1*74536a76Sjmcneill /* $NetBSD: sun8i_crypto.c,v 1.19 2021/01/15 22:47:32 jmcneill Exp $ */ 2defaca02Sriastradh 3defaca02Sriastradh /*- 4defaca02Sriastradh * Copyright (c) 2019 The NetBSD Foundation, Inc. 5defaca02Sriastradh * All rights reserved. 6defaca02Sriastradh * 7defaca02Sriastradh * This code is derived from software contributed to The NetBSD Foundation 8defaca02Sriastradh * by Taylor R. Campbell. 9defaca02Sriastradh * 10defaca02Sriastradh * Redistribution and use in source and binary forms, with or without 11defaca02Sriastradh * modification, are permitted provided that the following conditions 12defaca02Sriastradh * are met: 13defaca02Sriastradh * 1. Redistributions of source code must retain the above copyright 14defaca02Sriastradh * notice, this list of conditions and the following disclaimer. 15defaca02Sriastradh * 2. Redistributions in binary form must reproduce the above copyright 16defaca02Sriastradh * notice, this list of conditions and the following disclaimer in the 17defaca02Sriastradh * documentation and/or other materials provided with the distribution. 18defaca02Sriastradh * 19defaca02Sriastradh * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20defaca02Sriastradh * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21defaca02Sriastradh * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22defaca02Sriastradh * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23defaca02Sriastradh * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24defaca02Sriastradh * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25defaca02Sriastradh * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26defaca02Sriastradh * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27defaca02Sriastradh * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28defaca02Sriastradh * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29defaca02Sriastradh * POSSIBILITY OF SUCH DAMAGE. 30defaca02Sriastradh */ 31defaca02Sriastradh 32defaca02Sriastradh /* 33defaca02Sriastradh * sun8i_crypto -- Allwinner Crypto Engine driver 34defaca02Sriastradh * 35defaca02Sriastradh * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64 36defaca02Sriastradh * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at 37defaca02Sriastradh * the moment, but in principle it could be wired up with opencrypto(9) 38defaca02Sriastradh * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1, 39defaca02Sriastradh * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support 40defaca02Sriastradh * AES keys in SRAM (for some kind of HDMI HDCP stuff?). 41defaca02Sriastradh * 42defaca02Sriastradh * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf 43defaca02Sriastradh */ 44defaca02Sriastradh 45defaca02Sriastradh #include <sys/cdefs.h> 46*74536a76Sjmcneill __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.19 2021/01/15 22:47:32 jmcneill Exp $"); 47defaca02Sriastradh 48defaca02Sriastradh #include <sys/types.h> 49defaca02Sriastradh #include <sys/param.h> 50defaca02Sriastradh #include <sys/atomic.h> 51defaca02Sriastradh #include <sys/bus.h> 52defaca02Sriastradh #include <sys/callout.h> 53defaca02Sriastradh #include <sys/conf.h> 5486953edcSriastradh #include <sys/cprng.h> 55defaca02Sriastradh #include <sys/device.h> 56defaca02Sriastradh #include <sys/kernel.h> 57defaca02Sriastradh #include <sys/kmem.h> 5886953edcSriastradh #include <sys/mbuf.h> 59defaca02Sriastradh #include <sys/mutex.h> 60defaca02Sriastradh #include <sys/rndsource.h> 6186953edcSriastradh #include <sys/sdt.h> 62defaca02Sriastradh #include <sys/sysctl.h> 63defaca02Sriastradh #include <sys/workqueue.h> 64defaca02Sriastradh 65defaca02Sriastradh #include <dev/fdt/fdtvar.h> 66defaca02Sriastradh 6786953edcSriastradh #include <opencrypto/cryptodev.h> 6886953edcSriastradh 69defaca02Sriastradh #include <arm/sunxi/sun8i_crypto.h> 70defaca02Sriastradh 71defaca02Sriastradh #define SUN8I_CRYPTO_TIMEOUT hz 72f9ae7d17Sriastradh #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */ 73f9ae7d17Sriastradh #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE 74defaca02Sriastradh 75defaca02Sriastradh struct sun8i_crypto_task; 76defaca02Sriastradh 77defaca02Sriastradh struct sun8i_crypto_buf { 78defaca02Sriastradh bus_dma_segment_t cb_seg[1]; 79defaca02Sriastradh int cb_nsegs; 80defaca02Sriastradh void *cb_kva; 81defaca02Sriastradh }; 82defaca02Sriastradh 83defaca02Sriastradh struct sun8i_crypto_softc { 84defaca02Sriastradh device_t sc_dev; 85defaca02Sriastradh bus_space_tag_t sc_bst; 86defaca02Sriastradh bus_space_handle_t sc_bsh; 87defaca02Sriastradh bus_dma_tag_t sc_dmat; 8842356a74Sriastradh struct pool_cache *sc_taskpool; 89defaca02Sriastradh kmutex_t sc_lock; 90defaca02Sriastradh struct sun8i_crypto_chan { 91defaca02Sriastradh struct sun8i_crypto_task *cc_task; 92defaca02Sriastradh unsigned cc_starttime; 93defaca02Sriastradh } sc_chan[SUN8I_CRYPTO_NCHAN]; 94defaca02Sriastradh struct callout sc_timeout; 95defaca02Sriastradh struct workqueue *sc_wq; 96defaca02Sriastradh struct work sc_work; 97defaca02Sriastradh void *sc_ih; 98defaca02Sriastradh uint32_t sc_done; 99defaca02Sriastradh uint32_t sc_esr; 100defaca02Sriastradh bool sc_work_pending; 101defaca02Sriastradh struct sun8i_crypto_rng { 102defaca02Sriastradh struct sun8i_crypto_buf cr_buf; 103defaca02Sriastradh struct sun8i_crypto_task *cr_task; 104defaca02Sriastradh struct krndsource cr_rndsource; 105defaca02Sriastradh bool cr_pending; 106defaca02Sriastradh } sc_rng; 107defaca02Sriastradh struct sun8i_crypto_selftest { 108defaca02Sriastradh struct sun8i_crypto_buf cs_in; 109defaca02Sriastradh struct sun8i_crypto_buf cs_key; 110defaca02Sriastradh struct sun8i_crypto_buf cs_out; 111defaca02Sriastradh struct sun8i_crypto_task *cs_task; 112defaca02Sriastradh } sc_selftest; 113defaca02Sriastradh struct sun8i_crypto_sysctl { 114defaca02Sriastradh struct sysctllog *cy_log; 115defaca02Sriastradh const struct sysctlnode *cy_root_node; 116defaca02Sriastradh const struct sysctlnode *cy_trng_node; 117defaca02Sriastradh } sc_sysctl; 11886953edcSriastradh struct sun8i_crypto_opencrypto { 11986953edcSriastradh uint32_t co_driverid; 12086953edcSriastradh } sc_opencrypto; 121defaca02Sriastradh }; 122defaca02Sriastradh 123defaca02Sriastradh struct sun8i_crypto_task { 12442356a74Sriastradh struct sun8i_crypto_buf ct_descbuf; 125defaca02Sriastradh struct sun8i_crypto_taskdesc *ct_desc; 12686953edcSriastradh struct sun8i_crypto_buf ct_ivbuf; 12786953edcSriastradh void *ct_iv; 12886953edcSriastradh struct sun8i_crypto_buf ct_ctrbuf; 12986953edcSriastradh void *ct_ctr; 13042356a74Sriastradh bus_dmamap_t ct_descmap; 13142356a74Sriastradh bus_dmamap_t ct_keymap; 13286953edcSriastradh bus_dmamap_t ct_ivmap; /* IV input */ 13386953edcSriastradh bus_dmamap_t ct_ctrmap; /* updated IV output */ 13442356a74Sriastradh bus_dmamap_t ct_srcmap; 13542356a74Sriastradh bus_dmamap_t ct_dstmap; 13642356a74Sriastradh uint32_t ct_nbytes; 13742356a74Sriastradh int ct_flags; 13842356a74Sriastradh #define TASK_KEY __BIT(0) 13942356a74Sriastradh #define TASK_IV __BIT(1) 14042356a74Sriastradh #define TASK_CTR __BIT(2) 14142356a74Sriastradh #define TASK_SRC __BIT(3) 14242356a74Sriastradh #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */ 143defaca02Sriastradh void (*ct_callback)(struct sun8i_crypto_softc *, 144defaca02Sriastradh struct sun8i_crypto_task *, void *, int); 145defaca02Sriastradh void *ct_cookie; 146defaca02Sriastradh }; 147defaca02Sriastradh 14842356a74Sriastradh #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE 14942356a74Sriastradh #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE 15042356a74Sriastradh 15142356a74Sriastradh CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN); 15242356a74Sriastradh CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN); 15342356a74Sriastradh 154defaca02Sriastradh /* 155defaca02Sriastradh * Forward declarations 156defaca02Sriastradh */ 157defaca02Sriastradh 158defaca02Sriastradh static int sun8i_crypto_match(device_t, cfdata_t, void *); 159defaca02Sriastradh static void sun8i_crypto_attach(device_t, device_t, void *); 160defaca02Sriastradh 16142356a74Sriastradh static int sun8i_crypto_task_ctor(void *, void *, int); 16242356a74Sriastradh static void sun8i_crypto_task_dtor(void *, void *); 163defaca02Sriastradh static struct sun8i_crypto_task * 164defaca02Sriastradh sun8i_crypto_task_get(struct sun8i_crypto_softc *, 165defaca02Sriastradh void (*)(struct sun8i_crypto_softc *, 166defaca02Sriastradh struct sun8i_crypto_task *, void *, int), 16742356a74Sriastradh void *, int); 168defaca02Sriastradh static void sun8i_crypto_task_put(struct sun8i_crypto_softc *, 169defaca02Sriastradh struct sun8i_crypto_task *); 170defaca02Sriastradh 17142356a74Sriastradh static int sun8i_crypto_task_load(struct sun8i_crypto_softc *, 17242356a74Sriastradh struct sun8i_crypto_task *, uint32_t, 17342356a74Sriastradh uint32_t, uint32_t, uint32_t); 17486953edcSriastradh static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *, 17586953edcSriastradh struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t); 176defaca02Sriastradh 17742356a74Sriastradh static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *, 178defaca02Sriastradh struct sun8i_crypto_task *, uint32_t); 17942356a74Sriastradh static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *, 180defaca02Sriastradh struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t); 18142356a74Sriastradh 182defaca02Sriastradh static int sun8i_crypto_submit(struct sun8i_crypto_softc *, 183defaca02Sriastradh struct sun8i_crypto_task *); 184defaca02Sriastradh 185defaca02Sriastradh static void sun8i_crypto_timeout(void *); 186defaca02Sriastradh static int sun8i_crypto_intr(void *); 187defaca02Sriastradh static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *); 188defaca02Sriastradh static void sun8i_crypto_worker(struct work *, void *); 189defaca02Sriastradh static void sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned, 190defaca02Sriastradh int); 191defaca02Sriastradh 192defaca02Sriastradh static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t, 19342356a74Sriastradh struct sun8i_crypto_buf *, int); 194defaca02Sriastradh static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t, 195defaca02Sriastradh struct sun8i_crypto_buf *); 196defaca02Sriastradh 197defaca02Sriastradh static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *); 198defaca02Sriastradh static void sun8i_crypto_rng_get(size_t, void *); 199defaca02Sriastradh static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *, 200defaca02Sriastradh struct sun8i_crypto_task *, void *, int); 201defaca02Sriastradh 202defaca02Sriastradh static void sun8i_crypto_selftest(device_t); 203defaca02Sriastradh static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *, 204defaca02Sriastradh struct sun8i_crypto_task *, void *, int); 205defaca02Sriastradh 206defaca02Sriastradh static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *); 207defaca02Sriastradh static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS); 208defaca02Sriastradh static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *, 209defaca02Sriastradh struct sun8i_crypto_task *, void *, int); 210defaca02Sriastradh 21186953edcSriastradh static void sun8i_crypto_register(struct sun8i_crypto_softc *); 21286953edcSriastradh static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t); 21386953edcSriastradh static int sun8i_crypto_newsession(void *, uint32_t *, 21486953edcSriastradh struct cryptoini *); 21586953edcSriastradh static int sun8i_crypto_freesession(void *, uint64_t); 21686953edcSriastradh static u_int sun8i_crypto_ivlen(const struct cryptodesc *); 21786953edcSriastradh static int sun8i_crypto_process(void *, struct cryptop *, int); 21886953edcSriastradh static void sun8i_crypto_callback(struct sun8i_crypto_softc *, 21986953edcSriastradh struct sun8i_crypto_task *, void *, int); 22086953edcSriastradh 22186953edcSriastradh /* 22286953edcSriastradh * Probes 22386953edcSriastradh */ 22486953edcSriastradh 22586953edcSriastradh SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read, 22686953edcSriastradh "bus_size_t"/*reg*/, 22786953edcSriastradh "uint32_t"/*value*/); 22886953edcSriastradh SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write, 22986953edcSriastradh "bus_size_t"/*reg*/, 23086953edcSriastradh "uint32_t"/*write*/); 23186953edcSriastradh 23286953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success, 23386953edcSriastradh "struct sun8i_crypto_task *"/*task*/); 23486953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure, 23586953edcSriastradh "int"/*error*/); 23686953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor, 23786953edcSriastradh "struct sun8i_crypto_task *"/*task*/); 23886953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get, 23986953edcSriastradh "struct sun8i_crypto_task *"/*task*/); 24086953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put, 24186953edcSriastradh "struct sun8i_crypto_task *"/*task*/); 24286953edcSriastradh 24386953edcSriastradh SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load, 24486953edcSriastradh "struct sun8i_crypto_task *"/*task*/, 24586953edcSriastradh "uint32_t"/*tdqc*/, 24686953edcSriastradh "uint32_t"/*tdqs*/, 24786953edcSriastradh "uint32_t"/*tdqa*/, 24886953edcSriastradh "struct sun8i_crypto_taskdesc *"/*desc*/, 24986953edcSriastradh "int"/*error*/); 25086953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned, 25186953edcSriastradh "struct sun8i_crypto_task *"/*task*/, 25286953edcSriastradh "bus_addr_t"/*ds_addr*/, 25386953edcSriastradh "bus_size_t"/*ds_len*/); 25486953edcSriastradh SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done, 25586953edcSriastradh "struct sun8i_crypto_task *"/*task*/, 25686953edcSriastradh "int"/*error*/); 25786953edcSriastradh 25886953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure, 25986953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/, 26086953edcSriastradh "struct sun8i_crypto_task *"/*task*/, 26186953edcSriastradh "int"/*error*/); 26286953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success, 26386953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/, 26486953edcSriastradh "struct sun8i_crypto_task *"/*task*/, 26586953edcSriastradh "unsigned"/*chan*/); 26686953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr, 26786953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/, 26886953edcSriastradh "uint32_t"/*isr*/, 26986953edcSriastradh "uint32_t"/*esr*/); 27086953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done, 27186953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/, 27286953edcSriastradh "unsigned"/*chan*/, 27386953edcSriastradh "int"/*error*/); 27486953edcSriastradh 27586953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry, 27686953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/, 27786953edcSriastradh "struct cryptop *"/*crp*/, 27886953edcSriastradh "int"/*hint*/); 27986953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy, 28086953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/, 28186953edcSriastradh "struct cryptop *"/*crp*/, 28286953edcSriastradh "int"/*hint*/); 28386953edcSriastradh SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued, 28486953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/, 28586953edcSriastradh "struct cryptop *"/*crp*/, 28686953edcSriastradh "int"/*hint*/, 28786953edcSriastradh "struct sun8i_crypto_task *"/*task*/); 28886953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done, 28986953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/, 29086953edcSriastradh "struct cryptop *"/*crp*/, 29186953edcSriastradh "int"/*error*/); 29286953edcSriastradh 293defaca02Sriastradh /* 294defaca02Sriastradh * Register access 295defaca02Sriastradh */ 296defaca02Sriastradh 297defaca02Sriastradh static uint32_t 29886953edcSriastradh sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg) 299defaca02Sriastradh { 30086953edcSriastradh uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg); 30186953edcSriastradh 30286953edcSriastradh SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v); 30386953edcSriastradh return v; 304defaca02Sriastradh } 305defaca02Sriastradh 306defaca02Sriastradh static void 30786953edcSriastradh sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v) 308defaca02Sriastradh { 30986953edcSriastradh 31086953edcSriastradh SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v); 311defaca02Sriastradh bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v); 312defaca02Sriastradh } 313defaca02Sriastradh 314defaca02Sriastradh /* 315defaca02Sriastradh * Autoconf goo 316defaca02Sriastradh */ 317defaca02Sriastradh 318defaca02Sriastradh CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc), 319defaca02Sriastradh sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL); 320defaca02Sriastradh 321defaca02Sriastradh static const struct of_compat_data compat_data[] = { 322defaca02Sriastradh {"allwinner,sun50i-a64-crypto", 0}, 323defaca02Sriastradh {NULL} 324defaca02Sriastradh }; 325defaca02Sriastradh 326defaca02Sriastradh static int 327defaca02Sriastradh sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux) 328defaca02Sriastradh { 329defaca02Sriastradh const struct fdt_attach_args *const faa = aux; 330defaca02Sriastradh 331defaca02Sriastradh return of_match_compat_data(faa->faa_phandle, compat_data); 332defaca02Sriastradh } 333defaca02Sriastradh 334defaca02Sriastradh static void 335defaca02Sriastradh sun8i_crypto_attach(device_t parent, device_t self, void *aux) 336defaca02Sriastradh { 337defaca02Sriastradh struct sun8i_crypto_softc *const sc = device_private(self); 338defaca02Sriastradh const struct fdt_attach_args *const faa = aux; 339defaca02Sriastradh bus_addr_t addr; 340defaca02Sriastradh bus_size_t size; 341defaca02Sriastradh const int phandle = faa->faa_phandle; 342defaca02Sriastradh char intrstr[128]; 343defaca02Sriastradh struct clk *clk; 344defaca02Sriastradh struct fdtbus_reset *rst; 345defaca02Sriastradh 346defaca02Sriastradh sc->sc_dev = self; 347defaca02Sriastradh sc->sc_dmat = faa->faa_dmat; 348defaca02Sriastradh sc->sc_bst = faa->faa_bst; 34942356a74Sriastradh sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task), 35042356a74Sriastradh 0, 0, 0, "sun8icry", NULL, IPL_VM, 35142356a74Sriastradh &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc); 352defaca02Sriastradh mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_VM); 353defaca02Sriastradh callout_init(&sc->sc_timeout, CALLOUT_MPSAFE); 354defaca02Sriastradh callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc); 355defaca02Sriastradh if (workqueue_create(&sc->sc_wq, device_xname(self), 356defaca02Sriastradh &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) { 357defaca02Sriastradh aprint_error(": couldn't create workqueue\n"); 358defaca02Sriastradh return; 359defaca02Sriastradh } 360defaca02Sriastradh 36186953edcSriastradh /* 36286953edcSriastradh * Prime the pool with enough tasks that each channel can be 36386953edcSriastradh * busy with a task as we prepare another task for when it's 36486953edcSriastradh * done. 36586953edcSriastradh */ 36686953edcSriastradh pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN); 36786953edcSriastradh 368defaca02Sriastradh /* Get and map device registers. */ 369defaca02Sriastradh if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 370defaca02Sriastradh aprint_error(": couldn't get registers\n"); 371defaca02Sriastradh return; 372defaca02Sriastradh } 373defaca02Sriastradh if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) { 374defaca02Sriastradh aprint_error(": couldn't map registers\n"); 375defaca02Sriastradh return; 376defaca02Sriastradh } 377defaca02Sriastradh 378defaca02Sriastradh /* Get an interrupt handle. */ 379defaca02Sriastradh if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 380defaca02Sriastradh aprint_error(": failed to decode interrupt\n"); 381defaca02Sriastradh return; 382defaca02Sriastradh } 383defaca02Sriastradh 384defaca02Sriastradh /* Enable the bus clock. */ 385defaca02Sriastradh if (fdtbus_clock_enable(phandle, "bus", true) != 0) { 386defaca02Sriastradh aprint_error(": couldn't enable bus clock\n"); 387defaca02Sriastradh return; 388defaca02Sriastradh } 389defaca02Sriastradh 390defaca02Sriastradh /* Get the module clock and set it to 300 MHz. */ 391defaca02Sriastradh if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) { 392defaca02Sriastradh if (clk_enable(clk) != 0) { 393defaca02Sriastradh aprint_error(": couldn't enable CE clock\n"); 394defaca02Sriastradh return; 395defaca02Sriastradh } 396defaca02Sriastradh if (clk_set_rate(clk, 300*1000*1000) != 0) { 397defaca02Sriastradh aprint_error(": couldn't set CE clock to 300MHz\n"); 398defaca02Sriastradh return; 399defaca02Sriastradh } 400defaca02Sriastradh } 401defaca02Sriastradh 402defaca02Sriastradh /* Get a reset handle if we need and try to deassert it. */ 403defaca02Sriastradh if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) { 404defaca02Sriastradh if (fdtbus_reset_deassert(rst) != 0) { 405defaca02Sriastradh aprint_error(": couldn't de-assert reset\n"); 406defaca02Sriastradh return; 407defaca02Sriastradh } 408defaca02Sriastradh } 409defaca02Sriastradh 410defaca02Sriastradh aprint_naive("\n"); 411defaca02Sriastradh aprint_normal(": Crypto Engine\n"); 412defaca02Sriastradh aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk)); 413defaca02Sriastradh 414defaca02Sriastradh /* Disable and clear interrupts. */ 415defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0); 416defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0); 417defaca02Sriastradh 418defaca02Sriastradh /* Establish an interrupt handler. */ 419*74536a76Sjmcneill sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM, 420*74536a76Sjmcneill FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self)); 421defaca02Sriastradh if (sc->sc_ih == NULL) { 422defaca02Sriastradh aprint_error_dev(self, "failed to establish interrupt on %s\n", 423defaca02Sriastradh intrstr); 424defaca02Sriastradh return; 425defaca02Sriastradh } 426defaca02Sriastradh aprint_normal_dev(self, "interrupting on %s\n", intrstr); 427defaca02Sriastradh 428defaca02Sriastradh /* Set up the RNG. */ 429defaca02Sriastradh sun8i_crypto_rng_attach(sc); 430defaca02Sriastradh 431defaca02Sriastradh /* Attach the sysctl. */ 432defaca02Sriastradh sun8i_crypto_sysctl_attach(sc); 433defaca02Sriastradh 434defaca02Sriastradh /* Perform self-tests. */ 435defaca02Sriastradh config_interrupts(self, sun8i_crypto_selftest); 43686953edcSriastradh 43786953edcSriastradh /* Register opencrypto handlers. */ 43886953edcSriastradh sun8i_crypto_register(sc); 439defaca02Sriastradh } 440defaca02Sriastradh 44142356a74Sriastradh static int 44242356a74Sriastradh sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags) 44342356a74Sriastradh { 44442356a74Sriastradh struct sun8i_crypto_softc *sc = cookie; 44542356a74Sriastradh struct sun8i_crypto_task *task = vtask; 44642356a74Sriastradh int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT; 44742356a74Sriastradh int error; 448defaca02Sriastradh 44942356a74Sriastradh /* Create a DMA buffer for the task descriptor. */ 45042356a74Sriastradh error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc), 45142356a74Sriastradh &task->ct_descbuf, dmaflags); 45242356a74Sriastradh if (error) 45342356a74Sriastradh goto fail0; 45442356a74Sriastradh task->ct_desc = task->ct_descbuf.cb_kva; 45542356a74Sriastradh 45686953edcSriastradh /* Create DMA buffers for the IV and CTR. */ 45786953edcSriastradh error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES, 45886953edcSriastradh &task->ct_ivbuf, dmaflags); 45986953edcSriastradh if (error) 46086953edcSriastradh goto fail1; 46186953edcSriastradh task->ct_iv = task->ct_ivbuf.cb_kva; 46286953edcSriastradh error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, 46386953edcSriastradh &task->ct_ctrbuf, dmaflags); 46486953edcSriastradh if (error) 46586953edcSriastradh goto fail2; 46686953edcSriastradh task->ct_ctr = task->ct_ctrbuf.cb_kva; 46786953edcSriastradh 46842356a74Sriastradh /* Create a DMA map for the task descriptor and preload it. */ 46942356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1, 47042356a74Sriastradh sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap); 47142356a74Sriastradh if (error) 47286953edcSriastradh goto fail3; 47342356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc, 47442356a74Sriastradh sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK); 47542356a74Sriastradh if (error) 47686953edcSriastradh goto fail4; 47742356a74Sriastradh 47842356a74Sriastradh /* Create DMA maps for the key, IV, and CTR. */ 47942356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1, 48042356a74Sriastradh SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap); 48142356a74Sriastradh if (error) 48286953edcSriastradh goto fail5; 48342356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1, 48442356a74Sriastradh SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap); 48542356a74Sriastradh if (error) 48686953edcSriastradh goto fail6; 48742356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1, 48842356a74Sriastradh SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap); 48942356a74Sriastradh if (error) 49086953edcSriastradh goto fail7; 49142356a74Sriastradh 49242356a74Sriastradh /* Create DMA maps for the src and dst scatter/gather vectors. */ 49342356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 49442356a74Sriastradh SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 49542356a74Sriastradh &task->ct_srcmap); 49642356a74Sriastradh if (error) 49786953edcSriastradh goto fail8; 49842356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 49942356a74Sriastradh SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 50042356a74Sriastradh &task->ct_dstmap); 50142356a74Sriastradh if (error) 50286953edcSriastradh goto fail9; 50342356a74Sriastradh 50442356a74Sriastradh /* Success! */ 50586953edcSriastradh SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task); 50642356a74Sriastradh return 0; 50742356a74Sriastradh 50886953edcSriastradh fail10: __unused 50942356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 51086953edcSriastradh fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 51186953edcSriastradh fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 51286953edcSriastradh fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 51386953edcSriastradh fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 51486953edcSriastradh fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 51586953edcSriastradh fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 51686953edcSriastradh fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 51786953edcSriastradh fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 51842356a74Sriastradh fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 51986953edcSriastradh fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error); 52086953edcSriastradh return error; 52142356a74Sriastradh } 52242356a74Sriastradh 52342356a74Sriastradh static void 52442356a74Sriastradh sun8i_crypto_task_dtor(void *cookie, void *vtask) 52542356a74Sriastradh { 52642356a74Sriastradh struct sun8i_crypto_softc *sc = cookie; 52742356a74Sriastradh struct sun8i_crypto_task *task = vtask; 52842356a74Sriastradh 52986953edcSriastradh SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task); 53086953edcSriastradh 53142356a74Sriastradh /* XXX Zero the bounce buffers if there are any. */ 53242356a74Sriastradh 53342356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 53442356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 53542356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 53642356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 53742356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 53842356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 53942356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 54086953edcSriastradh sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 54186953edcSriastradh sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 54242356a74Sriastradh sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 54342356a74Sriastradh } 54442356a74Sriastradh 54542356a74Sriastradh /* 54642356a74Sriastradh * sun8i_crypto_task_get(sc, callback, cookie, pflags) 54742356a74Sriastradh * 54842356a74Sriastradh * Allocate a task that will call callback(sc, task, cookie, 54942356a74Sriastradh * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if 55042356a74Sriastradh * PR_NOWAIT, may fail and return NULL. No further allocation is 55142356a74Sriastradh * needed to submit the task if this succeeds (although task 55242356a74Sriastradh * submission may still fail if all channels are busy). 55342356a74Sriastradh */ 554defaca02Sriastradh static struct sun8i_crypto_task * 555defaca02Sriastradh sun8i_crypto_task_get(struct sun8i_crypto_softc *sc, 556defaca02Sriastradh void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *, 557defaca02Sriastradh void *, int), 55842356a74Sriastradh void *cookie, int pflags) 559defaca02Sriastradh { 560defaca02Sriastradh struct sun8i_crypto_task *task; 561defaca02Sriastradh 56242356a74Sriastradh /* Allocate a task, or fail if we can't. */ 56342356a74Sriastradh task = pool_cache_get(sc->sc_taskpool, pflags); 56442356a74Sriastradh if (task == NULL) 56586953edcSriastradh goto out; 566defaca02Sriastradh 56742356a74Sriastradh /* Set up flags and the callback. */ 56842356a74Sriastradh task->ct_flags = 0; 569defaca02Sriastradh task->ct_callback = callback; 570defaca02Sriastradh task->ct_cookie = cookie; 57186953edcSriastradh 57286953edcSriastradh out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task); 573defaca02Sriastradh return task; 574defaca02Sriastradh } 575defaca02Sriastradh 57642356a74Sriastradh /* 57742356a74Sriastradh * sun8i_crypto_task_invalid(sc, task, cookie, error) 57842356a74Sriastradh * 57942356a74Sriastradh * Callback for a task not currently in use, to detect errors. 58042356a74Sriastradh */ 58142356a74Sriastradh static void 58242356a74Sriastradh sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc, 58342356a74Sriastradh struct sun8i_crypto_task *task, void *cookie, int error) 58442356a74Sriastradh { 58542356a74Sriastradh void (*callback)(struct sun8i_crypto_softc *, 58642356a74Sriastradh struct sun8i_crypto_task *, void *, int) = cookie; 58742356a74Sriastradh 58842356a74Sriastradh panic("task for callback %p used after free", callback); 58942356a74Sriastradh } 59042356a74Sriastradh 59142356a74Sriastradh /* 59242356a74Sriastradh * sun8i_crypto_task_put(sc, task) 59342356a74Sriastradh * 59442356a74Sriastradh * Free a task obtained with sun8i_crypto_task_get. 59542356a74Sriastradh */ 596defaca02Sriastradh static void 597defaca02Sriastradh sun8i_crypto_task_put(struct sun8i_crypto_softc *sc, 598defaca02Sriastradh struct sun8i_crypto_task *task) 599defaca02Sriastradh { 600defaca02Sriastradh 60186953edcSriastradh SDT_PROBE1(sdt, sun8i_crypto, task, put, task); 60286953edcSriastradh 60342356a74Sriastradh task->ct_cookie = task->ct_callback; 60442356a74Sriastradh task->ct_callback = &sun8i_crypto_task_invalid; 60542356a74Sriastradh pool_cache_put(sc->sc_taskpool, task); 606defaca02Sriastradh } 607defaca02Sriastradh 608defaca02Sriastradh /* 60942356a74Sriastradh * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa) 610defaca02Sriastradh * 61142356a74Sriastradh * Set up the task descriptor after the relevant DMA maps have 61242356a74Sriastradh * been loaded for a transfer of nbytes. bus_dmamap_sync matches 61342356a74Sriastradh * sun8i_crypto_chan_done. May fail if input is inadequately 61442356a74Sriastradh * aligned. 61542356a74Sriastradh * 61642356a74Sriastradh * XXX Teach this to support task chains. 617defaca02Sriastradh */ 61842356a74Sriastradh static int 61942356a74Sriastradh sun8i_crypto_task_load(struct sun8i_crypto_softc *sc, 62042356a74Sriastradh struct sun8i_crypto_task *task, uint32_t nbytes, 62142356a74Sriastradh uint32_t tdqc, uint32_t tdqs, uint32_t tdqa) 622defaca02Sriastradh { 62342356a74Sriastradh struct sun8i_crypto_taskdesc *desc = task->ct_desc; 62442356a74Sriastradh int error; 625defaca02Sriastradh 62642356a74Sriastradh KASSERT(tdqs == 0 || tdqa == 0); 62742356a74Sriastradh KASSERT(nbytes % 4 == 0); 62842356a74Sriastradh 62942356a74Sriastradh memset(desc, 0, sizeof(*desc)); 63042356a74Sriastradh 63186953edcSriastradh /* Always enable interrupt for the task. */ 63286953edcSriastradh tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN; 63386953edcSriastradh 63442356a74Sriastradh desc->td_tdqc = htole32(tdqc); 63542356a74Sriastradh desc->td_tdqs = htole32(tdqs); 63642356a74Sriastradh desc->td_tdqa = htole32(tdqa); 63742356a74Sriastradh 63842356a74Sriastradh if (task->ct_flags & TASK_KEY) { 63942356a74Sriastradh bus_dmamap_t keymap = task->ct_keymap; 64042356a74Sriastradh KASSERT(keymap->dm_nsegs == 1); 64142356a74Sriastradh desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr); 64242356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, keymap, 0, 64342356a74Sriastradh keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 64442356a74Sriastradh } 64542356a74Sriastradh if (task->ct_flags & TASK_IV) { 64642356a74Sriastradh bus_dmamap_t ivmap = task->ct_ivmap; 64742356a74Sriastradh KASSERT(ivmap->dm_nsegs == 1); 64842356a74Sriastradh desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr); 64942356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, ivmap, 0, 65042356a74Sriastradh ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 65142356a74Sriastradh } 65242356a74Sriastradh if (task->ct_flags & TASK_CTR) { 65342356a74Sriastradh bus_dmamap_t ctrmap = task->ct_ctrmap; 65442356a74Sriastradh KASSERT(ctrmap->dm_nsegs == 1); 65542356a74Sriastradh desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr); 65642356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, ctrmap, 0, 65786953edcSriastradh ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD); 658defaca02Sriastradh } 659defaca02Sriastradh 66042356a74Sriastradh if (task->ct_flags & TASK_BYTES) 66142356a74Sriastradh desc->td_datalen = htole32(nbytes); 66242356a74Sriastradh else 66342356a74Sriastradh desc->td_datalen = htole32(nbytes/4); 664defaca02Sriastradh 66542356a74Sriastradh if (task->ct_flags & TASK_SRC) { 66642356a74Sriastradh bus_dmamap_t srcmap = task->ct_srcmap; 66742356a74Sriastradh KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize); 66886953edcSriastradh error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap, 66942356a74Sriastradh nbytes); 67042356a74Sriastradh if (error) 67142356a74Sriastradh return error; 67242356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes, 67342356a74Sriastradh BUS_DMASYNC_PREWRITE); 674defaca02Sriastradh } 675defaca02Sriastradh 67686953edcSriastradh error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap, 67742356a74Sriastradh nbytes); 67842356a74Sriastradh if (error) 67986953edcSriastradh goto out; 68042356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 68142356a74Sriastradh BUS_DMASYNC_PREREAD); 682defaca02Sriastradh 68342356a74Sriastradh task->ct_nbytes = nbytes; 68442356a74Sriastradh 68542356a74Sriastradh /* Success! */ 68686953edcSriastradh error = 0; 68786953edcSriastradh 68886953edcSriastradh out: SDT_PROBE6(sdt, sun8i_crypto, task, load, 68986953edcSriastradh task, tdqc, tdqs, tdqa, desc, error); 69086953edcSriastradh return error; 691defaca02Sriastradh } 692defaca02Sriastradh 69342356a74Sriastradh /* 69486953edcSriastradh * sun8i_crypto_task_scatter(task, adrlen, map, nbytes) 69542356a74Sriastradh * 69642356a74Sriastradh * Set up a task's scatter/gather vector -- src or dst -- with the 69742356a74Sriastradh * given DMA map for a transfer of nbytes. May fail if input is 69842356a74Sriastradh * inadequately aligned. 69942356a74Sriastradh */ 70042356a74Sriastradh static int 70186953edcSriastradh sun8i_crypto_task_scatter(struct sun8i_crypto_task *task, 70286953edcSriastradh struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map, 70342356a74Sriastradh uint32_t nbytes __diagused) 704defaca02Sriastradh { 705defaca02Sriastradh uint32_t total __diagused = 0; 706defaca02Sriastradh unsigned i; 707defaca02Sriastradh 70886953edcSriastradh /* 70986953edcSriastradh * Verify that the alignment is correct and initialize the 71086953edcSriastradh * scatter/gather vector. 71186953edcSriastradh */ 712defaca02Sriastradh KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS); 713defaca02Sriastradh for (i = 0; i < map->dm_nsegs; i++) { 71486953edcSriastradh if ((map->dm_segs[i].ds_addr % 4) | 71586953edcSriastradh (map->dm_segs[i].ds_len % 4)) { 71686953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, task, misaligned, 71786953edcSriastradh task, 71886953edcSriastradh map->dm_segs[i].ds_addr, 71986953edcSriastradh map->dm_segs[i].ds_len); 72086953edcSriastradh return EINVAL; 72186953edcSriastradh } 722defaca02Sriastradh KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX); 723defaca02Sriastradh KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total); 724defaca02Sriastradh adrlen[i].adr = htole32(map->dm_segs[i].ds_addr); 725defaca02Sriastradh adrlen[i].len = htole32(map->dm_segs[i].ds_len/4); 726defaca02Sriastradh total += map->dm_segs[i].ds_len; 727defaca02Sriastradh } 728defaca02Sriastradh 72986953edcSriastradh /* Set the remainder to zero. */ 730defaca02Sriastradh for (; i < SUN8I_CRYPTO_MAXSEGS; i++) { 73186953edcSriastradh adrlen[i].adr = 0; 73286953edcSriastradh adrlen[i].len = 0; 733defaca02Sriastradh } 734defaca02Sriastradh 73542356a74Sriastradh /* Verify the total size matches the transfer length. */ 73642356a74Sriastradh KASSERT(total == nbytes); 73742356a74Sriastradh 73842356a74Sriastradh /* Success! */ 73942356a74Sriastradh return 0; 740defaca02Sriastradh } 741defaca02Sriastradh 742defaca02Sriastradh /* 74342356a74Sriastradh * sun8i_crypto_task_load_trng(task, nbytes) 744defaca02Sriastradh * 74542356a74Sriastradh * Set up the task descriptor for a transfer of nbytes from the 74642356a74Sriastradh * TRNG. 747defaca02Sriastradh */ 748defaca02Sriastradh static int 74942356a74Sriastradh sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc, 75042356a74Sriastradh struct sun8i_crypto_task *task, uint32_t nbytes) 751defaca02Sriastradh { 752defaca02Sriastradh uint32_t tdqc = 0; 753defaca02Sriastradh 75442356a74Sriastradh /* Caller must provide dst only. */ 75542356a74Sriastradh KASSERT((task->ct_flags & TASK_KEY) == 0); 75642356a74Sriastradh KASSERT((task->ct_flags & TASK_IV) == 0); 75742356a74Sriastradh KASSERT((task->ct_flags & TASK_CTR) == 0); 75842356a74Sriastradh KASSERT((task->ct_flags & TASK_SRC) == 0); 759defaca02Sriastradh 760defaca02Sriastradh /* Set up the task descriptor queue control words. */ 761defaca02Sriastradh tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG, 762defaca02Sriastradh SUN8I_CRYPTO_TDQC_METHOD); 763defaca02Sriastradh 76442356a74Sriastradh /* Fill in the descriptor. */ 76542356a74Sriastradh return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0); 766defaca02Sriastradh } 767defaca02Sriastradh 768defaca02Sriastradh static int 76942356a74Sriastradh sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc, 770defaca02Sriastradh struct sun8i_crypto_task *task, 77142356a74Sriastradh uint32_t nbytes, uint32_t keysize, uint32_t dir) 772defaca02Sriastradh { 773defaca02Sriastradh uint32_t tdqc = 0, tdqs = 0; 774defaca02Sriastradh 77542356a74Sriastradh /* Caller must provide key, src, and dst only. */ 77642356a74Sriastradh KASSERT(task->ct_flags & TASK_KEY); 77742356a74Sriastradh KASSERT((task->ct_flags & TASK_IV) == 0); 77842356a74Sriastradh KASSERT((task->ct_flags & TASK_CTR) == 0); 77942356a74Sriastradh KASSERT(task->ct_flags & TASK_SRC); 780defaca02Sriastradh 781defaca02Sriastradh /* Set up the task descriptor queue control word. */ 782defaca02Sriastradh tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES, 783defaca02Sriastradh SUN8I_CRYPTO_TDQC_METHOD); 78486953edcSriastradh tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 78542356a74Sriastradh 78642356a74Sriastradh #ifdef DIAGNOSTIC 78742356a74Sriastradh switch (keysize) { 78842356a74Sriastradh case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128: 78942356a74Sriastradh KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16); 79042356a74Sriastradh break; 79142356a74Sriastradh case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192: 79242356a74Sriastradh KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24); 79342356a74Sriastradh break; 79442356a74Sriastradh case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256: 79542356a74Sriastradh KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32); 79642356a74Sriastradh break; 79742356a74Sriastradh } 79842356a74Sriastradh #endif 799defaca02Sriastradh 800defaca02Sriastradh /* Set up the symmetric control word. */ 801defaca02Sriastradh tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 802defaca02Sriastradh SUN8I_CRYPTO_TDQS_SKEY_SELECT); 803defaca02Sriastradh tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB, 804defaca02Sriastradh SUN8I_CRYPTO_TDQS_OP_MODE); 80542356a74Sriastradh tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 806defaca02Sriastradh 80742356a74Sriastradh /* Fill in the descriptor. */ 80842356a74Sriastradh return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0); 809defaca02Sriastradh } 810defaca02Sriastradh 81142356a74Sriastradh /* 81242356a74Sriastradh * sun8i_crypto_submit(sc, task) 81342356a74Sriastradh * 81442356a74Sriastradh * Submit a task to the crypto engine after it has been loaded 81542356a74Sriastradh * with sun8i_crypto_task_load. On success, guarantees to 81642356a74Sriastradh * eventually call the task's callback. 81742356a74Sriastradh */ 818defaca02Sriastradh static int 819defaca02Sriastradh sun8i_crypto_submit(struct sun8i_crypto_softc *sc, 820defaca02Sriastradh struct sun8i_crypto_task *task) 821defaca02Sriastradh { 822defaca02Sriastradh unsigned i, retries = 0; 823defaca02Sriastradh uint32_t icr; 824defaca02Sriastradh int error = 0; 825defaca02Sriastradh 826defaca02Sriastradh /* One at a time at the device registers, please. */ 827defaca02Sriastradh mutex_enter(&sc->sc_lock); 828defaca02Sriastradh 829defaca02Sriastradh /* Find a channel. */ 830defaca02Sriastradh for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 831defaca02Sriastradh if (sc->sc_chan[i].cc_task == NULL) 832defaca02Sriastradh break; 833defaca02Sriastradh } 834defaca02Sriastradh if (i == SUN8I_CRYPTO_NCHAN) { 835defaca02Sriastradh device_printf(sc->sc_dev, "no free channels\n"); 836defaca02Sriastradh error = ERESTART; 837defaca02Sriastradh goto out; 838defaca02Sriastradh } 839defaca02Sriastradh 840defaca02Sriastradh /* 841defaca02Sriastradh * Set the channel id. Caller is responsible for setting up 842defaca02Sriastradh * all other parts of the descriptor. 843defaca02Sriastradh */ 844defaca02Sriastradh task->ct_desc->td_cid = htole32(i); 845defaca02Sriastradh 84642356a74Sriastradh /* 84742356a74Sriastradh * Prepare to send the descriptor to the device by DMA. 84842356a74Sriastradh * Matches POSTWRITE in sun8i_crypto_chan_done. 84942356a74Sriastradh */ 85042356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 851defaca02Sriastradh sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE); 852defaca02Sriastradh 853defaca02Sriastradh /* Confirm we're ready to go. */ 854defaca02Sriastradh if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) { 855defaca02Sriastradh device_printf(sc->sc_dev, "TLR not clear\n"); 856defaca02Sriastradh error = EIO; 857defaca02Sriastradh goto out; 858defaca02Sriastradh } 859defaca02Sriastradh 860defaca02Sriastradh /* Enable interrupts for this channel. */ 861defaca02Sriastradh icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 862defaca02Sriastradh icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 863defaca02Sriastradh SUN8I_CRYPTO_ICR_INTR_EN); 864defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 865defaca02Sriastradh 866defaca02Sriastradh /* Set the task descriptor queue address. */ 867defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ, 86842356a74Sriastradh task->ct_descmap->dm_segs[0].ds_addr); 869defaca02Sriastradh 870defaca02Sriastradh /* Notify the engine to load it, and wait for acknowledgement. */ 871defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD); 872defaca02Sriastradh while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) 873defaca02Sriastradh { 874defaca02Sriastradh /* 875defaca02Sriastradh * XXX Timeout pulled from arse. Is it even important 876defaca02Sriastradh * to wait here? 877defaca02Sriastradh */ 878defaca02Sriastradh if (++retries == 1000) { 879defaca02Sriastradh device_printf(sc->sc_dev, "TLR didn't clear: %08x\n", 880defaca02Sriastradh sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR)); 881defaca02Sriastradh /* 882defaca02Sriastradh * Hope it clears eventually; if not, we'll 883defaca02Sriastradh * time out. 884defaca02Sriastradh */ 885defaca02Sriastradh break; 886defaca02Sriastradh } 887defaca02Sriastradh DELAY(1); 888defaca02Sriastradh } 889defaca02Sriastradh 89042356a74Sriastradh /* 89142356a74Sriastradh * Loaded up and ready to go. Start a timer ticking if it's 89242356a74Sriastradh * not already. 89342356a74Sriastradh */ 894defaca02Sriastradh sc->sc_chan[i].cc_task = task; 8952d8f3966Smaxv sc->sc_chan[i].cc_starttime = getticks(); 89642356a74Sriastradh if (!callout_pending(&sc->sc_timeout)) 897defaca02Sriastradh callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 898defaca02Sriastradh 899defaca02Sriastradh /* XXX Consider polling if cold to get entropy earlier. */ 900defaca02Sriastradh 901defaca02Sriastradh out: /* Done! */ 90286953edcSriastradh if (error) 90386953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure, 90486953edcSriastradh sc, task, error); 90586953edcSriastradh else 90686953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success, 90786953edcSriastradh sc, task, i); 908defaca02Sriastradh mutex_exit(&sc->sc_lock); 909defaca02Sriastradh return error; 910defaca02Sriastradh } 911defaca02Sriastradh 91242356a74Sriastradh /* 91342356a74Sriastradh * sun8i_crypto_timeout(cookie) 91442356a74Sriastradh * 91542356a74Sriastradh * Timeout handler. Schedules work in a thread to cancel all 91642356a74Sriastradh * pending tasks that were started long enough ago we're bored of 91742356a74Sriastradh * waiting for them, and reschedules another timeout unless the 91842356a74Sriastradh * channels are all idle. 91942356a74Sriastradh */ 920defaca02Sriastradh static void 921defaca02Sriastradh sun8i_crypto_timeout(void *cookie) 922defaca02Sriastradh { 923defaca02Sriastradh struct sun8i_crypto_softc *sc = cookie; 924defaca02Sriastradh unsigned i; 925defaca02Sriastradh 926defaca02Sriastradh mutex_enter(&sc->sc_lock); 927defaca02Sriastradh 928defaca02Sriastradh /* Check whether there are any tasks pending. */ 929defaca02Sriastradh for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 930defaca02Sriastradh if (sc->sc_chan[i].cc_task) 931defaca02Sriastradh break; 932defaca02Sriastradh } 933defaca02Sriastradh if (i == SUN8I_CRYPTO_NCHAN) 934defaca02Sriastradh /* None pending, so nothing to do. */ 935defaca02Sriastradh goto out; 936defaca02Sriastradh 937defaca02Sriastradh /* 938defaca02Sriastradh * Schedule the worker to check for timeouts, and schedule 939defaca02Sriastradh * another timeout in case we need it. 940defaca02Sriastradh */ 941defaca02Sriastradh sun8i_crypto_schedule_worker(sc); 942defaca02Sriastradh callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 943defaca02Sriastradh 944defaca02Sriastradh out: mutex_exit(&sc->sc_lock); 945defaca02Sriastradh } 946defaca02Sriastradh 94742356a74Sriastradh /* 94842356a74Sriastradh * sun8i_crypto_intr(cookie) 94942356a74Sriastradh * 95042356a74Sriastradh * Device interrupt handler. Find what channels have completed, 95142356a74Sriastradh * whether with success or with failure, and schedule work in 95242356a74Sriastradh * thread context to invoke the appropriate callbacks. 95342356a74Sriastradh */ 954defaca02Sriastradh static int 955defaca02Sriastradh sun8i_crypto_intr(void *cookie) 956defaca02Sriastradh { 957defaca02Sriastradh struct sun8i_crypto_softc *sc = cookie; 958defaca02Sriastradh uint32_t isr, esr; 959defaca02Sriastradh 960defaca02Sriastradh mutex_enter(&sc->sc_lock); 961defaca02Sriastradh 962defaca02Sriastradh /* 963defaca02Sriastradh * Get and acknowledge the interrupts and error status. 964defaca02Sriastradh * 965defaca02Sriastradh * XXX Data sheet says the error status register is read-only, 966defaca02Sriastradh * but then advises writing 1 to bit x1xx (keysram access error 967defaca02Sriastradh * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do? 968defaca02Sriastradh */ 969defaca02Sriastradh isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR); 970defaca02Sriastradh esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR); 971defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr); 9725e40da3dSriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr); 973defaca02Sriastradh 97486953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr); 97586953edcSriastradh 976defaca02Sriastradh /* Start the worker if necessary. */ 977defaca02Sriastradh sun8i_crypto_schedule_worker(sc); 978defaca02Sriastradh 979defaca02Sriastradh /* Tell the worker what to do. */ 980defaca02Sriastradh sc->sc_done |= __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE); 981defaca02Sriastradh sc->sc_esr |= esr; 982defaca02Sriastradh 983defaca02Sriastradh mutex_exit(&sc->sc_lock); 984defaca02Sriastradh 985defaca02Sriastradh return __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE) != 0; 986defaca02Sriastradh } 987defaca02Sriastradh 98842356a74Sriastradh /* 98942356a74Sriastradh * sun8i_crypto_schedule_worker(sc) 99042356a74Sriastradh * 99142356a74Sriastradh * Ensure that crypto engine thread context work to invoke task 99242356a74Sriastradh * callbacks will run promptly. Idempotent. 99342356a74Sriastradh */ 994defaca02Sriastradh static void 995defaca02Sriastradh sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc) 996defaca02Sriastradh { 997defaca02Sriastradh 998defaca02Sriastradh KASSERT(mutex_owned(&sc->sc_lock)); 999defaca02Sriastradh 1000defaca02Sriastradh /* Start the worker if necessary. */ 1001defaca02Sriastradh if (!sc->sc_work_pending) { 1002defaca02Sriastradh workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL); 1003defaca02Sriastradh sc->sc_work_pending = true; 1004defaca02Sriastradh } 1005defaca02Sriastradh } 1006defaca02Sriastradh 100742356a74Sriastradh /* 100842356a74Sriastradh * sun8i_crypto_worker(wk, cookie) 100942356a74Sriastradh * 101042356a74Sriastradh * Thread-context worker: Invoke all task callbacks for which the 101142356a74Sriastradh * device has notified us of completion or for which we gave up 101242356a74Sriastradh * waiting. 101342356a74Sriastradh */ 1014defaca02Sriastradh static void 1015defaca02Sriastradh sun8i_crypto_worker(struct work *wk, void *cookie) 1016defaca02Sriastradh { 1017defaca02Sriastradh struct sun8i_crypto_softc *sc = cookie; 1018defaca02Sriastradh uint32_t done, esr, esr_chan; 1019defaca02Sriastradh unsigned i, now; 1020defaca02Sriastradh int error; 1021defaca02Sriastradh 1022defaca02Sriastradh /* 1023defaca02Sriastradh * Acquire the lock. Note: We will be releasing and 1024defaca02Sriastradh * reacquiring it throughout the loop. 1025defaca02Sriastradh */ 1026defaca02Sriastradh mutex_enter(&sc->sc_lock); 1027defaca02Sriastradh 1028defaca02Sriastradh /* Acknowledge the work. */ 1029defaca02Sriastradh KASSERT(sc->sc_work_pending); 1030defaca02Sriastradh sc->sc_work_pending = false; 1031defaca02Sriastradh 1032defaca02Sriastradh /* 1033defaca02Sriastradh * Claim the done mask and error status once; we will be 1034defaca02Sriastradh * releasing and reacquiring the lock for the callbacks, so 1035defaca02Sriastradh * they may change. 1036defaca02Sriastradh */ 1037defaca02Sriastradh done = sc->sc_done; 1038defaca02Sriastradh esr = sc->sc_esr; 1039defaca02Sriastradh sc->sc_done = 0; 1040defaca02Sriastradh sc->sc_esr = 0; 1041defaca02Sriastradh 1042defaca02Sriastradh /* Check the time to determine what's timed out. */ 10432d8f3966Smaxv now = getticks(); 1044defaca02Sriastradh 1045defaca02Sriastradh /* Process the channels. */ 1046defaca02Sriastradh for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 104744c3f65aSriastradh /* Check whether the channel is done. */ 1048defaca02Sriastradh if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) { 104944c3f65aSriastradh /* Nope. Do we have a task to time out? */ 1050defaca02Sriastradh if ((sc->sc_chan[i].cc_task != NULL) && 1051defaca02Sriastradh ((now - sc->sc_chan[i].cc_starttime) >= 1052defaca02Sriastradh SUN8I_CRYPTO_TIMEOUT)) 1053defaca02Sriastradh sun8i_crypto_chan_done(sc, i, ETIMEDOUT); 1054defaca02Sriastradh continue; 1055defaca02Sriastradh } 105644c3f65aSriastradh 105744c3f65aSriastradh /* Channel is done. Interpret the error if any. */ 1058defaca02Sriastradh esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i)); 1059defaca02Sriastradh if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) { 1060defaca02Sriastradh device_printf(sc->sc_dev, "channel %u:" 1061defaca02Sriastradh " alg not supported\n", i); 1062defaca02Sriastradh error = ENODEV; 1063defaca02Sriastradh } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) { 1064defaca02Sriastradh device_printf(sc->sc_dev, "channel %u:" 1065defaca02Sriastradh " data length error\n", i); 1066defaca02Sriastradh error = EIO; /* XXX */ 1067defaca02Sriastradh } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) { 1068defaca02Sriastradh device_printf(sc->sc_dev, "channel %u:" 1069defaca02Sriastradh " key sram error\n", i); 1070defaca02Sriastradh error = EIO; /* XXX */ 1071defaca02Sriastradh } else if (esr_chan != 0) { 1072defaca02Sriastradh error = EIO; /* generic I/O error */ 1073defaca02Sriastradh } else { 1074defaca02Sriastradh error = 0; 1075defaca02Sriastradh } 1076defaca02Sriastradh 107744c3f65aSriastradh /* 107844c3f65aSriastradh * Notify the task of completion. May release the lock 107944c3f65aSriastradh * to invoke a callback. 108044c3f65aSriastradh */ 1081defaca02Sriastradh sun8i_crypto_chan_done(sc, i, error); 1082defaca02Sriastradh } 1083defaca02Sriastradh 1084defaca02Sriastradh /* All one; release the lock one last time. */ 1085defaca02Sriastradh mutex_exit(&sc->sc_lock); 1086defaca02Sriastradh } 1087defaca02Sriastradh 108842356a74Sriastradh /* 108942356a74Sriastradh * sun8i_crypto_chan_done(sc, i, error) 109042356a74Sriastradh * 109142356a74Sriastradh * Notify the callback for the task on channel i, if there is one, 109242356a74Sriastradh * of the specified error, or 0 for success. 109342356a74Sriastradh */ 1094defaca02Sriastradh static void 1095defaca02Sriastradh sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error) 1096defaca02Sriastradh { 1097defaca02Sriastradh struct sun8i_crypto_task *task; 109842356a74Sriastradh uint32_t nbytes; 1099defaca02Sriastradh uint32_t icr; 1100defaca02Sriastradh 1101defaca02Sriastradh KASSERT(mutex_owned(&sc->sc_lock)); 1102defaca02Sriastradh 110386953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error); 110486953edcSriastradh 1105defaca02Sriastradh /* Claim the task if there is one; bail if not. */ 1106defaca02Sriastradh if ((task = sc->sc_chan[i].cc_task) == NULL) { 1107defaca02Sriastradh device_printf(sc->sc_dev, "channel %u: no task but error=%d\n", 1108defaca02Sriastradh i, error); 1109defaca02Sriastradh return; 1110defaca02Sriastradh } 1111defaca02Sriastradh sc->sc_chan[i].cc_task = NULL; 1112defaca02Sriastradh 1113defaca02Sriastradh /* Disable interrupts on this channel. */ 1114defaca02Sriastradh icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 1115defaca02Sriastradh icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 1116defaca02Sriastradh SUN8I_CRYPTO_ICR_INTR_EN); 1117defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 1118defaca02Sriastradh 111942356a74Sriastradh /* 112042356a74Sriastradh * Finished sending the descriptor to the device by DMA. 112142356a74Sriastradh * Matches PREWRITE in sun8i_crypto_task_submit. 112242356a74Sriastradh */ 112342356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 1124defaca02Sriastradh sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE); 1125defaca02Sriastradh 112642356a74Sriastradh /* 112742356a74Sriastradh * Finished with all the other bits of DMA too. Matches 112842356a74Sriastradh * sun8i_crypto_task_load. 112942356a74Sriastradh */ 113042356a74Sriastradh nbytes = task->ct_nbytes; 113142356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 113242356a74Sriastradh BUS_DMASYNC_POSTREAD); 113342356a74Sriastradh if (task->ct_flags & TASK_SRC) 113442356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes, 113542356a74Sriastradh BUS_DMASYNC_POSTWRITE); 113642356a74Sriastradh if (task->ct_flags & TASK_CTR) 113742356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0, 113886953edcSriastradh task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD); 113942356a74Sriastradh if (task->ct_flags & TASK_IV) 114042356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0, 114142356a74Sriastradh task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 114242356a74Sriastradh if (task->ct_flags & TASK_KEY) 114342356a74Sriastradh /* XXX Can we zero the bounce buffer if there is one? */ 114442356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0, 114542356a74Sriastradh task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 114642356a74Sriastradh 1147defaca02Sriastradh /* Temporarily release the lock to invoke the callback. */ 1148defaca02Sriastradh mutex_exit(&sc->sc_lock); 114986953edcSriastradh SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error); 1150defaca02Sriastradh (*task->ct_callback)(sc, task, task->ct_cookie, error); 1151defaca02Sriastradh mutex_enter(&sc->sc_lock); 1152defaca02Sriastradh } 1153defaca02Sriastradh 1154defaca02Sriastradh /* 115542356a74Sriastradh * sun8i_crypto_allocbuf(sc, size, buf, dmaflags) 115642356a74Sriastradh * 115742356a74Sriastradh * Allocate a single-segment DMA-safe buffer and map it into KVA. 115842356a74Sriastradh * May fail if dmaflags is BUS_DMA_NOWAIT. 1159defaca02Sriastradh */ 1160defaca02Sriastradh static int 1161defaca02Sriastradh sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size, 116242356a74Sriastradh struct sun8i_crypto_buf *buf, int dmaflags) 1163defaca02Sriastradh { 1164defaca02Sriastradh int error; 1165defaca02Sriastradh 1166defaca02Sriastradh /* Allocate a DMA-safe buffer. */ 116786953edcSriastradh error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0, 116886953edcSriastradh buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags); 1169defaca02Sriastradh if (error) 1170defaca02Sriastradh goto fail0; 1171defaca02Sriastradh 1172defaca02Sriastradh /* Map the buffer into kernel virtual address space. */ 1173defaca02Sriastradh error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs, 117442356a74Sriastradh size, &buf->cb_kva, dmaflags); 1175defaca02Sriastradh if (error) 1176defaca02Sriastradh goto fail1; 1177defaca02Sriastradh 1178defaca02Sriastradh /* Success! */ 1179defaca02Sriastradh return 0; 1180defaca02Sriastradh 118142356a74Sriastradh fail2: __unused 118242356a74Sriastradh bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1183defaca02Sriastradh fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1184defaca02Sriastradh fail0: return error; 1185defaca02Sriastradh } 1186defaca02Sriastradh 118742356a74Sriastradh /* 118842356a74Sriastradh * sun8i_crypto_freebuf(sc, buf) 118942356a74Sriastradh * 119042356a74Sriastradh * Unmap buf and free it. 119142356a74Sriastradh */ 1192defaca02Sriastradh static void 1193defaca02Sriastradh sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size, 1194defaca02Sriastradh struct sun8i_crypto_buf *buf) 1195defaca02Sriastradh { 1196defaca02Sriastradh 1197defaca02Sriastradh bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1198defaca02Sriastradh bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1199defaca02Sriastradh } 1200defaca02Sriastradh 1201defaca02Sriastradh /* 120242356a74Sriastradh * sun8i_crypto_rng_attach(sc) 120342356a74Sriastradh * 120442356a74Sriastradh * Attach an rndsource for the crypto engine's TRNG. 1205defaca02Sriastradh */ 1206defaca02Sriastradh static void 1207defaca02Sriastradh sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc) 1208defaca02Sriastradh { 1209defaca02Sriastradh device_t self = sc->sc_dev; 1210defaca02Sriastradh struct sun8i_crypto_rng *rng = &sc->sc_rng; 121142356a74Sriastradh struct sun8i_crypto_task *task; 1212defaca02Sriastradh int error; 1213defaca02Sriastradh 1214defaca02Sriastradh /* Preallocate a buffer to reuse. */ 121542356a74Sriastradh error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf, 121642356a74Sriastradh BUS_DMA_WAITOK); 121786953edcSriastradh if (error) { 121886953edcSriastradh aprint_error_dev(self, "failed to allocate RNG buffer: %d\n", 121986953edcSriastradh error); 1220defaca02Sriastradh goto fail0; 122186953edcSriastradh } 1222defaca02Sriastradh 1223defaca02Sriastradh /* Create a task to reuse. */ 122442356a74Sriastradh task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done, 122542356a74Sriastradh rng, PR_WAITOK); 122642356a74Sriastradh if (rng->cr_task == NULL) { 122786953edcSriastradh aprint_error_dev(self, "failed to allocate RNG task\n"); 122842356a74Sriastradh error = ENOMEM; 1229defaca02Sriastradh goto fail1; 123042356a74Sriastradh } 123142356a74Sriastradh 123242356a74Sriastradh /* Preload the destination map. */ 123342356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 123442356a74Sriastradh rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 123586953edcSriastradh if (error) { 123686953edcSriastradh aprint_error_dev(self, "failed to load RNG buffer: %d\n", 123786953edcSriastradh error); 123842356a74Sriastradh goto fail2; 123986953edcSriastradh } 1240defaca02Sriastradh 1241defaca02Sriastradh /* 1242defaca02Sriastradh * Attach the rndsource. This is _not_ marked as RND_TYPE_RNG 1243defaca02Sriastradh * because the output is not uniformly distributed. The bits 1244defaca02Sriastradh * are heavily weighted toward 0 or 1, at different times, and 1245defaca02Sriastradh * I haven't scienced a satisfactory story out of it yet. 1246defaca02Sriastradh */ 1247defaca02Sriastradh rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc); 1248defaca02Sriastradh rnd_attach_source(&rng->cr_rndsource, device_xname(self), 1249defaca02Sriastradh RND_TYPE_UNKNOWN, 1250defaca02Sriastradh RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB); 1251defaca02Sriastradh 1252defaca02Sriastradh /* Success! */ 1253defaca02Sriastradh return; 1254defaca02Sriastradh 125542356a74Sriastradh fail3: __unused 125642356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 125742356a74Sriastradh fail2: sun8i_crypto_task_put(sc, task); 1258defaca02Sriastradh fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf); 125986953edcSriastradh fail0: return; 1260defaca02Sriastradh } 1261defaca02Sriastradh 126242356a74Sriastradh /* 126342356a74Sriastradh * sun8i_crypto_rng_get(nbytes, cookie) 126442356a74Sriastradh * 126542356a74Sriastradh * On-demand rndsource callback: try to gather nbytes of entropy 126642356a74Sriastradh * and enter them into the pool ASAP. 126742356a74Sriastradh */ 1268defaca02Sriastradh static void 1269defaca02Sriastradh sun8i_crypto_rng_get(size_t nbytes, void *cookie) 1270defaca02Sriastradh { 1271defaca02Sriastradh struct sun8i_crypto_softc *sc = cookie; 1272defaca02Sriastradh struct sun8i_crypto_rng *rng = &sc->sc_rng; 127342356a74Sriastradh struct sun8i_crypto_task *task = rng->cr_task; 1274defaca02Sriastradh bool pending; 1275defaca02Sriastradh int error; 1276defaca02Sriastradh 1277defaca02Sriastradh /* 1278defaca02Sriastradh * Test and set the RNG-pending flag. If it's already in 1279defaca02Sriastradh * progress, nothing to do here. 1280defaca02Sriastradh */ 1281defaca02Sriastradh mutex_enter(&sc->sc_lock); 1282defaca02Sriastradh pending = rng->cr_pending; 1283defaca02Sriastradh rng->cr_pending = true; 1284defaca02Sriastradh mutex_exit(&sc->sc_lock); 1285defaca02Sriastradh if (pending) 1286defaca02Sriastradh return; 1287defaca02Sriastradh 128842356a74Sriastradh /* Load the task descriptor. */ 128942356a74Sriastradh error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 129042356a74Sriastradh if (error) 129142356a74Sriastradh goto fail; 1292defaca02Sriastradh 129342356a74Sriastradh /* Submit! */ 129442356a74Sriastradh error = sun8i_crypto_submit(sc, task); 1295defaca02Sriastradh if (error) 1296defaca02Sriastradh goto fail; 1297defaca02Sriastradh 1298defaca02Sriastradh /* All done! */ 1299defaca02Sriastradh return; 1300defaca02Sriastradh 1301defaca02Sriastradh fail: mutex_enter(&sc->sc_lock); 1302defaca02Sriastradh rng->cr_pending = false; 1303defaca02Sriastradh mutex_exit(&sc->sc_lock); 1304defaca02Sriastradh } 1305defaca02Sriastradh 1306defaca02Sriastradh static void 1307defaca02Sriastradh sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc, 1308defaca02Sriastradh struct sun8i_crypto_task *task, void *cookie, int error) 1309defaca02Sriastradh { 1310defaca02Sriastradh struct sun8i_crypto_rng *rng = cookie; 1311defaca02Sriastradh uint8_t *buf = rng->cr_buf.cb_kva; 1312defaca02Sriastradh uint32_t entropybits; 1313defaca02Sriastradh 1314defaca02Sriastradh KASSERT(rng == &sc->sc_rng); 1315defaca02Sriastradh 1316defaca02Sriastradh /* If anything went wrong, forget about it. */ 1317defaca02Sriastradh if (error) 1318defaca02Sriastradh goto out; 1319defaca02Sriastradh 1320defaca02Sriastradh /* 1321defaca02Sriastradh * This TRNG has quite low entropy at best. But if it fails a 1322defaca02Sriastradh * repeated output test, then assume it's busted. 1323defaca02Sriastradh */ 1324f9ae7d17Sriastradh CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY); 1325f9ae7d17Sriastradh entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY; 1326defaca02Sriastradh if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2, 1327defaca02Sriastradh SUN8I_CRYPTO_RNGBYTES/2)) { 1328defaca02Sriastradh device_printf(sc->sc_dev, "failed repeated output test\n"); 1329defaca02Sriastradh entropybits = 0; 1330defaca02Sriastradh } 1331defaca02Sriastradh 1332560536a1Sriastradh /* 1333560536a1Sriastradh * Actually we don't believe in any of the entropy until this 1334560536a1Sriastradh * device has had more scrutiny. 1335560536a1Sriastradh */ 1336560536a1Sriastradh entropybits = 0; 1337560536a1Sriastradh 1338defaca02Sriastradh /* Success! Enter and erase the data. */ 1339defaca02Sriastradh rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES, 1340defaca02Sriastradh entropybits); 1341defaca02Sriastradh explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES); 1342defaca02Sriastradh 1343defaca02Sriastradh out: /* Done -- clear the RNG-pending flag. */ 1344defaca02Sriastradh mutex_enter(&sc->sc_lock); 1345defaca02Sriastradh rng->cr_pending = false; 1346defaca02Sriastradh mutex_exit(&sc->sc_lock); 1347defaca02Sriastradh } 1348defaca02Sriastradh 1349defaca02Sriastradh /* 1350defaca02Sriastradh * Self-test 1351defaca02Sriastradh */ 1352defaca02Sriastradh 1353dc50386fSriastradh static const uint8_t selftest_input[16]; 1354dc50386fSriastradh static const uint8_t selftest_key[16]; 1355dc50386fSriastradh static const uint8_t selftest_output[16] = { 1356defaca02Sriastradh 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b, 1357defaca02Sriastradh 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e, 1358defaca02Sriastradh }; 1359defaca02Sriastradh 1360defaca02Sriastradh static void 1361defaca02Sriastradh sun8i_crypto_selftest(device_t self) 1362defaca02Sriastradh { 136342356a74Sriastradh const size_t keybytes = sizeof selftest_key; 136442356a74Sriastradh const size_t nbytes = sizeof selftest_input; 1365defaca02Sriastradh struct sun8i_crypto_softc *sc = device_private(self); 1366defaca02Sriastradh struct sun8i_crypto_selftest *selftest = &sc->sc_selftest; 136742356a74Sriastradh struct sun8i_crypto_task *task; 1368defaca02Sriastradh int error; 1369defaca02Sriastradh 1370defaca02Sriastradh CTASSERT(sizeof selftest_input == sizeof selftest_output); 1371defaca02Sriastradh 1372defaca02Sriastradh /* Allocate an input buffer. */ 137342356a74Sriastradh error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in, 137442356a74Sriastradh BUS_DMA_WAITOK); 1375defaca02Sriastradh if (error) 1376defaca02Sriastradh goto fail0; 1377defaca02Sriastradh 1378defaca02Sriastradh /* Allocate a key buffer. */ 137942356a74Sriastradh error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key, 138042356a74Sriastradh BUS_DMA_WAITOK); 1381defaca02Sriastradh if (error) 1382defaca02Sriastradh goto fail1; 1383defaca02Sriastradh 1384defaca02Sriastradh /* Allocate an output buffer. */ 138542356a74Sriastradh error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out, 138642356a74Sriastradh BUS_DMA_WAITOK); 1387defaca02Sriastradh if (error) 1388defaca02Sriastradh goto fail2; 1389defaca02Sriastradh 1390defaca02Sriastradh /* Allocate a task descriptor. */ 139142356a74Sriastradh task = selftest->cs_task = sun8i_crypto_task_get(sc, 139242356a74Sriastradh sun8i_crypto_selftest_done, selftest, PR_WAITOK); 139342356a74Sriastradh if (selftest->cs_task == NULL) { 139442356a74Sriastradh error = ENOMEM; 1395defaca02Sriastradh goto fail3; 139642356a74Sriastradh } 1397defaca02Sriastradh 1398defaca02Sriastradh /* Copy the input and key into their buffers. */ 139942356a74Sriastradh memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes); 140042356a74Sriastradh memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes); 1401defaca02Sriastradh 140242356a74Sriastradh /* Load the key, src, and dst for DMA transfers. */ 140342356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 140442356a74Sriastradh selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK); 1405defaca02Sriastradh if (error) 1406defaca02Sriastradh goto fail4; 140742356a74Sriastradh task->ct_flags |= TASK_KEY; 140842356a74Sriastradh 140942356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap, 141042356a74Sriastradh selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 141142356a74Sriastradh if (error) 141242356a74Sriastradh goto fail5; 141342356a74Sriastradh task->ct_flags |= TASK_SRC; 141442356a74Sriastradh 141542356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 141642356a74Sriastradh selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 141742356a74Sriastradh if (error) 141842356a74Sriastradh goto fail6; 141942356a74Sriastradh 142042356a74Sriastradh /* Set up the task descriptor. */ 142186953edcSriastradh error = sun8i_crypto_task_load_aesecb(sc, task, nbytes, 142242356a74Sriastradh SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC); 142386953edcSriastradh if (error) 142486953edcSriastradh goto fail7; 142542356a74Sriastradh 142642356a74Sriastradh /* Submit! */ 142742356a74Sriastradh error = sun8i_crypto_submit(sc, task); 142842356a74Sriastradh if (error) 142942356a74Sriastradh goto fail7; 1430defaca02Sriastradh 1431defaca02Sriastradh device_printf(sc->sc_dev, "AES-128 self-test initiated\n"); 1432defaca02Sriastradh 1433defaca02Sriastradh /* Success! */ 1434defaca02Sriastradh return; 1435defaca02Sriastradh 143642356a74Sriastradh fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 143742356a74Sriastradh fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 143842356a74Sriastradh fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 143942356a74Sriastradh fail4: sun8i_crypto_task_put(sc, task); 144042356a74Sriastradh fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 144142356a74Sriastradh fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 144242356a74Sriastradh fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1443defaca02Sriastradh fail0: aprint_error_dev(self, "failed to run self-test, error=%d\n", error); 1444defaca02Sriastradh } 1445defaca02Sriastradh 1446dc50386fSriastradh static bool 1447dc50386fSriastradh sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title, 1448dc50386fSriastradh size_t n, const void *expected, const void *actual) 1449dc50386fSriastradh { 1450dc50386fSriastradh const uint8_t *e = expected; 1451dc50386fSriastradh const uint8_t *a = actual; 1452dc50386fSriastradh size_t i; 1453dc50386fSriastradh 1454dc50386fSriastradh if (memcmp(e, a, n) == 0) 1455dc50386fSriastradh return true; 1456dc50386fSriastradh 1457dc50386fSriastradh device_printf(sc->sc_dev, "self-test: %s\n", title); 1458dc50386fSriastradh printf("expected: "); 1459dc50386fSriastradh for (i = 0; i < n; i++) 1460dc50386fSriastradh printf("%02hhx", e[i]); 1461dc50386fSriastradh printf("\n"); 1462dc50386fSriastradh printf("actual: "); 1463dc50386fSriastradh for (i = 0; i < n; i++) 1464dc50386fSriastradh printf("%02hhx", a[i]); 1465dc50386fSriastradh printf("\n"); 1466dc50386fSriastradh return false; 1467dc50386fSriastradh } 1468dc50386fSriastradh 1469defaca02Sriastradh static void 1470defaca02Sriastradh sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc, 1471defaca02Sriastradh struct sun8i_crypto_task *task, void *cookie, int error) 1472defaca02Sriastradh { 147342356a74Sriastradh const size_t keybytes = sizeof selftest_key; 147442356a74Sriastradh const size_t nbytes = sizeof selftest_input; 1475defaca02Sriastradh struct sun8i_crypto_selftest *selftest = cookie; 1476defaca02Sriastradh bool ok = true; 1477defaca02Sriastradh 1478defaca02Sriastradh KASSERT(selftest == &sc->sc_selftest); 1479defaca02Sriastradh 1480dc50386fSriastradh /* If anything went wrong, fail now. */ 1481defaca02Sriastradh if (error) { 1482defaca02Sriastradh device_printf(sc->sc_dev, "self-test error=%d\n", error); 1483defaca02Sriastradh goto out; 1484defaca02Sriastradh } 1485defaca02Sriastradh 1486dc50386fSriastradh /* 1487dc50386fSriastradh * Verify the input and key weren't clobbered, and verify the 1488dc50386fSriastradh * output matches what we expect. 1489dc50386fSriastradh */ 149042356a74Sriastradh ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes, 149142356a74Sriastradh selftest_input, selftest->cs_in.cb_kva); 149242356a74Sriastradh ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes, 149342356a74Sriastradh selftest_key, selftest->cs_key.cb_kva); 149442356a74Sriastradh ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes, 149542356a74Sriastradh selftest_output, selftest->cs_out.cb_kva); 1496defaca02Sriastradh 1497defaca02Sriastradh /* XXX Disable the RNG and other stuff if this fails... */ 1498defaca02Sriastradh if (ok) 1499defaca02Sriastradh device_printf(sc->sc_dev, "AES-128 self-test passed\n"); 1500defaca02Sriastradh 150142356a74Sriastradh out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 150242356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 150342356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 150442356a74Sriastradh sun8i_crypto_task_put(sc, task); 150542356a74Sriastradh sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 150642356a74Sriastradh sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 150742356a74Sriastradh sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1508defaca02Sriastradh } 1509defaca02Sriastradh 1510defaca02Sriastradh /* 1511defaca02Sriastradh * Sysctl for testing 1512defaca02Sriastradh */ 1513defaca02Sriastradh 1514defaca02Sriastradh struct sun8i_crypto_userreq { 1515defaca02Sriastradh kmutex_t cu_lock; 1516defaca02Sriastradh kcondvar_t cu_cv; 1517defaca02Sriastradh size_t cu_size; 1518defaca02Sriastradh struct sun8i_crypto_buf cu_buf; 1519defaca02Sriastradh struct sun8i_crypto_task *cu_task; 1520defaca02Sriastradh int cu_error; 1521defaca02Sriastradh bool cu_done; 1522defaca02Sriastradh bool cu_cancel; 1523defaca02Sriastradh }; 1524defaca02Sriastradh 1525defaca02Sriastradh static void 1526defaca02Sriastradh sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc) 1527defaca02Sriastradh { 1528defaca02Sriastradh struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl; 1529defaca02Sriastradh int error; 1530defaca02Sriastradh 153144c3f65aSriastradh /* hw.sun8icryptoN (node) */ 1532defaca02Sriastradh error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node, 1533defaca02Sriastradh CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev), 1534defaca02Sriastradh SYSCTL_DESCR("sun8i crypto engine knobs"), 1535defaca02Sriastradh NULL, 0, NULL, 0, 1536defaca02Sriastradh CTL_HW, CTL_CREATE, CTL_EOL); 1537defaca02Sriastradh if (error) { 1538defaca02Sriastradh aprint_error_dev(sc->sc_dev, 1539defaca02Sriastradh "failed to set up sysctl hw.%s: %d\n", 1540defaca02Sriastradh device_xname(sc->sc_dev), error); 1541defaca02Sriastradh return; 1542defaca02Sriastradh } 1543defaca02Sriastradh 1544f9ae7d17Sriastradh /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */ 1545827c081cSriastradh sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node, 1546defaca02Sriastradh CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT, 1547f9ae7d17Sriastradh "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"), 1548defaca02Sriastradh &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL); 1549defaca02Sriastradh if (error) { 1550defaca02Sriastradh aprint_error_dev(sc->sc_dev, 1551defaca02Sriastradh "failed to set up sysctl hw.%s.rng: %d\n", 1552defaca02Sriastradh device_xname(sc->sc_dev), error); 1553defaca02Sriastradh return; 1554defaca02Sriastradh } 1555defaca02Sriastradh } 1556defaca02Sriastradh 1557defaca02Sriastradh static int 1558defaca02Sriastradh sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS) 1559defaca02Sriastradh { 1560defaca02Sriastradh struct sysctlnode node = *rnode; 1561defaca02Sriastradh struct sun8i_crypto_softc *sc = node.sysctl_data; 1562defaca02Sriastradh struct sun8i_crypto_userreq *req; 156342356a74Sriastradh struct sun8i_crypto_task *task; 1564defaca02Sriastradh size_t size; 1565defaca02Sriastradh int error; 1566defaca02Sriastradh 1567defaca02Sriastradh /* If oldp == NULL, the caller wants to learn the size. */ 1568defaca02Sriastradh if (oldp == NULL) { 1569f9ae7d17Sriastradh *oldlenp = 4096; 1570defaca02Sriastradh return 0; 1571defaca02Sriastradh } 1572defaca02Sriastradh 1573843b783fSriastradh /* Truncate to 4096 bytes. */ 1574843b783fSriastradh size = MIN(4096, *oldlenp); 1575defaca02Sriastradh if (size == 0) 1576defaca02Sriastradh return 0; /* nothing to do */ 1577defaca02Sriastradh 1578defaca02Sriastradh /* Allocate a request context. */ 1579defaca02Sriastradh req = kmem_alloc(sizeof(*req), KM_NOSLEEP); 1580defaca02Sriastradh if (req == NULL) 1581defaca02Sriastradh return ENOMEM; 1582defaca02Sriastradh 1583defaca02Sriastradh /* Initialize the request context. */ 1584defaca02Sriastradh mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE); 1585defaca02Sriastradh cv_init(&req->cu_cv, "sun8isy"); 1586defaca02Sriastradh req->cu_size = size; 1587defaca02Sriastradh req->cu_error = EIO; 1588defaca02Sriastradh req->cu_done = false; 1589defaca02Sriastradh req->cu_cancel = false; 1590defaca02Sriastradh 1591defaca02Sriastradh /* Allocate a buffer for the RNG output. */ 159242356a74Sriastradh error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT); 1593defaca02Sriastradh if (error) 1594defaca02Sriastradh goto out0; 1595defaca02Sriastradh 1596defaca02Sriastradh /* Allocate a task. */ 159742356a74Sriastradh task = req->cu_task = sun8i_crypto_task_get(sc, 159842356a74Sriastradh sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT); 159942356a74Sriastradh if (task == NULL) { 160042356a74Sriastradh error = ENOMEM; 1601defaca02Sriastradh goto out1; 160242356a74Sriastradh } 1603defaca02Sriastradh 1604defaca02Sriastradh /* Set the task up for TRNG to our buffer. */ 160542356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 160642356a74Sriastradh req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 160742356a74Sriastradh if (error) 160842356a74Sriastradh goto out2; 160942356a74Sriastradh error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 161042356a74Sriastradh if (error) 161142356a74Sriastradh goto out3; 1612defaca02Sriastradh 161342356a74Sriastradh /* Submit! */ 161442356a74Sriastradh error = sun8i_crypto_submit(sc, task); 1615443fde03Sriastradh if (error) { 161642356a74Sriastradh /* Make sure we don't restart the syscall -- just fail. */ 1617443fde03Sriastradh if (error == ERESTART) 1618443fde03Sriastradh error = EBUSY; 161942356a74Sriastradh goto out3; 1620443fde03Sriastradh } 1621defaca02Sriastradh 1622defaca02Sriastradh /* Wait for the request to complete. */ 1623defaca02Sriastradh mutex_enter(&req->cu_lock); 1624defaca02Sriastradh while (!req->cu_done) { 1625defaca02Sriastradh error = cv_wait_sig(&req->cu_cv, &req->cu_lock); 1626defaca02Sriastradh if (error) { 1627defaca02Sriastradh /* 16284a726746Sriastradh * If we finished while waiting to acquire the 16294a726746Sriastradh * lock, ignore the error and just return now. 16304a726746Sriastradh * Otherwise, notify the callback that it has 16314a726746Sriastradh * to clean up after us. 1632defaca02Sriastradh */ 16334a726746Sriastradh if (req->cu_done) 16344a726746Sriastradh error = 0; 16354a726746Sriastradh else 1636defaca02Sriastradh req->cu_cancel = true; 1637defaca02Sriastradh break; 1638defaca02Sriastradh } 1639defaca02Sriastradh } 1640defaca02Sriastradh mutex_exit(&req->cu_lock); 1641defaca02Sriastradh 1642defaca02Sriastradh /* 1643defaca02Sriastradh * Return early on error from cv_wait_sig, which means 1644defaca02Sriastradh * interruption; the callback will clean up instead. 1645defaca02Sriastradh */ 1646defaca02Sriastradh if (error) 1647defaca02Sriastradh return error; 1648defaca02Sriastradh 1649defaca02Sriastradh /* Check for error from the device. */ 1650defaca02Sriastradh error = req->cu_error; 1651defaca02Sriastradh if (error) 165242356a74Sriastradh goto out3; 1653defaca02Sriastradh 1654defaca02Sriastradh /* Copy out the data. */ 1655defaca02Sriastradh node.sysctl_data = req->cu_buf.cb_kva; 1656defaca02Sriastradh node.sysctl_size = size; 1657defaca02Sriastradh error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1658defaca02Sriastradh 1659defaca02Sriastradh /* Clear the buffer. */ 1660defaca02Sriastradh explicit_memset(req->cu_buf.cb_kva, 0, size); 1661defaca02Sriastradh 16624a726746Sriastradh /* Clean up. */ 166342356a74Sriastradh out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 166442356a74Sriastradh out2: sun8i_crypto_task_put(sc, task); 1665defaca02Sriastradh out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1666defaca02Sriastradh out0: cv_destroy(&req->cu_cv); 1667defaca02Sriastradh mutex_destroy(&req->cu_lock); 1668845298d6Sriastradh kmem_free(req, sizeof(*req)); 1669defaca02Sriastradh return error; 1670defaca02Sriastradh } 1671defaca02Sriastradh 1672defaca02Sriastradh static void 1673defaca02Sriastradh sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc, 1674defaca02Sriastradh struct sun8i_crypto_task *task, void *cookie, int error) 1675defaca02Sriastradh { 1676defaca02Sriastradh struct sun8i_crypto_userreq *req = cookie; 1677defaca02Sriastradh bool cancel; 1678defaca02Sriastradh 1679defaca02Sriastradh /* 1680defaca02Sriastradh * Notify the waiting thread of the error, and find out whether 1681defaca02Sriastradh * that thread cancelled. 1682defaca02Sriastradh */ 1683defaca02Sriastradh mutex_enter(&req->cu_lock); 1684defaca02Sriastradh cancel = req->cu_cancel; 1685defaca02Sriastradh req->cu_error = error; 1686defaca02Sriastradh req->cu_done = true; 1687defaca02Sriastradh cv_broadcast(&req->cu_cv); 1688defaca02Sriastradh mutex_exit(&req->cu_lock); 1689defaca02Sriastradh 1690defaca02Sriastradh /* 1691defaca02Sriastradh * If it wasn't cancelled, we're done -- the main thread will 1692defaca02Sriastradh * clean up after itself. 1693defaca02Sriastradh */ 1694defaca02Sriastradh if (!cancel) 1695defaca02Sriastradh return; 1696defaca02Sriastradh 16974a726746Sriastradh /* Clean up after the main thread cancelled. */ 169842356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 169942356a74Sriastradh sun8i_crypto_task_put(sc, task); 1700defaca02Sriastradh sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1701defaca02Sriastradh cv_destroy(&req->cu_cv); 1702defaca02Sriastradh mutex_destroy(&req->cu_lock); 1703845298d6Sriastradh kmem_free(req, sizeof(*req)); 1704defaca02Sriastradh } 170586953edcSriastradh 170686953edcSriastradh /* 170786953edcSriastradh * sun8i_crypto_register(sc) 170886953edcSriastradh * 170986953edcSriastradh * Register opencrypto algorithms supported by the crypto engine. 171086953edcSriastradh */ 171186953edcSriastradh static void 171286953edcSriastradh sun8i_crypto_register(struct sun8i_crypto_softc *sc) 171386953edcSriastradh { 171486953edcSriastradh struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto; 171586953edcSriastradh 171686953edcSriastradh co->co_driverid = crypto_get_driverid(0); 171786953edcSriastradh if (co->co_driverid == (uint32_t)-1) { 171886953edcSriastradh aprint_error_dev(sc->sc_dev, 171986953edcSriastradh "failed to register crypto driver\n"); 172086953edcSriastradh return; 172186953edcSriastradh } 172286953edcSriastradh 172386953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_AES_CBC); 172486953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_AES_CTR); 172586953edcSriastradh #ifdef CRYPTO_AES_ECB 172686953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_AES_ECB); 172786953edcSriastradh #endif 172886953edcSriastradh #ifdef CRYPTO_AES_XTS 172986953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_AES_XTS); 173086953edcSriastradh #endif 173186953edcSriastradh #ifdef CRYPTO_DES_CBC 173286953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_DES_CBC); 173386953edcSriastradh #endif 173486953edcSriastradh #ifdef CRYPTO_DES_ECB 173586953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_DES_ECB); 173686953edcSriastradh #endif 173786953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_3DES_CBC); 173886953edcSriastradh #ifdef CRYPTO_3DES_ECB 173986953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_3DES_ECB); 174086953edcSriastradh #endif 174186953edcSriastradh 174286953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_MD5); 174386953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA1); 174486953edcSriastradh #ifdef CRYPTO_SHA224 174586953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA224); 174686953edcSriastradh #endif 174786953edcSriastradh #ifdef CRYPTO_SHA256 174886953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA256); 174986953edcSriastradh #endif 175086953edcSriastradh 175186953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC); 175286953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC); 175386953edcSriastradh 175486953edcSriastradh //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */ 175586953edcSriastradh } 175686953edcSriastradh 175786953edcSriastradh /* 175886953edcSriastradh * sun8i_crypto_register1(sc, alg) 175986953edcSriastradh * 176086953edcSriastradh * Register support for one algorithm alg using 176186953edcSriastradh * sun8i_crypto_newsession/freesession/process. 176286953edcSriastradh */ 176386953edcSriastradh static void 176486953edcSriastradh sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg) 176586953edcSriastradh { 176686953edcSriastradh 176786953edcSriastradh crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0, 176886953edcSriastradh sun8i_crypto_newsession, 176986953edcSriastradh sun8i_crypto_freesession, 177086953edcSriastradh sun8i_crypto_process, 177186953edcSriastradh sc); 177286953edcSriastradh } 177386953edcSriastradh 177486953edcSriastradh /* 177586953edcSriastradh * sun8i_crypto_newsession(cookie, sidp, cri) 177686953edcSriastradh * 177786953edcSriastradh * Called by opencrypto to allocate a new session. We don't keep 177886953edcSriastradh * track of sessions, since there are no persistent keys in the 177986953edcSriastradh * hardware that we take advantage of, so this only validates the 178086953edcSriastradh * crypto operations and returns a zero session id. 178186953edcSriastradh */ 178286953edcSriastradh static int 178386953edcSriastradh sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri) 178486953edcSriastradh { 178586953edcSriastradh 178686953edcSriastradh /* No composition of operations is supported here. */ 178786953edcSriastradh if (cri->cri_next) 178886953edcSriastradh return EINVAL; 178986953edcSriastradh 179086953edcSriastradh /* 179186953edcSriastradh * No variation of rounds is supported here. (XXX Unused and 179286953edcSriastradh * unimplemented in opencrypto(9) altogether? 179386953edcSriastradh */ 179486953edcSriastradh if (cri->cri_rnd) 179586953edcSriastradh return EINVAL; 179686953edcSriastradh 179786953edcSriastradh /* 179886953edcSriastradh * Validate per-algorithm key length. 179986953edcSriastradh * 180086953edcSriastradh * XXX Does opencrypto(9) do this internally? 180186953edcSriastradh */ 180286953edcSriastradh switch (cri->cri_alg) { 180386953edcSriastradh case CRYPTO_MD5: 180486953edcSriastradh case CRYPTO_SHA1: 180586953edcSriastradh #ifdef CRYPTO_SHA224 180686953edcSriastradh case CRYPTO_SHA224: 180786953edcSriastradh #endif 180886953edcSriastradh #ifdef CRYPTO_SHA256 180986953edcSriastradh case CRYPTO_SHA256: 181086953edcSriastradh #endif 181186953edcSriastradh if (cri->cri_klen) 181286953edcSriastradh return EINVAL; 181386953edcSriastradh break; 181486953edcSriastradh case CRYPTO_AES_CBC: 181586953edcSriastradh #ifdef CRYPTO_AES_ECB 181686953edcSriastradh case CRYPTO_AES_ECB: 181786953edcSriastradh #endif 181886953edcSriastradh switch (cri->cri_klen) { 181986953edcSriastradh case 128: 182086953edcSriastradh case 192: 182186953edcSriastradh case 256: 182286953edcSriastradh break; 182386953edcSriastradh default: 182486953edcSriastradh return EINVAL; 182586953edcSriastradh } 182686953edcSriastradh break; 182786953edcSriastradh case CRYPTO_AES_CTR: 182886953edcSriastradh /* 182986953edcSriastradh * opencrypto `AES-CTR' takes four bytes of the input 183086953edcSriastradh * block as the last four bytes of the key, for reasons 183186953edcSriastradh * that are not entirely clear. 183286953edcSriastradh */ 183386953edcSriastradh switch (cri->cri_klen) { 183486953edcSriastradh case 128 + 32: 183586953edcSriastradh case 192 + 32: 183686953edcSriastradh case 256 + 32: 183786953edcSriastradh break; 183886953edcSriastradh default: 183986953edcSriastradh return EINVAL; 184086953edcSriastradh } 184186953edcSriastradh break; 184286953edcSriastradh #ifdef CRYPTO_AES_XTS 184386953edcSriastradh case CRYPTO_AES_XTS: 184486953edcSriastradh switch (cri->cri_klen) { 184586953edcSriastradh case 256: 184686953edcSriastradh case 384: 184786953edcSriastradh case 512: 184886953edcSriastradh break; 184986953edcSriastradh default: 185086953edcSriastradh return EINVAL; 185186953edcSriastradh } 185286953edcSriastradh break; 185386953edcSriastradh #endif 185486953edcSriastradh case CRYPTO_DES_CBC: 185586953edcSriastradh #ifdef CRYPTO_DES_ECB 185686953edcSriastradh case CRYPTO_DES_ECB: 185786953edcSriastradh #endif 185886953edcSriastradh switch (cri->cri_klen) { 185986953edcSriastradh case 64: 186086953edcSriastradh break; 186186953edcSriastradh default: 186286953edcSriastradh return EINVAL; 186386953edcSriastradh } 186486953edcSriastradh break; 186586953edcSriastradh case CRYPTO_3DES_CBC: 186686953edcSriastradh #ifdef CRYPTO_3DES_ECB 186786953edcSriastradh case CRYPTO_3DES_ECB: 186886953edcSriastradh #endif 186986953edcSriastradh switch (cri->cri_klen) { 187086953edcSriastradh case 192: 187186953edcSriastradh break; 187286953edcSriastradh default: 187386953edcSriastradh return EINVAL; 187486953edcSriastradh } 187586953edcSriastradh break; 187686953edcSriastradh case CRYPTO_SHA1_HMAC: 187786953edcSriastradh /* 187886953edcSriastradh * XXX Unclear what the length limit is, but since HMAC 187986953edcSriastradh * behaves qualitatively different for a key of at 188086953edcSriastradh * least the full block size -- and is generally best 188186953edcSriastradh * to use with half the block size -- let's limit it to 188286953edcSriastradh * one block. 188386953edcSriastradh */ 188486953edcSriastradh if (cri->cri_klen % 8) 188586953edcSriastradh return EINVAL; 188686953edcSriastradh if (cri->cri_klen > 512) 188786953edcSriastradh return EINVAL; 188886953edcSriastradh break; 188986953edcSriastradh case CRYPTO_SHA2_256_HMAC: 189086953edcSriastradh if (cri->cri_klen % 8) 189186953edcSriastradh return EINVAL; 189286953edcSriastradh if (cri->cri_klen > 512) 189386953edcSriastradh return EINVAL; 189486953edcSriastradh break; 189586953edcSriastradh default: 189686953edcSriastradh panic("unsupported algorithm %d", cri->cri_alg); 189786953edcSriastradh } 189886953edcSriastradh 189986953edcSriastradh KASSERT(cri->cri_klen % 8 == 0); 190086953edcSriastradh 190186953edcSriastradh /* Success! */ 190286953edcSriastradh *sidp = 1; 190386953edcSriastradh return 0; 190486953edcSriastradh } 190586953edcSriastradh 190686953edcSriastradh /* 190786953edcSriastradh * sun8i_crypto_freesession(cookie, dsid) 190886953edcSriastradh * 190986953edcSriastradh * Called by opencrypto to free a session. We don't keep track of 191086953edcSriastradh * sessions, since there are no persistent keys in the hardware 191186953edcSriastradh * that we take advantage of, so this is a no-op. 191286953edcSriastradh * 191386953edcSriastradh * Note: dsid is actually a 64-bit quantity containing both the 191486953edcSriastradh * driver id in the high half and the session id in the low half. 191586953edcSriastradh */ 191686953edcSriastradh static int 191786953edcSriastradh sun8i_crypto_freesession(void *cookie, uint64_t dsid) 191886953edcSriastradh { 191986953edcSriastradh 192086953edcSriastradh KASSERT((dsid & 0xffffffff) == 1); 192186953edcSriastradh 192286953edcSriastradh /* Success! */ 192386953edcSriastradh return 0; 192486953edcSriastradh } 192586953edcSriastradh 192686953edcSriastradh /* 192786953edcSriastradh * sun8i_crypto_ivlen(crd) 192886953edcSriastradh * 192986953edcSriastradh * Return the crypto engine's notion of `IV length', in bytes, for 193086953edcSriastradh * an opencrypto operation. 193186953edcSriastradh */ 193286953edcSriastradh static u_int 193386953edcSriastradh sun8i_crypto_ivlen(const struct cryptodesc *crd) 193486953edcSriastradh { 193586953edcSriastradh 193686953edcSriastradh switch (crd->crd_alg) { 193786953edcSriastradh case CRYPTO_AES_CBC: 193886953edcSriastradh return 16; 193986953edcSriastradh #ifdef CRYPTO_AES_XTS 194086953edcSriastradh case CRYPTO_AES_XTS: 194186953edcSriastradh return 16; 194286953edcSriastradh #endif 194386953edcSriastradh case CRYPTO_AES_CTR: /* XXX opencrypto quirk */ 194486953edcSriastradh return 8; 194586953edcSriastradh #ifdef CRYPTO_DES_CBC 194686953edcSriastradh case CRYPTO_DES_CBC: 194786953edcSriastradh return 8; 194886953edcSriastradh #endif 194986953edcSriastradh case CRYPTO_3DES_CBC: 195086953edcSriastradh return 8; 195186953edcSriastradh case CRYPTO_MD5: 195286953edcSriastradh return 16; 195386953edcSriastradh #ifdef CRYPTO_SHA224 195486953edcSriastradh case CRYPTO_SHA224: 195586953edcSriastradh return 32; 195686953edcSriastradh #endif 195786953edcSriastradh #ifdef CRYPTO_SHA256 195886953edcSriastradh case CRYPTO_SHA256: 195986953edcSriastradh return 32; 196086953edcSriastradh #endif 196186953edcSriastradh case CRYPTO_SHA1_HMAC: 196286953edcSriastradh return 20; 196386953edcSriastradh case CRYPTO_SHA2_256_HMAC: 196486953edcSriastradh return 32; 196586953edcSriastradh default: 196686953edcSriastradh return 0; 196786953edcSriastradh } 196886953edcSriastradh } 196986953edcSriastradh 197086953edcSriastradh /* 197186953edcSriastradh * sun8i_crypto_process(cookie, crp, hint) 197286953edcSriastradh * 197386953edcSriastradh * Main opencrypto processing dispatch. 197486953edcSriastradh */ 197586953edcSriastradh static int 197686953edcSriastradh sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint) 197786953edcSriastradh { 197886953edcSriastradh struct sun8i_crypto_softc *sc = cookie; 197986953edcSriastradh struct sun8i_crypto_task *task; 198086953edcSriastradh struct cryptodesc *crd = crp->crp_desc; 198186953edcSriastradh unsigned klen, ivlen; 198286953edcSriastradh uint32_t tdqc = 0, tdqs = 0; 198386953edcSriastradh uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0; 198486953edcSriastradh const uint32_t tdqa = 0; 198586953edcSriastradh int error; 198686953edcSriastradh 198786953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint); 198886953edcSriastradh 198986953edcSriastradh /* Reject compositions -- we do not handle them. */ 199086953edcSriastradh if (crd->crd_next != NULL) { 199186953edcSriastradh error = EOPNOTSUPP; 199286953edcSriastradh goto fail0; 199386953edcSriastradh } 199486953edcSriastradh 199586953edcSriastradh /* Reject transfers with nonsense skip. */ 199686953edcSriastradh if (crd->crd_skip < 0) { 199786953edcSriastradh error = EINVAL; 199886953edcSriastradh goto fail0; 199986953edcSriastradh } 200086953edcSriastradh 200186953edcSriastradh /* 200286953edcSriastradh * Actually just reject any nonzero skip, because it requires 200386953edcSriastradh * DMA segment bookkeeping that we don't do yet. 200486953edcSriastradh */ 200586953edcSriastradh if (crd->crd_skip) { 200686953edcSriastradh error = EOPNOTSUPP; 200786953edcSriastradh goto fail0; 200886953edcSriastradh } 200986953edcSriastradh 201086953edcSriastradh /* Reject large transfers. */ 201186953edcSriastradh if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) { 201286953edcSriastradh error = EFBIG; 201386953edcSriastradh goto fail0; 201486953edcSriastradh } 201586953edcSriastradh 201686953edcSriastradh /* Reject nonsense, unaligned, or mismatched lengths. */ 201786953edcSriastradh if (crd->crd_len < 0 || 201886953edcSriastradh crd->crd_len % 4 || 201986953edcSriastradh crd->crd_len != crp->crp_ilen) { 202086953edcSriastradh error = EINVAL; 202186953edcSriastradh goto fail0; 202286953edcSriastradh } 202386953edcSriastradh 202486953edcSriastradh /* Reject mismatched buffer lengths. */ 202586953edcSriastradh /* XXX Handle crd_skip. */ 202686953edcSriastradh if (crp->crp_flags & CRYPTO_F_IMBUF) { 202786953edcSriastradh struct mbuf *m = crp->crp_buf; 202886953edcSriastradh uint32_t nbytes = 0; 202986953edcSriastradh while (m != NULL) { 203086953edcSriastradh KASSERT(m->m_len >= 0); 203186953edcSriastradh if (m->m_len > crd->crd_len || 203286953edcSriastradh nbytes > crd->crd_len - m->m_len) { 203386953edcSriastradh error = EINVAL; 203486953edcSriastradh goto fail0; 203586953edcSriastradh } 203686953edcSriastradh nbytes += m->m_len; 203786953edcSriastradh m = m->m_next; 203886953edcSriastradh } 203986953edcSriastradh if (nbytes != crd->crd_len) { 204086953edcSriastradh error = EINVAL; 204186953edcSriastradh goto fail0; 204286953edcSriastradh } 204386953edcSriastradh } else if (crp->crp_flags & CRYPTO_F_IOV) { 204486953edcSriastradh struct uio *uio = crp->crp_buf; 204586953edcSriastradh if (uio->uio_resid != crd->crd_len) { 204686953edcSriastradh error = EINVAL; 204786953edcSriastradh goto fail0; 204886953edcSriastradh } 204986953edcSriastradh } 205086953edcSriastradh 205186953edcSriastradh /* Get a task, or fail with ERESTART if we can't. */ 205286953edcSriastradh task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp, 205386953edcSriastradh PR_NOWAIT); 205486953edcSriastradh if (task == NULL) { 205586953edcSriastradh /* 205686953edcSriastradh * Don't invoke crypto_done -- we are asking the 205786953edcSriastradh * opencrypto(9) machinery to queue the request and get 205886953edcSriastradh * back to us. 205986953edcSriastradh */ 206086953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint); 206186953edcSriastradh return ERESTART; 206286953edcSriastradh } 206386953edcSriastradh 206486953edcSriastradh /* Load key in, if relevant. */ 206586953edcSriastradh klen = crd->crd_klen; 206686953edcSriastradh if (klen) { 206786953edcSriastradh if (crd->crd_alg == CRYPTO_AES_CTR) 206886953edcSriastradh /* AES-CTR is special -- see IV processing below. */ 206986953edcSriastradh klen -= 32; 207086953edcSriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 207186953edcSriastradh crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT); 207286953edcSriastradh if (error) 207386953edcSriastradh goto fail1; 207486953edcSriastradh task->ct_flags |= TASK_KEY; 207586953edcSriastradh } 207686953edcSriastradh 207786953edcSriastradh /* Handle the IV, if relevant. */ 207886953edcSriastradh ivlen = sun8i_crypto_ivlen(crd); 207986953edcSriastradh if (ivlen) { 208086953edcSriastradh void *iv; 208186953edcSriastradh 208286953edcSriastradh /* 208386953edcSriastradh * If there's an explicit IV, use it; otherwise 208486953edcSriastradh * randomly generate one. 208586953edcSriastradh */ 208686953edcSriastradh if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 208786953edcSriastradh iv = crd->crd_iv; 208886953edcSriastradh } else { 208986953edcSriastradh cprng_fast(task->ct_iv, ivlen); 209086953edcSriastradh iv = task->ct_iv; 209186953edcSriastradh } 209286953edcSriastradh 209386953edcSriastradh /* 209486953edcSriastradh * If the IV is not already present in the user's 209586953edcSriastradh * buffer, copy it over. 209686953edcSriastradh */ 209786953edcSriastradh if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 209886953edcSriastradh if (crp->crp_flags & CRYPTO_F_IMBUF) { 209986953edcSriastradh m_copyback(crp->crp_buf, crd->crd_inject, 210086953edcSriastradh ivlen, iv); 210186953edcSriastradh } else if (crp->crp_flags & CRYPTO_F_IOV) { 210286953edcSriastradh cuio_copyback(crp->crp_buf, crd->crd_inject, 210386953edcSriastradh ivlen, iv); 210486953edcSriastradh } else { 210586953edcSriastradh panic("invalid buffer type %x", 210686953edcSriastradh crp->crp_flags); 210786953edcSriastradh } 210886953edcSriastradh } 210986953edcSriastradh 211086953edcSriastradh /* 211186953edcSriastradh * opencrypto's idea of `AES-CTR' is special. 211286953edcSriastradh * 211386953edcSriastradh * - The low 4 bytes of the input block are drawn from 211486953edcSriastradh * an extra 4 bytes at the end of the key. 211586953edcSriastradh * 211686953edcSriastradh * - The next 8 bytes of the input block are drawn from 211786953edcSriastradh * the opencrypto iv. 211886953edcSriastradh * 211986953edcSriastradh * - The high 4 bytes are the big-endian block counter, 212086953edcSriastradh * which starts at 1 because why not. 212186953edcSriastradh */ 212286953edcSriastradh if (crd->crd_alg == CRYPTO_AES_CTR) { 212386953edcSriastradh uint8_t block[16]; 212486953edcSriastradh uint32_t blkno = 1; 212586953edcSriastradh 212686953edcSriastradh /* Format the initial input block. */ 212786953edcSriastradh memcpy(block, crd->crd_key + klen/8, 4); 212886953edcSriastradh memcpy(block + 4, iv, 8); 212986953edcSriastradh be32enc(block + 12, blkno); 213086953edcSriastradh 213186953edcSriastradh /* Copy it into the DMA buffer. */ 213286953edcSriastradh memcpy(task->ct_iv, block, 16); 213386953edcSriastradh iv = task->ct_iv; 213486953edcSriastradh ivlen = 16; 213586953edcSriastradh } 213686953edcSriastradh 213786953edcSriastradh /* Load the IV. */ 213886953edcSriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen, 213986953edcSriastradh NULL, BUS_DMA_NOWAIT); 214086953edcSriastradh if (error) 214186953edcSriastradh goto fail1; 214286953edcSriastradh task->ct_flags |= TASK_IV; 214386953edcSriastradh } 214486953edcSriastradh 214586953edcSriastradh /* Load the src and dst. */ 214686953edcSriastradh if (crp->crp_flags & CRYPTO_F_IMBUF) { 214786953edcSriastradh struct mbuf *m = crp->crp_buf; 214886953edcSriastradh 214986953edcSriastradh /* XXX Handle crd_skip. */ 215086953edcSriastradh KASSERT(crd->crd_skip == 0); 215186953edcSriastradh error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m, 215286953edcSriastradh BUS_DMA_NOWAIT); 215386953edcSriastradh if (error) 215486953edcSriastradh goto fail1; 215586953edcSriastradh task->ct_flags |= TASK_SRC; 215686953edcSriastradh 215786953edcSriastradh /* XXX Handle crd_skip. */ 215886953edcSriastradh KASSERT(crd->crd_skip == 0); 215986953edcSriastradh error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m, 216086953edcSriastradh BUS_DMA_NOWAIT); 216186953edcSriastradh if (error) 216286953edcSriastradh goto fail1; 216386953edcSriastradh } else if (crp->crp_flags & CRYPTO_F_IOV) { 216486953edcSriastradh struct uio *uio = crp->crp_buf; 216586953edcSriastradh 216686953edcSriastradh /* XXX Handle crd_skip. */ 216786953edcSriastradh KASSERT(crd->crd_skip == 0); 216886953edcSriastradh error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio, 216986953edcSriastradh BUS_DMA_NOWAIT); 217086953edcSriastradh if (error) 217186953edcSriastradh goto fail1; 217286953edcSriastradh task->ct_flags |= TASK_SRC; 217386953edcSriastradh 217486953edcSriastradh /* XXX Handle crd_skip. */ 217586953edcSriastradh KASSERT(crd->crd_skip == 0); 217686953edcSriastradh error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio, 217786953edcSriastradh BUS_DMA_NOWAIT); 217886953edcSriastradh if (error) 217986953edcSriastradh goto fail1; 218086953edcSriastradh } else { 218186953edcSriastradh panic("invalid buffer type %x", crp->crp_flags); 218286953edcSriastradh } 218386953edcSriastradh 218486953edcSriastradh /* Set the encryption direction. */ 218586953edcSriastradh if (crd->crd_flags & CRD_F_ENCRYPT) 218686953edcSriastradh dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC; 218786953edcSriastradh else 218886953edcSriastradh dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC; 218986953edcSriastradh tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 219086953edcSriastradh 219186953edcSriastradh /* Set the method. */ 219286953edcSriastradh switch (crd->crd_alg) { 219386953edcSriastradh case CRYPTO_AES_CBC: 219486953edcSriastradh case CRYPTO_AES_CTR: 219586953edcSriastradh #ifdef CRYPTO_AES_ECB 219686953edcSriastradh case CRYPTO_AES_ECB: 219786953edcSriastradh #endif 219886953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_AES; 219986953edcSriastradh break; 220086953edcSriastradh #ifdef CRYPTO_AES_XTS 220186953edcSriastradh case CRYPTO_AES_XTS: 220286953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_AES; 220386953edcSriastradh break; 220486953edcSriastradh #endif 220586953edcSriastradh case CRYPTO_DES_CBC: 220686953edcSriastradh #ifdef CRYPTO_DES_ECB 220786953edcSriastradh case CRYPTO_DES_ECB: 220886953edcSriastradh #endif 220986953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_DES; 221086953edcSriastradh break; 221186953edcSriastradh case CRYPTO_3DES_CBC: 221286953edcSriastradh #ifdef CRYPTO_3DES_ECB 221386953edcSriastradh case CRYPTO_3DES_ECB: 221486953edcSriastradh #endif 221586953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_3DES; 221686953edcSriastradh break; 221786953edcSriastradh case CRYPTO_MD5: 221886953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_MD5; 221986953edcSriastradh break; 222086953edcSriastradh case CRYPTO_SHA1: 222186953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_SHA1; 222286953edcSriastradh break; 222386953edcSriastradh #ifdef CRYPTO_SHA224 222486953edcSriastradh case CRYPTO_SHA224: 222586953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_SHA224; 222686953edcSriastradh break; 222786953edcSriastradh #endif 222886953edcSriastradh #ifdef CRYPTO_SHA256 222986953edcSriastradh case CRYPTO_SHA256: 223086953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_SHA256; 223186953edcSriastradh break; 223286953edcSriastradh #endif 223386953edcSriastradh case CRYPTO_SHA1_HMAC: 223486953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1; 223586953edcSriastradh break; 223686953edcSriastradh case CRYPTO_SHA2_256_HMAC: 223786953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256; 223886953edcSriastradh break; 223986953edcSriastradh default: 224086953edcSriastradh panic("unknown algorithm %d", crd->crd_alg); 224186953edcSriastradh } 224286953edcSriastradh tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD); 224386953edcSriastradh 224486953edcSriastradh /* Set the key selector. No idea how to use the internal keys. */ 224586953edcSriastradh tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 224686953edcSriastradh SUN8I_CRYPTO_TDQS_SKEY_SELECT); 224786953edcSriastradh 224886953edcSriastradh /* XXX Deal with AES_CTS_Last_Block_Flag. */ 224986953edcSriastradh 225086953edcSriastradh /* Set the mode. */ 225186953edcSriastradh switch (crd->crd_alg) { 225286953edcSriastradh #ifdef CRYPTO_AES_ECB 225386953edcSriastradh case CRYPTO_AES_ECB: 225486953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 225586953edcSriastradh break; 225686953edcSriastradh #endif 225786953edcSriastradh #ifdef CRYPTO_DES_ECB 225886953edcSriastradh case CRYPTO_DES_ECB: 225986953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 226086953edcSriastradh break; 226186953edcSriastradh #endif 226286953edcSriastradh #ifdef CRYPTO_3DES_ECB 226386953edcSriastradh case CRYPTO_3DES_ECB: 226486953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 226586953edcSriastradh break; 226686953edcSriastradh #endif 226786953edcSriastradh case CRYPTO_AES_CBC: 226886953edcSriastradh case CRYPTO_DES_CBC: 226986953edcSriastradh case CRYPTO_3DES_CBC: 227086953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC; 227186953edcSriastradh break; 227286953edcSriastradh case CRYPTO_AES_CTR: 227386953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR; 227486953edcSriastradh break; 227586953edcSriastradh #ifdef CRYPTO_AES_XTS 227686953edcSriastradh case CRYPTO_AES_XTS: 227786953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS; 227886953edcSriastradh break; 227986953edcSriastradh #endif 228086953edcSriastradh default: 228186953edcSriastradh panic("unknown algorithm %d", crd->crd_alg); 228286953edcSriastradh } 228386953edcSriastradh tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE); 228486953edcSriastradh 228586953edcSriastradh /* Set the CTR width. */ 228686953edcSriastradh switch (crd->crd_alg) { 228786953edcSriastradh case CRYPTO_AES_CTR: 228886953edcSriastradh ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32; 228986953edcSriastradh break; 229086953edcSriastradh } 229186953edcSriastradh tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH); 229286953edcSriastradh 229386953edcSriastradh /* Set the AES key size. */ 229486953edcSriastradh switch (crd->crd_alg) { 229586953edcSriastradh case CRYPTO_AES_CBC: 229686953edcSriastradh #ifdef CRYPTO_AES_ECB 229786953edcSriastradh case CRYPTO_AES_ECB: 229886953edcSriastradh #endif 229986953edcSriastradh switch (crd->crd_klen) { 230086953edcSriastradh case 128: 230186953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 230286953edcSriastradh break; 230386953edcSriastradh case 192: 230486953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 230586953edcSriastradh break; 230686953edcSriastradh case 256: 230786953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 230886953edcSriastradh break; 230986953edcSriastradh default: 231086953edcSriastradh panic("invalid AES key size in bits: %u", 231186953edcSriastradh crd->crd_klen); 231286953edcSriastradh } 231386953edcSriastradh break; 231486953edcSriastradh case CRYPTO_AES_CTR: 231586953edcSriastradh switch (crd->crd_klen) { 231686953edcSriastradh case 128 + 32: 231786953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 231886953edcSriastradh break; 231986953edcSriastradh case 192 + 32: 232086953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 232186953edcSriastradh break; 232286953edcSriastradh case 256 + 32: 232386953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 232486953edcSriastradh break; 232586953edcSriastradh default: 232686953edcSriastradh panic("invalid `AES-CTR' ` ``key'' size' in bits: %u", 232786953edcSriastradh crd->crd_klen); 232886953edcSriastradh } 232986953edcSriastradh break; 233086953edcSriastradh #ifdef CRYPTO_AES_XTS 233186953edcSriastradh case CRYPTO_AES_XTS: 233286953edcSriastradh switch (crd->crd_klen) { 233386953edcSriastradh case 256: 233486953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 233586953edcSriastradh break; 233686953edcSriastradh case 384: 233786953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 233886953edcSriastradh break; 233986953edcSriastradh case 512: 234086953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 234186953edcSriastradh break; 234286953edcSriastradh default: 234386953edcSriastradh panic("invalid AES-XTS key size in bits: %u", 234486953edcSriastradh crd->crd_klen); 234586953edcSriastradh } 234686953edcSriastradh break; 234786953edcSriastradh #endif 234886953edcSriastradh } 234986953edcSriastradh tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 235086953edcSriastradh 235186953edcSriastradh /* Set up the task descriptor. */ 235286953edcSriastradh error = sun8i_crypto_task_load(sc, task, crd->crd_len, 235386953edcSriastradh tdqc, tdqs, tdqa); 235486953edcSriastradh if (error) 235586953edcSriastradh goto fail2; 235686953edcSriastradh 235786953edcSriastradh /* Submit! */ 235886953edcSriastradh error = sun8i_crypto_submit(sc, task); 235986953edcSriastradh if (error) 236086953edcSriastradh goto fail2; 236186953edcSriastradh 236286953edcSriastradh /* Success! */ 236386953edcSriastradh SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task); 236486953edcSriastradh return 0; 236586953edcSriastradh 236686953edcSriastradh fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 236786953edcSriastradh fail1: if (task->ct_flags & TASK_SRC) 236886953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 236986953edcSriastradh if (task->ct_flags & TASK_CTR) 237086953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 237186953edcSriastradh if (task->ct_flags & TASK_IV) 237286953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 237386953edcSriastradh if (task->ct_flags & TASK_KEY) 237486953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 237586953edcSriastradh sun8i_crypto_task_put(sc, task); 237686953edcSriastradh fail0: KASSERT(error); 237786953edcSriastradh KASSERT(error != ERESTART); 237886953edcSriastradh crp->crp_etype = error; 237986953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 238086953edcSriastradh crypto_done(crp); 238186953edcSriastradh return 0; 238286953edcSriastradh } 238386953edcSriastradh 238486953edcSriastradh /* 238586953edcSriastradh * sun8i_crypto_callback(sc, task, cookie, error) 238686953edcSriastradh * 238786953edcSriastradh * Completion callback for a task submitted via opencrypto. 238886953edcSriastradh * Release the task and pass the error on to opencrypto with 238986953edcSriastradh * crypto_done. 239086953edcSriastradh */ 239186953edcSriastradh static void 239286953edcSriastradh sun8i_crypto_callback(struct sun8i_crypto_softc *sc, 239386953edcSriastradh struct sun8i_crypto_task *task, void *cookie, int error) 239486953edcSriastradh { 239586953edcSriastradh struct cryptop *crp = cookie; 23965170c8cfSad struct cryptodesc *crd __diagused = crp->crp_desc; 239786953edcSriastradh 239886953edcSriastradh KASSERT(error != ERESTART); 239986953edcSriastradh KASSERT(crd != NULL); 240086953edcSriastradh KASSERT(crd->crd_next == NULL); 240186953edcSriastradh 240286953edcSriastradh /* Return the number of bytes processed. */ 240386953edcSriastradh crp->crp_olen = error ? 0 : crp->crp_ilen; 240486953edcSriastradh 240586953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 240686953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 240786953edcSriastradh if (task->ct_flags & TASK_CTR) 240886953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 240986953edcSriastradh if (task->ct_flags & TASK_IV) 241086953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 241186953edcSriastradh if (task->ct_flags & TASK_KEY) 241286953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 241386953edcSriastradh sun8i_crypto_task_put(sc, task); 241486953edcSriastradh KASSERT(error != ERESTART); 241586953edcSriastradh crp->crp_etype = error; 241686953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 241786953edcSriastradh crypto_done(crp); 241886953edcSriastradh } 2419