1091d81d1SSam Leffler /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2091d81d1SSam Leffler 360727d8bSWarner Losh /*- 4091d81d1SSam Leffler * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 56810ad6fSSam Leffler * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6091d81d1SSam Leffler * 7091d81d1SSam Leffler * This code was written by Angelos D. Keromytis in Athens, Greece, in 8091d81d1SSam Leffler * February 2000. Network Security Technologies Inc. (NSTI) kindly 9091d81d1SSam Leffler * supported the development of this code. 10091d81d1SSam Leffler * 11091d81d1SSam Leffler * Copyright (c) 2000, 2001 Angelos D. Keromytis 121833d604SJohn Baldwin * Copyright (c) 2014-2021 The FreeBSD Foundation 1308fca7a5SJohn-Mark Gurney * All rights reserved. 1408fca7a5SJohn-Mark Gurney * 1508fca7a5SJohn-Mark Gurney * Portions of this software were developed by John-Mark Gurney 1608fca7a5SJohn-Mark Gurney * under sponsorship of the FreeBSD Foundation and 1708fca7a5SJohn-Mark Gurney * Rubicon Communications, LLC (Netgate). 18091d81d1SSam Leffler * 191833d604SJohn Baldwin * Portions of this software were developed by Ararat River 201833d604SJohn Baldwin * Consulting, LLC under sponsorship of the FreeBSD Foundation. 211833d604SJohn Baldwin * 22091d81d1SSam Leffler * Permission to use, copy, and modify this software with or without fee 23091d81d1SSam Leffler * is hereby granted, provided that this entire notice is included in 24091d81d1SSam Leffler * all source code copies of any software which is or includes a copy or 25091d81d1SSam Leffler * modification of this software. 26091d81d1SSam Leffler * 27091d81d1SSam Leffler * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28091d81d1SSam Leffler * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29091d81d1SSam Leffler * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30091d81d1SSam Leffler * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31091d81d1SSam Leffler * PURPOSE. 32091d81d1SSam Leffler */ 33091d81d1SSam Leffler 342c446514SDavid E. O'Brien #include <sys/cdefs.h> 352c446514SDavid E. O'Brien __FBSDID("$FreeBSD$"); 362c446514SDavid E. O'Brien 37091d81d1SSam Leffler #include <sys/param.h> 38091d81d1SSam Leffler #include <sys/systm.h> 39091d81d1SSam Leffler #include <sys/malloc.h> 40091d81d1SSam Leffler #include <sys/mbuf.h> 416810ad6fSSam Leffler #include <sys/module.h> 42091d81d1SSam Leffler #include <sys/sysctl.h> 43091d81d1SSam Leffler #include <sys/errno.h> 44091d81d1SSam Leffler #include <sys/random.h> 45091d81d1SSam Leffler #include <sys/kernel.h> 46091d81d1SSam Leffler #include <sys/uio.h> 4708fca7a5SJohn-Mark Gurney #include <sys/endian.h> 4808fca7a5SJohn-Mark Gurney #include <sys/limits.h> 49091d81d1SSam Leffler 50091d81d1SSam Leffler #include <crypto/sha1.h> 51091d81d1SSam Leffler #include <opencrypto/rmd160.h> 52091d81d1SSam Leffler 53091d81d1SSam Leffler #include <opencrypto/cryptodev.h> 54091d81d1SSam Leffler #include <opencrypto/xform.h> 55091d81d1SSam Leffler 566810ad6fSSam Leffler #include <sys/kobj.h> 576810ad6fSSam Leffler #include <sys/bus.h> 586810ad6fSSam Leffler #include "cryptodev_if.h" 59091d81d1SSam Leffler 60c0341432SJohn Baldwin struct swcr_auth { 61c0341432SJohn Baldwin void *sw_ictx; 62c0341432SJohn Baldwin void *sw_octx; 63d8787d4fSMark Johnston const struct auth_hash *sw_axf; 64c0341432SJohn Baldwin uint16_t sw_mlen; 656113a08bSJohn Baldwin bool sw_hmac; 66c0341432SJohn Baldwin }; 67c0341432SJohn Baldwin 68c0341432SJohn Baldwin struct swcr_encdec { 69f313909aSJohn Baldwin void *sw_ctx; 70d8787d4fSMark Johnston const struct enc_xform *sw_exf; 71c0341432SJohn Baldwin }; 72c0341432SJohn Baldwin 73c0341432SJohn Baldwin struct swcr_compdec { 74d8787d4fSMark Johnston const struct comp_algo *sw_cxf; 75c0341432SJohn Baldwin }; 76c0341432SJohn Baldwin 77c0341432SJohn Baldwin struct swcr_session { 78dc475c9bSJohn Baldwin int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79c0341432SJohn Baldwin 80c0341432SJohn Baldwin struct swcr_auth swcr_auth; 81c0341432SJohn Baldwin struct swcr_encdec swcr_encdec; 82c0341432SJohn Baldwin struct swcr_compdec swcr_compdec; 83c0341432SJohn Baldwin }; 84507281e5SSean Eric Fagan 856810ad6fSSam Leffler static int32_t swcr_id; 866810ad6fSSam Leffler 871b0909d5SConrad Meyer static void swcr_freesession(device_t dev, crypto_session_t cses); 88091d81d1SSam Leffler 89c0341432SJohn Baldwin /* Used for CRYPTO_NULL_CBC. */ 90c0341432SJohn Baldwin static int 91dc475c9bSJohn Baldwin swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92c0341432SJohn Baldwin { 93c0341432SJohn Baldwin 94c0341432SJohn Baldwin return (0); 95c0341432SJohn Baldwin } 96c0341432SJohn Baldwin 97091d81d1SSam Leffler /* 98091d81d1SSam Leffler * Apply a symmetric encryption/decryption algorithm. 99091d81d1SSam Leffler */ 100091d81d1SSam Leffler static int 101dc475c9bSJohn Baldwin swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102091d81d1SSam Leffler { 1032cd83828SJohn Baldwin unsigned char blk[EALG_MAX_BLOCK_LEN]; 104c0341432SJohn Baldwin const struct crypto_session_params *csp; 105d8787d4fSMark Johnston const struct enc_xform *exf; 106b54d1284SJohn Baldwin const struct swcr_encdec *sw; 107b54d1284SJohn Baldwin void *ctx; 10886be314dSJohn Baldwin size_t inlen, outlen; 1092cd83828SJohn Baldwin int blks, resid; 1109c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 11126d292d3SJohn Baldwin const unsigned char *inblk; 11226d292d3SJohn Baldwin unsigned char *outblk; 11308fca7a5SJohn-Mark Gurney int error; 114c0341432SJohn Baldwin bool encrypting; 11508fca7a5SJohn-Mark Gurney 11608fca7a5SJohn-Mark Gurney error = 0; 117091d81d1SSam Leffler 118c0341432SJohn Baldwin sw = &ses->swcr_encdec; 119091d81d1SSam Leffler exf = sw->sw_exf; 1201833d604SJohn Baldwin csp = crypto_get_params(crp->crp_session); 121091d81d1SSam Leffler 122723d8764SJohn Baldwin if (exf->native_blocksize == 0) { 123091d81d1SSam Leffler /* Check for non-padded data */ 124723d8764SJohn Baldwin if ((crp->crp_payload_length % exf->blocksize) != 0) 125723d8764SJohn Baldwin return (EINVAL); 126723d8764SJohn Baldwin 127723d8764SJohn Baldwin blks = exf->blocksize; 128723d8764SJohn Baldwin } else 129723d8764SJohn Baldwin blks = exf->native_blocksize; 130091d81d1SSam Leffler 131c0341432SJohn Baldwin if (exf == &enc_xform_aes_icm && 132c0341432SJohn Baldwin (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 13308fca7a5SJohn-Mark Gurney return (EINVAL); 13408fca7a5SJohn-Mark Gurney 135b54d1284SJohn Baldwin ctx = __builtin_alloca(exf->ctxsize); 136c0341432SJohn Baldwin if (crp->crp_cipher_key != NULL) { 137b54d1284SJohn Baldwin error = exf->setkey(ctx, crp->crp_cipher_key, 138b54d1284SJohn Baldwin csp->csp_cipher_klen); 139c740ae4bSPoul-Henning Kamp if (error) 140c740ae4bSPoul-Henning Kamp return (error); 141b54d1284SJohn Baldwin } else 142b54d1284SJohn Baldwin memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143d295bdeeSPawel Jakub Dawidek 1442cd83828SJohn Baldwin crypto_read_iv(crp, blk); 1452cd83828SJohn Baldwin exf->reinit(ctx, blk, csp->csp_ivlen); 146091d81d1SSam Leffler 1479c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 1489c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 14986be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 1509c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1519c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 1529c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1539c0e3d3aSJohn Baldwin } else 1549c0e3d3aSJohn Baldwin cc_out = cc_in; 15586be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 1569c0e3d3aSJohn Baldwin 1579c0e3d3aSJohn Baldwin resid = crp->crp_payload_length; 158c0341432SJohn Baldwin encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 159091d81d1SSam Leffler 160091d81d1SSam Leffler /* 1619c0e3d3aSJohn Baldwin * Loop through encrypting blocks. 'inlen' is the remaining 1629c0e3d3aSJohn Baldwin * length of the current segment in the input buffer. 1639c0e3d3aSJohn Baldwin * 'outlen' is the remaining length of current segment in the 1649c0e3d3aSJohn Baldwin * output buffer. 165091d81d1SSam Leffler */ 1669c0e3d3aSJohn Baldwin while (resid >= blks) { 1679c0e3d3aSJohn Baldwin /* 1689c0e3d3aSJohn Baldwin * If the current block is not contained within the 1699c0e3d3aSJohn Baldwin * current input/output segment, use 'blk' as a local 1709c0e3d3aSJohn Baldwin * buffer. 1719c0e3d3aSJohn Baldwin */ 1729c0e3d3aSJohn Baldwin if (inlen < blks) { 1739c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, blks, blk); 1749c0e3d3aSJohn Baldwin inblk = blk; 175d295bdeeSPawel Jakub Dawidek } 1769c0e3d3aSJohn Baldwin if (outlen < blks) 1779c0e3d3aSJohn Baldwin outblk = blk; 1789c0e3d3aSJohn Baldwin 1799c0e3d3aSJohn Baldwin if (encrypting) 180b54d1284SJohn Baldwin exf->encrypt(ctx, inblk, outblk); 1819c0e3d3aSJohn Baldwin else 182b54d1284SJohn Baldwin exf->decrypt(ctx, inblk, outblk); 183091d81d1SSam Leffler 1849c0e3d3aSJohn Baldwin if (inlen < blks) { 18586be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 1869c0e3d3aSJohn Baldwin } else { 1879c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, blks); 1889c0e3d3aSJohn Baldwin inlen -= blks; 1899c0e3d3aSJohn Baldwin inblk += blks; 19008fca7a5SJohn-Mark Gurney } 191091d81d1SSam Leffler 1929c0e3d3aSJohn Baldwin if (outlen < blks) { 1939c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, blks, blk); 19486be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 1959c0e3d3aSJohn Baldwin } else { 1969c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, blks); 1979c0e3d3aSJohn Baldwin outlen -= blks; 1989c0e3d3aSJohn Baldwin outblk += blks; 199091d81d1SSam Leffler } 200091d81d1SSam Leffler 2019c0e3d3aSJohn Baldwin resid -= blks; 202f34a967bSPawel Jakub Dawidek } 203f34a967bSPawel Jakub Dawidek 204723d8764SJohn Baldwin /* Handle trailing partial block for stream ciphers. */ 2059c0e3d3aSJohn Baldwin if (resid > 0) { 206723d8764SJohn Baldwin KASSERT(exf->native_blocksize != 0, 207723d8764SJohn Baldwin ("%s: partial block of %d bytes for cipher %s", 2082cd83828SJohn Baldwin __func__, resid, exf->name)); 2099c0e3d3aSJohn Baldwin KASSERT(resid < blks, ("%s: partial block too big", __func__)); 210723d8764SJohn Baldwin 21186be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 21286be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 2139c0e3d3aSJohn Baldwin if (inlen < resid) { 2149c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 2159c0e3d3aSJohn Baldwin inblk = blk; 21686be314dSJohn Baldwin } 2179c0e3d3aSJohn Baldwin if (outlen < resid) 2189c0e3d3aSJohn Baldwin outblk = blk; 2199c0e3d3aSJohn Baldwin if (encrypting) 220b54d1284SJohn Baldwin exf->encrypt_last(ctx, inblk, outblk, 2219c0e3d3aSJohn Baldwin resid); 2229c0e3d3aSJohn Baldwin else 223b54d1284SJohn Baldwin exf->decrypt_last(ctx, inblk, outblk, 2249c0e3d3aSJohn Baldwin resid); 2259c0e3d3aSJohn Baldwin if (outlen < resid) 2269c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 227723d8764SJohn Baldwin } 228723d8764SJohn Baldwin 229b54d1284SJohn Baldwin explicit_bzero(ctx, exf->ctxsize); 23020c128daSJohn Baldwin explicit_bzero(blk, sizeof(blk)); 2319c0e3d3aSJohn Baldwin return (0); 232091d81d1SSam Leffler } 233091d81d1SSam Leffler 234091d81d1SSam Leffler /* 235c0341432SJohn Baldwin * Compute or verify hash. 236091d81d1SSam Leffler */ 237091d81d1SSam Leffler static int 238dc475c9bSJohn Baldwin swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 239091d81d1SSam Leffler { 240c0341432SJohn Baldwin u_char aalg[HASH_MAX_LEN]; 241c0341432SJohn Baldwin const struct crypto_session_params *csp; 2426113a08bSJohn Baldwin const struct swcr_auth *sw; 243d8787d4fSMark Johnston const struct auth_hash *axf; 244091d81d1SSam Leffler union authctx ctx; 245091d81d1SSam Leffler int err; 246091d81d1SSam Leffler 247c0341432SJohn Baldwin sw = &ses->swcr_auth; 248091d81d1SSam Leffler 249091d81d1SSam Leffler axf = sw->sw_axf; 250091d81d1SSam Leffler 251c0341432SJohn Baldwin csp = crypto_get_params(crp->crp_session); 2526038018aSMarcin Wojtas if (crp->crp_auth_key != NULL) { 2536113a08bSJohn Baldwin if (sw->sw_hmac) { 2546113a08bSJohn Baldwin hmac_init_ipad(axf, crp->crp_auth_key, 2556113a08bSJohn Baldwin csp->csp_auth_klen, &ctx); 2566113a08bSJohn Baldwin } else { 2576113a08bSJohn Baldwin axf->Init(&ctx); 2586113a08bSJohn Baldwin axf->Setkey(&ctx, crp->crp_auth_key, 259c0341432SJohn Baldwin csp->csp_auth_klen); 26025b7033bSConrad Meyer } 2616113a08bSJohn Baldwin } else 2626113a08bSJohn Baldwin memcpy(&ctx, sw->sw_ictx, axf->ctxsize); 263091d81d1SSam Leffler 2649b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 2659b774dc0SJohn Baldwin err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 2669b774dc0SJohn Baldwin else 267c0341432SJohn Baldwin err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 2689b6b2f86SJohn Baldwin axf->Update, &ctx); 269091d81d1SSam Leffler if (err) 270e0b155feSJohn Baldwin goto out; 271091d81d1SSam Leffler 2729c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 2739c0e3d3aSJohn Baldwin CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 2749c0e3d3aSJohn Baldwin err = crypto_apply_buf(&crp->crp_obuf, 2759c0e3d3aSJohn Baldwin crp->crp_payload_output_start, crp->crp_payload_length, 2769b6b2f86SJohn Baldwin axf->Update, &ctx); 2779c0e3d3aSJohn Baldwin else 2789c0e3d3aSJohn Baldwin err = crypto_apply(crp, crp->crp_payload_start, 2799b6b2f86SJohn Baldwin crp->crp_payload_length, axf->Update, &ctx); 280c0341432SJohn Baldwin if (err) 281e0b155feSJohn Baldwin goto out; 282091d81d1SSam Leffler 2836038018aSMarcin Wojtas if (csp->csp_flags & CSP_F_ESN) 2846038018aSMarcin Wojtas axf->Update(&ctx, crp->crp_esn, 4); 2856038018aSMarcin Wojtas 286091d81d1SSam Leffler axf->Final(aalg, &ctx); 2876113a08bSJohn Baldwin if (sw->sw_hmac) { 2886113a08bSJohn Baldwin if (crp->crp_auth_key != NULL) 2896113a08bSJohn Baldwin hmac_init_opad(axf, crp->crp_auth_key, 2906113a08bSJohn Baldwin csp->csp_auth_klen, &ctx); 2916113a08bSJohn Baldwin else 2926113a08bSJohn Baldwin memcpy(&ctx, sw->sw_octx, axf->ctxsize); 293091d81d1SSam Leffler axf->Update(&ctx, aalg, axf->hashsize); 294091d81d1SSam Leffler axf->Final(aalg, &ctx); 295091d81d1SSam Leffler } 296091d81d1SSam Leffler 297c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 29820c128daSJohn Baldwin u_char uaalg[HASH_MAX_LEN]; 29920c128daSJohn Baldwin 300c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 301c0341432SJohn Baldwin if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 30220c128daSJohn Baldwin err = EBADMSG; 30320c128daSJohn Baldwin explicit_bzero(uaalg, sizeof(uaalg)); 304c0341432SJohn Baldwin } else { 305091d81d1SSam Leffler /* Inject the authentication data */ 306c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 307c0341432SJohn Baldwin } 30820c128daSJohn Baldwin explicit_bzero(aalg, sizeof(aalg)); 309e0b155feSJohn Baldwin out: 310e0b155feSJohn Baldwin explicit_bzero(&ctx, sizeof(ctx)); 31120c128daSJohn Baldwin return (err); 312091d81d1SSam Leffler } 313091d81d1SSam Leffler 31408fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 31508fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 31608fca7a5SJohn-Mark Gurney 31708fca7a5SJohn-Mark Gurney static int 318dc475c9bSJohn Baldwin swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 31908fca7a5SJohn-Mark Gurney { 32026d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 32108fca7a5SJohn-Mark Gurney u_char *blk = (u_char *)blkbuf; 32226d292d3SJohn Baldwin u_char tag[GMAC_DIGEST_LEN]; 3239c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc; 32426d292d3SJohn Baldwin const u_char *inblk; 32508fca7a5SJohn-Mark Gurney union authctx ctx; 3266113a08bSJohn Baldwin const struct swcr_auth *swa; 327d8787d4fSMark Johnston const struct auth_hash *axf; 32808fca7a5SJohn-Mark Gurney uint32_t *blkp; 32986be314dSJohn Baldwin size_t len; 33086be314dSJohn Baldwin int blksz, error, ivlen, resid; 33108fca7a5SJohn-Mark Gurney 332c0341432SJohn Baldwin swa = &ses->swcr_auth; 33308fca7a5SJohn-Mark Gurney axf = swa->sw_axf; 33426d292d3SJohn Baldwin blksz = GMAC_BLOCK_LEN; 33526d292d3SJohn Baldwin KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 33626d292d3SJohn Baldwin __func__)); 33708fca7a5SJohn-Mark Gurney 3386113a08bSJohn Baldwin if (crp->crp_auth_key != NULL) { 3396113a08bSJohn Baldwin axf->Init(&ctx); 3406113a08bSJohn Baldwin axf->Setkey(&ctx, crp->crp_auth_key, 3416113a08bSJohn Baldwin crypto_get_params(crp->crp_session)->csp_auth_klen); 3426113a08bSJohn Baldwin } else 3436113a08bSJohn Baldwin memcpy(&ctx, swa->sw_ictx, axf->ctxsize); 3446113a08bSJohn Baldwin 34508fca7a5SJohn-Mark Gurney /* Initialize the IV */ 346c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 3471c9b25a5SJohn Baldwin crypto_read_iv(crp, blk); 34808fca7a5SJohn-Mark Gurney 3491c9b25a5SJohn Baldwin axf->Reinit(&ctx, blk, ivlen); 3509c0e3d3aSJohn Baldwin crypto_cursor_init(&cc, &crp->crp_buf); 3519c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc, crp->crp_payload_start); 35226d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 35386be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc, &len); 35426d292d3SJohn Baldwin if (len >= blksz) { 35526d292d3SJohn Baldwin len = rounddown(MIN(len, resid), blksz); 35626d292d3SJohn Baldwin crypto_cursor_advance(&cc, len); 35726d292d3SJohn Baldwin } else { 35826d292d3SJohn Baldwin len = blksz; 3599c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc, len, blk); 36026d292d3SJohn Baldwin inblk = blk; 36126d292d3SJohn Baldwin } 36226d292d3SJohn Baldwin axf->Update(&ctx, inblk, len); 36326d292d3SJohn Baldwin } 36426d292d3SJohn Baldwin if (resid > 0) { 36526d292d3SJohn Baldwin memset(blk, 0, blksz); 36626d292d3SJohn Baldwin crypto_cursor_copydata(&cc, resid, blk); 36708fca7a5SJohn-Mark Gurney axf->Update(&ctx, blk, blksz); 36808fca7a5SJohn-Mark Gurney } 36908fca7a5SJohn-Mark Gurney 37008fca7a5SJohn-Mark Gurney /* length block */ 37126d292d3SJohn Baldwin memset(blk, 0, blksz); 37208fca7a5SJohn-Mark Gurney blkp = (uint32_t *)blk + 1; 373c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 37408fca7a5SJohn-Mark Gurney axf->Update(&ctx, blk, blksz); 375c0341432SJohn Baldwin 376c0341432SJohn Baldwin /* Finalize MAC */ 37726d292d3SJohn Baldwin axf->Final(tag, &ctx); 378c0341432SJohn Baldwin 37920c128daSJohn Baldwin error = 0; 380c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 38126d292d3SJohn Baldwin u_char tag2[GMAC_DIGEST_LEN]; 38220c128daSJohn Baldwin 383c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 38426d292d3SJohn Baldwin tag2); 38526d292d3SJohn Baldwin if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 38620c128daSJohn Baldwin error = EBADMSG; 38726d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 388c0341432SJohn Baldwin } else { 389c0341432SJohn Baldwin /* Inject the authentication data */ 39026d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 391c0341432SJohn Baldwin } 39220c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 39326d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 39420c128daSJohn Baldwin return (error); 395c0341432SJohn Baldwin } 396c0341432SJohn Baldwin 397c0341432SJohn Baldwin static int 398dc475c9bSJohn Baldwin swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 399c0341432SJohn Baldwin { 40026d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 401c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 40226d292d3SJohn Baldwin u_char tag[GMAC_DIGEST_LEN]; 4039c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 40426d292d3SJohn Baldwin const u_char *inblk; 40526d292d3SJohn Baldwin u_char *outblk; 406b54d1284SJohn Baldwin const struct swcr_auth *swa; 407b54d1284SJohn Baldwin const struct swcr_encdec *swe; 408d8787d4fSMark Johnston const struct enc_xform *exf; 409b54d1284SJohn Baldwin void *ctx; 410c0341432SJohn Baldwin uint32_t *blkp; 41186be314dSJohn Baldwin size_t len; 41286be314dSJohn Baldwin int blksz, error, ivlen, r, resid; 413c0341432SJohn Baldwin 414c0341432SJohn Baldwin swa = &ses->swcr_auth; 415c0341432SJohn Baldwin swe = &ses->swcr_encdec; 416c0341432SJohn Baldwin exf = swe->sw_exf; 417ab91fb6cSJohn Baldwin blksz = GMAC_BLOCK_LEN; 418ab91fb6cSJohn Baldwin KASSERT(blksz == exf->native_blocksize, 419723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 420c0341432SJohn Baldwin 421c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 422c0341432SJohn Baldwin return (EINVAL); 423c0341432SJohn Baldwin 424c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 425c0341432SJohn Baldwin 426b54d1284SJohn Baldwin ctx = __builtin_alloca(exf->ctxsize); 427ab91fb6cSJohn Baldwin if (crp->crp_cipher_key != NULL) 428b54d1284SJohn Baldwin exf->setkey(ctx, crp->crp_cipher_key, 429ab91fb6cSJohn Baldwin crypto_get_params(crp->crp_session)->csp_cipher_klen); 430b54d1284SJohn Baldwin else 431b54d1284SJohn Baldwin memcpy(ctx, swe->sw_ctx, exf->ctxsize); 432b54d1284SJohn Baldwin exf->reinit(ctx, crp->crp_iv, ivlen); 433c0341432SJohn Baldwin 434c0341432SJohn Baldwin /* Supply MAC with AAD */ 4359b774dc0SJohn Baldwin if (crp->crp_aad != NULL) { 4369b774dc0SJohn Baldwin len = rounddown(crp->crp_aad_length, blksz); 4379b774dc0SJohn Baldwin if (len != 0) 438b54d1284SJohn Baldwin exf->update(ctx, crp->crp_aad, len); 4399b774dc0SJohn Baldwin if (crp->crp_aad_length != len) { 4409b774dc0SJohn Baldwin memset(blk, 0, blksz); 4419b774dc0SJohn Baldwin memcpy(blk, (char *)crp->crp_aad + len, 4429b774dc0SJohn Baldwin crp->crp_aad_length - len); 443b54d1284SJohn Baldwin exf->update(ctx, blk, blksz); 4449b774dc0SJohn Baldwin } 4459b774dc0SJohn Baldwin } else { 4469c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 4479c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_aad_start); 4489b774dc0SJohn Baldwin for (resid = crp->crp_aad_length; resid >= blksz; 4499b774dc0SJohn Baldwin resid -= len) { 45086be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 45126d292d3SJohn Baldwin if (len >= blksz) { 45226d292d3SJohn Baldwin len = rounddown(MIN(len, resid), blksz); 45326d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, len); 45426d292d3SJohn Baldwin } else { 45526d292d3SJohn Baldwin len = blksz; 4569c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 45726d292d3SJohn Baldwin inblk = blk; 45826d292d3SJohn Baldwin } 459b54d1284SJohn Baldwin exf->update(ctx, inblk, len); 46026d292d3SJohn Baldwin } 46126d292d3SJohn Baldwin if (resid > 0) { 46226d292d3SJohn Baldwin memset(blk, 0, blksz); 46326d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 464b54d1284SJohn Baldwin exf->update(ctx, blk, blksz); 465c0341432SJohn Baldwin } 4669b774dc0SJohn Baldwin } 467c0341432SJohn Baldwin 468c0341432SJohn Baldwin /* Do encryption with MAC */ 4699c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 4709c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 4719c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 4729c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 4739c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 4749c0e3d3aSJohn Baldwin } else 4759c0e3d3aSJohn Baldwin cc_out = cc_in; 47626d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 47786be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 47886be314dSJohn Baldwin if (len < blksz) { 47926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 48026d292d3SJohn Baldwin inblk = blk; 481c0341432SJohn Baldwin } else { 48226d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 483c0341432SJohn Baldwin } 48426d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 48586be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 48686be314dSJohn Baldwin if (len < blksz) 48726d292d3SJohn Baldwin outblk = blk; 488b54d1284SJohn Baldwin exf->encrypt(ctx, inblk, outblk); 489b54d1284SJohn Baldwin exf->update(ctx, outblk, blksz); 49026d292d3SJohn Baldwin if (outblk == blk) 49126d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 49226d292d3SJohn Baldwin else 49326d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 49426d292d3SJohn Baldwin } else { 495b54d1284SJohn Baldwin exf->update(ctx, inblk, blksz); 49626d292d3SJohn Baldwin } 49726d292d3SJohn Baldwin } 49826d292d3SJohn Baldwin if (resid > 0) { 49926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 50026d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 501b54d1284SJohn Baldwin exf->encrypt_last(ctx, blk, blk, resid); 50226d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 50326d292d3SJohn Baldwin } 504b54d1284SJohn Baldwin exf->update(ctx, blk, resid); 505c0341432SJohn Baldwin } 506c0341432SJohn Baldwin 507c0341432SJohn Baldwin /* length block */ 50826d292d3SJohn Baldwin memset(blk, 0, blksz); 509c0341432SJohn Baldwin blkp = (uint32_t *)blk + 1; 510c0341432SJohn Baldwin *blkp = htobe32(crp->crp_aad_length * 8); 511c0341432SJohn Baldwin blkp = (uint32_t *)blk + 3; 512c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 513b54d1284SJohn Baldwin exf->update(ctx, blk, blksz); 514c0341432SJohn Baldwin 515c0341432SJohn Baldwin /* Finalize MAC */ 516b54d1284SJohn Baldwin exf->final(tag, ctx); 517c0341432SJohn Baldwin 518c0341432SJohn Baldwin /* Validate tag */ 51920c128daSJohn Baldwin error = 0; 520c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 52126d292d3SJohn Baldwin u_char tag2[GMAC_DIGEST_LEN]; 52220c128daSJohn Baldwin 52326d292d3SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 524c0341432SJohn Baldwin 52526d292d3SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 52626d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 52720c128daSJohn Baldwin if (r != 0) { 52820c128daSJohn Baldwin error = EBADMSG; 52920c128daSJohn Baldwin goto out; 53020c128daSJohn Baldwin } 531c0341432SJohn Baldwin 532c0341432SJohn Baldwin /* tag matches, decrypt data */ 5339c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5349c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 53526d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 53626d292d3SJohn Baldwin resid -= blksz) { 53786be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 53886be314dSJohn Baldwin if (len < blksz) { 53926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 54026d292d3SJohn Baldwin inblk = blk; 54186be314dSJohn Baldwin } else 54226d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 54386be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 54486be314dSJohn Baldwin if (len < blksz) 54526d292d3SJohn Baldwin outblk = blk; 546b54d1284SJohn Baldwin exf->decrypt(ctx, inblk, outblk); 54726d292d3SJohn Baldwin if (outblk == blk) 54826d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 54926d292d3SJohn Baldwin else 55026d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 55126d292d3SJohn Baldwin } 55226d292d3SJohn Baldwin if (resid > 0) { 55326d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 554b54d1284SJohn Baldwin exf->decrypt_last(ctx, blk, blk, resid); 55526d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 556c0341432SJohn Baldwin } 557c0341432SJohn Baldwin } else { 558c0341432SJohn Baldwin /* Inject the authentication data */ 55926d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 560c0341432SJohn Baldwin } 561c0341432SJohn Baldwin 56220c128daSJohn Baldwin out: 563b54d1284SJohn Baldwin explicit_bzero(ctx, exf->ctxsize); 56420c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 56526d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 56620c128daSJohn Baldwin 56720c128daSJohn Baldwin return (error); 568c0341432SJohn Baldwin } 569c0341432SJohn Baldwin 5704361c4ebSJohn Baldwin static void 5714361c4ebSJohn Baldwin build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 5724361c4ebSJohn Baldwin u_int data_length, u_int tag_length, uint8_t *b0) 5734361c4ebSJohn Baldwin { 5744361c4ebSJohn Baldwin uint8_t *bp; 5754361c4ebSJohn Baldwin uint8_t flags, L; 5764361c4ebSJohn Baldwin 5774361c4ebSJohn Baldwin KASSERT(nonce_length >= 7 && nonce_length <= 13, 5784361c4ebSJohn Baldwin ("nonce_length must be between 7 and 13 bytes")); 5794361c4ebSJohn Baldwin 5804361c4ebSJohn Baldwin /* 5814361c4ebSJohn Baldwin * Need to determine the L field value. This is the number of 5824361c4ebSJohn Baldwin * bytes needed to specify the length of the message; the length 5834361c4ebSJohn Baldwin * is whatever is left in the 16 bytes after specifying flags and 5844361c4ebSJohn Baldwin * the nonce. 5854361c4ebSJohn Baldwin */ 5864361c4ebSJohn Baldwin L = 15 - nonce_length; 5874361c4ebSJohn Baldwin 5884361c4ebSJohn Baldwin flags = ((aad_length > 0) << 6) + 5894361c4ebSJohn Baldwin (((tag_length - 2) / 2) << 3) + 5904361c4ebSJohn Baldwin L - 1; 5914361c4ebSJohn Baldwin 5924361c4ebSJohn Baldwin /* 5934361c4ebSJohn Baldwin * Now we need to set up the first block, which has flags, nonce, 5944361c4ebSJohn Baldwin * and the message length. 5954361c4ebSJohn Baldwin */ 5964361c4ebSJohn Baldwin b0[0] = flags; 5974361c4ebSJohn Baldwin memcpy(b0 + 1, nonce, nonce_length); 5984361c4ebSJohn Baldwin bp = b0 + 1 + nonce_length; 5994361c4ebSJohn Baldwin 6004361c4ebSJohn Baldwin /* Need to copy L' [aka L-1] bytes of data_length */ 6014361c4ebSJohn Baldwin for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 6024361c4ebSJohn Baldwin *dst = data_length; 6034361c4ebSJohn Baldwin data_length >>= 8; 6044361c4ebSJohn Baldwin } 6054361c4ebSJohn Baldwin } 6064361c4ebSJohn Baldwin 6074361c4ebSJohn Baldwin /* NB: OCF only supports AAD lengths < 2^32. */ 6084361c4ebSJohn Baldwin static int 6094361c4ebSJohn Baldwin build_ccm_aad_length(u_int aad_length, uint8_t *blk) 6104361c4ebSJohn Baldwin { 6114361c4ebSJohn Baldwin if (aad_length < ((1 << 16) - (1 << 8))) { 6124361c4ebSJohn Baldwin be16enc(blk, aad_length); 6134361c4ebSJohn Baldwin return (sizeof(uint16_t)); 6144361c4ebSJohn Baldwin } else { 6154361c4ebSJohn Baldwin blk[0] = 0xff; 6164361c4ebSJohn Baldwin blk[1] = 0xfe; 6174361c4ebSJohn Baldwin be32enc(blk + 2, aad_length); 6184361c4ebSJohn Baldwin return (2 + sizeof(uint32_t)); 6194361c4ebSJohn Baldwin } 6204361c4ebSJohn Baldwin } 6214361c4ebSJohn Baldwin 622c0341432SJohn Baldwin static int 623dc475c9bSJohn Baldwin swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 624c0341432SJohn Baldwin { 6254361c4ebSJohn Baldwin u_char blk[CCM_CBC_BLOCK_LEN]; 6264361c4ebSJohn Baldwin u_char tag[AES_CBC_MAC_HASH_LEN]; 627c0341432SJohn Baldwin union authctx ctx; 628ae18720dSJohn Baldwin const struct crypto_session_params *csp; 6296113a08bSJohn Baldwin const struct swcr_auth *swa; 630d8787d4fSMark Johnston const struct auth_hash *axf; 6314361c4ebSJohn Baldwin int error, ivlen, len; 632c0341432SJohn Baldwin 633ae18720dSJohn Baldwin csp = crypto_get_params(crp->crp_session); 634c0341432SJohn Baldwin swa = &ses->swcr_auth; 635c0341432SJohn Baldwin axf = swa->sw_axf; 636c0341432SJohn Baldwin 6376113a08bSJohn Baldwin if (crp->crp_auth_key != NULL) { 6386113a08bSJohn Baldwin axf->Init(&ctx); 6396113a08bSJohn Baldwin axf->Setkey(&ctx, crp->crp_auth_key, csp->csp_auth_klen); 6406113a08bSJohn Baldwin } else 6416113a08bSJohn Baldwin memcpy(&ctx, swa->sw_ictx, axf->ctxsize); 642c0341432SJohn Baldwin 643c0341432SJohn Baldwin /* Initialize the IV */ 644ae18720dSJohn Baldwin ivlen = csp->csp_ivlen; 645c0341432SJohn Baldwin 6464361c4ebSJohn Baldwin /* Supply MAC with IV */ 6474361c4ebSJohn Baldwin axf->Reinit(&ctx, crp->crp_iv, ivlen); 648c0341432SJohn Baldwin 6494361c4ebSJohn Baldwin /* Supply MAC with b0. */ 6504361c4ebSJohn Baldwin build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 6514361c4ebSJohn Baldwin swa->sw_mlen, blk); 6524361c4ebSJohn Baldwin axf->Update(&ctx, blk, CCM_CBC_BLOCK_LEN); 6534361c4ebSJohn Baldwin 6544361c4ebSJohn Baldwin len = build_ccm_aad_length(crp->crp_payload_length, blk); 6554361c4ebSJohn Baldwin axf->Update(&ctx, blk, len); 6564361c4ebSJohn Baldwin 6574361c4ebSJohn Baldwin crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 6584361c4ebSJohn Baldwin axf->Update, &ctx); 659c0341432SJohn Baldwin 660c0341432SJohn Baldwin /* Finalize MAC */ 66126d292d3SJohn Baldwin axf->Final(tag, &ctx); 662c0341432SJohn Baldwin 6634361c4ebSJohn Baldwin error = 0; 664c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 66526d292d3SJohn Baldwin u_char tag2[AES_CBC_MAC_HASH_LEN]; 66620c128daSJohn Baldwin 667c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 66826d292d3SJohn Baldwin tag2); 66926d292d3SJohn Baldwin if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 67020c128daSJohn Baldwin error = EBADMSG; 67126d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag)); 672c0341432SJohn Baldwin } else { 673c0341432SJohn Baldwin /* Inject the authentication data */ 67426d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 675c0341432SJohn Baldwin } 67626d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 6774361c4ebSJohn Baldwin explicit_bzero(blk, sizeof(blk)); 67820c128daSJohn Baldwin return (error); 679c0341432SJohn Baldwin } 680c0341432SJohn Baldwin 681c0341432SJohn Baldwin static int 682dc475c9bSJohn Baldwin swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 683c0341432SJohn Baldwin { 684ae18720dSJohn Baldwin const struct crypto_session_params *csp; 68526d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 686c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 68726d292d3SJohn Baldwin u_char tag[AES_CBC_MAC_HASH_LEN]; 6889c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 68926d292d3SJohn Baldwin const u_char *inblk; 69026d292d3SJohn Baldwin u_char *outblk; 691b54d1284SJohn Baldwin const struct swcr_auth *swa; 692b54d1284SJohn Baldwin const struct swcr_encdec *swe; 693d8787d4fSMark Johnston const struct enc_xform *exf; 694b54d1284SJohn Baldwin void *ctx; 69586be314dSJohn Baldwin size_t len; 69626d292d3SJohn Baldwin int blksz, error, ivlen, r, resid; 697c0341432SJohn Baldwin 698ae18720dSJohn Baldwin csp = crypto_get_params(crp->crp_session); 699c0341432SJohn Baldwin swa = &ses->swcr_auth; 700c0341432SJohn Baldwin swe = &ses->swcr_encdec; 701c0341432SJohn Baldwin exf = swe->sw_exf; 702ab91fb6cSJohn Baldwin blksz = AES_BLOCK_LEN; 703ab91fb6cSJohn Baldwin KASSERT(blksz == exf->native_blocksize, 704723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 705c0341432SJohn Baldwin 706ae18720dSJohn Baldwin if (crp->crp_payload_length > ccm_max_payload_length(csp)) 707ae18720dSJohn Baldwin return (EMSGSIZE); 708ae18720dSJohn Baldwin 709c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 710c0341432SJohn Baldwin return (EINVAL); 711c0341432SJohn Baldwin 712ae18720dSJohn Baldwin ivlen = csp->csp_ivlen; 713c0341432SJohn Baldwin 714b54d1284SJohn Baldwin ctx = __builtin_alloca(exf->ctxsize); 715ab91fb6cSJohn Baldwin if (crp->crp_cipher_key != NULL) 716b54d1284SJohn Baldwin exf->setkey(ctx, crp->crp_cipher_key, 717ab91fb6cSJohn Baldwin crypto_get_params(crp->crp_session)->csp_cipher_klen); 718b54d1284SJohn Baldwin else 719b54d1284SJohn Baldwin memcpy(ctx, swe->sw_ctx, exf->ctxsize); 720b54d1284SJohn Baldwin exf->reinit(ctx, crp->crp_iv, ivlen); 721c0341432SJohn Baldwin 7224361c4ebSJohn Baldwin /* Supply MAC with b0. */ 7234361c4ebSJohn Baldwin _Static_assert(sizeof(blkbuf) >= CCM_CBC_BLOCK_LEN, 7244361c4ebSJohn Baldwin "blkbuf too small for b0"); 7254361c4ebSJohn Baldwin build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 7264361c4ebSJohn Baldwin crp->crp_payload_length, swa->sw_mlen, blk); 727b54d1284SJohn Baldwin exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 7284361c4ebSJohn Baldwin 729c0341432SJohn Baldwin /* Supply MAC with AAD */ 7304361c4ebSJohn Baldwin if (crp->crp_aad_length != 0) { 7314361c4ebSJohn Baldwin len = build_ccm_aad_length(crp->crp_aad_length, blk); 732b54d1284SJohn Baldwin exf->update(ctx, blk, len); 7339b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 734b54d1284SJohn Baldwin exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 7359b774dc0SJohn Baldwin else 7364361c4ebSJohn Baldwin crypto_apply(crp, crp->crp_aad_start, 737b54d1284SJohn Baldwin crp->crp_aad_length, exf->update, ctx); 7384361c4ebSJohn Baldwin 7394361c4ebSJohn Baldwin /* Pad the AAD (including length field) to a full block. */ 7404361c4ebSJohn Baldwin len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 7414361c4ebSJohn Baldwin if (len != 0) { 7424361c4ebSJohn Baldwin len = CCM_CBC_BLOCK_LEN - len; 7434361c4ebSJohn Baldwin memset(blk, 0, CCM_CBC_BLOCK_LEN); 744b54d1284SJohn Baldwin exf->update(ctx, blk, len); 7454361c4ebSJohn Baldwin } 7464361c4ebSJohn Baldwin } 747c0341432SJohn Baldwin 748c0341432SJohn Baldwin /* Do encryption/decryption with MAC */ 7499c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 7509c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 7519c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 7529c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 7539c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 7549c0e3d3aSJohn Baldwin } else 7559c0e3d3aSJohn Baldwin cc_out = cc_in; 75626d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 75786be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 75886be314dSJohn Baldwin if (len < blksz) { 75926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 76026d292d3SJohn Baldwin inblk = blk; 76186be314dSJohn Baldwin } else 76226d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 763c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 76486be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 76586be314dSJohn Baldwin if (len < blksz) 76626d292d3SJohn Baldwin outblk = blk; 767b54d1284SJohn Baldwin exf->update(ctx, inblk, blksz); 768b54d1284SJohn Baldwin exf->encrypt(ctx, inblk, outblk); 76926d292d3SJohn Baldwin if (outblk == blk) 77026d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 77126d292d3SJohn Baldwin else 77226d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 773c0341432SJohn Baldwin } else { 774c0341432SJohn Baldwin /* 775c0341432SJohn Baldwin * One of the problems with CCM+CBC is that 776c0341432SJohn Baldwin * the authentication is done on the 77726d292d3SJohn Baldwin * unencrypted data. As a result, we have to 778c0341432SJohn Baldwin * decrypt the data twice: once to generate 779c0341432SJohn Baldwin * the tag and a second time after the tag is 780c0341432SJohn Baldwin * verified. 781c0341432SJohn Baldwin */ 782b54d1284SJohn Baldwin exf->decrypt(ctx, inblk, blk); 783b54d1284SJohn Baldwin exf->update(ctx, blk, blksz); 78426d292d3SJohn Baldwin } 78526d292d3SJohn Baldwin } 78626d292d3SJohn Baldwin if (resid > 0) { 78726d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 78826d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 789b54d1284SJohn Baldwin exf->update(ctx, blk, resid); 790b54d1284SJohn Baldwin exf->encrypt_last(ctx, blk, blk, resid); 79126d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 79226d292d3SJohn Baldwin } else { 793b54d1284SJohn Baldwin exf->decrypt_last(ctx, blk, blk, resid); 794b54d1284SJohn Baldwin exf->update(ctx, blk, resid); 795c0341432SJohn Baldwin } 79608fca7a5SJohn-Mark Gurney } 79708fca7a5SJohn-Mark Gurney 79808fca7a5SJohn-Mark Gurney /* Finalize MAC */ 799b54d1284SJohn Baldwin exf->final(tag, ctx); 80008fca7a5SJohn-Mark Gurney 80108fca7a5SJohn-Mark Gurney /* Validate tag */ 80220c128daSJohn Baldwin error = 0; 803c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 80426d292d3SJohn Baldwin u_char tag2[AES_CBC_MAC_HASH_LEN]; 80520c128daSJohn Baldwin 806c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 80726d292d3SJohn Baldwin tag2); 80808fca7a5SJohn-Mark Gurney 80926d292d3SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 81026d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 81120c128daSJohn Baldwin if (r != 0) { 81220c128daSJohn Baldwin error = EBADMSG; 81320c128daSJohn Baldwin goto out; 81420c128daSJohn Baldwin } 815c0341432SJohn Baldwin 81608fca7a5SJohn-Mark Gurney /* tag matches, decrypt data */ 817b54d1284SJohn Baldwin exf->reinit(ctx, crp->crp_iv, ivlen); 8189c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 8199c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 82026d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 82126d292d3SJohn Baldwin resid -= blksz) { 82286be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 82386be314dSJohn Baldwin if (len < blksz) { 82426d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 82526d292d3SJohn Baldwin inblk = blk; 82686be314dSJohn Baldwin } else 82726d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 82886be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 82986be314dSJohn Baldwin if (len < blksz) 83026d292d3SJohn Baldwin outblk = blk; 831b54d1284SJohn Baldwin exf->decrypt(ctx, inblk, outblk); 83226d292d3SJohn Baldwin if (outblk == blk) 83326d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 83426d292d3SJohn Baldwin else 83526d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 83626d292d3SJohn Baldwin } 83726d292d3SJohn Baldwin if (resid > 0) { 83826d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 839b54d1284SJohn Baldwin exf->decrypt_last(ctx, blk, blk, resid); 84026d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 84108fca7a5SJohn-Mark Gurney } 84208fca7a5SJohn-Mark Gurney } else { 84308fca7a5SJohn-Mark Gurney /* Inject the authentication data */ 84426d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 84508fca7a5SJohn-Mark Gurney } 84608fca7a5SJohn-Mark Gurney 84720c128daSJohn Baldwin out: 848b54d1284SJohn Baldwin explicit_bzero(ctx, exf->ctxsize); 84920c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 85026d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 85120c128daSJohn Baldwin return (error); 85208fca7a5SJohn-Mark Gurney } 85308fca7a5SJohn-Mark Gurney 854dd2e1352SJohn Baldwin static int 855dc475c9bSJohn Baldwin swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 856dd2e1352SJohn Baldwin { 857dd2e1352SJohn Baldwin const struct crypto_session_params *csp; 858dd2e1352SJohn Baldwin uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 859dd2e1352SJohn Baldwin u_char *blk = (u_char *)blkbuf; 860dd2e1352SJohn Baldwin u_char tag[POLY1305_HASH_LEN]; 861dd2e1352SJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 862dd2e1352SJohn Baldwin const u_char *inblk; 863dd2e1352SJohn Baldwin u_char *outblk; 864dd2e1352SJohn Baldwin uint64_t *blkp; 865b54d1284SJohn Baldwin const struct swcr_auth *swa; 866b54d1284SJohn Baldwin const struct swcr_encdec *swe; 867d8787d4fSMark Johnston const struct enc_xform *exf; 868b54d1284SJohn Baldwin void *ctx; 86986be314dSJohn Baldwin size_t len; 870dd2e1352SJohn Baldwin int blksz, error, r, resid; 871dd2e1352SJohn Baldwin 872dd2e1352SJohn Baldwin swa = &ses->swcr_auth; 873dd2e1352SJohn Baldwin swe = &ses->swcr_encdec; 874dd2e1352SJohn Baldwin exf = swe->sw_exf; 875dd2e1352SJohn Baldwin blksz = exf->native_blocksize; 876dd2e1352SJohn Baldwin KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); 877dd2e1352SJohn Baldwin 878dd2e1352SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 879dd2e1352SJohn Baldwin return (EINVAL); 880dd2e1352SJohn Baldwin 881dd2e1352SJohn Baldwin csp = crypto_get_params(crp->crp_session); 882dd2e1352SJohn Baldwin 883b54d1284SJohn Baldwin ctx = __builtin_alloca(exf->ctxsize); 884dd2e1352SJohn Baldwin if (crp->crp_cipher_key != NULL) 885b54d1284SJohn Baldwin exf->setkey(ctx, crp->crp_cipher_key, 886dd2e1352SJohn Baldwin csp->csp_cipher_klen); 887b54d1284SJohn Baldwin else 888b54d1284SJohn Baldwin memcpy(ctx, swe->sw_ctx, exf->ctxsize); 889b54d1284SJohn Baldwin exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 890dd2e1352SJohn Baldwin 891ab91fb6cSJohn Baldwin /* Supply MAC with AAD */ 892ab91fb6cSJohn Baldwin if (crp->crp_aad != NULL) 893b54d1284SJohn Baldwin exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 894ab91fb6cSJohn Baldwin else 895b54d1284SJohn Baldwin crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 896b54d1284SJohn Baldwin exf->update, ctx); 897ab91fb6cSJohn Baldwin if (crp->crp_aad_length % 16 != 0) { 898ab91fb6cSJohn Baldwin /* padding1 */ 899ab91fb6cSJohn Baldwin memset(blk, 0, 16); 900b54d1284SJohn Baldwin exf->update(ctx, blk, 16 - crp->crp_aad_length % 16); 901ab91fb6cSJohn Baldwin } 902ab91fb6cSJohn Baldwin 903dd2e1352SJohn Baldwin /* Do encryption with MAC */ 904dd2e1352SJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 905dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 906dd2e1352SJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 907dd2e1352SJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 908dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 909dd2e1352SJohn Baldwin } else 910dd2e1352SJohn Baldwin cc_out = cc_in; 911dd2e1352SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 91286be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 91386be314dSJohn Baldwin if (len < blksz) { 914dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 915dd2e1352SJohn Baldwin inblk = blk; 91686be314dSJohn Baldwin } else 917dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 918dd2e1352SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 91986be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 92086be314dSJohn Baldwin if (len < blksz) 921dd2e1352SJohn Baldwin outblk = blk; 922b54d1284SJohn Baldwin exf->encrypt(ctx, inblk, outblk); 923b54d1284SJohn Baldwin exf->update(ctx, outblk, blksz); 924dd2e1352SJohn Baldwin if (outblk == blk) 925dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 926dd2e1352SJohn Baldwin else 927dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 928dd2e1352SJohn Baldwin } else { 929b54d1284SJohn Baldwin exf->update(ctx, inblk, blksz); 930dd2e1352SJohn Baldwin } 931dd2e1352SJohn Baldwin } 932dd2e1352SJohn Baldwin if (resid > 0) { 933dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 934dd2e1352SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 935b54d1284SJohn Baldwin exf->encrypt_last(ctx, blk, blk, resid); 936dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 937dd2e1352SJohn Baldwin } 938b54d1284SJohn Baldwin exf->update(ctx, blk, resid); 939dd2e1352SJohn Baldwin if (resid % 16 != 0) { 940dd2e1352SJohn Baldwin /* padding2 */ 941dd2e1352SJohn Baldwin memset(blk, 0, 16); 942b54d1284SJohn Baldwin exf->update(ctx, blk, 16 - resid % 16); 943dd2e1352SJohn Baldwin } 944dd2e1352SJohn Baldwin } 945dd2e1352SJohn Baldwin 946dd2e1352SJohn Baldwin /* lengths */ 947dd2e1352SJohn Baldwin blkp = (uint64_t *)blk; 948dd2e1352SJohn Baldwin blkp[0] = htole64(crp->crp_aad_length); 949dd2e1352SJohn Baldwin blkp[1] = htole64(crp->crp_payload_length); 950b54d1284SJohn Baldwin exf->update(ctx, blk, sizeof(uint64_t) * 2); 951dd2e1352SJohn Baldwin 952dd2e1352SJohn Baldwin /* Finalize MAC */ 953b54d1284SJohn Baldwin exf->final(tag, ctx); 954dd2e1352SJohn Baldwin 955dd2e1352SJohn Baldwin /* Validate tag */ 956dd2e1352SJohn Baldwin error = 0; 957dd2e1352SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 958dd2e1352SJohn Baldwin u_char tag2[POLY1305_HASH_LEN]; 959dd2e1352SJohn Baldwin 960dd2e1352SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 961dd2e1352SJohn Baldwin 962dd2e1352SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 963dd2e1352SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 964dd2e1352SJohn Baldwin if (r != 0) { 965dd2e1352SJohn Baldwin error = EBADMSG; 966dd2e1352SJohn Baldwin goto out; 967dd2e1352SJohn Baldwin } 968dd2e1352SJohn Baldwin 969dd2e1352SJohn Baldwin /* tag matches, decrypt data */ 970dd2e1352SJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 971dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 972dd2e1352SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 973dd2e1352SJohn Baldwin resid -= blksz) { 97486be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 97586be314dSJohn Baldwin if (len < blksz) { 976dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 977dd2e1352SJohn Baldwin inblk = blk; 97886be314dSJohn Baldwin } else 979dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 98086be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 98186be314dSJohn Baldwin if (len < blksz) 982dd2e1352SJohn Baldwin outblk = blk; 983b54d1284SJohn Baldwin exf->decrypt(ctx, inblk, outblk); 984dd2e1352SJohn Baldwin if (outblk == blk) 985dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 986dd2e1352SJohn Baldwin else 987dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 988dd2e1352SJohn Baldwin } 989dd2e1352SJohn Baldwin if (resid > 0) { 990dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 991b54d1284SJohn Baldwin exf->decrypt_last(ctx, blk, blk, resid); 992dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 993dd2e1352SJohn Baldwin } 994dd2e1352SJohn Baldwin } else { 995dd2e1352SJohn Baldwin /* Inject the authentication data */ 996dd2e1352SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 997dd2e1352SJohn Baldwin } 998dd2e1352SJohn Baldwin 999dd2e1352SJohn Baldwin out: 1000b54d1284SJohn Baldwin explicit_bzero(ctx, exf->ctxsize); 1001dd2e1352SJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 1002dd2e1352SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 1003dd2e1352SJohn Baldwin return (error); 1004dd2e1352SJohn Baldwin } 1005dd2e1352SJohn Baldwin 1006091d81d1SSam Leffler /* 1007c0341432SJohn Baldwin * Apply a cipher and a digest to perform EtA. 1008c0341432SJohn Baldwin */ 1009c0341432SJohn Baldwin static int 1010dc475c9bSJohn Baldwin swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1011c0341432SJohn Baldwin { 1012c0341432SJohn Baldwin int error; 1013c0341432SJohn Baldwin 1014c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1015c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 1016c0341432SJohn Baldwin if (error == 0) 1017c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 1018c0341432SJohn Baldwin } else { 1019c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 1020c0341432SJohn Baldwin if (error == 0) 1021c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 1022c0341432SJohn Baldwin } 1023c0341432SJohn Baldwin return (error); 1024c0341432SJohn Baldwin } 1025c0341432SJohn Baldwin 1026c0341432SJohn Baldwin /* 1027091d81d1SSam Leffler * Apply a compression/decompression algorithm 1028091d81d1SSam Leffler */ 1029091d81d1SSam Leffler static int 1030dc475c9bSJohn Baldwin swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1031091d81d1SSam Leffler { 1032d8787d4fSMark Johnston const struct comp_algo *cxf; 1033d3d79e96SJohn Baldwin uint8_t *data, *out; 1034091d81d1SSam Leffler int adj; 1035d3d79e96SJohn Baldwin uint32_t result; 1036091d81d1SSam Leffler 1037c0341432SJohn Baldwin cxf = ses->swcr_compdec.sw_cxf; 1038091d81d1SSam Leffler 1039091d81d1SSam Leffler /* We must handle the whole buffer of data in one time 1040091d81d1SSam Leffler * then if there is not all the data in the mbuf, we must 1041091d81d1SSam Leffler * copy in a buffer. 1042091d81d1SSam Leffler */ 1043091d81d1SSam Leffler 1044c0341432SJohn Baldwin data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1045091d81d1SSam Leffler if (data == NULL) 1046091d81d1SSam Leffler return (EINVAL); 1047c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1048c0341432SJohn Baldwin data); 1049091d81d1SSam Leffler 1050c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1051c0341432SJohn Baldwin result = cxf->compress(data, crp->crp_payload_length, &out); 1052091d81d1SSam Leffler else 1053c0341432SJohn Baldwin result = cxf->decompress(data, crp->crp_payload_length, &out); 1054091d81d1SSam Leffler 10551ede983cSDag-Erling Smørgrav free(data, M_CRYPTO_DATA); 1056091d81d1SSam Leffler if (result == 0) 1057c0341432SJohn Baldwin return (EINVAL); 1058c0341432SJohn Baldwin crp->crp_olen = result; 1059c0341432SJohn Baldwin 1060c0341432SJohn Baldwin /* Check the compressed size when doing compression */ 1061c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1062c0341432SJohn Baldwin if (result >= crp->crp_payload_length) { 1063c0341432SJohn Baldwin /* Compression was useless, we lost time */ 1064c0341432SJohn Baldwin free(out, M_CRYPTO_DATA); 1065c0341432SJohn Baldwin return (0); 1066c0341432SJohn Baldwin } 1067c0341432SJohn Baldwin } 1068091d81d1SSam Leffler 1069091d81d1SSam Leffler /* Copy back the (de)compressed data. m_copyback is 1070091d81d1SSam Leffler * extending the mbuf as necessary. 1071091d81d1SSam Leffler */ 1072c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_payload_start, result, out); 1073c0341432SJohn Baldwin if (result < crp->crp_payload_length) { 10749c0e3d3aSJohn Baldwin switch (crp->crp_buf.cb_type) { 1075c0341432SJohn Baldwin case CRYPTO_BUF_MBUF: 1076883a0196SJohn Baldwin case CRYPTO_BUF_SINGLE_MBUF: 1077c0341432SJohn Baldwin adj = result - crp->crp_payload_length; 10789c0e3d3aSJohn Baldwin m_adj(crp->crp_buf.cb_mbuf, adj); 1079c0341432SJohn Baldwin break; 1080c0341432SJohn Baldwin case CRYPTO_BUF_UIO: { 10819c0e3d3aSJohn Baldwin struct uio *uio = crp->crp_buf.cb_uio; 1082091d81d1SSam Leffler int ind; 1083091d81d1SSam Leffler 1084c0341432SJohn Baldwin adj = crp->crp_payload_length - result; 1085091d81d1SSam Leffler ind = uio->uio_iovcnt - 1; 1086091d81d1SSam Leffler 1087091d81d1SSam Leffler while (adj > 0 && ind >= 0) { 1088091d81d1SSam Leffler if (adj < uio->uio_iov[ind].iov_len) { 1089091d81d1SSam Leffler uio->uio_iov[ind].iov_len -= adj; 1090091d81d1SSam Leffler break; 1091091d81d1SSam Leffler } 1092091d81d1SSam Leffler 1093091d81d1SSam Leffler adj -= uio->uio_iov[ind].iov_len; 1094091d81d1SSam Leffler uio->uio_iov[ind].iov_len = 0; 1095091d81d1SSam Leffler ind--; 1096091d81d1SSam Leffler uio->uio_iovcnt--; 1097091d81d1SSam Leffler } 1098091d81d1SSam Leffler } 1099c0341432SJohn Baldwin break; 1100e6f6d0c9SAlan Somers case CRYPTO_BUF_VMPAGE: 1101e6f6d0c9SAlan Somers adj = crp->crp_payload_length - result; 1102e6f6d0c9SAlan Somers crp->crp_buf.cb_vm_page_len -= adj; 1103e6f6d0c9SAlan Somers break; 11049c0e3d3aSJohn Baldwin default: 11059c0e3d3aSJohn Baldwin break; 1106c0341432SJohn Baldwin } 1107091d81d1SSam Leffler } 11081ede983cSDag-Erling Smørgrav free(out, M_CRYPTO_DATA); 1109091d81d1SSam Leffler return 0; 1110091d81d1SSam Leffler } 1111091d81d1SSam Leffler 1112091d81d1SSam Leffler static int 11133e947048SJohn Baldwin swcr_setup_cipher(struct swcr_session *ses, 1114c0341432SJohn Baldwin const struct crypto_session_params *csp) 1115091d81d1SSam Leffler { 1116c0341432SJohn Baldwin struct swcr_encdec *swe; 1117d8787d4fSMark Johnston const struct enc_xform *txf; 1118f6c4bc3bSPawel Jakub Dawidek int error; 1119091d81d1SSam Leffler 1120c0341432SJohn Baldwin swe = &ses->swcr_encdec; 1121c0341432SJohn Baldwin txf = crypto_cipher(csp); 1122b54d1284SJohn Baldwin if (csp->csp_cipher_key != NULL) { 11233e947048SJohn Baldwin if (txf->ctxsize != 0) { 1124f313909aSJohn Baldwin swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 11253e947048SJohn Baldwin M_NOWAIT); 1126f313909aSJohn Baldwin if (swe->sw_ctx == NULL) 11273e947048SJohn Baldwin return (ENOMEM); 11283e947048SJohn Baldwin } 1129f313909aSJohn Baldwin error = txf->setkey(swe->sw_ctx, 1130c0341432SJohn Baldwin csp->csp_cipher_key, csp->csp_cipher_klen); 1131c0341432SJohn Baldwin if (error) 1132c0341432SJohn Baldwin return (error); 1133091d81d1SSam Leffler } 1134c0341432SJohn Baldwin swe->sw_exf = txf; 1135c0341432SJohn Baldwin return (0); 1136f6c4bc3bSPawel Jakub Dawidek } 1137091d81d1SSam Leffler 1138c0341432SJohn Baldwin static int 1139c0341432SJohn Baldwin swcr_setup_auth(struct swcr_session *ses, 1140c0341432SJohn Baldwin const struct crypto_session_params *csp) 1141c0341432SJohn Baldwin { 1142c0341432SJohn Baldwin struct swcr_auth *swa; 1143d8787d4fSMark Johnston const struct auth_hash *axf; 1144c0341432SJohn Baldwin 1145c0341432SJohn Baldwin swa = &ses->swcr_auth; 1146c0341432SJohn Baldwin 1147c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1148c0341432SJohn Baldwin swa->sw_axf = axf; 1149c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1150c0341432SJohn Baldwin return (EINVAL); 1151c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1152c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1153c0341432SJohn Baldwin else 1154c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 11556113a08bSJohn Baldwin if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 11566113a08bSJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 11576113a08bSJohn Baldwin M_NOWAIT); 1158c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1159c0341432SJohn Baldwin return (ENOBUFS); 11606113a08bSJohn Baldwin } 1161c0341432SJohn Baldwin 1162c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1163091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1164c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1165f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1166f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1167f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1168091d81d1SSam Leffler case CRYPTO_RIPEMD160_HMAC: 11696113a08bSJohn Baldwin swa->sw_hmac = true; 11706113a08bSJohn Baldwin if (csp->csp_auth_key != NULL) { 11713a0b6a93SJohn Baldwin swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1172091d81d1SSam Leffler M_NOWAIT); 1173c0341432SJohn Baldwin if (swa->sw_octx == NULL) 1174c0341432SJohn Baldwin return (ENOBUFS); 11756113a08bSJohn Baldwin hmac_init_ipad(axf, csp->csp_auth_key, 11766113a08bSJohn Baldwin csp->csp_auth_klen, swa->sw_ictx); 11776113a08bSJohn Baldwin hmac_init_opad(axf, csp->csp_auth_key, 11786113a08bSJohn Baldwin csp->csp_auth_klen, swa->sw_octx); 1179091d81d1SSam Leffler } 1180091d81d1SSam Leffler break; 1181091d81d1SSam Leffler case CRYPTO_SHA1: 1182c4729f6eSConrad Meyer case CRYPTO_SHA2_224: 1183c4729f6eSConrad Meyer case CRYPTO_SHA2_256: 1184c4729f6eSConrad Meyer case CRYPTO_SHA2_384: 1185c4729f6eSConrad Meyer case CRYPTO_SHA2_512: 1186aa268175SJohn Baldwin case CRYPTO_NULL_HMAC: 1187c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1188c0341432SJohn Baldwin break; 1189c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 11906113a08bSJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 11916113a08bSJohn Baldwin case CRYPTO_POLY1305: 11926113a08bSJohn Baldwin if (csp->csp_auth_key != NULL) { 1193c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1194c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1195c0341432SJohn Baldwin csp->csp_auth_klen); 11966113a08bSJohn Baldwin } 1197c0341432SJohn Baldwin break; 1198c0341432SJohn Baldwin case CRYPTO_BLAKE2B: 1199c0341432SJohn Baldwin case CRYPTO_BLAKE2S: 1200c0341432SJohn Baldwin /* 1201c0341432SJohn Baldwin * Blake2b and Blake2s support an optional key but do 1202c0341432SJohn Baldwin * not require one. 1203c0341432SJohn Baldwin */ 12046113a08bSJohn Baldwin if (csp->csp_auth_klen == 0) 12056113a08bSJohn Baldwin axf->Init(swa->sw_ictx); 12066113a08bSJohn Baldwin else if (csp->csp_auth_key != NULL) 1207c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1208c0341432SJohn Baldwin csp->csp_auth_klen); 12096113a08bSJohn Baldwin break; 12106113a08bSJohn Baldwin } 12116113a08bSJohn Baldwin 12126113a08bSJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) { 12136113a08bSJohn Baldwin switch (csp->csp_auth_alg) { 12146113a08bSJohn Baldwin case CRYPTO_AES_NIST_GMAC: 12156113a08bSJohn Baldwin ses->swcr_process = swcr_gmac; 1216c0341432SJohn Baldwin break; 1217c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1218c0341432SJohn Baldwin ses->swcr_process = swcr_ccm_cbc_mac; 1219c0341432SJohn Baldwin break; 12206113a08bSJohn Baldwin default: 12216113a08bSJohn Baldwin ses->swcr_process = swcr_authcompute; 12226113a08bSJohn Baldwin } 1223091d81d1SSam Leffler } 1224091d81d1SSam Leffler 1225c0341432SJohn Baldwin return (0); 1226c0341432SJohn Baldwin } 122708fca7a5SJohn-Mark Gurney 1228c0341432SJohn Baldwin static int 1229ab91fb6cSJohn Baldwin swcr_setup_aead(struct swcr_session *ses, 1230c0341432SJohn Baldwin const struct crypto_session_params *csp) 1231c0341432SJohn Baldwin { 1232c0341432SJohn Baldwin struct swcr_auth *swa; 1233ab91fb6cSJohn Baldwin int error; 1234c0341432SJohn Baldwin 1235ab91fb6cSJohn Baldwin error = swcr_setup_cipher(ses, csp); 1236ab91fb6cSJohn Baldwin if (error) 1237ab91fb6cSJohn Baldwin return (error); 1238ab91fb6cSJohn Baldwin 1239c0341432SJohn Baldwin swa = &ses->swcr_auth; 1240c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1241ab91fb6cSJohn Baldwin swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1242c0341432SJohn Baldwin else 1243c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1244ab91fb6cSJohn Baldwin return (0); 1245dd2e1352SJohn Baldwin } 1246dd2e1352SJohn Baldwin 1247c0341432SJohn Baldwin static bool 1248c0341432SJohn Baldwin swcr_auth_supported(const struct crypto_session_params *csp) 1249109919c6SBenno Rice { 1250d8787d4fSMark Johnston const struct auth_hash *axf; 1251091d81d1SSam Leffler 1252c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1253c0341432SJohn Baldwin if (axf == NULL) 1254c0341432SJohn Baldwin return (false); 1255c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1256091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1257c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1258f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1259f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1260f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1261091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 1262c0341432SJohn Baldwin case CRYPTO_RIPEMD160_HMAC: 1263091d81d1SSam Leffler break; 1264c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1265c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1266c0341432SJohn Baldwin case 128: 1267c0341432SJohn Baldwin case 192: 1268c0341432SJohn Baldwin case 256: 1269c0341432SJohn Baldwin break; 1270c0341432SJohn Baldwin default: 1271c0341432SJohn Baldwin return (false); 1272c0341432SJohn Baldwin } 1273c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1274c0341432SJohn Baldwin return (false); 1275c0341432SJohn Baldwin if (csp->csp_ivlen != AES_GCM_IV_LEN) 1276c0341432SJohn Baldwin return (false); 1277c0341432SJohn Baldwin break; 127825b7033bSConrad Meyer case CRYPTO_POLY1305: 1279c0341432SJohn Baldwin if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1280c0341432SJohn Baldwin return (false); 1281c0341432SJohn Baldwin break; 1282c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1283c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1284c0341432SJohn Baldwin case 128: 1285c0341432SJohn Baldwin case 192: 1286c0341432SJohn Baldwin case 256: 1287c0341432SJohn Baldwin break; 1288c0341432SJohn Baldwin default: 1289c0341432SJohn Baldwin return (false); 1290c0341432SJohn Baldwin } 1291c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1292c0341432SJohn Baldwin return (false); 1293c0341432SJohn Baldwin break; 1294c0341432SJohn Baldwin } 1295c0341432SJohn Baldwin return (true); 1296c0341432SJohn Baldwin } 1297091d81d1SSam Leffler 1298c0341432SJohn Baldwin static bool 1299c0341432SJohn Baldwin swcr_cipher_supported(const struct crypto_session_params *csp) 1300c0341432SJohn Baldwin { 1301d8787d4fSMark Johnston const struct enc_xform *txf; 1302c0341432SJohn Baldwin 1303c0341432SJohn Baldwin txf = crypto_cipher(csp); 1304c0341432SJohn Baldwin if (txf == NULL) 1305c0341432SJohn Baldwin return (false); 1306c0341432SJohn Baldwin if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1307c0341432SJohn Baldwin txf->ivsize != csp->csp_ivlen) 1308c0341432SJohn Baldwin return (false); 1309c0341432SJohn Baldwin return (true); 1310c0341432SJohn Baldwin } 1311c0341432SJohn Baldwin 13126038018aSMarcin Wojtas #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 13136038018aSMarcin Wojtas 1314c0341432SJohn Baldwin static int 1315c0341432SJohn Baldwin swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1316c0341432SJohn Baldwin { 13176038018aSMarcin Wojtas if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1318c0341432SJohn Baldwin return (EINVAL); 1319c0341432SJohn Baldwin switch (csp->csp_mode) { 1320c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1321c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1322c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1323c0341432SJohn Baldwin break; 1324c0341432SJohn Baldwin default: 1325c0341432SJohn Baldwin return (EINVAL); 13265fbc5b5aSConrad Meyer } 1327091d81d1SSam Leffler break; 1328c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1329c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1330c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1331c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1332dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1333c0341432SJohn Baldwin return (EINVAL); 1334c0341432SJohn Baldwin default: 1335c0341432SJohn Baldwin if (!swcr_cipher_supported(csp)) 1336c0341432SJohn Baldwin return (EINVAL); 1337091d81d1SSam Leffler break; 1338091d81d1SSam Leffler } 1339c0341432SJohn Baldwin break; 1340c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1341c0341432SJohn Baldwin if (!swcr_auth_supported(csp)) 1342c0341432SJohn Baldwin return (EINVAL); 1343c0341432SJohn Baldwin break; 1344c0341432SJohn Baldwin case CSP_MODE_AEAD: 1345c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1346c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1347c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1348c172a407SJohn Baldwin switch (csp->csp_cipher_klen * 8) { 1349c172a407SJohn Baldwin case 128: 1350c172a407SJohn Baldwin case 192: 1351c172a407SJohn Baldwin case 256: 1352c172a407SJohn Baldwin break; 1353c172a407SJohn Baldwin default: 1354c172a407SJohn Baldwin return (EINVAL); 1355c172a407SJohn Baldwin } 1356c172a407SJohn Baldwin break; 1357dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1358c0341432SJohn Baldwin break; 1359c0341432SJohn Baldwin default: 1360c0341432SJohn Baldwin return (EINVAL); 1361c0341432SJohn Baldwin } 1362c0341432SJohn Baldwin break; 1363c0341432SJohn Baldwin case CSP_MODE_ETA: 1364c0341432SJohn Baldwin /* AEAD algorithms cannot be used for EtA. */ 1365c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1366c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1367c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1368dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1369c0341432SJohn Baldwin return (EINVAL); 1370c0341432SJohn Baldwin } 1371c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1372c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1373c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1374c0341432SJohn Baldwin return (EINVAL); 1375c0341432SJohn Baldwin } 1376c0341432SJohn Baldwin 1377c0341432SJohn Baldwin if (!swcr_cipher_supported(csp) || 1378c0341432SJohn Baldwin !swcr_auth_supported(csp)) 1379c0341432SJohn Baldwin return (EINVAL); 1380c0341432SJohn Baldwin break; 1381c0341432SJohn Baldwin default: 1382c0341432SJohn Baldwin return (EINVAL); 1383c0341432SJohn Baldwin } 1384c0341432SJohn Baldwin 1385c0341432SJohn Baldwin return (CRYPTODEV_PROBE_SOFTWARE); 1386c0341432SJohn Baldwin } 1387c0341432SJohn Baldwin 1388c0341432SJohn Baldwin /* 1389c0341432SJohn Baldwin * Generate a new software session. 1390c0341432SJohn Baldwin */ 1391c0341432SJohn Baldwin static int 1392c0341432SJohn Baldwin swcr_newsession(device_t dev, crypto_session_t cses, 1393c0341432SJohn Baldwin const struct crypto_session_params *csp) 1394c0341432SJohn Baldwin { 1395c0341432SJohn Baldwin struct swcr_session *ses; 1396d8787d4fSMark Johnston const struct comp_algo *cxf; 1397c0341432SJohn Baldwin int error; 1398c0341432SJohn Baldwin 1399c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1400c0341432SJohn Baldwin 1401c0341432SJohn Baldwin error = 0; 1402c0341432SJohn Baldwin switch (csp->csp_mode) { 1403c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1404c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1405c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1406c0341432SJohn Baldwin cxf = &comp_algo_deflate; 1407c0341432SJohn Baldwin break; 1408c0341432SJohn Baldwin #ifdef INVARIANTS 1409c0341432SJohn Baldwin default: 1410c0341432SJohn Baldwin panic("bad compression algo"); 1411c0341432SJohn Baldwin #endif 1412c0341432SJohn Baldwin } 1413c0341432SJohn Baldwin ses->swcr_compdec.sw_cxf = cxf; 1414c0341432SJohn Baldwin ses->swcr_process = swcr_compdec; 1415c0341432SJohn Baldwin break; 1416c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1417c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1418c0341432SJohn Baldwin case CRYPTO_NULL_CBC: 1419c0341432SJohn Baldwin ses->swcr_process = swcr_null; 1420c0341432SJohn Baldwin break; 1421c0341432SJohn Baldwin #ifdef INVARIANTS 1422c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1423c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1424dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1425c0341432SJohn Baldwin panic("bad cipher algo"); 1426c0341432SJohn Baldwin #endif 1427c0341432SJohn Baldwin default: 14283e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1429c0341432SJohn Baldwin if (error == 0) 1430c0341432SJohn Baldwin ses->swcr_process = swcr_encdec; 1431c0341432SJohn Baldwin } 1432c0341432SJohn Baldwin break; 1433c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1434c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1435c0341432SJohn Baldwin break; 1436c0341432SJohn Baldwin case CSP_MODE_AEAD: 1437c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1438c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1439ab91fb6cSJohn Baldwin error = swcr_setup_aead(ses, csp); 1440c0341432SJohn Baldwin if (error == 0) 1441c0341432SJohn Baldwin ses->swcr_process = swcr_gcm; 1442c0341432SJohn Baldwin break; 1443c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1444ab91fb6cSJohn Baldwin error = swcr_setup_aead(ses, csp); 1445c0341432SJohn Baldwin if (error == 0) 1446c0341432SJohn Baldwin ses->swcr_process = swcr_ccm; 1447c0341432SJohn Baldwin break; 1448dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1449ab91fb6cSJohn Baldwin error = swcr_setup_aead(ses, csp); 1450dd2e1352SJohn Baldwin if (error == 0) 1451dd2e1352SJohn Baldwin ses->swcr_process = swcr_chacha20_poly1305; 1452dd2e1352SJohn Baldwin break; 1453c0341432SJohn Baldwin #ifdef INVARIANTS 1454c0341432SJohn Baldwin default: 1455c0341432SJohn Baldwin panic("bad aead algo"); 1456c0341432SJohn Baldwin #endif 1457c0341432SJohn Baldwin } 1458c0341432SJohn Baldwin break; 1459c0341432SJohn Baldwin case CSP_MODE_ETA: 1460c0341432SJohn Baldwin #ifdef INVARIANTS 1461c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1462c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1463c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1464dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1465c0341432SJohn Baldwin panic("bad eta cipher algo"); 1466c0341432SJohn Baldwin } 1467c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1468c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1469c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1470c0341432SJohn Baldwin panic("bad eta auth algo"); 1471c0341432SJohn Baldwin } 1472c0341432SJohn Baldwin #endif 1473c0341432SJohn Baldwin 1474c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1475c0341432SJohn Baldwin if (error) 1476c0341432SJohn Baldwin break; 1477c0341432SJohn Baldwin if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1478c0341432SJohn Baldwin /* Effectively degrade to digest mode. */ 1479c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1480c0341432SJohn Baldwin break; 1481c0341432SJohn Baldwin } 1482c0341432SJohn Baldwin 14833e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1484c0341432SJohn Baldwin if (error == 0) 1485c0341432SJohn Baldwin ses->swcr_process = swcr_eta; 1486c0341432SJohn Baldwin break; 1487c0341432SJohn Baldwin default: 1488c0341432SJohn Baldwin error = EINVAL; 1489c0341432SJohn Baldwin } 1490c0341432SJohn Baldwin 1491c0341432SJohn Baldwin if (error) 1492c0341432SJohn Baldwin swcr_freesession(dev, cses); 1493c0341432SJohn Baldwin return (error); 1494c0341432SJohn Baldwin } 1495c0341432SJohn Baldwin 1496c0341432SJohn Baldwin static void 1497c0341432SJohn Baldwin swcr_freesession(device_t dev, crypto_session_t cses) 1498c0341432SJohn Baldwin { 1499c0341432SJohn Baldwin struct swcr_session *ses; 1500c0341432SJohn Baldwin 1501c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1502c0341432SJohn Baldwin 1503f313909aSJohn Baldwin zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 15044a711b8dSJohn Baldwin zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 15054a711b8dSJohn Baldwin zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1506091d81d1SSam Leffler } 1507091d81d1SSam Leffler 1508091d81d1SSam Leffler /* 1509091d81d1SSam Leffler * Process a software request. 1510091d81d1SSam Leffler */ 1511091d81d1SSam Leffler static int 15126810ad6fSSam Leffler swcr_process(device_t dev, struct cryptop *crp, int hint) 1513091d81d1SSam Leffler { 1514c0341432SJohn Baldwin struct swcr_session *ses; 1515091d81d1SSam Leffler 15161b0909d5SConrad Meyer ses = crypto_get_driver_session(crp->crp_session); 1517091d81d1SSam Leffler 1518c0341432SJohn Baldwin crp->crp_etype = ses->swcr_process(ses, crp); 1519091d81d1SSam Leffler 1520091d81d1SSam Leffler crypto_done(crp); 1521c0341432SJohn Baldwin return (0); 1522091d81d1SSam Leffler } 1523091d81d1SSam Leffler 1524091d81d1SSam Leffler static void 15253f147ab2SWarner Losh swcr_identify(driver_t *drv, device_t parent) 1526091d81d1SSam Leffler { 15276810ad6fSSam Leffler /* NB: order 10 is so we get attached after h/w devices */ 15286810ad6fSSam Leffler if (device_find_child(parent, "cryptosoft", -1) == NULL && 152986c585d9SMarius Strobl BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 15306810ad6fSSam Leffler panic("cryptosoft: could not attach"); 15316810ad6fSSam Leffler } 1532f6c4bc3bSPawel Jakub Dawidek 15336810ad6fSSam Leffler static int 15346810ad6fSSam Leffler swcr_probe(device_t dev) 15356810ad6fSSam Leffler { 15366810ad6fSSam Leffler device_set_desc(dev, "software crypto"); 15374fc60fa9SMark Johnston device_quiet(dev); 153886c585d9SMarius Strobl return (BUS_PROBE_NOWILDCARD); 15396810ad6fSSam Leffler } 1540f6c4bc3bSPawel Jakub Dawidek 15416810ad6fSSam Leffler static int 15426810ad6fSSam Leffler swcr_attach(device_t dev) 15436810ad6fSSam Leffler { 15446810ad6fSSam Leffler 15459ebbebe4SConrad Meyer swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 15466810ad6fSSam Leffler CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 15476810ad6fSSam Leffler if (swcr_id < 0) { 15486810ad6fSSam Leffler device_printf(dev, "cannot initialize!"); 1549c0341432SJohn Baldwin return (ENXIO); 15506810ad6fSSam Leffler } 15516810ad6fSSam Leffler 1552c0341432SJohn Baldwin return (0); 1553091d81d1SSam Leffler } 15544b465da2SPawel Jakub Dawidek 15553f147ab2SWarner Losh static int 15566810ad6fSSam Leffler swcr_detach(device_t dev) 15574b465da2SPawel Jakub Dawidek { 15586810ad6fSSam Leffler crypto_unregister_all(swcr_id); 15593f147ab2SWarner Losh return 0; 15604b465da2SPawel Jakub Dawidek } 15616810ad6fSSam Leffler 15626810ad6fSSam Leffler static device_method_t swcr_methods[] = { 15636810ad6fSSam Leffler DEVMETHOD(device_identify, swcr_identify), 15646810ad6fSSam Leffler DEVMETHOD(device_probe, swcr_probe), 15656810ad6fSSam Leffler DEVMETHOD(device_attach, swcr_attach), 15666810ad6fSSam Leffler DEVMETHOD(device_detach, swcr_detach), 15676810ad6fSSam Leffler 1568c0341432SJohn Baldwin DEVMETHOD(cryptodev_probesession, swcr_probesession), 15696810ad6fSSam Leffler DEVMETHOD(cryptodev_newsession, swcr_newsession), 15706810ad6fSSam Leffler DEVMETHOD(cryptodev_freesession,swcr_freesession), 15716810ad6fSSam Leffler DEVMETHOD(cryptodev_process, swcr_process), 15726810ad6fSSam Leffler 15736810ad6fSSam Leffler {0, 0}, 15746810ad6fSSam Leffler }; 15756810ad6fSSam Leffler 15766810ad6fSSam Leffler static driver_t swcr_driver = { 15776810ad6fSSam Leffler "cryptosoft", 15786810ad6fSSam Leffler swcr_methods, 15796810ad6fSSam Leffler 0, /* NB: no softc */ 15806810ad6fSSam Leffler }; 15816810ad6fSSam Leffler static devclass_t swcr_devclass; 15826810ad6fSSam Leffler 15836810ad6fSSam Leffler /* 15846810ad6fSSam Leffler * NB: We explicitly reference the crypto module so we 15856810ad6fSSam Leffler * get the necessary ordering when built as a loadable 15866810ad6fSSam Leffler * module. This is required because we bundle the crypto 15876810ad6fSSam Leffler * module code together with the cryptosoft driver (otherwise 15886810ad6fSSam Leffler * normal module dependencies would handle things). 15896810ad6fSSam Leffler */ 15906810ad6fSSam Leffler extern int crypto_modevent(struct module *, int, void *); 15916810ad6fSSam Leffler /* XXX where to attach */ 15926810ad6fSSam Leffler DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 15936810ad6fSSam Leffler MODULE_VERSION(cryptosoft, 1); 15946810ad6fSSam Leffler MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1595