1091d81d1SSam Leffler /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2091d81d1SSam Leffler 360727d8bSWarner Losh /*- 4091d81d1SSam Leffler * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 56810ad6fSSam Leffler * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6091d81d1SSam Leffler * 7091d81d1SSam Leffler * This code was written by Angelos D. Keromytis in Athens, Greece, in 8091d81d1SSam Leffler * February 2000. Network Security Technologies Inc. (NSTI) kindly 9091d81d1SSam Leffler * supported the development of this code. 10091d81d1SSam Leffler * 11091d81d1SSam Leffler * Copyright (c) 2000, 2001 Angelos D. Keromytis 121833d604SJohn Baldwin * Copyright (c) 2014-2021 The FreeBSD Foundation 1308fca7a5SJohn-Mark Gurney * All rights reserved. 1408fca7a5SJohn-Mark Gurney * 1508fca7a5SJohn-Mark Gurney * Portions of this software were developed by John-Mark Gurney 1608fca7a5SJohn-Mark Gurney * under sponsorship of the FreeBSD Foundation and 1708fca7a5SJohn-Mark Gurney * Rubicon Communications, LLC (Netgate). 18091d81d1SSam Leffler * 191833d604SJohn Baldwin * Portions of this software were developed by Ararat River 201833d604SJohn Baldwin * Consulting, LLC under sponsorship of the FreeBSD Foundation. 211833d604SJohn Baldwin * 22091d81d1SSam Leffler * Permission to use, copy, and modify this software with or without fee 23091d81d1SSam Leffler * is hereby granted, provided that this entire notice is included in 24091d81d1SSam Leffler * all source code copies of any software which is or includes a copy or 25091d81d1SSam Leffler * modification of this software. 26091d81d1SSam Leffler * 27091d81d1SSam Leffler * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28091d81d1SSam Leffler * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29091d81d1SSam Leffler * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30091d81d1SSam Leffler * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31091d81d1SSam Leffler * PURPOSE. 32091d81d1SSam Leffler */ 33091d81d1SSam Leffler 342c446514SDavid E. O'Brien #include <sys/cdefs.h> 352c446514SDavid E. O'Brien __FBSDID("$FreeBSD$"); 362c446514SDavid E. O'Brien 37091d81d1SSam Leffler #include <sys/param.h> 38091d81d1SSam Leffler #include <sys/systm.h> 39091d81d1SSam Leffler #include <sys/malloc.h> 40091d81d1SSam Leffler #include <sys/mbuf.h> 416810ad6fSSam Leffler #include <sys/module.h> 42091d81d1SSam Leffler #include <sys/sysctl.h> 43091d81d1SSam Leffler #include <sys/errno.h> 44091d81d1SSam Leffler #include <sys/random.h> 45091d81d1SSam Leffler #include <sys/kernel.h> 46091d81d1SSam Leffler #include <sys/uio.h> 4708fca7a5SJohn-Mark Gurney #include <sys/endian.h> 4808fca7a5SJohn-Mark Gurney #include <sys/limits.h> 49091d81d1SSam Leffler 50091d81d1SSam Leffler #include <crypto/sha1.h> 51091d81d1SSam Leffler #include <opencrypto/rmd160.h> 52091d81d1SSam Leffler 53091d81d1SSam Leffler #include <opencrypto/cryptodev.h> 54091d81d1SSam Leffler #include <opencrypto/xform.h> 55091d81d1SSam Leffler 566810ad6fSSam Leffler #include <sys/kobj.h> 576810ad6fSSam Leffler #include <sys/bus.h> 586810ad6fSSam Leffler #include "cryptodev_if.h" 59091d81d1SSam Leffler 60c0341432SJohn Baldwin struct swcr_auth { 61c0341432SJohn Baldwin void *sw_ictx; 62c0341432SJohn Baldwin void *sw_octx; 63d8787d4fSMark Johnston const struct auth_hash *sw_axf; 64c0341432SJohn Baldwin uint16_t sw_mlen; 656113a08bSJohn Baldwin bool sw_hmac; 66c0341432SJohn Baldwin }; 67c0341432SJohn Baldwin 68c0341432SJohn Baldwin struct swcr_encdec { 69f313909aSJohn Baldwin void *sw_ctx; 70d8787d4fSMark Johnston const struct enc_xform *sw_exf; 71c0341432SJohn Baldwin }; 72c0341432SJohn Baldwin 73c0341432SJohn Baldwin struct swcr_compdec { 74d8787d4fSMark Johnston const struct comp_algo *sw_cxf; 75c0341432SJohn Baldwin }; 76c0341432SJohn Baldwin 77c0341432SJohn Baldwin struct swcr_session { 78dc475c9bSJohn Baldwin int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79c0341432SJohn Baldwin 80c0341432SJohn Baldwin struct swcr_auth swcr_auth; 81c0341432SJohn Baldwin struct swcr_encdec swcr_encdec; 82c0341432SJohn Baldwin struct swcr_compdec swcr_compdec; 83c0341432SJohn Baldwin }; 84507281e5SSean Eric Fagan 856810ad6fSSam Leffler static int32_t swcr_id; 866810ad6fSSam Leffler 871b0909d5SConrad Meyer static void swcr_freesession(device_t dev, crypto_session_t cses); 88091d81d1SSam Leffler 89c0341432SJohn Baldwin /* Used for CRYPTO_NULL_CBC. */ 90c0341432SJohn Baldwin static int 91dc475c9bSJohn Baldwin swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92c0341432SJohn Baldwin { 93c0341432SJohn Baldwin 94c0341432SJohn Baldwin return (0); 95c0341432SJohn Baldwin } 96c0341432SJohn Baldwin 97091d81d1SSam Leffler /* 98091d81d1SSam Leffler * Apply a symmetric encryption/decryption algorithm. 99091d81d1SSam Leffler */ 100091d81d1SSam Leffler static int 101dc475c9bSJohn Baldwin swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102091d81d1SSam Leffler { 1032cd83828SJohn Baldwin unsigned char blk[EALG_MAX_BLOCK_LEN]; 104c0341432SJohn Baldwin const struct crypto_session_params *csp; 105d8787d4fSMark Johnston const struct enc_xform *exf; 106b54d1284SJohn Baldwin const struct swcr_encdec *sw; 107b54d1284SJohn Baldwin void *ctx; 108cfb7b942SJohn Baldwin size_t inlen, outlen, todo; 10969e533c3SJohn Baldwin int blksz, resid; 1109c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 11126d292d3SJohn Baldwin const unsigned char *inblk; 11226d292d3SJohn Baldwin unsigned char *outblk; 11308fca7a5SJohn-Mark Gurney int error; 114c0341432SJohn Baldwin bool encrypting; 11508fca7a5SJohn-Mark Gurney 11608fca7a5SJohn-Mark Gurney error = 0; 117091d81d1SSam Leffler 118c0341432SJohn Baldwin sw = &ses->swcr_encdec; 119091d81d1SSam Leffler exf = sw->sw_exf; 1201833d604SJohn Baldwin csp = crypto_get_params(crp->crp_session); 121091d81d1SSam Leffler 122723d8764SJohn Baldwin if (exf->native_blocksize == 0) { 123091d81d1SSam Leffler /* Check for non-padded data */ 124723d8764SJohn Baldwin if ((crp->crp_payload_length % exf->blocksize) != 0) 125723d8764SJohn Baldwin return (EINVAL); 126723d8764SJohn Baldwin 12769e533c3SJohn Baldwin blksz = exf->blocksize; 128723d8764SJohn Baldwin } else 12969e533c3SJohn Baldwin blksz = exf->native_blocksize; 130091d81d1SSam Leffler 131c0341432SJohn Baldwin if (exf == &enc_xform_aes_icm && 132c0341432SJohn Baldwin (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 13308fca7a5SJohn-Mark Gurney return (EINVAL); 13408fca7a5SJohn-Mark Gurney 135b54d1284SJohn Baldwin ctx = __builtin_alloca(exf->ctxsize); 136c0341432SJohn Baldwin if (crp->crp_cipher_key != NULL) { 137b54d1284SJohn Baldwin error = exf->setkey(ctx, crp->crp_cipher_key, 138b54d1284SJohn Baldwin csp->csp_cipher_klen); 139c740ae4bSPoul-Henning Kamp if (error) 140c740ae4bSPoul-Henning Kamp return (error); 141b54d1284SJohn Baldwin } else 142b54d1284SJohn Baldwin memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143d295bdeeSPawel Jakub Dawidek 1442cd83828SJohn Baldwin crypto_read_iv(crp, blk); 1452cd83828SJohn Baldwin exf->reinit(ctx, blk, csp->csp_ivlen); 146091d81d1SSam Leffler 1479c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 1489c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 14986be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 1509c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1519c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 1529c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1539c0e3d3aSJohn Baldwin } else 1549c0e3d3aSJohn Baldwin cc_out = cc_in; 15586be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 1569c0e3d3aSJohn Baldwin 157c0341432SJohn Baldwin encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 158091d81d1SSam Leffler 159091d81d1SSam Leffler /* 1609c0e3d3aSJohn Baldwin * Loop through encrypting blocks. 'inlen' is the remaining 1619c0e3d3aSJohn Baldwin * length of the current segment in the input buffer. 1629c0e3d3aSJohn Baldwin * 'outlen' is the remaining length of current segment in the 1639c0e3d3aSJohn Baldwin * output buffer. 164091d81d1SSam Leffler */ 16569e533c3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 1669c0e3d3aSJohn Baldwin /* 1679c0e3d3aSJohn Baldwin * If the current block is not contained within the 1689c0e3d3aSJohn Baldwin * current input/output segment, use 'blk' as a local 1699c0e3d3aSJohn Baldwin * buffer. 1709c0e3d3aSJohn Baldwin */ 17169e533c3SJohn Baldwin if (inlen < blksz) { 17269e533c3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 1739c0e3d3aSJohn Baldwin inblk = blk; 17469e533c3SJohn Baldwin inlen = blksz; 175d295bdeeSPawel Jakub Dawidek } 17669e533c3SJohn Baldwin if (outlen < blksz) { 1779c0e3d3aSJohn Baldwin outblk = blk; 17869e533c3SJohn Baldwin outlen = blksz; 179cfb7b942SJohn Baldwin } 180cfb7b942SJohn Baldwin 18169e533c3SJohn Baldwin todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blksz); 1829c0e3d3aSJohn Baldwin 1839c0e3d3aSJohn Baldwin if (encrypting) 184cfb7b942SJohn Baldwin exf->encrypt_multi(ctx, inblk, outblk, todo); 1859c0e3d3aSJohn Baldwin else 186cfb7b942SJohn Baldwin exf->decrypt_multi(ctx, inblk, outblk, todo); 187091d81d1SSam Leffler 188cfb7b942SJohn Baldwin if (inblk == blk) { 18986be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 1909c0e3d3aSJohn Baldwin } else { 191cfb7b942SJohn Baldwin crypto_cursor_advance(&cc_in, todo); 192cfb7b942SJohn Baldwin inlen -= todo; 193cfb7b942SJohn Baldwin inblk += todo; 194cfb7b942SJohn Baldwin if (inlen == 0) 195cfb7b942SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 19608fca7a5SJohn-Mark Gurney } 197091d81d1SSam Leffler 198cfb7b942SJohn Baldwin if (outblk == blk) { 19969e533c3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 20086be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 2019c0e3d3aSJohn Baldwin } else { 202cfb7b942SJohn Baldwin crypto_cursor_advance(&cc_out, todo); 203cfb7b942SJohn Baldwin outlen -= todo; 204cfb7b942SJohn Baldwin outblk += todo; 205cfb7b942SJohn Baldwin if (outlen == 0) 206cfb7b942SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 207cfb7b942SJohn Baldwin &outlen); 208091d81d1SSam Leffler } 209f34a967bSPawel Jakub Dawidek } 210f34a967bSPawel Jakub Dawidek 211723d8764SJohn Baldwin /* Handle trailing partial block for stream ciphers. */ 2129c0e3d3aSJohn Baldwin if (resid > 0) { 213723d8764SJohn Baldwin KASSERT(exf->native_blocksize != 0, 214723d8764SJohn Baldwin ("%s: partial block of %d bytes for cipher %s", 2152cd83828SJohn Baldwin __func__, resid, exf->name)); 21669e533c3SJohn Baldwin KASSERT(resid < blksz, ("%s: partial block too big", __func__)); 217723d8764SJohn Baldwin 21886be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 21986be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 2209c0e3d3aSJohn Baldwin if (inlen < resid) { 2219c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 2229c0e3d3aSJohn Baldwin inblk = blk; 22386be314dSJohn Baldwin } 2249c0e3d3aSJohn Baldwin if (outlen < resid) 2259c0e3d3aSJohn Baldwin outblk = blk; 2269c0e3d3aSJohn Baldwin if (encrypting) 227b54d1284SJohn Baldwin exf->encrypt_last(ctx, inblk, outblk, 2289c0e3d3aSJohn Baldwin resid); 2299c0e3d3aSJohn Baldwin else 230b54d1284SJohn Baldwin exf->decrypt_last(ctx, inblk, outblk, 2319c0e3d3aSJohn Baldwin resid); 2329c0e3d3aSJohn Baldwin if (outlen < resid) 2339c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 234723d8764SJohn Baldwin } 235723d8764SJohn Baldwin 236b54d1284SJohn Baldwin explicit_bzero(ctx, exf->ctxsize); 23720c128daSJohn Baldwin explicit_bzero(blk, sizeof(blk)); 2389c0e3d3aSJohn Baldwin return (0); 239091d81d1SSam Leffler } 240091d81d1SSam Leffler 241091d81d1SSam Leffler /* 242c0341432SJohn Baldwin * Compute or verify hash. 243091d81d1SSam Leffler */ 244091d81d1SSam Leffler static int 245dc475c9bSJohn Baldwin swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 246091d81d1SSam Leffler { 2477051c579SJohn Baldwin struct { 2487051c579SJohn Baldwin union authctx ctx; 249c0341432SJohn Baldwin u_char aalg[HASH_MAX_LEN]; 2507051c579SJohn Baldwin u_char uaalg[HASH_MAX_LEN]; 2517051c579SJohn Baldwin } s; 252c0341432SJohn Baldwin const struct crypto_session_params *csp; 2536113a08bSJohn Baldwin const struct swcr_auth *sw; 254d8787d4fSMark Johnston const struct auth_hash *axf; 255091d81d1SSam Leffler int err; 256091d81d1SSam Leffler 257c0341432SJohn Baldwin sw = &ses->swcr_auth; 258091d81d1SSam Leffler 259091d81d1SSam Leffler axf = sw->sw_axf; 260091d81d1SSam Leffler 261c0341432SJohn Baldwin csp = crypto_get_params(crp->crp_session); 2626038018aSMarcin Wojtas if (crp->crp_auth_key != NULL) { 2636113a08bSJohn Baldwin if (sw->sw_hmac) { 2646113a08bSJohn Baldwin hmac_init_ipad(axf, crp->crp_auth_key, 2657051c579SJohn Baldwin csp->csp_auth_klen, &s.ctx); 2666113a08bSJohn Baldwin } else { 2677051c579SJohn Baldwin axf->Init(&s.ctx); 2687051c579SJohn Baldwin axf->Setkey(&s.ctx, crp->crp_auth_key, 269c0341432SJohn Baldwin csp->csp_auth_klen); 27025b7033bSConrad Meyer } 2716113a08bSJohn Baldwin } else 2727051c579SJohn Baldwin memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize); 273091d81d1SSam Leffler 2749b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 2757051c579SJohn Baldwin err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length); 2769b774dc0SJohn Baldwin else 277c0341432SJohn Baldwin err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 2787051c579SJohn Baldwin axf->Update, &s.ctx); 279091d81d1SSam Leffler if (err) 280e0b155feSJohn Baldwin goto out; 281091d81d1SSam Leffler 2829c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 2839c0e3d3aSJohn Baldwin CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 2849c0e3d3aSJohn Baldwin err = crypto_apply_buf(&crp->crp_obuf, 2859c0e3d3aSJohn Baldwin crp->crp_payload_output_start, crp->crp_payload_length, 2867051c579SJohn Baldwin axf->Update, &s.ctx); 2879c0e3d3aSJohn Baldwin else 2889c0e3d3aSJohn Baldwin err = crypto_apply(crp, crp->crp_payload_start, 2897051c579SJohn Baldwin crp->crp_payload_length, axf->Update, &s.ctx); 290c0341432SJohn Baldwin if (err) 291e0b155feSJohn Baldwin goto out; 292091d81d1SSam Leffler 2936038018aSMarcin Wojtas if (csp->csp_flags & CSP_F_ESN) 2947051c579SJohn Baldwin axf->Update(&s.ctx, crp->crp_esn, 4); 2956038018aSMarcin Wojtas 2967051c579SJohn Baldwin axf->Final(s.aalg, &s.ctx); 2976113a08bSJohn Baldwin if (sw->sw_hmac) { 2986113a08bSJohn Baldwin if (crp->crp_auth_key != NULL) 2996113a08bSJohn Baldwin hmac_init_opad(axf, crp->crp_auth_key, 3007051c579SJohn Baldwin csp->csp_auth_klen, &s.ctx); 3016113a08bSJohn Baldwin else 3027051c579SJohn Baldwin memcpy(&s.ctx, sw->sw_octx, axf->ctxsize); 3037051c579SJohn Baldwin axf->Update(&s.ctx, s.aalg, axf->hashsize); 3047051c579SJohn Baldwin axf->Final(s.aalg, &s.ctx); 305091d81d1SSam Leffler } 306091d81d1SSam Leffler 307c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 3087051c579SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg); 3097051c579SJohn Baldwin if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0) 31020c128daSJohn Baldwin err = EBADMSG; 311c0341432SJohn Baldwin } else { 312091d81d1SSam Leffler /* Inject the authentication data */ 3137051c579SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg); 314c0341432SJohn Baldwin } 315e0b155feSJohn Baldwin out: 3167051c579SJohn Baldwin explicit_bzero(&s, sizeof(s)); 31720c128daSJohn Baldwin return (err); 318091d81d1SSam Leffler } 319091d81d1SSam Leffler 32008fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 32108fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 32208fca7a5SJohn-Mark Gurney 32308fca7a5SJohn-Mark Gurney static int 324dc475c9bSJohn Baldwin swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 32508fca7a5SJohn-Mark Gurney { 3267051c579SJohn Baldwin struct { 3277051c579SJohn Baldwin union authctx ctx; 32826d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 32926d292d3SJohn Baldwin u_char tag[GMAC_DIGEST_LEN]; 3307051c579SJohn Baldwin u_char tag2[GMAC_DIGEST_LEN]; 3317051c579SJohn Baldwin } s; 3327051c579SJohn Baldwin u_char *blk = (u_char *)s.blkbuf; 3339c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc; 33426d292d3SJohn Baldwin const u_char *inblk; 3356113a08bSJohn Baldwin const struct swcr_auth *swa; 336d8787d4fSMark Johnston const struct auth_hash *axf; 33708fca7a5SJohn-Mark Gurney uint32_t *blkp; 33886be314dSJohn Baldwin size_t len; 33986be314dSJohn Baldwin int blksz, error, ivlen, resid; 34008fca7a5SJohn-Mark Gurney 341c0341432SJohn Baldwin swa = &ses->swcr_auth; 34208fca7a5SJohn-Mark Gurney axf = swa->sw_axf; 34326d292d3SJohn Baldwin blksz = GMAC_BLOCK_LEN; 34426d292d3SJohn Baldwin KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 34526d292d3SJohn Baldwin __func__)); 34608fca7a5SJohn-Mark Gurney 3476113a08bSJohn Baldwin if (crp->crp_auth_key != NULL) { 3487051c579SJohn Baldwin axf->Init(&s.ctx); 3497051c579SJohn Baldwin axf->Setkey(&s.ctx, crp->crp_auth_key, 3506113a08bSJohn Baldwin crypto_get_params(crp->crp_session)->csp_auth_klen); 3516113a08bSJohn Baldwin } else 3527051c579SJohn Baldwin memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 3536113a08bSJohn Baldwin 35408fca7a5SJohn-Mark Gurney /* Initialize the IV */ 355c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 3561c9b25a5SJohn Baldwin crypto_read_iv(crp, blk); 35708fca7a5SJohn-Mark Gurney 3587051c579SJohn Baldwin axf->Reinit(&s.ctx, blk, ivlen); 3599c0e3d3aSJohn Baldwin crypto_cursor_init(&cc, &crp->crp_buf); 3609c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc, crp->crp_payload_start); 36126d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 36286be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc, &len); 36326d292d3SJohn Baldwin if (len >= blksz) { 36426d292d3SJohn Baldwin len = rounddown(MIN(len, resid), blksz); 36526d292d3SJohn Baldwin crypto_cursor_advance(&cc, len); 36626d292d3SJohn Baldwin } else { 36726d292d3SJohn Baldwin len = blksz; 3689c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc, len, blk); 36926d292d3SJohn Baldwin inblk = blk; 37026d292d3SJohn Baldwin } 3717051c579SJohn Baldwin axf->Update(&s.ctx, inblk, len); 37226d292d3SJohn Baldwin } 37326d292d3SJohn Baldwin if (resid > 0) { 37426d292d3SJohn Baldwin memset(blk, 0, blksz); 37526d292d3SJohn Baldwin crypto_cursor_copydata(&cc, resid, blk); 3767051c579SJohn Baldwin axf->Update(&s.ctx, blk, blksz); 37708fca7a5SJohn-Mark Gurney } 37808fca7a5SJohn-Mark Gurney 37908fca7a5SJohn-Mark Gurney /* length block */ 38026d292d3SJohn Baldwin memset(blk, 0, blksz); 38108fca7a5SJohn-Mark Gurney blkp = (uint32_t *)blk + 1; 382c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 3837051c579SJohn Baldwin axf->Update(&s.ctx, blk, blksz); 384c0341432SJohn Baldwin 385c0341432SJohn Baldwin /* Finalize MAC */ 3867051c579SJohn Baldwin axf->Final(s.tag, &s.ctx); 387c0341432SJohn Baldwin 38820c128daSJohn Baldwin error = 0; 389c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 390c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 3917051c579SJohn Baldwin s.tag2); 3927051c579SJohn Baldwin if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 39320c128daSJohn Baldwin error = EBADMSG; 394c0341432SJohn Baldwin } else { 395c0341432SJohn Baldwin /* Inject the authentication data */ 3967051c579SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag); 397c0341432SJohn Baldwin } 3987051c579SJohn Baldwin explicit_bzero(&s, sizeof(s)); 39920c128daSJohn Baldwin return (error); 400c0341432SJohn Baldwin } 401c0341432SJohn Baldwin 402c0341432SJohn Baldwin static int 403dc475c9bSJohn Baldwin swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 404c0341432SJohn Baldwin { 4057051c579SJohn Baldwin struct { 40626d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 40726d292d3SJohn Baldwin u_char tag[GMAC_DIGEST_LEN]; 4087051c579SJohn Baldwin u_char tag2[GMAC_DIGEST_LEN]; 4097051c579SJohn Baldwin } s; 4107051c579SJohn Baldwin u_char *blk = (u_char *)s.blkbuf; 4119c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 41226d292d3SJohn Baldwin const u_char *inblk; 41326d292d3SJohn Baldwin u_char *outblk; 414a221a8f4SJohn Baldwin size_t inlen, outlen, todo; 415b54d1284SJohn Baldwin const struct swcr_auth *swa; 416b54d1284SJohn Baldwin const struct swcr_encdec *swe; 417d8787d4fSMark Johnston const struct enc_xform *exf; 418b54d1284SJohn Baldwin void *ctx; 419c0341432SJohn Baldwin uint32_t *blkp; 42086be314dSJohn Baldwin int blksz, error, ivlen, r, resid; 421c0341432SJohn Baldwin 422c0341432SJohn Baldwin swa = &ses->swcr_auth; 423c0341432SJohn Baldwin swe = &ses->swcr_encdec; 424c0341432SJohn Baldwin exf = swe->sw_exf; 425ab91fb6cSJohn Baldwin blksz = GMAC_BLOCK_LEN; 426ab91fb6cSJohn Baldwin KASSERT(blksz == exf->native_blocksize, 427723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 428c0341432SJohn Baldwin 429c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 430c0341432SJohn Baldwin return (EINVAL); 431c0341432SJohn Baldwin 432c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 433c0341432SJohn Baldwin 434b54d1284SJohn Baldwin ctx = __builtin_alloca(exf->ctxsize); 435ab91fb6cSJohn Baldwin if (crp->crp_cipher_key != NULL) 436b54d1284SJohn Baldwin exf->setkey(ctx, crp->crp_cipher_key, 437ab91fb6cSJohn Baldwin crypto_get_params(crp->crp_session)->csp_cipher_klen); 438b54d1284SJohn Baldwin else 439b54d1284SJohn Baldwin memcpy(ctx, swe->sw_ctx, exf->ctxsize); 440b54d1284SJohn Baldwin exf->reinit(ctx, crp->crp_iv, ivlen); 441c0341432SJohn Baldwin 442c0341432SJohn Baldwin /* Supply MAC with AAD */ 4439b774dc0SJohn Baldwin if (crp->crp_aad != NULL) { 444a221a8f4SJohn Baldwin inlen = rounddown2(crp->crp_aad_length, blksz); 445a221a8f4SJohn Baldwin if (inlen != 0) 446a221a8f4SJohn Baldwin exf->update(ctx, crp->crp_aad, inlen); 447a221a8f4SJohn Baldwin if (crp->crp_aad_length != inlen) { 4489b774dc0SJohn Baldwin memset(blk, 0, blksz); 449a221a8f4SJohn Baldwin memcpy(blk, (char *)crp->crp_aad + inlen, 450a221a8f4SJohn Baldwin crp->crp_aad_length - inlen); 451b54d1284SJohn Baldwin exf->update(ctx, blk, blksz); 4529b774dc0SJohn Baldwin } 4539b774dc0SJohn Baldwin } else { 4549c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 4559c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_aad_start); 4569b774dc0SJohn Baldwin for (resid = crp->crp_aad_length; resid >= blksz; 457a221a8f4SJohn Baldwin resid -= inlen) { 458a221a8f4SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 459a221a8f4SJohn Baldwin if (inlen >= blksz) { 460a221a8f4SJohn Baldwin inlen = rounddown2(MIN(inlen, resid), blksz); 461a221a8f4SJohn Baldwin crypto_cursor_advance(&cc_in, inlen); 46226d292d3SJohn Baldwin } else { 463a221a8f4SJohn Baldwin inlen = blksz; 464a221a8f4SJohn Baldwin crypto_cursor_copydata(&cc_in, inlen, blk); 46526d292d3SJohn Baldwin inblk = blk; 46626d292d3SJohn Baldwin } 467a221a8f4SJohn Baldwin exf->update(ctx, inblk, inlen); 46826d292d3SJohn Baldwin } 46926d292d3SJohn Baldwin if (resid > 0) { 47026d292d3SJohn Baldwin memset(blk, 0, blksz); 47126d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 472b54d1284SJohn Baldwin exf->update(ctx, blk, blksz); 473c0341432SJohn Baldwin } 4749b774dc0SJohn Baldwin } 475c0341432SJohn Baldwin 476c0341432SJohn Baldwin /* Do encryption with MAC */ 4779c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 4789c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 479a221a8f4SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 4809c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 4819c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 4829c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 4839c0e3d3aSJohn Baldwin } else 4849c0e3d3aSJohn Baldwin cc_out = cc_in; 485a221a8f4SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 486a221a8f4SJohn Baldwin 487a221a8f4SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 488a221a8f4SJohn Baldwin if (inlen < blksz) { 48926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 49026d292d3SJohn Baldwin inblk = blk; 491a221a8f4SJohn Baldwin inlen = blksz; 492c0341432SJohn Baldwin } 493a221a8f4SJohn Baldwin 49426d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 495a221a8f4SJohn Baldwin if (outlen < blksz) { 49626d292d3SJohn Baldwin outblk = blk; 497a221a8f4SJohn Baldwin outlen = blksz; 498a221a8f4SJohn Baldwin } 499a221a8f4SJohn Baldwin 500a221a8f4SJohn Baldwin todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 501a221a8f4SJohn Baldwin blksz); 502a221a8f4SJohn Baldwin 503a221a8f4SJohn Baldwin exf->encrypt_multi(ctx, inblk, outblk, todo); 504a221a8f4SJohn Baldwin exf->update(ctx, outblk, todo); 505a221a8f4SJohn Baldwin 506a221a8f4SJohn Baldwin if (outblk == blk) { 50726d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 508a221a8f4SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 50926d292d3SJohn Baldwin } else { 510a221a8f4SJohn Baldwin crypto_cursor_advance(&cc_out, todo); 511a221a8f4SJohn Baldwin outlen -= todo; 512a221a8f4SJohn Baldwin outblk += todo; 513a221a8f4SJohn Baldwin if (outlen == 0) 514a221a8f4SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 515a221a8f4SJohn Baldwin &outlen); 516a221a8f4SJohn Baldwin } 517a221a8f4SJohn Baldwin } else { 518a221a8f4SJohn Baldwin todo = rounddown2(MIN(resid, inlen), blksz); 519a221a8f4SJohn Baldwin exf->update(ctx, inblk, todo); 520a221a8f4SJohn Baldwin } 521a221a8f4SJohn Baldwin 522a221a8f4SJohn Baldwin if (inblk == blk) { 523a221a8f4SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 524a221a8f4SJohn Baldwin } else { 525a221a8f4SJohn Baldwin crypto_cursor_advance(&cc_in, todo); 526a221a8f4SJohn Baldwin inlen -= todo; 527a221a8f4SJohn Baldwin inblk += todo; 528a221a8f4SJohn Baldwin if (inlen == 0) 529a221a8f4SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 53026d292d3SJohn Baldwin } 53126d292d3SJohn Baldwin } 53226d292d3SJohn Baldwin if (resid > 0) { 53326d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 53426d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 535b54d1284SJohn Baldwin exf->encrypt_last(ctx, blk, blk, resid); 53626d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 53726d292d3SJohn Baldwin } 538b54d1284SJohn Baldwin exf->update(ctx, blk, resid); 539c0341432SJohn Baldwin } 540c0341432SJohn Baldwin 541c0341432SJohn Baldwin /* length block */ 54226d292d3SJohn Baldwin memset(blk, 0, blksz); 543c0341432SJohn Baldwin blkp = (uint32_t *)blk + 1; 544c0341432SJohn Baldwin *blkp = htobe32(crp->crp_aad_length * 8); 545c0341432SJohn Baldwin blkp = (uint32_t *)blk + 3; 546c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 547b54d1284SJohn Baldwin exf->update(ctx, blk, blksz); 548c0341432SJohn Baldwin 549c0341432SJohn Baldwin /* Finalize MAC */ 5507051c579SJohn Baldwin exf->final(s.tag, ctx); 551c0341432SJohn Baldwin 552c0341432SJohn Baldwin /* Validate tag */ 55320c128daSJohn Baldwin error = 0; 554c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 5557051c579SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 5567051c579SJohn Baldwin s.tag2); 5577051c579SJohn Baldwin r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 55820c128daSJohn Baldwin if (r != 0) { 55920c128daSJohn Baldwin error = EBADMSG; 56020c128daSJohn Baldwin goto out; 56120c128daSJohn Baldwin } 562c0341432SJohn Baldwin 563c0341432SJohn Baldwin /* tag matches, decrypt data */ 5649c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5659c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 566a221a8f4SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 567a221a8f4SJohn Baldwin 56826d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 569a221a8f4SJohn Baldwin resid -= todo) { 570a221a8f4SJohn Baldwin if (inlen < blksz) { 57126d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 57226d292d3SJohn Baldwin inblk = blk; 573a221a8f4SJohn Baldwin inlen = blksz; 574a221a8f4SJohn Baldwin } 575a221a8f4SJohn Baldwin if (outlen < blksz) { 57626d292d3SJohn Baldwin outblk = blk; 577a221a8f4SJohn Baldwin outlen = blksz; 578a221a8f4SJohn Baldwin } 579a221a8f4SJohn Baldwin 580a221a8f4SJohn Baldwin todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 581a221a8f4SJohn Baldwin blksz); 582a221a8f4SJohn Baldwin 583a221a8f4SJohn Baldwin exf->decrypt_multi(ctx, inblk, outblk, todo); 584a221a8f4SJohn Baldwin 585a221a8f4SJohn Baldwin if (inblk == blk) { 586a221a8f4SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 587a221a8f4SJohn Baldwin } else { 588a221a8f4SJohn Baldwin crypto_cursor_advance(&cc_in, todo); 589a221a8f4SJohn Baldwin inlen -= todo; 590a221a8f4SJohn Baldwin inblk += todo; 591a221a8f4SJohn Baldwin if (inlen == 0) 592a221a8f4SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, 593a221a8f4SJohn Baldwin &inlen); 594a221a8f4SJohn Baldwin } 595a221a8f4SJohn Baldwin 596a221a8f4SJohn Baldwin if (outblk == blk) { 59726d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 598a221a8f4SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 599a221a8f4SJohn Baldwin &outlen); 600a221a8f4SJohn Baldwin } else { 601a221a8f4SJohn Baldwin crypto_cursor_advance(&cc_out, todo); 602a221a8f4SJohn Baldwin outlen -= todo; 603a221a8f4SJohn Baldwin outblk += todo; 604a221a8f4SJohn Baldwin if (outlen == 0) 605a221a8f4SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 606a221a8f4SJohn Baldwin &outlen); 607a221a8f4SJohn Baldwin } 60826d292d3SJohn Baldwin } 60926d292d3SJohn Baldwin if (resid > 0) { 61026d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 611b54d1284SJohn Baldwin exf->decrypt_last(ctx, blk, blk, resid); 61226d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 613c0341432SJohn Baldwin } 614c0341432SJohn Baldwin } else { 615c0341432SJohn Baldwin /* Inject the authentication data */ 6167051c579SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 6177051c579SJohn Baldwin s.tag); 618c0341432SJohn Baldwin } 619c0341432SJohn Baldwin 62020c128daSJohn Baldwin out: 621b54d1284SJohn Baldwin explicit_bzero(ctx, exf->ctxsize); 6227051c579SJohn Baldwin explicit_bzero(&s, sizeof(s)); 62320c128daSJohn Baldwin 62420c128daSJohn Baldwin return (error); 625c0341432SJohn Baldwin } 626c0341432SJohn Baldwin 6274361c4ebSJohn Baldwin static void 6284361c4ebSJohn Baldwin build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 6294361c4ebSJohn Baldwin u_int data_length, u_int tag_length, uint8_t *b0) 6304361c4ebSJohn Baldwin { 6314361c4ebSJohn Baldwin uint8_t *bp; 6324361c4ebSJohn Baldwin uint8_t flags, L; 6334361c4ebSJohn Baldwin 6344361c4ebSJohn Baldwin KASSERT(nonce_length >= 7 && nonce_length <= 13, 6354361c4ebSJohn Baldwin ("nonce_length must be between 7 and 13 bytes")); 6364361c4ebSJohn Baldwin 6374361c4ebSJohn Baldwin /* 6384361c4ebSJohn Baldwin * Need to determine the L field value. This is the number of 6394361c4ebSJohn Baldwin * bytes needed to specify the length of the message; the length 6404361c4ebSJohn Baldwin * is whatever is left in the 16 bytes after specifying flags and 6414361c4ebSJohn Baldwin * the nonce. 6424361c4ebSJohn Baldwin */ 6434361c4ebSJohn Baldwin L = 15 - nonce_length; 6444361c4ebSJohn Baldwin 6454361c4ebSJohn Baldwin flags = ((aad_length > 0) << 6) + 6464361c4ebSJohn Baldwin (((tag_length - 2) / 2) << 3) + 6474361c4ebSJohn Baldwin L - 1; 6484361c4ebSJohn Baldwin 6494361c4ebSJohn Baldwin /* 6504361c4ebSJohn Baldwin * Now we need to set up the first block, which has flags, nonce, 6514361c4ebSJohn Baldwin * and the message length. 6524361c4ebSJohn Baldwin */ 6534361c4ebSJohn Baldwin b0[0] = flags; 6544361c4ebSJohn Baldwin memcpy(b0 + 1, nonce, nonce_length); 6554361c4ebSJohn Baldwin bp = b0 + 1 + nonce_length; 6564361c4ebSJohn Baldwin 6574361c4ebSJohn Baldwin /* Need to copy L' [aka L-1] bytes of data_length */ 6584361c4ebSJohn Baldwin for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 6594361c4ebSJohn Baldwin *dst = data_length; 6604361c4ebSJohn Baldwin data_length >>= 8; 6614361c4ebSJohn Baldwin } 6624361c4ebSJohn Baldwin } 6634361c4ebSJohn Baldwin 6644361c4ebSJohn Baldwin /* NB: OCF only supports AAD lengths < 2^32. */ 6654361c4ebSJohn Baldwin static int 6664361c4ebSJohn Baldwin build_ccm_aad_length(u_int aad_length, uint8_t *blk) 6674361c4ebSJohn Baldwin { 6684361c4ebSJohn Baldwin if (aad_length < ((1 << 16) - (1 << 8))) { 6694361c4ebSJohn Baldwin be16enc(blk, aad_length); 6704361c4ebSJohn Baldwin return (sizeof(uint16_t)); 6714361c4ebSJohn Baldwin } else { 6724361c4ebSJohn Baldwin blk[0] = 0xff; 6734361c4ebSJohn Baldwin blk[1] = 0xfe; 6744361c4ebSJohn Baldwin be32enc(blk + 2, aad_length); 6754361c4ebSJohn Baldwin return (2 + sizeof(uint32_t)); 6764361c4ebSJohn Baldwin } 6774361c4ebSJohn Baldwin } 6784361c4ebSJohn Baldwin 679c0341432SJohn Baldwin static int 680dc475c9bSJohn Baldwin swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 681c0341432SJohn Baldwin { 6827051c579SJohn Baldwin struct { 6837051c579SJohn Baldwin union authctx ctx; 6844361c4ebSJohn Baldwin u_char blk[CCM_CBC_BLOCK_LEN]; 6854361c4ebSJohn Baldwin u_char tag[AES_CBC_MAC_HASH_LEN]; 6867051c579SJohn Baldwin u_char tag2[AES_CBC_MAC_HASH_LEN]; 6877051c579SJohn Baldwin } s; 688ae18720dSJohn Baldwin const struct crypto_session_params *csp; 6896113a08bSJohn Baldwin const struct swcr_auth *swa; 690d8787d4fSMark Johnston const struct auth_hash *axf; 6914361c4ebSJohn Baldwin int error, ivlen, len; 692c0341432SJohn Baldwin 693ae18720dSJohn Baldwin csp = crypto_get_params(crp->crp_session); 694c0341432SJohn Baldwin swa = &ses->swcr_auth; 695c0341432SJohn Baldwin axf = swa->sw_axf; 696c0341432SJohn Baldwin 6976113a08bSJohn Baldwin if (crp->crp_auth_key != NULL) { 6987051c579SJohn Baldwin axf->Init(&s.ctx); 6997051c579SJohn Baldwin axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen); 7006113a08bSJohn Baldwin } else 7017051c579SJohn Baldwin memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 702c0341432SJohn Baldwin 703c0341432SJohn Baldwin /* Initialize the IV */ 704ae18720dSJohn Baldwin ivlen = csp->csp_ivlen; 705c0341432SJohn Baldwin 7064361c4ebSJohn Baldwin /* Supply MAC with IV */ 7077051c579SJohn Baldwin axf->Reinit(&s.ctx, crp->crp_iv, ivlen); 708c0341432SJohn Baldwin 7094361c4ebSJohn Baldwin /* Supply MAC with b0. */ 7104361c4ebSJohn Baldwin build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 7117051c579SJohn Baldwin swa->sw_mlen, s.blk); 7127051c579SJohn Baldwin axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN); 7134361c4ebSJohn Baldwin 7147051c579SJohn Baldwin len = build_ccm_aad_length(crp->crp_payload_length, s.blk); 7157051c579SJohn Baldwin axf->Update(&s.ctx, s.blk, len); 7164361c4ebSJohn Baldwin 7174361c4ebSJohn Baldwin crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 7187051c579SJohn Baldwin axf->Update, &s.ctx); 719c0341432SJohn Baldwin 720c0341432SJohn Baldwin /* Finalize MAC */ 7217051c579SJohn Baldwin axf->Final(s.tag, &s.ctx); 722c0341432SJohn Baldwin 7234361c4ebSJohn Baldwin error = 0; 724c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 725c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 7267051c579SJohn Baldwin s.tag2); 7277051c579SJohn Baldwin if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 72820c128daSJohn Baldwin error = EBADMSG; 729c0341432SJohn Baldwin } else { 730c0341432SJohn Baldwin /* Inject the authentication data */ 7317051c579SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 7327051c579SJohn Baldwin s.tag); 733c0341432SJohn Baldwin } 7347051c579SJohn Baldwin explicit_bzero(&s, sizeof(s)); 73520c128daSJohn Baldwin return (error); 736c0341432SJohn Baldwin } 737c0341432SJohn Baldwin 738c0341432SJohn Baldwin static int 739dc475c9bSJohn Baldwin swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 740c0341432SJohn Baldwin { 741ae18720dSJohn Baldwin const struct crypto_session_params *csp; 7427051c579SJohn Baldwin struct { 74326d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 74426d292d3SJohn Baldwin u_char tag[AES_CBC_MAC_HASH_LEN]; 7457051c579SJohn Baldwin u_char tag2[AES_CBC_MAC_HASH_LEN]; 7467051c579SJohn Baldwin } s; 7477051c579SJohn Baldwin u_char *blk = (u_char *)s.blkbuf; 7489c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 74926d292d3SJohn Baldwin const u_char *inblk; 75026d292d3SJohn Baldwin u_char *outblk; 751f8580fcaSJohn Baldwin size_t inlen, outlen, todo; 752b54d1284SJohn Baldwin const struct swcr_auth *swa; 753b54d1284SJohn Baldwin const struct swcr_encdec *swe; 754d8787d4fSMark Johnston const struct enc_xform *exf; 755b54d1284SJohn Baldwin void *ctx; 75686be314dSJohn Baldwin size_t len; 75726d292d3SJohn Baldwin int blksz, error, ivlen, r, resid; 758c0341432SJohn Baldwin 759ae18720dSJohn Baldwin csp = crypto_get_params(crp->crp_session); 760c0341432SJohn Baldwin swa = &ses->swcr_auth; 761c0341432SJohn Baldwin swe = &ses->swcr_encdec; 762c0341432SJohn Baldwin exf = swe->sw_exf; 763ab91fb6cSJohn Baldwin blksz = AES_BLOCK_LEN; 764ab91fb6cSJohn Baldwin KASSERT(blksz == exf->native_blocksize, 765723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 766c0341432SJohn Baldwin 767ae18720dSJohn Baldwin if (crp->crp_payload_length > ccm_max_payload_length(csp)) 768ae18720dSJohn Baldwin return (EMSGSIZE); 769ae18720dSJohn Baldwin 770c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 771c0341432SJohn Baldwin return (EINVAL); 772c0341432SJohn Baldwin 773ae18720dSJohn Baldwin ivlen = csp->csp_ivlen; 774c0341432SJohn Baldwin 775b54d1284SJohn Baldwin ctx = __builtin_alloca(exf->ctxsize); 776ab91fb6cSJohn Baldwin if (crp->crp_cipher_key != NULL) 777b54d1284SJohn Baldwin exf->setkey(ctx, crp->crp_cipher_key, 778ab91fb6cSJohn Baldwin crypto_get_params(crp->crp_session)->csp_cipher_klen); 779b54d1284SJohn Baldwin else 780b54d1284SJohn Baldwin memcpy(ctx, swe->sw_ctx, exf->ctxsize); 781b54d1284SJohn Baldwin exf->reinit(ctx, crp->crp_iv, ivlen); 782c0341432SJohn Baldwin 7834361c4ebSJohn Baldwin /* Supply MAC with b0. */ 7847051c579SJohn Baldwin _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN, 7854361c4ebSJohn Baldwin "blkbuf too small for b0"); 7864361c4ebSJohn Baldwin build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 7874361c4ebSJohn Baldwin crp->crp_payload_length, swa->sw_mlen, blk); 788b54d1284SJohn Baldwin exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 7894361c4ebSJohn Baldwin 790c0341432SJohn Baldwin /* Supply MAC with AAD */ 7914361c4ebSJohn Baldwin if (crp->crp_aad_length != 0) { 7924361c4ebSJohn Baldwin len = build_ccm_aad_length(crp->crp_aad_length, blk); 793b54d1284SJohn Baldwin exf->update(ctx, blk, len); 7949b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 795b54d1284SJohn Baldwin exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 7969b774dc0SJohn Baldwin else 7974361c4ebSJohn Baldwin crypto_apply(crp, crp->crp_aad_start, 798b54d1284SJohn Baldwin crp->crp_aad_length, exf->update, ctx); 7994361c4ebSJohn Baldwin 8004361c4ebSJohn Baldwin /* Pad the AAD (including length field) to a full block. */ 8014361c4ebSJohn Baldwin len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 8024361c4ebSJohn Baldwin if (len != 0) { 8034361c4ebSJohn Baldwin len = CCM_CBC_BLOCK_LEN - len; 8044361c4ebSJohn Baldwin memset(blk, 0, CCM_CBC_BLOCK_LEN); 805b54d1284SJohn Baldwin exf->update(ctx, blk, len); 8064361c4ebSJohn Baldwin } 8074361c4ebSJohn Baldwin } 808c0341432SJohn Baldwin 809c0341432SJohn Baldwin /* Do encryption/decryption with MAC */ 8109c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 8119c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 812f8580fcaSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 8139c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 8149c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 8159c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 8169c0e3d3aSJohn Baldwin } else 8179c0e3d3aSJohn Baldwin cc_out = cc_in; 818f8580fcaSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 819f8580fcaSJohn Baldwin 820f8580fcaSJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 821f8580fcaSJohn Baldwin if (inlen < blksz) { 82226d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 82326d292d3SJohn Baldwin inblk = blk; 824f8580fcaSJohn Baldwin inlen = blksz; 825f8580fcaSJohn Baldwin } 826f8580fcaSJohn Baldwin 827c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 828f8580fcaSJohn Baldwin if (outlen < blksz) { 82926d292d3SJohn Baldwin outblk = blk; 830f8580fcaSJohn Baldwin outlen = blksz; 831f8580fcaSJohn Baldwin } 832f8580fcaSJohn Baldwin 833f8580fcaSJohn Baldwin todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 834f8580fcaSJohn Baldwin blksz); 835f8580fcaSJohn Baldwin 836f8580fcaSJohn Baldwin exf->update(ctx, inblk, todo); 837f8580fcaSJohn Baldwin exf->encrypt_multi(ctx, inblk, outblk, todo); 838f8580fcaSJohn Baldwin 839f8580fcaSJohn Baldwin if (outblk == blk) { 84026d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 841f8580fcaSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 842f8580fcaSJohn Baldwin } else { 843f8580fcaSJohn Baldwin crypto_cursor_advance(&cc_out, todo); 844f8580fcaSJohn Baldwin outlen -= todo; 845f8580fcaSJohn Baldwin outblk += todo; 846f8580fcaSJohn Baldwin if (outlen == 0) 847f8580fcaSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 848f8580fcaSJohn Baldwin &outlen); 849f8580fcaSJohn Baldwin } 850c0341432SJohn Baldwin } else { 851c0341432SJohn Baldwin /* 852c0341432SJohn Baldwin * One of the problems with CCM+CBC is that 853c0341432SJohn Baldwin * the authentication is done on the 85426d292d3SJohn Baldwin * unencrypted data. As a result, we have to 855c0341432SJohn Baldwin * decrypt the data twice: once to generate 856c0341432SJohn Baldwin * the tag and a second time after the tag is 857c0341432SJohn Baldwin * verified. 858c0341432SJohn Baldwin */ 859f8580fcaSJohn Baldwin todo = blksz; 860b54d1284SJohn Baldwin exf->decrypt(ctx, inblk, blk); 861f8580fcaSJohn Baldwin exf->update(ctx, blk, todo); 862f8580fcaSJohn Baldwin } 863f8580fcaSJohn Baldwin 864f8580fcaSJohn Baldwin if (inblk == blk) { 865f8580fcaSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 866f8580fcaSJohn Baldwin } else { 867f8580fcaSJohn Baldwin crypto_cursor_advance(&cc_in, todo); 868f8580fcaSJohn Baldwin inlen -= todo; 869f8580fcaSJohn Baldwin inblk += todo; 870f8580fcaSJohn Baldwin if (inlen == 0) 871f8580fcaSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 87226d292d3SJohn Baldwin } 87326d292d3SJohn Baldwin } 87426d292d3SJohn Baldwin if (resid > 0) { 87526d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 87626d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 877b54d1284SJohn Baldwin exf->update(ctx, blk, resid); 878b54d1284SJohn Baldwin exf->encrypt_last(ctx, blk, blk, resid); 87926d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 88026d292d3SJohn Baldwin } else { 881b54d1284SJohn Baldwin exf->decrypt_last(ctx, blk, blk, resid); 882b54d1284SJohn Baldwin exf->update(ctx, blk, resid); 883c0341432SJohn Baldwin } 88408fca7a5SJohn-Mark Gurney } 88508fca7a5SJohn-Mark Gurney 88608fca7a5SJohn-Mark Gurney /* Finalize MAC */ 8877051c579SJohn Baldwin exf->final(s.tag, ctx); 88808fca7a5SJohn-Mark Gurney 88908fca7a5SJohn-Mark Gurney /* Validate tag */ 89020c128daSJohn Baldwin error = 0; 891c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 892c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 8937051c579SJohn Baldwin s.tag2); 8947051c579SJohn Baldwin r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 89520c128daSJohn Baldwin if (r != 0) { 89620c128daSJohn Baldwin error = EBADMSG; 89720c128daSJohn Baldwin goto out; 89820c128daSJohn Baldwin } 899c0341432SJohn Baldwin 90008fca7a5SJohn-Mark Gurney /* tag matches, decrypt data */ 901b54d1284SJohn Baldwin exf->reinit(ctx, crp->crp_iv, ivlen); 9029c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 9039c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 904f8580fcaSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 905f8580fcaSJohn Baldwin 906f8580fcaSJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; 907f8580fcaSJohn Baldwin resid -= todo) { 908f8580fcaSJohn Baldwin if (inlen < blksz) { 90926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 91026d292d3SJohn Baldwin inblk = blk; 911f8580fcaSJohn Baldwin inlen = blksz; 912f8580fcaSJohn Baldwin } 913f8580fcaSJohn Baldwin if (outlen < blksz) { 91426d292d3SJohn Baldwin outblk = blk; 915f8580fcaSJohn Baldwin outlen = blksz; 916f8580fcaSJohn Baldwin } 917f8580fcaSJohn Baldwin 918f8580fcaSJohn Baldwin todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 919f8580fcaSJohn Baldwin blksz); 920f8580fcaSJohn Baldwin 921f8580fcaSJohn Baldwin exf->decrypt_multi(ctx, inblk, outblk, todo); 922f8580fcaSJohn Baldwin 923f8580fcaSJohn Baldwin if (inblk == blk) { 924f8580fcaSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 925f8580fcaSJohn Baldwin } else { 926f8580fcaSJohn Baldwin crypto_cursor_advance(&cc_in, todo); 927f8580fcaSJohn Baldwin inlen -= todo; 928f8580fcaSJohn Baldwin inblk += todo; 929f8580fcaSJohn Baldwin if (inlen == 0) 930f8580fcaSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, 931f8580fcaSJohn Baldwin &inlen); 932f8580fcaSJohn Baldwin } 933f8580fcaSJohn Baldwin 934f8580fcaSJohn Baldwin if (outblk == blk) { 93526d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 936f8580fcaSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 937f8580fcaSJohn Baldwin &outlen); 938f8580fcaSJohn Baldwin } else { 939f8580fcaSJohn Baldwin crypto_cursor_advance(&cc_out, todo); 940f8580fcaSJohn Baldwin outlen -= todo; 941f8580fcaSJohn Baldwin outblk += todo; 942f8580fcaSJohn Baldwin if (outlen == 0) 943f8580fcaSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 944f8580fcaSJohn Baldwin &outlen); 945f8580fcaSJohn Baldwin } 94626d292d3SJohn Baldwin } 94726d292d3SJohn Baldwin if (resid > 0) { 94826d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 949b54d1284SJohn Baldwin exf->decrypt_last(ctx, blk, blk, resid); 95026d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 95108fca7a5SJohn-Mark Gurney } 95208fca7a5SJohn-Mark Gurney } else { 95308fca7a5SJohn-Mark Gurney /* Inject the authentication data */ 9547051c579SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 9557051c579SJohn Baldwin s.tag); 95608fca7a5SJohn-Mark Gurney } 95708fca7a5SJohn-Mark Gurney 95820c128daSJohn Baldwin out: 959b54d1284SJohn Baldwin explicit_bzero(ctx, exf->ctxsize); 9607051c579SJohn Baldwin explicit_bzero(&s, sizeof(s)); 96120c128daSJohn Baldwin return (error); 96208fca7a5SJohn-Mark Gurney } 96308fca7a5SJohn-Mark Gurney 964dd2e1352SJohn Baldwin static int 965dc475c9bSJohn Baldwin swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 966dd2e1352SJohn Baldwin { 967dd2e1352SJohn Baldwin const struct crypto_session_params *csp; 9687051c579SJohn Baldwin struct { 969dd2e1352SJohn Baldwin uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 970dd2e1352SJohn Baldwin u_char tag[POLY1305_HASH_LEN]; 9717051c579SJohn Baldwin u_char tag2[POLY1305_HASH_LEN]; 9727051c579SJohn Baldwin } s; 9737051c579SJohn Baldwin u_char *blk = (u_char *)s.blkbuf; 974dd2e1352SJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 975dd2e1352SJohn Baldwin const u_char *inblk; 976dd2e1352SJohn Baldwin u_char *outblk; 9775022c687SJohn Baldwin size_t inlen, outlen, todo; 978dd2e1352SJohn Baldwin uint64_t *blkp; 979b54d1284SJohn Baldwin const struct swcr_auth *swa; 980b54d1284SJohn Baldwin const struct swcr_encdec *swe; 981d8787d4fSMark Johnston const struct enc_xform *exf; 982b54d1284SJohn Baldwin void *ctx; 983dd2e1352SJohn Baldwin int blksz, error, r, resid; 984dd2e1352SJohn Baldwin 985dd2e1352SJohn Baldwin swa = &ses->swcr_auth; 986dd2e1352SJohn Baldwin swe = &ses->swcr_encdec; 987dd2e1352SJohn Baldwin exf = swe->sw_exf; 988dd2e1352SJohn Baldwin blksz = exf->native_blocksize; 9897051c579SJohn Baldwin KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__)); 990dd2e1352SJohn Baldwin 991dd2e1352SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 992dd2e1352SJohn Baldwin return (EINVAL); 993dd2e1352SJohn Baldwin 994dd2e1352SJohn Baldwin csp = crypto_get_params(crp->crp_session); 995dd2e1352SJohn Baldwin 996b54d1284SJohn Baldwin ctx = __builtin_alloca(exf->ctxsize); 997dd2e1352SJohn Baldwin if (crp->crp_cipher_key != NULL) 998b54d1284SJohn Baldwin exf->setkey(ctx, crp->crp_cipher_key, 999dd2e1352SJohn Baldwin csp->csp_cipher_klen); 1000b54d1284SJohn Baldwin else 1001b54d1284SJohn Baldwin memcpy(ctx, swe->sw_ctx, exf->ctxsize); 1002b54d1284SJohn Baldwin exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 1003dd2e1352SJohn Baldwin 1004ab91fb6cSJohn Baldwin /* Supply MAC with AAD */ 1005ab91fb6cSJohn Baldwin if (crp->crp_aad != NULL) 1006b54d1284SJohn Baldwin exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 1007ab91fb6cSJohn Baldwin else 1008b54d1284SJohn Baldwin crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 1009b54d1284SJohn Baldwin exf->update, ctx); 101004781697SJohn Baldwin if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) { 1011ab91fb6cSJohn Baldwin /* padding1 */ 101204781697SJohn Baldwin memset(blk, 0, POLY1305_BLOCK_LEN); 101304781697SJohn Baldwin exf->update(ctx, blk, POLY1305_BLOCK_LEN - 101404781697SJohn Baldwin crp->crp_aad_length % POLY1305_BLOCK_LEN); 1015ab91fb6cSJohn Baldwin } 1016ab91fb6cSJohn Baldwin 1017dd2e1352SJohn Baldwin /* Do encryption with MAC */ 1018dd2e1352SJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 1019dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 10205022c687SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 1021dd2e1352SJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1022dd2e1352SJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 1023dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1024dd2e1352SJohn Baldwin } else 1025dd2e1352SJohn Baldwin cc_out = cc_in; 10265022c687SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 10275022c687SJohn Baldwin 10285022c687SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 10295022c687SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; 10305022c687SJohn Baldwin resid -= todo) { 10315022c687SJohn Baldwin if (inlen < blksz) { 1032dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 1033dd2e1352SJohn Baldwin inblk = blk; 10345022c687SJohn Baldwin inlen = blksz; 10355022c687SJohn Baldwin } 10365022c687SJohn Baldwin 10375022c687SJohn Baldwin if (outlen < blksz) { 1038dd2e1352SJohn Baldwin outblk = blk; 10395022c687SJohn Baldwin outlen = blksz; 10405022c687SJohn Baldwin } 10415022c687SJohn Baldwin 10425022c687SJohn Baldwin todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 10435022c687SJohn Baldwin blksz); 10445022c687SJohn Baldwin 10455022c687SJohn Baldwin exf->encrypt_multi(ctx, inblk, outblk, todo); 10465022c687SJohn Baldwin exf->update(ctx, outblk, todo); 10475022c687SJohn Baldwin 10485022c687SJohn Baldwin if (inblk == blk) { 10495022c687SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 1050dd2e1352SJohn Baldwin } else { 10515022c687SJohn Baldwin crypto_cursor_advance(&cc_in, todo); 10525022c687SJohn Baldwin inlen -= todo; 10535022c687SJohn Baldwin inblk += todo; 10545022c687SJohn Baldwin if (inlen == 0) 10555022c687SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, 10565022c687SJohn Baldwin &inlen); 10575022c687SJohn Baldwin } 10585022c687SJohn Baldwin 10595022c687SJohn Baldwin if (outblk == blk) { 10605022c687SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 10615022c687SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 10625022c687SJohn Baldwin } else { 10635022c687SJohn Baldwin crypto_cursor_advance(&cc_out, todo); 10645022c687SJohn Baldwin outlen -= todo; 10655022c687SJohn Baldwin outblk += todo; 10665022c687SJohn Baldwin if (outlen == 0) 10675022c687SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 10685022c687SJohn Baldwin &outlen); 1069dd2e1352SJohn Baldwin } 1070dd2e1352SJohn Baldwin } 1071dd2e1352SJohn Baldwin if (resid > 0) { 1072dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 1073b54d1284SJohn Baldwin exf->encrypt_last(ctx, blk, blk, resid); 1074dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 1075b54d1284SJohn Baldwin exf->update(ctx, blk, resid); 10765022c687SJohn Baldwin } 10775022c687SJohn Baldwin } else 10785022c687SJohn Baldwin crypto_apply(crp, crp->crp_payload_start, 10795022c687SJohn Baldwin crp->crp_payload_length, exf->update, ctx); 10805022c687SJohn Baldwin if (crp->crp_payload_length % POLY1305_BLOCK_LEN != 0) { 1081dd2e1352SJohn Baldwin /* padding2 */ 108204781697SJohn Baldwin memset(blk, 0, POLY1305_BLOCK_LEN); 108304781697SJohn Baldwin exf->update(ctx, blk, POLY1305_BLOCK_LEN - 10845022c687SJohn Baldwin crp->crp_payload_length % POLY1305_BLOCK_LEN); 1085dd2e1352SJohn Baldwin } 1086dd2e1352SJohn Baldwin 1087dd2e1352SJohn Baldwin /* lengths */ 1088dd2e1352SJohn Baldwin blkp = (uint64_t *)blk; 1089dd2e1352SJohn Baldwin blkp[0] = htole64(crp->crp_aad_length); 1090dd2e1352SJohn Baldwin blkp[1] = htole64(crp->crp_payload_length); 1091b54d1284SJohn Baldwin exf->update(ctx, blk, sizeof(uint64_t) * 2); 1092dd2e1352SJohn Baldwin 1093dd2e1352SJohn Baldwin /* Finalize MAC */ 10947051c579SJohn Baldwin exf->final(s.tag, ctx); 1095dd2e1352SJohn Baldwin 1096dd2e1352SJohn Baldwin /* Validate tag */ 1097dd2e1352SJohn Baldwin error = 0; 1098dd2e1352SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 10997051c579SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 11007051c579SJohn Baldwin s.tag2); 11017051c579SJohn Baldwin r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 1102dd2e1352SJohn Baldwin if (r != 0) { 1103dd2e1352SJohn Baldwin error = EBADMSG; 1104dd2e1352SJohn Baldwin goto out; 1105dd2e1352SJohn Baldwin } 1106dd2e1352SJohn Baldwin 1107dd2e1352SJohn Baldwin /* tag matches, decrypt data */ 1108dd2e1352SJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 1109dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 11105022c687SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 11115022c687SJohn Baldwin 1112dd2e1352SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 11135022c687SJohn Baldwin resid -= todo) { 11145022c687SJohn Baldwin if (inlen < blksz) { 1115dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 1116dd2e1352SJohn Baldwin inblk = blk; 11175022c687SJohn Baldwin inlen = blksz; 11185022c687SJohn Baldwin } 11195022c687SJohn Baldwin if (outlen < blksz) { 1120dd2e1352SJohn Baldwin outblk = blk; 11215022c687SJohn Baldwin outlen = blksz; 11225022c687SJohn Baldwin } 11235022c687SJohn Baldwin 11245022c687SJohn Baldwin todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 11255022c687SJohn Baldwin blksz); 11265022c687SJohn Baldwin 11275022c687SJohn Baldwin exf->decrypt_multi(ctx, inblk, outblk, todo); 11285022c687SJohn Baldwin 11295022c687SJohn Baldwin if (inblk == blk) { 11305022c687SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 11315022c687SJohn Baldwin } else { 11325022c687SJohn Baldwin crypto_cursor_advance(&cc_in, todo); 11335022c687SJohn Baldwin inlen -= todo; 11345022c687SJohn Baldwin inblk += todo; 11355022c687SJohn Baldwin if (inlen == 0) 11365022c687SJohn Baldwin inblk = crypto_cursor_segment(&cc_in, 11375022c687SJohn Baldwin &inlen); 11385022c687SJohn Baldwin } 11395022c687SJohn Baldwin 11405022c687SJohn Baldwin if (outblk == blk) { 1141dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 11425022c687SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 11435022c687SJohn Baldwin &outlen); 11445022c687SJohn Baldwin } else { 11455022c687SJohn Baldwin crypto_cursor_advance(&cc_out, todo); 11465022c687SJohn Baldwin outlen -= todo; 11475022c687SJohn Baldwin outblk += todo; 11485022c687SJohn Baldwin if (outlen == 0) 11495022c687SJohn Baldwin outblk = crypto_cursor_segment(&cc_out, 11505022c687SJohn Baldwin &outlen); 11515022c687SJohn Baldwin } 1152dd2e1352SJohn Baldwin } 1153dd2e1352SJohn Baldwin if (resid > 0) { 1154dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 1155b54d1284SJohn Baldwin exf->decrypt_last(ctx, blk, blk, resid); 1156dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 1157dd2e1352SJohn Baldwin } 1158dd2e1352SJohn Baldwin } else { 1159dd2e1352SJohn Baldwin /* Inject the authentication data */ 11607051c579SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 11617051c579SJohn Baldwin s.tag); 1162dd2e1352SJohn Baldwin } 1163dd2e1352SJohn Baldwin 1164dd2e1352SJohn Baldwin out: 1165b54d1284SJohn Baldwin explicit_bzero(ctx, exf->ctxsize); 11667051c579SJohn Baldwin explicit_bzero(&s, sizeof(s)); 1167dd2e1352SJohn Baldwin return (error); 1168dd2e1352SJohn Baldwin } 1169dd2e1352SJohn Baldwin 1170091d81d1SSam Leffler /* 1171c0341432SJohn Baldwin * Apply a cipher and a digest to perform EtA. 1172c0341432SJohn Baldwin */ 1173c0341432SJohn Baldwin static int 1174dc475c9bSJohn Baldwin swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1175c0341432SJohn Baldwin { 1176c0341432SJohn Baldwin int error; 1177c0341432SJohn Baldwin 1178c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1179c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 1180c0341432SJohn Baldwin if (error == 0) 1181c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 1182c0341432SJohn Baldwin } else { 1183c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 1184c0341432SJohn Baldwin if (error == 0) 1185c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 1186c0341432SJohn Baldwin } 1187c0341432SJohn Baldwin return (error); 1188c0341432SJohn Baldwin } 1189c0341432SJohn Baldwin 1190c0341432SJohn Baldwin /* 1191091d81d1SSam Leffler * Apply a compression/decompression algorithm 1192091d81d1SSam Leffler */ 1193091d81d1SSam Leffler static int 1194dc475c9bSJohn Baldwin swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1195091d81d1SSam Leffler { 1196d8787d4fSMark Johnston const struct comp_algo *cxf; 1197d3d79e96SJohn Baldwin uint8_t *data, *out; 1198091d81d1SSam Leffler int adj; 1199d3d79e96SJohn Baldwin uint32_t result; 1200091d81d1SSam Leffler 1201c0341432SJohn Baldwin cxf = ses->swcr_compdec.sw_cxf; 1202091d81d1SSam Leffler 1203091d81d1SSam Leffler /* We must handle the whole buffer of data in one time 1204091d81d1SSam Leffler * then if there is not all the data in the mbuf, we must 1205091d81d1SSam Leffler * copy in a buffer. 1206091d81d1SSam Leffler */ 1207091d81d1SSam Leffler 1208c0341432SJohn Baldwin data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1209091d81d1SSam Leffler if (data == NULL) 1210091d81d1SSam Leffler return (EINVAL); 1211c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1212c0341432SJohn Baldwin data); 1213091d81d1SSam Leffler 1214c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1215c0341432SJohn Baldwin result = cxf->compress(data, crp->crp_payload_length, &out); 1216091d81d1SSam Leffler else 1217c0341432SJohn Baldwin result = cxf->decompress(data, crp->crp_payload_length, &out); 1218091d81d1SSam Leffler 12191ede983cSDag-Erling Smørgrav free(data, M_CRYPTO_DATA); 1220091d81d1SSam Leffler if (result == 0) 1221c0341432SJohn Baldwin return (EINVAL); 1222c0341432SJohn Baldwin crp->crp_olen = result; 1223c0341432SJohn Baldwin 1224c0341432SJohn Baldwin /* Check the compressed size when doing compression */ 1225c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1226c0341432SJohn Baldwin if (result >= crp->crp_payload_length) { 1227c0341432SJohn Baldwin /* Compression was useless, we lost time */ 1228c0341432SJohn Baldwin free(out, M_CRYPTO_DATA); 1229c0341432SJohn Baldwin return (0); 1230c0341432SJohn Baldwin } 1231c0341432SJohn Baldwin } 1232091d81d1SSam Leffler 1233091d81d1SSam Leffler /* Copy back the (de)compressed data. m_copyback is 1234091d81d1SSam Leffler * extending the mbuf as necessary. 1235091d81d1SSam Leffler */ 1236c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_payload_start, result, out); 1237c0341432SJohn Baldwin if (result < crp->crp_payload_length) { 12389c0e3d3aSJohn Baldwin switch (crp->crp_buf.cb_type) { 1239c0341432SJohn Baldwin case CRYPTO_BUF_MBUF: 1240883a0196SJohn Baldwin case CRYPTO_BUF_SINGLE_MBUF: 1241c0341432SJohn Baldwin adj = result - crp->crp_payload_length; 12429c0e3d3aSJohn Baldwin m_adj(crp->crp_buf.cb_mbuf, adj); 1243c0341432SJohn Baldwin break; 1244c0341432SJohn Baldwin case CRYPTO_BUF_UIO: { 12459c0e3d3aSJohn Baldwin struct uio *uio = crp->crp_buf.cb_uio; 1246091d81d1SSam Leffler int ind; 1247091d81d1SSam Leffler 1248c0341432SJohn Baldwin adj = crp->crp_payload_length - result; 1249091d81d1SSam Leffler ind = uio->uio_iovcnt - 1; 1250091d81d1SSam Leffler 1251091d81d1SSam Leffler while (adj > 0 && ind >= 0) { 1252091d81d1SSam Leffler if (adj < uio->uio_iov[ind].iov_len) { 1253091d81d1SSam Leffler uio->uio_iov[ind].iov_len -= adj; 1254091d81d1SSam Leffler break; 1255091d81d1SSam Leffler } 1256091d81d1SSam Leffler 1257091d81d1SSam Leffler adj -= uio->uio_iov[ind].iov_len; 1258091d81d1SSam Leffler uio->uio_iov[ind].iov_len = 0; 1259091d81d1SSam Leffler ind--; 1260091d81d1SSam Leffler uio->uio_iovcnt--; 1261091d81d1SSam Leffler } 1262091d81d1SSam Leffler } 1263c0341432SJohn Baldwin break; 1264e6f6d0c9SAlan Somers case CRYPTO_BUF_VMPAGE: 1265e6f6d0c9SAlan Somers adj = crp->crp_payload_length - result; 1266e6f6d0c9SAlan Somers crp->crp_buf.cb_vm_page_len -= adj; 1267e6f6d0c9SAlan Somers break; 12689c0e3d3aSJohn Baldwin default: 12699c0e3d3aSJohn Baldwin break; 1270c0341432SJohn Baldwin } 1271091d81d1SSam Leffler } 12721ede983cSDag-Erling Smørgrav free(out, M_CRYPTO_DATA); 1273091d81d1SSam Leffler return 0; 1274091d81d1SSam Leffler } 1275091d81d1SSam Leffler 1276091d81d1SSam Leffler static int 12773e947048SJohn Baldwin swcr_setup_cipher(struct swcr_session *ses, 1278c0341432SJohn Baldwin const struct crypto_session_params *csp) 1279091d81d1SSam Leffler { 1280c0341432SJohn Baldwin struct swcr_encdec *swe; 1281d8787d4fSMark Johnston const struct enc_xform *txf; 1282f6c4bc3bSPawel Jakub Dawidek int error; 1283091d81d1SSam Leffler 1284c0341432SJohn Baldwin swe = &ses->swcr_encdec; 1285c0341432SJohn Baldwin txf = crypto_cipher(csp); 1286b54d1284SJohn Baldwin if (csp->csp_cipher_key != NULL) { 12873e947048SJohn Baldwin if (txf->ctxsize != 0) { 1288f313909aSJohn Baldwin swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 12893e947048SJohn Baldwin M_NOWAIT); 1290f313909aSJohn Baldwin if (swe->sw_ctx == NULL) 12913e947048SJohn Baldwin return (ENOMEM); 12923e947048SJohn Baldwin } 1293f313909aSJohn Baldwin error = txf->setkey(swe->sw_ctx, 1294c0341432SJohn Baldwin csp->csp_cipher_key, csp->csp_cipher_klen); 1295c0341432SJohn Baldwin if (error) 1296c0341432SJohn Baldwin return (error); 1297091d81d1SSam Leffler } 1298c0341432SJohn Baldwin swe->sw_exf = txf; 1299c0341432SJohn Baldwin return (0); 1300f6c4bc3bSPawel Jakub Dawidek } 1301091d81d1SSam Leffler 1302c0341432SJohn Baldwin static int 1303c0341432SJohn Baldwin swcr_setup_auth(struct swcr_session *ses, 1304c0341432SJohn Baldwin const struct crypto_session_params *csp) 1305c0341432SJohn Baldwin { 1306c0341432SJohn Baldwin struct swcr_auth *swa; 1307d8787d4fSMark Johnston const struct auth_hash *axf; 1308c0341432SJohn Baldwin 1309c0341432SJohn Baldwin swa = &ses->swcr_auth; 1310c0341432SJohn Baldwin 1311c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1312c0341432SJohn Baldwin swa->sw_axf = axf; 1313c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1314c0341432SJohn Baldwin return (EINVAL); 1315c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1316c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1317c0341432SJohn Baldwin else 1318c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 13196113a08bSJohn Baldwin if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 13206113a08bSJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 13216113a08bSJohn Baldwin M_NOWAIT); 1322c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1323c0341432SJohn Baldwin return (ENOBUFS); 13246113a08bSJohn Baldwin } 1325c0341432SJohn Baldwin 1326c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1327091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1328c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1329f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1330f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1331f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1332091d81d1SSam Leffler case CRYPTO_RIPEMD160_HMAC: 13336113a08bSJohn Baldwin swa->sw_hmac = true; 13346113a08bSJohn Baldwin if (csp->csp_auth_key != NULL) { 13353a0b6a93SJohn Baldwin swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1336091d81d1SSam Leffler M_NOWAIT); 1337c0341432SJohn Baldwin if (swa->sw_octx == NULL) 1338c0341432SJohn Baldwin return (ENOBUFS); 13396113a08bSJohn Baldwin hmac_init_ipad(axf, csp->csp_auth_key, 13406113a08bSJohn Baldwin csp->csp_auth_klen, swa->sw_ictx); 13416113a08bSJohn Baldwin hmac_init_opad(axf, csp->csp_auth_key, 13426113a08bSJohn Baldwin csp->csp_auth_klen, swa->sw_octx); 1343091d81d1SSam Leffler } 1344091d81d1SSam Leffler break; 1345c3a688efSJohn Baldwin case CRYPTO_RIPEMD160: 1346091d81d1SSam Leffler case CRYPTO_SHA1: 1347c4729f6eSConrad Meyer case CRYPTO_SHA2_224: 1348c4729f6eSConrad Meyer case CRYPTO_SHA2_256: 1349c4729f6eSConrad Meyer case CRYPTO_SHA2_384: 1350c4729f6eSConrad Meyer case CRYPTO_SHA2_512: 1351aa268175SJohn Baldwin case CRYPTO_NULL_HMAC: 1352c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1353c0341432SJohn Baldwin break; 1354c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 13556113a08bSJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 13566113a08bSJohn Baldwin case CRYPTO_POLY1305: 13576113a08bSJohn Baldwin if (csp->csp_auth_key != NULL) { 1358c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1359c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1360c0341432SJohn Baldwin csp->csp_auth_klen); 13616113a08bSJohn Baldwin } 1362c0341432SJohn Baldwin break; 1363c0341432SJohn Baldwin case CRYPTO_BLAKE2B: 1364c0341432SJohn Baldwin case CRYPTO_BLAKE2S: 1365c0341432SJohn Baldwin /* 1366c0341432SJohn Baldwin * Blake2b and Blake2s support an optional key but do 1367c0341432SJohn Baldwin * not require one. 1368c0341432SJohn Baldwin */ 13696113a08bSJohn Baldwin if (csp->csp_auth_klen == 0) 13706113a08bSJohn Baldwin axf->Init(swa->sw_ictx); 13716113a08bSJohn Baldwin else if (csp->csp_auth_key != NULL) 1372c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1373c0341432SJohn Baldwin csp->csp_auth_klen); 13746113a08bSJohn Baldwin break; 13756113a08bSJohn Baldwin } 13766113a08bSJohn Baldwin 13776113a08bSJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) { 13786113a08bSJohn Baldwin switch (csp->csp_auth_alg) { 13796113a08bSJohn Baldwin case CRYPTO_AES_NIST_GMAC: 13806113a08bSJohn Baldwin ses->swcr_process = swcr_gmac; 1381c0341432SJohn Baldwin break; 1382c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1383c0341432SJohn Baldwin ses->swcr_process = swcr_ccm_cbc_mac; 1384c0341432SJohn Baldwin break; 13856113a08bSJohn Baldwin default: 13866113a08bSJohn Baldwin ses->swcr_process = swcr_authcompute; 13876113a08bSJohn Baldwin } 1388091d81d1SSam Leffler } 1389091d81d1SSam Leffler 1390c0341432SJohn Baldwin return (0); 1391c0341432SJohn Baldwin } 139208fca7a5SJohn-Mark Gurney 1393c0341432SJohn Baldwin static int 1394ab91fb6cSJohn Baldwin swcr_setup_aead(struct swcr_session *ses, 1395c0341432SJohn Baldwin const struct crypto_session_params *csp) 1396c0341432SJohn Baldwin { 1397c0341432SJohn Baldwin struct swcr_auth *swa; 1398ab91fb6cSJohn Baldwin int error; 1399c0341432SJohn Baldwin 1400ab91fb6cSJohn Baldwin error = swcr_setup_cipher(ses, csp); 1401ab91fb6cSJohn Baldwin if (error) 1402ab91fb6cSJohn Baldwin return (error); 1403ab91fb6cSJohn Baldwin 1404c0341432SJohn Baldwin swa = &ses->swcr_auth; 1405c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1406ab91fb6cSJohn Baldwin swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1407c0341432SJohn Baldwin else 1408c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1409ab91fb6cSJohn Baldwin return (0); 1410dd2e1352SJohn Baldwin } 1411dd2e1352SJohn Baldwin 1412c0341432SJohn Baldwin static bool 1413c0341432SJohn Baldwin swcr_auth_supported(const struct crypto_session_params *csp) 1414109919c6SBenno Rice { 1415d8787d4fSMark Johnston const struct auth_hash *axf; 1416091d81d1SSam Leffler 1417c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1418c0341432SJohn Baldwin if (axf == NULL) 1419c0341432SJohn Baldwin return (false); 1420c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1421091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1422c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1423f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1424f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1425f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1426091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 1427c0341432SJohn Baldwin case CRYPTO_RIPEMD160_HMAC: 1428091d81d1SSam Leffler break; 1429c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1430c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1431c0341432SJohn Baldwin case 128: 1432c0341432SJohn Baldwin case 192: 1433c0341432SJohn Baldwin case 256: 1434c0341432SJohn Baldwin break; 1435c0341432SJohn Baldwin default: 1436c0341432SJohn Baldwin return (false); 1437c0341432SJohn Baldwin } 1438c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1439c0341432SJohn Baldwin return (false); 1440c0341432SJohn Baldwin if (csp->csp_ivlen != AES_GCM_IV_LEN) 1441c0341432SJohn Baldwin return (false); 1442c0341432SJohn Baldwin break; 144325b7033bSConrad Meyer case CRYPTO_POLY1305: 1444c0341432SJohn Baldwin if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1445c0341432SJohn Baldwin return (false); 1446c0341432SJohn Baldwin break; 1447c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1448c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1449c0341432SJohn Baldwin case 128: 1450c0341432SJohn Baldwin case 192: 1451c0341432SJohn Baldwin case 256: 1452c0341432SJohn Baldwin break; 1453c0341432SJohn Baldwin default: 1454c0341432SJohn Baldwin return (false); 1455c0341432SJohn Baldwin } 1456c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1457c0341432SJohn Baldwin return (false); 1458c0341432SJohn Baldwin break; 1459c0341432SJohn Baldwin } 1460c0341432SJohn Baldwin return (true); 1461c0341432SJohn Baldwin } 1462091d81d1SSam Leffler 1463c0341432SJohn Baldwin static bool 1464c0341432SJohn Baldwin swcr_cipher_supported(const struct crypto_session_params *csp) 1465c0341432SJohn Baldwin { 1466d8787d4fSMark Johnston const struct enc_xform *txf; 1467c0341432SJohn Baldwin 1468c0341432SJohn Baldwin txf = crypto_cipher(csp); 1469c0341432SJohn Baldwin if (txf == NULL) 1470c0341432SJohn Baldwin return (false); 1471c0341432SJohn Baldwin if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1472c0341432SJohn Baldwin txf->ivsize != csp->csp_ivlen) 1473c0341432SJohn Baldwin return (false); 1474c0341432SJohn Baldwin return (true); 1475c0341432SJohn Baldwin } 1476c0341432SJohn Baldwin 14776038018aSMarcin Wojtas #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 14786038018aSMarcin Wojtas 1479c0341432SJohn Baldwin static int 1480c0341432SJohn Baldwin swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1481c0341432SJohn Baldwin { 14826038018aSMarcin Wojtas if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1483c0341432SJohn Baldwin return (EINVAL); 1484c0341432SJohn Baldwin switch (csp->csp_mode) { 1485c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1486c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1487c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1488c0341432SJohn Baldwin break; 1489c0341432SJohn Baldwin default: 1490c0341432SJohn Baldwin return (EINVAL); 14915fbc5b5aSConrad Meyer } 1492091d81d1SSam Leffler break; 1493c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1494c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1495c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1496c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1497dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 14988f35841fSJohn Baldwin case CRYPTO_XCHACHA20_POLY1305: 1499c0341432SJohn Baldwin return (EINVAL); 1500c0341432SJohn Baldwin default: 1501c0341432SJohn Baldwin if (!swcr_cipher_supported(csp)) 1502c0341432SJohn Baldwin return (EINVAL); 1503091d81d1SSam Leffler break; 1504091d81d1SSam Leffler } 1505c0341432SJohn Baldwin break; 1506c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1507c0341432SJohn Baldwin if (!swcr_auth_supported(csp)) 1508c0341432SJohn Baldwin return (EINVAL); 1509c0341432SJohn Baldwin break; 1510c0341432SJohn Baldwin case CSP_MODE_AEAD: 1511c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1512c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1513c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1514c172a407SJohn Baldwin switch (csp->csp_cipher_klen * 8) { 1515c172a407SJohn Baldwin case 128: 1516c172a407SJohn Baldwin case 192: 1517c172a407SJohn Baldwin case 256: 1518c172a407SJohn Baldwin break; 1519c172a407SJohn Baldwin default: 1520c172a407SJohn Baldwin return (EINVAL); 1521c172a407SJohn Baldwin } 1522c172a407SJohn Baldwin break; 1523dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 15248f35841fSJohn Baldwin case CRYPTO_XCHACHA20_POLY1305: 1525c0341432SJohn Baldwin break; 1526c0341432SJohn Baldwin default: 1527c0341432SJohn Baldwin return (EINVAL); 1528c0341432SJohn Baldwin } 1529c0341432SJohn Baldwin break; 1530c0341432SJohn Baldwin case CSP_MODE_ETA: 1531c0341432SJohn Baldwin /* AEAD algorithms cannot be used for EtA. */ 1532c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1533c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1534c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1535dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 15368f35841fSJohn Baldwin case CRYPTO_XCHACHA20_POLY1305: 1537c0341432SJohn Baldwin return (EINVAL); 1538c0341432SJohn Baldwin } 1539c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1540c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1541c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1542c0341432SJohn Baldwin return (EINVAL); 1543c0341432SJohn Baldwin } 1544c0341432SJohn Baldwin 1545c0341432SJohn Baldwin if (!swcr_cipher_supported(csp) || 1546c0341432SJohn Baldwin !swcr_auth_supported(csp)) 1547c0341432SJohn Baldwin return (EINVAL); 1548c0341432SJohn Baldwin break; 1549c0341432SJohn Baldwin default: 1550c0341432SJohn Baldwin return (EINVAL); 1551c0341432SJohn Baldwin } 1552c0341432SJohn Baldwin 1553c0341432SJohn Baldwin return (CRYPTODEV_PROBE_SOFTWARE); 1554c0341432SJohn Baldwin } 1555c0341432SJohn Baldwin 1556c0341432SJohn Baldwin /* 1557c0341432SJohn Baldwin * Generate a new software session. 1558c0341432SJohn Baldwin */ 1559c0341432SJohn Baldwin static int 1560c0341432SJohn Baldwin swcr_newsession(device_t dev, crypto_session_t cses, 1561c0341432SJohn Baldwin const struct crypto_session_params *csp) 1562c0341432SJohn Baldwin { 1563c0341432SJohn Baldwin struct swcr_session *ses; 1564d8787d4fSMark Johnston const struct comp_algo *cxf; 1565c0341432SJohn Baldwin int error; 1566c0341432SJohn Baldwin 1567c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1568c0341432SJohn Baldwin 1569c0341432SJohn Baldwin error = 0; 1570c0341432SJohn Baldwin switch (csp->csp_mode) { 1571c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1572c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1573c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1574c0341432SJohn Baldwin cxf = &comp_algo_deflate; 1575c0341432SJohn Baldwin break; 1576c0341432SJohn Baldwin #ifdef INVARIANTS 1577c0341432SJohn Baldwin default: 1578c0341432SJohn Baldwin panic("bad compression algo"); 1579c0341432SJohn Baldwin #endif 1580c0341432SJohn Baldwin } 1581c0341432SJohn Baldwin ses->swcr_compdec.sw_cxf = cxf; 1582c0341432SJohn Baldwin ses->swcr_process = swcr_compdec; 1583c0341432SJohn Baldwin break; 1584c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1585c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1586c0341432SJohn Baldwin case CRYPTO_NULL_CBC: 1587c0341432SJohn Baldwin ses->swcr_process = swcr_null; 1588c0341432SJohn Baldwin break; 1589c0341432SJohn Baldwin #ifdef INVARIANTS 1590c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1591c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1592dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 15938f35841fSJohn Baldwin case CRYPTO_XCHACHA20_POLY1305: 1594c0341432SJohn Baldwin panic("bad cipher algo"); 1595c0341432SJohn Baldwin #endif 1596c0341432SJohn Baldwin default: 15973e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1598c0341432SJohn Baldwin if (error == 0) 1599c0341432SJohn Baldwin ses->swcr_process = swcr_encdec; 1600c0341432SJohn Baldwin } 1601c0341432SJohn Baldwin break; 1602c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1603c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1604c0341432SJohn Baldwin break; 1605c0341432SJohn Baldwin case CSP_MODE_AEAD: 1606c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1607c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1608ab91fb6cSJohn Baldwin error = swcr_setup_aead(ses, csp); 1609c0341432SJohn Baldwin if (error == 0) 1610c0341432SJohn Baldwin ses->swcr_process = swcr_gcm; 1611c0341432SJohn Baldwin break; 1612c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1613ab91fb6cSJohn Baldwin error = swcr_setup_aead(ses, csp); 1614c0341432SJohn Baldwin if (error == 0) 1615c0341432SJohn Baldwin ses->swcr_process = swcr_ccm; 1616c0341432SJohn Baldwin break; 1617dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 16188f35841fSJohn Baldwin case CRYPTO_XCHACHA20_POLY1305: 1619ab91fb6cSJohn Baldwin error = swcr_setup_aead(ses, csp); 1620dd2e1352SJohn Baldwin if (error == 0) 1621dd2e1352SJohn Baldwin ses->swcr_process = swcr_chacha20_poly1305; 1622dd2e1352SJohn Baldwin break; 1623c0341432SJohn Baldwin #ifdef INVARIANTS 1624c0341432SJohn Baldwin default: 1625c0341432SJohn Baldwin panic("bad aead algo"); 1626c0341432SJohn Baldwin #endif 1627c0341432SJohn Baldwin } 1628c0341432SJohn Baldwin break; 1629c0341432SJohn Baldwin case CSP_MODE_ETA: 1630c0341432SJohn Baldwin #ifdef INVARIANTS 1631c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1632c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1633c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1634dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 16358f35841fSJohn Baldwin case CRYPTO_XCHACHA20_POLY1305: 1636c0341432SJohn Baldwin panic("bad eta cipher algo"); 1637c0341432SJohn Baldwin } 1638c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1639c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1640c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1641c0341432SJohn Baldwin panic("bad eta auth algo"); 1642c0341432SJohn Baldwin } 1643c0341432SJohn Baldwin #endif 1644c0341432SJohn Baldwin 1645c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1646c0341432SJohn Baldwin if (error) 1647c0341432SJohn Baldwin break; 1648c0341432SJohn Baldwin if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1649c0341432SJohn Baldwin /* Effectively degrade to digest mode. */ 1650c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1651c0341432SJohn Baldwin break; 1652c0341432SJohn Baldwin } 1653c0341432SJohn Baldwin 16543e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1655c0341432SJohn Baldwin if (error == 0) 1656c0341432SJohn Baldwin ses->swcr_process = swcr_eta; 1657c0341432SJohn Baldwin break; 1658c0341432SJohn Baldwin default: 1659c0341432SJohn Baldwin error = EINVAL; 1660c0341432SJohn Baldwin } 1661c0341432SJohn Baldwin 1662c0341432SJohn Baldwin if (error) 1663c0341432SJohn Baldwin swcr_freesession(dev, cses); 1664c0341432SJohn Baldwin return (error); 1665c0341432SJohn Baldwin } 1666c0341432SJohn Baldwin 1667c0341432SJohn Baldwin static void 1668c0341432SJohn Baldwin swcr_freesession(device_t dev, crypto_session_t cses) 1669c0341432SJohn Baldwin { 1670c0341432SJohn Baldwin struct swcr_session *ses; 1671c0341432SJohn Baldwin 1672c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1673c0341432SJohn Baldwin 1674f313909aSJohn Baldwin zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 16754a711b8dSJohn Baldwin zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 16764a711b8dSJohn Baldwin zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1677091d81d1SSam Leffler } 1678091d81d1SSam Leffler 1679091d81d1SSam Leffler /* 1680091d81d1SSam Leffler * Process a software request. 1681091d81d1SSam Leffler */ 1682091d81d1SSam Leffler static int 16836810ad6fSSam Leffler swcr_process(device_t dev, struct cryptop *crp, int hint) 1684091d81d1SSam Leffler { 1685c0341432SJohn Baldwin struct swcr_session *ses; 1686091d81d1SSam Leffler 16871b0909d5SConrad Meyer ses = crypto_get_driver_session(crp->crp_session); 1688091d81d1SSam Leffler 1689c0341432SJohn Baldwin crp->crp_etype = ses->swcr_process(ses, crp); 1690091d81d1SSam Leffler 1691091d81d1SSam Leffler crypto_done(crp); 1692c0341432SJohn Baldwin return (0); 1693091d81d1SSam Leffler } 1694091d81d1SSam Leffler 1695091d81d1SSam Leffler static void 16963f147ab2SWarner Losh swcr_identify(driver_t *drv, device_t parent) 1697091d81d1SSam Leffler { 16986810ad6fSSam Leffler /* NB: order 10 is so we get attached after h/w devices */ 16996810ad6fSSam Leffler if (device_find_child(parent, "cryptosoft", -1) == NULL && 170086c585d9SMarius Strobl BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 17016810ad6fSSam Leffler panic("cryptosoft: could not attach"); 17026810ad6fSSam Leffler } 1703f6c4bc3bSPawel Jakub Dawidek 17046810ad6fSSam Leffler static int 17056810ad6fSSam Leffler swcr_probe(device_t dev) 17066810ad6fSSam Leffler { 17076810ad6fSSam Leffler device_set_desc(dev, "software crypto"); 17084fc60fa9SMark Johnston device_quiet(dev); 170986c585d9SMarius Strobl return (BUS_PROBE_NOWILDCARD); 17106810ad6fSSam Leffler } 1711f6c4bc3bSPawel Jakub Dawidek 17126810ad6fSSam Leffler static int 17136810ad6fSSam Leffler swcr_attach(device_t dev) 17146810ad6fSSam Leffler { 17156810ad6fSSam Leffler 17169ebbebe4SConrad Meyer swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 17176810ad6fSSam Leffler CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 17186810ad6fSSam Leffler if (swcr_id < 0) { 17196810ad6fSSam Leffler device_printf(dev, "cannot initialize!"); 1720c0341432SJohn Baldwin return (ENXIO); 17216810ad6fSSam Leffler } 17226810ad6fSSam Leffler 1723c0341432SJohn Baldwin return (0); 1724091d81d1SSam Leffler } 17254b465da2SPawel Jakub Dawidek 17263f147ab2SWarner Losh static int 17276810ad6fSSam Leffler swcr_detach(device_t dev) 17284b465da2SPawel Jakub Dawidek { 17296810ad6fSSam Leffler crypto_unregister_all(swcr_id); 17303f147ab2SWarner Losh return 0; 17314b465da2SPawel Jakub Dawidek } 17326810ad6fSSam Leffler 17336810ad6fSSam Leffler static device_method_t swcr_methods[] = { 17346810ad6fSSam Leffler DEVMETHOD(device_identify, swcr_identify), 17356810ad6fSSam Leffler DEVMETHOD(device_probe, swcr_probe), 17366810ad6fSSam Leffler DEVMETHOD(device_attach, swcr_attach), 17376810ad6fSSam Leffler DEVMETHOD(device_detach, swcr_detach), 17386810ad6fSSam Leffler 1739c0341432SJohn Baldwin DEVMETHOD(cryptodev_probesession, swcr_probesession), 17406810ad6fSSam Leffler DEVMETHOD(cryptodev_newsession, swcr_newsession), 17416810ad6fSSam Leffler DEVMETHOD(cryptodev_freesession,swcr_freesession), 17426810ad6fSSam Leffler DEVMETHOD(cryptodev_process, swcr_process), 17436810ad6fSSam Leffler 17446810ad6fSSam Leffler {0, 0}, 17456810ad6fSSam Leffler }; 17466810ad6fSSam Leffler 17476810ad6fSSam Leffler static driver_t swcr_driver = { 17486810ad6fSSam Leffler "cryptosoft", 17496810ad6fSSam Leffler swcr_methods, 17506810ad6fSSam Leffler 0, /* NB: no softc */ 17516810ad6fSSam Leffler }; 17526810ad6fSSam Leffler static devclass_t swcr_devclass; 17536810ad6fSSam Leffler 17546810ad6fSSam Leffler /* 17556810ad6fSSam Leffler * NB: We explicitly reference the crypto module so we 17566810ad6fSSam Leffler * get the necessary ordering when built as a loadable 17576810ad6fSSam Leffler * module. This is required because we bundle the crypto 17586810ad6fSSam Leffler * module code together with the cryptosoft driver (otherwise 17596810ad6fSSam Leffler * normal module dependencies would handle things). 17606810ad6fSSam Leffler */ 17616810ad6fSSam Leffler extern int crypto_modevent(struct module *, int, void *); 17626810ad6fSSam Leffler /* XXX where to attach */ 17636810ad6fSSam Leffler DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 17646810ad6fSSam Leffler MODULE_VERSION(cryptosoft, 1); 17656810ad6fSSam Leffler MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1766