1091d81d1SSam Leffler /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2091d81d1SSam Leffler 360727d8bSWarner Losh /*- 4091d81d1SSam Leffler * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 56810ad6fSSam Leffler * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6091d81d1SSam Leffler * 7091d81d1SSam Leffler * This code was written by Angelos D. Keromytis in Athens, Greece, in 8091d81d1SSam Leffler * February 2000. Network Security Technologies Inc. (NSTI) kindly 9091d81d1SSam Leffler * supported the development of this code. 10091d81d1SSam Leffler * 11091d81d1SSam Leffler * Copyright (c) 2000, 2001 Angelos D. Keromytis 121833d604SJohn Baldwin * Copyright (c) 2014-2021 The FreeBSD Foundation 1308fca7a5SJohn-Mark Gurney * All rights reserved. 1408fca7a5SJohn-Mark Gurney * 1508fca7a5SJohn-Mark Gurney * Portions of this software were developed by John-Mark Gurney 1608fca7a5SJohn-Mark Gurney * under sponsorship of the FreeBSD Foundation and 1708fca7a5SJohn-Mark Gurney * Rubicon Communications, LLC (Netgate). 18091d81d1SSam Leffler * 191833d604SJohn Baldwin * Portions of this software were developed by Ararat River 201833d604SJohn Baldwin * Consulting, LLC under sponsorship of the FreeBSD Foundation. 211833d604SJohn Baldwin * 22091d81d1SSam Leffler * Permission to use, copy, and modify this software with or without fee 23091d81d1SSam Leffler * is hereby granted, provided that this entire notice is included in 24091d81d1SSam Leffler * all source code copies of any software which is or includes a copy or 25091d81d1SSam Leffler * modification of this software. 26091d81d1SSam Leffler * 27091d81d1SSam Leffler * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28091d81d1SSam Leffler * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29091d81d1SSam Leffler * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30091d81d1SSam Leffler * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31091d81d1SSam Leffler * PURPOSE. 32091d81d1SSam Leffler */ 33091d81d1SSam Leffler 342c446514SDavid E. O'Brien #include <sys/cdefs.h> 352c446514SDavid E. O'Brien __FBSDID("$FreeBSD$"); 362c446514SDavid E. O'Brien 37091d81d1SSam Leffler #include <sys/param.h> 38091d81d1SSam Leffler #include <sys/systm.h> 39091d81d1SSam Leffler #include <sys/malloc.h> 40091d81d1SSam Leffler #include <sys/mbuf.h> 416810ad6fSSam Leffler #include <sys/module.h> 42091d81d1SSam Leffler #include <sys/sysctl.h> 43091d81d1SSam Leffler #include <sys/errno.h> 44091d81d1SSam Leffler #include <sys/random.h> 45091d81d1SSam Leffler #include <sys/kernel.h> 46091d81d1SSam Leffler #include <sys/uio.h> 47109919c6SBenno Rice #include <sys/lock.h> 48109919c6SBenno Rice #include <sys/rwlock.h> 4908fca7a5SJohn-Mark Gurney #include <sys/endian.h> 5008fca7a5SJohn-Mark Gurney #include <sys/limits.h> 51a7fcb1afSSean Eric Fagan #include <sys/mutex.h> 52091d81d1SSam Leffler 53091d81d1SSam Leffler #include <crypto/sha1.h> 54091d81d1SSam Leffler #include <opencrypto/rmd160.h> 55091d81d1SSam Leffler 56091d81d1SSam Leffler #include <opencrypto/cryptodev.h> 57091d81d1SSam Leffler #include <opencrypto/xform.h> 58091d81d1SSam Leffler 596810ad6fSSam Leffler #include <sys/kobj.h> 606810ad6fSSam Leffler #include <sys/bus.h> 616810ad6fSSam Leffler #include "cryptodev_if.h" 62091d81d1SSam Leffler 63c0341432SJohn Baldwin struct swcr_auth { 64c0341432SJohn Baldwin void *sw_ictx; 65c0341432SJohn Baldwin void *sw_octx; 66d8787d4fSMark Johnston const struct auth_hash *sw_axf; 67c0341432SJohn Baldwin uint16_t sw_mlen; 68c0341432SJohn Baldwin }; 69c0341432SJohn Baldwin 70c0341432SJohn Baldwin struct swcr_encdec { 713e947048SJohn Baldwin void *sw_kschedule; 72d8787d4fSMark Johnston const struct enc_xform *sw_exf; 73c0341432SJohn Baldwin }; 74c0341432SJohn Baldwin 75c0341432SJohn Baldwin struct swcr_compdec { 76d8787d4fSMark Johnston const struct comp_algo *sw_cxf; 77c0341432SJohn Baldwin }; 78c0341432SJohn Baldwin 79c0341432SJohn Baldwin struct swcr_session { 80c0341432SJohn Baldwin struct mtx swcr_lock; 81c0341432SJohn Baldwin int (*swcr_process)(struct swcr_session *, struct cryptop *); 82c0341432SJohn Baldwin 83c0341432SJohn Baldwin struct swcr_auth swcr_auth; 84c0341432SJohn Baldwin struct swcr_encdec swcr_encdec; 85c0341432SJohn Baldwin struct swcr_compdec swcr_compdec; 86c0341432SJohn Baldwin }; 87507281e5SSean Eric Fagan 886810ad6fSSam Leffler static int32_t swcr_id; 896810ad6fSSam Leffler 901b0909d5SConrad Meyer static void swcr_freesession(device_t dev, crypto_session_t cses); 91091d81d1SSam Leffler 92c0341432SJohn Baldwin /* Used for CRYPTO_NULL_CBC. */ 93c0341432SJohn Baldwin static int 94c0341432SJohn Baldwin swcr_null(struct swcr_session *ses, struct cryptop *crp) 95c0341432SJohn Baldwin { 96c0341432SJohn Baldwin 97c0341432SJohn Baldwin return (0); 98c0341432SJohn Baldwin } 99c0341432SJohn Baldwin 100091d81d1SSam Leffler /* 101091d81d1SSam Leffler * Apply a symmetric encryption/decryption algorithm. 102091d81d1SSam Leffler */ 103091d81d1SSam Leffler static int 104c0341432SJohn Baldwin swcr_encdec(struct swcr_session *ses, struct cryptop *crp) 105091d81d1SSam Leffler { 1065d7ae54aSConrad Meyer unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; 10708fca7a5SJohn-Mark Gurney unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; 108c0341432SJohn Baldwin const struct crypto_session_params *csp; 109d8787d4fSMark Johnston const struct enc_xform *exf; 110c0341432SJohn Baldwin struct swcr_encdec *sw; 11186be314dSJohn Baldwin size_t inlen, outlen; 1121833d604SJohn Baldwin int i, blks, resid; 1139c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 11426d292d3SJohn Baldwin const unsigned char *inblk; 11526d292d3SJohn Baldwin unsigned char *outblk; 11608fca7a5SJohn-Mark Gurney int error; 117c0341432SJohn Baldwin bool encrypting; 11808fca7a5SJohn-Mark Gurney 11908fca7a5SJohn-Mark Gurney error = 0; 120091d81d1SSam Leffler 121c0341432SJohn Baldwin sw = &ses->swcr_encdec; 122091d81d1SSam Leffler exf = sw->sw_exf; 1231833d604SJohn Baldwin csp = crypto_get_params(crp->crp_session); 124091d81d1SSam Leffler 125723d8764SJohn Baldwin if (exf->native_blocksize == 0) { 126091d81d1SSam Leffler /* Check for non-padded data */ 127723d8764SJohn Baldwin if ((crp->crp_payload_length % exf->blocksize) != 0) 128723d8764SJohn Baldwin return (EINVAL); 129723d8764SJohn Baldwin 130723d8764SJohn Baldwin blks = exf->blocksize; 131723d8764SJohn Baldwin } else 132723d8764SJohn Baldwin blks = exf->native_blocksize; 133091d81d1SSam Leffler 134c0341432SJohn Baldwin if (exf == &enc_xform_aes_icm && 135c0341432SJohn Baldwin (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 13608fca7a5SJohn-Mark Gurney return (EINVAL); 13708fca7a5SJohn-Mark Gurney 138c0341432SJohn Baldwin if (crp->crp_cipher_key != NULL) { 1393e947048SJohn Baldwin error = exf->setkey(sw->sw_kschedule, 140c0341432SJohn Baldwin crp->crp_cipher_key, csp->csp_cipher_klen); 141c740ae4bSPoul-Henning Kamp if (error) 142c740ae4bSPoul-Henning Kamp return (error); 143c740ae4bSPoul-Henning Kamp } 144d295bdeeSPawel Jakub Dawidek 14520c128daSJohn Baldwin crypto_read_iv(crp, iv); 14620c128daSJohn Baldwin 14708fca7a5SJohn-Mark Gurney if (exf->reinit) { 148d295bdeeSPawel Jakub Dawidek /* 149d295bdeeSPawel Jakub Dawidek * xforms that provide a reinit method perform all IV 150d295bdeeSPawel Jakub Dawidek * handling themselves. 151d295bdeeSPawel Jakub Dawidek */ 1521833d604SJohn Baldwin exf->reinit(sw->sw_kschedule, iv, csp->csp_ivlen); 153091d81d1SSam Leffler } 154091d81d1SSam Leffler 1559c0e3d3aSJohn Baldwin ivp = iv; 156091d81d1SSam Leffler 1579c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 1589c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 15986be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 1609c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1619c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 1629c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1639c0e3d3aSJohn Baldwin } else 1649c0e3d3aSJohn Baldwin cc_out = cc_in; 16586be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 1669c0e3d3aSJohn Baldwin 1679c0e3d3aSJohn Baldwin resid = crp->crp_payload_length; 168c0341432SJohn Baldwin encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 169091d81d1SSam Leffler 170091d81d1SSam Leffler /* 1719c0e3d3aSJohn Baldwin * Loop through encrypting blocks. 'inlen' is the remaining 1729c0e3d3aSJohn Baldwin * length of the current segment in the input buffer. 1739c0e3d3aSJohn Baldwin * 'outlen' is the remaining length of current segment in the 1749c0e3d3aSJohn Baldwin * output buffer. 175091d81d1SSam Leffler */ 1769c0e3d3aSJohn Baldwin while (resid >= blks) { 1779c0e3d3aSJohn Baldwin /* 1789c0e3d3aSJohn Baldwin * If the current block is not contained within the 1799c0e3d3aSJohn Baldwin * current input/output segment, use 'blk' as a local 1809c0e3d3aSJohn Baldwin * buffer. 1819c0e3d3aSJohn Baldwin */ 1829c0e3d3aSJohn Baldwin if (inlen < blks) { 1839c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, blks, blk); 1849c0e3d3aSJohn Baldwin inblk = blk; 185d295bdeeSPawel Jakub Dawidek } 1869c0e3d3aSJohn Baldwin if (outlen < blks) 1879c0e3d3aSJohn Baldwin outblk = blk; 1889c0e3d3aSJohn Baldwin 1899c0e3d3aSJohn Baldwin /* 1909c0e3d3aSJohn Baldwin * Ciphers without a 'reinit' hook are assumed to be 1919c0e3d3aSJohn Baldwin * used in CBC mode where the chaining is done here. 1929c0e3d3aSJohn Baldwin */ 1939c0e3d3aSJohn Baldwin if (exf->reinit != NULL) { 1949c0e3d3aSJohn Baldwin if (encrypting) 1959c0e3d3aSJohn Baldwin exf->encrypt(sw->sw_kschedule, inblk, outblk); 1969c0e3d3aSJohn Baldwin else 1979c0e3d3aSJohn Baldwin exf->decrypt(sw->sw_kschedule, inblk, outblk); 198c0341432SJohn Baldwin } else if (encrypting) { 199091d81d1SSam Leffler /* XOR with previous block */ 2009c0e3d3aSJohn Baldwin for (i = 0; i < blks; i++) 2019c0e3d3aSJohn Baldwin outblk[i] = inblk[i] ^ ivp[i]; 202091d81d1SSam Leffler 2039c0e3d3aSJohn Baldwin exf->encrypt(sw->sw_kschedule, outblk, outblk); 204091d81d1SSam Leffler 205091d81d1SSam Leffler /* 206091d81d1SSam Leffler * Keep encrypted block for XOR'ing 207091d81d1SSam Leffler * with next block 208091d81d1SSam Leffler */ 2099c0e3d3aSJohn Baldwin memcpy(iv, outblk, blks); 210091d81d1SSam Leffler ivp = iv; 211091d81d1SSam Leffler } else { /* decrypt */ 212091d81d1SSam Leffler /* 213091d81d1SSam Leffler * Keep encrypted block for XOR'ing 214091d81d1SSam Leffler * with next block 215091d81d1SSam Leffler */ 21608fca7a5SJohn-Mark Gurney nivp = (ivp == iv) ? iv2 : iv; 2179c0e3d3aSJohn Baldwin memcpy(nivp, inblk, blks); 218091d81d1SSam Leffler 2199c0e3d3aSJohn Baldwin exf->decrypt(sw->sw_kschedule, inblk, outblk); 220091d81d1SSam Leffler 221091d81d1SSam Leffler /* XOR with previous block */ 2229c0e3d3aSJohn Baldwin for (i = 0; i < blks; i++) 2239c0e3d3aSJohn Baldwin outblk[i] ^= ivp[i]; 224091d81d1SSam Leffler 22508fca7a5SJohn-Mark Gurney ivp = nivp; 226091d81d1SSam Leffler } 227091d81d1SSam Leffler 2289c0e3d3aSJohn Baldwin if (inlen < blks) { 22986be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 2309c0e3d3aSJohn Baldwin } else { 2319c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, blks); 2329c0e3d3aSJohn Baldwin inlen -= blks; 2339c0e3d3aSJohn Baldwin inblk += blks; 23408fca7a5SJohn-Mark Gurney } 235091d81d1SSam Leffler 2369c0e3d3aSJohn Baldwin if (outlen < blks) { 2379c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, blks, blk); 23886be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 2399c0e3d3aSJohn Baldwin } else { 2409c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, blks); 2419c0e3d3aSJohn Baldwin outlen -= blks; 2429c0e3d3aSJohn Baldwin outblk += blks; 243091d81d1SSam Leffler } 244091d81d1SSam Leffler 2459c0e3d3aSJohn Baldwin resid -= blks; 246f34a967bSPawel Jakub Dawidek } 247f34a967bSPawel Jakub Dawidek 248723d8764SJohn Baldwin /* Handle trailing partial block for stream ciphers. */ 2499c0e3d3aSJohn Baldwin if (resid > 0) { 250723d8764SJohn Baldwin KASSERT(exf->native_blocksize != 0, 251723d8764SJohn Baldwin ("%s: partial block of %d bytes for cipher %s", 252723d8764SJohn Baldwin __func__, i, exf->name)); 253723d8764SJohn Baldwin KASSERT(exf->reinit != NULL, 254723d8764SJohn Baldwin ("%s: partial block cipher %s without reinit hook", 255723d8764SJohn Baldwin __func__, exf->name)); 2569c0e3d3aSJohn Baldwin KASSERT(resid < blks, ("%s: partial block too big", __func__)); 257723d8764SJohn Baldwin 25886be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &inlen); 25986be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &outlen); 2609c0e3d3aSJohn Baldwin if (inlen < resid) { 2619c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 2629c0e3d3aSJohn Baldwin inblk = blk; 26386be314dSJohn Baldwin } 2649c0e3d3aSJohn Baldwin if (outlen < resid) 2659c0e3d3aSJohn Baldwin outblk = blk; 2669c0e3d3aSJohn Baldwin if (encrypting) 2679c0e3d3aSJohn Baldwin exf->encrypt_last(sw->sw_kschedule, inblk, outblk, 2689c0e3d3aSJohn Baldwin resid); 2699c0e3d3aSJohn Baldwin else 2709c0e3d3aSJohn Baldwin exf->decrypt_last(sw->sw_kschedule, inblk, outblk, 2719c0e3d3aSJohn Baldwin resid); 2729c0e3d3aSJohn Baldwin if (outlen < resid) 2739c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 274723d8764SJohn Baldwin } 275723d8764SJohn Baldwin 27620c128daSJohn Baldwin explicit_bzero(blk, sizeof(blk)); 27720c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 27820c128daSJohn Baldwin explicit_bzero(iv2, sizeof(iv2)); 2799c0e3d3aSJohn Baldwin return (0); 280091d81d1SSam Leffler } 281091d81d1SSam Leffler 282c0341432SJohn Baldwin static void 283d8787d4fSMark Johnston swcr_authprepare(const struct auth_hash *axf, struct swcr_auth *sw, 284c0341432SJohn Baldwin const uint8_t *key, int klen) 285f6c4bc3bSPawel Jakub Dawidek { 286f6c4bc3bSPawel Jakub Dawidek 287f6c4bc3bSPawel Jakub Dawidek switch (axf->type) { 288f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA1_HMAC: 289c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 290f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 291f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 292f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 293f6c4bc3bSPawel Jakub Dawidek case CRYPTO_NULL_HMAC: 294f6c4bc3bSPawel Jakub Dawidek case CRYPTO_RIPEMD160_HMAC: 295c0341432SJohn Baldwin hmac_init_ipad(axf, key, klen, sw->sw_ictx); 296c0341432SJohn Baldwin hmac_init_opad(axf, key, klen, sw->sw_octx); 297f6c4bc3bSPawel Jakub Dawidek break; 29825b7033bSConrad Meyer case CRYPTO_POLY1305: 2990e33efe4SConrad Meyer case CRYPTO_BLAKE2B: 3000e33efe4SConrad Meyer case CRYPTO_BLAKE2S: 3010e33efe4SConrad Meyer axf->Setkey(sw->sw_ictx, key, klen); 3020e33efe4SConrad Meyer axf->Init(sw->sw_ictx); 3030e33efe4SConrad Meyer break; 304f6c4bc3bSPawel Jakub Dawidek default: 305c0341432SJohn Baldwin panic("%s: algorithm %d doesn't use keys", __func__, axf->type); 306f6c4bc3bSPawel Jakub Dawidek } 307f6c4bc3bSPawel Jakub Dawidek } 308f6c4bc3bSPawel Jakub Dawidek 309091d81d1SSam Leffler /* 310c0341432SJohn Baldwin * Compute or verify hash. 311091d81d1SSam Leffler */ 312091d81d1SSam Leffler static int 313c0341432SJohn Baldwin swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) 314091d81d1SSam Leffler { 315c0341432SJohn Baldwin u_char aalg[HASH_MAX_LEN]; 316c0341432SJohn Baldwin const struct crypto_session_params *csp; 317c0341432SJohn Baldwin struct swcr_auth *sw; 318d8787d4fSMark Johnston const struct auth_hash *axf; 319091d81d1SSam Leffler union authctx ctx; 320091d81d1SSam Leffler int err; 321091d81d1SSam Leffler 322c0341432SJohn Baldwin sw = &ses->swcr_auth; 323091d81d1SSam Leffler 324091d81d1SSam Leffler axf = sw->sw_axf; 325091d81d1SSam Leffler 326c0341432SJohn Baldwin csp = crypto_get_params(crp->crp_session); 3276038018aSMarcin Wojtas if (crp->crp_auth_key != NULL) { 328c0341432SJohn Baldwin swcr_authprepare(axf, sw, crp->crp_auth_key, 329c0341432SJohn Baldwin csp->csp_auth_klen); 33025b7033bSConrad Meyer } 331f6c4bc3bSPawel Jakub Dawidek 332091d81d1SSam Leffler bcopy(sw->sw_ictx, &ctx, axf->ctxsize); 333091d81d1SSam Leffler 3349b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 3359b774dc0SJohn Baldwin err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 3369b774dc0SJohn Baldwin else 337c0341432SJohn Baldwin err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 3389b6b2f86SJohn Baldwin axf->Update, &ctx); 339091d81d1SSam Leffler if (err) 340e0b155feSJohn Baldwin goto out; 341091d81d1SSam Leffler 3429c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 3439c0e3d3aSJohn Baldwin CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 3449c0e3d3aSJohn Baldwin err = crypto_apply_buf(&crp->crp_obuf, 3459c0e3d3aSJohn Baldwin crp->crp_payload_output_start, crp->crp_payload_length, 3469b6b2f86SJohn Baldwin axf->Update, &ctx); 3479c0e3d3aSJohn Baldwin else 3489c0e3d3aSJohn Baldwin err = crypto_apply(crp, crp->crp_payload_start, 3499b6b2f86SJohn Baldwin crp->crp_payload_length, axf->Update, &ctx); 350c0341432SJohn Baldwin if (err) 351e0b155feSJohn Baldwin goto out; 352091d81d1SSam Leffler 3536038018aSMarcin Wojtas if (csp->csp_flags & CSP_F_ESN) 3546038018aSMarcin Wojtas axf->Update(&ctx, crp->crp_esn, 4); 3556038018aSMarcin Wojtas 356091d81d1SSam Leffler axf->Final(aalg, &ctx); 357e0b155feSJohn Baldwin if (sw->sw_octx != NULL) { 358091d81d1SSam Leffler bcopy(sw->sw_octx, &ctx, axf->ctxsize); 359091d81d1SSam Leffler axf->Update(&ctx, aalg, axf->hashsize); 360091d81d1SSam Leffler axf->Final(aalg, &ctx); 361091d81d1SSam Leffler } 362091d81d1SSam Leffler 363c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 36420c128daSJohn Baldwin u_char uaalg[HASH_MAX_LEN]; 36520c128daSJohn Baldwin 366c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 367c0341432SJohn Baldwin if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 36820c128daSJohn Baldwin err = EBADMSG; 36920c128daSJohn Baldwin explicit_bzero(uaalg, sizeof(uaalg)); 370c0341432SJohn Baldwin } else { 371091d81d1SSam Leffler /* Inject the authentication data */ 372c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 373c0341432SJohn Baldwin } 37420c128daSJohn Baldwin explicit_bzero(aalg, sizeof(aalg)); 375e0b155feSJohn Baldwin out: 376e0b155feSJohn Baldwin explicit_bzero(&ctx, sizeof(ctx)); 37720c128daSJohn Baldwin return (err); 378091d81d1SSam Leffler } 379091d81d1SSam Leffler 38008fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 38108fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 38208fca7a5SJohn-Mark Gurney 38308fca7a5SJohn-Mark Gurney static int 384c0341432SJohn Baldwin swcr_gmac(struct swcr_session *ses, struct cryptop *crp) 38508fca7a5SJohn-Mark Gurney { 38626d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 38708fca7a5SJohn-Mark Gurney u_char *blk = (u_char *)blkbuf; 38826d292d3SJohn Baldwin u_char tag[GMAC_DIGEST_LEN]; 38926d292d3SJohn Baldwin u_char iv[AES_BLOCK_LEN]; 3909c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc; 39126d292d3SJohn Baldwin const u_char *inblk; 39208fca7a5SJohn-Mark Gurney union authctx ctx; 393c0341432SJohn Baldwin struct swcr_auth *swa; 394d8787d4fSMark Johnston const struct auth_hash *axf; 39508fca7a5SJohn-Mark Gurney uint32_t *blkp; 39686be314dSJohn Baldwin size_t len; 39786be314dSJohn Baldwin int blksz, error, ivlen, resid; 39808fca7a5SJohn-Mark Gurney 399c0341432SJohn Baldwin swa = &ses->swcr_auth; 40008fca7a5SJohn-Mark Gurney axf = swa->sw_axf; 401c0341432SJohn Baldwin 40208fca7a5SJohn-Mark Gurney bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 40326d292d3SJohn Baldwin blksz = GMAC_BLOCK_LEN; 40426d292d3SJohn Baldwin KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 40526d292d3SJohn Baldwin __func__)); 40608fca7a5SJohn-Mark Gurney 40708fca7a5SJohn-Mark Gurney /* Initialize the IV */ 408c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 40929fe41ddSJohn Baldwin crypto_read_iv(crp, iv); 41008fca7a5SJohn-Mark Gurney 41108fca7a5SJohn-Mark Gurney axf->Reinit(&ctx, iv, ivlen); 4129c0e3d3aSJohn Baldwin crypto_cursor_init(&cc, &crp->crp_buf); 4139c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc, crp->crp_payload_start); 41426d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 41586be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc, &len); 41626d292d3SJohn Baldwin if (len >= blksz) { 41726d292d3SJohn Baldwin len = rounddown(MIN(len, resid), blksz); 41826d292d3SJohn Baldwin crypto_cursor_advance(&cc, len); 41926d292d3SJohn Baldwin } else { 42026d292d3SJohn Baldwin len = blksz; 4219c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc, len, blk); 42226d292d3SJohn Baldwin inblk = blk; 42326d292d3SJohn Baldwin } 42426d292d3SJohn Baldwin axf->Update(&ctx, inblk, len); 42526d292d3SJohn Baldwin } 42626d292d3SJohn Baldwin if (resid > 0) { 42726d292d3SJohn Baldwin memset(blk, 0, blksz); 42826d292d3SJohn Baldwin crypto_cursor_copydata(&cc, resid, blk); 42908fca7a5SJohn-Mark Gurney axf->Update(&ctx, blk, blksz); 43008fca7a5SJohn-Mark Gurney } 43108fca7a5SJohn-Mark Gurney 43208fca7a5SJohn-Mark Gurney /* length block */ 43326d292d3SJohn Baldwin memset(blk, 0, blksz); 43408fca7a5SJohn-Mark Gurney blkp = (uint32_t *)blk + 1; 435c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 43608fca7a5SJohn-Mark Gurney axf->Update(&ctx, blk, blksz); 437c0341432SJohn Baldwin 438c0341432SJohn Baldwin /* Finalize MAC */ 43926d292d3SJohn Baldwin axf->Final(tag, &ctx); 440c0341432SJohn Baldwin 44120c128daSJohn Baldwin error = 0; 442c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 44326d292d3SJohn Baldwin u_char tag2[GMAC_DIGEST_LEN]; 44420c128daSJohn Baldwin 445c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 44626d292d3SJohn Baldwin tag2); 44726d292d3SJohn Baldwin if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 44820c128daSJohn Baldwin error = EBADMSG; 44926d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 450c0341432SJohn Baldwin } else { 451c0341432SJohn Baldwin /* Inject the authentication data */ 45226d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 453c0341432SJohn Baldwin } 45420c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 45526d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 45620c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 45720c128daSJohn Baldwin return (error); 458c0341432SJohn Baldwin } 459c0341432SJohn Baldwin 460c0341432SJohn Baldwin static int 461c0341432SJohn Baldwin swcr_gcm(struct swcr_session *ses, struct cryptop *crp) 462c0341432SJohn Baldwin { 46326d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 464c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 46526d292d3SJohn Baldwin u_char tag[GMAC_DIGEST_LEN]; 4669c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 46726d292d3SJohn Baldwin const u_char *inblk; 46826d292d3SJohn Baldwin u_char *outblk; 469c0341432SJohn Baldwin union authctx ctx; 470c0341432SJohn Baldwin struct swcr_auth *swa; 471c0341432SJohn Baldwin struct swcr_encdec *swe; 472d8787d4fSMark Johnston const struct auth_hash *axf; 473d8787d4fSMark Johnston const struct enc_xform *exf; 474c0341432SJohn Baldwin uint32_t *blkp; 47586be314dSJohn Baldwin size_t len; 47686be314dSJohn Baldwin int blksz, error, ivlen, r, resid; 477c0341432SJohn Baldwin 478c0341432SJohn Baldwin swa = &ses->swcr_auth; 479c0341432SJohn Baldwin axf = swa->sw_axf; 480c0341432SJohn Baldwin 481c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 48226d292d3SJohn Baldwin blksz = GMAC_BLOCK_LEN; 48326d292d3SJohn Baldwin KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 48426d292d3SJohn Baldwin __func__)); 485c0341432SJohn Baldwin 486c0341432SJohn Baldwin swe = &ses->swcr_encdec; 487c0341432SJohn Baldwin exf = swe->sw_exf; 488723d8764SJohn Baldwin KASSERT(axf->blocksize == exf->native_blocksize, 489723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 490c0341432SJohn Baldwin 491c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 492c0341432SJohn Baldwin return (EINVAL); 493c0341432SJohn Baldwin 494c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 495c0341432SJohn Baldwin 496c0341432SJohn Baldwin /* Supply MAC with IV */ 4975ae5ed5bSJohn Baldwin axf->Reinit(&ctx, crp->crp_iv, ivlen); 498c0341432SJohn Baldwin 499c0341432SJohn Baldwin /* Supply MAC with AAD */ 5009b774dc0SJohn Baldwin if (crp->crp_aad != NULL) { 5019b774dc0SJohn Baldwin len = rounddown(crp->crp_aad_length, blksz); 5029b774dc0SJohn Baldwin if (len != 0) 5039b774dc0SJohn Baldwin axf->Update(&ctx, crp->crp_aad, len); 5049b774dc0SJohn Baldwin if (crp->crp_aad_length != len) { 5059b774dc0SJohn Baldwin memset(blk, 0, blksz); 5069b774dc0SJohn Baldwin memcpy(blk, (char *)crp->crp_aad + len, 5079b774dc0SJohn Baldwin crp->crp_aad_length - len); 5089b774dc0SJohn Baldwin axf->Update(&ctx, blk, blksz); 5099b774dc0SJohn Baldwin } 5109b774dc0SJohn Baldwin } else { 5119c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5129c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_aad_start); 5139b774dc0SJohn Baldwin for (resid = crp->crp_aad_length; resid >= blksz; 5149b774dc0SJohn Baldwin resid -= len) { 51586be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 51626d292d3SJohn Baldwin if (len >= blksz) { 51726d292d3SJohn Baldwin len = rounddown(MIN(len, resid), blksz); 51826d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, len); 51926d292d3SJohn Baldwin } else { 52026d292d3SJohn Baldwin len = blksz; 5219c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 52226d292d3SJohn Baldwin inblk = blk; 52326d292d3SJohn Baldwin } 52426d292d3SJohn Baldwin axf->Update(&ctx, inblk, len); 52526d292d3SJohn Baldwin } 52626d292d3SJohn Baldwin if (resid > 0) { 52726d292d3SJohn Baldwin memset(blk, 0, blksz); 52826d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 529c0341432SJohn Baldwin axf->Update(&ctx, blk, blksz); 530c0341432SJohn Baldwin } 5319b774dc0SJohn Baldwin } 532c0341432SJohn Baldwin 533a10020cfSJohn Baldwin if (crp->crp_cipher_key != NULL) 534a10020cfSJohn Baldwin exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 535a10020cfSJohn Baldwin crypto_get_params(crp->crp_session)->csp_cipher_klen); 5365ae5ed5bSJohn Baldwin exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); 537c0341432SJohn Baldwin 538c0341432SJohn Baldwin /* Do encryption with MAC */ 5399c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5409c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 5419c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 5429c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 5439c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 5449c0e3d3aSJohn Baldwin } else 5459c0e3d3aSJohn Baldwin cc_out = cc_in; 54626d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 54786be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 54886be314dSJohn Baldwin if (len < blksz) { 54926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 55026d292d3SJohn Baldwin inblk = blk; 551c0341432SJohn Baldwin } else { 55226d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 553c0341432SJohn Baldwin } 55426d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 55586be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 55686be314dSJohn Baldwin if (len < blksz) 55726d292d3SJohn Baldwin outblk = blk; 55826d292d3SJohn Baldwin exf->encrypt(swe->sw_kschedule, inblk, outblk); 55926d292d3SJohn Baldwin axf->Update(&ctx, outblk, blksz); 56026d292d3SJohn Baldwin if (outblk == blk) 56126d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 56226d292d3SJohn Baldwin else 56326d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 56426d292d3SJohn Baldwin } else { 56526d292d3SJohn Baldwin axf->Update(&ctx, inblk, blksz); 56626d292d3SJohn Baldwin } 56726d292d3SJohn Baldwin } 56826d292d3SJohn Baldwin if (resid > 0) { 56926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 57026d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 57126d292d3SJohn Baldwin exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 57226d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 57326d292d3SJohn Baldwin } 57426d292d3SJohn Baldwin axf->Update(&ctx, blk, resid); 575c0341432SJohn Baldwin } 576c0341432SJohn Baldwin 577c0341432SJohn Baldwin /* length block */ 57826d292d3SJohn Baldwin memset(blk, 0, blksz); 579c0341432SJohn Baldwin blkp = (uint32_t *)blk + 1; 580c0341432SJohn Baldwin *blkp = htobe32(crp->crp_aad_length * 8); 581c0341432SJohn Baldwin blkp = (uint32_t *)blk + 3; 582c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 583c0341432SJohn Baldwin axf->Update(&ctx, blk, blksz); 584c0341432SJohn Baldwin 585c0341432SJohn Baldwin /* Finalize MAC */ 58626d292d3SJohn Baldwin axf->Final(tag, &ctx); 587c0341432SJohn Baldwin 588c0341432SJohn Baldwin /* Validate tag */ 58920c128daSJohn Baldwin error = 0; 590c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 59126d292d3SJohn Baldwin u_char tag2[GMAC_DIGEST_LEN]; 59220c128daSJohn Baldwin 59326d292d3SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 594c0341432SJohn Baldwin 59526d292d3SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 59626d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 59720c128daSJohn Baldwin if (r != 0) { 59820c128daSJohn Baldwin error = EBADMSG; 59920c128daSJohn Baldwin goto out; 60020c128daSJohn Baldwin } 601c0341432SJohn Baldwin 602c0341432SJohn Baldwin /* tag matches, decrypt data */ 6039c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 6049c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 60526d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 60626d292d3SJohn Baldwin resid -= blksz) { 60786be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 60886be314dSJohn Baldwin if (len < blksz) { 60926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 61026d292d3SJohn Baldwin inblk = blk; 61186be314dSJohn Baldwin } else 61226d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 61386be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 61486be314dSJohn Baldwin if (len < blksz) 61526d292d3SJohn Baldwin outblk = blk; 61626d292d3SJohn Baldwin exf->decrypt(swe->sw_kschedule, inblk, outblk); 61726d292d3SJohn Baldwin if (outblk == blk) 61826d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 61926d292d3SJohn Baldwin else 62026d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 62126d292d3SJohn Baldwin } 62226d292d3SJohn Baldwin if (resid > 0) { 62326d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 62426d292d3SJohn Baldwin exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 62526d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 626c0341432SJohn Baldwin } 627c0341432SJohn Baldwin } else { 628c0341432SJohn Baldwin /* Inject the authentication data */ 62926d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 630c0341432SJohn Baldwin } 631c0341432SJohn Baldwin 63220c128daSJohn Baldwin out: 63320c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 63426d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 63520c128daSJohn Baldwin 63620c128daSJohn Baldwin return (error); 637c0341432SJohn Baldwin } 638c0341432SJohn Baldwin 639c0341432SJohn Baldwin static int 640c0341432SJohn Baldwin swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) 641c0341432SJohn Baldwin { 64226d292d3SJohn Baldwin u_char tag[AES_CBC_MAC_HASH_LEN]; 64326d292d3SJohn Baldwin u_char iv[AES_BLOCK_LEN]; 644c0341432SJohn Baldwin union authctx ctx; 645ae18720dSJohn Baldwin const struct crypto_session_params *csp; 646c0341432SJohn Baldwin struct swcr_auth *swa; 647d8787d4fSMark Johnston const struct auth_hash *axf; 64826d292d3SJohn Baldwin int error, ivlen; 649c0341432SJohn Baldwin 650ae18720dSJohn Baldwin csp = crypto_get_params(crp->crp_session); 651c0341432SJohn Baldwin swa = &ses->swcr_auth; 652c0341432SJohn Baldwin axf = swa->sw_axf; 653c0341432SJohn Baldwin 654c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 655c0341432SJohn Baldwin 656c0341432SJohn Baldwin /* Initialize the IV */ 657ae18720dSJohn Baldwin ivlen = csp->csp_ivlen; 65829fe41ddSJohn Baldwin crypto_read_iv(crp, iv); 659c0341432SJohn Baldwin 660c0341432SJohn Baldwin /* 661c0341432SJohn Baldwin * AES CCM-CBC-MAC needs to know the length of both the auth 662c0341432SJohn Baldwin * data and payload data before doing the auth computation. 663c0341432SJohn Baldwin */ 664c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_payload_length; 665c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.cryptDataLength = 0; 666c0341432SJohn Baldwin 667c0341432SJohn Baldwin axf->Reinit(&ctx, iv, ivlen); 6689b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 6699b774dc0SJohn Baldwin error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 6709b774dc0SJohn Baldwin else 67126d292d3SJohn Baldwin error = crypto_apply(crp, crp->crp_payload_start, 67226d292d3SJohn Baldwin crp->crp_payload_length, axf->Update, &ctx); 67326d292d3SJohn Baldwin if (error) 67426d292d3SJohn Baldwin return (error); 675c0341432SJohn Baldwin 676c0341432SJohn Baldwin /* Finalize MAC */ 67726d292d3SJohn Baldwin axf->Final(tag, &ctx); 678c0341432SJohn Baldwin 679c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 68026d292d3SJohn Baldwin u_char tag2[AES_CBC_MAC_HASH_LEN]; 68120c128daSJohn Baldwin 682c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 68326d292d3SJohn Baldwin tag2); 68426d292d3SJohn Baldwin if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 68520c128daSJohn Baldwin error = EBADMSG; 68626d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag)); 687c0341432SJohn Baldwin } else { 688c0341432SJohn Baldwin /* Inject the authentication data */ 68926d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 690c0341432SJohn Baldwin } 69126d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 69220c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 69320c128daSJohn Baldwin return (error); 694c0341432SJohn Baldwin } 695c0341432SJohn Baldwin 696c0341432SJohn Baldwin static int 697c0341432SJohn Baldwin swcr_ccm(struct swcr_session *ses, struct cryptop *crp) 698c0341432SJohn Baldwin { 699ae18720dSJohn Baldwin const struct crypto_session_params *csp; 70026d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 701c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 70226d292d3SJohn Baldwin u_char tag[AES_CBC_MAC_HASH_LEN]; 7039c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 70426d292d3SJohn Baldwin const u_char *inblk; 70526d292d3SJohn Baldwin u_char *outblk; 706c0341432SJohn Baldwin union authctx ctx; 707c0341432SJohn Baldwin struct swcr_auth *swa; 708c0341432SJohn Baldwin struct swcr_encdec *swe; 709d8787d4fSMark Johnston const struct auth_hash *axf; 710d8787d4fSMark Johnston const struct enc_xform *exf; 71186be314dSJohn Baldwin size_t len; 71226d292d3SJohn Baldwin int blksz, error, ivlen, r, resid; 713c0341432SJohn Baldwin 714ae18720dSJohn Baldwin csp = crypto_get_params(crp->crp_session); 715c0341432SJohn Baldwin swa = &ses->swcr_auth; 716c0341432SJohn Baldwin axf = swa->sw_axf; 717c0341432SJohn Baldwin 718c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 71926d292d3SJohn Baldwin blksz = AES_BLOCK_LEN; 72026d292d3SJohn Baldwin KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 72126d292d3SJohn Baldwin __func__)); 722c0341432SJohn Baldwin 723c0341432SJohn Baldwin swe = &ses->swcr_encdec; 724c0341432SJohn Baldwin exf = swe->sw_exf; 725723d8764SJohn Baldwin KASSERT(axf->blocksize == exf->native_blocksize, 726723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 727c0341432SJohn Baldwin 728ae18720dSJohn Baldwin if (crp->crp_payload_length > ccm_max_payload_length(csp)) 729ae18720dSJohn Baldwin return (EMSGSIZE); 730ae18720dSJohn Baldwin 731c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 732c0341432SJohn Baldwin return (EINVAL); 733c0341432SJohn Baldwin 734ae18720dSJohn Baldwin ivlen = csp->csp_ivlen; 735c0341432SJohn Baldwin 736c0341432SJohn Baldwin /* 737c0341432SJohn Baldwin * AES CCM-CBC-MAC needs to know the length of both the auth 738c0341432SJohn Baldwin * data and payload data before doing the auth computation. 739c0341432SJohn Baldwin */ 740c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; 741c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; 742c0341432SJohn Baldwin 743c0341432SJohn Baldwin /* Supply MAC with IV */ 7445ae5ed5bSJohn Baldwin axf->Reinit(&ctx, crp->crp_iv, ivlen); 745c0341432SJohn Baldwin 746c0341432SJohn Baldwin /* Supply MAC with AAD */ 7479b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 7489b774dc0SJohn Baldwin error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 7499b774dc0SJohn Baldwin else 7509b774dc0SJohn Baldwin error = crypto_apply(crp, crp->crp_aad_start, 7519b774dc0SJohn Baldwin crp->crp_aad_length, axf->Update, &ctx); 75226d292d3SJohn Baldwin if (error) 75326d292d3SJohn Baldwin return (error); 754c0341432SJohn Baldwin 755a10020cfSJohn Baldwin if (crp->crp_cipher_key != NULL) 756a10020cfSJohn Baldwin exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 757a10020cfSJohn Baldwin crypto_get_params(crp->crp_session)->csp_cipher_klen); 7585ae5ed5bSJohn Baldwin exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); 759c0341432SJohn Baldwin 760c0341432SJohn Baldwin /* Do encryption/decryption with MAC */ 7619c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 7629c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 7639c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 7649c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 7659c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 7669c0e3d3aSJohn Baldwin } else 7679c0e3d3aSJohn Baldwin cc_out = cc_in; 76826d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 76986be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 77086be314dSJohn Baldwin if (len < blksz) { 77126d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 77226d292d3SJohn Baldwin inblk = blk; 77386be314dSJohn Baldwin } else 77426d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 775c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 77686be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 77786be314dSJohn Baldwin if (len < blksz) 77826d292d3SJohn Baldwin outblk = blk; 77926d292d3SJohn Baldwin axf->Update(&ctx, inblk, blksz); 78026d292d3SJohn Baldwin exf->encrypt(swe->sw_kschedule, inblk, outblk); 78126d292d3SJohn Baldwin if (outblk == blk) 78226d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 78326d292d3SJohn Baldwin else 78426d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 785c0341432SJohn Baldwin } else { 786c0341432SJohn Baldwin /* 787c0341432SJohn Baldwin * One of the problems with CCM+CBC is that 788c0341432SJohn Baldwin * the authentication is done on the 78926d292d3SJohn Baldwin * unencrypted data. As a result, we have to 790c0341432SJohn Baldwin * decrypt the data twice: once to generate 791c0341432SJohn Baldwin * the tag and a second time after the tag is 792c0341432SJohn Baldwin * verified. 793c0341432SJohn Baldwin */ 79426d292d3SJohn Baldwin exf->decrypt(swe->sw_kschedule, inblk, blk); 79526d292d3SJohn Baldwin axf->Update(&ctx, blk, blksz); 79626d292d3SJohn Baldwin } 79726d292d3SJohn Baldwin } 79826d292d3SJohn Baldwin if (resid > 0) { 79926d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 80026d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 80126d292d3SJohn Baldwin axf->Update(&ctx, blk, resid); 80226d292d3SJohn Baldwin exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 80326d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 80426d292d3SJohn Baldwin } else { 80526d292d3SJohn Baldwin exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 80626d292d3SJohn Baldwin axf->Update(&ctx, blk, resid); 807c0341432SJohn Baldwin } 80808fca7a5SJohn-Mark Gurney } 80908fca7a5SJohn-Mark Gurney 81008fca7a5SJohn-Mark Gurney /* Finalize MAC */ 81126d292d3SJohn Baldwin axf->Final(tag, &ctx); 81208fca7a5SJohn-Mark Gurney 81308fca7a5SJohn-Mark Gurney /* Validate tag */ 81420c128daSJohn Baldwin error = 0; 815c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 81626d292d3SJohn Baldwin u_char tag2[AES_CBC_MAC_HASH_LEN]; 81720c128daSJohn Baldwin 818c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 81926d292d3SJohn Baldwin tag2); 82008fca7a5SJohn-Mark Gurney 82126d292d3SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 82226d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 82320c128daSJohn Baldwin if (r != 0) { 82420c128daSJohn Baldwin error = EBADMSG; 82520c128daSJohn Baldwin goto out; 82620c128daSJohn Baldwin } 827c0341432SJohn Baldwin 82808fca7a5SJohn-Mark Gurney /* tag matches, decrypt data */ 8295ae5ed5bSJohn Baldwin exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); 8309c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 8319c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 83226d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 83326d292d3SJohn Baldwin resid -= blksz) { 83486be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 83586be314dSJohn Baldwin if (len < blksz) { 83626d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 83726d292d3SJohn Baldwin inblk = blk; 83886be314dSJohn Baldwin } else 83926d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 84086be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 84186be314dSJohn Baldwin if (len < blksz) 84226d292d3SJohn Baldwin outblk = blk; 84326d292d3SJohn Baldwin exf->decrypt(swe->sw_kschedule, inblk, outblk); 84426d292d3SJohn Baldwin if (outblk == blk) 84526d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 84626d292d3SJohn Baldwin else 84726d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 84826d292d3SJohn Baldwin } 84926d292d3SJohn Baldwin if (resid > 0) { 85026d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 85126d292d3SJohn Baldwin exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 85226d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 85308fca7a5SJohn-Mark Gurney } 85408fca7a5SJohn-Mark Gurney } else { 85508fca7a5SJohn-Mark Gurney /* Inject the authentication data */ 85626d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 85708fca7a5SJohn-Mark Gurney } 85808fca7a5SJohn-Mark Gurney 85920c128daSJohn Baldwin out: 86020c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 86126d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 86220c128daSJohn Baldwin return (error); 86308fca7a5SJohn-Mark Gurney } 86408fca7a5SJohn-Mark Gurney 865dd2e1352SJohn Baldwin static int 866dd2e1352SJohn Baldwin swcr_chacha20_poly1305(struct swcr_session *ses, struct cryptop *crp) 867dd2e1352SJohn Baldwin { 868dd2e1352SJohn Baldwin const struct crypto_session_params *csp; 869dd2e1352SJohn Baldwin uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 870dd2e1352SJohn Baldwin u_char *blk = (u_char *)blkbuf; 871dd2e1352SJohn Baldwin u_char tag[POLY1305_HASH_LEN]; 872dd2e1352SJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 873dd2e1352SJohn Baldwin const u_char *inblk; 874dd2e1352SJohn Baldwin u_char *outblk; 875dd2e1352SJohn Baldwin uint64_t *blkp; 876dd2e1352SJohn Baldwin union authctx ctx; 877dd2e1352SJohn Baldwin struct swcr_auth *swa; 878dd2e1352SJohn Baldwin struct swcr_encdec *swe; 879d8787d4fSMark Johnston const struct auth_hash *axf; 880d8787d4fSMark Johnston const struct enc_xform *exf; 88186be314dSJohn Baldwin size_t len; 882dd2e1352SJohn Baldwin int blksz, error, r, resid; 883dd2e1352SJohn Baldwin 884dd2e1352SJohn Baldwin swa = &ses->swcr_auth; 885dd2e1352SJohn Baldwin axf = swa->sw_axf; 886dd2e1352SJohn Baldwin 887dd2e1352SJohn Baldwin swe = &ses->swcr_encdec; 888dd2e1352SJohn Baldwin exf = swe->sw_exf; 889dd2e1352SJohn Baldwin blksz = exf->native_blocksize; 890dd2e1352SJohn Baldwin KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); 891dd2e1352SJohn Baldwin 892dd2e1352SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 893dd2e1352SJohn Baldwin return (EINVAL); 894dd2e1352SJohn Baldwin 895dd2e1352SJohn Baldwin csp = crypto_get_params(crp->crp_session); 896dd2e1352SJohn Baldwin 897dd2e1352SJohn Baldwin /* Generate Poly1305 key. */ 898dd2e1352SJohn Baldwin if (crp->crp_cipher_key != NULL) 899dd2e1352SJohn Baldwin axf->Setkey(&ctx, crp->crp_cipher_key, csp->csp_cipher_klen); 900dd2e1352SJohn Baldwin else 901dd2e1352SJohn Baldwin axf->Setkey(&ctx, csp->csp_cipher_key, csp->csp_cipher_klen); 902dd2e1352SJohn Baldwin axf->Reinit(&ctx, crp->crp_iv, csp->csp_ivlen); 903dd2e1352SJohn Baldwin 904dd2e1352SJohn Baldwin /* Supply MAC with AAD */ 905dd2e1352SJohn Baldwin if (crp->crp_aad != NULL) 906dd2e1352SJohn Baldwin axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 907dd2e1352SJohn Baldwin else 908dd2e1352SJohn Baldwin crypto_apply(crp, crp->crp_aad_start, 909dd2e1352SJohn Baldwin crp->crp_aad_length, axf->Update, &ctx); 910dd2e1352SJohn Baldwin if (crp->crp_aad_length % 16 != 0) { 911dd2e1352SJohn Baldwin /* padding1 */ 912dd2e1352SJohn Baldwin memset(blk, 0, 16); 913dd2e1352SJohn Baldwin axf->Update(&ctx, blk, 16 - crp->crp_aad_length % 16); 914dd2e1352SJohn Baldwin } 915dd2e1352SJohn Baldwin 916dd2e1352SJohn Baldwin if (crp->crp_cipher_key != NULL) 917dd2e1352SJohn Baldwin exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 918dd2e1352SJohn Baldwin csp->csp_cipher_klen); 9191833d604SJohn Baldwin exf->reinit(swe->sw_kschedule, crp->crp_iv, csp->csp_ivlen); 920dd2e1352SJohn Baldwin 921dd2e1352SJohn Baldwin /* Do encryption with MAC */ 922dd2e1352SJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 923dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 924dd2e1352SJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 925dd2e1352SJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 926dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 927dd2e1352SJohn Baldwin } else 928dd2e1352SJohn Baldwin cc_out = cc_in; 929dd2e1352SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 93086be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 93186be314dSJohn Baldwin if (len < blksz) { 932dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 933dd2e1352SJohn Baldwin inblk = blk; 93486be314dSJohn Baldwin } else 935dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 936dd2e1352SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 93786be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 93886be314dSJohn Baldwin if (len < blksz) 939dd2e1352SJohn Baldwin outblk = blk; 940dd2e1352SJohn Baldwin exf->encrypt(swe->sw_kschedule, inblk, outblk); 941dd2e1352SJohn Baldwin axf->Update(&ctx, outblk, blksz); 942dd2e1352SJohn Baldwin if (outblk == blk) 943dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 944dd2e1352SJohn Baldwin else 945dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 946dd2e1352SJohn Baldwin } else { 947dd2e1352SJohn Baldwin axf->Update(&ctx, inblk, blksz); 948dd2e1352SJohn Baldwin } 949dd2e1352SJohn Baldwin } 950dd2e1352SJohn Baldwin if (resid > 0) { 951dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 952dd2e1352SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 953dd2e1352SJohn Baldwin exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 954dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 955dd2e1352SJohn Baldwin } 956dd2e1352SJohn Baldwin axf->Update(&ctx, blk, resid); 957dd2e1352SJohn Baldwin if (resid % 16 != 0) { 958dd2e1352SJohn Baldwin /* padding2 */ 959dd2e1352SJohn Baldwin memset(blk, 0, 16); 960dd2e1352SJohn Baldwin axf->Update(&ctx, blk, 16 - resid % 16); 961dd2e1352SJohn Baldwin } 962dd2e1352SJohn Baldwin } 963dd2e1352SJohn Baldwin 964dd2e1352SJohn Baldwin /* lengths */ 965dd2e1352SJohn Baldwin blkp = (uint64_t *)blk; 966dd2e1352SJohn Baldwin blkp[0] = htole64(crp->crp_aad_length); 967dd2e1352SJohn Baldwin blkp[1] = htole64(crp->crp_payload_length); 968dd2e1352SJohn Baldwin axf->Update(&ctx, blk, sizeof(uint64_t) * 2); 969dd2e1352SJohn Baldwin 970dd2e1352SJohn Baldwin /* Finalize MAC */ 971dd2e1352SJohn Baldwin axf->Final(tag, &ctx); 972dd2e1352SJohn Baldwin 973dd2e1352SJohn Baldwin /* Validate tag */ 974dd2e1352SJohn Baldwin error = 0; 975dd2e1352SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 976dd2e1352SJohn Baldwin u_char tag2[POLY1305_HASH_LEN]; 977dd2e1352SJohn Baldwin 978dd2e1352SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 979dd2e1352SJohn Baldwin 980dd2e1352SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 981dd2e1352SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 982dd2e1352SJohn Baldwin if (r != 0) { 983dd2e1352SJohn Baldwin error = EBADMSG; 984dd2e1352SJohn Baldwin goto out; 985dd2e1352SJohn Baldwin } 986dd2e1352SJohn Baldwin 987dd2e1352SJohn Baldwin /* tag matches, decrypt data */ 988dd2e1352SJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 989dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 990dd2e1352SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 991dd2e1352SJohn Baldwin resid -= blksz) { 99286be314dSJohn Baldwin inblk = crypto_cursor_segment(&cc_in, &len); 99386be314dSJohn Baldwin if (len < blksz) { 994dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 995dd2e1352SJohn Baldwin inblk = blk; 99686be314dSJohn Baldwin } else 997dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 99886be314dSJohn Baldwin outblk = crypto_cursor_segment(&cc_out, &len); 99986be314dSJohn Baldwin if (len < blksz) 1000dd2e1352SJohn Baldwin outblk = blk; 1001dd2e1352SJohn Baldwin exf->decrypt(swe->sw_kschedule, inblk, outblk); 1002dd2e1352SJohn Baldwin if (outblk == blk) 1003dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 1004dd2e1352SJohn Baldwin else 1005dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 1006dd2e1352SJohn Baldwin } 1007dd2e1352SJohn Baldwin if (resid > 0) { 1008dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 1009dd2e1352SJohn Baldwin exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 1010dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 1011dd2e1352SJohn Baldwin } 1012dd2e1352SJohn Baldwin } else { 1013dd2e1352SJohn Baldwin /* Inject the authentication data */ 1014dd2e1352SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 1015dd2e1352SJohn Baldwin } 1016dd2e1352SJohn Baldwin 1017dd2e1352SJohn Baldwin out: 1018dd2e1352SJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 1019dd2e1352SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 1020dd2e1352SJohn Baldwin explicit_bzero(&ctx, sizeof(ctx)); 1021dd2e1352SJohn Baldwin return (error); 1022dd2e1352SJohn Baldwin } 1023dd2e1352SJohn Baldwin 1024091d81d1SSam Leffler /* 1025c0341432SJohn Baldwin * Apply a cipher and a digest to perform EtA. 1026c0341432SJohn Baldwin */ 1027c0341432SJohn Baldwin static int 1028c0341432SJohn Baldwin swcr_eta(struct swcr_session *ses, struct cryptop *crp) 1029c0341432SJohn Baldwin { 1030c0341432SJohn Baldwin int error; 1031c0341432SJohn Baldwin 1032c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1033c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 1034c0341432SJohn Baldwin if (error == 0) 1035c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 1036c0341432SJohn Baldwin } else { 1037c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 1038c0341432SJohn Baldwin if (error == 0) 1039c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 1040c0341432SJohn Baldwin } 1041c0341432SJohn Baldwin return (error); 1042c0341432SJohn Baldwin } 1043c0341432SJohn Baldwin 1044c0341432SJohn Baldwin /* 1045091d81d1SSam Leffler * Apply a compression/decompression algorithm 1046091d81d1SSam Leffler */ 1047091d81d1SSam Leffler static int 1048c0341432SJohn Baldwin swcr_compdec(struct swcr_session *ses, struct cryptop *crp) 1049091d81d1SSam Leffler { 1050d8787d4fSMark Johnston const struct comp_algo *cxf; 1051d3d79e96SJohn Baldwin uint8_t *data, *out; 1052091d81d1SSam Leffler int adj; 1053d3d79e96SJohn Baldwin uint32_t result; 1054091d81d1SSam Leffler 1055c0341432SJohn Baldwin cxf = ses->swcr_compdec.sw_cxf; 1056091d81d1SSam Leffler 1057091d81d1SSam Leffler /* We must handle the whole buffer of data in one time 1058091d81d1SSam Leffler * then if there is not all the data in the mbuf, we must 1059091d81d1SSam Leffler * copy in a buffer. 1060091d81d1SSam Leffler */ 1061091d81d1SSam Leffler 1062c0341432SJohn Baldwin data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1063091d81d1SSam Leffler if (data == NULL) 1064091d81d1SSam Leffler return (EINVAL); 1065c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1066c0341432SJohn Baldwin data); 1067091d81d1SSam Leffler 1068c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1069c0341432SJohn Baldwin result = cxf->compress(data, crp->crp_payload_length, &out); 1070091d81d1SSam Leffler else 1071c0341432SJohn Baldwin result = cxf->decompress(data, crp->crp_payload_length, &out); 1072091d81d1SSam Leffler 10731ede983cSDag-Erling Smørgrav free(data, M_CRYPTO_DATA); 1074091d81d1SSam Leffler if (result == 0) 1075c0341432SJohn Baldwin return (EINVAL); 1076c0341432SJohn Baldwin crp->crp_olen = result; 1077c0341432SJohn Baldwin 1078c0341432SJohn Baldwin /* Check the compressed size when doing compression */ 1079c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1080c0341432SJohn Baldwin if (result >= crp->crp_payload_length) { 1081c0341432SJohn Baldwin /* Compression was useless, we lost time */ 1082c0341432SJohn Baldwin free(out, M_CRYPTO_DATA); 1083c0341432SJohn Baldwin return (0); 1084c0341432SJohn Baldwin } 1085c0341432SJohn Baldwin } 1086091d81d1SSam Leffler 1087091d81d1SSam Leffler /* Copy back the (de)compressed data. m_copyback is 1088091d81d1SSam Leffler * extending the mbuf as necessary. 1089091d81d1SSam Leffler */ 1090c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_payload_start, result, out); 1091c0341432SJohn Baldwin if (result < crp->crp_payload_length) { 10929c0e3d3aSJohn Baldwin switch (crp->crp_buf.cb_type) { 1093c0341432SJohn Baldwin case CRYPTO_BUF_MBUF: 1094883a0196SJohn Baldwin case CRYPTO_BUF_SINGLE_MBUF: 1095c0341432SJohn Baldwin adj = result - crp->crp_payload_length; 10969c0e3d3aSJohn Baldwin m_adj(crp->crp_buf.cb_mbuf, adj); 1097c0341432SJohn Baldwin break; 1098c0341432SJohn Baldwin case CRYPTO_BUF_UIO: { 10999c0e3d3aSJohn Baldwin struct uio *uio = crp->crp_buf.cb_uio; 1100091d81d1SSam Leffler int ind; 1101091d81d1SSam Leffler 1102c0341432SJohn Baldwin adj = crp->crp_payload_length - result; 1103091d81d1SSam Leffler ind = uio->uio_iovcnt - 1; 1104091d81d1SSam Leffler 1105091d81d1SSam Leffler while (adj > 0 && ind >= 0) { 1106091d81d1SSam Leffler if (adj < uio->uio_iov[ind].iov_len) { 1107091d81d1SSam Leffler uio->uio_iov[ind].iov_len -= adj; 1108091d81d1SSam Leffler break; 1109091d81d1SSam Leffler } 1110091d81d1SSam Leffler 1111091d81d1SSam Leffler adj -= uio->uio_iov[ind].iov_len; 1112091d81d1SSam Leffler uio->uio_iov[ind].iov_len = 0; 1113091d81d1SSam Leffler ind--; 1114091d81d1SSam Leffler uio->uio_iovcnt--; 1115091d81d1SSam Leffler } 1116091d81d1SSam Leffler } 1117c0341432SJohn Baldwin break; 1118e6f6d0c9SAlan Somers case CRYPTO_BUF_VMPAGE: 1119e6f6d0c9SAlan Somers adj = crp->crp_payload_length - result; 1120e6f6d0c9SAlan Somers crp->crp_buf.cb_vm_page_len -= adj; 1121e6f6d0c9SAlan Somers break; 11229c0e3d3aSJohn Baldwin default: 11239c0e3d3aSJohn Baldwin break; 1124c0341432SJohn Baldwin } 1125091d81d1SSam Leffler } 11261ede983cSDag-Erling Smørgrav free(out, M_CRYPTO_DATA); 1127091d81d1SSam Leffler return 0; 1128091d81d1SSam Leffler } 1129091d81d1SSam Leffler 1130091d81d1SSam Leffler static int 11313e947048SJohn Baldwin swcr_setup_cipher(struct swcr_session *ses, 1132c0341432SJohn Baldwin const struct crypto_session_params *csp) 1133091d81d1SSam Leffler { 1134c0341432SJohn Baldwin struct swcr_encdec *swe; 1135d8787d4fSMark Johnston const struct enc_xform *txf; 1136f6c4bc3bSPawel Jakub Dawidek int error; 1137091d81d1SSam Leffler 1138c0341432SJohn Baldwin swe = &ses->swcr_encdec; 1139c0341432SJohn Baldwin txf = crypto_cipher(csp); 11403e947048SJohn Baldwin if (txf->ctxsize != 0) { 11413e947048SJohn Baldwin swe->sw_kschedule = malloc(txf->ctxsize, M_CRYPTO_DATA, 11423e947048SJohn Baldwin M_NOWAIT); 11433e947048SJohn Baldwin if (swe->sw_kschedule == NULL) 11443e947048SJohn Baldwin return (ENOMEM); 11453e947048SJohn Baldwin } 1146c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) { 11473e947048SJohn Baldwin error = txf->setkey(swe->sw_kschedule, 1148c0341432SJohn Baldwin csp->csp_cipher_key, csp->csp_cipher_klen); 1149c0341432SJohn Baldwin if (error) 1150c0341432SJohn Baldwin return (error); 1151091d81d1SSam Leffler } 1152c0341432SJohn Baldwin swe->sw_exf = txf; 1153c0341432SJohn Baldwin return (0); 1154f6c4bc3bSPawel Jakub Dawidek } 1155091d81d1SSam Leffler 1156c0341432SJohn Baldwin static int 1157c0341432SJohn Baldwin swcr_setup_auth(struct swcr_session *ses, 1158c0341432SJohn Baldwin const struct crypto_session_params *csp) 1159c0341432SJohn Baldwin { 1160c0341432SJohn Baldwin struct swcr_auth *swa; 1161d8787d4fSMark Johnston const struct auth_hash *axf; 1162c0341432SJohn Baldwin 1163c0341432SJohn Baldwin swa = &ses->swcr_auth; 1164c0341432SJohn Baldwin 1165c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1166c0341432SJohn Baldwin swa->sw_axf = axf; 1167c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1168c0341432SJohn Baldwin return (EINVAL); 1169c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1170c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1171c0341432SJohn Baldwin else 1172c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1173c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1174c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1175c0341432SJohn Baldwin return (ENOBUFS); 1176c0341432SJohn Baldwin 1177c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1178091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1179c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1180f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1181f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1182f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1183091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 1184091d81d1SSam Leffler case CRYPTO_RIPEMD160_HMAC: 11853a0b6a93SJohn Baldwin swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1186091d81d1SSam Leffler M_NOWAIT); 1187c0341432SJohn Baldwin if (swa->sw_octx == NULL) 1188c0341432SJohn Baldwin return (ENOBUFS); 1189c0341432SJohn Baldwin 1190c0341432SJohn Baldwin if (csp->csp_auth_key != NULL) { 1191c0341432SJohn Baldwin swcr_authprepare(axf, swa, csp->csp_auth_key, 1192c0341432SJohn Baldwin csp->csp_auth_klen); 1193091d81d1SSam Leffler } 1194091d81d1SSam Leffler 1195c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1196c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1197091d81d1SSam Leffler break; 1198091d81d1SSam Leffler case CRYPTO_SHA1: 1199c4729f6eSConrad Meyer case CRYPTO_SHA2_224: 1200c4729f6eSConrad Meyer case CRYPTO_SHA2_256: 1201c4729f6eSConrad Meyer case CRYPTO_SHA2_384: 1202c4729f6eSConrad Meyer case CRYPTO_SHA2_512: 1203c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1204c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1205c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1206c0341432SJohn Baldwin break; 1207c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1208c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1209c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1210c0341432SJohn Baldwin csp->csp_auth_klen); 1211c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1212c0341432SJohn Baldwin ses->swcr_process = swcr_gmac; 1213c0341432SJohn Baldwin break; 1214c0341432SJohn Baldwin case CRYPTO_POLY1305: 1215c0341432SJohn Baldwin case CRYPTO_BLAKE2B: 1216c0341432SJohn Baldwin case CRYPTO_BLAKE2S: 1217c0341432SJohn Baldwin /* 1218c0341432SJohn Baldwin * Blake2b and Blake2s support an optional key but do 1219c0341432SJohn Baldwin * not require one. 1220c0341432SJohn Baldwin */ 1221c0341432SJohn Baldwin if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) 1222c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1223c0341432SJohn Baldwin csp->csp_auth_klen); 1224c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1225c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1226c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1227c0341432SJohn Baldwin break; 1228c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1229c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1230c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1231c0341432SJohn Baldwin csp->csp_auth_klen); 1232c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1233c0341432SJohn Baldwin ses->swcr_process = swcr_ccm_cbc_mac; 1234c0341432SJohn Baldwin break; 1235091d81d1SSam Leffler } 1236091d81d1SSam Leffler 1237c0341432SJohn Baldwin return (0); 1238c0341432SJohn Baldwin } 123908fca7a5SJohn-Mark Gurney 1240c0341432SJohn Baldwin static int 1241c0341432SJohn Baldwin swcr_setup_gcm(struct swcr_session *ses, 1242c0341432SJohn Baldwin const struct crypto_session_params *csp) 1243c0341432SJohn Baldwin { 1244c0341432SJohn Baldwin struct swcr_auth *swa; 1245d8787d4fSMark Johnston const struct auth_hash *axf; 1246c0341432SJohn Baldwin 1247c0341432SJohn Baldwin if (csp->csp_ivlen != AES_GCM_IV_LEN) 1248c0341432SJohn Baldwin return (EINVAL); 1249c0341432SJohn Baldwin 1250c0341432SJohn Baldwin /* First, setup the auth side. */ 1251c0341432SJohn Baldwin swa = &ses->swcr_auth; 1252c0341432SJohn Baldwin switch (csp->csp_cipher_klen * 8) { 1253c0341432SJohn Baldwin case 128: 1254c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_128; 1255c0341432SJohn Baldwin break; 1256c0341432SJohn Baldwin case 192: 1257c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_192; 1258c0341432SJohn Baldwin break; 1259c0341432SJohn Baldwin case 256: 1260c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_256; 1261c0341432SJohn Baldwin break; 1262c0341432SJohn Baldwin default: 1263c0341432SJohn Baldwin return (EINVAL); 1264c0341432SJohn Baldwin } 1265c0341432SJohn Baldwin swa->sw_axf = axf; 1266c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1267c0341432SJohn Baldwin return (EINVAL); 1268c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1269c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1270c0341432SJohn Baldwin else 1271c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1272c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1273c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1274c0341432SJohn Baldwin return (ENOBUFS); 1275c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1276c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) 1277c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, 1278c0341432SJohn Baldwin csp->csp_cipher_klen); 1279c0341432SJohn Baldwin 1280c0341432SJohn Baldwin /* Second, setup the cipher side. */ 12813e947048SJohn Baldwin return (swcr_setup_cipher(ses, csp)); 1282c0341432SJohn Baldwin } 1283c0341432SJohn Baldwin 1284c0341432SJohn Baldwin static int 1285c0341432SJohn Baldwin swcr_setup_ccm(struct swcr_session *ses, 1286c0341432SJohn Baldwin const struct crypto_session_params *csp) 1287c0341432SJohn Baldwin { 1288c0341432SJohn Baldwin struct swcr_auth *swa; 1289d8787d4fSMark Johnston const struct auth_hash *axf; 1290c0341432SJohn Baldwin 1291c0341432SJohn Baldwin /* First, setup the auth side. */ 1292c0341432SJohn Baldwin swa = &ses->swcr_auth; 1293c0341432SJohn Baldwin switch (csp->csp_cipher_klen * 8) { 1294507281e5SSean Eric Fagan case 128: 1295507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_128; 1296507281e5SSean Eric Fagan break; 1297507281e5SSean Eric Fagan case 192: 1298507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_192; 1299507281e5SSean Eric Fagan break; 1300507281e5SSean Eric Fagan case 256: 1301507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_256; 1302507281e5SSean Eric Fagan break; 1303507281e5SSean Eric Fagan default: 1304c0341432SJohn Baldwin return (EINVAL); 1305507281e5SSean Eric Fagan } 1306c0341432SJohn Baldwin swa->sw_axf = axf; 1307c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1308c0341432SJohn Baldwin return (EINVAL); 1309c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1310c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1311c0341432SJohn Baldwin else 1312c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1313c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1314c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1315c0341432SJohn Baldwin return (ENOBUFS); 1316c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1317c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) 1318c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, 1319c0341432SJohn Baldwin csp->csp_cipher_klen); 132008fca7a5SJohn-Mark Gurney 1321c0341432SJohn Baldwin /* Second, setup the cipher side. */ 13223e947048SJohn Baldwin return (swcr_setup_cipher(ses, csp)); 13232e2e26d1SJohn Baldwin } 1324a2bc81bfSJohn-Mark Gurney 1325dd2e1352SJohn Baldwin static int 1326dd2e1352SJohn Baldwin swcr_setup_chacha20_poly1305(struct swcr_session *ses, 1327dd2e1352SJohn Baldwin const struct crypto_session_params *csp) 1328dd2e1352SJohn Baldwin { 1329dd2e1352SJohn Baldwin struct swcr_auth *swa; 1330d8787d4fSMark Johnston const struct auth_hash *axf; 1331dd2e1352SJohn Baldwin 1332dd2e1352SJohn Baldwin if (csp->csp_ivlen != CHACHA20_POLY1305_IV_LEN) 1333dd2e1352SJohn Baldwin return (EINVAL); 1334dd2e1352SJohn Baldwin 1335dd2e1352SJohn Baldwin /* First, setup the auth side. */ 1336dd2e1352SJohn Baldwin swa = &ses->swcr_auth; 1337dd2e1352SJohn Baldwin axf = &auth_hash_chacha20_poly1305; 1338dd2e1352SJohn Baldwin swa->sw_axf = axf; 1339dd2e1352SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1340dd2e1352SJohn Baldwin return (EINVAL); 1341dd2e1352SJohn Baldwin if (csp->csp_auth_mlen == 0) 1342dd2e1352SJohn Baldwin swa->sw_mlen = axf->hashsize; 1343dd2e1352SJohn Baldwin else 1344dd2e1352SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1345dd2e1352SJohn Baldwin 1346dd2e1352SJohn Baldwin /* The auth state is regenerated for each nonce. */ 1347dd2e1352SJohn Baldwin 1348dd2e1352SJohn Baldwin /* Second, setup the cipher side. */ 1349dd2e1352SJohn Baldwin return (swcr_setup_cipher(ses, csp)); 1350dd2e1352SJohn Baldwin } 1351dd2e1352SJohn Baldwin 1352c0341432SJohn Baldwin static bool 1353c0341432SJohn Baldwin swcr_auth_supported(const struct crypto_session_params *csp) 1354109919c6SBenno Rice { 1355d8787d4fSMark Johnston const struct auth_hash *axf; 1356091d81d1SSam Leffler 1357c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1358c0341432SJohn Baldwin if (axf == NULL) 1359c0341432SJohn Baldwin return (false); 1360c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1361091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1362c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1363f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1364f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1365f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1366091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 1367c0341432SJohn Baldwin case CRYPTO_RIPEMD160_HMAC: 1368091d81d1SSam Leffler break; 1369c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1370c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1371c0341432SJohn Baldwin case 128: 1372c0341432SJohn Baldwin case 192: 1373c0341432SJohn Baldwin case 256: 1374c0341432SJohn Baldwin break; 1375c0341432SJohn Baldwin default: 1376c0341432SJohn Baldwin return (false); 1377c0341432SJohn Baldwin } 1378c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1379c0341432SJohn Baldwin return (false); 1380c0341432SJohn Baldwin if (csp->csp_ivlen != AES_GCM_IV_LEN) 1381c0341432SJohn Baldwin return (false); 1382c0341432SJohn Baldwin break; 138325b7033bSConrad Meyer case CRYPTO_POLY1305: 1384c0341432SJohn Baldwin if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1385c0341432SJohn Baldwin return (false); 1386c0341432SJohn Baldwin break; 1387c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1388c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1389c0341432SJohn Baldwin case 128: 1390c0341432SJohn Baldwin case 192: 1391c0341432SJohn Baldwin case 256: 1392c0341432SJohn Baldwin break; 1393c0341432SJohn Baldwin default: 1394c0341432SJohn Baldwin return (false); 1395c0341432SJohn Baldwin } 1396c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1397c0341432SJohn Baldwin return (false); 1398c0341432SJohn Baldwin break; 1399c0341432SJohn Baldwin } 1400c0341432SJohn Baldwin return (true); 1401c0341432SJohn Baldwin } 1402091d81d1SSam Leffler 1403c0341432SJohn Baldwin static bool 1404c0341432SJohn Baldwin swcr_cipher_supported(const struct crypto_session_params *csp) 1405c0341432SJohn Baldwin { 1406d8787d4fSMark Johnston const struct enc_xform *txf; 1407c0341432SJohn Baldwin 1408c0341432SJohn Baldwin txf = crypto_cipher(csp); 1409c0341432SJohn Baldwin if (txf == NULL) 1410c0341432SJohn Baldwin return (false); 1411c0341432SJohn Baldwin if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1412c0341432SJohn Baldwin txf->ivsize != csp->csp_ivlen) 1413c0341432SJohn Baldwin return (false); 1414c0341432SJohn Baldwin return (true); 1415c0341432SJohn Baldwin } 1416c0341432SJohn Baldwin 14176038018aSMarcin Wojtas #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 14186038018aSMarcin Wojtas 1419c0341432SJohn Baldwin static int 1420c0341432SJohn Baldwin swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1421c0341432SJohn Baldwin { 14226038018aSMarcin Wojtas if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1423c0341432SJohn Baldwin return (EINVAL); 1424c0341432SJohn Baldwin switch (csp->csp_mode) { 1425c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1426c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1427c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1428c0341432SJohn Baldwin break; 1429c0341432SJohn Baldwin default: 1430c0341432SJohn Baldwin return (EINVAL); 14315fbc5b5aSConrad Meyer } 1432091d81d1SSam Leffler break; 1433c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1434c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1435c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1436c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1437dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1438c0341432SJohn Baldwin return (EINVAL); 1439c0341432SJohn Baldwin default: 1440c0341432SJohn Baldwin if (!swcr_cipher_supported(csp)) 1441c0341432SJohn Baldwin return (EINVAL); 1442091d81d1SSam Leffler break; 1443091d81d1SSam Leffler } 1444c0341432SJohn Baldwin break; 1445c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1446c0341432SJohn Baldwin if (!swcr_auth_supported(csp)) 1447c0341432SJohn Baldwin return (EINVAL); 1448c0341432SJohn Baldwin break; 1449c0341432SJohn Baldwin case CSP_MODE_AEAD: 1450c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1451c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1452c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1453dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1454c0341432SJohn Baldwin break; 1455c0341432SJohn Baldwin default: 1456c0341432SJohn Baldwin return (EINVAL); 1457c0341432SJohn Baldwin } 1458c0341432SJohn Baldwin break; 1459c0341432SJohn Baldwin case CSP_MODE_ETA: 1460c0341432SJohn Baldwin /* AEAD algorithms cannot be used for EtA. */ 1461c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1462c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1463c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1464dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1465c0341432SJohn Baldwin return (EINVAL); 1466c0341432SJohn Baldwin } 1467c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1468c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1469c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1470c0341432SJohn Baldwin return (EINVAL); 1471c0341432SJohn Baldwin } 1472c0341432SJohn Baldwin 1473c0341432SJohn Baldwin if (!swcr_cipher_supported(csp) || 1474c0341432SJohn Baldwin !swcr_auth_supported(csp)) 1475c0341432SJohn Baldwin return (EINVAL); 1476c0341432SJohn Baldwin break; 1477c0341432SJohn Baldwin default: 1478c0341432SJohn Baldwin return (EINVAL); 1479c0341432SJohn Baldwin } 1480c0341432SJohn Baldwin 1481c0341432SJohn Baldwin return (CRYPTODEV_PROBE_SOFTWARE); 1482c0341432SJohn Baldwin } 1483c0341432SJohn Baldwin 1484c0341432SJohn Baldwin /* 1485c0341432SJohn Baldwin * Generate a new software session. 1486c0341432SJohn Baldwin */ 1487c0341432SJohn Baldwin static int 1488c0341432SJohn Baldwin swcr_newsession(device_t dev, crypto_session_t cses, 1489c0341432SJohn Baldwin const struct crypto_session_params *csp) 1490c0341432SJohn Baldwin { 1491c0341432SJohn Baldwin struct swcr_session *ses; 1492c0341432SJohn Baldwin struct swcr_encdec *swe; 1493c0341432SJohn Baldwin struct swcr_auth *swa; 1494d8787d4fSMark Johnston const struct comp_algo *cxf; 1495c0341432SJohn Baldwin int error; 1496c0341432SJohn Baldwin 1497c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1498c0341432SJohn Baldwin mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); 1499c0341432SJohn Baldwin 1500c0341432SJohn Baldwin error = 0; 1501c0341432SJohn Baldwin swe = &ses->swcr_encdec; 1502c0341432SJohn Baldwin swa = &ses->swcr_auth; 1503c0341432SJohn Baldwin switch (csp->csp_mode) { 1504c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1505c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1506c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1507c0341432SJohn Baldwin cxf = &comp_algo_deflate; 1508c0341432SJohn Baldwin break; 1509c0341432SJohn Baldwin #ifdef INVARIANTS 1510c0341432SJohn Baldwin default: 1511c0341432SJohn Baldwin panic("bad compression algo"); 1512c0341432SJohn Baldwin #endif 1513c0341432SJohn Baldwin } 1514c0341432SJohn Baldwin ses->swcr_compdec.sw_cxf = cxf; 1515c0341432SJohn Baldwin ses->swcr_process = swcr_compdec; 1516c0341432SJohn Baldwin break; 1517c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1518c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1519c0341432SJohn Baldwin case CRYPTO_NULL_CBC: 1520c0341432SJohn Baldwin ses->swcr_process = swcr_null; 1521c0341432SJohn Baldwin break; 1522c0341432SJohn Baldwin #ifdef INVARIANTS 1523c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1524c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1525dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1526c0341432SJohn Baldwin panic("bad cipher algo"); 1527c0341432SJohn Baldwin #endif 1528c0341432SJohn Baldwin default: 15293e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1530c0341432SJohn Baldwin if (error == 0) 1531c0341432SJohn Baldwin ses->swcr_process = swcr_encdec; 1532c0341432SJohn Baldwin } 1533c0341432SJohn Baldwin break; 1534c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1535c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1536c0341432SJohn Baldwin break; 1537c0341432SJohn Baldwin case CSP_MODE_AEAD: 1538c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1539c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1540c0341432SJohn Baldwin error = swcr_setup_gcm(ses, csp); 1541c0341432SJohn Baldwin if (error == 0) 1542c0341432SJohn Baldwin ses->swcr_process = swcr_gcm; 1543c0341432SJohn Baldwin break; 1544c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1545c0341432SJohn Baldwin error = swcr_setup_ccm(ses, csp); 1546c0341432SJohn Baldwin if (error == 0) 1547c0341432SJohn Baldwin ses->swcr_process = swcr_ccm; 1548c0341432SJohn Baldwin break; 1549dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1550dd2e1352SJohn Baldwin error = swcr_setup_chacha20_poly1305(ses, csp); 1551dd2e1352SJohn Baldwin if (error == 0) 1552dd2e1352SJohn Baldwin ses->swcr_process = swcr_chacha20_poly1305; 1553dd2e1352SJohn Baldwin break; 1554c0341432SJohn Baldwin #ifdef INVARIANTS 1555c0341432SJohn Baldwin default: 1556c0341432SJohn Baldwin panic("bad aead algo"); 1557c0341432SJohn Baldwin #endif 1558c0341432SJohn Baldwin } 1559c0341432SJohn Baldwin break; 1560c0341432SJohn Baldwin case CSP_MODE_ETA: 1561c0341432SJohn Baldwin #ifdef INVARIANTS 1562c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1563c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1564c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1565dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1566c0341432SJohn Baldwin panic("bad eta cipher algo"); 1567c0341432SJohn Baldwin } 1568c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1569c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1570c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1571c0341432SJohn Baldwin panic("bad eta auth algo"); 1572c0341432SJohn Baldwin } 1573c0341432SJohn Baldwin #endif 1574c0341432SJohn Baldwin 1575c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1576c0341432SJohn Baldwin if (error) 1577c0341432SJohn Baldwin break; 1578c0341432SJohn Baldwin if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1579c0341432SJohn Baldwin /* Effectively degrade to digest mode. */ 1580c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1581c0341432SJohn Baldwin break; 1582c0341432SJohn Baldwin } 1583c0341432SJohn Baldwin 15843e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1585c0341432SJohn Baldwin if (error == 0) 1586c0341432SJohn Baldwin ses->swcr_process = swcr_eta; 1587c0341432SJohn Baldwin break; 1588c0341432SJohn Baldwin default: 1589c0341432SJohn Baldwin error = EINVAL; 1590c0341432SJohn Baldwin } 1591c0341432SJohn Baldwin 1592c0341432SJohn Baldwin if (error) 1593c0341432SJohn Baldwin swcr_freesession(dev, cses); 1594c0341432SJohn Baldwin return (error); 1595c0341432SJohn Baldwin } 1596c0341432SJohn Baldwin 1597c0341432SJohn Baldwin static void 1598c0341432SJohn Baldwin swcr_freesession(device_t dev, crypto_session_t cses) 1599c0341432SJohn Baldwin { 1600c0341432SJohn Baldwin struct swcr_session *ses; 1601c0341432SJohn Baldwin 1602c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1603c0341432SJohn Baldwin 1604c0341432SJohn Baldwin mtx_destroy(&ses->swcr_lock); 1605c0341432SJohn Baldwin 16063e947048SJohn Baldwin zfree(ses->swcr_encdec.sw_kschedule, M_CRYPTO_DATA); 16074a711b8dSJohn Baldwin zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 16084a711b8dSJohn Baldwin zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1609091d81d1SSam Leffler } 1610091d81d1SSam Leffler 1611091d81d1SSam Leffler /* 1612091d81d1SSam Leffler * Process a software request. 1613091d81d1SSam Leffler */ 1614091d81d1SSam Leffler static int 16156810ad6fSSam Leffler swcr_process(device_t dev, struct cryptop *crp, int hint) 1616091d81d1SSam Leffler { 1617c0341432SJohn Baldwin struct swcr_session *ses; 1618091d81d1SSam Leffler 16191b0909d5SConrad Meyer ses = crypto_get_driver_session(crp->crp_session); 1620a7fcb1afSSean Eric Fagan mtx_lock(&ses->swcr_lock); 1621091d81d1SSam Leffler 1622c0341432SJohn Baldwin crp->crp_etype = ses->swcr_process(ses, crp); 1623091d81d1SSam Leffler 1624a7fcb1afSSean Eric Fagan mtx_unlock(&ses->swcr_lock); 1625091d81d1SSam Leffler crypto_done(crp); 1626c0341432SJohn Baldwin return (0); 1627091d81d1SSam Leffler } 1628091d81d1SSam Leffler 1629091d81d1SSam Leffler static void 16303f147ab2SWarner Losh swcr_identify(driver_t *drv, device_t parent) 1631091d81d1SSam Leffler { 16326810ad6fSSam Leffler /* NB: order 10 is so we get attached after h/w devices */ 16336810ad6fSSam Leffler if (device_find_child(parent, "cryptosoft", -1) == NULL && 163486c585d9SMarius Strobl BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 16356810ad6fSSam Leffler panic("cryptosoft: could not attach"); 16366810ad6fSSam Leffler } 1637f6c4bc3bSPawel Jakub Dawidek 16386810ad6fSSam Leffler static int 16396810ad6fSSam Leffler swcr_probe(device_t dev) 16406810ad6fSSam Leffler { 16416810ad6fSSam Leffler device_set_desc(dev, "software crypto"); 16424fc60fa9SMark Johnston device_quiet(dev); 164386c585d9SMarius Strobl return (BUS_PROBE_NOWILDCARD); 16446810ad6fSSam Leffler } 1645f6c4bc3bSPawel Jakub Dawidek 16466810ad6fSSam Leffler static int 16476810ad6fSSam Leffler swcr_attach(device_t dev) 16486810ad6fSSam Leffler { 16496810ad6fSSam Leffler 16509ebbebe4SConrad Meyer swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 16516810ad6fSSam Leffler CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 16526810ad6fSSam Leffler if (swcr_id < 0) { 16536810ad6fSSam Leffler device_printf(dev, "cannot initialize!"); 1654c0341432SJohn Baldwin return (ENXIO); 16556810ad6fSSam Leffler } 16566810ad6fSSam Leffler 1657c0341432SJohn Baldwin return (0); 1658091d81d1SSam Leffler } 16594b465da2SPawel Jakub Dawidek 16603f147ab2SWarner Losh static int 16616810ad6fSSam Leffler swcr_detach(device_t dev) 16624b465da2SPawel Jakub Dawidek { 16636810ad6fSSam Leffler crypto_unregister_all(swcr_id); 16643f147ab2SWarner Losh return 0; 16654b465da2SPawel Jakub Dawidek } 16666810ad6fSSam Leffler 16676810ad6fSSam Leffler static device_method_t swcr_methods[] = { 16686810ad6fSSam Leffler DEVMETHOD(device_identify, swcr_identify), 16696810ad6fSSam Leffler DEVMETHOD(device_probe, swcr_probe), 16706810ad6fSSam Leffler DEVMETHOD(device_attach, swcr_attach), 16716810ad6fSSam Leffler DEVMETHOD(device_detach, swcr_detach), 16726810ad6fSSam Leffler 1673c0341432SJohn Baldwin DEVMETHOD(cryptodev_probesession, swcr_probesession), 16746810ad6fSSam Leffler DEVMETHOD(cryptodev_newsession, swcr_newsession), 16756810ad6fSSam Leffler DEVMETHOD(cryptodev_freesession,swcr_freesession), 16766810ad6fSSam Leffler DEVMETHOD(cryptodev_process, swcr_process), 16776810ad6fSSam Leffler 16786810ad6fSSam Leffler {0, 0}, 16796810ad6fSSam Leffler }; 16806810ad6fSSam Leffler 16816810ad6fSSam Leffler static driver_t swcr_driver = { 16826810ad6fSSam Leffler "cryptosoft", 16836810ad6fSSam Leffler swcr_methods, 16846810ad6fSSam Leffler 0, /* NB: no softc */ 16856810ad6fSSam Leffler }; 16866810ad6fSSam Leffler static devclass_t swcr_devclass; 16876810ad6fSSam Leffler 16886810ad6fSSam Leffler /* 16896810ad6fSSam Leffler * NB: We explicitly reference the crypto module so we 16906810ad6fSSam Leffler * get the necessary ordering when built as a loadable 16916810ad6fSSam Leffler * module. This is required because we bundle the crypto 16926810ad6fSSam Leffler * module code together with the cryptosoft driver (otherwise 16936810ad6fSSam Leffler * normal module dependencies would handle things). 16946810ad6fSSam Leffler */ 16956810ad6fSSam Leffler extern int crypto_modevent(struct module *, int, void *); 16966810ad6fSSam Leffler /* XXX where to attach */ 16976810ad6fSSam Leffler DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 16986810ad6fSSam Leffler MODULE_VERSION(cryptosoft, 1); 16996810ad6fSSam Leffler MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1700