1091d81d1SSam Leffler /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2091d81d1SSam Leffler 360727d8bSWarner Losh /*- 4091d81d1SSam Leffler * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 56810ad6fSSam Leffler * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6091d81d1SSam Leffler * 7091d81d1SSam Leffler * This code was written by Angelos D. Keromytis in Athens, Greece, in 8091d81d1SSam Leffler * February 2000. Network Security Technologies Inc. (NSTI) kindly 9091d81d1SSam Leffler * supported the development of this code. 10091d81d1SSam Leffler * 11091d81d1SSam Leffler * Copyright (c) 2000, 2001 Angelos D. Keromytis 1208fca7a5SJohn-Mark Gurney * Copyright (c) 2014 The FreeBSD Foundation 1308fca7a5SJohn-Mark Gurney * All rights reserved. 1408fca7a5SJohn-Mark Gurney * 1508fca7a5SJohn-Mark Gurney * Portions of this software were developed by John-Mark Gurney 1608fca7a5SJohn-Mark Gurney * under sponsorship of the FreeBSD Foundation and 1708fca7a5SJohn-Mark Gurney * Rubicon Communications, LLC (Netgate). 18091d81d1SSam Leffler * 19091d81d1SSam Leffler * Permission to use, copy, and modify this software with or without fee 20091d81d1SSam Leffler * is hereby granted, provided that this entire notice is included in 21091d81d1SSam Leffler * all source code copies of any software which is or includes a copy or 22091d81d1SSam Leffler * modification of this software. 23091d81d1SSam Leffler * 24091d81d1SSam Leffler * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 25091d81d1SSam Leffler * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 26091d81d1SSam Leffler * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 27091d81d1SSam Leffler * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 28091d81d1SSam Leffler * PURPOSE. 29091d81d1SSam Leffler */ 30091d81d1SSam Leffler 312c446514SDavid E. O'Brien #include <sys/cdefs.h> 322c446514SDavid E. O'Brien __FBSDID("$FreeBSD$"); 332c446514SDavid E. O'Brien 34091d81d1SSam Leffler #include <sys/param.h> 35091d81d1SSam Leffler #include <sys/systm.h> 36091d81d1SSam Leffler #include <sys/malloc.h> 37091d81d1SSam Leffler #include <sys/mbuf.h> 386810ad6fSSam Leffler #include <sys/module.h> 39091d81d1SSam Leffler #include <sys/sysctl.h> 40091d81d1SSam Leffler #include <sys/errno.h> 41091d81d1SSam Leffler #include <sys/random.h> 42091d81d1SSam Leffler #include <sys/kernel.h> 43091d81d1SSam Leffler #include <sys/uio.h> 44109919c6SBenno Rice #include <sys/lock.h> 45109919c6SBenno Rice #include <sys/rwlock.h> 4608fca7a5SJohn-Mark Gurney #include <sys/endian.h> 4708fca7a5SJohn-Mark Gurney #include <sys/limits.h> 48a7fcb1afSSean Eric Fagan #include <sys/mutex.h> 49091d81d1SSam Leffler 50091d81d1SSam Leffler #include <crypto/sha1.h> 51091d81d1SSam Leffler #include <opencrypto/rmd160.h> 52091d81d1SSam Leffler 53091d81d1SSam Leffler #include <opencrypto/cryptodev.h> 54091d81d1SSam Leffler #include <opencrypto/xform.h> 55091d81d1SSam Leffler 566810ad6fSSam Leffler #include <sys/kobj.h> 576810ad6fSSam Leffler #include <sys/bus.h> 586810ad6fSSam Leffler #include "cryptodev_if.h" 59091d81d1SSam Leffler 60c0341432SJohn Baldwin struct swcr_auth { 61c0341432SJohn Baldwin void *sw_ictx; 62c0341432SJohn Baldwin void *sw_octx; 63c0341432SJohn Baldwin struct auth_hash *sw_axf; 64c0341432SJohn Baldwin uint16_t sw_mlen; 65c0341432SJohn Baldwin }; 66c0341432SJohn Baldwin 67c0341432SJohn Baldwin struct swcr_encdec { 683e947048SJohn Baldwin void *sw_kschedule; 69c0341432SJohn Baldwin struct enc_xform *sw_exf; 70c0341432SJohn Baldwin }; 71c0341432SJohn Baldwin 72c0341432SJohn Baldwin struct swcr_compdec { 73c0341432SJohn Baldwin struct comp_algo *sw_cxf; 74c0341432SJohn Baldwin }; 75c0341432SJohn Baldwin 76c0341432SJohn Baldwin struct swcr_session { 77c0341432SJohn Baldwin struct mtx swcr_lock; 78c0341432SJohn Baldwin int (*swcr_process)(struct swcr_session *, struct cryptop *); 79c0341432SJohn Baldwin 80c0341432SJohn Baldwin struct swcr_auth swcr_auth; 81c0341432SJohn Baldwin struct swcr_encdec swcr_encdec; 82c0341432SJohn Baldwin struct swcr_compdec swcr_compdec; 83c0341432SJohn Baldwin }; 84507281e5SSean Eric Fagan 856810ad6fSSam Leffler static int32_t swcr_id; 866810ad6fSSam Leffler 871b0909d5SConrad Meyer static void swcr_freesession(device_t dev, crypto_session_t cses); 88091d81d1SSam Leffler 89c0341432SJohn Baldwin /* Used for CRYPTO_NULL_CBC. */ 90c0341432SJohn Baldwin static int 91c0341432SJohn Baldwin swcr_null(struct swcr_session *ses, struct cryptop *crp) 92c0341432SJohn Baldwin { 93c0341432SJohn Baldwin 94c0341432SJohn Baldwin return (0); 95c0341432SJohn Baldwin } 96c0341432SJohn Baldwin 97091d81d1SSam Leffler /* 98091d81d1SSam Leffler * Apply a symmetric encryption/decryption algorithm. 99091d81d1SSam Leffler */ 100091d81d1SSam Leffler static int 101c0341432SJohn Baldwin swcr_encdec(struct swcr_session *ses, struct cryptop *crp) 102091d81d1SSam Leffler { 1035d7ae54aSConrad Meyer unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; 10408fca7a5SJohn-Mark Gurney unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; 105c0341432SJohn Baldwin const struct crypto_session_params *csp; 106c0341432SJohn Baldwin struct swcr_encdec *sw; 107091d81d1SSam Leffler struct enc_xform *exf; 1089c0e3d3aSJohn Baldwin int i, blks, inlen, ivlen, outlen, resid; 1099c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 11026d292d3SJohn Baldwin const unsigned char *inblk; 11126d292d3SJohn Baldwin unsigned char *outblk; 11208fca7a5SJohn-Mark Gurney int error; 113c0341432SJohn Baldwin bool encrypting; 11408fca7a5SJohn-Mark Gurney 11508fca7a5SJohn-Mark Gurney error = 0; 116091d81d1SSam Leffler 117c0341432SJohn Baldwin sw = &ses->swcr_encdec; 118091d81d1SSam Leffler exf = sw->sw_exf; 11908fca7a5SJohn-Mark Gurney ivlen = exf->ivsize; 120091d81d1SSam Leffler 121723d8764SJohn Baldwin if (exf->native_blocksize == 0) { 122091d81d1SSam Leffler /* Check for non-padded data */ 123723d8764SJohn Baldwin if ((crp->crp_payload_length % exf->blocksize) != 0) 124723d8764SJohn Baldwin return (EINVAL); 125723d8764SJohn Baldwin 126723d8764SJohn Baldwin blks = exf->blocksize; 127723d8764SJohn Baldwin } else 128723d8764SJohn Baldwin blks = exf->native_blocksize; 129091d81d1SSam Leffler 130c0341432SJohn Baldwin if (exf == &enc_xform_aes_icm && 131c0341432SJohn Baldwin (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 13208fca7a5SJohn-Mark Gurney return (EINVAL); 13308fca7a5SJohn-Mark Gurney 134c0341432SJohn Baldwin if (crp->crp_cipher_key != NULL) { 135c0341432SJohn Baldwin csp = crypto_get_params(crp->crp_session); 1363e947048SJohn Baldwin error = exf->setkey(sw->sw_kschedule, 137c0341432SJohn Baldwin crp->crp_cipher_key, csp->csp_cipher_klen); 138c740ae4bSPoul-Henning Kamp if (error) 139c740ae4bSPoul-Henning Kamp return (error); 140c740ae4bSPoul-Henning Kamp } 141d295bdeeSPawel Jakub Dawidek 14220c128daSJohn Baldwin crypto_read_iv(crp, iv); 14320c128daSJohn Baldwin 14408fca7a5SJohn-Mark Gurney if (exf->reinit) { 145d295bdeeSPawel Jakub Dawidek /* 146d295bdeeSPawel Jakub Dawidek * xforms that provide a reinit method perform all IV 147d295bdeeSPawel Jakub Dawidek * handling themselves. 148d295bdeeSPawel Jakub Dawidek */ 149d295bdeeSPawel Jakub Dawidek exf->reinit(sw->sw_kschedule, iv); 150091d81d1SSam Leffler } 151091d81d1SSam Leffler 1529c0e3d3aSJohn Baldwin ivp = iv; 153091d81d1SSam Leffler 1549c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 1559c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1569c0e3d3aSJohn Baldwin inlen = crypto_cursor_seglen(&cc_in); 1579c0e3d3aSJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 1589c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1599c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 1609c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1619c0e3d3aSJohn Baldwin } else 1629c0e3d3aSJohn Baldwin cc_out = cc_in; 1639c0e3d3aSJohn Baldwin outlen = crypto_cursor_seglen(&cc_out); 1649c0e3d3aSJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 1659c0e3d3aSJohn Baldwin 1669c0e3d3aSJohn Baldwin resid = crp->crp_payload_length; 167c0341432SJohn Baldwin encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 168091d81d1SSam Leffler 169091d81d1SSam Leffler /* 1709c0e3d3aSJohn Baldwin * Loop through encrypting blocks. 'inlen' is the remaining 1719c0e3d3aSJohn Baldwin * length of the current segment in the input buffer. 1729c0e3d3aSJohn Baldwin * 'outlen' is the remaining length of current segment in the 1739c0e3d3aSJohn Baldwin * output buffer. 174091d81d1SSam Leffler */ 1759c0e3d3aSJohn Baldwin while (resid >= blks) { 1769c0e3d3aSJohn Baldwin /* 1779c0e3d3aSJohn Baldwin * If the current block is not contained within the 1789c0e3d3aSJohn Baldwin * current input/output segment, use 'blk' as a local 1799c0e3d3aSJohn Baldwin * buffer. 1809c0e3d3aSJohn Baldwin */ 1819c0e3d3aSJohn Baldwin if (inlen < blks) { 1829c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, blks, blk); 1839c0e3d3aSJohn Baldwin inblk = blk; 184d295bdeeSPawel Jakub Dawidek } 1859c0e3d3aSJohn Baldwin if (outlen < blks) 1869c0e3d3aSJohn Baldwin outblk = blk; 1879c0e3d3aSJohn Baldwin 1889c0e3d3aSJohn Baldwin /* 1899c0e3d3aSJohn Baldwin * Ciphers without a 'reinit' hook are assumed to be 1909c0e3d3aSJohn Baldwin * used in CBC mode where the chaining is done here. 1919c0e3d3aSJohn Baldwin */ 1929c0e3d3aSJohn Baldwin if (exf->reinit != NULL) { 1939c0e3d3aSJohn Baldwin if (encrypting) 1949c0e3d3aSJohn Baldwin exf->encrypt(sw->sw_kschedule, inblk, outblk); 1959c0e3d3aSJohn Baldwin else 1969c0e3d3aSJohn Baldwin exf->decrypt(sw->sw_kschedule, inblk, outblk); 197c0341432SJohn Baldwin } else if (encrypting) { 198091d81d1SSam Leffler /* XOR with previous block */ 1999c0e3d3aSJohn Baldwin for (i = 0; i < blks; i++) 2009c0e3d3aSJohn Baldwin outblk[i] = inblk[i] ^ ivp[i]; 201091d81d1SSam Leffler 2029c0e3d3aSJohn Baldwin exf->encrypt(sw->sw_kschedule, outblk, outblk); 203091d81d1SSam Leffler 204091d81d1SSam Leffler /* 205091d81d1SSam Leffler * Keep encrypted block for XOR'ing 206091d81d1SSam Leffler * with next block 207091d81d1SSam Leffler */ 2089c0e3d3aSJohn Baldwin memcpy(iv, outblk, blks); 209091d81d1SSam Leffler ivp = iv; 210091d81d1SSam Leffler } else { /* decrypt */ 211091d81d1SSam Leffler /* 212091d81d1SSam Leffler * Keep encrypted block for XOR'ing 213091d81d1SSam Leffler * with next block 214091d81d1SSam Leffler */ 21508fca7a5SJohn-Mark Gurney nivp = (ivp == iv) ? iv2 : iv; 2169c0e3d3aSJohn Baldwin memcpy(nivp, inblk, blks); 217091d81d1SSam Leffler 2189c0e3d3aSJohn Baldwin exf->decrypt(sw->sw_kschedule, inblk, outblk); 219091d81d1SSam Leffler 220091d81d1SSam Leffler /* XOR with previous block */ 2219c0e3d3aSJohn Baldwin for (i = 0; i < blks; i++) 2229c0e3d3aSJohn Baldwin outblk[i] ^= ivp[i]; 223091d81d1SSam Leffler 22408fca7a5SJohn-Mark Gurney ivp = nivp; 225091d81d1SSam Leffler } 226091d81d1SSam Leffler 2279c0e3d3aSJohn Baldwin if (inlen < blks) { 2289c0e3d3aSJohn Baldwin inlen = crypto_cursor_seglen(&cc_in); 2299c0e3d3aSJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 2309c0e3d3aSJohn Baldwin } else { 2319c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, blks); 2329c0e3d3aSJohn Baldwin inlen -= blks; 2339c0e3d3aSJohn Baldwin inblk += blks; 23408fca7a5SJohn-Mark Gurney } 235091d81d1SSam Leffler 2369c0e3d3aSJohn Baldwin if (outlen < blks) { 2379c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, blks, blk); 2389c0e3d3aSJohn Baldwin outlen = crypto_cursor_seglen(&cc_out); 2399c0e3d3aSJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 2409c0e3d3aSJohn Baldwin } else { 2419c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, blks); 2429c0e3d3aSJohn Baldwin outlen -= blks; 2439c0e3d3aSJohn Baldwin outblk += blks; 244091d81d1SSam Leffler } 245091d81d1SSam Leffler 2469c0e3d3aSJohn Baldwin resid -= blks; 247f34a967bSPawel Jakub Dawidek } 248f34a967bSPawel Jakub Dawidek 249723d8764SJohn Baldwin /* Handle trailing partial block for stream ciphers. */ 2509c0e3d3aSJohn Baldwin if (resid > 0) { 251723d8764SJohn Baldwin KASSERT(exf->native_blocksize != 0, 252723d8764SJohn Baldwin ("%s: partial block of %d bytes for cipher %s", 253723d8764SJohn Baldwin __func__, i, exf->name)); 254723d8764SJohn Baldwin KASSERT(exf->reinit != NULL, 255723d8764SJohn Baldwin ("%s: partial block cipher %s without reinit hook", 256723d8764SJohn Baldwin __func__, exf->name)); 2579c0e3d3aSJohn Baldwin KASSERT(resid < blks, ("%s: partial block too big", __func__)); 258723d8764SJohn Baldwin 2599c0e3d3aSJohn Baldwin inlen = crypto_cursor_seglen(&cc_in); 2609c0e3d3aSJohn Baldwin outlen = crypto_cursor_seglen(&cc_out); 2619c0e3d3aSJohn Baldwin if (inlen < resid) { 2629c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 2639c0e3d3aSJohn Baldwin inblk = blk; 2649c0e3d3aSJohn Baldwin } else 2659c0e3d3aSJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 2669c0e3d3aSJohn Baldwin if (outlen < resid) 2679c0e3d3aSJohn Baldwin outblk = blk; 2689c0e3d3aSJohn Baldwin else 2699c0e3d3aSJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 2709c0e3d3aSJohn Baldwin if (encrypting) 2719c0e3d3aSJohn Baldwin exf->encrypt_last(sw->sw_kschedule, inblk, outblk, 2729c0e3d3aSJohn Baldwin resid); 2739c0e3d3aSJohn Baldwin else 2749c0e3d3aSJohn Baldwin exf->decrypt_last(sw->sw_kschedule, inblk, outblk, 2759c0e3d3aSJohn Baldwin resid); 2769c0e3d3aSJohn Baldwin if (outlen < resid) 2779c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 278723d8764SJohn Baldwin } 279723d8764SJohn Baldwin 28020c128daSJohn Baldwin explicit_bzero(blk, sizeof(blk)); 28120c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 28220c128daSJohn Baldwin explicit_bzero(iv2, sizeof(iv2)); 2839c0e3d3aSJohn Baldwin return (0); 284091d81d1SSam Leffler } 285091d81d1SSam Leffler 286c0341432SJohn Baldwin static void 287c0341432SJohn Baldwin swcr_authprepare(struct auth_hash *axf, struct swcr_auth *sw, 288c0341432SJohn Baldwin const uint8_t *key, int klen) 289f6c4bc3bSPawel Jakub Dawidek { 290f6c4bc3bSPawel Jakub Dawidek 291f6c4bc3bSPawel Jakub Dawidek switch (axf->type) { 292f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA1_HMAC: 293c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 294f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 295f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 296f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 297f6c4bc3bSPawel Jakub Dawidek case CRYPTO_NULL_HMAC: 298f6c4bc3bSPawel Jakub Dawidek case CRYPTO_RIPEMD160_HMAC: 299c0341432SJohn Baldwin hmac_init_ipad(axf, key, klen, sw->sw_ictx); 300c0341432SJohn Baldwin hmac_init_opad(axf, key, klen, sw->sw_octx); 301f6c4bc3bSPawel Jakub Dawidek break; 30225b7033bSConrad Meyer case CRYPTO_POLY1305: 3030e33efe4SConrad Meyer case CRYPTO_BLAKE2B: 3040e33efe4SConrad Meyer case CRYPTO_BLAKE2S: 3050e33efe4SConrad Meyer axf->Setkey(sw->sw_ictx, key, klen); 3060e33efe4SConrad Meyer axf->Init(sw->sw_ictx); 3070e33efe4SConrad Meyer break; 308f6c4bc3bSPawel Jakub Dawidek default: 309c0341432SJohn Baldwin panic("%s: algorithm %d doesn't use keys", __func__, axf->type); 310f6c4bc3bSPawel Jakub Dawidek } 311f6c4bc3bSPawel Jakub Dawidek } 312f6c4bc3bSPawel Jakub Dawidek 313091d81d1SSam Leffler /* 314c0341432SJohn Baldwin * Compute or verify hash. 315091d81d1SSam Leffler */ 316091d81d1SSam Leffler static int 317c0341432SJohn Baldwin swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) 318091d81d1SSam Leffler { 319c0341432SJohn Baldwin u_char aalg[HASH_MAX_LEN]; 320c0341432SJohn Baldwin const struct crypto_session_params *csp; 321c0341432SJohn Baldwin struct swcr_auth *sw; 322091d81d1SSam Leffler struct auth_hash *axf; 323091d81d1SSam Leffler union authctx ctx; 324091d81d1SSam Leffler int err; 325091d81d1SSam Leffler 326c0341432SJohn Baldwin sw = &ses->swcr_auth; 327091d81d1SSam Leffler 328091d81d1SSam Leffler axf = sw->sw_axf; 329091d81d1SSam Leffler 330c0341432SJohn Baldwin csp = crypto_get_params(crp->crp_session); 3316038018aSMarcin Wojtas if (crp->crp_auth_key != NULL) { 332c0341432SJohn Baldwin swcr_authprepare(axf, sw, crp->crp_auth_key, 333c0341432SJohn Baldwin csp->csp_auth_klen); 33425b7033bSConrad Meyer } 335f6c4bc3bSPawel Jakub Dawidek 336091d81d1SSam Leffler bcopy(sw->sw_ictx, &ctx, axf->ctxsize); 337091d81d1SSam Leffler 3389b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 3399b774dc0SJohn Baldwin err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 3409b774dc0SJohn Baldwin else 341c0341432SJohn Baldwin err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 3429b6b2f86SJohn Baldwin axf->Update, &ctx); 343091d81d1SSam Leffler if (err) 344e0b155feSJohn Baldwin goto out; 345091d81d1SSam Leffler 3469c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 3479c0e3d3aSJohn Baldwin CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 3489c0e3d3aSJohn Baldwin err = crypto_apply_buf(&crp->crp_obuf, 3499c0e3d3aSJohn Baldwin crp->crp_payload_output_start, crp->crp_payload_length, 3509b6b2f86SJohn Baldwin axf->Update, &ctx); 3519c0e3d3aSJohn Baldwin else 3529c0e3d3aSJohn Baldwin err = crypto_apply(crp, crp->crp_payload_start, 3539b6b2f86SJohn Baldwin crp->crp_payload_length, axf->Update, &ctx); 354c0341432SJohn Baldwin if (err) 355e0b155feSJohn Baldwin goto out; 356091d81d1SSam Leffler 3576038018aSMarcin Wojtas if (csp->csp_flags & CSP_F_ESN) 3586038018aSMarcin Wojtas axf->Update(&ctx, crp->crp_esn, 4); 3596038018aSMarcin Wojtas 360091d81d1SSam Leffler axf->Final(aalg, &ctx); 361e0b155feSJohn Baldwin if (sw->sw_octx != NULL) { 362091d81d1SSam Leffler bcopy(sw->sw_octx, &ctx, axf->ctxsize); 363091d81d1SSam Leffler axf->Update(&ctx, aalg, axf->hashsize); 364091d81d1SSam Leffler axf->Final(aalg, &ctx); 365091d81d1SSam Leffler } 366091d81d1SSam Leffler 367c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 36820c128daSJohn Baldwin u_char uaalg[HASH_MAX_LEN]; 36920c128daSJohn Baldwin 370c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 371c0341432SJohn Baldwin if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 37220c128daSJohn Baldwin err = EBADMSG; 37320c128daSJohn Baldwin explicit_bzero(uaalg, sizeof(uaalg)); 374c0341432SJohn Baldwin } else { 375091d81d1SSam Leffler /* Inject the authentication data */ 376c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 377c0341432SJohn Baldwin } 37820c128daSJohn Baldwin explicit_bzero(aalg, sizeof(aalg)); 379e0b155feSJohn Baldwin out: 380e0b155feSJohn Baldwin explicit_bzero(&ctx, sizeof(ctx)); 38120c128daSJohn Baldwin return (err); 382091d81d1SSam Leffler } 383091d81d1SSam Leffler 38408fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 38508fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 38608fca7a5SJohn-Mark Gurney 38708fca7a5SJohn-Mark Gurney static int 388c0341432SJohn Baldwin swcr_gmac(struct swcr_session *ses, struct cryptop *crp) 38908fca7a5SJohn-Mark Gurney { 39026d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 39108fca7a5SJohn-Mark Gurney u_char *blk = (u_char *)blkbuf; 39226d292d3SJohn Baldwin u_char tag[GMAC_DIGEST_LEN]; 39326d292d3SJohn Baldwin u_char iv[AES_BLOCK_LEN]; 3949c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc; 39526d292d3SJohn Baldwin const u_char *inblk; 39608fca7a5SJohn-Mark Gurney union authctx ctx; 397c0341432SJohn Baldwin struct swcr_auth *swa; 398c0341432SJohn Baldwin struct auth_hash *axf; 39908fca7a5SJohn-Mark Gurney uint32_t *blkp; 40020c128daSJohn Baldwin int blksz, error, ivlen, len, resid; 40108fca7a5SJohn-Mark Gurney 402c0341432SJohn Baldwin swa = &ses->swcr_auth; 40308fca7a5SJohn-Mark Gurney axf = swa->sw_axf; 404c0341432SJohn Baldwin 40508fca7a5SJohn-Mark Gurney bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 40626d292d3SJohn Baldwin blksz = GMAC_BLOCK_LEN; 40726d292d3SJohn Baldwin KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 40826d292d3SJohn Baldwin __func__)); 40908fca7a5SJohn-Mark Gurney 41008fca7a5SJohn-Mark Gurney /* Initialize the IV */ 411c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 41229fe41ddSJohn Baldwin crypto_read_iv(crp, iv); 41308fca7a5SJohn-Mark Gurney 41408fca7a5SJohn-Mark Gurney axf->Reinit(&ctx, iv, ivlen); 4159c0e3d3aSJohn Baldwin crypto_cursor_init(&cc, &crp->crp_buf); 4169c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc, crp->crp_payload_start); 41726d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 41826d292d3SJohn Baldwin len = crypto_cursor_seglen(&cc); 41926d292d3SJohn Baldwin if (len >= blksz) { 42026d292d3SJohn Baldwin inblk = crypto_cursor_segbase(&cc); 42126d292d3SJohn Baldwin len = rounddown(MIN(len, resid), blksz); 42226d292d3SJohn Baldwin crypto_cursor_advance(&cc, len); 42326d292d3SJohn Baldwin } else { 42426d292d3SJohn Baldwin len = blksz; 4259c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc, len, blk); 42626d292d3SJohn Baldwin inblk = blk; 42726d292d3SJohn Baldwin } 42826d292d3SJohn Baldwin axf->Update(&ctx, inblk, len); 42926d292d3SJohn Baldwin } 43026d292d3SJohn Baldwin if (resid > 0) { 43126d292d3SJohn Baldwin memset(blk, 0, blksz); 43226d292d3SJohn Baldwin crypto_cursor_copydata(&cc, resid, blk); 43308fca7a5SJohn-Mark Gurney axf->Update(&ctx, blk, blksz); 43408fca7a5SJohn-Mark Gurney } 43508fca7a5SJohn-Mark Gurney 43608fca7a5SJohn-Mark Gurney /* length block */ 43726d292d3SJohn Baldwin memset(blk, 0, blksz); 43808fca7a5SJohn-Mark Gurney blkp = (uint32_t *)blk + 1; 439c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 44008fca7a5SJohn-Mark Gurney axf->Update(&ctx, blk, blksz); 441c0341432SJohn Baldwin 442c0341432SJohn Baldwin /* Finalize MAC */ 44326d292d3SJohn Baldwin axf->Final(tag, &ctx); 444c0341432SJohn Baldwin 44520c128daSJohn Baldwin error = 0; 446c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 44726d292d3SJohn Baldwin u_char tag2[GMAC_DIGEST_LEN]; 44820c128daSJohn Baldwin 449c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 45026d292d3SJohn Baldwin tag2); 45126d292d3SJohn Baldwin if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 45220c128daSJohn Baldwin error = EBADMSG; 45326d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 454c0341432SJohn Baldwin } else { 455c0341432SJohn Baldwin /* Inject the authentication data */ 45626d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 457c0341432SJohn Baldwin } 45820c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 45926d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 46020c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 46120c128daSJohn Baldwin return (error); 462c0341432SJohn Baldwin } 463c0341432SJohn Baldwin 464c0341432SJohn Baldwin static int 465c0341432SJohn Baldwin swcr_gcm(struct swcr_session *ses, struct cryptop *crp) 466c0341432SJohn Baldwin { 46726d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 468c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 46926d292d3SJohn Baldwin u_char tag[GMAC_DIGEST_LEN]; 47026d292d3SJohn Baldwin u_char iv[AES_BLOCK_LEN]; 4719c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 47226d292d3SJohn Baldwin const u_char *inblk; 47326d292d3SJohn Baldwin u_char *outblk; 474c0341432SJohn Baldwin union authctx ctx; 475c0341432SJohn Baldwin struct swcr_auth *swa; 476c0341432SJohn Baldwin struct swcr_encdec *swe; 477c0341432SJohn Baldwin struct auth_hash *axf; 478c0341432SJohn Baldwin struct enc_xform *exf; 479c0341432SJohn Baldwin uint32_t *blkp; 48020c128daSJohn Baldwin int blksz, error, ivlen, len, r, resid; 481c0341432SJohn Baldwin 482c0341432SJohn Baldwin swa = &ses->swcr_auth; 483c0341432SJohn Baldwin axf = swa->sw_axf; 484c0341432SJohn Baldwin 485c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 48626d292d3SJohn Baldwin blksz = GMAC_BLOCK_LEN; 48726d292d3SJohn Baldwin KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 48826d292d3SJohn Baldwin __func__)); 489c0341432SJohn Baldwin 490c0341432SJohn Baldwin swe = &ses->swcr_encdec; 491c0341432SJohn Baldwin exf = swe->sw_exf; 492723d8764SJohn Baldwin KASSERT(axf->blocksize == exf->native_blocksize, 493723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 494c0341432SJohn Baldwin 495c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 496c0341432SJohn Baldwin return (EINVAL); 497c0341432SJohn Baldwin 498c0341432SJohn Baldwin /* Initialize the IV */ 499c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 500c0341432SJohn Baldwin bcopy(crp->crp_iv, iv, ivlen); 501c0341432SJohn Baldwin 502c0341432SJohn Baldwin /* Supply MAC with IV */ 503c0341432SJohn Baldwin axf->Reinit(&ctx, iv, ivlen); 504c0341432SJohn Baldwin 505c0341432SJohn Baldwin /* Supply MAC with AAD */ 5069b774dc0SJohn Baldwin if (crp->crp_aad != NULL) { 5079b774dc0SJohn Baldwin len = rounddown(crp->crp_aad_length, blksz); 5089b774dc0SJohn Baldwin if (len != 0) 5099b774dc0SJohn Baldwin axf->Update(&ctx, crp->crp_aad, len); 5109b774dc0SJohn Baldwin if (crp->crp_aad_length != len) { 5119b774dc0SJohn Baldwin memset(blk, 0, blksz); 5129b774dc0SJohn Baldwin memcpy(blk, (char *)crp->crp_aad + len, 5139b774dc0SJohn Baldwin crp->crp_aad_length - len); 5149b774dc0SJohn Baldwin axf->Update(&ctx, blk, blksz); 5159b774dc0SJohn Baldwin } 5169b774dc0SJohn Baldwin } else { 5179c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5189c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_aad_start); 5199b774dc0SJohn Baldwin for (resid = crp->crp_aad_length; resid >= blksz; 5209b774dc0SJohn Baldwin resid -= len) { 52126d292d3SJohn Baldwin len = crypto_cursor_seglen(&cc_in); 52226d292d3SJohn Baldwin if (len >= blksz) { 52326d292d3SJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 52426d292d3SJohn Baldwin len = rounddown(MIN(len, resid), blksz); 52526d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, len); 52626d292d3SJohn Baldwin } else { 52726d292d3SJohn Baldwin len = blksz; 5289c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 52926d292d3SJohn Baldwin inblk = blk; 53026d292d3SJohn Baldwin } 53126d292d3SJohn Baldwin axf->Update(&ctx, inblk, len); 53226d292d3SJohn Baldwin } 53326d292d3SJohn Baldwin if (resid > 0) { 53426d292d3SJohn Baldwin memset(blk, 0, blksz); 53526d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 536c0341432SJohn Baldwin axf->Update(&ctx, blk, blksz); 537c0341432SJohn Baldwin } 5389b774dc0SJohn Baldwin } 539c0341432SJohn Baldwin 540a10020cfSJohn Baldwin if (crp->crp_cipher_key != NULL) 541a10020cfSJohn Baldwin exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 542a10020cfSJohn Baldwin crypto_get_params(crp->crp_session)->csp_cipher_klen); 543c0341432SJohn Baldwin exf->reinit(swe->sw_kschedule, iv); 544c0341432SJohn Baldwin 545c0341432SJohn Baldwin /* Do encryption with MAC */ 5469c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5479c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 5489c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 5499c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 5509c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 5519c0e3d3aSJohn Baldwin } else 5529c0e3d3aSJohn Baldwin cc_out = cc_in; 55326d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 55426d292d3SJohn Baldwin if (crypto_cursor_seglen(&cc_in) < blksz) { 55526d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 55626d292d3SJohn Baldwin inblk = blk; 557c0341432SJohn Baldwin } else { 55826d292d3SJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 55926d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 560c0341432SJohn Baldwin } 56126d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 56226d292d3SJohn Baldwin if (crypto_cursor_seglen(&cc_out) < blksz) 56326d292d3SJohn Baldwin outblk = blk; 56426d292d3SJohn Baldwin else 56526d292d3SJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 56626d292d3SJohn Baldwin exf->encrypt(swe->sw_kschedule, inblk, outblk); 56726d292d3SJohn Baldwin axf->Update(&ctx, outblk, blksz); 56826d292d3SJohn Baldwin if (outblk == blk) 56926d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 57026d292d3SJohn Baldwin else 57126d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 57226d292d3SJohn Baldwin } else { 57326d292d3SJohn Baldwin axf->Update(&ctx, inblk, blksz); 57426d292d3SJohn Baldwin } 57526d292d3SJohn Baldwin } 57626d292d3SJohn Baldwin if (resid > 0) { 57726d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 57826d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 57926d292d3SJohn Baldwin exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 58026d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 58126d292d3SJohn Baldwin } 58226d292d3SJohn Baldwin axf->Update(&ctx, blk, resid); 583c0341432SJohn Baldwin } 584c0341432SJohn Baldwin 585c0341432SJohn Baldwin /* length block */ 58626d292d3SJohn Baldwin memset(blk, 0, blksz); 587c0341432SJohn Baldwin blkp = (uint32_t *)blk + 1; 588c0341432SJohn Baldwin *blkp = htobe32(crp->crp_aad_length * 8); 589c0341432SJohn Baldwin blkp = (uint32_t *)blk + 3; 590c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 591c0341432SJohn Baldwin axf->Update(&ctx, blk, blksz); 592c0341432SJohn Baldwin 593c0341432SJohn Baldwin /* Finalize MAC */ 59426d292d3SJohn Baldwin axf->Final(tag, &ctx); 595c0341432SJohn Baldwin 596c0341432SJohn Baldwin /* Validate tag */ 59720c128daSJohn Baldwin error = 0; 598c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 59926d292d3SJohn Baldwin u_char tag2[GMAC_DIGEST_LEN]; 60020c128daSJohn Baldwin 60126d292d3SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 602c0341432SJohn Baldwin 60326d292d3SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 60426d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 60520c128daSJohn Baldwin if (r != 0) { 60620c128daSJohn Baldwin error = EBADMSG; 60720c128daSJohn Baldwin goto out; 60820c128daSJohn Baldwin } 609c0341432SJohn Baldwin 610c0341432SJohn Baldwin /* tag matches, decrypt data */ 6119c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 6129c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 61326d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 61426d292d3SJohn Baldwin resid -= blksz) { 61526d292d3SJohn Baldwin if (crypto_cursor_seglen(&cc_in) < blksz) { 61626d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 61726d292d3SJohn Baldwin inblk = blk; 61826d292d3SJohn Baldwin } else { 61926d292d3SJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 62026d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 62126d292d3SJohn Baldwin } 62226d292d3SJohn Baldwin if (crypto_cursor_seglen(&cc_out) < blksz) 62326d292d3SJohn Baldwin outblk = blk; 62426d292d3SJohn Baldwin else 62526d292d3SJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 62626d292d3SJohn Baldwin exf->decrypt(swe->sw_kschedule, inblk, outblk); 62726d292d3SJohn Baldwin if (outblk == blk) 62826d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 62926d292d3SJohn Baldwin else 63026d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 63126d292d3SJohn Baldwin } 63226d292d3SJohn Baldwin if (resid > 0) { 63326d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 63426d292d3SJohn Baldwin exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 63526d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 636c0341432SJohn Baldwin } 637c0341432SJohn Baldwin } else { 638c0341432SJohn Baldwin /* Inject the authentication data */ 63926d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 640c0341432SJohn Baldwin } 641c0341432SJohn Baldwin 64220c128daSJohn Baldwin out: 64320c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 64426d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 64520c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 64620c128daSJohn Baldwin 64720c128daSJohn Baldwin return (error); 648c0341432SJohn Baldwin } 649c0341432SJohn Baldwin 650c0341432SJohn Baldwin static int 651c0341432SJohn Baldwin swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) 652c0341432SJohn Baldwin { 65326d292d3SJohn Baldwin u_char tag[AES_CBC_MAC_HASH_LEN]; 65426d292d3SJohn Baldwin u_char iv[AES_BLOCK_LEN]; 655c0341432SJohn Baldwin union authctx ctx; 656c0341432SJohn Baldwin struct swcr_auth *swa; 657c0341432SJohn Baldwin struct auth_hash *axf; 65826d292d3SJohn Baldwin int error, ivlen; 659c0341432SJohn Baldwin 660c0341432SJohn Baldwin swa = &ses->swcr_auth; 661c0341432SJohn Baldwin axf = swa->sw_axf; 662c0341432SJohn Baldwin 663c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 664c0341432SJohn Baldwin 665c0341432SJohn Baldwin /* Initialize the IV */ 666c0341432SJohn Baldwin ivlen = AES_CCM_IV_LEN; 66729fe41ddSJohn Baldwin crypto_read_iv(crp, iv); 668c0341432SJohn Baldwin 669c0341432SJohn Baldwin /* 670c0341432SJohn Baldwin * AES CCM-CBC-MAC needs to know the length of both the auth 671c0341432SJohn Baldwin * data and payload data before doing the auth computation. 672c0341432SJohn Baldwin */ 673c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_payload_length; 674c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.cryptDataLength = 0; 675c0341432SJohn Baldwin 676c0341432SJohn Baldwin axf->Reinit(&ctx, iv, ivlen); 6779b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 6789b774dc0SJohn Baldwin error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 6799b774dc0SJohn Baldwin else 68026d292d3SJohn Baldwin error = crypto_apply(crp, crp->crp_payload_start, 68126d292d3SJohn Baldwin crp->crp_payload_length, axf->Update, &ctx); 68226d292d3SJohn Baldwin if (error) 68326d292d3SJohn Baldwin return (error); 684c0341432SJohn Baldwin 685c0341432SJohn Baldwin /* Finalize MAC */ 68626d292d3SJohn Baldwin axf->Final(tag, &ctx); 687c0341432SJohn Baldwin 688c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 68926d292d3SJohn Baldwin u_char tag2[AES_CBC_MAC_HASH_LEN]; 69020c128daSJohn Baldwin 691c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 69226d292d3SJohn Baldwin tag2); 69326d292d3SJohn Baldwin if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 69420c128daSJohn Baldwin error = EBADMSG; 69526d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag)); 696c0341432SJohn Baldwin } else { 697c0341432SJohn Baldwin /* Inject the authentication data */ 69826d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 699c0341432SJohn Baldwin } 70026d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 70120c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 70220c128daSJohn Baldwin return (error); 703c0341432SJohn Baldwin } 704c0341432SJohn Baldwin 705c0341432SJohn Baldwin static int 706c0341432SJohn Baldwin swcr_ccm(struct swcr_session *ses, struct cryptop *crp) 707c0341432SJohn Baldwin { 70826d292d3SJohn Baldwin uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 709c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 71026d292d3SJohn Baldwin u_char tag[AES_CBC_MAC_HASH_LEN]; 71126d292d3SJohn Baldwin u_char iv[AES_BLOCK_LEN]; 7129c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 71326d292d3SJohn Baldwin const u_char *inblk; 71426d292d3SJohn Baldwin u_char *outblk; 715c0341432SJohn Baldwin union authctx ctx; 716c0341432SJohn Baldwin struct swcr_auth *swa; 717c0341432SJohn Baldwin struct swcr_encdec *swe; 718c0341432SJohn Baldwin struct auth_hash *axf; 719c0341432SJohn Baldwin struct enc_xform *exf; 72026d292d3SJohn Baldwin int blksz, error, ivlen, r, resid; 721c0341432SJohn Baldwin 722c0341432SJohn Baldwin swa = &ses->swcr_auth; 723c0341432SJohn Baldwin axf = swa->sw_axf; 724c0341432SJohn Baldwin 725c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 72626d292d3SJohn Baldwin blksz = AES_BLOCK_LEN; 72726d292d3SJohn Baldwin KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 72826d292d3SJohn Baldwin __func__)); 729c0341432SJohn Baldwin 730c0341432SJohn Baldwin swe = &ses->swcr_encdec; 731c0341432SJohn Baldwin exf = swe->sw_exf; 732723d8764SJohn Baldwin KASSERT(axf->blocksize == exf->native_blocksize, 733723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 734c0341432SJohn Baldwin 735c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 736c0341432SJohn Baldwin return (EINVAL); 737c0341432SJohn Baldwin 738c0341432SJohn Baldwin /* Initialize the IV */ 739c0341432SJohn Baldwin ivlen = AES_CCM_IV_LEN; 740c0341432SJohn Baldwin bcopy(crp->crp_iv, iv, ivlen); 741c0341432SJohn Baldwin 742c0341432SJohn Baldwin /* 743c0341432SJohn Baldwin * AES CCM-CBC-MAC needs to know the length of both the auth 744c0341432SJohn Baldwin * data and payload data before doing the auth computation. 745c0341432SJohn Baldwin */ 746c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; 747c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; 748c0341432SJohn Baldwin 749c0341432SJohn Baldwin /* Supply MAC with IV */ 750c0341432SJohn Baldwin axf->Reinit(&ctx, iv, ivlen); 751c0341432SJohn Baldwin 752c0341432SJohn Baldwin /* Supply MAC with AAD */ 7539b774dc0SJohn Baldwin if (crp->crp_aad != NULL) 7549b774dc0SJohn Baldwin error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 7559b774dc0SJohn Baldwin else 7569b774dc0SJohn Baldwin error = crypto_apply(crp, crp->crp_aad_start, 7579b774dc0SJohn Baldwin crp->crp_aad_length, axf->Update, &ctx); 75826d292d3SJohn Baldwin if (error) 75926d292d3SJohn Baldwin return (error); 760c0341432SJohn Baldwin 761a10020cfSJohn Baldwin if (crp->crp_cipher_key != NULL) 762a10020cfSJohn Baldwin exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 763a10020cfSJohn Baldwin crypto_get_params(crp->crp_session)->csp_cipher_klen); 764c0341432SJohn Baldwin exf->reinit(swe->sw_kschedule, iv); 765c0341432SJohn Baldwin 766c0341432SJohn Baldwin /* Do encryption/decryption with MAC */ 7679c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 7689c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 7699c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 7709c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 7719c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 7729c0e3d3aSJohn Baldwin } else 7739c0e3d3aSJohn Baldwin cc_out = cc_in; 77426d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 77526d292d3SJohn Baldwin if (crypto_cursor_seglen(&cc_in) < blksz) { 77626d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 77726d292d3SJohn Baldwin inblk = blk; 77826d292d3SJohn Baldwin } else { 77926d292d3SJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 78026d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 78126d292d3SJohn Baldwin } 782c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 78326d292d3SJohn Baldwin if (crypto_cursor_seglen(&cc_out) < blksz) 78426d292d3SJohn Baldwin outblk = blk; 78526d292d3SJohn Baldwin else 78626d292d3SJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 78726d292d3SJohn Baldwin axf->Update(&ctx, inblk, blksz); 78826d292d3SJohn Baldwin exf->encrypt(swe->sw_kschedule, inblk, outblk); 78926d292d3SJohn Baldwin if (outblk == blk) 79026d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 79126d292d3SJohn Baldwin else 79226d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 793c0341432SJohn Baldwin } else { 794c0341432SJohn Baldwin /* 795c0341432SJohn Baldwin * One of the problems with CCM+CBC is that 796c0341432SJohn Baldwin * the authentication is done on the 79726d292d3SJohn Baldwin * unencrypted data. As a result, we have to 798c0341432SJohn Baldwin * decrypt the data twice: once to generate 799c0341432SJohn Baldwin * the tag and a second time after the tag is 800c0341432SJohn Baldwin * verified. 801c0341432SJohn Baldwin */ 80226d292d3SJohn Baldwin exf->decrypt(swe->sw_kschedule, inblk, blk); 80326d292d3SJohn Baldwin axf->Update(&ctx, blk, blksz); 80426d292d3SJohn Baldwin } 80526d292d3SJohn Baldwin } 80626d292d3SJohn Baldwin if (resid > 0) { 80726d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 80826d292d3SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 80926d292d3SJohn Baldwin axf->Update(&ctx, blk, resid); 81026d292d3SJohn Baldwin exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 81126d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 81226d292d3SJohn Baldwin } else { 81326d292d3SJohn Baldwin exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 81426d292d3SJohn Baldwin axf->Update(&ctx, blk, resid); 815c0341432SJohn Baldwin } 81608fca7a5SJohn-Mark Gurney } 81708fca7a5SJohn-Mark Gurney 81808fca7a5SJohn-Mark Gurney /* Finalize MAC */ 81926d292d3SJohn Baldwin axf->Final(tag, &ctx); 82008fca7a5SJohn-Mark Gurney 82108fca7a5SJohn-Mark Gurney /* Validate tag */ 82220c128daSJohn Baldwin error = 0; 823c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 82426d292d3SJohn Baldwin u_char tag2[AES_CBC_MAC_HASH_LEN]; 82520c128daSJohn Baldwin 826c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 82726d292d3SJohn Baldwin tag2); 82808fca7a5SJohn-Mark Gurney 82926d292d3SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 83026d292d3SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 83120c128daSJohn Baldwin if (r != 0) { 83220c128daSJohn Baldwin error = EBADMSG; 83320c128daSJohn Baldwin goto out; 83420c128daSJohn Baldwin } 835c0341432SJohn Baldwin 83608fca7a5SJohn-Mark Gurney /* tag matches, decrypt data */ 837507281e5SSean Eric Fagan exf->reinit(swe->sw_kschedule, iv); 8389c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 8399c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 84026d292d3SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 84126d292d3SJohn Baldwin resid -= blksz) { 84226d292d3SJohn Baldwin if (crypto_cursor_seglen(&cc_in) < blksz) { 84326d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 84426d292d3SJohn Baldwin inblk = blk; 84526d292d3SJohn Baldwin } else { 84626d292d3SJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 84726d292d3SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 84826d292d3SJohn Baldwin } 84926d292d3SJohn Baldwin if (crypto_cursor_seglen(&cc_out) < blksz) 85026d292d3SJohn Baldwin outblk = blk; 85126d292d3SJohn Baldwin else 85226d292d3SJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 85326d292d3SJohn Baldwin exf->decrypt(swe->sw_kschedule, inblk, outblk); 85426d292d3SJohn Baldwin if (outblk == blk) 85526d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 85626d292d3SJohn Baldwin else 85726d292d3SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 85826d292d3SJohn Baldwin } 85926d292d3SJohn Baldwin if (resid > 0) { 86026d292d3SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 86126d292d3SJohn Baldwin exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 86226d292d3SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 86308fca7a5SJohn-Mark Gurney } 86408fca7a5SJohn-Mark Gurney } else { 86508fca7a5SJohn-Mark Gurney /* Inject the authentication data */ 86626d292d3SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 86708fca7a5SJohn-Mark Gurney } 86808fca7a5SJohn-Mark Gurney 86920c128daSJohn Baldwin out: 87020c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 87126d292d3SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 87220c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 87320c128daSJohn Baldwin return (error); 87408fca7a5SJohn-Mark Gurney } 87508fca7a5SJohn-Mark Gurney 876dd2e1352SJohn Baldwin static int 877dd2e1352SJohn Baldwin swcr_chacha20_poly1305(struct swcr_session *ses, struct cryptop *crp) 878dd2e1352SJohn Baldwin { 879dd2e1352SJohn Baldwin const struct crypto_session_params *csp; 880dd2e1352SJohn Baldwin uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 881dd2e1352SJohn Baldwin u_char *blk = (u_char *)blkbuf; 882dd2e1352SJohn Baldwin u_char tag[POLY1305_HASH_LEN]; 883dd2e1352SJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 884dd2e1352SJohn Baldwin const u_char *inblk; 885dd2e1352SJohn Baldwin u_char *outblk; 886dd2e1352SJohn Baldwin uint64_t *blkp; 887dd2e1352SJohn Baldwin union authctx ctx; 888dd2e1352SJohn Baldwin struct swcr_auth *swa; 889dd2e1352SJohn Baldwin struct swcr_encdec *swe; 890dd2e1352SJohn Baldwin struct auth_hash *axf; 891dd2e1352SJohn Baldwin struct enc_xform *exf; 892dd2e1352SJohn Baldwin int blksz, error, r, resid; 893dd2e1352SJohn Baldwin 894dd2e1352SJohn Baldwin swa = &ses->swcr_auth; 895dd2e1352SJohn Baldwin axf = swa->sw_axf; 896dd2e1352SJohn Baldwin 897dd2e1352SJohn Baldwin swe = &ses->swcr_encdec; 898dd2e1352SJohn Baldwin exf = swe->sw_exf; 899dd2e1352SJohn Baldwin blksz = exf->native_blocksize; 900dd2e1352SJohn Baldwin KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); 901dd2e1352SJohn Baldwin 902dd2e1352SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 903dd2e1352SJohn Baldwin return (EINVAL); 904dd2e1352SJohn Baldwin 905dd2e1352SJohn Baldwin csp = crypto_get_params(crp->crp_session); 906dd2e1352SJohn Baldwin 907dd2e1352SJohn Baldwin /* Generate Poly1305 key. */ 908dd2e1352SJohn Baldwin if (crp->crp_cipher_key != NULL) 909dd2e1352SJohn Baldwin axf->Setkey(&ctx, crp->crp_cipher_key, csp->csp_cipher_klen); 910dd2e1352SJohn Baldwin else 911dd2e1352SJohn Baldwin axf->Setkey(&ctx, csp->csp_cipher_key, csp->csp_cipher_klen); 912dd2e1352SJohn Baldwin axf->Reinit(&ctx, crp->crp_iv, csp->csp_ivlen); 913dd2e1352SJohn Baldwin 914dd2e1352SJohn Baldwin /* Supply MAC with AAD */ 915dd2e1352SJohn Baldwin if (crp->crp_aad != NULL) 916dd2e1352SJohn Baldwin axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 917dd2e1352SJohn Baldwin else 918dd2e1352SJohn Baldwin crypto_apply(crp, crp->crp_aad_start, 919dd2e1352SJohn Baldwin crp->crp_aad_length, axf->Update, &ctx); 920dd2e1352SJohn Baldwin if (crp->crp_aad_length % 16 != 0) { 921dd2e1352SJohn Baldwin /* padding1 */ 922dd2e1352SJohn Baldwin memset(blk, 0, 16); 923dd2e1352SJohn Baldwin axf->Update(&ctx, blk, 16 - crp->crp_aad_length % 16); 924dd2e1352SJohn Baldwin } 925dd2e1352SJohn Baldwin 926dd2e1352SJohn Baldwin if (crp->crp_cipher_key != NULL) 927dd2e1352SJohn Baldwin exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 928dd2e1352SJohn Baldwin csp->csp_cipher_klen); 929dd2e1352SJohn Baldwin exf->reinit(swe->sw_kschedule, crp->crp_iv); 930dd2e1352SJohn Baldwin 931dd2e1352SJohn Baldwin /* Do encryption with MAC */ 932dd2e1352SJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 933dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 934dd2e1352SJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 935dd2e1352SJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 936dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 937dd2e1352SJohn Baldwin } else 938dd2e1352SJohn Baldwin cc_out = cc_in; 939dd2e1352SJohn Baldwin for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 940dd2e1352SJohn Baldwin if (crypto_cursor_seglen(&cc_in) < blksz) { 941dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 942dd2e1352SJohn Baldwin inblk = blk; 943dd2e1352SJohn Baldwin } else { 944dd2e1352SJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 945dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 946dd2e1352SJohn Baldwin } 947dd2e1352SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 948dd2e1352SJohn Baldwin if (crypto_cursor_seglen(&cc_out) < blksz) 949dd2e1352SJohn Baldwin outblk = blk; 950dd2e1352SJohn Baldwin else 951dd2e1352SJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 952dd2e1352SJohn Baldwin exf->encrypt(swe->sw_kschedule, inblk, outblk); 953dd2e1352SJohn Baldwin axf->Update(&ctx, outblk, blksz); 954dd2e1352SJohn Baldwin if (outblk == blk) 955dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 956dd2e1352SJohn Baldwin else 957dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 958dd2e1352SJohn Baldwin } else { 959dd2e1352SJohn Baldwin axf->Update(&ctx, inblk, blksz); 960dd2e1352SJohn Baldwin } 961dd2e1352SJohn Baldwin } 962dd2e1352SJohn Baldwin if (resid > 0) { 963dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 964dd2e1352SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 965dd2e1352SJohn Baldwin exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 966dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 967dd2e1352SJohn Baldwin } 968dd2e1352SJohn Baldwin axf->Update(&ctx, blk, resid); 969dd2e1352SJohn Baldwin if (resid % 16 != 0) { 970dd2e1352SJohn Baldwin /* padding2 */ 971dd2e1352SJohn Baldwin memset(blk, 0, 16); 972dd2e1352SJohn Baldwin axf->Update(&ctx, blk, 16 - resid % 16); 973dd2e1352SJohn Baldwin } 974dd2e1352SJohn Baldwin } 975dd2e1352SJohn Baldwin 976dd2e1352SJohn Baldwin /* lengths */ 977dd2e1352SJohn Baldwin blkp = (uint64_t *)blk; 978dd2e1352SJohn Baldwin blkp[0] = htole64(crp->crp_aad_length); 979dd2e1352SJohn Baldwin blkp[1] = htole64(crp->crp_payload_length); 980dd2e1352SJohn Baldwin axf->Update(&ctx, blk, sizeof(uint64_t) * 2); 981dd2e1352SJohn Baldwin 982dd2e1352SJohn Baldwin /* Finalize MAC */ 983dd2e1352SJohn Baldwin axf->Final(tag, &ctx); 984dd2e1352SJohn Baldwin 985dd2e1352SJohn Baldwin /* Validate tag */ 986dd2e1352SJohn Baldwin error = 0; 987dd2e1352SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 988dd2e1352SJohn Baldwin u_char tag2[POLY1305_HASH_LEN]; 989dd2e1352SJohn Baldwin 990dd2e1352SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 991dd2e1352SJohn Baldwin 992dd2e1352SJohn Baldwin r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 993dd2e1352SJohn Baldwin explicit_bzero(tag2, sizeof(tag2)); 994dd2e1352SJohn Baldwin if (r != 0) { 995dd2e1352SJohn Baldwin error = EBADMSG; 996dd2e1352SJohn Baldwin goto out; 997dd2e1352SJohn Baldwin } 998dd2e1352SJohn Baldwin 999dd2e1352SJohn Baldwin /* tag matches, decrypt data */ 1000dd2e1352SJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 1001dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1002dd2e1352SJohn Baldwin for (resid = crp->crp_payload_length; resid > blksz; 1003dd2e1352SJohn Baldwin resid -= blksz) { 1004dd2e1352SJohn Baldwin if (crypto_cursor_seglen(&cc_in) < blksz) { 1005dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, blksz, blk); 1006dd2e1352SJohn Baldwin inblk = blk; 1007dd2e1352SJohn Baldwin } else { 1008dd2e1352SJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 1009dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_in, blksz); 1010dd2e1352SJohn Baldwin } 1011dd2e1352SJohn Baldwin if (crypto_cursor_seglen(&cc_out) < blksz) 1012dd2e1352SJohn Baldwin outblk = blk; 1013dd2e1352SJohn Baldwin else 1014dd2e1352SJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 1015dd2e1352SJohn Baldwin exf->decrypt(swe->sw_kschedule, inblk, outblk); 1016dd2e1352SJohn Baldwin if (outblk == blk) 1017dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, blksz, blk); 1018dd2e1352SJohn Baldwin else 1019dd2e1352SJohn Baldwin crypto_cursor_advance(&cc_out, blksz); 1020dd2e1352SJohn Baldwin } 1021dd2e1352SJohn Baldwin if (resid > 0) { 1022dd2e1352SJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 1023dd2e1352SJohn Baldwin exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 1024dd2e1352SJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 1025dd2e1352SJohn Baldwin } 1026dd2e1352SJohn Baldwin } else { 1027dd2e1352SJohn Baldwin /* Inject the authentication data */ 1028dd2e1352SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 1029dd2e1352SJohn Baldwin } 1030dd2e1352SJohn Baldwin 1031dd2e1352SJohn Baldwin out: 1032dd2e1352SJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 1033dd2e1352SJohn Baldwin explicit_bzero(tag, sizeof(tag)); 1034dd2e1352SJohn Baldwin explicit_bzero(&ctx, sizeof(ctx)); 1035dd2e1352SJohn Baldwin return (error); 1036dd2e1352SJohn Baldwin } 1037dd2e1352SJohn Baldwin 1038091d81d1SSam Leffler /* 1039c0341432SJohn Baldwin * Apply a cipher and a digest to perform EtA. 1040c0341432SJohn Baldwin */ 1041c0341432SJohn Baldwin static int 1042c0341432SJohn Baldwin swcr_eta(struct swcr_session *ses, struct cryptop *crp) 1043c0341432SJohn Baldwin { 1044c0341432SJohn Baldwin int error; 1045c0341432SJohn Baldwin 1046c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1047c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 1048c0341432SJohn Baldwin if (error == 0) 1049c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 1050c0341432SJohn Baldwin } else { 1051c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 1052c0341432SJohn Baldwin if (error == 0) 1053c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 1054c0341432SJohn Baldwin } 1055c0341432SJohn Baldwin return (error); 1056c0341432SJohn Baldwin } 1057c0341432SJohn Baldwin 1058c0341432SJohn Baldwin /* 1059091d81d1SSam Leffler * Apply a compression/decompression algorithm 1060091d81d1SSam Leffler */ 1061091d81d1SSam Leffler static int 1062c0341432SJohn Baldwin swcr_compdec(struct swcr_session *ses, struct cryptop *crp) 1063091d81d1SSam Leffler { 1064d3d79e96SJohn Baldwin uint8_t *data, *out; 1065091d81d1SSam Leffler struct comp_algo *cxf; 1066091d81d1SSam Leffler int adj; 1067d3d79e96SJohn Baldwin uint32_t result; 1068091d81d1SSam Leffler 1069c0341432SJohn Baldwin cxf = ses->swcr_compdec.sw_cxf; 1070091d81d1SSam Leffler 1071091d81d1SSam Leffler /* We must handle the whole buffer of data in one time 1072091d81d1SSam Leffler * then if there is not all the data in the mbuf, we must 1073091d81d1SSam Leffler * copy in a buffer. 1074091d81d1SSam Leffler */ 1075091d81d1SSam Leffler 1076c0341432SJohn Baldwin data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1077091d81d1SSam Leffler if (data == NULL) 1078091d81d1SSam Leffler return (EINVAL); 1079c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1080c0341432SJohn Baldwin data); 1081091d81d1SSam Leffler 1082c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1083c0341432SJohn Baldwin result = cxf->compress(data, crp->crp_payload_length, &out); 1084091d81d1SSam Leffler else 1085c0341432SJohn Baldwin result = cxf->decompress(data, crp->crp_payload_length, &out); 1086091d81d1SSam Leffler 10871ede983cSDag-Erling Smørgrav free(data, M_CRYPTO_DATA); 1088091d81d1SSam Leffler if (result == 0) 1089c0341432SJohn Baldwin return (EINVAL); 1090c0341432SJohn Baldwin crp->crp_olen = result; 1091c0341432SJohn Baldwin 1092c0341432SJohn Baldwin /* Check the compressed size when doing compression */ 1093c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1094c0341432SJohn Baldwin if (result >= crp->crp_payload_length) { 1095c0341432SJohn Baldwin /* Compression was useless, we lost time */ 1096c0341432SJohn Baldwin free(out, M_CRYPTO_DATA); 1097c0341432SJohn Baldwin return (0); 1098c0341432SJohn Baldwin } 1099c0341432SJohn Baldwin } 1100091d81d1SSam Leffler 1101091d81d1SSam Leffler /* Copy back the (de)compressed data. m_copyback is 1102091d81d1SSam Leffler * extending the mbuf as necessary. 1103091d81d1SSam Leffler */ 1104c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_payload_start, result, out); 1105c0341432SJohn Baldwin if (result < crp->crp_payload_length) { 11069c0e3d3aSJohn Baldwin switch (crp->crp_buf.cb_type) { 1107c0341432SJohn Baldwin case CRYPTO_BUF_MBUF: 1108c0341432SJohn Baldwin adj = result - crp->crp_payload_length; 11099c0e3d3aSJohn Baldwin m_adj(crp->crp_buf.cb_mbuf, adj); 1110c0341432SJohn Baldwin break; 1111c0341432SJohn Baldwin case CRYPTO_BUF_UIO: { 11129c0e3d3aSJohn Baldwin struct uio *uio = crp->crp_buf.cb_uio; 1113091d81d1SSam Leffler int ind; 1114091d81d1SSam Leffler 1115c0341432SJohn Baldwin adj = crp->crp_payload_length - result; 1116091d81d1SSam Leffler ind = uio->uio_iovcnt - 1; 1117091d81d1SSam Leffler 1118091d81d1SSam Leffler while (adj > 0 && ind >= 0) { 1119091d81d1SSam Leffler if (adj < uio->uio_iov[ind].iov_len) { 1120091d81d1SSam Leffler uio->uio_iov[ind].iov_len -= adj; 1121091d81d1SSam Leffler break; 1122091d81d1SSam Leffler } 1123091d81d1SSam Leffler 1124091d81d1SSam Leffler adj -= uio->uio_iov[ind].iov_len; 1125091d81d1SSam Leffler uio->uio_iov[ind].iov_len = 0; 1126091d81d1SSam Leffler ind--; 1127091d81d1SSam Leffler uio->uio_iovcnt--; 1128091d81d1SSam Leffler } 1129091d81d1SSam Leffler } 1130c0341432SJohn Baldwin break; 1131e6f6d0c9SAlan Somers case CRYPTO_BUF_VMPAGE: 1132e6f6d0c9SAlan Somers adj = crp->crp_payload_length - result; 1133e6f6d0c9SAlan Somers crp->crp_buf.cb_vm_page_len -= adj; 1134e6f6d0c9SAlan Somers break; 11359c0e3d3aSJohn Baldwin default: 11369c0e3d3aSJohn Baldwin break; 1137c0341432SJohn Baldwin } 1138091d81d1SSam Leffler } 11391ede983cSDag-Erling Smørgrav free(out, M_CRYPTO_DATA); 1140091d81d1SSam Leffler return 0; 1141091d81d1SSam Leffler } 1142091d81d1SSam Leffler 1143091d81d1SSam Leffler static int 11443e947048SJohn Baldwin swcr_setup_cipher(struct swcr_session *ses, 1145c0341432SJohn Baldwin const struct crypto_session_params *csp) 1146091d81d1SSam Leffler { 1147c0341432SJohn Baldwin struct swcr_encdec *swe; 1148091d81d1SSam Leffler struct enc_xform *txf; 1149f6c4bc3bSPawel Jakub Dawidek int error; 1150091d81d1SSam Leffler 1151c0341432SJohn Baldwin swe = &ses->swcr_encdec; 1152c0341432SJohn Baldwin txf = crypto_cipher(csp); 1153c0341432SJohn Baldwin MPASS(txf->ivsize == csp->csp_ivlen); 11543e947048SJohn Baldwin if (txf->ctxsize != 0) { 11553e947048SJohn Baldwin swe->sw_kschedule = malloc(txf->ctxsize, M_CRYPTO_DATA, 11563e947048SJohn Baldwin M_NOWAIT); 11573e947048SJohn Baldwin if (swe->sw_kschedule == NULL) 11583e947048SJohn Baldwin return (ENOMEM); 11593e947048SJohn Baldwin } 1160c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) { 11613e947048SJohn Baldwin error = txf->setkey(swe->sw_kschedule, 1162c0341432SJohn Baldwin csp->csp_cipher_key, csp->csp_cipher_klen); 1163c0341432SJohn Baldwin if (error) 1164c0341432SJohn Baldwin return (error); 1165091d81d1SSam Leffler } 1166c0341432SJohn Baldwin swe->sw_exf = txf; 1167c0341432SJohn Baldwin return (0); 1168f6c4bc3bSPawel Jakub Dawidek } 1169091d81d1SSam Leffler 1170c0341432SJohn Baldwin static int 1171c0341432SJohn Baldwin swcr_setup_auth(struct swcr_session *ses, 1172c0341432SJohn Baldwin const struct crypto_session_params *csp) 1173c0341432SJohn Baldwin { 1174c0341432SJohn Baldwin struct swcr_auth *swa; 1175c0341432SJohn Baldwin struct auth_hash *axf; 1176c0341432SJohn Baldwin 1177c0341432SJohn Baldwin swa = &ses->swcr_auth; 1178c0341432SJohn Baldwin 1179c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1180c0341432SJohn Baldwin swa->sw_axf = axf; 1181c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1182c0341432SJohn Baldwin return (EINVAL); 1183c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1184c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1185c0341432SJohn Baldwin else 1186c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1187c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1188c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1189c0341432SJohn Baldwin return (ENOBUFS); 1190c0341432SJohn Baldwin 1191c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1192091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1193c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1194f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1195f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1196f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1197091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 1198091d81d1SSam Leffler case CRYPTO_RIPEMD160_HMAC: 11993a0b6a93SJohn Baldwin swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1200091d81d1SSam Leffler M_NOWAIT); 1201c0341432SJohn Baldwin if (swa->sw_octx == NULL) 1202c0341432SJohn Baldwin return (ENOBUFS); 1203c0341432SJohn Baldwin 1204c0341432SJohn Baldwin if (csp->csp_auth_key != NULL) { 1205c0341432SJohn Baldwin swcr_authprepare(axf, swa, csp->csp_auth_key, 1206c0341432SJohn Baldwin csp->csp_auth_klen); 1207091d81d1SSam Leffler } 1208091d81d1SSam Leffler 1209c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1210c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1211091d81d1SSam Leffler break; 1212091d81d1SSam Leffler case CRYPTO_SHA1: 1213c4729f6eSConrad Meyer case CRYPTO_SHA2_224: 1214c4729f6eSConrad Meyer case CRYPTO_SHA2_256: 1215c4729f6eSConrad Meyer case CRYPTO_SHA2_384: 1216c4729f6eSConrad Meyer case CRYPTO_SHA2_512: 1217c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1218c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1219c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1220c0341432SJohn Baldwin break; 1221c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1222c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1223c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1224c0341432SJohn Baldwin csp->csp_auth_klen); 1225c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1226c0341432SJohn Baldwin ses->swcr_process = swcr_gmac; 1227c0341432SJohn Baldwin break; 1228c0341432SJohn Baldwin case CRYPTO_POLY1305: 1229c0341432SJohn Baldwin case CRYPTO_BLAKE2B: 1230c0341432SJohn Baldwin case CRYPTO_BLAKE2S: 1231c0341432SJohn Baldwin /* 1232c0341432SJohn Baldwin * Blake2b and Blake2s support an optional key but do 1233c0341432SJohn Baldwin * not require one. 1234c0341432SJohn Baldwin */ 1235c0341432SJohn Baldwin if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) 1236c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1237c0341432SJohn Baldwin csp->csp_auth_klen); 1238c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1239c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1240c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1241c0341432SJohn Baldwin break; 1242c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1243c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1244c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1245c0341432SJohn Baldwin csp->csp_auth_klen); 1246c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 1247c0341432SJohn Baldwin ses->swcr_process = swcr_ccm_cbc_mac; 1248c0341432SJohn Baldwin break; 1249091d81d1SSam Leffler } 1250091d81d1SSam Leffler 1251c0341432SJohn Baldwin return (0); 1252c0341432SJohn Baldwin } 125308fca7a5SJohn-Mark Gurney 1254c0341432SJohn Baldwin static int 1255c0341432SJohn Baldwin swcr_setup_gcm(struct swcr_session *ses, 1256c0341432SJohn Baldwin const struct crypto_session_params *csp) 1257c0341432SJohn Baldwin { 1258c0341432SJohn Baldwin struct swcr_auth *swa; 1259c0341432SJohn Baldwin struct auth_hash *axf; 1260c0341432SJohn Baldwin 1261c0341432SJohn Baldwin if (csp->csp_ivlen != AES_GCM_IV_LEN) 1262c0341432SJohn Baldwin return (EINVAL); 1263c0341432SJohn Baldwin 1264c0341432SJohn Baldwin /* First, setup the auth side. */ 1265c0341432SJohn Baldwin swa = &ses->swcr_auth; 1266c0341432SJohn Baldwin switch (csp->csp_cipher_klen * 8) { 1267c0341432SJohn Baldwin case 128: 1268c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_128; 1269c0341432SJohn Baldwin break; 1270c0341432SJohn Baldwin case 192: 1271c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_192; 1272c0341432SJohn Baldwin break; 1273c0341432SJohn Baldwin case 256: 1274c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_256; 1275c0341432SJohn Baldwin break; 1276c0341432SJohn Baldwin default: 1277c0341432SJohn Baldwin return (EINVAL); 1278c0341432SJohn Baldwin } 1279c0341432SJohn Baldwin swa->sw_axf = axf; 1280c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1281c0341432SJohn Baldwin return (EINVAL); 1282c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1283c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1284c0341432SJohn Baldwin else 1285c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1286c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1287c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1288c0341432SJohn Baldwin return (ENOBUFS); 1289c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1290c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) 1291c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, 1292c0341432SJohn Baldwin csp->csp_cipher_klen); 1293c0341432SJohn Baldwin 1294c0341432SJohn Baldwin /* Second, setup the cipher side. */ 12953e947048SJohn Baldwin return (swcr_setup_cipher(ses, csp)); 1296c0341432SJohn Baldwin } 1297c0341432SJohn Baldwin 1298c0341432SJohn Baldwin static int 1299c0341432SJohn Baldwin swcr_setup_ccm(struct swcr_session *ses, 1300c0341432SJohn Baldwin const struct crypto_session_params *csp) 1301c0341432SJohn Baldwin { 1302c0341432SJohn Baldwin struct swcr_auth *swa; 1303c0341432SJohn Baldwin struct auth_hash *axf; 1304c0341432SJohn Baldwin 1305c0341432SJohn Baldwin if (csp->csp_ivlen != AES_CCM_IV_LEN) 1306c0341432SJohn Baldwin return (EINVAL); 1307c0341432SJohn Baldwin 1308c0341432SJohn Baldwin /* First, setup the auth side. */ 1309c0341432SJohn Baldwin swa = &ses->swcr_auth; 1310c0341432SJohn Baldwin switch (csp->csp_cipher_klen * 8) { 1311507281e5SSean Eric Fagan case 128: 1312507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_128; 1313507281e5SSean Eric Fagan break; 1314507281e5SSean Eric Fagan case 192: 1315507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_192; 1316507281e5SSean Eric Fagan break; 1317507281e5SSean Eric Fagan case 256: 1318507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_256; 1319507281e5SSean Eric Fagan break; 1320507281e5SSean Eric Fagan default: 1321c0341432SJohn Baldwin return (EINVAL); 1322507281e5SSean Eric Fagan } 1323c0341432SJohn Baldwin swa->sw_axf = axf; 1324c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1325c0341432SJohn Baldwin return (EINVAL); 1326c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1327c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1328c0341432SJohn Baldwin else 1329c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1330c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1331c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1332c0341432SJohn Baldwin return (ENOBUFS); 1333c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1334c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) 1335c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, 1336c0341432SJohn Baldwin csp->csp_cipher_klen); 133708fca7a5SJohn-Mark Gurney 1338c0341432SJohn Baldwin /* Second, setup the cipher side. */ 13393e947048SJohn Baldwin return (swcr_setup_cipher(ses, csp)); 13402e2e26d1SJohn Baldwin } 1341a2bc81bfSJohn-Mark Gurney 1342dd2e1352SJohn Baldwin static int 1343dd2e1352SJohn Baldwin swcr_setup_chacha20_poly1305(struct swcr_session *ses, 1344dd2e1352SJohn Baldwin const struct crypto_session_params *csp) 1345dd2e1352SJohn Baldwin { 1346dd2e1352SJohn Baldwin struct swcr_auth *swa; 1347dd2e1352SJohn Baldwin struct auth_hash *axf; 1348dd2e1352SJohn Baldwin 1349dd2e1352SJohn Baldwin if (csp->csp_ivlen != CHACHA20_POLY1305_IV_LEN) 1350dd2e1352SJohn Baldwin return (EINVAL); 1351dd2e1352SJohn Baldwin 1352dd2e1352SJohn Baldwin /* First, setup the auth side. */ 1353dd2e1352SJohn Baldwin swa = &ses->swcr_auth; 1354dd2e1352SJohn Baldwin axf = &auth_hash_chacha20_poly1305; 1355dd2e1352SJohn Baldwin swa->sw_axf = axf; 1356dd2e1352SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1357dd2e1352SJohn Baldwin return (EINVAL); 1358dd2e1352SJohn Baldwin if (csp->csp_auth_mlen == 0) 1359dd2e1352SJohn Baldwin swa->sw_mlen = axf->hashsize; 1360dd2e1352SJohn Baldwin else 1361dd2e1352SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1362dd2e1352SJohn Baldwin 1363dd2e1352SJohn Baldwin /* The auth state is regenerated for each nonce. */ 1364dd2e1352SJohn Baldwin 1365dd2e1352SJohn Baldwin /* Second, setup the cipher side. */ 1366dd2e1352SJohn Baldwin return (swcr_setup_cipher(ses, csp)); 1367dd2e1352SJohn Baldwin } 1368dd2e1352SJohn Baldwin 1369c0341432SJohn Baldwin static bool 1370c0341432SJohn Baldwin swcr_auth_supported(const struct crypto_session_params *csp) 1371109919c6SBenno Rice { 1372091d81d1SSam Leffler struct auth_hash *axf; 1373091d81d1SSam Leffler 1374c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1375c0341432SJohn Baldwin if (axf == NULL) 1376c0341432SJohn Baldwin return (false); 1377c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1378091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1379c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1380f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1381f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1382f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1383091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 1384c0341432SJohn Baldwin case CRYPTO_RIPEMD160_HMAC: 1385091d81d1SSam Leffler break; 1386c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1387c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1388c0341432SJohn Baldwin case 128: 1389c0341432SJohn Baldwin case 192: 1390c0341432SJohn Baldwin case 256: 1391c0341432SJohn Baldwin break; 1392c0341432SJohn Baldwin default: 1393c0341432SJohn Baldwin return (false); 1394c0341432SJohn Baldwin } 1395c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1396c0341432SJohn Baldwin return (false); 1397c0341432SJohn Baldwin if (csp->csp_ivlen != AES_GCM_IV_LEN) 1398c0341432SJohn Baldwin return (false); 1399c0341432SJohn Baldwin break; 140025b7033bSConrad Meyer case CRYPTO_POLY1305: 1401c0341432SJohn Baldwin if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1402c0341432SJohn Baldwin return (false); 1403c0341432SJohn Baldwin break; 1404c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1405c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1406c0341432SJohn Baldwin case 128: 1407c0341432SJohn Baldwin case 192: 1408c0341432SJohn Baldwin case 256: 1409c0341432SJohn Baldwin break; 1410c0341432SJohn Baldwin default: 1411c0341432SJohn Baldwin return (false); 1412c0341432SJohn Baldwin } 1413c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1414c0341432SJohn Baldwin return (false); 1415c0341432SJohn Baldwin if (csp->csp_ivlen != AES_CCM_IV_LEN) 1416c0341432SJohn Baldwin return (false); 1417c0341432SJohn Baldwin break; 1418c0341432SJohn Baldwin } 1419c0341432SJohn Baldwin return (true); 1420c0341432SJohn Baldwin } 1421091d81d1SSam Leffler 1422c0341432SJohn Baldwin static bool 1423c0341432SJohn Baldwin swcr_cipher_supported(const struct crypto_session_params *csp) 1424c0341432SJohn Baldwin { 1425c0341432SJohn Baldwin struct enc_xform *txf; 1426c0341432SJohn Baldwin 1427c0341432SJohn Baldwin txf = crypto_cipher(csp); 1428c0341432SJohn Baldwin if (txf == NULL) 1429c0341432SJohn Baldwin return (false); 1430c0341432SJohn Baldwin if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1431c0341432SJohn Baldwin txf->ivsize != csp->csp_ivlen) 1432c0341432SJohn Baldwin return (false); 1433c0341432SJohn Baldwin return (true); 1434c0341432SJohn Baldwin } 1435c0341432SJohn Baldwin 14366038018aSMarcin Wojtas #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 14376038018aSMarcin Wojtas 1438c0341432SJohn Baldwin static int 1439c0341432SJohn Baldwin swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1440c0341432SJohn Baldwin { 14416038018aSMarcin Wojtas if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1442c0341432SJohn Baldwin return (EINVAL); 1443c0341432SJohn Baldwin switch (csp->csp_mode) { 1444c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1445c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1446c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1447c0341432SJohn Baldwin break; 1448c0341432SJohn Baldwin default: 1449c0341432SJohn Baldwin return (EINVAL); 14505fbc5b5aSConrad Meyer } 1451091d81d1SSam Leffler break; 1452c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1453c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1454c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1455c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1456dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1457c0341432SJohn Baldwin return (EINVAL); 1458c0341432SJohn Baldwin default: 1459c0341432SJohn Baldwin if (!swcr_cipher_supported(csp)) 1460c0341432SJohn Baldwin return (EINVAL); 1461091d81d1SSam Leffler break; 1462091d81d1SSam Leffler } 1463c0341432SJohn Baldwin break; 1464c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1465c0341432SJohn Baldwin if (!swcr_auth_supported(csp)) 1466c0341432SJohn Baldwin return (EINVAL); 1467c0341432SJohn Baldwin break; 1468c0341432SJohn Baldwin case CSP_MODE_AEAD: 1469c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1470c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1471c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1472dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1473c0341432SJohn Baldwin break; 1474c0341432SJohn Baldwin default: 1475c0341432SJohn Baldwin return (EINVAL); 1476c0341432SJohn Baldwin } 1477c0341432SJohn Baldwin break; 1478c0341432SJohn Baldwin case CSP_MODE_ETA: 1479c0341432SJohn Baldwin /* AEAD algorithms cannot be used for EtA. */ 1480c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1481c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1482c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1483dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1484c0341432SJohn Baldwin return (EINVAL); 1485c0341432SJohn Baldwin } 1486c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1487c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1488c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1489c0341432SJohn Baldwin return (EINVAL); 1490c0341432SJohn Baldwin } 1491c0341432SJohn Baldwin 1492c0341432SJohn Baldwin if (!swcr_cipher_supported(csp) || 1493c0341432SJohn Baldwin !swcr_auth_supported(csp)) 1494c0341432SJohn Baldwin return (EINVAL); 1495c0341432SJohn Baldwin break; 1496c0341432SJohn Baldwin default: 1497c0341432SJohn Baldwin return (EINVAL); 1498c0341432SJohn Baldwin } 1499c0341432SJohn Baldwin 1500c0341432SJohn Baldwin return (CRYPTODEV_PROBE_SOFTWARE); 1501c0341432SJohn Baldwin } 1502c0341432SJohn Baldwin 1503c0341432SJohn Baldwin /* 1504c0341432SJohn Baldwin * Generate a new software session. 1505c0341432SJohn Baldwin */ 1506c0341432SJohn Baldwin static int 1507c0341432SJohn Baldwin swcr_newsession(device_t dev, crypto_session_t cses, 1508c0341432SJohn Baldwin const struct crypto_session_params *csp) 1509c0341432SJohn Baldwin { 1510c0341432SJohn Baldwin struct swcr_session *ses; 1511c0341432SJohn Baldwin struct swcr_encdec *swe; 1512c0341432SJohn Baldwin struct swcr_auth *swa; 1513c0341432SJohn Baldwin struct comp_algo *cxf; 1514c0341432SJohn Baldwin int error; 1515c0341432SJohn Baldwin 1516c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1517c0341432SJohn Baldwin mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); 1518c0341432SJohn Baldwin 1519c0341432SJohn Baldwin error = 0; 1520c0341432SJohn Baldwin swe = &ses->swcr_encdec; 1521c0341432SJohn Baldwin swa = &ses->swcr_auth; 1522c0341432SJohn Baldwin switch (csp->csp_mode) { 1523c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1524c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1525c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1526c0341432SJohn Baldwin cxf = &comp_algo_deflate; 1527c0341432SJohn Baldwin break; 1528c0341432SJohn Baldwin #ifdef INVARIANTS 1529c0341432SJohn Baldwin default: 1530c0341432SJohn Baldwin panic("bad compression algo"); 1531c0341432SJohn Baldwin #endif 1532c0341432SJohn Baldwin } 1533c0341432SJohn Baldwin ses->swcr_compdec.sw_cxf = cxf; 1534c0341432SJohn Baldwin ses->swcr_process = swcr_compdec; 1535c0341432SJohn Baldwin break; 1536c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1537c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1538c0341432SJohn Baldwin case CRYPTO_NULL_CBC: 1539c0341432SJohn Baldwin ses->swcr_process = swcr_null; 1540c0341432SJohn Baldwin break; 1541c0341432SJohn Baldwin #ifdef INVARIANTS 1542c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1543c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1544dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1545c0341432SJohn Baldwin panic("bad cipher algo"); 1546c0341432SJohn Baldwin #endif 1547c0341432SJohn Baldwin default: 15483e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1549c0341432SJohn Baldwin if (error == 0) 1550c0341432SJohn Baldwin ses->swcr_process = swcr_encdec; 1551c0341432SJohn Baldwin } 1552c0341432SJohn Baldwin break; 1553c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1554c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1555c0341432SJohn Baldwin break; 1556c0341432SJohn Baldwin case CSP_MODE_AEAD: 1557c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1558c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1559c0341432SJohn Baldwin error = swcr_setup_gcm(ses, csp); 1560c0341432SJohn Baldwin if (error == 0) 1561c0341432SJohn Baldwin ses->swcr_process = swcr_gcm; 1562c0341432SJohn Baldwin break; 1563c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1564c0341432SJohn Baldwin error = swcr_setup_ccm(ses, csp); 1565c0341432SJohn Baldwin if (error == 0) 1566c0341432SJohn Baldwin ses->swcr_process = swcr_ccm; 1567c0341432SJohn Baldwin break; 1568dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1569dd2e1352SJohn Baldwin error = swcr_setup_chacha20_poly1305(ses, csp); 1570dd2e1352SJohn Baldwin if (error == 0) 1571dd2e1352SJohn Baldwin ses->swcr_process = swcr_chacha20_poly1305; 1572dd2e1352SJohn Baldwin break; 1573c0341432SJohn Baldwin #ifdef INVARIANTS 1574c0341432SJohn Baldwin default: 1575c0341432SJohn Baldwin panic("bad aead algo"); 1576c0341432SJohn Baldwin #endif 1577c0341432SJohn Baldwin } 1578c0341432SJohn Baldwin break; 1579c0341432SJohn Baldwin case CSP_MODE_ETA: 1580c0341432SJohn Baldwin #ifdef INVARIANTS 1581c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1582c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1583c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1584dd2e1352SJohn Baldwin case CRYPTO_CHACHA20_POLY1305: 1585c0341432SJohn Baldwin panic("bad eta cipher algo"); 1586c0341432SJohn Baldwin } 1587c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1588c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1589c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1590c0341432SJohn Baldwin panic("bad eta auth algo"); 1591c0341432SJohn Baldwin } 1592c0341432SJohn Baldwin #endif 1593c0341432SJohn Baldwin 1594c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1595c0341432SJohn Baldwin if (error) 1596c0341432SJohn Baldwin break; 1597c0341432SJohn Baldwin if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1598c0341432SJohn Baldwin /* Effectively degrade to digest mode. */ 1599c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1600c0341432SJohn Baldwin break; 1601c0341432SJohn Baldwin } 1602c0341432SJohn Baldwin 16033e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1604c0341432SJohn Baldwin if (error == 0) 1605c0341432SJohn Baldwin ses->swcr_process = swcr_eta; 1606c0341432SJohn Baldwin break; 1607c0341432SJohn Baldwin default: 1608c0341432SJohn Baldwin error = EINVAL; 1609c0341432SJohn Baldwin } 1610c0341432SJohn Baldwin 1611c0341432SJohn Baldwin if (error) 1612c0341432SJohn Baldwin swcr_freesession(dev, cses); 1613c0341432SJohn Baldwin return (error); 1614c0341432SJohn Baldwin } 1615c0341432SJohn Baldwin 1616c0341432SJohn Baldwin static void 1617c0341432SJohn Baldwin swcr_freesession(device_t dev, crypto_session_t cses) 1618c0341432SJohn Baldwin { 1619c0341432SJohn Baldwin struct swcr_session *ses; 1620c0341432SJohn Baldwin 1621c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1622c0341432SJohn Baldwin 1623c0341432SJohn Baldwin mtx_destroy(&ses->swcr_lock); 1624c0341432SJohn Baldwin 16253e947048SJohn Baldwin zfree(ses->swcr_encdec.sw_kschedule, M_CRYPTO_DATA); 16264a711b8dSJohn Baldwin zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 16274a711b8dSJohn Baldwin zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1628091d81d1SSam Leffler } 1629091d81d1SSam Leffler 1630091d81d1SSam Leffler /* 1631091d81d1SSam Leffler * Process a software request. 1632091d81d1SSam Leffler */ 1633091d81d1SSam Leffler static int 16346810ad6fSSam Leffler swcr_process(device_t dev, struct cryptop *crp, int hint) 1635091d81d1SSam Leffler { 1636c0341432SJohn Baldwin struct swcr_session *ses; 1637091d81d1SSam Leffler 16381b0909d5SConrad Meyer ses = crypto_get_driver_session(crp->crp_session); 1639a7fcb1afSSean Eric Fagan mtx_lock(&ses->swcr_lock); 1640091d81d1SSam Leffler 1641c0341432SJohn Baldwin crp->crp_etype = ses->swcr_process(ses, crp); 1642091d81d1SSam Leffler 1643a7fcb1afSSean Eric Fagan mtx_unlock(&ses->swcr_lock); 1644091d81d1SSam Leffler crypto_done(crp); 1645c0341432SJohn Baldwin return (0); 1646091d81d1SSam Leffler } 1647091d81d1SSam Leffler 1648091d81d1SSam Leffler static void 16493f147ab2SWarner Losh swcr_identify(driver_t *drv, device_t parent) 1650091d81d1SSam Leffler { 16516810ad6fSSam Leffler /* NB: order 10 is so we get attached after h/w devices */ 16526810ad6fSSam Leffler if (device_find_child(parent, "cryptosoft", -1) == NULL && 165386c585d9SMarius Strobl BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 16546810ad6fSSam Leffler panic("cryptosoft: could not attach"); 16556810ad6fSSam Leffler } 1656f6c4bc3bSPawel Jakub Dawidek 16576810ad6fSSam Leffler static int 16586810ad6fSSam Leffler swcr_probe(device_t dev) 16596810ad6fSSam Leffler { 16606810ad6fSSam Leffler device_set_desc(dev, "software crypto"); 16614fc60fa9SMark Johnston device_quiet(dev); 166286c585d9SMarius Strobl return (BUS_PROBE_NOWILDCARD); 16636810ad6fSSam Leffler } 1664f6c4bc3bSPawel Jakub Dawidek 16656810ad6fSSam Leffler static int 16666810ad6fSSam Leffler swcr_attach(device_t dev) 16676810ad6fSSam Leffler { 16686810ad6fSSam Leffler 16699ebbebe4SConrad Meyer swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 16706810ad6fSSam Leffler CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 16716810ad6fSSam Leffler if (swcr_id < 0) { 16726810ad6fSSam Leffler device_printf(dev, "cannot initialize!"); 1673c0341432SJohn Baldwin return (ENXIO); 16746810ad6fSSam Leffler } 16756810ad6fSSam Leffler 1676c0341432SJohn Baldwin return (0); 1677091d81d1SSam Leffler } 16784b465da2SPawel Jakub Dawidek 16793f147ab2SWarner Losh static int 16806810ad6fSSam Leffler swcr_detach(device_t dev) 16814b465da2SPawel Jakub Dawidek { 16826810ad6fSSam Leffler crypto_unregister_all(swcr_id); 16833f147ab2SWarner Losh return 0; 16844b465da2SPawel Jakub Dawidek } 16856810ad6fSSam Leffler 16866810ad6fSSam Leffler static device_method_t swcr_methods[] = { 16876810ad6fSSam Leffler DEVMETHOD(device_identify, swcr_identify), 16886810ad6fSSam Leffler DEVMETHOD(device_probe, swcr_probe), 16896810ad6fSSam Leffler DEVMETHOD(device_attach, swcr_attach), 16906810ad6fSSam Leffler DEVMETHOD(device_detach, swcr_detach), 16916810ad6fSSam Leffler 1692c0341432SJohn Baldwin DEVMETHOD(cryptodev_probesession, swcr_probesession), 16936810ad6fSSam Leffler DEVMETHOD(cryptodev_newsession, swcr_newsession), 16946810ad6fSSam Leffler DEVMETHOD(cryptodev_freesession,swcr_freesession), 16956810ad6fSSam Leffler DEVMETHOD(cryptodev_process, swcr_process), 16966810ad6fSSam Leffler 16976810ad6fSSam Leffler {0, 0}, 16986810ad6fSSam Leffler }; 16996810ad6fSSam Leffler 17006810ad6fSSam Leffler static driver_t swcr_driver = { 17016810ad6fSSam Leffler "cryptosoft", 17026810ad6fSSam Leffler swcr_methods, 17036810ad6fSSam Leffler 0, /* NB: no softc */ 17046810ad6fSSam Leffler }; 17056810ad6fSSam Leffler static devclass_t swcr_devclass; 17066810ad6fSSam Leffler 17076810ad6fSSam Leffler /* 17086810ad6fSSam Leffler * NB: We explicitly reference the crypto module so we 17096810ad6fSSam Leffler * get the necessary ordering when built as a loadable 17106810ad6fSSam Leffler * module. This is required because we bundle the crypto 17116810ad6fSSam Leffler * module code together with the cryptosoft driver (otherwise 17126810ad6fSSam Leffler * normal module dependencies would handle things). 17136810ad6fSSam Leffler */ 17146810ad6fSSam Leffler extern int crypto_modevent(struct module *, int, void *); 17156810ad6fSSam Leffler /* XXX where to attach */ 17166810ad6fSSam Leffler DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 17176810ad6fSSam Leffler MODULE_VERSION(cryptosoft, 1); 17186810ad6fSSam Leffler MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1719