1091d81d1SSam Leffler /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2091d81d1SSam Leffler 360727d8bSWarner Losh /*- 4091d81d1SSam Leffler * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 56810ad6fSSam Leffler * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6091d81d1SSam Leffler * 7091d81d1SSam Leffler * This code was written by Angelos D. Keromytis in Athens, Greece, in 8091d81d1SSam Leffler * February 2000. Network Security Technologies Inc. (NSTI) kindly 9091d81d1SSam Leffler * supported the development of this code. 10091d81d1SSam Leffler * 11091d81d1SSam Leffler * Copyright (c) 2000, 2001 Angelos D. Keromytis 1208fca7a5SJohn-Mark Gurney * Copyright (c) 2014 The FreeBSD Foundation 1308fca7a5SJohn-Mark Gurney * All rights reserved. 1408fca7a5SJohn-Mark Gurney * 1508fca7a5SJohn-Mark Gurney * Portions of this software were developed by John-Mark Gurney 1608fca7a5SJohn-Mark Gurney * under sponsorship of the FreeBSD Foundation and 1708fca7a5SJohn-Mark Gurney * Rubicon Communications, LLC (Netgate). 18091d81d1SSam Leffler * 19091d81d1SSam Leffler * Permission to use, copy, and modify this software with or without fee 20091d81d1SSam Leffler * is hereby granted, provided that this entire notice is included in 21091d81d1SSam Leffler * all source code copies of any software which is or includes a copy or 22091d81d1SSam Leffler * modification of this software. 23091d81d1SSam Leffler * 24091d81d1SSam Leffler * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 25091d81d1SSam Leffler * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 26091d81d1SSam Leffler * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 27091d81d1SSam Leffler * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 28091d81d1SSam Leffler * PURPOSE. 29091d81d1SSam Leffler */ 30091d81d1SSam Leffler 312c446514SDavid E. O'Brien #include <sys/cdefs.h> 322c446514SDavid E. O'Brien __FBSDID("$FreeBSD$"); 332c446514SDavid E. O'Brien 34091d81d1SSam Leffler #include <sys/param.h> 35091d81d1SSam Leffler #include <sys/systm.h> 36091d81d1SSam Leffler #include <sys/malloc.h> 37091d81d1SSam Leffler #include <sys/mbuf.h> 386810ad6fSSam Leffler #include <sys/module.h> 39091d81d1SSam Leffler #include <sys/sysctl.h> 40091d81d1SSam Leffler #include <sys/errno.h> 41091d81d1SSam Leffler #include <sys/random.h> 42091d81d1SSam Leffler #include <sys/kernel.h> 43091d81d1SSam Leffler #include <sys/uio.h> 44109919c6SBenno Rice #include <sys/lock.h> 45109919c6SBenno Rice #include <sys/rwlock.h> 4608fca7a5SJohn-Mark Gurney #include <sys/endian.h> 4708fca7a5SJohn-Mark Gurney #include <sys/limits.h> 48a7fcb1afSSean Eric Fagan #include <sys/mutex.h> 49091d81d1SSam Leffler 50091d81d1SSam Leffler #include <crypto/sha1.h> 51091d81d1SSam Leffler #include <opencrypto/rmd160.h> 52091d81d1SSam Leffler 53091d81d1SSam Leffler #include <opencrypto/cryptodev.h> 54091d81d1SSam Leffler #include <opencrypto/xform.h> 55091d81d1SSam Leffler 566810ad6fSSam Leffler #include <sys/kobj.h> 576810ad6fSSam Leffler #include <sys/bus.h> 586810ad6fSSam Leffler #include "cryptodev_if.h" 59091d81d1SSam Leffler 60c0341432SJohn Baldwin struct swcr_auth { 61c0341432SJohn Baldwin void *sw_ictx; 62c0341432SJohn Baldwin void *sw_octx; 63c0341432SJohn Baldwin struct auth_hash *sw_axf; 64c0341432SJohn Baldwin uint16_t sw_mlen; 65c0341432SJohn Baldwin }; 66c0341432SJohn Baldwin 67c0341432SJohn Baldwin struct swcr_encdec { 683e947048SJohn Baldwin void *sw_kschedule; 69c0341432SJohn Baldwin struct enc_xform *sw_exf; 70c0341432SJohn Baldwin }; 71c0341432SJohn Baldwin 72c0341432SJohn Baldwin struct swcr_compdec { 73c0341432SJohn Baldwin struct comp_algo *sw_cxf; 74c0341432SJohn Baldwin }; 75c0341432SJohn Baldwin 76c0341432SJohn Baldwin struct swcr_session { 77c0341432SJohn Baldwin struct mtx swcr_lock; 78c0341432SJohn Baldwin int (*swcr_process)(struct swcr_session *, struct cryptop *); 79c0341432SJohn Baldwin 80c0341432SJohn Baldwin struct swcr_auth swcr_auth; 81c0341432SJohn Baldwin struct swcr_encdec swcr_encdec; 82c0341432SJohn Baldwin struct swcr_compdec swcr_compdec; 83c0341432SJohn Baldwin }; 84507281e5SSean Eric Fagan 856810ad6fSSam Leffler static int32_t swcr_id; 866810ad6fSSam Leffler 871b0909d5SConrad Meyer static void swcr_freesession(device_t dev, crypto_session_t cses); 88091d81d1SSam Leffler 89c0341432SJohn Baldwin /* Used for CRYPTO_NULL_CBC. */ 90c0341432SJohn Baldwin static int 91c0341432SJohn Baldwin swcr_null(struct swcr_session *ses, struct cryptop *crp) 92c0341432SJohn Baldwin { 93c0341432SJohn Baldwin 94c0341432SJohn Baldwin return (0); 95c0341432SJohn Baldwin } 96c0341432SJohn Baldwin 97091d81d1SSam Leffler /* 98091d81d1SSam Leffler * Apply a symmetric encryption/decryption algorithm. 99091d81d1SSam Leffler */ 100091d81d1SSam Leffler static int 101c0341432SJohn Baldwin swcr_encdec(struct swcr_session *ses, struct cryptop *crp) 102091d81d1SSam Leffler { 1035d7ae54aSConrad Meyer unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; 10408fca7a5SJohn-Mark Gurney unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; 105c0341432SJohn Baldwin const struct crypto_session_params *csp; 106c0341432SJohn Baldwin struct swcr_encdec *sw; 107091d81d1SSam Leffler struct enc_xform *exf; 1089c0e3d3aSJohn Baldwin int i, blks, inlen, ivlen, outlen, resid; 1099c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 1109c0e3d3aSJohn Baldwin const char *inblk; 1119c0e3d3aSJohn Baldwin char *outblk; 11208fca7a5SJohn-Mark Gurney int error; 113c0341432SJohn Baldwin bool encrypting; 11408fca7a5SJohn-Mark Gurney 11508fca7a5SJohn-Mark Gurney error = 0; 116091d81d1SSam Leffler 117c0341432SJohn Baldwin sw = &ses->swcr_encdec; 118091d81d1SSam Leffler exf = sw->sw_exf; 11908fca7a5SJohn-Mark Gurney ivlen = exf->ivsize; 120091d81d1SSam Leffler 121723d8764SJohn Baldwin if (exf->native_blocksize == 0) { 122091d81d1SSam Leffler /* Check for non-padded data */ 123723d8764SJohn Baldwin if ((crp->crp_payload_length % exf->blocksize) != 0) 124723d8764SJohn Baldwin return (EINVAL); 125723d8764SJohn Baldwin 126723d8764SJohn Baldwin blks = exf->blocksize; 127723d8764SJohn Baldwin } else 128723d8764SJohn Baldwin blks = exf->native_blocksize; 129091d81d1SSam Leffler 130c0341432SJohn Baldwin if (exf == &enc_xform_aes_icm && 131c0341432SJohn Baldwin (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 13208fca7a5SJohn-Mark Gurney return (EINVAL); 13308fca7a5SJohn-Mark Gurney 134c0341432SJohn Baldwin if (crp->crp_cipher_key != NULL) { 135c0341432SJohn Baldwin csp = crypto_get_params(crp->crp_session); 1363e947048SJohn Baldwin error = exf->setkey(sw->sw_kschedule, 137c0341432SJohn Baldwin crp->crp_cipher_key, csp->csp_cipher_klen); 138c740ae4bSPoul-Henning Kamp if (error) 139c740ae4bSPoul-Henning Kamp return (error); 140c740ae4bSPoul-Henning Kamp } 141d295bdeeSPawel Jakub Dawidek 14220c128daSJohn Baldwin crypto_read_iv(crp, iv); 14320c128daSJohn Baldwin 14408fca7a5SJohn-Mark Gurney if (exf->reinit) { 145d295bdeeSPawel Jakub Dawidek /* 146d295bdeeSPawel Jakub Dawidek * xforms that provide a reinit method perform all IV 147d295bdeeSPawel Jakub Dawidek * handling themselves. 148d295bdeeSPawel Jakub Dawidek */ 149d295bdeeSPawel Jakub Dawidek exf->reinit(sw->sw_kschedule, iv); 150091d81d1SSam Leffler } 151091d81d1SSam Leffler 1529c0e3d3aSJohn Baldwin ivp = iv; 153091d81d1SSam Leffler 1549c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 1559c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1569c0e3d3aSJohn Baldwin inlen = crypto_cursor_seglen(&cc_in); 1579c0e3d3aSJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 1589c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1599c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 1609c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1619c0e3d3aSJohn Baldwin } else 1629c0e3d3aSJohn Baldwin cc_out = cc_in; 1639c0e3d3aSJohn Baldwin outlen = crypto_cursor_seglen(&cc_out); 1649c0e3d3aSJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 1659c0e3d3aSJohn Baldwin 1669c0e3d3aSJohn Baldwin resid = crp->crp_payload_length; 167c0341432SJohn Baldwin encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 168091d81d1SSam Leffler 169091d81d1SSam Leffler /* 1709c0e3d3aSJohn Baldwin * Loop through encrypting blocks. 'inlen' is the remaining 1719c0e3d3aSJohn Baldwin * length of the current segment in the input buffer. 1729c0e3d3aSJohn Baldwin * 'outlen' is the remaining length of current segment in the 1739c0e3d3aSJohn Baldwin * output buffer. 174091d81d1SSam Leffler */ 1759c0e3d3aSJohn Baldwin while (resid >= blks) { 1769c0e3d3aSJohn Baldwin /* 1779c0e3d3aSJohn Baldwin * If the current block is not contained within the 1789c0e3d3aSJohn Baldwin * current input/output segment, use 'blk' as a local 1799c0e3d3aSJohn Baldwin * buffer. 1809c0e3d3aSJohn Baldwin */ 1819c0e3d3aSJohn Baldwin if (inlen < blks) { 1829c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, blks, blk); 1839c0e3d3aSJohn Baldwin inblk = blk; 184d295bdeeSPawel Jakub Dawidek } 1859c0e3d3aSJohn Baldwin if (outlen < blks) 1869c0e3d3aSJohn Baldwin outblk = blk; 1879c0e3d3aSJohn Baldwin 1889c0e3d3aSJohn Baldwin /* 1899c0e3d3aSJohn Baldwin * Ciphers without a 'reinit' hook are assumed to be 1909c0e3d3aSJohn Baldwin * used in CBC mode where the chaining is done here. 1919c0e3d3aSJohn Baldwin */ 1929c0e3d3aSJohn Baldwin if (exf->reinit != NULL) { 1939c0e3d3aSJohn Baldwin if (encrypting) 1949c0e3d3aSJohn Baldwin exf->encrypt(sw->sw_kschedule, inblk, outblk); 1959c0e3d3aSJohn Baldwin else 1969c0e3d3aSJohn Baldwin exf->decrypt(sw->sw_kschedule, inblk, outblk); 197c0341432SJohn Baldwin } else if (encrypting) { 198091d81d1SSam Leffler /* XOR with previous block */ 1999c0e3d3aSJohn Baldwin for (i = 0; i < blks; i++) 2009c0e3d3aSJohn Baldwin outblk[i] = inblk[i] ^ ivp[i]; 201091d81d1SSam Leffler 2029c0e3d3aSJohn Baldwin exf->encrypt(sw->sw_kschedule, outblk, outblk); 203091d81d1SSam Leffler 204091d81d1SSam Leffler /* 205091d81d1SSam Leffler * Keep encrypted block for XOR'ing 206091d81d1SSam Leffler * with next block 207091d81d1SSam Leffler */ 2089c0e3d3aSJohn Baldwin memcpy(iv, outblk, blks); 209091d81d1SSam Leffler ivp = iv; 210091d81d1SSam Leffler } else { /* decrypt */ 211091d81d1SSam Leffler /* 212091d81d1SSam Leffler * Keep encrypted block for XOR'ing 213091d81d1SSam Leffler * with next block 214091d81d1SSam Leffler */ 21508fca7a5SJohn-Mark Gurney nivp = (ivp == iv) ? iv2 : iv; 2169c0e3d3aSJohn Baldwin memcpy(nivp, inblk, blks); 217091d81d1SSam Leffler 2189c0e3d3aSJohn Baldwin exf->decrypt(sw->sw_kschedule, inblk, outblk); 219091d81d1SSam Leffler 220091d81d1SSam Leffler /* XOR with previous block */ 2219c0e3d3aSJohn Baldwin for (i = 0; i < blks; i++) 2229c0e3d3aSJohn Baldwin outblk[i] ^= ivp[i]; 223091d81d1SSam Leffler 22408fca7a5SJohn-Mark Gurney ivp = nivp; 225091d81d1SSam Leffler } 226091d81d1SSam Leffler 2279c0e3d3aSJohn Baldwin if (inlen < blks) { 2289c0e3d3aSJohn Baldwin inlen = crypto_cursor_seglen(&cc_in); 2299c0e3d3aSJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 2309c0e3d3aSJohn Baldwin } else { 2319c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, blks); 2329c0e3d3aSJohn Baldwin inlen -= blks; 2339c0e3d3aSJohn Baldwin inblk += blks; 23408fca7a5SJohn-Mark Gurney } 235091d81d1SSam Leffler 2369c0e3d3aSJohn Baldwin if (outlen < blks) { 2379c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, blks, blk); 2389c0e3d3aSJohn Baldwin outlen = crypto_cursor_seglen(&cc_out); 2399c0e3d3aSJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 2409c0e3d3aSJohn Baldwin } else { 2419c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, blks); 2429c0e3d3aSJohn Baldwin outlen -= blks; 2439c0e3d3aSJohn Baldwin outblk += blks; 244091d81d1SSam Leffler } 245091d81d1SSam Leffler 2469c0e3d3aSJohn Baldwin resid -= blks; 247f34a967bSPawel Jakub Dawidek } 248f34a967bSPawel Jakub Dawidek 249723d8764SJohn Baldwin /* Handle trailing partial block for stream ciphers. */ 2509c0e3d3aSJohn Baldwin if (resid > 0) { 251723d8764SJohn Baldwin KASSERT(exf->native_blocksize != 0, 252723d8764SJohn Baldwin ("%s: partial block of %d bytes for cipher %s", 253723d8764SJohn Baldwin __func__, i, exf->name)); 254723d8764SJohn Baldwin KASSERT(exf->reinit != NULL, 255723d8764SJohn Baldwin ("%s: partial block cipher %s without reinit hook", 256723d8764SJohn Baldwin __func__, exf->name)); 2579c0e3d3aSJohn Baldwin KASSERT(resid < blks, ("%s: partial block too big", __func__)); 258723d8764SJohn Baldwin 2599c0e3d3aSJohn Baldwin inlen = crypto_cursor_seglen(&cc_in); 2609c0e3d3aSJohn Baldwin outlen = crypto_cursor_seglen(&cc_out); 2619c0e3d3aSJohn Baldwin if (inlen < resid) { 2629c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, resid, blk); 2639c0e3d3aSJohn Baldwin inblk = blk; 2649c0e3d3aSJohn Baldwin } else 2659c0e3d3aSJohn Baldwin inblk = crypto_cursor_segbase(&cc_in); 2669c0e3d3aSJohn Baldwin if (outlen < resid) 2679c0e3d3aSJohn Baldwin outblk = blk; 2689c0e3d3aSJohn Baldwin else 2699c0e3d3aSJohn Baldwin outblk = crypto_cursor_segbase(&cc_out); 2709c0e3d3aSJohn Baldwin if (encrypting) 2719c0e3d3aSJohn Baldwin exf->encrypt_last(sw->sw_kschedule, inblk, outblk, 2729c0e3d3aSJohn Baldwin resid); 2739c0e3d3aSJohn Baldwin else 2749c0e3d3aSJohn Baldwin exf->decrypt_last(sw->sw_kschedule, inblk, outblk, 2759c0e3d3aSJohn Baldwin resid); 2769c0e3d3aSJohn Baldwin if (outlen < resid) 2779c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, resid, blk); 278723d8764SJohn Baldwin } 279723d8764SJohn Baldwin 28020c128daSJohn Baldwin explicit_bzero(blk, sizeof(blk)); 28120c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 28220c128daSJohn Baldwin explicit_bzero(iv2, sizeof(iv2)); 2839c0e3d3aSJohn Baldwin return (0); 284091d81d1SSam Leffler } 285091d81d1SSam Leffler 286c0341432SJohn Baldwin static void 287c0341432SJohn Baldwin swcr_authprepare(struct auth_hash *axf, struct swcr_auth *sw, 288c0341432SJohn Baldwin const uint8_t *key, int klen) 289f6c4bc3bSPawel Jakub Dawidek { 290f6c4bc3bSPawel Jakub Dawidek 291f6c4bc3bSPawel Jakub Dawidek switch (axf->type) { 292f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA1_HMAC: 293c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 294f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 295f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 296f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 297f6c4bc3bSPawel Jakub Dawidek case CRYPTO_NULL_HMAC: 298f6c4bc3bSPawel Jakub Dawidek case CRYPTO_RIPEMD160_HMAC: 299c0341432SJohn Baldwin hmac_init_ipad(axf, key, klen, sw->sw_ictx); 300c0341432SJohn Baldwin hmac_init_opad(axf, key, klen, sw->sw_octx); 301f6c4bc3bSPawel Jakub Dawidek break; 30225b7033bSConrad Meyer case CRYPTO_POLY1305: 3030e33efe4SConrad Meyer case CRYPTO_BLAKE2B: 3040e33efe4SConrad Meyer case CRYPTO_BLAKE2S: 3050e33efe4SConrad Meyer axf->Setkey(sw->sw_ictx, key, klen); 3060e33efe4SConrad Meyer axf->Init(sw->sw_ictx); 3070e33efe4SConrad Meyer break; 308f6c4bc3bSPawel Jakub Dawidek default: 309c0341432SJohn Baldwin panic("%s: algorithm %d doesn't use keys", __func__, axf->type); 310f6c4bc3bSPawel Jakub Dawidek } 311f6c4bc3bSPawel Jakub Dawidek } 312f6c4bc3bSPawel Jakub Dawidek 313091d81d1SSam Leffler /* 314c0341432SJohn Baldwin * Compute or verify hash. 315091d81d1SSam Leffler */ 316091d81d1SSam Leffler static int 317c0341432SJohn Baldwin swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) 318091d81d1SSam Leffler { 319c0341432SJohn Baldwin u_char aalg[HASH_MAX_LEN]; 320c0341432SJohn Baldwin const struct crypto_session_params *csp; 321c0341432SJohn Baldwin struct swcr_auth *sw; 322091d81d1SSam Leffler struct auth_hash *axf; 323091d81d1SSam Leffler union authctx ctx; 324091d81d1SSam Leffler int err; 325091d81d1SSam Leffler 326c0341432SJohn Baldwin sw = &ses->swcr_auth; 327091d81d1SSam Leffler 328091d81d1SSam Leffler axf = sw->sw_axf; 329091d81d1SSam Leffler 330c0341432SJohn Baldwin if (crp->crp_auth_key != NULL) { 331c0341432SJohn Baldwin csp = crypto_get_params(crp->crp_session); 332c0341432SJohn Baldwin swcr_authprepare(axf, sw, crp->crp_auth_key, 333c0341432SJohn Baldwin csp->csp_auth_klen); 33425b7033bSConrad Meyer } 335f6c4bc3bSPawel Jakub Dawidek 336091d81d1SSam Leffler bcopy(sw->sw_ictx, &ctx, axf->ctxsize); 337091d81d1SSam Leffler 338c0341432SJohn Baldwin err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 339c0341432SJohn Baldwin (int (*)(void *, void *, unsigned int))axf->Update, &ctx); 340091d81d1SSam Leffler if (err) 341091d81d1SSam Leffler return err; 342091d81d1SSam Leffler 3439c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 3449c0e3d3aSJohn Baldwin CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 3459c0e3d3aSJohn Baldwin err = crypto_apply_buf(&crp->crp_obuf, 3469c0e3d3aSJohn Baldwin crp->crp_payload_output_start, crp->crp_payload_length, 3479c0e3d3aSJohn Baldwin (int (*)(void *, void *, unsigned int))axf->Update, &ctx); 3489c0e3d3aSJohn Baldwin else 3499c0e3d3aSJohn Baldwin err = crypto_apply(crp, crp->crp_payload_start, 3509c0e3d3aSJohn Baldwin crp->crp_payload_length, 351c0341432SJohn Baldwin (int (*)(void *, void *, unsigned int))axf->Update, &ctx); 352c0341432SJohn Baldwin if (err) 353c0341432SJohn Baldwin return err; 354c0341432SJohn Baldwin 355c0341432SJohn Baldwin switch (axf->type) { 356c4729f6eSConrad Meyer case CRYPTO_SHA1: 357c4729f6eSConrad Meyer case CRYPTO_SHA2_224: 358c4729f6eSConrad Meyer case CRYPTO_SHA2_256: 359c4729f6eSConrad Meyer case CRYPTO_SHA2_384: 360c4729f6eSConrad Meyer case CRYPTO_SHA2_512: 361c4729f6eSConrad Meyer axf->Final(aalg, &ctx); 362c4729f6eSConrad Meyer break; 363c4729f6eSConrad Meyer 364091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 365c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 366f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 367f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 368f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 369091d81d1SSam Leffler case CRYPTO_RIPEMD160_HMAC: 370091d81d1SSam Leffler if (sw->sw_octx == NULL) 371091d81d1SSam Leffler return EINVAL; 372091d81d1SSam Leffler 373091d81d1SSam Leffler axf->Final(aalg, &ctx); 374091d81d1SSam Leffler bcopy(sw->sw_octx, &ctx, axf->ctxsize); 375091d81d1SSam Leffler axf->Update(&ctx, aalg, axf->hashsize); 376091d81d1SSam Leffler axf->Final(aalg, &ctx); 377091d81d1SSam Leffler break; 378091d81d1SSam Leffler 3790e33efe4SConrad Meyer case CRYPTO_BLAKE2B: 3800e33efe4SConrad Meyer case CRYPTO_BLAKE2S: 381091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 38225b7033bSConrad Meyer case CRYPTO_POLY1305: 383091d81d1SSam Leffler axf->Final(aalg, &ctx); 384091d81d1SSam Leffler break; 385091d81d1SSam Leffler } 386091d81d1SSam Leffler 387c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 38820c128daSJohn Baldwin u_char uaalg[HASH_MAX_LEN]; 38920c128daSJohn Baldwin 390c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 391c0341432SJohn Baldwin if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 39220c128daSJohn Baldwin err = EBADMSG; 39320c128daSJohn Baldwin explicit_bzero(uaalg, sizeof(uaalg)); 394c0341432SJohn Baldwin } else { 395091d81d1SSam Leffler /* Inject the authentication data */ 396c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 397c0341432SJohn Baldwin } 39820c128daSJohn Baldwin explicit_bzero(aalg, sizeof(aalg)); 39920c128daSJohn Baldwin return (err); 400091d81d1SSam Leffler } 401091d81d1SSam Leffler 40208fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 40308fca7a5SJohn-Mark Gurney CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 40408fca7a5SJohn-Mark Gurney 40508fca7a5SJohn-Mark Gurney static int 406c0341432SJohn Baldwin swcr_gmac(struct swcr_session *ses, struct cryptop *crp) 40708fca7a5SJohn-Mark Gurney { 40808fca7a5SJohn-Mark Gurney uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; 40908fca7a5SJohn-Mark Gurney u_char *blk = (u_char *)blkbuf; 41008fca7a5SJohn-Mark Gurney u_char aalg[AALG_MAX_RESULT_LEN]; 41108fca7a5SJohn-Mark Gurney u_char iv[EALG_MAX_BLOCK_LEN]; 4129c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc; 41308fca7a5SJohn-Mark Gurney union authctx ctx; 414c0341432SJohn Baldwin struct swcr_auth *swa; 415c0341432SJohn Baldwin struct auth_hash *axf; 41608fca7a5SJohn-Mark Gurney uint32_t *blkp; 41720c128daSJohn Baldwin int blksz, error, ivlen, len, resid; 41808fca7a5SJohn-Mark Gurney 419c0341432SJohn Baldwin swa = &ses->swcr_auth; 42008fca7a5SJohn-Mark Gurney axf = swa->sw_axf; 421c0341432SJohn Baldwin 42208fca7a5SJohn-Mark Gurney bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 42308fca7a5SJohn-Mark Gurney blksz = axf->blocksize; 42408fca7a5SJohn-Mark Gurney 42508fca7a5SJohn-Mark Gurney /* Initialize the IV */ 426c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 42729fe41ddSJohn Baldwin crypto_read_iv(crp, iv); 42808fca7a5SJohn-Mark Gurney 42908fca7a5SJohn-Mark Gurney axf->Reinit(&ctx, iv, ivlen); 4309c0e3d3aSJohn Baldwin crypto_cursor_init(&cc, &crp->crp_buf); 4319c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc, crp->crp_payload_start); 4329c0e3d3aSJohn Baldwin for (resid = crp->crp_payload_length; resid > 0; resid -= len) { 4339c0e3d3aSJohn Baldwin len = MIN(resid, blksz); 4349c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc, len, blk); 435c0341432SJohn Baldwin bzero(blk + len, blksz - len); 43608fca7a5SJohn-Mark Gurney axf->Update(&ctx, blk, blksz); 43708fca7a5SJohn-Mark Gurney } 43808fca7a5SJohn-Mark Gurney 43908fca7a5SJohn-Mark Gurney /* length block */ 44008fca7a5SJohn-Mark Gurney bzero(blk, blksz); 44108fca7a5SJohn-Mark Gurney blkp = (uint32_t *)blk + 1; 442c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 44308fca7a5SJohn-Mark Gurney axf->Update(&ctx, blk, blksz); 444c0341432SJohn Baldwin 445c0341432SJohn Baldwin /* Finalize MAC */ 446c0341432SJohn Baldwin axf->Final(aalg, &ctx); 447c0341432SJohn Baldwin 44820c128daSJohn Baldwin error = 0; 449c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 45020c128daSJohn Baldwin u_char uaalg[AALG_MAX_RESULT_LEN]; 45120c128daSJohn Baldwin 452c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 453c0341432SJohn Baldwin uaalg); 454c0341432SJohn Baldwin if (timingsafe_bcmp(aalg, uaalg, swa->sw_mlen) != 0) 45520c128daSJohn Baldwin error = EBADMSG; 45620c128daSJohn Baldwin explicit_bzero(uaalg, sizeof(uaalg)); 457c0341432SJohn Baldwin } else { 458c0341432SJohn Baldwin /* Inject the authentication data */ 459c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, aalg); 460c0341432SJohn Baldwin } 46120c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 46220c128daSJohn Baldwin explicit_bzero(aalg, sizeof(aalg)); 46320c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 46420c128daSJohn Baldwin return (error); 465c0341432SJohn Baldwin } 466c0341432SJohn Baldwin 467c0341432SJohn Baldwin static int 468c0341432SJohn Baldwin swcr_gcm(struct swcr_session *ses, struct cryptop *crp) 469c0341432SJohn Baldwin { 470c0341432SJohn Baldwin uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; 471c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 472c0341432SJohn Baldwin u_char aalg[AALG_MAX_RESULT_LEN]; 473c0341432SJohn Baldwin u_char iv[EALG_MAX_BLOCK_LEN]; 4749c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 475c0341432SJohn Baldwin union authctx ctx; 476c0341432SJohn Baldwin struct swcr_auth *swa; 477c0341432SJohn Baldwin struct swcr_encdec *swe; 478c0341432SJohn Baldwin struct auth_hash *axf; 479c0341432SJohn Baldwin struct enc_xform *exf; 480c0341432SJohn Baldwin uint32_t *blkp; 48120c128daSJohn Baldwin int blksz, error, ivlen, len, r, resid; 482c0341432SJohn Baldwin 483c0341432SJohn Baldwin swa = &ses->swcr_auth; 484c0341432SJohn Baldwin axf = swa->sw_axf; 485c0341432SJohn Baldwin 486c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 487c0341432SJohn Baldwin blksz = axf->blocksize; 488c0341432SJohn Baldwin 489c0341432SJohn Baldwin swe = &ses->swcr_encdec; 490c0341432SJohn Baldwin exf = swe->sw_exf; 491723d8764SJohn Baldwin KASSERT(axf->blocksize == exf->native_blocksize, 492723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 493c0341432SJohn Baldwin 494c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 495c0341432SJohn Baldwin return (EINVAL); 496c0341432SJohn Baldwin 497c0341432SJohn Baldwin /* Initialize the IV */ 498c0341432SJohn Baldwin ivlen = AES_GCM_IV_LEN; 499c0341432SJohn Baldwin bcopy(crp->crp_iv, iv, ivlen); 500c0341432SJohn Baldwin 501c0341432SJohn Baldwin /* Supply MAC with IV */ 502c0341432SJohn Baldwin axf->Reinit(&ctx, iv, ivlen); 503c0341432SJohn Baldwin 504c0341432SJohn Baldwin /* Supply MAC with AAD */ 5059c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5069c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_aad_start); 5079c0e3d3aSJohn Baldwin for (resid = crp->crp_aad_length; resid > 0; resid -= len) { 5089c0e3d3aSJohn Baldwin len = MIN(resid, blksz); 5099c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 510c0341432SJohn Baldwin bzero(blk + len, blksz - len); 511c0341432SJohn Baldwin axf->Update(&ctx, blk, blksz); 512c0341432SJohn Baldwin } 513c0341432SJohn Baldwin 514c0341432SJohn Baldwin exf->reinit(swe->sw_kschedule, iv); 515c0341432SJohn Baldwin 516c0341432SJohn Baldwin /* Do encryption with MAC */ 5179c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5189c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 5199c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 5209c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 5219c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 5229c0e3d3aSJohn Baldwin } else 5239c0e3d3aSJohn Baldwin cc_out = cc_in; 5249c0e3d3aSJohn Baldwin for (resid = crp->crp_payload_length; resid > 0; resid -= len) { 5259c0e3d3aSJohn Baldwin len = MIN(resid, blksz); 526c0341432SJohn Baldwin if (len < blksz) 527c0341432SJohn Baldwin bzero(blk, blksz); 5289c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 529c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 5303e947048SJohn Baldwin exf->encrypt(swe->sw_kschedule, blk, blk); 531c0341432SJohn Baldwin axf->Update(&ctx, blk, len); 5329c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, len, blk); 533c0341432SJohn Baldwin } else { 534c0341432SJohn Baldwin axf->Update(&ctx, blk, len); 535c0341432SJohn Baldwin } 536c0341432SJohn Baldwin } 537c0341432SJohn Baldwin 538c0341432SJohn Baldwin /* length block */ 539c0341432SJohn Baldwin bzero(blk, blksz); 540c0341432SJohn Baldwin blkp = (uint32_t *)blk + 1; 541c0341432SJohn Baldwin *blkp = htobe32(crp->crp_aad_length * 8); 542c0341432SJohn Baldwin blkp = (uint32_t *)blk + 3; 543c0341432SJohn Baldwin *blkp = htobe32(crp->crp_payload_length * 8); 544c0341432SJohn Baldwin axf->Update(&ctx, blk, blksz); 545c0341432SJohn Baldwin 546c0341432SJohn Baldwin /* Finalize MAC */ 547c0341432SJohn Baldwin axf->Final(aalg, &ctx); 548c0341432SJohn Baldwin 549c0341432SJohn Baldwin /* Validate tag */ 55020c128daSJohn Baldwin error = 0; 551c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 55220c128daSJohn Baldwin u_char uaalg[AALG_MAX_RESULT_LEN]; 55320c128daSJohn Baldwin 554c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 555c0341432SJohn Baldwin uaalg); 556c0341432SJohn Baldwin 557c0341432SJohn Baldwin r = timingsafe_bcmp(aalg, uaalg, swa->sw_mlen); 55820c128daSJohn Baldwin explicit_bzero(uaalg, sizeof(uaalg)); 55920c128daSJohn Baldwin if (r != 0) { 56020c128daSJohn Baldwin error = EBADMSG; 56120c128daSJohn Baldwin goto out; 56220c128daSJohn Baldwin } 563c0341432SJohn Baldwin 564c0341432SJohn Baldwin /* tag matches, decrypt data */ 5659c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 5669c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 5679c0e3d3aSJohn Baldwin for (resid = crp->crp_payload_length; resid > 0; 5689c0e3d3aSJohn Baldwin resid -= len) { 5699c0e3d3aSJohn Baldwin len = MIN(resid, blksz); 570c0341432SJohn Baldwin if (len < blksz) 571c0341432SJohn Baldwin bzero(blk, blksz); 5729c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 5733e947048SJohn Baldwin exf->decrypt(swe->sw_kschedule, blk, blk); 5749c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, len, blk); 575c0341432SJohn Baldwin } 576c0341432SJohn Baldwin } else { 577c0341432SJohn Baldwin /* Inject the authentication data */ 578c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 579c0341432SJohn Baldwin aalg); 580c0341432SJohn Baldwin } 581c0341432SJohn Baldwin 58220c128daSJohn Baldwin out: 58320c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 58420c128daSJohn Baldwin explicit_bzero(aalg, sizeof(aalg)); 58520c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 58620c128daSJohn Baldwin 58720c128daSJohn Baldwin return (error); 588c0341432SJohn Baldwin } 589c0341432SJohn Baldwin 590c0341432SJohn Baldwin static int 591c0341432SJohn Baldwin swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) 592c0341432SJohn Baldwin { 593c0341432SJohn Baldwin uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; 594c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 595c0341432SJohn Baldwin u_char aalg[AALG_MAX_RESULT_LEN]; 596c0341432SJohn Baldwin u_char iv[EALG_MAX_BLOCK_LEN]; 5979c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc; 598c0341432SJohn Baldwin union authctx ctx; 599c0341432SJohn Baldwin struct swcr_auth *swa; 600c0341432SJohn Baldwin struct auth_hash *axf; 60120c128daSJohn Baldwin int blksz, error, ivlen, len, resid; 602c0341432SJohn Baldwin 603c0341432SJohn Baldwin swa = &ses->swcr_auth; 604c0341432SJohn Baldwin axf = swa->sw_axf; 605c0341432SJohn Baldwin 606c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 607c0341432SJohn Baldwin blksz = axf->blocksize; 608c0341432SJohn Baldwin 609c0341432SJohn Baldwin /* Initialize the IV */ 610c0341432SJohn Baldwin ivlen = AES_CCM_IV_LEN; 61129fe41ddSJohn Baldwin crypto_read_iv(crp, iv); 612c0341432SJohn Baldwin 613c0341432SJohn Baldwin /* 614c0341432SJohn Baldwin * AES CCM-CBC-MAC needs to know the length of both the auth 615c0341432SJohn Baldwin * data and payload data before doing the auth computation. 616c0341432SJohn Baldwin */ 617c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_payload_length; 618c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.cryptDataLength = 0; 619c0341432SJohn Baldwin 620c0341432SJohn Baldwin axf->Reinit(&ctx, iv, ivlen); 6219c0e3d3aSJohn Baldwin crypto_cursor_init(&cc, &crp->crp_buf); 6229c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc, crp->crp_aad_start); 6239c0e3d3aSJohn Baldwin for (resid = crp->crp_payload_length; resid > 0; resid -= len) { 6249c0e3d3aSJohn Baldwin len = MIN(resid, blksz); 6259c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc, len, blk); 626c0341432SJohn Baldwin bzero(blk + len, blksz - len); 627c0341432SJohn Baldwin axf->Update(&ctx, blk, blksz); 628c0341432SJohn Baldwin } 629c0341432SJohn Baldwin 630c0341432SJohn Baldwin /* Finalize MAC */ 631c0341432SJohn Baldwin axf->Final(aalg, &ctx); 632c0341432SJohn Baldwin 63320c128daSJohn Baldwin error = 0; 634c0341432SJohn Baldwin if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 63520c128daSJohn Baldwin u_char uaalg[AALG_MAX_RESULT_LEN]; 63620c128daSJohn Baldwin 637c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 638c0341432SJohn Baldwin uaalg); 639c0341432SJohn Baldwin if (timingsafe_bcmp(aalg, uaalg, swa->sw_mlen) != 0) 64020c128daSJohn Baldwin error = EBADMSG; 64120c128daSJohn Baldwin explicit_bzero(uaalg, sizeof(uaalg)); 642c0341432SJohn Baldwin } else { 643c0341432SJohn Baldwin /* Inject the authentication data */ 644c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, aalg); 645c0341432SJohn Baldwin } 64620c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 64720c128daSJohn Baldwin explicit_bzero(aalg, sizeof(aalg)); 64820c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 64920c128daSJohn Baldwin return (error); 650c0341432SJohn Baldwin } 651c0341432SJohn Baldwin 652c0341432SJohn Baldwin static int 653c0341432SJohn Baldwin swcr_ccm(struct swcr_session *ses, struct cryptop *crp) 654c0341432SJohn Baldwin { 655c0341432SJohn Baldwin uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; 656c0341432SJohn Baldwin u_char *blk = (u_char *)blkbuf; 657c0341432SJohn Baldwin u_char aalg[AALG_MAX_RESULT_LEN]; 658c0341432SJohn Baldwin u_char iv[EALG_MAX_BLOCK_LEN]; 6599c0e3d3aSJohn Baldwin struct crypto_buffer_cursor cc_in, cc_out; 660c0341432SJohn Baldwin union authctx ctx; 661c0341432SJohn Baldwin struct swcr_auth *swa; 662c0341432SJohn Baldwin struct swcr_encdec *swe; 663c0341432SJohn Baldwin struct auth_hash *axf; 664c0341432SJohn Baldwin struct enc_xform *exf; 66520c128daSJohn Baldwin int blksz, error, ivlen, len, r, resid; 666c0341432SJohn Baldwin 667c0341432SJohn Baldwin swa = &ses->swcr_auth; 668c0341432SJohn Baldwin axf = swa->sw_axf; 669c0341432SJohn Baldwin 670c0341432SJohn Baldwin bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 671c0341432SJohn Baldwin blksz = axf->blocksize; 672c0341432SJohn Baldwin 673c0341432SJohn Baldwin swe = &ses->swcr_encdec; 674c0341432SJohn Baldwin exf = swe->sw_exf; 675723d8764SJohn Baldwin KASSERT(axf->blocksize == exf->native_blocksize, 676723d8764SJohn Baldwin ("%s: blocksize mismatch", __func__)); 677c0341432SJohn Baldwin 678c0341432SJohn Baldwin if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 679c0341432SJohn Baldwin return (EINVAL); 680c0341432SJohn Baldwin 681c0341432SJohn Baldwin /* Initialize the IV */ 682c0341432SJohn Baldwin ivlen = AES_CCM_IV_LEN; 683c0341432SJohn Baldwin bcopy(crp->crp_iv, iv, ivlen); 684c0341432SJohn Baldwin 685c0341432SJohn Baldwin /* 686c0341432SJohn Baldwin * AES CCM-CBC-MAC needs to know the length of both the auth 687c0341432SJohn Baldwin * data and payload data before doing the auth computation. 688c0341432SJohn Baldwin */ 689c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; 690c0341432SJohn Baldwin ctx.aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; 691c0341432SJohn Baldwin 692c0341432SJohn Baldwin /* Supply MAC with IV */ 693c0341432SJohn Baldwin axf->Reinit(&ctx, iv, ivlen); 694c0341432SJohn Baldwin 695c0341432SJohn Baldwin /* Supply MAC with AAD */ 6969c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 6979c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_aad_start); 6989c0e3d3aSJohn Baldwin for (resid = crp->crp_aad_length; resid > 0; resid -= len) { 6999c0e3d3aSJohn Baldwin len = MIN(resid, blksz); 7009c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 701c0341432SJohn Baldwin bzero(blk + len, blksz - len); 702c0341432SJohn Baldwin axf->Update(&ctx, blk, blksz); 703c0341432SJohn Baldwin } 704c0341432SJohn Baldwin 705c0341432SJohn Baldwin exf->reinit(swe->sw_kschedule, iv); 706c0341432SJohn Baldwin 707c0341432SJohn Baldwin /* Do encryption/decryption with MAC */ 7089c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 7099c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 7109c0e3d3aSJohn Baldwin if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 7119c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_out, &crp->crp_obuf); 7129c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 7139c0e3d3aSJohn Baldwin } else 7149c0e3d3aSJohn Baldwin cc_out = cc_in; 7159c0e3d3aSJohn Baldwin for (resid = crp->crp_payload_length; resid > 0; resid -= len) { 7169c0e3d3aSJohn Baldwin len = MIN(resid, blksz); 717c0341432SJohn Baldwin if (len < blksz) 718c0341432SJohn Baldwin bzero(blk, blksz); 7199c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 720c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 721c0341432SJohn Baldwin axf->Update(&ctx, blk, len); 7223e947048SJohn Baldwin exf->encrypt(swe->sw_kschedule, blk, blk); 7239c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, len, blk); 724c0341432SJohn Baldwin } else { 725c0341432SJohn Baldwin /* 726c0341432SJohn Baldwin * One of the problems with CCM+CBC is that 727c0341432SJohn Baldwin * the authentication is done on the 728c0341432SJohn Baldwin * unecncrypted data. As a result, we have to 729c0341432SJohn Baldwin * decrypt the data twice: once to generate 730c0341432SJohn Baldwin * the tag and a second time after the tag is 731c0341432SJohn Baldwin * verified. 732c0341432SJohn Baldwin */ 7333e947048SJohn Baldwin exf->decrypt(swe->sw_kschedule, blk, blk); 734c0341432SJohn Baldwin axf->Update(&ctx, blk, len); 735c0341432SJohn Baldwin } 73608fca7a5SJohn-Mark Gurney } 73708fca7a5SJohn-Mark Gurney 73808fca7a5SJohn-Mark Gurney /* Finalize MAC */ 73908fca7a5SJohn-Mark Gurney axf->Final(aalg, &ctx); 74008fca7a5SJohn-Mark Gurney 74108fca7a5SJohn-Mark Gurney /* Validate tag */ 74220c128daSJohn Baldwin error = 0; 743c0341432SJohn Baldwin if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 74420c128daSJohn Baldwin u_char uaalg[AALG_MAX_RESULT_LEN]; 74520c128daSJohn Baldwin 746c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 747c0341432SJohn Baldwin uaalg); 74808fca7a5SJohn-Mark Gurney 749c0341432SJohn Baldwin r = timingsafe_bcmp(aalg, uaalg, swa->sw_mlen); 75020c128daSJohn Baldwin explicit_bzero(uaalg, sizeof(uaalg)); 75120c128daSJohn Baldwin if (r != 0) { 75220c128daSJohn Baldwin error = EBADMSG; 75320c128daSJohn Baldwin goto out; 75420c128daSJohn Baldwin } 755c0341432SJohn Baldwin 75608fca7a5SJohn-Mark Gurney /* tag matches, decrypt data */ 757507281e5SSean Eric Fagan exf->reinit(swe->sw_kschedule, iv); 7589c0e3d3aSJohn Baldwin crypto_cursor_init(&cc_in, &crp->crp_buf); 7599c0e3d3aSJohn Baldwin crypto_cursor_advance(&cc_in, crp->crp_payload_start); 7609c0e3d3aSJohn Baldwin for (resid = crp->crp_payload_length; resid > 0; 7619c0e3d3aSJohn Baldwin resid -= len) { 7629c0e3d3aSJohn Baldwin len = MIN(resid, blksz); 76308fca7a5SJohn-Mark Gurney if (len < blksz) 76408fca7a5SJohn-Mark Gurney bzero(blk, blksz); 7659c0e3d3aSJohn Baldwin crypto_cursor_copydata(&cc_in, len, blk); 7663e947048SJohn Baldwin exf->decrypt(swe->sw_kschedule, blk, blk); 7679c0e3d3aSJohn Baldwin crypto_cursor_copyback(&cc_out, len, blk); 76808fca7a5SJohn-Mark Gurney } 76908fca7a5SJohn-Mark Gurney } else { 77008fca7a5SJohn-Mark Gurney /* Inject the authentication data */ 771c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 772c0341432SJohn Baldwin aalg); 77308fca7a5SJohn-Mark Gurney } 77408fca7a5SJohn-Mark Gurney 77520c128daSJohn Baldwin out: 77620c128daSJohn Baldwin explicit_bzero(blkbuf, sizeof(blkbuf)); 77720c128daSJohn Baldwin explicit_bzero(aalg, sizeof(aalg)); 77820c128daSJohn Baldwin explicit_bzero(iv, sizeof(iv)); 77920c128daSJohn Baldwin return (error); 78008fca7a5SJohn-Mark Gurney } 78108fca7a5SJohn-Mark Gurney 782091d81d1SSam Leffler /* 783c0341432SJohn Baldwin * Apply a cipher and a digest to perform EtA. 784c0341432SJohn Baldwin */ 785c0341432SJohn Baldwin static int 786c0341432SJohn Baldwin swcr_eta(struct swcr_session *ses, struct cryptop *crp) 787c0341432SJohn Baldwin { 788c0341432SJohn Baldwin int error; 789c0341432SJohn Baldwin 790c0341432SJohn Baldwin if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 791c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 792c0341432SJohn Baldwin if (error == 0) 793c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 794c0341432SJohn Baldwin } else { 795c0341432SJohn Baldwin error = swcr_authcompute(ses, crp); 796c0341432SJohn Baldwin if (error == 0) 797c0341432SJohn Baldwin error = swcr_encdec(ses, crp); 798c0341432SJohn Baldwin } 799c0341432SJohn Baldwin return (error); 800c0341432SJohn Baldwin } 801c0341432SJohn Baldwin 802c0341432SJohn Baldwin /* 803091d81d1SSam Leffler * Apply a compression/decompression algorithm 804091d81d1SSam Leffler */ 805091d81d1SSam Leffler static int 806c0341432SJohn Baldwin swcr_compdec(struct swcr_session *ses, struct cryptop *crp) 807091d81d1SSam Leffler { 808091d81d1SSam Leffler u_int8_t *data, *out; 809091d81d1SSam Leffler struct comp_algo *cxf; 810091d81d1SSam Leffler int adj; 811091d81d1SSam Leffler u_int32_t result; 812091d81d1SSam Leffler 813c0341432SJohn Baldwin cxf = ses->swcr_compdec.sw_cxf; 814091d81d1SSam Leffler 815091d81d1SSam Leffler /* We must handle the whole buffer of data in one time 816091d81d1SSam Leffler * then if there is not all the data in the mbuf, we must 817091d81d1SSam Leffler * copy in a buffer. 818091d81d1SSam Leffler */ 819091d81d1SSam Leffler 820c0341432SJohn Baldwin data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 821091d81d1SSam Leffler if (data == NULL) 822091d81d1SSam Leffler return (EINVAL); 823c0341432SJohn Baldwin crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 824c0341432SJohn Baldwin data); 825091d81d1SSam Leffler 826c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 827c0341432SJohn Baldwin result = cxf->compress(data, crp->crp_payload_length, &out); 828091d81d1SSam Leffler else 829c0341432SJohn Baldwin result = cxf->decompress(data, crp->crp_payload_length, &out); 830091d81d1SSam Leffler 8311ede983cSDag-Erling Smørgrav free(data, M_CRYPTO_DATA); 832091d81d1SSam Leffler if (result == 0) 833c0341432SJohn Baldwin return (EINVAL); 834c0341432SJohn Baldwin crp->crp_olen = result; 835c0341432SJohn Baldwin 836c0341432SJohn Baldwin /* Check the compressed size when doing compression */ 837c0341432SJohn Baldwin if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 838c0341432SJohn Baldwin if (result >= crp->crp_payload_length) { 839c0341432SJohn Baldwin /* Compression was useless, we lost time */ 840c0341432SJohn Baldwin free(out, M_CRYPTO_DATA); 841c0341432SJohn Baldwin return (0); 842c0341432SJohn Baldwin } 843c0341432SJohn Baldwin } 844091d81d1SSam Leffler 845091d81d1SSam Leffler /* Copy back the (de)compressed data. m_copyback is 846091d81d1SSam Leffler * extending the mbuf as necessary. 847091d81d1SSam Leffler */ 848c0341432SJohn Baldwin crypto_copyback(crp, crp->crp_payload_start, result, out); 849c0341432SJohn Baldwin if (result < crp->crp_payload_length) { 8509c0e3d3aSJohn Baldwin switch (crp->crp_buf.cb_type) { 851c0341432SJohn Baldwin case CRYPTO_BUF_MBUF: 852c0341432SJohn Baldwin adj = result - crp->crp_payload_length; 8539c0e3d3aSJohn Baldwin m_adj(crp->crp_buf.cb_mbuf, adj); 854c0341432SJohn Baldwin break; 855c0341432SJohn Baldwin case CRYPTO_BUF_UIO: { 8569c0e3d3aSJohn Baldwin struct uio *uio = crp->crp_buf.cb_uio; 857091d81d1SSam Leffler int ind; 858091d81d1SSam Leffler 859c0341432SJohn Baldwin adj = crp->crp_payload_length - result; 860091d81d1SSam Leffler ind = uio->uio_iovcnt - 1; 861091d81d1SSam Leffler 862091d81d1SSam Leffler while (adj > 0 && ind >= 0) { 863091d81d1SSam Leffler if (adj < uio->uio_iov[ind].iov_len) { 864091d81d1SSam Leffler uio->uio_iov[ind].iov_len -= adj; 865091d81d1SSam Leffler break; 866091d81d1SSam Leffler } 867091d81d1SSam Leffler 868091d81d1SSam Leffler adj -= uio->uio_iov[ind].iov_len; 869091d81d1SSam Leffler uio->uio_iov[ind].iov_len = 0; 870091d81d1SSam Leffler ind--; 871091d81d1SSam Leffler uio->uio_iovcnt--; 872091d81d1SSam Leffler } 873091d81d1SSam Leffler } 874c0341432SJohn Baldwin break; 8759c0e3d3aSJohn Baldwin default: 8769c0e3d3aSJohn Baldwin break; 877c0341432SJohn Baldwin } 878091d81d1SSam Leffler } 8791ede983cSDag-Erling Smørgrav free(out, M_CRYPTO_DATA); 880091d81d1SSam Leffler return 0; 881091d81d1SSam Leffler } 882091d81d1SSam Leffler 883091d81d1SSam Leffler static int 8843e947048SJohn Baldwin swcr_setup_cipher(struct swcr_session *ses, 885c0341432SJohn Baldwin const struct crypto_session_params *csp) 886091d81d1SSam Leffler { 887c0341432SJohn Baldwin struct swcr_encdec *swe; 888091d81d1SSam Leffler struct enc_xform *txf; 889f6c4bc3bSPawel Jakub Dawidek int error; 890091d81d1SSam Leffler 891c0341432SJohn Baldwin swe = &ses->swcr_encdec; 892c0341432SJohn Baldwin txf = crypto_cipher(csp); 893c0341432SJohn Baldwin MPASS(txf->ivsize == csp->csp_ivlen); 8943e947048SJohn Baldwin if (txf->ctxsize != 0) { 8953e947048SJohn Baldwin swe->sw_kschedule = malloc(txf->ctxsize, M_CRYPTO_DATA, 8963e947048SJohn Baldwin M_NOWAIT); 8973e947048SJohn Baldwin if (swe->sw_kschedule == NULL) 8983e947048SJohn Baldwin return (ENOMEM); 8993e947048SJohn Baldwin } 900c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) { 9013e947048SJohn Baldwin error = txf->setkey(swe->sw_kschedule, 902c0341432SJohn Baldwin csp->csp_cipher_key, csp->csp_cipher_klen); 903c0341432SJohn Baldwin if (error) 904c0341432SJohn Baldwin return (error); 905091d81d1SSam Leffler } 906c0341432SJohn Baldwin swe->sw_exf = txf; 907c0341432SJohn Baldwin return (0); 908f6c4bc3bSPawel Jakub Dawidek } 909091d81d1SSam Leffler 910c0341432SJohn Baldwin static int 911c0341432SJohn Baldwin swcr_setup_auth(struct swcr_session *ses, 912c0341432SJohn Baldwin const struct crypto_session_params *csp) 913c0341432SJohn Baldwin { 914c0341432SJohn Baldwin struct swcr_auth *swa; 915c0341432SJohn Baldwin struct auth_hash *axf; 916c0341432SJohn Baldwin 917c0341432SJohn Baldwin swa = &ses->swcr_auth; 918c0341432SJohn Baldwin 919c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 920c0341432SJohn Baldwin swa->sw_axf = axf; 921c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 922c0341432SJohn Baldwin return (EINVAL); 923c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 924c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 925c0341432SJohn Baldwin else 926c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 927c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 928c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 929c0341432SJohn Baldwin return (ENOBUFS); 930c0341432SJohn Baldwin 931c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 932091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 933c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 934f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 935f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 936f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 937091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 938091d81d1SSam Leffler case CRYPTO_RIPEMD160_HMAC: 9393a0b6a93SJohn Baldwin swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 940091d81d1SSam Leffler M_NOWAIT); 941c0341432SJohn Baldwin if (swa->sw_octx == NULL) 942c0341432SJohn Baldwin return (ENOBUFS); 943c0341432SJohn Baldwin 944c0341432SJohn Baldwin if (csp->csp_auth_key != NULL) { 945c0341432SJohn Baldwin swcr_authprepare(axf, swa, csp->csp_auth_key, 946c0341432SJohn Baldwin csp->csp_auth_klen); 947091d81d1SSam Leffler } 948091d81d1SSam Leffler 949c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 950c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 951091d81d1SSam Leffler break; 952091d81d1SSam Leffler case CRYPTO_SHA1: 953c4729f6eSConrad Meyer case CRYPTO_SHA2_224: 954c4729f6eSConrad Meyer case CRYPTO_SHA2_256: 955c4729f6eSConrad Meyer case CRYPTO_SHA2_384: 956c4729f6eSConrad Meyer case CRYPTO_SHA2_512: 957c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 958c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 959c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 960c0341432SJohn Baldwin break; 961c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 962c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 963c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 964c0341432SJohn Baldwin csp->csp_auth_klen); 965c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 966c0341432SJohn Baldwin ses->swcr_process = swcr_gmac; 967c0341432SJohn Baldwin break; 968c0341432SJohn Baldwin case CRYPTO_POLY1305: 969c0341432SJohn Baldwin case CRYPTO_BLAKE2B: 970c0341432SJohn Baldwin case CRYPTO_BLAKE2S: 971c0341432SJohn Baldwin /* 972c0341432SJohn Baldwin * Blake2b and Blake2s support an optional key but do 973c0341432SJohn Baldwin * not require one. 974c0341432SJohn Baldwin */ 975c0341432SJohn Baldwin if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) 976c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 977c0341432SJohn Baldwin csp->csp_auth_klen); 978c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 979c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 980c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 981c0341432SJohn Baldwin break; 982c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 983c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 984c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 985c0341432SJohn Baldwin csp->csp_auth_klen); 986c0341432SJohn Baldwin if (csp->csp_mode == CSP_MODE_DIGEST) 987c0341432SJohn Baldwin ses->swcr_process = swcr_ccm_cbc_mac; 988c0341432SJohn Baldwin break; 989091d81d1SSam Leffler } 990091d81d1SSam Leffler 991c0341432SJohn Baldwin return (0); 992c0341432SJohn Baldwin } 99308fca7a5SJohn-Mark Gurney 994c0341432SJohn Baldwin static int 995c0341432SJohn Baldwin swcr_setup_gcm(struct swcr_session *ses, 996c0341432SJohn Baldwin const struct crypto_session_params *csp) 997c0341432SJohn Baldwin { 998c0341432SJohn Baldwin struct swcr_auth *swa; 999c0341432SJohn Baldwin struct auth_hash *axf; 1000c0341432SJohn Baldwin 1001c0341432SJohn Baldwin if (csp->csp_ivlen != AES_GCM_IV_LEN) 1002c0341432SJohn Baldwin return (EINVAL); 1003c0341432SJohn Baldwin 1004c0341432SJohn Baldwin /* First, setup the auth side. */ 1005c0341432SJohn Baldwin swa = &ses->swcr_auth; 1006c0341432SJohn Baldwin switch (csp->csp_cipher_klen * 8) { 1007c0341432SJohn Baldwin case 128: 1008c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_128; 1009c0341432SJohn Baldwin break; 1010c0341432SJohn Baldwin case 192: 1011c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_192; 1012c0341432SJohn Baldwin break; 1013c0341432SJohn Baldwin case 256: 1014c0341432SJohn Baldwin axf = &auth_hash_nist_gmac_aes_256; 1015c0341432SJohn Baldwin break; 1016c0341432SJohn Baldwin default: 1017c0341432SJohn Baldwin return (EINVAL); 1018c0341432SJohn Baldwin } 1019c0341432SJohn Baldwin swa->sw_axf = axf; 1020c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1021c0341432SJohn Baldwin return (EINVAL); 1022c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1023c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1024c0341432SJohn Baldwin else 1025c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1026c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1027c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1028c0341432SJohn Baldwin return (ENOBUFS); 1029c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1030c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) 1031c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, 1032c0341432SJohn Baldwin csp->csp_cipher_klen); 1033c0341432SJohn Baldwin 1034c0341432SJohn Baldwin /* Second, setup the cipher side. */ 10353e947048SJohn Baldwin return (swcr_setup_cipher(ses, csp)); 1036c0341432SJohn Baldwin } 1037c0341432SJohn Baldwin 1038c0341432SJohn Baldwin static int 1039c0341432SJohn Baldwin swcr_setup_ccm(struct swcr_session *ses, 1040c0341432SJohn Baldwin const struct crypto_session_params *csp) 1041c0341432SJohn Baldwin { 1042c0341432SJohn Baldwin struct swcr_auth *swa; 1043c0341432SJohn Baldwin struct auth_hash *axf; 1044c0341432SJohn Baldwin 1045c0341432SJohn Baldwin if (csp->csp_ivlen != AES_CCM_IV_LEN) 1046c0341432SJohn Baldwin return (EINVAL); 1047c0341432SJohn Baldwin 1048c0341432SJohn Baldwin /* First, setup the auth side. */ 1049c0341432SJohn Baldwin swa = &ses->swcr_auth; 1050c0341432SJohn Baldwin switch (csp->csp_cipher_klen * 8) { 1051507281e5SSean Eric Fagan case 128: 1052507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_128; 1053507281e5SSean Eric Fagan break; 1054507281e5SSean Eric Fagan case 192: 1055507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_192; 1056507281e5SSean Eric Fagan break; 1057507281e5SSean Eric Fagan case 256: 1058507281e5SSean Eric Fagan axf = &auth_hash_ccm_cbc_mac_256; 1059507281e5SSean Eric Fagan break; 1060507281e5SSean Eric Fagan default: 1061c0341432SJohn Baldwin return (EINVAL); 1062507281e5SSean Eric Fagan } 1063c0341432SJohn Baldwin swa->sw_axf = axf; 1064c0341432SJohn Baldwin if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1065c0341432SJohn Baldwin return (EINVAL); 1066c0341432SJohn Baldwin if (csp->csp_auth_mlen == 0) 1067c0341432SJohn Baldwin swa->sw_mlen = axf->hashsize; 1068c0341432SJohn Baldwin else 1069c0341432SJohn Baldwin swa->sw_mlen = csp->csp_auth_mlen; 1070c0341432SJohn Baldwin swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1071c0341432SJohn Baldwin if (swa->sw_ictx == NULL) 1072c0341432SJohn Baldwin return (ENOBUFS); 1073c0341432SJohn Baldwin axf->Init(swa->sw_ictx); 1074c0341432SJohn Baldwin if (csp->csp_cipher_key != NULL) 1075c0341432SJohn Baldwin axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, 1076c0341432SJohn Baldwin csp->csp_cipher_klen); 107708fca7a5SJohn-Mark Gurney 1078c0341432SJohn Baldwin /* Second, setup the cipher side. */ 10793e947048SJohn Baldwin return (swcr_setup_cipher(ses, csp)); 10802e2e26d1SJohn Baldwin } 1081a2bc81bfSJohn-Mark Gurney 1082c0341432SJohn Baldwin static bool 1083c0341432SJohn Baldwin swcr_auth_supported(const struct crypto_session_params *csp) 1084109919c6SBenno Rice { 1085091d81d1SSam Leffler struct auth_hash *axf; 1086091d81d1SSam Leffler 1087c0341432SJohn Baldwin axf = crypto_auth_hash(csp); 1088c0341432SJohn Baldwin if (axf == NULL) 1089c0341432SJohn Baldwin return (false); 1090c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1091091d81d1SSam Leffler case CRYPTO_SHA1_HMAC: 1092c97f39ceSConrad Meyer case CRYPTO_SHA2_224_HMAC: 1093f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_256_HMAC: 1094f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_384_HMAC: 1095f6c4bc3bSPawel Jakub Dawidek case CRYPTO_SHA2_512_HMAC: 1096091d81d1SSam Leffler case CRYPTO_NULL_HMAC: 1097c0341432SJohn Baldwin case CRYPTO_RIPEMD160_HMAC: 1098091d81d1SSam Leffler break; 1099c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1100c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1101c0341432SJohn Baldwin case 128: 1102c0341432SJohn Baldwin case 192: 1103c0341432SJohn Baldwin case 256: 1104c0341432SJohn Baldwin break; 1105c0341432SJohn Baldwin default: 1106c0341432SJohn Baldwin return (false); 1107c0341432SJohn Baldwin } 1108c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1109c0341432SJohn Baldwin return (false); 1110c0341432SJohn Baldwin if (csp->csp_ivlen != AES_GCM_IV_LEN) 1111c0341432SJohn Baldwin return (false); 1112c0341432SJohn Baldwin break; 111325b7033bSConrad Meyer case CRYPTO_POLY1305: 1114c0341432SJohn Baldwin if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1115c0341432SJohn Baldwin return (false); 1116c0341432SJohn Baldwin break; 1117c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1118c0341432SJohn Baldwin switch (csp->csp_auth_klen * 8) { 1119c0341432SJohn Baldwin case 128: 1120c0341432SJohn Baldwin case 192: 1121c0341432SJohn Baldwin case 256: 1122c0341432SJohn Baldwin break; 1123c0341432SJohn Baldwin default: 1124c0341432SJohn Baldwin return (false); 1125c0341432SJohn Baldwin } 1126c0341432SJohn Baldwin if (csp->csp_auth_key == NULL) 1127c0341432SJohn Baldwin return (false); 1128c0341432SJohn Baldwin if (csp->csp_ivlen != AES_CCM_IV_LEN) 1129c0341432SJohn Baldwin return (false); 1130c0341432SJohn Baldwin break; 1131c0341432SJohn Baldwin } 1132c0341432SJohn Baldwin return (true); 1133c0341432SJohn Baldwin } 1134091d81d1SSam Leffler 1135c0341432SJohn Baldwin static bool 1136c0341432SJohn Baldwin swcr_cipher_supported(const struct crypto_session_params *csp) 1137c0341432SJohn Baldwin { 1138c0341432SJohn Baldwin struct enc_xform *txf; 1139c0341432SJohn Baldwin 1140c0341432SJohn Baldwin txf = crypto_cipher(csp); 1141c0341432SJohn Baldwin if (txf == NULL) 1142c0341432SJohn Baldwin return (false); 1143c0341432SJohn Baldwin if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1144c0341432SJohn Baldwin txf->ivsize != csp->csp_ivlen) 1145c0341432SJohn Baldwin return (false); 1146c0341432SJohn Baldwin return (true); 1147c0341432SJohn Baldwin } 1148c0341432SJohn Baldwin 1149c0341432SJohn Baldwin static int 1150c0341432SJohn Baldwin swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1151c0341432SJohn Baldwin { 1152c0341432SJohn Baldwin 11539c0e3d3aSJohn Baldwin if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT)) != 0) 1154c0341432SJohn Baldwin return (EINVAL); 1155c0341432SJohn Baldwin switch (csp->csp_mode) { 1156c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1157c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1158c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1159c0341432SJohn Baldwin break; 1160c0341432SJohn Baldwin default: 1161c0341432SJohn Baldwin return (EINVAL); 11625fbc5b5aSConrad Meyer } 1163091d81d1SSam Leffler break; 1164c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1165c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1166c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1167c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1168c0341432SJohn Baldwin return (EINVAL); 1169c0341432SJohn Baldwin default: 1170c0341432SJohn Baldwin if (!swcr_cipher_supported(csp)) 1171c0341432SJohn Baldwin return (EINVAL); 1172091d81d1SSam Leffler break; 1173091d81d1SSam Leffler } 1174c0341432SJohn Baldwin break; 1175c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1176c0341432SJohn Baldwin if (!swcr_auth_supported(csp)) 1177c0341432SJohn Baldwin return (EINVAL); 1178c0341432SJohn Baldwin break; 1179c0341432SJohn Baldwin case CSP_MODE_AEAD: 1180c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1181c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1182c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1183c0341432SJohn Baldwin break; 1184c0341432SJohn Baldwin default: 1185c0341432SJohn Baldwin return (EINVAL); 1186c0341432SJohn Baldwin } 1187c0341432SJohn Baldwin break; 1188c0341432SJohn Baldwin case CSP_MODE_ETA: 1189c0341432SJohn Baldwin /* AEAD algorithms cannot be used for EtA. */ 1190c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1191c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1192c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1193c0341432SJohn Baldwin return (EINVAL); 1194c0341432SJohn Baldwin } 1195c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1196c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1197c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1198c0341432SJohn Baldwin return (EINVAL); 1199c0341432SJohn Baldwin } 1200c0341432SJohn Baldwin 1201c0341432SJohn Baldwin if (!swcr_cipher_supported(csp) || 1202c0341432SJohn Baldwin !swcr_auth_supported(csp)) 1203c0341432SJohn Baldwin return (EINVAL); 1204c0341432SJohn Baldwin break; 1205c0341432SJohn Baldwin default: 1206c0341432SJohn Baldwin return (EINVAL); 1207c0341432SJohn Baldwin } 1208c0341432SJohn Baldwin 1209c0341432SJohn Baldwin return (CRYPTODEV_PROBE_SOFTWARE); 1210c0341432SJohn Baldwin } 1211c0341432SJohn Baldwin 1212c0341432SJohn Baldwin /* 1213c0341432SJohn Baldwin * Generate a new software session. 1214c0341432SJohn Baldwin */ 1215c0341432SJohn Baldwin static int 1216c0341432SJohn Baldwin swcr_newsession(device_t dev, crypto_session_t cses, 1217c0341432SJohn Baldwin const struct crypto_session_params *csp) 1218c0341432SJohn Baldwin { 1219c0341432SJohn Baldwin struct swcr_session *ses; 1220c0341432SJohn Baldwin struct swcr_encdec *swe; 1221c0341432SJohn Baldwin struct swcr_auth *swa; 1222c0341432SJohn Baldwin struct comp_algo *cxf; 1223c0341432SJohn Baldwin int error; 1224c0341432SJohn Baldwin 1225c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1226c0341432SJohn Baldwin mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); 1227c0341432SJohn Baldwin 1228c0341432SJohn Baldwin error = 0; 1229c0341432SJohn Baldwin swe = &ses->swcr_encdec; 1230c0341432SJohn Baldwin swa = &ses->swcr_auth; 1231c0341432SJohn Baldwin switch (csp->csp_mode) { 1232c0341432SJohn Baldwin case CSP_MODE_COMPRESS: 1233c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1234c0341432SJohn Baldwin case CRYPTO_DEFLATE_COMP: 1235c0341432SJohn Baldwin cxf = &comp_algo_deflate; 1236c0341432SJohn Baldwin break; 1237c0341432SJohn Baldwin #ifdef INVARIANTS 1238c0341432SJohn Baldwin default: 1239c0341432SJohn Baldwin panic("bad compression algo"); 1240c0341432SJohn Baldwin #endif 1241c0341432SJohn Baldwin } 1242c0341432SJohn Baldwin ses->swcr_compdec.sw_cxf = cxf; 1243c0341432SJohn Baldwin ses->swcr_process = swcr_compdec; 1244c0341432SJohn Baldwin break; 1245c0341432SJohn Baldwin case CSP_MODE_CIPHER: 1246c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1247c0341432SJohn Baldwin case CRYPTO_NULL_CBC: 1248c0341432SJohn Baldwin ses->swcr_process = swcr_null; 1249c0341432SJohn Baldwin break; 1250c0341432SJohn Baldwin #ifdef INVARIANTS 1251c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1252c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1253c0341432SJohn Baldwin panic("bad cipher algo"); 1254c0341432SJohn Baldwin #endif 1255c0341432SJohn Baldwin default: 12563e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1257c0341432SJohn Baldwin if (error == 0) 1258c0341432SJohn Baldwin ses->swcr_process = swcr_encdec; 1259c0341432SJohn Baldwin } 1260c0341432SJohn Baldwin break; 1261c0341432SJohn Baldwin case CSP_MODE_DIGEST: 1262c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1263c0341432SJohn Baldwin break; 1264c0341432SJohn Baldwin case CSP_MODE_AEAD: 1265c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1266c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1267c0341432SJohn Baldwin error = swcr_setup_gcm(ses, csp); 1268c0341432SJohn Baldwin if (error == 0) 1269c0341432SJohn Baldwin ses->swcr_process = swcr_gcm; 1270c0341432SJohn Baldwin break; 1271c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1272c0341432SJohn Baldwin error = swcr_setup_ccm(ses, csp); 1273c0341432SJohn Baldwin if (error == 0) 1274c0341432SJohn Baldwin ses->swcr_process = swcr_ccm; 1275c0341432SJohn Baldwin break; 1276c0341432SJohn Baldwin #ifdef INVARIANTS 1277c0341432SJohn Baldwin default: 1278c0341432SJohn Baldwin panic("bad aead algo"); 1279c0341432SJohn Baldwin #endif 1280c0341432SJohn Baldwin } 1281c0341432SJohn Baldwin break; 1282c0341432SJohn Baldwin case CSP_MODE_ETA: 1283c0341432SJohn Baldwin #ifdef INVARIANTS 1284c0341432SJohn Baldwin switch (csp->csp_cipher_alg) { 1285c0341432SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 1286c0341432SJohn Baldwin case CRYPTO_AES_CCM_16: 1287c0341432SJohn Baldwin panic("bad eta cipher algo"); 1288c0341432SJohn Baldwin } 1289c0341432SJohn Baldwin switch (csp->csp_auth_alg) { 1290c0341432SJohn Baldwin case CRYPTO_AES_NIST_GMAC: 1291c0341432SJohn Baldwin case CRYPTO_AES_CCM_CBC_MAC: 1292c0341432SJohn Baldwin panic("bad eta auth algo"); 1293c0341432SJohn Baldwin } 1294c0341432SJohn Baldwin #endif 1295c0341432SJohn Baldwin 1296c0341432SJohn Baldwin error = swcr_setup_auth(ses, csp); 1297c0341432SJohn Baldwin if (error) 1298c0341432SJohn Baldwin break; 1299c0341432SJohn Baldwin if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1300c0341432SJohn Baldwin /* Effectively degrade to digest mode. */ 1301c0341432SJohn Baldwin ses->swcr_process = swcr_authcompute; 1302c0341432SJohn Baldwin break; 1303c0341432SJohn Baldwin } 1304c0341432SJohn Baldwin 13053e947048SJohn Baldwin error = swcr_setup_cipher(ses, csp); 1306c0341432SJohn Baldwin if (error == 0) 1307c0341432SJohn Baldwin ses->swcr_process = swcr_eta; 1308c0341432SJohn Baldwin break; 1309c0341432SJohn Baldwin default: 1310c0341432SJohn Baldwin error = EINVAL; 1311c0341432SJohn Baldwin } 1312c0341432SJohn Baldwin 1313c0341432SJohn Baldwin if (error) 1314c0341432SJohn Baldwin swcr_freesession(dev, cses); 1315c0341432SJohn Baldwin return (error); 1316c0341432SJohn Baldwin } 1317c0341432SJohn Baldwin 1318c0341432SJohn Baldwin static void 1319c0341432SJohn Baldwin swcr_freesession(device_t dev, crypto_session_t cses) 1320c0341432SJohn Baldwin { 1321c0341432SJohn Baldwin struct swcr_session *ses; 1322c0341432SJohn Baldwin struct swcr_auth *swa; 1323c0341432SJohn Baldwin struct auth_hash *axf; 1324c0341432SJohn Baldwin 1325c0341432SJohn Baldwin ses = crypto_get_driver_session(cses); 1326c0341432SJohn Baldwin 1327c0341432SJohn Baldwin mtx_destroy(&ses->swcr_lock); 1328c0341432SJohn Baldwin 13293e947048SJohn Baldwin zfree(ses->swcr_encdec.sw_kschedule, M_CRYPTO_DATA); 1330c0341432SJohn Baldwin 1331c0341432SJohn Baldwin axf = ses->swcr_auth.sw_axf; 1332c0341432SJohn Baldwin if (axf != NULL) { 1333c0341432SJohn Baldwin swa = &ses->swcr_auth; 1334c0341432SJohn Baldwin if (swa->sw_ictx != NULL) { 1335c0341432SJohn Baldwin explicit_bzero(swa->sw_ictx, axf->ctxsize); 1336c0341432SJohn Baldwin free(swa->sw_ictx, M_CRYPTO_DATA); 1337c0341432SJohn Baldwin } 1338c0341432SJohn Baldwin if (swa->sw_octx != NULL) { 13393a0b6a93SJohn Baldwin explicit_bzero(swa->sw_octx, axf->ctxsize); 1340c0341432SJohn Baldwin free(swa->sw_octx, M_CRYPTO_DATA); 1341c0341432SJohn Baldwin } 1342091d81d1SSam Leffler } 1343091d81d1SSam Leffler } 1344091d81d1SSam Leffler 1345091d81d1SSam Leffler /* 1346091d81d1SSam Leffler * Process a software request. 1347091d81d1SSam Leffler */ 1348091d81d1SSam Leffler static int 13496810ad6fSSam Leffler swcr_process(device_t dev, struct cryptop *crp, int hint) 1350091d81d1SSam Leffler { 1351c0341432SJohn Baldwin struct swcr_session *ses; 1352091d81d1SSam Leffler 13531b0909d5SConrad Meyer ses = crypto_get_driver_session(crp->crp_session); 1354a7fcb1afSSean Eric Fagan mtx_lock(&ses->swcr_lock); 1355091d81d1SSam Leffler 1356c0341432SJohn Baldwin crp->crp_etype = ses->swcr_process(ses, crp); 1357091d81d1SSam Leffler 1358a7fcb1afSSean Eric Fagan mtx_unlock(&ses->swcr_lock); 1359091d81d1SSam Leffler crypto_done(crp); 1360c0341432SJohn Baldwin return (0); 1361091d81d1SSam Leffler } 1362091d81d1SSam Leffler 1363091d81d1SSam Leffler static void 13643f147ab2SWarner Losh swcr_identify(driver_t *drv, device_t parent) 1365091d81d1SSam Leffler { 13666810ad6fSSam Leffler /* NB: order 10 is so we get attached after h/w devices */ 13676810ad6fSSam Leffler if (device_find_child(parent, "cryptosoft", -1) == NULL && 136886c585d9SMarius Strobl BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 13696810ad6fSSam Leffler panic("cryptosoft: could not attach"); 13706810ad6fSSam Leffler } 1371f6c4bc3bSPawel Jakub Dawidek 13726810ad6fSSam Leffler static int 13736810ad6fSSam Leffler swcr_probe(device_t dev) 13746810ad6fSSam Leffler { 13756810ad6fSSam Leffler device_set_desc(dev, "software crypto"); 137686c585d9SMarius Strobl return (BUS_PROBE_NOWILDCARD); 13776810ad6fSSam Leffler } 1378f6c4bc3bSPawel Jakub Dawidek 13796810ad6fSSam Leffler static int 13806810ad6fSSam Leffler swcr_attach(device_t dev) 13816810ad6fSSam Leffler { 13826810ad6fSSam Leffler 13839ebbebe4SConrad Meyer swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 13846810ad6fSSam Leffler CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 13856810ad6fSSam Leffler if (swcr_id < 0) { 13866810ad6fSSam Leffler device_printf(dev, "cannot initialize!"); 1387c0341432SJohn Baldwin return (ENXIO); 13886810ad6fSSam Leffler } 13896810ad6fSSam Leffler 1390c0341432SJohn Baldwin return (0); 1391091d81d1SSam Leffler } 13924b465da2SPawel Jakub Dawidek 13933f147ab2SWarner Losh static int 13946810ad6fSSam Leffler swcr_detach(device_t dev) 13954b465da2SPawel Jakub Dawidek { 13966810ad6fSSam Leffler crypto_unregister_all(swcr_id); 13973f147ab2SWarner Losh return 0; 13984b465da2SPawel Jakub Dawidek } 13996810ad6fSSam Leffler 14006810ad6fSSam Leffler static device_method_t swcr_methods[] = { 14016810ad6fSSam Leffler DEVMETHOD(device_identify, swcr_identify), 14026810ad6fSSam Leffler DEVMETHOD(device_probe, swcr_probe), 14036810ad6fSSam Leffler DEVMETHOD(device_attach, swcr_attach), 14046810ad6fSSam Leffler DEVMETHOD(device_detach, swcr_detach), 14056810ad6fSSam Leffler 1406c0341432SJohn Baldwin DEVMETHOD(cryptodev_probesession, swcr_probesession), 14076810ad6fSSam Leffler DEVMETHOD(cryptodev_newsession, swcr_newsession), 14086810ad6fSSam Leffler DEVMETHOD(cryptodev_freesession,swcr_freesession), 14096810ad6fSSam Leffler DEVMETHOD(cryptodev_process, swcr_process), 14106810ad6fSSam Leffler 14116810ad6fSSam Leffler {0, 0}, 14126810ad6fSSam Leffler }; 14136810ad6fSSam Leffler 14146810ad6fSSam Leffler static driver_t swcr_driver = { 14156810ad6fSSam Leffler "cryptosoft", 14166810ad6fSSam Leffler swcr_methods, 14176810ad6fSSam Leffler 0, /* NB: no softc */ 14186810ad6fSSam Leffler }; 14196810ad6fSSam Leffler static devclass_t swcr_devclass; 14206810ad6fSSam Leffler 14216810ad6fSSam Leffler /* 14226810ad6fSSam Leffler * NB: We explicitly reference the crypto module so we 14236810ad6fSSam Leffler * get the necessary ordering when built as a loadable 14246810ad6fSSam Leffler * module. This is required because we bundle the crypto 14256810ad6fSSam Leffler * module code together with the cryptosoft driver (otherwise 14266810ad6fSSam Leffler * normal module dependencies would handle things). 14276810ad6fSSam Leffler */ 14286810ad6fSSam Leffler extern int crypto_modevent(struct module *, int, void *); 14296810ad6fSSam Leffler /* XXX where to attach */ 14306810ad6fSSam Leffler DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 14316810ad6fSSam Leffler MODULE_VERSION(cryptosoft, 1); 14326810ad6fSSam Leffler MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1433