1 /*- 2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 4 * Copyright (c) 2014,2016 The FreeBSD Foundation 5 * Copyright (c) 2020 Ampere Computing 6 * All rights reserved. 7 * 8 * Portions of this software were developed by John-Mark Gurney 9 * under sponsorship of the FreeBSD Foundation and 10 * Rubicon Communications, LLC (Netgate). 11 * 12 * This software was developed by Andrew Turner under 13 * sponsorship from the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37 /* 38 * This is based on the aesni code. 39 */ 40 41 #include <sys/cdefs.h> 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/bus.h> 46 #include <sys/endian.h> 47 #include <sys/lock.h> 48 #include <sys/malloc.h> 49 #include <sys/mbuf.h> 50 #include <sys/module.h> 51 #include <sys/mutex.h> 52 #include <sys/queue.h> 53 #include <sys/rwlock.h> 54 #include <sys/smp.h> 55 #include <sys/uio.h> 56 57 #include <machine/vfp.h> 58 59 #include <opencrypto/cryptodev.h> 60 #include <opencrypto/gmac.h> 61 #include <cryptodev_if.h> 62 #include <crypto/armv8/armv8_crypto.h> 63 #include <crypto/rijndael/rijndael.h> 64 65 struct armv8_crypto_softc { 66 int dieing; 67 int32_t cid; 68 struct rwlock lock; 69 bool has_pmul; 70 }; 71 72 static struct mtx *ctx_mtx; 73 static struct fpu_kern_ctx **ctx_vfp; 74 75 #define AQUIRE_CTX(i, ctx) \ 76 do { \ 77 (i) = PCPU_GET(cpuid); \ 78 mtx_lock(&ctx_mtx[(i)]); \ 79 (ctx) = ctx_vfp[(i)]; \ 80 } while (0) 81 #define RELEASE_CTX(i, ctx) \ 82 do { \ 83 mtx_unlock(&ctx_mtx[(i)]); \ 84 (i) = -1; \ 85 (ctx) = NULL; \ 86 } while (0) 87 88 static int armv8_crypto_cipher_process(struct armv8_crypto_session *, 89 struct cryptop *); 90 91 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data"); 92 93 static void 94 armv8_crypto_identify(driver_t *drv, device_t parent) 95 { 96 97 /* NB: order 10 is so we get attached after h/w devices */ 98 if (device_find_child(parent, "armv8crypto", -1) == NULL && 99 BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0) 100 panic("ARMv8 crypto: could not attach"); 101 } 102 103 static int 104 armv8_crypto_probe(device_t dev) 105 { 106 uint64_t reg; 107 int ret = ENXIO; 108 109 reg = READ_SPECIALREG(id_aa64isar0_el1); 110 111 switch (ID_AA64ISAR0_AES_VAL(reg)) { 112 case ID_AA64ISAR0_AES_BASE: 113 ret = 0; 114 device_set_desc(dev, "AES-CBC,AES-XTS"); 115 break; 116 case ID_AA64ISAR0_AES_PMULL: 117 ret = 0; 118 device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM"); 119 break; 120 default: 121 break; 122 case ID_AA64ISAR0_AES_NONE: 123 device_printf(dev, "CPU lacks AES instructions\n"); 124 break; 125 } 126 127 /* TODO: Check more fields as we support more features */ 128 129 return (ret); 130 } 131 132 static int 133 armv8_crypto_attach(device_t dev) 134 { 135 struct armv8_crypto_softc *sc; 136 uint64_t reg; 137 int i; 138 139 sc = device_get_softc(dev); 140 sc->dieing = 0; 141 142 reg = READ_SPECIALREG(id_aa64isar0_el1); 143 144 if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL) 145 sc->has_pmul = true; 146 147 sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session), 148 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); 149 if (sc->cid < 0) { 150 device_printf(dev, "Could not get crypto driver id.\n"); 151 return (ENOMEM); 152 } 153 154 rw_init(&sc->lock, "armv8crypto"); 155 156 ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO, 157 M_WAITOK|M_ZERO); 158 ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO, 159 M_WAITOK|M_ZERO); 160 161 CPU_FOREACH(i) { 162 ctx_vfp[i] = fpu_kern_alloc_ctx(0); 163 mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW); 164 } 165 166 return (0); 167 } 168 169 static int 170 armv8_crypto_detach(device_t dev) 171 { 172 struct armv8_crypto_softc *sc; 173 int i; 174 175 sc = device_get_softc(dev); 176 177 rw_wlock(&sc->lock); 178 sc->dieing = 1; 179 rw_wunlock(&sc->lock); 180 crypto_unregister_all(sc->cid); 181 182 rw_destroy(&sc->lock); 183 184 CPU_FOREACH(i) { 185 if (ctx_vfp[i] != NULL) { 186 mtx_destroy(&ctx_mtx[i]); 187 fpu_kern_free_ctx(ctx_vfp[i]); 188 } 189 ctx_vfp[i] = NULL; 190 } 191 free(ctx_mtx, M_ARMV8_CRYPTO); 192 ctx_mtx = NULL; 193 free(ctx_vfp, M_ARMV8_CRYPTO); 194 ctx_vfp = NULL; 195 196 return (0); 197 } 198 199 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD) 200 201 static int 202 armv8_crypto_probesession(device_t dev, 203 const struct crypto_session_params *csp) 204 { 205 struct armv8_crypto_softc *sc; 206 207 sc = device_get_softc(dev); 208 209 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 210 return (EINVAL); 211 212 switch (csp->csp_mode) { 213 case CSP_MODE_AEAD: 214 switch (csp->csp_cipher_alg) { 215 case CRYPTO_AES_NIST_GCM_16: 216 if (!sc->has_pmul) 217 return (EINVAL); 218 if (csp->csp_auth_mlen != 0 && 219 csp->csp_auth_mlen != GMAC_DIGEST_LEN) 220 return (EINVAL); 221 switch (csp->csp_cipher_klen * 8) { 222 case 128: 223 case 192: 224 case 256: 225 break; 226 default: 227 return (EINVAL); 228 } 229 break; 230 default: 231 return (EINVAL); 232 } 233 break; 234 case CSP_MODE_CIPHER: 235 switch (csp->csp_cipher_alg) { 236 case CRYPTO_AES_CBC: 237 if (csp->csp_ivlen != AES_BLOCK_LEN) 238 return (EINVAL); 239 switch (csp->csp_cipher_klen * 8) { 240 case 128: 241 case 192: 242 case 256: 243 break; 244 default: 245 return (EINVAL); 246 } 247 break; 248 case CRYPTO_AES_XTS: 249 if (csp->csp_ivlen != AES_XTS_IV_LEN) 250 return (EINVAL); 251 switch (csp->csp_cipher_klen * 8) { 252 case 256: 253 case 512: 254 break; 255 default: 256 return (EINVAL); 257 } 258 break; 259 default: 260 return (EINVAL); 261 } 262 break; 263 default: 264 return (EINVAL); 265 } 266 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); 267 } 268 269 static int 270 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, 271 const struct crypto_session_params *csp, const uint8_t *key, int keylen) 272 { 273 __uint128_val_t H; 274 struct fpu_kern_ctx *ctx; 275 int kt, i; 276 277 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 278 keylen /= 2; 279 280 switch (keylen * 8) { 281 case 128: 282 case 192: 283 case 256: 284 break; 285 default: 286 return (EINVAL); 287 } 288 289 kt = is_fpu_kern_thread(0); 290 if (!kt) { 291 AQUIRE_CTX(i, ctx); 292 fpu_kern_enter(curthread, ctx, 293 FPU_KERN_NORMAL | FPU_KERN_KTHR); 294 } 295 296 aes_v8_set_encrypt_key(key, 297 keylen * 8, &ses->enc_schedule); 298 299 if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) || 300 (csp->csp_cipher_alg == CRYPTO_AES_CBC)) 301 aes_v8_set_decrypt_key(key, 302 keylen * 8, &ses->dec_schedule); 303 304 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 305 aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule); 306 307 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { 308 memset(H.c, 0, sizeof(H.c)); 309 aes_v8_encrypt(H.c, H.c, &ses->enc_schedule); 310 H.u[0] = bswap64(H.u[0]); 311 H.u[1] = bswap64(H.u[1]); 312 gcm_init_v8(ses->Htable, H.u); 313 } 314 315 if (!kt) { 316 fpu_kern_leave(curthread, ctx); 317 RELEASE_CTX(i, ctx); 318 } 319 320 return (0); 321 } 322 323 static int 324 armv8_crypto_newsession(device_t dev, crypto_session_t cses, 325 const struct crypto_session_params *csp) 326 { 327 struct armv8_crypto_softc *sc; 328 struct armv8_crypto_session *ses; 329 int error; 330 331 sc = device_get_softc(dev); 332 rw_wlock(&sc->lock); 333 if (sc->dieing) { 334 rw_wunlock(&sc->lock); 335 return (EINVAL); 336 } 337 338 ses = crypto_get_driver_session(cses); 339 error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key, 340 csp->csp_cipher_klen); 341 rw_wunlock(&sc->lock); 342 return (error); 343 } 344 345 static int 346 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused) 347 { 348 struct armv8_crypto_session *ses; 349 350 ses = crypto_get_driver_session(crp->crp_session); 351 crp->crp_etype = armv8_crypto_cipher_process(ses, crp); 352 crypto_done(crp); 353 return (0); 354 } 355 356 static uint8_t * 357 armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated) 358 { 359 uint8_t *addr; 360 361 addr = crypto_contiguous_subsegment(crp, start, length); 362 if (addr != NULL) { 363 *allocated = 0; 364 return (addr); 365 } 366 addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT); 367 if (addr != NULL) { 368 *allocated = 1; 369 crypto_copydata(crp, start, length, addr); 370 } else 371 *allocated = 0; 372 return (addr); 373 } 374 375 static int 376 armv8_crypto_cipher_process(struct armv8_crypto_session *ses, 377 struct cryptop *crp) 378 { 379 struct crypto_buffer_cursor fromc, toc; 380 const struct crypto_session_params *csp; 381 struct fpu_kern_ctx *ctx; 382 uint8_t *authbuf; 383 uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN]; 384 int authallocated, i; 385 int encflag; 386 int kt; 387 int error; 388 389 csp = crypto_get_params(crp->crp_session); 390 encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 391 392 authallocated = 0; 393 authbuf = NULL; 394 kt = 1; 395 396 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { 397 if (crp->crp_aad != NULL) 398 authbuf = crp->crp_aad; 399 else 400 authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start, 401 crp->crp_aad_length, &authallocated); 402 if (authbuf == NULL) { 403 error = ENOMEM; 404 goto out; 405 } 406 } 407 crypto_cursor_init(&fromc, &crp->crp_buf); 408 crypto_cursor_advance(&fromc, crp->crp_payload_start); 409 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 410 crypto_cursor_init(&toc, &crp->crp_obuf); 411 crypto_cursor_advance(&toc, crp->crp_payload_output_start); 412 } else { 413 crypto_cursor_copy(&fromc, &toc); 414 } 415 416 kt = is_fpu_kern_thread(0); 417 if (!kt) { 418 AQUIRE_CTX(i, ctx); 419 fpu_kern_enter(curthread, ctx, 420 FPU_KERN_NORMAL | FPU_KERN_KTHR); 421 } 422 423 if (crp->crp_cipher_key != NULL) { 424 armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key, 425 csp->csp_cipher_klen); 426 } 427 428 crypto_read_iv(crp, iv); 429 430 switch (csp->csp_cipher_alg) { 431 case CRYPTO_AES_CBC: 432 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) { 433 error = EINVAL; 434 goto out; 435 } 436 if (encflag) 437 armv8_aes_encrypt_cbc(&ses->enc_schedule, 438 crp->crp_payload_length, &fromc, &toc, iv); 439 else 440 armv8_aes_decrypt_cbc(&ses->dec_schedule, 441 crp->crp_payload_length, &fromc, &toc, iv); 442 break; 443 case CRYPTO_AES_XTS: 444 if (encflag) 445 armv8_aes_encrypt_xts(&ses->enc_schedule, 446 &ses->xts_schedule.aes_key, crp->crp_payload_length, 447 &fromc, &toc, iv); 448 else 449 armv8_aes_decrypt_xts(&ses->dec_schedule, 450 &ses->xts_schedule.aes_key, crp->crp_payload_length, 451 &fromc, &toc, iv); 452 break; 453 case CRYPTO_AES_NIST_GCM_16: 454 if (encflag) { 455 memset(tag, 0, sizeof(tag)); 456 armv8_aes_encrypt_gcm(&ses->enc_schedule, 457 crp->crp_payload_length, &fromc, &toc, 458 crp->crp_aad_length, authbuf, tag, iv, ses->Htable); 459 crypto_copyback(crp, crp->crp_digest_start, sizeof(tag), 460 tag); 461 } else { 462 crypto_copydata(crp, crp->crp_digest_start, sizeof(tag), 463 tag); 464 error = armv8_aes_decrypt_gcm(&ses->enc_schedule, 465 crp->crp_payload_length, &fromc, &toc, 466 crp->crp_aad_length, authbuf, tag, iv, ses->Htable); 467 if (error != 0) 468 goto out; 469 } 470 break; 471 } 472 473 error = 0; 474 out: 475 if (!kt) { 476 fpu_kern_leave(curthread, ctx); 477 RELEASE_CTX(i, ctx); 478 } 479 480 if (authallocated) 481 zfree(authbuf, M_ARMV8_CRYPTO); 482 explicit_bzero(iv, sizeof(iv)); 483 explicit_bzero(tag, sizeof(tag)); 484 485 return (error); 486 } 487 488 static device_method_t armv8_crypto_methods[] = { 489 DEVMETHOD(device_identify, armv8_crypto_identify), 490 DEVMETHOD(device_probe, armv8_crypto_probe), 491 DEVMETHOD(device_attach, armv8_crypto_attach), 492 DEVMETHOD(device_detach, armv8_crypto_detach), 493 494 DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession), 495 DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession), 496 DEVMETHOD(cryptodev_process, armv8_crypto_process), 497 498 DEVMETHOD_END, 499 }; 500 501 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods, 502 sizeof(struct armv8_crypto_softc)); 503 504 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, 0, 0); 505