1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Support for Intel AES-NI instructions. This file contains glue 4 * code, the real AES implementation is in intel-aes_asm.S. 5 * 6 * Copyright (C) 2008, Intel Corp. 7 * Author: Huang Ying <ying.huang@intel.com> 8 * 9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 10 * interface for 64-bit kernels. 11 * Authors: Adrian Hoban <adrian.hoban@intel.com> 12 * Gabriele Paoloni <gabriele.paoloni@intel.com> 13 * Tadeusz Struk (tadeusz.struk@intel.com) 14 * Aidan O'Mahony (aidan.o.mahony@intel.com) 15 * Copyright (c) 2010, Intel Corporation. 16 */ 17 18 #include <linux/hardirq.h> 19 #include <linux/types.h> 20 #include <linux/module.h> 21 #include <linux/err.h> 22 #include <crypto/algapi.h> 23 #include <crypto/aes.h> 24 #include <crypto/ctr.h> 25 #include <crypto/b128ops.h> 26 #include <crypto/gcm.h> 27 #include <crypto/xts.h> 28 #include <asm/cpu_device_id.h> 29 #include <asm/simd.h> 30 #include <crypto/scatterwalk.h> 31 #include <crypto/internal/aead.h> 32 #include <crypto/internal/simd.h> 33 #include <crypto/internal/skcipher.h> 34 #include <linux/jump_label.h> 35 #include <linux/workqueue.h> 36 #include <linux/spinlock.h> 37 #include <linux/static_call.h> 38 39 40 #define AESNI_ALIGN 16 41 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN))) 42 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) 43 #define RFC4106_HASH_SUBKEY_SIZE 16 44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) 45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) 46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA) 47 48 /* This data is stored at the end of the crypto_tfm struct. 49 * It's a type of per "session" data storage location. 50 * This needs to be 16 byte aligned. 51 */ 52 struct aesni_rfc4106_gcm_ctx { 53 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 54 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 55 u8 nonce[4]; 56 }; 57 58 struct generic_gcmaes_ctx { 59 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 60 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 61 }; 62 63 struct aesni_xts_ctx { 64 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 65 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 66 }; 67 68 #define GCM_BLOCK_LEN 16 69 70 struct gcm_context_data { 71 /* init, update and finalize context data */ 72 u8 aad_hash[GCM_BLOCK_LEN]; 73 u64 aad_length; 74 u64 in_length; 75 u8 partial_block_enc_key[GCM_BLOCK_LEN]; 76 u8 orig_IV[GCM_BLOCK_LEN]; 77 u8 current_counter[GCM_BLOCK_LEN]; 78 u64 partial_block_len; 79 u64 unused; 80 u8 hash_keys[GCM_BLOCK_LEN * 16]; 81 }; 82 83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 84 unsigned int key_len); 85 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in); 86 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in); 87 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 88 const u8 *in, unsigned int len); 89 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 90 const u8 *in, unsigned int len); 91 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 92 const u8 *in, unsigned int len, u8 *iv); 93 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 94 const u8 *in, unsigned int len, u8 *iv); 95 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 96 const u8 *in, unsigned int len, u8 *iv); 97 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 98 const u8 *in, unsigned int len, u8 *iv); 99 100 #define AVX_GEN2_OPTSIZE 640 101 #define AVX_GEN4_OPTSIZE 4096 102 103 asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, 104 const u8 *in, unsigned int len, u8 *iv); 105 106 asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, 107 const u8 *in, unsigned int len, u8 *iv); 108 109 #ifdef CONFIG_X86_64 110 111 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 112 const u8 *in, unsigned int len, u8 *iv); 113 DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc); 114 115 /* Scatter / Gather routines, with args similar to above */ 116 asmlinkage void aesni_gcm_init(void *ctx, 117 struct gcm_context_data *gdata, 118 u8 *iv, 119 u8 *hash_subkey, const u8 *aad, 120 unsigned long aad_len); 121 asmlinkage void aesni_gcm_enc_update(void *ctx, 122 struct gcm_context_data *gdata, u8 *out, 123 const u8 *in, unsigned long plaintext_len); 124 asmlinkage void aesni_gcm_dec_update(void *ctx, 125 struct gcm_context_data *gdata, u8 *out, 126 const u8 *in, 127 unsigned long ciphertext_len); 128 asmlinkage void aesni_gcm_finalize(void *ctx, 129 struct gcm_context_data *gdata, 130 u8 *auth_tag, unsigned long auth_tag_len); 131 132 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, 133 void *keys, u8 *out, unsigned int num_bytes); 134 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, 135 void *keys, u8 *out, unsigned int num_bytes); 136 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, 137 void *keys, u8 *out, unsigned int num_bytes); 138 139 140 asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv, 141 const void *keys, u8 *out, unsigned int num_bytes, 142 unsigned int byte_ctr); 143 144 asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv, 145 const void *keys, u8 *out, unsigned int num_bytes, 146 unsigned int byte_ctr); 147 148 asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv, 149 const void *keys, u8 *out, unsigned int num_bytes, 150 unsigned int byte_ctr); 151 152 /* 153 * asmlinkage void aesni_gcm_init_avx_gen2() 154 * gcm_data *my_ctx_data, context data 155 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 156 */ 157 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data, 158 struct gcm_context_data *gdata, 159 u8 *iv, 160 u8 *hash_subkey, 161 const u8 *aad, 162 unsigned long aad_len); 163 164 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx, 165 struct gcm_context_data *gdata, u8 *out, 166 const u8 *in, unsigned long plaintext_len); 167 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx, 168 struct gcm_context_data *gdata, u8 *out, 169 const u8 *in, 170 unsigned long ciphertext_len); 171 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx, 172 struct gcm_context_data *gdata, 173 u8 *auth_tag, unsigned long auth_tag_len); 174 175 /* 176 * asmlinkage void aesni_gcm_init_avx_gen4() 177 * gcm_data *my_ctx_data, context data 178 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 179 */ 180 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data, 181 struct gcm_context_data *gdata, 182 u8 *iv, 183 u8 *hash_subkey, 184 const u8 *aad, 185 unsigned long aad_len); 186 187 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx, 188 struct gcm_context_data *gdata, u8 *out, 189 const u8 *in, unsigned long plaintext_len); 190 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx, 191 struct gcm_context_data *gdata, u8 *out, 192 const u8 *in, 193 unsigned long ciphertext_len); 194 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx, 195 struct gcm_context_data *gdata, 196 u8 *auth_tag, unsigned long auth_tag_len); 197 198 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx); 199 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2); 200 201 static inline struct 202 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 203 { 204 unsigned long align = AESNI_ALIGN; 205 206 if (align <= crypto_tfm_ctx_alignment()) 207 align = 1; 208 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 209 } 210 211 static inline struct 212 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm) 213 { 214 unsigned long align = AESNI_ALIGN; 215 216 if (align <= crypto_tfm_ctx_alignment()) 217 align = 1; 218 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 219 } 220 #endif 221 222 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 223 { 224 unsigned long addr = (unsigned long)raw_ctx; 225 unsigned long align = AESNI_ALIGN; 226 227 if (align <= crypto_tfm_ctx_alignment()) 228 align = 1; 229 return (struct crypto_aes_ctx *)ALIGN(addr, align); 230 } 231 232 static int aes_set_key_common(struct crypto_aes_ctx *ctx, 233 const u8 *in_key, unsigned int key_len) 234 { 235 int err; 236 237 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 238 key_len != AES_KEYSIZE_256) 239 return -EINVAL; 240 241 if (!crypto_simd_usable()) 242 err = aes_expandkey(ctx, in_key, key_len); 243 else { 244 kernel_fpu_begin(); 245 err = aesni_set_key(ctx, in_key, key_len); 246 kernel_fpu_end(); 247 } 248 249 return err; 250 } 251 252 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 253 unsigned int key_len) 254 { 255 return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key, 256 key_len); 257 } 258 259 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 260 { 261 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 262 263 if (!crypto_simd_usable()) { 264 aes_encrypt(ctx, dst, src); 265 } else { 266 kernel_fpu_begin(); 267 aesni_enc(ctx, dst, src); 268 kernel_fpu_end(); 269 } 270 } 271 272 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 273 { 274 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 275 276 if (!crypto_simd_usable()) { 277 aes_decrypt(ctx, dst, src); 278 } else { 279 kernel_fpu_begin(); 280 aesni_dec(ctx, dst, src); 281 kernel_fpu_end(); 282 } 283 } 284 285 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 286 unsigned int len) 287 { 288 return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len); 289 } 290 291 static int ecb_encrypt(struct skcipher_request *req) 292 { 293 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 294 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 295 struct skcipher_walk walk; 296 unsigned int nbytes; 297 int err; 298 299 err = skcipher_walk_virt(&walk, req, false); 300 301 while ((nbytes = walk.nbytes)) { 302 kernel_fpu_begin(); 303 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 304 nbytes & AES_BLOCK_MASK); 305 kernel_fpu_end(); 306 nbytes &= AES_BLOCK_SIZE - 1; 307 err = skcipher_walk_done(&walk, nbytes); 308 } 309 310 return err; 311 } 312 313 static int ecb_decrypt(struct skcipher_request *req) 314 { 315 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 316 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 317 struct skcipher_walk walk; 318 unsigned int nbytes; 319 int err; 320 321 err = skcipher_walk_virt(&walk, req, false); 322 323 while ((nbytes = walk.nbytes)) { 324 kernel_fpu_begin(); 325 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 326 nbytes & AES_BLOCK_MASK); 327 kernel_fpu_end(); 328 nbytes &= AES_BLOCK_SIZE - 1; 329 err = skcipher_walk_done(&walk, nbytes); 330 } 331 332 return err; 333 } 334 335 static int cbc_encrypt(struct skcipher_request *req) 336 { 337 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 338 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 339 struct skcipher_walk walk; 340 unsigned int nbytes; 341 int err; 342 343 err = skcipher_walk_virt(&walk, req, false); 344 345 while ((nbytes = walk.nbytes)) { 346 kernel_fpu_begin(); 347 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 348 nbytes & AES_BLOCK_MASK, walk.iv); 349 kernel_fpu_end(); 350 nbytes &= AES_BLOCK_SIZE - 1; 351 err = skcipher_walk_done(&walk, nbytes); 352 } 353 354 return err; 355 } 356 357 static int cbc_decrypt(struct skcipher_request *req) 358 { 359 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 360 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 361 struct skcipher_walk walk; 362 unsigned int nbytes; 363 int err; 364 365 err = skcipher_walk_virt(&walk, req, false); 366 367 while ((nbytes = walk.nbytes)) { 368 kernel_fpu_begin(); 369 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 370 nbytes & AES_BLOCK_MASK, walk.iv); 371 kernel_fpu_end(); 372 nbytes &= AES_BLOCK_SIZE - 1; 373 err = skcipher_walk_done(&walk, nbytes); 374 } 375 376 return err; 377 } 378 379 static int cts_cbc_encrypt(struct skcipher_request *req) 380 { 381 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 382 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 383 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 384 struct scatterlist *src = req->src, *dst = req->dst; 385 struct scatterlist sg_src[2], sg_dst[2]; 386 struct skcipher_request subreq; 387 struct skcipher_walk walk; 388 int err; 389 390 skcipher_request_set_tfm(&subreq, tfm); 391 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 392 NULL, NULL); 393 394 if (req->cryptlen <= AES_BLOCK_SIZE) { 395 if (req->cryptlen < AES_BLOCK_SIZE) 396 return -EINVAL; 397 cbc_blocks = 1; 398 } 399 400 if (cbc_blocks > 0) { 401 skcipher_request_set_crypt(&subreq, req->src, req->dst, 402 cbc_blocks * AES_BLOCK_SIZE, 403 req->iv); 404 405 err = cbc_encrypt(&subreq); 406 if (err) 407 return err; 408 409 if (req->cryptlen == AES_BLOCK_SIZE) 410 return 0; 411 412 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 413 if (req->dst != req->src) 414 dst = scatterwalk_ffwd(sg_dst, req->dst, 415 subreq.cryptlen); 416 } 417 418 /* handle ciphertext stealing */ 419 skcipher_request_set_crypt(&subreq, src, dst, 420 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 421 req->iv); 422 423 err = skcipher_walk_virt(&walk, &subreq, false); 424 if (err) 425 return err; 426 427 kernel_fpu_begin(); 428 aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 429 walk.nbytes, walk.iv); 430 kernel_fpu_end(); 431 432 return skcipher_walk_done(&walk, 0); 433 } 434 435 static int cts_cbc_decrypt(struct skcipher_request *req) 436 { 437 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 438 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 439 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 440 struct scatterlist *src = req->src, *dst = req->dst; 441 struct scatterlist sg_src[2], sg_dst[2]; 442 struct skcipher_request subreq; 443 struct skcipher_walk walk; 444 int err; 445 446 skcipher_request_set_tfm(&subreq, tfm); 447 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 448 NULL, NULL); 449 450 if (req->cryptlen <= AES_BLOCK_SIZE) { 451 if (req->cryptlen < AES_BLOCK_SIZE) 452 return -EINVAL; 453 cbc_blocks = 1; 454 } 455 456 if (cbc_blocks > 0) { 457 skcipher_request_set_crypt(&subreq, req->src, req->dst, 458 cbc_blocks * AES_BLOCK_SIZE, 459 req->iv); 460 461 err = cbc_decrypt(&subreq); 462 if (err) 463 return err; 464 465 if (req->cryptlen == AES_BLOCK_SIZE) 466 return 0; 467 468 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 469 if (req->dst != req->src) 470 dst = scatterwalk_ffwd(sg_dst, req->dst, 471 subreq.cryptlen); 472 } 473 474 /* handle ciphertext stealing */ 475 skcipher_request_set_crypt(&subreq, src, dst, 476 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 477 req->iv); 478 479 err = skcipher_walk_virt(&walk, &subreq, false); 480 if (err) 481 return err; 482 483 kernel_fpu_begin(); 484 aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 485 walk.nbytes, walk.iv); 486 kernel_fpu_end(); 487 488 return skcipher_walk_done(&walk, 0); 489 } 490 491 #ifdef CONFIG_X86_64 492 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 493 const u8 *in, unsigned int len, u8 *iv) 494 { 495 /* 496 * based on key length, override with the by8 version 497 * of ctr mode encryption/decryption for improved performance 498 * aes_set_key_common() ensures that key length is one of 499 * {128,192,256} 500 */ 501 if (ctx->key_length == AES_KEYSIZE_128) 502 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); 503 else if (ctx->key_length == AES_KEYSIZE_192) 504 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); 505 else 506 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); 507 } 508 509 static int ctr_crypt(struct skcipher_request *req) 510 { 511 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 512 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 513 u8 keystream[AES_BLOCK_SIZE]; 514 struct skcipher_walk walk; 515 unsigned int nbytes; 516 int err; 517 518 err = skcipher_walk_virt(&walk, req, false); 519 520 while ((nbytes = walk.nbytes) > 0) { 521 kernel_fpu_begin(); 522 if (nbytes & AES_BLOCK_MASK) 523 static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr, 524 walk.src.virt.addr, 525 nbytes & AES_BLOCK_MASK, 526 walk.iv); 527 nbytes &= ~AES_BLOCK_MASK; 528 529 if (walk.nbytes == walk.total && nbytes > 0) { 530 aesni_enc(ctx, keystream, walk.iv); 531 crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes, 532 walk.src.virt.addr + walk.nbytes - nbytes, 533 keystream, nbytes); 534 crypto_inc(walk.iv, AES_BLOCK_SIZE); 535 nbytes = 0; 536 } 537 kernel_fpu_end(); 538 err = skcipher_walk_done(&walk, nbytes); 539 } 540 return err; 541 } 542 543 static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 544 const u8 *in, unsigned int len, u8 *iv, 545 unsigned int byte_ctr) 546 { 547 if (ctx->key_length == AES_KEYSIZE_128) 548 aes_xctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len, 549 byte_ctr); 550 else if (ctx->key_length == AES_KEYSIZE_192) 551 aes_xctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len, 552 byte_ctr); 553 else 554 aes_xctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len, 555 byte_ctr); 556 } 557 558 static int xctr_crypt(struct skcipher_request *req) 559 { 560 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 561 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 562 u8 keystream[AES_BLOCK_SIZE]; 563 struct skcipher_walk walk; 564 unsigned int nbytes; 565 unsigned int byte_ctr = 0; 566 int err; 567 __le32 block[AES_BLOCK_SIZE / sizeof(__le32)]; 568 569 err = skcipher_walk_virt(&walk, req, false); 570 571 while ((nbytes = walk.nbytes) > 0) { 572 kernel_fpu_begin(); 573 if (nbytes & AES_BLOCK_MASK) 574 aesni_xctr_enc_avx_tfm(ctx, walk.dst.virt.addr, 575 walk.src.virt.addr, nbytes & AES_BLOCK_MASK, 576 walk.iv, byte_ctr); 577 nbytes &= ~AES_BLOCK_MASK; 578 byte_ctr += walk.nbytes - nbytes; 579 580 if (walk.nbytes == walk.total && nbytes > 0) { 581 memcpy(block, walk.iv, AES_BLOCK_SIZE); 582 block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE); 583 aesni_enc(ctx, keystream, (u8 *)block); 584 crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - 585 nbytes, walk.src.virt.addr + walk.nbytes 586 - nbytes, keystream, nbytes); 587 byte_ctr += nbytes; 588 nbytes = 0; 589 } 590 kernel_fpu_end(); 591 err = skcipher_walk_done(&walk, nbytes); 592 } 593 return err; 594 } 595 596 static int 597 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 598 { 599 struct crypto_aes_ctx ctx; 600 int ret; 601 602 ret = aes_expandkey(&ctx, key, key_len); 603 if (ret) 604 return ret; 605 606 /* Clear the data in the hash sub key container to zero.*/ 607 /* We want to cipher all zeros to create the hash sub key. */ 608 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 609 610 aes_encrypt(&ctx, hash_subkey, hash_subkey); 611 612 memzero_explicit(&ctx, sizeof(ctx)); 613 return 0; 614 } 615 616 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, 617 unsigned int key_len) 618 { 619 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); 620 621 if (key_len < 4) 622 return -EINVAL; 623 624 /*Account for 4 byte nonce at the end.*/ 625 key_len -= 4; 626 627 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 628 629 return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?: 630 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 631 } 632 633 /* This is the Integrity Check Value (aka the authentication tag) length and can 634 * be 8, 12 or 16 bytes long. */ 635 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 636 unsigned int authsize) 637 { 638 switch (authsize) { 639 case 8: 640 case 12: 641 case 16: 642 break; 643 default: 644 return -EINVAL; 645 } 646 647 return 0; 648 } 649 650 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, 651 unsigned int authsize) 652 { 653 switch (authsize) { 654 case 4: 655 case 8: 656 case 12: 657 case 13: 658 case 14: 659 case 15: 660 case 16: 661 break; 662 default: 663 return -EINVAL; 664 } 665 666 return 0; 667 } 668 669 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, 670 unsigned int assoclen, u8 *hash_subkey, 671 u8 *iv, void *aes_ctx, u8 *auth_tag, 672 unsigned long auth_tag_len) 673 { 674 u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8); 675 struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN); 676 unsigned long left = req->cryptlen; 677 struct scatter_walk assoc_sg_walk; 678 struct skcipher_walk walk; 679 bool do_avx, do_avx2; 680 u8 *assocmem = NULL; 681 u8 *assoc; 682 int err; 683 684 if (!enc) 685 left -= auth_tag_len; 686 687 do_avx = (left >= AVX_GEN2_OPTSIZE); 688 do_avx2 = (left >= AVX_GEN4_OPTSIZE); 689 690 /* Linearize assoc, if not already linear */ 691 if (req->src->length >= assoclen && req->src->length) { 692 scatterwalk_start(&assoc_sg_walk, req->src); 693 assoc = scatterwalk_map(&assoc_sg_walk); 694 } else { 695 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 696 GFP_KERNEL : GFP_ATOMIC; 697 698 /* assoc can be any length, so must be on heap */ 699 assocmem = kmalloc(assoclen, flags); 700 if (unlikely(!assocmem)) 701 return -ENOMEM; 702 assoc = assocmem; 703 704 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); 705 } 706 707 kernel_fpu_begin(); 708 if (static_branch_likely(&gcm_use_avx2) && do_avx2) 709 aesni_gcm_init_avx_gen4(aes_ctx, data, iv, hash_subkey, assoc, 710 assoclen); 711 else if (static_branch_likely(&gcm_use_avx) && do_avx) 712 aesni_gcm_init_avx_gen2(aes_ctx, data, iv, hash_subkey, assoc, 713 assoclen); 714 else 715 aesni_gcm_init(aes_ctx, data, iv, hash_subkey, assoc, assoclen); 716 kernel_fpu_end(); 717 718 if (!assocmem) 719 scatterwalk_unmap(assoc); 720 else 721 kfree(assocmem); 722 723 err = enc ? skcipher_walk_aead_encrypt(&walk, req, false) 724 : skcipher_walk_aead_decrypt(&walk, req, false); 725 726 while (walk.nbytes > 0) { 727 kernel_fpu_begin(); 728 if (static_branch_likely(&gcm_use_avx2) && do_avx2) { 729 if (enc) 730 aesni_gcm_enc_update_avx_gen4(aes_ctx, data, 731 walk.dst.virt.addr, 732 walk.src.virt.addr, 733 walk.nbytes); 734 else 735 aesni_gcm_dec_update_avx_gen4(aes_ctx, data, 736 walk.dst.virt.addr, 737 walk.src.virt.addr, 738 walk.nbytes); 739 } else if (static_branch_likely(&gcm_use_avx) && do_avx) { 740 if (enc) 741 aesni_gcm_enc_update_avx_gen2(aes_ctx, data, 742 walk.dst.virt.addr, 743 walk.src.virt.addr, 744 walk.nbytes); 745 else 746 aesni_gcm_dec_update_avx_gen2(aes_ctx, data, 747 walk.dst.virt.addr, 748 walk.src.virt.addr, 749 walk.nbytes); 750 } else if (enc) { 751 aesni_gcm_enc_update(aes_ctx, data, walk.dst.virt.addr, 752 walk.src.virt.addr, walk.nbytes); 753 } else { 754 aesni_gcm_dec_update(aes_ctx, data, walk.dst.virt.addr, 755 walk.src.virt.addr, walk.nbytes); 756 } 757 kernel_fpu_end(); 758 759 err = skcipher_walk_done(&walk, 0); 760 } 761 762 if (err) 763 return err; 764 765 kernel_fpu_begin(); 766 if (static_branch_likely(&gcm_use_avx2) && do_avx2) 767 aesni_gcm_finalize_avx_gen4(aes_ctx, data, auth_tag, 768 auth_tag_len); 769 else if (static_branch_likely(&gcm_use_avx) && do_avx) 770 aesni_gcm_finalize_avx_gen2(aes_ctx, data, auth_tag, 771 auth_tag_len); 772 else 773 aesni_gcm_finalize(aes_ctx, data, auth_tag, auth_tag_len); 774 kernel_fpu_end(); 775 776 return 0; 777 } 778 779 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, 780 u8 *hash_subkey, u8 *iv, void *aes_ctx) 781 { 782 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 783 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 784 u8 auth_tag[16]; 785 int err; 786 787 err = gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, aes_ctx, 788 auth_tag, auth_tag_len); 789 if (err) 790 return err; 791 792 scatterwalk_map_and_copy(auth_tag, req->dst, 793 req->assoclen + req->cryptlen, 794 auth_tag_len, 1); 795 return 0; 796 } 797 798 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, 799 u8 *hash_subkey, u8 *iv, void *aes_ctx) 800 { 801 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 802 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 803 u8 auth_tag_msg[16]; 804 u8 auth_tag[16]; 805 int err; 806 807 err = gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, aes_ctx, 808 auth_tag, auth_tag_len); 809 if (err) 810 return err; 811 812 /* Copy out original auth_tag */ 813 scatterwalk_map_and_copy(auth_tag_msg, req->src, 814 req->assoclen + req->cryptlen - auth_tag_len, 815 auth_tag_len, 0); 816 817 /* Compare generated tag with passed in tag. */ 818 if (crypto_memneq(auth_tag_msg, auth_tag, auth_tag_len)) { 819 memzero_explicit(auth_tag, sizeof(auth_tag)); 820 return -EBADMSG; 821 } 822 return 0; 823 } 824 825 static int helper_rfc4106_encrypt(struct aead_request *req) 826 { 827 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 828 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 829 void *aes_ctx = &(ctx->aes_key_expanded); 830 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); 831 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); 832 unsigned int i; 833 __be32 counter = cpu_to_be32(1); 834 835 /* Assuming we are supporting rfc4106 64-bit extended */ 836 /* sequence numbers We need to have the AAD length equal */ 837 /* to 16 or 20 bytes */ 838 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 839 return -EINVAL; 840 841 /* IV below built */ 842 for (i = 0; i < 4; i++) 843 *(iv+i) = ctx->nonce[i]; 844 for (i = 0; i < 8; i++) 845 *(iv+4+i) = req->iv[i]; 846 *((__be32 *)(iv+12)) = counter; 847 848 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 849 aes_ctx); 850 } 851 852 static int helper_rfc4106_decrypt(struct aead_request *req) 853 { 854 __be32 counter = cpu_to_be32(1); 855 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 856 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 857 void *aes_ctx = &(ctx->aes_key_expanded); 858 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); 859 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); 860 unsigned int i; 861 862 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 863 return -EINVAL; 864 865 /* Assuming we are supporting rfc4106 64-bit extended */ 866 /* sequence numbers We need to have the AAD length */ 867 /* equal to 16 or 20 bytes */ 868 869 /* IV below built */ 870 for (i = 0; i < 4; i++) 871 *(iv+i) = ctx->nonce[i]; 872 for (i = 0; i < 8; i++) 873 *(iv+4+i) = req->iv[i]; 874 *((__be32 *)(iv+12)) = counter; 875 876 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 877 aes_ctx); 878 } 879 #endif 880 881 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, 882 unsigned int keylen) 883 { 884 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 885 int err; 886 887 err = xts_verify_key(tfm, key, keylen); 888 if (err) 889 return err; 890 891 keylen /= 2; 892 893 /* first half of xts-key is for crypt */ 894 err = aes_set_key_common(aes_ctx(ctx->raw_crypt_ctx), key, keylen); 895 if (err) 896 return err; 897 898 /* second half of xts-key is for tweak */ 899 return aes_set_key_common(aes_ctx(ctx->raw_tweak_ctx), key + keylen, 900 keylen); 901 } 902 903 static int xts_crypt(struct skcipher_request *req, bool encrypt) 904 { 905 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 906 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 907 int tail = req->cryptlen % AES_BLOCK_SIZE; 908 struct skcipher_request subreq; 909 struct skcipher_walk walk; 910 int err; 911 912 if (req->cryptlen < AES_BLOCK_SIZE) 913 return -EINVAL; 914 915 err = skcipher_walk_virt(&walk, req, false); 916 if (!walk.nbytes) 917 return err; 918 919 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 920 int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 921 922 skcipher_walk_abort(&walk); 923 924 skcipher_request_set_tfm(&subreq, tfm); 925 skcipher_request_set_callback(&subreq, 926 skcipher_request_flags(req), 927 NULL, NULL); 928 skcipher_request_set_crypt(&subreq, req->src, req->dst, 929 blocks * AES_BLOCK_SIZE, req->iv); 930 req = &subreq; 931 932 err = skcipher_walk_virt(&walk, req, false); 933 if (!walk.nbytes) 934 return err; 935 } else { 936 tail = 0; 937 } 938 939 kernel_fpu_begin(); 940 941 /* calculate first value of T */ 942 aesni_enc(aes_ctx(ctx->raw_tweak_ctx), walk.iv, walk.iv); 943 944 while (walk.nbytes > 0) { 945 int nbytes = walk.nbytes; 946 947 if (nbytes < walk.total) 948 nbytes &= ~(AES_BLOCK_SIZE - 1); 949 950 if (encrypt) 951 aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx), 952 walk.dst.virt.addr, walk.src.virt.addr, 953 nbytes, walk.iv); 954 else 955 aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx), 956 walk.dst.virt.addr, walk.src.virt.addr, 957 nbytes, walk.iv); 958 kernel_fpu_end(); 959 960 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 961 962 if (walk.nbytes > 0) 963 kernel_fpu_begin(); 964 } 965 966 if (unlikely(tail > 0 && !err)) { 967 struct scatterlist sg_src[2], sg_dst[2]; 968 struct scatterlist *src, *dst; 969 970 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 971 if (req->dst != req->src) 972 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 973 974 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 975 req->iv); 976 977 err = skcipher_walk_virt(&walk, &subreq, false); 978 if (err) 979 return err; 980 981 kernel_fpu_begin(); 982 if (encrypt) 983 aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx), 984 walk.dst.virt.addr, walk.src.virt.addr, 985 walk.nbytes, walk.iv); 986 else 987 aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx), 988 walk.dst.virt.addr, walk.src.virt.addr, 989 walk.nbytes, walk.iv); 990 kernel_fpu_end(); 991 992 err = skcipher_walk_done(&walk, 0); 993 } 994 return err; 995 } 996 997 static int xts_encrypt(struct skcipher_request *req) 998 { 999 return xts_crypt(req, true); 1000 } 1001 1002 static int xts_decrypt(struct skcipher_request *req) 1003 { 1004 return xts_crypt(req, false); 1005 } 1006 1007 static struct crypto_alg aesni_cipher_alg = { 1008 .cra_name = "aes", 1009 .cra_driver_name = "aes-aesni", 1010 .cra_priority = 300, 1011 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1012 .cra_blocksize = AES_BLOCK_SIZE, 1013 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1014 .cra_module = THIS_MODULE, 1015 .cra_u = { 1016 .cipher = { 1017 .cia_min_keysize = AES_MIN_KEY_SIZE, 1018 .cia_max_keysize = AES_MAX_KEY_SIZE, 1019 .cia_setkey = aes_set_key, 1020 .cia_encrypt = aesni_encrypt, 1021 .cia_decrypt = aesni_decrypt 1022 } 1023 } 1024 }; 1025 1026 static struct skcipher_alg aesni_skciphers[] = { 1027 { 1028 .base = { 1029 .cra_name = "__ecb(aes)", 1030 .cra_driver_name = "__ecb-aes-aesni", 1031 .cra_priority = 400, 1032 .cra_flags = CRYPTO_ALG_INTERNAL, 1033 .cra_blocksize = AES_BLOCK_SIZE, 1034 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1035 .cra_module = THIS_MODULE, 1036 }, 1037 .min_keysize = AES_MIN_KEY_SIZE, 1038 .max_keysize = AES_MAX_KEY_SIZE, 1039 .setkey = aesni_skcipher_setkey, 1040 .encrypt = ecb_encrypt, 1041 .decrypt = ecb_decrypt, 1042 }, { 1043 .base = { 1044 .cra_name = "__cbc(aes)", 1045 .cra_driver_name = "__cbc-aes-aesni", 1046 .cra_priority = 400, 1047 .cra_flags = CRYPTO_ALG_INTERNAL, 1048 .cra_blocksize = AES_BLOCK_SIZE, 1049 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1050 .cra_module = THIS_MODULE, 1051 }, 1052 .min_keysize = AES_MIN_KEY_SIZE, 1053 .max_keysize = AES_MAX_KEY_SIZE, 1054 .ivsize = AES_BLOCK_SIZE, 1055 .setkey = aesni_skcipher_setkey, 1056 .encrypt = cbc_encrypt, 1057 .decrypt = cbc_decrypt, 1058 }, { 1059 .base = { 1060 .cra_name = "__cts(cbc(aes))", 1061 .cra_driver_name = "__cts-cbc-aes-aesni", 1062 .cra_priority = 400, 1063 .cra_flags = CRYPTO_ALG_INTERNAL, 1064 .cra_blocksize = AES_BLOCK_SIZE, 1065 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1066 .cra_module = THIS_MODULE, 1067 }, 1068 .min_keysize = AES_MIN_KEY_SIZE, 1069 .max_keysize = AES_MAX_KEY_SIZE, 1070 .ivsize = AES_BLOCK_SIZE, 1071 .walksize = 2 * AES_BLOCK_SIZE, 1072 .setkey = aesni_skcipher_setkey, 1073 .encrypt = cts_cbc_encrypt, 1074 .decrypt = cts_cbc_decrypt, 1075 #ifdef CONFIG_X86_64 1076 }, { 1077 .base = { 1078 .cra_name = "__ctr(aes)", 1079 .cra_driver_name = "__ctr-aes-aesni", 1080 .cra_priority = 400, 1081 .cra_flags = CRYPTO_ALG_INTERNAL, 1082 .cra_blocksize = 1, 1083 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1084 .cra_module = THIS_MODULE, 1085 }, 1086 .min_keysize = AES_MIN_KEY_SIZE, 1087 .max_keysize = AES_MAX_KEY_SIZE, 1088 .ivsize = AES_BLOCK_SIZE, 1089 .chunksize = AES_BLOCK_SIZE, 1090 .setkey = aesni_skcipher_setkey, 1091 .encrypt = ctr_crypt, 1092 .decrypt = ctr_crypt, 1093 #endif 1094 }, { 1095 .base = { 1096 .cra_name = "__xts(aes)", 1097 .cra_driver_name = "__xts-aes-aesni", 1098 .cra_priority = 401, 1099 .cra_flags = CRYPTO_ALG_INTERNAL, 1100 .cra_blocksize = AES_BLOCK_SIZE, 1101 .cra_ctxsize = XTS_AES_CTX_SIZE, 1102 .cra_module = THIS_MODULE, 1103 }, 1104 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1105 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1106 .ivsize = AES_BLOCK_SIZE, 1107 .walksize = 2 * AES_BLOCK_SIZE, 1108 .setkey = xts_aesni_setkey, 1109 .encrypt = xts_encrypt, 1110 .decrypt = xts_decrypt, 1111 } 1112 }; 1113 1114 static 1115 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; 1116 1117 #ifdef CONFIG_X86_64 1118 /* 1119 * XCTR does not have a non-AVX implementation, so it must be enabled 1120 * conditionally. 1121 */ 1122 static struct skcipher_alg aesni_xctr = { 1123 .base = { 1124 .cra_name = "__xctr(aes)", 1125 .cra_driver_name = "__xctr-aes-aesni", 1126 .cra_priority = 400, 1127 .cra_flags = CRYPTO_ALG_INTERNAL, 1128 .cra_blocksize = 1, 1129 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1130 .cra_module = THIS_MODULE, 1131 }, 1132 .min_keysize = AES_MIN_KEY_SIZE, 1133 .max_keysize = AES_MAX_KEY_SIZE, 1134 .ivsize = AES_BLOCK_SIZE, 1135 .chunksize = AES_BLOCK_SIZE, 1136 .setkey = aesni_skcipher_setkey, 1137 .encrypt = xctr_crypt, 1138 .decrypt = xctr_crypt, 1139 }; 1140 1141 static struct simd_skcipher_alg *aesni_simd_xctr; 1142 #endif /* CONFIG_X86_64 */ 1143 1144 #ifdef CONFIG_X86_64 1145 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key, 1146 unsigned int key_len) 1147 { 1148 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead); 1149 1150 return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?: 1151 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 1152 } 1153 1154 static int generic_gcmaes_encrypt(struct aead_request *req) 1155 { 1156 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1157 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1158 void *aes_ctx = &(ctx->aes_key_expanded); 1159 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); 1160 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); 1161 __be32 counter = cpu_to_be32(1); 1162 1163 memcpy(iv, req->iv, 12); 1164 *((__be32 *)(iv+12)) = counter; 1165 1166 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv, 1167 aes_ctx); 1168 } 1169 1170 static int generic_gcmaes_decrypt(struct aead_request *req) 1171 { 1172 __be32 counter = cpu_to_be32(1); 1173 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1174 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1175 void *aes_ctx = &(ctx->aes_key_expanded); 1176 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); 1177 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); 1178 1179 memcpy(iv, req->iv, 12); 1180 *((__be32 *)(iv+12)) = counter; 1181 1182 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv, 1183 aes_ctx); 1184 } 1185 1186 static struct aead_alg aesni_aeads[] = { { 1187 .setkey = common_rfc4106_set_key, 1188 .setauthsize = common_rfc4106_set_authsize, 1189 .encrypt = helper_rfc4106_encrypt, 1190 .decrypt = helper_rfc4106_decrypt, 1191 .ivsize = GCM_RFC4106_IV_SIZE, 1192 .maxauthsize = 16, 1193 .base = { 1194 .cra_name = "__rfc4106(gcm(aes))", 1195 .cra_driver_name = "__rfc4106-gcm-aesni", 1196 .cra_priority = 400, 1197 .cra_flags = CRYPTO_ALG_INTERNAL, 1198 .cra_blocksize = 1, 1199 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), 1200 .cra_alignmask = 0, 1201 .cra_module = THIS_MODULE, 1202 }, 1203 }, { 1204 .setkey = generic_gcmaes_set_key, 1205 .setauthsize = generic_gcmaes_set_authsize, 1206 .encrypt = generic_gcmaes_encrypt, 1207 .decrypt = generic_gcmaes_decrypt, 1208 .ivsize = GCM_AES_IV_SIZE, 1209 .maxauthsize = 16, 1210 .base = { 1211 .cra_name = "__gcm(aes)", 1212 .cra_driver_name = "__generic-gcm-aesni", 1213 .cra_priority = 400, 1214 .cra_flags = CRYPTO_ALG_INTERNAL, 1215 .cra_blocksize = 1, 1216 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), 1217 .cra_alignmask = 0, 1218 .cra_module = THIS_MODULE, 1219 }, 1220 } }; 1221 #else 1222 static struct aead_alg aesni_aeads[0]; 1223 #endif 1224 1225 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)]; 1226 1227 static const struct x86_cpu_id aesni_cpu_id[] = { 1228 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL), 1229 {} 1230 }; 1231 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1232 1233 static int __init aesni_init(void) 1234 { 1235 int err; 1236 1237 if (!x86_match_cpu(aesni_cpu_id)) 1238 return -ENODEV; 1239 #ifdef CONFIG_X86_64 1240 if (boot_cpu_has(X86_FEATURE_AVX2)) { 1241 pr_info("AVX2 version of gcm_enc/dec engaged.\n"); 1242 static_branch_enable(&gcm_use_avx); 1243 static_branch_enable(&gcm_use_avx2); 1244 } else 1245 if (boot_cpu_has(X86_FEATURE_AVX)) { 1246 pr_info("AVX version of gcm_enc/dec engaged.\n"); 1247 static_branch_enable(&gcm_use_avx); 1248 } else { 1249 pr_info("SSE version of gcm_enc/dec engaged.\n"); 1250 } 1251 if (boot_cpu_has(X86_FEATURE_AVX)) { 1252 /* optimize performance of ctr mode encryption transform */ 1253 static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm); 1254 pr_info("AES CTR mode by8 optimization enabled\n"); 1255 } 1256 #endif /* CONFIG_X86_64 */ 1257 1258 err = crypto_register_alg(&aesni_cipher_alg); 1259 if (err) 1260 return err; 1261 1262 err = simd_register_skciphers_compat(aesni_skciphers, 1263 ARRAY_SIZE(aesni_skciphers), 1264 aesni_simd_skciphers); 1265 if (err) 1266 goto unregister_cipher; 1267 1268 err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads), 1269 aesni_simd_aeads); 1270 if (err) 1271 goto unregister_skciphers; 1272 1273 #ifdef CONFIG_X86_64 1274 if (boot_cpu_has(X86_FEATURE_AVX)) 1275 err = simd_register_skciphers_compat(&aesni_xctr, 1, 1276 &aesni_simd_xctr); 1277 if (err) 1278 goto unregister_aeads; 1279 #endif /* CONFIG_X86_64 */ 1280 1281 return 0; 1282 1283 #ifdef CONFIG_X86_64 1284 unregister_aeads: 1285 simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads), 1286 aesni_simd_aeads); 1287 #endif /* CONFIG_X86_64 */ 1288 1289 unregister_skciphers: 1290 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), 1291 aesni_simd_skciphers); 1292 unregister_cipher: 1293 crypto_unregister_alg(&aesni_cipher_alg); 1294 return err; 1295 } 1296 1297 static void __exit aesni_exit(void) 1298 { 1299 simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads), 1300 aesni_simd_aeads); 1301 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), 1302 aesni_simd_skciphers); 1303 crypto_unregister_alg(&aesni_cipher_alg); 1304 #ifdef CONFIG_X86_64 1305 if (boot_cpu_has(X86_FEATURE_AVX)) 1306 simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr); 1307 #endif /* CONFIG_X86_64 */ 1308 } 1309 1310 late_initcall(aesni_init); 1311 module_exit(aesni_exit); 1312 1313 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1314 MODULE_LICENSE("GPL"); 1315 MODULE_ALIAS_CRYPTO("aes"); 1316