1 /******************************************************************************* 2 Copyright (c) 2012-2020, Intel Corporation 3 4 Redistribution and use in source and binary forms, with or without 5 modification, are permitted provided that the following conditions are met: 6 7 * Redistributions of source code must retain the above copyright notice, 8 this list of conditions and the following disclaimer. 9 * Redistributions in binary form must reproduce the above copyright 10 notice, this list of conditions and the following disclaimer in the 11 documentation and/or other materials provided with the distribution. 12 * Neither the name of Intel Corporation nor the names of its contributors 13 may be used to endorse or promote products derived from this software 14 without specific prior written permission. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 20 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 *******************************************************************************/ 27 28 #ifndef IMB_IPSEC_MB_H 29 #define IMB_IPSEC_MB_H 30 31 #include <stdlib.h> 32 #include <stdint.h> 33 #include <errno.h> 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 /* 128-bit data type that is not in sdtint.h */ 40 typedef struct { 41 uint64_t low; 42 uint64_t high; 43 } imb_uint128_t; 44 45 /* 46 * Macros for aligning data structures and function inlines 47 */ 48 #if defined __linux__ || defined __FreeBSD__ || defined __DragonFly__ 49 /* Linux/FreeBSD */ 50 #define DECLARE_ALIGNED(decl, alignval) \ 51 decl __attribute__((aligned(alignval))) 52 #define __forceinline \ 53 static inline __attribute__((always_inline)) 54 55 #if __GNUC__ >= 4 56 #define IMB_DLL_EXPORT __attribute__((visibility("default"))) 57 #define IMB_DLL_LOCAL __attribute__((visibility("hidden"))) 58 #else /* GNU C 4.0 and later */ 59 #define IMB_DLL_EXPORT 60 #define IMB_DLL_LOCAL 61 #endif /* different C compiler */ 62 63 #else 64 /* Windows */ 65 #define DECLARE_ALIGNED(decl, alignval) \ 66 __declspec(align(alignval)) decl 67 #define __forceinline \ 68 static __forceinline 69 70 /* Windows DLL export is done via DEF file */ 71 #define IMB_DLL_EXPORT 72 #define IMB_DLL_LOCAL 73 #endif 74 75 /* Library version */ 76 #define IMB_VERSION_STR "0.55.0" 77 #define IMB_VERSION_NUM 0x3700 78 79 /* Macro to translate version number */ 80 #define IMB_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c)) 81 82 /* 83 * Custom ASSERT and DIM macros 84 */ 85 #ifdef DEBUG 86 #include <assert.h> 87 #define IMB_ASSERT(x) assert(x) 88 #else 89 #define IMB_ASSERT(x) 90 #endif 91 92 #ifndef IMB_DIM 93 #define IMB_DIM(x) (sizeof(x) / sizeof(x[0])) 94 #endif 95 96 /* 97 * Architecture definitions 98 */ 99 typedef enum { 100 IMB_ARCH_NONE = 0, 101 IMB_ARCH_NOAESNI, 102 IMB_ARCH_SSE, 103 IMB_ARCH_AVX, 104 IMB_ARCH_AVX2, 105 IMB_ARCH_AVX512, 106 IMB_ARCH_NUM, 107 } IMB_ARCH; 108 109 /* 110 * Algorithm constants 111 */ 112 #define DES_KEY_SCHED_SIZE (16 * 8) /* 16 rounds x 8 bytes */ 113 #define DES_BLOCK_SIZE 8 114 115 #define AES_BLOCK_SIZE 16 116 117 #define SHA1_DIGEST_SIZE_IN_BYTES 20 118 #define SHA224_DIGEST_SIZE_IN_BYTES 28 119 #define SHA256_DIGEST_SIZE_IN_BYTES 32 120 #define SHA384_DIGEST_SIZE_IN_BYTES 48 121 #define SHA512_DIGEST_SIZE_IN_BYTES 64 122 123 #define SHA1_BLOCK_SIZE 64 /* 512 bits is 64 byte blocks */ 124 #define SHA_256_BLOCK_SIZE 64 /* 512 bits is 64 byte blocks */ 125 #define SHA_384_BLOCK_SIZE 128 126 #define SHA_512_BLOCK_SIZE 128 127 128 #define KASUMI_KEY_SIZE 16 129 #define KASUMI_IV_SIZE 8 130 #define KASUMI_BLOCK_SIZE 8 131 #define KASUMI_DIGEST_SIZE 4 132 133 /** 134 * Minimum Ethernet frame size to calculate CRC32 135 * Source Address (6 bytes) + Destination Address (6 bytes) + Type/Len (2 bytes) 136 */ 137 #define DOCSIS_CRC32_MIN_ETH_PDU_SIZE 14 138 #define DOCSIS_CRC32_TAG_SIZE 4 139 140 /* 141 * Job structure definitions 142 */ 143 144 typedef enum { 145 STS_BEING_PROCESSED = 0, 146 STS_COMPLETED_AES = 1, 147 STS_COMPLETED_HMAC = 2, 148 STS_COMPLETED = 3, /* COMPLETED_AES | COMPLETED_HMAC */ 149 STS_INVALID_ARGS = 4, 150 STS_INTERNAL_ERROR, 151 STS_ERROR 152 } JOB_STS; 153 154 /* 155 * Library error types 156 */ 157 enum { 158 IMB_ERR_MIN = 2000, 159 IMB_ERR_NULL_MBMGR, 160 IMB_ERR_JOB_NULL_SRC, 161 IMB_ERR_JOB_NULL_DST, 162 IMB_ERR_JOB_NULL_KEY, 163 IMB_ERR_JOB_NULL_IV, 164 IMB_ERR_JOB_NULL_AUTH, 165 IMB_ERR_JOB_NULL_AAD, 166 IMB_ERR_JOB_CIPH_LEN, 167 IMB_ERR_JOB_AUTH_LEN, 168 IMB_ERR_JOB_IV_LEN, 169 IMB_ERR_JOB_KEY_LEN, 170 IMB_ERR_JOB_AUTH_TAG_LEN, 171 IMB_ERR_JOB_AAD_LEN, 172 IMB_ERR_JOB_SRC_OFFSET, 173 IMB_ERR_JOB_CHAIN_ORDER, 174 IMB_ERR_CIPH_MODE, 175 IMB_ERR_HASH_ALGO, 176 IMB_ERR_JOB_NULL_AUTH_KEY, 177 IMB_ERR_MAX /* don't move this one */ 178 }; 179 180 /* 181 * IMB_ERR_MIN should be higher than __ELASTERROR 182 * to avoid overlap with standard error values 183 */ 184 #ifdef __ELASTERROR 185 #if __ELASTERROR > 2000 186 #error "Library error codes conflict with errno.h - please update IMB_ERR_MIN!" 187 #endif 188 #endif 189 190 /* 191 * Define enums from API v0.53, so applications that were using this version 192 * will still be compiled successfully. 193 * This list does not need to be extended for new enums. 194 */ 195 #ifndef NO_COMPAT_IMB_API_053 196 /***** Previous cipher mode enums *****/ 197 #define CBC IMB_CIPHER_CBC 198 #define CNTR IMB_CIPHER_CNTR 199 #define NULL_CIPHER IMB_CIPHER_NULL 200 #define DOCSIS_SEC_BPI IMB_CIPHER_DOCSIS_SEC_BPI 201 #define GCM IMB_CIPHER_GCM 202 #define CUSTOM_CIPHER IMB_CIPHER_CUSTOM 203 #define DES IMB_CIPHER_DES 204 #define DOCSIS_DES IMB_CIPHER_DOCSIS_DES 205 #define CCM IMB_CIPHER_CCM 206 #define DES3 IMB_CIPHER_DES3 207 #define PON_AES_CNTR IMB_CIPHER_PON_AES_CNTR 208 #define ECB IMB_CIPHER_ECB 209 #define CNTR_BITLEN IMB_CIPHER_CNTR_BITLEN 210 211 /***** Previous hash algo enums *****/ 212 #define SHA1 IMB_AUTH_HMAC_SHA_1 213 #define SHA_224 IMB_AUTH_HMAC_SHA_224 214 #define SHA_256 IMB_AUTH_HMAC_SHA_256 215 #define SHA_384 IMB_AUTH_HMAC_SHA_384 216 #define SHA_512 IMB_AUTH_HMAC_SHA_512 217 #define AES_XCBC IMB_AUTH_AES_XCBC 218 #define MD5 IMB_AUTH_MD5 219 #define NULL_HASH IMB_AUTH_NULL 220 #define AES_GMAC IMB_AUTH_AES_GMAC 221 #define CUSTOM_HASH IMB_AUTH_CUSTOM 222 #define AES_CCM IMB_AUTH_AES_CCM 223 #define AES_CMAC IMB_AUTH_AES_CMAC 224 #define PLAIN_SHA1 IMB_AUTH_SHA_1 225 #define PLAIN_SHA_224 IMB_AUTH_SHA_224 226 #define PLAIN_SHA_256 IMB_AUTH_SHA_256 227 #define PLAIN_SHA_384 IMB_AUTH_SHA_384 228 #define PLAIN_SHA_512 IMB_AUTH_SHA_512 229 #define AES_CMAC_BITLEN IMB_AUTH_AES_CMAC_BITLEN 230 #define PON_CRC_BIP IMB_AUTH_PON_CRC_BIP 231 232 /***** Previous cipher direction enums *****/ 233 #define ENCRYPT IMB_DIR_ENCRYPT 234 #define DECRYPT IMB_DIR_DECRYPT 235 236 /***** Previous chain order enums *****/ 237 #define HASH_CIPHER IMB_ORDER_HASH_CIPHER 238 #define CIPHER_HASH IMB_ORDER_CIPHER_HASH 239 240 /***** Previous key size enums *****/ 241 #define AES_128_BYTES IMB_KEY_AES_128_BYTES 242 #define AES_192_BYTES IMB_KEY_AES_192_BYTES 243 #define AES_256_BYTES IMB_KEY_AES_256_BYTES 244 245 #define MB_MGR IMB_MGR 246 #define JOB_AES_HMAC IMB_JOB 247 248 /***** Previous fields in IMB_JOB/JOB_AES_HMAC *****/ 249 #define aes_enc_key_expanded enc_keys 250 #define aes_dec_key_expanded dec_keys 251 #define aes_key_len_in_bytes key_len_in_bytes 252 #endif /* !NO_COMPAT_IMB_API_053 */ 253 254 typedef enum { 255 IMB_CIPHER_CBC = 1, 256 IMB_CIPHER_CNTR, 257 IMB_CIPHER_NULL, 258 IMB_CIPHER_DOCSIS_SEC_BPI, 259 IMB_CIPHER_GCM, 260 IMB_CIPHER_CUSTOM, 261 IMB_CIPHER_DES, 262 IMB_CIPHER_DOCSIS_DES, 263 IMB_CIPHER_CCM, 264 IMB_CIPHER_DES3, 265 IMB_CIPHER_PON_AES_CNTR, 266 IMB_CIPHER_ECB, 267 IMB_CIPHER_CNTR_BITLEN, /* 128-EEA2/NEA2 (3GPP) */ 268 IMB_CIPHER_ZUC_EEA3, /* 128-EEA3/NEA3 (3GPP) */ 269 IMB_CIPHER_SNOW3G_UEA2_BITLEN,/* 128-UEA2 (3GPP) */ 270 IMB_CIPHER_KASUMI_UEA1_BITLEN,/* 128-UEA1 (3GPP) */ 271 IMB_CIPHER_CBCS_1_9, /* MPEG CENC (ISO 23001-7) */ 272 IMB_CIPHER_CHACHA20, 273 IMB_CIPHER_CHACHA20_POLY1305, /* AEAD CHACHA20 */ 274 IMB_CIPHER_NUM 275 } JOB_CIPHER_MODE; 276 277 typedef enum { 278 IMB_DIR_ENCRYPT = 1, 279 IMB_DIR_DECRYPT 280 } JOB_CIPHER_DIRECTION; 281 282 typedef enum { 283 IMB_AUTH_HMAC_SHA_1 = 1, /* HMAC-SHA1 */ 284 IMB_AUTH_HMAC_SHA_224, /* HMAC-SHA224 */ 285 IMB_AUTH_HMAC_SHA_256, /* HMAC-SHA256 */ 286 IMB_AUTH_HMAC_SHA_384, /* HMAC-SHA384 */ 287 IMB_AUTH_HMAC_SHA_512, /* HMAC-SHA512 */ 288 IMB_AUTH_AES_XCBC, 289 IMB_AUTH_MD5, /* HMAC-MD5 */ 290 IMB_AUTH_NULL, 291 IMB_AUTH_AES_GMAC, 292 IMB_AUTH_CUSTOM, 293 IMB_AUTH_AES_CCM, /* AES128-CCM */ 294 IMB_AUTH_AES_CMAC, /* AES128-CMAC */ 295 IMB_AUTH_SHA_1, /* SHA1 */ 296 IMB_AUTH_SHA_224, /* SHA224 */ 297 IMB_AUTH_SHA_256, /* SHA256 */ 298 IMB_AUTH_SHA_384, /* SHA384 */ 299 IMB_AUTH_SHA_512, /* SHA512 */ 300 IMB_AUTH_AES_CMAC_BITLEN, /* 128-EIA2/NIA2 (3GPP) */ 301 IMB_AUTH_PON_CRC_BIP, 302 IMB_AUTH_ZUC_EIA3_BITLEN, /* 128-EIA3/NIA3 (3GPP) */ 303 IMB_AUTH_DOCSIS_CRC32, /* with DOCSIS_SEC_BPI only */ 304 IMB_AUTH_SNOW3G_UIA2_BITLEN, /* 128-UIA2 (3GPP) */ 305 IMB_AUTH_KASUMI_UIA1, /* 128-UIA1 (3GPP) */ 306 IMB_AUTH_AES_GMAC_128, /* AES-GMAC (128-bit key) */ 307 IMB_AUTH_AES_GMAC_192, /* AES-GMAC (192-bit key) */ 308 IMB_AUTH_AES_GMAC_256, /* AES-GMAC (256-bit key) */ 309 IMB_AUTH_AES_CMAC_256, /* AES256-CMAC */ 310 IMB_AUTH_POLY1305, /* POLY1305 */ 311 IMB_AUTH_CHACHA20_POLY1305, /* AEAD POLY1305 */ 312 IMB_AUTH_NUM 313 } JOB_HASH_ALG; 314 315 typedef enum { 316 IMB_ORDER_CIPHER_HASH = 1, 317 IMB_ORDER_HASH_CIPHER 318 } JOB_CHAIN_ORDER; 319 320 typedef enum { 321 IMB_KEY_AES_128_BYTES = 16, 322 IMB_KEY_AES_192_BYTES = 24, 323 IMB_KEY_AES_256_BYTES = 32 324 } AES_KEY_SIZE_BYTES; 325 326 typedef struct IMB_JOB { 327 /* 328 * For AES, enc_keys and dec_keys are 329 * expected to point to expanded keys structure. 330 * - AES-CTR, AES-ECB and AES-CCM, only enc_keys is used 331 * - DOCSIS (AES-CBC + AES-CFB), both pointers are used 332 * enc_keys has to be set always for the partial block 333 * 334 * For DES, enc_keys and dec_keys are 335 * expected to point to DES key schedule. 336 * - same key schedule used for enc and dec operations 337 * 338 * For 3DES, enc_keys and dec_keys are 339 * expected to point to an array of 3 pointers for 340 * the corresponding 3 key schedules. 341 * - same key schedule used for enc and dec operations 342 */ 343 const void *enc_keys; /* 16-byte aligned pointer. */ 344 const void *dec_keys; 345 uint64_t key_len_in_bytes; 346 const uint8_t *src; /* Input. May be cipher text or plaintext. 347 * In-place ciphering allowed. */ 348 uint8_t *dst; /*Output. May be cipher text or plaintext. 349 * In-place ciphering allowed, i.e. dst = src. */ 350 union { 351 uint64_t cipher_start_src_offset_in_bytes; 352 uint64_t cipher_start_src_offset_in_bits; 353 }; 354 /* Max len = 65472 bytes. 355 * IPSec case, the maximum cipher 356 * length would be: 357 * 65535 - 358 * 20 (outer IP header) - 359 * 24 (ESP header + IV) - 360 * 12 (supported ICV length) */ 361 union { 362 uint64_t msg_len_to_cipher_in_bytes; 363 uint64_t msg_len_to_cipher_in_bits; 364 }; 365 uint64_t hash_start_src_offset_in_bytes; 366 /* Max len = 65496 bytes. 367 * (Max cipher len + 368 * 24 bytes ESP header) */ 369 union { 370 uint64_t msg_len_to_hash_in_bytes; 371 uint64_t msg_len_to_hash_in_bits; 372 }; 373 const uint8_t *iv; /* Initialization Vector (IV) */ 374 uint64_t iv_len_in_bytes; /* IV length in bytes. */ 375 uint8_t *auth_tag_output; /* Tag output. This may point to 376 * a location in the src buffer 377 * (for in place)*/ 378 uint64_t auth_tag_output_len_in_bytes; /* Authentication (i.e. HMAC) tag 379 * output length in bytes 380 * (may be a truncated value) */ 381 382 /* Start algorithm-specific fields */ 383 union { 384 struct _HMAC_specific_fields { 385 /* Hashed result of HMAC key xor'd with ipad (0x36). */ 386 const uint8_t *_hashed_auth_key_xor_ipad; 387 /* Hashed result of HMAC key xor'd with opad (0x5c). */ 388 const uint8_t *_hashed_auth_key_xor_opad; 389 } HMAC; 390 struct _AES_XCBC_specific_fields { 391 /* 16-byte aligned pointers */ 392 const uint32_t *_k1_expanded; 393 const uint8_t *_k2; 394 const uint8_t *_k3; 395 } XCBC; 396 struct _AES_CCM_specific_fields { 397 /* Additional Authentication Data (AAD) */ 398 const void *aad; 399 uint64_t aad_len_in_bytes; /* Length of AAD */ 400 } CCM; 401 struct _AES_CMAC_specific_fields { 402 const void *_key_expanded; /* 16-byte aligned */ 403 const void *_skey1; 404 const void *_skey2; 405 } CMAC; 406 struct _AES_GCM_specific_fields { 407 /* Additional Authentication Data (AAD) */ 408 const void *aad; 409 uint64_t aad_len_in_bytes; /* Length of AAD */ 410 } GCM; 411 struct _ZUC_EIA3_specific_fields { 412 /* 16-byte aligned pointers */ 413 const uint8_t *_key; 414 const uint8_t *_iv; 415 } ZUC_EIA3; 416 struct _SNOW3G_UIA2_specific_fields { 417 /* 16-byte aligned pointers */ 418 const void *_key; 419 const void *_iv; 420 } SNOW3G_UIA2; 421 struct _KASUMI_UIA1_specific_fields { 422 /* 16-byte aligned pointers */ 423 const void *_key; 424 } KASUMI_UIA1; 425 struct _AES_GMAC_specific_fields { 426 const struct gcm_key_data *_key; 427 const void *_iv; 428 uint64_t iv_len_in_bytes; 429 } GMAC; /* Used with AES_GMAC_128/192/256 */ 430 struct _POLY1305_specific_fields { 431 const void *_key; /* pointer to 32 byte key */ 432 } POLY1305; 433 struct _CHACHA20_POLY1305_specific_fields { 434 /* Additional Authentication Data (AAD) */ 435 const void *aad; 436 uint64_t aad_len_in_bytes; /* Length of AAD */ 437 } CHACHA20_POLY1305; 438 } u; 439 440 JOB_STS status; 441 /* IMB_CIPHER_CBC, IMB_CIPHER_CNTR, IMB_CIPHER_GCM, etc. */ 442 JOB_CIPHER_MODE cipher_mode; 443 /* IMB_DIR_ENCRYPT/IMB_DIR_DECRYPT */ 444 JOB_CIPHER_DIRECTION cipher_direction; 445 JOB_HASH_ALG hash_alg; /* IMB_AUTH_SHA_1 or others... */ 446 /* IMB_ORDER_CIPHER_HASH or IMB_ORDER_HASH_CIPHER. 447 * For AES-CCM, when encrypting, IMB_ORDER_HASH_CIPHER 448 * must be selected, and when decrypting, 449 * IMB_ORDER_CIPHER_HASH must be selected. */ 450 JOB_CHAIN_ORDER chain_order; 451 452 void *user_data; 453 void *user_data2; 454 455 /* 456 * stateless custom cipher and hash 457 * Return: 458 * success: 0 459 * fail: other 460 */ 461 int (*cipher_func)(struct IMB_JOB *); 462 int (*hash_func)(struct IMB_JOB *); 463 } IMB_JOB; 464 465 466 /* KASUMI */ 467 468 /* 64 precomputed words for key schedule */ 469 #define KASUMI_KEY_SCHEDULE_SIZE 64 470 471 /** 472 * Structure to maintain internal key scheduling 473 */ 474 typedef struct kasumi_key_sched_s { 475 /* Kasumi internal scheduling */ 476 uint16_t sk16[KASUMI_KEY_SCHEDULE_SIZE]; /* key schedule */ 477 uint16_t msk16[KASUMI_KEY_SCHEDULE_SIZE]; /* modified key schedule */ 478 } kasumi_key_sched_t; 479 480 /* GCM data structures */ 481 #define GCM_BLOCK_LEN 16 482 483 /** 484 * @brief holds GCM operation context 485 */ 486 struct gcm_context_data { 487 /* init, update and finalize context data */ 488 uint8_t aad_hash[GCM_BLOCK_LEN]; 489 uint64_t aad_length; 490 uint64_t in_length; 491 uint8_t partial_block_enc_key[GCM_BLOCK_LEN]; 492 uint8_t orig_IV[GCM_BLOCK_LEN]; 493 uint8_t current_counter[GCM_BLOCK_LEN]; 494 uint64_t partial_block_length; 495 }; 496 497 /* Authenticated Tag Length in bytes. 498 * Valid values are 16 (most likely), 12 or 8. */ 499 #define MAX_TAG_LEN (16) 500 501 /* 502 * IV data is limited to 16 bytes as follows: 503 * 12 bytes is provided by an application - 504 * pre-counter block j0: 4 byte salt (from Security Association) 505 * concatenated with 8 byte Initialization Vector (from IPSec ESP 506 * Payload). 507 * 4 byte value 0x00000001 is padded automatically by the library - 508 * there is no need to add these 4 bytes on application side anymore. 509 */ 510 #define GCM_IV_DATA_LEN (12) 511 512 #define LONGEST_TESTED_AAD_LENGTH (2 * 1024) 513 514 /* Key lengths of 128 and 256 supported */ 515 #define GCM_128_KEY_LEN (16) 516 #define GCM_192_KEY_LEN (24) 517 #define GCM_256_KEY_LEN (32) 518 519 /* #define GCM_BLOCK_LEN 16 */ 520 #define GCM_ENC_KEY_LEN 16 521 #define GCM_KEY_SETS (15) /*exp key + 14 exp round keys*/ 522 523 /** 524 * @brief holds intermediate key data needed to improve performance 525 * 526 * gcm_key_data hold internal key information used by gcm128, gcm192 and gcm256. 527 */ 528 #ifdef __WIN32 529 __declspec(align(64)) 530 #endif /* WIN32 */ 531 struct gcm_key_data { 532 uint8_t expanded_keys[GCM_ENC_KEY_LEN * GCM_KEY_SETS]; 533 union { 534 /* Storage for precomputed hash keys */ 535 struct { 536 /* 537 * This is needed for schoolbook multiply purposes. 538 * (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., 539 * (Hashkey^48<<1 mod poly) 540 */ 541 uint8_t shifted_hkey[GCM_ENC_KEY_LEN * 8]; 542 /* 543 * This is needed for Karatsuba multiply purposes. 544 * Storage for XOR of High 64 bits and low 64 bits 545 * of HashKey mod poly. 546 * 547 * (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., 548 * (Hashkey^128<<1 mod poly) 549 */ 550 uint8_t shifted_hkey_k[GCM_ENC_KEY_LEN * 8]; 551 } sse_avx; 552 struct { 553 /* 554 * This is needed for schoolbook multiply purposes. 555 * (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., 556 * (Hashkey^48<<1 mod poly) 557 */ 558 uint8_t shifted_hkey[GCM_ENC_KEY_LEN * 8]; 559 } avx2_avx512; 560 struct { 561 /* 562 * (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., 563 * (Hashkey^48<<1 mod poly) 564 */ 565 uint8_t shifted_hkey[GCM_ENC_KEY_LEN * 48]; 566 } vaes_avx512; 567 } ghash_keys; 568 } 569 #ifdef LINUX 570 __attribute__((aligned(64))); 571 #else 572 ; 573 #endif 574 575 /* ========================================================================== */ 576 /* API data type definitions */ 577 struct IMB_MGR; 578 579 typedef void (*init_mb_mgr_t)(struct IMB_MGR *); 580 typedef IMB_JOB *(*get_next_job_t)(struct IMB_MGR *); 581 typedef IMB_JOB *(*submit_job_t)(struct IMB_MGR *); 582 typedef IMB_JOB *(*get_completed_job_t)(struct IMB_MGR *); 583 typedef IMB_JOB *(*flush_job_t)(struct IMB_MGR *); 584 typedef uint32_t (*queue_size_t)(struct IMB_MGR *); 585 typedef void (*keyexp_t)(const void *, void *, void *); 586 typedef void (*cmac_subkey_gen_t)(const void *, void *, void *); 587 typedef void (*hash_one_block_t)(const void *, void *); 588 typedef void (*hash_fn_t)(const void *, const uint64_t, void *); 589 typedef void (*xcbc_keyexp_t)(const void *, void *, void *, void *); 590 typedef int (*des_keysched_t)(uint64_t *, const void *); 591 typedef void (*aes_cfb_t)(void *, const void *, const void *, const void *, 592 uint64_t); 593 typedef void (*aes_gcm_enc_dec_t)(const struct gcm_key_data *, 594 struct gcm_context_data *, 595 uint8_t *, uint8_t const *, uint64_t, 596 const uint8_t *, uint8_t const *, uint64_t, 597 uint8_t *, uint64_t); 598 typedef void (*aes_gcm_enc_dec_iv_t)(const struct gcm_key_data *, 599 struct gcm_context_data *, uint8_t *, 600 uint8_t const *, const uint64_t, 601 const uint8_t *, uint8_t const *, 602 const uint64_t, uint8_t *, 603 const uint64_t, const uint64_t); 604 typedef void (*aes_gcm_init_t)(const struct gcm_key_data *, 605 struct gcm_context_data *, 606 const uint8_t *, uint8_t const *, uint64_t); 607 typedef void (*aes_gcm_init_var_iv_t)(const struct gcm_key_data *, 608 struct gcm_context_data *, 609 const uint8_t *, const uint64_t, 610 const uint8_t *, const uint64_t); 611 typedef void (*aes_gcm_enc_dec_update_t)(const struct gcm_key_data *, 612 struct gcm_context_data *, 613 uint8_t *, const uint8_t *, uint64_t); 614 typedef void (*aes_gcm_enc_dec_finalize_t)(const struct gcm_key_data *, 615 struct gcm_context_data *, 616 uint8_t *, uint64_t); 617 typedef void (*aes_gcm_precomp_t)(struct gcm_key_data *); 618 typedef void (*aes_gcm_pre_t)(const void *, struct gcm_key_data *); 619 620 typedef void (*aes_gmac_init_t)(const struct gcm_key_data *, 621 struct gcm_context_data *, 622 const uint8_t *, const uint64_t); 623 typedef void (*aes_gmac_update_t)(const struct gcm_key_data *, 624 struct gcm_context_data *, 625 const uint8_t *, const uint64_t); 626 typedef void (*aes_gmac_finalize_t)(const struct gcm_key_data *, 627 struct gcm_context_data *, 628 uint8_t *, const uint64_t); 629 630 typedef void (*ghash_t)(struct gcm_key_data *, const void *, 631 const uint64_t, void *, const uint64_t); 632 633 typedef void (*zuc_eea3_1_buffer_t)(const void *, const void *, const void *, 634 void *, const uint32_t); 635 636 typedef void (*zuc_eea3_4_buffer_t)(const void * const *, const void * const *, 637 const void * const *, void **, 638 const uint32_t *); 639 640 typedef void (*zuc_eea3_n_buffer_t)(const void * const *, const void * const *, 641 const void * const *, void **, 642 const uint32_t *, const uint32_t); 643 644 typedef void (*zuc_eia3_1_buffer_t)(const void *, const void *, const void *, 645 const uint32_t, uint32_t *); 646 647 typedef void (*zuc_eia3_n_buffer_t)(const void * const *, const void * const *, 648 const void * const *, 649 const uint32_t *, uint32_t **, 650 const uint32_t); 651 652 653 typedef void (*kasumi_f8_1_buffer_t)(const kasumi_key_sched_t *, 654 const uint64_t, const void *, void *, 655 const uint32_t); 656 typedef void (*kasumi_f8_1_buffer_bit_t)(const kasumi_key_sched_t *, 657 const uint64_t, const void *, 658 void *, 659 const uint32_t, const uint32_t); 660 typedef void (*kasumi_f8_2_buffer_t)(const kasumi_key_sched_t *, 661 const uint64_t, const uint64_t, 662 const void *, void *, 663 const uint32_t, 664 const void *, void *, 665 const uint32_t); 666 typedef void (*kasumi_f8_3_buffer_t)(const kasumi_key_sched_t *, 667 const uint64_t, const uint64_t, 668 const uint64_t, 669 const void *, void *, 670 const void *, void *, 671 const void *, void *, 672 const uint32_t); 673 typedef void (*kasumi_f8_4_buffer_t)(const kasumi_key_sched_t *, 674 const uint64_t, const uint64_t, 675 const uint64_t, const uint64_t, 676 const void *, void *, 677 const void *, void *, 678 const void *, void *, 679 const void *, void *, 680 const uint32_t); 681 typedef void (*kasumi_f8_n_buffer_t)(const kasumi_key_sched_t *, 682 const uint64_t *, const void * const *, 683 void **, const uint32_t *, 684 const uint32_t); 685 typedef void (*kasumi_f9_1_buffer_user_t)(const kasumi_key_sched_t *, 686 const uint64_t, const void *, 687 const uint32_t, void *, 688 const uint32_t); 689 typedef void (*kasumi_f9_1_buffer_t)(const kasumi_key_sched_t *, 690 const void *, 691 const uint32_t, void *); 692 typedef int (*kasumi_init_f8_key_sched_t)(const void *, 693 kasumi_key_sched_t *); 694 typedef int (*kasumi_init_f9_key_sched_t)(const void *, 695 kasumi_key_sched_t *); 696 typedef size_t (*kasumi_key_sched_size_t)(void); 697 698 699 /** 700 * Snow3G key scheduling structure 701 */ 702 typedef struct snow3g_key_schedule_s { 703 /* KEY */ 704 uint32_t k[4]; 705 } snow3g_key_schedule_t; 706 707 typedef void (*snow3g_f8_1_buffer_t)(const snow3g_key_schedule_t *, 708 const void *, const void *, 709 void *, const uint32_t); 710 711 typedef void (*snow3g_f8_1_buffer_bit_t)(const snow3g_key_schedule_t *, 712 const void *, const void *, void *, 713 const uint32_t, const uint32_t); 714 715 typedef void (*snow3g_f8_2_buffer_t)(const snow3g_key_schedule_t *, 716 const void *, const void *, 717 const void *, void *, const uint32_t, 718 const void *, void *,const uint32_t); 719 720 typedef void (*snow3g_f8_4_buffer_t)(const snow3g_key_schedule_t *, 721 const void *, const void *,const void *, 722 const void *, const void *, void *, 723 const uint32_t, const void *, void *, 724 const uint32_t, const void *, void *, 725 const uint32_t, const void *, void *, 726 const uint32_t); 727 728 typedef void (*snow3g_f8_8_buffer_t)(const snow3g_key_schedule_t *, 729 const void *, const void *,const void *, 730 const void *, const void *, const void *, 731 const void *, const void *, const void *, 732 void *, const uint32_t, const void *, 733 void *, const uint32_t, const void *, 734 void *, const uint32_t, const void *, 735 void *, const uint32_t, const void *, 736 void *, const uint32_t, const void *, 737 void *, const uint32_t, const void *, 738 void *, const uint32_t, const void *, 739 void *, const uint32_t); 740 741 typedef void 742 (*snow3g_f8_8_buffer_multikey_t)(const snow3g_key_schedule_t * const [], 743 const void * const [], const void * const [], 744 void *[], const uint32_t[]); 745 746 typedef void (*snow3g_f8_n_buffer_t)(const snow3g_key_schedule_t *, 747 const void * const [], 748 const void * const [], 749 void *[], const uint32_t[], 750 const uint32_t); 751 752 typedef void 753 (*snow3g_f8_n_buffer_multikey_t)(const snow3g_key_schedule_t * const [], 754 const void * const [], 755 const void * const [], 756 void *[], const uint32_t[], 757 const uint32_t); 758 759 typedef void (*snow3g_f9_1_buffer_t)(const snow3g_key_schedule_t *, 760 const void *, const void *, 761 const uint64_t, void *); 762 763 typedef int (*snow3g_init_key_sched_t)(const void *, 764 snow3g_key_schedule_t *); 765 766 typedef size_t (*snow3g_key_sched_size_t)(void); 767 768 typedef uint32_t (*hec_32_t)(const uint8_t *); 769 typedef uint64_t (*hec_64_t)(const uint8_t *); 770 771 typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); 772 /* ========================================================================== */ 773 /* Multi-buffer manager flags passed to alloc_mb_mgr() */ 774 775 #define IMB_FLAG_SHANI_OFF (1ULL << 0) /* disable use of SHANI extension */ 776 #define IMB_FLAG_AESNI_OFF (1ULL << 1) /* disable use of AESNI extension */ 777 778 /* ========================================================================== */ 779 /* Multi-buffer manager detected features 780 * - if bit is set then hardware supports given extension 781 * - valid after call to init_mb_mgr() or alloc_mb_mgr() 782 * - some HW supported features can be disabled via IMB_FLAG_xxx (see above) 783 */ 784 785 #define IMB_FEATURE_SHANI (1ULL << 0) 786 #define IMB_FEATURE_AESNI (1ULL << 1) 787 #define IMB_FEATURE_PCLMULQDQ (1ULL << 2) 788 #define IMB_FEATURE_CMOV (1ULL << 3) 789 #define IMB_FEATURE_SSE4_2 (1ULL << 4) 790 #define IMB_FEATURE_AVX (1ULL << 5) 791 #define IMB_FEATURE_AVX2 (1ULL << 6) 792 #define IMB_FEATURE_AVX512F (1ULL << 7) 793 #define IMB_FEATURE_AVX512DQ (1ULL << 8) 794 #define IMB_FEATURE_AVX512CD (1ULL << 9) 795 #define IMB_FEATURE_AVX512BW (1ULL << 10) 796 #define IMB_FEATURE_AVX512VL (1ULL << 11) 797 #define IMB_FEATURE_AVX512_SKX (IMB_FEATURE_AVX512F | IMB_FEATURE_AVX512DQ | \ 798 IMB_FEATURE_AVX512CD | IMB_FEATURE_AVX512BW | \ 799 IMB_FEATURE_AVX512VL) 800 #define IMB_FEATURE_VAES (1ULL << 12) 801 #define IMB_FEATURE_VPCLMULQDQ (1ULL << 13) 802 #define IMB_FEATURE_SAFE_DATA (1ULL << 14) 803 #define IMB_FEATURE_SAFE_PARAM (1ULL << 15) 804 #define IMB_FEATURE_GFNI (1ULL << 16) 805 806 /* ========================================================================== */ 807 /* TOP LEVEL (IMB_MGR) Data structure fields */ 808 809 #define MAX_JOBS 128 810 811 typedef struct IMB_MGR { 812 /* 813 * flags - passed to alloc_mb_mgr() 814 * features - reflects features of multi-buffer instance 815 */ 816 uint64_t flags; 817 uint64_t features; 818 819 /* 820 * Reserved for the future 821 */ 822 uint64_t reserved[5]; 823 uint32_t reserved2[1]; 824 825 /* per mb_mgr error status */ 826 int imb_errno; 827 828 /* 829 * ARCH handlers / API 830 * Careful as changes here can break ABI compatibility 831 * (always include function pointers at the end of the list, 832 * before "earliest_job") 833 */ 834 get_next_job_t get_next_job; 835 submit_job_t submit_job; 836 submit_job_t submit_job_nocheck; 837 get_completed_job_t get_completed_job; 838 flush_job_t flush_job; 839 queue_size_t queue_size; 840 keyexp_t keyexp_128; 841 keyexp_t keyexp_192; 842 keyexp_t keyexp_256; 843 cmac_subkey_gen_t cmac_subkey_gen_128; 844 xcbc_keyexp_t xcbc_keyexp; 845 des_keysched_t des_key_sched; 846 hash_one_block_t sha1_one_block; 847 hash_one_block_t sha224_one_block; 848 hash_one_block_t sha256_one_block; 849 hash_one_block_t sha384_one_block; 850 hash_one_block_t sha512_one_block; 851 hash_one_block_t md5_one_block; 852 hash_fn_t sha1; 853 hash_fn_t sha224; 854 hash_fn_t sha256; 855 hash_fn_t sha384; 856 hash_fn_t sha512; 857 aes_cfb_t aes128_cfb_one; 858 859 aes_gcm_enc_dec_t gcm128_enc; 860 aes_gcm_enc_dec_t gcm192_enc; 861 aes_gcm_enc_dec_t gcm256_enc; 862 aes_gcm_enc_dec_t gcm128_dec; 863 aes_gcm_enc_dec_t gcm192_dec; 864 aes_gcm_enc_dec_t gcm256_dec; 865 aes_gcm_init_t gcm128_init; 866 aes_gcm_init_t gcm192_init; 867 aes_gcm_init_t gcm256_init; 868 aes_gcm_enc_dec_update_t gcm128_enc_update; 869 aes_gcm_enc_dec_update_t gcm192_enc_update; 870 aes_gcm_enc_dec_update_t gcm256_enc_update; 871 aes_gcm_enc_dec_update_t gcm128_dec_update; 872 aes_gcm_enc_dec_update_t gcm192_dec_update; 873 aes_gcm_enc_dec_update_t gcm256_dec_update; 874 aes_gcm_enc_dec_finalize_t gcm128_enc_finalize; 875 aes_gcm_enc_dec_finalize_t gcm192_enc_finalize; 876 aes_gcm_enc_dec_finalize_t gcm256_enc_finalize; 877 aes_gcm_enc_dec_finalize_t gcm128_dec_finalize; 878 aes_gcm_enc_dec_finalize_t gcm192_dec_finalize; 879 aes_gcm_enc_dec_finalize_t gcm256_dec_finalize; 880 aes_gcm_precomp_t gcm128_precomp; 881 aes_gcm_precomp_t gcm192_precomp; 882 aes_gcm_precomp_t gcm256_precomp; 883 aes_gcm_pre_t gcm128_pre; 884 aes_gcm_pre_t gcm192_pre; 885 aes_gcm_pre_t gcm256_pre; 886 887 zuc_eea3_1_buffer_t eea3_1_buffer; 888 zuc_eea3_4_buffer_t eea3_4_buffer; 889 zuc_eea3_n_buffer_t eea3_n_buffer; 890 zuc_eia3_1_buffer_t eia3_1_buffer; 891 892 kasumi_f8_1_buffer_t f8_1_buffer; 893 kasumi_f8_1_buffer_bit_t f8_1_buffer_bit; 894 kasumi_f8_2_buffer_t f8_2_buffer; 895 kasumi_f8_3_buffer_t f8_3_buffer; 896 kasumi_f8_4_buffer_t f8_4_buffer; 897 kasumi_f8_n_buffer_t f8_n_buffer; 898 kasumi_f9_1_buffer_t f9_1_buffer; 899 kasumi_f9_1_buffer_user_t f9_1_buffer_user; 900 kasumi_init_f8_key_sched_t kasumi_init_f8_key_sched; 901 kasumi_init_f9_key_sched_t kasumi_init_f9_key_sched; 902 kasumi_key_sched_size_t kasumi_key_sched_size; 903 904 snow3g_f8_1_buffer_bit_t snow3g_f8_1_buffer_bit; 905 snow3g_f8_1_buffer_t snow3g_f8_1_buffer; 906 snow3g_f8_2_buffer_t snow3g_f8_2_buffer; 907 snow3g_f8_4_buffer_t snow3g_f8_4_buffer; 908 snow3g_f8_8_buffer_t snow3g_f8_8_buffer; 909 snow3g_f8_n_buffer_t snow3g_f8_n_buffer; 910 snow3g_f8_8_buffer_multikey_t snow3g_f8_8_buffer_multikey; 911 snow3g_f8_n_buffer_multikey_t snow3g_f8_n_buffer_multikey; 912 snow3g_f9_1_buffer_t snow3g_f9_1_buffer; 913 snow3g_init_key_sched_t snow3g_init_key_sched; 914 snow3g_key_sched_size_t snow3g_key_sched_size; 915 916 ghash_t ghash; 917 zuc_eia3_n_buffer_t eia3_n_buffer; 918 aes_gcm_init_var_iv_t gcm128_init_var_iv; 919 aes_gcm_init_var_iv_t gcm192_init_var_iv; 920 aes_gcm_init_var_iv_t gcm256_init_var_iv; 921 922 aes_gmac_init_t gmac128_init; 923 aes_gmac_init_t gmac192_init; 924 aes_gmac_init_t gmac256_init; 925 aes_gmac_update_t gmac128_update; 926 aes_gmac_update_t gmac192_update; 927 aes_gmac_update_t gmac256_update; 928 aes_gmac_finalize_t gmac128_finalize; 929 aes_gmac_finalize_t gmac192_finalize; 930 aes_gmac_finalize_t gmac256_finalize; 931 hec_32_t hec_32; 932 hec_64_t hec_64; 933 cmac_subkey_gen_t cmac_subkey_gen_256; 934 aes_gcm_pre_t ghash_pre; 935 crc32_fn_t crc32_ethernet_fcs; 936 crc32_fn_t crc16_x25; 937 crc32_fn_t crc32_sctp; 938 crc32_fn_t crc24_lte_a; 939 crc32_fn_t crc24_lte_b; 940 crc32_fn_t crc16_fp_data; 941 crc32_fn_t crc11_fp_header; 942 crc32_fn_t crc7_fp_header; 943 crc32_fn_t crc10_iuup_data; 944 crc32_fn_t crc6_iuup_header; 945 crc32_fn_t crc32_wimax_ofdma_data; 946 crc32_fn_t crc8_wimax_ofdma_hcs; 947 948 /* in-order scheduler fields */ 949 int earliest_job; /* byte offset, -1 if none */ 950 int next_job; /* byte offset */ 951 IMB_JOB jobs[MAX_JOBS]; 952 953 /* out of order managers */ 954 void *aes128_ooo; 955 void *aes192_ooo; 956 void *aes256_ooo; 957 void *docsis128_sec_ooo; 958 void *docsis128_crc32_sec_ooo; 959 void *docsis256_sec_ooo; 960 void *docsis256_crc32_sec_ooo; 961 void *des_enc_ooo; 962 void *des_dec_ooo; 963 void *des3_enc_ooo; 964 void *des3_dec_ooo; 965 void *docsis_des_enc_ooo; 966 void *docsis_des_dec_ooo; 967 968 void *hmac_sha_1_ooo; 969 void *hmac_sha_224_ooo; 970 void *hmac_sha_256_ooo; 971 void *hmac_sha_384_ooo; 972 void *hmac_sha_512_ooo; 973 void *hmac_md5_ooo; 974 void *aes_xcbc_ooo; 975 void *aes_ccm_ooo; 976 void *aes_cmac_ooo; 977 void *zuc_eea3_ooo; 978 void *zuc_eia3_ooo; 979 void *aes128_cbcs_ooo; 980 } IMB_MGR; 981 982 /* ========================================================================== */ 983 /* API definitions */ 984 985 /** 986 * @brief Get library version in string format 987 * 988 * @return library version string 989 */ 990 IMB_DLL_EXPORT const char *imb_get_version_str(void); 991 992 /** 993 * @brief Get library version in numerical format 994 * 995 * Use IMB_VERSION() macro to compare this 996 * numerical version against known library version. 997 * 998 * @return library version number 999 */ 1000 IMB_DLL_EXPORT unsigned imb_get_version(void); 1001 1002 1003 /** 1004 * @brief API to get error status 1005 * 1006 * @param mb_mgr Pointer to multi-buffer manager 1007 * 1008 * @retval Integer error type 1009 */ 1010 IMB_DLL_EXPORT int imb_get_errno(IMB_MGR *mb_mgr); 1011 1012 /** 1013 * @brief API to get description for \a errnum 1014 * 1015 * @param errnum error type 1016 * 1017 * @retval String description of \a errnum 1018 */ 1019 IMB_DLL_EXPORT const char *imb_get_strerror(int errnum); 1020 1021 /* 1022 * get_next_job returns a job object. This must be filled in and returned 1023 * via submit_job before get_next_job is called again. 1024 * After submit_job is called, one should call get_completed_job() at least 1025 * once (and preferably until it returns NULL). 1026 * get_completed_job and flush_job returns a job object. This job object ceases 1027 * to be usable at the next call to get_next_job 1028 */ 1029 IMB_DLL_EXPORT IMB_MGR *alloc_mb_mgr(uint64_t flags); 1030 IMB_DLL_EXPORT void free_mb_mgr(IMB_MGR *state); 1031 1032 IMB_DLL_EXPORT void init_mb_mgr_avx(IMB_MGR *state); 1033 IMB_DLL_EXPORT IMB_JOB *submit_job_avx(IMB_MGR *state); 1034 IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx(IMB_MGR *state); 1035 IMB_DLL_EXPORT IMB_JOB *flush_job_avx(IMB_MGR *state); 1036 IMB_DLL_EXPORT uint32_t queue_size_avx(IMB_MGR *state); 1037 IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx(IMB_MGR *state); 1038 IMB_DLL_EXPORT IMB_JOB *get_next_job_avx(IMB_MGR *state); 1039 1040 IMB_DLL_EXPORT void init_mb_mgr_avx2(IMB_MGR *state); 1041 IMB_DLL_EXPORT IMB_JOB *submit_job_avx2(IMB_MGR *state); 1042 IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2(IMB_MGR *state); 1043 IMB_DLL_EXPORT IMB_JOB *flush_job_avx2(IMB_MGR *state); 1044 IMB_DLL_EXPORT uint32_t queue_size_avx2(IMB_MGR *state); 1045 IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2(IMB_MGR *state); 1046 IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2(IMB_MGR *state); 1047 1048 IMB_DLL_EXPORT void init_mb_mgr_avx512(IMB_MGR *state); 1049 IMB_DLL_EXPORT IMB_JOB *submit_job_avx512(IMB_MGR *state); 1050 IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx512(IMB_MGR *state); 1051 IMB_DLL_EXPORT IMB_JOB *flush_job_avx512(IMB_MGR *state); 1052 IMB_DLL_EXPORT uint32_t queue_size_avx512(IMB_MGR *state); 1053 IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx512(IMB_MGR *state); 1054 IMB_DLL_EXPORT IMB_JOB *get_next_job_avx512(IMB_MGR *state); 1055 1056 IMB_DLL_EXPORT void init_mb_mgr_sse(IMB_MGR *state); 1057 IMB_DLL_EXPORT IMB_JOB *submit_job_sse(IMB_MGR *state); 1058 IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse(IMB_MGR *state); 1059 IMB_DLL_EXPORT IMB_JOB *flush_job_sse(IMB_MGR *state); 1060 IMB_DLL_EXPORT uint32_t queue_size_sse(IMB_MGR *state); 1061 IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse(IMB_MGR *state); 1062 IMB_DLL_EXPORT IMB_JOB *get_next_job_sse(IMB_MGR *state); 1063 1064 /** 1065 * @brief Automatically initialize most performant 1066 * Multi-buffer manager based on CPU features 1067 * 1068 * @param [in] state Pointer to MB_MGR struct 1069 * @param [out] arch Pointer to arch enum to be set (can be NULL) 1070 */ 1071 IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); 1072 1073 /* 1074 * Wrapper macros to call arch API's set up 1075 * at init phase of multi-buffer manager. 1076 * 1077 * For example, after calling init_mb_mgr_sse(&mgr) 1078 * The 'mgr' structure be set up so that: 1079 * mgr.get_next_job will point to get_next_job_sse(), 1080 * mgr.submit_job will point to submit_job_sse(), 1081 * mgr.submit_job_nocheck will point to submit_job_nocheck_sse(), 1082 * mgr.get_completed_job will point to get_completed_job_sse(), 1083 * mgr.flush_job will point to flush_job_sse(), 1084 * mgr.queue_size will point to queue_size_sse() 1085 * mgr.keyexp_128 will point to aes_keyexp_128_sse() 1086 * mgr.keyexp_192 will point to aes_keyexp_192_sse() 1087 * mgr.keyexp_256 will point to aes_keyexp_256_sse() 1088 * etc. 1089 * 1090 * Direct use of arch API's may result in better performance. 1091 * Using below indirect interface may produce slightly worse performance but 1092 * it can simplify application implementation. 1093 * The test app provides example of using the indirect interface. 1094 */ 1095 #define IMB_GET_NEXT_JOB(_mgr) ((_mgr)->get_next_job((_mgr))) 1096 #define IMB_SUBMIT_JOB(_mgr) ((_mgr)->submit_job((_mgr))) 1097 #define IMB_SUBMIT_JOB_NOCHECK(_mgr) ((_mgr)->submit_job_nocheck((_mgr))) 1098 #define IMB_GET_COMPLETED_JOB(_mgr) ((_mgr)->get_completed_job((_mgr))) 1099 #define IMB_FLUSH_JOB(_mgr) ((_mgr)->flush_job((_mgr))) 1100 #define IMB_QUEUE_SIZE(_mgr) ((_mgr)->queue_size((_mgr))) 1101 1102 /* Key expansion and generation API's */ 1103 #define IMB_AES_KEYEXP_128(_mgr, _raw, _enc, _dec) \ 1104 ((_mgr)->keyexp_128((_raw), (_enc), (_dec))) 1105 #define IMB_AES_KEYEXP_192(_mgr, _raw, _enc, _dec) \ 1106 ((_mgr)->keyexp_192((_raw), (_enc), (_dec))) 1107 #define IMB_AES_KEYEXP_256(_mgr, _raw, _enc, _dec) \ 1108 ((_mgr)->keyexp_256((_raw), (_enc), (_dec))) 1109 1110 #define IMB_AES_CMAC_SUBKEY_GEN_128(_mgr, _key_exp, _k1, _k2) \ 1111 ((_mgr)->cmac_subkey_gen_128((_key_exp), (_k1), (_k2))) 1112 1113 #define IMB_AES_CMAC_SUBKEY_GEN_256(_mgr, _key_exp, _k1, _k2) \ 1114 ((_mgr)->cmac_subkey_gen_256((_key_exp), (_k1), (_k2))) 1115 1116 #define IMB_AES_XCBC_KEYEXP(_mgr, _key, _k1_exp, _k2, _k3) \ 1117 ((_mgr)->xcbc_keyexp((_key), (_k1_exp), (_k2), (_k3))) 1118 1119 #define IMB_DES_KEYSCHED(_mgr, _ks, _key) \ 1120 ((_mgr)->des_key_sched((_ks), (_key))) 1121 1122 /* Hash API's */ 1123 #define IMB_SHA1_ONE_BLOCK(_mgr, _data, _digest) \ 1124 ((_mgr)->sha1_one_block((_data), (_digest))) 1125 #define IMB_SHA1(_mgr, _data, _length, _digest) \ 1126 ((_mgr)->sha1((_data), (_length), (_digest))) 1127 #define IMB_SHA224_ONE_BLOCK(_mgr, _data, _digest) \ 1128 ((_mgr)->sha224_one_block((_data), (_digest))) 1129 #define IMB_SHA224(_mgr, _data, _length, _digest) \ 1130 ((_mgr)->sha224((_data), (_length), (_digest))) 1131 #define IMB_SHA256_ONE_BLOCK(_mgr, _data, _digest) \ 1132 ((_mgr)->sha256_one_block((_data), (_digest))) 1133 #define IMB_SHA256(_mgr, _data, _length, _digest) \ 1134 ((_mgr)->sha256((_data), (_length), (_digest))) 1135 #define IMB_SHA384_ONE_BLOCK(_mgr, _data, _digest) \ 1136 ((_mgr)->sha384_one_block((_data), (_digest))) 1137 #define IMB_SHA384(_mgr, _data, _length, _digest) \ 1138 ((_mgr)->sha384((_data), (_length), (_digest))) 1139 #define IMB_SHA512_ONE_BLOCK(_mgr, _data, _digest) \ 1140 ((_mgr)->sha512_one_block((_data), (_digest))) 1141 #define IMB_SHA512(_mgr, _data, _length, _digest) \ 1142 ((_mgr)->sha512((_data), (_length), (_digest))) 1143 #define IMB_MD5_ONE_BLOCK(_mgr, _data, _digest) \ 1144 ((_mgr)->md5_one_block((_data), (_digest))) 1145 1146 /* AES-CFB API */ 1147 #define IMB_AES128_CFB_ONE(_mgr, _out, _in, _iv, _enc, _len) \ 1148 ((_mgr)->aes128_cfb_one((_out), (_in), (_iv), (_enc), (_len))) 1149 1150 /* AES-GCM API's */ 1151 #define IMB_AES128_GCM_ENC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\ 1152 _tag, _tagl) \ 1153 ((_mgr)->gcm128_enc((_key), (_ctx), (_out), (_in), (_len), (_iv), \ 1154 (_aad), (_aadl), (_tag), (_tagl))) 1155 #define IMB_AES192_GCM_ENC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\ 1156 _tag, _tagl) \ 1157 ((_mgr)->gcm192_enc((_key), (_ctx), (_out), (_in), (_len), (_iv), \ 1158 (_aad), (_aadl), (_tag), (_tagl))) 1159 #define IMB_AES256_GCM_ENC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\ 1160 _tag, _tagl) \ 1161 ((_mgr)->gcm256_enc((_key), (_ctx), (_out), (_in), (_len), (_iv), \ 1162 (_aad), (_aadl), (_tag), (_tagl))) 1163 1164 #define IMB_AES128_GCM_DEC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\ 1165 _tag, _tagl) \ 1166 ((_mgr)->gcm128_dec((_key), (_ctx), (_out), (_in), (_len), (_iv), \ 1167 (_aad), (_aadl), (_tag), (_tagl))) 1168 #define IMB_AES192_GCM_DEC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\ 1169 _tag, _tagl) \ 1170 ((_mgr)->gcm192_dec((_key), (_ctx), (_out), (_in), (_len), (_iv), \ 1171 (_aad), (_aadl), (_tag), (_tagl))) 1172 #define IMB_AES256_GCM_DEC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\ 1173 _tag, _tagl) \ 1174 ((_mgr)->gcm256_dec((_key), (_ctx), (_out), (_in), (_len), (_iv), \ 1175 (_aad), (_aadl), (_tag), (_tagl))) 1176 1177 #define IMB_AES128_GCM_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl) \ 1178 ((_mgr)->gcm128_init((_key), (_ctx), (_iv), (_aad), (_aadl))) 1179 #define IMB_AES192_GCM_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl) \ 1180 ((_mgr)->gcm192_init((_key), (_ctx), (_iv), (_aad), (_aadl))) 1181 #define IMB_AES256_GCM_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl) \ 1182 ((_mgr)->gcm256_init((_key), (_ctx), (_iv), (_aad), (_aadl))) 1183 1184 #define IMB_AES128_GCM_INIT_VAR_IV(_mgr, _key, _ctx, _iv, _ivl, _aad, _aadl) \ 1185 ((_mgr)->gcm128_init_var_iv((_key), (_ctx), (_iv), (_ivl), \ 1186 (_aad), (_aadl))) 1187 #define IMB_AES192_GCM_INIT_VAR_IV(_mgr, _key, _ctx, _iv, _ivl, _aad, _aadl) \ 1188 ((_mgr)->gcm192_init_var_iv((_key), (_ctx), (_iv), (_ivl), \ 1189 (_aad), (_aadl))) 1190 #define IMB_AES256_GCM_INIT_VAR_IV(_mgr, _key, _ctx, _iv, _ivl, _aad, _aadl) \ 1191 ((_mgr)->gcm256_init_var_iv((_key), (_ctx), (_iv), (_ivl), \ 1192 (_aad), (_aadl))) 1193 1194 #define IMB_AES128_GCM_ENC_UPDATE(_mgr, _key, _ctx, _out, _in, _len) \ 1195 ((_mgr)->gcm128_enc_update((_key), (_ctx), (_out), (_in), (_len))) 1196 #define IMB_AES192_GCM_ENC_UPDATE(_mgr, _key, _ctx, _out, _in, _len) \ 1197 ((_mgr)->gcm192_enc_update((_key), (_ctx), (_out), (_in), (_len))) 1198 #define IMB_AES256_GCM_ENC_UPDATE(_mgr, _key, _ctx, _out, _in, _len) \ 1199 ((_mgr)->gcm256_enc_update((_key), (_ctx), (_out), (_in), (_len))) 1200 1201 #define IMB_AES128_GCM_DEC_UPDATE(_mgr, _key, _ctx, _out, _in, _len) \ 1202 ((_mgr)->gcm128_dec_update((_key), (_ctx), (_out), (_in), (_len))) 1203 #define IMB_AES192_GCM_DEC_UPDATE(_mgr, _key, _ctx, _out, _in, _len) \ 1204 ((_mgr)->gcm192_dec_update((_key), (_ctx), (_out), (_in), (_len))) 1205 #define IMB_AES256_GCM_DEC_UPDATE(_mgr, _key, _ctx, _out, _in, _len) \ 1206 ((_mgr)->gcm256_dec_update((_key), (_ctx), (_out), (_in), (_len))) 1207 1208 #define IMB_AES128_GCM_ENC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1209 ((_mgr)->gcm128_enc_finalize((_key), (_ctx), (_tag), (_tagl))) 1210 #define IMB_AES192_GCM_ENC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1211 ((_mgr)->gcm192_enc_finalize((_key), (_ctx), (_tag), (_tagl))) 1212 #define IMB_AES256_GCM_ENC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1213 ((_mgr)->gcm256_enc_finalize((_key), (_ctx), (_tag), (_tagl))) 1214 1215 #define IMB_AES128_GCM_DEC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1216 ((_mgr)->gcm128_dec_finalize((_key), (_ctx), (_tag), (_tagl))) 1217 #define IMB_AES192_GCM_DEC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1218 ((_mgr)->gcm192_dec_finalize((_key), (_ctx), (_tag), (_tagl))) 1219 #define IMB_AES256_GCM_DEC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1220 ((_mgr)->gcm256_dec_finalize((_key), (_ctx), (_tag), (_tagl))) 1221 1222 #define IMB_AES128_GMAC_INIT(_mgr, _key, _ctx, _iv, _ivl) \ 1223 ((_mgr)->gmac128_init((_key), (_ctx), (_iv), (_ivl))) 1224 #define IMB_AES192_GMAC_INIT(_mgr, _key, _ctx, _iv, _ivl) \ 1225 ((_mgr)->gmac192_init((_key), (_ctx), (_iv), (_ivl))) 1226 #define IMB_AES256_GMAC_INIT(_mgr, _key, _ctx, _iv, _ivl) \ 1227 ((_mgr)->gmac256_init((_key), (_ctx), (_iv), (_ivl))) 1228 1229 #define IMB_AES128_GMAC_UPDATE(_mgr, _key, _ctx, _in, _len) \ 1230 ((_mgr)->gmac128_update((_key), (_ctx), (_in), (_len))) 1231 #define IMB_AES192_GMAC_UPDATE(_mgr, _key, _ctx, _in, _len) \ 1232 ((_mgr)->gmac192_update((_key), (_ctx), (_in), (_len))) 1233 #define IMB_AES256_GMAC_UPDATE(_mgr, _key, _ctx, _in, _len) \ 1234 ((_mgr)->gmac256_update((_key), (_ctx), (_in), (_len))) 1235 1236 #define IMB_AES128_GMAC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1237 ((_mgr)->gmac128_finalize((_key), (_ctx), (_tag), (_tagl))) 1238 #define IMB_AES192_GMAC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1239 ((_mgr)->gmac192_finalize((_key), (_ctx), (_tag), (_tagl))) 1240 #define IMB_AES256_GMAC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl) \ 1241 ((_mgr)->gmac256_finalize((_key), (_ctx), (_tag), (_tagl))) 1242 1243 #define IMB_AES128_GCM_PRECOMP(_mgr, _key) \ 1244 ((_mgr)->gcm128_precomp((_key))) 1245 #define IMB_AES192_GCM_PRECOMP(_mgr, _key) \ 1246 ((_mgr)->gcm192_precomp((_key))) 1247 #define IMB_AES256_GCM_PRECOMP(_mgr, _key) \ 1248 ((_mgr)->gcm256_precomp((_key))) 1249 1250 #define IMB_AES128_GCM_PRE(_mgr, _key_in, _key_exp) \ 1251 ((_mgr)->gcm128_pre((_key_in), (_key_exp))) 1252 #define IMB_AES192_GCM_PRE(_mgr, _key_in, _key_exp) \ 1253 ((_mgr)->gcm192_pre((_key_in), (_key_exp))) 1254 #define IMB_AES256_GCM_PRE(_mgr, _key_in, _key_exp) \ 1255 ((_mgr)->gcm256_pre((_key_in), (_key_exp))) 1256 1257 #define IMB_GHASH_PRE(_mgr, _key_in, _key_exp) \ 1258 ((_mgr)->ghash_pre((_key_in), (_key_exp))) 1259 #define IMB_GHASH(_mgr, _key, _in, _in_len, _out, _out_len) \ 1260 ((_mgr)->ghash((_key), (_in), (_in_len), (_out), (_out_len))) 1261 1262 /* ZUC EEA3/EIA3 functions */ 1263 1264 /** 1265 * @brief ZUC EEA3 Confidentiality functions 1266 * 1267 * @param mgr Pointer to multi-buffer structure 1268 * @param key Pointer to key 1269 * @param iv Pointer to 16-byte IV 1270 * @param in Pointer to Plaintext/Ciphertext input. 1271 * @param out Pointer to Ciphertext/Plaintext output. 1272 * @param len Length of input data in bytes. 1273 */ 1274 #define IMB_ZUC_EEA3_1_BUFFER(_mgr, _key, _iv, _in, _out, _len) \ 1275 ((_mgr)->eea3_1_buffer((_key), (_iv), (_in), (_out), (_len))) 1276 #define IMB_ZUC_EEA3_4_BUFFER(_mgr, _key, _iv, _in, _out, _len) \ 1277 ((_mgr)->eea3_4_buffer((_key), (_iv), (_in), (_out), (_len))) 1278 #define IMB_ZUC_EEA3_N_BUFFER(_mgr, _key, _iv, _in, _out, _len, _num) \ 1279 ((_mgr)->eea3_n_buffer((_key), (_iv), (_in), (_out), (_len), (_num))) 1280 1281 1282 /** 1283 * @brief ZUC EIA3 Integrity function 1284 * 1285 * @param mgr Pointer to multi-buffer structure 1286 * @param key Pointer to key 1287 * @param iv Pointer to 16-byte IV 1288 * @param in Pointer to Plaintext/Ciphertext input. 1289 * @param len Length of input data in bits. 1290 * @param tag Pointer to Authenticated Tag output (4 bytes) 1291 */ 1292 #define IMB_ZUC_EIA3_1_BUFFER(_mgr, _key, _iv, _in, _len, _tag) \ 1293 ((_mgr)->eia3_1_buffer((_key), (_iv), (_in), (_len), (_tag))) 1294 #define IMB_ZUC_EIA3_N_BUFFER(_mgr, _key, _iv, _in, _len, _tag, _num) \ 1295 ((_mgr)->eia3_n_buffer((_key), (_iv), (_in), (_len), (_tag), (_num))) 1296 1297 1298 /* KASUMI F8/F9 functions */ 1299 1300 /** 1301 * @brief Kasumi byte-level f8 operation on a single buffer 1302 * 1303 * This function performs kasumi f8 operation on a single buffer. The key has 1304 * already been scheduled with kasumi_init_f8_key_sched(). 1305 * No extra bits are modified. 1306 * 1307 * @param [in] ctx Context where the scheduled keys are stored 1308 * @param [in] iv Initialization vector 1309 * @param [in] in Input buffer 1310 * @param [out] out Output buffer 1311 * @param [in] len Length in BYTES 1312 * 1313 ******************************************************************************/ 1314 #define IMB_KASUMI_F8_1_BUFFER(_mgr, _ctx, _iv, _in, _out, _len) \ 1315 ((_mgr)->f8_1_buffer((_ctx), (_iv), (_in), (_out), (_len))) 1316 1317 /** 1318 * @brief Kasumi bit-level f8 operation on a single buffer 1319 * 1320 * This function performs kasumi f8 operation on a single buffer. The key has 1321 * already been scheduled with kasumi_init_f8_key_sched(). 1322 * No extra bits are modified. 1323 * 1324 * @param [in] ctx Context where the scheduled keys are stored 1325 * @param [in] iv Initialization vector 1326 * @param [in] in Input buffer 1327 * @param [out] out Output buffer 1328 * @param [in] len Length in BITS 1329 * @param [in] offset Offset in BITS from begin of input buffer 1330 * 1331 ******************************************************************************/ 1332 #define IMB_KASUMI_F8_1_BUFFER_BIT(_mgr, _ctx, _iv, _in, _out, _len, _offset) \ 1333 ((_mgr)->f8_1_buffer_bit((_ctx), (_iv), (_in), (_out), (_len), \ 1334 (_offset))) 1335 1336 /** 1337 * @brief Kasumi byte-level f8 operation in parallel on two buffers 1338 * 1339 * This function performs kasumi f8 operation on a two buffers. 1340 * They will be processed with the same key, which has already been scheduled 1341 * with kasumi_init_f8_key_sched(). 1342 * 1343 * @param [in] ctx Context where the scheduled keys are stored 1344 * @param [in] iv1 Initialization vector for buffer in1 1345 * @param [in] iv2 Initialization vector for buffer in2 1346 * @param [in] in1 Input buffer 1 1347 * @param [out] out1 Output buffer 1 1348 * @param [in] len1 Length in BYTES of input buffer 1 1349 * @param [in] in2 Input buffer 2 1350 * @param [out] out2 Output buffer 2 1351 * @param [in] len2 Length in BYTES of input buffer 2 1352 * 1353 ******************************************************************************/ 1354 #define IMB_KASUMI_F8_2_BUFFER(_mgr, _ctx, _iv1, _iv2, _in1, _out1, _len1, \ 1355 _in2, _out2, _len2) \ 1356 ((_mgr)->f8_2_buffer((_ctx), (_iv1), (_iv2), (_in1), (_out1), (_len1), \ 1357 (_in2), (_out2), (_len2))) 1358 /** 1359 * @brief kasumi byte-level f8 operation in parallel on three buffers 1360 * 1361 * This function performs kasumi f8 operation on a three buffers. 1362 * They must all have the same length and they will be processed with the same 1363 * key, which has already been scheduled with kasumi_init_f8_key_sched(). 1364 * 1365 * @param [in] ctx Context where the scheduled keys are stored 1366 * @param [in] iv1 Initialization vector for buffer in1 1367 * @param [in] iv2 Initialization vector for buffer in2 1368 * @param [in] iv3 Initialization vector for buffer in3 1369 * @param [in] in1 Input buffer 1 1370 * @param [out] out1 Output buffer 1 1371 * @param [in] in2 Input buffer 2 1372 * @param [out] out2 Output buffer 2 1373 * @param [in] in3 Input buffer 3 1374 * @param [out] out3 Output buffer 3 1375 * @param [in] len Common length in bytes for all buffers 1376 * 1377 ******************************************************************************/ 1378 #define IMB_KASUMI_F8_3_BUFFER(_mgr, _ctx, _iv1, _iv2, _iv3, _in1, _out1, \ 1379 _in2, _out2, _in3, _out3, _len) \ 1380 ((_mgr)->f8_3_buffer((_ctx), (_iv1), (_iv2), (_iv3), (_in1), (_out1), \ 1381 (_in2), (_out2), (_in3), (_out3), (_len))) 1382 /** 1383 * @brief kasumi byte-level f8 operation in parallel on four buffers 1384 * 1385 * This function performs kasumi f8 operation on four buffers. 1386 * They must all have the same length and they will be processed with the same 1387 * key, which has already been scheduled with kasumi_init_f8_key_sched(). 1388 * 1389 * @param [in] ctx Context where the scheduled keys are stored 1390 * @param [in] iv1 Initialization vector for buffer in1 1391 * @param [in] iv2 Initialization vector for buffer in2 1392 * @param [in] iv3 Initialization vector for buffer in3 1393 * @param [in] iv4 Initialization vector for buffer in4 1394 * @param [in] in1 Input buffer 1 1395 * @param [out] out1 Output buffer 1 1396 * @param [in] in2 Input buffer 2 1397 * @param [out] out2 Output buffer 2 1398 * @param [in] in3 Input buffer 3 1399 * @param [out] out3 Output buffer 3 1400 * @param [in] in4 Input buffer 4 1401 * @param [out] out4 Output buffer 4 1402 * @param [in] len Common length in bytes for all buffers 1403 * 1404 ******************************************************************************/ 1405 #define IMB_KASUMI_F8_4_BUFFER(_mgr, _ctx, _iv1, _iv2, _iv3, _iv4, \ 1406 _in1, _out1, _in2, _out2, _in3, _out3, \ 1407 _in4, _out4, _len) \ 1408 ((_mgr)->f8_4_buffer((_ctx), (_iv1), (_iv2), (_iv3), (_iv4), \ 1409 (_in1), (_out1), (_in2), (_out2), \ 1410 (_in3), (_out3), (_in4), (_out4), (_len))) 1411 /** 1412 * @brief Kasumi f8 operation on N buffers 1413 * 1414 * All input buffers can have different lengths and they will be processed 1415 * with the same key, which has already been scheduled 1416 * with kasumi_init_f8_key_sched(). 1417 * 1418 * @param [in] ctx Context where the scheduled keys are stored 1419 * @param [in] iv Array of IV values 1420 * @param [in] in Array of input buffers 1421 * @param [out] out Array of output buffers 1422 * @param [in] len Array of corresponding input buffer lengths in BITS 1423 * @param [in] count Number of input buffers 1424 */ 1425 #define IMB_KASUMI_F8_N_BUFFER(_mgr, _ctx, _iv, _in, _out, _len, _count) \ 1426 ((_mgr)->f8_n_buffer((_ctx), (_iv), (_in), (_out), (_len), \ 1427 (_count))) 1428 /** 1429 * @brief Kasumi bit-level f9 operation on a single buffer. 1430 * 1431 * The first QWORD of in represents the COUNT and FRESH, the last QWORD 1432 * represents the DIRECTION and PADDING. (See 3GPP TS 35.201 v10.0 section 4) 1433 * 1434 * The key has already been scheduled with kasumi_init_f9_key_sched(). 1435 * 1436 * @param [in] ctx Context where the scheduled keys are stored 1437 * @param [in] in Input buffer 1438 * @param [in] len Length in BYTES of the data to be hashed 1439 * @param [out] tag Computed digest 1440 * 1441 */ 1442 #define IMB_KASUMI_F9_1_BUFFER(_mgr, _ctx, _in, _len, _tag) \ 1443 ((_mgr)->f9_1_buffer((_ctx), (_in), (_len), (_tag))) 1444 1445 /** 1446 * @brief Kasumi bit-level f9 operation on a single buffer. 1447 * 1448 * The key has already been scheduled with kasumi_init_f9_key_sched(). 1449 * 1450 * @param [in] ctx Context where the scheduled keys are stored 1451 * @param [in] iv Initialization vector 1452 * @param [in] in Input buffer 1453 * @param [in] len Length in BITS of the data to be hashed 1454 * @param [out] tag Computed digest 1455 * @param [in] dir Direction bit 1456 * 1457 */ 1458 #define IMB_KASUMI_F9_1_BUFFER_USER(_mgr, _ctx, _iv, _in, _len, _tag, _dir) \ 1459 ((_mgr)->f9_1_buffer_user((_ctx), (_iv), (_in), (_len), \ 1460 (_tag), (_dir))) 1461 1462 /** 1463 * KASUMI F8 key schedule init function. 1464 * 1465 * @param[in] mgr Pointer to multi-buffer structure 1466 * @param[in] key Confidentiality key (expected in LE format) 1467 * @param[out] ctx Key schedule context to be initialised 1468 * @return 0 on success, -1 on failure 1469 * 1470 ******************************************************************************/ 1471 #define IMB_KASUMI_INIT_F8_KEY_SCHED(_mgr, _key, _ctx) \ 1472 ((_mgr)->kasumi_init_f8_key_sched((_key), (_ctx))) 1473 1474 /** 1475 * KASUMI F9 key schedule init function. 1476 * 1477 * @param[in] mgr Pointer to multi-buffer structure 1478 * @param[in] key Integrity key (expected in LE format) 1479 * @param[out] ctx Key schedule context to be initialised 1480 * @return 0 on success, -1 on failure 1481 * 1482 ******************************************************************************/ 1483 #define IMB_KASUMI_INIT_F9_KEY_SCHED(_mgr, _key, _ctx) \ 1484 ((_mgr)->kasumi_init_f9_key_sched((_key), (_ctx))) 1485 1486 /** 1487 ******************************************************************************* 1488 * This function returns the size of the kasumi_key_sched_t, used 1489 * to store the key schedule. 1490 * 1491 * @param[in] mgr Pointer to multi-buffer structure 1492 * @return size of kasumi_key_sched_t type success 1493 * 1494 ******************************************************************************/ 1495 #define IMB_KASUMI_KEY_SCHED_SIZE(_mgr)((_mgr)->kasumi_key_sched_size()) 1496 1497 1498 /* SNOW3G F8/F9 functions */ 1499 1500 /** 1501 * This function performs snow3g f8 operation on a single buffer. The key has 1502 * already been scheduled with snow3g_init_key_sched(). 1503 * 1504 * @param[in] mgr Pointer to multi-buffer structure 1505 * @param[in] ctx Context where the scheduled keys are stored 1506 * @param[in] iv iv[3] = count 1507 * iv[2] = (bearer << 27) | ((dir & 0x1) << 26) 1508 * iv[1] = pIV[3] 1509 * iv[0] = pIV[2] 1510 * @param[in] in Input buffer 1511 * @param[out] out Output buffer 1512 * @param[in] len Length in bits of input buffer 1513 * @param[in] offset Offset in input/output buffer (in bits) 1514 */ 1515 #define IMB_SNOW3G_F8_1_BUFFER_BIT(_mgr, _ctx, _iv, _in, _out, _len, _offset) \ 1516 ((_mgr)->snow3g_f8_1_buffer_bit((_ctx), (_iv), (_in), \ 1517 (_out), (_len), (_offset))) 1518 1519 /** 1520 * This function performs snow3g f8 operation on a single buffer. The key has 1521 * already been scheduled with snow3g_init_key_sched(). 1522 * 1523 * @param[in] mgr Pointer to multi-buffer structure 1524 * @param[in] ctx Context where the scheduled keys are stored 1525 * @param[in] iv iv[3] = count 1526 * iv[2] = (bearer << 27) | ((dir & 0x1) << 26) 1527 * iv[1] = pIV[3] 1528 * iv[0] = pIV[2] 1529 * @param[in] in Input buffer 1530 * @param[out] out Output buffer 1531 * @param[in] len Length in bits of input buffer 1532 */ 1533 #define IMB_SNOW3G_F8_1_BUFFER(_mgr, _ctx, _iv, _in, _out, _len) \ 1534 ((_mgr)->snow3g_f8_1_buffer((_ctx), (_iv), (_in), (_out), (_len))) 1535 1536 /** 1537 * This function performs snow3g f8 operation on two buffers. They will 1538 * be processed with the same key, which has already been scheduled with 1539 * snow3g_init_key_sched(). 1540 * 1541 * @param[in] mgr Pointer to multi-buffer structure 1542 * @param[in] ctx Context where the scheduled keys are stored 1543 * @param[in] iv1 IV to use for buffer pBufferIn1 1544 * @param[in] iv2 IV to use for buffer pBufferIn2 1545 * @param[in] in1 Input buffer 1 1546 * @param[out] out1 Output buffer 1 1547 * @param[in] len1 Length in bytes of input buffer 1 1548 * @param[in] in2 Input buffer 2 1549 * @param[out] out2 Output buffer 2 1550 * @param[in] len2 Length in bytes of input buffer 2 1551 */ 1552 #define IMB_SNOW3G_F8_2_BUFFER(_mgr, _ctx, _iv1, _iv2, \ 1553 _in1, _out1, _len1, \ 1554 _in2, _out2, _len2) \ 1555 ((_mgr)->snow3g_f8_2_buffer((_ctx), (_iv1), (_iv2), \ 1556 (_in1), (_out1), (_len1), \ 1557 (_in2), (_out2), (_len2))) 1558 1559 /** 1560 ******************************************************************************* 1561 * This function performs snow3g f8 operation on four buffers. They will 1562 * be processed with the same key, which has already been scheduled with 1563 * snow3g_init_key_sched(). 1564 * 1565 * @param[in] mgr Pointer to multi-buffer structure 1566 * @param[in] ctx Context where the scheduled keys are stored 1567 * @param[in] iv1 IV to use for buffer pBufferIn1 1568 * @param[in] iv2 IV to use for buffer pBufferIn2 1569 * @param[in] iv3 IV to use for buffer pBufferIn3 1570 * @param[in] iv4 IV to use for buffer pBufferIn4 1571 * @param[in] in1 Input buffer 1 1572 * @param[out] out1 Output buffer 1 1573 * @param[in] len1 Length in bytes of input buffer 1 1574 * @param[in] in2 Input buffer 2 1575 * @param[out] out2 Output buffer 2 1576 * @param[in] len2 Length in bytes of input buffer 2 1577 * @param[in] in3 Input buffer 3 1578 * @param[out] out3 Output buffer 3 1579 * @param[in] len3 Length in bytes of input buffer 3 1580 * @param[in] in4 Input buffer 4 1581 * @param[out] out4 Output buffer 4 1582 * @param[in] len4 Length in bytes of input buffer 4 1583 */ 1584 #define IMB_SNOW3G_F8_4_BUFFER(_mgr, _ctx, _iv1, _iv2, _iv3, _iv4, \ 1585 _in1, _out1, _len1, \ 1586 _in2, _out2, _len2, \ 1587 _in3, _out3, _len3, \ 1588 _in4, _out4, _len4) \ 1589 ((_mgr)->snow3g_f8_4_buffer((_ctx), (_iv1), (_iv2), (_iv3), (_iv4), \ 1590 (_in1), (_out1), (_len1), \ 1591 (_in2), (_out2), (_len2), \ 1592 (_in3), (_out3), (_len3), \ 1593 (_in4), (_out4), (_len4))) 1594 1595 /** 1596 ******************************************************************************* 1597 * This function performs snow3g f8 operation on eight buffers. They will 1598 * be processed with the same key, which has already been scheduled with 1599 * snow3g_init_key_sched(). 1600 * 1601 * @param[in] mgr Pointer to multi-buffer structure 1602 * @param[in] ctx Context where the scheduled keys are stored 1603 * @param[in] iv1 IV to use for buffer pBufferIn1 1604 * @param[in] iv2 IV to use for buffer pBufferIn2 1605 * @param[in] iv3 IV to use for buffer pBufferIn3 1606 * @param[in] iv4 IV to use for buffer pBufferIn4 1607 * @param[in] iv5 IV to use for buffer pBufferIn5 1608 * @param[in] iv6 IV to use for buffer pBufferIn6 1609 * @param[in] iv7 IV to use for buffer pBufferIn7 1610 * @param[in] iv8 IV to use for buffer pBufferIn8 1611 * @param[in] in1 Input buffer 1 1612 * @param[out] out1 Output buffer 1 1613 * @param[in] len1 Length in bytes of input buffer 1 1614 * @param[in] in2 Input buffer 2 1615 * @param[out] out2 Output buffer 2 1616 * @param[in] len2 Length in bytes of input buffer 2 1617 * @param[in] in3 Input buffer 3 1618 * @param[out] out3 Output buffer 3 1619 * @param[in] len3 Length in bytes of input buffer 3 1620 * @param[in] in4 Input buffer 4 1621 * @param[out] out4 Output buffer 4 1622 * @param[in] len4 Length in bytes of input buffer 4 1623 * @param[in] in5 Input buffer 5 1624 * @param[out] out5 Output buffer 5 1625 * @param[in] len5 Length in bytes of input buffer 5 1626 * @param[in] in6 Input buffer 6 1627 * @param[out] out6 Output buffer 6 1628 * @param[in] len6 Length in bytes of input buffer 6 1629 * @param[in] in7 Input buffer 7 1630 * @param[out] out7 Output buffer 7 1631 * @param[in] len7 Length in bytes of input buffer 7 1632 * @param[in] in8 Input buffer 8 1633 * @param[out] out8 Output buffer 8 1634 * @param[in] len8 Length in bytes of input buffer 8 1635 */ 1636 #define IMB_SNOW3G_F8_8_BUFFER(_mgr, _ctx, _iv1, _iv2, _iv3, _iv4, \ 1637 _iv5, _iv6, _iv7, _iv8, \ 1638 _in1, _out1, _len1, \ 1639 _in2, _out2, _len2, \ 1640 _in3, _out3, _len3, \ 1641 _in4, _out4, _len4, \ 1642 _in5, _out5, _len5, \ 1643 _in6, _out6, _len6, \ 1644 _in7, _out7, _len7, \ 1645 _in8, _out8, _len8) \ 1646 ((_mgr)->snow3g_f8_8_buffer((_ctx), (_iv1), (_iv2), (_iv3), (_iv4), \ 1647 (_iv5), (_iv6), (_iv7), (_iv8), \ 1648 (_in1), (_out1), (_len1), \ 1649 (_in2), (_out2), (_len2), \ 1650 (_in3), (_out3), (_len3), \ 1651 (_in4), (_out4), (_len4), \ 1652 (_in5), (_out5), (_len5), \ 1653 (_in6), (_out6), (_len6), \ 1654 (_in7), (_out7), (_len7), \ 1655 (_in8), (_out8), (_len8))) 1656 /** 1657 * This function performs snow3g f8 operation on eight buffers. They will 1658 * be processed with individual keys, which have already been scheduled 1659 * with snow3g_init_key_sched(). 1660 * 1661 * @param[in] mgr Pointer to multi-buffer structure 1662 * @param[in] ctx Array of 8 Contexts, where the scheduled keys are stored 1663 * @param[in] iv Array of 8 IV values 1664 * @param[in] in Array of 8 input buffers 1665 * @param[out] out Array of 8 output buffers 1666 * @param[in] lens Array of 8 corresponding input buffer lengths 1667 */ 1668 #define IMB_SNOW3G_F8_8_BUFFER_MULTIKEY(_mgr, _ctx, _iv, _in, _out, _len) \ 1669 ((_mgr)->snow3g_f8_8_buffer_multikey((_ctx), (_iv), (_in), (_out),\ 1670 (_len))) 1671 1672 /** 1673 * This function performs snow3g f8 operation in parallel on N buffers. All 1674 * input buffers can have different lengths and they will be processed with the 1675 * same key, which has already been scheduled with snow3g_init_key_sched(). 1676 * 1677 * @param[in] mgr Pointer to multi-buffer structure 1678 * @param[in] ctx Context where the scheduled keys are stored 1679 * @param[in] iv Array of IV values 1680 * @param[in] in Array of input buffers 1681 * @param[out] out Array of output buffers - out[0] set to NULL on failure 1682 * @param[in] len Array of corresponding input buffer lengths 1683 * @param[in] count Number of input buffers 1684 * 1685 ******************************************************************************/ 1686 #define IMB_SNOW3G_F8_N_BUFFER(_mgr, _ctx, _iv, _in, _out, _len, _count) \ 1687 ((_mgr)->snow3g_f8_n_buffer((_ctx), (_iv), (_in), \ 1688 (_out), (_len), (_count))) 1689 1690 /** 1691 * This function performs snow3g f8 operation in parallel on N buffers. All 1692 * input buffers can have different lengths. Confidentiallity keys can vary, 1693 * schedules with snow3g_init_key_sched_multi(). 1694 * 1695 * @param[in] mgr Pointer to multi-buffer structure 1696 * @param[in] ctx Array of Contexts, where the scheduled keys are stored 1697 * @param[in] iv Array of IV values 1698 * @param[in] in Array of input buffers 1699 * @param[out] out Array of output buffers 1700 * - out[0] set to NULL on failure 1701 * @param[in] len Array of corresponding input buffer lengths 1702 * @param[in] count Number of input buffers 1703 */ 1704 #define IMB_SNOW3G_F8_N_BUFFER_MULTIKEY(_mgr, _ctx, _iv, _in, \ 1705 _out, _len, _count) \ 1706 ((_mgr)->snow3g_f8_n_buffer_multikey((_ctx), (_iv), (_in), \ 1707 (_out), (_len), (_count))) 1708 1709 /** 1710 * This function performs a snow3g f9 operation on a single block of data. The 1711 * key has already been scheduled with snow3g_init_f8_key_sched(). 1712 * 1713 * @param[in] mgr Pointer to multi-buffer structure 1714 * @param[in] ctx Context where the scheduled keys are stored 1715 * @param[in] iv iv[3] = _BSWAP32(fresh^(dir<<15)) 1716 * iv[2] = _BSWAP32(count^(dir<<31)) 1717 * iv[1] = _BSWAP32(fresh) 1718 * iv[0] = _BSWAP32(count) 1719 * 1720 * @param[in] in Input buffer 1721 * @param[in] len Length in bits of the data to be hashed 1722 * @param[out] digest Computed digest 1723 */ 1724 #define IMB_SNOW3G_F9_1_BUFFER(_mgr, _ctx, _iv, _in, _len, _digest) \ 1725 ((_mgr)->snow3g_f9_1_buffer((_ctx), (_iv), (_in), (_len), (_digest))) 1726 1727 /** 1728 * Snow3g key schedule init function. 1729 * 1730 * @param[in] mgr Pointer to multi-buffer structure 1731 * @param[in] key Confidentiality/Integrity key (expected in LE format) 1732 * @param[out] ctx Key schedule context to be initialised 1733 * @return 0 on success 1734 * @return -1 on error 1735 * 1736 ******************************************************************************/ 1737 #define IMB_SNOW3G_INIT_KEY_SCHED(_mgr, _key, _ctx) \ 1738 ((_mgr)->snow3g_init_key_sched((_key), (_ctx))) 1739 1740 /** 1741 ******************************************************************************* 1742 * This function returns the size of the snow3g_key_schedule_t, used 1743 * to store the key schedule. 1744 * 1745 * @param[in] mgr Pointer to multi-buffer structure 1746 * @return size of snow3g_key_schedule_t type 1747 * 1748 ******************************************************************************/ 1749 #define IMB_SNOW3G_KEY_SCHED_SIZE(_mgr)((_mgr)->snow3g_key_sched_size()) 1750 1751 /* HEC compute functions */ 1752 #define IMB_HEC_32(_mgr, _in)((_mgr)->hec_32(_in)) 1753 #define IMB_HEC_64(_mgr, _in)((_mgr)->hec_64(_in)) 1754 1755 /* CRC32 Ethernet FCS function */ 1756 #define IMB_CRC32_ETHERNET_FCS(_mgr,_in,_len) \ 1757 (_mgr)->crc32_ethernet_fcs(_in,_len) 1758 1759 /* CRC16 X25 function */ 1760 #define IMB_CRC16_X25(_mgr,_in,_len) \ 1761 (_mgr)->crc16_x25(_in,_len) 1762 1763 /* CRC32 SCTP function */ 1764 #define IMB_CRC32_SCTP(_mgr,_in,_len) \ 1765 (_mgr)->crc32_sctp(_in,_len) 1766 1767 /* LTE CRC24A function */ 1768 #define IMB_CRC24_LTE_A(_mgr,_in,_len) \ 1769 (_mgr)->crc24_lte_a(_in,_len) 1770 1771 /* LTE CRC24B function */ 1772 #define IMB_CRC24_LTE_B(_mgr,_in,_len) \ 1773 (_mgr)->crc24_lte_b(_in,_len) 1774 1775 /* Framing Protocol CRC16 function (3GPP TS 25.435, 3GPP TS 25.427) */ 1776 #define IMB_CRC16_FP_DATA(_mgr,_in,_len) \ 1777 (_mgr)->crc16_fp_data(_in,_len) 1778 1779 /* Framing Protocol CRC11 function (3GPP TS 25.435, 3GPP TS 25.427) */ 1780 #define IMB_CRC11_FP_HEADER(_mgr,_in,_len) \ 1781 (_mgr)->crc11_fp_header(_in,_len) 1782 1783 /* Framing Protocol CRC7 function (3GPP TS 25.435, 3GPP TS 25.427) */ 1784 #define IMB_CRC7_FP_HEADER(_mgr,_in,_len) \ 1785 (_mgr)->crc7_fp_header(_in,_len) 1786 1787 /* IUUP CRC10 function (3GPP TS 25.415) */ 1788 #define IMB_CRC10_IUUP_DATA(_mgr,_in,_len) \ 1789 (_mgr)->crc10_iuup_data(_in,_len) 1790 1791 /* IUUP CRC6 function (3GPP TS 25.415) */ 1792 #define IMB_CRC6_IUUP_HEADER(_mgr,_in,_len) \ 1793 (_mgr)->crc6_iuup_header(_in,_len) 1794 1795 /* WIMAX OFDMA DATA CRC32 function (IEEE 802.16) */ 1796 #define IMB_CRC32_WIMAX_OFDMA_DATA(_mgr,_in,_len) \ 1797 (_mgr)->crc32_wimax_ofdma_data(_in,_len) 1798 1799 /* WIMAX OFDMA HCS CRC8 function (IEEE 802.16) */ 1800 #define IMB_CRC8_WIMAX_OFDMA_HCS(_mgr,_in,_len) \ 1801 (_mgr)->crc8_wimax_ofdma_hcs(_in,_len) 1802 1803 /* Auxiliary functions */ 1804 1805 /** 1806 * @brief DES key schedule set up 1807 * 1808 * \a ks buffer needs to accommodate \a DES_KEY_SCHED_SIZE (128) bytes of data. 1809 * 1810 * @param ks destination buffer to accommodate DES key schedule 1811 * @param key a pointer to an 8 byte DES key 1812 * 1813 * @return Operation status 1814 * @retval 0 success 1815 * @retval !0 error 1816 */ 1817 IMB_DLL_EXPORT int 1818 des_key_schedule(uint64_t *ks, const void *key); 1819 1820 /* SSE */ 1821 IMB_DLL_EXPORT void sha1_sse(const void *data, const uint64_t length, 1822 void *digest); 1823 IMB_DLL_EXPORT void sha1_one_block_sse(const void *data, void *digest); 1824 IMB_DLL_EXPORT void sha224_sse(const void *data, const uint64_t length, 1825 void *digest); 1826 IMB_DLL_EXPORT void sha224_one_block_sse(const void *data, void *digest); 1827 IMB_DLL_EXPORT void sha256_sse(const void *data, const uint64_t length, 1828 void *digest); 1829 IMB_DLL_EXPORT void sha256_one_block_sse(const void *data, void *digest); 1830 IMB_DLL_EXPORT void sha384_sse(const void *data, const uint64_t length, 1831 void *digest); 1832 IMB_DLL_EXPORT void sha384_one_block_sse(const void *data, void *digest); 1833 IMB_DLL_EXPORT void sha512_sse(const void *data, const uint64_t length, 1834 void *digest); 1835 IMB_DLL_EXPORT void sha512_one_block_sse(const void *data, void *digest); 1836 IMB_DLL_EXPORT void md5_one_block_sse(const void *data, void *digest); 1837 IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys, 1838 void *dec_exp_keys); 1839 IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys, 1840 void *dec_exp_keys); 1841 IMB_DLL_EXPORT void aes_keyexp_256_sse(const void *key, void *enc_exp_keys, 1842 void *dec_exp_keys); 1843 IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp, 1844 void *k2, void *k3); 1845 IMB_DLL_EXPORT void aes_keyexp_128_enc_sse(const void *key, 1846 void *enc_exp_keys); 1847 IMB_DLL_EXPORT void aes_keyexp_192_enc_sse(const void *key, 1848 void *enc_exp_keys); 1849 IMB_DLL_EXPORT void aes_keyexp_256_enc_sse(const void *key, 1850 void *enc_exp_keys); 1851 IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1, 1852 void *key2); 1853 IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in, 1854 const void *iv, const void *keys, 1855 uint64_t len); 1856 /* AVX */ 1857 IMB_DLL_EXPORT void sha1_avx(const void *data, const uint64_t length, 1858 void *digest); 1859 IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest); 1860 IMB_DLL_EXPORT void sha224_avx(const void *data, const uint64_t length, 1861 void *digest); 1862 IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest); 1863 IMB_DLL_EXPORT void sha256_avx(const void *data, const uint64_t length, 1864 void *digest); 1865 IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest); 1866 IMB_DLL_EXPORT void sha384_avx(const void *data, const uint64_t length, 1867 void *digest); 1868 IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest); 1869 IMB_DLL_EXPORT void sha512_avx(const void *data, const uint64_t length, 1870 void *digest); 1871 IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest); 1872 IMB_DLL_EXPORT void md5_one_block_avx(const void *data, void *digest); 1873 IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys, 1874 void *dec_exp_keys); 1875 IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys, 1876 void *dec_exp_keys); 1877 IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys, 1878 void *dec_exp_keys); 1879 IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp, 1880 void *k2, void *k3); 1881 IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key, 1882 void *enc_exp_keys); 1883 IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key, 1884 void *enc_exp_keys); 1885 IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key, 1886 void *enc_exp_keys); 1887 IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1, 1888 void *key2); 1889 IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in, 1890 const void *iv, const void *keys, 1891 uint64_t len); 1892 /* AVX2 */ 1893 IMB_DLL_EXPORT void sha1_avx2(const void *data, const uint64_t length, 1894 void *digest); 1895 IMB_DLL_EXPORT void sha1_one_block_avx2(const void *data, void *digest); 1896 IMB_DLL_EXPORT void sha224_avx2(const void *data, const uint64_t length, 1897 void *digest); 1898 IMB_DLL_EXPORT void sha224_one_block_avx2(const void *data, void *digest); 1899 IMB_DLL_EXPORT void sha256_avx2(const void *data, const uint64_t length, 1900 void *digest); 1901 IMB_DLL_EXPORT void sha256_one_block_avx2(const void *data, void *digest); 1902 IMB_DLL_EXPORT void sha384_avx2(const void *data, const uint64_t length, 1903 void *digest); 1904 IMB_DLL_EXPORT void sha384_one_block_avx2(const void *data, void *digest); 1905 IMB_DLL_EXPORT void sha512_avx2(const void *data, const uint64_t length, 1906 void *digest); 1907 IMB_DLL_EXPORT void sha512_one_block_avx2(const void *data, void *digest); 1908 IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest); 1909 IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys, 1910 void *dec_exp_keys); 1911 IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys, 1912 void *dec_exp_keys); 1913 IMB_DLL_EXPORT void aes_keyexp_256_avx2(const void *key, void *enc_exp_keys, 1914 void *dec_exp_keys); 1915 IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2(const void *key, void *k1_exp, 1916 void *k2, void *k3); 1917 IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2(const void *key, 1918 void *enc_exp_keys); 1919 IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2(const void *key, 1920 void *enc_exp_keys); 1921 IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2(const void *key, 1922 void *enc_exp_keys); 1923 IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1, 1924 void *key2); 1925 IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in, 1926 const void *iv, const void *keys, 1927 uint64_t len); 1928 1929 /* AVX512 */ 1930 IMB_DLL_EXPORT void sha1_avx512(const void *data, const uint64_t length, 1931 void *digest); 1932 IMB_DLL_EXPORT void sha1_one_block_avx512(const void *data, void *digest); 1933 IMB_DLL_EXPORT void sha224_avx512(const void *data, const uint64_t length, 1934 void *digest); 1935 IMB_DLL_EXPORT void sha224_one_block_avx512(const void *data, void *digest); 1936 IMB_DLL_EXPORT void sha256_avx512(const void *data, const uint64_t length, 1937 void *digest); 1938 IMB_DLL_EXPORT void sha256_one_block_avx512(const void *data, void *digest); 1939 IMB_DLL_EXPORT void sha384_avx512(const void *data, const uint64_t length, 1940 void *digest); 1941 IMB_DLL_EXPORT void sha384_one_block_avx512(const void *data, void *digest); 1942 IMB_DLL_EXPORT void sha512_avx512(const void *data, const uint64_t length, 1943 void *digest); 1944 IMB_DLL_EXPORT void sha512_one_block_avx512(const void *data, void *digest); 1945 IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest); 1946 IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys, 1947 void *dec_exp_keys); 1948 IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys, 1949 void *dec_exp_keys); 1950 IMB_DLL_EXPORT void aes_keyexp_256_avx512(const void *key, void *enc_exp_keys, 1951 void *dec_exp_keys); 1952 IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512(const void *key, void *k1_exp, 1953 void *k2, void *k3); 1954 IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512(const void *key, 1955 void *enc_exp_keys); 1956 IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512(const void *key, 1957 void *enc_exp_keys); 1958 IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512(const void *key, 1959 void *enc_exp_keys); 1960 IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1, 1961 void *key2); 1962 IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in, 1963 const void *iv, const void *keys, 1964 uint64_t len); 1965 1966 /* 1967 * Direct GCM API. 1968 * Note that GCM is also available through job API. 1969 */ 1970 /** 1971 * @brief GCM-AES Encryption 1972 * 1973 * @param key_data GCM expanded key data 1974 * @param context_data GCM operation context data 1975 * @param out Ciphertext output. Encrypt in-place is allowed. 1976 * @param in Plaintext input. 1977 * @param len Length of data in Bytes for encryption. 1978 * @param iv pointer to 12 byte IV structure. Internally, library 1979 * concates 0x00000001 value to it. 1980 * @param aad Additional Authentication Data (AAD). 1981 * @param aad_len Length of AAD. 1982 * @param auth_tag Authenticated Tag output. 1983 * @param auth_tag_len Authenticated Tag Length in bytes (must be 1984 * a multiple of 4 bytes). Valid values are 1985 * 16 (most likely), 12 or 8. 1986 */ 1987 IMB_DLL_EXPORT void 1988 aes_gcm_enc_128_sse(const struct gcm_key_data *key_data, 1989 struct gcm_context_data *context_data, 1990 uint8_t *out, uint8_t const *in, uint64_t len, 1991 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, 1992 uint8_t *auth_tag, uint64_t auth_tag_len); 1993 IMB_DLL_EXPORT void 1994 aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data, 1995 struct gcm_context_data *context_data, 1996 uint8_t *out, uint8_t const *in, uint64_t len, 1997 const uint8_t *iv, 1998 uint8_t const *aad, uint64_t aad_len, 1999 uint8_t *auth_tag, uint64_t auth_tag_len); 2000 IMB_DLL_EXPORT void 2001 aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data, 2002 struct gcm_context_data *context_data, 2003 uint8_t *out, uint8_t const *in, uint64_t len, 2004 const uint8_t *iv, 2005 uint8_t const *aad, uint64_t aad_len, 2006 uint8_t *auth_tag, uint64_t auth_tag_len); 2007 2008 IMB_DLL_EXPORT void 2009 aes_gcm_enc_192_sse(const struct gcm_key_data *key_data, 2010 struct gcm_context_data *context_data, 2011 uint8_t *out, uint8_t const *in, uint64_t len, 2012 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, 2013 uint8_t *auth_tag, uint64_t auth_tag_len); 2014 IMB_DLL_EXPORT void 2015 aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data, 2016 struct gcm_context_data *context_data, 2017 uint8_t *out, uint8_t const *in, uint64_t len, 2018 const uint8_t *iv, 2019 uint8_t const *aad, uint64_t aad_len, 2020 uint8_t *auth_tag, uint64_t auth_tag_len); 2021 IMB_DLL_EXPORT void 2022 aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data, 2023 struct gcm_context_data *context_data, 2024 uint8_t *out, uint8_t const *in, uint64_t len, 2025 const uint8_t *iv, 2026 uint8_t const *aad, uint64_t aad_len, 2027 uint8_t *auth_tag, uint64_t auth_tag_len); 2028 2029 IMB_DLL_EXPORT void 2030 aes_gcm_enc_256_sse(const struct gcm_key_data *key_data, 2031 struct gcm_context_data *context_data, 2032 uint8_t *out, uint8_t const *in, uint64_t len, 2033 const uint8_t *iv, 2034 uint8_t const *aad, uint64_t aad_len, 2035 uint8_t *auth_tag, uint64_t auth_tag_len); 2036 IMB_DLL_EXPORT void 2037 aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data, 2038 struct gcm_context_data *context_data, 2039 uint8_t *out, uint8_t const *in, uint64_t len, 2040 const uint8_t *iv, 2041 uint8_t const *aad, uint64_t aad_len, 2042 uint8_t *auth_tag, uint64_t auth_tag_len); 2043 IMB_DLL_EXPORT void 2044 aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data, 2045 struct gcm_context_data *context_data, 2046 uint8_t *out, uint8_t const *in, uint64_t len, 2047 const uint8_t *iv, 2048 uint8_t const *aad, uint64_t aad_len, 2049 uint8_t *auth_tag, uint64_t auth_tag_len); 2050 2051 /** 2052 * @brief GCM-AES Decryption 2053 * 2054 * @param key_data GCM expanded keys data 2055 * @param context_data GCM operation context data 2056 * @param out Plaintext output. Decrypt in-place is allowed. 2057 * @param in Ciphertext input. 2058 * @param len Length of data in Bytes for decryption. 2059 * @param iv pointer to 12 byte IV structure. Internally, library 2060 * concates 0x00000001 value to it. 2061 * @param aad Additional Authentication Data (AAD). 2062 * @param aad_len Length of AAD. 2063 * @param auth_tag Authenticated Tag output. 2064 * @param auth_tag_len Authenticated Tag Length in bytes (must be 2065 * a multiple of 4 bytes). Valid values are 2066 * 16 (most likely), 12 or 8. 2067 */ 2068 IMB_DLL_EXPORT void 2069 aes_gcm_dec_128_sse(const struct gcm_key_data *key_data, 2070 struct gcm_context_data *context_data, 2071 uint8_t *out, uint8_t const *in, uint64_t len, 2072 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, 2073 uint8_t *auth_tag, uint64_t auth_tag_len); 2074 IMB_DLL_EXPORT void 2075 aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data, 2076 struct gcm_context_data *context_data, 2077 uint8_t *out, uint8_t const *in, uint64_t len, 2078 const uint8_t *iv, 2079 uint8_t const *aad, uint64_t aad_len, 2080 uint8_t *auth_tag, uint64_t auth_tag_len); 2081 IMB_DLL_EXPORT void 2082 aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data, 2083 struct gcm_context_data *context_data, 2084 uint8_t *out, uint8_t const *in, uint64_t len, 2085 const uint8_t *iv, 2086 uint8_t const *aad, uint64_t aad_len, 2087 uint8_t *auth_tag, uint64_t auth_tag_len); 2088 2089 IMB_DLL_EXPORT void 2090 aes_gcm_dec_192_sse(const struct gcm_key_data *key_data, 2091 struct gcm_context_data *context_data, 2092 uint8_t *out, uint8_t const *in, uint64_t len, 2093 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, 2094 uint8_t *auth_tag, uint64_t auth_tag_len); 2095 IMB_DLL_EXPORT void 2096 aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data, 2097 struct gcm_context_data *context_data, 2098 uint8_t *out, uint8_t const *in, uint64_t len, 2099 const uint8_t *iv, 2100 uint8_t const *aad, uint64_t aad_len, 2101 uint8_t *auth_tag, uint64_t auth_tag_len); 2102 IMB_DLL_EXPORT void 2103 aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data, 2104 struct gcm_context_data *context_data, 2105 uint8_t *out, uint8_t const *in, uint64_t len, 2106 const uint8_t *iv, 2107 uint8_t const *aad, uint64_t aad_len, 2108 uint8_t *auth_tag, uint64_t auth_tag_len); 2109 2110 IMB_DLL_EXPORT void 2111 aes_gcm_dec_256_sse(const struct gcm_key_data *key_data, 2112 struct gcm_context_data *context_data, 2113 uint8_t *out, uint8_t const *in, uint64_t len, 2114 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, 2115 uint8_t *auth_tag, uint64_t auth_tag_len); 2116 IMB_DLL_EXPORT void 2117 aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data, 2118 struct gcm_context_data *context_data, 2119 uint8_t *out, uint8_t const *in, uint64_t len, 2120 const uint8_t *iv, 2121 uint8_t const *aad, uint64_t aad_len, 2122 uint8_t *auth_tag, uint64_t auth_tag_len); 2123 IMB_DLL_EXPORT void 2124 aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data, 2125 struct gcm_context_data *context_data, 2126 uint8_t *out, uint8_t const *in, uint64_t len, 2127 const uint8_t *iv, 2128 uint8_t const *aad, uint64_t aad_len, 2129 uint8_t *auth_tag, uint64_t auth_tag_len); 2130 2131 /** 2132 * @brief Start a AES-GCM Encryption message 2133 * 2134 * @param key_data GCM expanded key data 2135 * @param context_data GCM operation context data 2136 * @param iv pointer to 12 byte IV structure. Internally, library 2137 * concates 0x00000001 value to it. 2138 * @param aad Additional Authentication Data (AAD). 2139 * @param aad_len Length of AAD. 2140 * 2141 */ 2142 IMB_DLL_EXPORT void 2143 aes_gcm_init_128_sse(const struct gcm_key_data *key_data, 2144 struct gcm_context_data *context_data, 2145 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); 2146 IMB_DLL_EXPORT void 2147 aes_gcm_init_128_avx_gen2(const struct gcm_key_data *key_data, 2148 struct gcm_context_data *context_data, 2149 const uint8_t *iv, 2150 uint8_t const *aad, uint64_t aad_len); 2151 IMB_DLL_EXPORT void 2152 aes_gcm_init_128_avx_gen4(const struct gcm_key_data *key_data, 2153 struct gcm_context_data *context_data, 2154 const uint8_t *iv, 2155 uint8_t const *aad, uint64_t aad_len); 2156 2157 IMB_DLL_EXPORT void 2158 aes_gcm_init_192_sse(const struct gcm_key_data *key_data, 2159 struct gcm_context_data *context_data, 2160 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); 2161 IMB_DLL_EXPORT void 2162 aes_gcm_init_192_avx_gen2(const struct gcm_key_data *key_data, 2163 struct gcm_context_data *context_data, 2164 const uint8_t *iv, 2165 uint8_t const *aad, uint64_t aad_len); 2166 IMB_DLL_EXPORT void 2167 aes_gcm_init_192_avx_gen4(const struct gcm_key_data *key_data, 2168 struct gcm_context_data *context_data, 2169 const uint8_t *iv, 2170 uint8_t const *aad, uint64_t aad_len); 2171 2172 IMB_DLL_EXPORT void 2173 aes_gcm_init_256_sse(const struct gcm_key_data *key_data, 2174 struct gcm_context_data *context_data, 2175 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); 2176 IMB_DLL_EXPORT void 2177 aes_gcm_init_256_avx_gen2(const struct gcm_key_data *key_data, 2178 struct gcm_context_data *context_data, 2179 const uint8_t *iv, 2180 uint8_t const *aad, uint64_t aad_len); 2181 IMB_DLL_EXPORT void 2182 aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data, 2183 struct gcm_context_data *context_data, 2184 const uint8_t *iv, 2185 uint8_t const *aad, uint64_t aad_len); 2186 2187 /** 2188 * @brief encrypt a block of a AES-GCM Encryption message 2189 * 2190 * @param key_data GCM expanded key data 2191 * @param context_data GCM operation context data 2192 * @param out Ciphertext output. Encrypt in-place is allowed. 2193 * @param in Plaintext input. 2194 * @param len Length of data in Bytes for decryption. 2195 */ 2196 IMB_DLL_EXPORT void 2197 aes_gcm_enc_128_update_sse(const struct gcm_key_data *key_data, 2198 struct gcm_context_data *context_data, 2199 uint8_t *out, const uint8_t *in, uint64_t len); 2200 IMB_DLL_EXPORT void 2201 aes_gcm_enc_128_update_avx_gen2(const struct gcm_key_data *key_data, 2202 struct gcm_context_data *context_data, 2203 uint8_t *out, const uint8_t *in, uint64_t len); 2204 IMB_DLL_EXPORT void 2205 aes_gcm_enc_128_update_avx_gen4(const struct gcm_key_data *key_data, 2206 struct gcm_context_data *context_data, 2207 uint8_t *out, const uint8_t *in, uint64_t len); 2208 2209 IMB_DLL_EXPORT void 2210 aes_gcm_enc_192_update_sse(const struct gcm_key_data *key_data, 2211 struct gcm_context_data *context_data, 2212 uint8_t *out, const uint8_t *in, uint64_t len); 2213 IMB_DLL_EXPORT void 2214 aes_gcm_enc_192_update_avx_gen2(const struct gcm_key_data *key_data, 2215 struct gcm_context_data *context_data, 2216 uint8_t *out, const uint8_t *in, uint64_t len); 2217 IMB_DLL_EXPORT void 2218 aes_gcm_enc_192_update_avx_gen4(const struct gcm_key_data *key_data, 2219 struct gcm_context_data *context_data, 2220 uint8_t *out, const uint8_t *in, uint64_t len); 2221 2222 IMB_DLL_EXPORT void 2223 aes_gcm_enc_256_update_sse(const struct gcm_key_data *key_data, 2224 struct gcm_context_data *context_data, 2225 uint8_t *out, const uint8_t *in, uint64_t len); 2226 IMB_DLL_EXPORT void 2227 aes_gcm_enc_256_update_avx_gen2(const struct gcm_key_data *key_data, 2228 struct gcm_context_data *context_data, 2229 uint8_t *out, const uint8_t *in, uint64_t len); 2230 IMB_DLL_EXPORT void 2231 aes_gcm_enc_256_update_avx_gen4(const struct gcm_key_data *key_data, 2232 struct gcm_context_data *context_data, 2233 uint8_t *out, const uint8_t *in, uint64_t len); 2234 2235 /** 2236 * @brief decrypt a block of a AES-GCM Encryption message 2237 * 2238 * @param key_data GCM expanded key data 2239 * @param context_data GCM operation context data 2240 * @param out Plaintext output. Decrypt in-place is allowed. 2241 * @param in Ciphertext input. 2242 * @param len Length of data in Bytes for decryption. 2243 */ 2244 IMB_DLL_EXPORT void 2245 aes_gcm_dec_128_update_sse(const struct gcm_key_data *key_data, 2246 struct gcm_context_data *context_data, 2247 uint8_t *out, const uint8_t *in, uint64_t len); 2248 IMB_DLL_EXPORT void 2249 aes_gcm_dec_128_update_avx_gen2(const struct gcm_key_data *key_data, 2250 struct gcm_context_data *context_data, 2251 uint8_t *out, const uint8_t *in, uint64_t len); 2252 IMB_DLL_EXPORT void 2253 aes_gcm_dec_128_update_avx_gen4(const struct gcm_key_data *key_data, 2254 struct gcm_context_data *context_data, 2255 uint8_t *out, const uint8_t *in, uint64_t len); 2256 2257 IMB_DLL_EXPORT void 2258 aes_gcm_dec_192_update_sse(const struct gcm_key_data *key_data, 2259 struct gcm_context_data *context_data, 2260 uint8_t *out, const uint8_t *in, uint64_t len); 2261 IMB_DLL_EXPORT void 2262 aes_gcm_dec_192_update_avx_gen2(const struct gcm_key_data *key_data, 2263 struct gcm_context_data *context_data, 2264 uint8_t *out, const uint8_t *in, uint64_t len); 2265 IMB_DLL_EXPORT void 2266 aes_gcm_dec_192_update_avx_gen4(const struct gcm_key_data *key_data, 2267 struct gcm_context_data *context_data, 2268 uint8_t *out, const uint8_t *in, uint64_t len); 2269 2270 IMB_DLL_EXPORT void 2271 aes_gcm_dec_256_update_sse(const struct gcm_key_data *key_data, 2272 struct gcm_context_data *context_data, 2273 uint8_t *out, const uint8_t *in, uint64_t len); 2274 IMB_DLL_EXPORT void 2275 aes_gcm_dec_256_update_avx_gen2(const struct gcm_key_data *key_data, 2276 struct gcm_context_data *context_data, 2277 uint8_t *out, const uint8_t *in, uint64_t len); 2278 IMB_DLL_EXPORT void 2279 aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data, 2280 struct gcm_context_data *context_data, 2281 uint8_t *out, const uint8_t *in, uint64_t len); 2282 2283 /** 2284 * @brief End encryption of a AES-GCM Encryption message 2285 * 2286 * @param key_data GCM expanded key data 2287 * @param context_data GCM operation context data 2288 * @param auth_tag Authenticated Tag output. 2289 * @param auth_tag_len Authenticated Tag Length in bytes (must be 2290 * a multiple of 4 bytes). Valid values are 2291 * 16 (most likely), 12 or 8. 2292 */ 2293 IMB_DLL_EXPORT void 2294 aes_gcm_enc_128_finalize_sse(const struct gcm_key_data *key_data, 2295 struct gcm_context_data *context_data, 2296 uint8_t *auth_tag, uint64_t auth_tag_len); 2297 IMB_DLL_EXPORT void 2298 aes_gcm_enc_128_finalize_avx_gen2(const struct gcm_key_data *key_data, 2299 struct gcm_context_data *context_data, 2300 uint8_t *auth_tag, uint64_t auth_tag_len); 2301 IMB_DLL_EXPORT void 2302 aes_gcm_enc_128_finalize_avx_gen4(const struct gcm_key_data *key_data, 2303 struct gcm_context_data *context_data, 2304 uint8_t *auth_tag, uint64_t auth_tag_len); 2305 2306 IMB_DLL_EXPORT void 2307 aes_gcm_enc_192_finalize_sse(const struct gcm_key_data *key_data, 2308 struct gcm_context_data *context_data, 2309 uint8_t *auth_tag, uint64_t auth_tag_len); 2310 IMB_DLL_EXPORT void 2311 aes_gcm_enc_192_finalize_avx_gen2(const struct gcm_key_data *key_data, 2312 struct gcm_context_data *context_data, 2313 uint8_t *auth_tag, uint64_t auth_tag_len); 2314 IMB_DLL_EXPORT void 2315 aes_gcm_enc_192_finalize_avx_gen4(const struct gcm_key_data *key_data, 2316 struct gcm_context_data *context_data, 2317 uint8_t *auth_tag, uint64_t auth_tag_len); 2318 2319 IMB_DLL_EXPORT void 2320 aes_gcm_enc_256_finalize_sse(const struct gcm_key_data *key_data, 2321 struct gcm_context_data *context_data, 2322 uint8_t *auth_tag, uint64_t auth_tag_len); 2323 IMB_DLL_EXPORT void 2324 aes_gcm_enc_256_finalize_avx_gen2(const struct gcm_key_data *key_data, 2325 struct gcm_context_data *context_data, 2326 uint8_t *auth_tag, uint64_t auth_tag_len); 2327 IMB_DLL_EXPORT void 2328 aes_gcm_enc_256_finalize_avx_gen4(const struct gcm_key_data *key_data, 2329 struct gcm_context_data *context_data, 2330 uint8_t *auth_tag, uint64_t auth_tag_len); 2331 2332 /** 2333 * @brief End decryption of a AES-GCM Encryption message 2334 * 2335 * @param key_data GCM expanded key data 2336 * @param context_data GCM operation context data 2337 * @param auth_tag Authenticated Tag output. 2338 * @param auth_tag_len Authenticated Tag Length in bytes (must be 2339 * a multiple of 4 bytes). Valid values are 2340 * 16 (most likely), 12 or 8. 2341 */ 2342 IMB_DLL_EXPORT void 2343 aes_gcm_dec_128_finalize_sse(const struct gcm_key_data *key_data, 2344 struct gcm_context_data *context_data, 2345 uint8_t *auth_tag, uint64_t auth_tag_len); 2346 IMB_DLL_EXPORT void 2347 aes_gcm_dec_128_finalize_avx_gen2(const struct gcm_key_data *key_data, 2348 struct gcm_context_data *context_data, 2349 uint8_t *auth_tag, uint64_t auth_tag_len); 2350 IMB_DLL_EXPORT void 2351 aes_gcm_dec_128_finalize_avx_gen4(const struct gcm_key_data *key_data, 2352 struct gcm_context_data *context_data, 2353 uint8_t *auth_tag, uint64_t auth_tag_len); 2354 2355 IMB_DLL_EXPORT void 2356 aes_gcm_dec_192_finalize_sse(const struct gcm_key_data *key_data, 2357 struct gcm_context_data *context_data, 2358 uint8_t *auth_tag, uint64_t auth_tag_len); 2359 IMB_DLL_EXPORT void 2360 aes_gcm_dec_192_finalize_avx_gen2(const struct gcm_key_data *key_data, 2361 struct gcm_context_data *context_data, 2362 uint8_t *auth_tag, uint64_t auth_tag_len); 2363 IMB_DLL_EXPORT void 2364 aes_gcm_dec_192_finalize_avx_gen4(const struct gcm_key_data *key_data, 2365 struct gcm_context_data *context_data, 2366 uint8_t *auth_tag, uint64_t auth_tag_len); 2367 2368 IMB_DLL_EXPORT void 2369 aes_gcm_dec_256_finalize_sse(const struct gcm_key_data *key_data, 2370 struct gcm_context_data *context_data, 2371 uint8_t *auth_tag, uint64_t auth_tag_len); 2372 IMB_DLL_EXPORT void 2373 aes_gcm_dec_256_finalize_avx_gen2(const struct gcm_key_data *key_data, 2374 struct gcm_context_data *context_data, 2375 uint8_t *auth_tag, uint64_t auth_tag_len); 2376 IMB_DLL_EXPORT void 2377 aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data, 2378 struct gcm_context_data *context_data, 2379 uint8_t *auth_tag, uint64_t auth_tag_len); 2380 2381 /** 2382 * @brief Precomputation of HashKey constants 2383 * 2384 * Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and 2385 * shifted_hkey_X_k). 2386 * 2387 * @param gdata GCM context data 2388 */ 2389 IMB_DLL_EXPORT void aes_gcm_precomp_128_sse(struct gcm_key_data *key_data); 2390 IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2(struct gcm_key_data *key_data); 2391 IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4(struct gcm_key_data *key_data); 2392 2393 IMB_DLL_EXPORT void aes_gcm_precomp_192_sse(struct gcm_key_data *key_data); 2394 IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2(struct gcm_key_data *key_data); 2395 IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4(struct gcm_key_data *key_data); 2396 2397 IMB_DLL_EXPORT void aes_gcm_precomp_256_sse(struct gcm_key_data *key_data); 2398 IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2(struct gcm_key_data *key_data); 2399 IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4(struct gcm_key_data *key_data); 2400 2401 /** 2402 * @brief Pre-processes GCM key data 2403 * 2404 * Prefills the gcm key data with key values for each round and 2405 * the initial sub hash key for tag encoding 2406 * 2407 * @param key pointer to key data 2408 * @param key_data GCM expanded key data 2409 * 2410 */ 2411 IMB_DLL_EXPORT void aes_gcm_pre_128_sse(const void *key, 2412 struct gcm_key_data *key_data); 2413 IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2(const void *key, 2414 struct gcm_key_data *key_data); 2415 IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4(const void *key, 2416 struct gcm_key_data *key_data); 2417 IMB_DLL_EXPORT void aes_gcm_pre_192_sse(const void *key, 2418 struct gcm_key_data *key_data); 2419 IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2(const void *key, 2420 struct gcm_key_data *key_data); 2421 IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4(const void *key, 2422 struct gcm_key_data *key_data); 2423 IMB_DLL_EXPORT void aes_gcm_pre_256_sse(const void *key, 2424 struct gcm_key_data *key_data); 2425 IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2(const void *key, 2426 struct gcm_key_data *key_data); 2427 IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4(const void *key, 2428 struct gcm_key_data *key_data); 2429 2430 /** 2431 * @brief Generation of ZUC Initialization Vectors (for EEA3 and EIA3) 2432 * 2433 * @param [in] count COUNT (4 bytes in Little Endian) 2434 * @param [in] bearer BEARER (5 bits) 2435 * @param [in] dir DIRECTION (1 bit) 2436 * @param [out] iv_ptr Pointer to generated IV (16 bytes) 2437 * 2438 * @return 2439 * - 0 if success 2440 * - 1 if one or more parameters are wrong 2441 */ 2442 IMB_DLL_EXPORT int zuc_eea3_iv_gen(const uint32_t count, 2443 const uint8_t bearer, 2444 const uint8_t dir, 2445 void *iv_ptr); 2446 IMB_DLL_EXPORT int zuc_eia3_iv_gen(const uint32_t count, 2447 const uint8_t bearer, 2448 const uint8_t dir, 2449 void *iv_ptr); 2450 2451 /** 2452 * @brief Generation of KASUMI F8 Initialization Vector 2453 * 2454 * @param [in] count COUNT (4 bytes in Little Endian) 2455 * @param [in] bearer BEARER (5 bits) 2456 * @param [in] dir DIRECTION (1 bit) 2457 * @param [out] iv_ptr Pointer to generated IV (16 bytes) 2458 * 2459 * @return 2460 * - 0 if success 2461 * - 1 if one or more parameters are wrong 2462 */ 2463 IMB_DLL_EXPORT int kasumi_f8_iv_gen(const uint32_t count, 2464 const uint8_t bearer, 2465 const uint8_t dir, 2466 void *iv_ptr); 2467 /** 2468 * @brief Generation of KASUMI F9 Initialization Vector 2469 * 2470 * @param [in] count COUNT (4 bytes in Little Endian) 2471 * @param [in] fresh FRESH (4 bytes in Little Endian) 2472 * @param [out] iv_ptr Pointer to generated IV (16 bytes) 2473 * 2474 * @return 2475 * - 0 if success 2476 * - 1 if one or more parameters are wrong 2477 */ 2478 IMB_DLL_EXPORT int kasumi_f9_iv_gen(const uint32_t count, 2479 const uint32_t fresh, 2480 void *iv_ptr); 2481 2482 /** 2483 * @brief Generation of SNOW3G F8 Initialization Vector 2484 * 2485 * Parameters are passed in Little Endian format and 2486 * used to generate the IV in Big Endian format 2487 * 2488 * @param [in] count COUNT (4 bytes in Little Endian) 2489 * @param [in] bearer BEARER (5 bits) 2490 * @param [in] dir DIRECTION (1 bit) 2491 * @param [out] iv_ptr Pointer to generated IV (16 bytes) in Big Endian format 2492 * 2493 * @return 2494 * - 0 if success 2495 * - 1 if one or more parameters are wrong 2496 */ 2497 IMB_DLL_EXPORT int snow3g_f8_iv_gen(const uint32_t count, 2498 const uint8_t bearer, 2499 const uint8_t dir, 2500 void *iv_ptr); 2501 /** 2502 * @brief Generation of SNOW3G F9 Initialization Vector 2503 * 2504 * Parameters are passed in Little Endian format and 2505 * used to generate the IV in Big Endian format 2506 * 2507 * @param [in] count COUNT (4 bytes in Little Endian) 2508 * @param [in] fresh FRESH (4 bytes in Little Endian) 2509 * @param [in] dir DIRECTION (1 bit) 2510 * @param [out] iv_ptr Pointer to generated IV (16 bytes) in Big Endian format 2511 * 2512 * @return 2513 * - 0 if success 2514 * - 1 if one or more parameters are wrong 2515 */ 2516 IMB_DLL_EXPORT int snow3g_f9_iv_gen(const uint32_t count, 2517 const uint32_t fresh, 2518 const uint8_t dir, 2519 void *iv_ptr); 2520 /** 2521 * @brief Force clearing/zeroing of memory 2522 * 2523 * @param [in] mem Pointer to memory address to clear 2524 * @param [in] size Size of memory to clear (in bytes) 2525 */ 2526 IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size); 2527 2528 #ifdef __cplusplus 2529 } 2530 #endif 2531 2532 #endif /* IMB_IPSEC_MB_H */ 2533