1 /* aes.c
2 *
3 * Copyright (C) 2006-2021 wolfSSL Inc.
4 *
5 * This file is part of wolfSSL.
6 *
7 * wolfSSL is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * wolfSSL is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
20 */
21
22 /*
23
24 DESCRIPTION
25 This library provides the interfaces to the Advanced Encryption Standard (AES)
26 for encrypting and decrypting data. AES is the standard known for a symmetric
27 block cipher mechanism that uses n-bit binary string parameter key with 128-bits,
28 192-bits, and 256-bits of key sizes.
29
30 */
31 #ifdef HAVE_CONFIG_H
32 #include <config.h>
33 #endif
34
35 #include <wolfssl/wolfcrypt/settings.h>
36 #include <wolfssl/wolfcrypt/error-crypt.h>
37
38 #if !defined(NO_AES)
39
40 /* Tip: Locate the software cipher modes by searching for "Software AES" */
41
42 #if defined(HAVE_FIPS) && \
43 defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2)
44
45 /* set NO_WRAPPERS before headers, use direct internal f()s not wrappers */
46 #define FIPS_NO_WRAPPERS
47
48 #ifdef USE_WINDOWS_API
49 #pragma code_seg(".fipsA$g")
50 #pragma const_seg(".fipsB$g")
51 #endif
52 #endif
53
54 #include <wolfssl/wolfcrypt/aes.h>
55
56 #ifdef WOLFSSL_AESNI
57 #include <wmmintrin.h>
58 #include <emmintrin.h>
59 #include <smmintrin.h>
60 #endif /* WOLFSSL_AESNI */
61
62 #include <wolfssl/wolfcrypt/cpuid.h>
63
64 #ifdef WOLF_CRYPTO_CB
65 #include <wolfssl/wolfcrypt/cryptocb.h>
66 #endif
67
68 #ifdef WOLFSSL_IMXRT_DCP
69 #include <wolfssl/wolfcrypt/port/nxp/dcp_port.h>
70 #endif
71 #if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
72 #include <wolfssl/wolfcrypt/port/nxp/se050_port.h>
73 #endif
74
75 /* fips wrapper calls, user can call direct */
76 #if defined(HAVE_FIPS) && \
77 (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))
78
wc_AesSetKey(Aes * aes,const byte * key,word32 len,const byte * iv,int dir)79 int wc_AesSetKey(Aes* aes, const byte* key, word32 len, const byte* iv,
80 int dir)
81 {
82 if (aes == NULL || !( (len == 16) || (len == 24) || (len == 32)) ) {
83 return BAD_FUNC_ARG;
84 }
85
86 return AesSetKey_fips(aes, key, len, iv, dir);
87 }
wc_AesSetIV(Aes * aes,const byte * iv)88 int wc_AesSetIV(Aes* aes, const byte* iv)
89 {
90 if (aes == NULL) {
91 return BAD_FUNC_ARG;
92 }
93
94 return AesSetIV_fips(aes, iv);
95 }
96 #ifdef HAVE_AES_CBC
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)97 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
98 {
99 if (aes == NULL || out == NULL || in == NULL) {
100 return BAD_FUNC_ARG;
101 }
102
103 return AesCbcEncrypt_fips(aes, out, in, sz);
104 }
105 #ifdef HAVE_AES_DECRYPT
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)106 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
107 {
108 if (aes == NULL || out == NULL || in == NULL
109 || sz % AES_BLOCK_SIZE != 0) {
110 return BAD_FUNC_ARG;
111 }
112
113 return AesCbcDecrypt_fips(aes, out, in, sz);
114 }
115 #endif /* HAVE_AES_DECRYPT */
116 #endif /* HAVE_AES_CBC */
117
118 /* AES-CTR */
119 #ifdef WOLFSSL_AES_COUNTER
wc_AesCtrEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)120 int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
121 {
122 if (aes == NULL || out == NULL || in == NULL) {
123 return BAD_FUNC_ARG;
124 }
125
126 return AesCtrEncrypt(aes, out, in, sz);
127 }
128 #endif
129
130 /* AES-DIRECT */
131 #if defined(WOLFSSL_AES_DIRECT)
wc_AesEncryptDirect(Aes * aes,byte * out,const byte * in)132 void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
133 {
134 AesEncryptDirect(aes, out, in);
135 }
136
137 #ifdef HAVE_AES_DECRYPT
wc_AesDecryptDirect(Aes * aes,byte * out,const byte * in)138 void wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
139 {
140 AesDecryptDirect(aes, out, in);
141 }
142 #endif /* HAVE_AES_DECRYPT */
143
wc_AesSetKeyDirect(Aes * aes,const byte * key,word32 len,const byte * iv,int dir)144 int wc_AesSetKeyDirect(Aes* aes, const byte* key, word32 len,
145 const byte* iv, int dir)
146 {
147 return AesSetKeyDirect(aes, key, len, iv, dir);
148 }
149 #endif /* WOLFSSL_AES_DIRECT */
150
151 /* AES-GCM */
152 #ifdef HAVE_AESGCM
wc_AesGcmSetKey(Aes * aes,const byte * key,word32 len)153 int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
154 {
155 if (aes == NULL || !( (len == 16) || (len == 24) || (len == 32)) ) {
156 return BAD_FUNC_ARG;
157 }
158
159 return AesGcmSetKey_fips(aes, key, len);
160 }
wc_AesGcmEncrypt(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)161 int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
162 const byte* iv, word32 ivSz,
163 byte* authTag, word32 authTagSz,
164 const byte* authIn, word32 authInSz)
165 {
166 if (aes == NULL || authTagSz > AES_BLOCK_SIZE ||
167 authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ ||
168 ivSz == 0 || ivSz > AES_BLOCK_SIZE) {
169 return BAD_FUNC_ARG;
170 }
171
172 return AesGcmEncrypt_fips(aes, out, in, sz, iv, ivSz, authTag,
173 authTagSz, authIn, authInSz);
174 }
175
176 #ifdef HAVE_AES_DECRYPT
wc_AesGcmDecrypt(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)177 int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
178 const byte* iv, word32 ivSz,
179 const byte* authTag, word32 authTagSz,
180 const byte* authIn, word32 authInSz)
181 {
182 if (aes == NULL || out == NULL || in == NULL || iv == NULL
183 || authTag == NULL || authTagSz > AES_BLOCK_SIZE ||
184 ivSz == 0 || ivSz > AES_BLOCK_SIZE) {
185 return BAD_FUNC_ARG;
186 }
187
188 return AesGcmDecrypt_fips(aes, out, in, sz, iv, ivSz, authTag,
189 authTagSz, authIn, authInSz);
190 }
191 #endif /* HAVE_AES_DECRYPT */
192
wc_GmacSetKey(Gmac * gmac,const byte * key,word32 len)193 int wc_GmacSetKey(Gmac* gmac, const byte* key, word32 len)
194 {
195 if (gmac == NULL || key == NULL || !((len == 16) ||
196 (len == 24) || (len == 32)) ) {
197 return BAD_FUNC_ARG;
198 }
199
200 return GmacSetKey(gmac, key, len);
201 }
wc_GmacUpdate(Gmac * gmac,const byte * iv,word32 ivSz,const byte * authIn,word32 authInSz,byte * authTag,word32 authTagSz)202 int wc_GmacUpdate(Gmac* gmac, const byte* iv, word32 ivSz,
203 const byte* authIn, word32 authInSz,
204 byte* authTag, word32 authTagSz)
205 {
206 if (gmac == NULL || authTagSz > AES_BLOCK_SIZE ||
207 authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
208 return BAD_FUNC_ARG;
209 }
210
211 return GmacUpdate(gmac, iv, ivSz, authIn, authInSz,
212 authTag, authTagSz);
213 }
214 #endif /* HAVE_AESGCM */
215
216 /* AES-CCM */
217 #if defined(HAVE_AESCCM) && \
218 defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2)
wc_AesCcmSetKey(Aes * aes,const byte * key,word32 keySz)219 int wc_AesCcmSetKey(Aes* aes, const byte* key, word32 keySz)
220 {
221 return AesCcmSetKey(aes, key, keySz);
222 }
wc_AesCcmEncrypt(Aes * aes,byte * out,const byte * in,word32 inSz,const byte * nonce,word32 nonceSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)223 int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
224 const byte* nonce, word32 nonceSz,
225 byte* authTag, word32 authTagSz,
226 const byte* authIn, word32 authInSz)
227 {
228 /* sanity check on arguments */
229 if (aes == NULL || out == NULL || in == NULL || nonce == NULL
230 || authTag == NULL || nonceSz < 7 || nonceSz > 13)
231 return BAD_FUNC_ARG;
232
233 AesCcmEncrypt(aes, out, in, inSz, nonce, nonceSz, authTag,
234 authTagSz, authIn, authInSz);
235 return 0;
236 }
237
238 #ifdef HAVE_AES_DECRYPT
wc_AesCcmDecrypt(Aes * aes,byte * out,const byte * in,word32 inSz,const byte * nonce,word32 nonceSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)239 int wc_AesCcmDecrypt(Aes* aes, byte* out,
240 const byte* in, word32 inSz,
241 const byte* nonce, word32 nonceSz,
242 const byte* authTag, word32 authTagSz,
243 const byte* authIn, word32 authInSz)
244 {
245
246 if (aes == NULL || out == NULL || in == NULL || nonce == NULL
247 || authTag == NULL || nonceSz < 7 || nonceSz > 13) {
248 return BAD_FUNC_ARG;
249 }
250
251 return AesCcmDecrypt(aes, out, in, inSz, nonce, nonceSz,
252 authTag, authTagSz, authIn, authInSz);
253 }
254 #endif /* HAVE_AES_DECRYPT */
255 #endif /* HAVE_AESCCM && HAVE_FIPS_VERSION 2 */
256
wc_AesInit(Aes * aes,void * h,int i)257 int wc_AesInit(Aes* aes, void* h, int i)
258 {
259 if (aes == NULL)
260 return BAD_FUNC_ARG;
261
262 (void)h;
263 (void)i;
264
265 /* FIPS doesn't support */
266 #ifdef WOLFSSL_KCAPI_AES
267 return AesInit(aes, h, i);
268 #else
269 return 0;
270 #endif
271 }
wc_AesFree(Aes * aes)272 void wc_AesFree(Aes* aes)
273 {
274 (void)aes;
275 /* FIPS doesn't support */
276 #ifdef WOLFSSL_KCAPI_AES
277 AesFree(aes);
278 #endif
279 }
280
281 #else /* else build without fips, or for FIPS v2 */
282
283
284 #if defined(WOLFSSL_TI_CRYPT)
285 #include <wolfcrypt/src/port/ti/ti-aes.c>
286 #else
287
288 #include <wolfssl/wolfcrypt/logging.h>
289
290 #ifdef NO_INLINE
291 #include <wolfssl/wolfcrypt/misc.h>
292 #else
293 #define WOLFSSL_MISC_INCLUDED
294 #include <wolfcrypt/src/misc.c>
295 #endif
296
297 #if !defined(WOLFSSL_ARMASM)
298
299 #ifdef WOLFSSL_IMX6_CAAM_BLOB
300 /* case of possibly not using hardware acceleration for AES but using key
301 blobs */
302 #include <wolfssl/wolfcrypt/port/caam/wolfcaam.h>
303 #endif
304
305 #ifdef DEBUG_AESNI
306 #include <stdio.h>
307 #endif
308
309 #ifdef _MSC_VER
310 /* 4127 warning constant while(1) */
311 #pragma warning(disable: 4127)
312 #endif
313
314
315 /* Define AES implementation includes and functions */
316 #if defined(STM32_CRYPTO)
317 /* STM32F2/F4/F7/L4/L5/H7/WB55 hardware AES support for ECB, CBC, CTR and GCM modes */
318
319 #if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AESGCM) || defined(HAVE_AESCCM)
320
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)321 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
322 {
323 int ret = 0;
324 #ifdef WOLFSSL_STM32_CUBEMX
325 CRYP_HandleTypeDef hcryp;
326 #else
327 CRYP_InitTypeDef cryptInit;
328 CRYP_KeyInitTypeDef keyInit;
329 #endif
330
331 #ifdef WOLFSSL_STM32_CUBEMX
332 ret = wc_Stm32_Aes_Init(aes, &hcryp);
333 if (ret != 0)
334 return ret;
335
336 ret = wolfSSL_CryptHwMutexLock();
337 if (ret != 0)
338 return ret;
339
340 #if defined(STM32_HAL_V2)
341 hcryp.Init.Algorithm = CRYP_AES_ECB;
342 #elif defined(STM32_CRYPTO_AES_ONLY)
343 hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
344 hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_ECB;
345 hcryp.Init.KeyWriteFlag = CRYP_KEY_WRITE_ENABLE;
346 #endif
347 HAL_CRYP_Init(&hcryp);
348
349 #if defined(STM32_HAL_V2)
350 ret = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)inBlock, AES_BLOCK_SIZE,
351 (uint32_t*)outBlock, STM32_HAL_TIMEOUT);
352 #elif defined(STM32_CRYPTO_AES_ONLY)
353 ret = HAL_CRYPEx_AES(&hcryp, (uint8_t*)inBlock, AES_BLOCK_SIZE,
354 outBlock, STM32_HAL_TIMEOUT);
355 #else
356 ret = HAL_CRYP_AESECB_Encrypt(&hcryp, (uint8_t*)inBlock, AES_BLOCK_SIZE,
357 outBlock, STM32_HAL_TIMEOUT);
358 #endif
359 if (ret != HAL_OK) {
360 ret = WC_TIMEOUT_E;
361 }
362 HAL_CRYP_DeInit(&hcryp);
363
364 #else /* Standard Peripheral Library */
365 ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
366 if (ret != 0)
367 return ret;
368
369 ret = wolfSSL_CryptHwMutexLock();
370 if (ret != 0)
371 return ret;
372
373 /* reset registers to their default values */
374 CRYP_DeInit();
375
376 /* setup key */
377 CRYP_KeyInit(&keyInit);
378
379 /* set direction and mode */
380 cryptInit.CRYP_AlgoDir = CRYP_AlgoDir_Encrypt;
381 cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_ECB;
382 CRYP_Init(&cryptInit);
383
384 /* enable crypto processor */
385 CRYP_Cmd(ENABLE);
386
387 /* flush IN/OUT FIFOs */
388 CRYP_FIFOFlush();
389
390 CRYP_DataIn(*(uint32_t*)&inBlock[0]);
391 CRYP_DataIn(*(uint32_t*)&inBlock[4]);
392 CRYP_DataIn(*(uint32_t*)&inBlock[8]);
393 CRYP_DataIn(*(uint32_t*)&inBlock[12]);
394
395 /* wait until the complete message has been processed */
396 while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
397
398 *(uint32_t*)&outBlock[0] = CRYP_DataOut();
399 *(uint32_t*)&outBlock[4] = CRYP_DataOut();
400 *(uint32_t*)&outBlock[8] = CRYP_DataOut();
401 *(uint32_t*)&outBlock[12] = CRYP_DataOut();
402
403 /* disable crypto processor */
404 CRYP_Cmd(DISABLE);
405 #endif /* WOLFSSL_STM32_CUBEMX */
406 wolfSSL_CryptHwMutexUnLock();
407
408 return ret;
409 }
410 #endif /* WOLFSSL_AES_DIRECT || HAVE_AESGCM || HAVE_AESCCM */
411
412 #ifdef HAVE_AES_DECRYPT
413 #if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AESCCM)
wc_AesDecrypt(Aes * aes,const byte * inBlock,byte * outBlock)414 static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
415 {
416 int ret = 0;
417 #ifdef WOLFSSL_STM32_CUBEMX
418 CRYP_HandleTypeDef hcryp;
419 #else
420 CRYP_InitTypeDef cryptInit;
421 CRYP_KeyInitTypeDef keyInit;
422 #endif
423
424 #ifdef WOLFSSL_STM32_CUBEMX
425 ret = wc_Stm32_Aes_Init(aes, &hcryp);
426 if (ret != 0)
427 return ret;
428
429 ret = wolfSSL_CryptHwMutexLock();
430 if (ret != 0)
431 return ret;
432
433 #if defined(STM32_HAL_V2)
434 hcryp.Init.Algorithm = CRYP_AES_ECB;
435 #elif defined(STM32_CRYPTO_AES_ONLY)
436 hcryp.Init.OperatingMode = CRYP_ALGOMODE_KEYDERIVATION_DECRYPT;
437 hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_ECB;
438 hcryp.Init.KeyWriteFlag = CRYP_KEY_WRITE_ENABLE;
439 #endif
440 HAL_CRYP_Init(&hcryp);
441
442 #if defined(STM32_HAL_V2)
443 ret = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)inBlock, AES_BLOCK_SIZE,
444 (uint32_t*)outBlock, STM32_HAL_TIMEOUT);
445 #elif defined(STM32_CRYPTO_AES_ONLY)
446 ret = HAL_CRYPEx_AES(&hcryp, (uint8_t*)inBlock, AES_BLOCK_SIZE,
447 outBlock, STM32_HAL_TIMEOUT);
448 #else
449 ret = HAL_CRYP_AESECB_Decrypt(&hcryp, (uint8_t*)inBlock, AES_BLOCK_SIZE,
450 outBlock, STM32_HAL_TIMEOUT);
451 #endif
452 if (ret != HAL_OK) {
453 ret = WC_TIMEOUT_E;
454 }
455 HAL_CRYP_DeInit(&hcryp);
456
457 #else /* Standard Peripheral Library */
458 ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
459 if (ret != 0)
460 return ret;
461
462 ret = wolfSSL_CryptHwMutexLock();
463 if (ret != 0)
464 return ret;
465
466 /* reset registers to their default values */
467 CRYP_DeInit();
468
469 /* set direction and key */
470 CRYP_KeyInit(&keyInit);
471 cryptInit.CRYP_AlgoDir = CRYP_AlgoDir_Decrypt;
472 cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_Key;
473 CRYP_Init(&cryptInit);
474
475 /* enable crypto processor */
476 CRYP_Cmd(ENABLE);
477
478 /* wait until decrypt key has been initialized */
479 while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
480
481 /* set direction and mode */
482 cryptInit.CRYP_AlgoDir = CRYP_AlgoDir_Decrypt;
483 cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_ECB;
484 CRYP_Init(&cryptInit);
485
486 /* enable crypto processor */
487 CRYP_Cmd(ENABLE);
488
489 /* flush IN/OUT FIFOs */
490 CRYP_FIFOFlush();
491
492 CRYP_DataIn(*(uint32_t*)&inBlock[0]);
493 CRYP_DataIn(*(uint32_t*)&inBlock[4]);
494 CRYP_DataIn(*(uint32_t*)&inBlock[8]);
495 CRYP_DataIn(*(uint32_t*)&inBlock[12]);
496
497 /* wait until the complete message has been processed */
498 while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
499
500 *(uint32_t*)&outBlock[0] = CRYP_DataOut();
501 *(uint32_t*)&outBlock[4] = CRYP_DataOut();
502 *(uint32_t*)&outBlock[8] = CRYP_DataOut();
503 *(uint32_t*)&outBlock[12] = CRYP_DataOut();
504
505 /* disable crypto processor */
506 CRYP_Cmd(DISABLE);
507 #endif /* WOLFSSL_STM32_CUBEMX */
508 wolfSSL_CryptHwMutexUnLock();
509
510 return ret;
511 }
512 #endif /* WOLFSSL_AES_DIRECT || HAVE_AESCCM */
513 #endif /* HAVE_AES_DECRYPT */
514
515 #elif defined(HAVE_COLDFIRE_SEC)
516 /* Freescale Coldfire SEC support for CBC mode.
517 * NOTE: no support for AES-CTR/GCM/CCM/Direct */
518 #include <wolfssl/wolfcrypt/types.h>
519 #include "sec.h"
520 #include "mcf5475_sec.h"
521 #include "mcf5475_siu.h"
522 #elif defined(FREESCALE_LTC)
523 #include "fsl_ltc.h"
524 #if defined(FREESCALE_LTC_AES_GCM)
525 #undef NEED_AES_TABLES
526 #undef GCM_TABLE
527 #endif
528
529 /* if LTC doesn't have GCM, use software with LTC AES ECB mode */
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)530 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
531 {
532 word32 keySize = 0;
533 byte* key = (byte*)aes->key;
534 wc_AesGetKeySize(aes, &keySize);
535
536 if (wolfSSL_CryptHwMutexLock() == 0) {
537 LTC_AES_EncryptEcb(LTC_BASE, inBlock, outBlock, AES_BLOCK_SIZE,
538 key, keySize);
539 wolfSSL_CryptHwMutexUnLock();
540 }
541 return 0;
542 }
543 #ifdef HAVE_AES_DECRYPT
wc_AesDecrypt(Aes * aes,const byte * inBlock,byte * outBlock)544 static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
545 {
546 word32 keySize = 0;
547 byte* key = (byte*)aes->key;
548 wc_AesGetKeySize(aes, &keySize);
549
550 if (wolfSSL_CryptHwMutexLock() == 0) {
551 LTC_AES_DecryptEcb(LTC_BASE, inBlock, outBlock, AES_BLOCK_SIZE,
552 key, keySize, kLTC_EncryptKey);
553 wolfSSL_CryptHwMutexUnLock();
554 }
555 return 0;
556 }
557 #endif
558
559 #elif defined(FREESCALE_MMCAU)
560 /* Freescale mmCAU hardware AES support for Direct, CBC, CCM, GCM modes
561 * through the CAU/mmCAU library. Documentation located in
562 * ColdFire/ColdFire+ CAU and Kinetis mmCAU Software Library User
563 * Guide (See note in README). */
564 #ifdef FREESCALE_MMCAU_CLASSIC
565 /* MMCAU 1.4 library used with non-KSDK / classic MQX builds */
566 #include "cau_api.h"
567 #else
568 #include "fsl_mmcau.h"
569 #endif
570
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)571 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
572 {
573 if (wolfSSL_CryptHwMutexLock() == 0) {
574 #ifdef FREESCALE_MMCAU_CLASSIC
575 if ((wc_ptr_t)outBlock % WOLFSSL_MMCAU_ALIGNMENT) {
576 WOLFSSL_MSG("Bad cau_aes_encrypt alignment");
577 return BAD_ALIGN_E;
578 }
579 cau_aes_encrypt(inBlock, (byte*)aes->key, aes->rounds, outBlock);
580 #else
581 MMCAU_AES_EncryptEcb(inBlock, (byte*)aes->key, aes->rounds,
582 outBlock);
583 #endif
584 wolfSSL_CryptHwMutexUnLock();
585 }
586 return 0;
587 }
588 #ifdef HAVE_AES_DECRYPT
wc_AesDecrypt(Aes * aes,const byte * inBlock,byte * outBlock)589 static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
590 {
591 if (wolfSSL_CryptHwMutexLock() == 0) {
592 #ifdef FREESCALE_MMCAU_CLASSIC
593 if ((wc_ptr_t)outBlock % WOLFSSL_MMCAU_ALIGNMENT) {
594 WOLFSSL_MSG("Bad cau_aes_decrypt alignment");
595 return BAD_ALIGN_E;
596 }
597 cau_aes_decrypt(inBlock, (byte*)aes->key, aes->rounds, outBlock);
598 #else
599 MMCAU_AES_DecryptEcb(inBlock, (byte*)aes->key, aes->rounds,
600 outBlock);
601 #endif
602 wolfSSL_CryptHwMutexUnLock();
603 }
604 return 0;
605 }
606 #endif /* HAVE_AES_DECRYPT */
607
608 #elif defined(WOLFSSL_PIC32MZ_CRYPT)
609
610 #include <wolfssl/wolfcrypt/port/pic32/pic32mz-crypt.h>
611
612 #if defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT)
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)613 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
614 {
615 /* Thread mutex protection handled in Pic32Crypto */
616 return wc_Pic32AesCrypt(aes->key, aes->keylen, NULL, 0,
617 outBlock, inBlock, AES_BLOCK_SIZE,
618 PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RECB);
619 }
620 #endif
621
622 #if defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT)
wc_AesDecrypt(Aes * aes,const byte * inBlock,byte * outBlock)623 static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
624 {
625 /* Thread mutex protection handled in Pic32Crypto */
626 return wc_Pic32AesCrypt(aes->key, aes->keylen, NULL, 0,
627 outBlock, inBlock, AES_BLOCK_SIZE,
628 PIC32_DECRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RECB);
629 }
630 #endif
631
632 #elif defined(WOLFSSL_NRF51_AES)
633 /* Use built-in AES hardware - AES 128 ECB Encrypt Only */
634 #include "wolfssl/wolfcrypt/port/nrf51.h"
635
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)636 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
637 {
638 int ret;
639 ret = wolfSSL_CryptHwMutexLock();
640 if (ret == 0) {
641 ret = nrf51_aes_encrypt(inBlock, (byte*)aes->key, aes->rounds, outBlock);
642 wolfSSL_CryptHwMutexUnLock();
643 }
644 return ret;
645 }
646
647 #ifdef HAVE_AES_DECRYPT
648 #error nRF51 AES Hardware does not support decrypt
649 #endif /* HAVE_AES_DECRYPT */
650
651 #elif defined(WOLFSSL_ESP32WROOM32_CRYPT) && \
652 !defined(NO_WOLFSSL_ESP32WROOM32_CRYPT_AES)
653
654 #include "wolfssl/wolfcrypt/port/Espressif/esp32-crypt.h"
655
656 #if defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT)
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)657 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
658 {
659 /* Thread mutex protection handled in esp_aes_hw_InUse */
660 return wc_esp32AesEncrypt(aes, inBlock, outBlock);
661 }
662 #endif
663
664 #if defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT)
wc_AesDecrypt(Aes * aes,const byte * inBlock,byte * outBlock)665 static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
666 {
667 /* Thread mutex protection handled in esp_aes_hw_InUse */
668 return wc_esp32AesDecrypt(aes, inBlock, outBlock);
669 }
670 #endif
671
672 #elif defined(WOLFSSL_AESNI)
673
674 #define NEED_AES_TABLES
675
676 /* Each platform needs to query info type 1 from cpuid to see if aesni is
677 * supported. Also, let's setup a macro for proper linkage w/o ABI conflicts
678 */
679
680 #ifndef AESNI_ALIGN
681 #define AESNI_ALIGN 16
682 #endif
683
684 #ifdef _MSC_VER
685 #define XASM_LINK(f)
686 #elif defined(__APPLE__)
687 #define XASM_LINK(f) asm("_" f)
688 #else
689 #define XASM_LINK(f) asm(f)
690 #endif /* _MSC_VER */
691
692 static int checkAESNI = 0;
693 static int haveAESNI = 0;
694 static word32 intel_flags = 0;
695
Check_CPU_support_AES(void)696 static int Check_CPU_support_AES(void)
697 {
698 intel_flags = cpuid_get_flags();
699
700 return IS_INTEL_AESNI(intel_flags) != 0;
701 }
702
703
704 /* tell C compiler these are asm functions in case any mix up of ABI underscore
705 prefix between clang/gcc/llvm etc */
706 #ifdef HAVE_AES_CBC
707 void AES_CBC_encrypt(const unsigned char* in, unsigned char* out,
708 unsigned char* ivec, unsigned long length,
709 const unsigned char* KS, int nr)
710 XASM_LINK("AES_CBC_encrypt");
711
712 #ifdef HAVE_AES_DECRYPT
713 #if defined(WOLFSSL_AESNI_BY4)
714 void AES_CBC_decrypt_by4(const unsigned char* in, unsigned char* out,
715 unsigned char* ivec, unsigned long length,
716 const unsigned char* KS, int nr)
717 XASM_LINK("AES_CBC_decrypt_by4");
718 #elif defined(WOLFSSL_AESNI_BY6)
719 void AES_CBC_decrypt_by6(const unsigned char* in, unsigned char* out,
720 unsigned char* ivec, unsigned long length,
721 const unsigned char* KS, int nr)
722 XASM_LINK("AES_CBC_decrypt_by6");
723 #else /* WOLFSSL_AESNI_BYx */
724 void AES_CBC_decrypt_by8(const unsigned char* in, unsigned char* out,
725 unsigned char* ivec, unsigned long length,
726 const unsigned char* KS, int nr)
727 XASM_LINK("AES_CBC_decrypt_by8");
728 #endif /* WOLFSSL_AESNI_BYx */
729 #endif /* HAVE_AES_DECRYPT */
730 #endif /* HAVE_AES_CBC */
731
732 void AES_ECB_encrypt(const unsigned char* in, unsigned char* out,
733 unsigned long length, const unsigned char* KS, int nr)
734 XASM_LINK("AES_ECB_encrypt");
735
736 #ifdef HAVE_AES_DECRYPT
737 void AES_ECB_decrypt(const unsigned char* in, unsigned char* out,
738 unsigned long length, const unsigned char* KS, int nr)
739 XASM_LINK("AES_ECB_decrypt");
740 #endif
741
742 void AES_128_Key_Expansion(const unsigned char* userkey,
743 unsigned char* key_schedule)
744 XASM_LINK("AES_128_Key_Expansion");
745
746 void AES_192_Key_Expansion(const unsigned char* userkey,
747 unsigned char* key_schedule)
748 XASM_LINK("AES_192_Key_Expansion");
749
750 void AES_256_Key_Expansion(const unsigned char* userkey,
751 unsigned char* key_schedule)
752 XASM_LINK("AES_256_Key_Expansion");
753
754
AES_set_encrypt_key(const unsigned char * userKey,const int bits,Aes * aes)755 static int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
756 Aes* aes)
757 {
758 int ret;
759
760 if (!userKey || !aes)
761 return BAD_FUNC_ARG;
762
763 switch (bits) {
764 case 128:
765 AES_128_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 10;
766 return 0;
767 case 192:
768 AES_192_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 12;
769 return 0;
770 case 256:
771 AES_256_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 14;
772 return 0;
773 default:
774 ret = BAD_FUNC_ARG;
775 }
776
777 return ret;
778 }
779
780 #ifdef HAVE_AES_DECRYPT
AES_set_decrypt_key(const unsigned char * userKey,const int bits,Aes * aes)781 static int AES_set_decrypt_key(const unsigned char* userKey,
782 const int bits, Aes* aes)
783 {
784 int nr;
785 #ifdef WOLFSSL_SMALL_STACK
786 Aes *temp_key;
787 #else
788 Aes temp_key[1];
789 #endif
790 __m128i *Key_Schedule;
791 __m128i *Temp_Key_Schedule;
792
793 if (!userKey || !aes)
794 return BAD_FUNC_ARG;
795
796 #ifdef WOLFSSL_SMALL_STACK
797 if ((temp_key = (Aes *)XMALLOC(sizeof *aes, aes->heap,
798 DYNAMIC_TYPE_AES)) == NULL)
799 return MEMORY_E;
800 #endif
801
802 if (AES_set_encrypt_key(userKey,bits,temp_key) == BAD_FUNC_ARG) {
803 #ifdef WOLFSSL_SMALL_STACK
804 XFREE(temp_key, aes->heap, DYNAMIC_TYPE_AES);
805 #endif
806 return BAD_FUNC_ARG;
807 }
808
809 Key_Schedule = (__m128i*)aes->key;
810 Temp_Key_Schedule = (__m128i*)temp_key->key;
811
812 nr = temp_key->rounds;
813 aes->rounds = nr;
814
815 #ifdef WOLFSSL_SMALL_STACK
816 SAVE_VECTOR_REGISTERS(XFREE(temp_key, aes->heap, DYNAMIC_TYPE_AES); return _svr_ret;);
817 #else
818 SAVE_VECTOR_REGISTERS(return _svr_ret;);
819 #endif
820
821 Key_Schedule[nr] = Temp_Key_Schedule[0];
822 Key_Schedule[nr-1] = _mm_aesimc_si128(Temp_Key_Schedule[1]);
823 Key_Schedule[nr-2] = _mm_aesimc_si128(Temp_Key_Schedule[2]);
824 Key_Schedule[nr-3] = _mm_aesimc_si128(Temp_Key_Schedule[3]);
825 Key_Schedule[nr-4] = _mm_aesimc_si128(Temp_Key_Schedule[4]);
826 Key_Schedule[nr-5] = _mm_aesimc_si128(Temp_Key_Schedule[5]);
827 Key_Schedule[nr-6] = _mm_aesimc_si128(Temp_Key_Schedule[6]);
828 Key_Schedule[nr-7] = _mm_aesimc_si128(Temp_Key_Schedule[7]);
829 Key_Schedule[nr-8] = _mm_aesimc_si128(Temp_Key_Schedule[8]);
830 Key_Schedule[nr-9] = _mm_aesimc_si128(Temp_Key_Schedule[9]);
831
832 if (nr>10) {
833 Key_Schedule[nr-10] = _mm_aesimc_si128(Temp_Key_Schedule[10]);
834 Key_Schedule[nr-11] = _mm_aesimc_si128(Temp_Key_Schedule[11]);
835 }
836
837 if (nr>12) {
838 Key_Schedule[nr-12] = _mm_aesimc_si128(Temp_Key_Schedule[12]);
839 Key_Schedule[nr-13] = _mm_aesimc_si128(Temp_Key_Schedule[13]);
840 }
841
842 Key_Schedule[0] = Temp_Key_Schedule[nr];
843
844 RESTORE_VECTOR_REGISTERS();
845
846 #ifdef WOLFSSL_SMALL_STACK
847 XFREE(temp_key, aes->heap, DYNAMIC_TYPE_AES);
848 #endif
849
850 return 0;
851 }
852 #endif /* HAVE_AES_DECRYPT */
853
854 #elif (defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) \
855 && !defined(WOLFSSL_QNX_CAAM)) || \
856 ((defined(WOLFSSL_AFALG) || defined(WOLFSSL_DEVCRYPTO_AES)) && \
857 defined(HAVE_AESCCM))
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)858 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
859 {
860 wc_AesEncryptDirect(aes, outBlock, inBlock);
861 return 0;
862 }
863
864 #elif defined(WOLFSSL_AFALG)
865 /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
866
867 #elif defined(WOLFSSL_DEVCRYPTO_AES)
868 /* implemented in wolfcrypt/src/port/devcrypto/devcrypto_aes.c */
869
870 #elif defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
AES_ECB_encrypt(Aes * aes,const byte * inBlock,byte * outBlock,int sz)871 static int AES_ECB_encrypt(Aes* aes, const byte* inBlock, byte* outBlock,
872 int sz)
873 {
874 return se050_aes_crypt(aes, inBlock, outBlock, sz, AES_ENCRYPTION,
875 kAlgorithm_SSS_AES_ECB);
876 }
AES_ECB_decrypt(Aes * aes,const byte * inBlock,byte * outBlock,int sz)877 static int AES_ECB_decrypt(Aes* aes, const byte* inBlock, byte* outBlock,
878 int sz)
879 {
880 return se050_aes_crypt(aes, inBlock, outBlock, sz, AES_DECRYPTION,
881 kAlgorithm_SSS_AES_ECB);
882 }
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)883 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
884 {
885 return AES_ECB_encrypt(aes, inBlock, outBlock, AES_BLOCK_SIZE);
886 }
wc_AesDecrypt(Aes * aes,const byte * inBlock,byte * outBlock)887 static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
888 {
889 return AES_ECB_decrypt(aes, inBlock, outBlock, AES_BLOCK_SIZE);
890 }
891
892 #elif defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
893 #include "hal_data.h"
894
895 #ifndef WOLFSSL_SCE_AES256_HANDLE
896 #define WOLFSSL_SCE_AES256_HANDLE g_sce_aes_256
897 #endif
898
899 #ifndef WOLFSSL_SCE_AES192_HANDLE
900 #define WOLFSSL_SCE_AES192_HANDLE g_sce_aes_192
901 #endif
902
903 #ifndef WOLFSSL_SCE_AES128_HANDLE
904 #define WOLFSSL_SCE_AES128_HANDLE g_sce_aes_128
905 #endif
906
AES_ECB_encrypt(Aes * aes,const byte * inBlock,byte * outBlock,int sz)907 static int AES_ECB_encrypt(Aes* aes, const byte* inBlock, byte* outBlock,
908 int sz)
909 {
910 word32 ret;
911
912 if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag ==
913 CRYPTO_WORD_ENDIAN_BIG) {
914 ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
915 }
916
917 switch (aes->keylen) {
918 #ifdef WOLFSSL_AES_128
919 case AES_128_KEY_SIZE:
920 ret = WOLFSSL_SCE_AES128_HANDLE.p_api->encrypt(
921 WOLFSSL_SCE_AES128_HANDLE.p_ctrl, aes->key,
922 NULL, (sz / sizeof(word32)), (word32*)inBlock,
923 (word32*)outBlock);
924 break;
925 #endif
926 #ifdef WOLFSSL_AES_192
927 case AES_192_KEY_SIZE:
928 ret = WOLFSSL_SCE_AES192_HANDLE.p_api->encrypt(
929 WOLFSSL_SCE_AES192_HANDLE.p_ctrl, aes->key,
930 NULL, (sz / sizeof(word32)), (word32*)inBlock,
931 (word32*)outBlock);
932 break;
933 #endif
934 #ifdef WOLFSSL_AES_256
935 case AES_256_KEY_SIZE:
936 ret = WOLFSSL_SCE_AES256_HANDLE.p_api->encrypt(
937 WOLFSSL_SCE_AES256_HANDLE.p_ctrl, aes->key,
938 NULL, (sz / sizeof(word32)), (word32*)inBlock,
939 (word32*)outBlock);
940 break;
941 #endif
942 default:
943 WOLFSSL_MSG("Unknown key size");
944 return BAD_FUNC_ARG;
945 }
946
947 if (ret != SSP_SUCCESS) {
948 /* revert input */
949 ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
950 return WC_HW_E;
951 }
952
953 if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag ==
954 CRYPTO_WORD_ENDIAN_BIG) {
955 ByteReverseWords((word32*)outBlock, (word32*)outBlock, sz);
956 if (inBlock != outBlock) {
957 /* revert input */
958 ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
959 }
960 }
961 return 0;
962 }
963
964 #if defined(HAVE_AES_DECRYPT)
AES_ECB_decrypt(Aes * aes,const byte * inBlock,byte * outBlock,int sz)965 static int AES_ECB_decrypt(Aes* aes, const byte* inBlock, byte* outBlock,
966 int sz)
967 {
968 word32 ret;
969
970 if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag ==
971 CRYPTO_WORD_ENDIAN_BIG) {
972 ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
973 }
974
975 switch (aes->keylen) {
976 #ifdef WOLFSSL_AES_128
977 case AES_128_KEY_SIZE:
978 ret = WOLFSSL_SCE_AES128_HANDLE.p_api->decrypt(
979 WOLFSSL_SCE_AES128_HANDLE.p_ctrl, aes->key, aes->reg,
980 (sz / sizeof(word32)), (word32*)inBlock,
981 (word32*)outBlock);
982 break;
983 #endif
984 #ifdef WOLFSSL_AES_192
985 case AES_192_KEY_SIZE:
986 ret = WOLFSSL_SCE_AES192_HANDLE.p_api->decrypt(
987 WOLFSSL_SCE_AES192_HANDLE.p_ctrl, aes->key, aes->reg,
988 (sz / sizeof(word32)), (word32*)inBlock,
989 (word32*)outBlock);
990 break;
991 #endif
992 #ifdef WOLFSSL_AES_256
993 case AES_256_KEY_SIZE:
994 ret = WOLFSSL_SCE_AES256_HANDLE.p_api->decrypt(
995 WOLFSSL_SCE_AES256_HANDLE.p_ctrl, aes->key, aes->reg,
996 (sz / sizeof(word32)), (word32*)inBlock,
997 (word32*)outBlock);
998 break;
999 #endif
1000 default:
1001 WOLFSSL_MSG("Unknown key size");
1002 return BAD_FUNC_ARG;
1003 }
1004 if (ret != SSP_SUCCESS) {
1005 return WC_HW_E;
1006 }
1007
1008 if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag ==
1009 CRYPTO_WORD_ENDIAN_BIG) {
1010 ByteReverseWords((word32*)outBlock, (word32*)outBlock, sz);
1011 if (inBlock != outBlock) {
1012 /* revert input */
1013 ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
1014 }
1015 }
1016
1017 return 0;
1018 }
1019 #endif /* HAVE_AES_DECRYPT */
1020
1021 #if defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT)
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)1022 static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
1023 {
1024 return AES_ECB_encrypt(aes, inBlock, outBlock, AES_BLOCK_SIZE);
1025 }
1026 #endif
1027
1028 #if defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT)
wc_AesDecrypt(Aes * aes,const byte * inBlock,byte * outBlock)1029 static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
1030 {
1031 return AES_ECB_decrypt(aes, inBlock, outBlock, AES_BLOCK_SIZE);
1032 }
1033 #endif
1034
1035 #elif defined(WOLFSSL_KCAPI_AES)
1036 /* Only CBC and GCM that are in wolfcrypt/src/port/kcapi/kcapi_aes.c */
1037 #if defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AESCCM) || \
1038 defined(WOLFSSL_CMAC) || defined(WOLFSSL_AES_OFB) || \
1039 defined(WOLFSSL_AES_CFB) || defined(HAVE_AES_ECB)
1040 #define NEED_AES_TABLES
1041 #endif
1042 #else
1043
1044 /* using wolfCrypt software implementation */
1045 #define NEED_AES_TABLES
1046 #endif
1047
1048
1049
1050 #ifdef NEED_AES_TABLES
1051
1052 static const FLASH_QUALIFIER word32 rcon[] = {
1053 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1054 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1055 0x1B000000, 0x36000000,
1056 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
1057 };
1058
1059 #ifndef WOLFSSL_AES_SMALL_TABLES
1060 static const FLASH_QUALIFIER word32 Te[4][256] = {
1061 {
1062 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
1063 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
1064 0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
1065 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
1066 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
1067 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
1068 0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
1069 0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
1070 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
1071 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
1072 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
1073 0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
1074 0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
1075 0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
1076 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
1077 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
1078 0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
1079 0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
1080 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
1081 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
1082 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
1083 0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
1084 0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
1085 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
1086 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
1087 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
1088 0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
1089 0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
1090 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
1091 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
1092 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
1093 0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
1094 0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
1095 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
1096 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
1097 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
1098 0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
1099 0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
1100 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
1101 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
1102 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
1103 0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
1104 0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
1105 0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
1106 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
1107 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
1108 0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
1109 0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
1110 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
1111 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
1112 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
1113 0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
1114 0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
1115 0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
1116 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
1117 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
1118 0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
1119 0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
1120 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
1121 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
1122 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
1123 0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
1124 0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
1125 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
1126 },
1127 {
1128 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
1129 0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
1130 0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
1131 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
1132 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
1133 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
1134 0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
1135 0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
1136 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
1137 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
1138 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
1139 0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
1140 0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
1141 0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
1142 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
1143 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
1144 0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
1145 0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
1146 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
1147 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
1148 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
1149 0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
1150 0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
1151 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
1152 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
1153 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
1154 0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
1155 0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
1156 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
1157 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
1158 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
1159 0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
1160 0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
1161 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
1162 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
1163 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
1164 0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
1165 0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
1166 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
1167 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
1168 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
1169 0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
1170 0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
1171 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
1172 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
1173 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
1174 0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
1175 0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
1176 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
1177 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
1178 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
1179 0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
1180 0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
1181 0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
1182 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
1183 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
1184 0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
1185 0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
1186 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
1187 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
1188 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
1189 0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
1190 0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
1191 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
1192 },
1193 {
1194 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
1195 0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
1196 0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
1197 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
1198 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
1199 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
1200 0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
1201 0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
1202 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
1203 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
1204 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
1205 0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
1206 0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
1207 0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
1208 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
1209 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
1210 0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
1211 0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
1212 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
1213 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
1214 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
1215 0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
1216 0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
1217 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
1218 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
1219 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
1220 0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
1221 0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
1222 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
1223 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
1224 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
1225 0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
1226 0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
1227 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
1228 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
1229 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
1230 0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
1231 0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
1232 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
1233 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
1234 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
1235 0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
1236 0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
1237 0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
1238 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
1239 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
1240 0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
1241 0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
1242 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
1243 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
1244 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
1245 0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
1246 0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
1247 0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
1248 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
1249 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
1250 0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
1251 0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
1252 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
1253 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
1254 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
1255 0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
1256 0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
1257 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
1258 },
1259 {
1260 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
1261 0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
1262 0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
1263 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
1264 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
1265 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
1266 0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
1267 0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
1268 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
1269 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
1270 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
1271 0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
1272 0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
1273 0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
1274 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
1275 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
1276 0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
1277 0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
1278 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
1279 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
1280 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
1281 0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
1282 0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
1283 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
1284 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
1285 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
1286 0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
1287 0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
1288 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
1289 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
1290 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
1291 0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
1292 0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
1293 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
1294 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
1295 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
1296 0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
1297 0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
1298 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
1299 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
1300 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
1301 0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
1302 0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
1303 0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
1304 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
1305 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
1306 0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
1307 0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
1308 0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
1309 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
1310 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
1311 0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
1312 0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
1313 0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
1314 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
1315 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
1316 0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
1317 0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
1318 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
1319 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
1320 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
1321 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
1322 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
1323 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
1324 }
1325 };
1326
1327 #ifdef HAVE_AES_DECRYPT
1328 static const FLASH_QUALIFIER word32 Td[4][256] = {
1329 {
1330 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
1331 0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
1332 0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
1333 0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
1334 0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
1335 0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
1336 0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
1337 0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
1338 0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
1339 0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
1340 0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
1341 0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
1342 0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
1343 0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
1344 0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
1345 0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
1346 0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
1347 0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
1348 0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
1349 0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
1350 0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
1351 0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
1352 0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
1353 0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
1354 0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
1355 0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
1356 0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
1357 0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
1358 0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
1359 0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
1360 0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
1361 0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
1362 0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
1363 0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
1364 0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
1365 0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
1366 0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
1367 0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
1368 0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
1369 0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
1370 0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
1371 0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
1372 0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
1373 0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
1374 0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
1375 0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
1376 0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
1377 0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
1378 0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
1379 0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
1380 0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
1381 0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
1382 0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
1383 0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
1384 0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
1385 0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
1386 0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
1387 0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
1388 0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
1389 0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
1390 0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
1391 0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
1392 0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
1393 0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
1394 },
1395 {
1396 0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
1397 0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
1398 0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
1399 0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
1400 0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
1401 0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
1402 0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
1403 0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
1404 0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
1405 0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
1406 0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
1407 0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
1408 0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
1409 0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
1410 0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
1411 0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
1412 0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
1413 0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
1414 0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
1415 0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
1416 0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
1417 0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
1418 0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
1419 0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
1420 0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
1421 0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
1422 0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
1423 0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
1424 0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
1425 0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
1426 0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
1427 0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
1428 0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
1429 0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
1430 0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
1431 0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
1432 0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
1433 0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
1434 0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
1435 0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
1436 0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
1437 0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
1438 0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
1439 0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
1440 0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
1441 0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
1442 0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
1443 0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
1444 0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
1445 0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
1446 0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
1447 0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
1448 0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
1449 0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
1450 0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
1451 0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
1452 0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
1453 0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
1454 0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
1455 0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
1456 0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
1457 0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
1458 0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
1459 0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
1460 },
1461 {
1462 0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
1463 0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
1464 0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
1465 0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
1466 0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
1467 0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
1468 0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
1469 0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
1470 0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
1471 0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
1472 0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
1473 0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
1474 0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
1475 0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
1476 0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
1477 0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
1478 0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
1479 0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
1480 0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
1481 0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
1482
1483 0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
1484 0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
1485 0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
1486 0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
1487 0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
1488 0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
1489 0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
1490 0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
1491 0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
1492 0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
1493 0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
1494 0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
1495 0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
1496 0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
1497 0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
1498 0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
1499 0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
1500 0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
1501 0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
1502 0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
1503 0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
1504 0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
1505 0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
1506 0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
1507 0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
1508 0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
1509 0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
1510 0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
1511 0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
1512 0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
1513 0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
1514 0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
1515 0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
1516 0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
1517 0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
1518 0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
1519 0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
1520 0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
1521 0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
1522 0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
1523 0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
1524 0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
1525 0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
1526 0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
1527 },
1528 {
1529 0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
1530 0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
1531 0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
1532 0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
1533 0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
1534 0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
1535 0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
1536 0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
1537 0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
1538 0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
1539 0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
1540 0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
1541 0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
1542 0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
1543 0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
1544 0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
1545 0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
1546 0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
1547 0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
1548 0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
1549 0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
1550 0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
1551 0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
1552 0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
1553 0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
1554 0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
1555 0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
1556 0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
1557 0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
1558 0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
1559 0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
1560 0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
1561 0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
1562 0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
1563 0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
1564 0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
1565 0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
1566 0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
1567 0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
1568 0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
1569 0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
1570 0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
1571 0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
1572 0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
1573 0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
1574 0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
1575 0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
1576 0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
1577 0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
1578 0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
1579 0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
1580 0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
1581 0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
1582 0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
1583 0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
1584 0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
1585 0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
1586 0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
1587 0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
1588 0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
1589 0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
1590 0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
1591 0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
1592 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
1593 }
1594 };
1595 #endif /* HAVE_AES_DECRYPT */
1596 #endif /* WOLFSSL_AES_SMALL_TABLES */
1597
1598 #ifdef HAVE_AES_DECRYPT
1599 #if (defined(HAVE_AES_CBC) && !defined(WOLFSSL_DEVCRYPTO_CBC)) \
1600 || defined(WOLFSSL_AES_DIRECT)
1601 static const FLASH_QUALIFIER byte Td4[256] =
1602 {
1603 0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U,
1604 0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU,
1605 0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U,
1606 0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU,
1607 0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU,
1608 0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU,
1609 0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U,
1610 0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U,
1611 0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U,
1612 0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U,
1613 0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU,
1614 0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U,
1615 0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU,
1616 0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U,
1617 0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U,
1618 0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU,
1619 0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU,
1620 0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U,
1621 0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U,
1622 0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU,
1623 0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U,
1624 0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU,
1625 0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U,
1626 0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U,
1627 0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U,
1628 0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU,
1629 0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU,
1630 0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU,
1631 0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U,
1632 0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U,
1633 0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U,
1634 0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU,
1635 };
1636 #endif /* HAVE_AES_CBC || WOLFSSL_AES_DIRECT */
1637 #endif /* HAVE_AES_DECRYPT */
1638
1639 #define GETBYTE(x, y) (word32)((byte)((x) >> (8 * (y))))
1640
1641 #ifdef WOLFSSL_AES_SMALL_TABLES
1642 static const byte Tsbox[256] = {
1643 0x63U, 0x7cU, 0x77U, 0x7bU, 0xf2U, 0x6bU, 0x6fU, 0xc5U,
1644 0x30U, 0x01U, 0x67U, 0x2bU, 0xfeU, 0xd7U, 0xabU, 0x76U,
1645 0xcaU, 0x82U, 0xc9U, 0x7dU, 0xfaU, 0x59U, 0x47U, 0xf0U,
1646 0xadU, 0xd4U, 0xa2U, 0xafU, 0x9cU, 0xa4U, 0x72U, 0xc0U,
1647 0xb7U, 0xfdU, 0x93U, 0x26U, 0x36U, 0x3fU, 0xf7U, 0xccU,
1648 0x34U, 0xa5U, 0xe5U, 0xf1U, 0x71U, 0xd8U, 0x31U, 0x15U,
1649 0x04U, 0xc7U, 0x23U, 0xc3U, 0x18U, 0x96U, 0x05U, 0x9aU,
1650 0x07U, 0x12U, 0x80U, 0xe2U, 0xebU, 0x27U, 0xb2U, 0x75U,
1651 0x09U, 0x83U, 0x2cU, 0x1aU, 0x1bU, 0x6eU, 0x5aU, 0xa0U,
1652 0x52U, 0x3bU, 0xd6U, 0xb3U, 0x29U, 0xe3U, 0x2fU, 0x84U,
1653 0x53U, 0xd1U, 0x00U, 0xedU, 0x20U, 0xfcU, 0xb1U, 0x5bU,
1654 0x6aU, 0xcbU, 0xbeU, 0x39U, 0x4aU, 0x4cU, 0x58U, 0xcfU,
1655 0xd0U, 0xefU, 0xaaU, 0xfbU, 0x43U, 0x4dU, 0x33U, 0x85U,
1656 0x45U, 0xf9U, 0x02U, 0x7fU, 0x50U, 0x3cU, 0x9fU, 0xa8U,
1657 0x51U, 0xa3U, 0x40U, 0x8fU, 0x92U, 0x9dU, 0x38U, 0xf5U,
1658 0xbcU, 0xb6U, 0xdaU, 0x21U, 0x10U, 0xffU, 0xf3U, 0xd2U,
1659 0xcdU, 0x0cU, 0x13U, 0xecU, 0x5fU, 0x97U, 0x44U, 0x17U,
1660 0xc4U, 0xa7U, 0x7eU, 0x3dU, 0x64U, 0x5dU, 0x19U, 0x73U,
1661 0x60U, 0x81U, 0x4fU, 0xdcU, 0x22U, 0x2aU, 0x90U, 0x88U,
1662 0x46U, 0xeeU, 0xb8U, 0x14U, 0xdeU, 0x5eU, 0x0bU, 0xdbU,
1663 0xe0U, 0x32U, 0x3aU, 0x0aU, 0x49U, 0x06U, 0x24U, 0x5cU,
1664 0xc2U, 0xd3U, 0xacU, 0x62U, 0x91U, 0x95U, 0xe4U, 0x79U,
1665 0xe7U, 0xc8U, 0x37U, 0x6dU, 0x8dU, 0xd5U, 0x4eU, 0xa9U,
1666 0x6cU, 0x56U, 0xf4U, 0xeaU, 0x65U, 0x7aU, 0xaeU, 0x08U,
1667 0xbaU, 0x78U, 0x25U, 0x2eU, 0x1cU, 0xa6U, 0xb4U, 0xc6U,
1668 0xe8U, 0xddU, 0x74U, 0x1fU, 0x4bU, 0xbdU, 0x8bU, 0x8aU,
1669 0x70U, 0x3eU, 0xb5U, 0x66U, 0x48U, 0x03U, 0xf6U, 0x0eU,
1670 0x61U, 0x35U, 0x57U, 0xb9U, 0x86U, 0xc1U, 0x1dU, 0x9eU,
1671 0xe1U, 0xf8U, 0x98U, 0x11U, 0x69U, 0xd9U, 0x8eU, 0x94U,
1672 0x9bU, 0x1eU, 0x87U, 0xe9U, 0xceU, 0x55U, 0x28U, 0xdfU,
1673 0x8cU, 0xa1U, 0x89U, 0x0dU, 0xbfU, 0xe6U, 0x42U, 0x68U,
1674 0x41U, 0x99U, 0x2dU, 0x0fU, 0xb0U, 0x54U, 0xbbU, 0x16U
1675 };
1676
1677 #define AES_XTIME(x) ((byte)((byte)((x) << 1) ^ ((0 - ((x) >> 7)) & 0x1b)))
1678
col_mul(word32 t,int i2,int i3,int ia,int ib)1679 static word32 col_mul(word32 t, int i2, int i3, int ia, int ib)
1680 {
1681 byte t3 = GETBYTE(t, i3);
1682 byte tm = AES_XTIME(GETBYTE(t, i2) ^ t3);
1683
1684 return GETBYTE(t, ia) ^ GETBYTE(t, ib) ^ t3 ^ tm;
1685 }
1686
1687 #if defined(HAVE_AES_CBC) || defined(WOLFSSL_AES_DIRECT)
inv_col_mul(word32 t,int i9,int ib,int id,int ie)1688 static word32 inv_col_mul(word32 t, int i9, int ib, int id, int ie)
1689 {
1690 byte t9 = GETBYTE(t, i9);
1691 byte tb = GETBYTE(t, ib);
1692 byte td = GETBYTE(t, id);
1693 byte te = GETBYTE(t, ie);
1694 byte t0 = t9 ^ tb ^ td;
1695 return t0 ^ AES_XTIME(AES_XTIME(AES_XTIME(t0 ^ te) ^ td ^ te) ^ tb ^ te);
1696 }
1697 #endif
1698 #endif
1699
1700 #if defined(HAVE_AES_CBC) || defined(WOLFSSL_AES_DIRECT) || \
1701 defined(HAVE_AESCCM) || defined(HAVE_AESGCM)
1702
1703 #ifndef WC_CACHE_LINE_SZ
1704 #if defined(__x86_64__) || defined(_M_X64) || \
1705 (defined(__ILP32__) && (__ILP32__ >= 1))
1706 #define WC_CACHE_LINE_SZ 64
1707 #else
1708 /* default cache line size */
1709 #define WC_CACHE_LINE_SZ 32
1710 #endif
1711 #endif
1712
1713
1714 #ifndef WC_NO_CACHE_RESISTANT
1715 #ifndef WOLFSSL_AES_SMALL_TABLES
1716 /* load 4 Te Tables into cache by cache line stride */
PreFetchTe(void)1717 static WC_INLINE word32 PreFetchTe(void)
1718 {
1719 word32 x = 0;
1720 int i,j;
1721
1722 for (i = 0; i < 4; i++) {
1723 /* 256 elements, each one is 4 bytes */
1724 for (j = 0; j < 256; j += WC_CACHE_LINE_SZ/4) {
1725 x &= Te[i][j];
1726 }
1727 }
1728 return x;
1729 }
1730 #else
1731 /* load sbox into cache by cache line stride */
PreFetchSBox(void)1732 static WC_INLINE word32 PreFetchSBox(void)
1733 {
1734 word32 x = 0;
1735 int i;
1736
1737 for (i = 0; i < 256; i += WC_CACHE_LINE_SZ/4) {
1738 x &= Tsbox[i];
1739 }
1740 return x;
1741 }
1742 #endif
1743 #endif
1744
1745 /* Software AES - ECB Encrypt */
wc_AesEncrypt(Aes * aes,const byte * inBlock,byte * outBlock)1746 static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
1747 {
1748 word32 s0, s1, s2, s3;
1749 word32 t0, t1, t2, t3;
1750 word32 r = aes->rounds >> 1;
1751 const word32* rk = aes->key;
1752
1753 #ifdef DEBUG_WOLFSSL
1754 if (r > 7 || r == 0) {
1755 WOLFSSL_MSG("AesEncrypt encountered improper key, set it up");
1756 return; /* stop instead of seg-faulting, set up your keys! */
1757 }
1758 #endif
1759
1760 #ifdef WOLFSSL_AESNI
1761 if (haveAESNI && aes->use_aesni) {
1762 #ifdef DEBUG_AESNI
1763 printf("about to aes encrypt\n");
1764 printf("in = %p\n", inBlock);
1765 printf("out = %p\n", outBlock);
1766 printf("aes->key = %p\n", aes->key);
1767 printf("aes->rounds = %d\n", aes->rounds);
1768 printf("sz = %d\n", AES_BLOCK_SIZE);
1769 #endif
1770
1771 /* check alignment, decrypt doesn't need alignment */
1772 if ((wc_ptr_t)inBlock % AESNI_ALIGN) {
1773 #ifndef NO_WOLFSSL_ALLOC_ALIGN
1774 byte* tmp = (byte*)XMALLOC(AES_BLOCK_SIZE + AESNI_ALIGN, aes->heap,
1775 DYNAMIC_TYPE_TMP_BUFFER);
1776 byte* tmp_align;
1777 if (tmp == NULL) return;
1778
1779 tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN));
1780
1781 XMEMCPY(tmp_align, inBlock, AES_BLOCK_SIZE);
1782 AES_ECB_encrypt(tmp_align, tmp_align, AES_BLOCK_SIZE,
1783 (byte*)aes->key, aes->rounds);
1784 XMEMCPY(outBlock, tmp_align, AES_BLOCK_SIZE);
1785 XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
1786 return;
1787 #else
1788 WOLFSSL_MSG("AES-ECB encrypt with bad alignment");
1789 return;
1790 #endif
1791 }
1792
1793 AES_ECB_encrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key,
1794 aes->rounds);
1795
1796 return;
1797 }
1798 else {
1799 #ifdef DEBUG_AESNI
1800 printf("Skipping AES-NI\n");
1801 #endif
1802 }
1803 #endif
1804 #if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
1805 AES_ECB_encrypt(aes, inBlock, outBlock, AES_BLOCK_SIZE);
1806 return;
1807 #endif
1808
1809 #if defined(WOLFSSL_IMXRT_DCP)
1810 if (aes->keylen == 16) {
1811 DCPAesEcbEncrypt(aes, outBlock, inBlock, AES_BLOCK_SIZE);
1812 return;
1813 }
1814 #endif
1815
1816 /*
1817 * map byte array block to cipher state
1818 * and add initial round key:
1819 */
1820 XMEMCPY(&s0, inBlock, sizeof(s0));
1821 XMEMCPY(&s1, inBlock + sizeof(s0), sizeof(s1));
1822 XMEMCPY(&s2, inBlock + 2 * sizeof(s0), sizeof(s2));
1823 XMEMCPY(&s3, inBlock + 3 * sizeof(s0), sizeof(s3));
1824
1825 #ifdef LITTLE_ENDIAN_ORDER
1826 s0 = ByteReverseWord32(s0);
1827 s1 = ByteReverseWord32(s1);
1828 s2 = ByteReverseWord32(s2);
1829 s3 = ByteReverseWord32(s3);
1830 #endif
1831
1832 /* AddRoundKey */
1833 s0 ^= rk[0];
1834 s1 ^= rk[1];
1835 s2 ^= rk[2];
1836 s3 ^= rk[3];
1837
1838 #ifndef WOLFSSL_AES_SMALL_TABLES
1839 #ifndef WC_NO_CACHE_RESISTANT
1840 s0 |= PreFetchTe();
1841 #endif
1842
1843 #ifndef WOLFSSL_AES_NO_UNROLL
1844 /* Unroll the loop. */
1845 #define ENC_ROUND_T_S(o) \
1846 t0 = Te[0][GETBYTE(s0, 3)] ^ Te[1][GETBYTE(s1, 2)] ^ \
1847 Te[2][GETBYTE(s2, 1)] ^ Te[3][GETBYTE(s3, 0)] ^ rk[o+4]; \
1848 t1 = Te[0][GETBYTE(s1, 3)] ^ Te[1][GETBYTE(s2, 2)] ^ \
1849 Te[2][GETBYTE(s3, 1)] ^ Te[3][GETBYTE(s0, 0)] ^ rk[o+5]; \
1850 t2 = Te[0][GETBYTE(s2, 3)] ^ Te[1][GETBYTE(s3, 2)] ^ \
1851 Te[2][GETBYTE(s0, 1)] ^ Te[3][GETBYTE(s1, 0)] ^ rk[o+6]; \
1852 t3 = Te[0][GETBYTE(s3, 3)] ^ Te[1][GETBYTE(s0, 2)] ^ \
1853 Te[2][GETBYTE(s1, 1)] ^ Te[3][GETBYTE(s2, 0)] ^ rk[o+7]
1854 #define ENC_ROUND_S_T(o) \
1855 s0 = Te[0][GETBYTE(t0, 3)] ^ Te[1][GETBYTE(t1, 2)] ^ \
1856 Te[2][GETBYTE(t2, 1)] ^ Te[3][GETBYTE(t3, 0)] ^ rk[o+0]; \
1857 s1 = Te[0][GETBYTE(t1, 3)] ^ Te[1][GETBYTE(t2, 2)] ^ \
1858 Te[2][GETBYTE(t3, 1)] ^ Te[3][GETBYTE(t0, 0)] ^ rk[o+1]; \
1859 s2 = Te[0][GETBYTE(t2, 3)] ^ Te[1][GETBYTE(t3, 2)] ^ \
1860 Te[2][GETBYTE(t0, 1)] ^ Te[3][GETBYTE(t1, 0)] ^ rk[o+2]; \
1861 s3 = Te[0][GETBYTE(t3, 3)] ^ Te[1][GETBYTE(t0, 2)] ^ \
1862 Te[2][GETBYTE(t1, 1)] ^ Te[3][GETBYTE(t2, 0)] ^ rk[o+3]
1863
1864 ENC_ROUND_T_S( 0);
1865 ENC_ROUND_S_T( 8); ENC_ROUND_T_S( 8);
1866 ENC_ROUND_S_T(16); ENC_ROUND_T_S(16);
1867 ENC_ROUND_S_T(24); ENC_ROUND_T_S(24);
1868 ENC_ROUND_S_T(32); ENC_ROUND_T_S(32);
1869 if (r > 5) {
1870 ENC_ROUND_S_T(40); ENC_ROUND_T_S(40);
1871 if (r > 6) {
1872 ENC_ROUND_S_T(48); ENC_ROUND_T_S(48);
1873 }
1874 }
1875 rk += r * 8;
1876 #else
1877 /*
1878 * Nr - 1 full rounds:
1879 */
1880
1881 for (;;) {
1882 t0 =
1883 Te[0][GETBYTE(s0, 3)] ^
1884 Te[1][GETBYTE(s1, 2)] ^
1885 Te[2][GETBYTE(s2, 1)] ^
1886 Te[3][GETBYTE(s3, 0)] ^
1887 rk[4];
1888 t1 =
1889 Te[0][GETBYTE(s1, 3)] ^
1890 Te[1][GETBYTE(s2, 2)] ^
1891 Te[2][GETBYTE(s3, 1)] ^
1892 Te[3][GETBYTE(s0, 0)] ^
1893 rk[5];
1894 t2 =
1895 Te[0][GETBYTE(s2, 3)] ^
1896 Te[1][GETBYTE(s3, 2)] ^
1897 Te[2][GETBYTE(s0, 1)] ^
1898 Te[3][GETBYTE(s1, 0)] ^
1899 rk[6];
1900 t3 =
1901 Te[0][GETBYTE(s3, 3)] ^
1902 Te[1][GETBYTE(s0, 2)] ^
1903 Te[2][GETBYTE(s1, 1)] ^
1904 Te[3][GETBYTE(s2, 0)] ^
1905 rk[7];
1906
1907 rk += 8;
1908 if (--r == 0) {
1909 break;
1910 }
1911
1912 s0 =
1913 Te[0][GETBYTE(t0, 3)] ^
1914 Te[1][GETBYTE(t1, 2)] ^
1915 Te[2][GETBYTE(t2, 1)] ^
1916 Te[3][GETBYTE(t3, 0)] ^
1917 rk[0];
1918 s1 =
1919 Te[0][GETBYTE(t1, 3)] ^
1920 Te[1][GETBYTE(t2, 2)] ^
1921 Te[2][GETBYTE(t3, 1)] ^
1922 Te[3][GETBYTE(t0, 0)] ^
1923 rk[1];
1924 s2 =
1925 Te[0][GETBYTE(t2, 3)] ^
1926 Te[1][GETBYTE(t3, 2)] ^
1927 Te[2][GETBYTE(t0, 1)] ^
1928 Te[3][GETBYTE(t1, 0)] ^
1929 rk[2];
1930 s3 =
1931 Te[0][GETBYTE(t3, 3)] ^
1932 Te[1][GETBYTE(t0, 2)] ^
1933 Te[2][GETBYTE(t1, 1)] ^
1934 Te[3][GETBYTE(t2, 0)] ^
1935 rk[3];
1936 }
1937 #endif
1938
1939 /*
1940 * apply last round and
1941 * map cipher state to byte array block:
1942 */
1943
1944 s0 =
1945 (Te[2][GETBYTE(t0, 3)] & 0xff000000) ^
1946 (Te[3][GETBYTE(t1, 2)] & 0x00ff0000) ^
1947 (Te[0][GETBYTE(t2, 1)] & 0x0000ff00) ^
1948 (Te[1][GETBYTE(t3, 0)] & 0x000000ff) ^
1949 rk[0];
1950 s1 =
1951 (Te[2][GETBYTE(t1, 3)] & 0xff000000) ^
1952 (Te[3][GETBYTE(t2, 2)] & 0x00ff0000) ^
1953 (Te[0][GETBYTE(t3, 1)] & 0x0000ff00) ^
1954 (Te[1][GETBYTE(t0, 0)] & 0x000000ff) ^
1955 rk[1];
1956 s2 =
1957 (Te[2][GETBYTE(t2, 3)] & 0xff000000) ^
1958 (Te[3][GETBYTE(t3, 2)] & 0x00ff0000) ^
1959 (Te[0][GETBYTE(t0, 1)] & 0x0000ff00) ^
1960 (Te[1][GETBYTE(t1, 0)] & 0x000000ff) ^
1961 rk[2];
1962 s3 =
1963 (Te[2][GETBYTE(t3, 3)] & 0xff000000) ^
1964 (Te[3][GETBYTE(t0, 2)] & 0x00ff0000) ^
1965 (Te[0][GETBYTE(t1, 1)] & 0x0000ff00) ^
1966 (Te[1][GETBYTE(t2, 0)] & 0x000000ff) ^
1967 rk[3];
1968 #else
1969 #ifndef WC_NO_CACHE_RESISTANT
1970 s0 |= PreFetchSBox();
1971 #endif
1972
1973 r *= 2;
1974 /* Two rounds at a time */
1975 for (rk += 4; r > 1; r--, rk += 4) {
1976 t0 =
1977 ((word32)Tsbox[GETBYTE(s0, 3)] << 24) ^
1978 ((word32)Tsbox[GETBYTE(s1, 2)] << 16) ^
1979 ((word32)Tsbox[GETBYTE(s2, 1)] << 8) ^
1980 ((word32)Tsbox[GETBYTE(s3, 0)]);
1981 t1 =
1982 ((word32)Tsbox[GETBYTE(s1, 3)] << 24) ^
1983 ((word32)Tsbox[GETBYTE(s2, 2)] << 16) ^
1984 ((word32)Tsbox[GETBYTE(s3, 1)] << 8) ^
1985 ((word32)Tsbox[GETBYTE(s0, 0)]);
1986 t2 =
1987 ((word32)Tsbox[GETBYTE(s2, 3)] << 24) ^
1988 ((word32)Tsbox[GETBYTE(s3, 2)] << 16) ^
1989 ((word32)Tsbox[GETBYTE(s0, 1)] << 8) ^
1990 ((word32)Tsbox[GETBYTE(s1, 0)]);
1991 t3 =
1992 ((word32)Tsbox[GETBYTE(s3, 3)] << 24) ^
1993 ((word32)Tsbox[GETBYTE(s0, 2)] << 16) ^
1994 ((word32)Tsbox[GETBYTE(s1, 1)] << 8) ^
1995 ((word32)Tsbox[GETBYTE(s2, 0)]);
1996
1997 s0 =
1998 (col_mul(t0, 3, 2, 0, 1) << 24) ^
1999 (col_mul(t0, 2, 1, 0, 3) << 16) ^
2000 (col_mul(t0, 1, 0, 2, 3) << 8) ^
2001 (col_mul(t0, 0, 3, 2, 1) ) ^
2002 rk[0];
2003 s1 =
2004 (col_mul(t1, 3, 2, 0, 1) << 24) ^
2005 (col_mul(t1, 2, 1, 0, 3) << 16) ^
2006 (col_mul(t1, 1, 0, 2, 3) << 8) ^
2007 (col_mul(t1, 0, 3, 2, 1) ) ^
2008 rk[1];
2009 s2 =
2010 (col_mul(t2, 3, 2, 0, 1) << 24) ^
2011 (col_mul(t2, 2, 1, 0, 3) << 16) ^
2012 (col_mul(t2, 1, 0, 2, 3) << 8) ^
2013 (col_mul(t2, 0, 3, 2, 1) ) ^
2014 rk[2];
2015 s3 =
2016 (col_mul(t3, 3, 2, 0, 1) << 24) ^
2017 (col_mul(t3, 2, 1, 0, 3) << 16) ^
2018 (col_mul(t3, 1, 0, 2, 3) << 8) ^
2019 (col_mul(t3, 0, 3, 2, 1) ) ^
2020 rk[3];
2021 }
2022
2023 t0 =
2024 ((word32)Tsbox[GETBYTE(s0, 3)] << 24) ^
2025 ((word32)Tsbox[GETBYTE(s1, 2)] << 16) ^
2026 ((word32)Tsbox[GETBYTE(s2, 1)] << 8) ^
2027 ((word32)Tsbox[GETBYTE(s3, 0)]);
2028 t1 =
2029 ((word32)Tsbox[GETBYTE(s1, 3)] << 24) ^
2030 ((word32)Tsbox[GETBYTE(s2, 2)] << 16) ^
2031 ((word32)Tsbox[GETBYTE(s3, 1)] << 8) ^
2032 ((word32)Tsbox[GETBYTE(s0, 0)]);
2033 t2 =
2034 ((word32)Tsbox[GETBYTE(s2, 3)] << 24) ^
2035 ((word32)Tsbox[GETBYTE(s3, 2)] << 16) ^
2036 ((word32)Tsbox[GETBYTE(s0, 1)] << 8) ^
2037 ((word32)Tsbox[GETBYTE(s1, 0)]);
2038 t3 =
2039 ((word32)Tsbox[GETBYTE(s3, 3)] << 24) ^
2040 ((word32)Tsbox[GETBYTE(s0, 2)] << 16) ^
2041 ((word32)Tsbox[GETBYTE(s1, 1)] << 8) ^
2042 ((word32)Tsbox[GETBYTE(s2, 0)]);
2043 s0 = t0 ^ rk[0];
2044 s1 = t1 ^ rk[1];
2045 s2 = t2 ^ rk[2];
2046 s3 = t3 ^ rk[3];
2047 #endif
2048
2049 /* write out */
2050 #ifdef LITTLE_ENDIAN_ORDER
2051 s0 = ByteReverseWord32(s0);
2052 s1 = ByteReverseWord32(s1);
2053 s2 = ByteReverseWord32(s2);
2054 s3 = ByteReverseWord32(s3);
2055 #endif
2056
2057 XMEMCPY(outBlock, &s0, sizeof(s0));
2058 XMEMCPY(outBlock + sizeof(s0), &s1, sizeof(s1));
2059 XMEMCPY(outBlock + 2 * sizeof(s0), &s2, sizeof(s2));
2060 XMEMCPY(outBlock + 3 * sizeof(s0), &s3, sizeof(s3));
2061
2062 }
2063 #endif /* HAVE_AES_CBC || WOLFSSL_AES_DIRECT || HAVE_AESGCM */
2064
2065 #if defined(HAVE_AES_DECRYPT)
2066 #if (defined(HAVE_AES_CBC) && !defined(WOLFSSL_DEVCRYPTO_CBC)) || \
2067 defined(WOLFSSL_AES_DIRECT)
2068
2069 #ifndef WC_NO_CACHE_RESISTANT
2070 #ifndef WOLFSSL_AES_SMALL_TABLES
2071 /* load 4 Td Tables into cache by cache line stride */
PreFetchTd(void)2072 static WC_INLINE word32 PreFetchTd(void)
2073 {
2074 word32 x = 0;
2075 int i,j;
2076
2077 for (i = 0; i < 4; i++) {
2078 /* 256 elements, each one is 4 bytes */
2079 for (j = 0; j < 256; j += WC_CACHE_LINE_SZ/4) {
2080 x &= Td[i][j];
2081 }
2082 }
2083 return x;
2084 }
2085 #endif
2086
2087 /* load Td Table4 into cache by cache line stride */
PreFetchTd4(void)2088 static WC_INLINE word32 PreFetchTd4(void)
2089 {
2090 word32 x = 0;
2091 int i;
2092
2093 for (i = 0; i < 256; i += WC_CACHE_LINE_SZ) {
2094 x &= (word32)Td4[i];
2095 }
2096 return x;
2097 }
2098 #endif
2099
2100 /* Software AES - ECB Decrypt */
wc_AesDecrypt(Aes * aes,const byte * inBlock,byte * outBlock)2101 static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
2102 {
2103 word32 s0, s1, s2, s3;
2104 word32 t0, t1, t2, t3;
2105 word32 r = aes->rounds >> 1;
2106 const word32* rk = aes->key;
2107
2108 #ifdef DEBUG_WOLFSSL
2109 if (r > 7 || r == 0) {
2110 WOLFSSL_MSG("AesDecrypt encountered improper key, set it up");
2111 return; /* stop instead of seg-faulting, set up your keys! */
2112 }
2113 #endif
2114
2115 #ifdef WOLFSSL_AESNI
2116 if (haveAESNI && aes->use_aesni) {
2117 #ifdef DEBUG_AESNI
2118 printf("about to aes decrypt\n");
2119 printf("in = %p\n", inBlock);
2120 printf("out = %p\n", outBlock);
2121 printf("aes->key = %p\n", aes->key);
2122 printf("aes->rounds = %d\n", aes->rounds);
2123 printf("sz = %d\n", AES_BLOCK_SIZE);
2124 #endif
2125
2126 /* if input and output same will overwrite input iv */
2127 if ((const byte*)aes->tmp != inBlock)
2128 XMEMCPY(aes->tmp, inBlock, AES_BLOCK_SIZE);
2129 AES_ECB_decrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key,
2130 aes->rounds);
2131 return;
2132 }
2133 else {
2134 #ifdef DEBUG_AESNI
2135 printf("Skipping AES-NI\n");
2136 #endif
2137 }
2138 #endif /* WOLFSSL_AESNI */
2139 #if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
2140 return AES_ECB_decrypt(aes, inBlock, outBlock, AES_BLOCK_SIZE);
2141 #endif
2142 #if defined(WOLFSSL_IMXRT_DCP)
2143 if (aes->keylen == 16) {
2144 DCPAesEcbDecrypt(aes, outBlock, inBlock, AES_BLOCK_SIZE);
2145 return;
2146 }
2147 #endif
2148
2149 /*
2150 * map byte array block to cipher state
2151 * and add initial round key:
2152 */
2153 XMEMCPY(&s0, inBlock, sizeof(s0));
2154 XMEMCPY(&s1, inBlock + sizeof(s0), sizeof(s1));
2155 XMEMCPY(&s2, inBlock + 2 * sizeof(s0), sizeof(s2));
2156 XMEMCPY(&s3, inBlock + 3 * sizeof(s0), sizeof(s3));
2157
2158 #ifdef LITTLE_ENDIAN_ORDER
2159 s0 = ByteReverseWord32(s0);
2160 s1 = ByteReverseWord32(s1);
2161 s2 = ByteReverseWord32(s2);
2162 s3 = ByteReverseWord32(s3);
2163 #endif
2164
2165 s0 ^= rk[0];
2166 s1 ^= rk[1];
2167 s2 ^= rk[2];
2168 s3 ^= rk[3];
2169
2170 #ifndef WOLFSSL_AES_SMALL_TABLES
2171 #ifndef WC_NO_CACHE_RESISTANT
2172 s0 |= PreFetchTd();
2173 #endif
2174
2175 #ifndef WOLFSSL_AES_NO_UNROLL
2176 /* Unroll the loop. */
2177 #define DEC_ROUND_T_S(o) \
2178 t0 = Td[0][GETBYTE(s0, 3)] ^ Td[1][GETBYTE(s3, 2)] ^ \
2179 Td[2][GETBYTE(s2, 1)] ^ Td[3][GETBYTE(s1, 0)] ^ rk[o+4]; \
2180 t1 = Td[0][GETBYTE(s1, 3)] ^ Td[1][GETBYTE(s0, 2)] ^ \
2181 Td[2][GETBYTE(s3, 1)] ^ Td[3][GETBYTE(s2, 0)] ^ rk[o+5]; \
2182 t2 = Td[0][GETBYTE(s2, 3)] ^ Td[1][GETBYTE(s1, 2)] ^ \
2183 Td[2][GETBYTE(s0, 1)] ^ Td[3][GETBYTE(s3, 0)] ^ rk[o+6]; \
2184 t3 = Td[0][GETBYTE(s3, 3)] ^ Td[1][GETBYTE(s2, 2)] ^ \
2185 Td[2][GETBYTE(s1, 1)] ^ Td[3][GETBYTE(s0, 0)] ^ rk[o+7]
2186 #define DEC_ROUND_S_T(o) \
2187 s0 = Td[0][GETBYTE(t0, 3)] ^ Td[1][GETBYTE(t3, 2)] ^ \
2188 Td[2][GETBYTE(t2, 1)] ^ Td[3][GETBYTE(t1, 0)] ^ rk[o+0]; \
2189 s1 = Td[0][GETBYTE(t1, 3)] ^ Td[1][GETBYTE(t0, 2)] ^ \
2190 Td[2][GETBYTE(t3, 1)] ^ Td[3][GETBYTE(t2, 0)] ^ rk[o+1]; \
2191 s2 = Td[0][GETBYTE(t2, 3)] ^ Td[1][GETBYTE(t1, 2)] ^ \
2192 Td[2][GETBYTE(t0, 1)] ^ Td[3][GETBYTE(t3, 0)] ^ rk[o+2]; \
2193 s3 = Td[0][GETBYTE(t3, 3)] ^ Td[1][GETBYTE(t2, 2)] ^ \
2194 Td[2][GETBYTE(t1, 1)] ^ Td[3][GETBYTE(t0, 0)] ^ rk[o+3]
2195
2196 DEC_ROUND_T_S( 0);
2197 DEC_ROUND_S_T( 8); DEC_ROUND_T_S( 8);
2198 DEC_ROUND_S_T(16); DEC_ROUND_T_S(16);
2199 DEC_ROUND_S_T(24); DEC_ROUND_T_S(24);
2200 DEC_ROUND_S_T(32); DEC_ROUND_T_S(32);
2201 if (r > 5) {
2202 DEC_ROUND_S_T(40); DEC_ROUND_T_S(40);
2203 if (r > 6) {
2204 DEC_ROUND_S_T(48); DEC_ROUND_T_S(48);
2205 }
2206 }
2207 rk += r * 8;
2208 #else
2209
2210 /*
2211 * Nr - 1 full rounds:
2212 */
2213
2214 for (;;) {
2215 t0 =
2216 Td[0][GETBYTE(s0, 3)] ^
2217 Td[1][GETBYTE(s3, 2)] ^
2218 Td[2][GETBYTE(s2, 1)] ^
2219 Td[3][GETBYTE(s1, 0)] ^
2220 rk[4];
2221 t1 =
2222 Td[0][GETBYTE(s1, 3)] ^
2223 Td[1][GETBYTE(s0, 2)] ^
2224 Td[2][GETBYTE(s3, 1)] ^
2225 Td[3][GETBYTE(s2, 0)] ^
2226 rk[5];
2227 t2 =
2228 Td[0][GETBYTE(s2, 3)] ^
2229 Td[1][GETBYTE(s1, 2)] ^
2230 Td[2][GETBYTE(s0, 1)] ^
2231 Td[3][GETBYTE(s3, 0)] ^
2232 rk[6];
2233 t3 =
2234 Td[0][GETBYTE(s3, 3)] ^
2235 Td[1][GETBYTE(s2, 2)] ^
2236 Td[2][GETBYTE(s1, 1)] ^
2237 Td[3][GETBYTE(s0, 0)] ^
2238 rk[7];
2239
2240 rk += 8;
2241 if (--r == 0) {
2242 break;
2243 }
2244
2245 s0 =
2246 Td[0][GETBYTE(t0, 3)] ^
2247 Td[1][GETBYTE(t3, 2)] ^
2248 Td[2][GETBYTE(t2, 1)] ^
2249 Td[3][GETBYTE(t1, 0)] ^
2250 rk[0];
2251 s1 =
2252 Td[0][GETBYTE(t1, 3)] ^
2253 Td[1][GETBYTE(t0, 2)] ^
2254 Td[2][GETBYTE(t3, 1)] ^
2255 Td[3][GETBYTE(t2, 0)] ^
2256 rk[1];
2257 s2 =
2258 Td[0][GETBYTE(t2, 3)] ^
2259 Td[1][GETBYTE(t1, 2)] ^
2260 Td[2][GETBYTE(t0, 1)] ^
2261 Td[3][GETBYTE(t3, 0)] ^
2262 rk[2];
2263 s3 =
2264 Td[0][GETBYTE(t3, 3)] ^
2265 Td[1][GETBYTE(t2, 2)] ^
2266 Td[2][GETBYTE(t1, 1)] ^
2267 Td[3][GETBYTE(t0, 0)] ^
2268 rk[3];
2269 }
2270 #endif
2271 /*
2272 * apply last round and
2273 * map cipher state to byte array block:
2274 */
2275
2276 #ifndef WC_NO_CACHE_RESISTANT
2277 t0 |= PreFetchTd4();
2278 #endif
2279
2280 s0 =
2281 ((word32)Td4[GETBYTE(t0, 3)] << 24) ^
2282 ((word32)Td4[GETBYTE(t3, 2)] << 16) ^
2283 ((word32)Td4[GETBYTE(t2, 1)] << 8) ^
2284 ((word32)Td4[GETBYTE(t1, 0)]) ^
2285 rk[0];
2286 s1 =
2287 ((word32)Td4[GETBYTE(t1, 3)] << 24) ^
2288 ((word32)Td4[GETBYTE(t0, 2)] << 16) ^
2289 ((word32)Td4[GETBYTE(t3, 1)] << 8) ^
2290 ((word32)Td4[GETBYTE(t2, 0)]) ^
2291 rk[1];
2292 s2 =
2293 ((word32)Td4[GETBYTE(t2, 3)] << 24) ^
2294 ((word32)Td4[GETBYTE(t1, 2)] << 16) ^
2295 ((word32)Td4[GETBYTE(t0, 1)] << 8) ^
2296 ((word32)Td4[GETBYTE(t3, 0)]) ^
2297 rk[2];
2298 s3 =
2299 ((word32)Td4[GETBYTE(t3, 3)] << 24) ^
2300 ((word32)Td4[GETBYTE(t2, 2)] << 16) ^
2301 ((word32)Td4[GETBYTE(t1, 1)] << 8) ^
2302 ((word32)Td4[GETBYTE(t0, 0)]) ^
2303 rk[3];
2304 #else
2305 #ifndef WC_NO_CACHE_RESISTANT
2306 s0 |= PreFetchTd4();
2307 #endif
2308
2309 r *= 2;
2310 for (rk += 4; r > 1; r--, rk += 4) {
2311 t0 =
2312 ((word32)Td4[GETBYTE(s0, 3)] << 24) ^
2313 ((word32)Td4[GETBYTE(s3, 2)] << 16) ^
2314 ((word32)Td4[GETBYTE(s2, 1)] << 8) ^
2315 ((word32)Td4[GETBYTE(s1, 0)]) ^
2316 rk[0];
2317 t1 =
2318 ((word32)Td4[GETBYTE(s1, 3)] << 24) ^
2319 ((word32)Td4[GETBYTE(s0, 2)] << 16) ^
2320 ((word32)Td4[GETBYTE(s3, 1)] << 8) ^
2321 ((word32)Td4[GETBYTE(s2, 0)]) ^
2322 rk[1];
2323 t2 =
2324 ((word32)Td4[GETBYTE(s2, 3)] << 24) ^
2325 ((word32)Td4[GETBYTE(s1, 2)] << 16) ^
2326 ((word32)Td4[GETBYTE(s0, 1)] << 8) ^
2327 ((word32)Td4[GETBYTE(s3, 0)]) ^
2328 rk[2];
2329 t3 =
2330 ((word32)Td4[GETBYTE(s3, 3)] << 24) ^
2331 ((word32)Td4[GETBYTE(s2, 2)] << 16) ^
2332 ((word32)Td4[GETBYTE(s1, 1)] << 8) ^
2333 ((word32)Td4[GETBYTE(s0, 0)]) ^
2334 rk[3];
2335
2336 s0 =
2337 (inv_col_mul(t0, 0, 2, 1, 3) << 24) ^
2338 (inv_col_mul(t0, 3, 1, 0, 2) << 16) ^
2339 (inv_col_mul(t0, 2, 0, 3, 1) << 8) ^
2340 (inv_col_mul(t0, 1, 3, 2, 0) );
2341 s1 =
2342 (inv_col_mul(t1, 0, 2, 1, 3) << 24) ^
2343 (inv_col_mul(t1, 3, 1, 0, 2) << 16) ^
2344 (inv_col_mul(t1, 2, 0, 3, 1) << 8) ^
2345 (inv_col_mul(t1, 1, 3, 2, 0) );
2346 s2 =
2347 (inv_col_mul(t2, 0, 2, 1, 3) << 24) ^
2348 (inv_col_mul(t2, 3, 1, 0, 2) << 16) ^
2349 (inv_col_mul(t2, 2, 0, 3, 1) << 8) ^
2350 (inv_col_mul(t2, 1, 3, 2, 0) );
2351 s3 =
2352 (inv_col_mul(t3, 0, 2, 1, 3) << 24) ^
2353 (inv_col_mul(t3, 3, 1, 0, 2) << 16) ^
2354 (inv_col_mul(t3, 2, 0, 3, 1) << 8) ^
2355 (inv_col_mul(t3, 1, 3, 2, 0) );
2356 }
2357
2358 t0 =
2359 ((word32)Td4[GETBYTE(s0, 3)] << 24) ^
2360 ((word32)Td4[GETBYTE(s3, 2)] << 16) ^
2361 ((word32)Td4[GETBYTE(s2, 1)] << 8) ^
2362 ((word32)Td4[GETBYTE(s1, 0)]);
2363 t1 =
2364 ((word32)Td4[GETBYTE(s1, 3)] << 24) ^
2365 ((word32)Td4[GETBYTE(s0, 2)] << 16) ^
2366 ((word32)Td4[GETBYTE(s3, 1)] << 8) ^
2367 ((word32)Td4[GETBYTE(s2, 0)]);
2368 t2 =
2369 ((word32)Td4[GETBYTE(s2, 3)] << 24) ^
2370 ((word32)Td4[GETBYTE(s1, 2)] << 16) ^
2371 ((word32)Td4[GETBYTE(s0, 1)] << 8) ^
2372 ((word32)Td4[GETBYTE(s3, 0)]);
2373 t3 =
2374 ((word32)Td4[GETBYTE(s3, 3)] << 24) ^
2375 ((word32)Td4[GETBYTE(s2, 2)] << 16) ^
2376 ((word32)Td4[GETBYTE(s1, 1)] << 8) ^
2377 ((word32)Td4[GETBYTE(s0, 0)]);
2378 s0 = t0 ^ rk[0];
2379 s1 = t1 ^ rk[1];
2380 s2 = t2 ^ rk[2];
2381 s3 = t3 ^ rk[3];
2382 #endif
2383
2384 /* write out */
2385 #ifdef LITTLE_ENDIAN_ORDER
2386 s0 = ByteReverseWord32(s0);
2387 s1 = ByteReverseWord32(s1);
2388 s2 = ByteReverseWord32(s2);
2389 s3 = ByteReverseWord32(s3);
2390 #endif
2391
2392 XMEMCPY(outBlock, &s0, sizeof(s0));
2393 XMEMCPY(outBlock + sizeof(s0), &s1, sizeof(s1));
2394 XMEMCPY(outBlock + 2 * sizeof(s0), &s2, sizeof(s2));
2395 XMEMCPY(outBlock + 3 * sizeof(s0), &s3, sizeof(s3));
2396 }
2397 #endif /* HAVE_AES_CBC || WOLFSSL_AES_DIRECT */
2398 #endif /* HAVE_AES_DECRYPT */
2399
2400 #endif /* NEED_AES_TABLES */
2401
2402
2403
2404 /* wc_AesSetKey */
2405 #if defined(STM32_CRYPTO)
2406
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2407 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
2408 const byte* iv, int dir)
2409 {
2410 word32 *rk;
2411
2412 (void)dir;
2413
2414 if (aes == NULL || (keylen != 16 &&
2415 #ifdef WOLFSSL_AES_192
2416 keylen != 24 &&
2417 #endif
2418 keylen != 32)) {
2419 return BAD_FUNC_ARG;
2420 }
2421
2422 rk = aes->key;
2423 aes->keylen = keylen;
2424 aes->rounds = keylen/4 + 6;
2425 XMEMCPY(rk, userKey, keylen);
2426 #if !defined(WOLFSSL_STM32_CUBEMX) || defined(STM32_HAL_V2)
2427 ByteReverseWords(rk, rk, keylen);
2428 #endif
2429 #if defined(WOLFSSL_AES_CFB) || defined(WOLFSSL_AES_COUNTER) || \
2430 defined(WOLFSSL_AES_OFB)
2431 aes->left = 0;
2432 #endif
2433 return wc_AesSetIV(aes, iv);
2434 }
2435 #if defined(WOLFSSL_AES_DIRECT)
wc_AesSetKeyDirect(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2436 int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
2437 const byte* iv, int dir)
2438 {
2439 return wc_AesSetKey(aes, userKey, keylen, iv, dir);
2440 }
2441 #endif
2442
2443 #elif defined(HAVE_COLDFIRE_SEC)
2444 #if defined (HAVE_THREADX)
2445 #include "memory_pools.h"
2446 extern TX_BYTE_POOL mp_ncached; /* Non Cached memory pool */
2447 #endif
2448
2449 #define AES_BUFFER_SIZE (AES_BLOCK_SIZE * 64)
2450 static unsigned char *AESBuffIn = NULL;
2451 static unsigned char *AESBuffOut = NULL;
2452 static byte *secReg;
2453 static byte *secKey;
2454 static volatile SECdescriptorType *secDesc;
2455
2456 static wolfSSL_Mutex Mutex_AesSEC;
2457
2458 #define SEC_DESC_AES_CBC_ENCRYPT 0x60300010
2459 #define SEC_DESC_AES_CBC_DECRYPT 0x60200010
2460
2461 extern volatile unsigned char __MBAR[];
2462
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2463 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
2464 const byte* iv, int dir)
2465 {
2466 if (AESBuffIn == NULL) {
2467 #if defined (HAVE_THREADX)
2468 int s1, s2, s3, s4, s5;
2469 s5 = tx_byte_allocate(&mp_ncached,(void *)&secDesc,
2470 sizeof(SECdescriptorType), TX_NO_WAIT);
2471 s1 = tx_byte_allocate(&mp_ncached, (void *)&AESBuffIn,
2472 AES_BUFFER_SIZE, TX_NO_WAIT);
2473 s2 = tx_byte_allocate(&mp_ncached, (void *)&AESBuffOut,
2474 AES_BUFFER_SIZE, TX_NO_WAIT);
2475 s3 = tx_byte_allocate(&mp_ncached, (void *)&secKey,
2476 AES_BLOCK_SIZE*2, TX_NO_WAIT);
2477 s4 = tx_byte_allocate(&mp_ncached, (void *)&secReg,
2478 AES_BLOCK_SIZE, TX_NO_WAIT);
2479
2480 if (s1 || s2 || s3 || s4 || s5)
2481 return BAD_FUNC_ARG;
2482 #else
2483 #warning "Allocate non-Cache buffers"
2484 #endif
2485
2486 wc_InitMutex(&Mutex_AesSEC);
2487 }
2488
2489 if (!((keylen == 16) || (keylen == 24) || (keylen == 32)))
2490 return BAD_FUNC_ARG;
2491
2492 if (aes == NULL)
2493 return BAD_FUNC_ARG;
2494
2495 aes->keylen = keylen;
2496 aes->rounds = keylen/4 + 6;
2497 XMEMCPY(aes->key, userKey, keylen);
2498
2499 if (iv)
2500 XMEMCPY(aes->reg, iv, AES_BLOCK_SIZE);
2501
2502 #if defined(WOLFSSL_AES_CFB) || defined(WOLFSSL_AES_COUNTER) || \
2503 defined(WOLFSSL_AES_OFB)
2504 aes->left = 0;
2505 #endif
2506
2507 return 0;
2508 }
2509 #elif defined(FREESCALE_LTC)
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2510 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen, const byte* iv,
2511 int dir)
2512 {
2513 if (aes == NULL || !((keylen == 16) || (keylen == 24) || (keylen == 32)))
2514 return BAD_FUNC_ARG;
2515
2516 aes->rounds = keylen/4 + 6;
2517 XMEMCPY(aes->key, userKey, keylen);
2518
2519 #if defined(WOLFSSL_AES_CFB) || defined(WOLFSSL_AES_COUNTER) || \
2520 defined(WOLFSSL_AES_OFB)
2521 aes->left = 0;
2522 #endif
2523
2524 return wc_AesSetIV(aes, iv);
2525 }
2526
wc_AesSetKeyDirect(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2527 int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
2528 const byte* iv, int dir)
2529 {
2530 return wc_AesSetKey(aes, userKey, keylen, iv, dir);
2531 }
2532 #elif defined(FREESCALE_MMCAU)
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2533 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
2534 const byte* iv, int dir)
2535 {
2536 int ret;
2537 byte* rk;
2538 byte* tmpKey = (byte*)userKey;
2539 int tmpKeyDynamic = 0;
2540 word32 alignOffset = 0;
2541
2542 (void)dir;
2543
2544 if (!((keylen == 16) || (keylen == 24) || (keylen == 32)))
2545 return BAD_FUNC_ARG;
2546 if (aes == NULL)
2547 return BAD_FUNC_ARG;
2548
2549 rk = (byte*)aes->key;
2550 if (rk == NULL)
2551 return BAD_FUNC_ARG;
2552
2553 #if defined(WOLFSSL_AES_CFB) || defined(WOLFSSL_AES_COUNTER) || \
2554 defined(WOLFSSL_AES_OFB)
2555 aes->left = 0;
2556 #endif
2557
2558 aes->rounds = keylen/4 + 6;
2559
2560 #ifdef FREESCALE_MMCAU_CLASSIC
2561 if ((wc_ptr_t)userKey % WOLFSSL_MMCAU_ALIGNMENT) {
2562 #ifndef NO_WOLFSSL_ALLOC_ALIGN
2563 byte* tmp = (byte*)XMALLOC(keylen + WOLFSSL_MMCAU_ALIGNMENT,
2564 aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
2565 if (tmp == NULL) {
2566 return MEMORY_E;
2567 }
2568 alignOffset = WOLFSSL_MMCAU_ALIGNMENT -
2569 ((wc_ptr_t)tmp % WOLFSSL_MMCAU_ALIGNMENT);
2570 tmpKey = tmp + alignOffset;
2571 XMEMCPY(tmpKey, userKey, keylen);
2572 tmpKeyDynamic = 1;
2573 #else
2574 WOLFSSL_MSG("Bad cau_aes_set_key alignment");
2575 return BAD_ALIGN_E;
2576 #endif
2577 }
2578 #endif
2579
2580 ret = wolfSSL_CryptHwMutexLock();
2581 if(ret == 0) {
2582 #ifdef FREESCALE_MMCAU_CLASSIC
2583 cau_aes_set_key(tmpKey, keylen*8, rk);
2584 #else
2585 MMCAU_AES_SetKey(tmpKey, keylen, rk);
2586 #endif
2587 wolfSSL_CryptHwMutexUnLock();
2588
2589 ret = wc_AesSetIV(aes, iv);
2590 }
2591
2592 if (tmpKeyDynamic == 1) {
2593 XFREE(tmpKey - alignOffset, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
2594 }
2595
2596 return ret;
2597 }
2598
wc_AesSetKeyDirect(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2599 int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
2600 const byte* iv, int dir)
2601 {
2602 return wc_AesSetKey(aes, userKey, keylen, iv, dir);
2603 }
2604
2605 #elif defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2606 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen, const byte* iv,
2607 int dir)
2608 {
2609 int ret;
2610
2611 if (aes == NULL || (keylen != 16 && keylen != 24 && keylen != 32)) {
2612 return BAD_FUNC_ARG;
2613 }
2614
2615 aes->ctxInitDone = 0;
2616 #if defined(WOLFSSL_AES_CFB) || defined(WOLFSSL_AES_COUNTER) || \
2617 defined(WOLFSSL_AES_OFB)
2618 aes->left = 0;
2619 #endif
2620
2621 ret = se050_aes_set_key(aes, userKey, keylen, iv, dir);
2622 if (ret == 0) {
2623 ret = wc_AesSetIV(aes, iv);
2624 }
2625 return ret;
2626 }
2627
wc_AesSetKeyDirect(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2628 int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
2629 const byte* iv, int dir)
2630 {
2631 return wc_AesSetKey(aes, userKey, keylen, iv, dir);
2632 }
2633
2634 #elif defined(WOLFSSL_NRF51_AES)
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2635 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
2636 const byte* iv, int dir)
2637 {
2638 int ret;
2639
2640 (void)dir;
2641 (void)iv;
2642
2643 if (aes == NULL || keylen != 16)
2644 return BAD_FUNC_ARG;
2645
2646 aes->keylen = keylen;
2647 aes->rounds = keylen/4 + 6;
2648 XMEMCPY(aes->key, userKey, keylen);
2649 ret = nrf51_aes_set_key(userKey);
2650
2651 #if defined(WOLFSSL_AES_CFB) || defined(WOLFSSL_AES_COUNTER) || \
2652 defined(WOLFSSL_AES_OFB)
2653 aes->left = 0;
2654 #endif
2655
2656 return ret;
2657 }
2658
wc_AesSetKeyDirect(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2659 int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
2660 const byte* iv, int dir)
2661 {
2662 return wc_AesSetKey(aes, userKey, keylen, iv, dir);
2663 }
2664 #elif defined(WOLFSSL_ESP32WROOM32_CRYPT) && \
2665 !defined(NO_WOLFSSL_ESP32WROOM32_CRYPT_AES)
2666
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2667 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
2668 const byte* iv, int dir)
2669 {
2670 (void)dir;
2671 (void)iv;
2672
2673 if (aes == NULL || (keylen != 16 && keylen != 24 && keylen != 32)) {
2674 return BAD_FUNC_ARG;
2675 }
2676
2677 aes->keylen = keylen;
2678 aes->rounds = keylen/4 + 6;
2679
2680 XMEMCPY(aes->key, userKey, keylen);
2681 #if defined(WOLFSSL_AES_COUNTER)
2682 aes->left = 0;
2683 #endif
2684 return wc_AesSetIV(aes, iv);
2685 }
2686
wc_AesSetKeyDirect(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2687 int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
2688 const byte* iv, int dir)
2689 {
2690 return wc_AesSetKey(aes, userKey, keylen, iv, dir);
2691 }
2692 #elif defined(WOLFSSL_CRYPTOCELL) && defined(WOLFSSL_CRYPTOCELL_AES)
2693
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2694 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen, const byte* iv,
2695 int dir)
2696 {
2697 SaSiError_t ret = SASI_OK;
2698 SaSiAesIv_t iv_aes;
2699
2700 if (aes == NULL ||
2701 (keylen != AES_128_KEY_SIZE &&
2702 keylen != AES_192_KEY_SIZE &&
2703 keylen != AES_256_KEY_SIZE)) {
2704 return BAD_FUNC_ARG;
2705 }
2706 #if defined(AES_MAX_KEY_SIZE)
2707 if (keylen > (AES_MAX_KEY_SIZE/8)) {
2708 return BAD_FUNC_ARG;
2709 }
2710 #endif
2711 if (dir != AES_ENCRYPTION &&
2712 dir != AES_DECRYPTION) {
2713 return BAD_FUNC_ARG;
2714 }
2715
2716 if (dir == AES_ENCRYPTION) {
2717 aes->ctx.mode = SASI_AES_ENCRYPT;
2718 SaSi_AesInit(&aes->ctx.user_ctx,
2719 SASI_AES_ENCRYPT,
2720 SASI_AES_MODE_CBC,
2721 SASI_AES_PADDING_NONE);
2722 }
2723 else {
2724 aes->ctx.mode = SASI_AES_DECRYPT;
2725 SaSi_AesInit(&aes->ctx.user_ctx,
2726 SASI_AES_DECRYPT,
2727 SASI_AES_MODE_CBC,
2728 SASI_AES_PADDING_NONE);
2729 }
2730
2731 aes->keylen = keylen;
2732 aes->rounds = keylen/4 + 6;
2733 XMEMCPY(aes->key, userKey, keylen);
2734
2735 aes->ctx.key.pKey = (byte*)aes->key;
2736 aes->ctx.key.keySize= keylen;
2737
2738 ret = SaSi_AesSetKey(&aes->ctx.user_ctx,
2739 SASI_AES_USER_KEY,
2740 &aes->ctx.key,
2741 sizeof(aes->ctx.key));
2742 if (ret != SASI_OK) {
2743 return BAD_FUNC_ARG;
2744 }
2745
2746 ret = wc_AesSetIV(aes, iv);
2747
2748 if (iv)
2749 XMEMCPY(iv_aes, iv, AES_BLOCK_SIZE);
2750 else
2751 XMEMSET(iv_aes, 0, AES_BLOCK_SIZE);
2752
2753
2754 ret = SaSi_AesSetIv(&aes->ctx.user_ctx, iv_aes);
2755 if (ret != SASI_OK) {
2756 return ret;
2757 }
2758 return ret;
2759 }
2760 #if defined(WOLFSSL_AES_DIRECT)
wc_AesSetKeyDirect(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)2761 int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
2762 const byte* iv, int dir)
2763 {
2764 return wc_AesSetKey(aes, userKey, keylen, iv, dir);
2765 }
2766 #endif
2767
2768 #elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) \
2769 && !defined(WOLFSSL_QNX_CAAM)
2770 /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
2771
2772 #elif defined(WOLFSSL_AFALG)
2773 /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
2774
2775 #elif defined(WOLFSSL_DEVCRYPTO_AES)
2776 /* implemented in wolfcrypt/src/port/devcrypto/devcrypto_aes.c */
2777
2778 #elif defined(WOLFSSL_SILABS_SE_ACCEL)
2779 /* implemented in wolfcrypt/src/port/silabs/silabs_hash.c */
2780
2781 #else
2782
2783 /* Software AES - SetKey */
wc_AesSetKeyLocal(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir,int checkKeyLen)2784 static int wc_AesSetKeyLocal(Aes* aes, const byte* userKey, word32 keylen,
2785 const byte* iv, int dir, int checkKeyLen)
2786 {
2787 int ret;
2788 word32 *rk;
2789 #ifdef NEED_AES_TABLES
2790 word32 temp;
2791 unsigned int i = 0;
2792 #endif
2793 #ifdef WOLFSSL_IMX6_CAAM_BLOB
2794 byte local[32];
2795 word32 localSz = 32;
2796 #endif
2797
2798 #ifdef WOLFSSL_IMX6_CAAM_BLOB
2799 if (keylen == (16 + WC_CAAM_BLOB_SZ) ||
2800 keylen == (24 + WC_CAAM_BLOB_SZ) ||
2801 keylen == (32 + WC_CAAM_BLOB_SZ)) {
2802 if (wc_caamOpenBlob((byte*)userKey, keylen, local, &localSz) != 0) {
2803 return BAD_FUNC_ARG;
2804 }
2805
2806 /* set local values */
2807 userKey = local;
2808 keylen = localSz;
2809 }
2810 #endif
2811
2812 #if defined(WOLF_CRYPTO_CB) || (defined(WOLFSSL_DEVCRYPTO) && \
2813 (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))) || \
2814 (defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES))
2815 #ifdef WOLF_CRYPTO_CB
2816 if (aes->devId != INVALID_DEVID)
2817 #endif
2818 {
2819 if (keylen > sizeof(aes->devKey)) {
2820 return BAD_FUNC_ARG;
2821 }
2822 XMEMCPY(aes->devKey, userKey, keylen);
2823 }
2824 #endif
2825
2826 if (checkKeyLen) {
2827 if (keylen != 16 && keylen != 24 && keylen != 32) {
2828 return BAD_FUNC_ARG;
2829 }
2830 #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE < 256
2831 /* Check key length only when AES_MAX_KEY_SIZE doesn't allow
2832 * all key sizes. Otherwise this condition is never true. */
2833 if (keylen > (AES_MAX_KEY_SIZE / 8)) {
2834 return BAD_FUNC_ARG;
2835 }
2836 #endif
2837 }
2838
2839 #if defined(WOLFSSL_AES_CFB) || defined(WOLFSSL_AES_COUNTER) || \
2840 defined(WOLFSSL_AES_OFB)
2841 aes->left = 0;
2842 #endif
2843
2844 aes->keylen = keylen;
2845 aes->rounds = (keylen/4) + 6;
2846
2847 #ifdef WOLFSSL_AESNI
2848 aes->use_aesni = 0;
2849 if (checkAESNI == 0) {
2850 haveAESNI = Check_CPU_support_AES();
2851 checkAESNI = 1;
2852 }
2853 if (haveAESNI) {
2854 aes->use_aesni = 1;
2855 if (iv)
2856 XMEMCPY(aes->reg, iv, AES_BLOCK_SIZE);
2857 else
2858 XMEMSET(aes->reg, 0, AES_BLOCK_SIZE);
2859 if (dir == AES_ENCRYPTION)
2860 return AES_set_encrypt_key(userKey, keylen * 8, aes);
2861 #ifdef HAVE_AES_DECRYPT
2862 else
2863 return AES_set_decrypt_key(userKey, keylen * 8, aes);
2864 #endif
2865 }
2866 #endif /* WOLFSSL_AESNI */
2867
2868 #ifdef WOLFSSL_KCAPI_AES
2869 XMEMCPY(aes->devKey, userKey, keylen);
2870 if (aes->init != 0) {
2871 kcapi_cipher_destroy(aes->handle);
2872 aes->handle = NULL;
2873 aes->init = 0;
2874 }
2875 (void)dir;
2876 #endif
2877
2878 if (keylen > sizeof(aes->key)) {
2879 return BAD_FUNC_ARG;
2880 }
2881 rk = aes->key;
2882 XMEMCPY(rk, userKey, keylen);
2883 #if defined(LITTLE_ENDIAN_ORDER) && !defined(WOLFSSL_PIC32MZ_CRYPT) && \
2884 (!defined(WOLFSSL_ESP32WROOM32_CRYPT) || \
2885 defined(NO_WOLFSSL_ESP32WROOM32_CRYPT_AES))
2886 ByteReverseWords(rk, rk, keylen);
2887 #endif
2888
2889 #ifdef WOLFSSL_IMXRT_DCP
2890 /* Implemented in wolfcrypt/src/port/nxp/dcp_port.c */
2891 temp = 0;
2892 if (keylen == 16)
2893 temp = DCPAesSetKey(aes, userKey, keylen, iv, dir);
2894 if (temp != 0)
2895 return WC_HW_E;
2896 #endif
2897
2898 #ifdef NEED_AES_TABLES
2899 switch (keylen) {
2900 #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 128 && \
2901 defined(WOLFSSL_AES_128)
2902 case 16:
2903 while (1)
2904 {
2905 temp = rk[3];
2906 rk[4] = rk[0] ^
2907 #ifndef WOLFSSL_AES_SMALL_TABLES
2908 (Te[2][GETBYTE(temp, 2)] & 0xff000000) ^
2909 (Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^
2910 (Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^
2911 (Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^
2912 #else
2913 ((word32)Tsbox[GETBYTE(temp, 2)] << 24) ^
2914 ((word32)Tsbox[GETBYTE(temp, 1)] << 16) ^
2915 ((word32)Tsbox[GETBYTE(temp, 0)] << 8) ^
2916 ((word32)Tsbox[GETBYTE(temp, 3)]) ^
2917 #endif
2918 rcon[i];
2919 rk[5] = rk[1] ^ rk[4];
2920 rk[6] = rk[2] ^ rk[5];
2921 rk[7] = rk[3] ^ rk[6];
2922 if (++i == 10)
2923 break;
2924 rk += 4;
2925 }
2926 break;
2927 #endif /* 128 */
2928
2929 #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 192 && \
2930 defined(WOLFSSL_AES_192)
2931 case 24:
2932 /* for (;;) here triggers a bug in VC60 SP4 w/ Pro Pack */
2933 while (1)
2934 {
2935 temp = rk[ 5];
2936 rk[ 6] = rk[ 0] ^
2937 #ifndef WOLFSSL_AES_SMALL_TABLES
2938 (Te[2][GETBYTE(temp, 2)] & 0xff000000) ^
2939 (Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^
2940 (Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^
2941 (Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^
2942 #else
2943 ((word32)Tsbox[GETBYTE(temp, 2)] << 24) ^
2944 ((word32)Tsbox[GETBYTE(temp, 1)] << 16) ^
2945 ((word32)Tsbox[GETBYTE(temp, 0)] << 8) ^
2946 ((word32)Tsbox[GETBYTE(temp, 3)]) ^
2947 #endif
2948 rcon[i];
2949 rk[ 7] = rk[ 1] ^ rk[ 6];
2950 rk[ 8] = rk[ 2] ^ rk[ 7];
2951 rk[ 9] = rk[ 3] ^ rk[ 8];
2952 if (++i == 8)
2953 break;
2954 rk[10] = rk[ 4] ^ rk[ 9];
2955 rk[11] = rk[ 5] ^ rk[10];
2956 rk += 6;
2957 }
2958 break;
2959 #endif /* 192 */
2960
2961 #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 256 && \
2962 defined(WOLFSSL_AES_256)
2963 case 32:
2964 while (1)
2965 {
2966 temp = rk[ 7];
2967 rk[ 8] = rk[ 0] ^
2968 #ifndef WOLFSSL_AES_SMALL_TABLES
2969 (Te[2][GETBYTE(temp, 2)] & 0xff000000) ^
2970 (Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^
2971 (Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^
2972 (Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^
2973 #else
2974 ((word32)Tsbox[GETBYTE(temp, 2)] << 24) ^
2975 ((word32)Tsbox[GETBYTE(temp, 1)] << 16) ^
2976 ((word32)Tsbox[GETBYTE(temp, 0)] << 8) ^
2977 ((word32)Tsbox[GETBYTE(temp, 3)]) ^
2978 #endif
2979 rcon[i];
2980 rk[ 9] = rk[ 1] ^ rk[ 8];
2981 rk[10] = rk[ 2] ^ rk[ 9];
2982 rk[11] = rk[ 3] ^ rk[10];
2983 if (++i == 7)
2984 break;
2985 temp = rk[11];
2986 rk[12] = rk[ 4] ^
2987 #ifndef WOLFSSL_AES_SMALL_TABLES
2988 (Te[2][GETBYTE(temp, 3)] & 0xff000000) ^
2989 (Te[3][GETBYTE(temp, 2)] & 0x00ff0000) ^
2990 (Te[0][GETBYTE(temp, 1)] & 0x0000ff00) ^
2991 (Te[1][GETBYTE(temp, 0)] & 0x000000ff);
2992 #else
2993 ((word32)Tsbox[GETBYTE(temp, 3)] << 24) ^
2994 ((word32)Tsbox[GETBYTE(temp, 2)] << 16) ^
2995 ((word32)Tsbox[GETBYTE(temp, 1)] << 8) ^
2996 ((word32)Tsbox[GETBYTE(temp, 0)]);
2997 #endif
2998 rk[13] = rk[ 5] ^ rk[12];
2999 rk[14] = rk[ 6] ^ rk[13];
3000 rk[15] = rk[ 7] ^ rk[14];
3001
3002 rk += 8;
3003 }
3004 break;
3005 #endif /* 256 */
3006
3007 default:
3008 return BAD_FUNC_ARG;
3009 } /* switch */
3010 ForceZero(&temp, sizeof(temp));
3011
3012 #if defined(HAVE_AES_DECRYPT)
3013 if (dir == AES_DECRYPTION) {
3014 unsigned int j;
3015 rk = aes->key;
3016
3017 /* invert the order of the round keys: */
3018 for (i = 0, j = 4* aes->rounds; i < j; i += 4, j -= 4) {
3019 temp = rk[i ]; rk[i ] = rk[j ]; rk[j ] = temp;
3020 temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp;
3021 temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp;
3022 temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp;
3023 }
3024 ForceZero(&temp, sizeof(temp));
3025 #if !defined(WOLFSSL_AES_SMALL_TABLES)
3026 /* apply the inverse MixColumn transform to all round keys but the
3027 first and the last: */
3028 for (i = 1; i < aes->rounds; i++) {
3029 rk += 4;
3030 rk[0] =
3031 Td[0][Te[1][GETBYTE(rk[0], 3)] & 0xff] ^
3032 Td[1][Te[1][GETBYTE(rk[0], 2)] & 0xff] ^
3033 Td[2][Te[1][GETBYTE(rk[0], 1)] & 0xff] ^
3034 Td[3][Te[1][GETBYTE(rk[0], 0)] & 0xff];
3035 rk[1] =
3036 Td[0][Te[1][GETBYTE(rk[1], 3)] & 0xff] ^
3037 Td[1][Te[1][GETBYTE(rk[1], 2)] & 0xff] ^
3038 Td[2][Te[1][GETBYTE(rk[1], 1)] & 0xff] ^
3039 Td[3][Te[1][GETBYTE(rk[1], 0)] & 0xff];
3040 rk[2] =
3041 Td[0][Te[1][GETBYTE(rk[2], 3)] & 0xff] ^
3042 Td[1][Te[1][GETBYTE(rk[2], 2)] & 0xff] ^
3043 Td[2][Te[1][GETBYTE(rk[2], 1)] & 0xff] ^
3044 Td[3][Te[1][GETBYTE(rk[2], 0)] & 0xff];
3045 rk[3] =
3046 Td[0][Te[1][GETBYTE(rk[3], 3)] & 0xff] ^
3047 Td[1][Te[1][GETBYTE(rk[3], 2)] & 0xff] ^
3048 Td[2][Te[1][GETBYTE(rk[3], 1)] & 0xff] ^
3049 Td[3][Te[1][GETBYTE(rk[3], 0)] & 0xff];
3050 }
3051 #endif
3052 }
3053 #else
3054 (void)dir;
3055 #endif /* HAVE_AES_DECRYPT */
3056 (void)temp;
3057 #endif /* NEED_AES_TABLES */
3058
3059 #if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
3060 XMEMCPY((byte*)aes->key, userKey, keylen);
3061 if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag == CRYPTO_WORD_ENDIAN_BIG) {
3062 ByteReverseWords(aes->key, aes->key, 32);
3063 }
3064 #endif
3065
3066 ret = wc_AesSetIV(aes, iv);
3067
3068 #if defined(WOLFSSL_DEVCRYPTO) && \
3069 (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))
3070 aes->ctx.cfd = -1;
3071 #endif
3072 #ifdef WOLFSSL_IMX6_CAAM_BLOB
3073 ForceZero(local, sizeof(local));
3074 #endif
3075 return ret;
3076 }
3077
wc_AesSetKey(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)3078 int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
3079 const byte* iv, int dir)
3080 {
3081 if (aes == NULL) {
3082 return BAD_FUNC_ARG;
3083 }
3084 if (keylen > sizeof(aes->key)) {
3085 return BAD_FUNC_ARG;
3086 }
3087
3088 return wc_AesSetKeyLocal(aes, userKey, keylen, iv, dir, 1);
3089 }
3090
3091 #if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER)
3092 /* AES-CTR and AES-DIRECT need to use this for key setup */
3093 /* This function allows key sizes that are not 128/192/256 bits */
wc_AesSetKeyDirect(Aes * aes,const byte * userKey,word32 keylen,const byte * iv,int dir)3094 int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
3095 const byte* iv, int dir)
3096 {
3097 if (aes == NULL) {
3098 return BAD_FUNC_ARG;
3099 }
3100 if (keylen > sizeof(aes->key)) {
3101 return BAD_FUNC_ARG;
3102 }
3103
3104 return wc_AesSetKeyLocal(aes, userKey, keylen, iv, dir, 0);
3105 }
3106 #endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */
3107 #endif /* wc_AesSetKey block */
3108
3109
3110 /* wc_AesSetIV is shared between software and hardware */
wc_AesSetIV(Aes * aes,const byte * iv)3111 int wc_AesSetIV(Aes* aes, const byte* iv)
3112 {
3113 if (aes == NULL)
3114 return BAD_FUNC_ARG;
3115
3116 if (iv)
3117 XMEMCPY(aes->reg, iv, AES_BLOCK_SIZE);
3118 else
3119 XMEMSET(aes->reg, 0, AES_BLOCK_SIZE);
3120 return 0;
3121 }
3122
3123 /* AES-DIRECT */
3124 #if defined(WOLFSSL_AES_DIRECT)
3125 #if defined(HAVE_COLDFIRE_SEC)
3126 #error "Coldfire SEC doesn't yet support AES direct"
3127
3128 #elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
3129 !defined(WOLFSSL_QNX_CAAM)
3130 /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
3131
3132 #elif defined(WOLFSSL_AFALG)
3133 /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
3134
3135 #elif defined(WOLFSSL_DEVCRYPTO_AES)
3136 /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
3137
3138 #elif defined(WOLFSSL_LINUXKM)
3139
3140 #ifdef WOLFSSL_AESNI
3141
wc_AesEncryptDirect(Aes * aes,byte * out,const byte * in)3142 __must_check int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
3143 {
3144 if (haveAESNI && aes->use_aesni)
3145 SAVE_VECTOR_REGISTERS(return _svr_ret;);
3146 wc_AesEncrypt(aes, in, out);
3147 if (haveAESNI && aes->use_aesni)
3148 RESTORE_VECTOR_REGISTERS();
3149 return 0;
3150 }
3151 /* vector reg save/restore is explicit in all below calls to
3152 * wc_Aes{En,De}cryptDirect(), so bypass the public version with a
3153 * macro.
3154 */
3155 #define wc_AesEncryptDirect(aes, out, in) wc_AesEncrypt(aes, in, out)
3156 #ifdef HAVE_AES_DECRYPT
3157 /* Allow direct access to one block decrypt */
wc_AesDecryptDirect(Aes * aes,byte * out,const byte * in)3158 __must_check int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
3159 {
3160 if (haveAESNI && aes->use_aesni)
3161 SAVE_VECTOR_REGISTERS(return _svr_ret;);
3162 wc_AesDecrypt(aes, in, out);
3163 if (haveAESNI && aes->use_aesni)
3164 RESTORE_VECTOR_REGISTERS();
3165 return 0;
3166 }
3167 #define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out)
3168 #endif /* HAVE_AES_DECRYPT */
3169
3170 #else /* !WOLFSSL_AESNI */
3171
wc_AesEncryptDirect(Aes * aes,byte * out,const byte * in)3172 __must_check int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
3173 {
3174 wc_AesEncrypt(aes, in, out);
3175 return 0;
3176 }
3177 #define wc_AesEncryptDirect(aes, out, in) wc_AesEncrypt(aes, in, out)
3178 #ifdef HAVE_AES_DECRYPT
3179 /* Allow direct access to one block decrypt */
wc_AesDecryptDirect(Aes * aes,byte * out,const byte * in)3180 __must_check int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
3181 {
3182 wc_AesDecrypt(aes, in, out);
3183 return 0;
3184 }
3185 #define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out)
3186 #endif /* HAVE_AES_DECRYPT */
3187
3188 #endif /* WOLFSSL_AESNI */
3189
3190 #else
3191 /* Allow direct access to one block encrypt */
wc_AesEncryptDirect(Aes * aes,byte * out,const byte * in)3192 void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
3193 {
3194 wc_AesEncrypt(aes, in, out);
3195 }
3196 #ifdef HAVE_AES_DECRYPT
3197 /* Allow direct access to one block decrypt */
wc_AesDecryptDirect(Aes * aes,byte * out,const byte * in)3198 void wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
3199 {
3200 wc_AesDecrypt(aes, in, out);
3201 }
3202 #endif /* HAVE_AES_DECRYPT */
3203 #endif /* AES direct block */
3204 #endif /* WOLFSSL_AES_DIRECT */
3205
3206
3207 /* AES-CBC */
3208 #ifdef HAVE_AES_CBC
3209 #if defined(STM32_CRYPTO)
3210
3211 #ifdef WOLFSSL_STM32_CUBEMX
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3212 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3213 {
3214 int ret = 0;
3215 CRYP_HandleTypeDef hcryp;
3216 word32 blocks = (sz / AES_BLOCK_SIZE);
3217
3218 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3219 if (sz % AES_BLOCK_SIZE) {
3220 return BAD_LENGTH_E;
3221 }
3222 #endif
3223 if (blocks == 0)
3224 return 0;
3225
3226 ret = wc_Stm32_Aes_Init(aes, &hcryp);
3227 if (ret != 0)
3228 return ret;
3229
3230 ret = wolfSSL_CryptHwMutexLock();
3231 if (ret != 0) {
3232 return ret;
3233 }
3234
3235 #if defined(STM32_HAL_V2)
3236 hcryp.Init.Algorithm = CRYP_AES_CBC;
3237 ByteReverseWords(aes->reg, aes->reg, AES_BLOCK_SIZE);
3238 #elif defined(STM32_CRYPTO_AES_ONLY)
3239 hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
3240 hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_CBC;
3241 hcryp.Init.KeyWriteFlag = CRYP_KEY_WRITE_ENABLE;
3242 #endif
3243 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
3244 HAL_CRYP_Init(&hcryp);
3245
3246 #if defined(STM32_HAL_V2)
3247 ret = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in, blocks * AES_BLOCK_SIZE,
3248 (uint32_t*)out, STM32_HAL_TIMEOUT);
3249 #elif defined(STM32_CRYPTO_AES_ONLY)
3250 ret = HAL_CRYPEx_AES(&hcryp, (uint8_t*)in, blocks * AES_BLOCK_SIZE,
3251 out, STM32_HAL_TIMEOUT);
3252 #else
3253 ret = HAL_CRYP_AESCBC_Encrypt(&hcryp, (uint8_t*)in, blocks * AES_BLOCK_SIZE,
3254 out, STM32_HAL_TIMEOUT);
3255 #endif
3256 if (ret != HAL_OK) {
3257 ret = WC_TIMEOUT_E;
3258 }
3259
3260 /* store iv for next call */
3261 XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3262
3263 HAL_CRYP_DeInit(&hcryp);
3264
3265 wolfSSL_CryptHwMutexUnLock();
3266
3267 return ret;
3268 }
3269 #ifdef HAVE_AES_DECRYPT
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)3270 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3271 {
3272 int ret = 0;
3273 CRYP_HandleTypeDef hcryp;
3274 word32 blocks = (sz / AES_BLOCK_SIZE);
3275
3276 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3277 if (sz % AES_BLOCK_SIZE) {
3278 return BAD_LENGTH_E;
3279 }
3280 #endif
3281 if (blocks == 0)
3282 return 0;
3283
3284 ret = wc_Stm32_Aes_Init(aes, &hcryp);
3285 if (ret != 0)
3286 return ret;
3287
3288 ret = wolfSSL_CryptHwMutexLock();
3289 if (ret != 0) {
3290 return ret;
3291 }
3292
3293 /* if input and output same will overwrite input iv */
3294 XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3295
3296 #if defined(STM32_HAL_V2)
3297 hcryp.Init.Algorithm = CRYP_AES_CBC;
3298 ByteReverseWords(aes->reg, aes->reg, AES_BLOCK_SIZE);
3299 #elif defined(STM32_CRYPTO_AES_ONLY)
3300 hcryp.Init.OperatingMode = CRYP_ALGOMODE_KEYDERIVATION_DECRYPT;
3301 hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_CBC;
3302 hcryp.Init.KeyWriteFlag = CRYP_KEY_WRITE_ENABLE;
3303 #endif
3304
3305 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
3306 HAL_CRYP_Init(&hcryp);
3307
3308 #if defined(STM32_HAL_V2)
3309 ret = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)in, blocks * AES_BLOCK_SIZE,
3310 (uint32_t*)out, STM32_HAL_TIMEOUT);
3311 #elif defined(STM32_CRYPTO_AES_ONLY)
3312 ret = HAL_CRYPEx_AES(&hcryp, (uint8_t*)in, blocks * AES_BLOCK_SIZE,
3313 out, STM32_HAL_TIMEOUT);
3314 #else
3315 ret = HAL_CRYP_AESCBC_Decrypt(&hcryp, (uint8_t*)in, blocks * AES_BLOCK_SIZE,
3316 out, STM32_HAL_TIMEOUT);
3317 #endif
3318 if (ret != HAL_OK) {
3319 ret = WC_TIMEOUT_E;
3320 }
3321
3322 /* store iv for next call */
3323 XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
3324
3325 HAL_CRYP_DeInit(&hcryp);
3326 wolfSSL_CryptHwMutexUnLock();
3327
3328 return ret;
3329 }
3330 #endif /* HAVE_AES_DECRYPT */
3331
3332 #else /* Standard Peripheral Library */
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3333 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3334 {
3335 int ret;
3336 word32 *iv;
3337 CRYP_InitTypeDef cryptInit;
3338 CRYP_KeyInitTypeDef keyInit;
3339 CRYP_IVInitTypeDef ivInit;
3340 word32 blocks = (sz / AES_BLOCK_SIZE);
3341
3342 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3343 if (sz % AES_BLOCK_SIZE) {
3344 return BAD_LENGTH_E;
3345 }
3346 #endif
3347 if (blocks == 0)
3348 return 0;
3349
3350 ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
3351 if (ret != 0)
3352 return ret;
3353
3354 ret = wolfSSL_CryptHwMutexLock();
3355 if (ret != 0) {
3356 return ret;
3357 }
3358
3359 /* reset registers to their default values */
3360 CRYP_DeInit();
3361
3362 /* set key */
3363 CRYP_KeyInit(&keyInit);
3364
3365 /* set iv */
3366 iv = aes->reg;
3367 CRYP_IVStructInit(&ivInit);
3368 ByteReverseWords(iv, iv, AES_BLOCK_SIZE);
3369 ivInit.CRYP_IV0Left = iv[0];
3370 ivInit.CRYP_IV0Right = iv[1];
3371 ivInit.CRYP_IV1Left = iv[2];
3372 ivInit.CRYP_IV1Right = iv[3];
3373 CRYP_IVInit(&ivInit);
3374
3375 /* set direction and mode */
3376 cryptInit.CRYP_AlgoDir = CRYP_AlgoDir_Encrypt;
3377 cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_CBC;
3378 CRYP_Init(&cryptInit);
3379
3380 /* enable crypto processor */
3381 CRYP_Cmd(ENABLE);
3382
3383 while (blocks--) {
3384 /* flush IN/OUT FIFOs */
3385 CRYP_FIFOFlush();
3386
3387 CRYP_DataIn(*(uint32_t*)&in[0]);
3388 CRYP_DataIn(*(uint32_t*)&in[4]);
3389 CRYP_DataIn(*(uint32_t*)&in[8]);
3390 CRYP_DataIn(*(uint32_t*)&in[12]);
3391
3392 /* wait until the complete message has been processed */
3393 while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
3394
3395 *(uint32_t*)&out[0] = CRYP_DataOut();
3396 *(uint32_t*)&out[4] = CRYP_DataOut();
3397 *(uint32_t*)&out[8] = CRYP_DataOut();
3398 *(uint32_t*)&out[12] = CRYP_DataOut();
3399
3400 /* store iv for next call */
3401 XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3402
3403 sz -= AES_BLOCK_SIZE;
3404 in += AES_BLOCK_SIZE;
3405 out += AES_BLOCK_SIZE;
3406 }
3407
3408 /* disable crypto processor */
3409 CRYP_Cmd(DISABLE);
3410 wolfSSL_CryptHwMutexUnLock();
3411
3412 return ret;
3413 }
3414
3415 #ifdef HAVE_AES_DECRYPT
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)3416 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3417 {
3418 int ret;
3419 word32 *iv;
3420 CRYP_InitTypeDef cryptInit;
3421 CRYP_KeyInitTypeDef keyInit;
3422 CRYP_IVInitTypeDef ivInit;
3423 word32 blocks = (sz / AES_BLOCK_SIZE);
3424
3425 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3426 if (sz % AES_BLOCK_SIZE) {
3427 return BAD_LENGTH_E;
3428 }
3429 #endif
3430 if (blocks == 0)
3431 return 0;
3432
3433 ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
3434 if (ret != 0)
3435 return ret;
3436
3437 ret = wolfSSL_CryptHwMutexLock();
3438 if (ret != 0) {
3439 return ret;
3440 }
3441
3442 /* if input and output same will overwrite input iv */
3443 XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3444
3445 /* reset registers to their default values */
3446 CRYP_DeInit();
3447
3448 /* set direction and key */
3449 CRYP_KeyInit(&keyInit);
3450 cryptInit.CRYP_AlgoDir = CRYP_AlgoDir_Decrypt;
3451 cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_Key;
3452 CRYP_Init(&cryptInit);
3453
3454 /* enable crypto processor */
3455 CRYP_Cmd(ENABLE);
3456
3457 /* wait until key has been prepared */
3458 while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
3459
3460 /* set direction and mode */
3461 cryptInit.CRYP_AlgoDir = CRYP_AlgoDir_Decrypt;
3462 cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_CBC;
3463 CRYP_Init(&cryptInit);
3464
3465 /* set iv */
3466 iv = aes->reg;
3467 CRYP_IVStructInit(&ivInit);
3468 ByteReverseWords(iv, iv, AES_BLOCK_SIZE);
3469 ivInit.CRYP_IV0Left = iv[0];
3470 ivInit.CRYP_IV0Right = iv[1];
3471 ivInit.CRYP_IV1Left = iv[2];
3472 ivInit.CRYP_IV1Right = iv[3];
3473 CRYP_IVInit(&ivInit);
3474
3475 /* enable crypto processor */
3476 CRYP_Cmd(ENABLE);
3477
3478 while (blocks--) {
3479 /* flush IN/OUT FIFOs */
3480 CRYP_FIFOFlush();
3481
3482 CRYP_DataIn(*(uint32_t*)&in[0]);
3483 CRYP_DataIn(*(uint32_t*)&in[4]);
3484 CRYP_DataIn(*(uint32_t*)&in[8]);
3485 CRYP_DataIn(*(uint32_t*)&in[12]);
3486
3487 /* wait until the complete message has been processed */
3488 while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
3489
3490 *(uint32_t*)&out[0] = CRYP_DataOut();
3491 *(uint32_t*)&out[4] = CRYP_DataOut();
3492 *(uint32_t*)&out[8] = CRYP_DataOut();
3493 *(uint32_t*)&out[12] = CRYP_DataOut();
3494
3495 /* store iv for next call */
3496 XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
3497
3498 in += AES_BLOCK_SIZE;
3499 out += AES_BLOCK_SIZE;
3500 }
3501
3502 /* disable crypto processor */
3503 CRYP_Cmd(DISABLE);
3504 wolfSSL_CryptHwMutexUnLock();
3505
3506 return ret;
3507 }
3508 #endif /* HAVE_AES_DECRYPT */
3509 #endif /* WOLFSSL_STM32_CUBEMX */
3510
3511 #elif defined(HAVE_COLDFIRE_SEC)
wc_AesCbcCrypt(Aes * aes,byte * po,const byte * pi,word32 sz,word32 descHeader)3512 static int wc_AesCbcCrypt(Aes* aes, byte* po, const byte* pi, word32 sz,
3513 word32 descHeader)
3514 {
3515 #ifdef DEBUG_WOLFSSL
3516 int i; int stat1, stat2; int ret;
3517 #endif
3518
3519 int size;
3520 volatile int v;
3521
3522 if ((pi == NULL) || (po == NULL))
3523 return BAD_FUNC_ARG; /*wrong pointer*/
3524
3525 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3526 if (sz % AES_BLOCK_SIZE) {
3527 return BAD_LENGTH_E;
3528 }
3529 #endif
3530
3531 wc_LockMutex(&Mutex_AesSEC);
3532
3533 /* Set descriptor for SEC */
3534 secDesc->length1 = 0x0;
3535 secDesc->pointer1 = NULL;
3536
3537 secDesc->length2 = AES_BLOCK_SIZE;
3538 secDesc->pointer2 = (byte *)secReg; /* Initial Vector */
3539
3540 switch(aes->rounds) {
3541 case 10: secDesc->length3 = 16; break;
3542 case 12: secDesc->length3 = 24; break;
3543 case 14: secDesc->length3 = 32; break;
3544 }
3545 XMEMCPY(secKey, aes->key, secDesc->length3);
3546
3547 secDesc->pointer3 = (byte *)secKey;
3548 secDesc->pointer4 = AESBuffIn;
3549 secDesc->pointer5 = AESBuffOut;
3550 secDesc->length6 = 0x0;
3551 secDesc->pointer6 = NULL;
3552 secDesc->length7 = 0x0;
3553 secDesc->pointer7 = NULL;
3554 secDesc->nextDescriptorPtr = NULL;
3555
3556 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3557 size = AES_BUFFER_SIZE;
3558 #endif
3559 while (sz) {
3560 secDesc->header = descHeader;
3561 XMEMCPY(secReg, aes->reg, AES_BLOCK_SIZE);
3562 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3563 sz -= AES_BUFFER_SIZE;
3564 #else
3565 if (sz < AES_BUFFER_SIZE) {
3566 size = sz;
3567 sz = 0;
3568 } else {
3569 size = AES_BUFFER_SIZE;
3570 sz -= AES_BUFFER_SIZE;
3571 }
3572 #endif
3573
3574 secDesc->length4 = size;
3575 secDesc->length5 = size;
3576
3577 XMEMCPY(AESBuffIn, pi, size);
3578 if(descHeader == SEC_DESC_AES_CBC_DECRYPT) {
3579 XMEMCPY((void*)aes->tmp, (void*)&(pi[size-AES_BLOCK_SIZE]),
3580 AES_BLOCK_SIZE);
3581 }
3582
3583 /* Point SEC to the location of the descriptor */
3584 MCF_SEC_FR0 = (uint32)secDesc;
3585 /* Initialize SEC and wait for encryption to complete */
3586 MCF_SEC_CCCR0 = 0x0000001a;
3587 /* poll SISR to determine when channel is complete */
3588 v=0;
3589
3590 while ((secDesc->header>> 24) != 0xff) v++;
3591
3592 #ifdef DEBUG_WOLFSSL
3593 ret = MCF_SEC_SISRH;
3594 stat1 = MCF_SEC_AESSR;
3595 stat2 = MCF_SEC_AESISR;
3596 if (ret & 0xe0000000) {
3597 db_printf("Aes_Cbc(i=%d):ISRH=%08x, AESSR=%08x, "
3598 "AESISR=%08x\n", i, ret, stat1, stat2);
3599 }
3600 #endif
3601
3602 XMEMCPY(po, AESBuffOut, size);
3603
3604 if (descHeader == SEC_DESC_AES_CBC_ENCRYPT) {
3605 XMEMCPY((void*)aes->reg, (void*)&(po[size-AES_BLOCK_SIZE]),
3606 AES_BLOCK_SIZE);
3607 } else {
3608 XMEMCPY((void*)aes->reg, (void*)aes->tmp, AES_BLOCK_SIZE);
3609 }
3610
3611 pi += size;
3612 po += size;
3613 }
3614
3615 wc_UnLockMutex(&Mutex_AesSEC);
3616 return 0;
3617 }
3618
wc_AesCbcEncrypt(Aes * aes,byte * po,const byte * pi,word32 sz)3619 int wc_AesCbcEncrypt(Aes* aes, byte* po, const byte* pi, word32 sz)
3620 {
3621 return (wc_AesCbcCrypt(aes, po, pi, sz, SEC_DESC_AES_CBC_ENCRYPT));
3622 }
3623
3624 #ifdef HAVE_AES_DECRYPT
wc_AesCbcDecrypt(Aes * aes,byte * po,const byte * pi,word32 sz)3625 int wc_AesCbcDecrypt(Aes* aes, byte* po, const byte* pi, word32 sz)
3626 {
3627 return (wc_AesCbcCrypt(aes, po, pi, sz, SEC_DESC_AES_CBC_DECRYPT));
3628 }
3629 #endif /* HAVE_AES_DECRYPT */
3630
3631 #elif defined(FREESCALE_LTC)
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3632 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3633 {
3634 word32 keySize;
3635 status_t status;
3636 byte *iv, *enc_key;
3637 word32 blocks = (sz / AES_BLOCK_SIZE);
3638
3639 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3640 if (sz % AES_BLOCK_SIZE) {
3641 return BAD_LENGTH_E;
3642 }
3643 #endif
3644 if (blocks == 0)
3645 return 0;
3646
3647 iv = (byte*)aes->reg;
3648 enc_key = (byte*)aes->key;
3649
3650 status = wc_AesGetKeySize(aes, &keySize);
3651 if (status != 0) {
3652 return status;
3653 }
3654
3655 status = wolfSSL_CryptHwMutexLock();
3656 if (status != 0)
3657 return status;
3658 status = LTC_AES_EncryptCbc(LTC_BASE, in, out, blocks * AES_BLOCK_SIZE,
3659 iv, enc_key, keySize);
3660 wolfSSL_CryptHwMutexUnLock();
3661
3662 /* store iv for next call */
3663 if (status == kStatus_Success) {
3664 XMEMCPY(iv, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3665 }
3666
3667 return (status == kStatus_Success) ? 0 : -1;
3668 }
3669
3670 #ifdef HAVE_AES_DECRYPT
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)3671 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3672 {
3673 word32 keySize;
3674 status_t status;
3675 byte* iv, *dec_key;
3676 byte temp_block[AES_BLOCK_SIZE];
3677 word32 blocks = (sz / AES_BLOCK_SIZE);
3678
3679 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3680 if (sz % AES_BLOCK_SIZE) {
3681 return BAD_LENGTH_E;
3682 }
3683 #endif
3684 if (blocks == 0)
3685 return 0;
3686
3687 iv = (byte*)aes->reg;
3688 dec_key = (byte*)aes->key;
3689
3690 status = wc_AesGetKeySize(aes, &keySize);
3691 if (status != 0) {
3692 return status;
3693 }
3694
3695 /* get IV for next call */
3696 XMEMCPY(temp_block, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3697
3698 status = wolfSSL_CryptHwMutexLock();
3699 if (status != 0)
3700 return status;
3701 status = LTC_AES_DecryptCbc(LTC_BASE, in, out, blocks * AES_BLOCK_SIZE,
3702 iv, dec_key, keySize, kLTC_EncryptKey);
3703 wolfSSL_CryptHwMutexUnLock();
3704
3705 /* store IV for next call */
3706 if (status == kStatus_Success) {
3707 XMEMCPY(iv, temp_block, AES_BLOCK_SIZE);
3708 }
3709
3710 return (status == kStatus_Success) ? 0 : -1;
3711 }
3712 #endif /* HAVE_AES_DECRYPT */
3713
3714 #elif defined(FREESCALE_MMCAU)
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3715 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3716 {
3717 int i;
3718 int offset = 0;
3719 byte *iv;
3720 byte temp_block[AES_BLOCK_SIZE];
3721 word32 blocks = (sz / AES_BLOCK_SIZE);
3722
3723 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3724 if (sz % AES_BLOCK_SIZE) {
3725 return BAD_LENGTH_E;
3726 }
3727 #endif
3728 if (blocks == 0)
3729 return 0;
3730
3731 iv = (byte*)aes->reg;
3732
3733 while (blocks--) {
3734 XMEMCPY(temp_block, in + offset, AES_BLOCK_SIZE);
3735
3736 /* XOR block with IV for CBC */
3737 for (i = 0; i < AES_BLOCK_SIZE; i++)
3738 temp_block[i] ^= iv[i];
3739
3740 wc_AesEncrypt(aes, temp_block, out + offset);
3741
3742 offset += AES_BLOCK_SIZE;
3743
3744 /* store IV for next block */
3745 XMEMCPY(iv, out + offset - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3746 }
3747
3748 return 0;
3749 }
3750 #ifdef HAVE_AES_DECRYPT
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)3751 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3752 {
3753 int i;
3754 int offset = 0;
3755 byte* iv;
3756 byte temp_block[AES_BLOCK_SIZE];
3757 word32 blocks = (sz / AES_BLOCK_SIZE);
3758
3759 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3760 if (sz % AES_BLOCK_SIZE) {
3761 return BAD_LENGTH_E;
3762 }
3763 #endif
3764 if (blocks == 0)
3765 return 0;
3766
3767 iv = (byte*)aes->reg;
3768
3769 while (blocks--) {
3770 XMEMCPY(temp_block, in + offset, AES_BLOCK_SIZE);
3771
3772 wc_AesDecrypt(aes, in + offset, out + offset);
3773
3774 /* XOR block with IV for CBC */
3775 for (i = 0; i < AES_BLOCK_SIZE; i++)
3776 (out + offset)[i] ^= iv[i];
3777
3778 /* store IV for next block */
3779 XMEMCPY(iv, temp_block, AES_BLOCK_SIZE);
3780
3781 offset += AES_BLOCK_SIZE;
3782 }
3783
3784 return 0;
3785 }
3786 #endif /* HAVE_AES_DECRYPT */
3787
3788 #elif defined(WOLFSSL_PIC32MZ_CRYPT)
3789
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3790 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3791 {
3792 int ret;
3793
3794 if (sz == 0)
3795 return 0;
3796
3797 /* hardware fails on input that is not a multiple of AES block size */
3798 if (sz % AES_BLOCK_SIZE != 0) {
3799 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3800 return BAD_LENGTH_E;
3801 #else
3802 return BAD_FUNC_ARG;
3803 #endif
3804 }
3805
3806 ret = wc_Pic32AesCrypt(
3807 aes->key, aes->keylen, aes->reg, AES_BLOCK_SIZE,
3808 out, in, sz, PIC32_ENCRYPTION,
3809 PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCBC);
3810
3811 /* store iv for next call */
3812 if (ret == 0) {
3813 XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3814 }
3815
3816 return ret;
3817 }
3818 #ifdef HAVE_AES_DECRYPT
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)3819 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3820 {
3821 int ret;
3822 byte scratch[AES_BLOCK_SIZE];
3823
3824 if (sz == 0)
3825 return 0;
3826
3827 /* hardware fails on input that is not a multiple of AES block size */
3828 if (sz % AES_BLOCK_SIZE != 0) {
3829 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3830 return BAD_LENGTH_E;
3831 #else
3832 return BAD_FUNC_ARG;
3833 #endif
3834 }
3835 XMEMCPY(scratch, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3836
3837 ret = wc_Pic32AesCrypt(
3838 aes->key, aes->keylen, aes->reg, AES_BLOCK_SIZE,
3839 out, in, sz, PIC32_DECRYPTION,
3840 PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCBC);
3841
3842 /* store iv for next call */
3843 if (ret == 0) {
3844 XMEMCPY((byte*)aes->reg, scratch, AES_BLOCK_SIZE);
3845 }
3846
3847 return ret;
3848 }
3849 #endif /* HAVE_AES_DECRYPT */
3850 #elif defined(WOLFSSL_ESP32WROOM32_CRYPT) && \
3851 !defined(NO_WOLFSSL_ESP32WROOM32_CRYPT_AES)
3852
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3853 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3854 {
3855 return wc_esp32AesCbcEncrypt(aes, out, in, sz);
3856 }
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)3857 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3858 {
3859 return wc_esp32AesCbcDecrypt(aes, out, in, sz);
3860 }
3861 #elif defined(WOLFSSL_CRYPTOCELL) && defined(WOLFSSL_CRYPTOCELL_AES)
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3862 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3863 {
3864 return SaSi_AesBlock(&aes->ctx.user_ctx, (uint8_t*)in, sz, out);
3865 }
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)3866 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3867 {
3868 return SaSi_AesBlock(&aes->ctx.user_ctx, (uint8_t*)in, sz, out);
3869 }
3870 #elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
3871 !defined(WOLFSSL_QNX_CAAM)
3872 /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
3873
3874 #elif defined(WOLFSSL_AFALG)
3875 /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
3876
3877 #elif defined(WOLFSSL_KCAPI_AES) && !defined(WOLFSSL_NO_KCAPI_AES_CBC)
3878 /* implemented in wolfcrypt/src/port/kcapi/kcapi_aes.c */
3879
3880 #elif defined(WOLFSSL_DEVCRYPTO_CBC)
3881 /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
3882
3883 #elif defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3884 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3885 {
3886 return se050_aes_crypt(aes, in, out, sz, AES_ENCRYPTION,
3887 kAlgorithm_SSS_AES_CBC);
3888 }
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)3889 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3890 {
3891 return se050_aes_crypt(aes, in, out, sz, AES_DECRYPTION,
3892 kAlgorithm_SSS_AES_CBC);
3893 }
3894
3895 #elif defined(WOLFSSL_SILABS_SE_ACCEL)
3896 /* implemented in wolfcrypt/src/port/silabs/silabs_hash.c */
3897
3898 #else
3899
3900 /* Software AES - CBC Encrypt */
wc_AesCbcEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)3901 int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3902 {
3903 word32 blocks;
3904
3905 if (aes == NULL || out == NULL || in == NULL) {
3906 return BAD_FUNC_ARG;
3907 }
3908
3909 if (sz == 0) {
3910 return 0;
3911 }
3912
3913 blocks = sz / AES_BLOCK_SIZE;
3914 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3915 if (sz % AES_BLOCK_SIZE) {
3916 return BAD_LENGTH_E;
3917 }
3918 #endif
3919
3920 #ifdef WOLFSSL_IMXRT_DCP
3921 /* Implemented in wolfcrypt/src/port/nxp/dcp_port.c */
3922 if (aes->keylen == 16)
3923 return DCPAesCbcEncrypt(aes, out, in, sz);
3924 #endif
3925
3926 #ifdef WOLF_CRYPTO_CB
3927 if (aes->devId != INVALID_DEVID) {
3928 int ret = wc_CryptoCb_AesCbcEncrypt(aes, out, in, sz);
3929 if (ret != CRYPTOCB_UNAVAILABLE)
3930 return ret;
3931 /* fall-through when unavailable */
3932 }
3933 #endif
3934 #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
3935 /* if async and byte count above threshold */
3936 if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
3937 sz >= WC_ASYNC_THRESH_AES_CBC) {
3938 #if defined(HAVE_CAVIUM)
3939 return NitroxAesCbcEncrypt(aes, out, in, sz);
3940 #elif defined(HAVE_INTEL_QA)
3941 return IntelQaSymAesCbcEncrypt(&aes->asyncDev, out, in, sz,
3942 (const byte*)aes->devKey, aes->keylen,
3943 (byte*)aes->reg, AES_BLOCK_SIZE);
3944 #else /* WOLFSSL_ASYNC_CRYPT_TEST */
3945 if (wc_AsyncTestInit(&aes->asyncDev, ASYNC_TEST_AES_CBC_ENCRYPT)) {
3946 WC_ASYNC_TEST* testDev = &aes->asyncDev.test;
3947 testDev->aes.aes = aes;
3948 testDev->aes.out = out;
3949 testDev->aes.in = in;
3950 testDev->aes.sz = sz;
3951 return WC_PENDING_E;
3952 }
3953 #endif
3954 }
3955 #endif /* WOLFSSL_ASYNC_CRYPT */
3956
3957 #ifdef WOLFSSL_AESNI
3958 if (haveAESNI) {
3959 #ifdef DEBUG_AESNI
3960 printf("about to aes cbc encrypt\n");
3961 printf("in = %p\n", in);
3962 printf("out = %p\n", out);
3963 printf("aes->key = %p\n", aes->key);
3964 printf("aes->reg = %p\n", aes->reg);
3965 printf("aes->rounds = %d\n", aes->rounds);
3966 printf("sz = %d\n", sz);
3967 #endif
3968
3969 /* check alignment, decrypt doesn't need alignment */
3970 if ((wc_ptr_t)in % AESNI_ALIGN) {
3971 #ifndef NO_WOLFSSL_ALLOC_ALIGN
3972 byte* tmp = (byte*)XMALLOC(sz + AES_BLOCK_SIZE + AESNI_ALIGN,
3973 aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
3974 byte* tmp_align;
3975 if (tmp == NULL) return MEMORY_E;
3976
3977 tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN));
3978 XMEMCPY(tmp_align, in, sz);
3979 SAVE_VECTOR_REGISTERS(XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER); return _svr_ret;);
3980 AES_CBC_encrypt(tmp_align, tmp_align, (byte*)aes->reg, sz,
3981 (byte*)aes->key, aes->rounds);
3982 RESTORE_VECTOR_REGISTERS();
3983 /* store iv for next call */
3984 XMEMCPY(aes->reg, tmp_align + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
3985
3986 XMEMCPY(out, tmp_align, sz);
3987 XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
3988 return 0;
3989 #else
3990 WOLFSSL_MSG("AES-CBC encrypt with bad alignment");
3991 return BAD_ALIGN_E;
3992 #endif
3993 }
3994
3995 SAVE_VECTOR_REGISTERS(return _svr_ret;);
3996 AES_CBC_encrypt(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
3997 aes->rounds);
3998 RESTORE_VECTOR_REGISTERS();
3999 /* store iv for next call */
4000 XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
4001
4002 return 0;
4003 }
4004 #endif
4005
4006 while (blocks--) {
4007 xorbuf((byte*)aes->reg, in, AES_BLOCK_SIZE);
4008 wc_AesEncrypt(aes, (byte*)aes->reg, (byte*)aes->reg);
4009 XMEMCPY(out, aes->reg, AES_BLOCK_SIZE);
4010
4011 out += AES_BLOCK_SIZE;
4012 in += AES_BLOCK_SIZE;
4013 }
4014
4015 return 0;
4016 }
4017
4018 #ifdef HAVE_AES_DECRYPT
4019 /* Software AES - CBC Decrypt */
wc_AesCbcDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)4020 int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
4021 {
4022 word32 blocks;
4023
4024 if (aes == NULL || out == NULL || in == NULL) {
4025 return BAD_FUNC_ARG;
4026 }
4027
4028 if (sz == 0) {
4029 return 0;
4030 }
4031
4032 blocks = sz / AES_BLOCK_SIZE;
4033 if (sz % AES_BLOCK_SIZE) {
4034 #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
4035 return BAD_LENGTH_E;
4036 #else
4037 return BAD_FUNC_ARG;
4038 #endif
4039 }
4040
4041 #ifdef WOLFSSL_IMXRT_DCP
4042 /* Implemented in wolfcrypt/src/port/nxp/dcp_port.c */
4043 if (aes->keylen == 16)
4044 return DCPAesCbcDecrypt(aes, out, in, sz);
4045 #endif
4046
4047 #ifdef WOLF_CRYPTO_CB
4048 if (aes->devId != INVALID_DEVID) {
4049 int ret = wc_CryptoCb_AesCbcDecrypt(aes, out, in, sz);
4050 if (ret != CRYPTOCB_UNAVAILABLE)
4051 return ret;
4052 /* fall-through when unavailable */
4053 }
4054 #endif
4055 #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
4056 /* if async and byte count above threshold */
4057 if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
4058 sz >= WC_ASYNC_THRESH_AES_CBC) {
4059 #if defined(HAVE_CAVIUM)
4060 return NitroxAesCbcDecrypt(aes, out, in, sz);
4061 #elif defined(HAVE_INTEL_QA)
4062 return IntelQaSymAesCbcDecrypt(&aes->asyncDev, out, in, sz,
4063 (const byte*)aes->devKey, aes->keylen,
4064 (byte*)aes->reg, AES_BLOCK_SIZE);
4065 #else /* WOLFSSL_ASYNC_CRYPT_TEST */
4066 if (wc_AsyncTestInit(&aes->asyncDev, ASYNC_TEST_AES_CBC_DECRYPT)) {
4067 WC_ASYNC_TEST* testDev = &aes->asyncDev.test;
4068 testDev->aes.aes = aes;
4069 testDev->aes.out = out;
4070 testDev->aes.in = in;
4071 testDev->aes.sz = sz;
4072 return WC_PENDING_E;
4073 }
4074 #endif
4075 }
4076 #endif
4077
4078 #ifdef WOLFSSL_AESNI
4079 if (haveAESNI) {
4080 #ifdef DEBUG_AESNI
4081 printf("about to aes cbc decrypt\n");
4082 printf("in = %p\n", in);
4083 printf("out = %p\n", out);
4084 printf("aes->key = %p\n", aes->key);
4085 printf("aes->reg = %p\n", aes->reg);
4086 printf("aes->rounds = %d\n", aes->rounds);
4087 printf("sz = %d\n", sz);
4088 #endif
4089
4090 /* if input and output same will overwrite input iv */
4091 XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
4092 SAVE_VECTOR_REGISTERS(return _svr_ret;);
4093 #if defined(WOLFSSL_AESNI_BY4)
4094 AES_CBC_decrypt_by4(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
4095 aes->rounds);
4096 #elif defined(WOLFSSL_AESNI_BY6)
4097 AES_CBC_decrypt_by6(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
4098 aes->rounds);
4099 #else /* WOLFSSL_AESNI_BYx */
4100 AES_CBC_decrypt_by8(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
4101 aes->rounds);
4102 #endif /* WOLFSSL_AESNI_BYx */
4103 /* store iv for next call */
4104 RESTORE_VECTOR_REGISTERS();
4105 XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
4106 return 0;
4107 }
4108 #endif
4109
4110 while (blocks--) {
4111 XMEMCPY(aes->tmp, in, AES_BLOCK_SIZE);
4112 wc_AesDecrypt(aes, (byte*)aes->tmp, out);
4113 xorbuf(out, (byte*)aes->reg, AES_BLOCK_SIZE);
4114 /* store iv for next call */
4115 XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
4116
4117 out += AES_BLOCK_SIZE;
4118 in += AES_BLOCK_SIZE;
4119 }
4120
4121 return 0;
4122 }
4123 #endif
4124
4125 #endif /* AES-CBC block */
4126 #endif /* HAVE_AES_CBC */
4127
4128 /* AES-CTR */
4129 #if defined(WOLFSSL_AES_COUNTER)
4130
4131 #ifdef STM32_CRYPTO
4132 #define NEED_AES_CTR_SOFT
4133 #define XTRANSFORM_AESCTRBLOCK wc_AesCtrEncryptBlock
4134
wc_AesCtrEncryptBlock(Aes * aes,byte * out,const byte * in)4135 int wc_AesCtrEncryptBlock(Aes* aes, byte* out, const byte* in)
4136 {
4137 int ret = 0;
4138 #ifdef WOLFSSL_STM32_CUBEMX
4139 CRYP_HandleTypeDef hcryp;
4140 #ifdef STM32_HAL_V2
4141 word32 iv[AES_BLOCK_SIZE/sizeof(word32)];
4142 #endif
4143 #else
4144 word32 *iv;
4145 CRYP_InitTypeDef cryptInit;
4146 CRYP_KeyInitTypeDef keyInit;
4147 CRYP_IVInitTypeDef ivInit;
4148 #endif
4149
4150 #ifdef WOLFSSL_STM32_CUBEMX
4151 ret = wc_Stm32_Aes_Init(aes, &hcryp);
4152 if (ret != 0) {
4153 return ret;
4154 }
4155
4156 ret = wolfSSL_CryptHwMutexLock();
4157 if (ret != 0) {
4158 return ret;
4159 }
4160
4161 #if defined(STM32_HAL_V2)
4162 hcryp.Init.Algorithm = CRYP_AES_CTR;
4163 ByteReverseWords(iv, aes->reg, AES_BLOCK_SIZE);
4164 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)iv;
4165 #elif defined(STM32_CRYPTO_AES_ONLY)
4166 hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
4167 hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_CTR;
4168 hcryp.Init.KeyWriteFlag = CRYP_KEY_WRITE_ENABLE;
4169 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
4170 #else
4171 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
4172 #endif
4173 HAL_CRYP_Init(&hcryp);
4174
4175 #if defined(STM32_HAL_V2)
4176 ret = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in, AES_BLOCK_SIZE,
4177 (uint32_t*)out, STM32_HAL_TIMEOUT);
4178 #elif defined(STM32_CRYPTO_AES_ONLY)
4179 ret = HAL_CRYPEx_AES(&hcryp, (byte*)in, AES_BLOCK_SIZE,
4180 out, STM32_HAL_TIMEOUT);
4181 #else
4182 ret = HAL_CRYP_AESCTR_Encrypt(&hcryp, (byte*)in, AES_BLOCK_SIZE,
4183 out, STM32_HAL_TIMEOUT);
4184 #endif
4185 if (ret != HAL_OK) {
4186 ret = WC_TIMEOUT_E;
4187 }
4188 HAL_CRYP_DeInit(&hcryp);
4189
4190 #else /* Standard Peripheral Library */
4191 ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
4192 if (ret != 0) {
4193 return ret;
4194 }
4195
4196 ret = wolfSSL_CryptHwMutexLock();
4197 if (ret != 0) {
4198 return ret;
4199 }
4200
4201 /* reset registers to their default values */
4202 CRYP_DeInit();
4203
4204 /* set key */
4205 CRYP_KeyInit(&keyInit);
4206
4207 /* set iv */
4208 iv = aes->reg;
4209 CRYP_IVStructInit(&ivInit);
4210 ivInit.CRYP_IV0Left = ByteReverseWord32(iv[0]);
4211 ivInit.CRYP_IV0Right = ByteReverseWord32(iv[1]);
4212 ivInit.CRYP_IV1Left = ByteReverseWord32(iv[2]);
4213 ivInit.CRYP_IV1Right = ByteReverseWord32(iv[3]);
4214 CRYP_IVInit(&ivInit);
4215
4216 /* set direction and mode */
4217 cryptInit.CRYP_AlgoDir = CRYP_AlgoDir_Encrypt;
4218 cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_CTR;
4219 CRYP_Init(&cryptInit);
4220
4221 /* enable crypto processor */
4222 CRYP_Cmd(ENABLE);
4223
4224 /* flush IN/OUT FIFOs */
4225 CRYP_FIFOFlush();
4226
4227 CRYP_DataIn(*(uint32_t*)&in[0]);
4228 CRYP_DataIn(*(uint32_t*)&in[4]);
4229 CRYP_DataIn(*(uint32_t*)&in[8]);
4230 CRYP_DataIn(*(uint32_t*)&in[12]);
4231
4232 /* wait until the complete message has been processed */
4233 while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
4234
4235 *(uint32_t*)&out[0] = CRYP_DataOut();
4236 *(uint32_t*)&out[4] = CRYP_DataOut();
4237 *(uint32_t*)&out[8] = CRYP_DataOut();
4238 *(uint32_t*)&out[12] = CRYP_DataOut();
4239
4240 /* disable crypto processor */
4241 CRYP_Cmd(DISABLE);
4242 #endif /* WOLFSSL_STM32_CUBEMX */
4243
4244 wolfSSL_CryptHwMutexUnLock();
4245 return ret;
4246 }
4247
4248
4249 #elif defined(WOLFSSL_PIC32MZ_CRYPT)
4250
4251 #define NEED_AES_CTR_SOFT
4252 #define XTRANSFORM_AESCTRBLOCK wc_AesCtrEncryptBlock
4253
wc_AesCtrEncryptBlock(Aes * aes,byte * out,const byte * in)4254 int wc_AesCtrEncryptBlock(Aes* aes, byte* out, const byte* in)
4255 {
4256 word32 tmpIv[AES_BLOCK_SIZE / sizeof(word32)];
4257 XMEMCPY(tmpIv, aes->reg, AES_BLOCK_SIZE);
4258 return wc_Pic32AesCrypt(
4259 aes->key, aes->keylen, tmpIv, AES_BLOCK_SIZE,
4260 out, in, AES_BLOCK_SIZE,
4261 PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCTR);
4262 }
4263
4264 #elif defined(HAVE_COLDFIRE_SEC)
4265 #error "Coldfire SEC doesn't currently support AES-CTR mode"
4266
4267 #elif defined(FREESCALE_LTC)
wc_AesCtrEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)4268 int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
4269 {
4270 int ret = 0;
4271 word32 keySize;
4272 byte *iv, *enc_key;
4273 byte* tmp;
4274
4275 if (aes == NULL || out == NULL || in == NULL) {
4276 return BAD_FUNC_ARG;
4277 }
4278
4279 /* consume any unused bytes left in aes->tmp */
4280 tmp = (byte*)aes->tmp + AES_BLOCK_SIZE - aes->left;
4281 while (aes->left && sz) {
4282 *(out++) = *(in++) ^ *(tmp++);
4283 aes->left--;
4284 sz--;
4285 }
4286
4287 if (sz) {
4288 iv = (byte*)aes->reg;
4289 enc_key = (byte*)aes->key;
4290
4291 wc_AesGetKeySize(aes, &keySize);
4292
4293 ret = wolfSSL_CryptHwMutexLock();
4294 if (ret != 0)
4295 return ret;
4296 LTC_AES_CryptCtr(LTC_BASE, in, out, sz,
4297 iv, enc_key, keySize, (byte*)aes->tmp,
4298 (uint32_t*)&aes->left);
4299 wolfSSL_CryptHwMutexUnLock();
4300 }
4301
4302 return ret;
4303 }
4304
4305 #elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
4306 !defined(WOLFSSL_QNX_CAAM)
4307 /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
4308
4309 #elif defined(WOLFSSL_AFALG)
4310 /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
4311
4312 #elif defined(WOLFSSL_DEVCRYPTO_AES)
4313 /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
4314
4315 #elif defined(WOLFSSL_ESP32WROOM32_CRYPT) && \
4316 !defined(NO_WOLFSSL_ESP32WROOM32_CRYPT_AES)
4317 /* esp32 doesn't support CRT mode by hw. */
4318 /* use aes ecnryption plus sw implementation */
4319 #define NEED_AES_CTR_SOFT
4320
4321 #else
4322
4323 /* Use software based AES counter */
4324 #define NEED_AES_CTR_SOFT
4325 #endif
4326
4327 #ifdef NEED_AES_CTR_SOFT
4328 /* Increment AES counter */
IncrementAesCounter(byte * inOutCtr)4329 static WC_INLINE void IncrementAesCounter(byte* inOutCtr)
4330 {
4331 /* in network byte order so start at end and work back */
4332 int i;
4333 for (i = AES_BLOCK_SIZE - 1; i >= 0; i--) {
4334 if (++inOutCtr[i]) /* we're done unless we overflow */
4335 return;
4336 }
4337 }
4338
4339 /* Software AES - CTR Encrypt */
wc_AesCtrEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)4340 int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
4341 {
4342 byte* tmp;
4343 byte scratch[AES_BLOCK_SIZE];
4344
4345 if (aes == NULL || out == NULL || in == NULL) {
4346 return BAD_FUNC_ARG;
4347 }
4348
4349 /* consume any unused bytes left in aes->tmp */
4350 tmp = (byte*)aes->tmp + AES_BLOCK_SIZE - aes->left;
4351 while (aes->left && sz) {
4352 *(out++) = *(in++) ^ *(tmp++);
4353 aes->left--;
4354 sz--;
4355 }
4356
4357 /* do as many block size ops as possible */
4358 while (sz >= AES_BLOCK_SIZE) {
4359 #ifdef XTRANSFORM_AESCTRBLOCK
4360 XTRANSFORM_AESCTRBLOCK(aes, out, in);
4361 #else
4362 wc_AesEncrypt(aes, (byte*)aes->reg, scratch);
4363 xorbuf(scratch, in, AES_BLOCK_SIZE);
4364 XMEMCPY(out, scratch, AES_BLOCK_SIZE);
4365 #endif
4366 IncrementAesCounter((byte*)aes->reg);
4367
4368 out += AES_BLOCK_SIZE;
4369 in += AES_BLOCK_SIZE;
4370 sz -= AES_BLOCK_SIZE;
4371 aes->left = 0;
4372 }
4373 ForceZero(scratch, AES_BLOCK_SIZE);
4374
4375 /* handle non block size remaining and store unused byte count in left */
4376 if (sz) {
4377 wc_AesEncrypt(aes, (byte*)aes->reg, (byte*)aes->tmp);
4378 IncrementAesCounter((byte*)aes->reg);
4379
4380 aes->left = AES_BLOCK_SIZE;
4381 tmp = (byte*)aes->tmp;
4382
4383 while (sz--) {
4384 *(out++) = *(in++) ^ *(tmp++);
4385 aes->left--;
4386 }
4387 }
4388
4389 return 0;
4390 }
4391
4392 #endif /* NEED_AES_CTR_SOFT */
4393
4394 #endif /* WOLFSSL_AES_COUNTER */
4395 #endif /* !WOLFSSL_ARMASM */
4396
4397
4398 /*
4399 * The IV for AES GCM and CCM, stored in struct Aes's member reg, is comprised
4400 * of two parts in order:
4401 * 1. The fixed field which may be 0 or 4 bytes long. In TLS, this is set
4402 * to the implicit IV.
4403 * 2. The explicit IV is generated by wolfCrypt. It needs to be managed
4404 * by wolfCrypt to ensure the IV is unique for each call to encrypt.
4405 * The IV may be a 96-bit random value, or the 32-bit fixed value and a
4406 * 64-bit set of 0 or random data. The final 32-bits of reg is used as a
4407 * block counter during the encryption.
4408 */
4409
4410 #if (defined(HAVE_AESGCM) && !defined(WC_NO_RNG)) || defined(HAVE_AESCCM)
IncCtr(byte * ctr,word32 ctrSz)4411 static WC_INLINE void IncCtr(byte* ctr, word32 ctrSz)
4412 {
4413 int i;
4414 for (i = ctrSz-1; i >= 0; i--) {
4415 if (++ctr[i])
4416 break;
4417 }
4418 }
4419 #endif /* HAVE_AESGCM || HAVE_AESCCM */
4420
4421
4422 #ifdef HAVE_AESGCM
4423
4424 #ifdef WOLFSSL_AESGCM_STREAM
4425 /* Access initialization counter data. */
4426 #define AES_INITCTR(aes) ((aes)->streamData + 0 * AES_BLOCK_SIZE)
4427 /* Access counter data. */
4428 #define AES_COUNTER(aes) ((aes)->streamData + 1 * AES_BLOCK_SIZE)
4429 /* Access tag data. */
4430 #define AES_TAG(aes) ((aes)->streamData + 2 * AES_BLOCK_SIZE)
4431 /* Access last GHASH block. */
4432 #define AES_LASTGBLOCK(aes) ((aes)->streamData + 3 * AES_BLOCK_SIZE)
4433 /* Access last encrypted block. */
4434 #define AES_LASTBLOCK(aes) ((aes)->streamData + 4 * AES_BLOCK_SIZE)
4435 #endif
4436
4437 #if defined(HAVE_COLDFIRE_SEC)
4438 #error "Coldfire SEC doesn't currently support AES-GCM mode"
4439
4440 #endif
4441
4442 #ifdef WOLFSSL_ARMASM
4443 /* implementation is located in wolfcrypt/src/port/arm/armv8-aes.c */
4444
4445 #elif defined(WOLFSSL_AFALG)
4446 /* implemented in wolfcrypt/src/port/afalg/afalg_aes.c */
4447
4448 #elif defined(WOLFSSL_KCAPI_AES)
4449 /* implemented in wolfcrypt/src/port/kcapi/kcapi_aes.c */
4450
4451 #elif defined(WOLFSSL_DEVCRYPTO_AES)
4452 /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
4453
4454 #else /* software + AESNI implementation */
4455
4456 #if !defined(FREESCALE_LTC_AES_GCM)
IncrementGcmCounter(byte * inOutCtr)4457 static WC_INLINE void IncrementGcmCounter(byte* inOutCtr)
4458 {
4459 int i;
4460
4461 /* in network byte order so start at end and work back */
4462 for (i = AES_BLOCK_SIZE - 1; i >= AES_BLOCK_SIZE - CTR_SZ; i--) {
4463 if (++inOutCtr[i]) /* we're done unless we overflow */
4464 return;
4465 }
4466 }
4467 #endif /* !FREESCALE_LTC_AES_GCM */
4468
4469 #if defined(GCM_SMALL) || defined(GCM_TABLE) || defined(GCM_TABLE_4BIT)
4470
FlattenSzInBits(byte * buf,word32 sz)4471 static WC_INLINE void FlattenSzInBits(byte* buf, word32 sz)
4472 {
4473 /* Multiply the sz by 8 */
4474 word32 szHi = (sz >> (8*sizeof(sz) - 3));
4475 sz <<= 3;
4476
4477 /* copy over the words of the sz into the destination buffer */
4478 buf[0] = (szHi >> 24) & 0xff;
4479 buf[1] = (szHi >> 16) & 0xff;
4480 buf[2] = (szHi >> 8) & 0xff;
4481 buf[3] = szHi & 0xff;
4482 buf[4] = (sz >> 24) & 0xff;
4483 buf[5] = (sz >> 16) & 0xff;
4484 buf[6] = (sz >> 8) & 0xff;
4485 buf[7] = sz & 0xff;
4486 }
4487
4488
RIGHTSHIFTX(byte * x)4489 static WC_INLINE void RIGHTSHIFTX(byte* x)
4490 {
4491 int i;
4492 int carryIn = 0;
4493 int borrow = x[15] & 0x01;
4494
4495 for (i = 0; i < AES_BLOCK_SIZE; i++) {
4496 int carryOut = x[i] & 0x01;
4497 x[i] = (x[i] >> 1) | (carryIn ? 0x80 : 0);
4498 carryIn = carryOut;
4499 }
4500 if (borrow) x[0] ^= 0xE1;
4501 }
4502
4503 #endif /* defined(GCM_SMALL) || defined(GCM_TABLE) || defined(GCM_TABLE_4BIT) */
4504
4505
4506 #ifdef GCM_TABLE
4507
GenerateM0(Aes * aes)4508 static void GenerateM0(Aes* aes)
4509 {
4510 int i, j;
4511 byte (*m)[AES_BLOCK_SIZE] = aes->M0;
4512
4513 XMEMCPY(m[128], aes->H, AES_BLOCK_SIZE);
4514
4515 for (i = 64; i > 0; i /= 2) {
4516 XMEMCPY(m[i], m[i*2], AES_BLOCK_SIZE);
4517 RIGHTSHIFTX(m[i]);
4518 }
4519
4520 for (i = 2; i < 256; i *= 2) {
4521 for (j = 1; j < i; j++) {
4522 XMEMCPY(m[i+j], m[i], AES_BLOCK_SIZE);
4523 xorbuf(m[i+j], m[j], AES_BLOCK_SIZE);
4524 }
4525 }
4526
4527 XMEMSET(m[0], 0, AES_BLOCK_SIZE);
4528 }
4529
4530 #elif defined(GCM_TABLE_4BIT)
4531
Shift4_M0(byte * r8,byte * z8)4532 static WC_INLINE void Shift4_M0(byte *r8, byte* z8)
4533 {
4534 int i;
4535 for (i = 15; i > 0; i--)
4536 r8[i] = (z8[i-1] << 4) | (z8[i] >> 4);
4537 r8[0] = z8[0] >> 4;
4538 }
4539
GenerateM0(Aes * aes)4540 static void GenerateM0(Aes* aes)
4541 {
4542 #if !defined(BIG_ENDIAN_ORDER) && !defined(WC_16BIT_CPU)
4543 int i;
4544 #endif
4545 byte (*m)[AES_BLOCK_SIZE] = aes->M0;
4546
4547 /* 0 times -> 0x0 */
4548 XMEMSET(m[0x0], 0, AES_BLOCK_SIZE);
4549 /* 1 times -> 0x8 */
4550 XMEMCPY(m[0x8], aes->H, AES_BLOCK_SIZE);
4551 /* 2 times -> 0x4 */
4552 XMEMCPY(m[0x4], m[0x8], AES_BLOCK_SIZE);
4553 RIGHTSHIFTX(m[0x4]);
4554 /* 4 times -> 0x2 */
4555 XMEMCPY(m[0x2], m[0x4], AES_BLOCK_SIZE);
4556 RIGHTSHIFTX(m[0x2]);
4557 /* 8 times -> 0x1 */
4558 XMEMCPY(m[0x1], m[0x2], AES_BLOCK_SIZE);
4559 RIGHTSHIFTX(m[0x1]);
4560
4561 /* 0x3 */
4562 XMEMCPY(m[0x3], m[0x2], AES_BLOCK_SIZE);
4563 xorbuf (m[0x3], m[0x1], AES_BLOCK_SIZE);
4564
4565 /* 0x5 -> 0x7 */
4566 XMEMCPY(m[0x5], m[0x4], AES_BLOCK_SIZE);
4567 xorbuf (m[0x5], m[0x1], AES_BLOCK_SIZE);
4568 XMEMCPY(m[0x6], m[0x4], AES_BLOCK_SIZE);
4569 xorbuf (m[0x6], m[0x2], AES_BLOCK_SIZE);
4570 XMEMCPY(m[0x7], m[0x4], AES_BLOCK_SIZE);
4571 xorbuf (m[0x7], m[0x3], AES_BLOCK_SIZE);
4572
4573 /* 0x9 -> 0xf */
4574 XMEMCPY(m[0x9], m[0x8], AES_BLOCK_SIZE);
4575 xorbuf (m[0x9], m[0x1], AES_BLOCK_SIZE);
4576 XMEMCPY(m[0xa], m[0x8], AES_BLOCK_SIZE);
4577 xorbuf (m[0xa], m[0x2], AES_BLOCK_SIZE);
4578 XMEMCPY(m[0xb], m[0x8], AES_BLOCK_SIZE);
4579 xorbuf (m[0xb], m[0x3], AES_BLOCK_SIZE);
4580 XMEMCPY(m[0xc], m[0x8], AES_BLOCK_SIZE);
4581 xorbuf (m[0xc], m[0x4], AES_BLOCK_SIZE);
4582 XMEMCPY(m[0xd], m[0x8], AES_BLOCK_SIZE);
4583 xorbuf (m[0xd], m[0x5], AES_BLOCK_SIZE);
4584 XMEMCPY(m[0xe], m[0x8], AES_BLOCK_SIZE);
4585 xorbuf (m[0xe], m[0x6], AES_BLOCK_SIZE);
4586 XMEMCPY(m[0xf], m[0x8], AES_BLOCK_SIZE);
4587 xorbuf (m[0xf], m[0x7], AES_BLOCK_SIZE);
4588
4589 #if !defined(BIG_ENDIAN_ORDER) && !defined(WC_16BIT_CPU)
4590 for (i = 0; i < 16; i++) {
4591 Shift4_M0(m[16+i], m[i]);
4592 }
4593 #endif
4594 }
4595
4596 #endif /* GCM_TABLE */
4597
4598 /* Software AES - GCM SetKey */
wc_AesGcmSetKey(Aes * aes,const byte * key,word32 len)4599 int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
4600 {
4601 int ret;
4602 byte iv[AES_BLOCK_SIZE];
4603
4604 #ifdef WOLFSSL_IMX6_CAAM_BLOB
4605 byte local[32];
4606 word32 localSz = 32;
4607
4608 if (len == (16 + WC_CAAM_BLOB_SZ) ||
4609 len == (24 + WC_CAAM_BLOB_SZ) ||
4610 len == (32 + WC_CAAM_BLOB_SZ)) {
4611 if (wc_caamOpenBlob((byte*)key, len, local, &localSz) != 0) {
4612 return BAD_FUNC_ARG;
4613 }
4614
4615 /* set local values */
4616 key = local;
4617 len = localSz;
4618 }
4619 #endif
4620
4621 if (!((len == 16) || (len == 24) || (len == 32)))
4622 return BAD_FUNC_ARG;
4623 if (aes == NULL)
4624 return BAD_FUNC_ARG;
4625
4626 #ifdef OPENSSL_EXTRA
4627 XMEMSET(aes->aadH, 0, sizeof(aes->aadH));
4628 aes->aadLen = 0;
4629 #endif
4630 XMEMSET(iv, 0, AES_BLOCK_SIZE);
4631 ret = wc_AesSetKey(aes, key, len, iv, AES_ENCRYPTION);
4632 #ifdef WOLFSSL_AESGCM_STREAM
4633 aes->gcmKeySet = 1;
4634 #endif
4635
4636 #ifdef WOLFSSL_AESNI
4637 /* AES-NI code generates its own H value. */
4638 if (haveAESNI)
4639 return ret;
4640 #endif /* WOLFSSL_AESNI */
4641
4642 #if !defined(FREESCALE_LTC_AES_GCM)
4643 if (ret == 0) {
4644 wc_AesEncrypt(aes, iv, aes->H);
4645 #if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT)
4646 GenerateM0(aes);
4647 #endif /* GCM_TABLE */
4648 }
4649 #endif /* FREESCALE_LTC_AES_GCM */
4650
4651 #if defined(WOLFSSL_XILINX_CRYPT)
4652 wc_AesGcmSetKey_ex(aes, key, len, XSECURE_CSU_AES_KEY_SRC_KUP);
4653 #elif defined(WOLFSSL_AFALG_XILINX_AES)
4654 wc_AesGcmSetKey_ex(aes, key, len, 0);
4655 #endif
4656
4657 #ifdef WOLF_CRYPTO_CB
4658 if (aes->devId != INVALID_DEVID) {
4659 XMEMCPY(aes->devKey, key, len);
4660 }
4661 #endif
4662
4663 #ifdef WOLFSSL_IMX6_CAAM_BLOB
4664 ForceZero(local, sizeof(local));
4665 #endif
4666 return ret;
4667 }
4668
4669
4670 #ifdef WOLFSSL_AESNI
4671
4672 #if defined(USE_INTEL_SPEEDUP)
4673 #define HAVE_INTEL_AVX1
4674 #define HAVE_INTEL_AVX2
4675 #endif /* USE_INTEL_SPEEDUP */
4676
4677 #ifndef _MSC_VER
4678
4679 void AES_GCM_encrypt(const unsigned char *in, unsigned char *out,
4680 const unsigned char* addt, const unsigned char* ivec,
4681 unsigned char *tag, word32 nbytes,
4682 word32 abytes, word32 ibytes,
4683 word32 tbytes, const unsigned char* key, int nr)
4684 XASM_LINK("AES_GCM_encrypt");
4685 #ifdef HAVE_INTEL_AVX1
4686 void AES_GCM_encrypt_avx1(const unsigned char *in, unsigned char *out,
4687 const unsigned char* addt, const unsigned char* ivec,
4688 unsigned char *tag, word32 nbytes,
4689 word32 abytes, word32 ibytes,
4690 word32 tbytes, const unsigned char* key,
4691 int nr)
4692 XASM_LINK("AES_GCM_encrypt_avx1");
4693 #ifdef HAVE_INTEL_AVX2
4694 void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out,
4695 const unsigned char* addt, const unsigned char* ivec,
4696 unsigned char *tag, word32 nbytes,
4697 word32 abytes, word32 ibytes,
4698 word32 tbytes, const unsigned char* key,
4699 int nr)
4700 XASM_LINK("AES_GCM_encrypt_avx2");
4701 #endif /* HAVE_INTEL_AVX2 */
4702 #endif /* HAVE_INTEL_AVX1 */
4703
4704 #ifdef HAVE_AES_DECRYPT
4705 void AES_GCM_decrypt(const unsigned char *in, unsigned char *out,
4706 const unsigned char* addt, const unsigned char* ivec,
4707 const unsigned char *tag, word32 nbytes, word32 abytes,
4708 word32 ibytes, word32 tbytes, const unsigned char* key,
4709 int nr, int* res)
4710 XASM_LINK("AES_GCM_decrypt");
4711 #ifdef HAVE_INTEL_AVX1
4712 void AES_GCM_decrypt_avx1(const unsigned char *in, unsigned char *out,
4713 const unsigned char* addt, const unsigned char* ivec,
4714 const unsigned char *tag, word32 nbytes,
4715 word32 abytes, word32 ibytes, word32 tbytes,
4716 const unsigned char* key, int nr, int* res)
4717 XASM_LINK("AES_GCM_decrypt_avx1");
4718 #ifdef HAVE_INTEL_AVX2
4719 void AES_GCM_decrypt_avx2(const unsigned char *in, unsigned char *out,
4720 const unsigned char* addt, const unsigned char* ivec,
4721 const unsigned char *tag, word32 nbytes,
4722 word32 abytes, word32 ibytes, word32 tbytes,
4723 const unsigned char* key, int nr, int* res)
4724 XASM_LINK("AES_GCM_decrypt_avx2");
4725 #endif /* HAVE_INTEL_AVX2 */
4726 #endif /* HAVE_INTEL_AVX1 */
4727 #endif /* HAVE_AES_DECRYPT */
4728
4729 #else /* _MSC_VER */
4730
4731 #define S(w,z) ((char)((unsigned long long)(w) >> (8*(7-(z))) & 0xFF))
4732 #define M128_INIT(x,y) { S((x),7), S((x),6), S((x),5), S((x),4), \
4733 S((x),3), S((x),2), S((x),1), S((x),0), \
4734 S((y),7), S((y),6), S((y),5), S((y),4), \
4735 S((y),3), S((y),2), S((y),1), S((y),0) }
4736
4737 static const __m128i MOD2_128 =
4738 M128_INIT(0x1, (long long int)0xc200000000000000UL);
4739
4740
4741 /* See Intel Carry-Less Multiplication Instruction
4742 * and its Usage for Computing the GCM Mode White Paper
4743 * by Shay Gueron, Intel Mobility Group, Israel Development Center;
4744 * and Michael E. Kounavis, Intel Labs, Circuits and Systems Research */
4745
4746
4747 /* Figure 9. AES-GCM - Encrypt With Single Block Ghash at a Time */
4748
4749 static const __m128i ONE = M128_INIT(0x0, 0x1);
4750 #ifndef AES_GCM_AESNI_NO_UNROLL
4751 static const __m128i TWO = M128_INIT(0x0, 0x2);
4752 static const __m128i THREE = M128_INIT(0x0, 0x3);
4753 static const __m128i FOUR = M128_INIT(0x0, 0x4);
4754 static const __m128i FIVE = M128_INIT(0x0, 0x5);
4755 static const __m128i SIX = M128_INIT(0x0, 0x6);
4756 static const __m128i SEVEN = M128_INIT(0x0, 0x7);
4757 static const __m128i EIGHT = M128_INIT(0x0, 0x8);
4758 #endif
4759 static const __m128i BSWAP_EPI64 =
4760 M128_INIT(0x0001020304050607, 0x08090a0b0c0d0e0f);
4761 static const __m128i BSWAP_MASK =
4762 M128_INIT(0x08090a0b0c0d0e0f, 0x0001020304050607);
4763
4764
4765 /* The following are for MSC based builds which do not allow
4766 * inline assembly. Intrinsic functions are used instead. */
4767
4768 #define aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T) \
4769 do \
4770 { \
4771 word32 iv12[4]; \
4772 iv12[0] = *(word32*)&ivec[0]; \
4773 iv12[1] = *(word32*)&ivec[4]; \
4774 iv12[2] = *(word32*)&ivec[8]; \
4775 iv12[3] = 0x01000000; \
4776 Y = _mm_loadu_si128((__m128i*)iv12); \
4777 \
4778 /* (Compute E[ZERO, KS] and E[Y0, KS] together */ \
4779 tmp1 = _mm_load_si128(&KEY[0]); \
4780 tmp2 = _mm_xor_si128(Y, KEY[0]); \
4781 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \
4782 tmp2 = _mm_aesenc_si128(tmp2, KEY[1]); \
4783 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \
4784 tmp2 = _mm_aesenc_si128(tmp2, KEY[2]); \
4785 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \
4786 tmp2 = _mm_aesenc_si128(tmp2, KEY[3]); \
4787 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \
4788 tmp2 = _mm_aesenc_si128(tmp2, KEY[4]); \
4789 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \
4790 tmp2 = _mm_aesenc_si128(tmp2, KEY[5]); \
4791 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \
4792 tmp2 = _mm_aesenc_si128(tmp2, KEY[6]); \
4793 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \
4794 tmp2 = _mm_aesenc_si128(tmp2, KEY[7]); \
4795 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \
4796 tmp2 = _mm_aesenc_si128(tmp2, KEY[8]); \
4797 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \
4798 tmp2 = _mm_aesenc_si128(tmp2, KEY[9]); \
4799 lastKey = KEY[10]; \
4800 if (nr > 10) { \
4801 tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
4802 tmp2 = _mm_aesenc_si128(tmp2, lastKey); \
4803 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \
4804 tmp2 = _mm_aesenc_si128(tmp2, KEY[11]); \
4805 lastKey = KEY[12]; \
4806 if (nr > 12) { \
4807 tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
4808 tmp2 = _mm_aesenc_si128(tmp2, lastKey); \
4809 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \
4810 tmp2 = _mm_aesenc_si128(tmp2, KEY[13]); \
4811 lastKey = KEY[14]; \
4812 } \
4813 } \
4814 H = _mm_aesenclast_si128(tmp1, lastKey); \
4815 T = _mm_aesenclast_si128(tmp2, lastKey); \
4816 H = _mm_shuffle_epi8(H, BSWAP_MASK); \
4817 } \
4818 while (0)
4819
4820 #define aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T) \
4821 do \
4822 { \
4823 if (ibytes % 16) { \
4824 i = ibytes / 16; \
4825 for (j=0; j < (int)(ibytes%16); j++) \
4826 ((unsigned char*)&last_block)[j] = ivec[i*16+j]; \
4827 } \
4828 tmp1 = _mm_load_si128(&KEY[0]); \
4829 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \
4830 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \
4831 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \
4832 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \
4833 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \
4834 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \
4835 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \
4836 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \
4837 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \
4838 lastKey = KEY[10]; \
4839 if (nr > 10) { \
4840 tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
4841 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \
4842 lastKey = KEY[12]; \
4843 if (nr > 12) { \
4844 tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
4845 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \
4846 lastKey = KEY[14]; \
4847 } \
4848 } \
4849 H = _mm_aesenclast_si128(tmp1, lastKey); \
4850 H = _mm_shuffle_epi8(H, BSWAP_MASK); \
4851 Y = _mm_setzero_si128(); \
4852 for (i=0; i < (int)(ibytes/16); i++) { \
4853 tmp1 = _mm_loadu_si128(&((__m128i*)ivec)[i]); \
4854 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); \
4855 Y = _mm_xor_si128(Y, tmp1); \
4856 Y = gfmul_sw(Y, H); \
4857 } \
4858 if (ibytes % 16) { \
4859 tmp1 = last_block; \
4860 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); \
4861 Y = _mm_xor_si128(Y, tmp1); \
4862 Y = gfmul_sw(Y, H); \
4863 } \
4864 tmp1 = _mm_insert_epi64(tmp1, ibytes*8, 0); \
4865 tmp1 = _mm_insert_epi64(tmp1, 0, 1); \
4866 Y = _mm_xor_si128(Y, tmp1); \
4867 Y = gfmul_sw(Y, H); \
4868 Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /* Compute E(K, Y0) */ \
4869 tmp1 = _mm_xor_si128(Y, KEY[0]); \
4870 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \
4871 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \
4872 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \
4873 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \
4874 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \
4875 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \
4876 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \
4877 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \
4878 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \
4879 lastKey = KEY[10]; \
4880 if (nr > 10) { \
4881 tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
4882 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \
4883 lastKey = KEY[12]; \
4884 if (nr > 12) { \
4885 tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
4886 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \
4887 lastKey = KEY[14]; \
4888 } \
4889 } \
4890 T = _mm_aesenclast_si128(tmp1, lastKey); \
4891 } \
4892 while (0)
4893
4894 #define AES_ENC_8(j) \
4895 tmp1 = _mm_aesenc_si128(tmp1, KEY[j]); \
4896 tmp2 = _mm_aesenc_si128(tmp2, KEY[j]); \
4897 tmp3 = _mm_aesenc_si128(tmp3, KEY[j]); \
4898 tmp4 = _mm_aesenc_si128(tmp4, KEY[j]); \
4899 tmp5 = _mm_aesenc_si128(tmp5, KEY[j]); \
4900 tmp6 = _mm_aesenc_si128(tmp6, KEY[j]); \
4901 tmp7 = _mm_aesenc_si128(tmp7, KEY[j]); \
4902 tmp8 = _mm_aesenc_si128(tmp8, KEY[j]);
4903
4904 #define AES_ENC_LAST_8() \
4905 tmp1 =_mm_aesenclast_si128(tmp1, lastKey); \
4906 tmp2 =_mm_aesenclast_si128(tmp2, lastKey); \
4907 tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[i*8+0])); \
4908 tmp2 = _mm_xor_si128(tmp2, _mm_loadu_si128(&((__m128i*)in)[i*8+1])); \
4909 _mm_storeu_si128(&((__m128i*)out)[i*8+0], tmp1); \
4910 _mm_storeu_si128(&((__m128i*)out)[i*8+1], tmp2); \
4911 tmp3 =_mm_aesenclast_si128(tmp3, lastKey); \
4912 tmp4 =_mm_aesenclast_si128(tmp4, lastKey); \
4913 tmp3 = _mm_xor_si128(tmp3, _mm_loadu_si128(&((__m128i*)in)[i*8+2])); \
4914 tmp4 = _mm_xor_si128(tmp4, _mm_loadu_si128(&((__m128i*)in)[i*8+3])); \
4915 _mm_storeu_si128(&((__m128i*)out)[i*8+2], tmp3); \
4916 _mm_storeu_si128(&((__m128i*)out)[i*8+3], tmp4); \
4917 tmp5 =_mm_aesenclast_si128(tmp5, lastKey); \
4918 tmp6 =_mm_aesenclast_si128(tmp6, lastKey); \
4919 tmp5 = _mm_xor_si128(tmp5, _mm_loadu_si128(&((__m128i*)in)[i*8+4])); \
4920 tmp6 = _mm_xor_si128(tmp6, _mm_loadu_si128(&((__m128i*)in)[i*8+5])); \
4921 _mm_storeu_si128(&((__m128i*)out)[i*8+4], tmp5); \
4922 _mm_storeu_si128(&((__m128i*)out)[i*8+5], tmp6); \
4923 tmp7 =_mm_aesenclast_si128(tmp7, lastKey); \
4924 tmp8 =_mm_aesenclast_si128(tmp8, lastKey); \
4925 tmp7 = _mm_xor_si128(tmp7, _mm_loadu_si128(&((__m128i*)in)[i*8+6])); \
4926 tmp8 = _mm_xor_si128(tmp8, _mm_loadu_si128(&((__m128i*)in)[i*8+7])); \
4927 _mm_storeu_si128(&((__m128i*)out)[i*8+6], tmp7); \
4928 _mm_storeu_si128(&((__m128i*)out)[i*8+7], tmp8);
4929
4930
gfmul_sw(__m128i a,__m128i b)4931 static __m128i gfmul_sw(__m128i a, __m128i b)
4932 {
4933 __m128i r, t1, t2, t3, t4, t5, t6, t7;
4934 t2 = _mm_shuffle_epi32(b, 78);
4935 t3 = _mm_shuffle_epi32(a, 78);
4936 t2 = _mm_xor_si128(t2, b);
4937 t3 = _mm_xor_si128(t3, a);
4938 t4 = _mm_clmulepi64_si128(b, a, 0x11);
4939 t1 = _mm_clmulepi64_si128(b, a, 0x00);
4940 t2 = _mm_clmulepi64_si128(t2, t3, 0x00);
4941 t2 = _mm_xor_si128(t2, t1);
4942 t2 = _mm_xor_si128(t2, t4);
4943 t3 = _mm_slli_si128(t2, 8);
4944 t2 = _mm_srli_si128(t2, 8);
4945 t1 = _mm_xor_si128(t1, t3);
4946 t4 = _mm_xor_si128(t4, t2);
4947
4948 t5 = _mm_srli_epi32(t1, 31);
4949 t6 = _mm_srli_epi32(t4, 31);
4950 t1 = _mm_slli_epi32(t1, 1);
4951 t4 = _mm_slli_epi32(t4, 1);
4952 t7 = _mm_srli_si128(t5, 12);
4953 t5 = _mm_slli_si128(t5, 4);
4954 t6 = _mm_slli_si128(t6, 4);
4955 t4 = _mm_or_si128(t4, t7);
4956 t1 = _mm_or_si128(t1, t5);
4957 t4 = _mm_or_si128(t4, t6);
4958
4959 t5 = _mm_slli_epi32(t1, 31);
4960 t6 = _mm_slli_epi32(t1, 30);
4961 t7 = _mm_slli_epi32(t1, 25);
4962 t5 = _mm_xor_si128(t5, t6);
4963 t5 = _mm_xor_si128(t5, t7);
4964
4965 t6 = _mm_srli_si128(t5, 4);
4966 t5 = _mm_slli_si128(t5, 12);
4967 t1 = _mm_xor_si128(t1, t5);
4968 t7 = _mm_srli_epi32(t1, 1);
4969 t3 = _mm_srli_epi32(t1, 2);
4970 t2 = _mm_srli_epi32(t1, 7);
4971
4972 t7 = _mm_xor_si128(t7, t3);
4973 t7 = _mm_xor_si128(t7, t2);
4974 t7 = _mm_xor_si128(t7, t6);
4975 t7 = _mm_xor_si128(t7, t1);
4976 r = _mm_xor_si128(t4, t7);
4977
4978 return r;
4979 }
4980
gfmul_only(__m128i a,__m128i b,__m128i * r0,__m128i * r1)4981 static void gfmul_only(__m128i a, __m128i b, __m128i* r0, __m128i* r1)
4982 {
4983 __m128i t1, t2, t3, t4;
4984
4985 /* 128 x 128 Carryless Multiply */
4986 t2 = _mm_shuffle_epi32(b, 78);
4987 t3 = _mm_shuffle_epi32(a, 78);
4988 t2 = _mm_xor_si128(t2, b);
4989 t3 = _mm_xor_si128(t3, a);
4990 t4 = _mm_clmulepi64_si128(b, a, 0x11);
4991 t1 = _mm_clmulepi64_si128(b, a, 0x00);
4992 t2 = _mm_clmulepi64_si128(t2, t3, 0x00);
4993 t2 = _mm_xor_si128(t2, t1);
4994 t2 = _mm_xor_si128(t2, t4);
4995 t3 = _mm_slli_si128(t2, 8);
4996 t2 = _mm_srli_si128(t2, 8);
4997 t1 = _mm_xor_si128(t1, t3);
4998 t4 = _mm_xor_si128(t4, t2);
4999 *r0 = _mm_xor_si128(t1, *r0);
5000 *r1 = _mm_xor_si128(t4, *r1);
5001 }
5002
gfmul_shl1(__m128i a)5003 static __m128i gfmul_shl1(__m128i a)
5004 {
5005 __m128i t1 = a, t2;
5006 t2 = _mm_srli_epi64(t1, 63);
5007 t1 = _mm_slli_epi64(t1, 1);
5008 t2 = _mm_slli_si128(t2, 8);
5009 t1 = _mm_or_si128(t1, t2);
5010 /* if (a[1] >> 63) t1 = _mm_xor_si128(t1, MOD2_128); */
5011 a = _mm_shuffle_epi32(a, 0xff);
5012 a = _mm_srai_epi32(a, 31);
5013 a = _mm_and_si128(a, MOD2_128);
5014 t1 = _mm_xor_si128(t1, a);
5015 return t1;
5016 }
5017
ghash_red(__m128i r0,__m128i r1)5018 static __m128i ghash_red(__m128i r0, __m128i r1)
5019 {
5020 __m128i t2, t3;
5021 __m128i t5, t6, t7;
5022
5023 t5 = _mm_slli_epi32(r0, 31);
5024 t6 = _mm_slli_epi32(r0, 30);
5025 t7 = _mm_slli_epi32(r0, 25);
5026 t5 = _mm_xor_si128(t5, t6);
5027 t5 = _mm_xor_si128(t5, t7);
5028
5029 t6 = _mm_srli_si128(t5, 4);
5030 t5 = _mm_slli_si128(t5, 12);
5031 r0 = _mm_xor_si128(r0, t5);
5032 t7 = _mm_srli_epi32(r0, 1);
5033 t3 = _mm_srli_epi32(r0, 2);
5034 t2 = _mm_srli_epi32(r0, 7);
5035
5036 t7 = _mm_xor_si128(t7, t3);
5037 t7 = _mm_xor_si128(t7, t2);
5038 t7 = _mm_xor_si128(t7, t6);
5039 t7 = _mm_xor_si128(t7, r0);
5040 return _mm_xor_si128(r1, t7);
5041 }
5042
gfmul_shifted(__m128i a,__m128i b)5043 static __m128i gfmul_shifted(__m128i a, __m128i b)
5044 {
5045 __m128i t0 = _mm_setzero_si128(), t1 = _mm_setzero_si128();
5046 gfmul_only(a, b, &t0, &t1);
5047 return ghash_red(t0, t1);
5048 }
5049
5050 #ifndef AES_GCM_AESNI_NO_UNROLL
gfmul8(__m128i a1,__m128i a2,__m128i a3,__m128i a4,__m128i a5,__m128i a6,__m128i a7,__m128i a8,__m128i b1,__m128i b2,__m128i b3,__m128i b4,__m128i b5,__m128i b6,__m128i b7,__m128i b8)5051 static __m128i gfmul8(__m128i a1, __m128i a2, __m128i a3, __m128i a4,
5052 __m128i a5, __m128i a6, __m128i a7, __m128i a8,
5053 __m128i b1, __m128i b2, __m128i b3, __m128i b4,
5054 __m128i b5, __m128i b6, __m128i b7, __m128i b8)
5055 {
5056 __m128i t0 = _mm_setzero_si128(), t1 = _mm_setzero_si128();
5057 gfmul_only(a1, b8, &t0, &t1);
5058 gfmul_only(a2, b7, &t0, &t1);
5059 gfmul_only(a3, b6, &t0, &t1);
5060 gfmul_only(a4, b5, &t0, &t1);
5061 gfmul_only(a5, b4, &t0, &t1);
5062 gfmul_only(a6, b3, &t0, &t1);
5063 gfmul_only(a7, b2, &t0, &t1);
5064 gfmul_only(a8, b1, &t0, &t1);
5065 return ghash_red(t0, t1);
5066 }
5067 #endif
5068
5069
AES_GCM_encrypt(const unsigned char * in,unsigned char * out,const unsigned char * addt,const unsigned char * ivec,unsigned char * tag,word32 nbytes,word32 abytes,word32 ibytes,word32 tbytes,const unsigned char * key,int nr)5070 static void AES_GCM_encrypt(const unsigned char *in, unsigned char *out,
5071 const unsigned char* addt,
5072 const unsigned char* ivec, unsigned char *tag,
5073 word32 nbytes, word32 abytes, word32 ibytes,
5074 word32 tbytes, const unsigned char* key, int nr)
5075 {
5076 int i, j ,k;
5077 __m128i ctr1;
5078 __m128i H, Y, T;
5079 __m128i X = _mm_setzero_si128();
5080 __m128i *KEY = (__m128i*)key, lastKey;
5081 __m128i last_block = _mm_setzero_si128();
5082 __m128i tmp1, tmp2;
5083 #ifndef AES_GCM_AESNI_NO_UNROLL
5084 __m128i HT[8];
5085 __m128i r0, r1;
5086 __m128i XV;
5087 __m128i tmp3, tmp4, tmp5, tmp6, tmp7, tmp8;
5088 #endif
5089
5090 if (ibytes == GCM_NONCE_MID_SZ)
5091 aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T);
5092 else
5093 aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T);
5094
5095 for (i=0; i < (int)(abytes/16); i++) {
5096 tmp1 = _mm_loadu_si128(&((__m128i*)addt)[i]);
5097 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5098 X = _mm_xor_si128(X, tmp1);
5099 X = gfmul_sw(X, H);
5100 }
5101 if (abytes%16) {
5102 last_block = _mm_setzero_si128();
5103 for (j=0; j < (int)(abytes%16); j++)
5104 ((unsigned char*)&last_block)[j] = addt[i*16+j];
5105 tmp1 = last_block;
5106 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5107 X = _mm_xor_si128(X, tmp1);
5108 X = gfmul_sw(X, H);
5109 }
5110 tmp1 = _mm_shuffle_epi8(Y, BSWAP_EPI64);
5111 ctr1 = _mm_add_epi32(tmp1, ONE);
5112 H = gfmul_shl1(H);
5113
5114 #ifndef AES_GCM_AESNI_NO_UNROLL
5115 i = 0;
5116 if (nbytes >= 16*8) {
5117 HT[0] = H;
5118 HT[1] = gfmul_shifted(H, H);
5119 HT[2] = gfmul_shifted(H, HT[1]);
5120 HT[3] = gfmul_shifted(HT[1], HT[1]);
5121 HT[4] = gfmul_shifted(HT[1], HT[2]);
5122 HT[5] = gfmul_shifted(HT[2], HT[2]);
5123 HT[6] = gfmul_shifted(HT[2], HT[3]);
5124 HT[7] = gfmul_shifted(HT[3], HT[3]);
5125
5126 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5127 tmp2 = _mm_add_epi32(ctr1, ONE);
5128 tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_EPI64);
5129 tmp3 = _mm_add_epi32(ctr1, TWO);
5130 tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_EPI64);
5131 tmp4 = _mm_add_epi32(ctr1, THREE);
5132 tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_EPI64);
5133 tmp5 = _mm_add_epi32(ctr1, FOUR);
5134 tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_EPI64);
5135 tmp6 = _mm_add_epi32(ctr1, FIVE);
5136 tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_EPI64);
5137 tmp7 = _mm_add_epi32(ctr1, SIX);
5138 tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_EPI64);
5139 tmp8 = _mm_add_epi32(ctr1, SEVEN);
5140 tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_EPI64);
5141 ctr1 = _mm_add_epi32(ctr1, EIGHT);
5142 tmp1 =_mm_xor_si128(tmp1, KEY[0]);
5143 tmp2 =_mm_xor_si128(tmp2, KEY[0]);
5144 tmp3 =_mm_xor_si128(tmp3, KEY[0]);
5145 tmp4 =_mm_xor_si128(tmp4, KEY[0]);
5146 tmp5 =_mm_xor_si128(tmp5, KEY[0]);
5147 tmp6 =_mm_xor_si128(tmp6, KEY[0]);
5148 tmp7 =_mm_xor_si128(tmp7, KEY[0]);
5149 tmp8 =_mm_xor_si128(tmp8, KEY[0]);
5150 AES_ENC_8(1);
5151 AES_ENC_8(2);
5152 AES_ENC_8(3);
5153 AES_ENC_8(4);
5154 AES_ENC_8(5);
5155 AES_ENC_8(6);
5156 AES_ENC_8(7);
5157 AES_ENC_8(8);
5158 AES_ENC_8(9);
5159 lastKey = KEY[10];
5160 if (nr > 10) {
5161 AES_ENC_8(10);
5162 AES_ENC_8(11);
5163 lastKey = KEY[12];
5164 if (nr > 12) {
5165 AES_ENC_8(12);
5166 AES_ENC_8(13);
5167 lastKey = KEY[14];
5168 }
5169 }
5170 AES_ENC_LAST_8();
5171
5172 for (i=1; i < (int)(nbytes/16/8); i++) {
5173 r0 = _mm_setzero_si128();
5174 r1 = _mm_setzero_si128();
5175 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5176 tmp2 = _mm_add_epi32(ctr1, ONE);
5177 tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_EPI64);
5178 tmp3 = _mm_add_epi32(ctr1, TWO);
5179 tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_EPI64);
5180 tmp4 = _mm_add_epi32(ctr1, THREE);
5181 tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_EPI64);
5182 tmp5 = _mm_add_epi32(ctr1, FOUR);
5183 tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_EPI64);
5184 tmp6 = _mm_add_epi32(ctr1, FIVE);
5185 tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_EPI64);
5186 tmp7 = _mm_add_epi32(ctr1, SIX);
5187 tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_EPI64);
5188 tmp8 = _mm_add_epi32(ctr1, SEVEN);
5189 tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_EPI64);
5190 ctr1 = _mm_add_epi32(ctr1, EIGHT);
5191 tmp1 =_mm_xor_si128(tmp1, KEY[0]);
5192 tmp2 =_mm_xor_si128(tmp2, KEY[0]);
5193 tmp3 =_mm_xor_si128(tmp3, KEY[0]);
5194 tmp4 =_mm_xor_si128(tmp4, KEY[0]);
5195 tmp5 =_mm_xor_si128(tmp5, KEY[0]);
5196 tmp6 =_mm_xor_si128(tmp6, KEY[0]);
5197 tmp7 =_mm_xor_si128(tmp7, KEY[0]);
5198 tmp8 =_mm_xor_si128(tmp8, KEY[0]);
5199 /* 128 x 128 Carryless Multiply */
5200 XV = _mm_loadu_si128(&((__m128i*)out)[(i-1)*8+0]);
5201 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5202 XV = _mm_xor_si128(XV, X);
5203 gfmul_only(XV, HT[7], &r0, &r1);
5204 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
5205 tmp2 = _mm_aesenc_si128(tmp2, KEY[1]);
5206 tmp3 = _mm_aesenc_si128(tmp3, KEY[1]);
5207 tmp4 = _mm_aesenc_si128(tmp4, KEY[1]);
5208 tmp5 = _mm_aesenc_si128(tmp5, KEY[1]);
5209 tmp6 = _mm_aesenc_si128(tmp6, KEY[1]);
5210 tmp7 = _mm_aesenc_si128(tmp7, KEY[1]);
5211 tmp8 = _mm_aesenc_si128(tmp8, KEY[1]);
5212 /* 128 x 128 Carryless Multiply */
5213 XV = _mm_loadu_si128(&((__m128i*)out)[(i-1)*8+1]);
5214 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5215 gfmul_only(XV, HT[6], &r0, &r1);
5216 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
5217 tmp2 = _mm_aesenc_si128(tmp2, KEY[2]);
5218 tmp3 = _mm_aesenc_si128(tmp3, KEY[2]);
5219 tmp4 = _mm_aesenc_si128(tmp4, KEY[2]);
5220 tmp5 = _mm_aesenc_si128(tmp5, KEY[2]);
5221 tmp6 = _mm_aesenc_si128(tmp6, KEY[2]);
5222 tmp7 = _mm_aesenc_si128(tmp7, KEY[2]);
5223 tmp8 = _mm_aesenc_si128(tmp8, KEY[2]);
5224 /* 128 x 128 Carryless Multiply */
5225 XV = _mm_loadu_si128(&((__m128i*)out)[(i-1)*8+2]);
5226 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5227 gfmul_only(XV, HT[5], &r0, &r1);
5228 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
5229 tmp2 = _mm_aesenc_si128(tmp2, KEY[3]);
5230 tmp3 = _mm_aesenc_si128(tmp3, KEY[3]);
5231 tmp4 = _mm_aesenc_si128(tmp4, KEY[3]);
5232 tmp5 = _mm_aesenc_si128(tmp5, KEY[3]);
5233 tmp6 = _mm_aesenc_si128(tmp6, KEY[3]);
5234 tmp7 = _mm_aesenc_si128(tmp7, KEY[3]);
5235 tmp8 = _mm_aesenc_si128(tmp8, KEY[3]);
5236 /* 128 x 128 Carryless Multiply */
5237 XV = _mm_loadu_si128(&((__m128i*)out)[(i-1)*8+3]);
5238 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5239 gfmul_only(XV, HT[4], &r0, &r1);
5240 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
5241 tmp2 = _mm_aesenc_si128(tmp2, KEY[4]);
5242 tmp3 = _mm_aesenc_si128(tmp3, KEY[4]);
5243 tmp4 = _mm_aesenc_si128(tmp4, KEY[4]);
5244 tmp5 = _mm_aesenc_si128(tmp5, KEY[4]);
5245 tmp6 = _mm_aesenc_si128(tmp6, KEY[4]);
5246 tmp7 = _mm_aesenc_si128(tmp7, KEY[4]);
5247 tmp8 = _mm_aesenc_si128(tmp8, KEY[4]);
5248 /* 128 x 128 Carryless Multiply */
5249 XV = _mm_loadu_si128(&((__m128i*)out)[(i-1)*8+4]);
5250 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5251 gfmul_only(XV, HT[3], &r0, &r1);
5252 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
5253 tmp2 = _mm_aesenc_si128(tmp2, KEY[5]);
5254 tmp3 = _mm_aesenc_si128(tmp3, KEY[5]);
5255 tmp4 = _mm_aesenc_si128(tmp4, KEY[5]);
5256 tmp5 = _mm_aesenc_si128(tmp5, KEY[5]);
5257 tmp6 = _mm_aesenc_si128(tmp6, KEY[5]);
5258 tmp7 = _mm_aesenc_si128(tmp7, KEY[5]);
5259 tmp8 = _mm_aesenc_si128(tmp8, KEY[5]);
5260 /* 128 x 128 Carryless Multiply */
5261 XV = _mm_loadu_si128(&((__m128i*)out)[(i-1)*8+5]);
5262 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5263 gfmul_only(XV, HT[2], &r0, &r1);
5264 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
5265 tmp2 = _mm_aesenc_si128(tmp2, KEY[6]);
5266 tmp3 = _mm_aesenc_si128(tmp3, KEY[6]);
5267 tmp4 = _mm_aesenc_si128(tmp4, KEY[6]);
5268 tmp5 = _mm_aesenc_si128(tmp5, KEY[6]);
5269 tmp6 = _mm_aesenc_si128(tmp6, KEY[6]);
5270 tmp7 = _mm_aesenc_si128(tmp7, KEY[6]);
5271 tmp8 = _mm_aesenc_si128(tmp8, KEY[6]);
5272 /* 128 x 128 Carryless Multiply */
5273 XV = _mm_loadu_si128(&((__m128i*)out)[(i-1)*8+6]);
5274 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5275 gfmul_only(XV, HT[1], &r0, &r1);
5276 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
5277 tmp2 = _mm_aesenc_si128(tmp2, KEY[7]);
5278 tmp3 = _mm_aesenc_si128(tmp3, KEY[7]);
5279 tmp4 = _mm_aesenc_si128(tmp4, KEY[7]);
5280 tmp5 = _mm_aesenc_si128(tmp5, KEY[7]);
5281 tmp6 = _mm_aesenc_si128(tmp6, KEY[7]);
5282 tmp7 = _mm_aesenc_si128(tmp7, KEY[7]);
5283 tmp8 = _mm_aesenc_si128(tmp8, KEY[7]);
5284 /* 128 x 128 Carryless Multiply */
5285 XV = _mm_loadu_si128(&((__m128i*)out)[(i-1)*8+7]);
5286 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5287 gfmul_only(XV, HT[0], &r0, &r1);
5288 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
5289 tmp2 = _mm_aesenc_si128(tmp2, KEY[8]);
5290 tmp3 = _mm_aesenc_si128(tmp3, KEY[8]);
5291 tmp4 = _mm_aesenc_si128(tmp4, KEY[8]);
5292 tmp5 = _mm_aesenc_si128(tmp5, KEY[8]);
5293 tmp6 = _mm_aesenc_si128(tmp6, KEY[8]);
5294 tmp7 = _mm_aesenc_si128(tmp7, KEY[8]);
5295 tmp8 = _mm_aesenc_si128(tmp8, KEY[8]);
5296 /* Reduction */
5297 X = ghash_red(r0, r1);
5298 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
5299 tmp2 = _mm_aesenc_si128(tmp2, KEY[9]);
5300 tmp3 = _mm_aesenc_si128(tmp3, KEY[9]);
5301 tmp4 = _mm_aesenc_si128(tmp4, KEY[9]);
5302 tmp5 = _mm_aesenc_si128(tmp5, KEY[9]);
5303 tmp6 = _mm_aesenc_si128(tmp6, KEY[9]);
5304 tmp7 = _mm_aesenc_si128(tmp7, KEY[9]);
5305 tmp8 = _mm_aesenc_si128(tmp8, KEY[9]);
5306 lastKey = KEY[10];
5307 if (nr > 10) {
5308 tmp1 = _mm_aesenc_si128(tmp1, KEY[10]);
5309 tmp2 = _mm_aesenc_si128(tmp2, KEY[10]);
5310 tmp3 = _mm_aesenc_si128(tmp3, KEY[10]);
5311 tmp4 = _mm_aesenc_si128(tmp4, KEY[10]);
5312 tmp5 = _mm_aesenc_si128(tmp5, KEY[10]);
5313 tmp6 = _mm_aesenc_si128(tmp6, KEY[10]);
5314 tmp7 = _mm_aesenc_si128(tmp7, KEY[10]);
5315 tmp8 = _mm_aesenc_si128(tmp8, KEY[10]);
5316 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
5317 tmp2 = _mm_aesenc_si128(tmp2, KEY[11]);
5318 tmp3 = _mm_aesenc_si128(tmp3, KEY[11]);
5319 tmp4 = _mm_aesenc_si128(tmp4, KEY[11]);
5320 tmp5 = _mm_aesenc_si128(tmp5, KEY[11]);
5321 tmp6 = _mm_aesenc_si128(tmp6, KEY[11]);
5322 tmp7 = _mm_aesenc_si128(tmp7, KEY[11]);
5323 tmp8 = _mm_aesenc_si128(tmp8, KEY[11]);
5324 lastKey = KEY[12];
5325 if (nr > 12) {
5326 tmp1 = _mm_aesenc_si128(tmp1, KEY[12]);
5327 tmp2 = _mm_aesenc_si128(tmp2, KEY[12]);
5328 tmp3 = _mm_aesenc_si128(tmp3, KEY[12]);
5329 tmp4 = _mm_aesenc_si128(tmp4, KEY[12]);
5330 tmp5 = _mm_aesenc_si128(tmp5, KEY[12]);
5331 tmp6 = _mm_aesenc_si128(tmp6, KEY[12]);
5332 tmp7 = _mm_aesenc_si128(tmp7, KEY[12]);
5333 tmp8 = _mm_aesenc_si128(tmp8, KEY[12]);
5334 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
5335 tmp2 = _mm_aesenc_si128(tmp2, KEY[13]);
5336 tmp3 = _mm_aesenc_si128(tmp3, KEY[13]);
5337 tmp4 = _mm_aesenc_si128(tmp4, KEY[13]);
5338 tmp5 = _mm_aesenc_si128(tmp5, KEY[13]);
5339 tmp6 = _mm_aesenc_si128(tmp6, KEY[13]);
5340 tmp7 = _mm_aesenc_si128(tmp7, KEY[13]);
5341 tmp8 = _mm_aesenc_si128(tmp8, KEY[13]);
5342 lastKey = KEY[14];
5343 }
5344 }
5345 AES_ENC_LAST_8();
5346 }
5347
5348 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5349 tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_MASK);
5350 tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_MASK);
5351 tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_MASK);
5352 tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_MASK);
5353 tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_MASK);
5354 tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_MASK);
5355 tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_MASK);
5356 tmp1 = _mm_xor_si128(X, tmp1);
5357 X = gfmul8(tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8,
5358 HT[0], HT[1], HT[2], HT[3], HT[4], HT[5], HT[6], HT[7]);
5359 }
5360 for (k = i*8; k < (int)(nbytes/16); k++) {
5361 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5362 ctr1 = _mm_add_epi32(ctr1, ONE);
5363 tmp1 = _mm_xor_si128(tmp1, KEY[0]);
5364 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
5365 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
5366 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
5367 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
5368 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
5369 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
5370 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
5371 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
5372 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
5373 lastKey = KEY[10];
5374 if (nr > 10) {
5375 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5376 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
5377 lastKey = KEY[12];
5378 if (nr > 12) {
5379 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5380 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
5381 lastKey = KEY[14];
5382 }
5383 }
5384 tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
5385 tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[k]));
5386 _mm_storeu_si128(&((__m128i*)out)[k], tmp1);
5387 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5388 X =_mm_xor_si128(X, tmp1);
5389 X = gfmul_shifted(X, H);
5390 }
5391 #else /* AES_GCM_AESNI_NO_UNROLL */
5392 for (k = 0; k < (int)(nbytes/16) && k < 1; k++) {
5393 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5394 ctr1 = _mm_add_epi32(ctr1, ONE);
5395 tmp1 = _mm_xor_si128(tmp1, KEY[0]);
5396 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
5397 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
5398 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
5399 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
5400 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
5401 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
5402 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
5403 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
5404 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
5405 lastKey = KEY[10];
5406 if (nr > 10) {
5407 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5408 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
5409 lastKey = KEY[12];
5410 if (nr > 12) {
5411 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5412 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
5413 lastKey = KEY[14];
5414 }
5415 }
5416 tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
5417 tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[k]));
5418 _mm_storeu_si128(&((__m128i*)out)[k], tmp1);
5419 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5420 X =_mm_xor_si128(X, tmp1);
5421 }
5422 for (; k < (int)(nbytes/16); k++) {
5423 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5424 ctr1 = _mm_add_epi32(ctr1, ONE);
5425 tmp1 = _mm_xor_si128(tmp1, KEY[0]);
5426 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
5427 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
5428 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
5429 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
5430 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
5431 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
5432 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
5433 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
5434 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
5435 X = gfmul_shifted(X, H);
5436 lastKey = KEY[10];
5437 if (nr > 10) {
5438 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5439 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
5440 lastKey = KEY[12];
5441 if (nr > 12) {
5442 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5443 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
5444 lastKey = KEY[14];
5445 }
5446 }
5447 tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
5448 tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[k]));
5449 _mm_storeu_si128(&((__m128i*)out)[k], tmp1);
5450 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5451 X =_mm_xor_si128(X, tmp1);
5452 }
5453 if (k > 0) {
5454 X = gfmul_shifted(X, H);
5455 }
5456 #endif /* AES_GCM_AESNI_NO_UNROLL */
5457
5458 /* If one partial block remains */
5459 if (nbytes % 16) {
5460 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5461 tmp1 = _mm_xor_si128(tmp1, KEY[0]);
5462 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
5463 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
5464 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
5465 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
5466 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
5467 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
5468 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
5469 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
5470 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
5471 lastKey = KEY[10];
5472 if (nr > 10) {
5473 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5474 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
5475 lastKey = KEY[12];
5476 if (nr > 12) {
5477 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5478 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
5479 lastKey = KEY[14];
5480 }
5481 }
5482 tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
5483 last_block = tmp1;
5484 for (j=0; j < (int)(nbytes%16); j++)
5485 ((unsigned char*)&last_block)[j] = in[k*16+j];
5486 tmp1 = _mm_xor_si128(tmp1, last_block);
5487 last_block = tmp1;
5488 for (j=0; j < (int)(nbytes%16); j++)
5489 out[k*16+j] = ((unsigned char*)&last_block)[j];
5490 tmp1 = last_block;
5491 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5492 X =_mm_xor_si128(X, tmp1);
5493 X = gfmul_shifted(X, H);
5494 }
5495 tmp1 = _mm_insert_epi64(tmp1, ((word64)nbytes)*8, 0);
5496 tmp1 = _mm_insert_epi64(tmp1, ((word64)abytes)*8, 1);
5497 X = _mm_xor_si128(X, tmp1);
5498 X = gfmul_shifted(X, H);
5499 X = _mm_shuffle_epi8(X, BSWAP_MASK);
5500 T = _mm_xor_si128(X, T);
5501 /*_mm_storeu_si128((__m128i*)tag, T);*/
5502 XMEMCPY(tag, &T, tbytes);
5503 ForceZero(&lastKey, sizeof(lastKey));
5504 }
5505
5506 #ifdef HAVE_AES_DECRYPT
5507
AES_GCM_decrypt(const unsigned char * in,unsigned char * out,const unsigned char * addt,const unsigned char * ivec,const unsigned char * tag,word32 nbytes,word32 abytes,word32 ibytes,word32 tbytes,const unsigned char * key,int nr,int * res)5508 static void AES_GCM_decrypt(const unsigned char *in, unsigned char *out,
5509 const unsigned char* addt,
5510 const unsigned char* ivec, const unsigned char *tag,
5511 word32 nbytes, word32 abytes, word32 ibytes,
5512 word32 tbytes, const unsigned char* key, int nr,
5513 int* res)
5514 {
5515 int i, j ,k;
5516 __m128i H, Y, T;
5517 __m128i *KEY = (__m128i*)key, lastKey;
5518 __m128i ctr1;
5519 __m128i last_block = _mm_setzero_si128();
5520 __m128i X = _mm_setzero_si128();
5521 __m128i tmp1, tmp2, XV;
5522 #ifndef AES_GCM_AESNI_NO_UNROLL
5523 __m128i HT[8];
5524 __m128i r0, r1;
5525 __m128i tmp3, tmp4, tmp5, tmp6, tmp7, tmp8;
5526 #endif /* AES_GCM_AESNI_NO_UNROLL */
5527
5528 if (ibytes == GCM_NONCE_MID_SZ)
5529 aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T);
5530 else
5531 aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T);
5532
5533 for (i=0; i<(int)(abytes/16); i++) {
5534 tmp1 = _mm_loadu_si128(&((__m128i*)addt)[i]);
5535 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5536 X = _mm_xor_si128(X, tmp1);
5537 X = gfmul_sw(X, H);
5538 }
5539 if (abytes%16) {
5540 last_block = _mm_setzero_si128();
5541 for (j=0; j<(int)(abytes%16); j++)
5542 ((unsigned char*)&last_block)[j] = addt[i*16+j];
5543 tmp1 = last_block;
5544 tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
5545 X = _mm_xor_si128(X, tmp1);
5546 X = gfmul_sw(X, H);
5547 }
5548
5549 tmp1 = _mm_shuffle_epi8(Y, BSWAP_EPI64);
5550 ctr1 = _mm_add_epi32(tmp1, ONE);
5551 H = gfmul_shl1(H);
5552 i = 0;
5553
5554 #ifndef AES_GCM_AESNI_NO_UNROLL
5555
5556 if (0 < nbytes/16/8) {
5557 HT[0] = H;
5558 HT[1] = gfmul_shifted(H, H);
5559 HT[2] = gfmul_shifted(H, HT[1]);
5560 HT[3] = gfmul_shifted(HT[1], HT[1]);
5561 HT[4] = gfmul_shifted(HT[1], HT[2]);
5562 HT[5] = gfmul_shifted(HT[2], HT[2]);
5563 HT[6] = gfmul_shifted(HT[2], HT[3]);
5564 HT[7] = gfmul_shifted(HT[3], HT[3]);
5565
5566 for (; i < (int)(nbytes/16/8); i++) {
5567 r0 = _mm_setzero_si128();
5568 r1 = _mm_setzero_si128();
5569
5570 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5571 tmp2 = _mm_add_epi32(ctr1, ONE);
5572 tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_EPI64);
5573 tmp3 = _mm_add_epi32(ctr1, TWO);
5574 tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_EPI64);
5575 tmp4 = _mm_add_epi32(ctr1, THREE);
5576 tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_EPI64);
5577 tmp5 = _mm_add_epi32(ctr1, FOUR);
5578 tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_EPI64);
5579 tmp6 = _mm_add_epi32(ctr1, FIVE);
5580 tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_EPI64);
5581 tmp7 = _mm_add_epi32(ctr1, SIX);
5582 tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_EPI64);
5583 tmp8 = _mm_add_epi32(ctr1, SEVEN);
5584 tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_EPI64);
5585 ctr1 = _mm_add_epi32(ctr1, EIGHT);
5586 tmp1 =_mm_xor_si128(tmp1, KEY[0]);
5587 tmp2 =_mm_xor_si128(tmp2, KEY[0]);
5588 tmp3 =_mm_xor_si128(tmp3, KEY[0]);
5589 tmp4 =_mm_xor_si128(tmp4, KEY[0]);
5590 tmp5 =_mm_xor_si128(tmp5, KEY[0]);
5591 tmp6 =_mm_xor_si128(tmp6, KEY[0]);
5592 tmp7 =_mm_xor_si128(tmp7, KEY[0]);
5593 tmp8 =_mm_xor_si128(tmp8, KEY[0]);
5594 /* 128 x 128 Carryless Multiply */
5595 XV = _mm_loadu_si128(&((__m128i*)in)[i*8+0]);
5596 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5597 XV = _mm_xor_si128(XV, X);
5598 gfmul_only(XV, HT[7], &r0, &r1);
5599 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
5600 tmp2 = _mm_aesenc_si128(tmp2, KEY[1]);
5601 tmp3 = _mm_aesenc_si128(tmp3, KEY[1]);
5602 tmp4 = _mm_aesenc_si128(tmp4, KEY[1]);
5603 tmp5 = _mm_aesenc_si128(tmp5, KEY[1]);
5604 tmp6 = _mm_aesenc_si128(tmp6, KEY[1]);
5605 tmp7 = _mm_aesenc_si128(tmp7, KEY[1]);
5606 tmp8 = _mm_aesenc_si128(tmp8, KEY[1]);
5607 /* 128 x 128 Carryless Multiply */
5608 XV = _mm_loadu_si128(&((__m128i*)in)[i*8+1]);
5609 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5610 gfmul_only(XV, HT[6], &r0, &r1);
5611 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
5612 tmp2 = _mm_aesenc_si128(tmp2, KEY[2]);
5613 tmp3 = _mm_aesenc_si128(tmp3, KEY[2]);
5614 tmp4 = _mm_aesenc_si128(tmp4, KEY[2]);
5615 tmp5 = _mm_aesenc_si128(tmp5, KEY[2]);
5616 tmp6 = _mm_aesenc_si128(tmp6, KEY[2]);
5617 tmp7 = _mm_aesenc_si128(tmp7, KEY[2]);
5618 tmp8 = _mm_aesenc_si128(tmp8, KEY[2]);
5619 /* 128 x 128 Carryless Multiply */
5620 XV = _mm_loadu_si128(&((__m128i*)in)[i*8+2]);
5621 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5622 gfmul_only(XV, HT[5], &r0, &r1);
5623 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
5624 tmp2 = _mm_aesenc_si128(tmp2, KEY[3]);
5625 tmp3 = _mm_aesenc_si128(tmp3, KEY[3]);
5626 tmp4 = _mm_aesenc_si128(tmp4, KEY[3]);
5627 tmp5 = _mm_aesenc_si128(tmp5, KEY[3]);
5628 tmp6 = _mm_aesenc_si128(tmp6, KEY[3]);
5629 tmp7 = _mm_aesenc_si128(tmp7, KEY[3]);
5630 tmp8 = _mm_aesenc_si128(tmp8, KEY[3]);
5631 /* 128 x 128 Carryless Multiply */
5632 XV = _mm_loadu_si128(&((__m128i*)in)[i*8+3]);
5633 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5634 gfmul_only(XV, HT[4], &r0, &r1);
5635 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
5636 tmp2 = _mm_aesenc_si128(tmp2, KEY[4]);
5637 tmp3 = _mm_aesenc_si128(tmp3, KEY[4]);
5638 tmp4 = _mm_aesenc_si128(tmp4, KEY[4]);
5639 tmp5 = _mm_aesenc_si128(tmp5, KEY[4]);
5640 tmp6 = _mm_aesenc_si128(tmp6, KEY[4]);
5641 tmp7 = _mm_aesenc_si128(tmp7, KEY[4]);
5642 tmp8 = _mm_aesenc_si128(tmp8, KEY[4]);
5643 /* 128 x 128 Carryless Multiply */
5644 XV = _mm_loadu_si128(&((__m128i*)in)[i*8+4]);
5645 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5646 gfmul_only(XV, HT[3], &r0, &r1);
5647 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
5648 tmp2 = _mm_aesenc_si128(tmp2, KEY[5]);
5649 tmp3 = _mm_aesenc_si128(tmp3, KEY[5]);
5650 tmp4 = _mm_aesenc_si128(tmp4, KEY[5]);
5651 tmp5 = _mm_aesenc_si128(tmp5, KEY[5]);
5652 tmp6 = _mm_aesenc_si128(tmp6, KEY[5]);
5653 tmp7 = _mm_aesenc_si128(tmp7, KEY[5]);
5654 tmp8 = _mm_aesenc_si128(tmp8, KEY[5]);
5655 /* 128 x 128 Carryless Multiply */
5656 XV = _mm_loadu_si128(&((__m128i*)in)[i*8+5]);
5657 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5658 gfmul_only(XV, HT[2], &r0, &r1);
5659 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
5660 tmp2 = _mm_aesenc_si128(tmp2, KEY[6]);
5661 tmp3 = _mm_aesenc_si128(tmp3, KEY[6]);
5662 tmp4 = _mm_aesenc_si128(tmp4, KEY[6]);
5663 tmp5 = _mm_aesenc_si128(tmp5, KEY[6]);
5664 tmp6 = _mm_aesenc_si128(tmp6, KEY[6]);
5665 tmp7 = _mm_aesenc_si128(tmp7, KEY[6]);
5666 tmp8 = _mm_aesenc_si128(tmp8, KEY[6]);
5667 /* 128 x 128 Carryless Multiply */
5668 XV = _mm_loadu_si128(&((__m128i*)in)[i*8+6]);
5669 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5670 gfmul_only(XV, HT[1], &r0, &r1);
5671 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
5672 tmp2 = _mm_aesenc_si128(tmp2, KEY[7]);
5673 tmp3 = _mm_aesenc_si128(tmp3, KEY[7]);
5674 tmp4 = _mm_aesenc_si128(tmp4, KEY[7]);
5675 tmp5 = _mm_aesenc_si128(tmp5, KEY[7]);
5676 tmp6 = _mm_aesenc_si128(tmp6, KEY[7]);
5677 tmp7 = _mm_aesenc_si128(tmp7, KEY[7]);
5678 tmp8 = _mm_aesenc_si128(tmp8, KEY[7]);
5679 /* 128 x 128 Carryless Multiply */
5680 XV = _mm_loadu_si128(&((__m128i*)in)[i*8+7]);
5681 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5682 gfmul_only(XV, HT[0], &r0, &r1);
5683 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
5684 tmp2 = _mm_aesenc_si128(tmp2, KEY[8]);
5685 tmp3 = _mm_aesenc_si128(tmp3, KEY[8]);
5686 tmp4 = _mm_aesenc_si128(tmp4, KEY[8]);
5687 tmp5 = _mm_aesenc_si128(tmp5, KEY[8]);
5688 tmp6 = _mm_aesenc_si128(tmp6, KEY[8]);
5689 tmp7 = _mm_aesenc_si128(tmp7, KEY[8]);
5690 tmp8 = _mm_aesenc_si128(tmp8, KEY[8]);
5691 /* Reduction */
5692 X = ghash_red(r0, r1);
5693 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
5694 tmp2 = _mm_aesenc_si128(tmp2, KEY[9]);
5695 tmp3 = _mm_aesenc_si128(tmp3, KEY[9]);
5696 tmp4 = _mm_aesenc_si128(tmp4, KEY[9]);
5697 tmp5 = _mm_aesenc_si128(tmp5, KEY[9]);
5698 tmp6 = _mm_aesenc_si128(tmp6, KEY[9]);
5699 tmp7 = _mm_aesenc_si128(tmp7, KEY[9]);
5700 tmp8 = _mm_aesenc_si128(tmp8, KEY[9]);
5701 lastKey = KEY[10];
5702 if (nr > 10) {
5703 tmp1 = _mm_aesenc_si128(tmp1, KEY[10]);
5704 tmp2 = _mm_aesenc_si128(tmp2, KEY[10]);
5705 tmp3 = _mm_aesenc_si128(tmp3, KEY[10]);
5706 tmp4 = _mm_aesenc_si128(tmp4, KEY[10]);
5707 tmp5 = _mm_aesenc_si128(tmp5, KEY[10]);
5708 tmp6 = _mm_aesenc_si128(tmp6, KEY[10]);
5709 tmp7 = _mm_aesenc_si128(tmp7, KEY[10]);
5710 tmp8 = _mm_aesenc_si128(tmp8, KEY[10]);
5711 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
5712 tmp2 = _mm_aesenc_si128(tmp2, KEY[11]);
5713 tmp3 = _mm_aesenc_si128(tmp3, KEY[11]);
5714 tmp4 = _mm_aesenc_si128(tmp4, KEY[11]);
5715 tmp5 = _mm_aesenc_si128(tmp5, KEY[11]);
5716 tmp6 = _mm_aesenc_si128(tmp6, KEY[11]);
5717 tmp7 = _mm_aesenc_si128(tmp7, KEY[11]);
5718 tmp8 = _mm_aesenc_si128(tmp8, KEY[11]);
5719 lastKey = KEY[12];
5720 if (nr > 12) {
5721 tmp1 = _mm_aesenc_si128(tmp1, KEY[12]);
5722 tmp2 = _mm_aesenc_si128(tmp2, KEY[12]);
5723 tmp3 = _mm_aesenc_si128(tmp3, KEY[12]);
5724 tmp4 = _mm_aesenc_si128(tmp4, KEY[12]);
5725 tmp5 = _mm_aesenc_si128(tmp5, KEY[12]);
5726 tmp6 = _mm_aesenc_si128(tmp6, KEY[12]);
5727 tmp7 = _mm_aesenc_si128(tmp7, KEY[12]);
5728 tmp8 = _mm_aesenc_si128(tmp8, KEY[12]);
5729 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
5730 tmp2 = _mm_aesenc_si128(tmp2, KEY[13]);
5731 tmp3 = _mm_aesenc_si128(tmp3, KEY[13]);
5732 tmp4 = _mm_aesenc_si128(tmp4, KEY[13]);
5733 tmp5 = _mm_aesenc_si128(tmp5, KEY[13]);
5734 tmp6 = _mm_aesenc_si128(tmp6, KEY[13]);
5735 tmp7 = _mm_aesenc_si128(tmp7, KEY[13]);
5736 tmp8 = _mm_aesenc_si128(tmp8, KEY[13]);
5737 lastKey = KEY[14];
5738 }
5739 }
5740 AES_ENC_LAST_8();
5741 }
5742 }
5743
5744 #endif /* AES_GCM_AESNI_NO_UNROLL */
5745
5746 for (k = i*8; k < (int)(nbytes/16); k++) {
5747 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5748 ctr1 = _mm_add_epi32(ctr1, ONE);
5749 tmp1 = _mm_xor_si128(tmp1, KEY[0]);
5750 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
5751 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
5752 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
5753 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
5754 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
5755 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
5756 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
5757 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
5758 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
5759 /* 128 x 128 Carryless Multiply */
5760 XV = _mm_loadu_si128(&((__m128i*)in)[k]);
5761 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5762 XV = _mm_xor_si128(XV, X);
5763 X = gfmul_shifted(XV, H);
5764 lastKey = KEY[10];
5765 if (nr > 10) {
5766 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5767 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
5768 lastKey = KEY[12];
5769 if (nr > 12) {
5770 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5771 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
5772 lastKey = KEY[14];
5773 }
5774 }
5775 tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
5776 tmp2 = _mm_loadu_si128(&((__m128i*)in)[k]);
5777 tmp1 = _mm_xor_si128(tmp1, tmp2);
5778 _mm_storeu_si128(&((__m128i*)out)[k], tmp1);
5779 }
5780
5781 /* If one partial block remains */
5782 if (nbytes % 16) {
5783 tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
5784 tmp1 = _mm_xor_si128(tmp1, KEY[0]);
5785 tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
5786 tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
5787 tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
5788 tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
5789 tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
5790 tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
5791 tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
5792 tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
5793 tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
5794 lastKey = KEY[10];
5795 if (nr > 10) {
5796 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5797 tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
5798 lastKey = KEY[12];
5799 if (nr > 12) {
5800 tmp1 = _mm_aesenc_si128(tmp1, lastKey);
5801 tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
5802 lastKey = KEY[14];
5803 }
5804 }
5805 tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
5806 last_block = _mm_setzero_si128();
5807 for (j=0; j < (int)(nbytes%16); j++)
5808 ((unsigned char*)&last_block)[j] = in[k*16+j];
5809 XV = last_block;
5810 tmp1 = _mm_xor_si128(tmp1, last_block);
5811 last_block = tmp1;
5812 for (j=0; j < (int)(nbytes%16); j++)
5813 out[k*16+j] = ((unsigned char*)&last_block)[j];
5814 XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
5815 XV = _mm_xor_si128(XV, X);
5816 X = gfmul_shifted(XV, H);
5817 }
5818
5819 tmp1 = _mm_insert_epi64(tmp1, ((word64)nbytes)*8, 0);
5820 tmp1 = _mm_insert_epi64(tmp1, ((word64)abytes)*8, 1);
5821 /* 128 x 128 Carryless Multiply */
5822 X = _mm_xor_si128(X, tmp1);
5823 X = gfmul_shifted(X, H);
5824 X = _mm_shuffle_epi8(X, BSWAP_MASK);
5825 T = _mm_xor_si128(X, T);
5826
5827 /* if (0xffff !=
5828 _mm_movemask_epi8(_mm_cmpeq_epi8(T, _mm_loadu_si128((__m128i*)tag)))) */
5829 if (XMEMCMP(tag, &T, tbytes) != 0)
5830 *res = 0; /* in case the authentication failed */
5831 else
5832 *res = 1; /* when successful returns 1 */
5833 ForceZero(&lastKey, sizeof(lastKey));
5834 }
5835
5836 #endif /* HAVE_AES_DECRYPT */
5837 #endif /* _MSC_VER */
5838 #endif /* WOLFSSL_AESNI */
5839
5840 #if defined(GCM_SMALL)
GMULT(byte * X,byte * Y)5841 static void GMULT(byte* X, byte* Y)
5842 {
5843 byte Z[AES_BLOCK_SIZE];
5844 byte V[AES_BLOCK_SIZE];
5845 int i, j;
5846
5847 XMEMSET(Z, 0, AES_BLOCK_SIZE);
5848 XMEMCPY(V, X, AES_BLOCK_SIZE);
5849 for (i = 0; i < AES_BLOCK_SIZE; i++)
5850 {
5851 byte y = Y[i];
5852 for (j = 0; j < 8; j++)
5853 {
5854 if (y & 0x80) {
5855 xorbuf(Z, V, AES_BLOCK_SIZE);
5856 }
5857
5858 RIGHTSHIFTX(V);
5859 y = y << 1;
5860 }
5861 }
5862 XMEMCPY(X, Z, AES_BLOCK_SIZE);
5863 }
5864
5865
GHASH(Aes * aes,const byte * a,word32 aSz,const byte * c,word32 cSz,byte * s,word32 sSz)5866 void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
5867 word32 cSz, byte* s, word32 sSz)
5868 {
5869 byte x[AES_BLOCK_SIZE];
5870 byte scratch[AES_BLOCK_SIZE];
5871 word32 blocks, partial;
5872 byte* h;
5873
5874 if (aes == NULL) {
5875 return;
5876 }
5877
5878 h = aes->H;
5879 XMEMSET(x, 0, AES_BLOCK_SIZE);
5880
5881 /* Hash in A, the Additional Authentication Data */
5882 if (aSz != 0 && a != NULL) {
5883 blocks = aSz / AES_BLOCK_SIZE;
5884 partial = aSz % AES_BLOCK_SIZE;
5885 while (blocks--) {
5886 xorbuf(x, a, AES_BLOCK_SIZE);
5887 GMULT(x, h);
5888 a += AES_BLOCK_SIZE;
5889 }
5890 if (partial != 0) {
5891 XMEMSET(scratch, 0, AES_BLOCK_SIZE);
5892 XMEMCPY(scratch, a, partial);
5893 xorbuf(x, scratch, AES_BLOCK_SIZE);
5894 GMULT(x, h);
5895 }
5896 }
5897
5898 /* Hash in C, the Ciphertext */
5899 if (cSz != 0 && c != NULL) {
5900 blocks = cSz / AES_BLOCK_SIZE;
5901 partial = cSz % AES_BLOCK_SIZE;
5902 while (blocks--) {
5903 xorbuf(x, c, AES_BLOCK_SIZE);
5904 GMULT(x, h);
5905 c += AES_BLOCK_SIZE;
5906 }
5907 if (partial != 0) {
5908 XMEMSET(scratch, 0, AES_BLOCK_SIZE);
5909 XMEMCPY(scratch, c, partial);
5910 xorbuf(x, scratch, AES_BLOCK_SIZE);
5911 GMULT(x, h);
5912 }
5913 }
5914
5915 /* Hash in the lengths of A and C in bits */
5916 FlattenSzInBits(&scratch[0], aSz);
5917 FlattenSzInBits(&scratch[8], cSz);
5918 xorbuf(x, scratch, AES_BLOCK_SIZE);
5919 GMULT(x, h);
5920
5921 /* Copy the result into s. */
5922 XMEMCPY(s, x, sSz);
5923 }
5924
5925 #ifdef WOLFSSL_AESGCM_STREAM
5926 /* No extra initialization for small implementation.
5927 *
5928 * @param [in] aes AES GCM object.
5929 */
5930 #define GHASH_INIT_EXTRA(aes)
5931
5932 /* GHASH one block of data..
5933 *
5934 * XOR block into tag and GMULT with H.
5935 *
5936 * @param [in, out] aes AES GCM object.
5937 * @param [in] block Block of AAD or cipher text.
5938 */
5939 #define GHASH_ONE_BLOCK(aes, block) \
5940 do { \
5941 xorbuf(AES_TAG(aes), block, AES_BLOCK_SIZE); \
5942 GMULT(AES_TAG(aes), aes->H); \
5943 } \
5944 while (0)
5945 #endif /* WOLFSSL_AESGCM_STREAM */
5946 /* end GCM_SMALL */
5947 #elif defined(GCM_TABLE)
5948
5949 static const byte R[256][2] = {
5950 {0x00, 0x00}, {0x01, 0xc2}, {0x03, 0x84}, {0x02, 0x46},
5951 {0x07, 0x08}, {0x06, 0xca}, {0x04, 0x8c}, {0x05, 0x4e},
5952 {0x0e, 0x10}, {0x0f, 0xd2}, {0x0d, 0x94}, {0x0c, 0x56},
5953 {0x09, 0x18}, {0x08, 0xda}, {0x0a, 0x9c}, {0x0b, 0x5e},
5954 {0x1c, 0x20}, {0x1d, 0xe2}, {0x1f, 0xa4}, {0x1e, 0x66},
5955 {0x1b, 0x28}, {0x1a, 0xea}, {0x18, 0xac}, {0x19, 0x6e},
5956 {0x12, 0x30}, {0x13, 0xf2}, {0x11, 0xb4}, {0x10, 0x76},
5957 {0x15, 0x38}, {0x14, 0xfa}, {0x16, 0xbc}, {0x17, 0x7e},
5958 {0x38, 0x40}, {0x39, 0x82}, {0x3b, 0xc4}, {0x3a, 0x06},
5959 {0x3f, 0x48}, {0x3e, 0x8a}, {0x3c, 0xcc}, {0x3d, 0x0e},
5960 {0x36, 0x50}, {0x37, 0x92}, {0x35, 0xd4}, {0x34, 0x16},
5961 {0x31, 0x58}, {0x30, 0x9a}, {0x32, 0xdc}, {0x33, 0x1e},
5962 {0x24, 0x60}, {0x25, 0xa2}, {0x27, 0xe4}, {0x26, 0x26},
5963 {0x23, 0x68}, {0x22, 0xaa}, {0x20, 0xec}, {0x21, 0x2e},
5964 {0x2a, 0x70}, {0x2b, 0xb2}, {0x29, 0xf4}, {0x28, 0x36},
5965 {0x2d, 0x78}, {0x2c, 0xba}, {0x2e, 0xfc}, {0x2f, 0x3e},
5966 {0x70, 0x80}, {0x71, 0x42}, {0x73, 0x04}, {0x72, 0xc6},
5967 {0x77, 0x88}, {0x76, 0x4a}, {0x74, 0x0c}, {0x75, 0xce},
5968 {0x7e, 0x90}, {0x7f, 0x52}, {0x7d, 0x14}, {0x7c, 0xd6},
5969 {0x79, 0x98}, {0x78, 0x5a}, {0x7a, 0x1c}, {0x7b, 0xde},
5970 {0x6c, 0xa0}, {0x6d, 0x62}, {0x6f, 0x24}, {0x6e, 0xe6},
5971 {0x6b, 0xa8}, {0x6a, 0x6a}, {0x68, 0x2c}, {0x69, 0xee},
5972 {0x62, 0xb0}, {0x63, 0x72}, {0x61, 0x34}, {0x60, 0xf6},
5973 {0x65, 0xb8}, {0x64, 0x7a}, {0x66, 0x3c}, {0x67, 0xfe},
5974 {0x48, 0xc0}, {0x49, 0x02}, {0x4b, 0x44}, {0x4a, 0x86},
5975 {0x4f, 0xc8}, {0x4e, 0x0a}, {0x4c, 0x4c}, {0x4d, 0x8e},
5976 {0x46, 0xd0}, {0x47, 0x12}, {0x45, 0x54}, {0x44, 0x96},
5977 {0x41, 0xd8}, {0x40, 0x1a}, {0x42, 0x5c}, {0x43, 0x9e},
5978 {0x54, 0xe0}, {0x55, 0x22}, {0x57, 0x64}, {0x56, 0xa6},
5979 {0x53, 0xe8}, {0x52, 0x2a}, {0x50, 0x6c}, {0x51, 0xae},
5980 {0x5a, 0xf0}, {0x5b, 0x32}, {0x59, 0x74}, {0x58, 0xb6},
5981 {0x5d, 0xf8}, {0x5c, 0x3a}, {0x5e, 0x7c}, {0x5f, 0xbe},
5982 {0xe1, 0x00}, {0xe0, 0xc2}, {0xe2, 0x84}, {0xe3, 0x46},
5983 {0xe6, 0x08}, {0xe7, 0xca}, {0xe5, 0x8c}, {0xe4, 0x4e},
5984 {0xef, 0x10}, {0xee, 0xd2}, {0xec, 0x94}, {0xed, 0x56},
5985 {0xe8, 0x18}, {0xe9, 0xda}, {0xeb, 0x9c}, {0xea, 0x5e},
5986 {0xfd, 0x20}, {0xfc, 0xe2}, {0xfe, 0xa4}, {0xff, 0x66},
5987 {0xfa, 0x28}, {0xfb, 0xea}, {0xf9, 0xac}, {0xf8, 0x6e},
5988 {0xf3, 0x30}, {0xf2, 0xf2}, {0xf0, 0xb4}, {0xf1, 0x76},
5989 {0xf4, 0x38}, {0xf5, 0xfa}, {0xf7, 0xbc}, {0xf6, 0x7e},
5990 {0xd9, 0x40}, {0xd8, 0x82}, {0xda, 0xc4}, {0xdb, 0x06},
5991 {0xde, 0x48}, {0xdf, 0x8a}, {0xdd, 0xcc}, {0xdc, 0x0e},
5992 {0xd7, 0x50}, {0xd6, 0x92}, {0xd4, 0xd4}, {0xd5, 0x16},
5993 {0xd0, 0x58}, {0xd1, 0x9a}, {0xd3, 0xdc}, {0xd2, 0x1e},
5994 {0xc5, 0x60}, {0xc4, 0xa2}, {0xc6, 0xe4}, {0xc7, 0x26},
5995 {0xc2, 0x68}, {0xc3, 0xaa}, {0xc1, 0xec}, {0xc0, 0x2e},
5996 {0xcb, 0x70}, {0xca, 0xb2}, {0xc8, 0xf4}, {0xc9, 0x36},
5997 {0xcc, 0x78}, {0xcd, 0xba}, {0xcf, 0xfc}, {0xce, 0x3e},
5998 {0x91, 0x80}, {0x90, 0x42}, {0x92, 0x04}, {0x93, 0xc6},
5999 {0x96, 0x88}, {0x97, 0x4a}, {0x95, 0x0c}, {0x94, 0xce},
6000 {0x9f, 0x90}, {0x9e, 0x52}, {0x9c, 0x14}, {0x9d, 0xd6},
6001 {0x98, 0x98}, {0x99, 0x5a}, {0x9b, 0x1c}, {0x9a, 0xde},
6002 {0x8d, 0xa0}, {0x8c, 0x62}, {0x8e, 0x24}, {0x8f, 0xe6},
6003 {0x8a, 0xa8}, {0x8b, 0x6a}, {0x89, 0x2c}, {0x88, 0xee},
6004 {0x83, 0xb0}, {0x82, 0x72}, {0x80, 0x34}, {0x81, 0xf6},
6005 {0x84, 0xb8}, {0x85, 0x7a}, {0x87, 0x3c}, {0x86, 0xfe},
6006 {0xa9, 0xc0}, {0xa8, 0x02}, {0xaa, 0x44}, {0xab, 0x86},
6007 {0xae, 0xc8}, {0xaf, 0x0a}, {0xad, 0x4c}, {0xac, 0x8e},
6008 {0xa7, 0xd0}, {0xa6, 0x12}, {0xa4, 0x54}, {0xa5, 0x96},
6009 {0xa0, 0xd8}, {0xa1, 0x1a}, {0xa3, 0x5c}, {0xa2, 0x9e},
6010 {0xb5, 0xe0}, {0xb4, 0x22}, {0xb6, 0x64}, {0xb7, 0xa6},
6011 {0xb2, 0xe8}, {0xb3, 0x2a}, {0xb1, 0x6c}, {0xb0, 0xae},
6012 {0xbb, 0xf0}, {0xba, 0x32}, {0xb8, 0x74}, {0xb9, 0xb6},
6013 {0xbc, 0xf8}, {0xbd, 0x3a}, {0xbf, 0x7c}, {0xbe, 0xbe} };
6014
6015
GMULT(byte * x,byte m[256][AES_BLOCK_SIZE])6016 static void GMULT(byte *x, byte m[256][AES_BLOCK_SIZE])
6017 {
6018 #if !defined(WORD64_AVAILABLE) || defined(BIG_ENDIAN_ORDER)
6019 int i, j;
6020 byte Z[AES_BLOCK_SIZE];
6021 byte a;
6022
6023 XMEMSET(Z, 0, sizeof(Z));
6024
6025 for (i = 15; i > 0; i--) {
6026 xorbuf(Z, m[x[i]], AES_BLOCK_SIZE);
6027 a = Z[15];
6028
6029 for (j = 15; j > 0; j--) {
6030 Z[j] = Z[j-1];
6031 }
6032
6033 Z[0] = R[a][0];
6034 Z[1] ^= R[a][1];
6035 }
6036 xorbuf(Z, m[x[0]], AES_BLOCK_SIZE);
6037
6038 XMEMCPY(x, Z, AES_BLOCK_SIZE);
6039 #else
6040 byte Z[AES_BLOCK_SIZE + AES_BLOCK_SIZE];
6041 byte a;
6042 word64* pZ;
6043 word64* pm;
6044 word64* px = (word64*)(x);
6045 int i;
6046
6047 pZ = (word64*)(Z + 15 + 1);
6048 pm = (word64*)(m[x[15]]);
6049 pZ[0] = pm[0];
6050 pZ[1] = pm[1];
6051 a = Z[16 + 15];
6052 Z[15] = R[a][0];
6053 Z[16] ^= R[a][1];
6054 for (i = 14; i > 0; i--) {
6055 pZ = (word64*)(Z + i + 1);
6056 pm = (word64*)(m[x[i]]);
6057 pZ[0] ^= pm[0];
6058 pZ[1] ^= pm[1];
6059 a = Z[16 + i];
6060 Z[i] = R[a][0];
6061 Z[i+1] ^= R[a][1];
6062 }
6063 pZ = (word64*)(Z + 1);
6064 pm = (word64*)(m[x[0]]);
6065 px[0] = pZ[0] ^ pm[0]; px[1] = pZ[1] ^ pm[1];
6066 #endif
6067 }
6068
GHASH(Aes * aes,const byte * a,word32 aSz,const byte * c,word32 cSz,byte * s,word32 sSz)6069 void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
6070 word32 cSz, byte* s, word32 sSz)
6071 {
6072 byte x[AES_BLOCK_SIZE];
6073 byte scratch[AES_BLOCK_SIZE];
6074 word32 blocks, partial;
6075
6076 if (aes == NULL) {
6077 return;
6078 }
6079
6080 XMEMSET(x, 0, AES_BLOCK_SIZE);
6081
6082 /* Hash in A, the Additional Authentication Data */
6083 if (aSz != 0 && a != NULL) {
6084 blocks = aSz / AES_BLOCK_SIZE;
6085 partial = aSz % AES_BLOCK_SIZE;
6086 while (blocks--) {
6087 xorbuf(x, a, AES_BLOCK_SIZE);
6088 GMULT(x, aes->M0);
6089 a += AES_BLOCK_SIZE;
6090 }
6091 if (partial != 0) {
6092 XMEMSET(scratch, 0, AES_BLOCK_SIZE);
6093 XMEMCPY(scratch, a, partial);
6094 xorbuf(x, scratch, AES_BLOCK_SIZE);
6095 GMULT(x, aes->M0);
6096 }
6097 }
6098
6099 /* Hash in C, the Ciphertext */
6100 if (cSz != 0 && c != NULL) {
6101 blocks = cSz / AES_BLOCK_SIZE;
6102 partial = cSz % AES_BLOCK_SIZE;
6103 while (blocks--) {
6104 xorbuf(x, c, AES_BLOCK_SIZE);
6105 GMULT(x, aes->M0);
6106 c += AES_BLOCK_SIZE;
6107 }
6108 if (partial != 0) {
6109 XMEMSET(scratch, 0, AES_BLOCK_SIZE);
6110 XMEMCPY(scratch, c, partial);
6111 xorbuf(x, scratch, AES_BLOCK_SIZE);
6112 GMULT(x, aes->M0);
6113 }
6114 }
6115
6116 /* Hash in the lengths of A and C in bits */
6117 FlattenSzInBits(&scratch[0], aSz);
6118 FlattenSzInBits(&scratch[8], cSz);
6119 xorbuf(x, scratch, AES_BLOCK_SIZE);
6120 GMULT(x, aes->M0);
6121
6122 /* Copy the result into s. */
6123 XMEMCPY(s, x, sSz);
6124 }
6125
6126 #ifdef WOLFSSL_AESGCM_STREAM
6127 /* No extra initialization for table implementation.
6128 *
6129 * @param [in] aes AES GCM object.
6130 */
6131 #define GHASH_INIT_EXTRA(aes)
6132
6133 /* GHASH one block of data..
6134 *
6135 * XOR block into tag and GMULT with H using pre-computed table.
6136 *
6137 * @param [in, out] aes AES GCM object.
6138 * @param [in] block Block of AAD or cipher text.
6139 */
6140 #define GHASH_ONE_BLOCK(aes, block) \
6141 do { \
6142 xorbuf(AES_TAG(aes), block, AES_BLOCK_SIZE); \
6143 GMULT(AES_TAG(aes), aes->M0); \
6144 } \
6145 while (0)
6146 #endif /* WOLFSSL_AESGCM_STREAM */
6147 /* end GCM_TABLE */
6148 #elif defined(GCM_TABLE_4BIT)
6149
6150 /* remainder = x^7 + x^2 + x^1 + 1 => 0xe1
6151 * R shifts right a reverse bit pair of bytes such that:
6152 * R(b0, b1) => b1 = (b1 >> 1) | (b0 << 7); b0 >>= 1
6153 * 0 => 0, 0, 0, 0 => R(R(R(00,00) ^ 00,00) ^ 00,00) ^ 00,00 = 00,00
6154 * 8 => 0, 0, 0, 1 => R(R(R(00,00) ^ 00,00) ^ 00,00) ^ e1,00 = e1,00
6155 * 4 => 0, 0, 1, 0 => R(R(R(00,00) ^ 00,00) ^ e1,00) ^ 00,00 = 70,80
6156 * 2 => 0, 1, 0, 0 => R(R(R(00,00) ^ e1,00) ^ 00,00) ^ 00,00 = 38,40
6157 * 1 => 1, 0, 0, 0 => R(R(R(e1,00) ^ 00,00) ^ 00,00) ^ 00,00 = 1c,20
6158 * To calculate te rest, XOR result for each bit.
6159 * e.g. 6 = 4 ^ 2 => 48,c0
6160 *
6161 * Second half is same values rotated by 4-bits.
6162 */
6163 #if defined(BIG_ENDIAN_ORDER) || defined(WC_16BIT_CPU)
6164 static const byte R[16][2] = {
6165 {0x00, 0x00}, {0x1c, 0x20}, {0x38, 0x40}, {0x24, 0x60},
6166 {0x70, 0x80}, {0x6c, 0xa0}, {0x48, 0xc0}, {0x54, 0xe0},
6167 {0xe1, 0x00}, {0xfd, 0x20}, {0xd9, 0x40}, {0xc5, 0x60},
6168 {0x91, 0x80}, {0x8d, 0xa0}, {0xa9, 0xc0}, {0xb5, 0xe0},
6169 };
6170 #else
6171 static const word16 R[32] = {
6172 0x0000, 0x201c, 0x4038, 0x6024,
6173 0x8070, 0xa06c, 0xc048, 0xe054,
6174 0x00e1, 0x20fd, 0x40d9, 0x60c5,
6175 0x8091, 0xa08d, 0xc0a9, 0xe0b5,
6176
6177 0x0000, 0xc201, 0x8403, 0x4602,
6178 0x0807, 0xca06, 0x8c04, 0x4e05,
6179 0x100e, 0xd20f, 0x940d, 0x560c,
6180 0x1809, 0xda08, 0x9c0a, 0x5e0b,
6181 };
6182 #endif
6183
6184 /* Multiply in GF(2^128) defined by polynomial:
6185 * x^128 + x^7 + x^2 + x^1 + 1.
6186 *
6187 * H: hash key = encrypt(key, 0)
6188 * x = x * H in field
6189 *
6190 * x: cumlative result
6191 * m: 4-bit table
6192 * [0..15] * H
6193 */
6194 #if defined(BIG_ENDIAN_ORDER) || defined(WC_16BIT_CPU)
GMULT(byte * x,byte m[16][AES_BLOCK_SIZE])6195 static void GMULT(byte *x, byte m[16][AES_BLOCK_SIZE])
6196 {
6197 int i, j, n;
6198 byte Z[AES_BLOCK_SIZE];
6199 byte a;
6200
6201 XMEMSET(Z, 0, sizeof(Z));
6202
6203 for (i = 15; i >= 0; i--) {
6204 for (n = 0; n < 2; n++) {
6205 if (n == 0)
6206 xorbuf(Z, m[x[i] & 0xf], AES_BLOCK_SIZE);
6207 else {
6208 xorbuf(Z, m[x[i] >> 4], AES_BLOCK_SIZE);
6209 if (i == 0)
6210 break;
6211 }
6212 a = Z[15] & 0xf;
6213
6214 for (j = 15; j > 0; j--)
6215 Z[j] = (Z[j-1] << 4) | (Z[j] >> 4);
6216 Z[0] >>= 4;
6217
6218 Z[0] ^= R[a][0];
6219 Z[1] ^= R[a][1];
6220 }
6221 }
6222
6223 XMEMCPY(x, Z, AES_BLOCK_SIZE);
6224 }
6225 #elif defined(WC_32BIT_CPU)
GMULT(byte * x,byte m[32][AES_BLOCK_SIZE])6226 static WC_INLINE void GMULT(byte *x, byte m[32][AES_BLOCK_SIZE])
6227 {
6228 int i;
6229 word32 z8[4] = {0, 0, 0, 0};
6230 byte a;
6231 word32* x8 = (word32*)x;
6232 word32* m8;
6233 byte xi;
6234 word32 n7, n6, n5, n4, n3, n2, n1, n0;
6235
6236 for (i = 15; i > 0; i--) {
6237 xi = x[i];
6238
6239 /* XOR in (msn * H) */
6240 m8 = (word32*)m[xi & 0xf];
6241 z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
6242
6243 /* Cache top byte for remainder calculations - lost in rotate. */
6244 a = z8[3] >> 24;
6245
6246 /* Rotate Z by 8-bits */
6247 z8[3] = (z8[2] >> 24) | (z8[3] << 8);
6248 z8[2] = (z8[1] >> 24) | (z8[2] << 8);
6249 z8[1] = (z8[0] >> 24) | (z8[1] << 8);
6250 z8[0] <<= 8;
6251
6252 /* XOR in (msn * remainder) [pre-rotated by 4 bits] */
6253 z8[0] ^= (word32)R[16 + (a & 0xf)];
6254
6255 xi >>= 4;
6256 /* XOR in next significant nibble (XORed with H) * remainder */
6257 m8 = (word32*)m[xi];
6258 a ^= (byte)(m8[3] >> 20);
6259 z8[0] ^= (word32)R[a >> 4];
6260
6261 /* XOR in (next significant nibble * H) [pre-rotated by 4 bits] */
6262 m8 = (word32*)m[16 + xi];
6263 z8[0] ^= m8[0]; z8[1] ^= m8[1];
6264 z8[2] ^= m8[2]; z8[3] ^= m8[3];
6265 }
6266
6267 xi = x[0];
6268
6269 /* XOR in most significant nibble * H */
6270 m8 = (word32*)m[xi & 0xf];
6271 z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
6272
6273 /* Cache top byte for remainder calculations - lost in rotate. */
6274 a = (z8[3] >> 24) & 0xf;
6275
6276 /* Rotate z by 4-bits */
6277 n7 = z8[3] & 0xf0f0f0f0ULL;
6278 n6 = z8[3] & 0x0f0f0f0fULL;
6279 n5 = z8[2] & 0xf0f0f0f0ULL;
6280 n4 = z8[2] & 0x0f0f0f0fULL;
6281 n3 = z8[1] & 0xf0f0f0f0ULL;
6282 n2 = z8[1] & 0x0f0f0f0fULL;
6283 n1 = z8[0] & 0xf0f0f0f0ULL;
6284 n0 = z8[0] & 0x0f0f0f0fULL;
6285 z8[3] = (n7 >> 4) | (n6 << 12) | (n4 >> 20);
6286 z8[2] = (n5 >> 4) | (n4 << 12) | (n2 >> 20);
6287 z8[1] = (n3 >> 4) | (n2 << 12) | (n0 >> 20);
6288 z8[0] = (n1 >> 4) | (n0 << 12);
6289
6290 /* XOR in most significant nibble * remainder */
6291 z8[0] ^= (word32)R[a];
6292 /* XOR in next significant nibble * H */
6293 m8 = (word32*)m[xi >> 4];
6294 z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
6295
6296 /* Write back result. */
6297 x8[0] = z8[0]; x8[1] = z8[1]; x8[2] = z8[2]; x8[3] = z8[3];
6298 }
6299 #else
GMULT(byte * x,byte m[32][AES_BLOCK_SIZE])6300 static WC_INLINE void GMULT(byte *x, byte m[32][AES_BLOCK_SIZE])
6301 {
6302 int i;
6303 word64 z8[2] = {0, 0};
6304 byte a;
6305 word64* x8 = (word64*)x;
6306 word64* m8;
6307 word64 n0, n1, n2, n3;
6308 byte xi;
6309
6310 for (i = 15; i > 0; i--) {
6311 xi = x[i];
6312
6313 /* XOR in (msn * H) */
6314 m8 = (word64*)m[xi & 0xf];
6315 z8[0] ^= m8[0];
6316 z8[1] ^= m8[1];
6317
6318 /* Cache top byte for remainder calculations - lost in rotate. */
6319 a = z8[1] >> 56;
6320
6321 /* Rotate Z by 8-bits */
6322 z8[1] = (z8[0] >> 56) | (z8[1] << 8);
6323 z8[0] <<= 8;
6324
6325 /* XOR in (next significant nibble * H) [pre-rotated by 4 bits] */
6326 m8 = (word64*)m[16 + (xi >> 4)];
6327 z8[0] ^= m8[0];
6328 z8[1] ^= m8[1];
6329
6330 /* XOR in (msn * remainder) [pre-rotated by 4 bits] */
6331 z8[0] ^= (word64)R[16 + (a & 0xf)];
6332 /* XOR in next significant nibble (XORed with H) * remainder */
6333 m8 = (word64*)m[xi >> 4];
6334 a ^= (byte)(m8[1] >> 52);
6335 z8[0] ^= (word64)R[a >> 4];
6336 }
6337
6338 xi = x[0];
6339
6340 /* XOR in most significant nibble * H */
6341 m8 = (word64*)m[xi & 0xf];
6342 z8[0] ^= m8[0];
6343 z8[1] ^= m8[1];
6344
6345 /* Cache top byte for remainder calculations - lost in rotate. */
6346 a = (z8[1] >> 56) & 0xf;
6347
6348 /* Rotate z by 4-bits */
6349 n3 = z8[1] & 0xf0f0f0f0f0f0f0f0ULL;
6350 n2 = z8[1] & 0x0f0f0f0f0f0f0f0fULL;
6351 n1 = z8[0] & 0xf0f0f0f0f0f0f0f0ULL;
6352 n0 = z8[0] & 0x0f0f0f0f0f0f0f0fULL;
6353 z8[1] = (n3 >> 4) | (n2 << 12) | (n0 >> 52);
6354 z8[0] = (n1 >> 4) | (n0 << 12);
6355
6356 /* XOR in next significant nibble * H */
6357 m8 = (word64*)m[xi >> 4];
6358 z8[0] ^= m8[0];
6359 z8[1] ^= m8[1];
6360 /* XOR in most significant nibble * remainder */
6361 z8[0] ^= (word64)R[a];
6362
6363 /* Write back result. */
6364 x8[0] = z8[0];
6365 x8[1] = z8[1];
6366 }
6367 #endif
6368
GHASH(Aes * aes,const byte * a,word32 aSz,const byte * c,word32 cSz,byte * s,word32 sSz)6369 void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
6370 word32 cSz, byte* s, word32 sSz)
6371 {
6372 byte x[AES_BLOCK_SIZE];
6373 byte scratch[AES_BLOCK_SIZE];
6374 word32 blocks, partial;
6375
6376 if (aes == NULL) {
6377 return;
6378 }
6379
6380 XMEMSET(x, 0, AES_BLOCK_SIZE);
6381
6382 /* Hash in A, the Additional Authentication Data */
6383 if (aSz != 0 && a != NULL) {
6384 blocks = aSz / AES_BLOCK_SIZE;
6385 partial = aSz % AES_BLOCK_SIZE;
6386 while (blocks--) {
6387 xorbuf(x, a, AES_BLOCK_SIZE);
6388 GMULT(x, aes->M0);
6389 a += AES_BLOCK_SIZE;
6390 }
6391 if (partial != 0) {
6392 XMEMSET(scratch, 0, AES_BLOCK_SIZE);
6393 XMEMCPY(scratch, a, partial);
6394 xorbuf(x, scratch, AES_BLOCK_SIZE);
6395 GMULT(x, aes->M0);
6396 }
6397 }
6398
6399 /* Hash in C, the Ciphertext */
6400 if (cSz != 0 && c != NULL) {
6401 blocks = cSz / AES_BLOCK_SIZE;
6402 partial = cSz % AES_BLOCK_SIZE;
6403 while (blocks--) {
6404 xorbuf(x, c, AES_BLOCK_SIZE);
6405 GMULT(x, aes->M0);
6406 c += AES_BLOCK_SIZE;
6407 }
6408 if (partial != 0) {
6409 XMEMSET(scratch, 0, AES_BLOCK_SIZE);
6410 XMEMCPY(scratch, c, partial);
6411 xorbuf(x, scratch, AES_BLOCK_SIZE);
6412 GMULT(x, aes->M0);
6413 }
6414 }
6415
6416 /* Hash in the lengths of A and C in bits */
6417 FlattenSzInBits(&scratch[0], aSz);
6418 FlattenSzInBits(&scratch[8], cSz);
6419 xorbuf(x, scratch, AES_BLOCK_SIZE);
6420 GMULT(x, aes->M0);
6421
6422 /* Copy the result into s. */
6423 XMEMCPY(s, x, sSz);
6424 }
6425
6426 #ifdef WOLFSSL_AESGCM_STREAM
6427 /* No extra initialization for 4-bit table implementation.
6428 *
6429 * @param [in] aes AES GCM object.
6430 */
6431 #define GHASH_INIT_EXTRA(aes)
6432
6433 /* GHASH one block of data..
6434 *
6435 * XOR block into tag and GMULT with H using pre-computed table.
6436 *
6437 * @param [in, out] aes AES GCM object.
6438 * @param [in] block Block of AAD or cipher text.
6439 */
6440 #define GHASH_ONE_BLOCK(aes, block) \
6441 do { \
6442 xorbuf(AES_TAG(aes), block, AES_BLOCK_SIZE); \
6443 GMULT(AES_TAG(aes), aes->M0); \
6444 } \
6445 while (0)
6446 #endif /* WOLFSSL_AESGCM_STREAM */
6447 #elif defined(WORD64_AVAILABLE) && !defined(GCM_WORD32)
6448
6449 #if !defined(FREESCALE_LTC_AES_GCM)
GMULT(word64 * X,word64 * Y)6450 static void GMULT(word64* X, word64* Y)
6451 {
6452 word64 Z[2] = {0,0};
6453 word64 V[2];
6454 int i, j;
6455 #ifdef AES_GCM_GMULT_CT
6456 word64 v1;
6457 #endif
6458 V[0] = X[0]; V[1] = X[1];
6459
6460 for (i = 0; i < 2; i++)
6461 {
6462 word64 y = Y[i];
6463 for (j = 0; j < 64; j++)
6464 {
6465 #ifdef AES_GCM_GMULT_CT
6466 word64 mask = 0 - (y >> 63);
6467 Z[0] ^= V[0] & mask;
6468 Z[1] ^= V[1] & mask;
6469 #else
6470 if (y & 0x8000000000000000ULL) {
6471 Z[0] ^= V[0];
6472 Z[1] ^= V[1];
6473 }
6474 #endif
6475
6476 #ifdef AES_GCM_GMULT_CT
6477 v1 = (0 - (V[1] & 1)) & 0xE100000000000000ULL;
6478 V[1] >>= 1;
6479 V[1] |= V[0] << 63;
6480 V[0] >>= 1;
6481 V[0] ^= v1;
6482 #else
6483 if (V[1] & 0x0000000000000001) {
6484 V[1] >>= 1;
6485 V[1] |= ((V[0] & 0x0000000000000001) ?
6486 0x8000000000000000ULL : 0);
6487 V[0] >>= 1;
6488 V[0] ^= 0xE100000000000000ULL;
6489 }
6490 else {
6491 V[1] >>= 1;
6492 V[1] |= ((V[0] & 0x0000000000000001) ?
6493 0x8000000000000000ULL : 0);
6494 V[0] >>= 1;
6495 }
6496 #endif
6497 y <<= 1;
6498 }
6499 }
6500 X[0] = Z[0];
6501 X[1] = Z[1];
6502 }
6503
6504
GHASH(Aes * aes,const byte * a,word32 aSz,const byte * c,word32 cSz,byte * s,word32 sSz)6505 void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
6506 word32 cSz, byte* s, word32 sSz)
6507 {
6508 word64 x[2] = {0,0};
6509 word32 blocks, partial;
6510 word64 bigH[2];
6511
6512 if (aes == NULL) {
6513 return;
6514 }
6515
6516 XMEMCPY(bigH, aes->H, AES_BLOCK_SIZE);
6517 #ifdef LITTLE_ENDIAN_ORDER
6518 ByteReverseWords64(bigH, bigH, AES_BLOCK_SIZE);
6519 #endif
6520
6521 /* Hash in A, the Additional Authentication Data */
6522 if (aSz != 0 && a != NULL) {
6523 word64 bigA[2];
6524 blocks = aSz / AES_BLOCK_SIZE;
6525 partial = aSz % AES_BLOCK_SIZE;
6526 while (blocks--) {
6527 XMEMCPY(bigA, a, AES_BLOCK_SIZE);
6528 #ifdef LITTLE_ENDIAN_ORDER
6529 ByteReverseWords64(bigA, bigA, AES_BLOCK_SIZE);
6530 #endif
6531 x[0] ^= bigA[0];
6532 x[1] ^= bigA[1];
6533 GMULT(x, bigH);
6534 a += AES_BLOCK_SIZE;
6535 }
6536 if (partial != 0) {
6537 XMEMSET(bigA, 0, AES_BLOCK_SIZE);
6538 XMEMCPY(bigA, a, partial);
6539 #ifdef LITTLE_ENDIAN_ORDER
6540 ByteReverseWords64(bigA, bigA, AES_BLOCK_SIZE);
6541 #endif
6542 x[0] ^= bigA[0];
6543 x[1] ^= bigA[1];
6544 GMULT(x, bigH);
6545 }
6546 #ifdef OPENSSL_EXTRA
6547 /* store AAD partial tag for next call */
6548 aes->aadH[0] = (word32)((x[0] & 0xFFFFFFFF00000000) >> 32);
6549 aes->aadH[1] = (word32)(x[0] & 0xFFFFFFFF);
6550 aes->aadH[2] = (word32)((x[1] & 0xFFFFFFFF00000000) >> 32);
6551 aes->aadH[3] = (word32)(x[1] & 0xFFFFFFFF);
6552 #endif
6553 }
6554
6555 /* Hash in C, the Ciphertext */
6556 if (cSz != 0 && c != NULL) {
6557 word64 bigC[2];
6558 blocks = cSz / AES_BLOCK_SIZE;
6559 partial = cSz % AES_BLOCK_SIZE;
6560 #ifdef OPENSSL_EXTRA
6561 /* Start from last AAD partial tag */
6562 if(aes->aadLen) {
6563 x[0] = ((word64)aes->aadH[0]) << 32 | aes->aadH[1];
6564 x[1] = ((word64)aes->aadH[2]) << 32 | aes->aadH[3];
6565 }
6566 #endif
6567 while (blocks--) {
6568 XMEMCPY(bigC, c, AES_BLOCK_SIZE);
6569 #ifdef LITTLE_ENDIAN_ORDER
6570 ByteReverseWords64(bigC, bigC, AES_BLOCK_SIZE);
6571 #endif
6572 x[0] ^= bigC[0];
6573 x[1] ^= bigC[1];
6574 GMULT(x, bigH);
6575 c += AES_BLOCK_SIZE;
6576 }
6577 if (partial != 0) {
6578 XMEMSET(bigC, 0, AES_BLOCK_SIZE);
6579 XMEMCPY(bigC, c, partial);
6580 #ifdef LITTLE_ENDIAN_ORDER
6581 ByteReverseWords64(bigC, bigC, AES_BLOCK_SIZE);
6582 #endif
6583 x[0] ^= bigC[0];
6584 x[1] ^= bigC[1];
6585 GMULT(x, bigH);
6586 }
6587 }
6588
6589 /* Hash in the lengths in bits of A and C */
6590 {
6591 word64 len[2];
6592 len[0] = aSz; len[1] = cSz;
6593 #ifdef OPENSSL_EXTRA
6594 if (aes->aadLen)
6595 len[0] = (word64)aes->aadLen;
6596 #endif
6597 /* Lengths are in bytes. Convert to bits. */
6598 len[0] *= 8;
6599 len[1] *= 8;
6600
6601 x[0] ^= len[0];
6602 x[1] ^= len[1];
6603 GMULT(x, bigH);
6604 }
6605 #ifdef LITTLE_ENDIAN_ORDER
6606 ByteReverseWords64(x, x, AES_BLOCK_SIZE);
6607 #endif
6608 XMEMCPY(s, x, sSz);
6609 }
6610 #endif /* !FREESCALE_LTC_AES_GCM */
6611
6612 #ifdef WOLFSSL_AESGCM_STREAM
6613
6614 #ifdef LITTLE_ENDIAN_ORDER
6615
6616 /* No extra initialization for small implementation.
6617 *
6618 * @param [in] aes AES GCM object.
6619 */
6620 #define GHASH_INIT_EXTRA(aes) \
6621 ByteReverseWords64((word64*)aes->H, (word64*)aes->H, AES_BLOCK_SIZE)
6622
6623 /* GHASH one block of data..
6624 *
6625 * XOR block into tag and GMULT with H.
6626 *
6627 * @param [in, out] aes AES GCM object.
6628 * @param [in] block Block of AAD or cipher text.
6629 */
6630 #define GHASH_ONE_BLOCK(aes, block) \
6631 do { \
6632 word64* x = (word64*)AES_TAG(aes); \
6633 word64* h = (word64*)aes->H; \
6634 word64 block64[2]; \
6635 XMEMCPY(block64, block, AES_BLOCK_SIZE); \
6636 ByteReverseWords64(block64, block64, AES_BLOCK_SIZE); \
6637 x[0] ^= block64[0]; \
6638 x[1] ^= block64[1]; \
6639 GMULT(x, h); \
6640 } \
6641 while (0)
6642
6643 #ifdef OPENSSL_EXTRA
6644 /* GHASH in AAD and cipher text lengths in bits.
6645 *
6646 * Convert tag back to little-endian.
6647 *
6648 * @param [in, out] aes AES GCM object.
6649 */
6650 #define GHASH_LEN_BLOCK(aes) \
6651 do { \
6652 word64* x = (word64*)AES_TAG(aes); \
6653 word64* h = (word64*)aes->H; \
6654 word64 len[2]; \
6655 len[0] = aes->aSz; len[1] = aes->cSz; \
6656 if (aes->aadLen) \
6657 len[0] = (word64)aes->aadLen; \
6658 /* Lengths are in bytes. Convert to bits. */ \
6659 len[0] *= 8; \
6660 len[1] *= 8; \
6661 \
6662 x[0] ^= len[0]; \
6663 x[1] ^= len[1]; \
6664 GMULT(x, h); \
6665 ByteReverseWords64(x, x, AES_BLOCK_SIZE); \
6666 } \
6667 while (0)
6668 #else
6669 /* GHASH in AAD and cipher text lengths in bits.
6670 *
6671 * Convert tag back to little-endian.
6672 *
6673 * @param [in, out] aes AES GCM object.
6674 */
6675 #define GHASH_LEN_BLOCK(aes) \
6676 do { \
6677 word64* x = (word64*)AES_TAG(aes); \
6678 word64* h = (word64*)aes->H; \
6679 word64 len[2]; \
6680 len[0] = aes->aSz; len[1] = aes->cSz; \
6681 /* Lengths are in bytes. Convert to bits. */ \
6682 len[0] *= 8; \
6683 len[1] *= 8; \
6684 \
6685 x[0] ^= len[0]; \
6686 x[1] ^= len[1]; \
6687 GMULT(x, h); \
6688 ByteReverseWords64(x, x, AES_BLOCK_SIZE); \
6689 } \
6690 while (0)
6691 #endif
6692
6693 #else
6694
6695 /* No extra initialization for small implementation.
6696 *
6697 * @param [in] aes AES GCM object.
6698 */
6699 #define GHASH_INIT_EXTRA(aes)
6700
6701 /* GHASH one block of data..
6702 *
6703 * XOR block into tag and GMULT with H.
6704 *
6705 * @param [in, out] aes AES GCM object.
6706 * @param [in] block Block of AAD or cipher text.
6707 */
6708 #define GHASH_ONE_BLOCK(aes, block) \
6709 do { \
6710 word64* x = (word64*)AES_TAG(aes); \
6711 word64* h = (word64*)aes->H; \
6712 word64 block64[2]; \
6713 XMEMCPY(block64, block, AES_BLOCK_SIZE); \
6714 x[0] ^= block64[0]; \
6715 x[1] ^= block64[1]; \
6716 GMULT(x, h); \
6717 } \
6718 while (0)
6719
6720 #ifdef OPENSSL_EXTRA
6721 /* GHASH in AAD and cipher text lengths in bits.
6722 *
6723 * Convert tag back to little-endian.
6724 *
6725 * @param [in, out] aes AES GCM object.
6726 */
6727 #define GHASH_LEN_BLOCK(aes) \
6728 do { \
6729 word64* x = (word64*)AES_TAG(aes); \
6730 word64* h = (word64*)aes->H; \
6731 word64 len[2]; \
6732 len[0] = aes->aSz; len[1] = aes->cSz; \
6733 if (aes->aadLen) \
6734 len[0] = (word64)aes->aadLen; \
6735 /* Lengths are in bytes. Convert to bits. */ \
6736 len[0] *= 8; \
6737 len[1] *= 8; \
6738 \
6739 x[0] ^= len[0]; \
6740 x[1] ^= len[1]; \
6741 GMULT(x, h); \
6742 } \
6743 while (0)
6744 #else
6745 /* GHASH in AAD and cipher text lengths in bits.
6746 *
6747 * Convert tag back to little-endian.
6748 *
6749 * @param [in, out] aes AES GCM object.
6750 */
6751 #define GHASH_LEN_BLOCK(aes) \
6752 do { \
6753 word64* x = (word64*)AES_TAG(aes); \
6754 word64* h = (word64*)aes->H; \
6755 word64 len[2]; \
6756 len[0] = aes->aSz; len[1] = aes->cSz; \
6757 /* Lengths are in bytes. Convert to bits. */ \
6758 len[0] *= 8; \
6759 len[1] *= 8; \
6760 \
6761 x[0] ^= len[0]; \
6762 x[1] ^= len[1]; \
6763 GMULT(x, h); \
6764 } \
6765 while (0)
6766 #endif
6767
6768 #endif /* !LITTLE_ENDIAN_ORDER */
6769
6770 #endif /* WOLFSSL_AESGCM_STREAM */
6771 /* end defined(WORD64_AVAILABLE) && !defined(GCM_WORD32) */
6772 #else /* GCM_WORD32 */
6773
GMULT(word32 * X,word32 * Y)6774 static void GMULT(word32* X, word32* Y)
6775 {
6776 word32 Z[4] = {0,0,0,0};
6777 word32 V[4];
6778 int i, j;
6779
6780 V[0] = X[0]; V[1] = X[1]; V[2] = X[2]; V[3] = X[3];
6781
6782 for (i = 0; i < 4; i++)
6783 {
6784 word32 y = Y[i];
6785 for (j = 0; j < 32; j++)
6786 {
6787 if (y & 0x80000000) {
6788 Z[0] ^= V[0];
6789 Z[1] ^= V[1];
6790 Z[2] ^= V[2];
6791 Z[3] ^= V[3];
6792 }
6793
6794 if (V[3] & 0x00000001) {
6795 V[3] >>= 1;
6796 V[3] |= ((V[2] & 0x00000001) ? 0x80000000 : 0);
6797 V[2] >>= 1;
6798 V[2] |= ((V[1] & 0x00000001) ? 0x80000000 : 0);
6799 V[1] >>= 1;
6800 V[1] |= ((V[0] & 0x00000001) ? 0x80000000 : 0);
6801 V[0] >>= 1;
6802 V[0] ^= 0xE1000000;
6803 } else {
6804 V[3] >>= 1;
6805 V[3] |= ((V[2] & 0x00000001) ? 0x80000000 : 0);
6806 V[2] >>= 1;
6807 V[2] |= ((V[1] & 0x00000001) ? 0x80000000 : 0);
6808 V[1] >>= 1;
6809 V[1] |= ((V[0] & 0x00000001) ? 0x80000000 : 0);
6810 V[0] >>= 1;
6811 }
6812 y <<= 1;
6813 }
6814 }
6815 X[0] = Z[0];
6816 X[1] = Z[1];
6817 X[2] = Z[2];
6818 X[3] = Z[3];
6819 }
6820
6821
GHASH(Aes * aes,const byte * a,word32 aSz,const byte * c,word32 cSz,byte * s,word32 sSz)6822 void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
6823 word32 cSz, byte* s, word32 sSz)
6824 {
6825 word32 x[4] = {0,0,0,0};
6826 word32 blocks, partial;
6827 word32 bigH[4];
6828
6829 if (aes == NULL) {
6830 return;
6831 }
6832
6833 XMEMCPY(bigH, aes->H, AES_BLOCK_SIZE);
6834 #ifdef LITTLE_ENDIAN_ORDER
6835 ByteReverseWords(bigH, bigH, AES_BLOCK_SIZE);
6836 #endif
6837
6838 /* Hash in A, the Additional Authentication Data */
6839 if (aSz != 0 && a != NULL) {
6840 word32 bigA[4];
6841 blocks = aSz / AES_BLOCK_SIZE;
6842 partial = aSz % AES_BLOCK_SIZE;
6843 while (blocks--) {
6844 XMEMCPY(bigA, a, AES_BLOCK_SIZE);
6845 #ifdef LITTLE_ENDIAN_ORDER
6846 ByteReverseWords(bigA, bigA, AES_BLOCK_SIZE);
6847 #endif
6848 x[0] ^= bigA[0];
6849 x[1] ^= bigA[1];
6850 x[2] ^= bigA[2];
6851 x[3] ^= bigA[3];
6852 GMULT(x, bigH);
6853 a += AES_BLOCK_SIZE;
6854 }
6855 if (partial != 0) {
6856 XMEMSET(bigA, 0, AES_BLOCK_SIZE);
6857 XMEMCPY(bigA, a, partial);
6858 #ifdef LITTLE_ENDIAN_ORDER
6859 ByteReverseWords(bigA, bigA, AES_BLOCK_SIZE);
6860 #endif
6861 x[0] ^= bigA[0];
6862 x[1] ^= bigA[1];
6863 x[2] ^= bigA[2];
6864 x[3] ^= bigA[3];
6865 GMULT(x, bigH);
6866 }
6867 }
6868
6869 /* Hash in C, the Ciphertext */
6870 if (cSz != 0 && c != NULL) {
6871 word32 bigC[4];
6872 blocks = cSz / AES_BLOCK_SIZE;
6873 partial = cSz % AES_BLOCK_SIZE;
6874 while (blocks--) {
6875 XMEMCPY(bigC, c, AES_BLOCK_SIZE);
6876 #ifdef LITTLE_ENDIAN_ORDER
6877 ByteReverseWords(bigC, bigC, AES_BLOCK_SIZE);
6878 #endif
6879 x[0] ^= bigC[0];
6880 x[1] ^= bigC[1];
6881 x[2] ^= bigC[2];
6882 x[3] ^= bigC[3];
6883 GMULT(x, bigH);
6884 c += AES_BLOCK_SIZE;
6885 }
6886 if (partial != 0) {
6887 XMEMSET(bigC, 0, AES_BLOCK_SIZE);
6888 XMEMCPY(bigC, c, partial);
6889 #ifdef LITTLE_ENDIAN_ORDER
6890 ByteReverseWords(bigC, bigC, AES_BLOCK_SIZE);
6891 #endif
6892 x[0] ^= bigC[0];
6893 x[1] ^= bigC[1];
6894 x[2] ^= bigC[2];
6895 x[3] ^= bigC[3];
6896 GMULT(x, bigH);
6897 }
6898 }
6899
6900 /* Hash in the lengths in bits of A and C */
6901 {
6902 word32 len[4];
6903
6904 /* Lengths are in bytes. Convert to bits. */
6905 len[0] = (aSz >> (8*sizeof(aSz) - 3));
6906 len[1] = aSz << 3;
6907 len[2] = (cSz >> (8*sizeof(cSz) - 3));
6908 len[3] = cSz << 3;
6909
6910 x[0] ^= len[0];
6911 x[1] ^= len[1];
6912 x[2] ^= len[2];
6913 x[3] ^= len[3];
6914 GMULT(x, bigH);
6915 }
6916 #ifdef LITTLE_ENDIAN_ORDER
6917 ByteReverseWords(x, x, AES_BLOCK_SIZE);
6918 #endif
6919 XMEMCPY(s, x, sSz);
6920 }
6921
6922 #ifdef WOLFSSL_AESGCM_STREAM
6923 #ifdef LITTLE_ENDIAN_ORDER
6924 /* Little-endian 32-bit word implementation requires byte reversal of H.
6925 *
6926 * H is all-zeros block encrypted with key.
6927 *
6928 * @param [in, out] aes AES GCM object.
6929 */
6930 #define GHASH_INIT_EXTRA(aes) \
6931 ByteReverseWords((word32*)aes->H, (word32*)aes->H, AES_BLOCK_SIZE)
6932
6933 /* GHASH one block of data..
6934 *
6935 * XOR block, in big-endian form, into tag and GMULT with H.
6936 *
6937 * @param [in, out] aes AES GCM object.
6938 * @param [in] block Block of AAD or cipher text.
6939 */
6940 #define GHASH_ONE_BLOCK(aes, block) \
6941 do { \
6942 word32* x = (word32*)AES_TAG(aes); \
6943 word32* h = (word32*)aes->H; \
6944 word32 bigEnd[4]; \
6945 XMEMCPY(bigEnd, block, AES_BLOCK_SIZE); \
6946 ByteReverseWords(bigEnd, bigEnd, AES_BLOCK_SIZE); \
6947 x[0] ^= bigEnd[0]; \
6948 x[1] ^= bigEnd[1]; \
6949 x[2] ^= bigEnd[2]; \
6950 x[3] ^= bigEnd[3]; \
6951 GMULT(x, h); \
6952 } \
6953 while (0)
6954
6955 /* GHASH in AAD and cipher text lengths in bits.
6956 *
6957 * Convert tag back to little-endian.
6958 *
6959 * @param [in, out] aes AES GCM object.
6960 */
6961 #define GHASH_LEN_BLOCK(aes) \
6962 do { \
6963 word32 len[4]; \
6964 word32* x = (word32*)AES_TAG(aes); \
6965 word32* h = (word32*)aes->H; \
6966 len[0] = (aes->aSz >> (8*sizeof(aes->aSz) - 3)); \
6967 len[1] = aes->aSz << 3; \
6968 len[2] = (aes->cSz >> (8*sizeof(aes->cSz) - 3)); \
6969 len[3] = aes->cSz << 3; \
6970 x[0] ^= len[0]; \
6971 x[1] ^= len[1]; \
6972 x[2] ^= len[2]; \
6973 x[3] ^= len[3]; \
6974 GMULT(x, h); \
6975 ByteReverseWords(x, x, AES_BLOCK_SIZE); \
6976 } \
6977 while (0)
6978 #else
6979 /* No extra initialization for 32-bit word implementation.
6980 *
6981 * @param [in] aes AES GCM object.
6982 */
6983 #define GHASH_INIT_EXTRA(aes)
6984
6985 /* GHASH one block of data..
6986 *
6987 * XOR block into tag and GMULT with H.
6988 *
6989 * @param [in, out] aes AES GCM object.
6990 * @param [in] block Block of AAD or cipher text.
6991 */
6992 #define GHASH_ONE_BLOCK(aes, block) \
6993 do { \
6994 word32* x = (word32*)AES_TAG(aes); \
6995 word32* h = (word32*)aes->H; \
6996 word32 block32[4]; \
6997 XMEMCPY(block32, block, AES_BLOCK_SIZE); \
6998 x[0] ^= block32[0]; \
6999 x[1] ^= block32[1]; \
7000 x[2] ^= block32[2]; \
7001 x[3] ^= block32[3]; \
7002 GMULT(x, h); \
7003 } \
7004 while (0)
7005
7006 /* GHASH in AAD and cipher text lengths in bits.
7007 *
7008 * @param [in, out] aes AES GCM object.
7009 */
7010 #define GHASH_LEN_BLOCK(aes) \
7011 do { \
7012 word32 len[4]; \
7013 word32* x = (word32*)AES_TAG(aes); \
7014 word32* h = (word32*)aes->H; \
7015 len[0] = (aes->aSz >> (8*sizeof(aes->aSz) - 3)); \
7016 len[1] = aes->aSz << 3; \
7017 len[2] = (aes->cSz >> (8*sizeof(aes->cSz) - 3)); \
7018 len[3] = aes->cSz << 3; \
7019 x[0] ^= len[0]; \
7020 x[1] ^= len[1]; \
7021 x[2] ^= len[2]; \
7022 x[3] ^= len[3]; \
7023 GMULT(x, h); \
7024 } \
7025 while (0)
7026 #endif /* LITTLE_ENDIAN_ORDER */
7027 #endif /* WOLFSSL_AESGCM_STREAM */
7028 #endif /* end GCM_WORD32 */
7029
7030 #if !defined(WOLFSSL_XILINX_CRYPT) && !defined(WOLFSSL_AFALG_XILINX_AES)
7031 #ifdef WOLFSSL_AESGCM_STREAM
7032 #ifndef GHASH_LEN_BLOCK
7033 /* Hash in the lengths of the AAD and cipher text in bits.
7034 *
7035 * Default implementation.
7036 *
7037 * @param [in, out] aes AES GCM object.
7038 */
7039 #define GHASH_LEN_BLOCK(aes) \
7040 do { \
7041 byte scratch[AES_BLOCK_SIZE]; \
7042 FlattenSzInBits(&scratch[0], aes->aSz); \
7043 FlattenSzInBits(&scratch[8], aes->cSz); \
7044 GHASH_ONE_BLOCK(aes, scratch); \
7045 } \
7046 while (0)
7047 #endif
7048
7049 /* Initialize a GHASH for streaming operations.
7050 *
7051 * @param [in, out] aes AES GCM object.
7052 */
GHASH_INIT(Aes * aes)7053 static void GHASH_INIT(Aes* aes) {
7054 /* Set tag to all zeros as initial value. */
7055 XMEMSET(AES_TAG(aes), 0, AES_BLOCK_SIZE);
7056 /* Reset counts of AAD and cipher text. */
7057 aes->aOver = 0;
7058 aes->cOver = 0;
7059 /* Extra initialization baed on implementation. */
7060 GHASH_INIT_EXTRA(aes);
7061 }
7062
7063 /* Update the GHASH with AAD and/or cipher text.
7064 *
7065 * @param [in,out] aes AES GCM object.
7066 * @param [in] a Additional authentication data buffer.
7067 * @param [in] aSz Size of data in AAD buffer.
7068 * @param [in] c Cipher text buffer.
7069 * @param [in] cSz Size of data in cipher text buffer.
7070 */
GHASH_UPDATE(Aes * aes,const byte * a,word32 aSz,const byte * c,word32 cSz)7071 static void GHASH_UPDATE(Aes* aes, const byte* a, word32 aSz, const byte* c,
7072 word32 cSz)
7073 {
7074 word32 blocks;
7075 word32 partial;
7076
7077 /* Hash in A, the Additional Authentication Data */
7078 if (aSz != 0 && a != NULL) {
7079 /* Update count of AAD we have hashed. */
7080 aes->aSz += aSz;
7081 /* Check if we have unprocessed data. */
7082 if (aes->aOver > 0) {
7083 /* Calculate amount we can use - fill up the block. */
7084 byte sz = AES_BLOCK_SIZE - aes->aOver;
7085 if (sz > aSz) {
7086 sz = aSz;
7087 }
7088 /* Copy extra into last GHASH block array and update count. */
7089 XMEMCPY(AES_LASTGBLOCK(aes) + aes->aOver, a, sz);
7090 aes->aOver += sz;
7091 if (aes->aOver == AES_BLOCK_SIZE) {
7092 /* We have filled up the block and can process. */
7093 GHASH_ONE_BLOCK(aes, AES_LASTGBLOCK(aes));
7094 /* Reset count. */
7095 aes->aOver = 0;
7096 }
7097 /* Used up some data. */
7098 aSz -= sz;
7099 a += sz;
7100 }
7101
7102 /* Calculate number of blocks of AAD and the leftover. */
7103 blocks = aSz / AES_BLOCK_SIZE;
7104 partial = aSz % AES_BLOCK_SIZE;
7105 /* GHASH full blocks now. */
7106 while (blocks--) {
7107 GHASH_ONE_BLOCK(aes, a);
7108 a += AES_BLOCK_SIZE;
7109 }
7110 if (partial != 0) {
7111 /* Cache the partial block. */
7112 XMEMCPY(AES_LASTGBLOCK(aes), a, partial);
7113 aes->aOver = (byte)partial;
7114 }
7115 }
7116 if (aes->aOver > 0 && cSz > 0 && c != NULL) {
7117 /* No more AAD coming and we have a partial block. */
7118 /* Fill the rest of the block with zeros. */
7119 byte sz = AES_BLOCK_SIZE - aes->aOver;
7120 XMEMSET(AES_LASTGBLOCK(aes) + aes->aOver, 0, sz);
7121 /* GHASH last AAD block. */
7122 GHASH_ONE_BLOCK(aes, AES_LASTGBLOCK(aes));
7123 /* Clear partial count for next time through. */
7124 aes->aOver = 0;
7125 }
7126
7127 /* Hash in C, the Ciphertext */
7128 if (cSz != 0 && c != NULL) {
7129 /* Update count of cipher text we have hashed. */
7130 aes->cSz += cSz;
7131 if (aes->cOver > 0) {
7132 /* Calculate amount we can use - fill up the block. */
7133 byte sz = AES_BLOCK_SIZE - aes->cOver;
7134 if (sz > cSz) {
7135 sz = cSz;
7136 }
7137 XMEMCPY(AES_LASTGBLOCK(aes) + aes->cOver, c, sz);
7138 /* Update count of unsed encrypted counter. */
7139 aes->cOver += sz;
7140 if (aes->cOver == AES_BLOCK_SIZE) {
7141 /* We have filled up the block and can process. */
7142 GHASH_ONE_BLOCK(aes, AES_LASTGBLOCK(aes));
7143 /* Reset count. */
7144 aes->cOver = 0;
7145 }
7146 /* Used up some data. */
7147 cSz -= sz;
7148 c += sz;
7149 }
7150
7151 /* Calculate number of blocks of cipher text and the leftover. */
7152 blocks = cSz / AES_BLOCK_SIZE;
7153 partial = cSz % AES_BLOCK_SIZE;
7154 /* GHASH full blocks now. */
7155 while (blocks--) {
7156 GHASH_ONE_BLOCK(aes, c);
7157 c += AES_BLOCK_SIZE;
7158 }
7159 if (partial != 0) {
7160 /* Cache the partial block. */
7161 XMEMCPY(AES_LASTGBLOCK(aes), c, partial);
7162 aes->cOver = (byte)partial;
7163 }
7164 }
7165 }
7166
7167 /* Finalize the GHASH calculation.
7168 *
7169 * Complete hashing cipher text and hash the AAD and cipher text lengths.
7170 *
7171 * @param [in, out] aes AES GCM object.
7172 * @param [out] s Authentication tag.
7173 * @param [in] sSz Size of authentication tag required.
7174 */
GHASH_FINAL(Aes * aes,byte * s,word32 sSz)7175 static void GHASH_FINAL(Aes* aes, byte* s, word32 sSz)
7176 {
7177 /* AAD block incomplete when > 0 */
7178 byte over = aes->aOver;
7179
7180 if (aes->cOver > 0) {
7181 /* Cipher text block incomplete. */
7182 over = aes->cOver;
7183 }
7184 if (over > 0) {
7185 /* Zeroize the unused part of the block. */
7186 XMEMSET(AES_LASTGBLOCK(aes) + over, 0, AES_BLOCK_SIZE - over);
7187 /* Hash the last block of cipher text. */
7188 GHASH_ONE_BLOCK(aes, AES_LASTGBLOCK(aes));
7189 }
7190 /* Hash in the lengths of AAD and cipher text in bits */
7191 GHASH_LEN_BLOCK(aes);
7192 /* Copy the result into s. */
7193 XMEMCPY(s, AES_TAG(aes), sSz);
7194 }
7195 #endif /* WOLFSSL_AESGCM_STREAM */
7196
7197
7198 #ifdef FREESCALE_LTC_AES_GCM
wc_AesGcmEncrypt(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)7199 int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
7200 const byte* iv, word32 ivSz,
7201 byte* authTag, word32 authTagSz,
7202 const byte* authIn, word32 authInSz)
7203 {
7204 status_t status;
7205 word32 keySize;
7206
7207 /* argument checks */
7208 if (aes == NULL || authTagSz > AES_BLOCK_SIZE || ivSz == 0) {
7209 return BAD_FUNC_ARG;
7210 }
7211
7212 if (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
7213 WOLFSSL_MSG("GcmEncrypt authTagSz too small error");
7214 return BAD_FUNC_ARG;
7215 }
7216
7217 status = wc_AesGetKeySize(aes, &keySize);
7218 if (status)
7219 return status;
7220
7221 status = wolfSSL_CryptHwMutexLock();
7222 if (status != 0)
7223 return status;
7224
7225 status = LTC_AES_EncryptTagGcm(LTC_BASE, in, out, sz, iv, ivSz,
7226 authIn, authInSz, (byte*)aes->key, keySize, authTag, authTagSz);
7227 wolfSSL_CryptHwMutexUnLock();
7228
7229 return (status == kStatus_Success) ? 0 : AES_GCM_AUTH_E;
7230 }
7231
7232 #else
7233
7234 #ifdef STM32_CRYPTO_AES_GCM
7235
7236 /* this function supports inline encrypt */
7237 /* define STM32_AESGCM_PARTIAL for newer STM Cube HAL's with workaround
7238 for handling partial packets to improve auth tag calculation performance by
7239 using hardware */
wc_AesGcmEncrypt_STM32(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)7240 static int wc_AesGcmEncrypt_STM32(Aes* aes, byte* out, const byte* in, word32 sz,
7241 const byte* iv, word32 ivSz,
7242 byte* authTag, word32 authTagSz,
7243 const byte* authIn, word32 authInSz)
7244 {
7245 int ret;
7246 #ifdef WOLFSSL_STM32_CUBEMX
7247 CRYP_HandleTypeDef hcryp;
7248 #else
7249 word32 keyCopy[AES_256_KEY_SIZE/sizeof(word32)];
7250 #endif
7251 word32 keySize;
7252 #ifdef WOLFSSL_STM32_CUBEMX
7253 int status = HAL_OK;
7254 word32 blocks = sz / AES_BLOCK_SIZE;
7255 word32 partialBlock[AES_BLOCK_SIZE/sizeof(word32)];
7256 #else
7257 int status = SUCCESS;
7258 #endif
7259 word32 partial = sz % AES_BLOCK_SIZE;
7260 word32 tag[AES_BLOCK_SIZE/sizeof(word32)];
7261 word32 ctrInit[AES_BLOCK_SIZE/sizeof(word32)];
7262 word32 ctr[AES_BLOCK_SIZE/sizeof(word32)];
7263 word32 authhdr[AES_BLOCK_SIZE/sizeof(word32)];
7264 byte* authInPadded = NULL;
7265 int authPadSz, wasAlloc = 0, useSwGhash = 0;
7266
7267 ret = wc_AesGetKeySize(aes, &keySize);
7268 if (ret != 0)
7269 return ret;
7270
7271 #ifdef WOLFSSL_STM32_CUBEMX
7272 ret = wc_Stm32_Aes_Init(aes, &hcryp);
7273 if (ret != 0)
7274 return ret;
7275 #endif
7276
7277 XMEMSET(ctr, 0, AES_BLOCK_SIZE);
7278 if (ivSz == GCM_NONCE_MID_SZ) {
7279 byte* pCtr = (byte*)ctr;
7280 XMEMCPY(ctr, iv, ivSz);
7281 pCtr[AES_BLOCK_SIZE - 1] = 1;
7282 }
7283 else {
7284 GHASH(aes, NULL, 0, iv, ivSz, (byte*)ctr, AES_BLOCK_SIZE);
7285 }
7286 XMEMCPY(ctrInit, ctr, sizeof(ctr)); /* save off initial counter for GMAC */
7287
7288 /* Authentication buffer - must be 4-byte multiple zero padded */
7289 authPadSz = authInSz % sizeof(word32);
7290 if (authPadSz != 0) {
7291 authPadSz = authInSz + sizeof(word32) - authPadSz;
7292 if (authPadSz <= sizeof(authhdr)) {
7293 authInPadded = (byte*)authhdr;
7294 }
7295 else {
7296 authInPadded = (byte*)XMALLOC(authPadSz, aes->heap,
7297 DYNAMIC_TYPE_TMP_BUFFER);
7298 if (authInPadded == NULL) {
7299 wolfSSL_CryptHwMutexUnLock();
7300 return MEMORY_E;
7301 }
7302 wasAlloc = 1;
7303 }
7304 XMEMSET(authInPadded, 0, authPadSz);
7305 XMEMCPY(authInPadded, authIn, authInSz);
7306 } else {
7307 authPadSz = authInSz;
7308 authInPadded = (byte*)authIn;
7309 }
7310
7311 /* for cases where hardware cannot be used for authTag calculate it */
7312 /* if IV is not 12 calculate GHASH using software */
7313 if (ivSz != GCM_NONCE_MID_SZ
7314 #ifndef CRYP_HEADERWIDTHUNIT_BYTE
7315 /* or harware that does not support partial block */
7316 || sz == 0 || partial != 0
7317 #endif
7318 #if !defined(CRYP_HEADERWIDTHUNIT_BYTE) && !defined(STM32_AESGCM_PARTIAL)
7319 /* or authIn is not a multiple of 4 */
7320 || authPadSz != authInSz
7321 #endif
7322 ) {
7323 useSwGhash = 1;
7324 }
7325
7326 /* Hardware requires counter + 1 */
7327 IncrementGcmCounter((byte*)ctr);
7328
7329 ret = wolfSSL_CryptHwMutexLock();
7330 if (ret != 0) {
7331 return ret;
7332 }
7333 #ifdef WOLFSSL_STM32_CUBEMX
7334 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)ctr;
7335 hcryp.Init.Header = (STM_CRYPT_TYPE*)authInPadded;
7336
7337 #if defined(STM32_HAL_V2)
7338 hcryp.Init.Algorithm = CRYP_AES_GCM;
7339 #ifdef CRYP_HEADERWIDTHUNIT_BYTE
7340 /* V2 with CRYP_HEADERWIDTHUNIT_BYTE uses byte size for header */
7341 hcryp.Init.HeaderSize = authInSz;
7342 #else
7343 hcryp.Init.HeaderSize = authPadSz/sizeof(word32);
7344 #endif
7345 #ifdef STM32_AESGCM_PARTIAL
7346 hcryp.Init.HeaderPadSize = authPadSz - authInSz;
7347 #endif
7348 #ifdef CRYP_KEYIVCONFIG_ONCE
7349 /* allows repeated calls to HAL_CRYP_Encrypt */
7350 hcryp.Init.KeyIVConfigSkip = CRYP_KEYIVCONFIG_ONCE;
7351 #endif
7352 ByteReverseWords(ctr, ctr, AES_BLOCK_SIZE);
7353 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)ctr;
7354 HAL_CRYP_Init(&hcryp);
7355
7356 #ifndef CRYP_KEYIVCONFIG_ONCE
7357 /* GCM payload phase - can handle partial blocks */
7358 status = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in,
7359 (blocks * AES_BLOCK_SIZE) + partial, (uint32_t*)out, STM32_HAL_TIMEOUT);
7360 #else
7361 /* GCM payload phase - blocks */
7362 if (blocks) {
7363 status = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in,
7364 (blocks * AES_BLOCK_SIZE), (uint32_t*)out, STM32_HAL_TIMEOUT);
7365 }
7366 /* GCM payload phase - partial remainder */
7367 if (status == HAL_OK && (partial != 0 || blocks == 0)) {
7368 XMEMSET(partialBlock, 0, sizeof(partialBlock));
7369 XMEMCPY(partialBlock, in + (blocks * AES_BLOCK_SIZE), partial);
7370 status = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)partialBlock, partial,
7371 (uint32_t*)partialBlock, STM32_HAL_TIMEOUT);
7372 XMEMCPY(out + (blocks * AES_BLOCK_SIZE), partialBlock, partial);
7373 }
7374 #endif
7375 if (status == HAL_OK && !useSwGhash) {
7376 /* Compute the authTag */
7377 status = HAL_CRYPEx_AESGCM_GenerateAuthTAG(&hcryp, (uint32_t*)tag,
7378 STM32_HAL_TIMEOUT);
7379 }
7380 #elif defined(STM32_CRYPTO_AES_ONLY)
7381 /* Set the CRYP parameters */
7382 hcryp.Init.HeaderSize = authPadSz;
7383 if (authPadSz == 0)
7384 hcryp.Init.Header = NULL; /* cannot pass pointer here when authIn == 0 */
7385 hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_GCM_GMAC;
7386 hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
7387 hcryp.Init.GCMCMACPhase = CRYP_INIT_PHASE;
7388 HAL_CRYP_Init(&hcryp);
7389
7390 /* GCM init phase */
7391 status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
7392 if (status == HAL_OK) {
7393 /* GCM header phase */
7394 hcryp.Init.GCMCMACPhase = CRYP_HEADER_PHASE;
7395 status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
7396 }
7397 if (status == HAL_OK) {
7398 /* GCM payload phase - blocks */
7399 hcryp.Init.GCMCMACPhase = CRYP_PAYLOAD_PHASE;
7400 if (blocks) {
7401 status = HAL_CRYPEx_AES_Auth(&hcryp, (byte*)in,
7402 (blocks * AES_BLOCK_SIZE), out, STM32_HAL_TIMEOUT);
7403 }
7404 }
7405 if (status == HAL_OK && (partial != 0 || (sz > 0 && blocks == 0))) {
7406 /* GCM payload phase - partial remainder */
7407 XMEMSET(partialBlock, 0, sizeof(partialBlock));
7408 XMEMCPY(partialBlock, in + (blocks * AES_BLOCK_SIZE), partial);
7409 status = HAL_CRYPEx_AES_Auth(&hcryp, (uint8_t*)partialBlock, partial,
7410 (uint8_t*)partialBlock, STM32_HAL_TIMEOUT);
7411 XMEMCPY(out + (blocks * AES_BLOCK_SIZE), partialBlock, partial);
7412 }
7413 if (status == HAL_OK && !useSwGhash) {
7414 /* GCM final phase */
7415 hcryp.Init.GCMCMACPhase = CRYP_FINAL_PHASE;
7416 status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, sz, (uint8_t*)tag, STM32_HAL_TIMEOUT);
7417 }
7418 #else
7419 hcryp.Init.HeaderSize = authPadSz;
7420 HAL_CRYP_Init(&hcryp);
7421 if (blocks) {
7422 /* GCM payload phase - blocks */
7423 status = HAL_CRYPEx_AESGCM_Encrypt(&hcryp, (byte*)in,
7424 (blocks * AES_BLOCK_SIZE), out, STM32_HAL_TIMEOUT);
7425 }
7426 if (status == HAL_OK && (partial != 0 || blocks == 0)) {
7427 /* GCM payload phase - partial remainder */
7428 XMEMSET(partialBlock, 0, sizeof(partialBlock));
7429 XMEMCPY(partialBlock, in + (blocks * AES_BLOCK_SIZE), partial);
7430 status = HAL_CRYPEx_AESGCM_Encrypt(&hcryp, (uint8_t*)partialBlock, partial,
7431 (uint8_t*)partialBlock, STM32_HAL_TIMEOUT);
7432 XMEMCPY(out + (blocks * AES_BLOCK_SIZE), partialBlock, partial);
7433 }
7434 if (status == HAL_OK && !useSwGhash) {
7435 /* Compute the authTag */
7436 status = HAL_CRYPEx_AESGCM_Finish(&hcryp, sz, (uint8_t*)tag, STM32_HAL_TIMEOUT);
7437 }
7438 #endif
7439
7440 if (status != HAL_OK)
7441 ret = AES_GCM_AUTH_E;
7442 HAL_CRYP_DeInit(&hcryp);
7443
7444 #else /* Standard Peripheral Library */
7445 ByteReverseWords(keyCopy, (word32*)aes->key, keySize);
7446 status = CRYP_AES_GCM(MODE_ENCRYPT, (uint8_t*)ctr,
7447 (uint8_t*)keyCopy, keySize * 8,
7448 (uint8_t*)in, sz,
7449 (uint8_t*)authInPadded, authInSz,
7450 (uint8_t*)out, (uint8_t*)tag);
7451 if (status != SUCCESS)
7452 ret = AES_GCM_AUTH_E;
7453 #endif /* WOLFSSL_STM32_CUBEMX */
7454 wolfSSL_CryptHwMutexUnLock();
7455
7456 if (ret == 0) {
7457 /* return authTag */
7458 if (authTag) {
7459 if (useSwGhash) {
7460 GHASH(aes, authIn, authInSz, out, sz, authTag, authTagSz);
7461 wc_AesEncrypt(aes, (byte*)ctrInit, (byte*)tag);
7462 xorbuf(authTag, tag, authTagSz);
7463 }
7464 else {
7465 /* use hardware calculated tag */
7466 XMEMCPY(authTag, tag, authTagSz);
7467 }
7468 }
7469 }
7470
7471 /* Free memory */
7472 if (wasAlloc) {
7473 XFREE(authInPadded, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
7474 }
7475
7476 return ret;
7477 }
7478
7479 #endif /* STM32_CRYPTO_AES_GCM */
7480
7481 #ifdef WOLFSSL_AESNI
7482 /* For performance reasons, this code needs to be not inlined. */
7483 int AES_GCM_encrypt_C(Aes* aes, byte* out, const byte* in, word32 sz,
7484 const byte* iv, word32 ivSz,
7485 byte* authTag, word32 authTagSz,
7486 const byte* authIn, word32 authInSz);
7487 #else
7488 static
7489 #endif
AES_GCM_encrypt_C(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)7490 int AES_GCM_encrypt_C(Aes* aes, byte* out, const byte* in, word32 sz,
7491 const byte* iv, word32 ivSz,
7492 byte* authTag, word32 authTagSz,
7493 const byte* authIn, word32 authInSz)
7494 {
7495 int ret = 0;
7496 word32 blocks = sz / AES_BLOCK_SIZE;
7497 word32 partial = sz % AES_BLOCK_SIZE;
7498 const byte* p = in;
7499 byte* c = out;
7500 ALIGN32 byte counter[AES_BLOCK_SIZE];
7501 ALIGN32 byte initialCounter[AES_BLOCK_SIZE];
7502 ALIGN32 byte scratch[AES_BLOCK_SIZE];
7503
7504 if (ivSz == GCM_NONCE_MID_SZ) {
7505 /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */
7506 XMEMCPY(counter, iv, ivSz);
7507 XMEMSET(counter + GCM_NONCE_MID_SZ, 0,
7508 AES_BLOCK_SIZE - GCM_NONCE_MID_SZ - 1);
7509 counter[AES_BLOCK_SIZE - 1] = 1;
7510 }
7511 else {
7512 /* Counter is GHASH of IV. */
7513 #ifdef OPENSSL_EXTRA
7514 word32 aadTemp = aes->aadLen;
7515 aes->aadLen = 0;
7516 #endif
7517 GHASH(aes, NULL, 0, iv, ivSz, counter, AES_BLOCK_SIZE);
7518 #ifdef OPENSSL_EXTRA
7519 aes->aadLen = aadTemp;
7520 #endif
7521 }
7522 XMEMCPY(initialCounter, counter, AES_BLOCK_SIZE);
7523
7524 #ifdef WOLFSSL_PIC32MZ_CRYPT
7525 if (blocks) {
7526 /* use initial IV for HW, but don't use it below */
7527 XMEMCPY(aes->reg, counter, AES_BLOCK_SIZE);
7528
7529 ret = wc_Pic32AesCrypt(
7530 aes->key, aes->keylen, aes->reg, AES_BLOCK_SIZE,
7531 out, in, (blocks * AES_BLOCK_SIZE),
7532 PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_AES_GCM);
7533 if (ret != 0)
7534 return ret;
7535 }
7536 /* process remainder using partial handling */
7537 #endif
7538
7539 #if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT)
7540 /* some hardware acceleration can gain performance from doing AES encryption
7541 * of the whole buffer at once */
7542 if (c != p && blocks > 0) { /* can not handle inline encryption */
7543 while (blocks--) {
7544 IncrementGcmCounter(counter);
7545 XMEMCPY(c, counter, AES_BLOCK_SIZE);
7546 c += AES_BLOCK_SIZE;
7547 }
7548
7549 /* reset number of blocks and then do encryption */
7550 blocks = sz / AES_BLOCK_SIZE;
7551 wc_AesEcbEncrypt(aes, out, out, AES_BLOCK_SIZE * blocks);
7552 xorbuf(out, p, AES_BLOCK_SIZE * blocks);
7553 p += AES_BLOCK_SIZE * blocks;
7554 }
7555 else
7556 #endif /* HAVE_AES_ECB && !WOLFSSL_PIC32MZ_CRYPT */
7557 {
7558 while (blocks--) {
7559 IncrementGcmCounter(counter);
7560 #if !defined(WOLFSSL_PIC32MZ_CRYPT)
7561 wc_AesEncrypt(aes, counter, scratch);
7562 xorbufout(c, scratch, p, AES_BLOCK_SIZE);
7563 #endif
7564 p += AES_BLOCK_SIZE;
7565 c += AES_BLOCK_SIZE;
7566 }
7567 }
7568
7569 if (partial != 0) {
7570 IncrementGcmCounter(counter);
7571 wc_AesEncrypt(aes, counter, scratch);
7572 xorbufout(c, scratch, p, partial);
7573 }
7574 if (authTag) {
7575 GHASH(aes, authIn, authInSz, out, sz, authTag, authTagSz);
7576 wc_AesEncrypt(aes, initialCounter, scratch);
7577 xorbuf(authTag, scratch, authTagSz);
7578 #ifdef OPENSSL_EXTRA
7579 if (!in && !sz)
7580 /* store AAD size for next call */
7581 aes->aadLen = authInSz;
7582 #endif
7583 }
7584
7585 return ret;
7586 }
7587
7588 /* Software AES - GCM Encrypt */
wc_AesGcmEncrypt(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)7589 int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
7590 const byte* iv, word32 ivSz,
7591 byte* authTag, word32 authTagSz,
7592 const byte* authIn, word32 authInSz)
7593 {
7594 /* argument checks */
7595 if (aes == NULL || authTagSz > AES_BLOCK_SIZE || ivSz == 0) {
7596 return BAD_FUNC_ARG;
7597 }
7598
7599 if (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
7600 WOLFSSL_MSG("GcmEncrypt authTagSz too small error");
7601 return BAD_FUNC_ARG;
7602 }
7603
7604 #ifdef WOLF_CRYPTO_CB
7605 if (aes->devId != INVALID_DEVID) {
7606 int ret = wc_CryptoCb_AesGcmEncrypt(aes, out, in, sz, iv, ivSz,
7607 authTag, authTagSz, authIn, authInSz);
7608 if (ret != CRYPTOCB_UNAVAILABLE)
7609 return ret;
7610 /* fall-through when unavailable */
7611 }
7612 #endif
7613
7614 #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
7615 /* if async and byte count above threshold */
7616 /* only 12-byte IV is supported in HW */
7617 if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
7618 sz >= WC_ASYNC_THRESH_AES_GCM && ivSz == GCM_NONCE_MID_SZ) {
7619 #if defined(HAVE_CAVIUM)
7620 #ifdef HAVE_CAVIUM_V
7621 if (authInSz == 20) { /* Nitrox V GCM is only working with 20 byte AAD */
7622 return NitroxAesGcmEncrypt(aes, out, in, sz,
7623 (const byte*)aes->devKey, aes->keylen, iv, ivSz,
7624 authTag, authTagSz, authIn, authInSz);
7625 }
7626 #endif
7627 #elif defined(HAVE_INTEL_QA)
7628 return IntelQaSymAesGcmEncrypt(&aes->asyncDev, out, in, sz,
7629 (const byte*)aes->devKey, aes->keylen, iv, ivSz,
7630 authTag, authTagSz, authIn, authInSz);
7631 #else /* WOLFSSL_ASYNC_CRYPT_TEST */
7632 if (wc_AsyncTestInit(&aes->asyncDev, ASYNC_TEST_AES_GCM_ENCRYPT)) {
7633 WC_ASYNC_TEST* testDev = &aes->asyncDev.test;
7634 testDev->aes.aes = aes;
7635 testDev->aes.out = out;
7636 testDev->aes.in = in;
7637 testDev->aes.sz = sz;
7638 testDev->aes.iv = iv;
7639 testDev->aes.ivSz = ivSz;
7640 testDev->aes.authTag = authTag;
7641 testDev->aes.authTagSz = authTagSz;
7642 testDev->aes.authIn = authIn;
7643 testDev->aes.authInSz = authInSz;
7644 return WC_PENDING_E;
7645 }
7646 #endif
7647 }
7648 #endif /* WOLFSSL_ASYNC_CRYPT */
7649
7650 #ifdef WOLFSSL_SILABS_SE_ACCEL
7651 return wc_AesGcmEncrypt_silabs(
7652 aes, out, in, sz,
7653 iv, ivSz,
7654 authTag, authTagSz,
7655 authIn, authInSz);
7656 #endif
7657
7658 #ifdef STM32_CRYPTO_AES_GCM
7659 return wc_AesGcmEncrypt_STM32(
7660 aes, out, in, sz, iv, ivSz,
7661 authTag, authTagSz, authIn, authInSz);
7662 #endif /* STM32_CRYPTO_AES_GCM */
7663
7664 #ifdef WOLFSSL_AESNI
7665 #ifdef HAVE_INTEL_AVX2
7666 if (IS_INTEL_AVX2(intel_flags)) {
7667 SAVE_VECTOR_REGISTERS(return _svr_ret;);
7668 AES_GCM_encrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
7669 authTagSz, (const byte*)aes->key, aes->rounds);
7670 RESTORE_VECTOR_REGISTERS();
7671 return 0;
7672 }
7673 else
7674 #endif
7675 #ifdef HAVE_INTEL_AVX1
7676 if (IS_INTEL_AVX1(intel_flags)) {
7677 SAVE_VECTOR_REGISTERS(return _svr_ret;);
7678 AES_GCM_encrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
7679 authTagSz, (const byte*)aes->key, aes->rounds);
7680 RESTORE_VECTOR_REGISTERS();
7681 return 0;
7682 }
7683 else
7684 #endif
7685 if (haveAESNI) {
7686 AES_GCM_encrypt(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
7687 authTagSz, (const byte*)aes->key, aes->rounds);
7688 return 0;
7689 }
7690 else
7691 #endif
7692 {
7693 return AES_GCM_encrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
7694 authIn, authInSz);
7695 }
7696 }
7697 #endif
7698
7699
7700 /* AES GCM Decrypt */
7701 #if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
7702 #ifdef FREESCALE_LTC_AES_GCM
wc_AesGcmDecrypt(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)7703 int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
7704 const byte* iv, word32 ivSz,
7705 const byte* authTag, word32 authTagSz,
7706 const byte* authIn, word32 authInSz)
7707 {
7708 int ret;
7709 word32 keySize;
7710 status_t status;
7711
7712 /* argument checks */
7713 /* If the sz is non-zero, both in and out must be set. If sz is 0,
7714 * in and out are don't cares, as this is is the GMAC case. */
7715 if (aes == NULL || iv == NULL || (sz != 0 && (in == NULL || out == NULL)) ||
7716 authTag == NULL || authTagSz > AES_BLOCK_SIZE || authTagSz == 0 ||
7717 ivSz == 0) {
7718
7719 return BAD_FUNC_ARG;
7720 }
7721
7722 ret = wc_AesGetKeySize(aes, &keySize);
7723 if (ret != 0) {
7724 return ret;
7725 }
7726
7727 status = wolfSSL_CryptHwMutexLock();
7728 if (status != 0)
7729 return status;
7730
7731 status = LTC_AES_DecryptTagGcm(LTC_BASE, in, out, sz, iv, ivSz,
7732 authIn, authInSz, (byte*)aes->key, keySize, authTag, authTagSz);
7733 wolfSSL_CryptHwMutexUnLock();
7734
7735 return (status == kStatus_Success) ? 0 : AES_GCM_AUTH_E;
7736 }
7737
7738 #else
7739
7740 #ifdef STM32_CRYPTO_AES_GCM
7741 /* this function supports inline decrypt */
wc_AesGcmDecrypt_STM32(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)7742 static int wc_AesGcmDecrypt_STM32(Aes* aes, byte* out,
7743 const byte* in, word32 sz,
7744 const byte* iv, word32 ivSz,
7745 const byte* authTag, word32 authTagSz,
7746 const byte* authIn, word32 authInSz)
7747 {
7748 int ret;
7749 #ifdef WOLFSSL_STM32_CUBEMX
7750 int status = HAL_OK;
7751 CRYP_HandleTypeDef hcryp;
7752 word32 blocks = sz / AES_BLOCK_SIZE;
7753 #else
7754 int status = SUCCESS;
7755 word32 keyCopy[AES_256_KEY_SIZE/sizeof(word32)];
7756 #endif
7757 word32 keySize;
7758 word32 partial = sz % AES_BLOCK_SIZE;
7759 word32 tag[AES_BLOCK_SIZE/sizeof(word32)];
7760 word32 tagExpected[AES_BLOCK_SIZE/sizeof(word32)];
7761 word32 partialBlock[AES_BLOCK_SIZE/sizeof(word32)];
7762 word32 ctr[AES_BLOCK_SIZE/sizeof(word32)];
7763 word32 authhdr[AES_BLOCK_SIZE/sizeof(word32)];
7764 byte* authInPadded = NULL;
7765 int authPadSz, wasAlloc = 0, tagComputed = 0;
7766
7767 ret = wc_AesGetKeySize(aes, &keySize);
7768 if (ret != 0)
7769 return ret;
7770
7771 #ifdef WOLFSSL_STM32_CUBEMX
7772 ret = wc_Stm32_Aes_Init(aes, &hcryp);
7773 if (ret != 0)
7774 return ret;
7775 #endif
7776
7777 XMEMSET(ctr, 0, AES_BLOCK_SIZE);
7778 if (ivSz == GCM_NONCE_MID_SZ) {
7779 byte* pCtr = (byte*)ctr;
7780 XMEMCPY(ctr, iv, ivSz);
7781 pCtr[AES_BLOCK_SIZE - 1] = 1;
7782 }
7783 else {
7784 GHASH(aes, NULL, 0, iv, ivSz, (byte*)ctr, AES_BLOCK_SIZE);
7785 }
7786
7787 /* Make copy of expected authTag, which could get corrupted in some
7788 * Cube HAL versions without proper partial block support.
7789 * For TLS blocks the authTag is after the output buffer, so save it */
7790 XMEMCPY(tagExpected, authTag, authTagSz);
7791
7792 /* Authentication buffer - must be 4-byte multiple zero padded */
7793 authPadSz = authInSz % sizeof(word32);
7794 if (authPadSz != 0) {
7795 authPadSz = authInSz + sizeof(word32) - authPadSz;
7796 }
7797 else {
7798 authPadSz = authInSz;
7799 }
7800
7801 /* for cases where hardware cannot be used for authTag calculate it */
7802 /* if IV is not 12 calculate GHASH using software */
7803 if (ivSz != GCM_NONCE_MID_SZ
7804 #ifndef CRYP_HEADERWIDTHUNIT_BYTE
7805 /* or harware that does not support partial block */
7806 || sz == 0 || partial != 0
7807 #endif
7808 #if !defined(CRYP_HEADERWIDTHUNIT_BYTE) && !defined(STM32_AESGCM_PARTIAL)
7809 /* or authIn is not a multiple of 4 */
7810 || authPadSz != authInSz
7811 #endif
7812 ) {
7813 GHASH(aes, authIn, authInSz, in, sz, (byte*)tag, sizeof(tag));
7814 wc_AesEncrypt(aes, (byte*)ctr, (byte*)partialBlock);
7815 xorbuf(tag, partialBlock, sizeof(tag));
7816 tagComputed = 1;
7817 }
7818
7819 /* if using hardware for authentication tag make sure its aligned and zero padded */
7820 if (authPadSz != authInSz && !tagComputed) {
7821 if (authPadSz <= sizeof(authhdr)) {
7822 authInPadded = (byte*)authhdr;
7823 }
7824 else {
7825 authInPadded = (byte*)XMALLOC(authPadSz, aes->heap,
7826 DYNAMIC_TYPE_TMP_BUFFER);
7827 if (authInPadded == NULL) {
7828 wolfSSL_CryptHwMutexUnLock();
7829 return MEMORY_E;
7830 }
7831 wasAlloc = 1;
7832 }
7833 XMEMSET(authInPadded, 0, authPadSz);
7834 XMEMCPY(authInPadded, authIn, authInSz);
7835 } else {
7836 authInPadded = (byte*)authIn;
7837 }
7838
7839 /* Hardware requires counter + 1 */
7840 IncrementGcmCounter((byte*)ctr);
7841
7842 ret = wolfSSL_CryptHwMutexLock();
7843 if (ret != 0) {
7844 return ret;
7845 }
7846 #ifdef WOLFSSL_STM32_CUBEMX
7847 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)ctr;
7848 hcryp.Init.Header = (STM_CRYPT_TYPE*)authInPadded;
7849
7850 #if defined(STM32_HAL_V2)
7851 hcryp.Init.Algorithm = CRYP_AES_GCM;
7852 #ifdef CRYP_HEADERWIDTHUNIT_BYTE
7853 /* V2 with CRYP_HEADERWIDTHUNIT_BYTE uses byte size for header */
7854 hcryp.Init.HeaderSize = authInSz;
7855 #else
7856 hcryp.Init.HeaderSize = authPadSz/sizeof(word32);
7857 #endif
7858 #ifdef STM32_AESGCM_PARTIAL
7859 hcryp.Init.HeaderPadSize = authPadSz - authInSz;
7860 #endif
7861 #ifdef CRYP_KEYIVCONFIG_ONCE
7862 /* allows repeated calls to HAL_CRYP_Decrypt */
7863 hcryp.Init.KeyIVConfigSkip = CRYP_KEYIVCONFIG_ONCE;
7864 #endif
7865 ByteReverseWords(ctr, ctr, AES_BLOCK_SIZE);
7866 hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)ctr;
7867 HAL_CRYP_Init(&hcryp);
7868
7869 #ifndef CRYP_KEYIVCONFIG_ONCE
7870 status = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)in,
7871 (blocks * AES_BLOCK_SIZE) + partial, (uint32_t*)out, STM32_HAL_TIMEOUT);
7872 #else
7873 /* GCM payload phase - blocks */
7874 if (blocks) {
7875 status = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)in,
7876 (blocks * AES_BLOCK_SIZE), (uint32_t*)out, STM32_HAL_TIMEOUT);
7877 }
7878 /* GCM payload phase - partial remainder */
7879 if (status == HAL_OK && (partial != 0 || blocks == 0)) {
7880 XMEMSET(partialBlock, 0, sizeof(partialBlock));
7881 XMEMCPY(partialBlock, in + (blocks * AES_BLOCK_SIZE), partial);
7882 status = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)partialBlock, partial,
7883 ( uint32_t*)partialBlock, STM32_HAL_TIMEOUT);
7884 XMEMCPY(out + (blocks * AES_BLOCK_SIZE), partialBlock, partial);
7885 }
7886 #endif
7887 if (status == HAL_OK && !tagComputed) {
7888 /* Compute the authTag */
7889 status = HAL_CRYPEx_AESGCM_GenerateAuthTAG(&hcryp, (uint32_t*)tag,
7890 STM32_HAL_TIMEOUT);
7891 }
7892 #elif defined(STM32_CRYPTO_AES_ONLY)
7893 /* Set the CRYP parameters */
7894 hcryp.Init.HeaderSize = authPadSz;
7895 if (authPadSz == 0)
7896 hcryp.Init.Header = NULL; /* cannot pass pointer when authIn == 0 */
7897 hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_GCM_GMAC;
7898 hcryp.Init.OperatingMode = CRYP_ALGOMODE_DECRYPT;
7899 hcryp.Init.GCMCMACPhase = CRYP_INIT_PHASE;
7900 HAL_CRYP_Init(&hcryp);
7901
7902 /* GCM init phase */
7903 status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
7904 if (status == HAL_OK) {
7905 /* GCM header phase */
7906 hcryp.Init.GCMCMACPhase = CRYP_HEADER_PHASE;
7907 status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
7908 }
7909 if (status == HAL_OK) {
7910 /* GCM payload phase - blocks */
7911 hcryp.Init.GCMCMACPhase = CRYP_PAYLOAD_PHASE;
7912 if (blocks) {
7913 status = HAL_CRYPEx_AES_Auth(&hcryp, (byte*)in,
7914 (blocks * AES_BLOCK_SIZE), out, STM32_HAL_TIMEOUT);
7915 }
7916 }
7917 if (status == HAL_OK && (partial != 0 || (sz > 0 && blocks == 0))) {
7918 /* GCM payload phase - partial remainder */
7919 XMEMSET(partialBlock, 0, sizeof(partialBlock));
7920 XMEMCPY(partialBlock, in + (blocks * AES_BLOCK_SIZE), partial);
7921 status = HAL_CRYPEx_AES_Auth(&hcryp, (byte*)partialBlock, partial,
7922 (byte*)partialBlock, STM32_HAL_TIMEOUT);
7923 XMEMCPY(out + (blocks * AES_BLOCK_SIZE), partialBlock, partial);
7924 }
7925 if (status == HAL_OK && tagComputed == 0) {
7926 /* GCM final phase */
7927 hcryp.Init.GCMCMACPhase = CRYP_FINAL_PHASE;
7928 status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, sz, (byte*)tag, STM32_HAL_TIMEOUT);
7929 }
7930 #else
7931 hcryp.Init.HeaderSize = authPadSz;
7932 HAL_CRYP_Init(&hcryp);
7933 if (blocks) {
7934 /* GCM payload phase - blocks */
7935 status = HAL_CRYPEx_AESGCM_Decrypt(&hcryp, (byte*)in,
7936 (blocks * AES_BLOCK_SIZE), out, STM32_HAL_TIMEOUT);
7937 }
7938 if (status == HAL_OK && (partial != 0 || blocks == 0)) {
7939 /* GCM payload phase - partial remainder */
7940 XMEMSET(partialBlock, 0, sizeof(partialBlock));
7941 XMEMCPY(partialBlock, in + (blocks * AES_BLOCK_SIZE), partial);
7942 status = HAL_CRYPEx_AESGCM_Decrypt(&hcryp, (byte*)partialBlock, partial,
7943 (byte*)partialBlock, STM32_HAL_TIMEOUT);
7944 XMEMCPY(out + (blocks * AES_BLOCK_SIZE), partialBlock, partial);
7945 }
7946 if (status == HAL_OK && tagComputed == 0) {
7947 /* Compute the authTag */
7948 status = HAL_CRYPEx_AESGCM_Finish(&hcryp, sz, (byte*)tag, STM32_HAL_TIMEOUT);
7949 }
7950 #endif
7951
7952 if (status != HAL_OK)
7953 ret = AES_GCM_AUTH_E;
7954
7955 HAL_CRYP_DeInit(&hcryp);
7956
7957 #else /* Standard Peripheral Library */
7958 ByteReverseWords(keyCopy, (word32*)aes->key, aes->keylen);
7959
7960 /* Input size and auth size need to be the actual sizes, even though
7961 * they are not block aligned, because this length (in bits) is used
7962 * in the final GHASH. */
7963 XMEMSET(partialBlock, 0, sizeof(partialBlock)); /* use this to get tag */
7964 status = CRYP_AES_GCM(MODE_DECRYPT, (uint8_t*)ctr,
7965 (uint8_t*)keyCopy, keySize * 8,
7966 (uint8_t*)in, sz,
7967 (uint8_t*)authInPadded, authInSz,
7968 (uint8_t*)out, (uint8_t*)partialBlock);
7969 if (status != SUCCESS)
7970 ret = AES_GCM_AUTH_E;
7971 if (tagComputed == 0)
7972 XMEMCPY(tag, partialBlock, authTagSz);
7973 #endif /* WOLFSSL_STM32_CUBEMX */
7974 wolfSSL_CryptHwMutexUnLock();
7975
7976 /* Check authentication tag */
7977 if (ConstantCompare((const byte*)tagExpected, (byte*)tag, authTagSz) != 0) {
7978 ret = AES_GCM_AUTH_E;
7979 }
7980
7981 /* Free memory */
7982 if (wasAlloc) {
7983 XFREE(authInPadded, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
7984 }
7985
7986 return ret;
7987 }
7988
7989 #endif /* STM32_CRYPTO_AES_GCM */
7990
7991 #ifdef WOLFSSL_AESNI
7992 /* For performance reasons, this code needs to be not inlined. */
7993 int AES_GCM_decrypt_C(Aes* aes, byte* out, const byte* in, word32 sz,
7994 const byte* iv, word32 ivSz,
7995 const byte* authTag, word32 authTagSz,
7996 const byte* authIn, word32 authInSz);
7997 #else
7998 static
7999 #endif
AES_GCM_decrypt_C(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)8000 int AES_GCM_decrypt_C(Aes* aes, byte* out, const byte* in, word32 sz,
8001 const byte* iv, word32 ivSz,
8002 const byte* authTag, word32 authTagSz,
8003 const byte* authIn, word32 authInSz)
8004 {
8005 int ret = 0;
8006 word32 blocks = sz / AES_BLOCK_SIZE;
8007 word32 partial = sz % AES_BLOCK_SIZE;
8008 const byte* c = in;
8009 byte* p = out;
8010 ALIGN32 byte counter[AES_BLOCK_SIZE];
8011 ALIGN32 byte scratch[AES_BLOCK_SIZE];
8012 ALIGN32 byte Tprime[AES_BLOCK_SIZE];
8013 ALIGN32 byte EKY0[AES_BLOCK_SIZE];
8014
8015 if (ivSz == GCM_NONCE_MID_SZ) {
8016 /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */
8017 XMEMCPY(counter, iv, ivSz);
8018 XMEMSET(counter + GCM_NONCE_MID_SZ, 0,
8019 AES_BLOCK_SIZE - GCM_NONCE_MID_SZ - 1);
8020 counter[AES_BLOCK_SIZE - 1] = 1;
8021 }
8022 else {
8023 /* Counter is GHASH of IV. */
8024 #ifdef OPENSSL_EXTRA
8025 word32 aadTemp = aes->aadLen;
8026 aes->aadLen = 0;
8027 #endif
8028 GHASH(aes, NULL, 0, iv, ivSz, counter, AES_BLOCK_SIZE);
8029 #ifdef OPENSSL_EXTRA
8030 aes->aadLen = aadTemp;
8031 #endif
8032 }
8033
8034 /* Calc the authTag again using received auth data and the cipher text */
8035 GHASH(aes, authIn, authInSz, in, sz, Tprime, sizeof(Tprime));
8036 wc_AesEncrypt(aes, counter, EKY0);
8037 xorbuf(Tprime, EKY0, sizeof(Tprime));
8038
8039 #ifdef OPENSSL_EXTRA
8040 if (!out) {
8041 /* authenticated, non-confidential data */
8042 /* store AAD size for next call */
8043 aes->aadLen = authInSz;
8044 }
8045 #endif
8046 if (ConstantCompare(authTag, Tprime, authTagSz) != 0) {
8047 return AES_GCM_AUTH_E;
8048 }
8049
8050 #if defined(WOLFSSL_PIC32MZ_CRYPT)
8051 if (blocks) {
8052 /* use initial IV for HW, but don't use it below */
8053 XMEMCPY(aes->reg, counter, AES_BLOCK_SIZE);
8054
8055 ret = wc_Pic32AesCrypt(
8056 aes->key, aes->keylen, aes->reg, AES_BLOCK_SIZE,
8057 out, in, (blocks * AES_BLOCK_SIZE),
8058 PIC32_DECRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_AES_GCM);
8059 if (ret != 0)
8060 return ret;
8061 }
8062 /* process remainder using partial handling */
8063 #endif
8064
8065 #if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT)
8066 /* some hardware acceleration can gain performance from doing AES encryption
8067 * of the whole buffer at once */
8068 if (c != p && blocks > 0) { /* can not handle inline decryption */
8069 while (blocks--) {
8070 IncrementGcmCounter(counter);
8071 XMEMCPY(p, counter, AES_BLOCK_SIZE);
8072 p += AES_BLOCK_SIZE;
8073 }
8074
8075 /* reset number of blocks and then do encryption */
8076 blocks = sz / AES_BLOCK_SIZE;
8077
8078 wc_AesEcbEncrypt(aes, out, out, AES_BLOCK_SIZE * blocks);
8079 xorbuf(out, c, AES_BLOCK_SIZE * blocks);
8080 c += AES_BLOCK_SIZE * blocks;
8081 }
8082 else
8083 #endif /* HAVE_AES_ECB && !PIC32MZ */
8084 {
8085 while (blocks--) {
8086 IncrementGcmCounter(counter);
8087 #if !defined(WOLFSSL_PIC32MZ_CRYPT)
8088 wc_AesEncrypt(aes, counter, scratch);
8089 xorbufout(p, scratch, c, AES_BLOCK_SIZE);
8090 #endif
8091 p += AES_BLOCK_SIZE;
8092 c += AES_BLOCK_SIZE;
8093 }
8094 }
8095
8096 if (partial != 0) {
8097 IncrementGcmCounter(counter);
8098 wc_AesEncrypt(aes, counter, scratch);
8099 xorbuf(scratch, c, partial);
8100 XMEMCPY(p, scratch, partial);
8101 }
8102
8103 return ret;
8104 }
8105
8106 /* Software AES - GCM Decrypt */
wc_AesGcmDecrypt(Aes * aes,byte * out,const byte * in,word32 sz,const byte * iv,word32 ivSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)8107 int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
8108 const byte* iv, word32 ivSz,
8109 const byte* authTag, word32 authTagSz,
8110 const byte* authIn, word32 authInSz)
8111 {
8112 #ifdef WOLFSSL_AESNI
8113 int res = AES_GCM_AUTH_E;
8114 #endif
8115
8116 /* argument checks */
8117 /* If the sz is non-zero, both in and out must be set. If sz is 0,
8118 * in and out are don't cares, as this is is the GMAC case. */
8119 if (aes == NULL || iv == NULL || (sz != 0 && (in == NULL || out == NULL)) ||
8120 authTag == NULL || authTagSz > AES_BLOCK_SIZE || authTagSz == 0 ||
8121 ivSz == 0) {
8122
8123 return BAD_FUNC_ARG;
8124 }
8125
8126 #ifdef WOLF_CRYPTO_CB
8127 if (aes->devId != INVALID_DEVID) {
8128 int ret = wc_CryptoCb_AesGcmDecrypt(aes, out, in, sz, iv, ivSz,
8129 authTag, authTagSz, authIn, authInSz);
8130 if (ret != CRYPTOCB_UNAVAILABLE)
8131 return ret;
8132 /* fall-through when unavailable */
8133 }
8134 #endif
8135
8136 #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
8137 /* if async and byte count above threshold */
8138 /* only 12-byte IV is supported in HW */
8139 if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
8140 sz >= WC_ASYNC_THRESH_AES_GCM && ivSz == GCM_NONCE_MID_SZ) {
8141 #if defined(HAVE_CAVIUM)
8142 #ifdef HAVE_CAVIUM_V
8143 if (authInSz == 20) { /* Nitrox V GCM is only working with 20 byte AAD */
8144 return NitroxAesGcmDecrypt(aes, out, in, sz,
8145 (const byte*)aes->devKey, aes->keylen, iv, ivSz,
8146 authTag, authTagSz, authIn, authInSz);
8147 }
8148 #endif
8149 #elif defined(HAVE_INTEL_QA)
8150 return IntelQaSymAesGcmDecrypt(&aes->asyncDev, out, in, sz,
8151 (const byte*)aes->devKey, aes->keylen, iv, ivSz,
8152 authTag, authTagSz, authIn, authInSz);
8153 #else /* WOLFSSL_ASYNC_CRYPT_TEST */
8154 if (wc_AsyncTestInit(&aes->asyncDev, ASYNC_TEST_AES_GCM_DECRYPT)) {
8155 WC_ASYNC_TEST* testDev = &aes->asyncDev.test;
8156 testDev->aes.aes = aes;
8157 testDev->aes.out = out;
8158 testDev->aes.in = in;
8159 testDev->aes.sz = sz;
8160 testDev->aes.iv = iv;
8161 testDev->aes.ivSz = ivSz;
8162 testDev->aes.authTag = (byte*)authTag;
8163 testDev->aes.authTagSz = authTagSz;
8164 testDev->aes.authIn = authIn;
8165 testDev->aes.authInSz = authInSz;
8166 return WC_PENDING_E;
8167 }
8168 #endif
8169 }
8170 #endif /* WOLFSSL_ASYNC_CRYPT */
8171
8172 #ifdef WOLFSSL_SILABS_SE_ACCEL
8173 return wc_AesGcmDecrypt_silabs(
8174 aes, out, in, sz, iv, ivSz,
8175 authTag, authTagSz, authIn, authInSz);
8176
8177 #endif
8178
8179 #ifdef STM32_CRYPTO_AES_GCM
8180 /* The STM standard peripheral library API's doesn't support partial blocks */
8181 return wc_AesGcmDecrypt_STM32(
8182 aes, out, in, sz, iv, ivSz,
8183 authTag, authTagSz, authIn, authInSz);
8184 #endif /* STM32_CRYPTO_AES_GCM */
8185
8186 #ifdef WOLFSSL_AESNI
8187 #ifdef HAVE_INTEL_AVX2
8188 if (IS_INTEL_AVX2(intel_flags)) {
8189 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8190 AES_GCM_decrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
8191 authTagSz, (byte*)aes->key, aes->rounds, &res);
8192 RESTORE_VECTOR_REGISTERS();
8193 if (res == 0)
8194 return AES_GCM_AUTH_E;
8195 return 0;
8196 }
8197 else
8198 #endif
8199 #ifdef HAVE_INTEL_AVX1
8200 if (IS_INTEL_AVX1(intel_flags)) {
8201 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8202 AES_GCM_decrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
8203 authTagSz, (byte*)aes->key, aes->rounds, &res);
8204 RESTORE_VECTOR_REGISTERS();
8205 if (res == 0)
8206 return AES_GCM_AUTH_E;
8207 return 0;
8208 }
8209 else
8210 #endif
8211 if (haveAESNI) {
8212 AES_GCM_decrypt(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
8213 authTagSz, (byte*)aes->key, aes->rounds, &res);
8214 if (res == 0)
8215 return AES_GCM_AUTH_E;
8216 return 0;
8217 }
8218 else
8219 #endif
8220 {
8221 return AES_GCM_decrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
8222 authIn, authInSz);
8223 }
8224 }
8225 #endif
8226 #endif /* HAVE_AES_DECRYPT || HAVE_AESGCM_DECRYPT */
8227
8228 #ifdef WOLFSSL_AESGCM_STREAM
8229 /* Initialize the AES GCM cipher with an IV. C implementation.
8230 *
8231 * @param [in, out] aes AES object.
8232 * @param [in] iv IV/nonce buffer.
8233 * @param [in] ivSz Length of IV/nonce data.
8234 */
AesGcmInit_C(Aes * aes,const byte * iv,word32 ivSz)8235 static void AesGcmInit_C(Aes* aes, const byte* iv, word32 ivSz)
8236 {
8237 ALIGN32 byte counter[AES_BLOCK_SIZE];
8238
8239 if (ivSz == GCM_NONCE_MID_SZ) {
8240 /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */
8241 XMEMCPY(counter, iv, ivSz);
8242 XMEMSET(counter + GCM_NONCE_MID_SZ, 0,
8243 AES_BLOCK_SIZE - GCM_NONCE_MID_SZ - 1);
8244 counter[AES_BLOCK_SIZE - 1] = 1;
8245 }
8246 else {
8247 /* Counter is GHASH of IV. */
8248 #ifdef OPENSSL_EXTRA
8249 word32 aadTemp = aes->aadLen;
8250 aes->aadLen = 0;
8251 #endif
8252 GHASH(aes, NULL, 0, iv, ivSz, counter, AES_BLOCK_SIZE);
8253 #ifdef OPENSSL_EXTRA
8254 aes->aadLen = aadTemp;
8255 #endif
8256 }
8257
8258 /* Copy in the counter for use with cipher. */
8259 XMEMCPY(AES_COUNTER(aes), counter, AES_BLOCK_SIZE);
8260 /* Encrypt initial counter into a buffer for GCM. */
8261 wc_AesEncrypt(aes, counter, AES_INITCTR(aes));
8262 /* Reset state fields. */
8263 aes->over = 0;
8264 aes->aSz = 0;
8265 aes->cSz = 0;
8266 /* Initialization for GHASH. */
8267 GHASH_INIT(aes);
8268 }
8269
8270 /* Update the AES GCM cipher with data. C implementation.
8271 *
8272 * Only enciphers data.
8273 *
8274 * @param [in, out] aes AES object.
8275 * @param [in] out Cipher text or plaintext buffer.
8276 * @param [in] in Plaintext or cipher text buffer.
8277 * @param [in] sz Length of data.
8278 */
AesGcmCryptUpdate_C(Aes * aes,byte * out,const byte * in,word32 sz)8279 static void AesGcmCryptUpdate_C(Aes* aes, byte* out, const byte* in, word32 sz)
8280 {
8281 word32 blocks;
8282 word32 partial;
8283
8284 /* Check if previous encrypted block was not used up. */
8285 if (aes->over > 0) {
8286 byte pSz = AES_BLOCK_SIZE - aes->over;
8287 if (pSz > sz) pSz = sz;
8288
8289 /* Use some/all of last encrypted block. */
8290 xorbufout(out, AES_LASTBLOCK(aes) + aes->over, in, pSz);
8291 aes->over = (aes->over + pSz) & (AES_BLOCK_SIZE - 1);
8292
8293 /* Some data used. */
8294 sz -= pSz;
8295 in += pSz;
8296 out += pSz;
8297 }
8298
8299 /* Calculate the number of blocks needing to be encrypted and any leftover.
8300 */
8301 blocks = sz / AES_BLOCK_SIZE;
8302 partial = sz & (AES_BLOCK_SIZE - 1);
8303
8304 #if defined(HAVE_AES_ECB)
8305 /* Some hardware acceleration can gain performance from doing AES encryption
8306 * of the whole buffer at once.
8307 * Overwrites the cipher text before using plaintext - no inline encryption.
8308 */
8309 if ((out != in) && blocks > 0) {
8310 word32 b;
8311 /* Place incrementing counter blocks into cipher text. */
8312 for (b = 0; b < blocks; b++) {
8313 IncrementGcmCounter(AES_COUNTER(aes));
8314 XMEMCPY(out + b * AES_BLOCK_SIZE, AES_COUNTER(aes), AES_BLOCK_SIZE);
8315 }
8316
8317 /* Encrypt counter blocks. */
8318 wc_AesEcbEncrypt(aes, out, out, AES_BLOCK_SIZE * blocks);
8319 /* XOR in plaintext. */
8320 xorbuf(out, in, AES_BLOCK_SIZE * blocks);
8321 /* Skip over processed data. */
8322 in += AES_BLOCK_SIZE * blocks;
8323 out += AES_BLOCK_SIZE * blocks;
8324 }
8325 else
8326 #endif /* HAVE_AES_ECB */
8327 {
8328 /* Encrypt block by block. */
8329 while (blocks--) {
8330 ALIGN32 byte scratch[AES_BLOCK_SIZE];
8331 IncrementGcmCounter(AES_COUNTER(aes));
8332 /* Encrypt counter into a buffer. */
8333 wc_AesEncrypt(aes, AES_COUNTER(aes), scratch);
8334 /* XOR plain text into encrypted counter into cipher text buffer. */
8335 xorbufout(out, scratch, in, AES_BLOCK_SIZE);
8336 /* Data complete. */
8337 in += AES_BLOCK_SIZE;
8338 out += AES_BLOCK_SIZE;
8339 }
8340 }
8341
8342 if (partial != 0) {
8343 /* Generate an extra block and use up as much as needed. */
8344 IncrementGcmCounter(AES_COUNTER(aes));
8345 /* Encrypt counter into cache. */
8346 wc_AesEncrypt(aes, AES_COUNTER(aes), AES_LASTBLOCK(aes));
8347 /* XOR plain text into encrypted counter into cipher text buffer. */
8348 xorbufout(out, AES_LASTBLOCK(aes), in, partial);
8349 /* Keep amount of encrypted block used. */
8350 aes->over = partial;
8351 }
8352 }
8353
8354 /* Calculates authentication tag for AES GCM. C implementation.
8355 *
8356 * @param [in, out] aes AES object.
8357 * @param [out] authTag Buffer to store authentication tag in.
8358 * @param [in] authTagSz Length of tag to create.
8359 */
AesGcmFinal_C(Aes * aes,byte * authTag,word32 authTagSz)8360 static void AesGcmFinal_C(Aes* aes, byte* authTag, word32 authTagSz)
8361 {
8362 /* Calculate authentication tag. */
8363 GHASH_FINAL(aes, authTag, authTagSz);
8364 /* XOR in as much of encrypted counter as is required. */
8365 xorbuf(authTag, AES_INITCTR(aes), authTagSz);
8366 #ifdef OPENSSL_EXTRA
8367 /* store AAD size for next call */
8368 aes->aadLen = aes->aSz;
8369 #endif
8370 /* Zeroize last block to protect sensitive data. */
8371 ForceZero(AES_LASTBLOCK(aes), AES_BLOCK_SIZE);
8372 }
8373
8374 #ifdef WOLFSSL_AESNI
8375
8376 #ifdef __cplusplus
8377 extern "C" {
8378 #endif
8379
8380 /* Assembly code implementations in: aes_gcm_asm.S */
8381 #ifdef HAVE_INTEL_AVX2
8382 extern void AES_GCM_init_avx2(const unsigned char* key, int nr,
8383 const unsigned char* ivec, unsigned int ibytes, unsigned char* h,
8384 unsigned char* counter, unsigned char* initCtr);
8385 extern void AES_GCM_aad_update_avx2(const unsigned char* addt,
8386 unsigned int abytes, unsigned char* tag, unsigned char* h);
8387 extern void AES_GCM_encrypt_block_avx2(const unsigned char* key, int nr,
8388 unsigned char* out, const unsigned char* in, unsigned char* counter);
8389 extern void AES_GCM_ghash_block_avx2(const unsigned char* data,
8390 unsigned char* tag, unsigned char* h);
8391
8392 extern void AES_GCM_encrypt_update_avx2(const unsigned char* key, int nr,
8393 unsigned char* out, const unsigned char* in, unsigned int nbytes,
8394 unsigned char* tag, unsigned char* h, unsigned char* counter);
8395 extern void AES_GCM_encrypt_final_avx2(unsigned char* tag,
8396 unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
8397 unsigned int abytes, unsigned char* h, unsigned char* initCtr);
8398 #endif
8399 #ifdef HAVE_INTEL_AVX1
8400 extern void AES_GCM_init_avx1(const unsigned char* key, int nr,
8401 const unsigned char* ivec, unsigned int ibytes, unsigned char* h,
8402 unsigned char* counter, unsigned char* initCtr);
8403 extern void AES_GCM_aad_update_avx1(const unsigned char* addt,
8404 unsigned int abytes, unsigned char* tag, unsigned char* h);
8405 extern void AES_GCM_encrypt_block_avx1(const unsigned char* key, int nr,
8406 unsigned char* out, const unsigned char* in, unsigned char* counter);
8407 extern void AES_GCM_ghash_block_avx1(const unsigned char* data,
8408 unsigned char* tag, unsigned char* h);
8409
8410 extern void AES_GCM_encrypt_update_avx1(const unsigned char* key, int nr,
8411 unsigned char* out, const unsigned char* in, unsigned int nbytes,
8412 unsigned char* tag, unsigned char* h, unsigned char* counter);
8413 extern void AES_GCM_encrypt_final_avx1(unsigned char* tag,
8414 unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
8415 unsigned int abytes, unsigned char* h, unsigned char* initCtr);
8416 #endif
8417 extern void AES_GCM_init_aesni(const unsigned char* key, int nr,
8418 const unsigned char* ivec, unsigned int ibytes, unsigned char* h,
8419 unsigned char* counter, unsigned char* initCtr);
8420 extern void AES_GCM_aad_update_aesni(const unsigned char* addt,
8421 unsigned int abytes, unsigned char* tag, unsigned char* h);
8422 extern void AES_GCM_encrypt_block_aesni(const unsigned char* key, int nr,
8423 unsigned char* out, const unsigned char* in, unsigned char* counter);
8424 extern void AES_GCM_ghash_block_aesni(const unsigned char* data,
8425 unsigned char* tag, unsigned char* h);
8426
8427 extern void AES_GCM_encrypt_update_aesni(const unsigned char* key, int nr,
8428 unsigned char* out, const unsigned char* in, unsigned int nbytes,
8429 unsigned char* tag, unsigned char* h, unsigned char* counter);
8430 extern void AES_GCM_encrypt_final_aesni(unsigned char* tag,
8431 unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
8432 unsigned int abytes, unsigned char* h, unsigned char* initCtr);
8433
8434 #ifdef __cplusplus
8435 } /* extern "C" */
8436 #endif
8437
8438 /* Initialize the AES GCM cipher with an IV. AES-NI implementations.
8439 *
8440 * @param [in, out] aes AES object.
8441 * @param [in] iv IV/nonce buffer.
8442 * @param [in] ivSz Length of IV/nonce data.
8443 */
AesGcmInit_aesni(Aes * aes,const byte * iv,word32 ivSz)8444 static int AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
8445 {
8446 /* Reset state fields. */
8447 aes->aSz = 0;
8448 aes->cSz = 0;
8449 /* Set tag to all zeros as initial value. */
8450 XMEMSET(AES_TAG(aes), 0, AES_BLOCK_SIZE);
8451 /* Reset counts of AAD and cipher text. */
8452 aes->aOver = 0;
8453 aes->cOver = 0;
8454
8455 #ifdef HAVE_INTEL_AVX2
8456 if (IS_INTEL_AVX2(intel_flags)) {
8457 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8458 AES_GCM_init_avx2((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
8459 AES_COUNTER(aes), AES_INITCTR(aes));
8460 RESTORE_VECTOR_REGISTERS();
8461 }
8462 else
8463 #endif
8464 #ifdef HAVE_INTEL_AVX1
8465 if (IS_INTEL_AVX1(intel_flags)) {
8466 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8467 AES_GCM_init_avx1((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
8468 AES_COUNTER(aes), AES_INITCTR(aes));
8469 RESTORE_VECTOR_REGISTERS();
8470 }
8471 else
8472 #endif
8473 {
8474 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8475 AES_GCM_init_aesni((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
8476 AES_COUNTER(aes), AES_INITCTR(aes));
8477 RESTORE_VECTOR_REGISTERS();
8478 }
8479 return 0;
8480 }
8481
8482 /* Update the AES GCM for encryption with authentication data.
8483 *
8484 * Implementation uses AVX2, AVX1 or straight AES-NI optimized assembly code.
8485 *
8486 * @param [in, out] aes AES object.
8487 * @param [in] a Buffer holding authentication data.
8488 * @param [in] aSz Length of authentication data in bytes.
8489 * @param [in] endA Whether no more authentication data is expected.
8490 */
AesGcmAadUpdate_aesni(Aes * aes,const byte * a,word32 aSz,int endA)8491 static void AesGcmAadUpdate_aesni(Aes* aes, const byte* a, word32 aSz, int endA)
8492 {
8493 word32 blocks;
8494 int partial;
8495
8496 ASSERT_SAVED_VECTOR_REGISTERS();
8497
8498 if (aSz != 0 && a != NULL) {
8499 /* Total count of AAD updated. */
8500 aes->aSz += aSz;
8501 /* Check if we have unprocessed data. */
8502 if (aes->aOver > 0) {
8503 /* Calculate amount we can use - fill up the block. */
8504 byte sz = AES_BLOCK_SIZE - aes->aOver;
8505 if (sz > aSz) {
8506 sz = aSz;
8507 }
8508 /* Copy extra into last GHASH block array and update count. */
8509 XMEMCPY(AES_LASTGBLOCK(aes) + aes->aOver, a, sz);
8510 aes->aOver += sz;
8511 if (aes->aOver == AES_BLOCK_SIZE) {
8512 /* We have filled up the block and can process. */
8513 #ifdef HAVE_INTEL_AVX2
8514 if (IS_INTEL_AVX2(intel_flags)) {
8515 AES_GCM_ghash_block_avx2(AES_LASTGBLOCK(aes), AES_TAG(aes),
8516 aes->H);
8517 }
8518 else
8519 #endif
8520 #ifdef HAVE_INTEL_AVX1
8521 if (IS_INTEL_AVX1(intel_flags)) {
8522 AES_GCM_ghash_block_avx1(AES_LASTGBLOCK(aes), AES_TAG(aes),
8523 aes->H);
8524 }
8525 else
8526 #endif
8527 {
8528 AES_GCM_ghash_block_aesni(AES_LASTGBLOCK(aes), AES_TAG(aes),
8529 aes->H);
8530 }
8531 /* Reset count. */
8532 aes->aOver = 0;
8533 }
8534 /* Used up some data. */
8535 aSz -= sz;
8536 a += sz;
8537 }
8538
8539 /* Calculate number of blocks of AAD and the leftover. */
8540 blocks = aSz / AES_BLOCK_SIZE;
8541 partial = aSz % AES_BLOCK_SIZE;
8542 if (blocks > 0) {
8543 /* GHASH full blocks now. */
8544 #ifdef HAVE_INTEL_AVX2
8545 if (IS_INTEL_AVX2(intel_flags)) {
8546 AES_GCM_aad_update_avx2(a, blocks * AES_BLOCK_SIZE,
8547 AES_TAG(aes), aes->H);
8548 }
8549 else
8550 #endif
8551 #ifdef HAVE_INTEL_AVX1
8552 if (IS_INTEL_AVX1(intel_flags)) {
8553 AES_GCM_aad_update_avx1(a, blocks * AES_BLOCK_SIZE,
8554 AES_TAG(aes), aes->H);
8555 }
8556 else
8557 #endif
8558 {
8559 AES_GCM_aad_update_aesni(a, blocks * AES_BLOCK_SIZE,
8560 AES_TAG(aes), aes->H);
8561 }
8562 /* Skip over to end of AAD blocks. */
8563 a += blocks * AES_BLOCK_SIZE;
8564 }
8565 if (partial != 0) {
8566 /* Cache the partial block. */
8567 XMEMCPY(AES_LASTGBLOCK(aes), a, partial);
8568 aes->aOver = (byte)partial;
8569 }
8570 }
8571 if (endA && (aes->aOver > 0)) {
8572 /* No more AAD coming and we have a partial block. */
8573 /* Fill the rest of the block with zeros. */
8574 XMEMSET(AES_LASTGBLOCK(aes) + aes->aOver, 0,
8575 AES_BLOCK_SIZE - aes->aOver);
8576 /* GHASH last AAD block. */
8577 #ifdef HAVE_INTEL_AVX2
8578 if (IS_INTEL_AVX2(intel_flags)) {
8579 AES_GCM_ghash_block_avx2(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->H);
8580 }
8581 else
8582 #endif
8583 #ifdef HAVE_INTEL_AVX1
8584 if (IS_INTEL_AVX1(intel_flags)) {
8585 AES_GCM_ghash_block_avx1(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->H);
8586 }
8587 else
8588 #endif
8589 {
8590 AES_GCM_ghash_block_aesni(AES_LASTGBLOCK(aes), AES_TAG(aes),
8591 aes->H);
8592 }
8593 /* Clear partial count for next time through. */
8594 aes->aOver = 0;
8595 }
8596 }
8597
8598 /* Update the AES GCM for encryption with data and/or authentication data.
8599 *
8600 * Implementation uses AVX2, AVX1 or straight AES-NI optimized assembly code.
8601 *
8602 * @param [in, out] aes AES object.
8603 * @param [out] c Buffer to hold cipher text.
8604 * @param [in] p Buffer holding plaintext.
8605 * @param [in] cSz Length of cipher text/plaintext in bytes.
8606 * @param [in] a Buffer holding authentication data.
8607 * @param [in] aSz Length of authentication data in bytes.
8608 */
AesGcmEncryptUpdate_aesni(Aes * aes,byte * c,const byte * p,word32 cSz,const byte * a,word32 aSz)8609 static int AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
8610 word32 cSz, const byte* a, word32 aSz)
8611 {
8612 word32 blocks;
8613 int partial;
8614
8615 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8616 /* Hash in A, the Authentication Data */
8617 AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL));
8618
8619 /* Encrypt plaintext and Hash in C, the Cipher text */
8620 if (cSz != 0 && c != NULL) {
8621 /* Update count of cipher text we have hashed. */
8622 aes->cSz += cSz;
8623 if (aes->cOver > 0) {
8624 /* Calculate amount we can use - fill up the block. */
8625 byte sz = AES_BLOCK_SIZE - aes->cOver;
8626 if (sz > cSz) {
8627 sz = cSz;
8628 }
8629 /* Encrypt some of the plaintext. */
8630 xorbuf(AES_LASTGBLOCK(aes) + aes->cOver, p, sz);
8631 XMEMCPY(c, AES_LASTGBLOCK(aes) + aes->cOver, sz);
8632 /* Update count of unsed encrypted counter. */
8633 aes->cOver += sz;
8634 if (aes->cOver == AES_BLOCK_SIZE) {
8635 /* We have filled up the block and can process. */
8636 #ifdef HAVE_INTEL_AVX2
8637 if (IS_INTEL_AVX2(intel_flags)) {
8638 AES_GCM_ghash_block_avx2(AES_LASTGBLOCK(aes), AES_TAG(aes),
8639 aes->H);
8640 }
8641 else
8642 #endif
8643 #ifdef HAVE_INTEL_AVX1
8644 if (IS_INTEL_AVX1(intel_flags)) {
8645 AES_GCM_ghash_block_avx1(AES_LASTGBLOCK(aes), AES_TAG(aes),
8646 aes->H);
8647 }
8648 else
8649 #endif
8650 {
8651 AES_GCM_ghash_block_aesni(AES_LASTGBLOCK(aes), AES_TAG(aes),
8652 aes->H);
8653 }
8654 /* Reset count. */
8655 aes->cOver = 0;
8656 }
8657 /* Used up some data. */
8658 cSz -= sz;
8659 p += sz;
8660 c += sz;
8661 }
8662
8663 /* Calculate number of blocks of plaintext and the leftover. */
8664 blocks = cSz / AES_BLOCK_SIZE;
8665 partial = cSz % AES_BLOCK_SIZE;
8666 if (blocks > 0) {
8667 /* Encrypt and GHASH full blocks now. */
8668 #ifdef HAVE_INTEL_AVX2
8669 if (IS_INTEL_AVX2(intel_flags)) {
8670 AES_GCM_encrypt_update_avx2((byte*)aes->key, aes->rounds, c, p,
8671 blocks * AES_BLOCK_SIZE, AES_TAG(aes), aes->H,
8672 AES_COUNTER(aes));
8673 }
8674 else
8675 #endif
8676 #ifdef HAVE_INTEL_AVX1
8677 if (IS_INTEL_AVX1(intel_flags)) {
8678 AES_GCM_encrypt_update_avx1((byte*)aes->key, aes->rounds, c, p,
8679 blocks * AES_BLOCK_SIZE, AES_TAG(aes), aes->H,
8680 AES_COUNTER(aes));
8681 }
8682 else
8683 #endif
8684 {
8685 AES_GCM_encrypt_update_aesni((byte*)aes->key, aes->rounds, c, p,
8686 blocks * AES_BLOCK_SIZE, AES_TAG(aes), aes->H,
8687 AES_COUNTER(aes));
8688 }
8689 /* Skip over to end of blocks. */
8690 p += blocks * AES_BLOCK_SIZE;
8691 c += blocks * AES_BLOCK_SIZE;
8692 }
8693 if (partial != 0) {
8694 /* Encrypt the counter - XOR in zeros as proxy for plaintext. */
8695 XMEMSET(AES_LASTGBLOCK(aes), 0, AES_BLOCK_SIZE);
8696 #ifdef HAVE_INTEL_AVX2
8697 if (IS_INTEL_AVX2(intel_flags)) {
8698 AES_GCM_encrypt_block_avx2((byte*)aes->key, aes->rounds,
8699 AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
8700 }
8701 else
8702 #endif
8703 #ifdef HAVE_INTEL_AVX1
8704 if (IS_INTEL_AVX1(intel_flags)) {
8705 AES_GCM_encrypt_block_avx1((byte*)aes->key, aes->rounds,
8706 AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
8707 }
8708 else
8709 #endif
8710 {
8711 AES_GCM_encrypt_block_aesni((byte*)aes->key, aes->rounds,
8712 AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
8713 }
8714 /* XOR the remaining plaintext to calculate cipher text.
8715 * Keep cipher text for GHASH of last partial block.
8716 */
8717 xorbuf(AES_LASTGBLOCK(aes), p, partial);
8718 XMEMCPY(c, AES_LASTGBLOCK(aes), partial);
8719 /* Update count of the block used. */
8720 aes->cOver = (byte)partial;
8721 }
8722 }
8723 RESTORE_VECTOR_REGISTERS();
8724 return 0;
8725 }
8726
8727 /* Finalize the AES GCM for encryption and calculate the authentication tag.
8728 *
8729 * Calls AVX2, AVX1 or straight AES-NI optimized assembly code.
8730 *
8731 * @param [in, out] aes AES object.
8732 * @param [in] authTag Buffer to hold authentication tag.
8733 * @param [in] authTagSz Length of authentication tag in bytes.
8734 * @return 0 on success.
8735 */
AesGcmEncryptFinal_aesni(Aes * aes,byte * authTag,word32 authTagSz)8736 static int AesGcmEncryptFinal_aesni(Aes* aes, byte* authTag, word32 authTagSz)
8737 {
8738 /* AAD block incomplete when > 0 */
8739 byte over = aes->aOver;
8740
8741 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8742 if (aes->cOver > 0) {
8743 /* Cipher text block incomplete. */
8744 over = aes->cOver;
8745 }
8746 if (over > 0) {
8747 /* Fill the rest of the block with zeros. */
8748 XMEMSET(AES_LASTGBLOCK(aes) + over, 0, AES_BLOCK_SIZE - over);
8749 /* GHASH last cipher block. */
8750 #ifdef HAVE_INTEL_AVX2
8751 if (IS_INTEL_AVX2(intel_flags)) {
8752 AES_GCM_ghash_block_avx2(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->H);
8753 }
8754 else
8755 #endif
8756 #ifdef HAVE_INTEL_AVX1
8757 if (IS_INTEL_AVX1(intel_flags)) {
8758 AES_GCM_ghash_block_avx1(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->H);
8759 }
8760 else
8761 #endif
8762 {
8763 AES_GCM_ghash_block_aesni(AES_LASTGBLOCK(aes), AES_TAG(aes),
8764 aes->H);
8765 }
8766 }
8767 /* Calculate the authentication tag. */
8768 #ifdef HAVE_INTEL_AVX2
8769 if (IS_INTEL_AVX2(intel_flags)) {
8770 AES_GCM_encrypt_final_avx2(AES_TAG(aes), authTag, authTagSz, aes->cSz,
8771 aes->aSz, aes->H, AES_INITCTR(aes));
8772 }
8773 else
8774 #endif
8775 #ifdef HAVE_INTEL_AVX1
8776 if (IS_INTEL_AVX1(intel_flags)) {
8777 AES_GCM_encrypt_final_avx1(AES_TAG(aes), authTag, authTagSz, aes->cSz,
8778 aes->aSz, aes->H, AES_INITCTR(aes));
8779 }
8780 else
8781 #endif
8782 {
8783 AES_GCM_encrypt_final_aesni(AES_TAG(aes), authTag, authTagSz, aes->cSz,
8784 aes->aSz, aes->H, AES_INITCTR(aes));
8785 }
8786 RESTORE_VECTOR_REGISTERS();
8787 return 0;
8788 }
8789
8790 #if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
8791
8792 #ifdef __cplusplus
8793 extern "C" {
8794 #endif
8795
8796 /* Assembly code implementations in: aes_gcm_asm.S */
8797 #ifdef HAVE_INTEL_AVX2
8798 extern void AES_GCM_decrypt_update_avx2(const unsigned char* key, int nr,
8799 unsigned char* out, const unsigned char* in, unsigned int nbytes,
8800 unsigned char* tag, unsigned char* h, unsigned char* counter);
8801 extern void AES_GCM_decrypt_final_avx2(unsigned char* tag,
8802 const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
8803 unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res);
8804 #endif
8805 #ifdef HAVE_INTEL_AVX1
8806 extern void AES_GCM_decrypt_update_avx1(const unsigned char* key, int nr,
8807 unsigned char* out, const unsigned char* in, unsigned int nbytes,
8808 unsigned char* tag, unsigned char* h, unsigned char* counter);
8809 extern void AES_GCM_decrypt_final_avx1(unsigned char* tag,
8810 const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
8811 unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res);
8812 #endif
8813 extern void AES_GCM_decrypt_update_aesni(const unsigned char* key, int nr,
8814 unsigned char* out, const unsigned char* in, unsigned int nbytes,
8815 unsigned char* tag, unsigned char* h, unsigned char* counter);
8816 extern void AES_GCM_decrypt_final_aesni(unsigned char* tag,
8817 const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
8818 unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res);
8819
8820 #ifdef __cplusplus
8821 } /* extern "C" */
8822 #endif
8823
8824 /* Update the AES GCM for decryption with data and/or authentication data.
8825 *
8826 * @param [in, out] aes AES object.
8827 * @param [out] p Buffer to hold plaintext.
8828 * @param [in] c Buffer holding ciper text.
8829 * @param [in] cSz Length of cipher text/plaintext in bytes.
8830 * @param [in] a Buffer holding authentication data.
8831 * @param [in] aSz Length of authentication data in bytes.
8832 */
AesGcmDecryptUpdate_aesni(Aes * aes,byte * p,const byte * c,word32 cSz,const byte * a,word32 aSz)8833 static int AesGcmDecryptUpdate_aesni(Aes* aes, byte* p, const byte* c,
8834 word32 cSz, const byte* a, word32 aSz)
8835 {
8836 word32 blocks;
8837 int partial;
8838
8839 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8840 /* Hash in A, the Authentication Data */
8841 AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL));
8842
8843 /* Hash in C, the Cipher text, and decrypt. */
8844 if (cSz != 0 && p != NULL) {
8845 /* Update count of cipher text we have hashed. */
8846 aes->cSz += cSz;
8847 if (aes->cOver > 0) {
8848 /* Calculate amount we can use - fill up the block. */
8849 byte sz = AES_BLOCK_SIZE - aes->cOver;
8850 if (sz > cSz) {
8851 sz = cSz;
8852 }
8853 /* Keep a copy of the cipher text for GHASH. */
8854 XMEMCPY(AES_LASTBLOCK(aes) + aes->cOver, c, sz);
8855 /* Decrypt some of the cipher text. */
8856 xorbuf(AES_LASTGBLOCK(aes) + aes->cOver, c, sz);
8857 XMEMCPY(p, AES_LASTGBLOCK(aes) + aes->cOver, sz);
8858 /* Update count of unsed encrypted counter. */
8859 aes->cOver += sz;
8860 if (aes->cOver == AES_BLOCK_SIZE) {
8861 /* We have filled up the block and can process. */
8862 #ifdef HAVE_INTEL_AVX2
8863 if (IS_INTEL_AVX2(intel_flags)) {
8864 AES_GCM_ghash_block_avx2(AES_LASTBLOCK(aes), AES_TAG(aes),
8865 aes->H);
8866 }
8867 else
8868 #endif
8869 #ifdef HAVE_INTEL_AVX1
8870 if (IS_INTEL_AVX1(intel_flags)) {
8871 AES_GCM_ghash_block_avx1(AES_LASTBLOCK(aes), AES_TAG(aes),
8872 aes->H);
8873 }
8874 else
8875 #endif
8876 {
8877 AES_GCM_ghash_block_aesni(AES_LASTBLOCK(aes), AES_TAG(aes),
8878 aes->H);
8879 }
8880 /* Reset count. */
8881 aes->cOver = 0;
8882 }
8883 /* Used up some data. */
8884 cSz -= sz;
8885 c += sz;
8886 p += sz;
8887 }
8888
8889 /* Calculate number of blocks of plaintext and the leftover. */
8890 blocks = cSz / AES_BLOCK_SIZE;
8891 partial = cSz % AES_BLOCK_SIZE;
8892 if (blocks > 0) {
8893 /* Decrypt and GHASH full blocks now. */
8894 #ifdef HAVE_INTEL_AVX2
8895 if (IS_INTEL_AVX2(intel_flags)) {
8896 AES_GCM_decrypt_update_avx2((byte*)aes->key, aes->rounds, p, c,
8897 blocks * AES_BLOCK_SIZE, AES_TAG(aes), aes->H,
8898 AES_COUNTER(aes));
8899 }
8900 else
8901 #endif
8902 #ifdef HAVE_INTEL_AVX1
8903 if (IS_INTEL_AVX1(intel_flags)) {
8904 AES_GCM_decrypt_update_avx1((byte*)aes->key, aes->rounds, p, c,
8905 blocks * AES_BLOCK_SIZE, AES_TAG(aes), aes->H,
8906 AES_COUNTER(aes));
8907 }
8908 else
8909 #endif
8910 {
8911 AES_GCM_decrypt_update_aesni((byte*)aes->key, aes->rounds, p, c,
8912 blocks * AES_BLOCK_SIZE, AES_TAG(aes), aes->H,
8913 AES_COUNTER(aes));
8914 }
8915 /* Skip over to end of blocks. */
8916 c += blocks * AES_BLOCK_SIZE;
8917 p += blocks * AES_BLOCK_SIZE;
8918 }
8919 if (partial != 0) {
8920 /* Encrypt the counter - XOR in zeros as proxy for cipher text. */
8921 XMEMSET(AES_LASTGBLOCK(aes), 0, AES_BLOCK_SIZE);
8922 #ifdef HAVE_INTEL_AVX2
8923 if (IS_INTEL_AVX2(intel_flags)) {
8924 AES_GCM_encrypt_block_avx2((byte*)aes->key, aes->rounds,
8925 AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
8926 }
8927 else
8928 #endif
8929 #ifdef HAVE_INTEL_AVX1
8930 if (IS_INTEL_AVX1(intel_flags)) {
8931 AES_GCM_encrypt_block_avx1((byte*)aes->key, aes->rounds,
8932 AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
8933 }
8934 else
8935 #endif
8936 {
8937 AES_GCM_encrypt_block_aesni((byte*)aes->key, aes->rounds,
8938 AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
8939 }
8940 /* Keep cipher text for GHASH of last partial block. */
8941 XMEMCPY(AES_LASTBLOCK(aes), c, partial);
8942 /* XOR the remaining cipher text to calculate plaintext. */
8943 xorbuf(AES_LASTGBLOCK(aes), c, partial);
8944 XMEMCPY(p, AES_LASTGBLOCK(aes), partial);
8945 /* Update count of the block used. */
8946 aes->cOver = (byte)partial;
8947 }
8948 }
8949 RESTORE_VECTOR_REGISTERS();
8950 return 0;
8951 }
8952
8953 /* Finalize the AES GCM for decryption and check the authentication tag.
8954 *
8955 * Calls AVX2, AVX1 or straight AES-NI optimized assembly code.
8956 *
8957 * @param [in, out] aes AES object.
8958 * @param [in] authTag Buffer holding authentication tag.
8959 * @param [in] authTagSz Length of authentication tag in bytes.
8960 * @return 0 on success.
8961 * @return AES_GCM_AUTH_E when authentication tag doesn't match calculated
8962 * value.
8963 */
AesGcmDecryptFinal_aesni(Aes * aes,const byte * authTag,word32 authTagSz)8964 static int AesGcmDecryptFinal_aesni(Aes* aes, const byte* authTag,
8965 word32 authTagSz)
8966 {
8967 int ret = 0;
8968 int res;
8969 /* AAD block incomplete when > 0 */
8970 byte over = aes->aOver;
8971 byte *lastBlock = AES_LASTGBLOCK(aes);
8972
8973 SAVE_VECTOR_REGISTERS(return _svr_ret;);
8974 if (aes->cOver > 0) {
8975 /* Cipher text block incomplete. */
8976 over = aes->cOver;
8977 lastBlock = AES_LASTBLOCK(aes);
8978 }
8979 if (over > 0) {
8980 /* Zeroize the unused part of the block. */
8981 XMEMSET(lastBlock + over, 0, AES_BLOCK_SIZE - over);
8982 /* Hash the last block of cipher text. */
8983 #ifdef HAVE_INTEL_AVX2
8984 if (IS_INTEL_AVX2(intel_flags)) {
8985 AES_GCM_ghash_block_avx2(lastBlock, AES_TAG(aes), aes->H);
8986 }
8987 else
8988 #endif
8989 #ifdef HAVE_INTEL_AVX1
8990 if (IS_INTEL_AVX1(intel_flags)) {
8991 AES_GCM_ghash_block_avx1(lastBlock, AES_TAG(aes), aes->H);
8992 }
8993 else
8994 #endif
8995 {
8996 AES_GCM_ghash_block_aesni(lastBlock, AES_TAG(aes), aes->H);
8997 }
8998 }
8999 /* Calculate and compare the authentication tag. */
9000 #ifdef HAVE_INTEL_AVX2
9001 if (IS_INTEL_AVX2(intel_flags)) {
9002 AES_GCM_decrypt_final_avx2(AES_TAG(aes), authTag, authTagSz, aes->cSz,
9003 aes->aSz, aes->H, AES_INITCTR(aes), &res);
9004 }
9005 else
9006 #endif
9007 #ifdef HAVE_INTEL_AVX1
9008 if (IS_INTEL_AVX1(intel_flags)) {
9009 AES_GCM_decrypt_final_avx1(AES_TAG(aes), authTag, authTagSz, aes->cSz,
9010 aes->aSz, aes->H, AES_INITCTR(aes), &res);
9011 }
9012 else
9013 #endif
9014 {
9015 AES_GCM_decrypt_final_aesni(AES_TAG(aes), authTag, authTagSz, aes->cSz,
9016 aes->aSz, aes->H, AES_INITCTR(aes), &res);
9017 }
9018 RESTORE_VECTOR_REGISTERS();
9019 /* Return error code when calculated doesn't match input. */
9020 if (res == 0) {
9021 ret = AES_GCM_AUTH_E;
9022 }
9023 return ret;
9024 }
9025 #endif /* HAVE_AES_DECRYPT || HAVE_AESGCM_DECRYPT */
9026 #endif /* WOLFSSL_AESNI */
9027
9028 /* Initialize an AES GCM cipher for encryption or decryption.
9029 *
9030 * Must call wc_AesInit() before calling this function.
9031 *
9032 * @param [in, out] aes AES object.
9033 * @param [in] key Buffer holding key.
9034 * @param [in] len Length of key in bytes.
9035 * @param [in] iv Buffer holding IV/nonce.
9036 * @param [in] ivSz Length of IV/nonce in bytes.
9037 * @return 0 on success.
9038 * @return BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
9039 * is NULL, or the IV is NULL and no previous IV has been set.
9040 * @return MEMORY_E when dynamic memory allocation fails. (WOLFSSL_SMALL_STACK)
9041 */
wc_AesGcmInit(Aes * aes,const byte * key,word32 len,const byte * iv,word32 ivSz)9042 int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv,
9043 word32 ivSz)
9044 {
9045 int ret = 0;
9046
9047 /* Check validity of parameters. */
9048 if ((aes == NULL) || ((len > 0) && (key == NULL)) ||
9049 ((ivSz == 0) && (iv != NULL)) || (ivSz > AES_BLOCK_SIZE) ||
9050 ((ivSz > 0) && (iv == NULL))) {
9051 ret = BAD_FUNC_ARG;
9052 }
9053
9054 #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI)
9055 if ((ret == 0) && (aes->streamData == NULL)) {
9056 /* Allocate buffers for streaming. */
9057 aes->streamData = (byte*)XMALLOC(5 * AES_BLOCK_SIZE, aes->heap,
9058 DYNAMIC_TYPE_AES);
9059 if (aes->streamData == NULL) {
9060 ret = MEMORY_E;
9061 }
9062 }
9063 #endif
9064
9065 /* Set the key if passed in. */
9066 if ((ret == 0) && (key != NULL)) {
9067 ret = wc_AesGcmSetKey(aes, key, len);
9068 }
9069
9070 if (ret == 0) {
9071 /* Setup with IV if needed. */
9072 if (iv != NULL) {
9073 /* Cache the IV in AES GCM object. */
9074 XMEMCPY((byte*)aes->reg, iv, ivSz);
9075 aes->nonceSz = ivSz;
9076 }
9077 else if (aes->nonceSz != 0) {
9078 /* Copy out the cached copy. */
9079 iv = (byte*)aes->reg;
9080 ivSz = aes->nonceSz;
9081 }
9082
9083 if (iv != NULL) {
9084 /* Initialize with the IV. */
9085 #ifdef WOLFSSL_AESNI
9086 if (haveAESNI
9087 #ifdef HAVE_INTEL_AVX2
9088 || IS_INTEL_AVX2(intel_flags)
9089 #endif
9090 #ifdef HAVE_INTEL_AVX1
9091 || IS_INTEL_AVX1(intel_flags)
9092 #endif
9093 ) {
9094 ret = AesGcmInit_aesni(aes, iv, ivSz);
9095 }
9096 else
9097 #endif
9098 {
9099 AesGcmInit_C(aes, iv, ivSz);
9100 }
9101
9102 aes->nonceSet = 1;
9103 }
9104 }
9105
9106 return ret;
9107 }
9108
9109 /* Initialize an AES GCM cipher for encryption.
9110 *
9111 * Must call wc_AesInit() before calling this function.
9112 *
9113 * @param [in, out] aes AES object.
9114 * @param [in] key Buffer holding key.
9115 * @param [in] len Length of key in bytes.
9116 * @param [in] iv Buffer holding IV/nonce.
9117 * @param [in] ivSz Length of IV/nonce in bytes.
9118 * @return 0 on success.
9119 * @return BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
9120 * is NULL, or the IV is NULL and no previous IV has been set.
9121 */
wc_AesGcmEncryptInit(Aes * aes,const byte * key,word32 len,const byte * iv,word32 ivSz)9122 int wc_AesGcmEncryptInit(Aes* aes, const byte* key, word32 len, const byte* iv,
9123 word32 ivSz)
9124 {
9125 return wc_AesGcmInit(aes, key, len, iv, ivSz);
9126 }
9127
9128 /* Initialize an AES GCM cipher for encryption or decryption. Get IV.
9129 *
9130 * Must call wc_AesInit() before calling this function.
9131 *
9132 * @param [in, out] aes AES object.
9133 * @param [in] key Buffer holding key.
9134 * @param [in] len Length of key in bytes.
9135 * @param [in] iv Buffer holding IV/nonce.
9136 * @param [in] ivSz Length of IV/nonce in bytes.
9137 * @return 0 on success.
9138 * @return BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
9139 * is NULL, or the IV is NULL and no previous IV has been set.
9140 */
wc_AesGcmEncryptInit_ex(Aes * aes,const byte * key,word32 len,byte * ivOut,word32 ivOutSz)9141 int wc_AesGcmEncryptInit_ex(Aes* aes, const byte* key, word32 len, byte* ivOut,
9142 word32 ivOutSz)
9143 {
9144 XMEMCPY(ivOut, aes->reg, ivOutSz);
9145 return wc_AesGcmInit(aes, key, len, NULL, 0);
9146 }
9147
9148 /* Update the AES GCM for encryption with data and/or authentication data.
9149 *
9150 * All the AAD must be passed to update before the plaintext.
9151 * Last part of AAD can be passed with first part of plaintext.
9152 *
9153 * Must set key and IV before calling this function.
9154 * Must call wc_AesGcmInit() before calling this function.
9155 *
9156 * @param [in, out] aes AES object.
9157 * @param [out] out Buffer to hold cipher text.
9158 * @param [in] in Buffer holding plaintext.
9159 * @param [in] sz Length of plaintext in bytes.
9160 * @param [in] authIn Buffer holding authentication data.
9161 * @param [in] authInSz Length of authentication data in bytes.
9162 * @return 0 on success.
9163 * @return BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
9164 * is NULL.
9165 */
wc_AesGcmEncryptUpdate(Aes * aes,byte * out,const byte * in,word32 sz,const byte * authIn,word32 authInSz)9166 int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz,
9167 const byte* authIn, word32 authInSz)
9168 {
9169 int ret = 0;
9170
9171 /* Check validity of parameters. */
9172 if ((aes == NULL) || ((authInSz > 0) && (authIn == NULL)) || ((sz > 0) &&
9173 ((out == NULL) || (in == NULL)))) {
9174 ret = BAD_FUNC_ARG;
9175 }
9176
9177 /* Check key has been set. */
9178 if ((ret == 0) && (!aes->gcmKeySet)) {
9179 ret = MISSING_KEY;
9180 }
9181 /* Check IV has been set. */
9182 if ((ret == 0) && (!aes->nonceSet)) {
9183 ret = MISSING_IV;
9184 }
9185
9186 if ((ret == 0) && aes->ctrSet && (aes->aSz == 0) && (aes->cSz == 0)) {
9187 aes->invokeCtr[0]++;
9188 if (aes->invokeCtr[0] == 0) {
9189 aes->invokeCtr[1]++;
9190 if (aes->invokeCtr[1] == 0)
9191 ret = AES_GCM_OVERFLOW_E;
9192 }
9193 }
9194
9195 if (ret == 0) {
9196 /* Encrypt with AAD and/or plaintext. */
9197 #if defined(WOLFSSL_AESNI)
9198 if (haveAESNI
9199 #ifdef HAVE_INTEL_AVX2
9200 || IS_INTEL_AVX2(intel_flags)
9201 #endif
9202 #ifdef HAVE_INTEL_AVX1
9203 || IS_INTEL_AVX1(intel_flags)
9204 #endif
9205 ) {
9206 ret = AesGcmEncryptUpdate_aesni(aes, out, in, sz, authIn, authInSz);
9207 }
9208 else
9209 #endif
9210 {
9211 /* Encrypt the plaintext. */
9212 AesGcmCryptUpdate_C(aes, out, in, sz);
9213 /* Update the authenication tag with any authentication data and the
9214 * new cipher text. */
9215 GHASH_UPDATE(aes, authIn, authInSz, out, sz);
9216 }
9217 }
9218
9219 return ret;
9220 }
9221
9222 /* Finalize the AES GCM for encryption and return the authentication tag.
9223 *
9224 * Must set key and IV before calling this function.
9225 * Must call wc_AesGcmInit() before calling this function.
9226 *
9227 * @param [in, out] aes AES object.
9228 * @param [out] authTag Buffer to hold authentication tag.
9229 * @param [in] authTagSz Length of authentication tag in bytes.
9230 * @return 0 on success.
9231 */
wc_AesGcmEncryptFinal(Aes * aes,byte * authTag,word32 authTagSz)9232 int wc_AesGcmEncryptFinal(Aes* aes, byte* authTag, word32 authTagSz)
9233 {
9234 int ret = 0;
9235
9236 /* Check validity of parameters. */
9237 if ((aes == NULL) || (authTag == NULL) || (authTagSz > AES_BLOCK_SIZE) ||
9238 (authTagSz == 0)) {
9239 ret = BAD_FUNC_ARG;
9240 }
9241
9242 /* Check key has been set. */
9243 if ((ret == 0) && (!aes->gcmKeySet)) {
9244 ret = MISSING_KEY;
9245 }
9246 /* Check IV has been set. */
9247 if ((ret == 0) && (!aes->nonceSet)) {
9248 ret = MISSING_IV;
9249 }
9250
9251 if (ret == 0) {
9252 /* Calculate authentication tag. */
9253 #ifdef WOLFSSL_AESNI
9254 if (haveAESNI
9255 #ifdef HAVE_INTEL_AVX2
9256 || IS_INTEL_AVX2(intel_flags)
9257 #endif
9258 #ifdef HAVE_INTEL_AVX1
9259 || IS_INTEL_AVX1(intel_flags)
9260 #endif
9261 ) {
9262 AesGcmEncryptFinal_aesni(aes, authTag, authTagSz);
9263 }
9264 else
9265 #endif
9266 {
9267 AesGcmFinal_C(aes, authTag, authTagSz);
9268 }
9269 }
9270
9271 if ((ret == 0) && aes->ctrSet) {
9272 IncCtr((byte*)aes->reg, aes->nonceSz);
9273 }
9274
9275 return ret;
9276 }
9277
9278 #if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
9279 /* Initialize an AES GCM cipher for decryption.
9280 *
9281 * Must call wc_AesInit() before calling this function.
9282 *
9283 * @param [in, out] aes AES object.
9284 * @param [in] key Buffer holding key.
9285 * @param [in] len Length of key in bytes.
9286 * @param [in] iv Buffer holding IV/nonce.
9287 * @param [in] ivSz Length of IV/nonce in bytes.
9288 * @return 0 on success.
9289 * @return BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
9290 * is NULL, or the IV is NULL and no previous IV has been set.
9291 */
wc_AesGcmDecryptInit(Aes * aes,const byte * key,word32 len,const byte * iv,word32 ivSz)9292 int wc_AesGcmDecryptInit(Aes* aes, const byte* key, word32 len, const byte* iv,
9293 word32 ivSz)
9294 {
9295 return wc_AesGcmInit(aes, key, len, iv, ivSz);
9296 }
9297
9298 /* Update the AES GCM for decryption with data and/or authentication data.
9299 *
9300 * All the AAD must be passed to update before the cipher text.
9301 * Last part of AAD can be passed with first part of cipher text.
9302 *
9303 * Must set key and IV before calling this function.
9304 * Must call wc_AesGcmInit() before calling this function.
9305 *
9306 * @param [in, out] aes AES object.
9307 * @param [out] out Buffer to hold plaintext.
9308 * @param [in] in Buffer holding cipher text.
9309 * @param [in] sz Length of cipher text in bytes.
9310 * @param [in] authIn Buffer holding authentication data.
9311 * @param [in] authInSz Length of authentication data in bytes.
9312 * @return 0 on success.
9313 * @return BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
9314 * is NULL.
9315 */
wc_AesGcmDecryptUpdate(Aes * aes,byte * out,const byte * in,word32 sz,const byte * authIn,word32 authInSz)9316 int wc_AesGcmDecryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz,
9317 const byte* authIn, word32 authInSz)
9318 {
9319 int ret = 0;
9320
9321 /* Check validity of parameters. */
9322 if ((aes == NULL) || ((authInSz > 0) && (authIn == NULL)) || ((sz > 0) &&
9323 ((out == NULL) || (in == NULL)))) {
9324 ret = BAD_FUNC_ARG;
9325 }
9326
9327 /* Check key has been set. */
9328 if ((ret == 0) && (!aes->gcmKeySet)) {
9329 ret = MISSING_KEY;
9330 }
9331 /* Check IV has been set. */
9332 if ((ret == 0) && (!aes->nonceSet)) {
9333 ret = MISSING_IV;
9334 }
9335
9336 if (ret == 0) {
9337 /* Decrypt with AAD and/or cipher text. */
9338 #if defined(WOLFSSL_AESNI)
9339 if (haveAESNI
9340 #ifdef HAVE_INTEL_AVX2
9341 || IS_INTEL_AVX2(intel_flags)
9342 #endif
9343 #ifdef HAVE_INTEL_AVX1
9344 || IS_INTEL_AVX1(intel_flags)
9345 #endif
9346 ) {
9347 AesGcmDecryptUpdate_aesni(aes, out, in, sz, authIn, authInSz);
9348 }
9349 else
9350 #endif
9351 {
9352 /* Update the authenication tag with any authentication data and
9353 * cipher text. */
9354 GHASH_UPDATE(aes, authIn, authInSz, in, sz);
9355 /* Decrypt the cipher text. */
9356 AesGcmCryptUpdate_C(aes, out, in, sz);
9357 }
9358 }
9359
9360 return ret;
9361 }
9362
9363 /* Finalize the AES GCM for decryption and check the authentication tag.
9364 *
9365 * Must set key and IV before calling this function.
9366 * Must call wc_AesGcmInit() before calling this function.
9367 *
9368 * @param [in, out] aes AES object.
9369 * @param [in] authTag Buffer holding authentication tag.
9370 * @param [in] authTagSz Length of authentication tag in bytes.
9371 * @return 0 on success.
9372 */
wc_AesGcmDecryptFinal(Aes * aes,const byte * authTag,word32 authTagSz)9373 int wc_AesGcmDecryptFinal(Aes* aes, const byte* authTag, word32 authTagSz)
9374 {
9375 int ret = 0;
9376
9377 /* Check validity of parameters. */
9378 if ((aes == NULL) || (authTag == NULL) || (authTagSz > AES_BLOCK_SIZE) ||
9379 (authTagSz == 0)) {
9380 ret = BAD_FUNC_ARG;
9381 }
9382
9383 /* Check key has been set. */
9384 if ((ret == 0) && (!aes->gcmKeySet)) {
9385 ret = MISSING_KEY;
9386 }
9387 /* Check IV has been set. */
9388 if ((ret == 0) && (!aes->nonceSet)) {
9389 ret = MISSING_IV;
9390 }
9391
9392 if (ret == 0) {
9393 /* Calculate authentication tag and compare with one passed in.. */
9394 #ifdef WOLFSSL_AESNI
9395 if (haveAESNI
9396 #ifdef HAVE_INTEL_AVX2
9397 || IS_INTEL_AVX2(intel_flags)
9398 #endif
9399 #ifdef HAVE_INTEL_AVX1
9400 || IS_INTEL_AVX1(intel_flags)
9401 #endif
9402 ) {
9403 ret = AesGcmDecryptFinal_aesni(aes, authTag, authTagSz);
9404 }
9405 else
9406 #endif
9407 {
9408 ALIGN32 byte calcTag[AES_BLOCK_SIZE];
9409 /* Calculate authentication tag. */
9410 AesGcmFinal_C(aes, calcTag, authTagSz);
9411 /* Check calculated tag matches the one passed in. */
9412 if (ConstantCompare(authTag, calcTag, authTagSz) != 0) {
9413 ret = AES_GCM_AUTH_E;
9414 }
9415 }
9416 }
9417
9418 return ret;
9419 }
9420 #endif /* HAVE_AES_DECRYPT || HAVE_AESGCM_DECRYPT */
9421 #endif /* WOLFSSL_AESGCM_STREAM */
9422 #endif /* WOLFSSL_XILINX_CRYPT */
9423 #endif /* end of block for AESGCM implementation selection */
9424
9425
9426 /* Common to all, abstract functions that build off of lower level AESGCM
9427 * functions */
9428 #ifndef WC_NO_RNG
9429
CheckAesGcmIvSize(int ivSz)9430 static WC_INLINE int CheckAesGcmIvSize(int ivSz) {
9431 return (ivSz == GCM_NONCE_MIN_SZ ||
9432 ivSz == GCM_NONCE_MID_SZ ||
9433 ivSz == GCM_NONCE_MAX_SZ);
9434 }
9435
9436
wc_AesGcmSetExtIV(Aes * aes,const byte * iv,word32 ivSz)9437 int wc_AesGcmSetExtIV(Aes* aes, const byte* iv, word32 ivSz)
9438 {
9439 int ret = 0;
9440
9441 if (aes == NULL || iv == NULL || !CheckAesGcmIvSize(ivSz)) {
9442 ret = BAD_FUNC_ARG;
9443 }
9444
9445 if (ret == 0) {
9446 XMEMCPY((byte*)aes->reg, iv, ivSz);
9447
9448 /* If the IV is 96, allow for a 2^64 invocation counter.
9449 * For any other size for the nonce, limit the invocation
9450 * counter to 32-bits. (SP 800-38D 8.3) */
9451 aes->invokeCtr[0] = 0;
9452 aes->invokeCtr[1] = (ivSz == GCM_NONCE_MID_SZ) ? 0 : 0xFFFFFFFF;
9453 #ifdef WOLFSSL_AESGCM_STREAM
9454 aes->ctrSet = 1;
9455 #endif
9456 aes->nonceSz = ivSz;
9457 }
9458
9459 return ret;
9460 }
9461
9462
wc_AesGcmSetIV(Aes * aes,word32 ivSz,const byte * ivFixed,word32 ivFixedSz,WC_RNG * rng)9463 int wc_AesGcmSetIV(Aes* aes, word32 ivSz,
9464 const byte* ivFixed, word32 ivFixedSz,
9465 WC_RNG* rng)
9466 {
9467 int ret = 0;
9468
9469 if (aes == NULL || rng == NULL || !CheckAesGcmIvSize(ivSz) ||
9470 (ivFixed == NULL && ivFixedSz != 0) ||
9471 (ivFixed != NULL && ivFixedSz != AES_IV_FIXED_SZ)) {
9472
9473 ret = BAD_FUNC_ARG;
9474 }
9475
9476 if (ret == 0) {
9477 byte* iv = (byte*)aes->reg;
9478
9479 if (ivFixedSz)
9480 XMEMCPY(iv, ivFixed, ivFixedSz);
9481
9482 ret = wc_RNG_GenerateBlock(rng, iv + ivFixedSz, ivSz - ivFixedSz);
9483 }
9484
9485 if (ret == 0) {
9486 /* If the IV is 96, allow for a 2^64 invocation counter.
9487 * For any other size for the nonce, limit the invocation
9488 * counter to 32-bits. (SP 800-38D 8.3) */
9489 aes->invokeCtr[0] = 0;
9490 aes->invokeCtr[1] = (ivSz == GCM_NONCE_MID_SZ) ? 0 : 0xFFFFFFFF;
9491 #ifdef WOLFSSL_AESGCM_STREAM
9492 aes->ctrSet = 1;
9493 #endif
9494 aes->nonceSz = ivSz;
9495 }
9496
9497 return ret;
9498 }
9499
9500
wc_AesGcmEncrypt_ex(Aes * aes,byte * out,const byte * in,word32 sz,byte * ivOut,word32 ivOutSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)9501 int wc_AesGcmEncrypt_ex(Aes* aes, byte* out, const byte* in, word32 sz,
9502 byte* ivOut, word32 ivOutSz,
9503 byte* authTag, word32 authTagSz,
9504 const byte* authIn, word32 authInSz)
9505 {
9506 int ret = 0;
9507
9508 if (aes == NULL || (sz != 0 && (in == NULL || out == NULL)) ||
9509 ivOut == NULL || ivOutSz != aes->nonceSz ||
9510 (authIn == NULL && authInSz != 0)) {
9511
9512 ret = BAD_FUNC_ARG;
9513 }
9514
9515 if (ret == 0) {
9516 aes->invokeCtr[0]++;
9517 if (aes->invokeCtr[0] == 0) {
9518 aes->invokeCtr[1]++;
9519 if (aes->invokeCtr[1] == 0)
9520 ret = AES_GCM_OVERFLOW_E;
9521 }
9522 }
9523
9524 if (ret == 0) {
9525 XMEMCPY(ivOut, aes->reg, ivOutSz);
9526 ret = wc_AesGcmEncrypt(aes, out, in, sz,
9527 (byte*)aes->reg, ivOutSz,
9528 authTag, authTagSz,
9529 authIn, authInSz);
9530 if (ret == 0)
9531 IncCtr((byte*)aes->reg, ivOutSz);
9532 }
9533
9534 return ret;
9535 }
9536
wc_Gmac(const byte * key,word32 keySz,byte * iv,word32 ivSz,const byte * authIn,word32 authInSz,byte * authTag,word32 authTagSz,WC_RNG * rng)9537 int wc_Gmac(const byte* key, word32 keySz, byte* iv, word32 ivSz,
9538 const byte* authIn, word32 authInSz,
9539 byte* authTag, word32 authTagSz, WC_RNG* rng)
9540 {
9541 #ifdef WOLFSSL_SMALL_STACK
9542 Aes *aes = NULL;
9543 #else
9544 Aes aes[1];
9545 #endif
9546 int ret;
9547
9548 if (key == NULL || iv == NULL || (authIn == NULL && authInSz != 0) ||
9549 authTag == NULL || authTagSz == 0 || rng == NULL) {
9550
9551 return BAD_FUNC_ARG;
9552 }
9553
9554 #ifdef WOLFSSL_SMALL_STACK
9555 if ((aes = (Aes *)XMALLOC(sizeof *aes, NULL,
9556 DYNAMIC_TYPE_AES)) == NULL)
9557 return MEMORY_E;
9558 #endif
9559
9560 ret = wc_AesInit(aes, NULL, INVALID_DEVID);
9561 if (ret == 0) {
9562 ret = wc_AesGcmSetKey(aes, key, keySz);
9563 if (ret == 0)
9564 ret = wc_AesGcmSetIV(aes, ivSz, NULL, 0, rng);
9565 if (ret == 0)
9566 ret = wc_AesGcmEncrypt_ex(aes, NULL, NULL, 0, iv, ivSz,
9567 authTag, authTagSz, authIn, authInSz);
9568 wc_AesFree(aes);
9569 }
9570 ForceZero(aes, sizeof *aes);
9571 #ifdef WOLFSSL_SMALL_STACK
9572 XFREE(aes, NULL, DYNAMIC_TYPE_AES);
9573 #endif
9574
9575 return ret;
9576 }
9577
wc_GmacVerify(const byte * key,word32 keySz,const byte * iv,word32 ivSz,const byte * authIn,word32 authInSz,const byte * authTag,word32 authTagSz)9578 int wc_GmacVerify(const byte* key, word32 keySz,
9579 const byte* iv, word32 ivSz,
9580 const byte* authIn, word32 authInSz,
9581 const byte* authTag, word32 authTagSz)
9582 {
9583 int ret;
9584 #ifdef HAVE_AES_DECRYPT
9585 #ifdef WOLFSSL_SMALL_STACK
9586 Aes *aes = NULL;
9587 #else
9588 Aes aes[1];
9589 #endif
9590
9591 if (key == NULL || iv == NULL || (authIn == NULL && authInSz != 0) ||
9592 authTag == NULL || authTagSz == 0 || authTagSz > AES_BLOCK_SIZE) {
9593
9594 return BAD_FUNC_ARG;
9595 }
9596
9597 #ifdef WOLFSSL_SMALL_STACK
9598 if ((aes = (Aes *)XMALLOC(sizeof *aes, NULL,
9599 DYNAMIC_TYPE_AES)) == NULL)
9600 return MEMORY_E;
9601 #endif
9602
9603 ret = wc_AesInit(aes, NULL, INVALID_DEVID);
9604 if (ret == 0) {
9605 ret = wc_AesGcmSetKey(aes, key, keySz);
9606 if (ret == 0)
9607 ret = wc_AesGcmDecrypt(aes, NULL, NULL, 0, iv, ivSz,
9608 authTag, authTagSz, authIn, authInSz);
9609 wc_AesFree(aes);
9610 }
9611 ForceZero(aes, sizeof *aes);
9612 #ifdef WOLFSSL_SMALL_STACK
9613 XFREE(aes, NULL, DYNAMIC_TYPE_AES);
9614 #endif
9615 #else
9616 (void)key;
9617 (void)keySz;
9618 (void)iv;
9619 (void)ivSz;
9620 (void)authIn;
9621 (void)authInSz;
9622 (void)authTag;
9623 (void)authTagSz;
9624 ret = NOT_COMPILED_IN;
9625 #endif
9626 return ret;
9627 }
9628
9629 #endif /* WC_NO_RNG */
9630
9631
wc_GmacSetKey(Gmac * gmac,const byte * key,word32 len)9632 WOLFSSL_API int wc_GmacSetKey(Gmac* gmac, const byte* key, word32 len)
9633 {
9634 if (gmac == NULL || key == NULL) {
9635 return BAD_FUNC_ARG;
9636 }
9637 return wc_AesGcmSetKey(&gmac->aes, key, len);
9638 }
9639
9640
wc_GmacUpdate(Gmac * gmac,const byte * iv,word32 ivSz,const byte * authIn,word32 authInSz,byte * authTag,word32 authTagSz)9641 WOLFSSL_API int wc_GmacUpdate(Gmac* gmac, const byte* iv, word32 ivSz,
9642 const byte* authIn, word32 authInSz,
9643 byte* authTag, word32 authTagSz)
9644 {
9645 if (gmac == NULL) {
9646 return BAD_FUNC_ARG;
9647 }
9648
9649 return wc_AesGcmEncrypt(&gmac->aes, NULL, NULL, 0, iv, ivSz,
9650 authTag, authTagSz, authIn, authInSz);
9651 }
9652
9653 #endif /* HAVE_AESGCM */
9654
9655
9656 #ifdef HAVE_AESCCM
9657
wc_AesCcmSetKey(Aes * aes,const byte * key,word32 keySz)9658 int wc_AesCcmSetKey(Aes* aes, const byte* key, word32 keySz)
9659 {
9660 if (!((keySz == 16) || (keySz == 24) || (keySz == 32)))
9661 return BAD_FUNC_ARG;
9662
9663 return wc_AesSetKey(aes, key, keySz, NULL, AES_ENCRYPTION);
9664 }
9665
9666
9667 /* Checks if the tag size is an accepted value based on RFC 3610 section 2
9668 * returns 0 if tag size is ok
9669 */
wc_AesCcmCheckTagSize(int sz)9670 int wc_AesCcmCheckTagSize(int sz)
9671 {
9672 /* values here are from RFC 3610 section 2 */
9673 if (sz != 4 && sz != 6 && sz != 8 && sz != 10 && sz != 12 && sz != 14
9674 && sz != 16) {
9675 WOLFSSL_MSG("Bad auth tag size AES-CCM");
9676 return BAD_FUNC_ARG;
9677 }
9678 return 0;
9679 }
9680
9681 #ifdef WOLFSSL_ARMASM
9682 /* implementation located in wolfcrypt/src/port/arm/armv8-aes.c */
9683
9684 #elif defined(HAVE_COLDFIRE_SEC)
9685 #error "Coldfire SEC doesn't currently support AES-CCM mode"
9686
9687 #elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
9688 !defined(WOLFSSL_QNX_CAAM)
9689 /* implemented in wolfcrypt/src/port/caam_aes.c */
9690
9691 #elif defined(WOLFSSL_SILABS_SE_ACCEL)
9692 /* implemented in wolfcrypt/src/port/silabs/silabs_aes.c */
wc_AesCcmEncrypt(Aes * aes,byte * out,const byte * in,word32 inSz,const byte * nonce,word32 nonceSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)9693 int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
9694 const byte* nonce, word32 nonceSz,
9695 byte* authTag, word32 authTagSz,
9696 const byte* authIn, word32 authInSz)
9697 {
9698 return wc_AesCcmEncrypt_silabs(
9699 aes, out, in, inSz,
9700 nonce, nonceSz,
9701 authTag, authTagSz,
9702 authIn, authInSz);
9703 }
9704
9705 #ifdef HAVE_AES_DECRYPT
wc_AesCcmDecrypt(Aes * aes,byte * out,const byte * in,word32 inSz,const byte * nonce,word32 nonceSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)9706 int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
9707 const byte* nonce, word32 nonceSz,
9708 const byte* authTag, word32 authTagSz,
9709 const byte* authIn, word32 authInSz)
9710 {
9711 return wc_AesCcmDecrypt_silabs(
9712 aes, out, in, inSz,
9713 nonce, nonceSz,
9714 authTag, authTagSz,
9715 authIn, authInSz);
9716 }
9717 #endif
9718 #elif defined(FREESCALE_LTC)
9719
9720 /* return 0 on success */
wc_AesCcmEncrypt(Aes * aes,byte * out,const byte * in,word32 inSz,const byte * nonce,word32 nonceSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)9721 int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
9722 const byte* nonce, word32 nonceSz,
9723 byte* authTag, word32 authTagSz,
9724 const byte* authIn, word32 authInSz)
9725 {
9726 byte *key;
9727 word32 keySize;
9728 status_t status;
9729
9730 /* sanity check on arguments */
9731 /* note, LTC_AES_EncryptTagCcm() doesn't allow null src or dst
9732 * ptrs even if inSz is zero (ltc_aes_ccm_check_input_args()), so
9733 * don't allow it here either.
9734 */
9735 if (aes == NULL || out == NULL || in == NULL || nonce == NULL
9736 || authTag == NULL || nonceSz < 7 || nonceSz > 13) {
9737 return BAD_FUNC_ARG;
9738 }
9739
9740 if (wc_AesCcmCheckTagSize(authTagSz) != 0) {
9741 return BAD_FUNC_ARG;
9742 }
9743
9744 key = (byte*)aes->key;
9745
9746 status = wc_AesGetKeySize(aes, &keySize);
9747 if (status != 0) {
9748 return status;
9749 }
9750
9751 status = wolfSSL_CryptHwMutexLock();
9752 if (status != 0)
9753 return status;
9754
9755 status = LTC_AES_EncryptTagCcm(LTC_BASE, in, out, inSz,
9756 nonce, nonceSz, authIn, authInSz, key, keySize, authTag, authTagSz);
9757 wolfSSL_CryptHwMutexUnLock();
9758
9759 return (kStatus_Success == status) ? 0 : BAD_FUNC_ARG;
9760 }
9761
9762 #ifdef HAVE_AES_DECRYPT
wc_AesCcmDecrypt(Aes * aes,byte * out,const byte * in,word32 inSz,const byte * nonce,word32 nonceSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)9763 int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
9764 const byte* nonce, word32 nonceSz,
9765 const byte* authTag, word32 authTagSz,
9766 const byte* authIn, word32 authInSz)
9767 {
9768 byte *key;
9769 word32 keySize;
9770 status_t status;
9771
9772 /* sanity check on arguments */
9773 if (aes == NULL || out == NULL || in == NULL || nonce == NULL
9774 || authTag == NULL || nonceSz < 7 || nonceSz > 13) {
9775 return BAD_FUNC_ARG;
9776 }
9777
9778 key = (byte*)aes->key;
9779
9780 status = wc_AesGetKeySize(aes, &keySize);
9781 if (status != 0) {
9782 return status;
9783 }
9784
9785 status = wolfSSL_CryptHwMutexLock();
9786 if (status != 0)
9787 return status;
9788 status = LTC_AES_DecryptTagCcm(LTC_BASE, in, out, inSz,
9789 nonce, nonceSz, authIn, authInSz, key, keySize, authTag, authTagSz);
9790 wolfSSL_CryptHwMutexUnLock();
9791
9792 if (status != kStatus_Success) {
9793 XMEMSET(out, 0, inSz);
9794 return AES_CCM_AUTH_E;
9795 }
9796 return 0;
9797 }
9798 #endif /* HAVE_AES_DECRYPT */
9799
9800 #else
9801
9802 /* Software CCM */
roll_x(Aes * aes,const byte * in,word32 inSz,byte * out)9803 static void roll_x(Aes* aes, const byte* in, word32 inSz, byte* out)
9804 {
9805 /* process the bulk of the data */
9806 while (inSz >= AES_BLOCK_SIZE) {
9807 xorbuf(out, in, AES_BLOCK_SIZE);
9808 in += AES_BLOCK_SIZE;
9809 inSz -= AES_BLOCK_SIZE;
9810
9811 wc_AesEncrypt(aes, out, out);
9812 }
9813
9814 /* process remainder of the data */
9815 if (inSz > 0) {
9816 xorbuf(out, in, inSz);
9817 wc_AesEncrypt(aes, out, out);
9818 }
9819 }
9820
roll_auth(Aes * aes,const byte * in,word32 inSz,byte * out)9821 static void roll_auth(Aes* aes, const byte* in, word32 inSz, byte* out)
9822 {
9823 word32 authLenSz;
9824 word32 remainder;
9825
9826 /* encode the length in */
9827 if (inSz <= 0xFEFF) {
9828 authLenSz = 2;
9829 out[0] ^= ((inSz & 0xFF00) >> 8);
9830 out[1] ^= (inSz & 0x00FF);
9831 }
9832 else if (inSz <= 0xFFFFFFFF) {
9833 authLenSz = 6;
9834 out[0] ^= 0xFF; out[1] ^= 0xFE;
9835 out[2] ^= ((inSz & 0xFF000000) >> 24);
9836 out[3] ^= ((inSz & 0x00FF0000) >> 16);
9837 out[4] ^= ((inSz & 0x0000FF00) >> 8);
9838 out[5] ^= (inSz & 0x000000FF);
9839 }
9840 /* Note, the protocol handles auth data up to 2^64, but we are
9841 * using 32-bit sizes right now, so the bigger data isn't handled
9842 * else if (inSz <= 0xFFFFFFFFFFFFFFFF) {} */
9843 else
9844 return;
9845
9846 /* start fill out the rest of the first block */
9847 remainder = AES_BLOCK_SIZE - authLenSz;
9848 if (inSz >= remainder) {
9849 /* plenty of bulk data to fill the remainder of this block */
9850 xorbuf(out + authLenSz, in, remainder);
9851 inSz -= remainder;
9852 in += remainder;
9853 }
9854 else {
9855 /* not enough bulk data, copy what is available, and pad zero */
9856 xorbuf(out + authLenSz, in, inSz);
9857 inSz = 0;
9858 }
9859 wc_AesEncrypt(aes, out, out);
9860
9861 if (inSz > 0)
9862 roll_x(aes, in, inSz, out);
9863 }
9864
9865
AesCcmCtrInc(byte * B,word32 lenSz)9866 static WC_INLINE void AesCcmCtrInc(byte* B, word32 lenSz)
9867 {
9868 word32 i;
9869
9870 for (i = 0; i < lenSz; i++) {
9871 if (++B[AES_BLOCK_SIZE - 1 - i] != 0) return;
9872 }
9873 }
9874
9875 #ifdef WOLFSSL_AESNI
AesCcmCtrIncSet4(byte * B,word32 lenSz)9876 static WC_INLINE void AesCcmCtrIncSet4(byte* B, word32 lenSz)
9877 {
9878 word32 i;
9879
9880 /* B+1 = B */
9881 XMEMCPY(B + AES_BLOCK_SIZE * 1, B, AES_BLOCK_SIZE);
9882 /* B+2,B+3 = B,B+1 */
9883 XMEMCPY(B + AES_BLOCK_SIZE * 2, B, AES_BLOCK_SIZE * 2);
9884
9885 for (i = 0; i < lenSz; i++) {
9886 if (++B[AES_BLOCK_SIZE * 2 - 1 - i] != 0) break;
9887 }
9888 B[AES_BLOCK_SIZE * 3 - 1] += 2;
9889 if (B[AES_BLOCK_SIZE * 3 - 1] < 2) {
9890 for (i = 1; i < lenSz; i++) {
9891 if (++B[AES_BLOCK_SIZE * 3 - 1 - i] != 0) break;
9892 }
9893 }
9894 B[AES_BLOCK_SIZE * 4 - 1] += 3;
9895 if (B[AES_BLOCK_SIZE * 4 - 1] < 3) {
9896 for (i = 1; i < lenSz; i++) {
9897 if (++B[AES_BLOCK_SIZE * 4 - 1 - i] != 0) break;
9898 }
9899 }
9900 }
9901
AesCcmCtrInc4(byte * B,word32 lenSz)9902 static WC_INLINE void AesCcmCtrInc4(byte* B, word32 lenSz)
9903 {
9904 word32 i;
9905
9906 B[AES_BLOCK_SIZE - 1] += 4;
9907 if (B[AES_BLOCK_SIZE - 1] < 4) {
9908 for (i = 1; i < lenSz; i++) {
9909 if (++B[AES_BLOCK_SIZE - 1 - i] != 0) break;
9910 }
9911 }
9912 }
9913 #endif
9914
9915 /* Software AES - CCM Encrypt */
9916 /* return 0 on success */
wc_AesCcmEncrypt(Aes * aes,byte * out,const byte * in,word32 inSz,const byte * nonce,word32 nonceSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)9917 int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
9918 const byte* nonce, word32 nonceSz,
9919 byte* authTag, word32 authTagSz,
9920 const byte* authIn, word32 authInSz)
9921 {
9922 #ifndef WOLFSSL_AESNI
9923 byte A[AES_BLOCK_SIZE];
9924 byte B[AES_BLOCK_SIZE];
9925 #else
9926 ALIGN128 byte A[AES_BLOCK_SIZE * 4];
9927 ALIGN128 byte B[AES_BLOCK_SIZE * 4];
9928 #endif
9929 byte lenSz;
9930 word32 i;
9931 byte mask = 0xFF;
9932 const word32 wordSz = (word32)sizeof(word32);
9933
9934 /* sanity check on arguments */
9935 if (aes == NULL || (inSz != 0 && (in == NULL || out == NULL)) ||
9936 nonce == NULL || authTag == NULL || nonceSz < 7 || nonceSz > 13 ||
9937 authTagSz > AES_BLOCK_SIZE)
9938 return BAD_FUNC_ARG;
9939
9940 /* sanity check on tag size */
9941 if (wc_AesCcmCheckTagSize(authTagSz) != 0) {
9942 return BAD_FUNC_ARG;
9943 }
9944
9945 #ifdef WOLF_CRYPTO_CB
9946 if (aes->devId != INVALID_DEVID) {
9947 int ret = wc_CryptoCb_AesCcmEncrypt(aes, out, in, inSz, nonce, nonceSz,
9948 authTag, authTagSz, authIn, authInSz);
9949 if (ret != CRYPTOCB_UNAVAILABLE)
9950 return ret;
9951 /* fall-through when unavailable */
9952 }
9953 #endif
9954
9955 XMEMSET(A, 0, sizeof(A));
9956 XMEMCPY(B+1, nonce, nonceSz);
9957 lenSz = AES_BLOCK_SIZE - 1 - (byte)nonceSz;
9958 B[0] = (authInSz > 0 ? 64 : 0)
9959 + (8 * (((byte)authTagSz - 2) / 2))
9960 + (lenSz - 1);
9961 for (i = 0; i < lenSz; i++) {
9962 if (mask && i >= wordSz)
9963 mask = 0x00;
9964 B[AES_BLOCK_SIZE - 1 - i] = (inSz >> ((8 * i) & mask)) & mask;
9965 }
9966
9967 wc_AesEncrypt(aes, B, A);
9968
9969 if (authInSz > 0)
9970 roll_auth(aes, authIn, authInSz, A);
9971 if (inSz > 0)
9972 roll_x(aes, in, inSz, A);
9973 XMEMCPY(authTag, A, authTagSz);
9974
9975 B[0] = lenSz - 1;
9976 for (i = 0; i < lenSz; i++)
9977 B[AES_BLOCK_SIZE - 1 - i] = 0;
9978 wc_AesEncrypt(aes, B, A);
9979 xorbuf(authTag, A, authTagSz);
9980
9981 B[15] = 1;
9982 #ifdef WOLFSSL_AESNI
9983 if (haveAESNI && aes->use_aesni) {
9984 SAVE_VECTOR_REGISTERS(return _svr_ret;);
9985 while (inSz >= AES_BLOCK_SIZE * 4) {
9986 AesCcmCtrIncSet4(B, lenSz);
9987
9988 AES_ECB_encrypt(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key,
9989 aes->rounds);
9990
9991 xorbuf(A, in, AES_BLOCK_SIZE * 4);
9992 XMEMCPY(out, A, AES_BLOCK_SIZE * 4);
9993
9994 inSz -= AES_BLOCK_SIZE * 4;
9995 in += AES_BLOCK_SIZE * 4;
9996 out += AES_BLOCK_SIZE * 4;
9997
9998 AesCcmCtrInc4(B, lenSz);
9999 }
10000 RESTORE_VECTOR_REGISTERS();
10001 }
10002 #endif
10003 while (inSz >= AES_BLOCK_SIZE) {
10004 wc_AesEncrypt(aes, B, A);
10005 xorbuf(A, in, AES_BLOCK_SIZE);
10006 XMEMCPY(out, A, AES_BLOCK_SIZE);
10007
10008 AesCcmCtrInc(B, lenSz);
10009 inSz -= AES_BLOCK_SIZE;
10010 in += AES_BLOCK_SIZE;
10011 out += AES_BLOCK_SIZE;
10012 }
10013 if (inSz > 0) {
10014 wc_AesEncrypt(aes, B, A);
10015 xorbuf(A, in, inSz);
10016 XMEMCPY(out, A, inSz);
10017 }
10018
10019 ForceZero(A, AES_BLOCK_SIZE);
10020 ForceZero(B, AES_BLOCK_SIZE);
10021
10022 return 0;
10023 }
10024
10025 #ifdef HAVE_AES_DECRYPT
10026 /* Software AES - CCM Decrypt */
wc_AesCcmDecrypt(Aes * aes,byte * out,const byte * in,word32 inSz,const byte * nonce,word32 nonceSz,const byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)10027 int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
10028 const byte* nonce, word32 nonceSz,
10029 const byte* authTag, word32 authTagSz,
10030 const byte* authIn, word32 authInSz)
10031 {
10032 #ifndef WOLFSSL_AESNI
10033 byte A[AES_BLOCK_SIZE];
10034 byte B[AES_BLOCK_SIZE];
10035 #else
10036 ALIGN128 byte B[AES_BLOCK_SIZE * 4];
10037 ALIGN128 byte A[AES_BLOCK_SIZE * 4];
10038 #endif
10039 byte* o;
10040 byte lenSz;
10041 word32 i, oSz;
10042 int result = 0;
10043 byte mask = 0xFF;
10044 const word32 wordSz = (word32)sizeof(word32);
10045
10046 /* sanity check on arguments */
10047 if (aes == NULL || (inSz != 0 && (in == NULL || out == NULL)) ||
10048 nonce == NULL || authTag == NULL || nonceSz < 7 || nonceSz > 13 ||
10049 authTagSz > AES_BLOCK_SIZE)
10050 return BAD_FUNC_ARG;
10051
10052 /* sanity check on tag size */
10053 if (wc_AesCcmCheckTagSize(authTagSz) != 0) {
10054 return BAD_FUNC_ARG;
10055 }
10056
10057 #ifdef WOLF_CRYPTO_CB
10058 if (aes->devId != INVALID_DEVID) {
10059 int ret = wc_CryptoCb_AesCcmDecrypt(aes, out, in, inSz, nonce, nonceSz,
10060 authTag, authTagSz, authIn, authInSz);
10061 if (ret != CRYPTOCB_UNAVAILABLE)
10062 return ret;
10063 /* fall-through when unavailable */
10064 }
10065 #endif
10066
10067 o = out;
10068 oSz = inSz;
10069 XMEMSET(A, 0, sizeof A);
10070 XMEMCPY(B+1, nonce, nonceSz);
10071 lenSz = AES_BLOCK_SIZE - 1 - (byte)nonceSz;
10072
10073 B[0] = lenSz - 1;
10074 for (i = 0; i < lenSz; i++)
10075 B[AES_BLOCK_SIZE - 1 - i] = 0;
10076 B[15] = 1;
10077
10078 #ifdef WOLFSSL_AESNI
10079 if (haveAESNI && aes->use_aesni) {
10080 SAVE_VECTOR_REGISTERS(return _svr_ret;);
10081 while (oSz >= AES_BLOCK_SIZE * 4) {
10082 AesCcmCtrIncSet4(B, lenSz);
10083
10084 AES_ECB_encrypt(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key,
10085 aes->rounds);
10086
10087 xorbuf(A, in, AES_BLOCK_SIZE * 4);
10088 XMEMCPY(o, A, AES_BLOCK_SIZE * 4);
10089
10090 oSz -= AES_BLOCK_SIZE * 4;
10091 in += AES_BLOCK_SIZE * 4;
10092 o += AES_BLOCK_SIZE * 4;
10093
10094 AesCcmCtrInc4(B, lenSz);
10095 }
10096 RESTORE_VECTOR_REGISTERS();
10097 }
10098 #endif
10099 while (oSz >= AES_BLOCK_SIZE) {
10100 wc_AesEncrypt(aes, B, A);
10101 xorbuf(A, in, AES_BLOCK_SIZE);
10102 XMEMCPY(o, A, AES_BLOCK_SIZE);
10103
10104 AesCcmCtrInc(B, lenSz);
10105 oSz -= AES_BLOCK_SIZE;
10106 in += AES_BLOCK_SIZE;
10107 o += AES_BLOCK_SIZE;
10108 }
10109 if (inSz > 0) {
10110 wc_AesEncrypt(aes, B, A);
10111 xorbuf(A, in, oSz);
10112 XMEMCPY(o, A, oSz);
10113 }
10114
10115 for (i = 0; i < lenSz; i++)
10116 B[AES_BLOCK_SIZE - 1 - i] = 0;
10117 wc_AesEncrypt(aes, B, A);
10118
10119 o = out;
10120 oSz = inSz;
10121
10122 B[0] = (authInSz > 0 ? 64 : 0)
10123 + (8 * (((byte)authTagSz - 2) / 2))
10124 + (lenSz - 1);
10125 for (i = 0; i < lenSz; i++) {
10126 if (mask && i >= wordSz)
10127 mask = 0x00;
10128 B[AES_BLOCK_SIZE - 1 - i] = (inSz >> ((8 * i) & mask)) & mask;
10129 }
10130
10131 wc_AesEncrypt(aes, B, A);
10132
10133 if (authInSz > 0)
10134 roll_auth(aes, authIn, authInSz, A);
10135 if (inSz > 0)
10136 roll_x(aes, o, oSz, A);
10137
10138 B[0] = lenSz - 1;
10139 for (i = 0; i < lenSz; i++)
10140 B[AES_BLOCK_SIZE - 1 - i] = 0;
10141 wc_AesEncrypt(aes, B, B);
10142 xorbuf(A, B, authTagSz);
10143
10144 if (ConstantCompare(A, authTag, authTagSz) != 0) {
10145 /* If the authTag check fails, don't keep the decrypted data.
10146 * Unfortunately, you need the decrypted data to calculate the
10147 * check value. */
10148 #if defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2) && \
10149 defined(ACVP_VECTOR_TESTING)
10150 WOLFSSL_MSG("Preserve output for vector responses");
10151 #else
10152 if (inSz > 0)
10153 XMEMSET(out, 0, inSz);
10154 #endif
10155 result = AES_CCM_AUTH_E;
10156 }
10157
10158 ForceZero(A, AES_BLOCK_SIZE);
10159 ForceZero(B, AES_BLOCK_SIZE);
10160 o = NULL;
10161
10162 return result;
10163 }
10164
10165 #endif /* HAVE_AES_DECRYPT */
10166 #endif /* software CCM */
10167
10168 /* abstract functions that call lower level AESCCM functions */
10169 #ifndef WC_NO_RNG
10170
wc_AesCcmSetNonce(Aes * aes,const byte * nonce,word32 nonceSz)10171 int wc_AesCcmSetNonce(Aes* aes, const byte* nonce, word32 nonceSz)
10172 {
10173 int ret = 0;
10174
10175 if (aes == NULL || nonce == NULL ||
10176 nonceSz < CCM_NONCE_MIN_SZ || nonceSz > CCM_NONCE_MAX_SZ) {
10177
10178 ret = BAD_FUNC_ARG;
10179 }
10180
10181 if (ret == 0) {
10182 XMEMCPY(aes->reg, nonce, nonceSz);
10183 aes->nonceSz = nonceSz;
10184
10185 /* Invocation counter should be 2^61 */
10186 aes->invokeCtr[0] = 0;
10187 aes->invokeCtr[1] = 0xE0000000;
10188 }
10189
10190 return ret;
10191 }
10192
10193
wc_AesCcmEncrypt_ex(Aes * aes,byte * out,const byte * in,word32 sz,byte * ivOut,word32 ivOutSz,byte * authTag,word32 authTagSz,const byte * authIn,word32 authInSz)10194 int wc_AesCcmEncrypt_ex(Aes* aes, byte* out, const byte* in, word32 sz,
10195 byte* ivOut, word32 ivOutSz,
10196 byte* authTag, word32 authTagSz,
10197 const byte* authIn, word32 authInSz)
10198 {
10199 int ret = 0;
10200
10201 if (aes == NULL || out == NULL ||
10202 (in == NULL && sz != 0) ||
10203 ivOut == NULL ||
10204 (authIn == NULL && authInSz != 0) ||
10205 (ivOutSz != aes->nonceSz)) {
10206
10207 ret = BAD_FUNC_ARG;
10208 }
10209
10210 if (ret == 0) {
10211 aes->invokeCtr[0]++;
10212 if (aes->invokeCtr[0] == 0) {
10213 aes->invokeCtr[1]++;
10214 if (aes->invokeCtr[1] == 0)
10215 ret = AES_CCM_OVERFLOW_E;
10216 }
10217 }
10218
10219 if (ret == 0) {
10220 ret = wc_AesCcmEncrypt(aes, out, in, sz,
10221 (byte*)aes->reg, aes->nonceSz,
10222 authTag, authTagSz,
10223 authIn, authInSz);
10224 if (ret == 0) {
10225 XMEMCPY(ivOut, aes->reg, aes->nonceSz);
10226 IncCtr((byte*)aes->reg, aes->nonceSz);
10227 }
10228 }
10229
10230 return ret;
10231 }
10232
10233 #endif /* WC_NO_RNG */
10234
10235 #endif /* HAVE_AESCCM */
10236
10237
10238 /* Initialize Aes for use with async hardware */
wc_AesInit(Aes * aes,void * heap,int devId)10239 int wc_AesInit(Aes* aes, void* heap, int devId)
10240 {
10241 int ret = 0;
10242
10243 if (aes == NULL)
10244 return BAD_FUNC_ARG;
10245
10246 aes->heap = heap;
10247
10248 #ifdef WOLF_CRYPTO_CB
10249 aes->devId = devId;
10250 aes->devCtx = NULL;
10251 #else
10252 (void)devId;
10253 #endif
10254 #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
10255 ret = wolfAsync_DevCtxInit(&aes->asyncDev, WOLFSSL_ASYNC_MARKER_AES,
10256 aes->heap, devId);
10257 #endif /* WOLFSSL_ASYNC_CRYPT */
10258
10259 #ifdef WOLFSSL_AFALG
10260 aes->alFd = -1;
10261 aes->rdFd = -1;
10262 #endif
10263 #ifdef WOLFSSL_KCAPI_AES
10264 aes->handle = NULL;
10265 aes->init = 0;
10266 #endif
10267 #if defined(WOLFSSL_DEVCRYPTO) && \
10268 (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))
10269 aes->ctx.cfd = -1;
10270 #endif
10271 #if defined(WOLFSSL_CRYPTOCELL) && defined(WOLFSSL_CRYPTOCELL_AES)
10272 XMEMSET(&aes->ctx, 0, sizeof(aes->ctx));
10273 #endif
10274 #if defined(WOLFSSL_IMXRT_DCP)
10275 DCPAesInit(aes);
10276 #endif
10277
10278
10279 #ifdef HAVE_AESGCM
10280 #ifdef OPENSSL_EXTRA
10281 XMEMSET(aes->aadH, 0, sizeof(aes->aadH));
10282 aes->aadLen = 0;
10283 #endif
10284 #endif
10285
10286 #ifdef WOLFSSL_AESGCM_STREAM
10287 #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI)
10288 aes->streamData = NULL;
10289 #endif
10290 aes->keylen = 0;
10291 aes->nonceSz = 0;
10292 aes->gcmKeySet = 0;
10293 aes->nonceSet = 0;
10294 aes->ctrSet = 0;
10295 #endif
10296
10297 return ret;
10298 }
10299
10300 #ifdef HAVE_PKCS11
wc_AesInit_Id(Aes * aes,unsigned char * id,int len,void * heap,int devId)10301 int wc_AesInit_Id(Aes* aes, unsigned char* id, int len, void* heap, int devId)
10302 {
10303 int ret = 0;
10304
10305 if (aes == NULL)
10306 ret = BAD_FUNC_ARG;
10307 if (ret == 0 && (len < 0 || len > AES_MAX_ID_LEN))
10308 ret = BUFFER_E;
10309
10310 if (ret == 0)
10311 ret = wc_AesInit(aes, heap, devId);
10312 if (ret == 0) {
10313 XMEMCPY(aes->id, id, len);
10314 aes->idLen = len;
10315 aes->labelLen = 0;
10316 }
10317
10318 return ret;
10319 }
10320
wc_AesInit_Label(Aes * aes,const char * label,void * heap,int devId)10321 int wc_AesInit_Label(Aes* aes, const char* label, void* heap, int devId)
10322 {
10323 int ret = 0;
10324 int labelLen = 0;
10325
10326 if (aes == NULL || label == NULL)
10327 ret = BAD_FUNC_ARG;
10328 if (ret == 0) {
10329 labelLen = (int)XSTRLEN(label);
10330 if (labelLen == 0 || labelLen > AES_MAX_LABEL_LEN)
10331 ret = BUFFER_E;
10332 }
10333
10334 if (ret == 0)
10335 ret = wc_AesInit(aes, heap, devId);
10336 if (ret == 0) {
10337 XMEMCPY(aes->label, label, labelLen);
10338 aes->labelLen = labelLen;
10339 aes->idLen = 0;
10340 }
10341
10342 return ret;
10343 }
10344 #endif
10345
10346 /* Free Aes from use with async hardware */
wc_AesFree(Aes * aes)10347 void wc_AesFree(Aes* aes)
10348 {
10349 if (aes == NULL)
10350 return;
10351
10352 #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
10353 wolfAsync_DevCtxFree(&aes->asyncDev, WOLFSSL_ASYNC_MARKER_AES);
10354 #endif /* WOLFSSL_ASYNC_CRYPT */
10355 #if defined(WOLFSSL_AFALG) || defined(WOLFSSL_AFALG_XILINX_AES)
10356 if (aes->rdFd > 0) { /* negative is error case */
10357 close(aes->rdFd);
10358 }
10359 if (aes->alFd > 0) {
10360 close(aes->alFd);
10361 }
10362 #endif /* WOLFSSL_AFALG */
10363 #ifdef WOLFSSL_KCAPI_AES
10364 ForceZero((byte*)aes->devKey, AES_MAX_KEY_SIZE/WOLFSSL_BIT_SIZE);
10365 if (aes->init == 1) {
10366 kcapi_cipher_destroy(aes->handle);
10367 }
10368 aes->init = 0;
10369 aes->handle = NULL;
10370 #endif
10371 #if defined(WOLFSSL_DEVCRYPTO) && \
10372 (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))
10373 wc_DevCryptoFree(&aes->ctx);
10374 #endif
10375 #if defined(WOLF_CRYPTO_CB) || (defined(WOLFSSL_DEVCRYPTO) && \
10376 (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))) || \
10377 (defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES))
10378 ForceZero((byte*)aes->devKey, AES_MAX_KEY_SIZE/WOLFSSL_BIT_SIZE);
10379 #endif
10380 #if defined(WOLFSSL_IMXRT_DCP)
10381 DCPAesFree(aes);
10382 #endif
10383 #if defined(WOLFSSL_AESGCM_STREAM) && defined(WOLFSSL_SMALL_STACK) && \
10384 !defined(WOLFSSL_AESNI)
10385 if (aes->streamData != NULL) {
10386 XFREE(aes->streamData, aes->heap, DYNAMIC_TYPE_AES);
10387 }
10388 #endif
10389
10390 #if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
10391 se050_aes_free(aes);
10392 #endif
10393
10394 }
10395
10396
wc_AesGetKeySize(Aes * aes,word32 * keySize)10397 int wc_AesGetKeySize(Aes* aes, word32* keySize)
10398 {
10399 int ret = 0;
10400
10401 if (aes == NULL || keySize == NULL) {
10402 return BAD_FUNC_ARG;
10403 }
10404 #if defined(WOLFSSL_CRYPTOCELL) && defined(WOLFSSL_CRYPTOCELL_AES)
10405 *keySize = aes->ctx.key.keySize;
10406 return ret;
10407 #endif
10408 switch (aes->rounds) {
10409 #ifdef WOLFSSL_AES_128
10410 case 10:
10411 *keySize = 16;
10412 break;
10413 #endif
10414 #ifdef WOLFSSL_AES_192
10415 case 12:
10416 *keySize = 24;
10417 break;
10418 #endif
10419 #ifdef WOLFSSL_AES_256
10420 case 14:
10421 *keySize = 32;
10422 break;
10423 #endif
10424 default:
10425 *keySize = 0;
10426 ret = BAD_FUNC_ARG;
10427 }
10428
10429 return ret;
10430 }
10431
10432 #endif /* !WOLFSSL_TI_CRYPT */
10433
10434 #ifdef HAVE_AES_ECB
10435 #if defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
10436 !defined(WOLFSSL_QNX_CAAM)
10437 /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
10438
10439 #elif defined(WOLFSSL_AFALG)
10440 /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
10441
10442 #elif defined(WOLFSSL_DEVCRYPTO_AES)
10443 /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
10444
10445 #elif defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
10446
10447 /* Software AES - ECB */
wc_AesEcbEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)10448 int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10449 {
10450 if ((in == NULL) || (out == NULL) || (aes == NULL))
10451 return BAD_FUNC_ARG;
10452
10453 return AES_ECB_encrypt(aes, in, out, sz);
10454 }
10455
10456
wc_AesEcbDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)10457 int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10458 {
10459 if ((in == NULL) || (out == NULL) || (aes == NULL))
10460 return BAD_FUNC_ARG;
10461
10462 return AES_ECB_decrypt(aes, in, out, sz);
10463 }
10464
10465 #else
10466
10467 /* Software AES - ECB */
_AesEcbEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)10468 static int _AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10469 {
10470 word32 blocks = sz / AES_BLOCK_SIZE;
10471
10472 #ifdef WOLFSSL_IMXRT_DCP
10473 if (aes->keylen == 16)
10474 return DCPAesEcbEncrypt(aes, out, in, sz);
10475 #endif
10476 while (blocks > 0) {
10477 wc_AesEncryptDirect(aes, out, in);
10478 out += AES_BLOCK_SIZE;
10479 in += AES_BLOCK_SIZE;
10480 blocks--;
10481 }
10482 return 0;
10483 }
10484
_AesEcbDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)10485 static int _AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10486 {
10487 word32 blocks = sz / AES_BLOCK_SIZE;
10488
10489 #ifdef WOLFSSL_IMXRT_DCP
10490 if (aes->keylen == 16)
10491 return DCPAesEcbDecrypt(aes, out, in, sz);
10492 #endif
10493 while (blocks > 0) {
10494 wc_AesDecryptDirect(aes, out, in);
10495 out += AES_BLOCK_SIZE;
10496 in += AES_BLOCK_SIZE;
10497 blocks--;
10498 }
10499 return 0;
10500 }
10501
wc_AesEcbEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)10502 int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10503 {
10504 int ret;
10505
10506 if ((in == NULL) || (out == NULL) || (aes == NULL))
10507 return BAD_FUNC_ARG;
10508
10509 SAVE_VECTOR_REGISTERS(return _svr_ret;);
10510 ret = _AesEcbEncrypt(aes, out, in, sz);
10511 RESTORE_VECTOR_REGISTERS();
10512
10513 return ret;
10514 }
10515
wc_AesEcbDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)10516 int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10517 {
10518 int ret;
10519
10520 if ((in == NULL) || (out == NULL) || (aes == NULL))
10521 return BAD_FUNC_ARG;
10522
10523 SAVE_VECTOR_REGISTERS(return _svr_ret;);
10524 ret = _AesEcbDecrypt(aes, out, in, sz);
10525 RESTORE_VECTOR_REGISTERS();
10526
10527 return ret;
10528 }
10529 #endif
10530 #endif /* HAVE_AES_ECB */
10531
10532 #if defined(WOLFSSL_AES_CFB) || defined(WOLFSSL_AES_OFB)
10533 /* Feedback AES mode
10534 *
10535 * aes structure holding key to use for encryption
10536 * out buffer to hold result of encryption (must be at least as large as input
10537 * buffer)
10538 * in buffer to encrypt
10539 * sz size of input buffer
10540 * mode flag to specify AES mode
10541 *
10542 * returns 0 on success and negative error values on failure
10543 */
10544 /* Software AES - CFB Encrypt */
wc_AesFeedbackEncrypt(Aes * aes,byte * out,const byte * in,word32 sz,byte mode)10545 static int wc_AesFeedbackEncrypt(Aes* aes, byte* out, const byte* in,
10546 word32 sz, byte mode)
10547 {
10548 byte* tmp = NULL;
10549 #ifdef WOLFSSL_AES_CFB
10550 byte* reg = NULL;
10551 #endif
10552
10553 if (aes == NULL || out == NULL || in == NULL) {
10554 return BAD_FUNC_ARG;
10555 }
10556
10557 #ifdef WOLFSSL_AES_CFB
10558 if (aes->left && sz) {
10559 reg = (byte*)aes->reg + AES_BLOCK_SIZE - aes->left;
10560 }
10561 #endif
10562
10563 /* consume any unused bytes left in aes->tmp */
10564 tmp = (byte*)aes->tmp + AES_BLOCK_SIZE - aes->left;
10565 while (aes->left && sz) {
10566 *(out) = *(in++) ^ *(tmp++);
10567 #ifdef WOLFSSL_AES_CFB
10568 if (mode == AES_CFB_MODE) {
10569 *(reg++) = *out;
10570 }
10571 #endif
10572 out++;
10573 aes->left--;
10574 sz--;
10575 }
10576
10577 SAVE_VECTOR_REGISTERS(return _svr_ret;);
10578
10579 while (sz >= AES_BLOCK_SIZE) {
10580 /* Using aes->tmp here for inline case i.e. in=out */
10581 wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
10582 #ifdef WOLFSSL_AES_OFB
10583 if (mode == AES_OFB_MODE) {
10584 XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
10585 }
10586 #endif
10587 xorbuf((byte*)aes->tmp, in, AES_BLOCK_SIZE);
10588 #ifdef WOLFSSL_AES_CFB
10589 if (mode == AES_CFB_MODE) {
10590 XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
10591 }
10592 #endif
10593 XMEMCPY(out, aes->tmp, AES_BLOCK_SIZE);
10594 out += AES_BLOCK_SIZE;
10595 in += AES_BLOCK_SIZE;
10596 sz -= AES_BLOCK_SIZE;
10597 aes->left = 0;
10598 }
10599
10600 /* encrypt left over data */
10601 if (sz) {
10602 wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
10603 aes->left = AES_BLOCK_SIZE;
10604 tmp = (byte*)aes->tmp;
10605 #ifdef WOLFSSL_AES_OFB
10606 if (mode == AES_OFB_MODE) {
10607 XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
10608 }
10609 #endif
10610 #ifdef WOLFSSL_AES_CFB
10611 reg = (byte*)aes->reg;
10612 #endif
10613
10614 while (sz--) {
10615 *(out) = *(in++) ^ *(tmp++);
10616 #ifdef WOLFSSL_AES_CFB
10617 if (mode == AES_CFB_MODE) {
10618 *(reg++) = *out;
10619 }
10620 #endif
10621 out++;
10622 aes->left--;
10623 }
10624 }
10625 RESTORE_VECTOR_REGISTERS();
10626
10627 return 0;
10628 }
10629
10630
10631 #ifdef HAVE_AES_DECRYPT
10632 /* CFB 128
10633 *
10634 * aes structure holding key to use for decryption
10635 * out buffer to hold result of decryption (must be at least as large as input
10636 * buffer)
10637 * in buffer to decrypt
10638 * sz size of input buffer
10639 *
10640 * returns 0 on success and negative error values on failure
10641 */
10642 /* Software AES - CFB Decrypt */
wc_AesFeedbackDecrypt(Aes * aes,byte * out,const byte * in,word32 sz,byte mode)10643 static int wc_AesFeedbackDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
10644 byte mode)
10645 {
10646 byte* tmp;
10647
10648 if (aes == NULL || out == NULL || in == NULL) {
10649 return BAD_FUNC_ARG;
10650 }
10651
10652 #ifdef WOLFSSL_AES_CFB
10653 /* check if more input needs copied over to aes->reg */
10654 if (aes->left && sz && mode == AES_CFB_MODE) {
10655 int size = min(aes->left, sz);
10656 XMEMCPY((byte*)aes->reg + AES_BLOCK_SIZE - aes->left, in, size);
10657 }
10658 #endif
10659
10660 /* consume any unused bytes left in aes->tmp */
10661 tmp = (byte*)aes->tmp + AES_BLOCK_SIZE - aes->left;
10662 while (aes->left && sz) {
10663 *(out++) = *(in++) ^ *(tmp++);
10664 aes->left--;
10665 sz--;
10666 }
10667
10668 SAVE_VECTOR_REGISTERS(return _svr_ret;);
10669
10670 while (sz > AES_BLOCK_SIZE) {
10671 /* Using aes->tmp here for inline case i.e. in=out */
10672 wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
10673 #ifdef WOLFSSL_AES_OFB
10674 if (mode == AES_OFB_MODE) {
10675 XMEMCPY((byte*)aes->reg, (byte*)aes->tmp, AES_BLOCK_SIZE);
10676 }
10677 #endif
10678 xorbuf((byte*)aes->tmp, in, AES_BLOCK_SIZE);
10679 #ifdef WOLFSSL_AES_CFB
10680 if (mode == AES_CFB_MODE) {
10681 XMEMCPY(aes->reg, in, AES_BLOCK_SIZE);
10682 }
10683 #endif
10684 XMEMCPY(out, (byte*)aes->tmp, AES_BLOCK_SIZE);
10685 out += AES_BLOCK_SIZE;
10686 in += AES_BLOCK_SIZE;
10687 sz -= AES_BLOCK_SIZE;
10688 aes->left = 0;
10689 }
10690
10691 /* decrypt left over data */
10692 if (sz) {
10693 wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
10694 #ifdef WOLFSSL_AES_CFB
10695 if (mode == AES_CFB_MODE) {
10696 XMEMCPY(aes->reg, in, sz);
10697 }
10698 #endif
10699 #ifdef WOLFSSL_AES_OFB
10700 if (mode == AES_OFB_MODE) {
10701 XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
10702 }
10703 #endif
10704
10705 aes->left = AES_BLOCK_SIZE;
10706 tmp = (byte*)aes->tmp;
10707
10708 while (sz--) {
10709 *(out++) = *(in++) ^ *(tmp++);
10710 aes->left--;
10711 }
10712 }
10713 RESTORE_VECTOR_REGISTERS();
10714
10715 return 0;
10716 }
10717 #endif /* HAVE_AES_DECRYPT */
10718 #endif /* WOLFSSL_AES_CFB */
10719
10720 #ifdef WOLFSSL_AES_CFB
10721 /* CFB 128
10722 *
10723 * aes structure holding key to use for encryption
10724 * out buffer to hold result of encryption (must be at least as large as input
10725 * buffer)
10726 * in buffer to encrypt
10727 * sz size of input buffer
10728 *
10729 * returns 0 on success and negative error values on failure
10730 */
10731 /* Software AES - CFB Encrypt */
wc_AesCfbEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)10732 int wc_AesCfbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10733 {
10734 return wc_AesFeedbackEncrypt(aes, out, in, sz, AES_CFB_MODE);
10735 }
10736
10737
10738 #ifdef HAVE_AES_DECRYPT
10739 /* CFB 128
10740 *
10741 * aes structure holding key to use for decryption
10742 * out buffer to hold result of decryption (must be at least as large as input
10743 * buffer)
10744 * in buffer to decrypt
10745 * sz size of input buffer
10746 *
10747 * returns 0 on success and negative error values on failure
10748 */
10749 /* Software AES - CFB Decrypt */
wc_AesCfbDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)10750 int wc_AesCfbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10751 {
10752 return wc_AesFeedbackDecrypt(aes, out, in, sz, AES_CFB_MODE);
10753 }
10754 #endif /* HAVE_AES_DECRYPT */
10755
10756
10757 /* shift the whole AES_BLOCK_SIZE array left by 8 or 1 bits */
shiftLeftArray(byte * ary,byte shift)10758 static void shiftLeftArray(byte* ary, byte shift)
10759 {
10760 int i;
10761
10762 if (shift == WOLFSSL_BIT_SIZE) {
10763 /* shifting over by 8 bits */
10764 for (i = 0; i < AES_BLOCK_SIZE - 1; i++) {
10765 ary[i] = ary[i+1];
10766 }
10767 ary[i] = 0;
10768 }
10769 else {
10770 /* shifting over by 7 or less bits */
10771 for (i = 0; i < AES_BLOCK_SIZE - 1; i++) {
10772 byte carry = ary[i+1] & (0XFF << (WOLFSSL_BIT_SIZE - shift));
10773 carry >>= (WOLFSSL_BIT_SIZE - shift);
10774 ary[i] = (ary[i] << shift) + carry;
10775 }
10776 ary[i] = ary[i] << shift;
10777 }
10778 }
10779
10780
10781 /* returns 0 on success and negative values on failure */
wc_AesFeedbackCFB8(Aes * aes,byte * out,const byte * in,word32 sz,byte dir)10782 static int wc_AesFeedbackCFB8(Aes* aes, byte* out, const byte* in,
10783 word32 sz, byte dir)
10784 {
10785 byte *pt;
10786
10787 if (aes == NULL || out == NULL || in == NULL) {
10788 return BAD_FUNC_ARG;
10789 }
10790
10791 if (sz == 0) {
10792 return 0;
10793 }
10794
10795 SAVE_VECTOR_REGISTERS(return _svr_ret;);
10796
10797 while (sz > 0) {
10798 wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
10799 if (dir == AES_DECRYPTION) {
10800 pt = (byte*)aes->reg;
10801
10802 /* LSB + CAT */
10803 shiftLeftArray(pt, WOLFSSL_BIT_SIZE);
10804 pt[AES_BLOCK_SIZE - 1] = in[0];
10805 }
10806
10807 /* MSB + XOR */
10808 #ifdef BIG_ENDIAN_ORDER
10809 ByteReverseWords(aes->tmp, aes->tmp, AES_BLOCK_SIZE);
10810 #endif
10811 out[0] = aes->tmp[0] ^ in[0];
10812 if (dir == AES_ENCRYPTION) {
10813 pt = (byte*)aes->reg;
10814
10815 /* LSB + CAT */
10816 shiftLeftArray(pt, WOLFSSL_BIT_SIZE);
10817 pt[AES_BLOCK_SIZE - 1] = out[0];
10818 }
10819
10820 out += 1;
10821 in += 1;
10822 sz -= 1;
10823 }
10824
10825 RESTORE_VECTOR_REGISTERS();
10826
10827 return 0;
10828 }
10829
10830
10831 /* returns 0 on success and negative values on failure */
wc_AesFeedbackCFB1(Aes * aes,byte * out,const byte * in,word32 sz,byte dir)10832 static int wc_AesFeedbackCFB1(Aes* aes, byte* out, const byte* in,
10833 word32 sz, byte dir)
10834 {
10835 byte tmp;
10836 byte cur = 0; /* hold current work in order to handle inline in=out */
10837 byte* pt;
10838 int bit = 7;
10839
10840 if (aes == NULL || out == NULL || in == NULL) {
10841 return BAD_FUNC_ARG;
10842 }
10843
10844 if (sz == 0) {
10845 return 0;
10846 }
10847
10848 SAVE_VECTOR_REGISTERS(return _svr_ret;);
10849
10850 while (sz > 0) {
10851 wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
10852 if (dir == AES_DECRYPTION) {
10853 pt = (byte*)aes->reg;
10854
10855 /* LSB + CAT */
10856 tmp = (0X01 << bit) & in[0];
10857 tmp = tmp >> bit;
10858 tmp &= 0x01;
10859 shiftLeftArray((byte*)aes->reg, 1);
10860 pt[AES_BLOCK_SIZE - 1] |= tmp;
10861 }
10862
10863 /* MSB + XOR */
10864 tmp = (0X01 << bit) & in[0];
10865 pt = (byte*)aes->tmp;
10866 tmp = (pt[0] >> 7) ^ (tmp >> bit);
10867 tmp &= 0x01;
10868 cur |= (tmp << bit);
10869
10870
10871 if (dir == AES_ENCRYPTION) {
10872 pt = (byte*)aes->reg;
10873
10874 /* LSB + CAT */
10875 shiftLeftArray((byte*)aes->reg, 1);
10876 pt[AES_BLOCK_SIZE - 1] |= tmp;
10877 }
10878
10879 bit--;
10880 if (bit < 0) {
10881 out[0] = cur;
10882 out += 1;
10883 in += 1;
10884 sz -= 1;
10885 bit = 7;
10886 cur = 0;
10887 }
10888 else {
10889 sz -= 1;
10890 }
10891 }
10892
10893 if (bit > 0 && bit < 7) {
10894 out[0] = cur;
10895 }
10896 RESTORE_VECTOR_REGISTERS();
10897
10898 return 0;
10899 }
10900
10901
10902 /* CFB 1
10903 *
10904 * aes structure holding key to use for encryption
10905 * out buffer to hold result of encryption (must be at least as large as input
10906 * buffer)
10907 * in buffer to encrypt (packed to left, i.e. 101 is 0x90)
10908 * sz size of input buffer in bits (0x1 would be size of 1 and 0xFF size of 8)
10909 *
10910 * returns 0 on success and negative values on failure
10911 */
wc_AesCfb1Encrypt(Aes * aes,byte * out,const byte * in,word32 sz)10912 int wc_AesCfb1Encrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10913 {
10914 return wc_AesFeedbackCFB1(aes, out, in, sz, AES_ENCRYPTION);
10915 }
10916
10917
10918 /* CFB 8
10919 *
10920 * aes structure holding key to use for encryption
10921 * out buffer to hold result of encryption (must be at least as large as input
10922 * buffer)
10923 * in buffer to encrypt
10924 * sz size of input buffer
10925 *
10926 * returns 0 on success and negative values on failure
10927 */
wc_AesCfb8Encrypt(Aes * aes,byte * out,const byte * in,word32 sz)10928 int wc_AesCfb8Encrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10929 {
10930 return wc_AesFeedbackCFB8(aes, out, in, sz, AES_ENCRYPTION);
10931 }
10932 #ifdef HAVE_AES_DECRYPT
10933
10934 /* CFB 1
10935 *
10936 * aes structure holding key to use for encryption
10937 * out buffer to hold result of encryption (must be at least as large as input
10938 * buffer)
10939 * in buffer to encrypt
10940 * sz size of input buffer in bits (0x1 would be size of 1 and 0xFF size of 8)
10941 *
10942 * returns 0 on success and negative values on failure
10943 */
wc_AesCfb1Decrypt(Aes * aes,byte * out,const byte * in,word32 sz)10944 int wc_AesCfb1Decrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10945 {
10946 return wc_AesFeedbackCFB1(aes, out, in, sz, AES_DECRYPTION);
10947 }
10948
10949
10950 /* CFB 8
10951 *
10952 * aes structure holding key to use for encryption
10953 * out buffer to hold result of encryption (must be at least as large as input
10954 * buffer)
10955 * in buffer to encrypt
10956 * sz size of input buffer
10957 *
10958 * returns 0 on success and negative values on failure
10959 */
wc_AesCfb8Decrypt(Aes * aes,byte * out,const byte * in,word32 sz)10960 int wc_AesCfb8Decrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10961 {
10962 return wc_AesFeedbackCFB8(aes, out, in, sz, AES_DECRYPTION);
10963 }
10964 #endif /* HAVE_AES_DECRYPT */
10965 #endif /* WOLFSSL_AES_CFB */
10966
10967 #ifdef WOLFSSL_AES_OFB
10968 /* OFB
10969 *
10970 * aes structure holding key to use for encryption
10971 * out buffer to hold result of encryption (must be at least as large as input
10972 * buffer)
10973 * in buffer to encrypt
10974 * sz size of input buffer
10975 *
10976 * returns 0 on success and negative error values on failure
10977 */
10978 /* Software AES - CFB Encrypt */
wc_AesOfbEncrypt(Aes * aes,byte * out,const byte * in,word32 sz)10979 int wc_AesOfbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10980 {
10981 return wc_AesFeedbackEncrypt(aes, out, in, sz, AES_OFB_MODE);
10982 }
10983
10984
10985 #ifdef HAVE_AES_DECRYPT
10986 /* OFB
10987 *
10988 * aes structure holding key to use for decryption
10989 * out buffer to hold result of decryption (must be at least as large as input
10990 * buffer)
10991 * in buffer to decrypt
10992 * sz size of input buffer
10993 *
10994 * returns 0 on success and negative error values on failure
10995 */
10996 /* Software AES - OFB Decrypt */
wc_AesOfbDecrypt(Aes * aes,byte * out,const byte * in,word32 sz)10997 int wc_AesOfbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
10998 {
10999 return wc_AesFeedbackDecrypt(aes, out, in, sz, AES_OFB_MODE);
11000 }
11001 #endif /* HAVE_AES_DECRYPT */
11002 #endif /* WOLFSSL_AES_OFB */
11003
11004
11005 #ifdef HAVE_AES_KEYWRAP
11006
11007 /* Initialize key wrap counter with value */
InitKeyWrapCounter(byte * inOutCtr,word32 value)11008 static WC_INLINE void InitKeyWrapCounter(byte* inOutCtr, word32 value)
11009 {
11010 int i;
11011 word32 bytes;
11012
11013 bytes = sizeof(word32);
11014 for (i = 0; i < (int)sizeof(word32); i++) {
11015 inOutCtr[i+sizeof(word32)] = (value >> ((bytes - 1) * 8)) & 0xFF;
11016 bytes--;
11017 }
11018 }
11019
11020 /* Increment key wrap counter */
IncrementKeyWrapCounter(byte * inOutCtr)11021 static WC_INLINE void IncrementKeyWrapCounter(byte* inOutCtr)
11022 {
11023 int i;
11024
11025 /* in network byte order so start at end and work back */
11026 for (i = KEYWRAP_BLOCK_SIZE - 1; i >= 0; i--) {
11027 if (++inOutCtr[i]) /* we're done unless we overflow */
11028 return;
11029 }
11030 }
11031
11032 /* Decrement key wrap counter */
DecrementKeyWrapCounter(byte * inOutCtr)11033 static WC_INLINE void DecrementKeyWrapCounter(byte* inOutCtr)
11034 {
11035 int i;
11036
11037 for (i = KEYWRAP_BLOCK_SIZE - 1; i >= 0; i--) {
11038 if (--inOutCtr[i] != 0xFF) /* we're done unless we underflow */
11039 return;
11040 }
11041 }
11042
wc_AesKeyWrap_ex(Aes * aes,const byte * in,word32 inSz,byte * out,word32 outSz,const byte * iv)11043 int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
11044 word32 outSz, const byte* iv)
11045 {
11046 word32 i;
11047 byte* r;
11048 int j;
11049
11050 byte t[KEYWRAP_BLOCK_SIZE];
11051 byte tmp[AES_BLOCK_SIZE];
11052
11053 /* n must be at least 2 64-bit blocks, output size is (n + 1) 8 bytes (64-bit) */
11054 if (aes == NULL || in == NULL || inSz < 2*KEYWRAP_BLOCK_SIZE ||
11055 out == NULL || outSz < (inSz + KEYWRAP_BLOCK_SIZE))
11056 return BAD_FUNC_ARG;
11057
11058 /* input must be multiple of 64-bits */
11059 if (inSz % KEYWRAP_BLOCK_SIZE != 0)
11060 return BAD_FUNC_ARG;
11061
11062 r = out + 8;
11063 XMEMCPY(r, in, inSz);
11064 XMEMSET(t, 0, sizeof(t));
11065
11066 /* user IV is optional */
11067 if (iv == NULL) {
11068 XMEMSET(tmp, 0xA6, KEYWRAP_BLOCK_SIZE);
11069 } else {
11070 XMEMCPY(tmp, iv, KEYWRAP_BLOCK_SIZE);
11071 }
11072
11073 SAVE_VECTOR_REGISTERS(return _svr_ret;);
11074
11075 for (j = 0; j <= 5; j++) {
11076 for (i = 1; i <= inSz / KEYWRAP_BLOCK_SIZE; i++) {
11077 /* load R[i] */
11078 XMEMCPY(tmp + KEYWRAP_BLOCK_SIZE, r, KEYWRAP_BLOCK_SIZE);
11079
11080 wc_AesEncryptDirect(aes, tmp, tmp);
11081
11082 /* calculate new A */
11083 IncrementKeyWrapCounter(t);
11084 xorbuf(tmp, t, KEYWRAP_BLOCK_SIZE);
11085
11086 /* save R[i] */
11087 XMEMCPY(r, tmp + KEYWRAP_BLOCK_SIZE, KEYWRAP_BLOCK_SIZE);
11088 r += KEYWRAP_BLOCK_SIZE;
11089 }
11090 r = out + KEYWRAP_BLOCK_SIZE;
11091 }
11092 RESTORE_VECTOR_REGISTERS();
11093
11094 /* C[0] = A */
11095 XMEMCPY(out, tmp, KEYWRAP_BLOCK_SIZE);
11096
11097 return inSz + KEYWRAP_BLOCK_SIZE;
11098 }
11099
11100 /* perform AES key wrap (RFC3394), return out sz on success, negative on err */
wc_AesKeyWrap(const byte * key,word32 keySz,const byte * in,word32 inSz,byte * out,word32 outSz,const byte * iv)11101 int wc_AesKeyWrap(const byte* key, word32 keySz, const byte* in, word32 inSz,
11102 byte* out, word32 outSz, const byte* iv)
11103 {
11104 #ifdef WOLFSSL_SMALL_STACK
11105 Aes *aes = NULL;
11106 #else
11107 Aes aes[1];
11108 #endif
11109 int ret;
11110
11111 if (key == NULL)
11112 return BAD_FUNC_ARG;
11113
11114 #ifdef WOLFSSL_SMALL_STACK
11115 if ((aes = (Aes *)XMALLOC(sizeof *aes, NULL,
11116 DYNAMIC_TYPE_AES)) == NULL)
11117 return MEMORY_E;
11118 #endif
11119
11120 ret = wc_AesInit(aes, NULL, INVALID_DEVID);
11121 if (ret != 0)
11122 goto out;
11123
11124 ret = wc_AesSetKey(aes, key, keySz, NULL, AES_ENCRYPTION);
11125 if (ret != 0) {
11126 wc_AesFree(aes);
11127 goto out;
11128 }
11129
11130 ret = wc_AesKeyWrap_ex(aes, in, inSz, out, outSz, iv);
11131
11132 wc_AesFree(aes);
11133
11134 out:
11135 #ifdef WOLFSSL_SMALL_STACK
11136 if (aes != NULL)
11137 XFREE(aes, NULL, DYNAMIC_TYPE_AES);
11138 #endif
11139
11140 return ret;
11141 }
11142
wc_AesKeyUnWrap_ex(Aes * aes,const byte * in,word32 inSz,byte * out,word32 outSz,const byte * iv)11143 int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
11144 word32 outSz, const byte* iv)
11145 {
11146 byte* r;
11147 word32 i, n;
11148 int j;
11149
11150 byte t[KEYWRAP_BLOCK_SIZE];
11151 byte tmp[AES_BLOCK_SIZE];
11152
11153 const byte* expIv;
11154 const byte defaultIV[] = {
11155 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6
11156 };
11157
11158 if (aes == NULL || in == NULL || inSz < 3 * KEYWRAP_BLOCK_SIZE ||
11159 out == NULL || outSz < (inSz - KEYWRAP_BLOCK_SIZE))
11160 return BAD_FUNC_ARG;
11161
11162 /* input must be multiple of 64-bits */
11163 if (inSz % KEYWRAP_BLOCK_SIZE != 0)
11164 return BAD_FUNC_ARG;
11165
11166 /* user IV optional */
11167 if (iv != NULL)
11168 expIv = iv;
11169 else
11170 expIv = defaultIV;
11171
11172 /* A = C[0], R[i] = C[i] */
11173 XMEMCPY(tmp, in, KEYWRAP_BLOCK_SIZE);
11174 XMEMCPY(out, in + KEYWRAP_BLOCK_SIZE, inSz - KEYWRAP_BLOCK_SIZE);
11175 XMEMSET(t, 0, sizeof(t));
11176
11177 SAVE_VECTOR_REGISTERS(return _svr_ret;);
11178
11179 /* initialize counter to 6n */
11180 n = (inSz - 1) / KEYWRAP_BLOCK_SIZE;
11181 InitKeyWrapCounter(t, 6 * n);
11182
11183 for (j = 5; j >= 0; j--) {
11184 for (i = n; i >= 1; i--) {
11185
11186 /* calculate A */
11187 xorbuf(tmp, t, KEYWRAP_BLOCK_SIZE);
11188 DecrementKeyWrapCounter(t);
11189
11190 /* load R[i], starting at end of R */
11191 r = out + ((i - 1) * KEYWRAP_BLOCK_SIZE);
11192 XMEMCPY(tmp + KEYWRAP_BLOCK_SIZE, r, KEYWRAP_BLOCK_SIZE);
11193 wc_AesDecryptDirect(aes, tmp, tmp);
11194
11195 /* save R[i] */
11196 XMEMCPY(r, tmp + KEYWRAP_BLOCK_SIZE, KEYWRAP_BLOCK_SIZE);
11197 }
11198 }
11199 RESTORE_VECTOR_REGISTERS();
11200
11201 /* verify IV */
11202 if (XMEMCMP(tmp, expIv, KEYWRAP_BLOCK_SIZE) != 0)
11203 return BAD_KEYWRAP_IV_E;
11204
11205 return inSz - KEYWRAP_BLOCK_SIZE;
11206 }
11207
wc_AesKeyUnWrap(const byte * key,word32 keySz,const byte * in,word32 inSz,byte * out,word32 outSz,const byte * iv)11208 int wc_AesKeyUnWrap(const byte* key, word32 keySz, const byte* in, word32 inSz,
11209 byte* out, word32 outSz, const byte* iv)
11210 {
11211 #ifdef WOLFSSL_SMALL_STACK
11212 Aes *aes = NULL;
11213 #else
11214 Aes aes[1];
11215 #endif
11216 int ret;
11217
11218 (void)iv;
11219
11220 if (key == NULL)
11221 return BAD_FUNC_ARG;
11222
11223 #ifdef WOLFSSL_SMALL_STACK
11224 if ((aes = (Aes *)XMALLOC(sizeof *aes, NULL,
11225 DYNAMIC_TYPE_AES)) == NULL)
11226 return MEMORY_E;
11227 #endif
11228
11229
11230 ret = wc_AesInit(aes, NULL, INVALID_DEVID);
11231 if (ret != 0)
11232 goto out;
11233
11234 ret = wc_AesSetKey(aes, key, keySz, NULL, AES_DECRYPTION);
11235 if (ret != 0) {
11236 wc_AesFree(aes);
11237 goto out;
11238 }
11239
11240 ret = wc_AesKeyUnWrap_ex(aes, in, inSz, out, outSz, iv);
11241
11242 wc_AesFree(aes);
11243
11244 out:
11245 #ifdef WOLFSSL_SMALL_STACK
11246 if (aes)
11247 XFREE(aes, NULL, DYNAMIC_TYPE_AES);
11248 #endif
11249
11250 return ret;
11251 }
11252
11253 #endif /* HAVE_AES_KEYWRAP */
11254
11255 #ifdef WOLFSSL_AES_XTS
11256
11257 /* Galios Field to use */
11258 #define GF_XTS 0x87
11259
11260 /* This is to help with setting keys to correct encrypt or decrypt type.
11261 *
11262 * tweak AES key for tweak in XTS
11263 * aes AES key for encrypt/decrypt process
11264 * key buffer holding aes key | tweak key
11265 * len length of key buffer in bytes. Should be twice that of key size. i.e.
11266 * 32 for a 16 byte key.
11267 * dir direction, either AES_ENCRYPTION or AES_DECRYPTION
11268 * heap heap hint to use for memory. Can be NULL
11269 * devId id to use with async crypto. Can be 0
11270 *
11271 * Note: is up to user to call wc_AesFree on tweak and aes key when done.
11272 *
11273 * return 0 on success
11274 */
wc_AesXtsSetKey(XtsAes * aes,const byte * key,word32 len,int dir,void * heap,int devId)11275 int wc_AesXtsSetKey(XtsAes* aes, const byte* key, word32 len, int dir,
11276 void* heap, int devId)
11277 {
11278 word32 keySz;
11279 int ret = 0;
11280
11281 if (aes == NULL || key == NULL) {
11282 return BAD_FUNC_ARG;
11283 }
11284
11285 if ((ret = wc_AesInit(&aes->tweak, heap, devId)) != 0) {
11286 return ret;
11287 }
11288 if ((ret = wc_AesInit(&aes->aes, heap, devId)) != 0) {
11289 return ret;
11290 }
11291
11292 keySz = len/2;
11293 if (keySz != 16 && keySz != 32) {
11294 WOLFSSL_MSG("Unsupported key size");
11295 return WC_KEY_SIZE_E;
11296 }
11297
11298 if ((ret = wc_AesSetKey(&aes->aes, key, keySz, NULL, dir)) == 0) {
11299 ret = wc_AesSetKey(&aes->tweak, key + keySz, keySz, NULL,
11300 AES_ENCRYPTION);
11301 if (ret != 0) {
11302 wc_AesFree(&aes->aes);
11303 }
11304 }
11305
11306 return ret;
11307 }
11308
11309
11310 /* This is used to free up resources used by Aes structs
11311 *
11312 * aes AES keys to free
11313 *
11314 * return 0 on success
11315 */
wc_AesXtsFree(XtsAes * aes)11316 int wc_AesXtsFree(XtsAes* aes)
11317 {
11318 if (aes != NULL) {
11319 wc_AesFree(&aes->aes);
11320 wc_AesFree(&aes->tweak);
11321 }
11322
11323 return 0;
11324 }
11325
11326
11327 /* Same process as wc_AesXtsEncrypt but uses a word64 type as the tweak value
11328 * instead of a byte array. This just converts the word64 to a byte array and
11329 * calls wc_AesXtsEncrypt.
11330 *
11331 * aes AES keys to use for block encrypt/decrypt
11332 * out output buffer to hold cipher text
11333 * in input plain text buffer to encrypt
11334 * sz size of both out and in buffers
11335 * sector value to use for tweak
11336 *
11337 * returns 0 on success
11338 */
wc_AesXtsEncryptSector(XtsAes * aes,byte * out,const byte * in,word32 sz,word64 sector)11339 int wc_AesXtsEncryptSector(XtsAes* aes, byte* out, const byte* in,
11340 word32 sz, word64 sector)
11341 {
11342 byte* pt;
11343 byte i[AES_BLOCK_SIZE];
11344
11345 XMEMSET(i, 0, AES_BLOCK_SIZE);
11346 #ifdef BIG_ENDIAN_ORDER
11347 sector = ByteReverseWord64(sector);
11348 #endif
11349 pt = (byte*)§or;
11350 XMEMCPY(i, pt, sizeof(word64));
11351
11352 return wc_AesXtsEncrypt(aes, out, in, sz, (const byte*)i, AES_BLOCK_SIZE);
11353 }
11354
11355
11356 /* Same process as wc_AesXtsDecrypt but uses a word64 type as the tweak value
11357 * instead of a byte array. This just converts the word64 to a byte array.
11358 *
11359 * aes AES keys to use for block encrypt/decrypt
11360 * out output buffer to hold plain text
11361 * in input cipher text buffer to encrypt
11362 * sz size of both out and in buffers
11363 * sector value to use for tweak
11364 *
11365 * returns 0 on success
11366 */
wc_AesXtsDecryptSector(XtsAes * aes,byte * out,const byte * in,word32 sz,word64 sector)11367 int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz,
11368 word64 sector)
11369 {
11370 byte* pt;
11371 byte i[AES_BLOCK_SIZE];
11372
11373 XMEMSET(i, 0, AES_BLOCK_SIZE);
11374 #ifdef BIG_ENDIAN_ORDER
11375 sector = ByteReverseWord64(sector);
11376 #endif
11377 pt = (byte*)§or;
11378 XMEMCPY(i, pt, sizeof(word64));
11379
11380 return wc_AesXtsDecrypt(aes, out, in, sz, (const byte*)i, AES_BLOCK_SIZE);
11381 }
11382
11383 #ifdef HAVE_AES_ECB
11384 /* helper function for encrypting / decrypting full buffer at once */
_AesXtsHelper(Aes * aes,byte * out,const byte * in,word32 sz,int dir)11385 static int _AesXtsHelper(Aes* aes, byte* out, const byte* in, word32 sz, int dir)
11386 {
11387 word32 outSz = sz;
11388 word32 totalSz = (sz / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; /* total bytes */
11389 byte* pt = out;
11390
11391 outSz -= AES_BLOCK_SIZE;
11392
11393 while (outSz > 0) {
11394 word32 j;
11395 byte carry = 0;
11396
11397 /* multiply by shift left and propagate carry */
11398 for (j = 0; j < AES_BLOCK_SIZE && outSz > 0; j++, outSz--) {
11399 byte tmpC;
11400
11401 tmpC = (pt[j] >> 7) & 0x01;
11402 pt[j+AES_BLOCK_SIZE] = ((pt[j] << 1) + carry) & 0xFF;
11403 carry = tmpC;
11404 }
11405 if (carry) {
11406 pt[AES_BLOCK_SIZE] ^= GF_XTS;
11407 }
11408
11409 pt += AES_BLOCK_SIZE;
11410 }
11411
11412 xorbuf(out, in, totalSz);
11413 if (dir == AES_ENCRYPTION) {
11414 return _AesEcbEncrypt(aes, out, out, totalSz);
11415 }
11416 else {
11417 return _AesEcbDecrypt(aes, out, out, totalSz);
11418 }
11419 }
11420 #endif /* HAVE_AES_ECB */
11421
11422
11423 /* AES with XTS mode. (XTS) XEX encryption with Tweak and cipher text Stealing.
11424 *
11425 * xaes AES keys to use for block encrypt/decrypt
11426 * out output buffer to hold cipher text
11427 * in input plain text buffer to encrypt
11428 * sz size of both out and in buffers
11429 * i value to use for tweak
11430 * iSz size of i buffer, should always be AES_BLOCK_SIZE but having this input
11431 * adds a sanity check on how the user calls the function.
11432 *
11433 * returns 0 on success
11434 */
11435 /* Software AES - XTS Encrypt */
wc_AesXtsEncrypt(XtsAes * xaes,byte * out,const byte * in,word32 sz,const byte * i,word32 iSz)11436 int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
11437 const byte* i, word32 iSz)
11438 {
11439 int ret = 0;
11440 word32 blocks = (sz / AES_BLOCK_SIZE);
11441 Aes *aes, *tweak;
11442
11443 if (xaes == NULL || out == NULL || in == NULL) {
11444 return BAD_FUNC_ARG;
11445 }
11446
11447 aes = &xaes->aes;
11448 tweak = &xaes->tweak;
11449
11450 if (iSz < AES_BLOCK_SIZE) {
11451 return BAD_FUNC_ARG;
11452 }
11453
11454 if (blocks > 0) {
11455 byte tmp[AES_BLOCK_SIZE];
11456
11457 XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES
11458 * key setup passed to encrypt direct*/
11459
11460 SAVE_VECTOR_REGISTERS(return _svr_ret;);
11461
11462 wc_AesEncryptDirect(tweak, tmp, i);
11463
11464 #ifdef HAVE_AES_ECB
11465 /* encrypt all of buffer at once when possible */
11466 if (in != out) { /* can not handle inline */
11467 XMEMCPY(out, tmp, AES_BLOCK_SIZE);
11468 if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0) {
11469 RESTORE_VECTOR_REGISTERS();
11470 return ret;
11471 }
11472 }
11473 #endif
11474
11475 while (blocks > 0) {
11476 word32 j;
11477 byte carry = 0;
11478
11479 #ifdef HAVE_AES_ECB
11480 if (in == out)
11481 #endif
11482 { /* check for if inline */
11483 byte buf[AES_BLOCK_SIZE];
11484
11485 XMEMCPY(buf, in, AES_BLOCK_SIZE);
11486 xorbuf(buf, tmp, AES_BLOCK_SIZE);
11487 wc_AesEncryptDirect(aes, out, buf);
11488 }
11489 xorbuf(out, tmp, AES_BLOCK_SIZE);
11490
11491 /* multiply by shift left and propagate carry */
11492 for (j = 0; j < AES_BLOCK_SIZE; j++) {
11493 byte tmpC;
11494
11495 tmpC = (tmp[j] >> 7) & 0x01;
11496 tmp[j] = ((tmp[j] << 1) + carry) & 0xFF;
11497 carry = tmpC;
11498 }
11499 if (carry) {
11500 tmp[0] ^= GF_XTS;
11501 }
11502
11503 in += AES_BLOCK_SIZE;
11504 out += AES_BLOCK_SIZE;
11505 sz -= AES_BLOCK_SIZE;
11506 blocks--;
11507 }
11508
11509 /* stealing operation of XTS to handle left overs */
11510 if (sz > 0) {
11511 byte buf[AES_BLOCK_SIZE];
11512
11513 XMEMCPY(buf, out - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
11514 if (sz >= AES_BLOCK_SIZE) { /* extra sanity check before copy */
11515 return BUFFER_E;
11516 }
11517 XMEMCPY(out, buf, sz);
11518 XMEMCPY(buf, in, sz);
11519
11520 xorbuf(buf, tmp, AES_BLOCK_SIZE);
11521 wc_AesEncryptDirect(aes, out - AES_BLOCK_SIZE, buf);
11522 xorbuf(out - AES_BLOCK_SIZE, tmp, AES_BLOCK_SIZE);
11523 }
11524 RESTORE_VECTOR_REGISTERS();
11525 }
11526 else {
11527 WOLFSSL_MSG("Plain text input too small for encryption");
11528 return BAD_FUNC_ARG;
11529 }
11530
11531 return ret;
11532 }
11533
11534
11535 /* Same process as encryption but Aes key is AES_DECRYPTION type.
11536 *
11537 * xaes AES keys to use for block encrypt/decrypt
11538 * out output buffer to hold plain text
11539 * in input cipher text buffer to decrypt
11540 * sz size of both out and in buffers
11541 * i value to use for tweak
11542 * iSz size of i buffer, should always be AES_BLOCK_SIZE but having this input
11543 * adds a sanity check on how the user calls the function.
11544 *
11545 * returns 0 on success
11546 */
11547 /* Software AES - XTS Decrypt */
wc_AesXtsDecrypt(XtsAes * xaes,byte * out,const byte * in,word32 sz,const byte * i,word32 iSz)11548 int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
11549 const byte* i, word32 iSz)
11550 {
11551 int ret = 0;
11552 word32 blocks = (sz / AES_BLOCK_SIZE);
11553 Aes *aes, *tweak;
11554
11555 if (xaes == NULL || out == NULL || in == NULL) {
11556 return BAD_FUNC_ARG;
11557 }
11558
11559 aes = &xaes->aes;
11560 tweak = &xaes->tweak;
11561
11562 if (iSz < AES_BLOCK_SIZE) {
11563 return BAD_FUNC_ARG;
11564 }
11565
11566 if (blocks > 0) {
11567 word32 j;
11568 byte carry = 0;
11569 byte tmp[AES_BLOCK_SIZE];
11570 byte stl = (sz % AES_BLOCK_SIZE);
11571
11572 XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES
11573 * key setup passed to decrypt direct*/
11574
11575 SAVE_VECTOR_REGISTERS(return _svr_ret;);
11576
11577 wc_AesEncryptDirect(tweak, tmp, i);
11578
11579 /* if Stealing then break out of loop one block early to handle special
11580 * case */
11581 if (stl > 0) {
11582 blocks--;
11583 }
11584
11585 #ifdef HAVE_AES_ECB
11586 /* decrypt all of buffer at once when possible */
11587 if (in != out) { /* can not handle inline */
11588 XMEMCPY(out, tmp, AES_BLOCK_SIZE);
11589 if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0) {
11590 RESTORE_VECTOR_REGISTERS();
11591 return ret;
11592 }
11593 }
11594 #endif
11595
11596 while (blocks > 0) {
11597 #ifdef HAVE_AES_ECB
11598 if (in == out)
11599 #endif
11600 { /* check for if inline */
11601 byte buf[AES_BLOCK_SIZE];
11602
11603 XMEMCPY(buf, in, AES_BLOCK_SIZE);
11604 xorbuf(buf, tmp, AES_BLOCK_SIZE);
11605 wc_AesDecryptDirect(aes, out, buf);
11606 }
11607 xorbuf(out, tmp, AES_BLOCK_SIZE);
11608
11609 /* multiply by shift left and propagate carry */
11610 for (j = 0; j < AES_BLOCK_SIZE; j++) {
11611 byte tmpC;
11612
11613 tmpC = (tmp[j] >> 7) & 0x01;
11614 tmp[j] = ((tmp[j] << 1) + carry) & 0xFF;
11615 carry = tmpC;
11616 }
11617 if (carry) {
11618 tmp[0] ^= GF_XTS;
11619 }
11620 carry = 0;
11621
11622 in += AES_BLOCK_SIZE;
11623 out += AES_BLOCK_SIZE;
11624 sz -= AES_BLOCK_SIZE;
11625 blocks--;
11626 }
11627
11628 /* stealing operation of XTS to handle left overs */
11629 if (sz >= AES_BLOCK_SIZE) {
11630 byte buf[AES_BLOCK_SIZE];
11631 byte tmp2[AES_BLOCK_SIZE];
11632
11633 /* multiply by shift left and propagate carry */
11634 for (j = 0; j < AES_BLOCK_SIZE; j++) {
11635 byte tmpC;
11636
11637 tmpC = (tmp[j] >> 7) & 0x01;
11638 tmp2[j] = ((tmp[j] << 1) + carry) & 0xFF;
11639 carry = tmpC;
11640 }
11641 if (carry) {
11642 tmp2[0] ^= GF_XTS;
11643 }
11644
11645 XMEMCPY(buf, in, AES_BLOCK_SIZE);
11646 xorbuf(buf, tmp2, AES_BLOCK_SIZE);
11647 wc_AesDecryptDirect(aes, out, buf);
11648 xorbuf(out, tmp2, AES_BLOCK_SIZE);
11649
11650 /* tmp2 holds partial | last */
11651 XMEMCPY(tmp2, out, AES_BLOCK_SIZE);
11652 in += AES_BLOCK_SIZE;
11653 out += AES_BLOCK_SIZE;
11654 sz -= AES_BLOCK_SIZE;
11655
11656 /* Make buffer with end of cipher text | last */
11657 XMEMCPY(buf, tmp2, AES_BLOCK_SIZE);
11658 if (sz >= AES_BLOCK_SIZE) { /* extra sanity check before copy */
11659 RESTORE_VECTOR_REGISTERS();
11660 return BUFFER_E;
11661 }
11662 XMEMCPY(buf, in, sz);
11663 XMEMCPY(out, tmp2, sz);
11664
11665 xorbuf(buf, tmp, AES_BLOCK_SIZE);
11666 wc_AesDecryptDirect(aes, tmp2, buf);
11667 xorbuf(tmp2, tmp, AES_BLOCK_SIZE);
11668 XMEMCPY(out - AES_BLOCK_SIZE, tmp2, AES_BLOCK_SIZE);
11669 }
11670 RESTORE_VECTOR_REGISTERS();
11671 }
11672 else {
11673 WOLFSSL_MSG("Plain text input too small for encryption");
11674 return BAD_FUNC_ARG;
11675 }
11676
11677 return ret;
11678 }
11679
11680 #endif /* WOLFSSL_AES_XTS */
11681
11682 #endif /* HAVE_FIPS */
11683 #endif /* !NO_AES */
11684