1 // rijndael_simd.cpp - written and placed in the public domain by
2 // Jeffrey Walton, Uri Blumenthal and Marcel Raad.
3 // AES-NI code originally written by Wei Dai.
4 //
5 // This source file uses intrinsics and built-ins to gain access to
6 // AES-NI, ARMv8a AES and Power8 AES instructions. A separate source
7 // file is needed because additional CXXFLAGS are required to enable
8 // the appropriate instructions sets in some build configurations.
9 //
10 // ARMv8a AES code based on CriticalBlue code from Johannes Schneiders,
11 // Skip Hovsmith and Barry O'Rourke for the mbedTLS project. Stepping
12 // mbedTLS under a debugger was helped for us to determine problems
13 // with our subkey generation and scheduling.
14 //
15 // AltiVec and Power8 code based on http://github.com/noloader/AES-Intrinsics and
16 // http://www.ibm.com/developerworks/library/se-power8-in-core-cryptography/
17 // For Power8 do not remove the casts, even when const-ness is cast away. It causes
18 // failed compiles and a 0.3 to 0.6 cpb drop in performance. The IBM documentation
19 // absolutely sucks. Thanks to Andy Polyakov, Paul R and Trudeaun for answering
20 // questions and filling the gaps in the IBM documentation.
21 //
22
23 #include "pch.h"
24 #include "config.h"
25 #include "misc.h"
26
27 #if (CRYPTOPP_AESNI_AVAILABLE)
28 # include "adv_simd.h"
29 # include <emmintrin.h>
30 # include <smmintrin.h>
31 # include <wmmintrin.h>
32 #endif
33
34 #if (CRYPTOPP_ARM_NEON_HEADER)
35 # include "adv_simd.h"
36 # include <arm_neon.h>
37 #endif
38
39 #if (CRYPTOPP_ARM_ACLE_HEADER)
40 # include "adv_simd.h"
41 # include <stdint.h>
42 # include <arm_acle.h>
43 #endif
44
45 #if defined(_M_ARM64)
46 # include "adv_simd.h"
47 #endif
48
49 #if defined(CRYPTOPP_POWER8_AES_AVAILABLE)
50 # include "adv_simd.h"
51 # include "ppc_simd.h"
52 #endif
53
54 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
55 # include <signal.h>
56 # include <setjmp.h>
57 #endif
58
59 #ifndef EXCEPTION_EXECUTE_HANDLER
60 # define EXCEPTION_EXECUTE_HANDLER 1
61 #endif
62
63 // Clang intrinsic casts, http://bugs.llvm.org/show_bug.cgi?id=20670
64 #define M128_CAST(x) ((__m128i *)(void *)(x))
65 #define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
66
67 // Squash MS LNK4221 and libtool warnings
68 extern const char RIJNDAEL_SIMD_FNAME[] = __FILE__;
69
NAMESPACE_BEGIN(CryptoPP)70 NAMESPACE_BEGIN(CryptoPP)
71
72 // ************************* Feature Probes ************************* //
73
74 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
75 extern "C" {
76 typedef void (*SigHandler)(int);
77
78 static jmp_buf s_jmpSIGILL;
79 static void SigIllHandler(int)
80 {
81 longjmp(s_jmpSIGILL, 1);
82 }
83 }
84 #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
85
86 #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
CPU_ProbeAES()87 bool CPU_ProbeAES()
88 {
89 #if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES)
90 return false;
91 #elif (CRYPTOPP_ARM_AES_AVAILABLE)
92 # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
93 volatile bool result = true;
94 __try
95 {
96 // AES encrypt and decrypt
97 uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
98 uint8x16_t r1 = vaeseq_u8(data, key);
99 uint8x16_t r2 = vaesdq_u8(data, key);
100 r1 = vaesmcq_u8(r1);
101 r2 = vaesimcq_u8(r2);
102
103 result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
104 }
105 __except (EXCEPTION_EXECUTE_HANDLER)
106 {
107 return false;
108 }
109 return result;
110 # else
111
112 // longjmp and clobber warnings. Volatile is required.
113 // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
114 volatile bool result = true;
115
116 volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
117 if (oldHandler == SIG_ERR)
118 return false;
119
120 volatile sigset_t oldMask;
121 if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask))
122 {
123 signal(SIGILL, oldHandler);
124 return false;
125 }
126
127 if (setjmp(s_jmpSIGILL))
128 result = false;
129 else
130 {
131 uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
132 uint8x16_t r1 = vaeseq_u8(data, key);
133 uint8x16_t r2 = vaesdq_u8(data, key);
134 r1 = vaesmcq_u8(r1);
135 r2 = vaesimcq_u8(r2);
136
137 // Hack... GCC optimizes away the code and returns true
138 result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
139 }
140
141 sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
142 signal(SIGILL, oldHandler);
143 return result;
144 # endif
145 #else
146 return false;
147 #endif // CRYPTOPP_ARM_AES_AVAILABLE
148 }
149 #endif // ARM32 or ARM64
150
151 // ***************************** ARMv8 ***************************** //
152
153 #if (CRYPTOPP_ARM_AES_AVAILABLE)
154
155 ANONYMOUS_NAMESPACE_BEGIN
156
ARMV8_Enc_Block(uint64x2_t & data,const word32 * subkeys,unsigned int rounds)157 inline void ARMV8_Enc_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
158 {
159 CRYPTOPP_ASSERT(subkeys);
160 const byte *keys = reinterpret_cast<const byte*>(subkeys);
161 uint8x16_t block = vreinterpretq_u8_u64(data);
162
163 // AES single round encryption
164 block = vaeseq_u8(block, vld1q_u8(keys+0*16));
165 // AES mix columns
166 block = vaesmcq_u8(block);
167
168 for (unsigned int i=1; i<rounds-1; i+=2)
169 {
170 // AES single round encryption
171 block = vaeseq_u8(block, vld1q_u8(keys+i*16));
172 // AES mix columns
173 block = vaesmcq_u8(block);
174 // AES single round encryption
175 block = vaeseq_u8(block, vld1q_u8(keys+(i+1)*16));
176 // AES mix columns
177 block = vaesmcq_u8(block);
178 }
179
180 // AES single round encryption
181 block = vaeseq_u8(block, vld1q_u8(keys+(rounds-1)*16));
182 // Final Add (bitwise Xor)
183 block = veorq_u8(block, vld1q_u8(keys+rounds*16));
184
185 data = vreinterpretq_u64_u8(block);
186 }
187
ARMV8_Enc_6_Blocks(uint64x2_t & data0,uint64x2_t & data1,uint64x2_t & data2,uint64x2_t & data3,uint64x2_t & data4,uint64x2_t & data5,const word32 * subkeys,unsigned int rounds)188 inline void ARMV8_Enc_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
189 uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
190 const word32 *subkeys, unsigned int rounds)
191 {
192 CRYPTOPP_ASSERT(subkeys);
193 const byte *keys = reinterpret_cast<const byte*>(subkeys);
194
195 uint8x16_t block0 = vreinterpretq_u8_u64(data0);
196 uint8x16_t block1 = vreinterpretq_u8_u64(data1);
197 uint8x16_t block2 = vreinterpretq_u8_u64(data2);
198 uint8x16_t block3 = vreinterpretq_u8_u64(data3);
199 uint8x16_t block4 = vreinterpretq_u8_u64(data4);
200 uint8x16_t block5 = vreinterpretq_u8_u64(data5);
201
202 uint8x16_t key;
203 for (unsigned int i=0; i<rounds-1; ++i)
204 {
205 key = vld1q_u8(keys+i*16);
206 // AES single round encryption
207 block0 = vaeseq_u8(block0, key);
208 // AES mix columns
209 block0 = vaesmcq_u8(block0);
210 // AES single round encryption
211 block1 = vaeseq_u8(block1, key);
212 // AES mix columns
213 block1 = vaesmcq_u8(block1);
214 // AES single round encryption
215 block2 = vaeseq_u8(block2, key);
216 // AES mix columns
217 block2 = vaesmcq_u8(block2);
218 // AES single round encryption
219 block3 = vaeseq_u8(block3, key);
220 // AES mix columns
221 block3 = vaesmcq_u8(block3);
222 // AES single round encryption
223 block4 = vaeseq_u8(block4, key);
224 // AES mix columns
225 block4 = vaesmcq_u8(block4);
226 // AES single round encryption
227 block5 = vaeseq_u8(block5, key);
228 // AES mix columns
229 block5 = vaesmcq_u8(block5);
230 }
231
232 // AES single round encryption
233 key = vld1q_u8(keys+(rounds-1)*16);
234 block0 = vaeseq_u8(block0, key);
235 block1 = vaeseq_u8(block1, key);
236 block2 = vaeseq_u8(block2, key);
237 block3 = vaeseq_u8(block3, key);
238 block4 = vaeseq_u8(block4, key);
239 block5 = vaeseq_u8(block5, key);
240
241 // Final Add (bitwise Xor)
242 key = vld1q_u8(keys+rounds*16);
243 data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
244 data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
245 data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
246 data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
247 data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
248 data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
249 }
250
ARMV8_Dec_Block(uint64x2_t & data,const word32 * subkeys,unsigned int rounds)251 inline void ARMV8_Dec_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
252 {
253 CRYPTOPP_ASSERT(subkeys);
254 const byte *keys = reinterpret_cast<const byte*>(subkeys);
255 uint8x16_t block = vreinterpretq_u8_u64(data);
256
257 // AES single round decryption
258 block = vaesdq_u8(block, vld1q_u8(keys+0*16));
259 // AES inverse mix columns
260 block = vaesimcq_u8(block);
261
262 for (unsigned int i=1; i<rounds-1; i+=2)
263 {
264 // AES single round decryption
265 block = vaesdq_u8(block, vld1q_u8(keys+i*16));
266 // AES inverse mix columns
267 block = vaesimcq_u8(block);
268 // AES single round decryption
269 block = vaesdq_u8(block, vld1q_u8(keys+(i+1)*16));
270 // AES inverse mix columns
271 block = vaesimcq_u8(block);
272 }
273
274 // AES single round decryption
275 block = vaesdq_u8(block, vld1q_u8(keys+(rounds-1)*16));
276 // Final Add (bitwise Xor)
277 block = veorq_u8(block, vld1q_u8(keys+rounds*16));
278
279 data = vreinterpretq_u64_u8(block);
280 }
281
ARMV8_Dec_6_Blocks(uint64x2_t & data0,uint64x2_t & data1,uint64x2_t & data2,uint64x2_t & data3,uint64x2_t & data4,uint64x2_t & data5,const word32 * subkeys,unsigned int rounds)282 inline void ARMV8_Dec_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
283 uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
284 const word32 *subkeys, unsigned int rounds)
285 {
286 CRYPTOPP_ASSERT(subkeys);
287 const byte *keys = reinterpret_cast<const byte*>(subkeys);
288
289 uint8x16_t block0 = vreinterpretq_u8_u64(data0);
290 uint8x16_t block1 = vreinterpretq_u8_u64(data1);
291 uint8x16_t block2 = vreinterpretq_u8_u64(data2);
292 uint8x16_t block3 = vreinterpretq_u8_u64(data3);
293 uint8x16_t block4 = vreinterpretq_u8_u64(data4);
294 uint8x16_t block5 = vreinterpretq_u8_u64(data5);
295
296 uint8x16_t key;
297 for (unsigned int i=0; i<rounds-1; ++i)
298 {
299 key = vld1q_u8(keys+i*16);
300 // AES single round decryption
301 block0 = vaesdq_u8(block0, key);
302 // AES inverse mix columns
303 block0 = vaesimcq_u8(block0);
304 // AES single round decryption
305 block1 = vaesdq_u8(block1, key);
306 // AES inverse mix columns
307 block1 = vaesimcq_u8(block1);
308 // AES single round decryption
309 block2 = vaesdq_u8(block2, key);
310 // AES inverse mix columns
311 block2 = vaesimcq_u8(block2);
312 // AES single round decryption
313 block3 = vaesdq_u8(block3, key);
314 // AES inverse mix columns
315 block3 = vaesimcq_u8(block3);
316 // AES single round decryption
317 block4 = vaesdq_u8(block4, key);
318 // AES inverse mix columns
319 block4 = vaesimcq_u8(block4);
320 // AES single round decryption
321 block5 = vaesdq_u8(block5, key);
322 // AES inverse mix columns
323 block5 = vaesimcq_u8(block5);
324 }
325
326 // AES single round decryption
327 key = vld1q_u8(keys+(rounds-1)*16);
328 block0 = vaesdq_u8(block0, key);
329 block1 = vaesdq_u8(block1, key);
330 block2 = vaesdq_u8(block2, key);
331 block3 = vaesdq_u8(block3, key);
332 block4 = vaesdq_u8(block4, key);
333 block5 = vaesdq_u8(block5, key);
334
335 // Final Add (bitwise Xor)
336 key = vld1q_u8(keys+rounds*16);
337 data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
338 data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
339 data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
340 data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
341 data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
342 data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
343 }
344
345 ANONYMOUS_NAMESPACE_END
346
Rijndael_Enc_AdvancedProcessBlocks_ARMV8(const word32 * subKeys,size_t rounds,const byte * inBlocks,const byte * xorBlocks,byte * outBlocks,size_t length,word32 flags)347 size_t Rijndael_Enc_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
348 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
349 {
350 return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Enc_Block, ARMV8_Enc_6_Blocks,
351 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
352 }
353
Rijndael_Dec_AdvancedProcessBlocks_ARMV8(const word32 * subKeys,size_t rounds,const byte * inBlocks,const byte * xorBlocks,byte * outBlocks,size_t length,word32 flags)354 size_t Rijndael_Dec_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
355 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
356 {
357 return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Dec_Block, ARMV8_Dec_6_Blocks,
358 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
359 }
360
361 #endif // CRYPTOPP_ARM_AES_AVAILABLE
362
363 // ***************************** AES-NI ***************************** //
364
365 #if (CRYPTOPP_AESNI_AVAILABLE)
366
367 ANONYMOUS_NAMESPACE_BEGIN
368
369 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
370 CRYPTOPP_ALIGN_DATA(16)
371 const word32 s_rconLE[] = {
372 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36
373 };
374
AESNI_Enc_Block(__m128i & block,MAYBE_CONST word32 * subkeys,unsigned int rounds)375 inline void AESNI_Enc_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
376 {
377 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
378
379 block = _mm_xor_si128(block, skeys[0]);
380 for (unsigned int i=1; i<rounds-1; i+=2)
381 {
382 block = _mm_aesenc_si128(block, skeys[i]);
383 block = _mm_aesenc_si128(block, skeys[i+1]);
384 }
385 block = _mm_aesenc_si128(block, skeys[rounds-1]);
386 block = _mm_aesenclast_si128(block, skeys[rounds]);
387 }
388
AESNI_Enc_4_Blocks(__m128i & block0,__m128i & block1,__m128i & block2,__m128i & block3,MAYBE_CONST word32 * subkeys,unsigned int rounds)389 inline void AESNI_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
390 MAYBE_CONST word32 *subkeys, unsigned int rounds)
391 {
392 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
393
394 __m128i rk = skeys[0];
395 block0 = _mm_xor_si128(block0, rk);
396 block1 = _mm_xor_si128(block1, rk);
397 block2 = _mm_xor_si128(block2, rk);
398 block3 = _mm_xor_si128(block3, rk);
399 for (unsigned int i=1; i<rounds; i++)
400 {
401 rk = skeys[i];
402 block0 = _mm_aesenc_si128(block0, rk);
403 block1 = _mm_aesenc_si128(block1, rk);
404 block2 = _mm_aesenc_si128(block2, rk);
405 block3 = _mm_aesenc_si128(block3, rk);
406 }
407 rk = skeys[rounds];
408 block0 = _mm_aesenclast_si128(block0, rk);
409 block1 = _mm_aesenclast_si128(block1, rk);
410 block2 = _mm_aesenclast_si128(block2, rk);
411 block3 = _mm_aesenclast_si128(block3, rk);
412 }
413
AESNI_Dec_Block(__m128i & block,MAYBE_CONST word32 * subkeys,unsigned int rounds)414 inline void AESNI_Dec_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
415 {
416 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
417
418 block = _mm_xor_si128(block, skeys[0]);
419 for (unsigned int i=1; i<rounds-1; i+=2)
420 {
421 block = _mm_aesdec_si128(block, skeys[i]);
422 block = _mm_aesdec_si128(block, skeys[i+1]);
423 }
424 block = _mm_aesdec_si128(block, skeys[rounds-1]);
425 block = _mm_aesdeclast_si128(block, skeys[rounds]);
426 }
427
AESNI_Dec_4_Blocks(__m128i & block0,__m128i & block1,__m128i & block2,__m128i & block3,MAYBE_CONST word32 * subkeys,unsigned int rounds)428 inline void AESNI_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
429 MAYBE_CONST word32 *subkeys, unsigned int rounds)
430 {
431 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
432
433 __m128i rk = skeys[0];
434 block0 = _mm_xor_si128(block0, rk);
435 block1 = _mm_xor_si128(block1, rk);
436 block2 = _mm_xor_si128(block2, rk);
437 block3 = _mm_xor_si128(block3, rk);
438 for (unsigned int i=1; i<rounds; i++)
439 {
440 rk = skeys[i];
441 block0 = _mm_aesdec_si128(block0, rk);
442 block1 = _mm_aesdec_si128(block1, rk);
443 block2 = _mm_aesdec_si128(block2, rk);
444 block3 = _mm_aesdec_si128(block3, rk);
445 }
446 rk = skeys[rounds];
447 block0 = _mm_aesdeclast_si128(block0, rk);
448 block1 = _mm_aesdeclast_si128(block1, rk);
449 block2 = _mm_aesdeclast_si128(block2, rk);
450 block3 = _mm_aesdeclast_si128(block3, rk);
451 }
452
453 ANONYMOUS_NAMESPACE_END
454
Rijndael_UncheckedSetKey_SSE4_AESNI(const byte * userKey,size_t keyLen,word32 * rk)455 void Rijndael_UncheckedSetKey_SSE4_AESNI(const byte *userKey, size_t keyLen, word32 *rk)
456 {
457 const size_t rounds = keyLen / 4 + 6;
458 const word32 *rc = s_rconLE;
459
460 __m128i temp = _mm_loadu_si128(M128_CAST(userKey+keyLen-16));
461 std::memcpy(rk, userKey, keyLen);
462
463 // keySize: m_key allocates 4*(rounds+1) word32's.
464 const size_t keySize = 4*(rounds+1);
465 const word32* end = rk + keySize;
466
467 while (true)
468 {
469 rk[keyLen/4] = rk[0] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 3) ^ *(rc++);
470 rk[keyLen/4+1] = rk[1] ^ rk[keyLen/4];
471 rk[keyLen/4+2] = rk[2] ^ rk[keyLen/4+1];
472 rk[keyLen/4+3] = rk[3] ^ rk[keyLen/4+2];
473
474 if (rk + keyLen/4 + 4 == end)
475 break;
476
477 if (keyLen == 24)
478 {
479 rk[10] = rk[ 4] ^ rk[ 9];
480 rk[11] = rk[ 5] ^ rk[10];
481 temp = _mm_insert_epi32(temp, rk[11], 3);
482 }
483 else if (keyLen == 32)
484 {
485 temp = _mm_insert_epi32(temp, rk[11], 3);
486 rk[12] = rk[ 4] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 2);
487 rk[13] = rk[ 5] ^ rk[12];
488 rk[14] = rk[ 6] ^ rk[13];
489 rk[15] = rk[ 7] ^ rk[14];
490 temp = _mm_insert_epi32(temp, rk[15], 3);
491 }
492 else
493 {
494 temp = _mm_insert_epi32(temp, rk[7], 3);
495 }
496
497 rk += keyLen/4;
498 }
499 }
500
Rijndael_UncheckedSetKeyRev_AESNI(word32 * key,unsigned int rounds)501 void Rijndael_UncheckedSetKeyRev_AESNI(word32 *key, unsigned int rounds)
502 {
503 unsigned int i, j;
504 __m128i temp;
505
506 vec_swap(*M128_CAST(key), *M128_CAST(key+4*rounds));
507
508 for (i = 4, j = 4*rounds-4; i < j; i += 4, j -= 4)
509 {
510 temp = _mm_aesimc_si128(*M128_CAST(key+i));
511 *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+j));
512 *M128_CAST(key+j) = temp;
513 }
514
515 *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+i));
516 }
517
Rijndael_Enc_AdvancedProcessBlocks_AESNI(const word32 * subKeys,size_t rounds,const byte * inBlocks,const byte * xorBlocks,byte * outBlocks,size_t length,word32 flags)518 size_t Rijndael_Enc_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
519 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
520 {
521 // SunCC workaround
522 MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
523 MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
524 MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
525
526 return AdvancedProcessBlocks128_4x1_SSE(AESNI_Enc_Block, AESNI_Enc_4_Blocks,
527 sk, rounds, ib, xb, outBlocks, length, flags);
528 }
529
Rijndael_Dec_AdvancedProcessBlocks_AESNI(const word32 * subKeys,size_t rounds,const byte * inBlocks,const byte * xorBlocks,byte * outBlocks,size_t length,word32 flags)530 size_t Rijndael_Dec_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
531 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
532 {
533 MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
534 MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
535 MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
536
537 return AdvancedProcessBlocks128_4x1_SSE(AESNI_Dec_Block, AESNI_Dec_4_Blocks,
538 sk, rounds, ib, xb, outBlocks, length, flags);
539 }
540
541 #endif // CRYPTOPP_AESNI_AVAILABLE
542
543 // ************************** Power 8 Crypto ************************** //
544
545 #if (CRYPTOPP_POWER8_AES_AVAILABLE)
546
547 ANONYMOUS_NAMESPACE_BEGIN
548
549 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
550 CRYPTOPP_ALIGN_DATA(16)
551 static const uint32_t s_rconBE[] = {
552 0x01000000, 0x02000000, 0x04000000, 0x08000000,
553 0x10000000, 0x20000000, 0x40000000, 0x80000000,
554 0x1B000000, 0x36000000
555 };
556
POWER8_Enc_Block(uint32x4_p & block,const word32 * subkeys,unsigned int rounds)557 inline void POWER8_Enc_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
558 {
559 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
560 const byte *keys = reinterpret_cast<const byte*>(subkeys);
561
562 uint32x4_p k = VecLoadAligned(keys);
563 block = VecXor(block, k);
564
565 for (size_t i=1; i<rounds-1; i+=2)
566 {
567 block = VecEncrypt(block, VecLoadAligned( i*16, keys));
568 block = VecEncrypt(block, VecLoadAligned((i+1)*16, keys));
569 }
570
571 block = VecEncrypt(block, VecLoadAligned((rounds-1)*16, keys));
572 block = VecEncryptLast(block, VecLoadAligned(rounds*16, keys));
573 }
574
POWER8_Enc_6_Blocks(uint32x4_p & block0,uint32x4_p & block1,uint32x4_p & block2,uint32x4_p & block3,uint32x4_p & block4,uint32x4_p & block5,const word32 * subkeys,unsigned int rounds)575 inline void POWER8_Enc_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
576 uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
577 uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
578 {
579 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
580 const byte *keys = reinterpret_cast<const byte*>(subkeys);
581
582 uint32x4_p k = VecLoadAligned(keys);
583 block0 = VecXor(block0, k);
584 block1 = VecXor(block1, k);
585 block2 = VecXor(block2, k);
586 block3 = VecXor(block3, k);
587 block4 = VecXor(block4, k);
588 block5 = VecXor(block5, k);
589
590 for (size_t i=1; i<rounds; ++i)
591 {
592 k = VecLoadAligned(i*16, keys);
593 block0 = VecEncrypt(block0, k);
594 block1 = VecEncrypt(block1, k);
595 block2 = VecEncrypt(block2, k);
596 block3 = VecEncrypt(block3, k);
597 block4 = VecEncrypt(block4, k);
598 block5 = VecEncrypt(block5, k);
599 }
600
601 k = VecLoadAligned(rounds*16, keys);
602 block0 = VecEncryptLast(block0, k);
603 block1 = VecEncryptLast(block1, k);
604 block2 = VecEncryptLast(block2, k);
605 block3 = VecEncryptLast(block3, k);
606 block4 = VecEncryptLast(block4, k);
607 block5 = VecEncryptLast(block5, k);
608 }
609
POWER8_Dec_Block(uint32x4_p & block,const word32 * subkeys,unsigned int rounds)610 inline void POWER8_Dec_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
611 {
612 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
613 const byte *keys = reinterpret_cast<const byte*>(subkeys);
614
615 uint32x4_p k = VecLoadAligned(rounds*16, keys);
616 block = VecXor(block, k);
617
618 for (size_t i=rounds-1; i>1; i-=2)
619 {
620 block = VecDecrypt(block, VecLoadAligned( i*16, keys));
621 block = VecDecrypt(block, VecLoadAligned((i-1)*16, keys));
622 }
623
624 block = VecDecrypt(block, VecLoadAligned(16, keys));
625 block = VecDecryptLast(block, VecLoadAligned(0, keys));
626 }
627
POWER8_Dec_6_Blocks(uint32x4_p & block0,uint32x4_p & block1,uint32x4_p & block2,uint32x4_p & block3,uint32x4_p & block4,uint32x4_p & block5,const word32 * subkeys,unsigned int rounds)628 inline void POWER8_Dec_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
629 uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
630 uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
631 {
632 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
633 const byte *keys = reinterpret_cast<const byte*>(subkeys);
634
635 uint32x4_p k = VecLoadAligned(rounds*16, keys);
636 block0 = VecXor(block0, k);
637 block1 = VecXor(block1, k);
638 block2 = VecXor(block2, k);
639 block3 = VecXor(block3, k);
640 block4 = VecXor(block4, k);
641 block5 = VecXor(block5, k);
642
643 for (size_t i=rounds-1; i>0; --i)
644 {
645 k = VecLoadAligned(i*16, keys);
646 block0 = VecDecrypt(block0, k);
647 block1 = VecDecrypt(block1, k);
648 block2 = VecDecrypt(block2, k);
649 block3 = VecDecrypt(block3, k);
650 block4 = VecDecrypt(block4, k);
651 block5 = VecDecrypt(block5, k);
652 }
653
654 k = VecLoadAligned(0, keys);
655 block0 = VecDecryptLast(block0, k);
656 block1 = VecDecryptLast(block1, k);
657 block2 = VecDecryptLast(block2, k);
658 block3 = VecDecryptLast(block3, k);
659 block4 = VecDecryptLast(block4, k);
660 block5 = VecDecryptLast(block5, k);
661 }
662
663 ANONYMOUS_NAMESPACE_END
664
Rijndael_UncheckedSetKey_POWER8(const byte * userKey,size_t keyLen,word32 * rk,const byte * Se)665 void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32* rk, const byte* Se)
666 {
667 const size_t rounds = keyLen / 4 + 6;
668 const word32 *rc = s_rconBE;
669 word32 *rkey = rk, temp;
670
671 GetUserKey(BIG_ENDIAN_ORDER, rkey, keyLen/4, userKey, keyLen);
672
673 // keySize: m_key allocates 4*(rounds+1) word32's.
674 const size_t keySize = 4*(rounds+1);
675 const word32* end = rkey + keySize;
676
677 while (true)
678 {
679 temp = rkey[keyLen/4-1];
680 word32 x = (word32(Se[GETBYTE(temp, 2)]) << 24) ^ (word32(Se[GETBYTE(temp, 1)]) << 16) ^
681 (word32(Se[GETBYTE(temp, 0)]) << 8) ^ Se[GETBYTE(temp, 3)];
682 rkey[keyLen/4] = rkey[0] ^ x ^ *(rc++);
683 rkey[keyLen/4+1] = rkey[1] ^ rkey[keyLen/4];
684 rkey[keyLen/4+2] = rkey[2] ^ rkey[keyLen/4+1];
685 rkey[keyLen/4+3] = rkey[3] ^ rkey[keyLen/4+2];
686
687 if (rkey + keyLen/4 + 4 == end)
688 break;
689
690 if (keyLen == 24)
691 {
692 rkey[10] = rkey[ 4] ^ rkey[ 9];
693 rkey[11] = rkey[ 5] ^ rkey[10];
694 }
695 else if (keyLen == 32)
696 {
697 temp = rkey[11];
698 rkey[12] = rkey[ 4] ^ (word32(Se[GETBYTE(temp, 3)]) << 24) ^ (word32(Se[GETBYTE(temp, 2)]) << 16) ^ (word32(Se[GETBYTE(temp, 1)]) << 8) ^ Se[GETBYTE(temp, 0)];
699 rkey[13] = rkey[ 5] ^ rkey[12];
700 rkey[14] = rkey[ 6] ^ rkey[13];
701 rkey[15] = rkey[ 7] ^ rkey[14];
702 }
703 rkey += keyLen/4;
704 }
705
706 #if (CRYPTOPP_LITTLE_ENDIAN)
707 rkey = rk;
708 const uint8x16_p mask = {12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3};
709
710 unsigned int i=0;
711 for (i=0; i<rounds; i+=2, rkey+=8)
712 {
713 VecStore(VecPermute(VecLoad(rkey+0), mask), rkey+0);
714 VecStore(VecPermute(VecLoad(rkey+4), mask), rkey+4);
715 }
716
717 for ( ; i<rounds+1; i++, rkey+=4)
718 VecStore(VecPermute(VecLoad(rkey), mask), rkey);
719 #endif
720 }
721
Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 * subKeys,size_t rounds,const byte * inBlocks,const byte * xorBlocks,byte * outBlocks,size_t length,word32 flags)722 size_t Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
723 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
724 {
725 return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Enc_Block, POWER8_Enc_6_Blocks,
726 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
727 }
728
Rijndael_Dec_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 * subKeys,size_t rounds,const byte * inBlocks,const byte * xorBlocks,byte * outBlocks,size_t length,word32 flags)729 size_t Rijndael_Dec_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
730 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
731 {
732 return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Dec_Block, POWER8_Dec_6_Blocks,
733 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
734 }
735
736 #endif // CRYPTOPP_POWER8_AES_AVAILABLE
737 NAMESPACE_END
738