1 // cpu.h - originally written and placed in the public domain by Wei Dai
2 // updated for ARM and PowerPC by Jeffrey Walton.
3 // updated to split CPU_Query() and CPU_Probe() by Jeffrey Walton.
4
5 /// \file cpu.h
6 /// \brief Functions for CPU features and intrinsics
7 /// \details The CPU functions are used in IA-32, ARM and PowerPC code paths. The
8 /// functions provide cpu specific feature testing on IA-32, ARM and PowerPC machines.
9 /// \details Feature detection uses CPUID on IA-32, like Intel and AMD. On other platforms
10 /// a two-part strategy is used. First, the library attempts to *Query* the OS for a feature,
11 /// like using Linux getauxval() or android_getCpuFeatures(). If that fails, then *Probe*
12 /// the cpu executing an instruction and an observe a SIGILL if unsupported. The general
13 /// pattern used by the library is:
14 /// <pre>
15 /// g_hasCRC32 = CPU_QueryCRC32() || CPU_ProbeCRC32();
16 /// g_hasPMULL = CPU_QueryPMULL() || CPU_ProbePMULL();
17 /// g_hasAES = CPU_QueryAES() || CPU_ProbeAES();
18 /// </pre>
19 /// \details Generally speaking, CPU_Query() is in the source file <tt>cpu.cpp</tt> because it
20 /// does not require special architectural flags. CPU_Probe() is in a source file that receives
21 /// architectural flags, like <tt>sse_simd.cpp</tt>, <tt>neon_simd.cpp</tt> and
22 /// <tt>ppc_simd.cpp</tt>. For example, compiling <tt>neon_simd.cpp</tt> on an ARM64 machine will
23 /// have <tt>-march=armv8-a</tt> applied during a compile to make the instruction set architecture
24 /// (ISA) available.
25 /// \details The cpu probes are expensive when compared to a standard OS feature query. The library
26 /// also avoids probes on Apple platforms because Apple's signal handling for SIGILLs appears to
27 /// corrupt memory. CPU_Probe() will unconditionally return false for Apple platforms. OpenSSL
28 /// experienced the same problem and moved away from SIGILL probes on Apple.
29
30 #ifndef CRYPTOPP_CPU_H
31 #define CRYPTOPP_CPU_H
32
33 #include "config.h"
34
35 // Issue 340
36 #if CRYPTOPP_GCC_DIAGNOSTIC_AVAILABLE
37 # pragma GCC diagnostic push
38 # pragma GCC diagnostic ignored "-Wconversion"
39 # pragma GCC diagnostic ignored "-Wsign-conversion"
40 #endif
41
42 // Applies to both X86/X32/X64 and ARM32/ARM64
43 #if defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)
44 #define NEW_LINE "\n"
45 #define INTEL_PREFIX ".intel_syntax;"
46 #define INTEL_NOPREFIX ".intel_syntax;"
47 #define ATT_PREFIX ".att_syntax;"
48 #define ATT_NOPREFIX ".att_syntax;"
49 #elif defined(__GNUC__)
50 #define NEW_LINE
51 #define INTEL_PREFIX ".intel_syntax prefix;"
52 #define INTEL_NOPREFIX ".intel_syntax noprefix;"
53 #define ATT_PREFIX ".att_syntax prefix;"
54 #define ATT_NOPREFIX ".att_syntax noprefix;"
55 #else
56 #define NEW_LINE
57 #define INTEL_PREFIX
58 #define INTEL_NOPREFIX
59 #define ATT_PREFIX
60 #define ATT_NOPREFIX
61 #endif
62
63 #ifdef CRYPTOPP_GENERATE_X64_MASM
64
65 #define CRYPTOPP_X86_ASM_AVAILABLE
66 #define CRYPTOPP_BOOL_X64 1
67 #define CRYPTOPP_SSE2_ASM_AVAILABLE 1
68 #define NAMESPACE_END
69
70 #else
71
72 NAMESPACE_BEGIN(CryptoPP)
73
74 // ***************************** IA-32 ***************************** //
75
76 #if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64 || CRYPTOPP_DOXYGEN_PROCESSING
77
78 #define CRYPTOPP_CPUID_AVAILABLE 1
79
80 // Hide from Doxygen
81 #ifndef CRYPTOPP_DOXYGEN_PROCESSING
82 // These should not be used directly
83 extern CRYPTOPP_DLL bool g_x86DetectionDone;
84 extern CRYPTOPP_DLL bool g_hasSSE2;
85 extern CRYPTOPP_DLL bool g_hasSSSE3;
86 extern CRYPTOPP_DLL bool g_hasSSE41;
87 extern CRYPTOPP_DLL bool g_hasSSE42;
88 extern CRYPTOPP_DLL bool g_hasMOVBE;
89 extern CRYPTOPP_DLL bool g_hasAESNI;
90 extern CRYPTOPP_DLL bool g_hasCLMUL;
91 extern CRYPTOPP_DLL bool g_hasAVX;
92 extern CRYPTOPP_DLL bool g_hasAVX2;
93 extern CRYPTOPP_DLL bool g_hasSHA;
94 extern CRYPTOPP_DLL bool g_hasADX;
95 extern CRYPTOPP_DLL bool g_isP4;
96 extern CRYPTOPP_DLL bool g_hasRDRAND;
97 extern CRYPTOPP_DLL bool g_hasRDSEED;
98 extern CRYPTOPP_DLL bool g_hasPadlockRNG;
99 extern CRYPTOPP_DLL bool g_hasPadlockACE;
100 extern CRYPTOPP_DLL bool g_hasPadlockACE2;
101 extern CRYPTOPP_DLL bool g_hasPadlockPHE;
102 extern CRYPTOPP_DLL bool g_hasPadlockPMM;
103 extern CRYPTOPP_DLL word32 g_cacheLineSize;
104
105 CRYPTOPP_DLL void CRYPTOPP_API DetectX86Features();
106 CRYPTOPP_DLL bool CRYPTOPP_API CpuId(word32 func, word32 subfunc, word32 output[4]);
107 #endif // CRYPTOPP_DOXYGEN_PROCESSING
108
109 /// \name IA-32 CPU FEATURES
110 //@{
111
112 /// \brief Determine SSE2 availability
113 /// \return true if SSE2 is determined to be available, false otherwise
114 /// \details MMX, SSE and SSE2 are core processor features for x86_64, and
115 /// the function return value is based on OSXSAVE. On i386 both
116 /// SSE2 and OSXSAVE are used for the return value.
117 /// \note This function is only available on Intel IA-32 platforms
HasSSE2()118 inline bool HasSSE2()
119 {
120 #if (CRYPTOPP_SSE2_ASM_AVAILABLE || CRYPTOPP_SSE2_INTRIN_AVAILABLE)
121 if (!g_x86DetectionDone)
122 DetectX86Features();
123 return g_hasSSE2;
124 #else
125 return false;
126 #endif
127 }
128
129 /// \brief Determine SSSE3 availability
130 /// \return true if SSSE3 is determined to be available, false otherwise
131 /// \details HasSSSE3() is a runtime check performed using CPUID
132 /// \note This function is only available on Intel IA-32 platforms
HasSSSE3()133 inline bool HasSSSE3()
134 {
135 #if CRYPTOPP_SSSE3_AVAILABLE
136 if (!g_x86DetectionDone)
137 DetectX86Features();
138 return g_hasSSSE3;
139 #else
140 return false;
141 #endif
142 }
143
144 /// \brief Determine SSE4.1 availability
145 /// \return true if SSE4.1 is determined to be available, false otherwise
146 /// \details HasSSE41() is a runtime check performed using CPUID
147 /// \note This function is only available on Intel IA-32 platforms
HasSSE41()148 inline bool HasSSE41()
149 {
150 #if CRYPTOPP_SSE41_AVAILABLE
151 if (!g_x86DetectionDone)
152 DetectX86Features();
153 return g_hasSSE41;
154 #else
155 return false;
156 #endif
157 }
158
159 /// \brief Determine SSE4.2 availability
160 /// \return true if SSE4.2 is determined to be available, false otherwise
161 /// \details HasSSE42() is a runtime check performed using CPUID
162 /// \note This function is only available on Intel IA-32 platforms
HasSSE42()163 inline bool HasSSE42()
164 {
165 #if CRYPTOPP_SSE42_AVAILABLE
166 if (!g_x86DetectionDone)
167 DetectX86Features();
168 return g_hasSSE42;
169 #else
170 return false;
171 #endif
172 }
173
174 /// \brief Determine MOVBE availability
175 /// \return true if MOVBE is determined to be available, false otherwise
176 /// \details HasMOVBE() is a runtime check performed using CPUID
177 /// \since Crypto++ 8.3
178 /// \note This function is only available on Intel IA-32 platforms
HasMOVBE()179 inline bool HasMOVBE()
180 {
181 #if CRYPTOPP_SSE42_AVAILABLE
182 if (!g_x86DetectionDone)
183 DetectX86Features();
184 return g_hasMOVBE;
185 #else
186 return false;
187 #endif
188 }
189
190 /// \brief Determine AES-NI availability
191 /// \return true if AES-NI is determined to be available, false otherwise
192 /// \details HasAESNI() is a runtime check performed using CPUID
193 /// \since Crypto++ 5.6.1
194 /// \note This function is only available on Intel IA-32 platforms
HasAESNI()195 inline bool HasAESNI()
196 {
197 #if CRYPTOPP_AESNI_AVAILABLE
198 if (!g_x86DetectionDone)
199 DetectX86Features();
200 return g_hasAESNI;
201 #else
202 return false;
203 #endif
204 }
205
206 /// \brief Determine Carryless Multiply availability
207 /// \return true if pclmulqdq is determined to be available, false otherwise
208 /// \details HasCLMUL() is a runtime check performed using CPUID
209 /// \since Crypto++ 5.6.1
210 /// \note This function is only available on Intel IA-32 platforms
HasCLMUL()211 inline bool HasCLMUL()
212 {
213 #if CRYPTOPP_CLMUL_AVAILABLE
214 if (!g_x86DetectionDone)
215 DetectX86Features();
216 return g_hasCLMUL;
217 #else
218 return false;
219 #endif
220 }
221
222 /// \brief Determine SHA availability
223 /// \return true if SHA is determined to be available, false otherwise
224 /// \details HasSHA() is a runtime check performed using CPUID
225 /// \since Crypto++ 6.0
226 /// \note This function is only available on Intel IA-32 platforms
HasSHA()227 inline bool HasSHA()
228 {
229 #if CRYPTOPP_SHANI_AVAILABLE
230 if (!g_x86DetectionDone)
231 DetectX86Features();
232 return g_hasSHA;
233 #else
234 return false;
235 #endif
236 }
237
238 /// \brief Determine ADX availability
239 /// \return true if ADX is determined to be available, false otherwise
240 /// \details HasADX() is a runtime check performed using CPUID
241 /// \since Crypto++ 7.0
242 /// \note This function is only available on Intel IA-32 platforms
HasADX()243 inline bool HasADX()
244 {
245 #if CRYPTOPP_ADX_AVAILABLE
246 if (!g_x86DetectionDone)
247 DetectX86Features();
248 return g_hasADX;
249 #else
250 return false;
251 #endif
252 }
253
254 /// \brief Determine AVX availability
255 /// \return true if AVX is determined to be available, false otherwise
256 /// \details HasAVX() is a runtime check performed using CPUID
257 /// \since Crypto++ 8.0
258 /// \note This function is only available on Intel IA-32 platforms
HasAVX()259 inline bool HasAVX()
260 {
261 #if CRYPTOPP_AVX_AVAILABLE
262 if (!g_x86DetectionDone)
263 DetectX86Features();
264 return g_hasAVX;
265 #else
266 return false;
267 #endif
268 }
269
270 /// \brief Determine AVX2 availability
271 /// \return true if AVX2 is determined to be available, false otherwise
272 /// \details HasAVX2() is a runtime check performed using CPUID
273 /// \since Crypto++ 8.0
274 /// \note This function is only available on Intel IA-32 platforms
HasAVX2()275 inline bool HasAVX2()
276 {
277 #if CRYPTOPP_AVX2_AVAILABLE
278 if (!g_x86DetectionDone)
279 DetectX86Features();
280 return g_hasAVX2;
281 #else
282 return false;
283 #endif
284 }
285
286 /// \brief Determine RDRAND availability
287 /// \return true if RDRAND is determined to be available, false otherwise
288 /// \details HasRDRAND() is a runtime check performed using CPUID
289 /// \note This function is only available on Intel IA-32 platforms
HasRDRAND()290 inline bool HasRDRAND()
291 {
292 #if CRYPTOPP_RDRAND_AVAILABLE
293 if (!g_x86DetectionDone)
294 DetectX86Features();
295 return g_hasRDRAND;
296 #else
297 return false;
298 #endif
299 }
300
301 /// \brief Determine RDSEED availability
302 /// \return true if RDSEED is determined to be available, false otherwise
303 /// \details HasRDSEED() is a runtime check performed using CPUID
304 /// \note This function is only available on Intel IA-32 platforms
HasRDSEED()305 inline bool HasRDSEED()
306 {
307 #if CRYPTOPP_RDSEED_AVAILABLE
308 if (!g_x86DetectionDone)
309 DetectX86Features();
310 return g_hasRDSEED;
311 #else
312 return false;
313 #endif
314 }
315
316 /// \brief Determine Padlock RNG availability
317 /// \return true if VIA Padlock RNG is determined to be available, false otherwise
318 /// \details HasPadlockRNG() is a runtime check performed using CPUID
319 /// \note This function is only available on Intel IA-32 platforms
HasPadlockRNG()320 inline bool HasPadlockRNG()
321 {
322 #if CRYPTOPP_PADLOCK_RNG_AVAILABLE
323 if (!g_x86DetectionDone)
324 DetectX86Features();
325 return g_hasPadlockRNG;
326 #else
327 return false;
328 #endif
329 }
330
331 /// \brief Determine Padlock ACE availability
332 /// \return true if VIA Padlock ACE is determined to be available, false otherwise
333 /// \details HasPadlockACE() is a runtime check performed using CPUID
334 /// \note This function is only available on Intel IA-32 platforms
HasPadlockACE()335 inline bool HasPadlockACE()
336 {
337 #if CRYPTOPP_PADLOCK_ACE_AVAILABLE
338 if (!g_x86DetectionDone)
339 DetectX86Features();
340 return g_hasPadlockACE;
341 #else
342 return false;
343 #endif
344 }
345
346 /// \brief Determine Padlock ACE2 availability
347 /// \return true if VIA Padlock ACE2 is determined to be available, false otherwise
348 /// \details HasPadlockACE2() is a runtime check performed using CPUID
349 /// \note This function is only available on Intel IA-32 platforms
HasPadlockACE2()350 inline bool HasPadlockACE2()
351 {
352 #if CRYPTOPP_PADLOCK_ACE2_AVAILABLE
353 if (!g_x86DetectionDone)
354 DetectX86Features();
355 return g_hasPadlockACE2;
356 #else
357 return false;
358 #endif
359 }
360
361 /// \brief Determine Padlock PHE availability
362 /// \return true if VIA Padlock PHE is determined to be available, false otherwise
363 /// \details HasPadlockPHE() is a runtime check performed using CPUID
364 /// \note This function is only available on Intel IA-32 platforms
HasPadlockPHE()365 inline bool HasPadlockPHE()
366 {
367 #if CRYPTOPP_PADLOCK_PHE_AVAILABLE
368 if (!g_x86DetectionDone)
369 DetectX86Features();
370 return g_hasPadlockPHE;
371 #else
372 return false;
373 #endif
374 }
375
376 /// \brief Determine Padlock PMM availability
377 /// \return true if VIA Padlock PMM is determined to be available, false otherwise
378 /// \details HasPadlockPMM() is a runtime check performed using CPUID
379 /// \note This function is only available on Intel IA-32 platforms
HasPadlockPMM()380 inline bool HasPadlockPMM()
381 {
382 #if CRYPTOPP_PADLOCK_PMM_AVAILABLE
383 if (!g_x86DetectionDone)
384 DetectX86Features();
385 return g_hasPadlockPMM;
386 #else
387 return false;
388 #endif
389 }
390
391 /// \brief Determine if the CPU is an Intel P4
392 /// \return true if the CPU is a P4, false otherwise
393 /// \details IsP4() is a runtime check performed using CPUID
394 /// \note This function is only available on Intel IA-32 platforms
IsP4()395 inline bool IsP4()
396 {
397 if (!g_x86DetectionDone)
398 DetectX86Features();
399 return g_isP4;
400 }
401
402 /// \brief Provides the cache line size
403 /// \return lower bound on the size of a cache line in bytes, if available
404 /// \details GetCacheLineSize() returns the lower bound on the size of a cache line, if it
405 /// is available. If the value is not available at runtime, then 32 is returned for a 32-bit
406 /// processor and 64 is returned for a 64-bit processor.
407 /// \details x86/x32/x64 uses CPUID to determine the value and it is usually accurate. PowerPC
408 /// and AIX also makes the value available to user space and it is also usually accurate. The
409 /// ARM processor equivalent is a privileged instruction, so a compile time value is returned.
GetCacheLineSize()410 inline int GetCacheLineSize()
411 {
412 if (!g_x86DetectionDone)
413 DetectX86Features();
414 return g_cacheLineSize;
415 }
416 //@}
417
418 #endif // CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
419
420 // ***************************** ARM-32, Aarch32 and Aarch64 ***************************** //
421
422 #if CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8 || CRYPTOPP_DOXYGEN_PROCESSING
423
424 // Hide from Doxygen
425 #ifndef CRYPTOPP_DOXYGEN_PROCESSING
426 extern bool g_ArmDetectionDone;
427 extern bool g_hasARMv7;
428 extern bool g_hasNEON;
429 extern bool g_hasPMULL;
430 extern bool g_hasCRC32;
431 extern bool g_hasAES;
432 extern bool g_hasSHA1;
433 extern bool g_hasSHA2;
434 extern bool g_hasSHA512;
435 extern bool g_hasSHA3;
436 extern bool g_hasSM3;
437 extern bool g_hasSM4;
438 void CRYPTOPP_API DetectArmFeatures();
439 #endif // CRYPTOPP_DOXYGEN_PROCESSING
440
441 /// \name ARM A-32, Aarch32 and AArch64 CPU FEATURES
442 //@{
443
444 /// \brief Determine if an ARM processor is ARMv7 or above
445 /// \return true if the hardware is ARMv7 or above, false otherwise.
446 /// \details Some AES code requires ARMv7 or above
447 /// \since Crypto++ 8.0
448 /// \note This function is only available on ARM-32, Aarch32 and Aarch64 platforms
HasARMv7()449 inline bool HasARMv7()
450 {
451 // ASIMD is a core feature on Aarch32 and Aarch64 like SSE2 is a core feature on x86_64
452 #if defined(__aarch32__) || defined(__aarch64__)
453 return true;
454 #else
455 if (!g_ArmDetectionDone)
456 DetectArmFeatures();
457 return g_hasARMv7;
458 #endif
459 }
460
461 /// \brief Determine if an ARM processor has Advanced SIMD available
462 /// \return true if the hardware is capable of Advanced SIMD at runtime, false otherwise.
463 /// \details Advanced SIMD instructions are available under most ARMv7, Aarch32 and Aarch64.
464 /// \details Runtime support requires compile time support. When compiling with GCC, you may
465 /// need to compile with <tt>-mfpu=neon</tt> (32-bit) or <tt>-march=armv8-a</tt>
466 /// (64-bit). Also see ARM's <tt>__ARM_NEON</tt> preprocessor macro.
467 /// \since Crypto++ 5.6.4
468 /// \note This function is only available on ARM-32, Aarch32 and Aarch64 platforms
HasNEON()469 inline bool HasNEON()
470 {
471 // ASIMD is a core feature on Aarch32 and Aarch64 like SSE2 is a core feature on x86_64
472 #if defined(CRYPTOPP_ARM_ASIMD_AVAILABLE)
473 return true;
474 #elif defined(CRYPTOPP_ARM_NEON_AVAILABLE)
475 if (!g_ArmDetectionDone)
476 DetectArmFeatures();
477 return g_hasNEON;
478 #else
479 return false;
480 #endif
481 }
482
483 /// \brief Determine if an ARM processor has CRC32 available
484 /// \return true if the hardware is capable of CRC32 at runtime, false otherwise.
485 /// \details CRC32 instructions provide access to the processor's CRC-32 and CRC-32C
486 /// instructions. They are provided by ARM C Language Extensions 2.0 (ACLE 2.0) and
487 /// available under Aarch32 and Aarch64.
488 /// \details Runtime support requires compile time support. When compiling with GCC,
489 /// you may need to compile with <tt>-march=armv8-a+crc</tt>; while Apple requires
490 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRC32</tt> preprocessor macro.
491 /// \since Crypto++ 5.6.4
492 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasCRC32()493 inline bool HasCRC32()
494 {
495 #if defined(CRYPTOPP_ARM_CRC32_AVAILABLE)
496 if (!g_ArmDetectionDone)
497 DetectArmFeatures();
498 return g_hasCRC32;
499 #else
500 return false;
501 #endif
502 }
503
504 /// \brief Determine if an ARM processor has AES available
505 /// \return true if the hardware is capable of AES at runtime, false otherwise.
506 /// \details AES is part of the optional Crypto extensions on Aarch32 and Aarch64. They are
507 /// accessed using ARM C Language Extensions 2.0 (ACLE 2.0).
508 /// \details Runtime support requires compile time support. When compiling with GCC, you may
509 /// need to compile with <tt>-march=armv8-a+crypto</tt>; while Apple requires
510 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRYPTO</tt> preprocessor macro.
511 /// \since Crypto++ 5.6.4
512 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasAES()513 inline bool HasAES()
514 {
515 #if defined(CRYPTOPP_ARM_AES_AVAILABLE)
516 if (!g_ArmDetectionDone)
517 DetectArmFeatures();
518 return g_hasAES;
519 #else
520 return false;
521 #endif
522 }
523
524 /// \brief Determine if an ARM processor provides Polynomial Multiplication
525 /// \return true if the hardware is capable of polynomial multiplications at runtime,
526 /// false otherwise.
527 /// \details The multiplication instructions are available under Aarch32 and Aarch64.
528 /// \details Runtime support requires compile time support. When compiling with GCC,
529 /// you may need to compile with <tt>-march=armv8-a+crypto</tt>; while Apple requires
530 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRYPTO</tt> preprocessor macro.
531 /// \since Crypto++ 5.6.4
532 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasPMULL()533 inline bool HasPMULL()
534 {
535 #if defined(CRYPTOPP_ARM_PMULL_AVAILABLE)
536 if (!g_ArmDetectionDone)
537 DetectArmFeatures();
538 return g_hasPMULL;
539 #else
540 return false;
541 #endif
542 }
543
544 /// \brief Determine if an ARM processor has SHA1 available
545 /// \return true if the hardware is capable of SHA1 at runtime, false otherwise.
546 /// \details SHA1 is part of the optional Crypto extensions on Aarch32 and Aarch64. They are
547 /// accessed using ARM C Language Extensions 2.0 (ACLE 2.0).
548 /// \details Runtime support requires compile time support. When compiling with GCC, you may
549 /// need to compile with <tt>-march=armv8-a+crypto</tt>; while Apple requires
550 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRYPTO</tt> preprocessor macro.
551 /// \since Crypto++ 5.6.4
552 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasSHA1()553 inline bool HasSHA1()
554 {
555 #if defined(CRYPTOPP_ARM_SHA1_AVAILABLE)
556 if (!g_ArmDetectionDone)
557 DetectArmFeatures();
558 return g_hasSHA1;
559 #else
560 return false;
561 #endif
562 }
563
564 /// \brief Determine if an ARM processor has SHA256 available
565 /// \return true if the hardware is capable of SHA256 at runtime, false otherwise.
566 /// \details SHA256 is part of the optional Crypto extensions on Aarch32 and Aarch64. They are
567 /// accessed using ARM C Language Extensions 2.0 (ACLE 2.0).
568 /// \details Runtime support requires compile time support. When compiling with GCC, you may
569 /// need to compile with <tt>-march=armv8-a+crypto</tt>; while Apple requires
570 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRYPTO</tt> preprocessor macro.
571 /// \since Crypto++ 5.6.4
572 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasSHA2()573 inline bool HasSHA2()
574 {
575 #if defined(CRYPTOPP_ARM_SHA2_AVAILABLE)
576 if (!g_ArmDetectionDone)
577 DetectArmFeatures();
578 return g_hasSHA2;
579 #else
580 return false;
581 #endif
582 }
583
584 /// \brief Determine if an ARM processor has SHA3 available
585 /// \return true if the hardware is capable of SHA3 at runtime, false otherwise.
586 /// \details SHA3 is part of the ARMv8.2 Crypto extensions on Aarch32 and Aarch64. They
587 /// are accessed using ARM C Language Extensions 2.0 (ACLE 2.0).
588 /// \details Runtime support requires compile time support. When compiling with GCC, you
589 /// may need to compile with <tt>-march=armv8.2-a+crypto</tt>; while Apple requires
590 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRYPTO</tt> preprocessor macro.
591 /// \since Crypto++ 8.0
592 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasSHA3()593 inline bool HasSHA3()
594 {
595 #if defined(CRYPTOPP_ARM_SHA3_AVAILABLE)
596 if (!g_ArmDetectionDone)
597 DetectArmFeatures();
598 return g_hasSHA3;
599 #else
600 return false;
601 #endif
602 }
603
604 /// \brief Determine if an ARM processor has SHA512 available
605 /// \return true if the hardware is capable of SHA512 at runtime, false otherwise.
606 /// \details SHA512 is part of the ARMv8.2 Crypto extensions on Aarch32 and Aarch64. They
607 /// are accessed using ARM C Language Extensions 2.0 (ACLE 2.0).
608 /// \details Runtime support requires compile time support. When compiling with GCC, you
609 /// may need to compile with <tt>-march=armv8.2-a+crypto</tt>; while Apple requires
610 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRYPTO</tt> preprocessor macro.
611 /// \since Crypto++ 8.0
612 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasSHA512()613 inline bool HasSHA512()
614 {
615 #if defined(CRYPTOPP_ARM_SHA512_AVAILABLE)
616 if (!g_ArmDetectionDone)
617 DetectArmFeatures();
618 return g_hasSHA512;
619 #else
620 return false;
621 #endif
622 }
623
624 /// \brief Determine if an ARM processor has SM3 available
625 /// \return true if the hardware is capable of SM3 at runtime, false otherwise.
626 /// \details SM3 is part of the ARMv8.2 Crypto extensions on Aarch32 and Aarch64. They
627 /// are accessed using ARM C Language Extensions 2.0 (ACLE 2.0).
628 /// \details Runtime support requires compile time support. When compiling with GCC, you
629 /// may need to compile with <tt>-march=armv8.2-a+crypto</tt>; while Apple requires
630 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRYPTO</tt> preprocessor macro.
631 /// \since Crypto++ 8.0
632 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasSM3()633 inline bool HasSM3()
634 {
635 #if defined(CRYPTOPP_ARM_SM3_AVAILABLE)
636 if (!g_ArmDetectionDone)
637 DetectArmFeatures();
638 return g_hasSM3;
639 #else
640 return false;
641 #endif
642 }
643
644 /// \brief Determine if an ARM processor has SM4 available
645 /// \return true if the hardware is capable of SM4 at runtime, false otherwise.
646 /// \details SM4 is part of the ARMv8.2 Crypto extensions on Aarch32 and Aarch64. They
647 /// are accessed using ARM C Language Extensions 2.0 (ACLE 2.0).
648 /// \details Runtime support requires compile time support. When compiling with GCC, you
649 /// may need to compile with <tt>-march=armv8.2-a+crypto</tt>; while Apple requires
650 /// <tt>-arch arm64</tt>. Also see ARM's <tt>__ARM_FEATURE_CRYPTO</tt> preprocessor macro.
651 /// \since Crypto++ 8.0
652 /// \note This function is only available on Aarch32 and Aarch64 platforms
HasSM4()653 inline bool HasSM4()
654 {
655 #if defined(CRYPTOPP_ARM_SM4_AVAILABLE)
656 if (!g_ArmDetectionDone)
657 DetectArmFeatures();
658 return g_hasSM4;
659 #else
660 return false;
661 #endif
662 }
663
664 //@}
665
666 #endif // CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8
667
668 // ***************************** PowerPC ***************************** //
669
670 #if CRYPTOPP_BOOL_PPC32 || CRYPTOPP_BOOL_PPC64 || CRYPTOPP_DOXYGEN_PROCESSING
671
672 // Hide from Doxygen
673 #ifndef CRYPTOPP_DOXYGEN_PROCESSING
674 extern bool g_PowerpcDetectionDone;
675 extern bool g_hasAltivec;
676 extern bool g_hasPower7;
677 extern bool g_hasPower8;
678 extern bool g_hasPower9;
679 extern bool g_hasAES;
680 extern bool g_hasPMULL;
681 extern bool g_hasSHA256;
682 extern bool g_hasSHA512;
683 extern bool g_hasDARN;
684 extern word32 g_cacheLineSize;
685 void CRYPTOPP_API DetectPowerpcFeatures();
686 #endif // CRYPTOPP_DOXYGEN_PROCESSING
687
688 /// \name POWERPC CPU FEATURES
689 //@{
690
691 /// \brief Determine if a PowerPC processor has Altivec available
692 /// \return true if the hardware is capable of Altivec at runtime, false otherwise.
693 /// \details Altivec instructions are available on modern PowerPCs.
694 /// \details Runtime support requires compile time support. When compiling with GCC, you may
695 /// need to compile with <tt>-mcpu=power4</tt>; while IBM XL C/C++ compilers require
696 /// <tt>-qarch=pwr6 -qaltivec</tt>. Also see PowerPC's <tt>_ALTIVEC_</tt> preprocessor macro.
697 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasAltivec()698 inline bool HasAltivec()
699 {
700 #if CRYPTOPP_ALTIVEC_AVAILABLE
701 if (!g_PowerpcDetectionDone)
702 DetectPowerpcFeatures();
703 return g_hasAltivec;
704 #else
705 return false;
706 #endif
707 }
708
709 /// \brief Determine if a PowerPC processor has Power7 available
710 /// \return true if the hardware is capable of Power7 at runtime, false otherwise.
711 /// \details Runtime support requires compile time support. When compiling with GCC, you may
712 /// need to compile with <tt>-mcpu=power7</tt>; while IBM XL C/C++ compilers require
713 /// <tt>-qarch=pwr7 -qaltivec</tt>. Also see PowerPC's <tt>_ALTIVEC_</tt> preprocessor macro.
714 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasPower7()715 inline bool HasPower7()
716 {
717 #if CRYPTOPP_POWER7_AVAILABLE
718 if (!g_PowerpcDetectionDone)
719 DetectPowerpcFeatures();
720 return g_hasPower7;
721 #else
722 return false;
723 #endif
724 }
725
726 /// \brief Determine if a PowerPC processor has Power8 available
727 /// \return true if the hardware is capable of Power8 at runtime, false otherwise.
728 /// \details Runtime support requires compile time support. When compiling with GCC, you may
729 /// need to compile with <tt>-mcpu=power8</tt>; while IBM XL C/C++ compilers require
730 /// <tt>-qarch=pwr8 -qaltivec</tt>. Also see PowerPC's <tt>_ALTIVEC_</tt> preprocessor macro.
731 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasPower8()732 inline bool HasPower8()
733 {
734 #if CRYPTOPP_POWER8_AVAILABLE
735 if (!g_PowerpcDetectionDone)
736 DetectPowerpcFeatures();
737 return g_hasPower8;
738 #else
739 return false;
740 #endif
741 }
742
743 /// \brief Determine if a PowerPC processor has Power9 available
744 /// \return true if the hardware is capable of Power9 at runtime, false otherwise.
745 /// \details Runtime support requires compile time support. When compiling with GCC, you may
746 /// need to compile with <tt>-mcpu=power9</tt>; while IBM XL C/C++ compilers require
747 /// <tt>-qarch=pwr9 -qaltivec</tt>. Also see PowerPC's <tt>_ALTIVEC_</tt> preprocessor macro.
748 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasPower9()749 inline bool HasPower9()
750 {
751 #if CRYPTOPP_POWER9_AVAILABLE
752 if (!g_PowerpcDetectionDone)
753 DetectPowerpcFeatures();
754 return g_hasPower9;
755 #else
756 return false;
757 #endif
758 }
759
760 /// \brief Determine if a PowerPC processor has AES available
761 /// \return true if the hardware is capable of AES at runtime, false otherwise.
762 /// \details AES is part of the in-crypto extensions on Power8 and Power9.
763 /// \details Runtime support requires compile time support. When compiling with GCC, you may
764 /// need to compile with <tt>-mcpu=power8</tt>; while IBM XL C/C++ compilers require
765 /// <tt>-qarch=pwr8 -qaltivec</tt>. Also see PowerPC's <tt>__CRYPTO</tt> preprocessor macro.
766 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasAES()767 inline bool HasAES()
768 {
769 #if CRYPTOPP_POWER8_AES_AVAILABLE
770 if (!g_PowerpcDetectionDone)
771 DetectPowerpcFeatures();
772 return g_hasAES;
773 #else
774 return false;
775 #endif
776 }
777
778 /// \brief Determine if a PowerPC processor has Polynomial Multiply available
779 /// \return true if the hardware is capable of PMULL at runtime, false otherwise.
780 /// \details PMULL is part of the in-crypto extensions on Power8 and Power9.
781 /// \details Runtime support requires compile time support. When compiling with GCC, you may
782 /// need to compile with <tt>-mcpu=power8</tt>; while IBM XL C/C++ compilers require
783 /// <tt>-qarch=pwr8 -qaltivec</tt>. Also see PowerPC's <tt>__CRYPTO</tt> preprocessor macro.
784 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasPMULL()785 inline bool HasPMULL()
786 {
787 #if CRYPTOPP_POWER8_VMULL_AVAILABLE
788 if (!g_PowerpcDetectionDone)
789 DetectPowerpcFeatures();
790 return g_hasPMULL;
791 #else
792 return false;
793 #endif
794 }
795
796 /// \brief Determine if a PowerPC processor has SHA256 available
797 /// \return true if the hardware is capable of SHA256 at runtime, false otherwise.
798 /// \details SHA is part of the in-crypto extensions on Power8 and Power9.
799 /// \details Runtime support requires compile time support. When compiling with GCC, you may
800 /// need to compile with <tt>-mcpu=power8</tt>; while IBM XL C/C++ compilers require
801 /// <tt>-qarch=pwr8 -qaltivec</tt>. Also see PowerPC's <tt>__CRYPTO</tt> preprocessor macro.
802 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasSHA256()803 inline bool HasSHA256()
804 {
805 #if CRYPTOPP_POWER8_SHA_AVAILABLE
806 if (!g_PowerpcDetectionDone)
807 DetectPowerpcFeatures();
808 return g_hasSHA256;
809 #else
810 return false;
811 #endif
812 }
813
814 /// \brief Determine if a PowerPC processor has SHA512 available
815 /// \return true if the hardware is capable of SHA512 at runtime, false otherwise.
816 /// \details SHA is part of the in-crypto extensions on Power8 and Power9.
817 /// \details Runtime support requires compile time support. When compiling with GCC, you may
818 /// need to compile with <tt>-mcpu=power8</tt>; while IBM XL C/C++ compilers require
819 /// <tt>-qarch=pwr8 -qaltivec</tt>. Also see PowerPC's <tt>__CRYPTO</tt> preprocessor macro.
820 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasSHA512()821 inline bool HasSHA512()
822 {
823 #if CRYPTOPP_POWER8_SHA_AVAILABLE
824 if (!g_PowerpcDetectionDone)
825 DetectPowerpcFeatures();
826 return g_hasSHA512;
827 #else
828 return false;
829 #endif
830 }
831
832 /// \brief Determine if a PowerPC processor has DARN available
833 /// \return true if the hardware is capable of DARN at runtime, false otherwise.
834 /// \details Runtime support requires compile time support. When compiling with GCC, you may
835 /// need to compile with <tt>-mcpu=power9</tt>; while IBM XL C/C++ compilers require
836 /// <tt>-qarch=pwr9 -qaltivec</tt>. Also see PowerPC's <tt>_ALTIVEC_</tt> preprocessor macro.
837 /// \note This function is only available on PowerPC and PowerPC-64 platforms
HasDARN()838 inline bool HasDARN()
839 {
840 #if CRYPTOPP_POWER9_AVAILABLE
841 if (!g_PowerpcDetectionDone)
842 DetectPowerpcFeatures();
843 // see comments in cpu.cpp
844 # if defined(__ibmxl__) && defined(__linux__)
845 return false;
846 # else
847 return g_hasDARN;
848 # endif
849 #else
850 return false;
851 #endif
852 }
853
854 /// \brief Provides the cache line size
855 /// \return lower bound on the size of a cache line in bytes, if available
856 /// \details GetCacheLineSize() returns the lower bound on the size of a cache line, if it
857 /// is available. If the value is not available at runtime, then 32 is returned for a 32-bit
858 /// processor and 64 is returned for a 64-bit processor.
859 /// \details x86/x32/x64 uses CPUID to determine the value and it is usually accurate. PowerPC
860 /// and AIX also makes the value available to user space and it is also usually accurate. The
861 /// ARM processor equivalent is a privileged instruction, so a compile time value is returned.
GetCacheLineSize()862 inline int GetCacheLineSize()
863 {
864 if (!g_PowerpcDetectionDone)
865 DetectPowerpcFeatures();
866 return g_cacheLineSize;
867 }
868
869 //@}
870
871 #endif // CRYPTOPP_BOOL_PPC32 || CRYPTOPP_BOOL_PPC64
872
873 // ***************************** L1 cache line ***************************** //
874
875 // Non-Intel systems
876 #if !(CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64 || CRYPTOPP_BOOL_PPC32 || CRYPTOPP_BOOL_PPC64)
877 /// \brief Provides the cache line size
878 /// \return lower bound on the size of a cache line in bytes, if available
879 /// \details GetCacheLineSize() returns the lower bound on the size of a cache line, if it
880 /// is available. If the value is not available at runtime, then 32 is returned for a 32-bit
881 /// processor and 64 is returned for a 64-bit processor.
882 /// \details x86/x32/x64 uses CPUID to determine the value and it is usually accurate. PowerPC
883 /// and AIX also makes the value available to user space and it is also usually accurate. The
884 /// ARM processor equivalent is a privileged instruction, so a compile time value is returned.
GetCacheLineSize()885 inline int GetCacheLineSize()
886 {
887 return CRYPTOPP_L1_CACHE_LINE_SIZE;
888 }
889 #endif // Non-Intel systems
890
891 #endif // CRYPTOPP_GENERATE_X64_MASM
892
893 // ***************************** Inline ASM Helper ***************************** //
894
895 #ifndef CRYPTOPP_DOXYGEN_PROCESSING
896
897 #if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
898
899 #ifdef CRYPTOPP_GENERATE_X64_MASM
900 #define AS1(x) x*newline*
901 #define AS2(x, y) x, y*newline*
902 #define AS3(x, y, z) x, y, z*newline*
903 #define ASS(x, y, a, b, c, d) x, y, a*64+b*16+c*4+d*newline*
904 #define ASL(x) label##x:*newline*
905 #define ASJ(x, y, z) x label##y*newline*
906 #define ASC(x, y) x label##y*newline*
907 #define AS_HEX(y) 0##y##h
908 #elif defined(_MSC_VER) || defined(__BORLANDC__)
909 #define AS1(x) __asm {x}
910 #define AS2(x, y) __asm {x, y}
911 #define AS3(x, y, z) __asm {x, y, z}
912 #define ASS(x, y, a, b, c, d) __asm {x, y, (a)*64+(b)*16+(c)*4+(d)}
913 #define ASL(x) __asm {label##x:}
914 #define ASJ(x, y, z) __asm {x label##y}
915 #define ASC(x, y) __asm {x label##y}
916 #define CRYPTOPP_NAKED __declspec(naked)
917 #define AS_HEX(y) 0x##y
918 #else
919 // define these in two steps to allow arguments to be expanded
920 #define GNU_AS1(x) #x ";" NEW_LINE
921 #define GNU_AS2(x, y) #x ", " #y ";" NEW_LINE
922 #define GNU_AS3(x, y, z) #x ", " #y ", " #z ";" NEW_LINE
923 #define GNU_ASL(x) "\n" #x ":" NEW_LINE
924 // clang 5.0.0 and apple clang 9.0.0 don't support numerical backward jumps
925 #if (CRYPTOPP_LLVM_CLANG_VERSION >= 50000) || (CRYPTOPP_APPLE_CLANG_VERSION >= 90000)
926 #define GNU_ASJ(x, y, z) ATT_PREFIX ";" NEW_LINE #x " " #y #z ";" NEW_LINE INTEL_PREFIX ";" NEW_LINE
927 #else
928 #define GNU_ASJ(x, y, z) #x " " #y #z ";" NEW_LINE
929 #endif
930 #define AS1(x) GNU_AS1(x)
931 #define AS2(x, y) GNU_AS2(x, y)
932 #define AS3(x, y, z) GNU_AS3(x, y, z)
933 #define ASS(x, y, a, b, c, d) #x ", " #y ", " #a "*64+" #b "*16+" #c "*4+" #d ";"
934 #define ASL(x) GNU_ASL(x)
935 #define ASJ(x, y, z) GNU_ASJ(x, y, z)
936 #define ASC(x, y) #x " " #y ";"
937 #define CRYPTOPP_NAKED
938 #define AS_HEX(y) 0x##y
939 #endif
940
941 #define IF0(y)
942 #define IF1(y) y
943
944 #ifdef CRYPTOPP_GENERATE_X64_MASM
945 #define ASM_MOD(x, y) ((x) MOD (y))
946 #define XMMWORD_PTR XMMWORD PTR
947 #else
948 // GNU assembler doesn't seem to have mod operator
949 #define ASM_MOD(x, y) ((x)-((x)/(y))*(y))
950 // GAS 2.15 doesn't support XMMWORD PTR. it seems necessary only for MASM
951 #define XMMWORD_PTR
952 #endif
953
954 #if CRYPTOPP_BOOL_X86
955 #define AS_REG_1 ecx
956 #define AS_REG_2 edx
957 #define AS_REG_3 esi
958 #define AS_REG_4 edi
959 #define AS_REG_5 eax
960 #define AS_REG_6 ebx
961 #define AS_REG_7 ebp
962 #define AS_REG_1d ecx
963 #define AS_REG_2d edx
964 #define AS_REG_3d esi
965 #define AS_REG_4d edi
966 #define AS_REG_5d eax
967 #define AS_REG_6d ebx
968 #define AS_REG_7d ebp
969 #define WORD_SZ 4
970 #define WORD_REG(x) e##x
971 #define WORD_PTR DWORD PTR
972 #define AS_PUSH_IF86(x) AS1(push e##x)
973 #define AS_POP_IF86(x) AS1(pop e##x)
974 #define AS_JCXZ jecxz
975 #elif CRYPTOPP_BOOL_X32
976 #define AS_REG_1 ecx
977 #define AS_REG_2 edx
978 #define AS_REG_3 r8d
979 #define AS_REG_4 r9d
980 #define AS_REG_5 eax
981 #define AS_REG_6 r10d
982 #define AS_REG_7 r11d
983 #define AS_REG_1d ecx
984 #define AS_REG_2d edx
985 #define AS_REG_3d r8d
986 #define AS_REG_4d r9d
987 #define AS_REG_5d eax
988 #define AS_REG_6d r10d
989 #define AS_REG_7d r11d
990 #define WORD_SZ 4
991 #define WORD_REG(x) e##x
992 #define WORD_PTR DWORD PTR
993 #define AS_PUSH_IF86(x) AS1(push r##x)
994 #define AS_POP_IF86(x) AS1(pop r##x)
995 #define AS_JCXZ jecxz
996 #elif CRYPTOPP_BOOL_X64
997 #ifdef CRYPTOPP_GENERATE_X64_MASM
998 #define AS_REG_1 rcx
999 #define AS_REG_2 rdx
1000 #define AS_REG_3 r8
1001 #define AS_REG_4 r9
1002 #define AS_REG_5 rax
1003 #define AS_REG_6 r10
1004 #define AS_REG_7 r11
1005 #define AS_REG_1d ecx
1006 #define AS_REG_2d edx
1007 #define AS_REG_3d r8d
1008 #define AS_REG_4d r9d
1009 #define AS_REG_5d eax
1010 #define AS_REG_6d r10d
1011 #define AS_REG_7d r11d
1012 #else
1013 #define AS_REG_1 rdi
1014 #define AS_REG_2 rsi
1015 #define AS_REG_3 rdx
1016 #define AS_REG_4 rcx
1017 #define AS_REG_5 r8
1018 #define AS_REG_6 r9
1019 #define AS_REG_7 r10
1020 #define AS_REG_1d edi
1021 #define AS_REG_2d esi
1022 #define AS_REG_3d edx
1023 #define AS_REG_4d ecx
1024 #define AS_REG_5d r8d
1025 #define AS_REG_6d r9d
1026 #define AS_REG_7d r10d
1027 #endif
1028 #define WORD_SZ 8
1029 #define WORD_REG(x) r##x
1030 #define WORD_PTR QWORD PTR
1031 #define AS_PUSH_IF86(x)
1032 #define AS_POP_IF86(x)
1033 #define AS_JCXZ jrcxz
1034 #endif
1035
1036 // helper macro for stream cipher output
1037 #define AS_XMM_OUTPUT4(labelPrefix, inputPtr, outputPtr, x0, x1, x2, x3, t, p0, p1, p2, p3, increment)\
1038 AS2( test inputPtr, inputPtr)\
1039 ASC( jz, labelPrefix##3)\
1040 AS2( test inputPtr, 15)\
1041 ASC( jnz, labelPrefix##7)\
1042 AS2( pxor xmm##x0, [inputPtr+p0*16])\
1043 AS2( pxor xmm##x1, [inputPtr+p1*16])\
1044 AS2( pxor xmm##x2, [inputPtr+p2*16])\
1045 AS2( pxor xmm##x3, [inputPtr+p3*16])\
1046 AS2( add inputPtr, increment*16)\
1047 ASC( jmp, labelPrefix##3)\
1048 ASL(labelPrefix##7)\
1049 AS2( movdqu xmm##t, [inputPtr+p0*16])\
1050 AS2( pxor xmm##x0, xmm##t)\
1051 AS2( movdqu xmm##t, [inputPtr+p1*16])\
1052 AS2( pxor xmm##x1, xmm##t)\
1053 AS2( movdqu xmm##t, [inputPtr+p2*16])\
1054 AS2( pxor xmm##x2, xmm##t)\
1055 AS2( movdqu xmm##t, [inputPtr+p3*16])\
1056 AS2( pxor xmm##x3, xmm##t)\
1057 AS2( add inputPtr, increment*16)\
1058 ASL(labelPrefix##3)\
1059 AS2( test outputPtr, 15)\
1060 ASC( jnz, labelPrefix##8)\
1061 AS2( movdqa [outputPtr+p0*16], xmm##x0)\
1062 AS2( movdqa [outputPtr+p1*16], xmm##x1)\
1063 AS2( movdqa [outputPtr+p2*16], xmm##x2)\
1064 AS2( movdqa [outputPtr+p3*16], xmm##x3)\
1065 ASC( jmp, labelPrefix##9)\
1066 ASL(labelPrefix##8)\
1067 AS2( movdqu [outputPtr+p0*16], xmm##x0)\
1068 AS2( movdqu [outputPtr+p1*16], xmm##x1)\
1069 AS2( movdqu [outputPtr+p2*16], xmm##x2)\
1070 AS2( movdqu [outputPtr+p3*16], xmm##x3)\
1071 ASL(labelPrefix##9)\
1072 AS2( add outputPtr, increment*16)
1073
1074 #endif // CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
1075
1076 #endif // Not CRYPTOPP_DOXYGEN_PROCESSING
1077
1078 NAMESPACE_END
1079
1080 // Issue 340
1081 #if CRYPTOPP_GCC_DIAGNOSTIC_AVAILABLE
1082 # pragma GCC diagnostic pop
1083 #endif
1084
1085 #endif // CRYPTOPP_CPU_H
1086