1 /* 2 * x86 specific aes acceleration. 3 * SPDX-License-Identifier: GPL-2.0-or-later 4 */ 5 6 #ifndef X86_HOST_CRYPTO_AES_ROUND_H 7 #define X86_HOST_CRYPTO_AES_ROUND_H 8 9 #include "host/cpuinfo.h" 10 #include <immintrin.h> 11 12 #if defined(__AES__) && defined(__SSSE3__) 13 # define HAVE_AES_ACCEL true 14 # define ATTR_AES_ACCEL 15 #else 16 # define HAVE_AES_ACCEL likely(cpuinfo & CPUINFO_AES) 17 # define ATTR_AES_ACCEL __attribute__((target("aes,ssse3"))) 18 #endif 19 20 static inline __m128i ATTR_AES_ACCEL 21 aes_accel_bswap(__m128i x) 22 { 23 return _mm_shuffle_epi8(x, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 24 9, 10, 11, 12, 13, 14, 15)); 25 } 26 27 static inline void ATTR_AES_ACCEL 28 aesenc_MC_accel(AESState *ret, const AESState *st, bool be) 29 { 30 __m128i t = (__m128i)st->v; 31 __m128i z = _mm_setzero_si128(); 32 33 if (be) { 34 t = aes_accel_bswap(t); 35 t = _mm_aesdeclast_si128(t, z); 36 t = _mm_aesenc_si128(t, z); 37 t = aes_accel_bswap(t); 38 } else { 39 t = _mm_aesdeclast_si128(t, z); 40 t = _mm_aesenc_si128(t, z); 41 } 42 ret->v = (AESStateVec)t; 43 } 44 45 static inline void ATTR_AES_ACCEL 46 aesenc_SB_SR_AK_accel(AESState *ret, const AESState *st, 47 const AESState *rk, bool be) 48 { 49 __m128i t = (__m128i)st->v; 50 __m128i k = (__m128i)rk->v; 51 52 if (be) { 53 t = aes_accel_bswap(t); 54 k = aes_accel_bswap(k); 55 t = _mm_aesenclast_si128(t, k); 56 t = aes_accel_bswap(t); 57 } else { 58 t = _mm_aesenclast_si128(t, k); 59 } 60 ret->v = (AESStateVec)t; 61 } 62 63 static inline void ATTR_AES_ACCEL 64 aesenc_SB_SR_MC_AK_accel(AESState *ret, const AESState *st, 65 const AESState *rk, bool be) 66 { 67 __m128i t = (__m128i)st->v; 68 __m128i k = (__m128i)rk->v; 69 70 if (be) { 71 t = aes_accel_bswap(t); 72 k = aes_accel_bswap(k); 73 t = _mm_aesenc_si128(t, k); 74 t = aes_accel_bswap(t); 75 } else { 76 t = _mm_aesenc_si128(t, k); 77 } 78 ret->v = (AESStateVec)t; 79 } 80 81 static inline void ATTR_AES_ACCEL 82 aesdec_IMC_accel(AESState *ret, const AESState *st, bool be) 83 { 84 __m128i t = (__m128i)st->v; 85 86 if (be) { 87 t = aes_accel_bswap(t); 88 t = _mm_aesimc_si128(t); 89 t = aes_accel_bswap(t); 90 } else { 91 t = _mm_aesimc_si128(t); 92 } 93 ret->v = (AESStateVec)t; 94 } 95 96 static inline void ATTR_AES_ACCEL 97 aesdec_ISB_ISR_AK_accel(AESState *ret, const AESState *st, 98 const AESState *rk, bool be) 99 { 100 __m128i t = (__m128i)st->v; 101 __m128i k = (__m128i)rk->v; 102 103 if (be) { 104 t = aes_accel_bswap(t); 105 k = aes_accel_bswap(k); 106 t = _mm_aesdeclast_si128(t, k); 107 t = aes_accel_bswap(t); 108 } else { 109 t = _mm_aesdeclast_si128(t, k); 110 } 111 ret->v = (AESStateVec)t; 112 } 113 114 static inline void ATTR_AES_ACCEL 115 aesdec_ISB_ISR_AK_IMC_accel(AESState *ret, const AESState *st, 116 const AESState *rk, bool be) 117 { 118 __m128i t = (__m128i)st->v; 119 __m128i k = (__m128i)rk->v; 120 121 if (be) { 122 t = aes_accel_bswap(t); 123 k = aes_accel_bswap(k); 124 t = _mm_aesdeclast_si128(t, k); 125 t = _mm_aesimc_si128(t); 126 t = aes_accel_bswap(t); 127 } else { 128 t = _mm_aesdeclast_si128(t, k); 129 t = _mm_aesimc_si128(t); 130 } 131 ret->v = (AESStateVec)t; 132 } 133 134 static inline void ATTR_AES_ACCEL 135 aesdec_ISB_ISR_IMC_AK_accel(AESState *ret, const AESState *st, 136 const AESState *rk, bool be) 137 { 138 __m128i t = (__m128i)st->v; 139 __m128i k = (__m128i)rk->v; 140 141 if (be) { 142 t = aes_accel_bswap(t); 143 k = aes_accel_bswap(k); 144 t = _mm_aesdec_si128(t, k); 145 t = aes_accel_bswap(t); 146 } else { 147 t = _mm_aesdec_si128(t, k); 148 } 149 ret->v = (AESStateVec)t; 150 } 151 152 #endif /* X86_HOST_CRYPTO_AES_ROUND_H */ 153