1 /* 2 * Power v2.07 specific aes acceleration. 3 * SPDX-License-Identifier: GPL-2.0-or-later 4 */ 5 6 #ifndef PPC_HOST_CRYPTO_AES_ROUND_H 7 #define PPC_HOST_CRYPTO_AES_ROUND_H 8 9 #ifdef __ALTIVEC__ 10 #include "host/cpuinfo.h" 11 12 #ifdef __CRYPTO__ 13 # define HAVE_AES_ACCEL true 14 #else 15 # define HAVE_AES_ACCEL likely(cpuinfo & CPUINFO_CRYPTO) 16 #endif 17 #define ATTR_AES_ACCEL 18 19 /* 20 * While there is <altivec.h>, both gcc and clang "aid" with the 21 * endianness issues in different ways. Just use inline asm instead. 22 */ 23 24 /* Bytes in memory are host-endian; bytes in register are @be. */ 25 static inline AESStateVec aes_accel_ld(const AESState *p, bool be) 26 { 27 AESStateVec r; 28 29 if (be) { 30 asm("lvx %0, 0, %1" : "=v"(r) : "r"(p), "m"(*p)); 31 } else if (HOST_BIG_ENDIAN) { 32 AESStateVec rev = { 33 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 34 }; 35 asm("lvx %0, 0, %1\n\t" 36 "vperm %0, %0, %0, %2" 37 : "=v"(r) : "r"(p), "v"(rev), "m"(*p)); 38 } else { 39 #ifdef __POWER9_VECTOR__ 40 asm("lxvb16x %x0, 0, %1" : "=v"(r) : "r"(p), "m"(*p)); 41 #else 42 asm("lxvd2x %x0, 0, %1\n\t" 43 "xxpermdi %x0, %x0, %x0, 2" 44 : "=v"(r) : "r"(p), "m"(*p)); 45 #endif 46 } 47 return r; 48 } 49 50 static void aes_accel_st(AESState *p, AESStateVec r, bool be) 51 { 52 if (be) { 53 asm("stvx %1, 0, %2" : "=m"(*p) : "v"(r), "r"(p)); 54 } else if (HOST_BIG_ENDIAN) { 55 AESStateVec rev = { 56 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 57 }; 58 asm("vperm %1, %1, %1, %2\n\t" 59 "stvx %1, 0, %3" 60 : "=m"(*p), "+v"(r) : "v"(rev), "r"(p)); 61 } else { 62 #ifdef __POWER9_VECTOR__ 63 asm("stxvb16x %x1, 0, %2" : "=m"(*p) : "v"(r), "r"(p)); 64 #else 65 asm("xxpermdi %x1, %x1, %x1, 2\n\t" 66 "stxvd2x %x1, 0, %2" 67 : "=m"(*p), "+v"(r) : "r"(p)); 68 #endif 69 } 70 } 71 72 static inline AESStateVec aes_accel_vcipher(AESStateVec d, AESStateVec k) 73 { 74 asm("vcipher %0, %0, %1" : "+v"(d) : "v"(k)); 75 return d; 76 } 77 78 static inline AESStateVec aes_accel_vncipher(AESStateVec d, AESStateVec k) 79 { 80 asm("vncipher %0, %0, %1" : "+v"(d) : "v"(k)); 81 return d; 82 } 83 84 static inline AESStateVec aes_accel_vcipherlast(AESStateVec d, AESStateVec k) 85 { 86 asm("vcipherlast %0, %0, %1" : "+v"(d) : "v"(k)); 87 return d; 88 } 89 90 static inline AESStateVec aes_accel_vncipherlast(AESStateVec d, AESStateVec k) 91 { 92 asm("vncipherlast %0, %0, %1" : "+v"(d) : "v"(k)); 93 return d; 94 } 95 96 static inline void 97 aesenc_MC_accel(AESState *ret, const AESState *st, bool be) 98 { 99 AESStateVec t, z = { }; 100 101 t = aes_accel_ld(st, be); 102 t = aes_accel_vncipherlast(t, z); 103 t = aes_accel_vcipher(t, z); 104 aes_accel_st(ret, t, be); 105 } 106 107 static inline void 108 aesenc_SB_SR_AK_accel(AESState *ret, const AESState *st, 109 const AESState *rk, bool be) 110 { 111 AESStateVec t, k; 112 113 t = aes_accel_ld(st, be); 114 k = aes_accel_ld(rk, be); 115 t = aes_accel_vcipherlast(t, k); 116 aes_accel_st(ret, t, be); 117 } 118 119 static inline void 120 aesenc_SB_SR_MC_AK_accel(AESState *ret, const AESState *st, 121 const AESState *rk, bool be) 122 { 123 AESStateVec t, k; 124 125 t = aes_accel_ld(st, be); 126 k = aes_accel_ld(rk, be); 127 t = aes_accel_vcipher(t, k); 128 aes_accel_st(ret, t, be); 129 } 130 131 static inline void 132 aesdec_IMC_accel(AESState *ret, const AESState *st, bool be) 133 { 134 AESStateVec t, z = { }; 135 136 t = aes_accel_ld(st, be); 137 t = aes_accel_vcipherlast(t, z); 138 t = aes_accel_vncipher(t, z); 139 aes_accel_st(ret, t, be); 140 } 141 142 static inline void 143 aesdec_ISB_ISR_AK_accel(AESState *ret, const AESState *st, 144 const AESState *rk, bool be) 145 { 146 AESStateVec t, k; 147 148 t = aes_accel_ld(st, be); 149 k = aes_accel_ld(rk, be); 150 t = aes_accel_vncipherlast(t, k); 151 aes_accel_st(ret, t, be); 152 } 153 154 static inline void 155 aesdec_ISB_ISR_AK_IMC_accel(AESState *ret, const AESState *st, 156 const AESState *rk, bool be) 157 { 158 AESStateVec t, k; 159 160 t = aes_accel_ld(st, be); 161 k = aes_accel_ld(rk, be); 162 t = aes_accel_vncipher(t, k); 163 aes_accel_st(ret, t, be); 164 } 165 166 static inline void 167 aesdec_ISB_ISR_IMC_AK_accel(AESState *ret, const AESState *st, 168 const AESState *rk, bool be) 169 { 170 AESStateVec t, k, z = { }; 171 172 t = aes_accel_ld(st, be); 173 k = aes_accel_ld(rk, be); 174 t = aes_accel_vncipher(t, z); 175 aes_accel_st(ret, t ^ k, be); 176 } 177 #else 178 /* Without ALTIVEC, we can't even write inline assembly. */ 179 #include "host/include/generic/host/crypto/aes-round.h" 180 #endif 181 182 #endif /* PPC_HOST_CRYPTO_AES_ROUND_H */ 183