xref: /qemu/target/arm/tcg/crypto_helper.c (revision 83ecdb18) 
1 /*
2  * crypto_helper.c - emulate v8 Crypto Extensions instructions
3  *
4  * Copyright (C) 2013 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  */
11 
12 #include "qemu/osdep.h"
13 
14 #include "cpu.h"
15 #include "exec/helper-proto.h"
16 #include "tcg/tcg-gvec-desc.h"
17 #include "crypto/aes.h"
18 #include "crypto/sm4.h"
19 #include "vec_internal.h"
20 
21 union CRYPTO_STATE {
22     uint8_t    bytes[16];
23     uint32_t   words[4];
24     uint64_t   l[2];
25 };
26 
27 #if HOST_BIG_ENDIAN
28 #define CR_ST_BYTE(state, i)   ((state).bytes[(15 - (i)) ^ 8])
29 #define CR_ST_WORD(state, i)   ((state).words[(3 - (i)) ^ 2])
30 #else
31 #define CR_ST_BYTE(state, i)   ((state).bytes[i])
32 #define CR_ST_WORD(state, i)   ((state).words[i])
33 #endif
34 
35 /*
36  * The caller has not been converted to full gvec, and so only
37  * modifies the low 16 bytes of the vector register.
38  */
39 static void clear_tail_16(void *vd, uint32_t desc)
40 {
41     int opr_sz = simd_oprsz(desc);
42     int max_sz = simd_maxsz(desc);
43 
44     assert(opr_sz == 16);
45     clear_tail(vd, opr_sz, max_sz);
46 }
47 
48 static void do_crypto_aese(uint64_t *rd, uint64_t *rn,
49                            uint64_t *rm, bool decrypt)
50 {
51     static uint8_t const * const sbox[2] = { AES_sbox, AES_isbox };
52     static uint8_t const * const shift[2] = { AES_shifts, AES_ishifts };
53     union CRYPTO_STATE rk = { .l = { rm[0], rm[1] } };
54     union CRYPTO_STATE st = { .l = { rn[0], rn[1] } };
55     int i;
56 
57     /* xor state vector with round key */
58     rk.l[0] ^= st.l[0];
59     rk.l[1] ^= st.l[1];
60 
61     /* combine ShiftRows operation and sbox substitution */
62     for (i = 0; i < 16; i++) {
63         CR_ST_BYTE(st, i) = sbox[decrypt][CR_ST_BYTE(rk, shift[decrypt][i])];
64     }
65 
66     rd[0] = st.l[0];
67     rd[1] = st.l[1];
68 }
69 
70 void HELPER(crypto_aese)(void *vd, void *vn, void *vm, uint32_t desc)
71 {
72     intptr_t i, opr_sz = simd_oprsz(desc);
73     bool decrypt = simd_data(desc);
74 
75     for (i = 0; i < opr_sz; i += 16) {
76         do_crypto_aese(vd + i, vn + i, vm + i, decrypt);
77     }
78     clear_tail(vd, opr_sz, simd_maxsz(desc));
79 }
80 
81 static void do_crypto_aesmc(uint64_t *rd, uint64_t *rm, bool decrypt)
82 {
83     static uint32_t const mc[][256] = { {
84         /* MixColumns lookup table */
85         0x00000000, 0x03010102, 0x06020204, 0x05030306,
86         0x0c040408, 0x0f05050a, 0x0a06060c, 0x0907070e,
87         0x18080810, 0x1b090912, 0x1e0a0a14, 0x1d0b0b16,
88         0x140c0c18, 0x170d0d1a, 0x120e0e1c, 0x110f0f1e,
89         0x30101020, 0x33111122, 0x36121224, 0x35131326,
90         0x3c141428, 0x3f15152a, 0x3a16162c, 0x3917172e,
91         0x28181830, 0x2b191932, 0x2e1a1a34, 0x2d1b1b36,
92         0x241c1c38, 0x271d1d3a, 0x221e1e3c, 0x211f1f3e,
93         0x60202040, 0x63212142, 0x66222244, 0x65232346,
94         0x6c242448, 0x6f25254a, 0x6a26264c, 0x6927274e,
95         0x78282850, 0x7b292952, 0x7e2a2a54, 0x7d2b2b56,
96         0x742c2c58, 0x772d2d5a, 0x722e2e5c, 0x712f2f5e,
97         0x50303060, 0x53313162, 0x56323264, 0x55333366,
98         0x5c343468, 0x5f35356a, 0x5a36366c, 0x5937376e,
99         0x48383870, 0x4b393972, 0x4e3a3a74, 0x4d3b3b76,
100         0x443c3c78, 0x473d3d7a, 0x423e3e7c, 0x413f3f7e,
101         0xc0404080, 0xc3414182, 0xc6424284, 0xc5434386,
102         0xcc444488, 0xcf45458a, 0xca46468c, 0xc947478e,
103         0xd8484890, 0xdb494992, 0xde4a4a94, 0xdd4b4b96,
104         0xd44c4c98, 0xd74d4d9a, 0xd24e4e9c, 0xd14f4f9e,
105         0xf05050a0, 0xf35151a2, 0xf65252a4, 0xf55353a6,
106         0xfc5454a8, 0xff5555aa, 0xfa5656ac, 0xf95757ae,
107         0xe85858b0, 0xeb5959b2, 0xee5a5ab4, 0xed5b5bb6,
108         0xe45c5cb8, 0xe75d5dba, 0xe25e5ebc, 0xe15f5fbe,
109         0xa06060c0, 0xa36161c2, 0xa66262c4, 0xa56363c6,
110         0xac6464c8, 0xaf6565ca, 0xaa6666cc, 0xa96767ce,
111         0xb86868d0, 0xbb6969d2, 0xbe6a6ad4, 0xbd6b6bd6,
112         0xb46c6cd8, 0xb76d6dda, 0xb26e6edc, 0xb16f6fde,
113         0x907070e0, 0x937171e2, 0x967272e4, 0x957373e6,
114         0x9c7474e8, 0x9f7575ea, 0x9a7676ec, 0x997777ee,
115         0x887878f0, 0x8b7979f2, 0x8e7a7af4, 0x8d7b7bf6,
116         0x847c7cf8, 0x877d7dfa, 0x827e7efc, 0x817f7ffe,
117         0x9b80801b, 0x98818119, 0x9d82821f, 0x9e83831d,
118         0x97848413, 0x94858511, 0x91868617, 0x92878715,
119         0x8388880b, 0x80898909, 0x858a8a0f, 0x868b8b0d,
120         0x8f8c8c03, 0x8c8d8d01, 0x898e8e07, 0x8a8f8f05,
121         0xab90903b, 0xa8919139, 0xad92923f, 0xae93933d,
122         0xa7949433, 0xa4959531, 0xa1969637, 0xa2979735,
123         0xb398982b, 0xb0999929, 0xb59a9a2f, 0xb69b9b2d,
124         0xbf9c9c23, 0xbc9d9d21, 0xb99e9e27, 0xba9f9f25,
125         0xfba0a05b, 0xf8a1a159, 0xfda2a25f, 0xfea3a35d,
126         0xf7a4a453, 0xf4a5a551, 0xf1a6a657, 0xf2a7a755,
127         0xe3a8a84b, 0xe0a9a949, 0xe5aaaa4f, 0xe6abab4d,
128         0xefacac43, 0xecadad41, 0xe9aeae47, 0xeaafaf45,
129         0xcbb0b07b, 0xc8b1b179, 0xcdb2b27f, 0xceb3b37d,
130         0xc7b4b473, 0xc4b5b571, 0xc1b6b677, 0xc2b7b775,
131         0xd3b8b86b, 0xd0b9b969, 0xd5baba6f, 0xd6bbbb6d,
132         0xdfbcbc63, 0xdcbdbd61, 0xd9bebe67, 0xdabfbf65,
133         0x5bc0c09b, 0x58c1c199, 0x5dc2c29f, 0x5ec3c39d,
134         0x57c4c493, 0x54c5c591, 0x51c6c697, 0x52c7c795,
135         0x43c8c88b, 0x40c9c989, 0x45caca8f, 0x46cbcb8d,
136         0x4fcccc83, 0x4ccdcd81, 0x49cece87, 0x4acfcf85,
137         0x6bd0d0bb, 0x68d1d1b9, 0x6dd2d2bf, 0x6ed3d3bd,
138         0x67d4d4b3, 0x64d5d5b1, 0x61d6d6b7, 0x62d7d7b5,
139         0x73d8d8ab, 0x70d9d9a9, 0x75dadaaf, 0x76dbdbad,
140         0x7fdcdca3, 0x7cdddda1, 0x79dedea7, 0x7adfdfa5,
141         0x3be0e0db, 0x38e1e1d9, 0x3de2e2df, 0x3ee3e3dd,
142         0x37e4e4d3, 0x34e5e5d1, 0x31e6e6d7, 0x32e7e7d5,
143         0x23e8e8cb, 0x20e9e9c9, 0x25eaeacf, 0x26ebebcd,
144         0x2fececc3, 0x2cededc1, 0x29eeeec7, 0x2aefefc5,
145         0x0bf0f0fb, 0x08f1f1f9, 0x0df2f2ff, 0x0ef3f3fd,
146         0x07f4f4f3, 0x04f5f5f1, 0x01f6f6f7, 0x02f7f7f5,
147         0x13f8f8eb, 0x10f9f9e9, 0x15fafaef, 0x16fbfbed,
148         0x1ffcfce3, 0x1cfdfde1, 0x19fefee7, 0x1affffe5,
149     }, {
150         /* Inverse MixColumns lookup table */
151         0x00000000, 0x0b0d090e, 0x161a121c, 0x1d171b12,
152         0x2c342438, 0x27392d36, 0x3a2e3624, 0x31233f2a,
153         0x58684870, 0x5365417e, 0x4e725a6c, 0x457f5362,
154         0x745c6c48, 0x7f516546, 0x62467e54, 0x694b775a,
155         0xb0d090e0, 0xbbdd99ee, 0xa6ca82fc, 0xadc78bf2,
156         0x9ce4b4d8, 0x97e9bdd6, 0x8afea6c4, 0x81f3afca,
157         0xe8b8d890, 0xe3b5d19e, 0xfea2ca8c, 0xf5afc382,
158         0xc48cfca8, 0xcf81f5a6, 0xd296eeb4, 0xd99be7ba,
159         0x7bbb3bdb, 0x70b632d5, 0x6da129c7, 0x66ac20c9,
160         0x578f1fe3, 0x5c8216ed, 0x41950dff, 0x4a9804f1,
161         0x23d373ab, 0x28de7aa5, 0x35c961b7, 0x3ec468b9,
162         0x0fe75793, 0x04ea5e9d, 0x19fd458f, 0x12f04c81,
163         0xcb6bab3b, 0xc066a235, 0xdd71b927, 0xd67cb029,
164         0xe75f8f03, 0xec52860d, 0xf1459d1f, 0xfa489411,
165         0x9303e34b, 0x980eea45, 0x8519f157, 0x8e14f859,
166         0xbf37c773, 0xb43ace7d, 0xa92dd56f, 0xa220dc61,
167         0xf66d76ad, 0xfd607fa3, 0xe07764b1, 0xeb7a6dbf,
168         0xda595295, 0xd1545b9b, 0xcc434089, 0xc74e4987,
169         0xae053edd, 0xa50837d3, 0xb81f2cc1, 0xb31225cf,
170         0x82311ae5, 0x893c13eb, 0x942b08f9, 0x9f2601f7,
171         0x46bde64d, 0x4db0ef43, 0x50a7f451, 0x5baafd5f,
172         0x6a89c275, 0x6184cb7b, 0x7c93d069, 0x779ed967,
173         0x1ed5ae3d, 0x15d8a733, 0x08cfbc21, 0x03c2b52f,
174         0x32e18a05, 0x39ec830b, 0x24fb9819, 0x2ff69117,
175         0x8dd64d76, 0x86db4478, 0x9bcc5f6a, 0x90c15664,
176         0xa1e2694e, 0xaaef6040, 0xb7f87b52, 0xbcf5725c,
177         0xd5be0506, 0xdeb30c08, 0xc3a4171a, 0xc8a91e14,
178         0xf98a213e, 0xf2872830, 0xef903322, 0xe49d3a2c,
179         0x3d06dd96, 0x360bd498, 0x2b1ccf8a, 0x2011c684,
180         0x1132f9ae, 0x1a3ff0a0, 0x0728ebb2, 0x0c25e2bc,
181         0x656e95e6, 0x6e639ce8, 0x737487fa, 0x78798ef4,
182         0x495ab1de, 0x4257b8d0, 0x5f40a3c2, 0x544daacc,
183         0xf7daec41, 0xfcd7e54f, 0xe1c0fe5d, 0xeacdf753,
184         0xdbeec879, 0xd0e3c177, 0xcdf4da65, 0xc6f9d36b,
185         0xafb2a431, 0xa4bfad3f, 0xb9a8b62d, 0xb2a5bf23,
186         0x83868009, 0x888b8907, 0x959c9215, 0x9e919b1b,
187         0x470a7ca1, 0x4c0775af, 0x51106ebd, 0x5a1d67b3,
188         0x6b3e5899, 0x60335197, 0x7d244a85, 0x7629438b,
189         0x1f6234d1, 0x146f3ddf, 0x097826cd, 0x02752fc3,
190         0x335610e9, 0x385b19e7, 0x254c02f5, 0x2e410bfb,
191         0x8c61d79a, 0x876cde94, 0x9a7bc586, 0x9176cc88,
192         0xa055f3a2, 0xab58faac, 0xb64fe1be, 0xbd42e8b0,
193         0xd4099fea, 0xdf0496e4, 0xc2138df6, 0xc91e84f8,
194         0xf83dbbd2, 0xf330b2dc, 0xee27a9ce, 0xe52aa0c0,
195         0x3cb1477a, 0x37bc4e74, 0x2aab5566, 0x21a65c68,
196         0x10856342, 0x1b886a4c, 0x069f715e, 0x0d927850,
197         0x64d90f0a, 0x6fd40604, 0x72c31d16, 0x79ce1418,
198         0x48ed2b32, 0x43e0223c, 0x5ef7392e, 0x55fa3020,
199         0x01b79aec, 0x0aba93e2, 0x17ad88f0, 0x1ca081fe,
200         0x2d83bed4, 0x268eb7da, 0x3b99acc8, 0x3094a5c6,
201         0x59dfd29c, 0x52d2db92, 0x4fc5c080, 0x44c8c98e,
202         0x75ebf6a4, 0x7ee6ffaa, 0x63f1e4b8, 0x68fcedb6,
203         0xb1670a0c, 0xba6a0302, 0xa77d1810, 0xac70111e,
204         0x9d532e34, 0x965e273a, 0x8b493c28, 0x80443526,
205         0xe90f427c, 0xe2024b72, 0xff155060, 0xf418596e,
206         0xc53b6644, 0xce366f4a, 0xd3217458, 0xd82c7d56,
207         0x7a0ca137, 0x7101a839, 0x6c16b32b, 0x671bba25,
208         0x5638850f, 0x5d358c01, 0x40229713, 0x4b2f9e1d,
209         0x2264e947, 0x2969e049, 0x347efb5b, 0x3f73f255,
210         0x0e50cd7f, 0x055dc471, 0x184adf63, 0x1347d66d,
211         0xcadc31d7, 0xc1d138d9, 0xdcc623cb, 0xd7cb2ac5,
212         0xe6e815ef, 0xede51ce1, 0xf0f207f3, 0xfbff0efd,
213         0x92b479a7, 0x99b970a9, 0x84ae6bbb, 0x8fa362b5,
214         0xbe805d9f, 0xb58d5491, 0xa89a4f83, 0xa397468d,
215     } };
216 
217     union CRYPTO_STATE st = { .l = { rm[0], rm[1] } };
218     int i;
219 
220     for (i = 0; i < 16; i += 4) {
221         CR_ST_WORD(st, i >> 2) =
222             mc[decrypt][CR_ST_BYTE(st, i)] ^
223             rol32(mc[decrypt][CR_ST_BYTE(st, i + 1)], 8) ^
224             rol32(mc[decrypt][CR_ST_BYTE(st, i + 2)], 16) ^
225             rol32(mc[decrypt][CR_ST_BYTE(st, i + 3)], 24);
226     }
227 
228     rd[0] = st.l[0];
229     rd[1] = st.l[1];
230 }
231 
232 void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t desc)
233 {
234     intptr_t i, opr_sz = simd_oprsz(desc);
235     bool decrypt = simd_data(desc);
236 
237     for (i = 0; i < opr_sz; i += 16) {
238         do_crypto_aesmc(vd + i, vm + i, decrypt);
239     }
240     clear_tail(vd, opr_sz, simd_maxsz(desc));
241 }
242 
243 /*
244  * SHA-1 logical functions
245  */
246 
247 static uint32_t cho(uint32_t x, uint32_t y, uint32_t z)
248 {
249     return (x & (y ^ z)) ^ z;
250 }
251 
252 static uint32_t par(uint32_t x, uint32_t y, uint32_t z)
253 {
254     return x ^ y ^ z;
255 }
256 
257 static uint32_t maj(uint32_t x, uint32_t y, uint32_t z)
258 {
259     return (x & y) | ((x | y) & z);
260 }
261 
262 void HELPER(crypto_sha1su0)(void *vd, void *vn, void *vm, uint32_t desc)
263 {
264     uint64_t *d = vd, *n = vn, *m = vm;
265     uint64_t d0, d1;
266 
267     d0 = d[1] ^ d[0] ^ m[0];
268     d1 = n[0] ^ d[1] ^ m[1];
269     d[0] = d0;
270     d[1] = d1;
271 
272     clear_tail_16(vd, desc);
273 }
274 
275 static inline void crypto_sha1_3reg(uint64_t *rd, uint64_t *rn,
276                                     uint64_t *rm, uint32_t desc,
277                                     uint32_t (*fn)(union CRYPTO_STATE *d))
278 {
279     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
280     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
281     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
282     int i;
283 
284     for (i = 0; i < 4; i++) {
285         uint32_t t = fn(&d);
286 
287         t += rol32(CR_ST_WORD(d, 0), 5) + CR_ST_WORD(n, 0)
288              + CR_ST_WORD(m, i);
289 
290         CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3);
291         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
292         CR_ST_WORD(d, 2) = ror32(CR_ST_WORD(d, 1), 2);
293         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
294         CR_ST_WORD(d, 0) = t;
295     }
296     rd[0] = d.l[0];
297     rd[1] = d.l[1];
298 
299     clear_tail_16(rd, desc);
300 }
301 
302 static uint32_t do_sha1c(union CRYPTO_STATE *d)
303 {
304     return cho(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
305 }
306 
307 void HELPER(crypto_sha1c)(void *vd, void *vn, void *vm, uint32_t desc)
308 {
309     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1c);
310 }
311 
312 static uint32_t do_sha1p(union CRYPTO_STATE *d)
313 {
314     return par(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
315 }
316 
317 void HELPER(crypto_sha1p)(void *vd, void *vn, void *vm, uint32_t desc)
318 {
319     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1p);
320 }
321 
322 static uint32_t do_sha1m(union CRYPTO_STATE *d)
323 {
324     return maj(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
325 }
326 
327 void HELPER(crypto_sha1m)(void *vd, void *vn, void *vm, uint32_t desc)
328 {
329     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1m);
330 }
331 
332 void HELPER(crypto_sha1h)(void *vd, void *vm, uint32_t desc)
333 {
334     uint64_t *rd = vd;
335     uint64_t *rm = vm;
336     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
337 
338     CR_ST_WORD(m, 0) = ror32(CR_ST_WORD(m, 0), 2);
339     CR_ST_WORD(m, 1) = CR_ST_WORD(m, 2) = CR_ST_WORD(m, 3) = 0;
340 
341     rd[0] = m.l[0];
342     rd[1] = m.l[1];
343 
344     clear_tail_16(vd, desc);
345 }
346 
347 void HELPER(crypto_sha1su1)(void *vd, void *vm, uint32_t desc)
348 {
349     uint64_t *rd = vd;
350     uint64_t *rm = vm;
351     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
352     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
353 
354     CR_ST_WORD(d, 0) = rol32(CR_ST_WORD(d, 0) ^ CR_ST_WORD(m, 1), 1);
355     CR_ST_WORD(d, 1) = rol32(CR_ST_WORD(d, 1) ^ CR_ST_WORD(m, 2), 1);
356     CR_ST_WORD(d, 2) = rol32(CR_ST_WORD(d, 2) ^ CR_ST_WORD(m, 3), 1);
357     CR_ST_WORD(d, 3) = rol32(CR_ST_WORD(d, 3) ^ CR_ST_WORD(d, 0), 1);
358 
359     rd[0] = d.l[0];
360     rd[1] = d.l[1];
361 
362     clear_tail_16(vd, desc);
363 }
364 
365 /*
366  * The SHA-256 logical functions, according to
367  * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf
368  */
369 
370 static uint32_t S0(uint32_t x)
371 {
372     return ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22);
373 }
374 
375 static uint32_t S1(uint32_t x)
376 {
377     return ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25);
378 }
379 
380 static uint32_t s0(uint32_t x)
381 {
382     return ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3);
383 }
384 
385 static uint32_t s1(uint32_t x)
386 {
387     return ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10);
388 }
389 
390 void HELPER(crypto_sha256h)(void *vd, void *vn, void *vm, uint32_t desc)
391 {
392     uint64_t *rd = vd;
393     uint64_t *rn = vn;
394     uint64_t *rm = vm;
395     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
396     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
397     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
398     int i;
399 
400     for (i = 0; i < 4; i++) {
401         uint32_t t = cho(CR_ST_WORD(n, 0), CR_ST_WORD(n, 1), CR_ST_WORD(n, 2))
402                      + CR_ST_WORD(n, 3) + S1(CR_ST_WORD(n, 0))
403                      + CR_ST_WORD(m, i);
404 
405         CR_ST_WORD(n, 3) = CR_ST_WORD(n, 2);
406         CR_ST_WORD(n, 2) = CR_ST_WORD(n, 1);
407         CR_ST_WORD(n, 1) = CR_ST_WORD(n, 0);
408         CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3) + t;
409 
410         t += maj(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
411              + S0(CR_ST_WORD(d, 0));
412 
413         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
414         CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
415         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
416         CR_ST_WORD(d, 0) = t;
417     }
418 
419     rd[0] = d.l[0];
420     rd[1] = d.l[1];
421 
422     clear_tail_16(vd, desc);
423 }
424 
425 void HELPER(crypto_sha256h2)(void *vd, void *vn, void *vm, uint32_t desc)
426 {
427     uint64_t *rd = vd;
428     uint64_t *rn = vn;
429     uint64_t *rm = vm;
430     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
431     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
432     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
433     int i;
434 
435     for (i = 0; i < 4; i++) {
436         uint32_t t = cho(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
437                      + CR_ST_WORD(d, 3) + S1(CR_ST_WORD(d, 0))
438                      + CR_ST_WORD(m, i);
439 
440         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
441         CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
442         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
443         CR_ST_WORD(d, 0) = CR_ST_WORD(n, 3 - i) + t;
444     }
445 
446     rd[0] = d.l[0];
447     rd[1] = d.l[1];
448 
449     clear_tail_16(vd, desc);
450 }
451 
452 void HELPER(crypto_sha256su0)(void *vd, void *vm, uint32_t desc)
453 {
454     uint64_t *rd = vd;
455     uint64_t *rm = vm;
456     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
457     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
458 
459     CR_ST_WORD(d, 0) += s0(CR_ST_WORD(d, 1));
460     CR_ST_WORD(d, 1) += s0(CR_ST_WORD(d, 2));
461     CR_ST_WORD(d, 2) += s0(CR_ST_WORD(d, 3));
462     CR_ST_WORD(d, 3) += s0(CR_ST_WORD(m, 0));
463 
464     rd[0] = d.l[0];
465     rd[1] = d.l[1];
466 
467     clear_tail_16(vd, desc);
468 }
469 
470 void HELPER(crypto_sha256su1)(void *vd, void *vn, void *vm, uint32_t desc)
471 {
472     uint64_t *rd = vd;
473     uint64_t *rn = vn;
474     uint64_t *rm = vm;
475     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
476     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
477     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
478 
479     CR_ST_WORD(d, 0) += s1(CR_ST_WORD(m, 2)) + CR_ST_WORD(n, 1);
480     CR_ST_WORD(d, 1) += s1(CR_ST_WORD(m, 3)) + CR_ST_WORD(n, 2);
481     CR_ST_WORD(d, 2) += s1(CR_ST_WORD(d, 0)) + CR_ST_WORD(n, 3);
482     CR_ST_WORD(d, 3) += s1(CR_ST_WORD(d, 1)) + CR_ST_WORD(m, 0);
483 
484     rd[0] = d.l[0];
485     rd[1] = d.l[1];
486 
487     clear_tail_16(vd, desc);
488 }
489 
490 /*
491  * The SHA-512 logical functions (same as above but using 64-bit operands)
492  */
493 
494 static uint64_t cho512(uint64_t x, uint64_t y, uint64_t z)
495 {
496     return (x & (y ^ z)) ^ z;
497 }
498 
499 static uint64_t maj512(uint64_t x, uint64_t y, uint64_t z)
500 {
501     return (x & y) | ((x | y) & z);
502 }
503 
504 static uint64_t S0_512(uint64_t x)
505 {
506     return ror64(x, 28) ^ ror64(x, 34) ^ ror64(x, 39);
507 }
508 
509 static uint64_t S1_512(uint64_t x)
510 {
511     return ror64(x, 14) ^ ror64(x, 18) ^ ror64(x, 41);
512 }
513 
514 static uint64_t s0_512(uint64_t x)
515 {
516     return ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7);
517 }
518 
519 static uint64_t s1_512(uint64_t x)
520 {
521     return ror64(x, 19) ^ ror64(x, 61) ^ (x >> 6);
522 }
523 
524 void HELPER(crypto_sha512h)(void *vd, void *vn, void *vm, uint32_t desc)
525 {
526     uint64_t *rd = vd;
527     uint64_t *rn = vn;
528     uint64_t *rm = vm;
529     uint64_t d0 = rd[0];
530     uint64_t d1 = rd[1];
531 
532     d1 += S1_512(rm[1]) + cho512(rm[1], rn[0], rn[1]);
533     d0 += S1_512(d1 + rm[0]) + cho512(d1 + rm[0], rm[1], rn[0]);
534 
535     rd[0] = d0;
536     rd[1] = d1;
537 
538     clear_tail_16(vd, desc);
539 }
540 
541 void HELPER(crypto_sha512h2)(void *vd, void *vn, void *vm, uint32_t desc)
542 {
543     uint64_t *rd = vd;
544     uint64_t *rn = vn;
545     uint64_t *rm = vm;
546     uint64_t d0 = rd[0];
547     uint64_t d1 = rd[1];
548 
549     d1 += S0_512(rm[0]) + maj512(rn[0], rm[1], rm[0]);
550     d0 += S0_512(d1) + maj512(d1, rm[0], rm[1]);
551 
552     rd[0] = d0;
553     rd[1] = d1;
554 
555     clear_tail_16(vd, desc);
556 }
557 
558 void HELPER(crypto_sha512su0)(void *vd, void *vn, uint32_t desc)
559 {
560     uint64_t *rd = vd;
561     uint64_t *rn = vn;
562     uint64_t d0 = rd[0];
563     uint64_t d1 = rd[1];
564 
565     d0 += s0_512(rd[1]);
566     d1 += s0_512(rn[0]);
567 
568     rd[0] = d0;
569     rd[1] = d1;
570 
571     clear_tail_16(vd, desc);
572 }
573 
574 void HELPER(crypto_sha512su1)(void *vd, void *vn, void *vm, uint32_t desc)
575 {
576     uint64_t *rd = vd;
577     uint64_t *rn = vn;
578     uint64_t *rm = vm;
579 
580     rd[0] += s1_512(rn[0]) + rm[0];
581     rd[1] += s1_512(rn[1]) + rm[1];
582 
583     clear_tail_16(vd, desc);
584 }
585 
586 void HELPER(crypto_sm3partw1)(void *vd, void *vn, void *vm, uint32_t desc)
587 {
588     uint64_t *rd = vd;
589     uint64_t *rn = vn;
590     uint64_t *rm = vm;
591     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
592     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
593     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
594     uint32_t t;
595 
596     t = CR_ST_WORD(d, 0) ^ CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 1), 17);
597     CR_ST_WORD(d, 0) = t ^ ror32(t, 17) ^ ror32(t, 9);
598 
599     t = CR_ST_WORD(d, 1) ^ CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 2), 17);
600     CR_ST_WORD(d, 1) = t ^ ror32(t, 17) ^ ror32(t, 9);
601 
602     t = CR_ST_WORD(d, 2) ^ CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 3), 17);
603     CR_ST_WORD(d, 2) = t ^ ror32(t, 17) ^ ror32(t, 9);
604 
605     t = CR_ST_WORD(d, 3) ^ CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 0), 17);
606     CR_ST_WORD(d, 3) = t ^ ror32(t, 17) ^ ror32(t, 9);
607 
608     rd[0] = d.l[0];
609     rd[1] = d.l[1];
610 
611     clear_tail_16(vd, desc);
612 }
613 
614 void HELPER(crypto_sm3partw2)(void *vd, void *vn, void *vm, uint32_t desc)
615 {
616     uint64_t *rd = vd;
617     uint64_t *rn = vn;
618     uint64_t *rm = vm;
619     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
620     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
621     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
622     uint32_t t = CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 0), 25);
623 
624     CR_ST_WORD(d, 0) ^= t;
625     CR_ST_WORD(d, 1) ^= CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 1), 25);
626     CR_ST_WORD(d, 2) ^= CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 2), 25);
627     CR_ST_WORD(d, 3) ^= CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(m, 3), 25) ^
628                         ror32(t, 17) ^ ror32(t, 2) ^ ror32(t, 26);
629 
630     rd[0] = d.l[0];
631     rd[1] = d.l[1];
632 
633     clear_tail_16(vd, desc);
634 }
635 
636 static inline void QEMU_ALWAYS_INLINE
637 crypto_sm3tt(uint64_t *rd, uint64_t *rn, uint64_t *rm,
638              uint32_t desc, uint32_t opcode)
639 {
640     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
641     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
642     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
643     uint32_t imm2 = simd_data(desc);
644     uint32_t t;
645 
646     assert(imm2 < 4);
647 
648     if (opcode == 0 || opcode == 2) {
649         /* SM3TT1A, SM3TT2A */
650         t = par(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
651     } else if (opcode == 1) {
652         /* SM3TT1B */
653         t = maj(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
654     } else if (opcode == 3) {
655         /* SM3TT2B */
656         t = cho(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
657     } else {
658         qemu_build_not_reached();
659     }
660 
661     t += CR_ST_WORD(d, 0) + CR_ST_WORD(m, imm2);
662 
663     CR_ST_WORD(d, 0) = CR_ST_WORD(d, 1);
664 
665     if (opcode < 2) {
666         /* SM3TT1A, SM3TT1B */
667         t += CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 3), 20);
668 
669         CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 23);
670     } else {
671         /* SM3TT2A, SM3TT2B */
672         t += CR_ST_WORD(n, 3);
673         t ^= rol32(t, 9) ^ rol32(t, 17);
674 
675         CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 13);
676     }
677 
678     CR_ST_WORD(d, 2) = CR_ST_WORD(d, 3);
679     CR_ST_WORD(d, 3) = t;
680 
681     rd[0] = d.l[0];
682     rd[1] = d.l[1];
683 
684     clear_tail_16(rd, desc);
685 }
686 
687 #define DO_SM3TT(NAME, OPCODE) \
688     void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
689     { crypto_sm3tt(vd, vn, vm, desc, OPCODE); }
690 
691 DO_SM3TT(crypto_sm3tt1a, 0)
692 DO_SM3TT(crypto_sm3tt1b, 1)
693 DO_SM3TT(crypto_sm3tt2a, 2)
694 DO_SM3TT(crypto_sm3tt2b, 3)
695 
696 #undef DO_SM3TT
697 
698 static void do_crypto_sm4e(uint64_t *rd, uint64_t *rn, uint64_t *rm)
699 {
700     union CRYPTO_STATE d = { .l = { rn[0], rn[1] } };
701     union CRYPTO_STATE n = { .l = { rm[0], rm[1] } };
702     uint32_t t, i;
703 
704     for (i = 0; i < 4; i++) {
705         t = CR_ST_WORD(d, (i + 1) % 4) ^
706             CR_ST_WORD(d, (i + 2) % 4) ^
707             CR_ST_WORD(d, (i + 3) % 4) ^
708             CR_ST_WORD(n, i);
709 
710         t = sm4_sbox[t & 0xff] |
711             sm4_sbox[(t >> 8) & 0xff] << 8 |
712             sm4_sbox[(t >> 16) & 0xff] << 16 |
713             sm4_sbox[(t >> 24) & 0xff] << 24;
714 
715         CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^
716                             rol32(t, 24);
717     }
718 
719     rd[0] = d.l[0];
720     rd[1] = d.l[1];
721 }
722 
723 void HELPER(crypto_sm4e)(void *vd, void *vn, void *vm, uint32_t desc)
724 {
725     intptr_t i, opr_sz = simd_oprsz(desc);
726 
727     for (i = 0; i < opr_sz; i += 16) {
728         do_crypto_sm4e(vd + i, vn + i, vm + i);
729     }
730     clear_tail(vd, opr_sz, simd_maxsz(desc));
731 }
732 
733 static void do_crypto_sm4ekey(uint64_t *rd, uint64_t *rn, uint64_t *rm)
734 {
735     union CRYPTO_STATE d;
736     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
737     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
738     uint32_t t, i;
739 
740     d = n;
741     for (i = 0; i < 4; i++) {
742         t = CR_ST_WORD(d, (i + 1) % 4) ^
743             CR_ST_WORD(d, (i + 2) % 4) ^
744             CR_ST_WORD(d, (i + 3) % 4) ^
745             CR_ST_WORD(m, i);
746 
747         t = sm4_sbox[t & 0xff] |
748             sm4_sbox[(t >> 8) & 0xff] << 8 |
749             sm4_sbox[(t >> 16) & 0xff] << 16 |
750             sm4_sbox[(t >> 24) & 0xff] << 24;
751 
752         CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23);
753     }
754 
755     rd[0] = d.l[0];
756     rd[1] = d.l[1];
757 }
758 
759 void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm, uint32_t desc)
760 {
761     intptr_t i, opr_sz = simd_oprsz(desc);
762 
763     for (i = 0; i < opr_sz; i += 16) {
764         do_crypto_sm4ekey(vd + i, vn + i, vm + i);
765     }
766     clear_tail(vd, opr_sz, simd_maxsz(desc));
767 }
768 
769 void HELPER(crypto_rax1)(void *vd, void *vn, void *vm, uint32_t desc)
770 {
771     intptr_t i, opr_sz = simd_oprsz(desc);
772     uint64_t *d = vd, *n = vn, *m = vm;
773 
774     for (i = 0; i < opr_sz / 8; ++i) {
775         d[i] = n[i] ^ rol64(m[i], 1);
776     }
777     clear_tail(vd, opr_sz, simd_maxsz(desc));
778 }
779