1 /*******************************************************************************
2 Copyright (c) 2012-2020, Intel Corporation
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice,
8 this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in the
11 documentation and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors
13 may be used to endorse or promote products derived from this software
14 without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *******************************************************************************/
27
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31
32 #define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_avx
33
34 #include "intel-ipsec-mb.h"
35 #include "include/ipsec_ooo_mgr.h"
36 #include "include/kasumi_internal.h"
37 #include "include/zuc_internal.h"
38 #include "include/snow3g.h"
39 #include "include/gcm.h"
40 #include "include/chacha20_poly1305.h"
41
42 #include "include/save_xmms.h"
43 #include "include/asm.h"
44 #include "include/des.h"
45 #include "include/cpu_feature.h"
46 #include "include/noaesni.h"
47 #include "include/error.h"
48
49 IMB_JOB *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state,
50 IMB_JOB *job);
51 IMB_JOB *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state);
52
53 IMB_JOB *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state,
54 IMB_JOB *job);
55 IMB_JOB *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state);
56
57 IMB_JOB *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state,
58 IMB_JOB *job);
59 IMB_JOB *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state);
60
61 IMB_JOB *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state,
62 IMB_JOB *job);
63 IMB_JOB *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state);
64
65 IMB_JOB *submit_job_aes_cntr_avx(IMB_JOB *job);
66
67 IMB_JOB *submit_job_aes_cntr_bit_avx(IMB_JOB *job);
68
69 IMB_JOB *submit_job_zuc_eea3_avx(MB_MGR_ZUC_OOO *state,
70 IMB_JOB *job);
71 IMB_JOB *flush_job_zuc_eea3_avx(MB_MGR_ZUC_OOO *state);
72
73 IMB_JOB *submit_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state,
74 IMB_JOB *job);
75 IMB_JOB *flush_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state);
76
77 uint32_t hec_32_avx(const uint8_t *in);
78 uint64_t hec_64_avx(const uint8_t *in);
79
80 IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state,
81 IMB_JOB *job);
82 IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state);
83
84 IMB_JOB *submit_job_chacha20_enc_dec_avx(IMB_JOB *job);
85
86 #define SAVE_XMMS save_xmms_avx
87 #define RESTORE_XMMS restore_xmms_avx
88
89 #define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx
90 #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx
91 #define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx
92 #define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx
93 #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx
94 #define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx
95 #define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx
96 #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx
97 #define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx
98 #define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx
99 #define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx
100 #define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx
101 #define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx
102 #define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx
103 #define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx
104
105 #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx
106 #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx
107
108 #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx
109 #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx
110 #define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx
111 #define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx
112
113 #define AES_CBC_DEC_128 aes_cbc_dec_128_avx
114 #define AES_CBC_DEC_192 aes_cbc_dec_192_avx
115 #define AES_CBC_DEC_256 aes_cbc_dec_256_avx
116
117 #define AES_CNTR_128 aes_cntr_128_avx
118 #define AES_CNTR_192 aes_cntr_192_avx
119 #define AES_CNTR_256 aes_cntr_256_avx
120
121 #define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx
122 #define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx
123
124 #define AES_ECB_ENC_128 aes_ecb_enc_128_avx
125 #define AES_ECB_ENC_192 aes_ecb_enc_192_avx
126 #define AES_ECB_ENC_256 aes_ecb_enc_256_avx
127 #define AES_ECB_DEC_128 aes_ecb_dec_128_avx
128 #define AES_ECB_DEC_192 aes_ecb_dec_192_avx
129 #define AES_ECB_DEC_256 aes_ecb_dec_256_avx
130
131 #define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx
132 #define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx
133 #define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx
134 #define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx
135
136 #define AES_GCM_DEC_128 aes_gcm_dec_128_avx_gen2
137 #define AES_GCM_ENC_128 aes_gcm_enc_128_avx_gen2
138 #define AES_GCM_DEC_192 aes_gcm_dec_192_avx_gen2
139 #define AES_GCM_ENC_192 aes_gcm_enc_192_avx_gen2
140 #define AES_GCM_DEC_256 aes_gcm_dec_256_avx_gen2
141 #define AES_GCM_ENC_256 aes_gcm_enc_256_avx_gen2
142
143 #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen2
144 #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen2
145 #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen2
146 #define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen2
147 #define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen2
148 #define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen2
149
150 #define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx
151 #define FLUSH_JOB_AES_GCM_DEC flush_job_aes_gcm_dec_avx
152 #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx
153 #define FLUSH_JOB_AES_GCM_ENC flush_job_aes_gcm_enc_avx
154
155 #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx
156 #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx
157
158 #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx
159 #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx
160 #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx
161 #define QUEUE_SIZE queue_size_avx
162
163 #define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX
164 #define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX
165 #define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX
166 #define FLUSH_JOB_AES_DEC FLUSH_JOB_AES_DEC_AVX
167
168 #define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx
169
170 #define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx
171
172 IMB_JOB *submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state,
173 IMB_JOB *job);
174 IMB_JOB *flush_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state);
175
176 IMB_JOB *submit_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state,
177 IMB_JOB *job);
178 IMB_JOB *flush_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state);
179
180 IMB_JOB *submit_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state,
181 IMB_JOB *job);
182 IMB_JOB *flush_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state);
183
184 IMB_JOB *submit_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state,
185 IMB_JOB *job);
186 IMB_JOB *flush_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state);
187
188 IMB_JOB *submit_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state,
189 IMB_JOB *job);
190 IMB_JOB *flush_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state);
191
192 IMB_JOB *submit_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state,
193 IMB_JOB *job);
194 IMB_JOB *flush_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state);
195
196 IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state,
197 IMB_JOB *job);
198
199 IMB_JOB *flush_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state);
200
201 IMB_JOB *submit_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state,
202 IMB_JOB *job);
203
204 IMB_JOB *flush_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state);
205
206 IMB_JOB *submit_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state,
207 IMB_JOB *job);
208
209 IMB_JOB *flush_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state);
210
211 IMB_JOB *submit_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state,
212 IMB_JOB *job);
213
214 IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state);
215
216 void aes_cmac_256_subkey_gen_avx(const void *key_exp,
217 void *key1, void *key2);
218
219 #define SUBMIT_JOB_HMAC submit_job_hmac_avx
220 #define FLUSH_JOB_HMAC flush_job_hmac_avx
221 #define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx
222 #define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx
223 #define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx
224 #define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx
225 #define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx
226 #define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx
227 #define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx
228 #define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx
229 #define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx
230 #define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx
231
232 /* ====================================================================== */
233
234 #define SUBMIT_JOB submit_job_avx
235 #define FLUSH_JOB flush_job_avx
236 #define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx
237 #define GET_NEXT_JOB get_next_job_avx
238 #define GET_COMPLETED_JOB get_completed_job_avx
239
240 /* ====================================================================== */
241
242
243 #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX
244 #define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX
245
246 /* ====================================================================== */
247
248 #define AES_CFB_128_ONE aes_cfb_128_one_avx
249 #define AES_CFB_256_ONE aes_cfb_256_one_avx
250
251 void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len);
252
253 #define AES128_CBC_MAC aes128_cbc_mac_x8
254
255 #define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx
256 #define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx
257
258 #define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx
259 #define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx
260
261 #define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx
262 #define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx
263
264 #define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx
265 #define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx
266
267 /* ====================================================================== */
268
269 uint32_t
270 ethernet_fcs_avx_local(const void *msg, const uint64_t len,
271 const void *tag_ouput);
272
273 #define ETHERNET_FCS ethernet_fcs_avx_local
274
275 uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len);
276 uint32_t crc16_x25_avx(const void *msg, const uint64_t len);
277 uint32_t crc32_sctp_avx(const void *msg, const uint64_t len);
278 uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len);
279 uint32_t crc24_lte_b_avx(const void *msg, const uint64_t len);
280 uint32_t crc16_fp_data_avx(const void *msg, const uint64_t len);
281 uint32_t crc11_fp_header_avx(const void *msg, const uint64_t len);
282 uint32_t crc7_fp_header_avx(const void *msg, const uint64_t len);
283 uint32_t crc10_iuup_data_avx(const void *msg, const uint64_t len);
284 uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len);
285 uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len);
286 uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len);
287
288 /* ====================================================================== */
289
290 #define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx
291 #define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx
292 #define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx
293 #define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx
294
295 /* ====================================================================== */
296
297 /*
298 * GCM submit / flush API for AVX arch
299 */
300 static IMB_JOB *
submit_job_aes_gcm_dec_avx(IMB_MGR * state,IMB_JOB * job)301 submit_job_aes_gcm_dec_avx(IMB_MGR *state, IMB_JOB *job)
302 {
303 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
304 (void) state;
305
306 if (16 == job->key_len_in_bytes) {
307 AES_GCM_DEC_IV_128(job->dec_keys,
308 &ctx, job->dst,
309 job->src +
310 job->cipher_start_src_offset_in_bytes,
311 job->msg_len_to_cipher_in_bytes,
312 job->iv, job->iv_len_in_bytes,
313 job->u.GCM.aad,
314 job->u.GCM.aad_len_in_bytes,
315 job->auth_tag_output,
316 job->auth_tag_output_len_in_bytes);
317 } else if (24 == job->key_len_in_bytes) {
318 AES_GCM_DEC_IV_192(job->dec_keys,
319 &ctx, job->dst,
320 job->src +
321 job->cipher_start_src_offset_in_bytes,
322 job->msg_len_to_cipher_in_bytes,
323 job->iv, job->iv_len_in_bytes,
324 job->u.GCM.aad,
325 job->u.GCM.aad_len_in_bytes,
326 job->auth_tag_output,
327 job->auth_tag_output_len_in_bytes);
328 } else { /* assume 32 bytes */
329 AES_GCM_DEC_IV_256(job->dec_keys,
330 &ctx, job->dst,
331 job->src +
332 job->cipher_start_src_offset_in_bytes,
333 job->msg_len_to_cipher_in_bytes,
334 job->iv, job->iv_len_in_bytes,
335 job->u.GCM.aad,
336 job->u.GCM.aad_len_in_bytes,
337 job->auth_tag_output,
338 job->auth_tag_output_len_in_bytes);
339 }
340
341 job->status = STS_COMPLETED;
342 return job;
343 }
344
345 static IMB_JOB *
flush_job_aes_gcm_dec_avx(IMB_MGR * state,IMB_JOB * job)346 flush_job_aes_gcm_dec_avx(IMB_MGR *state, IMB_JOB *job)
347 {
348 (void) state;
349 (void) job;
350 return NULL;
351 }
352
353 static IMB_JOB *
submit_job_aes_gcm_enc_avx(IMB_MGR * state,IMB_JOB * job)354 submit_job_aes_gcm_enc_avx(IMB_MGR *state, IMB_JOB *job)
355 {
356 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
357 (void) state;
358
359 if (16 == job->key_len_in_bytes) {
360 AES_GCM_ENC_IV_128(job->enc_keys,
361 &ctx, job->dst,
362 job->src +
363 job->cipher_start_src_offset_in_bytes,
364 job->msg_len_to_cipher_in_bytes,
365 job->iv, job->iv_len_in_bytes,
366 job->u.GCM.aad,
367 job->u.GCM.aad_len_in_bytes,
368 job->auth_tag_output,
369 job->auth_tag_output_len_in_bytes);
370 } else if (24 == job->key_len_in_bytes) {
371 AES_GCM_ENC_IV_192(job->enc_keys,
372 &ctx, job->dst,
373 job->src +
374 job->cipher_start_src_offset_in_bytes,
375 job->msg_len_to_cipher_in_bytes,
376 job->iv, job->iv_len_in_bytes,
377 job->u.GCM.aad,
378 job->u.GCM.aad_len_in_bytes,
379 job->auth_tag_output,
380 job->auth_tag_output_len_in_bytes);
381 } else { /* assume 32 bytes */
382 AES_GCM_ENC_IV_256(job->enc_keys,
383 &ctx, job->dst,
384 job->src +
385 job->cipher_start_src_offset_in_bytes,
386 job->msg_len_to_cipher_in_bytes,
387 job->iv, job->iv_len_in_bytes,
388 job->u.GCM.aad,
389 job->u.GCM.aad_len_in_bytes,
390 job->auth_tag_output,
391 job->auth_tag_output_len_in_bytes);
392 }
393
394 job->status = STS_COMPLETED;
395 return job;
396 }
397
398 static IMB_JOB *
flush_job_aes_gcm_enc_avx(IMB_MGR * state,IMB_JOB * job)399 flush_job_aes_gcm_enc_avx(IMB_MGR *state, IMB_JOB *job)
400 {
401 (void) state;
402 (void) job;
403 return NULL;
404 }
405
406 /* ====================================================================== */
407
408 IMB_DLL_LOCAL IMB_JOB *
submit_job_aes_cntr_avx(IMB_JOB * job)409 submit_job_aes_cntr_avx(IMB_JOB *job)
410 {
411 if (16 == job->key_len_in_bytes)
412 AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes,
413 job->iv,
414 job->enc_keys,
415 job->dst,
416 job->msg_len_to_cipher_in_bytes,
417 job->iv_len_in_bytes);
418 else if (24 == job->key_len_in_bytes)
419 AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes,
420 job->iv,
421 job->enc_keys,
422 job->dst,
423 job->msg_len_to_cipher_in_bytes,
424 job->iv_len_in_bytes);
425 else /* assume 32 bytes */
426 AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes,
427 job->iv,
428 job->enc_keys,
429 job->dst,
430 job->msg_len_to_cipher_in_bytes,
431 job->iv_len_in_bytes);
432
433 job->status |= STS_COMPLETED_AES;
434 return job;
435 }
436
437 IMB_DLL_LOCAL IMB_JOB *
submit_job_aes_cntr_bit_avx(IMB_JOB * job)438 submit_job_aes_cntr_bit_avx(IMB_JOB *job)
439 {
440 if (16 == job->key_len_in_bytes)
441 aes_cntr_bit_128_avx(job->src +
442 job->cipher_start_src_offset_in_bytes,
443 job->iv,
444 job->enc_keys,
445 job->dst,
446 job->msg_len_to_cipher_in_bits,
447 job->iv_len_in_bytes);
448 else if (24 == job->key_len_in_bytes)
449 aes_cntr_bit_192_avx(job->src +
450 job->cipher_start_src_offset_in_bytes,
451 job->iv,
452 job->enc_keys,
453 job->dst,
454 job->msg_len_to_cipher_in_bits,
455 job->iv_len_in_bytes);
456 else /* assume 32 bytes */
457 aes_cntr_bit_256_avx(job->src +
458 job->cipher_start_src_offset_in_bytes,
459 job->iv,
460 job->enc_keys,
461 job->dst,
462 job->msg_len_to_cipher_in_bits,
463 job->iv_len_in_bytes);
464
465 job->status |= STS_COMPLETED_AES;
466 return job;
467 }
468
469 void
init_mb_mgr_avx(IMB_MGR * state)470 init_mb_mgr_avx(IMB_MGR *state)
471 {
472 unsigned int j;
473 uint8_t *p;
474 size_t size;
475
476 #ifdef SAFE_PARAM
477 if (state == NULL) {
478 imb_set_errno(NULL, IMB_ERR_NULL_MBMGR);
479 return;
480 }
481 #endif
482
483 MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo;
484 MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo;
485 MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo;
486 MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo;
487 MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo =
488 state->docsis128_crc32_sec_ooo;
489 MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo;
490 MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo =
491 state->docsis256_crc32_sec_ooo;
492 MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo;
493 MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo;
494 MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo;
495 MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo;
496 MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo;
497 MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo;
498 MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo;
499 MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo;
500 MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo;
501 MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo;
502 MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo;
503 MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo;
504
505 /* reset error status */
506 imb_set_errno(state, 0);
507
508 state->features = cpu_feature_adjust(state->flags,
509 cpu_feature_detect());
510
511 if (!(state->features & IMB_FEATURE_AESNI)) {
512 init_mb_mgr_sse_no_aesni(state);
513 return;
514 }
515
516 /* Init AES out-of-order fields */
517 memset(aes128_ooo->lens, 0xFF,
518 sizeof(aes128_ooo->lens));
519 memset(&aes128_ooo->lens[0], 0,
520 sizeof(aes128_ooo->lens[0]) * 8);
521 memset(aes128_ooo->job_in_lane, 0,
522 sizeof(aes128_ooo->job_in_lane));
523 aes128_ooo->unused_lanes = 0xF76543210;
524 aes128_ooo->num_lanes_inuse = 0;
525
526 memset(aes192_ooo->lens, 0xFF,
527 sizeof(aes192_ooo->lens));
528 memset(&aes192_ooo->lens[0], 0,
529 sizeof(aes192_ooo->lens[0]) * 8);
530 memset(aes192_ooo->job_in_lane, 0,
531 sizeof(aes192_ooo->job_in_lane));
532 aes192_ooo->unused_lanes = 0xF76543210;
533 aes192_ooo->num_lanes_inuse = 0;
534
535 memset(&aes256_ooo->lens, 0xFF,
536 sizeof(aes256_ooo->lens));
537 memset(&aes256_ooo->lens[0], 0,
538 sizeof(aes256_ooo->lens[0]) * 8);
539 memset(aes256_ooo->job_in_lane, 0,
540 sizeof(aes256_ooo->job_in_lane));
541 aes256_ooo->unused_lanes = 0xF76543210;
542 aes256_ooo->num_lanes_inuse = 0;
543
544 /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block)
545 * uses same settings as AES CBC.
546 */
547 memset(docsis128_sec_ooo->lens, 0xFF,
548 sizeof(docsis128_sec_ooo->lens));
549 memset(&docsis128_sec_ooo->lens[0], 0,
550 sizeof(docsis128_sec_ooo->lens[0]) * 8);
551 memset(docsis128_sec_ooo->job_in_lane, 0,
552 sizeof(docsis128_sec_ooo->job_in_lane));
553 docsis128_sec_ooo->unused_lanes = 0xF76543210;
554 docsis128_sec_ooo->num_lanes_inuse = 0;
555
556 memset(docsis128_crc32_sec_ooo->lens, 0xFF,
557 sizeof(docsis128_crc32_sec_ooo->lens));
558 memset(&docsis128_crc32_sec_ooo->lens[0], 0,
559 sizeof(docsis128_crc32_sec_ooo->lens[0]) * 8);
560 memset(docsis128_crc32_sec_ooo->job_in_lane, 0,
561 sizeof(docsis128_crc32_sec_ooo->job_in_lane));
562 docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210;
563 docsis128_crc32_sec_ooo->num_lanes_inuse = 0;
564
565 memset(docsis256_sec_ooo->lens, 0xFF,
566 sizeof(docsis256_sec_ooo->lens));
567 memset(&docsis256_sec_ooo->lens[0], 0,
568 sizeof(docsis256_sec_ooo->lens[0]) * 8);
569 memset(docsis256_sec_ooo->job_in_lane, 0,
570 sizeof(docsis256_sec_ooo->job_in_lane));
571 docsis256_sec_ooo->unused_lanes = 0xF76543210;
572 docsis256_sec_ooo->num_lanes_inuse = 0;
573
574 memset(docsis256_crc32_sec_ooo->lens, 0xFF,
575 sizeof(docsis256_crc32_sec_ooo->lens));
576 memset(&docsis256_crc32_sec_ooo->lens[0], 0,
577 sizeof(docsis256_crc32_sec_ooo->lens[0]) * 8);
578 memset(docsis256_crc32_sec_ooo->job_in_lane, 0,
579 sizeof(docsis256_crc32_sec_ooo->job_in_lane));
580 docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210;
581 docsis256_crc32_sec_ooo->num_lanes_inuse = 0;
582
583 /* Init ZUC out-of-order fields */
584 memset(zuc_eea3_ooo->lens, 0,
585 sizeof(zuc_eea3_ooo->lens));
586 memset(zuc_eea3_ooo->job_in_lane, 0,
587 sizeof(zuc_eea3_ooo->job_in_lane));
588 zuc_eea3_ooo->unused_lanes = 0xFF03020100;
589 zuc_eea3_ooo->num_lanes_inuse = 0;
590 memset(&zuc_eea3_ooo->state, 0,
591 sizeof(zuc_eea3_ooo->state));
592 zuc_eea3_ooo->init_not_done = 0;
593 zuc_eea3_ooo->unused_lane_bitmask = 0x0f;
594
595 memset(zuc_eia3_ooo->lens, 0xFF,
596 sizeof(zuc_eia3_ooo->lens));
597 memset(zuc_eia3_ooo->job_in_lane, 0,
598 sizeof(zuc_eia3_ooo->job_in_lane));
599 zuc_eia3_ooo->unused_lanes = 0xFF03020100;
600 zuc_eia3_ooo->num_lanes_inuse = 0;
601 memset(&zuc_eia3_ooo->state, 0,
602 sizeof(zuc_eia3_ooo->state));
603 zuc_eia3_ooo->init_not_done = 0;
604 zuc_eia3_ooo->unused_lane_bitmask = 0x0f;
605
606 /* Init HMAC/SHA1 out-of-order fields */
607 hmac_sha_1_ooo->lens[0] = 0;
608 hmac_sha_1_ooo->lens[1] = 0;
609 hmac_sha_1_ooo->lens[2] = 0;
610 hmac_sha_1_ooo->lens[3] = 0;
611 hmac_sha_1_ooo->lens[4] = 0xFFFF;
612 hmac_sha_1_ooo->lens[5] = 0xFFFF;
613 hmac_sha_1_ooo->lens[6] = 0xFFFF;
614 hmac_sha_1_ooo->lens[7] = 0xFFFF;
615 hmac_sha_1_ooo->unused_lanes = 0xFF03020100;
616 for (j = 0; j < AVX_NUM_SHA1_LANES; j++) {
617 hmac_sha_1_ooo->ldata[j].job_in_lane = NULL;
618 hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80;
619 memset(hmac_sha_1_ooo->ldata[j].extra_block + 65,
620 0x00,
621 64+7);
622 p = hmac_sha_1_ooo->ldata[j].outer_block;
623 memset(p + 5*4 + 1,
624 0x00,
625 64 - 5*4 - 1 - 2);
626 p[5*4] = 0x80;
627 p[64-2] = 0x02;
628 p[64-1] = 0xA0;
629 }
630 /* Init HMAC/SHA224 out-of-order fields */
631 hmac_sha_224_ooo->lens[0] = 0;
632 hmac_sha_224_ooo->lens[1] = 0;
633 hmac_sha_224_ooo->lens[2] = 0;
634 hmac_sha_224_ooo->lens[3] = 0;
635 hmac_sha_224_ooo->lens[4] = 0xFFFF;
636 hmac_sha_224_ooo->lens[5] = 0xFFFF;
637 hmac_sha_224_ooo->lens[6] = 0xFFFF;
638 hmac_sha_224_ooo->lens[7] = 0xFFFF;
639 hmac_sha_224_ooo->unused_lanes = 0xFF03020100;
640 for (j = 0; j < AVX_NUM_SHA256_LANES; j++) {
641 hmac_sha_224_ooo->ldata[j].job_in_lane = NULL;
642
643 p = hmac_sha_224_ooo->ldata[j].extra_block;
644 size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block);
645 memset (p, 0x00, size);
646 p[64] = 0x80;
647
648 p = hmac_sha_224_ooo->ldata[j].outer_block;
649 size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block);
650 memset(p, 0x00, size);
651 p[7 * 4] = 0x80; /* digest 7 words long */
652 p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */
653 p[64 - 1] = 0xE0;
654 }
655
656 /* Init HMAC/SHA256 out-of-order fields */
657 hmac_sha_256_ooo->lens[0] = 0;
658 hmac_sha_256_ooo->lens[1] = 0;
659 hmac_sha_256_ooo->lens[2] = 0;
660 hmac_sha_256_ooo->lens[3] = 0;
661 hmac_sha_256_ooo->lens[4] = 0xFFFF;
662 hmac_sha_256_ooo->lens[5] = 0xFFFF;
663 hmac_sha_256_ooo->lens[6] = 0xFFFF;
664 hmac_sha_256_ooo->lens[7] = 0xFFFF;
665 hmac_sha_256_ooo->unused_lanes = 0xFF03020100;
666 for (j = 0; j < AVX_NUM_SHA256_LANES; j++) {
667 hmac_sha_256_ooo->ldata[j].job_in_lane = NULL;
668 hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80;
669 memset(hmac_sha_256_ooo->ldata[j].extra_block + 65,
670 0x00,
671 64+7);
672 p = hmac_sha_256_ooo->ldata[j].outer_block;
673 memset(p + 8*4 + 1,
674 0x00,
675 64 - 8*4 - 1 - 2);
676 p[8 * 4] = 0x80; /* 8 digest words */
677 p[64 - 2] = 0x03; /* length */
678 p[64 - 1] = 0x00;
679 }
680
681
682 /* Init HMAC/SHA384 out-of-order fields */
683 hmac_sha_384_ooo->lens[0] = 0;
684 hmac_sha_384_ooo->lens[1] = 0;
685 hmac_sha_384_ooo->lens[2] = 0xFFFF;
686 hmac_sha_384_ooo->lens[3] = 0xFFFF;
687 hmac_sha_384_ooo->lens[4] = 0xFFFF;
688 hmac_sha_384_ooo->lens[5] = 0xFFFF;
689 hmac_sha_384_ooo->lens[6] = 0xFFFF;
690 hmac_sha_384_ooo->lens[7] = 0xFFFF;
691 hmac_sha_384_ooo->unused_lanes = 0xFF0100;
692 for (j = 0; j < AVX_NUM_SHA512_LANES; j++) {
693 MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo;
694
695 ctx->ldata[j].job_in_lane = NULL;
696 ctx->ldata[j].extra_block[SHA_384_BLOCK_SIZE] = 0x80;
697 memset(ctx->ldata[j].extra_block + (SHA_384_BLOCK_SIZE + 1),
698 0x00, SHA_384_BLOCK_SIZE + 7);
699
700 p = ctx->ldata[j].outer_block;
701 memset(p + SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00,
702 /* special end point because this length is constant */
703 SHA_384_BLOCK_SIZE -
704 SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2);
705 /* mark the end */
706 p[SHA384_DIGEST_SIZE_IN_BYTES] = 0x80;
707 /* hmac outer block length always of fixed size,
708 * it is OKey length, a whole message block length, 1024 bits,
709 * with padding plus the length of the inner digest,
710 * which is 384 bits, 1408 bits == 0x0580.
711 * The input message block needs to be converted to big endian
712 * within the sha implementation before use.
713 */
714 p[SHA_384_BLOCK_SIZE - 2] = 0x05;
715 p[SHA_384_BLOCK_SIZE - 1] = 0x80;
716 }
717
718 /* Init HMAC/SHA512 out-of-order fields */
719 hmac_sha_512_ooo->lens[0] = 0;
720 hmac_sha_512_ooo->lens[1] = 0;
721 hmac_sha_512_ooo->lens[2] = 0xFFFF;
722 hmac_sha_512_ooo->lens[3] = 0xFFFF;
723 hmac_sha_512_ooo->lens[4] = 0xFFFF;
724 hmac_sha_512_ooo->lens[5] = 0xFFFF;
725 hmac_sha_512_ooo->lens[6] = 0xFFFF;
726 hmac_sha_512_ooo->lens[7] = 0xFFFF;
727 hmac_sha_512_ooo->unused_lanes = 0xFF0100;
728 for (j = 0; j < AVX_NUM_SHA512_LANES; j++) {
729 MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo;
730
731 ctx->ldata[j].job_in_lane = NULL;
732 ctx->ldata[j].extra_block[SHA_512_BLOCK_SIZE] = 0x80;
733 memset(ctx->ldata[j].extra_block + (SHA_512_BLOCK_SIZE + 1),
734 0x00, SHA_512_BLOCK_SIZE + 7);
735 p = ctx->ldata[j].outer_block;
736 memset(p + SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00,
737 /* special end point because this length is constant */
738 SHA_512_BLOCK_SIZE -
739 SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2);
740 /* mark the end */
741 p[SHA512_DIGEST_SIZE_IN_BYTES] = 0x80;
742 /*
743 * hmac outer block length always of fixed size,
744 * it is OKey length, a whole message block length, 1024 bits,
745 * with padding plus the length of the inner digest,
746 * which is 512 bits, 1536 bits == 0x600.
747 * The input message block needs to be converted to big endian
748 * within the sha implementation before use.
749 */
750 p[SHA_512_BLOCK_SIZE - 2] = 0x06;
751 p[SHA_512_BLOCK_SIZE - 1] = 0x00;
752 }
753
754
755 /* Init HMAC/MD5 out-of-order fields */
756 hmac_md5_ooo->lens[0] = 0;
757 hmac_md5_ooo->lens[1] = 0;
758 hmac_md5_ooo->lens[2] = 0;
759 hmac_md5_ooo->lens[3] = 0;
760 hmac_md5_ooo->lens[4] = 0;
761 hmac_md5_ooo->lens[5] = 0;
762 hmac_md5_ooo->lens[6] = 0;
763 hmac_md5_ooo->lens[7] = 0;
764 hmac_md5_ooo->lens[8] = 0xFFFF;
765 hmac_md5_ooo->lens[9] = 0xFFFF;
766 hmac_md5_ooo->lens[10] = 0xFFFF;
767 hmac_md5_ooo->lens[11] = 0xFFFF;
768 hmac_md5_ooo->lens[12] = 0xFFFF;
769 hmac_md5_ooo->lens[13] = 0xFFFF;
770 hmac_md5_ooo->lens[14] = 0xFFFF;
771 hmac_md5_ooo->lens[15] = 0xFFFF;
772 hmac_md5_ooo->unused_lanes = 0xF76543210;
773 for (j = 0; j < AVX_NUM_MD5_LANES; j++) {
774 hmac_md5_ooo->ldata[j].job_in_lane = NULL;
775
776 p = hmac_md5_ooo->ldata[j].extra_block;
777 size = sizeof(hmac_md5_ooo->ldata[j].extra_block);
778 memset (p, 0x00, size);
779 p[64] = 0x80;
780
781 p = hmac_md5_ooo->ldata[j].outer_block;
782 size = sizeof(hmac_md5_ooo->ldata[j].outer_block);
783 memset(p, 0x00, size);
784 p[4 * 4] = 0x80;
785 p[64 - 7] = 0x02;
786 p[64 - 8] = 0x80;
787 }
788
789 /* Init AES/XCBC OOO fields */
790 memset(aes_xcbc_ooo->lens, 0xff,
791 sizeof(aes_xcbc_ooo->lens));
792 aes_xcbc_ooo->unused_lanes = 0xF76543210;
793 for (j = 0; j < 8; j++) {
794 aes_xcbc_ooo->lens[j] = 0;
795 aes_xcbc_ooo->ldata[j].job_in_lane = NULL;
796 aes_xcbc_ooo->ldata[j].final_block[16] = 0x80;
797 memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15);
798 }
799 aes_xcbc_ooo->num_lanes_inuse = 0;
800
801 /* Init AES-CCM auth out-of-order fields */
802 for (j = 0; j < 8; j++) {
803 aes_ccm_ooo->init_done[j] = 0;
804 aes_ccm_ooo->lens[j] = 0;
805 aes_ccm_ooo->job_in_lane[j] = NULL;
806 }
807 for (; j < 16; j++)
808 aes_ccm_ooo->lens[j] = 0xFFFF;
809
810 aes_ccm_ooo->unused_lanes = 0xF76543210;
811 aes_ccm_ooo->num_lanes_inuse = 0;
812
813 /* Init AES-CMAC auth out-of-order fields */
814 for (j = 0; j < 8; j++) {
815 aes_cmac_ooo->init_done[j] = 0;
816 aes_cmac_ooo->lens[j] = 0;
817 aes_cmac_ooo->job_in_lane[j] = NULL;
818 }
819 aes_cmac_ooo->unused_lanes = 0xF76543210;
820 aes_cmac_ooo->num_lanes_inuse = 0;
821
822 /* Init AES CBC-S out-of-order fields */
823 memset(aes128_cbcs_ooo->lens, 0xFF,
824 sizeof(aes128_cbcs_ooo->lens));
825 memset(&aes128_cbcs_ooo->lens[0], 0,
826 sizeof(aes128_cbcs_ooo->lens[0]) * 8);
827 memset(aes128_cbcs_ooo->job_in_lane, 0,
828 sizeof(aes128_cbcs_ooo->job_in_lane));
829 aes128_cbcs_ooo->unused_lanes = 0xF76543210;
830 aes128_cbcs_ooo->num_lanes_inuse = 0;
831
832 /* Init "in order" components */
833 state->next_job = 0;
834 state->earliest_job = -1;
835
836 /* set AVX handlers */
837 state->get_next_job = get_next_job_avx;
838 state->submit_job = submit_job_avx;
839 state->submit_job_nocheck = submit_job_nocheck_avx;
840 state->get_completed_job = get_completed_job_avx;
841 state->flush_job = flush_job_avx;
842 state->queue_size = queue_size_avx;
843 state->keyexp_128 = aes_keyexp_128_avx;
844 state->keyexp_192 = aes_keyexp_192_avx;
845 state->keyexp_256 = aes_keyexp_256_avx;
846 state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx;
847 state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx;
848 state->xcbc_keyexp = aes_xcbc_expand_key_avx;
849 state->des_key_sched = des_key_schedule;
850 state->sha1_one_block = sha1_one_block_avx;
851 state->sha1 = sha1_avx;
852 state->sha224_one_block = sha224_one_block_avx;
853 state->sha224 = sha224_avx;
854 state->sha256_one_block = sha256_one_block_avx;
855 state->sha256 = sha256_avx;
856 state->sha384_one_block = sha384_one_block_avx;
857 state->sha384 = sha384_avx;
858 state->sha512_one_block = sha512_one_block_avx;
859 state->sha512 = sha512_avx;
860 state->md5_one_block = md5_one_block_avx;
861 state->aes128_cfb_one = aes_cfb_128_one_avx;
862
863 state->eea3_1_buffer = zuc_eea3_1_buffer_avx;
864 state->eea3_4_buffer = zuc_eea3_4_buffer_avx;
865 state->eea3_n_buffer = zuc_eea3_n_buffer_avx;
866 state->eia3_1_buffer = zuc_eia3_1_buffer_avx;
867 state->eia3_n_buffer = zuc_eia3_n_buffer_avx;
868
869 state->f8_1_buffer = kasumi_f8_1_buffer_avx;
870 state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx;
871 state->f8_2_buffer = kasumi_f8_2_buffer_avx;
872 state->f8_3_buffer = kasumi_f8_3_buffer_avx;
873 state->f8_4_buffer = kasumi_f8_4_buffer_avx;
874 state->f8_n_buffer = kasumi_f8_n_buffer_avx;
875 state->f9_1_buffer = kasumi_f9_1_buffer_avx;
876 state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx;
877 state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx;
878 state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx;
879 state->kasumi_key_sched_size = kasumi_key_sched_size_avx;
880
881 state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx;
882 state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx;
883 state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx;
884 state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx;
885 state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx;
886 state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx;
887 state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx;
888 state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx;
889 state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx;
890 state->snow3g_init_key_sched = snow3g_init_key_sched_avx;
891 state->snow3g_key_sched_size = snow3g_key_sched_size_avx;
892
893 state->gcm128_enc = aes_gcm_enc_128_avx_gen2;
894 state->gcm192_enc = aes_gcm_enc_192_avx_gen2;
895 state->gcm256_enc = aes_gcm_enc_256_avx_gen2;
896 state->gcm128_dec = aes_gcm_dec_128_avx_gen2;
897 state->gcm192_dec = aes_gcm_dec_192_avx_gen2;
898 state->gcm256_dec = aes_gcm_dec_256_avx_gen2;
899 state->gcm128_init = aes_gcm_init_128_avx_gen2;
900 state->gcm192_init = aes_gcm_init_192_avx_gen2;
901 state->gcm256_init = aes_gcm_init_256_avx_gen2;
902 state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen2;
903 state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen2;
904 state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen2;
905 state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen2;
906 state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen2;
907 state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen2;
908 state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen2;
909 state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen2;
910 state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen2;
911 state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen2;
912 state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen2;
913 state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen2;
914 state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen2;
915 state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen2;
916 state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen2;
917 state->gcm128_precomp = aes_gcm_precomp_128_avx_gen2;
918 state->gcm192_precomp = aes_gcm_precomp_192_avx_gen2;
919 state->gcm256_precomp = aes_gcm_precomp_256_avx_gen2;
920 state->gcm128_pre = aes_gcm_pre_128_avx_gen2;
921 state->gcm192_pre = aes_gcm_pre_192_avx_gen2;
922 state->gcm256_pre = aes_gcm_pre_256_avx_gen2;
923 state->ghash = ghash_avx_gen2;
924 state->ghash_pre = ghash_pre_avx_gen2;
925
926 state->gmac128_init = imb_aes_gmac_init_128_avx_gen2;
927 state->gmac192_init = imb_aes_gmac_init_192_avx_gen2;
928 state->gmac256_init = imb_aes_gmac_init_256_avx_gen2;
929 state->gmac128_update = imb_aes_gmac_update_128_avx_gen2;
930 state->gmac192_update = imb_aes_gmac_update_192_avx_gen2;
931 state->gmac256_update = imb_aes_gmac_update_256_avx_gen2;
932 state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen2;
933 state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen2;
934 state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen2;
935 state->hec_32 = hec_32_avx;
936 state->hec_64 = hec_64_avx;
937 state->crc32_ethernet_fcs = ethernet_fcs_avx;
938 state->crc16_x25 = crc16_x25_avx;
939 state->crc32_sctp = crc32_sctp_avx;
940 state->crc24_lte_a = crc24_lte_a_avx;
941 state->crc24_lte_b = crc24_lte_b_avx;
942 state->crc16_fp_data = crc16_fp_data_avx;
943 state->crc11_fp_header = crc11_fp_header_avx;
944 state->crc7_fp_header = crc7_fp_header_avx;
945 state->crc10_iuup_data = crc10_iuup_data_avx;
946 state->crc6_iuup_header = crc6_iuup_header_avx;
947 state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx;
948 state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx;
949 }
950
951 #include "mb_mgr_code.h"
952