1 /*******************************************************************************
2 Copyright (c) 2018-2020, Intel Corporation
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice,
8 this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in the
11 documentation and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors
13 may be used to endorse or promote products derived from this software
14 without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *******************************************************************************/
27
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31
32 #define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_sse
33
34 #include "intel-ipsec-mb.h"
35 #include "include/ipsec_ooo_mgr.h"
36 #include "include/kasumi_internal.h"
37 #include "include/zuc_internal.h"
38 #include "include/snow3g.h"
39 #include "include/chacha20_poly1305.h"
40
41 #include "include/save_xmms.h"
42 #include "include/asm.h"
43 #include "include/des.h"
44 #include "include/gcm.h"
45 #include "include/noaesni.h"
46 #include "include/error.h"
47
48 /* ====================================================================== */
49
50 IMB_JOB *submit_job_aes128_enc_sse_no_aesni(MB_MGR_AES_OOO *state,
51 IMB_JOB *job);
52 IMB_JOB *flush_job_aes128_enc_sse_no_aesni(MB_MGR_AES_OOO *state);
53
54 IMB_JOB *submit_job_aes192_enc_sse_no_aesni(MB_MGR_AES_OOO *state,
55 IMB_JOB *job);
56 IMB_JOB *flush_job_aes192_enc_sse_no_aesni(MB_MGR_AES_OOO *state);
57
58 IMB_JOB *submit_job_aes256_enc_sse_no_aesni(MB_MGR_AES_OOO *state,
59 IMB_JOB *job);
60 IMB_JOB *flush_job_aes256_enc_sse_no_aesni(MB_MGR_AES_OOO *state);
61
62 IMB_JOB *submit_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state,
63 IMB_JOB *job);
64 IMB_JOB *flush_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state);
65
66 IMB_JOB *submit_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state,
67 IMB_JOB *job);
68 IMB_JOB *flush_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state);
69
70 IMB_JOB *submit_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state,
71 IMB_JOB *job);
72 IMB_JOB *flush_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state);
73
74 IMB_JOB *submit_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state,
75 IMB_JOB *job);
76 IMB_JOB *flush_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state);
77
78 IMB_JOB *submit_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state,
79 IMB_JOB *job);
80 IMB_JOB *flush_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state);
81
82 IMB_JOB *submit_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state,
83 IMB_JOB *job);
84 IMB_JOB *flush_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state);
85
86 IMB_JOB *submit_job_aes_xcbc_sse_no_aesni(MB_MGR_AES_XCBC_OOO *state,
87 IMB_JOB *job);
88 IMB_JOB *flush_job_aes_xcbc_sse_no_aesni(MB_MGR_AES_XCBC_OOO *state);
89
90 IMB_JOB *submit_job_aes128_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state,
91 IMB_JOB *job);
92 IMB_JOB *flush_job_aes128_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state);
93
94 IMB_JOB *submit_job_aes256_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state,
95 IMB_JOB *job);
96 IMB_JOB *flush_job_aes256_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state);
97
98 IMB_JOB *submit_job_aes128_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state,
99 IMB_JOB *job);
100
101 IMB_JOB *flush_job_aes128_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state);
102
103 IMB_JOB *submit_job_aes256_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state,
104 IMB_JOB *job);
105
106 IMB_JOB *flush_job_aes256_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state);
107
108 IMB_JOB *submit_job_aes_cntr_sse_no_aesni(IMB_JOB *job);
109
110 IMB_JOB *submit_job_aes_cntr_bit_sse_no_aesni(IMB_JOB *job);
111
112 JOB_AES_HMAC *submit_job_zuc_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state,
113 IMB_JOB *job);
114 JOB_AES_HMAC *flush_job_zuc_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state);
115
116 JOB_AES_HMAC *submit_job_zuc_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state,
117 IMB_JOB *job);
118 JOB_AES_HMAC *flush_job_zuc_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state);
119
120 uint32_t hec_32_sse_no_aesni(const uint8_t *in);
121 uint64_t hec_64_sse_no_aesni(const uint8_t *in);
122
123 IMB_JOB *submit_job_aes128_cbcs_1_9_enc_sse_no_aesni(MB_MGR_AES_OOO *state,
124 IMB_JOB *job);
125 IMB_JOB *flush_job_aes128_cbcs_1_9_enc_sse_no_aesni(MB_MGR_AES_OOO *state);
126
127 IMB_JOB *submit_job_chacha20_enc_dec_sse(IMB_JOB *job);
128
129 #define SAVE_XMMS save_xmms
130 #define RESTORE_XMMS restore_xmms
131
132 #define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse_no_aesni
133 #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni
134 #define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse_no_aesni
135 #define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_sse_no_aesni
136 #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni
137 #define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_sse_no_aesni
138 #define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse_no_aesni
139 #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse_no_aesni
140 #define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse_no_aesni
141 #define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse_no_aesni
142 #define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse_no_aesni
143 #define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse_no_aesni
144 #define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_sse_no_aesni
145 #define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_sse_no_aesni
146 #define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_sse_no_aesni
147 #define SUBMIT_JOB_HMAC submit_job_hmac_sse
148 #define FLUSH_JOB_HMAC flush_job_hmac_sse
149 #define SUBMIT_JOB_HMAC_NI submit_job_hmac_sse
150 #define FLUSH_JOB_HMAC_NI flush_job_hmac_sse
151 #define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_sse
152 #define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_sse
153 #define SUBMIT_JOB_HMAC_SHA_224_NI submit_job_hmac_sha_224_sse
154 #define FLUSH_JOB_HMAC_SHA_224_NI flush_job_hmac_sha_224_sse
155 #define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_sse
156 #define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_sse
157 #define SUBMIT_JOB_HMAC_SHA_256_NI submit_job_hmac_sha_256_sse
158 #define FLUSH_JOB_HMAC_SHA_256_NI flush_job_hmac_sha_256_sse
159 #define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_sse
160 #define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_sse
161 #define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_sse
162 #define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_sse
163 #define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_sse
164 #define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse
165 #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse_no_aesni
166 #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse_no_aesni
167
168 #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse_no_aesni
169 #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse_no_aesni
170
171 #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_sse_no_aesni
172 #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_sse_no_aesni
173 #define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_sse_no_aesni
174 #define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_sse_no_aesni
175
176 #define AES_CBC_DEC_128 aes_cbc_dec_128_sse_no_aesni
177 #define AES_CBC_DEC_192 aes_cbc_dec_192_sse_no_aesni
178 #define AES_CBC_DEC_256 aes_cbc_dec_256_sse_no_aesni
179
180 #define AES_CNTR_128 aes_cntr_128_sse_no_aesni
181 #define AES_CNTR_192 aes_cntr_192_sse_no_aesni
182 #define AES_CNTR_256 aes_cntr_256_sse_no_aesni
183
184 #define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse_no_aesni
185 #define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse_no_aesni
186
187 #define AES_ECB_ENC_128 aes_ecb_enc_128_sse_no_aesni
188 #define AES_ECB_ENC_192 aes_ecb_enc_192_sse_no_aesni
189 #define AES_ECB_ENC_256 aes_ecb_enc_256_sse_no_aesni
190 #define AES_ECB_DEC_128 aes_ecb_dec_128_sse_no_aesni
191 #define AES_ECB_DEC_192 aes_ecb_dec_192_sse_no_aesni
192 #define AES_ECB_DEC_256 aes_ecb_dec_256_sse_no_aesni
193
194 #define SUBMIT_JOB_PON_ENC submit_job_pon_enc_sse_no_aesni
195 #define SUBMIT_JOB_PON_DEC submit_job_pon_dec_sse_no_aesni
196 #define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse_no_aesni
197 #define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse_no_aesni
198
199 #define AES_GCM_DEC_128 aes_gcm_dec_128_sse_no_aesni
200 #define AES_GCM_ENC_128 aes_gcm_enc_128_sse_no_aesni
201 #define AES_GCM_DEC_192 aes_gcm_dec_192_sse_no_aesni
202 #define AES_GCM_ENC_192 aes_gcm_enc_192_sse_no_aesni
203 #define AES_GCM_DEC_256 aes_gcm_dec_256_sse_no_aesni
204 #define AES_GCM_ENC_256 aes_gcm_enc_256_sse_no_aesni
205
206 #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse_no_aesni
207 #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse_no_aesni
208 #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse_no_aesni
209 #define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_sse_no_aesni
210 #define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse_no_aesni
211 #define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse_no_aesni
212
213 #define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse_no_aesni
214 #define FLUSH_JOB_AES_GCM_DEC flush_job_aes_gcm_dec_sse_no_aesni
215 #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse_no_aesni
216 #define FLUSH_JOB_AES_GCM_ENC flush_job_aes_gcm_enc_sse_no_aesni
217
218 /* ====================================================================== */
219
220 #define SUBMIT_JOB submit_job_sse_no_aesni
221 #define FLUSH_JOB flush_job_sse_no_aesni
222 #define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse_no_aesni
223 #define GET_NEXT_JOB get_next_job_sse_no_aesni
224 #define GET_COMPLETED_JOB get_completed_job_sse_no_aesni
225
226 #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni
227 #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni
228 #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse_no_aesni
229 #define QUEUE_SIZE queue_size_sse_no_aesni
230
231 /* ====================================================================== */
232
233 #define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE
234 #define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE
235 #define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE
236 #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE
237 #define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE
238
239 /* ====================================================================== */
240
241 #define AES_CFB_128_ONE aes_cfb_128_one_sse_no_aesni
242 #define AES_CFB_256_ONE aes_cfb_256_one_sse_no_aesni
243
244 void aes128_cbc_mac_x4_no_aesni(AES_ARGS *args, uint64_t len);
245
246 #define AES128_CBC_MAC aes128_cbc_mac_x4_no_aesni
247
248 #define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_sse_no_aesni
249 #define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_sse_no_aesni
250
251 #define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_sse_no_aesni
252 #define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_sse_no_aesni
253
254 #define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_sse_no_aesni
255 #define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_sse_no_aesni
256
257 #define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_sse_no_aesni
258 #define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_sse_no_aesni
259
260 /* ====================================================================== */
261
262 #define SUBMIT_JOB_AES128_CBCS_1_9_ENC \
263 submit_job_aes128_cbcs_1_9_enc_sse_no_aesni
264 #define FLUSH_JOB_AES128_CBCS_1_9_ENC \
265 flush_job_aes128_cbcs_1_9_enc_sse_no_aesni
266 #define SUBMIT_JOB_AES128_CBCS_1_9_DEC \
267 submit_job_aes128_cbcs_1_9_dec_sse_no_aesni
268 #define AES_CBCS_1_9_DEC_128 \
269 aes_cbcs_1_9_dec_128_sse_no_aesni
270 #define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_sse
271 #define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_sse
272
273 /* ====================================================================== */
274
275 uint32_t
276 ethernet_fcs_sse_no_aesni_local(const void *msg, const uint64_t len,
277 const void *tag_ouput);
278
279 #define ETHERNET_FCS ethernet_fcs_sse_no_aesni_local
280
281 uint32_t ethernet_fcs_sse_no_aesni(const void *msg, const uint64_t len);
282 uint32_t crc16_x25_sse_no_aesni(const void *msg, const uint64_t len);
283 uint32_t crc32_sctp_sse_no_aesni(const void *msg, const uint64_t len);
284 uint32_t crc24_lte_a_sse_no_aesni(const void *msg, const uint64_t len);
285 uint32_t crc24_lte_b_sse_no_aesni(const void *msg, const uint64_t len);
286 uint32_t crc16_fp_data_sse_no_aesni(const void *msg, const uint64_t len);
287 uint32_t crc11_fp_header_sse_no_aesni(const void *msg, const uint64_t len);
288 uint32_t crc7_fp_header_sse_no_aesni(const void *msg, const uint64_t len);
289 uint32_t crc10_iuup_data_sse_no_aesni(const void *msg, const uint64_t len);
290 uint32_t crc6_iuup_header_sse_no_aesni(const void *msg, const uint64_t len);
291 uint32_t
292 crc32_wimax_ofdma_data_sse_no_aesni(const void *msg, const uint64_t len);
293 uint32_t crc8_wimax_ofdma_hcs_sse_no_aesni(const void *msg, const uint64_t len);
294
295 /* ====================================================================== */
296
297 /*
298 * GCM submit / flush API for SSE arch without AESNI
299 */
300 static IMB_JOB *
submit_job_aes_gcm_dec_sse_no_aesni(IMB_MGR * state,IMB_JOB * job)301 submit_job_aes_gcm_dec_sse_no_aesni(IMB_MGR *state, IMB_JOB *job)
302 {
303 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
304 (void) state;
305
306 if (16 == job->key_len_in_bytes) {
307 AES_GCM_DEC_IV_128(job->dec_keys,
308 &ctx, job->dst,
309 job->src +
310 job->cipher_start_src_offset_in_bytes,
311 job->msg_len_to_cipher_in_bytes,
312 job->iv, job->iv_len_in_bytes,
313 job->u.GCM.aad,
314 job->u.GCM.aad_len_in_bytes,
315 job->auth_tag_output,
316 job->auth_tag_output_len_in_bytes);
317 } else if (24 == job->key_len_in_bytes) {
318 AES_GCM_DEC_IV_192(job->dec_keys,
319 &ctx, job->dst,
320 job->src +
321 job->cipher_start_src_offset_in_bytes,
322 job->msg_len_to_cipher_in_bytes,
323 job->iv, job->iv_len_in_bytes,
324 job->u.GCM.aad,
325 job->u.GCM.aad_len_in_bytes,
326 job->auth_tag_output,
327 job->auth_tag_output_len_in_bytes);
328 } else { /* assume 32 bytes */
329 AES_GCM_DEC_IV_256(job->dec_keys,
330 &ctx, job->dst,
331 job->src +
332 job->cipher_start_src_offset_in_bytes,
333 job->msg_len_to_cipher_in_bytes,
334 job->iv, job->iv_len_in_bytes,
335 job->u.GCM.aad,
336 job->u.GCM.aad_len_in_bytes,
337 job->auth_tag_output,
338 job->auth_tag_output_len_in_bytes);
339 }
340
341 job->status = STS_COMPLETED;
342 return job;
343 }
344
345 static IMB_JOB *
flush_job_aes_gcm_dec_sse_no_aesni(IMB_MGR * state,IMB_JOB * job)346 flush_job_aes_gcm_dec_sse_no_aesni(IMB_MGR *state, IMB_JOB *job)
347 {
348 (void) state;
349 (void) job;
350 return NULL;
351 }
352
353 static IMB_JOB *
submit_job_aes_gcm_enc_sse_no_aesni(IMB_MGR * state,IMB_JOB * job)354 submit_job_aes_gcm_enc_sse_no_aesni(IMB_MGR *state, IMB_JOB *job)
355 {
356 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
357 (void) state;
358
359 if (16 == job->key_len_in_bytes) {
360 AES_GCM_ENC_IV_128(job->enc_keys,
361 &ctx, job->dst,
362 job->src +
363 job->cipher_start_src_offset_in_bytes,
364 job->msg_len_to_cipher_in_bytes,
365 job->iv, job->iv_len_in_bytes,
366 job->u.GCM.aad,
367 job->u.GCM.aad_len_in_bytes,
368 job->auth_tag_output,
369 job->auth_tag_output_len_in_bytes);
370 } else if (24 == job->key_len_in_bytes) {
371 AES_GCM_ENC_IV_192(job->enc_keys,
372 &ctx, job->dst,
373 job->src +
374 job->cipher_start_src_offset_in_bytes,
375 job->msg_len_to_cipher_in_bytes,
376 job->iv, job->iv_len_in_bytes,
377 job->u.GCM.aad,
378 job->u.GCM.aad_len_in_bytes,
379 job->auth_tag_output,
380 job->auth_tag_output_len_in_bytes);
381 } else { /* assume 32 bytes */
382 AES_GCM_ENC_IV_256(job->enc_keys,
383 &ctx, job->dst,
384 job->src +
385 job->cipher_start_src_offset_in_bytes,
386 job->msg_len_to_cipher_in_bytes,
387 job->iv, job->iv_len_in_bytes,
388 job->u.GCM.aad,
389 job->u.GCM.aad_len_in_bytes,
390 job->auth_tag_output,
391 job->auth_tag_output_len_in_bytes);
392 }
393
394 job->status = STS_COMPLETED;
395 return job;
396 }
397
398 static IMB_JOB *
flush_job_aes_gcm_enc_sse_no_aesni(IMB_MGR * state,IMB_JOB * job)399 flush_job_aes_gcm_enc_sse_no_aesni(IMB_MGR *state, IMB_JOB *job)
400 {
401 (void) state;
402 (void) job;
403 return NULL;
404 }
405
406 IMB_DLL_LOCAL IMB_JOB *
submit_job_aes_cntr_sse_no_aesni(IMB_JOB * job)407 submit_job_aes_cntr_sse_no_aesni(IMB_JOB *job)
408 {
409 if (16 == job->key_len_in_bytes)
410 AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes,
411 job->iv,
412 job->enc_keys,
413 job->dst,
414 job->msg_len_to_cipher_in_bytes,
415 job->iv_len_in_bytes);
416 else if (24 == job->key_len_in_bytes)
417 AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes,
418 job->iv,
419 job->enc_keys,
420 job->dst,
421 job->msg_len_to_cipher_in_bytes,
422 job->iv_len_in_bytes);
423 else /* assume 32 bytes */
424 AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes,
425 job->iv,
426 job->enc_keys,
427 job->dst,
428 job->msg_len_to_cipher_in_bytes,
429 job->iv_len_in_bytes);
430
431 job->status |= STS_COMPLETED_AES;
432 return job;
433 }
434
435 IMB_DLL_LOCAL IMB_JOB *
submit_job_aes_cntr_bit_sse_no_aesni(IMB_JOB * job)436 submit_job_aes_cntr_bit_sse_no_aesni(IMB_JOB *job)
437 {
438 const uint64_t offset = job->cipher_start_src_offset_in_bytes;
439
440 if (16 == job->key_len_in_bytes)
441 aes_cntr_bit_128_sse_no_aesni(job->src + offset,
442 job->iv,
443 job->enc_keys,
444 job->dst,
445 job->msg_len_to_cipher_in_bits,
446 job->iv_len_in_bytes);
447 else if (24 == job->key_len_in_bytes)
448 aes_cntr_bit_192_sse_no_aesni(job->src + offset,
449 job->iv,
450 job->enc_keys,
451 job->dst,
452 job->msg_len_to_cipher_in_bits,
453 job->iv_len_in_bytes);
454 else /* assume 32 bytes */
455 aes_cntr_bit_256_sse_no_aesni(job->src + offset,
456 job->iv,
457 job->enc_keys,
458 job->dst,
459 job->msg_len_to_cipher_in_bits,
460 job->iv_len_in_bytes);
461
462 job->status |= STS_COMPLETED_AES;
463 return job;
464 }
465
466 /* ====================================================================== */
467
468 void
init_mb_mgr_sse_no_aesni(IMB_MGR * state)469 init_mb_mgr_sse_no_aesni(IMB_MGR *state)
470 {
471 unsigned int j;
472 uint8_t *p;
473 size_t size;
474
475 #ifdef SAFE_PARAM
476 if (state == NULL) {
477 imb_set_errno(NULL, IMB_ERR_NULL_MBMGR);
478 return;
479 }
480 #endif
481
482 MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo;
483 MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo;
484 MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo;
485 MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo;
486 MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo =
487 state->docsis128_crc32_sec_ooo;
488 MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo;
489 MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo =
490 state->docsis256_crc32_sec_ooo;
491 MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo;
492 MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo;
493 MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo;
494 MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo;
495 MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo;
496 MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo;
497 MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo;
498 MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo;
499 MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo;
500 MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo;
501 MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo;
502 MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo;
503
504 /* reset error status */
505 imb_set_errno(state, 0);
506
507 /* Init AES out-of-order fields */
508 memset(aes128_ooo->lens, 0xFF,
509 sizeof(aes128_ooo->lens));
510 memset(&aes128_ooo->lens[0], 0,
511 sizeof(aes128_ooo->lens[0]) * 4);
512 memset(aes128_ooo->job_in_lane, 0,
513 sizeof(aes128_ooo->job_in_lane));
514 aes128_ooo->unused_lanes = 0xF3210;
515 aes128_ooo->num_lanes_inuse = 0;
516
517
518 memset(aes192_ooo->lens, 0xFF,
519 sizeof(aes192_ooo->lens));
520 memset(&aes192_ooo->lens[0], 0,
521 sizeof(aes192_ooo->lens[0]) * 4);
522 memset(aes192_ooo->job_in_lane, 0,
523 sizeof(aes192_ooo->job_in_lane));
524 aes192_ooo->unused_lanes = 0xF3210;
525 aes192_ooo->num_lanes_inuse = 0;
526
527
528 memset(aes256_ooo->lens, 0xFF,
529 sizeof(aes256_ooo->lens));
530 memset(&aes256_ooo->lens[0], 0,
531 sizeof(aes256_ooo->lens[0]) * 4);
532 memset(aes256_ooo->job_in_lane, 0,
533 sizeof(aes256_ooo->job_in_lane));
534 aes256_ooo->unused_lanes = 0xF3210;
535 aes256_ooo->num_lanes_inuse = 0;
536
537
538 /* DOCSIS SEC BPI uses same settings as AES CBC */
539 memset(docsis128_sec_ooo->lens, 0xFF,
540 sizeof(docsis128_sec_ooo->lens));
541 memset(&docsis128_sec_ooo->lens[0], 0,
542 sizeof(docsis128_sec_ooo->lens[0]) * 4);
543 memset(docsis128_sec_ooo->job_in_lane, 0,
544 sizeof(docsis128_sec_ooo->job_in_lane));
545 docsis128_sec_ooo->unused_lanes = 0xF3210;
546 docsis128_sec_ooo->num_lanes_inuse = 0;
547
548 memset(docsis128_crc32_sec_ooo->lens, 0xFF,
549 sizeof(docsis128_crc32_sec_ooo->lens));
550 memset(&docsis128_crc32_sec_ooo->lens[0], 0,
551 sizeof(docsis128_crc32_sec_ooo->lens[0]) * 4);
552 memset(docsis128_crc32_sec_ooo->job_in_lane, 0,
553 sizeof(docsis128_crc32_sec_ooo->job_in_lane));
554 docsis128_crc32_sec_ooo->unused_lanes = 0xF3210;
555 docsis128_crc32_sec_ooo->num_lanes_inuse = 0;
556
557 memset(docsis256_sec_ooo->lens, 0xFF,
558 sizeof(docsis256_sec_ooo->lens));
559 memset(&docsis256_sec_ooo->lens[0], 0,
560 sizeof(docsis256_sec_ooo->lens[0]) * 4);
561 memset(docsis256_sec_ooo->job_in_lane, 0,
562 sizeof(docsis256_sec_ooo->job_in_lane));
563 docsis256_sec_ooo->unused_lanes = 0xF3210;
564 docsis256_sec_ooo->num_lanes_inuse = 0;
565
566 memset(docsis256_crc32_sec_ooo->lens, 0xFF,
567 sizeof(docsis256_crc32_sec_ooo->lens));
568 memset(&docsis256_crc32_sec_ooo->lens[0], 0,
569 sizeof(docsis256_crc32_sec_ooo->lens[0]) * 4);
570 memset(docsis256_crc32_sec_ooo->job_in_lane, 0,
571 sizeof(docsis256_crc32_sec_ooo->job_in_lane));
572 docsis256_crc32_sec_ooo->unused_lanes = 0xF3210;
573 docsis256_crc32_sec_ooo->num_lanes_inuse = 0;
574
575 /* Init ZUC out-of-order fields */
576 memset(zuc_eea3_ooo->lens, 0,
577 sizeof(zuc_eea3_ooo->lens));
578 memset(zuc_eea3_ooo->job_in_lane, 0,
579 sizeof(zuc_eea3_ooo->job_in_lane));
580 zuc_eea3_ooo->unused_lanes = 0xFF03020100;
581 zuc_eea3_ooo->num_lanes_inuse = 0;
582 memset(&zuc_eea3_ooo->state, 0,
583 sizeof(zuc_eea3_ooo->state));
584 zuc_eea3_ooo->init_not_done = 0;
585 zuc_eea3_ooo->unused_lane_bitmask = 0x0f;
586
587 memset(zuc_eia3_ooo->lens, 0xFF,
588 sizeof(zuc_eia3_ooo->lens));
589 memset(zuc_eia3_ooo->job_in_lane, 0,
590 sizeof(zuc_eia3_ooo->job_in_lane));
591 zuc_eia3_ooo->unused_lanes = 0xFF03020100;
592 zuc_eia3_ooo->num_lanes_inuse = 0;
593 memset(&zuc_eia3_ooo->state, 0,
594 sizeof(zuc_eia3_ooo->state));
595 zuc_eia3_ooo->init_not_done = 0;
596 zuc_eia3_ooo->unused_lane_bitmask = 0x0f;
597
598 /* Init HMAC/SHA1 out-of-order fields */
599 hmac_sha_1_ooo->lens[0] = 0;
600 hmac_sha_1_ooo->lens[1] = 0;
601 hmac_sha_1_ooo->lens[2] = 0;
602 hmac_sha_1_ooo->lens[3] = 0;
603 hmac_sha_1_ooo->lens[4] = 0xFFFF;
604 hmac_sha_1_ooo->lens[5] = 0xFFFF;
605 hmac_sha_1_ooo->lens[6] = 0xFFFF;
606 hmac_sha_1_ooo->lens[7] = 0xFFFF;
607 hmac_sha_1_ooo->unused_lanes = 0xFF03020100;
608 for (j = 0; j < SSE_NUM_SHA1_LANES; j++) {
609 hmac_sha_1_ooo->ldata[j].job_in_lane = NULL;
610 hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80;
611 memset(hmac_sha_1_ooo->ldata[j].extra_block + 65,
612 0x00,
613 64+7);
614 p = hmac_sha_1_ooo->ldata[j].outer_block;
615 memset(p + 5*4 + 1,
616 0x00,
617 64 - 5*4 - 1 - 2);
618 p[5*4] = 0x80;
619 p[64-2] = 0x02;
620 p[64-1] = 0xA0;
621 }
622
623 /* Init HMAC/SHA224 out-of-order fields */
624 hmac_sha_224_ooo->lens[0] = 0;
625 hmac_sha_224_ooo->lens[1] = 0;
626 hmac_sha_224_ooo->lens[2] = 0;
627 hmac_sha_224_ooo->lens[3] = 0;
628 hmac_sha_224_ooo->lens[4] = 0xFFFF;
629 hmac_sha_224_ooo->lens[5] = 0xFFFF;
630 hmac_sha_224_ooo->lens[6] = 0xFFFF;
631 hmac_sha_224_ooo->lens[7] = 0xFFFF;
632 hmac_sha_224_ooo->unused_lanes = 0xFF03020100;
633 for (j = 0; j < SSE_NUM_SHA256_LANES; j++) {
634 hmac_sha_224_ooo->ldata[j].job_in_lane = NULL;
635
636 p = hmac_sha_224_ooo->ldata[j].extra_block;
637 size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block);
638 memset (p, 0x00, size);
639 p[64] = 0x80;
640
641 p = hmac_sha_224_ooo->ldata[j].outer_block;
642 size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block);
643 memset(p, 0x00, size);
644 p[7*4] = 0x80; /* digest 7 words long */
645 p[64-2] = 0x02; /* length in little endian = 0x02E0 */
646 p[64-1] = 0xE0;
647 }
648
649 /* Init HMAC/SHA_256 out-of-order fields */
650 hmac_sha_256_ooo->lens[0] = 0;
651 hmac_sha_256_ooo->lens[1] = 0;
652 hmac_sha_256_ooo->lens[2] = 0;
653 hmac_sha_256_ooo->lens[3] = 0;
654 hmac_sha_256_ooo->lens[4] = 0xFFFF;
655 hmac_sha_256_ooo->lens[5] = 0xFFFF;
656 hmac_sha_256_ooo->lens[6] = 0xFFFF;
657 hmac_sha_256_ooo->lens[7] = 0xFFFF;
658 hmac_sha_256_ooo->unused_lanes = 0xFF03020100;
659 for (j = 0; j < SSE_NUM_SHA256_LANES; j++) {
660 hmac_sha_256_ooo->ldata[j].job_in_lane = NULL;
661 hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80;
662 memset(hmac_sha_256_ooo->ldata[j].extra_block + 65,
663 0x00,
664 64+7);
665 p = hmac_sha_256_ooo->ldata[j].outer_block;
666 memset(p + 8*4 + 1,
667 0x00,
668 64 - 8*4 - 1 - 2); /* digest is 8*4 bytes long */
669 p[8*4] = 0x80;
670 p[64-2] = 0x03; /* length of (opad (64*8) bits + 256 bits)
671 * in hex is 0x300 */
672 p[64-1] = 0x00;
673 }
674
675 /* Init HMAC/SHA384 out-of-order fields */
676 hmac_sha_384_ooo->lens[0] = 0;
677 hmac_sha_384_ooo->lens[1] = 0;
678 hmac_sha_384_ooo->lens[2] = 0xFFFF;
679 hmac_sha_384_ooo->lens[3] = 0xFFFF;
680 hmac_sha_384_ooo->lens[4] = 0xFFFF;
681 hmac_sha_384_ooo->lens[5] = 0xFFFF;
682 hmac_sha_384_ooo->lens[6] = 0xFFFF;
683 hmac_sha_384_ooo->lens[7] = 0xFFFF;
684 hmac_sha_384_ooo->unused_lanes = 0xFF0100;
685 for (j = 0; j < SSE_NUM_SHA512_LANES; j++) {
686 MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo;
687
688 ctx->ldata[j].job_in_lane = NULL;
689 ctx->ldata[j].extra_block[SHA_384_BLOCK_SIZE] = 0x80;
690 memset(ctx->ldata[j].extra_block + (SHA_384_BLOCK_SIZE + 1),
691 0x00, SHA_384_BLOCK_SIZE + 7);
692
693 p = ctx->ldata[j].outer_block;
694 memset(p + SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00,
695 /* special end point because this length is constant */
696 SHA_384_BLOCK_SIZE -
697 SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2);
698 p[SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */
699 /*
700 * hmac outer block length always of fixed size, it is OKey
701 * length, a whole message block length, 1024 bits, with padding
702 * plus the length of the inner digest, which is 384 bits
703 * 1408 bits == 0x0580. The input message block needs to be
704 * converted to big endian within the sha implementation
705 * before use.
706 */
707 p[SHA_384_BLOCK_SIZE - 2] = 0x05;
708 p[SHA_384_BLOCK_SIZE - 1] = 0x80;
709 }
710
711 /* Init HMAC/SHA512 out-of-order fields */
712 hmac_sha_512_ooo->lens[0] = 0;
713 hmac_sha_512_ooo->lens[1] = 0;
714 hmac_sha_512_ooo->lens[2] = 0xFFFF;
715 hmac_sha_512_ooo->lens[3] = 0xFFFF;
716 hmac_sha_512_ooo->lens[4] = 0xFFFF;
717 hmac_sha_512_ooo->lens[5] = 0xFFFF;
718 hmac_sha_512_ooo->lens[6] = 0xFFFF;
719 hmac_sha_512_ooo->lens[7] = 0xFFFF;
720 hmac_sha_512_ooo->unused_lanes = 0xFF0100;
721 for (j = 0; j < SSE_NUM_SHA512_LANES; j++) {
722 MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo;
723
724 ctx->ldata[j].job_in_lane = NULL;
725 ctx->ldata[j].extra_block[SHA_512_BLOCK_SIZE] = 0x80;
726 memset(ctx->ldata[j].extra_block + (SHA_512_BLOCK_SIZE + 1),
727 0x00, SHA_512_BLOCK_SIZE + 7);
728
729 p = ctx->ldata[j].outer_block;
730 memset(p + SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00,
731 /* special end point because this length is constant */
732 SHA_512_BLOCK_SIZE -
733 SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2);
734 p[SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */
735 /*
736 * hmac outer block length always of fixed size, it is OKey
737 * length, a whole message block length, 1024 bits, with padding
738 * plus the length of the inner digest, which is 512 bits
739 * 1536 bits == 0x600. The input message block needs to be
740 * converted to big endian within the sha implementation
741 * before use.
742 */
743 p[SHA_512_BLOCK_SIZE - 2] = 0x06;
744 p[SHA_512_BLOCK_SIZE - 1] = 0x00;
745 }
746
747 /* Init HMAC/MD5 out-of-order fields */
748 hmac_md5_ooo->lens[0] = 0;
749 hmac_md5_ooo->lens[1] = 0;
750 hmac_md5_ooo->lens[2] = 0;
751 hmac_md5_ooo->lens[3] = 0;
752 hmac_md5_ooo->lens[4] = 0;
753 hmac_md5_ooo->lens[5] = 0;
754 hmac_md5_ooo->lens[6] = 0;
755 hmac_md5_ooo->lens[7] = 0;
756 hmac_md5_ooo->lens[8] = 0xFFFF;
757 hmac_md5_ooo->lens[9] = 0xFFFF;
758 hmac_md5_ooo->lens[10] = 0xFFFF;
759 hmac_md5_ooo->lens[11] = 0xFFFF;
760 hmac_md5_ooo->lens[12] = 0xFFFF;
761 hmac_md5_ooo->lens[13] = 0xFFFF;
762 hmac_md5_ooo->lens[14] = 0xFFFF;
763 hmac_md5_ooo->lens[15] = 0xFFFF;
764 hmac_md5_ooo->unused_lanes = 0xF76543210;
765 for (j = 0; j < SSE_NUM_MD5_LANES; j++) {
766 hmac_md5_ooo->ldata[j].job_in_lane = NULL;
767
768 p = hmac_md5_ooo->ldata[j].extra_block;
769 size = sizeof(hmac_md5_ooo->ldata[j].extra_block);
770 memset (p, 0x00, size);
771 p[64] = 0x80;
772
773 p = hmac_md5_ooo->ldata[j].outer_block;
774 size = sizeof(hmac_md5_ooo->ldata[j].outer_block);
775 memset(p, 0x00, size);
776 p[4*4] = 0x80;
777 p[64-7] = 0x02;
778 p[64-8] = 0x80;
779 }
780
781 /* Init AES/XCBC OOO fields */
782 memset(aes_xcbc_ooo->lens, 0xff,
783 sizeof(aes_xcbc_ooo->lens));
784 aes_xcbc_ooo->unused_lanes = 0xFF03020100;
785 for (j = 0; j < 4; j++) {
786 aes_xcbc_ooo->lens[j] = 0xFFFF;
787 aes_xcbc_ooo->ldata[j].job_in_lane = NULL;
788 aes_xcbc_ooo->ldata[j].final_block[16] = 0x80;
789 memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15);
790 }
791 aes_xcbc_ooo->num_lanes_inuse = 0;
792
793 /* Init AES-CCM auth out-of-order fields */
794 memset(aes_ccm_ooo, 0, sizeof(MB_MGR_CCM_OOO));
795 for (j = 4; j < 16; j++)
796 aes_ccm_ooo->lens[j] = 0xFFFF;
797 aes_ccm_ooo->unused_lanes = 0xF3210;
798 aes_ccm_ooo->num_lanes_inuse = 0;
799
800 /* Init AES-CMAC auth out-of-order fields */
801 aes_cmac_ooo->lens[0] = 0;
802 aes_cmac_ooo->lens[1] = 0;
803 aes_cmac_ooo->lens[2] = 0;
804 aes_cmac_ooo->lens[3] = 0;
805 aes_cmac_ooo->lens[4] = 0xFFFF;
806 aes_cmac_ooo->lens[5] = 0xFFFF;
807 aes_cmac_ooo->lens[6] = 0xFFFF;
808 aes_cmac_ooo->lens[7] = 0xFFFF;
809 for (j = 0; j < 4; j++) {
810 aes_cmac_ooo->init_done[j] = 0;
811 aes_cmac_ooo->job_in_lane[j] = NULL;
812 }
813 aes_cmac_ooo->unused_lanes = 0xF3210;
814 aes_cmac_ooo->num_lanes_inuse = 0;
815
816 /* Init AES-CBCS out-of-order fields */
817 memset(aes128_cbcs_ooo->lens, 0xFF, sizeof(aes128_cbcs_ooo->lens));
818 memset(aes128_cbcs_ooo->job_in_lane, 0,
819 sizeof(aes128_cbcs_ooo->job_in_lane));
820 aes128_cbcs_ooo->num_lanes_inuse = 0;
821 aes128_cbcs_ooo->unused_lanes = 0xF3210;
822
823 /* Init "in order" components */
824 state->next_job = 0;
825 state->earliest_job = -1;
826
827 /* set SSE NO AESNI handlers */
828 state->get_next_job = get_next_job_sse_no_aesni;
829 state->submit_job = submit_job_sse_no_aesni;
830 state->submit_job_nocheck = submit_job_nocheck_sse_no_aesni;
831 state->get_completed_job = get_completed_job_sse_no_aesni;
832 state->flush_job = flush_job_sse_no_aesni;
833 state->queue_size = queue_size_sse_no_aesni;
834 state->keyexp_128 = aes_keyexp_128_sse_no_aesni;
835 state->keyexp_192 = aes_keyexp_192_sse_no_aesni;
836 state->keyexp_256 = aes_keyexp_256_sse_no_aesni;
837 state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_sse_no_aesni;
838 state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_sse_no_aesni;
839 state->xcbc_keyexp = aes_xcbc_expand_key_sse_no_aesni;
840 state->des_key_sched = des_key_schedule;
841 state->sha1_one_block = sha1_one_block_sse;
842 state->sha1 = sha1_sse;
843 state->sha224_one_block = sha224_one_block_sse;
844 state->sha224 = sha224_sse;
845 state->sha256_one_block = sha256_one_block_sse;
846 state->sha256 = sha256_sse;
847 state->sha384_one_block = sha384_one_block_sse;
848 state->sha384 = sha384_sse;
849 state->sha512_one_block = sha512_one_block_sse;
850 state->sha512 = sha512_sse;
851 state->md5_one_block = md5_one_block_sse;
852 state->aes128_cfb_one = aes_cfb_128_one_sse_no_aesni;
853
854 state->eea3_1_buffer = zuc_eea3_1_buffer_sse_no_aesni;
855 state->eea3_4_buffer = zuc_eea3_4_buffer_sse_no_aesni;
856 state->eea3_n_buffer = zuc_eea3_n_buffer_sse_no_aesni;
857 state->eia3_1_buffer = zuc_eia3_1_buffer_sse_no_aesni;
858 state->eia3_n_buffer = zuc_eia3_n_buffer_sse_no_aesni;
859
860 state->f8_1_buffer = kasumi_f8_1_buffer_sse;
861 state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_sse;
862 state->f8_2_buffer = kasumi_f8_2_buffer_sse;
863 state->f8_3_buffer = kasumi_f8_3_buffer_sse;
864 state->f8_4_buffer = kasumi_f8_4_buffer_sse;
865 state->f8_n_buffer = kasumi_f8_n_buffer_sse;
866 state->f9_1_buffer = kasumi_f9_1_buffer_sse;
867 state->f9_1_buffer_user = kasumi_f9_1_buffer_user_sse;
868 state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_sse;
869 state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_sse;
870 state->kasumi_key_sched_size = kasumi_key_sched_size_sse;
871
872 state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_sse_no_aesni;
873 state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_sse_no_aesni;
874 state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_sse_no_aesni;
875 state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_sse_no_aesni;
876 state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_sse_no_aesni;
877 state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_sse_no_aesni;
878 state->snow3g_f8_8_buffer_multikey =
879 snow3g_f8_8_buffer_multikey_sse_no_aesni;
880 state->snow3g_f8_n_buffer_multikey =
881 snow3g_f8_n_buffer_multikey_sse_no_aesni;
882 state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_sse_no_aesni;
883 state->snow3g_init_key_sched = snow3g_init_key_sched_sse_no_aesni;
884 state->snow3g_key_sched_size = snow3g_key_sched_size_sse_no_aesni;
885
886 state->gcm128_enc = aes_gcm_enc_128_sse_no_aesni;
887 state->gcm192_enc = aes_gcm_enc_192_sse_no_aesni;
888 state->gcm256_enc = aes_gcm_enc_256_sse_no_aesni;
889 state->gcm128_dec = aes_gcm_dec_128_sse_no_aesni;
890 state->gcm192_dec = aes_gcm_dec_192_sse_no_aesni;
891 state->gcm256_dec = aes_gcm_dec_256_sse_no_aesni;
892 state->gcm128_init = aes_gcm_init_128_sse_no_aesni;
893 state->gcm192_init = aes_gcm_init_192_sse_no_aesni;
894 state->gcm256_init = aes_gcm_init_256_sse_no_aesni;
895 state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_sse_no_aesni;
896 state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_sse_no_aesni;
897 state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_sse_no_aesni;
898 state->gcm128_enc_update = aes_gcm_enc_128_update_sse_no_aesni;
899 state->gcm192_enc_update = aes_gcm_enc_192_update_sse_no_aesni;
900 state->gcm256_enc_update = aes_gcm_enc_256_update_sse_no_aesni;
901 state->gcm128_dec_update = aes_gcm_dec_128_update_sse_no_aesni;
902 state->gcm192_dec_update = aes_gcm_dec_192_update_sse_no_aesni;
903 state->gcm256_dec_update = aes_gcm_dec_256_update_sse_no_aesni;
904 state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse_no_aesni;
905 state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse_no_aesni;
906 state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse_no_aesni;
907 state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse_no_aesni;
908 state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse_no_aesni;
909 state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse_no_aesni;
910 state->gcm128_precomp = aes_gcm_precomp_128_sse_no_aesni;
911 state->gcm192_precomp = aes_gcm_precomp_192_sse_no_aesni;
912 state->gcm256_precomp = aes_gcm_precomp_256_sse_no_aesni;
913 state->gcm128_pre = aes_gcm_pre_128_sse_no_aesni;
914 state->gcm192_pre = aes_gcm_pre_192_sse_no_aesni;
915 state->gcm256_pre = aes_gcm_pre_256_sse_no_aesni;
916 state->ghash = ghash_sse_no_aesni;
917 state->ghash_pre = ghash_pre_sse_no_aesni;
918
919 state->gmac128_init = imb_aes_gmac_init_128_sse_no_aesni;
920 state->gmac192_init = imb_aes_gmac_init_192_sse_no_aesni;
921 state->gmac256_init = imb_aes_gmac_init_256_sse_no_aesni;
922 state->gmac128_update = imb_aes_gmac_update_128_sse_no_aesni;
923 state->gmac192_update = imb_aes_gmac_update_192_sse_no_aesni;
924 state->gmac256_update = imb_aes_gmac_update_256_sse_no_aesni;
925 state->gmac128_finalize = imb_aes_gmac_finalize_128_sse_no_aesni;
926 state->gmac192_finalize = imb_aes_gmac_finalize_192_sse_no_aesni;
927 state->gmac256_finalize = imb_aes_gmac_finalize_256_sse_no_aesni;
928
929 state->hec_32 = hec_32_sse_no_aesni;
930 state->hec_64 = hec_64_sse_no_aesni;
931 state->crc32_ethernet_fcs = ethernet_fcs_sse_no_aesni;
932 state->crc16_x25 = crc16_x25_sse_no_aesni;
933 state->crc32_sctp = crc32_sctp_sse_no_aesni;
934 state->crc24_lte_a = crc24_lte_a_sse_no_aesni;
935 state->crc24_lte_b = crc24_lte_b_sse_no_aesni;
936 state->crc16_fp_data = crc16_fp_data_sse_no_aesni;
937 state->crc11_fp_header = crc11_fp_header_sse_no_aesni;
938 state->crc7_fp_header = crc7_fp_header_sse_no_aesni;
939 state->crc10_iuup_data = crc10_iuup_data_sse_no_aesni;
940 state->crc6_iuup_header = crc6_iuup_header_sse_no_aesni;
941 state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_sse_no_aesni;
942 state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_sse_no_aesni;
943 }
944
945 #include "mb_mgr_code.h"
946