1#include <machine/asm.h>
2.text
3.globl	aesni_encrypt
4.type	aesni_encrypt,@function
5.align	16
6aesni_encrypt:
7.L_aesni_encrypt_begin:
8	movl	4(%esp),%eax
9	movl	12(%esp),%edx
10	movups	(%eax),%xmm2
11	movl	240(%edx),%ecx
12	movl	8(%esp),%eax
13	movups	(%edx),%xmm0
14	movups	16(%edx),%xmm1
15	leal	32(%edx),%edx
16	xorps	%xmm0,%xmm2
17.L000enc1_loop_1:
18.byte	102,15,56,220,209
19	decl	%ecx
20	movups	(%edx),%xmm1
21	leal	16(%edx),%edx
22	jnz	.L000enc1_loop_1
23.byte	102,15,56,221,209
24	movups	%xmm2,(%eax)
25	ret
26.size	aesni_encrypt,.-.L_aesni_encrypt_begin
27.globl	aesni_decrypt
28.type	aesni_decrypt,@function
29.align	16
30aesni_decrypt:
31.L_aesni_decrypt_begin:
32	movl	4(%esp),%eax
33	movl	12(%esp),%edx
34	movups	(%eax),%xmm2
35	movl	240(%edx),%ecx
36	movl	8(%esp),%eax
37	movups	(%edx),%xmm0
38	movups	16(%edx),%xmm1
39	leal	32(%edx),%edx
40	xorps	%xmm0,%xmm2
41.L001dec1_loop_2:
42.byte	102,15,56,222,209
43	decl	%ecx
44	movups	(%edx),%xmm1
45	leal	16(%edx),%edx
46	jnz	.L001dec1_loop_2
47.byte	102,15,56,223,209
48	movups	%xmm2,(%eax)
49	ret
50.size	aesni_decrypt,.-.L_aesni_decrypt_begin
51.type	_aesni_encrypt3,@function
52.align	16
53_aesni_encrypt3:
54	movups	(%edx),%xmm0
55	shrl	$1,%ecx
56	movups	16(%edx),%xmm1
57	leal	32(%edx),%edx
58	xorps	%xmm0,%xmm2
59	pxor	%xmm0,%xmm3
60	pxor	%xmm0,%xmm4
61	movups	(%edx),%xmm0
62.L002enc3_loop:
63.byte	102,15,56,220,209
64.byte	102,15,56,220,217
65	decl	%ecx
66.byte	102,15,56,220,225
67	movups	16(%edx),%xmm1
68.byte	102,15,56,220,208
69.byte	102,15,56,220,216
70	leal	32(%edx),%edx
71.byte	102,15,56,220,224
72	movups	(%edx),%xmm0
73	jnz	.L002enc3_loop
74.byte	102,15,56,220,209
75.byte	102,15,56,220,217
76.byte	102,15,56,220,225
77.byte	102,15,56,221,208
78.byte	102,15,56,221,216
79.byte	102,15,56,221,224
80	ret
81.size	_aesni_encrypt3,.-_aesni_encrypt3
82.type	_aesni_decrypt3,@function
83.align	16
84_aesni_decrypt3:
85	movups	(%edx),%xmm0
86	shrl	$1,%ecx
87	movups	16(%edx),%xmm1
88	leal	32(%edx),%edx
89	xorps	%xmm0,%xmm2
90	pxor	%xmm0,%xmm3
91	pxor	%xmm0,%xmm4
92	movups	(%edx),%xmm0
93.L003dec3_loop:
94.byte	102,15,56,222,209
95.byte	102,15,56,222,217
96	decl	%ecx
97.byte	102,15,56,222,225
98	movups	16(%edx),%xmm1
99.byte	102,15,56,222,208
100.byte	102,15,56,222,216
101	leal	32(%edx),%edx
102.byte	102,15,56,222,224
103	movups	(%edx),%xmm0
104	jnz	.L003dec3_loop
105.byte	102,15,56,222,209
106.byte	102,15,56,222,217
107.byte	102,15,56,222,225
108.byte	102,15,56,223,208
109.byte	102,15,56,223,216
110.byte	102,15,56,223,224
111	ret
112.size	_aesni_decrypt3,.-_aesni_decrypt3
113.type	_aesni_encrypt4,@function
114.align	16
115_aesni_encrypt4:
116	movups	(%edx),%xmm0
117	movups	16(%edx),%xmm1
118	shrl	$1,%ecx
119	leal	32(%edx),%edx
120	xorps	%xmm0,%xmm2
121	pxor	%xmm0,%xmm3
122	pxor	%xmm0,%xmm4
123	pxor	%xmm0,%xmm5
124	movups	(%edx),%xmm0
125.L004enc4_loop:
126.byte	102,15,56,220,209
127.byte	102,15,56,220,217
128	decl	%ecx
129.byte	102,15,56,220,225
130.byte	102,15,56,220,233
131	movups	16(%edx),%xmm1
132.byte	102,15,56,220,208
133.byte	102,15,56,220,216
134	leal	32(%edx),%edx
135.byte	102,15,56,220,224
136.byte	102,15,56,220,232
137	movups	(%edx),%xmm0
138	jnz	.L004enc4_loop
139.byte	102,15,56,220,209
140.byte	102,15,56,220,217
141.byte	102,15,56,220,225
142.byte	102,15,56,220,233
143.byte	102,15,56,221,208
144.byte	102,15,56,221,216
145.byte	102,15,56,221,224
146.byte	102,15,56,221,232
147	ret
148.size	_aesni_encrypt4,.-_aesni_encrypt4
149.type	_aesni_decrypt4,@function
150.align	16
151_aesni_decrypt4:
152	movups	(%edx),%xmm0
153	movups	16(%edx),%xmm1
154	shrl	$1,%ecx
155	leal	32(%edx),%edx
156	xorps	%xmm0,%xmm2
157	pxor	%xmm0,%xmm3
158	pxor	%xmm0,%xmm4
159	pxor	%xmm0,%xmm5
160	movups	(%edx),%xmm0
161.L005dec4_loop:
162.byte	102,15,56,222,209
163.byte	102,15,56,222,217
164	decl	%ecx
165.byte	102,15,56,222,225
166.byte	102,15,56,222,233
167	movups	16(%edx),%xmm1
168.byte	102,15,56,222,208
169.byte	102,15,56,222,216
170	leal	32(%edx),%edx
171.byte	102,15,56,222,224
172.byte	102,15,56,222,232
173	movups	(%edx),%xmm0
174	jnz	.L005dec4_loop
175.byte	102,15,56,222,209
176.byte	102,15,56,222,217
177.byte	102,15,56,222,225
178.byte	102,15,56,222,233
179.byte	102,15,56,223,208
180.byte	102,15,56,223,216
181.byte	102,15,56,223,224
182.byte	102,15,56,223,232
183	ret
184.size	_aesni_decrypt4,.-_aesni_decrypt4
185.type	_aesni_encrypt6,@function
186.align	16
187_aesni_encrypt6:
188	movups	(%edx),%xmm0
189	shrl	$1,%ecx
190	movups	16(%edx),%xmm1
191	leal	32(%edx),%edx
192	xorps	%xmm0,%xmm2
193	pxor	%xmm0,%xmm3
194.byte	102,15,56,220,209
195	pxor	%xmm0,%xmm4
196.byte	102,15,56,220,217
197	pxor	%xmm0,%xmm5
198	decl	%ecx
199.byte	102,15,56,220,225
200	pxor	%xmm0,%xmm6
201.byte	102,15,56,220,233
202	pxor	%xmm0,%xmm7
203.byte	102,15,56,220,241
204	movups	(%edx),%xmm0
205.byte	102,15,56,220,249
206	jmp	.L_aesni_encrypt6_enter
207.align	16
208.L006enc6_loop:
209.byte	102,15,56,220,209
210.byte	102,15,56,220,217
211	decl	%ecx
212.byte	102,15,56,220,225
213.byte	102,15,56,220,233
214.byte	102,15,56,220,241
215.byte	102,15,56,220,249
216.align	16
217.L_aesni_encrypt6_enter:
218	movups	16(%edx),%xmm1
219.byte	102,15,56,220,208
220.byte	102,15,56,220,216
221	leal	32(%edx),%edx
222.byte	102,15,56,220,224
223.byte	102,15,56,220,232
224.byte	102,15,56,220,240
225.byte	102,15,56,220,248
226	movups	(%edx),%xmm0
227	jnz	.L006enc6_loop
228.byte	102,15,56,220,209
229.byte	102,15,56,220,217
230.byte	102,15,56,220,225
231.byte	102,15,56,220,233
232.byte	102,15,56,220,241
233.byte	102,15,56,220,249
234.byte	102,15,56,221,208
235.byte	102,15,56,221,216
236.byte	102,15,56,221,224
237.byte	102,15,56,221,232
238.byte	102,15,56,221,240
239.byte	102,15,56,221,248
240	ret
241.size	_aesni_encrypt6,.-_aesni_encrypt6
242.type	_aesni_decrypt6,@function
243.align	16
244_aesni_decrypt6:
245	movups	(%edx),%xmm0
246	shrl	$1,%ecx
247	movups	16(%edx),%xmm1
248	leal	32(%edx),%edx
249	xorps	%xmm0,%xmm2
250	pxor	%xmm0,%xmm3
251.byte	102,15,56,222,209
252	pxor	%xmm0,%xmm4
253.byte	102,15,56,222,217
254	pxor	%xmm0,%xmm5
255	decl	%ecx
256.byte	102,15,56,222,225
257	pxor	%xmm0,%xmm6
258.byte	102,15,56,222,233
259	pxor	%xmm0,%xmm7
260.byte	102,15,56,222,241
261	movups	(%edx),%xmm0
262.byte	102,15,56,222,249
263	jmp	.L_aesni_decrypt6_enter
264.align	16
265.L007dec6_loop:
266.byte	102,15,56,222,209
267.byte	102,15,56,222,217
268	decl	%ecx
269.byte	102,15,56,222,225
270.byte	102,15,56,222,233
271.byte	102,15,56,222,241
272.byte	102,15,56,222,249
273.align	16
274.L_aesni_decrypt6_enter:
275	movups	16(%edx),%xmm1
276.byte	102,15,56,222,208
277.byte	102,15,56,222,216
278	leal	32(%edx),%edx
279.byte	102,15,56,222,224
280.byte	102,15,56,222,232
281.byte	102,15,56,222,240
282.byte	102,15,56,222,248
283	movups	(%edx),%xmm0
284	jnz	.L007dec6_loop
285.byte	102,15,56,222,209
286.byte	102,15,56,222,217
287.byte	102,15,56,222,225
288.byte	102,15,56,222,233
289.byte	102,15,56,222,241
290.byte	102,15,56,222,249
291.byte	102,15,56,223,208
292.byte	102,15,56,223,216
293.byte	102,15,56,223,224
294.byte	102,15,56,223,232
295.byte	102,15,56,223,240
296.byte	102,15,56,223,248
297	ret
298.size	_aesni_decrypt6,.-_aesni_decrypt6
299.globl	aesni_ecb_encrypt
300.type	aesni_ecb_encrypt,@function
301.align	16
302aesni_ecb_encrypt:
303.L_aesni_ecb_encrypt_begin:
304	pushl	%ebp
305	pushl	%ebx
306	pushl	%esi
307	pushl	%edi
308	movl	20(%esp),%esi
309	movl	24(%esp),%edi
310	movl	28(%esp),%eax
311	movl	32(%esp),%edx
312	movl	36(%esp),%ebx
313	andl	$-16,%eax
314	jz	.L008ecb_ret
315	movl	240(%edx),%ecx
316	testl	%ebx,%ebx
317	jz	.L009ecb_decrypt
318	movl	%edx,%ebp
319	movl	%ecx,%ebx
320	cmpl	$96,%eax
321	jb	.L010ecb_enc_tail
322	movdqu	(%esi),%xmm2
323	movdqu	16(%esi),%xmm3
324	movdqu	32(%esi),%xmm4
325	movdqu	48(%esi),%xmm5
326	movdqu	64(%esi),%xmm6
327	movdqu	80(%esi),%xmm7
328	leal	96(%esi),%esi
329	subl	$96,%eax
330	jmp	.L011ecb_enc_loop6_enter
331.align	16
332.L012ecb_enc_loop6:
333	movups	%xmm2,(%edi)
334	movdqu	(%esi),%xmm2
335	movups	%xmm3,16(%edi)
336	movdqu	16(%esi),%xmm3
337	movups	%xmm4,32(%edi)
338	movdqu	32(%esi),%xmm4
339	movups	%xmm5,48(%edi)
340	movdqu	48(%esi),%xmm5
341	movups	%xmm6,64(%edi)
342	movdqu	64(%esi),%xmm6
343	movups	%xmm7,80(%edi)
344	leal	96(%edi),%edi
345	movdqu	80(%esi),%xmm7
346	leal	96(%esi),%esi
347.L011ecb_enc_loop6_enter:
348	call	_aesni_encrypt6
349	movl	%ebp,%edx
350	movl	%ebx,%ecx
351	subl	$96,%eax
352	jnc	.L012ecb_enc_loop6
353	movups	%xmm2,(%edi)
354	movups	%xmm3,16(%edi)
355	movups	%xmm4,32(%edi)
356	movups	%xmm5,48(%edi)
357	movups	%xmm6,64(%edi)
358	movups	%xmm7,80(%edi)
359	leal	96(%edi),%edi
360	addl	$96,%eax
361	jz	.L008ecb_ret
362.L010ecb_enc_tail:
363	movups	(%esi),%xmm2
364	cmpl	$32,%eax
365	jb	.L013ecb_enc_one
366	movups	16(%esi),%xmm3
367	je	.L014ecb_enc_two
368	movups	32(%esi),%xmm4
369	cmpl	$64,%eax
370	jb	.L015ecb_enc_three
371	movups	48(%esi),%xmm5
372	je	.L016ecb_enc_four
373	movups	64(%esi),%xmm6
374	xorps	%xmm7,%xmm7
375	call	_aesni_encrypt6
376	movups	%xmm2,(%edi)
377	movups	%xmm3,16(%edi)
378	movups	%xmm4,32(%edi)
379	movups	%xmm5,48(%edi)
380	movups	%xmm6,64(%edi)
381	jmp	.L008ecb_ret
382.align	16
383.L013ecb_enc_one:
384	movups	(%edx),%xmm0
385	movups	16(%edx),%xmm1
386	leal	32(%edx),%edx
387	xorps	%xmm0,%xmm2
388.L017enc1_loop_3:
389.byte	102,15,56,220,209
390	decl	%ecx
391	movups	(%edx),%xmm1
392	leal	16(%edx),%edx
393	jnz	.L017enc1_loop_3
394.byte	102,15,56,221,209
395	movups	%xmm2,(%edi)
396	jmp	.L008ecb_ret
397.align	16
398.L014ecb_enc_two:
399	xorps	%xmm4,%xmm4
400	call	_aesni_encrypt3
401	movups	%xmm2,(%edi)
402	movups	%xmm3,16(%edi)
403	jmp	.L008ecb_ret
404.align	16
405.L015ecb_enc_three:
406	call	_aesni_encrypt3
407	movups	%xmm2,(%edi)
408	movups	%xmm3,16(%edi)
409	movups	%xmm4,32(%edi)
410	jmp	.L008ecb_ret
411.align	16
412.L016ecb_enc_four:
413	call	_aesni_encrypt4
414	movups	%xmm2,(%edi)
415	movups	%xmm3,16(%edi)
416	movups	%xmm4,32(%edi)
417	movups	%xmm5,48(%edi)
418	jmp	.L008ecb_ret
419.align	16
420.L009ecb_decrypt:
421	movl	%edx,%ebp
422	movl	%ecx,%ebx
423	cmpl	$96,%eax
424	jb	.L018ecb_dec_tail
425	movdqu	(%esi),%xmm2
426	movdqu	16(%esi),%xmm3
427	movdqu	32(%esi),%xmm4
428	movdqu	48(%esi),%xmm5
429	movdqu	64(%esi),%xmm6
430	movdqu	80(%esi),%xmm7
431	leal	96(%esi),%esi
432	subl	$96,%eax
433	jmp	.L019ecb_dec_loop6_enter
434.align	16
435.L020ecb_dec_loop6:
436	movups	%xmm2,(%edi)
437	movdqu	(%esi),%xmm2
438	movups	%xmm3,16(%edi)
439	movdqu	16(%esi),%xmm3
440	movups	%xmm4,32(%edi)
441	movdqu	32(%esi),%xmm4
442	movups	%xmm5,48(%edi)
443	movdqu	48(%esi),%xmm5
444	movups	%xmm6,64(%edi)
445	movdqu	64(%esi),%xmm6
446	movups	%xmm7,80(%edi)
447	leal	96(%edi),%edi
448	movdqu	80(%esi),%xmm7
449	leal	96(%esi),%esi
450.L019ecb_dec_loop6_enter:
451	call	_aesni_decrypt6
452	movl	%ebp,%edx
453	movl	%ebx,%ecx
454	subl	$96,%eax
455	jnc	.L020ecb_dec_loop6
456	movups	%xmm2,(%edi)
457	movups	%xmm3,16(%edi)
458	movups	%xmm4,32(%edi)
459	movups	%xmm5,48(%edi)
460	movups	%xmm6,64(%edi)
461	movups	%xmm7,80(%edi)
462	leal	96(%edi),%edi
463	addl	$96,%eax
464	jz	.L008ecb_ret
465.L018ecb_dec_tail:
466	movups	(%esi),%xmm2
467	cmpl	$32,%eax
468	jb	.L021ecb_dec_one
469	movups	16(%esi),%xmm3
470	je	.L022ecb_dec_two
471	movups	32(%esi),%xmm4
472	cmpl	$64,%eax
473	jb	.L023ecb_dec_three
474	movups	48(%esi),%xmm5
475	je	.L024ecb_dec_four
476	movups	64(%esi),%xmm6
477	xorps	%xmm7,%xmm7
478	call	_aesni_decrypt6
479	movups	%xmm2,(%edi)
480	movups	%xmm3,16(%edi)
481	movups	%xmm4,32(%edi)
482	movups	%xmm5,48(%edi)
483	movups	%xmm6,64(%edi)
484	jmp	.L008ecb_ret
485.align	16
486.L021ecb_dec_one:
487	movups	(%edx),%xmm0
488	movups	16(%edx),%xmm1
489	leal	32(%edx),%edx
490	xorps	%xmm0,%xmm2
491.L025dec1_loop_4:
492.byte	102,15,56,222,209
493	decl	%ecx
494	movups	(%edx),%xmm1
495	leal	16(%edx),%edx
496	jnz	.L025dec1_loop_4
497.byte	102,15,56,223,209
498	movups	%xmm2,(%edi)
499	jmp	.L008ecb_ret
500.align	16
501.L022ecb_dec_two:
502	xorps	%xmm4,%xmm4
503	call	_aesni_decrypt3
504	movups	%xmm2,(%edi)
505	movups	%xmm3,16(%edi)
506	jmp	.L008ecb_ret
507.align	16
508.L023ecb_dec_three:
509	call	_aesni_decrypt3
510	movups	%xmm2,(%edi)
511	movups	%xmm3,16(%edi)
512	movups	%xmm4,32(%edi)
513	jmp	.L008ecb_ret
514.align	16
515.L024ecb_dec_four:
516	call	_aesni_decrypt4
517	movups	%xmm2,(%edi)
518	movups	%xmm3,16(%edi)
519	movups	%xmm4,32(%edi)
520	movups	%xmm5,48(%edi)
521.L008ecb_ret:
522	popl	%edi
523	popl	%esi
524	popl	%ebx
525	popl	%ebp
526	ret
527.size	aesni_ecb_encrypt,.-.L_aesni_ecb_encrypt_begin
528.globl	aesni_ccm64_encrypt_blocks
529.type	aesni_ccm64_encrypt_blocks,@function
530.align	16
531aesni_ccm64_encrypt_blocks:
532.L_aesni_ccm64_encrypt_blocks_begin:
533	pushl	%ebp
534	pushl	%ebx
535	pushl	%esi
536	pushl	%edi
537	movl	20(%esp),%esi
538	movl	24(%esp),%edi
539	movl	28(%esp),%eax
540	movl	32(%esp),%edx
541	movl	36(%esp),%ebx
542	movl	40(%esp),%ecx
543	movl	%esp,%ebp
544	subl	$60,%esp
545	andl	$-16,%esp
546	movl	%ebp,48(%esp)
547	movdqu	(%ebx),%xmm7
548	movdqu	(%ecx),%xmm3
549	movl	240(%edx),%ecx
550	movl	$202182159,(%esp)
551	movl	$134810123,4(%esp)
552	movl	$67438087,8(%esp)
553	movl	$66051,12(%esp)
554	movl	$1,%ebx
555	xorl	%ebp,%ebp
556	movl	%ebx,16(%esp)
557	movl	%ebp,20(%esp)
558	movl	%ebp,24(%esp)
559	movl	%ebp,28(%esp)
560	shrl	$1,%ecx
561	leal	(%edx),%ebp
562	movdqa	(%esp),%xmm5
563	movdqa	%xmm7,%xmm2
564	movl	%ecx,%ebx
565.byte	102,15,56,0,253
566.L026ccm64_enc_outer:
567	movups	(%ebp),%xmm0
568	movl	%ebx,%ecx
569	movups	(%esi),%xmm6
570	xorps	%xmm0,%xmm2
571	movups	16(%ebp),%xmm1
572	xorps	%xmm6,%xmm0
573	leal	32(%ebp),%edx
574	xorps	%xmm0,%xmm3
575	movups	(%edx),%xmm0
576.L027ccm64_enc2_loop:
577.byte	102,15,56,220,209
578	decl	%ecx
579.byte	102,15,56,220,217
580	movups	16(%edx),%xmm1
581.byte	102,15,56,220,208
582	leal	32(%edx),%edx
583.byte	102,15,56,220,216
584	movups	(%edx),%xmm0
585	jnz	.L027ccm64_enc2_loop
586.byte	102,15,56,220,209
587.byte	102,15,56,220,217
588	paddq	16(%esp),%xmm7
589.byte	102,15,56,221,208
590.byte	102,15,56,221,216
591	decl	%eax
592	leal	16(%esi),%esi
593	xorps	%xmm2,%xmm6
594	movdqa	%xmm7,%xmm2
595	movups	%xmm6,(%edi)
596	leal	16(%edi),%edi
597.byte	102,15,56,0,213
598	jnz	.L026ccm64_enc_outer
599	movl	48(%esp),%esp
600	movl	40(%esp),%edi
601	movups	%xmm3,(%edi)
602	popl	%edi
603	popl	%esi
604	popl	%ebx
605	popl	%ebp
606	ret
607.size	aesni_ccm64_encrypt_blocks,.-.L_aesni_ccm64_encrypt_blocks_begin
608.globl	aesni_ccm64_decrypt_blocks
609.type	aesni_ccm64_decrypt_blocks,@function
610.align	16
611aesni_ccm64_decrypt_blocks:
612.L_aesni_ccm64_decrypt_blocks_begin:
613	pushl	%ebp
614	pushl	%ebx
615	pushl	%esi
616	pushl	%edi
617	movl	20(%esp),%esi
618	movl	24(%esp),%edi
619	movl	28(%esp),%eax
620	movl	32(%esp),%edx
621	movl	36(%esp),%ebx
622	movl	40(%esp),%ecx
623	movl	%esp,%ebp
624	subl	$60,%esp
625	andl	$-16,%esp
626	movl	%ebp,48(%esp)
627	movdqu	(%ebx),%xmm7
628	movdqu	(%ecx),%xmm3
629	movl	240(%edx),%ecx
630	movl	$202182159,(%esp)
631	movl	$134810123,4(%esp)
632	movl	$67438087,8(%esp)
633	movl	$66051,12(%esp)
634	movl	$1,%ebx
635	xorl	%ebp,%ebp
636	movl	%ebx,16(%esp)
637	movl	%ebp,20(%esp)
638	movl	%ebp,24(%esp)
639	movl	%ebp,28(%esp)
640	movdqa	(%esp),%xmm5
641	movdqa	%xmm7,%xmm2
642	movl	%edx,%ebp
643	movl	%ecx,%ebx
644.byte	102,15,56,0,253
645	movups	(%edx),%xmm0
646	movups	16(%edx),%xmm1
647	leal	32(%edx),%edx
648	xorps	%xmm0,%xmm2
649.L028enc1_loop_5:
650.byte	102,15,56,220,209
651	decl	%ecx
652	movups	(%edx),%xmm1
653	leal	16(%edx),%edx
654	jnz	.L028enc1_loop_5
655.byte	102,15,56,221,209
656	movups	(%esi),%xmm6
657	paddq	16(%esp),%xmm7
658	leal	16(%esi),%esi
659	jmp	.L029ccm64_dec_outer
660.align	16
661.L029ccm64_dec_outer:
662	xorps	%xmm2,%xmm6
663	movdqa	%xmm7,%xmm2
664	movl	%ebx,%ecx
665	movups	%xmm6,(%edi)
666	leal	16(%edi),%edi
667.byte	102,15,56,0,213
668	subl	$1,%eax
669	jz	.L030ccm64_dec_break
670	movups	(%ebp),%xmm0
671	shrl	$1,%ecx
672	movups	16(%ebp),%xmm1
673	xorps	%xmm0,%xmm6
674	leal	32(%ebp),%edx
675	xorps	%xmm0,%xmm2
676	xorps	%xmm6,%xmm3
677	movups	(%edx),%xmm0
678.L031ccm64_dec2_loop:
679.byte	102,15,56,220,209
680	decl	%ecx
681.byte	102,15,56,220,217
682	movups	16(%edx),%xmm1
683.byte	102,15,56,220,208
684	leal	32(%edx),%edx
685.byte	102,15,56,220,216
686	movups	(%edx),%xmm0
687	jnz	.L031ccm64_dec2_loop
688	movups	(%esi),%xmm6
689	paddq	16(%esp),%xmm7
690.byte	102,15,56,220,209
691.byte	102,15,56,220,217
692	leal	16(%esi),%esi
693.byte	102,15,56,221,208
694.byte	102,15,56,221,216
695	jmp	.L029ccm64_dec_outer
696.align	16
697.L030ccm64_dec_break:
698	movl	%ebp,%edx
699	movups	(%edx),%xmm0
700	movups	16(%edx),%xmm1
701	xorps	%xmm0,%xmm6
702	leal	32(%edx),%edx
703	xorps	%xmm6,%xmm3
704.L032enc1_loop_6:
705.byte	102,15,56,220,217
706	decl	%ecx
707	movups	(%edx),%xmm1
708	leal	16(%edx),%edx
709	jnz	.L032enc1_loop_6
710.byte	102,15,56,221,217
711	movl	48(%esp),%esp
712	movl	40(%esp),%edi
713	movups	%xmm3,(%edi)
714	popl	%edi
715	popl	%esi
716	popl	%ebx
717	popl	%ebp
718	ret
719.size	aesni_ccm64_decrypt_blocks,.-.L_aesni_ccm64_decrypt_blocks_begin
720.globl	aesni_ctr32_encrypt_blocks
721.type	aesni_ctr32_encrypt_blocks,@function
722.align	16
723aesni_ctr32_encrypt_blocks:
724.L_aesni_ctr32_encrypt_blocks_begin:
725	pushl	%ebp
726	pushl	%ebx
727	pushl	%esi
728	pushl	%edi
729	movl	20(%esp),%esi
730	movl	24(%esp),%edi
731	movl	28(%esp),%eax
732	movl	32(%esp),%edx
733	movl	36(%esp),%ebx
734	movl	%esp,%ebp
735	subl	$88,%esp
736	andl	$-16,%esp
737	movl	%ebp,80(%esp)
738	cmpl	$1,%eax
739	je	.L033ctr32_one_shortcut
740	movdqu	(%ebx),%xmm7
741	movl	$202182159,(%esp)
742	movl	$134810123,4(%esp)
743	movl	$67438087,8(%esp)
744	movl	$66051,12(%esp)
745	movl	$6,%ecx
746	xorl	%ebp,%ebp
747	movl	%ecx,16(%esp)
748	movl	%ecx,20(%esp)
749	movl	%ecx,24(%esp)
750	movl	%ebp,28(%esp)
751.byte	102,15,58,22,251,3
752.byte	102,15,58,34,253,3
753	movl	240(%edx),%ecx
754	bswap	%ebx
755	pxor	%xmm1,%xmm1
756	pxor	%xmm0,%xmm0
757	movdqa	(%esp),%xmm2
758.byte	102,15,58,34,203,0
759	leal	3(%ebx),%ebp
760.byte	102,15,58,34,197,0
761	incl	%ebx
762.byte	102,15,58,34,203,1
763	incl	%ebp
764.byte	102,15,58,34,197,1
765	incl	%ebx
766.byte	102,15,58,34,203,2
767	incl	%ebp
768.byte	102,15,58,34,197,2
769	movdqa	%xmm1,48(%esp)
770.byte	102,15,56,0,202
771	movdqa	%xmm0,64(%esp)
772.byte	102,15,56,0,194
773	pshufd	$192,%xmm1,%xmm2
774	pshufd	$128,%xmm1,%xmm3
775	cmpl	$6,%eax
776	jb	.L034ctr32_tail
777	movdqa	%xmm7,32(%esp)
778	shrl	$1,%ecx
779	movl	%edx,%ebp
780	movl	%ecx,%ebx
781	subl	$6,%eax
782	jmp	.L035ctr32_loop6
783.align	16
784.L035ctr32_loop6:
785	pshufd	$64,%xmm1,%xmm4
786	movdqa	32(%esp),%xmm1
787	pshufd	$192,%xmm0,%xmm5
788	por	%xmm1,%xmm2
789	pshufd	$128,%xmm0,%xmm6
790	por	%xmm1,%xmm3
791	pshufd	$64,%xmm0,%xmm7
792	por	%xmm1,%xmm4
793	por	%xmm1,%xmm5
794	por	%xmm1,%xmm6
795	por	%xmm1,%xmm7
796	movups	(%ebp),%xmm0
797	movups	16(%ebp),%xmm1
798	leal	32(%ebp),%edx
799	decl	%ecx
800	pxor	%xmm0,%xmm2
801	pxor	%xmm0,%xmm3
802.byte	102,15,56,220,209
803	pxor	%xmm0,%xmm4
804.byte	102,15,56,220,217
805	pxor	%xmm0,%xmm5
806.byte	102,15,56,220,225
807	pxor	%xmm0,%xmm6
808.byte	102,15,56,220,233
809	pxor	%xmm0,%xmm7
810.byte	102,15,56,220,241
811	movups	(%edx),%xmm0
812.byte	102,15,56,220,249
813	call	.L_aesni_encrypt6_enter
814	movups	(%esi),%xmm1
815	movups	16(%esi),%xmm0
816	xorps	%xmm1,%xmm2
817	movups	32(%esi),%xmm1
818	xorps	%xmm0,%xmm3
819	movups	%xmm2,(%edi)
820	movdqa	16(%esp),%xmm0
821	xorps	%xmm1,%xmm4
822	movdqa	48(%esp),%xmm1
823	movups	%xmm3,16(%edi)
824	movups	%xmm4,32(%edi)
825	paddd	%xmm0,%xmm1
826	paddd	64(%esp),%xmm0
827	movdqa	(%esp),%xmm2
828	movups	48(%esi),%xmm3
829	movups	64(%esi),%xmm4
830	xorps	%xmm3,%xmm5
831	movups	80(%esi),%xmm3
832	leal	96(%esi),%esi
833	movdqa	%xmm1,48(%esp)
834.byte	102,15,56,0,202
835	xorps	%xmm4,%xmm6
836	movups	%xmm5,48(%edi)
837	xorps	%xmm3,%xmm7
838	movdqa	%xmm0,64(%esp)
839.byte	102,15,56,0,194
840	movups	%xmm6,64(%edi)
841	pshufd	$192,%xmm1,%xmm2
842	movups	%xmm7,80(%edi)
843	leal	96(%edi),%edi
844	movl	%ebx,%ecx
845	pshufd	$128,%xmm1,%xmm3
846	subl	$6,%eax
847	jnc	.L035ctr32_loop6
848	addl	$6,%eax
849	jz	.L036ctr32_ret
850	movl	%ebp,%edx
851	leal	1(,%ecx,2),%ecx
852	movdqa	32(%esp),%xmm7
853.L034ctr32_tail:
854	por	%xmm7,%xmm2
855	cmpl	$2,%eax
856	jb	.L037ctr32_one
857	pshufd	$64,%xmm1,%xmm4
858	por	%xmm7,%xmm3
859	je	.L038ctr32_two
860	pshufd	$192,%xmm0,%xmm5
861	por	%xmm7,%xmm4
862	cmpl	$4,%eax
863	jb	.L039ctr32_three
864	pshufd	$128,%xmm0,%xmm6
865	por	%xmm7,%xmm5
866	je	.L040ctr32_four
867	por	%xmm7,%xmm6
868	call	_aesni_encrypt6
869	movups	(%esi),%xmm1
870	movups	16(%esi),%xmm0
871	xorps	%xmm1,%xmm2
872	movups	32(%esi),%xmm1
873	xorps	%xmm0,%xmm3
874	movups	48(%esi),%xmm0
875	xorps	%xmm1,%xmm4
876	movups	64(%esi),%xmm1
877	xorps	%xmm0,%xmm5
878	movups	%xmm2,(%edi)
879	xorps	%xmm1,%xmm6
880	movups	%xmm3,16(%edi)
881	movups	%xmm4,32(%edi)
882	movups	%xmm5,48(%edi)
883	movups	%xmm6,64(%edi)
884	jmp	.L036ctr32_ret
885.align	16
886.L033ctr32_one_shortcut:
887	movups	(%ebx),%xmm2
888	movl	240(%edx),%ecx
889.L037ctr32_one:
890	movups	(%edx),%xmm0
891	movups	16(%edx),%xmm1
892	leal	32(%edx),%edx
893	xorps	%xmm0,%xmm2
894.L041enc1_loop_7:
895.byte	102,15,56,220,209
896	decl	%ecx
897	movups	(%edx),%xmm1
898	leal	16(%edx),%edx
899	jnz	.L041enc1_loop_7
900.byte	102,15,56,221,209
901	movups	(%esi),%xmm6
902	xorps	%xmm2,%xmm6
903	movups	%xmm6,(%edi)
904	jmp	.L036ctr32_ret
905.align	16
906.L038ctr32_two:
907	call	_aesni_encrypt3
908	movups	(%esi),%xmm5
909	movups	16(%esi),%xmm6
910	xorps	%xmm5,%xmm2
911	xorps	%xmm6,%xmm3
912	movups	%xmm2,(%edi)
913	movups	%xmm3,16(%edi)
914	jmp	.L036ctr32_ret
915.align	16
916.L039ctr32_three:
917	call	_aesni_encrypt3
918	movups	(%esi),%xmm5
919	movups	16(%esi),%xmm6
920	xorps	%xmm5,%xmm2
921	movups	32(%esi),%xmm7
922	xorps	%xmm6,%xmm3
923	movups	%xmm2,(%edi)
924	xorps	%xmm7,%xmm4
925	movups	%xmm3,16(%edi)
926	movups	%xmm4,32(%edi)
927	jmp	.L036ctr32_ret
928.align	16
929.L040ctr32_four:
930	call	_aesni_encrypt4
931	movups	(%esi),%xmm6
932	movups	16(%esi),%xmm7
933	movups	32(%esi),%xmm1
934	xorps	%xmm6,%xmm2
935	movups	48(%esi),%xmm0
936	xorps	%xmm7,%xmm3
937	movups	%xmm2,(%edi)
938	xorps	%xmm1,%xmm4
939	movups	%xmm3,16(%edi)
940	xorps	%xmm0,%xmm5
941	movups	%xmm4,32(%edi)
942	movups	%xmm5,48(%edi)
943.L036ctr32_ret:
944	movl	80(%esp),%esp
945	popl	%edi
946	popl	%esi
947	popl	%ebx
948	popl	%ebp
949	ret
950.size	aesni_ctr32_encrypt_blocks,.-.L_aesni_ctr32_encrypt_blocks_begin
951.globl	aesni_xts_encrypt
952.type	aesni_xts_encrypt,@function
953.align	16
954aesni_xts_encrypt:
955.L_aesni_xts_encrypt_begin:
956	pushl	%ebp
957	pushl	%ebx
958	pushl	%esi
959	pushl	%edi
960	movl	36(%esp),%edx
961	movl	40(%esp),%esi
962	movl	240(%edx),%ecx
963	movups	(%esi),%xmm2
964	movups	(%edx),%xmm0
965	movups	16(%edx),%xmm1
966	leal	32(%edx),%edx
967	xorps	%xmm0,%xmm2
968.L042enc1_loop_8:
969.byte	102,15,56,220,209
970	decl	%ecx
971	movups	(%edx),%xmm1
972	leal	16(%edx),%edx
973	jnz	.L042enc1_loop_8
974.byte	102,15,56,221,209
975	movl	20(%esp),%esi
976	movl	24(%esp),%edi
977	movl	28(%esp),%eax
978	movl	32(%esp),%edx
979	movl	%esp,%ebp
980	subl	$120,%esp
981	movl	240(%edx),%ecx
982	andl	$-16,%esp
983	movl	$135,96(%esp)
984	movl	$0,100(%esp)
985	movl	$1,104(%esp)
986	movl	$0,108(%esp)
987	movl	%eax,112(%esp)
988	movl	%ebp,116(%esp)
989	movdqa	%xmm2,%xmm1
990	pxor	%xmm0,%xmm0
991	movdqa	96(%esp),%xmm3
992	pcmpgtd	%xmm1,%xmm0
993	andl	$-16,%eax
994	movl	%edx,%ebp
995	movl	%ecx,%ebx
996	subl	$96,%eax
997	jc	.L043xts_enc_short
998	shrl	$1,%ecx
999	movl	%ecx,%ebx
1000	jmp	.L044xts_enc_loop6
1001.align	16
1002.L044xts_enc_loop6:
1003	pshufd	$19,%xmm0,%xmm2
1004	pxor	%xmm0,%xmm0
1005	movdqa	%xmm1,(%esp)
1006	paddq	%xmm1,%xmm1
1007	pand	%xmm3,%xmm2
1008	pcmpgtd	%xmm1,%xmm0
1009	pxor	%xmm2,%xmm1
1010	pshufd	$19,%xmm0,%xmm2
1011	pxor	%xmm0,%xmm0
1012	movdqa	%xmm1,16(%esp)
1013	paddq	%xmm1,%xmm1
1014	pand	%xmm3,%xmm2
1015	pcmpgtd	%xmm1,%xmm0
1016	pxor	%xmm2,%xmm1
1017	pshufd	$19,%xmm0,%xmm2
1018	pxor	%xmm0,%xmm0
1019	movdqa	%xmm1,32(%esp)
1020	paddq	%xmm1,%xmm1
1021	pand	%xmm3,%xmm2
1022	pcmpgtd	%xmm1,%xmm0
1023	pxor	%xmm2,%xmm1
1024	pshufd	$19,%xmm0,%xmm2
1025	pxor	%xmm0,%xmm0
1026	movdqa	%xmm1,48(%esp)
1027	paddq	%xmm1,%xmm1
1028	pand	%xmm3,%xmm2
1029	pcmpgtd	%xmm1,%xmm0
1030	pxor	%xmm2,%xmm1
1031	pshufd	$19,%xmm0,%xmm7
1032	movdqa	%xmm1,64(%esp)
1033	paddq	%xmm1,%xmm1
1034	movups	(%ebp),%xmm0
1035	pand	%xmm3,%xmm7
1036	movups	(%esi),%xmm2
1037	pxor	%xmm1,%xmm7
1038	movdqu	16(%esi),%xmm3
1039	xorps	%xmm0,%xmm2
1040	movdqu	32(%esi),%xmm4
1041	pxor	%xmm0,%xmm3
1042	movdqu	48(%esi),%xmm5
1043	pxor	%xmm0,%xmm4
1044	movdqu	64(%esi),%xmm6
1045	pxor	%xmm0,%xmm5
1046	movdqu	80(%esi),%xmm1
1047	pxor	%xmm0,%xmm6
1048	leal	96(%esi),%esi
1049	pxor	(%esp),%xmm2
1050	movdqa	%xmm7,80(%esp)
1051	pxor	%xmm1,%xmm7
1052	movups	16(%ebp),%xmm1
1053	leal	32(%ebp),%edx
1054	pxor	16(%esp),%xmm3
1055.byte	102,15,56,220,209
1056	pxor	32(%esp),%xmm4
1057.byte	102,15,56,220,217
1058	pxor	48(%esp),%xmm5
1059	decl	%ecx
1060.byte	102,15,56,220,225
1061	pxor	64(%esp),%xmm6
1062.byte	102,15,56,220,233
1063	pxor	%xmm0,%xmm7
1064.byte	102,15,56,220,241
1065	movups	(%edx),%xmm0
1066.byte	102,15,56,220,249
1067	call	.L_aesni_encrypt6_enter
1068	movdqa	80(%esp),%xmm1
1069	pxor	%xmm0,%xmm0
1070	xorps	(%esp),%xmm2
1071	pcmpgtd	%xmm1,%xmm0
1072	xorps	16(%esp),%xmm3
1073	movups	%xmm2,(%edi)
1074	xorps	32(%esp),%xmm4
1075	movups	%xmm3,16(%edi)
1076	xorps	48(%esp),%xmm5
1077	movups	%xmm4,32(%edi)
1078	xorps	64(%esp),%xmm6
1079	movups	%xmm5,48(%edi)
1080	xorps	%xmm1,%xmm7
1081	movups	%xmm6,64(%edi)
1082	pshufd	$19,%xmm0,%xmm2
1083	movups	%xmm7,80(%edi)
1084	leal	96(%edi),%edi
1085	movdqa	96(%esp),%xmm3
1086	pxor	%xmm0,%xmm0
1087	paddq	%xmm1,%xmm1
1088	pand	%xmm3,%xmm2
1089	pcmpgtd	%xmm1,%xmm0
1090	movl	%ebx,%ecx
1091	pxor	%xmm2,%xmm1
1092	subl	$96,%eax
1093	jnc	.L044xts_enc_loop6
1094	leal	1(,%ecx,2),%ecx
1095	movl	%ebp,%edx
1096	movl	%ecx,%ebx
1097.L043xts_enc_short:
1098	addl	$96,%eax
1099	jz	.L045xts_enc_done6x
1100	movdqa	%xmm1,%xmm5
1101	cmpl	$32,%eax
1102	jb	.L046xts_enc_one
1103	pshufd	$19,%xmm0,%xmm2
1104	pxor	%xmm0,%xmm0
1105	paddq	%xmm1,%xmm1
1106	pand	%xmm3,%xmm2
1107	pcmpgtd	%xmm1,%xmm0
1108	pxor	%xmm2,%xmm1
1109	je	.L047xts_enc_two
1110	pshufd	$19,%xmm0,%xmm2
1111	pxor	%xmm0,%xmm0
1112	movdqa	%xmm1,%xmm6
1113	paddq	%xmm1,%xmm1
1114	pand	%xmm3,%xmm2
1115	pcmpgtd	%xmm1,%xmm0
1116	pxor	%xmm2,%xmm1
1117	cmpl	$64,%eax
1118	jb	.L048xts_enc_three
1119	pshufd	$19,%xmm0,%xmm2
1120	pxor	%xmm0,%xmm0
1121	movdqa	%xmm1,%xmm7
1122	paddq	%xmm1,%xmm1
1123	pand	%xmm3,%xmm2
1124	pcmpgtd	%xmm1,%xmm0
1125	pxor	%xmm2,%xmm1
1126	movdqa	%xmm5,(%esp)
1127	movdqa	%xmm6,16(%esp)
1128	je	.L049xts_enc_four
1129	movdqa	%xmm7,32(%esp)
1130	pshufd	$19,%xmm0,%xmm7
1131	movdqa	%xmm1,48(%esp)
1132	paddq	%xmm1,%xmm1
1133	pand	%xmm3,%xmm7
1134	pxor	%xmm1,%xmm7
1135	movdqu	(%esi),%xmm2
1136	movdqu	16(%esi),%xmm3
1137	movdqu	32(%esi),%xmm4
1138	pxor	(%esp),%xmm2
1139	movdqu	48(%esi),%xmm5
1140	pxor	16(%esp),%xmm3
1141	movdqu	64(%esi),%xmm6
1142	pxor	32(%esp),%xmm4
1143	leal	80(%esi),%esi
1144	pxor	48(%esp),%xmm5
1145	movdqa	%xmm7,64(%esp)
1146	pxor	%xmm7,%xmm6
1147	call	_aesni_encrypt6
1148	movaps	64(%esp),%xmm1
1149	xorps	(%esp),%xmm2
1150	xorps	16(%esp),%xmm3
1151	xorps	32(%esp),%xmm4
1152	movups	%xmm2,(%edi)
1153	xorps	48(%esp),%xmm5
1154	movups	%xmm3,16(%edi)
1155	xorps	%xmm1,%xmm6
1156	movups	%xmm4,32(%edi)
1157	movups	%xmm5,48(%edi)
1158	movups	%xmm6,64(%edi)
1159	leal	80(%edi),%edi
1160	jmp	.L050xts_enc_done
1161.align	16
1162.L046xts_enc_one:
1163	movups	(%esi),%xmm2
1164	leal	16(%esi),%esi
1165	xorps	%xmm5,%xmm2
1166	movups	(%edx),%xmm0
1167	movups	16(%edx),%xmm1
1168	leal	32(%edx),%edx
1169	xorps	%xmm0,%xmm2
1170.L051enc1_loop_9:
1171.byte	102,15,56,220,209
1172	decl	%ecx
1173	movups	(%edx),%xmm1
1174	leal	16(%edx),%edx
1175	jnz	.L051enc1_loop_9
1176.byte	102,15,56,221,209
1177	xorps	%xmm5,%xmm2
1178	movups	%xmm2,(%edi)
1179	leal	16(%edi),%edi
1180	movdqa	%xmm5,%xmm1
1181	jmp	.L050xts_enc_done
1182.align	16
1183.L047xts_enc_two:
1184	movaps	%xmm1,%xmm6
1185	movups	(%esi),%xmm2
1186	movups	16(%esi),%xmm3
1187	leal	32(%esi),%esi
1188	xorps	%xmm5,%xmm2
1189	xorps	%xmm6,%xmm3
1190	xorps	%xmm4,%xmm4
1191	call	_aesni_encrypt3
1192	xorps	%xmm5,%xmm2
1193	xorps	%xmm6,%xmm3
1194	movups	%xmm2,(%edi)
1195	movups	%xmm3,16(%edi)
1196	leal	32(%edi),%edi
1197	movdqa	%xmm6,%xmm1
1198	jmp	.L050xts_enc_done
1199.align	16
1200.L048xts_enc_three:
1201	movaps	%xmm1,%xmm7
1202	movups	(%esi),%xmm2
1203	movups	16(%esi),%xmm3
1204	movups	32(%esi),%xmm4
1205	leal	48(%esi),%esi
1206	xorps	%xmm5,%xmm2
1207	xorps	%xmm6,%xmm3
1208	xorps	%xmm7,%xmm4
1209	call	_aesni_encrypt3
1210	xorps	%xmm5,%xmm2
1211	xorps	%xmm6,%xmm3
1212	xorps	%xmm7,%xmm4
1213	movups	%xmm2,(%edi)
1214	movups	%xmm3,16(%edi)
1215	movups	%xmm4,32(%edi)
1216	leal	48(%edi),%edi
1217	movdqa	%xmm7,%xmm1
1218	jmp	.L050xts_enc_done
1219.align	16
1220.L049xts_enc_four:
1221	movaps	%xmm1,%xmm6
1222	movups	(%esi),%xmm2
1223	movups	16(%esi),%xmm3
1224	movups	32(%esi),%xmm4
1225	xorps	(%esp),%xmm2
1226	movups	48(%esi),%xmm5
1227	leal	64(%esi),%esi
1228	xorps	16(%esp),%xmm3
1229	xorps	%xmm7,%xmm4
1230	xorps	%xmm6,%xmm5
1231	call	_aesni_encrypt4
1232	xorps	(%esp),%xmm2
1233	xorps	16(%esp),%xmm3
1234	xorps	%xmm7,%xmm4
1235	movups	%xmm2,(%edi)
1236	xorps	%xmm6,%xmm5
1237	movups	%xmm3,16(%edi)
1238	movups	%xmm4,32(%edi)
1239	movups	%xmm5,48(%edi)
1240	leal	64(%edi),%edi
1241	movdqa	%xmm6,%xmm1
1242	jmp	.L050xts_enc_done
1243.align	16
1244.L045xts_enc_done6x:
1245	movl	112(%esp),%eax
1246	andl	$15,%eax
1247	jz	.L052xts_enc_ret
1248	movdqa	%xmm1,%xmm5
1249	movl	%eax,112(%esp)
1250	jmp	.L053xts_enc_steal
1251.align	16
1252.L050xts_enc_done:
1253	movl	112(%esp),%eax
1254	pxor	%xmm0,%xmm0
1255	andl	$15,%eax
1256	jz	.L052xts_enc_ret
1257	pcmpgtd	%xmm1,%xmm0
1258	movl	%eax,112(%esp)
1259	pshufd	$19,%xmm0,%xmm5
1260	paddq	%xmm1,%xmm1
1261	pand	96(%esp),%xmm5
1262	pxor	%xmm1,%xmm5
1263.L053xts_enc_steal:
1264	movzbl	(%esi),%ecx
1265	movzbl	-16(%edi),%edx
1266	leal	1(%esi),%esi
1267	movb	%cl,-16(%edi)
1268	movb	%dl,(%edi)
1269	leal	1(%edi),%edi
1270	subl	$1,%eax
1271	jnz	.L053xts_enc_steal
1272	subl	112(%esp),%edi
1273	movl	%ebp,%edx
1274	movl	%ebx,%ecx
1275	movups	-16(%edi),%xmm2
1276	xorps	%xmm5,%xmm2
1277	movups	(%edx),%xmm0
1278	movups	16(%edx),%xmm1
1279	leal	32(%edx),%edx
1280	xorps	%xmm0,%xmm2
1281.L054enc1_loop_10:
1282.byte	102,15,56,220,209
1283	decl	%ecx
1284	movups	(%edx),%xmm1
1285	leal	16(%edx),%edx
1286	jnz	.L054enc1_loop_10
1287.byte	102,15,56,221,209
1288	xorps	%xmm5,%xmm2
1289	movups	%xmm2,-16(%edi)
1290.L052xts_enc_ret:
1291	movl	116(%esp),%esp
1292	popl	%edi
1293	popl	%esi
1294	popl	%ebx
1295	popl	%ebp
1296	ret
1297.size	aesni_xts_encrypt,.-.L_aesni_xts_encrypt_begin
1298.globl	aesni_xts_decrypt
1299.type	aesni_xts_decrypt,@function
1300.align	16
1301aesni_xts_decrypt:
1302.L_aesni_xts_decrypt_begin:
1303	pushl	%ebp
1304	pushl	%ebx
1305	pushl	%esi
1306	pushl	%edi
1307	movl	36(%esp),%edx
1308	movl	40(%esp),%esi
1309	movl	240(%edx),%ecx
1310	movups	(%esi),%xmm2
1311	movups	(%edx),%xmm0
1312	movups	16(%edx),%xmm1
1313	leal	32(%edx),%edx
1314	xorps	%xmm0,%xmm2
1315.L055enc1_loop_11:
1316.byte	102,15,56,220,209
1317	decl	%ecx
1318	movups	(%edx),%xmm1
1319	leal	16(%edx),%edx
1320	jnz	.L055enc1_loop_11
1321.byte	102,15,56,221,209
1322	movl	20(%esp),%esi
1323	movl	24(%esp),%edi
1324	movl	28(%esp),%eax
1325	movl	32(%esp),%edx
1326	movl	%esp,%ebp
1327	subl	$120,%esp
1328	andl	$-16,%esp
1329	xorl	%ebx,%ebx
1330	testl	$15,%eax
1331	setnz	%bl
1332	shll	$4,%ebx
1333	subl	%ebx,%eax
1334	movl	$135,96(%esp)
1335	movl	$0,100(%esp)
1336	movl	$1,104(%esp)
1337	movl	$0,108(%esp)
1338	movl	%eax,112(%esp)
1339	movl	%ebp,116(%esp)
1340	movl	240(%edx),%ecx
1341	movl	%edx,%ebp
1342	movl	%ecx,%ebx
1343	movdqa	%xmm2,%xmm1
1344	pxor	%xmm0,%xmm0
1345	movdqa	96(%esp),%xmm3
1346	pcmpgtd	%xmm1,%xmm0
1347	andl	$-16,%eax
1348	subl	$96,%eax
1349	jc	.L056xts_dec_short
1350	shrl	$1,%ecx
1351	movl	%ecx,%ebx
1352	jmp	.L057xts_dec_loop6
1353.align	16
1354.L057xts_dec_loop6:
1355	pshufd	$19,%xmm0,%xmm2
1356	pxor	%xmm0,%xmm0
1357	movdqa	%xmm1,(%esp)
1358	paddq	%xmm1,%xmm1
1359	pand	%xmm3,%xmm2
1360	pcmpgtd	%xmm1,%xmm0
1361	pxor	%xmm2,%xmm1
1362	pshufd	$19,%xmm0,%xmm2
1363	pxor	%xmm0,%xmm0
1364	movdqa	%xmm1,16(%esp)
1365	paddq	%xmm1,%xmm1
1366	pand	%xmm3,%xmm2
1367	pcmpgtd	%xmm1,%xmm0
1368	pxor	%xmm2,%xmm1
1369	pshufd	$19,%xmm0,%xmm2
1370	pxor	%xmm0,%xmm0
1371	movdqa	%xmm1,32(%esp)
1372	paddq	%xmm1,%xmm1
1373	pand	%xmm3,%xmm2
1374	pcmpgtd	%xmm1,%xmm0
1375	pxor	%xmm2,%xmm1
1376	pshufd	$19,%xmm0,%xmm2
1377	pxor	%xmm0,%xmm0
1378	movdqa	%xmm1,48(%esp)
1379	paddq	%xmm1,%xmm1
1380	pand	%xmm3,%xmm2
1381	pcmpgtd	%xmm1,%xmm0
1382	pxor	%xmm2,%xmm1
1383	pshufd	$19,%xmm0,%xmm7
1384	movdqa	%xmm1,64(%esp)
1385	paddq	%xmm1,%xmm1
1386	movups	(%ebp),%xmm0
1387	pand	%xmm3,%xmm7
1388	movups	(%esi),%xmm2
1389	pxor	%xmm1,%xmm7
1390	movdqu	16(%esi),%xmm3
1391	xorps	%xmm0,%xmm2
1392	movdqu	32(%esi),%xmm4
1393	pxor	%xmm0,%xmm3
1394	movdqu	48(%esi),%xmm5
1395	pxor	%xmm0,%xmm4
1396	movdqu	64(%esi),%xmm6
1397	pxor	%xmm0,%xmm5
1398	movdqu	80(%esi),%xmm1
1399	pxor	%xmm0,%xmm6
1400	leal	96(%esi),%esi
1401	pxor	(%esp),%xmm2
1402	movdqa	%xmm7,80(%esp)
1403	pxor	%xmm1,%xmm7
1404	movups	16(%ebp),%xmm1
1405	leal	32(%ebp),%edx
1406	pxor	16(%esp),%xmm3
1407.byte	102,15,56,222,209
1408	pxor	32(%esp),%xmm4
1409.byte	102,15,56,222,217
1410	pxor	48(%esp),%xmm5
1411	decl	%ecx
1412.byte	102,15,56,222,225
1413	pxor	64(%esp),%xmm6
1414.byte	102,15,56,222,233
1415	pxor	%xmm0,%xmm7
1416.byte	102,15,56,222,241
1417	movups	(%edx),%xmm0
1418.byte	102,15,56,222,249
1419	call	.L_aesni_decrypt6_enter
1420	movdqa	80(%esp),%xmm1
1421	pxor	%xmm0,%xmm0
1422	xorps	(%esp),%xmm2
1423	pcmpgtd	%xmm1,%xmm0
1424	xorps	16(%esp),%xmm3
1425	movups	%xmm2,(%edi)
1426	xorps	32(%esp),%xmm4
1427	movups	%xmm3,16(%edi)
1428	xorps	48(%esp),%xmm5
1429	movups	%xmm4,32(%edi)
1430	xorps	64(%esp),%xmm6
1431	movups	%xmm5,48(%edi)
1432	xorps	%xmm1,%xmm7
1433	movups	%xmm6,64(%edi)
1434	pshufd	$19,%xmm0,%xmm2
1435	movups	%xmm7,80(%edi)
1436	leal	96(%edi),%edi
1437	movdqa	96(%esp),%xmm3
1438	pxor	%xmm0,%xmm0
1439	paddq	%xmm1,%xmm1
1440	pand	%xmm3,%xmm2
1441	pcmpgtd	%xmm1,%xmm0
1442	movl	%ebx,%ecx
1443	pxor	%xmm2,%xmm1
1444	subl	$96,%eax
1445	jnc	.L057xts_dec_loop6
1446	leal	1(,%ecx,2),%ecx
1447	movl	%ebp,%edx
1448	movl	%ecx,%ebx
1449.L056xts_dec_short:
1450	addl	$96,%eax
1451	jz	.L058xts_dec_done6x
1452	movdqa	%xmm1,%xmm5
1453	cmpl	$32,%eax
1454	jb	.L059xts_dec_one
1455	pshufd	$19,%xmm0,%xmm2
1456	pxor	%xmm0,%xmm0
1457	paddq	%xmm1,%xmm1
1458	pand	%xmm3,%xmm2
1459	pcmpgtd	%xmm1,%xmm0
1460	pxor	%xmm2,%xmm1
1461	je	.L060xts_dec_two
1462	pshufd	$19,%xmm0,%xmm2
1463	pxor	%xmm0,%xmm0
1464	movdqa	%xmm1,%xmm6
1465	paddq	%xmm1,%xmm1
1466	pand	%xmm3,%xmm2
1467	pcmpgtd	%xmm1,%xmm0
1468	pxor	%xmm2,%xmm1
1469	cmpl	$64,%eax
1470	jb	.L061xts_dec_three
1471	pshufd	$19,%xmm0,%xmm2
1472	pxor	%xmm0,%xmm0
1473	movdqa	%xmm1,%xmm7
1474	paddq	%xmm1,%xmm1
1475	pand	%xmm3,%xmm2
1476	pcmpgtd	%xmm1,%xmm0
1477	pxor	%xmm2,%xmm1
1478	movdqa	%xmm5,(%esp)
1479	movdqa	%xmm6,16(%esp)
1480	je	.L062xts_dec_four
1481	movdqa	%xmm7,32(%esp)
1482	pshufd	$19,%xmm0,%xmm7
1483	movdqa	%xmm1,48(%esp)
1484	paddq	%xmm1,%xmm1
1485	pand	%xmm3,%xmm7
1486	pxor	%xmm1,%xmm7
1487	movdqu	(%esi),%xmm2
1488	movdqu	16(%esi),%xmm3
1489	movdqu	32(%esi),%xmm4
1490	pxor	(%esp),%xmm2
1491	movdqu	48(%esi),%xmm5
1492	pxor	16(%esp),%xmm3
1493	movdqu	64(%esi),%xmm6
1494	pxor	32(%esp),%xmm4
1495	leal	80(%esi),%esi
1496	pxor	48(%esp),%xmm5
1497	movdqa	%xmm7,64(%esp)
1498	pxor	%xmm7,%xmm6
1499	call	_aesni_decrypt6
1500	movaps	64(%esp),%xmm1
1501	xorps	(%esp),%xmm2
1502	xorps	16(%esp),%xmm3
1503	xorps	32(%esp),%xmm4
1504	movups	%xmm2,(%edi)
1505	xorps	48(%esp),%xmm5
1506	movups	%xmm3,16(%edi)
1507	xorps	%xmm1,%xmm6
1508	movups	%xmm4,32(%edi)
1509	movups	%xmm5,48(%edi)
1510	movups	%xmm6,64(%edi)
1511	leal	80(%edi),%edi
1512	jmp	.L063xts_dec_done
1513.align	16
1514.L059xts_dec_one:
1515	movups	(%esi),%xmm2
1516	leal	16(%esi),%esi
1517	xorps	%xmm5,%xmm2
1518	movups	(%edx),%xmm0
1519	movups	16(%edx),%xmm1
1520	leal	32(%edx),%edx
1521	xorps	%xmm0,%xmm2
1522.L064dec1_loop_12:
1523.byte	102,15,56,222,209
1524	decl	%ecx
1525	movups	(%edx),%xmm1
1526	leal	16(%edx),%edx
1527	jnz	.L064dec1_loop_12
1528.byte	102,15,56,223,209
1529	xorps	%xmm5,%xmm2
1530	movups	%xmm2,(%edi)
1531	leal	16(%edi),%edi
1532	movdqa	%xmm5,%xmm1
1533	jmp	.L063xts_dec_done
1534.align	16
1535.L060xts_dec_two:
1536	movaps	%xmm1,%xmm6
1537	movups	(%esi),%xmm2
1538	movups	16(%esi),%xmm3
1539	leal	32(%esi),%esi
1540	xorps	%xmm5,%xmm2
1541	xorps	%xmm6,%xmm3
1542	call	_aesni_decrypt3
1543	xorps	%xmm5,%xmm2
1544	xorps	%xmm6,%xmm3
1545	movups	%xmm2,(%edi)
1546	movups	%xmm3,16(%edi)
1547	leal	32(%edi),%edi
1548	movdqa	%xmm6,%xmm1
1549	jmp	.L063xts_dec_done
1550.align	16
1551.L061xts_dec_three:
1552	movaps	%xmm1,%xmm7
1553	movups	(%esi),%xmm2
1554	movups	16(%esi),%xmm3
1555	movups	32(%esi),%xmm4
1556	leal	48(%esi),%esi
1557	xorps	%xmm5,%xmm2
1558	xorps	%xmm6,%xmm3
1559	xorps	%xmm7,%xmm4
1560	call	_aesni_decrypt3
1561	xorps	%xmm5,%xmm2
1562	xorps	%xmm6,%xmm3
1563	xorps	%xmm7,%xmm4
1564	movups	%xmm2,(%edi)
1565	movups	%xmm3,16(%edi)
1566	movups	%xmm4,32(%edi)
1567	leal	48(%edi),%edi
1568	movdqa	%xmm7,%xmm1
1569	jmp	.L063xts_dec_done
1570.align	16
1571.L062xts_dec_four:
1572	movaps	%xmm1,%xmm6
1573	movups	(%esi),%xmm2
1574	movups	16(%esi),%xmm3
1575	movups	32(%esi),%xmm4
1576	xorps	(%esp),%xmm2
1577	movups	48(%esi),%xmm5
1578	leal	64(%esi),%esi
1579	xorps	16(%esp),%xmm3
1580	xorps	%xmm7,%xmm4
1581	xorps	%xmm6,%xmm5
1582	call	_aesni_decrypt4
1583	xorps	(%esp),%xmm2
1584	xorps	16(%esp),%xmm3
1585	xorps	%xmm7,%xmm4
1586	movups	%xmm2,(%edi)
1587	xorps	%xmm6,%xmm5
1588	movups	%xmm3,16(%edi)
1589	movups	%xmm4,32(%edi)
1590	movups	%xmm5,48(%edi)
1591	leal	64(%edi),%edi
1592	movdqa	%xmm6,%xmm1
1593	jmp	.L063xts_dec_done
1594.align	16
1595.L058xts_dec_done6x:
1596	movl	112(%esp),%eax
1597	andl	$15,%eax
1598	jz	.L065xts_dec_ret
1599	movl	%eax,112(%esp)
1600	jmp	.L066xts_dec_only_one_more
1601.align	16
1602.L063xts_dec_done:
1603	movl	112(%esp),%eax
1604	pxor	%xmm0,%xmm0
1605	andl	$15,%eax
1606	jz	.L065xts_dec_ret
1607	pcmpgtd	%xmm1,%xmm0
1608	movl	%eax,112(%esp)
1609	pshufd	$19,%xmm0,%xmm2
1610	pxor	%xmm0,%xmm0
1611	movdqa	96(%esp),%xmm3
1612	paddq	%xmm1,%xmm1
1613	pand	%xmm3,%xmm2
1614	pcmpgtd	%xmm1,%xmm0
1615	pxor	%xmm2,%xmm1
1616.L066xts_dec_only_one_more:
1617	pshufd	$19,%xmm0,%xmm5
1618	movdqa	%xmm1,%xmm6
1619	paddq	%xmm1,%xmm1
1620	pand	%xmm3,%xmm5
1621	pxor	%xmm1,%xmm5
1622	movl	%ebp,%edx
1623	movl	%ebx,%ecx
1624	movups	(%esi),%xmm2
1625	xorps	%xmm5,%xmm2
1626	movups	(%edx),%xmm0
1627	movups	16(%edx),%xmm1
1628	leal	32(%edx),%edx
1629	xorps	%xmm0,%xmm2
1630.L067dec1_loop_13:
1631.byte	102,15,56,222,209
1632	decl	%ecx
1633	movups	(%edx),%xmm1
1634	leal	16(%edx),%edx
1635	jnz	.L067dec1_loop_13
1636.byte	102,15,56,223,209
1637	xorps	%xmm5,%xmm2
1638	movups	%xmm2,(%edi)
1639.L068xts_dec_steal:
1640	movzbl	16(%esi),%ecx
1641	movzbl	(%edi),%edx
1642	leal	1(%esi),%esi
1643	movb	%cl,(%edi)
1644	movb	%dl,16(%edi)
1645	leal	1(%edi),%edi
1646	subl	$1,%eax
1647	jnz	.L068xts_dec_steal
1648	subl	112(%esp),%edi
1649	movl	%ebp,%edx
1650	movl	%ebx,%ecx
1651	movups	(%edi),%xmm2
1652	xorps	%xmm6,%xmm2
1653	movups	(%edx),%xmm0
1654	movups	16(%edx),%xmm1
1655	leal	32(%edx),%edx
1656	xorps	%xmm0,%xmm2
1657.L069dec1_loop_14:
1658.byte	102,15,56,222,209
1659	decl	%ecx
1660	movups	(%edx),%xmm1
1661	leal	16(%edx),%edx
1662	jnz	.L069dec1_loop_14
1663.byte	102,15,56,223,209
1664	xorps	%xmm6,%xmm2
1665	movups	%xmm2,(%edi)
1666.L065xts_dec_ret:
1667	movl	116(%esp),%esp
1668	popl	%edi
1669	popl	%esi
1670	popl	%ebx
1671	popl	%ebp
1672	ret
1673.size	aesni_xts_decrypt,.-.L_aesni_xts_decrypt_begin
1674.globl	aesni_cbc_encrypt
1675.type	aesni_cbc_encrypt,@function
1676.align	16
1677aesni_cbc_encrypt:
1678.L_aesni_cbc_encrypt_begin:
1679	pushl	%ebp
1680	pushl	%ebx
1681	pushl	%esi
1682	pushl	%edi
1683	movl	20(%esp),%esi
1684	movl	%esp,%ebx
1685	movl	24(%esp),%edi
1686	subl	$24,%ebx
1687	movl	28(%esp),%eax
1688	andl	$-16,%ebx
1689	movl	32(%esp),%edx
1690	movl	36(%esp),%ebp
1691	testl	%eax,%eax
1692	jz	.L070cbc_abort
1693	cmpl	$0,40(%esp)
1694	xchgl	%esp,%ebx
1695	movups	(%ebp),%xmm7
1696	movl	240(%edx),%ecx
1697	movl	%edx,%ebp
1698	movl	%ebx,16(%esp)
1699	movl	%ecx,%ebx
1700	je	.L071cbc_decrypt
1701	movaps	%xmm7,%xmm2
1702	cmpl	$16,%eax
1703	jb	.L072cbc_enc_tail
1704	subl	$16,%eax
1705	jmp	.L073cbc_enc_loop
1706.align	16
1707.L073cbc_enc_loop:
1708	movups	(%esi),%xmm7
1709	leal	16(%esi),%esi
1710	movups	(%edx),%xmm0
1711	movups	16(%edx),%xmm1
1712	xorps	%xmm0,%xmm7
1713	leal	32(%edx),%edx
1714	xorps	%xmm7,%xmm2
1715.L074enc1_loop_15:
1716.byte	102,15,56,220,209
1717	decl	%ecx
1718	movups	(%edx),%xmm1
1719	leal	16(%edx),%edx
1720	jnz	.L074enc1_loop_15
1721.byte	102,15,56,221,209
1722	movl	%ebx,%ecx
1723	movl	%ebp,%edx
1724	movups	%xmm2,(%edi)
1725	leal	16(%edi),%edi
1726	subl	$16,%eax
1727	jnc	.L073cbc_enc_loop
1728	addl	$16,%eax
1729	jnz	.L072cbc_enc_tail
1730	movaps	%xmm2,%xmm7
1731	jmp	.L075cbc_ret
1732.L072cbc_enc_tail:
1733	movl	%eax,%ecx
1734.long	2767451785
1735	movl	$16,%ecx
1736	subl	%eax,%ecx
1737	xorl	%eax,%eax
1738.long	2868115081
1739	leal	-16(%edi),%edi
1740	movl	%ebx,%ecx
1741	movl	%edi,%esi
1742	movl	%ebp,%edx
1743	jmp	.L073cbc_enc_loop
1744.align	16
1745.L071cbc_decrypt:
1746	cmpl	$80,%eax
1747	jbe	.L076cbc_dec_tail
1748	movaps	%xmm7,(%esp)
1749	subl	$80,%eax
1750	jmp	.L077cbc_dec_loop6_enter
1751.align	16
1752.L078cbc_dec_loop6:
1753	movaps	%xmm0,(%esp)
1754	movups	%xmm7,(%edi)
1755	leal	16(%edi),%edi
1756.L077cbc_dec_loop6_enter:
1757	movdqu	(%esi),%xmm2
1758	movdqu	16(%esi),%xmm3
1759	movdqu	32(%esi),%xmm4
1760	movdqu	48(%esi),%xmm5
1761	movdqu	64(%esi),%xmm6
1762	movdqu	80(%esi),%xmm7
1763	call	_aesni_decrypt6
1764	movups	(%esi),%xmm1
1765	movups	16(%esi),%xmm0
1766	xorps	(%esp),%xmm2
1767	xorps	%xmm1,%xmm3
1768	movups	32(%esi),%xmm1
1769	xorps	%xmm0,%xmm4
1770	movups	48(%esi),%xmm0
1771	xorps	%xmm1,%xmm5
1772	movups	64(%esi),%xmm1
1773	xorps	%xmm0,%xmm6
1774	movups	80(%esi),%xmm0
1775	xorps	%xmm1,%xmm7
1776	movups	%xmm2,(%edi)
1777	movups	%xmm3,16(%edi)
1778	leal	96(%esi),%esi
1779	movups	%xmm4,32(%edi)
1780	movl	%ebx,%ecx
1781	movups	%xmm5,48(%edi)
1782	movl	%ebp,%edx
1783	movups	%xmm6,64(%edi)
1784	leal	80(%edi),%edi
1785	subl	$96,%eax
1786	ja	.L078cbc_dec_loop6
1787	movaps	%xmm7,%xmm2
1788	movaps	%xmm0,%xmm7
1789	addl	$80,%eax
1790	jle	.L079cbc_dec_tail_collected
1791	movups	%xmm2,(%edi)
1792	leal	16(%edi),%edi
1793.L076cbc_dec_tail:
1794	movups	(%esi),%xmm2
1795	movaps	%xmm2,%xmm6
1796	cmpl	$16,%eax
1797	jbe	.L080cbc_dec_one
1798	movups	16(%esi),%xmm3
1799	movaps	%xmm3,%xmm5
1800	cmpl	$32,%eax
1801	jbe	.L081cbc_dec_two
1802	movups	32(%esi),%xmm4
1803	cmpl	$48,%eax
1804	jbe	.L082cbc_dec_three
1805	movups	48(%esi),%xmm5
1806	cmpl	$64,%eax
1807	jbe	.L083cbc_dec_four
1808	movups	64(%esi),%xmm6
1809	movaps	%xmm7,(%esp)
1810	movups	(%esi),%xmm2
1811	xorps	%xmm7,%xmm7
1812	call	_aesni_decrypt6
1813	movups	(%esi),%xmm1
1814	movups	16(%esi),%xmm0
1815	xorps	(%esp),%xmm2
1816	xorps	%xmm1,%xmm3
1817	movups	32(%esi),%xmm1
1818	xorps	%xmm0,%xmm4
1819	movups	48(%esi),%xmm0
1820	xorps	%xmm1,%xmm5
1821	movups	64(%esi),%xmm7
1822	xorps	%xmm0,%xmm6
1823	movups	%xmm2,(%edi)
1824	movups	%xmm3,16(%edi)
1825	movups	%xmm4,32(%edi)
1826	movups	%xmm5,48(%edi)
1827	leal	64(%edi),%edi
1828	movaps	%xmm6,%xmm2
1829	subl	$80,%eax
1830	jmp	.L079cbc_dec_tail_collected
1831.align	16
1832.L080cbc_dec_one:
1833	movups	(%edx),%xmm0
1834	movups	16(%edx),%xmm1
1835	leal	32(%edx),%edx
1836	xorps	%xmm0,%xmm2
1837.L084dec1_loop_16:
1838.byte	102,15,56,222,209
1839	decl	%ecx
1840	movups	(%edx),%xmm1
1841	leal	16(%edx),%edx
1842	jnz	.L084dec1_loop_16
1843.byte	102,15,56,223,209
1844	xorps	%xmm7,%xmm2
1845	movaps	%xmm6,%xmm7
1846	subl	$16,%eax
1847	jmp	.L079cbc_dec_tail_collected
1848.align	16
1849.L081cbc_dec_two:
1850	xorps	%xmm4,%xmm4
1851	call	_aesni_decrypt3
1852	xorps	%xmm7,%xmm2
1853	xorps	%xmm6,%xmm3
1854	movups	%xmm2,(%edi)
1855	movaps	%xmm3,%xmm2
1856	leal	16(%edi),%edi
1857	movaps	%xmm5,%xmm7
1858	subl	$32,%eax
1859	jmp	.L079cbc_dec_tail_collected
1860.align	16
1861.L082cbc_dec_three:
1862	call	_aesni_decrypt3
1863	xorps	%xmm7,%xmm2
1864	xorps	%xmm6,%xmm3
1865	xorps	%xmm5,%xmm4
1866	movups	%xmm2,(%edi)
1867	movaps	%xmm4,%xmm2
1868	movups	%xmm3,16(%edi)
1869	leal	32(%edi),%edi
1870	movups	32(%esi),%xmm7
1871	subl	$48,%eax
1872	jmp	.L079cbc_dec_tail_collected
1873.align	16
1874.L083cbc_dec_four:
1875	call	_aesni_decrypt4
1876	movups	16(%esi),%xmm1
1877	movups	32(%esi),%xmm0
1878	xorps	%xmm7,%xmm2
1879	movups	48(%esi),%xmm7
1880	xorps	%xmm6,%xmm3
1881	movups	%xmm2,(%edi)
1882	xorps	%xmm1,%xmm4
1883	movups	%xmm3,16(%edi)
1884	xorps	%xmm0,%xmm5
1885	movups	%xmm4,32(%edi)
1886	leal	48(%edi),%edi
1887	movaps	%xmm5,%xmm2
1888	subl	$64,%eax
1889.L079cbc_dec_tail_collected:
1890	andl	$15,%eax
1891	jnz	.L085cbc_dec_tail_partial
1892	movups	%xmm2,(%edi)
1893	jmp	.L075cbc_ret
1894.align	16
1895.L085cbc_dec_tail_partial:
1896	movaps	%xmm2,(%esp)
1897	movl	$16,%ecx
1898	movl	%esp,%esi
1899	subl	%eax,%ecx
1900.long	2767451785
1901.L075cbc_ret:
1902	movl	16(%esp),%esp
1903	movl	36(%esp),%ebp
1904	movups	%xmm7,(%ebp)
1905.L070cbc_abort:
1906	popl	%edi
1907	popl	%esi
1908	popl	%ebx
1909	popl	%ebp
1910	ret
1911.size	aesni_cbc_encrypt,.-.L_aesni_cbc_encrypt_begin
1912.type	_aesni_set_encrypt_key,@function
1913.align	16
1914_aesni_set_encrypt_key:
1915	testl	%eax,%eax
1916	jz	.L086bad_pointer
1917	testl	%edx,%edx
1918	jz	.L086bad_pointer
1919	movups	(%eax),%xmm0
1920	xorps	%xmm4,%xmm4
1921	leal	16(%edx),%edx
1922	cmpl	$256,%ecx
1923	je	.L08714rounds
1924	cmpl	$192,%ecx
1925	je	.L08812rounds
1926	cmpl	$128,%ecx
1927	jne	.L089bad_keybits
1928.align	16
1929.L09010rounds:
1930	movl	$9,%ecx
1931	movups	%xmm0,-16(%edx)
1932.byte	102,15,58,223,200,1
1933	call	.L091key_128_cold
1934.byte	102,15,58,223,200,2
1935	call	.L092key_128
1936.byte	102,15,58,223,200,4
1937	call	.L092key_128
1938.byte	102,15,58,223,200,8
1939	call	.L092key_128
1940.byte	102,15,58,223,200,16
1941	call	.L092key_128
1942.byte	102,15,58,223,200,32
1943	call	.L092key_128
1944.byte	102,15,58,223,200,64
1945	call	.L092key_128
1946.byte	102,15,58,223,200,128
1947	call	.L092key_128
1948.byte	102,15,58,223,200,27
1949	call	.L092key_128
1950.byte	102,15,58,223,200,54
1951	call	.L092key_128
1952	movups	%xmm0,(%edx)
1953	movl	%ecx,80(%edx)
1954	xorl	%eax,%eax
1955	ret
1956.align	16
1957.L092key_128:
1958	movups	%xmm0,(%edx)
1959	leal	16(%edx),%edx
1960.L091key_128_cold:
1961	shufps	$16,%xmm0,%xmm4
1962	xorps	%xmm4,%xmm0
1963	shufps	$140,%xmm0,%xmm4
1964	xorps	%xmm4,%xmm0
1965	shufps	$255,%xmm1,%xmm1
1966	xorps	%xmm1,%xmm0
1967	ret
1968.align	16
1969.L08812rounds:
1970	movq	16(%eax),%xmm2
1971	movl	$11,%ecx
1972	movups	%xmm0,-16(%edx)
1973.byte	102,15,58,223,202,1
1974	call	.L093key_192a_cold
1975.byte	102,15,58,223,202,2
1976	call	.L094key_192b
1977.byte	102,15,58,223,202,4
1978	call	.L095key_192a
1979.byte	102,15,58,223,202,8
1980	call	.L094key_192b
1981.byte	102,15,58,223,202,16
1982	call	.L095key_192a
1983.byte	102,15,58,223,202,32
1984	call	.L094key_192b
1985.byte	102,15,58,223,202,64
1986	call	.L095key_192a
1987.byte	102,15,58,223,202,128
1988	call	.L094key_192b
1989	movups	%xmm0,(%edx)
1990	movl	%ecx,48(%edx)
1991	xorl	%eax,%eax
1992	ret
1993.align	16
1994.L095key_192a:
1995	movups	%xmm0,(%edx)
1996	leal	16(%edx),%edx
1997.align	16
1998.L093key_192a_cold:
1999	movaps	%xmm2,%xmm5
2000.L096key_192b_warm:
2001	shufps	$16,%xmm0,%xmm4
2002	movdqa	%xmm2,%xmm3
2003	xorps	%xmm4,%xmm0
2004	shufps	$140,%xmm0,%xmm4
2005	pslldq	$4,%xmm3
2006	xorps	%xmm4,%xmm0
2007	pshufd	$85,%xmm1,%xmm1
2008	pxor	%xmm3,%xmm2
2009	pxor	%xmm1,%xmm0
2010	pshufd	$255,%xmm0,%xmm3
2011	pxor	%xmm3,%xmm2
2012	ret
2013.align	16
2014.L094key_192b:
2015	movaps	%xmm0,%xmm3
2016	shufps	$68,%xmm0,%xmm5
2017	movups	%xmm5,(%edx)
2018	shufps	$78,%xmm2,%xmm3
2019	movups	%xmm3,16(%edx)
2020	leal	32(%edx),%edx
2021	jmp	.L096key_192b_warm
2022.align	16
2023.L08714rounds:
2024	movups	16(%eax),%xmm2
2025	movl	$13,%ecx
2026	leal	16(%edx),%edx
2027	movups	%xmm0,-32(%edx)
2028	movups	%xmm2,-16(%edx)
2029.byte	102,15,58,223,202,1
2030	call	.L097key_256a_cold
2031.byte	102,15,58,223,200,1
2032	call	.L098key_256b
2033.byte	102,15,58,223,202,2
2034	call	.L099key_256a
2035.byte	102,15,58,223,200,2
2036	call	.L098key_256b
2037.byte	102,15,58,223,202,4
2038	call	.L099key_256a
2039.byte	102,15,58,223,200,4
2040	call	.L098key_256b
2041.byte	102,15,58,223,202,8
2042	call	.L099key_256a
2043.byte	102,15,58,223,200,8
2044	call	.L098key_256b
2045.byte	102,15,58,223,202,16
2046	call	.L099key_256a
2047.byte	102,15,58,223,200,16
2048	call	.L098key_256b
2049.byte	102,15,58,223,202,32
2050	call	.L099key_256a
2051.byte	102,15,58,223,200,32
2052	call	.L098key_256b
2053.byte	102,15,58,223,202,64
2054	call	.L099key_256a
2055	movups	%xmm0,(%edx)
2056	movl	%ecx,16(%edx)
2057	xorl	%eax,%eax
2058	ret
2059.align	16
2060.L099key_256a:
2061	movups	%xmm2,(%edx)
2062	leal	16(%edx),%edx
2063.L097key_256a_cold:
2064	shufps	$16,%xmm0,%xmm4
2065	xorps	%xmm4,%xmm0
2066	shufps	$140,%xmm0,%xmm4
2067	xorps	%xmm4,%xmm0
2068	shufps	$255,%xmm1,%xmm1
2069	xorps	%xmm1,%xmm0
2070	ret
2071.align	16
2072.L098key_256b:
2073	movups	%xmm0,(%edx)
2074	leal	16(%edx),%edx
2075	shufps	$16,%xmm2,%xmm4
2076	xorps	%xmm4,%xmm2
2077	shufps	$140,%xmm2,%xmm4
2078	xorps	%xmm4,%xmm2
2079	shufps	$170,%xmm1,%xmm1
2080	xorps	%xmm1,%xmm2
2081	ret
2082.align	4
2083.L086bad_pointer:
2084	movl	$-1,%eax
2085	ret
2086.align	4
2087.L089bad_keybits:
2088	movl	$-2,%eax
2089	ret
2090.size	_aesni_set_encrypt_key,.-_aesni_set_encrypt_key
2091.globl	aesni_set_encrypt_key
2092.type	aesni_set_encrypt_key,@function
2093.align	16
2094aesni_set_encrypt_key:
2095.L_aesni_set_encrypt_key_begin:
2096	movl	4(%esp),%eax
2097	movl	8(%esp),%ecx
2098	movl	12(%esp),%edx
2099	call	_aesni_set_encrypt_key
2100	ret
2101.size	aesni_set_encrypt_key,.-.L_aesni_set_encrypt_key_begin
2102.globl	aesni_set_decrypt_key
2103.type	aesni_set_decrypt_key,@function
2104.align	16
2105aesni_set_decrypt_key:
2106.L_aesni_set_decrypt_key_begin:
2107	movl	4(%esp),%eax
2108	movl	8(%esp),%ecx
2109	movl	12(%esp),%edx
2110	call	_aesni_set_encrypt_key
2111	movl	12(%esp),%edx
2112	shll	$4,%ecx
2113	testl	%eax,%eax
2114	jnz	.L100dec_key_ret
2115	leal	16(%edx,%ecx,1),%eax
2116	movups	(%edx),%xmm0
2117	movups	(%eax),%xmm1
2118	movups	%xmm0,(%eax)
2119	movups	%xmm1,(%edx)
2120	leal	16(%edx),%edx
2121	leal	-16(%eax),%eax
2122.L101dec_key_inverse:
2123	movups	(%edx),%xmm0
2124	movups	(%eax),%xmm1
2125.byte	102,15,56,219,192
2126.byte	102,15,56,219,201
2127	leal	16(%edx),%edx
2128	leal	-16(%eax),%eax
2129	movups	%xmm0,16(%eax)
2130	movups	%xmm1,-16(%edx)
2131	cmpl	%edx,%eax
2132	ja	.L101dec_key_inverse
2133	movups	(%edx),%xmm0
2134.byte	102,15,56,219,192
2135	movups	%xmm0,(%edx)
2136	xorl	%eax,%eax
2137.L100dec_key_ret:
2138	ret
2139.size	aesni_set_decrypt_key,.-.L_aesni_set_decrypt_key_begin
2140.byte	65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69
2141.byte	83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83
2142.byte	32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115
2143.byte	115,108,46,111,114,103,62,0
2144