1########################################################################
2# Copyright (c) 2013, Intel Corporation
3#
4# This software is available to you under a choice of one of two
5# licenses.  You may choose to be licensed under the terms of the GNU
6# General Public License (GPL) Version 2, available from the file
7# COPYING in the main directory of this source tree, or the
8# OpenIB.org BSD license below:
9#
10# Redistribution and use in source and binary forms, with or without
11# modification, are permitted provided that the following conditions are
12# met:
13#
14# * Redistributions of source code must retain the above copyright
15#   notice, this list of conditions and the following disclaimer.
16#
17# * Redistributions in binary form must reproduce the above copyright
18#   notice, this list of conditions and the following disclaimer in the
19#   documentation and/or other materials provided with the
20#   distribution.
21#
22# * Neither the name of the Intel Corporation nor the names of its
23#   contributors may be used to endorse or promote products derived from
24#   this software without specific prior written permission.
25#
26#
27# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
28# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
31# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES# LOSS OF USE, DATA, OR
34# PROFITS# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38########################################################################
39##
40## Authors:
41##	Erdinc Ozturk <erdinc.ozturk@intel.com>
42##	Vinodh Gopal <vinodh.gopal@intel.com>
43##	James Guilford <james.guilford@intel.com>
44##	Tim Chen <tim.c.chen@linux.intel.com>
45##
46## References:
47##       This code was derived and highly optimized from the code described in paper:
48##               Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation
49##			on Intel Architecture Processors. August, 2010
50##       The details of the implementation is explained in:
51##               Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode
52##			on Intel Architecture Processors. October, 2012.
53##
54## Assumptions:
55##
56##
57##
58## iv:
59##       0                   1                   2                   3
60##       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
61##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
62##       |                             Salt  (From the SA)               |
63##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
64##       |                     Initialization Vector                     |
65##       |         (This is the sequence number from IPSec header)       |
66##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
67##       |                              0x1                              |
68##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
69##
70##
71##
72## AAD:
73##       AAD padded to 128 bits with 0
74##       for example, assume AAD is a u32 vector
75##
76##       if AAD is 8 bytes:
77##       AAD[3] = {A0, A1}#
78##       padded AAD in xmm register = {A1 A0 0 0}
79##
80##       0                   1                   2                   3
81##       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
82##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
83##       |                               SPI (A1)                        |
84##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
85##       |                     32-bit Sequence Number (A0)               |
86##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
87##       |                              0x0                              |
88##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
89##
90##                                       AAD Format with 32-bit Sequence Number
91##
92##       if AAD is 12 bytes:
93##       AAD[3] = {A0, A1, A2}#
94##       padded AAD in xmm register = {A2 A1 A0 0}
95##
96##       0                   1                   2                   3
97##       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
98##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
99##       |                               SPI (A2)                        |
100##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
101##       |                 64-bit Extended Sequence Number {A1,A0}       |
102##       |                                                               |
103##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
104##       |                              0x0                              |
105##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
106##
107##        AAD Format with 64-bit Extended Sequence Number
108##
109##
110## aadLen:
111##       from the definition of the spec, aadLen can only be 8 or 12 bytes.
112##	 The code additionally supports aadLen of length 16 bytes.
113##
114## TLen:
115##       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
116##
117## poly = x^128 + x^127 + x^126 + x^121 + 1
118## throughout the code, one tab and two tab indentations are used. one tab is
119## for GHASH part, two tabs is for AES part.
120##
121
122#include <linux/linkage.h>
123
124# constants in mergeable sections, linker can reorder and merge
125.section	.rodata.cst16.POLY, "aM", @progbits, 16
126.align 16
127POLY:            .octa     0xC2000000000000000000000000000001
128
129.section	.rodata.cst16.POLY2, "aM", @progbits, 16
130.align 16
131POLY2:           .octa     0xC20000000000000000000001C2000000
132
133.section	.rodata.cst16.TWOONE, "aM", @progbits, 16
134.align 16
135TWOONE:          .octa     0x00000001000000000000000000000001
136
137.section	.rodata.cst16.SHUF_MASK, "aM", @progbits, 16
138.align 16
139SHUF_MASK:       .octa     0x000102030405060708090A0B0C0D0E0F
140
141.section	.rodata.cst16.ONE, "aM", @progbits, 16
142.align 16
143ONE:             .octa     0x00000000000000000000000000000001
144
145.section	.rodata.cst16.ONEf, "aM", @progbits, 16
146.align 16
147ONEf:            .octa     0x01000000000000000000000000000000
148
149# order of these constants should not change.
150# more specifically, ALL_F should follow SHIFT_MASK, and zero should follow ALL_F
151.section	.rodata, "a", @progbits
152.align 16
153SHIFT_MASK:      .octa     0x0f0e0d0c0b0a09080706050403020100
154ALL_F:           .octa     0xffffffffffffffffffffffffffffffff
155                 .octa     0x00000000000000000000000000000000
156
157.text
158
159
160#define AadHash 16*0
161#define AadLen 16*1
162#define InLen (16*1)+8
163#define PBlockEncKey 16*2
164#define OrigIV 16*3
165#define CurCount 16*4
166#define PBlockLen 16*5
167
168HashKey        = 16*6   # store HashKey <<1 mod poly here
169HashKey_2      = 16*7   # store HashKey^2 <<1 mod poly here
170HashKey_3      = 16*8   # store HashKey^3 <<1 mod poly here
171HashKey_4      = 16*9   # store HashKey^4 <<1 mod poly here
172HashKey_5      = 16*10   # store HashKey^5 <<1 mod poly here
173HashKey_6      = 16*11   # store HashKey^6 <<1 mod poly here
174HashKey_7      = 16*12   # store HashKey^7 <<1 mod poly here
175HashKey_8      = 16*13   # store HashKey^8 <<1 mod poly here
176HashKey_k      = 16*14   # store XOR of HashKey <<1 mod poly here (for Karatsuba purposes)
177HashKey_2_k    = 16*15   # store XOR of HashKey^2 <<1 mod poly here (for Karatsuba purposes)
178HashKey_3_k    = 16*16   # store XOR of HashKey^3 <<1 mod poly here (for Karatsuba purposes)
179HashKey_4_k    = 16*17   # store XOR of HashKey^4 <<1 mod poly here (for Karatsuba purposes)
180HashKey_5_k    = 16*18   # store XOR of HashKey^5 <<1 mod poly here (for Karatsuba purposes)
181HashKey_6_k    = 16*19   # store XOR of HashKey^6 <<1 mod poly here (for Karatsuba purposes)
182HashKey_7_k    = 16*20   # store XOR of HashKey^7 <<1 mod poly here (for Karatsuba purposes)
183HashKey_8_k    = 16*21   # store XOR of HashKey^8 <<1 mod poly here (for Karatsuba purposes)
184
185#define arg1 %rdi
186#define arg2 %rsi
187#define arg3 %rdx
188#define arg4 %rcx
189#define arg5 %r8
190#define arg6 %r9
191#define keysize 2*15*16(arg1)
192
193i = 0
194j = 0
195
196out_order = 0
197in_order = 1
198DEC = 0
199ENC = 1
200
201.macro define_reg r n
202reg_\r = %xmm\n
203.endm
204
205.macro setreg
206.altmacro
207define_reg i %i
208define_reg j %j
209.noaltmacro
210.endm
211
212TMP1 =   16*0    # Temporary storage for AAD
213TMP2 =   16*1    # Temporary storage for AES State 2 (State 1 is stored in an XMM register)
214TMP3 =   16*2    # Temporary storage for AES State 3
215TMP4 =   16*3    # Temporary storage for AES State 4
216TMP5 =   16*4    # Temporary storage for AES State 5
217TMP6 =   16*5    # Temporary storage for AES State 6
218TMP7 =   16*6    # Temporary storage for AES State 7
219TMP8 =   16*7    # Temporary storage for AES State 8
220
221VARIABLE_OFFSET = 16*8
222
223################################
224# Utility Macros
225################################
226
227.macro FUNC_SAVE
228        push    %r12
229        push    %r13
230        push    %r15
231
232	push	%rbp
233	mov	%rsp, %rbp
234
235        sub     $VARIABLE_OFFSET, %rsp
236        and     $~63, %rsp                    # align rsp to 64 bytes
237.endm
238
239.macro FUNC_RESTORE
240        mov     %rbp, %rsp
241	pop	%rbp
242
243        pop     %r15
244        pop     %r13
245        pop     %r12
246.endm
247
248# Encryption of a single block
249.macro ENCRYPT_SINGLE_BLOCK REP XMM0
250                vpxor    (arg1), \XMM0, \XMM0
251               i = 1
252               setreg
253.rep \REP
254                vaesenc  16*i(arg1), \XMM0, \XMM0
255               i = (i+1)
256               setreg
257.endr
258                vaesenclast 16*i(arg1), \XMM0, \XMM0
259.endm
260
261# combined for GCM encrypt and decrypt functions
262# clobbering all xmm registers
263# clobbering r10, r11, r12, r13, r15, rax
264.macro  GCM_ENC_DEC INITIAL_BLOCKS GHASH_8_ENCRYPT_8_PARALLEL GHASH_LAST_8 GHASH_MUL ENC_DEC REP
265        vmovdqu AadHash(arg2), %xmm8
266        vmovdqu  HashKey(arg2), %xmm13      # xmm13 = HashKey
267        add arg5, InLen(arg2)
268
269        # initialize the data pointer offset as zero
270        xor     %r11d, %r11d
271
272        PARTIAL_BLOCK \GHASH_MUL, arg3, arg4, arg5, %r11, %xmm8, \ENC_DEC
273        sub %r11, arg5
274
275        mov     arg5, %r13                  # save the number of bytes of plaintext/ciphertext
276        and     $-16, %r13                  # r13 = r13 - (r13 mod 16)
277
278        mov     %r13, %r12
279        shr     $4, %r12
280        and     $7, %r12
281        jz      .L_initial_num_blocks_is_0\@
282
283        cmp     $7, %r12
284        je      .L_initial_num_blocks_is_7\@
285        cmp     $6, %r12
286        je      .L_initial_num_blocks_is_6\@
287        cmp     $5, %r12
288        je      .L_initial_num_blocks_is_5\@
289        cmp     $4, %r12
290        je      .L_initial_num_blocks_is_4\@
291        cmp     $3, %r12
292        je      .L_initial_num_blocks_is_3\@
293        cmp     $2, %r12
294        je      .L_initial_num_blocks_is_2\@
295
296        jmp     .L_initial_num_blocks_is_1\@
297
298.L_initial_num_blocks_is_7\@:
299        \INITIAL_BLOCKS  \REP, 7, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
300        sub     $16*7, %r13
301        jmp     .L_initial_blocks_encrypted\@
302
303.L_initial_num_blocks_is_6\@:
304        \INITIAL_BLOCKS  \REP, 6, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
305        sub     $16*6, %r13
306        jmp     .L_initial_blocks_encrypted\@
307
308.L_initial_num_blocks_is_5\@:
309        \INITIAL_BLOCKS  \REP, 5, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
310        sub     $16*5, %r13
311        jmp     .L_initial_blocks_encrypted\@
312
313.L_initial_num_blocks_is_4\@:
314        \INITIAL_BLOCKS  \REP, 4, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
315        sub     $16*4, %r13
316        jmp     .L_initial_blocks_encrypted\@
317
318.L_initial_num_blocks_is_3\@:
319        \INITIAL_BLOCKS  \REP, 3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
320        sub     $16*3, %r13
321        jmp     .L_initial_blocks_encrypted\@
322
323.L_initial_num_blocks_is_2\@:
324        \INITIAL_BLOCKS  \REP, 2, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
325        sub     $16*2, %r13
326        jmp     .L_initial_blocks_encrypted\@
327
328.L_initial_num_blocks_is_1\@:
329        \INITIAL_BLOCKS  \REP, 1, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
330        sub     $16*1, %r13
331        jmp     .L_initial_blocks_encrypted\@
332
333.L_initial_num_blocks_is_0\@:
334        \INITIAL_BLOCKS  \REP, 0, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
335
336
337.L_initial_blocks_encrypted\@:
338        test    %r13, %r13
339        je      .L_zero_cipher_left\@
340
341        sub     $128, %r13
342        je      .L_eight_cipher_left\@
343
344
345
346
347        vmovd   %xmm9, %r15d
348        and     $255, %r15d
349        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
350
351
352.L_encrypt_by_8_new\@:
353        cmp     $(255-8), %r15d
354        jg      .L_encrypt_by_8\@
355
356
357
358        add     $8, %r15b
359        \GHASH_8_ENCRYPT_8_PARALLEL      \REP, %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, out_order, \ENC_DEC
360        add     $128, %r11
361        sub     $128, %r13
362        jne     .L_encrypt_by_8_new\@
363
364        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
365        jmp     .L_eight_cipher_left\@
366
367.L_encrypt_by_8\@:
368        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
369        add     $8, %r15b
370        \GHASH_8_ENCRYPT_8_PARALLEL      \REP, %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, in_order, \ENC_DEC
371        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
372        add     $128, %r11
373        sub     $128, %r13
374        jne     .L_encrypt_by_8_new\@
375
376        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
377
378
379
380
381.L_eight_cipher_left\@:
382        \GHASH_LAST_8    %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8
383
384
385.L_zero_cipher_left\@:
386        vmovdqu %xmm14, AadHash(arg2)
387        vmovdqu %xmm9, CurCount(arg2)
388
389        # check for 0 length
390        mov     arg5, %r13
391        and     $15, %r13                            # r13 = (arg5 mod 16)
392
393        je      .L_multiple_of_16_bytes\@
394
395        # handle the last <16 Byte block separately
396
397        mov %r13, PBlockLen(arg2)
398
399        vpaddd  ONE(%rip), %xmm9, %xmm9              # INCR CNT to get Yn
400        vmovdqu %xmm9, CurCount(arg2)
401        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
402
403        ENCRYPT_SINGLE_BLOCK    \REP, %xmm9                # E(K, Yn)
404        vmovdqu %xmm9, PBlockEncKey(arg2)
405
406        cmp $16, arg5
407        jge .L_large_enough_update\@
408
409        lea (arg4,%r11,1), %r10
410        mov %r13, %r12
411
412        READ_PARTIAL_BLOCK %r10 %r12 %xmm1
413
414        lea     SHIFT_MASK+16(%rip), %r12
415        sub     %r13, %r12                           # adjust the shuffle mask pointer to be
416						     # able to shift 16-r13 bytes (r13 is the
417	# number of bytes in plaintext mod 16)
418
419        jmp .L_final_ghash_mul\@
420
421.L_large_enough_update\@:
422        sub $16, %r11
423        add %r13, %r11
424
425        # receive the last <16 Byte block
426        vmovdqu	(arg4, %r11, 1), %xmm1
427
428        sub	%r13, %r11
429        add	$16, %r11
430
431        lea	SHIFT_MASK+16(%rip), %r12
432        # adjust the shuffle mask pointer to be able to shift 16-r13 bytes
433        # (r13 is the number of bytes in plaintext mod 16)
434        sub	%r13, %r12
435        # get the appropriate shuffle mask
436        vmovdqu	(%r12), %xmm2
437        # shift right 16-r13 bytes
438        vpshufb  %xmm2, %xmm1, %xmm1
439
440.L_final_ghash_mul\@:
441        .if  \ENC_DEC ==  DEC
442        vmovdqa %xmm1, %xmm2
443        vpxor   %xmm1, %xmm9, %xmm9                  # Plaintext XOR E(K, Yn)
444        vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1        # get the appropriate mask to
445						     # mask out top 16-r13 bytes of xmm9
446        vpand   %xmm1, %xmm9, %xmm9                  # mask out top 16-r13 bytes of xmm9
447        vpand   %xmm1, %xmm2, %xmm2
448        vpshufb SHUF_MASK(%rip), %xmm2, %xmm2
449        vpxor   %xmm2, %xmm14, %xmm14
450
451        vmovdqu %xmm14, AadHash(arg2)
452        .else
453        vpxor   %xmm1, %xmm9, %xmm9                  # Plaintext XOR E(K, Yn)
454        vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1        # get the appropriate mask to
455						     # mask out top 16-r13 bytes of xmm9
456        vpand   %xmm1, %xmm9, %xmm9                  # mask out top 16-r13 bytes of xmm9
457        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
458        vpxor   %xmm9, %xmm14, %xmm14
459
460        vmovdqu %xmm14, AadHash(arg2)
461        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9        # shuffle xmm9 back to output as ciphertext
462        .endif
463
464
465        #############################
466        # output r13 Bytes
467        vmovq   %xmm9, %rax
468        cmp     $8, %r13
469        jle     .L_less_than_8_bytes_left\@
470
471        mov     %rax, (arg3 , %r11)
472        add     $8, %r11
473        vpsrldq $8, %xmm9, %xmm9
474        vmovq   %xmm9, %rax
475        sub     $8, %r13
476
477.L_less_than_8_bytes_left\@:
478        movb    %al, (arg3 , %r11)
479        add     $1, %r11
480        shr     $8, %rax
481        sub     $1, %r13
482        jne     .L_less_than_8_bytes_left\@
483        #############################
484
485.L_multiple_of_16_bytes\@:
486.endm
487
488
489# GCM_COMPLETE Finishes update of tag of last partial block
490# Output: Authorization Tag (AUTH_TAG)
491# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15
492.macro GCM_COMPLETE GHASH_MUL REP AUTH_TAG AUTH_TAG_LEN
493        vmovdqu AadHash(arg2), %xmm14
494        vmovdqu HashKey(arg2), %xmm13
495
496        mov PBlockLen(arg2), %r12
497        test %r12, %r12
498        je .L_partial_done\@
499
500	#GHASH computation for the last <16 Byte block
501        \GHASH_MUL       %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
502
503.L_partial_done\@:
504        mov AadLen(arg2), %r12                          # r12 = aadLen (number of bytes)
505        shl     $3, %r12                             # convert into number of bits
506        vmovd   %r12d, %xmm15                        # len(A) in xmm15
507
508        mov InLen(arg2), %r12
509        shl     $3, %r12                        # len(C) in bits  (*128)
510        vmovq   %r12, %xmm1
511        vpslldq $8, %xmm15, %xmm15                   # xmm15 = len(A)|| 0x0000000000000000
512        vpxor   %xmm1, %xmm15, %xmm15                # xmm15 = len(A)||len(C)
513
514        vpxor   %xmm15, %xmm14, %xmm14
515        \GHASH_MUL       %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6    # final GHASH computation
516        vpshufb SHUF_MASK(%rip), %xmm14, %xmm14      # perform a 16Byte swap
517
518        vmovdqu OrigIV(arg2), %xmm9
519
520        ENCRYPT_SINGLE_BLOCK    \REP, %xmm9                # E(K, Y0)
521
522        vpxor   %xmm14, %xmm9, %xmm9
523
524
525
526.L_return_T\@:
527        mov     \AUTH_TAG, %r10              # r10 = authTag
528        mov     \AUTH_TAG_LEN, %r11              # r11 = auth_tag_len
529
530        cmp     $16, %r11
531        je      .L_T_16\@
532
533        cmp     $8, %r11
534        jl      .L_T_4\@
535
536.L_T_8\@:
537        vmovq   %xmm9, %rax
538        mov     %rax, (%r10)
539        add     $8, %r10
540        sub     $8, %r11
541        vpsrldq $8, %xmm9, %xmm9
542        test    %r11, %r11
543        je     .L_return_T_done\@
544.L_T_4\@:
545        vmovd   %xmm9, %eax
546        mov     %eax, (%r10)
547        add     $4, %r10
548        sub     $4, %r11
549        vpsrldq     $4, %xmm9, %xmm9
550        test    %r11, %r11
551        je     .L_return_T_done\@
552.L_T_123\@:
553        vmovd     %xmm9, %eax
554        cmp     $2, %r11
555        jl     .L_T_1\@
556        mov     %ax, (%r10)
557        cmp     $2, %r11
558        je     .L_return_T_done\@
559        add     $2, %r10
560        sar     $16, %eax
561.L_T_1\@:
562        mov     %al, (%r10)
563        jmp     .L_return_T_done\@
564
565.L_T_16\@:
566        vmovdqu %xmm9, (%r10)
567
568.L_return_T_done\@:
569.endm
570
571.macro CALC_AAD_HASH GHASH_MUL AAD AADLEN T1 T2 T3 T4 T5 T6 T7 T8
572
573	mov     \AAD, %r10                      # r10 = AAD
574	mov     \AADLEN, %r12                      # r12 = aadLen
575
576
577	mov     %r12, %r11
578
579	vpxor   \T8, \T8, \T8
580	vpxor   \T7, \T7, \T7
581	cmp     $16, %r11
582	jl      .L_get_AAD_rest8\@
583.L_get_AAD_blocks\@:
584	vmovdqu (%r10), \T7
585	vpshufb SHUF_MASK(%rip), \T7, \T7
586	vpxor   \T7, \T8, \T8
587	\GHASH_MUL       \T8, \T2, \T1, \T3, \T4, \T5, \T6
588	add     $16, %r10
589	sub     $16, %r12
590	sub     $16, %r11
591	cmp     $16, %r11
592	jge     .L_get_AAD_blocks\@
593	vmovdqu \T8, \T7
594	test    %r11, %r11
595	je      .L_get_AAD_done\@
596
597	vpxor   \T7, \T7, \T7
598
599	/* read the last <16B of AAD. since we have at least 4B of
600	data right after the AAD (the ICV, and maybe some CT), we can
601	read 4B/8B blocks safely, and then get rid of the extra stuff */
602.L_get_AAD_rest8\@:
603	cmp     $4, %r11
604	jle     .L_get_AAD_rest4\@
605	movq    (%r10), \T1
606	add     $8, %r10
607	sub     $8, %r11
608	vpslldq $8, \T1, \T1
609	vpsrldq $8, \T7, \T7
610	vpxor   \T1, \T7, \T7
611	jmp     .L_get_AAD_rest8\@
612.L_get_AAD_rest4\@:
613	test    %r11, %r11
614	jle     .L_get_AAD_rest0\@
615	mov     (%r10), %eax
616	movq    %rax, \T1
617	add     $4, %r10
618	sub     $4, %r11
619	vpslldq $12, \T1, \T1
620	vpsrldq $4, \T7, \T7
621	vpxor   \T1, \T7, \T7
622.L_get_AAD_rest0\@:
623	/* finalize: shift out the extra bytes we read, and align
624	left. since pslldq can only shift by an immediate, we use
625	vpshufb and a pair of shuffle masks */
626	leaq	ALL_F(%rip), %r11
627	subq	%r12, %r11
628	vmovdqu	16(%r11), \T1
629	andq	$~3, %r11
630	vpshufb (%r11), \T7, \T7
631	vpand	\T1, \T7, \T7
632.L_get_AAD_rest_final\@:
633	vpshufb SHUF_MASK(%rip), \T7, \T7
634	vpxor   \T8, \T7, \T7
635	\GHASH_MUL       \T7, \T2, \T1, \T3, \T4, \T5, \T6
636
637.L_get_AAD_done\@:
638        vmovdqu \T7, AadHash(arg2)
639.endm
640
641.macro INIT GHASH_MUL PRECOMPUTE
642        mov arg6, %r11
643        mov %r11, AadLen(arg2) # ctx_data.aad_length = aad_length
644        xor %r11d, %r11d
645        mov %r11, InLen(arg2) # ctx_data.in_length = 0
646
647        mov %r11, PBlockLen(arg2) # ctx_data.partial_block_length = 0
648        mov %r11, PBlockEncKey(arg2) # ctx_data.partial_block_enc_key = 0
649        mov arg3, %rax
650        movdqu (%rax), %xmm0
651        movdqu %xmm0, OrigIV(arg2) # ctx_data.orig_IV = iv
652
653        vpshufb SHUF_MASK(%rip), %xmm0, %xmm0
654        movdqu %xmm0, CurCount(arg2) # ctx_data.current_counter = iv
655
656        vmovdqu  (arg4), %xmm6              # xmm6 = HashKey
657
658        vpshufb  SHUF_MASK(%rip), %xmm6, %xmm6
659        ###############  PRECOMPUTATION of HashKey<<1 mod poly from the HashKey
660        vmovdqa  %xmm6, %xmm2
661        vpsllq   $1, %xmm6, %xmm6
662        vpsrlq   $63, %xmm2, %xmm2
663        vmovdqa  %xmm2, %xmm1
664        vpslldq  $8, %xmm2, %xmm2
665        vpsrldq  $8, %xmm1, %xmm1
666        vpor     %xmm2, %xmm6, %xmm6
667        #reduction
668        vpshufd  $0b00100100, %xmm1, %xmm2
669        vpcmpeqd TWOONE(%rip), %xmm2, %xmm2
670        vpand    POLY(%rip), %xmm2, %xmm2
671        vpxor    %xmm2, %xmm6, %xmm6        # xmm6 holds the HashKey<<1 mod poly
672        #######################################################################
673        vmovdqu  %xmm6, HashKey(arg2)       # store HashKey<<1 mod poly
674
675        CALC_AAD_HASH \GHASH_MUL, arg5, arg6, %xmm2, %xmm6, %xmm3, %xmm4, %xmm5, %xmm7, %xmm1, %xmm0
676
677        \PRECOMPUTE  %xmm6, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5
678.endm
679
680
681# Reads DLEN bytes starting at DPTR and stores in XMMDst
682# where 0 < DLEN < 16
683# Clobbers %rax, DLEN
684.macro READ_PARTIAL_BLOCK DPTR DLEN XMMDst
685        vpxor \XMMDst, \XMMDst, \XMMDst
686
687        cmp $8, \DLEN
688        jl .L_read_lt8_\@
689        mov (\DPTR), %rax
690        vpinsrq $0, %rax, \XMMDst, \XMMDst
691        sub $8, \DLEN
692        jz .L_done_read_partial_block_\@
693        xor %eax, %eax
694.L_read_next_byte_\@:
695        shl $8, %rax
696        mov 7(\DPTR, \DLEN, 1), %al
697        dec \DLEN
698        jnz .L_read_next_byte_\@
699        vpinsrq $1, %rax, \XMMDst, \XMMDst
700        jmp .L_done_read_partial_block_\@
701.L_read_lt8_\@:
702        xor %eax, %eax
703.L_read_next_byte_lt8_\@:
704        shl $8, %rax
705        mov -1(\DPTR, \DLEN, 1), %al
706        dec \DLEN
707        jnz .L_read_next_byte_lt8_\@
708        vpinsrq $0, %rax, \XMMDst, \XMMDst
709.L_done_read_partial_block_\@:
710.endm
711
712# PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks
713# between update calls.
714# Requires the input data be at least 1 byte long due to READ_PARTIAL_BLOCK
715# Outputs encrypted bytes, and updates hash and partial info in gcm_data_context
716# Clobbers rax, r10, r12, r13, xmm0-6, xmm9-13
717.macro PARTIAL_BLOCK GHASH_MUL CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \
718        AAD_HASH ENC_DEC
719        mov 	PBlockLen(arg2), %r13
720        test	%r13, %r13
721        je	.L_partial_block_done_\@	# Leave Macro if no partial blocks
722        # Read in input data without over reading
723        cmp	$16, \PLAIN_CYPH_LEN
724        jl	.L_fewer_than_16_bytes_\@
725        vmovdqu	(\PLAIN_CYPH_IN), %xmm1	# If more than 16 bytes, just fill xmm
726        jmp	.L_data_read_\@
727
728.L_fewer_than_16_bytes_\@:
729        lea	(\PLAIN_CYPH_IN, \DATA_OFFSET, 1), %r10
730        mov	\PLAIN_CYPH_LEN, %r12
731        READ_PARTIAL_BLOCK %r10 %r12 %xmm1
732
733        mov PBlockLen(arg2), %r13
734
735.L_data_read_\@:				# Finished reading in data
736
737        vmovdqu	PBlockEncKey(arg2), %xmm9
738        vmovdqu	HashKey(arg2), %xmm13
739
740        lea	SHIFT_MASK(%rip), %r12
741
742        # adjust the shuffle mask pointer to be able to shift r13 bytes
743        # r16-r13 is the number of bytes in plaintext mod 16)
744        add	%r13, %r12
745        vmovdqu	(%r12), %xmm2		# get the appropriate shuffle mask
746        vpshufb %xmm2, %xmm9, %xmm9		# shift right r13 bytes
747
748.if  \ENC_DEC ==  DEC
749        vmovdqa	%xmm1, %xmm3
750        pxor	%xmm1, %xmm9		# Cyphertext XOR E(K, Yn)
751
752        mov	\PLAIN_CYPH_LEN, %r10
753        add	%r13, %r10
754        # Set r10 to be the amount of data left in CYPH_PLAIN_IN after filling
755        sub	$16, %r10
756        # Determine if if partial block is not being filled and
757        # shift mask accordingly
758        jge	.L_no_extra_mask_1_\@
759        sub	%r10, %r12
760.L_no_extra_mask_1_\@:
761
762        vmovdqu	ALL_F-SHIFT_MASK(%r12), %xmm1
763        # get the appropriate mask to mask out bottom r13 bytes of xmm9
764        vpand	%xmm1, %xmm9, %xmm9		# mask out bottom r13 bytes of xmm9
765
766        vpand	%xmm1, %xmm3, %xmm3
767        vmovdqa	SHUF_MASK(%rip), %xmm10
768        vpshufb	%xmm10, %xmm3, %xmm3
769        vpshufb	%xmm2, %xmm3, %xmm3
770        vpxor	%xmm3, \AAD_HASH, \AAD_HASH
771
772        test	%r10, %r10
773        jl	.L_partial_incomplete_1_\@
774
775        # GHASH computation for the last <16 Byte block
776        \GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
777        xor	%eax,%eax
778
779        mov	%rax, PBlockLen(arg2)
780        jmp	.L_dec_done_\@
781.L_partial_incomplete_1_\@:
782        add	\PLAIN_CYPH_LEN, PBlockLen(arg2)
783.L_dec_done_\@:
784        vmovdqu	\AAD_HASH, AadHash(arg2)
785.else
786        vpxor	%xmm1, %xmm9, %xmm9			# Plaintext XOR E(K, Yn)
787
788        mov	\PLAIN_CYPH_LEN, %r10
789        add	%r13, %r10
790        # Set r10 to be the amount of data left in CYPH_PLAIN_IN after filling
791        sub	$16, %r10
792        # Determine if if partial block is not being filled and
793        # shift mask accordingly
794        jge	.L_no_extra_mask_2_\@
795        sub	%r10, %r12
796.L_no_extra_mask_2_\@:
797
798        vmovdqu	ALL_F-SHIFT_MASK(%r12), %xmm1
799        # get the appropriate mask to mask out bottom r13 bytes of xmm9
800        vpand	%xmm1, %xmm9, %xmm9
801
802        vmovdqa	SHUF_MASK(%rip), %xmm1
803        vpshufb %xmm1, %xmm9, %xmm9
804        vpshufb %xmm2, %xmm9, %xmm9
805        vpxor	%xmm9, \AAD_HASH, \AAD_HASH
806
807        test	%r10, %r10
808        jl	.L_partial_incomplete_2_\@
809
810        # GHASH computation for the last <16 Byte block
811        \GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
812        xor	%eax,%eax
813
814        mov	%rax, PBlockLen(arg2)
815        jmp	.L_encode_done_\@
816.L_partial_incomplete_2_\@:
817        add	\PLAIN_CYPH_LEN, PBlockLen(arg2)
818.L_encode_done_\@:
819        vmovdqu	\AAD_HASH, AadHash(arg2)
820
821        vmovdqa	SHUF_MASK(%rip), %xmm10
822        # shuffle xmm9 back to output as ciphertext
823        vpshufb	%xmm10, %xmm9, %xmm9
824        vpshufb	%xmm2, %xmm9, %xmm9
825.endif
826        # output encrypted Bytes
827        test	%r10, %r10
828        jl	.L_partial_fill_\@
829        mov	%r13, %r12
830        mov	$16, %r13
831        # Set r13 to be the number of bytes to write out
832        sub	%r12, %r13
833        jmp	.L_count_set_\@
834.L_partial_fill_\@:
835        mov	\PLAIN_CYPH_LEN, %r13
836.L_count_set_\@:
837        vmovdqa	%xmm9, %xmm0
838        vmovq	%xmm0, %rax
839        cmp	$8, %r13
840        jle	.L_less_than_8_bytes_left_\@
841
842        mov	%rax, (\CYPH_PLAIN_OUT, \DATA_OFFSET, 1)
843        add	$8, \DATA_OFFSET
844        psrldq	$8, %xmm0
845        vmovq	%xmm0, %rax
846        sub	$8, %r13
847.L_less_than_8_bytes_left_\@:
848        movb	%al, (\CYPH_PLAIN_OUT, \DATA_OFFSET, 1)
849        add	$1, \DATA_OFFSET
850        shr	$8, %rax
851        sub	$1, %r13
852        jne	.L_less_than_8_bytes_left_\@
853.L_partial_block_done_\@:
854.endm # PARTIAL_BLOCK
855
856###############################################################################
857# GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
858# Input: A and B (128-bits each, bit-reflected)
859# Output: C = A*B*x mod poly, (i.e. >>1 )
860# To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
861# GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
862###############################################################################
863.macro  GHASH_MUL_AVX GH HK T1 T2 T3 T4 T5
864
865        vpshufd         $0b01001110, \GH, \T2
866        vpshufd         $0b01001110, \HK, \T3
867        vpxor           \GH     , \T2, \T2      # T2 = (a1+a0)
868        vpxor           \HK     , \T3, \T3      # T3 = (b1+b0)
869
870        vpclmulqdq      $0x11, \HK, \GH, \T1    # T1 = a1*b1
871        vpclmulqdq      $0x00, \HK, \GH, \GH    # GH = a0*b0
872        vpclmulqdq      $0x00, \T3, \T2, \T2    # T2 = (a1+a0)*(b1+b0)
873        vpxor           \GH, \T2,\T2
874        vpxor           \T1, \T2,\T2            # T2 = a0*b1+a1*b0
875
876        vpslldq         $8, \T2,\T3             # shift-L T3 2 DWs
877        vpsrldq         $8, \T2,\T2             # shift-R T2 2 DWs
878        vpxor           \T3, \GH, \GH
879        vpxor           \T2, \T1, \T1           # <T1:GH> = GH x HK
880
881        #first phase of the reduction
882        vpslld  $31, \GH, \T2                   # packed right shifting << 31
883        vpslld  $30, \GH, \T3                   # packed right shifting shift << 30
884        vpslld  $25, \GH, \T4                   # packed right shifting shift << 25
885
886        vpxor   \T3, \T2, \T2                   # xor the shifted versions
887        vpxor   \T4, \T2, \T2
888
889        vpsrldq $4, \T2, \T5                    # shift-R T5 1 DW
890
891        vpslldq $12, \T2, \T2                   # shift-L T2 3 DWs
892        vpxor   \T2, \GH, \GH                   # first phase of the reduction complete
893
894        #second phase of the reduction
895
896        vpsrld  $1,\GH, \T2                     # packed left shifting >> 1
897        vpsrld  $2,\GH, \T3                     # packed left shifting >> 2
898        vpsrld  $7,\GH, \T4                     # packed left shifting >> 7
899        vpxor   \T3, \T2, \T2                   # xor the shifted versions
900        vpxor   \T4, \T2, \T2
901
902        vpxor   \T5, \T2, \T2
903        vpxor   \T2, \GH, \GH
904        vpxor   \T1, \GH, \GH                   # the result is in GH
905
906
907.endm
908
909.macro PRECOMPUTE_AVX HK T1 T2 T3 T4 T5 T6
910
911        # Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
912        vmovdqa  \HK, \T5
913
914        vpshufd  $0b01001110, \T5, \T1
915        vpxor    \T5, \T1, \T1
916        vmovdqu  \T1, HashKey_k(arg2)
917
918        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^2<<1 mod poly
919        vmovdqu  \T5, HashKey_2(arg2)                    #  [HashKey_2] = HashKey^2<<1 mod poly
920        vpshufd  $0b01001110, \T5, \T1
921        vpxor    \T5, \T1, \T1
922        vmovdqu  \T1, HashKey_2_k(arg2)
923
924        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^3<<1 mod poly
925        vmovdqu  \T5, HashKey_3(arg2)
926        vpshufd  $0b01001110, \T5, \T1
927        vpxor    \T5, \T1, \T1
928        vmovdqu  \T1, HashKey_3_k(arg2)
929
930        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^4<<1 mod poly
931        vmovdqu  \T5, HashKey_4(arg2)
932        vpshufd  $0b01001110, \T5, \T1
933        vpxor    \T5, \T1, \T1
934        vmovdqu  \T1, HashKey_4_k(arg2)
935
936        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^5<<1 mod poly
937        vmovdqu  \T5, HashKey_5(arg2)
938        vpshufd  $0b01001110, \T5, \T1
939        vpxor    \T5, \T1, \T1
940        vmovdqu  \T1, HashKey_5_k(arg2)
941
942        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^6<<1 mod poly
943        vmovdqu  \T5, HashKey_6(arg2)
944        vpshufd  $0b01001110, \T5, \T1
945        vpxor    \T5, \T1, \T1
946        vmovdqu  \T1, HashKey_6_k(arg2)
947
948        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^7<<1 mod poly
949        vmovdqu  \T5, HashKey_7(arg2)
950        vpshufd  $0b01001110, \T5, \T1
951        vpxor    \T5, \T1, \T1
952        vmovdqu  \T1, HashKey_7_k(arg2)
953
954        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^8<<1 mod poly
955        vmovdqu  \T5, HashKey_8(arg2)
956        vpshufd  $0b01001110, \T5, \T1
957        vpxor    \T5, \T1, \T1
958        vmovdqu  \T1, HashKey_8_k(arg2)
959
960.endm
961
962## if a = number of total plaintext bytes
963## b = floor(a/16)
964## num_initial_blocks = b mod 4#
965## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext
966## r10, r11, r12, rax are clobbered
967## arg1, arg2, arg3, arg4 are used as pointers only, not modified
968
969.macro INITIAL_BLOCKS_AVX REP num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC
970	i = (8-\num_initial_blocks)
971	setreg
972        vmovdqu AadHash(arg2), reg_i
973
974	# start AES for num_initial_blocks blocks
975	vmovdqu CurCount(arg2), \CTR
976
977	i = (9-\num_initial_blocks)
978	setreg
979.rep \num_initial_blocks
980                vpaddd  ONE(%rip), \CTR, \CTR		# INCR Y0
981                vmovdqa \CTR, reg_i
982                vpshufb SHUF_MASK(%rip), reg_i, reg_i   # perform a 16Byte swap
983	i = (i+1)
984	setreg
985.endr
986
987	vmovdqa  (arg1), \T_key
988	i = (9-\num_initial_blocks)
989	setreg
990.rep \num_initial_blocks
991                vpxor   \T_key, reg_i, reg_i
992	i = (i+1)
993	setreg
994.endr
995
996       j = 1
997       setreg
998.rep \REP
999       vmovdqa  16*j(arg1), \T_key
1000	i = (9-\num_initial_blocks)
1001	setreg
1002.rep \num_initial_blocks
1003        vaesenc \T_key, reg_i, reg_i
1004	i = (i+1)
1005	setreg
1006.endr
1007
1008       j = (j+1)
1009       setreg
1010.endr
1011
1012	vmovdqa  16*j(arg1), \T_key
1013	i = (9-\num_initial_blocks)
1014	setreg
1015.rep \num_initial_blocks
1016        vaesenclast      \T_key, reg_i, reg_i
1017	i = (i+1)
1018	setreg
1019.endr
1020
1021	i = (9-\num_initial_blocks)
1022	setreg
1023.rep \num_initial_blocks
1024                vmovdqu (arg4, %r11), \T1
1025                vpxor   \T1, reg_i, reg_i
1026                vmovdqu reg_i, (arg3 , %r11)           # write back ciphertext for num_initial_blocks blocks
1027                add     $16, %r11
1028.if  \ENC_DEC == DEC
1029                vmovdqa \T1, reg_i
1030.endif
1031                vpshufb SHUF_MASK(%rip), reg_i, reg_i  # prepare ciphertext for GHASH computations
1032	i = (i+1)
1033	setreg
1034.endr
1035
1036
1037	i = (8-\num_initial_blocks)
1038	j = (9-\num_initial_blocks)
1039	setreg
1040
1041.rep \num_initial_blocks
1042        vpxor    reg_i, reg_j, reg_j
1043        GHASH_MUL_AVX       reg_j, \T2, \T1, \T3, \T4, \T5, \T6 # apply GHASH on num_initial_blocks blocks
1044	i = (i+1)
1045	j = (j+1)
1046	setreg
1047.endr
1048        # XMM8 has the combined result here
1049
1050        vmovdqa  \XMM8, TMP1(%rsp)
1051        vmovdqa  \XMM8, \T3
1052
1053        cmp     $128, %r13
1054        jl      .L_initial_blocks_done\@                  # no need for precomputed constants
1055
1056###############################################################################
1057# Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
1058                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1059                vmovdqa  \CTR, \XMM1
1060                vpshufb  SHUF_MASK(%rip), \XMM1, \XMM1  # perform a 16Byte swap
1061
1062                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1063                vmovdqa  \CTR, \XMM2
1064                vpshufb  SHUF_MASK(%rip), \XMM2, \XMM2  # perform a 16Byte swap
1065
1066                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1067                vmovdqa  \CTR, \XMM3
1068                vpshufb  SHUF_MASK(%rip), \XMM3, \XMM3  # perform a 16Byte swap
1069
1070                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1071                vmovdqa  \CTR, \XMM4
1072                vpshufb  SHUF_MASK(%rip), \XMM4, \XMM4  # perform a 16Byte swap
1073
1074                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1075                vmovdqa  \CTR, \XMM5
1076                vpshufb  SHUF_MASK(%rip), \XMM5, \XMM5  # perform a 16Byte swap
1077
1078                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1079                vmovdqa  \CTR, \XMM6
1080                vpshufb  SHUF_MASK(%rip), \XMM6, \XMM6  # perform a 16Byte swap
1081
1082                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1083                vmovdqa  \CTR, \XMM7
1084                vpshufb  SHUF_MASK(%rip), \XMM7, \XMM7  # perform a 16Byte swap
1085
1086                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1087                vmovdqa  \CTR, \XMM8
1088                vpshufb  SHUF_MASK(%rip), \XMM8, \XMM8  # perform a 16Byte swap
1089
1090                vmovdqa  (arg1), \T_key
1091                vpxor    \T_key, \XMM1, \XMM1
1092                vpxor    \T_key, \XMM2, \XMM2
1093                vpxor    \T_key, \XMM3, \XMM3
1094                vpxor    \T_key, \XMM4, \XMM4
1095                vpxor    \T_key, \XMM5, \XMM5
1096                vpxor    \T_key, \XMM6, \XMM6
1097                vpxor    \T_key, \XMM7, \XMM7
1098                vpxor    \T_key, \XMM8, \XMM8
1099
1100               i = 1
1101               setreg
1102.rep    \REP       # do REP rounds
1103                vmovdqa  16*i(arg1), \T_key
1104                vaesenc  \T_key, \XMM1, \XMM1
1105                vaesenc  \T_key, \XMM2, \XMM2
1106                vaesenc  \T_key, \XMM3, \XMM3
1107                vaesenc  \T_key, \XMM4, \XMM4
1108                vaesenc  \T_key, \XMM5, \XMM5
1109                vaesenc  \T_key, \XMM6, \XMM6
1110                vaesenc  \T_key, \XMM7, \XMM7
1111                vaesenc  \T_key, \XMM8, \XMM8
1112               i = (i+1)
1113               setreg
1114.endr
1115
1116                vmovdqa  16*i(arg1), \T_key
1117                vaesenclast  \T_key, \XMM1, \XMM1
1118                vaesenclast  \T_key, \XMM2, \XMM2
1119                vaesenclast  \T_key, \XMM3, \XMM3
1120                vaesenclast  \T_key, \XMM4, \XMM4
1121                vaesenclast  \T_key, \XMM5, \XMM5
1122                vaesenclast  \T_key, \XMM6, \XMM6
1123                vaesenclast  \T_key, \XMM7, \XMM7
1124                vaesenclast  \T_key, \XMM8, \XMM8
1125
1126                vmovdqu  (arg4, %r11), \T1
1127                vpxor    \T1, \XMM1, \XMM1
1128                vmovdqu  \XMM1, (arg3 , %r11)
1129                .if   \ENC_DEC == DEC
1130                vmovdqa  \T1, \XMM1
1131                .endif
1132
1133                vmovdqu  16*1(arg4, %r11), \T1
1134                vpxor    \T1, \XMM2, \XMM2
1135                vmovdqu  \XMM2, 16*1(arg3 , %r11)
1136                .if   \ENC_DEC == DEC
1137                vmovdqa  \T1, \XMM2
1138                .endif
1139
1140                vmovdqu  16*2(arg4, %r11), \T1
1141                vpxor    \T1, \XMM3, \XMM3
1142                vmovdqu  \XMM3, 16*2(arg3 , %r11)
1143                .if   \ENC_DEC == DEC
1144                vmovdqa  \T1, \XMM3
1145                .endif
1146
1147                vmovdqu  16*3(arg4, %r11), \T1
1148                vpxor    \T1, \XMM4, \XMM4
1149                vmovdqu  \XMM4, 16*3(arg3 , %r11)
1150                .if   \ENC_DEC == DEC
1151                vmovdqa  \T1, \XMM4
1152                .endif
1153
1154                vmovdqu  16*4(arg4, %r11), \T1
1155                vpxor    \T1, \XMM5, \XMM5
1156                vmovdqu  \XMM5, 16*4(arg3 , %r11)
1157                .if   \ENC_DEC == DEC
1158                vmovdqa  \T1, \XMM5
1159                .endif
1160
1161                vmovdqu  16*5(arg4, %r11), \T1
1162                vpxor    \T1, \XMM6, \XMM6
1163                vmovdqu  \XMM6, 16*5(arg3 , %r11)
1164                .if   \ENC_DEC == DEC
1165                vmovdqa  \T1, \XMM6
1166                .endif
1167
1168                vmovdqu  16*6(arg4, %r11), \T1
1169                vpxor    \T1, \XMM7, \XMM7
1170                vmovdqu  \XMM7, 16*6(arg3 , %r11)
1171                .if   \ENC_DEC == DEC
1172                vmovdqa  \T1, \XMM7
1173                .endif
1174
1175                vmovdqu  16*7(arg4, %r11), \T1
1176                vpxor    \T1, \XMM8, \XMM8
1177                vmovdqu  \XMM8, 16*7(arg3 , %r11)
1178                .if   \ENC_DEC == DEC
1179                vmovdqa  \T1, \XMM8
1180                .endif
1181
1182                add     $128, %r11
1183
1184                vpshufb  SHUF_MASK(%rip), \XMM1, \XMM1     # perform a 16Byte swap
1185                vpxor    TMP1(%rsp), \XMM1, \XMM1          # combine GHASHed value with the corresponding ciphertext
1186                vpshufb  SHUF_MASK(%rip), \XMM2, \XMM2     # perform a 16Byte swap
1187                vpshufb  SHUF_MASK(%rip), \XMM3, \XMM3     # perform a 16Byte swap
1188                vpshufb  SHUF_MASK(%rip), \XMM4, \XMM4     # perform a 16Byte swap
1189                vpshufb  SHUF_MASK(%rip), \XMM5, \XMM5     # perform a 16Byte swap
1190                vpshufb  SHUF_MASK(%rip), \XMM6, \XMM6     # perform a 16Byte swap
1191                vpshufb  SHUF_MASK(%rip), \XMM7, \XMM7     # perform a 16Byte swap
1192                vpshufb  SHUF_MASK(%rip), \XMM8, \XMM8     # perform a 16Byte swap
1193
1194###############################################################################
1195
1196.L_initial_blocks_done\@:
1197
1198.endm
1199
1200# encrypt 8 blocks at a time
1201# ghash the 8 previously encrypted ciphertext blocks
1202# arg1, arg2, arg3, arg4 are used as pointers only, not modified
1203# r11 is the data offset value
1204.macro GHASH_8_ENCRYPT_8_PARALLEL_AVX REP T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC
1205
1206        vmovdqa \XMM1, \T2
1207        vmovdqa \XMM2, TMP2(%rsp)
1208        vmovdqa \XMM3, TMP3(%rsp)
1209        vmovdqa \XMM4, TMP4(%rsp)
1210        vmovdqa \XMM5, TMP5(%rsp)
1211        vmovdqa \XMM6, TMP6(%rsp)
1212        vmovdqa \XMM7, TMP7(%rsp)
1213        vmovdqa \XMM8, TMP8(%rsp)
1214
1215.if \loop_idx == in_order
1216                vpaddd  ONE(%rip), \CTR, \XMM1           # INCR CNT
1217                vpaddd  ONE(%rip), \XMM1, \XMM2
1218                vpaddd  ONE(%rip), \XMM2, \XMM3
1219                vpaddd  ONE(%rip), \XMM3, \XMM4
1220                vpaddd  ONE(%rip), \XMM4, \XMM5
1221                vpaddd  ONE(%rip), \XMM5, \XMM6
1222                vpaddd  ONE(%rip), \XMM6, \XMM7
1223                vpaddd  ONE(%rip), \XMM7, \XMM8
1224                vmovdqa \XMM8, \CTR
1225
1226                vpshufb SHUF_MASK(%rip), \XMM1, \XMM1    # perform a 16Byte swap
1227                vpshufb SHUF_MASK(%rip), \XMM2, \XMM2    # perform a 16Byte swap
1228                vpshufb SHUF_MASK(%rip), \XMM3, \XMM3    # perform a 16Byte swap
1229                vpshufb SHUF_MASK(%rip), \XMM4, \XMM4    # perform a 16Byte swap
1230                vpshufb SHUF_MASK(%rip), \XMM5, \XMM5    # perform a 16Byte swap
1231                vpshufb SHUF_MASK(%rip), \XMM6, \XMM6    # perform a 16Byte swap
1232                vpshufb SHUF_MASK(%rip), \XMM7, \XMM7    # perform a 16Byte swap
1233                vpshufb SHUF_MASK(%rip), \XMM8, \XMM8    # perform a 16Byte swap
1234.else
1235                vpaddd  ONEf(%rip), \CTR, \XMM1           # INCR CNT
1236                vpaddd  ONEf(%rip), \XMM1, \XMM2
1237                vpaddd  ONEf(%rip), \XMM2, \XMM3
1238                vpaddd  ONEf(%rip), \XMM3, \XMM4
1239                vpaddd  ONEf(%rip), \XMM4, \XMM5
1240                vpaddd  ONEf(%rip), \XMM5, \XMM6
1241                vpaddd  ONEf(%rip), \XMM6, \XMM7
1242                vpaddd  ONEf(%rip), \XMM7, \XMM8
1243                vmovdqa \XMM8, \CTR
1244.endif
1245
1246
1247        #######################################################################
1248
1249                vmovdqu (arg1), \T1
1250                vpxor   \T1, \XMM1, \XMM1
1251                vpxor   \T1, \XMM2, \XMM2
1252                vpxor   \T1, \XMM3, \XMM3
1253                vpxor   \T1, \XMM4, \XMM4
1254                vpxor   \T1, \XMM5, \XMM5
1255                vpxor   \T1, \XMM6, \XMM6
1256                vpxor   \T1, \XMM7, \XMM7
1257                vpxor   \T1, \XMM8, \XMM8
1258
1259        #######################################################################
1260
1261
1262
1263
1264
1265                vmovdqu 16*1(arg1), \T1
1266                vaesenc \T1, \XMM1, \XMM1
1267                vaesenc \T1, \XMM2, \XMM2
1268                vaesenc \T1, \XMM3, \XMM3
1269                vaesenc \T1, \XMM4, \XMM4
1270                vaesenc \T1, \XMM5, \XMM5
1271                vaesenc \T1, \XMM6, \XMM6
1272                vaesenc \T1, \XMM7, \XMM7
1273                vaesenc \T1, \XMM8, \XMM8
1274
1275                vmovdqu 16*2(arg1), \T1
1276                vaesenc \T1, \XMM1, \XMM1
1277                vaesenc \T1, \XMM2, \XMM2
1278                vaesenc \T1, \XMM3, \XMM3
1279                vaesenc \T1, \XMM4, \XMM4
1280                vaesenc \T1, \XMM5, \XMM5
1281                vaesenc \T1, \XMM6, \XMM6
1282                vaesenc \T1, \XMM7, \XMM7
1283                vaesenc \T1, \XMM8, \XMM8
1284
1285
1286        #######################################################################
1287
1288        vmovdqu         HashKey_8(arg2), \T5
1289        vpclmulqdq      $0x11, \T5, \T2, \T4             # T4 = a1*b1
1290        vpclmulqdq      $0x00, \T5, \T2, \T7             # T7 = a0*b0
1291
1292        vpshufd         $0b01001110, \T2, \T6
1293        vpxor           \T2, \T6, \T6
1294
1295        vmovdqu         HashKey_8_k(arg2), \T5
1296        vpclmulqdq      $0x00, \T5, \T6, \T6
1297
1298                vmovdqu 16*3(arg1), \T1
1299                vaesenc \T1, \XMM1, \XMM1
1300                vaesenc \T1, \XMM2, \XMM2
1301                vaesenc \T1, \XMM3, \XMM3
1302                vaesenc \T1, \XMM4, \XMM4
1303                vaesenc \T1, \XMM5, \XMM5
1304                vaesenc \T1, \XMM6, \XMM6
1305                vaesenc \T1, \XMM7, \XMM7
1306                vaesenc \T1, \XMM8, \XMM8
1307
1308        vmovdqa         TMP2(%rsp), \T1
1309        vmovdqu         HashKey_7(arg2), \T5
1310        vpclmulqdq      $0x11, \T5, \T1, \T3
1311        vpxor           \T3, \T4, \T4
1312        vpclmulqdq      $0x00, \T5, \T1, \T3
1313        vpxor           \T3, \T7, \T7
1314
1315        vpshufd         $0b01001110, \T1, \T3
1316        vpxor           \T1, \T3, \T3
1317        vmovdqu         HashKey_7_k(arg2), \T5
1318        vpclmulqdq      $0x10, \T5, \T3, \T3
1319        vpxor           \T3, \T6, \T6
1320
1321                vmovdqu 16*4(arg1), \T1
1322                vaesenc \T1, \XMM1, \XMM1
1323                vaesenc \T1, \XMM2, \XMM2
1324                vaesenc \T1, \XMM3, \XMM3
1325                vaesenc \T1, \XMM4, \XMM4
1326                vaesenc \T1, \XMM5, \XMM5
1327                vaesenc \T1, \XMM6, \XMM6
1328                vaesenc \T1, \XMM7, \XMM7
1329                vaesenc \T1, \XMM8, \XMM8
1330
1331        #######################################################################
1332
1333        vmovdqa         TMP3(%rsp), \T1
1334        vmovdqu         HashKey_6(arg2), \T5
1335        vpclmulqdq      $0x11, \T5, \T1, \T3
1336        vpxor           \T3, \T4, \T4
1337        vpclmulqdq      $0x00, \T5, \T1, \T3
1338        vpxor           \T3, \T7, \T7
1339
1340        vpshufd         $0b01001110, \T1, \T3
1341        vpxor           \T1, \T3, \T3
1342        vmovdqu         HashKey_6_k(arg2), \T5
1343        vpclmulqdq      $0x10, \T5, \T3, \T3
1344        vpxor           \T3, \T6, \T6
1345
1346                vmovdqu 16*5(arg1), \T1
1347                vaesenc \T1, \XMM1, \XMM1
1348                vaesenc \T1, \XMM2, \XMM2
1349                vaesenc \T1, \XMM3, \XMM3
1350                vaesenc \T1, \XMM4, \XMM4
1351                vaesenc \T1, \XMM5, \XMM5
1352                vaesenc \T1, \XMM6, \XMM6
1353                vaesenc \T1, \XMM7, \XMM7
1354                vaesenc \T1, \XMM8, \XMM8
1355
1356        vmovdqa         TMP4(%rsp), \T1
1357        vmovdqu         HashKey_5(arg2), \T5
1358        vpclmulqdq      $0x11, \T5, \T1, \T3
1359        vpxor           \T3, \T4, \T4
1360        vpclmulqdq      $0x00, \T5, \T1, \T3
1361        vpxor           \T3, \T7, \T7
1362
1363        vpshufd         $0b01001110, \T1, \T3
1364        vpxor           \T1, \T3, \T3
1365        vmovdqu         HashKey_5_k(arg2), \T5
1366        vpclmulqdq      $0x10, \T5, \T3, \T3
1367        vpxor           \T3, \T6, \T6
1368
1369                vmovdqu 16*6(arg1), \T1
1370                vaesenc \T1, \XMM1, \XMM1
1371                vaesenc \T1, \XMM2, \XMM2
1372                vaesenc \T1, \XMM3, \XMM3
1373                vaesenc \T1, \XMM4, \XMM4
1374                vaesenc \T1, \XMM5, \XMM5
1375                vaesenc \T1, \XMM6, \XMM6
1376                vaesenc \T1, \XMM7, \XMM7
1377                vaesenc \T1, \XMM8, \XMM8
1378
1379
1380        vmovdqa         TMP5(%rsp), \T1
1381        vmovdqu         HashKey_4(arg2), \T5
1382        vpclmulqdq      $0x11, \T5, \T1, \T3
1383        vpxor           \T3, \T4, \T4
1384        vpclmulqdq      $0x00, \T5, \T1, \T3
1385        vpxor           \T3, \T7, \T7
1386
1387        vpshufd         $0b01001110, \T1, \T3
1388        vpxor           \T1, \T3, \T3
1389        vmovdqu         HashKey_4_k(arg2), \T5
1390        vpclmulqdq      $0x10, \T5, \T3, \T3
1391        vpxor           \T3, \T6, \T6
1392
1393                vmovdqu 16*7(arg1), \T1
1394                vaesenc \T1, \XMM1, \XMM1
1395                vaesenc \T1, \XMM2, \XMM2
1396                vaesenc \T1, \XMM3, \XMM3
1397                vaesenc \T1, \XMM4, \XMM4
1398                vaesenc \T1, \XMM5, \XMM5
1399                vaesenc \T1, \XMM6, \XMM6
1400                vaesenc \T1, \XMM7, \XMM7
1401                vaesenc \T1, \XMM8, \XMM8
1402
1403        vmovdqa         TMP6(%rsp), \T1
1404        vmovdqu         HashKey_3(arg2), \T5
1405        vpclmulqdq      $0x11, \T5, \T1, \T3
1406        vpxor           \T3, \T4, \T4
1407        vpclmulqdq      $0x00, \T5, \T1, \T3
1408        vpxor           \T3, \T7, \T7
1409
1410        vpshufd         $0b01001110, \T1, \T3
1411        vpxor           \T1, \T3, \T3
1412        vmovdqu         HashKey_3_k(arg2), \T5
1413        vpclmulqdq      $0x10, \T5, \T3, \T3
1414        vpxor           \T3, \T6, \T6
1415
1416
1417                vmovdqu 16*8(arg1), \T1
1418                vaesenc \T1, \XMM1, \XMM1
1419                vaesenc \T1, \XMM2, \XMM2
1420                vaesenc \T1, \XMM3, \XMM3
1421                vaesenc \T1, \XMM4, \XMM4
1422                vaesenc \T1, \XMM5, \XMM5
1423                vaesenc \T1, \XMM6, \XMM6
1424                vaesenc \T1, \XMM7, \XMM7
1425                vaesenc \T1, \XMM8, \XMM8
1426
1427        vmovdqa         TMP7(%rsp), \T1
1428        vmovdqu         HashKey_2(arg2), \T5
1429        vpclmulqdq      $0x11, \T5, \T1, \T3
1430        vpxor           \T3, \T4, \T4
1431        vpclmulqdq      $0x00, \T5, \T1, \T3
1432        vpxor           \T3, \T7, \T7
1433
1434        vpshufd         $0b01001110, \T1, \T3
1435        vpxor           \T1, \T3, \T3
1436        vmovdqu         HashKey_2_k(arg2), \T5
1437        vpclmulqdq      $0x10, \T5, \T3, \T3
1438        vpxor           \T3, \T6, \T6
1439
1440        #######################################################################
1441
1442                vmovdqu 16*9(arg1), \T5
1443                vaesenc \T5, \XMM1, \XMM1
1444                vaesenc \T5, \XMM2, \XMM2
1445                vaesenc \T5, \XMM3, \XMM3
1446                vaesenc \T5, \XMM4, \XMM4
1447                vaesenc \T5, \XMM5, \XMM5
1448                vaesenc \T5, \XMM6, \XMM6
1449                vaesenc \T5, \XMM7, \XMM7
1450                vaesenc \T5, \XMM8, \XMM8
1451
1452        vmovdqa         TMP8(%rsp), \T1
1453        vmovdqu         HashKey(arg2), \T5
1454        vpclmulqdq      $0x11, \T5, \T1, \T3
1455        vpxor           \T3, \T4, \T4
1456        vpclmulqdq      $0x00, \T5, \T1, \T3
1457        vpxor           \T3, \T7, \T7
1458
1459        vpshufd         $0b01001110, \T1, \T3
1460        vpxor           \T1, \T3, \T3
1461        vmovdqu         HashKey_k(arg2), \T5
1462        vpclmulqdq      $0x10, \T5, \T3, \T3
1463        vpxor           \T3, \T6, \T6
1464
1465        vpxor           \T4, \T6, \T6
1466        vpxor           \T7, \T6, \T6
1467
1468                vmovdqu 16*10(arg1), \T5
1469
1470        i = 11
1471        setreg
1472.rep (\REP-9)
1473
1474        vaesenc \T5, \XMM1, \XMM1
1475        vaesenc \T5, \XMM2, \XMM2
1476        vaesenc \T5, \XMM3, \XMM3
1477        vaesenc \T5, \XMM4, \XMM4
1478        vaesenc \T5, \XMM5, \XMM5
1479        vaesenc \T5, \XMM6, \XMM6
1480        vaesenc \T5, \XMM7, \XMM7
1481        vaesenc \T5, \XMM8, \XMM8
1482
1483        vmovdqu 16*i(arg1), \T5
1484        i = i + 1
1485        setreg
1486.endr
1487
1488	i = 0
1489	j = 1
1490	setreg
1491.rep 8
1492		vpxor	16*i(arg4, %r11), \T5, \T2
1493                .if \ENC_DEC == ENC
1494                vaesenclast     \T2, reg_j, reg_j
1495                .else
1496                vaesenclast     \T2, reg_j, \T3
1497                vmovdqu 16*i(arg4, %r11), reg_j
1498                vmovdqu \T3, 16*i(arg3, %r11)
1499                .endif
1500	i = (i+1)
1501	j = (j+1)
1502	setreg
1503.endr
1504	#######################################################################
1505
1506
1507	vpslldq	$8, \T6, \T3				# shift-L T3 2 DWs
1508	vpsrldq	$8, \T6, \T6				# shift-R T2 2 DWs
1509	vpxor	\T3, \T7, \T7
1510	vpxor	\T4, \T6, \T6				# accumulate the results in T6:T7
1511
1512
1513
1514	#######################################################################
1515	#first phase of the reduction
1516	#######################################################################
1517        vpslld  $31, \T7, \T2                           # packed right shifting << 31
1518        vpslld  $30, \T7, \T3                           # packed right shifting shift << 30
1519        vpslld  $25, \T7, \T4                           # packed right shifting shift << 25
1520
1521        vpxor   \T3, \T2, \T2                           # xor the shifted versions
1522        vpxor   \T4, \T2, \T2
1523
1524        vpsrldq $4, \T2, \T1                            # shift-R T1 1 DW
1525
1526        vpslldq $12, \T2, \T2                           # shift-L T2 3 DWs
1527        vpxor   \T2, \T7, \T7                           # first phase of the reduction complete
1528	#######################################################################
1529                .if \ENC_DEC == ENC
1530		vmovdqu	 \XMM1,	16*0(arg3,%r11)		# Write to the Ciphertext buffer
1531		vmovdqu	 \XMM2,	16*1(arg3,%r11)		# Write to the Ciphertext buffer
1532		vmovdqu	 \XMM3,	16*2(arg3,%r11)		# Write to the Ciphertext buffer
1533		vmovdqu	 \XMM4,	16*3(arg3,%r11)		# Write to the Ciphertext buffer
1534		vmovdqu	 \XMM5,	16*4(arg3,%r11)		# Write to the Ciphertext buffer
1535		vmovdqu	 \XMM6,	16*5(arg3,%r11)		# Write to the Ciphertext buffer
1536		vmovdqu	 \XMM7,	16*6(arg3,%r11)		# Write to the Ciphertext buffer
1537		vmovdqu	 \XMM8,	16*7(arg3,%r11)		# Write to the Ciphertext buffer
1538                .endif
1539
1540	#######################################################################
1541	#second phase of the reduction
1542        vpsrld  $1, \T7, \T2                            # packed left shifting >> 1
1543        vpsrld  $2, \T7, \T3                            # packed left shifting >> 2
1544        vpsrld  $7, \T7, \T4                            # packed left shifting >> 7
1545        vpxor   \T3, \T2, \T2                           # xor the shifted versions
1546        vpxor   \T4, \T2, \T2
1547
1548        vpxor   \T1, \T2, \T2
1549        vpxor   \T2, \T7, \T7
1550        vpxor   \T7, \T6, \T6                           # the result is in T6
1551	#######################################################################
1552
1553		vpshufb	SHUF_MASK(%rip), \XMM1, \XMM1	# perform a 16Byte swap
1554		vpshufb	SHUF_MASK(%rip), \XMM2, \XMM2	# perform a 16Byte swap
1555		vpshufb	SHUF_MASK(%rip), \XMM3, \XMM3	# perform a 16Byte swap
1556		vpshufb	SHUF_MASK(%rip), \XMM4, \XMM4	# perform a 16Byte swap
1557		vpshufb	SHUF_MASK(%rip), \XMM5, \XMM5	# perform a 16Byte swap
1558		vpshufb	SHUF_MASK(%rip), \XMM6, \XMM6	# perform a 16Byte swap
1559		vpshufb	SHUF_MASK(%rip), \XMM7, \XMM7	# perform a 16Byte swap
1560		vpshufb	SHUF_MASK(%rip), \XMM8, \XMM8	# perform a 16Byte swap
1561
1562
1563	vpxor	\T6, \XMM1, \XMM1
1564
1565
1566
1567.endm
1568
1569
1570# GHASH the last 4 ciphertext blocks.
1571.macro  GHASH_LAST_8_AVX T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8
1572
1573        ## Karatsuba Method
1574
1575
1576        vpshufd         $0b01001110, \XMM1, \T2
1577        vpxor           \XMM1, \T2, \T2
1578        vmovdqu         HashKey_8(arg2), \T5
1579        vpclmulqdq      $0x11, \T5, \XMM1, \T6
1580        vpclmulqdq      $0x00, \T5, \XMM1, \T7
1581
1582        vmovdqu         HashKey_8_k(arg2), \T3
1583        vpclmulqdq      $0x00, \T3, \T2, \XMM1
1584
1585        ######################
1586
1587        vpshufd         $0b01001110, \XMM2, \T2
1588        vpxor           \XMM2, \T2, \T2
1589        vmovdqu         HashKey_7(arg2), \T5
1590        vpclmulqdq      $0x11, \T5, \XMM2, \T4
1591        vpxor           \T4, \T6, \T6
1592
1593        vpclmulqdq      $0x00, \T5, \XMM2, \T4
1594        vpxor           \T4, \T7, \T7
1595
1596        vmovdqu         HashKey_7_k(arg2), \T3
1597        vpclmulqdq      $0x00, \T3, \T2, \T2
1598        vpxor           \T2, \XMM1, \XMM1
1599
1600        ######################
1601
1602        vpshufd         $0b01001110, \XMM3, \T2
1603        vpxor           \XMM3, \T2, \T2
1604        vmovdqu         HashKey_6(arg2), \T5
1605        vpclmulqdq      $0x11, \T5, \XMM3, \T4
1606        vpxor           \T4, \T6, \T6
1607
1608        vpclmulqdq      $0x00, \T5, \XMM3, \T4
1609        vpxor           \T4, \T7, \T7
1610
1611        vmovdqu         HashKey_6_k(arg2), \T3
1612        vpclmulqdq      $0x00, \T3, \T2, \T2
1613        vpxor           \T2, \XMM1, \XMM1
1614
1615        ######################
1616
1617        vpshufd         $0b01001110, \XMM4, \T2
1618        vpxor           \XMM4, \T2, \T2
1619        vmovdqu         HashKey_5(arg2), \T5
1620        vpclmulqdq      $0x11, \T5, \XMM4, \T4
1621        vpxor           \T4, \T6, \T6
1622
1623        vpclmulqdq      $0x00, \T5, \XMM4, \T4
1624        vpxor           \T4, \T7, \T7
1625
1626        vmovdqu         HashKey_5_k(arg2), \T3
1627        vpclmulqdq      $0x00, \T3, \T2, \T2
1628        vpxor           \T2, \XMM1, \XMM1
1629
1630        ######################
1631
1632        vpshufd         $0b01001110, \XMM5, \T2
1633        vpxor           \XMM5, \T2, \T2
1634        vmovdqu         HashKey_4(arg2), \T5
1635        vpclmulqdq      $0x11, \T5, \XMM5, \T4
1636        vpxor           \T4, \T6, \T6
1637
1638        vpclmulqdq      $0x00, \T5, \XMM5, \T4
1639        vpxor           \T4, \T7, \T7
1640
1641        vmovdqu         HashKey_4_k(arg2), \T3
1642        vpclmulqdq      $0x00, \T3, \T2, \T2
1643        vpxor           \T2, \XMM1, \XMM1
1644
1645        ######################
1646
1647        vpshufd         $0b01001110, \XMM6, \T2
1648        vpxor           \XMM6, \T2, \T2
1649        vmovdqu         HashKey_3(arg2), \T5
1650        vpclmulqdq      $0x11, \T5, \XMM6, \T4
1651        vpxor           \T4, \T6, \T6
1652
1653        vpclmulqdq      $0x00, \T5, \XMM6, \T4
1654        vpxor           \T4, \T7, \T7
1655
1656        vmovdqu         HashKey_3_k(arg2), \T3
1657        vpclmulqdq      $0x00, \T3, \T2, \T2
1658        vpxor           \T2, \XMM1, \XMM1
1659
1660        ######################
1661
1662        vpshufd         $0b01001110, \XMM7, \T2
1663        vpxor           \XMM7, \T2, \T2
1664        vmovdqu         HashKey_2(arg2), \T5
1665        vpclmulqdq      $0x11, \T5, \XMM7, \T4
1666        vpxor           \T4, \T6, \T6
1667
1668        vpclmulqdq      $0x00, \T5, \XMM7, \T4
1669        vpxor           \T4, \T7, \T7
1670
1671        vmovdqu         HashKey_2_k(arg2), \T3
1672        vpclmulqdq      $0x00, \T3, \T2, \T2
1673        vpxor           \T2, \XMM1, \XMM1
1674
1675        ######################
1676
1677        vpshufd         $0b01001110, \XMM8, \T2
1678        vpxor           \XMM8, \T2, \T2
1679        vmovdqu         HashKey(arg2), \T5
1680        vpclmulqdq      $0x11, \T5, \XMM8, \T4
1681        vpxor           \T4, \T6, \T6
1682
1683        vpclmulqdq      $0x00, \T5, \XMM8, \T4
1684        vpxor           \T4, \T7, \T7
1685
1686        vmovdqu         HashKey_k(arg2), \T3
1687        vpclmulqdq      $0x00, \T3, \T2, \T2
1688
1689        vpxor           \T2, \XMM1, \XMM1
1690        vpxor           \T6, \XMM1, \XMM1
1691        vpxor           \T7, \XMM1, \T2
1692
1693
1694
1695
1696        vpslldq $8, \T2, \T4
1697        vpsrldq $8, \T2, \T2
1698
1699        vpxor   \T4, \T7, \T7
1700        vpxor   \T2, \T6, \T6   # <T6:T7> holds the result of
1701				# the accumulated carry-less multiplications
1702
1703        #######################################################################
1704        #first phase of the reduction
1705        vpslld  $31, \T7, \T2   # packed right shifting << 31
1706        vpslld  $30, \T7, \T3   # packed right shifting shift << 30
1707        vpslld  $25, \T7, \T4   # packed right shifting shift << 25
1708
1709        vpxor   \T3, \T2, \T2   # xor the shifted versions
1710        vpxor   \T4, \T2, \T2
1711
1712        vpsrldq $4, \T2, \T1    # shift-R T1 1 DW
1713
1714        vpslldq $12, \T2, \T2   # shift-L T2 3 DWs
1715        vpxor   \T2, \T7, \T7   # first phase of the reduction complete
1716        #######################################################################
1717
1718
1719        #second phase of the reduction
1720        vpsrld  $1, \T7, \T2    # packed left shifting >> 1
1721        vpsrld  $2, \T7, \T3    # packed left shifting >> 2
1722        vpsrld  $7, \T7, \T4    # packed left shifting >> 7
1723        vpxor   \T3, \T2, \T2   # xor the shifted versions
1724        vpxor   \T4, \T2, \T2
1725
1726        vpxor   \T1, \T2, \T2
1727        vpxor   \T2, \T7, \T7
1728        vpxor   \T7, \T6, \T6   # the result is in T6
1729
1730.endm
1731
1732#############################################################
1733#void   aesni_gcm_precomp_avx_gen2
1734#        (gcm_data     *my_ctx_data,
1735#         gcm_context_data *data,
1736#        u8     *hash_subkey# /* H, the Hash sub key input. Data starts on a 16-byte boundary. */
1737#        u8      *iv, /* Pre-counter block j0: 4 byte salt
1738#			(from Security Association) concatenated with 8 byte
1739#			Initialisation Vector (from IPSec ESP Payload)
1740#			concatenated with 0x00000001. 16-byte aligned pointer. */
1741#        const   u8 *aad, /* Additional Authentication Data (AAD)*/
1742#        u64     aad_len) /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
1743#############################################################
1744SYM_FUNC_START(aesni_gcm_init_avx_gen2)
1745        FUNC_SAVE
1746        INIT GHASH_MUL_AVX, PRECOMPUTE_AVX
1747        FUNC_RESTORE
1748        RET
1749SYM_FUNC_END(aesni_gcm_init_avx_gen2)
1750
1751###############################################################################
1752#void   aesni_gcm_enc_update_avx_gen2(
1753#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
1754#        gcm_context_data *data,
1755#        u8      *out, /* Ciphertext output. Encrypt in-place is allowed.  */
1756#        const   u8 *in, /* Plaintext input */
1757#        u64     plaintext_len) /* Length of data in Bytes for encryption. */
1758###############################################################################
1759SYM_FUNC_START(aesni_gcm_enc_update_avx_gen2)
1760        FUNC_SAVE
1761        mov     keysize, %eax
1762        cmp     $32, %eax
1763        je      key_256_enc_update
1764        cmp     $16, %eax
1765        je      key_128_enc_update
1766        # must be 192
1767        GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 11
1768        FUNC_RESTORE
1769        RET
1770key_128_enc_update:
1771        GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 9
1772        FUNC_RESTORE
1773        RET
1774key_256_enc_update:
1775        GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 13
1776        FUNC_RESTORE
1777        RET
1778SYM_FUNC_END(aesni_gcm_enc_update_avx_gen2)
1779
1780###############################################################################
1781#void   aesni_gcm_dec_update_avx_gen2(
1782#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
1783#        gcm_context_data *data,
1784#        u8      *out, /* Plaintext output. Decrypt in-place is allowed.  */
1785#        const   u8 *in, /* Ciphertext input */
1786#        u64     plaintext_len) /* Length of data in Bytes for encryption. */
1787###############################################################################
1788SYM_FUNC_START(aesni_gcm_dec_update_avx_gen2)
1789        FUNC_SAVE
1790        mov     keysize,%eax
1791        cmp     $32, %eax
1792        je      key_256_dec_update
1793        cmp     $16, %eax
1794        je      key_128_dec_update
1795        # must be 192
1796        GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 11
1797        FUNC_RESTORE
1798        RET
1799key_128_dec_update:
1800        GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 9
1801        FUNC_RESTORE
1802        RET
1803key_256_dec_update:
1804        GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 13
1805        FUNC_RESTORE
1806        RET
1807SYM_FUNC_END(aesni_gcm_dec_update_avx_gen2)
1808
1809###############################################################################
1810#void   aesni_gcm_finalize_avx_gen2(
1811#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
1812#        gcm_context_data *data,
1813#        u8      *auth_tag, /* Authenticated Tag output. */
1814#        u64     auth_tag_len)# /* Authenticated Tag Length in bytes.
1815#				Valid values are 16 (most likely), 12 or 8. */
1816###############################################################################
1817SYM_FUNC_START(aesni_gcm_finalize_avx_gen2)
1818        FUNC_SAVE
1819        mov	keysize,%eax
1820        cmp     $32, %eax
1821        je      key_256_finalize
1822        cmp     $16, %eax
1823        je      key_128_finalize
1824        # must be 192
1825        GCM_COMPLETE GHASH_MUL_AVX, 11, arg3, arg4
1826        FUNC_RESTORE
1827        RET
1828key_128_finalize:
1829        GCM_COMPLETE GHASH_MUL_AVX, 9, arg3, arg4
1830        FUNC_RESTORE
1831        RET
1832key_256_finalize:
1833        GCM_COMPLETE GHASH_MUL_AVX, 13, arg3, arg4
1834        FUNC_RESTORE
1835        RET
1836SYM_FUNC_END(aesni_gcm_finalize_avx_gen2)
1837
1838###############################################################################
1839# GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
1840# Input: A and B (128-bits each, bit-reflected)
1841# Output: C = A*B*x mod poly, (i.e. >>1 )
1842# To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
1843# GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
1844###############################################################################
1845.macro  GHASH_MUL_AVX2 GH HK T1 T2 T3 T4 T5
1846
1847        vpclmulqdq      $0x11,\HK,\GH,\T1      # T1 = a1*b1
1848        vpclmulqdq      $0x00,\HK,\GH,\T2      # T2 = a0*b0
1849        vpclmulqdq      $0x01,\HK,\GH,\T3      # T3 = a1*b0
1850        vpclmulqdq      $0x10,\HK,\GH,\GH      # GH = a0*b1
1851        vpxor           \T3, \GH, \GH
1852
1853
1854        vpsrldq         $8 , \GH, \T3          # shift-R GH 2 DWs
1855        vpslldq         $8 , \GH, \GH          # shift-L GH 2 DWs
1856
1857        vpxor           \T3, \T1, \T1
1858        vpxor           \T2, \GH, \GH
1859
1860        #######################################################################
1861        #first phase of the reduction
1862        vmovdqa         POLY2(%rip), \T3
1863
1864        vpclmulqdq      $0x01, \GH, \T3, \T2
1865        vpslldq         $8, \T2, \T2           # shift-L T2 2 DWs
1866
1867        vpxor           \T2, \GH, \GH          # first phase of the reduction complete
1868        #######################################################################
1869        #second phase of the reduction
1870        vpclmulqdq      $0x00, \GH, \T3, \T2
1871        vpsrldq         $4, \T2, \T2           # shift-R T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
1872
1873        vpclmulqdq      $0x10, \GH, \T3, \GH
1874        vpslldq         $4, \GH, \GH           # shift-L GH 1 DW (Shift-L 1-DW to obtain result with no shifts)
1875
1876        vpxor           \T2, \GH, \GH          # second phase of the reduction complete
1877        #######################################################################
1878        vpxor           \T1, \GH, \GH          # the result is in GH
1879
1880
1881.endm
1882
1883.macro PRECOMPUTE_AVX2 HK T1 T2 T3 T4 T5 T6
1884
1885        # Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
1886        vmovdqa  \HK, \T5
1887        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^2<<1 mod poly
1888        vmovdqu  \T5, HashKey_2(arg2)                       #  [HashKey_2] = HashKey^2<<1 mod poly
1889
1890        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^3<<1 mod poly
1891        vmovdqu  \T5, HashKey_3(arg2)
1892
1893        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^4<<1 mod poly
1894        vmovdqu  \T5, HashKey_4(arg2)
1895
1896        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^5<<1 mod poly
1897        vmovdqu  \T5, HashKey_5(arg2)
1898
1899        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^6<<1 mod poly
1900        vmovdqu  \T5, HashKey_6(arg2)
1901
1902        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^7<<1 mod poly
1903        vmovdqu  \T5, HashKey_7(arg2)
1904
1905        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^8<<1 mod poly
1906        vmovdqu  \T5, HashKey_8(arg2)
1907
1908.endm
1909
1910## if a = number of total plaintext bytes
1911## b = floor(a/16)
1912## num_initial_blocks = b mod 4#
1913## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext
1914## r10, r11, r12, rax are clobbered
1915## arg1, arg2, arg3, arg4 are used as pointers only, not modified
1916
1917.macro INITIAL_BLOCKS_AVX2 REP num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC VER
1918	i = (8-\num_initial_blocks)
1919	setreg
1920	vmovdqu AadHash(arg2), reg_i
1921
1922	# start AES for num_initial_blocks blocks
1923	vmovdqu CurCount(arg2), \CTR
1924
1925	i = (9-\num_initial_blocks)
1926	setreg
1927.rep \num_initial_blocks
1928                vpaddd  ONE(%rip), \CTR, \CTR   # INCR Y0
1929                vmovdqa \CTR, reg_i
1930                vpshufb SHUF_MASK(%rip), reg_i, reg_i     # perform a 16Byte swap
1931	i = (i+1)
1932	setreg
1933.endr
1934
1935	vmovdqa  (arg1), \T_key
1936	i = (9-\num_initial_blocks)
1937	setreg
1938.rep \num_initial_blocks
1939                vpxor   \T_key, reg_i, reg_i
1940	i = (i+1)
1941	setreg
1942.endr
1943
1944	j = 1
1945	setreg
1946.rep \REP
1947	vmovdqa  16*j(arg1), \T_key
1948	i = (9-\num_initial_blocks)
1949	setreg
1950.rep \num_initial_blocks
1951        vaesenc \T_key, reg_i, reg_i
1952	i = (i+1)
1953	setreg
1954.endr
1955
1956	j = (j+1)
1957	setreg
1958.endr
1959
1960
1961	vmovdqa  16*j(arg1), \T_key
1962	i = (9-\num_initial_blocks)
1963	setreg
1964.rep \num_initial_blocks
1965        vaesenclast      \T_key, reg_i, reg_i
1966	i = (i+1)
1967	setreg
1968.endr
1969
1970	i = (9-\num_initial_blocks)
1971	setreg
1972.rep \num_initial_blocks
1973                vmovdqu (arg4, %r11), \T1
1974                vpxor   \T1, reg_i, reg_i
1975                vmovdqu reg_i, (arg3 , %r11)           # write back ciphertext for
1976						       # num_initial_blocks blocks
1977                add     $16, %r11
1978.if  \ENC_DEC == DEC
1979                vmovdqa \T1, reg_i
1980.endif
1981                vpshufb SHUF_MASK(%rip), reg_i, reg_i  # prepare ciphertext for GHASH computations
1982	i = (i+1)
1983	setreg
1984.endr
1985
1986
1987	i = (8-\num_initial_blocks)
1988	j = (9-\num_initial_blocks)
1989	setreg
1990
1991.rep \num_initial_blocks
1992        vpxor    reg_i, reg_j, reg_j
1993        GHASH_MUL_AVX2       reg_j, \T2, \T1, \T3, \T4, \T5, \T6  # apply GHASH on num_initial_blocks blocks
1994	i = (i+1)
1995	j = (j+1)
1996	setreg
1997.endr
1998        # XMM8 has the combined result here
1999
2000        vmovdqa  \XMM8, TMP1(%rsp)
2001        vmovdqa  \XMM8, \T3
2002
2003        cmp     $128, %r13
2004        jl      .L_initial_blocks_done\@                  # no need for precomputed constants
2005
2006###############################################################################
2007# Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
2008                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
2009                vmovdqa  \CTR, \XMM1
2010                vpshufb  SHUF_MASK(%rip), \XMM1, \XMM1  # perform a 16Byte swap
2011
2012                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
2013                vmovdqa  \CTR, \XMM2
2014                vpshufb  SHUF_MASK(%rip), \XMM2, \XMM2  # perform a 16Byte swap
2015
2016                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
2017                vmovdqa  \CTR, \XMM3
2018                vpshufb  SHUF_MASK(%rip), \XMM3, \XMM3  # perform a 16Byte swap
2019
2020                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
2021                vmovdqa  \CTR, \XMM4
2022                vpshufb  SHUF_MASK(%rip), \XMM4, \XMM4  # perform a 16Byte swap
2023
2024                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
2025                vmovdqa  \CTR, \XMM5
2026                vpshufb  SHUF_MASK(%rip), \XMM5, \XMM5  # perform a 16Byte swap
2027
2028                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
2029                vmovdqa  \CTR, \XMM6
2030                vpshufb  SHUF_MASK(%rip), \XMM6, \XMM6  # perform a 16Byte swap
2031
2032                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
2033                vmovdqa  \CTR, \XMM7
2034                vpshufb  SHUF_MASK(%rip), \XMM7, \XMM7  # perform a 16Byte swap
2035
2036                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
2037                vmovdqa  \CTR, \XMM8
2038                vpshufb  SHUF_MASK(%rip), \XMM8, \XMM8  # perform a 16Byte swap
2039
2040                vmovdqa  (arg1), \T_key
2041                vpxor    \T_key, \XMM1, \XMM1
2042                vpxor    \T_key, \XMM2, \XMM2
2043                vpxor    \T_key, \XMM3, \XMM3
2044                vpxor    \T_key, \XMM4, \XMM4
2045                vpxor    \T_key, \XMM5, \XMM5
2046                vpxor    \T_key, \XMM6, \XMM6
2047                vpxor    \T_key, \XMM7, \XMM7
2048                vpxor    \T_key, \XMM8, \XMM8
2049
2050		i = 1
2051		setreg
2052.rep    \REP       # do REP rounds
2053                vmovdqa  16*i(arg1), \T_key
2054                vaesenc  \T_key, \XMM1, \XMM1
2055                vaesenc  \T_key, \XMM2, \XMM2
2056                vaesenc  \T_key, \XMM3, \XMM3
2057                vaesenc  \T_key, \XMM4, \XMM4
2058                vaesenc  \T_key, \XMM5, \XMM5
2059                vaesenc  \T_key, \XMM6, \XMM6
2060                vaesenc  \T_key, \XMM7, \XMM7
2061                vaesenc  \T_key, \XMM8, \XMM8
2062		i = (i+1)
2063		setreg
2064.endr
2065
2066
2067                vmovdqa  16*i(arg1), \T_key
2068                vaesenclast  \T_key, \XMM1, \XMM1
2069                vaesenclast  \T_key, \XMM2, \XMM2
2070                vaesenclast  \T_key, \XMM3, \XMM3
2071                vaesenclast  \T_key, \XMM4, \XMM4
2072                vaesenclast  \T_key, \XMM5, \XMM5
2073                vaesenclast  \T_key, \XMM6, \XMM6
2074                vaesenclast  \T_key, \XMM7, \XMM7
2075                vaesenclast  \T_key, \XMM8, \XMM8
2076
2077                vmovdqu  (arg4, %r11), \T1
2078                vpxor    \T1, \XMM1, \XMM1
2079                vmovdqu  \XMM1, (arg3 , %r11)
2080                .if   \ENC_DEC == DEC
2081                vmovdqa  \T1, \XMM1
2082                .endif
2083
2084                vmovdqu  16*1(arg4, %r11), \T1
2085                vpxor    \T1, \XMM2, \XMM2
2086                vmovdqu  \XMM2, 16*1(arg3 , %r11)
2087                .if   \ENC_DEC == DEC
2088                vmovdqa  \T1, \XMM2
2089                .endif
2090
2091                vmovdqu  16*2(arg4, %r11), \T1
2092                vpxor    \T1, \XMM3, \XMM3
2093                vmovdqu  \XMM3, 16*2(arg3 , %r11)
2094                .if   \ENC_DEC == DEC
2095                vmovdqa  \T1, \XMM3
2096                .endif
2097
2098                vmovdqu  16*3(arg4, %r11), \T1
2099                vpxor    \T1, \XMM4, \XMM4
2100                vmovdqu  \XMM4, 16*3(arg3 , %r11)
2101                .if   \ENC_DEC == DEC
2102                vmovdqa  \T1, \XMM4
2103                .endif
2104
2105                vmovdqu  16*4(arg4, %r11), \T1
2106                vpxor    \T1, \XMM5, \XMM5
2107                vmovdqu  \XMM5, 16*4(arg3 , %r11)
2108                .if   \ENC_DEC == DEC
2109                vmovdqa  \T1, \XMM5
2110                .endif
2111
2112                vmovdqu  16*5(arg4, %r11), \T1
2113                vpxor    \T1, \XMM6, \XMM6
2114                vmovdqu  \XMM6, 16*5(arg3 , %r11)
2115                .if   \ENC_DEC == DEC
2116                vmovdqa  \T1, \XMM6
2117                .endif
2118
2119                vmovdqu  16*6(arg4, %r11), \T1
2120                vpxor    \T1, \XMM7, \XMM7
2121                vmovdqu  \XMM7, 16*6(arg3 , %r11)
2122                .if   \ENC_DEC == DEC
2123                vmovdqa  \T1, \XMM7
2124                .endif
2125
2126                vmovdqu  16*7(arg4, %r11), \T1
2127                vpxor    \T1, \XMM8, \XMM8
2128                vmovdqu  \XMM8, 16*7(arg3 , %r11)
2129                .if   \ENC_DEC == DEC
2130                vmovdqa  \T1, \XMM8
2131                .endif
2132
2133                add     $128, %r11
2134
2135                vpshufb  SHUF_MASK(%rip), \XMM1, \XMM1     # perform a 16Byte swap
2136                vpxor    TMP1(%rsp), \XMM1, \XMM1          # combine GHASHed value with
2137							   # the corresponding ciphertext
2138                vpshufb  SHUF_MASK(%rip), \XMM2, \XMM2     # perform a 16Byte swap
2139                vpshufb  SHUF_MASK(%rip), \XMM3, \XMM3     # perform a 16Byte swap
2140                vpshufb  SHUF_MASK(%rip), \XMM4, \XMM4     # perform a 16Byte swap
2141                vpshufb  SHUF_MASK(%rip), \XMM5, \XMM5     # perform a 16Byte swap
2142                vpshufb  SHUF_MASK(%rip), \XMM6, \XMM6     # perform a 16Byte swap
2143                vpshufb  SHUF_MASK(%rip), \XMM7, \XMM7     # perform a 16Byte swap
2144                vpshufb  SHUF_MASK(%rip), \XMM8, \XMM8     # perform a 16Byte swap
2145
2146###############################################################################
2147
2148.L_initial_blocks_done\@:
2149
2150
2151.endm
2152
2153
2154
2155# encrypt 8 blocks at a time
2156# ghash the 8 previously encrypted ciphertext blocks
2157# arg1, arg2, arg3, arg4 are used as pointers only, not modified
2158# r11 is the data offset value
2159.macro GHASH_8_ENCRYPT_8_PARALLEL_AVX2 REP T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC
2160
2161        vmovdqa \XMM1, \T2
2162        vmovdqa \XMM2, TMP2(%rsp)
2163        vmovdqa \XMM3, TMP3(%rsp)
2164        vmovdqa \XMM4, TMP4(%rsp)
2165        vmovdqa \XMM5, TMP5(%rsp)
2166        vmovdqa \XMM6, TMP6(%rsp)
2167        vmovdqa \XMM7, TMP7(%rsp)
2168        vmovdqa \XMM8, TMP8(%rsp)
2169
2170.if \loop_idx == in_order
2171                vpaddd  ONE(%rip), \CTR, \XMM1            # INCR CNT
2172                vpaddd  ONE(%rip), \XMM1, \XMM2
2173                vpaddd  ONE(%rip), \XMM2, \XMM3
2174                vpaddd  ONE(%rip), \XMM3, \XMM4
2175                vpaddd  ONE(%rip), \XMM4, \XMM5
2176                vpaddd  ONE(%rip), \XMM5, \XMM6
2177                vpaddd  ONE(%rip), \XMM6, \XMM7
2178                vpaddd  ONE(%rip), \XMM7, \XMM8
2179                vmovdqa \XMM8, \CTR
2180
2181                vpshufb SHUF_MASK(%rip), \XMM1, \XMM1     # perform a 16Byte swap
2182                vpshufb SHUF_MASK(%rip), \XMM2, \XMM2     # perform a 16Byte swap
2183                vpshufb SHUF_MASK(%rip), \XMM3, \XMM3     # perform a 16Byte swap
2184                vpshufb SHUF_MASK(%rip), \XMM4, \XMM4     # perform a 16Byte swap
2185                vpshufb SHUF_MASK(%rip), \XMM5, \XMM5     # perform a 16Byte swap
2186                vpshufb SHUF_MASK(%rip), \XMM6, \XMM6     # perform a 16Byte swap
2187                vpshufb SHUF_MASK(%rip), \XMM7, \XMM7     # perform a 16Byte swap
2188                vpshufb SHUF_MASK(%rip), \XMM8, \XMM8     # perform a 16Byte swap
2189.else
2190                vpaddd  ONEf(%rip), \CTR, \XMM1            # INCR CNT
2191                vpaddd  ONEf(%rip), \XMM1, \XMM2
2192                vpaddd  ONEf(%rip), \XMM2, \XMM3
2193                vpaddd  ONEf(%rip), \XMM3, \XMM4
2194                vpaddd  ONEf(%rip), \XMM4, \XMM5
2195                vpaddd  ONEf(%rip), \XMM5, \XMM6
2196                vpaddd  ONEf(%rip), \XMM6, \XMM7
2197                vpaddd  ONEf(%rip), \XMM7, \XMM8
2198                vmovdqa \XMM8, \CTR
2199.endif
2200
2201
2202        #######################################################################
2203
2204                vmovdqu (arg1), \T1
2205                vpxor   \T1, \XMM1, \XMM1
2206                vpxor   \T1, \XMM2, \XMM2
2207                vpxor   \T1, \XMM3, \XMM3
2208                vpxor   \T1, \XMM4, \XMM4
2209                vpxor   \T1, \XMM5, \XMM5
2210                vpxor   \T1, \XMM6, \XMM6
2211                vpxor   \T1, \XMM7, \XMM7
2212                vpxor   \T1, \XMM8, \XMM8
2213
2214        #######################################################################
2215
2216
2217
2218
2219
2220                vmovdqu 16*1(arg1), \T1
2221                vaesenc \T1, \XMM1, \XMM1
2222                vaesenc \T1, \XMM2, \XMM2
2223                vaesenc \T1, \XMM3, \XMM3
2224                vaesenc \T1, \XMM4, \XMM4
2225                vaesenc \T1, \XMM5, \XMM5
2226                vaesenc \T1, \XMM6, \XMM6
2227                vaesenc \T1, \XMM7, \XMM7
2228                vaesenc \T1, \XMM8, \XMM8
2229
2230                vmovdqu 16*2(arg1), \T1
2231                vaesenc \T1, \XMM1, \XMM1
2232                vaesenc \T1, \XMM2, \XMM2
2233                vaesenc \T1, \XMM3, \XMM3
2234                vaesenc \T1, \XMM4, \XMM4
2235                vaesenc \T1, \XMM5, \XMM5
2236                vaesenc \T1, \XMM6, \XMM6
2237                vaesenc \T1, \XMM7, \XMM7
2238                vaesenc \T1, \XMM8, \XMM8
2239
2240
2241        #######################################################################
2242
2243        vmovdqu         HashKey_8(arg2), \T5
2244        vpclmulqdq      $0x11, \T5, \T2, \T4              # T4 = a1*b1
2245        vpclmulqdq      $0x00, \T5, \T2, \T7              # T7 = a0*b0
2246        vpclmulqdq      $0x01, \T5, \T2, \T6              # T6 = a1*b0
2247        vpclmulqdq      $0x10, \T5, \T2, \T5              # T5 = a0*b1
2248        vpxor           \T5, \T6, \T6
2249
2250                vmovdqu 16*3(arg1), \T1
2251                vaesenc \T1, \XMM1, \XMM1
2252                vaesenc \T1, \XMM2, \XMM2
2253                vaesenc \T1, \XMM3, \XMM3
2254                vaesenc \T1, \XMM4, \XMM4
2255                vaesenc \T1, \XMM5, \XMM5
2256                vaesenc \T1, \XMM6, \XMM6
2257                vaesenc \T1, \XMM7, \XMM7
2258                vaesenc \T1, \XMM8, \XMM8
2259
2260        vmovdqa         TMP2(%rsp), \T1
2261        vmovdqu         HashKey_7(arg2), \T5
2262        vpclmulqdq      $0x11, \T5, \T1, \T3
2263        vpxor           \T3, \T4, \T4
2264
2265        vpclmulqdq      $0x00, \T5, \T1, \T3
2266        vpxor           \T3, \T7, \T7
2267
2268        vpclmulqdq      $0x01, \T5, \T1, \T3
2269        vpxor           \T3, \T6, \T6
2270
2271        vpclmulqdq      $0x10, \T5, \T1, \T3
2272        vpxor           \T3, \T6, \T6
2273
2274                vmovdqu 16*4(arg1), \T1
2275                vaesenc \T1, \XMM1, \XMM1
2276                vaesenc \T1, \XMM2, \XMM2
2277                vaesenc \T1, \XMM3, \XMM3
2278                vaesenc \T1, \XMM4, \XMM4
2279                vaesenc \T1, \XMM5, \XMM5
2280                vaesenc \T1, \XMM6, \XMM6
2281                vaesenc \T1, \XMM7, \XMM7
2282                vaesenc \T1, \XMM8, \XMM8
2283
2284        #######################################################################
2285
2286        vmovdqa         TMP3(%rsp), \T1
2287        vmovdqu         HashKey_6(arg2), \T5
2288        vpclmulqdq      $0x11, \T5, \T1, \T3
2289        vpxor           \T3, \T4, \T4
2290
2291        vpclmulqdq      $0x00, \T5, \T1, \T3
2292        vpxor           \T3, \T7, \T7
2293
2294        vpclmulqdq      $0x01, \T5, \T1, \T3
2295        vpxor           \T3, \T6, \T6
2296
2297        vpclmulqdq      $0x10, \T5, \T1, \T3
2298        vpxor           \T3, \T6, \T6
2299
2300                vmovdqu 16*5(arg1), \T1
2301                vaesenc \T1, \XMM1, \XMM1
2302                vaesenc \T1, \XMM2, \XMM2
2303                vaesenc \T1, \XMM3, \XMM3
2304                vaesenc \T1, \XMM4, \XMM4
2305                vaesenc \T1, \XMM5, \XMM5
2306                vaesenc \T1, \XMM6, \XMM6
2307                vaesenc \T1, \XMM7, \XMM7
2308                vaesenc \T1, \XMM8, \XMM8
2309
2310        vmovdqa         TMP4(%rsp), \T1
2311        vmovdqu         HashKey_5(arg2), \T5
2312        vpclmulqdq      $0x11, \T5, \T1, \T3
2313        vpxor           \T3, \T4, \T4
2314
2315        vpclmulqdq      $0x00, \T5, \T1, \T3
2316        vpxor           \T3, \T7, \T7
2317
2318        vpclmulqdq      $0x01, \T5, \T1, \T3
2319        vpxor           \T3, \T6, \T6
2320
2321        vpclmulqdq      $0x10, \T5, \T1, \T3
2322        vpxor           \T3, \T6, \T6
2323
2324                vmovdqu 16*6(arg1), \T1
2325                vaesenc \T1, \XMM1, \XMM1
2326                vaesenc \T1, \XMM2, \XMM2
2327                vaesenc \T1, \XMM3, \XMM3
2328                vaesenc \T1, \XMM4, \XMM4
2329                vaesenc \T1, \XMM5, \XMM5
2330                vaesenc \T1, \XMM6, \XMM6
2331                vaesenc \T1, \XMM7, \XMM7
2332                vaesenc \T1, \XMM8, \XMM8
2333
2334
2335        vmovdqa         TMP5(%rsp), \T1
2336        vmovdqu         HashKey_4(arg2), \T5
2337        vpclmulqdq      $0x11, \T5, \T1, \T3
2338        vpxor           \T3, \T4, \T4
2339
2340        vpclmulqdq      $0x00, \T5, \T1, \T3
2341        vpxor           \T3, \T7, \T7
2342
2343        vpclmulqdq      $0x01, \T5, \T1, \T3
2344        vpxor           \T3, \T6, \T6
2345
2346        vpclmulqdq      $0x10, \T5, \T1, \T3
2347        vpxor           \T3, \T6, \T6
2348
2349                vmovdqu 16*7(arg1), \T1
2350                vaesenc \T1, \XMM1, \XMM1
2351                vaesenc \T1, \XMM2, \XMM2
2352                vaesenc \T1, \XMM3, \XMM3
2353                vaesenc \T1, \XMM4, \XMM4
2354                vaesenc \T1, \XMM5, \XMM5
2355                vaesenc \T1, \XMM6, \XMM6
2356                vaesenc \T1, \XMM7, \XMM7
2357                vaesenc \T1, \XMM8, \XMM8
2358
2359        vmovdqa         TMP6(%rsp), \T1
2360        vmovdqu         HashKey_3(arg2), \T5
2361        vpclmulqdq      $0x11, \T5, \T1, \T3
2362        vpxor           \T3, \T4, \T4
2363
2364        vpclmulqdq      $0x00, \T5, \T1, \T3
2365        vpxor           \T3, \T7, \T7
2366
2367        vpclmulqdq      $0x01, \T5, \T1, \T3
2368        vpxor           \T3, \T6, \T6
2369
2370        vpclmulqdq      $0x10, \T5, \T1, \T3
2371        vpxor           \T3, \T6, \T6
2372
2373                vmovdqu 16*8(arg1), \T1
2374                vaesenc \T1, \XMM1, \XMM1
2375                vaesenc \T1, \XMM2, \XMM2
2376                vaesenc \T1, \XMM3, \XMM3
2377                vaesenc \T1, \XMM4, \XMM4
2378                vaesenc \T1, \XMM5, \XMM5
2379                vaesenc \T1, \XMM6, \XMM6
2380                vaesenc \T1, \XMM7, \XMM7
2381                vaesenc \T1, \XMM8, \XMM8
2382
2383        vmovdqa         TMP7(%rsp), \T1
2384        vmovdqu         HashKey_2(arg2), \T5
2385        vpclmulqdq      $0x11, \T5, \T1, \T3
2386        vpxor           \T3, \T4, \T4
2387
2388        vpclmulqdq      $0x00, \T5, \T1, \T3
2389        vpxor           \T3, \T7, \T7
2390
2391        vpclmulqdq      $0x01, \T5, \T1, \T3
2392        vpxor           \T3, \T6, \T6
2393
2394        vpclmulqdq      $0x10, \T5, \T1, \T3
2395        vpxor           \T3, \T6, \T6
2396
2397
2398        #######################################################################
2399
2400                vmovdqu 16*9(arg1), \T5
2401                vaesenc \T5, \XMM1, \XMM1
2402                vaesenc \T5, \XMM2, \XMM2
2403                vaesenc \T5, \XMM3, \XMM3
2404                vaesenc \T5, \XMM4, \XMM4
2405                vaesenc \T5, \XMM5, \XMM5
2406                vaesenc \T5, \XMM6, \XMM6
2407                vaesenc \T5, \XMM7, \XMM7
2408                vaesenc \T5, \XMM8, \XMM8
2409
2410        vmovdqa         TMP8(%rsp), \T1
2411        vmovdqu         HashKey(arg2), \T5
2412
2413        vpclmulqdq      $0x00, \T5, \T1, \T3
2414        vpxor           \T3, \T7, \T7
2415
2416        vpclmulqdq      $0x01, \T5, \T1, \T3
2417        vpxor           \T3, \T6, \T6
2418
2419        vpclmulqdq      $0x10, \T5, \T1, \T3
2420        vpxor           \T3, \T6, \T6
2421
2422        vpclmulqdq      $0x11, \T5, \T1, \T3
2423        vpxor           \T3, \T4, \T1
2424
2425
2426                vmovdqu 16*10(arg1), \T5
2427
2428        i = 11
2429        setreg
2430.rep (\REP-9)
2431        vaesenc \T5, \XMM1, \XMM1
2432        vaesenc \T5, \XMM2, \XMM2
2433        vaesenc \T5, \XMM3, \XMM3
2434        vaesenc \T5, \XMM4, \XMM4
2435        vaesenc \T5, \XMM5, \XMM5
2436        vaesenc \T5, \XMM6, \XMM6
2437        vaesenc \T5, \XMM7, \XMM7
2438        vaesenc \T5, \XMM8, \XMM8
2439
2440        vmovdqu 16*i(arg1), \T5
2441        i = i + 1
2442        setreg
2443.endr
2444
2445	i = 0
2446	j = 1
2447	setreg
2448.rep 8
2449		vpxor	16*i(arg4, %r11), \T5, \T2
2450                .if \ENC_DEC == ENC
2451                vaesenclast     \T2, reg_j, reg_j
2452                .else
2453                vaesenclast     \T2, reg_j, \T3
2454                vmovdqu 16*i(arg4, %r11), reg_j
2455                vmovdqu \T3, 16*i(arg3, %r11)
2456                .endif
2457	i = (i+1)
2458	j = (j+1)
2459	setreg
2460.endr
2461	#######################################################################
2462
2463
2464	vpslldq	$8, \T6, \T3				# shift-L T3 2 DWs
2465	vpsrldq	$8, \T6, \T6				# shift-R T2 2 DWs
2466	vpxor	\T3, \T7, \T7
2467	vpxor	\T6, \T1, \T1				# accumulate the results in T1:T7
2468
2469
2470
2471	#######################################################################
2472	#first phase of the reduction
2473	vmovdqa         POLY2(%rip), \T3
2474
2475	vpclmulqdq	$0x01, \T7, \T3, \T2
2476	vpslldq		$8, \T2, \T2			# shift-L xmm2 2 DWs
2477
2478	vpxor		\T2, \T7, \T7			# first phase of the reduction complete
2479	#######################################################################
2480                .if \ENC_DEC == ENC
2481		vmovdqu	 \XMM1,	16*0(arg3,%r11)		# Write to the Ciphertext buffer
2482		vmovdqu	 \XMM2,	16*1(arg3,%r11)		# Write to the Ciphertext buffer
2483		vmovdqu	 \XMM3,	16*2(arg3,%r11)		# Write to the Ciphertext buffer
2484		vmovdqu	 \XMM4,	16*3(arg3,%r11)		# Write to the Ciphertext buffer
2485		vmovdqu	 \XMM5,	16*4(arg3,%r11)		# Write to the Ciphertext buffer
2486		vmovdqu	 \XMM6,	16*5(arg3,%r11)		# Write to the Ciphertext buffer
2487		vmovdqu	 \XMM7,	16*6(arg3,%r11)		# Write to the Ciphertext buffer
2488		vmovdqu	 \XMM8,	16*7(arg3,%r11)		# Write to the Ciphertext buffer
2489                .endif
2490
2491	#######################################################################
2492	#second phase of the reduction
2493	vpclmulqdq	$0x00, \T7, \T3, \T2
2494	vpsrldq		$4, \T2, \T2			# shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
2495
2496	vpclmulqdq	$0x10, \T7, \T3, \T4
2497	vpslldq		$4, \T4, \T4			# shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts)
2498
2499	vpxor		\T2, \T4, \T4			# second phase of the reduction complete
2500	#######################################################################
2501	vpxor		\T4, \T1, \T1			# the result is in T1
2502
2503		vpshufb	SHUF_MASK(%rip), \XMM1, \XMM1	# perform a 16Byte swap
2504		vpshufb	SHUF_MASK(%rip), \XMM2, \XMM2	# perform a 16Byte swap
2505		vpshufb	SHUF_MASK(%rip), \XMM3, \XMM3	# perform a 16Byte swap
2506		vpshufb	SHUF_MASK(%rip), \XMM4, \XMM4	# perform a 16Byte swap
2507		vpshufb	SHUF_MASK(%rip), \XMM5, \XMM5	# perform a 16Byte swap
2508		vpshufb	SHUF_MASK(%rip), \XMM6, \XMM6	# perform a 16Byte swap
2509		vpshufb	SHUF_MASK(%rip), \XMM7, \XMM7	# perform a 16Byte swap
2510		vpshufb	SHUF_MASK(%rip), \XMM8, \XMM8	# perform a 16Byte swap
2511
2512
2513	vpxor	\T1, \XMM1, \XMM1
2514
2515
2516
2517.endm
2518
2519
2520# GHASH the last 4 ciphertext blocks.
2521.macro  GHASH_LAST_8_AVX2 T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8
2522
2523        ## Karatsuba Method
2524
2525        vmovdqu         HashKey_8(arg2), \T5
2526
2527        vpshufd         $0b01001110, \XMM1, \T2
2528        vpshufd         $0b01001110, \T5, \T3
2529        vpxor           \XMM1, \T2, \T2
2530        vpxor           \T5, \T3, \T3
2531
2532        vpclmulqdq      $0x11, \T5, \XMM1, \T6
2533        vpclmulqdq      $0x00, \T5, \XMM1, \T7
2534
2535        vpclmulqdq      $0x00, \T3, \T2, \XMM1
2536
2537        ######################
2538
2539        vmovdqu         HashKey_7(arg2), \T5
2540        vpshufd         $0b01001110, \XMM2, \T2
2541        vpshufd         $0b01001110, \T5, \T3
2542        vpxor           \XMM2, \T2, \T2
2543        vpxor           \T5, \T3, \T3
2544
2545        vpclmulqdq      $0x11, \T5, \XMM2, \T4
2546        vpxor           \T4, \T6, \T6
2547
2548        vpclmulqdq      $0x00, \T5, \XMM2, \T4
2549        vpxor           \T4, \T7, \T7
2550
2551        vpclmulqdq      $0x00, \T3, \T2, \T2
2552
2553        vpxor           \T2, \XMM1, \XMM1
2554
2555        ######################
2556
2557        vmovdqu         HashKey_6(arg2), \T5
2558        vpshufd         $0b01001110, \XMM3, \T2
2559        vpshufd         $0b01001110, \T5, \T3
2560        vpxor           \XMM3, \T2, \T2
2561        vpxor           \T5, \T3, \T3
2562
2563        vpclmulqdq      $0x11, \T5, \XMM3, \T4
2564        vpxor           \T4, \T6, \T6
2565
2566        vpclmulqdq      $0x00, \T5, \XMM3, \T4
2567        vpxor           \T4, \T7, \T7
2568
2569        vpclmulqdq      $0x00, \T3, \T2, \T2
2570
2571        vpxor           \T2, \XMM1, \XMM1
2572
2573        ######################
2574
2575        vmovdqu         HashKey_5(arg2), \T5
2576        vpshufd         $0b01001110, \XMM4, \T2
2577        vpshufd         $0b01001110, \T5, \T3
2578        vpxor           \XMM4, \T2, \T2
2579        vpxor           \T5, \T3, \T3
2580
2581        vpclmulqdq      $0x11, \T5, \XMM4, \T4
2582        vpxor           \T4, \T6, \T6
2583
2584        vpclmulqdq      $0x00, \T5, \XMM4, \T4
2585        vpxor           \T4, \T7, \T7
2586
2587        vpclmulqdq      $0x00, \T3, \T2, \T2
2588
2589        vpxor           \T2, \XMM1, \XMM1
2590
2591        ######################
2592
2593        vmovdqu         HashKey_4(arg2), \T5
2594        vpshufd         $0b01001110, \XMM5, \T2
2595        vpshufd         $0b01001110, \T5, \T3
2596        vpxor           \XMM5, \T2, \T2
2597        vpxor           \T5, \T3, \T3
2598
2599        vpclmulqdq      $0x11, \T5, \XMM5, \T4
2600        vpxor           \T4, \T6, \T6
2601
2602        vpclmulqdq      $0x00, \T5, \XMM5, \T4
2603        vpxor           \T4, \T7, \T7
2604
2605        vpclmulqdq      $0x00, \T3, \T2, \T2
2606
2607        vpxor           \T2, \XMM1, \XMM1
2608
2609        ######################
2610
2611        vmovdqu         HashKey_3(arg2), \T5
2612        vpshufd         $0b01001110, \XMM6, \T2
2613        vpshufd         $0b01001110, \T5, \T3
2614        vpxor           \XMM6, \T2, \T2
2615        vpxor           \T5, \T3, \T3
2616
2617        vpclmulqdq      $0x11, \T5, \XMM6, \T4
2618        vpxor           \T4, \T6, \T6
2619
2620        vpclmulqdq      $0x00, \T5, \XMM6, \T4
2621        vpxor           \T4, \T7, \T7
2622
2623        vpclmulqdq      $0x00, \T3, \T2, \T2
2624
2625        vpxor           \T2, \XMM1, \XMM1
2626
2627        ######################
2628
2629        vmovdqu         HashKey_2(arg2), \T5
2630        vpshufd         $0b01001110, \XMM7, \T2
2631        vpshufd         $0b01001110, \T5, \T3
2632        vpxor           \XMM7, \T2, \T2
2633        vpxor           \T5, \T3, \T3
2634
2635        vpclmulqdq      $0x11, \T5, \XMM7, \T4
2636        vpxor           \T4, \T6, \T6
2637
2638        vpclmulqdq      $0x00, \T5, \XMM7, \T4
2639        vpxor           \T4, \T7, \T7
2640
2641        vpclmulqdq      $0x00, \T3, \T2, \T2
2642
2643        vpxor           \T2, \XMM1, \XMM1
2644
2645        ######################
2646
2647        vmovdqu         HashKey(arg2), \T5
2648        vpshufd         $0b01001110, \XMM8, \T2
2649        vpshufd         $0b01001110, \T5, \T3
2650        vpxor           \XMM8, \T2, \T2
2651        vpxor           \T5, \T3, \T3
2652
2653        vpclmulqdq      $0x11, \T5, \XMM8, \T4
2654        vpxor           \T4, \T6, \T6
2655
2656        vpclmulqdq      $0x00, \T5, \XMM8, \T4
2657        vpxor           \T4, \T7, \T7
2658
2659        vpclmulqdq      $0x00, \T3, \T2, \T2
2660
2661        vpxor           \T2, \XMM1, \XMM1
2662        vpxor           \T6, \XMM1, \XMM1
2663        vpxor           \T7, \XMM1, \T2
2664
2665
2666
2667
2668        vpslldq $8, \T2, \T4
2669        vpsrldq $8, \T2, \T2
2670
2671        vpxor   \T4, \T7, \T7
2672        vpxor   \T2, \T6, \T6                      # <T6:T7> holds the result of the
2673						   # accumulated carry-less multiplications
2674
2675        #######################################################################
2676        #first phase of the reduction
2677        vmovdqa         POLY2(%rip), \T3
2678
2679        vpclmulqdq      $0x01, \T7, \T3, \T2
2680        vpslldq         $8, \T2, \T2               # shift-L xmm2 2 DWs
2681
2682        vpxor           \T2, \T7, \T7              # first phase of the reduction complete
2683        #######################################################################
2684
2685
2686        #second phase of the reduction
2687        vpclmulqdq      $0x00, \T7, \T3, \T2
2688        vpsrldq         $4, \T2, \T2               # shift-R T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
2689
2690        vpclmulqdq      $0x10, \T7, \T3, \T4
2691        vpslldq         $4, \T4, \T4               # shift-L T4 1 DW (Shift-L 1-DW to obtain result with no shifts)
2692
2693        vpxor           \T2, \T4, \T4              # second phase of the reduction complete
2694        #######################################################################
2695        vpxor           \T4, \T6, \T6              # the result is in T6
2696.endm
2697
2698
2699
2700#############################################################
2701#void   aesni_gcm_init_avx_gen4
2702#        (gcm_data     *my_ctx_data,
2703#         gcm_context_data *data,
2704#        u8      *iv, /* Pre-counter block j0: 4 byte salt
2705#			(from Security Association) concatenated with 8 byte
2706#			Initialisation Vector (from IPSec ESP Payload)
2707#			concatenated with 0x00000001. 16-byte aligned pointer. */
2708#        u8     *hash_subkey# /* H, the Hash sub key input. Data starts on a 16-byte boundary. */
2709#        const   u8 *aad, /* Additional Authentication Data (AAD)*/
2710#        u64     aad_len) /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
2711#############################################################
2712SYM_FUNC_START(aesni_gcm_init_avx_gen4)
2713        FUNC_SAVE
2714        INIT GHASH_MUL_AVX2, PRECOMPUTE_AVX2
2715        FUNC_RESTORE
2716        RET
2717SYM_FUNC_END(aesni_gcm_init_avx_gen4)
2718
2719###############################################################################
2720#void   aesni_gcm_enc_avx_gen4(
2721#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
2722#        gcm_context_data *data,
2723#        u8      *out, /* Ciphertext output. Encrypt in-place is allowed.  */
2724#        const   u8 *in, /* Plaintext input */
2725#        u64     plaintext_len) /* Length of data in Bytes for encryption. */
2726###############################################################################
2727SYM_FUNC_START(aesni_gcm_enc_update_avx_gen4)
2728        FUNC_SAVE
2729        mov     keysize,%eax
2730        cmp     $32, %eax
2731        je      key_256_enc_update4
2732        cmp     $16, %eax
2733        je      key_128_enc_update4
2734        # must be 192
2735        GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 11
2736        FUNC_RESTORE
2737	RET
2738key_128_enc_update4:
2739        GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 9
2740        FUNC_RESTORE
2741	RET
2742key_256_enc_update4:
2743        GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 13
2744        FUNC_RESTORE
2745	RET
2746SYM_FUNC_END(aesni_gcm_enc_update_avx_gen4)
2747
2748###############################################################################
2749#void   aesni_gcm_dec_update_avx_gen4(
2750#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
2751#        gcm_context_data *data,
2752#        u8      *out, /* Plaintext output. Decrypt in-place is allowed.  */
2753#        const   u8 *in, /* Ciphertext input */
2754#        u64     plaintext_len) /* Length of data in Bytes for encryption. */
2755###############################################################################
2756SYM_FUNC_START(aesni_gcm_dec_update_avx_gen4)
2757        FUNC_SAVE
2758        mov     keysize,%eax
2759        cmp     $32, %eax
2760        je      key_256_dec_update4
2761        cmp     $16, %eax
2762        je      key_128_dec_update4
2763        # must be 192
2764        GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 11
2765        FUNC_RESTORE
2766        RET
2767key_128_dec_update4:
2768        GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 9
2769        FUNC_RESTORE
2770        RET
2771key_256_dec_update4:
2772        GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 13
2773        FUNC_RESTORE
2774        RET
2775SYM_FUNC_END(aesni_gcm_dec_update_avx_gen4)
2776
2777###############################################################################
2778#void   aesni_gcm_finalize_avx_gen4(
2779#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
2780#        gcm_context_data *data,
2781#        u8      *auth_tag, /* Authenticated Tag output. */
2782#        u64     auth_tag_len)# /* Authenticated Tag Length in bytes.
2783#                              Valid values are 16 (most likely), 12 or 8. */
2784###############################################################################
2785SYM_FUNC_START(aesni_gcm_finalize_avx_gen4)
2786        FUNC_SAVE
2787        mov	keysize,%eax
2788        cmp     $32, %eax
2789        je      key_256_finalize4
2790        cmp     $16, %eax
2791        je      key_128_finalize4
2792        # must be 192
2793        GCM_COMPLETE GHASH_MUL_AVX2, 11, arg3, arg4
2794        FUNC_RESTORE
2795        RET
2796key_128_finalize4:
2797        GCM_COMPLETE GHASH_MUL_AVX2, 9, arg3, arg4
2798        FUNC_RESTORE
2799        RET
2800key_256_finalize4:
2801        GCM_COMPLETE GHASH_MUL_AVX2, 13, arg3, arg4
2802        FUNC_RESTORE
2803        RET
2804SYM_FUNC_END(aesni_gcm_finalize_avx_gen4)
2805