1// Copyright 2016 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// +build amd64,!gccgo,!appengine
6
7#include "textflag.h"
8
9DATA iv0<>+0x00(SB)/4, $0x6a09e667
10DATA iv0<>+0x04(SB)/4, $0xbb67ae85
11DATA iv0<>+0x08(SB)/4, $0x3c6ef372
12DATA iv0<>+0x0c(SB)/4, $0xa54ff53a
13GLOBL iv0<>(SB), (NOPTR+RODATA), $16
14
15DATA iv1<>+0x00(SB)/4, $0x510e527f
16DATA iv1<>+0x04(SB)/4, $0x9b05688c
17DATA iv1<>+0x08(SB)/4, $0x1f83d9ab
18DATA iv1<>+0x0c(SB)/4, $0x5be0cd19
19GLOBL iv1<>(SB), (NOPTR+RODATA), $16
20
21DATA rol16<>+0x00(SB)/8, $0x0504070601000302
22DATA rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A
23GLOBL rol16<>(SB), (NOPTR+RODATA), $16
24
25DATA rol8<>+0x00(SB)/8, $0x0407060500030201
26DATA rol8<>+0x08(SB)/8, $0x0C0F0E0D080B0A09
27GLOBL rol8<>(SB), (NOPTR+RODATA), $16
28
29DATA counter<>+0x00(SB)/8, $0x40
30DATA counter<>+0x08(SB)/8, $0x0
31GLOBL counter<>(SB), (NOPTR+RODATA), $16
32
33#define ROTL_SSE2(n, t, v) \
34	MOVO  v, t;       \
35	PSLLL $n, t;      \
36	PSRLL $(32-n), v; \
37	PXOR  t, v
38
39#define ROTL_SSSE3(c, v) \
40	PSHUFB c, v
41
42#define ROUND_SSE2(v0, v1, v2, v3, m0, m1, m2, m3, t) \
43	PADDL  m0, v0;        \
44	PADDL  v1, v0;        \
45	PXOR   v0, v3;        \
46	ROTL_SSE2(16, t, v3); \
47	PADDL  v3, v2;        \
48	PXOR   v2, v1;        \
49	ROTL_SSE2(20, t, v1); \
50	PADDL  m1, v0;        \
51	PADDL  v1, v0;        \
52	PXOR   v0, v3;        \
53	ROTL_SSE2(24, t, v3); \
54	PADDL  v3, v2;        \
55	PXOR   v2, v1;        \
56	ROTL_SSE2(25, t, v1); \
57	PSHUFL $0x39, v1, v1; \
58	PSHUFL $0x4E, v2, v2; \
59	PSHUFL $0x93, v3, v3; \
60	PADDL  m2, v0;        \
61	PADDL  v1, v0;        \
62	PXOR   v0, v3;        \
63	ROTL_SSE2(16, t, v3); \
64	PADDL  v3, v2;        \
65	PXOR   v2, v1;        \
66	ROTL_SSE2(20, t, v1); \
67	PADDL  m3, v0;        \
68	PADDL  v1, v0;        \
69	PXOR   v0, v3;        \
70	ROTL_SSE2(24, t, v3); \
71	PADDL  v3, v2;        \
72	PXOR   v2, v1;        \
73	ROTL_SSE2(25, t, v1); \
74	PSHUFL $0x39, v3, v3; \
75	PSHUFL $0x4E, v2, v2; \
76	PSHUFL $0x93, v1, v1
77
78#define ROUND_SSSE3(v0, v1, v2, v3, m0, m1, m2, m3, t, c16, c8) \
79	PADDL  m0, v0;        \
80	PADDL  v1, v0;        \
81	PXOR   v0, v3;        \
82	ROTL_SSSE3(c16, v3);  \
83	PADDL  v3, v2;        \
84	PXOR   v2, v1;        \
85	ROTL_SSE2(20, t, v1); \
86	PADDL  m1, v0;        \
87	PADDL  v1, v0;        \
88	PXOR   v0, v3;        \
89	ROTL_SSSE3(c8, v3);   \
90	PADDL  v3, v2;        \
91	PXOR   v2, v1;        \
92	ROTL_SSE2(25, t, v1); \
93	PSHUFL $0x39, v1, v1; \
94	PSHUFL $0x4E, v2, v2; \
95	PSHUFL $0x93, v3, v3; \
96	PADDL  m2, v0;        \
97	PADDL  v1, v0;        \
98	PXOR   v0, v3;        \
99	ROTL_SSSE3(c16, v3);  \
100	PADDL  v3, v2;        \
101	PXOR   v2, v1;        \
102	ROTL_SSE2(20, t, v1); \
103	PADDL  m3, v0;        \
104	PADDL  v1, v0;        \
105	PXOR   v0, v3;        \
106	ROTL_SSSE3(c8, v3);   \
107	PADDL  v3, v2;        \
108	PXOR   v2, v1;        \
109	ROTL_SSE2(25, t, v1); \
110	PSHUFL $0x39, v3, v3; \
111	PSHUFL $0x4E, v2, v2; \
112	PSHUFL $0x93, v1, v1
113
114
115#define LOAD_MSG_SSE4(m0, m1, m2, m3, src, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15) \
116	MOVL   i0*4(src), m0;      \
117	PINSRD $1, i1*4(src), m0;  \
118	PINSRD $2, i2*4(src), m0;  \
119	PINSRD $3, i3*4(src), m0;  \
120	MOVL   i4*4(src), m1;      \
121	PINSRD $1, i5*4(src), m1;  \
122	PINSRD $2, i6*4(src), m1;  \
123	PINSRD $3, i7*4(src), m1;  \
124	MOVL   i8*4(src), m2;      \
125	PINSRD $1, i9*4(src), m2;  \
126	PINSRD $2, i10*4(src), m2; \
127	PINSRD $3, i11*4(src), m2; \
128	MOVL   i12*4(src), m3;     \
129	PINSRD $1, i13*4(src), m3; \
130	PINSRD $2, i14*4(src), m3; \
131	PINSRD $3, i15*4(src), m3
132
133#define PRECOMPUTE_MSG(dst, off, src, R8, R9, R10, R11, R12, R13, R14, R15) \
134	MOVQ 0*4(src), R8;           \
135	MOVQ 2*4(src), R9;           \
136	MOVQ 4*4(src), R10;          \
137	MOVQ 6*4(src), R11;          \
138	MOVQ 8*4(src), R12;          \
139	MOVQ 10*4(src), R13;         \
140	MOVQ 12*4(src), R14;         \
141	MOVQ 14*4(src), R15;         \
142	                             \
143	MOVL R8, 0*4+off+0(dst);     \
144	MOVL R8, 9*4+off+64(dst);    \
145	MOVL R8, 5*4+off+128(dst);   \
146	MOVL R8, 14*4+off+192(dst);  \
147	MOVL R8, 4*4+off+256(dst);   \
148	MOVL R8, 2*4+off+320(dst);   \
149	MOVL R8, 8*4+off+384(dst);   \
150	MOVL R8, 12*4+off+448(dst);  \
151	MOVL R8, 3*4+off+512(dst);   \
152	MOVL R8, 15*4+off+576(dst);  \
153	SHRQ $32, R8;                \
154	MOVL R8, 4*4+off+0(dst);     \
155	MOVL R8, 8*4+off+64(dst);    \
156	MOVL R8, 14*4+off+128(dst);  \
157	MOVL R8, 5*4+off+192(dst);   \
158	MOVL R8, 12*4+off+256(dst);  \
159	MOVL R8, 11*4+off+320(dst);  \
160	MOVL R8, 1*4+off+384(dst);   \
161	MOVL R8, 6*4+off+448(dst);   \
162	MOVL R8, 10*4+off+512(dst);  \
163	MOVL R8, 3*4+off+576(dst);   \
164	                             \
165	MOVL R9, 1*4+off+0(dst);     \
166	MOVL R9, 13*4+off+64(dst);   \
167	MOVL R9, 6*4+off+128(dst);   \
168	MOVL R9, 8*4+off+192(dst);   \
169	MOVL R9, 2*4+off+256(dst);   \
170	MOVL R9, 0*4+off+320(dst);   \
171	MOVL R9, 14*4+off+384(dst);  \
172	MOVL R9, 11*4+off+448(dst);  \
173	MOVL R9, 12*4+off+512(dst);  \
174	MOVL R9, 4*4+off+576(dst);   \
175	SHRQ $32, R9;                \
176	MOVL R9, 5*4+off+0(dst);     \
177	MOVL R9, 15*4+off+64(dst);   \
178	MOVL R9, 9*4+off+128(dst);   \
179	MOVL R9, 1*4+off+192(dst);   \
180	MOVL R9, 11*4+off+256(dst);  \
181	MOVL R9, 7*4+off+320(dst);   \
182	MOVL R9, 13*4+off+384(dst);  \
183	MOVL R9, 3*4+off+448(dst);   \
184	MOVL R9, 6*4+off+512(dst);   \
185	MOVL R9, 10*4+off+576(dst);  \
186	                             \
187	MOVL R10, 2*4+off+0(dst);    \
188	MOVL R10, 1*4+off+64(dst);   \
189	MOVL R10, 15*4+off+128(dst); \
190	MOVL R10, 10*4+off+192(dst); \
191	MOVL R10, 6*4+off+256(dst);  \
192	MOVL R10, 8*4+off+320(dst);  \
193	MOVL R10, 3*4+off+384(dst);  \
194	MOVL R10, 13*4+off+448(dst); \
195	MOVL R10, 14*4+off+512(dst); \
196	MOVL R10, 5*4+off+576(dst);  \
197	SHRQ $32, R10;               \
198	MOVL R10, 6*4+off+0(dst);    \
199	MOVL R10, 11*4+off+64(dst);  \
200	MOVL R10, 2*4+off+128(dst);  \
201	MOVL R10, 9*4+off+192(dst);  \
202	MOVL R10, 1*4+off+256(dst);  \
203	MOVL R10, 13*4+off+320(dst); \
204	MOVL R10, 4*4+off+384(dst);  \
205	MOVL R10, 8*4+off+448(dst);  \
206	MOVL R10, 15*4+off+512(dst); \
207	MOVL R10, 7*4+off+576(dst);  \
208	                             \
209	MOVL R11, 3*4+off+0(dst);    \
210	MOVL R11, 7*4+off+64(dst);   \
211	MOVL R11, 13*4+off+128(dst); \
212	MOVL R11, 12*4+off+192(dst); \
213	MOVL R11, 10*4+off+256(dst); \
214	MOVL R11, 1*4+off+320(dst);  \
215	MOVL R11, 9*4+off+384(dst);  \
216	MOVL R11, 14*4+off+448(dst); \
217	MOVL R11, 0*4+off+512(dst);  \
218	MOVL R11, 6*4+off+576(dst);  \
219	SHRQ $32, R11;               \
220	MOVL R11, 7*4+off+0(dst);    \
221	MOVL R11, 14*4+off+64(dst);  \
222	MOVL R11, 10*4+off+128(dst); \
223	MOVL R11, 0*4+off+192(dst);  \
224	MOVL R11, 5*4+off+256(dst);  \
225	MOVL R11, 9*4+off+320(dst);  \
226	MOVL R11, 12*4+off+384(dst); \
227	MOVL R11, 1*4+off+448(dst);  \
228	MOVL R11, 13*4+off+512(dst); \
229	MOVL R11, 2*4+off+576(dst);  \
230	                             \
231	MOVL R12, 8*4+off+0(dst);    \
232	MOVL R12, 5*4+off+64(dst);   \
233	MOVL R12, 4*4+off+128(dst);  \
234	MOVL R12, 15*4+off+192(dst); \
235	MOVL R12, 14*4+off+256(dst); \
236	MOVL R12, 3*4+off+320(dst);  \
237	MOVL R12, 11*4+off+384(dst); \
238	MOVL R12, 10*4+off+448(dst); \
239	MOVL R12, 7*4+off+512(dst);  \
240	MOVL R12, 1*4+off+576(dst);  \
241	SHRQ $32, R12;               \
242	MOVL R12, 12*4+off+0(dst);   \
243	MOVL R12, 2*4+off+64(dst);   \
244	MOVL R12, 11*4+off+128(dst); \
245	MOVL R12, 4*4+off+192(dst);  \
246	MOVL R12, 0*4+off+256(dst);  \
247	MOVL R12, 15*4+off+320(dst); \
248	MOVL R12, 10*4+off+384(dst); \
249	MOVL R12, 7*4+off+448(dst);  \
250	MOVL R12, 5*4+off+512(dst);  \
251	MOVL R12, 9*4+off+576(dst);  \
252	                             \
253	MOVL R13, 9*4+off+0(dst);    \
254	MOVL R13, 4*4+off+64(dst);   \
255	MOVL R13, 8*4+off+128(dst);  \
256	MOVL R13, 13*4+off+192(dst); \
257	MOVL R13, 3*4+off+256(dst);  \
258	MOVL R13, 5*4+off+320(dst);  \
259	MOVL R13, 7*4+off+384(dst);  \
260	MOVL R13, 15*4+off+448(dst); \
261	MOVL R13, 11*4+off+512(dst); \
262	MOVL R13, 0*4+off+576(dst);  \
263	SHRQ $32, R13;               \
264	MOVL R13, 13*4+off+0(dst);   \
265	MOVL R13, 10*4+off+64(dst);  \
266	MOVL R13, 0*4+off+128(dst);  \
267	MOVL R13, 3*4+off+192(dst);  \
268	MOVL R13, 9*4+off+256(dst);  \
269	MOVL R13, 6*4+off+320(dst);  \
270	MOVL R13, 15*4+off+384(dst); \
271	MOVL R13, 4*4+off+448(dst);  \
272	MOVL R13, 2*4+off+512(dst);  \
273	MOVL R13, 12*4+off+576(dst); \
274	                             \
275	MOVL R14, 10*4+off+0(dst);   \
276	MOVL R14, 12*4+off+64(dst);  \
277	MOVL R14, 1*4+off+128(dst);  \
278	MOVL R14, 6*4+off+192(dst);  \
279	MOVL R14, 13*4+off+256(dst); \
280	MOVL R14, 4*4+off+320(dst);  \
281	MOVL R14, 0*4+off+384(dst);  \
282	MOVL R14, 2*4+off+448(dst);  \
283	MOVL R14, 8*4+off+512(dst);  \
284	MOVL R14, 14*4+off+576(dst); \
285	SHRQ $32, R14;               \
286	MOVL R14, 14*4+off+0(dst);   \
287	MOVL R14, 3*4+off+64(dst);   \
288	MOVL R14, 7*4+off+128(dst);  \
289	MOVL R14, 2*4+off+192(dst);  \
290	MOVL R14, 15*4+off+256(dst); \
291	MOVL R14, 12*4+off+320(dst); \
292	MOVL R14, 6*4+off+384(dst);  \
293	MOVL R14, 0*4+off+448(dst);  \
294	MOVL R14, 9*4+off+512(dst);  \
295	MOVL R14, 11*4+off+576(dst); \
296	                             \
297	MOVL R15, 11*4+off+0(dst);   \
298	MOVL R15, 0*4+off+64(dst);   \
299	MOVL R15, 12*4+off+128(dst); \
300	MOVL R15, 7*4+off+192(dst);  \
301	MOVL R15, 8*4+off+256(dst);  \
302	MOVL R15, 14*4+off+320(dst); \
303	MOVL R15, 2*4+off+384(dst);  \
304	MOVL R15, 5*4+off+448(dst);  \
305	MOVL R15, 1*4+off+512(dst);  \
306	MOVL R15, 13*4+off+576(dst); \
307	SHRQ $32, R15;               \
308	MOVL R15, 15*4+off+0(dst);   \
309	MOVL R15, 6*4+off+64(dst);   \
310	MOVL R15, 3*4+off+128(dst);  \
311	MOVL R15, 11*4+off+192(dst); \
312	MOVL R15, 7*4+off+256(dst);  \
313	MOVL R15, 10*4+off+320(dst); \
314	MOVL R15, 5*4+off+384(dst);  \
315	MOVL R15, 9*4+off+448(dst);  \
316	MOVL R15, 4*4+off+512(dst);  \
317	MOVL R15, 8*4+off+576(dst)
318
319#define BLAKE2s_SSE2() \
320	PRECOMPUTE_MSG(SP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15);               \
321	ROUND_SSE2(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X8);                 \
322	ROUND_SSE2(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X8);     \
323	ROUND_SSE2(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X8); \
324	ROUND_SSE2(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X8); \
325	ROUND_SSE2(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X8); \
326	ROUND_SSE2(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X8); \
327	ROUND_SSE2(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X8); \
328	ROUND_SSE2(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X8); \
329	ROUND_SSE2(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X8); \
330	ROUND_SSE2(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X8)
331
332#define BLAKE2s_SSSE3() \
333	PRECOMPUTE_MSG(SP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15);                          \
334	ROUND_SSSE3(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X8, X13, X14);                 \
335	ROUND_SSSE3(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X8, X13, X14);     \
336	ROUND_SSSE3(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X8, X13, X14); \
337	ROUND_SSSE3(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X8, X13, X14); \
338	ROUND_SSSE3(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X8, X13, X14); \
339	ROUND_SSSE3(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X8, X13, X14); \
340	ROUND_SSSE3(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X8, X13, X14); \
341	ROUND_SSSE3(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X8, X13, X14); \
342	ROUND_SSSE3(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X8, X13, X14); \
343	ROUND_SSSE3(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X8, X13, X14)
344
345#define BLAKE2s_SSE4() \
346	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15); \
347	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
348	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3); \
349	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
350	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4); \
351	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
352	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8); \
353	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
354	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13); \
355	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
356	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9); \
357	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
358	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11); \
359	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
360	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10); \
361	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
362	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5); \
363	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
364	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0); \
365	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
366
367#define HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_FUNC) \
368	MOVQ  h, AX;                   \
369	MOVQ  c, BX;                   \
370	MOVL  flag, CX;                \
371	MOVQ  blocks_base, SI;         \
372	MOVQ  blocks_len, DX;          \
373	                               \
374	MOVQ  SP, BP;                  \
375	MOVQ  SP, R9;                  \
376	ADDQ  $15, R9;                 \
377	ANDQ  $~15, R9;                \
378	MOVQ  R9, SP;                  \
379	                               \
380	MOVQ  0(BX), R9;               \
381	MOVQ  R9, 0(SP);               \
382	XORQ  R9, R9;                  \
383	MOVQ  R9, 8(SP);               \
384	MOVL  CX, 8(SP);               \
385	                               \
386	MOVOU 0(AX), X0;               \
387	MOVOU 16(AX), X1;              \
388	MOVOU iv0<>(SB), X2;           \
389	MOVOU iv1<>(SB), X3            \
390	                               \
391	MOVOU counter<>(SB), X12;      \
392	MOVOU rol16<>(SB), X13;        \
393	MOVOU rol8<>(SB), X14;         \
394	MOVO  0(SP), X15;              \
395	                               \
396	loop:                          \
397	MOVO  X0, X4;                  \
398	MOVO  X1, X5;                  \
399	MOVO  X2, X6;                  \
400	MOVO  X3, X7;                  \
401	                               \
402	PADDQ X12, X15;                \
403	PXOR  X15, X7;                 \
404	                               \
405	BLAKE2s_FUNC();                \
406	                               \
407	PXOR  X4, X0;                  \
408	PXOR  X5, X1;                  \
409	PXOR  X6, X0;                  \
410	PXOR  X7, X1;                  \
411	                               \
412	LEAQ  64(SI), SI;              \
413	SUBQ  $64, DX;                 \
414	JNE   loop;                    \
415	                               \
416	MOVO  X15, 0(SP);              \
417	MOVQ  0(SP), R9;               \
418	MOVQ  R9, 0(BX);               \
419	                               \
420	MOVOU X0, 0(AX);               \
421	MOVOU X1, 16(AX);              \
422	                               \
423	MOVQ  BP, SP
424
425// func hashBlocksSSE2(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
426TEXT ·hashBlocksSSE2(SB), 0, $672-48 // frame = 656 + 16 byte alignment
427	HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSE2)
428	RET
429
430// func hashBlocksSSSE3(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
431TEXT ·hashBlocksSSSE3(SB), 0, $672-48 // frame = 656 + 16 byte alignment
432	HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSSE3)
433	RET
434
435// func hashBlocksSSE4(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
436TEXT ·hashBlocksSSE4(SB), 0, $32-48 // frame = 16 + 16 byte alignment
437	HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSE4)
438	RET
439