1 /*
2
3 BLIS
4 An object-based framework for developing high-performance BLAS-like
5 libraries.
6
7 Copyright (C) 2014, The University of Texas at Austin
8
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are
11 met:
12 - Redistributions of source code must retain the above copyright
13 notice, this list of conditions and the following disclaimer.
14 - Redistributions in binary form must reproduce the above copyright
15 notice, this list of conditions and the following disclaimer in the
16 documentation and/or other materials provided with the distribution.
17 - Neither the name of The University of Texas at Austin nor the names
18 of its contributors may be used to endorse or promote products
19 derived derived from this software without specific prior written permission.
20
21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY
25 OF TEXAS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
29 OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32
33 */
34
35 #include "blis.h"
36 #include <assert.h>
37
38 #include "bli_avx512_macros.h"
39
40 #define A_L1_PREFETCH_DIST 4
41 #define B_L1_PREFETCH_DIST 2
42 #define L2_PREFETCH_DIST 16 // Must be greater than 10, because of the way the loop is constructed.
43
44 //Alternate code path uused if C is not row-major
45 // r9 = c
46 // zmm30 = cs_c * 1...16
47 // r11 = rs_c
48 // r12 = &alpha
49 // r13 = &beta
50 #define UPDATE_C_ROW_SCATTERED_(NUM,BNZ1,BNZ2) \
51 \
52 BNZ1 KXNORW(K(2), K(0), K(0)) BNZ2 \
53 KXNORW(K(3), K(0), K(0)) \
54 BNZ1 VGATHERDPS(ZMM(31) MASK_K(2), MEM(R(9),ZMM(30),4)) BNZ2 \
55 VMULPS(ZMM(NUM), ZMM(NUM), MEM_1TO16(R(12))) /*scale by alpha*/ \
56 BNZ1 VFMADD231PS(ZMM(NUM), ZMM(31), MEM_1TO16(R(13))) BNZ2 /*scale by beta, add in result*/ \
57 VSCATTERDPS(MEM(R(9),ZMM(30),4) MASK_K(3), ZMM(NUM)) \
58 ADD(R(9), R(11))
59
60 #define UPDATE_C_ROW_SCATTERED(NUM) UPDATE_C_ROW_SCATTERED_(NUM,,)
61 #define UPDATE_C_BZ_ROW_SCATTERED(NUM) UPDATE_C_ROW_SCATTERED_(NUM,COMMENT_BEGIN,COMMENT_END)
62
63 // r12 = &alpha
64 // zmm31 = beta
65 // r9 = c
66 // r11 = rs_c
67 // r10 = 3*rs_c
68 // rdi = 4*rs_c
69 #define UPDATE_C_4_ROWS_(R1,R2,R3,R4,BNZ1,BNZ2) \
70 \
71 VMULPS(ZMM(R1), ZMM(R1), MEM_1TO16(R(12))) \
72 VMULPS(ZMM(R2), ZMM(R2), MEM_1TO16(R(12))) \
73 VMULPS(ZMM(R3), ZMM(R3), MEM_1TO16(R(12))) \
74 VMULPS(ZMM(R4), ZMM(R4), MEM_1TO16(R(12))) \
75 BNZ1 VFMADD231PS(ZMM(R1), ZMM(31), MEM(R(9) )) BNZ2 \
76 BNZ1 VFMADD231PS(ZMM(R2), ZMM(31), MEM(R(9),R(11),1)) BNZ2 \
77 BNZ1 VFMADD231PS(ZMM(R3), ZMM(31), MEM(R(9),R(11),2)) BNZ2 \
78 BNZ1 VFMADD231PS(ZMM(R4), ZMM(31), MEM(R(9),R(10),1)) BNZ2 \
79 VMOVUPS(MEM(R(9) ), ZMM(R1)) \
80 VMOVUPS(MEM(R(9),R(11),1), ZMM(R2)) \
81 VMOVUPS(MEM(R(9),R(11),2), ZMM(R3)) \
82 VMOVUPS(MEM(R(9),R(10),1), ZMM(R4)) \
83 ADD(R(9), RDI)
84
85 // r12 = &alpha
86 // zmm31 = beta
87 // r9 = c
88 // r11 = rs_c
89 #define UPDATE_C_2_ROWS_(R1,R2,BNZ1,BNZ2) \
90 \
91 VMULPS(ZMM(R1), ZMM(R1), MEM_1TO16(R(12))) \
92 VMULPS(ZMM(R2), ZMM(R2), MEM_1TO16(R(12))) \
93 BNZ1 VFMADD231PS(ZMM(R1), ZMM(31), MEM(R(9) )) BNZ2 \
94 BNZ1 VFMADD231PS(ZMM(R2), ZMM(31), MEM(R(9),R(11),1)) BNZ2 \
95 VMOVUPS(MEM(R(9) ), ZMM(R1)) \
96 VMOVUPS(MEM(R(9),R(11),1), ZMM(R2))
97
98 #define UPDATE_C_4_ROWS(R1,R2,R3,R4) UPDATE_C_4_ROWS_(R1,R2,R3,R4,,)
99 #define UPDATE_C_2_ROWS(R1,R2) UPDATE_C_2_ROWS_(R1,R2,,)
100 #define UPDATE_C_BZ_4_ROWS(R1,R2,R3,R4) UPDATE_C_4_ROWS_(R1,R2,R3,R4,COMMENT_BEGIN,COMMENT_END)
101 #define UPDATE_C_BZ_2_ROWS(R1,R2) UPDATE_C_2_ROWS_(R1,R2,COMMENT_BEGIN,COMMENT_END)
102
103 #define A_TIMES_B_ROW(n) VFMADD231PS(ZMM(n), ZMM(31), MEM_1TO16(R(15),n*4))
104 #define A_TIMES_B_ROW_PREV(n) VFMADD231PS(ZMM(n), ZMM(31), MEM_1TO16(R(15),(n-32)*4))
105 #define PREFETCH_A_L1(n) PREFETCH(0, MEM(R(15),A_L1_PREFETCH_DIST*4*32+n*64))
106 #define PREFETCH_A_L2(n) PREFETCH(1, MEM(R(15),R(14),1,n*64))
107 #define PREFETCH_B_L1 PREFETCH(0, MEM(RBX,B_L1_PREFETCH_DIST*4*16))
108 #define PREFETCH_B_L2 PREFETCH(1, MEM(RBX,R(13),1))
109
110 //One iteration of the k_r loop.
111 //Each iteration, we prefetch A into L1 and into L2
112 // r15 = a
113 // rbx = b
114 // rcx = c
115 // r11 = rs_c
116 // r13 = L2_PREFETCH_DIST*4*16
117 // r14 = L2_PREFETCH_DIST*4*32
118 // r12 = 32*4 = dist. to next sliver of a
119 // r9 = 16*4 = dist. to next sliver of b
120 #define MAIN_LOOP_(COUNTER, PC_L1_1, PC_L1_2, PC_L2_1, PC_L2_2) \
121 \
122 /* Can this be pre-loaded for next it. in zmm30? */ \
123 VMOVAPS(ZMM(31), MEM(RBX)) \
124 \
125 A_TIMES_B_ROW ( 0) \
126 A_TIMES_B_ROW ( 1) PREFETCH_A_L1(0) \
127 A_TIMES_B_ROW ( 2) PREFETCH_A_L1(1) \
128 A_TIMES_B_ROW ( 3) PREFETCH_A_L1(2) \
129 A_TIMES_B_ROW ( 4) PREFETCH_A_L1(3) \
130 A_TIMES_B_ROW ( 5) PREFETCH_A_L2(0) \
131 A_TIMES_B_ROW ( 6) PC_L1_1 PREFETCH(0, MEM(RCX)) PC_L1_2 \
132 A_TIMES_B_ROW ( 7) PC_L1_1 ADD(RCX, R(11)) PC_L1_2 \
133 A_TIMES_B_ROW ( 8) \
134 A_TIMES_B_ROW ( 9) PC_L2_1 PREFETCH(1, MEM(RCX)) PC_L2_2 \
135 A_TIMES_B_ROW (10) PREFETCH_A_L2(1) \
136 A_TIMES_B_ROW (11) PC_L1_1 PREFETCH(0, MEM(RCX)) PC_L1_2 \
137 A_TIMES_B_ROW (12) PC_L1_1 ADD(RCX, R(11)) PC_L1_2 \
138 A_TIMES_B_ROW (13) \
139 A_TIMES_B_ROW (14) \
140 A_TIMES_B_ROW (15) PREFETCH_A_L2(2) \
141 A_TIMES_B_ROW (16) PC_L1_1 PREFETCH(0, MEM(RCX)) PC_L1_2 \
142 A_TIMES_B_ROW (17) PC_L1_1 ADD(RCX, R(11)) PC_L1_2 \
143 A_TIMES_B_ROW (18) \
144 A_TIMES_B_ROW (19) \
145 A_TIMES_B_ROW (20) PREFETCH_A_L2(3) \
146 A_TIMES_B_ROW (21) ADD(R(15), R(12)) \
147 A_TIMES_B_ROW_PREV(22) \
148 A_TIMES_B_ROW_PREV(23) PC_L2_1 ADD(RCX, R(11)) PC_L2_2 \
149 A_TIMES_B_ROW_PREV(24) DEC(COUNTER) \
150 A_TIMES_B_ROW_PREV(25) PREFETCH_B_L2 \
151 A_TIMES_B_ROW_PREV(26) PREFETCH_B_L1 \
152 A_TIMES_B_ROW_PREV(27) ADD(RBX, R(9)) \
153 A_TIMES_B_ROW_PREV(28) CMP(COUNTER, IMM(0)) \
154 A_TIMES_B_ROW_PREV(29)
155
156 #define MAIN_LOOP(COUNTER) MAIN_LOOP_(COUNTER,COMMENT_BEGIN,COMMENT_END,COMMENT_BEGIN,COMMENT_END)
157 #define MAIN_LOOP_PC_L1(COUNTER) MAIN_LOOP_(COUNTER,,,COMMENT_BEGIN,COMMENT_END)
158 #define MAIN_LOOP_PC_L2(COUNTER) MAIN_LOOP_(COUNTER,COMMENT_BEGIN,COMMENT_END,,)
159
160 //This is an array used for the scatter/gather instructions.
161 static int32_t offsets[32] __attribute__((aligned(64))) =
162 { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,
163 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31};
164
165 //#define MONITORS
166 //#define LOOPMON
bli_sgemm_opt_30x16_knc(dim_t k_,float * restrict alpha,float * restrict a,float * restrict b,float * restrict beta,float * restrict c,inc_t rs_c_,inc_t cs_c_,auxinfo_t * data,cntx_t * restrict cntx)167 void bli_sgemm_opt_30x16_knc(
168 dim_t k_,
169 float* restrict alpha,
170 float* restrict a,
171 float* restrict b,
172 float* restrict beta,
173 float* restrict c, inc_t rs_c_, inc_t cs_c_,
174 auxinfo_t* data,
175 cntx_t* restrict cntx
176 )
177 {
178 (void)data;
179 (void)cntx;
180
181 const float * a_next = bli_auxinfo_next_a( data );
182 const float * b_next = bli_auxinfo_next_b( data );
183
184 const int32_t * offsetPtr = &offsets[0];
185 const int64_t k = k_;
186 const int64_t rs_c = rs_c_;
187 const int64_t cs_c = cs_c_;
188
189 #ifdef MONITORS
190 int toph, topl, both, botl, midl, midh, mid2l, mid2h;
191 #endif
192 #ifdef LOOPMON
193 int tlooph, tloopl, blooph, bloopl;
194 #endif
195
196 __asm__ volatile
197 (
198 #ifdef MONITORS
199 RDTSC
200 MOV(VAR(topl), EAX)
201 MOV(VAR(toph), EDX)
202 #endif
203
204 VPXORD(ZMM(0), ZMM(0), ZMM(0)) //clear out registers
205
206 VMOVAPS(ZMM( 1), ZMM(0))
207 VMOVAPS(ZMM( 2), ZMM(0)) MOV(RSI, VAR(k)) //loop index
208 VMOVAPS(ZMM( 3), ZMM(0)) MOV(R(11), VAR(rs_c)) //load row stride
209 VMOVAPS(ZMM( 4), ZMM(0)) SAL(R(11), IMM(2)) //scale row stride
210 VMOVAPS(ZMM( 5), ZMM(0)) MOV(R(15), VAR(a)) //load address of a
211 VMOVAPS(ZMM( 6), ZMM(0)) MOV(RBX, VAR(b)) //load address of b
212 VMOVAPS(ZMM( 7), ZMM(0))
213 VMOVAPS(ZMM( 8), ZMM(0)) LEA(R(10), MEM(R(11),R(11),2)) //r10 has 3 * r11
214 VMOVAPS(ZMM( 9), ZMM(0))
215 VMOVAPS(ZMM(10), ZMM(0)) MOV(RDI, R(11))
216 VMOVAPS(ZMM(11), ZMM(0)) SAL(RDI, IMM(2)) //rdi has 4*r11
217 VMOVAPS(ZMM(12), ZMM(0)) MOV(RCX, VAR(c)) //load address of c for prefetching
218 VMOVAPS(ZMM(13), ZMM(0))
219 VMOVAPS(ZMM(14), ZMM(0)) MOV(R(8), VAR(k))
220 VMOVAPS(ZMM(15), ZMM(0))
221 VMOVAPS(ZMM(16), ZMM(0))
222 VMOVAPS(ZMM(17), ZMM(0)) MOV(R(13), IMM(4*16*L2_PREFETCH_DIST))
223 VMOVAPS(ZMM(18), ZMM(0)) MOV(R(14), IMM(4*32*L2_PREFETCH_DIST))
224 VMOVAPS(ZMM(19), ZMM(0))
225 VMOVAPS(ZMM(20), ZMM(0))
226 VMOVAPS(ZMM(21), ZMM(0))
227 VMOVAPS(ZMM(22), ZMM(0))
228 VMOVAPS(ZMM(23), ZMM(0)) SUB(R(8), IMM(30+L2_PREFETCH_DIST)) //Check if we have over 40 operations to do.
229 VMOVAPS(ZMM(24), ZMM(0)) MOV(R(8), IMM(30))
230 VMOVAPS(ZMM(25), ZMM(0)) MOV(R(9), IMM(4*16)) //amount to increment b* by each iteration
231 VMOVAPS(ZMM(26), ZMM(0)) MOV(R(12), IMM(4*32)) //amount to increment a* by each iteration
232 VMOVAPS(ZMM(27), ZMM(0))
233 VMOVAPS(ZMM(28), ZMM(0))
234 VMOVAPS(ZMM(29), ZMM(0))
235
236 #ifdef MONITORS
237 RDTSC
238 MOV(VAR(midl), EAX)
239 MOV(VAR(midh), EDX)
240 #endif
241
242 JLE(CONSIDER_UNDER_40)
243 SUB(RSI, IMM(30+L2_PREFETCH_DIST))
244
245 //First 30 iterations
246 LABEL(LOOPREFECHCL2)
247 MAIN_LOOP_PC_L2(R(8))
248 JNZ(LOOPREFECHCL2)
249 MOV(RCX, VAR(c))
250
251 //Main Loop.
252 LABEL(LOOPMAIN)
253 MAIN_LOOP(RSI)
254 JNZ(LOOPMAIN)
255
256 //Penultimate 22 iterations.
257 //Break these off from the main loop to avoid prefetching extra shit.
258 MOV(R(14), VAR(a_next))
259 MOV(R(13), VAR(b_next))
260 SUB(R(14), R(15))
261 SUB(R(13), RBX)
262 //Yes, I know 10-20 = -10
263 MOV(RSI, IMM(10+L2_PREFETCH_DIST-20))
264
265 LABEL(LOOPMAIN2)
266 MAIN_LOOP(RSI)
267 JNZ(LOOPMAIN2)
268
269 //Last 10 iterations
270 MOV(R(8), IMM(10))
271
272 LABEL(LOOPREFETCHCL1)
273 MAIN_LOOP_PC_L1(R(8))
274 JNZ(LOOPREFETCHCL1)
275
276 JMP(POSTACCUM)
277
278 //Alternate main loop, with no prefetching of C
279 //Used when <= 40 iterations
280 LABEL(CONSIDER_UNDER_40)
281
282 MOV(RSI, VAR(k))
283 TEST(RSI, RSI)
284 JZ(POSTACCUM)
285
286 LABEL(LOOP_UNDER_40)
287 MAIN_LOOP(RSI)
288 JNZ(LOOP_UNDER_40)
289
290 LABEL(POSTACCUM)
291
292 #ifdef MONITORS
293 RDTSC
294 MOV(VAR(mid2l), EAX)
295 MOV(VAR(mid2h), EDX)
296 #endif
297
298 MOV(R(9), VAR(c)) //load address of c for update
299 MOV(R(12), VAR(alpha)) //load address of alpha
300
301 // Check if C is row stride. If not, jump to the slow scattered update
302 MOV(R(14), VAR(cs_c))
303 DEC(R(14))
304 JNZ(SCATTEREDUPDATE)
305
306 MOV(R(14), VAR(beta))
307 VBROADCASTSS(ZMM(31), MEM(R(14)))
308
309 MOV(EBX, MEM(R(14)))
310 TEST(EBX, EBX)
311 JZ(COLSTORBZ)
312
313 UPDATE_C_4_ROWS( 0, 1, 2, 3)
314 UPDATE_C_4_ROWS( 4, 5, 6, 7)
315 UPDATE_C_4_ROWS( 8, 9,10,11)
316 UPDATE_C_4_ROWS(12,13,14,15)
317 UPDATE_C_4_ROWS(16,17,18,19)
318 UPDATE_C_4_ROWS(20,21,22,23)
319 UPDATE_C_4_ROWS(24,25,26,27)
320 UPDATE_C_2_ROWS(28,29)
321
322 JMP(END)
323
324 LABEL(COLSTORBZ)
325
326 UPDATE_C_BZ_4_ROWS( 0, 1, 2, 3)
327 UPDATE_C_BZ_4_ROWS( 4, 5, 6, 7)
328 UPDATE_C_BZ_4_ROWS( 8, 9,10,11)
329 UPDATE_C_BZ_4_ROWS(12,13,14,15)
330 UPDATE_C_BZ_4_ROWS(16,17,18,19)
331 UPDATE_C_BZ_4_ROWS(20,21,22,23)
332 UPDATE_C_BZ_4_ROWS(24,25,26,27)
333 UPDATE_C_BZ_2_ROWS(28,29)
334
335 JMP(END)
336
337 LABEL(SCATTEREDUPDATE)
338
339 MOV(R(13), VAR(beta))
340 MOV(R(10), VAR(offsetPtr))
341 VMOVAPS(ZMM(30), MEM(R(10)))
342 MOV(EBX, MEM(R(13)))
343 /* Note that this ignores the upper 32 bits in cs_c */
344 VPBROADCASTD(ZMM(31), VAR(cs_c))
345 VPMULLD(ZMM(30), ZMM(31), ZMM(30))
346
347 TEST(EBX, EBX)
348 JZ(SCATTERBZ)
349
350 UPDATE_C_ROW_SCATTERED( 0)
351 UPDATE_C_ROW_SCATTERED( 1)
352 UPDATE_C_ROW_SCATTERED( 2)
353 UPDATE_C_ROW_SCATTERED( 3)
354 UPDATE_C_ROW_SCATTERED( 4)
355 UPDATE_C_ROW_SCATTERED( 5)
356 UPDATE_C_ROW_SCATTERED( 6)
357 UPDATE_C_ROW_SCATTERED( 7)
358 UPDATE_C_ROW_SCATTERED( 8)
359 UPDATE_C_ROW_SCATTERED( 9)
360 UPDATE_C_ROW_SCATTERED(10)
361 UPDATE_C_ROW_SCATTERED(11)
362 UPDATE_C_ROW_SCATTERED(12)
363 UPDATE_C_ROW_SCATTERED(13)
364 UPDATE_C_ROW_SCATTERED(14)
365 UPDATE_C_ROW_SCATTERED(15)
366 UPDATE_C_ROW_SCATTERED(16)
367 UPDATE_C_ROW_SCATTERED(17)
368 UPDATE_C_ROW_SCATTERED(18)
369 UPDATE_C_ROW_SCATTERED(19)
370 UPDATE_C_ROW_SCATTERED(20)
371 UPDATE_C_ROW_SCATTERED(21)
372 UPDATE_C_ROW_SCATTERED(22)
373 UPDATE_C_ROW_SCATTERED(23)
374 UPDATE_C_ROW_SCATTERED(24)
375 UPDATE_C_ROW_SCATTERED(25)
376 UPDATE_C_ROW_SCATTERED(26)
377 UPDATE_C_ROW_SCATTERED(27)
378 UPDATE_C_ROW_SCATTERED(28)
379 UPDATE_C_ROW_SCATTERED(29)
380
381 JMP(END)
382
383 LABEL(SCATTERBZ)
384
385 UPDATE_C_BZ_ROW_SCATTERED( 0)
386 UPDATE_C_BZ_ROW_SCATTERED( 1)
387 UPDATE_C_BZ_ROW_SCATTERED( 2)
388 UPDATE_C_BZ_ROW_SCATTERED( 3)
389 UPDATE_C_BZ_ROW_SCATTERED( 4)
390 UPDATE_C_BZ_ROW_SCATTERED( 5)
391 UPDATE_C_BZ_ROW_SCATTERED( 6)
392 UPDATE_C_BZ_ROW_SCATTERED( 7)
393 UPDATE_C_BZ_ROW_SCATTERED( 8)
394 UPDATE_C_BZ_ROW_SCATTERED( 9)
395 UPDATE_C_BZ_ROW_SCATTERED(10)
396 UPDATE_C_BZ_ROW_SCATTERED(11)
397 UPDATE_C_BZ_ROW_SCATTERED(12)
398 UPDATE_C_BZ_ROW_SCATTERED(13)
399 UPDATE_C_BZ_ROW_SCATTERED(14)
400 UPDATE_C_BZ_ROW_SCATTERED(15)
401 UPDATE_C_BZ_ROW_SCATTERED(16)
402 UPDATE_C_BZ_ROW_SCATTERED(17)
403 UPDATE_C_BZ_ROW_SCATTERED(18)
404 UPDATE_C_BZ_ROW_SCATTERED(19)
405 UPDATE_C_BZ_ROW_SCATTERED(20)
406 UPDATE_C_BZ_ROW_SCATTERED(21)
407 UPDATE_C_BZ_ROW_SCATTERED(22)
408 UPDATE_C_BZ_ROW_SCATTERED(23)
409 UPDATE_C_BZ_ROW_SCATTERED(24)
410 UPDATE_C_BZ_ROW_SCATTERED(25)
411 UPDATE_C_BZ_ROW_SCATTERED(26)
412 UPDATE_C_BZ_ROW_SCATTERED(27)
413 UPDATE_C_BZ_ROW_SCATTERED(28)
414 UPDATE_C_BZ_ROW_SCATTERED(29)
415
416 LABEL(END)
417
418 #ifdef MONITORS
419 RDTSC
420 MOV(VAR(botl), EAX)
421 MOV(VAR(both), EDX)
422 #endif
423 : // output operands
424 #ifdef MONITORS
425 [topl] "=m" (topl),
426 [toph] "=m" (toph),
427 [midl] "=m" (midl),
428 [midh] "=m" (midh),
429 [mid2l] "=m" (mid2l),
430 [mid2h] "=m" (mid2h),
431 [botl] "=m" (botl),
432 [both] "=m" (both)
433 #endif
434 : // input operands
435 [k] "m" (k),
436 [a] "m" (a),
437 [b] "m" (b),
438 [alpha] "m" (alpha),
439 [beta] "m" (beta),
440 [c] "m" (c),
441 [rs_c] "m" (rs_c),
442 [cs_c] "m" (cs_c),
443 [a_next] "m" (a_next),
444 [b_next] "m" (b_next),
445 [offsetPtr] "m" (offsetPtr)
446 : // register clobber list
447 "rax", "rbx", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12",
448 "r13", "r14", "r15", "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5",
449 "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", "zmm11", "zmm12", "zmm13",
450 "zmm14", "zmm15", "zmm16", "zmm17", "zmm18", "zmm19", "zmm20", "zmm21",
451 "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", "zmm27", "zmm28", "zmm29",
452 "zmm30", "zmm31", "memory"
453 );
454
455 #ifdef LOOPMON
456 printf("looptime = \t%d\n", bloopl - tloopl);
457 #endif
458 #ifdef MONITORS
459 dim_t top = ((dim_t)toph << 32) | topl;
460 dim_t mid = ((dim_t)midh << 32) | midl;
461 dim_t mid2 = ((dim_t)mid2h << 32) | mid2l;
462 dim_t bot = ((dim_t)both << 32) | botl;
463 printf("setup =\t%u\tmain loop =\t%u\tcleanup=\t%u\ttotal=\t%u\n", mid - top, mid2 - mid, bot - mid2, bot - top);
464 #endif
465 }
466