1 /*
2
3 BLIS
4 An object-based framework for developing high-performance BLAS-like
5 libraries.
6
7 Copyright (C) 2014, The University of Texas at Austin
8
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are
11 met:
12 - Redistributions of source code must retain the above copyright
13 notice, this list of conditions and the following disclaimer.
14 - Redistributions in binary form must reproduce the above copyright
15 notice, this list of conditions and the following disclaimer in the
16 documentation and/or other materials provided with the distribution.
17 - Neither the name(s) of the copyright holder(s) nor the names of its
18 contributors may be used to endorse or promote products derived
19 from this software without specific prior written permission.
20
21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32
33 */
34
35 #include "blis.h"
36
bli_sgemm_opt_d4x2(dim_t k,float * restrict alpha,float * restrict a,float * restrict b,float * restrict beta,float * restrict c,inc_t rs_c,inc_t cs_c,float * restrict a_next,float * restrict b_next)37 void bli_sgemm_opt_d4x2(
38 dim_t k,
39 float* restrict alpha,
40 float* restrict a,
41 float* restrict b,
42 float* restrict beta,
43 float* restrict c, inc_t rs_c, inc_t cs_c,
44 float* restrict a_next,
45 float* restrict b_next
46 )
47 {
48 bli_check_error_code( BLIS_NOT_YET_IMPLEMENTED );
49 }
50
bli_dgemm_opt_d4x2(dim_t k,double * restrict alpha,double * restrict a,double * restrict b,double * restrict beta,double * restrict c,inc_t rs_c,inc_t cs_c,double * restrict a_next,double * restrict b_next)51 void bli_dgemm_opt_d4x2(
52 dim_t k,
53 double* restrict alpha,
54 double* restrict a,
55 double* restrict b,
56 double* restrict beta,
57 double* restrict c, inc_t rs_c, inc_t cs_c,
58 double* restrict a_next,
59 double* restrict b_next
60 )
61 {
62 dim_t k_iter;
63 dim_t k_left;
64
65 k_iter = k / 8;
66 k_left = k % 8;
67
68 __asm__ volatile
69 (
70 " \n\t"
71 "movl %6, %%ecx \n\t" // load address of c
72 " \n\t"
73 "movl %8, %%edi \n\t" // load cs_c
74 "sall $3, %%edi \n\t" // cs_c *= sizeof(double)
75 " \n\t"
76 "prefetcht0 (%%ecx) \n\t" // give a T0 prefetch hint for c00.
77 "prefetcht0 (%%ecx,%%edi) \n\t" // give a T0 prefetch hint for c01.
78 " \n\t"
79 "movl %2, %%eax \n\t" // load address of a.
80 "movl %3, %%ebx \n\t" // load address of b.
81 " \n\t"
82 "addl $8 * 16, %%eax \n\t" // increment pointers to allow byte
83 "addl $8 * 16, %%ebx \n\t" // offsets in the unrolled iterations.
84 " \n\t"
85 "movapd -8 * 16(%%eax), %%xmm0 \n\t" // initialize loop by pre-loading elements
86 "movapd -4 * 16(%%eax), %%xmm3 \n\t" // of a.
87 " \n\t"
88 "pxor %%xmm4, %%xmm4 \n\t"
89 "pxor %%xmm5, %%xmm5 \n\t"
90 "pxor %%xmm6, %%xmm6 \n\t"
91 "pxor %%xmm7, %%xmm7 \n\t"
92 " \n\t"
93 " \n\t"
94 " \n\t"
95 "movl %0, %%esi \n\t" // i = k_iter;
96 "testl %%esi, %%esi \n\t" // check i via logical AND.
97 "je .CONSIDERKLEFT \n\t" // if i == 0, jump to code that
98 " \n\t" // contains the k_left loop.
99 " \n\t"
100 " \n\t"
101 ".LOOPKITER: \n\t" // MAIN LOOP
102 " \n\t"
103 "movapd -8 * 16(%%ebx), %%xmm1 \n\t" // iteration 0
104 "movapd %%xmm1, %%xmm2 \n\t"
105 "mulpd %%xmm0, %%xmm1 \n\t"
106 "addpd %%xmm1, %%xmm4 \n\t"
107 "movapd -7 * 16(%%ebx), %%xmm1 \n\t"
108 "mulpd %%xmm1, %%xmm0 \n\t"
109 "addpd %%xmm0, %%xmm5 \n\t"
110 "movapd -7 * 16(%%eax), %%xmm0 \n\t"
111 "mulpd %%xmm0, %%xmm2 \n\t"
112 "mulpd %%xmm0, %%xmm1 \n\t"
113 "movapd -6 * 16(%%eax), %%xmm0 \n\t"
114 "addpd %%xmm2, %%xmm6 \n\t"
115 "addpd %%xmm1, %%xmm7 \n\t"
116 " \n\t"
117 "movapd -6 * 16(%%ebx), %%xmm1 \n\t" // iteration 1
118 "movapd %%xmm1, %%xmm2 \n\t"
119 "mulpd %%xmm0, %%xmm1 \n\t"
120 "addpd %%xmm1, %%xmm4 \n\t"
121 "movapd -5 * 16(%%ebx), %%xmm1 \n\t"
122 "mulpd %%xmm1, %%xmm0 \n\t"
123 "addpd %%xmm0, %%xmm5 \n\t"
124 "movapd -5 * 16(%%eax), %%xmm0 \n\t"
125 "mulpd %%xmm0, %%xmm2 \n\t"
126 "mulpd %%xmm0, %%xmm1 \n\t"
127 "movapd 0 * 16(%%eax), %%xmm0 \n\t"
128 "addpd %%xmm2, %%xmm6 \n\t"
129 "addpd %%xmm1, %%xmm7 \n\t"
130 " \n\t"
131 "movapd -4 * 16(%%ebx), %%xmm1 \n\t" // iteration 2
132 "movapd %%xmm1, %%xmm2 \n\t"
133 "mulpd %%xmm3, %%xmm1 \n\t"
134 "addpd %%xmm1, %%xmm4 \n\t"
135 "movapd -3 * 16(%%ebx), %%xmm1 \n\t"
136 "mulpd %%xmm1, %%xmm3 \n\t"
137 "addpd %%xmm3, %%xmm5 \n\t"
138 "movapd -3 * 16(%%eax), %%xmm3 \n\t"
139 "mulpd %%xmm3, %%xmm2 \n\t"
140 "mulpd %%xmm3, %%xmm1 \n\t"
141 "movapd -2 * 16(%%eax), %%xmm3 \n\t"
142 "addpd %%xmm2, %%xmm6 \n\t"
143 "addpd %%xmm1, %%xmm7 \n\t"
144 " \n\t"
145 "movapd -2 * 16(%%ebx), %%xmm1 \n\t" // iteration 3
146 "movapd %%xmm1, %%xmm2 \n\t"
147 "mulpd %%xmm3, %%xmm1 \n\t"
148 "addpd %%xmm1, %%xmm4 \n\t"
149 "movapd -1 * 16(%%ebx), %%xmm1 \n\t"
150 "mulpd %%xmm1, %%xmm3 \n\t"
151 "addpd %%xmm3, %%xmm5 \n\t"
152 "movapd -1 * 16(%%eax), %%xmm3 \n\t"
153 "mulpd %%xmm3, %%xmm2 \n\t"
154 "mulpd %%xmm3, %%xmm1 \n\t"
155 "movapd 4 * 16(%%eax), %%xmm3 \n\t"
156 "addpd %%xmm2, %%xmm6 \n\t"
157 "addpd %%xmm1, %%xmm7 \n\t"
158 " \n\t"
159 "movapd 0 * 16(%%ebx), %%xmm1 \n\t" // iteration 4
160 "movapd %%xmm1, %%xmm2 \n\t"
161 "mulpd %%xmm0, %%xmm1 \n\t"
162 "addpd %%xmm1, %%xmm4 \n\t"
163 "movapd 1 * 16(%%ebx), %%xmm1 \n\t"
164 "mulpd %%xmm1, %%xmm0 \n\t"
165 "addpd %%xmm0, %%xmm5 \n\t"
166 "movapd 1 * 16(%%eax), %%xmm0 \n\t"
167 "mulpd %%xmm0, %%xmm2 \n\t"
168 "mulpd %%xmm0, %%xmm1 \n\t"
169 "movapd 2 * 16(%%eax), %%xmm0 \n\t"
170 "addpd %%xmm2, %%xmm6 \n\t"
171 "addpd %%xmm1, %%xmm7 \n\t"
172 " \n\t"
173 "movapd 2 * 16(%%ebx), %%xmm1 \n\t" // iteration 5
174 "movapd %%xmm1, %%xmm2 \n\t"
175 "mulpd %%xmm0, %%xmm1 \n\t"
176 "addpd %%xmm1, %%xmm4 \n\t"
177 "movapd 3 * 16(%%ebx), %%xmm1 \n\t"
178 "mulpd %%xmm1, %%xmm0 \n\t"
179 "addpd %%xmm0, %%xmm5 \n\t"
180 "movapd 3 * 16(%%eax), %%xmm0 \n\t"
181 "mulpd %%xmm0, %%xmm2 \n\t"
182 "mulpd %%xmm0, %%xmm1 \n\t"
183 "movapd 8 * 16(%%eax), %%xmm0 \n\t"
184 "addpd %%xmm2, %%xmm6 \n\t"
185 "addpd %%xmm1, %%xmm7 \n\t"
186 " \n\t"
187 "movapd 4 * 16(%%ebx), %%xmm1 \n\t" // iteration 6
188 "movapd %%xmm1, %%xmm2 \n\t"
189 "mulpd %%xmm3, %%xmm1 \n\t"
190 "addpd %%xmm1, %%xmm4 \n\t"
191 "movapd 5 * 16(%%ebx), %%xmm1 \n\t"
192 "mulpd %%xmm1, %%xmm3 \n\t"
193 "addpd %%xmm3, %%xmm5 \n\t"
194 "movapd 5 * 16(%%eax), %%xmm3 \n\t"
195 "mulpd %%xmm3, %%xmm2 \n\t"
196 "mulpd %%xmm3, %%xmm1 \n\t"
197 "movapd 6 * 16(%%eax), %%xmm3 \n\t"
198 "addl $8 * 4 * 8, %%eax \n\t" // a += 8*4 (unroll x mr)
199 "addpd %%xmm2, %%xmm6 \n\t"
200 "addpd %%xmm1, %%xmm7 \n\t"
201 " \n\t"
202 "movapd 6 * 16(%%ebx), %%xmm1 \n\t" // iteration 7
203 "movapd %%xmm1, %%xmm2 \n\t"
204 "mulpd %%xmm3, %%xmm1 \n\t"
205 "addpd %%xmm1, %%xmm4 \n\t"
206 "movapd 7 * 16(%%ebx), %%xmm1 \n\t"
207 "addl $8 * 2 * 2 * 8, %%ebx \n\t" // b += 8*2*2 (unroll x nr x ndup)
208 "mulpd %%xmm1, %%xmm3 \n\t"
209 "addpd %%xmm3, %%xmm5 \n\t"
210 "movapd -9 * 16(%%eax), %%xmm3 \n\t"
211 "mulpd %%xmm3, %%xmm2 \n\t"
212 "mulpd %%xmm3, %%xmm1 \n\t"
213 "decl %%esi \n\t" // i -= 1;
214 "movapd -4 * 16(%%eax), %%xmm3 \n\t"
215 "addpd %%xmm2, %%xmm6 \n\t"
216 "addpd %%xmm1, %%xmm7 \n\t"
217 " \n\t"
218 "jne .LOOPKITER \n\t" // iterate again if i != 0.
219 " \n\t"
220 " \n\t"
221 " \n\t"
222 ".CONSIDERKLEFT: \n\t"
223 " \n\t"
224 "movl %1, %%esi \n\t" // i = k_left;
225 "testl %%esi, %%esi \n\t" // check i via logical AND.
226 "je .POSTACCUM \n\t" // if i == 0, we're done; jump to end.
227 " \n\t" // else, we prepare to enter k_left loop.
228 " \n\t"
229 " \n\t"
230 ".LOOPKLEFT: \n\t" // EDGE LOOP
231 " \n\t"
232 "movapd -8 * 16(%%ebx), %%xmm1 \n\t" // iteration i
233 "movapd %%xmm1, %%xmm2 \n\t"
234 "mulpd %%xmm0, %%xmm1 \n\t"
235 "addpd %%xmm1, %%xmm4 \n\t"
236 "movapd -7 * 16(%%ebx), %%xmm1 \n\t"
237 "addl $1 * 2 * 2 * 8, %%ebx \n\t" // b += 2*2 (1 x nr x ndup)
238 "mulpd %%xmm1, %%xmm0 \n\t"
239 "addpd %%xmm0, %%xmm5 \n\t"
240 "movapd -7 * 16(%%eax), %%xmm0 \n\t"
241 "mulpd %%xmm0, %%xmm2 \n\t"
242 "mulpd %%xmm0, %%xmm1 \n\t"
243 "movapd -6 * 16(%%eax), %%xmm0 \n\t"
244 "addl $1 * 4 * 8, %%eax \n\t" // a += 4 (1 x mr)
245 "addpd %%xmm2, %%xmm6 \n\t"
246 "addpd %%xmm1, %%xmm7 \n\t"
247 " \n\t"
248 "decl %%esi \n\t" // i -= 1;
249 "jne .LOOPKLEFT \n\t" // iterate again if i != 0.
250 " \n\t"
251 " \n\t"
252 " \n\t"
253 ".POSTACCUM: \n\t"
254 " \n\t"
255 "movl %4, %%eax \n\t" // load address of alpha
256 "movl %5, %%ebx \n\t" // load address of beta
257 "movddup (%%eax), %%xmm2 \n\t" // load alpha and duplicate
258 "movddup (%%ebx), %%xmm3 \n\t" // load beta and duplicate
259 " \n\t"
260 " \n\t"
261 " \n\t"
262 "movl %7, %%esi \n\t" // load rs_c
263 "sall $3, %%esi \n\t" // rs_c *= sizeof(double)
264 " \n\t"
265 "leal (%%ecx,%%esi,2), %%edx \n\t" // load address of c + 2*rs_c;
266 " \n\t"
267 " \n\t"
268 " \n\t"
269 "movlpd (%%ecx), %%xmm0 \n\t" // load c00 and c10,
270 "movhpd (%%ecx,%%esi), %%xmm0 \n\t"
271 "mulpd %%xmm2, %%xmm4 \n\t" // scale by alpha,
272 "mulpd %%xmm3, %%xmm0 \n\t" // scale by beta,
273 "addpd %%xmm4, %%xmm0 \n\t" // add the gemm result,
274 "movlpd %%xmm0, (%%ecx) \n\t" // and store back to memory.
275 "movhpd %%xmm0, (%%ecx,%%esi) \n\t"
276 "addl %%edi, %%ecx \n\t"
277 " \n\t"
278 "movlpd (%%edx), %%xmm1 \n\t" // load c01 and c11,
279 "movhpd (%%edx,%%esi), %%xmm1 \n\t"
280 "mulpd %%xmm2, %%xmm6 \n\t" // scale by alpha,
281 "mulpd %%xmm3, %%xmm1 \n\t" // scale by beta,
282 "addpd %%xmm6, %%xmm1 \n\t" // add the gemm result,
283 "movlpd %%xmm1, (%%edx) \n\t" // and store back to memory.
284 "movhpd %%xmm1, (%%edx,%%esi) \n\t"
285 "addl %%edi, %%edx \n\t"
286 " \n\t"
287 "movlpd (%%ecx), %%xmm0 \n\t" // load c20 and c30,
288 "movhpd (%%ecx,%%esi), %%xmm0 \n\t"
289 "mulpd %%xmm2, %%xmm5 \n\t" // scale by alpha,
290 "mulpd %%xmm3, %%xmm0 \n\t" // scale by beta,
291 "addpd %%xmm5, %%xmm0 \n\t" // add the gemm result,
292 "movlpd %%xmm0, (%%ecx) \n\t" // and store back to memory.
293 "movhpd %%xmm0, (%%ecx,%%esi) \n\t"
294 " \n\t"
295 "movlpd (%%edx), %%xmm1 \n\t" // load c21 and c31,
296 "movhpd (%%edx,%%esi), %%xmm1 \n\t"
297 "mulpd %%xmm2, %%xmm7 \n\t" // scale by alpha,
298 "mulpd %%xmm3, %%xmm1 \n\t" // scale by beta,
299 "addpd %%xmm7, %%xmm1 \n\t" // add the gemm result,
300 "movlpd %%xmm1, (%%edx) \n\t" // and store back to memory.
301 "movhpd %%xmm1, (%%edx,%%esi) \n\t"
302 " \n\t"
303 " \n\t"
304 " \n\t"
305
306 : // output operands (none)
307 : // input operands
308 "m" (k_iter),
309 "m" (k_left),
310 "m" (a),
311 "m" (b),
312 "m" (alpha),
313 "m" (beta),
314 "m" (c),
315 "m" (rs_c),
316 "m" (cs_c)
317 : // register clobber list
318 "eax", "ebx", "ecx", "edx", "esi", "edi",
319 "xmm0", "xmm1", "xmm2", "xmm3",
320 "xmm4", "xmm5", "xmm6", "xmm7",
321 "memory"
322 );
323
324 }
325
bli_cgemm_opt_d4x2(dim_t k,scomplex * restrict alpha,scomplex * restrict a,scomplex * restrict b,scomplex * restrict beta,scomplex * restrict c,inc_t rs_c,inc_t cs_c,scomplex * restrict a_next,scomplex * restrict b_next)326 void bli_cgemm_opt_d4x2(
327 dim_t k,
328 scomplex* restrict alpha,
329 scomplex* restrict a,
330 scomplex* restrict b,
331 scomplex* restrict beta,
332 scomplex* restrict c, inc_t rs_c, inc_t cs_c,
333 scomplex* restrict a_next,
334 scomplex* restrict b_next
335 )
336 {
337 bli_check_error_code( BLIS_NOT_YET_IMPLEMENTED );
338 }
339
bli_zgemm_opt_d4x2(dim_t k,dcomplex * restrict alpha,dcomplex * restrict a,dcomplex * restrict b,dcomplex * restrict beta,dcomplex * restrict c,inc_t rs_c,inc_t cs_c,dcomplex * restrict a_next,dcomplex * restrict b_next)340 void bli_zgemm_opt_d4x2(
341 dim_t k,
342 dcomplex* restrict alpha,
343 dcomplex* restrict a,
344 dcomplex* restrict b,
345 dcomplex* restrict beta,
346 dcomplex* restrict c, inc_t rs_c, inc_t cs_c,
347 dcomplex* restrict a_next,
348 dcomplex* restrict b_next
349 )
350 {
351 bli_check_error_code( BLIS_NOT_YET_IMPLEMENTED );
352 }
353
354