1 /**************************************************************************************************
2 * *
3 * This file is part of BLASFEO. *
4 * *
5 * BLASFEO -- BLAS For Embedded Optimization. *
6 * Copyright (C) 2019 by Gianluca Frison. *
7 * Developed at IMTEK (University of Freiburg) under the supervision of Moritz Diehl. *
8 * All rights reserved. *
9 * *
10 * The 2-Clause BSD License *
11 * *
12 * Redistribution and use in source and binary forms, with or without *
13 * modification, are permitted provided that the following conditions are met: *
14 * *
15 * 1. Redistributions of source code must retain the above copyright notice, this *
16 * list of conditions and the following disclaimer. *
17 * 2. Redistributions in binary form must reproduce the above copyright notice, *
18 * this list of conditions and the following disclaimer in the documentation *
19 * and/or other materials provided with the distribution. *
20 * *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND *
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED *
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE *
24 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR *
25 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES *
26 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; *
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND *
28 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT *
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS *
30 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *
31 * *
32 * Author: Gianluca Frison, gianluca.frison (at) imtek.uni-freiburg.de *
33 * *
34 **************************************************************************************************/
35
36
37 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_sgemm_nt_4x4_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)38 void kernel_sgemm_nt_4x4_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
39 {
40
41 const int bs = 4;
42
43 float
44 a_0, a_1, a_2, a_3,
45 b_0, b_1, b_2, b_3;
46
47 #if defined(TARGET_GENERIC)
48 float CC[16] = {0};
49 #else
50 ALIGNED( float CC[16], 64 ) = {0};
51 #endif
52
53 int k;
54
55 for(k=0; k<kmax-3; k+=4)
56 {
57
58 // k = 0
59
60 a_0 = A[0+lda*0];
61 a_1 = A[1+lda*0];
62 a_2 = A[2+lda*0];
63 a_3 = A[3+lda*0];
64
65 b_0 = B[0+ldb*0];
66 b_1 = B[1+ldb*0];
67 b_2 = B[2+ldb*0];
68 b_3 = B[3+ldb*0];
69
70 CC[0+bs*0] += a_0 * b_0;
71 CC[1+bs*0] += a_1 * b_0;
72 CC[2+bs*0] += a_2 * b_0;
73 CC[3+bs*0] += a_3 * b_0;
74
75 CC[0+bs*1] += a_0 * b_1;
76 CC[1+bs*1] += a_1 * b_1;
77 CC[2+bs*1] += a_2 * b_1;
78 CC[3+bs*1] += a_3 * b_1;
79
80 CC[0+bs*2] += a_0 * b_2;
81 CC[1+bs*2] += a_1 * b_2;
82 CC[2+bs*2] += a_2 * b_2;
83 CC[3+bs*2] += a_3 * b_2;
84
85 CC[0+bs*3] += a_0 * b_3;
86 CC[1+bs*3] += a_1 * b_3;
87 CC[2+bs*3] += a_2 * b_3;
88 CC[3+bs*3] += a_3 * b_3;
89
90
91 // k = 1
92
93 a_0 = A[0+lda*1];
94 a_1 = A[1+lda*1];
95 a_2 = A[2+lda*1];
96 a_3 = A[3+lda*1];
97
98 b_0 = B[0+ldb*1];
99 b_1 = B[1+ldb*1];
100 b_2 = B[2+ldb*1];
101 b_3 = B[3+ldb*1];
102
103 CC[0+bs*0] += a_0 * b_0;
104 CC[1+bs*0] += a_1 * b_0;
105 CC[2+bs*0] += a_2 * b_0;
106 CC[3+bs*0] += a_3 * b_0;
107
108 CC[0+bs*1] += a_0 * b_1;
109 CC[1+bs*1] += a_1 * b_1;
110 CC[2+bs*1] += a_2 * b_1;
111 CC[3+bs*1] += a_3 * b_1;
112
113 CC[0+bs*2] += a_0 * b_2;
114 CC[1+bs*2] += a_1 * b_2;
115 CC[2+bs*2] += a_2 * b_2;
116 CC[3+bs*2] += a_3 * b_2;
117
118 CC[0+bs*3] += a_0 * b_3;
119 CC[1+bs*3] += a_1 * b_3;
120 CC[2+bs*3] += a_2 * b_3;
121 CC[3+bs*3] += a_3 * b_3;
122
123
124 // k = 2
125
126 a_0 = A[0+lda*2];
127 a_1 = A[1+lda*2];
128 a_2 = A[2+lda*2];
129 a_3 = A[3+lda*2];
130
131 b_0 = B[0+ldb*2];
132 b_1 = B[1+ldb*2];
133 b_2 = B[2+ldb*2];
134 b_3 = B[3+ldb*2];
135
136 CC[0+bs*0] += a_0 * b_0;
137 CC[1+bs*0] += a_1 * b_0;
138 CC[2+bs*0] += a_2 * b_0;
139 CC[3+bs*0] += a_3 * b_0;
140
141 CC[0+bs*1] += a_0 * b_1;
142 CC[1+bs*1] += a_1 * b_1;
143 CC[2+bs*1] += a_2 * b_1;
144 CC[3+bs*1] += a_3 * b_1;
145
146 CC[0+bs*2] += a_0 * b_2;
147 CC[1+bs*2] += a_1 * b_2;
148 CC[2+bs*2] += a_2 * b_2;
149 CC[3+bs*2] += a_3 * b_2;
150
151 CC[0+bs*3] += a_0 * b_3;
152 CC[1+bs*3] += a_1 * b_3;
153 CC[2+bs*3] += a_2 * b_3;
154 CC[3+bs*3] += a_3 * b_3;
155
156
157 // k = 3
158
159 a_0 = A[0+lda*3];
160 a_1 = A[1+lda*3];
161 a_2 = A[2+lda*3];
162 a_3 = A[3+lda*3];
163
164 b_0 = B[0+ldb*3];
165 b_1 = B[1+ldb*3];
166 b_2 = B[2+ldb*3];
167 b_3 = B[3+ldb*3];
168
169 CC[0+bs*0] += a_0 * b_0;
170 CC[1+bs*0] += a_1 * b_0;
171 CC[2+bs*0] += a_2 * b_0;
172 CC[3+bs*0] += a_3 * b_0;
173
174 CC[0+bs*1] += a_0 * b_1;
175 CC[1+bs*1] += a_1 * b_1;
176 CC[2+bs*1] += a_2 * b_1;
177 CC[3+bs*1] += a_3 * b_1;
178
179 CC[0+bs*2] += a_0 * b_2;
180 CC[1+bs*2] += a_1 * b_2;
181 CC[2+bs*2] += a_2 * b_2;
182 CC[3+bs*2] += a_3 * b_2;
183
184 CC[0+bs*3] += a_0 * b_3;
185 CC[1+bs*3] += a_1 * b_3;
186 CC[2+bs*3] += a_2 * b_3;
187 CC[3+bs*3] += a_3 * b_3;
188
189 A += 4*lda;
190 B += 4*ldb;
191
192 }
193
194 for(; k<kmax; k++)
195 {
196
197 // k = 0
198
199 a_0 = A[0+lda*0];
200 a_1 = A[1+lda*0];
201 a_2 = A[2+lda*0];
202 a_3 = A[3+lda*0];
203
204 b_0 = B[0+ldb*0];
205 b_1 = B[1+ldb*0];
206 b_2 = B[2+ldb*0];
207 b_3 = B[3+ldb*0];
208
209 CC[0+bs*0] += a_0 * b_0;
210 CC[1+bs*0] += a_1 * b_0;
211 CC[2+bs*0] += a_2 * b_0;
212 CC[3+bs*0] += a_3 * b_0;
213
214 CC[0+bs*1] += a_0 * b_1;
215 CC[1+bs*1] += a_1 * b_1;
216 CC[2+bs*1] += a_2 * b_1;
217 CC[3+bs*1] += a_3 * b_1;
218
219 CC[0+bs*2] += a_0 * b_2;
220 CC[1+bs*2] += a_1 * b_2;
221 CC[2+bs*2] += a_2 * b_2;
222 CC[3+bs*2] += a_3 * b_2;
223
224 CC[0+bs*3] += a_0 * b_3;
225 CC[1+bs*3] += a_1 * b_3;
226 CC[2+bs*3] += a_2 * b_3;
227 CC[3+bs*3] += a_3 * b_3;
228
229 A += 1*lda;
230 B += 1*ldb;
231
232 }
233
234 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
235 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
236 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
237 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
238
239 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
240 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
241 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
242 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
243
244 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
245 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
246 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
247 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
248
249 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
250 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
251 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
252 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
253
254 return;
255
256 }
257 //#endif
258
259
260
kernel_sgemm_nt_4x3_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)261 static void kernel_sgemm_nt_4x3_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
262 {
263
264 const int bs = 4;
265
266 float
267 a_0, a_1, a_2, a_3,
268 b_0, b_1, b_2;
269
270 #if defined(TARGET_GENERIC)
271 float CC[16] = {0};
272 #else
273 ALIGNED( float CC[16], 64 ) = {0};
274 #endif
275
276 int k;
277
278 for(k=0; k<kmax-3; k+=4)
279 {
280
281 // k = 0
282
283 a_0 = A[0+lda*0];
284 a_1 = A[1+lda*0];
285 a_2 = A[2+lda*0];
286 a_3 = A[3+lda*0];
287
288 b_0 = B[0+ldb*0];
289 b_1 = B[1+ldb*0];
290 b_2 = B[2+ldb*0];
291
292 CC[0+bs*0] += a_0 * b_0;
293 CC[1+bs*0] += a_1 * b_0;
294 CC[2+bs*0] += a_2 * b_0;
295 CC[3+bs*0] += a_3 * b_0;
296
297 CC[0+bs*1] += a_0 * b_1;
298 CC[1+bs*1] += a_1 * b_1;
299 CC[2+bs*1] += a_2 * b_1;
300 CC[3+bs*1] += a_3 * b_1;
301
302 CC[0+bs*2] += a_0 * b_2;
303 CC[1+bs*2] += a_1 * b_2;
304 CC[2+bs*2] += a_2 * b_2;
305 CC[3+bs*2] += a_3 * b_2;
306
307
308 // k = 1
309
310 a_0 = A[0+lda*1];
311 a_1 = A[1+lda*1];
312 a_2 = A[2+lda*1];
313 a_3 = A[3+lda*1];
314
315 b_0 = B[0+ldb*1];
316 b_1 = B[1+ldb*1];
317 b_2 = B[2+ldb*1];
318
319 CC[0+bs*0] += a_0 * b_0;
320 CC[1+bs*0] += a_1 * b_0;
321 CC[2+bs*0] += a_2 * b_0;
322 CC[3+bs*0] += a_3 * b_0;
323
324 CC[0+bs*1] += a_0 * b_1;
325 CC[1+bs*1] += a_1 * b_1;
326 CC[2+bs*1] += a_2 * b_1;
327 CC[3+bs*1] += a_3 * b_1;
328
329 CC[0+bs*2] += a_0 * b_2;
330 CC[1+bs*2] += a_1 * b_2;
331 CC[2+bs*2] += a_2 * b_2;
332 CC[3+bs*2] += a_3 * b_2;
333
334
335 // k = 2
336
337 a_0 = A[0+lda*2];
338 a_1 = A[1+lda*2];
339 a_2 = A[2+lda*2];
340 a_3 = A[3+lda*2];
341
342 b_0 = B[0+ldb*2];
343 b_1 = B[1+ldb*2];
344 b_2 = B[2+ldb*2];
345
346 CC[0+bs*0] += a_0 * b_0;
347 CC[1+bs*0] += a_1 * b_0;
348 CC[2+bs*0] += a_2 * b_0;
349 CC[3+bs*0] += a_3 * b_0;
350
351 CC[0+bs*1] += a_0 * b_1;
352 CC[1+bs*1] += a_1 * b_1;
353 CC[2+bs*1] += a_2 * b_1;
354 CC[3+bs*1] += a_3 * b_1;
355
356 CC[0+bs*2] += a_0 * b_2;
357 CC[1+bs*2] += a_1 * b_2;
358 CC[2+bs*2] += a_2 * b_2;
359 CC[3+bs*2] += a_3 * b_2;
360
361
362 // k = 3
363
364 a_0 = A[0+lda*3];
365 a_1 = A[1+lda*3];
366 a_2 = A[2+lda*3];
367 a_3 = A[3+lda*3];
368
369 b_0 = B[0+ldb*3];
370 b_1 = B[1+ldb*3];
371 b_2 = B[2+ldb*3];
372
373 CC[0+bs*0] += a_0 * b_0;
374 CC[1+bs*0] += a_1 * b_0;
375 CC[2+bs*0] += a_2 * b_0;
376 CC[3+bs*0] += a_3 * b_0;
377
378 CC[0+bs*1] += a_0 * b_1;
379 CC[1+bs*1] += a_1 * b_1;
380 CC[2+bs*1] += a_2 * b_1;
381 CC[3+bs*1] += a_3 * b_1;
382
383 CC[0+bs*2] += a_0 * b_2;
384 CC[1+bs*2] += a_1 * b_2;
385 CC[2+bs*2] += a_2 * b_2;
386 CC[3+bs*2] += a_3 * b_2;
387
388 A += 4*lda;
389 B += 4*ldb;
390
391 }
392
393 for(; k<kmax; k++)
394 {
395
396 // k = 0
397
398 a_0 = A[0+lda*0];
399 a_1 = A[1+lda*0];
400 a_2 = A[2+lda*0];
401 a_3 = A[3+lda*0];
402
403 b_0 = B[0+ldb*0];
404 b_1 = B[1+ldb*0];
405 b_2 = B[2+ldb*0];
406
407 CC[0+bs*0] += a_0 * b_0;
408 CC[1+bs*0] += a_1 * b_0;
409 CC[2+bs*0] += a_2 * b_0;
410 CC[3+bs*0] += a_3 * b_0;
411
412 CC[0+bs*1] += a_0 * b_1;
413 CC[1+bs*1] += a_1 * b_1;
414 CC[2+bs*1] += a_2 * b_1;
415 CC[3+bs*1] += a_3 * b_1;
416
417 CC[0+bs*2] += a_0 * b_2;
418 CC[1+bs*2] += a_1 * b_2;
419 CC[2+bs*2] += a_2 * b_2;
420 CC[3+bs*2] += a_3 * b_2;
421
422 A += 1*lda;
423 B += 1*ldb;
424
425 }
426
427 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
428 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
429 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
430 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
431
432 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
433 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
434 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
435 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
436
437 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
438 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
439 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
440 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
441
442 return;
443
444 }
445
446
447
kernel_sgemm_nt_4x2_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)448 static void kernel_sgemm_nt_4x2_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
449 {
450
451 const int bs = 4;
452
453 float
454 a_0, a_1, a_2, a_3,
455 b_0, b_1;
456
457 #if defined(TARGET_GENERIC)
458 float CC[16] = {0};
459 #else
460 ALIGNED( float CC[16], 64 ) = {0};
461 #endif
462
463 int k;
464
465 for(k=0; k<kmax-3; k+=4)
466 {
467
468 // k = 0
469
470 a_0 = A[0+lda*0];
471 a_1 = A[1+lda*0];
472 a_2 = A[2+lda*0];
473 a_3 = A[3+lda*0];
474
475 b_0 = B[0+ldb*0];
476 b_1 = B[1+ldb*0];
477
478 CC[0+bs*0] += a_0 * b_0;
479 CC[1+bs*0] += a_1 * b_0;
480 CC[2+bs*0] += a_2 * b_0;
481 CC[3+bs*0] += a_3 * b_0;
482
483 CC[0+bs*1] += a_0 * b_1;
484 CC[1+bs*1] += a_1 * b_1;
485 CC[2+bs*1] += a_2 * b_1;
486 CC[3+bs*1] += a_3 * b_1;
487
488
489 // k = 1
490
491 a_0 = A[0+lda*1];
492 a_1 = A[1+lda*1];
493 a_2 = A[2+lda*1];
494 a_3 = A[3+lda*1];
495
496 b_0 = B[0+ldb*1];
497 b_1 = B[1+ldb*1];
498
499 CC[0+bs*0] += a_0 * b_0;
500 CC[1+bs*0] += a_1 * b_0;
501 CC[2+bs*0] += a_2 * b_0;
502 CC[3+bs*0] += a_3 * b_0;
503
504 CC[0+bs*1] += a_0 * b_1;
505 CC[1+bs*1] += a_1 * b_1;
506 CC[2+bs*1] += a_2 * b_1;
507 CC[3+bs*1] += a_3 * b_1;
508
509
510 // k = 2
511
512 a_0 = A[0+lda*2];
513 a_1 = A[1+lda*2];
514 a_2 = A[2+lda*2];
515 a_3 = A[3+lda*2];
516
517 b_0 = B[0+ldb*2];
518 b_1 = B[1+ldb*2];
519
520 CC[0+bs*0] += a_0 * b_0;
521 CC[1+bs*0] += a_1 * b_0;
522 CC[2+bs*0] += a_2 * b_0;
523 CC[3+bs*0] += a_3 * b_0;
524
525 CC[0+bs*1] += a_0 * b_1;
526 CC[1+bs*1] += a_1 * b_1;
527 CC[2+bs*1] += a_2 * b_1;
528 CC[3+bs*1] += a_3 * b_1;
529
530
531 // k = 3
532
533 a_0 = A[0+lda*3];
534 a_1 = A[1+lda*3];
535 a_2 = A[2+lda*3];
536 a_3 = A[3+lda*3];
537
538 b_0 = B[0+ldb*3];
539 b_1 = B[1+ldb*3];
540
541 CC[0+bs*0] += a_0 * b_0;
542 CC[1+bs*0] += a_1 * b_0;
543 CC[2+bs*0] += a_2 * b_0;
544 CC[3+bs*0] += a_3 * b_0;
545
546 CC[0+bs*1] += a_0 * b_1;
547 CC[1+bs*1] += a_1 * b_1;
548 CC[2+bs*1] += a_2 * b_1;
549 CC[3+bs*1] += a_3 * b_1;
550
551 A += 4*lda;
552 B += 4*ldb;
553
554 }
555
556 for(; k<kmax; k++)
557 {
558
559 // k = 0
560
561 a_0 = A[0+lda*0];
562 a_1 = A[1+lda*0];
563 a_2 = A[2+lda*0];
564 a_3 = A[3+lda*0];
565
566 b_0 = B[0+ldb*0];
567 b_1 = B[1+ldb*0];
568
569 CC[0+bs*0] += a_0 * b_0;
570 CC[1+bs*0] += a_1 * b_0;
571 CC[2+bs*0] += a_2 * b_0;
572 CC[3+bs*0] += a_3 * b_0;
573
574 CC[0+bs*1] += a_0 * b_1;
575 CC[1+bs*1] += a_1 * b_1;
576 CC[2+bs*1] += a_2 * b_1;
577 CC[3+bs*1] += a_3 * b_1;
578
579 A += 1*lda;
580 B += 1*ldb;
581
582 }
583
584 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
585 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
586 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
587 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
588
589 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
590 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
591 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
592 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
593
594 return;
595
596 }
597
598
599
kernel_sgemm_nt_4x1_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)600 static void kernel_sgemm_nt_4x1_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
601 {
602
603 const int bs = 4;
604
605 float
606 a_0, a_1, a_2, a_3,
607 b_0;
608
609 #if defined(TARGET_GENERIC)
610 float CC[16] = {0};
611 #else
612 ALIGNED( float CC[16], 64 ) = {0};
613 #endif
614
615 int k;
616
617 for(k=0; k<kmax-3; k+=4)
618 {
619
620 // k = 0
621
622 a_0 = A[0+lda*0];
623 a_1 = A[1+lda*0];
624 a_2 = A[2+lda*0];
625 a_3 = A[3+lda*0];
626
627 b_0 = B[0+ldb*0];
628
629 CC[0+bs*0] += a_0 * b_0;
630 CC[1+bs*0] += a_1 * b_0;
631 CC[2+bs*0] += a_2 * b_0;
632 CC[3+bs*0] += a_3 * b_0;
633
634
635 // k = 1
636
637 a_0 = A[0+lda*1];
638 a_1 = A[1+lda*1];
639 a_2 = A[2+lda*1];
640 a_3 = A[3+lda*1];
641
642 b_0 = B[0+ldb*1];
643
644 CC[0+bs*0] += a_0 * b_0;
645 CC[1+bs*0] += a_1 * b_0;
646 CC[2+bs*0] += a_2 * b_0;
647 CC[3+bs*0] += a_3 * b_0;
648
649
650 // k = 2
651
652 a_0 = A[0+lda*2];
653 a_1 = A[1+lda*2];
654 a_2 = A[2+lda*2];
655 a_3 = A[3+lda*2];
656
657 b_0 = B[0+ldb*2];
658
659 CC[0+bs*0] += a_0 * b_0;
660 CC[1+bs*0] += a_1 * b_0;
661 CC[2+bs*0] += a_2 * b_0;
662 CC[3+bs*0] += a_3 * b_0;
663
664
665 // k = 3
666
667 a_0 = A[0+lda*3];
668 a_1 = A[1+lda*3];
669 a_2 = A[2+lda*3];
670 a_3 = A[3+lda*3];
671
672 b_0 = B[0+ldb*3];
673
674 CC[0+bs*0] += a_0 * b_0;
675 CC[1+bs*0] += a_1 * b_0;
676 CC[2+bs*0] += a_2 * b_0;
677 CC[3+bs*0] += a_3 * b_0;
678
679 A += 4*lda;
680 B += 4*ldb;
681
682 }
683
684 for(; k<kmax; k++)
685 {
686
687 // k = 0
688
689 a_0 = A[0+lda*0];
690 a_1 = A[1+lda*0];
691 a_2 = A[2+lda*0];
692 a_3 = A[3+lda*0];
693
694 b_0 = B[0+ldb*0];
695
696 CC[0+bs*0] += a_0 * b_0;
697 CC[1+bs*0] += a_1 * b_0;
698 CC[2+bs*0] += a_2 * b_0;
699 CC[3+bs*0] += a_3 * b_0;
700
701 A += 1*lda;
702 B += 1*ldb;
703
704 }
705
706 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
707 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
708 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
709 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
710
711 return;
712
713 }
714
715
716
717 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_sgemm_nt_4x4_vs_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)718 void kernel_sgemm_nt_4x4_vs_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
719 {
720
721 const int bs = 4;
722
723 #if defined(TARGET_GENERIC)
724 float CC[16] = {0};
725 #else
726 ALIGNED( float CC[16], 64 ) = {0};
727 #endif
728
729 float alpha1 = 1.0;
730 float beta1 = 0.0;
731
732 if(n1<=1)
733 {
734 kernel_sgemm_nt_4x1_libcccc(kmax, &alpha1, A, lda, B, ldb, &beta1, CC, bs, CC, bs);
735 }
736 else if(n1==2)
737 {
738 kernel_sgemm_nt_4x2_libcccc(kmax, &alpha1, A, lda, B, ldb, &beta1, CC, bs, CC, bs);
739 }
740 else if(n1==3)
741 {
742 kernel_sgemm_nt_4x3_libcccc(kmax, &alpha1, A, lda, B, ldb, &beta1, CC, bs, CC, bs);
743 }
744 else //if(n1==1)
745 {
746 kernel_sgemm_nt_4x4_libcccc(kmax, &alpha1, A, lda, B, ldb, &beta1, CC, bs, CC, bs);
747 }
748
749 if(m1>=4)
750 {
751 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
752 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
753 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
754 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
755
756 if(n1==1)
757 return;
758
759 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
760 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
761 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
762 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
763
764 if(n1==2)
765 return;
766
767 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
768 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
769 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
770 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
771
772 if(n1==3)
773 return;
774
775 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
776 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
777 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
778 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
779 }
780 else if(m1>=3)
781 {
782 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
783 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
784 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
785
786 if(n1==1)
787 return;
788
789 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
790 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
791 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
792
793 if(n1==2)
794 return;
795
796 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
797 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
798 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
799
800 if(n1==3)
801 return;
802
803 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
804 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
805 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
806 }
807 else if(m1>=2)
808 {
809 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
810 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
811
812 if(n1==1)
813 return;
814
815 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
816 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
817
818 if(n1==2)
819 return;
820
821 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
822 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
823
824 if(n1==3)
825 return;
826
827 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
828 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
829 }
830 else //if(m1>=1)
831 {
832 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
833
834 if(n1==1)
835 return;
836
837 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
838
839 if(n1==2)
840 return;
841
842 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
843
844 if(n1==3)
845 return;
846
847 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
848 }
849
850 return;
851
852 }
853 //#endif
854
855
856
857 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_nt_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)858 void kernel_sgemm_nt_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
859 {
860
861 kernel_sgemm_nt_4x4_libcccc(kmax, alpha, A, 4, B, ldb, beta, C, ldc, D, ldd);
862
863 return;
864
865 }
866 //#endif
867
868
869
870 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_nt_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)871 void kernel_sgemm_nt_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
872 {
873
874 kernel_sgemm_nt_4x4_vs_libcccc(kmax, alpha, A, 4, B, ldb, beta, C, ldc, D, ldd, m1, n1);
875
876 return;
877
878 }
879 //#endif
880
881
882
883 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_nt_4x4_libc4cc(int kmax,float * alpha,float * A,int lda,float * B,float * beta,float * C,int ldc,float * D,int ldd)884 void kernel_sgemm_nt_4x4_libc4cc(int kmax, float *alpha, float *A, int lda, float *B, float *beta, float *C, int ldc, float *D, int ldd)
885 {
886
887 kernel_sgemm_nt_4x4_libcccc(kmax, alpha, A, lda, B, 4, beta, C, ldc, D, ldd);
888
889 return;
890
891 }
892 //#endif
893
894
895
896 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_nt_4x4_vs_libc4cc(int kmax,float * alpha,float * A,int lda,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)897 void kernel_sgemm_nt_4x4_vs_libc4cc(int kmax, float *alpha, float *A, int lda, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
898 {
899
900 kernel_sgemm_nt_4x4_vs_libcccc(kmax, alpha, A, lda, B, 4, beta, C, ldc, D, ldd, m1, n1);
901
902 return;
903
904 }
905 //#endif
906
907
908
909 //#if defined(TARGET_GENERIC) | defined(TARGET_X86_AMD_BARCELONA)
910 #if ! (defined(TARGET_X64_INTEL_CORE) | defined(TARGET_ARMV7A_ARM_CORTEX_A15) | defined(TARGET_ARMV7A_ARM_CORTEX_A9) | defined(TARGET_ARMV7A_ARM_CORTEX_A7) )
kernel_sgemm_nt_4x4_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd)911 void kernel_sgemm_nt_4x4_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd)
912 {
913
914 const int bs = 4;
915
916 #if defined(TARGET_GENERIC)
917 float CC[16] = {0};
918 #else
919 ALIGNED( float CC[16], 64 ) = {0};
920 #endif
921
922 float alpha1 = 1.0;
923 float beta1 = 0.0;
924
925 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
926
927 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
928 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
929 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
930 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
931
932 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
933 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
934 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
935 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
936
937 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
938 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
939 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
940 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
941
942 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
943 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
944 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
945 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
946
947 return;
948
949 }
950 #endif
951
952
953
954 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_nt_4x4_vs_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)955 void kernel_sgemm_nt_4x4_vs_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
956 {
957
958 const int bs = 4;
959
960 #if defined(TARGET_GENERIC)
961 float CC[16] = {0};
962 #else
963 ALIGNED( float CC[16], 64 ) = {0};
964 #endif
965
966 float alpha1 = 1.0;
967 float beta1 = 0.0;
968
969 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
970
971 if(m1>=4)
972 {
973 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
974 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
975 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
976 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
977
978 if(n1==1)
979 return;
980
981 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
982 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
983 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
984 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
985
986 if(n1==2)
987 return;
988
989 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
990 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
991 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
992 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
993
994 if(n1==3)
995 return;
996
997 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
998 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
999 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
1000 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
1001 }
1002 else if(m1>=3)
1003 {
1004 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1005 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1006 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
1007
1008 if(n1==1)
1009 return;
1010
1011 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1012 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1013 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
1014
1015 if(n1==2)
1016 return;
1017
1018 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1019 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
1020 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
1021
1022 if(n1==3)
1023 return;
1024
1025 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1026 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
1027 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
1028 }
1029 else if(m1>=2)
1030 {
1031 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1032 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1033
1034 if(n1==1)
1035 return;
1036
1037 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1038 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1039
1040 if(n1==2)
1041 return;
1042
1043 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1044 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
1045
1046 if(n1==3)
1047 return;
1048
1049 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1050 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
1051 }
1052 else //if(m1>=1)
1053 {
1054 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1055
1056 if(n1==1)
1057 return;
1058
1059 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1060
1061 if(n1==2)
1062 return;
1063
1064 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1065
1066 if(n1==3)
1067 return;
1068
1069 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1070 }
1071
1072 return;
1073
1074 }
1075 //#endif
1076
1077
1078
1079 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_sgemm_nn_4x4_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)1080 void kernel_sgemm_nn_4x4_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
1081 {
1082
1083 const int bs = 4;
1084
1085 float
1086 a_0, a_1, a_2, a_3,
1087 b_0, b_1, b_2, b_3;
1088
1089 #if defined(TARGET_GENERIC)
1090 float CC[16] = {0};
1091 #else
1092 ALIGNED( float CC[16], 64 ) = {0};
1093 #endif
1094
1095 int k;
1096
1097 for(k=0; k<kmax-3; k+=4)
1098 {
1099
1100 // k = 0
1101
1102 a_0 = A[0+lda*0];
1103 a_1 = A[1+lda*0];
1104 a_2 = A[2+lda*0];
1105 a_3 = A[3+lda*0];
1106
1107 b_0 = B[0+ldb*0];
1108 b_1 = B[0+ldb*1];
1109 b_2 = B[0+ldb*2];
1110 b_3 = B[0+ldb*3];
1111
1112 CC[0+bs*0] += a_0 * b_0;
1113 CC[1+bs*0] += a_1 * b_0;
1114 CC[2+bs*0] += a_2 * b_0;
1115 CC[3+bs*0] += a_3 * b_0;
1116
1117 CC[0+bs*1] += a_0 * b_1;
1118 CC[1+bs*1] += a_1 * b_1;
1119 CC[2+bs*1] += a_2 * b_1;
1120 CC[3+bs*1] += a_3 * b_1;
1121
1122 CC[0+bs*2] += a_0 * b_2;
1123 CC[1+bs*2] += a_1 * b_2;
1124 CC[2+bs*2] += a_2 * b_2;
1125 CC[3+bs*2] += a_3 * b_2;
1126
1127 CC[0+bs*3] += a_0 * b_3;
1128 CC[1+bs*3] += a_1 * b_3;
1129 CC[2+bs*3] += a_2 * b_3;
1130 CC[3+bs*3] += a_3 * b_3;
1131
1132
1133 // k = 1
1134
1135 a_0 = A[0+lda*1];
1136 a_1 = A[1+lda*1];
1137 a_2 = A[2+lda*1];
1138 a_3 = A[3+lda*1];
1139
1140 b_0 = B[1+ldb*0];
1141 b_1 = B[1+ldb*1];
1142 b_2 = B[1+ldb*2];
1143 b_3 = B[1+ldb*3];
1144
1145 CC[0+bs*0] += a_0 * b_0;
1146 CC[1+bs*0] += a_1 * b_0;
1147 CC[2+bs*0] += a_2 * b_0;
1148 CC[3+bs*0] += a_3 * b_0;
1149
1150 CC[0+bs*1] += a_0 * b_1;
1151 CC[1+bs*1] += a_1 * b_1;
1152 CC[2+bs*1] += a_2 * b_1;
1153 CC[3+bs*1] += a_3 * b_1;
1154
1155 CC[0+bs*2] += a_0 * b_2;
1156 CC[1+bs*2] += a_1 * b_2;
1157 CC[2+bs*2] += a_2 * b_2;
1158 CC[3+bs*2] += a_3 * b_2;
1159
1160 CC[0+bs*3] += a_0 * b_3;
1161 CC[1+bs*3] += a_1 * b_3;
1162 CC[2+bs*3] += a_2 * b_3;
1163 CC[3+bs*3] += a_3 * b_3;
1164
1165
1166 // k = 2
1167
1168 a_0 = A[0+lda*2];
1169 a_1 = A[1+lda*2];
1170 a_2 = A[2+lda*2];
1171 a_3 = A[3+lda*2];
1172
1173 b_0 = B[2+ldb*0];
1174 b_1 = B[2+ldb*1];
1175 b_2 = B[2+ldb*2];
1176 b_3 = B[2+ldb*3];
1177
1178 CC[0+bs*0] += a_0 * b_0;
1179 CC[1+bs*0] += a_1 * b_0;
1180 CC[2+bs*0] += a_2 * b_0;
1181 CC[3+bs*0] += a_3 * b_0;
1182
1183 CC[0+bs*1] += a_0 * b_1;
1184 CC[1+bs*1] += a_1 * b_1;
1185 CC[2+bs*1] += a_2 * b_1;
1186 CC[3+bs*1] += a_3 * b_1;
1187
1188 CC[0+bs*2] += a_0 * b_2;
1189 CC[1+bs*2] += a_1 * b_2;
1190 CC[2+bs*2] += a_2 * b_2;
1191 CC[3+bs*2] += a_3 * b_2;
1192
1193 CC[0+bs*3] += a_0 * b_3;
1194 CC[1+bs*3] += a_1 * b_3;
1195 CC[2+bs*3] += a_2 * b_3;
1196 CC[3+bs*3] += a_3 * b_3;
1197
1198
1199 // k = 3
1200
1201 a_0 = A[0+lda*3];
1202 a_1 = A[1+lda*3];
1203 a_2 = A[2+lda*3];
1204 a_3 = A[3+lda*3];
1205
1206 b_0 = B[3+ldb*0];
1207 b_1 = B[3+ldb*1];
1208 b_2 = B[3+ldb*2];
1209 b_3 = B[3+ldb*3];
1210
1211 CC[0+bs*0] += a_0 * b_0;
1212 CC[1+bs*0] += a_1 * b_0;
1213 CC[2+bs*0] += a_2 * b_0;
1214 CC[3+bs*0] += a_3 * b_0;
1215
1216 CC[0+bs*1] += a_0 * b_1;
1217 CC[1+bs*1] += a_1 * b_1;
1218 CC[2+bs*1] += a_2 * b_1;
1219 CC[3+bs*1] += a_3 * b_1;
1220
1221 CC[0+bs*2] += a_0 * b_2;
1222 CC[1+bs*2] += a_1 * b_2;
1223 CC[2+bs*2] += a_2 * b_2;
1224 CC[3+bs*2] += a_3 * b_2;
1225
1226 CC[0+bs*3] += a_0 * b_3;
1227 CC[1+bs*3] += a_1 * b_3;
1228 CC[2+bs*3] += a_2 * b_3;
1229 CC[3+bs*3] += a_3 * b_3;
1230
1231 A += 4*lda;
1232 B += 4;
1233
1234 }
1235
1236 for(; k<kmax; k++)
1237 {
1238
1239 // k = 0
1240
1241 a_0 = A[0+lda*0];
1242 a_1 = A[1+lda*0];
1243 a_2 = A[2+lda*0];
1244 a_3 = A[3+lda*0];
1245
1246 b_0 = B[0+ldb*0];
1247 b_1 = B[0+ldb*1];
1248 b_2 = B[0+ldb*2];
1249 b_3 = B[0+ldb*3];
1250
1251 CC[0+bs*0] += a_0 * b_0;
1252 CC[1+bs*0] += a_1 * b_0;
1253 CC[2+bs*0] += a_2 * b_0;
1254 CC[3+bs*0] += a_3 * b_0;
1255
1256 CC[0+bs*1] += a_0 * b_1;
1257 CC[1+bs*1] += a_1 * b_1;
1258 CC[2+bs*1] += a_2 * b_1;
1259 CC[3+bs*1] += a_3 * b_1;
1260
1261 CC[0+bs*2] += a_0 * b_2;
1262 CC[1+bs*2] += a_1 * b_2;
1263 CC[2+bs*2] += a_2 * b_2;
1264 CC[3+bs*2] += a_3 * b_2;
1265
1266 CC[0+bs*3] += a_0 * b_3;
1267 CC[1+bs*3] += a_1 * b_3;
1268 CC[2+bs*3] += a_2 * b_3;
1269 CC[3+bs*3] += a_3 * b_3;
1270
1271 A += 1*lda;
1272 B += 1;
1273
1274 }
1275
1276 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1277 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1278 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
1279 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
1280
1281 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1282 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1283 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
1284 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
1285
1286 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1287 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
1288 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
1289 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
1290
1291 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1292 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
1293 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
1294 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
1295
1296 return;
1297
1298 }
1299 //#endif
1300
1301
1302
kernel_sgemm_nn_4x3_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)1303 static void kernel_sgemm_nn_4x3_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
1304 {
1305
1306 const int bs = 4;
1307
1308 float
1309 a_0, a_1, a_2, a_3,
1310 b_0, b_1, b_2;
1311
1312 #if defined(TARGET_GENERIC)
1313 float CC[16] = {0};
1314 #else
1315 ALIGNED( float CC[16], 64 ) = {0};
1316 #endif
1317
1318 int k;
1319
1320 for(k=0; k<kmax-3; k+=4)
1321 {
1322
1323 // k = 0
1324
1325 a_0 = A[0+lda*0];
1326 a_1 = A[1+lda*0];
1327 a_2 = A[2+lda*0];
1328 a_3 = A[3+lda*0];
1329
1330 b_0 = B[0+ldb*0];
1331 b_1 = B[0+ldb*1];
1332 b_2 = B[0+ldb*2];
1333
1334 CC[0+bs*0] += a_0 * b_0;
1335 CC[1+bs*0] += a_1 * b_0;
1336 CC[2+bs*0] += a_2 * b_0;
1337 CC[3+bs*0] += a_3 * b_0;
1338
1339 CC[0+bs*1] += a_0 * b_1;
1340 CC[1+bs*1] += a_1 * b_1;
1341 CC[2+bs*1] += a_2 * b_1;
1342 CC[3+bs*1] += a_3 * b_1;
1343
1344 CC[0+bs*2] += a_0 * b_2;
1345 CC[1+bs*2] += a_1 * b_2;
1346 CC[2+bs*2] += a_2 * b_2;
1347 CC[3+bs*2] += a_3 * b_2;
1348
1349
1350 // k = 1
1351
1352 a_0 = A[0+lda*1];
1353 a_1 = A[1+lda*1];
1354 a_2 = A[2+lda*1];
1355 a_3 = A[3+lda*1];
1356
1357 b_0 = B[1+ldb*0];
1358 b_1 = B[1+ldb*1];
1359 b_2 = B[1+ldb*2];
1360
1361 CC[0+bs*0] += a_0 * b_0;
1362 CC[1+bs*0] += a_1 * b_0;
1363 CC[2+bs*0] += a_2 * b_0;
1364 CC[3+bs*0] += a_3 * b_0;
1365
1366 CC[0+bs*1] += a_0 * b_1;
1367 CC[1+bs*1] += a_1 * b_1;
1368 CC[2+bs*1] += a_2 * b_1;
1369 CC[3+bs*1] += a_3 * b_1;
1370
1371 CC[0+bs*2] += a_0 * b_2;
1372 CC[1+bs*2] += a_1 * b_2;
1373 CC[2+bs*2] += a_2 * b_2;
1374 CC[3+bs*2] += a_3 * b_2;
1375
1376
1377 // k = 2
1378
1379 a_0 = A[0+lda*2];
1380 a_1 = A[1+lda*2];
1381 a_2 = A[2+lda*2];
1382 a_3 = A[3+lda*2];
1383
1384 b_0 = B[2+ldb*0];
1385 b_1 = B[2+ldb*1];
1386 b_2 = B[2+ldb*2];
1387
1388 CC[0+bs*0] += a_0 * b_0;
1389 CC[1+bs*0] += a_1 * b_0;
1390 CC[2+bs*0] += a_2 * b_0;
1391 CC[3+bs*0] += a_3 * b_0;
1392
1393 CC[0+bs*1] += a_0 * b_1;
1394 CC[1+bs*1] += a_1 * b_1;
1395 CC[2+bs*1] += a_2 * b_1;
1396 CC[3+bs*1] += a_3 * b_1;
1397
1398 CC[0+bs*2] += a_0 * b_2;
1399 CC[1+bs*2] += a_1 * b_2;
1400 CC[2+bs*2] += a_2 * b_2;
1401 CC[3+bs*2] += a_3 * b_2;
1402
1403
1404 // k = 3
1405
1406 a_0 = A[0+lda*3];
1407 a_1 = A[1+lda*3];
1408 a_2 = A[2+lda*3];
1409 a_3 = A[3+lda*3];
1410
1411 b_0 = B[3+ldb*0];
1412 b_1 = B[3+ldb*1];
1413 b_2 = B[3+ldb*2];
1414
1415 CC[0+bs*0] += a_0 * b_0;
1416 CC[1+bs*0] += a_1 * b_0;
1417 CC[2+bs*0] += a_2 * b_0;
1418 CC[3+bs*0] += a_3 * b_0;
1419
1420 CC[0+bs*1] += a_0 * b_1;
1421 CC[1+bs*1] += a_1 * b_1;
1422 CC[2+bs*1] += a_2 * b_1;
1423 CC[3+bs*1] += a_3 * b_1;
1424
1425 CC[0+bs*2] += a_0 * b_2;
1426 CC[1+bs*2] += a_1 * b_2;
1427 CC[2+bs*2] += a_2 * b_2;
1428 CC[3+bs*2] += a_3 * b_2;
1429
1430 A += 4*lda;
1431 B += 4;
1432
1433 }
1434
1435 for(; k<kmax; k++)
1436 {
1437
1438 // k = 0
1439
1440 a_0 = A[0+lda*0];
1441 a_1 = A[1+lda*0];
1442 a_2 = A[2+lda*0];
1443 a_3 = A[3+lda*0];
1444
1445 b_0 = B[0+ldb*0];
1446 b_1 = B[0+ldb*1];
1447 b_2 = B[0+ldb*2];
1448
1449 CC[0+bs*0] += a_0 * b_0;
1450 CC[1+bs*0] += a_1 * b_0;
1451 CC[2+bs*0] += a_2 * b_0;
1452 CC[3+bs*0] += a_3 * b_0;
1453
1454 CC[0+bs*1] += a_0 * b_1;
1455 CC[1+bs*1] += a_1 * b_1;
1456 CC[2+bs*1] += a_2 * b_1;
1457 CC[3+bs*1] += a_3 * b_1;
1458
1459 CC[0+bs*2] += a_0 * b_2;
1460 CC[1+bs*2] += a_1 * b_2;
1461 CC[2+bs*2] += a_2 * b_2;
1462 CC[3+bs*2] += a_3 * b_2;
1463
1464 A += 1*lda;
1465 B += 1;
1466
1467 }
1468
1469 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1470 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1471 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
1472 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
1473
1474 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1475 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1476 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
1477 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
1478
1479 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1480 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
1481 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
1482 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
1483
1484 return;
1485
1486 }
1487
1488
1489
kernel_sgemm_nn_4x2_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)1490 static void kernel_sgemm_nn_4x2_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
1491 {
1492
1493 const int bs = 4;
1494
1495 float
1496 a_0, a_1, a_2, a_3,
1497 b_0, b_1;
1498
1499 #if defined(TARGET_GENERIC)
1500 float CC[16] = {0};
1501 #else
1502 ALIGNED( float CC[16], 64 ) = {0};
1503 #endif
1504
1505 int k;
1506
1507 for(k=0; k<kmax-3; k+=4)
1508 {
1509
1510 // k = 0
1511
1512 a_0 = A[0+lda*0];
1513 a_1 = A[1+lda*0];
1514 a_2 = A[2+lda*0];
1515 a_3 = A[3+lda*0];
1516
1517 b_0 = B[0+ldb*0];
1518 b_1 = B[0+ldb*1];
1519
1520 CC[0+bs*0] += a_0 * b_0;
1521 CC[1+bs*0] += a_1 * b_0;
1522 CC[2+bs*0] += a_2 * b_0;
1523 CC[3+bs*0] += a_3 * b_0;
1524
1525 CC[0+bs*1] += a_0 * b_1;
1526 CC[1+bs*1] += a_1 * b_1;
1527 CC[2+bs*1] += a_2 * b_1;
1528 CC[3+bs*1] += a_3 * b_1;
1529
1530
1531 // k = 1
1532
1533 a_0 = A[0+lda*1];
1534 a_1 = A[1+lda*1];
1535 a_2 = A[2+lda*1];
1536 a_3 = A[3+lda*1];
1537
1538 b_0 = B[1+ldb*0];
1539 b_1 = B[1+ldb*1];
1540
1541 CC[0+bs*0] += a_0 * b_0;
1542 CC[1+bs*0] += a_1 * b_0;
1543 CC[2+bs*0] += a_2 * b_0;
1544 CC[3+bs*0] += a_3 * b_0;
1545
1546 CC[0+bs*1] += a_0 * b_1;
1547 CC[1+bs*1] += a_1 * b_1;
1548 CC[2+bs*1] += a_2 * b_1;
1549 CC[3+bs*1] += a_3 * b_1;
1550
1551
1552 // k = 2
1553
1554 a_0 = A[0+lda*2];
1555 a_1 = A[1+lda*2];
1556 a_2 = A[2+lda*2];
1557 a_3 = A[3+lda*2];
1558
1559 b_0 = B[2+ldb*0];
1560 b_1 = B[2+ldb*1];
1561
1562 CC[0+bs*0] += a_0 * b_0;
1563 CC[1+bs*0] += a_1 * b_0;
1564 CC[2+bs*0] += a_2 * b_0;
1565 CC[3+bs*0] += a_3 * b_0;
1566
1567 CC[0+bs*1] += a_0 * b_1;
1568 CC[1+bs*1] += a_1 * b_1;
1569 CC[2+bs*1] += a_2 * b_1;
1570 CC[3+bs*1] += a_3 * b_1;
1571
1572
1573 // k = 3
1574
1575 a_0 = A[0+lda*3];
1576 a_1 = A[1+lda*3];
1577 a_2 = A[2+lda*3];
1578 a_3 = A[3+lda*3];
1579
1580 b_0 = B[3+ldb*0];
1581 b_1 = B[3+ldb*1];
1582
1583 CC[0+bs*0] += a_0 * b_0;
1584 CC[1+bs*0] += a_1 * b_0;
1585 CC[2+bs*0] += a_2 * b_0;
1586 CC[3+bs*0] += a_3 * b_0;
1587
1588 CC[0+bs*1] += a_0 * b_1;
1589 CC[1+bs*1] += a_1 * b_1;
1590 CC[2+bs*1] += a_2 * b_1;
1591 CC[3+bs*1] += a_3 * b_1;
1592
1593 A += 4*lda;
1594 B += 4;
1595
1596 }
1597
1598 for(; k<kmax; k++)
1599 {
1600
1601 // k = 0
1602
1603 a_0 = A[0+lda*0];
1604 a_1 = A[1+lda*0];
1605 a_2 = A[2+lda*0];
1606 a_3 = A[3+lda*0];
1607
1608 b_0 = B[0+ldb*0];
1609 b_1 = B[0+ldb*1];
1610
1611 CC[0+bs*0] += a_0 * b_0;
1612 CC[1+bs*0] += a_1 * b_0;
1613 CC[2+bs*0] += a_2 * b_0;
1614 CC[3+bs*0] += a_3 * b_0;
1615
1616 CC[0+bs*1] += a_0 * b_1;
1617 CC[1+bs*1] += a_1 * b_1;
1618 CC[2+bs*1] += a_2 * b_1;
1619 CC[3+bs*1] += a_3 * b_1;
1620
1621 A += 1*lda;
1622 B += 1;
1623
1624 }
1625
1626 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1627 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1628 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
1629 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
1630
1631 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1632 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1633 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
1634 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
1635
1636 return;
1637
1638 }
1639
1640
1641
kernel_sgemm_nn_4x1_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)1642 static void kernel_sgemm_nn_4x1_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
1643 {
1644
1645 const int bs = 4;
1646
1647 float
1648 a_0, a_1, a_2, a_3,
1649 b_0;
1650
1651 #if defined(TARGET_GENERIC)
1652 float CC[16] = {0};
1653 #else
1654 ALIGNED( float CC[16], 64 ) = {0};
1655 #endif
1656
1657 int k;
1658
1659 for(k=0; k<kmax-3; k+=4)
1660 {
1661
1662 // k = 0
1663
1664 a_0 = A[0+lda*0];
1665 a_1 = A[1+lda*0];
1666 a_2 = A[2+lda*0];
1667 a_3 = A[3+lda*0];
1668
1669 b_0 = B[0+ldb*0];
1670
1671 CC[0+bs*0] += a_0 * b_0;
1672 CC[1+bs*0] += a_1 * b_0;
1673 CC[2+bs*0] += a_2 * b_0;
1674 CC[3+bs*0] += a_3 * b_0;
1675
1676
1677 // k = 1
1678
1679 a_0 = A[0+lda*1];
1680 a_1 = A[1+lda*1];
1681 a_2 = A[2+lda*1];
1682 a_3 = A[3+lda*1];
1683
1684 b_0 = B[1+ldb*0];
1685
1686 CC[0+bs*0] += a_0 * b_0;
1687 CC[1+bs*0] += a_1 * b_0;
1688 CC[2+bs*0] += a_2 * b_0;
1689 CC[3+bs*0] += a_3 * b_0;
1690
1691
1692 // k = 2
1693
1694 a_0 = A[0+lda*2];
1695 a_1 = A[1+lda*2];
1696 a_2 = A[2+lda*2];
1697 a_3 = A[3+lda*2];
1698
1699 b_0 = B[2+ldb*0];
1700
1701 CC[0+bs*0] += a_0 * b_0;
1702 CC[1+bs*0] += a_1 * b_0;
1703 CC[2+bs*0] += a_2 * b_0;
1704 CC[3+bs*0] += a_3 * b_0;
1705
1706
1707 // k = 3
1708
1709 a_0 = A[0+lda*3];
1710 a_1 = A[1+lda*3];
1711 a_2 = A[2+lda*3];
1712 a_3 = A[3+lda*3];
1713
1714 b_0 = B[3+ldb*0];
1715
1716 CC[0+bs*0] += a_0 * b_0;
1717 CC[1+bs*0] += a_1 * b_0;
1718 CC[2+bs*0] += a_2 * b_0;
1719 CC[3+bs*0] += a_3 * b_0;
1720
1721 A += 4*lda;
1722 B += 4;
1723
1724 }
1725
1726 for(; k<kmax; k++)
1727 {
1728
1729 // k = 0
1730
1731 a_0 = A[0+lda*0];
1732 a_1 = A[1+lda*0];
1733 a_2 = A[2+lda*0];
1734 a_3 = A[3+lda*0];
1735
1736 b_0 = B[0+ldb*0];
1737
1738 CC[0+bs*0] += a_0 * b_0;
1739 CC[1+bs*0] += a_1 * b_0;
1740 CC[2+bs*0] += a_2 * b_0;
1741 CC[3+bs*0] += a_3 * b_0;
1742
1743 A += 1*lda;
1744 B += 1;
1745
1746 }
1747
1748 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1749 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1750 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
1751 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
1752
1753 return;
1754
1755 }
1756
1757
1758
1759 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_sgemm_nn_4x4_vs_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)1760 void kernel_sgemm_nn_4x4_vs_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
1761 {
1762
1763 const int bs = 4;
1764
1765 #if defined(TARGET_GENERIC)
1766 float CC[16] = {0};
1767 #else
1768 ALIGNED( float CC[16], 64 ) = {0};
1769 #endif
1770
1771 float alpha1 = 1.0;
1772 float beta1 = 0.0;
1773
1774 if(n1<=1)
1775 {
1776 kernel_sgemm_nn_4x1_libcccc(kmax, &alpha1, A, lda, B, ldb, &beta1, CC, bs, CC, bs);
1777 }
1778 else if(n1==2)
1779 {
1780 kernel_sgemm_nn_4x2_libcccc(kmax, &alpha1, A, lda, B, ldb, &beta1, CC, bs, CC, bs);
1781 }
1782 else if(n1==3)
1783 {
1784 kernel_sgemm_nn_4x3_libcccc(kmax, &alpha1, A, lda, B, ldb, &beta1, CC, bs, CC, bs);
1785 }
1786 else //if(n1==4)
1787 {
1788 kernel_sgemm_nn_4x4_libcccc(kmax, &alpha1, A, lda, B, ldb, &beta1, CC, bs, CC, bs);
1789 }
1790
1791 if(m1>=4)
1792 {
1793 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1794 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1795 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
1796 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
1797
1798 if(n1==1)
1799 return;
1800
1801 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1802 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1803 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
1804 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
1805
1806 if(n1==2)
1807 return;
1808
1809 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1810 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
1811 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
1812 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
1813
1814 if(n1==3)
1815 return;
1816
1817 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1818 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
1819 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
1820 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
1821 }
1822 else if(m1>=3)
1823 {
1824 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1825 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1826 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
1827
1828 if(n1==1)
1829 return;
1830
1831 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1832 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1833 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
1834
1835 if(n1==2)
1836 return;
1837
1838 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1839 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
1840 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
1841
1842 if(n1==3)
1843 return;
1844
1845 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1846 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
1847 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
1848 }
1849 else if(m1>=2)
1850 {
1851 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1852 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1853
1854 if(n1==1)
1855 return;
1856
1857 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1858 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1859
1860 if(n1==2)
1861 return;
1862
1863 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1864 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
1865
1866 if(n1==3)
1867 return;
1868
1869 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1870 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
1871 }
1872 else //if(m1>=1)
1873 {
1874 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1875
1876 if(n1==1)
1877 return;
1878
1879 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1880
1881 if(n1==2)
1882 return;
1883
1884 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1885
1886 if(n1==3)
1887 return;
1888
1889 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1890 }
1891
1892 return;
1893
1894 }
1895 //#endif
1896
1897
1898 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_nn_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)1899 void kernel_sgemm_nn_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
1900 {
1901
1902 kernel_sgemm_nn_4x4_libcccc(kmax, alpha, A, 4, B, ldb, beta, C, ldc, D, ldd);
1903
1904 return;
1905
1906 }
1907 //#endif
1908
1909
1910
1911 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_nn_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)1912 void kernel_sgemm_nn_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
1913 {
1914
1915 kernel_sgemm_nn_4x4_vs_libcccc(kmax, alpha, A, 4, B, ldb, beta, C, ldc, D, ldd, m1, n1);
1916
1917 return;
1918
1919 }
1920 //#endif
1921
1922
1923 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_sgemm_tt_4x4_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)1924 void kernel_sgemm_tt_4x4_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
1925 {
1926
1927 const int bs = 4;
1928
1929 #if defined(TARGET_GENERIC)
1930 float CC[16] = {0};
1931 #else
1932 ALIGNED( float CC[16], 64 ) = {0};
1933 #endif
1934
1935 float alpha1 = 1.0;
1936 float beta1 = 0.0;
1937
1938 kernel_sgemm_nn_4x4_libcccc(kmax, &alpha1, B, ldb, A, lda, &beta1, CC, bs, CC, bs);
1939
1940 float tmp;
1941 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
1942 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
1943 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
1944 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
1945 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
1946 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
1947
1948 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
1949 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
1950 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
1951 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
1952
1953 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
1954 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
1955 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
1956 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
1957
1958 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
1959 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
1960 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
1961 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
1962
1963 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
1964 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
1965 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
1966 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
1967
1968 return;
1969
1970 }
1971 //#endif
1972
1973
1974
1975 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_sgemm_tt_4x4_vs_libcccc(int kmax,float * alpha,float * A,int lda,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)1976 void kernel_sgemm_tt_4x4_vs_libcccc(int kmax, float *alpha, float *A, int lda, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
1977 {
1978
1979 const int bs = 4;
1980
1981 #if defined(TARGET_GENERIC)
1982 float CC[16] = {0};
1983 #else
1984 ALIGNED( float CC[16], 64 ) = {0};
1985 #endif
1986
1987 float alpha1 = 1.0;
1988 float beta1 = 0.0;
1989
1990 kernel_sgemm_nn_4x4_libcccc(kmax, &alpha1, B, ldb, A, lda, &beta1, CC, bs, CC, bs);
1991
1992 float tmp;
1993 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
1994 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
1995 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
1996 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
1997 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
1998 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
1999
2000 if(m1>=4)
2001 {
2002 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2003 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2004 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2005 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
2006
2007 if(n1==1)
2008 return;
2009
2010 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2011 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2012 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2013 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
2014
2015 if(n1==2)
2016 return;
2017
2018 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2019 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2020 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2021 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
2022
2023 if(n1==3)
2024 return;
2025
2026 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2027 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2028 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
2029 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
2030 }
2031 else if(m1>=3)
2032 {
2033 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2034 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2035 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2036
2037 if(n1==1)
2038 return;
2039
2040 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2041 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2042 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2043
2044 if(n1==2)
2045 return;
2046
2047 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2048 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2049 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2050
2051 if(n1==3)
2052 return;
2053
2054 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2055 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2056 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
2057 }
2058 else if(m1>=2)
2059 {
2060 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2061 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2062
2063 if(n1==1)
2064 return;
2065
2066 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2067 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2068
2069 if(n1==2)
2070 return;
2071
2072 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2073 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2074
2075 if(n1==3)
2076 return;
2077
2078 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2079 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2080 }
2081 else //if(m1>=1)
2082 {
2083 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2084
2085 if(n1==1)
2086 return;
2087
2088 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2089
2090 if(n1==2)
2091 return;
2092
2093 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2094
2095 if(n1==3)
2096 return;
2097
2098 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2099 }
2100
2101 return;
2102
2103 }
2104 //#endif
2105
2106
2107
2108 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_tt_4x4_libc4cc(int kmax,float * alpha,float * A,int lda,float * B,float * beta,float * C,int ldc,float * D,int ldd)2109 void kernel_sgemm_tt_4x4_libc4cc(int kmax, float *alpha, float *A, int lda, float *B, float *beta, float *C, int ldc, float *D, int ldd)
2110 {
2111
2112 const int bs = 4;
2113
2114 #if defined(TARGET_GENERIC)
2115 float CC[16] = {0};
2116 #else
2117 ALIGNED( float CC[16], 64 ) = {0};
2118 #endif
2119
2120 float alpha1 = 1.0;
2121 float beta1 = 0.0;
2122
2123 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, B, A, lda, &beta1, CC, bs, CC, bs);
2124
2125 float tmp;
2126 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
2127 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
2128 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
2129 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
2130 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
2131 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
2132
2133 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2134 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2135 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2136 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
2137
2138 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2139 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2140 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2141 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
2142
2143 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2144 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2145 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2146 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
2147
2148 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2149 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2150 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
2151 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
2152
2153 return;
2154
2155 }
2156 //#endif
2157
2158
2159
2160 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_sgemm_tt_4x4_vs_libc4cc(int kmax,float * alpha,float * A,int lda,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)2161 void kernel_sgemm_tt_4x4_vs_libc4cc(int kmax, float *alpha, float *A, int lda, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
2162 {
2163
2164 const int bs = 4;
2165
2166 #if defined(TARGET_GENERIC)
2167 float CC[16] = {0};
2168 #else
2169 ALIGNED( float CC[16], 64 ) = {0};
2170 #endif
2171
2172 float alpha1 = 1.0;
2173 float beta1 = 0.0;
2174
2175 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, B, A, lda, &beta1, CC, bs, CC, bs);
2176
2177 float tmp;
2178 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
2179 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
2180 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
2181 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
2182 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
2183 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
2184
2185 if(m1>=4)
2186 {
2187 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2188 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2189 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2190 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
2191
2192 if(n1==1)
2193 return;
2194
2195 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2196 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2197 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2198 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
2199
2200 if(n1==2)
2201 return;
2202
2203 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2204 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2205 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2206 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
2207
2208 if(n1==3)
2209 return;
2210
2211 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2212 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2213 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
2214 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
2215 }
2216 else if(m1>=3)
2217 {
2218 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2219 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2220 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2221
2222 if(n1==1)
2223 return;
2224
2225 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2226 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2227 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2228
2229 if(n1==2)
2230 return;
2231
2232 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2233 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2234 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2235
2236 if(n1==3)
2237 return;
2238
2239 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2240 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2241 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
2242 }
2243 else if(m1>=2)
2244 {
2245 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2246 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2247
2248 if(n1==1)
2249 return;
2250
2251 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2252 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2253
2254 if(n1==2)
2255 return;
2256
2257 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2258 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2259
2260 if(n1==3)
2261 return;
2262
2263 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2264 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2265 }
2266 else //if(m1>=1)
2267 {
2268 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2269
2270 if(n1==1)
2271 return;
2272
2273 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2274
2275 if(n1==2)
2276 return;
2277
2278 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2279
2280 if(n1==3)
2281 return;
2282
2283 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2284 }
2285
2286 return;
2287
2288 }
2289 //#endif
2290
2291
2292
2293 //#if defined(TARGET_GENERIC) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_ssyrk_nt_l_4x4_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd)2294 void kernel_ssyrk_nt_l_4x4_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd)
2295 {
2296
2297 const int bs = 4;
2298
2299 #if defined(TARGET_GENERIC)
2300 float CC[16] = {0};
2301 #else
2302 ALIGNED( float CC[16], 64 ) = {0};
2303 #endif
2304
2305 float alpha1 = 1.0;
2306 float beta1 = 0.0;
2307
2308 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
2309
2310 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2311 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2312 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2313 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
2314
2315 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2316 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2317 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
2318
2319 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2320 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
2321
2322 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
2323
2324 return;
2325
2326 }
2327 //#endif
2328
2329
2330
2331 //#if defined(TARGET_GENERIC) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_ssyrk_nt_l_4x4_vs_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)2332 void kernel_ssyrk_nt_l_4x4_vs_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
2333 {
2334
2335 const int bs = 4;
2336
2337 #if defined(TARGET_GENERIC)
2338 float CC[16] = {0};
2339 #else
2340 ALIGNED( float CC[16], 64 ) = {0};
2341 #endif
2342
2343 float alpha1 = 1.0;
2344 float beta1 = 0.0;
2345
2346 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
2347
2348 if(m1>=4)
2349 {
2350 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2351 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2352 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2353 D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
2354
2355 if(n1==1)
2356 return;
2357
2358 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2359 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2360 D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
2361
2362 if(n1==2)
2363 return;
2364
2365 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2366 D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
2367
2368 if(n1==3)
2369 return;
2370
2371 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
2372 }
2373 else if(m1>=3)
2374 {
2375 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2376 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2377 D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2378
2379 if(n1==1)
2380 return;
2381
2382 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2383 D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2384
2385 if(n1==2)
2386 return;
2387
2388 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2389 }
2390 else if(m1>=2)
2391 {
2392 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2393 D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2394
2395 if(n1==1)
2396 return;
2397
2398 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2399 }
2400 else //if(m1>=1)
2401 {
2402 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2403 }
2404
2405 return;
2406
2407 }
2408 //#endif
2409
2410
2411
2412 //#if defined(TARGET_GENERIC) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_ssyrk_nt_u_4x4_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd)2413 void kernel_ssyrk_nt_u_4x4_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd)
2414 {
2415
2416 const int bs = 4;
2417
2418 #if defined(TARGET_GENERIC)
2419 float CC[16] = {0};
2420 #else
2421 ALIGNED( float CC[16], 64 ) = {0};
2422 #endif
2423
2424 float alpha1 = 1.0;
2425 float beta1 = 0.0;
2426
2427 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
2428
2429 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2430 // D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2431 // D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2432 // D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
2433
2434 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2435 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2436 // D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2437 // D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
2438
2439 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2440 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2441 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2442 // D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
2443
2444 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2445 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2446 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
2447 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
2448
2449 return;
2450
2451 }
2452 //#endif
2453
2454
2455
2456 //#if defined(TARGET_GENERIC) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_ssyrk_nt_u_4x4_vs_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)2457 void kernel_ssyrk_nt_u_4x4_vs_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
2458 {
2459
2460 const int bs = 4;
2461
2462 #if defined(TARGET_GENERIC)
2463 float CC[16] = {0};
2464 #else
2465 ALIGNED( float CC[16], 64 ) = {0};
2466 #endif
2467
2468 float alpha1 = 1.0;
2469 float beta1 = 0.0;
2470
2471 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
2472
2473 if(m1>=4)
2474 {
2475 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2476 // D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2477 // D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2478 // D[3+ldd*0] = beta[0]*C[3+ldc*0] + alpha[0]*CC[3+bs*0];
2479
2480 if(n1==1)
2481 return;
2482
2483 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2484 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2485 // D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2486 // D[3+ldd*1] = beta[0]*C[3+ldc*1] + alpha[0]*CC[3+bs*1];
2487
2488 if(n1==2)
2489 return;
2490
2491 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2492 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2493 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2494 // D[3+ldd*2] = beta[0]*C[3+ldc*2] + alpha[0]*CC[3+bs*2];
2495
2496 if(n1==3)
2497 return;
2498
2499 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2500 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2501 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
2502 D[3+ldd*3] = beta[0]*C[3+ldc*3] + alpha[0]*CC[3+bs*3];
2503 }
2504 else if(m1>=3)
2505 {
2506 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2507 // D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2508 // D[2+ldd*0] = beta[0]*C[2+ldc*0] + alpha[0]*CC[2+bs*0];
2509
2510 if(n1==1)
2511 return;
2512
2513 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2514 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2515 // D[2+ldd*1] = beta[0]*C[2+ldc*1] + alpha[0]*CC[2+bs*1];
2516
2517 if(n1==2)
2518 return;
2519
2520 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2521 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2522 D[2+ldd*2] = beta[0]*C[2+ldc*2] + alpha[0]*CC[2+bs*2];
2523
2524 if(n1==3)
2525 return;
2526
2527 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2528 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2529 D[2+ldd*3] = beta[0]*C[2+ldc*3] + alpha[0]*CC[2+bs*3];
2530 }
2531 else if(m1>=2)
2532 {
2533 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2534 // D[1+ldd*0] = beta[0]*C[1+ldc*0] + alpha[0]*CC[1+bs*0];
2535
2536 if(n1==1)
2537 return;
2538
2539 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2540 D[1+ldd*1] = beta[0]*C[1+ldc*1] + alpha[0]*CC[1+bs*1];
2541
2542 if(n1==2)
2543 return;
2544
2545 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2546 D[1+ldd*2] = beta[0]*C[1+ldc*2] + alpha[0]*CC[1+bs*2];
2547
2548 if(n1==3)
2549 return;
2550
2551 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2552 D[1+ldd*3] = beta[0]*C[1+ldc*3] + alpha[0]*CC[1+bs*3];
2553 }
2554 else //if(m1>=1)
2555 {
2556 D[0+ldd*0] = beta[0]*C[0+ldc*0] + alpha[0]*CC[0+bs*0];
2557
2558 if(n1==1)
2559 return;
2560
2561 D[0+ldd*1] = beta[0]*C[0+ldc*1] + alpha[0]*CC[0+bs*1];
2562
2563 if(n1==2)
2564 return;
2565
2566 D[0+ldd*2] = beta[0]*C[0+ldc*2] + alpha[0]*CC[0+bs*2];
2567
2568 if(n1==3)
2569 return;
2570
2571 D[0+ldd*3] = beta[0]*C[0+ldc*3] + alpha[0]*CC[0+bs*3];
2572 }
2573
2574 return;
2575
2576 }
2577 //#endif
2578
2579
2580
2581 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_rl_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)2582 void kernel_strmm_nn_rl_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
2583 {
2584
2585 const int bs = 4;
2586
2587 float
2588 a_0, a_1, a_2, a_3,
2589 b_0, b_1, b_2, b_3;
2590
2591 #if defined(TARGET_GENERIC)
2592 float CC[16] = {0};
2593 #else
2594 ALIGNED( float CC[16], 64 ) = {0};
2595 #endif
2596
2597 int k;
2598
2599 k = 0;
2600
2601 // k = 0
2602
2603 a_0 = A[0];
2604 a_1 = A[1];
2605 a_2 = A[2];
2606 a_3 = A[3];
2607
2608 b_0 = B[0+0*ldb];
2609 CC[0+bs*0] += a_0 * b_0;
2610 CC[1+bs*0] += a_1 * b_0;
2611 CC[2+bs*0] += a_2 * b_0;
2612 CC[3+bs*0] += a_3 * b_0;
2613
2614 A += bs;
2615 B += 1;
2616 k += 1;
2617
2618 if(k>=kmax)
2619 goto store;
2620
2621 // k = 1
2622
2623 a_0 = A[0];
2624 a_1 = A[1];
2625 a_2 = A[2];
2626 a_3 = A[3];
2627
2628 b_0 = B[0+0*ldb];
2629 CC[0+bs*0] += a_0 * b_0;
2630 CC[1+bs*0] += a_1 * b_0;
2631 CC[2+bs*0] += a_2 * b_0;
2632 CC[3+bs*0] += a_3 * b_0;
2633
2634 b_1 = B[0+1*ldb];
2635 CC[0+bs*1] += a_0 * b_1;
2636 CC[1+bs*1] += a_1 * b_1;
2637 CC[2+bs*1] += a_2 * b_1;
2638 CC[3+bs*1] += a_3 * b_1;
2639
2640 A += bs;
2641 B += 1;
2642 k += 1;
2643
2644 if(k>=kmax)
2645 goto store;
2646
2647 // k = 2
2648
2649 a_0 = A[0];
2650 a_1 = A[1];
2651 a_2 = A[2];
2652 a_3 = A[3];
2653
2654 b_0 = B[0+0*ldb];
2655 CC[0+bs*0] += a_0 * b_0;
2656 CC[1+bs*0] += a_1 * b_0;
2657 CC[2+bs*0] += a_2 * b_0;
2658 CC[3+bs*0] += a_3 * b_0;
2659
2660 b_1 = B[0+1*ldb];
2661 CC[0+bs*1] += a_0 * b_1;
2662 CC[1+bs*1] += a_1 * b_1;
2663 CC[2+bs*1] += a_2 * b_1;
2664 CC[3+bs*1] += a_3 * b_1;
2665
2666 b_2 = B[0+2*ldb];
2667 CC[0+bs*2] += a_0 * b_2;
2668 CC[1+bs*2] += a_1 * b_2;
2669 CC[2+bs*2] += a_2 * b_2;
2670 CC[3+bs*2] += a_3 * b_2;
2671
2672 A += bs;
2673 B += 1;
2674 k += 1;
2675
2676 if(k>=kmax)
2677 goto store;
2678
2679 // k = 3
2680
2681 a_0 = A[0];
2682 a_1 = A[1];
2683 a_2 = A[2];
2684 a_3 = A[3];
2685
2686 b_0 = B[0+0*ldb];
2687 CC[0+bs*0] += a_0 * b_0;
2688 CC[1+bs*0] += a_1 * b_0;
2689 CC[2+bs*0] += a_2 * b_0;
2690 CC[3+bs*0] += a_3 * b_0;
2691
2692 b_1 = B[0+1*ldb];
2693 CC[0+bs*1] += a_0 * b_1;
2694 CC[1+bs*1] += a_1 * b_1;
2695 CC[2+bs*1] += a_2 * b_1;
2696 CC[3+bs*1] += a_3 * b_1;
2697
2698 b_2 = B[0+2*ldb];
2699 CC[0+bs*2] += a_0 * b_2;
2700 CC[1+bs*2] += a_1 * b_2;
2701 CC[2+bs*2] += a_2 * b_2;
2702 CC[3+bs*2] += a_3 * b_2;
2703
2704 b_3 = B[0+3*ldb];
2705 CC[0+bs*3] += a_0 * b_3;
2706 CC[1+bs*3] += a_1 * b_3;
2707 CC[2+bs*3] += a_2 * b_3;
2708 CC[3+bs*3] += a_3 * b_3;
2709
2710 A += bs;
2711 B += 1;
2712 k += 1;
2713
2714 store:
2715
2716 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
2717 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
2718 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
2719 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
2720
2721 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
2722 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
2723 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
2724 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
2725
2726 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
2727 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
2728 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
2729 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
2730
2731 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
2732 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
2733 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
2734 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
2735
2736 float beta1 = 1.0;
2737
2738 kernel_sgemm_nn_4x4_lib4ccc(kmax-k, alpha, A, B, ldb, &beta1, CC, bs, D, ldd);
2739
2740 return;
2741
2742 }
2743 //#endif
2744
2745
2746
2747 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_rl_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)2748 void kernel_strmm_nn_rl_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
2749 {
2750
2751 const int bs = 4;
2752
2753 float
2754 a_0, a_1, a_2, a_3,
2755 b_0, b_1, b_2, b_3;
2756
2757 #if defined(TARGET_GENERIC)
2758 float CC[16] = {0};
2759 #else
2760 ALIGNED( float CC[16], 64 ) = {0};
2761 #endif
2762
2763 int k;
2764
2765 k = 0;
2766
2767 // k = 0
2768
2769 a_0 = A[0];
2770 a_1 = A[1];
2771 a_2 = A[2];
2772 a_3 = A[3];
2773
2774 b_0 = B[0+0*ldb];
2775 CC[0+bs*0] += a_0 * b_0;
2776 CC[1+bs*0] += a_1 * b_0;
2777 CC[2+bs*0] += a_2 * b_0;
2778 CC[3+bs*0] += a_3 * b_0;
2779
2780 A += bs;
2781 B += 1;
2782 k += 1;
2783
2784 if(k>=kmax)
2785 goto store;
2786
2787 // k = 1
2788
2789 a_0 = A[0];
2790 a_1 = A[1];
2791 a_2 = A[2];
2792 a_3 = A[3];
2793
2794 b_0 = B[0+0*ldb];
2795 CC[0+bs*0] += a_0 * b_0;
2796 CC[1+bs*0] += a_1 * b_0;
2797 CC[2+bs*0] += a_2 * b_0;
2798 CC[3+bs*0] += a_3 * b_0;
2799
2800 b_1 = B[0+1*ldb];
2801 CC[0+bs*1] += a_0 * b_1;
2802 CC[1+bs*1] += a_1 * b_1;
2803 CC[2+bs*1] += a_2 * b_1;
2804 CC[3+bs*1] += a_3 * b_1;
2805
2806 A += bs;
2807 B += 1;
2808 k += 1;
2809
2810 if(k>=kmax)
2811 goto store;
2812
2813 // k = 2
2814
2815 a_0 = A[0];
2816 a_1 = A[1];
2817 a_2 = A[2];
2818 a_3 = A[3];
2819
2820 b_0 = B[0+0*ldb];
2821 CC[0+bs*0] += a_0 * b_0;
2822 CC[1+bs*0] += a_1 * b_0;
2823 CC[2+bs*0] += a_2 * b_0;
2824 CC[3+bs*0] += a_3 * b_0;
2825
2826 b_1 = B[0+1*ldb];
2827 CC[0+bs*1] += a_0 * b_1;
2828 CC[1+bs*1] += a_1 * b_1;
2829 CC[2+bs*1] += a_2 * b_1;
2830 CC[3+bs*1] += a_3 * b_1;
2831
2832 b_2 = B[0+2*ldb];
2833 CC[0+bs*2] += a_0 * b_2;
2834 CC[1+bs*2] += a_1 * b_2;
2835 CC[2+bs*2] += a_2 * b_2;
2836 CC[3+bs*2] += a_3 * b_2;
2837
2838 A += bs;
2839 B += 1;
2840 k += 1;
2841
2842 if(k>=kmax)
2843 goto store;
2844
2845 // k = 3
2846
2847 a_0 = A[0];
2848 a_1 = A[1];
2849 a_2 = A[2];
2850 a_3 = A[3];
2851
2852 b_0 = B[0+0*ldb];
2853 CC[0+bs*0] += a_0 * b_0;
2854 CC[1+bs*0] += a_1 * b_0;
2855 CC[2+bs*0] += a_2 * b_0;
2856 CC[3+bs*0] += a_3 * b_0;
2857
2858 b_1 = B[0+1*ldb];
2859 CC[0+bs*1] += a_0 * b_1;
2860 CC[1+bs*1] += a_1 * b_1;
2861 CC[2+bs*1] += a_2 * b_1;
2862 CC[3+bs*1] += a_3 * b_1;
2863
2864 b_2 = B[0+2*ldb];
2865 CC[0+bs*2] += a_0 * b_2;
2866 CC[1+bs*2] += a_1 * b_2;
2867 CC[2+bs*2] += a_2 * b_2;
2868 CC[3+bs*2] += a_3 * b_2;
2869
2870 b_3 = B[0+3*ldb];
2871 CC[0+bs*3] += a_0 * b_3;
2872 CC[1+bs*3] += a_1 * b_3;
2873 CC[2+bs*3] += a_2 * b_3;
2874 CC[3+bs*3] += a_3 * b_3;
2875
2876 A += bs;
2877 B += 1;
2878 k += 1;
2879
2880 store:
2881
2882 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
2883 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
2884 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
2885 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
2886
2887 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
2888 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
2889 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
2890 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
2891
2892 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
2893 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
2894 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
2895 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
2896
2897 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
2898 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
2899 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
2900 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
2901
2902 float beta1 = 1.0;
2903
2904 kernel_sgemm_nn_4x4_lib4ccc(kmax-k, alpha, A, B, ldb, &beta1, CC, bs, CC, bs);
2905
2906 if(m1>=4)
2907 {
2908 D[0+ldd*0] = CC[0+bs*0];
2909 D[1+ldd*0] = CC[1+bs*0];
2910 D[2+ldd*0] = CC[2+bs*0];
2911 D[3+ldd*0] = CC[3+bs*0];
2912
2913 if(n1==1)
2914 return;
2915
2916 D[0+ldd*1] = CC[0+bs*1];
2917 D[1+ldd*1] = CC[1+bs*1];
2918 D[2+ldd*1] = CC[2+bs*1];
2919 D[3+ldd*1] = CC[3+bs*1];
2920
2921 if(n1==2)
2922 return;
2923
2924 D[0+ldd*2] = CC[0+bs*2];
2925 D[1+ldd*2] = CC[1+bs*2];
2926 D[2+ldd*2] = CC[2+bs*2];
2927 D[3+ldd*2] = CC[3+bs*2];
2928
2929 if(n1==3)
2930 return;
2931
2932 D[0+ldd*3] = CC[0+bs*3];
2933 D[1+ldd*3] = CC[1+bs*3];
2934 D[2+ldd*3] = CC[2+bs*3];
2935 D[3+ldd*3] = CC[3+bs*3];
2936 }
2937 else if(m1>=3)
2938 {
2939 D[0+ldd*0] = CC[0+bs*0];
2940 D[1+ldd*0] = CC[1+bs*0];
2941 D[2+ldd*0] = CC[2+bs*0];
2942
2943 if(n1==1)
2944 return;
2945
2946 D[0+ldd*1] = CC[0+bs*1];
2947 D[1+ldd*1] = CC[1+bs*1];
2948 D[2+ldd*1] = CC[2+bs*1];
2949
2950 if(n1==2)
2951 return;
2952
2953 D[0+ldd*2] = CC[0+bs*2];
2954 D[1+ldd*2] = CC[1+bs*2];
2955 D[2+ldd*2] = CC[2+bs*2];
2956
2957 if(n1==3)
2958 return;
2959
2960 D[0+ldd*3] = CC[0+bs*3];
2961 D[1+ldd*3] = CC[1+bs*3];
2962 D[2+ldd*3] = CC[2+bs*3];
2963 }
2964 else if(m1>=2)
2965 {
2966 D[0+ldd*0] = CC[0+bs*0];
2967 D[1+ldd*0] = CC[1+bs*0];
2968
2969 if(n1==1)
2970 return;
2971
2972 D[0+ldd*1] = CC[0+bs*1];
2973 D[1+ldd*1] = CC[1+bs*1];
2974
2975 if(n1==2)
2976 return;
2977
2978 D[0+ldd*2] = CC[0+bs*2];
2979 D[1+ldd*2] = CC[1+bs*2];
2980
2981 if(n1==3)
2982 return;
2983
2984 D[0+ldd*3] = CC[0+bs*3];
2985 D[1+ldd*3] = CC[1+bs*3];
2986 }
2987 else //if(m1>=1)
2988 {
2989 D[0+ldd*0] = CC[0+bs*0];
2990
2991 if(n1==1)
2992 return;
2993
2994 D[0+ldd*1] = CC[0+bs*1];
2995
2996 if(n1==2)
2997 return;
2998
2999 D[0+ldd*2] = CC[0+bs*2];
3000
3001 if(n1==3)
3002 return;
3003
3004 D[0+ldd*3] = CC[0+bs*3];
3005 }
3006
3007 return;
3008
3009 }
3010 //#endif
3011
3012
3013
3014 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_rl_4x4_tran_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd)3015 void kernel_strmm_nn_rl_4x4_tran_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd)
3016 {
3017
3018 const int bs = 4;
3019
3020 float
3021 a_0, a_1, a_2, a_3,
3022 b_0, b_1, b_2, b_3;
3023
3024 #if defined(TARGET_GENERIC)
3025 float CC[16] = {0};
3026 #else
3027 ALIGNED( float CC[16], 64 ) = {0};
3028 #endif
3029
3030 int k;
3031
3032 k = 0;
3033
3034 // k = 0
3035
3036 a_0 = A[0];
3037 a_1 = A[1];
3038 a_2 = A[2];
3039 a_3 = A[3];
3040
3041 b_0 = B[0+0*ldb];
3042 CC[0+bs*0] += a_0 * b_0;
3043 CC[1+bs*0] += a_1 * b_0;
3044 CC[2+bs*0] += a_2 * b_0;
3045 CC[3+bs*0] += a_3 * b_0;
3046
3047 A += bs;
3048 B += 1;
3049 k += 1;
3050
3051 if(k>=kmax)
3052 goto store;
3053
3054 // k = 1
3055
3056 a_0 = A[0];
3057 a_1 = A[1];
3058 a_2 = A[2];
3059 a_3 = A[3];
3060
3061 b_0 = B[0+0*ldb];
3062 CC[0+bs*0] += a_0 * b_0;
3063 CC[1+bs*0] += a_1 * b_0;
3064 CC[2+bs*0] += a_2 * b_0;
3065 CC[3+bs*0] += a_3 * b_0;
3066
3067 b_1 = B[0+1*ldb];
3068 CC[0+bs*1] += a_0 * b_1;
3069 CC[1+bs*1] += a_1 * b_1;
3070 CC[2+bs*1] += a_2 * b_1;
3071 CC[3+bs*1] += a_3 * b_1;
3072
3073 A += bs;
3074 B += 1;
3075 k += 1;
3076
3077 if(k>=kmax)
3078 goto store;
3079
3080 // k = 2
3081
3082 a_0 = A[0];
3083 a_1 = A[1];
3084 a_2 = A[2];
3085 a_3 = A[3];
3086
3087 b_0 = B[0+0*ldb];
3088 CC[0+bs*0] += a_0 * b_0;
3089 CC[1+bs*0] += a_1 * b_0;
3090 CC[2+bs*0] += a_2 * b_0;
3091 CC[3+bs*0] += a_3 * b_0;
3092
3093 b_1 = B[0+1*ldb];
3094 CC[0+bs*1] += a_0 * b_1;
3095 CC[1+bs*1] += a_1 * b_1;
3096 CC[2+bs*1] += a_2 * b_1;
3097 CC[3+bs*1] += a_3 * b_1;
3098
3099 b_2 = B[0+2*ldb];
3100 CC[0+bs*2] += a_0 * b_2;
3101 CC[1+bs*2] += a_1 * b_2;
3102 CC[2+bs*2] += a_2 * b_2;
3103 CC[3+bs*2] += a_3 * b_2;
3104
3105 A += bs;
3106 B += 1;
3107 k += 1;
3108
3109 if(k>=kmax)
3110 goto store;
3111
3112 // k = 3
3113
3114 a_0 = A[0];
3115 a_1 = A[1];
3116 a_2 = A[2];
3117 a_3 = A[3];
3118
3119 b_0 = B[0+0*ldb];
3120 CC[0+bs*0] += a_0 * b_0;
3121 CC[1+bs*0] += a_1 * b_0;
3122 CC[2+bs*0] += a_2 * b_0;
3123 CC[3+bs*0] += a_3 * b_0;
3124
3125 b_1 = B[0+1*ldb];
3126 CC[0+bs*1] += a_0 * b_1;
3127 CC[1+bs*1] += a_1 * b_1;
3128 CC[2+bs*1] += a_2 * b_1;
3129 CC[3+bs*1] += a_3 * b_1;
3130
3131 b_2 = B[0+2*ldb];
3132 CC[0+bs*2] += a_0 * b_2;
3133 CC[1+bs*2] += a_1 * b_2;
3134 CC[2+bs*2] += a_2 * b_2;
3135 CC[3+bs*2] += a_3 * b_2;
3136
3137 b_3 = B[0+3*ldb];
3138 CC[0+bs*3] += a_0 * b_3;
3139 CC[1+bs*3] += a_1 * b_3;
3140 CC[2+bs*3] += a_2 * b_3;
3141 CC[3+bs*3] += a_3 * b_3;
3142
3143 A += bs;
3144 B += 1;
3145 k += 1;
3146
3147 float alpha1 = 1.0;
3148 float beta1 = 1.0;
3149
3150 kernel_sgemm_nn_4x4_lib4ccc(kmax-k, &alpha1, A, B, ldb, &beta1, CC, bs, CC, bs);
3151
3152 store:
3153
3154 // scale & tranpose & store
3155 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
3156 D[0+ldd*1] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
3157 D[0+ldd*2] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
3158 D[0+ldd*3] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
3159
3160 D[1+ldd*0] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
3161 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
3162 D[1+ldd*2] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
3163 D[1+ldd*3] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
3164
3165 D[2+ldd*0] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
3166 D[2+ldd*1] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
3167 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
3168 D[2+ldd*3] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
3169
3170 D[3+ldd*0] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
3171 D[3+ldd*1] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
3172 D[3+ldd*2] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
3173 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
3174
3175 return;
3176
3177 }
3178 //#endif
3179
3180
3181
3182 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_rl_4x4_tran_vs_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd,int m1,int n1)3183 void kernel_strmm_nn_rl_4x4_tran_vs_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd, int m1, int n1)
3184 {
3185
3186 const int bs = 4;
3187
3188 float
3189 tmp,
3190 a_0, a_1, a_2, a_3,
3191 b_0, b_1, b_2, b_3;
3192
3193 #if defined(TARGET_GENERIC)
3194 float CC[16] = {0};
3195 #else
3196 ALIGNED( float CC[16], 64 ) = {0};
3197 #endif
3198
3199 int k;
3200
3201 k = 0;
3202
3203 // k = 0
3204
3205 a_0 = A[0];
3206 a_1 = A[1];
3207 a_2 = A[2];
3208 a_3 = A[3];
3209
3210 b_0 = B[0+0*ldb];
3211 CC[0+bs*0] += a_0 * b_0;
3212 CC[1+bs*0] += a_1 * b_0;
3213 CC[2+bs*0] += a_2 * b_0;
3214 CC[3+bs*0] += a_3 * b_0;
3215
3216 A += bs;
3217 B += 1;
3218 k += 1;
3219
3220 if(k>=kmax)
3221 goto store;
3222
3223 // k = 1
3224
3225 a_0 = A[0];
3226 a_1 = A[1];
3227 a_2 = A[2];
3228 a_3 = A[3];
3229
3230 b_0 = B[0+0*ldb];
3231 CC[0+bs*0] += a_0 * b_0;
3232 CC[1+bs*0] += a_1 * b_0;
3233 CC[2+bs*0] += a_2 * b_0;
3234 CC[3+bs*0] += a_3 * b_0;
3235
3236 b_1 = B[0+1*ldb];
3237 CC[0+bs*1] += a_0 * b_1;
3238 CC[1+bs*1] += a_1 * b_1;
3239 CC[2+bs*1] += a_2 * b_1;
3240 CC[3+bs*1] += a_3 * b_1;
3241
3242 A += bs;
3243 B += 1;
3244 k += 1;
3245
3246 if(k>=kmax)
3247 goto store;
3248
3249 // k = 2
3250
3251 a_0 = A[0];
3252 a_1 = A[1];
3253 a_2 = A[2];
3254 a_3 = A[3];
3255
3256 b_0 = B[0+0*ldb];
3257 CC[0+bs*0] += a_0 * b_0;
3258 CC[1+bs*0] += a_1 * b_0;
3259 CC[2+bs*0] += a_2 * b_0;
3260 CC[3+bs*0] += a_3 * b_0;
3261
3262 b_1 = B[0+1*ldb];
3263 CC[0+bs*1] += a_0 * b_1;
3264 CC[1+bs*1] += a_1 * b_1;
3265 CC[2+bs*1] += a_2 * b_1;
3266 CC[3+bs*1] += a_3 * b_1;
3267
3268 b_2 = B[0+2*ldb];
3269 CC[0+bs*2] += a_0 * b_2;
3270 CC[1+bs*2] += a_1 * b_2;
3271 CC[2+bs*2] += a_2 * b_2;
3272 CC[3+bs*2] += a_3 * b_2;
3273
3274 A += bs;
3275 B += 1;
3276 k += 1;
3277
3278 if(k>=kmax)
3279 goto store;
3280
3281 // k = 3
3282
3283 a_0 = A[0];
3284 a_1 = A[1];
3285 a_2 = A[2];
3286 a_3 = A[3];
3287
3288 b_0 = B[0+0*ldb];
3289 CC[0+bs*0] += a_0 * b_0;
3290 CC[1+bs*0] += a_1 * b_0;
3291 CC[2+bs*0] += a_2 * b_0;
3292 CC[3+bs*0] += a_3 * b_0;
3293
3294 b_1 = B[0+1*ldb];
3295 CC[0+bs*1] += a_0 * b_1;
3296 CC[1+bs*1] += a_1 * b_1;
3297 CC[2+bs*1] += a_2 * b_1;
3298 CC[3+bs*1] += a_3 * b_1;
3299
3300 b_2 = B[0+2*ldb];
3301 CC[0+bs*2] += a_0 * b_2;
3302 CC[1+bs*2] += a_1 * b_2;
3303 CC[2+bs*2] += a_2 * b_2;
3304 CC[3+bs*2] += a_3 * b_2;
3305
3306 b_3 = B[0+3*ldb];
3307 CC[0+bs*3] += a_0 * b_3;
3308 CC[1+bs*3] += a_1 * b_3;
3309 CC[2+bs*3] += a_2 * b_3;
3310 CC[3+bs*3] += a_3 * b_3;
3311
3312 A += bs;
3313 B += 1;
3314 k += 1;
3315
3316 float alpha1 = 1.0;
3317 float beta1 = 1.0;
3318
3319 kernel_sgemm_nn_4x4_lib4ccc(kmax-k, &alpha1, A, B, ldb, &beta1, CC, bs, CC, bs);
3320
3321 store:
3322
3323 // scale
3324 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
3325 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
3326 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
3327 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
3328
3329 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
3330 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
3331 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
3332 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
3333
3334 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
3335 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
3336 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
3337 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
3338
3339 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
3340 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
3341 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
3342 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
3343
3344 // transpose
3345 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
3346 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
3347 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
3348
3349 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
3350 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
3351
3352 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
3353
3354 // store
3355 if(m1>=4)
3356 {
3357 D[0+ldd*0] = CC[0+bs*0];
3358 D[1+ldd*0] = CC[1+bs*0];
3359 D[2+ldd*0] = CC[2+bs*0];
3360 D[3+ldd*0] = CC[3+bs*0];
3361
3362 if(n1==1)
3363 return;
3364
3365 D[0+ldd*1] = CC[0+bs*1];
3366 D[1+ldd*1] = CC[1+bs*1];
3367 D[2+ldd*1] = CC[2+bs*1];
3368 D[3+ldd*1] = CC[3+bs*1];
3369
3370 if(n1==2)
3371 return;
3372
3373 D[0+ldd*2] = CC[0+bs*2];
3374 D[1+ldd*2] = CC[1+bs*2];
3375 D[2+ldd*2] = CC[2+bs*2];
3376 D[3+ldd*2] = CC[3+bs*2];
3377
3378 if(n1==3)
3379 return;
3380
3381 D[0+ldd*3] = CC[0+bs*3];
3382 D[1+ldd*3] = CC[1+bs*3];
3383 D[2+ldd*3] = CC[2+bs*3];
3384 D[3+ldd*3] = CC[3+bs*3];
3385 }
3386 else if(m1>=3)
3387 {
3388 D[0+ldd*0] = CC[0+bs*0];
3389 D[1+ldd*0] = CC[1+bs*0];
3390 D[2+ldd*0] = CC[2+bs*0];
3391
3392 if(n1==1)
3393 return;
3394
3395 D[0+ldd*1] = CC[0+bs*1];
3396 D[1+ldd*1] = CC[1+bs*1];
3397 D[2+ldd*1] = CC[2+bs*1];
3398
3399 if(n1==2)
3400 return;
3401
3402 D[0+ldd*2] = CC[0+bs*2];
3403 D[1+ldd*2] = CC[1+bs*2];
3404 D[2+ldd*2] = CC[2+bs*2];
3405
3406 if(n1==3)
3407 return;
3408
3409 D[0+ldd*3] = CC[0+bs*3];
3410 D[1+ldd*3] = CC[1+bs*3];
3411 D[2+ldd*3] = CC[2+bs*3];
3412 }
3413 else if(m1>=2)
3414 {
3415 D[0+ldd*0] = CC[0+bs*0];
3416 D[1+ldd*0] = CC[1+bs*0];
3417
3418 if(n1==1)
3419 return;
3420
3421 D[0+ldd*1] = CC[0+bs*1];
3422 D[1+ldd*1] = CC[1+bs*1];
3423
3424 if(n1==2)
3425 return;
3426
3427 D[0+ldd*2] = CC[0+bs*2];
3428 D[1+ldd*2] = CC[1+bs*2];
3429
3430 if(n1==3)
3431 return;
3432
3433 D[0+ldd*3] = CC[0+bs*3];
3434 D[1+ldd*3] = CC[1+bs*3];
3435 }
3436 else //if(m1>=1)
3437 {
3438 D[0+ldd*0] = CC[0+bs*0];
3439
3440 if(n1==1)
3441 return;
3442
3443 D[0+ldd*1] = CC[0+bs*1];
3444
3445 if(n1==2)
3446 return;
3447
3448 D[0+ldd*2] = CC[0+bs*2];
3449
3450 if(n1==3)
3451 return;
3452
3453 D[0+ldd*3] = CC[0+bs*3];
3454 }
3455
3456 return;
3457
3458 }
3459 //#endif
3460
3461
3462
3463 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_rl_one_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)3464 void kernel_strmm_nn_rl_one_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
3465 {
3466
3467 const int bs = 4;
3468
3469 float
3470 a_0, a_1, a_2, a_3,
3471 b_0, b_1, b_2, b_3;
3472
3473 #if defined(TARGET_GENERIC)
3474 float CC[16] = {0};
3475 #else
3476 ALIGNED( float CC[16], 64 ) = {0};
3477 #endif
3478
3479 int k;
3480
3481 k = 0;
3482
3483 // k = 0
3484
3485 a_0 = A[0];
3486 a_1 = A[1];
3487 a_2 = A[2];
3488 a_3 = A[3];
3489
3490 CC[0+bs*0] += a_0;
3491 CC[1+bs*0] += a_1;
3492 CC[2+bs*0] += a_2;
3493 CC[3+bs*0] += a_3;
3494
3495 A += bs;
3496 B += 1;
3497 k += 1;
3498
3499 if(k>=kmax)
3500 goto store;
3501
3502 // k = 1
3503
3504 a_0 = A[0];
3505 a_1 = A[1];
3506 a_2 = A[2];
3507 a_3 = A[3];
3508
3509 b_0 = B[0+0*ldb];
3510 CC[0+bs*0] += a_0 * b_0;
3511 CC[1+bs*0] += a_1 * b_0;
3512 CC[2+bs*0] += a_2 * b_0;
3513 CC[3+bs*0] += a_3 * b_0;
3514
3515 CC[0+bs*1] += a_0;
3516 CC[1+bs*1] += a_1;
3517 CC[2+bs*1] += a_2;
3518 CC[3+bs*1] += a_3;
3519
3520 A += bs;
3521 B += 1;
3522 k += 1;
3523
3524 if(k>=kmax)
3525 goto store;
3526
3527 // k = 2
3528
3529 a_0 = A[0];
3530 a_1 = A[1];
3531 a_2 = A[2];
3532 a_3 = A[3];
3533
3534 b_0 = B[0+0*ldb];
3535 CC[0+bs*0] += a_0 * b_0;
3536 CC[1+bs*0] += a_1 * b_0;
3537 CC[2+bs*0] += a_2 * b_0;
3538 CC[3+bs*0] += a_3 * b_0;
3539
3540 b_1 = B[0+1*ldb];
3541 CC[0+bs*1] += a_0 * b_1;
3542 CC[1+bs*1] += a_1 * b_1;
3543 CC[2+bs*1] += a_2 * b_1;
3544 CC[3+bs*1] += a_3 * b_1;
3545
3546 CC[0+bs*2] += a_0;
3547 CC[1+bs*2] += a_1;
3548 CC[2+bs*2] += a_2;
3549 CC[3+bs*2] += a_3;
3550
3551 A += bs;
3552 B += 1;
3553 k += 1;
3554
3555 if(k>=kmax)
3556 goto store;
3557
3558 // k = 3
3559
3560 a_0 = A[0];
3561 a_1 = A[1];
3562 a_2 = A[2];
3563 a_3 = A[3];
3564
3565 b_0 = B[0+0*ldb];
3566 CC[0+bs*0] += a_0 * b_0;
3567 CC[1+bs*0] += a_1 * b_0;
3568 CC[2+bs*0] += a_2 * b_0;
3569 CC[3+bs*0] += a_3 * b_0;
3570
3571 b_1 = B[0+1*ldb];
3572 CC[0+bs*1] += a_0 * b_1;
3573 CC[1+bs*1] += a_1 * b_1;
3574 CC[2+bs*1] += a_2 * b_1;
3575 CC[3+bs*1] += a_3 * b_1;
3576
3577 b_2 = B[0+2*ldb];
3578 CC[0+bs*2] += a_0 * b_2;
3579 CC[1+bs*2] += a_1 * b_2;
3580 CC[2+bs*2] += a_2 * b_2;
3581 CC[3+bs*2] += a_3 * b_2;
3582
3583 CC[0+bs*3] += a_0;
3584 CC[1+bs*3] += a_1;
3585 CC[2+bs*3] += a_2;
3586 CC[3+bs*3] += a_3;
3587
3588 A += bs;
3589 B += 1;
3590 k += 1;
3591
3592 store:
3593
3594 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
3595 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
3596 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
3597 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
3598
3599 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
3600 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
3601 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
3602 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
3603
3604 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
3605 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
3606 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
3607 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
3608
3609 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
3610 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
3611 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
3612 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
3613
3614 float beta1 = 1.0;
3615
3616 kernel_sgemm_nn_4x4_lib4ccc(kmax-k, alpha, A, B, ldb, &beta1, CC, bs, D, ldd);
3617
3618 return;
3619
3620 }
3621 //#endif
3622
3623
3624
3625 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_rl_one_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)3626 void kernel_strmm_nn_rl_one_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
3627 {
3628
3629 const int bs = 4;
3630
3631 float
3632 a_0, a_1, a_2, a_3,
3633 b_0, b_1, b_2, b_3;
3634
3635 #if defined(TARGET_GENERIC)
3636 float CC[16] = {0};
3637 #else
3638 ALIGNED( float CC[16], 64 ) = {0};
3639 #endif
3640
3641 int k;
3642
3643 k = 0;
3644
3645 // k = 0
3646
3647 a_0 = A[0];
3648 a_1 = A[1];
3649 a_2 = A[2];
3650 a_3 = A[3];
3651
3652 CC[0+bs*0] += a_0;
3653 CC[1+bs*0] += a_1;
3654 CC[2+bs*0] += a_2;
3655 CC[3+bs*0] += a_3;
3656
3657 A += bs;
3658 B += 1;
3659 k += 1;
3660
3661 if(k>=kmax)
3662 goto store;
3663
3664 // k = 1
3665
3666 a_0 = A[0];
3667 a_1 = A[1];
3668 a_2 = A[2];
3669 a_3 = A[3];
3670
3671 b_0 = B[0+0*ldb];
3672 CC[0+bs*0] += a_0 * b_0;
3673 CC[1+bs*0] += a_1 * b_0;
3674 CC[2+bs*0] += a_2 * b_0;
3675 CC[3+bs*0] += a_3 * b_0;
3676
3677 CC[0+bs*1] += a_0;
3678 CC[1+bs*1] += a_1;
3679 CC[2+bs*1] += a_2;
3680 CC[3+bs*1] += a_3;
3681
3682 A += bs;
3683 B += 1;
3684 k += 1;
3685
3686 if(k>=kmax)
3687 goto store;
3688
3689 // k = 2
3690
3691 a_0 = A[0];
3692 a_1 = A[1];
3693 a_2 = A[2];
3694 a_3 = A[3];
3695
3696 b_0 = B[0+0*ldb];
3697 CC[0+bs*0] += a_0 * b_0;
3698 CC[1+bs*0] += a_1 * b_0;
3699 CC[2+bs*0] += a_2 * b_0;
3700 CC[3+bs*0] += a_3 * b_0;
3701
3702 b_1 = B[0+1*ldb];
3703 CC[0+bs*1] += a_0 * b_1;
3704 CC[1+bs*1] += a_1 * b_1;
3705 CC[2+bs*1] += a_2 * b_1;
3706 CC[3+bs*1] += a_3 * b_1;
3707
3708 CC[0+bs*2] += a_0;
3709 CC[1+bs*2] += a_1;
3710 CC[2+bs*2] += a_2;
3711 CC[3+bs*2] += a_3;
3712
3713 A += bs;
3714 B += 1;
3715 k += 1;
3716
3717 if(k>=kmax)
3718 goto store;
3719
3720 // k = 3
3721
3722 a_0 = A[0];
3723 a_1 = A[1];
3724 a_2 = A[2];
3725 a_3 = A[3];
3726
3727 b_0 = B[0+0*ldb];
3728 CC[0+bs*0] += a_0 * b_0;
3729 CC[1+bs*0] += a_1 * b_0;
3730 CC[2+bs*0] += a_2 * b_0;
3731 CC[3+bs*0] += a_3 * b_0;
3732
3733 b_1 = B[0+1*ldb];
3734 CC[0+bs*1] += a_0 * b_1;
3735 CC[1+bs*1] += a_1 * b_1;
3736 CC[2+bs*1] += a_2 * b_1;
3737 CC[3+bs*1] += a_3 * b_1;
3738
3739 b_2 = B[0+2*ldb];
3740 CC[0+bs*2] += a_0 * b_2;
3741 CC[1+bs*2] += a_1 * b_2;
3742 CC[2+bs*2] += a_2 * b_2;
3743 CC[3+bs*2] += a_3 * b_2;
3744
3745 CC[0+bs*3] += a_0;
3746 CC[1+bs*3] += a_1;
3747 CC[2+bs*3] += a_2;
3748 CC[3+bs*3] += a_3;
3749
3750 A += bs;
3751 B += 1;
3752 k += 1;
3753
3754 store:
3755
3756 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
3757 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
3758 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
3759 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
3760
3761 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
3762 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
3763 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
3764 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
3765
3766 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
3767 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
3768 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
3769 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
3770
3771 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
3772 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
3773 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
3774 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
3775
3776 float beta1 = 1.0;
3777
3778 kernel_sgemm_nn_4x4_lib4ccc(kmax-k, alpha, A, B, ldb, &beta1, CC, bs, CC, bs);
3779
3780 if(m1>=4)
3781 {
3782 D[0+ldd*0] = CC[0+bs*0];
3783 D[1+ldd*0] = CC[1+bs*0];
3784 D[2+ldd*0] = CC[2+bs*0];
3785 D[3+ldd*0] = CC[3+bs*0];
3786
3787 if(n1==1)
3788 return;
3789
3790 D[0+ldd*1] = CC[0+bs*1];
3791 D[1+ldd*1] = CC[1+bs*1];
3792 D[2+ldd*1] = CC[2+bs*1];
3793 D[3+ldd*1] = CC[3+bs*1];
3794
3795 if(n1==2)
3796 return;
3797
3798 D[0+ldd*2] = CC[0+bs*2];
3799 D[1+ldd*2] = CC[1+bs*2];
3800 D[2+ldd*2] = CC[2+bs*2];
3801 D[3+ldd*2] = CC[3+bs*2];
3802
3803 if(n1==3)
3804 return;
3805
3806 D[0+ldd*3] = CC[0+bs*3];
3807 D[1+ldd*3] = CC[1+bs*3];
3808 D[2+ldd*3] = CC[2+bs*3];
3809 D[3+ldd*3] = CC[3+bs*3];
3810 }
3811 else if(m1>=3)
3812 {
3813 D[0+ldd*0] = CC[0+bs*0];
3814 D[1+ldd*0] = CC[1+bs*0];
3815 D[2+ldd*0] = CC[2+bs*0];
3816
3817 if(n1==1)
3818 return;
3819
3820 D[0+ldd*1] = CC[0+bs*1];
3821 D[1+ldd*1] = CC[1+bs*1];
3822 D[2+ldd*1] = CC[2+bs*1];
3823
3824 if(n1==2)
3825 return;
3826
3827 D[0+ldd*2] = CC[0+bs*2];
3828 D[1+ldd*2] = CC[1+bs*2];
3829 D[2+ldd*2] = CC[2+bs*2];
3830
3831 if(n1==3)
3832 return;
3833
3834 D[0+ldd*3] = CC[0+bs*3];
3835 D[1+ldd*3] = CC[1+bs*3];
3836 D[2+ldd*3] = CC[2+bs*3];
3837 }
3838 else if(m1>=2)
3839 {
3840 D[0+ldd*0] = CC[0+bs*0];
3841 D[1+ldd*0] = CC[1+bs*0];
3842
3843 if(n1==1)
3844 return;
3845
3846 D[0+ldd*1] = CC[0+bs*1];
3847 D[1+ldd*1] = CC[1+bs*1];
3848
3849 if(n1==2)
3850 return;
3851
3852 D[0+ldd*2] = CC[0+bs*2];
3853 D[1+ldd*2] = CC[1+bs*2];
3854
3855 if(n1==3)
3856 return;
3857
3858 D[0+ldd*3] = CC[0+bs*3];
3859 D[1+ldd*3] = CC[1+bs*3];
3860 }
3861 else //if(m1>=1)
3862 {
3863 D[0+ldd*0] = CC[0+bs*0];
3864
3865 if(n1==1)
3866 return;
3867
3868 D[0+ldd*1] = CC[0+bs*1];
3869
3870 if(n1==2)
3871 return;
3872
3873 D[0+ldd*2] = CC[0+bs*2];
3874
3875 if(n1==3)
3876 return;
3877
3878 D[0+ldd*3] = CC[0+bs*3];
3879 }
3880
3881 return;
3882
3883 }
3884 //#endif
3885
3886
3887
3888 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_rl_one_4x4_tran_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd)3889 void kernel_strmm_nn_rl_one_4x4_tran_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd)
3890 {
3891
3892 const int bs = 4;
3893
3894 float
3895 a_0, a_1, a_2, a_3,
3896 b_0, b_1, b_2, b_3;
3897
3898 #if defined(TARGET_GENERIC)
3899 float CC[16] = {0};
3900 #else
3901 ALIGNED( float CC[16], 64 ) = {0};
3902 #endif
3903
3904 int k;
3905
3906 k = 0;
3907
3908 // k = 0
3909
3910 a_0 = A[0];
3911 a_1 = A[1];
3912 a_2 = A[2];
3913 a_3 = A[3];
3914
3915 CC[0+bs*0] += a_0;
3916 CC[1+bs*0] += a_1;
3917 CC[2+bs*0] += a_2;
3918 CC[3+bs*0] += a_3;
3919
3920 A += bs;
3921 B += 1;
3922 k += 1;
3923
3924 if(k>=kmax)
3925 goto store;
3926
3927 // k = 1
3928
3929 a_0 = A[0];
3930 a_1 = A[1];
3931 a_2 = A[2];
3932 a_3 = A[3];
3933
3934 b_0 = B[0+0*ldb];
3935 CC[0+bs*0] += a_0 * b_0;
3936 CC[1+bs*0] += a_1 * b_0;
3937 CC[2+bs*0] += a_2 * b_0;
3938 CC[3+bs*0] += a_3 * b_0;
3939
3940 CC[0+bs*1] += a_0;
3941 CC[1+bs*1] += a_1;
3942 CC[2+bs*1] += a_2;
3943 CC[3+bs*1] += a_3;
3944
3945 A += bs;
3946 B += 1;
3947 k += 1;
3948
3949 if(k>=kmax)
3950 goto store;
3951
3952 // k = 2
3953
3954 a_0 = A[0];
3955 a_1 = A[1];
3956 a_2 = A[2];
3957 a_3 = A[3];
3958
3959 b_0 = B[0+0*ldb];
3960 CC[0+bs*0] += a_0 * b_0;
3961 CC[1+bs*0] += a_1 * b_0;
3962 CC[2+bs*0] += a_2 * b_0;
3963 CC[3+bs*0] += a_3 * b_0;
3964
3965 b_1 = B[0+1*ldb];
3966 CC[0+bs*1] += a_0 * b_1;
3967 CC[1+bs*1] += a_1 * b_1;
3968 CC[2+bs*1] += a_2 * b_1;
3969 CC[3+bs*1] += a_3 * b_1;
3970
3971 CC[0+bs*2] += a_0;
3972 CC[1+bs*2] += a_1;
3973 CC[2+bs*2] += a_2;
3974 CC[3+bs*2] += a_3;
3975
3976 A += bs;
3977 B += 1;
3978 k += 1;
3979
3980 if(k>=kmax)
3981 goto store;
3982
3983 // k = 3
3984
3985 a_0 = A[0];
3986 a_1 = A[1];
3987 a_2 = A[2];
3988 a_3 = A[3];
3989
3990 b_0 = B[0+0*ldb];
3991 CC[0+bs*0] += a_0 * b_0;
3992 CC[1+bs*0] += a_1 * b_0;
3993 CC[2+bs*0] += a_2 * b_0;
3994 CC[3+bs*0] += a_3 * b_0;
3995
3996 b_1 = B[0+1*ldb];
3997 CC[0+bs*1] += a_0 * b_1;
3998 CC[1+bs*1] += a_1 * b_1;
3999 CC[2+bs*1] += a_2 * b_1;
4000 CC[3+bs*1] += a_3 * b_1;
4001
4002 b_2 = B[0+2*ldb];
4003 CC[0+bs*2] += a_0 * b_2;
4004 CC[1+bs*2] += a_1 * b_2;
4005 CC[2+bs*2] += a_2 * b_2;
4006 CC[3+bs*2] += a_3 * b_2;
4007
4008 CC[0+bs*3] += a_0;
4009 CC[1+bs*3] += a_1;
4010 CC[2+bs*3] += a_2;
4011 CC[3+bs*3] += a_3;
4012
4013 A += bs;
4014 B += 1;
4015 k += 1;
4016
4017 float alpha1 = 1.0;
4018 float beta1 = 1.0;
4019
4020 kernel_sgemm_nn_4x4_lib4ccc(kmax-k, &alpha1, A, B, ldb, &beta1, CC, bs, CC, bs);
4021
4022 store:
4023
4024 // scale & tranpose & store
4025 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
4026 D[0+ldd*1] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
4027 D[0+ldd*2] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
4028 D[0+ldd*3] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
4029
4030 D[1+ldd*0] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
4031 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
4032 D[1+ldd*2] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
4033 D[1+ldd*3] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
4034
4035 D[2+ldd*0] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
4036 D[2+ldd*1] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
4037 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
4038 D[2+ldd*3] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
4039
4040 D[3+ldd*0] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
4041 D[3+ldd*1] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
4042 D[3+ldd*2] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
4043 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
4044
4045 return;
4046
4047 }
4048 //#endif
4049
4050
4051
4052 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_rl_one_4x4_tran_vs_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd,int m1,int n1)4053 void kernel_strmm_nn_rl_one_4x4_tran_vs_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd, int m1, int n1)
4054 {
4055
4056 const int bs = 4;
4057
4058 float
4059 tmp,
4060 a_0, a_1, a_2, a_3,
4061 b_0, b_1, b_2, b_3;
4062
4063 #if defined(TARGET_GENERIC)
4064 float CC[16] = {0};
4065 #else
4066 ALIGNED( float CC[16], 64 ) = {0};
4067 #endif
4068
4069 int k;
4070
4071 k = 0;
4072
4073 // k = 0
4074
4075 a_0 = A[0];
4076 a_1 = A[1];
4077 a_2 = A[2];
4078 a_3 = A[3];
4079
4080 CC[0+bs*0] += a_0;
4081 CC[1+bs*0] += a_1;
4082 CC[2+bs*0] += a_2;
4083 CC[3+bs*0] += a_3;
4084
4085 A += bs;
4086 B += 1;
4087 k += 1;
4088
4089 if(k>=kmax)
4090 goto store;
4091
4092 // k = 1
4093
4094 a_0 = A[0];
4095 a_1 = A[1];
4096 a_2 = A[2];
4097 a_3 = A[3];
4098
4099 b_0 = B[0+0*ldb];
4100 CC[0+bs*0] += a_0 * b_0;
4101 CC[1+bs*0] += a_1 * b_0;
4102 CC[2+bs*0] += a_2 * b_0;
4103 CC[3+bs*0] += a_3 * b_0;
4104
4105 CC[0+bs*1] += a_0;
4106 CC[1+bs*1] += a_1;
4107 CC[2+bs*1] += a_2;
4108 CC[3+bs*1] += a_3;
4109
4110 A += bs;
4111 B += 1;
4112 k += 1;
4113
4114 if(k>=kmax)
4115 goto store;
4116
4117 // k = 2
4118
4119 a_0 = A[0];
4120 a_1 = A[1];
4121 a_2 = A[2];
4122 a_3 = A[3];
4123
4124 b_0 = B[0+0*ldb];
4125 CC[0+bs*0] += a_0 * b_0;
4126 CC[1+bs*0] += a_1 * b_0;
4127 CC[2+bs*0] += a_2 * b_0;
4128 CC[3+bs*0] += a_3 * b_0;
4129
4130 b_1 = B[0+1*ldb];
4131 CC[0+bs*1] += a_0 * b_1;
4132 CC[1+bs*1] += a_1 * b_1;
4133 CC[2+bs*1] += a_2 * b_1;
4134 CC[3+bs*1] += a_3 * b_1;
4135
4136 CC[0+bs*2] += a_0;
4137 CC[1+bs*2] += a_1;
4138 CC[2+bs*2] += a_2;
4139 CC[3+bs*2] += a_3;
4140
4141 A += bs;
4142 B += 1;
4143 k += 1;
4144
4145 if(k>=kmax)
4146 goto store;
4147
4148 // k = 3
4149
4150 a_0 = A[0];
4151 a_1 = A[1];
4152 a_2 = A[2];
4153 a_3 = A[3];
4154
4155 b_0 = B[0+0*ldb];
4156 CC[0+bs*0] += a_0 * b_0;
4157 CC[1+bs*0] += a_1 * b_0;
4158 CC[2+bs*0] += a_2 * b_0;
4159 CC[3+bs*0] += a_3 * b_0;
4160
4161 b_1 = B[0+1*ldb];
4162 CC[0+bs*1] += a_0 * b_1;
4163 CC[1+bs*1] += a_1 * b_1;
4164 CC[2+bs*1] += a_2 * b_1;
4165 CC[3+bs*1] += a_3 * b_1;
4166
4167 b_2 = B[0+2*ldb];
4168 CC[0+bs*2] += a_0 * b_2;
4169 CC[1+bs*2] += a_1 * b_2;
4170 CC[2+bs*2] += a_2 * b_2;
4171 CC[3+bs*2] += a_3 * b_2;
4172
4173 CC[0+bs*3] += a_0;
4174 CC[1+bs*3] += a_1;
4175 CC[2+bs*3] += a_2;
4176 CC[3+bs*3] += a_3;
4177
4178 A += bs;
4179 B += 1;
4180 k += 1;
4181
4182 float alpha1 = 1.0;
4183 float beta1 = 1.0;
4184
4185 kernel_sgemm_nn_4x4_lib4ccc(kmax-k, &alpha1, A, B, ldb, &beta1, CC, bs, CC, bs);
4186
4187 store:
4188
4189 // scale
4190 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
4191 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
4192 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
4193 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
4194
4195 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
4196 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
4197 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
4198 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
4199
4200 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
4201 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
4202 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
4203 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
4204
4205 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
4206 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
4207 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
4208 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
4209
4210 // transpose
4211 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
4212 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
4213 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
4214
4215 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
4216 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
4217
4218 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
4219
4220 // store
4221 if(m1>=4)
4222 {
4223 D[0+ldd*0] = CC[0+bs*0];
4224 D[1+ldd*0] = CC[1+bs*0];
4225 D[2+ldd*0] = CC[2+bs*0];
4226 D[3+ldd*0] = CC[3+bs*0];
4227
4228 if(n1==1)
4229 return;
4230
4231 D[0+ldd*1] = CC[0+bs*1];
4232 D[1+ldd*1] = CC[1+bs*1];
4233 D[2+ldd*1] = CC[2+bs*1];
4234 D[3+ldd*1] = CC[3+bs*1];
4235
4236 if(n1==2)
4237 return;
4238
4239 D[0+ldd*2] = CC[0+bs*2];
4240 D[1+ldd*2] = CC[1+bs*2];
4241 D[2+ldd*2] = CC[2+bs*2];
4242 D[3+ldd*2] = CC[3+bs*2];
4243
4244 if(n1==3)
4245 return;
4246
4247 D[0+ldd*3] = CC[0+bs*3];
4248 D[1+ldd*3] = CC[1+bs*3];
4249 D[2+ldd*3] = CC[2+bs*3];
4250 D[3+ldd*3] = CC[3+bs*3];
4251 }
4252 else if(m1>=3)
4253 {
4254 D[0+ldd*0] = CC[0+bs*0];
4255 D[1+ldd*0] = CC[1+bs*0];
4256 D[2+ldd*0] = CC[2+bs*0];
4257
4258 if(n1==1)
4259 return;
4260
4261 D[0+ldd*1] = CC[0+bs*1];
4262 D[1+ldd*1] = CC[1+bs*1];
4263 D[2+ldd*1] = CC[2+bs*1];
4264
4265 if(n1==2)
4266 return;
4267
4268 D[0+ldd*2] = CC[0+bs*2];
4269 D[1+ldd*2] = CC[1+bs*2];
4270 D[2+ldd*2] = CC[2+bs*2];
4271
4272 if(n1==3)
4273 return;
4274
4275 D[0+ldd*3] = CC[0+bs*3];
4276 D[1+ldd*3] = CC[1+bs*3];
4277 D[2+ldd*3] = CC[2+bs*3];
4278 }
4279 else if(m1>=2)
4280 {
4281 D[0+ldd*0] = CC[0+bs*0];
4282 D[1+ldd*0] = CC[1+bs*0];
4283
4284 if(n1==1)
4285 return;
4286
4287 D[0+ldd*1] = CC[0+bs*1];
4288 D[1+ldd*1] = CC[1+bs*1];
4289
4290 if(n1==2)
4291 return;
4292
4293 D[0+ldd*2] = CC[0+bs*2];
4294 D[1+ldd*2] = CC[1+bs*2];
4295
4296 if(n1==3)
4297 return;
4298
4299 D[0+ldd*3] = CC[0+bs*3];
4300 D[1+ldd*3] = CC[1+bs*3];
4301 }
4302 else //if(m1>=1)
4303 {
4304 D[0+ldd*0] = CC[0+bs*0];
4305
4306 if(n1==1)
4307 return;
4308
4309 D[0+ldd*1] = CC[0+bs*1];
4310
4311 if(n1==2)
4312 return;
4313
4314 D[0+ldd*2] = CC[0+bs*2];
4315
4316 if(n1==3)
4317 return;
4318
4319 D[0+ldd*3] = CC[0+bs*3];
4320 }
4321
4322 return;
4323
4324 }
4325 //#endif
4326
4327
4328
4329 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_ru_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)4330 void kernel_strmm_nn_ru_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
4331 {
4332
4333 const int bs = 4;
4334
4335 float
4336 a_0, a_1, a_2, a_3,
4337 b_0, b_1, b_2, b_3;
4338
4339 #if defined(TARGET_GENERIC)
4340 float CC[16] = {0};
4341 #else
4342 ALIGNED( float CC[16], 64 ) = {0};
4343 #endif
4344
4345 float alpha1 = 1.0;
4346
4347 // assume always kmax>=4 !!!
4348
4349 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
4350
4351 A += kmax*bs;
4352 B += kmax;
4353
4354 // k = 0
4355
4356 a_0 = A[0];
4357 a_1 = A[1];
4358 a_2 = A[2];
4359 a_3 = A[3];
4360
4361 b_0 = B[0+0*ldb];
4362 CC[0+bs*0] += a_0 * b_0;
4363 CC[1+bs*0] += a_1 * b_0;
4364 CC[2+bs*0] += a_2 * b_0;
4365 CC[3+bs*0] += a_3 * b_0;
4366
4367 b_0 = B[0+1*ldb];
4368 CC[0+bs*1] += a_0 * b_0;
4369 CC[1+bs*1] += a_1 * b_0;
4370 CC[2+bs*1] += a_2 * b_0;
4371 CC[3+bs*1] += a_3 * b_0;
4372
4373 b_0 = B[0+2*ldb];
4374 CC[0+bs*2] += a_0 * b_0;
4375 CC[1+bs*2] += a_1 * b_0;
4376 CC[2+bs*2] += a_2 * b_0;
4377 CC[3+bs*2] += a_3 * b_0;
4378
4379 b_0 = B[0+3*ldb];
4380 CC[0+bs*3] += a_0 * b_0;
4381 CC[1+bs*3] += a_1 * b_0;
4382 CC[2+bs*3] += a_2 * b_0;
4383 CC[3+bs*3] += a_3 * b_0;
4384
4385 A += bs;
4386 B += 1;
4387 // k += 1;
4388
4389 // k = 1
4390
4391 a_0 = A[0];
4392 a_1 = A[1];
4393 a_2 = A[2];
4394 a_3 = A[3];
4395
4396 b_0 = B[0+1*ldb];
4397 CC[0+bs*1] += a_0 * b_0;
4398 CC[1+bs*1] += a_1 * b_0;
4399 CC[2+bs*1] += a_2 * b_0;
4400 CC[3+bs*1] += a_3 * b_0;
4401
4402 b_0 = B[0+2*ldb];
4403 CC[0+bs*2] += a_0 * b_0;
4404 CC[1+bs*2] += a_1 * b_0;
4405 CC[2+bs*2] += a_2 * b_0;
4406 CC[3+bs*2] += a_3 * b_0;
4407
4408 b_0 = B[0+3*ldb];
4409 CC[0+bs*3] += a_0 * b_0;
4410 CC[1+bs*3] += a_1 * b_0;
4411 CC[2+bs*3] += a_2 * b_0;
4412 CC[3+bs*3] += a_3 * b_0;
4413
4414 A += bs;
4415 B += 1;
4416 // k += 1;
4417
4418 // k = 2
4419
4420 a_0 = A[0];
4421 a_1 = A[1];
4422 a_2 = A[2];
4423 a_3 = A[3];
4424
4425 b_0 = B[0+2*ldb];
4426 CC[0+bs*2] += a_0 * b_0;
4427 CC[1+bs*2] += a_1 * b_0;
4428 CC[2+bs*2] += a_2 * b_0;
4429 CC[3+bs*2] += a_3 * b_0;
4430
4431 b_0 = B[0+3*ldb];
4432 CC[0+bs*3] += a_0 * b_0;
4433 CC[1+bs*3] += a_1 * b_0;
4434 CC[2+bs*3] += a_2 * b_0;
4435 CC[3+bs*3] += a_3 * b_0;
4436
4437 A += bs;
4438 B += 1;
4439 // k += 1;
4440
4441 // k = 3
4442
4443 a_0 = A[0];
4444 a_1 = A[1];
4445 a_2 = A[2];
4446 a_3 = A[3];
4447
4448 b_0 = B[0+3*ldb];
4449 CC[0+bs*3] += a_0 * b_0;
4450 CC[1+bs*3] += a_1 * b_0;
4451 CC[2+bs*3] += a_2 * b_0;
4452 CC[3+bs*3] += a_3 * b_0;
4453
4454 A += bs;
4455 B += 1;
4456 // k += 1;
4457
4458 store:
4459
4460 D[0+ldd*0] = alpha[0]*CC[0+bs*0];
4461 D[1+ldd*0] = alpha[0]*CC[1+bs*0];
4462 D[2+ldd*0] = alpha[0]*CC[2+bs*0];
4463 D[3+ldd*0] = alpha[0]*CC[3+bs*0];
4464
4465 D[0+ldd*1] = alpha[0]*CC[0+bs*1];
4466 D[1+ldd*1] = alpha[0]*CC[1+bs*1];
4467 D[2+ldd*1] = alpha[0]*CC[2+bs*1];
4468 D[3+ldd*1] = alpha[0]*CC[3+bs*1];
4469
4470 D[0+ldd*2] = alpha[0]*CC[0+bs*2];
4471 D[1+ldd*2] = alpha[0]*CC[1+bs*2];
4472 D[2+ldd*2] = alpha[0]*CC[2+bs*2];
4473 D[3+ldd*2] = alpha[0]*CC[3+bs*2];
4474
4475 D[0+ldd*3] = alpha[0]*CC[0+bs*3];
4476 D[1+ldd*3] = alpha[0]*CC[1+bs*3];
4477 D[2+ldd*3] = alpha[0]*CC[2+bs*3];
4478 D[3+ldd*3] = alpha[0]*CC[3+bs*3];
4479
4480 return;
4481
4482 }
4483 //#endif
4484
4485
4486
4487 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_ru_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)4488 void kernel_strmm_nn_ru_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
4489 {
4490
4491 const int bs = 4;
4492
4493 float
4494 a_0, a_1, a_2, a_3,
4495 b_0, b_1, b_2, b_3;
4496
4497 #if defined(TARGET_GENERIC)
4498 float CC[16] = {0};
4499 #else
4500 ALIGNED( float CC[16], 64 ) = {0};
4501 #endif
4502
4503 float alpha1 = 1.0;
4504
4505 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
4506
4507 A += kmax*bs;
4508 B += kmax;
4509
4510 if(n1>=4)
4511 {
4512
4513 // k = 0
4514
4515 a_0 = A[0];
4516 a_1 = A[1];
4517 a_2 = A[2];
4518 a_3 = A[3];
4519
4520 b_0 = B[0+0*ldb];
4521 CC[0+bs*0] += a_0 * b_0;
4522 CC[1+bs*0] += a_1 * b_0;
4523 CC[2+bs*0] += a_2 * b_0;
4524 CC[3+bs*0] += a_3 * b_0;
4525
4526 b_0 = B[0+1*ldb];
4527 CC[0+bs*1] += a_0 * b_0;
4528 CC[1+bs*1] += a_1 * b_0;
4529 CC[2+bs*1] += a_2 * b_0;
4530 CC[3+bs*1] += a_3 * b_0;
4531
4532 b_0 = B[0+2*ldb];
4533 CC[0+bs*2] += a_0 * b_0;
4534 CC[1+bs*2] += a_1 * b_0;
4535 CC[2+bs*2] += a_2 * b_0;
4536 CC[3+bs*2] += a_3 * b_0;
4537
4538 b_0 = B[0+3*ldb];
4539 CC[0+bs*3] += a_0 * b_0;
4540 CC[1+bs*3] += a_1 * b_0;
4541 CC[2+bs*3] += a_2 * b_0;
4542 CC[3+bs*3] += a_3 * b_0;
4543
4544 A += bs;
4545 B += 1;
4546 // k += 1;
4547
4548 // k = 1
4549
4550 a_0 = A[0];
4551 a_1 = A[1];
4552 a_2 = A[2];
4553 a_3 = A[3];
4554
4555 b_0 = B[0+1*ldb];
4556 CC[0+bs*1] += a_0 * b_0;
4557 CC[1+bs*1] += a_1 * b_0;
4558 CC[2+bs*1] += a_2 * b_0;
4559 CC[3+bs*1] += a_3 * b_0;
4560
4561 b_0 = B[0+2*ldb];
4562 CC[0+bs*2] += a_0 * b_0;
4563 CC[1+bs*2] += a_1 * b_0;
4564 CC[2+bs*2] += a_2 * b_0;
4565 CC[3+bs*2] += a_3 * b_0;
4566
4567 b_0 = B[0+3*ldb];
4568 CC[0+bs*3] += a_0 * b_0;
4569 CC[1+bs*3] += a_1 * b_0;
4570 CC[2+bs*3] += a_2 * b_0;
4571 CC[3+bs*3] += a_3 * b_0;
4572
4573 A += bs;
4574 B += 1;
4575 // k += 1;
4576
4577 // k = 2
4578
4579 a_0 = A[0];
4580 a_1 = A[1];
4581 a_2 = A[2];
4582 a_3 = A[3];
4583
4584 b_0 = B[0+2*ldb];
4585 CC[0+bs*2] += a_0 * b_0;
4586 CC[1+bs*2] += a_1 * b_0;
4587 CC[2+bs*2] += a_2 * b_0;
4588 CC[3+bs*2] += a_3 * b_0;
4589
4590 b_0 = B[0+3*ldb];
4591 CC[0+bs*3] += a_0 * b_0;
4592 CC[1+bs*3] += a_1 * b_0;
4593 CC[2+bs*3] += a_2 * b_0;
4594 CC[3+bs*3] += a_3 * b_0;
4595
4596 A += bs;
4597 B += 1;
4598 // k += 1;
4599
4600 // k = 3
4601
4602 a_0 = A[0];
4603 a_1 = A[1];
4604 a_2 = A[2];
4605 a_3 = A[3];
4606
4607 b_0 = B[0+3*ldb];
4608 CC[0+bs*3] += a_0 * b_0;
4609 CC[1+bs*3] += a_1 * b_0;
4610 CC[2+bs*3] += a_2 * b_0;
4611 CC[3+bs*3] += a_3 * b_0;
4612
4613 A += bs;
4614 B += 1;
4615 // k += 1;
4616
4617 }
4618 else if(n1==3)
4619 {
4620
4621 // k = 0
4622
4623 a_0 = A[0];
4624 a_1 = A[1];
4625 a_2 = A[2];
4626 a_3 = A[3];
4627
4628 b_0 = B[0+0*ldb];
4629 CC[0+bs*0] += a_0 * b_0;
4630 CC[1+bs*0] += a_1 * b_0;
4631 CC[2+bs*0] += a_2 * b_0;
4632 CC[3+bs*0] += a_3 * b_0;
4633
4634 b_0 = B[0+1*ldb];
4635 CC[0+bs*1] += a_0 * b_0;
4636 CC[1+bs*1] += a_1 * b_0;
4637 CC[2+bs*1] += a_2 * b_0;
4638 CC[3+bs*1] += a_3 * b_0;
4639
4640 b_0 = B[0+2*ldb];
4641 CC[0+bs*2] += a_0 * b_0;
4642 CC[1+bs*2] += a_1 * b_0;
4643 CC[2+bs*2] += a_2 * b_0;
4644 CC[3+bs*2] += a_3 * b_0;
4645
4646 A += bs;
4647 B += 1;
4648 // k += 1;
4649
4650 // k = 1
4651
4652 a_0 = A[0];
4653 a_1 = A[1];
4654 a_2 = A[2];
4655 a_3 = A[3];
4656
4657 b_0 = B[0+1*ldb];
4658 CC[0+bs*1] += a_0 * b_0;
4659 CC[1+bs*1] += a_1 * b_0;
4660 CC[2+bs*1] += a_2 * b_0;
4661 CC[3+bs*1] += a_3 * b_0;
4662
4663 b_0 = B[0+2*ldb];
4664 CC[0+bs*2] += a_0 * b_0;
4665 CC[1+bs*2] += a_1 * b_0;
4666 CC[2+bs*2] += a_2 * b_0;
4667 CC[3+bs*2] += a_3 * b_0;
4668
4669 A += bs;
4670 B += 1;
4671 // k += 1;
4672
4673 // k = 2
4674
4675 a_0 = A[0];
4676 a_1 = A[1];
4677 a_2 = A[2];
4678 a_3 = A[3];
4679
4680 b_0 = B[0+2*ldb];
4681 CC[0+bs*2] += a_0 * b_0;
4682 CC[1+bs*2] += a_1 * b_0;
4683 CC[2+bs*2] += a_2 * b_0;
4684 CC[3+bs*2] += a_3 * b_0;
4685
4686 A += bs;
4687 B += 1;
4688 // k += 1;
4689
4690 // k = 3
4691
4692 A += bs;
4693 B += 1;
4694 // k += 1;
4695
4696 }
4697 else if(n1==2)
4698 {
4699
4700 // k = 0
4701
4702 a_0 = A[0];
4703 a_1 = A[1];
4704 a_2 = A[2];
4705 a_3 = A[3];
4706
4707 b_0 = B[0+0*ldb];
4708 CC[0+bs*0] += a_0 * b_0;
4709 CC[1+bs*0] += a_1 * b_0;
4710 CC[2+bs*0] += a_2 * b_0;
4711 CC[3+bs*0] += a_3 * b_0;
4712
4713 b_0 = B[0+1*ldb];
4714 CC[0+bs*1] += a_0 * b_0;
4715 CC[1+bs*1] += a_1 * b_0;
4716 CC[2+bs*1] += a_2 * b_0;
4717 CC[3+bs*1] += a_3 * b_0;
4718
4719 A += bs;
4720 B += 1;
4721 // k += 1;
4722
4723 // k = 1
4724
4725 a_0 = A[0];
4726 a_1 = A[1];
4727 a_2 = A[2];
4728 a_3 = A[3];
4729
4730 b_0 = B[0+1*ldb];
4731 CC[0+bs*1] += a_0 * b_0;
4732 CC[1+bs*1] += a_1 * b_0;
4733 CC[2+bs*1] += a_2 * b_0;
4734 CC[3+bs*1] += a_3 * b_0;
4735
4736 A += bs;
4737 B += 1;
4738 // k += 1;
4739
4740 // k = 2
4741
4742 A += bs;
4743 B += 1;
4744 // k += 1;
4745
4746 // k = 3
4747
4748 A += bs;
4749 B += 1;
4750 // k += 1;
4751
4752 }
4753 else if(n1==1)
4754 {
4755
4756 // k = 0
4757
4758 a_0 = A[0];
4759 a_1 = A[1];
4760 a_2 = A[2];
4761 a_3 = A[3];
4762
4763 b_0 = B[0+0*ldb];
4764 CC[0+bs*0] += a_0 * b_0;
4765 CC[1+bs*0] += a_1 * b_0;
4766 CC[2+bs*0] += a_2 * b_0;
4767 CC[3+bs*0] += a_3 * b_0;
4768
4769 A += bs;
4770 B += 1;
4771 // k += 1;
4772
4773 // k = 1
4774
4775 A += bs;
4776 B += 1;
4777 // k += 1;
4778
4779 // k = 2
4780
4781 A += bs;
4782 B += 1;
4783 // k += 1;
4784
4785 // k = 3
4786
4787 A += bs;
4788 B += 1;
4789 // k += 1;
4790
4791 }
4792
4793 store:
4794
4795 CC[0+bs*0] = alpha[0]*CC[0+bs*0];
4796 CC[1+bs*0] = alpha[0]*CC[1+bs*0];
4797 CC[2+bs*0] = alpha[0]*CC[2+bs*0];
4798 CC[3+bs*0] = alpha[0]*CC[3+bs*0];
4799
4800 CC[0+bs*1] = alpha[0]*CC[0+bs*1];
4801 CC[1+bs*1] = alpha[0]*CC[1+bs*1];
4802 CC[2+bs*1] = alpha[0]*CC[2+bs*1];
4803 CC[3+bs*1] = alpha[0]*CC[3+bs*1];
4804
4805 CC[0+bs*2] = alpha[0]*CC[0+bs*2];
4806 CC[1+bs*2] = alpha[0]*CC[1+bs*2];
4807 CC[2+bs*2] = alpha[0]*CC[2+bs*2];
4808 CC[3+bs*2] = alpha[0]*CC[3+bs*2];
4809
4810 CC[0+bs*3] = alpha[0]*CC[0+bs*3];
4811 CC[1+bs*3] = alpha[0]*CC[1+bs*3];
4812 CC[2+bs*3] = alpha[0]*CC[2+bs*3];
4813 CC[3+bs*3] = alpha[0]*CC[3+bs*3];
4814
4815 if(m1>=4)
4816 {
4817 D[0+ldd*0] = CC[0+bs*0];
4818 D[1+ldd*0] = CC[1+bs*0];
4819 D[2+ldd*0] = CC[2+bs*0];
4820 D[3+ldd*0] = CC[3+bs*0];
4821
4822 if(n1==1)
4823 return;
4824
4825 D[0+ldd*1] = CC[0+bs*1];
4826 D[1+ldd*1] = CC[1+bs*1];
4827 D[2+ldd*1] = CC[2+bs*1];
4828 D[3+ldd*1] = CC[3+bs*1];
4829
4830 if(n1==2)
4831 return;
4832
4833 D[0+ldd*2] = CC[0+bs*2];
4834 D[1+ldd*2] = CC[1+bs*2];
4835 D[2+ldd*2] = CC[2+bs*2];
4836 D[3+ldd*2] = CC[3+bs*2];
4837
4838 if(n1==3)
4839 return;
4840
4841 D[0+ldd*3] = CC[0+bs*3];
4842 D[1+ldd*3] = CC[1+bs*3];
4843 D[2+ldd*3] = CC[2+bs*3];
4844 D[3+ldd*3] = CC[3+bs*3];
4845 }
4846 else if(m1>=3)
4847 {
4848 D[0+ldd*0] = CC[0+bs*0];
4849 D[1+ldd*0] = CC[1+bs*0];
4850 D[2+ldd*0] = CC[2+bs*0];
4851
4852 if(n1==1)
4853 return;
4854
4855 D[0+ldd*1] = CC[0+bs*1];
4856 D[1+ldd*1] = CC[1+bs*1];
4857 D[2+ldd*1] = CC[2+bs*1];
4858
4859 if(n1==2)
4860 return;
4861
4862 D[0+ldd*2] = CC[0+bs*2];
4863 D[1+ldd*2] = CC[1+bs*2];
4864 D[2+ldd*2] = CC[2+bs*2];
4865
4866 if(n1==3)
4867 return;
4868
4869 D[0+ldd*3] = CC[0+bs*3];
4870 D[1+ldd*3] = CC[1+bs*3];
4871 D[2+ldd*3] = CC[2+bs*3];
4872 }
4873 else if(m1>=2)
4874 {
4875 D[0+ldd*0] = CC[0+bs*0];
4876 D[1+ldd*0] = CC[1+bs*0];
4877
4878 if(n1==1)
4879 return;
4880
4881 D[0+ldd*1] = CC[0+bs*1];
4882 D[1+ldd*1] = CC[1+bs*1];
4883
4884 if(n1==2)
4885 return;
4886
4887 D[0+ldd*2] = CC[0+bs*2];
4888 D[1+ldd*2] = CC[1+bs*2];
4889
4890 if(n1==3)
4891 return;
4892
4893 D[0+ldd*3] = CC[0+bs*3];
4894 D[1+ldd*3] = CC[1+bs*3];
4895 }
4896 else //if(m1>=1)
4897 {
4898 D[0+ldd*0] = CC[0+bs*0];
4899
4900 if(n1==1)
4901 return;
4902
4903 D[0+ldd*1] = CC[0+bs*1];
4904
4905 if(n1==2)
4906 return;
4907
4908 D[0+ldd*2] = CC[0+bs*2];
4909
4910 if(n1==3)
4911 return;
4912
4913 D[0+ldd*3] = CC[0+bs*3];
4914 }
4915
4916 return;
4917
4918 }
4919 //#endif
4920
4921
4922
4923 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_ru_4x4_tran_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd)4924 void kernel_strmm_nn_ru_4x4_tran_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd)
4925 {
4926
4927 const int bs = 4;
4928
4929 float
4930 a_0, a_1, a_2, a_3,
4931 b_0, b_1, b_2, b_3;
4932
4933 #if defined(TARGET_GENERIC)
4934 float CC[16] = {0};
4935 #else
4936 ALIGNED( float CC[16], 64 ) = {0};
4937 #endif
4938
4939 float alpha1 = 1.0;
4940
4941 // assume always kmax>=4 !!!
4942
4943 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
4944
4945 A += kmax*bs;
4946 B += kmax;
4947
4948 // k = 0
4949
4950 a_0 = A[0];
4951 a_1 = A[1];
4952 a_2 = A[2];
4953 a_3 = A[3];
4954
4955 b_0 = B[0+0*ldb];
4956 CC[0+bs*0] += a_0 * b_0;
4957 CC[1+bs*0] += a_1 * b_0;
4958 CC[2+bs*0] += a_2 * b_0;
4959 CC[3+bs*0] += a_3 * b_0;
4960
4961 b_0 = B[0+1*ldb];
4962 CC[0+bs*1] += a_0 * b_0;
4963 CC[1+bs*1] += a_1 * b_0;
4964 CC[2+bs*1] += a_2 * b_0;
4965 CC[3+bs*1] += a_3 * b_0;
4966
4967 b_0 = B[0+2*ldb];
4968 CC[0+bs*2] += a_0 * b_0;
4969 CC[1+bs*2] += a_1 * b_0;
4970 CC[2+bs*2] += a_2 * b_0;
4971 CC[3+bs*2] += a_3 * b_0;
4972
4973 b_0 = B[0+3*ldb];
4974 CC[0+bs*3] += a_0 * b_0;
4975 CC[1+bs*3] += a_1 * b_0;
4976 CC[2+bs*3] += a_2 * b_0;
4977 CC[3+bs*3] += a_3 * b_0;
4978
4979 A += bs;
4980 B += 1;
4981 // k += 1;
4982
4983 // k = 1
4984
4985 a_0 = A[0];
4986 a_1 = A[1];
4987 a_2 = A[2];
4988 a_3 = A[3];
4989
4990 b_0 = B[0+1*ldb];
4991 CC[0+bs*1] += a_0 * b_0;
4992 CC[1+bs*1] += a_1 * b_0;
4993 CC[2+bs*1] += a_2 * b_0;
4994 CC[3+bs*1] += a_3 * b_0;
4995
4996 b_0 = B[0+2*ldb];
4997 CC[0+bs*2] += a_0 * b_0;
4998 CC[1+bs*2] += a_1 * b_0;
4999 CC[2+bs*2] += a_2 * b_0;
5000 CC[3+bs*2] += a_3 * b_0;
5001
5002 b_0 = B[0+3*ldb];
5003 CC[0+bs*3] += a_0 * b_0;
5004 CC[1+bs*3] += a_1 * b_0;
5005 CC[2+bs*3] += a_2 * b_0;
5006 CC[3+bs*3] += a_3 * b_0;
5007
5008 A += bs;
5009 B += 1;
5010 // k += 1;
5011
5012 // k = 2
5013
5014 a_0 = A[0];
5015 a_1 = A[1];
5016 a_2 = A[2];
5017 a_3 = A[3];
5018
5019 b_0 = B[0+2*ldb];
5020 CC[0+bs*2] += a_0 * b_0;
5021 CC[1+bs*2] += a_1 * b_0;
5022 CC[2+bs*2] += a_2 * b_0;
5023 CC[3+bs*2] += a_3 * b_0;
5024
5025 b_0 = B[0+3*ldb];
5026 CC[0+bs*3] += a_0 * b_0;
5027 CC[1+bs*3] += a_1 * b_0;
5028 CC[2+bs*3] += a_2 * b_0;
5029 CC[3+bs*3] += a_3 * b_0;
5030
5031 A += bs;
5032 B += 1;
5033 // k += 1;
5034
5035 // k = 3
5036
5037 a_0 = A[0];
5038 a_1 = A[1];
5039 a_2 = A[2];
5040 a_3 = A[3];
5041
5042 b_0 = B[0+3*ldb];
5043 CC[0+bs*3] += a_0 * b_0;
5044 CC[1+bs*3] += a_1 * b_0;
5045 CC[2+bs*3] += a_2 * b_0;
5046 CC[3+bs*3] += a_3 * b_0;
5047
5048 A += bs;
5049 B += 1;
5050 // k += 1;
5051
5052 store:
5053
5054 // scale & tranpose & store
5055 D[0+ldd*0] = alpha[0]*CC[0+bs*0];
5056 D[0+ldd*1] = alpha[0]*CC[1+bs*0];
5057 D[0+ldd*2] = alpha[0]*CC[2+bs*0];
5058 D[0+ldd*3] = alpha[0]*CC[3+bs*0];
5059
5060 D[1+ldd*0] = alpha[0]*CC[0+bs*1];
5061 D[1+ldd*1] = alpha[0]*CC[1+bs*1];
5062 D[1+ldd*2] = alpha[0]*CC[2+bs*1];
5063 D[1+ldd*3] = alpha[0]*CC[3+bs*1];
5064
5065 D[2+ldd*0] = alpha[0]*CC[0+bs*2];
5066 D[2+ldd*1] = alpha[0]*CC[1+bs*2];
5067 D[2+ldd*2] = alpha[0]*CC[2+bs*2];
5068 D[2+ldd*3] = alpha[0]*CC[3+bs*2];
5069
5070 D[3+ldd*0] = alpha[0]*CC[0+bs*3];
5071 D[3+ldd*1] = alpha[0]*CC[1+bs*3];
5072 D[3+ldd*2] = alpha[0]*CC[2+bs*3];
5073 D[3+ldd*3] = alpha[0]*CC[3+bs*3];
5074
5075 return;
5076
5077 }
5078 //#endif
5079
5080
5081
5082 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_ru_4x4_tran_vs_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd,int m1,int n1)5083 void kernel_strmm_nn_ru_4x4_tran_vs_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd, int m1, int n1)
5084 {
5085
5086 const int bs = 4;
5087
5088 float
5089 tmp,
5090 a_0, a_1, a_2, a_3,
5091 b_0, b_1, b_2, b_3;
5092
5093 #if defined(TARGET_GENERIC)
5094 float CC[16] = {0};
5095 #else
5096 ALIGNED( float CC[16], 64 ) = {0};
5097 #endif
5098
5099 float alpha1 = 1.0;
5100
5101 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
5102
5103 A += kmax*bs;
5104 B += kmax;
5105
5106 // XXX m1 and n1 are swapped !!!!!
5107 if(m1>=4)
5108 {
5109
5110 // k = 0
5111
5112 a_0 = A[0];
5113 a_1 = A[1];
5114 a_2 = A[2];
5115 a_3 = A[3];
5116
5117 b_0 = B[0+0*ldb];
5118 CC[0+bs*0] += a_0 * b_0;
5119 CC[1+bs*0] += a_1 * b_0;
5120 CC[2+bs*0] += a_2 * b_0;
5121 CC[3+bs*0] += a_3 * b_0;
5122
5123 b_0 = B[0+1*ldb];
5124 CC[0+bs*1] += a_0 * b_0;
5125 CC[1+bs*1] += a_1 * b_0;
5126 CC[2+bs*1] += a_2 * b_0;
5127 CC[3+bs*1] += a_3 * b_0;
5128
5129 b_0 = B[0+2*ldb];
5130 CC[0+bs*2] += a_0 * b_0;
5131 CC[1+bs*2] += a_1 * b_0;
5132 CC[2+bs*2] += a_2 * b_0;
5133 CC[3+bs*2] += a_3 * b_0;
5134
5135 b_0 = B[0+3*ldb];
5136 CC[0+bs*3] += a_0 * b_0;
5137 CC[1+bs*3] += a_1 * b_0;
5138 CC[2+bs*3] += a_2 * b_0;
5139 CC[3+bs*3] += a_3 * b_0;
5140
5141 A += bs;
5142 B += 1;
5143 // k += 1;
5144
5145 // k = 1
5146
5147 a_0 = A[0];
5148 a_1 = A[1];
5149 a_2 = A[2];
5150 a_3 = A[3];
5151
5152 b_0 = B[0+1*ldb];
5153 CC[0+bs*1] += a_0 * b_0;
5154 CC[1+bs*1] += a_1 * b_0;
5155 CC[2+bs*1] += a_2 * b_0;
5156 CC[3+bs*1] += a_3 * b_0;
5157
5158 b_0 = B[0+2*ldb];
5159 CC[0+bs*2] += a_0 * b_0;
5160 CC[1+bs*2] += a_1 * b_0;
5161 CC[2+bs*2] += a_2 * b_0;
5162 CC[3+bs*2] += a_3 * b_0;
5163
5164 b_0 = B[0+3*ldb];
5165 CC[0+bs*3] += a_0 * b_0;
5166 CC[1+bs*3] += a_1 * b_0;
5167 CC[2+bs*3] += a_2 * b_0;
5168 CC[3+bs*3] += a_3 * b_0;
5169
5170 A += bs;
5171 B += 1;
5172 // k += 1;
5173
5174 // k = 2
5175
5176 a_0 = A[0];
5177 a_1 = A[1];
5178 a_2 = A[2];
5179 a_3 = A[3];
5180
5181 b_0 = B[0+2*ldb];
5182 CC[0+bs*2] += a_0 * b_0;
5183 CC[1+bs*2] += a_1 * b_0;
5184 CC[2+bs*2] += a_2 * b_0;
5185 CC[3+bs*2] += a_3 * b_0;
5186
5187 b_0 = B[0+3*ldb];
5188 CC[0+bs*3] += a_0 * b_0;
5189 CC[1+bs*3] += a_1 * b_0;
5190 CC[2+bs*3] += a_2 * b_0;
5191 CC[3+bs*3] += a_3 * b_0;
5192
5193 A += bs;
5194 B += 1;
5195 // k += 1;
5196
5197 // k = 3
5198
5199 a_0 = A[0];
5200 a_1 = A[1];
5201 a_2 = A[2];
5202 a_3 = A[3];
5203
5204 b_0 = B[0+3*ldb];
5205 CC[0+bs*3] += a_0 * b_0;
5206 CC[1+bs*3] += a_1 * b_0;
5207 CC[2+bs*3] += a_2 * b_0;
5208 CC[3+bs*3] += a_3 * b_0;
5209
5210 A += bs;
5211 B += 1;
5212 // k += 1;
5213
5214 }
5215 else if(m1==3)
5216 {
5217
5218 // k = 0
5219
5220 a_0 = A[0];
5221 a_1 = A[1];
5222 a_2 = A[2];
5223 a_3 = A[3];
5224
5225 b_0 = B[0+0*ldb];
5226 CC[0+bs*0] += a_0 * b_0;
5227 CC[1+bs*0] += a_1 * b_0;
5228 CC[2+bs*0] += a_2 * b_0;
5229 CC[3+bs*0] += a_3 * b_0;
5230
5231 b_0 = B[0+1*ldb];
5232 CC[0+bs*1] += a_0 * b_0;
5233 CC[1+bs*1] += a_1 * b_0;
5234 CC[2+bs*1] += a_2 * b_0;
5235 CC[3+bs*1] += a_3 * b_0;
5236
5237 b_0 = B[0+2*ldb];
5238 CC[0+bs*2] += a_0 * b_0;
5239 CC[1+bs*2] += a_1 * b_0;
5240 CC[2+bs*2] += a_2 * b_0;
5241 CC[3+bs*2] += a_3 * b_0;
5242
5243 A += bs;
5244 B += 1;
5245 // k += 1;
5246
5247 // k = 1
5248
5249 a_0 = A[0];
5250 a_1 = A[1];
5251 a_2 = A[2];
5252 a_3 = A[3];
5253
5254 b_0 = B[0+1*ldb];
5255 CC[0+bs*1] += a_0 * b_0;
5256 CC[1+bs*1] += a_1 * b_0;
5257 CC[2+bs*1] += a_2 * b_0;
5258 CC[3+bs*1] += a_3 * b_0;
5259
5260 b_0 = B[0+2*ldb];
5261 CC[0+bs*2] += a_0 * b_0;
5262 CC[1+bs*2] += a_1 * b_0;
5263 CC[2+bs*2] += a_2 * b_0;
5264 CC[3+bs*2] += a_3 * b_0;
5265
5266 A += bs;
5267 B += 1;
5268 // k += 1;
5269
5270 // k = 2
5271
5272 a_0 = A[0];
5273 a_1 = A[1];
5274 a_2 = A[2];
5275 a_3 = A[3];
5276
5277 b_0 = B[0+2*ldb];
5278 CC[0+bs*2] += a_0 * b_0;
5279 CC[1+bs*2] += a_1 * b_0;
5280 CC[2+bs*2] += a_2 * b_0;
5281 CC[3+bs*2] += a_3 * b_0;
5282
5283 A += bs;
5284 B += 1;
5285 // k += 1;
5286
5287 // k = 3
5288
5289 A += bs;
5290 B += 1;
5291 // k += 1;
5292
5293 }
5294 else if(m1==2)
5295 {
5296
5297 // k = 0
5298
5299 a_0 = A[0];
5300 a_1 = A[1];
5301 a_2 = A[2];
5302 a_3 = A[3];
5303
5304 b_0 = B[0+0*ldb];
5305 CC[0+bs*0] += a_0 * b_0;
5306 CC[1+bs*0] += a_1 * b_0;
5307 CC[2+bs*0] += a_2 * b_0;
5308 CC[3+bs*0] += a_3 * b_0;
5309
5310 b_0 = B[0+1*ldb];
5311 CC[0+bs*1] += a_0 * b_0;
5312 CC[1+bs*1] += a_1 * b_0;
5313 CC[2+bs*1] += a_2 * b_0;
5314 CC[3+bs*1] += a_3 * b_0;
5315
5316 A += bs;
5317 B += 1;
5318 // k += 1;
5319
5320 // k = 1
5321
5322 a_0 = A[0];
5323 a_1 = A[1];
5324 a_2 = A[2];
5325 a_3 = A[3];
5326
5327 b_0 = B[0+1*ldb];
5328 CC[0+bs*1] += a_0 * b_0;
5329 CC[1+bs*1] += a_1 * b_0;
5330 CC[2+bs*1] += a_2 * b_0;
5331 CC[3+bs*1] += a_3 * b_0;
5332
5333 A += bs;
5334 B += 1;
5335 // k += 1;
5336
5337 // k = 2
5338
5339 A += bs;
5340 B += 1;
5341 // k += 1;
5342
5343 // k = 3
5344
5345 A += bs;
5346 B += 1;
5347 // k += 1;
5348
5349 }
5350 else if(m1==1)
5351 {
5352
5353 // k = 0
5354
5355 a_0 = A[0];
5356 a_1 = A[1];
5357 a_2 = A[2];
5358 a_3 = A[3];
5359
5360 b_0 = B[0+0*ldb];
5361 CC[0+bs*0] += a_0 * b_0;
5362 CC[1+bs*0] += a_1 * b_0;
5363 CC[2+bs*0] += a_2 * b_0;
5364 CC[3+bs*0] += a_3 * b_0;
5365
5366 A += bs;
5367 B += 1;
5368 // k += 1;
5369
5370 // k = 1
5371
5372 A += bs;
5373 B += 1;
5374 // k += 1;
5375
5376 // k = 2
5377
5378 A += bs;
5379 B += 1;
5380 // k += 1;
5381
5382 // k = 3
5383
5384 A += bs;
5385 B += 1;
5386 // k += 1;
5387
5388 }
5389
5390 store:
5391
5392 // scale
5393 CC[0+bs*0] = alpha[0]*CC[0+bs*0];
5394 CC[1+bs*0] = alpha[0]*CC[1+bs*0];
5395 CC[2+bs*0] = alpha[0]*CC[2+bs*0];
5396 CC[3+bs*0] = alpha[0]*CC[3+bs*0];
5397
5398 CC[0+bs*1] = alpha[0]*CC[0+bs*1];
5399 CC[1+bs*1] = alpha[0]*CC[1+bs*1];
5400 CC[2+bs*1] = alpha[0]*CC[2+bs*1];
5401 CC[3+bs*1] = alpha[0]*CC[3+bs*1];
5402
5403 CC[0+bs*2] = alpha[0]*CC[0+bs*2];
5404 CC[1+bs*2] = alpha[0]*CC[1+bs*2];
5405 CC[2+bs*2] = alpha[0]*CC[2+bs*2];
5406 CC[3+bs*2] = alpha[0]*CC[3+bs*2];
5407
5408 CC[0+bs*3] = alpha[0]*CC[0+bs*3];
5409 CC[1+bs*3] = alpha[0]*CC[1+bs*3];
5410 CC[2+bs*3] = alpha[0]*CC[2+bs*3];
5411 CC[3+bs*3] = alpha[0]*CC[3+bs*3];
5412
5413 // transpose
5414 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
5415 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
5416 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
5417
5418 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
5419 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
5420
5421 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
5422
5423 // store
5424 if(m1>=4)
5425 {
5426 D[0+ldd*0] = CC[0+bs*0];
5427 D[1+ldd*0] = CC[1+bs*0];
5428 D[2+ldd*0] = CC[2+bs*0];
5429 D[3+ldd*0] = CC[3+bs*0];
5430
5431 if(n1==1)
5432 return;
5433
5434 D[0+ldd*1] = CC[0+bs*1];
5435 D[1+ldd*1] = CC[1+bs*1];
5436 D[2+ldd*1] = CC[2+bs*1];
5437 D[3+ldd*1] = CC[3+bs*1];
5438
5439 if(n1==2)
5440 return;
5441
5442 D[0+ldd*2] = CC[0+bs*2];
5443 D[1+ldd*2] = CC[1+bs*2];
5444 D[2+ldd*2] = CC[2+bs*2];
5445 D[3+ldd*2] = CC[3+bs*2];
5446
5447 if(n1==3)
5448 return;
5449
5450 D[0+ldd*3] = CC[0+bs*3];
5451 D[1+ldd*3] = CC[1+bs*3];
5452 D[2+ldd*3] = CC[2+bs*3];
5453 D[3+ldd*3] = CC[3+bs*3];
5454 }
5455 else if(m1>=3)
5456 {
5457 D[0+ldd*0] = CC[0+bs*0];
5458 D[1+ldd*0] = CC[1+bs*0];
5459 D[2+ldd*0] = CC[2+bs*0];
5460
5461 if(n1==1)
5462 return;
5463
5464 D[0+ldd*1] = CC[0+bs*1];
5465 D[1+ldd*1] = CC[1+bs*1];
5466 D[2+ldd*1] = CC[2+bs*1];
5467
5468 if(n1==2)
5469 return;
5470
5471 D[0+ldd*2] = CC[0+bs*2];
5472 D[1+ldd*2] = CC[1+bs*2];
5473 D[2+ldd*2] = CC[2+bs*2];
5474
5475 if(n1==3)
5476 return;
5477
5478 D[0+ldd*3] = CC[0+bs*3];
5479 D[1+ldd*3] = CC[1+bs*3];
5480 D[2+ldd*3] = CC[2+bs*3];
5481 }
5482 else if(m1>=2)
5483 {
5484 D[0+ldd*0] = CC[0+bs*0];
5485 D[1+ldd*0] = CC[1+bs*0];
5486
5487 if(n1==1)
5488 return;
5489
5490 D[0+ldd*1] = CC[0+bs*1];
5491 D[1+ldd*1] = CC[1+bs*1];
5492
5493 if(n1==2)
5494 return;
5495
5496 D[0+ldd*2] = CC[0+bs*2];
5497 D[1+ldd*2] = CC[1+bs*2];
5498
5499 if(n1==3)
5500 return;
5501
5502 D[0+ldd*3] = CC[0+bs*3];
5503 D[1+ldd*3] = CC[1+bs*3];
5504 }
5505 else //if(m1>=1)
5506 {
5507 D[0+ldd*0] = CC[0+bs*0];
5508
5509 if(n1==1)
5510 return;
5511
5512 D[0+ldd*1] = CC[0+bs*1];
5513
5514 if(n1==2)
5515 return;
5516
5517 D[0+ldd*2] = CC[0+bs*2];
5518
5519 if(n1==3)
5520 return;
5521
5522 D[0+ldd*3] = CC[0+bs*3];
5523 }
5524
5525 return;
5526
5527 }
5528 //#endif
5529
5530
5531
5532 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_ru_one_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)5533 void kernel_strmm_nn_ru_one_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
5534 {
5535
5536 const int bs = 4;
5537
5538 float
5539 a_0, a_1, a_2, a_3,
5540 b_0, b_1, b_2, b_3;
5541
5542 #if defined(TARGET_GENERIC)
5543 float CC[16] = {0};
5544 #else
5545 ALIGNED( float CC[16], 64 ) = {0};
5546 #endif
5547
5548 float alpha1 = 1.0;
5549
5550 // assume always kmax>=4 !!!
5551
5552 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
5553
5554 A += kmax*bs;
5555 B += kmax;
5556
5557 // k = 0
5558
5559 a_0 = A[0];
5560 a_1 = A[1];
5561 a_2 = A[2];
5562 a_3 = A[3];
5563
5564 CC[0+bs*0] += a_0;
5565 CC[1+bs*0] += a_1;
5566 CC[2+bs*0] += a_2;
5567 CC[3+bs*0] += a_3;
5568
5569 b_0 = B[0+1*ldb];
5570 CC[0+bs*1] += a_0 * b_0;
5571 CC[1+bs*1] += a_1 * b_0;
5572 CC[2+bs*1] += a_2 * b_0;
5573 CC[3+bs*1] += a_3 * b_0;
5574
5575 b_0 = B[0+2*ldb];
5576 CC[0+bs*2] += a_0 * b_0;
5577 CC[1+bs*2] += a_1 * b_0;
5578 CC[2+bs*2] += a_2 * b_0;
5579 CC[3+bs*2] += a_3 * b_0;
5580
5581 b_0 = B[0+3*ldb];
5582 CC[0+bs*3] += a_0 * b_0;
5583 CC[1+bs*3] += a_1 * b_0;
5584 CC[2+bs*3] += a_2 * b_0;
5585 CC[3+bs*3] += a_3 * b_0;
5586
5587 A += bs;
5588 B += 1;
5589 // k += 1;
5590
5591 // k = 1
5592
5593 a_0 = A[0];
5594 a_1 = A[1];
5595 a_2 = A[2];
5596 a_3 = A[3];
5597
5598 CC[0+bs*1] += a_0;
5599 CC[1+bs*1] += a_1;
5600 CC[2+bs*1] += a_2;
5601 CC[3+bs*1] += a_3;
5602
5603 b_0 = B[0+2*ldb];
5604 CC[0+bs*2] += a_0 * b_0;
5605 CC[1+bs*2] += a_1 * b_0;
5606 CC[2+bs*2] += a_2 * b_0;
5607 CC[3+bs*2] += a_3 * b_0;
5608
5609 b_0 = B[0+3*ldb];
5610 CC[0+bs*3] += a_0 * b_0;
5611 CC[1+bs*3] += a_1 * b_0;
5612 CC[2+bs*3] += a_2 * b_0;
5613 CC[3+bs*3] += a_3 * b_0;
5614
5615 A += bs;
5616 B += 1;
5617 // k += 1;
5618
5619 // k = 2
5620
5621 a_0 = A[0];
5622 a_1 = A[1];
5623 a_2 = A[2];
5624 a_3 = A[3];
5625
5626 CC[0+bs*2] += a_0;
5627 CC[1+bs*2] += a_1;
5628 CC[2+bs*2] += a_2;
5629 CC[3+bs*2] += a_3;
5630
5631 b_0 = B[0+3*ldb];
5632 CC[0+bs*3] += a_0 * b_0;
5633 CC[1+bs*3] += a_1 * b_0;
5634 CC[2+bs*3] += a_2 * b_0;
5635 CC[3+bs*3] += a_3 * b_0;
5636
5637 A += bs;
5638 B += 1;
5639 // k += 1;
5640
5641 // k = 3
5642
5643 a_0 = A[0];
5644 a_1 = A[1];
5645 a_2 = A[2];
5646 a_3 = A[3];
5647
5648 CC[0+bs*3] += a_0;
5649 CC[1+bs*3] += a_1;
5650 CC[2+bs*3] += a_2;
5651 CC[3+bs*3] += a_3;
5652
5653 A += bs;
5654 B += 1;
5655 // k += 1;
5656
5657 store:
5658
5659 D[0+ldd*0] = alpha[0]*CC[0+bs*0];
5660 D[1+ldd*0] = alpha[0]*CC[1+bs*0];
5661 D[2+ldd*0] = alpha[0]*CC[2+bs*0];
5662 D[3+ldd*0] = alpha[0]*CC[3+bs*0];
5663
5664 D[0+ldd*1] = alpha[0]*CC[0+bs*1];
5665 D[1+ldd*1] = alpha[0]*CC[1+bs*1];
5666 D[2+ldd*1] = alpha[0]*CC[2+bs*1];
5667 D[3+ldd*1] = alpha[0]*CC[3+bs*1];
5668
5669 D[0+ldd*2] = alpha[0]*CC[0+bs*2];
5670 D[1+ldd*2] = alpha[0]*CC[1+bs*2];
5671 D[2+ldd*2] = alpha[0]*CC[2+bs*2];
5672 D[3+ldd*2] = alpha[0]*CC[3+bs*2];
5673
5674 D[0+ldd*3] = alpha[0]*CC[0+bs*3];
5675 D[1+ldd*3] = alpha[0]*CC[1+bs*3];
5676 D[2+ldd*3] = alpha[0]*CC[2+bs*3];
5677 D[3+ldd*3] = alpha[0]*CC[3+bs*3];
5678
5679 return;
5680
5681 }
5682 //#endif
5683
5684
5685
5686 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_ru_one_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)5687 void kernel_strmm_nn_ru_one_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
5688 {
5689
5690 const int bs = 4;
5691
5692 float
5693 a_0, a_1, a_2, a_3,
5694 b_0, b_1, b_2, b_3;
5695
5696 #if defined(TARGET_GENERIC)
5697 float CC[16] = {0};
5698 #else
5699 ALIGNED( float CC[16], 64 ) = {0};
5700 #endif
5701
5702 float alpha1 = 1.0;
5703
5704 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
5705
5706 A += kmax*bs;
5707 B += kmax;
5708
5709 if(n1>=4)
5710 {
5711
5712 // k = 0
5713
5714 a_0 = A[0];
5715 a_1 = A[1];
5716 a_2 = A[2];
5717 a_3 = A[3];
5718
5719 CC[0+bs*0] += a_0;
5720 CC[1+bs*0] += a_1;
5721 CC[2+bs*0] += a_2;
5722 CC[3+bs*0] += a_3;
5723
5724 b_0 = B[0+1*ldb];
5725 CC[0+bs*1] += a_0 * b_0;
5726 CC[1+bs*1] += a_1 * b_0;
5727 CC[2+bs*1] += a_2 * b_0;
5728 CC[3+bs*1] += a_3 * b_0;
5729
5730 b_0 = B[0+2*ldb];
5731 CC[0+bs*2] += a_0 * b_0;
5732 CC[1+bs*2] += a_1 * b_0;
5733 CC[2+bs*2] += a_2 * b_0;
5734 CC[3+bs*2] += a_3 * b_0;
5735
5736 b_0 = B[0+3*ldb];
5737 CC[0+bs*3] += a_0 * b_0;
5738 CC[1+bs*3] += a_1 * b_0;
5739 CC[2+bs*3] += a_2 * b_0;
5740 CC[3+bs*3] += a_3 * b_0;
5741
5742 A += bs;
5743 B += 1;
5744 // k += 1;
5745
5746 // k = 1
5747
5748 a_0 = A[0];
5749 a_1 = A[1];
5750 a_2 = A[2];
5751 a_3 = A[3];
5752
5753 CC[0+bs*1] += a_0;
5754 CC[1+bs*1] += a_1;
5755 CC[2+bs*1] += a_2;
5756 CC[3+bs*1] += a_3;
5757
5758 b_0 = B[0+2*ldb];
5759 CC[0+bs*2] += a_0 * b_0;
5760 CC[1+bs*2] += a_1 * b_0;
5761 CC[2+bs*2] += a_2 * b_0;
5762 CC[3+bs*2] += a_3 * b_0;
5763
5764 b_0 = B[0+3*ldb];
5765 CC[0+bs*3] += a_0 * b_0;
5766 CC[1+bs*3] += a_1 * b_0;
5767 CC[2+bs*3] += a_2 * b_0;
5768 CC[3+bs*3] += a_3 * b_0;
5769
5770 A += bs;
5771 B += 1;
5772 // k += 1;
5773
5774 // k = 2
5775
5776 a_0 = A[0];
5777 a_1 = A[1];
5778 a_2 = A[2];
5779 a_3 = A[3];
5780
5781 CC[0+bs*2] += a_0;
5782 CC[1+bs*2] += a_1;
5783 CC[2+bs*2] += a_2;
5784 CC[3+bs*2] += a_3;
5785
5786 b_0 = B[0+3*ldb];
5787 CC[0+bs*3] += a_0 * b_0;
5788 CC[1+bs*3] += a_1 * b_0;
5789 CC[2+bs*3] += a_2 * b_0;
5790 CC[3+bs*3] += a_3 * b_0;
5791
5792 A += bs;
5793 B += 1;
5794 // k += 1;
5795
5796 // k = 3
5797
5798 a_0 = A[0];
5799 a_1 = A[1];
5800 a_2 = A[2];
5801 a_3 = A[3];
5802
5803 CC[0+bs*3] += a_0;
5804 CC[1+bs*3] += a_1;
5805 CC[2+bs*3] += a_2;
5806 CC[3+bs*3] += a_3;
5807
5808 A += bs;
5809 B += 1;
5810 // k += 1;
5811
5812 }
5813 else if(n1==3)
5814 {
5815
5816 // k = 0
5817
5818 a_0 = A[0];
5819 a_1 = A[1];
5820 a_2 = A[2];
5821 a_3 = A[3];
5822
5823 CC[0+bs*0] += a_0;
5824 CC[1+bs*0] += a_1;
5825 CC[2+bs*0] += a_2;
5826 CC[3+bs*0] += a_3;
5827
5828 b_0 = B[0+1*ldb];
5829 CC[0+bs*1] += a_0 * b_0;
5830 CC[1+bs*1] += a_1 * b_0;
5831 CC[2+bs*1] += a_2 * b_0;
5832 CC[3+bs*1] += a_3 * b_0;
5833
5834 b_0 = B[0+2*ldb];
5835 CC[0+bs*2] += a_0 * b_0;
5836 CC[1+bs*2] += a_1 * b_0;
5837 CC[2+bs*2] += a_2 * b_0;
5838 CC[3+bs*2] += a_3 * b_0;
5839
5840 A += bs;
5841 B += 1;
5842 // k += 1;
5843
5844 // k = 1
5845
5846 a_0 = A[0];
5847 a_1 = A[1];
5848 a_2 = A[2];
5849 a_3 = A[3];
5850
5851 CC[0+bs*1] += a_0;
5852 CC[1+bs*1] += a_1;
5853 CC[2+bs*1] += a_2;
5854 CC[3+bs*1] += a_3;
5855
5856 b_0 = B[0+2*ldb];
5857 CC[0+bs*2] += a_0 * b_0;
5858 CC[1+bs*2] += a_1 * b_0;
5859 CC[2+bs*2] += a_2 * b_0;
5860 CC[3+bs*2] += a_3 * b_0;
5861
5862 A += bs;
5863 B += 1;
5864 // k += 1;
5865
5866 // k = 2
5867
5868 a_0 = A[0];
5869 a_1 = A[1];
5870 a_2 = A[2];
5871 a_3 = A[3];
5872
5873 CC[0+bs*2] += a_0;
5874 CC[1+bs*2] += a_1;
5875 CC[2+bs*2] += a_2;
5876 CC[3+bs*2] += a_3;
5877
5878 A += bs;
5879 B += 1;
5880 // k += 1;
5881
5882 // k = 3
5883
5884 A += bs;
5885 B += 1;
5886 // k += 1;
5887
5888 }
5889 else if(n1==2)
5890 {
5891
5892 // k = 0
5893
5894 a_0 = A[0];
5895 a_1 = A[1];
5896 a_2 = A[2];
5897 a_3 = A[3];
5898
5899 CC[0+bs*0] += a_0;
5900 CC[1+bs*0] += a_1;
5901 CC[2+bs*0] += a_2;
5902 CC[3+bs*0] += a_3;
5903
5904 b_0 = B[0+1*ldb];
5905 CC[0+bs*1] += a_0 * b_0;
5906 CC[1+bs*1] += a_1 * b_0;
5907 CC[2+bs*1] += a_2 * b_0;
5908 CC[3+bs*1] += a_3 * b_0;
5909
5910 A += bs;
5911 B += 1;
5912 // k += 1;
5913
5914 // k = 1
5915
5916 a_0 = A[0];
5917 a_1 = A[1];
5918 a_2 = A[2];
5919 a_3 = A[3];
5920
5921 CC[0+bs*1] += a_0;
5922 CC[1+bs*1] += a_1;
5923 CC[2+bs*1] += a_2;
5924 CC[3+bs*1] += a_3;
5925
5926 A += bs;
5927 B += 1;
5928 // k += 1;
5929
5930 // k = 2
5931
5932 A += bs;
5933 B += 1;
5934 // k += 1;
5935
5936 // k = 3
5937
5938 A += bs;
5939 B += 1;
5940 // k += 1;
5941
5942 }
5943 else if(n1==1)
5944 {
5945
5946 // k = 0
5947
5948 a_0 = A[0];
5949 a_1 = A[1];
5950 a_2 = A[2];
5951 a_3 = A[3];
5952
5953 CC[0+bs*0] += a_0;
5954 CC[1+bs*0] += a_1;
5955 CC[2+bs*0] += a_2;
5956 CC[3+bs*0] += a_3;
5957
5958 A += bs;
5959 B += 1;
5960 // k += 1;
5961
5962 // k = 1
5963
5964 A += bs;
5965 B += 1;
5966 // k += 1;
5967
5968 // k = 2
5969
5970 A += bs;
5971 B += 1;
5972 // k += 1;
5973
5974 // k = 3
5975
5976 A += bs;
5977 B += 1;
5978 // k += 1;
5979
5980 }
5981
5982 store:
5983
5984 CC[0+bs*0] = alpha[0]*CC[0+bs*0];
5985 CC[1+bs*0] = alpha[0]*CC[1+bs*0];
5986 CC[2+bs*0] = alpha[0]*CC[2+bs*0];
5987 CC[3+bs*0] = alpha[0]*CC[3+bs*0];
5988
5989 CC[0+bs*1] = alpha[0]*CC[0+bs*1];
5990 CC[1+bs*1] = alpha[0]*CC[1+bs*1];
5991 CC[2+bs*1] = alpha[0]*CC[2+bs*1];
5992 CC[3+bs*1] = alpha[0]*CC[3+bs*1];
5993
5994 CC[0+bs*2] = alpha[0]*CC[0+bs*2];
5995 CC[1+bs*2] = alpha[0]*CC[1+bs*2];
5996 CC[2+bs*2] = alpha[0]*CC[2+bs*2];
5997 CC[3+bs*2] = alpha[0]*CC[3+bs*2];
5998
5999 CC[0+bs*3] = alpha[0]*CC[0+bs*3];
6000 CC[1+bs*3] = alpha[0]*CC[1+bs*3];
6001 CC[2+bs*3] = alpha[0]*CC[2+bs*3];
6002 CC[3+bs*3] = alpha[0]*CC[3+bs*3];
6003
6004 if(m1>=4)
6005 {
6006 D[0+ldd*0] = CC[0+bs*0];
6007 D[1+ldd*0] = CC[1+bs*0];
6008 D[2+ldd*0] = CC[2+bs*0];
6009 D[3+ldd*0] = CC[3+bs*0];
6010
6011 if(n1==1)
6012 return;
6013
6014 D[0+ldd*1] = CC[0+bs*1];
6015 D[1+ldd*1] = CC[1+bs*1];
6016 D[2+ldd*1] = CC[2+bs*1];
6017 D[3+ldd*1] = CC[3+bs*1];
6018
6019 if(n1==2)
6020 return;
6021
6022 D[0+ldd*2] = CC[0+bs*2];
6023 D[1+ldd*2] = CC[1+bs*2];
6024 D[2+ldd*2] = CC[2+bs*2];
6025 D[3+ldd*2] = CC[3+bs*2];
6026
6027 if(n1==3)
6028 return;
6029
6030 D[0+ldd*3] = CC[0+bs*3];
6031 D[1+ldd*3] = CC[1+bs*3];
6032 D[2+ldd*3] = CC[2+bs*3];
6033 D[3+ldd*3] = CC[3+bs*3];
6034 }
6035 else if(m1>=3)
6036 {
6037 D[0+ldd*0] = CC[0+bs*0];
6038 D[1+ldd*0] = CC[1+bs*0];
6039 D[2+ldd*0] = CC[2+bs*0];
6040
6041 if(n1==1)
6042 return;
6043
6044 D[0+ldd*1] = CC[0+bs*1];
6045 D[1+ldd*1] = CC[1+bs*1];
6046 D[2+ldd*1] = CC[2+bs*1];
6047
6048 if(n1==2)
6049 return;
6050
6051 D[0+ldd*2] = CC[0+bs*2];
6052 D[1+ldd*2] = CC[1+bs*2];
6053 D[2+ldd*2] = CC[2+bs*2];
6054
6055 if(n1==3)
6056 return;
6057
6058 D[0+ldd*3] = CC[0+bs*3];
6059 D[1+ldd*3] = CC[1+bs*3];
6060 D[2+ldd*3] = CC[2+bs*3];
6061 }
6062 else if(m1>=2)
6063 {
6064 D[0+ldd*0] = CC[0+bs*0];
6065 D[1+ldd*0] = CC[1+bs*0];
6066
6067 if(n1==1)
6068 return;
6069
6070 D[0+ldd*1] = CC[0+bs*1];
6071 D[1+ldd*1] = CC[1+bs*1];
6072
6073 if(n1==2)
6074 return;
6075
6076 D[0+ldd*2] = CC[0+bs*2];
6077 D[1+ldd*2] = CC[1+bs*2];
6078
6079 if(n1==3)
6080 return;
6081
6082 D[0+ldd*3] = CC[0+bs*3];
6083 D[1+ldd*3] = CC[1+bs*3];
6084 }
6085 else //if(m1>=1)
6086 {
6087 D[0+ldd*0] = CC[0+bs*0];
6088
6089 if(n1==1)
6090 return;
6091
6092 D[0+ldd*1] = CC[0+bs*1];
6093
6094 if(n1==2)
6095 return;
6096
6097 D[0+ldd*2] = CC[0+bs*2];
6098
6099 if(n1==3)
6100 return;
6101
6102 D[0+ldd*3] = CC[0+bs*3];
6103 }
6104
6105 return;
6106
6107 }
6108 //#endif
6109
6110
6111
6112 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_ru_one_4x4_tran_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd)6113 void kernel_strmm_nn_ru_one_4x4_tran_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd)
6114 {
6115
6116 const int bs = 4;
6117
6118 float
6119 a_0, a_1, a_2, a_3,
6120 b_0, b_1, b_2, b_3;
6121
6122 #if defined(TARGET_GENERIC)
6123 float CC[16] = {0};
6124 #else
6125 ALIGNED( float CC[16], 64 ) = {0};
6126 #endif
6127
6128 float alpha1 = 1.0;
6129
6130 // assume always kmax>=4 !!!
6131
6132 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
6133
6134 A += kmax*bs;
6135 B += kmax;
6136
6137 // k = 0
6138
6139 a_0 = A[0];
6140 a_1 = A[1];
6141 a_2 = A[2];
6142 a_3 = A[3];
6143
6144 CC[0+bs*0] += a_0;
6145 CC[1+bs*0] += a_1;
6146 CC[2+bs*0] += a_2;
6147 CC[3+bs*0] += a_3;
6148
6149 b_0 = B[0+1*ldb];
6150 CC[0+bs*1] += a_0 * b_0;
6151 CC[1+bs*1] += a_1 * b_0;
6152 CC[2+bs*1] += a_2 * b_0;
6153 CC[3+bs*1] += a_3 * b_0;
6154
6155 b_0 = B[0+2*ldb];
6156 CC[0+bs*2] += a_0 * b_0;
6157 CC[1+bs*2] += a_1 * b_0;
6158 CC[2+bs*2] += a_2 * b_0;
6159 CC[3+bs*2] += a_3 * b_0;
6160
6161 b_0 = B[0+3*ldb];
6162 CC[0+bs*3] += a_0 * b_0;
6163 CC[1+bs*3] += a_1 * b_0;
6164 CC[2+bs*3] += a_2 * b_0;
6165 CC[3+bs*3] += a_3 * b_0;
6166
6167 A += bs;
6168 B += 1;
6169 // k += 1;
6170
6171 // k = 1
6172
6173 a_0 = A[0];
6174 a_1 = A[1];
6175 a_2 = A[2];
6176 a_3 = A[3];
6177
6178 CC[0+bs*1] += a_0;
6179 CC[1+bs*1] += a_1;
6180 CC[2+bs*1] += a_2;
6181 CC[3+bs*1] += a_3;
6182
6183 b_0 = B[0+2*ldb];
6184 CC[0+bs*2] += a_0 * b_0;
6185 CC[1+bs*2] += a_1 * b_0;
6186 CC[2+bs*2] += a_2 * b_0;
6187 CC[3+bs*2] += a_3 * b_0;
6188
6189 b_0 = B[0+3*ldb];
6190 CC[0+bs*3] += a_0 * b_0;
6191 CC[1+bs*3] += a_1 * b_0;
6192 CC[2+bs*3] += a_2 * b_0;
6193 CC[3+bs*3] += a_3 * b_0;
6194
6195 A += bs;
6196 B += 1;
6197 // k += 1;
6198
6199 // k = 2
6200
6201 a_0 = A[0];
6202 a_1 = A[1];
6203 a_2 = A[2];
6204 a_3 = A[3];
6205
6206 CC[0+bs*2] += a_0;
6207 CC[1+bs*2] += a_1;
6208 CC[2+bs*2] += a_2;
6209 CC[3+bs*2] += a_3;
6210
6211 b_0 = B[0+3*ldb];
6212 CC[0+bs*3] += a_0 * b_0;
6213 CC[1+bs*3] += a_1 * b_0;
6214 CC[2+bs*3] += a_2 * b_0;
6215 CC[3+bs*3] += a_3 * b_0;
6216
6217 A += bs;
6218 B += 1;
6219 // k += 1;
6220
6221 // k = 3
6222
6223 a_0 = A[0];
6224 a_1 = A[1];
6225 a_2 = A[2];
6226 a_3 = A[3];
6227
6228 CC[0+bs*3] += a_0;
6229 CC[1+bs*3] += a_1;
6230 CC[2+bs*3] += a_2;
6231 CC[3+bs*3] += a_3;
6232
6233 A += bs;
6234 B += 1;
6235 // k += 1;
6236
6237 store:
6238
6239 // scale & tranpose & store
6240 D[0+ldd*0] = alpha[0]*CC[0+bs*0];
6241 D[0+ldd*1] = alpha[0]*CC[1+bs*0];
6242 D[0+ldd*2] = alpha[0]*CC[2+bs*0];
6243 D[0+ldd*3] = alpha[0]*CC[3+bs*0];
6244
6245 D[1+ldd*0] = alpha[0]*CC[0+bs*1];
6246 D[1+ldd*1] = alpha[0]*CC[1+bs*1];
6247 D[1+ldd*2] = alpha[0]*CC[2+bs*1];
6248 D[1+ldd*3] = alpha[0]*CC[3+bs*1];
6249
6250 D[2+ldd*0] = alpha[0]*CC[0+bs*2];
6251 D[2+ldd*1] = alpha[0]*CC[1+bs*2];
6252 D[2+ldd*2] = alpha[0]*CC[2+bs*2];
6253 D[2+ldd*3] = alpha[0]*CC[3+bs*2];
6254
6255 D[3+ldd*0] = alpha[0]*CC[0+bs*3];
6256 D[3+ldd*1] = alpha[0]*CC[1+bs*3];
6257 D[3+ldd*2] = alpha[0]*CC[2+bs*3];
6258 D[3+ldd*3] = alpha[0]*CC[3+bs*3];
6259
6260 return;
6261
6262 }
6263 //#endif
6264
6265
6266
6267 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nn_ru_one_4x4_tran_vs_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd,int m1,int n1)6268 void kernel_strmm_nn_ru_one_4x4_tran_vs_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd, int m1, int n1)
6269 {
6270
6271 const int bs = 4;
6272
6273 float
6274 tmp,
6275 a_0, a_1, a_2, a_3,
6276 b_0, b_1, b_2, b_3;
6277
6278 #if defined(TARGET_GENERIC)
6279 float CC[16] = {0};
6280 #else
6281 ALIGNED( float CC[16], 64 ) = {0};
6282 #endif
6283
6284 float alpha1 = 1.0;
6285
6286 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
6287
6288 A += kmax*bs;
6289 B += kmax;
6290
6291 // XXX m1 and n1 are swapped !!!!!
6292 if(m1>=4)
6293 {
6294
6295 // k = 0
6296
6297 a_0 = A[0];
6298 a_1 = A[1];
6299 a_2 = A[2];
6300 a_3 = A[3];
6301
6302 CC[0+bs*0] += a_0;
6303 CC[1+bs*0] += a_1;
6304 CC[2+bs*0] += a_2;
6305 CC[3+bs*0] += a_3;
6306
6307 b_0 = B[0+1*ldb];
6308 CC[0+bs*1] += a_0 * b_0;
6309 CC[1+bs*1] += a_1 * b_0;
6310 CC[2+bs*1] += a_2 * b_0;
6311 CC[3+bs*1] += a_3 * b_0;
6312
6313 b_0 = B[0+2*ldb];
6314 CC[0+bs*2] += a_0 * b_0;
6315 CC[1+bs*2] += a_1 * b_0;
6316 CC[2+bs*2] += a_2 * b_0;
6317 CC[3+bs*2] += a_3 * b_0;
6318
6319 b_0 = B[0+3*ldb];
6320 CC[0+bs*3] += a_0 * b_0;
6321 CC[1+bs*3] += a_1 * b_0;
6322 CC[2+bs*3] += a_2 * b_0;
6323 CC[3+bs*3] += a_3 * b_0;
6324
6325 A += bs;
6326 B += 1;
6327 // k += 1;
6328
6329 // k = 1
6330
6331 a_0 = A[0];
6332 a_1 = A[1];
6333 a_2 = A[2];
6334 a_3 = A[3];
6335
6336 CC[0+bs*1] += a_0;
6337 CC[1+bs*1] += a_1;
6338 CC[2+bs*1] += a_2;
6339 CC[3+bs*1] += a_3;
6340
6341 b_0 = B[0+2*ldb];
6342 CC[0+bs*2] += a_0 * b_0;
6343 CC[1+bs*2] += a_1 * b_0;
6344 CC[2+bs*2] += a_2 * b_0;
6345 CC[3+bs*2] += a_3 * b_0;
6346
6347 b_0 = B[0+3*ldb];
6348 CC[0+bs*3] += a_0 * b_0;
6349 CC[1+bs*3] += a_1 * b_0;
6350 CC[2+bs*3] += a_2 * b_0;
6351 CC[3+bs*3] += a_3 * b_0;
6352
6353 A += bs;
6354 B += 1;
6355 // k += 1;
6356
6357 // k = 2
6358
6359 a_0 = A[0];
6360 a_1 = A[1];
6361 a_2 = A[2];
6362 a_3 = A[3];
6363
6364 CC[0+bs*2] += a_0;
6365 CC[1+bs*2] += a_1;
6366 CC[2+bs*2] += a_2;
6367 CC[3+bs*2] += a_3;
6368
6369 b_0 = B[0+3*ldb];
6370 CC[0+bs*3] += a_0 * b_0;
6371 CC[1+bs*3] += a_1 * b_0;
6372 CC[2+bs*3] += a_2 * b_0;
6373 CC[3+bs*3] += a_3 * b_0;
6374
6375 A += bs;
6376 B += 1;
6377 // k += 1;
6378
6379 // k = 3
6380
6381 a_0 = A[0];
6382 a_1 = A[1];
6383 a_2 = A[2];
6384 a_3 = A[3];
6385
6386 CC[0+bs*3] += a_0;
6387 CC[1+bs*3] += a_1;
6388 CC[2+bs*3] += a_2;
6389 CC[3+bs*3] += a_3;
6390
6391 A += bs;
6392 B += 1;
6393 // k += 1;
6394
6395 }
6396 else if(m1==3)
6397 {
6398
6399 // k = 0
6400
6401 a_0 = A[0];
6402 a_1 = A[1];
6403 a_2 = A[2];
6404 a_3 = A[3];
6405
6406 CC[0+bs*0] += a_0;
6407 CC[1+bs*0] += a_1;
6408 CC[2+bs*0] += a_2;
6409 CC[3+bs*0] += a_3;
6410
6411 b_0 = B[0+1*ldb];
6412 CC[0+bs*1] += a_0 * b_0;
6413 CC[1+bs*1] += a_1 * b_0;
6414 CC[2+bs*1] += a_2 * b_0;
6415 CC[3+bs*1] += a_3 * b_0;
6416
6417 b_0 = B[0+2*ldb];
6418 CC[0+bs*2] += a_0 * b_0;
6419 CC[1+bs*2] += a_1 * b_0;
6420 CC[2+bs*2] += a_2 * b_0;
6421 CC[3+bs*2] += a_3 * b_0;
6422
6423 A += bs;
6424 B += 1;
6425 // k += 1;
6426
6427 // k = 1
6428
6429 a_0 = A[0];
6430 a_1 = A[1];
6431 a_2 = A[2];
6432 a_3 = A[3];
6433
6434 CC[0+bs*1] += a_0;
6435 CC[1+bs*1] += a_1;
6436 CC[2+bs*1] += a_2;
6437 CC[3+bs*1] += a_3;
6438
6439 b_0 = B[0+2*ldb];
6440 CC[0+bs*2] += a_0 * b_0;
6441 CC[1+bs*2] += a_1 * b_0;
6442 CC[2+bs*2] += a_2 * b_0;
6443 CC[3+bs*2] += a_3 * b_0;
6444
6445 A += bs;
6446 B += 1;
6447 // k += 1;
6448
6449 // k = 2
6450
6451 a_0 = A[0];
6452 a_1 = A[1];
6453 a_2 = A[2];
6454 a_3 = A[3];
6455
6456 CC[0+bs*2] += a_0;
6457 CC[1+bs*2] += a_1;
6458 CC[2+bs*2] += a_2;
6459 CC[3+bs*2] += a_3;
6460
6461 A += bs;
6462 B += 1;
6463 // k += 1;
6464
6465 // k = 3
6466
6467 A += bs;
6468 B += 1;
6469 // k += 1;
6470
6471 }
6472 else if(m1==2)
6473 {
6474
6475 // k = 0
6476
6477 a_0 = A[0];
6478 a_1 = A[1];
6479 a_2 = A[2];
6480 a_3 = A[3];
6481
6482 CC[0+bs*0] += a_0;
6483 CC[1+bs*0] += a_1;
6484 CC[2+bs*0] += a_2;
6485 CC[3+bs*0] += a_3;
6486
6487 b_0 = B[0+1*ldb];
6488 CC[0+bs*1] += a_0 * b_0;
6489 CC[1+bs*1] += a_1 * b_0;
6490 CC[2+bs*1] += a_2 * b_0;
6491 CC[3+bs*1] += a_3 * b_0;
6492
6493 A += bs;
6494 B += 1;
6495 // k += 1;
6496
6497 // k = 1
6498
6499 a_0 = A[0];
6500 a_1 = A[1];
6501 a_2 = A[2];
6502 a_3 = A[3];
6503
6504 CC[0+bs*1] += a_0;
6505 CC[1+bs*1] += a_1;
6506 CC[2+bs*1] += a_2;
6507 CC[3+bs*1] += a_3;
6508
6509 A += bs;
6510 B += 1;
6511 // k += 1;
6512
6513 // k = 2
6514
6515 A += bs;
6516 B += 1;
6517 // k += 1;
6518
6519 // k = 3
6520
6521 A += bs;
6522 B += 1;
6523 // k += 1;
6524
6525 }
6526 else if(m1==1)
6527 {
6528
6529 // k = 0
6530
6531 a_0 = A[0];
6532 a_1 = A[1];
6533 a_2 = A[2];
6534 a_3 = A[3];
6535
6536 CC[0+bs*0] += a_0;
6537 CC[1+bs*0] += a_1;
6538 CC[2+bs*0] += a_2;
6539 CC[3+bs*0] += a_3;
6540
6541 A += bs;
6542 B += 1;
6543 // k += 1;
6544
6545 // k = 1
6546
6547 A += bs;
6548 B += 1;
6549 // k += 1;
6550
6551 // k = 2
6552
6553 A += bs;
6554 B += 1;
6555 // k += 1;
6556
6557 // k = 3
6558
6559 A += bs;
6560 B += 1;
6561 // k += 1;
6562
6563 }
6564
6565 store:
6566
6567 // scale
6568 CC[0+bs*0] = alpha[0]*CC[0+bs*0];
6569 CC[1+bs*0] = alpha[0]*CC[1+bs*0];
6570 CC[2+bs*0] = alpha[0]*CC[2+bs*0];
6571 CC[3+bs*0] = alpha[0]*CC[3+bs*0];
6572
6573 CC[0+bs*1] = alpha[0]*CC[0+bs*1];
6574 CC[1+bs*1] = alpha[0]*CC[1+bs*1];
6575 CC[2+bs*1] = alpha[0]*CC[2+bs*1];
6576 CC[3+bs*1] = alpha[0]*CC[3+bs*1];
6577
6578 CC[0+bs*2] = alpha[0]*CC[0+bs*2];
6579 CC[1+bs*2] = alpha[0]*CC[1+bs*2];
6580 CC[2+bs*2] = alpha[0]*CC[2+bs*2];
6581 CC[3+bs*2] = alpha[0]*CC[3+bs*2];
6582
6583 CC[0+bs*3] = alpha[0]*CC[0+bs*3];
6584 CC[1+bs*3] = alpha[0]*CC[1+bs*3];
6585 CC[2+bs*3] = alpha[0]*CC[2+bs*3];
6586 CC[3+bs*3] = alpha[0]*CC[3+bs*3];
6587
6588 // transpose
6589 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
6590 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
6591 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
6592
6593 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
6594 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
6595
6596 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
6597
6598 // store
6599 if(m1>=4)
6600 {
6601 D[0+ldd*0] = CC[0+bs*0];
6602 D[1+ldd*0] = CC[1+bs*0];
6603 D[2+ldd*0] = CC[2+bs*0];
6604 D[3+ldd*0] = CC[3+bs*0];
6605
6606 if(n1==1)
6607 return;
6608
6609 D[0+ldd*1] = CC[0+bs*1];
6610 D[1+ldd*1] = CC[1+bs*1];
6611 D[2+ldd*1] = CC[2+bs*1];
6612 D[3+ldd*1] = CC[3+bs*1];
6613
6614 if(n1==2)
6615 return;
6616
6617 D[0+ldd*2] = CC[0+bs*2];
6618 D[1+ldd*2] = CC[1+bs*2];
6619 D[2+ldd*2] = CC[2+bs*2];
6620 D[3+ldd*2] = CC[3+bs*2];
6621
6622 if(n1==3)
6623 return;
6624
6625 D[0+ldd*3] = CC[0+bs*3];
6626 D[1+ldd*3] = CC[1+bs*3];
6627 D[2+ldd*3] = CC[2+bs*3];
6628 D[3+ldd*3] = CC[3+bs*3];
6629 }
6630 else if(m1>=3)
6631 {
6632 D[0+ldd*0] = CC[0+bs*0];
6633 D[1+ldd*0] = CC[1+bs*0];
6634 D[2+ldd*0] = CC[2+bs*0];
6635
6636 if(n1==1)
6637 return;
6638
6639 D[0+ldd*1] = CC[0+bs*1];
6640 D[1+ldd*1] = CC[1+bs*1];
6641 D[2+ldd*1] = CC[2+bs*1];
6642
6643 if(n1==2)
6644 return;
6645
6646 D[0+ldd*2] = CC[0+bs*2];
6647 D[1+ldd*2] = CC[1+bs*2];
6648 D[2+ldd*2] = CC[2+bs*2];
6649
6650 if(n1==3)
6651 return;
6652
6653 D[0+ldd*3] = CC[0+bs*3];
6654 D[1+ldd*3] = CC[1+bs*3];
6655 D[2+ldd*3] = CC[2+bs*3];
6656 }
6657 else if(m1>=2)
6658 {
6659 D[0+ldd*0] = CC[0+bs*0];
6660 D[1+ldd*0] = CC[1+bs*0];
6661
6662 if(n1==1)
6663 return;
6664
6665 D[0+ldd*1] = CC[0+bs*1];
6666 D[1+ldd*1] = CC[1+bs*1];
6667
6668 if(n1==2)
6669 return;
6670
6671 D[0+ldd*2] = CC[0+bs*2];
6672 D[1+ldd*2] = CC[1+bs*2];
6673
6674 if(n1==3)
6675 return;
6676
6677 D[0+ldd*3] = CC[0+bs*3];
6678 D[1+ldd*3] = CC[1+bs*3];
6679 }
6680 else //if(m1>=1)
6681 {
6682 D[0+ldd*0] = CC[0+bs*0];
6683
6684 if(n1==1)
6685 return;
6686
6687 D[0+ldd*1] = CC[0+bs*1];
6688
6689 if(n1==2)
6690 return;
6691
6692 D[0+ldd*2] = CC[0+bs*2];
6693
6694 if(n1==3)
6695 return;
6696
6697 D[0+ldd*3] = CC[0+bs*3];
6698 }
6699
6700 return;
6701
6702 }
6703 //#endif
6704
6705
6706
6707 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_4x4_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd)6708 void kernel_strmm_nt_rl_4x4_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd)
6709 {
6710
6711 const int bs = 4;
6712
6713 float
6714 a_0, a_1, a_2, a_3,
6715 b_0, b_1, b_2, b_3;
6716
6717 #if defined(TARGET_GENERIC)
6718 float CC[16] = {0};
6719 #else
6720 ALIGNED( float CC[16], 64 ) = {0};
6721 #endif
6722
6723 float alpha1 = 1.0;
6724 float beta1 = 0.0;
6725
6726 // assume always kmax>=4 !!!
6727
6728 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
6729
6730 A += kmax*bs;
6731 B += kmax*bs;
6732
6733 // k = 0
6734
6735 a_0 = A[0];
6736 a_1 = A[1];
6737 a_2 = A[2];
6738 a_3 = A[3];
6739
6740 b_0 = B[0+0*bs];
6741 CC[0+bs*0] += a_0 * b_0;
6742 CC[1+bs*0] += a_1 * b_0;
6743 CC[2+bs*0] += a_2 * b_0;
6744 CC[3+bs*0] += a_3 * b_0;
6745
6746 b_0 = B[1+0*bs];
6747 CC[0+bs*1] += a_0 * b_0;
6748 CC[1+bs*1] += a_1 * b_0;
6749 CC[2+bs*1] += a_2 * b_0;
6750 CC[3+bs*1] += a_3 * b_0;
6751
6752 b_0 = B[2+0*bs];
6753 CC[0+bs*2] += a_0 * b_0;
6754 CC[1+bs*2] += a_1 * b_0;
6755 CC[2+bs*2] += a_2 * b_0;
6756 CC[3+bs*2] += a_3 * b_0;
6757
6758 b_0 = B[3+0*bs];
6759 CC[0+bs*3] += a_0 * b_0;
6760 CC[1+bs*3] += a_1 * b_0;
6761 CC[2+bs*3] += a_2 * b_0;
6762 CC[3+bs*3] += a_3 * b_0;
6763
6764 A += bs;
6765 B += bs;
6766 // k += 1;
6767
6768 // k = 1
6769
6770 a_0 = A[0];
6771 a_1 = A[1];
6772 a_2 = A[2];
6773 a_3 = A[3];
6774
6775 b_0 = B[1+0*bs];
6776 CC[0+bs*1] += a_0 * b_0;
6777 CC[1+bs*1] += a_1 * b_0;
6778 CC[2+bs*1] += a_2 * b_0;
6779 CC[3+bs*1] += a_3 * b_0;
6780
6781 b_0 = B[2+0*bs];
6782 CC[0+bs*2] += a_0 * b_0;
6783 CC[1+bs*2] += a_1 * b_0;
6784 CC[2+bs*2] += a_2 * b_0;
6785 CC[3+bs*2] += a_3 * b_0;
6786
6787 b_0 = B[3+0*bs];
6788 CC[0+bs*3] += a_0 * b_0;
6789 CC[1+bs*3] += a_1 * b_0;
6790 CC[2+bs*3] += a_2 * b_0;
6791 CC[3+bs*3] += a_3 * b_0;
6792
6793 A += bs;
6794 B += bs;
6795 // k += 1;
6796
6797 // k = 2
6798
6799 a_0 = A[0];
6800 a_1 = A[1];
6801 a_2 = A[2];
6802 a_3 = A[3];
6803
6804 b_0 = B[2+0*bs];
6805 CC[0+bs*2] += a_0 * b_0;
6806 CC[1+bs*2] += a_1 * b_0;
6807 CC[2+bs*2] += a_2 * b_0;
6808 CC[3+bs*2] += a_3 * b_0;
6809
6810 b_0 = B[3+0*bs];
6811 CC[0+bs*3] += a_0 * b_0;
6812 CC[1+bs*3] += a_1 * b_0;
6813 CC[2+bs*3] += a_2 * b_0;
6814 CC[3+bs*3] += a_3 * b_0;
6815
6816 A += bs;
6817 B += bs;
6818 // k += 1;
6819
6820 // k = 3
6821
6822 a_0 = A[0];
6823 a_1 = A[1];
6824 a_2 = A[2];
6825 a_3 = A[3];
6826
6827 b_0 = B[3+0*bs];
6828 CC[0+bs*3] += a_0 * b_0;
6829 CC[1+bs*3] += a_1 * b_0;
6830 CC[2+bs*3] += a_2 * b_0;
6831 CC[3+bs*3] += a_3 * b_0;
6832
6833 A += bs;
6834 B += bs;
6835 // k += 1;
6836
6837 store:
6838
6839 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
6840 D[1+ldd*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
6841 D[2+ldd*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
6842 D[3+ldd*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
6843
6844 D[0+ldd*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
6845 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
6846 D[2+ldd*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
6847 D[3+ldd*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
6848
6849 D[0+ldd*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
6850 D[1+ldd*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
6851 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
6852 D[3+ldd*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
6853
6854 D[0+ldd*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
6855 D[1+ldd*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
6856 D[2+ldd*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
6857 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
6858
6859 return;
6860
6861 }
6862 //#endif
6863
6864
6865
6866 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_4x4_vs_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)6867 void kernel_strmm_nt_rl_4x4_vs_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
6868 {
6869
6870 const int bs = 4;
6871
6872 float
6873 a_0, a_1, a_2, a_3,
6874 b_0, b_1, b_2, b_3;
6875
6876 #if defined(TARGET_GENERIC)
6877 float CC[16] = {0};
6878 #else
6879 ALIGNED( float CC[16], 64 ) = {0};
6880 #endif
6881
6882 float alpha1 = 1.0;
6883 float beta1 = 0.0;
6884
6885 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
6886
6887 A += kmax*bs;
6888 B += kmax*bs;
6889
6890 if(n1>=4)
6891 {
6892
6893 // k = 0
6894
6895 a_0 = A[0];
6896 a_1 = A[1];
6897 a_2 = A[2];
6898 a_3 = A[3];
6899
6900 b_0 = B[0+0*bs];
6901 CC[0+bs*0] += a_0 * b_0;
6902 CC[1+bs*0] += a_1 * b_0;
6903 CC[2+bs*0] += a_2 * b_0;
6904 CC[3+bs*0] += a_3 * b_0;
6905
6906 b_0 = B[1+0*bs];
6907 CC[0+bs*1] += a_0 * b_0;
6908 CC[1+bs*1] += a_1 * b_0;
6909 CC[2+bs*1] += a_2 * b_0;
6910 CC[3+bs*1] += a_3 * b_0;
6911
6912 b_0 = B[2+0*bs];
6913 CC[0+bs*2] += a_0 * b_0;
6914 CC[1+bs*2] += a_1 * b_0;
6915 CC[2+bs*2] += a_2 * b_0;
6916 CC[3+bs*2] += a_3 * b_0;
6917
6918 b_0 = B[3+0*bs];
6919 CC[0+bs*3] += a_0 * b_0;
6920 CC[1+bs*3] += a_1 * b_0;
6921 CC[2+bs*3] += a_2 * b_0;
6922 CC[3+bs*3] += a_3 * b_0;
6923
6924 A += bs;
6925 B += bs;
6926 // k += 1;
6927
6928 // k = 1
6929
6930 a_0 = A[0];
6931 a_1 = A[1];
6932 a_2 = A[2];
6933 a_3 = A[3];
6934
6935 b_0 = B[1+0*bs];
6936 CC[0+bs*1] += a_0 * b_0;
6937 CC[1+bs*1] += a_1 * b_0;
6938 CC[2+bs*1] += a_2 * b_0;
6939 CC[3+bs*1] += a_3 * b_0;
6940
6941 b_0 = B[2+0*bs];
6942 CC[0+bs*2] += a_0 * b_0;
6943 CC[1+bs*2] += a_1 * b_0;
6944 CC[2+bs*2] += a_2 * b_0;
6945 CC[3+bs*2] += a_3 * b_0;
6946
6947 b_0 = B[3+0*bs];
6948 CC[0+bs*3] += a_0 * b_0;
6949 CC[1+bs*3] += a_1 * b_0;
6950 CC[2+bs*3] += a_2 * b_0;
6951 CC[3+bs*3] += a_3 * b_0;
6952
6953 A += bs;
6954 B += bs;
6955 // k += 1;
6956
6957 // k = 2
6958
6959 a_0 = A[0];
6960 a_1 = A[1];
6961 a_2 = A[2];
6962 a_3 = A[3];
6963
6964 b_0 = B[2+0*bs];
6965 CC[0+bs*2] += a_0 * b_0;
6966 CC[1+bs*2] += a_1 * b_0;
6967 CC[2+bs*2] += a_2 * b_0;
6968 CC[3+bs*2] += a_3 * b_0;
6969
6970 b_0 = B[3+0*bs];
6971 CC[0+bs*3] += a_0 * b_0;
6972 CC[1+bs*3] += a_1 * b_0;
6973 CC[2+bs*3] += a_2 * b_0;
6974 CC[3+bs*3] += a_3 * b_0;
6975
6976 A += bs;
6977 B += bs;
6978 // k += 1;
6979
6980 // k = 3
6981
6982 a_0 = A[0];
6983 a_1 = A[1];
6984 a_2 = A[2];
6985 a_3 = A[3];
6986
6987 b_0 = B[3+0*bs];
6988 CC[0+bs*3] += a_0 * b_0;
6989 CC[1+bs*3] += a_1 * b_0;
6990 CC[2+bs*3] += a_2 * b_0;
6991 CC[3+bs*3] += a_3 * b_0;
6992
6993 A += bs;
6994 B += bs;
6995 // k += 1;
6996
6997 }
6998 else if(n1==3)
6999 {
7000
7001 // k = 0
7002
7003 a_0 = A[0];
7004 a_1 = A[1];
7005 a_2 = A[2];
7006 a_3 = A[3];
7007
7008 b_0 = B[0+0*bs];
7009 CC[0+bs*0] += a_0 * b_0;
7010 CC[1+bs*0] += a_1 * b_0;
7011 CC[2+bs*0] += a_2 * b_0;
7012 CC[3+bs*0] += a_3 * b_0;
7013
7014 b_0 = B[1+0*bs];
7015 CC[0+bs*1] += a_0 * b_0;
7016 CC[1+bs*1] += a_1 * b_0;
7017 CC[2+bs*1] += a_2 * b_0;
7018 CC[3+bs*1] += a_3 * b_0;
7019
7020 b_0 = B[2+0*bs];
7021 CC[0+bs*2] += a_0 * b_0;
7022 CC[1+bs*2] += a_1 * b_0;
7023 CC[2+bs*2] += a_2 * b_0;
7024 CC[3+bs*2] += a_3 * b_0;
7025
7026 A += bs;
7027 B += bs;
7028 // k += 1;
7029
7030 // k = 1
7031
7032 a_0 = A[0];
7033 a_1 = A[1];
7034 a_2 = A[2];
7035 a_3 = A[3];
7036
7037 b_0 = B[1+0*bs];
7038 CC[0+bs*1] += a_0 * b_0;
7039 CC[1+bs*1] += a_1 * b_0;
7040 CC[2+bs*1] += a_2 * b_0;
7041 CC[3+bs*1] += a_3 * b_0;
7042
7043 b_0 = B[2+0*bs];
7044 CC[0+bs*2] += a_0 * b_0;
7045 CC[1+bs*2] += a_1 * b_0;
7046 CC[2+bs*2] += a_2 * b_0;
7047 CC[3+bs*2] += a_3 * b_0;
7048
7049 A += bs;
7050 B += bs;
7051 // k += 1;
7052
7053 // k = 2
7054
7055 a_0 = A[0];
7056 a_1 = A[1];
7057 a_2 = A[2];
7058 a_3 = A[3];
7059
7060 b_0 = B[2+0*bs];
7061 CC[0+bs*2] += a_0 * b_0;
7062 CC[1+bs*2] += a_1 * b_0;
7063 CC[2+bs*2] += a_2 * b_0;
7064 CC[3+bs*2] += a_3 * b_0;
7065
7066 A += bs;
7067 B += bs;
7068 // k += 1;
7069
7070 // k = 3
7071
7072 A += bs;
7073 B += bs;
7074 // k += 1;
7075
7076 }
7077 else if(n1==2)
7078 {
7079
7080 // k = 0
7081
7082 a_0 = A[0];
7083 a_1 = A[1];
7084 a_2 = A[2];
7085 a_3 = A[3];
7086
7087 b_0 = B[0+0*bs];
7088 CC[0+bs*0] += a_0 * b_0;
7089 CC[1+bs*0] += a_1 * b_0;
7090 CC[2+bs*0] += a_2 * b_0;
7091 CC[3+bs*0] += a_3 * b_0;
7092
7093 b_0 = B[1+0*bs];
7094 CC[0+bs*1] += a_0 * b_0;
7095 CC[1+bs*1] += a_1 * b_0;
7096 CC[2+bs*1] += a_2 * b_0;
7097 CC[3+bs*1] += a_3 * b_0;
7098
7099 A += bs;
7100 B += bs;
7101 // k += 1;
7102
7103 // k = 1
7104
7105 a_0 = A[0];
7106 a_1 = A[1];
7107 a_2 = A[2];
7108 a_3 = A[3];
7109
7110 b_0 = B[1+0*bs];
7111 CC[0+bs*1] += a_0 * b_0;
7112 CC[1+bs*1] += a_1 * b_0;
7113 CC[2+bs*1] += a_2 * b_0;
7114 CC[3+bs*1] += a_3 * b_0;
7115
7116 A += bs;
7117 B += bs;
7118 // k += 1;
7119
7120 // k = 2
7121
7122 A += bs;
7123 B += bs;
7124 // k += 1;
7125
7126 // k = 3
7127
7128 A += bs;
7129 B += bs;
7130 // k += 1;
7131
7132 }
7133 else if(n1==1)
7134 {
7135
7136 // k = 0
7137
7138 a_0 = A[0];
7139 a_1 = A[1];
7140 a_2 = A[2];
7141 a_3 = A[3];
7142
7143 b_0 = B[0+0*bs];
7144 CC[0+bs*0] += a_0 * b_0;
7145 CC[1+bs*0] += a_1 * b_0;
7146 CC[2+bs*0] += a_2 * b_0;
7147 CC[3+bs*0] += a_3 * b_0;
7148
7149 A += bs;
7150 B += bs;
7151 // k += 1;
7152
7153 // k = 1
7154
7155 A += bs;
7156 B += bs;
7157 // k += 1;
7158
7159 // k = 2
7160
7161 A += bs;
7162 B += bs;
7163 // k += 1;
7164
7165 // k = 3
7166
7167 A += bs;
7168 B += bs;
7169 // k += 1;
7170
7171 }
7172
7173 store:
7174
7175 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
7176 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
7177 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
7178 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
7179
7180 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
7181 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
7182 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
7183 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
7184
7185 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
7186 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
7187 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
7188 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
7189
7190 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
7191 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
7192 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
7193 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
7194
7195 if(m1>=4)
7196 {
7197 D[0+ldd*0] = CC[0+bs*0];
7198 D[1+ldd*0] = CC[1+bs*0];
7199 D[2+ldd*0] = CC[2+bs*0];
7200 D[3+ldd*0] = CC[3+bs*0];
7201
7202 if(n1==1)
7203 return;
7204
7205 D[0+ldd*1] = CC[0+bs*1];
7206 D[1+ldd*1] = CC[1+bs*1];
7207 D[2+ldd*1] = CC[2+bs*1];
7208 D[3+ldd*1] = CC[3+bs*1];
7209
7210 if(n1==2)
7211 return;
7212
7213 D[0+ldd*2] = CC[0+bs*2];
7214 D[1+ldd*2] = CC[1+bs*2];
7215 D[2+ldd*2] = CC[2+bs*2];
7216 D[3+ldd*2] = CC[3+bs*2];
7217
7218 if(n1==3)
7219 return;
7220
7221 D[0+ldd*3] = CC[0+bs*3];
7222 D[1+ldd*3] = CC[1+bs*3];
7223 D[2+ldd*3] = CC[2+bs*3];
7224 D[3+ldd*3] = CC[3+bs*3];
7225 }
7226 else if(m1>=3)
7227 {
7228 D[0+ldd*0] = CC[0+bs*0];
7229 D[1+ldd*0] = CC[1+bs*0];
7230 D[2+ldd*0] = CC[2+bs*0];
7231
7232 if(n1==1)
7233 return;
7234
7235 D[0+ldd*1] = CC[0+bs*1];
7236 D[1+ldd*1] = CC[1+bs*1];
7237 D[2+ldd*1] = CC[2+bs*1];
7238
7239 if(n1==2)
7240 return;
7241
7242 D[0+ldd*2] = CC[0+bs*2];
7243 D[1+ldd*2] = CC[1+bs*2];
7244 D[2+ldd*2] = CC[2+bs*2];
7245
7246 if(n1==3)
7247 return;
7248
7249 D[0+ldd*3] = CC[0+bs*3];
7250 D[1+ldd*3] = CC[1+bs*3];
7251 D[2+ldd*3] = CC[2+bs*3];
7252 }
7253 else if(m1>=2)
7254 {
7255 D[0+ldd*0] = CC[0+bs*0];
7256 D[1+ldd*0] = CC[1+bs*0];
7257
7258 if(n1==1)
7259 return;
7260
7261 D[0+ldd*1] = CC[0+bs*1];
7262 D[1+ldd*1] = CC[1+bs*1];
7263
7264 if(n1==2)
7265 return;
7266
7267 D[0+ldd*2] = CC[0+bs*2];
7268 D[1+ldd*2] = CC[1+bs*2];
7269
7270 if(n1==3)
7271 return;
7272
7273 D[0+ldd*3] = CC[0+bs*3];
7274 D[1+ldd*3] = CC[1+bs*3];
7275 }
7276 else //if(m1>=1)
7277 {
7278 D[0+ldd*0] = CC[0+bs*0];
7279
7280 if(n1==1)
7281 return;
7282
7283 D[0+ldd*1] = CC[0+bs*1];
7284
7285 if(n1==2)
7286 return;
7287
7288 D[0+ldd*2] = CC[0+bs*2];
7289
7290 if(n1==3)
7291 return;
7292
7293 D[0+ldd*3] = CC[0+bs*3];
7294 }
7295
7296 return;
7297
7298 }
7299 //#endif
7300
7301
7302
7303 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_4x4_tran_lib444c(int kmax,float * alpha,float * A,float * B,float * beta,float * C,float * D,int ldd)7304 void kernel_strmm_nt_rl_4x4_tran_lib444c(int kmax, float *alpha, float *A, float *B, float *beta, float *C, float *D, int ldd)
7305 {
7306
7307 const int bs = 4;
7308
7309 float
7310 tmp,
7311 a_0, a_1, a_2, a_3,
7312 b_0, b_1, b_2, b_3;
7313
7314 #if defined(TARGET_GENERIC)
7315 float CC[16] = {0};
7316 #else
7317 ALIGNED( float CC[16], 64 ) = {0};
7318 #endif
7319
7320 float alpha1 = 1.0;
7321 float beta1 = 0.0;
7322
7323 // assume always kmax>=4 !!!
7324
7325 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
7326
7327 A += kmax*bs;
7328 B += kmax*bs;
7329
7330 // k = 0
7331
7332 a_0 = A[0];
7333 a_1 = A[1];
7334 a_2 = A[2];
7335 a_3 = A[3];
7336
7337 b_0 = B[0+0*bs];
7338 CC[0+bs*0] += a_0 * b_0;
7339 CC[1+bs*0] += a_1 * b_0;
7340 CC[2+bs*0] += a_2 * b_0;
7341 CC[3+bs*0] += a_3 * b_0;
7342
7343 b_0 = B[1+0*bs];
7344 CC[0+bs*1] += a_0 * b_0;
7345 CC[1+bs*1] += a_1 * b_0;
7346 CC[2+bs*1] += a_2 * b_0;
7347 CC[3+bs*1] += a_3 * b_0;
7348
7349 b_0 = B[2+0*bs];
7350 CC[0+bs*2] += a_0 * b_0;
7351 CC[1+bs*2] += a_1 * b_0;
7352 CC[2+bs*2] += a_2 * b_0;
7353 CC[3+bs*2] += a_3 * b_0;
7354
7355 b_0 = B[3+0*bs];
7356 CC[0+bs*3] += a_0 * b_0;
7357 CC[1+bs*3] += a_1 * b_0;
7358 CC[2+bs*3] += a_2 * b_0;
7359 CC[3+bs*3] += a_3 * b_0;
7360
7361 A += bs;
7362 B += bs;
7363 // k += 1;
7364
7365 // k = 1
7366
7367 a_0 = A[0];
7368 a_1 = A[1];
7369 a_2 = A[2];
7370 a_3 = A[3];
7371
7372 b_0 = B[1+0*bs];
7373 CC[0+bs*1] += a_0 * b_0;
7374 CC[1+bs*1] += a_1 * b_0;
7375 CC[2+bs*1] += a_2 * b_0;
7376 CC[3+bs*1] += a_3 * b_0;
7377
7378 b_0 = B[2+0*bs];
7379 CC[0+bs*2] += a_0 * b_0;
7380 CC[1+bs*2] += a_1 * b_0;
7381 CC[2+bs*2] += a_2 * b_0;
7382 CC[3+bs*2] += a_3 * b_0;
7383
7384 b_0 = B[3+0*bs];
7385 CC[0+bs*3] += a_0 * b_0;
7386 CC[1+bs*3] += a_1 * b_0;
7387 CC[2+bs*3] += a_2 * b_0;
7388 CC[3+bs*3] += a_3 * b_0;
7389
7390 A += bs;
7391 B += bs;
7392 // k += 1;
7393
7394 // k = 2
7395
7396 a_0 = A[0];
7397 a_1 = A[1];
7398 a_2 = A[2];
7399 a_3 = A[3];
7400
7401 b_0 = B[2+0*bs];
7402 CC[0+bs*2] += a_0 * b_0;
7403 CC[1+bs*2] += a_1 * b_0;
7404 CC[2+bs*2] += a_2 * b_0;
7405 CC[3+bs*2] += a_3 * b_0;
7406
7407 b_0 = B[3+0*bs];
7408 CC[0+bs*3] += a_0 * b_0;
7409 CC[1+bs*3] += a_1 * b_0;
7410 CC[2+bs*3] += a_2 * b_0;
7411 CC[3+bs*3] += a_3 * b_0;
7412
7413 A += bs;
7414 B += bs;
7415 // k += 1;
7416
7417 // k = 3
7418
7419 a_0 = A[0];
7420 a_1 = A[1];
7421 a_2 = A[2];
7422 a_3 = A[3];
7423
7424 b_0 = B[3+0*bs];
7425 CC[0+bs*3] += a_0 * b_0;
7426 CC[1+bs*3] += a_1 * b_0;
7427 CC[2+bs*3] += a_2 * b_0;
7428 CC[3+bs*3] += a_3 * b_0;
7429
7430 A += bs;
7431 B += bs;
7432 // k += 1;
7433
7434 store:
7435
7436 // scale & transpose & store
7437 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
7438 D[0+ldd*1] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
7439 D[0+ldd*2] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
7440 D[0+ldd*3] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
7441
7442 D[1+ldd*0] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
7443 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
7444 D[1+ldd*2] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
7445 D[1+ldd*3] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
7446
7447 D[2+ldd*0] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
7448 D[2+ldd*1] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
7449 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
7450 D[2+ldd*3] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
7451
7452 D[3+ldd*0] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
7453 D[3+ldd*1] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
7454 D[3+ldd*2] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
7455 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
7456
7457 return;
7458
7459 }
7460 //#endif
7461
7462
7463
7464 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_4x4_tran_vs_lib444c(int kmax,float * alpha,float * A,float * B,float * beta,float * C,float * D,int ldd,int m1,int n1)7465 void kernel_strmm_nt_rl_4x4_tran_vs_lib444c(int kmax, float *alpha, float *A, float *B, float *beta, float *C, float *D, int ldd, int m1, int n1)
7466 {
7467
7468 const int bs = 4;
7469
7470 float
7471 tmp,
7472 a_0, a_1, a_2, a_3,
7473 b_0, b_1, b_2, b_3;
7474
7475 #if defined(TARGET_GENERIC)
7476 float CC[16] = {0};
7477 #else
7478 ALIGNED( float CC[16], 64 ) = {0};
7479 #endif
7480
7481 float alpha1 = 1.0;
7482 float beta1 = 0.0;
7483
7484 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
7485
7486 A += kmax*bs;
7487 B += kmax*bs;
7488
7489 // XXX m1 and n1 are swapped !!!!!
7490 if(m1>=4)
7491 {
7492
7493 // k = 0
7494
7495 a_0 = A[0];
7496 a_1 = A[1];
7497 a_2 = A[2];
7498 a_3 = A[3];
7499
7500 b_0 = B[0+0*bs];
7501 CC[0+bs*0] += a_0 * b_0;
7502 CC[1+bs*0] += a_1 * b_0;
7503 CC[2+bs*0] += a_2 * b_0;
7504 CC[3+bs*0] += a_3 * b_0;
7505
7506 b_0 = B[1+0*bs];
7507 CC[0+bs*1] += a_0 * b_0;
7508 CC[1+bs*1] += a_1 * b_0;
7509 CC[2+bs*1] += a_2 * b_0;
7510 CC[3+bs*1] += a_3 * b_0;
7511
7512 b_0 = B[2+0*bs];
7513 CC[0+bs*2] += a_0 * b_0;
7514 CC[1+bs*2] += a_1 * b_0;
7515 CC[2+bs*2] += a_2 * b_0;
7516 CC[3+bs*2] += a_3 * b_0;
7517
7518 b_0 = B[3+0*bs];
7519 CC[0+bs*3] += a_0 * b_0;
7520 CC[1+bs*3] += a_1 * b_0;
7521 CC[2+bs*3] += a_2 * b_0;
7522 CC[3+bs*3] += a_3 * b_0;
7523
7524 A += bs;
7525 B += bs;
7526 // k += 1;
7527
7528 // k = 1
7529
7530 a_0 = A[0];
7531 a_1 = A[1];
7532 a_2 = A[2];
7533 a_3 = A[3];
7534
7535 b_0 = B[1+0*bs];
7536 CC[0+bs*1] += a_0 * b_0;
7537 CC[1+bs*1] += a_1 * b_0;
7538 CC[2+bs*1] += a_2 * b_0;
7539 CC[3+bs*1] += a_3 * b_0;
7540
7541 b_0 = B[2+0*bs];
7542 CC[0+bs*2] += a_0 * b_0;
7543 CC[1+bs*2] += a_1 * b_0;
7544 CC[2+bs*2] += a_2 * b_0;
7545 CC[3+bs*2] += a_3 * b_0;
7546
7547 b_0 = B[3+0*bs];
7548 CC[0+bs*3] += a_0 * b_0;
7549 CC[1+bs*3] += a_1 * b_0;
7550 CC[2+bs*3] += a_2 * b_0;
7551 CC[3+bs*3] += a_3 * b_0;
7552
7553 A += bs;
7554 B += bs;
7555 // k += 1;
7556
7557 // k = 2
7558
7559 a_0 = A[0];
7560 a_1 = A[1];
7561 a_2 = A[2];
7562 a_3 = A[3];
7563
7564 b_0 = B[2+0*bs];
7565 CC[0+bs*2] += a_0 * b_0;
7566 CC[1+bs*2] += a_1 * b_0;
7567 CC[2+bs*2] += a_2 * b_0;
7568 CC[3+bs*2] += a_3 * b_0;
7569
7570 b_0 = B[3+0*bs];
7571 CC[0+bs*3] += a_0 * b_0;
7572 CC[1+bs*3] += a_1 * b_0;
7573 CC[2+bs*3] += a_2 * b_0;
7574 CC[3+bs*3] += a_3 * b_0;
7575
7576 A += bs;
7577 B += bs;
7578 // k += 1;
7579
7580 // k = 3
7581
7582 a_0 = A[0];
7583 a_1 = A[1];
7584 a_2 = A[2];
7585 a_3 = A[3];
7586
7587 b_0 = B[3+0*bs];
7588 CC[0+bs*3] += a_0 * b_0;
7589 CC[1+bs*3] += a_1 * b_0;
7590 CC[2+bs*3] += a_2 * b_0;
7591 CC[3+bs*3] += a_3 * b_0;
7592
7593 A += bs;
7594 B += bs;
7595 // k += 1;
7596
7597 }
7598 else if(m1==3)
7599 {
7600
7601 // k = 0
7602
7603 a_0 = A[0];
7604 a_1 = A[1];
7605 a_2 = A[2];
7606 a_3 = A[3];
7607
7608 b_0 = B[0+0*bs];
7609 CC[0+bs*0] += a_0 * b_0;
7610 CC[1+bs*0] += a_1 * b_0;
7611 CC[2+bs*0] += a_2 * b_0;
7612 CC[3+bs*0] += a_3 * b_0;
7613
7614 b_0 = B[1+0*bs];
7615 CC[0+bs*1] += a_0 * b_0;
7616 CC[1+bs*1] += a_1 * b_0;
7617 CC[2+bs*1] += a_2 * b_0;
7618 CC[3+bs*1] += a_3 * b_0;
7619
7620 b_0 = B[2+0*bs];
7621 CC[0+bs*2] += a_0 * b_0;
7622 CC[1+bs*2] += a_1 * b_0;
7623 CC[2+bs*2] += a_2 * b_0;
7624 CC[3+bs*2] += a_3 * b_0;
7625
7626 A += bs;
7627 B += bs;
7628 // k += 1;
7629
7630 // k = 1
7631
7632 a_0 = A[0];
7633 a_1 = A[1];
7634 a_2 = A[2];
7635 a_3 = A[3];
7636
7637 b_0 = B[1+0*bs];
7638 CC[0+bs*1] += a_0 * b_0;
7639 CC[1+bs*1] += a_1 * b_0;
7640 CC[2+bs*1] += a_2 * b_0;
7641 CC[3+bs*1] += a_3 * b_0;
7642
7643 b_0 = B[2+0*bs];
7644 CC[0+bs*2] += a_0 * b_0;
7645 CC[1+bs*2] += a_1 * b_0;
7646 CC[2+bs*2] += a_2 * b_0;
7647 CC[3+bs*2] += a_3 * b_0;
7648
7649 A += bs;
7650 B += bs;
7651 // k += 1;
7652
7653 // k = 2
7654
7655 a_0 = A[0];
7656 a_1 = A[1];
7657 a_2 = A[2];
7658 a_3 = A[3];
7659
7660 b_0 = B[2+0*bs];
7661 CC[0+bs*2] += a_0 * b_0;
7662 CC[1+bs*2] += a_1 * b_0;
7663 CC[2+bs*2] += a_2 * b_0;
7664 CC[3+bs*2] += a_3 * b_0;
7665
7666 A += bs;
7667 B += bs;
7668 // k += 1;
7669
7670 // k = 3
7671
7672 A += bs;
7673 B += bs;
7674 // k += 1;
7675
7676 }
7677 else if(m1==2)
7678 {
7679
7680 // k = 0
7681
7682 a_0 = A[0];
7683 a_1 = A[1];
7684 a_2 = A[2];
7685 a_3 = A[3];
7686
7687 b_0 = B[0+0*bs];
7688 CC[0+bs*0] += a_0 * b_0;
7689 CC[1+bs*0] += a_1 * b_0;
7690 CC[2+bs*0] += a_2 * b_0;
7691 CC[3+bs*0] += a_3 * b_0;
7692
7693 b_0 = B[1+0*bs];
7694 CC[0+bs*1] += a_0 * b_0;
7695 CC[1+bs*1] += a_1 * b_0;
7696 CC[2+bs*1] += a_2 * b_0;
7697 CC[3+bs*1] += a_3 * b_0;
7698
7699 A += bs;
7700 B += bs;
7701 // k += 1;
7702
7703 // k = 1
7704
7705 a_0 = A[0];
7706 a_1 = A[1];
7707 a_2 = A[2];
7708 a_3 = A[3];
7709
7710 b_0 = B[1+0*bs];
7711 CC[0+bs*1] += a_0 * b_0;
7712 CC[1+bs*1] += a_1 * b_0;
7713 CC[2+bs*1] += a_2 * b_0;
7714 CC[3+bs*1] += a_3 * b_0;
7715
7716 A += bs;
7717 B += bs;
7718 // k += 1;
7719
7720 // k = 2
7721
7722 A += bs;
7723 B += bs;
7724 // k += 1;
7725
7726 // k = 3
7727
7728 A += bs;
7729 B += bs;
7730 // k += 1;
7731
7732 }
7733 else if(m1==1)
7734 {
7735
7736 // k = 0
7737
7738 a_0 = A[0];
7739 a_1 = A[1];
7740 a_2 = A[2];
7741 a_3 = A[3];
7742
7743 b_0 = B[0+0*bs];
7744 CC[0+bs*0] += a_0 * b_0;
7745 CC[1+bs*0] += a_1 * b_0;
7746 CC[2+bs*0] += a_2 * b_0;
7747 CC[3+bs*0] += a_3 * b_0;
7748
7749 A += bs;
7750 B += bs;
7751 // k += 1;
7752
7753 // k = 1
7754
7755 A += bs;
7756 B += bs;
7757 // k += 1;
7758
7759 // k = 2
7760
7761 A += bs;
7762 B += bs;
7763 // k += 1;
7764
7765 // k = 3
7766
7767 A += bs;
7768 B += bs;
7769 // k += 1;
7770
7771 }
7772
7773 store:
7774
7775 // scale
7776 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
7777 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
7778 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
7779 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
7780
7781 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
7782 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
7783 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
7784 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
7785
7786 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
7787 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
7788 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
7789 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
7790
7791 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
7792 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
7793 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
7794 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
7795
7796 // transpose
7797 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
7798 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
7799 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
7800
7801 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
7802 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
7803
7804 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
7805
7806 // store
7807 if(m1>=4)
7808 {
7809 D[0+ldd*0] = CC[0+bs*0];
7810 D[1+ldd*0] = CC[1+bs*0];
7811 D[2+ldd*0] = CC[2+bs*0];
7812 D[3+ldd*0] = CC[3+bs*0];
7813
7814 if(n1==1)
7815 return;
7816
7817 D[0+ldd*1] = CC[0+bs*1];
7818 D[1+ldd*1] = CC[1+bs*1];
7819 D[2+ldd*1] = CC[2+bs*1];
7820 D[3+ldd*1] = CC[3+bs*1];
7821
7822 if(n1==2)
7823 return;
7824
7825 D[0+ldd*2] = CC[0+bs*2];
7826 D[1+ldd*2] = CC[1+bs*2];
7827 D[2+ldd*2] = CC[2+bs*2];
7828 D[3+ldd*2] = CC[3+bs*2];
7829
7830 if(n1==3)
7831 return;
7832
7833 D[0+ldd*3] = CC[0+bs*3];
7834 D[1+ldd*3] = CC[1+bs*3];
7835 D[2+ldd*3] = CC[2+bs*3];
7836 D[3+ldd*3] = CC[3+bs*3];
7837 }
7838 else if(m1>=3)
7839 {
7840 D[0+ldd*0] = CC[0+bs*0];
7841 D[1+ldd*0] = CC[1+bs*0];
7842 D[2+ldd*0] = CC[2+bs*0];
7843
7844 if(n1==1)
7845 return;
7846
7847 D[0+ldd*1] = CC[0+bs*1];
7848 D[1+ldd*1] = CC[1+bs*1];
7849 D[2+ldd*1] = CC[2+bs*1];
7850
7851 if(n1==2)
7852 return;
7853
7854 D[0+ldd*2] = CC[0+bs*2];
7855 D[1+ldd*2] = CC[1+bs*2];
7856 D[2+ldd*2] = CC[2+bs*2];
7857
7858 if(n1==3)
7859 return;
7860
7861 D[0+ldd*3] = CC[0+bs*3];
7862 D[1+ldd*3] = CC[1+bs*3];
7863 D[2+ldd*3] = CC[2+bs*3];
7864 }
7865 else if(m1>=2)
7866 {
7867 D[0+ldd*0] = CC[0+bs*0];
7868 D[1+ldd*0] = CC[1+bs*0];
7869
7870 if(n1==1)
7871 return;
7872
7873 D[0+ldd*1] = CC[0+bs*1];
7874 D[1+ldd*1] = CC[1+bs*1];
7875
7876 if(n1==2)
7877 return;
7878
7879 D[0+ldd*2] = CC[0+bs*2];
7880 D[1+ldd*2] = CC[1+bs*2];
7881
7882 if(n1==3)
7883 return;
7884
7885 D[0+ldd*3] = CC[0+bs*3];
7886 D[1+ldd*3] = CC[1+bs*3];
7887 }
7888 else //if(m1>=1)
7889 {
7890 D[0+ldd*0] = CC[0+bs*0];
7891
7892 if(n1==1)
7893 return;
7894
7895 D[0+ldd*1] = CC[0+bs*1];
7896
7897 if(n1==2)
7898 return;
7899
7900 D[0+ldd*2] = CC[0+bs*2];
7901
7902 if(n1==3)
7903 return;
7904
7905 D[0+ldd*3] = CC[0+bs*3];
7906 }
7907
7908 return;
7909
7910 }
7911 //#endif
7912
7913
7914
7915 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_one_4x4_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd)7916 void kernel_strmm_nt_rl_one_4x4_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd)
7917 {
7918
7919 const int bs = 4;
7920
7921 float
7922 a_0, a_1, a_2, a_3,
7923 b_0, b_1, b_2, b_3;
7924
7925 #if defined(TARGET_GENERIC)
7926 float CC[16] = {0};
7927 #else
7928 ALIGNED( float CC[16], 64 ) = {0};
7929 #endif
7930
7931 float alpha1 = 1.0;
7932 float beta1 = 0.0;
7933
7934 // assume always kmax>=4 !!!
7935
7936 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
7937
7938 A += kmax*bs;
7939 B += kmax*bs;
7940
7941 // k = 0
7942
7943 a_0 = A[0];
7944 a_1 = A[1];
7945 a_2 = A[2];
7946 a_3 = A[3];
7947
7948 CC[0+bs*0] += a_0;
7949 CC[1+bs*0] += a_1;
7950 CC[2+bs*0] += a_2;
7951 CC[3+bs*0] += a_3;
7952
7953 b_0 = B[1+0*bs];
7954 CC[0+bs*1] += a_0 * b_0;
7955 CC[1+bs*1] += a_1 * b_0;
7956 CC[2+bs*1] += a_2 * b_0;
7957 CC[3+bs*1] += a_3 * b_0;
7958
7959 b_0 = B[2+0*bs];
7960 CC[0+bs*2] += a_0 * b_0;
7961 CC[1+bs*2] += a_1 * b_0;
7962 CC[2+bs*2] += a_2 * b_0;
7963 CC[3+bs*2] += a_3 * b_0;
7964
7965 b_0 = B[3+0*bs];
7966 CC[0+bs*3] += a_0 * b_0;
7967 CC[1+bs*3] += a_1 * b_0;
7968 CC[2+bs*3] += a_2 * b_0;
7969 CC[3+bs*3] += a_3 * b_0;
7970
7971 A += bs;
7972 B += bs;
7973 // k += 1;
7974
7975 // k = 1
7976
7977 a_0 = A[0];
7978 a_1 = A[1];
7979 a_2 = A[2];
7980 a_3 = A[3];
7981
7982 CC[0+bs*1] += a_0;
7983 CC[1+bs*1] += a_1;
7984 CC[2+bs*1] += a_2;
7985 CC[3+bs*1] += a_3;
7986
7987 b_0 = B[2+0*bs];
7988 CC[0+bs*2] += a_0 * b_0;
7989 CC[1+bs*2] += a_1 * b_0;
7990 CC[2+bs*2] += a_2 * b_0;
7991 CC[3+bs*2] += a_3 * b_0;
7992
7993 b_0 = B[3+0*bs];
7994 CC[0+bs*3] += a_0 * b_0;
7995 CC[1+bs*3] += a_1 * b_0;
7996 CC[2+bs*3] += a_2 * b_0;
7997 CC[3+bs*3] += a_3 * b_0;
7998
7999 A += bs;
8000 B += bs;
8001 // k += 1;
8002
8003 // k = 2
8004
8005 a_0 = A[0];
8006 a_1 = A[1];
8007 a_2 = A[2];
8008 a_3 = A[3];
8009
8010 CC[0+bs*2] += a_0;
8011 CC[1+bs*2] += a_1;
8012 CC[2+bs*2] += a_2;
8013 CC[3+bs*2] += a_3;
8014
8015 b_0 = B[3+0*bs];
8016 CC[0+bs*3] += a_0 * b_0;
8017 CC[1+bs*3] += a_1 * b_0;
8018 CC[2+bs*3] += a_2 * b_0;
8019 CC[3+bs*3] += a_3 * b_0;
8020
8021 A += bs;
8022 B += bs;
8023 // k += 1;
8024
8025 // k = 3
8026
8027 a_0 = A[0];
8028 a_1 = A[1];
8029 a_2 = A[2];
8030 a_3 = A[3];
8031
8032 CC[0+bs*3] += a_0;
8033 CC[1+bs*3] += a_1;
8034 CC[2+bs*3] += a_2;
8035 CC[3+bs*3] += a_3;
8036
8037 A += bs;
8038 B += bs;
8039 // k += 1;
8040
8041 store:
8042
8043 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
8044 D[1+ldd*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
8045 D[2+ldd*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
8046 D[3+ldd*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
8047
8048 D[0+ldd*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
8049 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
8050 D[2+ldd*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
8051 D[3+ldd*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
8052
8053 D[0+ldd*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
8054 D[1+ldd*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
8055 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
8056 D[3+ldd*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
8057
8058 D[0+ldd*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
8059 D[1+ldd*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
8060 D[2+ldd*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
8061 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
8062
8063 return;
8064
8065 }
8066 //#endif
8067
8068
8069
8070 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_one_4x4_vs_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)8071 void kernel_strmm_nt_rl_one_4x4_vs_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
8072 {
8073
8074 const int bs = 4;
8075
8076 float
8077 a_0, a_1, a_2, a_3,
8078 b_0, b_1, b_2, b_3;
8079
8080 #if defined(TARGET_GENERIC)
8081 float CC[16] = {0};
8082 #else
8083 ALIGNED( float CC[16], 64 ) = {0};
8084 #endif
8085
8086 float alpha1 = 1.0;
8087 float beta1 = 0.0;
8088
8089 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
8090
8091 A += kmax*bs;
8092 B += kmax*bs;
8093
8094 if(n1>=4)
8095 {
8096
8097 // k = 0
8098
8099 a_0 = A[0];
8100 a_1 = A[1];
8101 a_2 = A[2];
8102 a_3 = A[3];
8103
8104 CC[0+bs*0] += a_0;
8105 CC[1+bs*0] += a_1;
8106 CC[2+bs*0] += a_2;
8107 CC[3+bs*0] += a_3;
8108
8109 b_0 = B[1+0*bs];
8110 CC[0+bs*1] += a_0 * b_0;
8111 CC[1+bs*1] += a_1 * b_0;
8112 CC[2+bs*1] += a_2 * b_0;
8113 CC[3+bs*1] += a_3 * b_0;
8114
8115 b_0 = B[2+0*bs];
8116 CC[0+bs*2] += a_0 * b_0;
8117 CC[1+bs*2] += a_1 * b_0;
8118 CC[2+bs*2] += a_2 * b_0;
8119 CC[3+bs*2] += a_3 * b_0;
8120
8121 b_0 = B[3+0*bs];
8122 CC[0+bs*3] += a_0 * b_0;
8123 CC[1+bs*3] += a_1 * b_0;
8124 CC[2+bs*3] += a_2 * b_0;
8125 CC[3+bs*3] += a_3 * b_0;
8126
8127 A += bs;
8128 B += bs;
8129 // k += 1;
8130
8131 // k = 1
8132
8133 a_0 = A[0];
8134 a_1 = A[1];
8135 a_2 = A[2];
8136 a_3 = A[3];
8137
8138 CC[0+bs*1] += a_0;
8139 CC[1+bs*1] += a_1;
8140 CC[2+bs*1] += a_2;
8141 CC[3+bs*1] += a_3;
8142
8143 b_0 = B[2+0*bs];
8144 CC[0+bs*2] += a_0 * b_0;
8145 CC[1+bs*2] += a_1 * b_0;
8146 CC[2+bs*2] += a_2 * b_0;
8147 CC[3+bs*2] += a_3 * b_0;
8148
8149 b_0 = B[3+0*bs];
8150 CC[0+bs*3] += a_0 * b_0;
8151 CC[1+bs*3] += a_1 * b_0;
8152 CC[2+bs*3] += a_2 * b_0;
8153 CC[3+bs*3] += a_3 * b_0;
8154
8155 A += bs;
8156 B += bs;
8157 // k += 1;
8158
8159 // k = 2
8160
8161 a_0 = A[0];
8162 a_1 = A[1];
8163 a_2 = A[2];
8164 a_3 = A[3];
8165
8166 CC[0+bs*2] += a_0;
8167 CC[1+bs*2] += a_1;
8168 CC[2+bs*2] += a_2;
8169 CC[3+bs*2] += a_3;
8170
8171 b_0 = B[3+0*bs];
8172 CC[0+bs*3] += a_0 * b_0;
8173 CC[1+bs*3] += a_1 * b_0;
8174 CC[2+bs*3] += a_2 * b_0;
8175 CC[3+bs*3] += a_3 * b_0;
8176
8177 A += bs;
8178 B += bs;
8179 // k += 1;
8180
8181 // k = 3
8182
8183 a_0 = A[0];
8184 a_1 = A[1];
8185 a_2 = A[2];
8186 a_3 = A[3];
8187
8188 CC[0+bs*3] += a_0;
8189 CC[1+bs*3] += a_1;
8190 CC[2+bs*3] += a_2;
8191 CC[3+bs*3] += a_3;
8192
8193 A += bs;
8194 B += bs;
8195 // k += 1;
8196
8197 }
8198 else if(n1==3)
8199 {
8200
8201 // k = 0
8202
8203 a_0 = A[0];
8204 a_1 = A[1];
8205 a_2 = A[2];
8206 a_3 = A[3];
8207
8208 CC[0+bs*0] += a_0;
8209 CC[1+bs*0] += a_1;
8210 CC[2+bs*0] += a_2;
8211 CC[3+bs*0] += a_3;
8212
8213 b_0 = B[1+0*bs];
8214 CC[0+bs*1] += a_0 * b_0;
8215 CC[1+bs*1] += a_1 * b_0;
8216 CC[2+bs*1] += a_2 * b_0;
8217 CC[3+bs*1] += a_3 * b_0;
8218
8219 b_0 = B[2+0*bs];
8220 CC[0+bs*2] += a_0 * b_0;
8221 CC[1+bs*2] += a_1 * b_0;
8222 CC[2+bs*2] += a_2 * b_0;
8223 CC[3+bs*2] += a_3 * b_0;
8224
8225 A += bs;
8226 B += bs;
8227 // k += 1;
8228
8229 // k = 1
8230
8231 a_0 = A[0];
8232 a_1 = A[1];
8233 a_2 = A[2];
8234 a_3 = A[3];
8235
8236 CC[0+bs*1] += a_0;
8237 CC[1+bs*1] += a_1;
8238 CC[2+bs*1] += a_2;
8239 CC[3+bs*1] += a_3;
8240
8241 b_0 = B[2+0*bs];
8242 CC[0+bs*2] += a_0 * b_0;
8243 CC[1+bs*2] += a_1 * b_0;
8244 CC[2+bs*2] += a_2 * b_0;
8245 CC[3+bs*2] += a_3 * b_0;
8246
8247 A += bs;
8248 B += bs;
8249 // k += 1;
8250
8251 // k = 2
8252
8253 a_0 = A[0];
8254 a_1 = A[1];
8255 a_2 = A[2];
8256 a_3 = A[3];
8257
8258 CC[0+bs*2] += a_0;
8259 CC[1+bs*2] += a_1;
8260 CC[2+bs*2] += a_2;
8261 CC[3+bs*2] += a_3;
8262
8263 A += bs;
8264 B += bs;
8265 // k += 1;
8266
8267 // k = 3
8268
8269 A += bs;
8270 B += bs;
8271 // k += 1;
8272
8273 }
8274 else if(n1==2)
8275 {
8276
8277 // k = 0
8278
8279 a_0 = A[0];
8280 a_1 = A[1];
8281 a_2 = A[2];
8282 a_3 = A[3];
8283
8284 CC[0+bs*0] += a_0;
8285 CC[1+bs*0] += a_1;
8286 CC[2+bs*0] += a_2;
8287 CC[3+bs*0] += a_3;
8288
8289 b_0 = B[1+0*bs];
8290 CC[0+bs*1] += a_0 * b_0;
8291 CC[1+bs*1] += a_1 * b_0;
8292 CC[2+bs*1] += a_2 * b_0;
8293 CC[3+bs*1] += a_3 * b_0;
8294
8295 A += bs;
8296 B += bs;
8297 // k += 1;
8298
8299 // k = 1
8300
8301 a_0 = A[0];
8302 a_1 = A[1];
8303 a_2 = A[2];
8304 a_3 = A[3];
8305
8306 CC[0+bs*1] += a_0;
8307 CC[1+bs*1] += a_1;
8308 CC[2+bs*1] += a_2;
8309 CC[3+bs*1] += a_3;
8310
8311 A += bs;
8312 B += bs;
8313 // k += 1;
8314
8315 // k = 2
8316
8317 A += bs;
8318 B += bs;
8319 // k += 1;
8320
8321 // k = 3
8322
8323 A += bs;
8324 B += bs;
8325 // k += 1;
8326
8327 }
8328 else if(n1==1)
8329 {
8330
8331 // k = 0
8332
8333 a_0 = A[0];
8334 a_1 = A[1];
8335 a_2 = A[2];
8336 a_3 = A[3];
8337
8338 CC[0+bs*0] += a_0;
8339 CC[1+bs*0] += a_1;
8340 CC[2+bs*0] += a_2;
8341 CC[3+bs*0] += a_3;
8342
8343 A += bs;
8344 B += bs;
8345 // k += 1;
8346
8347 // k = 1
8348
8349 A += bs;
8350 B += bs;
8351 // k += 1;
8352
8353 // k = 2
8354
8355 A += bs;
8356 B += bs;
8357 // k += 1;
8358
8359 // k = 3
8360
8361 A += bs;
8362 B += bs;
8363 // k += 1;
8364
8365 }
8366
8367 store:
8368
8369 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
8370 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
8371 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
8372 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
8373
8374 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
8375 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
8376 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
8377 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
8378
8379 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
8380 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
8381 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
8382 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
8383
8384 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
8385 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
8386 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
8387 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
8388
8389 if(m1>=4)
8390 {
8391 D[0+ldd*0] = CC[0+bs*0];
8392 D[1+ldd*0] = CC[1+bs*0];
8393 D[2+ldd*0] = CC[2+bs*0];
8394 D[3+ldd*0] = CC[3+bs*0];
8395
8396 if(n1==1)
8397 return;
8398
8399 D[0+ldd*1] = CC[0+bs*1];
8400 D[1+ldd*1] = CC[1+bs*1];
8401 D[2+ldd*1] = CC[2+bs*1];
8402 D[3+ldd*1] = CC[3+bs*1];
8403
8404 if(n1==2)
8405 return;
8406
8407 D[0+ldd*2] = CC[0+bs*2];
8408 D[1+ldd*2] = CC[1+bs*2];
8409 D[2+ldd*2] = CC[2+bs*2];
8410 D[3+ldd*2] = CC[3+bs*2];
8411
8412 if(n1==3)
8413 return;
8414
8415 D[0+ldd*3] = CC[0+bs*3];
8416 D[1+ldd*3] = CC[1+bs*3];
8417 D[2+ldd*3] = CC[2+bs*3];
8418 D[3+ldd*3] = CC[3+bs*3];
8419 }
8420 else if(m1>=3)
8421 {
8422 D[0+ldd*0] = CC[0+bs*0];
8423 D[1+ldd*0] = CC[1+bs*0];
8424 D[2+ldd*0] = CC[2+bs*0];
8425
8426 if(n1==1)
8427 return;
8428
8429 D[0+ldd*1] = CC[0+bs*1];
8430 D[1+ldd*1] = CC[1+bs*1];
8431 D[2+ldd*1] = CC[2+bs*1];
8432
8433 if(n1==2)
8434 return;
8435
8436 D[0+ldd*2] = CC[0+bs*2];
8437 D[1+ldd*2] = CC[1+bs*2];
8438 D[2+ldd*2] = CC[2+bs*2];
8439
8440 if(n1==3)
8441 return;
8442
8443 D[0+ldd*3] = CC[0+bs*3];
8444 D[1+ldd*3] = CC[1+bs*3];
8445 D[2+ldd*3] = CC[2+bs*3];
8446 }
8447 else if(m1>=2)
8448 {
8449 D[0+ldd*0] = CC[0+bs*0];
8450 D[1+ldd*0] = CC[1+bs*0];
8451
8452 if(n1==1)
8453 return;
8454
8455 D[0+ldd*1] = CC[0+bs*1];
8456 D[1+ldd*1] = CC[1+bs*1];
8457
8458 if(n1==2)
8459 return;
8460
8461 D[0+ldd*2] = CC[0+bs*2];
8462 D[1+ldd*2] = CC[1+bs*2];
8463
8464 if(n1==3)
8465 return;
8466
8467 D[0+ldd*3] = CC[0+bs*3];
8468 D[1+ldd*3] = CC[1+bs*3];
8469 }
8470 else //if(m1>=1)
8471 {
8472 D[0+ldd*0] = CC[0+bs*0];
8473
8474 if(n1==1)
8475 return;
8476
8477 D[0+ldd*1] = CC[0+bs*1];
8478
8479 if(n1==2)
8480 return;
8481
8482 D[0+ldd*2] = CC[0+bs*2];
8483
8484 if(n1==3)
8485 return;
8486
8487 D[0+ldd*3] = CC[0+bs*3];
8488 }
8489
8490 return;
8491
8492 }
8493 //#endif
8494
8495
8496
8497 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_one_4x4_tran_lib444c(int kmax,float * alpha,float * A,float * B,float * beta,float * C,float * D,int ldd)8498 void kernel_strmm_nt_rl_one_4x4_tran_lib444c(int kmax, float *alpha, float *A, float *B, float *beta, float *C, float *D, int ldd)
8499 {
8500
8501 const int bs = 4;
8502
8503 float
8504 tmp,
8505 a_0, a_1, a_2, a_3,
8506 b_0, b_1, b_2, b_3;
8507
8508 #if defined(TARGET_GENERIC)
8509 float CC[16] = {0};
8510 #else
8511 ALIGNED( float CC[16], 64 ) = {0};
8512 #endif
8513
8514 float alpha1 = 1.0;
8515 float beta1 = 0.0;
8516
8517 // assume always kmax>=4 !!!
8518
8519 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
8520
8521 A += kmax*bs;
8522 B += kmax*bs;
8523
8524 // k = 0
8525
8526 a_0 = A[0];
8527 a_1 = A[1];
8528 a_2 = A[2];
8529 a_3 = A[3];
8530
8531 CC[0+bs*0] += a_0;
8532 CC[1+bs*0] += a_1;
8533 CC[2+bs*0] += a_2;
8534 CC[3+bs*0] += a_3;
8535
8536 b_0 = B[1+0*bs];
8537 CC[0+bs*1] += a_0 * b_0;
8538 CC[1+bs*1] += a_1 * b_0;
8539 CC[2+bs*1] += a_2 * b_0;
8540 CC[3+bs*1] += a_3 * b_0;
8541
8542 b_0 = B[2+0*bs];
8543 CC[0+bs*2] += a_0 * b_0;
8544 CC[1+bs*2] += a_1 * b_0;
8545 CC[2+bs*2] += a_2 * b_0;
8546 CC[3+bs*2] += a_3 * b_0;
8547
8548 b_0 = B[3+0*bs];
8549 CC[0+bs*3] += a_0 * b_0;
8550 CC[1+bs*3] += a_1 * b_0;
8551 CC[2+bs*3] += a_2 * b_0;
8552 CC[3+bs*3] += a_3 * b_0;
8553
8554 A += bs;
8555 B += bs;
8556 // k += 1;
8557
8558 // k = 1
8559
8560 a_0 = A[0];
8561 a_1 = A[1];
8562 a_2 = A[2];
8563 a_3 = A[3];
8564
8565 CC[0+bs*1] += a_0;
8566 CC[1+bs*1] += a_1;
8567 CC[2+bs*1] += a_2;
8568 CC[3+bs*1] += a_3;
8569
8570 b_0 = B[2+0*bs];
8571 CC[0+bs*2] += a_0 * b_0;
8572 CC[1+bs*2] += a_1 * b_0;
8573 CC[2+bs*2] += a_2 * b_0;
8574 CC[3+bs*2] += a_3 * b_0;
8575
8576 b_0 = B[3+0*bs];
8577 CC[0+bs*3] += a_0 * b_0;
8578 CC[1+bs*3] += a_1 * b_0;
8579 CC[2+bs*3] += a_2 * b_0;
8580 CC[3+bs*3] += a_3 * b_0;
8581
8582 A += bs;
8583 B += bs;
8584 // k += 1;
8585
8586 // k = 2
8587
8588 a_0 = A[0];
8589 a_1 = A[1];
8590 a_2 = A[2];
8591 a_3 = A[3];
8592
8593 CC[0+bs*2] += a_0;
8594 CC[1+bs*2] += a_1;
8595 CC[2+bs*2] += a_2;
8596 CC[3+bs*2] += a_3;
8597
8598 b_0 = B[3+0*bs];
8599 CC[0+bs*3] += a_0 * b_0;
8600 CC[1+bs*3] += a_1 * b_0;
8601 CC[2+bs*3] += a_2 * b_0;
8602 CC[3+bs*3] += a_3 * b_0;
8603
8604 A += bs;
8605 B += bs;
8606 // k += 1;
8607
8608 // k = 3
8609
8610 a_0 = A[0];
8611 a_1 = A[1];
8612 a_2 = A[2];
8613 a_3 = A[3];
8614
8615 CC[0+bs*3] += a_0;
8616 CC[1+bs*3] += a_1;
8617 CC[2+bs*3] += a_2;
8618 CC[3+bs*3] += a_3;
8619
8620 A += bs;
8621 B += bs;
8622 // k += 1;
8623
8624 store:
8625
8626 // scale & transpose & store
8627 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
8628 D[0+ldd*1] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
8629 D[0+ldd*2] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
8630 D[0+ldd*3] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
8631
8632 D[1+ldd*0] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
8633 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
8634 D[1+ldd*2] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
8635 D[1+ldd*3] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
8636
8637 D[2+ldd*0] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
8638 D[2+ldd*1] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
8639 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
8640 D[2+ldd*3] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
8641
8642 D[3+ldd*0] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
8643 D[3+ldd*1] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
8644 D[3+ldd*2] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
8645 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
8646
8647 return;
8648
8649 }
8650 //#endif
8651
8652
8653
8654 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_one_4x4_tran_vs_lib444c(int kmax,float * alpha,float * A,float * B,float * beta,float * C,float * D,int ldd,int m1,int n1)8655 void kernel_strmm_nt_rl_one_4x4_tran_vs_lib444c(int kmax, float *alpha, float *A, float *B, float *beta, float *C, float *D, int ldd, int m1, int n1)
8656 {
8657
8658 const int bs = 4;
8659
8660 float
8661 tmp,
8662 a_0, a_1, a_2, a_3,
8663 b_0, b_1, b_2, b_3;
8664
8665 #if defined(TARGET_GENERIC)
8666 float CC[16] = {0};
8667 #else
8668 ALIGNED( float CC[16], 64 ) = {0};
8669 #endif
8670
8671 float alpha1 = 1.0;
8672 float beta1 = 0.0;
8673
8674 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
8675
8676 A += kmax*bs;
8677 B += kmax*bs;
8678
8679 // XXX m1 and n1 are swapped !!!!!
8680 if(m1>=4)
8681 {
8682
8683 // k = 0
8684
8685 a_0 = A[0];
8686 a_1 = A[1];
8687 a_2 = A[2];
8688 a_3 = A[3];
8689
8690 CC[0+bs*0] += a_0;
8691 CC[1+bs*0] += a_1;
8692 CC[2+bs*0] += a_2;
8693 CC[3+bs*0] += a_3;
8694
8695 b_0 = B[1+0*bs];
8696 CC[0+bs*1] += a_0 * b_0;
8697 CC[1+bs*1] += a_1 * b_0;
8698 CC[2+bs*1] += a_2 * b_0;
8699 CC[3+bs*1] += a_3 * b_0;
8700
8701 b_0 = B[2+0*bs];
8702 CC[0+bs*2] += a_0 * b_0;
8703 CC[1+bs*2] += a_1 * b_0;
8704 CC[2+bs*2] += a_2 * b_0;
8705 CC[3+bs*2] += a_3 * b_0;
8706
8707 b_0 = B[3+0*bs];
8708 CC[0+bs*3] += a_0 * b_0;
8709 CC[1+bs*3] += a_1 * b_0;
8710 CC[2+bs*3] += a_2 * b_0;
8711 CC[3+bs*3] += a_3 * b_0;
8712
8713 A += bs;
8714 B += bs;
8715 // k += 1;
8716
8717 // k = 1
8718
8719 a_0 = A[0];
8720 a_1 = A[1];
8721 a_2 = A[2];
8722 a_3 = A[3];
8723
8724 CC[0+bs*1] += a_0;
8725 CC[1+bs*1] += a_1;
8726 CC[2+bs*1] += a_2;
8727 CC[3+bs*1] += a_3;
8728
8729 b_0 = B[2+0*bs];
8730 CC[0+bs*2] += a_0 * b_0;
8731 CC[1+bs*2] += a_1 * b_0;
8732 CC[2+bs*2] += a_2 * b_0;
8733 CC[3+bs*2] += a_3 * b_0;
8734
8735 b_0 = B[3+0*bs];
8736 CC[0+bs*3] += a_0 * b_0;
8737 CC[1+bs*3] += a_1 * b_0;
8738 CC[2+bs*3] += a_2 * b_0;
8739 CC[3+bs*3] += a_3 * b_0;
8740
8741 A += bs;
8742 B += bs;
8743 // k += 1;
8744
8745 // k = 2
8746
8747 a_0 = A[0];
8748 a_1 = A[1];
8749 a_2 = A[2];
8750 a_3 = A[3];
8751
8752 CC[0+bs*2] += a_0;
8753 CC[1+bs*2] += a_1;
8754 CC[2+bs*2] += a_2;
8755 CC[3+bs*2] += a_3;
8756
8757 b_0 = B[3+0*bs];
8758 CC[0+bs*3] += a_0 * b_0;
8759 CC[1+bs*3] += a_1 * b_0;
8760 CC[2+bs*3] += a_2 * b_0;
8761 CC[3+bs*3] += a_3 * b_0;
8762
8763 A += bs;
8764 B += bs;
8765 // k += 1;
8766
8767 // k = 3
8768
8769 a_0 = A[0];
8770 a_1 = A[1];
8771 a_2 = A[2];
8772 a_3 = A[3];
8773
8774 CC[0+bs*3] += a_0;
8775 CC[1+bs*3] += a_1;
8776 CC[2+bs*3] += a_2;
8777 CC[3+bs*3] += a_3;
8778
8779 A += bs;
8780 B += bs;
8781 // k += 1;
8782
8783 }
8784 else if(m1==3)
8785 {
8786
8787 // k = 0
8788
8789 a_0 = A[0];
8790 a_1 = A[1];
8791 a_2 = A[2];
8792 a_3 = A[3];
8793
8794 CC[0+bs*0] += a_0;
8795 CC[1+bs*0] += a_1;
8796 CC[2+bs*0] += a_2;
8797 CC[3+bs*0] += a_3;
8798
8799 b_0 = B[1+0*bs];
8800 CC[0+bs*1] += a_0 * b_0;
8801 CC[1+bs*1] += a_1 * b_0;
8802 CC[2+bs*1] += a_2 * b_0;
8803 CC[3+bs*1] += a_3 * b_0;
8804
8805 b_0 = B[2+0*bs];
8806 CC[0+bs*2] += a_0 * b_0;
8807 CC[1+bs*2] += a_1 * b_0;
8808 CC[2+bs*2] += a_2 * b_0;
8809 CC[3+bs*2] += a_3 * b_0;
8810
8811 A += bs;
8812 B += bs;
8813 // k += 1;
8814
8815 // k = 1
8816
8817 a_0 = A[0];
8818 a_1 = A[1];
8819 a_2 = A[2];
8820 a_3 = A[3];
8821
8822 CC[0+bs*1] += a_0;
8823 CC[1+bs*1] += a_1;
8824 CC[2+bs*1] += a_2;
8825 CC[3+bs*1] += a_3;
8826
8827 b_0 = B[2+0*bs];
8828 CC[0+bs*2] += a_0 * b_0;
8829 CC[1+bs*2] += a_1 * b_0;
8830 CC[2+bs*2] += a_2 * b_0;
8831 CC[3+bs*2] += a_3 * b_0;
8832
8833 A += bs;
8834 B += bs;
8835 // k += 1;
8836
8837 // k = 2
8838
8839 a_0 = A[0];
8840 a_1 = A[1];
8841 a_2 = A[2];
8842 a_3 = A[3];
8843
8844 CC[0+bs*2] += a_0;
8845 CC[1+bs*2] += a_1;
8846 CC[2+bs*2] += a_2;
8847 CC[3+bs*2] += a_3;
8848
8849 A += bs;
8850 B += bs;
8851 // k += 1;
8852
8853 // k = 3
8854
8855 A += bs;
8856 B += bs;
8857 // k += 1;
8858
8859 }
8860 else if(m1==2)
8861 {
8862
8863 // k = 0
8864
8865 a_0 = A[0];
8866 a_1 = A[1];
8867 a_2 = A[2];
8868 a_3 = A[3];
8869
8870 CC[0+bs*0] += a_0;
8871 CC[1+bs*0] += a_1;
8872 CC[2+bs*0] += a_2;
8873 CC[3+bs*0] += a_3;
8874
8875 b_0 = B[1+0*bs];
8876 CC[0+bs*1] += a_0 * b_0;
8877 CC[1+bs*1] += a_1 * b_0;
8878 CC[2+bs*1] += a_2 * b_0;
8879 CC[3+bs*1] += a_3 * b_0;
8880
8881 A += bs;
8882 B += bs;
8883 // k += 1;
8884
8885 // k = 1
8886
8887 a_0 = A[0];
8888 a_1 = A[1];
8889 a_2 = A[2];
8890 a_3 = A[3];
8891
8892 CC[0+bs*1] += a_0;
8893 CC[1+bs*1] += a_1;
8894 CC[2+bs*1] += a_2;
8895 CC[3+bs*1] += a_3;
8896
8897 A += bs;
8898 B += bs;
8899 // k += 1;
8900
8901 // k = 2
8902
8903 A += bs;
8904 B += bs;
8905 // k += 1;
8906
8907 // k = 3
8908
8909 A += bs;
8910 B += bs;
8911 // k += 1;
8912
8913 }
8914 else if(m1==1)
8915 {
8916
8917 // k = 0
8918
8919 a_0 = A[0];
8920 a_1 = A[1];
8921 a_2 = A[2];
8922 a_3 = A[3];
8923
8924 CC[0+bs*0] += a_0;
8925 CC[1+bs*0] += a_1;
8926 CC[2+bs*0] += a_2;
8927 CC[3+bs*0] += a_3;
8928
8929 A += bs;
8930 B += bs;
8931 // k += 1;
8932
8933 // k = 1
8934
8935 A += bs;
8936 B += bs;
8937 // k += 1;
8938
8939 // k = 2
8940
8941 A += bs;
8942 B += bs;
8943 // k += 1;
8944
8945 // k = 3
8946
8947 A += bs;
8948 B += bs;
8949 // k += 1;
8950
8951 }
8952
8953 store:
8954
8955 // scale
8956 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
8957 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
8958 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
8959 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
8960
8961 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
8962 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
8963 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
8964 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
8965
8966 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
8967 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
8968 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
8969 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
8970
8971 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
8972 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
8973 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
8974 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
8975
8976 // transpose
8977 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
8978 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
8979 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
8980
8981 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
8982 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
8983
8984 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
8985
8986 // store
8987 if(m1>=4)
8988 {
8989 D[0+ldd*0] = CC[0+bs*0];
8990 D[1+ldd*0] = CC[1+bs*0];
8991 D[2+ldd*0] = CC[2+bs*0];
8992 D[3+ldd*0] = CC[3+bs*0];
8993
8994 if(n1==1)
8995 return;
8996
8997 D[0+ldd*1] = CC[0+bs*1];
8998 D[1+ldd*1] = CC[1+bs*1];
8999 D[2+ldd*1] = CC[2+bs*1];
9000 D[3+ldd*1] = CC[3+bs*1];
9001
9002 if(n1==2)
9003 return;
9004
9005 D[0+ldd*2] = CC[0+bs*2];
9006 D[1+ldd*2] = CC[1+bs*2];
9007 D[2+ldd*2] = CC[2+bs*2];
9008 D[3+ldd*2] = CC[3+bs*2];
9009
9010 if(n1==3)
9011 return;
9012
9013 D[0+ldd*3] = CC[0+bs*3];
9014 D[1+ldd*3] = CC[1+bs*3];
9015 D[2+ldd*3] = CC[2+bs*3];
9016 D[3+ldd*3] = CC[3+bs*3];
9017 }
9018 else if(m1>=3)
9019 {
9020 D[0+ldd*0] = CC[0+bs*0];
9021 D[1+ldd*0] = CC[1+bs*0];
9022 D[2+ldd*0] = CC[2+bs*0];
9023
9024 if(n1==1)
9025 return;
9026
9027 D[0+ldd*1] = CC[0+bs*1];
9028 D[1+ldd*1] = CC[1+bs*1];
9029 D[2+ldd*1] = CC[2+bs*1];
9030
9031 if(n1==2)
9032 return;
9033
9034 D[0+ldd*2] = CC[0+bs*2];
9035 D[1+ldd*2] = CC[1+bs*2];
9036 D[2+ldd*2] = CC[2+bs*2];
9037
9038 if(n1==3)
9039 return;
9040
9041 D[0+ldd*3] = CC[0+bs*3];
9042 D[1+ldd*3] = CC[1+bs*3];
9043 D[2+ldd*3] = CC[2+bs*3];
9044 }
9045 else if(m1>=2)
9046 {
9047 D[0+ldd*0] = CC[0+bs*0];
9048 D[1+ldd*0] = CC[1+bs*0];
9049
9050 if(n1==1)
9051 return;
9052
9053 D[0+ldd*1] = CC[0+bs*1];
9054 D[1+ldd*1] = CC[1+bs*1];
9055
9056 if(n1==2)
9057 return;
9058
9059 D[0+ldd*2] = CC[0+bs*2];
9060 D[1+ldd*2] = CC[1+bs*2];
9061
9062 if(n1==3)
9063 return;
9064
9065 D[0+ldd*3] = CC[0+bs*3];
9066 D[1+ldd*3] = CC[1+bs*3];
9067 }
9068 else //if(m1>=1)
9069 {
9070 D[0+ldd*0] = CC[0+bs*0];
9071
9072 if(n1==1)
9073 return;
9074
9075 D[0+ldd*1] = CC[0+bs*1];
9076
9077 if(n1==2)
9078 return;
9079
9080 D[0+ldd*2] = CC[0+bs*2];
9081
9082 if(n1==3)
9083 return;
9084
9085 D[0+ldd*3] = CC[0+bs*3];
9086 }
9087
9088 return;
9089
9090 }
9091 //#endif
9092
9093
9094
9095 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)9096 void kernel_strmm_nt_rl_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
9097 {
9098
9099 const int bs = 4;
9100
9101 float
9102 a_0, a_1, a_2, a_3,
9103 b_0, b_1, b_2, b_3;
9104
9105 #if defined(TARGET_GENERIC)
9106 float CC[16] = {0};
9107 #else
9108 ALIGNED( float CC[16], 64 ) = {0};
9109 #endif
9110
9111 float alpha1 = 1.0;
9112
9113 // assume always kmax>=4 !!!
9114
9115 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
9116
9117 A += kmax*bs;
9118 B += kmax*ldb;
9119
9120 // k = 0
9121
9122 a_0 = A[0];
9123 a_1 = A[1];
9124 a_2 = A[2];
9125 a_3 = A[3];
9126
9127 b_0 = B[0+0*ldb];
9128 CC[0+bs*0] += a_0 * b_0;
9129 CC[1+bs*0] += a_1 * b_0;
9130 CC[2+bs*0] += a_2 * b_0;
9131 CC[3+bs*0] += a_3 * b_0;
9132
9133 b_0 = B[1+0*ldb];
9134 CC[0+bs*1] += a_0 * b_0;
9135 CC[1+bs*1] += a_1 * b_0;
9136 CC[2+bs*1] += a_2 * b_0;
9137 CC[3+bs*1] += a_3 * b_0;
9138
9139 b_0 = B[2+0*ldb];
9140 CC[0+bs*2] += a_0 * b_0;
9141 CC[1+bs*2] += a_1 * b_0;
9142 CC[2+bs*2] += a_2 * b_0;
9143 CC[3+bs*2] += a_3 * b_0;
9144
9145 b_0 = B[3+0*ldb];
9146 CC[0+bs*3] += a_0 * b_0;
9147 CC[1+bs*3] += a_1 * b_0;
9148 CC[2+bs*3] += a_2 * b_0;
9149 CC[3+bs*3] += a_3 * b_0;
9150
9151 A += bs;
9152 B += ldb;
9153 // k += 1;
9154
9155 // k = 1
9156
9157 a_0 = A[0];
9158 a_1 = A[1];
9159 a_2 = A[2];
9160 a_3 = A[3];
9161
9162 b_0 = B[1+0*ldb];
9163 CC[0+bs*1] += a_0 * b_0;
9164 CC[1+bs*1] += a_1 * b_0;
9165 CC[2+bs*1] += a_2 * b_0;
9166 CC[3+bs*1] += a_3 * b_0;
9167
9168 b_0 = B[2+0*ldb];
9169 CC[0+bs*2] += a_0 * b_0;
9170 CC[1+bs*2] += a_1 * b_0;
9171 CC[2+bs*2] += a_2 * b_0;
9172 CC[3+bs*2] += a_3 * b_0;
9173
9174 b_0 = B[3+0*ldb];
9175 CC[0+bs*3] += a_0 * b_0;
9176 CC[1+bs*3] += a_1 * b_0;
9177 CC[2+bs*3] += a_2 * b_0;
9178 CC[3+bs*3] += a_3 * b_0;
9179
9180 A += bs;
9181 B += ldb;
9182 // k += 1;
9183
9184 // k = 2
9185
9186 a_0 = A[0];
9187 a_1 = A[1];
9188 a_2 = A[2];
9189 a_3 = A[3];
9190
9191 b_0 = B[2+0*ldb];
9192 CC[0+bs*2] += a_0 * b_0;
9193 CC[1+bs*2] += a_1 * b_0;
9194 CC[2+bs*2] += a_2 * b_0;
9195 CC[3+bs*2] += a_3 * b_0;
9196
9197 b_0 = B[3+0*ldb];
9198 CC[0+bs*3] += a_0 * b_0;
9199 CC[1+bs*3] += a_1 * b_0;
9200 CC[2+bs*3] += a_2 * b_0;
9201 CC[3+bs*3] += a_3 * b_0;
9202
9203 A += bs;
9204 B += ldb;
9205 // k += 1;
9206
9207 // k = 3
9208
9209 a_0 = A[0];
9210 a_1 = A[1];
9211 a_2 = A[2];
9212 a_3 = A[3];
9213
9214 b_0 = B[3+0*ldb];
9215 CC[0+bs*3] += a_0 * b_0;
9216 CC[1+bs*3] += a_1 * b_0;
9217 CC[2+bs*3] += a_2 * b_0;
9218 CC[3+bs*3] += a_3 * b_0;
9219
9220 A += bs;
9221 B += ldb;
9222 // k += 1;
9223
9224 store:
9225
9226 D[0+ldd*0] = alpha[0]*CC[0+bs*0];
9227 D[1+ldd*0] = alpha[0]*CC[1+bs*0];
9228 D[2+ldd*0] = alpha[0]*CC[2+bs*0];
9229 D[3+ldd*0] = alpha[0]*CC[3+bs*0];
9230
9231 D[0+ldd*1] = alpha[0]*CC[0+bs*1];
9232 D[1+ldd*1] = alpha[0]*CC[1+bs*1];
9233 D[2+ldd*1] = alpha[0]*CC[2+bs*1];
9234 D[3+ldd*1] = alpha[0]*CC[3+bs*1];
9235
9236 D[0+ldd*2] = alpha[0]*CC[0+bs*2];
9237 D[1+ldd*2] = alpha[0]*CC[1+bs*2];
9238 D[2+ldd*2] = alpha[0]*CC[2+bs*2];
9239 D[3+ldd*2] = alpha[0]*CC[3+bs*2];
9240
9241 D[0+ldd*3] = alpha[0]*CC[0+bs*3];
9242 D[1+ldd*3] = alpha[0]*CC[1+bs*3];
9243 D[2+ldd*3] = alpha[0]*CC[2+bs*3];
9244 D[3+ldd*3] = alpha[0]*CC[3+bs*3];
9245
9246 return;
9247
9248 }
9249 //#endif
9250
9251
9252
9253 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)9254 void kernel_strmm_nt_rl_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
9255 {
9256
9257 const int bs = 4;
9258
9259 float
9260 a_0, a_1, a_2, a_3,
9261 b_0, b_1, b_2, b_3;
9262
9263 #if defined(TARGET_GENERIC)
9264 float CC[16] = {0};
9265 #else
9266 ALIGNED( float CC[16], 64 ) = {0};
9267 #endif
9268
9269 float alpha1 = 1.0;
9270
9271 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
9272
9273 A += kmax*bs;
9274 B += kmax*ldb;
9275
9276 if(n1>=4)
9277 {
9278
9279 // k = 0
9280
9281 a_0 = A[0];
9282 a_1 = A[1];
9283 a_2 = A[2];
9284 a_3 = A[3];
9285
9286 b_0 = B[0+0*ldb];
9287 CC[0+bs*0] += a_0 * b_0;
9288 CC[1+bs*0] += a_1 * b_0;
9289 CC[2+bs*0] += a_2 * b_0;
9290 CC[3+bs*0] += a_3 * b_0;
9291
9292 b_0 = B[1+0*ldb];
9293 CC[0+bs*1] += a_0 * b_0;
9294 CC[1+bs*1] += a_1 * b_0;
9295 CC[2+bs*1] += a_2 * b_0;
9296 CC[3+bs*1] += a_3 * b_0;
9297
9298 b_0 = B[2+0*ldb];
9299 CC[0+bs*2] += a_0 * b_0;
9300 CC[1+bs*2] += a_1 * b_0;
9301 CC[2+bs*2] += a_2 * b_0;
9302 CC[3+bs*2] += a_3 * b_0;
9303
9304 b_0 = B[3+0*ldb];
9305 CC[0+bs*3] += a_0 * b_0;
9306 CC[1+bs*3] += a_1 * b_0;
9307 CC[2+bs*3] += a_2 * b_0;
9308 CC[3+bs*3] += a_3 * b_0;
9309
9310 A += bs;
9311 B += ldb;
9312 // k += 1;
9313
9314 // k = 1
9315
9316 a_0 = A[0];
9317 a_1 = A[1];
9318 a_2 = A[2];
9319 a_3 = A[3];
9320
9321 b_0 = B[1+0*ldb];
9322 CC[0+bs*1] += a_0 * b_0;
9323 CC[1+bs*1] += a_1 * b_0;
9324 CC[2+bs*1] += a_2 * b_0;
9325 CC[3+bs*1] += a_3 * b_0;
9326
9327 b_0 = B[2+0*ldb];
9328 CC[0+bs*2] += a_0 * b_0;
9329 CC[1+bs*2] += a_1 * b_0;
9330 CC[2+bs*2] += a_2 * b_0;
9331 CC[3+bs*2] += a_3 * b_0;
9332
9333 b_0 = B[3+0*ldb];
9334 CC[0+bs*3] += a_0 * b_0;
9335 CC[1+bs*3] += a_1 * b_0;
9336 CC[2+bs*3] += a_2 * b_0;
9337 CC[3+bs*3] += a_3 * b_0;
9338
9339 A += bs;
9340 B += ldb;
9341 // k += 1;
9342
9343 // k = 2
9344
9345 a_0 = A[0];
9346 a_1 = A[1];
9347 a_2 = A[2];
9348 a_3 = A[3];
9349
9350 b_0 = B[2+0*ldb];
9351 CC[0+bs*2] += a_0 * b_0;
9352 CC[1+bs*2] += a_1 * b_0;
9353 CC[2+bs*2] += a_2 * b_0;
9354 CC[3+bs*2] += a_3 * b_0;
9355
9356 b_0 = B[3+0*ldb];
9357 CC[0+bs*3] += a_0 * b_0;
9358 CC[1+bs*3] += a_1 * b_0;
9359 CC[2+bs*3] += a_2 * b_0;
9360 CC[3+bs*3] += a_3 * b_0;
9361
9362 A += bs;
9363 B += ldb;
9364 // k += 1;
9365
9366 // k = 3
9367
9368 a_0 = A[0];
9369 a_1 = A[1];
9370 a_2 = A[2];
9371 a_3 = A[3];
9372
9373 b_0 = B[3+0*ldb];
9374 CC[0+bs*3] += a_0 * b_0;
9375 CC[1+bs*3] += a_1 * b_0;
9376 CC[2+bs*3] += a_2 * b_0;
9377 CC[3+bs*3] += a_3 * b_0;
9378
9379 A += bs;
9380 B += ldb;
9381 // k += 1;
9382
9383 }
9384 else if(n1==3)
9385 {
9386
9387 // k = 0
9388
9389 a_0 = A[0];
9390 a_1 = A[1];
9391 a_2 = A[2];
9392 a_3 = A[3];
9393
9394 b_0 = B[0+0*ldb];
9395 CC[0+bs*0] += a_0 * b_0;
9396 CC[1+bs*0] += a_1 * b_0;
9397 CC[2+bs*0] += a_2 * b_0;
9398 CC[3+bs*0] += a_3 * b_0;
9399
9400 b_0 = B[1+0*ldb];
9401 CC[0+bs*1] += a_0 * b_0;
9402 CC[1+bs*1] += a_1 * b_0;
9403 CC[2+bs*1] += a_2 * b_0;
9404 CC[3+bs*1] += a_3 * b_0;
9405
9406 b_0 = B[2+0*ldb];
9407 CC[0+bs*2] += a_0 * b_0;
9408 CC[1+bs*2] += a_1 * b_0;
9409 CC[2+bs*2] += a_2 * b_0;
9410 CC[3+bs*2] += a_3 * b_0;
9411
9412 A += bs;
9413 B += ldb;
9414 // k += 1;
9415
9416 // k = 1
9417
9418 a_0 = A[0];
9419 a_1 = A[1];
9420 a_2 = A[2];
9421 a_3 = A[3];
9422
9423 b_0 = B[1+0*ldb];
9424 CC[0+bs*1] += a_0 * b_0;
9425 CC[1+bs*1] += a_1 * b_0;
9426 CC[2+bs*1] += a_2 * b_0;
9427 CC[3+bs*1] += a_3 * b_0;
9428
9429 b_0 = B[2+0*ldb];
9430 CC[0+bs*2] += a_0 * b_0;
9431 CC[1+bs*2] += a_1 * b_0;
9432 CC[2+bs*2] += a_2 * b_0;
9433 CC[3+bs*2] += a_3 * b_0;
9434
9435 A += bs;
9436 B += ldb;
9437 // k += 1;
9438
9439 // k = 2
9440
9441 a_0 = A[0];
9442 a_1 = A[1];
9443 a_2 = A[2];
9444 a_3 = A[3];
9445
9446 b_0 = B[2+0*ldb];
9447 CC[0+bs*2] += a_0 * b_0;
9448 CC[1+bs*2] += a_1 * b_0;
9449 CC[2+bs*2] += a_2 * b_0;
9450 CC[3+bs*2] += a_3 * b_0;
9451
9452 A += bs;
9453 B += ldb;
9454 // k += 1;
9455
9456 // k = 3
9457
9458 A += bs;
9459 B += ldb;
9460 // k += 1;
9461
9462 }
9463 else if(n1==2)
9464 {
9465
9466 // k = 0
9467
9468 a_0 = A[0];
9469 a_1 = A[1];
9470 a_2 = A[2];
9471 a_3 = A[3];
9472
9473 b_0 = B[0+0*ldb];
9474 CC[0+bs*0] += a_0 * b_0;
9475 CC[1+bs*0] += a_1 * b_0;
9476 CC[2+bs*0] += a_2 * b_0;
9477 CC[3+bs*0] += a_3 * b_0;
9478
9479 b_0 = B[1+0*ldb];
9480 CC[0+bs*1] += a_0 * b_0;
9481 CC[1+bs*1] += a_1 * b_0;
9482 CC[2+bs*1] += a_2 * b_0;
9483 CC[3+bs*1] += a_3 * b_0;
9484
9485 A += bs;
9486 B += ldb;
9487 // k += 1;
9488
9489 // k = 1
9490
9491 a_0 = A[0];
9492 a_1 = A[1];
9493 a_2 = A[2];
9494 a_3 = A[3];
9495
9496 b_0 = B[1+0*ldb];
9497 CC[0+bs*1] += a_0 * b_0;
9498 CC[1+bs*1] += a_1 * b_0;
9499 CC[2+bs*1] += a_2 * b_0;
9500 CC[3+bs*1] += a_3 * b_0;
9501
9502 A += bs;
9503 B += ldb;
9504 // k += 1;
9505
9506 // k = 2
9507
9508 A += bs;
9509 B += ldb;
9510 // k += 1;
9511
9512 // k = 3
9513
9514 A += bs;
9515 B += ldb;
9516 // k += 1;
9517
9518 }
9519 else if(n1==1)
9520 {
9521
9522 // k = 0
9523
9524 a_0 = A[0];
9525 a_1 = A[1];
9526 a_2 = A[2];
9527 a_3 = A[3];
9528
9529 b_0 = B[0+0*ldb];
9530 CC[0+bs*0] += a_0 * b_0;
9531 CC[1+bs*0] += a_1 * b_0;
9532 CC[2+bs*0] += a_2 * b_0;
9533 CC[3+bs*0] += a_3 * b_0;
9534
9535 A += bs;
9536 B += ldb;
9537 // k += 1;
9538
9539 // k = 1
9540
9541 A += bs;
9542 B += ldb;
9543 // k += 1;
9544
9545 // k = 2
9546
9547 A += bs;
9548 B += ldb;
9549 // k += 1;
9550
9551 // k = 3
9552
9553 A += bs;
9554 B += ldb;
9555 // k += 1;
9556
9557 }
9558
9559 store:
9560
9561 CC[0+bs*0] = alpha[0]*CC[0+bs*0];
9562 CC[1+bs*0] = alpha[0]*CC[1+bs*0];
9563 CC[2+bs*0] = alpha[0]*CC[2+bs*0];
9564 CC[3+bs*0] = alpha[0]*CC[3+bs*0];
9565
9566 CC[0+bs*1] = alpha[0]*CC[0+bs*1];
9567 CC[1+bs*1] = alpha[0]*CC[1+bs*1];
9568 CC[2+bs*1] = alpha[0]*CC[2+bs*1];
9569 CC[3+bs*1] = alpha[0]*CC[3+bs*1];
9570
9571 CC[0+bs*2] = alpha[0]*CC[0+bs*2];
9572 CC[1+bs*2] = alpha[0]*CC[1+bs*2];
9573 CC[2+bs*2] = alpha[0]*CC[2+bs*2];
9574 CC[3+bs*2] = alpha[0]*CC[3+bs*2];
9575
9576 CC[0+bs*3] = alpha[0]*CC[0+bs*3];
9577 CC[1+bs*3] = alpha[0]*CC[1+bs*3];
9578 CC[2+bs*3] = alpha[0]*CC[2+bs*3];
9579 CC[3+bs*3] = alpha[0]*CC[3+bs*3];
9580
9581 if(m1>=4)
9582 {
9583 D[0+ldd*0] = CC[0+bs*0];
9584 D[1+ldd*0] = CC[1+bs*0];
9585 D[2+ldd*0] = CC[2+bs*0];
9586 D[3+ldd*0] = CC[3+bs*0];
9587
9588 if(n1==1)
9589 return;
9590
9591 D[0+ldd*1] = CC[0+bs*1];
9592 D[1+ldd*1] = CC[1+bs*1];
9593 D[2+ldd*1] = CC[2+bs*1];
9594 D[3+ldd*1] = CC[3+bs*1];
9595
9596 if(n1==2)
9597 return;
9598
9599 D[0+ldd*2] = CC[0+bs*2];
9600 D[1+ldd*2] = CC[1+bs*2];
9601 D[2+ldd*2] = CC[2+bs*2];
9602 D[3+ldd*2] = CC[3+bs*2];
9603
9604 if(n1==3)
9605 return;
9606
9607 D[0+ldd*3] = CC[0+bs*3];
9608 D[1+ldd*3] = CC[1+bs*3];
9609 D[2+ldd*3] = CC[2+bs*3];
9610 D[3+ldd*3] = CC[3+bs*3];
9611 }
9612 else if(m1>=3)
9613 {
9614 D[0+ldd*0] = CC[0+bs*0];
9615 D[1+ldd*0] = CC[1+bs*0];
9616 D[2+ldd*0] = CC[2+bs*0];
9617
9618 if(n1==1)
9619 return;
9620
9621 D[0+ldd*1] = CC[0+bs*1];
9622 D[1+ldd*1] = CC[1+bs*1];
9623 D[2+ldd*1] = CC[2+bs*1];
9624
9625 if(n1==2)
9626 return;
9627
9628 D[0+ldd*2] = CC[0+bs*2];
9629 D[1+ldd*2] = CC[1+bs*2];
9630 D[2+ldd*2] = CC[2+bs*2];
9631
9632 if(n1==3)
9633 return;
9634
9635 D[0+ldd*3] = CC[0+bs*3];
9636 D[1+ldd*3] = CC[1+bs*3];
9637 D[2+ldd*3] = CC[2+bs*3];
9638 }
9639 else if(m1>=2)
9640 {
9641 D[0+ldd*0] = CC[0+bs*0];
9642 D[1+ldd*0] = CC[1+bs*0];
9643
9644 if(n1==1)
9645 return;
9646
9647 D[0+ldd*1] = CC[0+bs*1];
9648 D[1+ldd*1] = CC[1+bs*1];
9649
9650 if(n1==2)
9651 return;
9652
9653 D[0+ldd*2] = CC[0+bs*2];
9654 D[1+ldd*2] = CC[1+bs*2];
9655
9656 if(n1==3)
9657 return;
9658
9659 D[0+ldd*3] = CC[0+bs*3];
9660 D[1+ldd*3] = CC[1+bs*3];
9661 }
9662 else //if(m1>=1)
9663 {
9664 D[0+ldd*0] = CC[0+bs*0];
9665
9666 if(n1==1)
9667 return;
9668
9669 D[0+ldd*1] = CC[0+bs*1];
9670
9671 if(n1==2)
9672 return;
9673
9674 D[0+ldd*2] = CC[0+bs*2];
9675
9676 if(n1==3)
9677 return;
9678
9679 D[0+ldd*3] = CC[0+bs*3];
9680 }
9681
9682 return;
9683
9684 }
9685 //#endif
9686
9687
9688
9689 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_4x4_tran_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd)9690 void kernel_strmm_nt_rl_4x4_tran_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd)
9691 {
9692
9693 const int bs = 4;
9694
9695 float
9696 a_0, a_1, a_2, a_3,
9697 b_0, b_1, b_2, b_3;
9698
9699 #if defined(TARGET_GENERIC)
9700 float CC[16] = {0};
9701 #else
9702 ALIGNED( float CC[16], 64 ) = {0};
9703 #endif
9704
9705 float alpha1 = 1.0;
9706
9707 // assume always kmax>=4 !!!
9708
9709 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
9710
9711 A += kmax*bs;
9712 B += kmax*ldb;
9713
9714 // k = 0
9715
9716 a_0 = A[0];
9717 a_1 = A[1];
9718 a_2 = A[2];
9719 a_3 = A[3];
9720
9721 b_0 = B[0+0*ldb];
9722 CC[0+bs*0] += a_0 * b_0;
9723 CC[1+bs*0] += a_1 * b_0;
9724 CC[2+bs*0] += a_2 * b_0;
9725 CC[3+bs*0] += a_3 * b_0;
9726
9727 b_0 = B[1+0*ldb];
9728 CC[0+bs*1] += a_0 * b_0;
9729 CC[1+bs*1] += a_1 * b_0;
9730 CC[2+bs*1] += a_2 * b_0;
9731 CC[3+bs*1] += a_3 * b_0;
9732
9733 b_0 = B[2+0*ldb];
9734 CC[0+bs*2] += a_0 * b_0;
9735 CC[1+bs*2] += a_1 * b_0;
9736 CC[2+bs*2] += a_2 * b_0;
9737 CC[3+bs*2] += a_3 * b_0;
9738
9739 b_0 = B[3+0*ldb];
9740 CC[0+bs*3] += a_0 * b_0;
9741 CC[1+bs*3] += a_1 * b_0;
9742 CC[2+bs*3] += a_2 * b_0;
9743 CC[3+bs*3] += a_3 * b_0;
9744
9745 A += bs;
9746 B += ldb;
9747 // k += 1;
9748
9749 // k = 1
9750
9751 a_0 = A[0];
9752 a_1 = A[1];
9753 a_2 = A[2];
9754 a_3 = A[3];
9755
9756 b_0 = B[1+0*ldb];
9757 CC[0+bs*1] += a_0 * b_0;
9758 CC[1+bs*1] += a_1 * b_0;
9759 CC[2+bs*1] += a_2 * b_0;
9760 CC[3+bs*1] += a_3 * b_0;
9761
9762 b_0 = B[2+0*ldb];
9763 CC[0+bs*2] += a_0 * b_0;
9764 CC[1+bs*2] += a_1 * b_0;
9765 CC[2+bs*2] += a_2 * b_0;
9766 CC[3+bs*2] += a_3 * b_0;
9767
9768 b_0 = B[3+0*ldb];
9769 CC[0+bs*3] += a_0 * b_0;
9770 CC[1+bs*3] += a_1 * b_0;
9771 CC[2+bs*3] += a_2 * b_0;
9772 CC[3+bs*3] += a_3 * b_0;
9773
9774 A += bs;
9775 B += ldb;
9776 // k += 1;
9777
9778 // k = 2
9779
9780 a_0 = A[0];
9781 a_1 = A[1];
9782 a_2 = A[2];
9783 a_3 = A[3];
9784
9785 b_0 = B[2+0*ldb];
9786 CC[0+bs*2] += a_0 * b_0;
9787 CC[1+bs*2] += a_1 * b_0;
9788 CC[2+bs*2] += a_2 * b_0;
9789 CC[3+bs*2] += a_3 * b_0;
9790
9791 b_0 = B[3+0*ldb];
9792 CC[0+bs*3] += a_0 * b_0;
9793 CC[1+bs*3] += a_1 * b_0;
9794 CC[2+bs*3] += a_2 * b_0;
9795 CC[3+bs*3] += a_3 * b_0;
9796
9797 A += bs;
9798 B += ldb;
9799 // k += 1;
9800
9801 // k = 3
9802
9803 a_0 = A[0];
9804 a_1 = A[1];
9805 a_2 = A[2];
9806 a_3 = A[3];
9807
9808 b_0 = B[3+0*ldb];
9809 CC[0+bs*3] += a_0 * b_0;
9810 CC[1+bs*3] += a_1 * b_0;
9811 CC[2+bs*3] += a_2 * b_0;
9812 CC[3+bs*3] += a_3 * b_0;
9813
9814 A += bs;
9815 B += ldb;
9816 // k += 1;
9817
9818 store:
9819
9820 // scale & tranpose & store
9821 D[0+ldd*0] = alpha[0]*CC[0+bs*0];
9822 D[0+ldd*1] = alpha[0]*CC[1+bs*0];
9823 D[0+ldd*2] = alpha[0]*CC[2+bs*0];
9824 D[0+ldd*3] = alpha[0]*CC[3+bs*0];
9825
9826 D[1+ldd*0] = alpha[0]*CC[0+bs*1];
9827 D[1+ldd*1] = alpha[0]*CC[1+bs*1];
9828 D[1+ldd*2] = alpha[0]*CC[2+bs*1];
9829 D[1+ldd*3] = alpha[0]*CC[3+bs*1];
9830
9831 D[2+ldd*0] = alpha[0]*CC[0+bs*2];
9832 D[2+ldd*1] = alpha[0]*CC[1+bs*2];
9833 D[2+ldd*2] = alpha[0]*CC[2+bs*2];
9834 D[2+ldd*3] = alpha[0]*CC[3+bs*2];
9835
9836 D[3+ldd*0] = alpha[0]*CC[0+bs*3];
9837 D[3+ldd*1] = alpha[0]*CC[1+bs*3];
9838 D[3+ldd*2] = alpha[0]*CC[2+bs*3];
9839 D[3+ldd*3] = alpha[0]*CC[3+bs*3];
9840
9841 return;
9842
9843 }
9844 //#endif
9845
9846
9847
9848 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_4x4_tran_vs_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd,int m1,int n1)9849 void kernel_strmm_nt_rl_4x4_tran_vs_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd, int m1, int n1)
9850 {
9851
9852 const int bs = 4;
9853
9854 float
9855 tmp,
9856 a_0, a_1, a_2, a_3,
9857 b_0, b_1, b_2, b_3;
9858
9859 #if defined(TARGET_GENERIC)
9860 float CC[16] = {0};
9861 #else
9862 ALIGNED( float CC[16], 64 ) = {0};
9863 #endif
9864
9865 float alpha1 = 1.0;
9866
9867 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
9868
9869 A += kmax*bs;
9870 B += kmax*ldb;
9871
9872 // XXX m1 and n1 are swapped !!!!!
9873 if(m1>=4)
9874 {
9875
9876 // k = 0
9877
9878 a_0 = A[0];
9879 a_1 = A[1];
9880 a_2 = A[2];
9881 a_3 = A[3];
9882
9883 b_0 = B[0+0*ldb];
9884 CC[0+bs*0] += a_0 * b_0;
9885 CC[1+bs*0] += a_1 * b_0;
9886 CC[2+bs*0] += a_2 * b_0;
9887 CC[3+bs*0] += a_3 * b_0;
9888
9889 b_0 = B[1+0*ldb];
9890 CC[0+bs*1] += a_0 * b_0;
9891 CC[1+bs*1] += a_1 * b_0;
9892 CC[2+bs*1] += a_2 * b_0;
9893 CC[3+bs*1] += a_3 * b_0;
9894
9895 b_0 = B[2+0*ldb];
9896 CC[0+bs*2] += a_0 * b_0;
9897 CC[1+bs*2] += a_1 * b_0;
9898 CC[2+bs*2] += a_2 * b_0;
9899 CC[3+bs*2] += a_3 * b_0;
9900
9901 b_0 = B[3+0*ldb];
9902 CC[0+bs*3] += a_0 * b_0;
9903 CC[1+bs*3] += a_1 * b_0;
9904 CC[2+bs*3] += a_2 * b_0;
9905 CC[3+bs*3] += a_3 * b_0;
9906
9907 A += bs;
9908 B += ldb;
9909 // k += 1;
9910
9911 // k = 1
9912
9913 a_0 = A[0];
9914 a_1 = A[1];
9915 a_2 = A[2];
9916 a_3 = A[3];
9917
9918 b_0 = B[1+0*ldb];
9919 CC[0+bs*1] += a_0 * b_0;
9920 CC[1+bs*1] += a_1 * b_0;
9921 CC[2+bs*1] += a_2 * b_0;
9922 CC[3+bs*1] += a_3 * b_0;
9923
9924 b_0 = B[2+0*ldb];
9925 CC[0+bs*2] += a_0 * b_0;
9926 CC[1+bs*2] += a_1 * b_0;
9927 CC[2+bs*2] += a_2 * b_0;
9928 CC[3+bs*2] += a_3 * b_0;
9929
9930 b_0 = B[3+0*ldb];
9931 CC[0+bs*3] += a_0 * b_0;
9932 CC[1+bs*3] += a_1 * b_0;
9933 CC[2+bs*3] += a_2 * b_0;
9934 CC[3+bs*3] += a_3 * b_0;
9935
9936 A += bs;
9937 B += ldb;
9938 // k += 1;
9939
9940 // k = 2
9941
9942 a_0 = A[0];
9943 a_1 = A[1];
9944 a_2 = A[2];
9945 a_3 = A[3];
9946
9947 b_0 = B[2+0*ldb];
9948 CC[0+bs*2] += a_0 * b_0;
9949 CC[1+bs*2] += a_1 * b_0;
9950 CC[2+bs*2] += a_2 * b_0;
9951 CC[3+bs*2] += a_3 * b_0;
9952
9953 b_0 = B[3+0*ldb];
9954 CC[0+bs*3] += a_0 * b_0;
9955 CC[1+bs*3] += a_1 * b_0;
9956 CC[2+bs*3] += a_2 * b_0;
9957 CC[3+bs*3] += a_3 * b_0;
9958
9959 A += bs;
9960 B += ldb;
9961 // k += 1;
9962
9963 // k = 3
9964
9965 a_0 = A[0];
9966 a_1 = A[1];
9967 a_2 = A[2];
9968 a_3 = A[3];
9969
9970 b_0 = B[3+0*ldb];
9971 CC[0+bs*3] += a_0 * b_0;
9972 CC[1+bs*3] += a_1 * b_0;
9973 CC[2+bs*3] += a_2 * b_0;
9974 CC[3+bs*3] += a_3 * b_0;
9975
9976 A += bs;
9977 B += ldb;
9978 // k += 1;
9979
9980 }
9981 else if(m1==3)
9982 {
9983
9984 // k = 0
9985
9986 a_0 = A[0];
9987 a_1 = A[1];
9988 a_2 = A[2];
9989 a_3 = A[3];
9990
9991 b_0 = B[0+0*ldb];
9992 CC[0+bs*0] += a_0 * b_0;
9993 CC[1+bs*0] += a_1 * b_0;
9994 CC[2+bs*0] += a_2 * b_0;
9995 CC[3+bs*0] += a_3 * b_0;
9996
9997 b_0 = B[1+0*ldb];
9998 CC[0+bs*1] += a_0 * b_0;
9999 CC[1+bs*1] += a_1 * b_0;
10000 CC[2+bs*1] += a_2 * b_0;
10001 CC[3+bs*1] += a_3 * b_0;
10002
10003 b_0 = B[2+0*ldb];
10004 CC[0+bs*2] += a_0 * b_0;
10005 CC[1+bs*2] += a_1 * b_0;
10006 CC[2+bs*2] += a_2 * b_0;
10007 CC[3+bs*2] += a_3 * b_0;
10008
10009 A += bs;
10010 B += ldb;
10011 // k += 1;
10012
10013 // k = 1
10014
10015 a_0 = A[0];
10016 a_1 = A[1];
10017 a_2 = A[2];
10018 a_3 = A[3];
10019
10020 b_0 = B[1+0*ldb];
10021 CC[0+bs*1] += a_0 * b_0;
10022 CC[1+bs*1] += a_1 * b_0;
10023 CC[2+bs*1] += a_2 * b_0;
10024 CC[3+bs*1] += a_3 * b_0;
10025
10026 b_0 = B[2+0*ldb];
10027 CC[0+bs*2] += a_0 * b_0;
10028 CC[1+bs*2] += a_1 * b_0;
10029 CC[2+bs*2] += a_2 * b_0;
10030 CC[3+bs*2] += a_3 * b_0;
10031
10032 A += bs;
10033 B += ldb;
10034 // k += 1;
10035
10036 // k = 2
10037
10038 a_0 = A[0];
10039 a_1 = A[1];
10040 a_2 = A[2];
10041 a_3 = A[3];
10042
10043 b_0 = B[2+0*ldb];
10044 CC[0+bs*2] += a_0 * b_0;
10045 CC[1+bs*2] += a_1 * b_0;
10046 CC[2+bs*2] += a_2 * b_0;
10047 CC[3+bs*2] += a_3 * b_0;
10048
10049 A += bs;
10050 B += ldb;
10051 // k += 1;
10052
10053 // k = 3
10054
10055 A += bs;
10056 B += ldb;
10057 // k += 1;
10058
10059 }
10060 else if(m1==2)
10061 {
10062
10063 // k = 0
10064
10065 a_0 = A[0];
10066 a_1 = A[1];
10067 a_2 = A[2];
10068 a_3 = A[3];
10069
10070 b_0 = B[0+0*ldb];
10071 CC[0+bs*0] += a_0 * b_0;
10072 CC[1+bs*0] += a_1 * b_0;
10073 CC[2+bs*0] += a_2 * b_0;
10074 CC[3+bs*0] += a_3 * b_0;
10075
10076 b_0 = B[1+0*ldb];
10077 CC[0+bs*1] += a_0 * b_0;
10078 CC[1+bs*1] += a_1 * b_0;
10079 CC[2+bs*1] += a_2 * b_0;
10080 CC[3+bs*1] += a_3 * b_0;
10081
10082 A += bs;
10083 B += ldb;
10084 // k += 1;
10085
10086 // k = 1
10087
10088 a_0 = A[0];
10089 a_1 = A[1];
10090 a_2 = A[2];
10091 a_3 = A[3];
10092
10093 b_0 = B[1+0*ldb];
10094 CC[0+bs*1] += a_0 * b_0;
10095 CC[1+bs*1] += a_1 * b_0;
10096 CC[2+bs*1] += a_2 * b_0;
10097 CC[3+bs*1] += a_3 * b_0;
10098
10099 A += bs;
10100 B += ldb;
10101 // k += 1;
10102
10103 // k = 2
10104
10105 A += bs;
10106 B += ldb;
10107 // k += 1;
10108
10109 // k = 3
10110
10111 A += bs;
10112 B += ldb;
10113 // k += 1;
10114
10115 }
10116 else if(m1==1)
10117 {
10118
10119 // k = 0
10120
10121 a_0 = A[0];
10122 a_1 = A[1];
10123 a_2 = A[2];
10124 a_3 = A[3];
10125
10126 b_0 = B[0+0*ldb];
10127 CC[0+bs*0] += a_0 * b_0;
10128 CC[1+bs*0] += a_1 * b_0;
10129 CC[2+bs*0] += a_2 * b_0;
10130 CC[3+bs*0] += a_3 * b_0;
10131
10132 A += bs;
10133 B += ldb;
10134 // k += 1;
10135
10136 // k = 1
10137
10138 A += bs;
10139 B += ldb;
10140 // k += 1;
10141
10142 // k = 2
10143
10144 A += bs;
10145 B += ldb;
10146 // k += 1;
10147
10148 // k = 3
10149
10150 A += bs;
10151 B += ldb;
10152 // k += 1;
10153
10154 }
10155
10156 store:
10157
10158 // scale
10159 CC[0+bs*0] = alpha[0]*CC[0+bs*0];
10160 CC[1+bs*0] = alpha[0]*CC[1+bs*0];
10161 CC[2+bs*0] = alpha[0]*CC[2+bs*0];
10162 CC[3+bs*0] = alpha[0]*CC[3+bs*0];
10163
10164 CC[0+bs*1] = alpha[0]*CC[0+bs*1];
10165 CC[1+bs*1] = alpha[0]*CC[1+bs*1];
10166 CC[2+bs*1] = alpha[0]*CC[2+bs*1];
10167 CC[3+bs*1] = alpha[0]*CC[3+bs*1];
10168
10169 CC[0+bs*2] = alpha[0]*CC[0+bs*2];
10170 CC[1+bs*2] = alpha[0]*CC[1+bs*2];
10171 CC[2+bs*2] = alpha[0]*CC[2+bs*2];
10172 CC[3+bs*2] = alpha[0]*CC[3+bs*2];
10173
10174 CC[0+bs*3] = alpha[0]*CC[0+bs*3];
10175 CC[1+bs*3] = alpha[0]*CC[1+bs*3];
10176 CC[2+bs*3] = alpha[0]*CC[2+bs*3];
10177 CC[3+bs*3] = alpha[0]*CC[3+bs*3];
10178
10179 // transpose
10180 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
10181 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
10182 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
10183
10184 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
10185 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
10186
10187 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
10188
10189 // store
10190 if(m1>=4)
10191 {
10192 D[0+ldd*0] = CC[0+bs*0];
10193 D[1+ldd*0] = CC[1+bs*0];
10194 D[2+ldd*0] = CC[2+bs*0];
10195 D[3+ldd*0] = CC[3+bs*0];
10196
10197 if(n1==1)
10198 return;
10199
10200 D[0+ldd*1] = CC[0+bs*1];
10201 D[1+ldd*1] = CC[1+bs*1];
10202 D[2+ldd*1] = CC[2+bs*1];
10203 D[3+ldd*1] = CC[3+bs*1];
10204
10205 if(n1==2)
10206 return;
10207
10208 D[0+ldd*2] = CC[0+bs*2];
10209 D[1+ldd*2] = CC[1+bs*2];
10210 D[2+ldd*2] = CC[2+bs*2];
10211 D[3+ldd*2] = CC[3+bs*2];
10212
10213 if(n1==3)
10214 return;
10215
10216 D[0+ldd*3] = CC[0+bs*3];
10217 D[1+ldd*3] = CC[1+bs*3];
10218 D[2+ldd*3] = CC[2+bs*3];
10219 D[3+ldd*3] = CC[3+bs*3];
10220 }
10221 else if(m1>=3)
10222 {
10223 D[0+ldd*0] = CC[0+bs*0];
10224 D[1+ldd*0] = CC[1+bs*0];
10225 D[2+ldd*0] = CC[2+bs*0];
10226
10227 if(n1==1)
10228 return;
10229
10230 D[0+ldd*1] = CC[0+bs*1];
10231 D[1+ldd*1] = CC[1+bs*1];
10232 D[2+ldd*1] = CC[2+bs*1];
10233
10234 if(n1==2)
10235 return;
10236
10237 D[0+ldd*2] = CC[0+bs*2];
10238 D[1+ldd*2] = CC[1+bs*2];
10239 D[2+ldd*2] = CC[2+bs*2];
10240
10241 if(n1==3)
10242 return;
10243
10244 D[0+ldd*3] = CC[0+bs*3];
10245 D[1+ldd*3] = CC[1+bs*3];
10246 D[2+ldd*3] = CC[2+bs*3];
10247 }
10248 else if(m1>=2)
10249 {
10250 D[0+ldd*0] = CC[0+bs*0];
10251 D[1+ldd*0] = CC[1+bs*0];
10252
10253 if(n1==1)
10254 return;
10255
10256 D[0+ldd*1] = CC[0+bs*1];
10257 D[1+ldd*1] = CC[1+bs*1];
10258
10259 if(n1==2)
10260 return;
10261
10262 D[0+ldd*2] = CC[0+bs*2];
10263 D[1+ldd*2] = CC[1+bs*2];
10264
10265 if(n1==3)
10266 return;
10267
10268 D[0+ldd*3] = CC[0+bs*3];
10269 D[1+ldd*3] = CC[1+bs*3];
10270 }
10271 else //if(m1>=1)
10272 {
10273 D[0+ldd*0] = CC[0+bs*0];
10274
10275 if(n1==1)
10276 return;
10277
10278 D[0+ldd*1] = CC[0+bs*1];
10279
10280 if(n1==2)
10281 return;
10282
10283 D[0+ldd*2] = CC[0+bs*2];
10284
10285 if(n1==3)
10286 return;
10287
10288 D[0+ldd*3] = CC[0+bs*3];
10289 }
10290
10291 return;
10292
10293 }
10294 //#endif
10295
10296
10297
10298 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_one_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)10299 void kernel_strmm_nt_rl_one_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
10300 {
10301
10302 const int bs = 4;
10303
10304 float
10305 a_0, a_1, a_2, a_3,
10306 b_0, b_1, b_2, b_3;
10307
10308 #if defined(TARGET_GENERIC)
10309 float CC[16] = {0};
10310 #else
10311 ALIGNED( float CC[16], 64 ) = {0};
10312 #endif
10313
10314 float alpha1 = 1.0;
10315
10316 // assume always kmax>=4 !!!
10317
10318 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
10319
10320 A += kmax*bs;
10321 B += kmax*ldb;
10322
10323 // k = 0
10324
10325 a_0 = A[0];
10326 a_1 = A[1];
10327 a_2 = A[2];
10328 a_3 = A[3];
10329
10330 CC[0+bs*0] += a_0;
10331 CC[1+bs*0] += a_1;
10332 CC[2+bs*0] += a_2;
10333 CC[3+bs*0] += a_3;
10334
10335 b_0 = B[1+0*ldb];
10336 CC[0+bs*1] += a_0 * b_0;
10337 CC[1+bs*1] += a_1 * b_0;
10338 CC[2+bs*1] += a_2 * b_0;
10339 CC[3+bs*1] += a_3 * b_0;
10340
10341 b_0 = B[2+0*ldb];
10342 CC[0+bs*2] += a_0 * b_0;
10343 CC[1+bs*2] += a_1 * b_0;
10344 CC[2+bs*2] += a_2 * b_0;
10345 CC[3+bs*2] += a_3 * b_0;
10346
10347 b_0 = B[3+0*ldb];
10348 CC[0+bs*3] += a_0 * b_0;
10349 CC[1+bs*3] += a_1 * b_0;
10350 CC[2+bs*3] += a_2 * b_0;
10351 CC[3+bs*3] += a_3 * b_0;
10352
10353 A += bs;
10354 B += ldb;
10355 // k += 1;
10356
10357 // k = 1
10358
10359 a_0 = A[0];
10360 a_1 = A[1];
10361 a_2 = A[2];
10362 a_3 = A[3];
10363
10364 CC[0+bs*1] += a_0;
10365 CC[1+bs*1] += a_1;
10366 CC[2+bs*1] += a_2;
10367 CC[3+bs*1] += a_3;
10368
10369 b_0 = B[2+0*ldb];
10370 CC[0+bs*2] += a_0 * b_0;
10371 CC[1+bs*2] += a_1 * b_0;
10372 CC[2+bs*2] += a_2 * b_0;
10373 CC[3+bs*2] += a_3 * b_0;
10374
10375 b_0 = B[3+0*ldb];
10376 CC[0+bs*3] += a_0 * b_0;
10377 CC[1+bs*3] += a_1 * b_0;
10378 CC[2+bs*3] += a_2 * b_0;
10379 CC[3+bs*3] += a_3 * b_0;
10380
10381 A += bs;
10382 B += ldb;
10383 // k += 1;
10384
10385 // k = 2
10386
10387 a_0 = A[0];
10388 a_1 = A[1];
10389 a_2 = A[2];
10390 a_3 = A[3];
10391
10392 CC[0+bs*2] += a_0;
10393 CC[1+bs*2] += a_1;
10394 CC[2+bs*2] += a_2;
10395 CC[3+bs*2] += a_3;
10396
10397 b_0 = B[3+0*ldb];
10398 CC[0+bs*3] += a_0 * b_0;
10399 CC[1+bs*3] += a_1 * b_0;
10400 CC[2+bs*3] += a_2 * b_0;
10401 CC[3+bs*3] += a_3 * b_0;
10402
10403 A += bs;
10404 B += ldb;
10405 // k += 1;
10406
10407 // k = 3
10408
10409 a_0 = A[0];
10410 a_1 = A[1];
10411 a_2 = A[2];
10412 a_3 = A[3];
10413
10414 CC[0+bs*3] += a_0;
10415 CC[1+bs*3] += a_1;
10416 CC[2+bs*3] += a_2;
10417 CC[3+bs*3] += a_3;
10418
10419 A += bs;
10420 B += ldb;
10421 // k += 1;
10422
10423 store:
10424
10425 D[0+ldd*0] = alpha[0]*CC[0+bs*0];
10426 D[1+ldd*0] = alpha[0]*CC[1+bs*0];
10427 D[2+ldd*0] = alpha[0]*CC[2+bs*0];
10428 D[3+ldd*0] = alpha[0]*CC[3+bs*0];
10429
10430 D[0+ldd*1] = alpha[0]*CC[0+bs*1];
10431 D[1+ldd*1] = alpha[0]*CC[1+bs*1];
10432 D[2+ldd*1] = alpha[0]*CC[2+bs*1];
10433 D[3+ldd*1] = alpha[0]*CC[3+bs*1];
10434
10435 D[0+ldd*2] = alpha[0]*CC[0+bs*2];
10436 D[1+ldd*2] = alpha[0]*CC[1+bs*2];
10437 D[2+ldd*2] = alpha[0]*CC[2+bs*2];
10438 D[3+ldd*2] = alpha[0]*CC[3+bs*2];
10439
10440 D[0+ldd*3] = alpha[0]*CC[0+bs*3];
10441 D[1+ldd*3] = alpha[0]*CC[1+bs*3];
10442 D[2+ldd*3] = alpha[0]*CC[2+bs*3];
10443 D[3+ldd*3] = alpha[0]*CC[3+bs*3];
10444
10445 return;
10446
10447 }
10448 //#endif
10449
10450
10451
10452 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_one_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)10453 void kernel_strmm_nt_rl_one_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
10454 {
10455
10456 const int bs = 4;
10457
10458 float
10459 a_0, a_1, a_2, a_3,
10460 b_0, b_1, b_2, b_3;
10461
10462 #if defined(TARGET_GENERIC)
10463 float CC[16] = {0};
10464 #else
10465 ALIGNED( float CC[16], 64 ) = {0};
10466 #endif
10467
10468 float alpha1 = 1.0;
10469
10470 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
10471
10472 A += kmax*bs;
10473 B += kmax*ldb;
10474
10475 if(n1>=4)
10476 {
10477
10478 // k = 0
10479
10480 a_0 = A[0];
10481 a_1 = A[1];
10482 a_2 = A[2];
10483 a_3 = A[3];
10484
10485 CC[0+bs*0] += a_0;
10486 CC[1+bs*0] += a_1;
10487 CC[2+bs*0] += a_2;
10488 CC[3+bs*0] += a_3;
10489
10490 b_0 = B[1+0*ldb];
10491 CC[0+bs*1] += a_0 * b_0;
10492 CC[1+bs*1] += a_1 * b_0;
10493 CC[2+bs*1] += a_2 * b_0;
10494 CC[3+bs*1] += a_3 * b_0;
10495
10496 b_0 = B[2+0*ldb];
10497 CC[0+bs*2] += a_0 * b_0;
10498 CC[1+bs*2] += a_1 * b_0;
10499 CC[2+bs*2] += a_2 * b_0;
10500 CC[3+bs*2] += a_3 * b_0;
10501
10502 b_0 = B[3+0*ldb];
10503 CC[0+bs*3] += a_0 * b_0;
10504 CC[1+bs*3] += a_1 * b_0;
10505 CC[2+bs*3] += a_2 * b_0;
10506 CC[3+bs*3] += a_3 * b_0;
10507
10508 A += bs;
10509 B += ldb;
10510 // k += 1;
10511
10512 // k = 1
10513
10514 a_0 = A[0];
10515 a_1 = A[1];
10516 a_2 = A[2];
10517 a_3 = A[3];
10518
10519 CC[0+bs*1] += a_0;
10520 CC[1+bs*1] += a_1;
10521 CC[2+bs*1] += a_2;
10522 CC[3+bs*1] += a_3;
10523
10524 b_0 = B[2+0*ldb];
10525 CC[0+bs*2] += a_0 * b_0;
10526 CC[1+bs*2] += a_1 * b_0;
10527 CC[2+bs*2] += a_2 * b_0;
10528 CC[3+bs*2] += a_3 * b_0;
10529
10530 b_0 = B[3+0*ldb];
10531 CC[0+bs*3] += a_0 * b_0;
10532 CC[1+bs*3] += a_1 * b_0;
10533 CC[2+bs*3] += a_2 * b_0;
10534 CC[3+bs*3] += a_3 * b_0;
10535
10536 A += bs;
10537 B += ldb;
10538 // k += 1;
10539
10540 // k = 2
10541
10542 a_0 = A[0];
10543 a_1 = A[1];
10544 a_2 = A[2];
10545 a_3 = A[3];
10546
10547 CC[0+bs*2] += a_0;
10548 CC[1+bs*2] += a_1;
10549 CC[2+bs*2] += a_2;
10550 CC[3+bs*2] += a_3;
10551
10552 b_0 = B[3+0*ldb];
10553 CC[0+bs*3] += a_0 * b_0;
10554 CC[1+bs*3] += a_1 * b_0;
10555 CC[2+bs*3] += a_2 * b_0;
10556 CC[3+bs*3] += a_3 * b_0;
10557
10558 A += bs;
10559 B += ldb;
10560 // k += 1;
10561
10562 // k = 3
10563
10564 a_0 = A[0];
10565 a_1 = A[1];
10566 a_2 = A[2];
10567 a_3 = A[3];
10568
10569 CC[0+bs*3] += a_0;
10570 CC[1+bs*3] += a_1;
10571 CC[2+bs*3] += a_2;
10572 CC[3+bs*3] += a_3;
10573
10574 A += bs;
10575 B += ldb;
10576 // k += 1;
10577
10578 }
10579 else if(n1==3)
10580 {
10581
10582 // k = 0
10583
10584 a_0 = A[0];
10585 a_1 = A[1];
10586 a_2 = A[2];
10587 a_3 = A[3];
10588
10589 CC[0+bs*0] += a_0;
10590 CC[1+bs*0] += a_1;
10591 CC[2+bs*0] += a_2;
10592 CC[3+bs*0] += a_3;
10593
10594 b_0 = B[1+0*ldb];
10595 CC[0+bs*1] += a_0 * b_0;
10596 CC[1+bs*1] += a_1 * b_0;
10597 CC[2+bs*1] += a_2 * b_0;
10598 CC[3+bs*1] += a_3 * b_0;
10599
10600 b_0 = B[2+0*ldb];
10601 CC[0+bs*2] += a_0 * b_0;
10602 CC[1+bs*2] += a_1 * b_0;
10603 CC[2+bs*2] += a_2 * b_0;
10604 CC[3+bs*2] += a_3 * b_0;
10605
10606 A += bs;
10607 B += ldb;
10608 // k += 1;
10609
10610 // k = 1
10611
10612 a_0 = A[0];
10613 a_1 = A[1];
10614 a_2 = A[2];
10615 a_3 = A[3];
10616
10617 CC[0+bs*1] += a_0;
10618 CC[1+bs*1] += a_1;
10619 CC[2+bs*1] += a_2;
10620 CC[3+bs*1] += a_3;
10621
10622 b_0 = B[2+0*ldb];
10623 CC[0+bs*2] += a_0 * b_0;
10624 CC[1+bs*2] += a_1 * b_0;
10625 CC[2+bs*2] += a_2 * b_0;
10626 CC[3+bs*2] += a_3 * b_0;
10627
10628 A += bs;
10629 B += ldb;
10630 // k += 1;
10631
10632 // k = 2
10633
10634 a_0 = A[0];
10635 a_1 = A[1];
10636 a_2 = A[2];
10637 a_3 = A[3];
10638
10639 CC[0+bs*2] += a_0;
10640 CC[1+bs*2] += a_1;
10641 CC[2+bs*2] += a_2;
10642 CC[3+bs*2] += a_3;
10643
10644 A += bs;
10645 B += ldb;
10646 // k += 1;
10647
10648 // k = 3
10649
10650 A += bs;
10651 B += ldb;
10652 // k += 1;
10653
10654 }
10655 else if(n1==2)
10656 {
10657
10658 // k = 0
10659
10660 a_0 = A[0];
10661 a_1 = A[1];
10662 a_2 = A[2];
10663 a_3 = A[3];
10664
10665 CC[0+bs*0] += a_0;
10666 CC[1+bs*0] += a_1;
10667 CC[2+bs*0] += a_2;
10668 CC[3+bs*0] += a_3;
10669
10670 b_0 = B[1+0*ldb];
10671 CC[0+bs*1] += a_0 * b_0;
10672 CC[1+bs*1] += a_1 * b_0;
10673 CC[2+bs*1] += a_2 * b_0;
10674 CC[3+bs*1] += a_3 * b_0;
10675
10676 A += bs;
10677 B += ldb;
10678 // k += 1;
10679
10680 // k = 1
10681
10682 a_0 = A[0];
10683 a_1 = A[1];
10684 a_2 = A[2];
10685 a_3 = A[3];
10686
10687 CC[0+bs*1] += a_0;
10688 CC[1+bs*1] += a_1;
10689 CC[2+bs*1] += a_2;
10690 CC[3+bs*1] += a_3;
10691
10692 A += bs;
10693 B += ldb;
10694 // k += 1;
10695
10696 // k = 2
10697
10698 A += bs;
10699 B += ldb;
10700 // k += 1;
10701
10702 // k = 3
10703
10704 A += bs;
10705 B += ldb;
10706 // k += 1;
10707
10708 }
10709 else if(n1==1)
10710 {
10711
10712 // k = 0
10713
10714 a_0 = A[0];
10715 a_1 = A[1];
10716 a_2 = A[2];
10717 a_3 = A[3];
10718
10719 CC[0+bs*0] += a_0;
10720 CC[1+bs*0] += a_1;
10721 CC[2+bs*0] += a_2;
10722 CC[3+bs*0] += a_3;
10723
10724 A += bs;
10725 B += ldb;
10726 // k += 1;
10727
10728 // k = 1
10729
10730 A += bs;
10731 B += ldb;
10732 // k += 1;
10733
10734 // k = 2
10735
10736 A += bs;
10737 B += ldb;
10738 // k += 1;
10739
10740 // k = 3
10741
10742 A += bs;
10743 B += ldb;
10744 // k += 1;
10745
10746 }
10747
10748 store:
10749
10750 CC[0+bs*0] = alpha[0]*CC[0+bs*0];
10751 CC[1+bs*0] = alpha[0]*CC[1+bs*0];
10752 CC[2+bs*0] = alpha[0]*CC[2+bs*0];
10753 CC[3+bs*0] = alpha[0]*CC[3+bs*0];
10754
10755 CC[0+bs*1] = alpha[0]*CC[0+bs*1];
10756 CC[1+bs*1] = alpha[0]*CC[1+bs*1];
10757 CC[2+bs*1] = alpha[0]*CC[2+bs*1];
10758 CC[3+bs*1] = alpha[0]*CC[3+bs*1];
10759
10760 CC[0+bs*2] = alpha[0]*CC[0+bs*2];
10761 CC[1+bs*2] = alpha[0]*CC[1+bs*2];
10762 CC[2+bs*2] = alpha[0]*CC[2+bs*2];
10763 CC[3+bs*2] = alpha[0]*CC[3+bs*2];
10764
10765 CC[0+bs*3] = alpha[0]*CC[0+bs*3];
10766 CC[1+bs*3] = alpha[0]*CC[1+bs*3];
10767 CC[2+bs*3] = alpha[0]*CC[2+bs*3];
10768 CC[3+bs*3] = alpha[0]*CC[3+bs*3];
10769
10770 if(m1>=4)
10771 {
10772 D[0+ldd*0] = CC[0+bs*0];
10773 D[1+ldd*0] = CC[1+bs*0];
10774 D[2+ldd*0] = CC[2+bs*0];
10775 D[3+ldd*0] = CC[3+bs*0];
10776
10777 if(n1==1)
10778 return;
10779
10780 D[0+ldd*1] = CC[0+bs*1];
10781 D[1+ldd*1] = CC[1+bs*1];
10782 D[2+ldd*1] = CC[2+bs*1];
10783 D[3+ldd*1] = CC[3+bs*1];
10784
10785 if(n1==2)
10786 return;
10787
10788 D[0+ldd*2] = CC[0+bs*2];
10789 D[1+ldd*2] = CC[1+bs*2];
10790 D[2+ldd*2] = CC[2+bs*2];
10791 D[3+ldd*2] = CC[3+bs*2];
10792
10793 if(n1==3)
10794 return;
10795
10796 D[0+ldd*3] = CC[0+bs*3];
10797 D[1+ldd*3] = CC[1+bs*3];
10798 D[2+ldd*3] = CC[2+bs*3];
10799 D[3+ldd*3] = CC[3+bs*3];
10800 }
10801 else if(m1>=3)
10802 {
10803 D[0+ldd*0] = CC[0+bs*0];
10804 D[1+ldd*0] = CC[1+bs*0];
10805 D[2+ldd*0] = CC[2+bs*0];
10806
10807 if(n1==1)
10808 return;
10809
10810 D[0+ldd*1] = CC[0+bs*1];
10811 D[1+ldd*1] = CC[1+bs*1];
10812 D[2+ldd*1] = CC[2+bs*1];
10813
10814 if(n1==2)
10815 return;
10816
10817 D[0+ldd*2] = CC[0+bs*2];
10818 D[1+ldd*2] = CC[1+bs*2];
10819 D[2+ldd*2] = CC[2+bs*2];
10820
10821 if(n1==3)
10822 return;
10823
10824 D[0+ldd*3] = CC[0+bs*3];
10825 D[1+ldd*3] = CC[1+bs*3];
10826 D[2+ldd*3] = CC[2+bs*3];
10827 }
10828 else if(m1>=2)
10829 {
10830 D[0+ldd*0] = CC[0+bs*0];
10831 D[1+ldd*0] = CC[1+bs*0];
10832
10833 if(n1==1)
10834 return;
10835
10836 D[0+ldd*1] = CC[0+bs*1];
10837 D[1+ldd*1] = CC[1+bs*1];
10838
10839 if(n1==2)
10840 return;
10841
10842 D[0+ldd*2] = CC[0+bs*2];
10843 D[1+ldd*2] = CC[1+bs*2];
10844
10845 if(n1==3)
10846 return;
10847
10848 D[0+ldd*3] = CC[0+bs*3];
10849 D[1+ldd*3] = CC[1+bs*3];
10850 }
10851 else //if(m1>=1)
10852 {
10853 D[0+ldd*0] = CC[0+bs*0];
10854
10855 if(n1==1)
10856 return;
10857
10858 D[0+ldd*1] = CC[0+bs*1];
10859
10860 if(n1==2)
10861 return;
10862
10863 D[0+ldd*2] = CC[0+bs*2];
10864
10865 if(n1==3)
10866 return;
10867
10868 D[0+ldd*3] = CC[0+bs*3];
10869 }
10870
10871 return;
10872
10873 }
10874 //#endif
10875
10876
10877
10878 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_one_4x4_tran_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd)10879 void kernel_strmm_nt_rl_one_4x4_tran_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd)
10880 {
10881
10882 const int bs = 4;
10883
10884 float
10885 tmp,
10886 a_0, a_1, a_2, a_3,
10887 b_0, b_1, b_2, b_3;
10888
10889 #if defined(TARGET_GENERIC)
10890 float CC[16] = {0};
10891 #else
10892 ALIGNED( float CC[16], 64 ) = {0};
10893 #endif
10894
10895 float alpha1 = 1.0;
10896
10897 // assume always kmax>=4 !!!
10898
10899 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
10900
10901 A += kmax*bs;
10902 B += kmax*ldb;
10903
10904 // k = 0
10905
10906 a_0 = A[0];
10907 a_1 = A[1];
10908 a_2 = A[2];
10909 a_3 = A[3];
10910
10911 CC[0+bs*0] += a_0;
10912 CC[1+bs*0] += a_1;
10913 CC[2+bs*0] += a_2;
10914 CC[3+bs*0] += a_3;
10915
10916 b_0 = B[1+0*ldb];
10917 CC[0+bs*1] += a_0 * b_0;
10918 CC[1+bs*1] += a_1 * b_0;
10919 CC[2+bs*1] += a_2 * b_0;
10920 CC[3+bs*1] += a_3 * b_0;
10921
10922 b_0 = B[2+0*ldb];
10923 CC[0+bs*2] += a_0 * b_0;
10924 CC[1+bs*2] += a_1 * b_0;
10925 CC[2+bs*2] += a_2 * b_0;
10926 CC[3+bs*2] += a_3 * b_0;
10927
10928 b_0 = B[3+0*ldb];
10929 CC[0+bs*3] += a_0 * b_0;
10930 CC[1+bs*3] += a_1 * b_0;
10931 CC[2+bs*3] += a_2 * b_0;
10932 CC[3+bs*3] += a_3 * b_0;
10933
10934 A += bs;
10935 B += ldb;
10936 // k += 1;
10937
10938 // k = 1
10939
10940 a_0 = A[0];
10941 a_1 = A[1];
10942 a_2 = A[2];
10943 a_3 = A[3];
10944
10945 CC[0+bs*1] += a_0;
10946 CC[1+bs*1] += a_1;
10947 CC[2+bs*1] += a_2;
10948 CC[3+bs*1] += a_3;
10949
10950 b_0 = B[2+0*ldb];
10951 CC[0+bs*2] += a_0 * b_0;
10952 CC[1+bs*2] += a_1 * b_0;
10953 CC[2+bs*2] += a_2 * b_0;
10954 CC[3+bs*2] += a_3 * b_0;
10955
10956 b_0 = B[3+0*ldb];
10957 CC[0+bs*3] += a_0 * b_0;
10958 CC[1+bs*3] += a_1 * b_0;
10959 CC[2+bs*3] += a_2 * b_0;
10960 CC[3+bs*3] += a_3 * b_0;
10961
10962 A += bs;
10963 B += ldb;
10964 // k += 1;
10965
10966 // k = 2
10967
10968 a_0 = A[0];
10969 a_1 = A[1];
10970 a_2 = A[2];
10971 a_3 = A[3];
10972
10973 CC[0+bs*2] += a_0;
10974 CC[1+bs*2] += a_1;
10975 CC[2+bs*2] += a_2;
10976 CC[3+bs*2] += a_3;
10977
10978 b_0 = B[3+0*ldb];
10979 CC[0+bs*3] += a_0 * b_0;
10980 CC[1+bs*3] += a_1 * b_0;
10981 CC[2+bs*3] += a_2 * b_0;
10982 CC[3+bs*3] += a_3 * b_0;
10983
10984 A += bs;
10985 B += ldb;
10986 // k += 1;
10987
10988 // k = 3
10989
10990 a_0 = A[0];
10991 a_1 = A[1];
10992 a_2 = A[2];
10993 a_3 = A[3];
10994
10995 CC[0+bs*3] += a_0;
10996 CC[1+bs*3] += a_1;
10997 CC[2+bs*3] += a_2;
10998 CC[3+bs*3] += a_3;
10999
11000 A += bs;
11001 B += ldb;
11002 // k += 1;
11003
11004 store:
11005
11006 // scale & tranpose & store
11007 D[0+ldd*0] = alpha[0]*CC[0+bs*0];
11008 D[0+ldd*1] = alpha[0]*CC[1+bs*0];
11009 D[0+ldd*2] = alpha[0]*CC[2+bs*0];
11010 D[0+ldd*3] = alpha[0]*CC[3+bs*0];
11011
11012 D[1+ldd*0] = alpha[0]*CC[0+bs*1];
11013 D[1+ldd*1] = alpha[0]*CC[1+bs*1];
11014 D[1+ldd*2] = alpha[0]*CC[2+bs*1];
11015 D[1+ldd*3] = alpha[0]*CC[3+bs*1];
11016
11017 D[2+ldd*0] = alpha[0]*CC[0+bs*2];
11018 D[2+ldd*1] = alpha[0]*CC[1+bs*2];
11019 D[2+ldd*2] = alpha[0]*CC[2+bs*2];
11020 D[2+ldd*3] = alpha[0]*CC[3+bs*2];
11021
11022 D[3+ldd*0] = alpha[0]*CC[0+bs*3];
11023 D[3+ldd*1] = alpha[0]*CC[1+bs*3];
11024 D[3+ldd*2] = alpha[0]*CC[2+bs*3];
11025 D[3+ldd*3] = alpha[0]*CC[3+bs*3];
11026
11027 return;
11028
11029 }
11030 //#endif
11031
11032
11033
11034 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_rl_one_4x4_tran_vs_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd,int m1,int n1)11035 void kernel_strmm_nt_rl_one_4x4_tran_vs_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd, int m1, int n1)
11036 {
11037
11038 const int bs = 4;
11039
11040 float
11041 tmp,
11042 a_0, a_1, a_2, a_3,
11043 b_0, b_1, b_2, b_3;
11044
11045 #if defined(TARGET_GENERIC)
11046 float CC[16] = {0};
11047 #else
11048 ALIGNED( float CC[16], 64 ) = {0};
11049 #endif
11050
11051 float alpha1 = 1.0;
11052
11053 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
11054
11055 A += kmax*bs;
11056 B += kmax*ldb;
11057
11058 // XXX m1 and n1 are swapped !!!!!
11059 if(m1>=4)
11060 {
11061
11062 // k = 0
11063
11064 a_0 = A[0];
11065 a_1 = A[1];
11066 a_2 = A[2];
11067 a_3 = A[3];
11068
11069 CC[0+bs*0] += a_0;
11070 CC[1+bs*0] += a_1;
11071 CC[2+bs*0] += a_2;
11072 CC[3+bs*0] += a_3;
11073
11074 b_0 = B[1+0*ldb];
11075 CC[0+bs*1] += a_0 * b_0;
11076 CC[1+bs*1] += a_1 * b_0;
11077 CC[2+bs*1] += a_2 * b_0;
11078 CC[3+bs*1] += a_3 * b_0;
11079
11080 b_0 = B[2+0*ldb];
11081 CC[0+bs*2] += a_0 * b_0;
11082 CC[1+bs*2] += a_1 * b_0;
11083 CC[2+bs*2] += a_2 * b_0;
11084 CC[3+bs*2] += a_3 * b_0;
11085
11086 b_0 = B[3+0*ldb];
11087 CC[0+bs*3] += a_0 * b_0;
11088 CC[1+bs*3] += a_1 * b_0;
11089 CC[2+bs*3] += a_2 * b_0;
11090 CC[3+bs*3] += a_3 * b_0;
11091
11092 A += bs;
11093 B += ldb;
11094 // k += 1;
11095
11096 // k = 1
11097
11098 a_0 = A[0];
11099 a_1 = A[1];
11100 a_2 = A[2];
11101 a_3 = A[3];
11102
11103 CC[0+bs*1] += a_0;
11104 CC[1+bs*1] += a_1;
11105 CC[2+bs*1] += a_2;
11106 CC[3+bs*1] += a_3;
11107
11108 b_0 = B[2+0*ldb];
11109 CC[0+bs*2] += a_0 * b_0;
11110 CC[1+bs*2] += a_1 * b_0;
11111 CC[2+bs*2] += a_2 * b_0;
11112 CC[3+bs*2] += a_3 * b_0;
11113
11114 b_0 = B[3+0*ldb];
11115 CC[0+bs*3] += a_0 * b_0;
11116 CC[1+bs*3] += a_1 * b_0;
11117 CC[2+bs*3] += a_2 * b_0;
11118 CC[3+bs*3] += a_3 * b_0;
11119
11120 A += bs;
11121 B += ldb;
11122 // k += 1;
11123
11124 // k = 2
11125
11126 a_0 = A[0];
11127 a_1 = A[1];
11128 a_2 = A[2];
11129 a_3 = A[3];
11130
11131 CC[0+bs*2] += a_0;
11132 CC[1+bs*2] += a_1;
11133 CC[2+bs*2] += a_2;
11134 CC[3+bs*2] += a_3;
11135
11136 b_0 = B[3+0*ldb];
11137 CC[0+bs*3] += a_0 * b_0;
11138 CC[1+bs*3] += a_1 * b_0;
11139 CC[2+bs*3] += a_2 * b_0;
11140 CC[3+bs*3] += a_3 * b_0;
11141
11142 A += bs;
11143 B += ldb;
11144 // k += 1;
11145
11146 // k = 3
11147
11148 a_0 = A[0];
11149 a_1 = A[1];
11150 a_2 = A[2];
11151 a_3 = A[3];
11152
11153 CC[0+bs*3] += a_0;
11154 CC[1+bs*3] += a_1;
11155 CC[2+bs*3] += a_2;
11156 CC[3+bs*3] += a_3;
11157
11158 A += bs;
11159 B += ldb;
11160 // k += 1;
11161
11162 }
11163 else if(m1==3)
11164 {
11165
11166 // k = 0
11167
11168 a_0 = A[0];
11169 a_1 = A[1];
11170 a_2 = A[2];
11171 a_3 = A[3];
11172
11173 CC[0+bs*0] += a_0;
11174 CC[1+bs*0] += a_1;
11175 CC[2+bs*0] += a_2;
11176 CC[3+bs*0] += a_3;
11177
11178 b_0 = B[1+0*ldb];
11179 CC[0+bs*1] += a_0 * b_0;
11180 CC[1+bs*1] += a_1 * b_0;
11181 CC[2+bs*1] += a_2 * b_0;
11182 CC[3+bs*1] += a_3 * b_0;
11183
11184 b_0 = B[2+0*ldb];
11185 CC[0+bs*2] += a_0 * b_0;
11186 CC[1+bs*2] += a_1 * b_0;
11187 CC[2+bs*2] += a_2 * b_0;
11188 CC[3+bs*2] += a_3 * b_0;
11189
11190 A += bs;
11191 B += ldb;
11192 // k += 1;
11193
11194 // k = 1
11195
11196 a_0 = A[0];
11197 a_1 = A[1];
11198 a_2 = A[2];
11199 a_3 = A[3];
11200
11201 CC[0+bs*1] += a_0;
11202 CC[1+bs*1] += a_1;
11203 CC[2+bs*1] += a_2;
11204 CC[3+bs*1] += a_3;
11205
11206 b_0 = B[2+0*ldb];
11207 CC[0+bs*2] += a_0 * b_0;
11208 CC[1+bs*2] += a_1 * b_0;
11209 CC[2+bs*2] += a_2 * b_0;
11210 CC[3+bs*2] += a_3 * b_0;
11211
11212 A += bs;
11213 B += ldb;
11214 // k += 1;
11215
11216 // k = 2
11217
11218 a_0 = A[0];
11219 a_1 = A[1];
11220 a_2 = A[2];
11221 a_3 = A[3];
11222
11223 CC[0+bs*2] += a_0;
11224 CC[1+bs*2] += a_1;
11225 CC[2+bs*2] += a_2;
11226 CC[3+bs*2] += a_3;
11227
11228 A += bs;
11229 B += ldb;
11230 // k += 1;
11231
11232 // k = 3
11233
11234 A += bs;
11235 B += ldb;
11236 // k += 1;
11237
11238 }
11239 else if(m1==2)
11240 {
11241
11242 // k = 0
11243
11244 a_0 = A[0];
11245 a_1 = A[1];
11246 a_2 = A[2];
11247 a_3 = A[3];
11248
11249 CC[0+bs*0] += a_0;
11250 CC[1+bs*0] += a_1;
11251 CC[2+bs*0] += a_2;
11252 CC[3+bs*0] += a_3;
11253
11254 b_0 = B[1+0*ldb];
11255 CC[0+bs*1] += a_0 * b_0;
11256 CC[1+bs*1] += a_1 * b_0;
11257 CC[2+bs*1] += a_2 * b_0;
11258 CC[3+bs*1] += a_3 * b_0;
11259
11260 A += bs;
11261 B += ldb;
11262 // k += 1;
11263
11264 // k = 1
11265
11266 a_0 = A[0];
11267 a_1 = A[1];
11268 a_2 = A[2];
11269 a_3 = A[3];
11270
11271 CC[0+bs*1] += a_0;
11272 CC[1+bs*1] += a_1;
11273 CC[2+bs*1] += a_2;
11274 CC[3+bs*1] += a_3;
11275
11276 A += bs;
11277 B += ldb;
11278 // k += 1;
11279
11280 // k = 2
11281
11282 A += bs;
11283 B += ldb;
11284 // k += 1;
11285
11286 // k = 3
11287
11288 A += bs;
11289 B += ldb;
11290 // k += 1;
11291
11292 }
11293 else if(m1==1)
11294 {
11295
11296 // k = 0
11297
11298 a_0 = A[0];
11299 a_1 = A[1];
11300 a_2 = A[2];
11301 a_3 = A[3];
11302
11303 CC[0+bs*0] += a_0;
11304 CC[1+bs*0] += a_1;
11305 CC[2+bs*0] += a_2;
11306 CC[3+bs*0] += a_3;
11307
11308 A += bs;
11309 B += ldb;
11310 // k += 1;
11311
11312 // k = 1
11313
11314 A += bs;
11315 B += ldb;
11316 // k += 1;
11317
11318 // k = 2
11319
11320 A += bs;
11321 B += ldb;
11322 // k += 1;
11323
11324 // k = 3
11325
11326 A += bs;
11327 B += ldb;
11328 // k += 1;
11329
11330 }
11331
11332 store:
11333
11334 // scale
11335 CC[0+bs*0] = alpha[0]*CC[0+bs*0];
11336 CC[1+bs*0] = alpha[0]*CC[1+bs*0];
11337 CC[2+bs*0] = alpha[0]*CC[2+bs*0];
11338 CC[3+bs*0] = alpha[0]*CC[3+bs*0];
11339
11340 CC[0+bs*1] = alpha[0]*CC[0+bs*1];
11341 CC[1+bs*1] = alpha[0]*CC[1+bs*1];
11342 CC[2+bs*1] = alpha[0]*CC[2+bs*1];
11343 CC[3+bs*1] = alpha[0]*CC[3+bs*1];
11344
11345 CC[0+bs*2] = alpha[0]*CC[0+bs*2];
11346 CC[1+bs*2] = alpha[0]*CC[1+bs*2];
11347 CC[2+bs*2] = alpha[0]*CC[2+bs*2];
11348 CC[3+bs*2] = alpha[0]*CC[3+bs*2];
11349
11350 CC[0+bs*3] = alpha[0]*CC[0+bs*3];
11351 CC[1+bs*3] = alpha[0]*CC[1+bs*3];
11352 CC[2+bs*3] = alpha[0]*CC[2+bs*3];
11353 CC[3+bs*3] = alpha[0]*CC[3+bs*3];
11354
11355 // transpose
11356 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
11357 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
11358 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
11359
11360 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
11361 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
11362
11363 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
11364
11365 // store
11366 if(m1>=4)
11367 {
11368 D[0+ldd*0] = CC[0+bs*0];
11369 D[1+ldd*0] = CC[1+bs*0];
11370 D[2+ldd*0] = CC[2+bs*0];
11371 D[3+ldd*0] = CC[3+bs*0];
11372
11373 if(n1==1)
11374 return;
11375
11376 D[0+ldd*1] = CC[0+bs*1];
11377 D[1+ldd*1] = CC[1+bs*1];
11378 D[2+ldd*1] = CC[2+bs*1];
11379 D[3+ldd*1] = CC[3+bs*1];
11380
11381 if(n1==2)
11382 return;
11383
11384 D[0+ldd*2] = CC[0+bs*2];
11385 D[1+ldd*2] = CC[1+bs*2];
11386 D[2+ldd*2] = CC[2+bs*2];
11387 D[3+ldd*2] = CC[3+bs*2];
11388
11389 if(n1==3)
11390 return;
11391
11392 D[0+ldd*3] = CC[0+bs*3];
11393 D[1+ldd*3] = CC[1+bs*3];
11394 D[2+ldd*3] = CC[2+bs*3];
11395 D[3+ldd*3] = CC[3+bs*3];
11396 }
11397 else if(m1>=3)
11398 {
11399 D[0+ldd*0] = CC[0+bs*0];
11400 D[1+ldd*0] = CC[1+bs*0];
11401 D[2+ldd*0] = CC[2+bs*0];
11402
11403 if(n1==1)
11404 return;
11405
11406 D[0+ldd*1] = CC[0+bs*1];
11407 D[1+ldd*1] = CC[1+bs*1];
11408 D[2+ldd*1] = CC[2+bs*1];
11409
11410 if(n1==2)
11411 return;
11412
11413 D[0+ldd*2] = CC[0+bs*2];
11414 D[1+ldd*2] = CC[1+bs*2];
11415 D[2+ldd*2] = CC[2+bs*2];
11416
11417 if(n1==3)
11418 return;
11419
11420 D[0+ldd*3] = CC[0+bs*3];
11421 D[1+ldd*3] = CC[1+bs*3];
11422 D[2+ldd*3] = CC[2+bs*3];
11423 }
11424 else if(m1>=2)
11425 {
11426 D[0+ldd*0] = CC[0+bs*0];
11427 D[1+ldd*0] = CC[1+bs*0];
11428
11429 if(n1==1)
11430 return;
11431
11432 D[0+ldd*1] = CC[0+bs*1];
11433 D[1+ldd*1] = CC[1+bs*1];
11434
11435 if(n1==2)
11436 return;
11437
11438 D[0+ldd*2] = CC[0+bs*2];
11439 D[1+ldd*2] = CC[1+bs*2];
11440
11441 if(n1==3)
11442 return;
11443
11444 D[0+ldd*3] = CC[0+bs*3];
11445 D[1+ldd*3] = CC[1+bs*3];
11446 }
11447 else //if(m1>=1)
11448 {
11449 D[0+ldd*0] = CC[0+bs*0];
11450
11451 if(n1==1)
11452 return;
11453
11454 D[0+ldd*1] = CC[0+bs*1];
11455
11456 if(n1==2)
11457 return;
11458
11459 D[0+ldd*2] = CC[0+bs*2];
11460
11461 if(n1==3)
11462 return;
11463
11464 D[0+ldd*3] = CC[0+bs*3];
11465 }
11466
11467 return;
11468
11469 }
11470 //#endif
11471
11472
11473
11474 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_4x4_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd)11475 void kernel_strmm_nt_ru_4x4_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd)
11476 {
11477
11478 const int bs = 4;
11479
11480 float
11481 a_0, a_1, a_2, a_3,
11482 b_0, b_1, b_2, b_3;
11483
11484 #if defined(TARGET_GENERIC)
11485 float CC[16] = {0};
11486 #else
11487 ALIGNED( float CC[16], 64 ) = {0};
11488 #endif
11489
11490 int k;
11491
11492 k = 0;
11493
11494 // k = 0
11495
11496 a_0 = A[0];
11497 a_1 = A[1];
11498 a_2 = A[2];
11499 a_3 = A[3];
11500
11501 b_0 = B[0+0*bs];
11502 CC[0+bs*0] += a_0 * b_0;
11503 CC[1+bs*0] += a_1 * b_0;
11504 CC[2+bs*0] += a_2 * b_0;
11505 CC[3+bs*0] += a_3 * b_0;
11506
11507 A += bs;
11508 B += bs;
11509 k += 1;
11510
11511 if(k>=kmax)
11512 goto store;
11513
11514 // k = 1
11515
11516 a_0 = A[0];
11517 a_1 = A[1];
11518 a_2 = A[2];
11519 a_3 = A[3];
11520
11521 b_0 = B[0+0*bs];
11522 CC[0+bs*0] += a_0 * b_0;
11523 CC[1+bs*0] += a_1 * b_0;
11524 CC[2+bs*0] += a_2 * b_0;
11525 CC[3+bs*0] += a_3 * b_0;
11526
11527 b_1 = B[1+0*bs];
11528 CC[0+bs*1] += a_0 * b_1;
11529 CC[1+bs*1] += a_1 * b_1;
11530 CC[2+bs*1] += a_2 * b_1;
11531 CC[3+bs*1] += a_3 * b_1;
11532
11533 A += bs;
11534 B += bs;
11535 k += 1;
11536
11537 if(k>=kmax)
11538 goto store;
11539
11540 // k = 2
11541
11542 a_0 = A[0];
11543 a_1 = A[1];
11544 a_2 = A[2];
11545 a_3 = A[3];
11546
11547 b_0 = B[0+0*bs];
11548 CC[0+bs*0] += a_0 * b_0;
11549 CC[1+bs*0] += a_1 * b_0;
11550 CC[2+bs*0] += a_2 * b_0;
11551 CC[3+bs*0] += a_3 * b_0;
11552
11553 b_1 = B[1+0*bs];
11554 CC[0+bs*1] += a_0 * b_1;
11555 CC[1+bs*1] += a_1 * b_1;
11556 CC[2+bs*1] += a_2 * b_1;
11557 CC[3+bs*1] += a_3 * b_1;
11558
11559 b_2 = B[2+0*bs];
11560 CC[0+bs*2] += a_0 * b_2;
11561 CC[1+bs*2] += a_1 * b_2;
11562 CC[2+bs*2] += a_2 * b_2;
11563 CC[3+bs*2] += a_3 * b_2;
11564
11565 A += bs;
11566 B += bs;
11567 k += 1;
11568
11569 if(k>=kmax)
11570 goto store;
11571
11572 // k = 3
11573
11574 a_0 = A[0];
11575 a_1 = A[1];
11576 a_2 = A[2];
11577 a_3 = A[3];
11578
11579 b_0 = B[0+0*bs];
11580 CC[0+bs*0] += a_0 * b_0;
11581 CC[1+bs*0] += a_1 * b_0;
11582 CC[2+bs*0] += a_2 * b_0;
11583 CC[3+bs*0] += a_3 * b_0;
11584
11585 b_1 = B[1+0*bs];
11586 CC[0+bs*1] += a_0 * b_1;
11587 CC[1+bs*1] += a_1 * b_1;
11588 CC[2+bs*1] += a_2 * b_1;
11589 CC[3+bs*1] += a_3 * b_1;
11590
11591 b_2 = B[2+0*bs];
11592 CC[0+bs*2] += a_0 * b_2;
11593 CC[1+bs*2] += a_1 * b_2;
11594 CC[2+bs*2] += a_2 * b_2;
11595 CC[3+bs*2] += a_3 * b_2;
11596
11597 b_3 = B[3+0*bs];
11598 CC[0+bs*3] += a_0 * b_3;
11599 CC[1+bs*3] += a_1 * b_3;
11600 CC[2+bs*3] += a_2 * b_3;
11601 CC[3+bs*3] += a_3 * b_3;
11602
11603 A += bs;
11604 B += bs;
11605 k += 1;
11606
11607 store:
11608
11609 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
11610 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
11611 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
11612 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
11613
11614 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
11615 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
11616 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
11617 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
11618
11619 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
11620 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
11621 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
11622 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
11623
11624 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
11625 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
11626 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
11627 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
11628
11629 float beta1 = 1.0;
11630
11631 kernel_sgemm_nt_4x4_lib4(kmax-k, alpha, A, B, &beta1, CC, CC);
11632
11633 D[0+ldd*0] = CC[0+bs*0];
11634 D[1+ldd*0] = CC[1+bs*0];
11635 D[2+ldd*0] = CC[2+bs*0];
11636 D[3+ldd*0] = CC[3+bs*0];
11637
11638 D[0+ldd*1] = CC[0+bs*1];
11639 D[1+ldd*1] = CC[1+bs*1];
11640 D[2+ldd*1] = CC[2+bs*1];
11641 D[3+ldd*1] = CC[3+bs*1];
11642
11643 D[0+ldd*2] = CC[0+bs*2];
11644 D[1+ldd*2] = CC[1+bs*2];
11645 D[2+ldd*2] = CC[2+bs*2];
11646 D[3+ldd*2] = CC[3+bs*2];
11647
11648 D[0+ldd*3] = CC[0+bs*3];
11649 D[1+ldd*3] = CC[1+bs*3];
11650 D[2+ldd*3] = CC[2+bs*3];
11651 D[3+ldd*3] = CC[3+bs*3];
11652
11653 return;
11654
11655 }
11656 //#endif
11657
11658
11659
11660 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_4x4_vs_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)11661 void kernel_strmm_nt_ru_4x4_vs_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
11662 {
11663
11664 const int bs = 4;
11665
11666 float
11667 a_0, a_1, a_2, a_3,
11668 b_0, b_1, b_2, b_3;
11669
11670 #if defined(TARGET_GENERIC)
11671 float CC[16] = {0};
11672 #else
11673 ALIGNED( float CC[16], 64 ) = {0};
11674 #endif
11675
11676 int k;
11677
11678 k = 0;
11679
11680 // k = 0
11681
11682 a_0 = A[0];
11683 a_1 = A[1];
11684 a_2 = A[2];
11685 a_3 = A[3];
11686
11687 b_0 = B[0+0*bs];
11688 CC[0+bs*0] += a_0 * b_0;
11689 CC[1+bs*0] += a_1 * b_0;
11690 CC[2+bs*0] += a_2 * b_0;
11691 CC[3+bs*0] += a_3 * b_0;
11692
11693 A += bs;
11694 B += bs;
11695 k += 1;
11696
11697 if(k>=kmax)
11698 goto store;
11699
11700 // k = 1
11701
11702 a_0 = A[0];
11703 a_1 = A[1];
11704 a_2 = A[2];
11705 a_3 = A[3];
11706
11707 b_0 = B[0+0*bs];
11708 CC[0+bs*0] += a_0 * b_0;
11709 CC[1+bs*0] += a_1 * b_0;
11710 CC[2+bs*0] += a_2 * b_0;
11711 CC[3+bs*0] += a_3 * b_0;
11712
11713 b_1 = B[1+0*bs];
11714 CC[0+bs*1] += a_0 * b_1;
11715 CC[1+bs*1] += a_1 * b_1;
11716 CC[2+bs*1] += a_2 * b_1;
11717 CC[3+bs*1] += a_3 * b_1;
11718
11719 A += bs;
11720 B += bs;
11721 k += 1;
11722
11723 if(k>=kmax)
11724 goto store;
11725
11726 // k = 2
11727
11728 a_0 = A[0];
11729 a_1 = A[1];
11730 a_2 = A[2];
11731 a_3 = A[3];
11732
11733 b_0 = B[0+0*bs];
11734 CC[0+bs*0] += a_0 * b_0;
11735 CC[1+bs*0] += a_1 * b_0;
11736 CC[2+bs*0] += a_2 * b_0;
11737 CC[3+bs*0] += a_3 * b_0;
11738
11739 b_1 = B[1+0*bs];
11740 CC[0+bs*1] += a_0 * b_1;
11741 CC[1+bs*1] += a_1 * b_1;
11742 CC[2+bs*1] += a_2 * b_1;
11743 CC[3+bs*1] += a_3 * b_1;
11744
11745 b_2 = B[2+0*bs];
11746 CC[0+bs*2] += a_0 * b_2;
11747 CC[1+bs*2] += a_1 * b_2;
11748 CC[2+bs*2] += a_2 * b_2;
11749 CC[3+bs*2] += a_3 * b_2;
11750
11751 A += bs;
11752 B += bs;
11753 k += 1;
11754
11755 if(k>=kmax)
11756 goto store;
11757
11758 // k = 3
11759
11760 a_0 = A[0];
11761 a_1 = A[1];
11762 a_2 = A[2];
11763 a_3 = A[3];
11764
11765 b_0 = B[0+0*bs];
11766 CC[0+bs*0] += a_0 * b_0;
11767 CC[1+bs*0] += a_1 * b_0;
11768 CC[2+bs*0] += a_2 * b_0;
11769 CC[3+bs*0] += a_3 * b_0;
11770
11771 b_1 = B[1+0*bs];
11772 CC[0+bs*1] += a_0 * b_1;
11773 CC[1+bs*1] += a_1 * b_1;
11774 CC[2+bs*1] += a_2 * b_1;
11775 CC[3+bs*1] += a_3 * b_1;
11776
11777 b_2 = B[2+0*bs];
11778 CC[0+bs*2] += a_0 * b_2;
11779 CC[1+bs*2] += a_1 * b_2;
11780 CC[2+bs*2] += a_2 * b_2;
11781 CC[3+bs*2] += a_3 * b_2;
11782
11783 b_3 = B[3+0*bs];
11784 CC[0+bs*3] += a_0 * b_3;
11785 CC[1+bs*3] += a_1 * b_3;
11786 CC[2+bs*3] += a_2 * b_3;
11787 CC[3+bs*3] += a_3 * b_3;
11788
11789 A += bs;
11790 B += bs;
11791 k += 1;
11792
11793 store:
11794
11795 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
11796 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
11797 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
11798 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
11799
11800 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
11801 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
11802 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
11803 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
11804
11805 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
11806 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
11807 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
11808 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
11809
11810 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
11811 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
11812 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
11813 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
11814
11815 float beta1 = 1.0;
11816
11817 kernel_sgemm_nt_4x4_lib4(kmax-k, alpha, A, B, &beta1, CC, CC);
11818
11819 if(m1>=4)
11820 {
11821 D[0+ldd*0] = CC[0+bs*0];
11822 D[1+ldd*0] = CC[1+bs*0];
11823 D[2+ldd*0] = CC[2+bs*0];
11824 D[3+ldd*0] = CC[3+bs*0];
11825
11826 if(n1==1)
11827 return;
11828
11829 D[0+ldd*1] = CC[0+bs*1];
11830 D[1+ldd*1] = CC[1+bs*1];
11831 D[2+ldd*1] = CC[2+bs*1];
11832 D[3+ldd*1] = CC[3+bs*1];
11833
11834 if(n1==2)
11835 return;
11836
11837 D[0+ldd*2] = CC[0+bs*2];
11838 D[1+ldd*2] = CC[1+bs*2];
11839 D[2+ldd*2] = CC[2+bs*2];
11840 D[3+ldd*2] = CC[3+bs*2];
11841
11842 if(n1==3)
11843 return;
11844
11845 D[0+ldd*3] = CC[0+bs*3];
11846 D[1+ldd*3] = CC[1+bs*3];
11847 D[2+ldd*3] = CC[2+bs*3];
11848 D[3+ldd*3] = CC[3+bs*3];
11849 }
11850 else if(m1>=3)
11851 {
11852 D[0+ldd*0] = CC[0+bs*0];
11853 D[1+ldd*0] = CC[1+bs*0];
11854 D[2+ldd*0] = CC[2+bs*0];
11855
11856 if(n1==1)
11857 return;
11858
11859 D[0+ldd*1] = CC[0+bs*1];
11860 D[1+ldd*1] = CC[1+bs*1];
11861 D[2+ldd*1] = CC[2+bs*1];
11862
11863 if(n1==2)
11864 return;
11865
11866 D[0+ldd*2] = CC[0+bs*2];
11867 D[1+ldd*2] = CC[1+bs*2];
11868 D[2+ldd*2] = CC[2+bs*2];
11869
11870 if(n1==3)
11871 return;
11872
11873 D[0+ldd*3] = CC[0+bs*3];
11874 D[1+ldd*3] = CC[1+bs*3];
11875 D[2+ldd*3] = CC[2+bs*3];
11876 }
11877 else if(m1>=2)
11878 {
11879 D[0+ldd*0] = CC[0+bs*0];
11880 D[1+ldd*0] = CC[1+bs*0];
11881
11882 if(n1==1)
11883 return;
11884
11885 D[0+ldd*1] = CC[0+bs*1];
11886 D[1+ldd*1] = CC[1+bs*1];
11887
11888 if(n1==2)
11889 return;
11890
11891 D[0+ldd*2] = CC[0+bs*2];
11892 D[1+ldd*2] = CC[1+bs*2];
11893
11894 if(n1==3)
11895 return;
11896
11897 D[0+ldd*3] = CC[0+bs*3];
11898 D[1+ldd*3] = CC[1+bs*3];
11899 }
11900 else //if(m1>=1)
11901 {
11902 D[0+ldd*0] = CC[0+bs*0];
11903
11904 if(n1==1)
11905 return;
11906
11907 D[0+ldd*1] = CC[0+bs*1];
11908
11909 if(n1==2)
11910 return;
11911
11912 D[0+ldd*2] = CC[0+bs*2];
11913
11914 if(n1==3)
11915 return;
11916
11917 D[0+ldd*3] = CC[0+bs*3];
11918 }
11919
11920 return;
11921
11922 }
11923 //#endif
11924
11925
11926
11927 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_4x4_tran_lib444c(int kmax,float * alpha,float * A,float * B,float * beta,float * C,float * D,int ldd)11928 void kernel_strmm_nt_ru_4x4_tran_lib444c(int kmax, float *alpha, float *A, float *B, float *beta, float *C, float *D, int ldd)
11929 {
11930
11931 const int bs = 4;
11932
11933 float
11934 a_0, a_1, a_2, a_3,
11935 b_0, b_1, b_2, b_3;
11936
11937 #if defined(TARGET_GENERIC)
11938 float CC[16] = {0};
11939 #else
11940 ALIGNED( float CC[16], 64 ) = {0};
11941 #endif
11942
11943 int k;
11944
11945 k = 0;
11946
11947 // k = 0
11948
11949 a_0 = A[0];
11950 a_1 = A[1];
11951 a_2 = A[2];
11952 a_3 = A[3];
11953
11954 b_0 = B[0+0*bs];
11955 CC[0+bs*0] += a_0 * b_0;
11956 CC[1+bs*0] += a_1 * b_0;
11957 CC[2+bs*0] += a_2 * b_0;
11958 CC[3+bs*0] += a_3 * b_0;
11959
11960 A += bs;
11961 B += bs;
11962 k += 1;
11963
11964 if(k>=kmax)
11965 goto store;
11966
11967 // k = 1
11968
11969 a_0 = A[0];
11970 a_1 = A[1];
11971 a_2 = A[2];
11972 a_3 = A[3];
11973
11974 b_0 = B[0+0*bs];
11975 CC[0+bs*0] += a_0 * b_0;
11976 CC[1+bs*0] += a_1 * b_0;
11977 CC[2+bs*0] += a_2 * b_0;
11978 CC[3+bs*0] += a_3 * b_0;
11979
11980 b_1 = B[1+0*bs];
11981 CC[0+bs*1] += a_0 * b_1;
11982 CC[1+bs*1] += a_1 * b_1;
11983 CC[2+bs*1] += a_2 * b_1;
11984 CC[3+bs*1] += a_3 * b_1;
11985
11986 A += bs;
11987 B += bs;
11988 k += 1;
11989
11990 if(k>=kmax)
11991 goto store;
11992
11993 // k = 2
11994
11995 a_0 = A[0];
11996 a_1 = A[1];
11997 a_2 = A[2];
11998 a_3 = A[3];
11999
12000 b_0 = B[0+0*bs];
12001 CC[0+bs*0] += a_0 * b_0;
12002 CC[1+bs*0] += a_1 * b_0;
12003 CC[2+bs*0] += a_2 * b_0;
12004 CC[3+bs*0] += a_3 * b_0;
12005
12006 b_1 = B[1+0*bs];
12007 CC[0+bs*1] += a_0 * b_1;
12008 CC[1+bs*1] += a_1 * b_1;
12009 CC[2+bs*1] += a_2 * b_1;
12010 CC[3+bs*1] += a_3 * b_1;
12011
12012 b_2 = B[2+0*bs];
12013 CC[0+bs*2] += a_0 * b_2;
12014 CC[1+bs*2] += a_1 * b_2;
12015 CC[2+bs*2] += a_2 * b_2;
12016 CC[3+bs*2] += a_3 * b_2;
12017
12018 A += bs;
12019 B += bs;
12020 k += 1;
12021
12022 if(k>=kmax)
12023 goto store;
12024
12025 // k = 3
12026
12027 a_0 = A[0];
12028 a_1 = A[1];
12029 a_2 = A[2];
12030 a_3 = A[3];
12031
12032 b_0 = B[0+0*bs];
12033 CC[0+bs*0] += a_0 * b_0;
12034 CC[1+bs*0] += a_1 * b_0;
12035 CC[2+bs*0] += a_2 * b_0;
12036 CC[3+bs*0] += a_3 * b_0;
12037
12038 b_1 = B[1+0*bs];
12039 CC[0+bs*1] += a_0 * b_1;
12040 CC[1+bs*1] += a_1 * b_1;
12041 CC[2+bs*1] += a_2 * b_1;
12042 CC[3+bs*1] += a_3 * b_1;
12043
12044 b_2 = B[2+0*bs];
12045 CC[0+bs*2] += a_0 * b_2;
12046 CC[1+bs*2] += a_1 * b_2;
12047 CC[2+bs*2] += a_2 * b_2;
12048 CC[3+bs*2] += a_3 * b_2;
12049
12050 b_3 = B[3+0*bs];
12051 CC[0+bs*3] += a_0 * b_3;
12052 CC[1+bs*3] += a_1 * b_3;
12053 CC[2+bs*3] += a_2 * b_3;
12054 CC[3+bs*3] += a_3 * b_3;
12055
12056 A += bs;
12057 B += bs;
12058 k += 1;
12059
12060 float alpha1 = 1.0;
12061 float beta1 = 1.0;
12062
12063 kernel_sgemm_nt_4x4_lib4(kmax-k, &alpha1, A, B, &beta1, CC, CC);
12064
12065 store:
12066
12067 // scale & tranpose & store
12068 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
12069 D[0+ldd*1] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
12070 D[0+ldd*2] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
12071 D[0+ldd*3] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
12072
12073 D[1+ldd*0] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
12074 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
12075 D[1+ldd*2] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
12076 D[1+ldd*3] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
12077
12078 D[2+ldd*0] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
12079 D[2+ldd*1] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
12080 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
12081 D[2+ldd*3] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
12082
12083 D[3+ldd*0] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
12084 D[3+ldd*1] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
12085 D[3+ldd*2] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
12086 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
12087
12088 return;
12089
12090 }
12091 //#endif
12092
12093
12094
12095 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_4x4_tran_vs_lib444c(int kmax,float * alpha,float * A,float * B,float * beta,float * C,float * D,int ldd,int m1,int n1)12096 void kernel_strmm_nt_ru_4x4_tran_vs_lib444c(int kmax, float *alpha, float *A, float *B, float *beta, float *C, float *D, int ldd, int m1, int n1)
12097 {
12098
12099 const int bs = 4;
12100
12101 float
12102 tmp,
12103 a_0, a_1, a_2, a_3,
12104 b_0, b_1, b_2, b_3;
12105
12106 #if defined(TARGET_GENERIC)
12107 float CC[16] = {0};
12108 #else
12109 ALIGNED( float CC[16], 64 ) = {0};
12110 #endif
12111
12112 int k;
12113
12114 k = 0;
12115
12116 // k = 0
12117
12118 a_0 = A[0];
12119 a_1 = A[1];
12120 a_2 = A[2];
12121 a_3 = A[3];
12122
12123 b_0 = B[0+0*bs];
12124 CC[0+bs*0] += a_0 * b_0;
12125 CC[1+bs*0] += a_1 * b_0;
12126 CC[2+bs*0] += a_2 * b_0;
12127 CC[3+bs*0] += a_3 * b_0;
12128
12129 A += bs;
12130 B += bs;
12131 k += 1;
12132
12133 if(k>=kmax)
12134 goto store;
12135
12136 // k = 1
12137
12138 a_0 = A[0];
12139 a_1 = A[1];
12140 a_2 = A[2];
12141 a_3 = A[3];
12142
12143 b_0 = B[0+0*bs];
12144 CC[0+bs*0] += a_0 * b_0;
12145 CC[1+bs*0] += a_1 * b_0;
12146 CC[2+bs*0] += a_2 * b_0;
12147 CC[3+bs*0] += a_3 * b_0;
12148
12149 b_1 = B[1+0*bs];
12150 CC[0+bs*1] += a_0 * b_1;
12151 CC[1+bs*1] += a_1 * b_1;
12152 CC[2+bs*1] += a_2 * b_1;
12153 CC[3+bs*1] += a_3 * b_1;
12154
12155 A += bs;
12156 B += bs;
12157 k += 1;
12158
12159 if(k>=kmax)
12160 goto store;
12161
12162 // k = 2
12163
12164 a_0 = A[0];
12165 a_1 = A[1];
12166 a_2 = A[2];
12167 a_3 = A[3];
12168
12169 b_0 = B[0+0*bs];
12170 CC[0+bs*0] += a_0 * b_0;
12171 CC[1+bs*0] += a_1 * b_0;
12172 CC[2+bs*0] += a_2 * b_0;
12173 CC[3+bs*0] += a_3 * b_0;
12174
12175 b_1 = B[1+0*bs];
12176 CC[0+bs*1] += a_0 * b_1;
12177 CC[1+bs*1] += a_1 * b_1;
12178 CC[2+bs*1] += a_2 * b_1;
12179 CC[3+bs*1] += a_3 * b_1;
12180
12181 b_2 = B[2+0*bs];
12182 CC[0+bs*2] += a_0 * b_2;
12183 CC[1+bs*2] += a_1 * b_2;
12184 CC[2+bs*2] += a_2 * b_2;
12185 CC[3+bs*2] += a_3 * b_2;
12186
12187 A += bs;
12188 B += bs;
12189 k += 1;
12190
12191 if(k>=kmax)
12192 goto store;
12193
12194 // k = 3
12195
12196 a_0 = A[0];
12197 a_1 = A[1];
12198 a_2 = A[2];
12199 a_3 = A[3];
12200
12201 b_0 = B[0+0*bs];
12202 CC[0+bs*0] += a_0 * b_0;
12203 CC[1+bs*0] += a_1 * b_0;
12204 CC[2+bs*0] += a_2 * b_0;
12205 CC[3+bs*0] += a_3 * b_0;
12206
12207 b_1 = B[1+0*bs];
12208 CC[0+bs*1] += a_0 * b_1;
12209 CC[1+bs*1] += a_1 * b_1;
12210 CC[2+bs*1] += a_2 * b_1;
12211 CC[3+bs*1] += a_3 * b_1;
12212
12213 b_2 = B[2+0*bs];
12214 CC[0+bs*2] += a_0 * b_2;
12215 CC[1+bs*2] += a_1 * b_2;
12216 CC[2+bs*2] += a_2 * b_2;
12217 CC[3+bs*2] += a_3 * b_2;
12218
12219 b_3 = B[3+0*bs];
12220 CC[0+bs*3] += a_0 * b_3;
12221 CC[1+bs*3] += a_1 * b_3;
12222 CC[2+bs*3] += a_2 * b_3;
12223 CC[3+bs*3] += a_3 * b_3;
12224
12225 A += bs;
12226 B += bs;
12227 k += 1;
12228
12229 float alpha1 = 1.0;
12230 float beta1 = 1.0;
12231
12232 kernel_sgemm_nt_4x4_lib4(kmax-k, &alpha1, A, B, &beta1, CC, CC);
12233
12234 store:
12235
12236 // scale
12237 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
12238 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
12239 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
12240 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
12241
12242 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
12243 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
12244 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
12245 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
12246
12247 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
12248 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
12249 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
12250 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
12251
12252 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
12253 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
12254 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
12255 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
12256
12257 // transpose
12258 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
12259 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
12260 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
12261
12262 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
12263 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
12264
12265 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
12266
12267 // store
12268 if(m1>=4)
12269 {
12270 D[0+ldd*0] = CC[0+bs*0];
12271 D[1+ldd*0] = CC[1+bs*0];
12272 D[2+ldd*0] = CC[2+bs*0];
12273 D[3+ldd*0] = CC[3+bs*0];
12274
12275 if(n1==1)
12276 return;
12277
12278 D[0+ldd*1] = CC[0+bs*1];
12279 D[1+ldd*1] = CC[1+bs*1];
12280 D[2+ldd*1] = CC[2+bs*1];
12281 D[3+ldd*1] = CC[3+bs*1];
12282
12283 if(n1==2)
12284 return;
12285
12286 D[0+ldd*2] = CC[0+bs*2];
12287 D[1+ldd*2] = CC[1+bs*2];
12288 D[2+ldd*2] = CC[2+bs*2];
12289 D[3+ldd*2] = CC[3+bs*2];
12290
12291 if(n1==3)
12292 return;
12293
12294 D[0+ldd*3] = CC[0+bs*3];
12295 D[1+ldd*3] = CC[1+bs*3];
12296 D[2+ldd*3] = CC[2+bs*3];
12297 D[3+ldd*3] = CC[3+bs*3];
12298 }
12299 else if(m1>=3)
12300 {
12301 D[0+ldd*0] = CC[0+bs*0];
12302 D[1+ldd*0] = CC[1+bs*0];
12303 D[2+ldd*0] = CC[2+bs*0];
12304
12305 if(n1==1)
12306 return;
12307
12308 D[0+ldd*1] = CC[0+bs*1];
12309 D[1+ldd*1] = CC[1+bs*1];
12310 D[2+ldd*1] = CC[2+bs*1];
12311
12312 if(n1==2)
12313 return;
12314
12315 D[0+ldd*2] = CC[0+bs*2];
12316 D[1+ldd*2] = CC[1+bs*2];
12317 D[2+ldd*2] = CC[2+bs*2];
12318
12319 if(n1==3)
12320 return;
12321
12322 D[0+ldd*3] = CC[0+bs*3];
12323 D[1+ldd*3] = CC[1+bs*3];
12324 D[2+ldd*3] = CC[2+bs*3];
12325 }
12326 else if(m1>=2)
12327 {
12328 D[0+ldd*0] = CC[0+bs*0];
12329 D[1+ldd*0] = CC[1+bs*0];
12330
12331 if(n1==1)
12332 return;
12333
12334 D[0+ldd*1] = CC[0+bs*1];
12335 D[1+ldd*1] = CC[1+bs*1];
12336
12337 if(n1==2)
12338 return;
12339
12340 D[0+ldd*2] = CC[0+bs*2];
12341 D[1+ldd*2] = CC[1+bs*2];
12342
12343 if(n1==3)
12344 return;
12345
12346 D[0+ldd*3] = CC[0+bs*3];
12347 D[1+ldd*3] = CC[1+bs*3];
12348 }
12349 else //if(m1>=1)
12350 {
12351 D[0+ldd*0] = CC[0+bs*0];
12352
12353 if(n1==1)
12354 return;
12355
12356 D[0+ldd*1] = CC[0+bs*1];
12357
12358 if(n1==2)
12359 return;
12360
12361 D[0+ldd*2] = CC[0+bs*2];
12362
12363 if(n1==3)
12364 return;
12365
12366 D[0+ldd*3] = CC[0+bs*3];
12367 }
12368
12369 return;
12370
12371 }
12372 //#endif
12373
12374
12375
12376 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)12377 void kernel_strmm_nt_ru_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
12378 {
12379
12380 const int bs = 4;
12381
12382 float
12383 a_0, a_1, a_2, a_3,
12384 b_0, b_1, b_2, b_3;
12385
12386 #if defined(TARGET_GENERIC)
12387 float CC[16] = {0};
12388 #else
12389 ALIGNED( float CC[16], 64 ) = {0};
12390 #endif
12391
12392 int k;
12393
12394 k = 0;
12395
12396 // k = 0
12397
12398 a_0 = A[0];
12399 a_1 = A[1];
12400 a_2 = A[2];
12401 a_3 = A[3];
12402
12403 b_0 = B[0+0*ldb];
12404 CC[0+bs*0] += a_0 * b_0;
12405 CC[1+bs*0] += a_1 * b_0;
12406 CC[2+bs*0] += a_2 * b_0;
12407 CC[3+bs*0] += a_3 * b_0;
12408
12409 A += bs;
12410 B += ldb;
12411 k += 1;
12412
12413 if(k>=kmax)
12414 goto store;
12415
12416 // k = 1
12417
12418 a_0 = A[0];
12419 a_1 = A[1];
12420 a_2 = A[2];
12421 a_3 = A[3];
12422
12423 b_0 = B[0+0*ldb];
12424 CC[0+bs*0] += a_0 * b_0;
12425 CC[1+bs*0] += a_1 * b_0;
12426 CC[2+bs*0] += a_2 * b_0;
12427 CC[3+bs*0] += a_3 * b_0;
12428
12429 b_1 = B[1+0*ldb];
12430 CC[0+bs*1] += a_0 * b_1;
12431 CC[1+bs*1] += a_1 * b_1;
12432 CC[2+bs*1] += a_2 * b_1;
12433 CC[3+bs*1] += a_3 * b_1;
12434
12435 A += bs;
12436 B += ldb;
12437 k += 1;
12438
12439 if(k>=kmax)
12440 goto store;
12441
12442 // k = 2
12443
12444 a_0 = A[0];
12445 a_1 = A[1];
12446 a_2 = A[2];
12447 a_3 = A[3];
12448
12449 b_0 = B[0+0*ldb];
12450 CC[0+bs*0] += a_0 * b_0;
12451 CC[1+bs*0] += a_1 * b_0;
12452 CC[2+bs*0] += a_2 * b_0;
12453 CC[3+bs*0] += a_3 * b_0;
12454
12455 b_1 = B[1+0*ldb];
12456 CC[0+bs*1] += a_0 * b_1;
12457 CC[1+bs*1] += a_1 * b_1;
12458 CC[2+bs*1] += a_2 * b_1;
12459 CC[3+bs*1] += a_3 * b_1;
12460
12461 b_2 = B[2+0*ldb];
12462 CC[0+bs*2] += a_0 * b_2;
12463 CC[1+bs*2] += a_1 * b_2;
12464 CC[2+bs*2] += a_2 * b_2;
12465 CC[3+bs*2] += a_3 * b_2;
12466
12467 A += bs;
12468 B += ldb;
12469 k += 1;
12470
12471 if(k>=kmax)
12472 goto store;
12473
12474 // k = 3
12475
12476 a_0 = A[0];
12477 a_1 = A[1];
12478 a_2 = A[2];
12479 a_3 = A[3];
12480
12481 b_0 = B[0+0*ldb];
12482 CC[0+bs*0] += a_0 * b_0;
12483 CC[1+bs*0] += a_1 * b_0;
12484 CC[2+bs*0] += a_2 * b_0;
12485 CC[3+bs*0] += a_3 * b_0;
12486
12487 b_1 = B[1+0*ldb];
12488 CC[0+bs*1] += a_0 * b_1;
12489 CC[1+bs*1] += a_1 * b_1;
12490 CC[2+bs*1] += a_2 * b_1;
12491 CC[3+bs*1] += a_3 * b_1;
12492
12493 b_2 = B[2+0*ldb];
12494 CC[0+bs*2] += a_0 * b_2;
12495 CC[1+bs*2] += a_1 * b_2;
12496 CC[2+bs*2] += a_2 * b_2;
12497 CC[3+bs*2] += a_3 * b_2;
12498
12499 b_3 = B[3+0*ldb];
12500 CC[0+bs*3] += a_0 * b_3;
12501 CC[1+bs*3] += a_1 * b_3;
12502 CC[2+bs*3] += a_2 * b_3;
12503 CC[3+bs*3] += a_3 * b_3;
12504
12505 A += bs;
12506 B += ldb;
12507 k += 1;
12508
12509 store:
12510
12511 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
12512 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
12513 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
12514 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
12515
12516 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
12517 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
12518 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
12519 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
12520
12521 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
12522 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
12523 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
12524 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
12525
12526 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
12527 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
12528 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
12529 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
12530
12531 float beta1 = 1.0;
12532
12533 kernel_sgemm_nt_4x4_lib4ccc(kmax-k, alpha, A, B, ldb, &beta1, CC, bs, D, ldd);
12534
12535 return;
12536
12537 }
12538 //#endif
12539
12540
12541
12542 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)12543 void kernel_strmm_nt_ru_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
12544 {
12545
12546 const int bs = 4;
12547
12548 float
12549 a_0, a_1, a_2, a_3,
12550 b_0, b_1, b_2, b_3;
12551
12552 #if defined(TARGET_GENERIC)
12553 float CC[16] = {0};
12554 #else
12555 ALIGNED( float CC[16], 64 ) = {0};
12556 #endif
12557
12558 int k;
12559
12560 k = 0;
12561
12562 // k = 0
12563
12564 a_0 = A[0];
12565 a_1 = A[1];
12566 a_2 = A[2];
12567 a_3 = A[3];
12568
12569 b_0 = B[0+0*ldb];
12570 CC[0+bs*0] += a_0 * b_0;
12571 CC[1+bs*0] += a_1 * b_0;
12572 CC[2+bs*0] += a_2 * b_0;
12573 CC[3+bs*0] += a_3 * b_0;
12574
12575 A += bs;
12576 B += ldb;
12577 k += 1;
12578
12579 if(k>=kmax)
12580 goto store;
12581
12582 // k = 1
12583
12584 a_0 = A[0];
12585 a_1 = A[1];
12586 a_2 = A[2];
12587 a_3 = A[3];
12588
12589 b_0 = B[0+0*ldb];
12590 CC[0+bs*0] += a_0 * b_0;
12591 CC[1+bs*0] += a_1 * b_0;
12592 CC[2+bs*0] += a_2 * b_0;
12593 CC[3+bs*0] += a_3 * b_0;
12594
12595 b_1 = B[1+0*ldb];
12596 CC[0+bs*1] += a_0 * b_1;
12597 CC[1+bs*1] += a_1 * b_1;
12598 CC[2+bs*1] += a_2 * b_1;
12599 CC[3+bs*1] += a_3 * b_1;
12600
12601 A += bs;
12602 B += ldb;
12603 k += 1;
12604
12605 if(k>=kmax)
12606 goto store;
12607
12608 // k = 2
12609
12610 a_0 = A[0];
12611 a_1 = A[1];
12612 a_2 = A[2];
12613 a_3 = A[3];
12614
12615 b_0 = B[0+0*ldb];
12616 CC[0+bs*0] += a_0 * b_0;
12617 CC[1+bs*0] += a_1 * b_0;
12618 CC[2+bs*0] += a_2 * b_0;
12619 CC[3+bs*0] += a_3 * b_0;
12620
12621 b_1 = B[1+0*ldb];
12622 CC[0+bs*1] += a_0 * b_1;
12623 CC[1+bs*1] += a_1 * b_1;
12624 CC[2+bs*1] += a_2 * b_1;
12625 CC[3+bs*1] += a_3 * b_1;
12626
12627 b_2 = B[2+0*ldb];
12628 CC[0+bs*2] += a_0 * b_2;
12629 CC[1+bs*2] += a_1 * b_2;
12630 CC[2+bs*2] += a_2 * b_2;
12631 CC[3+bs*2] += a_3 * b_2;
12632
12633 A += bs;
12634 B += ldb;
12635 k += 1;
12636
12637 if(k>=kmax)
12638 goto store;
12639
12640 // k = 3
12641
12642 a_0 = A[0];
12643 a_1 = A[1];
12644 a_2 = A[2];
12645 a_3 = A[3];
12646
12647 b_0 = B[0+0*ldb];
12648 CC[0+bs*0] += a_0 * b_0;
12649 CC[1+bs*0] += a_1 * b_0;
12650 CC[2+bs*0] += a_2 * b_0;
12651 CC[3+bs*0] += a_3 * b_0;
12652
12653 b_1 = B[1+0*ldb];
12654 CC[0+bs*1] += a_0 * b_1;
12655 CC[1+bs*1] += a_1 * b_1;
12656 CC[2+bs*1] += a_2 * b_1;
12657 CC[3+bs*1] += a_3 * b_1;
12658
12659 b_2 = B[2+0*ldb];
12660 CC[0+bs*2] += a_0 * b_2;
12661 CC[1+bs*2] += a_1 * b_2;
12662 CC[2+bs*2] += a_2 * b_2;
12663 CC[3+bs*2] += a_3 * b_2;
12664
12665 b_3 = B[3+0*ldb];
12666 CC[0+bs*3] += a_0 * b_3;
12667 CC[1+bs*3] += a_1 * b_3;
12668 CC[2+bs*3] += a_2 * b_3;
12669 CC[3+bs*3] += a_3 * b_3;
12670
12671 A += bs;
12672 B += ldb;
12673 k += 1;
12674
12675 store:
12676
12677 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
12678 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
12679 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
12680 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
12681
12682 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
12683 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
12684 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
12685 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
12686
12687 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
12688 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
12689 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
12690 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
12691
12692 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
12693 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
12694 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
12695 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
12696
12697 float beta1 = 1.0;
12698
12699 kernel_sgemm_nt_4x4_lib4ccc(kmax-k, alpha, A, B, ldb, &beta1, CC, bs, CC, bs);
12700
12701 if(m1>=4)
12702 {
12703 D[0+ldd*0] = CC[0+bs*0];
12704 D[1+ldd*0] = CC[1+bs*0];
12705 D[2+ldd*0] = CC[2+bs*0];
12706 D[3+ldd*0] = CC[3+bs*0];
12707
12708 if(n1==1)
12709 return;
12710
12711 D[0+ldd*1] = CC[0+bs*1];
12712 D[1+ldd*1] = CC[1+bs*1];
12713 D[2+ldd*1] = CC[2+bs*1];
12714 D[3+ldd*1] = CC[3+bs*1];
12715
12716 if(n1==2)
12717 return;
12718
12719 D[0+ldd*2] = CC[0+bs*2];
12720 D[1+ldd*2] = CC[1+bs*2];
12721 D[2+ldd*2] = CC[2+bs*2];
12722 D[3+ldd*2] = CC[3+bs*2];
12723
12724 if(n1==3)
12725 return;
12726
12727 D[0+ldd*3] = CC[0+bs*3];
12728 D[1+ldd*3] = CC[1+bs*3];
12729 D[2+ldd*3] = CC[2+bs*3];
12730 D[3+ldd*3] = CC[3+bs*3];
12731 }
12732 else if(m1>=3)
12733 {
12734 D[0+ldd*0] = CC[0+bs*0];
12735 D[1+ldd*0] = CC[1+bs*0];
12736 D[2+ldd*0] = CC[2+bs*0];
12737
12738 if(n1==1)
12739 return;
12740
12741 D[0+ldd*1] = CC[0+bs*1];
12742 D[1+ldd*1] = CC[1+bs*1];
12743 D[2+ldd*1] = CC[2+bs*1];
12744
12745 if(n1==2)
12746 return;
12747
12748 D[0+ldd*2] = CC[0+bs*2];
12749 D[1+ldd*2] = CC[1+bs*2];
12750 D[2+ldd*2] = CC[2+bs*2];
12751
12752 if(n1==3)
12753 return;
12754
12755 D[0+ldd*3] = CC[0+bs*3];
12756 D[1+ldd*3] = CC[1+bs*3];
12757 D[2+ldd*3] = CC[2+bs*3];
12758 }
12759 else if(m1>=2)
12760 {
12761 D[0+ldd*0] = CC[0+bs*0];
12762 D[1+ldd*0] = CC[1+bs*0];
12763
12764 if(n1==1)
12765 return;
12766
12767 D[0+ldd*1] = CC[0+bs*1];
12768 D[1+ldd*1] = CC[1+bs*1];
12769
12770 if(n1==2)
12771 return;
12772
12773 D[0+ldd*2] = CC[0+bs*2];
12774 D[1+ldd*2] = CC[1+bs*2];
12775
12776 if(n1==3)
12777 return;
12778
12779 D[0+ldd*3] = CC[0+bs*3];
12780 D[1+ldd*3] = CC[1+bs*3];
12781 }
12782 else //if(m1>=1)
12783 {
12784 D[0+ldd*0] = CC[0+bs*0];
12785
12786 if(n1==1)
12787 return;
12788
12789 D[0+ldd*1] = CC[0+bs*1];
12790
12791 if(n1==2)
12792 return;
12793
12794 D[0+ldd*2] = CC[0+bs*2];
12795
12796 if(n1==3)
12797 return;
12798
12799 D[0+ldd*3] = CC[0+bs*3];
12800 }
12801
12802 return;
12803
12804 }
12805 //#endif
12806
12807
12808
12809 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_4x4_tran_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd)12810 void kernel_strmm_nt_ru_4x4_tran_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd)
12811 {
12812
12813 const int bs = 4;
12814
12815 float
12816 a_0, a_1, a_2, a_3,
12817 b_0, b_1, b_2, b_3;
12818
12819 #if defined(TARGET_GENERIC)
12820 float CC[16] = {0};
12821 #else
12822 ALIGNED( float CC[16], 64 ) = {0};
12823 #endif
12824
12825 int k;
12826
12827 k = 0;
12828
12829 // k = 0
12830
12831 a_0 = A[0];
12832 a_1 = A[1];
12833 a_2 = A[2];
12834 a_3 = A[3];
12835
12836 b_0 = B[0+0*ldb];
12837 CC[0+bs*0] += a_0 * b_0;
12838 CC[1+bs*0] += a_1 * b_0;
12839 CC[2+bs*0] += a_2 * b_0;
12840 CC[3+bs*0] += a_3 * b_0;
12841
12842 A += bs;
12843 B += ldb;
12844 k += 1;
12845
12846 if(k>=kmax)
12847 goto store;
12848
12849 // k = 1
12850
12851 a_0 = A[0];
12852 a_1 = A[1];
12853 a_2 = A[2];
12854 a_3 = A[3];
12855
12856 b_0 = B[0+0*ldb];
12857 CC[0+bs*0] += a_0 * b_0;
12858 CC[1+bs*0] += a_1 * b_0;
12859 CC[2+bs*0] += a_2 * b_0;
12860 CC[3+bs*0] += a_3 * b_0;
12861
12862 b_1 = B[1+0*ldb];
12863 CC[0+bs*1] += a_0 * b_1;
12864 CC[1+bs*1] += a_1 * b_1;
12865 CC[2+bs*1] += a_2 * b_1;
12866 CC[3+bs*1] += a_3 * b_1;
12867
12868 A += bs;
12869 B += ldb;
12870 k += 1;
12871
12872 if(k>=kmax)
12873 goto store;
12874
12875 // k = 2
12876
12877 a_0 = A[0];
12878 a_1 = A[1];
12879 a_2 = A[2];
12880 a_3 = A[3];
12881
12882 b_0 = B[0+0*ldb];
12883 CC[0+bs*0] += a_0 * b_0;
12884 CC[1+bs*0] += a_1 * b_0;
12885 CC[2+bs*0] += a_2 * b_0;
12886 CC[3+bs*0] += a_3 * b_0;
12887
12888 b_1 = B[1+0*ldb];
12889 CC[0+bs*1] += a_0 * b_1;
12890 CC[1+bs*1] += a_1 * b_1;
12891 CC[2+bs*1] += a_2 * b_1;
12892 CC[3+bs*1] += a_3 * b_1;
12893
12894 b_2 = B[2+0*ldb];
12895 CC[0+bs*2] += a_0 * b_2;
12896 CC[1+bs*2] += a_1 * b_2;
12897 CC[2+bs*2] += a_2 * b_2;
12898 CC[3+bs*2] += a_3 * b_2;
12899
12900 A += bs;
12901 B += ldb;
12902 k += 1;
12903
12904 if(k>=kmax)
12905 goto store;
12906
12907 // k = 3
12908
12909 a_0 = A[0];
12910 a_1 = A[1];
12911 a_2 = A[2];
12912 a_3 = A[3];
12913
12914 b_0 = B[0+0*ldb];
12915 CC[0+bs*0] += a_0 * b_0;
12916 CC[1+bs*0] += a_1 * b_0;
12917 CC[2+bs*0] += a_2 * b_0;
12918 CC[3+bs*0] += a_3 * b_0;
12919
12920 b_1 = B[1+0*ldb];
12921 CC[0+bs*1] += a_0 * b_1;
12922 CC[1+bs*1] += a_1 * b_1;
12923 CC[2+bs*1] += a_2 * b_1;
12924 CC[3+bs*1] += a_3 * b_1;
12925
12926 b_2 = B[2+0*ldb];
12927 CC[0+bs*2] += a_0 * b_2;
12928 CC[1+bs*2] += a_1 * b_2;
12929 CC[2+bs*2] += a_2 * b_2;
12930 CC[3+bs*2] += a_3 * b_2;
12931
12932 b_3 = B[3+0*ldb];
12933 CC[0+bs*3] += a_0 * b_3;
12934 CC[1+bs*3] += a_1 * b_3;
12935 CC[2+bs*3] += a_2 * b_3;
12936 CC[3+bs*3] += a_3 * b_3;
12937
12938 A += bs;
12939 B += ldb;
12940 k += 1;
12941
12942 float alpha1 = 1.0;
12943 float beta1 = 1.0;
12944
12945 kernel_sgemm_nt_4x4_lib4ccc(kmax-k, &alpha1, A, B, ldb, &beta1, CC, bs, CC, bs);
12946
12947 store:
12948
12949 // scale & tranpose & store
12950 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
12951 D[0+ldd*1] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
12952 D[0+ldd*2] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
12953 D[0+ldd*3] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
12954
12955 D[1+ldd*0] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
12956 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
12957 D[1+ldd*2] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
12958 D[1+ldd*3] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
12959
12960 D[2+ldd*0] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
12961 D[2+ldd*1] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
12962 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
12963 D[2+ldd*3] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
12964
12965 D[3+ldd*0] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
12966 D[3+ldd*1] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
12967 D[3+ldd*2] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
12968 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
12969
12970 return;
12971
12972 }
12973 //#endif
12974
12975
12976
12977 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_4x4_tran_vs_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd,int m1,int n1)12978 void kernel_strmm_nt_ru_4x4_tran_vs_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd, int m1, int n1)
12979 {
12980
12981 const int bs = 4;
12982
12983 float
12984 tmp,
12985 a_0, a_1, a_2, a_3,
12986 b_0, b_1, b_2, b_3;
12987
12988 #if defined(TARGET_GENERIC)
12989 float CC[16] = {0};
12990 #else
12991 ALIGNED( float CC[16], 64 ) = {0};
12992 #endif
12993
12994 int k;
12995
12996 k = 0;
12997
12998 // k = 0
12999
13000 a_0 = A[0];
13001 a_1 = A[1];
13002 a_2 = A[2];
13003 a_3 = A[3];
13004
13005 b_0 = B[0+0*ldb];
13006 CC[0+bs*0] += a_0 * b_0;
13007 CC[1+bs*0] += a_1 * b_0;
13008 CC[2+bs*0] += a_2 * b_0;
13009 CC[3+bs*0] += a_3 * b_0;
13010
13011 A += bs;
13012 B += ldb;
13013 k += 1;
13014
13015 if(k>=kmax)
13016 goto store;
13017
13018 // k = 1
13019
13020 a_0 = A[0];
13021 a_1 = A[1];
13022 a_2 = A[2];
13023 a_3 = A[3];
13024
13025 b_0 = B[0+0*ldb];
13026 CC[0+bs*0] += a_0 * b_0;
13027 CC[1+bs*0] += a_1 * b_0;
13028 CC[2+bs*0] += a_2 * b_0;
13029 CC[3+bs*0] += a_3 * b_0;
13030
13031 b_1 = B[1+0*ldb];
13032 CC[0+bs*1] += a_0 * b_1;
13033 CC[1+bs*1] += a_1 * b_1;
13034 CC[2+bs*1] += a_2 * b_1;
13035 CC[3+bs*1] += a_3 * b_1;
13036
13037 A += bs;
13038 B += ldb;
13039 k += 1;
13040
13041 if(k>=kmax)
13042 goto store;
13043
13044 // k = 2
13045
13046 a_0 = A[0];
13047 a_1 = A[1];
13048 a_2 = A[2];
13049 a_3 = A[3];
13050
13051 b_0 = B[0+0*ldb];
13052 CC[0+bs*0] += a_0 * b_0;
13053 CC[1+bs*0] += a_1 * b_0;
13054 CC[2+bs*0] += a_2 * b_0;
13055 CC[3+bs*0] += a_3 * b_0;
13056
13057 b_1 = B[1+0*ldb];
13058 CC[0+bs*1] += a_0 * b_1;
13059 CC[1+bs*1] += a_1 * b_1;
13060 CC[2+bs*1] += a_2 * b_1;
13061 CC[3+bs*1] += a_3 * b_1;
13062
13063 b_2 = B[2+0*ldb];
13064 CC[0+bs*2] += a_0 * b_2;
13065 CC[1+bs*2] += a_1 * b_2;
13066 CC[2+bs*2] += a_2 * b_2;
13067 CC[3+bs*2] += a_3 * b_2;
13068
13069 A += bs;
13070 B += ldb;
13071 k += 1;
13072
13073 if(k>=kmax)
13074 goto store;
13075
13076 // k = 3
13077
13078 a_0 = A[0];
13079 a_1 = A[1];
13080 a_2 = A[2];
13081 a_3 = A[3];
13082
13083 b_0 = B[0+0*ldb];
13084 CC[0+bs*0] += a_0 * b_0;
13085 CC[1+bs*0] += a_1 * b_0;
13086 CC[2+bs*0] += a_2 * b_0;
13087 CC[3+bs*0] += a_3 * b_0;
13088
13089 b_1 = B[1+0*ldb];
13090 CC[0+bs*1] += a_0 * b_1;
13091 CC[1+bs*1] += a_1 * b_1;
13092 CC[2+bs*1] += a_2 * b_1;
13093 CC[3+bs*1] += a_3 * b_1;
13094
13095 b_2 = B[2+0*ldb];
13096 CC[0+bs*2] += a_0 * b_2;
13097 CC[1+bs*2] += a_1 * b_2;
13098 CC[2+bs*2] += a_2 * b_2;
13099 CC[3+bs*2] += a_3 * b_2;
13100
13101 b_3 = B[3+0*ldb];
13102 CC[0+bs*3] += a_0 * b_3;
13103 CC[1+bs*3] += a_1 * b_3;
13104 CC[2+bs*3] += a_2 * b_3;
13105 CC[3+bs*3] += a_3 * b_3;
13106
13107 A += bs;
13108 B += ldb;
13109 k += 1;
13110
13111 float alpha1 = 1.0;
13112 float beta1 = 1.0;
13113
13114 kernel_sgemm_nt_4x4_lib4ccc(kmax-k, &alpha1, A, B, ldb, &beta1, CC, bs, CC, bs);
13115
13116 store:
13117
13118 // scale
13119 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
13120 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
13121 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
13122 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
13123
13124 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
13125 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
13126 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
13127 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
13128
13129 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
13130 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
13131 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
13132 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
13133
13134 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
13135 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
13136 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
13137 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
13138
13139 // transpose
13140 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
13141 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
13142 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
13143
13144 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
13145 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
13146
13147 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
13148
13149 // store
13150 if(m1>=4)
13151 {
13152 D[0+ldd*0] = CC[0+bs*0];
13153 D[1+ldd*0] = CC[1+bs*0];
13154 D[2+ldd*0] = CC[2+bs*0];
13155 D[3+ldd*0] = CC[3+bs*0];
13156
13157 if(n1==1)
13158 return;
13159
13160 D[0+ldd*1] = CC[0+bs*1];
13161 D[1+ldd*1] = CC[1+bs*1];
13162 D[2+ldd*1] = CC[2+bs*1];
13163 D[3+ldd*1] = CC[3+bs*1];
13164
13165 if(n1==2)
13166 return;
13167
13168 D[0+ldd*2] = CC[0+bs*2];
13169 D[1+ldd*2] = CC[1+bs*2];
13170 D[2+ldd*2] = CC[2+bs*2];
13171 D[3+ldd*2] = CC[3+bs*2];
13172
13173 if(n1==3)
13174 return;
13175
13176 D[0+ldd*3] = CC[0+bs*3];
13177 D[1+ldd*3] = CC[1+bs*3];
13178 D[2+ldd*3] = CC[2+bs*3];
13179 D[3+ldd*3] = CC[3+bs*3];
13180 }
13181 else if(m1>=3)
13182 {
13183 D[0+ldd*0] = CC[0+bs*0];
13184 D[1+ldd*0] = CC[1+bs*0];
13185 D[2+ldd*0] = CC[2+bs*0];
13186
13187 if(n1==1)
13188 return;
13189
13190 D[0+ldd*1] = CC[0+bs*1];
13191 D[1+ldd*1] = CC[1+bs*1];
13192 D[2+ldd*1] = CC[2+bs*1];
13193
13194 if(n1==2)
13195 return;
13196
13197 D[0+ldd*2] = CC[0+bs*2];
13198 D[1+ldd*2] = CC[1+bs*2];
13199 D[2+ldd*2] = CC[2+bs*2];
13200
13201 if(n1==3)
13202 return;
13203
13204 D[0+ldd*3] = CC[0+bs*3];
13205 D[1+ldd*3] = CC[1+bs*3];
13206 D[2+ldd*3] = CC[2+bs*3];
13207 }
13208 else if(m1>=2)
13209 {
13210 D[0+ldd*0] = CC[0+bs*0];
13211 D[1+ldd*0] = CC[1+bs*0];
13212
13213 if(n1==1)
13214 return;
13215
13216 D[0+ldd*1] = CC[0+bs*1];
13217 D[1+ldd*1] = CC[1+bs*1];
13218
13219 if(n1==2)
13220 return;
13221
13222 D[0+ldd*2] = CC[0+bs*2];
13223 D[1+ldd*2] = CC[1+bs*2];
13224
13225 if(n1==3)
13226 return;
13227
13228 D[0+ldd*3] = CC[0+bs*3];
13229 D[1+ldd*3] = CC[1+bs*3];
13230 }
13231 else //if(m1>=1)
13232 {
13233 D[0+ldd*0] = CC[0+bs*0];
13234
13235 if(n1==1)
13236 return;
13237
13238 D[0+ldd*1] = CC[0+bs*1];
13239
13240 if(n1==2)
13241 return;
13242
13243 D[0+ldd*2] = CC[0+bs*2];
13244
13245 if(n1==3)
13246 return;
13247
13248 D[0+ldd*3] = CC[0+bs*3];
13249 }
13250
13251 return;
13252
13253 }
13254 //#endif
13255
13256
13257
13258 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_one_4x4_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd)13259 void kernel_strmm_nt_ru_one_4x4_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd)
13260 {
13261
13262 const int bs = 4;
13263
13264 float
13265 a_0, a_1, a_2, a_3,
13266 b_0, b_1, b_2, b_3;
13267
13268 #if defined(TARGET_GENERIC)
13269 float CC[16] = {0};
13270 #else
13271 ALIGNED( float CC[16], 64 ) = {0};
13272 #endif
13273
13274 int k;
13275
13276 k = 0;
13277
13278 // k = 0
13279
13280 a_0 = A[0];
13281 a_1 = A[1];
13282 a_2 = A[2];
13283 a_3 = A[3];
13284
13285 CC[0+bs*0] += a_0;
13286 CC[1+bs*0] += a_1;
13287 CC[2+bs*0] += a_2;
13288 CC[3+bs*0] += a_3;
13289
13290 A += bs;
13291 B += bs;
13292 k += 1;
13293
13294 if(k>=kmax)
13295 goto store;
13296
13297 // k = 1
13298
13299 a_0 = A[0];
13300 a_1 = A[1];
13301 a_2 = A[2];
13302 a_3 = A[3];
13303
13304 b_0 = B[0+0*bs];
13305 CC[0+bs*0] += a_0 * b_0;
13306 CC[1+bs*0] += a_1 * b_0;
13307 CC[2+bs*0] += a_2 * b_0;
13308 CC[3+bs*0] += a_3 * b_0;
13309
13310 CC[0+bs*1] += a_0;
13311 CC[1+bs*1] += a_1;
13312 CC[2+bs*1] += a_2;
13313 CC[3+bs*1] += a_3;
13314
13315 A += bs;
13316 B += bs;
13317 k += 1;
13318
13319 if(k>=kmax)
13320 goto store;
13321
13322 // k = 2
13323
13324 a_0 = A[0];
13325 a_1 = A[1];
13326 a_2 = A[2];
13327 a_3 = A[3];
13328
13329 b_0 = B[0+0*bs];
13330 CC[0+bs*0] += a_0 * b_0;
13331 CC[1+bs*0] += a_1 * b_0;
13332 CC[2+bs*0] += a_2 * b_0;
13333 CC[3+bs*0] += a_3 * b_0;
13334
13335 b_1 = B[1+0*bs];
13336 CC[0+bs*1] += a_0 * b_1;
13337 CC[1+bs*1] += a_1 * b_1;
13338 CC[2+bs*1] += a_2 * b_1;
13339 CC[3+bs*1] += a_3 * b_1;
13340
13341 CC[0+bs*2] += a_0;
13342 CC[1+bs*2] += a_1;
13343 CC[2+bs*2] += a_2;
13344 CC[3+bs*2] += a_3;
13345
13346 A += bs;
13347 B += bs;
13348 k += 1;
13349
13350 if(k>=kmax)
13351 goto store;
13352
13353 // k = 3
13354
13355 a_0 = A[0];
13356 a_1 = A[1];
13357 a_2 = A[2];
13358 a_3 = A[3];
13359
13360 b_0 = B[0+0*bs];
13361 CC[0+bs*0] += a_0 * b_0;
13362 CC[1+bs*0] += a_1 * b_0;
13363 CC[2+bs*0] += a_2 * b_0;
13364 CC[3+bs*0] += a_3 * b_0;
13365
13366 b_1 = B[1+0*bs];
13367 CC[0+bs*1] += a_0 * b_1;
13368 CC[1+bs*1] += a_1 * b_1;
13369 CC[2+bs*1] += a_2 * b_1;
13370 CC[3+bs*1] += a_3 * b_1;
13371
13372 b_2 = B[2+0*bs];
13373 CC[0+bs*2] += a_0 * b_2;
13374 CC[1+bs*2] += a_1 * b_2;
13375 CC[2+bs*2] += a_2 * b_2;
13376 CC[3+bs*2] += a_3 * b_2;
13377
13378 CC[0+bs*3] += a_0;
13379 CC[1+bs*3] += a_1;
13380 CC[2+bs*3] += a_2;
13381 CC[3+bs*3] += a_3;
13382
13383 A += bs;
13384 B += bs;
13385 k += 1;
13386
13387 store:
13388
13389 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
13390 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
13391 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
13392 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
13393
13394 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
13395 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
13396 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
13397 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
13398
13399 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
13400 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
13401 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
13402 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
13403
13404 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
13405 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
13406 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
13407 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
13408
13409 float beta1 = 1.0;
13410
13411 kernel_sgemm_nt_4x4_lib4(kmax-k, alpha, A, B, &beta1, CC, CC);
13412
13413 D[0+ldd*0] = CC[0+bs*0];
13414 D[1+ldd*0] = CC[1+bs*0];
13415 D[2+ldd*0] = CC[2+bs*0];
13416 D[3+ldd*0] = CC[3+bs*0];
13417
13418 D[0+ldd*1] = CC[0+bs*1];
13419 D[1+ldd*1] = CC[1+bs*1];
13420 D[2+ldd*1] = CC[2+bs*1];
13421 D[3+ldd*1] = CC[3+bs*1];
13422
13423 D[0+ldd*2] = CC[0+bs*2];
13424 D[1+ldd*2] = CC[1+bs*2];
13425 D[2+ldd*2] = CC[2+bs*2];
13426 D[3+ldd*2] = CC[3+bs*2];
13427
13428 D[0+ldd*3] = CC[0+bs*3];
13429 D[1+ldd*3] = CC[1+bs*3];
13430 D[2+ldd*3] = CC[2+bs*3];
13431 D[3+ldd*3] = CC[3+bs*3];
13432
13433 return;
13434
13435 }
13436 //#endif
13437
13438
13439
13440 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_one_4x4_vs_lib44cc(int kmax,float * alpha,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)13441 void kernel_strmm_nt_ru_one_4x4_vs_lib44cc(int kmax, float *alpha, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
13442 {
13443
13444 const int bs = 4;
13445
13446 float
13447 a_0, a_1, a_2, a_3,
13448 b_0, b_1, b_2, b_3;
13449
13450 #if defined(TARGET_GENERIC)
13451 float CC[16] = {0};
13452 #else
13453 ALIGNED( float CC[16], 64 ) = {0};
13454 #endif
13455
13456 int k;
13457
13458 k = 0;
13459
13460 // k = 0
13461
13462 a_0 = A[0];
13463 a_1 = A[1];
13464 a_2 = A[2];
13465 a_3 = A[3];
13466
13467 CC[0+bs*0] += a_0;
13468 CC[1+bs*0] += a_1;
13469 CC[2+bs*0] += a_2;
13470 CC[3+bs*0] += a_3;
13471
13472 A += bs;
13473 B += bs;
13474 k += 1;
13475
13476 if(k>=kmax)
13477 goto store;
13478
13479 // k = 1
13480
13481 a_0 = A[0];
13482 a_1 = A[1];
13483 a_2 = A[2];
13484 a_3 = A[3];
13485
13486 b_0 = B[0+0*bs];
13487 CC[0+bs*0] += a_0 * b_0;
13488 CC[1+bs*0] += a_1 * b_0;
13489 CC[2+bs*0] += a_2 * b_0;
13490 CC[3+bs*0] += a_3 * b_0;
13491
13492 CC[0+bs*1] += a_0;
13493 CC[1+bs*1] += a_1;
13494 CC[2+bs*1] += a_2;
13495 CC[3+bs*1] += a_3;
13496
13497 A += bs;
13498 B += bs;
13499 k += 1;
13500
13501 if(k>=kmax)
13502 goto store;
13503
13504 // k = 2
13505
13506 a_0 = A[0];
13507 a_1 = A[1];
13508 a_2 = A[2];
13509 a_3 = A[3];
13510
13511 b_0 = B[0+0*bs];
13512 CC[0+bs*0] += a_0 * b_0;
13513 CC[1+bs*0] += a_1 * b_0;
13514 CC[2+bs*0] += a_2 * b_0;
13515 CC[3+bs*0] += a_3 * b_0;
13516
13517 b_1 = B[1+0*bs];
13518 CC[0+bs*1] += a_0 * b_1;
13519 CC[1+bs*1] += a_1 * b_1;
13520 CC[2+bs*1] += a_2 * b_1;
13521 CC[3+bs*1] += a_3 * b_1;
13522
13523 CC[0+bs*2] += a_0;
13524 CC[1+bs*2] += a_1;
13525 CC[2+bs*2] += a_2;
13526 CC[3+bs*2] += a_3;
13527
13528 A += bs;
13529 B += bs;
13530 k += 1;
13531
13532 if(k>=kmax)
13533 goto store;
13534
13535 // k = 3
13536
13537 a_0 = A[0];
13538 a_1 = A[1];
13539 a_2 = A[2];
13540 a_3 = A[3];
13541
13542 b_0 = B[0+0*bs];
13543 CC[0+bs*0] += a_0 * b_0;
13544 CC[1+bs*0] += a_1 * b_0;
13545 CC[2+bs*0] += a_2 * b_0;
13546 CC[3+bs*0] += a_3 * b_0;
13547
13548 b_1 = B[1+0*bs];
13549 CC[0+bs*1] += a_0 * b_1;
13550 CC[1+bs*1] += a_1 * b_1;
13551 CC[2+bs*1] += a_2 * b_1;
13552 CC[3+bs*1] += a_3 * b_1;
13553
13554 b_2 = B[2+0*bs];
13555 CC[0+bs*2] += a_0 * b_2;
13556 CC[1+bs*2] += a_1 * b_2;
13557 CC[2+bs*2] += a_2 * b_2;
13558 CC[3+bs*2] += a_3 * b_2;
13559
13560 CC[0+bs*3] += a_0;
13561 CC[1+bs*3] += a_1;
13562 CC[2+bs*3] += a_2;
13563 CC[3+bs*3] += a_3;
13564
13565 A += bs;
13566 B += bs;
13567 k += 1;
13568
13569 store:
13570
13571 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
13572 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
13573 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
13574 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
13575
13576 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
13577 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
13578 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
13579 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
13580
13581 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
13582 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
13583 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
13584 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
13585
13586 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
13587 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
13588 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
13589 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
13590
13591 float beta1 = 1.0;
13592
13593 kernel_sgemm_nt_4x4_lib4(kmax-k, alpha, A, B, &beta1, CC, CC);
13594
13595 if(m1>=4)
13596 {
13597 D[0+ldd*0] = CC[0+bs*0];
13598 D[1+ldd*0] = CC[1+bs*0];
13599 D[2+ldd*0] = CC[2+bs*0];
13600 D[3+ldd*0] = CC[3+bs*0];
13601
13602 if(n1==1)
13603 return;
13604
13605 D[0+ldd*1] = CC[0+bs*1];
13606 D[1+ldd*1] = CC[1+bs*1];
13607 D[2+ldd*1] = CC[2+bs*1];
13608 D[3+ldd*1] = CC[3+bs*1];
13609
13610 if(n1==2)
13611 return;
13612
13613 D[0+ldd*2] = CC[0+bs*2];
13614 D[1+ldd*2] = CC[1+bs*2];
13615 D[2+ldd*2] = CC[2+bs*2];
13616 D[3+ldd*2] = CC[3+bs*2];
13617
13618 if(n1==3)
13619 return;
13620
13621 D[0+ldd*3] = CC[0+bs*3];
13622 D[1+ldd*3] = CC[1+bs*3];
13623 D[2+ldd*3] = CC[2+bs*3];
13624 D[3+ldd*3] = CC[3+bs*3];
13625 }
13626 else if(m1>=3)
13627 {
13628 D[0+ldd*0] = CC[0+bs*0];
13629 D[1+ldd*0] = CC[1+bs*0];
13630 D[2+ldd*0] = CC[2+bs*0];
13631
13632 if(n1==1)
13633 return;
13634
13635 D[0+ldd*1] = CC[0+bs*1];
13636 D[1+ldd*1] = CC[1+bs*1];
13637 D[2+ldd*1] = CC[2+bs*1];
13638
13639 if(n1==2)
13640 return;
13641
13642 D[0+ldd*2] = CC[0+bs*2];
13643 D[1+ldd*2] = CC[1+bs*2];
13644 D[2+ldd*2] = CC[2+bs*2];
13645
13646 if(n1==3)
13647 return;
13648
13649 D[0+ldd*3] = CC[0+bs*3];
13650 D[1+ldd*3] = CC[1+bs*3];
13651 D[2+ldd*3] = CC[2+bs*3];
13652 }
13653 else if(m1>=2)
13654 {
13655 D[0+ldd*0] = CC[0+bs*0];
13656 D[1+ldd*0] = CC[1+bs*0];
13657
13658 if(n1==1)
13659 return;
13660
13661 D[0+ldd*1] = CC[0+bs*1];
13662 D[1+ldd*1] = CC[1+bs*1];
13663
13664 if(n1==2)
13665 return;
13666
13667 D[0+ldd*2] = CC[0+bs*2];
13668 D[1+ldd*2] = CC[1+bs*2];
13669
13670 if(n1==3)
13671 return;
13672
13673 D[0+ldd*3] = CC[0+bs*3];
13674 D[1+ldd*3] = CC[1+bs*3];
13675 }
13676 else //if(m1>=1)
13677 {
13678 D[0+ldd*0] = CC[0+bs*0];
13679
13680 if(n1==1)
13681 return;
13682
13683 D[0+ldd*1] = CC[0+bs*1];
13684
13685 if(n1==2)
13686 return;
13687
13688 D[0+ldd*2] = CC[0+bs*2];
13689
13690 if(n1==3)
13691 return;
13692
13693 D[0+ldd*3] = CC[0+bs*3];
13694 }
13695
13696 return;
13697
13698 }
13699 //#endif
13700
13701
13702
13703 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_one_4x4_tran_lib444c(int kmax,float * alpha,float * A,float * B,float * beta,float * C,float * D,int ldd)13704 void kernel_strmm_nt_ru_one_4x4_tran_lib444c(int kmax, float *alpha, float *A, float *B, float *beta, float *C, float *D, int ldd)
13705 {
13706
13707 const int bs = 4;
13708
13709 float
13710 a_0, a_1, a_2, a_3,
13711 b_0, b_1, b_2, b_3;
13712
13713 #if defined(TARGET_GENERIC)
13714 float CC[16] = {0};
13715 #else
13716 ALIGNED( float CC[16], 64 ) = {0};
13717 #endif
13718
13719 int k;
13720
13721 k = 0;
13722
13723 // k = 0
13724
13725 a_0 = A[0];
13726 a_1 = A[1];
13727 a_2 = A[2];
13728 a_3 = A[3];
13729
13730 CC[0+bs*0] += a_0;
13731 CC[1+bs*0] += a_1;
13732 CC[2+bs*0] += a_2;
13733 CC[3+bs*0] += a_3;
13734
13735 A += bs;
13736 B += bs;
13737 k += 1;
13738
13739 if(k>=kmax)
13740 goto store;
13741
13742 // k = 1
13743
13744 a_0 = A[0];
13745 a_1 = A[1];
13746 a_2 = A[2];
13747 a_3 = A[3];
13748
13749 b_0 = B[0+0*bs];
13750 CC[0+bs*0] += a_0 * b_0;
13751 CC[1+bs*0] += a_1 * b_0;
13752 CC[2+bs*0] += a_2 * b_0;
13753 CC[3+bs*0] += a_3 * b_0;
13754
13755 CC[0+bs*1] += a_0;
13756 CC[1+bs*1] += a_1;
13757 CC[2+bs*1] += a_2;
13758 CC[3+bs*1] += a_3;
13759
13760 A += bs;
13761 B += bs;
13762 k += 1;
13763
13764 if(k>=kmax)
13765 goto store;
13766
13767 // k = 2
13768
13769 a_0 = A[0];
13770 a_1 = A[1];
13771 a_2 = A[2];
13772 a_3 = A[3];
13773
13774 b_0 = B[0+0*bs];
13775 CC[0+bs*0] += a_0 * b_0;
13776 CC[1+bs*0] += a_1 * b_0;
13777 CC[2+bs*0] += a_2 * b_0;
13778 CC[3+bs*0] += a_3 * b_0;
13779
13780 b_1 = B[1+0*bs];
13781 CC[0+bs*1] += a_0 * b_1;
13782 CC[1+bs*1] += a_1 * b_1;
13783 CC[2+bs*1] += a_2 * b_1;
13784 CC[3+bs*1] += a_3 * b_1;
13785
13786 CC[0+bs*2] += a_0;
13787 CC[1+bs*2] += a_1;
13788 CC[2+bs*2] += a_2;
13789 CC[3+bs*2] += a_3;
13790
13791 A += bs;
13792 B += bs;
13793 k += 1;
13794
13795 if(k>=kmax)
13796 goto store;
13797
13798 // k = 3
13799
13800 a_0 = A[0];
13801 a_1 = A[1];
13802 a_2 = A[2];
13803 a_3 = A[3];
13804
13805 b_0 = B[0+0*bs];
13806 CC[0+bs*0] += a_0 * b_0;
13807 CC[1+bs*0] += a_1 * b_0;
13808 CC[2+bs*0] += a_2 * b_0;
13809 CC[3+bs*0] += a_3 * b_0;
13810
13811 b_1 = B[1+0*bs];
13812 CC[0+bs*1] += a_0 * b_1;
13813 CC[1+bs*1] += a_1 * b_1;
13814 CC[2+bs*1] += a_2 * b_1;
13815 CC[3+bs*1] += a_3 * b_1;
13816
13817 b_2 = B[2+0*bs];
13818 CC[0+bs*2] += a_0 * b_2;
13819 CC[1+bs*2] += a_1 * b_2;
13820 CC[2+bs*2] += a_2 * b_2;
13821 CC[3+bs*2] += a_3 * b_2;
13822
13823 CC[0+bs*3] += a_0;
13824 CC[1+bs*3] += a_1;
13825 CC[2+bs*3] += a_2;
13826 CC[3+bs*3] += a_3;
13827
13828 A += bs;
13829 B += bs;
13830 k += 1;
13831
13832 float alpha1 = 1.0;
13833 float beta1 = 1.0;
13834
13835 kernel_sgemm_nt_4x4_lib4(kmax-k, &alpha1, A, B, &beta1, CC, CC);
13836
13837 store:
13838
13839 // scale & tranpose & store
13840 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
13841 D[0+ldd*1] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
13842 D[0+ldd*2] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
13843 D[0+ldd*3] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
13844
13845 D[1+ldd*0] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
13846 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
13847 D[1+ldd*2] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
13848 D[1+ldd*3] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
13849
13850 D[2+ldd*0] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
13851 D[2+ldd*1] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
13852 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
13853 D[2+ldd*3] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
13854
13855 D[3+ldd*0] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
13856 D[3+ldd*1] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
13857 D[3+ldd*2] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
13858 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
13859
13860 return;
13861
13862 }
13863 //#endif
13864
13865
13866
13867 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_one_4x4_tran_vs_lib444c(int kmax,float * alpha,float * A,float * B,float * beta,float * C,float * D,int ldd,int m1,int n1)13868 void kernel_strmm_nt_ru_one_4x4_tran_vs_lib444c(int kmax, float *alpha, float *A, float *B, float *beta, float *C, float *D, int ldd, int m1, int n1)
13869 {
13870
13871 const int bs = 4;
13872
13873 float
13874 tmp,
13875 a_0, a_1, a_2, a_3,
13876 b_0, b_1, b_2, b_3;
13877
13878 #if defined(TARGET_GENERIC)
13879 float CC[16] = {0};
13880 #else
13881 ALIGNED( float CC[16], 64 ) = {0};
13882 #endif
13883
13884 int k;
13885
13886 k = 0;
13887
13888 // k = 0
13889
13890 a_0 = A[0];
13891 a_1 = A[1];
13892 a_2 = A[2];
13893 a_3 = A[3];
13894
13895 CC[0+bs*0] += a_0;
13896 CC[1+bs*0] += a_1;
13897 CC[2+bs*0] += a_2;
13898 CC[3+bs*0] += a_3;
13899
13900 A += bs;
13901 B += bs;
13902 k += 1;
13903
13904 if(k>=kmax)
13905 goto store;
13906
13907 // k = 1
13908
13909 a_0 = A[0];
13910 a_1 = A[1];
13911 a_2 = A[2];
13912 a_3 = A[3];
13913
13914 b_0 = B[0+0*bs];
13915 CC[0+bs*0] += a_0 * b_0;
13916 CC[1+bs*0] += a_1 * b_0;
13917 CC[2+bs*0] += a_2 * b_0;
13918 CC[3+bs*0] += a_3 * b_0;
13919
13920 CC[0+bs*1] += a_0;
13921 CC[1+bs*1] += a_1;
13922 CC[2+bs*1] += a_2;
13923 CC[3+bs*1] += a_3;
13924
13925 A += bs;
13926 B += bs;
13927 k += 1;
13928
13929 if(k>=kmax)
13930 goto store;
13931
13932 // k = 2
13933
13934 a_0 = A[0];
13935 a_1 = A[1];
13936 a_2 = A[2];
13937 a_3 = A[3];
13938
13939 b_0 = B[0+0*bs];
13940 CC[0+bs*0] += a_0 * b_0;
13941 CC[1+bs*0] += a_1 * b_0;
13942 CC[2+bs*0] += a_2 * b_0;
13943 CC[3+bs*0] += a_3 * b_0;
13944
13945 b_1 = B[1+0*bs];
13946 CC[0+bs*1] += a_0 * b_1;
13947 CC[1+bs*1] += a_1 * b_1;
13948 CC[2+bs*1] += a_2 * b_1;
13949 CC[3+bs*1] += a_3 * b_1;
13950
13951 CC[0+bs*2] += a_0;
13952 CC[1+bs*2] += a_1;
13953 CC[2+bs*2] += a_2;
13954 CC[3+bs*2] += a_3;
13955
13956 A += bs;
13957 B += bs;
13958 k += 1;
13959
13960 if(k>=kmax)
13961 goto store;
13962
13963 // k = 3
13964
13965 a_0 = A[0];
13966 a_1 = A[1];
13967 a_2 = A[2];
13968 a_3 = A[3];
13969
13970 b_0 = B[0+0*bs];
13971 CC[0+bs*0] += a_0 * b_0;
13972 CC[1+bs*0] += a_1 * b_0;
13973 CC[2+bs*0] += a_2 * b_0;
13974 CC[3+bs*0] += a_3 * b_0;
13975
13976 b_1 = B[1+0*bs];
13977 CC[0+bs*1] += a_0 * b_1;
13978 CC[1+bs*1] += a_1 * b_1;
13979 CC[2+bs*1] += a_2 * b_1;
13980 CC[3+bs*1] += a_3 * b_1;
13981
13982 b_2 = B[2+0*bs];
13983 CC[0+bs*2] += a_0 * b_2;
13984 CC[1+bs*2] += a_1 * b_2;
13985 CC[2+bs*2] += a_2 * b_2;
13986 CC[3+bs*2] += a_3 * b_2;
13987
13988 CC[0+bs*3] += a_0;
13989 CC[1+bs*3] += a_1;
13990 CC[2+bs*3] += a_2;
13991 CC[3+bs*3] += a_3;
13992
13993 A += bs;
13994 B += bs;
13995 k += 1;
13996
13997 float alpha1 = 1.0;
13998 float beta1 = 1.0;
13999
14000 kernel_sgemm_nt_4x4_lib4(kmax-k, &alpha1, A, B, &beta1, CC, CC);
14001
14002 store:
14003
14004 // scale
14005 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
14006 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
14007 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
14008 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
14009
14010 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
14011 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
14012 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
14013 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
14014
14015 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
14016 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
14017 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
14018 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
14019
14020 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
14021 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
14022 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
14023 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
14024
14025 // transpose
14026 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
14027 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
14028 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
14029
14030 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
14031 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
14032
14033 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
14034
14035 // store
14036 if(m1>=4)
14037 {
14038 D[0+ldd*0] = CC[0+bs*0];
14039 D[1+ldd*0] = CC[1+bs*0];
14040 D[2+ldd*0] = CC[2+bs*0];
14041 D[3+ldd*0] = CC[3+bs*0];
14042
14043 if(n1==1)
14044 return;
14045
14046 D[0+ldd*1] = CC[0+bs*1];
14047 D[1+ldd*1] = CC[1+bs*1];
14048 D[2+ldd*1] = CC[2+bs*1];
14049 D[3+ldd*1] = CC[3+bs*1];
14050
14051 if(n1==2)
14052 return;
14053
14054 D[0+ldd*2] = CC[0+bs*2];
14055 D[1+ldd*2] = CC[1+bs*2];
14056 D[2+ldd*2] = CC[2+bs*2];
14057 D[3+ldd*2] = CC[3+bs*2];
14058
14059 if(n1==3)
14060 return;
14061
14062 D[0+ldd*3] = CC[0+bs*3];
14063 D[1+ldd*3] = CC[1+bs*3];
14064 D[2+ldd*3] = CC[2+bs*3];
14065 D[3+ldd*3] = CC[3+bs*3];
14066 }
14067 else if(m1>=3)
14068 {
14069 D[0+ldd*0] = CC[0+bs*0];
14070 D[1+ldd*0] = CC[1+bs*0];
14071 D[2+ldd*0] = CC[2+bs*0];
14072
14073 if(n1==1)
14074 return;
14075
14076 D[0+ldd*1] = CC[0+bs*1];
14077 D[1+ldd*1] = CC[1+bs*1];
14078 D[2+ldd*1] = CC[2+bs*1];
14079
14080 if(n1==2)
14081 return;
14082
14083 D[0+ldd*2] = CC[0+bs*2];
14084 D[1+ldd*2] = CC[1+bs*2];
14085 D[2+ldd*2] = CC[2+bs*2];
14086
14087 if(n1==3)
14088 return;
14089
14090 D[0+ldd*3] = CC[0+bs*3];
14091 D[1+ldd*3] = CC[1+bs*3];
14092 D[2+ldd*3] = CC[2+bs*3];
14093 }
14094 else if(m1>=2)
14095 {
14096 D[0+ldd*0] = CC[0+bs*0];
14097 D[1+ldd*0] = CC[1+bs*0];
14098
14099 if(n1==1)
14100 return;
14101
14102 D[0+ldd*1] = CC[0+bs*1];
14103 D[1+ldd*1] = CC[1+bs*1];
14104
14105 if(n1==2)
14106 return;
14107
14108 D[0+ldd*2] = CC[0+bs*2];
14109 D[1+ldd*2] = CC[1+bs*2];
14110
14111 if(n1==3)
14112 return;
14113
14114 D[0+ldd*3] = CC[0+bs*3];
14115 D[1+ldd*3] = CC[1+bs*3];
14116 }
14117 else //if(m1>=1)
14118 {
14119 D[0+ldd*0] = CC[0+bs*0];
14120
14121 if(n1==1)
14122 return;
14123
14124 D[0+ldd*1] = CC[0+bs*1];
14125
14126 if(n1==2)
14127 return;
14128
14129 D[0+ldd*2] = CC[0+bs*2];
14130
14131 if(n1==3)
14132 return;
14133
14134 D[0+ldd*3] = CC[0+bs*3];
14135 }
14136
14137 return;
14138
14139 }
14140 //#endif
14141
14142
14143
14144 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_one_4x4_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd)14145 void kernel_strmm_nt_ru_one_4x4_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd)
14146 {
14147
14148 const int bs = 4;
14149
14150 float
14151 a_0, a_1, a_2, a_3,
14152 b_0, b_1, b_2, b_3;
14153
14154 #if defined(TARGET_GENERIC)
14155 float CC[16] = {0};
14156 #else
14157 ALIGNED( float CC[16], 64 ) = {0};
14158 #endif
14159
14160 int k;
14161
14162 k = 0;
14163
14164 // k = 0
14165
14166 a_0 = A[0];
14167 a_1 = A[1];
14168 a_2 = A[2];
14169 a_3 = A[3];
14170
14171 CC[0+bs*0] += a_0;
14172 CC[1+bs*0] += a_1;
14173 CC[2+bs*0] += a_2;
14174 CC[3+bs*0] += a_3;
14175
14176 A += bs;
14177 B += ldb;
14178 k += 1;
14179
14180 if(k>=kmax)
14181 goto store;
14182
14183 // k = 1
14184
14185 a_0 = A[0];
14186 a_1 = A[1];
14187 a_2 = A[2];
14188 a_3 = A[3];
14189
14190 b_0 = B[0+0*ldb];
14191 CC[0+bs*0] += a_0 * b_0;
14192 CC[1+bs*0] += a_1 * b_0;
14193 CC[2+bs*0] += a_2 * b_0;
14194 CC[3+bs*0] += a_3 * b_0;
14195
14196 CC[0+bs*1] += a_0;
14197 CC[1+bs*1] += a_1;
14198 CC[2+bs*1] += a_2;
14199 CC[3+bs*1] += a_3;
14200
14201 A += bs;
14202 B += ldb;
14203 k += 1;
14204
14205 if(k>=kmax)
14206 goto store;
14207
14208 // k = 2
14209
14210 a_0 = A[0];
14211 a_1 = A[1];
14212 a_2 = A[2];
14213 a_3 = A[3];
14214
14215 b_0 = B[0+0*ldb];
14216 CC[0+bs*0] += a_0 * b_0;
14217 CC[1+bs*0] += a_1 * b_0;
14218 CC[2+bs*0] += a_2 * b_0;
14219 CC[3+bs*0] += a_3 * b_0;
14220
14221 b_1 = B[1+0*ldb];
14222 CC[0+bs*1] += a_0 * b_1;
14223 CC[1+bs*1] += a_1 * b_1;
14224 CC[2+bs*1] += a_2 * b_1;
14225 CC[3+bs*1] += a_3 * b_1;
14226
14227 CC[0+bs*2] += a_0;
14228 CC[1+bs*2] += a_1;
14229 CC[2+bs*2] += a_2;
14230 CC[3+bs*2] += a_3;
14231
14232 A += bs;
14233 B += ldb;
14234 k += 1;
14235
14236 if(k>=kmax)
14237 goto store;
14238
14239 // k = 3
14240
14241 a_0 = A[0];
14242 a_1 = A[1];
14243 a_2 = A[2];
14244 a_3 = A[3];
14245
14246 b_0 = B[0+0*ldb];
14247 CC[0+bs*0] += a_0 * b_0;
14248 CC[1+bs*0] += a_1 * b_0;
14249 CC[2+bs*0] += a_2 * b_0;
14250 CC[3+bs*0] += a_3 * b_0;
14251
14252 b_1 = B[1+0*ldb];
14253 CC[0+bs*1] += a_0 * b_1;
14254 CC[1+bs*1] += a_1 * b_1;
14255 CC[2+bs*1] += a_2 * b_1;
14256 CC[3+bs*1] += a_3 * b_1;
14257
14258 b_2 = B[2+0*ldb];
14259 CC[0+bs*2] += a_0 * b_2;
14260 CC[1+bs*2] += a_1 * b_2;
14261 CC[2+bs*2] += a_2 * b_2;
14262 CC[3+bs*2] += a_3 * b_2;
14263
14264 CC[0+bs*3] += a_0;
14265 CC[1+bs*3] += a_1;
14266 CC[2+bs*3] += a_2;
14267 CC[3+bs*3] += a_3;
14268
14269 A += bs;
14270 B += ldb;
14271 k += 1;
14272
14273 store:
14274
14275 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
14276 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
14277 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
14278 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
14279
14280 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
14281 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
14282 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
14283 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
14284
14285 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
14286 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
14287 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
14288 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
14289
14290 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
14291 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
14292 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
14293 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
14294
14295 float beta1 = 1.0;
14296
14297 kernel_sgemm_nt_4x4_lib4ccc(kmax-k, alpha, A, B, ldb, &beta1, CC, bs, D, ldd);
14298
14299 return;
14300
14301 }
14302 //#endif
14303
14304
14305
14306 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_one_4x4_vs_lib4ccc(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,int m1,int n1)14307 void kernel_strmm_nt_ru_one_4x4_vs_lib4ccc(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, int m1, int n1)
14308 {
14309
14310 const int bs = 4;
14311
14312 float
14313 a_0, a_1, a_2, a_3,
14314 b_0, b_1, b_2, b_3;
14315
14316 #if defined(TARGET_GENERIC)
14317 float CC[16] = {0};
14318 #else
14319 ALIGNED( float CC[16], 64 ) = {0};
14320 #endif
14321
14322 int k;
14323
14324 k = 0;
14325
14326 // k = 0
14327
14328 a_0 = A[0];
14329 a_1 = A[1];
14330 a_2 = A[2];
14331 a_3 = A[3];
14332
14333 CC[0+bs*0] += a_0;
14334 CC[1+bs*0] += a_1;
14335 CC[2+bs*0] += a_2;
14336 CC[3+bs*0] += a_3;
14337
14338 A += bs;
14339 B += ldb;
14340 k += 1;
14341
14342 if(k>=kmax)
14343 goto store;
14344
14345 // k = 1
14346
14347 a_0 = A[0];
14348 a_1 = A[1];
14349 a_2 = A[2];
14350 a_3 = A[3];
14351
14352 b_0 = B[0+0*ldb];
14353 CC[0+bs*0] += a_0 * b_0;
14354 CC[1+bs*0] += a_1 * b_0;
14355 CC[2+bs*0] += a_2 * b_0;
14356 CC[3+bs*0] += a_3 * b_0;
14357
14358 CC[0+bs*1] += a_0;
14359 CC[1+bs*1] += a_1;
14360 CC[2+bs*1] += a_2;
14361 CC[3+bs*1] += a_3;
14362
14363 A += bs;
14364 B += ldb;
14365 k += 1;
14366
14367 if(k>=kmax)
14368 goto store;
14369
14370 // k = 2
14371
14372 a_0 = A[0];
14373 a_1 = A[1];
14374 a_2 = A[2];
14375 a_3 = A[3];
14376
14377 b_0 = B[0+0*ldb];
14378 CC[0+bs*0] += a_0 * b_0;
14379 CC[1+bs*0] += a_1 * b_0;
14380 CC[2+bs*0] += a_2 * b_0;
14381 CC[3+bs*0] += a_3 * b_0;
14382
14383 b_1 = B[1+0*ldb];
14384 CC[0+bs*1] += a_0 * b_1;
14385 CC[1+bs*1] += a_1 * b_1;
14386 CC[2+bs*1] += a_2 * b_1;
14387 CC[3+bs*1] += a_3 * b_1;
14388
14389 CC[0+bs*2] += a_0;
14390 CC[1+bs*2] += a_1;
14391 CC[2+bs*2] += a_2;
14392 CC[3+bs*2] += a_3;
14393
14394 A += bs;
14395 B += ldb;
14396 k += 1;
14397
14398 if(k>=kmax)
14399 goto store;
14400
14401 // k = 3
14402
14403 a_0 = A[0];
14404 a_1 = A[1];
14405 a_2 = A[2];
14406 a_3 = A[3];
14407
14408 b_0 = B[0+0*ldb];
14409 CC[0+bs*0] += a_0 * b_0;
14410 CC[1+bs*0] += a_1 * b_0;
14411 CC[2+bs*0] += a_2 * b_0;
14412 CC[3+bs*0] += a_3 * b_0;
14413
14414 b_1 = B[1+0*ldb];
14415 CC[0+bs*1] += a_0 * b_1;
14416 CC[1+bs*1] += a_1 * b_1;
14417 CC[2+bs*1] += a_2 * b_1;
14418 CC[3+bs*1] += a_3 * b_1;
14419
14420 b_2 = B[2+0*ldb];
14421 CC[0+bs*2] += a_0 * b_2;
14422 CC[1+bs*2] += a_1 * b_2;
14423 CC[2+bs*2] += a_2 * b_2;
14424 CC[3+bs*2] += a_3 * b_2;
14425
14426 CC[0+bs*3] += a_0;
14427 CC[1+bs*3] += a_1;
14428 CC[2+bs*3] += a_2;
14429 CC[3+bs*3] += a_3;
14430
14431 A += bs;
14432 B += ldb;
14433 k += 1;
14434
14435 store:
14436
14437 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+ldc*0];
14438 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+ldc*0];
14439 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+ldc*0];
14440 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+ldc*0];
14441
14442 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+ldc*1];
14443 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+ldc*1];
14444 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+ldc*1];
14445 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+ldc*1];
14446
14447 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+ldc*2];
14448 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+ldc*2];
14449 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+ldc*2];
14450 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+ldc*2];
14451
14452 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+ldc*3];
14453 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+ldc*3];
14454 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+ldc*3];
14455 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+ldc*3];
14456
14457 float beta1 = 1.0;
14458
14459 kernel_sgemm_nt_4x4_lib4ccc(kmax-k, alpha, A, B, ldb, &beta1, CC, bs, CC, bs);
14460
14461 if(m1>=4)
14462 {
14463 D[0+ldd*0] = CC[0+bs*0];
14464 D[1+ldd*0] = CC[1+bs*0];
14465 D[2+ldd*0] = CC[2+bs*0];
14466 D[3+ldd*0] = CC[3+bs*0];
14467
14468 if(n1==1)
14469 return;
14470
14471 D[0+ldd*1] = CC[0+bs*1];
14472 D[1+ldd*1] = CC[1+bs*1];
14473 D[2+ldd*1] = CC[2+bs*1];
14474 D[3+ldd*1] = CC[3+bs*1];
14475
14476 if(n1==2)
14477 return;
14478
14479 D[0+ldd*2] = CC[0+bs*2];
14480 D[1+ldd*2] = CC[1+bs*2];
14481 D[2+ldd*2] = CC[2+bs*2];
14482 D[3+ldd*2] = CC[3+bs*2];
14483
14484 if(n1==3)
14485 return;
14486
14487 D[0+ldd*3] = CC[0+bs*3];
14488 D[1+ldd*3] = CC[1+bs*3];
14489 D[2+ldd*3] = CC[2+bs*3];
14490 D[3+ldd*3] = CC[3+bs*3];
14491 }
14492 else if(m1>=3)
14493 {
14494 D[0+ldd*0] = CC[0+bs*0];
14495 D[1+ldd*0] = CC[1+bs*0];
14496 D[2+ldd*0] = CC[2+bs*0];
14497
14498 if(n1==1)
14499 return;
14500
14501 D[0+ldd*1] = CC[0+bs*1];
14502 D[1+ldd*1] = CC[1+bs*1];
14503 D[2+ldd*1] = CC[2+bs*1];
14504
14505 if(n1==2)
14506 return;
14507
14508 D[0+ldd*2] = CC[0+bs*2];
14509 D[1+ldd*2] = CC[1+bs*2];
14510 D[2+ldd*2] = CC[2+bs*2];
14511
14512 if(n1==3)
14513 return;
14514
14515 D[0+ldd*3] = CC[0+bs*3];
14516 D[1+ldd*3] = CC[1+bs*3];
14517 D[2+ldd*3] = CC[2+bs*3];
14518 }
14519 else if(m1>=2)
14520 {
14521 D[0+ldd*0] = CC[0+bs*0];
14522 D[1+ldd*0] = CC[1+bs*0];
14523
14524 if(n1==1)
14525 return;
14526
14527 D[0+ldd*1] = CC[0+bs*1];
14528 D[1+ldd*1] = CC[1+bs*1];
14529
14530 if(n1==2)
14531 return;
14532
14533 D[0+ldd*2] = CC[0+bs*2];
14534 D[1+ldd*2] = CC[1+bs*2];
14535
14536 if(n1==3)
14537 return;
14538
14539 D[0+ldd*3] = CC[0+bs*3];
14540 D[1+ldd*3] = CC[1+bs*3];
14541 }
14542 else //if(m1>=1)
14543 {
14544 D[0+ldd*0] = CC[0+bs*0];
14545
14546 if(n1==1)
14547 return;
14548
14549 D[0+ldd*1] = CC[0+bs*1];
14550
14551 if(n1==2)
14552 return;
14553
14554 D[0+ldd*2] = CC[0+bs*2];
14555
14556 if(n1==3)
14557 return;
14558
14559 D[0+ldd*3] = CC[0+bs*3];
14560 }
14561
14562 return;
14563
14564 }
14565 //#endif
14566
14567
14568
14569 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_one_4x4_tran_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd)14570 void kernel_strmm_nt_ru_one_4x4_tran_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd)
14571 {
14572
14573 const int bs = 4;
14574
14575 float
14576 a_0, a_1, a_2, a_3,
14577 b_0, b_1, b_2, b_3;
14578
14579 #if defined(TARGET_GENERIC)
14580 float CC[16] = {0};
14581 #else
14582 ALIGNED( float CC[16], 64 ) = {0};
14583 #endif
14584
14585 int k;
14586
14587 k = 0;
14588
14589 // k = 0
14590
14591 a_0 = A[0];
14592 a_1 = A[1];
14593 a_2 = A[2];
14594 a_3 = A[3];
14595
14596 CC[0+bs*0] += a_0;
14597 CC[1+bs*0] += a_1;
14598 CC[2+bs*0] += a_2;
14599 CC[3+bs*0] += a_3;
14600
14601 A += bs;
14602 B += ldb;
14603 k += 1;
14604
14605 if(k>=kmax)
14606 goto store;
14607
14608 // k = 1
14609
14610 a_0 = A[0];
14611 a_1 = A[1];
14612 a_2 = A[2];
14613 a_3 = A[3];
14614
14615 b_0 = B[0+0*ldb];
14616 CC[0+bs*0] += a_0 * b_0;
14617 CC[1+bs*0] += a_1 * b_0;
14618 CC[2+bs*0] += a_2 * b_0;
14619 CC[3+bs*0] += a_3 * b_0;
14620
14621 CC[0+bs*1] += a_0;
14622 CC[1+bs*1] += a_1;
14623 CC[2+bs*1] += a_2;
14624 CC[3+bs*1] += a_3;
14625
14626 A += bs;
14627 B += ldb;
14628 k += 1;
14629
14630 if(k>=kmax)
14631 goto store;
14632
14633 // k = 2
14634
14635 a_0 = A[0];
14636 a_1 = A[1];
14637 a_2 = A[2];
14638 a_3 = A[3];
14639
14640 b_0 = B[0+0*ldb];
14641 CC[0+bs*0] += a_0 * b_0;
14642 CC[1+bs*0] += a_1 * b_0;
14643 CC[2+bs*0] += a_2 * b_0;
14644 CC[3+bs*0] += a_3 * b_0;
14645
14646 b_1 = B[1+0*ldb];
14647 CC[0+bs*1] += a_0 * b_1;
14648 CC[1+bs*1] += a_1 * b_1;
14649 CC[2+bs*1] += a_2 * b_1;
14650 CC[3+bs*1] += a_3 * b_1;
14651
14652 CC[0+bs*2] += a_0;
14653 CC[1+bs*2] += a_1;
14654 CC[2+bs*2] += a_2;
14655 CC[3+bs*2] += a_3;
14656
14657 A += bs;
14658 B += ldb;
14659 k += 1;
14660
14661 if(k>=kmax)
14662 goto store;
14663
14664 // k = 3
14665
14666 a_0 = A[0];
14667 a_1 = A[1];
14668 a_2 = A[2];
14669 a_3 = A[3];
14670
14671 b_0 = B[0+0*ldb];
14672 CC[0+bs*0] += a_0 * b_0;
14673 CC[1+bs*0] += a_1 * b_0;
14674 CC[2+bs*0] += a_2 * b_0;
14675 CC[3+bs*0] += a_3 * b_0;
14676
14677 b_1 = B[1+0*ldb];
14678 CC[0+bs*1] += a_0 * b_1;
14679 CC[1+bs*1] += a_1 * b_1;
14680 CC[2+bs*1] += a_2 * b_1;
14681 CC[3+bs*1] += a_3 * b_1;
14682
14683 b_2 = B[2+0*ldb];
14684 CC[0+bs*2] += a_0 * b_2;
14685 CC[1+bs*2] += a_1 * b_2;
14686 CC[2+bs*2] += a_2 * b_2;
14687 CC[3+bs*2] += a_3 * b_2;
14688
14689 CC[0+bs*3] += a_0;
14690 CC[1+bs*3] += a_1;
14691 CC[2+bs*3] += a_2;
14692 CC[3+bs*3] += a_3;
14693
14694 A += bs;
14695 B += ldb;
14696 k += 1;
14697
14698 float alpha1 = 1.0;
14699 float beta1 = 1.0;
14700
14701 kernel_sgemm_nt_4x4_lib4ccc(kmax-k, &alpha1, A, B, ldb, &beta1, CC, bs, CC, bs);
14702
14703 store:
14704
14705 // scale & tranpose & store
14706 D[0+ldd*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
14707 D[0+ldd*1] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
14708 D[0+ldd*2] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
14709 D[0+ldd*3] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
14710
14711 D[1+ldd*0] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
14712 D[1+ldd*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
14713 D[1+ldd*2] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
14714 D[1+ldd*3] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
14715
14716 D[2+ldd*0] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
14717 D[2+ldd*1] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
14718 D[2+ldd*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
14719 D[2+ldd*3] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
14720
14721 D[3+ldd*0] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
14722 D[3+ldd*1] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
14723 D[3+ldd*2] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
14724 D[3+ldd*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
14725
14726 return;
14727
14728 }
14729 //#endif
14730
14731
14732
14733 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strmm_nt_ru_one_4x4_tran_vs_lib4c4c(int kmax,float * alpha,float * A,float * B,int ldb,float * beta,float * C,float * D,int ldd,int m1,int n1)14734 void kernel_strmm_nt_ru_one_4x4_tran_vs_lib4c4c(int kmax, float *alpha, float *A, float *B, int ldb, float *beta, float *C, float *D, int ldd, int m1, int n1)
14735 {
14736
14737 const int bs = 4;
14738
14739 float
14740 tmp,
14741 a_0, a_1, a_2, a_3,
14742 b_0, b_1, b_2, b_3;
14743
14744 #if defined(TARGET_GENERIC)
14745 float CC[16] = {0};
14746 #else
14747 ALIGNED( float CC[16], 64 ) = {0};
14748 #endif
14749
14750 int k;
14751
14752 k = 0;
14753
14754 // k = 0
14755
14756 a_0 = A[0];
14757 a_1 = A[1];
14758 a_2 = A[2];
14759 a_3 = A[3];
14760
14761 CC[0+bs*0] += a_0;
14762 CC[1+bs*0] += a_1;
14763 CC[2+bs*0] += a_2;
14764 CC[3+bs*0] += a_3;
14765
14766 A += bs;
14767 B += ldb;
14768 k += 1;
14769
14770 if(k>=kmax)
14771 goto store;
14772
14773 // k = 1
14774
14775 a_0 = A[0];
14776 a_1 = A[1];
14777 a_2 = A[2];
14778 a_3 = A[3];
14779
14780 b_0 = B[0+0*ldb];
14781 CC[0+bs*0] += a_0 * b_0;
14782 CC[1+bs*0] += a_1 * b_0;
14783 CC[2+bs*0] += a_2 * b_0;
14784 CC[3+bs*0] += a_3 * b_0;
14785
14786 CC[0+bs*1] += a_0;
14787 CC[1+bs*1] += a_1;
14788 CC[2+bs*1] += a_2;
14789 CC[3+bs*1] += a_3;
14790
14791 A += bs;
14792 B += ldb;
14793 k += 1;
14794
14795 if(k>=kmax)
14796 goto store;
14797
14798 // k = 2
14799
14800 a_0 = A[0];
14801 a_1 = A[1];
14802 a_2 = A[2];
14803 a_3 = A[3];
14804
14805 b_0 = B[0+0*ldb];
14806 CC[0+bs*0] += a_0 * b_0;
14807 CC[1+bs*0] += a_1 * b_0;
14808 CC[2+bs*0] += a_2 * b_0;
14809 CC[3+bs*0] += a_3 * b_0;
14810
14811 b_1 = B[1+0*ldb];
14812 CC[0+bs*1] += a_0 * b_1;
14813 CC[1+bs*1] += a_1 * b_1;
14814 CC[2+bs*1] += a_2 * b_1;
14815 CC[3+bs*1] += a_3 * b_1;
14816
14817 CC[0+bs*2] += a_0;
14818 CC[1+bs*2] += a_1;
14819 CC[2+bs*2] += a_2;
14820 CC[3+bs*2] += a_3;
14821
14822 A += bs;
14823 B += ldb;
14824 k += 1;
14825
14826 if(k>=kmax)
14827 goto store;
14828
14829 // k = 3
14830
14831 a_0 = A[0];
14832 a_1 = A[1];
14833 a_2 = A[2];
14834 a_3 = A[3];
14835
14836 b_0 = B[0+0*ldb];
14837 CC[0+bs*0] += a_0 * b_0;
14838 CC[1+bs*0] += a_1 * b_0;
14839 CC[2+bs*0] += a_2 * b_0;
14840 CC[3+bs*0] += a_3 * b_0;
14841
14842 b_1 = B[1+0*ldb];
14843 CC[0+bs*1] += a_0 * b_1;
14844 CC[1+bs*1] += a_1 * b_1;
14845 CC[2+bs*1] += a_2 * b_1;
14846 CC[3+bs*1] += a_3 * b_1;
14847
14848 b_2 = B[2+0*ldb];
14849 CC[0+bs*2] += a_0 * b_2;
14850 CC[1+bs*2] += a_1 * b_2;
14851 CC[2+bs*2] += a_2 * b_2;
14852 CC[3+bs*2] += a_3 * b_2;
14853
14854 CC[0+bs*3] += a_0;
14855 CC[1+bs*3] += a_1;
14856 CC[2+bs*3] += a_2;
14857 CC[3+bs*3] += a_3;
14858
14859 A += bs;
14860 B += ldb;
14861 k += 1;
14862
14863 float alpha1 = 1.0;
14864 float beta1 = 1.0;
14865
14866 kernel_sgemm_nt_4x4_lib4ccc(kmax-k, &alpha1, A, B, ldb, &beta1, CC, bs, CC, bs);
14867
14868 store:
14869
14870 // scale
14871 CC[0+bs*0] = alpha[0]*CC[0+bs*0] + beta[0]*C[0+bs*0];
14872 CC[1+bs*0] = alpha[0]*CC[1+bs*0] + beta[0]*C[1+bs*0];
14873 CC[2+bs*0] = alpha[0]*CC[2+bs*0] + beta[0]*C[2+bs*0];
14874 CC[3+bs*0] = alpha[0]*CC[3+bs*0] + beta[0]*C[3+bs*0];
14875
14876 CC[0+bs*1] = alpha[0]*CC[0+bs*1] + beta[0]*C[0+bs*1];
14877 CC[1+bs*1] = alpha[0]*CC[1+bs*1] + beta[0]*C[1+bs*1];
14878 CC[2+bs*1] = alpha[0]*CC[2+bs*1] + beta[0]*C[2+bs*1];
14879 CC[3+bs*1] = alpha[0]*CC[3+bs*1] + beta[0]*C[3+bs*1];
14880
14881 CC[0+bs*2] = alpha[0]*CC[0+bs*2] + beta[0]*C[0+bs*2];
14882 CC[1+bs*2] = alpha[0]*CC[1+bs*2] + beta[0]*C[1+bs*2];
14883 CC[2+bs*2] = alpha[0]*CC[2+bs*2] + beta[0]*C[2+bs*2];
14884 CC[3+bs*2] = alpha[0]*CC[3+bs*2] + beta[0]*C[3+bs*2];
14885
14886 CC[0+bs*3] = alpha[0]*CC[0+bs*3] + beta[0]*C[0+bs*3];
14887 CC[1+bs*3] = alpha[0]*CC[1+bs*3] + beta[0]*C[1+bs*3];
14888 CC[2+bs*3] = alpha[0]*CC[2+bs*3] + beta[0]*C[2+bs*3];
14889 CC[3+bs*3] = alpha[0]*CC[3+bs*3] + beta[0]*C[3+bs*3];
14890
14891 // transpose
14892 tmp = CC[1+bs*0]; CC[1+bs*0] = CC[0+bs*1]; CC[0+bs*1] = tmp;
14893 tmp = CC[2+bs*0]; CC[2+bs*0] = CC[0+bs*2]; CC[0+bs*2] = tmp;
14894 tmp = CC[3+bs*0]; CC[3+bs*0] = CC[0+bs*3]; CC[0+bs*3] = tmp;
14895
14896 tmp = CC[2+bs*1]; CC[2+bs*1] = CC[1+bs*2]; CC[1+bs*2] = tmp;
14897 tmp = CC[3+bs*1]; CC[3+bs*1] = CC[1+bs*3]; CC[1+bs*3] = tmp;
14898
14899 tmp = CC[3+bs*2]; CC[3+bs*2] = CC[2+bs*3]; CC[2+bs*3] = tmp;
14900
14901 // store
14902 if(m1>=4)
14903 {
14904 D[0+ldd*0] = CC[0+bs*0];
14905 D[1+ldd*0] = CC[1+bs*0];
14906 D[2+ldd*0] = CC[2+bs*0];
14907 D[3+ldd*0] = CC[3+bs*0];
14908
14909 if(n1==1)
14910 return;
14911
14912 D[0+ldd*1] = CC[0+bs*1];
14913 D[1+ldd*1] = CC[1+bs*1];
14914 D[2+ldd*1] = CC[2+bs*1];
14915 D[3+ldd*1] = CC[3+bs*1];
14916
14917 if(n1==2)
14918 return;
14919
14920 D[0+ldd*2] = CC[0+bs*2];
14921 D[1+ldd*2] = CC[1+bs*2];
14922 D[2+ldd*2] = CC[2+bs*2];
14923 D[3+ldd*2] = CC[3+bs*2];
14924
14925 if(n1==3)
14926 return;
14927
14928 D[0+ldd*3] = CC[0+bs*3];
14929 D[1+ldd*3] = CC[1+bs*3];
14930 D[2+ldd*3] = CC[2+bs*3];
14931 D[3+ldd*3] = CC[3+bs*3];
14932 }
14933 else if(m1>=3)
14934 {
14935 D[0+ldd*0] = CC[0+bs*0];
14936 D[1+ldd*0] = CC[1+bs*0];
14937 D[2+ldd*0] = CC[2+bs*0];
14938
14939 if(n1==1)
14940 return;
14941
14942 D[0+ldd*1] = CC[0+bs*1];
14943 D[1+ldd*1] = CC[1+bs*1];
14944 D[2+ldd*1] = CC[2+bs*1];
14945
14946 if(n1==2)
14947 return;
14948
14949 D[0+ldd*2] = CC[0+bs*2];
14950 D[1+ldd*2] = CC[1+bs*2];
14951 D[2+ldd*2] = CC[2+bs*2];
14952
14953 if(n1==3)
14954 return;
14955
14956 D[0+ldd*3] = CC[0+bs*3];
14957 D[1+ldd*3] = CC[1+bs*3];
14958 D[2+ldd*3] = CC[2+bs*3];
14959 }
14960 else if(m1>=2)
14961 {
14962 D[0+ldd*0] = CC[0+bs*0];
14963 D[1+ldd*0] = CC[1+bs*0];
14964
14965 if(n1==1)
14966 return;
14967
14968 D[0+ldd*1] = CC[0+bs*1];
14969 D[1+ldd*1] = CC[1+bs*1];
14970
14971 if(n1==2)
14972 return;
14973
14974 D[0+ldd*2] = CC[0+bs*2];
14975 D[1+ldd*2] = CC[1+bs*2];
14976
14977 if(n1==3)
14978 return;
14979
14980 D[0+ldd*3] = CC[0+bs*3];
14981 D[1+ldd*3] = CC[1+bs*3];
14982 }
14983 else //if(m1>=1)
14984 {
14985 D[0+ldd*0] = CC[0+bs*0];
14986
14987 if(n1==1)
14988 return;
14989
14990 D[0+ldd*1] = CC[0+bs*1];
14991
14992 if(n1==2)
14993 return;
14994
14995 D[0+ldd*2] = CC[0+bs*2];
14996
14997 if(n1==3)
14998 return;
14999
15000 D[0+ldd*3] = CC[0+bs*3];
15001 }
15002
15003 return;
15004
15005 }
15006 //#endif
15007
15008
15009
15010 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER)
kernel_spotrf_nt_l_4x4_lib44cc(int kmax,float * A,float * B,float * C,int ldc,float * D,int ldd,float * inv_diag_D)15011 void kernel_spotrf_nt_l_4x4_lib44cc(int kmax, float *A, float *B, float *C, int ldc, float *D, int ldd, float *inv_diag_D)
15012 {
15013
15014 const int bs = 4;
15015
15016 #if defined(TARGET_GENERIC)
15017 float CC[16] = {0};
15018 #else
15019 ALIGNED( float CC[16], 64 ) = {0};
15020 #endif
15021
15022 CC[0+bs*0] = C[0+ldc*0];
15023 CC[1+bs*0] = C[1+ldc*0];
15024 CC[2+bs*0] = C[2+ldc*0];
15025 CC[3+bs*0] = C[3+ldc*0];
15026
15027 CC[1+bs*1] = C[1+ldc*1];
15028 CC[2+bs*1] = C[2+ldc*1];
15029 CC[3+bs*1] = C[3+ldc*1];
15030
15031 CC[2+bs*2] = C[2+ldc*2];
15032 CC[3+bs*2] = C[3+ldc*2];
15033
15034 CC[3+bs*3] = C[3+ldc*3];
15035
15036 kernel_spotrf_nt_l_4x4_lib4(kmax, A, B, CC, CC, inv_diag_D);
15037
15038 D[0+ldd*0] = CC[0+bs*0];
15039 D[1+ldd*0] = CC[1+bs*0];
15040 D[2+ldd*0] = CC[2+bs*0];
15041 D[3+ldd*0] = CC[3+bs*0];
15042
15043 D[1+ldd*1] = CC[1+bs*1];
15044 D[2+ldd*1] = CC[2+bs*1];
15045 D[3+ldd*1] = CC[3+bs*1];
15046
15047 D[2+ldd*2] = CC[2+bs*2];
15048 D[3+ldd*2] = CC[3+bs*2];
15049
15050 D[3+ldd*3] = CC[3+bs*3];
15051
15052 return;
15053
15054 }
15055 //#endif
15056
15057
15058
15059 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_spotrf_nt_l_4x4_vs_lib44cc(int kmax,float * A,float * B,float * C,int ldc,float * D,int ldd,float * inv_diag_D,int m1,int n1)15060 void kernel_spotrf_nt_l_4x4_vs_lib44cc(int kmax, float *A, float *B, float *C, int ldc, float *D, int ldd, float *inv_diag_D, int m1, int n1)
15061 {
15062
15063 const int bs = 4;
15064
15065 #if defined(TARGET_GENERIC)
15066 float CC[16] = {0};
15067 #else
15068 ALIGNED( float CC[16], 64 ) = {0};
15069 #endif
15070
15071 if(m1>=4)
15072 {
15073 CC[0+bs*0] = C[0+ldc*0];
15074 CC[1+bs*0] = C[1+ldc*0];
15075 CC[2+bs*0] = C[2+ldc*0];
15076 CC[3+bs*0] = C[3+ldc*0];
15077
15078 if(n1==1)
15079 goto kernel;
15080
15081 CC[1+bs*1] = C[1+ldc*1];
15082 CC[2+bs*1] = C[2+ldc*1];
15083 CC[3+bs*1] = C[3+ldc*1];
15084
15085 if(n1==2)
15086 goto kernel;
15087
15088 CC[2+bs*2] = C[2+ldc*2];
15089 CC[3+bs*2] = C[3+ldc*2];
15090
15091 if(n1==3)
15092 goto kernel;
15093
15094 CC[3+bs*3] = C[3+ldc*3];
15095 }
15096 else if(m1>=3)
15097 {
15098 CC[0+bs*0] = C[0+ldc*0];
15099 CC[1+bs*0] = C[1+ldc*0];
15100 CC[2+bs*0] = C[2+ldc*0];
15101
15102 if(n1==1)
15103 goto kernel;
15104
15105 CC[1+bs*1] = C[1+ldc*1];
15106 CC[2+bs*1] = C[2+ldc*1];
15107
15108 if(n1==2)
15109 goto kernel;
15110
15111 CC[2+bs*2] = C[2+ldc*2];
15112 }
15113 else if(m1>=2)
15114 {
15115 CC[0+bs*0] = C[0+ldc*0];
15116 CC[1+bs*0] = C[1+ldc*0];
15117
15118 if(n1==1)
15119 goto kernel;
15120
15121 CC[1+bs*1] = C[1+ldc*1];
15122 }
15123 else //if(m1>=1)
15124 {
15125 CC[0+bs*0] = C[0+ldc*0];
15126 }
15127
15128 kernel:
15129 kernel_spotrf_nt_l_4x4_vs_lib4(kmax, A, B, CC, CC, inv_diag_D, m1, n1);
15130
15131 if(m1>=4)
15132 {
15133 D[0+ldd*0] = CC[0+bs*0];
15134 D[1+ldd*0] = CC[1+bs*0];
15135 D[2+ldd*0] = CC[2+bs*0];
15136 D[3+ldd*0] = CC[3+bs*0];
15137
15138 if(n1==1)
15139 goto end;
15140
15141 D[1+ldd*1] = CC[1+bs*1];
15142 D[2+ldd*1] = CC[2+bs*1];
15143 D[3+ldd*1] = CC[3+bs*1];
15144
15145 if(n1==2)
15146 goto end;
15147
15148 D[2+ldd*2] = CC[2+bs*2];
15149 D[3+ldd*2] = CC[3+bs*2];
15150
15151 if(n1==3)
15152 goto end;
15153
15154 D[3+ldd*3] = CC[3+bs*3];
15155 }
15156 else if(m1>=3)
15157 {
15158 D[0+ldd*0] = CC[0+bs*0];
15159 D[1+ldd*0] = CC[1+bs*0];
15160 D[2+ldd*0] = CC[2+bs*0];
15161
15162 if(n1==1)
15163 goto end;
15164
15165 D[1+ldd*1] = CC[1+bs*1];
15166 D[2+ldd*1] = CC[2+bs*1];
15167
15168 if(n1==2)
15169 goto end;
15170
15171 D[2+ldd*2] = CC[2+bs*2];
15172 }
15173 else if(m1>=2)
15174 {
15175 D[0+ldd*0] = CC[0+bs*0];
15176 D[1+ldd*0] = CC[1+bs*0];
15177
15178 if(n1==1)
15179 goto end;
15180
15181 D[1+ldd*1] = CC[1+bs*1];
15182 }
15183 else //if(m1>=1)
15184 {
15185 D[0+ldd*0] = CC[0+bs*0];
15186 }
15187
15188 end:
15189 return;
15190
15191 }
15192 //#endif
15193
15194
15195
15196 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_rl_inv_4x4_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,float * inv_diag_E)15197 void kernel_strsm_nn_rl_inv_4x4_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, float *inv_diag_E)
15198 {
15199
15200 const int bs = 4;
15201
15202 float tmp;
15203
15204 #if defined(TARGET_GENERIC)
15205 float CC[16] = {0};
15206 #else
15207 ALIGNED( float CC[16], 64 ) = {0};
15208 #endif
15209
15210 float alpha1 = -1.0;
15211
15212 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
15213
15214 tmp = inv_diag_E[3];
15215 CC[0+bs*3] *= tmp;
15216 CC[1+bs*3] *= tmp;
15217 CC[2+bs*3] *= tmp;
15218 CC[3+bs*3] *= tmp;
15219 tmp = E[3+lde*0];
15220 CC[0+bs*0] -= CC[0+bs*3] * tmp;
15221 CC[1+bs*0] -= CC[1+bs*3] * tmp;
15222 CC[2+bs*0] -= CC[2+bs*3] * tmp;
15223 CC[3+bs*0] -= CC[3+bs*3] * tmp;
15224 tmp = E[3+lde*1];
15225 CC[0+bs*1] -= CC[0+bs*3] * tmp;
15226 CC[1+bs*1] -= CC[1+bs*3] * tmp;
15227 CC[2+bs*1] -= CC[2+bs*3] * tmp;
15228 CC[3+bs*1] -= CC[3+bs*3] * tmp;
15229 tmp = E[3+lde*2];
15230 CC[0+bs*2] -= CC[0+bs*3] * tmp;
15231 CC[1+bs*2] -= CC[1+bs*3] * tmp;
15232 CC[2+bs*2] -= CC[2+bs*3] * tmp;
15233 CC[3+bs*2] -= CC[3+bs*3] * tmp;
15234
15235 tmp = inv_diag_E[2];
15236 CC[0+bs*2] *= tmp;
15237 CC[1+bs*2] *= tmp;
15238 CC[2+bs*2] *= tmp;
15239 CC[3+bs*2] *= tmp;
15240 tmp = E[2+lde*0];
15241 CC[0+bs*0] -= CC[0+bs*2] * tmp;
15242 CC[1+bs*0] -= CC[1+bs*2] * tmp;
15243 CC[2+bs*0] -= CC[2+bs*2] * tmp;
15244 CC[3+bs*0] -= CC[3+bs*2] * tmp;
15245 tmp = E[2+lde*1];
15246 CC[0+bs*1] -= CC[0+bs*2] * tmp;
15247 CC[1+bs*1] -= CC[1+bs*2] * tmp;
15248 CC[2+bs*1] -= CC[2+bs*2] * tmp;
15249 CC[3+bs*1] -= CC[3+bs*2] * tmp;
15250
15251 tmp = inv_diag_E[1];
15252 CC[0+bs*1] *= tmp;
15253 CC[1+bs*1] *= tmp;
15254 CC[2+bs*1] *= tmp;
15255 CC[3+bs*1] *= tmp;
15256 tmp = E[1+lde*0];
15257 CC[0+bs*0] -= CC[0+bs*1] * tmp;
15258 CC[1+bs*0] -= CC[1+bs*1] * tmp;
15259 CC[2+bs*0] -= CC[2+bs*1] * tmp;
15260 CC[3+bs*0] -= CC[3+bs*1] * tmp;
15261
15262 tmp = inv_diag_E[0];
15263 CC[0+bs*0] *= tmp;
15264 CC[1+bs*0] *= tmp;
15265 CC[2+bs*0] *= tmp;
15266 CC[3+bs*0] *= tmp;
15267
15268
15269 D[0+bs*0] = CC[0+bs*0];
15270 D[1+bs*0] = CC[1+bs*0];
15271 D[2+bs*0] = CC[2+bs*0];
15272 D[3+bs*0] = CC[3+bs*0];
15273
15274 D[0+bs*1] = CC[0+bs*1];
15275 D[1+bs*1] = CC[1+bs*1];
15276 D[2+bs*1] = CC[2+bs*1];
15277 D[3+bs*1] = CC[3+bs*1];
15278
15279 D[0+bs*2] = CC[0+bs*2];
15280 D[1+bs*2] = CC[1+bs*2];
15281 D[2+bs*2] = CC[2+bs*2];
15282 D[3+bs*2] = CC[3+bs*2];
15283
15284 D[0+bs*3] = CC[0+bs*3];
15285 D[1+bs*3] = CC[1+bs*3];
15286 D[2+bs*3] = CC[2+bs*3];
15287 D[3+bs*3] = CC[3+bs*3];
15288
15289 return;
15290
15291 }
15292 //#endif
15293
15294
15295
15296 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_rl_inv_4x4_vs_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,float * inv_diag_E,int m1,int n1)15297 void kernel_strsm_nn_rl_inv_4x4_vs_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, float *inv_diag_E, int m1, int n1)
15298 {
15299
15300 const int bs = 4;
15301
15302 float tmp;
15303
15304 #if defined(TARGET_GENERIC)
15305 float CC[16] = {0};
15306 #else
15307 ALIGNED( float CC[16], 64 ) = {0};
15308 #endif
15309
15310 float alpha1 = -1.0;
15311
15312 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
15313
15314 if(n1<=3)
15315 goto n3;
15316
15317 tmp = inv_diag_E[3];
15318 CC[0+bs*3] *= tmp;
15319 CC[1+bs*3] *= tmp;
15320 CC[2+bs*3] *= tmp;
15321 CC[3+bs*3] *= tmp;
15322 tmp = E[3+lde*0];
15323 CC[0+bs*0] -= CC[0+bs*3] * tmp;
15324 CC[1+bs*0] -= CC[1+bs*3] * tmp;
15325 CC[2+bs*0] -= CC[2+bs*3] * tmp;
15326 CC[3+bs*0] -= CC[3+bs*3] * tmp;
15327 tmp = E[3+lde*1];
15328 CC[0+bs*1] -= CC[0+bs*3] * tmp;
15329 CC[1+bs*1] -= CC[1+bs*3] * tmp;
15330 CC[2+bs*1] -= CC[2+bs*3] * tmp;
15331 CC[3+bs*1] -= CC[3+bs*3] * tmp;
15332 tmp = E[3+lde*2];
15333 CC[0+bs*2] -= CC[0+bs*3] * tmp;
15334 CC[1+bs*2] -= CC[1+bs*3] * tmp;
15335 CC[2+bs*2] -= CC[2+bs*3] * tmp;
15336 CC[3+bs*2] -= CC[3+bs*3] * tmp;
15337
15338 n3:
15339 if(n1<=2)
15340 goto n2;
15341
15342 tmp = inv_diag_E[2];
15343 CC[0+bs*2] *= tmp;
15344 CC[1+bs*2] *= tmp;
15345 CC[2+bs*2] *= tmp;
15346 CC[3+bs*2] *= tmp;
15347 tmp = E[2+lde*0];
15348 CC[0+bs*0] -= CC[0+bs*2] * tmp;
15349 CC[1+bs*0] -= CC[1+bs*2] * tmp;
15350 CC[2+bs*0] -= CC[2+bs*2] * tmp;
15351 CC[3+bs*0] -= CC[3+bs*2] * tmp;
15352 tmp = E[2+lde*1];
15353 CC[0+bs*1] -= CC[0+bs*2] * tmp;
15354 CC[1+bs*1] -= CC[1+bs*2] * tmp;
15355 CC[2+bs*1] -= CC[2+bs*2] * tmp;
15356 CC[3+bs*1] -= CC[3+bs*2] * tmp;
15357
15358 n2:
15359 if(n1<=1)
15360 goto n1;
15361
15362 tmp = inv_diag_E[1];
15363 CC[0+bs*1] *= tmp;
15364 CC[1+bs*1] *= tmp;
15365 CC[2+bs*1] *= tmp;
15366 CC[3+bs*1] *= tmp;
15367 tmp = E[1+lde*0];
15368 CC[0+bs*0] -= CC[0+bs*1] * tmp;
15369 CC[1+bs*0] -= CC[1+bs*1] * tmp;
15370 CC[2+bs*0] -= CC[2+bs*1] * tmp;
15371 CC[3+bs*0] -= CC[3+bs*1] * tmp;
15372
15373 n1:
15374
15375 tmp = inv_diag_E[0];
15376 CC[0+bs*0] *= tmp;
15377 CC[1+bs*0] *= tmp;
15378 CC[2+bs*0] *= tmp;
15379 CC[3+bs*0] *= tmp;
15380
15381 store:
15382
15383 if(m1>=4)
15384 {
15385 D[0+bs*0] = CC[0+bs*0];
15386 D[1+bs*0] = CC[1+bs*0];
15387 D[2+bs*0] = CC[2+bs*0];
15388 D[3+bs*0] = CC[3+bs*0];
15389
15390 if(n1==1)
15391 return;
15392
15393 D[0+bs*1] = CC[0+bs*1];
15394 D[1+bs*1] = CC[1+bs*1];
15395 D[2+bs*1] = CC[2+bs*1];
15396 D[3+bs*1] = CC[3+bs*1];
15397
15398 if(n1==2)
15399 return;
15400
15401 D[0+bs*2] = CC[0+bs*2];
15402 D[1+bs*2] = CC[1+bs*2];
15403 D[2+bs*2] = CC[2+bs*2];
15404 D[3+bs*2] = CC[3+bs*2];
15405
15406 if(n1==3)
15407 return;
15408
15409 D[0+bs*3] = CC[0+bs*3];
15410 D[1+bs*3] = CC[1+bs*3];
15411 D[2+bs*3] = CC[2+bs*3];
15412 D[3+bs*3] = CC[3+bs*3];
15413 }
15414 else if(m1>=3)
15415 {
15416 D[0+bs*0] = CC[0+bs*0];
15417 D[1+bs*0] = CC[1+bs*0];
15418 D[2+bs*0] = CC[2+bs*0];
15419
15420 if(n1==1)
15421 return;
15422
15423 D[0+bs*1] = CC[0+bs*1];
15424 D[1+bs*1] = CC[1+bs*1];
15425 D[2+bs*1] = CC[2+bs*1];
15426
15427 if(n1==2)
15428 return;
15429
15430 D[0+bs*2] = CC[0+bs*2];
15431 D[1+bs*2] = CC[1+bs*2];
15432 D[2+bs*2] = CC[2+bs*2];
15433
15434 if(n1==3)
15435 return;
15436
15437 D[0+bs*3] = CC[0+bs*3];
15438 D[1+bs*3] = CC[1+bs*3];
15439 D[2+bs*3] = CC[2+bs*3];
15440 }
15441 else if(m1>=2)
15442 {
15443 D[0+bs*0] = CC[0+bs*0];
15444 D[1+bs*0] = CC[1+bs*0];
15445
15446 if(n1==1)
15447 return;
15448
15449 D[0+bs*1] = CC[0+bs*1];
15450 D[1+bs*1] = CC[1+bs*1];
15451
15452 if(n1==2)
15453 return;
15454
15455 D[0+bs*2] = CC[0+bs*2];
15456 D[1+bs*2] = CC[1+bs*2];
15457
15458 if(n1==3)
15459 return;
15460
15461 D[0+bs*3] = CC[0+bs*3];
15462 D[1+bs*3] = CC[1+bs*3];
15463 }
15464 else //if(m1>=1)
15465 {
15466 D[0+bs*0] = CC[0+bs*0];
15467
15468 if(n1==1)
15469 return;
15470
15471 D[0+bs*1] = CC[0+bs*1];
15472
15473 if(n1==2)
15474 return;
15475
15476 D[0+bs*2] = CC[0+bs*2];
15477
15478 if(n1==3)
15479 return;
15480
15481 D[0+bs*3] = CC[0+bs*3];
15482 }
15483
15484 return;
15485
15486 }
15487 //#endif
15488
15489
15490
15491 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_rl_inv_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E)15492 void kernel_strsm_nn_rl_inv_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E)
15493 {
15494
15495 const int bs = 4;
15496
15497 float tmp;
15498
15499 #if defined(TARGET_GENERIC)
15500 float CC[16] = {0};
15501 #else
15502 ALIGNED( float CC[16], 64 ) = {0};
15503 #endif
15504
15505 float alpha1 = -1.0;
15506
15507 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
15508
15509 tmp = inv_diag_E[3];
15510 CC[0+bs*3] *= tmp;
15511 CC[1+bs*3] *= tmp;
15512 CC[2+bs*3] *= tmp;
15513 CC[3+bs*3] *= tmp;
15514 tmp = E[3+lde*0];
15515 CC[0+bs*0] -= CC[0+bs*3] * tmp;
15516 CC[1+bs*0] -= CC[1+bs*3] * tmp;
15517 CC[2+bs*0] -= CC[2+bs*3] * tmp;
15518 CC[3+bs*0] -= CC[3+bs*3] * tmp;
15519 tmp = E[3+lde*1];
15520 CC[0+bs*1] -= CC[0+bs*3] * tmp;
15521 CC[1+bs*1] -= CC[1+bs*3] * tmp;
15522 CC[2+bs*1] -= CC[2+bs*3] * tmp;
15523 CC[3+bs*1] -= CC[3+bs*3] * tmp;
15524 tmp = E[3+lde*2];
15525 CC[0+bs*2] -= CC[0+bs*3] * tmp;
15526 CC[1+bs*2] -= CC[1+bs*3] * tmp;
15527 CC[2+bs*2] -= CC[2+bs*3] * tmp;
15528 CC[3+bs*2] -= CC[3+bs*3] * tmp;
15529
15530 tmp = inv_diag_E[2];
15531 CC[0+bs*2] *= tmp;
15532 CC[1+bs*2] *= tmp;
15533 CC[2+bs*2] *= tmp;
15534 CC[3+bs*2] *= tmp;
15535 tmp = E[2+lde*0];
15536 CC[0+bs*0] -= CC[0+bs*2] * tmp;
15537 CC[1+bs*0] -= CC[1+bs*2] * tmp;
15538 CC[2+bs*0] -= CC[2+bs*2] * tmp;
15539 CC[3+bs*0] -= CC[3+bs*2] * tmp;
15540 tmp = E[2+lde*1];
15541 CC[0+bs*1] -= CC[0+bs*2] * tmp;
15542 CC[1+bs*1] -= CC[1+bs*2] * tmp;
15543 CC[2+bs*1] -= CC[2+bs*2] * tmp;
15544 CC[3+bs*1] -= CC[3+bs*2] * tmp;
15545
15546 tmp = inv_diag_E[1];
15547 CC[0+bs*1] *= tmp;
15548 CC[1+bs*1] *= tmp;
15549 CC[2+bs*1] *= tmp;
15550 CC[3+bs*1] *= tmp;
15551 tmp = E[1+lde*0];
15552 CC[0+bs*0] -= CC[0+bs*1] * tmp;
15553 CC[1+bs*0] -= CC[1+bs*1] * tmp;
15554 CC[2+bs*0] -= CC[2+bs*1] * tmp;
15555 CC[3+bs*0] -= CC[3+bs*1] * tmp;
15556
15557 tmp = inv_diag_E[0];
15558 CC[0+bs*0] *= tmp;
15559 CC[1+bs*0] *= tmp;
15560 CC[2+bs*0] *= tmp;
15561 CC[3+bs*0] *= tmp;
15562
15563
15564 D[0+ldd*0] = CC[0+bs*0];
15565 D[1+ldd*0] = CC[1+bs*0];
15566 D[2+ldd*0] = CC[2+bs*0];
15567 D[3+ldd*0] = CC[3+bs*0];
15568
15569 D[0+ldd*1] = CC[0+bs*1];
15570 D[1+ldd*1] = CC[1+bs*1];
15571 D[2+ldd*1] = CC[2+bs*1];
15572 D[3+ldd*1] = CC[3+bs*1];
15573
15574 D[0+ldd*2] = CC[0+bs*2];
15575 D[1+ldd*2] = CC[1+bs*2];
15576 D[2+ldd*2] = CC[2+bs*2];
15577 D[3+ldd*2] = CC[3+bs*2];
15578
15579 D[0+ldd*3] = CC[0+bs*3];
15580 D[1+ldd*3] = CC[1+bs*3];
15581 D[2+ldd*3] = CC[2+bs*3];
15582 D[3+ldd*3] = CC[3+bs*3];
15583
15584 return;
15585
15586 }
15587 //#endif
15588
15589
15590
15591 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_rl_inv_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E,int m1,int n1)15592 void kernel_strsm_nn_rl_inv_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E, int m1, int n1)
15593 {
15594
15595 const int bs = 4;
15596
15597 float tmp;
15598
15599 #if defined(TARGET_GENERIC)
15600 float CC[16] = {0};
15601 #else
15602 ALIGNED( float CC[16], 64 ) = {0};
15603 #endif
15604
15605 float alpha1 = -1.0;
15606
15607 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
15608
15609 if(n1<=3)
15610 goto n3;
15611
15612 tmp = inv_diag_E[3];
15613 CC[0+bs*3] *= tmp;
15614 CC[1+bs*3] *= tmp;
15615 CC[2+bs*3] *= tmp;
15616 CC[3+bs*3] *= tmp;
15617 tmp = E[3+lde*0];
15618 CC[0+bs*0] -= CC[0+bs*3] * tmp;
15619 CC[1+bs*0] -= CC[1+bs*3] * tmp;
15620 CC[2+bs*0] -= CC[2+bs*3] * tmp;
15621 CC[3+bs*0] -= CC[3+bs*3] * tmp;
15622 tmp = E[3+lde*1];
15623 CC[0+bs*1] -= CC[0+bs*3] * tmp;
15624 CC[1+bs*1] -= CC[1+bs*3] * tmp;
15625 CC[2+bs*1] -= CC[2+bs*3] * tmp;
15626 CC[3+bs*1] -= CC[3+bs*3] * tmp;
15627 tmp = E[3+lde*2];
15628 CC[0+bs*2] -= CC[0+bs*3] * tmp;
15629 CC[1+bs*2] -= CC[1+bs*3] * tmp;
15630 CC[2+bs*2] -= CC[2+bs*3] * tmp;
15631 CC[3+bs*2] -= CC[3+bs*3] * tmp;
15632
15633 n3:
15634 if(n1<=2)
15635 goto n2;
15636
15637 tmp = inv_diag_E[2];
15638 CC[0+bs*2] *= tmp;
15639 CC[1+bs*2] *= tmp;
15640 CC[2+bs*2] *= tmp;
15641 CC[3+bs*2] *= tmp;
15642 tmp = E[2+lde*0];
15643 CC[0+bs*0] -= CC[0+bs*2] * tmp;
15644 CC[1+bs*0] -= CC[1+bs*2] * tmp;
15645 CC[2+bs*0] -= CC[2+bs*2] * tmp;
15646 CC[3+bs*0] -= CC[3+bs*2] * tmp;
15647 tmp = E[2+lde*1];
15648 CC[0+bs*1] -= CC[0+bs*2] * tmp;
15649 CC[1+bs*1] -= CC[1+bs*2] * tmp;
15650 CC[2+bs*1] -= CC[2+bs*2] * tmp;
15651 CC[3+bs*1] -= CC[3+bs*2] * tmp;
15652
15653 n2:
15654 if(n1<=1)
15655 goto n1;
15656
15657 tmp = inv_diag_E[1];
15658 CC[0+bs*1] *= tmp;
15659 CC[1+bs*1] *= tmp;
15660 CC[2+bs*1] *= tmp;
15661 CC[3+bs*1] *= tmp;
15662 tmp = E[1+lde*0];
15663 CC[0+bs*0] -= CC[0+bs*1] * tmp;
15664 CC[1+bs*0] -= CC[1+bs*1] * tmp;
15665 CC[2+bs*0] -= CC[2+bs*1] * tmp;
15666 CC[3+bs*0] -= CC[3+bs*1] * tmp;
15667
15668 n1:
15669
15670 tmp = inv_diag_E[0];
15671 CC[0+bs*0] *= tmp;
15672 CC[1+bs*0] *= tmp;
15673 CC[2+bs*0] *= tmp;
15674 CC[3+bs*0] *= tmp;
15675
15676 store:
15677
15678 if(m1>=4)
15679 {
15680 D[0+ldd*0] = CC[0+bs*0];
15681 D[1+ldd*0] = CC[1+bs*0];
15682 D[2+ldd*0] = CC[2+bs*0];
15683 D[3+ldd*0] = CC[3+bs*0];
15684
15685 if(n1==1)
15686 return;
15687
15688 D[0+ldd*1] = CC[0+bs*1];
15689 D[1+ldd*1] = CC[1+bs*1];
15690 D[2+ldd*1] = CC[2+bs*1];
15691 D[3+ldd*1] = CC[3+bs*1];
15692
15693 if(n1==2)
15694 return;
15695
15696 D[0+ldd*2] = CC[0+bs*2];
15697 D[1+ldd*2] = CC[1+bs*2];
15698 D[2+ldd*2] = CC[2+bs*2];
15699 D[3+ldd*2] = CC[3+bs*2];
15700
15701 if(n1==3)
15702 return;
15703
15704 D[0+ldd*3] = CC[0+bs*3];
15705 D[1+ldd*3] = CC[1+bs*3];
15706 D[2+ldd*3] = CC[2+bs*3];
15707 D[3+ldd*3] = CC[3+bs*3];
15708 }
15709 else if(m1>=3)
15710 {
15711 D[0+ldd*0] = CC[0+bs*0];
15712 D[1+ldd*0] = CC[1+bs*0];
15713 D[2+ldd*0] = CC[2+bs*0];
15714
15715 if(n1==1)
15716 return;
15717
15718 D[0+ldd*1] = CC[0+bs*1];
15719 D[1+ldd*1] = CC[1+bs*1];
15720 D[2+ldd*1] = CC[2+bs*1];
15721
15722 if(n1==2)
15723 return;
15724
15725 D[0+ldd*2] = CC[0+bs*2];
15726 D[1+ldd*2] = CC[1+bs*2];
15727 D[2+ldd*2] = CC[2+bs*2];
15728
15729 if(n1==3)
15730 return;
15731
15732 D[0+ldd*3] = CC[0+bs*3];
15733 D[1+ldd*3] = CC[1+bs*3];
15734 D[2+ldd*3] = CC[2+bs*3];
15735 }
15736 else if(m1>=2)
15737 {
15738 D[0+ldd*0] = CC[0+bs*0];
15739 D[1+ldd*0] = CC[1+bs*0];
15740
15741 if(n1==1)
15742 return;
15743
15744 D[0+ldd*1] = CC[0+bs*1];
15745 D[1+ldd*1] = CC[1+bs*1];
15746
15747 if(n1==2)
15748 return;
15749
15750 D[0+ldd*2] = CC[0+bs*2];
15751 D[1+ldd*2] = CC[1+bs*2];
15752
15753 if(n1==3)
15754 return;
15755
15756 D[0+ldd*3] = CC[0+bs*3];
15757 D[1+ldd*3] = CC[1+bs*3];
15758 }
15759 else //if(m1>=1)
15760 {
15761 D[0+ldd*0] = CC[0+bs*0];
15762
15763 if(n1==1)
15764 return;
15765
15766 D[0+ldd*1] = CC[0+bs*1];
15767
15768 if(n1==2)
15769 return;
15770
15771 D[0+ldd*2] = CC[0+bs*2];
15772
15773 if(n1==3)
15774 return;
15775
15776 D[0+ldd*3] = CC[0+bs*3];
15777 }
15778
15779 return;
15780
15781 }
15782 //#endif
15783
15784
15785
15786 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_rl_one_4x4_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde)15787 void kernel_strsm_nn_rl_one_4x4_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde)
15788 {
15789
15790 const int bs = 4;
15791
15792 float tmp;
15793
15794 #if defined(TARGET_GENERIC)
15795 float CC[16] = {0};
15796 #else
15797 ALIGNED( float CC[16], 64 ) = {0};
15798 #endif
15799
15800 float alpha1 = -1.0;
15801
15802 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
15803
15804 tmp = E[3+lde*0];
15805 CC[0+bs*0] -= CC[0+bs*3] * tmp;
15806 CC[1+bs*0] -= CC[1+bs*3] * tmp;
15807 CC[2+bs*0] -= CC[2+bs*3] * tmp;
15808 CC[3+bs*0] -= CC[3+bs*3] * tmp;
15809 tmp = E[3+lde*1];
15810 CC[0+bs*1] -= CC[0+bs*3] * tmp;
15811 CC[1+bs*1] -= CC[1+bs*3] * tmp;
15812 CC[2+bs*1] -= CC[2+bs*3] * tmp;
15813 CC[3+bs*1] -= CC[3+bs*3] * tmp;
15814 tmp = E[3+lde*2];
15815 CC[0+bs*2] -= CC[0+bs*3] * tmp;
15816 CC[1+bs*2] -= CC[1+bs*3] * tmp;
15817 CC[2+bs*2] -= CC[2+bs*3] * tmp;
15818 CC[3+bs*2] -= CC[3+bs*3] * tmp;
15819
15820 tmp = E[2+lde*0];
15821 CC[0+bs*0] -= CC[0+bs*2] * tmp;
15822 CC[1+bs*0] -= CC[1+bs*2] * tmp;
15823 CC[2+bs*0] -= CC[2+bs*2] * tmp;
15824 CC[3+bs*0] -= CC[3+bs*2] * tmp;
15825 tmp = E[2+lde*1];
15826 CC[0+bs*1] -= CC[0+bs*2] * tmp;
15827 CC[1+bs*1] -= CC[1+bs*2] * tmp;
15828 CC[2+bs*1] -= CC[2+bs*2] * tmp;
15829 CC[3+bs*1] -= CC[3+bs*2] * tmp;
15830
15831 tmp = E[1+lde*0];
15832 CC[0+bs*0] -= CC[0+bs*1] * tmp;
15833 CC[1+bs*0] -= CC[1+bs*1] * tmp;
15834 CC[2+bs*0] -= CC[2+bs*1] * tmp;
15835 CC[3+bs*0] -= CC[3+bs*1] * tmp;
15836
15837
15838 D[0+bs*0] = CC[0+bs*0];
15839 D[1+bs*0] = CC[1+bs*0];
15840 D[2+bs*0] = CC[2+bs*0];
15841 D[3+bs*0] = CC[3+bs*0];
15842
15843 D[0+bs*1] = CC[0+bs*1];
15844 D[1+bs*1] = CC[1+bs*1];
15845 D[2+bs*1] = CC[2+bs*1];
15846 D[3+bs*1] = CC[3+bs*1];
15847
15848 D[0+bs*2] = CC[0+bs*2];
15849 D[1+bs*2] = CC[1+bs*2];
15850 D[2+bs*2] = CC[2+bs*2];
15851 D[3+bs*2] = CC[3+bs*2];
15852
15853 D[0+bs*3] = CC[0+bs*3];
15854 D[1+bs*3] = CC[1+bs*3];
15855 D[2+bs*3] = CC[2+bs*3];
15856 D[3+bs*3] = CC[3+bs*3];
15857
15858 return;
15859
15860 }
15861 //#endif
15862
15863
15864
15865 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_rl_one_4x4_vs_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,int m1,int n1)15866 void kernel_strsm_nn_rl_one_4x4_vs_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, int m1, int n1)
15867 {
15868
15869 const int bs = 4;
15870
15871 float tmp;
15872
15873 #if defined(TARGET_GENERIC)
15874 float CC[16] = {0};
15875 #else
15876 ALIGNED( float CC[16], 64 ) = {0};
15877 #endif
15878
15879 float alpha1 = -1.0;
15880
15881 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
15882
15883 if(n1<=3)
15884 goto n3;
15885
15886 tmp = E[3+lde*0];
15887 CC[0+bs*0] -= CC[0+bs*3] * tmp;
15888 CC[1+bs*0] -= CC[1+bs*3] * tmp;
15889 CC[2+bs*0] -= CC[2+bs*3] * tmp;
15890 CC[3+bs*0] -= CC[3+bs*3] * tmp;
15891 tmp = E[3+lde*1];
15892 CC[0+bs*1] -= CC[0+bs*3] * tmp;
15893 CC[1+bs*1] -= CC[1+bs*3] * tmp;
15894 CC[2+bs*1] -= CC[2+bs*3] * tmp;
15895 CC[3+bs*1] -= CC[3+bs*3] * tmp;
15896 tmp = E[3+lde*2];
15897 CC[0+bs*2] -= CC[0+bs*3] * tmp;
15898 CC[1+bs*2] -= CC[1+bs*3] * tmp;
15899 CC[2+bs*2] -= CC[2+bs*3] * tmp;
15900 CC[3+bs*2] -= CC[3+bs*3] * tmp;
15901
15902 n3:
15903 if(n1<=2)
15904 goto n2;
15905
15906 tmp = E[2+lde*0];
15907 CC[0+bs*0] -= CC[0+bs*2] * tmp;
15908 CC[1+bs*0] -= CC[1+bs*2] * tmp;
15909 CC[2+bs*0] -= CC[2+bs*2] * tmp;
15910 CC[3+bs*0] -= CC[3+bs*2] * tmp;
15911 tmp = E[2+lde*1];
15912 CC[0+bs*1] -= CC[0+bs*2] * tmp;
15913 CC[1+bs*1] -= CC[1+bs*2] * tmp;
15914 CC[2+bs*1] -= CC[2+bs*2] * tmp;
15915 CC[3+bs*1] -= CC[3+bs*2] * tmp;
15916
15917 n2:
15918 if(n1<=1)
15919 goto n1;
15920
15921 tmp = E[1+lde*0];
15922 CC[0+bs*0] -= CC[0+bs*1] * tmp;
15923 CC[1+bs*0] -= CC[1+bs*1] * tmp;
15924 CC[2+bs*0] -= CC[2+bs*1] * tmp;
15925 CC[3+bs*0] -= CC[3+bs*1] * tmp;
15926
15927 n1:
15928
15929 store:
15930
15931 if(m1>=4)
15932 {
15933 D[0+bs*0] = CC[0+bs*0];
15934 D[1+bs*0] = CC[1+bs*0];
15935 D[2+bs*0] = CC[2+bs*0];
15936 D[3+bs*0] = CC[3+bs*0];
15937
15938 if(n1==1)
15939 return;
15940
15941 D[0+bs*1] = CC[0+bs*1];
15942 D[1+bs*1] = CC[1+bs*1];
15943 D[2+bs*1] = CC[2+bs*1];
15944 D[3+bs*1] = CC[3+bs*1];
15945
15946 if(n1==2)
15947 return;
15948
15949 D[0+bs*2] = CC[0+bs*2];
15950 D[1+bs*2] = CC[1+bs*2];
15951 D[2+bs*2] = CC[2+bs*2];
15952 D[3+bs*2] = CC[3+bs*2];
15953
15954 if(n1==3)
15955 return;
15956
15957 D[0+bs*3] = CC[0+bs*3];
15958 D[1+bs*3] = CC[1+bs*3];
15959 D[2+bs*3] = CC[2+bs*3];
15960 D[3+bs*3] = CC[3+bs*3];
15961 }
15962 else if(m1>=3)
15963 {
15964 D[0+bs*0] = CC[0+bs*0];
15965 D[1+bs*0] = CC[1+bs*0];
15966 D[2+bs*0] = CC[2+bs*0];
15967
15968 if(n1==1)
15969 return;
15970
15971 D[0+bs*1] = CC[0+bs*1];
15972 D[1+bs*1] = CC[1+bs*1];
15973 D[2+bs*1] = CC[2+bs*1];
15974
15975 if(n1==2)
15976 return;
15977
15978 D[0+bs*2] = CC[0+bs*2];
15979 D[1+bs*2] = CC[1+bs*2];
15980 D[2+bs*2] = CC[2+bs*2];
15981
15982 if(n1==3)
15983 return;
15984
15985 D[0+bs*3] = CC[0+bs*3];
15986 D[1+bs*3] = CC[1+bs*3];
15987 D[2+bs*3] = CC[2+bs*3];
15988 }
15989 else if(m1>=2)
15990 {
15991 D[0+bs*0] = CC[0+bs*0];
15992 D[1+bs*0] = CC[1+bs*0];
15993
15994 if(n1==1)
15995 return;
15996
15997 D[0+bs*1] = CC[0+bs*1];
15998 D[1+bs*1] = CC[1+bs*1];
15999
16000 if(n1==2)
16001 return;
16002
16003 D[0+bs*2] = CC[0+bs*2];
16004 D[1+bs*2] = CC[1+bs*2];
16005
16006 if(n1==3)
16007 return;
16008
16009 D[0+bs*3] = CC[0+bs*3];
16010 D[1+bs*3] = CC[1+bs*3];
16011 }
16012 else //if(m1>=1)
16013 {
16014 D[0+bs*0] = CC[0+bs*0];
16015
16016 if(n1==1)
16017 return;
16018
16019 D[0+bs*1] = CC[0+bs*1];
16020
16021 if(n1==2)
16022 return;
16023
16024 D[0+bs*2] = CC[0+bs*2];
16025
16026 if(n1==3)
16027 return;
16028
16029 D[0+bs*3] = CC[0+bs*3];
16030 }
16031
16032 return;
16033
16034 }
16035 //#endif
16036
16037
16038
16039 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_rl_one_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde)16040 void kernel_strsm_nn_rl_one_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde)
16041 {
16042
16043 const int bs = 4;
16044
16045 float tmp;
16046
16047 #if defined(TARGET_GENERIC)
16048 float CC[16] = {0};
16049 #else
16050 ALIGNED( float CC[16], 64 ) = {0};
16051 #endif
16052
16053 float alpha1 = -1.0;
16054
16055 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
16056
16057 tmp = E[3+lde*0];
16058 CC[0+bs*0] -= CC[0+bs*3] * tmp;
16059 CC[1+bs*0] -= CC[1+bs*3] * tmp;
16060 CC[2+bs*0] -= CC[2+bs*3] * tmp;
16061 CC[3+bs*0] -= CC[3+bs*3] * tmp;
16062 tmp = E[3+lde*1];
16063 CC[0+bs*1] -= CC[0+bs*3] * tmp;
16064 CC[1+bs*1] -= CC[1+bs*3] * tmp;
16065 CC[2+bs*1] -= CC[2+bs*3] * tmp;
16066 CC[3+bs*1] -= CC[3+bs*3] * tmp;
16067 tmp = E[3+lde*2];
16068 CC[0+bs*2] -= CC[0+bs*3] * tmp;
16069 CC[1+bs*2] -= CC[1+bs*3] * tmp;
16070 CC[2+bs*2] -= CC[2+bs*3] * tmp;
16071 CC[3+bs*2] -= CC[3+bs*3] * tmp;
16072
16073 tmp = E[2+lde*0];
16074 CC[0+bs*0] -= CC[0+bs*2] * tmp;
16075 CC[1+bs*0] -= CC[1+bs*2] * tmp;
16076 CC[2+bs*0] -= CC[2+bs*2] * tmp;
16077 CC[3+bs*0] -= CC[3+bs*2] * tmp;
16078 tmp = E[2+lde*1];
16079 CC[0+bs*1] -= CC[0+bs*2] * tmp;
16080 CC[1+bs*1] -= CC[1+bs*2] * tmp;
16081 CC[2+bs*1] -= CC[2+bs*2] * tmp;
16082 CC[3+bs*1] -= CC[3+bs*2] * tmp;
16083
16084 tmp = E[1+lde*0];
16085 CC[0+bs*0] -= CC[0+bs*1] * tmp;
16086 CC[1+bs*0] -= CC[1+bs*1] * tmp;
16087 CC[2+bs*0] -= CC[2+bs*1] * tmp;
16088 CC[3+bs*0] -= CC[3+bs*1] * tmp;
16089
16090
16091 D[0+ldd*0] = CC[0+bs*0];
16092 D[1+ldd*0] = CC[1+bs*0];
16093 D[2+ldd*0] = CC[2+bs*0];
16094 D[3+ldd*0] = CC[3+bs*0];
16095
16096 D[0+ldd*1] = CC[0+bs*1];
16097 D[1+ldd*1] = CC[1+bs*1];
16098 D[2+ldd*1] = CC[2+bs*1];
16099 D[3+ldd*1] = CC[3+bs*1];
16100
16101 D[0+ldd*2] = CC[0+bs*2];
16102 D[1+ldd*2] = CC[1+bs*2];
16103 D[2+ldd*2] = CC[2+bs*2];
16104 D[3+ldd*2] = CC[3+bs*2];
16105
16106 D[0+ldd*3] = CC[0+bs*3];
16107 D[1+ldd*3] = CC[1+bs*3];
16108 D[2+ldd*3] = CC[2+bs*3];
16109 D[3+ldd*3] = CC[3+bs*3];
16110
16111 return;
16112
16113 }
16114 //#endif
16115
16116
16117
16118 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_rl_one_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,int m1,int n1)16119 void kernel_strsm_nn_rl_one_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, int m1, int n1)
16120 {
16121
16122 const int bs = 4;
16123
16124 float tmp;
16125
16126 #if defined(TARGET_GENERIC)
16127 float CC[16] = {0};
16128 #else
16129 ALIGNED( float CC[16], 64 ) = {0};
16130 #endif
16131
16132 float alpha1 = -1.0;
16133
16134 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
16135
16136 if(n1<=3)
16137 goto n3;
16138
16139 tmp = E[3+lde*0];
16140 CC[0+bs*0] -= CC[0+bs*3] * tmp;
16141 CC[1+bs*0] -= CC[1+bs*3] * tmp;
16142 CC[2+bs*0] -= CC[2+bs*3] * tmp;
16143 CC[3+bs*0] -= CC[3+bs*3] * tmp;
16144 tmp = E[3+lde*1];
16145 CC[0+bs*1] -= CC[0+bs*3] * tmp;
16146 CC[1+bs*1] -= CC[1+bs*3] * tmp;
16147 CC[2+bs*1] -= CC[2+bs*3] * tmp;
16148 CC[3+bs*1] -= CC[3+bs*3] * tmp;
16149 tmp = E[3+lde*2];
16150 CC[0+bs*2] -= CC[0+bs*3] * tmp;
16151 CC[1+bs*2] -= CC[1+bs*3] * tmp;
16152 CC[2+bs*2] -= CC[2+bs*3] * tmp;
16153 CC[3+bs*2] -= CC[3+bs*3] * tmp;
16154
16155 n3:
16156 if(n1<=2)
16157 goto n2;
16158
16159 tmp = E[2+lde*0];
16160 CC[0+bs*0] -= CC[0+bs*2] * tmp;
16161 CC[1+bs*0] -= CC[1+bs*2] * tmp;
16162 CC[2+bs*0] -= CC[2+bs*2] * tmp;
16163 CC[3+bs*0] -= CC[3+bs*2] * tmp;
16164 tmp = E[2+lde*1];
16165 CC[0+bs*1] -= CC[0+bs*2] * tmp;
16166 CC[1+bs*1] -= CC[1+bs*2] * tmp;
16167 CC[2+bs*1] -= CC[2+bs*2] * tmp;
16168 CC[3+bs*1] -= CC[3+bs*2] * tmp;
16169
16170 n2:
16171 if(n1<=1)
16172 goto n1;
16173
16174 tmp = E[1+lde*0];
16175 CC[0+bs*0] -= CC[0+bs*1] * tmp;
16176 CC[1+bs*0] -= CC[1+bs*1] * tmp;
16177 CC[2+bs*0] -= CC[2+bs*1] * tmp;
16178 CC[3+bs*0] -= CC[3+bs*1] * tmp;
16179
16180 n1:
16181
16182 store:
16183
16184 if(m1>=4)
16185 {
16186 D[0+ldd*0] = CC[0+bs*0];
16187 D[1+ldd*0] = CC[1+bs*0];
16188 D[2+ldd*0] = CC[2+bs*0];
16189 D[3+ldd*0] = CC[3+bs*0];
16190
16191 if(n1==1)
16192 return;
16193
16194 D[0+ldd*1] = CC[0+bs*1];
16195 D[1+ldd*1] = CC[1+bs*1];
16196 D[2+ldd*1] = CC[2+bs*1];
16197 D[3+ldd*1] = CC[3+bs*1];
16198
16199 if(n1==2)
16200 return;
16201
16202 D[0+ldd*2] = CC[0+bs*2];
16203 D[1+ldd*2] = CC[1+bs*2];
16204 D[2+ldd*2] = CC[2+bs*2];
16205 D[3+ldd*2] = CC[3+bs*2];
16206
16207 if(n1==3)
16208 return;
16209
16210 D[0+ldd*3] = CC[0+bs*3];
16211 D[1+ldd*3] = CC[1+bs*3];
16212 D[2+ldd*3] = CC[2+bs*3];
16213 D[3+ldd*3] = CC[3+bs*3];
16214 }
16215 else if(m1>=3)
16216 {
16217 D[0+ldd*0] = CC[0+bs*0];
16218 D[1+ldd*0] = CC[1+bs*0];
16219 D[2+ldd*0] = CC[2+bs*0];
16220
16221 if(n1==1)
16222 return;
16223
16224 D[0+ldd*1] = CC[0+bs*1];
16225 D[1+ldd*1] = CC[1+bs*1];
16226 D[2+ldd*1] = CC[2+bs*1];
16227
16228 if(n1==2)
16229 return;
16230
16231 D[0+ldd*2] = CC[0+bs*2];
16232 D[1+ldd*2] = CC[1+bs*2];
16233 D[2+ldd*2] = CC[2+bs*2];
16234
16235 if(n1==3)
16236 return;
16237
16238 D[0+ldd*3] = CC[0+bs*3];
16239 D[1+ldd*3] = CC[1+bs*3];
16240 D[2+ldd*3] = CC[2+bs*3];
16241 }
16242 else if(m1>=2)
16243 {
16244 D[0+ldd*0] = CC[0+bs*0];
16245 D[1+ldd*0] = CC[1+bs*0];
16246
16247 if(n1==1)
16248 return;
16249
16250 D[0+ldd*1] = CC[0+bs*1];
16251 D[1+ldd*1] = CC[1+bs*1];
16252
16253 if(n1==2)
16254 return;
16255
16256 D[0+ldd*2] = CC[0+bs*2];
16257 D[1+ldd*2] = CC[1+bs*2];
16258
16259 if(n1==3)
16260 return;
16261
16262 D[0+ldd*3] = CC[0+bs*3];
16263 D[1+ldd*3] = CC[1+bs*3];
16264 }
16265 else //if(m1>=1)
16266 {
16267 D[0+ldd*0] = CC[0+bs*0];
16268
16269 if(n1==1)
16270 return;
16271
16272 D[0+ldd*1] = CC[0+bs*1];
16273
16274 if(n1==2)
16275 return;
16276
16277 D[0+ldd*2] = CC[0+bs*2];
16278
16279 if(n1==3)
16280 return;
16281
16282 D[0+ldd*3] = CC[0+bs*3];
16283 }
16284
16285 return;
16286
16287 }
16288 //#endif
16289
16290
16291
16292 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_inv_4x4_lib44cc4(int kmax,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,float * E,float * inv_diag_E)16293 void kernel_strsm_nt_rl_inv_4x4_lib44cc4(int kmax, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, float *E, float *inv_diag_E)
16294 {
16295
16296 const int bs = 4;
16297
16298 float tmp;
16299
16300 #if defined(TARGET_GENERIC)
16301 float CC[16] = {0};
16302 #else
16303 ALIGNED( float CC[16], 64 ) = {0};
16304 #endif
16305
16306 float alpha1 = -1.0;
16307
16308 CC[0+bs*0] = C[0+ldc*0];
16309 CC[1+bs*0] = C[1+ldc*0];
16310 CC[2+bs*0] = C[2+ldc*0];
16311 CC[3+bs*0] = C[3+ldc*0];
16312
16313 CC[0+bs*1] = C[0+ldc*1];
16314 CC[1+bs*1] = C[1+ldc*1];
16315 CC[2+bs*1] = C[2+ldc*1];
16316 CC[3+bs*1] = C[3+ldc*1];
16317
16318 CC[0+bs*2] = C[0+ldc*2];
16319 CC[1+bs*2] = C[1+ldc*2];
16320 CC[2+bs*2] = C[2+ldc*2];
16321 CC[3+bs*2] = C[3+ldc*2];
16322
16323 CC[0+bs*3] = C[0+ldc*3];
16324 CC[1+bs*3] = C[1+ldc*3];
16325 CC[2+bs*3] = C[2+ldc*3];
16326 CC[3+bs*3] = C[3+ldc*3];
16327
16328 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, beta, CC, CC);
16329
16330 tmp = inv_diag_E[0];
16331 CC[0+bs*0] *= tmp;
16332 CC[1+bs*0] *= tmp;
16333 CC[2+bs*0] *= tmp;
16334 CC[3+bs*0] *= tmp;
16335
16336 tmp = E[1+bs*0];
16337 CC[0+bs*1] -= CC[0+bs*0] * tmp;
16338 CC[1+bs*1] -= CC[1+bs*0] * tmp;
16339 CC[2+bs*1] -= CC[2+bs*0] * tmp;
16340 CC[3+bs*1] -= CC[3+bs*0] * tmp;
16341 tmp = inv_diag_E[1];
16342 CC[0+bs*1] *= tmp;
16343 CC[1+bs*1] *= tmp;
16344 CC[2+bs*1] *= tmp;
16345 CC[3+bs*1] *= tmp;
16346
16347 tmp = E[2+bs*0];
16348 CC[0+bs*2] -= CC[0+bs*0] * tmp;
16349 CC[1+bs*2] -= CC[1+bs*0] * tmp;
16350 CC[2+bs*2] -= CC[2+bs*0] * tmp;
16351 CC[3+bs*2] -= CC[3+bs*0] * tmp;
16352 tmp = E[2+bs*1];
16353 CC[0+bs*2] -= CC[0+bs*1] * tmp;
16354 CC[1+bs*2] -= CC[1+bs*1] * tmp;
16355 CC[2+bs*2] -= CC[2+bs*1] * tmp;
16356 CC[3+bs*2] -= CC[3+bs*1] * tmp;
16357 tmp = inv_diag_E[2];
16358 CC[0+bs*2] *= tmp;
16359 CC[1+bs*2] *= tmp;
16360 CC[2+bs*2] *= tmp;
16361 CC[3+bs*2] *= tmp;
16362
16363 tmp = E[3+bs*0];
16364 CC[0+bs*3] -= CC[0+bs*0] * tmp;
16365 CC[1+bs*3] -= CC[1+bs*0] * tmp;
16366 CC[2+bs*3] -= CC[2+bs*0] * tmp;
16367 CC[3+bs*3] -= CC[3+bs*0] * tmp;
16368 tmp = E[3+bs*1];
16369 CC[0+bs*3] -= CC[0+bs*1] * tmp;
16370 CC[1+bs*3] -= CC[1+bs*1] * tmp;
16371 CC[2+bs*3] -= CC[2+bs*1] * tmp;
16372 CC[3+bs*3] -= CC[3+bs*1] * tmp;
16373 tmp = E[3+bs*2];
16374 CC[0+bs*3] -= CC[0+bs*2] * tmp;
16375 CC[1+bs*3] -= CC[1+bs*2] * tmp;
16376 CC[2+bs*3] -= CC[2+bs*2] * tmp;
16377 CC[3+bs*3] -= CC[3+bs*2] * tmp;
16378 tmp = inv_diag_E[3];
16379 CC[0+bs*3] *= tmp;
16380 CC[1+bs*3] *= tmp;
16381 CC[2+bs*3] *= tmp;
16382 CC[3+bs*3] *= tmp;
16383
16384 D[0+ldd*0] = CC[0+bs*0];
16385 D[1+ldd*0] = CC[1+bs*0];
16386 D[2+ldd*0] = CC[2+bs*0];
16387 D[3+ldd*0] = CC[3+bs*0];
16388
16389 D[0+ldd*1] = CC[0+bs*1];
16390 D[1+ldd*1] = CC[1+bs*1];
16391 D[2+ldd*1] = CC[2+bs*1];
16392 D[3+ldd*1] = CC[3+bs*1];
16393
16394 D[0+ldd*2] = CC[0+bs*2];
16395 D[1+ldd*2] = CC[1+bs*2];
16396 D[2+ldd*2] = CC[2+bs*2];
16397 D[3+ldd*2] = CC[3+bs*2];
16398
16399 D[0+ldd*3] = CC[0+bs*3];
16400 D[1+ldd*3] = CC[1+bs*3];
16401 D[2+ldd*3] = CC[2+bs*3];
16402 D[3+ldd*3] = CC[3+bs*3];
16403
16404 return;
16405
16406 }
16407 //#endif
16408
16409
16410
16411 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_inv_4x4_vs_lib44cc4(int kmax,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,float * E,float * inv_diag_E,int m1,int n1)16412 void kernel_strsm_nt_rl_inv_4x4_vs_lib44cc4(int kmax, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, float *E, float *inv_diag_E, int m1, int n1)
16413 {
16414
16415 const int bs = 4;
16416
16417 float tmp;
16418
16419 #if defined(TARGET_GENERIC)
16420 float CC[16] = {0};
16421 #else
16422 ALIGNED( float CC[16], 64 ) = {0};
16423 #endif
16424
16425 float alpha1 = -1.0;
16426
16427 if(m1>=4)
16428 {
16429 CC[0+bs*0] = C[0+ldc*0];
16430 CC[1+bs*0] = C[1+ldc*0];
16431 CC[2+bs*0] = C[2+ldc*0];
16432 CC[3+bs*0] = C[3+ldc*0];
16433
16434 if(n1==1)
16435 goto kernel;
16436
16437 CC[0+bs*1] = C[0+ldc*1];
16438 CC[1+bs*1] = C[1+ldc*1];
16439 CC[2+bs*1] = C[2+ldc*1];
16440 CC[3+bs*1] = C[3+ldc*1];
16441
16442 if(n1==2)
16443 goto kernel;
16444
16445 CC[0+bs*2] = C[0+ldc*2];
16446 CC[1+bs*2] = C[1+ldc*2];
16447 CC[2+bs*2] = C[2+ldc*2];
16448 CC[3+bs*2] = C[3+ldc*2];
16449
16450 if(n1==3)
16451 goto kernel;
16452
16453 CC[0+bs*3] = C[0+ldc*3];
16454 CC[1+bs*3] = C[1+ldc*3];
16455 CC[2+bs*3] = C[2+ldc*3];
16456 CC[3+bs*3] = C[3+ldc*3];
16457 }
16458 else if(m1>=3)
16459 {
16460 CC[0+bs*0] = C[0+ldc*0];
16461 CC[1+bs*0] = C[1+ldc*0];
16462 CC[2+bs*0] = C[2+ldc*0];
16463
16464 if(n1==1)
16465 goto kernel;
16466
16467 CC[0+bs*1] = C[0+ldc*1];
16468 CC[1+bs*1] = C[1+ldc*1];
16469 CC[2+bs*1] = C[2+ldc*1];
16470
16471 if(n1==2)
16472 goto kernel;
16473
16474 CC[0+bs*2] = C[0+ldc*2];
16475 CC[1+bs*2] = C[1+ldc*2];
16476 CC[2+bs*2] = C[2+ldc*2];
16477
16478 if(n1==3)
16479 goto kernel;
16480
16481 CC[0+bs*3] = C[0+ldc*3];
16482 CC[1+bs*3] = C[1+ldc*3];
16483 CC[2+bs*3] = C[2+ldc*3];
16484 }
16485 else if(m1>=2)
16486 {
16487 CC[0+bs*0] = C[0+ldc*0];
16488 CC[1+bs*0] = C[1+ldc*0];
16489
16490 if(n1==1)
16491 goto kernel;
16492
16493 CC[0+bs*1] = C[0+ldc*1];
16494 CC[1+bs*1] = C[1+ldc*1];
16495
16496 if(n1==2)
16497 goto kernel;
16498
16499 CC[0+bs*2] = C[0+ldc*2];
16500 CC[1+bs*2] = C[1+ldc*2];
16501
16502 if(n1==3)
16503 goto kernel;
16504
16505 CC[0+bs*3] = C[0+ldc*3];
16506 CC[1+bs*3] = C[1+ldc*3];
16507 }
16508 else //if(m1>=1)
16509 {
16510 CC[0+bs*0] = C[0+ldc*0];
16511
16512 if(n1==1)
16513 goto kernel;
16514
16515 CC[0+bs*1] = C[0+ldc*1];
16516
16517 if(n1==2)
16518 goto kernel;
16519
16520 CC[0+bs*2] = C[0+ldc*2];
16521
16522 if(n1==3)
16523 goto kernel;
16524
16525 CC[0+bs*3] = C[0+ldc*3];
16526 }
16527
16528 kernel:
16529 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, beta, CC, CC);
16530
16531 tmp = inv_diag_E[0];
16532 CC[0+bs*0] *= tmp;
16533 CC[1+bs*0] *= tmp;
16534 CC[2+bs*0] *= tmp;
16535 CC[3+bs*0] *= tmp;
16536
16537 if(n1==1)
16538 goto store;
16539
16540 tmp = E[1+bs*0];
16541 CC[0+bs*1] -= CC[0+bs*0] * tmp;
16542 CC[1+bs*1] -= CC[1+bs*0] * tmp;
16543 CC[2+bs*1] -= CC[2+bs*0] * tmp;
16544 CC[3+bs*1] -= CC[3+bs*0] * tmp;
16545 tmp = inv_diag_E[1];
16546 CC[0+bs*1] *= tmp;
16547 CC[1+bs*1] *= tmp;
16548 CC[2+bs*1] *= tmp;
16549 CC[3+bs*1] *= tmp;
16550
16551 if(n1==2)
16552 goto store;
16553
16554 tmp = E[2+bs*0];
16555 CC[0+bs*2] -= CC[0+bs*0] * tmp;
16556 CC[1+bs*2] -= CC[1+bs*0] * tmp;
16557 CC[2+bs*2] -= CC[2+bs*0] * tmp;
16558 CC[3+bs*2] -= CC[3+bs*0] * tmp;
16559 tmp = E[2+bs*1];
16560 CC[0+bs*2] -= CC[0+bs*1] * tmp;
16561 CC[1+bs*2] -= CC[1+bs*1] * tmp;
16562 CC[2+bs*2] -= CC[2+bs*1] * tmp;
16563 CC[3+bs*2] -= CC[3+bs*1] * tmp;
16564 tmp = inv_diag_E[2];
16565 CC[0+bs*2] *= tmp;
16566 CC[1+bs*2] *= tmp;
16567 CC[2+bs*2] *= tmp;
16568 CC[3+bs*2] *= tmp;
16569
16570 if(n1==3)
16571 goto store;
16572
16573 tmp = E[3+bs*0];
16574 CC[0+bs*3] -= CC[0+bs*0] * tmp;
16575 CC[1+bs*3] -= CC[1+bs*0] * tmp;
16576 CC[2+bs*3] -= CC[2+bs*0] * tmp;
16577 CC[3+bs*3] -= CC[3+bs*0] * tmp;
16578 tmp = E[3+bs*1];
16579 CC[0+bs*3] -= CC[0+bs*1] * tmp;
16580 CC[1+bs*3] -= CC[1+bs*1] * tmp;
16581 CC[2+bs*3] -= CC[2+bs*1] * tmp;
16582 CC[3+bs*3] -= CC[3+bs*1] * tmp;
16583 tmp = E[3+bs*2];
16584 CC[0+bs*3] -= CC[0+bs*2] * tmp;
16585 CC[1+bs*3] -= CC[1+bs*2] * tmp;
16586 CC[2+bs*3] -= CC[2+bs*2] * tmp;
16587 CC[3+bs*3] -= CC[3+bs*2] * tmp;
16588 tmp = inv_diag_E[3];
16589 CC[0+bs*3] *= tmp;
16590 CC[1+bs*3] *= tmp;
16591 CC[2+bs*3] *= tmp;
16592 CC[3+bs*3] *= tmp;
16593
16594 store:
16595
16596 if(m1>=4)
16597 {
16598 D[0+ldd*0] = CC[0+bs*0];
16599 D[1+ldd*0] = CC[1+bs*0];
16600 D[2+ldd*0] = CC[2+bs*0];
16601 D[3+ldd*0] = CC[3+bs*0];
16602
16603 if(n1==1)
16604 return;
16605
16606 D[0+ldd*1] = CC[0+bs*1];
16607 D[1+ldd*1] = CC[1+bs*1];
16608 D[2+ldd*1] = CC[2+bs*1];
16609 D[3+ldd*1] = CC[3+bs*1];
16610
16611 if(n1==2)
16612 return;
16613
16614 D[0+ldd*2] = CC[0+bs*2];
16615 D[1+ldd*2] = CC[1+bs*2];
16616 D[2+ldd*2] = CC[2+bs*2];
16617 D[3+ldd*2] = CC[3+bs*2];
16618
16619 if(n1==3)
16620 return;
16621
16622 D[0+ldd*3] = CC[0+bs*3];
16623 D[1+ldd*3] = CC[1+bs*3];
16624 D[2+ldd*3] = CC[2+bs*3];
16625 D[3+ldd*3] = CC[3+bs*3];
16626 }
16627 else if(m1>=3)
16628 {
16629 D[0+ldd*0] = CC[0+bs*0];
16630 D[1+ldd*0] = CC[1+bs*0];
16631 D[2+ldd*0] = CC[2+bs*0];
16632
16633 if(n1==1)
16634 return;
16635
16636 D[0+ldd*1] = CC[0+bs*1];
16637 D[1+ldd*1] = CC[1+bs*1];
16638 D[2+ldd*1] = CC[2+bs*1];
16639
16640 if(n1==2)
16641 return;
16642
16643 D[0+ldd*2] = CC[0+bs*2];
16644 D[1+ldd*2] = CC[1+bs*2];
16645 D[2+ldd*2] = CC[2+bs*2];
16646
16647 if(n1==3)
16648 return;
16649
16650 D[0+ldd*3] = CC[0+bs*3];
16651 D[1+ldd*3] = CC[1+bs*3];
16652 D[2+ldd*3] = CC[2+bs*3];
16653 }
16654 else if(m1>=2)
16655 {
16656 D[0+ldd*0] = CC[0+bs*0];
16657 D[1+ldd*0] = CC[1+bs*0];
16658
16659 if(n1==1)
16660 return;
16661
16662 D[0+ldd*1] = CC[0+bs*1];
16663 D[1+ldd*1] = CC[1+bs*1];
16664
16665 if(n1==2)
16666 return;
16667
16668 D[0+ldd*2] = CC[0+bs*2];
16669 D[1+ldd*2] = CC[1+bs*2];
16670
16671 if(n1==3)
16672 return;
16673
16674 D[0+ldd*3] = CC[0+bs*3];
16675 D[1+ldd*3] = CC[1+bs*3];
16676 }
16677 else //if(m1>=1)
16678 {
16679 D[0+ldd*0] = CC[0+bs*0];
16680
16681 if(n1==1)
16682 return;
16683
16684 D[0+ldd*1] = CC[0+bs*1];
16685
16686 if(n1==2)
16687 return;
16688
16689 D[0+ldd*2] = CC[0+bs*2];
16690
16691 if(n1==3)
16692 return;
16693
16694 D[0+ldd*3] = CC[0+bs*3];
16695 }
16696
16697 return;
16698
16699 }
16700 //#endif
16701
16702
16703
16704 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_AMD_BULLDOZER)
kernel_strsm_nt_rl_inv_4x4_lib44ccc(int kmax,float * A,float * B,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E)16705 void kernel_strsm_nt_rl_inv_4x4_lib44ccc(int kmax, float *A, float *B, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E)
16706 {
16707
16708 const int bs = 4;
16709
16710 float tmp;
16711
16712 #if defined(TARGET_GENERIC)
16713 float CC[16] = {0};
16714 #else
16715 ALIGNED( float CC[16], 64 ) = {0};
16716 #endif
16717
16718 float alpha1 = -1.0;
16719 float beta1 = 1.0;
16720
16721 CC[0+bs*0] = C[0+ldc*0];
16722 CC[1+bs*0] = C[1+ldc*0];
16723 CC[2+bs*0] = C[2+ldc*0];
16724 CC[3+bs*0] = C[3+ldc*0];
16725
16726 CC[0+bs*1] = C[0+ldc*1];
16727 CC[1+bs*1] = C[1+ldc*1];
16728 CC[2+bs*1] = C[2+ldc*1];
16729 CC[3+bs*1] = C[3+ldc*1];
16730
16731 CC[0+bs*2] = C[0+ldc*2];
16732 CC[1+bs*2] = C[1+ldc*2];
16733 CC[2+bs*2] = C[2+ldc*2];
16734 CC[3+bs*2] = C[3+ldc*2];
16735
16736 CC[0+bs*3] = C[0+ldc*3];
16737 CC[1+bs*3] = C[1+ldc*3];
16738 CC[2+bs*3] = C[2+ldc*3];
16739 CC[3+bs*3] = C[3+ldc*3];
16740
16741 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
16742
16743 tmp = inv_diag_E[0];
16744 CC[0+bs*0] *= tmp;
16745 CC[1+bs*0] *= tmp;
16746 CC[2+bs*0] *= tmp;
16747 CC[3+bs*0] *= tmp;
16748
16749 tmp = E[1+lde*0];
16750 CC[0+bs*1] -= CC[0+bs*0] * tmp;
16751 CC[1+bs*1] -= CC[1+bs*0] * tmp;
16752 CC[2+bs*1] -= CC[2+bs*0] * tmp;
16753 CC[3+bs*1] -= CC[3+bs*0] * tmp;
16754 tmp = inv_diag_E[1];
16755 CC[0+bs*1] *= tmp;
16756 CC[1+bs*1] *= tmp;
16757 CC[2+bs*1] *= tmp;
16758 CC[3+bs*1] *= tmp;
16759
16760 tmp = E[2+lde*0];
16761 CC[0+bs*2] -= CC[0+bs*0] * tmp;
16762 CC[1+bs*2] -= CC[1+bs*0] * tmp;
16763 CC[2+bs*2] -= CC[2+bs*0] * tmp;
16764 CC[3+bs*2] -= CC[3+bs*0] * tmp;
16765 tmp = E[2+lde*1];
16766 CC[0+bs*2] -= CC[0+bs*1] * tmp;
16767 CC[1+bs*2] -= CC[1+bs*1] * tmp;
16768 CC[2+bs*2] -= CC[2+bs*1] * tmp;
16769 CC[3+bs*2] -= CC[3+bs*1] * tmp;
16770 tmp = inv_diag_E[2];
16771 CC[0+bs*2] *= tmp;
16772 CC[1+bs*2] *= tmp;
16773 CC[2+bs*2] *= tmp;
16774 CC[3+bs*2] *= tmp;
16775
16776 tmp = E[3+lde*0];
16777 CC[0+bs*3] -= CC[0+bs*0] * tmp;
16778 CC[1+bs*3] -= CC[1+bs*0] * tmp;
16779 CC[2+bs*3] -= CC[2+bs*0] * tmp;
16780 CC[3+bs*3] -= CC[3+bs*0] * tmp;
16781 tmp = E[3+lde*1];
16782 CC[0+bs*3] -= CC[0+bs*1] * tmp;
16783 CC[1+bs*3] -= CC[1+bs*1] * tmp;
16784 CC[2+bs*3] -= CC[2+bs*1] * tmp;
16785 CC[3+bs*3] -= CC[3+bs*1] * tmp;
16786 tmp = E[3+lde*2];
16787 CC[0+bs*3] -= CC[0+bs*2] * tmp;
16788 CC[1+bs*3] -= CC[1+bs*2] * tmp;
16789 CC[2+bs*3] -= CC[2+bs*2] * tmp;
16790 CC[3+bs*3] -= CC[3+bs*2] * tmp;
16791 tmp = inv_diag_E[3];
16792 CC[0+bs*3] *= tmp;
16793 CC[1+bs*3] *= tmp;
16794 CC[2+bs*3] *= tmp;
16795 CC[3+bs*3] *= tmp;
16796
16797 D[0+ldd*0] = CC[0+bs*0];
16798 D[1+ldd*0] = CC[1+bs*0];
16799 D[2+ldd*0] = CC[2+bs*0];
16800 D[3+ldd*0] = CC[3+bs*0];
16801
16802 D[0+ldd*1] = CC[0+bs*1];
16803 D[1+ldd*1] = CC[1+bs*1];
16804 D[2+ldd*1] = CC[2+bs*1];
16805 D[3+ldd*1] = CC[3+bs*1];
16806
16807 D[0+ldd*2] = CC[0+bs*2];
16808 D[1+ldd*2] = CC[1+bs*2];
16809 D[2+ldd*2] = CC[2+bs*2];
16810 D[3+ldd*2] = CC[3+bs*2];
16811
16812 D[0+ldd*3] = CC[0+bs*3];
16813 D[1+ldd*3] = CC[1+bs*3];
16814 D[2+ldd*3] = CC[2+bs*3];
16815 D[3+ldd*3] = CC[3+bs*3];
16816
16817 return;
16818
16819 }
16820 //#endif
16821
16822
16823
16824 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_strsm_nt_rl_inv_4x4_vs_lib44ccc(int kmax,float * A,float * B,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E,int m1,int n1)16825 void kernel_strsm_nt_rl_inv_4x4_vs_lib44ccc(int kmax, float *A, float *B, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E, int m1, int n1)
16826 {
16827
16828 const int bs = 4;
16829
16830 float tmp;
16831
16832 #if defined(TARGET_GENERIC)
16833 float CC[16] = {0};
16834 #else
16835 ALIGNED( float CC[16], 64 ) = {0};
16836 #endif
16837
16838 float alpha1 = -1.0;
16839 float beta1 = 1.0;
16840
16841 if(m1>=4)
16842 {
16843 CC[0+bs*0] = C[0+ldc*0];
16844 CC[1+bs*0] = C[1+ldc*0];
16845 CC[2+bs*0] = C[2+ldc*0];
16846 CC[3+bs*0] = C[3+ldc*0];
16847
16848 if(n1==1)
16849 goto kernel;
16850
16851 CC[0+bs*1] = C[0+ldc*1];
16852 CC[1+bs*1] = C[1+ldc*1];
16853 CC[2+bs*1] = C[2+ldc*1];
16854 CC[3+bs*1] = C[3+ldc*1];
16855
16856 if(n1==2)
16857 goto kernel;
16858
16859 CC[0+bs*2] = C[0+ldc*2];
16860 CC[1+bs*2] = C[1+ldc*2];
16861 CC[2+bs*2] = C[2+ldc*2];
16862 CC[3+bs*2] = C[3+ldc*2];
16863
16864 if(n1==3)
16865 goto kernel;
16866
16867 CC[0+bs*3] = C[0+ldc*3];
16868 CC[1+bs*3] = C[1+ldc*3];
16869 CC[2+bs*3] = C[2+ldc*3];
16870 CC[3+bs*3] = C[3+ldc*3];
16871 }
16872 else if(m1>=3)
16873 {
16874 CC[0+bs*0] = C[0+ldc*0];
16875 CC[1+bs*0] = C[1+ldc*0];
16876 CC[2+bs*0] = C[2+ldc*0];
16877
16878 if(n1==1)
16879 goto kernel;
16880
16881 CC[0+bs*1] = C[0+ldc*1];
16882 CC[1+bs*1] = C[1+ldc*1];
16883 CC[2+bs*1] = C[2+ldc*1];
16884
16885 if(n1==2)
16886 goto kernel;
16887
16888 CC[0+bs*2] = C[0+ldc*2];
16889 CC[1+bs*2] = C[1+ldc*2];
16890 CC[2+bs*2] = C[2+ldc*2];
16891
16892 if(n1==3)
16893 goto kernel;
16894
16895 CC[0+bs*3] = C[0+ldc*3];
16896 CC[1+bs*3] = C[1+ldc*3];
16897 CC[2+bs*3] = C[2+ldc*3];
16898 }
16899 else if(m1>=2)
16900 {
16901 CC[0+bs*0] = C[0+ldc*0];
16902 CC[1+bs*0] = C[1+ldc*0];
16903
16904 if(n1==1)
16905 goto kernel;
16906
16907 CC[0+bs*1] = C[0+ldc*1];
16908 CC[1+bs*1] = C[1+ldc*1];
16909
16910 if(n1==2)
16911 goto kernel;
16912
16913 CC[0+bs*2] = C[0+ldc*2];
16914 CC[1+bs*2] = C[1+ldc*2];
16915
16916 if(n1==3)
16917 goto kernel;
16918
16919 CC[0+bs*3] = C[0+ldc*3];
16920 CC[1+bs*3] = C[1+ldc*3];
16921 }
16922 else //if(m1>=1)
16923 {
16924 CC[0+bs*0] = C[0+ldc*0];
16925
16926 if(n1==1)
16927 goto kernel;
16928
16929 CC[0+bs*1] = C[0+ldc*1];
16930
16931 if(n1==2)
16932 goto kernel;
16933
16934 CC[0+bs*2] = C[0+ldc*2];
16935
16936 if(n1==3)
16937 goto kernel;
16938
16939 CC[0+bs*3] = C[0+ldc*3];
16940 }
16941
16942 kernel:
16943 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, &beta1, CC, CC);
16944
16945 tmp = inv_diag_E[0];
16946 CC[0+bs*0] *= tmp;
16947 CC[1+bs*0] *= tmp;
16948 CC[2+bs*0] *= tmp;
16949 CC[3+bs*0] *= tmp;
16950
16951 if(n1==1)
16952 goto store;
16953
16954 tmp = E[1+lde*0];
16955 CC[0+bs*1] -= CC[0+bs*0] * tmp;
16956 CC[1+bs*1] -= CC[1+bs*0] * tmp;
16957 CC[2+bs*1] -= CC[2+bs*0] * tmp;
16958 CC[3+bs*1] -= CC[3+bs*0] * tmp;
16959 tmp = inv_diag_E[1];
16960 CC[0+bs*1] *= tmp;
16961 CC[1+bs*1] *= tmp;
16962 CC[2+bs*1] *= tmp;
16963 CC[3+bs*1] *= tmp;
16964
16965 if(n1==2)
16966 goto store;
16967
16968 tmp = E[2+lde*0];
16969 CC[0+bs*2] -= CC[0+bs*0] * tmp;
16970 CC[1+bs*2] -= CC[1+bs*0] * tmp;
16971 CC[2+bs*2] -= CC[2+bs*0] * tmp;
16972 CC[3+bs*2] -= CC[3+bs*0] * tmp;
16973 tmp = E[2+lde*1];
16974 CC[0+bs*2] -= CC[0+bs*1] * tmp;
16975 CC[1+bs*2] -= CC[1+bs*1] * tmp;
16976 CC[2+bs*2] -= CC[2+bs*1] * tmp;
16977 CC[3+bs*2] -= CC[3+bs*1] * tmp;
16978 tmp = inv_diag_E[2];
16979 CC[0+bs*2] *= tmp;
16980 CC[1+bs*2] *= tmp;
16981 CC[2+bs*2] *= tmp;
16982 CC[3+bs*2] *= tmp;
16983
16984 if(n1==3)
16985 goto store;
16986
16987 tmp = E[3+lde*0];
16988 CC[0+bs*3] -= CC[0+bs*0] * tmp;
16989 CC[1+bs*3] -= CC[1+bs*0] * tmp;
16990 CC[2+bs*3] -= CC[2+bs*0] * tmp;
16991 CC[3+bs*3] -= CC[3+bs*0] * tmp;
16992 tmp = E[3+lde*1];
16993 CC[0+bs*3] -= CC[0+bs*1] * tmp;
16994 CC[1+bs*3] -= CC[1+bs*1] * tmp;
16995 CC[2+bs*3] -= CC[2+bs*1] * tmp;
16996 CC[3+bs*3] -= CC[3+bs*1] * tmp;
16997 tmp = E[3+lde*2];
16998 CC[0+bs*3] -= CC[0+bs*2] * tmp;
16999 CC[1+bs*3] -= CC[1+bs*2] * tmp;
17000 CC[2+bs*3] -= CC[2+bs*2] * tmp;
17001 CC[3+bs*3] -= CC[3+bs*2] * tmp;
17002 tmp = inv_diag_E[3];
17003 CC[0+bs*3] *= tmp;
17004 CC[1+bs*3] *= tmp;
17005 CC[2+bs*3] *= tmp;
17006 CC[3+bs*3] *= tmp;
17007
17008 store:
17009
17010 if(m1>=4)
17011 {
17012 D[0+ldd*0] = CC[0+bs*0];
17013 D[1+ldd*0] = CC[1+bs*0];
17014 D[2+ldd*0] = CC[2+bs*0];
17015 D[3+ldd*0] = CC[3+bs*0];
17016
17017 if(n1==1)
17018 return;
17019
17020 D[0+ldd*1] = CC[0+bs*1];
17021 D[1+ldd*1] = CC[1+bs*1];
17022 D[2+ldd*1] = CC[2+bs*1];
17023 D[3+ldd*1] = CC[3+bs*1];
17024
17025 if(n1==2)
17026 return;
17027
17028 D[0+ldd*2] = CC[0+bs*2];
17029 D[1+ldd*2] = CC[1+bs*2];
17030 D[2+ldd*2] = CC[2+bs*2];
17031 D[3+ldd*2] = CC[3+bs*2];
17032
17033 if(n1==3)
17034 return;
17035
17036 D[0+ldd*3] = CC[0+bs*3];
17037 D[1+ldd*3] = CC[1+bs*3];
17038 D[2+ldd*3] = CC[2+bs*3];
17039 D[3+ldd*3] = CC[3+bs*3];
17040 }
17041 else if(m1>=3)
17042 {
17043 D[0+ldd*0] = CC[0+bs*0];
17044 D[1+ldd*0] = CC[1+bs*0];
17045 D[2+ldd*0] = CC[2+bs*0];
17046
17047 if(n1==1)
17048 return;
17049
17050 D[0+ldd*1] = CC[0+bs*1];
17051 D[1+ldd*1] = CC[1+bs*1];
17052 D[2+ldd*1] = CC[2+bs*1];
17053
17054 if(n1==2)
17055 return;
17056
17057 D[0+ldd*2] = CC[0+bs*2];
17058 D[1+ldd*2] = CC[1+bs*2];
17059 D[2+ldd*2] = CC[2+bs*2];
17060
17061 if(n1==3)
17062 return;
17063
17064 D[0+ldd*3] = CC[0+bs*3];
17065 D[1+ldd*3] = CC[1+bs*3];
17066 D[2+ldd*3] = CC[2+bs*3];
17067 }
17068 else if(m1>=2)
17069 {
17070 D[0+ldd*0] = CC[0+bs*0];
17071 D[1+ldd*0] = CC[1+bs*0];
17072
17073 if(n1==1)
17074 return;
17075
17076 D[0+ldd*1] = CC[0+bs*1];
17077 D[1+ldd*1] = CC[1+bs*1];
17078
17079 if(n1==2)
17080 return;
17081
17082 D[0+ldd*2] = CC[0+bs*2];
17083 D[1+ldd*2] = CC[1+bs*2];
17084
17085 if(n1==3)
17086 return;
17087
17088 D[0+ldd*3] = CC[0+bs*3];
17089 D[1+ldd*3] = CC[1+bs*3];
17090 }
17091 else //if(m1>=1)
17092 {
17093 D[0+ldd*0] = CC[0+bs*0];
17094
17095 if(n1==1)
17096 return;
17097
17098 D[0+ldd*1] = CC[0+bs*1];
17099
17100 if(n1==2)
17101 return;
17102
17103 D[0+ldd*2] = CC[0+bs*2];
17104
17105 if(n1==3)
17106 return;
17107
17108 D[0+ldd*3] = CC[0+bs*3];
17109 }
17110
17111 return;
17112
17113 }
17114 //#endif
17115
17116
17117
17118 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_inv_4x4_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,float * inv_diag_E)17119 void kernel_strsm_nt_rl_inv_4x4_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, float *inv_diag_E)
17120 {
17121
17122 const int bs = 4;
17123
17124 float tmp;
17125
17126 #if defined(TARGET_GENERIC)
17127 float CC[16] = {0};
17128 #else
17129 ALIGNED( float CC[16], 64 ) = {0};
17130 #endif
17131
17132 float alpha1 = -1.0;
17133
17134 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
17135
17136 tmp = inv_diag_E[0];
17137 CC[0+bs*0] *= tmp;
17138 CC[1+bs*0] *= tmp;
17139 CC[2+bs*0] *= tmp;
17140 CC[3+bs*0] *= tmp;
17141
17142 tmp = E[1+lde*0];
17143 CC[0+bs*1] -= CC[0+bs*0] * tmp;
17144 CC[1+bs*1] -= CC[1+bs*0] * tmp;
17145 CC[2+bs*1] -= CC[2+bs*0] * tmp;
17146 CC[3+bs*1] -= CC[3+bs*0] * tmp;
17147 tmp = inv_diag_E[1];
17148 CC[0+bs*1] *= tmp;
17149 CC[1+bs*1] *= tmp;
17150 CC[2+bs*1] *= tmp;
17151 CC[3+bs*1] *= tmp;
17152
17153 tmp = E[2+lde*0];
17154 CC[0+bs*2] -= CC[0+bs*0] * tmp;
17155 CC[1+bs*2] -= CC[1+bs*0] * tmp;
17156 CC[2+bs*2] -= CC[2+bs*0] * tmp;
17157 CC[3+bs*2] -= CC[3+bs*0] * tmp;
17158 tmp = E[2+lde*1];
17159 CC[0+bs*2] -= CC[0+bs*1] * tmp;
17160 CC[1+bs*2] -= CC[1+bs*1] * tmp;
17161 CC[2+bs*2] -= CC[2+bs*1] * tmp;
17162 CC[3+bs*2] -= CC[3+bs*1] * tmp;
17163 tmp = inv_diag_E[2];
17164 CC[0+bs*2] *= tmp;
17165 CC[1+bs*2] *= tmp;
17166 CC[2+bs*2] *= tmp;
17167 CC[3+bs*2] *= tmp;
17168
17169 tmp = E[3+lde*0];
17170 CC[0+bs*3] -= CC[0+bs*0] * tmp;
17171 CC[1+bs*3] -= CC[1+bs*0] * tmp;
17172 CC[2+bs*3] -= CC[2+bs*0] * tmp;
17173 CC[3+bs*3] -= CC[3+bs*0] * tmp;
17174 tmp = E[3+lde*1];
17175 CC[0+bs*3] -= CC[0+bs*1] * tmp;
17176 CC[1+bs*3] -= CC[1+bs*1] * tmp;
17177 CC[2+bs*3] -= CC[2+bs*1] * tmp;
17178 CC[3+bs*3] -= CC[3+bs*1] * tmp;
17179 tmp = E[3+lde*2];
17180 CC[0+bs*3] -= CC[0+bs*2] * tmp;
17181 CC[1+bs*3] -= CC[1+bs*2] * tmp;
17182 CC[2+bs*3] -= CC[2+bs*2] * tmp;
17183 CC[3+bs*3] -= CC[3+bs*2] * tmp;
17184 tmp = inv_diag_E[3];
17185 CC[0+bs*3] *= tmp;
17186 CC[1+bs*3] *= tmp;
17187 CC[2+bs*3] *= tmp;
17188 CC[3+bs*3] *= tmp;
17189
17190 D[0+bs*0] = CC[0+bs*0];
17191 D[1+bs*0] = CC[1+bs*0];
17192 D[2+bs*0] = CC[2+bs*0];
17193 D[3+bs*0] = CC[3+bs*0];
17194
17195 D[0+bs*1] = CC[0+bs*1];
17196 D[1+bs*1] = CC[1+bs*1];
17197 D[2+bs*1] = CC[2+bs*1];
17198 D[3+bs*1] = CC[3+bs*1];
17199
17200 D[0+bs*2] = CC[0+bs*2];
17201 D[1+bs*2] = CC[1+bs*2];
17202 D[2+bs*2] = CC[2+bs*2];
17203 D[3+bs*2] = CC[3+bs*2];
17204
17205 D[0+bs*3] = CC[0+bs*3];
17206 D[1+bs*3] = CC[1+bs*3];
17207 D[2+bs*3] = CC[2+bs*3];
17208 D[3+bs*3] = CC[3+bs*3];
17209
17210 return;
17211
17212 }
17213 //#endif
17214
17215
17216
17217 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_inv_4x4_vs_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,float * inv_diag_E,int m1,int n1)17218 void kernel_strsm_nt_rl_inv_4x4_vs_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, float *inv_diag_E, int m1, int n1)
17219 {
17220
17221 const int bs = 4;
17222
17223 float tmp;
17224
17225 #if defined(TARGET_GENERIC)
17226 float CC[16] = {0};
17227 #else
17228 ALIGNED( float CC[16], 64 ) = {0};
17229 #endif
17230
17231 float alpha1 = -1.0;
17232
17233 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
17234
17235 tmp = inv_diag_E[0];
17236 CC[0+bs*0] *= tmp;
17237 CC[1+bs*0] *= tmp;
17238 CC[2+bs*0] *= tmp;
17239 CC[3+bs*0] *= tmp;
17240
17241 if(n1==1)
17242 goto store;
17243
17244 tmp = E[1+lde*0];
17245 CC[0+bs*1] -= CC[0+bs*0] * tmp;
17246 CC[1+bs*1] -= CC[1+bs*0] * tmp;
17247 CC[2+bs*1] -= CC[2+bs*0] * tmp;
17248 CC[3+bs*1] -= CC[3+bs*0] * tmp;
17249 tmp = inv_diag_E[1];
17250 CC[0+bs*1] *= tmp;
17251 CC[1+bs*1] *= tmp;
17252 CC[2+bs*1] *= tmp;
17253 CC[3+bs*1] *= tmp;
17254
17255 if(n1==2)
17256 goto store;
17257
17258 tmp = E[2+lde*0];
17259 CC[0+bs*2] -= CC[0+bs*0] * tmp;
17260 CC[1+bs*2] -= CC[1+bs*0] * tmp;
17261 CC[2+bs*2] -= CC[2+bs*0] * tmp;
17262 CC[3+bs*2] -= CC[3+bs*0] * tmp;
17263 tmp = E[2+lde*1];
17264 CC[0+bs*2] -= CC[0+bs*1] * tmp;
17265 CC[1+bs*2] -= CC[1+bs*1] * tmp;
17266 CC[2+bs*2] -= CC[2+bs*1] * tmp;
17267 CC[3+bs*2] -= CC[3+bs*1] * tmp;
17268 tmp = inv_diag_E[2];
17269 CC[0+bs*2] *= tmp;
17270 CC[1+bs*2] *= tmp;
17271 CC[2+bs*2] *= tmp;
17272 CC[3+bs*2] *= tmp;
17273
17274 if(n1==3)
17275 goto store;
17276
17277 tmp = E[3+lde*0];
17278 CC[0+bs*3] -= CC[0+bs*0] * tmp;
17279 CC[1+bs*3] -= CC[1+bs*0] * tmp;
17280 CC[2+bs*3] -= CC[2+bs*0] * tmp;
17281 CC[3+bs*3] -= CC[3+bs*0] * tmp;
17282 tmp = E[3+lde*1];
17283 CC[0+bs*3] -= CC[0+bs*1] * tmp;
17284 CC[1+bs*3] -= CC[1+bs*1] * tmp;
17285 CC[2+bs*3] -= CC[2+bs*1] * tmp;
17286 CC[3+bs*3] -= CC[3+bs*1] * tmp;
17287 tmp = E[3+lde*2];
17288 CC[0+bs*3] -= CC[0+bs*2] * tmp;
17289 CC[1+bs*3] -= CC[1+bs*2] * tmp;
17290 CC[2+bs*3] -= CC[2+bs*2] * tmp;
17291 CC[3+bs*3] -= CC[3+bs*2] * tmp;
17292 tmp = inv_diag_E[3];
17293 CC[0+bs*3] *= tmp;
17294 CC[1+bs*3] *= tmp;
17295 CC[2+bs*3] *= tmp;
17296 CC[3+bs*3] *= tmp;
17297
17298 store:
17299
17300 if(m1>=4)
17301 {
17302 D[0+bs*0] = CC[0+bs*0];
17303 D[1+bs*0] = CC[1+bs*0];
17304 D[2+bs*0] = CC[2+bs*0];
17305 D[3+bs*0] = CC[3+bs*0];
17306
17307 if(n1==1)
17308 return;
17309
17310 D[0+bs*1] = CC[0+bs*1];
17311 D[1+bs*1] = CC[1+bs*1];
17312 D[2+bs*1] = CC[2+bs*1];
17313 D[3+bs*1] = CC[3+bs*1];
17314
17315 if(n1==2)
17316 return;
17317
17318 D[0+bs*2] = CC[0+bs*2];
17319 D[1+bs*2] = CC[1+bs*2];
17320 D[2+bs*2] = CC[2+bs*2];
17321 D[3+bs*2] = CC[3+bs*2];
17322
17323 if(n1==3)
17324 return;
17325
17326 D[0+bs*3] = CC[0+bs*3];
17327 D[1+bs*3] = CC[1+bs*3];
17328 D[2+bs*3] = CC[2+bs*3];
17329 D[3+bs*3] = CC[3+bs*3];
17330 }
17331 else if(m1>=3)
17332 {
17333 D[0+bs*0] = CC[0+bs*0];
17334 D[1+bs*0] = CC[1+bs*0];
17335 D[2+bs*0] = CC[2+bs*0];
17336
17337 if(n1==1)
17338 return;
17339
17340 D[0+bs*1] = CC[0+bs*1];
17341 D[1+bs*1] = CC[1+bs*1];
17342 D[2+bs*1] = CC[2+bs*1];
17343
17344 if(n1==2)
17345 return;
17346
17347 D[0+bs*2] = CC[0+bs*2];
17348 D[1+bs*2] = CC[1+bs*2];
17349 D[2+bs*2] = CC[2+bs*2];
17350
17351 if(n1==3)
17352 return;
17353
17354 D[0+bs*3] = CC[0+bs*3];
17355 D[1+bs*3] = CC[1+bs*3];
17356 D[2+bs*3] = CC[2+bs*3];
17357 }
17358 else if(m1>=2)
17359 {
17360 D[0+bs*0] = CC[0+bs*0];
17361 D[1+bs*0] = CC[1+bs*0];
17362
17363 if(n1==1)
17364 return;
17365
17366 D[0+bs*1] = CC[0+bs*1];
17367 D[1+bs*1] = CC[1+bs*1];
17368
17369 if(n1==2)
17370 return;
17371
17372 D[0+bs*2] = CC[0+bs*2];
17373 D[1+bs*2] = CC[1+bs*2];
17374
17375 if(n1==3)
17376 return;
17377
17378 D[0+bs*3] = CC[0+bs*3];
17379 D[1+bs*3] = CC[1+bs*3];
17380 }
17381 else //if(m1>=1)
17382 {
17383 D[0+bs*0] = CC[0+bs*0];
17384
17385 if(n1==1)
17386 return;
17387
17388 D[0+bs*1] = CC[0+bs*1];
17389
17390 if(n1==2)
17391 return;
17392
17393 D[0+bs*2] = CC[0+bs*2];
17394
17395 if(n1==3)
17396 return;
17397
17398 D[0+bs*3] = CC[0+bs*3];
17399 }
17400
17401 return;
17402
17403 }
17404 //#endif
17405
17406
17407
17408 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_inv_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E)17409 void kernel_strsm_nt_rl_inv_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E)
17410 {
17411
17412 const int bs = 4;
17413
17414 float tmp;
17415
17416 #if defined(TARGET_GENERIC)
17417 float CC[16] = {0};
17418 #else
17419 ALIGNED( float CC[16], 64 ) = {0};
17420 #endif
17421
17422 float alpha1 = -1.0;
17423
17424 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
17425
17426 tmp = inv_diag_E[0];
17427 CC[0+bs*0] *= tmp;
17428 CC[1+bs*0] *= tmp;
17429 CC[2+bs*0] *= tmp;
17430 CC[3+bs*0] *= tmp;
17431
17432 tmp = E[1+lde*0];
17433 CC[0+bs*1] -= CC[0+bs*0] * tmp;
17434 CC[1+bs*1] -= CC[1+bs*0] * tmp;
17435 CC[2+bs*1] -= CC[2+bs*0] * tmp;
17436 CC[3+bs*1] -= CC[3+bs*0] * tmp;
17437 tmp = inv_diag_E[1];
17438 CC[0+bs*1] *= tmp;
17439 CC[1+bs*1] *= tmp;
17440 CC[2+bs*1] *= tmp;
17441 CC[3+bs*1] *= tmp;
17442
17443 tmp = E[2+lde*0];
17444 CC[0+bs*2] -= CC[0+bs*0] * tmp;
17445 CC[1+bs*2] -= CC[1+bs*0] * tmp;
17446 CC[2+bs*2] -= CC[2+bs*0] * tmp;
17447 CC[3+bs*2] -= CC[3+bs*0] * tmp;
17448 tmp = E[2+lde*1];
17449 CC[0+bs*2] -= CC[0+bs*1] * tmp;
17450 CC[1+bs*2] -= CC[1+bs*1] * tmp;
17451 CC[2+bs*2] -= CC[2+bs*1] * tmp;
17452 CC[3+bs*2] -= CC[3+bs*1] * tmp;
17453 tmp = inv_diag_E[2];
17454 CC[0+bs*2] *= tmp;
17455 CC[1+bs*2] *= tmp;
17456 CC[2+bs*2] *= tmp;
17457 CC[3+bs*2] *= tmp;
17458
17459 tmp = E[3+lde*0];
17460 CC[0+bs*3] -= CC[0+bs*0] * tmp;
17461 CC[1+bs*3] -= CC[1+bs*0] * tmp;
17462 CC[2+bs*3] -= CC[2+bs*0] * tmp;
17463 CC[3+bs*3] -= CC[3+bs*0] * tmp;
17464 tmp = E[3+lde*1];
17465 CC[0+bs*3] -= CC[0+bs*1] * tmp;
17466 CC[1+bs*3] -= CC[1+bs*1] * tmp;
17467 CC[2+bs*3] -= CC[2+bs*1] * tmp;
17468 CC[3+bs*3] -= CC[3+bs*1] * tmp;
17469 tmp = E[3+lde*2];
17470 CC[0+bs*3] -= CC[0+bs*2] * tmp;
17471 CC[1+bs*3] -= CC[1+bs*2] * tmp;
17472 CC[2+bs*3] -= CC[2+bs*2] * tmp;
17473 CC[3+bs*3] -= CC[3+bs*2] * tmp;
17474 tmp = inv_diag_E[3];
17475 CC[0+bs*3] *= tmp;
17476 CC[1+bs*3] *= tmp;
17477 CC[2+bs*3] *= tmp;
17478 CC[3+bs*3] *= tmp;
17479
17480 D[0+ldd*0] = CC[0+bs*0];
17481 D[1+ldd*0] = CC[1+bs*0];
17482 D[2+ldd*0] = CC[2+bs*0];
17483 D[3+ldd*0] = CC[3+bs*0];
17484
17485 D[0+ldd*1] = CC[0+bs*1];
17486 D[1+ldd*1] = CC[1+bs*1];
17487 D[2+ldd*1] = CC[2+bs*1];
17488 D[3+ldd*1] = CC[3+bs*1];
17489
17490 D[0+ldd*2] = CC[0+bs*2];
17491 D[1+ldd*2] = CC[1+bs*2];
17492 D[2+ldd*2] = CC[2+bs*2];
17493 D[3+ldd*2] = CC[3+bs*2];
17494
17495 D[0+ldd*3] = CC[0+bs*3];
17496 D[1+ldd*3] = CC[1+bs*3];
17497 D[2+ldd*3] = CC[2+bs*3];
17498 D[3+ldd*3] = CC[3+bs*3];
17499
17500 return;
17501
17502 }
17503 //#endif
17504
17505
17506
17507 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_inv_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E,int m1,int n1)17508 void kernel_strsm_nt_rl_inv_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E, int m1, int n1)
17509 {
17510
17511 const int bs = 4;
17512
17513 float tmp;
17514
17515 #if defined(TARGET_GENERIC)
17516 float CC[16] = {0};
17517 #else
17518 ALIGNED( float CC[16], 64 ) = {0};
17519 #endif
17520
17521 float alpha1 = -1.0;
17522
17523 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
17524
17525 tmp = inv_diag_E[0];
17526 CC[0+bs*0] *= tmp;
17527 CC[1+bs*0] *= tmp;
17528 CC[2+bs*0] *= tmp;
17529 CC[3+bs*0] *= tmp;
17530
17531 if(n1==1)
17532 goto store;
17533
17534 tmp = E[1+lde*0];
17535 CC[0+bs*1] -= CC[0+bs*0] * tmp;
17536 CC[1+bs*1] -= CC[1+bs*0] * tmp;
17537 CC[2+bs*1] -= CC[2+bs*0] * tmp;
17538 CC[3+bs*1] -= CC[3+bs*0] * tmp;
17539 tmp = inv_diag_E[1];
17540 CC[0+bs*1] *= tmp;
17541 CC[1+bs*1] *= tmp;
17542 CC[2+bs*1] *= tmp;
17543 CC[3+bs*1] *= tmp;
17544
17545 if(n1==2)
17546 goto store;
17547
17548 tmp = E[2+lde*0];
17549 CC[0+bs*2] -= CC[0+bs*0] * tmp;
17550 CC[1+bs*2] -= CC[1+bs*0] * tmp;
17551 CC[2+bs*2] -= CC[2+bs*0] * tmp;
17552 CC[3+bs*2] -= CC[3+bs*0] * tmp;
17553 tmp = E[2+lde*1];
17554 CC[0+bs*2] -= CC[0+bs*1] * tmp;
17555 CC[1+bs*2] -= CC[1+bs*1] * tmp;
17556 CC[2+bs*2] -= CC[2+bs*1] * tmp;
17557 CC[3+bs*2] -= CC[3+bs*1] * tmp;
17558 tmp = inv_diag_E[2];
17559 CC[0+bs*2] *= tmp;
17560 CC[1+bs*2] *= tmp;
17561 CC[2+bs*2] *= tmp;
17562 CC[3+bs*2] *= tmp;
17563
17564 if(n1==3)
17565 goto store;
17566
17567 tmp = E[3+lde*0];
17568 CC[0+bs*3] -= CC[0+bs*0] * tmp;
17569 CC[1+bs*3] -= CC[1+bs*0] * tmp;
17570 CC[2+bs*3] -= CC[2+bs*0] * tmp;
17571 CC[3+bs*3] -= CC[3+bs*0] * tmp;
17572 tmp = E[3+lde*1];
17573 CC[0+bs*3] -= CC[0+bs*1] * tmp;
17574 CC[1+bs*3] -= CC[1+bs*1] * tmp;
17575 CC[2+bs*3] -= CC[2+bs*1] * tmp;
17576 CC[3+bs*3] -= CC[3+bs*1] * tmp;
17577 tmp = E[3+lde*2];
17578 CC[0+bs*3] -= CC[0+bs*2] * tmp;
17579 CC[1+bs*3] -= CC[1+bs*2] * tmp;
17580 CC[2+bs*3] -= CC[2+bs*2] * tmp;
17581 CC[3+bs*3] -= CC[3+bs*2] * tmp;
17582 tmp = inv_diag_E[3];
17583 CC[0+bs*3] *= tmp;
17584 CC[1+bs*3] *= tmp;
17585 CC[2+bs*3] *= tmp;
17586 CC[3+bs*3] *= tmp;
17587
17588 store:
17589
17590 if(m1>=4)
17591 {
17592 D[0+ldd*0] = CC[0+bs*0];
17593 D[1+ldd*0] = CC[1+bs*0];
17594 D[2+ldd*0] = CC[2+bs*0];
17595 D[3+ldd*0] = CC[3+bs*0];
17596
17597 if(n1==1)
17598 return;
17599
17600 D[0+ldd*1] = CC[0+bs*1];
17601 D[1+ldd*1] = CC[1+bs*1];
17602 D[2+ldd*1] = CC[2+bs*1];
17603 D[3+ldd*1] = CC[3+bs*1];
17604
17605 if(n1==2)
17606 return;
17607
17608 D[0+ldd*2] = CC[0+bs*2];
17609 D[1+ldd*2] = CC[1+bs*2];
17610 D[2+ldd*2] = CC[2+bs*2];
17611 D[3+ldd*2] = CC[3+bs*2];
17612
17613 if(n1==3)
17614 return;
17615
17616 D[0+ldd*3] = CC[0+bs*3];
17617 D[1+ldd*3] = CC[1+bs*3];
17618 D[2+ldd*3] = CC[2+bs*3];
17619 D[3+ldd*3] = CC[3+bs*3];
17620 }
17621 else if(m1>=3)
17622 {
17623 D[0+ldd*0] = CC[0+bs*0];
17624 D[1+ldd*0] = CC[1+bs*0];
17625 D[2+ldd*0] = CC[2+bs*0];
17626
17627 if(n1==1)
17628 return;
17629
17630 D[0+ldd*1] = CC[0+bs*1];
17631 D[1+ldd*1] = CC[1+bs*1];
17632 D[2+ldd*1] = CC[2+bs*1];
17633
17634 if(n1==2)
17635 return;
17636
17637 D[0+ldd*2] = CC[0+bs*2];
17638 D[1+ldd*2] = CC[1+bs*2];
17639 D[2+ldd*2] = CC[2+bs*2];
17640
17641 if(n1==3)
17642 return;
17643
17644 D[0+ldd*3] = CC[0+bs*3];
17645 D[1+ldd*3] = CC[1+bs*3];
17646 D[2+ldd*3] = CC[2+bs*3];
17647 }
17648 else if(m1>=2)
17649 {
17650 D[0+ldd*0] = CC[0+bs*0];
17651 D[1+ldd*0] = CC[1+bs*0];
17652
17653 if(n1==1)
17654 return;
17655
17656 D[0+ldd*1] = CC[0+bs*1];
17657 D[1+ldd*1] = CC[1+bs*1];
17658
17659 if(n1==2)
17660 return;
17661
17662 D[0+ldd*2] = CC[0+bs*2];
17663 D[1+ldd*2] = CC[1+bs*2];
17664
17665 if(n1==3)
17666 return;
17667
17668 D[0+ldd*3] = CC[0+bs*3];
17669 D[1+ldd*3] = CC[1+bs*3];
17670 }
17671 else //if(m1>=1)
17672 {
17673 D[0+ldd*0] = CC[0+bs*0];
17674
17675 if(n1==1)
17676 return;
17677
17678 D[0+ldd*1] = CC[0+bs*1];
17679
17680 if(n1==2)
17681 return;
17682
17683 D[0+ldd*2] = CC[0+bs*2];
17684
17685 if(n1==3)
17686 return;
17687
17688 D[0+ldd*3] = CC[0+bs*3];
17689 }
17690
17691 return;
17692
17693 }
17694 //#endif
17695
17696
17697
17698 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_strsm_nt_rl_one_4x4_lib44cc4(int kmax,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,float * E)17699 void kernel_strsm_nt_rl_one_4x4_lib44cc4(int kmax, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, float *E)
17700 {
17701
17702 const int bs = 4;
17703
17704 float tmp;
17705
17706 #if defined(TARGET_GENERIC)
17707 float CC[16] = {0};
17708 #else
17709 ALIGNED( float CC[16], 64 ) = {0};
17710 #endif
17711
17712 float alpha1 = -1.0;
17713
17714 CC[0+bs*0] = C[0+ldc*0];
17715 CC[1+bs*0] = C[1+ldc*0];
17716 CC[2+bs*0] = C[2+ldc*0];
17717 CC[3+bs*0] = C[3+ldc*0];
17718
17719 CC[0+bs*1] = C[0+ldc*1];
17720 CC[1+bs*1] = C[1+ldc*1];
17721 CC[2+bs*1] = C[2+ldc*1];
17722 CC[3+bs*1] = C[3+ldc*1];
17723
17724 CC[0+bs*2] = C[0+ldc*2];
17725 CC[1+bs*2] = C[1+ldc*2];
17726 CC[2+bs*2] = C[2+ldc*2];
17727 CC[3+bs*2] = C[3+ldc*2];
17728
17729 CC[0+bs*3] = C[0+ldc*3];
17730 CC[1+bs*3] = C[1+ldc*3];
17731 CC[2+bs*3] = C[2+ldc*3];
17732 CC[3+bs*3] = C[3+ldc*3];
17733
17734 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, beta, CC, CC);
17735
17736 tmp = E[1+bs*0];
17737 CC[0+bs*1] -= CC[0+bs*0] * tmp;
17738 CC[1+bs*1] -= CC[1+bs*0] * tmp;
17739 CC[2+bs*1] -= CC[2+bs*0] * tmp;
17740 CC[3+bs*1] -= CC[3+bs*0] * tmp;
17741
17742 tmp = E[2+bs*0];
17743 CC[0+bs*2] -= CC[0+bs*0] * tmp;
17744 CC[1+bs*2] -= CC[1+bs*0] * tmp;
17745 CC[2+bs*2] -= CC[2+bs*0] * tmp;
17746 CC[3+bs*2] -= CC[3+bs*0] * tmp;
17747 tmp = E[2+bs*1];
17748 CC[0+bs*2] -= CC[0+bs*1] * tmp;
17749 CC[1+bs*2] -= CC[1+bs*1] * tmp;
17750 CC[2+bs*2] -= CC[2+bs*1] * tmp;
17751 CC[3+bs*2] -= CC[3+bs*1] * tmp;
17752
17753 tmp = E[3+bs*0];
17754 CC[0+bs*3] -= CC[0+bs*0] * tmp;
17755 CC[1+bs*3] -= CC[1+bs*0] * tmp;
17756 CC[2+bs*3] -= CC[2+bs*0] * tmp;
17757 CC[3+bs*3] -= CC[3+bs*0] * tmp;
17758 tmp = E[3+bs*1];
17759 CC[0+bs*3] -= CC[0+bs*1] * tmp;
17760 CC[1+bs*3] -= CC[1+bs*1] * tmp;
17761 CC[2+bs*3] -= CC[2+bs*1] * tmp;
17762 CC[3+bs*3] -= CC[3+bs*1] * tmp;
17763 tmp = E[3+bs*2];
17764 CC[0+bs*3] -= CC[0+bs*2] * tmp;
17765 CC[1+bs*3] -= CC[1+bs*2] * tmp;
17766 CC[2+bs*3] -= CC[2+bs*2] * tmp;
17767 CC[3+bs*3] -= CC[3+bs*2] * tmp;
17768
17769 D[0+ldd*0] = CC[0+bs*0];
17770 D[1+ldd*0] = CC[1+bs*0];
17771 D[2+ldd*0] = CC[2+bs*0];
17772 D[3+ldd*0] = CC[3+bs*0];
17773
17774 D[0+ldd*1] = CC[0+bs*1];
17775 D[1+ldd*1] = CC[1+bs*1];
17776 D[2+ldd*1] = CC[2+bs*1];
17777 D[3+ldd*1] = CC[3+bs*1];
17778
17779 D[0+ldd*2] = CC[0+bs*2];
17780 D[1+ldd*2] = CC[1+bs*2];
17781 D[2+ldd*2] = CC[2+bs*2];
17782 D[3+ldd*2] = CC[3+bs*2];
17783
17784 D[0+ldd*3] = CC[0+bs*3];
17785 D[1+ldd*3] = CC[1+bs*3];
17786 D[2+ldd*3] = CC[2+bs*3];
17787 D[3+ldd*3] = CC[3+bs*3];
17788
17789 return;
17790
17791 }
17792 //#endif
17793
17794
17795
17796 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_strsm_nt_rl_one_4x4_vs_lib44cc4(int kmax,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,float * E,int m1,int n1)17797 void kernel_strsm_nt_rl_one_4x4_vs_lib44cc4(int kmax, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, float *E, int m1, int n1)
17798 {
17799
17800 const int bs = 4;
17801
17802 float tmp;
17803
17804 #if defined(TARGET_GENERIC)
17805 float CC[16] = {0};
17806 #else
17807 ALIGNED( float CC[16], 64 ) = {0};
17808 #endif
17809
17810 float alpha1 = -1.0;
17811
17812 if(m1>=4)
17813 {
17814 CC[0+bs*0] = C[0+ldc*0];
17815 CC[1+bs*0] = C[1+ldc*0];
17816 CC[2+bs*0] = C[2+ldc*0];
17817 CC[3+bs*0] = C[3+ldc*0];
17818
17819 if(n1==1)
17820 goto kernel;
17821
17822 CC[0+bs*1] = C[0+ldc*1];
17823 CC[1+bs*1] = C[1+ldc*1];
17824 CC[2+bs*1] = C[2+ldc*1];
17825 CC[3+bs*1] = C[3+ldc*1];
17826
17827 if(n1==2)
17828 goto kernel;
17829
17830 CC[0+bs*2] = C[0+ldc*2];
17831 CC[1+bs*2] = C[1+ldc*2];
17832 CC[2+bs*2] = C[2+ldc*2];
17833 CC[3+bs*2] = C[3+ldc*2];
17834
17835 if(n1==3)
17836 goto kernel;
17837
17838 CC[0+bs*3] = C[0+ldc*3];
17839 CC[1+bs*3] = C[1+ldc*3];
17840 CC[2+bs*3] = C[2+ldc*3];
17841 CC[3+bs*3] = C[3+ldc*3];
17842 }
17843 else if(m1>=3)
17844 {
17845 CC[0+bs*0] = C[0+ldc*0];
17846 CC[1+bs*0] = C[1+ldc*0];
17847 CC[2+bs*0] = C[2+ldc*0];
17848
17849 if(n1==1)
17850 goto kernel;
17851
17852 CC[0+bs*1] = C[0+ldc*1];
17853 CC[1+bs*1] = C[1+ldc*1];
17854 CC[2+bs*1] = C[2+ldc*1];
17855
17856 if(n1==2)
17857 goto kernel;
17858
17859 CC[0+bs*2] = C[0+ldc*2];
17860 CC[1+bs*2] = C[1+ldc*2];
17861 CC[2+bs*2] = C[2+ldc*2];
17862
17863 if(n1==3)
17864 goto kernel;
17865
17866 CC[0+bs*3] = C[0+ldc*3];
17867 CC[1+bs*3] = C[1+ldc*3];
17868 CC[2+bs*3] = C[2+ldc*3];
17869 }
17870 else if(m1>=2)
17871 {
17872 CC[0+bs*0] = C[0+ldc*0];
17873 CC[1+bs*0] = C[1+ldc*0];
17874
17875 if(n1==1)
17876 goto kernel;
17877
17878 CC[0+bs*1] = C[0+ldc*1];
17879 CC[1+bs*1] = C[1+ldc*1];
17880
17881 if(n1==2)
17882 goto kernel;
17883
17884 CC[0+bs*2] = C[0+ldc*2];
17885 CC[1+bs*2] = C[1+ldc*2];
17886
17887 if(n1==3)
17888 goto kernel;
17889
17890 CC[0+bs*3] = C[0+ldc*3];
17891 CC[1+bs*3] = C[1+ldc*3];
17892 }
17893 else //if(m1>=1)
17894 {
17895 CC[0+bs*0] = C[0+ldc*0];
17896
17897 if(n1==1)
17898 goto kernel;
17899
17900 CC[0+bs*1] = C[0+ldc*1];
17901
17902 if(n1==2)
17903 goto kernel;
17904
17905 CC[0+bs*2] = C[0+ldc*2];
17906
17907 if(n1==3)
17908 goto kernel;
17909
17910 CC[0+bs*3] = C[0+ldc*3];
17911 }
17912
17913 kernel:
17914 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, beta, CC, CC);
17915
17916 if(n1==1)
17917 goto store;
17918
17919 tmp = E[1+bs*0];
17920 CC[0+bs*1] -= CC[0+bs*0] * tmp;
17921 CC[1+bs*1] -= CC[1+bs*0] * tmp;
17922 CC[2+bs*1] -= CC[2+bs*0] * tmp;
17923 CC[3+bs*1] -= CC[3+bs*0] * tmp;
17924
17925 if(n1==2)
17926 goto store;
17927
17928 tmp = E[2+bs*0];
17929 CC[0+bs*2] -= CC[0+bs*0] * tmp;
17930 CC[1+bs*2] -= CC[1+bs*0] * tmp;
17931 CC[2+bs*2] -= CC[2+bs*0] * tmp;
17932 CC[3+bs*2] -= CC[3+bs*0] * tmp;
17933 tmp = E[2+bs*1];
17934 CC[0+bs*2] -= CC[0+bs*1] * tmp;
17935 CC[1+bs*2] -= CC[1+bs*1] * tmp;
17936 CC[2+bs*2] -= CC[2+bs*1] * tmp;
17937 CC[3+bs*2] -= CC[3+bs*1] * tmp;
17938
17939 if(n1==3)
17940 goto store;
17941
17942 tmp = E[3+bs*0];
17943 CC[0+bs*3] -= CC[0+bs*0] * tmp;
17944 CC[1+bs*3] -= CC[1+bs*0] * tmp;
17945 CC[2+bs*3] -= CC[2+bs*0] * tmp;
17946 CC[3+bs*3] -= CC[3+bs*0] * tmp;
17947 tmp = E[3+bs*1];
17948 CC[0+bs*3] -= CC[0+bs*1] * tmp;
17949 CC[1+bs*3] -= CC[1+bs*1] * tmp;
17950 CC[2+bs*3] -= CC[2+bs*1] * tmp;
17951 CC[3+bs*3] -= CC[3+bs*1] * tmp;
17952 tmp = E[3+bs*2];
17953 CC[0+bs*3] -= CC[0+bs*2] * tmp;
17954 CC[1+bs*3] -= CC[1+bs*2] * tmp;
17955 CC[2+bs*3] -= CC[2+bs*2] * tmp;
17956 CC[3+bs*3] -= CC[3+bs*2] * tmp;
17957
17958 store:
17959
17960 if(m1>=4)
17961 {
17962 D[0+ldd*0] = CC[0+bs*0];
17963 D[1+ldd*0] = CC[1+bs*0];
17964 D[2+ldd*0] = CC[2+bs*0];
17965 D[3+ldd*0] = CC[3+bs*0];
17966
17967 if(n1==1)
17968 return;
17969
17970 D[0+ldd*1] = CC[0+bs*1];
17971 D[1+ldd*1] = CC[1+bs*1];
17972 D[2+ldd*1] = CC[2+bs*1];
17973 D[3+ldd*1] = CC[3+bs*1];
17974
17975 if(n1==2)
17976 return;
17977
17978 D[0+ldd*2] = CC[0+bs*2];
17979 D[1+ldd*2] = CC[1+bs*2];
17980 D[2+ldd*2] = CC[2+bs*2];
17981 D[3+ldd*2] = CC[3+bs*2];
17982
17983 if(n1==3)
17984 return;
17985
17986 D[0+ldd*3] = CC[0+bs*3];
17987 D[1+ldd*3] = CC[1+bs*3];
17988 D[2+ldd*3] = CC[2+bs*3];
17989 D[3+ldd*3] = CC[3+bs*3];
17990 }
17991 else if(m1>=3)
17992 {
17993 D[0+ldd*0] = CC[0+bs*0];
17994 D[1+ldd*0] = CC[1+bs*0];
17995 D[2+ldd*0] = CC[2+bs*0];
17996
17997 if(n1==1)
17998 return;
17999
18000 D[0+ldd*1] = CC[0+bs*1];
18001 D[1+ldd*1] = CC[1+bs*1];
18002 D[2+ldd*1] = CC[2+bs*1];
18003
18004 if(n1==2)
18005 return;
18006
18007 D[0+ldd*2] = CC[0+bs*2];
18008 D[1+ldd*2] = CC[1+bs*2];
18009 D[2+ldd*2] = CC[2+bs*2];
18010
18011 if(n1==3)
18012 return;
18013
18014 D[0+ldd*3] = CC[0+bs*3];
18015 D[1+ldd*3] = CC[1+bs*3];
18016 D[2+ldd*3] = CC[2+bs*3];
18017 }
18018 else if(m1>=2)
18019 {
18020 D[0+ldd*0] = CC[0+bs*0];
18021 D[1+ldd*0] = CC[1+bs*0];
18022
18023 if(n1==1)
18024 return;
18025
18026 D[0+ldd*1] = CC[0+bs*1];
18027 D[1+ldd*1] = CC[1+bs*1];
18028
18029 if(n1==2)
18030 return;
18031
18032 D[0+ldd*2] = CC[0+bs*2];
18033 D[1+ldd*2] = CC[1+bs*2];
18034
18035 if(n1==3)
18036 return;
18037
18038 D[0+ldd*3] = CC[0+bs*3];
18039 D[1+ldd*3] = CC[1+bs*3];
18040 }
18041 else //if(m1>=1)
18042 {
18043 D[0+ldd*0] = CC[0+bs*0];
18044
18045 if(n1==1)
18046 return;
18047
18048 D[0+ldd*1] = CC[0+bs*1];
18049
18050 if(n1==2)
18051 return;
18052
18053 D[0+ldd*2] = CC[0+bs*2];
18054
18055 if(n1==3)
18056 return;
18057
18058 D[0+ldd*3] = CC[0+bs*3];
18059 }
18060
18061 return;
18062
18063 }
18064 //#endif
18065
18066
18067
18068 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_one_4x4_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde)18069 void kernel_strsm_nt_rl_one_4x4_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde)
18070 {
18071
18072 const int bs = 4;
18073
18074 float tmp;
18075
18076 #if defined(TARGET_GENERIC)
18077 float CC[16] = {0};
18078 #else
18079 ALIGNED( float CC[16], 64 ) = {0};
18080 #endif
18081
18082 float alpha1 = -1.0;
18083
18084 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
18085
18086 tmp = E[1+lde*0];
18087 CC[0+bs*1] -= CC[0+bs*0] * tmp;
18088 CC[1+bs*1] -= CC[1+bs*0] * tmp;
18089 CC[2+bs*1] -= CC[2+bs*0] * tmp;
18090 CC[3+bs*1] -= CC[3+bs*0] * tmp;
18091
18092 tmp = E[2+lde*0];
18093 CC[0+bs*2] -= CC[0+bs*0] * tmp;
18094 CC[1+bs*2] -= CC[1+bs*0] * tmp;
18095 CC[2+bs*2] -= CC[2+bs*0] * tmp;
18096 CC[3+bs*2] -= CC[3+bs*0] * tmp;
18097 tmp = E[2+lde*1];
18098 CC[0+bs*2] -= CC[0+bs*1] * tmp;
18099 CC[1+bs*2] -= CC[1+bs*1] * tmp;
18100 CC[2+bs*2] -= CC[2+bs*1] * tmp;
18101 CC[3+bs*2] -= CC[3+bs*1] * tmp;
18102
18103 tmp = E[3+lde*0];
18104 CC[0+bs*3] -= CC[0+bs*0] * tmp;
18105 CC[1+bs*3] -= CC[1+bs*0] * tmp;
18106 CC[2+bs*3] -= CC[2+bs*0] * tmp;
18107 CC[3+bs*3] -= CC[3+bs*0] * tmp;
18108 tmp = E[3+lde*1];
18109 CC[0+bs*3] -= CC[0+bs*1] * tmp;
18110 CC[1+bs*3] -= CC[1+bs*1] * tmp;
18111 CC[2+bs*3] -= CC[2+bs*1] * tmp;
18112 CC[3+bs*3] -= CC[3+bs*1] * tmp;
18113 tmp = E[3+lde*2];
18114 CC[0+bs*3] -= CC[0+bs*2] * tmp;
18115 CC[1+bs*3] -= CC[1+bs*2] * tmp;
18116 CC[2+bs*3] -= CC[2+bs*2] * tmp;
18117 CC[3+bs*3] -= CC[3+bs*2] * tmp;
18118
18119
18120 D[0+bs*0] = CC[0+bs*0];
18121 D[1+bs*0] = CC[1+bs*0];
18122 D[2+bs*0] = CC[2+bs*0];
18123 D[3+bs*0] = CC[3+bs*0];
18124
18125 D[0+bs*1] = CC[0+bs*1];
18126 D[1+bs*1] = CC[1+bs*1];
18127 D[2+bs*1] = CC[2+bs*1];
18128 D[3+bs*1] = CC[3+bs*1];
18129
18130 D[0+bs*2] = CC[0+bs*2];
18131 D[1+bs*2] = CC[1+bs*2];
18132 D[2+bs*2] = CC[2+bs*2];
18133 D[3+bs*2] = CC[3+bs*2];
18134
18135 D[0+bs*3] = CC[0+bs*3];
18136 D[1+bs*3] = CC[1+bs*3];
18137 D[2+bs*3] = CC[2+bs*3];
18138 D[3+bs*3] = CC[3+bs*3];
18139
18140 return;
18141
18142 }
18143 //#endif
18144
18145
18146
18147 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_one_4x4_vs_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,int m1,int n1)18148 void kernel_strsm_nt_rl_one_4x4_vs_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, int m1, int n1)
18149 {
18150
18151 const int bs = 4;
18152
18153 float tmp;
18154
18155 #if defined(TARGET_GENERIC)
18156 float CC[16] = {0};
18157 #else
18158 ALIGNED( float CC[16], 64 ) = {0};
18159 #endif
18160
18161 float alpha1 = -1.0;
18162
18163 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
18164
18165 if(n1==1)
18166 goto store;
18167
18168 tmp = E[1+lde*0];
18169 CC[0+bs*1] -= CC[0+bs*0] * tmp;
18170 CC[1+bs*1] -= CC[1+bs*0] * tmp;
18171 CC[2+bs*1] -= CC[2+bs*0] * tmp;
18172 CC[3+bs*1] -= CC[3+bs*0] * tmp;
18173
18174 if(n1==2)
18175 goto store;
18176
18177 tmp = E[2+lde*0];
18178 CC[0+bs*2] -= CC[0+bs*0] * tmp;
18179 CC[1+bs*2] -= CC[1+bs*0] * tmp;
18180 CC[2+bs*2] -= CC[2+bs*0] * tmp;
18181 CC[3+bs*2] -= CC[3+bs*0] * tmp;
18182 tmp = E[2+lde*1];
18183 CC[0+bs*2] -= CC[0+bs*1] * tmp;
18184 CC[1+bs*2] -= CC[1+bs*1] * tmp;
18185 CC[2+bs*2] -= CC[2+bs*1] * tmp;
18186 CC[3+bs*2] -= CC[3+bs*1] * tmp;
18187
18188 if(n1==3)
18189 goto store;
18190
18191 tmp = E[3+lde*0];
18192 CC[0+bs*3] -= CC[0+bs*0] * tmp;
18193 CC[1+bs*3] -= CC[1+bs*0] * tmp;
18194 CC[2+bs*3] -= CC[2+bs*0] * tmp;
18195 CC[3+bs*3] -= CC[3+bs*0] * tmp;
18196 tmp = E[3+lde*1];
18197 CC[0+bs*3] -= CC[0+bs*1] * tmp;
18198 CC[1+bs*3] -= CC[1+bs*1] * tmp;
18199 CC[2+bs*3] -= CC[2+bs*1] * tmp;
18200 CC[3+bs*3] -= CC[3+bs*1] * tmp;
18201 tmp = E[3+lde*2];
18202 CC[0+bs*3] -= CC[0+bs*2] * tmp;
18203 CC[1+bs*3] -= CC[1+bs*2] * tmp;
18204 CC[2+bs*3] -= CC[2+bs*2] * tmp;
18205 CC[3+bs*3] -= CC[3+bs*2] * tmp;
18206
18207 store:
18208
18209 if(m1>=4)
18210 {
18211 D[0+bs*0] = CC[0+bs*0];
18212 D[1+bs*0] = CC[1+bs*0];
18213 D[2+bs*0] = CC[2+bs*0];
18214 D[3+bs*0] = CC[3+bs*0];
18215
18216 if(n1==1)
18217 return;
18218
18219 D[0+bs*1] = CC[0+bs*1];
18220 D[1+bs*1] = CC[1+bs*1];
18221 D[2+bs*1] = CC[2+bs*1];
18222 D[3+bs*1] = CC[3+bs*1];
18223
18224 if(n1==2)
18225 return;
18226
18227 D[0+bs*2] = CC[0+bs*2];
18228 D[1+bs*2] = CC[1+bs*2];
18229 D[2+bs*2] = CC[2+bs*2];
18230 D[3+bs*2] = CC[3+bs*2];
18231
18232 if(n1==3)
18233 return;
18234
18235 D[0+bs*3] = CC[0+bs*3];
18236 D[1+bs*3] = CC[1+bs*3];
18237 D[2+bs*3] = CC[2+bs*3];
18238 D[3+bs*3] = CC[3+bs*3];
18239 }
18240 else if(m1>=3)
18241 {
18242 D[0+bs*0] = CC[0+bs*0];
18243 D[1+bs*0] = CC[1+bs*0];
18244 D[2+bs*0] = CC[2+bs*0];
18245
18246 if(n1==1)
18247 return;
18248
18249 D[0+bs*1] = CC[0+bs*1];
18250 D[1+bs*1] = CC[1+bs*1];
18251 D[2+bs*1] = CC[2+bs*1];
18252
18253 if(n1==2)
18254 return;
18255
18256 D[0+bs*2] = CC[0+bs*2];
18257 D[1+bs*2] = CC[1+bs*2];
18258 D[2+bs*2] = CC[2+bs*2];
18259
18260 if(n1==3)
18261 return;
18262
18263 D[0+bs*3] = CC[0+bs*3];
18264 D[1+bs*3] = CC[1+bs*3];
18265 D[2+bs*3] = CC[2+bs*3];
18266 }
18267 else if(m1>=2)
18268 {
18269 D[0+bs*0] = CC[0+bs*0];
18270 D[1+bs*0] = CC[1+bs*0];
18271
18272 if(n1==1)
18273 return;
18274
18275 D[0+bs*1] = CC[0+bs*1];
18276 D[1+bs*1] = CC[1+bs*1];
18277
18278 if(n1==2)
18279 return;
18280
18281 D[0+bs*2] = CC[0+bs*2];
18282 D[1+bs*2] = CC[1+bs*2];
18283
18284 if(n1==3)
18285 return;
18286
18287 D[0+bs*3] = CC[0+bs*3];
18288 D[1+bs*3] = CC[1+bs*3];
18289 }
18290 else //if(m1>=1)
18291 {
18292 D[0+bs*0] = CC[0+bs*0];
18293
18294 if(n1==1)
18295 return;
18296
18297 D[0+bs*1] = CC[0+bs*1];
18298
18299 if(n1==2)
18300 return;
18301
18302 D[0+bs*2] = CC[0+bs*2];
18303
18304 if(n1==3)
18305 return;
18306
18307 D[0+bs*3] = CC[0+bs*3];
18308 }
18309
18310 return;
18311
18312 }
18313 //#endif
18314
18315
18316
18317 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_one_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde)18318 void kernel_strsm_nt_rl_one_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde)
18319 {
18320
18321 const int bs = 4;
18322
18323 float tmp;
18324
18325 #if defined(TARGET_GENERIC)
18326 float CC[16] = {0};
18327 #else
18328 ALIGNED( float CC[16], 64 ) = {0};
18329 #endif
18330
18331 float alpha1 = -1.0;
18332
18333 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
18334
18335 tmp = E[1+lde*0];
18336 CC[0+bs*1] -= CC[0+bs*0] * tmp;
18337 CC[1+bs*1] -= CC[1+bs*0] * tmp;
18338 CC[2+bs*1] -= CC[2+bs*0] * tmp;
18339 CC[3+bs*1] -= CC[3+bs*0] * tmp;
18340
18341 tmp = E[2+lde*0];
18342 CC[0+bs*2] -= CC[0+bs*0] * tmp;
18343 CC[1+bs*2] -= CC[1+bs*0] * tmp;
18344 CC[2+bs*2] -= CC[2+bs*0] * tmp;
18345 CC[3+bs*2] -= CC[3+bs*0] * tmp;
18346 tmp = E[2+lde*1];
18347 CC[0+bs*2] -= CC[0+bs*1] * tmp;
18348 CC[1+bs*2] -= CC[1+bs*1] * tmp;
18349 CC[2+bs*2] -= CC[2+bs*1] * tmp;
18350 CC[3+bs*2] -= CC[3+bs*1] * tmp;
18351
18352 tmp = E[3+lde*0];
18353 CC[0+bs*3] -= CC[0+bs*0] * tmp;
18354 CC[1+bs*3] -= CC[1+bs*0] * tmp;
18355 CC[2+bs*3] -= CC[2+bs*0] * tmp;
18356 CC[3+bs*3] -= CC[3+bs*0] * tmp;
18357 tmp = E[3+lde*1];
18358 CC[0+bs*3] -= CC[0+bs*1] * tmp;
18359 CC[1+bs*3] -= CC[1+bs*1] * tmp;
18360 CC[2+bs*3] -= CC[2+bs*1] * tmp;
18361 CC[3+bs*3] -= CC[3+bs*1] * tmp;
18362 tmp = E[3+lde*2];
18363 CC[0+bs*3] -= CC[0+bs*2] * tmp;
18364 CC[1+bs*3] -= CC[1+bs*2] * tmp;
18365 CC[2+bs*3] -= CC[2+bs*2] * tmp;
18366 CC[3+bs*3] -= CC[3+bs*2] * tmp;
18367
18368 D[0+ldd*0] = CC[0+bs*0];
18369 D[1+ldd*0] = CC[1+bs*0];
18370 D[2+ldd*0] = CC[2+bs*0];
18371 D[3+ldd*0] = CC[3+bs*0];
18372
18373 D[0+ldd*1] = CC[0+bs*1];
18374 D[1+ldd*1] = CC[1+bs*1];
18375 D[2+ldd*1] = CC[2+bs*1];
18376 D[3+ldd*1] = CC[3+bs*1];
18377
18378 D[0+ldd*2] = CC[0+bs*2];
18379 D[1+ldd*2] = CC[1+bs*2];
18380 D[2+ldd*2] = CC[2+bs*2];
18381 D[3+ldd*2] = CC[3+bs*2];
18382
18383 D[0+ldd*3] = CC[0+bs*3];
18384 D[1+ldd*3] = CC[1+bs*3];
18385 D[2+ldd*3] = CC[2+bs*3];
18386 D[3+ldd*3] = CC[3+bs*3];
18387
18388 return;
18389
18390 }
18391 //#endif
18392
18393
18394
18395 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_rl_one_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,int m1,int n1)18396 void kernel_strsm_nt_rl_one_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, int m1, int n1)
18397 {
18398
18399 const int bs = 4;
18400
18401 float tmp;
18402
18403 #if defined(TARGET_GENERIC)
18404 float CC[16] = {0};
18405 #else
18406 ALIGNED( float CC[16], 64 ) = {0};
18407 #endif
18408
18409 float alpha1 = -1.0;
18410
18411 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
18412
18413 if(n1==1)
18414 goto store;
18415
18416 tmp = E[1+lde*0];
18417 CC[0+bs*1] -= CC[0+bs*0] * tmp;
18418 CC[1+bs*1] -= CC[1+bs*0] * tmp;
18419 CC[2+bs*1] -= CC[2+bs*0] * tmp;
18420 CC[3+bs*1] -= CC[3+bs*0] * tmp;
18421
18422 if(n1==2)
18423 goto store;
18424
18425 tmp = E[2+lde*0];
18426 CC[0+bs*2] -= CC[0+bs*0] * tmp;
18427 CC[1+bs*2] -= CC[1+bs*0] * tmp;
18428 CC[2+bs*2] -= CC[2+bs*0] * tmp;
18429 CC[3+bs*2] -= CC[3+bs*0] * tmp;
18430 tmp = E[2+lde*1];
18431 CC[0+bs*2] -= CC[0+bs*1] * tmp;
18432 CC[1+bs*2] -= CC[1+bs*1] * tmp;
18433 CC[2+bs*2] -= CC[2+bs*1] * tmp;
18434 CC[3+bs*2] -= CC[3+bs*1] * tmp;
18435
18436 if(n1==3)
18437 goto store;
18438
18439 tmp = E[3+lde*0];
18440 CC[0+bs*3] -= CC[0+bs*0] * tmp;
18441 CC[1+bs*3] -= CC[1+bs*0] * tmp;
18442 CC[2+bs*3] -= CC[2+bs*0] * tmp;
18443 CC[3+bs*3] -= CC[3+bs*0] * tmp;
18444 tmp = E[3+lde*1];
18445 CC[0+bs*3] -= CC[0+bs*1] * tmp;
18446 CC[1+bs*3] -= CC[1+bs*1] * tmp;
18447 CC[2+bs*3] -= CC[2+bs*1] * tmp;
18448 CC[3+bs*3] -= CC[3+bs*1] * tmp;
18449 tmp = E[3+lde*2];
18450 CC[0+bs*3] -= CC[0+bs*2] * tmp;
18451 CC[1+bs*3] -= CC[1+bs*2] * tmp;
18452 CC[2+bs*3] -= CC[2+bs*2] * tmp;
18453 CC[3+bs*3] -= CC[3+bs*2] * tmp;
18454
18455 store:
18456
18457 if(m1>=4)
18458 {
18459 D[0+ldd*0] = CC[0+bs*0];
18460 D[1+ldd*0] = CC[1+bs*0];
18461 D[2+ldd*0] = CC[2+bs*0];
18462 D[3+ldd*0] = CC[3+bs*0];
18463
18464 if(n1==1)
18465 return;
18466
18467 D[0+ldd*1] = CC[0+bs*1];
18468 D[1+ldd*1] = CC[1+bs*1];
18469 D[2+ldd*1] = CC[2+bs*1];
18470 D[3+ldd*1] = CC[3+bs*1];
18471
18472 if(n1==2)
18473 return;
18474
18475 D[0+ldd*2] = CC[0+bs*2];
18476 D[1+ldd*2] = CC[1+bs*2];
18477 D[2+ldd*2] = CC[2+bs*2];
18478 D[3+ldd*2] = CC[3+bs*2];
18479
18480 if(n1==3)
18481 return;
18482
18483 D[0+ldd*3] = CC[0+bs*3];
18484 D[1+ldd*3] = CC[1+bs*3];
18485 D[2+ldd*3] = CC[2+bs*3];
18486 D[3+ldd*3] = CC[3+bs*3];
18487 }
18488 else if(m1>=3)
18489 {
18490 D[0+ldd*0] = CC[0+bs*0];
18491 D[1+ldd*0] = CC[1+bs*0];
18492 D[2+ldd*0] = CC[2+bs*0];
18493
18494 if(n1==1)
18495 return;
18496
18497 D[0+ldd*1] = CC[0+bs*1];
18498 D[1+ldd*1] = CC[1+bs*1];
18499 D[2+ldd*1] = CC[2+bs*1];
18500
18501 if(n1==2)
18502 return;
18503
18504 D[0+ldd*2] = CC[0+bs*2];
18505 D[1+ldd*2] = CC[1+bs*2];
18506 D[2+ldd*2] = CC[2+bs*2];
18507
18508 if(n1==3)
18509 return;
18510
18511 D[0+ldd*3] = CC[0+bs*3];
18512 D[1+ldd*3] = CC[1+bs*3];
18513 D[2+ldd*3] = CC[2+bs*3];
18514 }
18515 else if(m1>=2)
18516 {
18517 D[0+ldd*0] = CC[0+bs*0];
18518 D[1+ldd*0] = CC[1+bs*0];
18519
18520 if(n1==1)
18521 return;
18522
18523 D[0+ldd*1] = CC[0+bs*1];
18524 D[1+ldd*1] = CC[1+bs*1];
18525
18526 if(n1==2)
18527 return;
18528
18529 D[0+ldd*2] = CC[0+bs*2];
18530 D[1+ldd*2] = CC[1+bs*2];
18531
18532 if(n1==3)
18533 return;
18534
18535 D[0+ldd*3] = CC[0+bs*3];
18536 D[1+ldd*3] = CC[1+bs*3];
18537 }
18538 else //if(m1>=1)
18539 {
18540 D[0+ldd*0] = CC[0+bs*0];
18541
18542 if(n1==1)
18543 return;
18544
18545 D[0+ldd*1] = CC[0+bs*1];
18546
18547 if(n1==2)
18548 return;
18549
18550 D[0+ldd*2] = CC[0+bs*2];
18551
18552 if(n1==3)
18553 return;
18554
18555 D[0+ldd*3] = CC[0+bs*3];
18556 }
18557
18558 return;
18559
18560 }
18561 //#endif
18562
18563
18564
18565 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_ru_inv_4x4_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,float * inv_diag_E)18566 void kernel_strsm_nn_ru_inv_4x4_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, float *inv_diag_E)
18567 {
18568
18569 const int bs = 4;
18570
18571 float tmp;
18572
18573 #if defined(TARGET_GENERIC)
18574 float CC[16] = {0};
18575 #else
18576 ALIGNED( float CC[16], 64 ) = {0};
18577 #endif
18578
18579 float alpha1 = -1.0;
18580
18581 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
18582
18583 tmp = inv_diag_E[0];
18584 CC[0+bs*0] *= tmp;
18585 CC[1+bs*0] *= tmp;
18586 CC[2+bs*0] *= tmp;
18587 CC[3+bs*0] *= tmp;
18588
18589 tmp = E[0+lde*1];
18590 CC[0+bs*1] -= CC[0+bs*0] * tmp;
18591 CC[1+bs*1] -= CC[1+bs*0] * tmp;
18592 CC[2+bs*1] -= CC[2+bs*0] * tmp;
18593 CC[3+bs*1] -= CC[3+bs*0] * tmp;
18594 tmp = inv_diag_E[1];
18595 CC[0+bs*1] *= tmp;
18596 CC[1+bs*1] *= tmp;
18597 CC[2+bs*1] *= tmp;
18598 CC[3+bs*1] *= tmp;
18599
18600 tmp = E[0+lde*2];
18601 CC[0+bs*2] -= CC[0+bs*0] * tmp;
18602 CC[1+bs*2] -= CC[1+bs*0] * tmp;
18603 CC[2+bs*2] -= CC[2+bs*0] * tmp;
18604 CC[3+bs*2] -= CC[3+bs*0] * tmp;
18605 tmp = E[1+lde*2];
18606 CC[0+bs*2] -= CC[0+bs*1] * tmp;
18607 CC[1+bs*2] -= CC[1+bs*1] * tmp;
18608 CC[2+bs*2] -= CC[2+bs*1] * tmp;
18609 CC[3+bs*2] -= CC[3+bs*1] * tmp;
18610 tmp = inv_diag_E[2];
18611 CC[0+bs*2] *= tmp;
18612 CC[1+bs*2] *= tmp;
18613 CC[2+bs*2] *= tmp;
18614 CC[3+bs*2] *= tmp;
18615
18616 tmp = E[0+lde*3];
18617 CC[0+bs*3] -= CC[0+bs*0] * tmp;
18618 CC[1+bs*3] -= CC[1+bs*0] * tmp;
18619 CC[2+bs*3] -= CC[2+bs*0] * tmp;
18620 CC[3+bs*3] -= CC[3+bs*0] * tmp;
18621 tmp = E[1+lde*3];
18622 CC[0+bs*3] -= CC[0+bs*1] * tmp;
18623 CC[1+bs*3] -= CC[1+bs*1] * tmp;
18624 CC[2+bs*3] -= CC[2+bs*1] * tmp;
18625 CC[3+bs*3] -= CC[3+bs*1] * tmp;
18626 tmp = E[2+lde*3];
18627 CC[0+bs*3] -= CC[0+bs*2] * tmp;
18628 CC[1+bs*3] -= CC[1+bs*2] * tmp;
18629 CC[2+bs*3] -= CC[2+bs*2] * tmp;
18630 CC[3+bs*3] -= CC[3+bs*2] * tmp;
18631 tmp = inv_diag_E[3];
18632 CC[0+bs*3] *= tmp;
18633 CC[1+bs*3] *= tmp;
18634 CC[2+bs*3] *= tmp;
18635 CC[3+bs*3] *= tmp;
18636
18637 D[0+bs*0] = CC[0+bs*0];
18638 D[1+bs*0] = CC[1+bs*0];
18639 D[2+bs*0] = CC[2+bs*0];
18640 D[3+bs*0] = CC[3+bs*0];
18641
18642 D[0+bs*1] = CC[0+bs*1];
18643 D[1+bs*1] = CC[1+bs*1];
18644 D[2+bs*1] = CC[2+bs*1];
18645 D[3+bs*1] = CC[3+bs*1];
18646
18647 D[0+bs*2] = CC[0+bs*2];
18648 D[1+bs*2] = CC[1+bs*2];
18649 D[2+bs*2] = CC[2+bs*2];
18650 D[3+bs*2] = CC[3+bs*2];
18651
18652 D[0+bs*3] = CC[0+bs*3];
18653 D[1+bs*3] = CC[1+bs*3];
18654 D[2+bs*3] = CC[2+bs*3];
18655 D[3+bs*3] = CC[3+bs*3];
18656
18657 return;
18658
18659 }
18660 //#endif
18661
18662
18663
18664 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_ru_inv_4x4_vs_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,float * inv_diag_E,int m1,int n1)18665 void kernel_strsm_nn_ru_inv_4x4_vs_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, float *inv_diag_E, int m1, int n1)
18666 {
18667
18668 const int bs = 4;
18669
18670 float tmp;
18671
18672 #if defined(TARGET_GENERIC)
18673 float CC[16] = {0};
18674 #else
18675 ALIGNED( float CC[16], 64 ) = {0};
18676 #endif
18677
18678 float alpha1 = -1.0;
18679
18680 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
18681
18682 tmp = inv_diag_E[0];
18683 CC[0+bs*0] *= tmp;
18684 CC[1+bs*0] *= tmp;
18685 CC[2+bs*0] *= tmp;
18686 CC[3+bs*0] *= tmp;
18687
18688 if(n1==1)
18689 goto store;
18690
18691 tmp = E[0+lde*1];
18692 CC[0+bs*1] -= CC[0+bs*0] * tmp;
18693 CC[1+bs*1] -= CC[1+bs*0] * tmp;
18694 CC[2+bs*1] -= CC[2+bs*0] * tmp;
18695 CC[3+bs*1] -= CC[3+bs*0] * tmp;
18696 tmp = inv_diag_E[1];
18697 CC[0+bs*1] *= tmp;
18698 CC[1+bs*1] *= tmp;
18699 CC[2+bs*1] *= tmp;
18700 CC[3+bs*1] *= tmp;
18701
18702 if(n1==2)
18703 goto store;
18704
18705 tmp = E[0+lde*2];
18706 CC[0+bs*2] -= CC[0+bs*0] * tmp;
18707 CC[1+bs*2] -= CC[1+bs*0] * tmp;
18708 CC[2+bs*2] -= CC[2+bs*0] * tmp;
18709 CC[3+bs*2] -= CC[3+bs*0] * tmp;
18710 tmp = E[1+lde*2];
18711 CC[0+bs*2] -= CC[0+bs*1] * tmp;
18712 CC[1+bs*2] -= CC[1+bs*1] * tmp;
18713 CC[2+bs*2] -= CC[2+bs*1] * tmp;
18714 CC[3+bs*2] -= CC[3+bs*1] * tmp;
18715 tmp = inv_diag_E[2];
18716 CC[0+bs*2] *= tmp;
18717 CC[1+bs*2] *= tmp;
18718 CC[2+bs*2] *= tmp;
18719 CC[3+bs*2] *= tmp;
18720
18721 if(n1==3)
18722 goto store;
18723
18724 tmp = E[0+lde*3];
18725 CC[0+bs*3] -= CC[0+bs*0] * tmp;
18726 CC[1+bs*3] -= CC[1+bs*0] * tmp;
18727 CC[2+bs*3] -= CC[2+bs*0] * tmp;
18728 CC[3+bs*3] -= CC[3+bs*0] * tmp;
18729 tmp = E[1+lde*3];
18730 CC[0+bs*3] -= CC[0+bs*1] * tmp;
18731 CC[1+bs*3] -= CC[1+bs*1] * tmp;
18732 CC[2+bs*3] -= CC[2+bs*1] * tmp;
18733 CC[3+bs*3] -= CC[3+bs*1] * tmp;
18734 tmp = E[2+lde*3];
18735 CC[0+bs*3] -= CC[0+bs*2] * tmp;
18736 CC[1+bs*3] -= CC[1+bs*2] * tmp;
18737 CC[2+bs*3] -= CC[2+bs*2] * tmp;
18738 CC[3+bs*3] -= CC[3+bs*2] * tmp;
18739 tmp = inv_diag_E[3];
18740 CC[0+bs*3] *= tmp;
18741 CC[1+bs*3] *= tmp;
18742 CC[2+bs*3] *= tmp;
18743 CC[3+bs*3] *= tmp;
18744
18745 store:
18746
18747 if(m1>=4)
18748 {
18749 D[0+bs*0] = CC[0+bs*0];
18750 D[1+bs*0] = CC[1+bs*0];
18751 D[2+bs*0] = CC[2+bs*0];
18752 D[3+bs*0] = CC[3+bs*0];
18753
18754 if(n1==1)
18755 return;
18756
18757 D[0+bs*1] = CC[0+bs*1];
18758 D[1+bs*1] = CC[1+bs*1];
18759 D[2+bs*1] = CC[2+bs*1];
18760 D[3+bs*1] = CC[3+bs*1];
18761
18762 if(n1==2)
18763 return;
18764
18765 D[0+bs*2] = CC[0+bs*2];
18766 D[1+bs*2] = CC[1+bs*2];
18767 D[2+bs*2] = CC[2+bs*2];
18768 D[3+bs*2] = CC[3+bs*2];
18769
18770 if(n1==3)
18771 return;
18772
18773 D[0+bs*3] = CC[0+bs*3];
18774 D[1+bs*3] = CC[1+bs*3];
18775 D[2+bs*3] = CC[2+bs*3];
18776 D[3+bs*3] = CC[3+bs*3];
18777 }
18778 else if(m1>=3)
18779 {
18780 D[0+bs*0] = CC[0+bs*0];
18781 D[1+bs*0] = CC[1+bs*0];
18782 D[2+bs*0] = CC[2+bs*0];
18783
18784 if(n1==1)
18785 return;
18786
18787 D[0+bs*1] = CC[0+bs*1];
18788 D[1+bs*1] = CC[1+bs*1];
18789 D[2+bs*1] = CC[2+bs*1];
18790
18791 if(n1==2)
18792 return;
18793
18794 D[0+bs*2] = CC[0+bs*2];
18795 D[1+bs*2] = CC[1+bs*2];
18796 D[2+bs*2] = CC[2+bs*2];
18797
18798 if(n1==3)
18799 return;
18800
18801 D[0+bs*3] = CC[0+bs*3];
18802 D[1+bs*3] = CC[1+bs*3];
18803 D[2+bs*3] = CC[2+bs*3];
18804 }
18805 else if(m1>=2)
18806 {
18807 D[0+bs*0] = CC[0+bs*0];
18808 D[1+bs*0] = CC[1+bs*0];
18809
18810 if(n1==1)
18811 return;
18812
18813 D[0+bs*1] = CC[0+bs*1];
18814 D[1+bs*1] = CC[1+bs*1];
18815
18816 if(n1==2)
18817 return;
18818
18819 D[0+bs*2] = CC[0+bs*2];
18820 D[1+bs*2] = CC[1+bs*2];
18821
18822 if(n1==3)
18823 return;
18824
18825 D[0+bs*3] = CC[0+bs*3];
18826 D[1+bs*3] = CC[1+bs*3];
18827 }
18828 else //if(m1>=1)
18829 {
18830 D[0+bs*0] = CC[0+bs*0];
18831
18832 if(n1==1)
18833 return;
18834
18835 D[0+bs*1] = CC[0+bs*1];
18836
18837 if(n1==2)
18838 return;
18839
18840 D[0+bs*2] = CC[0+bs*2];
18841
18842 if(n1==3)
18843 return;
18844
18845 D[0+bs*3] = CC[0+bs*3];
18846 }
18847
18848 return;
18849
18850 }
18851 //#endif
18852
18853
18854
18855 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_ru_inv_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E)18856 void kernel_strsm_nn_ru_inv_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E)
18857 {
18858
18859 const int bs = 4;
18860
18861 float tmp;
18862
18863 #if defined(TARGET_GENERIC)
18864 float CC[16] = {0};
18865 #else
18866 ALIGNED( float CC[16], 64 ) = {0};
18867 #endif
18868
18869 float alpha1 = -1.0;
18870
18871 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
18872
18873 tmp = inv_diag_E[0];
18874 CC[0+bs*0] *= tmp;
18875 CC[1+bs*0] *= tmp;
18876 CC[2+bs*0] *= tmp;
18877 CC[3+bs*0] *= tmp;
18878
18879 tmp = E[0+lde*1];
18880 CC[0+bs*1] -= CC[0+bs*0] * tmp;
18881 CC[1+bs*1] -= CC[1+bs*0] * tmp;
18882 CC[2+bs*1] -= CC[2+bs*0] * tmp;
18883 CC[3+bs*1] -= CC[3+bs*0] * tmp;
18884 tmp = inv_diag_E[1];
18885 CC[0+bs*1] *= tmp;
18886 CC[1+bs*1] *= tmp;
18887 CC[2+bs*1] *= tmp;
18888 CC[3+bs*1] *= tmp;
18889
18890 tmp = E[0+lde*2];
18891 CC[0+bs*2] -= CC[0+bs*0] * tmp;
18892 CC[1+bs*2] -= CC[1+bs*0] * tmp;
18893 CC[2+bs*2] -= CC[2+bs*0] * tmp;
18894 CC[3+bs*2] -= CC[3+bs*0] * tmp;
18895 tmp = E[1+lde*2];
18896 CC[0+bs*2] -= CC[0+bs*1] * tmp;
18897 CC[1+bs*2] -= CC[1+bs*1] * tmp;
18898 CC[2+bs*2] -= CC[2+bs*1] * tmp;
18899 CC[3+bs*2] -= CC[3+bs*1] * tmp;
18900 tmp = inv_diag_E[2];
18901 CC[0+bs*2] *= tmp;
18902 CC[1+bs*2] *= tmp;
18903 CC[2+bs*2] *= tmp;
18904 CC[3+bs*2] *= tmp;
18905
18906 tmp = E[0+lde*3];
18907 CC[0+bs*3] -= CC[0+bs*0] * tmp;
18908 CC[1+bs*3] -= CC[1+bs*0] * tmp;
18909 CC[2+bs*3] -= CC[2+bs*0] * tmp;
18910 CC[3+bs*3] -= CC[3+bs*0] * tmp;
18911 tmp = E[1+lde*3];
18912 CC[0+bs*3] -= CC[0+bs*1] * tmp;
18913 CC[1+bs*3] -= CC[1+bs*1] * tmp;
18914 CC[2+bs*3] -= CC[2+bs*1] * tmp;
18915 CC[3+bs*3] -= CC[3+bs*1] * tmp;
18916 tmp = E[2+lde*3];
18917 CC[0+bs*3] -= CC[0+bs*2] * tmp;
18918 CC[1+bs*3] -= CC[1+bs*2] * tmp;
18919 CC[2+bs*3] -= CC[2+bs*2] * tmp;
18920 CC[3+bs*3] -= CC[3+bs*2] * tmp;
18921 tmp = inv_diag_E[3];
18922 CC[0+bs*3] *= tmp;
18923 CC[1+bs*3] *= tmp;
18924 CC[2+bs*3] *= tmp;
18925 CC[3+bs*3] *= tmp;
18926
18927
18928 D[0+ldd*0] = CC[0+bs*0];
18929 D[1+ldd*0] = CC[1+bs*0];
18930 D[2+ldd*0] = CC[2+bs*0];
18931 D[3+ldd*0] = CC[3+bs*0];
18932
18933 D[0+ldd*1] = CC[0+bs*1];
18934 D[1+ldd*1] = CC[1+bs*1];
18935 D[2+ldd*1] = CC[2+bs*1];
18936 D[3+ldd*1] = CC[3+bs*1];
18937
18938 D[0+ldd*2] = CC[0+bs*2];
18939 D[1+ldd*2] = CC[1+bs*2];
18940 D[2+ldd*2] = CC[2+bs*2];
18941 D[3+ldd*2] = CC[3+bs*2];
18942
18943 D[0+ldd*3] = CC[0+bs*3];
18944 D[1+ldd*3] = CC[1+bs*3];
18945 D[2+ldd*3] = CC[2+bs*3];
18946 D[3+ldd*3] = CC[3+bs*3];
18947
18948 return;
18949
18950 }
18951 //#endif
18952
18953
18954
18955 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_ru_inv_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E,int m1,int n1)18956 void kernel_strsm_nn_ru_inv_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E, int m1, int n1)
18957 {
18958
18959 const int bs = 4;
18960
18961 float tmp;
18962
18963 #if defined(TARGET_GENERIC)
18964 float CC[16] = {0};
18965 #else
18966 ALIGNED( float CC[16], 64 ) = {0};
18967 #endif
18968
18969 float alpha1 = -1.0;
18970
18971 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
18972
18973 tmp = inv_diag_E[0];
18974 CC[0+bs*0] *= tmp;
18975 CC[1+bs*0] *= tmp;
18976 CC[2+bs*0] *= tmp;
18977 CC[3+bs*0] *= tmp;
18978
18979 if(n1==1)
18980 goto store;
18981
18982 tmp = E[0+lde*1];
18983 CC[0+bs*1] -= CC[0+bs*0] * tmp;
18984 CC[1+bs*1] -= CC[1+bs*0] * tmp;
18985 CC[2+bs*1] -= CC[2+bs*0] * tmp;
18986 CC[3+bs*1] -= CC[3+bs*0] * tmp;
18987 tmp = inv_diag_E[1];
18988 CC[0+bs*1] *= tmp;
18989 CC[1+bs*1] *= tmp;
18990 CC[2+bs*1] *= tmp;
18991 CC[3+bs*1] *= tmp;
18992
18993 if(n1==2)
18994 goto store;
18995
18996 tmp = E[0+lde*2];
18997 CC[0+bs*2] -= CC[0+bs*0] * tmp;
18998 CC[1+bs*2] -= CC[1+bs*0] * tmp;
18999 CC[2+bs*2] -= CC[2+bs*0] * tmp;
19000 CC[3+bs*2] -= CC[3+bs*0] * tmp;
19001 tmp = E[1+lde*2];
19002 CC[0+bs*2] -= CC[0+bs*1] * tmp;
19003 CC[1+bs*2] -= CC[1+bs*1] * tmp;
19004 CC[2+bs*2] -= CC[2+bs*1] * tmp;
19005 CC[3+bs*2] -= CC[3+bs*1] * tmp;
19006 tmp = inv_diag_E[2];
19007 CC[0+bs*2] *= tmp;
19008 CC[1+bs*2] *= tmp;
19009 CC[2+bs*2] *= tmp;
19010 CC[3+bs*2] *= tmp;
19011
19012 if(n1==3)
19013 goto store;
19014
19015 tmp = E[0+lde*3];
19016 CC[0+bs*3] -= CC[0+bs*0] * tmp;
19017 CC[1+bs*3] -= CC[1+bs*0] * tmp;
19018 CC[2+bs*3] -= CC[2+bs*0] * tmp;
19019 CC[3+bs*3] -= CC[3+bs*0] * tmp;
19020 tmp = E[1+lde*3];
19021 CC[0+bs*3] -= CC[0+bs*1] * tmp;
19022 CC[1+bs*3] -= CC[1+bs*1] * tmp;
19023 CC[2+bs*3] -= CC[2+bs*1] * tmp;
19024 CC[3+bs*3] -= CC[3+bs*1] * tmp;
19025 tmp = E[2+lde*3];
19026 CC[0+bs*3] -= CC[0+bs*2] * tmp;
19027 CC[1+bs*3] -= CC[1+bs*2] * tmp;
19028 CC[2+bs*3] -= CC[2+bs*2] * tmp;
19029 CC[3+bs*3] -= CC[3+bs*2] * tmp;
19030 tmp = inv_diag_E[3];
19031 CC[0+bs*3] *= tmp;
19032 CC[1+bs*3] *= tmp;
19033 CC[2+bs*3] *= tmp;
19034 CC[3+bs*3] *= tmp;
19035
19036 store:
19037
19038 if(m1>=4)
19039 {
19040 D[0+ldd*0] = CC[0+bs*0];
19041 D[1+ldd*0] = CC[1+bs*0];
19042 D[2+ldd*0] = CC[2+bs*0];
19043 D[3+ldd*0] = CC[3+bs*0];
19044
19045 if(n1==1)
19046 return;
19047
19048 D[0+ldd*1] = CC[0+bs*1];
19049 D[1+ldd*1] = CC[1+bs*1];
19050 D[2+ldd*1] = CC[2+bs*1];
19051 D[3+ldd*1] = CC[3+bs*1];
19052
19053 if(n1==2)
19054 return;
19055
19056 D[0+ldd*2] = CC[0+bs*2];
19057 D[1+ldd*2] = CC[1+bs*2];
19058 D[2+ldd*2] = CC[2+bs*2];
19059 D[3+ldd*2] = CC[3+bs*2];
19060
19061 if(n1==3)
19062 return;
19063
19064 D[0+ldd*3] = CC[0+bs*3];
19065 D[1+ldd*3] = CC[1+bs*3];
19066 D[2+ldd*3] = CC[2+bs*3];
19067 D[3+ldd*3] = CC[3+bs*3];
19068 }
19069 else if(m1>=3)
19070 {
19071 D[0+ldd*0] = CC[0+bs*0];
19072 D[1+ldd*0] = CC[1+bs*0];
19073 D[2+ldd*0] = CC[2+bs*0];
19074
19075 if(n1==1)
19076 return;
19077
19078 D[0+ldd*1] = CC[0+bs*1];
19079 D[1+ldd*1] = CC[1+bs*1];
19080 D[2+ldd*1] = CC[2+bs*1];
19081
19082 if(n1==2)
19083 return;
19084
19085 D[0+ldd*2] = CC[0+bs*2];
19086 D[1+ldd*2] = CC[1+bs*2];
19087 D[2+ldd*2] = CC[2+bs*2];
19088
19089 if(n1==3)
19090 return;
19091
19092 D[0+ldd*3] = CC[0+bs*3];
19093 D[1+ldd*3] = CC[1+bs*3];
19094 D[2+ldd*3] = CC[2+bs*3];
19095 }
19096 else if(m1>=2)
19097 {
19098 D[0+ldd*0] = CC[0+bs*0];
19099 D[1+ldd*0] = CC[1+bs*0];
19100
19101 if(n1==1)
19102 return;
19103
19104 D[0+ldd*1] = CC[0+bs*1];
19105 D[1+ldd*1] = CC[1+bs*1];
19106
19107 if(n1==2)
19108 return;
19109
19110 D[0+ldd*2] = CC[0+bs*2];
19111 D[1+ldd*2] = CC[1+bs*2];
19112
19113 if(n1==3)
19114 return;
19115
19116 D[0+ldd*3] = CC[0+bs*3];
19117 D[1+ldd*3] = CC[1+bs*3];
19118 }
19119 else //if(m1>=1)
19120 {
19121 D[0+ldd*0] = CC[0+bs*0];
19122
19123 if(n1==1)
19124 return;
19125
19126 D[0+ldd*1] = CC[0+bs*1];
19127
19128 if(n1==2)
19129 return;
19130
19131 D[0+ldd*2] = CC[0+bs*2];
19132
19133 if(n1==3)
19134 return;
19135
19136 D[0+ldd*3] = CC[0+bs*3];
19137 }
19138
19139 return;
19140
19141 }
19142 //#endif
19143
19144
19145
19146 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_ru_one_4x4_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde)19147 void kernel_strsm_nn_ru_one_4x4_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde)
19148 {
19149
19150 const int bs = 4;
19151
19152 float tmp;
19153
19154 #if defined(TARGET_GENERIC)
19155 float CC[16] = {0};
19156 #else
19157 ALIGNED( float CC[16], 64 ) = {0};
19158 #endif
19159
19160 float alpha1 = -1.0;
19161
19162 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
19163
19164 tmp = E[0+lde*1];
19165 CC[0+bs*1] -= CC[0+bs*0] * tmp;
19166 CC[1+bs*1] -= CC[1+bs*0] * tmp;
19167 CC[2+bs*1] -= CC[2+bs*0] * tmp;
19168 CC[3+bs*1] -= CC[3+bs*0] * tmp;
19169
19170 tmp = E[0+lde*2];
19171 CC[0+bs*2] -= CC[0+bs*0] * tmp;
19172 CC[1+bs*2] -= CC[1+bs*0] * tmp;
19173 CC[2+bs*2] -= CC[2+bs*0] * tmp;
19174 CC[3+bs*2] -= CC[3+bs*0] * tmp;
19175 tmp = E[1+lde*2];
19176 CC[0+bs*2] -= CC[0+bs*1] * tmp;
19177 CC[1+bs*2] -= CC[1+bs*1] * tmp;
19178 CC[2+bs*2] -= CC[2+bs*1] * tmp;
19179 CC[3+bs*2] -= CC[3+bs*1] * tmp;
19180
19181 tmp = E[0+lde*3];
19182 CC[0+bs*3] -= CC[0+bs*0] * tmp;
19183 CC[1+bs*3] -= CC[1+bs*0] * tmp;
19184 CC[2+bs*3] -= CC[2+bs*0] * tmp;
19185 CC[3+bs*3] -= CC[3+bs*0] * tmp;
19186 tmp = E[1+lde*3];
19187 CC[0+bs*3] -= CC[0+bs*1] * tmp;
19188 CC[1+bs*3] -= CC[1+bs*1] * tmp;
19189 CC[2+bs*3] -= CC[2+bs*1] * tmp;
19190 CC[3+bs*3] -= CC[3+bs*1] * tmp;
19191 tmp = E[2+lde*3];
19192 CC[0+bs*3] -= CC[0+bs*2] * tmp;
19193 CC[1+bs*3] -= CC[1+bs*2] * tmp;
19194 CC[2+bs*3] -= CC[2+bs*2] * tmp;
19195 CC[3+bs*3] -= CC[3+bs*2] * tmp;
19196
19197 D[0+bs*0] = CC[0+bs*0];
19198 D[1+bs*0] = CC[1+bs*0];
19199 D[2+bs*0] = CC[2+bs*0];
19200 D[3+bs*0] = CC[3+bs*0];
19201
19202 D[0+bs*1] = CC[0+bs*1];
19203 D[1+bs*1] = CC[1+bs*1];
19204 D[2+bs*1] = CC[2+bs*1];
19205 D[3+bs*1] = CC[3+bs*1];
19206
19207 D[0+bs*2] = CC[0+bs*2];
19208 D[1+bs*2] = CC[1+bs*2];
19209 D[2+bs*2] = CC[2+bs*2];
19210 D[3+bs*2] = CC[3+bs*2];
19211
19212 D[0+bs*3] = CC[0+bs*3];
19213 D[1+bs*3] = CC[1+bs*3];
19214 D[2+bs*3] = CC[2+bs*3];
19215 D[3+bs*3] = CC[3+bs*3];
19216
19217 return;
19218
19219 }
19220 //#endif
19221
19222
19223
19224 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_ru_one_4x4_vs_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,int m1,int n1)19225 void kernel_strsm_nn_ru_one_4x4_vs_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, int m1, int n1)
19226 {
19227
19228 const int bs = 4;
19229
19230 float tmp;
19231
19232 #if defined(TARGET_GENERIC)
19233 float CC[16] = {0};
19234 #else
19235 ALIGNED( float CC[16], 64 ) = {0};
19236 #endif
19237
19238 float alpha1 = -1.0;
19239
19240 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
19241
19242 if(n1==1)
19243 goto store;
19244
19245 tmp = E[0+lde*1];
19246 CC[0+bs*1] -= CC[0+bs*0] * tmp;
19247 CC[1+bs*1] -= CC[1+bs*0] * tmp;
19248 CC[2+bs*1] -= CC[2+bs*0] * tmp;
19249 CC[3+bs*1] -= CC[3+bs*0] * tmp;
19250
19251 if(n1==2)
19252 goto store;
19253
19254 tmp = E[0+lde*2];
19255 CC[0+bs*2] -= CC[0+bs*0] * tmp;
19256 CC[1+bs*2] -= CC[1+bs*0] * tmp;
19257 CC[2+bs*2] -= CC[2+bs*0] * tmp;
19258 CC[3+bs*2] -= CC[3+bs*0] * tmp;
19259 tmp = E[1+lde*2];
19260 CC[0+bs*2] -= CC[0+bs*1] * tmp;
19261 CC[1+bs*2] -= CC[1+bs*1] * tmp;
19262 CC[2+bs*2] -= CC[2+bs*1] * tmp;
19263 CC[3+bs*2] -= CC[3+bs*1] * tmp;
19264
19265 if(n1==3)
19266 goto store;
19267
19268 tmp = E[0+lde*3];
19269 CC[0+bs*3] -= CC[0+bs*0] * tmp;
19270 CC[1+bs*3] -= CC[1+bs*0] * tmp;
19271 CC[2+bs*3] -= CC[2+bs*0] * tmp;
19272 CC[3+bs*3] -= CC[3+bs*0] * tmp;
19273 tmp = E[1+lde*3];
19274 CC[0+bs*3] -= CC[0+bs*1] * tmp;
19275 CC[1+bs*3] -= CC[1+bs*1] * tmp;
19276 CC[2+bs*3] -= CC[2+bs*1] * tmp;
19277 CC[3+bs*3] -= CC[3+bs*1] * tmp;
19278 tmp = E[2+lde*3];
19279 CC[0+bs*3] -= CC[0+bs*2] * tmp;
19280 CC[1+bs*3] -= CC[1+bs*2] * tmp;
19281 CC[2+bs*3] -= CC[2+bs*2] * tmp;
19282 CC[3+bs*3] -= CC[3+bs*2] * tmp;
19283
19284 store:
19285
19286 if(m1>=4)
19287 {
19288 D[0+bs*0] = CC[0+bs*0];
19289 D[1+bs*0] = CC[1+bs*0];
19290 D[2+bs*0] = CC[2+bs*0];
19291 D[3+bs*0] = CC[3+bs*0];
19292
19293 if(n1==1)
19294 return;
19295
19296 D[0+bs*1] = CC[0+bs*1];
19297 D[1+bs*1] = CC[1+bs*1];
19298 D[2+bs*1] = CC[2+bs*1];
19299 D[3+bs*1] = CC[3+bs*1];
19300
19301 if(n1==2)
19302 return;
19303
19304 D[0+bs*2] = CC[0+bs*2];
19305 D[1+bs*2] = CC[1+bs*2];
19306 D[2+bs*2] = CC[2+bs*2];
19307 D[3+bs*2] = CC[3+bs*2];
19308
19309 if(n1==3)
19310 return;
19311
19312 D[0+bs*3] = CC[0+bs*3];
19313 D[1+bs*3] = CC[1+bs*3];
19314 D[2+bs*3] = CC[2+bs*3];
19315 D[3+bs*3] = CC[3+bs*3];
19316 }
19317 else if(m1>=3)
19318 {
19319 D[0+bs*0] = CC[0+bs*0];
19320 D[1+bs*0] = CC[1+bs*0];
19321 D[2+bs*0] = CC[2+bs*0];
19322
19323 if(n1==1)
19324 return;
19325
19326 D[0+bs*1] = CC[0+bs*1];
19327 D[1+bs*1] = CC[1+bs*1];
19328 D[2+bs*1] = CC[2+bs*1];
19329
19330 if(n1==2)
19331 return;
19332
19333 D[0+bs*2] = CC[0+bs*2];
19334 D[1+bs*2] = CC[1+bs*2];
19335 D[2+bs*2] = CC[2+bs*2];
19336
19337 if(n1==3)
19338 return;
19339
19340 D[0+bs*3] = CC[0+bs*3];
19341 D[1+bs*3] = CC[1+bs*3];
19342 D[2+bs*3] = CC[2+bs*3];
19343 }
19344 else if(m1>=2)
19345 {
19346 D[0+bs*0] = CC[0+bs*0];
19347 D[1+bs*0] = CC[1+bs*0];
19348
19349 if(n1==1)
19350 return;
19351
19352 D[0+bs*1] = CC[0+bs*1];
19353 D[1+bs*1] = CC[1+bs*1];
19354
19355 if(n1==2)
19356 return;
19357
19358 D[0+bs*2] = CC[0+bs*2];
19359 D[1+bs*2] = CC[1+bs*2];
19360
19361 if(n1==3)
19362 return;
19363
19364 D[0+bs*3] = CC[0+bs*3];
19365 D[1+bs*3] = CC[1+bs*3];
19366 }
19367 else //if(m1>=1)
19368 {
19369 D[0+bs*0] = CC[0+bs*0];
19370
19371 if(n1==1)
19372 return;
19373
19374 D[0+bs*1] = CC[0+bs*1];
19375
19376 if(n1==2)
19377 return;
19378
19379 D[0+bs*2] = CC[0+bs*2];
19380
19381 if(n1==3)
19382 return;
19383
19384 D[0+bs*3] = CC[0+bs*3];
19385 }
19386
19387 return;
19388
19389 }
19390 //#endif
19391
19392
19393
19394 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_ru_one_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde)19395 void kernel_strsm_nn_ru_one_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde)
19396 {
19397
19398 const int bs = 4;
19399
19400 float tmp;
19401
19402 #if defined(TARGET_GENERIC)
19403 float CC[16] = {0};
19404 #else
19405 ALIGNED( float CC[16], 64 ) = {0};
19406 #endif
19407
19408 float alpha1 = -1.0;
19409
19410 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
19411
19412 tmp = E[0+lde*1];
19413 CC[0+bs*1] -= CC[0+bs*0] * tmp;
19414 CC[1+bs*1] -= CC[1+bs*0] * tmp;
19415 CC[2+bs*1] -= CC[2+bs*0] * tmp;
19416 CC[3+bs*1] -= CC[3+bs*0] * tmp;
19417
19418 tmp = E[0+lde*2];
19419 CC[0+bs*2] -= CC[0+bs*0] * tmp;
19420 CC[1+bs*2] -= CC[1+bs*0] * tmp;
19421 CC[2+bs*2] -= CC[2+bs*0] * tmp;
19422 CC[3+bs*2] -= CC[3+bs*0] * tmp;
19423 tmp = E[1+lde*2];
19424 CC[0+bs*2] -= CC[0+bs*1] * tmp;
19425 CC[1+bs*2] -= CC[1+bs*1] * tmp;
19426 CC[2+bs*2] -= CC[2+bs*1] * tmp;
19427 CC[3+bs*2] -= CC[3+bs*1] * tmp;
19428
19429 tmp = E[0+lde*3];
19430 CC[0+bs*3] -= CC[0+bs*0] * tmp;
19431 CC[1+bs*3] -= CC[1+bs*0] * tmp;
19432 CC[2+bs*3] -= CC[2+bs*0] * tmp;
19433 CC[3+bs*3] -= CC[3+bs*0] * tmp;
19434 tmp = E[1+lde*3];
19435 CC[0+bs*3] -= CC[0+bs*1] * tmp;
19436 CC[1+bs*3] -= CC[1+bs*1] * tmp;
19437 CC[2+bs*3] -= CC[2+bs*1] * tmp;
19438 CC[3+bs*3] -= CC[3+bs*1] * tmp;
19439 tmp = E[2+lde*3];
19440 CC[0+bs*3] -= CC[0+bs*2] * tmp;
19441 CC[1+bs*3] -= CC[1+bs*2] * tmp;
19442 CC[2+bs*3] -= CC[2+bs*2] * tmp;
19443 CC[3+bs*3] -= CC[3+bs*2] * tmp;
19444
19445
19446 D[0+ldd*0] = CC[0+bs*0];
19447 D[1+ldd*0] = CC[1+bs*0];
19448 D[2+ldd*0] = CC[2+bs*0];
19449 D[3+ldd*0] = CC[3+bs*0];
19450
19451 D[0+ldd*1] = CC[0+bs*1];
19452 D[1+ldd*1] = CC[1+bs*1];
19453 D[2+ldd*1] = CC[2+bs*1];
19454 D[3+ldd*1] = CC[3+bs*1];
19455
19456 D[0+ldd*2] = CC[0+bs*2];
19457 D[1+ldd*2] = CC[1+bs*2];
19458 D[2+ldd*2] = CC[2+bs*2];
19459 D[3+ldd*2] = CC[3+bs*2];
19460
19461 D[0+ldd*3] = CC[0+bs*3];
19462 D[1+ldd*3] = CC[1+bs*3];
19463 D[2+ldd*3] = CC[2+bs*3];
19464 D[3+ldd*3] = CC[3+bs*3];
19465
19466 return;
19467
19468 }
19469 //#endif
19470
19471
19472
19473 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nn_ru_one_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,int m1,int n1)19474 void kernel_strsm_nn_ru_one_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, int m1, int n1)
19475 {
19476
19477 const int bs = 4;
19478
19479 float tmp;
19480
19481 #if defined(TARGET_GENERIC)
19482 float CC[16] = {0};
19483 #else
19484 ALIGNED( float CC[16], 64 ) = {0};
19485 #endif
19486
19487 float alpha1 = -1.0;
19488
19489 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
19490
19491 if(n1==1)
19492 goto store;
19493
19494 tmp = E[0+lde*1];
19495 CC[0+bs*1] -= CC[0+bs*0] * tmp;
19496 CC[1+bs*1] -= CC[1+bs*0] * tmp;
19497 CC[2+bs*1] -= CC[2+bs*0] * tmp;
19498 CC[3+bs*1] -= CC[3+bs*0] * tmp;
19499
19500 if(n1==2)
19501 goto store;
19502
19503 tmp = E[0+lde*2];
19504 CC[0+bs*2] -= CC[0+bs*0] * tmp;
19505 CC[1+bs*2] -= CC[1+bs*0] * tmp;
19506 CC[2+bs*2] -= CC[2+bs*0] * tmp;
19507 CC[3+bs*2] -= CC[3+bs*0] * tmp;
19508 tmp = E[1+lde*2];
19509 CC[0+bs*2] -= CC[0+bs*1] * tmp;
19510 CC[1+bs*2] -= CC[1+bs*1] * tmp;
19511 CC[2+bs*2] -= CC[2+bs*1] * tmp;
19512 CC[3+bs*2] -= CC[3+bs*1] * tmp;
19513
19514 if(n1==3)
19515 goto store;
19516
19517 tmp = E[0+lde*3];
19518 CC[0+bs*3] -= CC[0+bs*0] * tmp;
19519 CC[1+bs*3] -= CC[1+bs*0] * tmp;
19520 CC[2+bs*3] -= CC[2+bs*0] * tmp;
19521 CC[3+bs*3] -= CC[3+bs*0] * tmp;
19522 tmp = E[1+lde*3];
19523 CC[0+bs*3] -= CC[0+bs*1] * tmp;
19524 CC[1+bs*3] -= CC[1+bs*1] * tmp;
19525 CC[2+bs*3] -= CC[2+bs*1] * tmp;
19526 CC[3+bs*3] -= CC[3+bs*1] * tmp;
19527 tmp = E[2+lde*3];
19528 CC[0+bs*3] -= CC[0+bs*2] * tmp;
19529 CC[1+bs*3] -= CC[1+bs*2] * tmp;
19530 CC[2+bs*3] -= CC[2+bs*2] * tmp;
19531 CC[3+bs*3] -= CC[3+bs*2] * tmp;
19532
19533 store:
19534
19535 if(m1>=4)
19536 {
19537 D[0+ldd*0] = CC[0+bs*0];
19538 D[1+ldd*0] = CC[1+bs*0];
19539 D[2+ldd*0] = CC[2+bs*0];
19540 D[3+ldd*0] = CC[3+bs*0];
19541
19542 if(n1==1)
19543 return;
19544
19545 D[0+ldd*1] = CC[0+bs*1];
19546 D[1+ldd*1] = CC[1+bs*1];
19547 D[2+ldd*1] = CC[2+bs*1];
19548 D[3+ldd*1] = CC[3+bs*1];
19549
19550 if(n1==2)
19551 return;
19552
19553 D[0+ldd*2] = CC[0+bs*2];
19554 D[1+ldd*2] = CC[1+bs*2];
19555 D[2+ldd*2] = CC[2+bs*2];
19556 D[3+ldd*2] = CC[3+bs*2];
19557
19558 if(n1==3)
19559 return;
19560
19561 D[0+ldd*3] = CC[0+bs*3];
19562 D[1+ldd*3] = CC[1+bs*3];
19563 D[2+ldd*3] = CC[2+bs*3];
19564 D[3+ldd*3] = CC[3+bs*3];
19565 }
19566 else if(m1>=3)
19567 {
19568 D[0+ldd*0] = CC[0+bs*0];
19569 D[1+ldd*0] = CC[1+bs*0];
19570 D[2+ldd*0] = CC[2+bs*0];
19571
19572 if(n1==1)
19573 return;
19574
19575 D[0+ldd*1] = CC[0+bs*1];
19576 D[1+ldd*1] = CC[1+bs*1];
19577 D[2+ldd*1] = CC[2+bs*1];
19578
19579 if(n1==2)
19580 return;
19581
19582 D[0+ldd*2] = CC[0+bs*2];
19583 D[1+ldd*2] = CC[1+bs*2];
19584 D[2+ldd*2] = CC[2+bs*2];
19585
19586 if(n1==3)
19587 return;
19588
19589 D[0+ldd*3] = CC[0+bs*3];
19590 D[1+ldd*3] = CC[1+bs*3];
19591 D[2+ldd*3] = CC[2+bs*3];
19592 }
19593 else if(m1>=2)
19594 {
19595 D[0+ldd*0] = CC[0+bs*0];
19596 D[1+ldd*0] = CC[1+bs*0];
19597
19598 if(n1==1)
19599 return;
19600
19601 D[0+ldd*1] = CC[0+bs*1];
19602 D[1+ldd*1] = CC[1+bs*1];
19603
19604 if(n1==2)
19605 return;
19606
19607 D[0+ldd*2] = CC[0+bs*2];
19608 D[1+ldd*2] = CC[1+bs*2];
19609
19610 if(n1==3)
19611 return;
19612
19613 D[0+ldd*3] = CC[0+bs*3];
19614 D[1+ldd*3] = CC[1+bs*3];
19615 }
19616 else //if(m1>=1)
19617 {
19618 D[0+ldd*0] = CC[0+bs*0];
19619
19620 if(n1==1)
19621 return;
19622
19623 D[0+ldd*1] = CC[0+bs*1];
19624
19625 if(n1==2)
19626 return;
19627
19628 D[0+ldd*2] = CC[0+bs*2];
19629
19630 if(n1==3)
19631 return;
19632
19633 D[0+ldd*3] = CC[0+bs*3];
19634 }
19635
19636 return;
19637
19638 }
19639 //#endif
19640
19641
19642
19643 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_inv_4x4_lib44cc4(int kmax,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,float * E,float * inv_diag_E)19644 void kernel_strsm_nt_ru_inv_4x4_lib44cc4(int kmax, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, float *E, float *inv_diag_E)
19645 {
19646
19647 const int bs = 4;
19648
19649 float tmp;
19650
19651 #if defined(TARGET_GENERIC)
19652 float CC[16] = {0};
19653 #else
19654 ALIGNED( float CC[16], 64 ) = {0};
19655 #endif
19656
19657 float alpha1 = -1.0;
19658
19659 CC[0+bs*0] = C[0+ldc*0];
19660 CC[1+bs*0] = C[1+ldc*0];
19661 CC[2+bs*0] = C[2+ldc*0];
19662 CC[3+bs*0] = C[3+ldc*0];
19663
19664 CC[0+bs*1] = C[0+ldc*1];
19665 CC[1+bs*1] = C[1+ldc*1];
19666 CC[2+bs*1] = C[2+ldc*1];
19667 CC[3+bs*1] = C[3+ldc*1];
19668
19669 CC[0+bs*2] = C[0+ldc*2];
19670 CC[1+bs*2] = C[1+ldc*2];
19671 CC[2+bs*2] = C[2+ldc*2];
19672 CC[3+bs*2] = C[3+ldc*2];
19673
19674 CC[0+bs*3] = C[0+ldc*3];
19675 CC[1+bs*3] = C[1+ldc*3];
19676 CC[2+bs*3] = C[2+ldc*3];
19677 CC[3+bs*3] = C[3+ldc*3];
19678
19679 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, beta, CC, CC);
19680
19681 tmp = inv_diag_E[3];
19682 CC[0+bs*3] *= tmp;
19683 CC[1+bs*3] *= tmp;
19684 CC[2+bs*3] *= tmp;
19685 CC[3+bs*3] *= tmp;
19686 tmp = E[0+bs*3];
19687 CC[0+bs*0] -= CC[0+bs*3] * tmp;
19688 CC[1+bs*0] -= CC[1+bs*3] * tmp;
19689 CC[2+bs*0] -= CC[2+bs*3] * tmp;
19690 CC[3+bs*0] -= CC[3+bs*3] * tmp;
19691 tmp = E[1+bs*3];
19692 CC[0+bs*1] -= CC[0+bs*3] * tmp;
19693 CC[1+bs*1] -= CC[1+bs*3] * tmp;
19694 CC[2+bs*1] -= CC[2+bs*3] * tmp;
19695 CC[3+bs*1] -= CC[3+bs*3] * tmp;
19696 tmp = E[2+bs*3];
19697 CC[0+bs*2] -= CC[0+bs*3] * tmp;
19698 CC[1+bs*2] -= CC[1+bs*3] * tmp;
19699 CC[2+bs*2] -= CC[2+bs*3] * tmp;
19700 CC[3+bs*2] -= CC[3+bs*3] * tmp;
19701
19702 tmp = inv_diag_E[2];
19703 CC[0+bs*2] *= tmp;
19704 CC[1+bs*2] *= tmp;
19705 CC[2+bs*2] *= tmp;
19706 CC[3+bs*2] *= tmp;
19707 tmp = E[0+bs*2];
19708 CC[0+bs*0] -= CC[0+bs*2] * tmp;
19709 CC[1+bs*0] -= CC[1+bs*2] * tmp;
19710 CC[2+bs*0] -= CC[2+bs*2] * tmp;
19711 CC[3+bs*0] -= CC[3+bs*2] * tmp;
19712 tmp = E[1+bs*2];
19713 CC[0+bs*1] -= CC[0+bs*2] * tmp;
19714 CC[1+bs*1] -= CC[1+bs*2] * tmp;
19715 CC[2+bs*1] -= CC[2+bs*2] * tmp;
19716 CC[3+bs*1] -= CC[3+bs*2] * tmp;
19717
19718 tmp = inv_diag_E[1];
19719 CC[0+bs*1] *= tmp;
19720 CC[1+bs*1] *= tmp;
19721 CC[2+bs*1] *= tmp;
19722 CC[3+bs*1] *= tmp;
19723 tmp = E[0+bs*1];
19724 CC[0+bs*0] -= CC[0+bs*1] * tmp;
19725 CC[1+bs*0] -= CC[1+bs*1] * tmp;
19726 CC[2+bs*0] -= CC[2+bs*1] * tmp;
19727 CC[3+bs*0] -= CC[3+bs*1] * tmp;
19728
19729 tmp = inv_diag_E[0];
19730 CC[0+bs*0] *= tmp;
19731 CC[1+bs*0] *= tmp;
19732 CC[2+bs*0] *= tmp;
19733 CC[3+bs*0] *= tmp;
19734
19735
19736 D[0+ldd*0] = CC[0+bs*0];
19737 D[1+ldd*0] = CC[1+bs*0];
19738 D[2+ldd*0] = CC[2+bs*0];
19739 D[3+ldd*0] = CC[3+bs*0];
19740
19741 D[0+ldd*1] = CC[0+bs*1];
19742 D[1+ldd*1] = CC[1+bs*1];
19743 D[2+ldd*1] = CC[2+bs*1];
19744 D[3+ldd*1] = CC[3+bs*1];
19745
19746 D[0+ldd*2] = CC[0+bs*2];
19747 D[1+ldd*2] = CC[1+bs*2];
19748 D[2+ldd*2] = CC[2+bs*2];
19749 D[3+ldd*2] = CC[3+bs*2];
19750
19751 D[0+ldd*3] = CC[0+bs*3];
19752 D[1+ldd*3] = CC[1+bs*3];
19753 D[2+ldd*3] = CC[2+bs*3];
19754 D[3+ldd*3] = CC[3+bs*3];
19755
19756 return;
19757
19758 }
19759 //#endif
19760
19761
19762
19763 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_inv_4x4_vs_lib44cc4(int kmax,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,float * E,float * inv_diag_E,int m1,int n1)19764 void kernel_strsm_nt_ru_inv_4x4_vs_lib44cc4(int kmax, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, float *E, float *inv_diag_E, int m1, int n1)
19765 {
19766
19767 const int bs = 4;
19768
19769 float tmp;
19770
19771 #if defined(TARGET_GENERIC)
19772 float CC[16] = {0};
19773 #else
19774 ALIGNED( float CC[16], 64 ) = {0};
19775 #endif
19776
19777 float alpha1 = -1.0;
19778
19779 CC[0+bs*0] = C[0+ldc*0];
19780 CC[1+bs*0] = C[1+ldc*0];
19781 CC[2+bs*0] = C[2+ldc*0];
19782 CC[3+bs*0] = C[3+ldc*0];
19783
19784 CC[0+bs*1] = C[0+ldc*1];
19785 CC[1+bs*1] = C[1+ldc*1];
19786 CC[2+bs*1] = C[2+ldc*1];
19787 CC[3+bs*1] = C[3+ldc*1];
19788
19789 CC[0+bs*2] = C[0+ldc*2];
19790 CC[1+bs*2] = C[1+ldc*2];
19791 CC[2+bs*2] = C[2+ldc*2];
19792 CC[3+bs*2] = C[3+ldc*2];
19793
19794 CC[0+bs*3] = C[0+ldc*3];
19795 CC[1+bs*3] = C[1+ldc*3];
19796 CC[2+bs*3] = C[2+ldc*3];
19797 CC[3+bs*3] = C[3+ldc*3];
19798
19799 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, beta, CC, CC);
19800
19801 if(n1<=3)
19802 goto n3;
19803
19804 tmp = inv_diag_E[3];
19805 CC[0+bs*3] *= tmp;
19806 CC[1+bs*3] *= tmp;
19807 CC[2+bs*3] *= tmp;
19808 CC[3+bs*3] *= tmp;
19809 tmp = E[0+bs*3];
19810 CC[0+bs*0] -= CC[0+bs*3] * tmp;
19811 CC[1+bs*0] -= CC[1+bs*3] * tmp;
19812 CC[2+bs*0] -= CC[2+bs*3] * tmp;
19813 CC[3+bs*0] -= CC[3+bs*3] * tmp;
19814 tmp = E[1+bs*3];
19815 CC[0+bs*1] -= CC[0+bs*3] * tmp;
19816 CC[1+bs*1] -= CC[1+bs*3] * tmp;
19817 CC[2+bs*1] -= CC[2+bs*3] * tmp;
19818 CC[3+bs*1] -= CC[3+bs*3] * tmp;
19819 tmp = E[2+bs*3];
19820 CC[0+bs*2] -= CC[0+bs*3] * tmp;
19821 CC[1+bs*2] -= CC[1+bs*3] * tmp;
19822 CC[2+bs*2] -= CC[2+bs*3] * tmp;
19823 CC[3+bs*2] -= CC[3+bs*3] * tmp;
19824
19825 n3:
19826 if(n1<=2)
19827 goto n2;
19828
19829 tmp = inv_diag_E[2];
19830 CC[0+bs*2] *= tmp;
19831 CC[1+bs*2] *= tmp;
19832 CC[2+bs*2] *= tmp;
19833 CC[3+bs*2] *= tmp;
19834 tmp = E[0+bs*2];
19835 CC[0+bs*0] -= CC[0+bs*2] * tmp;
19836 CC[1+bs*0] -= CC[1+bs*2] * tmp;
19837 CC[2+bs*0] -= CC[2+bs*2] * tmp;
19838 CC[3+bs*0] -= CC[3+bs*2] * tmp;
19839 tmp = E[1+bs*2];
19840 CC[0+bs*1] -= CC[0+bs*2] * tmp;
19841 CC[1+bs*1] -= CC[1+bs*2] * tmp;
19842 CC[2+bs*1] -= CC[2+bs*2] * tmp;
19843 CC[3+bs*1] -= CC[3+bs*2] * tmp;
19844
19845 n2:
19846 if(n1<=1)
19847 goto n1;
19848
19849 tmp = inv_diag_E[1];
19850 CC[0+bs*1] *= tmp;
19851 CC[1+bs*1] *= tmp;
19852 CC[2+bs*1] *= tmp;
19853 CC[3+bs*1] *= tmp;
19854 tmp = E[0+bs*1];
19855 CC[0+bs*0] -= CC[0+bs*1] * tmp;
19856 CC[1+bs*0] -= CC[1+bs*1] * tmp;
19857 CC[2+bs*0] -= CC[2+bs*1] * tmp;
19858 CC[3+bs*0] -= CC[3+bs*1] * tmp;
19859
19860 n1:
19861
19862 tmp = inv_diag_E[0];
19863 CC[0+bs*0] *= tmp;
19864 CC[1+bs*0] *= tmp;
19865 CC[2+bs*0] *= tmp;
19866 CC[3+bs*0] *= tmp;
19867
19868 store:
19869
19870 if(m1>=4)
19871 {
19872 D[0+ldd*0] = CC[0+bs*0];
19873 D[1+ldd*0] = CC[1+bs*0];
19874 D[2+ldd*0] = CC[2+bs*0];
19875 D[3+ldd*0] = CC[3+bs*0];
19876
19877 if(n1==1)
19878 return;
19879
19880 D[0+ldd*1] = CC[0+bs*1];
19881 D[1+ldd*1] = CC[1+bs*1];
19882 D[2+ldd*1] = CC[2+bs*1];
19883 D[3+ldd*1] = CC[3+bs*1];
19884
19885 if(n1==2)
19886 return;
19887
19888 D[0+ldd*2] = CC[0+bs*2];
19889 D[1+ldd*2] = CC[1+bs*2];
19890 D[2+ldd*2] = CC[2+bs*2];
19891 D[3+ldd*2] = CC[3+bs*2];
19892
19893 if(n1==3)
19894 return;
19895
19896 D[0+ldd*3] = CC[0+bs*3];
19897 D[1+ldd*3] = CC[1+bs*3];
19898 D[2+ldd*3] = CC[2+bs*3];
19899 D[3+ldd*3] = CC[3+bs*3];
19900 }
19901 else if(m1>=3)
19902 {
19903 D[0+ldd*0] = CC[0+bs*0];
19904 D[1+ldd*0] = CC[1+bs*0];
19905 D[2+ldd*0] = CC[2+bs*0];
19906
19907 if(n1==1)
19908 return;
19909
19910 D[0+ldd*1] = CC[0+bs*1];
19911 D[1+ldd*1] = CC[1+bs*1];
19912 D[2+ldd*1] = CC[2+bs*1];
19913
19914 if(n1==2)
19915 return;
19916
19917 D[0+ldd*2] = CC[0+bs*2];
19918 D[1+ldd*2] = CC[1+bs*2];
19919 D[2+ldd*2] = CC[2+bs*2];
19920
19921 if(n1==3)
19922 return;
19923
19924 D[0+ldd*3] = CC[0+bs*3];
19925 D[1+ldd*3] = CC[1+bs*3];
19926 D[2+ldd*3] = CC[2+bs*3];
19927 }
19928 else if(m1>=2)
19929 {
19930 D[0+ldd*0] = CC[0+bs*0];
19931 D[1+ldd*0] = CC[1+bs*0];
19932
19933 if(n1==1)
19934 return;
19935
19936 D[0+ldd*1] = CC[0+bs*1];
19937 D[1+ldd*1] = CC[1+bs*1];
19938
19939 if(n1==2)
19940 return;
19941
19942 D[0+ldd*2] = CC[0+bs*2];
19943 D[1+ldd*2] = CC[1+bs*2];
19944
19945 if(n1==3)
19946 return;
19947
19948 D[0+ldd*3] = CC[0+bs*3];
19949 D[1+ldd*3] = CC[1+bs*3];
19950 }
19951 else //if(m1>=1)
19952 {
19953 D[0+ldd*0] = CC[0+bs*0];
19954
19955 if(n1==1)
19956 return;
19957
19958 D[0+ldd*1] = CC[0+bs*1];
19959
19960 if(n1==2)
19961 return;
19962
19963 D[0+ldd*2] = CC[0+bs*2];
19964
19965 if(n1==3)
19966 return;
19967
19968 D[0+ldd*3] = CC[0+bs*3];
19969 }
19970
19971 return;
19972
19973 }
19974 //#endif
19975
19976
19977
19978 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_inv_4x4_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,float * inv_diag_E)19979 void kernel_strsm_nt_ru_inv_4x4_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, float *inv_diag_E)
19980 {
19981
19982 const int bs = 4;
19983
19984 float tmp;
19985
19986 #if defined(TARGET_GENERIC)
19987 float CC[16] = {0};
19988 #else
19989 ALIGNED( float CC[16], 64 ) = {0};
19990 #endif
19991
19992 float alpha1 = -1.0;
19993
19994 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
19995
19996 tmp = inv_diag_E[3];
19997 CC[0+bs*3] *= tmp;
19998 CC[1+bs*3] *= tmp;
19999 CC[2+bs*3] *= tmp;
20000 CC[3+bs*3] *= tmp;
20001 tmp = E[0+lde*3];
20002 CC[0+bs*0] -= CC[0+bs*3] * tmp;
20003 CC[1+bs*0] -= CC[1+bs*3] * tmp;
20004 CC[2+bs*0] -= CC[2+bs*3] * tmp;
20005 CC[3+bs*0] -= CC[3+bs*3] * tmp;
20006 tmp = E[1+lde*3];
20007 CC[0+bs*1] -= CC[0+bs*3] * tmp;
20008 CC[1+bs*1] -= CC[1+bs*3] * tmp;
20009 CC[2+bs*1] -= CC[2+bs*3] * tmp;
20010 CC[3+bs*1] -= CC[3+bs*3] * tmp;
20011 tmp = E[2+lde*3];
20012 CC[0+bs*2] -= CC[0+bs*3] * tmp;
20013 CC[1+bs*2] -= CC[1+bs*3] * tmp;
20014 CC[2+bs*2] -= CC[2+bs*3] * tmp;
20015 CC[3+bs*2] -= CC[3+bs*3] * tmp;
20016
20017 tmp = inv_diag_E[2];
20018 CC[0+bs*2] *= tmp;
20019 CC[1+bs*2] *= tmp;
20020 CC[2+bs*2] *= tmp;
20021 CC[3+bs*2] *= tmp;
20022 tmp = E[0+lde*2];
20023 CC[0+bs*0] -= CC[0+bs*2] * tmp;
20024 CC[1+bs*0] -= CC[1+bs*2] * tmp;
20025 CC[2+bs*0] -= CC[2+bs*2] * tmp;
20026 CC[3+bs*0] -= CC[3+bs*2] * tmp;
20027 tmp = E[1+lde*2];
20028 CC[0+bs*1] -= CC[0+bs*2] * tmp;
20029 CC[1+bs*1] -= CC[1+bs*2] * tmp;
20030 CC[2+bs*1] -= CC[2+bs*2] * tmp;
20031 CC[3+bs*1] -= CC[3+bs*2] * tmp;
20032
20033 tmp = inv_diag_E[1];
20034 CC[0+bs*1] *= tmp;
20035 CC[1+bs*1] *= tmp;
20036 CC[2+bs*1] *= tmp;
20037 CC[3+bs*1] *= tmp;
20038 tmp = E[0+lde*1];
20039 CC[0+bs*0] -= CC[0+bs*1] * tmp;
20040 CC[1+bs*0] -= CC[1+bs*1] * tmp;
20041 CC[2+bs*0] -= CC[2+bs*1] * tmp;
20042 CC[3+bs*0] -= CC[3+bs*1] * tmp;
20043
20044 tmp = inv_diag_E[0];
20045 CC[0+bs*0] *= tmp;
20046 CC[1+bs*0] *= tmp;
20047 CC[2+bs*0] *= tmp;
20048 CC[3+bs*0] *= tmp;
20049
20050
20051 D[0+bs*0] = CC[0+bs*0];
20052 D[1+bs*0] = CC[1+bs*0];
20053 D[2+bs*0] = CC[2+bs*0];
20054 D[3+bs*0] = CC[3+bs*0];
20055
20056 D[0+bs*1] = CC[0+bs*1];
20057 D[1+bs*1] = CC[1+bs*1];
20058 D[2+bs*1] = CC[2+bs*1];
20059 D[3+bs*1] = CC[3+bs*1];
20060
20061 D[0+bs*2] = CC[0+bs*2];
20062 D[1+bs*2] = CC[1+bs*2];
20063 D[2+bs*2] = CC[2+bs*2];
20064 D[3+bs*2] = CC[3+bs*2];
20065
20066 D[0+bs*3] = CC[0+bs*3];
20067 D[1+bs*3] = CC[1+bs*3];
20068 D[2+bs*3] = CC[2+bs*3];
20069 D[3+bs*3] = CC[3+bs*3];
20070
20071 return;
20072
20073 }
20074 //#endif
20075
20076
20077
20078 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_inv_4x4_vs_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,float * inv_diag_E,int m1,int n1)20079 void kernel_strsm_nt_ru_inv_4x4_vs_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, float *inv_diag_E, int m1, int n1)
20080 {
20081
20082 const int bs = 4;
20083
20084 float tmp;
20085
20086 #if defined(TARGET_GENERIC)
20087 float CC[16] = {0};
20088 #else
20089 ALIGNED( float CC[16], 64 ) = {0};
20090 #endif
20091
20092 float alpha1 = -1.0;
20093
20094 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
20095
20096 if(n1<=3)
20097 goto n3;
20098
20099 tmp = inv_diag_E[3];
20100 CC[0+bs*3] *= tmp;
20101 CC[1+bs*3] *= tmp;
20102 CC[2+bs*3] *= tmp;
20103 CC[3+bs*3] *= tmp;
20104 tmp = E[0+lde*3];
20105 CC[0+bs*0] -= CC[0+bs*3] * tmp;
20106 CC[1+bs*0] -= CC[1+bs*3] * tmp;
20107 CC[2+bs*0] -= CC[2+bs*3] * tmp;
20108 CC[3+bs*0] -= CC[3+bs*3] * tmp;
20109 tmp = E[1+lde*3];
20110 CC[0+bs*1] -= CC[0+bs*3] * tmp;
20111 CC[1+bs*1] -= CC[1+bs*3] * tmp;
20112 CC[2+bs*1] -= CC[2+bs*3] * tmp;
20113 CC[3+bs*1] -= CC[3+bs*3] * tmp;
20114 tmp = E[2+lde*3];
20115 CC[0+bs*2] -= CC[0+bs*3] * tmp;
20116 CC[1+bs*2] -= CC[1+bs*3] * tmp;
20117 CC[2+bs*2] -= CC[2+bs*3] * tmp;
20118 CC[3+bs*2] -= CC[3+bs*3] * tmp;
20119
20120 n3:
20121 if(n1<=2)
20122 goto n2;
20123
20124 tmp = inv_diag_E[2];
20125 CC[0+bs*2] *= tmp;
20126 CC[1+bs*2] *= tmp;
20127 CC[2+bs*2] *= tmp;
20128 CC[3+bs*2] *= tmp;
20129 tmp = E[0+lde*2];
20130 CC[0+bs*0] -= CC[0+bs*2] * tmp;
20131 CC[1+bs*0] -= CC[1+bs*2] * tmp;
20132 CC[2+bs*0] -= CC[2+bs*2] * tmp;
20133 CC[3+bs*0] -= CC[3+bs*2] * tmp;
20134 tmp = E[1+lde*2];
20135 CC[0+bs*1] -= CC[0+bs*2] * tmp;
20136 CC[1+bs*1] -= CC[1+bs*2] * tmp;
20137 CC[2+bs*1] -= CC[2+bs*2] * tmp;
20138 CC[3+bs*1] -= CC[3+bs*2] * tmp;
20139
20140 n2:
20141 if(n1<=1)
20142 goto n1;
20143
20144 tmp = inv_diag_E[1];
20145 CC[0+bs*1] *= tmp;
20146 CC[1+bs*1] *= tmp;
20147 CC[2+bs*1] *= tmp;
20148 CC[3+bs*1] *= tmp;
20149 tmp = E[0+lde*1];
20150 CC[0+bs*0] -= CC[0+bs*1] * tmp;
20151 CC[1+bs*0] -= CC[1+bs*1] * tmp;
20152 CC[2+bs*0] -= CC[2+bs*1] * tmp;
20153 CC[3+bs*0] -= CC[3+bs*1] * tmp;
20154
20155 n1:
20156
20157 tmp = inv_diag_E[0];
20158 CC[0+bs*0] *= tmp;
20159 CC[1+bs*0] *= tmp;
20160 CC[2+bs*0] *= tmp;
20161 CC[3+bs*0] *= tmp;
20162
20163 store:
20164
20165 if(m1>=4)
20166 {
20167 D[0+bs*0] = CC[0+bs*0];
20168 D[1+bs*0] = CC[1+bs*0];
20169 D[2+bs*0] = CC[2+bs*0];
20170 D[3+bs*0] = CC[3+bs*0];
20171
20172 if(n1==1)
20173 return;
20174
20175 D[0+bs*1] = CC[0+bs*1];
20176 D[1+bs*1] = CC[1+bs*1];
20177 D[2+bs*1] = CC[2+bs*1];
20178 D[3+bs*1] = CC[3+bs*1];
20179
20180 if(n1==2)
20181 return;
20182
20183 D[0+bs*2] = CC[0+bs*2];
20184 D[1+bs*2] = CC[1+bs*2];
20185 D[2+bs*2] = CC[2+bs*2];
20186 D[3+bs*2] = CC[3+bs*2];
20187
20188 if(n1==3)
20189 return;
20190
20191 D[0+bs*3] = CC[0+bs*3];
20192 D[1+bs*3] = CC[1+bs*3];
20193 D[2+bs*3] = CC[2+bs*3];
20194 D[3+bs*3] = CC[3+bs*3];
20195 }
20196 else if(m1>=3)
20197 {
20198 D[0+bs*0] = CC[0+bs*0];
20199 D[1+bs*0] = CC[1+bs*0];
20200 D[2+bs*0] = CC[2+bs*0];
20201
20202 if(n1==1)
20203 return;
20204
20205 D[0+bs*1] = CC[0+bs*1];
20206 D[1+bs*1] = CC[1+bs*1];
20207 D[2+bs*1] = CC[2+bs*1];
20208
20209 if(n1==2)
20210 return;
20211
20212 D[0+bs*2] = CC[0+bs*2];
20213 D[1+bs*2] = CC[1+bs*2];
20214 D[2+bs*2] = CC[2+bs*2];
20215
20216 if(n1==3)
20217 return;
20218
20219 D[0+bs*3] = CC[0+bs*3];
20220 D[1+bs*3] = CC[1+bs*3];
20221 D[2+bs*3] = CC[2+bs*3];
20222 }
20223 else if(m1>=2)
20224 {
20225 D[0+bs*0] = CC[0+bs*0];
20226 D[1+bs*0] = CC[1+bs*0];
20227
20228 if(n1==1)
20229 return;
20230
20231 D[0+bs*1] = CC[0+bs*1];
20232 D[1+bs*1] = CC[1+bs*1];
20233
20234 if(n1==2)
20235 return;
20236
20237 D[0+bs*2] = CC[0+bs*2];
20238 D[1+bs*2] = CC[1+bs*2];
20239
20240 if(n1==3)
20241 return;
20242
20243 D[0+bs*3] = CC[0+bs*3];
20244 D[1+bs*3] = CC[1+bs*3];
20245 }
20246 else //if(m1>=1)
20247 {
20248 D[0+bs*0] = CC[0+bs*0];
20249
20250 if(n1==1)
20251 return;
20252
20253 D[0+bs*1] = CC[0+bs*1];
20254
20255 if(n1==2)
20256 return;
20257
20258 D[0+bs*2] = CC[0+bs*2];
20259
20260 if(n1==3)
20261 return;
20262
20263 D[0+bs*3] = CC[0+bs*3];
20264 }
20265
20266 return;
20267
20268 }
20269 //#endif
20270
20271
20272
20273 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_inv_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E)20274 void kernel_strsm_nt_ru_inv_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E)
20275 {
20276
20277 const int bs = 4;
20278
20279 float tmp;
20280
20281 #if defined(TARGET_GENERIC)
20282 float CC[16] = {0};
20283 #else
20284 ALIGNED( float CC[16], 64 ) = {0};
20285 #endif
20286
20287 float alpha1 = -1.0;
20288
20289 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
20290
20291 tmp = inv_diag_E[3];
20292 CC[0+bs*3] *= tmp;
20293 CC[1+bs*3] *= tmp;
20294 CC[2+bs*3] *= tmp;
20295 CC[3+bs*3] *= tmp;
20296 tmp = E[0+lde*3];
20297 CC[0+bs*0] -= CC[0+bs*3] * tmp;
20298 CC[1+bs*0] -= CC[1+bs*3] * tmp;
20299 CC[2+bs*0] -= CC[2+bs*3] * tmp;
20300 CC[3+bs*0] -= CC[3+bs*3] * tmp;
20301 tmp = E[1+lde*3];
20302 CC[0+bs*1] -= CC[0+bs*3] * tmp;
20303 CC[1+bs*1] -= CC[1+bs*3] * tmp;
20304 CC[2+bs*1] -= CC[2+bs*3] * tmp;
20305 CC[3+bs*1] -= CC[3+bs*3] * tmp;
20306 tmp = E[2+lde*3];
20307 CC[0+bs*2] -= CC[0+bs*3] * tmp;
20308 CC[1+bs*2] -= CC[1+bs*3] * tmp;
20309 CC[2+bs*2] -= CC[2+bs*3] * tmp;
20310 CC[3+bs*2] -= CC[3+bs*3] * tmp;
20311
20312 tmp = inv_diag_E[2];
20313 CC[0+bs*2] *= tmp;
20314 CC[1+bs*2] *= tmp;
20315 CC[2+bs*2] *= tmp;
20316 CC[3+bs*2] *= tmp;
20317 tmp = E[0+lde*2];
20318 CC[0+bs*0] -= CC[0+bs*2] * tmp;
20319 CC[1+bs*0] -= CC[1+bs*2] * tmp;
20320 CC[2+bs*0] -= CC[2+bs*2] * tmp;
20321 CC[3+bs*0] -= CC[3+bs*2] * tmp;
20322 tmp = E[1+lde*2];
20323 CC[0+bs*1] -= CC[0+bs*2] * tmp;
20324 CC[1+bs*1] -= CC[1+bs*2] * tmp;
20325 CC[2+bs*1] -= CC[2+bs*2] * tmp;
20326 CC[3+bs*1] -= CC[3+bs*2] * tmp;
20327
20328 tmp = inv_diag_E[1];
20329 CC[0+bs*1] *= tmp;
20330 CC[1+bs*1] *= tmp;
20331 CC[2+bs*1] *= tmp;
20332 CC[3+bs*1] *= tmp;
20333 tmp = E[0+lde*1];
20334 CC[0+bs*0] -= CC[0+bs*1] * tmp;
20335 CC[1+bs*0] -= CC[1+bs*1] * tmp;
20336 CC[2+bs*0] -= CC[2+bs*1] * tmp;
20337 CC[3+bs*0] -= CC[3+bs*1] * tmp;
20338
20339 tmp = inv_diag_E[0];
20340 CC[0+bs*0] *= tmp;
20341 CC[1+bs*0] *= tmp;
20342 CC[2+bs*0] *= tmp;
20343 CC[3+bs*0] *= tmp;
20344
20345
20346 D[0+ldd*0] = CC[0+bs*0];
20347 D[1+ldd*0] = CC[1+bs*0];
20348 D[2+ldd*0] = CC[2+bs*0];
20349 D[3+ldd*0] = CC[3+bs*0];
20350
20351 D[0+ldd*1] = CC[0+bs*1];
20352 D[1+ldd*1] = CC[1+bs*1];
20353 D[2+ldd*1] = CC[2+bs*1];
20354 D[3+ldd*1] = CC[3+bs*1];
20355
20356 D[0+ldd*2] = CC[0+bs*2];
20357 D[1+ldd*2] = CC[1+bs*2];
20358 D[2+ldd*2] = CC[2+bs*2];
20359 D[3+ldd*2] = CC[3+bs*2];
20360
20361 D[0+ldd*3] = CC[0+bs*3];
20362 D[1+ldd*3] = CC[1+bs*3];
20363 D[2+ldd*3] = CC[2+bs*3];
20364 D[3+ldd*3] = CC[3+bs*3];
20365
20366 return;
20367
20368 }
20369 //#endif
20370
20371
20372
20373 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_inv_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,float * inv_diag_E,int m1,int n1)20374 void kernel_strsm_nt_ru_inv_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, float *inv_diag_E, int m1, int n1)
20375 {
20376
20377 const int bs = 4;
20378
20379 float tmp;
20380
20381 #if defined(TARGET_GENERIC)
20382 float CC[16] = {0};
20383 #else
20384 ALIGNED( float CC[16], 64 ) = {0};
20385 #endif
20386
20387 float alpha1 = -1.0;
20388
20389 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
20390
20391 if(n1<=3)
20392 goto n3;
20393
20394 tmp = inv_diag_E[3];
20395 CC[0+bs*3] *= tmp;
20396 CC[1+bs*3] *= tmp;
20397 CC[2+bs*3] *= tmp;
20398 CC[3+bs*3] *= tmp;
20399 tmp = E[0+lde*3];
20400 CC[0+bs*0] -= CC[0+bs*3] * tmp;
20401 CC[1+bs*0] -= CC[1+bs*3] * tmp;
20402 CC[2+bs*0] -= CC[2+bs*3] * tmp;
20403 CC[3+bs*0] -= CC[3+bs*3] * tmp;
20404 tmp = E[1+lde*3];
20405 CC[0+bs*1] -= CC[0+bs*3] * tmp;
20406 CC[1+bs*1] -= CC[1+bs*3] * tmp;
20407 CC[2+bs*1] -= CC[2+bs*3] * tmp;
20408 CC[3+bs*1] -= CC[3+bs*3] * tmp;
20409 tmp = E[2+lde*3];
20410 CC[0+bs*2] -= CC[0+bs*3] * tmp;
20411 CC[1+bs*2] -= CC[1+bs*3] * tmp;
20412 CC[2+bs*2] -= CC[2+bs*3] * tmp;
20413 CC[3+bs*2] -= CC[3+bs*3] * tmp;
20414
20415 n3:
20416 if(n1<=2)
20417 goto n2;
20418
20419 tmp = inv_diag_E[2];
20420 CC[0+bs*2] *= tmp;
20421 CC[1+bs*2] *= tmp;
20422 CC[2+bs*2] *= tmp;
20423 CC[3+bs*2] *= tmp;
20424 tmp = E[0+lde*2];
20425 CC[0+bs*0] -= CC[0+bs*2] * tmp;
20426 CC[1+bs*0] -= CC[1+bs*2] * tmp;
20427 CC[2+bs*0] -= CC[2+bs*2] * tmp;
20428 CC[3+bs*0] -= CC[3+bs*2] * tmp;
20429 tmp = E[1+lde*2];
20430 CC[0+bs*1] -= CC[0+bs*2] * tmp;
20431 CC[1+bs*1] -= CC[1+bs*2] * tmp;
20432 CC[2+bs*1] -= CC[2+bs*2] * tmp;
20433 CC[3+bs*1] -= CC[3+bs*2] * tmp;
20434
20435 n2:
20436 if(n1<=1)
20437 goto n1;
20438
20439 tmp = inv_diag_E[1];
20440 CC[0+bs*1] *= tmp;
20441 CC[1+bs*1] *= tmp;
20442 CC[2+bs*1] *= tmp;
20443 CC[3+bs*1] *= tmp;
20444 tmp = E[0+lde*1];
20445 CC[0+bs*0] -= CC[0+bs*1] * tmp;
20446 CC[1+bs*0] -= CC[1+bs*1] * tmp;
20447 CC[2+bs*0] -= CC[2+bs*1] * tmp;
20448 CC[3+bs*0] -= CC[3+bs*1] * tmp;
20449
20450 n1:
20451
20452 tmp = inv_diag_E[0];
20453 CC[0+bs*0] *= tmp;
20454 CC[1+bs*0] *= tmp;
20455 CC[2+bs*0] *= tmp;
20456 CC[3+bs*0] *= tmp;
20457
20458 store:
20459
20460 if(m1>=4)
20461 {
20462 D[0+ldd*0] = CC[0+bs*0];
20463 D[1+ldd*0] = CC[1+bs*0];
20464 D[2+ldd*0] = CC[2+bs*0];
20465 D[3+ldd*0] = CC[3+bs*0];
20466
20467 if(n1==1)
20468 return;
20469
20470 D[0+ldd*1] = CC[0+bs*1];
20471 D[1+ldd*1] = CC[1+bs*1];
20472 D[2+ldd*1] = CC[2+bs*1];
20473 D[3+ldd*1] = CC[3+bs*1];
20474
20475 if(n1==2)
20476 return;
20477
20478 D[0+ldd*2] = CC[0+bs*2];
20479 D[1+ldd*2] = CC[1+bs*2];
20480 D[2+ldd*2] = CC[2+bs*2];
20481 D[3+ldd*2] = CC[3+bs*2];
20482
20483 if(n1==3)
20484 return;
20485
20486 D[0+ldd*3] = CC[0+bs*3];
20487 D[1+ldd*3] = CC[1+bs*3];
20488 D[2+ldd*3] = CC[2+bs*3];
20489 D[3+ldd*3] = CC[3+bs*3];
20490 }
20491 else if(m1>=3)
20492 {
20493 D[0+ldd*0] = CC[0+bs*0];
20494 D[1+ldd*0] = CC[1+bs*0];
20495 D[2+ldd*0] = CC[2+bs*0];
20496
20497 if(n1==1)
20498 return;
20499
20500 D[0+ldd*1] = CC[0+bs*1];
20501 D[1+ldd*1] = CC[1+bs*1];
20502 D[2+ldd*1] = CC[2+bs*1];
20503
20504 if(n1==2)
20505 return;
20506
20507 D[0+ldd*2] = CC[0+bs*2];
20508 D[1+ldd*2] = CC[1+bs*2];
20509 D[2+ldd*2] = CC[2+bs*2];
20510
20511 if(n1==3)
20512 return;
20513
20514 D[0+ldd*3] = CC[0+bs*3];
20515 D[1+ldd*3] = CC[1+bs*3];
20516 D[2+ldd*3] = CC[2+bs*3];
20517 }
20518 else if(m1>=2)
20519 {
20520 D[0+ldd*0] = CC[0+bs*0];
20521 D[1+ldd*0] = CC[1+bs*0];
20522
20523 if(n1==1)
20524 return;
20525
20526 D[0+ldd*1] = CC[0+bs*1];
20527 D[1+ldd*1] = CC[1+bs*1];
20528
20529 if(n1==2)
20530 return;
20531
20532 D[0+ldd*2] = CC[0+bs*2];
20533 D[1+ldd*2] = CC[1+bs*2];
20534
20535 if(n1==3)
20536 return;
20537
20538 D[0+ldd*3] = CC[0+bs*3];
20539 D[1+ldd*3] = CC[1+bs*3];
20540 }
20541 else //if(m1>=1)
20542 {
20543 D[0+ldd*0] = CC[0+bs*0];
20544
20545 if(n1==1)
20546 return;
20547
20548 D[0+ldd*1] = CC[0+bs*1];
20549
20550 if(n1==2)
20551 return;
20552
20553 D[0+ldd*2] = CC[0+bs*2];
20554
20555 if(n1==3)
20556 return;
20557
20558 D[0+ldd*3] = CC[0+bs*3];
20559 }
20560
20561 return;
20562
20563 }
20564 //#endif
20565
20566
20567
20568 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_one_4x4_lib44cc4(int kmax,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,float * E)20569 void kernel_strsm_nt_ru_one_4x4_lib44cc4(int kmax, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, float *E)
20570 {
20571
20572 const int bs = 4;
20573
20574 float tmp;
20575
20576 #if defined(TARGET_GENERIC)
20577 float CC[16] = {0};
20578 #else
20579 ALIGNED( float CC[16], 64 ) = {0};
20580 #endif
20581
20582 float alpha1 = -1.0;
20583
20584 CC[0+bs*0] = C[0+ldc*0];
20585 CC[1+bs*0] = C[1+ldc*0];
20586 CC[2+bs*0] = C[2+ldc*0];
20587 CC[3+bs*0] = C[3+ldc*0];
20588
20589 CC[0+bs*1] = C[0+ldc*1];
20590 CC[1+bs*1] = C[1+ldc*1];
20591 CC[2+bs*1] = C[2+ldc*1];
20592 CC[3+bs*1] = C[3+ldc*1];
20593
20594 CC[0+bs*2] = C[0+ldc*2];
20595 CC[1+bs*2] = C[1+ldc*2];
20596 CC[2+bs*2] = C[2+ldc*2];
20597 CC[3+bs*2] = C[3+ldc*2];
20598
20599 CC[0+bs*3] = C[0+ldc*3];
20600 CC[1+bs*3] = C[1+ldc*3];
20601 CC[2+bs*3] = C[2+ldc*3];
20602 CC[3+bs*3] = C[3+ldc*3];
20603
20604 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, beta, CC, CC);
20605
20606 tmp = E[0+bs*3];
20607 CC[0+bs*0] -= CC[0+bs*3] * tmp;
20608 CC[1+bs*0] -= CC[1+bs*3] * tmp;
20609 CC[2+bs*0] -= CC[2+bs*3] * tmp;
20610 CC[3+bs*0] -= CC[3+bs*3] * tmp;
20611 tmp = E[1+bs*3];
20612 CC[0+bs*1] -= CC[0+bs*3] * tmp;
20613 CC[1+bs*1] -= CC[1+bs*3] * tmp;
20614 CC[2+bs*1] -= CC[2+bs*3] * tmp;
20615 CC[3+bs*1] -= CC[3+bs*3] * tmp;
20616 tmp = E[2+bs*3];
20617 CC[0+bs*2] -= CC[0+bs*3] * tmp;
20618 CC[1+bs*2] -= CC[1+bs*3] * tmp;
20619 CC[2+bs*2] -= CC[2+bs*3] * tmp;
20620 CC[3+bs*2] -= CC[3+bs*3] * tmp;
20621
20622 tmp = E[0+bs*2];
20623 CC[0+bs*0] -= CC[0+bs*2] * tmp;
20624 CC[1+bs*0] -= CC[1+bs*2] * tmp;
20625 CC[2+bs*0] -= CC[2+bs*2] * tmp;
20626 CC[3+bs*0] -= CC[3+bs*2] * tmp;
20627 tmp = E[1+bs*2];
20628 CC[0+bs*1] -= CC[0+bs*2] * tmp;
20629 CC[1+bs*1] -= CC[1+bs*2] * tmp;
20630 CC[2+bs*1] -= CC[2+bs*2] * tmp;
20631 CC[3+bs*1] -= CC[3+bs*2] * tmp;
20632
20633 tmp = E[0+bs*1];
20634 CC[0+bs*0] -= CC[0+bs*1] * tmp;
20635 CC[1+bs*0] -= CC[1+bs*1] * tmp;
20636 CC[2+bs*0] -= CC[2+bs*1] * tmp;
20637 CC[3+bs*0] -= CC[3+bs*1] * tmp;
20638
20639
20640 D[0+ldd*0] = CC[0+bs*0];
20641 D[1+ldd*0] = CC[1+bs*0];
20642 D[2+ldd*0] = CC[2+bs*0];
20643 D[3+ldd*0] = CC[3+bs*0];
20644
20645 D[0+ldd*1] = CC[0+bs*1];
20646 D[1+ldd*1] = CC[1+bs*1];
20647 D[2+ldd*1] = CC[2+bs*1];
20648 D[3+ldd*1] = CC[3+bs*1];
20649
20650 D[0+ldd*2] = CC[0+bs*2];
20651 D[1+ldd*2] = CC[1+bs*2];
20652 D[2+ldd*2] = CC[2+bs*2];
20653 D[3+ldd*2] = CC[3+bs*2];
20654
20655 D[0+ldd*3] = CC[0+bs*3];
20656 D[1+ldd*3] = CC[1+bs*3];
20657 D[2+ldd*3] = CC[2+bs*3];
20658 D[3+ldd*3] = CC[3+bs*3];
20659
20660 return;
20661
20662 }
20663 //#endif
20664
20665
20666
20667 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_one_4x4_vs_lib44cc4(int kmax,float * A,float * B,float * beta,float * C,int ldc,float * D,int ldd,float * E,int m1,int n1)20668 void kernel_strsm_nt_ru_one_4x4_vs_lib44cc4(int kmax, float *A, float *B, float *beta, float *C, int ldc, float *D, int ldd, float *E, int m1, int n1)
20669 {
20670
20671 const int bs = 4;
20672
20673 float tmp;
20674
20675 #if defined(TARGET_GENERIC)
20676 float CC[16] = {0};
20677 #else
20678 ALIGNED( float CC[16], 64 ) = {0};
20679 #endif
20680
20681 float alpha1 = -1.0;
20682
20683 CC[0+bs*0] = C[0+ldc*0];
20684 CC[1+bs*0] = C[1+ldc*0];
20685 CC[2+bs*0] = C[2+ldc*0];
20686 CC[3+bs*0] = C[3+ldc*0];
20687
20688 CC[0+bs*1] = C[0+ldc*1];
20689 CC[1+bs*1] = C[1+ldc*1];
20690 CC[2+bs*1] = C[2+ldc*1];
20691 CC[3+bs*1] = C[3+ldc*1];
20692
20693 CC[0+bs*2] = C[0+ldc*2];
20694 CC[1+bs*2] = C[1+ldc*2];
20695 CC[2+bs*2] = C[2+ldc*2];
20696 CC[3+bs*2] = C[3+ldc*2];
20697
20698 CC[0+bs*3] = C[0+ldc*3];
20699 CC[1+bs*3] = C[1+ldc*3];
20700 CC[2+bs*3] = C[2+ldc*3];
20701 CC[3+bs*3] = C[3+ldc*3];
20702
20703 kernel_sgemm_nt_4x4_lib4(kmax, &alpha1, A, B, beta, CC, CC);
20704
20705 if(n1<=3)
20706 goto n3;
20707
20708 tmp = E[0+bs*3];
20709 CC[0+bs*0] -= CC[0+bs*3] * tmp;
20710 CC[1+bs*0] -= CC[1+bs*3] * tmp;
20711 CC[2+bs*0] -= CC[2+bs*3] * tmp;
20712 CC[3+bs*0] -= CC[3+bs*3] * tmp;
20713 tmp = E[1+bs*3];
20714 CC[0+bs*1] -= CC[0+bs*3] * tmp;
20715 CC[1+bs*1] -= CC[1+bs*3] * tmp;
20716 CC[2+bs*1] -= CC[2+bs*3] * tmp;
20717 CC[3+bs*1] -= CC[3+bs*3] * tmp;
20718 tmp = E[2+bs*3];
20719 CC[0+bs*2] -= CC[0+bs*3] * tmp;
20720 CC[1+bs*2] -= CC[1+bs*3] * tmp;
20721 CC[2+bs*2] -= CC[2+bs*3] * tmp;
20722 CC[3+bs*2] -= CC[3+bs*3] * tmp;
20723
20724 n3:
20725 if(n1<=2)
20726 goto n2;
20727
20728 tmp = E[0+bs*2];
20729 CC[0+bs*0] -= CC[0+bs*2] * tmp;
20730 CC[1+bs*0] -= CC[1+bs*2] * tmp;
20731 CC[2+bs*0] -= CC[2+bs*2] * tmp;
20732 CC[3+bs*0] -= CC[3+bs*2] * tmp;
20733 tmp = E[1+bs*2];
20734 CC[0+bs*1] -= CC[0+bs*2] * tmp;
20735 CC[1+bs*1] -= CC[1+bs*2] * tmp;
20736 CC[2+bs*1] -= CC[2+bs*2] * tmp;
20737 CC[3+bs*1] -= CC[3+bs*2] * tmp;
20738
20739 n2:
20740 if(n1<=1)
20741 goto n1;
20742
20743 tmp = E[0+bs*1];
20744 CC[0+bs*0] -= CC[0+bs*1] * tmp;
20745 CC[1+bs*0] -= CC[1+bs*1] * tmp;
20746 CC[2+bs*0] -= CC[2+bs*1] * tmp;
20747 CC[3+bs*0] -= CC[3+bs*1] * tmp;
20748
20749 n1:
20750
20751 store:
20752
20753 if(m1>=4)
20754 {
20755 D[0+ldd*0] = CC[0+bs*0];
20756 D[1+ldd*0] = CC[1+bs*0];
20757 D[2+ldd*0] = CC[2+bs*0];
20758 D[3+ldd*0] = CC[3+bs*0];
20759
20760 if(n1==1)
20761 return;
20762
20763 D[0+ldd*1] = CC[0+bs*1];
20764 D[1+ldd*1] = CC[1+bs*1];
20765 D[2+ldd*1] = CC[2+bs*1];
20766 D[3+ldd*1] = CC[3+bs*1];
20767
20768 if(n1==2)
20769 return;
20770
20771 D[0+ldd*2] = CC[0+bs*2];
20772 D[1+ldd*2] = CC[1+bs*2];
20773 D[2+ldd*2] = CC[2+bs*2];
20774 D[3+ldd*2] = CC[3+bs*2];
20775
20776 if(n1==3)
20777 return;
20778
20779 D[0+ldd*3] = CC[0+bs*3];
20780 D[1+ldd*3] = CC[1+bs*3];
20781 D[2+ldd*3] = CC[2+bs*3];
20782 D[3+ldd*3] = CC[3+bs*3];
20783 }
20784 else if(m1>=3)
20785 {
20786 D[0+ldd*0] = CC[0+bs*0];
20787 D[1+ldd*0] = CC[1+bs*0];
20788 D[2+ldd*0] = CC[2+bs*0];
20789
20790 if(n1==1)
20791 return;
20792
20793 D[0+ldd*1] = CC[0+bs*1];
20794 D[1+ldd*1] = CC[1+bs*1];
20795 D[2+ldd*1] = CC[2+bs*1];
20796
20797 if(n1==2)
20798 return;
20799
20800 D[0+ldd*2] = CC[0+bs*2];
20801 D[1+ldd*2] = CC[1+bs*2];
20802 D[2+ldd*2] = CC[2+bs*2];
20803
20804 if(n1==3)
20805 return;
20806
20807 D[0+ldd*3] = CC[0+bs*3];
20808 D[1+ldd*3] = CC[1+bs*3];
20809 D[2+ldd*3] = CC[2+bs*3];
20810 }
20811 else if(m1>=2)
20812 {
20813 D[0+ldd*0] = CC[0+bs*0];
20814 D[1+ldd*0] = CC[1+bs*0];
20815
20816 if(n1==1)
20817 return;
20818
20819 D[0+ldd*1] = CC[0+bs*1];
20820 D[1+ldd*1] = CC[1+bs*1];
20821
20822 if(n1==2)
20823 return;
20824
20825 D[0+ldd*2] = CC[0+bs*2];
20826 D[1+ldd*2] = CC[1+bs*2];
20827
20828 if(n1==3)
20829 return;
20830
20831 D[0+ldd*3] = CC[0+bs*3];
20832 D[1+ldd*3] = CC[1+bs*3];
20833 }
20834 else //if(m1>=1)
20835 {
20836 D[0+ldd*0] = CC[0+bs*0];
20837
20838 if(n1==1)
20839 return;
20840
20841 D[0+ldd*1] = CC[0+bs*1];
20842
20843 if(n1==2)
20844 return;
20845
20846 D[0+ldd*2] = CC[0+bs*2];
20847
20848 if(n1==3)
20849 return;
20850
20851 D[0+ldd*3] = CC[0+bs*3];
20852 }
20853
20854 return;
20855
20856 }
20857 //#endif
20858
20859
20860
20861 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_one_4x4_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde)20862 void kernel_strsm_nt_ru_one_4x4_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde)
20863 {
20864
20865 const int bs = 4;
20866
20867 float tmp;
20868
20869 #if defined(TARGET_GENERIC)
20870 float CC[16] = {0};
20871 #else
20872 ALIGNED( float CC[16], 64 ) = {0};
20873 #endif
20874
20875 float alpha1 = -1.0;
20876
20877 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
20878
20879 tmp = E[0+lde*3];
20880 CC[0+bs*0] -= CC[0+bs*3] * tmp;
20881 CC[1+bs*0] -= CC[1+bs*3] * tmp;
20882 CC[2+bs*0] -= CC[2+bs*3] * tmp;
20883 CC[3+bs*0] -= CC[3+bs*3] * tmp;
20884 tmp = E[1+lde*3];
20885 CC[0+bs*1] -= CC[0+bs*3] * tmp;
20886 CC[1+bs*1] -= CC[1+bs*3] * tmp;
20887 CC[2+bs*1] -= CC[2+bs*3] * tmp;
20888 CC[3+bs*1] -= CC[3+bs*3] * tmp;
20889 tmp = E[2+lde*3];
20890 CC[0+bs*2] -= CC[0+bs*3] * tmp;
20891 CC[1+bs*2] -= CC[1+bs*3] * tmp;
20892 CC[2+bs*2] -= CC[2+bs*3] * tmp;
20893 CC[3+bs*2] -= CC[3+bs*3] * tmp;
20894
20895 tmp = E[0+lde*2];
20896 CC[0+bs*0] -= CC[0+bs*2] * tmp;
20897 CC[1+bs*0] -= CC[1+bs*2] * tmp;
20898 CC[2+bs*0] -= CC[2+bs*2] * tmp;
20899 CC[3+bs*0] -= CC[3+bs*2] * tmp;
20900 tmp = E[1+lde*2];
20901 CC[0+bs*1] -= CC[0+bs*2] * tmp;
20902 CC[1+bs*1] -= CC[1+bs*2] * tmp;
20903 CC[2+bs*1] -= CC[2+bs*2] * tmp;
20904 CC[3+bs*1] -= CC[3+bs*2] * tmp;
20905
20906 tmp = E[0+lde*1];
20907 CC[0+bs*0] -= CC[0+bs*1] * tmp;
20908 CC[1+bs*0] -= CC[1+bs*1] * tmp;
20909 CC[2+bs*0] -= CC[2+bs*1] * tmp;
20910 CC[3+bs*0] -= CC[3+bs*1] * tmp;
20911
20912
20913 D[0+bs*0] = CC[0+bs*0];
20914 D[1+bs*0] = CC[1+bs*0];
20915 D[2+bs*0] = CC[2+bs*0];
20916 D[3+bs*0] = CC[3+bs*0];
20917
20918 D[0+bs*1] = CC[0+bs*1];
20919 D[1+bs*1] = CC[1+bs*1];
20920 D[2+bs*1] = CC[2+bs*1];
20921 D[3+bs*1] = CC[3+bs*1];
20922
20923 D[0+bs*2] = CC[0+bs*2];
20924 D[1+bs*2] = CC[1+bs*2];
20925 D[2+bs*2] = CC[2+bs*2];
20926 D[3+bs*2] = CC[3+bs*2];
20927
20928 D[0+bs*3] = CC[0+bs*3];
20929 D[1+bs*3] = CC[1+bs*3];
20930 D[2+bs*3] = CC[2+bs*3];
20931 D[3+bs*3] = CC[3+bs*3];
20932
20933 return;
20934
20935 }
20936 //#endif
20937
20938
20939
20940 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_one_4x4_vs_lib4c44c(int kmax,float * A,float * B,int ldb,float * beta,float * C,float * D,float * E,int lde,int m1,int n1)20941 void kernel_strsm_nt_ru_one_4x4_vs_lib4c44c(int kmax, float *A, float *B, int ldb, float *beta, float *C, float *D, float *E, int lde, int m1, int n1)
20942 {
20943
20944 const int bs = 4;
20945
20946 float tmp;
20947
20948 #if defined(TARGET_GENERIC)
20949 float CC[16] = {0};
20950 #else
20951 ALIGNED( float CC[16], 64 ) = {0};
20952 #endif
20953
20954 float alpha1 = -1.0;
20955
20956 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, bs, CC, bs);
20957
20958 if(n1<=3)
20959 goto n3;
20960
20961 tmp = E[0+lde*3];
20962 CC[0+bs*0] -= CC[0+bs*3] * tmp;
20963 CC[1+bs*0] -= CC[1+bs*3] * tmp;
20964 CC[2+bs*0] -= CC[2+bs*3] * tmp;
20965 CC[3+bs*0] -= CC[3+bs*3] * tmp;
20966 tmp = E[1+lde*3];
20967 CC[0+bs*1] -= CC[0+bs*3] * tmp;
20968 CC[1+bs*1] -= CC[1+bs*3] * tmp;
20969 CC[2+bs*1] -= CC[2+bs*3] * tmp;
20970 CC[3+bs*1] -= CC[3+bs*3] * tmp;
20971 tmp = E[2+lde*3];
20972 CC[0+bs*2] -= CC[0+bs*3] * tmp;
20973 CC[1+bs*2] -= CC[1+bs*3] * tmp;
20974 CC[2+bs*2] -= CC[2+bs*3] * tmp;
20975 CC[3+bs*2] -= CC[3+bs*3] * tmp;
20976
20977 n3:
20978 if(n1<=2)
20979 goto n2;
20980
20981 tmp = E[0+lde*2];
20982 CC[0+bs*0] -= CC[0+bs*2] * tmp;
20983 CC[1+bs*0] -= CC[1+bs*2] * tmp;
20984 CC[2+bs*0] -= CC[2+bs*2] * tmp;
20985 CC[3+bs*0] -= CC[3+bs*2] * tmp;
20986 tmp = E[1+lde*2];
20987 CC[0+bs*1] -= CC[0+bs*2] * tmp;
20988 CC[1+bs*1] -= CC[1+bs*2] * tmp;
20989 CC[2+bs*1] -= CC[2+bs*2] * tmp;
20990 CC[3+bs*1] -= CC[3+bs*2] * tmp;
20991
20992 n2:
20993 if(n1<=1)
20994 goto n1;
20995
20996 tmp = E[0+lde*1];
20997 CC[0+bs*0] -= CC[0+bs*1] * tmp;
20998 CC[1+bs*0] -= CC[1+bs*1] * tmp;
20999 CC[2+bs*0] -= CC[2+bs*1] * tmp;
21000 CC[3+bs*0] -= CC[3+bs*1] * tmp;
21001
21002 n1:
21003
21004 store:
21005
21006 if(m1>=4)
21007 {
21008 D[0+bs*0] = CC[0+bs*0];
21009 D[1+bs*0] = CC[1+bs*0];
21010 D[2+bs*0] = CC[2+bs*0];
21011 D[3+bs*0] = CC[3+bs*0];
21012
21013 if(n1==1)
21014 return;
21015
21016 D[0+bs*1] = CC[0+bs*1];
21017 D[1+bs*1] = CC[1+bs*1];
21018 D[2+bs*1] = CC[2+bs*1];
21019 D[3+bs*1] = CC[3+bs*1];
21020
21021 if(n1==2)
21022 return;
21023
21024 D[0+bs*2] = CC[0+bs*2];
21025 D[1+bs*2] = CC[1+bs*2];
21026 D[2+bs*2] = CC[2+bs*2];
21027 D[3+bs*2] = CC[3+bs*2];
21028
21029 if(n1==3)
21030 return;
21031
21032 D[0+bs*3] = CC[0+bs*3];
21033 D[1+bs*3] = CC[1+bs*3];
21034 D[2+bs*3] = CC[2+bs*3];
21035 D[3+bs*3] = CC[3+bs*3];
21036 }
21037 else if(m1>=3)
21038 {
21039 D[0+bs*0] = CC[0+bs*0];
21040 D[1+bs*0] = CC[1+bs*0];
21041 D[2+bs*0] = CC[2+bs*0];
21042
21043 if(n1==1)
21044 return;
21045
21046 D[0+bs*1] = CC[0+bs*1];
21047 D[1+bs*1] = CC[1+bs*1];
21048 D[2+bs*1] = CC[2+bs*1];
21049
21050 if(n1==2)
21051 return;
21052
21053 D[0+bs*2] = CC[0+bs*2];
21054 D[1+bs*2] = CC[1+bs*2];
21055 D[2+bs*2] = CC[2+bs*2];
21056
21057 if(n1==3)
21058 return;
21059
21060 D[0+bs*3] = CC[0+bs*3];
21061 D[1+bs*3] = CC[1+bs*3];
21062 D[2+bs*3] = CC[2+bs*3];
21063 }
21064 else if(m1>=2)
21065 {
21066 D[0+bs*0] = CC[0+bs*0];
21067 D[1+bs*0] = CC[1+bs*0];
21068
21069 if(n1==1)
21070 return;
21071
21072 D[0+bs*1] = CC[0+bs*1];
21073 D[1+bs*1] = CC[1+bs*1];
21074
21075 if(n1==2)
21076 return;
21077
21078 D[0+bs*2] = CC[0+bs*2];
21079 D[1+bs*2] = CC[1+bs*2];
21080
21081 if(n1==3)
21082 return;
21083
21084 D[0+bs*3] = CC[0+bs*3];
21085 D[1+bs*3] = CC[1+bs*3];
21086 }
21087 else //if(m1>=1)
21088 {
21089 D[0+bs*0] = CC[0+bs*0];
21090
21091 if(n1==1)
21092 return;
21093
21094 D[0+bs*1] = CC[0+bs*1];
21095
21096 if(n1==2)
21097 return;
21098
21099 D[0+bs*2] = CC[0+bs*2];
21100
21101 if(n1==3)
21102 return;
21103
21104 D[0+bs*3] = CC[0+bs*3];
21105 }
21106
21107 return;
21108
21109 }
21110 //#endif
21111
21112
21113
21114 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_one_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde)21115 void kernel_strsm_nt_ru_one_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde)
21116 {
21117
21118 const int bs = 4;
21119
21120 float tmp;
21121
21122 #if defined(TARGET_GENERIC)
21123 float CC[16] = {0};
21124 #else
21125 ALIGNED( float CC[16], 64 ) = {0};
21126 #endif
21127
21128 float alpha1 = -1.0;
21129
21130 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
21131
21132 tmp = E[0+lde*3];
21133 CC[0+bs*0] -= CC[0+bs*3] * tmp;
21134 CC[1+bs*0] -= CC[1+bs*3] * tmp;
21135 CC[2+bs*0] -= CC[2+bs*3] * tmp;
21136 CC[3+bs*0] -= CC[3+bs*3] * tmp;
21137 tmp = E[1+lde*3];
21138 CC[0+bs*1] -= CC[0+bs*3] * tmp;
21139 CC[1+bs*1] -= CC[1+bs*3] * tmp;
21140 CC[2+bs*1] -= CC[2+bs*3] * tmp;
21141 CC[3+bs*1] -= CC[3+bs*3] * tmp;
21142 tmp = E[2+lde*3];
21143 CC[0+bs*2] -= CC[0+bs*3] * tmp;
21144 CC[1+bs*2] -= CC[1+bs*3] * tmp;
21145 CC[2+bs*2] -= CC[2+bs*3] * tmp;
21146 CC[3+bs*2] -= CC[3+bs*3] * tmp;
21147
21148 tmp = E[0+lde*2];
21149 CC[0+bs*0] -= CC[0+bs*2] * tmp;
21150 CC[1+bs*0] -= CC[1+bs*2] * tmp;
21151 CC[2+bs*0] -= CC[2+bs*2] * tmp;
21152 CC[3+bs*0] -= CC[3+bs*2] * tmp;
21153 tmp = E[1+lde*2];
21154 CC[0+bs*1] -= CC[0+bs*2] * tmp;
21155 CC[1+bs*1] -= CC[1+bs*2] * tmp;
21156 CC[2+bs*1] -= CC[2+bs*2] * tmp;
21157 CC[3+bs*1] -= CC[3+bs*2] * tmp;
21158
21159 tmp = E[0+lde*1];
21160 CC[0+bs*0] -= CC[0+bs*1] * tmp;
21161 CC[1+bs*0] -= CC[1+bs*1] * tmp;
21162 CC[2+bs*0] -= CC[2+bs*1] * tmp;
21163 CC[3+bs*0] -= CC[3+bs*1] * tmp;
21164
21165
21166 D[0+ldd*0] = CC[0+bs*0];
21167 D[1+ldd*0] = CC[1+bs*0];
21168 D[2+ldd*0] = CC[2+bs*0];
21169 D[3+ldd*0] = CC[3+bs*0];
21170
21171 D[0+ldd*1] = CC[0+bs*1];
21172 D[1+ldd*1] = CC[1+bs*1];
21173 D[2+ldd*1] = CC[2+bs*1];
21174 D[3+ldd*1] = CC[3+bs*1];
21175
21176 D[0+ldd*2] = CC[0+bs*2];
21177 D[1+ldd*2] = CC[1+bs*2];
21178 D[2+ldd*2] = CC[2+bs*2];
21179 D[3+ldd*2] = CC[3+bs*2];
21180
21181 D[0+ldd*3] = CC[0+bs*3];
21182 D[1+ldd*3] = CC[1+bs*3];
21183 D[2+ldd*3] = CC[2+bs*3];
21184 D[3+ldd*3] = CC[3+bs*3];
21185
21186 return;
21187
21188 }
21189 //#endif
21190
21191
21192
21193 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9) || defined(TARGET_ARMV8A_ARM_CORTEX_A57) || defined(TARGET_ARMV8A_ARM_CORTEX_A53)
kernel_strsm_nt_ru_one_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,int m1,int n1)21194 void kernel_strsm_nt_ru_one_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, int m1, int n1)
21195 {
21196
21197 const int bs = 4;
21198
21199 float tmp;
21200
21201 #if defined(TARGET_GENERIC)
21202 float CC[16] = {0};
21203 #else
21204 ALIGNED( float CC[16], 64 ) = {0};
21205 #endif
21206
21207 float alpha1 = -1.0;
21208
21209 kernel_sgemm_nt_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
21210
21211 if(n1<=3)
21212 goto n3;
21213
21214 tmp = E[0+lde*3];
21215 CC[0+bs*0] -= CC[0+bs*3] * tmp;
21216 CC[1+bs*0] -= CC[1+bs*3] * tmp;
21217 CC[2+bs*0] -= CC[2+bs*3] * tmp;
21218 CC[3+bs*0] -= CC[3+bs*3] * tmp;
21219 tmp = E[1+lde*3];
21220 CC[0+bs*1] -= CC[0+bs*3] * tmp;
21221 CC[1+bs*1] -= CC[1+bs*3] * tmp;
21222 CC[2+bs*1] -= CC[2+bs*3] * tmp;
21223 CC[3+bs*1] -= CC[3+bs*3] * tmp;
21224 tmp = E[2+lde*3];
21225 CC[0+bs*2] -= CC[0+bs*3] * tmp;
21226 CC[1+bs*2] -= CC[1+bs*3] * tmp;
21227 CC[2+bs*2] -= CC[2+bs*3] * tmp;
21228 CC[3+bs*2] -= CC[3+bs*3] * tmp;
21229
21230 n3:
21231 if(n1<=2)
21232 goto n2;
21233
21234 tmp = E[0+lde*2];
21235 CC[0+bs*0] -= CC[0+bs*2] * tmp;
21236 CC[1+bs*0] -= CC[1+bs*2] * tmp;
21237 CC[2+bs*0] -= CC[2+bs*2] * tmp;
21238 CC[3+bs*0] -= CC[3+bs*2] * tmp;
21239 tmp = E[1+lde*2];
21240 CC[0+bs*1] -= CC[0+bs*2] * tmp;
21241 CC[1+bs*1] -= CC[1+bs*2] * tmp;
21242 CC[2+bs*1] -= CC[2+bs*2] * tmp;
21243 CC[3+bs*1] -= CC[3+bs*2] * tmp;
21244
21245 n2:
21246 if(n1<=1)
21247 goto n1;
21248
21249 tmp = E[0+lde*1];
21250 CC[0+bs*0] -= CC[0+bs*1] * tmp;
21251 CC[1+bs*0] -= CC[1+bs*1] * tmp;
21252 CC[2+bs*0] -= CC[2+bs*1] * tmp;
21253 CC[3+bs*0] -= CC[3+bs*1] * tmp;
21254
21255 n1:
21256
21257 store:
21258
21259 if(m1>=4)
21260 {
21261 D[0+ldd*0] = CC[0+bs*0];
21262 D[1+ldd*0] = CC[1+bs*0];
21263 D[2+ldd*0] = CC[2+bs*0];
21264 D[3+ldd*0] = CC[3+bs*0];
21265
21266 if(n1==1)
21267 return;
21268
21269 D[0+ldd*1] = CC[0+bs*1];
21270 D[1+ldd*1] = CC[1+bs*1];
21271 D[2+ldd*1] = CC[2+bs*1];
21272 D[3+ldd*1] = CC[3+bs*1];
21273
21274 if(n1==2)
21275 return;
21276
21277 D[0+ldd*2] = CC[0+bs*2];
21278 D[1+ldd*2] = CC[1+bs*2];
21279 D[2+ldd*2] = CC[2+bs*2];
21280 D[3+ldd*2] = CC[3+bs*2];
21281
21282 if(n1==3)
21283 return;
21284
21285 D[0+ldd*3] = CC[0+bs*3];
21286 D[1+ldd*3] = CC[1+bs*3];
21287 D[2+ldd*3] = CC[2+bs*3];
21288 D[3+ldd*3] = CC[3+bs*3];
21289 }
21290 else if(m1>=3)
21291 {
21292 D[0+ldd*0] = CC[0+bs*0];
21293 D[1+ldd*0] = CC[1+bs*0];
21294 D[2+ldd*0] = CC[2+bs*0];
21295
21296 if(n1==1)
21297 return;
21298
21299 D[0+ldd*1] = CC[0+bs*1];
21300 D[1+ldd*1] = CC[1+bs*1];
21301 D[2+ldd*1] = CC[2+bs*1];
21302
21303 if(n1==2)
21304 return;
21305
21306 D[0+ldd*2] = CC[0+bs*2];
21307 D[1+ldd*2] = CC[1+bs*2];
21308 D[2+ldd*2] = CC[2+bs*2];
21309
21310 if(n1==3)
21311 return;
21312
21313 D[0+ldd*3] = CC[0+bs*3];
21314 D[1+ldd*3] = CC[1+bs*3];
21315 D[2+ldd*3] = CC[2+bs*3];
21316 }
21317 else if(m1>=2)
21318 {
21319 D[0+ldd*0] = CC[0+bs*0];
21320 D[1+ldd*0] = CC[1+bs*0];
21321
21322 if(n1==1)
21323 return;
21324
21325 D[0+ldd*1] = CC[0+bs*1];
21326 D[1+ldd*1] = CC[1+bs*1];
21327
21328 if(n1==2)
21329 return;
21330
21331 D[0+ldd*2] = CC[0+bs*2];
21332 D[1+ldd*2] = CC[1+bs*2];
21333
21334 if(n1==3)
21335 return;
21336
21337 D[0+ldd*3] = CC[0+bs*3];
21338 D[1+ldd*3] = CC[1+bs*3];
21339 }
21340 else //if(m1>=1)
21341 {
21342 D[0+ldd*0] = CC[0+bs*0];
21343
21344 if(n1==1)
21345 return;
21346
21347 D[0+ldd*1] = CC[0+bs*1];
21348
21349 if(n1==2)
21350 return;
21351
21352 D[0+ldd*2] = CC[0+bs*2];
21353
21354 if(n1==3)
21355 return;
21356
21357 D[0+ldd*3] = CC[0+bs*3];
21358 }
21359
21360 return;
21361
21362 }
21363 //#endif
21364
21365
21366
21367 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_strsm_nn_ll_one_4x4_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde)21368 void kernel_strsm_nn_ll_one_4x4_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde)
21369 {
21370
21371 const int bs = 4;
21372
21373 float tmp,
21374 e_1, e_2, e_3;
21375
21376 #if defined(TARGET_GENERIC)
21377 float CC[16] = {0};
21378 #else
21379 ALIGNED( float CC[16], 64 ) = {0};
21380 #endif
21381
21382 float alpha1 = -1.0;
21383
21384 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
21385
21386 e_1 = E[1+lde*0];
21387 e_2 = E[2+lde*0];
21388 e_3 = E[3+lde*0];
21389 CC[1+bs*0] -= e_1 * CC[0+bs*0];
21390 CC[2+bs*0] -= e_2 * CC[0+bs*0];
21391 CC[3+bs*0] -= e_3 * CC[0+bs*0];
21392 CC[1+bs*1] -= e_1 * CC[0+bs*1];
21393 CC[2+bs*1] -= e_2 * CC[0+bs*1];
21394 CC[3+bs*1] -= e_3 * CC[0+bs*1];
21395 CC[1+bs*2] -= e_1 * CC[0+bs*2];
21396 CC[2+bs*2] -= e_2 * CC[0+bs*2];
21397 CC[3+bs*2] -= e_3 * CC[0+bs*2];
21398 CC[1+bs*3] -= e_1 * CC[0+bs*3];
21399 CC[2+bs*3] -= e_2 * CC[0+bs*3];
21400 CC[3+bs*3] -= e_3 * CC[0+bs*3];
21401
21402 e_2 = E[2+lde*1];
21403 e_3 = E[3+lde*1];
21404 CC[2+bs*0] -= e_2 * CC[1+bs*0];
21405 CC[3+bs*0] -= e_3 * CC[1+bs*0];
21406 CC[2+bs*1] -= e_2 * CC[1+bs*1];
21407 CC[3+bs*1] -= e_3 * CC[1+bs*1];
21408 CC[2+bs*2] -= e_2 * CC[1+bs*2];
21409 CC[3+bs*2] -= e_3 * CC[1+bs*2];
21410 CC[2+bs*3] -= e_2 * CC[1+bs*3];
21411 CC[3+bs*3] -= e_3 * CC[1+bs*3];
21412
21413 e_3 = E[3+lde*2];
21414 CC[3+bs*0] -= e_3 * CC[2+bs*0];
21415 CC[3+bs*1] -= e_3 * CC[2+bs*1];
21416 CC[3+bs*2] -= e_3 * CC[2+bs*2];
21417 CC[3+bs*3] -= e_3 * CC[2+bs*3];
21418
21419
21420 D[0+ldd*0] = CC[0+bs*0];
21421 D[1+ldd*0] = CC[1+bs*0];
21422 D[2+ldd*0] = CC[2+bs*0];
21423 D[3+ldd*0] = CC[3+bs*0];
21424
21425 D[0+ldd*1] = CC[0+bs*1];
21426 D[1+ldd*1] = CC[1+bs*1];
21427 D[2+ldd*1] = CC[2+bs*1];
21428 D[3+ldd*1] = CC[3+bs*1];
21429
21430 D[0+ldd*2] = CC[0+bs*2];
21431 D[1+ldd*2] = CC[1+bs*2];
21432 D[2+ldd*2] = CC[2+bs*2];
21433 D[3+ldd*2] = CC[3+bs*2];
21434
21435 D[0+ldd*3] = CC[0+bs*3];
21436 D[1+ldd*3] = CC[1+bs*3];
21437 D[2+ldd*3] = CC[2+bs*3];
21438 D[3+ldd*3] = CC[3+bs*3];
21439
21440 return;
21441
21442 }
21443 //#endif
21444
21445
21446
21447 //#if defined(TARGET_GENERIC) || defined(TARGET_X86_AMD_BARCELONA) || defined(TARGET_X86_AMD_JAGUAR) || defined(TARGET_X64_INTEL_CORE) || defined(TARGET_X64_AMD_BULLDOZER) || defined(TARGET_ARMV7A_ARM_CORTEX_A15) || defined(TARGET_ARMV7A_ARM_CORTEX_A7) || defined(TARGET_ARMV7A_ARM_CORTEX_A9)
kernel_strsm_nn_ll_one_4x4_vs_lib4cccc(int kmax,float * A,float * B,int ldb,float * beta,float * C,int ldc,float * D,int ldd,float * E,int lde,int m1,int n1)21448 void kernel_strsm_nn_ll_one_4x4_vs_lib4cccc(int kmax, float *A, float *B, int ldb, float *beta, float *C, int ldc, float *D, int ldd, float *E, int lde, int m1, int n1)
21449 {
21450
21451 const int bs = 4;
21452
21453 float tmp,
21454 e_1, e_2, e_3;
21455
21456 #if defined(TARGET_GENERIC)
21457 float CC[16] = {0};
21458 #else
21459 ALIGNED( float CC[16], 64 ) = {0};
21460 #endif
21461
21462 float alpha1 = -1.0;
21463
21464 kernel_sgemm_nn_4x4_lib4ccc(kmax, &alpha1, A, B, ldb, beta, C, ldc, CC, bs);
21465
21466 e_1 = E[1+lde*0];
21467 e_2 = E[2+lde*0];
21468 e_3 = E[3+lde*0];
21469 CC[1+bs*0] -= e_1 * CC[0+bs*0];
21470 CC[2+bs*0] -= e_2 * CC[0+bs*0];
21471 CC[3+bs*0] -= e_3 * CC[0+bs*0];
21472 CC[1+bs*1] -= e_1 * CC[0+bs*1];
21473 CC[2+bs*1] -= e_2 * CC[0+bs*1];
21474 CC[3+bs*1] -= e_3 * CC[0+bs*1];
21475 CC[1+bs*2] -= e_1 * CC[0+bs*2];
21476 CC[2+bs*2] -= e_2 * CC[0+bs*2];
21477 CC[3+bs*2] -= e_3 * CC[0+bs*2];
21478 CC[1+bs*3] -= e_1 * CC[0+bs*3];
21479 CC[2+bs*3] -= e_2 * CC[0+bs*3];
21480 CC[3+bs*3] -= e_3 * CC[0+bs*3];
21481
21482 e_2 = E[2+lde*1];
21483 e_3 = E[3+lde*1];
21484 CC[2+bs*0] -= e_2 * CC[1+bs*0];
21485 CC[3+bs*0] -= e_3 * CC[1+bs*0];
21486 CC[2+bs*1] -= e_2 * CC[1+bs*1];
21487 CC[3+bs*1] -= e_3 * CC[1+bs*1];
21488 CC[2+bs*2] -= e_2 * CC[1+bs*2];
21489 CC[3+bs*2] -= e_3 * CC[1+bs*2];
21490 CC[2+bs*3] -= e_2 * CC[1+bs*3];
21491 CC[3+bs*3] -= e_3 * CC[1+bs*3];
21492
21493 e_3 = E[3+lde*2];
21494 CC[3+bs*0] -= e_3 * CC[2+bs*0];
21495 CC[3+bs*1] -= e_3 * CC[2+bs*1];
21496 CC[3+bs*2] -= e_3 * CC[2+bs*2];
21497 CC[3+bs*3] -= e_3 * CC[2+bs*3];
21498
21499 store:
21500
21501 if(m1>=4)
21502 {
21503 D[0+ldd*0] = CC[0+bs*0];
21504 D[1+ldd*0] = CC[1+bs*0];
21505 D[2+ldd*0] = CC[2+bs*0];
21506 D[3+ldd*0] = CC[3+bs*0];
21507
21508 if(n1==1)
21509 return;
21510
21511 D[0+ldd*1] = CC[0+bs*1];
21512 D[1+ldd*1] = CC[1+bs*1];
21513 D[2+ldd*1] = CC[2+bs*1];
21514 D[3+ldd*1] = CC[3+bs*1];
21515
21516 if(n1==2)
21517 return;
21518
21519 D[0+ldd*2] = CC[0+bs*2];
21520 D[1+ldd*2] = CC[1+bs*2];
21521 D[2+ldd*2] = CC[2+bs*2];
21522 D[3+ldd*2] = CC[3+bs*2];
21523
21524 if(n1==3)
21525 return;
21526
21527 D[0+ldd*3] = CC[0+bs*3];
21528 D[1+ldd*3] = CC[1+bs*3];
21529 D[2+ldd*3] = CC[2+bs*3];
21530 D[3+ldd*3] = CC[3+bs*3];
21531 }
21532 else if(m1>=3)
21533 {
21534 D[0+ldd*0] = CC[0+bs*0];
21535 D[1+ldd*0] = CC[1+bs*0];
21536 D[2+ldd*0] = CC[2+bs*0];
21537
21538 if(n1==1)
21539 return;
21540
21541 D[0+ldd*1] = CC[0+bs*1];
21542 D[1+ldd*1] = CC[1+bs*1];
21543 D[2+ldd*1] = CC[2+bs*1];
21544
21545 if(n1==2)
21546 return;
21547
21548 D[0+ldd*2] = CC[0+bs*2];
21549 D[1+ldd*2] = CC[1+bs*2];
21550 D[2+ldd*2] = CC[2+bs*2];
21551
21552 if(n1==3)
21553 return;
21554
21555 D[0+ldd*3] = CC[0+bs*3];
21556 D[1+ldd*3] = CC[1+bs*3];
21557 D[2+ldd*3] = CC[2+bs*3];
21558 }
21559 else if(m1>=2)
21560 {
21561 D[0+ldd*0] = CC[0+bs*0];
21562 D[1+ldd*0] = CC[1+bs*0];
21563
21564 if(n1==1)
21565 return;
21566
21567 D[0+ldd*1] = CC[0+bs*1];
21568 D[1+ldd*1] = CC[1+bs*1];
21569
21570 if(n1==2)
21571 return;
21572
21573 D[0+ldd*2] = CC[0+bs*2];
21574 D[1+ldd*2] = CC[1+bs*2];
21575
21576 if(n1==3)
21577 return;
21578
21579 D[0+ldd*3] = CC[0+bs*3];
21580 D[1+ldd*3] = CC[1+bs*3];
21581 }
21582 else //if(m1>=1)
21583 {
21584 D[0+ldd*0] = CC[0+bs*0];
21585
21586 if(n1==1)
21587 return;
21588
21589 D[0+ldd*1] = CC[0+bs*1];
21590
21591 if(n1==2)
21592 return;
21593
21594 D[0+ldd*2] = CC[0+bs*2];
21595
21596 if(n1==3)
21597 return;
21598
21599 D[0+ldd*3] = CC[0+bs*3];
21600 }
21601
21602 return;
21603
21604 }
21605 //#endif
21606
21607
21608
21609
21610
21611