1 /*
2
3 BLIS
4 An object-based framework for developing high-performance BLAS-like
5 libraries.
6
7 Copyright (C) 2014, The University of Texas at Austin
8 Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc.
9
10 Redistribution and use in source and binary forms, with or without
11 modification, are permitted provided that the following conditions are
12 met:
13 - Redistributions of source code must retain the above copyright
14 notice, this list of conditions and the following disclaimer.
15 - Redistributions in binary form must reproduce the above copyright
16 notice, this list of conditions and the following disclaimer in the
17 documentation and/or other materials provided with the distribution.
18 - Neither the name(s) of the copyright holder(s) nor the names of its
19 contributors may be used to endorse or promote products derived
20 from this software without specific prior written permission.
21
22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33
34 */
35
36 #ifndef BLIS_PARAM_MACRO_DEFS_H
37 #define BLIS_PARAM_MACRO_DEFS_H
38
39
40 // -- Parameter query macros --
41
42 // buffer
43
bli_is_aligned_to(siz_t p,siz_t size)44 BLIS_INLINE bool bli_is_aligned_to( siz_t p, siz_t size )
45 {
46 return ( bool )
47 ( p % size == 0 );
48 }
49
bli_is_unaligned_to(siz_t p,siz_t size)50 BLIS_INLINE bool bli_is_unaligned_to( siz_t p, siz_t size )
51 {
52 return ( bool )
53 ( p % size != 0 );
54 }
55
bli_offset_past_alignment(siz_t p,siz_t size)56 BLIS_INLINE siz_t bli_offset_past_alignment( siz_t p, siz_t size )
57 {
58 return ( siz_t )
59 ( p % size );
60 }
61
62
63 // datatype
64
bli_is_float(num_t dt)65 BLIS_INLINE bool bli_is_float( num_t dt )
66 {
67 return ( bool )
68 ( dt == BLIS_FLOAT );
69 }
70
bli_is_double(num_t dt)71 BLIS_INLINE bool bli_is_double( num_t dt )
72 {
73 return ( bool )
74 ( dt == BLIS_DOUBLE );
75 }
76
bli_is_scomplex(num_t dt)77 BLIS_INLINE bool bli_is_scomplex( num_t dt )
78 {
79 return ( bool )
80 ( dt == BLIS_SCOMPLEX );
81 }
82
bli_is_dcomplex(num_t dt)83 BLIS_INLINE bool bli_is_dcomplex( num_t dt )
84 {
85 return ( bool )
86 ( dt == BLIS_DCOMPLEX );
87 }
88
bli_is_constant(num_t dt)89 BLIS_INLINE bool bli_is_constant( num_t dt )
90 {
91 return ( bool )
92 ( dt == BLIS_CONSTANT );
93 }
94
bli_is_int(num_t dt)95 BLIS_INLINE bool bli_is_int( num_t dt )
96 {
97 return ( bool )
98 ( dt == BLIS_INT );
99 }
100
bli_is_real(num_t dt)101 BLIS_INLINE bool bli_is_real( num_t dt )
102 {
103 return ( bool )
104 ( bli_is_float( dt ) ||
105 bli_is_double( dt ) );
106 }
107
bli_is_complex(num_t dt)108 BLIS_INLINE bool bli_is_complex( num_t dt )
109 {
110 return ( bool )
111 ( bli_is_scomplex( dt ) ||
112 bli_is_dcomplex( dt ) );
113 }
114
bli_is_single_prec(num_t dt)115 BLIS_INLINE bool bli_is_single_prec( num_t dt )
116 {
117 return ( bool )
118 ( bli_is_float( dt ) ||
119 bli_is_scomplex( dt ) );
120 }
121
bli_is_double_prec(num_t dt)122 BLIS_INLINE bool bli_is_double_prec( num_t dt )
123 {
124 return ( bool )
125 ( bli_is_double( dt ) ||
126 bli_is_dcomplex( dt ) );
127 }
128
bli_dt_domain(num_t dt)129 BLIS_INLINE dom_t bli_dt_domain( num_t dt )
130 {
131 return ( dom_t )
132 ( dt & BLIS_DOMAIN_BIT );
133 }
134
bli_dt_dom_is_real(num_t dt)135 BLIS_INLINE bool bli_dt_dom_is_real( num_t dt )
136 {
137 return ( bool )
138 ( ( dt & BLIS_DOMAIN_BIT ) == BLIS_REAL );
139 }
140
bli_dt_dom_is_complex(num_t dt)141 BLIS_INLINE bool bli_dt_dom_is_complex( num_t dt )
142 {
143 return ( bool )
144 ( ( dt & BLIS_DOMAIN_BIT ) == BLIS_COMPLEX );
145 }
146
bli_dt_prec(num_t dt)147 BLIS_INLINE prec_t bli_dt_prec( num_t dt )
148 {
149 return ( prec_t )
150 ( dt & BLIS_PRECISION_BIT );
151 }
152
bli_dt_prec_is_single(num_t dt)153 BLIS_INLINE bool bli_dt_prec_is_single( num_t dt )
154 {
155 return ( bool )
156 ( ( dt & BLIS_PRECISION_BIT ) == BLIS_SINGLE_PREC );
157 }
158
bli_dt_prec_is_double(num_t dt)159 BLIS_INLINE bool bli_dt_prec_is_double( num_t dt )
160 {
161 return ( bool )
162 ( ( dt & BLIS_PRECISION_BIT ) == BLIS_DOUBLE_PREC );
163 }
164
bli_dt_proj_to_real(num_t dt)165 BLIS_INLINE num_t bli_dt_proj_to_real( num_t dt )
166 {
167 return ( num_t )
168 ( dt & ~BLIS_BITVAL_COMPLEX );
169 }
170
bli_dt_proj_to_complex(num_t dt)171 BLIS_INLINE num_t bli_dt_proj_to_complex( num_t dt )
172 {
173 return ( num_t )
174 ( dt | BLIS_BITVAL_COMPLEX );
175 }
176
bli_dt_proj_to_single_prec(num_t dt)177 BLIS_INLINE num_t bli_dt_proj_to_single_prec( num_t dt )
178 {
179 return ( num_t )
180 ( dt & ~BLIS_BITVAL_DOUBLE_PREC );
181 }
182
bli_dt_proj_to_double_prec(num_t dt)183 BLIS_INLINE num_t bli_dt_proj_to_double_prec( num_t dt )
184 {
185 return ( num_t )
186 ( dt | BLIS_BITVAL_DOUBLE_PREC );
187 }
188
189
190 // trans
191
bli_is_notrans(trans_t trans)192 BLIS_INLINE bool bli_is_notrans( trans_t trans )
193 {
194 return ( bool )
195 ( trans == BLIS_NO_TRANSPOSE );
196 }
197
bli_is_trans(trans_t trans)198 BLIS_INLINE bool bli_is_trans( trans_t trans )
199 {
200 return ( bool )
201 ( trans == BLIS_TRANSPOSE );
202 }
203
bli_is_conjnotrans(trans_t trans)204 BLIS_INLINE bool bli_is_conjnotrans( trans_t trans )
205 {
206 return ( bool )
207 ( trans == BLIS_CONJ_NO_TRANSPOSE );
208 }
209
bli_is_conjtrans(trans_t trans)210 BLIS_INLINE bool bli_is_conjtrans( trans_t trans )
211 {
212 return ( bool )
213 ( trans == BLIS_CONJ_TRANSPOSE );
214 }
215
bli_does_notrans(trans_t trans)216 BLIS_INLINE bool bli_does_notrans( trans_t trans )
217 {
218 return ( bool )
219 ( (~trans & BLIS_TRANS_BIT ) == BLIS_BITVAL_TRANS );
220 }
221
bli_does_trans(trans_t trans)222 BLIS_INLINE bool bli_does_trans( trans_t trans )
223 {
224 return ( bool )
225 ( ( trans & BLIS_TRANS_BIT ) == BLIS_BITVAL_TRANS );
226 }
227
bli_does_noconj(trans_t trans)228 BLIS_INLINE bool bli_does_noconj( trans_t trans )
229 {
230 return ( bool )
231 ( (~trans & BLIS_CONJ_BIT ) == BLIS_BITVAL_CONJ );
232 }
233
bli_does_conj(trans_t trans)234 BLIS_INLINE bool bli_does_conj( trans_t trans )
235 {
236 return ( bool )
237 ( ( trans & BLIS_CONJ_BIT ) == BLIS_BITVAL_CONJ );
238 }
239
bli_extract_trans(trans_t trans)240 BLIS_INLINE trans_t bli_extract_trans( trans_t trans )
241 {
242 return ( trans_t )
243 ( trans & BLIS_TRANS_BIT );
244 }
245
bli_extract_conj(trans_t trans)246 BLIS_INLINE conj_t bli_extract_conj( trans_t trans )
247 {
248 return ( conj_t )
249 ( trans & BLIS_CONJ_BIT );
250 }
251
bli_trans_toggled(trans_t trans)252 BLIS_INLINE trans_t bli_trans_toggled( trans_t trans )
253 {
254 return ( trans_t )
255 ( trans ^ BLIS_TRANS_BIT );
256 }
257
bli_trans_toggled_conj(trans_t trans)258 BLIS_INLINE trans_t bli_trans_toggled_conj( trans_t trans )
259 {
260 return ( trans_t )
261 ( trans ^ BLIS_CONJ_BIT );
262 }
263
bli_toggle_trans(trans_t * trans)264 BLIS_INLINE void bli_toggle_trans( trans_t* trans )
265 {
266 *trans = bli_trans_toggled( *trans );
267 }
268
269
270 // side
271
bli_is_left(side_t side)272 BLIS_INLINE bool bli_is_left( side_t side )
273 {
274 return ( bool )
275 ( side == BLIS_LEFT );
276 }
277
bli_is_right(side_t side)278 BLIS_INLINE bool bli_is_right( side_t side )
279 {
280 return ( bool )
281 ( side == BLIS_RIGHT );
282 }
283
bli_side_toggled(side_t side)284 BLIS_INLINE side_t bli_side_toggled( side_t side )
285 {
286 return ( bli_is_left( side ) ? BLIS_RIGHT : BLIS_LEFT );
287 }
288
bli_toggle_side(side_t * side)289 BLIS_INLINE void bli_toggle_side( side_t* side )
290 {
291 *side = bli_side_toggled( *side );
292 }
293
294
295 // uplo
296
bli_is_lower(uplo_t uplo)297 BLIS_INLINE bool bli_is_lower( uplo_t uplo )
298 {
299 return ( bool )
300 ( uplo == BLIS_LOWER );
301 }
302
bli_is_upper(uplo_t uplo)303 BLIS_INLINE bool bli_is_upper( uplo_t uplo )
304 {
305 return ( bool )
306 ( uplo == BLIS_UPPER );
307 }
308
bli_is_upper_or_lower(uplo_t uplo)309 BLIS_INLINE bool bli_is_upper_or_lower( uplo_t uplo )
310 {
311 return ( bool )
312 ( bli_is_upper( uplo ) ||
313 bli_is_lower( uplo ) );
314 }
315
bli_is_dense(uplo_t uplo)316 BLIS_INLINE bool bli_is_dense( uplo_t uplo )
317 {
318 return ( bool )
319 ( uplo == BLIS_DENSE );
320 }
321
bli_is_zeros(uplo_t uplo)322 BLIS_INLINE bool bli_is_zeros( uplo_t uplo )
323 {
324 return ( bool )
325 ( uplo == BLIS_ZEROS );
326 }
327
bli_uplo_toggled(uplo_t uplo)328 BLIS_INLINE uplo_t bli_uplo_toggled( uplo_t uplo )
329 {
330 return ( uplo_t )
331 ( bli_is_upper_or_lower( uplo )
332 ? ( ( uplo ^ BLIS_LOWER_BIT ) ^ BLIS_UPPER_BIT )
333 : uplo
334 );
335 }
336
bli_toggle_uplo(uplo_t * uplo)337 BLIS_INLINE void bli_toggle_uplo( uplo_t* uplo )
338 {
339 *uplo = bli_uplo_toggled( *uplo );
340 }
341
342
343 // structure
344
bli_is_general(struc_t struc)345 BLIS_INLINE bool bli_is_general( struc_t struc )
346 {
347 return ( bool )
348 ( struc == BLIS_GENERAL );
349 }
350
bli_is_hermitian(struc_t struc)351 BLIS_INLINE bool bli_is_hermitian( struc_t struc )
352 {
353 return ( bool )
354 ( struc == BLIS_HERMITIAN );
355 }
356
bli_is_symmetric(struc_t struc)357 BLIS_INLINE bool bli_is_symmetric( struc_t struc )
358 {
359 return ( bool )
360 ( struc == BLIS_SYMMETRIC );
361 }
362
bli_is_triangular(struc_t struc)363 BLIS_INLINE bool bli_is_triangular( struc_t struc )
364 {
365 return ( bool )
366 ( struc == BLIS_TRIANGULAR );
367 }
368
bli_is_herm_or_symm(struc_t struc)369 BLIS_INLINE bool bli_is_herm_or_symm( struc_t struc )
370 {
371 return ( bool )
372 ( bli_is_hermitian( struc ) ||
373 bli_is_symmetric( struc ) );
374 }
375
376
377 // conj
378
bli_is_noconj(conj_t conj)379 BLIS_INLINE bool bli_is_noconj( conj_t conj )
380 {
381 return ( bool )
382 ( conj == BLIS_NO_CONJUGATE );
383 }
384
bli_is_conj(conj_t conj)385 BLIS_INLINE bool bli_is_conj( conj_t conj )
386 {
387 return ( bool )
388 ( conj == BLIS_CONJUGATE );
389 }
390
bli_conj_toggled(conj_t conj)391 BLIS_INLINE conj_t bli_conj_toggled( conj_t conj )
392 {
393 return ( conj_t )
394 ( conj ^ BLIS_CONJ_BIT );
395 }
396
bli_apply_conj(conj_t conjapp,conj_t conj)397 BLIS_INLINE conj_t bli_apply_conj( conj_t conjapp, conj_t conj )
398 {
399 return ( conj_t )
400 ( conj ^ conjapp );
401 }
402
bli_toggle_conj(conj_t * conj)403 BLIS_INLINE void bli_toggle_conj( conj_t* conj )
404 {
405 *conj = bli_conj_toggled( *conj );
406 }
407
408
409 // diag
410
bli_is_nonunit_diag(diag_t diag)411 BLIS_INLINE bool bli_is_nonunit_diag( diag_t diag )
412 {
413 return ( bool )
414 ( diag == BLIS_NONUNIT_DIAG );
415 }
416
bli_is_unit_diag(diag_t diag)417 BLIS_INLINE bool bli_is_unit_diag( diag_t diag )
418 {
419 return ( bool )
420 ( diag == BLIS_UNIT_DIAG );
421 }
422
423
424 // dimension-related
425
bli_zero_dim1(dim_t m)426 BLIS_INLINE bool bli_zero_dim1( dim_t m )
427 {
428 return ( bool )
429 ( m == 0 );
430 }
431
bli_zero_dim2(dim_t m,dim_t n)432 BLIS_INLINE bool bli_zero_dim2( dim_t m, dim_t n )
433 {
434 return ( bool )
435 ( m == 0 || n == 0 );
436 }
437
bli_zero_dim3(dim_t m,dim_t n,dim_t k)438 BLIS_INLINE bool bli_zero_dim3( dim_t m, dim_t n, dim_t k )
439 {
440 return ( bool )
441 ( m == 0 || n == 0 || k == 0 );
442 }
443
bli_nonzero_dim(dim_t m)444 BLIS_INLINE bool bli_nonzero_dim( dim_t m )
445 {
446 return ( bool )
447 ( m > 0 );
448 }
449
bli_vector_dim(dim_t m,dim_t n)450 BLIS_INLINE bool bli_vector_dim( dim_t m, dim_t n )
451 {
452 return ( bool )
453 ( m == 1 ? n : m );
454 }
455
bli_is_vector(dim_t m,dim_t n)456 BLIS_INLINE bool bli_is_vector( dim_t m, dim_t n )
457 {
458 return ( bool )
459 ( m == 1 || n == 1 );
460 }
461
bli_is_row_vector(dim_t m,dim_t n)462 BLIS_INLINE bool bli_is_row_vector( dim_t m, dim_t n )
463 {
464 return ( bool )
465 ( m == 1 );
466 }
467
bli_is_col_vector(dim_t m,dim_t n)468 BLIS_INLINE bool bli_is_col_vector( dim_t m, dim_t n )
469 {
470 return ( bool )
471 ( n == 1 );
472 }
473
bli_set_dim_with_side(side_t side,dim_t m,dim_t n,dim_t * dim)474 BLIS_INLINE void bli_set_dim_with_side( side_t side, dim_t m, dim_t n, dim_t* dim )
475 {
476 if ( bli_is_left( side ) ) *dim = m;
477 else *dim = n;
478 }
479
bli_set_dims_with_trans(trans_t trans,dim_t m,dim_t n,dim_t * mt,dim_t * nt)480 BLIS_INLINE void bli_set_dims_with_trans( trans_t trans, dim_t m, dim_t n, dim_t* mt, dim_t* nt )
481 {
482 if ( bli_does_notrans( trans ) ) { *mt = m; *nt = n; }
483 else { *mt = n; *nt = m; }
484 }
485
bli_set_dims_incs_with_trans(trans_t trans,dim_t m,dim_t n,inc_t rs,inc_t cs,dim_t * mt,dim_t * nt,inc_t * rst,inc_t * cst)486 BLIS_INLINE void bli_set_dims_incs_with_trans( trans_t trans,
487 dim_t m, dim_t n, inc_t rs, inc_t cs,
488 dim_t* mt, dim_t* nt, inc_t* rst, inc_t* cst )
489 {
490 if ( bli_does_notrans( trans ) ) { *mt = m; *nt = n; *rst = rs; *cst = cs; }
491 else { *mt = n; *nt = m; *rst = cs; *cst = rs; }
492 }
493
494
495 // blocksize-related
496
bli_determine_blocksize_dim_f(dim_t i,dim_t dim,dim_t b_alg)497 BLIS_INLINE dim_t bli_determine_blocksize_dim_f( dim_t i, dim_t dim, dim_t b_alg )
498 {
499 return ( dim_t )
500 ( bli_min( b_alg, dim - i ) );
501 }
502
bli_determine_blocksize_dim_b(dim_t i,dim_t dim,dim_t b_alg)503 BLIS_INLINE dim_t bli_determine_blocksize_dim_b( dim_t i, dim_t dim, dim_t b_alg )
504 {
505 return ( dim_t )
506 ( i == 0 && dim % b_alg != 0 ? dim % b_alg
507 : b_alg );
508 }
509
510
511 // stride-related
512
bli_vector_inc(trans_t trans,dim_t m,dim_t n,inc_t rs,inc_t cs)513 BLIS_INLINE inc_t bli_vector_inc( trans_t trans, dim_t m, dim_t n, inc_t rs, inc_t cs )
514 {
515 return ( inc_t )
516 ( bli_does_notrans( trans ) ? ( m == 1 ? cs : rs )
517 : ( m == 1 ? rs : cs ) );
518 }
519
bli_is_row_stored(inc_t rs,inc_t cs)520 BLIS_INLINE bool bli_is_row_stored( inc_t rs, inc_t cs )
521 {
522 return ( bool )
523 ( bli_abs( cs ) == 1 );
524 }
525
bli_is_col_stored(inc_t rs,inc_t cs)526 BLIS_INLINE bool bli_is_col_stored( inc_t rs, inc_t cs )
527 {
528 return ( bool )
529 ( bli_abs( rs ) == 1 );
530 }
531
bli_is_row_stored_f(dim_t m,dim_t n,inc_t rs,inc_t cs)532 BLIS_INLINE bool bli_is_row_stored_f( dim_t m, dim_t n, inc_t rs, inc_t cs )
533 {
534 return ( bool )
535 ( cs == 1 && ( rs > 1 || n == 1 ) );
536 }
537
bli_is_col_stored_f(dim_t m,dim_t n,inc_t rs,inc_t cs)538 BLIS_INLINE bool bli_is_col_stored_f( dim_t m, dim_t n, inc_t rs, inc_t cs )
539 {
540 return ( bool )
541 ( rs == 1 && ( cs > 1 || m == 1 ) );
542 }
543
bli_is_gen_stored(inc_t rs,inc_t cs)544 BLIS_INLINE bool bli_is_gen_stored( inc_t rs, inc_t cs )
545 {
546 return ( bool )
547 ( bli_abs( rs ) != 1 &&
548 bli_abs( cs ) != 1 );
549 }
550
bli_is_row_tilted(dim_t m,dim_t n,inc_t rs,inc_t cs)551 BLIS_INLINE bool bli_is_row_tilted( dim_t m, dim_t n, inc_t rs, inc_t cs )
552 {
553 return ( bool )
554 ( bli_abs( cs ) == bli_abs( rs )
555 ? n < m
556 : bli_abs( cs ) < bli_abs( rs ) );
557 }
558
bli_is_col_tilted(dim_t m,dim_t n,inc_t rs,inc_t cs)559 BLIS_INLINE bool bli_is_col_tilted( dim_t m, dim_t n, inc_t rs, inc_t cs )
560 {
561 return ( bool )
562 ( bli_abs( rs ) == bli_abs( cs )
563 ? m < n
564 : bli_abs( rs ) < bli_abs( cs ) );
565 }
566
bli_has_nonunit_inc1(inc_t s1)567 BLIS_INLINE bool bli_has_nonunit_inc1( inc_t s1 )
568 {
569 return ( bool )
570 ( s1 != 1 );
571 }
572
bli_has_nonunit_inc2(inc_t s1,inc_t s2)573 BLIS_INLINE bool bli_has_nonunit_inc2( inc_t s1, inc_t s2 )
574 {
575 return ( bool )
576 ( s1 != 1 || s2 != 1 );
577 }
578
bli_has_nonunit_inc3(inc_t s1,inc_t s2,inc_t s3)579 BLIS_INLINE bool bli_has_nonunit_inc3( inc_t s1, inc_t s2, inc_t s3 )
580 {
581 return ( bool )
582 ( s1 != 1 || s2 != 1 || s3 != 1 );
583 }
584
585
586 // diag offset-related
587
bli_negate_diag_offset(doff_t * diagoff)588 BLIS_INLINE void bli_negate_diag_offset( doff_t* diagoff )
589 {
590 *diagoff = -(*diagoff);
591 }
592
bli_shift_diag_offset_to_grow_uplo(uplo_t uplo,doff_t * diagoff)593 BLIS_INLINE void bli_shift_diag_offset_to_grow_uplo( uplo_t uplo, doff_t* diagoff )
594 {
595 if ( bli_is_upper( uplo ) ) *diagoff -= 1;
596 else if ( bli_is_lower( uplo ) ) *diagoff += 1;
597 }
598
bli_shift_diag_offset_to_shrink_uplo(uplo_t uplo,doff_t * diagoff)599 BLIS_INLINE void bli_shift_diag_offset_to_shrink_uplo( uplo_t uplo, doff_t* diagoff )
600 {
601 if ( bli_is_upper( uplo ) ) *diagoff += 1;
602 else if ( bli_is_lower( uplo ) ) *diagoff -= 1;
603 }
604
bli_diag_offset_with_trans(trans_t trans,doff_t diagoff)605 BLIS_INLINE doff_t bli_diag_offset_with_trans( trans_t trans, doff_t diagoff )
606 {
607 return ( doff_t )
608 ( bli_does_trans( trans ) ? -diagoff : diagoff );
609 }
610
bli_is_strictly_above_diag(doff_t diagoff,trans_t trans,dim_t m,dim_t n)611 BLIS_INLINE bool bli_is_strictly_above_diag( doff_t diagoff, trans_t trans, dim_t m, dim_t n )
612 {
613 return ( bool )
614 ( bli_does_trans( trans )
615 ? ( ( doff_t )n <= -diagoff )
616 : ( ( doff_t )m <= -diagoff ) );
617 }
618
bli_is_strictly_below_diag(doff_t diagoff,trans_t trans,dim_t m,dim_t n)619 BLIS_INLINE bool bli_is_strictly_below_diag( doff_t diagoff, trans_t trans, dim_t m, dim_t n )
620 {
621 return ( bool )
622 ( bli_does_trans( trans )
623 ? ( ( doff_t )m <= diagoff )
624 : ( ( doff_t )n <= diagoff ) );
625 }
626
bli_is_outside_diag(doff_t diagoff,trans_t trans,dim_t m,dim_t n)627 BLIS_INLINE bool bli_is_outside_diag( doff_t diagoff, trans_t trans, dim_t m, dim_t n )
628 {
629 return ( bool )
630 ( bli_is_strictly_above_diag( diagoff, trans, m, n ) ||
631 bli_is_strictly_below_diag( diagoff, trans, m, n ) );
632 }
633
bli_is_stored_subpart(doff_t diagoff,trans_t trans,uplo_t uplo,dim_t m,dim_t n)634 BLIS_INLINE bool bli_is_stored_subpart( doff_t diagoff, trans_t trans, uplo_t uplo, dim_t m, dim_t n )
635 {
636 return ( bool )
637 ( ( bli_is_upper( uplo ) && bli_is_strictly_above_diag( diagoff, trans, m, n ) ) ||
638 ( bli_is_lower( uplo ) && bli_is_strictly_below_diag( diagoff, trans, m, n ) ) );
639 }
640
bli_is_unstored_subpart(doff_t diagoff,trans_t trans,uplo_t uplo,dim_t m,dim_t n)641 BLIS_INLINE bool bli_is_unstored_subpart( doff_t diagoff, trans_t trans, uplo_t uplo, dim_t m, dim_t n )
642 {
643 return ( bool )
644 ( ( bli_is_upper( uplo ) && bli_is_strictly_below_diag( diagoff, trans, m, n ) ) ||
645 ( bli_is_lower( uplo ) && bli_is_strictly_above_diag( diagoff, trans, m, n ) ) );
646 }
647
bli_is_strictly_above_diag_n(doff_t diagoff,dim_t m,dim_t n)648 BLIS_INLINE bool bli_is_strictly_above_diag_n( doff_t diagoff, dim_t m, dim_t n )
649 {
650 return ( bool )
651 ( ( doff_t )m <= -diagoff );
652 }
653
bli_is_strictly_below_diag_n(doff_t diagoff,dim_t m,dim_t n)654 BLIS_INLINE bool bli_is_strictly_below_diag_n( doff_t diagoff, dim_t m, dim_t n )
655 {
656 return ( bool )
657 ( ( doff_t )n <= diagoff );
658 }
659
bli_intersects_diag_n(doff_t diagoff,dim_t m,dim_t n)660 BLIS_INLINE bool bli_intersects_diag_n( doff_t diagoff, dim_t m, dim_t n )
661 {
662 return ( bool )
663 ( !bli_is_strictly_above_diag_n( diagoff, m, n ) &&
664 !bli_is_strictly_below_diag_n( diagoff, m, n ) );
665 }
666
bli_is_outside_diag_n(doff_t diagoff,dim_t m,dim_t n)667 BLIS_INLINE bool bli_is_outside_diag_n( doff_t diagoff, dim_t m, dim_t n )
668 {
669 return ( bool )
670 ( bli_is_strictly_above_diag_n( diagoff, m, n ) ||
671 bli_is_strictly_below_diag_n( diagoff, m, n ) );
672 }
673
bli_is_stored_subpart_n(doff_t diagoff,uplo_t uplo,dim_t m,dim_t n)674 BLIS_INLINE bool bli_is_stored_subpart_n( doff_t diagoff, uplo_t uplo, dim_t m, dim_t n )
675 {
676 return ( bool )
677 ( ( bli_is_upper( uplo ) && bli_is_strictly_above_diag_n( diagoff, m, n ) ) ||
678 ( bli_is_lower( uplo ) && bli_is_strictly_below_diag_n( diagoff, m, n ) ) );
679 }
680
bli_is_unstored_subpart_n(doff_t diagoff,uplo_t uplo,dim_t m,dim_t n)681 BLIS_INLINE bool bli_is_unstored_subpart_n( doff_t diagoff, uplo_t uplo, dim_t m, dim_t n )
682 {
683 return ( bool )
684 ( ( bli_is_upper( uplo ) && bli_is_strictly_below_diag_n( diagoff, m, n ) ) ||
685 ( bli_is_lower( uplo ) && bli_is_strictly_above_diag_n( diagoff, m, n ) ) );
686 }
687
688
689 // pruning-related
690
bli_prune_unstored_region_top_l(doff_t * diagoff,dim_t * m,dim_t * n,dim_t * offm_inc)691 BLIS_INLINE void bli_prune_unstored_region_top_l( doff_t* diagoff, dim_t* m, dim_t* n, dim_t* offm_inc )
692 {
693 *offm_inc = 0;
694
695 // If the diagonal intersects the left side of the matrix,
696 // ignore the area above that intersection.
697 if ( *diagoff < 0 )
698 {
699 *m = *m + *diagoff;
700 *offm_inc = - *diagoff;
701 *diagoff = 0;
702 }
703 }
704
bli_prune_unstored_region_right_l(doff_t * diagoff,dim_t * m,dim_t * n,dim_t * offn_inc)705 BLIS_INLINE void bli_prune_unstored_region_right_l( doff_t* diagoff, dim_t* m, dim_t* n, dim_t* offn_inc )
706 {
707 *offn_inc = 0;
708
709 // If the diagonal intersects the bottom side of the matrix,
710 // ignore the area to the right of that intersection.
711 if ( *n > *diagoff + *m )
712 {
713 *n = *diagoff + *m;
714 }
715 }
716
bli_prune_unstored_region_left_u(doff_t * diagoff,dim_t * m,dim_t * n,dim_t * offn_inc)717 BLIS_INLINE void bli_prune_unstored_region_left_u( doff_t* diagoff, dim_t* m, dim_t* n, dim_t* offn_inc )
718 {
719 *offn_inc = 0;
720
721 // If the diagonal intersects the top side of the matrix,
722 // ignore the area to the left of that intersection.
723 if ( *diagoff > 0 )
724 {
725 *n = *n - *diagoff;
726 *offn_inc = + *diagoff;
727 *diagoff = 0;
728 }
729 }
730
bli_prune_unstored_region_bottom_u(doff_t * diagoff,dim_t * m,dim_t * n,dim_t * offm_inc)731 BLIS_INLINE void bli_prune_unstored_region_bottom_u( doff_t* diagoff, dim_t* m, dim_t* n, dim_t* offm_inc )
732 {
733 *offm_inc = 0;
734
735 // If the diagonal intersects the right side of the matrix,
736 // ignore the area below that intersection.
737 if ( *m > -(*diagoff) + *n )
738 {
739 *m = -(*diagoff) + *n;
740 }
741 }
742
743
744 // thread range-related
745
bli_rotate180_trapezoid(doff_t * diagoff,uplo_t * uplo,dim_t * m,dim_t * n)746 BLIS_INLINE void bli_rotate180_trapezoid( doff_t* diagoff, uplo_t* uplo, dim_t* m, dim_t* n )
747 {
748 *diagoff = *n - *diagoff - *m;
749 bli_toggle_uplo( uplo );
750 }
751
bli_reflect_about_diag(doff_t * diagoff,uplo_t * uplo,dim_t * m,dim_t * n)752 BLIS_INLINE void bli_reflect_about_diag( doff_t* diagoff, uplo_t* uplo, dim_t* m, dim_t* n )
753 {
754 bli_swap_dims( m, n );
755 bli_negate_diag_offset( diagoff );
756 bli_toggle_uplo( uplo );
757 }
758
bli_reverse_index_direction(dim_t n,dim_t * start,dim_t * end)759 BLIS_INLINE void bli_reverse_index_direction( dim_t n, dim_t* start, dim_t* end )
760 {
761 dim_t start2 = n - *start;
762 dim_t end2 = n - *end;
763 *start = end2;
764 *end = start2;
765 }
766
767
768 // mdim_t-related
769
bli_is_m_dim(mdim_t mdim)770 BLIS_INLINE bool bli_is_m_dim( mdim_t mdim )
771 {
772 return ( bool )
773 ( mdim == BLIS_M );
774 }
775
bli_is_n_dim(mdim_t mdim)776 BLIS_INLINE bool bli_is_n_dim( mdim_t mdim )
777 {
778 return ( bool )
779 ( mdim == BLIS_N );
780 }
781
bli_dim_toggled(mdim_t mdim)782 BLIS_INLINE mdim_t bli_dim_toggled( mdim_t mdim )
783 {
784 return ( mdim_t )
785 ( mdim == BLIS_M ? BLIS_N : BLIS_M );
786 }
787
bli_toggle_dim(mdim_t * mdim)788 BLIS_INLINE void bli_toggle_dim( mdim_t* mdim )
789 {
790 *mdim = bli_dim_toggled( *mdim );
791 }
792
793
794 // stor3_t-related
795
bli_stor3_from_strides(inc_t rs_c,inc_t cs_c,inc_t rs_a,inc_t cs_a,inc_t rs_b,inc_t cs_b)796 BLIS_INLINE stor3_t bli_stor3_from_strides( inc_t rs_c, inc_t cs_c,
797 inc_t rs_a, inc_t cs_a,
798 inc_t rs_b, inc_t cs_b )
799 {
800 // If any matrix is general-stored, return the stor3_t id for the
801 // general-purpose sup microkernel.
802 if ( bli_is_gen_stored( rs_c, cs_c ) ||
803 bli_is_gen_stored( rs_a, cs_a ) ||
804 bli_is_gen_stored( rs_b, cs_b ) ) return BLIS_XXX;
805
806 // Otherwise, compute and return the stor3_t id as follows.
807 const bool c_is_col = bli_is_col_stored( rs_c, cs_c );
808 const bool a_is_col = bli_is_col_stored( rs_a, cs_a );
809 const bool b_is_col = bli_is_col_stored( rs_b, cs_b );
810
811 return ( stor3_t )( 4 * c_is_col +
812 2 * a_is_col +
813 1 * b_is_col );
814 }
815
bli_stor3_trans(stor3_t id)816 BLIS_INLINE stor3_t bli_stor3_trans( stor3_t id )
817 {
818 #if 1
819 stor3_t map[ BLIS_NUM_3OP_RC_COMBOS ]
820 =
821 {
822 ( stor3_t )7, // BLIS_RRR = 0 -> BLIS_CCC = 7
823 ( stor3_t )5, // BLIS_RRC = 1 -> BLIS_CRC = 5
824 ( stor3_t )6, // BLIS_RCR = 2 -> BLIS_CCR = 6
825 ( stor3_t )4, // BLIS_RCC = 3 -> BLIS_CRR = 4
826 ( stor3_t )3, // BLIS_CRR = 4 -> BLIS_RCC = 3
827 ( stor3_t )1, // BLIS_CRC = 5 -> BLIS_RRC = 1
828 ( stor3_t )2, // BLIS_CCR = 6 -> BLIS_RCR = 2
829 ( stor3_t )0, // BLIS_CCC = 7 -> BLIS_RRR = 0
830 };
831
832 return map[id];
833 #else
834 return ( ( id & 0x4 ) ^ 0x4 ) | // flip c bit
835 ( ( ( id & 0x1 ) ^ 0x1 ) << 1 ) | // flip b bit and move to a position
836 ( ( ( id & 0x2 ) ^ 0x2 ) >> 1 ); // flip a bit and move to b position
837 #endif
838 }
839
bli_stor3_transa(stor3_t id)840 BLIS_INLINE stor3_t bli_stor3_transa( stor3_t id )
841 {
842 #if 0
843 stor3_t map[ BLIS_NUM_3OP_RC_COMBOS ]
844 =
845 {
846 ( stor3_t )1, // BLIS_RRR = 0 -> BLIS_RRC = 1
847 ( stor3_t )0, // BLIS_RRC = 1 -> BLIS_RRR = 0
848 ( stor3_t )3, // BLIS_RCR = 2 -> BLIS_RCC = 3
849 ( stor3_t )2, // BLIS_RCC = 3 -> BLIS_RCR = 2
850 ( stor3_t )5, // BLIS_CRR = 4 -> BLIS_CRC = 5
851 ( stor3_t )4, // BLIS_CRC = 5 -> BLIS_CRR = 4
852 ( stor3_t )7, // BLIS_CCR = 6 -> BLIS_CCC = 7
853 ( stor3_t )6, // BLIS_CCC = 7 -> BLIS_CCR = 6
854 };
855
856 return map[id];
857 #else
858 return ( stor3_t )( id ^ 0x1 );
859 #endif
860 }
861
bli_stor3_transb(stor3_t id)862 BLIS_INLINE stor3_t bli_stor3_transb( stor3_t id )
863 {
864 #if 0
865 stor3_t map[ BLIS_NUM_3OP_RC_COMBOS ]
866 =
867 {
868 ( stor3_t )2, // BLIS_RRR = 0 -> BLIS_RCR = 2
869 ( stor3_t )3, // BLIS_RRC = 1 -> BLIS_RCC = 3
870 ( stor3_t )0, // BLIS_RCR = 2 -> BLIS_RRR = 0
871 ( stor3_t )1, // BLIS_RCC = 3 -> BLIS_RRC = 1
872 ( stor3_t )6, // BLIS_CRR = 4 -> BLIS_CCR = 6
873 ( stor3_t )7, // BLIS_CRC = 5 -> BLIS_CCC = 7
874 ( stor3_t )4, // BLIS_CCR = 6 -> BLIS_CRR = 4
875 ( stor3_t )5, // BLIS_CCC = 7 -> BLIS_CRC = 5
876 };
877
878 return map[id];
879 #else
880 return ( stor3_t )( id ^ 0x2 );
881 #endif
882 }
883
884
885
886 // index-related
887
bli_is_edge_f(dim_t i,dim_t n_iter,dim_t n_left)888 BLIS_INLINE bool bli_is_edge_f( dim_t i, dim_t n_iter, dim_t n_left )
889 {
890 return ( bool )
891 ( i == n_iter - 1 && n_left != 0 );
892 }
893
bli_is_not_edge_f(dim_t i,dim_t n_iter,dim_t n_left)894 BLIS_INLINE bool bli_is_not_edge_f( dim_t i, dim_t n_iter, dim_t n_left )
895 {
896 return ( bool )
897 ( i != n_iter - 1 || n_left == 0 );
898 }
899
bli_is_edge_b(dim_t i,dim_t n_iter,dim_t n_left)900 BLIS_INLINE bool bli_is_edge_b( dim_t i, dim_t n_iter, dim_t n_left )
901 {
902 return ( bool )
903 ( i == 0 && n_left != 0 );
904 }
905
bli_is_not_edge_b(dim_t i,dim_t n_iter,dim_t n_left)906 BLIS_INLINE bool bli_is_not_edge_b( dim_t i, dim_t n_iter, dim_t n_left )
907 {
908 return ( bool )
909 ( i != 0 || n_left == 0 );
910 }
911
bli_is_last_iter_sl(dim_t i,dim_t end_iter,dim_t tid,dim_t nth)912 BLIS_INLINE bool bli_is_last_iter_sl( dim_t i, dim_t end_iter, dim_t tid, dim_t nth )
913 {
914 return ( bool )
915 ( i == end_iter - 1 );
916 }
917
bli_is_last_iter_rr(dim_t i,dim_t end_iter,dim_t tid,dim_t nth)918 BLIS_INLINE bool bli_is_last_iter_rr( dim_t i, dim_t end_iter, dim_t tid, dim_t nth )
919 {
920 return ( bool )
921 ( i == end_iter - 1 - ( ( end_iter - tid - 1 ) % nth ) );
922 }
923
bli_is_last_iter(dim_t i,dim_t end_iter,dim_t tid,dim_t nth)924 BLIS_INLINE bool bli_is_last_iter( dim_t i, dim_t end_iter, dim_t tid, dim_t nth )
925 {
926 #ifdef BLIS_ENABLE_JRIR_SLAB
927 return bli_is_last_iter_sl( i, end_iter, tid, nth );
928 #else // BLIS_ENABLE_JRIR_RR
929 return bli_is_last_iter_rr( i, end_iter, tid, nth );
930 #endif
931 }
932
933
934 // packbuf_t-related
935
bli_packbuf_index(packbuf_t buf_type)936 BLIS_INLINE guint_t bli_packbuf_index( packbuf_t buf_type )
937 {
938 return ( guint_t )
939 ( ( buf_type & BLIS_PACK_BUFFER_BITS ) >> BLIS_PACK_BUFFER_SHIFT );
940 }
941
942 // pack_t-related
943
bli_is_packed(pack_t schema)944 BLIS_INLINE bool bli_is_packed( pack_t schema )
945 {
946 return ( bool )
947 ( schema & BLIS_PACK_BIT );
948 }
949
bli_is_row_packed(pack_t schema)950 BLIS_INLINE bool bli_is_row_packed( pack_t schema )
951 {
952 return ( bool )
953 ( ( schema & BLIS_PACK_RC_BIT ) == ( BLIS_BITVAL_PACKED_UNSPEC ^
954 BLIS_BITVAL_PACKED_ROWS ) );
955 }
956
bli_is_col_packed(pack_t schema)957 BLIS_INLINE bool bli_is_col_packed( pack_t schema )
958 {
959 return ( bool )
960 ( ( schema & BLIS_PACK_RC_BIT ) == ( BLIS_BITVAL_PACKED_UNSPEC ^
961 BLIS_BITVAL_PACKED_COLUMNS ) );
962 }
963
bli_is_panel_packed(pack_t schema)964 BLIS_INLINE bool bli_is_panel_packed( pack_t schema )
965 {
966 return ( bool )
967 ( schema & BLIS_PACK_PANEL_BIT );
968 }
969
bli_is_4mi_packed(pack_t schema)970 BLIS_INLINE bool bli_is_4mi_packed( pack_t schema )
971 {
972 return ( bool )
973 ( ( schema & BLIS_PACK_FORMAT_BITS ) == BLIS_BITVAL_4MI );
974 }
975
bli_is_3mi_packed(pack_t schema)976 BLIS_INLINE bool bli_is_3mi_packed( pack_t schema )
977 {
978 return ( bool )
979 ( ( schema & BLIS_PACK_FORMAT_BITS ) == BLIS_BITVAL_3MI );
980 }
981
bli_is_3ms_packed(pack_t schema)982 BLIS_INLINE bool bli_is_3ms_packed( pack_t schema )
983 {
984 return ( bool )
985 ( ( schema & BLIS_PACK_FORMAT_BITS ) == BLIS_BITVAL_3MS );
986 }
987
bli_is_ro_packed(pack_t schema)988 BLIS_INLINE bool bli_is_ro_packed( pack_t schema )
989 {
990 return ( bool )
991 ( ( schema & BLIS_PACK_FORMAT_BITS ) == BLIS_BITVAL_RO );
992 }
993
bli_is_io_packed(pack_t schema)994 BLIS_INLINE bool bli_is_io_packed( pack_t schema )
995 {
996 return ( bool )
997 ( ( schema & BLIS_PACK_FORMAT_BITS ) == BLIS_BITVAL_IO );
998 }
999
bli_is_rpi_packed(pack_t schema)1000 BLIS_INLINE bool bli_is_rpi_packed( pack_t schema )
1001 {
1002 return ( bool )
1003 ( ( schema & BLIS_PACK_FORMAT_BITS ) == BLIS_BITVAL_RPI );
1004 }
1005
bli_is_rih_packed(pack_t schema)1006 BLIS_INLINE bool bli_is_rih_packed( pack_t schema )
1007 {
1008 return ( bool )
1009 ( bli_is_ro_packed( schema ) ||
1010 bli_is_io_packed( schema ) ||
1011 bli_is_rpi_packed( schema ) );
1012 }
1013
bli_is_1r_packed(pack_t schema)1014 BLIS_INLINE bool bli_is_1r_packed( pack_t schema )
1015 {
1016 return ( bool )
1017 ( ( schema & BLIS_PACK_FORMAT_BITS ) == BLIS_BITVAL_1R );
1018 }
1019
bli_is_1e_packed(pack_t schema)1020 BLIS_INLINE bool bli_is_1e_packed( pack_t schema )
1021 {
1022 return ( bool )
1023 ( ( schema & BLIS_PACK_FORMAT_BITS ) == BLIS_BITVAL_1E );
1024 }
1025
bli_is_1m_packed(pack_t schema)1026 BLIS_INLINE bool bli_is_1m_packed( pack_t schema )
1027 {
1028 return ( bool )
1029 ( bli_is_1r_packed( schema ) ||
1030 bli_is_1e_packed( schema ) );
1031 }
1032
bli_is_nat_packed(pack_t schema)1033 BLIS_INLINE bool bli_is_nat_packed( pack_t schema )
1034 {
1035 return ( bool )
1036 ( ( schema & BLIS_PACK_FORMAT_BITS ) == 0 );
1037 }
1038
bli_is_ind_packed(pack_t schema)1039 BLIS_INLINE bool bli_is_ind_packed( pack_t schema )
1040 {
1041 return ( bool )
1042 ( ( schema & BLIS_PACK_FORMAT_BITS ) != 0 );
1043 }
1044
bli_pack_schema_index(pack_t schema)1045 BLIS_INLINE guint_t bli_pack_schema_index( pack_t schema )
1046 {
1047 return ( guint_t )
1048 ( ( schema & BLIS_PACK_FORMAT_BITS ) >> BLIS_PACK_FORMAT_SHIFT );
1049 }
1050
1051
1052
1053 // pointer-related
1054
1055 // Increment a pointer by an integer fraction:
1056 // p0 + (num/dem)
1057 // where p0 is a pointer to a datatype of size sizeof_p0.
bli_ptr_inc_by_frac(void_fp p0,siz_t sizeof_p0,dim_t num,dim_t den)1058 BLIS_INLINE void_fp bli_ptr_inc_by_frac( void_fp p0, siz_t sizeof_p0, dim_t num, dim_t den )
1059 {
1060 return ( void_fp )
1061 ( ( char* )p0 + ( ( num * ( dim_t )sizeof_p0 ) / den ) );
1062 }
1063
1064
1065
1066 // Set dimensions, increments, effective uplo/diagoff, etc for ONE matrix
1067 // argument.
1068
bli_set_dims_incs_uplo_1m(doff_t diagoffa,diag_t diaga,uplo_t uploa,dim_t m,dim_t n,inc_t rs_a,inc_t cs_a,uplo_t * uplo_eff,dim_t * n_elem_max,dim_t * n_iter,inc_t * inca,inc_t * lda,dim_t * ij0,dim_t * n_shift)1069 BLIS_INLINE void bli_set_dims_incs_uplo_1m
1070 (
1071 doff_t diagoffa, diag_t diaga,
1072 uplo_t uploa, dim_t m, dim_t n, inc_t rs_a, inc_t cs_a,
1073 uplo_t* uplo_eff, dim_t* n_elem_max, dim_t* n_iter, inc_t* inca, inc_t* lda,
1074 dim_t* ij0, dim_t* n_shift
1075 )
1076 {
1077 // This is to prevent the compiler from warning about uninitialized
1078 // variables.
1079 *ij0 = 0;
1080 *n_shift = 0;
1081
1082 // If matrix A is entirely "unstored", that is, if either:
1083 // - A is lower-stored and entirely above the diagonal, or
1084 // - A is upper-stored and entirely below the diagonal
1085 // then we mark the storage as implicitly zero.
1086 if ( bli_is_unstored_subpart( diagoffa, BLIS_NO_TRANSPOSE, uploa, m, n ) )
1087 {
1088 *uplo_eff = BLIS_ZEROS;
1089 }
1090 else
1091 {
1092 doff_t diagoffa_use_ = diagoffa;
1093 doff_t diagoff_eff_;
1094 dim_t n_iter_max_;
1095
1096 if ( bli_is_unit_diag( diaga ) )
1097 bli_shift_diag_offset_to_shrink_uplo( uploa, &diagoffa_use_ );
1098
1099 // If matrix A is entirely "stored", that is, if either:
1100 // - A is upper-stored and entirely above the diagonal, or
1101 // - A is lower-stored and entirely below the diagonal
1102 // then we mark the storage as dense.
1103 if ( bli_is_stored_subpart( diagoffa_use_, BLIS_NO_TRANSPOSE, uploa, m, n ) )
1104 uploa = BLIS_DENSE;
1105
1106 n_iter_max_ = n;
1107 *n_elem_max = m;
1108 *inca = rs_a;
1109 *lda = cs_a;
1110 *uplo_eff = uploa;
1111 diagoff_eff_ = diagoffa_use_;
1112
1113 if ( bli_is_row_tilted( *n_elem_max, n_iter_max_, *inca, *lda ) )
1114 {
1115 bli_swap_dims( &n_iter_max_, n_elem_max );
1116 bli_swap_incs( inca, lda );
1117 bli_toggle_uplo( uplo_eff );
1118 bli_negate_diag_offset( &diagoff_eff_ );
1119 }
1120
1121 if ( bli_is_dense( *uplo_eff ) )
1122 {
1123 *n_iter = n_iter_max_;
1124 }
1125 else if ( bli_is_upper( *uplo_eff ) )
1126 {
1127 if ( diagoff_eff_ < 0 )
1128 {
1129 *ij0 = 0;
1130 *n_shift = -diagoff_eff_;
1131 *n_elem_max = bli_min( *n_elem_max, *n_shift + bli_min( m, n ) );
1132 *n_iter = n_iter_max_;
1133 }
1134 else
1135 {
1136 *ij0 = diagoff_eff_;
1137 *n_shift = 0;
1138 *n_iter = n_iter_max_ - diagoff_eff_;
1139 }
1140 }
1141 else // if ( bli_is_lower( *uplo_eff ) )
1142 {
1143 if ( diagoff_eff_ < 0 )
1144 {
1145 *ij0 = -diagoff_eff_;
1146 *n_shift = 0;
1147 *n_elem_max = *n_elem_max + diagoff_eff_;
1148 *n_iter = bli_min( *n_elem_max, bli_min( m, n ) );
1149 }
1150 else
1151 {
1152 *ij0 = 0;
1153 *n_shift = diagoff_eff_;
1154 *n_iter = bli_min( n_iter_max_, *n_shift + bli_min( m, n ) );
1155 }
1156 }
1157 }
1158 }
1159
1160 // Set dimensions, increments, effective uplo/diagoff, etc for ONE matrix
1161 // argument (without column-wise stride optimization).
1162
bli_set_dims_incs_uplo_1m_noswap(doff_t diagoffa,diag_t diaga,uplo_t uploa,dim_t m,dim_t n,inc_t rs_a,inc_t cs_a,uplo_t * uplo_eff,dim_t * n_elem_max,dim_t * n_iter,inc_t * inca,inc_t * lda,dim_t * ij0,dim_t * n_shift)1163 BLIS_INLINE void bli_set_dims_incs_uplo_1m_noswap
1164 (
1165 doff_t diagoffa, diag_t diaga,
1166 uplo_t uploa, dim_t m, dim_t n, inc_t rs_a, inc_t cs_a,
1167 uplo_t* uplo_eff, dim_t* n_elem_max, dim_t* n_iter, inc_t* inca, inc_t* lda,
1168 dim_t* ij0, dim_t* n_shift
1169 )
1170 {
1171 // This is to prevent the compiler from warning about uninitialized
1172 // variables.
1173 *ij0 = 0;
1174 *n_shift = 0;
1175
1176 // If matrix A is entirely "unstored", that is, if either:
1177 // - A is lower-stored and entirely above the diagonal, or
1178 // - A is upper-stored and entirely below the diagonal
1179 // then we mark the storage as implicitly zero.
1180 if ( bli_is_unstored_subpart( diagoffa, BLIS_NO_TRANSPOSE, uploa, m, n ) )
1181 {
1182 *uplo_eff = BLIS_ZEROS;
1183 }
1184 else
1185 {
1186 doff_t diagoffa_use_ = diagoffa;
1187 doff_t diagoff_eff_;
1188 dim_t n_iter_max_;
1189
1190 if ( bli_is_unit_diag( diaga ) )
1191 bli_shift_diag_offset_to_shrink_uplo( uploa, &diagoffa_use_ );
1192
1193 // If matrix A is entirely "stored", that is, if either:
1194 // - A is upper-stored and entirely above the diagonal, or
1195 // - A is lower-stored and entirely below the diagonal
1196 // then we mark the storage as dense.
1197 if ( bli_is_stored_subpart( diagoffa_use_, BLIS_NO_TRANSPOSE, uploa, m, n ) )
1198 uploa = BLIS_DENSE;
1199
1200 n_iter_max_ = n;
1201 *n_elem_max = m;
1202 *inca = rs_a;
1203 *lda = cs_a;
1204 *uplo_eff = uploa;
1205 diagoff_eff_ = diagoffa_use_;
1206
1207 if ( bli_is_dense( *uplo_eff ) )
1208 {
1209 *n_iter = n_iter_max_;
1210 }
1211 else if ( bli_is_upper( *uplo_eff ) )
1212 {
1213 if ( diagoff_eff_ < 0 )
1214 {
1215 *ij0 = 0;
1216 *n_shift = -diagoff_eff_;
1217 *n_elem_max = bli_min( *n_elem_max, *n_shift + bli_min( m, n ) );
1218 *n_iter = n_iter_max_;
1219 }
1220 else
1221 {
1222 *ij0 = diagoff_eff_;
1223 *n_shift = 0;
1224 *n_iter = n_iter_max_ - diagoff_eff_;
1225 }
1226 }
1227 else // if ( bli_is_lower( *uplo_eff ) )
1228 {
1229 if ( diagoff_eff_ < 0 )
1230 {
1231 *ij0 = -diagoff_eff_;
1232 *n_shift = 0;
1233 *n_elem_max = *n_elem_max + diagoff_eff_;
1234 *n_iter = bli_min( *n_elem_max, bli_min( m, n ) );
1235 }
1236 else
1237 {
1238 *ij0 = 0;
1239 *n_shift = diagoff_eff_;
1240 *n_iter = bli_min( n_iter_max_, *n_shift + bli_min( m, n ) );
1241 }
1242 }
1243 }
1244 }
1245
1246 // Set dimensions and increments for TWO matrix arguments.
1247
bli_set_dims_incs_2m(trans_t transa,dim_t m,dim_t n,inc_t rs_a,inc_t cs_a,inc_t rs_b,inc_t cs_b,dim_t * n_elem,dim_t * n_iter,inc_t * inca,inc_t * lda,inc_t * incb,inc_t * ldb)1248 BLIS_INLINE void bli_set_dims_incs_2m
1249 (
1250 trans_t transa,
1251 dim_t m, dim_t n, inc_t rs_a, inc_t cs_a,
1252 inc_t rs_b, inc_t cs_b,
1253 dim_t* n_elem, dim_t* n_iter, inc_t* inca, inc_t* lda,
1254 inc_t* incb, inc_t* ldb
1255 )
1256 {
1257 {
1258 *n_iter = n;
1259 *n_elem = m;
1260 *inca = rs_a;
1261 *lda = cs_a;
1262 *incb = rs_b;
1263 *ldb = cs_b;
1264
1265 if ( bli_does_trans( transa ) )
1266 {
1267 bli_swap_incs( inca, lda );
1268 }
1269
1270 if ( bli_is_row_tilted( *n_elem, *n_iter, *incb, *ldb ) &&
1271 bli_is_row_tilted( *n_elem, *n_iter, *inca, *lda ) )
1272 {
1273 bli_swap_dims( n_iter, n_elem );
1274 bli_swap_incs( inca, lda );
1275 bli_swap_incs( incb, ldb );
1276 }
1277 }
1278 }
1279
1280 // Set dimensions, increments, effective uplo/diagoff, etc for TWO matrix
1281 // arguments.
1282
bli_set_dims_incs_uplo_2m(doff_t diagoffa,diag_t diaga,trans_t transa,uplo_t uploa,dim_t m,dim_t n,inc_t rs_a,inc_t cs_a,inc_t rs_b,inc_t cs_b,uplo_t * uplo_eff,dim_t * n_elem_max,dim_t * n_iter,inc_t * inca,inc_t * lda,inc_t * incb,inc_t * ldb,dim_t * ij0,dim_t * n_shift)1283 BLIS_INLINE void bli_set_dims_incs_uplo_2m
1284 (
1285 doff_t diagoffa, diag_t diaga, trans_t transa,
1286 uplo_t uploa, dim_t m, dim_t n, inc_t rs_a, inc_t cs_a,
1287 inc_t rs_b, inc_t cs_b,
1288 uplo_t* uplo_eff, dim_t* n_elem_max, dim_t* n_iter, inc_t* inca, inc_t* lda,
1289 inc_t* incb, inc_t* ldb,
1290 dim_t* ij0, dim_t* n_shift
1291 )
1292 {
1293 // This is to prevent the compiler from warning about uninitialized
1294 // variables.
1295 *ij0 = 0;
1296 *n_shift = 0;
1297
1298 // If matrix A is entirely "unstored", that is, if either:
1299 // - A is lower-stored and entirely above the diagonal, or
1300 // - A is upper-stored and entirely below the diagonal
1301 // then we mark the storage as implicitly zero.
1302 if ( bli_is_unstored_subpart( diagoffa, transa, uploa, m, n ) )
1303 {
1304 *uplo_eff = BLIS_ZEROS;
1305 }
1306 else
1307 {
1308 doff_t diagoffa_use_ = diagoffa;
1309 doff_t diagoff_eff_;
1310 dim_t n_iter_max_;
1311
1312 if ( bli_is_unit_diag( diaga ) )
1313 bli_shift_diag_offset_to_shrink_uplo( uploa, &diagoffa_use_ );
1314
1315 // If matrix A is entirely "stored", that is, if either:
1316 // - A is upper-stored and entirely above the diagonal, or
1317 // - A is lower-stored and entirely below the diagonal
1318 // then we mark the storage as dense.
1319 if ( bli_is_stored_subpart( diagoffa_use_, transa, uploa, m, n ) )
1320 uploa = BLIS_DENSE;
1321
1322 n_iter_max_ = n;
1323 *n_elem_max = m;
1324 *inca = rs_a;
1325 *lda = cs_a;
1326 *incb = rs_b;
1327 *ldb = cs_b;
1328 *uplo_eff = uploa;
1329 diagoff_eff_ = diagoffa_use_;
1330
1331 if ( bli_does_trans( transa ) )
1332 {
1333 bli_swap_incs( inca, lda );
1334 bli_toggle_uplo( uplo_eff );
1335 bli_negate_diag_offset( &diagoff_eff_ );
1336 }
1337
1338 if ( bli_is_row_tilted( *n_elem_max, n_iter_max_, *incb, *ldb ) &&
1339 bli_is_row_tilted( *n_elem_max, n_iter_max_, *inca, *lda ) )
1340 {
1341 bli_swap_dims( &n_iter_max_, n_elem_max );
1342 bli_swap_incs( inca, lda );
1343 bli_swap_incs( incb, ldb );
1344 bli_toggle_uplo( uplo_eff );
1345 bli_negate_diag_offset( &diagoff_eff_ );
1346 }
1347
1348 if ( bli_is_dense( *uplo_eff ) )
1349 {
1350 *n_iter = n_iter_max_;
1351 }
1352 else if ( bli_is_upper( *uplo_eff ) )
1353 {
1354 if ( diagoff_eff_ < 0 )
1355 {
1356 *ij0 = 0;
1357 *n_shift = -diagoff_eff_;
1358 *n_elem_max = bli_min( *n_elem_max, *n_shift + bli_min( m, n ) );
1359 *n_iter = n_iter_max_;
1360 }
1361 else
1362 {
1363 *ij0 = diagoff_eff_;
1364 *n_shift = 0;
1365 *n_iter = n_iter_max_ - diagoff_eff_;
1366 }
1367 }
1368 else // if ( bli_is_lower( *uplo_eff ) )
1369 {
1370 if ( diagoff_eff_ < 0 )
1371 {
1372 *ij0 = -diagoff_eff_;
1373 *n_shift = 0;
1374 *n_elem_max = *n_elem_max + diagoff_eff_;
1375 *n_iter = bli_min( *n_elem_max, bli_min( m, n ) );
1376 }
1377 else
1378 {
1379 *ij0 = 0;
1380 *n_shift = diagoff_eff_;
1381 *n_iter = bli_min( n_iter_max_, *n_shift + bli_min( m, n ) );
1382 }
1383 }
1384 }
1385 }
1386
1387 // Set dimensions, increments, etc for ONE matrix argument when operating
1388 // on the diagonal.
1389
bli_set_dims_incs_1d(doff_t diagoffx,dim_t m,dim_t n,inc_t rs_x,inc_t cs_x,dim_t * offx,dim_t * n_elem,inc_t * incx)1390 BLIS_INLINE void bli_set_dims_incs_1d
1391 (
1392 doff_t diagoffx,
1393 dim_t m, dim_t n, inc_t rs_x, inc_t cs_x,
1394 dim_t* offx, dim_t* n_elem, inc_t* incx
1395 )
1396 {
1397 if ( diagoffx < 0 )
1398 {
1399 *n_elem = bli_min( m - ( dim_t )(-diagoffx), n );
1400 *offx = ( dim_t )(-diagoffx) * rs_x;
1401 }
1402 else
1403 {
1404 *n_elem = bli_min( n - ( dim_t )( diagoffx), m );
1405 *offx = ( dim_t )( diagoffx) * cs_x;
1406 }
1407
1408 *incx = rs_x + cs_x; \
1409 }
1410
1411 // Set dimensions, increments, etc for TWO matrix arguments when operating
1412 // on diagonals.
bli_set_dims_incs_2d(doff_t diagoffx,trans_t transx,dim_t m,dim_t n,inc_t rs_x,inc_t cs_x,inc_t rs_y,inc_t cs_y,dim_t * offx,dim_t * offy,dim_t * n_elem,inc_t * incx,inc_t * incy)1413 BLIS_INLINE void bli_set_dims_incs_2d
1414 (
1415 doff_t diagoffx, trans_t transx,
1416 dim_t m, dim_t n, inc_t rs_x, inc_t cs_x,
1417 inc_t rs_y, inc_t cs_y,
1418 dim_t* offx, dim_t* offy, dim_t* n_elem,
1419 inc_t* incx, inc_t* incy
1420 )
1421 {
1422 doff_t diagoffy_ = bli_diag_offset_with_trans( transx, diagoffx );
1423
1424 if ( diagoffx < 0 ) *offx = -diagoffx * rs_x;
1425 else *offx = diagoffx * cs_x;
1426
1427 if ( diagoffy_ < 0 )
1428 {
1429 *n_elem = bli_min( m - ( dim_t )(-diagoffy_), n );
1430 *offy = -diagoffy_ * rs_y;
1431 }
1432 else
1433 {
1434 *n_elem = bli_min( n - ( dim_t )( diagoffy_), m );
1435 *offy = diagoffy_ * cs_y;
1436 }
1437
1438 *incx = rs_x + cs_x;
1439 *incy = rs_y + cs_y;
1440 }
1441
1442
1443 #endif
1444