1 /*********************************************************************/
2 /* Copyright 2009, 2010 The University of Texas at Austin. */
3 /* All rights reserved. */
4 /* */
5 /* Redistribution and use in source and binary forms, with or */
6 /* without modification, are permitted provided that the following */
7 /* conditions are met: */
8 /* */
9 /* 1. Redistributions of source code must retain the above */
10 /* copyright notice, this list of conditions and the following */
11 /* disclaimer. */
12 /* */
13 /* 2. Redistributions in binary form must reproduce the above */
14 /* copyright notice, this list of conditions and the following */
15 /* disclaimer in the documentation and/or other materials */
16 /* provided with the distribution. */
17 /* */
18 /* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
19 /* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
20 /* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
21 /* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
22 /* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
23 /* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
24 /* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
25 /* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
26 /* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
27 /* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
28 /* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
29 /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
30 /* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
31 /* POSSIBILITY OF SUCH DAMAGE. */
32 /* */
33 /* The views and conclusions contained in the software and */
34 /* documentation are those of the authors and should not be */
35 /* interpreted as representing official policies, either expressed */
36 /* or implied, of The University of Texas at Austin. */
37 /*********************************************************************/
38
39 #ifndef COMMON_X86
40 #define COMMON_X86
41
42 #ifndef ASSEMBLER
43
44 #ifdef C_MSVC
45 #include <intrin.h>
46 #endif
47
48 #ifdef C_SUN
49 #define __asm__ __asm
50 #define __volatile__
51 #endif
52
53 /*
54 #ifdef HAVE_SSE2
55 #define MB __asm__ __volatile__ ("mfence");
56 #define WMB __asm__ __volatile__ ("sfence");
57 #else
58 #define MB
59 #define WMB
60 #endif
61 */
62
63 #ifdef __GNUC__
64 #define MB do { __asm__ __volatile__("": : :"memory"); } while (0)
65 #define WMB do { __asm__ __volatile__("": : :"memory"); } while (0)
66 #define RMB
67 #else
68 #define MB do {} while (0)
69 #define WMB do {} while (0)
70 #define RMB
71 #endif
72
blas_lock(volatile BLASULONG * address)73 static void __inline blas_lock(volatile BLASULONG *address){
74
75
76 #ifndef C_MSVC
77 int ret;
78 #else
79 BLASULONG ret;
80 #endif
81
82 do {
83 while (*address) {YIELDING;}
84
85 #ifndef C_MSVC
86 __asm__ __volatile__(
87 "xchgl %0, %1\n"
88 : "=r"(ret), "=m"(*address)
89 : "0"(1), "m"(*address)
90 : "memory");
91 #else
92 ret=InterlockedExchange64((volatile LONG64 *)(address), 1);
93 #endif
94 } while (ret);
95
96 }
97 #define BLAS_LOCK_DEFINED
98
rpcc(void)99 static __inline BLASULONG rpcc(void){
100 #ifdef C_MSVC
101 return __rdtsc();
102 #else
103 BLASULONG a, d;
104
105 __asm__ __volatile__ ("rdtsc" : "=a" (a), "=d" (d));
106
107 return ((BLASULONG)a + ((BLASULONG)d << 32));
108 #endif
109 }
110 #define RPCC_DEFINED
111
112 #define RPCC64BIT
113
114 #ifndef C_MSVC
getstackaddr(void)115 static __inline BLASULONG getstackaddr(void){
116 BLASULONG addr;
117
118 __asm__ __volatile__ ("movq %%rsp, %0"
119 : "=r"(addr) : : "memory");
120
121 return addr;
122 }
123 #endif
124
cpuid(int op,int * eax,int * ebx,int * ecx,int * edx)125 static __inline void cpuid(int op, int *eax, int *ebx, int *ecx, int *edx){
126
127 #ifdef C_MSVC
128 int cpuinfo[4];
129 __cpuid(cpuinfo, op);
130 *eax=cpuinfo[0];
131 *ebx=cpuinfo[1];
132 *ecx=cpuinfo[2];
133 *edx=cpuinfo[3];
134 #else
135 __asm__ __volatile__("mov $0, %%ecx;"
136 "cpuid"
137 : "=a" (*eax),
138 "=b" (*ebx),
139 "=c" (*ecx),
140 "=d" (*edx)
141 : "0" (op));
142 #endif
143 }
144
cpuid_count(int op,int count,int * eax,int * ebx,int * ecx,int * edx)145 static __inline void cpuid_count(int op, int count, int *eax, int *ebx, int *ecx, int *edx)
146 {
147 #ifdef C_MSVC
148 int cpuInfo[4] = {-1};
149 __cpuidex(cpuInfo, op, count);
150 *eax = cpuInfo[0];
151 *ebx = cpuInfo[1];
152 *ecx = cpuInfo[2];
153 *edx = cpuInfo[3];
154 #else
155 #if defined(__i386__) && defined(__PIC__)
156 __asm__ __volatile__
157 ("mov %%ebx, %%edi;"
158 "cpuid;"
159 "xchgl %%ebx, %%edi;"
160 : "=a" (*eax), "=D" (*ebx), "=c" (*ecx), "=d" (*edx) : "0" (op), "2" (count) : "cc");
161 #else
162 __asm__ __volatile__
163 ("cpuid": "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) : "0" (op), "2" (count) : "cc");
164 #endif
165 #endif
166 }
167
168 /*
169 #define WHEREAMI
170 */
171
WhereAmI(void)172 static __inline int WhereAmI(void){
173 int eax, ebx, ecx, edx;
174 int apicid;
175
176 cpuid(1, &eax, &ebx, &ecx, &edx);
177 apicid = BITMASK(ebx, 24, 0xff);
178
179 return apicid;
180 }
181
182
183 #ifdef CORE_BARCELONA
184 #define IFLUSH gotoblas_iflush()
185 #define IFLUSH_HALF gotoblas_iflush_half()
186 #endif
187
188 #ifdef ENABLE_SSE_EXCEPTION
189
190 #define IDEBUG_START \
191 { \
192 unsigned int fp_sse_mode, new_fp_mode; \
193 __asm__ __volatile__ ("stmxcsr %0" : "=m" (fp_sse_mode) : ); \
194 new_fp_mode = fp_sse_mode & ~0xd00; \
195 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (new_fp_mode) );
196
197 #define IDEBUG_END \
198 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (fp_sse_mode) ); \
199 }
200
201 #endif
202
203 #ifdef XDOUBLE
204 #define GET_IMAGE(res) __asm__ __volatile__("fstpt %0" : "=m"(res) : : "memory")
205 #elif defined(DOUBLE)
206 #define GET_IMAGE(res) __asm__ __volatile__("movsd %%xmm1, %0" : "=m"(res) : : "memory")
207 #else
208 #define GET_IMAGE(res) __asm__ __volatile__("movss %%xmm1, %0" : "=m"(res) : : "memory")
209 #endif
210
211 #define GET_IMAGE_CANCEL
212
213 #ifdef SMP
214 #if defined(USE64BITINT)
blas_quickdivide(blasint x,blasint y)215 static __inline blasint blas_quickdivide(blasint x, blasint y){
216 return x / y;
217 }
218 #elif defined (C_MSVC)
blas_quickdivide(BLASLONG x,BLASLONG y)219 static __inline BLASLONG blas_quickdivide(BLASLONG x, BLASLONG y){
220 return x / y;
221 }
222 #else
223 extern unsigned int blas_quick_divide_table[];
224
blas_quickdivide(unsigned int x,unsigned int y)225 static __inline unsigned int blas_quickdivide(unsigned int x, unsigned int y){
226
227 volatile unsigned int result;
228
229 if (y <= 1) return x;
230
231 #if (MAX_CPU_NUMBER > 64)
232 if (y > 64) {
233 result = x / y;
234 return result;
235 }
236 #endif
237
238 y = blas_quick_divide_table[y];
239
240 __asm__ __volatile__ ("mull %0" :"=d" (result), "+a"(x) : "0" (y));
241 return result;
242 }
243 #endif
244 #endif
245
246 #endif
247
248 #ifndef PAGESIZE
249 #define PAGESIZE ( 4 << 10)
250 #endif
251 #define HUGE_PAGESIZE ( 2 << 20)
252
253 #ifndef BUFFERSIZE
254 #define BUFFER_SIZE (32 << 22)
255 #else
256 #define BUFFER_SIZE (32 << BUFFERSIZE)
257 #endif
258
259 #define SEEK_ADDRESS
260
261 #ifdef F_INTERFACE_G77
262 #define RETURN_BY_STACK
263 #define NEED_F2CCONV
264 #endif
265
266 #ifdef F_INTERFACE_G95
267 #define RETURN_BY_PACKED
268 #endif
269
270 #ifdef F_INTERFACE_GFORT
271 #ifdef OS_WINDOWS
272 #ifndef DOUBLE
273 #define RETURN_BY_REGS
274 #else
275 #define RETURN_BY_STACK
276 #endif
277 #else
278 #define RETURN_BY_PACKED
279 #endif
280 #endif
281
282 #ifdef F_INTERFACE_INTEL
283 #define RETURN_BY_STACK
284 #endif
285
286 #ifdef F_INTERFACE_FUJITSU
287 #define RETURN_BY_STACK
288 #endif
289
290 #ifdef F_INTERFACE_FLANG
291 #define RETURN_BY_STACK
292 #endif
293
294 #ifdef F_INTERFACE_PGI
295 #define RETURN_BY_STACK
296 #endif
297
298 #ifdef F_INTERFACE_PATHSCALE
299 #define RETURN_BY_PACKED
300 #endif
301
302 #ifdef F_INTERFACE_SUN
303 #define RETURN_BY_PACKED
304 #endif
305
306 #ifdef ASSEMBLER
307
308 #if defined(PILEDRIVER) || defined(BULLDOZER) || defined(STEAMROLLER) || defined(EXCAVATOR)
309 //Enable some optimization for barcelona.
310 #define BARCELONA_OPTIMIZATION
311 #endif
312
313 #if defined(HAVE_3DNOW)
314 #define EMMS femms
315 #elif defined(HAVE_MMX)
316 #define EMMS emms
317 #endif
318
319 #ifndef EMMS
320 #define EMMS
321 #endif
322
323 #define BRANCH .byte 0x3e
324 #define NOBRANCH .byte 0x2e
325 #define PADDING .byte 0x66
326
327 #ifdef OS_WINDOWS
328 #define ARG1 %rcx
329 #define ARG2 %rdx
330 #define ARG3 %r8
331 #define ARG4 %r9
332 #else
333 #define ARG1 %rdi
334 #define ARG2 %rsi
335 #define ARG3 %rdx
336 #define ARG4 %rcx
337 #define ARG5 %r8
338 #define ARG6 %r9
339 #endif
340
341 #ifndef COMPLEX
342 #ifdef XDOUBLE
343 #define LOCAL_BUFFER_SIZE QLOCAL_BUFFER_SIZE
344 #elif defined DOUBLE
345 #define LOCAL_BUFFER_SIZE DLOCAL_BUFFER_SIZE
346 #else
347 #define LOCAL_BUFFER_SIZE SLOCAL_BUFFER_SIZE
348 #endif
349 #else
350 #ifdef XDOUBLE
351 #define LOCAL_BUFFER_SIZE XLOCAL_BUFFER_SIZE
352 #elif defined DOUBLE
353 #define LOCAL_BUFFER_SIZE ZLOCAL_BUFFER_SIZE
354 #else
355 #define LOCAL_BUFFER_SIZE CLOCAL_BUFFER_SIZE
356 #endif
357 #endif
358
359 #if defined(OS_WINDOWS)
360 #if LOCAL_BUFFER_SIZE > 16384
361 #define STACK_TOUCHING \
362 movl $0, 4096 * 4(%rsp);\
363 movl $0, 4096 * 3(%rsp);\
364 movl $0, 4096 * 2(%rsp);\
365 movl $0, 4096 * 1(%rsp);
366 #elif LOCAL_BUFFER_SIZE > 12288
367 #define STACK_TOUCHING \
368 movl $0, 4096 * 3(%rsp);\
369 movl $0, 4096 * 2(%rsp);\
370 movl $0, 4096 * 1(%rsp);
371 #elif LOCAL_BUFFER_SIZE > 8192
372 #define STACK_TOUCHING \
373 movl $0, 4096 * 2(%rsp);\
374 movl $0, 4096 * 1(%rsp);
375 #elif LOCAL_BUFFER_SIZE > 4096
376 #define STACK_TOUCHING \
377 movl $0, 4096 * 1(%rsp);
378 #else
379 #define STACK_TOUCHING
380 #endif
381 #else
382 #define STACK_TOUCHING
383 #endif
384
385 #if defined(CORE2)
386 #define movapd movaps
387 #define andpd andps
388 #define movlpd movlps
389 #define movhpd movhps
390 #endif
391
392 #ifndef F_INTERFACE
393 #define REALNAME ASMNAME
394 #else
395 #define REALNAME ASMFNAME
396 #endif
397
398 #ifdef OS_DARWIN
399 #define PROLOGUE .text;.align 5; .globl REALNAME; REALNAME:
400 #define EPILOGUE .subsections_via_symbols
401 #define PROFCODE
402 #endif
403
404 #ifdef OS_WINDOWS
405 #define SAVEREGISTERS \
406 subq $256, %rsp;\
407 movups %xmm6, 0(%rsp);\
408 movups %xmm7, 16(%rsp);\
409 movups %xmm8, 32(%rsp);\
410 movups %xmm9, 48(%rsp);\
411 movups %xmm10, 64(%rsp);\
412 movups %xmm11, 80(%rsp);\
413 movups %xmm12, 96(%rsp);\
414 movups %xmm13, 112(%rsp);\
415 movups %xmm14, 128(%rsp);\
416 movups %xmm15, 144(%rsp)
417
418 #define RESTOREREGISTERS \
419 movups 0(%rsp), %xmm6;\
420 movups 16(%rsp), %xmm7;\
421 movups 32(%rsp), %xmm8;\
422 movups 48(%rsp), %xmm9;\
423 movups 64(%rsp), %xmm10;\
424 movups 80(%rsp), %xmm11;\
425 movups 96(%rsp), %xmm12;\
426 movups 112(%rsp), %xmm13;\
427 movups 128(%rsp), %xmm14;\
428 movups 144(%rsp), %xmm15;\
429 addq $256, %rsp
430 #else
431 #define SAVEREGISTERS
432 #define RESTOREREGISTERS
433 #endif
434
435 #if defined(OS_WINDOWS) && !defined(C_PGI)
436 #define PROLOGUE \
437 .text; \
438 .align 16; \
439 .globl REALNAME ;\
440 .def REALNAME;.scl 2;.type 32;.endef; \
441 REALNAME:
442
443 #define PROFCODE
444
445 #define EPILOGUE .end
446 #endif
447
448 #if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLY) || defined(__ELF__) || defined(C_PGI)
449 #define PROLOGUE \
450 .text; \
451 .align 512; \
452 .globl REALNAME ;\
453 .type REALNAME, @function; \
454 REALNAME: \
455 _CET_ENDBR
456
457 #ifdef PROFILE
458 #define PROFCODE call *mcount@GOTPCREL(%rip)
459 #else
460 #define PROFCODE
461 #endif
462
463 #define EPILOGUE \
464 .size REALNAME, .-REALNAME; \
465 .section .note.GNU-stack,"",@progbits
466
467
468 #endif
469
470 #endif
471
472 #ifdef XDOUBLE
473 #define FLD fldt
474 #define FST fstpt
475 #define MOVQ movq
476 #elif defined(DOUBLE)
477 #define FLD fldl
478 #define FST fstpl
479 #define FSTU fstl
480 #define FMUL fmull
481 #define FADD faddl
482 #define MOVSD movsd
483 #define MULSD mulsd
484 #define MULPD mulpd
485 #define CMPEQPD cmpeqpd
486 #define COMISD comisd
487 #define PSRLQ psrlq
488 #define ANDPD andpd
489 #define ADDPD addpd
490 #define ADDSD addsd
491 #define SUBPD subpd
492 #define SUBSD subsd
493 #define MOVQ movq
494 #define MOVUPD movupd
495 #define XORPD xorpd
496 #else
497 #define FLD flds
498 #define FST fstps
499 #define FSTU fsts
500 #define FMUL fmuls
501 #define FADD fadds
502 #define MOVSD movss
503 #define MULSD mulss
504 #define MULPD mulps
505 #define CMPEQPD cmpeqps
506 #define COMISD comiss
507 #define PSRLQ psrld
508 #define ANDPD andps
509 #define ADDPD addps
510 #define ADDSD addss
511 #define SUBPD subps
512 #define SUBSD subss
513 #define MOVQ movd
514 #define MOVUPD movups
515 #define XORPD xorps
516 #endif
517
518 #define HALT hlt
519
520 #ifdef OS_DARWIN
521 #define ALIGN_2 .align 2
522 #define ALIGN_3 .align 3
523 #define ALIGN_4 .align 4
524 #define ALIGN_5 .align 5
525 #define ffreep fstp
526 #endif
527
528 #ifndef ALIGN_2
529 #define ALIGN_2 .align 4
530 #endif
531
532 #ifndef ALIGN_3
533 #define ALIGN_3 .align 8
534 #endif
535
536 #ifndef ALIGN_4
537 #define ALIGN_4 .align 16
538 #endif
539
540 #ifndef ALIGN_5
541 #define ALIGN_5 .align 32
542 #endif
543
544 #ifndef ALIGN_6
545 #define ALIGN_6 .align 64
546 #endif
547
548 // ffreep %st(0).
549 // Because Clang didn't support ffreep, we directly use the opcode.
550 // Please check out http://www.sandpile.org/x86/opc_fpu.htm
551 #ifndef ffreep
552 #define ffreep .byte 0xdf, 0xc0 #
553 #endif
554 #endif
555