1 /*********************************************************************/
2 /* Copyright 2009, 2010 The University of Texas at Austin. */
3 /* All rights reserved. */
4 /* */
5 /* Redistribution and use in source and binary forms, with or */
6 /* without modification, are permitted provided that the following */
7 /* conditions are met: */
8 /* */
9 /* 1. Redistributions of source code must retain the above */
10 /* copyright notice, this list of conditions and the following */
11 /* disclaimer. */
12 /* */
13 /* 2. Redistributions in binary form must reproduce the above */
14 /* copyright notice, this list of conditions and the following */
15 /* disclaimer in the documentation and/or other materials */
16 /* provided with the distribution. */
17 /* */
18 /* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
19 /* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
20 /* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
21 /* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
22 /* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
23 /* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
24 /* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
25 /* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
26 /* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
27 /* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
28 /* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
29 /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
30 /* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
31 /* POSSIBILITY OF SUCH DAMAGE. */
32 /* */
33 /* The views and conclusions contained in the software and */
34 /* documentation are those of the authors and should not be */
35 /* interpreted as representing official policies, either expressed */
36 /* or implied, of The University of Texas at Austin. */
37 /*********************************************************************/
38
39 #ifndef COMMON_X86
40 #define COMMON_X86
41
42 #ifndef ASSEMBLER
43
44 #ifdef C_SUN
45 #define __asm__ __asm
46 #define __volatile__
47 #endif
48
49 #ifdef HAVE_SSE2
50 #define MB __asm__ __volatile__ ("mfence");
51 #define WMB __asm__ __volatile__ ("sfence");
52 #else
53 #define MB
54 #define WMB
55 #endif
56
blas_lock(volatile BLASULONG * address)57 static void __inline blas_lock(volatile BLASULONG *address){
58
59 int ret;
60
61 do {
62 while (*address) {YIELDING;};
63
64 __asm__ __volatile__(
65 "xchgl %0, %1\n"
66 : "=r"(ret), "=m"(*address)
67 : "0"(1), "m"(*address)
68 : "memory");
69
70 } while (ret);
71 }
72
rpcc(void)73 static __inline BLASULONG rpcc(void){
74 BLASULONG a, d;
75
76 __asm__ __volatile__ ("rdtsc" : "=a" (a), "=d" (d));
77
78 return ((BLASULONG)a + ((BLASULONG)d << 32));
79 }
80
81 #define RPCC64BIT
82
getstackaddr(void)83 static __inline BLASULONG getstackaddr(void){
84 BLASULONG addr;
85
86 __asm__ __volatile__ ("movq %%rsp, %0"
87 : "=r"(addr) : : "memory");
88
89 return addr;
90 }
91
cpuid(int op,int * eax,int * ebx,int * ecx,int * edx)92 static __inline void cpuid(int op, int *eax, int *ebx, int *ecx, int *edx){
93
94 __asm__ __volatile__("cpuid"
95 : "=a" (*eax),
96 "=b" (*ebx),
97 "=c" (*ecx),
98 "=d" (*edx)
99 : "0" (op));
100 }
101
102 #define WHEREAMI
103
WhereAmI(void)104 static inline int WhereAmI(void){
105 int eax, ebx, ecx, edx;
106 int apicid;
107
108 cpuid(1, &eax, &ebx, &ecx, &edx);
109 apicid = BITMASK(ebx, 24, 0xff);
110
111 return apicid;
112 }
113
114 #ifdef CORE_BARCELONA
115 #define IFLUSH gotoblas_iflush()
116 #define IFLUSH_HALF gotoblas_iflush_half()
117 #endif
118
119 #ifdef ENABLE_SSE_EXCEPTION
120
121 #define IDEBUG_START \
122 { \
123 unsigned int fp_sse_mode, new_fp_mode; \
124 __asm__ __volatile__ ("stmxcsr %0" : "=m" (fp_sse_mode) : ); \
125 new_fp_mode = fp_sse_mode & ~0xd00; \
126 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (new_fp_mode) );
127
128 #define IDEBUG_END \
129 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (fp_sse_mode) ); \
130 }
131
132 #endif
133
134 #ifdef XDOUBLE
135 #define GET_IMAGE(res) __asm__ __volatile__("fstpt %0" : "=m"(res) : : "memory")
136 #elif defined(DOUBLE)
137 #define GET_IMAGE(res) __asm__ __volatile__("movsd %%xmm1, %0" : "=m"(res) : : "memory")
138 #else
139 #define GET_IMAGE(res) __asm__ __volatile__("movss %%xmm1, %0" : "=m"(res) : : "memory")
140 #endif
141
142 #define GET_IMAGE_CANCEL
143
144 #ifdef SMP
145 #ifdef USE64BITINT
blas_quickdivide(blasint x,blasint y)146 static __inline blasint blas_quickdivide(blasint x, blasint y){
147 return x / y;
148 }
149 #else
150 extern unsigned int blas_quick_divide_table[];
151
blas_quickdivide(unsigned int x,unsigned int y)152 static __inline int blas_quickdivide(unsigned int x, unsigned int y){
153
154 unsigned int result;
155
156 if (y <= 1) return x;
157
158 y = blas_quick_divide_table[y];
159
160 __asm__ __volatile__ ("mull %0" :"=d" (result) :"a"(x), "0" (y));
161
162 return result;
163 }
164 #endif
165 #endif
166
167 #endif
168
169 #ifndef PAGESIZE
170 #define PAGESIZE ( 4 << 10)
171 #endif
172 #define HUGE_PAGESIZE ( 2 << 20)
173
174 #define BUFFER_SIZE (32 << 20)
175
176 #define SEEK_ADDRESS
177
178 #ifdef F_INTERFACE_G77
179 #define RETURN_BY_STACK
180 #define NEED_F2CCONV
181 #endif
182
183 #ifdef F_INTERFACE_G95
184 #define RETURN_BY_PACKED
185 #endif
186
187 #ifdef F_INTERFACE_GFORT
188 #ifdef OS_WINDOWS
189 #ifndef DOUBLE
190 #define RETURN_BY_REGS
191 #else
192 #define RETURN_BY_STACK
193 #endif
194 #else
195 #define RETURN_BY_PACKED
196 #endif
197 #endif
198
199 #ifdef F_INTERFACE_INTEL
200 #define RETURN_BY_STACK
201 #endif
202
203 #ifdef F_INTERFACE_FUJITSU
204 #define RETURN_BY_STACK
205 #endif
206
207 #ifdef F_INTERFACE_PGI
208 #define RETURN_BY_STACK
209 #endif
210
211 #ifdef F_INTERFACE_PATHSCALE
212 #define RETURN_BY_PACKED
213 #endif
214
215 #ifdef F_INTERFACE_SUN
216 #define RETURN_BY_PACKED
217 #endif
218
219 #ifdef ASSEMBLER
220
221 #if defined(HAVE_3DNOW)
222 #define EMMS femms
223 #elif defined(HAVE_MMX)
224 #define EMMS emms
225 #endif
226
227 #ifndef EMMS
228 #define EMMS
229 #endif
230
231 #define BRANCH .byte 0x3e
232 #define NOBRANCH .byte 0x2e
233 #define PADDING .byte 0x66
234
235 #ifdef OS_WINDOWS
236 #define ARG1 %rcx
237 #define ARG2 %rdx
238 #define ARG3 %r8
239 #define ARG4 %r9
240 #else
241 #define ARG1 %rdi
242 #define ARG2 %rsi
243 #define ARG3 %rdx
244 #define ARG4 %rcx
245 #define ARG5 %r8
246 #define ARG6 %r9
247 #endif
248
249 #ifndef COMPLEX
250 #ifdef XDOUBLE
251 #define LOCAL_BUFFER_SIZE QLOCAL_BUFFER_SIZE
252 #elif defined DOUBLE
253 #define LOCAL_BUFFER_SIZE DLOCAL_BUFFER_SIZE
254 #else
255 #define LOCAL_BUFFER_SIZE SLOCAL_BUFFER_SIZE
256 #endif
257 #else
258 #ifdef XDOUBLE
259 #define LOCAL_BUFFER_SIZE XLOCAL_BUFFER_SIZE
260 #elif defined DOUBLE
261 #define LOCAL_BUFFER_SIZE ZLOCAL_BUFFER_SIZE
262 #else
263 #define LOCAL_BUFFER_SIZE CLOCAL_BUFFER_SIZE
264 #endif
265 #endif
266
267 #if defined(OS_WINDOWS)
268 #if LOCAL_BUFFER_SIZE > 16384
269 #define STACK_TOUCHING \
270 movl $0, 4096 * 4(%rsp);\
271 movl $0, 4096 * 3(%rsp);\
272 movl $0, 4096 * 2(%rsp);\
273 movl $0, 4096 * 1(%rsp);
274 #elif LOCAL_BUFFER_SIZE > 12288
275 #define STACK_TOUCHING \
276 movl $0, 4096 * 3(%rsp);\
277 movl $0, 4096 * 2(%rsp);\
278 movl $0, 4096 * 1(%rsp);
279 #elif LOCAL_BUFFER_SIZE > 8192
280 #define STACK_TOUCHING \
281 movl $0, 4096 * 2(%rsp);\
282 movl $0, 4096 * 1(%rsp);
283 #elif LOCAL_BUFFER_SIZE > 4096
284 #define STACK_TOUCHING \
285 movl $0, 4096 * 1(%rsp);
286 #else
287 #define STACK_TOUCHING
288 #endif
289 #else
290 #define STACK_TOUCHING
291 #endif
292
293 #if defined(CORE2)
294 #define movapd movaps
295 #define andpd andps
296 #define movlpd movlps
297 #define movhpd movhps
298 #endif
299
300 #ifndef F_INTERFACE
301 #define REALNAME ASMNAME
302 #else
303 #define REALNAME ASMFNAME
304 #endif
305
306 #ifdef OS_DARWIN
307 #define PROLOGUE .text;.align 5; .globl REALNAME; REALNAME:
308 #define EPILOGUE .subsections_via_symbols
309 #define PROFCODE
310 #endif
311
312 #ifdef OS_WINDOWS
313 #define SAVEREGISTERS \
314 subq $256, %rsp;\
315 movups %xmm6, 0(%rsp);\
316 movups %xmm7, 16(%rsp);\
317 movups %xmm8, 32(%rsp);\
318 movups %xmm9, 48(%rsp);\
319 movups %xmm10, 64(%rsp);\
320 movups %xmm11, 80(%rsp);\
321 movups %xmm12, 96(%rsp);\
322 movups %xmm13, 112(%rsp);\
323 movups %xmm14, 128(%rsp);\
324 movups %xmm15, 144(%rsp)
325
326 #define RESTOREREGISTERS \
327 movups 0(%rsp), %xmm6;\
328 movups 16(%rsp), %xmm7;\
329 movups 32(%rsp), %xmm8;\
330 movups 48(%rsp), %xmm9;\
331 movups 64(%rsp), %xmm10;\
332 movups 80(%rsp), %xmm11;\
333 movups 96(%rsp), %xmm12;\
334 movups 112(%rsp), %xmm13;\
335 movups 128(%rsp), %xmm14;\
336 movups 144(%rsp), %xmm15;\
337 addq $256, %rsp
338 #else
339 #define SAVEREGISTERS
340 #define RESTOREREGISTERS
341 #endif
342
343 #if defined(OS_WINDOWS) && !defined(C_PGI)
344 #define PROLOGUE \
345 .text; \
346 .align 16; \
347 .globl REALNAME ;\
348 .def REALNAME;.scl 2;.type 32;.endef; \
349 REALNAME:
350
351 #define PROFCODE
352
353 #define EPILOGUE .end REALNAME
354 #endif
355
356 #if defined(OS_LINUX) || defined(OS_FreeBSD) || defined(OS_NetBSD) || defined(__ELF__) || defined(C_PGI)
357 #define PROLOGUE \
358 .text; \
359 .align 512; \
360 .globl REALNAME ;\
361 .type REALNAME, @function; \
362 REALNAME:
363
364 #ifdef PROFILE
365 #define PROFCODE call *mcount@GOTPCREL(%rip)
366 #else
367 #define PROFCODE
368 #endif
369
370 #define EPILOGUE .size REALNAME, .-REALNAME
371
372 #endif
373
374 #endif
375
376 #ifdef XDOUBLE
377 #define FLD fldt
378 #define FST fstpt
379 #define MOVQ movq
380 #elif defined(DOUBLE)
381 #define FLD fldl
382 #define FST fstpl
383 #define FSTU fstl
384 #define FMUL fmull
385 #define FADD faddl
386 #define MOVSD movsd
387 #define MULSD mulsd
388 #define MULPD mulpd
389 #define CMPEQPD cmpeqpd
390 #define COMISD comisd
391 #define PSRLQ psrlq
392 #define ANDPD andpd
393 #define ADDPD addpd
394 #define ADDSD addsd
395 #define SUBPD subpd
396 #define SUBSD subsd
397 #define MOVQ movq
398 #define MOVUPD movupd
399 #define XORPD xorpd
400 #else
401 #define FLD flds
402 #define FST fstps
403 #define FSTU fsts
404 #define FMUL fmuls
405 #define FADD fadds
406 #define MOVSD movss
407 #define MULSD mulss
408 #define MULPD mulps
409 #define CMPEQPD cmpeqps
410 #define COMISD comiss
411 #define PSRLQ psrld
412 #define ANDPD andps
413 #define ADDPD addps
414 #define ADDSD addss
415 #define SUBPD subps
416 #define SUBSD subss
417 #define MOVQ movd
418 #define MOVUPD movups
419 #define XORPD xorps
420 #endif
421
422 #define HALT hlt
423
424 #ifdef OS_DARWIN
425 #define ALIGN_2 .align 2
426 #define ALIGN_3 .align 3
427 #define ALIGN_4 .align 4
428 #define ffreep fstp
429 #endif
430
431 #ifndef ALIGN_2
432 #define ALIGN_2 .align 4
433 #endif
434
435 #ifndef ALIGN_3
436 #define ALIGN_3 .align 8
437 #endif
438
439 #ifndef ALIGN_4
440 #define ALIGN_4 .align 16
441 #endif
442
443 #ifndef ALIGN_5
444 #define ALIGN_5 .align 32
445 #endif
446
447 #ifndef ALIGN_6
448 #define ALIGN_6 .align 64
449 #endif
450
451 #endif
452