1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2005 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * $COPYRIGHT$
13 *
14 * Additional copyrights may follow
15 *
16 * $HEADER$
17 */
18
19 #ifndef OPAL_SYS_ARCH_ATOMIC_H
20 #define OPAL_SYS_ARCH_ATOMIC_H 1
21
22
23 /* BWB - FIX ME! */
24 #ifdef __linux__
25 #define MB() __asm__ __volatile__(".set mips2; sync; .set mips0": : :"memory")
26 #define RMB() __asm__ __volatile__(".set mips2; sync; .set mips0": : :"memory")
27 #define WMB() __asm__ __volatile__(".set mips2; sync; .set mips0": : :"memory")
28 #define SMP_SYNC ".set mips2; sync; .set mips0"
29 #else
30 #define MB() __asm__ __volatile__("sync": : :"memory")
31 #define RMB() __asm__ __volatile__("sync": : :"memory")
32 #define WMB() __asm__ __volatile__("sync": : :"memory")
33 #define SMP_SYNC "sync"
34 #endif
35
36
37 /**********************************************************************
38 *
39 * Define constants for MIPS
40 *
41 *********************************************************************/
42 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
43
44 #define OPAL_HAVE_ATOMIC_CMPSET_32 1
45
46 #ifdef __mips64
47 #define OPAL_HAVE_ATOMIC_CMPSET_64 1
48 #endif
49
50 /**********************************************************************
51 *
52 * Memory Barriers
53 *
54 *********************************************************************/
55 #if OPAL_GCC_INLINE_ASSEMBLY
56
57 static inline
opal_atomic_mb(void)58 void opal_atomic_mb(void)
59 {
60 MB();
61 }
62
63
64 static inline
opal_atomic_rmb(void)65 void opal_atomic_rmb(void)
66 {
67 RMB();
68 }
69
70
71 static inline
opal_atomic_wmb(void)72 void opal_atomic_wmb(void)
73 {
74 WMB();
75 }
76
77 static inline
opal_atomic_isync(void)78 void opal_atomic_isync(void)
79 {
80 }
81
82 #endif
83
84 /**********************************************************************
85 *
86 * Atomic math operations
87 *
88 *********************************************************************/
89 #if OPAL_GCC_INLINE_ASSEMBLY
90
opal_atomic_cmpset_32(volatile int32_t * addr,int32_t oldval,int32_t newval)91 static inline int opal_atomic_cmpset_32(volatile int32_t *addr,
92 int32_t oldval, int32_t newval)
93 {
94 int32_t ret;
95
96 __asm__ __volatile__ (".set noreorder \n"
97 ".set noat \n"
98 "1: \n"
99 #ifdef __linux__
100 ".set mips2 \n\t"
101 #endif
102 "ll %0, %2 \n" /* load *addr into ret */
103 "bne %0, %z3, 2f \n" /* done if oldval != ret */
104 "or $1, %z4, 0 \n" /* tmp = newval (delay slot) */
105 "sc $1, %2 \n" /* store tmp in *addr */
106 #ifdef __linux__
107 ".set mips0 \n\t"
108 #endif
109 /* note: ret will be 0 if failed, 1 if succeeded */
110 "beqz $1, 1b \n" /* if 0 jump back to 1b */
111 "nop \n" /* fill delay slots */
112 "2: \n"
113 ".set reorder \n"
114 : "=&r"(ret), "=m"(*addr)
115 : "m"(*addr), "r"(oldval), "r"(newval)
116 : "cc", "memory");
117 return (ret == oldval);
118 }
119
120
121 /* these two functions aren't inlined in the non-gcc case because then
122 there would be two function calls (since neither cmpset_32 nor
123 atomic_?mb can be inlined). Instead, we "inline" them by hand in
124 the assembly, meaning there is one function call overhead instead
125 of two */
opal_atomic_cmpset_acq_32(volatile int32_t * addr,int32_t oldval,int32_t newval)126 static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr,
127 int32_t oldval, int32_t newval)
128 {
129 int rc;
130
131 rc = opal_atomic_cmpset_32(addr, oldval, newval);
132 opal_atomic_rmb();
133
134 return rc;
135 }
136
137
opal_atomic_cmpset_rel_32(volatile int32_t * addr,int32_t oldval,int32_t newval)138 static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr,
139 int32_t oldval, int32_t newval)
140 {
141 opal_atomic_wmb();
142 return opal_atomic_cmpset_32(addr, oldval, newval);
143 }
144
145 #ifdef OPAL_HAVE_ATOMIC_CMPSET_64
opal_atomic_cmpset_64(volatile int64_t * addr,int64_t oldval,int64_t newval)146 static inline int opal_atomic_cmpset_64(volatile int64_t *addr,
147 int64_t oldval, int64_t newval)
148 {
149 int64_t ret;
150
151 __asm__ __volatile__ (".set noreorder \n"
152 ".set noat \n"
153 "1: \n\t"
154 "lld %0, %2 \n\t" /* load *addr into ret */
155 "bne %0, %z3, 2f \n\t" /* done if oldval != ret */
156 "or $1, %4, 0 \n\t" /* tmp = newval (delay slot) */
157 "scd $1, %2 \n\t" /* store tmp in *addr */
158 /* note: ret will be 0 if failed, 1 if succeeded */
159 "beqz $1, 1b \n\t" /* if 0 jump back to 1b */
160 "nop \n\t" /* fill delay slot */
161 "2: \n\t"
162 ".set reorder \n"
163 : "=&r" (ret), "=m" (*addr)
164 : "m" (*addr), "r" (oldval), "r" (newval)
165 : "cc", "memory");
166
167 return (ret == oldval);
168 }
169
170
171 /* these two functions aren't inlined in the non-gcc case because then
172 there would be two function calls (since neither cmpset_64 nor
173 atomic_?mb can be inlined). Instead, we "inline" them by hand in
174 the assembly, meaning there is one function call overhead instead
175 of two */
opal_atomic_cmpset_acq_64(volatile int64_t * addr,int64_t oldval,int64_t newval)176 static inline int opal_atomic_cmpset_acq_64(volatile int64_t *addr,
177 int64_t oldval, int64_t newval)
178 {
179 int rc;
180
181 rc = opal_atomic_cmpset_64(addr, oldval, newval);
182 opal_atomic_rmb();
183
184 return rc;
185 }
186
187
opal_atomic_cmpset_rel_64(volatile int64_t * addr,int64_t oldval,int64_t newval)188 static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr,
189 int64_t oldval, int64_t newval)
190 {
191 opal_atomic_wmb();
192 return opal_atomic_cmpset_64(addr, oldval, newval);
193 }
194 #endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */
195
196 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
197
198 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
199