1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2005 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2010 IBM Corporation. All rights reserved.
13 * Copyright (c) 2010 ARM ltd. All rights reserved.
14 * $COPYRIGHT$
15 *
16 * Additional copyrights may follow
17 *
18 * $HEADER$
19 */
20
21 /*
22 * ARMv5 and earlier lack robust atomic operations and therefore this file uses
23 * Linux kernel support where needed. The kernel also provides memory barriers
24 * and this file uses them for ARMv5 and earlier processors, which lack the
25 * memory barrier instruction. These kernel functions are available on kernel
26 * versions 2.6.15 and greater; using them will result in undefined behavior on
27 * older kernels.
28 * See Documentation/arm/kernel_user_helpers.txt in the kernel tree for details
29 */
30
31 #ifndef OPAL_SYS_ARCH_ATOMIC_H
32 #define OPAL_SYS_ARCH_ATOMIC_H 1
33
34 #if (OPAL_ASM_ARM_VERSION >= 7)
35
36 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
37 /* use the DMB instruction if available... */
38
39 #define MB() __asm__ __volatile__ ("dmb" : : : "memory")
40 #define RMB() __asm__ __volatile__ ("dmb" : : : "memory")
41 #define WMB() __asm__ __volatile__ ("dmb" : : : "memory")
42
43 #elif (OPAL_ASM_ARM_VERSION == 6)
44
45 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
46 /* ...or the v6-specific equivalent... */
47
48 #define MB() __asm__ __volatile__ ("mcr p15, 0, r0, c7, c10, 5" : : : "memory")
49 #define RMB() MB()
50 #define WMB() MB()
51
52 #else
53
54 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
55 /* ...otherwise use the Linux kernel-provided barrier */
56
57 #define MB() (*((void (*)(void))(0xffff0fa0)))()
58 #define RMB() MB()
59 #define WMB() MB()
60
61 #endif
62
63 /**********************************************************************
64 *
65 * Memory Barriers
66 *
67 *********************************************************************/
68
69 #if (OPAL_HAVE_ATOMIC_MEM_BARRIER == 1)
70
71 static inline
opal_atomic_mb(void)72 void opal_atomic_mb(void)
73 {
74 MB();
75 }
76
77
78 static inline
opal_atomic_rmb(void)79 void opal_atomic_rmb(void)
80 {
81 RMB();
82 }
83
84
85 static inline
opal_atomic_wmb(void)86 void opal_atomic_wmb(void)
87 {
88 WMB();
89 }
90
91 static inline
opal_atomic_isync(void)92 void opal_atomic_isync(void)
93 {
94 }
95
96 #endif
97
98
99 /**********************************************************************
100 *
101 * Atomic math operations
102 *
103 *********************************************************************/
104
105 #if (OPAL_GCC_INLINE_ASSEMBLY && (OPAL_ASM_ARM_VERSION >= 6))
106
107 #define OPAL_HAVE_ATOMIC_CMPSET_32 1
108 #define OPAL_HAVE_ATOMIC_MATH_32 1
opal_atomic_cmpset_32(volatile int32_t * addr,int32_t oldval,int32_t newval)109 static inline int opal_atomic_cmpset_32(volatile int32_t *addr,
110 int32_t oldval, int32_t newval)
111 {
112 int32_t ret, tmp;
113
114 __asm__ __volatile__ (
115 "1: ldrex %0, [%2] \n"
116 " cmp %0, %3 \n"
117 " bne 2f \n"
118 " strex %1, %4, [%2] \n"
119 " cmp %1, #0 \n"
120 " bne 1b \n"
121 "2: \n"
122
123 : "=&r" (ret), "=&r" (tmp)
124 : "r" (addr), "r" (oldval), "r" (newval)
125 : "cc", "memory");
126
127 return (ret == oldval);
128 }
129
130 /* these two functions aren't inlined in the non-gcc case because then
131 there would be two function calls (since neither cmpset_32 nor
132 atomic_?mb can be inlined). Instead, we "inline" them by hand in
133 the assembly, meaning there is one function call overhead instead
134 of two */
opal_atomic_cmpset_acq_32(volatile int32_t * addr,int32_t oldval,int32_t newval)135 static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr,
136 int32_t oldval, int32_t newval)
137 {
138 int rc;
139
140 rc = opal_atomic_cmpset_32(addr, oldval, newval);
141 opal_atomic_rmb();
142
143 return rc;
144 }
145
146
opal_atomic_cmpset_rel_32(volatile int32_t * addr,int32_t oldval,int32_t newval)147 static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr,
148 int32_t oldval, int32_t newval)
149 {
150 opal_atomic_wmb();
151 return opal_atomic_cmpset_32(addr, oldval, newval);
152 }
153
154 #if (OPAL_ASM_SUPPORT_64BIT == 1)
155
156 #define OPAL_HAVE_ATOMIC_CMPSET_64 1
opal_atomic_cmpset_64(volatile int64_t * addr,int64_t oldval,int64_t newval)157 static inline int opal_atomic_cmpset_64(volatile int64_t *addr,
158 int64_t oldval, int64_t newval)
159 {
160 int64_t ret;
161 int tmp;
162
163
164 __asm__ __volatile__ (
165 "1: ldrexd %0, %H0, [%2] \n"
166 " cmp %0, %3 \n"
167 " it eq \n"
168 " cmpeq %H0, %H3 \n"
169 " bne 2f \n"
170 " strexd %1, %4, %H4, [%2] \n"
171 " cmp %1, #0 \n"
172 " bne 1b \n"
173 "2: \n"
174
175 : "=&r" (ret), "=&r" (tmp)
176 : "r" (addr), "r" (oldval), "r" (newval)
177 : "cc", "memory");
178
179 return (ret == oldval);
180 }
181
182 /* these two functions aren't inlined in the non-gcc case because then
183 there would be two function calls (since neither cmpset_64 nor
184 atomic_?mb can be inlined). Instead, we "inline" them by hand in
185 the assembly, meaning there is one function call overhead instead
186 of two */
opal_atomic_cmpset_acq_64(volatile int64_t * addr,int64_t oldval,int64_t newval)187 static inline int opal_atomic_cmpset_acq_64(volatile int64_t *addr,
188 int64_t oldval, int64_t newval)
189 {
190 int rc;
191
192 rc = opal_atomic_cmpset_64(addr, oldval, newval);
193 opal_atomic_rmb();
194
195 return rc;
196 }
197
198
opal_atomic_cmpset_rel_64(volatile int64_t * addr,int64_t oldval,int64_t newval)199 static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr,
200 int64_t oldval, int64_t newval)
201 {
202 opal_atomic_wmb();
203 return opal_atomic_cmpset_64(addr, oldval, newval);
204 }
205
206 #endif
207
208
209 #define OPAL_HAVE_ATOMIC_ADD_32 1
opal_atomic_add_32(volatile int32_t * v,int inc)210 static inline int32_t opal_atomic_add_32(volatile int32_t* v, int inc)
211 {
212 int32_t t;
213 int tmp;
214
215 __asm__ __volatile__(
216 "1: ldrex %0, [%2] \n"
217 " add %0, %0, %3 \n"
218 " strex %1, %0, [%2] \n"
219 " cmp %1, #0 \n"
220 " bne 1b \n"
221
222 : "=&r" (t), "=&r" (tmp)
223 : "r" (v), "r" (inc)
224 : "cc", "memory");
225
226
227 return t;
228 }
229
230 #define OPAL_HAVE_ATOMIC_SUB_32 1
opal_atomic_sub_32(volatile int32_t * v,int dec)231 static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec)
232 {
233 int32_t t;
234 int tmp;
235
236 __asm__ __volatile__(
237 "1: ldrex %0, [%2] \n"
238 " sub %0, %0, %3 \n"
239 " strex %1, %0, [%2] \n"
240 " cmp %1, #0 \n"
241 " bne 1b \n"
242
243 : "=&r" (t), "=&r" (tmp)
244 : "r" (v), "r" (dec)
245 : "cc", "memory");
246
247 return t;
248 }
249
250 #else /* OPAL_ASM_ARM_VERSION <=5 or no GCC inline assembly */
251
252 #define OPAL_HAVE_ATOMIC_CMPSET_32 1
253 #define __kuser_cmpxchg (*((int (*)(int, int, volatile int*))(0xffff0fc0)))
opal_atomic_cmpset_32(volatile int32_t * addr,int32_t oldval,int32_t newval)254 static inline int opal_atomic_cmpset_32(volatile int32_t *addr,
255 int32_t oldval, int32_t newval)
256 {
257 return !(__kuser_cmpxchg(oldval, newval, addr));
258 }
259
opal_atomic_cmpset_acq_32(volatile int32_t * addr,int32_t oldval,int32_t newval)260 static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr,
261 int32_t oldval, int32_t newval)
262 {
263 /* kernel function includes all necessary memory barriers */
264 return opal_atomic_cmpset_32(addr, oldval, newval);
265 }
266
opal_atomic_cmpset_rel_32(volatile int32_t * addr,int32_t oldval,int32_t newval)267 static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr,
268 int32_t oldval, int32_t newval)
269 {
270 /* kernel function includes all necessary memory barriers */
271 return opal_atomic_cmpset_32(addr, oldval, newval);
272 }
273
274 #endif
275
276 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
277