1 #ifndef _ATOMIC_H
2 #define _ATOMIC_H
3
4 #include <stdint.h>
5
6 #include "atomic_arch.h"
7
8 #ifdef a_ll
9
10 #ifndef a_pre_llsc
11 #define a_pre_llsc()
12 #endif
13
14 #ifndef a_post_llsc
15 #define a_post_llsc()
16 #endif
17
18 #ifndef a_cas
19 #define a_cas a_cas
a_cas(volatile int * p,int t,int s)20 static inline int a_cas(volatile int *p, int t, int s)
21 {
22 int old;
23 a_pre_llsc();
24 do old = a_ll(p);
25 while (old==t && !a_sc(p, s));
26 a_post_llsc();
27 return old;
28 }
29 #endif
30
31 #ifndef a_swap
32 #define a_swap a_swap
a_swap(volatile int * p,int v)33 static inline int a_swap(volatile int *p, int v)
34 {
35 int old;
36 a_pre_llsc();
37 do old = a_ll(p);
38 while (!a_sc(p, v));
39 a_post_llsc();
40 return old;
41 }
42 #endif
43
44 #ifndef a_fetch_add
45 #define a_fetch_add a_fetch_add
a_fetch_add(volatile int * p,int v)46 static inline int a_fetch_add(volatile int *p, int v)
47 {
48 int old;
49 a_pre_llsc();
50 do old = a_ll(p);
51 while (!a_sc(p, (unsigned)old + v));
52 a_post_llsc();
53 return old;
54 }
55 #endif
56
57 #ifndef a_fetch_and
58 #define a_fetch_and a_fetch_and
a_fetch_and(volatile int * p,int v)59 static inline int a_fetch_and(volatile int *p, int v)
60 {
61 int old;
62 a_pre_llsc();
63 do old = a_ll(p);
64 while (!a_sc(p, old & v));
65 a_post_llsc();
66 return old;
67 }
68 #endif
69
70 #ifndef a_fetch_or
71 #define a_fetch_or a_fetch_or
a_fetch_or(volatile int * p,int v)72 static inline int a_fetch_or(volatile int *p, int v)
73 {
74 int old;
75 a_pre_llsc();
76 do old = a_ll(p);
77 while (!a_sc(p, old | v));
78 a_post_llsc();
79 return old;
80 }
81 #endif
82
83 #endif
84
85 #ifdef a_ll_p
86
87 #ifndef a_cas_p
88 #define a_cas_p a_cas_p
a_cas_p(volatile void * p,void * t,void * s)89 static inline void *a_cas_p(volatile void *p, void *t, void *s)
90 {
91 void *old;
92 a_pre_llsc();
93 do old = a_ll_p(p);
94 while (old==t && !a_sc_p(p, s));
95 a_post_llsc();
96 return old;
97 }
98 #endif
99
100 #endif
101
102 #ifndef a_cas
103 #error missing definition of a_cas
104 #endif
105
106 #ifndef a_swap
107 #define a_swap a_swap
a_swap(volatile int * p,int v)108 static inline int a_swap(volatile int *p, int v)
109 {
110 int old;
111 do old = *p;
112 while (a_cas(p, old, v) != old);
113 return old;
114 }
115 #endif
116
117 #ifndef a_fetch_add
118 #define a_fetch_add a_fetch_add
a_fetch_add(volatile int * p,int v)119 static inline int a_fetch_add(volatile int *p, int v)
120 {
121 int old;
122 do old = *p;
123 while (a_cas(p, old, (unsigned)old+v) != old);
124 return old;
125 }
126 #endif
127
128 #ifndef a_fetch_and
129 #define a_fetch_and a_fetch_and
a_fetch_and(volatile int * p,int v)130 static inline int a_fetch_and(volatile int *p, int v)
131 {
132 int old;
133 do old = *p;
134 while (a_cas(p, old, old&v) != old);
135 return old;
136 }
137 #endif
138 #ifndef a_fetch_or
139 #define a_fetch_or a_fetch_or
a_fetch_or(volatile int * p,int v)140 static inline int a_fetch_or(volatile int *p, int v)
141 {
142 int old;
143 do old = *p;
144 while (a_cas(p, old, old|v) != old);
145 return old;
146 }
147 #endif
148
149 #ifndef a_and
150 #define a_and a_and
a_and(volatile int * p,int v)151 static inline void a_and(volatile int *p, int v)
152 {
153 a_fetch_and(p, v);
154 }
155 #endif
156
157 #ifndef a_or
158 #define a_or a_or
a_or(volatile int * p,int v)159 static inline void a_or(volatile int *p, int v)
160 {
161 a_fetch_or(p, v);
162 }
163 #endif
164
165 #ifndef a_inc
166 #define a_inc a_inc
a_inc(volatile int * p)167 static inline void a_inc(volatile int *p)
168 {
169 a_fetch_add(p, 1);
170 }
171 #endif
172
173 #ifndef a_dec
174 #define a_dec a_dec
a_dec(volatile int * p)175 static inline void a_dec(volatile int *p)
176 {
177 a_fetch_add(p, -1);
178 }
179 #endif
180
181 #ifndef a_store
182 #define a_store a_store
a_store(volatile int * p,int v)183 static inline void a_store(volatile int *p, int v)
184 {
185 #ifdef a_barrier
186 a_barrier();
187 *p = v;
188 a_barrier();
189 #else
190 a_swap(p, v);
191 #endif
192 }
193 #endif
194
195 #ifndef a_barrier
196 #define a_barrier a_barrier
a_barrier()197 static void a_barrier()
198 {
199 volatile int tmp = 0;
200 a_cas(&tmp, 0, 0);
201 }
202 #endif
203
204 #ifndef a_spin
205 #define a_spin a_barrier
206 #endif
207
208 #ifndef a_and_64
209 #define a_and_64 a_and_64
a_and_64(volatile uint64_t * p,uint64_t v)210 static inline void a_and_64(volatile uint64_t *p, uint64_t v)
211 {
212 union { uint64_t v; uint32_t r[2]; } u = { v };
213 if (u.r[0]+1) a_and((int *)p, u.r[0]);
214 if (u.r[1]+1) a_and((int *)p+1, u.r[1]);
215 }
216 #endif
217
218 #ifndef a_or_64
219 #define a_or_64 a_or_64
a_or_64(volatile uint64_t * p,uint64_t v)220 static inline void a_or_64(volatile uint64_t *p, uint64_t v)
221 {
222 union { uint64_t v; uint32_t r[2]; } u = { v };
223 if (u.r[0]) a_or((int *)p, u.r[0]);
224 if (u.r[1]) a_or((int *)p+1, u.r[1]);
225 }
226 #endif
227
228 #ifndef a_cas_p
229 typedef char a_cas_p_undefined_but_pointer_not_32bit[-sizeof(char) == 0xffffffff ? 1 : -1];
230 #define a_cas_p a_cas_p
a_cas_p(volatile void * p,void * t,void * s)231 static inline void *a_cas_p(volatile void *p, void *t, void *s)
232 {
233 return (void *)a_cas((volatile int *)p, (int)t, (int)s);
234 }
235 #endif
236
237 #ifndef a_or_l
238 #define a_or_l a_or_l
a_or_l(volatile void * p,long v)239 static inline void a_or_l(volatile void *p, long v)
240 {
241 if (sizeof(long) == sizeof(int)) a_or(p, v);
242 else a_or_64(p, v);
243 }
244 #endif
245
246 #ifndef a_crash
247 #define a_crash a_crash
a_crash()248 static inline void a_crash()
249 {
250 *(volatile char *)0=0;
251 }
252 #endif
253
254 #ifndef a_ctz_64
255 #define a_ctz_64 a_ctz_64
a_ctz_64(uint64_t x)256 static inline int a_ctz_64(uint64_t x)
257 {
258 static const char debruijn64[64] = {
259 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
260 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
261 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
262 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
263 };
264 static const char debruijn32[32] = {
265 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
266 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
267 };
268 if (sizeof(long) < 8) {
269 uint32_t y = x;
270 if (!y) {
271 y = x>>32;
272 return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
273 }
274 return debruijn32[(y&-y)*0x076be629 >> 27];
275 }
276 return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
277 }
278 #endif
279
280 #ifndef a_ctz_l
281 #define a_ctz_l a_ctz_l
a_ctz_l(unsigned long x)282 static inline int a_ctz_l(unsigned long x)
283 {
284 static const char debruijn32[32] = {
285 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
286 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
287 };
288 if (sizeof(long) == 8) return a_ctz_64(x);
289 return debruijn32[(x&-x)*0x076be629 >> 27];
290 }
291 #endif
292
293 #endif
294