1 /*-
2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #ifdef __arm__
28 #include <arm/atomic.h>
29 #else /* !__arm__ */
30
31 #ifndef _MACHINE_ATOMIC_H_
32 #define _MACHINE_ATOMIC_H_
33
34 #define isb() __asm __volatile("isb" : : : "memory")
35
36 /*
37 * Options for DMB and DSB:
38 * oshld Outer Shareable, load
39 * oshst Outer Shareable, store
40 * osh Outer Shareable, all
41 * nshld Non-shareable, load
42 * nshst Non-shareable, store
43 * nsh Non-shareable, all
44 * ishld Inner Shareable, load
45 * ishst Inner Shareable, store
46 * ish Inner Shareable, all
47 * ld Full system, load
48 * st Full system, store
49 * sy Full system, all
50 */
51 #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory")
52 #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory")
53
54 #define mb() dmb(sy) /* Full system memory barrier all */
55 #define wmb() dmb(st) /* Full system memory barrier store */
56 #define rmb() dmb(ld) /* Full system memory barrier load */
57
58 #ifdef _KERNEL
59 extern _Bool lse_supported;
60 #endif
61
62 #if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
63 #include <sys/atomic_san.h>
64 #else
65
66 #include <sys/atomic_common.h>
67
68 #ifdef _KERNEL
69
70 #ifdef LSE_ATOMICS
71 #define _ATOMIC_LSE_SUPPORTED 1
72 #else
73 #define _ATOMIC_LSE_SUPPORTED lse_supported
74 #endif
75 #else
76 #define _ATOMIC_LSE_SUPPORTED 0
77 #endif
78
79 #define _ATOMIC_OP_PROTO(t, op, bar, flav) \
80 static __inline void \
81 atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)
82
83 #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
84 _ATOMIC_OP_PROTO(t, op, bar, _llsc) \
85 { \
86 uint##t##_t tmp; \
87 int res; \
88 \
89 pre; \
90 __asm __volatile( \
91 "1: ld"#a"xr"#s" %"#w"0, [%2]\n" \
92 " "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \
93 " st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \
94 " cbnz %w1, 1b\n" \
95 : "=&r"(tmp), "=&r"(res) \
96 : "r" (p), "r" (val) \
97 : "memory" \
98 ); \
99 } \
100 \
101 _ATOMIC_OP_PROTO(t, op, bar, _lse) \
102 { \
103 uint##t##_t tmp; \
104 \
105 pre; \
106 __asm __volatile( \
107 ".arch_extension lse\n" \
108 "ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \
109 ".arch_extension nolse\n" \
110 : "=r" (tmp) \
111 : "r" (p), "r" (val) \
112 : "memory" \
113 ); \
114 } \
115 \
116 _ATOMIC_OP_PROTO(t, op, bar, ) \
117 { \
118 if (_ATOMIC_LSE_SUPPORTED) \
119 atomic_##op##_##bar##t##_lse(p, val); \
120 else \
121 atomic_##op##_##bar##t##_llsc(p, val); \
122 }
123
124 #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
125 _ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \
126 bar, a, l) \
127 _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \
128 bar, a, l) \
129 _ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \
130 bar, a, l) \
131 _ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \
132 bar, a, l)
133
134 #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \
135 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \
136 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \
137 __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l)
138
139 _ATOMIC_OP(add, add, add, )
140 _ATOMIC_OP(clear, bic, clr, )
141 _ATOMIC_OP(set, orr, set, )
142 _ATOMIC_OP(subtract, add, add, val = -val)
143
144 #define _ATOMIC_CMPSET_PROTO(t, bar, flav) \
145 static __inline int \
146 atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \
147 uint##t##_t cmpval, uint##t##_t newval)
148
149 #define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \
150 static __inline int \
151 atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \
152 uint##t##_t *cmpval, uint##t##_t newval)
153
154 #define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \
155 _ATOMIC_CMPSET_PROTO(t, bar, _llsc) \
156 { \
157 uint##t##_t tmp; \
158 int res; \
159 \
160 __asm __volatile( \
161 "1: mov %w1, #1\n" \
162 " ld"#a"xr"#s" %"#w"0, [%2]\n" \
163 " cmp %"#w"0, %"#w"3\n" \
164 " b.ne 2f\n" \
165 " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \
166 " cbnz %w1, 1b\n" \
167 "2:" \
168 : "=&r"(tmp), "=&r"(res) \
169 : "r" (p), "r" (cmpval), "r" (newval) \
170 : "cc", "memory" \
171 ); \
172 \
173 return (!res); \
174 } \
175 \
176 _ATOMIC_CMPSET_PROTO(t, bar, _lse) \
177 { \
178 uint##t##_t oldval; \
179 int res; \
180 \
181 oldval = cmpval; \
182 __asm __volatile( \
183 ".arch_extension lse\n" \
184 "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \
185 "cmp %"#w"1, %"#w"2\n" \
186 "cset %w0, eq\n" \
187 ".arch_extension nolse\n" \
188 : "=r" (res), "+&r" (cmpval) \
189 : "r" (oldval), "r" (p), "r" (newval) \
190 : "cc", "memory" \
191 ); \
192 \
193 return (res); \
194 } \
195 \
196 _ATOMIC_CMPSET_PROTO(t, bar, ) \
197 { \
198 if (_ATOMIC_LSE_SUPPORTED) \
199 return (atomic_cmpset_##bar##t##_lse(p, cmpval, \
200 newval)); \
201 else \
202 return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \
203 newval)); \
204 } \
205 \
206 _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \
207 { \
208 uint##t##_t _cmpval, tmp; \
209 int res; \
210 \
211 _cmpval = *cmpval; \
212 __asm __volatile( \
213 " mov %w1, #1\n" \
214 " ld"#a"xr"#s" %"#w"0, [%2]\n" \
215 " cmp %"#w"0, %"#w"3\n" \
216 " b.ne 1f\n" \
217 " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \
218 "1:" \
219 : "=&r"(tmp), "=&r"(res) \
220 : "r" (p), "r" (_cmpval), "r" (newval) \
221 : "cc", "memory" \
222 ); \
223 *cmpval = tmp; \
224 \
225 return (!res); \
226 } \
227 \
228 _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \
229 { \
230 uint##t##_t _cmpval, tmp; \
231 int res; \
232 \
233 _cmpval = tmp = *cmpval; \
234 __asm __volatile( \
235 ".arch_extension lse\n" \
236 "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \
237 "cmp %"#w"1, %"#w"2\n" \
238 "cset %w0, eq\n" \
239 ".arch_extension nolse\n" \
240 : "=r" (res), "+&r" (tmp) \
241 : "r" (_cmpval), "r" (p), "r" (newval) \
242 : "cc", "memory" \
243 ); \
244 *cmpval = tmp; \
245 \
246 return (res); \
247 } \
248 \
249 _ATOMIC_FCMPSET_PROTO(t, bar, ) \
250 { \
251 if (_ATOMIC_LSE_SUPPORTED) \
252 return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \
253 newval)); \
254 else \
255 return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \
256 newval)); \
257 }
258
259 #define _ATOMIC_CMPSET(bar, a, l) \
260 _ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \
261 _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \
262 _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \
263 _ATOMIC_CMPSET_IMPL(64, , , bar, a, l)
264
265 #define atomic_cmpset_8 atomic_cmpset_8
266 #define atomic_fcmpset_8 atomic_fcmpset_8
267 #define atomic_cmpset_16 atomic_cmpset_16
268 #define atomic_fcmpset_16 atomic_fcmpset_16
269
270 _ATOMIC_CMPSET( , , )
271 _ATOMIC_CMPSET(acq_, a, )
272 _ATOMIC_CMPSET(rel_, ,l)
273
274 #define _ATOMIC_FETCHADD_PROTO(t, flav) \
275 static __inline uint##t##_t \
276 atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
277
278 #define _ATOMIC_FETCHADD_IMPL(t, w) \
279 _ATOMIC_FETCHADD_PROTO(t, _llsc) \
280 { \
281 uint##t##_t ret, tmp; \
282 int res; \
283 \
284 __asm __volatile( \
285 "1: ldxr %"#w"2, [%3]\n" \
286 " add %"#w"0, %"#w"2, %"#w"4\n" \
287 " stxr %w1, %"#w"0, [%3]\n" \
288 " cbnz %w1, 1b\n" \
289 : "=&r" (tmp), "=&r" (res), "=&r" (ret) \
290 : "r" (p), "r" (val) \
291 : "memory" \
292 ); \
293 \
294 return (ret); \
295 } \
296 \
297 _ATOMIC_FETCHADD_PROTO(t, _lse) \
298 { \
299 uint##t##_t ret; \
300 \
301 __asm __volatile( \
302 ".arch_extension lse\n" \
303 "ldadd %"#w"2, %"#w"0, [%1]\n" \
304 ".arch_extension nolse\n" \
305 : "=r" (ret) \
306 : "r" (p), "r" (val) \
307 : "memory" \
308 ); \
309 \
310 return (ret); \
311 } \
312 \
313 _ATOMIC_FETCHADD_PROTO(t, ) \
314 { \
315 if (_ATOMIC_LSE_SUPPORTED) \
316 return (atomic_fetchadd_##t##_lse(p, val)); \
317 else \
318 return (atomic_fetchadd_##t##_llsc(p, val)); \
319 }
320
321 _ATOMIC_FETCHADD_IMPL(32, w)
322 _ATOMIC_FETCHADD_IMPL(64, )
323
324 #define _ATOMIC_SWAP_PROTO(t, flav) \
325 static __inline uint##t##_t \
326 atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
327
328 #define _ATOMIC_READANDCLEAR_PROTO(t, flav) \
329 static __inline uint##t##_t \
330 atomic_readandclear_##t##flav(volatile uint##t##_t *p)
331
332 #define _ATOMIC_SWAP_IMPL(t, w, zreg) \
333 _ATOMIC_SWAP_PROTO(t, _llsc) \
334 { \
335 uint##t##_t ret; \
336 int res; \
337 \
338 __asm __volatile( \
339 "1: ldxr %"#w"1, [%2]\n" \
340 " stxr %w0, %"#w"3, [%2]\n" \
341 " cbnz %w0, 1b\n" \
342 : "=&r" (res), "=&r" (ret) \
343 : "r" (p), "r" (val) \
344 : "memory" \
345 ); \
346 \
347 return (ret); \
348 } \
349 \
350 _ATOMIC_SWAP_PROTO(t, _lse) \
351 { \
352 uint##t##_t ret; \
353 \
354 __asm __volatile( \
355 ".arch_extension lse\n" \
356 "swp %"#w"2, %"#w"0, [%1]\n" \
357 ".arch_extension nolse\n" \
358 : "=r" (ret) \
359 : "r" (p), "r" (val) \
360 : "memory" \
361 ); \
362 \
363 return (ret); \
364 } \
365 \
366 _ATOMIC_SWAP_PROTO(t, ) \
367 { \
368 if (_ATOMIC_LSE_SUPPORTED) \
369 return (atomic_swap_##t##_lse(p, val)); \
370 else \
371 return (atomic_swap_##t##_llsc(p, val)); \
372 } \
373 \
374 _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \
375 { \
376 uint##t##_t ret; \
377 int res; \
378 \
379 __asm __volatile( \
380 "1: ldxr %"#w"1, [%2]\n" \
381 " stxr %w0, "#zreg", [%2]\n" \
382 " cbnz %w0, 1b\n" \
383 : "=&r" (res), "=&r" (ret) \
384 : "r" (p) \
385 : "memory" \
386 ); \
387 \
388 return (ret); \
389 } \
390 \
391 _ATOMIC_READANDCLEAR_PROTO(t, _lse) \
392 { \
393 return (atomic_swap_##t##_lse(p, 0)); \
394 } \
395 \
396 _ATOMIC_READANDCLEAR_PROTO(t, ) \
397 { \
398 if (_ATOMIC_LSE_SUPPORTED) \
399 return (atomic_readandclear_##t##_lse(p)); \
400 else \
401 return (atomic_readandclear_##t##_llsc(p)); \
402 }
403
404 _ATOMIC_SWAP_IMPL(32, w, wzr)
405 _ATOMIC_SWAP_IMPL(64, , xzr)
406
407 #define _ATOMIC_TEST_OP_PROTO(t, op, bar, flav) \
408 static __inline int \
409 atomic_testand##op##_##bar##t##flav(volatile uint##t##_t *p, u_int val)
410
411 #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op, bar, a) \
412 _ATOMIC_TEST_OP_PROTO(t, op, bar, _llsc) \
413 { \
414 uint##t##_t mask, old, tmp; \
415 int res; \
416 \
417 mask = ((uint##t##_t)1) << (val & (t - 1)); \
418 __asm __volatile( \
419 "1: ld"#a"xr %"#w"2, [%3]\n" \
420 " "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \
421 " stxr %w1, %"#w"0, [%3]\n" \
422 " cbnz %w1, 1b\n" \
423 : "=&r" (tmp), "=&r" (res), "=&r" (old) \
424 : "r" (p), "r" (mask) \
425 : "memory" \
426 ); \
427 \
428 return ((old & mask) != 0); \
429 } \
430 \
431 _ATOMIC_TEST_OP_PROTO(t, op, bar, _lse) \
432 { \
433 uint##t##_t mask, old; \
434 \
435 mask = ((uint##t##_t)1) << (val & (t - 1)); \
436 __asm __volatile( \
437 ".arch_extension lse\n" \
438 "ld"#lse_asm_op#a" %"#w"2, %"#w"0, [%1]\n" \
439 ".arch_extension nolse\n" \
440 : "=r" (old) \
441 : "r" (p), "r" (mask) \
442 : "memory" \
443 ); \
444 \
445 return ((old & mask) != 0); \
446 } \
447 \
448 _ATOMIC_TEST_OP_PROTO(t, op, bar, ) \
449 { \
450 if (_ATOMIC_LSE_SUPPORTED) \
451 return (atomic_testand##op##_##bar##t##_lse(p, val)); \
452 else \
453 return (atomic_testand##op##_##bar##t##_llsc(p, val)); \
454 }
455
456 #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \
457 _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, , ) \
458 _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, acq_, a) \
459 _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, , ) \
460 _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, acq_, a)
461
_ATOMIC_TEST_OP(clear,bic,clr)462 _ATOMIC_TEST_OP(clear, bic, clr)
463 _ATOMIC_TEST_OP(set, orr, set)
464
465 #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \
466 static __inline uint##t##_t \
467 atomic_load_acq_##t(volatile uint##t##_t *p) \
468 { \
469 uint##t##_t ret; \
470 \
471 __asm __volatile( \
472 "ldar"#s" %"#w"0, [%1]\n" \
473 : "=&r" (ret) \
474 : "r" (p) \
475 : "memory"); \
476 \
477 return (ret); \
478 }
479
480 #define atomic_load_acq_8 atomic_load_acq_8
481 #define atomic_load_acq_16 atomic_load_acq_16
482 _ATOMIC_LOAD_ACQ_IMPL(8, w, b)
483 _ATOMIC_LOAD_ACQ_IMPL(16, w, h)
484 _ATOMIC_LOAD_ACQ_IMPL(32, w, )
485 _ATOMIC_LOAD_ACQ_IMPL(64, , )
486
487 #define _ATOMIC_STORE_REL_IMPL(t, w, s) \
488 static __inline void \
489 atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val) \
490 { \
491 __asm __volatile( \
492 "stlr"#s" %"#w"0, [%1]\n" \
493 : \
494 : "r" (val), "r" (p) \
495 : "memory"); \
496 }
497
498 _ATOMIC_STORE_REL_IMPL(8, w, b)
499 _ATOMIC_STORE_REL_IMPL(16, w, h)
500 _ATOMIC_STORE_REL_IMPL(32, w, )
501 _ATOMIC_STORE_REL_IMPL(64, , )
502
503 #define atomic_add_char atomic_add_8
504 #define atomic_fcmpset_char atomic_fcmpset_8
505 #define atomic_clear_char atomic_clear_8
506 #define atomic_cmpset_char atomic_cmpset_8
507 #define atomic_fetchadd_char atomic_fetchadd_8
508 #define atomic_readandclear_char atomic_readandclear_8
509 #define atomic_set_char atomic_set_8
510 #define atomic_swap_char atomic_swap_8
511 #define atomic_subtract_char atomic_subtract_8
512 #define atomic_testandclear_char atomic_testandclear_8
513 #define atomic_testandset_char atomic_testandset_8
514
515 #define atomic_add_acq_char atomic_add_acq_8
516 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8
517 #define atomic_clear_acq_char atomic_clear_acq_8
518 #define atomic_cmpset_acq_char atomic_cmpset_acq_8
519 #define atomic_load_acq_char atomic_load_acq_8
520 #define atomic_set_acq_char atomic_set_acq_8
521 #define atomic_subtract_acq_char atomic_subtract_acq_8
522 #define atomic_testandset_acq_char atomic_testandset_acq_8
523
524 #define atomic_add_rel_char atomic_add_rel_8
525 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8
526 #define atomic_clear_rel_char atomic_clear_rel_8
527 #define atomic_cmpset_rel_char atomic_cmpset_rel_8
528 #define atomic_set_rel_char atomic_set_rel_8
529 #define atomic_subtract_rel_char atomic_subtract_rel_8
530 #define atomic_store_rel_char atomic_store_rel_8
531
532 #define atomic_add_short atomic_add_16
533 #define atomic_fcmpset_short atomic_fcmpset_16
534 #define atomic_clear_short atomic_clear_16
535 #define atomic_cmpset_short atomic_cmpset_16
536 #define atomic_fetchadd_short atomic_fetchadd_16
537 #define atomic_readandclear_short atomic_readandclear_16
538 #define atomic_set_short atomic_set_16
539 #define atomic_swap_short atomic_swap_16
540 #define atomic_subtract_short atomic_subtract_16
541 #define atomic_testandclear_short atomic_testandclear_16
542 #define atomic_testandset_short atomic_testandset_16
543
544 #define atomic_add_acq_short atomic_add_acq_16
545 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16
546 #define atomic_clear_acq_short atomic_clear_acq_16
547 #define atomic_cmpset_acq_short atomic_cmpset_acq_16
548 #define atomic_load_acq_short atomic_load_acq_16
549 #define atomic_set_acq_short atomic_set_acq_16
550 #define atomic_subtract_acq_short atomic_subtract_acq_16
551 #define atomic_testandset_acq_short atomic_testandset_acq_16
552
553 #define atomic_add_rel_short atomic_add_rel_16
554 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16
555 #define atomic_clear_rel_short atomic_clear_rel_16
556 #define atomic_cmpset_rel_short atomic_cmpset_rel_16
557 #define atomic_set_rel_short atomic_set_rel_16
558 #define atomic_subtract_rel_short atomic_subtract_rel_16
559 #define atomic_store_rel_short atomic_store_rel_16
560
561 #define atomic_add_int atomic_add_32
562 #define atomic_fcmpset_int atomic_fcmpset_32
563 #define atomic_clear_int atomic_clear_32
564 #define atomic_cmpset_int atomic_cmpset_32
565 #define atomic_fetchadd_int atomic_fetchadd_32
566 #define atomic_readandclear_int atomic_readandclear_32
567 #define atomic_set_int atomic_set_32
568 #define atomic_swap_int atomic_swap_32
569 #define atomic_subtract_int atomic_subtract_32
570 #define atomic_testandclear_int atomic_testandclear_32
571 #define atomic_testandset_int atomic_testandset_32
572
573 #define atomic_add_acq_int atomic_add_acq_32
574 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
575 #define atomic_clear_acq_int atomic_clear_acq_32
576 #define atomic_cmpset_acq_int atomic_cmpset_acq_32
577 #define atomic_load_acq_int atomic_load_acq_32
578 #define atomic_set_acq_int atomic_set_acq_32
579 #define atomic_subtract_acq_int atomic_subtract_acq_32
580 #define atomic_testandset_acq_int atomic_testandset_acq_32
581
582 #define atomic_add_rel_int atomic_add_rel_32
583 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
584 #define atomic_clear_rel_int atomic_clear_rel_32
585 #define atomic_cmpset_rel_int atomic_cmpset_rel_32
586 #define atomic_set_rel_int atomic_set_rel_32
587 #define atomic_subtract_rel_int atomic_subtract_rel_32
588 #define atomic_store_rel_int atomic_store_rel_32
589
590 #define atomic_add_long atomic_add_64
591 #define atomic_fcmpset_long atomic_fcmpset_64
592 #define atomic_clear_long atomic_clear_64
593 #define atomic_cmpset_long atomic_cmpset_64
594 #define atomic_fetchadd_long atomic_fetchadd_64
595 #define atomic_readandclear_long atomic_readandclear_64
596 #define atomic_set_long atomic_set_64
597 #define atomic_swap_long atomic_swap_64
598 #define atomic_subtract_long atomic_subtract_64
599 #define atomic_testandclear_long atomic_testandclear_64
600 #define atomic_testandset_long atomic_testandset_64
601
602 #define atomic_add_ptr atomic_add_64
603 #define atomic_fcmpset_ptr atomic_fcmpset_64
604 #define atomic_clear_ptr atomic_clear_64
605 #define atomic_cmpset_ptr atomic_cmpset_64
606 #define atomic_fetchadd_ptr atomic_fetchadd_64
607 #define atomic_readandclear_ptr atomic_readandclear_64
608 #define atomic_set_ptr atomic_set_64
609 #define atomic_swap_ptr atomic_swap_64
610 #define atomic_subtract_ptr atomic_subtract_64
611
612 #define atomic_add_acq_long atomic_add_acq_64
613 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
614 #define atomic_clear_acq_long atomic_clear_acq_64
615 #define atomic_cmpset_acq_long atomic_cmpset_acq_64
616 #define atomic_load_acq_long atomic_load_acq_64
617 #define atomic_set_acq_long atomic_set_acq_64
618 #define atomic_subtract_acq_long atomic_subtract_acq_64
619 #define atomic_testandset_acq_long atomic_testandset_acq_64
620
621 #define atomic_add_acq_ptr atomic_add_acq_64
622 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
623 #define atomic_clear_acq_ptr atomic_clear_acq_64
624 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
625 #define atomic_load_acq_ptr atomic_load_acq_64
626 #define atomic_set_acq_ptr atomic_set_acq_64
627 #define atomic_subtract_acq_ptr atomic_subtract_acq_64
628
629 #define atomic_add_rel_long atomic_add_rel_64
630 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
631 #define atomic_clear_rel_long atomic_clear_rel_64
632 #define atomic_cmpset_rel_long atomic_cmpset_rel_64
633 #define atomic_set_rel_long atomic_set_rel_64
634 #define atomic_subtract_rel_long atomic_subtract_rel_64
635 #define atomic_store_rel_long atomic_store_rel_64
636
637 #define atomic_add_rel_ptr atomic_add_rel_64
638 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
639 #define atomic_clear_rel_ptr atomic_clear_rel_64
640 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
641 #define atomic_set_rel_ptr atomic_set_rel_64
642 #define atomic_subtract_rel_ptr atomic_subtract_rel_64
643 #define atomic_store_rel_ptr atomic_store_rel_64
644
645 static __inline void
646 atomic_thread_fence_acq(void)
647 {
648
649 dmb(ld);
650 }
651
652 static __inline void
atomic_thread_fence_rel(void)653 atomic_thread_fence_rel(void)
654 {
655
656 dmb(sy);
657 }
658
659 static __inline void
atomic_thread_fence_acq_rel(void)660 atomic_thread_fence_acq_rel(void)
661 {
662
663 dmb(sy);
664 }
665
666 static __inline void
atomic_thread_fence_seq_cst(void)667 atomic_thread_fence_seq_cst(void)
668 {
669
670 dmb(sy);
671 }
672
673 #endif /* KCSAN && !KCSAN_RUNTIME */
674 #endif /* _MACHINE_ATOMIC_H_ */
675
676 #endif /* !__arm__ */
677