1412042e2SAndrew Turner /*-
2412042e2SAndrew Turner * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
3412042e2SAndrew Turner * All rights reserved.
4412042e2SAndrew Turner *
5412042e2SAndrew Turner * Redistribution and use in source and binary forms, with or without
6412042e2SAndrew Turner * modification, are permitted provided that the following conditions
7412042e2SAndrew Turner * are met:
8412042e2SAndrew Turner * 1. Redistributions of source code must retain the above copyright
9412042e2SAndrew Turner * notice, this list of conditions and the following disclaimer.
10412042e2SAndrew Turner * 2. Redistributions in binary form must reproduce the above copyright
11412042e2SAndrew Turner * notice, this list of conditions and the following disclaimer in the
12412042e2SAndrew Turner * documentation and/or other materials provided with the distribution.
13412042e2SAndrew Turner *
14412042e2SAndrew Turner * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15412042e2SAndrew Turner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16412042e2SAndrew Turner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17412042e2SAndrew Turner * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18412042e2SAndrew Turner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19412042e2SAndrew Turner * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20412042e2SAndrew Turner * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21412042e2SAndrew Turner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22412042e2SAndrew Turner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23412042e2SAndrew Turner * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24412042e2SAndrew Turner * SUCH DAMAGE.
25412042e2SAndrew Turner */
26412042e2SAndrew Turner
27d5d97bedSMike Karels #ifdef __arm__
28d5d97bedSMike Karels #include <arm/atomic.h>
29d5d97bedSMike Karels #else /* !__arm__ */
30d5d97bedSMike Karels
31412042e2SAndrew Turner #ifndef _MACHINE_ATOMIC_H_
32412042e2SAndrew Turner #define _MACHINE_ATOMIC_H_
33412042e2SAndrew Turner
34412042e2SAndrew Turner #define isb() __asm __volatile("isb" : : : "memory")
35412042e2SAndrew Turner
36458f2175SZbigniew Bodek /*
37458f2175SZbigniew Bodek * Options for DMB and DSB:
38458f2175SZbigniew Bodek * oshld Outer Shareable, load
39458f2175SZbigniew Bodek * oshst Outer Shareable, store
40458f2175SZbigniew Bodek * osh Outer Shareable, all
41458f2175SZbigniew Bodek * nshld Non-shareable, load
42458f2175SZbigniew Bodek * nshst Non-shareable, store
43458f2175SZbigniew Bodek * nsh Non-shareable, all
44458f2175SZbigniew Bodek * ishld Inner Shareable, load
45458f2175SZbigniew Bodek * ishst Inner Shareable, store
46458f2175SZbigniew Bodek * ish Inner Shareable, all
47458f2175SZbigniew Bodek * ld Full system, load
48458f2175SZbigniew Bodek * st Full system, store
49458f2175SZbigniew Bodek * sy Full system, all
50458f2175SZbigniew Bodek */
51458f2175SZbigniew Bodek #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory")
52458f2175SZbigniew Bodek #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory")
53458f2175SZbigniew Bodek
54458f2175SZbigniew Bodek #define mb() dmb(sy) /* Full system memory barrier all */
55458f2175SZbigniew Bodek #define wmb() dmb(st) /* Full system memory barrier store */
56458f2175SZbigniew Bodek #define rmb() dmb(ld) /* Full system memory barrier load */
57412042e2SAndrew Turner
5889c52f9dSKyle Evans #ifdef _KERNEL
5989c52f9dSKyle Evans extern _Bool lse_supported;
6089c52f9dSKyle Evans #endif
6189c52f9dSKyle Evans
62d8da59ddSAndrew Turner #if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
63435c7cfbSMark Johnston #include <sys/atomic_san.h>
64849aef49SAndrew Turner #else
65849aef49SAndrew Turner
66849aef49SAndrew Turner #include <sys/atomic_common.h>
67849aef49SAndrew Turner
68a83c682bSMark Johnston #ifdef _KERNEL
69a83c682bSMark Johnston
70a83c682bSMark Johnston #ifdef LSE_ATOMICS
71a83c682bSMark Johnston #define _ATOMIC_LSE_SUPPORTED 1
72a83c682bSMark Johnston #else
73a83c682bSMark Johnston #define _ATOMIC_LSE_SUPPORTED lse_supported
74a83c682bSMark Johnston #endif
75a83c682bSMark Johnston #else
76a83c682bSMark Johnston #define _ATOMIC_LSE_SUPPORTED 0
77a83c682bSMark Johnston #endif
78a83c682bSMark Johnston
79c1fced68SMark Johnston #define _ATOMIC_OP_PROTO(t, op, bar, flav) \
80119a353eSAndrew Turner static __inline void \
81c1fced68SMark Johnston atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)
82c1fced68SMark Johnston
83920de6a1SMark Johnston #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
84c1fced68SMark Johnston _ATOMIC_OP_PROTO(t, op, bar, _llsc) \
854ffa494eSAndrew Turner { \
863ad6c736SMark Johnston uint##t##_t tmp; \
874ffa494eSAndrew Turner int res; \
884ffa494eSAndrew Turner \
89920de6a1SMark Johnston pre; \
904ffa494eSAndrew Turner __asm __volatile( \
913ad6c736SMark Johnston "1: ld"#a"xr"#s" %"#w"0, [%2]\n" \
92920de6a1SMark Johnston " "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \
933ad6c736SMark Johnston " st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \
94119a353eSAndrew Turner " cbnz %w1, 1b\n" \
95119a353eSAndrew Turner : "=&r"(tmp), "=&r"(res) \
96119a353eSAndrew Turner : "r" (p), "r" (val) \
97119a353eSAndrew Turner : "memory" \
98119a353eSAndrew Turner ); \
99c1fced68SMark Johnston } \
100c1fced68SMark Johnston \
101920de6a1SMark Johnston _ATOMIC_OP_PROTO(t, op, bar, _lse) \
102920de6a1SMark Johnston { \
103920de6a1SMark Johnston uint##t##_t tmp; \
104920de6a1SMark Johnston \
105920de6a1SMark Johnston pre; \
106920de6a1SMark Johnston __asm __volatile( \
107920de6a1SMark Johnston ".arch_extension lse\n" \
108920de6a1SMark Johnston "ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \
109920de6a1SMark Johnston ".arch_extension nolse\n" \
110920de6a1SMark Johnston : "=r" (tmp) \
111920de6a1SMark Johnston : "r" (p), "r" (val) \
112920de6a1SMark Johnston : "memory" \
113920de6a1SMark Johnston ); \
114920de6a1SMark Johnston } \
115920de6a1SMark Johnston \
116c1fced68SMark Johnston _ATOMIC_OP_PROTO(t, op, bar, ) \
117c1fced68SMark Johnston { \
118a83c682bSMark Johnston if (_ATOMIC_LSE_SUPPORTED) \
119a83c682bSMark Johnston atomic_##op##_##bar##t##_lse(p, val); \
120a83c682bSMark Johnston else \
121c1fced68SMark Johnston atomic_##op##_##bar##t##_llsc(p, val); \
122412042e2SAndrew Turner }
123412042e2SAndrew Turner
124920de6a1SMark Johnston #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
125920de6a1SMark Johnston _ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \
126920de6a1SMark Johnston bar, a, l) \
127920de6a1SMark Johnston _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \
128920de6a1SMark Johnston bar, a, l) \
129920de6a1SMark Johnston _ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \
130920de6a1SMark Johnston bar, a, l) \
131920de6a1SMark Johnston _ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \
132920de6a1SMark Johnston bar, a, l)
133412042e2SAndrew Turner
134920de6a1SMark Johnston #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \
135920de6a1SMark Johnston __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \
136920de6a1SMark Johnston __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \
137920de6a1SMark Johnston __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l)
138119a353eSAndrew Turner
139920de6a1SMark Johnston _ATOMIC_OP(add, add, add, )
140920de6a1SMark Johnston _ATOMIC_OP(clear, bic, clr, )
141920de6a1SMark Johnston _ATOMIC_OP(set, orr, set, )
142920de6a1SMark Johnston _ATOMIC_OP(subtract, add, add, val = -val)
1433ad6c736SMark Johnston
144c1fced68SMark Johnston #define _ATOMIC_CMPSET_PROTO(t, bar, flav) \
145dc5f9fcdSOlivier Houchard static __inline int \
146c1fced68SMark Johnston atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \
147c1fced68SMark Johnston uint##t##_t cmpval, uint##t##_t newval)
148c1fced68SMark Johnston
149c1fced68SMark Johnston #define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \
150c1fced68SMark Johnston static __inline int \
151c1fced68SMark Johnston atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \
152c1fced68SMark Johnston uint##t##_t *cmpval, uint##t##_t newval)
153c1fced68SMark Johnston
154c1fced68SMark Johnston #define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \
155c1fced68SMark Johnston _ATOMIC_CMPSET_PROTO(t, bar, _llsc) \
156e39ce4caSEmmanuel Vadot { \
1573ad6c736SMark Johnston uint##t##_t tmp; \
158e39ce4caSEmmanuel Vadot int res; \
159e39ce4caSEmmanuel Vadot \
160e39ce4caSEmmanuel Vadot __asm __volatile( \
161e39ce4caSEmmanuel Vadot "1: mov %w1, #1\n" \
1623ad6c736SMark Johnston " ld"#a"xr"#s" %"#w"0, [%2]\n" \
1633ad6c736SMark Johnston " cmp %"#w"0, %"#w"3\n" \
164e39ce4caSEmmanuel Vadot " b.ne 2f\n" \
1653ad6c736SMark Johnston " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \
1664ffa494eSAndrew Turner " cbnz %w1, 1b\n" \
1674ffa494eSAndrew Turner "2:" \
1684ffa494eSAndrew Turner : "=&r"(tmp), "=&r"(res) \
1694ffa494eSAndrew Turner : "r" (p), "r" (cmpval), "r" (newval) \
1704ffa494eSAndrew Turner : "cc", "memory" \
1714ffa494eSAndrew Turner ); \
1724ffa494eSAndrew Turner \
1734ffa494eSAndrew Turner return (!res); \
1744ffa494eSAndrew Turner } \
1754ffa494eSAndrew Turner \
176920de6a1SMark Johnston _ATOMIC_CMPSET_PROTO(t, bar, _lse) \
177920de6a1SMark Johnston { \
178920de6a1SMark Johnston uint##t##_t oldval; \
179920de6a1SMark Johnston int res; \
180920de6a1SMark Johnston \
181920de6a1SMark Johnston oldval = cmpval; \
182920de6a1SMark Johnston __asm __volatile( \
183920de6a1SMark Johnston ".arch_extension lse\n" \
184920de6a1SMark Johnston "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \
185920de6a1SMark Johnston "cmp %"#w"1, %"#w"2\n" \
186920de6a1SMark Johnston "cset %w0, eq\n" \
187920de6a1SMark Johnston ".arch_extension nolse\n" \
188920de6a1SMark Johnston : "=r" (res), "+&r" (cmpval) \
189920de6a1SMark Johnston : "r" (oldval), "r" (p), "r" (newval) \
190920de6a1SMark Johnston : "cc", "memory" \
191920de6a1SMark Johnston ); \
192920de6a1SMark Johnston \
193920de6a1SMark Johnston return (res); \
194920de6a1SMark Johnston } \
195920de6a1SMark Johnston \
196c1fced68SMark Johnston _ATOMIC_CMPSET_PROTO(t, bar, ) \
197c1fced68SMark Johnston { \
198a83c682bSMark Johnston if (_ATOMIC_LSE_SUPPORTED) \
199a83c682bSMark Johnston return (atomic_cmpset_##bar##t##_lse(p, cmpval, \
200a83c682bSMark Johnston newval)); \
201a83c682bSMark Johnston else \
202a83c682bSMark Johnston return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \
203a83c682bSMark Johnston newval)); \
204c1fced68SMark Johnston } \
205c1fced68SMark Johnston \
206c1fced68SMark Johnston _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \
2074ffa494eSAndrew Turner { \
2083ad6c736SMark Johnston uint##t##_t _cmpval, tmp; \
2093ad6c736SMark Johnston int res; \
2103ad6c736SMark Johnston \
2113ad6c736SMark Johnston _cmpval = *cmpval; \
2123ad6c736SMark Johnston __asm __volatile( \
2133ad6c736SMark Johnston " mov %w1, #1\n" \
2143ad6c736SMark Johnston " ld"#a"xr"#s" %"#w"0, [%2]\n" \
2153ad6c736SMark Johnston " cmp %"#w"0, %"#w"3\n" \
2163ad6c736SMark Johnston " b.ne 1f\n" \
2173ad6c736SMark Johnston " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \
2183ad6c736SMark Johnston "1:" \
2193ad6c736SMark Johnston : "=&r"(tmp), "=&r"(res) \
2203ad6c736SMark Johnston : "r" (p), "r" (_cmpval), "r" (newval) \
2213ad6c736SMark Johnston : "cc", "memory" \
2223ad6c736SMark Johnston ); \
2233ad6c736SMark Johnston *cmpval = tmp; \
2243ad6c736SMark Johnston \
2253ad6c736SMark Johnston return (!res); \
226c1fced68SMark Johnston } \
227c1fced68SMark Johnston \
228920de6a1SMark Johnston _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \
229920de6a1SMark Johnston { \
230920de6a1SMark Johnston uint##t##_t _cmpval, tmp; \
231920de6a1SMark Johnston int res; \
232920de6a1SMark Johnston \
233920de6a1SMark Johnston _cmpval = tmp = *cmpval; \
234920de6a1SMark Johnston __asm __volatile( \
235920de6a1SMark Johnston ".arch_extension lse\n" \
236920de6a1SMark Johnston "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \
237920de6a1SMark Johnston "cmp %"#w"1, %"#w"2\n" \
238920de6a1SMark Johnston "cset %w0, eq\n" \
239920de6a1SMark Johnston ".arch_extension nolse\n" \
240920de6a1SMark Johnston : "=r" (res), "+&r" (tmp) \
241920de6a1SMark Johnston : "r" (_cmpval), "r" (p), "r" (newval) \
242920de6a1SMark Johnston : "cc", "memory" \
243920de6a1SMark Johnston ); \
244920de6a1SMark Johnston *cmpval = tmp; \
245920de6a1SMark Johnston \
246920de6a1SMark Johnston return (res); \
247920de6a1SMark Johnston } \
248920de6a1SMark Johnston \
249c1fced68SMark Johnston _ATOMIC_FCMPSET_PROTO(t, bar, ) \
250c1fced68SMark Johnston { \
251a83c682bSMark Johnston if (_ATOMIC_LSE_SUPPORTED) \
252a83c682bSMark Johnston return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \
253a83c682bSMark Johnston newval)); \
254a83c682bSMark Johnston else \
255a83c682bSMark Johnston return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \
256a83c682bSMark Johnston newval)); \
2573ad6c736SMark Johnston }
2583ad6c736SMark Johnston
2593ad6c736SMark Johnston #define _ATOMIC_CMPSET(bar, a, l) \
2603ad6c736SMark Johnston _ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \
2613ad6c736SMark Johnston _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \
2623ad6c736SMark Johnston _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \
2633ad6c736SMark Johnston _ATOMIC_CMPSET_IMPL(64, , , bar, a, l)
2643ad6c736SMark Johnston
265ca0ec73cSConrad Meyer #define atomic_cmpset_8 atomic_cmpset_8
266ca0ec73cSConrad Meyer #define atomic_fcmpset_8 atomic_fcmpset_8
267ca0ec73cSConrad Meyer #define atomic_cmpset_16 atomic_cmpset_16
268ca0ec73cSConrad Meyer #define atomic_fcmpset_16 atomic_fcmpset_16
269ca0ec73cSConrad Meyer
2703ad6c736SMark Johnston _ATOMIC_CMPSET( , , )
2713ad6c736SMark Johnston _ATOMIC_CMPSET(acq_, a, )
2723ad6c736SMark Johnston _ATOMIC_CMPSET(rel_, ,l)
2733ad6c736SMark Johnston
274c1fced68SMark Johnston #define _ATOMIC_FETCHADD_PROTO(t, flav) \
2753ad6c736SMark Johnston static __inline uint##t##_t \
276c1fced68SMark Johnston atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
277c1fced68SMark Johnston
278c1fced68SMark Johnston #define _ATOMIC_FETCHADD_IMPL(t, w) \
279c1fced68SMark Johnston _ATOMIC_FETCHADD_PROTO(t, _llsc) \
2803ad6c736SMark Johnston { \
281920de6a1SMark Johnston uint##t##_t ret, tmp; \
2824ffa494eSAndrew Turner int res; \
2834ffa494eSAndrew Turner \
2844ffa494eSAndrew Turner __asm __volatile( \
2853ad6c736SMark Johnston "1: ldxr %"#w"2, [%3]\n" \
2863ad6c736SMark Johnston " add %"#w"0, %"#w"2, %"#w"4\n" \
2873ad6c736SMark Johnston " stxr %w1, %"#w"0, [%3]\n" \
2884ffa494eSAndrew Turner " cbnz %w1, 1b\n" \
2893ad6c736SMark Johnston : "=&r" (tmp), "=&r" (res), "=&r" (ret) \
2903ad6c736SMark Johnston : "r" (p), "r" (val) \
2913ad6c736SMark Johnston : "memory" \
2924ffa494eSAndrew Turner ); \
2934ffa494eSAndrew Turner \
2943ad6c736SMark Johnston return (ret); \
295c1fced68SMark Johnston } \
296c1fced68SMark Johnston \
297920de6a1SMark Johnston _ATOMIC_FETCHADD_PROTO(t, _lse) \
298920de6a1SMark Johnston { \
299920de6a1SMark Johnston uint##t##_t ret; \
300920de6a1SMark Johnston \
301920de6a1SMark Johnston __asm __volatile( \
302920de6a1SMark Johnston ".arch_extension lse\n" \
303920de6a1SMark Johnston "ldadd %"#w"2, %"#w"0, [%1]\n" \
304920de6a1SMark Johnston ".arch_extension nolse\n" \
305920de6a1SMark Johnston : "=r" (ret) \
306920de6a1SMark Johnston : "r" (p), "r" (val) \
307920de6a1SMark Johnston : "memory" \
308920de6a1SMark Johnston ); \
309920de6a1SMark Johnston \
310920de6a1SMark Johnston return (ret); \
311920de6a1SMark Johnston } \
312920de6a1SMark Johnston \
313c1fced68SMark Johnston _ATOMIC_FETCHADD_PROTO(t, ) \
314c1fced68SMark Johnston { \
315a83c682bSMark Johnston if (_ATOMIC_LSE_SUPPORTED) \
316a83c682bSMark Johnston return (atomic_fetchadd_##t##_lse(p, val)); \
317a83c682bSMark Johnston else \
318c1fced68SMark Johnston return (atomic_fetchadd_##t##_llsc(p, val)); \
3193ad6c736SMark Johnston }
3203ad6c736SMark Johnston
3213ad6c736SMark Johnston _ATOMIC_FETCHADD_IMPL(32, w)
3223ad6c736SMark Johnston _ATOMIC_FETCHADD_IMPL(64, )
3233ad6c736SMark Johnston
324c1fced68SMark Johnston #define _ATOMIC_SWAP_PROTO(t, flav) \
3253ad6c736SMark Johnston static __inline uint##t##_t \
326c1fced68SMark Johnston atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
327c1fced68SMark Johnston
328c1fced68SMark Johnston #define _ATOMIC_READANDCLEAR_PROTO(t, flav) \
329c1fced68SMark Johnston static __inline uint##t##_t \
330c1fced68SMark Johnston atomic_readandclear_##t##flav(volatile uint##t##_t *p)
331c1fced68SMark Johnston
332c1fced68SMark Johnston #define _ATOMIC_SWAP_IMPL(t, w, zreg) \
333c1fced68SMark Johnston _ATOMIC_SWAP_PROTO(t, _llsc) \
3343ad6c736SMark Johnston { \
3353ad6c736SMark Johnston uint##t##_t ret; \
3363ad6c736SMark Johnston int res; \
3373ad6c736SMark Johnston \
3383ad6c736SMark Johnston __asm __volatile( \
3393ad6c736SMark Johnston "1: ldxr %"#w"1, [%2]\n" \
3403ad6c736SMark Johnston " stxr %w0, %"#w"3, [%2]\n" \
3413ad6c736SMark Johnston " cbnz %w0, 1b\n" \
3423ad6c736SMark Johnston : "=&r" (res), "=&r" (ret) \
3433ad6c736SMark Johnston : "r" (p), "r" (val) \
3443ad6c736SMark Johnston : "memory" \
3453ad6c736SMark Johnston ); \
3463ad6c736SMark Johnston \
3473ad6c736SMark Johnston return (ret); \
3484ffa494eSAndrew Turner } \
3494ffa494eSAndrew Turner \
350920de6a1SMark Johnston _ATOMIC_SWAP_PROTO(t, _lse) \
351920de6a1SMark Johnston { \
352920de6a1SMark Johnston uint##t##_t ret; \
353920de6a1SMark Johnston \
354920de6a1SMark Johnston __asm __volatile( \
355920de6a1SMark Johnston ".arch_extension lse\n" \
356920de6a1SMark Johnston "swp %"#w"2, %"#w"0, [%1]\n" \
357920de6a1SMark Johnston ".arch_extension nolse\n" \
358920de6a1SMark Johnston : "=r" (ret) \
359920de6a1SMark Johnston : "r" (p), "r" (val) \
360920de6a1SMark Johnston : "memory" \
361920de6a1SMark Johnston ); \
362920de6a1SMark Johnston \
363920de6a1SMark Johnston return (ret); \
364920de6a1SMark Johnston } \
365920de6a1SMark Johnston \
366c1fced68SMark Johnston _ATOMIC_SWAP_PROTO(t, ) \
367c1fced68SMark Johnston { \
368a83c682bSMark Johnston if (_ATOMIC_LSE_SUPPORTED) \
369a83c682bSMark Johnston return (atomic_swap_##t##_lse(p, val)); \
370a83c682bSMark Johnston else \
371c1fced68SMark Johnston return (atomic_swap_##t##_llsc(p, val)); \
372c1fced68SMark Johnston } \
373c1fced68SMark Johnston \
374c1fced68SMark Johnston _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \
375119a353eSAndrew Turner { \
3763ad6c736SMark Johnston uint##t##_t ret; \
377119a353eSAndrew Turner int res; \
378119a353eSAndrew Turner \
379119a353eSAndrew Turner __asm __volatile( \
3803ad6c736SMark Johnston "1: ldxr %"#w"1, [%2]\n" \
3813ad6c736SMark Johnston " stxr %w0, "#zreg", [%2]\n" \
3823ad6c736SMark Johnston " cbnz %w0, 1b\n" \
3833ad6c736SMark Johnston : "=&r" (res), "=&r" (ret) \
3843ad6c736SMark Johnston : "r" (p) \
3853ad6c736SMark Johnston : "memory" \
386119a353eSAndrew Turner ); \
387119a353eSAndrew Turner \
3883ad6c736SMark Johnston return (ret); \
389c1fced68SMark Johnston } \
390c1fced68SMark Johnston \
391920de6a1SMark Johnston _ATOMIC_READANDCLEAR_PROTO(t, _lse) \
392920de6a1SMark Johnston { \
393920de6a1SMark Johnston return (atomic_swap_##t##_lse(p, 0)); \
394920de6a1SMark Johnston } \
395920de6a1SMark Johnston \
396c1fced68SMark Johnston _ATOMIC_READANDCLEAR_PROTO(t, ) \
397c1fced68SMark Johnston { \
398a83c682bSMark Johnston if (_ATOMIC_LSE_SUPPORTED) \
399a83c682bSMark Johnston return (atomic_readandclear_##t##_lse(p)); \
400a83c682bSMark Johnston else \
401c1fced68SMark Johnston return (atomic_readandclear_##t##_llsc(p)); \
4023ad6c736SMark Johnston }
4033ad6c736SMark Johnston
4043ad6c736SMark Johnston _ATOMIC_SWAP_IMPL(32, w, wzr)
4053ad6c736SMark Johnston _ATOMIC_SWAP_IMPL(64, , xzr)
4063ad6c736SMark Johnston
40702c16e21SAndrew Turner #define _ATOMIC_TEST_OP_PROTO(t, op, bar, flav) \
408119a353eSAndrew Turner static __inline int \
40902c16e21SAndrew Turner atomic_testand##op##_##bar##t##flav(volatile uint##t##_t *p, u_int val)
410c1fced68SMark Johnston
41102c16e21SAndrew Turner #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op, bar, a) \
41202c16e21SAndrew Turner _ATOMIC_TEST_OP_PROTO(t, op, bar, _llsc) \
413119a353eSAndrew Turner { \
4143ad6c736SMark Johnston uint##t##_t mask, old, tmp; \
415119a353eSAndrew Turner int res; \
416119a353eSAndrew Turner \
41782661227SRyan Libby mask = ((uint##t##_t)1) << (val & (t - 1)); \
418119a353eSAndrew Turner __asm __volatile( \
41902c16e21SAndrew Turner "1: ld"#a"xr %"#w"2, [%3]\n" \
420920de6a1SMark Johnston " "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \
4213ad6c736SMark Johnston " stxr %w1, %"#w"0, [%3]\n" \
422119a353eSAndrew Turner " cbnz %w1, 1b\n" \
4233ad6c736SMark Johnston : "=&r" (tmp), "=&r" (res), "=&r" (old) \
4243ad6c736SMark Johnston : "r" (p), "r" (mask) \
4253ad6c736SMark Johnston : "memory" \
426119a353eSAndrew Turner ); \
427119a353eSAndrew Turner \
4283ad6c736SMark Johnston return ((old & mask) != 0); \
429c1fced68SMark Johnston } \
430c1fced68SMark Johnston \
43102c16e21SAndrew Turner _ATOMIC_TEST_OP_PROTO(t, op, bar, _lse) \
432920de6a1SMark Johnston { \
433920de6a1SMark Johnston uint##t##_t mask, old; \
434920de6a1SMark Johnston \
43582661227SRyan Libby mask = ((uint##t##_t)1) << (val & (t - 1)); \
436920de6a1SMark Johnston __asm __volatile( \
437920de6a1SMark Johnston ".arch_extension lse\n" \
43802c16e21SAndrew Turner "ld"#lse_asm_op#a" %"#w"2, %"#w"0, [%1]\n" \
439920de6a1SMark Johnston ".arch_extension nolse\n" \
440920de6a1SMark Johnston : "=r" (old) \
441920de6a1SMark Johnston : "r" (p), "r" (mask) \
442920de6a1SMark Johnston : "memory" \
443920de6a1SMark Johnston ); \
444920de6a1SMark Johnston \
445920de6a1SMark Johnston return ((old & mask) != 0); \
446920de6a1SMark Johnston } \
447920de6a1SMark Johnston \
44802c16e21SAndrew Turner _ATOMIC_TEST_OP_PROTO(t, op, bar, ) \
449c1fced68SMark Johnston { \
450a83c682bSMark Johnston if (_ATOMIC_LSE_SUPPORTED) \
45102c16e21SAndrew Turner return (atomic_testand##op##_##bar##t##_lse(p, val)); \
452a83c682bSMark Johnston else \
45302c16e21SAndrew Turner return (atomic_testand##op##_##bar##t##_llsc(p, val)); \
454412042e2SAndrew Turner }
455412042e2SAndrew Turner
456920de6a1SMark Johnston #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \
45702c16e21SAndrew Turner _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, , ) \
45802c16e21SAndrew Turner _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, acq_, a) \
45902c16e21SAndrew Turner _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, , ) \
46002c16e21SAndrew Turner _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, acq_, a)
461412042e2SAndrew Turner
_ATOMIC_TEST_OP(clear,bic,clr)462920de6a1SMark Johnston _ATOMIC_TEST_OP(clear, bic, clr)
463920de6a1SMark Johnston _ATOMIC_TEST_OP(set, orr, set)
464412042e2SAndrew Turner
4653ad6c736SMark Johnston #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \
4663ad6c736SMark Johnston static __inline uint##t##_t \
4673ad6c736SMark Johnston atomic_load_acq_##t(volatile uint##t##_t *p) \
4683ad6c736SMark Johnston { \
4693ad6c736SMark Johnston uint##t##_t ret; \
4703ad6c736SMark Johnston \
4713ad6c736SMark Johnston __asm __volatile( \
4723ad6c736SMark Johnston "ldar"#s" %"#w"0, [%1]\n" \
4733ad6c736SMark Johnston : "=&r" (ret) \
4743ad6c736SMark Johnston : "r" (p) \
4753ad6c736SMark Johnston : "memory"); \
4763ad6c736SMark Johnston \
4773ad6c736SMark Johnston return (ret); \
478119a353eSAndrew Turner }
479119a353eSAndrew Turner
480ca0ec73cSConrad Meyer #define atomic_load_acq_8 atomic_load_acq_8
481ca0ec73cSConrad Meyer #define atomic_load_acq_16 atomic_load_acq_16
4823ad6c736SMark Johnston _ATOMIC_LOAD_ACQ_IMPL(8, w, b)
4833ad6c736SMark Johnston _ATOMIC_LOAD_ACQ_IMPL(16, w, h)
4843ad6c736SMark Johnston _ATOMIC_LOAD_ACQ_IMPL(32, w, )
4853ad6c736SMark Johnston _ATOMIC_LOAD_ACQ_IMPL(64, , )
486119a353eSAndrew Turner
4873ad6c736SMark Johnston #define _ATOMIC_STORE_REL_IMPL(t, w, s) \
4883ad6c736SMark Johnston static __inline void \
4893ad6c736SMark Johnston atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val) \
4903ad6c736SMark Johnston { \
4913ad6c736SMark Johnston __asm __volatile( \
4923ad6c736SMark Johnston "stlr"#s" %"#w"0, [%1]\n" \
4933ad6c736SMark Johnston : \
4943ad6c736SMark Johnston : "r" (val), "r" (p) \
4953ad6c736SMark Johnston : "memory"); \
496412042e2SAndrew Turner }
497412042e2SAndrew Turner
4983ad6c736SMark Johnston _ATOMIC_STORE_REL_IMPL(8, w, b)
4993ad6c736SMark Johnston _ATOMIC_STORE_REL_IMPL(16, w, h)
5003ad6c736SMark Johnston _ATOMIC_STORE_REL_IMPL(32, w, )
5013ad6c736SMark Johnston _ATOMIC_STORE_REL_IMPL(64, , )
502119a353eSAndrew Turner
503ec1ecf78SAndrew Turner #define atomic_add_char atomic_add_8
504ec1ecf78SAndrew Turner #define atomic_fcmpset_char atomic_fcmpset_8
505ec1ecf78SAndrew Turner #define atomic_clear_char atomic_clear_8
506ec1ecf78SAndrew Turner #define atomic_cmpset_char atomic_cmpset_8
507ec1ecf78SAndrew Turner #define atomic_fetchadd_char atomic_fetchadd_8
508ec1ecf78SAndrew Turner #define atomic_readandclear_char atomic_readandclear_8
509ec1ecf78SAndrew Turner #define atomic_set_char atomic_set_8
510ec1ecf78SAndrew Turner #define atomic_swap_char atomic_swap_8
511ec1ecf78SAndrew Turner #define atomic_subtract_char atomic_subtract_8
512ec1ecf78SAndrew Turner #define atomic_testandclear_char atomic_testandclear_8
513ec1ecf78SAndrew Turner #define atomic_testandset_char atomic_testandset_8
514ec1ecf78SAndrew Turner
515ec1ecf78SAndrew Turner #define atomic_add_acq_char atomic_add_acq_8
516ec1ecf78SAndrew Turner #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8
517ec1ecf78SAndrew Turner #define atomic_clear_acq_char atomic_clear_acq_8
518ec1ecf78SAndrew Turner #define atomic_cmpset_acq_char atomic_cmpset_acq_8
519ec1ecf78SAndrew Turner #define atomic_load_acq_char atomic_load_acq_8
520ec1ecf78SAndrew Turner #define atomic_set_acq_char atomic_set_acq_8
521ec1ecf78SAndrew Turner #define atomic_subtract_acq_char atomic_subtract_acq_8
522ec1ecf78SAndrew Turner #define atomic_testandset_acq_char atomic_testandset_acq_8
523ec1ecf78SAndrew Turner
524ec1ecf78SAndrew Turner #define atomic_add_rel_char atomic_add_rel_8
525ec1ecf78SAndrew Turner #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8
526ec1ecf78SAndrew Turner #define atomic_clear_rel_char atomic_clear_rel_8
527ec1ecf78SAndrew Turner #define atomic_cmpset_rel_char atomic_cmpset_rel_8
528ec1ecf78SAndrew Turner #define atomic_set_rel_char atomic_set_rel_8
529ec1ecf78SAndrew Turner #define atomic_subtract_rel_char atomic_subtract_rel_8
530ec1ecf78SAndrew Turner #define atomic_store_rel_char atomic_store_rel_8
531ec1ecf78SAndrew Turner
532ec1ecf78SAndrew Turner #define atomic_add_short atomic_add_16
533ec1ecf78SAndrew Turner #define atomic_fcmpset_short atomic_fcmpset_16
534ec1ecf78SAndrew Turner #define atomic_clear_short atomic_clear_16
535ec1ecf78SAndrew Turner #define atomic_cmpset_short atomic_cmpset_16
536ec1ecf78SAndrew Turner #define atomic_fetchadd_short atomic_fetchadd_16
537ec1ecf78SAndrew Turner #define atomic_readandclear_short atomic_readandclear_16
538ec1ecf78SAndrew Turner #define atomic_set_short atomic_set_16
539ec1ecf78SAndrew Turner #define atomic_swap_short atomic_swap_16
540ec1ecf78SAndrew Turner #define atomic_subtract_short atomic_subtract_16
541ec1ecf78SAndrew Turner #define atomic_testandclear_short atomic_testandclear_16
542ec1ecf78SAndrew Turner #define atomic_testandset_short atomic_testandset_16
543ec1ecf78SAndrew Turner
544ec1ecf78SAndrew Turner #define atomic_add_acq_short atomic_add_acq_16
545ec1ecf78SAndrew Turner #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16
546ec1ecf78SAndrew Turner #define atomic_clear_acq_short atomic_clear_acq_16
547ec1ecf78SAndrew Turner #define atomic_cmpset_acq_short atomic_cmpset_acq_16
548ec1ecf78SAndrew Turner #define atomic_load_acq_short atomic_load_acq_16
549ec1ecf78SAndrew Turner #define atomic_set_acq_short atomic_set_acq_16
550ec1ecf78SAndrew Turner #define atomic_subtract_acq_short atomic_subtract_acq_16
551ec1ecf78SAndrew Turner #define atomic_testandset_acq_short atomic_testandset_acq_16
552ec1ecf78SAndrew Turner
553ec1ecf78SAndrew Turner #define atomic_add_rel_short atomic_add_rel_16
554ec1ecf78SAndrew Turner #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16
555ec1ecf78SAndrew Turner #define atomic_clear_rel_short atomic_clear_rel_16
556ec1ecf78SAndrew Turner #define atomic_cmpset_rel_short atomic_cmpset_rel_16
557ec1ecf78SAndrew Turner #define atomic_set_rel_short atomic_set_rel_16
558ec1ecf78SAndrew Turner #define atomic_subtract_rel_short atomic_subtract_rel_16
559ec1ecf78SAndrew Turner #define atomic_store_rel_short atomic_store_rel_16
560ec1ecf78SAndrew Turner
561412042e2SAndrew Turner #define atomic_add_int atomic_add_32
562dc5f9fcdSOlivier Houchard #define atomic_fcmpset_int atomic_fcmpset_32
563412042e2SAndrew Turner #define atomic_clear_int atomic_clear_32
564412042e2SAndrew Turner #define atomic_cmpset_int atomic_cmpset_32
565412042e2SAndrew Turner #define atomic_fetchadd_int atomic_fetchadd_32
566412042e2SAndrew Turner #define atomic_readandclear_int atomic_readandclear_32
567412042e2SAndrew Turner #define atomic_set_int atomic_set_32
5689b8c3c4fSAndrew Turner #define atomic_swap_int atomic_swap_32
569412042e2SAndrew Turner #define atomic_subtract_int atomic_subtract_32
570f2792e5eSAndrew Turner #define atomic_testandclear_int atomic_testandclear_32
571f2792e5eSAndrew Turner #define atomic_testandset_int atomic_testandset_32
572412042e2SAndrew Turner
57346f52b02SAndrew Turner #define atomic_add_acq_int atomic_add_acq_32
574dc5f9fcdSOlivier Houchard #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
57546f52b02SAndrew Turner #define atomic_clear_acq_int atomic_clear_acq_32
57646f52b02SAndrew Turner #define atomic_cmpset_acq_int atomic_cmpset_acq_32
57746f52b02SAndrew Turner #define atomic_load_acq_int atomic_load_acq_32
57846f52b02SAndrew Turner #define atomic_set_acq_int atomic_set_acq_32
57946f52b02SAndrew Turner #define atomic_subtract_acq_int atomic_subtract_acq_32
58002c16e21SAndrew Turner #define atomic_testandset_acq_int atomic_testandset_acq_32
58146f52b02SAndrew Turner
582412042e2SAndrew Turner #define atomic_add_rel_int atomic_add_rel_32
583dc5f9fcdSOlivier Houchard #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
584c90baf68SAndrew Turner #define atomic_clear_rel_int atomic_clear_rel_32
585412042e2SAndrew Turner #define atomic_cmpset_rel_int atomic_cmpset_rel_32
586412042e2SAndrew Turner #define atomic_set_rel_int atomic_set_rel_32
587412042e2SAndrew Turner #define atomic_subtract_rel_int atomic_subtract_rel_32
588412042e2SAndrew Turner #define atomic_store_rel_int atomic_store_rel_32
589412042e2SAndrew Turner
590412042e2SAndrew Turner #define atomic_add_long atomic_add_64
591dc5f9fcdSOlivier Houchard #define atomic_fcmpset_long atomic_fcmpset_64
592412042e2SAndrew Turner #define atomic_clear_long atomic_clear_64
593412042e2SAndrew Turner #define atomic_cmpset_long atomic_cmpset_64
594412042e2SAndrew Turner #define atomic_fetchadd_long atomic_fetchadd_64
595412042e2SAndrew Turner #define atomic_readandclear_long atomic_readandclear_64
596412042e2SAndrew Turner #define atomic_set_long atomic_set_64
5979b8c3c4fSAndrew Turner #define atomic_swap_long atomic_swap_64
598412042e2SAndrew Turner #define atomic_subtract_long atomic_subtract_64
599f2792e5eSAndrew Turner #define atomic_testandclear_long atomic_testandclear_64
600f2792e5eSAndrew Turner #define atomic_testandset_long atomic_testandset_64
601412042e2SAndrew Turner
602412042e2SAndrew Turner #define atomic_add_ptr atomic_add_64
603dc5f9fcdSOlivier Houchard #define atomic_fcmpset_ptr atomic_fcmpset_64
604412042e2SAndrew Turner #define atomic_clear_ptr atomic_clear_64
605412042e2SAndrew Turner #define atomic_cmpset_ptr atomic_cmpset_64
606412042e2SAndrew Turner #define atomic_fetchadd_ptr atomic_fetchadd_64
607412042e2SAndrew Turner #define atomic_readandclear_ptr atomic_readandclear_64
608412042e2SAndrew Turner #define atomic_set_ptr atomic_set_64
6099b8c3c4fSAndrew Turner #define atomic_swap_ptr atomic_swap_64
610412042e2SAndrew Turner #define atomic_subtract_ptr atomic_subtract_64
611412042e2SAndrew Turner
612412042e2SAndrew Turner #define atomic_add_acq_long atomic_add_acq_64
613dc5f9fcdSOlivier Houchard #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
614c90baf68SAndrew Turner #define atomic_clear_acq_long atomic_clear_acq_64
615412042e2SAndrew Turner #define atomic_cmpset_acq_long atomic_cmpset_acq_64
616412042e2SAndrew Turner #define atomic_load_acq_long atomic_load_acq_64
617412042e2SAndrew Turner #define atomic_set_acq_long atomic_set_acq_64
618412042e2SAndrew Turner #define atomic_subtract_acq_long atomic_subtract_acq_64
61902c16e21SAndrew Turner #define atomic_testandset_acq_long atomic_testandset_acq_64
620412042e2SAndrew Turner
621412042e2SAndrew Turner #define atomic_add_acq_ptr atomic_add_acq_64
622dc5f9fcdSOlivier Houchard #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
623c90baf68SAndrew Turner #define atomic_clear_acq_ptr atomic_clear_acq_64
624412042e2SAndrew Turner #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
625412042e2SAndrew Turner #define atomic_load_acq_ptr atomic_load_acq_64
626412042e2SAndrew Turner #define atomic_set_acq_ptr atomic_set_acq_64
627412042e2SAndrew Turner #define atomic_subtract_acq_ptr atomic_subtract_acq_64
628412042e2SAndrew Turner
629119a353eSAndrew Turner #define atomic_add_rel_long atomic_add_rel_64
630dc5f9fcdSOlivier Houchard #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
631119a353eSAndrew Turner #define atomic_clear_rel_long atomic_clear_rel_64
632119a353eSAndrew Turner #define atomic_cmpset_rel_long atomic_cmpset_rel_64
633119a353eSAndrew Turner #define atomic_set_rel_long atomic_set_rel_64
634119a353eSAndrew Turner #define atomic_subtract_rel_long atomic_subtract_rel_64
635119a353eSAndrew Turner #define atomic_store_rel_long atomic_store_rel_64
63646f52b02SAndrew Turner
637119a353eSAndrew Turner #define atomic_add_rel_ptr atomic_add_rel_64
638dc5f9fcdSOlivier Houchard #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
639119a353eSAndrew Turner #define atomic_clear_rel_ptr atomic_clear_rel_64
640119a353eSAndrew Turner #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
641119a353eSAndrew Turner #define atomic_set_rel_ptr atomic_set_rel_64
642119a353eSAndrew Turner #define atomic_subtract_rel_ptr atomic_subtract_rel_64
643119a353eSAndrew Turner #define atomic_store_rel_ptr atomic_store_rel_64
644412042e2SAndrew Turner
6458954a9a4SKonstantin Belousov static __inline void
6468954a9a4SKonstantin Belousov atomic_thread_fence_acq(void)
6478954a9a4SKonstantin Belousov {
6488954a9a4SKonstantin Belousov
6498954a9a4SKonstantin Belousov dmb(ld);
6508954a9a4SKonstantin Belousov }
6518954a9a4SKonstantin Belousov
6528954a9a4SKonstantin Belousov static __inline void
atomic_thread_fence_rel(void)6538954a9a4SKonstantin Belousov atomic_thread_fence_rel(void)
6548954a9a4SKonstantin Belousov {
6558954a9a4SKonstantin Belousov
6568954a9a4SKonstantin Belousov dmb(sy);
6578954a9a4SKonstantin Belousov }
6588954a9a4SKonstantin Belousov
6598954a9a4SKonstantin Belousov static __inline void
atomic_thread_fence_acq_rel(void)6608954a9a4SKonstantin Belousov atomic_thread_fence_acq_rel(void)
6618954a9a4SKonstantin Belousov {
6628954a9a4SKonstantin Belousov
6638954a9a4SKonstantin Belousov dmb(sy);
6648954a9a4SKonstantin Belousov }
6658954a9a4SKonstantin Belousov
6668954a9a4SKonstantin Belousov static __inline void
atomic_thread_fence_seq_cst(void)6678954a9a4SKonstantin Belousov atomic_thread_fence_seq_cst(void)
6688954a9a4SKonstantin Belousov {
6698954a9a4SKonstantin Belousov
6708954a9a4SKonstantin Belousov dmb(sy);
6718954a9a4SKonstantin Belousov }
6728954a9a4SKonstantin Belousov
673849aef49SAndrew Turner #endif /* KCSAN && !KCSAN_RUNTIME */
674412042e2SAndrew Turner #endif /* _MACHINE_ATOMIC_H_ */
675d5d97bedSMike Karels
676d5d97bedSMike Karels #endif /* !__arm__ */
677