18d7e7a98SRuslan Bukin /*-
28d7e7a98SRuslan Bukin * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
38d7e7a98SRuslan Bukin * All rights reserved.
48d7e7a98SRuslan Bukin *
58d7e7a98SRuslan Bukin * Portions of this software were developed by SRI International and the
68d7e7a98SRuslan Bukin * University of Cambridge Computer Laboratory under DARPA/AFRL contract
78d7e7a98SRuslan Bukin * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
88d7e7a98SRuslan Bukin *
98d7e7a98SRuslan Bukin * Portions of this software were developed by the University of Cambridge
108d7e7a98SRuslan Bukin * Computer Laboratory as part of the CTSRD Project, with support from the
118d7e7a98SRuslan Bukin * UK Higher Education Innovation Fund (HEIF).
128d7e7a98SRuslan Bukin *
138d7e7a98SRuslan Bukin * Redistribution and use in source and binary forms, with or without
148d7e7a98SRuslan Bukin * modification, are permitted provided that the following conditions
158d7e7a98SRuslan Bukin * are met:
168d7e7a98SRuslan Bukin * 1. Redistributions of source code must retain the above copyright
178d7e7a98SRuslan Bukin * notice, this list of conditions and the following disclaimer.
188d7e7a98SRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright
198d7e7a98SRuslan Bukin * notice, this list of conditions and the following disclaimer in the
208d7e7a98SRuslan Bukin * documentation and/or other materials provided with the distribution.
218d7e7a98SRuslan Bukin *
228d7e7a98SRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
238d7e7a98SRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
248d7e7a98SRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
258d7e7a98SRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
268d7e7a98SRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
278d7e7a98SRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
288d7e7a98SRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
298d7e7a98SRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
308d7e7a98SRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
318d7e7a98SRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
328d7e7a98SRuslan Bukin * SUCH DAMAGE.
338d7e7a98SRuslan Bukin */
348d7e7a98SRuslan Bukin
358d7e7a98SRuslan Bukin #ifndef _MACHINE_ATOMIC_H_
368d7e7a98SRuslan Bukin #define _MACHINE_ATOMIC_H_
378d7e7a98SRuslan Bukin
3830d4f9e8SKonstantin Belousov #include <sys/atomic_common.h>
3930d4f9e8SKonstantin Belousov
408d7e7a98SRuslan Bukin #define fence() __asm __volatile("fence" ::: "memory");
418d7e7a98SRuslan Bukin #define mb() fence()
428d7e7a98SRuslan Bukin #define rmb() fence()
438d7e7a98SRuslan Bukin #define wmb() fence()
448d7e7a98SRuslan Bukin
4529a5f639SKyle Evans static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t);
4629a5f639SKyle Evans static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t);
4729a5f639SKyle Evans static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t);
4829a5f639SKyle Evans static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *,
4929a5f639SKyle Evans uint16_t);
5029a5f639SKyle Evans
518d7e7a98SRuslan Bukin #define ATOMIC_ACQ_REL(NAME, WIDTH) \
528d7e7a98SRuslan Bukin static __inline void \
538d7e7a98SRuslan Bukin atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
548d7e7a98SRuslan Bukin { \
558d7e7a98SRuslan Bukin atomic_##NAME##_##WIDTH(p, v); \
568d7e7a98SRuslan Bukin fence(); \
578d7e7a98SRuslan Bukin } \
588d7e7a98SRuslan Bukin \
598d7e7a98SRuslan Bukin static __inline void \
608d7e7a98SRuslan Bukin atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
618d7e7a98SRuslan Bukin { \
628d7e7a98SRuslan Bukin fence(); \
638d7e7a98SRuslan Bukin atomic_##NAME##_##WIDTH(p, v); \
648d7e7a98SRuslan Bukin }
658d7e7a98SRuslan Bukin
6629a5f639SKyle Evans #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \
6729a5f639SKyle Evans static __inline int \
6829a5f639SKyle Evans atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \
6929a5f639SKyle Evans uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \
7029a5f639SKyle Evans { \
7129a5f639SKyle Evans int retval; \
7229a5f639SKyle Evans \
7329a5f639SKyle Evans retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \
7429a5f639SKyle Evans fence(); \
7529a5f639SKyle Evans return (retval); \
7629a5f639SKyle Evans } \
7729a5f639SKyle Evans \
7829a5f639SKyle Evans static __inline int \
7929a5f639SKyle Evans atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \
8029a5f639SKyle Evans uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \
8129a5f639SKyle Evans { \
8229a5f639SKyle Evans fence(); \
8329a5f639SKyle Evans return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \
8429a5f639SKyle Evans }
8529a5f639SKyle Evans
8629a5f639SKyle Evans #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \
8729a5f639SKyle Evans static __inline int \
8829a5f639SKyle Evans atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \
8929a5f639SKyle Evans uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \
9029a5f639SKyle Evans { \
9129a5f639SKyle Evans int retval; \
9229a5f639SKyle Evans \
9329a5f639SKyle Evans retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \
9429a5f639SKyle Evans fence(); \
9529a5f639SKyle Evans return (retval); \
9629a5f639SKyle Evans } \
9729a5f639SKyle Evans \
9829a5f639SKyle Evans static __inline int \
9929a5f639SKyle Evans atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \
10029a5f639SKyle Evans uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \
10129a5f639SKyle Evans { \
10229a5f639SKyle Evans fence(); \
10329a5f639SKyle Evans return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \
10429a5f639SKyle Evans }
10529a5f639SKyle Evans
10629a5f639SKyle Evans ATOMIC_CMPSET_ACQ_REL(8);
10729a5f639SKyle Evans ATOMIC_FCMPSET_ACQ_REL(8);
10829a5f639SKyle Evans ATOMIC_CMPSET_ACQ_REL(16);
10929a5f639SKyle Evans ATOMIC_FCMPSET_ACQ_REL(16);
11029a5f639SKyle Evans
11129a5f639SKyle Evans #define atomic_cmpset_char atomic_cmpset_8
11229a5f639SKyle Evans #define atomic_cmpset_acq_char atomic_cmpset_acq_8
11329a5f639SKyle Evans #define atomic_cmpset_rel_char atomic_cmpset_rel_8
11429a5f639SKyle Evans #define atomic_fcmpset_char atomic_fcmpset_8
11529a5f639SKyle Evans #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8
11629a5f639SKyle Evans #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8
11729a5f639SKyle Evans
11829a5f639SKyle Evans #define atomic_cmpset_short atomic_cmpset_16
11929a5f639SKyle Evans #define atomic_cmpset_acq_short atomic_cmpset_acq_16
12029a5f639SKyle Evans #define atomic_cmpset_rel_short atomic_cmpset_rel_16
12129a5f639SKyle Evans #define atomic_fcmpset_short atomic_fcmpset_16
12229a5f639SKyle Evans #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16
12329a5f639SKyle Evans #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16
12429a5f639SKyle Evans
1258d7e7a98SRuslan Bukin static __inline void
atomic_add_32(volatile uint32_t * p,uint32_t val)1268d7e7a98SRuslan Bukin atomic_add_32(volatile uint32_t *p, uint32_t val)
1278d7e7a98SRuslan Bukin {
1288d7e7a98SRuslan Bukin
1298d7e7a98SRuslan Bukin __asm __volatile("amoadd.w zero, %1, %0"
1308d7e7a98SRuslan Bukin : "+A" (*p)
1318d7e7a98SRuslan Bukin : "r" (val)
1328d7e7a98SRuslan Bukin : "memory");
1338d7e7a98SRuslan Bukin }
1348d7e7a98SRuslan Bukin
1358d7e7a98SRuslan Bukin static __inline void
atomic_subtract_32(volatile uint32_t * p,uint32_t val)1368d7e7a98SRuslan Bukin atomic_subtract_32(volatile uint32_t *p, uint32_t val)
1378d7e7a98SRuslan Bukin {
1388d7e7a98SRuslan Bukin
1398d7e7a98SRuslan Bukin __asm __volatile("amoadd.w zero, %1, %0"
1408d7e7a98SRuslan Bukin : "+A" (*p)
1418d7e7a98SRuslan Bukin : "r" (-val)
1428d7e7a98SRuslan Bukin : "memory");
1438d7e7a98SRuslan Bukin }
1448d7e7a98SRuslan Bukin
1458d7e7a98SRuslan Bukin static __inline void
atomic_set_32(volatile uint32_t * p,uint32_t val)1468d7e7a98SRuslan Bukin atomic_set_32(volatile uint32_t *p, uint32_t val)
1478d7e7a98SRuslan Bukin {
1488d7e7a98SRuslan Bukin
1498d7e7a98SRuslan Bukin __asm __volatile("amoor.w zero, %1, %0"
1508d7e7a98SRuslan Bukin : "+A" (*p)
1518d7e7a98SRuslan Bukin : "r" (val)
1528d7e7a98SRuslan Bukin : "memory");
1538d7e7a98SRuslan Bukin }
1548d7e7a98SRuslan Bukin
1558d7e7a98SRuslan Bukin static __inline void
atomic_clear_32(volatile uint32_t * p,uint32_t val)1568d7e7a98SRuslan Bukin atomic_clear_32(volatile uint32_t *p, uint32_t val)
1578d7e7a98SRuslan Bukin {
1588d7e7a98SRuslan Bukin
1598d7e7a98SRuslan Bukin __asm __volatile("amoand.w zero, %1, %0"
1608d7e7a98SRuslan Bukin : "+A" (*p)
1618d7e7a98SRuslan Bukin : "r" (~val)
1628d7e7a98SRuslan Bukin : "memory");
1638d7e7a98SRuslan Bukin }
1648d7e7a98SRuslan Bukin
1658d7e7a98SRuslan Bukin static __inline int
atomic_cmpset_32(volatile uint32_t * p,uint32_t cmpval,uint32_t newval)1668d7e7a98SRuslan Bukin atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
1678d7e7a98SRuslan Bukin {
1688d7e7a98SRuslan Bukin uint32_t tmp;
1698d7e7a98SRuslan Bukin int res;
1708d7e7a98SRuslan Bukin
1718d7e7a98SRuslan Bukin res = 0;
1728d7e7a98SRuslan Bukin
1738d7e7a98SRuslan Bukin __asm __volatile(
1748d7e7a98SRuslan Bukin "0:"
1758d7e7a98SRuslan Bukin "li %1, 1\n" /* Preset to fail */
1768d7e7a98SRuslan Bukin "lr.w %0, %2\n"
1778d7e7a98SRuslan Bukin "bne %0, %z3, 1f\n"
1788d7e7a98SRuslan Bukin "sc.w %1, %z4, %2\n"
1798d7e7a98SRuslan Bukin "bnez %1, 0b\n"
1808d7e7a98SRuslan Bukin "1:"
1818d7e7a98SRuslan Bukin : "=&r" (tmp), "=&r" (res), "+A" (*p)
182b96562ebSJohn Baldwin : "rJ" ((long)(int32_t)cmpval), "rJ" (newval)
1838d7e7a98SRuslan Bukin : "memory");
1848d7e7a98SRuslan Bukin
1858d7e7a98SRuslan Bukin return (!res);
1868d7e7a98SRuslan Bukin }
1878d7e7a98SRuslan Bukin
18843b595f6SRuslan Bukin static __inline int
atomic_fcmpset_32(volatile uint32_t * p,uint32_t * cmpval,uint32_t newval)18943b595f6SRuslan Bukin atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
19043b595f6SRuslan Bukin {
19143b595f6SRuslan Bukin uint32_t tmp;
19243b595f6SRuslan Bukin int res;
19343b595f6SRuslan Bukin
19443b595f6SRuslan Bukin res = 0;
19543b595f6SRuslan Bukin
19643b595f6SRuslan Bukin __asm __volatile(
19743b595f6SRuslan Bukin "0:"
19843b595f6SRuslan Bukin "li %1, 1\n" /* Preset to fail */
19943b595f6SRuslan Bukin "lr.w %0, %2\n" /* Load old value */
20043b595f6SRuslan Bukin "bne %0, %z4, 1f\n" /* Compare */
20143b595f6SRuslan Bukin "sc.w %1, %z5, %2\n" /* Try to store new value */
20243b595f6SRuslan Bukin "j 2f\n"
20343b595f6SRuslan Bukin "1:"
20443b595f6SRuslan Bukin "sw %0, %3\n" /* Save old value */
20543b595f6SRuslan Bukin "2:"
20643b595f6SRuslan Bukin : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
207b96562ebSJohn Baldwin : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval)
20843b595f6SRuslan Bukin : "memory");
20943b595f6SRuslan Bukin
21043b595f6SRuslan Bukin return (!res);
21143b595f6SRuslan Bukin }
21243b595f6SRuslan Bukin
2138d7e7a98SRuslan Bukin static __inline uint32_t
atomic_fetchadd_32(volatile uint32_t * p,uint32_t val)2148d7e7a98SRuslan Bukin atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
2158d7e7a98SRuslan Bukin {
2168d7e7a98SRuslan Bukin uint32_t ret;
2178d7e7a98SRuslan Bukin
2188d7e7a98SRuslan Bukin __asm __volatile("amoadd.w %0, %2, %1"
2198d7e7a98SRuslan Bukin : "=&r" (ret), "+A" (*p)
2208d7e7a98SRuslan Bukin : "r" (val)
2218d7e7a98SRuslan Bukin : "memory");
2228d7e7a98SRuslan Bukin
2238d7e7a98SRuslan Bukin return (ret);
2248d7e7a98SRuslan Bukin }
2258d7e7a98SRuslan Bukin
2268d7e7a98SRuslan Bukin static __inline uint32_t
atomic_readandclear_32(volatile uint32_t * p)2278d7e7a98SRuslan Bukin atomic_readandclear_32(volatile uint32_t *p)
2288d7e7a98SRuslan Bukin {
2298d7e7a98SRuslan Bukin uint32_t ret;
2308d7e7a98SRuslan Bukin uint32_t val;
2318d7e7a98SRuslan Bukin
2328d7e7a98SRuslan Bukin val = 0;
2338d7e7a98SRuslan Bukin
2348d7e7a98SRuslan Bukin __asm __volatile("amoswap.w %0, %2, %1"
2358d7e7a98SRuslan Bukin : "=&r"(ret), "+A" (*p)
2368d7e7a98SRuslan Bukin : "r" (val)
2378d7e7a98SRuslan Bukin : "memory");
2388d7e7a98SRuslan Bukin
2398d7e7a98SRuslan Bukin return (ret);
2408d7e7a98SRuslan Bukin }
2418d7e7a98SRuslan Bukin
2428d7e7a98SRuslan Bukin #define atomic_add_int atomic_add_32
2438d7e7a98SRuslan Bukin #define atomic_clear_int atomic_clear_32
2448d7e7a98SRuslan Bukin #define atomic_cmpset_int atomic_cmpset_32
24543b595f6SRuslan Bukin #define atomic_fcmpset_int atomic_fcmpset_32
2468d7e7a98SRuslan Bukin #define atomic_fetchadd_int atomic_fetchadd_32
2478d7e7a98SRuslan Bukin #define atomic_readandclear_int atomic_readandclear_32
2488d7e7a98SRuslan Bukin #define atomic_set_int atomic_set_32
2498d7e7a98SRuslan Bukin #define atomic_subtract_int atomic_subtract_32
2508d7e7a98SRuslan Bukin
2518d7e7a98SRuslan Bukin ATOMIC_ACQ_REL(set, 32)
2528d7e7a98SRuslan Bukin ATOMIC_ACQ_REL(clear, 32)
2538d7e7a98SRuslan Bukin ATOMIC_ACQ_REL(add, 32)
2548d7e7a98SRuslan Bukin ATOMIC_ACQ_REL(subtract, 32)
2558d7e7a98SRuslan Bukin
25629a5f639SKyle Evans ATOMIC_CMPSET_ACQ_REL(32);
25729a5f639SKyle Evans ATOMIC_FCMPSET_ACQ_REL(32);
25843b595f6SRuslan Bukin
2598d7e7a98SRuslan Bukin static __inline uint32_t
atomic_load_acq_32(volatile uint32_t * p)2608d7e7a98SRuslan Bukin atomic_load_acq_32(volatile uint32_t *p)
2618d7e7a98SRuslan Bukin {
2628d7e7a98SRuslan Bukin uint32_t ret;
2638d7e7a98SRuslan Bukin
2648d7e7a98SRuslan Bukin ret = *p;
2658d7e7a98SRuslan Bukin
2668d7e7a98SRuslan Bukin fence();
2678d7e7a98SRuslan Bukin
2688d7e7a98SRuslan Bukin return (ret);
2698d7e7a98SRuslan Bukin }
2708d7e7a98SRuslan Bukin
2718d7e7a98SRuslan Bukin static __inline void
atomic_store_rel_32(volatile uint32_t * p,uint32_t val)2728d7e7a98SRuslan Bukin atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
2738d7e7a98SRuslan Bukin {
2748d7e7a98SRuslan Bukin
2758d7e7a98SRuslan Bukin fence();
2768d7e7a98SRuslan Bukin
2778d7e7a98SRuslan Bukin *p = val;
2788d7e7a98SRuslan Bukin }
2798d7e7a98SRuslan Bukin
2808d7e7a98SRuslan Bukin #define atomic_add_acq_int atomic_add_acq_32
2818d7e7a98SRuslan Bukin #define atomic_clear_acq_int atomic_clear_acq_32
2828d7e7a98SRuslan Bukin #define atomic_cmpset_acq_int atomic_cmpset_acq_32
28343b595f6SRuslan Bukin #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
2848d7e7a98SRuslan Bukin #define atomic_load_acq_int atomic_load_acq_32
2858d7e7a98SRuslan Bukin #define atomic_set_acq_int atomic_set_acq_32
2868d7e7a98SRuslan Bukin #define atomic_subtract_acq_int atomic_subtract_acq_32
2878d7e7a98SRuslan Bukin
2888d7e7a98SRuslan Bukin #define atomic_add_rel_int atomic_add_rel_32
2890d3b3beeSMitchell Horne #define atomic_clear_rel_int atomic_clear_rel_32
2908d7e7a98SRuslan Bukin #define atomic_cmpset_rel_int atomic_cmpset_rel_32
29143b595f6SRuslan Bukin #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
2928d7e7a98SRuslan Bukin #define atomic_set_rel_int atomic_set_rel_32
2938d7e7a98SRuslan Bukin #define atomic_subtract_rel_int atomic_subtract_rel_32
2948d7e7a98SRuslan Bukin #define atomic_store_rel_int atomic_store_rel_32
2958d7e7a98SRuslan Bukin
2968d7e7a98SRuslan Bukin static __inline void
atomic_add_64(volatile uint64_t * p,uint64_t val)2978d7e7a98SRuslan Bukin atomic_add_64(volatile uint64_t *p, uint64_t val)
2988d7e7a98SRuslan Bukin {
2998d7e7a98SRuslan Bukin
3008d7e7a98SRuslan Bukin __asm __volatile("amoadd.d zero, %1, %0"
3018d7e7a98SRuslan Bukin : "+A" (*p)
3028d7e7a98SRuslan Bukin : "r" (val)
3038d7e7a98SRuslan Bukin : "memory");
3048d7e7a98SRuslan Bukin }
3058d7e7a98SRuslan Bukin
3068d7e7a98SRuslan Bukin static __inline void
atomic_subtract_64(volatile uint64_t * p,uint64_t val)3078d7e7a98SRuslan Bukin atomic_subtract_64(volatile uint64_t *p, uint64_t val)
3088d7e7a98SRuslan Bukin {
3098d7e7a98SRuslan Bukin
3108d7e7a98SRuslan Bukin __asm __volatile("amoadd.d zero, %1, %0"
3118d7e7a98SRuslan Bukin : "+A" (*p)
3128d7e7a98SRuslan Bukin : "r" (-val)
3138d7e7a98SRuslan Bukin : "memory");
3148d7e7a98SRuslan Bukin }
3158d7e7a98SRuslan Bukin
3168d7e7a98SRuslan Bukin static __inline void
atomic_set_64(volatile uint64_t * p,uint64_t val)3178d7e7a98SRuslan Bukin atomic_set_64(volatile uint64_t *p, uint64_t val)
3188d7e7a98SRuslan Bukin {
3198d7e7a98SRuslan Bukin
3208d7e7a98SRuslan Bukin __asm __volatile("amoor.d zero, %1, %0"
3218d7e7a98SRuslan Bukin : "+A" (*p)
3228d7e7a98SRuslan Bukin : "r" (val)
3238d7e7a98SRuslan Bukin : "memory");
3248d7e7a98SRuslan Bukin }
3258d7e7a98SRuslan Bukin
3268d7e7a98SRuslan Bukin static __inline void
atomic_clear_64(volatile uint64_t * p,uint64_t val)3278d7e7a98SRuslan Bukin atomic_clear_64(volatile uint64_t *p, uint64_t val)
3288d7e7a98SRuslan Bukin {
3298d7e7a98SRuslan Bukin
3308d7e7a98SRuslan Bukin __asm __volatile("amoand.d zero, %1, %0"
3318d7e7a98SRuslan Bukin : "+A" (*p)
3328d7e7a98SRuslan Bukin : "r" (~val)
3338d7e7a98SRuslan Bukin : "memory");
3348d7e7a98SRuslan Bukin }
3358d7e7a98SRuslan Bukin
3368d7e7a98SRuslan Bukin static __inline int
atomic_cmpset_64(volatile uint64_t * p,uint64_t cmpval,uint64_t newval)3378d7e7a98SRuslan Bukin atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
3388d7e7a98SRuslan Bukin {
3398d7e7a98SRuslan Bukin uint64_t tmp;
3408d7e7a98SRuslan Bukin int res;
3418d7e7a98SRuslan Bukin
3428d7e7a98SRuslan Bukin res = 0;
3438d7e7a98SRuslan Bukin
3448d7e7a98SRuslan Bukin __asm __volatile(
3458d7e7a98SRuslan Bukin "0:"
3468d7e7a98SRuslan Bukin "li %1, 1\n" /* Preset to fail */
3478d7e7a98SRuslan Bukin "lr.d %0, %2\n"
3488d7e7a98SRuslan Bukin "bne %0, %z3, 1f\n"
3498d7e7a98SRuslan Bukin "sc.d %1, %z4, %2\n"
3508d7e7a98SRuslan Bukin "bnez %1, 0b\n"
3518d7e7a98SRuslan Bukin "1:"
3528d7e7a98SRuslan Bukin : "=&r" (tmp), "=&r" (res), "+A" (*p)
3538d7e7a98SRuslan Bukin : "rJ" (cmpval), "rJ" (newval)
3548d7e7a98SRuslan Bukin : "memory");
3558d7e7a98SRuslan Bukin
3568d7e7a98SRuslan Bukin return (!res);
3578d7e7a98SRuslan Bukin }
3588d7e7a98SRuslan Bukin
35943b595f6SRuslan Bukin static __inline int
atomic_fcmpset_64(volatile uint64_t * p,uint64_t * cmpval,uint64_t newval)36043b595f6SRuslan Bukin atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
36143b595f6SRuslan Bukin {
36243b595f6SRuslan Bukin uint64_t tmp;
36343b595f6SRuslan Bukin int res;
36443b595f6SRuslan Bukin
36543b595f6SRuslan Bukin res = 0;
36643b595f6SRuslan Bukin
36743b595f6SRuslan Bukin __asm __volatile(
36843b595f6SRuslan Bukin "0:"
36943b595f6SRuslan Bukin "li %1, 1\n" /* Preset to fail */
37043b595f6SRuslan Bukin "lr.d %0, %2\n" /* Load old value */
37143b595f6SRuslan Bukin "bne %0, %z4, 1f\n" /* Compare */
37243b595f6SRuslan Bukin "sc.d %1, %z5, %2\n" /* Try to store new value */
37343b595f6SRuslan Bukin "j 2f\n"
37443b595f6SRuslan Bukin "1:"
37543b595f6SRuslan Bukin "sd %0, %3\n" /* Save old value */
37643b595f6SRuslan Bukin "2:"
37743b595f6SRuslan Bukin : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
37843b595f6SRuslan Bukin : "rJ" (*cmpval), "rJ" (newval)
37943b595f6SRuslan Bukin : "memory");
38043b595f6SRuslan Bukin
38143b595f6SRuslan Bukin return (!res);
38243b595f6SRuslan Bukin }
38343b595f6SRuslan Bukin
3848d7e7a98SRuslan Bukin static __inline uint64_t
atomic_fetchadd_64(volatile uint64_t * p,uint64_t val)3858d7e7a98SRuslan Bukin atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
3868d7e7a98SRuslan Bukin {
3878d7e7a98SRuslan Bukin uint64_t ret;
3888d7e7a98SRuslan Bukin
3898d7e7a98SRuslan Bukin __asm __volatile("amoadd.d %0, %2, %1"
3908d7e7a98SRuslan Bukin : "=&r" (ret), "+A" (*p)
3918d7e7a98SRuslan Bukin : "r" (val)
3928d7e7a98SRuslan Bukin : "memory");
3938d7e7a98SRuslan Bukin
3948d7e7a98SRuslan Bukin return (ret);
3958d7e7a98SRuslan Bukin }
3968d7e7a98SRuslan Bukin
3978d7e7a98SRuslan Bukin static __inline uint64_t
atomic_readandclear_64(volatile uint64_t * p)3988d7e7a98SRuslan Bukin atomic_readandclear_64(volatile uint64_t *p)
3998d7e7a98SRuslan Bukin {
4008d7e7a98SRuslan Bukin uint64_t ret;
4018d7e7a98SRuslan Bukin uint64_t val;
4028d7e7a98SRuslan Bukin
4038d7e7a98SRuslan Bukin val = 0;
4048d7e7a98SRuslan Bukin
4058d7e7a98SRuslan Bukin __asm __volatile("amoswap.d %0, %2, %1"
4068d7e7a98SRuslan Bukin : "=&r"(ret), "+A" (*p)
4078d7e7a98SRuslan Bukin : "r" (val)
4088d7e7a98SRuslan Bukin : "memory");
4098d7e7a98SRuslan Bukin
4108d7e7a98SRuslan Bukin return (ret);
4118d7e7a98SRuslan Bukin }
4128d7e7a98SRuslan Bukin
4135fe191b0SRuslan Bukin static __inline uint32_t
atomic_swap_32(volatile uint32_t * p,uint32_t val)4145fe191b0SRuslan Bukin atomic_swap_32(volatile uint32_t *p, uint32_t val)
4155fe191b0SRuslan Bukin {
4165fe191b0SRuslan Bukin uint32_t old;
4175fe191b0SRuslan Bukin
4185fe191b0SRuslan Bukin __asm __volatile("amoswap.w %0, %2, %1"
4195fe191b0SRuslan Bukin : "=&r"(old), "+A" (*p)
4205fe191b0SRuslan Bukin : "r" (val)
4215fe191b0SRuslan Bukin : "memory");
4225fe191b0SRuslan Bukin
4235fe191b0SRuslan Bukin return (old);
4245fe191b0SRuslan Bukin }
4255fe191b0SRuslan Bukin
4268d7e7a98SRuslan Bukin static __inline uint64_t
atomic_swap_64(volatile uint64_t * p,uint64_t val)4278d7e7a98SRuslan Bukin atomic_swap_64(volatile uint64_t *p, uint64_t val)
4288d7e7a98SRuslan Bukin {
4298d7e7a98SRuslan Bukin uint64_t old;
4308d7e7a98SRuslan Bukin
4318d7e7a98SRuslan Bukin __asm __volatile("amoswap.d %0, %2, %1"
4328d7e7a98SRuslan Bukin : "=&r"(old), "+A" (*p)
4338d7e7a98SRuslan Bukin : "r" (val)
4348d7e7a98SRuslan Bukin : "memory");
4358d7e7a98SRuslan Bukin
4368d7e7a98SRuslan Bukin return (old);
4378d7e7a98SRuslan Bukin }
4388d7e7a98SRuslan Bukin
43913a10f34SMarius Strobl #define atomic_swap_int atomic_swap_32
44013a10f34SMarius Strobl
4418d7e7a98SRuslan Bukin #define atomic_add_long atomic_add_64
4428d7e7a98SRuslan Bukin #define atomic_clear_long atomic_clear_64
4438d7e7a98SRuslan Bukin #define atomic_cmpset_long atomic_cmpset_64
44443b595f6SRuslan Bukin #define atomic_fcmpset_long atomic_fcmpset_64
4458d7e7a98SRuslan Bukin #define atomic_fetchadd_long atomic_fetchadd_64
4468d7e7a98SRuslan Bukin #define atomic_readandclear_long atomic_readandclear_64
4478d7e7a98SRuslan Bukin #define atomic_set_long atomic_set_64
4488d7e7a98SRuslan Bukin #define atomic_subtract_long atomic_subtract_64
44913a10f34SMarius Strobl #define atomic_swap_long atomic_swap_64
4508d7e7a98SRuslan Bukin
4518d7e7a98SRuslan Bukin #define atomic_add_ptr atomic_add_64
4528d7e7a98SRuslan Bukin #define atomic_clear_ptr atomic_clear_64
4538d7e7a98SRuslan Bukin #define atomic_cmpset_ptr atomic_cmpset_64
45443b595f6SRuslan Bukin #define atomic_fcmpset_ptr atomic_fcmpset_64
4558d7e7a98SRuslan Bukin #define atomic_fetchadd_ptr atomic_fetchadd_64
4568d7e7a98SRuslan Bukin #define atomic_readandclear_ptr atomic_readandclear_64
4578d7e7a98SRuslan Bukin #define atomic_set_ptr atomic_set_64
4588d7e7a98SRuslan Bukin #define atomic_subtract_ptr atomic_subtract_64
45913a10f34SMarius Strobl #define atomic_swap_ptr atomic_swap_64
4608d7e7a98SRuslan Bukin
4618d7e7a98SRuslan Bukin ATOMIC_ACQ_REL(set, 64)
4628d7e7a98SRuslan Bukin ATOMIC_ACQ_REL(clear, 64)
4638d7e7a98SRuslan Bukin ATOMIC_ACQ_REL(add, 64)
4648d7e7a98SRuslan Bukin ATOMIC_ACQ_REL(subtract, 64)
4658d7e7a98SRuslan Bukin
46629a5f639SKyle Evans ATOMIC_CMPSET_ACQ_REL(64);
46729a5f639SKyle Evans ATOMIC_FCMPSET_ACQ_REL(64);
46843b595f6SRuslan Bukin
4698d7e7a98SRuslan Bukin static __inline uint64_t
atomic_load_acq_64(volatile uint64_t * p)4708d7e7a98SRuslan Bukin atomic_load_acq_64(volatile uint64_t *p)
4718d7e7a98SRuslan Bukin {
4728d7e7a98SRuslan Bukin uint64_t ret;
4738d7e7a98SRuslan Bukin
4748d7e7a98SRuslan Bukin ret = *p;
4758d7e7a98SRuslan Bukin
4768d7e7a98SRuslan Bukin fence();
4778d7e7a98SRuslan Bukin
4788d7e7a98SRuslan Bukin return (ret);
4798d7e7a98SRuslan Bukin }
4808d7e7a98SRuslan Bukin
4818d7e7a98SRuslan Bukin static __inline void
atomic_store_rel_64(volatile uint64_t * p,uint64_t val)4828d7e7a98SRuslan Bukin atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
4838d7e7a98SRuslan Bukin {
4848d7e7a98SRuslan Bukin
4858d7e7a98SRuslan Bukin fence();
4868d7e7a98SRuslan Bukin
4878d7e7a98SRuslan Bukin *p = val;
4888d7e7a98SRuslan Bukin }
4898d7e7a98SRuslan Bukin
4908d7e7a98SRuslan Bukin #define atomic_add_acq_long atomic_add_acq_64
4910d3b3beeSMitchell Horne #define atomic_clear_acq_long atomic_clear_acq_64
4928d7e7a98SRuslan Bukin #define atomic_cmpset_acq_long atomic_cmpset_acq_64
49343b595f6SRuslan Bukin #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
4948d7e7a98SRuslan Bukin #define atomic_load_acq_long atomic_load_acq_64
4958d7e7a98SRuslan Bukin #define atomic_set_acq_long atomic_set_acq_64
4968d7e7a98SRuslan Bukin #define atomic_subtract_acq_long atomic_subtract_acq_64
4978d7e7a98SRuslan Bukin
4988d7e7a98SRuslan Bukin #define atomic_add_acq_ptr atomic_add_acq_64
4990d3b3beeSMitchell Horne #define atomic_clear_acq_ptr atomic_clear_acq_64
5008d7e7a98SRuslan Bukin #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
50143b595f6SRuslan Bukin #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
5028d7e7a98SRuslan Bukin #define atomic_load_acq_ptr atomic_load_acq_64
5038d7e7a98SRuslan Bukin #define atomic_set_acq_ptr atomic_set_acq_64
5048d7e7a98SRuslan Bukin #define atomic_subtract_acq_ptr atomic_subtract_acq_64
5058d7e7a98SRuslan Bukin
5065c118142SRuslan Bukin #undef ATOMIC_ACQ_REL
5075c118142SRuslan Bukin
5088d7e7a98SRuslan Bukin static __inline void
atomic_thread_fence_acq(void)5098d7e7a98SRuslan Bukin atomic_thread_fence_acq(void)
5108d7e7a98SRuslan Bukin {
5118d7e7a98SRuslan Bukin
5128d7e7a98SRuslan Bukin fence();
5138d7e7a98SRuslan Bukin }
5148d7e7a98SRuslan Bukin
5158d7e7a98SRuslan Bukin static __inline void
atomic_thread_fence_rel(void)5168d7e7a98SRuslan Bukin atomic_thread_fence_rel(void)
5178d7e7a98SRuslan Bukin {
5188d7e7a98SRuslan Bukin
5198d7e7a98SRuslan Bukin fence();
5208d7e7a98SRuslan Bukin }
5218d7e7a98SRuslan Bukin
5228d7e7a98SRuslan Bukin static __inline void
atomic_thread_fence_acq_rel(void)5238d7e7a98SRuslan Bukin atomic_thread_fence_acq_rel(void)
5248d7e7a98SRuslan Bukin {
5258d7e7a98SRuslan Bukin
5268d7e7a98SRuslan Bukin fence();
5278d7e7a98SRuslan Bukin }
5288d7e7a98SRuslan Bukin
5298d7e7a98SRuslan Bukin static __inline void
atomic_thread_fence_seq_cst(void)5308d7e7a98SRuslan Bukin atomic_thread_fence_seq_cst(void)
5318d7e7a98SRuslan Bukin {
5328d7e7a98SRuslan Bukin
5338d7e7a98SRuslan Bukin fence();
5348d7e7a98SRuslan Bukin }
5358d7e7a98SRuslan Bukin
5368d7e7a98SRuslan Bukin #define atomic_add_rel_long atomic_add_rel_64
5378d7e7a98SRuslan Bukin #define atomic_clear_rel_long atomic_clear_rel_64
5388d7e7a98SRuslan Bukin
5398d7e7a98SRuslan Bukin #define atomic_add_rel_long atomic_add_rel_64
5408d7e7a98SRuslan Bukin #define atomic_clear_rel_long atomic_clear_rel_64
5418d7e7a98SRuslan Bukin #define atomic_cmpset_rel_long atomic_cmpset_rel_64
54243b595f6SRuslan Bukin #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
5438d7e7a98SRuslan Bukin #define atomic_set_rel_long atomic_set_rel_64
5448d7e7a98SRuslan Bukin #define atomic_subtract_rel_long atomic_subtract_rel_64
5458d7e7a98SRuslan Bukin #define atomic_store_rel_long atomic_store_rel_64
5468d7e7a98SRuslan Bukin
5478d7e7a98SRuslan Bukin #define atomic_add_rel_ptr atomic_add_rel_64
5488d7e7a98SRuslan Bukin #define atomic_clear_rel_ptr atomic_clear_rel_64
5498d7e7a98SRuslan Bukin #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
55043b595f6SRuslan Bukin #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
5518d7e7a98SRuslan Bukin #define atomic_set_rel_ptr atomic_set_rel_64
5528d7e7a98SRuslan Bukin #define atomic_subtract_rel_ptr atomic_subtract_rel_64
5538d7e7a98SRuslan Bukin #define atomic_store_rel_ptr atomic_store_rel_64
5548d7e7a98SRuslan Bukin
55529a5f639SKyle Evans #include <sys/_atomic_subword.h>
55629a5f639SKyle Evans
5578d7e7a98SRuslan Bukin #endif /* _MACHINE_ATOMIC_H_ */
558