xref: /freebsd/sys/sys/atomic_san.h (revision 45cb7671)
1435c7cfbSMark Johnston /*-
2435c7cfbSMark Johnston  * SPDX-License-Identifier: BSD-2-Clause
3435c7cfbSMark Johnston  *
4435c7cfbSMark Johnston  * Copyright (c) 2019 Andrew Turner
53ead6023SMark Johnston  * Copyright (c) 2021 The FreeBSD Foundation
6435c7cfbSMark Johnston  *
7435c7cfbSMark Johnston  * This software was developed by SRI International and the University of
8435c7cfbSMark Johnston  * Cambridge Computer Laboratory (Department of Computer Science and
9435c7cfbSMark Johnston  * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
10435c7cfbSMark Johnston  * DARPA SSITH research programme.
11435c7cfbSMark Johnston  *
123ead6023SMark Johnston  * Portions of this software were written by Mark Johnston under sponsorship
133ead6023SMark Johnston  * by the FreeBSD Foundation.
143ead6023SMark Johnston  *
15435c7cfbSMark Johnston  * Redistribution and use in source and binary forms, with or without
16435c7cfbSMark Johnston  * modification, are permitted provided that the following conditions
17435c7cfbSMark Johnston  * are met:
18435c7cfbSMark Johnston  * 1. Redistributions of source code must retain the above copyright
19435c7cfbSMark Johnston  *    notice, this list of conditions and the following disclaimer.
20435c7cfbSMark Johnston  * 2. Redistributions in binary form must reproduce the above copyright
21435c7cfbSMark Johnston  *    notice, this list of conditions and the following disclaimer in the
22435c7cfbSMark Johnston  *    documentation and/or other materials provided with the distribution.
23435c7cfbSMark Johnston  *
24435c7cfbSMark Johnston  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
25435c7cfbSMark Johnston  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26435c7cfbSMark Johnston  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27435c7cfbSMark Johnston  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
28435c7cfbSMark Johnston  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29435c7cfbSMark Johnston  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30435c7cfbSMark Johnston  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31435c7cfbSMark Johnston  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32435c7cfbSMark Johnston  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33435c7cfbSMark Johnston  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34435c7cfbSMark Johnston  * SUCH DAMAGE.
35435c7cfbSMark Johnston  *
36435c7cfbSMark Johnston  * $FreeBSD$
37435c7cfbSMark Johnston  */
38435c7cfbSMark Johnston 
39435c7cfbSMark Johnston #ifndef _SYS_ATOMIC_SAN_H_
40435c7cfbSMark Johnston #define	_SYS_ATOMIC_SAN_H_
41435c7cfbSMark Johnston 
42435c7cfbSMark Johnston #ifndef _MACHINE_ATOMIC_H_
43435c7cfbSMark Johnston #error do not include this header, use machine/atomic.h
44435c7cfbSMark Johnston #endif
45435c7cfbSMark Johnston 
463ead6023SMark Johnston #define	ATOMIC_SAN_FUNC_1(sp, op, name, type)				\
473ead6023SMark Johnston 	void sp##_atomic_##op##_##name(volatile type *, type);		\
483ead6023SMark Johnston 	void sp##_atomic_##op##_acq_##name(volatile type *, type);	\
493ead6023SMark Johnston 	void sp##_atomic_##op##_rel_##name(volatile type *, type)
50435c7cfbSMark Johnston 
513ead6023SMark Johnston #define	ATOMIC_SAN_CMPSET(sp, name, type)				\
523ead6023SMark Johnston 	int sp##_atomic_cmpset_##name(volatile type *, type, type);	\
533ead6023SMark Johnston 	int sp##_atomic_cmpset_acq_##name(volatile type *, type, type); \
543ead6023SMark Johnston 	int sp##_atomic_cmpset_rel_##name(volatile type *, type, type)
55435c7cfbSMark Johnston 
563ead6023SMark Johnston #define	ATOMIC_SAN_FCMPSET(sp, name, type)				\
573ead6023SMark Johnston 	int sp##_atomic_fcmpset_##name(volatile type *, type *, type);	\
583ead6023SMark Johnston 	int sp##_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
593ead6023SMark Johnston 	int sp##_atomic_fcmpset_rel_##name(volatile type *, type *, type)
60435c7cfbSMark Johnston 
613ead6023SMark Johnston #define	ATOMIC_SAN_READ(sp, op, name, type)				\
623ead6023SMark Johnston 	type sp##_atomic_##op##_##name(volatile type *, type)
63435c7cfbSMark Johnston 
643ead6023SMark Johnston #define	ATOMIC_SAN_READANDCLEAR(sp, name, type)				\
653ead6023SMark Johnston 	type sp##_atomic_readandclear_##name(volatile type *)
66435c7cfbSMark Johnston 
673ead6023SMark Johnston #define	ATOMIC_SAN_LOAD(sp, name, type)					\
683ead6023SMark Johnston 	type sp##_atomic_load_##name(volatile type *);			\
693ead6023SMark Johnston 	type sp##_atomic_load_acq_##name(volatile type *)
70435c7cfbSMark Johnston 
713ead6023SMark Johnston #define	ATOMIC_SAN_STORE(sp, name, type)				\
723ead6023SMark Johnston 	void sp##_atomic_store_##name(volatile type *, type);		\
733ead6023SMark Johnston 	void sp##_atomic_store_rel_##name(volatile type *, type)
74435c7cfbSMark Johnston 
753ead6023SMark Johnston #define	ATOMIC_SAN_TEST(sp, op, name, type)				\
763ead6023SMark Johnston 	int sp##_atomic_##op##_##name(volatile type *, u_int);		\
773ead6023SMark Johnston 	int sp##_atomic_##op##_acq_##name(volatile type *, u_int)
78435c7cfbSMark Johnston 
7945cb7671SJohn Baldwin #define	_ATOMIC_SAN_THREAD_FENCE(sp)					\
803ead6023SMark Johnston 	void sp##_atomic_thread_fence_acq(void);			\
813ead6023SMark Johnston 	void sp##_atomic_thread_fence_rel(void);			\
823ead6023SMark Johnston 	void sp##_atomic_thread_fence_acq_rel(void);			\
833ead6023SMark Johnston 	void sp##_atomic_thread_fence_seq_cst(void);			\
843ead6023SMark Johnston 	void sp##_atomic_interrupt_fence(void)
85435c7cfbSMark Johnston 
8645cb7671SJohn Baldwin #define	ATOMIC_SAN_THREAD_FENCE(sp)					\
8745cb7671SJohn Baldwin 	_ATOMIC_SAN_THREAD_FENCE(sp)
8845cb7671SJohn Baldwin 
893ead6023SMark Johnston #define	_ATOMIC_SAN_FUNCS(sp, name, type)				\
903ead6023SMark Johnston 	ATOMIC_SAN_FUNC_1(sp, add, name, type);				\
913ead6023SMark Johnston 	ATOMIC_SAN_FUNC_1(sp, clear, name, type);			\
923ead6023SMark Johnston 	ATOMIC_SAN_CMPSET(sp, name, type);				\
933ead6023SMark Johnston 	ATOMIC_SAN_FCMPSET(sp, name, type);				\
943ead6023SMark Johnston 	ATOMIC_SAN_READ(sp, fetchadd, name, type);			\
953ead6023SMark Johnston 	ATOMIC_SAN_LOAD(sp, name, type);				\
963ead6023SMark Johnston 	ATOMIC_SAN_READANDCLEAR(sp, name, type);			\
973ead6023SMark Johnston 	ATOMIC_SAN_FUNC_1(sp, set, name, type);				\
983ead6023SMark Johnston 	ATOMIC_SAN_FUNC_1(sp, subtract, name, type);			\
993ead6023SMark Johnston 	ATOMIC_SAN_STORE(sp, name, type);				\
1003ead6023SMark Johnston 	ATOMIC_SAN_READ(sp, swap, name, type);				\
1013ead6023SMark Johnston 	ATOMIC_SAN_TEST(sp, testandclear, name, type);			\
10245cb7671SJohn Baldwin 	ATOMIC_SAN_TEST(sp, testandset, name, type)
103435c7cfbSMark Johnston 
1043ead6023SMark Johnston #define	ATOMIC_SAN_FUNCS(name, type)					\
105a90d053bSMark Johnston 	_ATOMIC_SAN_FUNCS(SAN_INTERCEPTOR_PREFIX, name, type)
106435c7cfbSMark Johnston 
1073ead6023SMark Johnston ATOMIC_SAN_FUNCS(char, uint8_t);
1083ead6023SMark Johnston ATOMIC_SAN_FUNCS(short, uint16_t);
1093ead6023SMark Johnston ATOMIC_SAN_FUNCS(int, u_int);
1103ead6023SMark Johnston ATOMIC_SAN_FUNCS(long, u_long);
1113ead6023SMark Johnston ATOMIC_SAN_FUNCS(ptr, uintptr_t);
1123ead6023SMark Johnston ATOMIC_SAN_FUNCS(8, uint8_t);
1133ead6023SMark Johnston ATOMIC_SAN_FUNCS(16, uint16_t);
1143ead6023SMark Johnston ATOMIC_SAN_FUNCS(32, uint32_t);
1153ead6023SMark Johnston ATOMIC_SAN_FUNCS(64, uint64_t);
11645cb7671SJohn Baldwin ATOMIC_SAN_THREAD_FENCE(SAN_INTERCEPTOR_PREFIX);
117435c7cfbSMark Johnston 
1183ead6023SMark Johnston #ifndef SAN_RUNTIME
119435c7cfbSMark Johnston 
1203ead6023SMark Johnston /*
1213ead6023SMark Johnston  * Redirect uses of an atomic(9) function to the sanitizer's interceptor.
1223ead6023SMark Johnston  * For instance, KASAN callers of atomic_add_char() will be redirected to
1233ead6023SMark Johnston  * kasan_atomic_add_char().
1243ead6023SMark Johnston  */
1253ead6023SMark Johnston #define	ATOMIC_SAN(func)						\
126a90d053bSMark Johnston 	__CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_atomic_, func))
127435c7cfbSMark Johnston 
1283ead6023SMark Johnston #define	atomic_add_char			ATOMIC_SAN(add_char)
1293ead6023SMark Johnston #define	atomic_add_acq_char		ATOMIC_SAN(add_acq_char)
1303ead6023SMark Johnston #define	atomic_add_rel_char		ATOMIC_SAN(add_rel_char)
1313ead6023SMark Johnston #define	atomic_clear_char		ATOMIC_SAN(clear_char)
1323ead6023SMark Johnston #define	atomic_clear_acq_char		ATOMIC_SAN(clear_acq_char)
1333ead6023SMark Johnston #define	atomic_clear_rel_char		ATOMIC_SAN(clear_rel_char)
1343ead6023SMark Johnston #define	atomic_cmpset_char		ATOMIC_SAN(cmpset_char)
1353ead6023SMark Johnston #define	atomic_cmpset_acq_char		ATOMIC_SAN(cmpset_acq_char)
1363ead6023SMark Johnston #define	atomic_cmpset_rel_char		ATOMIC_SAN(cmpset_rel_char)
1373ead6023SMark Johnston #define	atomic_fcmpset_char		ATOMIC_SAN(fcmpset_char)
1383ead6023SMark Johnston #define	atomic_fcmpset_acq_char		ATOMIC_SAN(fcmpset_acq_char)
1393ead6023SMark Johnston #define	atomic_fcmpset_rel_char		ATOMIC_SAN(fcmpset_rel_char)
1403ead6023SMark Johnston #define	atomic_fetchadd_char		ATOMIC_SAN(fetchadd_char)
1413ead6023SMark Johnston #define	atomic_load_char		ATOMIC_SAN(load_char)
1423ead6023SMark Johnston #define	atomic_load_acq_char		ATOMIC_SAN(load_acq_char)
1433ead6023SMark Johnston #define	atomic_readandclear_char	ATOMIC_SAN(readandclear_char)
1443ead6023SMark Johnston #define	atomic_set_char			ATOMIC_SAN(set_char)
1453ead6023SMark Johnston #define	atomic_set_acq_char		ATOMIC_SAN(set_acq_char)
1463ead6023SMark Johnston #define	atomic_set_rel_char		ATOMIC_SAN(set_rel_char)
1473ead6023SMark Johnston #define	atomic_subtract_char		ATOMIC_SAN(subtract_char)
1483ead6023SMark Johnston #define	atomic_subtract_acq_char	ATOMIC_SAN(subtract_acq_char)
1493ead6023SMark Johnston #define	atomic_subtract_rel_char	ATOMIC_SAN(subtract_rel_char)
1503ead6023SMark Johnston #define	atomic_store_char		ATOMIC_SAN(store_char)
1513ead6023SMark Johnston #define	atomic_store_rel_char		ATOMIC_SAN(store_rel_char)
1523ead6023SMark Johnston #define	atomic_swap_char		ATOMIC_SAN(swap_char)
1533ead6023SMark Johnston #define	atomic_testandclear_char	ATOMIC_SAN(testandclear_char)
1543ead6023SMark Johnston #define	atomic_testandset_char		ATOMIC_SAN(testandset_char)
155435c7cfbSMark Johnston 
1563ead6023SMark Johnston #define	atomic_add_short		ATOMIC_SAN(add_short)
1573ead6023SMark Johnston #define	atomic_add_acq_short		ATOMIC_SAN(add_acq_short)
1583ead6023SMark Johnston #define	atomic_add_rel_short		ATOMIC_SAN(add_rel_short)
1593ead6023SMark Johnston #define	atomic_clear_short		ATOMIC_SAN(clear_short)
1603ead6023SMark Johnston #define	atomic_clear_acq_short		ATOMIC_SAN(clear_acq_short)
1613ead6023SMark Johnston #define	atomic_clear_rel_short		ATOMIC_SAN(clear_rel_short)
1623ead6023SMark Johnston #define	atomic_cmpset_short		ATOMIC_SAN(cmpset_short)
1633ead6023SMark Johnston #define	atomic_cmpset_acq_short		ATOMIC_SAN(cmpset_acq_short)
1643ead6023SMark Johnston #define	atomic_cmpset_rel_short		ATOMIC_SAN(cmpset_rel_short)
1653ead6023SMark Johnston #define	atomic_fcmpset_short		ATOMIC_SAN(fcmpset_short)
1663ead6023SMark Johnston #define	atomic_fcmpset_acq_short	ATOMIC_SAN(fcmpset_acq_short)
1673ead6023SMark Johnston #define	atomic_fcmpset_rel_short	ATOMIC_SAN(fcmpset_rel_short)
1683ead6023SMark Johnston #define	atomic_fetchadd_short		ATOMIC_SAN(fetchadd_short)
1693ead6023SMark Johnston #define	atomic_load_short		ATOMIC_SAN(load_short)
1703ead6023SMark Johnston #define	atomic_load_acq_short		ATOMIC_SAN(load_acq_short)
1713ead6023SMark Johnston #define	atomic_readandclear_short	ATOMIC_SAN(readandclear_short)
1723ead6023SMark Johnston #define	atomic_set_short		ATOMIC_SAN(set_short)
1733ead6023SMark Johnston #define	atomic_set_acq_short		ATOMIC_SAN(set_acq_short)
1743ead6023SMark Johnston #define	atomic_set_rel_short		ATOMIC_SAN(set_rel_short)
1753ead6023SMark Johnston #define	atomic_subtract_short		ATOMIC_SAN(subtract_short)
1763ead6023SMark Johnston #define	atomic_subtract_acq_short	ATOMIC_SAN(subtract_acq_short)
1773ead6023SMark Johnston #define	atomic_subtract_rel_short	ATOMIC_SAN(subtract_rel_short)
1783ead6023SMark Johnston #define	atomic_store_short		ATOMIC_SAN(store_short)
1793ead6023SMark Johnston #define	atomic_store_rel_short		ATOMIC_SAN(store_rel_short)
1803ead6023SMark Johnston #define	atomic_swap_short		ATOMIC_SAN(swap_short)
1813ead6023SMark Johnston #define	atomic_testandclear_short	ATOMIC_SAN(testandclear_short)
1823ead6023SMark Johnston #define	atomic_testandset_short		ATOMIC_SAN(testandset_short)
1833ead6023SMark Johnston 
1843ead6023SMark Johnston #define	atomic_add_int			ATOMIC_SAN(add_int)
1853ead6023SMark Johnston #define	atomic_add_acq_int		ATOMIC_SAN(add_acq_int)
1863ead6023SMark Johnston #define	atomic_add_rel_int		ATOMIC_SAN(add_rel_int)
1873ead6023SMark Johnston #define	atomic_clear_int		ATOMIC_SAN(clear_int)
1883ead6023SMark Johnston #define	atomic_clear_acq_int		ATOMIC_SAN(clear_acq_int)
1893ead6023SMark Johnston #define	atomic_clear_rel_int		ATOMIC_SAN(clear_rel_int)
1903ead6023SMark Johnston #define	atomic_cmpset_int		ATOMIC_SAN(cmpset_int)
1913ead6023SMark Johnston #define	atomic_cmpset_acq_int		ATOMIC_SAN(cmpset_acq_int)
1923ead6023SMark Johnston #define	atomic_cmpset_rel_int		ATOMIC_SAN(cmpset_rel_int)
1933ead6023SMark Johnston #define	atomic_fcmpset_int		ATOMIC_SAN(fcmpset_int)
1943ead6023SMark Johnston #define	atomic_fcmpset_acq_int		ATOMIC_SAN(fcmpset_acq_int)
1953ead6023SMark Johnston #define	atomic_fcmpset_rel_int		ATOMIC_SAN(fcmpset_rel_int)
1963ead6023SMark Johnston #define	atomic_fetchadd_int		ATOMIC_SAN(fetchadd_int)
1973ead6023SMark Johnston #define	atomic_load_int			ATOMIC_SAN(load_int)
1983ead6023SMark Johnston #define	atomic_load_acq_int		ATOMIC_SAN(load_acq_int)
1993ead6023SMark Johnston #define	atomic_readandclear_int		ATOMIC_SAN(readandclear_int)
2003ead6023SMark Johnston #define	atomic_set_int			ATOMIC_SAN(set_int)
2013ead6023SMark Johnston #define	atomic_set_acq_int		ATOMIC_SAN(set_acq_int)
2023ead6023SMark Johnston #define	atomic_set_rel_int		ATOMIC_SAN(set_rel_int)
2033ead6023SMark Johnston #define	atomic_subtract_int		ATOMIC_SAN(subtract_int)
2043ead6023SMark Johnston #define	atomic_subtract_acq_int		ATOMIC_SAN(subtract_acq_int)
2053ead6023SMark Johnston #define	atomic_subtract_rel_int		ATOMIC_SAN(subtract_rel_int)
2063ead6023SMark Johnston #define	atomic_store_int		ATOMIC_SAN(store_int)
2073ead6023SMark Johnston #define	atomic_store_rel_int		ATOMIC_SAN(store_rel_int)
2083ead6023SMark Johnston #define	atomic_swap_int			ATOMIC_SAN(swap_int)
2093ead6023SMark Johnston #define	atomic_testandclear_int		ATOMIC_SAN(testandclear_int)
2103ead6023SMark Johnston #define	atomic_testandset_int		ATOMIC_SAN(testandset_int)
2113ead6023SMark Johnston 
2123ead6023SMark Johnston #define	atomic_add_long			ATOMIC_SAN(add_long)
2133ead6023SMark Johnston #define	atomic_add_acq_long		ATOMIC_SAN(add_acq_long)
2143ead6023SMark Johnston #define	atomic_add_rel_long		ATOMIC_SAN(add_rel_long)
2153ead6023SMark Johnston #define	atomic_clear_long		ATOMIC_SAN(clear_long)
2163ead6023SMark Johnston #define	atomic_clear_acq_long		ATOMIC_SAN(clear_acq_long)
2173ead6023SMark Johnston #define	atomic_clear_rel_long		ATOMIC_SAN(clear_rel_long)
2183ead6023SMark Johnston #define	atomic_cmpset_long		ATOMIC_SAN(cmpset_long)
2193ead6023SMark Johnston #define	atomic_cmpset_acq_long		ATOMIC_SAN(cmpset_acq_long)
2203ead6023SMark Johnston #define	atomic_cmpset_rel_long		ATOMIC_SAN(cmpset_rel_long)
2213ead6023SMark Johnston #define	atomic_fcmpset_long		ATOMIC_SAN(fcmpset_long)
2223ead6023SMark Johnston #define	atomic_fcmpset_acq_long		ATOMIC_SAN(fcmpset_acq_long)
2233ead6023SMark Johnston #define	atomic_fcmpset_rel_long		ATOMIC_SAN(fcmpset_rel_long)
2243ead6023SMark Johnston #define	atomic_fetchadd_long		ATOMIC_SAN(fetchadd_long)
2253ead6023SMark Johnston #define	atomic_load_long		ATOMIC_SAN(load_long)
2263ead6023SMark Johnston #define	atomic_load_acq_long		ATOMIC_SAN(load_acq_long)
2273ead6023SMark Johnston #define	atomic_readandclear_long	ATOMIC_SAN(readandclear_long)
2283ead6023SMark Johnston #define	atomic_set_long			ATOMIC_SAN(set_long)
2293ead6023SMark Johnston #define	atomic_set_acq_long		ATOMIC_SAN(set_acq_long)
2303ead6023SMark Johnston #define	atomic_set_rel_long		ATOMIC_SAN(set_rel_long)
2313ead6023SMark Johnston #define	atomic_subtract_long		ATOMIC_SAN(subtract_long)
2323ead6023SMark Johnston #define	atomic_subtract_acq_long	ATOMIC_SAN(subtract_acq_long)
2333ead6023SMark Johnston #define	atomic_subtract_rel_long	ATOMIC_SAN(subtract_rel_long)
2343ead6023SMark Johnston #define	atomic_store_long		ATOMIC_SAN(store_long)
2353ead6023SMark Johnston #define	atomic_store_rel_long		ATOMIC_SAN(store_rel_long)
2363ead6023SMark Johnston #define	atomic_swap_long		ATOMIC_SAN(swap_long)
2373ead6023SMark Johnston #define	atomic_testandclear_long	ATOMIC_SAN(testandclear_long)
2383ead6023SMark Johnston #define	atomic_testandset_long		ATOMIC_SAN(testandset_long)
2393ead6023SMark Johnston #define	atomic_testandset_acq_long	ATOMIC_SAN(testandset_acq_long)
2403ead6023SMark Johnston 
2413ead6023SMark Johnston #define	atomic_add_ptr			ATOMIC_SAN(add_ptr)
2423ead6023SMark Johnston #define	atomic_add_acq_ptr		ATOMIC_SAN(add_acq_ptr)
2433ead6023SMark Johnston #define	atomic_add_rel_ptr		ATOMIC_SAN(add_rel_ptr)
2443ead6023SMark Johnston #define	atomic_clear_ptr		ATOMIC_SAN(clear_ptr)
2453ead6023SMark Johnston #define	atomic_clear_acq_ptr		ATOMIC_SAN(clear_acq_ptr)
2463ead6023SMark Johnston #define	atomic_clear_rel_ptr		ATOMIC_SAN(clear_rel_ptr)
2473ead6023SMark Johnston #define	atomic_cmpset_ptr		ATOMIC_SAN(cmpset_ptr)
2483ead6023SMark Johnston #define	atomic_cmpset_acq_ptr		ATOMIC_SAN(cmpset_acq_ptr)
2493ead6023SMark Johnston #define	atomic_cmpset_rel_ptr		ATOMIC_SAN(cmpset_rel_ptr)
2503ead6023SMark Johnston #define	atomic_fcmpset_ptr		ATOMIC_SAN(fcmpset_ptr)
2513ead6023SMark Johnston #define	atomic_fcmpset_acq_ptr		ATOMIC_SAN(fcmpset_acq_ptr)
2523ead6023SMark Johnston #define	atomic_fcmpset_rel_ptr		ATOMIC_SAN(fcmpset_rel_ptr)
2533ead6023SMark Johnston #define	atomic_fetchadd_ptr		ATOMIC_SAN(fetchadd_ptr)
254435c7cfbSMark Johnston #define	atomic_load_ptr(x)		({					\
255435c7cfbSMark Johnston 	__typeof(*x) __retptr;							\
2563ead6023SMark Johnston 	__retptr = (void *)ATOMIC_SAN(load_ptr)((volatile uintptr_t *)(x));	\
257435c7cfbSMark Johnston 	__retptr;								\
258435c7cfbSMark Johnston })
2593ead6023SMark Johnston #define	atomic_load_acq_ptr		ATOMIC_SAN(load_acq_ptr)
260435c7cfbSMark Johnston #define	atomic_load_consume_ptr(x)	({					\
261435c7cfbSMark Johnston 	__typeof(*x) __retptr;							\
2623ead6023SMark Johnston 	__retptr = (void *)atomic_load_acq_ptr((volatile uintptr_t *)(x));\
263435c7cfbSMark Johnston 	__retptr;								\
264435c7cfbSMark Johnston })
2653ead6023SMark Johnston #define	atomic_readandclear_ptr		ATOMIC_SAN(readandclear_ptr)
2663ead6023SMark Johnston #define	atomic_set_ptr			ATOMIC_SAN(set_ptr)
2673ead6023SMark Johnston #define	atomic_set_acq_ptr		ATOMIC_SAN(set_acq_ptr)
2683ead6023SMark Johnston #define	atomic_set_rel_ptr		ATOMIC_SAN(set_rel_ptr)
2693ead6023SMark Johnston #define	atomic_subtract_ptr		ATOMIC_SAN(subtract_ptr)
2703ead6023SMark Johnston #define	atomic_subtract_acq_ptr		ATOMIC_SAN(subtract_acq_ptr)
2713ead6023SMark Johnston #define	atomic_subtract_rel_ptr		ATOMIC_SAN(subtract_rel_ptr)
272435c7cfbSMark Johnston #define	atomic_store_ptr(x, v)		({					\
273435c7cfbSMark Johnston 	__typeof(*x) __value = (v);						\
2743ead6023SMark Johnston 	ATOMIC_SAN(store_ptr)((volatile uintptr_t *)(x), (uintptr_t)(__value));\
275435c7cfbSMark Johnston })
2763ead6023SMark Johnston #define	atomic_store_rel_ptr		ATOMIC_SAN(store_rel_ptr)
2773ead6023SMark Johnston #define	atomic_swap_ptr			ATOMIC_SAN(swap_ptr)
2783ead6023SMark Johnston #define	atomic_testandclear_ptr		ATOMIC_SAN(testandclear_ptr)
2793ead6023SMark Johnston #define	atomic_testandset_ptr		ATOMIC_SAN(testandset_ptr)
280435c7cfbSMark Johnston 
2813ead6023SMark Johnston #define	atomic_add_8			ATOMIC_SAN(add_8)
2823ead6023SMark Johnston #define	atomic_add_acq_8		ATOMIC_SAN(add_acq_8)
2833ead6023SMark Johnston #define	atomic_add_rel_8		ATOMIC_SAN(add_rel_8)
2843ead6023SMark Johnston #define	atomic_clear_8			ATOMIC_SAN(clear_8)
2853ead6023SMark Johnston #define	atomic_clear_acq_8		ATOMIC_SAN(clear_acq_8)
2863ead6023SMark Johnston #define	atomic_clear_rel_8		ATOMIC_SAN(clear_rel_8)
2873ead6023SMark Johnston #define	atomic_cmpset_8			ATOMIC_SAN(cmpset_8)
2883ead6023SMark Johnston #define	atomic_cmpset_acq_8		ATOMIC_SAN(cmpset_acq_8)
2893ead6023SMark Johnston #define	atomic_cmpset_rel_8		ATOMIC_SAN(cmpset_rel_8)
2903ead6023SMark Johnston #define	atomic_fcmpset_8		ATOMIC_SAN(fcmpset_8)
2913ead6023SMark Johnston #define	atomic_fcmpset_acq_8		ATOMIC_SAN(fcmpset_acq_8)
2923ead6023SMark Johnston #define	atomic_fcmpset_rel_8		ATOMIC_SAN(fcmpset_rel_8)
2933ead6023SMark Johnston #define	atomic_fetchadd_8		ATOMIC_SAN(fetchadd_8)
2943ead6023SMark Johnston #define	atomic_load_8			ATOMIC_SAN(load_8)
2953ead6023SMark Johnston #define	atomic_load_acq_8		ATOMIC_SAN(load_acq_8)
2963ead6023SMark Johnston #define	atomic_readandclear_8		ATOMIC_SAN(readandclear_8)
2973ead6023SMark Johnston #define	atomic_set_8			ATOMIC_SAN(set_8)
2983ead6023SMark Johnston #define	atomic_set_acq_8		ATOMIC_SAN(set_acq_8)
2993ead6023SMark Johnston #define	atomic_set_rel_8		ATOMIC_SAN(set_rel_8)
3003ead6023SMark Johnston #define	atomic_subtract_8		ATOMIC_SAN(subtract_8)
3013ead6023SMark Johnston #define	atomic_subtract_acq_8		ATOMIC_SAN(subtract_acq_8)
3023ead6023SMark Johnston #define	atomic_subtract_rel_8		ATOMIC_SAN(subtract_rel_8)
3033ead6023SMark Johnston #define	atomic_store_8			ATOMIC_SAN(store_8)
3043ead6023SMark Johnston #define	atomic_store_rel_8		ATOMIC_SAN(store_rel_8)
3053ead6023SMark Johnston #define	atomic_swap_8			ATOMIC_SAN(swap_8)
3063ead6023SMark Johnston #define	atomic_testandclear_8		ATOMIC_SAN(testandclear_8)
3073ead6023SMark Johnston #define	atomic_testandset_8		ATOMIC_SAN(testandset_8)
308435c7cfbSMark Johnston 
3093ead6023SMark Johnston #define	atomic_add_16			ATOMIC_SAN(add_16)
3103ead6023SMark Johnston #define	atomic_add_acq_16		ATOMIC_SAN(add_acq_16)
3113ead6023SMark Johnston #define	atomic_add_rel_16		ATOMIC_SAN(add_rel_16)
3123ead6023SMark Johnston #define	atomic_clear_16			ATOMIC_SAN(clear_16)
3133ead6023SMark Johnston #define	atomic_clear_acq_16		ATOMIC_SAN(clear_acq_16)
3143ead6023SMark Johnston #define	atomic_clear_rel_16		ATOMIC_SAN(clear_rel_16)
3153ead6023SMark Johnston #define	atomic_cmpset_16		ATOMIC_SAN(cmpset_16)
3163ead6023SMark Johnston #define	atomic_cmpset_acq_16		ATOMIC_SAN(cmpset_acq_16)
3173ead6023SMark Johnston #define	atomic_cmpset_rel_16		ATOMIC_SAN(cmpset_rel_16)
3183ead6023SMark Johnston #define	atomic_fcmpset_16		ATOMIC_SAN(fcmpset_16)
3193ead6023SMark Johnston #define	atomic_fcmpset_acq_16		ATOMIC_SAN(fcmpset_acq_16)
3203ead6023SMark Johnston #define	atomic_fcmpset_rel_16		ATOMIC_SAN(fcmpset_rel_16)
3213ead6023SMark Johnston #define	atomic_fetchadd_16		ATOMIC_SAN(fetchadd_16)
3223ead6023SMark Johnston #define	atomic_load_16			ATOMIC_SAN(load_16)
3233ead6023SMark Johnston #define	atomic_load_acq_16		ATOMIC_SAN(load_acq_16)
3243ead6023SMark Johnston #define	atomic_readandclear_16		ATOMIC_SAN(readandclear_16)
3253ead6023SMark Johnston #define	atomic_set_16			ATOMIC_SAN(set_16)
3263ead6023SMark Johnston #define	atomic_set_acq_16		ATOMIC_SAN(set_acq_16)
3273ead6023SMark Johnston #define	atomic_set_rel_16		ATOMIC_SAN(set_rel_16)
3283ead6023SMark Johnston #define	atomic_subtract_16		ATOMIC_SAN(subtract_16)
3293ead6023SMark Johnston #define	atomic_subtract_acq_16		ATOMIC_SAN(subtract_acq_16)
3303ead6023SMark Johnston #define	atomic_subtract_rel_16		ATOMIC_SAN(subtract_rel_16)
3313ead6023SMark Johnston #define	atomic_store_16			ATOMIC_SAN(store_16)
3323ead6023SMark Johnston #define	atomic_store_rel_16		ATOMIC_SAN(store_rel_16)
3333ead6023SMark Johnston #define	atomic_swap_16			ATOMIC_SAN(swap_16)
3343ead6023SMark Johnston #define	atomic_testandclear_16		ATOMIC_SAN(testandclear_16)
3353ead6023SMark Johnston #define	atomic_testandset_16		ATOMIC_SAN(testandset_16)
336435c7cfbSMark Johnston 
3373ead6023SMark Johnston #define	atomic_add_32			ATOMIC_SAN(add_32)
3383ead6023SMark Johnston #define	atomic_add_acq_32		ATOMIC_SAN(add_acq_32)
3393ead6023SMark Johnston #define	atomic_add_rel_32		ATOMIC_SAN(add_rel_32)
3403ead6023SMark Johnston #define	atomic_clear_32			ATOMIC_SAN(clear_32)
3413ead6023SMark Johnston #define	atomic_clear_acq_32		ATOMIC_SAN(clear_acq_32)
3423ead6023SMark Johnston #define	atomic_clear_rel_32		ATOMIC_SAN(clear_rel_32)
3433ead6023SMark Johnston #define	atomic_cmpset_32		ATOMIC_SAN(cmpset_32)
3443ead6023SMark Johnston #define	atomic_cmpset_acq_32		ATOMIC_SAN(cmpset_acq_32)
3453ead6023SMark Johnston #define	atomic_cmpset_rel_32		ATOMIC_SAN(cmpset_rel_32)
3463ead6023SMark Johnston #define	atomic_fcmpset_32		ATOMIC_SAN(fcmpset_32)
3473ead6023SMark Johnston #define	atomic_fcmpset_acq_32		ATOMIC_SAN(fcmpset_acq_32)
3483ead6023SMark Johnston #define	atomic_fcmpset_rel_32		ATOMIC_SAN(fcmpset_rel_32)
3493ead6023SMark Johnston #define	atomic_fetchadd_32		ATOMIC_SAN(fetchadd_32)
3503ead6023SMark Johnston #define	atomic_load_32			ATOMIC_SAN(load_32)
3513ead6023SMark Johnston #define	atomic_load_acq_32		ATOMIC_SAN(load_acq_32)
3523ead6023SMark Johnston #define	atomic_readandclear_32		ATOMIC_SAN(readandclear_32)
3533ead6023SMark Johnston #define	atomic_set_32			ATOMIC_SAN(set_32)
3543ead6023SMark Johnston #define	atomic_set_acq_32		ATOMIC_SAN(set_acq_32)
3553ead6023SMark Johnston #define	atomic_set_rel_32		ATOMIC_SAN(set_rel_32)
3563ead6023SMark Johnston #define	atomic_subtract_32		ATOMIC_SAN(subtract_32)
3573ead6023SMark Johnston #define	atomic_subtract_acq_32		ATOMIC_SAN(subtract_acq_32)
3583ead6023SMark Johnston #define	atomic_subtract_rel_32		ATOMIC_SAN(subtract_rel_32)
3593ead6023SMark Johnston #define	atomic_store_32			ATOMIC_SAN(store_32)
3603ead6023SMark Johnston #define	atomic_store_rel_32		ATOMIC_SAN(store_rel_32)
3613ead6023SMark Johnston #define	atomic_swap_32			ATOMIC_SAN(swap_32)
3623ead6023SMark Johnston #define	atomic_testandclear_32		ATOMIC_SAN(testandclear_32)
3633ead6023SMark Johnston #define	atomic_testandset_32		ATOMIC_SAN(testandset_32)
364435c7cfbSMark Johnston 
3653ead6023SMark Johnston #define	atomic_add_64			ATOMIC_SAN(add_64)
3663ead6023SMark Johnston #define	atomic_add_acq_64		ATOMIC_SAN(add_acq_64)
3673ead6023SMark Johnston #define	atomic_add_rel_64		ATOMIC_SAN(add_rel_64)
3683ead6023SMark Johnston #define	atomic_clear_64			ATOMIC_SAN(clear_64)
3693ead6023SMark Johnston #define	atomic_clear_acq_64		ATOMIC_SAN(clear_acq_64)
3703ead6023SMark Johnston #define	atomic_clear_rel_64		ATOMIC_SAN(clear_rel_64)
3713ead6023SMark Johnston #define	atomic_cmpset_64		ATOMIC_SAN(cmpset_64)
3723ead6023SMark Johnston #define	atomic_cmpset_acq_64		ATOMIC_SAN(cmpset_acq_64)
3733ead6023SMark Johnston #define	atomic_cmpset_rel_64		ATOMIC_SAN(cmpset_rel_64)
3743ead6023SMark Johnston #define	atomic_fcmpset_64		ATOMIC_SAN(fcmpset_64)
3753ead6023SMark Johnston #define	atomic_fcmpset_acq_64		ATOMIC_SAN(fcmpset_acq_64)
3763ead6023SMark Johnston #define	atomic_fcmpset_rel_64		ATOMIC_SAN(fcmpset_rel_64)
3773ead6023SMark Johnston #define	atomic_fetchadd_64		ATOMIC_SAN(fetchadd_64)
3783ead6023SMark Johnston #define	atomic_load_64			ATOMIC_SAN(load_64)
3793ead6023SMark Johnston #define	atomic_load_acq_64		ATOMIC_SAN(load_acq_64)
3803ead6023SMark Johnston #define	atomic_readandclear_64		ATOMIC_SAN(readandclear_64)
3813ead6023SMark Johnston #define	atomic_set_64			ATOMIC_SAN(set_64)
3823ead6023SMark Johnston #define	atomic_set_acq_64		ATOMIC_SAN(set_acq_64)
3833ead6023SMark Johnston #define	atomic_set_rel_64		ATOMIC_SAN(set_rel_64)
3843ead6023SMark Johnston #define	atomic_subtract_64		ATOMIC_SAN(subtract_64)
3853ead6023SMark Johnston #define	atomic_subtract_acq_64		ATOMIC_SAN(subtract_acq_64)
3863ead6023SMark Johnston #define	atomic_subtract_rel_64		ATOMIC_SAN(subtract_rel_64)
3873ead6023SMark Johnston #define	atomic_store_64			ATOMIC_SAN(store_64)
3883ead6023SMark Johnston #define	atomic_store_rel_64		ATOMIC_SAN(store_rel_64)
3893ead6023SMark Johnston #define	atomic_swap_64			ATOMIC_SAN(swap_64)
3903ead6023SMark Johnston #define	atomic_testandclear_64		ATOMIC_SAN(testandclear_64)
3913ead6023SMark Johnston #define	atomic_testandset_64		ATOMIC_SAN(testandset_64)
392435c7cfbSMark Johnston 
3933ead6023SMark Johnston #define	atomic_thread_fence_acq		ATOMIC_SAN(thread_fence_acq)
3943ead6023SMark Johnston #define	atomic_thread_fence_acq_rel	ATOMIC_SAN(thread_fence_acq_rel)
3953ead6023SMark Johnston #define	atomic_thread_fence_rel		ATOMIC_SAN(thread_fence_rel)
3963ead6023SMark Johnston #define	atomic_thread_fence_seq_cst	ATOMIC_SAN(thread_fence_seq_cst)
3973ead6023SMark Johnston #define	atomic_interrupt_fence		ATOMIC_SAN(interrupt_fence)
398435c7cfbSMark Johnston 
3993ead6023SMark Johnston #endif /* !SAN_RUNTIME */
400435c7cfbSMark Johnston 
401435c7cfbSMark Johnston #endif /* !_SYS_ATOMIC_SAN_H_ */
402