1435c7cfbSMark Johnston /*- 2435c7cfbSMark Johnston * SPDX-License-Identifier: BSD-2-Clause 3435c7cfbSMark Johnston * 4435c7cfbSMark Johnston * Copyright (c) 2019 Andrew Turner 53ead6023SMark Johnston * Copyright (c) 2021 The FreeBSD Foundation 6435c7cfbSMark Johnston * 7435c7cfbSMark Johnston * This software was developed by SRI International and the University of 8435c7cfbSMark Johnston * Cambridge Computer Laboratory (Department of Computer Science and 9435c7cfbSMark Johnston * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the 10435c7cfbSMark Johnston * DARPA SSITH research programme. 11435c7cfbSMark Johnston * 123ead6023SMark Johnston * Portions of this software were written by Mark Johnston under sponsorship 133ead6023SMark Johnston * by the FreeBSD Foundation. 143ead6023SMark Johnston * 15435c7cfbSMark Johnston * Redistribution and use in source and binary forms, with or without 16435c7cfbSMark Johnston * modification, are permitted provided that the following conditions 17435c7cfbSMark Johnston * are met: 18435c7cfbSMark Johnston * 1. Redistributions of source code must retain the above copyright 19435c7cfbSMark Johnston * notice, this list of conditions and the following disclaimer. 20435c7cfbSMark Johnston * 2. Redistributions in binary form must reproduce the above copyright 21435c7cfbSMark Johnston * notice, this list of conditions and the following disclaimer in the 22435c7cfbSMark Johnston * documentation and/or other materials provided with the distribution. 23435c7cfbSMark Johnston * 24435c7cfbSMark Johnston * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 25435c7cfbSMark Johnston * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26435c7cfbSMark Johnston * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27435c7cfbSMark Johnston * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 28435c7cfbSMark Johnston * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29435c7cfbSMark Johnston * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30435c7cfbSMark Johnston * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31435c7cfbSMark Johnston * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32435c7cfbSMark Johnston * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33435c7cfbSMark Johnston * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34435c7cfbSMark Johnston * SUCH DAMAGE. 35435c7cfbSMark Johnston */ 36435c7cfbSMark Johnston 37435c7cfbSMark Johnston #ifndef _SYS_ATOMIC_SAN_H_ 38435c7cfbSMark Johnston #define _SYS_ATOMIC_SAN_H_ 39435c7cfbSMark Johnston 40435c7cfbSMark Johnston #ifndef _MACHINE_ATOMIC_H_ 41435c7cfbSMark Johnston #error do not include this header, use machine/atomic.h 42435c7cfbSMark Johnston #endif 43435c7cfbSMark Johnston 446ebd8466SMark Johnston #include <sys/types.h> 456ebd8466SMark Johnston 463ead6023SMark Johnston #define ATOMIC_SAN_FUNC_1(sp, op, name, type) \ 473ead6023SMark Johnston void sp##_atomic_##op##_##name(volatile type *, type); \ 483ead6023SMark Johnston void sp##_atomic_##op##_acq_##name(volatile type *, type); \ 493ead6023SMark Johnston void sp##_atomic_##op##_rel_##name(volatile type *, type) 50435c7cfbSMark Johnston 513ead6023SMark Johnston #define ATOMIC_SAN_CMPSET(sp, name, type) \ 523ead6023SMark Johnston int sp##_atomic_cmpset_##name(volatile type *, type, type); \ 533ead6023SMark Johnston int sp##_atomic_cmpset_acq_##name(volatile type *, type, type); \ 543ead6023SMark Johnston int sp##_atomic_cmpset_rel_##name(volatile type *, type, type) 55435c7cfbSMark Johnston 563ead6023SMark Johnston #define ATOMIC_SAN_FCMPSET(sp, name, type) \ 573ead6023SMark Johnston int sp##_atomic_fcmpset_##name(volatile type *, type *, type); \ 583ead6023SMark Johnston int sp##_atomic_fcmpset_acq_##name(volatile type *, type *, type); \ 593ead6023SMark Johnston int sp##_atomic_fcmpset_rel_##name(volatile type *, type *, type) 60435c7cfbSMark Johnston 613ead6023SMark Johnston #define ATOMIC_SAN_READ(sp, op, name, type) \ 623ead6023SMark Johnston type sp##_atomic_##op##_##name(volatile type *, type) 63435c7cfbSMark Johnston 643ead6023SMark Johnston #define ATOMIC_SAN_READANDCLEAR(sp, name, type) \ 653ead6023SMark Johnston type sp##_atomic_readandclear_##name(volatile type *) 66435c7cfbSMark Johnston 673ead6023SMark Johnston #define ATOMIC_SAN_LOAD(sp, name, type) \ 681f6b6cf1SMark Johnston type sp##_atomic_load_##name(volatile type *) 691f6b6cf1SMark Johnston 701f6b6cf1SMark Johnston #define ATOMIC_SAN_LOAD_ACQ(sp, name, type) \ 713ead6023SMark Johnston type sp##_atomic_load_acq_##name(volatile type *) 72435c7cfbSMark Johnston 733ead6023SMark Johnston #define ATOMIC_SAN_STORE(sp, name, type) \ 741f6b6cf1SMark Johnston void sp##_atomic_store_##name(volatile type *, type) 751f6b6cf1SMark Johnston 761f6b6cf1SMark Johnston #define ATOMIC_SAN_STORE_REL(sp, name, type) \ 773ead6023SMark Johnston void sp##_atomic_store_rel_##name(volatile type *, type) 78435c7cfbSMark Johnston 793ead6023SMark Johnston #define ATOMIC_SAN_TEST(sp, op, name, type) \ 803ead6023SMark Johnston int sp##_atomic_##op##_##name(volatile type *, u_int); \ 813ead6023SMark Johnston int sp##_atomic_##op##_acq_##name(volatile type *, u_int) 82435c7cfbSMark Johnston 8345cb7671SJohn Baldwin #define _ATOMIC_SAN_THREAD_FENCE(sp) \ 843ead6023SMark Johnston void sp##_atomic_thread_fence_acq(void); \ 853ead6023SMark Johnston void sp##_atomic_thread_fence_rel(void); \ 863ead6023SMark Johnston void sp##_atomic_thread_fence_acq_rel(void); \ 873ead6023SMark Johnston void sp##_atomic_thread_fence_seq_cst(void); \ 883ead6023SMark Johnston void sp##_atomic_interrupt_fence(void) 89435c7cfbSMark Johnston 9045cb7671SJohn Baldwin #define ATOMIC_SAN_THREAD_FENCE(sp) \ 9145cb7671SJohn Baldwin _ATOMIC_SAN_THREAD_FENCE(sp) 9245cb7671SJohn Baldwin 931f6b6cf1SMark Johnston #define ATOMIC_SAN_LOAD_STORE(sp, name, type) \ 941f6b6cf1SMark Johnston ATOMIC_SAN_LOAD(sp, name, type); \ 951f6b6cf1SMark Johnston ATOMIC_SAN_STORE(sp, name, type) 961f6b6cf1SMark Johnston 973ead6023SMark Johnston #define _ATOMIC_SAN_FUNCS(sp, name, type) \ 983ead6023SMark Johnston ATOMIC_SAN_FUNC_1(sp, add, name, type); \ 993ead6023SMark Johnston ATOMIC_SAN_FUNC_1(sp, clear, name, type); \ 1003ead6023SMark Johnston ATOMIC_SAN_CMPSET(sp, name, type); \ 1013ead6023SMark Johnston ATOMIC_SAN_FCMPSET(sp, name, type); \ 1023ead6023SMark Johnston ATOMIC_SAN_READ(sp, fetchadd, name, type); \ 1033ead6023SMark Johnston ATOMIC_SAN_LOAD(sp, name, type); \ 1041f6b6cf1SMark Johnston ATOMIC_SAN_LOAD_ACQ(sp, name, type); \ 1053ead6023SMark Johnston ATOMIC_SAN_READANDCLEAR(sp, name, type); \ 1063ead6023SMark Johnston ATOMIC_SAN_FUNC_1(sp, set, name, type); \ 1073ead6023SMark Johnston ATOMIC_SAN_FUNC_1(sp, subtract, name, type); \ 1083ead6023SMark Johnston ATOMIC_SAN_STORE(sp, name, type); \ 1091f6b6cf1SMark Johnston ATOMIC_SAN_STORE_REL(sp, name, type); \ 1103ead6023SMark Johnston ATOMIC_SAN_READ(sp, swap, name, type); \ 1113ead6023SMark Johnston ATOMIC_SAN_TEST(sp, testandclear, name, type); \ 11245cb7671SJohn Baldwin ATOMIC_SAN_TEST(sp, testandset, name, type) 113435c7cfbSMark Johnston 1143ead6023SMark Johnston #define ATOMIC_SAN_FUNCS(name, type) \ 115a90d053bSMark Johnston _ATOMIC_SAN_FUNCS(SAN_INTERCEPTOR_PREFIX, name, type) 116435c7cfbSMark Johnston 1173ead6023SMark Johnston ATOMIC_SAN_FUNCS(char, uint8_t); 1183ead6023SMark Johnston ATOMIC_SAN_FUNCS(short, uint16_t); 1193ead6023SMark Johnston ATOMIC_SAN_FUNCS(int, u_int); 1203ead6023SMark Johnston ATOMIC_SAN_FUNCS(long, u_long); 1213ead6023SMark Johnston ATOMIC_SAN_FUNCS(ptr, uintptr_t); 1223ead6023SMark Johnston ATOMIC_SAN_FUNCS(8, uint8_t); 1233ead6023SMark Johnston ATOMIC_SAN_FUNCS(16, uint16_t); 1243ead6023SMark Johnston ATOMIC_SAN_FUNCS(32, uint32_t); 1253ead6023SMark Johnston ATOMIC_SAN_FUNCS(64, uint64_t); 1261f6b6cf1SMark Johnston ATOMIC_SAN_LOAD_STORE(SAN_INTERCEPTOR_PREFIX, bool, bool); 12745cb7671SJohn Baldwin ATOMIC_SAN_THREAD_FENCE(SAN_INTERCEPTOR_PREFIX); 128435c7cfbSMark Johnston 1293ead6023SMark Johnston #ifndef SAN_RUNTIME 130435c7cfbSMark Johnston 1313ead6023SMark Johnston /* 1323ead6023SMark Johnston * Redirect uses of an atomic(9) function to the sanitizer's interceptor. 1333ead6023SMark Johnston * For instance, KASAN callers of atomic_add_char() will be redirected to 1343ead6023SMark Johnston * kasan_atomic_add_char(). 1353ead6023SMark Johnston */ 1363ead6023SMark Johnston #define ATOMIC_SAN(func) \ 137a90d053bSMark Johnston __CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_atomic_, func)) 138435c7cfbSMark Johnston 1391f6b6cf1SMark Johnston #define atomic_load_bool ATOMIC_SAN(load_bool) 1401f6b6cf1SMark Johnston #define atomic_store_bool ATOMIC_SAN(store_bool) 1411f6b6cf1SMark Johnston 1423ead6023SMark Johnston #define atomic_add_char ATOMIC_SAN(add_char) 1433ead6023SMark Johnston #define atomic_add_acq_char ATOMIC_SAN(add_acq_char) 1443ead6023SMark Johnston #define atomic_add_rel_char ATOMIC_SAN(add_rel_char) 1453ead6023SMark Johnston #define atomic_clear_char ATOMIC_SAN(clear_char) 1463ead6023SMark Johnston #define atomic_clear_acq_char ATOMIC_SAN(clear_acq_char) 1473ead6023SMark Johnston #define atomic_clear_rel_char ATOMIC_SAN(clear_rel_char) 1483ead6023SMark Johnston #define atomic_cmpset_char ATOMIC_SAN(cmpset_char) 1493ead6023SMark Johnston #define atomic_cmpset_acq_char ATOMIC_SAN(cmpset_acq_char) 1503ead6023SMark Johnston #define atomic_cmpset_rel_char ATOMIC_SAN(cmpset_rel_char) 1513ead6023SMark Johnston #define atomic_fcmpset_char ATOMIC_SAN(fcmpset_char) 1523ead6023SMark Johnston #define atomic_fcmpset_acq_char ATOMIC_SAN(fcmpset_acq_char) 1533ead6023SMark Johnston #define atomic_fcmpset_rel_char ATOMIC_SAN(fcmpset_rel_char) 1543ead6023SMark Johnston #define atomic_fetchadd_char ATOMIC_SAN(fetchadd_char) 1553ead6023SMark Johnston #define atomic_load_char ATOMIC_SAN(load_char) 1563ead6023SMark Johnston #define atomic_load_acq_char ATOMIC_SAN(load_acq_char) 1573ead6023SMark Johnston #define atomic_readandclear_char ATOMIC_SAN(readandclear_char) 1583ead6023SMark Johnston #define atomic_set_char ATOMIC_SAN(set_char) 1593ead6023SMark Johnston #define atomic_set_acq_char ATOMIC_SAN(set_acq_char) 1603ead6023SMark Johnston #define atomic_set_rel_char ATOMIC_SAN(set_rel_char) 1613ead6023SMark Johnston #define atomic_subtract_char ATOMIC_SAN(subtract_char) 1623ead6023SMark Johnston #define atomic_subtract_acq_char ATOMIC_SAN(subtract_acq_char) 1633ead6023SMark Johnston #define atomic_subtract_rel_char ATOMIC_SAN(subtract_rel_char) 1643ead6023SMark Johnston #define atomic_store_char ATOMIC_SAN(store_char) 1653ead6023SMark Johnston #define atomic_store_rel_char ATOMIC_SAN(store_rel_char) 1663ead6023SMark Johnston #define atomic_swap_char ATOMIC_SAN(swap_char) 1673ead6023SMark Johnston #define atomic_testandclear_char ATOMIC_SAN(testandclear_char) 1683ead6023SMark Johnston #define atomic_testandset_char ATOMIC_SAN(testandset_char) 169435c7cfbSMark Johnston 1703ead6023SMark Johnston #define atomic_add_short ATOMIC_SAN(add_short) 1713ead6023SMark Johnston #define atomic_add_acq_short ATOMIC_SAN(add_acq_short) 1723ead6023SMark Johnston #define atomic_add_rel_short ATOMIC_SAN(add_rel_short) 1733ead6023SMark Johnston #define atomic_clear_short ATOMIC_SAN(clear_short) 1743ead6023SMark Johnston #define atomic_clear_acq_short ATOMIC_SAN(clear_acq_short) 1753ead6023SMark Johnston #define atomic_clear_rel_short ATOMIC_SAN(clear_rel_short) 1763ead6023SMark Johnston #define atomic_cmpset_short ATOMIC_SAN(cmpset_short) 1773ead6023SMark Johnston #define atomic_cmpset_acq_short ATOMIC_SAN(cmpset_acq_short) 1783ead6023SMark Johnston #define atomic_cmpset_rel_short ATOMIC_SAN(cmpset_rel_short) 1793ead6023SMark Johnston #define atomic_fcmpset_short ATOMIC_SAN(fcmpset_short) 1803ead6023SMark Johnston #define atomic_fcmpset_acq_short ATOMIC_SAN(fcmpset_acq_short) 1813ead6023SMark Johnston #define atomic_fcmpset_rel_short ATOMIC_SAN(fcmpset_rel_short) 1823ead6023SMark Johnston #define atomic_fetchadd_short ATOMIC_SAN(fetchadd_short) 1833ead6023SMark Johnston #define atomic_load_short ATOMIC_SAN(load_short) 1843ead6023SMark Johnston #define atomic_load_acq_short ATOMIC_SAN(load_acq_short) 1853ead6023SMark Johnston #define atomic_readandclear_short ATOMIC_SAN(readandclear_short) 1863ead6023SMark Johnston #define atomic_set_short ATOMIC_SAN(set_short) 1873ead6023SMark Johnston #define atomic_set_acq_short ATOMIC_SAN(set_acq_short) 1883ead6023SMark Johnston #define atomic_set_rel_short ATOMIC_SAN(set_rel_short) 1893ead6023SMark Johnston #define atomic_subtract_short ATOMIC_SAN(subtract_short) 1903ead6023SMark Johnston #define atomic_subtract_acq_short ATOMIC_SAN(subtract_acq_short) 1913ead6023SMark Johnston #define atomic_subtract_rel_short ATOMIC_SAN(subtract_rel_short) 1923ead6023SMark Johnston #define atomic_store_short ATOMIC_SAN(store_short) 1933ead6023SMark Johnston #define atomic_store_rel_short ATOMIC_SAN(store_rel_short) 1943ead6023SMark Johnston #define atomic_swap_short ATOMIC_SAN(swap_short) 1953ead6023SMark Johnston #define atomic_testandclear_short ATOMIC_SAN(testandclear_short) 1963ead6023SMark Johnston #define atomic_testandset_short ATOMIC_SAN(testandset_short) 1973ead6023SMark Johnston 1983ead6023SMark Johnston #define atomic_add_int ATOMIC_SAN(add_int) 1993ead6023SMark Johnston #define atomic_add_acq_int ATOMIC_SAN(add_acq_int) 2003ead6023SMark Johnston #define atomic_add_rel_int ATOMIC_SAN(add_rel_int) 2013ead6023SMark Johnston #define atomic_clear_int ATOMIC_SAN(clear_int) 2023ead6023SMark Johnston #define atomic_clear_acq_int ATOMIC_SAN(clear_acq_int) 2033ead6023SMark Johnston #define atomic_clear_rel_int ATOMIC_SAN(clear_rel_int) 2043ead6023SMark Johnston #define atomic_cmpset_int ATOMIC_SAN(cmpset_int) 2053ead6023SMark Johnston #define atomic_cmpset_acq_int ATOMIC_SAN(cmpset_acq_int) 2063ead6023SMark Johnston #define atomic_cmpset_rel_int ATOMIC_SAN(cmpset_rel_int) 2073ead6023SMark Johnston #define atomic_fcmpset_int ATOMIC_SAN(fcmpset_int) 2083ead6023SMark Johnston #define atomic_fcmpset_acq_int ATOMIC_SAN(fcmpset_acq_int) 2093ead6023SMark Johnston #define atomic_fcmpset_rel_int ATOMIC_SAN(fcmpset_rel_int) 2103ead6023SMark Johnston #define atomic_fetchadd_int ATOMIC_SAN(fetchadd_int) 2113ead6023SMark Johnston #define atomic_load_int ATOMIC_SAN(load_int) 2123ead6023SMark Johnston #define atomic_load_acq_int ATOMIC_SAN(load_acq_int) 2133ead6023SMark Johnston #define atomic_readandclear_int ATOMIC_SAN(readandclear_int) 2143ead6023SMark Johnston #define atomic_set_int ATOMIC_SAN(set_int) 2153ead6023SMark Johnston #define atomic_set_acq_int ATOMIC_SAN(set_acq_int) 2163ead6023SMark Johnston #define atomic_set_rel_int ATOMIC_SAN(set_rel_int) 2173ead6023SMark Johnston #define atomic_subtract_int ATOMIC_SAN(subtract_int) 2183ead6023SMark Johnston #define atomic_subtract_acq_int ATOMIC_SAN(subtract_acq_int) 2193ead6023SMark Johnston #define atomic_subtract_rel_int ATOMIC_SAN(subtract_rel_int) 2203ead6023SMark Johnston #define atomic_store_int ATOMIC_SAN(store_int) 2213ead6023SMark Johnston #define atomic_store_rel_int ATOMIC_SAN(store_rel_int) 2223ead6023SMark Johnston #define atomic_swap_int ATOMIC_SAN(swap_int) 2233ead6023SMark Johnston #define atomic_testandclear_int ATOMIC_SAN(testandclear_int) 2243ead6023SMark Johnston #define atomic_testandset_int ATOMIC_SAN(testandset_int) 2253ead6023SMark Johnston 2263ead6023SMark Johnston #define atomic_add_long ATOMIC_SAN(add_long) 2273ead6023SMark Johnston #define atomic_add_acq_long ATOMIC_SAN(add_acq_long) 2283ead6023SMark Johnston #define atomic_add_rel_long ATOMIC_SAN(add_rel_long) 2293ead6023SMark Johnston #define atomic_clear_long ATOMIC_SAN(clear_long) 2303ead6023SMark Johnston #define atomic_clear_acq_long ATOMIC_SAN(clear_acq_long) 2313ead6023SMark Johnston #define atomic_clear_rel_long ATOMIC_SAN(clear_rel_long) 2323ead6023SMark Johnston #define atomic_cmpset_long ATOMIC_SAN(cmpset_long) 2333ead6023SMark Johnston #define atomic_cmpset_acq_long ATOMIC_SAN(cmpset_acq_long) 2343ead6023SMark Johnston #define atomic_cmpset_rel_long ATOMIC_SAN(cmpset_rel_long) 2353ead6023SMark Johnston #define atomic_fcmpset_long ATOMIC_SAN(fcmpset_long) 2363ead6023SMark Johnston #define atomic_fcmpset_acq_long ATOMIC_SAN(fcmpset_acq_long) 2373ead6023SMark Johnston #define atomic_fcmpset_rel_long ATOMIC_SAN(fcmpset_rel_long) 2383ead6023SMark Johnston #define atomic_fetchadd_long ATOMIC_SAN(fetchadd_long) 2393ead6023SMark Johnston #define atomic_load_long ATOMIC_SAN(load_long) 2403ead6023SMark Johnston #define atomic_load_acq_long ATOMIC_SAN(load_acq_long) 2413ead6023SMark Johnston #define atomic_readandclear_long ATOMIC_SAN(readandclear_long) 2423ead6023SMark Johnston #define atomic_set_long ATOMIC_SAN(set_long) 2433ead6023SMark Johnston #define atomic_set_acq_long ATOMIC_SAN(set_acq_long) 2443ead6023SMark Johnston #define atomic_set_rel_long ATOMIC_SAN(set_rel_long) 2453ead6023SMark Johnston #define atomic_subtract_long ATOMIC_SAN(subtract_long) 2463ead6023SMark Johnston #define atomic_subtract_acq_long ATOMIC_SAN(subtract_acq_long) 2473ead6023SMark Johnston #define atomic_subtract_rel_long ATOMIC_SAN(subtract_rel_long) 2483ead6023SMark Johnston #define atomic_store_long ATOMIC_SAN(store_long) 2493ead6023SMark Johnston #define atomic_store_rel_long ATOMIC_SAN(store_rel_long) 2503ead6023SMark Johnston #define atomic_swap_long ATOMIC_SAN(swap_long) 2513ead6023SMark Johnston #define atomic_testandclear_long ATOMIC_SAN(testandclear_long) 2523ead6023SMark Johnston #define atomic_testandset_long ATOMIC_SAN(testandset_long) 2533ead6023SMark Johnston #define atomic_testandset_acq_long ATOMIC_SAN(testandset_acq_long) 2543ead6023SMark Johnston 2553ead6023SMark Johnston #define atomic_add_ptr ATOMIC_SAN(add_ptr) 2563ead6023SMark Johnston #define atomic_add_acq_ptr ATOMIC_SAN(add_acq_ptr) 2573ead6023SMark Johnston #define atomic_add_rel_ptr ATOMIC_SAN(add_rel_ptr) 2583ead6023SMark Johnston #define atomic_clear_ptr ATOMIC_SAN(clear_ptr) 2593ead6023SMark Johnston #define atomic_clear_acq_ptr ATOMIC_SAN(clear_acq_ptr) 2603ead6023SMark Johnston #define atomic_clear_rel_ptr ATOMIC_SAN(clear_rel_ptr) 2613ead6023SMark Johnston #define atomic_cmpset_ptr ATOMIC_SAN(cmpset_ptr) 2623ead6023SMark Johnston #define atomic_cmpset_acq_ptr ATOMIC_SAN(cmpset_acq_ptr) 2633ead6023SMark Johnston #define atomic_cmpset_rel_ptr ATOMIC_SAN(cmpset_rel_ptr) 2643ead6023SMark Johnston #define atomic_fcmpset_ptr ATOMIC_SAN(fcmpset_ptr) 2653ead6023SMark Johnston #define atomic_fcmpset_acq_ptr ATOMIC_SAN(fcmpset_acq_ptr) 2663ead6023SMark Johnston #define atomic_fcmpset_rel_ptr ATOMIC_SAN(fcmpset_rel_ptr) 2673ead6023SMark Johnston #define atomic_fetchadd_ptr ATOMIC_SAN(fetchadd_ptr) 26808b0c980SMark Johnston #define atomic_load_ptr(x) \ 26908b0c980SMark Johnston ((void *)ATOMIC_SAN(load_ptr)(__DECONST(volatile uintptr_t *, (x)))) 2703ead6023SMark Johnston #define atomic_load_acq_ptr ATOMIC_SAN(load_acq_ptr) 27108b0c980SMark Johnston #define atomic_load_consume_ptr(x) \ 27208b0c980SMark Johnston ((void *)atomic_load_acq_ptr((volatile uintptr_t *)(x))) 2733ead6023SMark Johnston #define atomic_readandclear_ptr ATOMIC_SAN(readandclear_ptr) 2743ead6023SMark Johnston #define atomic_set_ptr ATOMIC_SAN(set_ptr) 2753ead6023SMark Johnston #define atomic_set_acq_ptr ATOMIC_SAN(set_acq_ptr) 2763ead6023SMark Johnston #define atomic_set_rel_ptr ATOMIC_SAN(set_rel_ptr) 2773ead6023SMark Johnston #define atomic_subtract_ptr ATOMIC_SAN(subtract_ptr) 2783ead6023SMark Johnston #define atomic_subtract_acq_ptr ATOMIC_SAN(subtract_acq_ptr) 2793ead6023SMark Johnston #define atomic_subtract_rel_ptr ATOMIC_SAN(subtract_rel_ptr) 280435c7cfbSMark Johnston #define atomic_store_ptr(x, v) ({ \ 281435c7cfbSMark Johnston __typeof(*x) __value = (v); \ 2823ead6023SMark Johnston ATOMIC_SAN(store_ptr)((volatile uintptr_t *)(x), (uintptr_t)(__value));\ 283435c7cfbSMark Johnston }) 2843ead6023SMark Johnston #define atomic_store_rel_ptr ATOMIC_SAN(store_rel_ptr) 2853ead6023SMark Johnston #define atomic_swap_ptr ATOMIC_SAN(swap_ptr) 2863ead6023SMark Johnston #define atomic_testandclear_ptr ATOMIC_SAN(testandclear_ptr) 2873ead6023SMark Johnston #define atomic_testandset_ptr ATOMIC_SAN(testandset_ptr) 288435c7cfbSMark Johnston 2893ead6023SMark Johnston #define atomic_add_8 ATOMIC_SAN(add_8) 2903ead6023SMark Johnston #define atomic_add_acq_8 ATOMIC_SAN(add_acq_8) 2913ead6023SMark Johnston #define atomic_add_rel_8 ATOMIC_SAN(add_rel_8) 2923ead6023SMark Johnston #define atomic_clear_8 ATOMIC_SAN(clear_8) 2933ead6023SMark Johnston #define atomic_clear_acq_8 ATOMIC_SAN(clear_acq_8) 2943ead6023SMark Johnston #define atomic_clear_rel_8 ATOMIC_SAN(clear_rel_8) 2953ead6023SMark Johnston #define atomic_cmpset_8 ATOMIC_SAN(cmpset_8) 2963ead6023SMark Johnston #define atomic_cmpset_acq_8 ATOMIC_SAN(cmpset_acq_8) 2973ead6023SMark Johnston #define atomic_cmpset_rel_8 ATOMIC_SAN(cmpset_rel_8) 2983ead6023SMark Johnston #define atomic_fcmpset_8 ATOMIC_SAN(fcmpset_8) 2993ead6023SMark Johnston #define atomic_fcmpset_acq_8 ATOMIC_SAN(fcmpset_acq_8) 3003ead6023SMark Johnston #define atomic_fcmpset_rel_8 ATOMIC_SAN(fcmpset_rel_8) 3013ead6023SMark Johnston #define atomic_fetchadd_8 ATOMIC_SAN(fetchadd_8) 3023ead6023SMark Johnston #define atomic_load_8 ATOMIC_SAN(load_8) 3033ead6023SMark Johnston #define atomic_load_acq_8 ATOMIC_SAN(load_acq_8) 3043ead6023SMark Johnston #define atomic_readandclear_8 ATOMIC_SAN(readandclear_8) 3053ead6023SMark Johnston #define atomic_set_8 ATOMIC_SAN(set_8) 3063ead6023SMark Johnston #define atomic_set_acq_8 ATOMIC_SAN(set_acq_8) 3073ead6023SMark Johnston #define atomic_set_rel_8 ATOMIC_SAN(set_rel_8) 3083ead6023SMark Johnston #define atomic_subtract_8 ATOMIC_SAN(subtract_8) 3093ead6023SMark Johnston #define atomic_subtract_acq_8 ATOMIC_SAN(subtract_acq_8) 3103ead6023SMark Johnston #define atomic_subtract_rel_8 ATOMIC_SAN(subtract_rel_8) 3113ead6023SMark Johnston #define atomic_store_8 ATOMIC_SAN(store_8) 3123ead6023SMark Johnston #define atomic_store_rel_8 ATOMIC_SAN(store_rel_8) 3133ead6023SMark Johnston #define atomic_swap_8 ATOMIC_SAN(swap_8) 3143ead6023SMark Johnston #define atomic_testandclear_8 ATOMIC_SAN(testandclear_8) 3153ead6023SMark Johnston #define atomic_testandset_8 ATOMIC_SAN(testandset_8) 316435c7cfbSMark Johnston 3173ead6023SMark Johnston #define atomic_add_16 ATOMIC_SAN(add_16) 3183ead6023SMark Johnston #define atomic_add_acq_16 ATOMIC_SAN(add_acq_16) 3193ead6023SMark Johnston #define atomic_add_rel_16 ATOMIC_SAN(add_rel_16) 3203ead6023SMark Johnston #define atomic_clear_16 ATOMIC_SAN(clear_16) 3213ead6023SMark Johnston #define atomic_clear_acq_16 ATOMIC_SAN(clear_acq_16) 3223ead6023SMark Johnston #define atomic_clear_rel_16 ATOMIC_SAN(clear_rel_16) 3233ead6023SMark Johnston #define atomic_cmpset_16 ATOMIC_SAN(cmpset_16) 3243ead6023SMark Johnston #define atomic_cmpset_acq_16 ATOMIC_SAN(cmpset_acq_16) 3253ead6023SMark Johnston #define atomic_cmpset_rel_16 ATOMIC_SAN(cmpset_rel_16) 3263ead6023SMark Johnston #define atomic_fcmpset_16 ATOMIC_SAN(fcmpset_16) 3273ead6023SMark Johnston #define atomic_fcmpset_acq_16 ATOMIC_SAN(fcmpset_acq_16) 3283ead6023SMark Johnston #define atomic_fcmpset_rel_16 ATOMIC_SAN(fcmpset_rel_16) 3293ead6023SMark Johnston #define atomic_fetchadd_16 ATOMIC_SAN(fetchadd_16) 3303ead6023SMark Johnston #define atomic_load_16 ATOMIC_SAN(load_16) 3313ead6023SMark Johnston #define atomic_load_acq_16 ATOMIC_SAN(load_acq_16) 3323ead6023SMark Johnston #define atomic_readandclear_16 ATOMIC_SAN(readandclear_16) 3333ead6023SMark Johnston #define atomic_set_16 ATOMIC_SAN(set_16) 3343ead6023SMark Johnston #define atomic_set_acq_16 ATOMIC_SAN(set_acq_16) 3353ead6023SMark Johnston #define atomic_set_rel_16 ATOMIC_SAN(set_rel_16) 3363ead6023SMark Johnston #define atomic_subtract_16 ATOMIC_SAN(subtract_16) 3373ead6023SMark Johnston #define atomic_subtract_acq_16 ATOMIC_SAN(subtract_acq_16) 3383ead6023SMark Johnston #define atomic_subtract_rel_16 ATOMIC_SAN(subtract_rel_16) 3393ead6023SMark Johnston #define atomic_store_16 ATOMIC_SAN(store_16) 3403ead6023SMark Johnston #define atomic_store_rel_16 ATOMIC_SAN(store_rel_16) 3413ead6023SMark Johnston #define atomic_swap_16 ATOMIC_SAN(swap_16) 3423ead6023SMark Johnston #define atomic_testandclear_16 ATOMIC_SAN(testandclear_16) 3433ead6023SMark Johnston #define atomic_testandset_16 ATOMIC_SAN(testandset_16) 344435c7cfbSMark Johnston 3453ead6023SMark Johnston #define atomic_add_32 ATOMIC_SAN(add_32) 3463ead6023SMark Johnston #define atomic_add_acq_32 ATOMIC_SAN(add_acq_32) 3473ead6023SMark Johnston #define atomic_add_rel_32 ATOMIC_SAN(add_rel_32) 3483ead6023SMark Johnston #define atomic_clear_32 ATOMIC_SAN(clear_32) 3493ead6023SMark Johnston #define atomic_clear_acq_32 ATOMIC_SAN(clear_acq_32) 3503ead6023SMark Johnston #define atomic_clear_rel_32 ATOMIC_SAN(clear_rel_32) 3513ead6023SMark Johnston #define atomic_cmpset_32 ATOMIC_SAN(cmpset_32) 3523ead6023SMark Johnston #define atomic_cmpset_acq_32 ATOMIC_SAN(cmpset_acq_32) 3533ead6023SMark Johnston #define atomic_cmpset_rel_32 ATOMIC_SAN(cmpset_rel_32) 3543ead6023SMark Johnston #define atomic_fcmpset_32 ATOMIC_SAN(fcmpset_32) 3553ead6023SMark Johnston #define atomic_fcmpset_acq_32 ATOMIC_SAN(fcmpset_acq_32) 3563ead6023SMark Johnston #define atomic_fcmpset_rel_32 ATOMIC_SAN(fcmpset_rel_32) 3573ead6023SMark Johnston #define atomic_fetchadd_32 ATOMIC_SAN(fetchadd_32) 3583ead6023SMark Johnston #define atomic_load_32 ATOMIC_SAN(load_32) 3593ead6023SMark Johnston #define atomic_load_acq_32 ATOMIC_SAN(load_acq_32) 3603ead6023SMark Johnston #define atomic_readandclear_32 ATOMIC_SAN(readandclear_32) 3613ead6023SMark Johnston #define atomic_set_32 ATOMIC_SAN(set_32) 3623ead6023SMark Johnston #define atomic_set_acq_32 ATOMIC_SAN(set_acq_32) 3633ead6023SMark Johnston #define atomic_set_rel_32 ATOMIC_SAN(set_rel_32) 3643ead6023SMark Johnston #define atomic_subtract_32 ATOMIC_SAN(subtract_32) 3653ead6023SMark Johnston #define atomic_subtract_acq_32 ATOMIC_SAN(subtract_acq_32) 3663ead6023SMark Johnston #define atomic_subtract_rel_32 ATOMIC_SAN(subtract_rel_32) 3673ead6023SMark Johnston #define atomic_store_32 ATOMIC_SAN(store_32) 3683ead6023SMark Johnston #define atomic_store_rel_32 ATOMIC_SAN(store_rel_32) 3693ead6023SMark Johnston #define atomic_swap_32 ATOMIC_SAN(swap_32) 3703ead6023SMark Johnston #define atomic_testandclear_32 ATOMIC_SAN(testandclear_32) 3713ead6023SMark Johnston #define atomic_testandset_32 ATOMIC_SAN(testandset_32) 372435c7cfbSMark Johnston 3733ead6023SMark Johnston #define atomic_add_64 ATOMIC_SAN(add_64) 3743ead6023SMark Johnston #define atomic_add_acq_64 ATOMIC_SAN(add_acq_64) 3753ead6023SMark Johnston #define atomic_add_rel_64 ATOMIC_SAN(add_rel_64) 3763ead6023SMark Johnston #define atomic_clear_64 ATOMIC_SAN(clear_64) 3773ead6023SMark Johnston #define atomic_clear_acq_64 ATOMIC_SAN(clear_acq_64) 3783ead6023SMark Johnston #define atomic_clear_rel_64 ATOMIC_SAN(clear_rel_64) 3793ead6023SMark Johnston #define atomic_cmpset_64 ATOMIC_SAN(cmpset_64) 3803ead6023SMark Johnston #define atomic_cmpset_acq_64 ATOMIC_SAN(cmpset_acq_64) 3813ead6023SMark Johnston #define atomic_cmpset_rel_64 ATOMIC_SAN(cmpset_rel_64) 3823ead6023SMark Johnston #define atomic_fcmpset_64 ATOMIC_SAN(fcmpset_64) 3833ead6023SMark Johnston #define atomic_fcmpset_acq_64 ATOMIC_SAN(fcmpset_acq_64) 3843ead6023SMark Johnston #define atomic_fcmpset_rel_64 ATOMIC_SAN(fcmpset_rel_64) 3853ead6023SMark Johnston #define atomic_fetchadd_64 ATOMIC_SAN(fetchadd_64) 3863ead6023SMark Johnston #define atomic_load_64 ATOMIC_SAN(load_64) 3873ead6023SMark Johnston #define atomic_load_acq_64 ATOMIC_SAN(load_acq_64) 3883ead6023SMark Johnston #define atomic_readandclear_64 ATOMIC_SAN(readandclear_64) 3893ead6023SMark Johnston #define atomic_set_64 ATOMIC_SAN(set_64) 3903ead6023SMark Johnston #define atomic_set_acq_64 ATOMIC_SAN(set_acq_64) 3913ead6023SMark Johnston #define atomic_set_rel_64 ATOMIC_SAN(set_rel_64) 3923ead6023SMark Johnston #define atomic_subtract_64 ATOMIC_SAN(subtract_64) 3933ead6023SMark Johnston #define atomic_subtract_acq_64 ATOMIC_SAN(subtract_acq_64) 3943ead6023SMark Johnston #define atomic_subtract_rel_64 ATOMIC_SAN(subtract_rel_64) 3953ead6023SMark Johnston #define atomic_store_64 ATOMIC_SAN(store_64) 3963ead6023SMark Johnston #define atomic_store_rel_64 ATOMIC_SAN(store_rel_64) 3973ead6023SMark Johnston #define atomic_swap_64 ATOMIC_SAN(swap_64) 3983ead6023SMark Johnston #define atomic_testandclear_64 ATOMIC_SAN(testandclear_64) 3993ead6023SMark Johnston #define atomic_testandset_64 ATOMIC_SAN(testandset_64) 400435c7cfbSMark Johnston 4013ead6023SMark Johnston #define atomic_thread_fence_acq ATOMIC_SAN(thread_fence_acq) 4023ead6023SMark Johnston #define atomic_thread_fence_acq_rel ATOMIC_SAN(thread_fence_acq_rel) 4033ead6023SMark Johnston #define atomic_thread_fence_rel ATOMIC_SAN(thread_fence_rel) 4043ead6023SMark Johnston #define atomic_thread_fence_seq_cst ATOMIC_SAN(thread_fence_seq_cst) 4053ead6023SMark Johnston #define atomic_interrupt_fence ATOMIC_SAN(interrupt_fence) 406435c7cfbSMark Johnston 4073ead6023SMark Johnston #endif /* !SAN_RUNTIME */ 408435c7cfbSMark Johnston 409435c7cfbSMark Johnston #endif /* !_SYS_ATOMIC_SAN_H_ */ 410