1 /* 2 * An implementation of C11 stdatomic.h directly borrowed from FreeBSD 3 * (original copyright follows), with minor modifications for 4 * portability to other systems. Works for recent Clang (that 5 * implement the feature c_atomic) and GCC 4.7+; includes 6 * compatibility for GCC below 4.7 but I wouldn't recommend it. 7 * 8 * Caveats and limitations: 9 * - Only the ``_Atomic parentheses'' notation is implemented, while 10 * the ``_Atomic space'' one is not. 11 * - _Atomic types must be typedef'ed, or programs using them will 12 * not type check correctly (incompatible anonymous structure 13 * types). 14 * - Non-scalar _Atomic types would require runtime support for 15 * runtime locking, which, as far as I know, is not currently 16 * available on any system. 17 */ 18 19 /*- 20 * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org> 21 * David Chisnall <theraven@FreeBSD.org> 22 * All rights reserved. 23 * 24 * Redistribution and use in source and binary forms, with or without 25 * modification, are permitted provided that the following conditions 26 * are met: 27 * 1. Redistributions of source code must retain the above copyright 28 * notice, this list of conditions and the following disclaimer. 29 * 2. Redistributions in binary form must reproduce the above copyright 30 * notice, this list of conditions and the following disclaimer in the 31 * documentation and/or other materials provided with the distribution. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * $FreeBSD: src/include/stdatomic.h,v 1.10.2.2 2012/05/30 19:21:54 theraven Exp $ 46 */ 47 48 #ifndef _STDATOMIC_H_ 49 #define _STDATOMIC_H_ 50 51 #include <stddef.h> 52 #include <stdint.h> 53 54 #if !defined(__has_feature) 55 #define __has_feature(x) 0 56 #endif 57 #if !defined(__has_builtin) 58 #define __has_builtin(x) 0 59 #endif 60 #if !defined(__GNUC_PREREQ__) 61 #if defined(__GNUC__) && defined(__GNUC_MINOR__) 62 #define __GNUC_PREREQ__(maj, min) \ 63 ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) 64 #else 65 #define __GNUC_PREREQ__(maj, min) 0 66 #endif 67 #endif 68 69 #if !defined(__CLANG_ATOMICS) && !defined(__GNUC_ATOMICS) 70 #if __has_feature(c_atomic) 71 #define __CLANG_ATOMICS 72 #elif __GNUC_PREREQ__(4, 7) 73 #define __GNUC_ATOMICS 74 #elif !defined(__GNUC__) 75 #error "stdatomic.h does not support your compiler" 76 #endif 77 #endif 78 79 #if !defined(__CLANG_ATOMICS) 80 #define _Atomic(T) struct { volatile __typeof__(T) __val; } 81 #endif 82 83 /* 84 * 7.17.2 Initialization. 85 */ 86 87 #if defined(__CLANG_ATOMICS) 88 #define ATOMIC_VAR_INIT(value) (value) 89 #define atomic_init(obj, value) __c11_atomic_init(obj, value) 90 #else 91 #define ATOMIC_VAR_INIT(value) { .__val = (value) } 92 #define atomic_init(obj, value) do { \ 93 (obj)->__val = (value); \ 94 } while (0) 95 #endif 96 97 /* 98 * Clang and recent GCC both provide predefined macros for the memory 99 * orderings. If we are using a compiler that doesn't define them, use the 100 * clang values - these will be ignored in the fallback path. 101 */ 102 103 #ifndef __ATOMIC_RELAXED 104 #define __ATOMIC_RELAXED 0 105 #endif 106 #ifndef __ATOMIC_CONSUME 107 #define __ATOMIC_CONSUME 1 108 #endif 109 #ifndef __ATOMIC_ACQUIRE 110 #define __ATOMIC_ACQUIRE 2 111 #endif 112 #ifndef __ATOMIC_RELEASE 113 #define __ATOMIC_RELEASE 3 114 #endif 115 #ifndef __ATOMIC_ACQ_REL 116 #define __ATOMIC_ACQ_REL 4 117 #endif 118 #ifndef __ATOMIC_SEQ_CST 119 #define __ATOMIC_SEQ_CST 5 120 #endif 121 122 /* 123 * 7.17.3 Order and consistency. 124 * 125 * The memory_order_* constants that denote the barrier behaviour of the 126 * atomic operations. 127 */ 128 129 enum memory_order { 130 memory_order_relaxed = __ATOMIC_RELAXED, 131 memory_order_consume = __ATOMIC_CONSUME, 132 memory_order_acquire = __ATOMIC_ACQUIRE, 133 memory_order_release = __ATOMIC_RELEASE, 134 memory_order_acq_rel = __ATOMIC_ACQ_REL, 135 memory_order_seq_cst = __ATOMIC_SEQ_CST 136 }; 137 138 typedef enum memory_order memory_order; 139 140 /* 141 * 7.17.4 Fences. 142 */ 143 144 #ifdef __CLANG_ATOMICS 145 #define atomic_thread_fence(order) __c11_atomic_thread_fence(order) 146 #define atomic_signal_fence(order) __c11_atomic_signal_fence(order) 147 #elif defined(__GNUC_ATOMICS) 148 #define atomic_thread_fence(order) __atomic_thread_fence(order) 149 #define atomic_signal_fence(order) __atomic_signal_fence(order) 150 #else 151 #define atomic_thread_fence(order) __sync_synchronize() 152 #define atomic_signal_fence(order) __asm volatile ("" : : : "memory") 153 #endif 154 155 /* 156 * 7.17.5 Lock-free property. 157 */ 158 159 #if defined(__CLANG_ATOMICS) 160 #define atomic_is_lock_free(obj) \ 161 __c11_atomic_is_lock_free(sizeof(obj)) 162 #elif defined(__GNUC_ATOMICS) 163 #define atomic_is_lock_free(obj) \ 164 __atomic_is_lock_free(sizeof((obj)->__val)) 165 #else 166 #define atomic_is_lock_free(obj) \ 167 (sizeof((obj)->__val) <= sizeof(void *)) 168 #endif 169 170 /* 171 * 7.17.6 Atomic integer types. 172 */ 173 174 typedef _Atomic(_Bool) atomic_bool; 175 typedef _Atomic(char) atomic_char; 176 typedef _Atomic(signed char) atomic_schar; 177 typedef _Atomic(unsigned char) atomic_uchar; 178 typedef _Atomic(short) atomic_short; 179 typedef _Atomic(unsigned short) atomic_ushort; 180 typedef _Atomic(int) atomic_int; 181 typedef _Atomic(unsigned int) atomic_uint; 182 typedef _Atomic(long) atomic_long; 183 typedef _Atomic(unsigned long) atomic_ulong; 184 typedef _Atomic(long long) atomic_llong; 185 typedef _Atomic(unsigned long long) atomic_ullong; 186 #if 0 187 typedef _Atomic(char16_t) atomic_char16_t; 188 typedef _Atomic(char32_t) atomic_char32_t; 189 #endif 190 typedef _Atomic(wchar_t) atomic_wchar_t; 191 typedef _Atomic(int_least8_t) atomic_int_least8_t; 192 typedef _Atomic(uint_least8_t) atomic_uint_least8_t; 193 typedef _Atomic(int_least16_t) atomic_int_least16_t; 194 typedef _Atomic(uint_least16_t) atomic_uint_least16_t; 195 typedef _Atomic(int_least32_t) atomic_int_least32_t; 196 typedef _Atomic(uint_least32_t) atomic_uint_least32_t; 197 typedef _Atomic(int_least64_t) atomic_int_least64_t; 198 typedef _Atomic(uint_least64_t) atomic_uint_least64_t; 199 typedef _Atomic(int_fast8_t) atomic_int_fast8_t; 200 typedef _Atomic(uint_fast8_t) atomic_uint_fast8_t; 201 typedef _Atomic(int_fast16_t) atomic_int_fast16_t; 202 typedef _Atomic(uint_fast16_t) atomic_uint_fast16_t; 203 typedef _Atomic(int_fast32_t) atomic_int_fast32_t; 204 typedef _Atomic(uint_fast32_t) atomic_uint_fast32_t; 205 typedef _Atomic(int_fast64_t) atomic_int_fast64_t; 206 typedef _Atomic(uint_fast64_t) atomic_uint_fast64_t; 207 typedef _Atomic(intptr_t) atomic_intptr_t; 208 typedef _Atomic(uintptr_t) atomic_uintptr_t; 209 typedef _Atomic(size_t) atomic_size_t; 210 typedef _Atomic(ptrdiff_t) atomic_ptrdiff_t; 211 typedef _Atomic(intmax_t) atomic_intmax_t; 212 typedef _Atomic(uintmax_t) atomic_uintmax_t; 213 214 /* 215 * 7.17.7 Operations on atomic types. 216 */ 217 218 /* 219 * Compiler-specific operations. 220 */ 221 222 #if defined(__CLANG_ATOMICS) 223 #define atomic_compare_exchange_strong_explicit(object, expected, \ 224 desired, success, failure) \ 225 __c11_atomic_compare_exchange_strong(object, expected, desired, \ 226 success, failure) 227 #define atomic_compare_exchange_weak_explicit(object, expected, \ 228 desired, success, failure) \ 229 __c11_atomic_compare_exchange_weak(object, expected, desired, \ 230 success, failure) 231 #define atomic_exchange_explicit(object, desired, order) \ 232 __c11_atomic_exchange(object, desired, order) 233 #define atomic_fetch_add_explicit(object, operand, order) \ 234 __c11_atomic_fetch_add(object, operand, order) 235 #define atomic_fetch_and_explicit(object, operand, order) \ 236 __c11_atomic_fetch_and(object, operand, order) 237 #define atomic_fetch_or_explicit(object, operand, order) \ 238 __c11_atomic_fetch_or(object, operand, order) 239 #define atomic_fetch_sub_explicit(object, operand, order) \ 240 __c11_atomic_fetch_sub(object, operand, order) 241 #define atomic_fetch_xor_explicit(object, operand, order) \ 242 __c11_atomic_fetch_xor(object, operand, order) 243 #define atomic_load_explicit(object, order) \ 244 __c11_atomic_load(object, order) 245 #define atomic_store_explicit(object, desired, order) \ 246 __c11_atomic_store(object, desired, order) 247 #elif defined(__GNUC_ATOMICS) 248 #define atomic_compare_exchange_strong_explicit(object, expected, \ 249 desired, success, failure) \ 250 __atomic_compare_exchange_n(&(object)->__val, expected, \ 251 desired, 0, success, failure) 252 #define atomic_compare_exchange_weak_explicit(object, expected, \ 253 desired, success, failure) \ 254 __atomic_compare_exchange_n(&(object)->__val, expected, \ 255 desired, 1, success, failure) 256 #define atomic_exchange_explicit(object, desired, order) \ 257 __atomic_exchange_n(&(object)->__val, desired, order) 258 #define atomic_fetch_add_explicit(object, operand, order) \ 259 __atomic_fetch_add(&(object)->__val, operand, order) 260 #define atomic_fetch_and_explicit(object, operand, order) \ 261 __atomic_fetch_and(&(object)->__val, operand, order) 262 #define atomic_fetch_or_explicit(object, operand, order) \ 263 __atomic_fetch_or(&(object)->__val, operand, order) 264 #define atomic_fetch_sub_explicit(object, operand, order) \ 265 __atomic_fetch_sub(&(object)->__val, operand, order) 266 #define atomic_fetch_xor_explicit(object, operand, order) \ 267 __atomic_fetch_xor(&(object)->__val, operand, order) 268 #define atomic_load_explicit(object, order) \ 269 __atomic_load_n(&(object)->__val, order) 270 #define atomic_store_explicit(object, desired, order) \ 271 __atomic_store_n(&(object)->__val, desired, order) 272 #else 273 #define atomic_compare_exchange_strong_explicit(object, expected, \ 274 desired, success, failure) ({ \ 275 __typeof__((object)->__val) __v; \ 276 _Bool __r; \ 277 __v = __sync_val_compare_and_swap(&(object)->__val, \ 278 *(expected), desired); \ 279 __r = *(expected) == __v; \ 280 *(expected) = __v; \ 281 __r; \ 282 }) 283 284 #define atomic_compare_exchange_weak_explicit(object, expected, \ 285 desired, success, failure) \ 286 atomic_compare_exchange_strong_explicit(object, expected, \ 287 desired, success, failure) 288 #if __has_builtin(__sync_swap) 289 /* Clang provides a full-barrier atomic exchange - use it if available. */ 290 #define atomic_exchange_explicit(object, desired, order) \ 291 __sync_swap(&(object)->__val, desired) 292 #else 293 /* 294 * __sync_lock_test_and_set() is only an acquire barrier in theory (although in 295 * practice it is usually a full barrier) so we need an explicit barrier after 296 * it. 297 */ 298 #define atomic_exchange_explicit(object, desired, order) ({ \ 299 __typeof__((object)->__val) __v; \ 300 __v = __sync_lock_test_and_set(&(object)->__val, desired); \ 301 __sync_synchronize(); \ 302 __v; \ 303 }) 304 #endif 305 #define atomic_fetch_add_explicit(object, operand, order) \ 306 __sync_fetch_and_add(&(object)->__val, operand) 307 #define atomic_fetch_and_explicit(object, operand, order) \ 308 __sync_fetch_and_and(&(object)->__val, operand) 309 #define atomic_fetch_or_explicit(object, operand, order) \ 310 __sync_fetch_and_or(&(object)->__val, operand) 311 #define atomic_fetch_sub_explicit(object, operand, order) \ 312 __sync_fetch_and_sub(&(object)->__val, operand) 313 #define atomic_fetch_xor_explicit(object, operand, order) \ 314 __sync_fetch_and_xor(&(object)->__val, operand) 315 #define atomic_load_explicit(object, order) \ 316 __sync_fetch_and_add(&(object)->__val, 0) 317 #define atomic_store_explicit(object, desired, order) do { \ 318 __sync_synchronize(); \ 319 (object)->__val = (desired); \ 320 __sync_synchronize(); \ 321 } while (0) 322 #endif 323 324 /* 325 * Convenience functions. 326 */ 327 328 #define atomic_compare_exchange_strong(object, expected, desired) \ 329 atomic_compare_exchange_strong_explicit(object, expected, \ 330 desired, memory_order_seq_cst, memory_order_seq_cst) 331 #define atomic_compare_exchange_weak(object, expected, desired) \ 332 atomic_compare_exchange_weak_explicit(object, expected, \ 333 desired, memory_order_seq_cst, memory_order_seq_cst) 334 #define atomic_exchange(object, desired) \ 335 atomic_exchange_explicit(object, desired, memory_order_seq_cst) 336 #define atomic_fetch_add(object, operand) \ 337 atomic_fetch_add_explicit(object, operand, memory_order_seq_cst) 338 #define atomic_fetch_and(object, operand) \ 339 atomic_fetch_and_explicit(object, operand, memory_order_seq_cst) 340 #define atomic_fetch_or(object, operand) \ 341 atomic_fetch_or_explicit(object, operand, memory_order_seq_cst) 342 #define atomic_fetch_sub(object, operand) \ 343 atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst) 344 #define atomic_fetch_xor(object, operand) \ 345 atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst) 346 #define atomic_load(object) \ 347 atomic_load_explicit(object, memory_order_seq_cst) 348 #define atomic_store(object, desired) \ 349 atomic_store_explicit(object, desired, memory_order_seq_cst) 350 351 /* 352 * 7.17.8 Atomic flag type and operations. 353 */ 354 355 typedef atomic_bool atomic_flag; 356 357 #define ATOMIC_FLAG_INIT ATOMIC_VAR_INIT(0) 358 359 #define atomic_flag_clear_explicit(object, order) \ 360 atomic_store_explicit(object, 0, order) 361 #define atomic_flag_test_and_set_explicit(object, order) \ 362 atomic_compare_exchange_strong_explicit(object, 0, 1, order, order) 363 364 #define atomic_flag_clear(object) \ 365 atomic_flag_clear_explicit(object, memory_order_seq_cst) 366 #define atomic_flag_test_and_set(object) \ 367 atomic_flag_test_and_set_explicit(object, memory_order_seq_cst) 368 369 #endif /* !_STDATOMIC_H_ */ 370