1 #ifndef AWS_COMMON_ATOMICS_H 2 #define AWS_COMMON_ATOMICS_H 3 4 #include <aws/common/common.h> 5 6 /** 7 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 8 * SPDX-License-Identifier: Apache-2.0. 9 */ 10 11 /** 12 * struct aws_atomic_var represents an atomic variable - a value which can hold an integer or pointer 13 * that can be manipulated atomically. struct aws_atomic_vars should normally only be manipulated 14 * with atomics methods defined in this header. 15 */ 16 struct aws_atomic_var { 17 void *value; 18 }; 19 /* Helpers for extracting the integer and pointer values from aws_atomic_var. */ 20 #define AWS_ATOMIC_VAR_PTRVAL(var) ((var)->value) 21 #define AWS_ATOMIC_VAR_INTVAL(var) (*(aws_atomic_impl_int_t *)(var)) 22 23 /* 24 * This enumeration specifies the memory ordering properties requested for a particular 25 * atomic operation. The atomic operation may provide stricter ordering than requested. 26 * Note that, within a single thread, all operations are still sequenced (that is, a thread 27 * sees its own atomic writes and reads happening in program order, but other threads may 28 * disagree on this ordering). 29 * 30 * The behavior of these memory orderings are the same as in the C11 atomics API; however, 31 * we only implement a subset that can be portably implemented on the compilers we target. 32 */ 33 34 enum aws_memory_order { 35 /** 36 * No particular ordering constraints are guaranteed relative to other 37 * operations at all; we merely ensure that the operation itself is atomic. 38 */ 39 aws_memory_order_relaxed = 0, 40 /* aws_memory_order_consume - not currently implemented */ 41 42 /** 43 * Specifies acquire ordering. No reads or writes on the current thread can be 44 * reordered to happen before this operation. This is typically paired with a release 45 * ordering; any writes that happened on the releasing operation will be visible 46 * after the paired acquire operation. 47 * 48 * Acquire ordering is only meaningful on load or load-store operations. 49 */ 50 aws_memory_order_acquire = 2, /* leave a spot for consume if we ever add it */ 51 52 /** 53 * Specifies release order. No reads or writes can be reordered to come after this 54 * operation. Typically paired with an acquire operation. 55 * 56 * Release ordering is only meaningful on store or load-store operations. 57 */ 58 aws_memory_order_release, 59 60 /** 61 * Specifies acquire-release order; if this operation acts as a load, it acts as an 62 * acquire operation; if it acts as a store, it acts as a release operation; if it's 63 * a load-store, it does both. 64 */ 65 aws_memory_order_acq_rel, 66 67 /* 68 * Specifies sequentially consistent order. This behaves as acq_rel, but in addition, 69 * all seq_cst operations appear to occur in some globally consistent order. 70 * 71 * TODO: Figure out how to correctly implement this in MSVC. It appears that interlocked 72 * functions provide only acq_rel ordering. 73 */ 74 aws_memory_order_seq_cst 75 }; 76 77 /** 78 * Statically initializes an aws_atomic_var to a given size_t value. 79 */ 80 #define AWS_ATOMIC_INIT_INT(x) \ 81 { .value = (void *)(uintptr_t)(x) } 82 83 /** 84 * Statically initializes an aws_atomic_var to a given void * value. 85 */ 86 #define AWS_ATOMIC_INIT_PTR(x) \ 87 { .value = (void *)(x) } 88 89 AWS_EXTERN_C_BEGIN 90 91 /* 92 * Note: We do not use the C11 atomics API; this is because we want to make sure the representation 93 * (and behavior) of atomic values is consistent, regardless of what --std= flag you pass to your compiler. 94 * Since C11 atomics can silently introduce locks, we run the risk of creating such ABI inconsistencies 95 * if we decide based on compiler features which atomics API to use, and in practice we expect to have 96 * either the GNU or MSVC atomics anyway. 97 * 98 * As future work, we could test to see if the C11 atomics API on this platform behaves consistently 99 * with the other APIs and use it if it does. 100 */ 101 102 /** 103 * Initializes an atomic variable with an integer value. This operation should be done before any 104 * other operations on this atomic variable, and must be done before attempting any parallel operations. 105 * 106 * This operation does not imply a barrier. Ensure that you use an acquire-release barrier (or stronger) 107 * when communicating the fact that initialization is complete to the other thread. Launching the thread 108 * implies a sufficiently strong barrier. 109 */ 110 AWS_STATIC_IMPL 111 void aws_atomic_init_int(volatile struct aws_atomic_var *var, size_t n); 112 113 /** 114 * Initializes an atomic variable with a pointer value. This operation should be done before any 115 * other operations on this atomic variable, and must be done before attempting any parallel operations. 116 * 117 * This operation does not imply a barrier. Ensure that you use an acquire-release barrier (or stronger) 118 * when communicating the fact that initialization is complete to the other thread. Launching the thread 119 * implies a sufficiently strong barrier. 120 */ 121 AWS_STATIC_IMPL 122 void aws_atomic_init_ptr(volatile struct aws_atomic_var *var, void *p); 123 124 /** 125 * Reads an atomic var as an integer, using the specified ordering, and returns the result. 126 */ 127 AWS_STATIC_IMPL 128 size_t aws_atomic_load_int_explicit(volatile const struct aws_atomic_var *var, enum aws_memory_order memory_order); 129 130 /** 131 * Reads an atomic var as an integer, using sequentially consistent ordering, and returns the result. 132 */ 133 AWS_STATIC_IMPL 134 size_t aws_atomic_load_int(volatile const struct aws_atomic_var *var); 135 /** 136 * Reads an atomic var as a pointer, using the specified ordering, and returns the result. 137 */ 138 AWS_STATIC_IMPL 139 void *aws_atomic_load_ptr_explicit(volatile const struct aws_atomic_var *var, enum aws_memory_order memory_order); 140 141 /** 142 * Reads an atomic var as a pointer, using sequentially consistent ordering, and returns the result. 143 */ 144 AWS_STATIC_IMPL 145 void *aws_atomic_load_ptr(volatile const struct aws_atomic_var *var); 146 147 /** 148 * Stores an integer into an atomic var, using the specified ordering. 149 */ 150 AWS_STATIC_IMPL 151 void aws_atomic_store_int_explicit(volatile struct aws_atomic_var *var, size_t n, enum aws_memory_order memory_order); 152 153 /** 154 * Stores an integer into an atomic var, using sequentially consistent ordering. 155 */ 156 AWS_STATIC_IMPL 157 void aws_atomic_store_int(volatile struct aws_atomic_var *var, size_t n); 158 159 /** 160 * Stores a pointer into an atomic var, using the specified ordering. 161 */ 162 AWS_STATIC_IMPL 163 void aws_atomic_store_ptr_explicit(volatile struct aws_atomic_var *var, void *p, enum aws_memory_order memory_order); 164 165 /** 166 * Stores a pointer into an atomic var, using sequentially consistent ordering. 167 */ 168 AWS_STATIC_IMPL 169 void aws_atomic_store_ptr(volatile struct aws_atomic_var *var, void *p); 170 171 /** 172 * Exchanges an integer with the value in an atomic_var, using the specified ordering. 173 * Returns the value that was previously in the atomic_var. 174 */ 175 AWS_STATIC_IMPL 176 size_t aws_atomic_exchange_int_explicit( 177 volatile struct aws_atomic_var *var, 178 size_t n, 179 enum aws_memory_order memory_order); 180 181 /** 182 * Exchanges an integer with the value in an atomic_var, using sequentially consistent ordering. 183 * Returns the value that was previously in the atomic_var. 184 */ 185 AWS_STATIC_IMPL 186 size_t aws_atomic_exchange_int(volatile struct aws_atomic_var *var, size_t n); 187 188 /** 189 * Exchanges a pointer with the value in an atomic_var, using the specified ordering. 190 * Returns the value that was previously in the atomic_var. 191 */ 192 AWS_STATIC_IMPL 193 void *aws_atomic_exchange_ptr_explicit( 194 volatile struct aws_atomic_var *var, 195 void *p, 196 enum aws_memory_order memory_order); 197 198 /** 199 * Exchanges an integer with the value in an atomic_var, using sequentially consistent ordering. 200 * Returns the value that was previously in the atomic_var. 201 */ 202 AWS_STATIC_IMPL 203 void *aws_atomic_exchange_ptr(volatile struct aws_atomic_var *var, void *p); 204 205 /** 206 * Atomically compares *var to *expected; if they are equal, atomically sets *var = desired. Otherwise, *expected is set 207 * to the value in *var. On success, the memory ordering used was order_success; otherwise, it was order_failure. 208 * order_failure must be no stronger than order_success, and must not be release or acq_rel. 209 * Returns true if the compare was successful and the variable updated to desired. 210 */ 211 AWS_STATIC_IMPL 212 bool aws_atomic_compare_exchange_int_explicit( 213 volatile struct aws_atomic_var *var, 214 size_t *expected, 215 size_t desired, 216 enum aws_memory_order order_success, 217 enum aws_memory_order order_failure); 218 219 /** 220 * Atomically compares *var to *expected; if they are equal, atomically sets *var = desired. Otherwise, *expected is set 221 * to the value in *var. Uses sequentially consistent memory ordering, regardless of success or failure. 222 * Returns true if the compare was successful and the variable updated to desired. 223 */ 224 AWS_STATIC_IMPL 225 bool aws_atomic_compare_exchange_int(volatile struct aws_atomic_var *var, size_t *expected, size_t desired); 226 227 /** 228 * Atomically compares *var to *expected; if they are equal, atomically sets *var = desired. Otherwise, *expected is set 229 * to the value in *var. On success, the memory ordering used was order_success; otherwise, it was order_failure. 230 * order_failure must be no stronger than order_success, and must not be release or acq_rel. 231 * Returns true if the compare was successful and the variable updated to desired. 232 */ 233 AWS_STATIC_IMPL 234 bool aws_atomic_compare_exchange_ptr_explicit( 235 volatile struct aws_atomic_var *var, 236 void **expected, 237 void *desired, 238 enum aws_memory_order order_success, 239 enum aws_memory_order order_failure); 240 241 /** 242 * Atomically compares *var to *expected; if they are equal, atomically sets *var = desired. Otherwise, *expected is set 243 * to the value in *var. Uses sequentially consistent memory ordering, regardless of success or failure. 244 * Returns true if the compare was successful and the variable updated to desired. 245 */ 246 AWS_STATIC_IMPL 247 bool aws_atomic_compare_exchange_ptr(volatile struct aws_atomic_var *var, void **expected, void *desired); 248 249 /** 250 * Atomically adds n to *var, and returns the previous value of *var. 251 */ 252 AWS_STATIC_IMPL 253 size_t aws_atomic_fetch_add_explicit(volatile struct aws_atomic_var *var, size_t n, enum aws_memory_order order); 254 255 /** 256 * Atomically subtracts n from *var, and returns the previous value of *var. 257 */ 258 AWS_STATIC_IMPL 259 size_t aws_atomic_fetch_sub_explicit(volatile struct aws_atomic_var *var, size_t n, enum aws_memory_order order); 260 261 /** 262 * Atomically ORs n with *var, and returns the previous value of *var. 263 */ 264 AWS_STATIC_IMPL 265 size_t aws_atomic_fetch_or_explicit(volatile struct aws_atomic_var *var, size_t n, enum aws_memory_order order); 266 267 /** 268 * Atomically ANDs n with *var, and returns the previous value of *var. 269 */ 270 AWS_STATIC_IMPL 271 size_t aws_atomic_fetch_and_explicit(volatile struct aws_atomic_var *var, size_t n, enum aws_memory_order order); 272 273 /** 274 * Atomically XORs n with *var, and returns the previous value of *var. 275 */ 276 AWS_STATIC_IMPL 277 size_t aws_atomic_fetch_xor_explicit(volatile struct aws_atomic_var *var, size_t n, enum aws_memory_order order); 278 279 /** 280 * Atomically adds n to *var, and returns the previous value of *var. 281 * Uses sequentially consistent ordering. 282 */ 283 AWS_STATIC_IMPL 284 size_t aws_atomic_fetch_add(volatile struct aws_atomic_var *var, size_t n); 285 286 /** 287 * Atomically subtracts n from *var, and returns the previous value of *var. 288 * Uses sequentially consistent ordering. 289 */ 290 AWS_STATIC_IMPL 291 size_t aws_atomic_fetch_sub(volatile struct aws_atomic_var *var, size_t n); 292 293 /** 294 * Atomically ands n into *var, and returns the previous value of *var. 295 * Uses sequentially consistent ordering. 296 */ 297 AWS_STATIC_IMPL 298 size_t aws_atomic_fetch_and(volatile struct aws_atomic_var *var, size_t n); 299 300 /** 301 * Atomically ors n into *var, and returns the previous value of *var. 302 * Uses sequentially consistent ordering. 303 */ 304 AWS_STATIC_IMPL 305 size_t aws_atomic_fetch_or(volatile struct aws_atomic_var *var, size_t n); 306 307 /** 308 * Atomically xors n into *var, and returns the previous value of *var. 309 * Uses sequentially consistent ordering. 310 */ 311 AWS_STATIC_IMPL 312 size_t aws_atomic_fetch_xor(volatile struct aws_atomic_var *var, size_t n); 313 314 /** 315 * Provides the same reordering guarantees as an atomic operation with the specified memory order, without 316 * needing to actually perform an atomic operation. 317 */ 318 AWS_STATIC_IMPL 319 void aws_atomic_thread_fence(enum aws_memory_order order); 320 321 #ifndef AWS_NO_STATIC_IMPL 322 # include <aws/common/atomics.inl> 323 #endif /* AWS_NO_STATIC_IMPL */ 324 325 AWS_EXTERN_C_END 326 327 #endif 328