1 /*
2 * --------------- DO NOT EDIT THIS FILE! ---------------
3 * This file was automatically generated by the
4 * $ERL_TOP/erts/lib_src/utils/make_atomics_api script.
5 * If you need to make changes, edit the script and
6 * regenerate this file.
7 * --------------- DO NOT EDIT THIS FILE! ---------------
8 */
9
10 /*
11 * %CopyrightBegin%
12 *
13 * Copyright Ericsson AB 2011-2016. All Rights Reserved.
14 *
15 * Licensed under the Apache License, Version 2.0 (the "License");
16 * you may not use this file except in compliance with the License.
17 * You may obtain a copy of the License at
18 *
19 * http://www.apache.org/licenses/LICENSE-2.0
20 *
21 * Unless required by applicable law or agreed to in writing, software
22 * distributed under the License is distributed on an "AS IS" BASIS,
23 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
24 * See the License for the specific language governing permissions and
25 * limitations under the License.
26 *
27 * %CopyrightEnd%
28 */
29
30 /*
31 * Description: The ethread atomics API
32 * Author: Rickard Green
33 */
34
35 /*
36 * This file maps native atomic implementations to ethread
37 * API atomics. If no native atomic implementation
38 * is available, a less efficient fallback is used instead.
39 * The API consists of 32-bit size, word size (pointer size),
40 * and double word size atomics.
41 *
42 * The following atomic operations are implemented for
43 * 32-bit size, and word size atomics:
44 * - cmpxchg
45 * - xchg
46 * - set
47 * - init
48 * - add_read
49 * - read
50 * - inc_read
51 * - dec_read
52 * - add
53 * - inc
54 * - dec
55 * - read_band
56 * - read_bor
57 *
58 * The following atomic operations are implemented for
59 * double word size atomics:
60 * - cmpxchg
61 * - set
62 * - read
63 * - init
64 *
65 * Appart from a function implementing the atomic operation
66 * with unspecified memory barrier semantics, there are
67 * functions implementing each operation with the following
68 * implied memory barrier semantics:
69 * - mb - Full memory barrier. Orders both loads, and
70 * stores before, and after the atomic operation.
71 * No load or store is allowed to be reordered
72 * over the atomic operation.
73 * - relb - Release barrier. Orders both loads, and
74 * stores appearing *before* the atomic
75 * operation. These are not allowed to be
76 * reordered over the atomic operation.
77 * - acqb - Acquire barrier. Orders both loads, and stores
78 * appearing *after* the atomic operation. These
79 * are not allowed to be reordered over the
80 * atomic operation.
81 * - wb - Write barrier. Orders *only* stores. These are
82 * not allowed to be reordered over the barrier.
83 * Store in atomic operation is ordered *after*
84 * the barrier.
85 * - rb - Read barrier. Orders *only* loads. These are
86 * not allowed to be reordered over the barrier.
87 * Load in atomic operation is ordered *before*
88 * the barrier.
89 * - ddrb - Data dependency read barrier. Orders *only*
90 * loads according to data dependency across the
91 * barrier. Load in atomic operation is ordered
92 * before the barrier.
93 *
94 * We implement all of these operation/barrier
95 * combinations, regardless of whether they are useful
96 * or not (some of them are useless).
97 *
98 * Double word size atomic functions are on the followning
99 * form:
100 * ethr_dw_atomic_<OP>[_<BARRIER>]
101 *
102 * Word size atomic functions are on the followning
103 * form:
104 * ethr_atomic_<OP>[_<BARRIER>]
105 *
106 * 32-bit size atomic functions are on the followning
107 * form:
108 * ethr_atomic32_<OP>[_<BARRIER>]
109 *
110 * Apart from the operation/barrier functions
111 * described above also 'addr' functions are implemented
112 * which return the actual memory address used of the
113 * atomic variable. The 'addr' functions have no barrier
114 * versions.
115 *
116 * The native atomic implementation does not need to
117 * implement all operation/barrier combinations.
118 * Functions that have no native implementation will be
119 * constructed from existing native functionality. These
120 * functions will perform the wanted operation and will
121 * produce sufficient memory barriers, but may
122 * in some cases be less efficient than pure native
123 * versions.
124 *
125 * When we create ethread API operation/barrier functions by
126 * adding barriers before and after native operations it is
127 * assumed that:
128 * - A native read operation begins, and ends with a load.
129 * - A native set operation begins, and ends with a store.
130 * - An init operation begins with either a load, or a store,
131 * and ends with either a load, or a store.
132 * - All other operations begins with a load, and ends with
133 * either a load, or a store.
134 *
135 * This is the minimum functionality that a native
136 * implementation needs to provide:
137 *
138 * - Functions that need to be implemented:
139 *
140 * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
141 * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
142 * (at least one cmpxchg of optional barrier)
143 *
144 * - Macros that needs to be defined:
145 *
146 * A macro informing about the presence of the native
147 * implementation:
148 *
149 * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
150 *
151 * A macro naming (a string constant) the implementation:
152 *
153 * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
154 *
155 * Each implemented native atomic function has to
156 * be accompanied by a defined macro on the following
157 * form informing about its presence:
158 *
159 * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
160 *
161 * A (sparc-v9 style) membar macro:
162 *
163 * - ETHR_MEMBAR(B)
164 *
165 * Which takes a combination of the following macros
166 * or:ed (using |) together:
167 *
168 * - ETHR_LoadLoad
169 * - ETHR_LoadStore
170 * - ETHR_StoreLoad
171 * - ETHR_StoreStore
172 *
173 */
174
175
176 #ifdef HAVE_CONFIG_H
177 #include "config.h"
178 #endif
179
180 #define ETHR_TRY_INLINE_FUNCS
181 #define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __
182 #define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __
183 #define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __
184 #define ETHR_ATOMIC_IMPL__
185
186 #include "ethread.h"
187 #include "ethr_internal.h"
188
189 #if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
190 || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
191 /*
192 * Spinlock based fallback for atomics used in absence of a native
193 * implementation.
194 */
195
196 #define ETHR_ATMC_FLLBK_ADDR_BITS 10
197 #define ETHR_ATMC_FLLBK_ADDR_SHIFT 6
198
199 typedef struct {
200 union {
201 ethr_spinlock_t lck;
202 char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))];
203 } u;
204 } ethr_atomic_protection_t;
205
206 extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
207
208 #define ETHR_ATOMIC_PTR2LCK__(PTR) \
209 (ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \
210 & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck)
211
212
213 #define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \
214 do { \
215 ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \
216 ethr_spin_lock(slp__); \
217 { EXPS; } \
218 ethr_spin_unlock(slp__); \
219 } while (0)
220
221 ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
222
223 #endif
224
225
226 #if defined(ETHR_AMC_FALLBACK__)
227
228 /*
229 * Fallback for large sized (word and/or double word size) atomics using
230 * an "Atomic Modification Counter" based on smaller sized native atomics.
231 *
232 * We use a 63-bit modification counter and a one bit exclusive flag.
233 * If 32-bit native atomics are used, we need two 32-bit native atomics.
234 * The exclusive flag is the least significant bit, or if multiple atomics
235 * are used, the least significant bit of the least significant atomic.
236 *
237 * When using the AMC fallback the following is true:
238 * - Reads of the same atomic variable can be done in parallel.
239 * - Uncontended reads doesn't cause any cache line invalidations,
240 * since no modifications are done.
241 * - Assuming that the AMC atomic(s) and the integer(s) containing the
242 * value of the implemented atomic resides in the same cache line,
243 * modifications will only cause invalidations of one cache line.
244 *
245 * When using the spinlock based fallback none of the above is true,
246 * however, the spinlock based fallback consumes less memory.
247 */
248
249 # if ETHR_AMC_NO_ATMCS__ != 1 && ETHR_AMC_NO_ATMCS__ != 2
250 # error "Not supported"
251 # endif
252 # define ETHR_AMC_MAX_TRY_READ__ 10
253 # ifdef ETHR_DEBUG
254 # define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \
255 do { \
256 ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \
257 ETHR_ASSERT(act == (S) + 1); \
258 ETHR_ASSERT(act & 1); \
259 } while (0)
260 # else
261 # define ETHR_DBG_CHK_EXCL_STATE(ASP, S)
262 # endif
263
264 static ETHR_INLINE void
amc_init(ethr_amc_t * amc,int dw,ethr_sint_t * avar,ethr_sint_t * val)265 amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
266 {
267 avar[0] = val[0];
268 if (dw)
269 avar[1] = val[1];
270 #if ETHR_AMC_NO_ATMCS__ == 2
271 ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0);
272 #endif
273 ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0);
274 }
275
276 static ETHR_INLINE ETHR_AMC_SINT_T__
amc_set_excl(ethr_amc_t * amc,ETHR_AMC_SINT_T__ prev_state0)277 amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0)
278 {
279 ETHR_AMC_SINT_T__ state0 = prev_state0;
280 /* Set exclusive flag. */
281 while (1) {
282 ETHR_AMC_SINT_T__ act_state0, new_state0;
283 while (state0 & 1) { /* Wait until exclusive bit has been cleared */
284 ETHR_SPIN_BODY;
285 state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
286 }
287 /* Try to set exclusive bit */
288 new_state0 = state0 + 1;
289 act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0],
290 new_state0,
291 state0);
292 if (state0 == act_state0)
293 return state0; /* old state0 */
294 state0 = act_state0;
295 }
296 }
297
298 static ETHR_INLINE void
amc_inc_mc_unset_excl(ethr_amc_t * amc,ETHR_AMC_SINT_T__ old_state0)299 amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
300 {
301 ETHR_AMC_SINT_T__ state0 = old_state0;
302
303 /* Increment modification counter and reset exclusive flag. */
304
305 ETHR_DBG_CHK_EXCL_STATE(amc, state0);
306
307 state0 += 2;
308
309 ETHR_ASSERT((state0 & 1) == 0);
310
311 #if ETHR_AMC_NO_ATMCS__ == 2
312 if (state0 == 0) {
313 /*
314 * state0 wrapped, so we need to increment state1. There is no need
315 * for atomic inc op, since this is always done while having exclusive
316 * flag.
317 */
318 ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]);
319 state1++;
320 ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1);
321 }
322 #endif
323 ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0);
324 }
325
326 static ETHR_INLINE void
amc_unset_excl(ethr_amc_t * amc,ETHR_AMC_SINT_T__ old_state0)327 amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
328 {
329 ETHR_DBG_CHK_EXCL_STATE(amc, old_state0);
330 /*
331 * Reset exclusive flag, but leave modification counter unchanged,
332 * i.e., restore state to what it was before setting exclusive
333 * flag.
334 */
335 ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0);
336 }
337
338 static ETHR_INLINE void
amc_set(ethr_amc_t * amc,int dw,ethr_sint_t * avar,ethr_sint_t * val)339 amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
340 {
341 ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
342
343 state0 = amc_set_excl(amc, state0);
344
345 avar[0] = val[0];
346 if (dw)
347 avar[1] = val[1];
348
349 amc_inc_mc_unset_excl(amc, state0);
350 }
351
352 static ETHR_INLINE int
amc_try_read(ethr_amc_t * amc,int dw,ethr_sint_t * avar,ethr_sint_t * val,ETHR_AMC_SINT_T__ * state0p)353 amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
354 ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p)
355 {
356 /* *state0p should contain last read value if aborting */
357 ETHR_AMC_SINT_T__ old_state0;
358 #if ETHR_AMC_NO_ATMCS__ == 2
359 ETHR_AMC_SINT_T__ state1;
360 int abrt;
361 #endif
362
363 *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
364 if ((*state0p) & 1)
365 return 0; /* exclusive flag set; abort */
366 #if ETHR_AMC_NO_ATMCS__ == 2
367 state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]);
368 #else
369 ETHR_COMPILER_BARRIER;
370 #endif
371
372 val[0] = avar[0];
373 if (dw)
374 val[1] = avar[1];
375
376 ETHR_READ_MEMORY_BARRIER;
377
378 /*
379 * Abort if state has changed (i.e, either the exclusive
380 * flag is set, or modification counter changed).
381 */
382 old_state0 = *state0p;
383 #if ETHR_AMC_NO_ATMCS__ == 2
384 *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
385 abrt = (old_state0 != *state0p);
386 abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]));
387 return abrt == 0;
388 #else
389 *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
390 return old_state0 == *state0p;
391 #endif
392 }
393
394 static ETHR_INLINE void
amc_read(ethr_amc_t * amc,int dw,ethr_sint_t * avar,ethr_sint_t * val)395 amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
396 {
397 ETHR_AMC_SINT_T__ state0;
398 int i;
399
400 #if ETHR_AMC_MAX_TRY_READ__ == 0
401 state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
402 #else
403 for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) {
404 if (amc_try_read(amc, dw, avar, val, &state0))
405 return; /* read success */
406 ETHR_SPIN_BODY;
407 }
408 #endif
409
410 state0 = amc_set_excl(amc, state0);
411
412 val[0] = avar[0];
413 if (dw)
414 val[1] = avar[1];
415
416 amc_unset_excl(amc, state0);
417 }
418
419 static ETHR_INLINE int
amc_cmpxchg(ethr_amc_t * amc,int dw,ethr_sint_t * avar,ethr_sint_t * new,ethr_sint_t * xchg)420 amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
421 ethr_sint_t *new, ethr_sint_t *xchg)
422 {
423 ethr_sint_t val[2];
424 ETHR_AMC_SINT_T__ state0;
425
426 if (amc_try_read(amc, dw, avar, val, &state0)) {
427 if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) {
428 xchg[0] = val[0];
429 if (dw)
430 xchg[1] = val[1];
431 return 0; /* failed */
432 }
433 /* Operation will succeed if not interrupted */
434 }
435
436 state0 = amc_set_excl(amc, state0);
437
438 if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) {
439 xchg[0] = avar[0];
440 if (dw)
441 xchg[1] = avar[1];
442
443 ETHR_DBG_CHK_EXCL_STATE(amc, state0);
444
445 amc_unset_excl(amc, state0);
446 return 0; /* failed */
447 }
448
449 avar[0] = new[0];
450 if (dw)
451 avar[1] = new[1];
452
453 amc_inc_mc_unset_excl(amc, state0);
454 return 1;
455 }
456
457
458 #define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS) \
459 do { \
460 ETHR_AMC_SINT_T__ state0__; \
461 state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]); \
462 state0__ = amc_set_excl((AMC), state0__); \
463 { OPS; } \
464 amc_inc_mc_unset_excl((AMC), state0__); \
465 } while (0)
466
467 #endif /* amc fallback */
468
469
470 int
ethr_init_atomics(void)471 ethr_init_atomics(void)
472 {
473 #if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
474 || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
475 int i;
476 for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) {
477 int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck);
478 if (res != 0)
479 return res;
480 }
481 #endif
482 return 0;
483 }
484
485
486 /* ---------- Double word size atomic implementation ---------- */
487
488
489
490 /*
491 * Double word atomics need runtime test.
492 */
493
ethr_have_native_dw_atomic(void)494 int ethr_have_native_dw_atomic(void)
495 {
496 return ethr_have_native_dw_atomic__();
497 }
498
499
500 /* --- addr() --- */
501
ETHR_DW_ATOMIC_FUNC__(addr)502 ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var)
503 {
504 ethr_sint_t *res;
505 ETHR_ASSERT(!ethr_not_inited__);
506 ETHR_ASSERT(var);
507 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
508 res = ethr_dw_atomic_addr__(var);
509 #elif defined(ETHR_AMC_FALLBACK__)
510 res = (ethr_sint_t *) ((&var->fallback))->sint;
511 #else
512 res = (ethr_sint_t *) (&var->fallback);
513 #endif
514 return res;
515 }
516
517 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_addr(ethr_dw_atomic_t * var)518 ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var)
519 {
520 ETHR_ASSERT(!ethr_not_inited__);
521 ETHR_ASSERT(var);
522 return ethr_dw_atomic_addr__(var);
523 }
524 #endif
525
526
527 /* -- cmpxchg() -- */
528
529
ETHR_DW_ATOMIC_FUNC__(cmpxchg)530 int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
531 {
532 int res;
533 ETHR_ASSERT(!ethr_not_inited__);
534 ETHR_ASSERT(var);
535 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
536 res = ethr_dw_atomic_cmpxchg__(var, val, old_val);
537 #elif defined(ETHR_AMC_FALLBACK__)
538 res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
539 #else
540 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
541 {
542 res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
543 if (res) {
544 (&var->fallback)->sint[0] = val->sint[0];
545 (&var->fallback)->sint[1] = val->sint[1];
546 }
547 else {
548 old_val->sint[0] = (&var->fallback)->sint[0];
549 old_val->sint[1] = (&var->fallback)->sint[1];
550 }
551 });
552 #endif
553 return res;
554 }
555
556 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t * var,ethr_dw_sint_t * val,ethr_dw_sint_t * old_val)557 int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
558 {
559 ETHR_ASSERT(!ethr_not_inited__);
560 ETHR_ASSERT(var);
561 return ethr_dw_atomic_cmpxchg__(var, val, old_val);
562 }
563 #endif
564
ethr_dw_atomic_cmpxchg_ddrb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val,ethr_dw_sint_t * old_val)565 int ethr_dw_atomic_cmpxchg_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
566 {
567 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS)
568 return ethr_dw_atomic_cmpxchg_ddrb__(var, val, old_val);
569 #elif defined(ETHR_ORDERED_READ_DEPEND)
570 return ethr_dw_atomic_cmpxchg(var, val, old_val);
571 #else
572 return ethr_dw_atomic_cmpxchg_rb(var, val, old_val);
573 #endif
574 }
575
ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)576 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
577 {
578 int res;
579 ETHR_ASSERT(!ethr_not_inited__);
580 ETHR_ASSERT(var);
581 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
582 res = ethr_dw_atomic_cmpxchg_rb__(var, val, old_val);
583 #elif defined(ETHR_AMC_FALLBACK__)
584 res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
585 ETHR_MEMBAR(ETHR_LoadLoad);
586 #else
587 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
588 {
589 res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
590 if (res) {
591 (&var->fallback)->sint[0] = val->sint[0];
592 (&var->fallback)->sint[1] = val->sint[1];
593 }
594 else {
595 old_val->sint[0] = (&var->fallback)->sint[0];
596 old_val->sint[1] = (&var->fallback)->sint[1];
597 }
598 });
599 ETHR_MEMBAR(ETHR_LoadLoad);
600 #endif
601 return res;
602 }
603
604 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val,ethr_dw_sint_t * old_val)605 int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
606 {
607 ETHR_ASSERT(!ethr_not_inited__);
608 ETHR_ASSERT(var);
609 return ethr_dw_atomic_cmpxchg_rb__(var, val, old_val);
610 }
611 #endif
612
ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)613 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
614 {
615 int res;
616 ETHR_ASSERT(!ethr_not_inited__);
617 ETHR_ASSERT(var);
618 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
619 res = ethr_dw_atomic_cmpxchg_wb__(var, val, old_val);
620 #elif defined(ETHR_AMC_FALLBACK__)
621 ETHR_MEMBAR(ETHR_StoreStore);
622 res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
623 #else
624 ETHR_MEMBAR(ETHR_StoreStore);
625 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
626 {
627 res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
628 if (res) {
629 (&var->fallback)->sint[0] = val->sint[0];
630 (&var->fallback)->sint[1] = val->sint[1];
631 }
632 else {
633 old_val->sint[0] = (&var->fallback)->sint[0];
634 old_val->sint[1] = (&var->fallback)->sint[1];
635 }
636 });
637 #endif
638 return res;
639 }
640
641 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val,ethr_dw_sint_t * old_val)642 int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
643 {
644 ETHR_ASSERT(!ethr_not_inited__);
645 ETHR_ASSERT(var);
646 return ethr_dw_atomic_cmpxchg_wb__(var, val, old_val);
647 }
648 #endif
649
ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)650 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
651 {
652 int res;
653 ETHR_ASSERT(!ethr_not_inited__);
654 ETHR_ASSERT(var);
655 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
656 res = ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val);
657 #elif defined(ETHR_AMC_FALLBACK__)
658 res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
659 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
660 #else
661 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
662 {
663 res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
664 if (res) {
665 (&var->fallback)->sint[0] = val->sint[0];
666 (&var->fallback)->sint[1] = val->sint[1];
667 }
668 else {
669 old_val->sint[0] = (&var->fallback)->sint[0];
670 old_val->sint[1] = (&var->fallback)->sint[1];
671 }
672 });
673 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
674 #endif
675 return res;
676 }
677
678 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val,ethr_dw_sint_t * old_val)679 int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
680 {
681 ETHR_ASSERT(!ethr_not_inited__);
682 ETHR_ASSERT(var);
683 return ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val);
684 }
685 #endif
686
ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)687 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
688 {
689 int res;
690 ETHR_ASSERT(!ethr_not_inited__);
691 ETHR_ASSERT(var);
692 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
693 res = ethr_dw_atomic_cmpxchg_relb__(var, val, old_val);
694 #elif defined(ETHR_AMC_FALLBACK__)
695 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
696 res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
697 #else
698 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
699 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
700 {
701 res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
702 if (res) {
703 (&var->fallback)->sint[0] = val->sint[0];
704 (&var->fallback)->sint[1] = val->sint[1];
705 }
706 else {
707 old_val->sint[0] = (&var->fallback)->sint[0];
708 old_val->sint[1] = (&var->fallback)->sint[1];
709 }
710 });
711 #endif
712 return res;
713 }
714
715 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val,ethr_dw_sint_t * old_val)716 int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
717 {
718 ETHR_ASSERT(!ethr_not_inited__);
719 ETHR_ASSERT(var);
720 return ethr_dw_atomic_cmpxchg_relb__(var, val, old_val);
721 }
722 #endif
723
ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)724 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
725 {
726 int res;
727 ETHR_ASSERT(!ethr_not_inited__);
728 ETHR_ASSERT(var);
729 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
730 res = ethr_dw_atomic_cmpxchg_mb__(var, val, old_val);
731 #elif defined(ETHR_AMC_FALLBACK__)
732 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
733 res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
734 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
735 #else
736 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
737 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
738 {
739 res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
740 if (res) {
741 (&var->fallback)->sint[0] = val->sint[0];
742 (&var->fallback)->sint[1] = val->sint[1];
743 }
744 else {
745 old_val->sint[0] = (&var->fallback)->sint[0];
746 old_val->sint[1] = (&var->fallback)->sint[1];
747 }
748 });
749 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
750 #endif
751 return res;
752 }
753
754 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val,ethr_dw_sint_t * old_val)755 int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
756 {
757 ETHR_ASSERT(!ethr_not_inited__);
758 ETHR_ASSERT(var);
759 return ethr_dw_atomic_cmpxchg_mb__(var, val, old_val);
760 }
761 #endif
762
763
764 /* -- set() -- */
765
766
ETHR_DW_ATOMIC_FUNC__(set)767 void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
768 {
769 ETHR_ASSERT(!ethr_not_inited__);
770 ETHR_ASSERT(var);
771 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
772 ethr_dw_atomic_set__(var, val);
773 #elif defined(ETHR_AMC_FALLBACK__)
774 amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
775 #else
776 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
777 #endif
778
779 }
780
781 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_set(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)782 void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
783 {
784 ETHR_ASSERT(!ethr_not_inited__);
785 ETHR_ASSERT(var);
786 ethr_dw_atomic_set__(var, val);
787 }
788 #endif
789
ethr_dw_atomic_set_ddrb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)790 void ethr_dw_atomic_set_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
791 {
792 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS)
793 ethr_dw_atomic_set_ddrb__(var, val);
794 #elif defined(ETHR_ORDERED_READ_DEPEND)
795 ethr_dw_atomic_set(var, val);
796 #else
797 ethr_dw_atomic_set_rb(var, val);
798 #endif
799 }
800
ETHR_DW_ATOMIC_FUNC__(set_rb)801 void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
802 {
803 ETHR_ASSERT(!ethr_not_inited__);
804 ETHR_ASSERT(var);
805 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
806 ethr_dw_atomic_set_rb__(var, val);
807 #elif defined(ETHR_AMC_FALLBACK__)
808 amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
809 ETHR_MEMBAR(ETHR_LoadLoad);
810 #else
811 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
812 ETHR_MEMBAR(ETHR_LoadLoad);
813 #endif
814
815 }
816
817 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_set_rb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)818 void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
819 {
820 ETHR_ASSERT(!ethr_not_inited__);
821 ETHR_ASSERT(var);
822 ethr_dw_atomic_set_rb__(var, val);
823 }
824 #endif
825
ETHR_DW_ATOMIC_FUNC__(set_wb)826 void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
827 {
828 ETHR_ASSERT(!ethr_not_inited__);
829 ETHR_ASSERT(var);
830 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
831 ethr_dw_atomic_set_wb__(var, val);
832 #elif defined(ETHR_AMC_FALLBACK__)
833 ETHR_MEMBAR(ETHR_StoreStore);
834 amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
835 #else
836 ETHR_MEMBAR(ETHR_StoreStore);
837 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
838 #endif
839
840 }
841
842 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_set_wb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)843 void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
844 {
845 ETHR_ASSERT(!ethr_not_inited__);
846 ETHR_ASSERT(var);
847 ethr_dw_atomic_set_wb__(var, val);
848 }
849 #endif
850
ETHR_DW_ATOMIC_FUNC__(set_acqb)851 void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
852 {
853 ETHR_ASSERT(!ethr_not_inited__);
854 ETHR_ASSERT(var);
855 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
856 ethr_dw_atomic_set_acqb__(var, val);
857 #elif defined(ETHR_AMC_FALLBACK__)
858 amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
859 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
860 #else
861 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
862 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
863 #endif
864
865 }
866
867 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_set_acqb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)868 void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
869 {
870 ETHR_ASSERT(!ethr_not_inited__);
871 ETHR_ASSERT(var);
872 ethr_dw_atomic_set_acqb__(var, val);
873 }
874 #endif
875
ETHR_DW_ATOMIC_FUNC__(set_relb)876 void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
877 {
878 ETHR_ASSERT(!ethr_not_inited__);
879 ETHR_ASSERT(var);
880 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
881 ethr_dw_atomic_set_relb__(var, val);
882 #elif defined(ETHR_AMC_FALLBACK__)
883 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
884 amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
885 #else
886 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
887 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
888 #endif
889
890 }
891
892 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_set_relb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)893 void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
894 {
895 ETHR_ASSERT(!ethr_not_inited__);
896 ETHR_ASSERT(var);
897 ethr_dw_atomic_set_relb__(var, val);
898 }
899 #endif
900
ETHR_DW_ATOMIC_FUNC__(set_mb)901 void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
902 {
903 ETHR_ASSERT(!ethr_not_inited__);
904 ETHR_ASSERT(var);
905 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
906 ethr_dw_atomic_set_mb__(var, val);
907 #elif defined(ETHR_AMC_FALLBACK__)
908 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
909 amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
910 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
911 #else
912 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
913 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
914 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
915 #endif
916
917 }
918
919 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_set_mb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)920 void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
921 {
922 ETHR_ASSERT(!ethr_not_inited__);
923 ETHR_ASSERT(var);
924 ethr_dw_atomic_set_mb__(var, val);
925 }
926 #endif
927
928
929 /* -- read() -- */
930
931
ETHR_DW_ATOMIC_FUNC__(read)932 void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
933 {
934 ETHR_ASSERT(!ethr_not_inited__);
935 ETHR_ASSERT(var);
936 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
937 ethr_dw_atomic_read__(var, val);
938 #elif defined(ETHR_AMC_FALLBACK__)
939 amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
940 #else
941 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
942 #endif
943
944 }
945
946 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_read(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)947 void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
948 {
949 ETHR_ASSERT(!ethr_not_inited__);
950 ETHR_ASSERT(var);
951 ethr_dw_atomic_read__(var, val);
952 }
953 #endif
954
ethr_dw_atomic_read_ddrb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)955 void ethr_dw_atomic_read_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
956 {
957 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS)
958 ethr_dw_atomic_read_ddrb__(var, val);
959 #elif defined(ETHR_ORDERED_READ_DEPEND)
960 ethr_dw_atomic_read(var, val);
961 #else
962 ethr_dw_atomic_read_rb(var, val);
963 #endif
964 }
965
ETHR_DW_ATOMIC_FUNC__(read_rb)966 void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
967 {
968 ETHR_ASSERT(!ethr_not_inited__);
969 ETHR_ASSERT(var);
970 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
971 ethr_dw_atomic_read_rb__(var, val);
972 #elif defined(ETHR_AMC_FALLBACK__)
973 amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
974 #else
975 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
976 ETHR_MEMBAR(ETHR_LoadLoad);
977 #endif
978
979 }
980
981 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_read_rb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)982 void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
983 {
984 ETHR_ASSERT(!ethr_not_inited__);
985 ETHR_ASSERT(var);
986 ethr_dw_atomic_read_rb__(var, val);
987 }
988 #endif
989
ETHR_DW_ATOMIC_FUNC__(read_wb)990 void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
991 {
992 ETHR_ASSERT(!ethr_not_inited__);
993 ETHR_ASSERT(var);
994 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
995 ethr_dw_atomic_read_wb__(var, val);
996 #elif defined(ETHR_AMC_FALLBACK__)
997 ETHR_MEMBAR(ETHR_StoreStore);
998 amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
999 #else
1000 ETHR_MEMBAR(ETHR_StoreStore);
1001 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
1002 #endif
1003
1004 }
1005
1006 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_read_wb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1007 void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1008 {
1009 ETHR_ASSERT(!ethr_not_inited__);
1010 ETHR_ASSERT(var);
1011 ethr_dw_atomic_read_wb__(var, val);
1012 }
1013 #endif
1014
ETHR_DW_ATOMIC_FUNC__(read_acqb)1015 void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1016 {
1017 ETHR_ASSERT(!ethr_not_inited__);
1018 ETHR_ASSERT(var);
1019 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1020 ethr_dw_atomic_read_acqb__(var, val);
1021 #elif defined(ETHR_AMC_FALLBACK__)
1022 amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1023 ETHR_MEMBAR(ETHR_LoadStore);
1024 #else
1025 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
1026 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
1027 #endif
1028
1029 }
1030
1031 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_read_acqb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1032 void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1033 {
1034 ETHR_ASSERT(!ethr_not_inited__);
1035 ETHR_ASSERT(var);
1036 ethr_dw_atomic_read_acqb__(var, val);
1037 }
1038 #endif
1039
ETHR_DW_ATOMIC_FUNC__(read_relb)1040 void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1041 {
1042 ETHR_ASSERT(!ethr_not_inited__);
1043 ETHR_ASSERT(var);
1044 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1045 ethr_dw_atomic_read_relb__(var, val);
1046 #elif defined(ETHR_AMC_FALLBACK__)
1047 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1048 amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1049 #else
1050 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1051 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
1052 #endif
1053
1054 }
1055
1056 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_read_relb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1057 void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1058 {
1059 ETHR_ASSERT(!ethr_not_inited__);
1060 ETHR_ASSERT(var);
1061 ethr_dw_atomic_read_relb__(var, val);
1062 }
1063 #endif
1064
ETHR_DW_ATOMIC_FUNC__(read_mb)1065 void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1066 {
1067 ETHR_ASSERT(!ethr_not_inited__);
1068 ETHR_ASSERT(var);
1069 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1070 ethr_dw_atomic_read_mb__(var, val);
1071 #elif defined(ETHR_AMC_FALLBACK__)
1072 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1073 amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1074 ETHR_MEMBAR(ETHR_LoadStore);
1075 #else
1076 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1077 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
1078 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
1079 #endif
1080
1081 }
1082
1083 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_read_mb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1084 void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1085 {
1086 ETHR_ASSERT(!ethr_not_inited__);
1087 ETHR_ASSERT(var);
1088 ethr_dw_atomic_read_mb__(var, val);
1089 }
1090 #endif
1091
1092
1093 /* -- init() -- */
1094
1095
ETHR_DW_ATOMIC_FUNC__(init)1096 void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1097 {
1098 ETHR_ASSERT(var);
1099 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1100 ethr_dw_atomic_init__(var, val);
1101 #elif defined(ETHR_AMC_FALLBACK__)
1102 amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1103 #else
1104 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
1105 #endif
1106
1107 }
1108
1109 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_init(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1110 void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1111 {
1112 ETHR_ASSERT(var);
1113 ethr_dw_atomic_init__(var, val);
1114 }
1115 #endif
1116
ethr_dw_atomic_init_ddrb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1117 void ethr_dw_atomic_init_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1118 {
1119 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS)
1120 ethr_dw_atomic_init_ddrb__(var, val);
1121 #elif defined(ETHR_ORDERED_READ_DEPEND)
1122 ethr_dw_atomic_init(var, val);
1123 #else
1124 ethr_dw_atomic_init_rb(var, val);
1125 #endif
1126 }
1127
ETHR_DW_ATOMIC_FUNC__(init_rb)1128 void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1129 {
1130 ETHR_ASSERT(var);
1131 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1132 ethr_dw_atomic_init_rb__(var, val);
1133 #elif defined(ETHR_AMC_FALLBACK__)
1134 amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1135 ETHR_MEMBAR(ETHR_LoadLoad);
1136 #else
1137 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
1138 ETHR_MEMBAR(ETHR_LoadLoad);
1139 #endif
1140
1141 }
1142
1143 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_init_rb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1144 void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1145 {
1146 ETHR_ASSERT(var);
1147 ethr_dw_atomic_init_rb__(var, val);
1148 }
1149 #endif
1150
ETHR_DW_ATOMIC_FUNC__(init_wb)1151 void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1152 {
1153 ETHR_ASSERT(var);
1154 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1155 ethr_dw_atomic_init_wb__(var, val);
1156 #elif defined(ETHR_AMC_FALLBACK__)
1157 ETHR_MEMBAR(ETHR_StoreStore);
1158 amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1159 #else
1160 ETHR_MEMBAR(ETHR_StoreStore);
1161 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
1162 #endif
1163
1164 }
1165
1166 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_init_wb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1167 void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1168 {
1169 ETHR_ASSERT(var);
1170 ethr_dw_atomic_init_wb__(var, val);
1171 }
1172 #endif
1173
ETHR_DW_ATOMIC_FUNC__(init_acqb)1174 void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1175 {
1176 ETHR_ASSERT(var);
1177 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1178 ethr_dw_atomic_init_acqb__(var, val);
1179 #elif defined(ETHR_AMC_FALLBACK__)
1180 amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1181 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1182 #else
1183 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
1184 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1185 #endif
1186
1187 }
1188
1189 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_init_acqb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1190 void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1191 {
1192 ETHR_ASSERT(var);
1193 ethr_dw_atomic_init_acqb__(var, val);
1194 }
1195 #endif
1196
ETHR_DW_ATOMIC_FUNC__(init_relb)1197 void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1198 {
1199 ETHR_ASSERT(var);
1200 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1201 ethr_dw_atomic_init_relb__(var, val);
1202 #elif defined(ETHR_AMC_FALLBACK__)
1203 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1204 amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1205 #else
1206 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1207 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
1208 #endif
1209
1210 }
1211
1212 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_init_relb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1213 void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1214 {
1215 ETHR_ASSERT(var);
1216 ethr_dw_atomic_init_relb__(var, val);
1217 }
1218 #endif
1219
ETHR_DW_ATOMIC_FUNC__(init_mb)1220 void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1221 {
1222 ETHR_ASSERT(var);
1223 #if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1224 ethr_dw_atomic_init_mb__(var, val);
1225 #elif defined(ETHR_AMC_FALLBACK__)
1226 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1227 amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
1228 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1229 #else
1230 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1231 ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
1232 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1233 #endif
1234
1235 }
1236
1237 #ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_dw_atomic_init_mb(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1238 void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1239 {
1240 ETHR_ASSERT(var);
1241 ethr_dw_atomic_init_mb__(var, val);
1242 }
1243 #endif
1244
1245
1246 /* ---------- Word size atomic implementation ---------- */
1247
1248
1249
1250
1251 /* --- addr() --- */
1252
ethr_atomic_addr(ethr_atomic_t * var)1253 ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var)
1254 {
1255 ethr_sint_t *res;
1256 ETHR_ASSERT(!ethr_not_inited__);
1257 ETHR_ASSERT(var);
1258 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1259 res = ethr_atomic_addr__(var);
1260 #elif defined(ETHR_AMC_FALLBACK__)
1261 res = (ethr_sint_t *) (var)->sint;
1262 #else
1263 res = (ethr_sint_t *) var;
1264 #endif
1265 return res;
1266 }
1267
1268
1269 /* -- cmpxchg() -- */
1270
1271
ethr_atomic_cmpxchg(ethr_atomic_t * var,ethr_sint_t val,ethr_sint_t old_val)1272 ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
1273 {
1274 ethr_sint_t res;
1275 ETHR_ASSERT(!ethr_not_inited__);
1276 ETHR_ASSERT(var);
1277 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1278 res = ethr_atomic_cmpxchg__(var, val, old_val);
1279 #elif defined(ETHR_AMC_FALLBACK__)
1280 res = old_val;
1281 (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
1282 #else
1283 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
1284 #endif
1285 return res;
1286 }
1287
ethr_atomic_cmpxchg_ddrb(ethr_atomic_t * var,ethr_sint_t val,ethr_sint_t old_val)1288 ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
1289 {
1290 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1291 return ethr_atomic_cmpxchg_ddrb__(var, val, old_val);
1292 #elif defined(ETHR_ORDERED_READ_DEPEND)
1293 return ethr_atomic_cmpxchg(var, val, old_val);
1294 #else
1295 return ethr_atomic_cmpxchg_rb(var, val, old_val);
1296 #endif
1297 }
1298
ethr_atomic_cmpxchg_rb(ethr_atomic_t * var,ethr_sint_t val,ethr_sint_t old_val)1299 ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
1300 {
1301 ethr_sint_t res;
1302 ETHR_ASSERT(!ethr_not_inited__);
1303 ETHR_ASSERT(var);
1304 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1305 res = ethr_atomic_cmpxchg_rb__(var, val, old_val);
1306 #elif defined(ETHR_AMC_FALLBACK__)
1307 res = old_val;
1308 (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
1309 ETHR_MEMBAR(ETHR_LoadLoad);
1310 #else
1311 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
1312 ETHR_MEMBAR(ETHR_LoadLoad);
1313 #endif
1314 return res;
1315 }
1316
ethr_atomic_cmpxchg_wb(ethr_atomic_t * var,ethr_sint_t val,ethr_sint_t old_val)1317 ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
1318 {
1319 ethr_sint_t res;
1320 ETHR_ASSERT(!ethr_not_inited__);
1321 ETHR_ASSERT(var);
1322 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1323 res = ethr_atomic_cmpxchg_wb__(var, val, old_val);
1324 #elif defined(ETHR_AMC_FALLBACK__)
1325 ETHR_MEMBAR(ETHR_StoreStore);
1326 res = old_val;
1327 (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
1328 #else
1329 ETHR_MEMBAR(ETHR_StoreStore);
1330 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
1331 #endif
1332 return res;
1333 }
1334
ethr_atomic_cmpxchg_acqb(ethr_atomic_t * var,ethr_sint_t val,ethr_sint_t old_val)1335 ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
1336 {
1337 ethr_sint_t res;
1338 ETHR_ASSERT(!ethr_not_inited__);
1339 ETHR_ASSERT(var);
1340 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1341 res = ethr_atomic_cmpxchg_acqb__(var, val, old_val);
1342 #elif defined(ETHR_AMC_FALLBACK__)
1343 res = old_val;
1344 (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
1345 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1346 #else
1347 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
1348 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1349 #endif
1350 return res;
1351 }
1352
ethr_atomic_cmpxchg_relb(ethr_atomic_t * var,ethr_sint_t val,ethr_sint_t old_val)1353 ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
1354 {
1355 ethr_sint_t res;
1356 ETHR_ASSERT(!ethr_not_inited__);
1357 ETHR_ASSERT(var);
1358 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1359 res = ethr_atomic_cmpxchg_relb__(var, val, old_val);
1360 #elif defined(ETHR_AMC_FALLBACK__)
1361 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1362 res = old_val;
1363 (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
1364 #else
1365 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1366 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
1367 #endif
1368 return res;
1369 }
1370
ethr_atomic_cmpxchg_mb(ethr_atomic_t * var,ethr_sint_t val,ethr_sint_t old_val)1371 ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
1372 {
1373 ethr_sint_t res;
1374 ETHR_ASSERT(!ethr_not_inited__);
1375 ETHR_ASSERT(var);
1376 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1377 res = ethr_atomic_cmpxchg_mb__(var, val, old_val);
1378 #elif defined(ETHR_AMC_FALLBACK__)
1379 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1380 res = old_val;
1381 (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
1382 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1383 #else
1384 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1385 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
1386 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1387 #endif
1388 return res;
1389 }
1390
1391
1392 /* -- xchg() -- */
1393
1394
ethr_atomic_xchg(ethr_atomic_t * var,ethr_sint_t val)1395 ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val)
1396 {
1397 ethr_sint_t res;
1398 ETHR_ASSERT(!ethr_not_inited__);
1399 ETHR_ASSERT(var);
1400 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1401 res = ethr_atomic_xchg__(var, val);
1402 #elif defined(ETHR_AMC_FALLBACK__)
1403 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
1404 #else
1405 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
1406 #endif
1407 return res;
1408 }
1409
ethr_atomic_xchg_ddrb(ethr_atomic_t * var,ethr_sint_t val)1410 ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val)
1411 {
1412 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1413 return ethr_atomic_xchg_ddrb__(var, val);
1414 #elif defined(ETHR_ORDERED_READ_DEPEND)
1415 return ethr_atomic_xchg(var, val);
1416 #else
1417 return ethr_atomic_xchg_rb(var, val);
1418 #endif
1419 }
1420
ethr_atomic_xchg_rb(ethr_atomic_t * var,ethr_sint_t val)1421 ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val)
1422 {
1423 ethr_sint_t res;
1424 ETHR_ASSERT(!ethr_not_inited__);
1425 ETHR_ASSERT(var);
1426 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1427 res = ethr_atomic_xchg_rb__(var, val);
1428 #elif defined(ETHR_AMC_FALLBACK__)
1429 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
1430 ETHR_MEMBAR(ETHR_LoadLoad);
1431 #else
1432 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
1433 ETHR_MEMBAR(ETHR_LoadLoad);
1434 #endif
1435 return res;
1436 }
1437
ethr_atomic_xchg_wb(ethr_atomic_t * var,ethr_sint_t val)1438 ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val)
1439 {
1440 ethr_sint_t res;
1441 ETHR_ASSERT(!ethr_not_inited__);
1442 ETHR_ASSERT(var);
1443 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1444 res = ethr_atomic_xchg_wb__(var, val);
1445 #elif defined(ETHR_AMC_FALLBACK__)
1446 ETHR_MEMBAR(ETHR_StoreStore);
1447 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
1448 #else
1449 ETHR_MEMBAR(ETHR_StoreStore);
1450 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
1451 #endif
1452 return res;
1453 }
1454
ethr_atomic_xchg_acqb(ethr_atomic_t * var,ethr_sint_t val)1455 ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val)
1456 {
1457 ethr_sint_t res;
1458 ETHR_ASSERT(!ethr_not_inited__);
1459 ETHR_ASSERT(var);
1460 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1461 res = ethr_atomic_xchg_acqb__(var, val);
1462 #elif defined(ETHR_AMC_FALLBACK__)
1463 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
1464 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1465 #else
1466 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
1467 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1468 #endif
1469 return res;
1470 }
1471
ethr_atomic_xchg_relb(ethr_atomic_t * var,ethr_sint_t val)1472 ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val)
1473 {
1474 ethr_sint_t res;
1475 ETHR_ASSERT(!ethr_not_inited__);
1476 ETHR_ASSERT(var);
1477 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1478 res = ethr_atomic_xchg_relb__(var, val);
1479 #elif defined(ETHR_AMC_FALLBACK__)
1480 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1481 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
1482 #else
1483 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1484 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
1485 #endif
1486 return res;
1487 }
1488
ethr_atomic_xchg_mb(ethr_atomic_t * var,ethr_sint_t val)1489 ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val)
1490 {
1491 ethr_sint_t res;
1492 ETHR_ASSERT(!ethr_not_inited__);
1493 ETHR_ASSERT(var);
1494 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1495 res = ethr_atomic_xchg_mb__(var, val);
1496 #elif defined(ETHR_AMC_FALLBACK__)
1497 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1498 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
1499 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1500 #else
1501 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1502 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
1503 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1504 #endif
1505 return res;
1506 }
1507
1508
1509 /* -- set() -- */
1510
1511
ethr_atomic_set(ethr_atomic_t * var,ethr_sint_t val)1512 void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val)
1513 {
1514 ETHR_ASSERT(!ethr_not_inited__);
1515 ETHR_ASSERT(var);
1516 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1517 ethr_atomic_set__(var, val);
1518 #elif defined(ETHR_AMC_FALLBACK__)
1519 amc_set(&var->amc, 0, &var->sint, &val);
1520 #else
1521 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1522 #endif
1523
1524 }
1525
ethr_atomic_set_ddrb(ethr_atomic_t * var,ethr_sint_t val)1526 void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val)
1527 {
1528 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1529 ethr_atomic_set_ddrb__(var, val);
1530 #elif defined(ETHR_ORDERED_READ_DEPEND)
1531 ethr_atomic_set(var, val);
1532 #else
1533 ethr_atomic_set_rb(var, val);
1534 #endif
1535 }
1536
ethr_atomic_set_rb(ethr_atomic_t * var,ethr_sint_t val)1537 void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val)
1538 {
1539 ETHR_ASSERT(!ethr_not_inited__);
1540 ETHR_ASSERT(var);
1541 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1542 ethr_atomic_set_rb__(var, val);
1543 #elif defined(ETHR_AMC_FALLBACK__)
1544 amc_set(&var->amc, 0, &var->sint, &val);
1545 ETHR_MEMBAR(ETHR_LoadLoad);
1546 #else
1547 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1548 ETHR_MEMBAR(ETHR_LoadLoad);
1549 #endif
1550
1551 }
1552
ethr_atomic_set_wb(ethr_atomic_t * var,ethr_sint_t val)1553 void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val)
1554 {
1555 ETHR_ASSERT(!ethr_not_inited__);
1556 ETHR_ASSERT(var);
1557 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1558 ethr_atomic_set_wb__(var, val);
1559 #elif defined(ETHR_AMC_FALLBACK__)
1560 ETHR_MEMBAR(ETHR_StoreStore);
1561 amc_set(&var->amc, 0, &var->sint, &val);
1562 #else
1563 ETHR_MEMBAR(ETHR_StoreStore);
1564 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1565 #endif
1566
1567 }
1568
ethr_atomic_set_acqb(ethr_atomic_t * var,ethr_sint_t val)1569 void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val)
1570 {
1571 ETHR_ASSERT(!ethr_not_inited__);
1572 ETHR_ASSERT(var);
1573 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1574 ethr_atomic_set_acqb__(var, val);
1575 #elif defined(ETHR_AMC_FALLBACK__)
1576 amc_set(&var->amc, 0, &var->sint, &val);
1577 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1578 #else
1579 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1580 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1581 #endif
1582
1583 }
1584
ethr_atomic_set_relb(ethr_atomic_t * var,ethr_sint_t val)1585 void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val)
1586 {
1587 ETHR_ASSERT(!ethr_not_inited__);
1588 ETHR_ASSERT(var);
1589 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1590 ethr_atomic_set_relb__(var, val);
1591 #elif defined(ETHR_AMC_FALLBACK__)
1592 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1593 amc_set(&var->amc, 0, &var->sint, &val);
1594 #else
1595 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1596 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1597 #endif
1598
1599 }
1600
ethr_atomic_set_mb(ethr_atomic_t * var,ethr_sint_t val)1601 void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val)
1602 {
1603 ETHR_ASSERT(!ethr_not_inited__);
1604 ETHR_ASSERT(var);
1605 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1606 ethr_atomic_set_mb__(var, val);
1607 #elif defined(ETHR_AMC_FALLBACK__)
1608 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1609 amc_set(&var->amc, 0, &var->sint, &val);
1610 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1611 #else
1612 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1613 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1614 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1615 #endif
1616
1617 }
1618
1619
1620 /* -- init() -- */
1621
1622
ethr_atomic_init(ethr_atomic_t * var,ethr_sint_t val)1623 void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val)
1624 {
1625 ETHR_ASSERT(var);
1626 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1627 ethr_atomic_init__(var, val);
1628 #elif defined(ETHR_AMC_FALLBACK__)
1629 amc_init(&var->amc, 0, &var->sint, &val);
1630 #else
1631 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1632 #endif
1633
1634 }
1635
ethr_atomic_init_ddrb(ethr_atomic_t * var,ethr_sint_t val)1636 void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val)
1637 {
1638 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1639 ethr_atomic_init_ddrb__(var, val);
1640 #elif defined(ETHR_ORDERED_READ_DEPEND)
1641 ethr_atomic_init(var, val);
1642 #else
1643 ethr_atomic_init_rb(var, val);
1644 #endif
1645 }
1646
ethr_atomic_init_rb(ethr_atomic_t * var,ethr_sint_t val)1647 void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val)
1648 {
1649 ETHR_ASSERT(var);
1650 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1651 ethr_atomic_init_rb__(var, val);
1652 #elif defined(ETHR_AMC_FALLBACK__)
1653 amc_init(&var->amc, 0, &var->sint, &val);
1654 ETHR_MEMBAR(ETHR_LoadLoad);
1655 #else
1656 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1657 ETHR_MEMBAR(ETHR_LoadLoad);
1658 #endif
1659
1660 }
1661
ethr_atomic_init_wb(ethr_atomic_t * var,ethr_sint_t val)1662 void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val)
1663 {
1664 ETHR_ASSERT(var);
1665 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1666 ethr_atomic_init_wb__(var, val);
1667 #elif defined(ETHR_AMC_FALLBACK__)
1668 ETHR_MEMBAR(ETHR_StoreStore);
1669 amc_init(&var->amc, 0, &var->sint, &val);
1670 #else
1671 ETHR_MEMBAR(ETHR_StoreStore);
1672 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1673 #endif
1674
1675 }
1676
ethr_atomic_init_acqb(ethr_atomic_t * var,ethr_sint_t val)1677 void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val)
1678 {
1679 ETHR_ASSERT(var);
1680 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1681 ethr_atomic_init_acqb__(var, val);
1682 #elif defined(ETHR_AMC_FALLBACK__)
1683 amc_init(&var->amc, 0, &var->sint, &val);
1684 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1685 #else
1686 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1687 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1688 #endif
1689
1690 }
1691
ethr_atomic_init_relb(ethr_atomic_t * var,ethr_sint_t val)1692 void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val)
1693 {
1694 ETHR_ASSERT(var);
1695 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1696 ethr_atomic_init_relb__(var, val);
1697 #elif defined(ETHR_AMC_FALLBACK__)
1698 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1699 amc_init(&var->amc, 0, &var->sint, &val);
1700 #else
1701 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1702 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1703 #endif
1704
1705 }
1706
ethr_atomic_init_mb(ethr_atomic_t * var,ethr_sint_t val)1707 void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val)
1708 {
1709 ETHR_ASSERT(var);
1710 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1711 ethr_atomic_init_mb__(var, val);
1712 #elif defined(ETHR_AMC_FALLBACK__)
1713 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1714 amc_init(&var->amc, 0, &var->sint, &val);
1715 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1716 #else
1717 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1718 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
1719 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1720 #endif
1721
1722 }
1723
1724
1725 /* -- add_read() -- */
1726
1727
ethr_atomic_add_read(ethr_atomic_t * var,ethr_sint_t val)1728 ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val)
1729 {
1730 ethr_sint_t res;
1731 ETHR_ASSERT(!ethr_not_inited__);
1732 ETHR_ASSERT(var);
1733 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1734 res = ethr_atomic_add_read__(var, val);
1735 #elif defined(ETHR_AMC_FALLBACK__)
1736 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
1737 #else
1738 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
1739 #endif
1740 return res;
1741 }
1742
ethr_atomic_add_read_ddrb(ethr_atomic_t * var,ethr_sint_t val)1743 ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val)
1744 {
1745 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1746 return ethr_atomic_add_read_ddrb__(var, val);
1747 #elif defined(ETHR_ORDERED_READ_DEPEND)
1748 return ethr_atomic_add_read(var, val);
1749 #else
1750 return ethr_atomic_add_read_rb(var, val);
1751 #endif
1752 }
1753
ethr_atomic_add_read_rb(ethr_atomic_t * var,ethr_sint_t val)1754 ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val)
1755 {
1756 ethr_sint_t res;
1757 ETHR_ASSERT(!ethr_not_inited__);
1758 ETHR_ASSERT(var);
1759 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1760 res = ethr_atomic_add_read_rb__(var, val);
1761 #elif defined(ETHR_AMC_FALLBACK__)
1762 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
1763 ETHR_MEMBAR(ETHR_LoadLoad);
1764 #else
1765 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
1766 ETHR_MEMBAR(ETHR_LoadLoad);
1767 #endif
1768 return res;
1769 }
1770
ethr_atomic_add_read_wb(ethr_atomic_t * var,ethr_sint_t val)1771 ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val)
1772 {
1773 ethr_sint_t res;
1774 ETHR_ASSERT(!ethr_not_inited__);
1775 ETHR_ASSERT(var);
1776 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1777 res = ethr_atomic_add_read_wb__(var, val);
1778 #elif defined(ETHR_AMC_FALLBACK__)
1779 ETHR_MEMBAR(ETHR_StoreStore);
1780 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
1781 #else
1782 ETHR_MEMBAR(ETHR_StoreStore);
1783 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
1784 #endif
1785 return res;
1786 }
1787
ethr_atomic_add_read_acqb(ethr_atomic_t * var,ethr_sint_t val)1788 ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val)
1789 {
1790 ethr_sint_t res;
1791 ETHR_ASSERT(!ethr_not_inited__);
1792 ETHR_ASSERT(var);
1793 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1794 res = ethr_atomic_add_read_acqb__(var, val);
1795 #elif defined(ETHR_AMC_FALLBACK__)
1796 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
1797 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1798 #else
1799 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
1800 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1801 #endif
1802 return res;
1803 }
1804
ethr_atomic_add_read_relb(ethr_atomic_t * var,ethr_sint_t val)1805 ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val)
1806 {
1807 ethr_sint_t res;
1808 ETHR_ASSERT(!ethr_not_inited__);
1809 ETHR_ASSERT(var);
1810 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1811 res = ethr_atomic_add_read_relb__(var, val);
1812 #elif defined(ETHR_AMC_FALLBACK__)
1813 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1814 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
1815 #else
1816 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1817 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
1818 #endif
1819 return res;
1820 }
1821
ethr_atomic_add_read_mb(ethr_atomic_t * var,ethr_sint_t val)1822 ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val)
1823 {
1824 ethr_sint_t res;
1825 ETHR_ASSERT(!ethr_not_inited__);
1826 ETHR_ASSERT(var);
1827 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1828 res = ethr_atomic_add_read_mb__(var, val);
1829 #elif defined(ETHR_AMC_FALLBACK__)
1830 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1831 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
1832 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1833 #else
1834 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1835 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
1836 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1837 #endif
1838 return res;
1839 }
1840
1841
1842 /* -- read() -- */
1843
1844
ethr_atomic_read(ethr_atomic_t * var)1845 ethr_sint_t ethr_atomic_read(ethr_atomic_t *var)
1846 {
1847 ethr_sint_t res;
1848 ETHR_ASSERT(!ethr_not_inited__);
1849 ETHR_ASSERT(var);
1850 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1851 res = ethr_atomic_read__(var);
1852 #elif defined(ETHR_AMC_FALLBACK__)
1853 amc_read(&var->amc, 0, &var->sint, &res);
1854 #else
1855 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
1856 #endif
1857 return res;
1858 }
1859
ethr_atomic_read_ddrb(ethr_atomic_t * var)1860 ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var)
1861 {
1862 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1863 return ethr_atomic_read_ddrb__(var);
1864 #elif defined(ETHR_ORDERED_READ_DEPEND)
1865 return ethr_atomic_read(var);
1866 #else
1867 return ethr_atomic_read_rb(var);
1868 #endif
1869 }
1870
ethr_atomic_read_rb(ethr_atomic_t * var)1871 ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var)
1872 {
1873 ethr_sint_t res;
1874 ETHR_ASSERT(!ethr_not_inited__);
1875 ETHR_ASSERT(var);
1876 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1877 res = ethr_atomic_read_rb__(var);
1878 #elif defined(ETHR_AMC_FALLBACK__)
1879 amc_read(&var->amc, 0, &var->sint, &res);
1880 #else
1881 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
1882 ETHR_MEMBAR(ETHR_LoadLoad);
1883 #endif
1884 return res;
1885 }
1886
ethr_atomic_read_wb(ethr_atomic_t * var)1887 ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var)
1888 {
1889 ethr_sint_t res;
1890 ETHR_ASSERT(!ethr_not_inited__);
1891 ETHR_ASSERT(var);
1892 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1893 res = ethr_atomic_read_wb__(var);
1894 #elif defined(ETHR_AMC_FALLBACK__)
1895 ETHR_MEMBAR(ETHR_StoreStore);
1896 amc_read(&var->amc, 0, &var->sint, &res);
1897 #else
1898 ETHR_MEMBAR(ETHR_StoreStore);
1899 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
1900 #endif
1901 return res;
1902 }
1903
ethr_atomic_read_acqb(ethr_atomic_t * var)1904 ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var)
1905 {
1906 ethr_sint_t res;
1907 ETHR_ASSERT(!ethr_not_inited__);
1908 ETHR_ASSERT(var);
1909 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1910 res = ethr_atomic_read_acqb__(var);
1911 #elif defined(ETHR_AMC_FALLBACK__)
1912 amc_read(&var->amc, 0, &var->sint, &res);
1913 ETHR_MEMBAR(ETHR_LoadStore);
1914 #else
1915 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
1916 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
1917 #endif
1918 return res;
1919 }
1920
ethr_atomic_read_relb(ethr_atomic_t * var)1921 ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var)
1922 {
1923 ethr_sint_t res;
1924 ETHR_ASSERT(!ethr_not_inited__);
1925 ETHR_ASSERT(var);
1926 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1927 res = ethr_atomic_read_relb__(var);
1928 #elif defined(ETHR_AMC_FALLBACK__)
1929 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1930 amc_read(&var->amc, 0, &var->sint, &res);
1931 #else
1932 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1933 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
1934 #endif
1935 return res;
1936 }
1937
ethr_atomic_read_mb(ethr_atomic_t * var)1938 ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var)
1939 {
1940 ethr_sint_t res;
1941 ETHR_ASSERT(!ethr_not_inited__);
1942 ETHR_ASSERT(var);
1943 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1944 res = ethr_atomic_read_mb__(var);
1945 #elif defined(ETHR_AMC_FALLBACK__)
1946 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1947 amc_read(&var->amc, 0, &var->sint, &res);
1948 ETHR_MEMBAR(ETHR_LoadStore);
1949 #else
1950 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1951 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
1952 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
1953 #endif
1954 return res;
1955 }
1956
1957
1958 /* -- inc_read() -- */
1959
1960
ethr_atomic_inc_read(ethr_atomic_t * var)1961 ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var)
1962 {
1963 ethr_sint_t res;
1964 ETHR_ASSERT(!ethr_not_inited__);
1965 ETHR_ASSERT(var);
1966 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1967 res = ethr_atomic_inc_read__(var);
1968 #elif defined(ETHR_AMC_FALLBACK__)
1969 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
1970 #else
1971 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
1972 #endif
1973 return res;
1974 }
1975
ethr_atomic_inc_read_ddrb(ethr_atomic_t * var)1976 ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var)
1977 {
1978 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1979 return ethr_atomic_inc_read_ddrb__(var);
1980 #elif defined(ETHR_ORDERED_READ_DEPEND)
1981 return ethr_atomic_inc_read(var);
1982 #else
1983 return ethr_atomic_inc_read_rb(var);
1984 #endif
1985 }
1986
ethr_atomic_inc_read_rb(ethr_atomic_t * var)1987 ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var)
1988 {
1989 ethr_sint_t res;
1990 ETHR_ASSERT(!ethr_not_inited__);
1991 ETHR_ASSERT(var);
1992 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
1993 res = ethr_atomic_inc_read_rb__(var);
1994 #elif defined(ETHR_AMC_FALLBACK__)
1995 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
1996 ETHR_MEMBAR(ETHR_LoadLoad);
1997 #else
1998 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
1999 ETHR_MEMBAR(ETHR_LoadLoad);
2000 #endif
2001 return res;
2002 }
2003
ethr_atomic_inc_read_wb(ethr_atomic_t * var)2004 ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var)
2005 {
2006 ethr_sint_t res;
2007 ETHR_ASSERT(!ethr_not_inited__);
2008 ETHR_ASSERT(var);
2009 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2010 res = ethr_atomic_inc_read_wb__(var);
2011 #elif defined(ETHR_AMC_FALLBACK__)
2012 ETHR_MEMBAR(ETHR_StoreStore);
2013 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
2014 #else
2015 ETHR_MEMBAR(ETHR_StoreStore);
2016 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
2017 #endif
2018 return res;
2019 }
2020
ethr_atomic_inc_read_acqb(ethr_atomic_t * var)2021 ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
2022 {
2023 ethr_sint_t res;
2024 ETHR_ASSERT(!ethr_not_inited__);
2025 ETHR_ASSERT(var);
2026 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2027 res = ethr_atomic_inc_read_acqb__(var);
2028 #elif defined(ETHR_AMC_FALLBACK__)
2029 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
2030 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2031 #else
2032 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
2033 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2034 #endif
2035 return res;
2036 }
2037
ethr_atomic_inc_read_relb(ethr_atomic_t * var)2038 ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var)
2039 {
2040 ethr_sint_t res;
2041 ETHR_ASSERT(!ethr_not_inited__);
2042 ETHR_ASSERT(var);
2043 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2044 res = ethr_atomic_inc_read_relb__(var);
2045 #elif defined(ETHR_AMC_FALLBACK__)
2046 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2047 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
2048 #else
2049 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2050 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
2051 #endif
2052 return res;
2053 }
2054
ethr_atomic_inc_read_mb(ethr_atomic_t * var)2055 ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var)
2056 {
2057 ethr_sint_t res;
2058 ETHR_ASSERT(!ethr_not_inited__);
2059 ETHR_ASSERT(var);
2060 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2061 res = ethr_atomic_inc_read_mb__(var);
2062 #elif defined(ETHR_AMC_FALLBACK__)
2063 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2064 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
2065 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2066 #else
2067 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2068 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
2069 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2070 #endif
2071 return res;
2072 }
2073
2074
2075 /* -- dec_read() -- */
2076
2077
ethr_atomic_dec_read(ethr_atomic_t * var)2078 ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var)
2079 {
2080 ethr_sint_t res;
2081 ETHR_ASSERT(!ethr_not_inited__);
2082 ETHR_ASSERT(var);
2083 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2084 res = ethr_atomic_dec_read__(var);
2085 #elif defined(ETHR_AMC_FALLBACK__)
2086 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
2087 #else
2088 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
2089 #endif
2090 return res;
2091 }
2092
ethr_atomic_dec_read_ddrb(ethr_atomic_t * var)2093 ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var)
2094 {
2095 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2096 return ethr_atomic_dec_read_ddrb__(var);
2097 #elif defined(ETHR_ORDERED_READ_DEPEND)
2098 return ethr_atomic_dec_read(var);
2099 #else
2100 return ethr_atomic_dec_read_rb(var);
2101 #endif
2102 }
2103
ethr_atomic_dec_read_rb(ethr_atomic_t * var)2104 ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var)
2105 {
2106 ethr_sint_t res;
2107 ETHR_ASSERT(!ethr_not_inited__);
2108 ETHR_ASSERT(var);
2109 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2110 res = ethr_atomic_dec_read_rb__(var);
2111 #elif defined(ETHR_AMC_FALLBACK__)
2112 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
2113 ETHR_MEMBAR(ETHR_LoadLoad);
2114 #else
2115 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
2116 ETHR_MEMBAR(ETHR_LoadLoad);
2117 #endif
2118 return res;
2119 }
2120
ethr_atomic_dec_read_wb(ethr_atomic_t * var)2121 ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var)
2122 {
2123 ethr_sint_t res;
2124 ETHR_ASSERT(!ethr_not_inited__);
2125 ETHR_ASSERT(var);
2126 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2127 res = ethr_atomic_dec_read_wb__(var);
2128 #elif defined(ETHR_AMC_FALLBACK__)
2129 ETHR_MEMBAR(ETHR_StoreStore);
2130 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
2131 #else
2132 ETHR_MEMBAR(ETHR_StoreStore);
2133 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
2134 #endif
2135 return res;
2136 }
2137
ethr_atomic_dec_read_acqb(ethr_atomic_t * var)2138 ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var)
2139 {
2140 ethr_sint_t res;
2141 ETHR_ASSERT(!ethr_not_inited__);
2142 ETHR_ASSERT(var);
2143 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2144 res = ethr_atomic_dec_read_acqb__(var);
2145 #elif defined(ETHR_AMC_FALLBACK__)
2146 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
2147 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2148 #else
2149 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
2150 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2151 #endif
2152 return res;
2153 }
2154
ethr_atomic_dec_read_relb(ethr_atomic_t * var)2155 ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var)
2156 {
2157 ethr_sint_t res;
2158 ETHR_ASSERT(!ethr_not_inited__);
2159 ETHR_ASSERT(var);
2160 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2161 res = ethr_atomic_dec_read_relb__(var);
2162 #elif defined(ETHR_AMC_FALLBACK__)
2163 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2164 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
2165 #else
2166 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2167 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
2168 #endif
2169 return res;
2170 }
2171
ethr_atomic_dec_read_mb(ethr_atomic_t * var)2172 ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var)
2173 {
2174 ethr_sint_t res;
2175 ETHR_ASSERT(!ethr_not_inited__);
2176 ETHR_ASSERT(var);
2177 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2178 res = ethr_atomic_dec_read_mb__(var);
2179 #elif defined(ETHR_AMC_FALLBACK__)
2180 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2181 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
2182 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2183 #else
2184 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2185 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
2186 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2187 #endif
2188 return res;
2189 }
2190
2191
2192 /* -- add() -- */
2193
2194
ethr_atomic_add(ethr_atomic_t * var,ethr_sint_t val)2195 void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val)
2196 {
2197 ETHR_ASSERT(!ethr_not_inited__);
2198 ETHR_ASSERT(var);
2199 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2200 ethr_atomic_add__(var, val);
2201 #elif defined(ETHR_AMC_FALLBACK__)
2202 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
2203 #else
2204 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
2205 #endif
2206
2207 }
2208
ethr_atomic_add_ddrb(ethr_atomic_t * var,ethr_sint_t val)2209 void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val)
2210 {
2211 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2212 ethr_atomic_add_ddrb__(var, val);
2213 #elif defined(ETHR_ORDERED_READ_DEPEND)
2214 ethr_atomic_add(var, val);
2215 #else
2216 ethr_atomic_add_rb(var, val);
2217 #endif
2218 }
2219
ethr_atomic_add_rb(ethr_atomic_t * var,ethr_sint_t val)2220 void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val)
2221 {
2222 ETHR_ASSERT(!ethr_not_inited__);
2223 ETHR_ASSERT(var);
2224 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2225 ethr_atomic_add_rb__(var, val);
2226 #elif defined(ETHR_AMC_FALLBACK__)
2227 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
2228 ETHR_MEMBAR(ETHR_LoadLoad);
2229 #else
2230 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
2231 ETHR_MEMBAR(ETHR_LoadLoad);
2232 #endif
2233
2234 }
2235
ethr_atomic_add_wb(ethr_atomic_t * var,ethr_sint_t val)2236 void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val)
2237 {
2238 ETHR_ASSERT(!ethr_not_inited__);
2239 ETHR_ASSERT(var);
2240 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2241 ethr_atomic_add_wb__(var, val);
2242 #elif defined(ETHR_AMC_FALLBACK__)
2243 ETHR_MEMBAR(ETHR_StoreStore);
2244 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
2245 #else
2246 ETHR_MEMBAR(ETHR_StoreStore);
2247 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
2248 #endif
2249
2250 }
2251
ethr_atomic_add_acqb(ethr_atomic_t * var,ethr_sint_t val)2252 void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val)
2253 {
2254 ETHR_ASSERT(!ethr_not_inited__);
2255 ETHR_ASSERT(var);
2256 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2257 ethr_atomic_add_acqb__(var, val);
2258 #elif defined(ETHR_AMC_FALLBACK__)
2259 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
2260 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2261 #else
2262 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
2263 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2264 #endif
2265
2266 }
2267
ethr_atomic_add_relb(ethr_atomic_t * var,ethr_sint_t val)2268 void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val)
2269 {
2270 ETHR_ASSERT(!ethr_not_inited__);
2271 ETHR_ASSERT(var);
2272 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2273 ethr_atomic_add_relb__(var, val);
2274 #elif defined(ETHR_AMC_FALLBACK__)
2275 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2276 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
2277 #else
2278 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2279 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
2280 #endif
2281
2282 }
2283
ethr_atomic_add_mb(ethr_atomic_t * var,ethr_sint_t val)2284 void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val)
2285 {
2286 ETHR_ASSERT(!ethr_not_inited__);
2287 ETHR_ASSERT(var);
2288 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2289 ethr_atomic_add_mb__(var, val);
2290 #elif defined(ETHR_AMC_FALLBACK__)
2291 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2292 ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
2293 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2294 #else
2295 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2296 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
2297 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2298 #endif
2299
2300 }
2301
2302
2303 /* -- inc() -- */
2304
2305
ethr_atomic_inc(ethr_atomic_t * var)2306 void ethr_atomic_inc(ethr_atomic_t *var)
2307 {
2308 ETHR_ASSERT(!ethr_not_inited__);
2309 ETHR_ASSERT(var);
2310 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2311 ethr_atomic_inc__(var);
2312 #elif defined(ETHR_AMC_FALLBACK__)
2313 ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
2314 #else
2315 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
2316 #endif
2317
2318 }
2319
ethr_atomic_inc_ddrb(ethr_atomic_t * var)2320 void ethr_atomic_inc_ddrb(ethr_atomic_t *var)
2321 {
2322 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2323 ethr_atomic_inc_ddrb__(var);
2324 #elif defined(ETHR_ORDERED_READ_DEPEND)
2325 ethr_atomic_inc(var);
2326 #else
2327 ethr_atomic_inc_rb(var);
2328 #endif
2329 }
2330
ethr_atomic_inc_rb(ethr_atomic_t * var)2331 void ethr_atomic_inc_rb(ethr_atomic_t *var)
2332 {
2333 ETHR_ASSERT(!ethr_not_inited__);
2334 ETHR_ASSERT(var);
2335 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2336 ethr_atomic_inc_rb__(var);
2337 #elif defined(ETHR_AMC_FALLBACK__)
2338 ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
2339 ETHR_MEMBAR(ETHR_LoadLoad);
2340 #else
2341 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
2342 ETHR_MEMBAR(ETHR_LoadLoad);
2343 #endif
2344
2345 }
2346
ethr_atomic_inc_wb(ethr_atomic_t * var)2347 void ethr_atomic_inc_wb(ethr_atomic_t *var)
2348 {
2349 ETHR_ASSERT(!ethr_not_inited__);
2350 ETHR_ASSERT(var);
2351 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2352 ethr_atomic_inc_wb__(var);
2353 #elif defined(ETHR_AMC_FALLBACK__)
2354 ETHR_MEMBAR(ETHR_StoreStore);
2355 ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
2356 #else
2357 ETHR_MEMBAR(ETHR_StoreStore);
2358 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
2359 #endif
2360
2361 }
2362
ethr_atomic_inc_acqb(ethr_atomic_t * var)2363 void ethr_atomic_inc_acqb(ethr_atomic_t *var)
2364 {
2365 ETHR_ASSERT(!ethr_not_inited__);
2366 ETHR_ASSERT(var);
2367 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2368 ethr_atomic_inc_acqb__(var);
2369 #elif defined(ETHR_AMC_FALLBACK__)
2370 ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
2371 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2372 #else
2373 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
2374 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2375 #endif
2376
2377 }
2378
ethr_atomic_inc_relb(ethr_atomic_t * var)2379 void ethr_atomic_inc_relb(ethr_atomic_t *var)
2380 {
2381 ETHR_ASSERT(!ethr_not_inited__);
2382 ETHR_ASSERT(var);
2383 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2384 ethr_atomic_inc_relb__(var);
2385 #elif defined(ETHR_AMC_FALLBACK__)
2386 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2387 ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
2388 #else
2389 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2390 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
2391 #endif
2392
2393 }
2394
ethr_atomic_inc_mb(ethr_atomic_t * var)2395 void ethr_atomic_inc_mb(ethr_atomic_t *var)
2396 {
2397 ETHR_ASSERT(!ethr_not_inited__);
2398 ETHR_ASSERT(var);
2399 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2400 ethr_atomic_inc_mb__(var);
2401 #elif defined(ETHR_AMC_FALLBACK__)
2402 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2403 ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
2404 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2405 #else
2406 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2407 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
2408 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2409 #endif
2410
2411 }
2412
2413
2414 /* -- dec() -- */
2415
2416
ethr_atomic_dec(ethr_atomic_t * var)2417 void ethr_atomic_dec(ethr_atomic_t *var)
2418 {
2419 ETHR_ASSERT(!ethr_not_inited__);
2420 ETHR_ASSERT(var);
2421 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2422 ethr_atomic_dec__(var);
2423 #elif defined(ETHR_AMC_FALLBACK__)
2424 ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
2425 #else
2426 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
2427 #endif
2428
2429 }
2430
ethr_atomic_dec_ddrb(ethr_atomic_t * var)2431 void ethr_atomic_dec_ddrb(ethr_atomic_t *var)
2432 {
2433 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2434 ethr_atomic_dec_ddrb__(var);
2435 #elif defined(ETHR_ORDERED_READ_DEPEND)
2436 ethr_atomic_dec(var);
2437 #else
2438 ethr_atomic_dec_rb(var);
2439 #endif
2440 }
2441
ethr_atomic_dec_rb(ethr_atomic_t * var)2442 void ethr_atomic_dec_rb(ethr_atomic_t *var)
2443 {
2444 ETHR_ASSERT(!ethr_not_inited__);
2445 ETHR_ASSERT(var);
2446 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2447 ethr_atomic_dec_rb__(var);
2448 #elif defined(ETHR_AMC_FALLBACK__)
2449 ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
2450 ETHR_MEMBAR(ETHR_LoadLoad);
2451 #else
2452 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
2453 ETHR_MEMBAR(ETHR_LoadLoad);
2454 #endif
2455
2456 }
2457
ethr_atomic_dec_wb(ethr_atomic_t * var)2458 void ethr_atomic_dec_wb(ethr_atomic_t *var)
2459 {
2460 ETHR_ASSERT(!ethr_not_inited__);
2461 ETHR_ASSERT(var);
2462 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2463 ethr_atomic_dec_wb__(var);
2464 #elif defined(ETHR_AMC_FALLBACK__)
2465 ETHR_MEMBAR(ETHR_StoreStore);
2466 ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
2467 #else
2468 ETHR_MEMBAR(ETHR_StoreStore);
2469 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
2470 #endif
2471
2472 }
2473
ethr_atomic_dec_acqb(ethr_atomic_t * var)2474 void ethr_atomic_dec_acqb(ethr_atomic_t *var)
2475 {
2476 ETHR_ASSERT(!ethr_not_inited__);
2477 ETHR_ASSERT(var);
2478 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2479 ethr_atomic_dec_acqb__(var);
2480 #elif defined(ETHR_AMC_FALLBACK__)
2481 ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
2482 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2483 #else
2484 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
2485 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2486 #endif
2487
2488 }
2489
ethr_atomic_dec_relb(ethr_atomic_t * var)2490 void ethr_atomic_dec_relb(ethr_atomic_t *var)
2491 {
2492 ETHR_ASSERT(!ethr_not_inited__);
2493 ETHR_ASSERT(var);
2494 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2495 ethr_atomic_dec_relb__(var);
2496 #elif defined(ETHR_AMC_FALLBACK__)
2497 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2498 ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
2499 #else
2500 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2501 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
2502 #endif
2503
2504 }
2505
ethr_atomic_dec_mb(ethr_atomic_t * var)2506 void ethr_atomic_dec_mb(ethr_atomic_t *var)
2507 {
2508 ETHR_ASSERT(!ethr_not_inited__);
2509 ETHR_ASSERT(var);
2510 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2511 ethr_atomic_dec_mb__(var);
2512 #elif defined(ETHR_AMC_FALLBACK__)
2513 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2514 ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
2515 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2516 #else
2517 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2518 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
2519 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2520 #endif
2521
2522 }
2523
2524
2525 /* -- read_band() -- */
2526
2527
ethr_atomic_read_band(ethr_atomic_t * var,ethr_sint_t val)2528 ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val)
2529 {
2530 ethr_sint_t res;
2531 ETHR_ASSERT(!ethr_not_inited__);
2532 ETHR_ASSERT(var);
2533 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2534 res = ethr_atomic_read_band__(var, val);
2535 #elif defined(ETHR_AMC_FALLBACK__)
2536 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
2537 #else
2538 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
2539 #endif
2540 return res;
2541 }
2542
ethr_atomic_read_band_ddrb(ethr_atomic_t * var,ethr_sint_t val)2543 ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val)
2544 {
2545 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2546 return ethr_atomic_read_band_ddrb__(var, val);
2547 #elif defined(ETHR_ORDERED_READ_DEPEND)
2548 return ethr_atomic_read_band(var, val);
2549 #else
2550 return ethr_atomic_read_band_rb(var, val);
2551 #endif
2552 }
2553
ethr_atomic_read_band_rb(ethr_atomic_t * var,ethr_sint_t val)2554 ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val)
2555 {
2556 ethr_sint_t res;
2557 ETHR_ASSERT(!ethr_not_inited__);
2558 ETHR_ASSERT(var);
2559 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2560 res = ethr_atomic_read_band_rb__(var, val);
2561 #elif defined(ETHR_AMC_FALLBACK__)
2562 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
2563 ETHR_MEMBAR(ETHR_LoadLoad);
2564 #else
2565 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
2566 ETHR_MEMBAR(ETHR_LoadLoad);
2567 #endif
2568 return res;
2569 }
2570
ethr_atomic_read_band_wb(ethr_atomic_t * var,ethr_sint_t val)2571 ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val)
2572 {
2573 ethr_sint_t res;
2574 ETHR_ASSERT(!ethr_not_inited__);
2575 ETHR_ASSERT(var);
2576 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2577 res = ethr_atomic_read_band_wb__(var, val);
2578 #elif defined(ETHR_AMC_FALLBACK__)
2579 ETHR_MEMBAR(ETHR_StoreStore);
2580 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
2581 #else
2582 ETHR_MEMBAR(ETHR_StoreStore);
2583 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
2584 #endif
2585 return res;
2586 }
2587
ethr_atomic_read_band_acqb(ethr_atomic_t * var,ethr_sint_t val)2588 ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val)
2589 {
2590 ethr_sint_t res;
2591 ETHR_ASSERT(!ethr_not_inited__);
2592 ETHR_ASSERT(var);
2593 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2594 res = ethr_atomic_read_band_acqb__(var, val);
2595 #elif defined(ETHR_AMC_FALLBACK__)
2596 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
2597 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2598 #else
2599 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
2600 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2601 #endif
2602 return res;
2603 }
2604
ethr_atomic_read_band_relb(ethr_atomic_t * var,ethr_sint_t val)2605 ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val)
2606 {
2607 ethr_sint_t res;
2608 ETHR_ASSERT(!ethr_not_inited__);
2609 ETHR_ASSERT(var);
2610 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2611 res = ethr_atomic_read_band_relb__(var, val);
2612 #elif defined(ETHR_AMC_FALLBACK__)
2613 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2614 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
2615 #else
2616 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2617 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
2618 #endif
2619 return res;
2620 }
2621
ethr_atomic_read_band_mb(ethr_atomic_t * var,ethr_sint_t val)2622 ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val)
2623 {
2624 ethr_sint_t res;
2625 ETHR_ASSERT(!ethr_not_inited__);
2626 ETHR_ASSERT(var);
2627 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2628 res = ethr_atomic_read_band_mb__(var, val);
2629 #elif defined(ETHR_AMC_FALLBACK__)
2630 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2631 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
2632 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2633 #else
2634 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2635 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
2636 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2637 #endif
2638 return res;
2639 }
2640
2641
2642 /* -- read_bor() -- */
2643
2644
ethr_atomic_read_bor(ethr_atomic_t * var,ethr_sint_t val)2645 ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val)
2646 {
2647 ethr_sint_t res;
2648 ETHR_ASSERT(!ethr_not_inited__);
2649 ETHR_ASSERT(var);
2650 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2651 res = ethr_atomic_read_bor__(var, val);
2652 #elif defined(ETHR_AMC_FALLBACK__)
2653 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
2654 #else
2655 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
2656 #endif
2657 return res;
2658 }
2659
ethr_atomic_read_bor_ddrb(ethr_atomic_t * var,ethr_sint_t val)2660 ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val)
2661 {
2662 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2663 return ethr_atomic_read_bor_ddrb__(var, val);
2664 #elif defined(ETHR_ORDERED_READ_DEPEND)
2665 return ethr_atomic_read_bor(var, val);
2666 #else
2667 return ethr_atomic_read_bor_rb(var, val);
2668 #endif
2669 }
2670
ethr_atomic_read_bor_rb(ethr_atomic_t * var,ethr_sint_t val)2671 ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val)
2672 {
2673 ethr_sint_t res;
2674 ETHR_ASSERT(!ethr_not_inited__);
2675 ETHR_ASSERT(var);
2676 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2677 res = ethr_atomic_read_bor_rb__(var, val);
2678 #elif defined(ETHR_AMC_FALLBACK__)
2679 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
2680 ETHR_MEMBAR(ETHR_LoadLoad);
2681 #else
2682 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
2683 ETHR_MEMBAR(ETHR_LoadLoad);
2684 #endif
2685 return res;
2686 }
2687
ethr_atomic_read_bor_wb(ethr_atomic_t * var,ethr_sint_t val)2688 ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val)
2689 {
2690 ethr_sint_t res;
2691 ETHR_ASSERT(!ethr_not_inited__);
2692 ETHR_ASSERT(var);
2693 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2694 res = ethr_atomic_read_bor_wb__(var, val);
2695 #elif defined(ETHR_AMC_FALLBACK__)
2696 ETHR_MEMBAR(ETHR_StoreStore);
2697 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
2698 #else
2699 ETHR_MEMBAR(ETHR_StoreStore);
2700 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
2701 #endif
2702 return res;
2703 }
2704
ethr_atomic_read_bor_acqb(ethr_atomic_t * var,ethr_sint_t val)2705 ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val)
2706 {
2707 ethr_sint_t res;
2708 ETHR_ASSERT(!ethr_not_inited__);
2709 ETHR_ASSERT(var);
2710 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2711 res = ethr_atomic_read_bor_acqb__(var, val);
2712 #elif defined(ETHR_AMC_FALLBACK__)
2713 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
2714 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2715 #else
2716 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
2717 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2718 #endif
2719 return res;
2720 }
2721
ethr_atomic_read_bor_relb(ethr_atomic_t * var,ethr_sint_t val)2722 ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val)
2723 {
2724 ethr_sint_t res;
2725 ETHR_ASSERT(!ethr_not_inited__);
2726 ETHR_ASSERT(var);
2727 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2728 res = ethr_atomic_read_bor_relb__(var, val);
2729 #elif defined(ETHR_AMC_FALLBACK__)
2730 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2731 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
2732 #else
2733 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2734 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
2735 #endif
2736 return res;
2737 }
2738
ethr_atomic_read_bor_mb(ethr_atomic_t * var,ethr_sint_t val)2739 ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val)
2740 {
2741 ethr_sint_t res;
2742 ETHR_ASSERT(!ethr_not_inited__);
2743 ETHR_ASSERT(var);
2744 #if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
2745 res = ethr_atomic_read_bor_mb__(var, val);
2746 #elif defined(ETHR_AMC_FALLBACK__)
2747 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2748 ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
2749 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2750 #else
2751 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2752 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
2753 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2754 #endif
2755 return res;
2756 }
2757
2758
2759 /* ---------- 32-bit atomic implementation ---------- */
2760
2761
2762
2763
2764 /* --- addr() --- */
2765
ethr_atomic32_addr(ethr_atomic32_t * var)2766 ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var)
2767 {
2768 ethr_sint32_t *res;
2769 ETHR_ASSERT(!ethr_not_inited__);
2770 ETHR_ASSERT(var);
2771 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2772 res = ethr_atomic32_addr__(var);
2773
2774 #else
2775 res = (ethr_sint32_t *) var;
2776 #endif
2777 return res;
2778 }
2779
2780
2781 /* -- cmpxchg() -- */
2782
2783
ethr_atomic32_cmpxchg(ethr_atomic32_t * var,ethr_sint32_t val,ethr_sint32_t old_val)2784 ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
2785 {
2786 ethr_sint32_t res;
2787 ETHR_ASSERT(!ethr_not_inited__);
2788 ETHR_ASSERT(var);
2789 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2790 res = ethr_atomic32_cmpxchg__(var, val, old_val);
2791 #else
2792 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
2793 #endif
2794 return res;
2795 }
2796
ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t * var,ethr_sint32_t val,ethr_sint32_t old_val)2797 ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
2798 {
2799 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2800 return ethr_atomic32_cmpxchg_ddrb__(var, val, old_val);
2801 #elif defined(ETHR_ORDERED_READ_DEPEND)
2802 return ethr_atomic32_cmpxchg(var, val, old_val);
2803 #else
2804 return ethr_atomic32_cmpxchg_rb(var, val, old_val);
2805 #endif
2806 }
2807
ethr_atomic32_cmpxchg_rb(ethr_atomic32_t * var,ethr_sint32_t val,ethr_sint32_t old_val)2808 ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
2809 {
2810 ethr_sint32_t res;
2811 ETHR_ASSERT(!ethr_not_inited__);
2812 ETHR_ASSERT(var);
2813 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2814 res = ethr_atomic32_cmpxchg_rb__(var, val, old_val);
2815 #else
2816 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
2817 ETHR_MEMBAR(ETHR_LoadLoad);
2818 #endif
2819 return res;
2820 }
2821
ethr_atomic32_cmpxchg_wb(ethr_atomic32_t * var,ethr_sint32_t val,ethr_sint32_t old_val)2822 ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
2823 {
2824 ethr_sint32_t res;
2825 ETHR_ASSERT(!ethr_not_inited__);
2826 ETHR_ASSERT(var);
2827 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2828 res = ethr_atomic32_cmpxchg_wb__(var, val, old_val);
2829 #else
2830 ETHR_MEMBAR(ETHR_StoreStore);
2831 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
2832 #endif
2833 return res;
2834 }
2835
ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t * var,ethr_sint32_t val,ethr_sint32_t old_val)2836 ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
2837 {
2838 ethr_sint32_t res;
2839 ETHR_ASSERT(!ethr_not_inited__);
2840 ETHR_ASSERT(var);
2841 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2842 res = ethr_atomic32_cmpxchg_acqb__(var, val, old_val);
2843 #else
2844 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
2845 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2846 #endif
2847 return res;
2848 }
2849
ethr_atomic32_cmpxchg_relb(ethr_atomic32_t * var,ethr_sint32_t val,ethr_sint32_t old_val)2850 ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
2851 {
2852 ethr_sint32_t res;
2853 ETHR_ASSERT(!ethr_not_inited__);
2854 ETHR_ASSERT(var);
2855 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2856 res = ethr_atomic32_cmpxchg_relb__(var, val, old_val);
2857 #else
2858 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2859 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
2860 #endif
2861 return res;
2862 }
2863
ethr_atomic32_cmpxchg_mb(ethr_atomic32_t * var,ethr_sint32_t val,ethr_sint32_t old_val)2864 ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
2865 {
2866 ethr_sint32_t res;
2867 ETHR_ASSERT(!ethr_not_inited__);
2868 ETHR_ASSERT(var);
2869 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2870 res = ethr_atomic32_cmpxchg_mb__(var, val, old_val);
2871 #else
2872 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2873 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
2874 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2875 #endif
2876 return res;
2877 }
2878
2879
2880 /* -- xchg() -- */
2881
2882
ethr_atomic32_xchg(ethr_atomic32_t * var,ethr_sint32_t val)2883 ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val)
2884 {
2885 ethr_sint32_t res;
2886 ETHR_ASSERT(!ethr_not_inited__);
2887 ETHR_ASSERT(var);
2888 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2889 res = ethr_atomic32_xchg__(var, val);
2890 #else
2891 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
2892 #endif
2893 return res;
2894 }
2895
ethr_atomic32_xchg_ddrb(ethr_atomic32_t * var,ethr_sint32_t val)2896 ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
2897 {
2898 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2899 return ethr_atomic32_xchg_ddrb__(var, val);
2900 #elif defined(ETHR_ORDERED_READ_DEPEND)
2901 return ethr_atomic32_xchg(var, val);
2902 #else
2903 return ethr_atomic32_xchg_rb(var, val);
2904 #endif
2905 }
2906
ethr_atomic32_xchg_rb(ethr_atomic32_t * var,ethr_sint32_t val)2907 ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val)
2908 {
2909 ethr_sint32_t res;
2910 ETHR_ASSERT(!ethr_not_inited__);
2911 ETHR_ASSERT(var);
2912 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2913 res = ethr_atomic32_xchg_rb__(var, val);
2914 #else
2915 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
2916 ETHR_MEMBAR(ETHR_LoadLoad);
2917 #endif
2918 return res;
2919 }
2920
ethr_atomic32_xchg_wb(ethr_atomic32_t * var,ethr_sint32_t val)2921 ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val)
2922 {
2923 ethr_sint32_t res;
2924 ETHR_ASSERT(!ethr_not_inited__);
2925 ETHR_ASSERT(var);
2926 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2927 res = ethr_atomic32_xchg_wb__(var, val);
2928 #else
2929 ETHR_MEMBAR(ETHR_StoreStore);
2930 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
2931 #endif
2932 return res;
2933 }
2934
ethr_atomic32_xchg_acqb(ethr_atomic32_t * var,ethr_sint32_t val)2935 ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
2936 {
2937 ethr_sint32_t res;
2938 ETHR_ASSERT(!ethr_not_inited__);
2939 ETHR_ASSERT(var);
2940 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2941 res = ethr_atomic32_xchg_acqb__(var, val);
2942 #else
2943 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
2944 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2945 #endif
2946 return res;
2947 }
2948
ethr_atomic32_xchg_relb(ethr_atomic32_t * var,ethr_sint32_t val)2949 ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val)
2950 {
2951 ethr_sint32_t res;
2952 ETHR_ASSERT(!ethr_not_inited__);
2953 ETHR_ASSERT(var);
2954 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2955 res = ethr_atomic32_xchg_relb__(var, val);
2956 #else
2957 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2958 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
2959 #endif
2960 return res;
2961 }
2962
ethr_atomic32_xchg_mb(ethr_atomic32_t * var,ethr_sint32_t val)2963 ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val)
2964 {
2965 ethr_sint32_t res;
2966 ETHR_ASSERT(!ethr_not_inited__);
2967 ETHR_ASSERT(var);
2968 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2969 res = ethr_atomic32_xchg_mb__(var, val);
2970 #else
2971 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2972 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
2973 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2974 #endif
2975 return res;
2976 }
2977
2978
2979 /* -- set() -- */
2980
2981
ethr_atomic32_set(ethr_atomic32_t * var,ethr_sint32_t val)2982 void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val)
2983 {
2984 ETHR_ASSERT(!ethr_not_inited__);
2985 ETHR_ASSERT(var);
2986 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2987 ethr_atomic32_set__(var, val);
2988 #else
2989 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
2990 #endif
2991
2992 }
2993
ethr_atomic32_set_ddrb(ethr_atomic32_t * var,ethr_sint32_t val)2994 void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
2995 {
2996 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
2997 ethr_atomic32_set_ddrb__(var, val);
2998 #elif defined(ETHR_ORDERED_READ_DEPEND)
2999 ethr_atomic32_set(var, val);
3000 #else
3001 ethr_atomic32_set_rb(var, val);
3002 #endif
3003 }
3004
ethr_atomic32_set_rb(ethr_atomic32_t * var,ethr_sint32_t val)3005 void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val)
3006 {
3007 ETHR_ASSERT(!ethr_not_inited__);
3008 ETHR_ASSERT(var);
3009 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3010 ethr_atomic32_set_rb__(var, val);
3011 #else
3012 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3013 ETHR_MEMBAR(ETHR_LoadLoad);
3014 #endif
3015
3016 }
3017
ethr_atomic32_set_wb(ethr_atomic32_t * var,ethr_sint32_t val)3018 void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val)
3019 {
3020 ETHR_ASSERT(!ethr_not_inited__);
3021 ETHR_ASSERT(var);
3022 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3023 ethr_atomic32_set_wb__(var, val);
3024 #else
3025 ETHR_MEMBAR(ETHR_StoreStore);
3026 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3027 #endif
3028
3029 }
3030
ethr_atomic32_set_acqb(ethr_atomic32_t * var,ethr_sint32_t val)3031 void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
3032 {
3033 ETHR_ASSERT(!ethr_not_inited__);
3034 ETHR_ASSERT(var);
3035 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3036 ethr_atomic32_set_acqb__(var, val);
3037 #else
3038 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3039 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
3040 #endif
3041
3042 }
3043
ethr_atomic32_set_relb(ethr_atomic32_t * var,ethr_sint32_t val)3044 void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val)
3045 {
3046 ETHR_ASSERT(!ethr_not_inited__);
3047 ETHR_ASSERT(var);
3048 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3049 ethr_atomic32_set_relb__(var, val);
3050 #else
3051 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
3052 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3053 #endif
3054
3055 }
3056
ethr_atomic32_set_mb(ethr_atomic32_t * var,ethr_sint32_t val)3057 void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val)
3058 {
3059 ETHR_ASSERT(!ethr_not_inited__);
3060 ETHR_ASSERT(var);
3061 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3062 ethr_atomic32_set_mb__(var, val);
3063 #else
3064 ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
3065 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3066 ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
3067 #endif
3068
3069 }
3070
3071
3072 /* -- init() -- */
3073
3074
ethr_atomic32_init(ethr_atomic32_t * var,ethr_sint32_t val)3075 void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val)
3076 {
3077 ETHR_ASSERT(var);
3078 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3079 ethr_atomic32_init__(var, val);
3080 #else
3081 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3082 #endif
3083
3084 }
3085
ethr_atomic32_init_ddrb(ethr_atomic32_t * var,ethr_sint32_t val)3086 void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
3087 {
3088 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3089 ethr_atomic32_init_ddrb__(var, val);
3090 #elif defined(ETHR_ORDERED_READ_DEPEND)
3091 ethr_atomic32_init(var, val);
3092 #else
3093 ethr_atomic32_init_rb(var, val);
3094 #endif
3095 }
3096
ethr_atomic32_init_rb(ethr_atomic32_t * var,ethr_sint32_t val)3097 void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val)
3098 {
3099 ETHR_ASSERT(var);
3100 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3101 ethr_atomic32_init_rb__(var, val);
3102 #else
3103 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3104 ETHR_MEMBAR(ETHR_LoadLoad);
3105 #endif
3106
3107 }
3108
ethr_atomic32_init_wb(ethr_atomic32_t * var,ethr_sint32_t val)3109 void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val)
3110 {
3111 ETHR_ASSERT(var);
3112 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3113 ethr_atomic32_init_wb__(var, val);
3114 #else
3115 ETHR_MEMBAR(ETHR_StoreStore);
3116 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3117 #endif
3118
3119 }
3120
ethr_atomic32_init_acqb(ethr_atomic32_t * var,ethr_sint32_t val)3121 void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
3122 {
3123 ETHR_ASSERT(var);
3124 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3125 ethr_atomic32_init_acqb__(var, val);
3126 #else
3127 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3128 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3129 #endif
3130
3131 }
3132
ethr_atomic32_init_relb(ethr_atomic32_t * var,ethr_sint32_t val)3133 void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val)
3134 {
3135 ETHR_ASSERT(var);
3136 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3137 ethr_atomic32_init_relb__(var, val);
3138 #else
3139 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3140 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3141 #endif
3142
3143 }
3144
ethr_atomic32_init_mb(ethr_atomic32_t * var,ethr_sint32_t val)3145 void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val)
3146 {
3147 ETHR_ASSERT(var);
3148 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3149 ethr_atomic32_init_mb__(var, val);
3150 #else
3151 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3152 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
3153 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3154 #endif
3155
3156 }
3157
3158
3159 /* -- add_read() -- */
3160
3161
ethr_atomic32_add_read(ethr_atomic32_t * var,ethr_sint32_t val)3162 ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val)
3163 {
3164 ethr_sint32_t res;
3165 ETHR_ASSERT(!ethr_not_inited__);
3166 ETHR_ASSERT(var);
3167 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3168 res = ethr_atomic32_add_read__(var, val);
3169 #else
3170 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
3171 #endif
3172 return res;
3173 }
3174
ethr_atomic32_add_read_ddrb(ethr_atomic32_t * var,ethr_sint32_t val)3175 ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
3176 {
3177 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3178 return ethr_atomic32_add_read_ddrb__(var, val);
3179 #elif defined(ETHR_ORDERED_READ_DEPEND)
3180 return ethr_atomic32_add_read(var, val);
3181 #else
3182 return ethr_atomic32_add_read_rb(var, val);
3183 #endif
3184 }
3185
ethr_atomic32_add_read_rb(ethr_atomic32_t * var,ethr_sint32_t val)3186 ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val)
3187 {
3188 ethr_sint32_t res;
3189 ETHR_ASSERT(!ethr_not_inited__);
3190 ETHR_ASSERT(var);
3191 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3192 res = ethr_atomic32_add_read_rb__(var, val);
3193 #else
3194 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
3195 ETHR_MEMBAR(ETHR_LoadLoad);
3196 #endif
3197 return res;
3198 }
3199
ethr_atomic32_add_read_wb(ethr_atomic32_t * var,ethr_sint32_t val)3200 ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val)
3201 {
3202 ethr_sint32_t res;
3203 ETHR_ASSERT(!ethr_not_inited__);
3204 ETHR_ASSERT(var);
3205 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3206 res = ethr_atomic32_add_read_wb__(var, val);
3207 #else
3208 ETHR_MEMBAR(ETHR_StoreStore);
3209 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
3210 #endif
3211 return res;
3212 }
3213
ethr_atomic32_add_read_acqb(ethr_atomic32_t * var,ethr_sint32_t val)3214 ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
3215 {
3216 ethr_sint32_t res;
3217 ETHR_ASSERT(!ethr_not_inited__);
3218 ETHR_ASSERT(var);
3219 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3220 res = ethr_atomic32_add_read_acqb__(var, val);
3221 #else
3222 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
3223 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3224 #endif
3225 return res;
3226 }
3227
ethr_atomic32_add_read_relb(ethr_atomic32_t * var,ethr_sint32_t val)3228 ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val)
3229 {
3230 ethr_sint32_t res;
3231 ETHR_ASSERT(!ethr_not_inited__);
3232 ETHR_ASSERT(var);
3233 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3234 res = ethr_atomic32_add_read_relb__(var, val);
3235 #else
3236 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3237 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
3238 #endif
3239 return res;
3240 }
3241
ethr_atomic32_add_read_mb(ethr_atomic32_t * var,ethr_sint32_t val)3242 ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val)
3243 {
3244 ethr_sint32_t res;
3245 ETHR_ASSERT(!ethr_not_inited__);
3246 ETHR_ASSERT(var);
3247 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3248 res = ethr_atomic32_add_read_mb__(var, val);
3249 #else
3250 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3251 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
3252 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3253 #endif
3254 return res;
3255 }
3256
3257
3258 /* -- read() -- */
3259
3260
ethr_atomic32_read(ethr_atomic32_t * var)3261 ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var)
3262 {
3263 ethr_sint32_t res;
3264 ETHR_ASSERT(!ethr_not_inited__);
3265 ETHR_ASSERT(var);
3266 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3267 res = ethr_atomic32_read__(var);
3268 #else
3269 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
3270 #endif
3271 return res;
3272 }
3273
ethr_atomic32_read_ddrb(ethr_atomic32_t * var)3274 ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var)
3275 {
3276 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3277 return ethr_atomic32_read_ddrb__(var);
3278 #elif defined(ETHR_ORDERED_READ_DEPEND)
3279 return ethr_atomic32_read(var);
3280 #else
3281 return ethr_atomic32_read_rb(var);
3282 #endif
3283 }
3284
ethr_atomic32_read_rb(ethr_atomic32_t * var)3285 ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var)
3286 {
3287 ethr_sint32_t res;
3288 ETHR_ASSERT(!ethr_not_inited__);
3289 ETHR_ASSERT(var);
3290 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3291 res = ethr_atomic32_read_rb__(var);
3292 #else
3293 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
3294 ETHR_MEMBAR(ETHR_LoadLoad);
3295 #endif
3296 return res;
3297 }
3298
ethr_atomic32_read_wb(ethr_atomic32_t * var)3299 ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var)
3300 {
3301 ethr_sint32_t res;
3302 ETHR_ASSERT(!ethr_not_inited__);
3303 ETHR_ASSERT(var);
3304 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3305 res = ethr_atomic32_read_wb__(var);
3306 #else
3307 ETHR_MEMBAR(ETHR_StoreStore);
3308 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
3309 #endif
3310 return res;
3311 }
3312
ethr_atomic32_read_acqb(ethr_atomic32_t * var)3313 ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var)
3314 {
3315 ethr_sint32_t res;
3316 ETHR_ASSERT(!ethr_not_inited__);
3317 ETHR_ASSERT(var);
3318 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3319 res = ethr_atomic32_read_acqb__(var);
3320 #else
3321 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
3322 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
3323 #endif
3324 return res;
3325 }
3326
ethr_atomic32_read_relb(ethr_atomic32_t * var)3327 ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var)
3328 {
3329 ethr_sint32_t res;
3330 ETHR_ASSERT(!ethr_not_inited__);
3331 ETHR_ASSERT(var);
3332 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3333 res = ethr_atomic32_read_relb__(var);
3334 #else
3335 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3336 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
3337 #endif
3338 return res;
3339 }
3340
ethr_atomic32_read_mb(ethr_atomic32_t * var)3341 ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var)
3342 {
3343 ethr_sint32_t res;
3344 ETHR_ASSERT(!ethr_not_inited__);
3345 ETHR_ASSERT(var);
3346 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3347 res = ethr_atomic32_read_mb__(var);
3348 #else
3349 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3350 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
3351 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
3352 #endif
3353 return res;
3354 }
3355
3356
3357 /* -- inc_read() -- */
3358
3359
ethr_atomic32_inc_read(ethr_atomic32_t * var)3360 ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var)
3361 {
3362 ethr_sint32_t res;
3363 ETHR_ASSERT(!ethr_not_inited__);
3364 ETHR_ASSERT(var);
3365 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3366 res = ethr_atomic32_inc_read__(var);
3367 #else
3368 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
3369 #endif
3370 return res;
3371 }
3372
ethr_atomic32_inc_read_ddrb(ethr_atomic32_t * var)3373 ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var)
3374 {
3375 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3376 return ethr_atomic32_inc_read_ddrb__(var);
3377 #elif defined(ETHR_ORDERED_READ_DEPEND)
3378 return ethr_atomic32_inc_read(var);
3379 #else
3380 return ethr_atomic32_inc_read_rb(var);
3381 #endif
3382 }
3383
ethr_atomic32_inc_read_rb(ethr_atomic32_t * var)3384 ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var)
3385 {
3386 ethr_sint32_t res;
3387 ETHR_ASSERT(!ethr_not_inited__);
3388 ETHR_ASSERT(var);
3389 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3390 res = ethr_atomic32_inc_read_rb__(var);
3391 #else
3392 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
3393 ETHR_MEMBAR(ETHR_LoadLoad);
3394 #endif
3395 return res;
3396 }
3397
ethr_atomic32_inc_read_wb(ethr_atomic32_t * var)3398 ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var)
3399 {
3400 ethr_sint32_t res;
3401 ETHR_ASSERT(!ethr_not_inited__);
3402 ETHR_ASSERT(var);
3403 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3404 res = ethr_atomic32_inc_read_wb__(var);
3405 #else
3406 ETHR_MEMBAR(ETHR_StoreStore);
3407 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
3408 #endif
3409 return res;
3410 }
3411
ethr_atomic32_inc_read_acqb(ethr_atomic32_t * var)3412 ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var)
3413 {
3414 ethr_sint32_t res;
3415 ETHR_ASSERT(!ethr_not_inited__);
3416 ETHR_ASSERT(var);
3417 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3418 res = ethr_atomic32_inc_read_acqb__(var);
3419 #else
3420 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
3421 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3422 #endif
3423 return res;
3424 }
3425
ethr_atomic32_inc_read_relb(ethr_atomic32_t * var)3426 ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var)
3427 {
3428 ethr_sint32_t res;
3429 ETHR_ASSERT(!ethr_not_inited__);
3430 ETHR_ASSERT(var);
3431 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3432 res = ethr_atomic32_inc_read_relb__(var);
3433 #else
3434 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3435 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
3436 #endif
3437 return res;
3438 }
3439
ethr_atomic32_inc_read_mb(ethr_atomic32_t * var)3440 ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var)
3441 {
3442 ethr_sint32_t res;
3443 ETHR_ASSERT(!ethr_not_inited__);
3444 ETHR_ASSERT(var);
3445 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3446 res = ethr_atomic32_inc_read_mb__(var);
3447 #else
3448 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3449 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
3450 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3451 #endif
3452 return res;
3453 }
3454
3455
3456 /* -- dec_read() -- */
3457
3458
ethr_atomic32_dec_read(ethr_atomic32_t * var)3459 ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var)
3460 {
3461 ethr_sint32_t res;
3462 ETHR_ASSERT(!ethr_not_inited__);
3463 ETHR_ASSERT(var);
3464 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3465 res = ethr_atomic32_dec_read__(var);
3466 #else
3467 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
3468 #endif
3469 return res;
3470 }
3471
ethr_atomic32_dec_read_ddrb(ethr_atomic32_t * var)3472 ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var)
3473 {
3474 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3475 return ethr_atomic32_dec_read_ddrb__(var);
3476 #elif defined(ETHR_ORDERED_READ_DEPEND)
3477 return ethr_atomic32_dec_read(var);
3478 #else
3479 return ethr_atomic32_dec_read_rb(var);
3480 #endif
3481 }
3482
ethr_atomic32_dec_read_rb(ethr_atomic32_t * var)3483 ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var)
3484 {
3485 ethr_sint32_t res;
3486 ETHR_ASSERT(!ethr_not_inited__);
3487 ETHR_ASSERT(var);
3488 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3489 res = ethr_atomic32_dec_read_rb__(var);
3490 #else
3491 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
3492 ETHR_MEMBAR(ETHR_LoadLoad);
3493 #endif
3494 return res;
3495 }
3496
ethr_atomic32_dec_read_wb(ethr_atomic32_t * var)3497 ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var)
3498 {
3499 ethr_sint32_t res;
3500 ETHR_ASSERT(!ethr_not_inited__);
3501 ETHR_ASSERT(var);
3502 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3503 res = ethr_atomic32_dec_read_wb__(var);
3504 #else
3505 ETHR_MEMBAR(ETHR_StoreStore);
3506 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
3507 #endif
3508 return res;
3509 }
3510
ethr_atomic32_dec_read_acqb(ethr_atomic32_t * var)3511 ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var)
3512 {
3513 ethr_sint32_t res;
3514 ETHR_ASSERT(!ethr_not_inited__);
3515 ETHR_ASSERT(var);
3516 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3517 res = ethr_atomic32_dec_read_acqb__(var);
3518 #else
3519 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
3520 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3521 #endif
3522 return res;
3523 }
3524
ethr_atomic32_dec_read_relb(ethr_atomic32_t * var)3525 ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var)
3526 {
3527 ethr_sint32_t res;
3528 ETHR_ASSERT(!ethr_not_inited__);
3529 ETHR_ASSERT(var);
3530 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3531 res = ethr_atomic32_dec_read_relb__(var);
3532 #else
3533 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3534 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
3535 #endif
3536 return res;
3537 }
3538
ethr_atomic32_dec_read_mb(ethr_atomic32_t * var)3539 ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var)
3540 {
3541 ethr_sint32_t res;
3542 ETHR_ASSERT(!ethr_not_inited__);
3543 ETHR_ASSERT(var);
3544 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3545 res = ethr_atomic32_dec_read_mb__(var);
3546 #else
3547 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3548 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
3549 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3550 #endif
3551 return res;
3552 }
3553
3554
3555 /* -- add() -- */
3556
3557
ethr_atomic32_add(ethr_atomic32_t * var,ethr_sint32_t val)3558 void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val)
3559 {
3560 ETHR_ASSERT(!ethr_not_inited__);
3561 ETHR_ASSERT(var);
3562 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3563 ethr_atomic32_add__(var, val);
3564 #else
3565 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
3566 #endif
3567
3568 }
3569
ethr_atomic32_add_ddrb(ethr_atomic32_t * var,ethr_sint32_t val)3570 void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
3571 {
3572 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3573 ethr_atomic32_add_ddrb__(var, val);
3574 #elif defined(ETHR_ORDERED_READ_DEPEND)
3575 ethr_atomic32_add(var, val);
3576 #else
3577 ethr_atomic32_add_rb(var, val);
3578 #endif
3579 }
3580
ethr_atomic32_add_rb(ethr_atomic32_t * var,ethr_sint32_t val)3581 void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val)
3582 {
3583 ETHR_ASSERT(!ethr_not_inited__);
3584 ETHR_ASSERT(var);
3585 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3586 ethr_atomic32_add_rb__(var, val);
3587 #else
3588 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
3589 ETHR_MEMBAR(ETHR_LoadLoad);
3590 #endif
3591
3592 }
3593
ethr_atomic32_add_wb(ethr_atomic32_t * var,ethr_sint32_t val)3594 void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val)
3595 {
3596 ETHR_ASSERT(!ethr_not_inited__);
3597 ETHR_ASSERT(var);
3598 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3599 ethr_atomic32_add_wb__(var, val);
3600 #else
3601 ETHR_MEMBAR(ETHR_StoreStore);
3602 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
3603 #endif
3604
3605 }
3606
ethr_atomic32_add_acqb(ethr_atomic32_t * var,ethr_sint32_t val)3607 void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
3608 {
3609 ETHR_ASSERT(!ethr_not_inited__);
3610 ETHR_ASSERT(var);
3611 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3612 ethr_atomic32_add_acqb__(var, val);
3613 #else
3614 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
3615 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3616 #endif
3617
3618 }
3619
ethr_atomic32_add_relb(ethr_atomic32_t * var,ethr_sint32_t val)3620 void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val)
3621 {
3622 ETHR_ASSERT(!ethr_not_inited__);
3623 ETHR_ASSERT(var);
3624 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3625 ethr_atomic32_add_relb__(var, val);
3626 #else
3627 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3628 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
3629 #endif
3630
3631 }
3632
ethr_atomic32_add_mb(ethr_atomic32_t * var,ethr_sint32_t val)3633 void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val)
3634 {
3635 ETHR_ASSERT(!ethr_not_inited__);
3636 ETHR_ASSERT(var);
3637 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3638 ethr_atomic32_add_mb__(var, val);
3639 #else
3640 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3641 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
3642 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3643 #endif
3644
3645 }
3646
3647
3648 /* -- inc() -- */
3649
3650
ethr_atomic32_inc(ethr_atomic32_t * var)3651 void ethr_atomic32_inc(ethr_atomic32_t *var)
3652 {
3653 ETHR_ASSERT(!ethr_not_inited__);
3654 ETHR_ASSERT(var);
3655 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3656 ethr_atomic32_inc__(var);
3657 #else
3658 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
3659 #endif
3660
3661 }
3662
ethr_atomic32_inc_ddrb(ethr_atomic32_t * var)3663 void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var)
3664 {
3665 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3666 ethr_atomic32_inc_ddrb__(var);
3667 #elif defined(ETHR_ORDERED_READ_DEPEND)
3668 ethr_atomic32_inc(var);
3669 #else
3670 ethr_atomic32_inc_rb(var);
3671 #endif
3672 }
3673
ethr_atomic32_inc_rb(ethr_atomic32_t * var)3674 void ethr_atomic32_inc_rb(ethr_atomic32_t *var)
3675 {
3676 ETHR_ASSERT(!ethr_not_inited__);
3677 ETHR_ASSERT(var);
3678 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3679 ethr_atomic32_inc_rb__(var);
3680 #else
3681 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
3682 ETHR_MEMBAR(ETHR_LoadLoad);
3683 #endif
3684
3685 }
3686
ethr_atomic32_inc_wb(ethr_atomic32_t * var)3687 void ethr_atomic32_inc_wb(ethr_atomic32_t *var)
3688 {
3689 ETHR_ASSERT(!ethr_not_inited__);
3690 ETHR_ASSERT(var);
3691 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3692 ethr_atomic32_inc_wb__(var);
3693 #else
3694 ETHR_MEMBAR(ETHR_StoreStore);
3695 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
3696 #endif
3697
3698 }
3699
ethr_atomic32_inc_acqb(ethr_atomic32_t * var)3700 void ethr_atomic32_inc_acqb(ethr_atomic32_t *var)
3701 {
3702 ETHR_ASSERT(!ethr_not_inited__);
3703 ETHR_ASSERT(var);
3704 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3705 ethr_atomic32_inc_acqb__(var);
3706 #else
3707 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
3708 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3709 #endif
3710
3711 }
3712
ethr_atomic32_inc_relb(ethr_atomic32_t * var)3713 void ethr_atomic32_inc_relb(ethr_atomic32_t *var)
3714 {
3715 ETHR_ASSERT(!ethr_not_inited__);
3716 ETHR_ASSERT(var);
3717 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3718 ethr_atomic32_inc_relb__(var);
3719 #else
3720 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3721 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
3722 #endif
3723
3724 }
3725
ethr_atomic32_inc_mb(ethr_atomic32_t * var)3726 void ethr_atomic32_inc_mb(ethr_atomic32_t *var)
3727 {
3728 ETHR_ASSERT(!ethr_not_inited__);
3729 ETHR_ASSERT(var);
3730 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3731 ethr_atomic32_inc_mb__(var);
3732 #else
3733 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3734 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
3735 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3736 #endif
3737
3738 }
3739
3740
3741 /* -- dec() -- */
3742
3743
ethr_atomic32_dec(ethr_atomic32_t * var)3744 void ethr_atomic32_dec(ethr_atomic32_t *var)
3745 {
3746 ETHR_ASSERT(!ethr_not_inited__);
3747 ETHR_ASSERT(var);
3748 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3749 ethr_atomic32_dec__(var);
3750 #else
3751 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
3752 #endif
3753
3754 }
3755
ethr_atomic32_dec_ddrb(ethr_atomic32_t * var)3756 void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var)
3757 {
3758 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3759 ethr_atomic32_dec_ddrb__(var);
3760 #elif defined(ETHR_ORDERED_READ_DEPEND)
3761 ethr_atomic32_dec(var);
3762 #else
3763 ethr_atomic32_dec_rb(var);
3764 #endif
3765 }
3766
ethr_atomic32_dec_rb(ethr_atomic32_t * var)3767 void ethr_atomic32_dec_rb(ethr_atomic32_t *var)
3768 {
3769 ETHR_ASSERT(!ethr_not_inited__);
3770 ETHR_ASSERT(var);
3771 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3772 ethr_atomic32_dec_rb__(var);
3773 #else
3774 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
3775 ETHR_MEMBAR(ETHR_LoadLoad);
3776 #endif
3777
3778 }
3779
ethr_atomic32_dec_wb(ethr_atomic32_t * var)3780 void ethr_atomic32_dec_wb(ethr_atomic32_t *var)
3781 {
3782 ETHR_ASSERT(!ethr_not_inited__);
3783 ETHR_ASSERT(var);
3784 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3785 ethr_atomic32_dec_wb__(var);
3786 #else
3787 ETHR_MEMBAR(ETHR_StoreStore);
3788 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
3789 #endif
3790
3791 }
3792
ethr_atomic32_dec_acqb(ethr_atomic32_t * var)3793 void ethr_atomic32_dec_acqb(ethr_atomic32_t *var)
3794 {
3795 ETHR_ASSERT(!ethr_not_inited__);
3796 ETHR_ASSERT(var);
3797 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3798 ethr_atomic32_dec_acqb__(var);
3799 #else
3800 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
3801 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3802 #endif
3803
3804 }
3805
ethr_atomic32_dec_relb(ethr_atomic32_t * var)3806 void ethr_atomic32_dec_relb(ethr_atomic32_t *var)
3807 {
3808 ETHR_ASSERT(!ethr_not_inited__);
3809 ETHR_ASSERT(var);
3810 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3811 ethr_atomic32_dec_relb__(var);
3812 #else
3813 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3814 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
3815 #endif
3816
3817 }
3818
ethr_atomic32_dec_mb(ethr_atomic32_t * var)3819 void ethr_atomic32_dec_mb(ethr_atomic32_t *var)
3820 {
3821 ETHR_ASSERT(!ethr_not_inited__);
3822 ETHR_ASSERT(var);
3823 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3824 ethr_atomic32_dec_mb__(var);
3825 #else
3826 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3827 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
3828 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3829 #endif
3830
3831 }
3832
3833
3834 /* -- read_band() -- */
3835
3836
ethr_atomic32_read_band(ethr_atomic32_t * var,ethr_sint32_t val)3837 ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val)
3838 {
3839 ethr_sint32_t res;
3840 ETHR_ASSERT(!ethr_not_inited__);
3841 ETHR_ASSERT(var);
3842 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3843 res = ethr_atomic32_read_band__(var, val);
3844 #else
3845 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
3846 #endif
3847 return res;
3848 }
3849
ethr_atomic32_read_band_ddrb(ethr_atomic32_t * var,ethr_sint32_t val)3850 ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
3851 {
3852 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3853 return ethr_atomic32_read_band_ddrb__(var, val);
3854 #elif defined(ETHR_ORDERED_READ_DEPEND)
3855 return ethr_atomic32_read_band(var, val);
3856 #else
3857 return ethr_atomic32_read_band_rb(var, val);
3858 #endif
3859 }
3860
ethr_atomic32_read_band_rb(ethr_atomic32_t * var,ethr_sint32_t val)3861 ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val)
3862 {
3863 ethr_sint32_t res;
3864 ETHR_ASSERT(!ethr_not_inited__);
3865 ETHR_ASSERT(var);
3866 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3867 res = ethr_atomic32_read_band_rb__(var, val);
3868 #else
3869 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
3870 ETHR_MEMBAR(ETHR_LoadLoad);
3871 #endif
3872 return res;
3873 }
3874
ethr_atomic32_read_band_wb(ethr_atomic32_t * var,ethr_sint32_t val)3875 ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val)
3876 {
3877 ethr_sint32_t res;
3878 ETHR_ASSERT(!ethr_not_inited__);
3879 ETHR_ASSERT(var);
3880 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3881 res = ethr_atomic32_read_band_wb__(var, val);
3882 #else
3883 ETHR_MEMBAR(ETHR_StoreStore);
3884 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
3885 #endif
3886 return res;
3887 }
3888
ethr_atomic32_read_band_acqb(ethr_atomic32_t * var,ethr_sint32_t val)3889 ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
3890 {
3891 ethr_sint32_t res;
3892 ETHR_ASSERT(!ethr_not_inited__);
3893 ETHR_ASSERT(var);
3894 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3895 res = ethr_atomic32_read_band_acqb__(var, val);
3896 #else
3897 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
3898 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3899 #endif
3900 return res;
3901 }
3902
ethr_atomic32_read_band_relb(ethr_atomic32_t * var,ethr_sint32_t val)3903 ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val)
3904 {
3905 ethr_sint32_t res;
3906 ETHR_ASSERT(!ethr_not_inited__);
3907 ETHR_ASSERT(var);
3908 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3909 res = ethr_atomic32_read_band_relb__(var, val);
3910 #else
3911 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3912 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
3913 #endif
3914 return res;
3915 }
3916
ethr_atomic32_read_band_mb(ethr_atomic32_t * var,ethr_sint32_t val)3917 ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val)
3918 {
3919 ethr_sint32_t res;
3920 ETHR_ASSERT(!ethr_not_inited__);
3921 ETHR_ASSERT(var);
3922 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3923 res = ethr_atomic32_read_band_mb__(var, val);
3924 #else
3925 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3926 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
3927 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3928 #endif
3929 return res;
3930 }
3931
3932
3933 /* -- read_bor() -- */
3934
3935
ethr_atomic32_read_bor(ethr_atomic32_t * var,ethr_sint32_t val)3936 ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val)
3937 {
3938 ethr_sint32_t res;
3939 ETHR_ASSERT(!ethr_not_inited__);
3940 ETHR_ASSERT(var);
3941 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3942 res = ethr_atomic32_read_bor__(var, val);
3943 #else
3944 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
3945 #endif
3946 return res;
3947 }
3948
ethr_atomic32_read_bor_ddrb(ethr_atomic32_t * var,ethr_sint32_t val)3949 ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
3950 {
3951 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3952 return ethr_atomic32_read_bor_ddrb__(var, val);
3953 #elif defined(ETHR_ORDERED_READ_DEPEND)
3954 return ethr_atomic32_read_bor(var, val);
3955 #else
3956 return ethr_atomic32_read_bor_rb(var, val);
3957 #endif
3958 }
3959
ethr_atomic32_read_bor_rb(ethr_atomic32_t * var,ethr_sint32_t val)3960 ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val)
3961 {
3962 ethr_sint32_t res;
3963 ETHR_ASSERT(!ethr_not_inited__);
3964 ETHR_ASSERT(var);
3965 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3966 res = ethr_atomic32_read_bor_rb__(var, val);
3967 #else
3968 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
3969 ETHR_MEMBAR(ETHR_LoadLoad);
3970 #endif
3971 return res;
3972 }
3973
ethr_atomic32_read_bor_wb(ethr_atomic32_t * var,ethr_sint32_t val)3974 ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val)
3975 {
3976 ethr_sint32_t res;
3977 ETHR_ASSERT(!ethr_not_inited__);
3978 ETHR_ASSERT(var);
3979 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3980 res = ethr_atomic32_read_bor_wb__(var, val);
3981 #else
3982 ETHR_MEMBAR(ETHR_StoreStore);
3983 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
3984 #endif
3985 return res;
3986 }
3987
ethr_atomic32_read_bor_acqb(ethr_atomic32_t * var,ethr_sint32_t val)3988 ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
3989 {
3990 ethr_sint32_t res;
3991 ETHR_ASSERT(!ethr_not_inited__);
3992 ETHR_ASSERT(var);
3993 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
3994 res = ethr_atomic32_read_bor_acqb__(var, val);
3995 #else
3996 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
3997 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3998 #endif
3999 return res;
4000 }
4001
ethr_atomic32_read_bor_relb(ethr_atomic32_t * var,ethr_sint32_t val)4002 ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val)
4003 {
4004 ethr_sint32_t res;
4005 ETHR_ASSERT(!ethr_not_inited__);
4006 ETHR_ASSERT(var);
4007 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
4008 res = ethr_atomic32_read_bor_relb__(var, val);
4009 #else
4010 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4011 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
4012 #endif
4013 return res;
4014 }
4015
ethr_atomic32_read_bor_mb(ethr_atomic32_t * var,ethr_sint32_t val)4016 ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val)
4017 {
4018 ethr_sint32_t res;
4019 ETHR_ASSERT(!ethr_not_inited__);
4020 ETHR_ASSERT(var);
4021 #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
4022 res = ethr_atomic32_read_bor_mb__(var, val);
4023 #else
4024 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4025 ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
4026 ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4027 #endif
4028 return res;
4029 }
4030
4031
4032
4033 /* --------- Info functions --------- */
4034
4035 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
4036 char *zero_ops[] = {NULL};
4037 #endif
4038
4039
4040 static char *native_su_dw_atomic_ops[] = {
4041 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG
4042 "cmpxchg",
4043 #endif
4044 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB
4045 "cmpxchg_rb",
4046 #endif
4047 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB
4048 "cmpxchg_wb",
4049 #endif
4050 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB
4051 "cmpxchg_acqb",
4052 #endif
4053 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB
4054 "cmpxchg_relb",
4055 #endif
4056 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
4057 "cmpxchg_mb",
4058 #endif
4059 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
4060 "set",
4061 #endif
4062 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB
4063 "set_rb",
4064 #endif
4065 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB
4066 "set_wb",
4067 #endif
4068 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB
4069 "set_acqb",
4070 #endif
4071 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB
4072 "set_relb",
4073 #endif
4074 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB
4075 "set_mb",
4076 #endif
4077 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
4078 "read",
4079 #endif
4080 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB
4081 "read_rb",
4082 #endif
4083 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB
4084 "read_wb",
4085 #endif
4086 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB
4087 "read_acqb",
4088 #endif
4089 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB
4090 "read_relb",
4091 #endif
4092 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB
4093 "read_mb",
4094 #endif
4095 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT
4096 "init",
4097 #endif
4098 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB
4099 "init_rb",
4100 #endif
4101 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB
4102 "init_wb",
4103 #endif
4104 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB
4105 "init_acqb",
4106 #endif
4107 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB
4108 "init_relb",
4109 #endif
4110 #ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB
4111 "init_mb",
4112 #endif
4113 NULL
4114 };
4115
4116 char **
ethr_native_su_dw_atomic_ops(void)4117 ethr_native_su_dw_atomic_ops(void)
4118 {
4119
4120 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
4121 if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
4122 return &zero_ops[0];
4123 #endif
4124 return &native_su_dw_atomic_ops[0];
4125 }
4126
4127
4128 static char *native_dw_atomic_ops[] = {
4129 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG
4130 "cmpxchg",
4131 #endif
4132 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB
4133 "cmpxchg_rb",
4134 #endif
4135 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB
4136 "cmpxchg_wb",
4137 #endif
4138 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB
4139 "cmpxchg_acqb",
4140 #endif
4141 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB
4142 "cmpxchg_relb",
4143 #endif
4144 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
4145 "cmpxchg_mb",
4146 #endif
4147 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET
4148 "set",
4149 #endif
4150 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB
4151 "set_rb",
4152 #endif
4153 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB
4154 "set_wb",
4155 #endif
4156 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB
4157 "set_acqb",
4158 #endif
4159 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB
4160 "set_relb",
4161 #endif
4162 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB
4163 "set_mb",
4164 #endif
4165 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ
4166 "read",
4167 #endif
4168 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB
4169 "read_rb",
4170 #endif
4171 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB
4172 "read_wb",
4173 #endif
4174 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB
4175 "read_acqb",
4176 #endif
4177 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB
4178 "read_relb",
4179 #endif
4180 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB
4181 "read_mb",
4182 #endif
4183 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT
4184 "init",
4185 #endif
4186 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB
4187 "init_rb",
4188 #endif
4189 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB
4190 "init_wb",
4191 #endif
4192 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB
4193 "init_acqb",
4194 #endif
4195 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB
4196 "init_relb",
4197 #endif
4198 #ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB
4199 "init_mb",
4200 #endif
4201 NULL
4202 };
4203
4204 char **
ethr_native_dw_atomic_ops(void)4205 ethr_native_dw_atomic_ops(void)
4206 {
4207
4208 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
4209 if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
4210 return &zero_ops[0];
4211 #endif
4212 return &native_dw_atomic_ops[0];
4213 }
4214
4215
4216 static char *native_atomic64_ops[] = {
4217 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
4218 "cmpxchg",
4219 #endif
4220 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
4221 "cmpxchg_rb",
4222 #endif
4223 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
4224 "cmpxchg_wb",
4225 #endif
4226 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
4227 "cmpxchg_acqb",
4228 #endif
4229 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
4230 "cmpxchg_relb",
4231 #endif
4232 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
4233 "cmpxchg_mb",
4234 #endif
4235 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
4236 "xchg",
4237 #endif
4238 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
4239 "xchg_rb",
4240 #endif
4241 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
4242 "xchg_wb",
4243 #endif
4244 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
4245 "xchg_acqb",
4246 #endif
4247 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
4248 "xchg_relb",
4249 #endif
4250 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
4251 "xchg_mb",
4252 #endif
4253 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
4254 "set",
4255 #endif
4256 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
4257 "set_rb",
4258 #endif
4259 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
4260 "set_wb",
4261 #endif
4262 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
4263 "set_acqb",
4264 #endif
4265 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
4266 "set_relb",
4267 #endif
4268 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
4269 "set_mb",
4270 #endif
4271 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
4272 "init",
4273 #endif
4274 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
4275 "init_rb",
4276 #endif
4277 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
4278 "init_wb",
4279 #endif
4280 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
4281 "init_acqb",
4282 #endif
4283 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
4284 "init_relb",
4285 #endif
4286 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
4287 "init_mb",
4288 #endif
4289 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
4290 "add_return",
4291 #endif
4292 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
4293 "add_return_rb",
4294 #endif
4295 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
4296 "add_return_wb",
4297 #endif
4298 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
4299 "add_return_acqb",
4300 #endif
4301 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
4302 "add_return_relb",
4303 #endif
4304 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
4305 "add_return_mb",
4306 #endif
4307 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
4308 "read",
4309 #endif
4310 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
4311 "read_rb",
4312 #endif
4313 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
4314 "read_wb",
4315 #endif
4316 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
4317 "read_acqb",
4318 #endif
4319 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
4320 "read_relb",
4321 #endif
4322 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
4323 "read_mb",
4324 #endif
4325 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
4326 "inc_return",
4327 #endif
4328 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
4329 "inc_return_rb",
4330 #endif
4331 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
4332 "inc_return_wb",
4333 #endif
4334 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
4335 "inc_return_acqb",
4336 #endif
4337 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
4338 "inc_return_relb",
4339 #endif
4340 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
4341 "inc_return_mb",
4342 #endif
4343 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
4344 "dec_return",
4345 #endif
4346 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
4347 "dec_return_rb",
4348 #endif
4349 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
4350 "dec_return_wb",
4351 #endif
4352 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
4353 "dec_return_acqb",
4354 #endif
4355 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
4356 "dec_return_relb",
4357 #endif
4358 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
4359 "dec_return_mb",
4360 #endif
4361 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
4362 "add",
4363 #endif
4364 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
4365 "add_rb",
4366 #endif
4367 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
4368 "add_wb",
4369 #endif
4370 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
4371 "add_acqb",
4372 #endif
4373 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
4374 "add_relb",
4375 #endif
4376 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
4377 "add_mb",
4378 #endif
4379 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
4380 "inc",
4381 #endif
4382 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
4383 "inc_rb",
4384 #endif
4385 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
4386 "inc_wb",
4387 #endif
4388 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
4389 "inc_acqb",
4390 #endif
4391 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
4392 "inc_relb",
4393 #endif
4394 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
4395 "inc_mb",
4396 #endif
4397 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
4398 "dec",
4399 #endif
4400 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
4401 "dec_rb",
4402 #endif
4403 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
4404 "dec_wb",
4405 #endif
4406 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
4407 "dec_acqb",
4408 #endif
4409 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
4410 "dec_relb",
4411 #endif
4412 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
4413 "dec_mb",
4414 #endif
4415 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
4416 "and_retold",
4417 #endif
4418 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
4419 "and_retold_rb",
4420 #endif
4421 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
4422 "and_retold_wb",
4423 #endif
4424 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
4425 "and_retold_acqb",
4426 #endif
4427 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
4428 "and_retold_relb",
4429 #endif
4430 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
4431 "and_retold_mb",
4432 #endif
4433 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
4434 "or_retold",
4435 #endif
4436 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
4437 "or_retold_rb",
4438 #endif
4439 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
4440 "or_retold_wb",
4441 #endif
4442 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
4443 "or_retold_acqb",
4444 #endif
4445 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
4446 "or_retold_relb",
4447 #endif
4448 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
4449 "or_retold_mb",
4450 #endif
4451 NULL
4452 };
4453
4454 char **
ethr_native_atomic64_ops(void)4455 ethr_native_atomic64_ops(void)
4456 {
4457
4458 return &native_atomic64_ops[0];
4459 }
4460
4461
4462 static char *native_atomic32_ops[] = {
4463 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
4464 "cmpxchg",
4465 #endif
4466 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
4467 "cmpxchg_rb",
4468 #endif
4469 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
4470 "cmpxchg_wb",
4471 #endif
4472 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
4473 "cmpxchg_acqb",
4474 #endif
4475 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
4476 "cmpxchg_relb",
4477 #endif
4478 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
4479 "cmpxchg_mb",
4480 #endif
4481 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
4482 "xchg",
4483 #endif
4484 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
4485 "xchg_rb",
4486 #endif
4487 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
4488 "xchg_wb",
4489 #endif
4490 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
4491 "xchg_acqb",
4492 #endif
4493 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
4494 "xchg_relb",
4495 #endif
4496 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
4497 "xchg_mb",
4498 #endif
4499 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
4500 "set",
4501 #endif
4502 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
4503 "set_rb",
4504 #endif
4505 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
4506 "set_wb",
4507 #endif
4508 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
4509 "set_acqb",
4510 #endif
4511 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
4512 "set_relb",
4513 #endif
4514 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
4515 "set_mb",
4516 #endif
4517 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
4518 "init",
4519 #endif
4520 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
4521 "init_rb",
4522 #endif
4523 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
4524 "init_wb",
4525 #endif
4526 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
4527 "init_acqb",
4528 #endif
4529 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
4530 "init_relb",
4531 #endif
4532 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
4533 "init_mb",
4534 #endif
4535 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
4536 "add_return",
4537 #endif
4538 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
4539 "add_return_rb",
4540 #endif
4541 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
4542 "add_return_wb",
4543 #endif
4544 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
4545 "add_return_acqb",
4546 #endif
4547 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
4548 "add_return_relb",
4549 #endif
4550 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
4551 "add_return_mb",
4552 #endif
4553 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
4554 "read",
4555 #endif
4556 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
4557 "read_rb",
4558 #endif
4559 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
4560 "read_wb",
4561 #endif
4562 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
4563 "read_acqb",
4564 #endif
4565 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
4566 "read_relb",
4567 #endif
4568 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
4569 "read_mb",
4570 #endif
4571 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
4572 "inc_return",
4573 #endif
4574 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
4575 "inc_return_rb",
4576 #endif
4577 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
4578 "inc_return_wb",
4579 #endif
4580 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
4581 "inc_return_acqb",
4582 #endif
4583 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
4584 "inc_return_relb",
4585 #endif
4586 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
4587 "inc_return_mb",
4588 #endif
4589 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
4590 "dec_return",
4591 #endif
4592 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
4593 "dec_return_rb",
4594 #endif
4595 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
4596 "dec_return_wb",
4597 #endif
4598 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
4599 "dec_return_acqb",
4600 #endif
4601 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
4602 "dec_return_relb",
4603 #endif
4604 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
4605 "dec_return_mb",
4606 #endif
4607 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
4608 "add",
4609 #endif
4610 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
4611 "add_rb",
4612 #endif
4613 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
4614 "add_wb",
4615 #endif
4616 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
4617 "add_acqb",
4618 #endif
4619 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
4620 "add_relb",
4621 #endif
4622 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
4623 "add_mb",
4624 #endif
4625 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
4626 "inc",
4627 #endif
4628 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
4629 "inc_rb",
4630 #endif
4631 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
4632 "inc_wb",
4633 #endif
4634 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
4635 "inc_acqb",
4636 #endif
4637 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
4638 "inc_relb",
4639 #endif
4640 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
4641 "inc_mb",
4642 #endif
4643 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
4644 "dec",
4645 #endif
4646 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
4647 "dec_rb",
4648 #endif
4649 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
4650 "dec_wb",
4651 #endif
4652 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
4653 "dec_acqb",
4654 #endif
4655 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
4656 "dec_relb",
4657 #endif
4658 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
4659 "dec_mb",
4660 #endif
4661 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
4662 "and_retold",
4663 #endif
4664 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
4665 "and_retold_rb",
4666 #endif
4667 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
4668 "and_retold_wb",
4669 #endif
4670 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
4671 "and_retold_acqb",
4672 #endif
4673 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
4674 "and_retold_relb",
4675 #endif
4676 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
4677 "and_retold_mb",
4678 #endif
4679 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
4680 "or_retold",
4681 #endif
4682 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
4683 "or_retold_rb",
4684 #endif
4685 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
4686 "or_retold_wb",
4687 #endif
4688 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
4689 "or_retold_acqb",
4690 #endif
4691 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
4692 "or_retold_relb",
4693 #endif
4694 #ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
4695 "or_retold_mb",
4696 #endif
4697 NULL
4698 };
4699
4700 char **
ethr_native_atomic32_ops(void)4701 ethr_native_atomic32_ops(void)
4702 {
4703
4704 return &native_atomic32_ops[0];
4705 }
4706