1 /*
2  * --------------- DO NOT EDIT THIS FILE! ---------------
3  * This file was automatically generated by the
4  * $ERL_TOP/erts/lib_src/utils/make_atomics_api script.
5  * If you need to make changes, edit the script and
6  * regenerate this file.
7  * --------------- DO NOT EDIT THIS FILE! ---------------
8  */
9 
10 /*
11  * %CopyrightBegin%
12  *
13  * Copyright Ericsson AB 2011-2016. All Rights Reserved.
14  *
15  * Licensed under the Apache License, Version 2.0 (the "License");
16  * you may not use this file except in compliance with the License.
17  * You may obtain a copy of the License at
18  *
19  *     http://www.apache.org/licenses/LICENSE-2.0
20  *
21  * Unless required by applicable law or agreed to in writing, software
22  * distributed under the License is distributed on an "AS IS" BASIS,
23  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
24  * See the License for the specific language governing permissions and
25  * limitations under the License.
26  *
27  * %CopyrightEnd%
28  */
29 
30 /*
31  * Description: The ethread atomics API
32  * Author: Rickard Green
33  */
34 
35 /*
36  * This file maps native atomic implementations to ethread
37  * API atomics. If no native atomic implementation
38  * is available, a less efficient fallback is used instead.
39  * The API consists of 32-bit size, word size (pointer size),
40  * and double word size atomics.
41  *
42  * The following atomic operations are implemented for
43  * 32-bit size, and word size atomics:
44  * - cmpxchg
45  * - xchg
46  * - set
47  * - init
48  * - add_read
49  * - read
50  * - inc_read
51  * - dec_read
52  * - add
53  * - inc
54  * - dec
55  * - read_band
56  * - read_bor
57  *
58  * The following atomic operations are implemented for
59  * double word size atomics:
60  * - cmpxchg
61  * - set
62  * - read
63  * - init
64  *
65  * Appart from a function implementing the atomic operation
66  * with unspecified memory barrier semantics, there are
67  * functions implementing each operation with the following
68  * implied memory barrier semantics:
69  * - mb   - Full memory barrier. Orders both loads, and
70  *          stores before, and after the atomic operation.
71  *          No load or store is allowed to be reordered
72  *          over the atomic operation.
73  * - relb - Release barrier. Orders both loads, and
74  *          stores appearing *before* the atomic
75  *          operation. These are not allowed to be
76  *          reordered over the atomic operation.
77  * - acqb - Acquire barrier. Orders both loads, and stores
78  *          appearing *after* the atomic operation. These
79  *          are not allowed to be reordered over the
80  *          atomic operation.
81  * - wb   - Write barrier. Orders *only* stores. These are
82  *          not allowed to be reordered over the barrier.
83  *          Store in atomic operation is ordered *after*
84  *          the barrier.
85  * - rb   - Read barrier. Orders *only* loads. These are
86  *          not allowed to be reordered over the barrier.
87  *          Load in atomic operation is ordered *before*
88  *          the barrier.
89  * - ddrb - Data dependency read barrier. Orders *only*
90  *          loads according to data dependency across the
91  *          barrier. Load in atomic operation is ordered
92  *          before the barrier.
93  *
94  * We implement all of these operation/barrier
95  * combinations, regardless of whether they are useful
96  * or not (some of them are useless).
97  *
98  * Double word size atomic functions are on the followning
99  * form:
100  *   ethr_dw_atomic_<OP>[_<BARRIER>]
101  *
102  * Word size atomic functions are on the followning
103  * form:
104  *   ethr_atomic_<OP>[_<BARRIER>]
105  *
106  * 32-bit size atomic functions are on the followning
107  * form:
108  *   ethr_atomic32_<OP>[_<BARRIER>]
109  *
110  * Apart from the operation/barrier functions
111  * described above also 'addr' functions are implemented
112  * which return the actual memory address used of the
113  * atomic variable. The 'addr' functions have no barrier
114  * versions.
115  *
116  * The native atomic implementation does not need to
117  * implement all operation/barrier combinations.
118  * Functions that have no native implementation will be
119  * constructed from existing native functionality. These
120  * functions will perform the wanted operation and will
121  * produce sufficient memory barriers, but may
122  * in some cases be less efficient than pure native
123  * versions.
124  *
125  * When we create ethread API operation/barrier functions by
126  * adding barriers before and after native operations it is
127  * assumed that:
128  * - A native read operation begins, and ends with a load.
129  * - A native set operation begins, and ends with a store.
130  * - An init operation begins with either a load, or a store,
131  *   and ends with either a load, or a store.
132  * - All other operations begins with a load, and ends with
133  *   either a load, or a store.
134  *
135  * This is the minimum functionality that a native
136  * implementation needs to provide:
137  *
138  * - Functions that need to be implemented:
139  *
140  *   - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
141  *   - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
142  *     (at least one cmpxchg of optional barrier)
143  *
144  * - Macros that needs to be defined:
145  *
146  *   A macro informing about the presence of the native
147  *   implementation:
148  *
149  *   - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
150  *
151  *   A macro naming (a string constant) the implementation:
152  *
153  *   - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
154  *
155  *   Each implemented native atomic function has to
156  *   be accompanied by a defined macro on the following
157  *   form informing about its presence:
158  *
159  *   - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
160  *
161  *   A (sparc-v9 style) membar macro:
162  *
163  *   - ETHR_MEMBAR(B)
164  *
165  *     Which takes a combination of the following macros
166  *     or:ed (using |) together:
167  *
168  *     - ETHR_LoadLoad
169  *     - ETHR_LoadStore
170  *     - ETHR_StoreLoad
171  *     - ETHR_StoreStore
172  *
173  */
174 
175 #ifndef ETHR_ATOMICS_H__
176 #define ETHR_ATOMICS_H__
177 
178 #undef ETHR_AMC_FALLBACK__
179 #undef ETHR_AMC_NO_ATMCS__
180 #undef ETHR_AMC_ATMC_T__
181 #undef ETHR_AMC_ATMC_FUNC__
182 
183 /* -- 32-bit atomics -- */
184 
185 #undef ETHR_NAINT32_T__
186 #undef ETHR_NATMC32_FUNC__
187 #undef ETHR_NATMC32_ADDR_FUNC__
188 #undef ETHR_NATMC32_BITS__
189 #if defined(ETHR_HAVE_NATIVE_ATOMIC32)
190 #    define ETHR_NEED_NATMC32_ADDR
191 #  define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr
192 typedef ethr_native_atomic32_t ethr_atomic32_t;
193 #  define ETHR_NAINT32_T__ ethr_sint32_t
194 #  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X
195 #  define ETHR_NATMC32_BITS__ 32
196 #elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
197 #  define ETHR_NEED_NATMC64_ADDR
198 #ifdef ETHR_BIGENDIAN
199 #  define ETHR_NATMC32_ADDR_FUNC__(VAR) \
200   (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
201 #else
202 #  define ETHR_NATMC32_ADDR_FUNC__(VAR) \
203   ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
204 #endif
205 typedef ethr_native_atomic64_t ethr_atomic32_t;
206 #  define ETHR_NAINT32_T__ ethr_sint64_t
207 #  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X
208 #  define ETHR_NATMC32_BITS__ 64
209 #else
210 /*
211  * No native atomics usable for 32-bits atomics :(
212  * Use fallback...
213  */
214 typedef ethr_sint32_t ethr_atomic32_t;
215 #endif
216 
217 #undef ETHR_ATMC32_INLINE__
218 #ifdef ETHR_NATMC32_BITS__
219 #  ifdef ETHR_TRY_INLINE_FUNCS
220 #    define ETHR_ATMC32_INLINE__
221 #  endif
222 #  define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS
223 #endif
224 
225 #if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
226 #  define ETHR_NEED_ATMC32_PROTOTYPES__
227 #endif
228 
229 #ifndef ETHR_INLINE_ATMC32_FUNC_NAME_
230 #  define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X
231 #endif
232 
233 #undef ETHR_ATMC32_FUNC__
234 #define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
235 
236 
237 /* -- Word size atomics -- */
238 
239 #undef ETHR_NEED_NATMC32_ADDR
240 #undef ETHR_NEED_NATMC64_ADDR
241 
242 #undef ETHR_NAINT_T__
243 #undef ETHR_NATMC_FUNC__
244 #undef ETHR_NATMC_ADDR_FUNC__
245 #undef ETHR_NATMC_BITS__
246 #if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
247 #  ifndef ETHR_NEED_NATMC64_ADDR
248 #    define ETHR_NEED_NATMC64_ADDR
249 #  endif
250 #  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr
251 typedef ethr_native_atomic64_t ethr_atomic_t;
252 #  define ETHR_NAINT_T__ ethr_sint64_t
253 #  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
254 #  define ETHR_NATMC_BITS__ 64
255 #elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32)
256 #  ifndef ETHR_NEED_NATMC64_ADDR
257 #    define ETHR_NEED_NATMC32_ADDR
258 #  endif
259 #  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr
260 typedef ethr_native_atomic32_t ethr_atomic_t;
261 #  define ETHR_NAINT_T__ ethr_sint32_t
262 #  define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
263 #  define ETHR_NATMC_BITS__ 32
264 #elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
265 #  ifndef ETHR_NEED_NATMC64_ADDR
266 #    define ETHR_NEED_NATMC64_ADDR
267 #  endif
268 #ifdef ETHR_BIGENDIAN
269 #  define ETHR_NATMC_ADDR_FUNC__(VAR) \
270   (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
271 #else
272 #  define ETHR_NATMC_ADDR_FUNC__(VAR) \
273   ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
274 #endif
275 typedef ethr_native_atomic64_t ethr_atomic_t;
276 #  define ETHR_NATMC_T__ ethr_native_atomic64_t
277 #  define ETHR_NAINT_T__ ethr_sint64_t
278 #  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
279 #  define ETHR_NATMC_BITS__ 64
280 #else
281 /*
282  * No native atomics usable for pointer size atomics :(
283  * Use fallback...
284  */
285 
286 #  if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
287 #    define ETHR_AMC_FALLBACK__
288 #    define ETHR_AMC_NO_ATMCS__ 2
289 #    define ETHR_AMC_SINT_T__ ethr_sint32_t
290 #    define ETHR_AMC_ATMC_T__ ethr_atomic32_t
291 #    define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
292 typedef struct {
293     ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
294 } ethr_amc_t;
295 typedef struct {
296     ethr_amc_t amc;
297     ethr_sint_t sint;
298 } ethr_atomic_t;
299 #  else /* locked fallback */
300 typedef ethr_sint_t ethr_atomic_t;
301 #  endif
302 #endif
303 
304 #undef ETHR_ATMC_INLINE__
305 #ifdef ETHR_NATMC_BITS__
306 #  ifdef ETHR_TRY_INLINE_FUNCS
307 #    define ETHR_ATMC_INLINE__
308 #  endif
309 #  define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS
310 #endif
311 
312 #if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
313 #  define ETHR_NEED_ATMC_PROTOTYPES__
314 #endif
315 
316 #ifndef ETHR_INLINE_ATMC_FUNC_NAME_
317 #  define ETHR_INLINE_ATMC_FUNC_NAME_(X) X
318 #endif
319 
320 #undef ETHR_ATMC_FUNC__
321 #define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
322 
323 /* -- Double word atomics -- */
324 
325 #undef ETHR_SU_DW_NAINT_T__
326 #undef ETHR_SU_DW_NATMC_FUNC__
327 #undef ETHR_SU_DW_NATMC_ADDR_FUNC__
328 #undef ETHR_DW_NATMC_FUNC__
329 #undef ETHR_DW_NATMC_ADDR_FUNC__
330 #undef ETHR_DW_NATMC_BITS__
331 #if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
332 #  define ETHR_NEED_DW_NATMC_ADDR
333 #  define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr
334 #  define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t
335 #  define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X
336 #  define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X
337 #  if ETHR_SIZEOF_PTR == 8
338 #    define ETHR_DW_NATMC_BITS__ 128
339 #  elif ETHR_SIZEOF_PTR == 4
340 #    define ETHR_DW_NATMC_BITS__ 64
341 #  else
342 #    error "Word size not supported"
343 #  endif
344 #  ifdef ETHR_NATIVE_SU_DW_SINT_T
345 #    define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T
346 #  endif
347 #elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
348 #  define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
349 #  ifndef ETHR_NEED_NATMC64_ADDR
350 #    define ETHR_NEED_NATMC64_ADDR
351 #  endif
352 #  define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \
353   ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR)))
354 #  define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t
355 #  define ETHR_SU_DW_NAINT_T__ ethr_sint64_t
356 #  define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
357 #  define ETHR_DW_NATMC_BITS__ 64
358 #endif
359 
360 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
361 #define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X ## _fallback__
362 #else
363 #define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X
364 #endif
365 
366 #if !defined(ETHR_DW_NATMC_BITS__) || defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
367 #  define ETHR_NEED_DW_FALLBACK__
368 #endif
369 
370 #if defined(ETHR_NEED_DW_FALLBACK__)
371 /*
372  * No native atomics usable for double word atomics :(
373  * Use fallback...
374  */
375 
376 #  ifndef ETHR_AMC_FALLBACK__
377 #    if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
378 #      define ETHR_AMC_FALLBACK__
379 #      define ETHR_AMC_NO_ATMCS__ 1
380 #      define ETHR_AMC_SINT_T__ ethr_sint_t
381 #      define ETHR_AMC_ATMC_T__ ethr_atomic_t
382 #      define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
383 #    elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
384 #      define ETHR_AMC_FALLBACK__
385 #      define ETHR_AMC_NO_ATMCS__ 2
386 #      define ETHR_AMC_SINT_T__ ethr_sint32_t
387 #      define ETHR_AMC_ATMC_T__ ethr_atomic32_t
388 #      define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
389 #    endif
390 #    ifdef ETHR_AMC_FALLBACK__
391 typedef struct {
392     ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
393 } ethr_amc_t;
394 #    endif
395 #  endif
396 
397 typedef struct {
398 #ifdef ETHR_AMC_FALLBACK__
399     ethr_amc_t amc;
400 #endif
401     ethr_sint_t sint[2];
402 } ethr_dw_atomic_fallback_t;
403 
404 #endif
405 
406 typedef union {
407 #ifdef ETHR_NATIVE_DW_ATOMIC_T__
408     ETHR_NATIVE_DW_ATOMIC_T__ native;
409 #endif
410 #ifdef ETHR_NEED_DW_FALLBACK__
411     ethr_dw_atomic_fallback_t fallback;
412 #endif
413     ethr_sint_t sint[2];
414 } ethr_dw_atomic_t;
415 
416 typedef union {
417 #ifdef ETHR_SU_DW_NAINT_T__
418     ETHR_SU_DW_NAINT_T__ dw_sint;
419 #endif
420     ethr_sint_t sint[2];
421 } ethr_dw_sint_t;
422 
423 #ifdef ETHR_BIGENDIAN
424 #  define ETHR_DW_SINT_LOW_WORD 1
425 #  define ETHR_DW_SINT_HIGH_WORD 0
426 #else
427 #  define ETHR_DW_SINT_LOW_WORD 0
428 #  define ETHR_DW_SINT_HIGH_WORD 1
429 #endif
430 
431 #undef ETHR_DW_ATMC_INLINE__
432 #ifdef ETHR_DW_NATMC_BITS__
433 #  ifdef ETHR_TRY_INLINE_FUNCS
434 #    define ETHR_ATMC32_INLINE__
435 #  endif
436 #  define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS
437 #endif
438 
439 #if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
440 #  define ETHR_NEED_DW_ATMC_PROTOTYPES__
441 #endif
442 
443 #ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_
444 #  define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X
445 #endif
446 
447 #undef ETHR_DW_ATMC_FUNC__
448 #define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X)
449 
450 #if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__)
451 int ethr_have_native_dw_atomic(void);
452 #endif
453 #if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
454 static ETHR_INLINE int
ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)455 ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void)
456 {
457 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
458     return ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__;
459 #elif defined(ETHR_DW_NATMC_BITS__)
460     return 1;
461 #else
462     return 0;
463 #endif
464 }
465 #endif
466 
467 /* -- Misc -- */
468 
469 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
470 /*
471  * Unusual values are used by read() fallbacks implemented via cmpxchg().
472  * We want to use an unusual value in hope that it is more efficient
473  * not to match the value in memory.
474  *
475  * - Negative integer values are probably more unusual.
476  * - Very large absolute integer values are probably more unusual.
477  * - Odd pointers are probably more unusual (only char pointers can be odd).
478  */
479 #  define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181)
480 #  if ETHR_SIZEOF_PTR == 4
481 #    define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__)
482 #  elif ETHR_SIZEOF_PTR == 8
483 #    define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L)
484 #  else
485 #    error "Word size not supported"
486 #  endif
487 #  if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR)
488 #    error "No ethr_native_dw_atomic_addr() available"
489 #  endif
490 #  if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR)
491 #    error "No ethr_native_atomic32_addr() available"
492 #  endif
493 #  if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR)
494 #    error "No ethr_native_atomic64_addr() available"
495 #  endif
496 #endif
497 
498 #if defined(__GNUC__)
499 #  ifndef ETHR_COMPILER_BARRIER
500 #    define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
501 #  endif
502 #elif defined(ETHR_WIN32_THREADS)
503 #  ifndef ETHR_COMPILER_BARRIER
504 #    include <intrin.h>
505 #    pragma intrinsic(_ReadWriteBarrier)
506 #    define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
507 #  endif
508 #endif
509 
510 void ethr_compiler_barrier_fallback(void);
511 #ifndef ETHR_COMPILER_BARRIER
512 #  define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback()
513 #endif
514 
515 int ethr_init_atomics(void);
516 
517 /* info */
518 char **ethr_native_atomic32_ops(void);
519 char **ethr_native_atomic64_ops(void);
520 char **ethr_native_dw_atomic_ops(void);
521 char **ethr_native_su_dw_atomic_ops(void);
522 
523 #if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__)
524 /*
525  * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only,
526  * i.e. when no native atomic implementation exist and only our lock
527  * based atomic fallback is used, a noop is sufficient.
528  */
529 #  undef ETHR_MEMORY_BARRIER
530 #  undef ETHR_WRITE_MEMORY_BARRIER
531 #  undef ETHR_READ_MEMORY_BARRIER
532 #  undef ETHR_READ_DEPEND_MEMORY_BARRIER
533 #  undef ETHR_MEMBAR
534 #  define ETHR_MEMBAR(B) do { } while (0)
535 #endif
536 
537 #ifndef ETHR_MEMBAR
538 #  error "No ETHR_MEMBAR defined"
539 #endif
540 
541 #define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore)
542 #define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore)
543 #define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad)
544 #ifdef ETHR_READ_DEPEND_MEMORY_BARRIER
545 #  undef ETHR_ORDERED_READ_DEPEND
546 #else
547 #  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
548 #  define ETHR_ORDERED_READ_DEPEND
549 #endif
550 
551 
552 /* ---------- Double word size atomic implementation ---------- */
553 
554 
555 #ifdef ETHR_NEED_DW_ATMC_PROTOTYPES__
556 ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var);
557 int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
558 int ethr_dw_atomic_cmpxchg_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
559 int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
560 int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
561 int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
562 int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
563 int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
564 void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
565 void ethr_dw_atomic_set_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
566 void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
567 void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
568 void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
569 void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
570 void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
571 void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
572 void ethr_dw_atomic_read_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
573 void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
574 void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
575 void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
576 void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
577 void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
578 void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
579 void ethr_dw_atomic_init_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
580 void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
581 void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
582 void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
583 void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
584 void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
585 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
586 ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var);
587 int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
588 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
589 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
590 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
591 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
592 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
593 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
594 void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
595 void ETHR_DW_ATOMIC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
596 void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
597 void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
598 void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
599 void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
600 void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
601 void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
602 void ETHR_DW_ATOMIC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
603 void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
604 void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
605 void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
606 void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
607 void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
608 void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
609 void ETHR_DW_ATOMIC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
610 void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
611 void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
612 void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
613 void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
614 void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
615 #endif
616 #endif /* ETHR_NEED_DW_ATMC_PROTOTYPES__ */
617 
618 #if (defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) \
619      && (defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
620 
621 #if !defined(ETHR_DW_NATMC_BITS__)
622 #  error "Missing native atomic implementation"
623 #elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC)  || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
624 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG
625 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG
626 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG
627 #    define ETHR_HAVE_DW_NATMC_CMPXCHG 1
628 #  endif
629 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG
630 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1
631 #  endif
632 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB
633 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB
634 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB
635 #    define ETHR_HAVE_DW_NATMC_CMPXCHG_RB 1
636 #  endif
637 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB
638 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1
639 #  endif
640 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB
641 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB
642 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB
643 #    define ETHR_HAVE_DW_NATMC_CMPXCHG_WB 1
644 #  endif
645 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB
646 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1
647 #  endif
648 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB
649 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB
650 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB
651 #    define ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB 1
652 #  endif
653 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB
654 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1
655 #  endif
656 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB
657 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB
658 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB
659 #    define ETHR_HAVE_DW_NATMC_CMPXCHG_RELB 1
660 #  endif
661 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB
662 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1
663 #  endif
664 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB
665 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB
666 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
667 #    define ETHR_HAVE_DW_NATMC_CMPXCHG_MB 1
668 #  endif
669 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
670 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1
671 #  endif
672 #  undef ETHR_HAVE_SU_DW_NATMC_SET
673 #  undef ETHR_HAVE_DW_NATMC_SET
674 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET
675 #    define ETHR_HAVE_DW_NATMC_SET 1
676 #  endif
677 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
678 #    define ETHR_HAVE_SU_DW_NATMC_SET 1
679 #  endif
680 #  undef ETHR_HAVE_SU_DW_NATMC_SET_RB
681 #  undef ETHR_HAVE_DW_NATMC_SET_RB
682 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB
683 #    define ETHR_HAVE_DW_NATMC_SET_RB 1
684 #  endif
685 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB
686 #    define ETHR_HAVE_SU_DW_NATMC_SET_RB 1
687 #  endif
688 #  undef ETHR_HAVE_SU_DW_NATMC_SET_WB
689 #  undef ETHR_HAVE_DW_NATMC_SET_WB
690 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB
691 #    define ETHR_HAVE_DW_NATMC_SET_WB 1
692 #  endif
693 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB
694 #    define ETHR_HAVE_SU_DW_NATMC_SET_WB 1
695 #  endif
696 #  undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB
697 #  undef ETHR_HAVE_DW_NATMC_SET_ACQB
698 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB
699 #    define ETHR_HAVE_DW_NATMC_SET_ACQB 1
700 #  endif
701 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB
702 #    define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1
703 #  endif
704 #  undef ETHR_HAVE_SU_DW_NATMC_SET_RELB
705 #  undef ETHR_HAVE_DW_NATMC_SET_RELB
706 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB
707 #    define ETHR_HAVE_DW_NATMC_SET_RELB 1
708 #  endif
709 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB
710 #    define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1
711 #  endif
712 #  undef ETHR_HAVE_SU_DW_NATMC_SET_MB
713 #  undef ETHR_HAVE_DW_NATMC_SET_MB
714 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB
715 #    define ETHR_HAVE_DW_NATMC_SET_MB 1
716 #  endif
717 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB
718 #    define ETHR_HAVE_SU_DW_NATMC_SET_MB 1
719 #  endif
720 #  undef ETHR_HAVE_SU_DW_NATMC_READ
721 #  undef ETHR_HAVE_DW_NATMC_READ
722 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ
723 #    define ETHR_HAVE_DW_NATMC_READ 1
724 #  endif
725 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
726 #    define ETHR_HAVE_SU_DW_NATMC_READ 1
727 #  endif
728 #  undef ETHR_HAVE_SU_DW_NATMC_READ_RB
729 #  undef ETHR_HAVE_DW_NATMC_READ_RB
730 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB
731 #    define ETHR_HAVE_DW_NATMC_READ_RB 1
732 #  endif
733 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB
734 #    define ETHR_HAVE_SU_DW_NATMC_READ_RB 1
735 #  endif
736 #  undef ETHR_HAVE_SU_DW_NATMC_READ_WB
737 #  undef ETHR_HAVE_DW_NATMC_READ_WB
738 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB
739 #    define ETHR_HAVE_DW_NATMC_READ_WB 1
740 #  endif
741 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB
742 #    define ETHR_HAVE_SU_DW_NATMC_READ_WB 1
743 #  endif
744 #  undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB
745 #  undef ETHR_HAVE_DW_NATMC_READ_ACQB
746 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB
747 #    define ETHR_HAVE_DW_NATMC_READ_ACQB 1
748 #  endif
749 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB
750 #    define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1
751 #  endif
752 #  undef ETHR_HAVE_SU_DW_NATMC_READ_RELB
753 #  undef ETHR_HAVE_DW_NATMC_READ_RELB
754 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB
755 #    define ETHR_HAVE_DW_NATMC_READ_RELB 1
756 #  endif
757 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB
758 #    define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1
759 #  endif
760 #  undef ETHR_HAVE_SU_DW_NATMC_READ_MB
761 #  undef ETHR_HAVE_DW_NATMC_READ_MB
762 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB
763 #    define ETHR_HAVE_DW_NATMC_READ_MB 1
764 #  endif
765 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB
766 #    define ETHR_HAVE_SU_DW_NATMC_READ_MB 1
767 #  endif
768 #  undef ETHR_HAVE_SU_DW_NATMC_INIT
769 #  undef ETHR_HAVE_DW_NATMC_INIT
770 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT
771 #    define ETHR_HAVE_DW_NATMC_INIT 1
772 #  endif
773 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT
774 #    define ETHR_HAVE_SU_DW_NATMC_INIT 1
775 #  endif
776 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_RB
777 #  undef ETHR_HAVE_DW_NATMC_INIT_RB
778 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB
779 #    define ETHR_HAVE_DW_NATMC_INIT_RB 1
780 #  endif
781 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB
782 #    define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1
783 #  endif
784 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_WB
785 #  undef ETHR_HAVE_DW_NATMC_INIT_WB
786 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB
787 #    define ETHR_HAVE_DW_NATMC_INIT_WB 1
788 #  endif
789 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB
790 #    define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1
791 #  endif
792 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB
793 #  undef ETHR_HAVE_DW_NATMC_INIT_ACQB
794 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB
795 #    define ETHR_HAVE_DW_NATMC_INIT_ACQB 1
796 #  endif
797 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB
798 #    define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1
799 #  endif
800 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB
801 #  undef ETHR_HAVE_DW_NATMC_INIT_RELB
802 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB
803 #    define ETHR_HAVE_DW_NATMC_INIT_RELB 1
804 #  endif
805 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB
806 #    define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1
807 #  endif
808 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_MB
809 #  undef ETHR_HAVE_DW_NATMC_INIT_MB
810 #  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB
811 #    define ETHR_HAVE_DW_NATMC_INIT_MB 1
812 #  endif
813 #  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB
814 #    define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1
815 #  endif
816 #elif ETHR_DW_NATMC_BITS__ == 64
817 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG
818 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG
819 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
820 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1
821 #  endif
822 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB
823 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB
824 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
825 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1
826 #  endif
827 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB
828 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB
829 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
830 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1
831 #  endif
832 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB
833 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB
834 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
835 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1
836 #  endif
837 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB
838 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB
839 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
840 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1
841 #  endif
842 #  undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB
843 #  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB
844 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
845 #    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1
846 #  endif
847 #  undef ETHR_HAVE_DW_NATMC_SET
848 #  undef ETHR_HAVE_SU_DW_NATMC_SET
849 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
850 #    define ETHR_HAVE_SU_DW_NATMC_SET 1
851 #  endif
852 #  undef ETHR_HAVE_DW_NATMC_SET_RB
853 #  undef ETHR_HAVE_SU_DW_NATMC_SET_RB
854 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
855 #    define ETHR_HAVE_SU_DW_NATMC_SET_RB 1
856 #  endif
857 #  undef ETHR_HAVE_DW_NATMC_SET_WB
858 #  undef ETHR_HAVE_SU_DW_NATMC_SET_WB
859 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
860 #    define ETHR_HAVE_SU_DW_NATMC_SET_WB 1
861 #  endif
862 #  undef ETHR_HAVE_DW_NATMC_SET_ACQB
863 #  undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB
864 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
865 #    define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1
866 #  endif
867 #  undef ETHR_HAVE_DW_NATMC_SET_RELB
868 #  undef ETHR_HAVE_SU_DW_NATMC_SET_RELB
869 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
870 #    define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1
871 #  endif
872 #  undef ETHR_HAVE_DW_NATMC_SET_MB
873 #  undef ETHR_HAVE_SU_DW_NATMC_SET_MB
874 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
875 #    define ETHR_HAVE_SU_DW_NATMC_SET_MB 1
876 #  endif
877 #  undef ETHR_HAVE_DW_NATMC_READ
878 #  undef ETHR_HAVE_SU_DW_NATMC_READ
879 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
880 #    define ETHR_HAVE_SU_DW_NATMC_READ 1
881 #  endif
882 #  undef ETHR_HAVE_DW_NATMC_READ_RB
883 #  undef ETHR_HAVE_SU_DW_NATMC_READ_RB
884 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
885 #    define ETHR_HAVE_SU_DW_NATMC_READ_RB 1
886 #  endif
887 #  undef ETHR_HAVE_DW_NATMC_READ_WB
888 #  undef ETHR_HAVE_SU_DW_NATMC_READ_WB
889 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
890 #    define ETHR_HAVE_SU_DW_NATMC_READ_WB 1
891 #  endif
892 #  undef ETHR_HAVE_DW_NATMC_READ_ACQB
893 #  undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB
894 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
895 #    define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1
896 #  endif
897 #  undef ETHR_HAVE_DW_NATMC_READ_RELB
898 #  undef ETHR_HAVE_SU_DW_NATMC_READ_RELB
899 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
900 #    define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1
901 #  endif
902 #  undef ETHR_HAVE_DW_NATMC_READ_MB
903 #  undef ETHR_HAVE_SU_DW_NATMC_READ_MB
904 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
905 #    define ETHR_HAVE_SU_DW_NATMC_READ_MB 1
906 #  endif
907 #  undef ETHR_HAVE_DW_NATMC_INIT
908 #  undef ETHR_HAVE_SU_DW_NATMC_INIT
909 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
910 #    define ETHR_HAVE_SU_DW_NATMC_INIT 1
911 #  endif
912 #  undef ETHR_HAVE_DW_NATMC_INIT_RB
913 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_RB
914 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
915 #    define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1
916 #  endif
917 #  undef ETHR_HAVE_DW_NATMC_INIT_WB
918 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_WB
919 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
920 #    define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1
921 #  endif
922 #  undef ETHR_HAVE_DW_NATMC_INIT_ACQB
923 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB
924 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
925 #    define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1
926 #  endif
927 #  undef ETHR_HAVE_DW_NATMC_INIT_RELB
928 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB
929 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
930 #    define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1
931 #  endif
932 #  undef ETHR_HAVE_DW_NATMC_INIT_MB
933 #  undef ETHR_HAVE_SU_DW_NATMC_INIT_MB
934 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
935 #    define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1
936 #  endif
937 #else
938 #  error "Invalid native atomic size"
939 #endif
940 
941 
942 #if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)
943 
944 #if (!defined(ETHR_HAVE_DW_NATMC_CMPXCHG) \
945      && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) \
946      && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) \
947      && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) \
948      && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) \
949      && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB))
950 #  error "No native cmpxchg() op available"
951 #endif
952 
953 
954 /*
955  * Read op used together with cmpxchg() fallback when no native op present.
956  */
957 #if defined(ETHR_HAVE_DW_NATMC_READ)
958 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
959   ETHR_DW_NATMC_FUNC__(read)(VAR, VAL)
960 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
961 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
962   VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(VAR)
963 #elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
964 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
965   ETHR_DW_NATMC_FUNC__(read_rb)(VAR, VAL)
966 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
967 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
968   VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR)
969 #elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
970 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
971   ETHR_DW_NATMC_FUNC__(read_wb)(VAR, VAL)
972 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
973 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
974   VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR)
975 #elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
976 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
977   ETHR_DW_NATMC_FUNC__(read_acqb)(VAR, VAL)
978 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
979 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
980   VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR)
981 #elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
982 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
983   ETHR_DW_NATMC_FUNC__(read_relb)(VAR, VAL)
984 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
985 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
986   VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR)
987 #elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
988 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
989   ETHR_DW_NATMC_FUNC__(read_mb)(VAR, VAL)
990 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
991 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
992   VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR)
993 #else
994 /*
995  * We have no native read() op; guess zero and then use the
996  * the atomics actual value returned from cmpxchg().
997  */
998 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
999 do { \
1000   VAL.sint[0] = (ethr_sint_t) 0; \
1001   VAL.sint[1] = (ethr_sint_t) 0; \
1002 } while (0)
1003 #endif
1004 
1005 /*
1006  * Native cmpxchg() fallback used when no native op present.
1007  */
1008 #define ETHR_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
1009 do { \
1010     int res__; \
1011     ethr_dw_sint_t AVAL, exp_act__; \
1012     ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \
1013     do { \
1014 	AVAL.sint[0] = exp_act__.sint[0]; \
1015 	AVAL.sint[1] = exp_act__.sint[1]; \
1016 	{ OPS; } \
1017 	res__ = CMPXCHG(VAR, AVAL.sint, exp_act__.sint); \
1018     } while (__builtin_expect(res__ == 0, 0)); \
1019 } while (0)
1020 
1021 
1022 #elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
1023 
1024 #if (!defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) \
1025      && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) \
1026      && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) \
1027      && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) \
1028      && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) \
1029      && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB))
1030 #  error "No native cmpxchg() op available"
1031 #endif
1032 
1033 
1034 /*
1035  * Read op used together with cmpxchg() fallback when no native op present.
1036  */
1037 #if defined(ETHR_HAVE_SU_DW_NATMC_READ)
1038 #define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
1039   ETHR_SU_DW_NATMC_FUNC__(read)(VAR)
1040 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
1041 #define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
1042   ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR)
1043 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
1044 #define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
1045   ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR)
1046 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
1047 #define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
1048   ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR)
1049 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
1050 #define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
1051   ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR)
1052 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
1053 #define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
1054   ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR)
1055 #else
1056 /*
1057  * We have no native read() op; guess zero and then use the
1058  * the atomics actual value returned from cmpxchg().
1059  */
1060 #define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
1061   ((ETHR_SU_DW_NAINT_T__) 0)
1062 #endif
1063 
1064 /*
1065  * Native cmpxchg() fallback used when no native op present.
1066  */
1067 #define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
1068 do { \
1069     ETHR_SU_DW_NAINT_T__ AVAL; \
1070     ETHR_SU_DW_NAINT_T__ new__, act__, exp__; \
1071     act__ = ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \
1072     do { \
1073 	exp__ = act__; \
1074 	AVAL = (ETHR_SU_DW_NAINT_T__) act__; \
1075 	{ OPS; } \
1076 	new__ = (ETHR_SU_DW_NAINT_T__) AVAL; \
1077 	act__ = CMPXCHG(VAR, new__, exp__); \
1078     } while (__builtin_expect(act__ != exp__, 0)); \
1079 } while (0)
1080 
1081 
1082 #else
1083 #  error "?!?"
1084 #endif
1085 
1086 
1087 
1088 /* --- addr() --- */
1089 
ETHR_DW_ATMC_FUNC__(addr)1090 static ETHR_INLINE ethr_sint_t *ETHR_DW_ATMC_FUNC__(addr)(ethr_dw_atomic_t *var)
1091 {
1092 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1093     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1094 #endif
1095 
1096     return (ethr_sint_t *) ETHR_DW_NATMC_ADDR_FUNC__((&var->native));
1097 
1098 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1099     } else { return ETHR_DW_ATOMIC_FUNC__(addr)(var); }
1100 #endif
1101 
1102 }
1103 
1104 
1105 /* --- cmpxchg() --- */
1106 
1107 
ETHR_DW_ATMC_FUNC__(cmpxchg)1108 static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
1109 {
1110     int res;
1111 
1112 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1113     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1114 #endif
1115 
1116 #if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1117     ETHR_SU_DW_NAINT_T__ act;
1118     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
1119     res = (act == old_val->dw_sint);
1120     old_val->dw_sint = act;
1121 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1122     ETHR_SU_DW_NAINT_T__ act;
1123     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
1124     res = (act == old_val->dw_sint);
1125     old_val->dw_sint = act;
1126 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1127     ETHR_SU_DW_NAINT_T__ act;
1128     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
1129     res = (act == old_val->dw_sint);
1130     old_val->dw_sint = act;
1131 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1132     ETHR_SU_DW_NAINT_T__ act;
1133     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
1134     res = (act == old_val->dw_sint);
1135     old_val->dw_sint = act;
1136 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1137     ETHR_SU_DW_NAINT_T__ act;
1138     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
1139     res = (act == old_val->dw_sint);
1140     old_val->dw_sint = act;
1141 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1142     ETHR_SU_DW_NAINT_T__ act;
1143     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
1144     res = (act == old_val->dw_sint);
1145     old_val->dw_sint = act;
1146 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1147     res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
1148 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1149     res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
1150 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1151     res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
1152 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1153     res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
1154 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1155     res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
1156 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1157     res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
1158 #else
1159 #error "Missing implementation of ethr_dw_atomic_cmpxchg()!"
1160 #endif
1161 
1162 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1163     } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val); }
1164 #endif
1165 
1166     return res;
1167 }
1168 
ETHR_DW_ATMC_FUNC__(cmpxchg_rb)1169 static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
1170 {
1171     int res;
1172 
1173 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1174     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1175 #endif
1176 
1177 #if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1178     ETHR_SU_DW_NAINT_T__ act;
1179     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
1180     res = (act == old_val->dw_sint);
1181     old_val->dw_sint = act;
1182 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1183     ETHR_SU_DW_NAINT_T__ act;
1184     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
1185     ETHR_MEMBAR(ETHR_LoadLoad);
1186     res = (act == old_val->dw_sint);
1187     old_val->dw_sint = act;
1188 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1189     ETHR_SU_DW_NAINT_T__ act;
1190     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
1191     res = (act == old_val->dw_sint);
1192     old_val->dw_sint = act;
1193 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1194     ETHR_SU_DW_NAINT_T__ act;
1195     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
1196     ETHR_MEMBAR(ETHR_LoadLoad);
1197     res = (act == old_val->dw_sint);
1198     old_val->dw_sint = act;
1199 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1200     ETHR_SU_DW_NAINT_T__ act;
1201     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
1202     ETHR_MEMBAR(ETHR_LoadLoad);
1203     res = (act == old_val->dw_sint);
1204     old_val->dw_sint = act;
1205 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1206     ETHR_SU_DW_NAINT_T__ act;
1207     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
1208     ETHR_MEMBAR(ETHR_LoadLoad);
1209     res = (act == old_val->dw_sint);
1210     old_val->dw_sint = act;
1211 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1212     res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
1213 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1214     res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
1215     ETHR_MEMBAR(ETHR_LoadLoad);
1216 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1217     res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
1218 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1219     res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
1220     ETHR_MEMBAR(ETHR_LoadLoad);
1221 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1222     res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
1223     ETHR_MEMBAR(ETHR_LoadLoad);
1224 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1225     res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
1226     ETHR_MEMBAR(ETHR_LoadLoad);
1227 #else
1228 #error "Missing implementation of ethr_dw_atomic_cmpxchg_rb()!"
1229 #endif
1230 
1231 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1232     } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val); }
1233 #endif
1234 
1235     return res;
1236 }
1237 
ETHR_DW_ATMC_FUNC__(cmpxchg_wb)1238 static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
1239 {
1240     int res;
1241 
1242 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1243     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1244 #endif
1245 
1246 #if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1247     ETHR_SU_DW_NAINT_T__ act;
1248     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
1249     res = (act == old_val->dw_sint);
1250     old_val->dw_sint = act;
1251 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1252     ETHR_SU_DW_NAINT_T__ act;
1253     ETHR_MEMBAR(ETHR_StoreStore);
1254     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
1255     res = (act == old_val->dw_sint);
1256     old_val->dw_sint = act;
1257 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1258     ETHR_SU_DW_NAINT_T__ act;
1259     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
1260     res = (act == old_val->dw_sint);
1261     old_val->dw_sint = act;
1262 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1263     ETHR_SU_DW_NAINT_T__ act;
1264     ETHR_MEMBAR(ETHR_StoreStore);
1265     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
1266     res = (act == old_val->dw_sint);
1267     old_val->dw_sint = act;
1268 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1269     ETHR_SU_DW_NAINT_T__ act;
1270     ETHR_MEMBAR(ETHR_StoreStore);
1271     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
1272     res = (act == old_val->dw_sint);
1273     old_val->dw_sint = act;
1274 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1275     ETHR_SU_DW_NAINT_T__ act;
1276     ETHR_MEMBAR(ETHR_StoreStore);
1277     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
1278     res = (act == old_val->dw_sint);
1279     old_val->dw_sint = act;
1280 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1281     res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
1282 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1283     ETHR_MEMBAR(ETHR_StoreStore);
1284     res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
1285 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1286     res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
1287 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1288     ETHR_MEMBAR(ETHR_StoreStore);
1289     res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
1290 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1291     ETHR_MEMBAR(ETHR_StoreStore);
1292     res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
1293 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1294     ETHR_MEMBAR(ETHR_StoreStore);
1295     res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
1296 #else
1297 #error "Missing implementation of ethr_dw_atomic_cmpxchg_wb()!"
1298 #endif
1299 
1300 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1301     } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(var, val, old_val); }
1302 #endif
1303 
1304     return res;
1305 }
1306 
ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)1307 static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
1308 {
1309     int res;
1310 
1311 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1312     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1313 #endif
1314 
1315 #if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1316     ETHR_SU_DW_NAINT_T__ act;
1317     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
1318     res = (act == old_val->dw_sint);
1319     old_val->dw_sint = act;
1320 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1321     ETHR_SU_DW_NAINT_T__ act;
1322     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
1323     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1324     res = (act == old_val->dw_sint);
1325     old_val->dw_sint = act;
1326 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1327     ETHR_SU_DW_NAINT_T__ act;
1328     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
1329     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1330     res = (act == old_val->dw_sint);
1331     old_val->dw_sint = act;
1332 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1333     ETHR_SU_DW_NAINT_T__ act;
1334     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
1335     res = (act == old_val->dw_sint);
1336     old_val->dw_sint = act;
1337 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1338     ETHR_SU_DW_NAINT_T__ act;
1339     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
1340     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1341     res = (act == old_val->dw_sint);
1342     old_val->dw_sint = act;
1343 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1344     ETHR_SU_DW_NAINT_T__ act;
1345     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
1346     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1347     res = (act == old_val->dw_sint);
1348     old_val->dw_sint = act;
1349 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1350     res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
1351 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1352     res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
1353     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1354 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1355     res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
1356     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1357 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1358     res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
1359 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1360     res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
1361     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1362 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1363     res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
1364     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1365 #else
1366 #error "Missing implementation of ethr_dw_atomic_cmpxchg_acqb()!"
1367 #endif
1368 
1369 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1370     } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(var, val, old_val); }
1371 #endif
1372 
1373     return res;
1374 }
1375 
ETHR_DW_ATMC_FUNC__(cmpxchg_relb)1376 static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
1377 {
1378     int res;
1379 
1380 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1381     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1382 #endif
1383 
1384 #if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1385     ETHR_SU_DW_NAINT_T__ act;
1386     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
1387     res = (act == old_val->dw_sint);
1388     old_val->dw_sint = act;
1389 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1390     ETHR_SU_DW_NAINT_T__ act;
1391     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1392     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
1393     res = (act == old_val->dw_sint);
1394     old_val->dw_sint = act;
1395 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1396     ETHR_SU_DW_NAINT_T__ act;
1397     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1398     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
1399     res = (act == old_val->dw_sint);
1400     old_val->dw_sint = act;
1401 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1402     ETHR_SU_DW_NAINT_T__ act;
1403     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
1404     res = (act == old_val->dw_sint);
1405     old_val->dw_sint = act;
1406 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1407     ETHR_SU_DW_NAINT_T__ act;
1408     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1409     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
1410     res = (act == old_val->dw_sint);
1411     old_val->dw_sint = act;
1412 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1413     ETHR_SU_DW_NAINT_T__ act;
1414     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1415     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
1416     res = (act == old_val->dw_sint);
1417     old_val->dw_sint = act;
1418 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1419     res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
1420 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1421     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1422     res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
1423 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1424     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1425     res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
1426 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1427     res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
1428 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1429     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1430     res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
1431 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1432     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1433     res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
1434 #else
1435 #error "Missing implementation of ethr_dw_atomic_cmpxchg_relb()!"
1436 #endif
1437 
1438 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1439     } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(var, val, old_val); }
1440 #endif
1441 
1442     return res;
1443 }
1444 
ETHR_DW_ATMC_FUNC__(cmpxchg_mb)1445 static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
1446 {
1447     int res;
1448 
1449 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1450     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1451 #endif
1452 
1453 #if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1454     ETHR_SU_DW_NAINT_T__ act;
1455     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
1456     res = (act == old_val->dw_sint);
1457     old_val->dw_sint = act;
1458 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1459     ETHR_SU_DW_NAINT_T__ act;
1460     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
1461     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1462     res = (act == old_val->dw_sint);
1463     old_val->dw_sint = act;
1464 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1465     ETHR_SU_DW_NAINT_T__ act;
1466     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1467     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
1468     res = (act == old_val->dw_sint);
1469     old_val->dw_sint = act;
1470 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1471     ETHR_SU_DW_NAINT_T__ act;
1472     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1473     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
1474     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1475     res = (act == old_val->dw_sint);
1476     old_val->dw_sint = act;
1477 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1478     ETHR_SU_DW_NAINT_T__ act;
1479     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1480     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
1481     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1482     res = (act == old_val->dw_sint);
1483     old_val->dw_sint = act;
1484 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1485     ETHR_SU_DW_NAINT_T__ act;
1486     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1487     act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
1488     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1489     res = (act == old_val->dw_sint);
1490     old_val->dw_sint = act;
1491 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1492     res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
1493 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1494     res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
1495     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1496 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1497     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1498     res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
1499 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1500     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1501     res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
1502     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1503 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1504     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1505     res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
1506     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1507 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1508     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1509     res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
1510     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1511 #else
1512 #error "Missing implementation of ethr_dw_atomic_cmpxchg_mb()!"
1513 #endif
1514 
1515 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1516     } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(var, val, old_val); }
1517 #endif
1518 
1519     return res;
1520 }
1521 
ETHR_DW_ATMC_FUNC__(cmpxchg_ddrb)1522 static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
1523 {
1524 #ifdef ETHR_ORDERED_READ_DEPEND
1525     return ETHR_DW_ATMC_FUNC__(cmpxchg)(var, val, old_val);
1526 #else
1527     return ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, val, old_val);
1528 #endif
1529 }
1530 
1531 
1532 /* --- set() --- */
1533 
1534 
ETHR_DW_ATMC_FUNC__(set)1535 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1536 {
1537 
1538 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1539     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1540 #endif
1541 
1542 #if defined(ETHR_HAVE_SU_DW_NATMC_SET)
1543     ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
1544 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
1545     ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
1546 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
1547     ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
1548 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
1549     ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
1550 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
1551     ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
1552 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
1553     ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
1554 #elif defined(ETHR_HAVE_DW_NATMC_SET)
1555     ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
1556 #elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
1557     ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
1558 #elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
1559     ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
1560 #elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
1561     ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
1562 #elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
1563     ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
1564 #elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
1565     ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
1566 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1567     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
1568 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1569     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
1570 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1571     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
1572 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1573     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
1574 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1575     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
1576 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1577     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
1578 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1579     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1580 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1581     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1582 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1583     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1584 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1585     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1586 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1587     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1588 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1589     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1590 #else
1591 #error "Missing implementation of ethr_dw_atomic_set()!"
1592 #endif
1593 
1594 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1595     } else { ETHR_DW_ATOMIC_FUNC__(set)(var, val); }
1596 #endif
1597 
1598 }
1599 
ETHR_DW_ATMC_FUNC__(set_rb)1600 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1601 {
1602 
1603 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1604     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1605 #endif
1606 
1607 #if defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
1608     ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
1609 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
1610     ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
1611     ETHR_MEMBAR(ETHR_LoadLoad);
1612 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
1613     ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
1614 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
1615     ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
1616     ETHR_MEMBAR(ETHR_LoadLoad);
1617 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
1618     ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
1619     ETHR_MEMBAR(ETHR_LoadLoad);
1620 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
1621     ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
1622     ETHR_MEMBAR(ETHR_LoadLoad);
1623 #elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
1624     ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
1625 #elif defined(ETHR_HAVE_DW_NATMC_SET)
1626     ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
1627     ETHR_MEMBAR(ETHR_LoadLoad);
1628 #elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
1629     ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
1630 #elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
1631     ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
1632     ETHR_MEMBAR(ETHR_LoadLoad);
1633 #elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
1634     ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
1635     ETHR_MEMBAR(ETHR_LoadLoad);
1636 #elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
1637     ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
1638     ETHR_MEMBAR(ETHR_LoadLoad);
1639 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1640     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
1641 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1642     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
1643     ETHR_MEMBAR(ETHR_LoadLoad);
1644 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1645     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
1646 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1647     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
1648     ETHR_MEMBAR(ETHR_LoadLoad);
1649 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1650     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
1651     ETHR_MEMBAR(ETHR_LoadLoad);
1652 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1653     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
1654     ETHR_MEMBAR(ETHR_LoadLoad);
1655 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1656     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1657 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1658     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1659     ETHR_MEMBAR(ETHR_LoadLoad);
1660 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1661     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1662 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1663     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1664     ETHR_MEMBAR(ETHR_LoadLoad);
1665 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1666     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1667     ETHR_MEMBAR(ETHR_LoadLoad);
1668 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1669     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1670     ETHR_MEMBAR(ETHR_LoadLoad);
1671 #else
1672 #error "Missing implementation of ethr_dw_atomic_set_rb()!"
1673 #endif
1674 
1675 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1676     } else { ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val); }
1677 #endif
1678 
1679 }
1680 
ETHR_DW_ATMC_FUNC__(set_wb)1681 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1682 {
1683 
1684 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1685     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1686 #endif
1687 
1688 #if defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
1689     ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
1690 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
1691     ETHR_MEMBAR(ETHR_StoreStore);
1692     ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
1693 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
1694     ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
1695 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
1696     ETHR_MEMBAR(ETHR_StoreStore);
1697     ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
1698 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
1699     ETHR_MEMBAR(ETHR_StoreStore);
1700     ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
1701 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
1702     ETHR_MEMBAR(ETHR_StoreStore);
1703     ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
1704 #elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
1705     ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
1706 #elif defined(ETHR_HAVE_DW_NATMC_SET)
1707     ETHR_MEMBAR(ETHR_StoreStore);
1708     ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
1709 #elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
1710     ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
1711 #elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
1712     ETHR_MEMBAR(ETHR_StoreStore);
1713     ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
1714 #elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
1715     ETHR_MEMBAR(ETHR_StoreStore);
1716     ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
1717 #elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
1718     ETHR_MEMBAR(ETHR_StoreStore);
1719     ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
1720 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1721     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
1722 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1723     ETHR_MEMBAR(ETHR_StoreStore);
1724     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
1725 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1726     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
1727 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1728     ETHR_MEMBAR(ETHR_StoreStore);
1729     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
1730 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1731     ETHR_MEMBAR(ETHR_StoreStore);
1732     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
1733 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1734     ETHR_MEMBAR(ETHR_StoreStore);
1735     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
1736 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1737     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1738 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1739     ETHR_MEMBAR(ETHR_StoreStore);
1740     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1741 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1742     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1743 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1744     ETHR_MEMBAR(ETHR_StoreStore);
1745     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1746 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1747     ETHR_MEMBAR(ETHR_StoreStore);
1748     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1749 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1750     ETHR_MEMBAR(ETHR_StoreStore);
1751     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1752 #else
1753 #error "Missing implementation of ethr_dw_atomic_set_wb()!"
1754 #endif
1755 
1756 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1757     } else { ETHR_DW_ATOMIC_FUNC__(set_wb)(var, val); }
1758 #endif
1759 
1760 }
1761 
ETHR_DW_ATMC_FUNC__(set_acqb)1762 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1763 {
1764 
1765 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1766     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1767 #endif
1768 
1769 #if defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
1770     ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
1771 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
1772     ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
1773     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1774 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
1775     ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
1776     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1777 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
1778     ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
1779 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
1780     ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
1781     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1782 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
1783     ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
1784     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1785 #elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
1786     ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
1787 #elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
1788     ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
1789     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1790 #elif defined(ETHR_HAVE_DW_NATMC_SET)
1791     ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
1792     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1793 #elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
1794     ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
1795 #elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
1796     ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
1797     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1798 #elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
1799     ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
1800     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1801 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1802     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
1803 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1804     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
1805     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1806 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1807     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
1808     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1809 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1810     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
1811 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1812     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
1813     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1814 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1815     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
1816     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1817 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1818     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1819 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1820     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1821     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1822 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1823     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1824     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1825 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1826     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1827 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1828     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1829     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1830 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1831     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1832     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1833 #else
1834 #error "Missing implementation of ethr_dw_atomic_set_acqb()!"
1835 #endif
1836 
1837 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1838     } else { ETHR_DW_ATOMIC_FUNC__(set_acqb)(var, val); }
1839 #endif
1840 
1841 }
1842 
ETHR_DW_ATMC_FUNC__(set_relb)1843 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1844 {
1845 
1846 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1847     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1848 #endif
1849 
1850 #if defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
1851     ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
1852 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
1853     ETHR_MEMBAR(ETHR_LoadStore);
1854     ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
1855 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
1856     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1857     ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
1858 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
1859     ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
1860 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
1861     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1862     ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
1863 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
1864     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1865     ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
1866 #elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
1867     ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
1868 #elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
1869     ETHR_MEMBAR(ETHR_LoadStore);
1870     ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
1871 #elif defined(ETHR_HAVE_DW_NATMC_SET)
1872     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1873     ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
1874 #elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
1875     ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
1876 #elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
1877     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1878     ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
1879 #elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
1880     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1881     ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
1882 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1883     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
1884 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1885     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1886     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
1887 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1888     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1889     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
1890 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1891     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
1892 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1893     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1894     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
1895 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1896     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1897     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
1898 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1899     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1900 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
1901     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1902     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1903 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
1904     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1905     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1906 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1907     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1908 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
1909     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1910     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1911 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1912     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1913     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1914 #else
1915 #error "Missing implementation of ethr_dw_atomic_set_relb()!"
1916 #endif
1917 
1918 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1919     } else { ETHR_DW_ATOMIC_FUNC__(set_relb)(var, val); }
1920 #endif
1921 
1922 }
1923 
ETHR_DW_ATMC_FUNC__(set_mb)1924 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
1925 {
1926 
1927 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
1928     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
1929 #endif
1930 
1931 #if defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
1932     ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
1933 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
1934     ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
1935     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1936 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
1937     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1938     ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
1939 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
1940     ETHR_MEMBAR(ETHR_LoadStore);
1941     ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
1942     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1943 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
1944     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1945     ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
1946     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1947 #elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
1948     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1949     ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
1950     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1951 #elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
1952     ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
1953 #elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
1954     ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
1955     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1956 #elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
1957     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1958     ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
1959 #elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
1960     ETHR_MEMBAR(ETHR_LoadStore);
1961     ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
1962     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1963 #elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
1964     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1965     ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
1966     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1967 #elif defined(ETHR_HAVE_DW_NATMC_SET)
1968     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
1969     ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
1970     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
1971 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
1972     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
1973 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
1974     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
1975     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1976 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
1977     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1978     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
1979 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
1980     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1981     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
1982     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1983 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
1984     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1985     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
1986     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1987 #elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
1988     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1989     ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
1990     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1991 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
1992     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1993 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
1994     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1995     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
1996 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
1997     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
1998     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
1999 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
2000     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2001     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
2002     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2003 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
2004     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2005     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
2006     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2007 #elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
2008     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2009     ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
2010     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2011 #else
2012 #error "Missing implementation of ethr_dw_atomic_set_mb()!"
2013 #endif
2014 
2015 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2016     } else { ETHR_DW_ATOMIC_FUNC__(set_mb)(var, val); }
2017 #endif
2018 
2019 }
2020 
ETHR_DW_ATMC_FUNC__(set_ddrb)2021 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2022 {
2023 #ifdef ETHR_ORDERED_READ_DEPEND
2024     ETHR_DW_ATMC_FUNC__(set)(var, val);
2025 #else
2026     ETHR_DW_ATMC_FUNC__(set_rb)(var, val);
2027 #endif
2028 }
2029 
2030 
2031 /* --- read() --- */
2032 
2033 
ETHR_DW_ATMC_FUNC__(read)2034 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2035 {
2036 
2037 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2038     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2039 #endif
2040 
2041 #if defined(ETHR_HAVE_SU_DW_NATMC_READ)
2042     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
2043 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
2044     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
2045 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
2046     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
2047 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
2048     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
2049 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
2050     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
2051 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
2052     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
2053 #elif defined(ETHR_HAVE_DW_NATMC_READ)
2054     ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
2055 #elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
2056     ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
2057 #elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
2058     ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
2059 #elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
2060     ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
2061 #elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
2062     ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
2063 #elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
2064     ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
2065 #else
2066     ethr_dw_sint_t tmp;
2067     tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2068     tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2069     val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2070     val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2071     (void) ETHR_DW_ATMC_FUNC__(cmpxchg)(var, &tmp, val);
2072 #endif
2073 
2074 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2075     } else { ETHR_DW_ATOMIC_FUNC__(read)(var, val); }
2076 #endif
2077 
2078 }
2079 
ETHR_DW_ATMC_FUNC__(read_rb)2080 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2081 {
2082 
2083 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2084     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2085 #endif
2086 
2087 #if defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
2088     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
2089 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
2090     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
2091     ETHR_MEMBAR(ETHR_LoadLoad);
2092 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
2093     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
2094 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
2095     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
2096     ETHR_MEMBAR(ETHR_LoadLoad);
2097 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
2098     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
2099     ETHR_MEMBAR(ETHR_LoadLoad);
2100 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
2101     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
2102     ETHR_MEMBAR(ETHR_LoadLoad);
2103 #elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
2104     ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
2105 #elif defined(ETHR_HAVE_DW_NATMC_READ)
2106     ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
2107     ETHR_MEMBAR(ETHR_LoadLoad);
2108 #elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
2109     ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
2110 #elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
2111     ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
2112     ETHR_MEMBAR(ETHR_LoadLoad);
2113 #elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
2114     ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
2115     ETHR_MEMBAR(ETHR_LoadLoad);
2116 #elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
2117     ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
2118     ETHR_MEMBAR(ETHR_LoadLoad);
2119 #else
2120     ethr_dw_sint_t tmp;
2121     tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2122     tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2123     val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2124     val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2125     (void) ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, &tmp, val);
2126 #endif
2127 
2128 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2129     } else { ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val); }
2130 #endif
2131 
2132 }
2133 
ETHR_DW_ATMC_FUNC__(read_wb)2134 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2135 {
2136 
2137 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2138     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2139 #endif
2140 
2141 #if defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
2142     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
2143 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
2144     ETHR_MEMBAR(ETHR_StoreStore);
2145     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
2146 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
2147     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
2148 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
2149     ETHR_MEMBAR(ETHR_StoreStore);
2150     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
2151 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
2152     ETHR_MEMBAR(ETHR_StoreStore);
2153     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
2154 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
2155     ETHR_MEMBAR(ETHR_StoreStore);
2156     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
2157 #elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
2158     ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
2159 #elif defined(ETHR_HAVE_DW_NATMC_READ)
2160     ETHR_MEMBAR(ETHR_StoreStore);
2161     ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
2162 #elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
2163     ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
2164 #elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
2165     ETHR_MEMBAR(ETHR_StoreStore);
2166     ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
2167 #elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
2168     ETHR_MEMBAR(ETHR_StoreStore);
2169     ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
2170 #elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
2171     ETHR_MEMBAR(ETHR_StoreStore);
2172     ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
2173 #else
2174     ethr_dw_sint_t tmp;
2175     tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2176     tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2177     val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2178     val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2179     (void) ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(var, &tmp, val);
2180 #endif
2181 
2182 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2183     } else { ETHR_DW_ATOMIC_FUNC__(read_wb)(var, val); }
2184 #endif
2185 
2186 }
2187 
ETHR_DW_ATMC_FUNC__(read_acqb)2188 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2189 {
2190 
2191 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2192     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2193 #endif
2194 
2195 #if defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
2196     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
2197 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
2198     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
2199     ETHR_MEMBAR(ETHR_LoadStore);
2200 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
2201     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
2202     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2203 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
2204     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
2205 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
2206     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
2207     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2208 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
2209     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
2210     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2211 #elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
2212     ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
2213 #elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
2214     ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
2215     ETHR_MEMBAR(ETHR_LoadStore);
2216 #elif defined(ETHR_HAVE_DW_NATMC_READ)
2217     ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
2218     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2219 #elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
2220     ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
2221 #elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
2222     ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
2223     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2224 #elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
2225     ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
2226     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2227 #else
2228     ethr_dw_sint_t tmp;
2229     tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2230     tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2231     val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2232     val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2233     (void) ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(var, &tmp, val);
2234 #endif
2235 
2236 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2237     } else { ETHR_DW_ATOMIC_FUNC__(read_acqb)(var, val); }
2238 #endif
2239 
2240 }
2241 
ETHR_DW_ATMC_FUNC__(read_relb)2242 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2243 {
2244 
2245 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2246     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2247 #endif
2248 
2249 #if defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
2250     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
2251 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
2252     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2253     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
2254 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
2255     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2256     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
2257 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
2258     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
2259 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
2260     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2261     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
2262 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
2263     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2264     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
2265 #elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
2266     ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
2267 #elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
2268     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2269     ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
2270 #elif defined(ETHR_HAVE_DW_NATMC_READ)
2271     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2272     ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
2273 #elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
2274     ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
2275 #elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
2276     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2277     ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
2278 #elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
2279     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2280     ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
2281 #else
2282     ethr_dw_sint_t tmp;
2283     tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2284     tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2285     val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2286     val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2287     (void) ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(var, &tmp, val);
2288 #endif
2289 
2290 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2291     } else { ETHR_DW_ATOMIC_FUNC__(read_relb)(var, val); }
2292 #endif
2293 
2294 }
2295 
ETHR_DW_ATMC_FUNC__(read_mb)2296 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2297 {
2298 
2299 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2300     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2301 #endif
2302 
2303 #if defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
2304     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
2305 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
2306     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
2307     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2308 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
2309     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2310     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
2311 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
2312     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2313     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
2314     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2315 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
2316     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2317     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
2318     ETHR_MEMBAR(ETHR_LoadStore);
2319 #elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
2320     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2321     val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
2322     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2323 #elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
2324     ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
2325 #elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
2326     ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
2327     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2328 #elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
2329     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2330     ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
2331 #elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
2332     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2333     ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
2334     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2335 #elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
2336     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2337     ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
2338     ETHR_MEMBAR(ETHR_LoadStore);
2339 #elif defined(ETHR_HAVE_DW_NATMC_READ)
2340     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
2341     ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
2342     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
2343 #else
2344     ethr_dw_sint_t tmp;
2345     tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2346     tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2347     val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
2348     val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
2349     (void) ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(var, &tmp, val);
2350 #endif
2351 
2352 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2353     } else { ETHR_DW_ATOMIC_FUNC__(read_mb)(var, val); }
2354 #endif
2355 
2356 }
2357 
ETHR_DW_ATMC_FUNC__(read_ddrb)2358 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2359 {
2360 #ifdef ETHR_ORDERED_READ_DEPEND
2361     ETHR_DW_ATMC_FUNC__(read)(var, val);
2362 #else
2363     ETHR_DW_ATMC_FUNC__(read_rb)(var, val);
2364 #endif
2365 }
2366 
2367 
2368 /* --- init() --- */
2369 
2370 
ETHR_DW_ATMC_FUNC__(init)2371 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2372 {
2373 
2374 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2375     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2376 #endif
2377 
2378 #if defined(ETHR_HAVE_SU_DW_NATMC_INIT)
2379     ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
2380 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
2381     ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
2382 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
2383     ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
2384 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
2385     ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
2386 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
2387     ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
2388 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
2389     ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
2390 #elif defined(ETHR_HAVE_DW_NATMC_INIT)
2391     ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
2392 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
2393     ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
2394 #elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
2395     ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
2396 #elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
2397     ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
2398 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
2399     ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
2400 #elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
2401     ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
2402 #else
2403     ETHR_DW_ATMC_FUNC__(set)(var, val);
2404 #endif
2405 
2406 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2407     } else { ETHR_DW_ATOMIC_FUNC__(init)(var, val); }
2408 #endif
2409 
2410 }
2411 
ETHR_DW_ATMC_FUNC__(init_rb)2412 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2413 {
2414 
2415 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2416     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2417 #endif
2418 
2419 #if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
2420     ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
2421 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
2422     ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
2423     ETHR_MEMBAR(ETHR_LoadLoad);
2424 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
2425     ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
2426 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
2427     ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
2428     ETHR_MEMBAR(ETHR_LoadLoad);
2429 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
2430     ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
2431     ETHR_MEMBAR(ETHR_LoadLoad);
2432 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
2433     ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
2434     ETHR_MEMBAR(ETHR_LoadLoad);
2435 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
2436     ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
2437 #elif defined(ETHR_HAVE_DW_NATMC_INIT)
2438     ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
2439     ETHR_MEMBAR(ETHR_LoadLoad);
2440 #elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
2441     ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
2442 #elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
2443     ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
2444     ETHR_MEMBAR(ETHR_LoadLoad);
2445 #elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
2446     ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
2447     ETHR_MEMBAR(ETHR_LoadLoad);
2448 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
2449     ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
2450     ETHR_MEMBAR(ETHR_LoadLoad);
2451 #else
2452     ETHR_DW_ATMC_FUNC__(set_rb)(var, val);
2453 #endif
2454 
2455 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2456     } else { ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val); }
2457 #endif
2458 
2459 }
2460 
ETHR_DW_ATMC_FUNC__(init_wb)2461 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2462 {
2463 
2464 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2465     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2466 #endif
2467 
2468 #if defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
2469     ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
2470 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
2471     ETHR_MEMBAR(ETHR_StoreStore);
2472     ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
2473 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
2474     ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
2475 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
2476     ETHR_MEMBAR(ETHR_StoreStore);
2477     ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
2478 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
2479     ETHR_MEMBAR(ETHR_StoreStore);
2480     ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
2481 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
2482     ETHR_MEMBAR(ETHR_StoreStore);
2483     ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
2484 #elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
2485     ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
2486 #elif defined(ETHR_HAVE_DW_NATMC_INIT)
2487     ETHR_MEMBAR(ETHR_StoreStore);
2488     ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
2489 #elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
2490     ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
2491 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
2492     ETHR_MEMBAR(ETHR_StoreStore);
2493     ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
2494 #elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
2495     ETHR_MEMBAR(ETHR_StoreStore);
2496     ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
2497 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
2498     ETHR_MEMBAR(ETHR_StoreStore);
2499     ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
2500 #else
2501     ETHR_DW_ATMC_FUNC__(set_wb)(var, val);
2502 #endif
2503 
2504 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2505     } else { ETHR_DW_ATOMIC_FUNC__(init_wb)(var, val); }
2506 #endif
2507 
2508 }
2509 
ETHR_DW_ATMC_FUNC__(init_acqb)2510 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2511 {
2512 
2513 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2514     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2515 #endif
2516 
2517 #if defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
2518     ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
2519 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
2520     ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
2521     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2522 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
2523     ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
2524     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2525 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
2526     ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
2527 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
2528     ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
2529     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2530 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
2531     ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
2532     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2533 #elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
2534     ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
2535 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
2536     ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
2537     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2538 #elif defined(ETHR_HAVE_DW_NATMC_INIT)
2539     ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
2540     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2541 #elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
2542     ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
2543 #elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
2544     ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
2545     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2546 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
2547     ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
2548     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2549 #else
2550     ETHR_DW_ATMC_FUNC__(set_acqb)(var, val);
2551 #endif
2552 
2553 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2554     } else { ETHR_DW_ATOMIC_FUNC__(init_acqb)(var, val); }
2555 #endif
2556 
2557 }
2558 
ETHR_DW_ATMC_FUNC__(init_relb)2559 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2560 {
2561 
2562 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2563     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2564 #endif
2565 
2566 #if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
2567     ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
2568 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
2569     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
2570     ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
2571 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
2572     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2573     ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
2574 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
2575     ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
2576 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
2577     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2578     ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
2579 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
2580     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2581     ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
2582 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
2583     ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
2584 #elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
2585     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
2586     ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
2587 #elif defined(ETHR_HAVE_DW_NATMC_INIT)
2588     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2589     ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
2590 #elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
2591     ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
2592 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
2593     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2594     ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
2595 #elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
2596     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2597     ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
2598 #else
2599     ETHR_DW_ATMC_FUNC__(set_relb)(var, val);
2600 #endif
2601 
2602 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2603     } else { ETHR_DW_ATOMIC_FUNC__(init_relb)(var, val); }
2604 #endif
2605 
2606 }
2607 
ETHR_DW_ATMC_FUNC__(init_mb)2608 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2609 {
2610 
2611 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2612     if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
2613 #endif
2614 
2615 #if defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
2616     ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
2617 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
2618     ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
2619     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2620 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
2621     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2622     ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
2623 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
2624     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
2625     ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
2626     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2627 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
2628     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2629     ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
2630     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2631 #elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
2632     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2633     ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
2634     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2635 #elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
2636     ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
2637 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
2638     ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
2639     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2640 #elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
2641     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2642     ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
2643 #elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
2644     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
2645     ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
2646     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2647 #elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
2648     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2649     ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
2650     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2651 #elif defined(ETHR_HAVE_DW_NATMC_INIT)
2652     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2653     ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
2654     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
2655 #else
2656     ETHR_DW_ATMC_FUNC__(set_mb)(var, val);
2657 #endif
2658 
2659 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
2660     } else { ETHR_DW_ATOMIC_FUNC__(init_mb)(var, val); }
2661 #endif
2662 
2663 }
2664 
ETHR_DW_ATMC_FUNC__(init_ddrb)2665 static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
2666 {
2667 #ifdef ETHR_ORDERED_READ_DEPEND
2668     ETHR_DW_ATMC_FUNC__(init)(var, val);
2669 #else
2670     ETHR_DW_ATMC_FUNC__(init_rb)(var, val);
2671 #endif
2672 }
2673 
2674 #endif /* ETHR_DW_ATMC_INLINE__ */
2675 
2676 
2677 /* ---------- Word size atomic implementation ---------- */
2678 
2679 
2680 #ifdef ETHR_NEED_ATMC_PROTOTYPES__
2681 ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var);
2682 ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
2683 ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
2684 ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
2685 ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
2686 ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
2687 ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
2688 ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
2689 ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val);
2690 ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val);
2691 ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val);
2692 ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val);
2693 ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val);
2694 ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val);
2695 ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val);
2696 void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val);
2697 void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val);
2698 void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val);
2699 void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val);
2700 void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val);
2701 void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val);
2702 void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val);
2703 void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val);
2704 void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val);
2705 void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val);
2706 void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val);
2707 void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val);
2708 void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val);
2709 void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val);
2710 ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val);
2711 ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val);
2712 ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val);
2713 ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val);
2714 ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val);
2715 ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val);
2716 ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val);
2717 ethr_sint_t ethr_atomic_read(ethr_atomic_t *var);
2718 ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var);
2719 ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var);
2720 ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var);
2721 ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var);
2722 ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var);
2723 ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var);
2724 ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var);
2725 ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var);
2726 ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var);
2727 ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var);
2728 ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var);
2729 ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var);
2730 ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var);
2731 ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var);
2732 ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var);
2733 ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var);
2734 ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var);
2735 ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var);
2736 ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var);
2737 ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var);
2738 void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val);
2739 void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val);
2740 void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val);
2741 void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val);
2742 void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val);
2743 void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val);
2744 void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val);
2745 void ethr_atomic_inc(ethr_atomic_t *var);
2746 void ethr_atomic_inc_ddrb(ethr_atomic_t *var);
2747 void ethr_atomic_inc_rb(ethr_atomic_t *var);
2748 void ethr_atomic_inc_wb(ethr_atomic_t *var);
2749 void ethr_atomic_inc_acqb(ethr_atomic_t *var);
2750 void ethr_atomic_inc_relb(ethr_atomic_t *var);
2751 void ethr_atomic_inc_mb(ethr_atomic_t *var);
2752 void ethr_atomic_dec(ethr_atomic_t *var);
2753 void ethr_atomic_dec_ddrb(ethr_atomic_t *var);
2754 void ethr_atomic_dec_rb(ethr_atomic_t *var);
2755 void ethr_atomic_dec_wb(ethr_atomic_t *var);
2756 void ethr_atomic_dec_acqb(ethr_atomic_t *var);
2757 void ethr_atomic_dec_relb(ethr_atomic_t *var);
2758 void ethr_atomic_dec_mb(ethr_atomic_t *var);
2759 ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val);
2760 ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val);
2761 ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val);
2762 ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val);
2763 ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val);
2764 ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val);
2765 ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val);
2766 ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val);
2767 ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val);
2768 ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val);
2769 ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val);
2770 ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val);
2771 ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val);
2772 ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val);
2773 #endif /* ETHR_NEED_ATMC_PROTOTYPES__ */
2774 
2775 #if (defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
2776      && (defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
2777 
2778 #if !defined(ETHR_NATMC_BITS__)
2779 #  error "Missing native atomic implementation"
2780 #elif ETHR_NATMC_BITS__ == 64
2781 #  undef ETHR_HAVE_NATMC_CMPXCHG
2782 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
2783 #    define ETHR_HAVE_NATMC_CMPXCHG 1
2784 #  endif
2785 #  undef ETHR_HAVE_NATMC_CMPXCHG_RB
2786 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
2787 #    define ETHR_HAVE_NATMC_CMPXCHG_RB 1
2788 #  endif
2789 #  undef ETHR_HAVE_NATMC_CMPXCHG_WB
2790 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
2791 #    define ETHR_HAVE_NATMC_CMPXCHG_WB 1
2792 #  endif
2793 #  undef ETHR_HAVE_NATMC_CMPXCHG_ACQB
2794 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
2795 #    define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1
2796 #  endif
2797 #  undef ETHR_HAVE_NATMC_CMPXCHG_RELB
2798 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
2799 #    define ETHR_HAVE_NATMC_CMPXCHG_RELB 1
2800 #  endif
2801 #  undef ETHR_HAVE_NATMC_CMPXCHG_MB
2802 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
2803 #    define ETHR_HAVE_NATMC_CMPXCHG_MB 1
2804 #  endif
2805 #  undef ETHR_HAVE_NATMC_XCHG
2806 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
2807 #    define ETHR_HAVE_NATMC_XCHG 1
2808 #  endif
2809 #  undef ETHR_HAVE_NATMC_XCHG_RB
2810 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
2811 #    define ETHR_HAVE_NATMC_XCHG_RB 1
2812 #  endif
2813 #  undef ETHR_HAVE_NATMC_XCHG_WB
2814 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
2815 #    define ETHR_HAVE_NATMC_XCHG_WB 1
2816 #  endif
2817 #  undef ETHR_HAVE_NATMC_XCHG_ACQB
2818 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
2819 #    define ETHR_HAVE_NATMC_XCHG_ACQB 1
2820 #  endif
2821 #  undef ETHR_HAVE_NATMC_XCHG_RELB
2822 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
2823 #    define ETHR_HAVE_NATMC_XCHG_RELB 1
2824 #  endif
2825 #  undef ETHR_HAVE_NATMC_XCHG_MB
2826 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
2827 #    define ETHR_HAVE_NATMC_XCHG_MB 1
2828 #  endif
2829 #  undef ETHR_HAVE_NATMC_SET
2830 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
2831 #    define ETHR_HAVE_NATMC_SET 1
2832 #  endif
2833 #  undef ETHR_HAVE_NATMC_SET_RB
2834 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
2835 #    define ETHR_HAVE_NATMC_SET_RB 1
2836 #  endif
2837 #  undef ETHR_HAVE_NATMC_SET_WB
2838 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
2839 #    define ETHR_HAVE_NATMC_SET_WB 1
2840 #  endif
2841 #  undef ETHR_HAVE_NATMC_SET_ACQB
2842 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
2843 #    define ETHR_HAVE_NATMC_SET_ACQB 1
2844 #  endif
2845 #  undef ETHR_HAVE_NATMC_SET_RELB
2846 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
2847 #    define ETHR_HAVE_NATMC_SET_RELB 1
2848 #  endif
2849 #  undef ETHR_HAVE_NATMC_SET_MB
2850 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
2851 #    define ETHR_HAVE_NATMC_SET_MB 1
2852 #  endif
2853 #  undef ETHR_HAVE_NATMC_INIT
2854 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
2855 #    define ETHR_HAVE_NATMC_INIT 1
2856 #  endif
2857 #  undef ETHR_HAVE_NATMC_INIT_RB
2858 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
2859 #    define ETHR_HAVE_NATMC_INIT_RB 1
2860 #  endif
2861 #  undef ETHR_HAVE_NATMC_INIT_WB
2862 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
2863 #    define ETHR_HAVE_NATMC_INIT_WB 1
2864 #  endif
2865 #  undef ETHR_HAVE_NATMC_INIT_ACQB
2866 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
2867 #    define ETHR_HAVE_NATMC_INIT_ACQB 1
2868 #  endif
2869 #  undef ETHR_HAVE_NATMC_INIT_RELB
2870 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
2871 #    define ETHR_HAVE_NATMC_INIT_RELB 1
2872 #  endif
2873 #  undef ETHR_HAVE_NATMC_INIT_MB
2874 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
2875 #    define ETHR_HAVE_NATMC_INIT_MB 1
2876 #  endif
2877 #  undef ETHR_HAVE_NATMC_ADD_RETURN
2878 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
2879 #    define ETHR_HAVE_NATMC_ADD_RETURN 1
2880 #  endif
2881 #  undef ETHR_HAVE_NATMC_ADD_RETURN_RB
2882 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
2883 #    define ETHR_HAVE_NATMC_ADD_RETURN_RB 1
2884 #  endif
2885 #  undef ETHR_HAVE_NATMC_ADD_RETURN_WB
2886 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
2887 #    define ETHR_HAVE_NATMC_ADD_RETURN_WB 1
2888 #  endif
2889 #  undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB
2890 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
2891 #    define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1
2892 #  endif
2893 #  undef ETHR_HAVE_NATMC_ADD_RETURN_RELB
2894 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
2895 #    define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1
2896 #  endif
2897 #  undef ETHR_HAVE_NATMC_ADD_RETURN_MB
2898 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
2899 #    define ETHR_HAVE_NATMC_ADD_RETURN_MB 1
2900 #  endif
2901 #  undef ETHR_HAVE_NATMC_READ
2902 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
2903 #    define ETHR_HAVE_NATMC_READ 1
2904 #  endif
2905 #  undef ETHR_HAVE_NATMC_READ_RB
2906 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
2907 #    define ETHR_HAVE_NATMC_READ_RB 1
2908 #  endif
2909 #  undef ETHR_HAVE_NATMC_READ_WB
2910 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
2911 #    define ETHR_HAVE_NATMC_READ_WB 1
2912 #  endif
2913 #  undef ETHR_HAVE_NATMC_READ_ACQB
2914 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
2915 #    define ETHR_HAVE_NATMC_READ_ACQB 1
2916 #  endif
2917 #  undef ETHR_HAVE_NATMC_READ_RELB
2918 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
2919 #    define ETHR_HAVE_NATMC_READ_RELB 1
2920 #  endif
2921 #  undef ETHR_HAVE_NATMC_READ_MB
2922 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
2923 #    define ETHR_HAVE_NATMC_READ_MB 1
2924 #  endif
2925 #  undef ETHR_HAVE_NATMC_INC_RETURN
2926 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
2927 #    define ETHR_HAVE_NATMC_INC_RETURN 1
2928 #  endif
2929 #  undef ETHR_HAVE_NATMC_INC_RETURN_RB
2930 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
2931 #    define ETHR_HAVE_NATMC_INC_RETURN_RB 1
2932 #  endif
2933 #  undef ETHR_HAVE_NATMC_INC_RETURN_WB
2934 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
2935 #    define ETHR_HAVE_NATMC_INC_RETURN_WB 1
2936 #  endif
2937 #  undef ETHR_HAVE_NATMC_INC_RETURN_ACQB
2938 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
2939 #    define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1
2940 #  endif
2941 #  undef ETHR_HAVE_NATMC_INC_RETURN_RELB
2942 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
2943 #    define ETHR_HAVE_NATMC_INC_RETURN_RELB 1
2944 #  endif
2945 #  undef ETHR_HAVE_NATMC_INC_RETURN_MB
2946 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
2947 #    define ETHR_HAVE_NATMC_INC_RETURN_MB 1
2948 #  endif
2949 #  undef ETHR_HAVE_NATMC_DEC_RETURN
2950 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
2951 #    define ETHR_HAVE_NATMC_DEC_RETURN 1
2952 #  endif
2953 #  undef ETHR_HAVE_NATMC_DEC_RETURN_RB
2954 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
2955 #    define ETHR_HAVE_NATMC_DEC_RETURN_RB 1
2956 #  endif
2957 #  undef ETHR_HAVE_NATMC_DEC_RETURN_WB
2958 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
2959 #    define ETHR_HAVE_NATMC_DEC_RETURN_WB 1
2960 #  endif
2961 #  undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB
2962 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
2963 #    define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1
2964 #  endif
2965 #  undef ETHR_HAVE_NATMC_DEC_RETURN_RELB
2966 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
2967 #    define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1
2968 #  endif
2969 #  undef ETHR_HAVE_NATMC_DEC_RETURN_MB
2970 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
2971 #    define ETHR_HAVE_NATMC_DEC_RETURN_MB 1
2972 #  endif
2973 #  undef ETHR_HAVE_NATMC_ADD
2974 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
2975 #    define ETHR_HAVE_NATMC_ADD 1
2976 #  endif
2977 #  undef ETHR_HAVE_NATMC_ADD_RB
2978 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
2979 #    define ETHR_HAVE_NATMC_ADD_RB 1
2980 #  endif
2981 #  undef ETHR_HAVE_NATMC_ADD_WB
2982 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
2983 #    define ETHR_HAVE_NATMC_ADD_WB 1
2984 #  endif
2985 #  undef ETHR_HAVE_NATMC_ADD_ACQB
2986 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
2987 #    define ETHR_HAVE_NATMC_ADD_ACQB 1
2988 #  endif
2989 #  undef ETHR_HAVE_NATMC_ADD_RELB
2990 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
2991 #    define ETHR_HAVE_NATMC_ADD_RELB 1
2992 #  endif
2993 #  undef ETHR_HAVE_NATMC_ADD_MB
2994 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
2995 #    define ETHR_HAVE_NATMC_ADD_MB 1
2996 #  endif
2997 #  undef ETHR_HAVE_NATMC_INC
2998 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
2999 #    define ETHR_HAVE_NATMC_INC 1
3000 #  endif
3001 #  undef ETHR_HAVE_NATMC_INC_RB
3002 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
3003 #    define ETHR_HAVE_NATMC_INC_RB 1
3004 #  endif
3005 #  undef ETHR_HAVE_NATMC_INC_WB
3006 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
3007 #    define ETHR_HAVE_NATMC_INC_WB 1
3008 #  endif
3009 #  undef ETHR_HAVE_NATMC_INC_ACQB
3010 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
3011 #    define ETHR_HAVE_NATMC_INC_ACQB 1
3012 #  endif
3013 #  undef ETHR_HAVE_NATMC_INC_RELB
3014 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
3015 #    define ETHR_HAVE_NATMC_INC_RELB 1
3016 #  endif
3017 #  undef ETHR_HAVE_NATMC_INC_MB
3018 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
3019 #    define ETHR_HAVE_NATMC_INC_MB 1
3020 #  endif
3021 #  undef ETHR_HAVE_NATMC_DEC
3022 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
3023 #    define ETHR_HAVE_NATMC_DEC 1
3024 #  endif
3025 #  undef ETHR_HAVE_NATMC_DEC_RB
3026 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
3027 #    define ETHR_HAVE_NATMC_DEC_RB 1
3028 #  endif
3029 #  undef ETHR_HAVE_NATMC_DEC_WB
3030 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
3031 #    define ETHR_HAVE_NATMC_DEC_WB 1
3032 #  endif
3033 #  undef ETHR_HAVE_NATMC_DEC_ACQB
3034 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
3035 #    define ETHR_HAVE_NATMC_DEC_ACQB 1
3036 #  endif
3037 #  undef ETHR_HAVE_NATMC_DEC_RELB
3038 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
3039 #    define ETHR_HAVE_NATMC_DEC_RELB 1
3040 #  endif
3041 #  undef ETHR_HAVE_NATMC_DEC_MB
3042 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
3043 #    define ETHR_HAVE_NATMC_DEC_MB 1
3044 #  endif
3045 #  undef ETHR_HAVE_NATMC_AND_RETOLD
3046 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
3047 #    define ETHR_HAVE_NATMC_AND_RETOLD 1
3048 #  endif
3049 #  undef ETHR_HAVE_NATMC_AND_RETOLD_RB
3050 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
3051 #    define ETHR_HAVE_NATMC_AND_RETOLD_RB 1
3052 #  endif
3053 #  undef ETHR_HAVE_NATMC_AND_RETOLD_WB
3054 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
3055 #    define ETHR_HAVE_NATMC_AND_RETOLD_WB 1
3056 #  endif
3057 #  undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB
3058 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
3059 #    define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1
3060 #  endif
3061 #  undef ETHR_HAVE_NATMC_AND_RETOLD_RELB
3062 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
3063 #    define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1
3064 #  endif
3065 #  undef ETHR_HAVE_NATMC_AND_RETOLD_MB
3066 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
3067 #    define ETHR_HAVE_NATMC_AND_RETOLD_MB 1
3068 #  endif
3069 #  undef ETHR_HAVE_NATMC_OR_RETOLD
3070 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
3071 #    define ETHR_HAVE_NATMC_OR_RETOLD 1
3072 #  endif
3073 #  undef ETHR_HAVE_NATMC_OR_RETOLD_RB
3074 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
3075 #    define ETHR_HAVE_NATMC_OR_RETOLD_RB 1
3076 #  endif
3077 #  undef ETHR_HAVE_NATMC_OR_RETOLD_WB
3078 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
3079 #    define ETHR_HAVE_NATMC_OR_RETOLD_WB 1
3080 #  endif
3081 #  undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB
3082 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
3083 #    define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1
3084 #  endif
3085 #  undef ETHR_HAVE_NATMC_OR_RETOLD_RELB
3086 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
3087 #    define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1
3088 #  endif
3089 #  undef ETHR_HAVE_NATMC_OR_RETOLD_MB
3090 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
3091 #    define ETHR_HAVE_NATMC_OR_RETOLD_MB 1
3092 #  endif
3093 #elif ETHR_NATMC_BITS__ == 32
3094 #  undef ETHR_HAVE_NATMC_CMPXCHG
3095 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
3096 #    define ETHR_HAVE_NATMC_CMPXCHG 1
3097 #  endif
3098 #  undef ETHR_HAVE_NATMC_CMPXCHG_RB
3099 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
3100 #    define ETHR_HAVE_NATMC_CMPXCHG_RB 1
3101 #  endif
3102 #  undef ETHR_HAVE_NATMC_CMPXCHG_WB
3103 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
3104 #    define ETHR_HAVE_NATMC_CMPXCHG_WB 1
3105 #  endif
3106 #  undef ETHR_HAVE_NATMC_CMPXCHG_ACQB
3107 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
3108 #    define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1
3109 #  endif
3110 #  undef ETHR_HAVE_NATMC_CMPXCHG_RELB
3111 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
3112 #    define ETHR_HAVE_NATMC_CMPXCHG_RELB 1
3113 #  endif
3114 #  undef ETHR_HAVE_NATMC_CMPXCHG_MB
3115 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
3116 #    define ETHR_HAVE_NATMC_CMPXCHG_MB 1
3117 #  endif
3118 #  undef ETHR_HAVE_NATMC_XCHG
3119 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
3120 #    define ETHR_HAVE_NATMC_XCHG 1
3121 #  endif
3122 #  undef ETHR_HAVE_NATMC_XCHG_RB
3123 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
3124 #    define ETHR_HAVE_NATMC_XCHG_RB 1
3125 #  endif
3126 #  undef ETHR_HAVE_NATMC_XCHG_WB
3127 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
3128 #    define ETHR_HAVE_NATMC_XCHG_WB 1
3129 #  endif
3130 #  undef ETHR_HAVE_NATMC_XCHG_ACQB
3131 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
3132 #    define ETHR_HAVE_NATMC_XCHG_ACQB 1
3133 #  endif
3134 #  undef ETHR_HAVE_NATMC_XCHG_RELB
3135 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
3136 #    define ETHR_HAVE_NATMC_XCHG_RELB 1
3137 #  endif
3138 #  undef ETHR_HAVE_NATMC_XCHG_MB
3139 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
3140 #    define ETHR_HAVE_NATMC_XCHG_MB 1
3141 #  endif
3142 #  undef ETHR_HAVE_NATMC_SET
3143 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
3144 #    define ETHR_HAVE_NATMC_SET 1
3145 #  endif
3146 #  undef ETHR_HAVE_NATMC_SET_RB
3147 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
3148 #    define ETHR_HAVE_NATMC_SET_RB 1
3149 #  endif
3150 #  undef ETHR_HAVE_NATMC_SET_WB
3151 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
3152 #    define ETHR_HAVE_NATMC_SET_WB 1
3153 #  endif
3154 #  undef ETHR_HAVE_NATMC_SET_ACQB
3155 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
3156 #    define ETHR_HAVE_NATMC_SET_ACQB 1
3157 #  endif
3158 #  undef ETHR_HAVE_NATMC_SET_RELB
3159 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
3160 #    define ETHR_HAVE_NATMC_SET_RELB 1
3161 #  endif
3162 #  undef ETHR_HAVE_NATMC_SET_MB
3163 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
3164 #    define ETHR_HAVE_NATMC_SET_MB 1
3165 #  endif
3166 #  undef ETHR_HAVE_NATMC_INIT
3167 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
3168 #    define ETHR_HAVE_NATMC_INIT 1
3169 #  endif
3170 #  undef ETHR_HAVE_NATMC_INIT_RB
3171 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
3172 #    define ETHR_HAVE_NATMC_INIT_RB 1
3173 #  endif
3174 #  undef ETHR_HAVE_NATMC_INIT_WB
3175 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
3176 #    define ETHR_HAVE_NATMC_INIT_WB 1
3177 #  endif
3178 #  undef ETHR_HAVE_NATMC_INIT_ACQB
3179 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
3180 #    define ETHR_HAVE_NATMC_INIT_ACQB 1
3181 #  endif
3182 #  undef ETHR_HAVE_NATMC_INIT_RELB
3183 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
3184 #    define ETHR_HAVE_NATMC_INIT_RELB 1
3185 #  endif
3186 #  undef ETHR_HAVE_NATMC_INIT_MB
3187 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
3188 #    define ETHR_HAVE_NATMC_INIT_MB 1
3189 #  endif
3190 #  undef ETHR_HAVE_NATMC_ADD_RETURN
3191 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
3192 #    define ETHR_HAVE_NATMC_ADD_RETURN 1
3193 #  endif
3194 #  undef ETHR_HAVE_NATMC_ADD_RETURN_RB
3195 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
3196 #    define ETHR_HAVE_NATMC_ADD_RETURN_RB 1
3197 #  endif
3198 #  undef ETHR_HAVE_NATMC_ADD_RETURN_WB
3199 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
3200 #    define ETHR_HAVE_NATMC_ADD_RETURN_WB 1
3201 #  endif
3202 #  undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB
3203 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
3204 #    define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1
3205 #  endif
3206 #  undef ETHR_HAVE_NATMC_ADD_RETURN_RELB
3207 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
3208 #    define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1
3209 #  endif
3210 #  undef ETHR_HAVE_NATMC_ADD_RETURN_MB
3211 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
3212 #    define ETHR_HAVE_NATMC_ADD_RETURN_MB 1
3213 #  endif
3214 #  undef ETHR_HAVE_NATMC_READ
3215 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
3216 #    define ETHR_HAVE_NATMC_READ 1
3217 #  endif
3218 #  undef ETHR_HAVE_NATMC_READ_RB
3219 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
3220 #    define ETHR_HAVE_NATMC_READ_RB 1
3221 #  endif
3222 #  undef ETHR_HAVE_NATMC_READ_WB
3223 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
3224 #    define ETHR_HAVE_NATMC_READ_WB 1
3225 #  endif
3226 #  undef ETHR_HAVE_NATMC_READ_ACQB
3227 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
3228 #    define ETHR_HAVE_NATMC_READ_ACQB 1
3229 #  endif
3230 #  undef ETHR_HAVE_NATMC_READ_RELB
3231 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
3232 #    define ETHR_HAVE_NATMC_READ_RELB 1
3233 #  endif
3234 #  undef ETHR_HAVE_NATMC_READ_MB
3235 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
3236 #    define ETHR_HAVE_NATMC_READ_MB 1
3237 #  endif
3238 #  undef ETHR_HAVE_NATMC_INC_RETURN
3239 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
3240 #    define ETHR_HAVE_NATMC_INC_RETURN 1
3241 #  endif
3242 #  undef ETHR_HAVE_NATMC_INC_RETURN_RB
3243 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
3244 #    define ETHR_HAVE_NATMC_INC_RETURN_RB 1
3245 #  endif
3246 #  undef ETHR_HAVE_NATMC_INC_RETURN_WB
3247 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
3248 #    define ETHR_HAVE_NATMC_INC_RETURN_WB 1
3249 #  endif
3250 #  undef ETHR_HAVE_NATMC_INC_RETURN_ACQB
3251 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
3252 #    define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1
3253 #  endif
3254 #  undef ETHR_HAVE_NATMC_INC_RETURN_RELB
3255 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
3256 #    define ETHR_HAVE_NATMC_INC_RETURN_RELB 1
3257 #  endif
3258 #  undef ETHR_HAVE_NATMC_INC_RETURN_MB
3259 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
3260 #    define ETHR_HAVE_NATMC_INC_RETURN_MB 1
3261 #  endif
3262 #  undef ETHR_HAVE_NATMC_DEC_RETURN
3263 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
3264 #    define ETHR_HAVE_NATMC_DEC_RETURN 1
3265 #  endif
3266 #  undef ETHR_HAVE_NATMC_DEC_RETURN_RB
3267 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
3268 #    define ETHR_HAVE_NATMC_DEC_RETURN_RB 1
3269 #  endif
3270 #  undef ETHR_HAVE_NATMC_DEC_RETURN_WB
3271 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
3272 #    define ETHR_HAVE_NATMC_DEC_RETURN_WB 1
3273 #  endif
3274 #  undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB
3275 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
3276 #    define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1
3277 #  endif
3278 #  undef ETHR_HAVE_NATMC_DEC_RETURN_RELB
3279 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
3280 #    define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1
3281 #  endif
3282 #  undef ETHR_HAVE_NATMC_DEC_RETURN_MB
3283 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
3284 #    define ETHR_HAVE_NATMC_DEC_RETURN_MB 1
3285 #  endif
3286 #  undef ETHR_HAVE_NATMC_ADD
3287 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
3288 #    define ETHR_HAVE_NATMC_ADD 1
3289 #  endif
3290 #  undef ETHR_HAVE_NATMC_ADD_RB
3291 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
3292 #    define ETHR_HAVE_NATMC_ADD_RB 1
3293 #  endif
3294 #  undef ETHR_HAVE_NATMC_ADD_WB
3295 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
3296 #    define ETHR_HAVE_NATMC_ADD_WB 1
3297 #  endif
3298 #  undef ETHR_HAVE_NATMC_ADD_ACQB
3299 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
3300 #    define ETHR_HAVE_NATMC_ADD_ACQB 1
3301 #  endif
3302 #  undef ETHR_HAVE_NATMC_ADD_RELB
3303 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
3304 #    define ETHR_HAVE_NATMC_ADD_RELB 1
3305 #  endif
3306 #  undef ETHR_HAVE_NATMC_ADD_MB
3307 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
3308 #    define ETHR_HAVE_NATMC_ADD_MB 1
3309 #  endif
3310 #  undef ETHR_HAVE_NATMC_INC
3311 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
3312 #    define ETHR_HAVE_NATMC_INC 1
3313 #  endif
3314 #  undef ETHR_HAVE_NATMC_INC_RB
3315 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
3316 #    define ETHR_HAVE_NATMC_INC_RB 1
3317 #  endif
3318 #  undef ETHR_HAVE_NATMC_INC_WB
3319 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
3320 #    define ETHR_HAVE_NATMC_INC_WB 1
3321 #  endif
3322 #  undef ETHR_HAVE_NATMC_INC_ACQB
3323 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
3324 #    define ETHR_HAVE_NATMC_INC_ACQB 1
3325 #  endif
3326 #  undef ETHR_HAVE_NATMC_INC_RELB
3327 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
3328 #    define ETHR_HAVE_NATMC_INC_RELB 1
3329 #  endif
3330 #  undef ETHR_HAVE_NATMC_INC_MB
3331 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
3332 #    define ETHR_HAVE_NATMC_INC_MB 1
3333 #  endif
3334 #  undef ETHR_HAVE_NATMC_DEC
3335 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
3336 #    define ETHR_HAVE_NATMC_DEC 1
3337 #  endif
3338 #  undef ETHR_HAVE_NATMC_DEC_RB
3339 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
3340 #    define ETHR_HAVE_NATMC_DEC_RB 1
3341 #  endif
3342 #  undef ETHR_HAVE_NATMC_DEC_WB
3343 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
3344 #    define ETHR_HAVE_NATMC_DEC_WB 1
3345 #  endif
3346 #  undef ETHR_HAVE_NATMC_DEC_ACQB
3347 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
3348 #    define ETHR_HAVE_NATMC_DEC_ACQB 1
3349 #  endif
3350 #  undef ETHR_HAVE_NATMC_DEC_RELB
3351 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
3352 #    define ETHR_HAVE_NATMC_DEC_RELB 1
3353 #  endif
3354 #  undef ETHR_HAVE_NATMC_DEC_MB
3355 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
3356 #    define ETHR_HAVE_NATMC_DEC_MB 1
3357 #  endif
3358 #  undef ETHR_HAVE_NATMC_AND_RETOLD
3359 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
3360 #    define ETHR_HAVE_NATMC_AND_RETOLD 1
3361 #  endif
3362 #  undef ETHR_HAVE_NATMC_AND_RETOLD_RB
3363 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
3364 #    define ETHR_HAVE_NATMC_AND_RETOLD_RB 1
3365 #  endif
3366 #  undef ETHR_HAVE_NATMC_AND_RETOLD_WB
3367 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
3368 #    define ETHR_HAVE_NATMC_AND_RETOLD_WB 1
3369 #  endif
3370 #  undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB
3371 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
3372 #    define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1
3373 #  endif
3374 #  undef ETHR_HAVE_NATMC_AND_RETOLD_RELB
3375 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
3376 #    define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1
3377 #  endif
3378 #  undef ETHR_HAVE_NATMC_AND_RETOLD_MB
3379 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
3380 #    define ETHR_HAVE_NATMC_AND_RETOLD_MB 1
3381 #  endif
3382 #  undef ETHR_HAVE_NATMC_OR_RETOLD
3383 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
3384 #    define ETHR_HAVE_NATMC_OR_RETOLD 1
3385 #  endif
3386 #  undef ETHR_HAVE_NATMC_OR_RETOLD_RB
3387 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
3388 #    define ETHR_HAVE_NATMC_OR_RETOLD_RB 1
3389 #  endif
3390 #  undef ETHR_HAVE_NATMC_OR_RETOLD_WB
3391 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
3392 #    define ETHR_HAVE_NATMC_OR_RETOLD_WB 1
3393 #  endif
3394 #  undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB
3395 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
3396 #    define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1
3397 #  endif
3398 #  undef ETHR_HAVE_NATMC_OR_RETOLD_RELB
3399 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
3400 #    define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1
3401 #  endif
3402 #  undef ETHR_HAVE_NATMC_OR_RETOLD_MB
3403 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
3404 #    define ETHR_HAVE_NATMC_OR_RETOLD_MB 1
3405 #  endif
3406 #else
3407 #  error "Invalid native atomic size"
3408 #endif
3409 
3410 #if (!defined(ETHR_HAVE_NATMC_CMPXCHG) \
3411      && !defined(ETHR_HAVE_NATMC_CMPXCHG_RB) \
3412      && !defined(ETHR_HAVE_NATMC_CMPXCHG_WB) \
3413      && !defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) \
3414      && !defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) \
3415      && !defined(ETHR_HAVE_NATMC_CMPXCHG_MB))
3416 #  error "No native cmpxchg() op available"
3417 #endif
3418 
3419 
3420 /*
3421  * Read op used together with cmpxchg() fallback when no native op present.
3422  */
3423 #if defined(ETHR_HAVE_NATMC_READ)
3424 #define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
3425   ETHR_NATMC_FUNC__(read)(VAR)
3426 #elif defined(ETHR_HAVE_NATMC_READ_RB)
3427 #define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
3428   ETHR_NATMC_FUNC__(read_rb)(VAR)
3429 #elif defined(ETHR_HAVE_NATMC_READ_WB)
3430 #define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
3431   ETHR_NATMC_FUNC__(read_wb)(VAR)
3432 #elif defined(ETHR_HAVE_NATMC_READ_ACQB)
3433 #define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
3434   ETHR_NATMC_FUNC__(read_acqb)(VAR)
3435 #elif defined(ETHR_HAVE_NATMC_READ_RELB)
3436 #define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
3437   ETHR_NATMC_FUNC__(read_relb)(VAR)
3438 #elif defined(ETHR_HAVE_NATMC_READ_MB)
3439 #define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
3440   ETHR_NATMC_FUNC__(read_mb)(VAR)
3441 #else
3442 /*
3443  * We have no native read() op; guess zero and then use the
3444  * the atomics actual value returned from cmpxchg().
3445  */
3446 #define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
3447   ((ETHR_NAINT_T__) 0)
3448 #endif
3449 
3450 /*
3451  * Native cmpxchg() fallback used when no native op present.
3452  */
3453 #define ETHR_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
3454 do { \
3455     ethr_sint_t AVAL; \
3456     ETHR_NAINT_T__ new__, act__, exp__; \
3457     act__ = ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \
3458     do { \
3459 	exp__ = act__; \
3460 	AVAL = (ethr_sint_t) act__; \
3461 	{ OPS; } \
3462 	new__ = (ETHR_NAINT_T__) AVAL; \
3463 	act__ = CMPXCHG(VAR, new__, exp__); \
3464     } while (__builtin_expect(act__ != exp__, 0)); \
3465 } while (0)
3466 
3467 
3468 
3469 /* --- addr() --- */
3470 
ETHR_ATMC_FUNC__(addr)3471 static ETHR_INLINE ethr_sint_t *ETHR_ATMC_FUNC__(addr)(ethr_atomic_t *var)
3472 {
3473     return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var);
3474 
3475 }
3476 
3477 
3478 /* --- cmpxchg() --- */
3479 
3480 
ETHR_ATMC_FUNC__(cmpxchg)3481 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
3482 {
3483     ethr_sint_t res;
3484 #if defined(ETHR_HAVE_NATMC_CMPXCHG)
3485     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3486 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3487     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3488 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3489     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3490 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3491     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3492 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3493     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3494 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3495     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3496 #else
3497 #error "Missing implementation of ethr_atomic_cmpxchg()!"
3498 #endif
3499     return res;
3500 }
3501 
ETHR_ATMC_FUNC__(cmpxchg_rb)3502 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_rb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
3503 {
3504     ethr_sint_t res;
3505 #if defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3506     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3507 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3508     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3509     ETHR_MEMBAR(ETHR_LoadLoad);
3510 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3511     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3512 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3513     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3514     ETHR_MEMBAR(ETHR_LoadLoad);
3515 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3516     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3517     ETHR_MEMBAR(ETHR_LoadLoad);
3518 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3519     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3520     ETHR_MEMBAR(ETHR_LoadLoad);
3521 #else
3522 #error "Missing implementation of ethr_atomic_cmpxchg_rb()!"
3523 #endif
3524     return res;
3525 }
3526 
ETHR_ATMC_FUNC__(cmpxchg_wb)3527 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_wb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
3528 {
3529     ethr_sint_t res;
3530 #if defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3531     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3532 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3533     ETHR_MEMBAR(ETHR_StoreStore);
3534     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3535 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3536     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3537 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3538     ETHR_MEMBAR(ETHR_StoreStore);
3539     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3540 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3541     ETHR_MEMBAR(ETHR_StoreStore);
3542     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3543 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3544     ETHR_MEMBAR(ETHR_StoreStore);
3545     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3546 #else
3547 #error "Missing implementation of ethr_atomic_cmpxchg_wb()!"
3548 #endif
3549     return res;
3550 }
3551 
ETHR_ATMC_FUNC__(cmpxchg_acqb)3552 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_acqb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
3553 {
3554     ethr_sint_t res;
3555 #if defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3556     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3557 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3558     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3559     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3560 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3561     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3562     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3563 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3564     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3565 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3566     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3567     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3568 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3569     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3570     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3571 #else
3572 #error "Missing implementation of ethr_atomic_cmpxchg_acqb()!"
3573 #endif
3574     return res;
3575 }
3576 
ETHR_ATMC_FUNC__(cmpxchg_relb)3577 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_relb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
3578 {
3579     ethr_sint_t res;
3580 #if defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3581     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3582 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3583     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3584     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3585 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3586     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3587     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3588 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3589     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3590 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3591     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3592     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3593 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3594     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3595     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3596 #else
3597 #error "Missing implementation of ethr_atomic_cmpxchg_relb()!"
3598 #endif
3599     return res;
3600 }
3601 
ETHR_ATMC_FUNC__(cmpxchg_mb)3602 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_mb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
3603 {
3604     ethr_sint_t res;
3605 #if defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3606     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3607 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3608     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3609     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3610 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3611     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3612     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3613 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3614     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3615     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3616     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3617 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3618     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3619     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3620     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3621 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3622     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3623     res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
3624     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3625 #else
3626 #error "Missing implementation of ethr_atomic_cmpxchg_mb()!"
3627 #endif
3628     return res;
3629 }
3630 
ETHR_ATMC_FUNC__(cmpxchg_ddrb)3631 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_ddrb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
3632 {
3633 #ifdef ETHR_ORDERED_READ_DEPEND
3634     return ETHR_ATMC_FUNC__(cmpxchg)(var, val, old_val);
3635 #else
3636     return ETHR_ATMC_FUNC__(cmpxchg_rb)(var, val, old_val);
3637 #endif
3638 }
3639 
3640 
3641 /* --- xchg() --- */
3642 
3643 
ETHR_ATMC_FUNC__(xchg)3644 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg)(ethr_atomic_t *var, ethr_sint_t val)
3645 {
3646     ethr_sint_t res;
3647 #if defined(ETHR_HAVE_NATMC_XCHG)
3648     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
3649 #elif defined(ETHR_HAVE_NATMC_XCHG_RB)
3650     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
3651 #elif defined(ETHR_HAVE_NATMC_XCHG_WB)
3652     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
3653 #elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
3654     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
3655 #elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
3656     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
3657 #elif defined(ETHR_HAVE_NATMC_XCHG_MB)
3658     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
3659 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3660     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
3661 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3662     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
3663 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3664     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
3665 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3666     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
3667 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3668     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
3669 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3670     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
3671 #else
3672 #error "Missing implementation of ethr_atomic_xchg()!"
3673 #endif
3674     return res;
3675 }
3676 
ETHR_ATMC_FUNC__(xchg_rb)3677 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_rb)(ethr_atomic_t *var, ethr_sint_t val)
3678 {
3679     ethr_sint_t res;
3680 #if defined(ETHR_HAVE_NATMC_XCHG_RB)
3681     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
3682 #elif defined(ETHR_HAVE_NATMC_XCHG)
3683     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
3684     ETHR_MEMBAR(ETHR_LoadLoad);
3685 #elif defined(ETHR_HAVE_NATMC_XCHG_MB)
3686     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
3687 #elif defined(ETHR_HAVE_NATMC_XCHG_WB)
3688     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
3689     ETHR_MEMBAR(ETHR_LoadLoad);
3690 #elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
3691     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
3692     ETHR_MEMBAR(ETHR_LoadLoad);
3693 #elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
3694     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
3695     ETHR_MEMBAR(ETHR_LoadLoad);
3696 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3697     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
3698 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3699     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
3700     ETHR_MEMBAR(ETHR_LoadLoad);
3701 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3702     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
3703 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3704     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
3705     ETHR_MEMBAR(ETHR_LoadLoad);
3706 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3707     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
3708     ETHR_MEMBAR(ETHR_LoadLoad);
3709 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3710     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
3711     ETHR_MEMBAR(ETHR_LoadLoad);
3712 #else
3713 #error "Missing implementation of ethr_atomic_xchg_rb()!"
3714 #endif
3715     return res;
3716 }
3717 
ETHR_ATMC_FUNC__(xchg_wb)3718 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_wb)(ethr_atomic_t *var, ethr_sint_t val)
3719 {
3720     ethr_sint_t res;
3721 #if defined(ETHR_HAVE_NATMC_XCHG_WB)
3722     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
3723 #elif defined(ETHR_HAVE_NATMC_XCHG)
3724     ETHR_MEMBAR(ETHR_StoreStore);
3725     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
3726 #elif defined(ETHR_HAVE_NATMC_XCHG_MB)
3727     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
3728 #elif defined(ETHR_HAVE_NATMC_XCHG_RB)
3729     ETHR_MEMBAR(ETHR_StoreStore);
3730     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
3731 #elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
3732     ETHR_MEMBAR(ETHR_StoreStore);
3733     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
3734 #elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
3735     ETHR_MEMBAR(ETHR_StoreStore);
3736     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
3737 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3738     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
3739 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3740     ETHR_MEMBAR(ETHR_StoreStore);
3741     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
3742 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3743     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
3744 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3745     ETHR_MEMBAR(ETHR_StoreStore);
3746     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
3747 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3748     ETHR_MEMBAR(ETHR_StoreStore);
3749     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
3750 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3751     ETHR_MEMBAR(ETHR_StoreStore);
3752     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
3753 #else
3754 #error "Missing implementation of ethr_atomic_xchg_wb()!"
3755 #endif
3756     return res;
3757 }
3758 
ETHR_ATMC_FUNC__(xchg_acqb)3759 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_acqb)(ethr_atomic_t *var, ethr_sint_t val)
3760 {
3761     ethr_sint_t res;
3762 #if defined(ETHR_HAVE_NATMC_XCHG_ACQB)
3763     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
3764 #elif defined(ETHR_HAVE_NATMC_XCHG_RB)
3765     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
3766     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3767 #elif defined(ETHR_HAVE_NATMC_XCHG)
3768     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
3769     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3770 #elif defined(ETHR_HAVE_NATMC_XCHG_MB)
3771     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
3772 #elif defined(ETHR_HAVE_NATMC_XCHG_WB)
3773     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
3774     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3775 #elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
3776     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
3777     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3778 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3779     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
3780 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3781     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
3782     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3783 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3784     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
3785     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3786 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3787     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
3788 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3789     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
3790     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3791 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3792     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
3793     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3794 #else
3795 #error "Missing implementation of ethr_atomic_xchg_acqb()!"
3796 #endif
3797     return res;
3798 }
3799 
ETHR_ATMC_FUNC__(xchg_relb)3800 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_relb)(ethr_atomic_t *var, ethr_sint_t val)
3801 {
3802     ethr_sint_t res;
3803 #if defined(ETHR_HAVE_NATMC_XCHG_RELB)
3804     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
3805 #elif defined(ETHR_HAVE_NATMC_XCHG_WB)
3806     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3807     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
3808 #elif defined(ETHR_HAVE_NATMC_XCHG)
3809     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3810     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
3811 #elif defined(ETHR_HAVE_NATMC_XCHG_MB)
3812     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
3813 #elif defined(ETHR_HAVE_NATMC_XCHG_RB)
3814     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3815     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
3816 #elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
3817     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3818     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
3819 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3820     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
3821 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3822     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3823     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
3824 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3825     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3826     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
3827 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3828     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
3829 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3830     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3831     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
3832 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3833     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3834     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
3835 #else
3836 #error "Missing implementation of ethr_atomic_xchg_relb()!"
3837 #endif
3838     return res;
3839 }
3840 
ETHR_ATMC_FUNC__(xchg_mb)3841 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_mb)(ethr_atomic_t *var, ethr_sint_t val)
3842 {
3843     ethr_sint_t res;
3844 #if defined(ETHR_HAVE_NATMC_XCHG_MB)
3845     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
3846 #elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
3847     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
3848     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3849 #elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
3850     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3851     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
3852 #elif defined(ETHR_HAVE_NATMC_XCHG_WB)
3853     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3854     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
3855     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3856 #elif defined(ETHR_HAVE_NATMC_XCHG_RB)
3857     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3858     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
3859     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3860 #elif defined(ETHR_HAVE_NATMC_XCHG)
3861     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3862     res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
3863     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3864 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
3865     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
3866 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
3867     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
3868     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3869 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
3870     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3871     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
3872 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
3873     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3874     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
3875     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3876 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
3877     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3878     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
3879     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3880 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
3881     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
3882     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
3883     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
3884 #else
3885 #error "Missing implementation of ethr_atomic_xchg_mb()!"
3886 #endif
3887     return res;
3888 }
3889 
ETHR_ATMC_FUNC__(xchg_ddrb)3890 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
3891 {
3892 #ifdef ETHR_ORDERED_READ_DEPEND
3893     return ETHR_ATMC_FUNC__(xchg)(var, val);
3894 #else
3895     return ETHR_ATMC_FUNC__(xchg_rb)(var, val);
3896 #endif
3897 }
3898 
3899 
3900 /* --- set() --- */
3901 
3902 
ETHR_ATMC_FUNC__(set)3903 static ETHR_INLINE void ETHR_ATMC_FUNC__(set)(ethr_atomic_t *var, ethr_sint_t val)
3904 {
3905 #if defined(ETHR_HAVE_NATMC_SET)
3906     ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
3907 #elif defined(ETHR_HAVE_NATMC_SET_RB)
3908     ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
3909 #elif defined(ETHR_HAVE_NATMC_SET_WB)
3910     ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
3911 #elif defined(ETHR_HAVE_NATMC_SET_ACQB)
3912     ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
3913 #elif defined(ETHR_HAVE_NATMC_SET_RELB)
3914     ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
3915 #elif defined(ETHR_HAVE_NATMC_SET_MB)
3916     ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
3917 #else
3918     (void) ETHR_ATMC_FUNC__(xchg)(var, val);
3919 #endif
3920 }
3921 
ETHR_ATMC_FUNC__(set_rb)3922 static ETHR_INLINE void ETHR_ATMC_FUNC__(set_rb)(ethr_atomic_t *var, ethr_sint_t val)
3923 {
3924 #if defined(ETHR_HAVE_NATMC_SET_RB)
3925     ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
3926 #elif defined(ETHR_HAVE_NATMC_SET)
3927     ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
3928     ETHR_MEMBAR(ETHR_LoadLoad);
3929 #elif defined(ETHR_HAVE_NATMC_SET_MB)
3930     ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
3931 #elif defined(ETHR_HAVE_NATMC_SET_WB)
3932     ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
3933     ETHR_MEMBAR(ETHR_LoadLoad);
3934 #elif defined(ETHR_HAVE_NATMC_SET_ACQB)
3935     ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
3936     ETHR_MEMBAR(ETHR_LoadLoad);
3937 #elif defined(ETHR_HAVE_NATMC_SET_RELB)
3938     ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
3939     ETHR_MEMBAR(ETHR_LoadLoad);
3940 #else
3941     (void) ETHR_ATMC_FUNC__(xchg_rb)(var, val);
3942 #endif
3943 }
3944 
ETHR_ATMC_FUNC__(set_wb)3945 static ETHR_INLINE void ETHR_ATMC_FUNC__(set_wb)(ethr_atomic_t *var, ethr_sint_t val)
3946 {
3947 #if defined(ETHR_HAVE_NATMC_SET_WB)
3948     ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
3949 #elif defined(ETHR_HAVE_NATMC_SET)
3950     ETHR_MEMBAR(ETHR_StoreStore);
3951     ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
3952 #elif defined(ETHR_HAVE_NATMC_SET_MB)
3953     ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
3954 #elif defined(ETHR_HAVE_NATMC_SET_RB)
3955     ETHR_MEMBAR(ETHR_StoreStore);
3956     ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
3957 #elif defined(ETHR_HAVE_NATMC_SET_ACQB)
3958     ETHR_MEMBAR(ETHR_StoreStore);
3959     ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
3960 #elif defined(ETHR_HAVE_NATMC_SET_RELB)
3961     ETHR_MEMBAR(ETHR_StoreStore);
3962     ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
3963 #else
3964     (void) ETHR_ATMC_FUNC__(xchg_wb)(var, val);
3965 #endif
3966 }
3967 
ETHR_ATMC_FUNC__(set_acqb)3968 static ETHR_INLINE void ETHR_ATMC_FUNC__(set_acqb)(ethr_atomic_t *var, ethr_sint_t val)
3969 {
3970 #if defined(ETHR_HAVE_NATMC_SET_ACQB)
3971     ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
3972 #elif defined(ETHR_HAVE_NATMC_SET_RB)
3973     ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
3974     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
3975 #elif defined(ETHR_HAVE_NATMC_SET)
3976     ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
3977     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
3978 #elif defined(ETHR_HAVE_NATMC_SET_MB)
3979     ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
3980 #elif defined(ETHR_HAVE_NATMC_SET_WB)
3981     ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
3982     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
3983 #elif defined(ETHR_HAVE_NATMC_SET_RELB)
3984     ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
3985     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
3986 #else
3987     (void) ETHR_ATMC_FUNC__(xchg_acqb)(var, val);
3988 #endif
3989 }
3990 
ETHR_ATMC_FUNC__(set_relb)3991 static ETHR_INLINE void ETHR_ATMC_FUNC__(set_relb)(ethr_atomic_t *var, ethr_sint_t val)
3992 {
3993 #if defined(ETHR_HAVE_NATMC_SET_RELB)
3994     ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
3995 #elif defined(ETHR_HAVE_NATMC_SET_WB)
3996     ETHR_MEMBAR(ETHR_LoadStore);
3997     ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
3998 #elif defined(ETHR_HAVE_NATMC_SET)
3999     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
4000     ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
4001 #elif defined(ETHR_HAVE_NATMC_SET_MB)
4002     ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
4003 #elif defined(ETHR_HAVE_NATMC_SET_RB)
4004     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
4005     ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
4006 #elif defined(ETHR_HAVE_NATMC_SET_ACQB)
4007     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
4008     ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
4009 #else
4010     (void) ETHR_ATMC_FUNC__(xchg_relb)(var, val);
4011 #endif
4012 }
4013 
ETHR_ATMC_FUNC__(set_mb)4014 static ETHR_INLINE void ETHR_ATMC_FUNC__(set_mb)(ethr_atomic_t *var, ethr_sint_t val)
4015 {
4016 #if defined(ETHR_HAVE_NATMC_SET_MB)
4017     ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
4018 #elif defined(ETHR_HAVE_NATMC_SET_RELB)
4019     ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
4020     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
4021 #elif defined(ETHR_HAVE_NATMC_SET_ACQB)
4022     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
4023     ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
4024 #elif defined(ETHR_HAVE_NATMC_SET_WB)
4025     ETHR_MEMBAR(ETHR_LoadStore);
4026     ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
4027     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
4028 #elif defined(ETHR_HAVE_NATMC_SET_RB)
4029     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
4030     ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
4031     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
4032 #elif defined(ETHR_HAVE_NATMC_SET)
4033     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
4034     ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
4035     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
4036 #else
4037     (void) ETHR_ATMC_FUNC__(xchg_mb)(var, val);
4038 #endif
4039 }
4040 
ETHR_ATMC_FUNC__(set_ddrb)4041 static ETHR_INLINE void ETHR_ATMC_FUNC__(set_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
4042 {
4043 #ifdef ETHR_ORDERED_READ_DEPEND
4044     ETHR_ATMC_FUNC__(set)(var, val);
4045 #else
4046     ETHR_ATMC_FUNC__(set_rb)(var, val);
4047 #endif
4048 }
4049 
4050 
4051 /* --- init() --- */
4052 
4053 
ETHR_ATMC_FUNC__(init)4054 static ETHR_INLINE void ETHR_ATMC_FUNC__(init)(ethr_atomic_t *var, ethr_sint_t val)
4055 {
4056 #if defined(ETHR_HAVE_NATMC_INIT)
4057     ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
4058 #elif defined(ETHR_HAVE_NATMC_INIT_RB)
4059     ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
4060 #elif defined(ETHR_HAVE_NATMC_INIT_WB)
4061     ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
4062 #elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
4063     ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
4064 #elif defined(ETHR_HAVE_NATMC_INIT_RELB)
4065     ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
4066 #elif defined(ETHR_HAVE_NATMC_INIT_MB)
4067     ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
4068 #else
4069     ETHR_ATMC_FUNC__(set)(var, val);
4070 #endif
4071 }
4072 
ETHR_ATMC_FUNC__(init_rb)4073 static ETHR_INLINE void ETHR_ATMC_FUNC__(init_rb)(ethr_atomic_t *var, ethr_sint_t val)
4074 {
4075 #if defined(ETHR_HAVE_NATMC_INIT_RB)
4076     ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
4077 #elif defined(ETHR_HAVE_NATMC_INIT)
4078     ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
4079     ETHR_MEMBAR(ETHR_LoadLoad);
4080 #elif defined(ETHR_HAVE_NATMC_INIT_MB)
4081     ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
4082 #elif defined(ETHR_HAVE_NATMC_INIT_WB)
4083     ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
4084     ETHR_MEMBAR(ETHR_LoadLoad);
4085 #elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
4086     ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
4087     ETHR_MEMBAR(ETHR_LoadLoad);
4088 #elif defined(ETHR_HAVE_NATMC_INIT_RELB)
4089     ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
4090     ETHR_MEMBAR(ETHR_LoadLoad);
4091 #else
4092     ETHR_ATMC_FUNC__(set_rb)(var, val);
4093 #endif
4094 }
4095 
ETHR_ATMC_FUNC__(init_wb)4096 static ETHR_INLINE void ETHR_ATMC_FUNC__(init_wb)(ethr_atomic_t *var, ethr_sint_t val)
4097 {
4098 #if defined(ETHR_HAVE_NATMC_INIT_WB)
4099     ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
4100 #elif defined(ETHR_HAVE_NATMC_INIT)
4101     ETHR_MEMBAR(ETHR_StoreStore);
4102     ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
4103 #elif defined(ETHR_HAVE_NATMC_INIT_MB)
4104     ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
4105 #elif defined(ETHR_HAVE_NATMC_INIT_RB)
4106     ETHR_MEMBAR(ETHR_StoreStore);
4107     ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
4108 #elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
4109     ETHR_MEMBAR(ETHR_StoreStore);
4110     ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
4111 #elif defined(ETHR_HAVE_NATMC_INIT_RELB)
4112     ETHR_MEMBAR(ETHR_StoreStore);
4113     ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
4114 #else
4115     ETHR_ATMC_FUNC__(set_wb)(var, val);
4116 #endif
4117 }
4118 
ETHR_ATMC_FUNC__(init_acqb)4119 static ETHR_INLINE void ETHR_ATMC_FUNC__(init_acqb)(ethr_atomic_t *var, ethr_sint_t val)
4120 {
4121 #if defined(ETHR_HAVE_NATMC_INIT_ACQB)
4122     ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
4123 #elif defined(ETHR_HAVE_NATMC_INIT_RB)
4124     ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
4125     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4126 #elif defined(ETHR_HAVE_NATMC_INIT)
4127     ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
4128     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4129 #elif defined(ETHR_HAVE_NATMC_INIT_MB)
4130     ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
4131 #elif defined(ETHR_HAVE_NATMC_INIT_WB)
4132     ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
4133     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4134 #elif defined(ETHR_HAVE_NATMC_INIT_RELB)
4135     ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
4136     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4137 #else
4138     ETHR_ATMC_FUNC__(set_acqb)(var, val);
4139 #endif
4140 }
4141 
ETHR_ATMC_FUNC__(init_relb)4142 static ETHR_INLINE void ETHR_ATMC_FUNC__(init_relb)(ethr_atomic_t *var, ethr_sint_t val)
4143 {
4144 #if defined(ETHR_HAVE_NATMC_INIT_RELB)
4145     ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
4146 #elif defined(ETHR_HAVE_NATMC_INIT_WB)
4147     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
4148     ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
4149 #elif defined(ETHR_HAVE_NATMC_INIT)
4150     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4151     ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
4152 #elif defined(ETHR_HAVE_NATMC_INIT_MB)
4153     ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
4154 #elif defined(ETHR_HAVE_NATMC_INIT_RB)
4155     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4156     ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
4157 #elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
4158     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4159     ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
4160 #else
4161     ETHR_ATMC_FUNC__(set_relb)(var, val);
4162 #endif
4163 }
4164 
ETHR_ATMC_FUNC__(init_mb)4165 static ETHR_INLINE void ETHR_ATMC_FUNC__(init_mb)(ethr_atomic_t *var, ethr_sint_t val)
4166 {
4167 #if defined(ETHR_HAVE_NATMC_INIT_MB)
4168     ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
4169 #elif defined(ETHR_HAVE_NATMC_INIT_RELB)
4170     ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
4171     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4172 #elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
4173     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4174     ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
4175 #elif defined(ETHR_HAVE_NATMC_INIT_WB)
4176     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
4177     ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
4178     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4179 #elif defined(ETHR_HAVE_NATMC_INIT_RB)
4180     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4181     ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
4182     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4183 #elif defined(ETHR_HAVE_NATMC_INIT)
4184     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4185     ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
4186     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4187 #else
4188     ETHR_ATMC_FUNC__(set_mb)(var, val);
4189 #endif
4190 }
4191 
ETHR_ATMC_FUNC__(init_ddrb)4192 static ETHR_INLINE void ETHR_ATMC_FUNC__(init_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
4193 {
4194 #ifdef ETHR_ORDERED_READ_DEPEND
4195     ETHR_ATMC_FUNC__(init)(var, val);
4196 #else
4197     ETHR_ATMC_FUNC__(init_rb)(var, val);
4198 #endif
4199 }
4200 
4201 
4202 /* --- add_read() --- */
4203 
4204 
ETHR_ATMC_FUNC__(add_read)4205 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read)(ethr_atomic_t *var, ethr_sint_t val)
4206 {
4207     ethr_sint_t res;
4208 #if defined(ETHR_HAVE_NATMC_ADD_RETURN)
4209     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
4210 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
4211     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
4212 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
4213     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
4214 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
4215     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
4216 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
4217     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
4218 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
4219     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
4220 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
4221     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
4222 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
4223     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
4224 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
4225     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
4226 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
4227     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
4228 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
4229     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
4230 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
4231     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
4232 #else
4233 #error "Missing implementation of ethr_atomic_add_read()!"
4234 #endif
4235     return res;
4236 }
4237 
ETHR_ATMC_FUNC__(add_read_rb)4238 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_rb)(ethr_atomic_t *var, ethr_sint_t val)
4239 {
4240     ethr_sint_t res;
4241 #if defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
4242     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
4243 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
4244     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
4245     ETHR_MEMBAR(ETHR_LoadLoad);
4246 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
4247     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
4248 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
4249     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
4250     ETHR_MEMBAR(ETHR_LoadLoad);
4251 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
4252     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
4253     ETHR_MEMBAR(ETHR_LoadLoad);
4254 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
4255     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
4256     ETHR_MEMBAR(ETHR_LoadLoad);
4257 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
4258     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
4259 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
4260     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
4261     ETHR_MEMBAR(ETHR_LoadLoad);
4262 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
4263     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
4264 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
4265     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
4266     ETHR_MEMBAR(ETHR_LoadLoad);
4267 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
4268     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
4269     ETHR_MEMBAR(ETHR_LoadLoad);
4270 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
4271     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
4272     ETHR_MEMBAR(ETHR_LoadLoad);
4273 #else
4274 #error "Missing implementation of ethr_atomic_add_read_rb()!"
4275 #endif
4276     return res;
4277 }
4278 
ETHR_ATMC_FUNC__(add_read_wb)4279 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_wb)(ethr_atomic_t *var, ethr_sint_t val)
4280 {
4281     ethr_sint_t res;
4282 #if defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
4283     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
4284 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
4285     ETHR_MEMBAR(ETHR_StoreStore);
4286     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
4287 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
4288     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
4289 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
4290     ETHR_MEMBAR(ETHR_StoreStore);
4291     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
4292 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
4293     ETHR_MEMBAR(ETHR_StoreStore);
4294     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
4295 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
4296     ETHR_MEMBAR(ETHR_StoreStore);
4297     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
4298 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
4299     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
4300 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
4301     ETHR_MEMBAR(ETHR_StoreStore);
4302     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
4303 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
4304     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
4305 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
4306     ETHR_MEMBAR(ETHR_StoreStore);
4307     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
4308 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
4309     ETHR_MEMBAR(ETHR_StoreStore);
4310     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
4311 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
4312     ETHR_MEMBAR(ETHR_StoreStore);
4313     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
4314 #else
4315 #error "Missing implementation of ethr_atomic_add_read_wb()!"
4316 #endif
4317     return res;
4318 }
4319 
ETHR_ATMC_FUNC__(add_read_acqb)4320 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_acqb)(ethr_atomic_t *var, ethr_sint_t val)
4321 {
4322     ethr_sint_t res;
4323 #if defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
4324     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
4325 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
4326     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
4327     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4328 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
4329     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
4330     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4331 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
4332     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
4333 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
4334     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
4335     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4336 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
4337     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
4338     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4339 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
4340     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
4341 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
4342     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
4343     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4344 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
4345     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
4346     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4347 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
4348     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
4349 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
4350     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
4351     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4352 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
4353     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
4354     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4355 #else
4356 #error "Missing implementation of ethr_atomic_add_read_acqb()!"
4357 #endif
4358     return res;
4359 }
4360 
ETHR_ATMC_FUNC__(add_read_relb)4361 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_relb)(ethr_atomic_t *var, ethr_sint_t val)
4362 {
4363     ethr_sint_t res;
4364 #if defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
4365     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
4366 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
4367     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4368     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
4369 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
4370     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4371     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
4372 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
4373     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
4374 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
4375     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4376     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
4377 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
4378     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4379     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
4380 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
4381     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
4382 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
4383     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4384     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
4385 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
4386     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4387     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
4388 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
4389     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
4390 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
4391     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4392     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
4393 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
4394     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4395     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
4396 #else
4397 #error "Missing implementation of ethr_atomic_add_read_relb()!"
4398 #endif
4399     return res;
4400 }
4401 
ETHR_ATMC_FUNC__(add_read_mb)4402 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_mb)(ethr_atomic_t *var, ethr_sint_t val)
4403 {
4404     ethr_sint_t res;
4405 #if defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
4406     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
4407 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
4408     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
4409     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4410 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
4411     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4412     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
4413 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
4414     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4415     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
4416     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4417 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
4418     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4419     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
4420     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4421 #elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
4422     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4423     res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
4424     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4425 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
4426     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
4427 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
4428     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
4429     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4430 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
4431     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4432     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
4433 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
4434     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4435     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
4436     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4437 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
4438     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4439     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
4440     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4441 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
4442     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4443     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
4444     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4445 #else
4446 #error "Missing implementation of ethr_atomic_add_read_mb()!"
4447 #endif
4448     return res;
4449 }
4450 
ETHR_ATMC_FUNC__(add_read_ddrb)4451 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
4452 {
4453 #ifdef ETHR_ORDERED_READ_DEPEND
4454     return ETHR_ATMC_FUNC__(add_read)(var, val);
4455 #else
4456     return ETHR_ATMC_FUNC__(add_read_rb)(var, val);
4457 #endif
4458 }
4459 
4460 
4461 /* --- read() --- */
4462 
4463 
ETHR_ATMC_FUNC__(read)4464 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read)(ethr_atomic_t *var)
4465 {
4466     ethr_sint_t res;
4467 #if defined(ETHR_HAVE_NATMC_READ)
4468     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
4469 #elif defined(ETHR_HAVE_NATMC_READ_RB)
4470     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
4471 #elif defined(ETHR_HAVE_NATMC_READ_WB)
4472     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
4473 #elif defined(ETHR_HAVE_NATMC_READ_ACQB)
4474     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
4475 #elif defined(ETHR_HAVE_NATMC_READ_RELB)
4476     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
4477 #elif defined(ETHR_HAVE_NATMC_READ_MB)
4478     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
4479 #else
4480     res = ETHR_ATMC_FUNC__(cmpxchg)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
4481 #endif
4482     return res;
4483 }
4484 
ETHR_ATMC_FUNC__(read_rb)4485 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_rb)(ethr_atomic_t *var)
4486 {
4487     ethr_sint_t res;
4488 #if defined(ETHR_HAVE_NATMC_READ_RB)
4489     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
4490 #elif defined(ETHR_HAVE_NATMC_READ)
4491     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
4492     ETHR_MEMBAR(ETHR_LoadLoad);
4493 #elif defined(ETHR_HAVE_NATMC_READ_MB)
4494     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
4495 #elif defined(ETHR_HAVE_NATMC_READ_WB)
4496     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
4497     ETHR_MEMBAR(ETHR_LoadLoad);
4498 #elif defined(ETHR_HAVE_NATMC_READ_ACQB)
4499     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
4500     ETHR_MEMBAR(ETHR_LoadLoad);
4501 #elif defined(ETHR_HAVE_NATMC_READ_RELB)
4502     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
4503     ETHR_MEMBAR(ETHR_LoadLoad);
4504 #else
4505     res = ETHR_ATMC_FUNC__(cmpxchg_rb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
4506 #endif
4507     return res;
4508 }
4509 
ETHR_ATMC_FUNC__(read_wb)4510 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_wb)(ethr_atomic_t *var)
4511 {
4512     ethr_sint_t res;
4513 #if defined(ETHR_HAVE_NATMC_READ_WB)
4514     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
4515 #elif defined(ETHR_HAVE_NATMC_READ)
4516     ETHR_MEMBAR(ETHR_StoreStore);
4517     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
4518 #elif defined(ETHR_HAVE_NATMC_READ_MB)
4519     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
4520 #elif defined(ETHR_HAVE_NATMC_READ_RB)
4521     ETHR_MEMBAR(ETHR_StoreStore);
4522     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
4523 #elif defined(ETHR_HAVE_NATMC_READ_ACQB)
4524     ETHR_MEMBAR(ETHR_StoreStore);
4525     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
4526 #elif defined(ETHR_HAVE_NATMC_READ_RELB)
4527     ETHR_MEMBAR(ETHR_StoreStore);
4528     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
4529 #else
4530     res = ETHR_ATMC_FUNC__(cmpxchg_wb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
4531 #endif
4532     return res;
4533 }
4534 
ETHR_ATMC_FUNC__(read_acqb)4535 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_acqb)(ethr_atomic_t *var)
4536 {
4537     ethr_sint_t res;
4538 #if defined(ETHR_HAVE_NATMC_READ_ACQB)
4539     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
4540 #elif defined(ETHR_HAVE_NATMC_READ_RB)
4541     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
4542     ETHR_MEMBAR(ETHR_LoadStore);
4543 #elif defined(ETHR_HAVE_NATMC_READ)
4544     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
4545     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
4546 #elif defined(ETHR_HAVE_NATMC_READ_MB)
4547     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
4548 #elif defined(ETHR_HAVE_NATMC_READ_WB)
4549     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
4550     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
4551 #elif defined(ETHR_HAVE_NATMC_READ_RELB)
4552     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
4553     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
4554 #else
4555     res = ETHR_ATMC_FUNC__(cmpxchg_acqb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
4556 #endif
4557     return res;
4558 }
4559 
ETHR_ATMC_FUNC__(read_relb)4560 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_relb)(ethr_atomic_t *var)
4561 {
4562     ethr_sint_t res;
4563 #if defined(ETHR_HAVE_NATMC_READ_RELB)
4564     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
4565 #elif defined(ETHR_HAVE_NATMC_READ_WB)
4566     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4567     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
4568 #elif defined(ETHR_HAVE_NATMC_READ)
4569     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4570     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
4571 #elif defined(ETHR_HAVE_NATMC_READ_MB)
4572     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
4573 #elif defined(ETHR_HAVE_NATMC_READ_RB)
4574     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4575     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
4576 #elif defined(ETHR_HAVE_NATMC_READ_ACQB)
4577     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4578     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
4579 #else
4580     res = ETHR_ATMC_FUNC__(cmpxchg_relb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
4581 #endif
4582     return res;
4583 }
4584 
ETHR_ATMC_FUNC__(read_mb)4585 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_mb)(ethr_atomic_t *var)
4586 {
4587     ethr_sint_t res;
4588 #if defined(ETHR_HAVE_NATMC_READ_MB)
4589     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
4590 #elif defined(ETHR_HAVE_NATMC_READ_RELB)
4591     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
4592     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
4593 #elif defined(ETHR_HAVE_NATMC_READ_ACQB)
4594     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4595     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
4596 #elif defined(ETHR_HAVE_NATMC_READ_WB)
4597     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4598     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
4599     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
4600 #elif defined(ETHR_HAVE_NATMC_READ_RB)
4601     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4602     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
4603     ETHR_MEMBAR(ETHR_LoadStore);
4604 #elif defined(ETHR_HAVE_NATMC_READ)
4605     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4606     res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
4607     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
4608 #else
4609     res = ETHR_ATMC_FUNC__(cmpxchg_mb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
4610 #endif
4611     return res;
4612 }
4613 
ETHR_ATMC_FUNC__(read_ddrb)4614 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_ddrb)(ethr_atomic_t *var)
4615 {
4616 #ifdef ETHR_ORDERED_READ_DEPEND
4617     return ETHR_ATMC_FUNC__(read)(var);
4618 #else
4619     return ETHR_ATMC_FUNC__(read_rb)(var);
4620 #endif
4621 }
4622 
4623 
4624 /* --- inc_read() --- */
4625 
4626 
ETHR_ATMC_FUNC__(inc_read)4627 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read)(ethr_atomic_t *var)
4628 {
4629     ethr_sint_t res;
4630 #if defined(ETHR_HAVE_NATMC_INC_RETURN)
4631     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
4632 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
4633     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
4634 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
4635     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
4636 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
4637     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
4638 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
4639     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
4640 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
4641     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
4642 #else
4643     res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) 1);
4644 #endif
4645     return res;
4646 }
4647 
ETHR_ATMC_FUNC__(inc_read_rb)4648 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_rb)(ethr_atomic_t *var)
4649 {
4650     ethr_sint_t res;
4651 #if defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
4652     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
4653 #elif defined(ETHR_HAVE_NATMC_INC_RETURN)
4654     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
4655     ETHR_MEMBAR(ETHR_LoadLoad);
4656 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
4657     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
4658 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
4659     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
4660     ETHR_MEMBAR(ETHR_LoadLoad);
4661 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
4662     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
4663     ETHR_MEMBAR(ETHR_LoadLoad);
4664 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
4665     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
4666     ETHR_MEMBAR(ETHR_LoadLoad);
4667 #else
4668     res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) 1);
4669 #endif
4670     return res;
4671 }
4672 
ETHR_ATMC_FUNC__(inc_read_wb)4673 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_wb)(ethr_atomic_t *var)
4674 {
4675     ethr_sint_t res;
4676 #if defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
4677     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
4678 #elif defined(ETHR_HAVE_NATMC_INC_RETURN)
4679     ETHR_MEMBAR(ETHR_StoreStore);
4680     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
4681 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
4682     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
4683 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
4684     ETHR_MEMBAR(ETHR_StoreStore);
4685     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
4686 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
4687     ETHR_MEMBAR(ETHR_StoreStore);
4688     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
4689 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
4690     ETHR_MEMBAR(ETHR_StoreStore);
4691     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
4692 #else
4693     res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) 1);
4694 #endif
4695     return res;
4696 }
4697 
ETHR_ATMC_FUNC__(inc_read_acqb)4698 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_acqb)(ethr_atomic_t *var)
4699 {
4700     ethr_sint_t res;
4701 #if defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
4702     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
4703 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
4704     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
4705     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4706 #elif defined(ETHR_HAVE_NATMC_INC_RETURN)
4707     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
4708     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4709 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
4710     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
4711 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
4712     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
4713     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4714 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
4715     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
4716     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4717 #else
4718     res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) 1);
4719 #endif
4720     return res;
4721 }
4722 
ETHR_ATMC_FUNC__(inc_read_relb)4723 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_relb)(ethr_atomic_t *var)
4724 {
4725     ethr_sint_t res;
4726 #if defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
4727     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
4728 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
4729     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4730     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
4731 #elif defined(ETHR_HAVE_NATMC_INC_RETURN)
4732     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4733     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
4734 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
4735     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
4736 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
4737     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4738     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
4739 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
4740     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4741     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
4742 #else
4743     res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) 1);
4744 #endif
4745     return res;
4746 }
4747 
ETHR_ATMC_FUNC__(inc_read_mb)4748 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_mb)(ethr_atomic_t *var)
4749 {
4750     ethr_sint_t res;
4751 #if defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
4752     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
4753 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
4754     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
4755     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4756 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
4757     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4758     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
4759 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
4760     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4761     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
4762     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4763 #elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
4764     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4765     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
4766     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4767 #elif defined(ETHR_HAVE_NATMC_INC_RETURN)
4768     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4769     res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
4770     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4771 #else
4772     res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) 1);
4773 #endif
4774     return res;
4775 }
4776 
ETHR_ATMC_FUNC__(inc_read_ddrb)4777 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_ddrb)(ethr_atomic_t *var)
4778 {
4779 #ifdef ETHR_ORDERED_READ_DEPEND
4780     return ETHR_ATMC_FUNC__(inc_read)(var);
4781 #else
4782     return ETHR_ATMC_FUNC__(inc_read_rb)(var);
4783 #endif
4784 }
4785 
4786 
4787 /* --- dec_read() --- */
4788 
4789 
ETHR_ATMC_FUNC__(dec_read)4790 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read)(ethr_atomic_t *var)
4791 {
4792     ethr_sint_t res;
4793 #if defined(ETHR_HAVE_NATMC_DEC_RETURN)
4794     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
4795 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
4796     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
4797 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
4798     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
4799 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
4800     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
4801 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
4802     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
4803 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
4804     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
4805 #else
4806     res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) -1);
4807 #endif
4808     return res;
4809 }
4810 
ETHR_ATMC_FUNC__(dec_read_rb)4811 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_rb)(ethr_atomic_t *var)
4812 {
4813     ethr_sint_t res;
4814 #if defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
4815     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
4816 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
4817     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
4818     ETHR_MEMBAR(ETHR_LoadLoad);
4819 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
4820     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
4821 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
4822     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
4823     ETHR_MEMBAR(ETHR_LoadLoad);
4824 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
4825     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
4826     ETHR_MEMBAR(ETHR_LoadLoad);
4827 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
4828     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
4829     ETHR_MEMBAR(ETHR_LoadLoad);
4830 #else
4831     res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) -1);
4832 #endif
4833     return res;
4834 }
4835 
ETHR_ATMC_FUNC__(dec_read_wb)4836 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_wb)(ethr_atomic_t *var)
4837 {
4838     ethr_sint_t res;
4839 #if defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
4840     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
4841 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
4842     ETHR_MEMBAR(ETHR_StoreStore);
4843     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
4844 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
4845     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
4846 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
4847     ETHR_MEMBAR(ETHR_StoreStore);
4848     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
4849 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
4850     ETHR_MEMBAR(ETHR_StoreStore);
4851     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
4852 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
4853     ETHR_MEMBAR(ETHR_StoreStore);
4854     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
4855 #else
4856     res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) -1);
4857 #endif
4858     return res;
4859 }
4860 
ETHR_ATMC_FUNC__(dec_read_acqb)4861 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_acqb)(ethr_atomic_t *var)
4862 {
4863     ethr_sint_t res;
4864 #if defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
4865     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
4866 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
4867     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
4868     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4869 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
4870     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
4871     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4872 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
4873     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
4874 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
4875     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
4876     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4877 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
4878     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
4879     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4880 #else
4881     res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) -1);
4882 #endif
4883     return res;
4884 }
4885 
ETHR_ATMC_FUNC__(dec_read_relb)4886 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_relb)(ethr_atomic_t *var)
4887 {
4888     ethr_sint_t res;
4889 #if defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
4890     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
4891 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
4892     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4893     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
4894 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
4895     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4896     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
4897 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
4898     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
4899 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
4900     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4901     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
4902 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
4903     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4904     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
4905 #else
4906     res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) -1);
4907 #endif
4908     return res;
4909 }
4910 
ETHR_ATMC_FUNC__(dec_read_mb)4911 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_mb)(ethr_atomic_t *var)
4912 {
4913     ethr_sint_t res;
4914 #if defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
4915     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
4916 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
4917     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
4918     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4919 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
4920     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4921     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
4922 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
4923     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4924     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
4925     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4926 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
4927     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4928     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
4929     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4930 #elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
4931     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
4932     res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
4933     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
4934 #else
4935     res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) -1);
4936 #endif
4937     return res;
4938 }
4939 
ETHR_ATMC_FUNC__(dec_read_ddrb)4940 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_ddrb)(ethr_atomic_t *var)
4941 {
4942 #ifdef ETHR_ORDERED_READ_DEPEND
4943     return ETHR_ATMC_FUNC__(dec_read)(var);
4944 #else
4945     return ETHR_ATMC_FUNC__(dec_read_rb)(var);
4946 #endif
4947 }
4948 
4949 
4950 /* --- add() --- */
4951 
4952 
ETHR_ATMC_FUNC__(add)4953 static ETHR_INLINE void ETHR_ATMC_FUNC__(add)(ethr_atomic_t *var, ethr_sint_t val)
4954 {
4955 #if defined(ETHR_HAVE_NATMC_ADD)
4956     ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
4957 #elif defined(ETHR_HAVE_NATMC_ADD_RB)
4958     ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
4959 #elif defined(ETHR_HAVE_NATMC_ADD_WB)
4960     ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
4961 #elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
4962     ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
4963 #elif defined(ETHR_HAVE_NATMC_ADD_RELB)
4964     ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
4965 #elif defined(ETHR_HAVE_NATMC_ADD_MB)
4966     ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
4967 #else
4968     (void) ETHR_ATMC_FUNC__(add_read)(var, val);
4969 #endif
4970 }
4971 
ETHR_ATMC_FUNC__(add_rb)4972 static ETHR_INLINE void ETHR_ATMC_FUNC__(add_rb)(ethr_atomic_t *var, ethr_sint_t val)
4973 {
4974 #if defined(ETHR_HAVE_NATMC_ADD_RB)
4975     ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
4976 #elif defined(ETHR_HAVE_NATMC_ADD)
4977     ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
4978     ETHR_MEMBAR(ETHR_LoadLoad);
4979 #elif defined(ETHR_HAVE_NATMC_ADD_MB)
4980     ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
4981 #elif defined(ETHR_HAVE_NATMC_ADD_WB)
4982     ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
4983     ETHR_MEMBAR(ETHR_LoadLoad);
4984 #elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
4985     ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
4986     ETHR_MEMBAR(ETHR_LoadLoad);
4987 #elif defined(ETHR_HAVE_NATMC_ADD_RELB)
4988     ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
4989     ETHR_MEMBAR(ETHR_LoadLoad);
4990 #else
4991     (void) ETHR_ATMC_FUNC__(add_read_rb)(var, val);
4992 #endif
4993 }
4994 
ETHR_ATMC_FUNC__(add_wb)4995 static ETHR_INLINE void ETHR_ATMC_FUNC__(add_wb)(ethr_atomic_t *var, ethr_sint_t val)
4996 {
4997 #if defined(ETHR_HAVE_NATMC_ADD_WB)
4998     ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
4999 #elif defined(ETHR_HAVE_NATMC_ADD)
5000     ETHR_MEMBAR(ETHR_StoreStore);
5001     ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
5002 #elif defined(ETHR_HAVE_NATMC_ADD_MB)
5003     ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
5004 #elif defined(ETHR_HAVE_NATMC_ADD_RB)
5005     ETHR_MEMBAR(ETHR_StoreStore);
5006     ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
5007 #elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
5008     ETHR_MEMBAR(ETHR_StoreStore);
5009     ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
5010 #elif defined(ETHR_HAVE_NATMC_ADD_RELB)
5011     ETHR_MEMBAR(ETHR_StoreStore);
5012     ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
5013 #else
5014     (void) ETHR_ATMC_FUNC__(add_read_wb)(var, val);
5015 #endif
5016 }
5017 
ETHR_ATMC_FUNC__(add_acqb)5018 static ETHR_INLINE void ETHR_ATMC_FUNC__(add_acqb)(ethr_atomic_t *var, ethr_sint_t val)
5019 {
5020 #if defined(ETHR_HAVE_NATMC_ADD_ACQB)
5021     ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
5022 #elif defined(ETHR_HAVE_NATMC_ADD_RB)
5023     ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
5024     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5025 #elif defined(ETHR_HAVE_NATMC_ADD)
5026     ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
5027     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5028 #elif defined(ETHR_HAVE_NATMC_ADD_MB)
5029     ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
5030 #elif defined(ETHR_HAVE_NATMC_ADD_WB)
5031     ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
5032     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5033 #elif defined(ETHR_HAVE_NATMC_ADD_RELB)
5034     ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
5035     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5036 #else
5037     (void) ETHR_ATMC_FUNC__(add_read_acqb)(var, val);
5038 #endif
5039 }
5040 
ETHR_ATMC_FUNC__(add_relb)5041 static ETHR_INLINE void ETHR_ATMC_FUNC__(add_relb)(ethr_atomic_t *var, ethr_sint_t val)
5042 {
5043 #if defined(ETHR_HAVE_NATMC_ADD_RELB)
5044     ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
5045 #elif defined(ETHR_HAVE_NATMC_ADD_WB)
5046     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5047     ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
5048 #elif defined(ETHR_HAVE_NATMC_ADD)
5049     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5050     ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
5051 #elif defined(ETHR_HAVE_NATMC_ADD_MB)
5052     ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
5053 #elif defined(ETHR_HAVE_NATMC_ADD_RB)
5054     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5055     ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
5056 #elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
5057     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5058     ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
5059 #else
5060     (void) ETHR_ATMC_FUNC__(add_read_relb)(var, val);
5061 #endif
5062 }
5063 
ETHR_ATMC_FUNC__(add_mb)5064 static ETHR_INLINE void ETHR_ATMC_FUNC__(add_mb)(ethr_atomic_t *var, ethr_sint_t val)
5065 {
5066 #if defined(ETHR_HAVE_NATMC_ADD_MB)
5067     ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
5068 #elif defined(ETHR_HAVE_NATMC_ADD_RELB)
5069     ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
5070     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5071 #elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
5072     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5073     ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
5074 #elif defined(ETHR_HAVE_NATMC_ADD_WB)
5075     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5076     ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
5077     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5078 #elif defined(ETHR_HAVE_NATMC_ADD_RB)
5079     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5080     ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
5081     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5082 #elif defined(ETHR_HAVE_NATMC_ADD)
5083     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5084     ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
5085     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5086 #else
5087     (void) ETHR_ATMC_FUNC__(add_read_mb)(var, val);
5088 #endif
5089 }
5090 
ETHR_ATMC_FUNC__(add_ddrb)5091 static ETHR_INLINE void ETHR_ATMC_FUNC__(add_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
5092 {
5093 #ifdef ETHR_ORDERED_READ_DEPEND
5094     ETHR_ATMC_FUNC__(add)(var, val);
5095 #else
5096     ETHR_ATMC_FUNC__(add_rb)(var, val);
5097 #endif
5098 }
5099 
5100 
5101 /* --- inc() --- */
5102 
5103 
ETHR_ATMC_FUNC__(inc)5104 static ETHR_INLINE void ETHR_ATMC_FUNC__(inc)(ethr_atomic_t *var)
5105 {
5106 #if defined(ETHR_HAVE_NATMC_INC)
5107     ETHR_NATMC_FUNC__(inc)(var);
5108 #elif defined(ETHR_HAVE_NATMC_INC_RB)
5109     ETHR_NATMC_FUNC__(inc_rb)(var);
5110 #elif defined(ETHR_HAVE_NATMC_INC_WB)
5111     ETHR_NATMC_FUNC__(inc_wb)(var);
5112 #elif defined(ETHR_HAVE_NATMC_INC_ACQB)
5113     ETHR_NATMC_FUNC__(inc_acqb)(var);
5114 #elif defined(ETHR_HAVE_NATMC_INC_RELB)
5115     ETHR_NATMC_FUNC__(inc_relb)(var);
5116 #elif defined(ETHR_HAVE_NATMC_INC_MB)
5117     ETHR_NATMC_FUNC__(inc_mb)(var);
5118 #else
5119     (void) ETHR_ATMC_FUNC__(inc_read)(var);
5120 #endif
5121 }
5122 
ETHR_ATMC_FUNC__(inc_rb)5123 static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_rb)(ethr_atomic_t *var)
5124 {
5125 #if defined(ETHR_HAVE_NATMC_INC_RB)
5126     ETHR_NATMC_FUNC__(inc_rb)(var);
5127 #elif defined(ETHR_HAVE_NATMC_INC)
5128     ETHR_NATMC_FUNC__(inc)(var);
5129     ETHR_MEMBAR(ETHR_LoadLoad);
5130 #elif defined(ETHR_HAVE_NATMC_INC_MB)
5131     ETHR_NATMC_FUNC__(inc_mb)(var);
5132 #elif defined(ETHR_HAVE_NATMC_INC_WB)
5133     ETHR_NATMC_FUNC__(inc_wb)(var);
5134     ETHR_MEMBAR(ETHR_LoadLoad);
5135 #elif defined(ETHR_HAVE_NATMC_INC_ACQB)
5136     ETHR_NATMC_FUNC__(inc_acqb)(var);
5137     ETHR_MEMBAR(ETHR_LoadLoad);
5138 #elif defined(ETHR_HAVE_NATMC_INC_RELB)
5139     ETHR_NATMC_FUNC__(inc_relb)(var);
5140     ETHR_MEMBAR(ETHR_LoadLoad);
5141 #else
5142     (void) ETHR_ATMC_FUNC__(inc_read_rb)(var);
5143 #endif
5144 }
5145 
ETHR_ATMC_FUNC__(inc_wb)5146 static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_wb)(ethr_atomic_t *var)
5147 {
5148 #if defined(ETHR_HAVE_NATMC_INC_WB)
5149     ETHR_NATMC_FUNC__(inc_wb)(var);
5150 #elif defined(ETHR_HAVE_NATMC_INC)
5151     ETHR_MEMBAR(ETHR_StoreStore);
5152     ETHR_NATMC_FUNC__(inc)(var);
5153 #elif defined(ETHR_HAVE_NATMC_INC_MB)
5154     ETHR_NATMC_FUNC__(inc_mb)(var);
5155 #elif defined(ETHR_HAVE_NATMC_INC_RB)
5156     ETHR_MEMBAR(ETHR_StoreStore);
5157     ETHR_NATMC_FUNC__(inc_rb)(var);
5158 #elif defined(ETHR_HAVE_NATMC_INC_ACQB)
5159     ETHR_MEMBAR(ETHR_StoreStore);
5160     ETHR_NATMC_FUNC__(inc_acqb)(var);
5161 #elif defined(ETHR_HAVE_NATMC_INC_RELB)
5162     ETHR_MEMBAR(ETHR_StoreStore);
5163     ETHR_NATMC_FUNC__(inc_relb)(var);
5164 #else
5165     (void) ETHR_ATMC_FUNC__(inc_read_wb)(var);
5166 #endif
5167 }
5168 
ETHR_ATMC_FUNC__(inc_acqb)5169 static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_acqb)(ethr_atomic_t *var)
5170 {
5171 #if defined(ETHR_HAVE_NATMC_INC_ACQB)
5172     ETHR_NATMC_FUNC__(inc_acqb)(var);
5173 #elif defined(ETHR_HAVE_NATMC_INC_RB)
5174     ETHR_NATMC_FUNC__(inc_rb)(var);
5175     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5176 #elif defined(ETHR_HAVE_NATMC_INC)
5177     ETHR_NATMC_FUNC__(inc)(var);
5178     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5179 #elif defined(ETHR_HAVE_NATMC_INC_MB)
5180     ETHR_NATMC_FUNC__(inc_mb)(var);
5181 #elif defined(ETHR_HAVE_NATMC_INC_WB)
5182     ETHR_NATMC_FUNC__(inc_wb)(var);
5183     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5184 #elif defined(ETHR_HAVE_NATMC_INC_RELB)
5185     ETHR_NATMC_FUNC__(inc_relb)(var);
5186     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5187 #else
5188     (void) ETHR_ATMC_FUNC__(inc_read_acqb)(var);
5189 #endif
5190 }
5191 
ETHR_ATMC_FUNC__(inc_relb)5192 static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_relb)(ethr_atomic_t *var)
5193 {
5194 #if defined(ETHR_HAVE_NATMC_INC_RELB)
5195     ETHR_NATMC_FUNC__(inc_relb)(var);
5196 #elif defined(ETHR_HAVE_NATMC_INC_WB)
5197     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5198     ETHR_NATMC_FUNC__(inc_wb)(var);
5199 #elif defined(ETHR_HAVE_NATMC_INC)
5200     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5201     ETHR_NATMC_FUNC__(inc)(var);
5202 #elif defined(ETHR_HAVE_NATMC_INC_MB)
5203     ETHR_NATMC_FUNC__(inc_mb)(var);
5204 #elif defined(ETHR_HAVE_NATMC_INC_RB)
5205     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5206     ETHR_NATMC_FUNC__(inc_rb)(var);
5207 #elif defined(ETHR_HAVE_NATMC_INC_ACQB)
5208     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5209     ETHR_NATMC_FUNC__(inc_acqb)(var);
5210 #else
5211     (void) ETHR_ATMC_FUNC__(inc_read_relb)(var);
5212 #endif
5213 }
5214 
ETHR_ATMC_FUNC__(inc_mb)5215 static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_mb)(ethr_atomic_t *var)
5216 {
5217 #if defined(ETHR_HAVE_NATMC_INC_MB)
5218     ETHR_NATMC_FUNC__(inc_mb)(var);
5219 #elif defined(ETHR_HAVE_NATMC_INC_RELB)
5220     ETHR_NATMC_FUNC__(inc_relb)(var);
5221     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5222 #elif defined(ETHR_HAVE_NATMC_INC_ACQB)
5223     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5224     ETHR_NATMC_FUNC__(inc_acqb)(var);
5225 #elif defined(ETHR_HAVE_NATMC_INC_WB)
5226     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5227     ETHR_NATMC_FUNC__(inc_wb)(var);
5228     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5229 #elif defined(ETHR_HAVE_NATMC_INC_RB)
5230     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5231     ETHR_NATMC_FUNC__(inc_rb)(var);
5232     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5233 #elif defined(ETHR_HAVE_NATMC_INC)
5234     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5235     ETHR_NATMC_FUNC__(inc)(var);
5236     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5237 #else
5238     (void) ETHR_ATMC_FUNC__(inc_read_mb)(var);
5239 #endif
5240 }
5241 
ETHR_ATMC_FUNC__(inc_ddrb)5242 static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_ddrb)(ethr_atomic_t *var)
5243 {
5244 #ifdef ETHR_ORDERED_READ_DEPEND
5245     ETHR_ATMC_FUNC__(inc)(var);
5246 #else
5247     ETHR_ATMC_FUNC__(inc_rb)(var);
5248 #endif
5249 }
5250 
5251 
5252 /* --- dec() --- */
5253 
5254 
ETHR_ATMC_FUNC__(dec)5255 static ETHR_INLINE void ETHR_ATMC_FUNC__(dec)(ethr_atomic_t *var)
5256 {
5257 #if defined(ETHR_HAVE_NATMC_DEC)
5258     ETHR_NATMC_FUNC__(dec)(var);
5259 #elif defined(ETHR_HAVE_NATMC_DEC_RB)
5260     ETHR_NATMC_FUNC__(dec_rb)(var);
5261 #elif defined(ETHR_HAVE_NATMC_DEC_WB)
5262     ETHR_NATMC_FUNC__(dec_wb)(var);
5263 #elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
5264     ETHR_NATMC_FUNC__(dec_acqb)(var);
5265 #elif defined(ETHR_HAVE_NATMC_DEC_RELB)
5266     ETHR_NATMC_FUNC__(dec_relb)(var);
5267 #elif defined(ETHR_HAVE_NATMC_DEC_MB)
5268     ETHR_NATMC_FUNC__(dec_mb)(var);
5269 #else
5270     (void) ETHR_ATMC_FUNC__(dec_read)(var);
5271 #endif
5272 }
5273 
ETHR_ATMC_FUNC__(dec_rb)5274 static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_rb)(ethr_atomic_t *var)
5275 {
5276 #if defined(ETHR_HAVE_NATMC_DEC_RB)
5277     ETHR_NATMC_FUNC__(dec_rb)(var);
5278 #elif defined(ETHR_HAVE_NATMC_DEC)
5279     ETHR_NATMC_FUNC__(dec)(var);
5280     ETHR_MEMBAR(ETHR_LoadLoad);
5281 #elif defined(ETHR_HAVE_NATMC_DEC_MB)
5282     ETHR_NATMC_FUNC__(dec_mb)(var);
5283 #elif defined(ETHR_HAVE_NATMC_DEC_WB)
5284     ETHR_NATMC_FUNC__(dec_wb)(var);
5285     ETHR_MEMBAR(ETHR_LoadLoad);
5286 #elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
5287     ETHR_NATMC_FUNC__(dec_acqb)(var);
5288     ETHR_MEMBAR(ETHR_LoadLoad);
5289 #elif defined(ETHR_HAVE_NATMC_DEC_RELB)
5290     ETHR_NATMC_FUNC__(dec_relb)(var);
5291     ETHR_MEMBAR(ETHR_LoadLoad);
5292 #else
5293     (void) ETHR_ATMC_FUNC__(dec_read_rb)(var);
5294 #endif
5295 }
5296 
ETHR_ATMC_FUNC__(dec_wb)5297 static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_wb)(ethr_atomic_t *var)
5298 {
5299 #if defined(ETHR_HAVE_NATMC_DEC_WB)
5300     ETHR_NATMC_FUNC__(dec_wb)(var);
5301 #elif defined(ETHR_HAVE_NATMC_DEC)
5302     ETHR_MEMBAR(ETHR_StoreStore);
5303     ETHR_NATMC_FUNC__(dec)(var);
5304 #elif defined(ETHR_HAVE_NATMC_DEC_MB)
5305     ETHR_NATMC_FUNC__(dec_mb)(var);
5306 #elif defined(ETHR_HAVE_NATMC_DEC_RB)
5307     ETHR_MEMBAR(ETHR_StoreStore);
5308     ETHR_NATMC_FUNC__(dec_rb)(var);
5309 #elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
5310     ETHR_MEMBAR(ETHR_StoreStore);
5311     ETHR_NATMC_FUNC__(dec_acqb)(var);
5312 #elif defined(ETHR_HAVE_NATMC_DEC_RELB)
5313     ETHR_MEMBAR(ETHR_StoreStore);
5314     ETHR_NATMC_FUNC__(dec_relb)(var);
5315 #else
5316     (void) ETHR_ATMC_FUNC__(dec_read_wb)(var);
5317 #endif
5318 }
5319 
ETHR_ATMC_FUNC__(dec_acqb)5320 static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_acqb)(ethr_atomic_t *var)
5321 {
5322 #if defined(ETHR_HAVE_NATMC_DEC_ACQB)
5323     ETHR_NATMC_FUNC__(dec_acqb)(var);
5324 #elif defined(ETHR_HAVE_NATMC_DEC_RB)
5325     ETHR_NATMC_FUNC__(dec_rb)(var);
5326     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5327 #elif defined(ETHR_HAVE_NATMC_DEC)
5328     ETHR_NATMC_FUNC__(dec)(var);
5329     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5330 #elif defined(ETHR_HAVE_NATMC_DEC_MB)
5331     ETHR_NATMC_FUNC__(dec_mb)(var);
5332 #elif defined(ETHR_HAVE_NATMC_DEC_WB)
5333     ETHR_NATMC_FUNC__(dec_wb)(var);
5334     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5335 #elif defined(ETHR_HAVE_NATMC_DEC_RELB)
5336     ETHR_NATMC_FUNC__(dec_relb)(var);
5337     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5338 #else
5339     (void) ETHR_ATMC_FUNC__(dec_read_acqb)(var);
5340 #endif
5341 }
5342 
ETHR_ATMC_FUNC__(dec_relb)5343 static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_relb)(ethr_atomic_t *var)
5344 {
5345 #if defined(ETHR_HAVE_NATMC_DEC_RELB)
5346     ETHR_NATMC_FUNC__(dec_relb)(var);
5347 #elif defined(ETHR_HAVE_NATMC_DEC_WB)
5348     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5349     ETHR_NATMC_FUNC__(dec_wb)(var);
5350 #elif defined(ETHR_HAVE_NATMC_DEC)
5351     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5352     ETHR_NATMC_FUNC__(dec)(var);
5353 #elif defined(ETHR_HAVE_NATMC_DEC_MB)
5354     ETHR_NATMC_FUNC__(dec_mb)(var);
5355 #elif defined(ETHR_HAVE_NATMC_DEC_RB)
5356     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5357     ETHR_NATMC_FUNC__(dec_rb)(var);
5358 #elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
5359     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5360     ETHR_NATMC_FUNC__(dec_acqb)(var);
5361 #else
5362     (void) ETHR_ATMC_FUNC__(dec_read_relb)(var);
5363 #endif
5364 }
5365 
ETHR_ATMC_FUNC__(dec_mb)5366 static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_mb)(ethr_atomic_t *var)
5367 {
5368 #if defined(ETHR_HAVE_NATMC_DEC_MB)
5369     ETHR_NATMC_FUNC__(dec_mb)(var);
5370 #elif defined(ETHR_HAVE_NATMC_DEC_RELB)
5371     ETHR_NATMC_FUNC__(dec_relb)(var);
5372     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5373 #elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
5374     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5375     ETHR_NATMC_FUNC__(dec_acqb)(var);
5376 #elif defined(ETHR_HAVE_NATMC_DEC_WB)
5377     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5378     ETHR_NATMC_FUNC__(dec_wb)(var);
5379     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5380 #elif defined(ETHR_HAVE_NATMC_DEC_RB)
5381     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5382     ETHR_NATMC_FUNC__(dec_rb)(var);
5383     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5384 #elif defined(ETHR_HAVE_NATMC_DEC)
5385     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5386     ETHR_NATMC_FUNC__(dec)(var);
5387     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5388 #else
5389     (void) ETHR_ATMC_FUNC__(dec_read_mb)(var);
5390 #endif
5391 }
5392 
ETHR_ATMC_FUNC__(dec_ddrb)5393 static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_ddrb)(ethr_atomic_t *var)
5394 {
5395 #ifdef ETHR_ORDERED_READ_DEPEND
5396     ETHR_ATMC_FUNC__(dec)(var);
5397 #else
5398     ETHR_ATMC_FUNC__(dec_rb)(var);
5399 #endif
5400 }
5401 
5402 
5403 /* --- read_band() --- */
5404 
5405 
ETHR_ATMC_FUNC__(read_band)5406 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band)(ethr_atomic_t *var, ethr_sint_t val)
5407 {
5408     ethr_sint_t res;
5409 #if defined(ETHR_HAVE_NATMC_AND_RETOLD)
5410     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
5411 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
5412     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
5413 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
5414     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
5415 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
5416     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
5417 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
5418     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
5419 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
5420     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
5421 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5422     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
5423 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5424     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
5425 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5426     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
5427 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5428     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
5429 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5430     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
5431 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5432     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
5433 #else
5434 #error "Missing implementation of ethr_atomic_read_band()!"
5435 #endif
5436     return res;
5437 }
5438 
ETHR_ATMC_FUNC__(read_band_rb)5439 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_rb)(ethr_atomic_t *var, ethr_sint_t val)
5440 {
5441     ethr_sint_t res;
5442 #if defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
5443     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
5444 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
5445     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
5446     ETHR_MEMBAR(ETHR_LoadLoad);
5447 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
5448     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
5449 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
5450     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
5451     ETHR_MEMBAR(ETHR_LoadLoad);
5452 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
5453     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
5454     ETHR_MEMBAR(ETHR_LoadLoad);
5455 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
5456     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
5457     ETHR_MEMBAR(ETHR_LoadLoad);
5458 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5459     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
5460 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5461     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
5462     ETHR_MEMBAR(ETHR_LoadLoad);
5463 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5464     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
5465 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5466     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
5467     ETHR_MEMBAR(ETHR_LoadLoad);
5468 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5469     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
5470     ETHR_MEMBAR(ETHR_LoadLoad);
5471 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5472     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
5473     ETHR_MEMBAR(ETHR_LoadLoad);
5474 #else
5475 #error "Missing implementation of ethr_atomic_read_band_rb()!"
5476 #endif
5477     return res;
5478 }
5479 
ETHR_ATMC_FUNC__(read_band_wb)5480 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_wb)(ethr_atomic_t *var, ethr_sint_t val)
5481 {
5482     ethr_sint_t res;
5483 #if defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
5484     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
5485 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
5486     ETHR_MEMBAR(ETHR_StoreStore);
5487     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
5488 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
5489     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
5490 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
5491     ETHR_MEMBAR(ETHR_StoreStore);
5492     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
5493 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
5494     ETHR_MEMBAR(ETHR_StoreStore);
5495     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
5496 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
5497     ETHR_MEMBAR(ETHR_StoreStore);
5498     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
5499 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5500     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
5501 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5502     ETHR_MEMBAR(ETHR_StoreStore);
5503     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
5504 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5505     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
5506 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5507     ETHR_MEMBAR(ETHR_StoreStore);
5508     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
5509 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5510     ETHR_MEMBAR(ETHR_StoreStore);
5511     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
5512 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5513     ETHR_MEMBAR(ETHR_StoreStore);
5514     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
5515 #else
5516 #error "Missing implementation of ethr_atomic_read_band_wb()!"
5517 #endif
5518     return res;
5519 }
5520 
ETHR_ATMC_FUNC__(read_band_acqb)5521 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_acqb)(ethr_atomic_t *var, ethr_sint_t val)
5522 {
5523     ethr_sint_t res;
5524 #if defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
5525     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
5526 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
5527     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
5528     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5529 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
5530     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
5531     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5532 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
5533     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
5534 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
5535     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
5536     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5537 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
5538     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
5539     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5540 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5541     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
5542 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5543     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
5544     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5545 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5546     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
5547     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5548 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5549     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
5550 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5551     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
5552     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5553 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5554     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
5555     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5556 #else
5557 #error "Missing implementation of ethr_atomic_read_band_acqb()!"
5558 #endif
5559     return res;
5560 }
5561 
ETHR_ATMC_FUNC__(read_band_relb)5562 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_relb)(ethr_atomic_t *var, ethr_sint_t val)
5563 {
5564     ethr_sint_t res;
5565 #if defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
5566     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
5567 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
5568     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5569     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
5570 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
5571     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5572     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
5573 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
5574     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
5575 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
5576     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5577     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
5578 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
5579     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5580     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
5581 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5582     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
5583 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5584     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5585     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
5586 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5587     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5588     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
5589 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5590     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
5591 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5592     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5593     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
5594 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5595     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5596     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
5597 #else
5598 #error "Missing implementation of ethr_atomic_read_band_relb()!"
5599 #endif
5600     return res;
5601 }
5602 
ETHR_ATMC_FUNC__(read_band_mb)5603 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_mb)(ethr_atomic_t *var, ethr_sint_t val)
5604 {
5605     ethr_sint_t res;
5606 #if defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
5607     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
5608 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
5609     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
5610     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5611 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
5612     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5613     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
5614 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
5615     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5616     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
5617     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5618 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
5619     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5620     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
5621     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5622 #elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
5623     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5624     res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
5625     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5626 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5627     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
5628 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5629     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
5630     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5631 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5632     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5633     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
5634 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5635     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5636     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
5637     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5638 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5639     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5640     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
5641     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5642 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5643     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5644     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
5645     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5646 #else
5647 #error "Missing implementation of ethr_atomic_read_band_mb()!"
5648 #endif
5649     return res;
5650 }
5651 
ETHR_ATMC_FUNC__(read_band_ddrb)5652 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
5653 {
5654 #ifdef ETHR_ORDERED_READ_DEPEND
5655     return ETHR_ATMC_FUNC__(read_band)(var, val);
5656 #else
5657     return ETHR_ATMC_FUNC__(read_band_rb)(var, val);
5658 #endif
5659 }
5660 
5661 
5662 /* --- read_bor() --- */
5663 
5664 
ETHR_ATMC_FUNC__(read_bor)5665 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor)(ethr_atomic_t *var, ethr_sint_t val)
5666 {
5667     ethr_sint_t res;
5668 #if defined(ETHR_HAVE_NATMC_OR_RETOLD)
5669     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
5670 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
5671     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
5672 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
5673     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
5674 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
5675     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
5676 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
5677     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
5678 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
5679     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
5680 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5681     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
5682 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5683     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
5684 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5685     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
5686 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5687     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
5688 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5689     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
5690 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5691     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
5692 #else
5693 #error "Missing implementation of ethr_atomic_read_bor()!"
5694 #endif
5695     return res;
5696 }
5697 
ETHR_ATMC_FUNC__(read_bor_rb)5698 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_rb)(ethr_atomic_t *var, ethr_sint_t val)
5699 {
5700     ethr_sint_t res;
5701 #if defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
5702     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
5703 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
5704     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
5705     ETHR_MEMBAR(ETHR_LoadLoad);
5706 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
5707     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
5708 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
5709     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
5710     ETHR_MEMBAR(ETHR_LoadLoad);
5711 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
5712     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
5713     ETHR_MEMBAR(ETHR_LoadLoad);
5714 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
5715     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
5716     ETHR_MEMBAR(ETHR_LoadLoad);
5717 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5718     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
5719 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5720     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
5721     ETHR_MEMBAR(ETHR_LoadLoad);
5722 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5723     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
5724 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5725     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
5726     ETHR_MEMBAR(ETHR_LoadLoad);
5727 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5728     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
5729     ETHR_MEMBAR(ETHR_LoadLoad);
5730 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5731     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
5732     ETHR_MEMBAR(ETHR_LoadLoad);
5733 #else
5734 #error "Missing implementation of ethr_atomic_read_bor_rb()!"
5735 #endif
5736     return res;
5737 }
5738 
ETHR_ATMC_FUNC__(read_bor_wb)5739 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_wb)(ethr_atomic_t *var, ethr_sint_t val)
5740 {
5741     ethr_sint_t res;
5742 #if defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
5743     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
5744 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
5745     ETHR_MEMBAR(ETHR_StoreStore);
5746     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
5747 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
5748     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
5749 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
5750     ETHR_MEMBAR(ETHR_StoreStore);
5751     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
5752 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
5753     ETHR_MEMBAR(ETHR_StoreStore);
5754     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
5755 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
5756     ETHR_MEMBAR(ETHR_StoreStore);
5757     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
5758 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5759     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
5760 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5761     ETHR_MEMBAR(ETHR_StoreStore);
5762     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
5763 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5764     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
5765 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5766     ETHR_MEMBAR(ETHR_StoreStore);
5767     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
5768 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5769     ETHR_MEMBAR(ETHR_StoreStore);
5770     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
5771 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5772     ETHR_MEMBAR(ETHR_StoreStore);
5773     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
5774 #else
5775 #error "Missing implementation of ethr_atomic_read_bor_wb()!"
5776 #endif
5777     return res;
5778 }
5779 
ETHR_ATMC_FUNC__(read_bor_acqb)5780 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_acqb)(ethr_atomic_t *var, ethr_sint_t val)
5781 {
5782     ethr_sint_t res;
5783 #if defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
5784     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
5785 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
5786     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
5787     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5788 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
5789     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
5790     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5791 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
5792     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
5793 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
5794     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
5795     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5796 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
5797     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
5798     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5799 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5800     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
5801 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5802     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
5803     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5804 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5805     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
5806     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5807 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5808     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
5809 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5810     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
5811     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5812 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5813     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
5814     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5815 #else
5816 #error "Missing implementation of ethr_atomic_read_bor_acqb()!"
5817 #endif
5818     return res;
5819 }
5820 
ETHR_ATMC_FUNC__(read_bor_relb)5821 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_relb)(ethr_atomic_t *var, ethr_sint_t val)
5822 {
5823     ethr_sint_t res;
5824 #if defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
5825     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
5826 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
5827     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5828     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
5829 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
5830     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5831     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
5832 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
5833     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
5834 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
5835     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5836     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
5837 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
5838     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5839     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
5840 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5841     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
5842 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5843     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5844     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
5845 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5846     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5847     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
5848 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5849     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
5850 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5851     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5852     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
5853 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5854     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5855     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
5856 #else
5857 #error "Missing implementation of ethr_atomic_read_bor_relb()!"
5858 #endif
5859     return res;
5860 }
5861 
ETHR_ATMC_FUNC__(read_bor_mb)5862 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_mb)(ethr_atomic_t *var, ethr_sint_t val)
5863 {
5864     ethr_sint_t res;
5865 #if defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
5866     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
5867 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
5868     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
5869     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5870 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
5871     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5872     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
5873 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
5874     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5875     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
5876     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5877 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
5878     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5879     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
5880     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5881 #elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
5882     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5883     res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
5884     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5885 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
5886     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
5887 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
5888     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
5889     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5890 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
5891     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5892     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
5893 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
5894     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5895     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
5896     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5897 #elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
5898     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5899     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
5900     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5901 #elif defined(ETHR_HAVE_NATMC_CMPXCHG)
5902     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
5903     ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
5904     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
5905 #else
5906 #error "Missing implementation of ethr_atomic_read_bor_mb()!"
5907 #endif
5908     return res;
5909 }
5910 
ETHR_ATMC_FUNC__(read_bor_ddrb)5911 static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
5912 {
5913 #ifdef ETHR_ORDERED_READ_DEPEND
5914     return ETHR_ATMC_FUNC__(read_bor)(var, val);
5915 #else
5916     return ETHR_ATMC_FUNC__(read_bor_rb)(var, val);
5917 #endif
5918 }
5919 
5920 #endif /* ETHR_ATMC_INLINE__ */
5921 
5922 
5923 /* ---------- 32-bit atomic implementation ---------- */
5924 
5925 
5926 #ifdef ETHR_NEED_ATMC32_PROTOTYPES__
5927 ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var);
5928 ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
5929 ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
5930 ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
5931 ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
5932 ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
5933 ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
5934 ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
5935 ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val);
5936 ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
5937 ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val);
5938 ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val);
5939 ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
5940 ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val);
5941 ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val);
5942 void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val);
5943 void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
5944 void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val);
5945 void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val);
5946 void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
5947 void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val);
5948 void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val);
5949 void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val);
5950 void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
5951 void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val);
5952 void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val);
5953 void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
5954 void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val);
5955 void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val);
5956 ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val);
5957 ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
5958 ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val);
5959 ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val);
5960 ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
5961 ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val);
5962 ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val);
5963 ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var);
5964 ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var);
5965 ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var);
5966 ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var);
5967 ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var);
5968 ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var);
5969 ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var);
5970 ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var);
5971 ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var);
5972 ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var);
5973 ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var);
5974 ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var);
5975 ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var);
5976 ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var);
5977 ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var);
5978 ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var);
5979 ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var);
5980 ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var);
5981 ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var);
5982 ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var);
5983 ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var);
5984 void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val);
5985 void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
5986 void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val);
5987 void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val);
5988 void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
5989 void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val);
5990 void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val);
5991 void ethr_atomic32_inc(ethr_atomic32_t *var);
5992 void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var);
5993 void ethr_atomic32_inc_rb(ethr_atomic32_t *var);
5994 void ethr_atomic32_inc_wb(ethr_atomic32_t *var);
5995 void ethr_atomic32_inc_acqb(ethr_atomic32_t *var);
5996 void ethr_atomic32_inc_relb(ethr_atomic32_t *var);
5997 void ethr_atomic32_inc_mb(ethr_atomic32_t *var);
5998 void ethr_atomic32_dec(ethr_atomic32_t *var);
5999 void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var);
6000 void ethr_atomic32_dec_rb(ethr_atomic32_t *var);
6001 void ethr_atomic32_dec_wb(ethr_atomic32_t *var);
6002 void ethr_atomic32_dec_acqb(ethr_atomic32_t *var);
6003 void ethr_atomic32_dec_relb(ethr_atomic32_t *var);
6004 void ethr_atomic32_dec_mb(ethr_atomic32_t *var);
6005 ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val);
6006 ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
6007 ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val);
6008 ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val);
6009 ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
6010 ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val);
6011 ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val);
6012 ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val);
6013 ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
6014 ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val);
6015 ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val);
6016 ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
6017 ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val);
6018 ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val);
6019 #endif /* ETHR_NEED_ATMC32_PROTOTYPES__ */
6020 
6021 #if (defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) \
6022      && (defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
6023 
6024 #if !defined(ETHR_NATMC32_BITS__)
6025 #  error "Missing native atomic implementation"
6026 #elif ETHR_NATMC32_BITS__ == 64
6027 #  undef ETHR_HAVE_NATMC32_CMPXCHG
6028 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
6029 #    define ETHR_HAVE_NATMC32_CMPXCHG 1
6030 #  endif
6031 #  undef ETHR_HAVE_NATMC32_CMPXCHG_RB
6032 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
6033 #    define ETHR_HAVE_NATMC32_CMPXCHG_RB 1
6034 #  endif
6035 #  undef ETHR_HAVE_NATMC32_CMPXCHG_WB
6036 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
6037 #    define ETHR_HAVE_NATMC32_CMPXCHG_WB 1
6038 #  endif
6039 #  undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB
6040 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
6041 #    define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1
6042 #  endif
6043 #  undef ETHR_HAVE_NATMC32_CMPXCHG_RELB
6044 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
6045 #    define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1
6046 #  endif
6047 #  undef ETHR_HAVE_NATMC32_CMPXCHG_MB
6048 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
6049 #    define ETHR_HAVE_NATMC32_CMPXCHG_MB 1
6050 #  endif
6051 #  undef ETHR_HAVE_NATMC32_XCHG
6052 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
6053 #    define ETHR_HAVE_NATMC32_XCHG 1
6054 #  endif
6055 #  undef ETHR_HAVE_NATMC32_XCHG_RB
6056 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
6057 #    define ETHR_HAVE_NATMC32_XCHG_RB 1
6058 #  endif
6059 #  undef ETHR_HAVE_NATMC32_XCHG_WB
6060 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
6061 #    define ETHR_HAVE_NATMC32_XCHG_WB 1
6062 #  endif
6063 #  undef ETHR_HAVE_NATMC32_XCHG_ACQB
6064 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
6065 #    define ETHR_HAVE_NATMC32_XCHG_ACQB 1
6066 #  endif
6067 #  undef ETHR_HAVE_NATMC32_XCHG_RELB
6068 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
6069 #    define ETHR_HAVE_NATMC32_XCHG_RELB 1
6070 #  endif
6071 #  undef ETHR_HAVE_NATMC32_XCHG_MB
6072 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
6073 #    define ETHR_HAVE_NATMC32_XCHG_MB 1
6074 #  endif
6075 #  undef ETHR_HAVE_NATMC32_SET
6076 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
6077 #    define ETHR_HAVE_NATMC32_SET 1
6078 #  endif
6079 #  undef ETHR_HAVE_NATMC32_SET_RB
6080 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
6081 #    define ETHR_HAVE_NATMC32_SET_RB 1
6082 #  endif
6083 #  undef ETHR_HAVE_NATMC32_SET_WB
6084 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
6085 #    define ETHR_HAVE_NATMC32_SET_WB 1
6086 #  endif
6087 #  undef ETHR_HAVE_NATMC32_SET_ACQB
6088 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
6089 #    define ETHR_HAVE_NATMC32_SET_ACQB 1
6090 #  endif
6091 #  undef ETHR_HAVE_NATMC32_SET_RELB
6092 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
6093 #    define ETHR_HAVE_NATMC32_SET_RELB 1
6094 #  endif
6095 #  undef ETHR_HAVE_NATMC32_SET_MB
6096 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
6097 #    define ETHR_HAVE_NATMC32_SET_MB 1
6098 #  endif
6099 #  undef ETHR_HAVE_NATMC32_INIT
6100 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
6101 #    define ETHR_HAVE_NATMC32_INIT 1
6102 #  endif
6103 #  undef ETHR_HAVE_NATMC32_INIT_RB
6104 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
6105 #    define ETHR_HAVE_NATMC32_INIT_RB 1
6106 #  endif
6107 #  undef ETHR_HAVE_NATMC32_INIT_WB
6108 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
6109 #    define ETHR_HAVE_NATMC32_INIT_WB 1
6110 #  endif
6111 #  undef ETHR_HAVE_NATMC32_INIT_ACQB
6112 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
6113 #    define ETHR_HAVE_NATMC32_INIT_ACQB 1
6114 #  endif
6115 #  undef ETHR_HAVE_NATMC32_INIT_RELB
6116 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
6117 #    define ETHR_HAVE_NATMC32_INIT_RELB 1
6118 #  endif
6119 #  undef ETHR_HAVE_NATMC32_INIT_MB
6120 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
6121 #    define ETHR_HAVE_NATMC32_INIT_MB 1
6122 #  endif
6123 #  undef ETHR_HAVE_NATMC32_ADD_RETURN
6124 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
6125 #    define ETHR_HAVE_NATMC32_ADD_RETURN 1
6126 #  endif
6127 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_RB
6128 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
6129 #    define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1
6130 #  endif
6131 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_WB
6132 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
6133 #    define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1
6134 #  endif
6135 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB
6136 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
6137 #    define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1
6138 #  endif
6139 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB
6140 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
6141 #    define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1
6142 #  endif
6143 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_MB
6144 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
6145 #    define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1
6146 #  endif
6147 #  undef ETHR_HAVE_NATMC32_READ
6148 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
6149 #    define ETHR_HAVE_NATMC32_READ 1
6150 #  endif
6151 #  undef ETHR_HAVE_NATMC32_READ_RB
6152 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
6153 #    define ETHR_HAVE_NATMC32_READ_RB 1
6154 #  endif
6155 #  undef ETHR_HAVE_NATMC32_READ_WB
6156 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
6157 #    define ETHR_HAVE_NATMC32_READ_WB 1
6158 #  endif
6159 #  undef ETHR_HAVE_NATMC32_READ_ACQB
6160 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
6161 #    define ETHR_HAVE_NATMC32_READ_ACQB 1
6162 #  endif
6163 #  undef ETHR_HAVE_NATMC32_READ_RELB
6164 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
6165 #    define ETHR_HAVE_NATMC32_READ_RELB 1
6166 #  endif
6167 #  undef ETHR_HAVE_NATMC32_READ_MB
6168 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
6169 #    define ETHR_HAVE_NATMC32_READ_MB 1
6170 #  endif
6171 #  undef ETHR_HAVE_NATMC32_INC_RETURN
6172 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
6173 #    define ETHR_HAVE_NATMC32_INC_RETURN 1
6174 #  endif
6175 #  undef ETHR_HAVE_NATMC32_INC_RETURN_RB
6176 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
6177 #    define ETHR_HAVE_NATMC32_INC_RETURN_RB 1
6178 #  endif
6179 #  undef ETHR_HAVE_NATMC32_INC_RETURN_WB
6180 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
6181 #    define ETHR_HAVE_NATMC32_INC_RETURN_WB 1
6182 #  endif
6183 #  undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB
6184 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
6185 #    define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1
6186 #  endif
6187 #  undef ETHR_HAVE_NATMC32_INC_RETURN_RELB
6188 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
6189 #    define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1
6190 #  endif
6191 #  undef ETHR_HAVE_NATMC32_INC_RETURN_MB
6192 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
6193 #    define ETHR_HAVE_NATMC32_INC_RETURN_MB 1
6194 #  endif
6195 #  undef ETHR_HAVE_NATMC32_DEC_RETURN
6196 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
6197 #    define ETHR_HAVE_NATMC32_DEC_RETURN 1
6198 #  endif
6199 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_RB
6200 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
6201 #    define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1
6202 #  endif
6203 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_WB
6204 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
6205 #    define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1
6206 #  endif
6207 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB
6208 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
6209 #    define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1
6210 #  endif
6211 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB
6212 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
6213 #    define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1
6214 #  endif
6215 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_MB
6216 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
6217 #    define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1
6218 #  endif
6219 #  undef ETHR_HAVE_NATMC32_ADD
6220 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
6221 #    define ETHR_HAVE_NATMC32_ADD 1
6222 #  endif
6223 #  undef ETHR_HAVE_NATMC32_ADD_RB
6224 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
6225 #    define ETHR_HAVE_NATMC32_ADD_RB 1
6226 #  endif
6227 #  undef ETHR_HAVE_NATMC32_ADD_WB
6228 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
6229 #    define ETHR_HAVE_NATMC32_ADD_WB 1
6230 #  endif
6231 #  undef ETHR_HAVE_NATMC32_ADD_ACQB
6232 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
6233 #    define ETHR_HAVE_NATMC32_ADD_ACQB 1
6234 #  endif
6235 #  undef ETHR_HAVE_NATMC32_ADD_RELB
6236 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
6237 #    define ETHR_HAVE_NATMC32_ADD_RELB 1
6238 #  endif
6239 #  undef ETHR_HAVE_NATMC32_ADD_MB
6240 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
6241 #    define ETHR_HAVE_NATMC32_ADD_MB 1
6242 #  endif
6243 #  undef ETHR_HAVE_NATMC32_INC
6244 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
6245 #    define ETHR_HAVE_NATMC32_INC 1
6246 #  endif
6247 #  undef ETHR_HAVE_NATMC32_INC_RB
6248 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
6249 #    define ETHR_HAVE_NATMC32_INC_RB 1
6250 #  endif
6251 #  undef ETHR_HAVE_NATMC32_INC_WB
6252 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
6253 #    define ETHR_HAVE_NATMC32_INC_WB 1
6254 #  endif
6255 #  undef ETHR_HAVE_NATMC32_INC_ACQB
6256 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
6257 #    define ETHR_HAVE_NATMC32_INC_ACQB 1
6258 #  endif
6259 #  undef ETHR_HAVE_NATMC32_INC_RELB
6260 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
6261 #    define ETHR_HAVE_NATMC32_INC_RELB 1
6262 #  endif
6263 #  undef ETHR_HAVE_NATMC32_INC_MB
6264 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
6265 #    define ETHR_HAVE_NATMC32_INC_MB 1
6266 #  endif
6267 #  undef ETHR_HAVE_NATMC32_DEC
6268 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
6269 #    define ETHR_HAVE_NATMC32_DEC 1
6270 #  endif
6271 #  undef ETHR_HAVE_NATMC32_DEC_RB
6272 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
6273 #    define ETHR_HAVE_NATMC32_DEC_RB 1
6274 #  endif
6275 #  undef ETHR_HAVE_NATMC32_DEC_WB
6276 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
6277 #    define ETHR_HAVE_NATMC32_DEC_WB 1
6278 #  endif
6279 #  undef ETHR_HAVE_NATMC32_DEC_ACQB
6280 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
6281 #    define ETHR_HAVE_NATMC32_DEC_ACQB 1
6282 #  endif
6283 #  undef ETHR_HAVE_NATMC32_DEC_RELB
6284 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
6285 #    define ETHR_HAVE_NATMC32_DEC_RELB 1
6286 #  endif
6287 #  undef ETHR_HAVE_NATMC32_DEC_MB
6288 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
6289 #    define ETHR_HAVE_NATMC32_DEC_MB 1
6290 #  endif
6291 #  undef ETHR_HAVE_NATMC32_AND_RETOLD
6292 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
6293 #    define ETHR_HAVE_NATMC32_AND_RETOLD 1
6294 #  endif
6295 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_RB
6296 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
6297 #    define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1
6298 #  endif
6299 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_WB
6300 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
6301 #    define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1
6302 #  endif
6303 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB
6304 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
6305 #    define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1
6306 #  endif
6307 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB
6308 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
6309 #    define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1
6310 #  endif
6311 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_MB
6312 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
6313 #    define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1
6314 #  endif
6315 #  undef ETHR_HAVE_NATMC32_OR_RETOLD
6316 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
6317 #    define ETHR_HAVE_NATMC32_OR_RETOLD 1
6318 #  endif
6319 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_RB
6320 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
6321 #    define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1
6322 #  endif
6323 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_WB
6324 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
6325 #    define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1
6326 #  endif
6327 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB
6328 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
6329 #    define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1
6330 #  endif
6331 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB
6332 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
6333 #    define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1
6334 #  endif
6335 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_MB
6336 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
6337 #    define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1
6338 #  endif
6339 #elif ETHR_NATMC32_BITS__ == 32
6340 #  undef ETHR_HAVE_NATMC32_CMPXCHG
6341 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
6342 #    define ETHR_HAVE_NATMC32_CMPXCHG 1
6343 #  endif
6344 #  undef ETHR_HAVE_NATMC32_CMPXCHG_RB
6345 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
6346 #    define ETHR_HAVE_NATMC32_CMPXCHG_RB 1
6347 #  endif
6348 #  undef ETHR_HAVE_NATMC32_CMPXCHG_WB
6349 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
6350 #    define ETHR_HAVE_NATMC32_CMPXCHG_WB 1
6351 #  endif
6352 #  undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB
6353 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
6354 #    define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1
6355 #  endif
6356 #  undef ETHR_HAVE_NATMC32_CMPXCHG_RELB
6357 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
6358 #    define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1
6359 #  endif
6360 #  undef ETHR_HAVE_NATMC32_CMPXCHG_MB
6361 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
6362 #    define ETHR_HAVE_NATMC32_CMPXCHG_MB 1
6363 #  endif
6364 #  undef ETHR_HAVE_NATMC32_XCHG
6365 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
6366 #    define ETHR_HAVE_NATMC32_XCHG 1
6367 #  endif
6368 #  undef ETHR_HAVE_NATMC32_XCHG_RB
6369 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
6370 #    define ETHR_HAVE_NATMC32_XCHG_RB 1
6371 #  endif
6372 #  undef ETHR_HAVE_NATMC32_XCHG_WB
6373 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
6374 #    define ETHR_HAVE_NATMC32_XCHG_WB 1
6375 #  endif
6376 #  undef ETHR_HAVE_NATMC32_XCHG_ACQB
6377 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
6378 #    define ETHR_HAVE_NATMC32_XCHG_ACQB 1
6379 #  endif
6380 #  undef ETHR_HAVE_NATMC32_XCHG_RELB
6381 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
6382 #    define ETHR_HAVE_NATMC32_XCHG_RELB 1
6383 #  endif
6384 #  undef ETHR_HAVE_NATMC32_XCHG_MB
6385 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
6386 #    define ETHR_HAVE_NATMC32_XCHG_MB 1
6387 #  endif
6388 #  undef ETHR_HAVE_NATMC32_SET
6389 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
6390 #    define ETHR_HAVE_NATMC32_SET 1
6391 #  endif
6392 #  undef ETHR_HAVE_NATMC32_SET_RB
6393 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
6394 #    define ETHR_HAVE_NATMC32_SET_RB 1
6395 #  endif
6396 #  undef ETHR_HAVE_NATMC32_SET_WB
6397 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
6398 #    define ETHR_HAVE_NATMC32_SET_WB 1
6399 #  endif
6400 #  undef ETHR_HAVE_NATMC32_SET_ACQB
6401 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
6402 #    define ETHR_HAVE_NATMC32_SET_ACQB 1
6403 #  endif
6404 #  undef ETHR_HAVE_NATMC32_SET_RELB
6405 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
6406 #    define ETHR_HAVE_NATMC32_SET_RELB 1
6407 #  endif
6408 #  undef ETHR_HAVE_NATMC32_SET_MB
6409 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
6410 #    define ETHR_HAVE_NATMC32_SET_MB 1
6411 #  endif
6412 #  undef ETHR_HAVE_NATMC32_INIT
6413 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
6414 #    define ETHR_HAVE_NATMC32_INIT 1
6415 #  endif
6416 #  undef ETHR_HAVE_NATMC32_INIT_RB
6417 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
6418 #    define ETHR_HAVE_NATMC32_INIT_RB 1
6419 #  endif
6420 #  undef ETHR_HAVE_NATMC32_INIT_WB
6421 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
6422 #    define ETHR_HAVE_NATMC32_INIT_WB 1
6423 #  endif
6424 #  undef ETHR_HAVE_NATMC32_INIT_ACQB
6425 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
6426 #    define ETHR_HAVE_NATMC32_INIT_ACQB 1
6427 #  endif
6428 #  undef ETHR_HAVE_NATMC32_INIT_RELB
6429 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
6430 #    define ETHR_HAVE_NATMC32_INIT_RELB 1
6431 #  endif
6432 #  undef ETHR_HAVE_NATMC32_INIT_MB
6433 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
6434 #    define ETHR_HAVE_NATMC32_INIT_MB 1
6435 #  endif
6436 #  undef ETHR_HAVE_NATMC32_ADD_RETURN
6437 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
6438 #    define ETHR_HAVE_NATMC32_ADD_RETURN 1
6439 #  endif
6440 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_RB
6441 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
6442 #    define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1
6443 #  endif
6444 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_WB
6445 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
6446 #    define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1
6447 #  endif
6448 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB
6449 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
6450 #    define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1
6451 #  endif
6452 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB
6453 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
6454 #    define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1
6455 #  endif
6456 #  undef ETHR_HAVE_NATMC32_ADD_RETURN_MB
6457 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
6458 #    define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1
6459 #  endif
6460 #  undef ETHR_HAVE_NATMC32_READ
6461 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
6462 #    define ETHR_HAVE_NATMC32_READ 1
6463 #  endif
6464 #  undef ETHR_HAVE_NATMC32_READ_RB
6465 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
6466 #    define ETHR_HAVE_NATMC32_READ_RB 1
6467 #  endif
6468 #  undef ETHR_HAVE_NATMC32_READ_WB
6469 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
6470 #    define ETHR_HAVE_NATMC32_READ_WB 1
6471 #  endif
6472 #  undef ETHR_HAVE_NATMC32_READ_ACQB
6473 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
6474 #    define ETHR_HAVE_NATMC32_READ_ACQB 1
6475 #  endif
6476 #  undef ETHR_HAVE_NATMC32_READ_RELB
6477 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
6478 #    define ETHR_HAVE_NATMC32_READ_RELB 1
6479 #  endif
6480 #  undef ETHR_HAVE_NATMC32_READ_MB
6481 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
6482 #    define ETHR_HAVE_NATMC32_READ_MB 1
6483 #  endif
6484 #  undef ETHR_HAVE_NATMC32_INC_RETURN
6485 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
6486 #    define ETHR_HAVE_NATMC32_INC_RETURN 1
6487 #  endif
6488 #  undef ETHR_HAVE_NATMC32_INC_RETURN_RB
6489 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
6490 #    define ETHR_HAVE_NATMC32_INC_RETURN_RB 1
6491 #  endif
6492 #  undef ETHR_HAVE_NATMC32_INC_RETURN_WB
6493 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
6494 #    define ETHR_HAVE_NATMC32_INC_RETURN_WB 1
6495 #  endif
6496 #  undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB
6497 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
6498 #    define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1
6499 #  endif
6500 #  undef ETHR_HAVE_NATMC32_INC_RETURN_RELB
6501 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
6502 #    define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1
6503 #  endif
6504 #  undef ETHR_HAVE_NATMC32_INC_RETURN_MB
6505 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
6506 #    define ETHR_HAVE_NATMC32_INC_RETURN_MB 1
6507 #  endif
6508 #  undef ETHR_HAVE_NATMC32_DEC_RETURN
6509 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
6510 #    define ETHR_HAVE_NATMC32_DEC_RETURN 1
6511 #  endif
6512 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_RB
6513 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
6514 #    define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1
6515 #  endif
6516 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_WB
6517 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
6518 #    define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1
6519 #  endif
6520 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB
6521 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
6522 #    define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1
6523 #  endif
6524 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB
6525 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
6526 #    define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1
6527 #  endif
6528 #  undef ETHR_HAVE_NATMC32_DEC_RETURN_MB
6529 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
6530 #    define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1
6531 #  endif
6532 #  undef ETHR_HAVE_NATMC32_ADD
6533 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
6534 #    define ETHR_HAVE_NATMC32_ADD 1
6535 #  endif
6536 #  undef ETHR_HAVE_NATMC32_ADD_RB
6537 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
6538 #    define ETHR_HAVE_NATMC32_ADD_RB 1
6539 #  endif
6540 #  undef ETHR_HAVE_NATMC32_ADD_WB
6541 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
6542 #    define ETHR_HAVE_NATMC32_ADD_WB 1
6543 #  endif
6544 #  undef ETHR_HAVE_NATMC32_ADD_ACQB
6545 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
6546 #    define ETHR_HAVE_NATMC32_ADD_ACQB 1
6547 #  endif
6548 #  undef ETHR_HAVE_NATMC32_ADD_RELB
6549 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
6550 #    define ETHR_HAVE_NATMC32_ADD_RELB 1
6551 #  endif
6552 #  undef ETHR_HAVE_NATMC32_ADD_MB
6553 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
6554 #    define ETHR_HAVE_NATMC32_ADD_MB 1
6555 #  endif
6556 #  undef ETHR_HAVE_NATMC32_INC
6557 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
6558 #    define ETHR_HAVE_NATMC32_INC 1
6559 #  endif
6560 #  undef ETHR_HAVE_NATMC32_INC_RB
6561 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
6562 #    define ETHR_HAVE_NATMC32_INC_RB 1
6563 #  endif
6564 #  undef ETHR_HAVE_NATMC32_INC_WB
6565 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
6566 #    define ETHR_HAVE_NATMC32_INC_WB 1
6567 #  endif
6568 #  undef ETHR_HAVE_NATMC32_INC_ACQB
6569 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
6570 #    define ETHR_HAVE_NATMC32_INC_ACQB 1
6571 #  endif
6572 #  undef ETHR_HAVE_NATMC32_INC_RELB
6573 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
6574 #    define ETHR_HAVE_NATMC32_INC_RELB 1
6575 #  endif
6576 #  undef ETHR_HAVE_NATMC32_INC_MB
6577 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
6578 #    define ETHR_HAVE_NATMC32_INC_MB 1
6579 #  endif
6580 #  undef ETHR_HAVE_NATMC32_DEC
6581 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
6582 #    define ETHR_HAVE_NATMC32_DEC 1
6583 #  endif
6584 #  undef ETHR_HAVE_NATMC32_DEC_RB
6585 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
6586 #    define ETHR_HAVE_NATMC32_DEC_RB 1
6587 #  endif
6588 #  undef ETHR_HAVE_NATMC32_DEC_WB
6589 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
6590 #    define ETHR_HAVE_NATMC32_DEC_WB 1
6591 #  endif
6592 #  undef ETHR_HAVE_NATMC32_DEC_ACQB
6593 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
6594 #    define ETHR_HAVE_NATMC32_DEC_ACQB 1
6595 #  endif
6596 #  undef ETHR_HAVE_NATMC32_DEC_RELB
6597 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
6598 #    define ETHR_HAVE_NATMC32_DEC_RELB 1
6599 #  endif
6600 #  undef ETHR_HAVE_NATMC32_DEC_MB
6601 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
6602 #    define ETHR_HAVE_NATMC32_DEC_MB 1
6603 #  endif
6604 #  undef ETHR_HAVE_NATMC32_AND_RETOLD
6605 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
6606 #    define ETHR_HAVE_NATMC32_AND_RETOLD 1
6607 #  endif
6608 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_RB
6609 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
6610 #    define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1
6611 #  endif
6612 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_WB
6613 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
6614 #    define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1
6615 #  endif
6616 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB
6617 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
6618 #    define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1
6619 #  endif
6620 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB
6621 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
6622 #    define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1
6623 #  endif
6624 #  undef ETHR_HAVE_NATMC32_AND_RETOLD_MB
6625 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
6626 #    define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1
6627 #  endif
6628 #  undef ETHR_HAVE_NATMC32_OR_RETOLD
6629 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
6630 #    define ETHR_HAVE_NATMC32_OR_RETOLD 1
6631 #  endif
6632 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_RB
6633 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
6634 #    define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1
6635 #  endif
6636 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_WB
6637 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
6638 #    define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1
6639 #  endif
6640 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB
6641 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
6642 #    define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1
6643 #  endif
6644 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB
6645 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
6646 #    define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1
6647 #  endif
6648 #  undef ETHR_HAVE_NATMC32_OR_RETOLD_MB
6649 #  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
6650 #    define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1
6651 #  endif
6652 #else
6653 #  error "Invalid native atomic size"
6654 #endif
6655 
6656 #if (!defined(ETHR_HAVE_NATMC32_CMPXCHG) \
6657      && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) \
6658      && !defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) \
6659      && !defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) \
6660      && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) \
6661      && !defined(ETHR_HAVE_NATMC32_CMPXCHG_MB))
6662 #  error "No native cmpxchg() op available"
6663 #endif
6664 
6665 
6666 /*
6667  * Read op used together with cmpxchg() fallback when no native op present.
6668  */
6669 #if defined(ETHR_HAVE_NATMC32_READ)
6670 #define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
6671   ETHR_NATMC32_FUNC__(read)(VAR)
6672 #elif defined(ETHR_HAVE_NATMC32_READ_RB)
6673 #define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
6674   ETHR_NATMC32_FUNC__(read_rb)(VAR)
6675 #elif defined(ETHR_HAVE_NATMC32_READ_WB)
6676 #define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
6677   ETHR_NATMC32_FUNC__(read_wb)(VAR)
6678 #elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
6679 #define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
6680   ETHR_NATMC32_FUNC__(read_acqb)(VAR)
6681 #elif defined(ETHR_HAVE_NATMC32_READ_RELB)
6682 #define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
6683   ETHR_NATMC32_FUNC__(read_relb)(VAR)
6684 #elif defined(ETHR_HAVE_NATMC32_READ_MB)
6685 #define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
6686   ETHR_NATMC32_FUNC__(read_mb)(VAR)
6687 #else
6688 /*
6689  * We have no native read() op; guess zero and then use the
6690  * the atomics actual value returned from cmpxchg().
6691  */
6692 #define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
6693   ((ETHR_NAINT32_T__) 0)
6694 #endif
6695 
6696 /*
6697  * Native cmpxchg() fallback used when no native op present.
6698  */
6699 #define ETHR_NATMC32_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
6700 do { \
6701     ethr_sint32_t AVAL; \
6702     ETHR_NAINT32_T__ new__, act__, exp__; \
6703     act__ = ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR); \
6704     do { \
6705 	exp__ = act__; \
6706 	AVAL = (ethr_sint32_t) act__; \
6707 	{ OPS; } \
6708 	new__ = (ETHR_NAINT32_T__) AVAL; \
6709 	act__ = CMPXCHG(VAR, new__, exp__); \
6710     } while (__builtin_expect(act__ != exp__, 0)); \
6711 } while (0)
6712 
6713 
6714 
6715 /* --- addr() --- */
6716 
ETHR_ATMC32_FUNC__(addr)6717 static ETHR_INLINE ethr_sint32_t *ETHR_ATMC32_FUNC__(addr)(ethr_atomic32_t *var)
6718 {
6719     return (ethr_sint32_t *) ETHR_NATMC32_ADDR_FUNC__(var);
6720 
6721 }
6722 
6723 
6724 /* --- cmpxchg() --- */
6725 
6726 
ETHR_ATMC32_FUNC__(cmpxchg)6727 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
6728 {
6729     ethr_sint32_t res;
6730 #if defined(ETHR_HAVE_NATMC32_CMPXCHG)
6731     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6732 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6733     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6734 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6735     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6736 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6737     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6738 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6739     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6740 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6741     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6742 #else
6743 #error "Missing implementation of ethr_atomic32_cmpxchg()!"
6744 #endif
6745     return res;
6746 }
6747 
ETHR_ATMC32_FUNC__(cmpxchg_rb)6748 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
6749 {
6750     ethr_sint32_t res;
6751 #if defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6752     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6753 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
6754     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6755     ETHR_MEMBAR(ETHR_LoadLoad);
6756 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6757     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6758 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6759     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6760     ETHR_MEMBAR(ETHR_LoadLoad);
6761 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6762     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6763     ETHR_MEMBAR(ETHR_LoadLoad);
6764 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6765     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6766     ETHR_MEMBAR(ETHR_LoadLoad);
6767 #else
6768 #error "Missing implementation of ethr_atomic32_cmpxchg_rb()!"
6769 #endif
6770     return res;
6771 }
6772 
ETHR_ATMC32_FUNC__(cmpxchg_wb)6773 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
6774 {
6775     ethr_sint32_t res;
6776 #if defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6777     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6778 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
6779     ETHR_MEMBAR(ETHR_StoreStore);
6780     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6781 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6782     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6783 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6784     ETHR_MEMBAR(ETHR_StoreStore);
6785     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6786 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6787     ETHR_MEMBAR(ETHR_StoreStore);
6788     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6789 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6790     ETHR_MEMBAR(ETHR_StoreStore);
6791     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6792 #else
6793 #error "Missing implementation of ethr_atomic32_cmpxchg_wb()!"
6794 #endif
6795     return res;
6796 }
6797 
ETHR_ATMC32_FUNC__(cmpxchg_acqb)6798 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
6799 {
6800     ethr_sint32_t res;
6801 #if defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6802     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6803 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6804     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6805     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
6806 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
6807     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6808     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
6809 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6810     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6811 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6812     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6813     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
6814 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6815     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6816     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
6817 #else
6818 #error "Missing implementation of ethr_atomic32_cmpxchg_acqb()!"
6819 #endif
6820     return res;
6821 }
6822 
ETHR_ATMC32_FUNC__(cmpxchg_relb)6823 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
6824 {
6825     ethr_sint32_t res;
6826 #if defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6827     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6828 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6829     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
6830     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6831 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
6832     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
6833     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6834 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6835     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6836 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6837     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
6838     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6839 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6840     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
6841     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6842 #else
6843 #error "Missing implementation of ethr_atomic32_cmpxchg_relb()!"
6844 #endif
6845     return res;
6846 }
6847 
ETHR_ATMC32_FUNC__(cmpxchg_mb)6848 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
6849 {
6850     ethr_sint32_t res;
6851 #if defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6852     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6853 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6854     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6855     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
6856 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6857     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
6858     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6859 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6860     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
6861     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6862     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
6863 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6864     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
6865     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6866     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
6867 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
6868     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
6869     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
6870     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
6871 #else
6872 #error "Missing implementation of ethr_atomic32_cmpxchg_mb()!"
6873 #endif
6874     return res;
6875 }
6876 
ETHR_ATMC32_FUNC__(cmpxchg_ddrb)6877 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
6878 {
6879 #ifdef ETHR_ORDERED_READ_DEPEND
6880     return ETHR_ATMC32_FUNC__(cmpxchg)(var, val, old_val);
6881 #else
6882     return ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, val, old_val);
6883 #endif
6884 }
6885 
6886 
6887 /* --- xchg() --- */
6888 
6889 
ETHR_ATMC32_FUNC__(xchg)6890 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg)(ethr_atomic32_t *var, ethr_sint32_t val)
6891 {
6892     ethr_sint32_t res;
6893 #if defined(ETHR_HAVE_NATMC32_XCHG)
6894     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
6895 #elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
6896     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
6897 #elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
6898     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
6899 #elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
6900     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
6901 #elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
6902     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
6903 #elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
6904     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
6905 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
6906     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
6907 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6908     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
6909 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6910     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
6911 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6912     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
6913 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6914     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
6915 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6916     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
6917 #else
6918 #error "Missing implementation of ethr_atomic32_xchg()!"
6919 #endif
6920     return res;
6921 }
6922 
ETHR_ATMC32_FUNC__(xchg_rb)6923 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
6924 {
6925     ethr_sint32_t res;
6926 #if defined(ETHR_HAVE_NATMC32_XCHG_RB)
6927     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
6928 #elif defined(ETHR_HAVE_NATMC32_XCHG)
6929     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
6930     ETHR_MEMBAR(ETHR_LoadLoad);
6931 #elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
6932     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
6933 #elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
6934     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
6935     ETHR_MEMBAR(ETHR_LoadLoad);
6936 #elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
6937     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
6938     ETHR_MEMBAR(ETHR_LoadLoad);
6939 #elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
6940     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
6941     ETHR_MEMBAR(ETHR_LoadLoad);
6942 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6943     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
6944 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
6945     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
6946     ETHR_MEMBAR(ETHR_LoadLoad);
6947 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6948     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
6949 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6950     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
6951     ETHR_MEMBAR(ETHR_LoadLoad);
6952 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6953     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
6954     ETHR_MEMBAR(ETHR_LoadLoad);
6955 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6956     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
6957     ETHR_MEMBAR(ETHR_LoadLoad);
6958 #else
6959 #error "Missing implementation of ethr_atomic32_xchg_rb()!"
6960 #endif
6961     return res;
6962 }
6963 
ETHR_ATMC32_FUNC__(xchg_wb)6964 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
6965 {
6966     ethr_sint32_t res;
6967 #if defined(ETHR_HAVE_NATMC32_XCHG_WB)
6968     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
6969 #elif defined(ETHR_HAVE_NATMC32_XCHG)
6970     ETHR_MEMBAR(ETHR_StoreStore);
6971     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
6972 #elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
6973     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
6974 #elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
6975     ETHR_MEMBAR(ETHR_StoreStore);
6976     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
6977 #elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
6978     ETHR_MEMBAR(ETHR_StoreStore);
6979     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
6980 #elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
6981     ETHR_MEMBAR(ETHR_StoreStore);
6982     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
6983 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
6984     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
6985 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
6986     ETHR_MEMBAR(ETHR_StoreStore);
6987     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
6988 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
6989     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
6990 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
6991     ETHR_MEMBAR(ETHR_StoreStore);
6992     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
6993 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
6994     ETHR_MEMBAR(ETHR_StoreStore);
6995     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
6996 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
6997     ETHR_MEMBAR(ETHR_StoreStore);
6998     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
6999 #else
7000 #error "Missing implementation of ethr_atomic32_xchg_wb()!"
7001 #endif
7002     return res;
7003 }
7004 
ETHR_ATMC32_FUNC__(xchg_acqb)7005 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
7006 {
7007     ethr_sint32_t res;
7008 #if defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
7009     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
7010 #elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
7011     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
7012     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7013 #elif defined(ETHR_HAVE_NATMC32_XCHG)
7014     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
7015     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7016 #elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
7017     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
7018 #elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
7019     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
7020     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7021 #elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
7022     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
7023     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7024 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7025     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
7026 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7027     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
7028     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7029 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7030     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
7031     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7032 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7033     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
7034 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7035     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
7036     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7037 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7038     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
7039     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7040 #else
7041 #error "Missing implementation of ethr_atomic32_xchg_acqb()!"
7042 #endif
7043     return res;
7044 }
7045 
ETHR_ATMC32_FUNC__(xchg_relb)7046 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
7047 {
7048     ethr_sint32_t res;
7049 #if defined(ETHR_HAVE_NATMC32_XCHG_RELB)
7050     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
7051 #elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
7052     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7053     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
7054 #elif defined(ETHR_HAVE_NATMC32_XCHG)
7055     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7056     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
7057 #elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
7058     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
7059 #elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
7060     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7061     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
7062 #elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
7063     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7064     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
7065 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7066     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
7067 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7068     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7069     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
7070 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7071     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7072     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
7073 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7074     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
7075 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7076     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7077     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
7078 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7079     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7080     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
7081 #else
7082 #error "Missing implementation of ethr_atomic32_xchg_relb()!"
7083 #endif
7084     return res;
7085 }
7086 
ETHR_ATMC32_FUNC__(xchg_mb)7087 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
7088 {
7089     ethr_sint32_t res;
7090 #if defined(ETHR_HAVE_NATMC32_XCHG_MB)
7091     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
7092 #elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
7093     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
7094     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7095 #elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
7096     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7097     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
7098 #elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
7099     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7100     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
7101     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7102 #elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
7103     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7104     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
7105     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7106 #elif defined(ETHR_HAVE_NATMC32_XCHG)
7107     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7108     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
7109     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7110 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7111     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
7112 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7113     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
7114     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7115 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7116     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7117     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
7118 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7119     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7120     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
7121     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7122 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7123     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7124     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
7125     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7126 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7127     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7128     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
7129     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7130 #else
7131 #error "Missing implementation of ethr_atomic32_xchg_mb()!"
7132 #endif
7133     return res;
7134 }
7135 
ETHR_ATMC32_FUNC__(xchg_ddrb)7136 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
7137 {
7138 #ifdef ETHR_ORDERED_READ_DEPEND
7139     return ETHR_ATMC32_FUNC__(xchg)(var, val);
7140 #else
7141     return ETHR_ATMC32_FUNC__(xchg_rb)(var, val);
7142 #endif
7143 }
7144 
7145 
7146 /* --- set() --- */
7147 
7148 
ETHR_ATMC32_FUNC__(set)7149 static ETHR_INLINE void ETHR_ATMC32_FUNC__(set)(ethr_atomic32_t *var, ethr_sint32_t val)
7150 {
7151 #if defined(ETHR_HAVE_NATMC32_SET)
7152     ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
7153 #elif defined(ETHR_HAVE_NATMC32_SET_RB)
7154     ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
7155 #elif defined(ETHR_HAVE_NATMC32_SET_WB)
7156     ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
7157 #elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
7158     ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
7159 #elif defined(ETHR_HAVE_NATMC32_SET_RELB)
7160     ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
7161 #elif defined(ETHR_HAVE_NATMC32_SET_MB)
7162     ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
7163 #else
7164     (void) ETHR_ATMC32_FUNC__(xchg)(var, val);
7165 #endif
7166 }
7167 
ETHR_ATMC32_FUNC__(set_rb)7168 static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
7169 {
7170 #if defined(ETHR_HAVE_NATMC32_SET_RB)
7171     ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
7172 #elif defined(ETHR_HAVE_NATMC32_SET)
7173     ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
7174     ETHR_MEMBAR(ETHR_LoadLoad);
7175 #elif defined(ETHR_HAVE_NATMC32_SET_MB)
7176     ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
7177 #elif defined(ETHR_HAVE_NATMC32_SET_WB)
7178     ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
7179     ETHR_MEMBAR(ETHR_LoadLoad);
7180 #elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
7181     ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
7182     ETHR_MEMBAR(ETHR_LoadLoad);
7183 #elif defined(ETHR_HAVE_NATMC32_SET_RELB)
7184     ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
7185     ETHR_MEMBAR(ETHR_LoadLoad);
7186 #else
7187     (void) ETHR_ATMC32_FUNC__(xchg_rb)(var, val);
7188 #endif
7189 }
7190 
ETHR_ATMC32_FUNC__(set_wb)7191 static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
7192 {
7193 #if defined(ETHR_HAVE_NATMC32_SET_WB)
7194     ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
7195 #elif defined(ETHR_HAVE_NATMC32_SET)
7196     ETHR_MEMBAR(ETHR_StoreStore);
7197     ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
7198 #elif defined(ETHR_HAVE_NATMC32_SET_MB)
7199     ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
7200 #elif defined(ETHR_HAVE_NATMC32_SET_RB)
7201     ETHR_MEMBAR(ETHR_StoreStore);
7202     ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
7203 #elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
7204     ETHR_MEMBAR(ETHR_StoreStore);
7205     ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
7206 #elif defined(ETHR_HAVE_NATMC32_SET_RELB)
7207     ETHR_MEMBAR(ETHR_StoreStore);
7208     ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
7209 #else
7210     (void) ETHR_ATMC32_FUNC__(xchg_wb)(var, val);
7211 #endif
7212 }
7213 
ETHR_ATMC32_FUNC__(set_acqb)7214 static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
7215 {
7216 #if defined(ETHR_HAVE_NATMC32_SET_ACQB)
7217     ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
7218 #elif defined(ETHR_HAVE_NATMC32_SET_RB)
7219     ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
7220     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
7221 #elif defined(ETHR_HAVE_NATMC32_SET)
7222     ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
7223     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
7224 #elif defined(ETHR_HAVE_NATMC32_SET_MB)
7225     ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
7226 #elif defined(ETHR_HAVE_NATMC32_SET_WB)
7227     ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
7228     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
7229 #elif defined(ETHR_HAVE_NATMC32_SET_RELB)
7230     ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
7231     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
7232 #else
7233     (void) ETHR_ATMC32_FUNC__(xchg_acqb)(var, val);
7234 #endif
7235 }
7236 
ETHR_ATMC32_FUNC__(set_relb)7237 static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
7238 {
7239 #if defined(ETHR_HAVE_NATMC32_SET_RELB)
7240     ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
7241 #elif defined(ETHR_HAVE_NATMC32_SET_WB)
7242     ETHR_MEMBAR(ETHR_LoadStore);
7243     ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
7244 #elif defined(ETHR_HAVE_NATMC32_SET)
7245     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
7246     ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
7247 #elif defined(ETHR_HAVE_NATMC32_SET_MB)
7248     ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
7249 #elif defined(ETHR_HAVE_NATMC32_SET_RB)
7250     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
7251     ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
7252 #elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
7253     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
7254     ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
7255 #else
7256     (void) ETHR_ATMC32_FUNC__(xchg_relb)(var, val);
7257 #endif
7258 }
7259 
ETHR_ATMC32_FUNC__(set_mb)7260 static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
7261 {
7262 #if defined(ETHR_HAVE_NATMC32_SET_MB)
7263     ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
7264 #elif defined(ETHR_HAVE_NATMC32_SET_RELB)
7265     ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
7266     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
7267 #elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
7268     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
7269     ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
7270 #elif defined(ETHR_HAVE_NATMC32_SET_WB)
7271     ETHR_MEMBAR(ETHR_LoadStore);
7272     ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
7273     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
7274 #elif defined(ETHR_HAVE_NATMC32_SET_RB)
7275     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
7276     ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
7277     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
7278 #elif defined(ETHR_HAVE_NATMC32_SET)
7279     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
7280     ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
7281     ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
7282 #else
7283     (void) ETHR_ATMC32_FUNC__(xchg_mb)(var, val);
7284 #endif
7285 }
7286 
ETHR_ATMC32_FUNC__(set_ddrb)7287 static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
7288 {
7289 #ifdef ETHR_ORDERED_READ_DEPEND
7290     ETHR_ATMC32_FUNC__(set)(var, val);
7291 #else
7292     ETHR_ATMC32_FUNC__(set_rb)(var, val);
7293 #endif
7294 }
7295 
7296 
7297 /* --- init() --- */
7298 
7299 
ETHR_ATMC32_FUNC__(init)7300 static ETHR_INLINE void ETHR_ATMC32_FUNC__(init)(ethr_atomic32_t *var, ethr_sint32_t val)
7301 {
7302 #if defined(ETHR_HAVE_NATMC32_INIT)
7303     ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
7304 #elif defined(ETHR_HAVE_NATMC32_INIT_RB)
7305     ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
7306 #elif defined(ETHR_HAVE_NATMC32_INIT_WB)
7307     ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
7308 #elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
7309     ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
7310 #elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
7311     ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
7312 #elif defined(ETHR_HAVE_NATMC32_INIT_MB)
7313     ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
7314 #else
7315     ETHR_ATMC32_FUNC__(set)(var, val);
7316 #endif
7317 }
7318 
ETHR_ATMC32_FUNC__(init_rb)7319 static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
7320 {
7321 #if defined(ETHR_HAVE_NATMC32_INIT_RB)
7322     ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
7323 #elif defined(ETHR_HAVE_NATMC32_INIT)
7324     ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
7325     ETHR_MEMBAR(ETHR_LoadLoad);
7326 #elif defined(ETHR_HAVE_NATMC32_INIT_MB)
7327     ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
7328 #elif defined(ETHR_HAVE_NATMC32_INIT_WB)
7329     ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
7330     ETHR_MEMBAR(ETHR_LoadLoad);
7331 #elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
7332     ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
7333     ETHR_MEMBAR(ETHR_LoadLoad);
7334 #elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
7335     ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
7336     ETHR_MEMBAR(ETHR_LoadLoad);
7337 #else
7338     ETHR_ATMC32_FUNC__(set_rb)(var, val);
7339 #endif
7340 }
7341 
ETHR_ATMC32_FUNC__(init_wb)7342 static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
7343 {
7344 #if defined(ETHR_HAVE_NATMC32_INIT_WB)
7345     ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
7346 #elif defined(ETHR_HAVE_NATMC32_INIT)
7347     ETHR_MEMBAR(ETHR_StoreStore);
7348     ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
7349 #elif defined(ETHR_HAVE_NATMC32_INIT_MB)
7350     ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
7351 #elif defined(ETHR_HAVE_NATMC32_INIT_RB)
7352     ETHR_MEMBAR(ETHR_StoreStore);
7353     ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
7354 #elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
7355     ETHR_MEMBAR(ETHR_StoreStore);
7356     ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
7357 #elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
7358     ETHR_MEMBAR(ETHR_StoreStore);
7359     ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
7360 #else
7361     ETHR_ATMC32_FUNC__(set_wb)(var, val);
7362 #endif
7363 }
7364 
ETHR_ATMC32_FUNC__(init_acqb)7365 static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
7366 {
7367 #if defined(ETHR_HAVE_NATMC32_INIT_ACQB)
7368     ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
7369 #elif defined(ETHR_HAVE_NATMC32_INIT_RB)
7370     ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
7371     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7372 #elif defined(ETHR_HAVE_NATMC32_INIT)
7373     ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
7374     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7375 #elif defined(ETHR_HAVE_NATMC32_INIT_MB)
7376     ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
7377 #elif defined(ETHR_HAVE_NATMC32_INIT_WB)
7378     ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
7379     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7380 #elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
7381     ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
7382     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7383 #else
7384     ETHR_ATMC32_FUNC__(set_acqb)(var, val);
7385 #endif
7386 }
7387 
ETHR_ATMC32_FUNC__(init_relb)7388 static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
7389 {
7390 #if defined(ETHR_HAVE_NATMC32_INIT_RELB)
7391     ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
7392 #elif defined(ETHR_HAVE_NATMC32_INIT_WB)
7393     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
7394     ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
7395 #elif defined(ETHR_HAVE_NATMC32_INIT)
7396     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7397     ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
7398 #elif defined(ETHR_HAVE_NATMC32_INIT_MB)
7399     ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
7400 #elif defined(ETHR_HAVE_NATMC32_INIT_RB)
7401     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7402     ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
7403 #elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
7404     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7405     ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
7406 #else
7407     ETHR_ATMC32_FUNC__(set_relb)(var, val);
7408 #endif
7409 }
7410 
ETHR_ATMC32_FUNC__(init_mb)7411 static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
7412 {
7413 #if defined(ETHR_HAVE_NATMC32_INIT_MB)
7414     ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
7415 #elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
7416     ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
7417     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7418 #elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
7419     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7420     ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
7421 #elif defined(ETHR_HAVE_NATMC32_INIT_WB)
7422     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
7423     ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
7424     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7425 #elif defined(ETHR_HAVE_NATMC32_INIT_RB)
7426     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7427     ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
7428     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7429 #elif defined(ETHR_HAVE_NATMC32_INIT)
7430     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7431     ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
7432     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7433 #else
7434     ETHR_ATMC32_FUNC__(set_mb)(var, val);
7435 #endif
7436 }
7437 
ETHR_ATMC32_FUNC__(init_ddrb)7438 static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
7439 {
7440 #ifdef ETHR_ORDERED_READ_DEPEND
7441     ETHR_ATMC32_FUNC__(init)(var, val);
7442 #else
7443     ETHR_ATMC32_FUNC__(init_rb)(var, val);
7444 #endif
7445 }
7446 
7447 
7448 /* --- add_read() --- */
7449 
7450 
ETHR_ATMC32_FUNC__(add_read)7451 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read)(ethr_atomic32_t *var, ethr_sint32_t val)
7452 {
7453     ethr_sint32_t res;
7454 #if defined(ETHR_HAVE_NATMC32_ADD_RETURN)
7455     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
7456 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
7457     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
7458 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
7459     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
7460 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
7461     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
7462 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
7463     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
7464 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
7465     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
7466 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7467     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
7468 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7469     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
7470 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7471     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
7472 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7473     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
7474 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7475     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
7476 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7477     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
7478 #else
7479 #error "Missing implementation of ethr_atomic32_add_read()!"
7480 #endif
7481     return res;
7482 }
7483 
ETHR_ATMC32_FUNC__(add_read_rb)7484 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
7485 {
7486     ethr_sint32_t res;
7487 #if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
7488     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
7489 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
7490     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
7491     ETHR_MEMBAR(ETHR_LoadLoad);
7492 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
7493     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
7494 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
7495     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
7496     ETHR_MEMBAR(ETHR_LoadLoad);
7497 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
7498     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
7499     ETHR_MEMBAR(ETHR_LoadLoad);
7500 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
7501     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
7502     ETHR_MEMBAR(ETHR_LoadLoad);
7503 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7504     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
7505 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7506     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
7507     ETHR_MEMBAR(ETHR_LoadLoad);
7508 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7509     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
7510 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7511     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
7512     ETHR_MEMBAR(ETHR_LoadLoad);
7513 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7514     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
7515     ETHR_MEMBAR(ETHR_LoadLoad);
7516 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7517     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
7518     ETHR_MEMBAR(ETHR_LoadLoad);
7519 #else
7520 #error "Missing implementation of ethr_atomic32_add_read_rb()!"
7521 #endif
7522     return res;
7523 }
7524 
ETHR_ATMC32_FUNC__(add_read_wb)7525 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
7526 {
7527     ethr_sint32_t res;
7528 #if defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
7529     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
7530 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
7531     ETHR_MEMBAR(ETHR_StoreStore);
7532     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
7533 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
7534     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
7535 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
7536     ETHR_MEMBAR(ETHR_StoreStore);
7537     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
7538 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
7539     ETHR_MEMBAR(ETHR_StoreStore);
7540     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
7541 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
7542     ETHR_MEMBAR(ETHR_StoreStore);
7543     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
7544 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7545     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
7546 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7547     ETHR_MEMBAR(ETHR_StoreStore);
7548     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
7549 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7550     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
7551 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7552     ETHR_MEMBAR(ETHR_StoreStore);
7553     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
7554 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7555     ETHR_MEMBAR(ETHR_StoreStore);
7556     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
7557 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7558     ETHR_MEMBAR(ETHR_StoreStore);
7559     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
7560 #else
7561 #error "Missing implementation of ethr_atomic32_add_read_wb()!"
7562 #endif
7563     return res;
7564 }
7565 
ETHR_ATMC32_FUNC__(add_read_acqb)7566 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
7567 {
7568     ethr_sint32_t res;
7569 #if defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
7570     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
7571 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
7572     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
7573     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7574 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
7575     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
7576     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7577 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
7578     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
7579 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
7580     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
7581     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7582 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
7583     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
7584     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7585 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7586     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
7587 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7588     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
7589     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7590 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7591     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
7592     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7593 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7594     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
7595 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7596     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
7597     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7598 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7599     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
7600     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7601 #else
7602 #error "Missing implementation of ethr_atomic32_add_read_acqb()!"
7603 #endif
7604     return res;
7605 }
7606 
ETHR_ATMC32_FUNC__(add_read_relb)7607 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
7608 {
7609     ethr_sint32_t res;
7610 #if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
7611     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
7612 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
7613     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7614     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
7615 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
7616     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7617     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
7618 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
7619     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
7620 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
7621     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7622     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
7623 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
7624     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7625     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
7626 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7627     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
7628 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7629     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7630     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
7631 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7632     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7633     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
7634 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7635     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
7636 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7637     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7638     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
7639 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7640     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7641     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
7642 #else
7643 #error "Missing implementation of ethr_atomic32_add_read_relb()!"
7644 #endif
7645     return res;
7646 }
7647 
ETHR_ATMC32_FUNC__(add_read_mb)7648 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
7649 {
7650     ethr_sint32_t res;
7651 #if defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
7652     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
7653 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
7654     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
7655     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7656 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
7657     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7658     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
7659 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
7660     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7661     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
7662     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7663 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
7664     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7665     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
7666     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7667 #elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
7668     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7669     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
7670     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7671 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
7672     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
7673 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
7674     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
7675     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7676 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
7677     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7678     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
7679 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
7680     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7681     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
7682     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7683 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
7684     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7685     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
7686     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7687 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
7688     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7689     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
7690     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7691 #else
7692 #error "Missing implementation of ethr_atomic32_add_read_mb()!"
7693 #endif
7694     return res;
7695 }
7696 
ETHR_ATMC32_FUNC__(add_read_ddrb)7697 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
7698 {
7699 #ifdef ETHR_ORDERED_READ_DEPEND
7700     return ETHR_ATMC32_FUNC__(add_read)(var, val);
7701 #else
7702     return ETHR_ATMC32_FUNC__(add_read_rb)(var, val);
7703 #endif
7704 }
7705 
7706 
7707 /* --- read() --- */
7708 
7709 
ETHR_ATMC32_FUNC__(read)7710 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read)(ethr_atomic32_t *var)
7711 {
7712     ethr_sint32_t res;
7713 #if defined(ETHR_HAVE_NATMC32_READ)
7714     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
7715 #elif defined(ETHR_HAVE_NATMC32_READ_RB)
7716     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
7717 #elif defined(ETHR_HAVE_NATMC32_READ_WB)
7718     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
7719 #elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
7720     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
7721 #elif defined(ETHR_HAVE_NATMC32_READ_RELB)
7722     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
7723 #elif defined(ETHR_HAVE_NATMC32_READ_MB)
7724     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
7725 #else
7726     res = ETHR_ATMC32_FUNC__(cmpxchg)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
7727 #endif
7728     return res;
7729 }
7730 
ETHR_ATMC32_FUNC__(read_rb)7731 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_rb)(ethr_atomic32_t *var)
7732 {
7733     ethr_sint32_t res;
7734 #if defined(ETHR_HAVE_NATMC32_READ_RB)
7735     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
7736 #elif defined(ETHR_HAVE_NATMC32_READ)
7737     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
7738     ETHR_MEMBAR(ETHR_LoadLoad);
7739 #elif defined(ETHR_HAVE_NATMC32_READ_MB)
7740     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
7741 #elif defined(ETHR_HAVE_NATMC32_READ_WB)
7742     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
7743     ETHR_MEMBAR(ETHR_LoadLoad);
7744 #elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
7745     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
7746     ETHR_MEMBAR(ETHR_LoadLoad);
7747 #elif defined(ETHR_HAVE_NATMC32_READ_RELB)
7748     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
7749     ETHR_MEMBAR(ETHR_LoadLoad);
7750 #else
7751     res = ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
7752 #endif
7753     return res;
7754 }
7755 
ETHR_ATMC32_FUNC__(read_wb)7756 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_wb)(ethr_atomic32_t *var)
7757 {
7758     ethr_sint32_t res;
7759 #if defined(ETHR_HAVE_NATMC32_READ_WB)
7760     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
7761 #elif defined(ETHR_HAVE_NATMC32_READ)
7762     ETHR_MEMBAR(ETHR_StoreStore);
7763     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
7764 #elif defined(ETHR_HAVE_NATMC32_READ_MB)
7765     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
7766 #elif defined(ETHR_HAVE_NATMC32_READ_RB)
7767     ETHR_MEMBAR(ETHR_StoreStore);
7768     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
7769 #elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
7770     ETHR_MEMBAR(ETHR_StoreStore);
7771     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
7772 #elif defined(ETHR_HAVE_NATMC32_READ_RELB)
7773     ETHR_MEMBAR(ETHR_StoreStore);
7774     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
7775 #else
7776     res = ETHR_ATMC32_FUNC__(cmpxchg_wb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
7777 #endif
7778     return res;
7779 }
7780 
ETHR_ATMC32_FUNC__(read_acqb)7781 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_acqb)(ethr_atomic32_t *var)
7782 {
7783     ethr_sint32_t res;
7784 #if defined(ETHR_HAVE_NATMC32_READ_ACQB)
7785     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
7786 #elif defined(ETHR_HAVE_NATMC32_READ_RB)
7787     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
7788     ETHR_MEMBAR(ETHR_LoadStore);
7789 #elif defined(ETHR_HAVE_NATMC32_READ)
7790     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
7791     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
7792 #elif defined(ETHR_HAVE_NATMC32_READ_MB)
7793     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
7794 #elif defined(ETHR_HAVE_NATMC32_READ_WB)
7795     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
7796     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
7797 #elif defined(ETHR_HAVE_NATMC32_READ_RELB)
7798     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
7799     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
7800 #else
7801     res = ETHR_ATMC32_FUNC__(cmpxchg_acqb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
7802 #endif
7803     return res;
7804 }
7805 
ETHR_ATMC32_FUNC__(read_relb)7806 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_relb)(ethr_atomic32_t *var)
7807 {
7808     ethr_sint32_t res;
7809 #if defined(ETHR_HAVE_NATMC32_READ_RELB)
7810     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
7811 #elif defined(ETHR_HAVE_NATMC32_READ_WB)
7812     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7813     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
7814 #elif defined(ETHR_HAVE_NATMC32_READ)
7815     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7816     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
7817 #elif defined(ETHR_HAVE_NATMC32_READ_MB)
7818     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
7819 #elif defined(ETHR_HAVE_NATMC32_READ_RB)
7820     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7821     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
7822 #elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
7823     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7824     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
7825 #else
7826     res = ETHR_ATMC32_FUNC__(cmpxchg_relb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
7827 #endif
7828     return res;
7829 }
7830 
ETHR_ATMC32_FUNC__(read_mb)7831 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_mb)(ethr_atomic32_t *var)
7832 {
7833     ethr_sint32_t res;
7834 #if defined(ETHR_HAVE_NATMC32_READ_MB)
7835     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
7836 #elif defined(ETHR_HAVE_NATMC32_READ_RELB)
7837     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
7838     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
7839 #elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
7840     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7841     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
7842 #elif defined(ETHR_HAVE_NATMC32_READ_WB)
7843     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7844     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
7845     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
7846 #elif defined(ETHR_HAVE_NATMC32_READ_RB)
7847     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7848     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
7849     ETHR_MEMBAR(ETHR_LoadStore);
7850 #elif defined(ETHR_HAVE_NATMC32_READ)
7851     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7852     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
7853     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
7854 #else
7855     res = ETHR_ATMC32_FUNC__(cmpxchg_mb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
7856 #endif
7857     return res;
7858 }
7859 
ETHR_ATMC32_FUNC__(read_ddrb)7860 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_ddrb)(ethr_atomic32_t *var)
7861 {
7862 #ifdef ETHR_ORDERED_READ_DEPEND
7863     return ETHR_ATMC32_FUNC__(read)(var);
7864 #else
7865     return ETHR_ATMC32_FUNC__(read_rb)(var);
7866 #endif
7867 }
7868 
7869 
7870 /* --- inc_read() --- */
7871 
7872 
ETHR_ATMC32_FUNC__(inc_read)7873 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read)(ethr_atomic32_t *var)
7874 {
7875     ethr_sint32_t res;
7876 #if defined(ETHR_HAVE_NATMC32_INC_RETURN)
7877     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
7878 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
7879     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
7880 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
7881     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
7882 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
7883     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
7884 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
7885     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
7886 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
7887     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
7888 #else
7889     res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) 1);
7890 #endif
7891     return res;
7892 }
7893 
ETHR_ATMC32_FUNC__(inc_read_rb)7894 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_rb)(ethr_atomic32_t *var)
7895 {
7896     ethr_sint32_t res;
7897 #if defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
7898     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
7899 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
7900     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
7901     ETHR_MEMBAR(ETHR_LoadLoad);
7902 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
7903     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
7904 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
7905     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
7906     ETHR_MEMBAR(ETHR_LoadLoad);
7907 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
7908     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
7909     ETHR_MEMBAR(ETHR_LoadLoad);
7910 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
7911     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
7912     ETHR_MEMBAR(ETHR_LoadLoad);
7913 #else
7914     res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) 1);
7915 #endif
7916     return res;
7917 }
7918 
ETHR_ATMC32_FUNC__(inc_read_wb)7919 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_wb)(ethr_atomic32_t *var)
7920 {
7921     ethr_sint32_t res;
7922 #if defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
7923     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
7924 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
7925     ETHR_MEMBAR(ETHR_StoreStore);
7926     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
7927 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
7928     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
7929 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
7930     ETHR_MEMBAR(ETHR_StoreStore);
7931     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
7932 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
7933     ETHR_MEMBAR(ETHR_StoreStore);
7934     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
7935 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
7936     ETHR_MEMBAR(ETHR_StoreStore);
7937     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
7938 #else
7939     res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) 1);
7940 #endif
7941     return res;
7942 }
7943 
ETHR_ATMC32_FUNC__(inc_read_acqb)7944 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_acqb)(ethr_atomic32_t *var)
7945 {
7946     ethr_sint32_t res;
7947 #if defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
7948     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
7949 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
7950     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
7951     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7952 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
7953     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
7954     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7955 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
7956     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
7957 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
7958     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
7959     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7960 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
7961     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
7962     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
7963 #else
7964     res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) 1);
7965 #endif
7966     return res;
7967 }
7968 
ETHR_ATMC32_FUNC__(inc_read_relb)7969 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_relb)(ethr_atomic32_t *var)
7970 {
7971     ethr_sint32_t res;
7972 #if defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
7973     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
7974 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
7975     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7976     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
7977 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
7978     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7979     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
7980 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
7981     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
7982 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
7983     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7984     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
7985 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
7986     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
7987     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
7988 #else
7989     res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) 1);
7990 #endif
7991     return res;
7992 }
7993 
ETHR_ATMC32_FUNC__(inc_read_mb)7994 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_mb)(ethr_atomic32_t *var)
7995 {
7996     ethr_sint32_t res;
7997 #if defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
7998     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
7999 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
8000     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
8001     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8002 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
8003     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8004     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
8005 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
8006     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8007     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
8008     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8009 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
8010     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8011     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
8012     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8013 #elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
8014     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8015     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
8016     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8017 #else
8018     res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) 1);
8019 #endif
8020     return res;
8021 }
8022 
ETHR_ATMC32_FUNC__(inc_read_ddrb)8023 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_ddrb)(ethr_atomic32_t *var)
8024 {
8025 #ifdef ETHR_ORDERED_READ_DEPEND
8026     return ETHR_ATMC32_FUNC__(inc_read)(var);
8027 #else
8028     return ETHR_ATMC32_FUNC__(inc_read_rb)(var);
8029 #endif
8030 }
8031 
8032 
8033 /* --- dec_read() --- */
8034 
8035 
ETHR_ATMC32_FUNC__(dec_read)8036 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read)(ethr_atomic32_t *var)
8037 {
8038     ethr_sint32_t res;
8039 #if defined(ETHR_HAVE_NATMC32_DEC_RETURN)
8040     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
8041 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
8042     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
8043 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
8044     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
8045 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
8046     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
8047 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
8048     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
8049 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
8050     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
8051 #else
8052     res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) -1);
8053 #endif
8054     return res;
8055 }
8056 
ETHR_ATMC32_FUNC__(dec_read_rb)8057 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_rb)(ethr_atomic32_t *var)
8058 {
8059     ethr_sint32_t res;
8060 #if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
8061     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
8062 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
8063     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
8064     ETHR_MEMBAR(ETHR_LoadLoad);
8065 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
8066     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
8067 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
8068     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
8069     ETHR_MEMBAR(ETHR_LoadLoad);
8070 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
8071     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
8072     ETHR_MEMBAR(ETHR_LoadLoad);
8073 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
8074     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
8075     ETHR_MEMBAR(ETHR_LoadLoad);
8076 #else
8077     res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) -1);
8078 #endif
8079     return res;
8080 }
8081 
ETHR_ATMC32_FUNC__(dec_read_wb)8082 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_wb)(ethr_atomic32_t *var)
8083 {
8084     ethr_sint32_t res;
8085 #if defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
8086     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
8087 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
8088     ETHR_MEMBAR(ETHR_StoreStore);
8089     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
8090 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
8091     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
8092 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
8093     ETHR_MEMBAR(ETHR_StoreStore);
8094     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
8095 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
8096     ETHR_MEMBAR(ETHR_StoreStore);
8097     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
8098 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
8099     ETHR_MEMBAR(ETHR_StoreStore);
8100     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
8101 #else
8102     res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) -1);
8103 #endif
8104     return res;
8105 }
8106 
ETHR_ATMC32_FUNC__(dec_read_acqb)8107 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_acqb)(ethr_atomic32_t *var)
8108 {
8109     ethr_sint32_t res;
8110 #if defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
8111     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
8112 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
8113     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
8114     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8115 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
8116     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
8117     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8118 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
8119     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
8120 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
8121     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
8122     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8123 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
8124     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
8125     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8126 #else
8127     res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) -1);
8128 #endif
8129     return res;
8130 }
8131 
ETHR_ATMC32_FUNC__(dec_read_relb)8132 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_relb)(ethr_atomic32_t *var)
8133 {
8134     ethr_sint32_t res;
8135 #if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
8136     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
8137 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
8138     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8139     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
8140 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
8141     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8142     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
8143 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
8144     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
8145 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
8146     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8147     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
8148 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
8149     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8150     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
8151 #else
8152     res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) -1);
8153 #endif
8154     return res;
8155 }
8156 
ETHR_ATMC32_FUNC__(dec_read_mb)8157 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_mb)(ethr_atomic32_t *var)
8158 {
8159     ethr_sint32_t res;
8160 #if defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
8161     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
8162 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
8163     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
8164     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8165 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
8166     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8167     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
8168 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
8169     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8170     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
8171     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8172 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
8173     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8174     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
8175     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8176 #elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
8177     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8178     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
8179     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8180 #else
8181     res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) -1);
8182 #endif
8183     return res;
8184 }
8185 
ETHR_ATMC32_FUNC__(dec_read_ddrb)8186 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_ddrb)(ethr_atomic32_t *var)
8187 {
8188 #ifdef ETHR_ORDERED_READ_DEPEND
8189     return ETHR_ATMC32_FUNC__(dec_read)(var);
8190 #else
8191     return ETHR_ATMC32_FUNC__(dec_read_rb)(var);
8192 #endif
8193 }
8194 
8195 
8196 /* --- add() --- */
8197 
8198 
ETHR_ATMC32_FUNC__(add)8199 static ETHR_INLINE void ETHR_ATMC32_FUNC__(add)(ethr_atomic32_t *var, ethr_sint32_t val)
8200 {
8201 #if defined(ETHR_HAVE_NATMC32_ADD)
8202     ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
8203 #elif defined(ETHR_HAVE_NATMC32_ADD_RB)
8204     ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
8205 #elif defined(ETHR_HAVE_NATMC32_ADD_WB)
8206     ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
8207 #elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
8208     ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
8209 #elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
8210     ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
8211 #elif defined(ETHR_HAVE_NATMC32_ADD_MB)
8212     ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
8213 #else
8214     (void) ETHR_ATMC32_FUNC__(add_read)(var, val);
8215 #endif
8216 }
8217 
ETHR_ATMC32_FUNC__(add_rb)8218 static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
8219 {
8220 #if defined(ETHR_HAVE_NATMC32_ADD_RB)
8221     ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
8222 #elif defined(ETHR_HAVE_NATMC32_ADD)
8223     ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
8224     ETHR_MEMBAR(ETHR_LoadLoad);
8225 #elif defined(ETHR_HAVE_NATMC32_ADD_MB)
8226     ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
8227 #elif defined(ETHR_HAVE_NATMC32_ADD_WB)
8228     ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
8229     ETHR_MEMBAR(ETHR_LoadLoad);
8230 #elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
8231     ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
8232     ETHR_MEMBAR(ETHR_LoadLoad);
8233 #elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
8234     ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
8235     ETHR_MEMBAR(ETHR_LoadLoad);
8236 #else
8237     (void) ETHR_ATMC32_FUNC__(add_read_rb)(var, val);
8238 #endif
8239 }
8240 
ETHR_ATMC32_FUNC__(add_wb)8241 static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
8242 {
8243 #if defined(ETHR_HAVE_NATMC32_ADD_WB)
8244     ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
8245 #elif defined(ETHR_HAVE_NATMC32_ADD)
8246     ETHR_MEMBAR(ETHR_StoreStore);
8247     ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
8248 #elif defined(ETHR_HAVE_NATMC32_ADD_MB)
8249     ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
8250 #elif defined(ETHR_HAVE_NATMC32_ADD_RB)
8251     ETHR_MEMBAR(ETHR_StoreStore);
8252     ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
8253 #elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
8254     ETHR_MEMBAR(ETHR_StoreStore);
8255     ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
8256 #elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
8257     ETHR_MEMBAR(ETHR_StoreStore);
8258     ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
8259 #else
8260     (void) ETHR_ATMC32_FUNC__(add_read_wb)(var, val);
8261 #endif
8262 }
8263 
ETHR_ATMC32_FUNC__(add_acqb)8264 static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
8265 {
8266 #if defined(ETHR_HAVE_NATMC32_ADD_ACQB)
8267     ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
8268 #elif defined(ETHR_HAVE_NATMC32_ADD_RB)
8269     ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
8270     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8271 #elif defined(ETHR_HAVE_NATMC32_ADD)
8272     ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
8273     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8274 #elif defined(ETHR_HAVE_NATMC32_ADD_MB)
8275     ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
8276 #elif defined(ETHR_HAVE_NATMC32_ADD_WB)
8277     ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
8278     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8279 #elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
8280     ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
8281     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8282 #else
8283     (void) ETHR_ATMC32_FUNC__(add_read_acqb)(var, val);
8284 #endif
8285 }
8286 
ETHR_ATMC32_FUNC__(add_relb)8287 static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
8288 {
8289 #if defined(ETHR_HAVE_NATMC32_ADD_RELB)
8290     ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
8291 #elif defined(ETHR_HAVE_NATMC32_ADD_WB)
8292     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8293     ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
8294 #elif defined(ETHR_HAVE_NATMC32_ADD)
8295     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8296     ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
8297 #elif defined(ETHR_HAVE_NATMC32_ADD_MB)
8298     ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
8299 #elif defined(ETHR_HAVE_NATMC32_ADD_RB)
8300     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8301     ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
8302 #elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
8303     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8304     ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
8305 #else
8306     (void) ETHR_ATMC32_FUNC__(add_read_relb)(var, val);
8307 #endif
8308 }
8309 
ETHR_ATMC32_FUNC__(add_mb)8310 static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
8311 {
8312 #if defined(ETHR_HAVE_NATMC32_ADD_MB)
8313     ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
8314 #elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
8315     ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
8316     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8317 #elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
8318     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8319     ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
8320 #elif defined(ETHR_HAVE_NATMC32_ADD_WB)
8321     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8322     ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
8323     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8324 #elif defined(ETHR_HAVE_NATMC32_ADD_RB)
8325     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8326     ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
8327     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8328 #elif defined(ETHR_HAVE_NATMC32_ADD)
8329     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8330     ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
8331     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8332 #else
8333     (void) ETHR_ATMC32_FUNC__(add_read_mb)(var, val);
8334 #endif
8335 }
8336 
ETHR_ATMC32_FUNC__(add_ddrb)8337 static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
8338 {
8339 #ifdef ETHR_ORDERED_READ_DEPEND
8340     ETHR_ATMC32_FUNC__(add)(var, val);
8341 #else
8342     ETHR_ATMC32_FUNC__(add_rb)(var, val);
8343 #endif
8344 }
8345 
8346 
8347 /* --- inc() --- */
8348 
8349 
ETHR_ATMC32_FUNC__(inc)8350 static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc)(ethr_atomic32_t *var)
8351 {
8352 #if defined(ETHR_HAVE_NATMC32_INC)
8353     ETHR_NATMC32_FUNC__(inc)(var);
8354 #elif defined(ETHR_HAVE_NATMC32_INC_RB)
8355     ETHR_NATMC32_FUNC__(inc_rb)(var);
8356 #elif defined(ETHR_HAVE_NATMC32_INC_WB)
8357     ETHR_NATMC32_FUNC__(inc_wb)(var);
8358 #elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
8359     ETHR_NATMC32_FUNC__(inc_acqb)(var);
8360 #elif defined(ETHR_HAVE_NATMC32_INC_RELB)
8361     ETHR_NATMC32_FUNC__(inc_relb)(var);
8362 #elif defined(ETHR_HAVE_NATMC32_INC_MB)
8363     ETHR_NATMC32_FUNC__(inc_mb)(var);
8364 #else
8365     (void) ETHR_ATMC32_FUNC__(inc_read)(var);
8366 #endif
8367 }
8368 
ETHR_ATMC32_FUNC__(inc_rb)8369 static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_rb)(ethr_atomic32_t *var)
8370 {
8371 #if defined(ETHR_HAVE_NATMC32_INC_RB)
8372     ETHR_NATMC32_FUNC__(inc_rb)(var);
8373 #elif defined(ETHR_HAVE_NATMC32_INC)
8374     ETHR_NATMC32_FUNC__(inc)(var);
8375     ETHR_MEMBAR(ETHR_LoadLoad);
8376 #elif defined(ETHR_HAVE_NATMC32_INC_MB)
8377     ETHR_NATMC32_FUNC__(inc_mb)(var);
8378 #elif defined(ETHR_HAVE_NATMC32_INC_WB)
8379     ETHR_NATMC32_FUNC__(inc_wb)(var);
8380     ETHR_MEMBAR(ETHR_LoadLoad);
8381 #elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
8382     ETHR_NATMC32_FUNC__(inc_acqb)(var);
8383     ETHR_MEMBAR(ETHR_LoadLoad);
8384 #elif defined(ETHR_HAVE_NATMC32_INC_RELB)
8385     ETHR_NATMC32_FUNC__(inc_relb)(var);
8386     ETHR_MEMBAR(ETHR_LoadLoad);
8387 #else
8388     (void) ETHR_ATMC32_FUNC__(inc_read_rb)(var);
8389 #endif
8390 }
8391 
ETHR_ATMC32_FUNC__(inc_wb)8392 static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_wb)(ethr_atomic32_t *var)
8393 {
8394 #if defined(ETHR_HAVE_NATMC32_INC_WB)
8395     ETHR_NATMC32_FUNC__(inc_wb)(var);
8396 #elif defined(ETHR_HAVE_NATMC32_INC)
8397     ETHR_MEMBAR(ETHR_StoreStore);
8398     ETHR_NATMC32_FUNC__(inc)(var);
8399 #elif defined(ETHR_HAVE_NATMC32_INC_MB)
8400     ETHR_NATMC32_FUNC__(inc_mb)(var);
8401 #elif defined(ETHR_HAVE_NATMC32_INC_RB)
8402     ETHR_MEMBAR(ETHR_StoreStore);
8403     ETHR_NATMC32_FUNC__(inc_rb)(var);
8404 #elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
8405     ETHR_MEMBAR(ETHR_StoreStore);
8406     ETHR_NATMC32_FUNC__(inc_acqb)(var);
8407 #elif defined(ETHR_HAVE_NATMC32_INC_RELB)
8408     ETHR_MEMBAR(ETHR_StoreStore);
8409     ETHR_NATMC32_FUNC__(inc_relb)(var);
8410 #else
8411     (void) ETHR_ATMC32_FUNC__(inc_read_wb)(var);
8412 #endif
8413 }
8414 
ETHR_ATMC32_FUNC__(inc_acqb)8415 static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_acqb)(ethr_atomic32_t *var)
8416 {
8417 #if defined(ETHR_HAVE_NATMC32_INC_ACQB)
8418     ETHR_NATMC32_FUNC__(inc_acqb)(var);
8419 #elif defined(ETHR_HAVE_NATMC32_INC_RB)
8420     ETHR_NATMC32_FUNC__(inc_rb)(var);
8421     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8422 #elif defined(ETHR_HAVE_NATMC32_INC)
8423     ETHR_NATMC32_FUNC__(inc)(var);
8424     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8425 #elif defined(ETHR_HAVE_NATMC32_INC_MB)
8426     ETHR_NATMC32_FUNC__(inc_mb)(var);
8427 #elif defined(ETHR_HAVE_NATMC32_INC_WB)
8428     ETHR_NATMC32_FUNC__(inc_wb)(var);
8429     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8430 #elif defined(ETHR_HAVE_NATMC32_INC_RELB)
8431     ETHR_NATMC32_FUNC__(inc_relb)(var);
8432     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8433 #else
8434     (void) ETHR_ATMC32_FUNC__(inc_read_acqb)(var);
8435 #endif
8436 }
8437 
ETHR_ATMC32_FUNC__(inc_relb)8438 static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_relb)(ethr_atomic32_t *var)
8439 {
8440 #if defined(ETHR_HAVE_NATMC32_INC_RELB)
8441     ETHR_NATMC32_FUNC__(inc_relb)(var);
8442 #elif defined(ETHR_HAVE_NATMC32_INC_WB)
8443     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8444     ETHR_NATMC32_FUNC__(inc_wb)(var);
8445 #elif defined(ETHR_HAVE_NATMC32_INC)
8446     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8447     ETHR_NATMC32_FUNC__(inc)(var);
8448 #elif defined(ETHR_HAVE_NATMC32_INC_MB)
8449     ETHR_NATMC32_FUNC__(inc_mb)(var);
8450 #elif defined(ETHR_HAVE_NATMC32_INC_RB)
8451     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8452     ETHR_NATMC32_FUNC__(inc_rb)(var);
8453 #elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
8454     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8455     ETHR_NATMC32_FUNC__(inc_acqb)(var);
8456 #else
8457     (void) ETHR_ATMC32_FUNC__(inc_read_relb)(var);
8458 #endif
8459 }
8460 
ETHR_ATMC32_FUNC__(inc_mb)8461 static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_mb)(ethr_atomic32_t *var)
8462 {
8463 #if defined(ETHR_HAVE_NATMC32_INC_MB)
8464     ETHR_NATMC32_FUNC__(inc_mb)(var);
8465 #elif defined(ETHR_HAVE_NATMC32_INC_RELB)
8466     ETHR_NATMC32_FUNC__(inc_relb)(var);
8467     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8468 #elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
8469     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8470     ETHR_NATMC32_FUNC__(inc_acqb)(var);
8471 #elif defined(ETHR_HAVE_NATMC32_INC_WB)
8472     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8473     ETHR_NATMC32_FUNC__(inc_wb)(var);
8474     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8475 #elif defined(ETHR_HAVE_NATMC32_INC_RB)
8476     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8477     ETHR_NATMC32_FUNC__(inc_rb)(var);
8478     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8479 #elif defined(ETHR_HAVE_NATMC32_INC)
8480     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8481     ETHR_NATMC32_FUNC__(inc)(var);
8482     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8483 #else
8484     (void) ETHR_ATMC32_FUNC__(inc_read_mb)(var);
8485 #endif
8486 }
8487 
ETHR_ATMC32_FUNC__(inc_ddrb)8488 static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_ddrb)(ethr_atomic32_t *var)
8489 {
8490 #ifdef ETHR_ORDERED_READ_DEPEND
8491     ETHR_ATMC32_FUNC__(inc)(var);
8492 #else
8493     ETHR_ATMC32_FUNC__(inc_rb)(var);
8494 #endif
8495 }
8496 
8497 
8498 /* --- dec() --- */
8499 
8500 
ETHR_ATMC32_FUNC__(dec)8501 static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec)(ethr_atomic32_t *var)
8502 {
8503 #if defined(ETHR_HAVE_NATMC32_DEC)
8504     ETHR_NATMC32_FUNC__(dec)(var);
8505 #elif defined(ETHR_HAVE_NATMC32_DEC_RB)
8506     ETHR_NATMC32_FUNC__(dec_rb)(var);
8507 #elif defined(ETHR_HAVE_NATMC32_DEC_WB)
8508     ETHR_NATMC32_FUNC__(dec_wb)(var);
8509 #elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
8510     ETHR_NATMC32_FUNC__(dec_acqb)(var);
8511 #elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
8512     ETHR_NATMC32_FUNC__(dec_relb)(var);
8513 #elif defined(ETHR_HAVE_NATMC32_DEC_MB)
8514     ETHR_NATMC32_FUNC__(dec_mb)(var);
8515 #else
8516     (void) ETHR_ATMC32_FUNC__(dec_read)(var);
8517 #endif
8518 }
8519 
ETHR_ATMC32_FUNC__(dec_rb)8520 static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_rb)(ethr_atomic32_t *var)
8521 {
8522 #if defined(ETHR_HAVE_NATMC32_DEC_RB)
8523     ETHR_NATMC32_FUNC__(dec_rb)(var);
8524 #elif defined(ETHR_HAVE_NATMC32_DEC)
8525     ETHR_NATMC32_FUNC__(dec)(var);
8526     ETHR_MEMBAR(ETHR_LoadLoad);
8527 #elif defined(ETHR_HAVE_NATMC32_DEC_MB)
8528     ETHR_NATMC32_FUNC__(dec_mb)(var);
8529 #elif defined(ETHR_HAVE_NATMC32_DEC_WB)
8530     ETHR_NATMC32_FUNC__(dec_wb)(var);
8531     ETHR_MEMBAR(ETHR_LoadLoad);
8532 #elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
8533     ETHR_NATMC32_FUNC__(dec_acqb)(var);
8534     ETHR_MEMBAR(ETHR_LoadLoad);
8535 #elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
8536     ETHR_NATMC32_FUNC__(dec_relb)(var);
8537     ETHR_MEMBAR(ETHR_LoadLoad);
8538 #else
8539     (void) ETHR_ATMC32_FUNC__(dec_read_rb)(var);
8540 #endif
8541 }
8542 
ETHR_ATMC32_FUNC__(dec_wb)8543 static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_wb)(ethr_atomic32_t *var)
8544 {
8545 #if defined(ETHR_HAVE_NATMC32_DEC_WB)
8546     ETHR_NATMC32_FUNC__(dec_wb)(var);
8547 #elif defined(ETHR_HAVE_NATMC32_DEC)
8548     ETHR_MEMBAR(ETHR_StoreStore);
8549     ETHR_NATMC32_FUNC__(dec)(var);
8550 #elif defined(ETHR_HAVE_NATMC32_DEC_MB)
8551     ETHR_NATMC32_FUNC__(dec_mb)(var);
8552 #elif defined(ETHR_HAVE_NATMC32_DEC_RB)
8553     ETHR_MEMBAR(ETHR_StoreStore);
8554     ETHR_NATMC32_FUNC__(dec_rb)(var);
8555 #elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
8556     ETHR_MEMBAR(ETHR_StoreStore);
8557     ETHR_NATMC32_FUNC__(dec_acqb)(var);
8558 #elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
8559     ETHR_MEMBAR(ETHR_StoreStore);
8560     ETHR_NATMC32_FUNC__(dec_relb)(var);
8561 #else
8562     (void) ETHR_ATMC32_FUNC__(dec_read_wb)(var);
8563 #endif
8564 }
8565 
ETHR_ATMC32_FUNC__(dec_acqb)8566 static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_acqb)(ethr_atomic32_t *var)
8567 {
8568 #if defined(ETHR_HAVE_NATMC32_DEC_ACQB)
8569     ETHR_NATMC32_FUNC__(dec_acqb)(var);
8570 #elif defined(ETHR_HAVE_NATMC32_DEC_RB)
8571     ETHR_NATMC32_FUNC__(dec_rb)(var);
8572     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8573 #elif defined(ETHR_HAVE_NATMC32_DEC)
8574     ETHR_NATMC32_FUNC__(dec)(var);
8575     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8576 #elif defined(ETHR_HAVE_NATMC32_DEC_MB)
8577     ETHR_NATMC32_FUNC__(dec_mb)(var);
8578 #elif defined(ETHR_HAVE_NATMC32_DEC_WB)
8579     ETHR_NATMC32_FUNC__(dec_wb)(var);
8580     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8581 #elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
8582     ETHR_NATMC32_FUNC__(dec_relb)(var);
8583     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8584 #else
8585     (void) ETHR_ATMC32_FUNC__(dec_read_acqb)(var);
8586 #endif
8587 }
8588 
ETHR_ATMC32_FUNC__(dec_relb)8589 static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_relb)(ethr_atomic32_t *var)
8590 {
8591 #if defined(ETHR_HAVE_NATMC32_DEC_RELB)
8592     ETHR_NATMC32_FUNC__(dec_relb)(var);
8593 #elif defined(ETHR_HAVE_NATMC32_DEC_WB)
8594     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8595     ETHR_NATMC32_FUNC__(dec_wb)(var);
8596 #elif defined(ETHR_HAVE_NATMC32_DEC)
8597     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8598     ETHR_NATMC32_FUNC__(dec)(var);
8599 #elif defined(ETHR_HAVE_NATMC32_DEC_MB)
8600     ETHR_NATMC32_FUNC__(dec_mb)(var);
8601 #elif defined(ETHR_HAVE_NATMC32_DEC_RB)
8602     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8603     ETHR_NATMC32_FUNC__(dec_rb)(var);
8604 #elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
8605     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8606     ETHR_NATMC32_FUNC__(dec_acqb)(var);
8607 #else
8608     (void) ETHR_ATMC32_FUNC__(dec_read_relb)(var);
8609 #endif
8610 }
8611 
ETHR_ATMC32_FUNC__(dec_mb)8612 static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_mb)(ethr_atomic32_t *var)
8613 {
8614 #if defined(ETHR_HAVE_NATMC32_DEC_MB)
8615     ETHR_NATMC32_FUNC__(dec_mb)(var);
8616 #elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
8617     ETHR_NATMC32_FUNC__(dec_relb)(var);
8618     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8619 #elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
8620     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8621     ETHR_NATMC32_FUNC__(dec_acqb)(var);
8622 #elif defined(ETHR_HAVE_NATMC32_DEC_WB)
8623     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8624     ETHR_NATMC32_FUNC__(dec_wb)(var);
8625     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8626 #elif defined(ETHR_HAVE_NATMC32_DEC_RB)
8627     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8628     ETHR_NATMC32_FUNC__(dec_rb)(var);
8629     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8630 #elif defined(ETHR_HAVE_NATMC32_DEC)
8631     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8632     ETHR_NATMC32_FUNC__(dec)(var);
8633     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8634 #else
8635     (void) ETHR_ATMC32_FUNC__(dec_read_mb)(var);
8636 #endif
8637 }
8638 
ETHR_ATMC32_FUNC__(dec_ddrb)8639 static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_ddrb)(ethr_atomic32_t *var)
8640 {
8641 #ifdef ETHR_ORDERED_READ_DEPEND
8642     ETHR_ATMC32_FUNC__(dec)(var);
8643 #else
8644     ETHR_ATMC32_FUNC__(dec_rb)(var);
8645 #endif
8646 }
8647 
8648 
8649 /* --- read_band() --- */
8650 
8651 
ETHR_ATMC32_FUNC__(read_band)8652 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band)(ethr_atomic32_t *var, ethr_sint32_t val)
8653 {
8654     ethr_sint32_t res;
8655 #if defined(ETHR_HAVE_NATMC32_AND_RETOLD)
8656     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
8657 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
8658     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
8659 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
8660     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
8661 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
8662     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
8663 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
8664     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
8665 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
8666     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
8667 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
8668     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
8669 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
8670     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
8671 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
8672     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
8673 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
8674     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
8675 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
8676     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
8677 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
8678     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
8679 #else
8680 #error "Missing implementation of ethr_atomic32_read_band()!"
8681 #endif
8682     return res;
8683 }
8684 
ETHR_ATMC32_FUNC__(read_band_rb)8685 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
8686 {
8687     ethr_sint32_t res;
8688 #if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
8689     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
8690 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
8691     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
8692     ETHR_MEMBAR(ETHR_LoadLoad);
8693 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
8694     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
8695 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
8696     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
8697     ETHR_MEMBAR(ETHR_LoadLoad);
8698 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
8699     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
8700     ETHR_MEMBAR(ETHR_LoadLoad);
8701 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
8702     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
8703     ETHR_MEMBAR(ETHR_LoadLoad);
8704 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
8705     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
8706 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
8707     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
8708     ETHR_MEMBAR(ETHR_LoadLoad);
8709 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
8710     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
8711 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
8712     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
8713     ETHR_MEMBAR(ETHR_LoadLoad);
8714 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
8715     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
8716     ETHR_MEMBAR(ETHR_LoadLoad);
8717 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
8718     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
8719     ETHR_MEMBAR(ETHR_LoadLoad);
8720 #else
8721 #error "Missing implementation of ethr_atomic32_read_band_rb()!"
8722 #endif
8723     return res;
8724 }
8725 
ETHR_ATMC32_FUNC__(read_band_wb)8726 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
8727 {
8728     ethr_sint32_t res;
8729 #if defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
8730     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
8731 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
8732     ETHR_MEMBAR(ETHR_StoreStore);
8733     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
8734 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
8735     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
8736 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
8737     ETHR_MEMBAR(ETHR_StoreStore);
8738     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
8739 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
8740     ETHR_MEMBAR(ETHR_StoreStore);
8741     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
8742 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
8743     ETHR_MEMBAR(ETHR_StoreStore);
8744     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
8745 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
8746     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
8747 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
8748     ETHR_MEMBAR(ETHR_StoreStore);
8749     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
8750 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
8751     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
8752 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
8753     ETHR_MEMBAR(ETHR_StoreStore);
8754     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
8755 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
8756     ETHR_MEMBAR(ETHR_StoreStore);
8757     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
8758 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
8759     ETHR_MEMBAR(ETHR_StoreStore);
8760     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
8761 #else
8762 #error "Missing implementation of ethr_atomic32_read_band_wb()!"
8763 #endif
8764     return res;
8765 }
8766 
ETHR_ATMC32_FUNC__(read_band_acqb)8767 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
8768 {
8769     ethr_sint32_t res;
8770 #if defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
8771     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
8772 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
8773     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
8774     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8775 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
8776     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
8777     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8778 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
8779     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
8780 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
8781     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
8782     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8783 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
8784     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
8785     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8786 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
8787     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
8788 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
8789     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
8790     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8791 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
8792     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
8793     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8794 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
8795     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
8796 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
8797     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
8798     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8799 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
8800     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
8801     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8802 #else
8803 #error "Missing implementation of ethr_atomic32_read_band_acqb()!"
8804 #endif
8805     return res;
8806 }
8807 
ETHR_ATMC32_FUNC__(read_band_relb)8808 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
8809 {
8810     ethr_sint32_t res;
8811 #if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
8812     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
8813 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
8814     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8815     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
8816 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
8817     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8818     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
8819 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
8820     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
8821 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
8822     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8823     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
8824 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
8825     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8826     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
8827 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
8828     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
8829 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
8830     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8831     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
8832 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
8833     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8834     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
8835 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
8836     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
8837 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
8838     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8839     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
8840 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
8841     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8842     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
8843 #else
8844 #error "Missing implementation of ethr_atomic32_read_band_relb()!"
8845 #endif
8846     return res;
8847 }
8848 
ETHR_ATMC32_FUNC__(read_band_mb)8849 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
8850 {
8851     ethr_sint32_t res;
8852 #if defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
8853     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
8854 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
8855     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
8856     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8857 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
8858     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8859     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
8860 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
8861     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8862     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
8863     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8864 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
8865     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8866     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
8867     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8868 #elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
8869     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8870     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
8871     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8872 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
8873     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
8874 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
8875     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
8876     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8877 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
8878     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8879     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
8880 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
8881     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8882     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
8883     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8884 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
8885     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8886     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
8887     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8888 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
8889     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
8890     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
8891     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
8892 #else
8893 #error "Missing implementation of ethr_atomic32_read_band_mb()!"
8894 #endif
8895     return res;
8896 }
8897 
ETHR_ATMC32_FUNC__(read_band_ddrb)8898 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
8899 {
8900 #ifdef ETHR_ORDERED_READ_DEPEND
8901     return ETHR_ATMC32_FUNC__(read_band)(var, val);
8902 #else
8903     return ETHR_ATMC32_FUNC__(read_band_rb)(var, val);
8904 #endif
8905 }
8906 
8907 
8908 /* --- read_bor() --- */
8909 
8910 
ETHR_ATMC32_FUNC__(read_bor)8911 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor)(ethr_atomic32_t *var, ethr_sint32_t val)
8912 {
8913     ethr_sint32_t res;
8914 #if defined(ETHR_HAVE_NATMC32_OR_RETOLD)
8915     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
8916 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
8917     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
8918 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
8919     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
8920 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
8921     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
8922 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
8923     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
8924 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
8925     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
8926 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
8927     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
8928 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
8929     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
8930 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
8931     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
8932 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
8933     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
8934 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
8935     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
8936 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
8937     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
8938 #else
8939 #error "Missing implementation of ethr_atomic32_read_bor()!"
8940 #endif
8941     return res;
8942 }
8943 
ETHR_ATMC32_FUNC__(read_bor_rb)8944 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
8945 {
8946     ethr_sint32_t res;
8947 #if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
8948     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
8949 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
8950     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
8951     ETHR_MEMBAR(ETHR_LoadLoad);
8952 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
8953     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
8954 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
8955     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
8956     ETHR_MEMBAR(ETHR_LoadLoad);
8957 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
8958     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
8959     ETHR_MEMBAR(ETHR_LoadLoad);
8960 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
8961     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
8962     ETHR_MEMBAR(ETHR_LoadLoad);
8963 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
8964     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
8965 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
8966     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
8967     ETHR_MEMBAR(ETHR_LoadLoad);
8968 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
8969     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
8970 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
8971     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
8972     ETHR_MEMBAR(ETHR_LoadLoad);
8973 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
8974     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
8975     ETHR_MEMBAR(ETHR_LoadLoad);
8976 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
8977     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
8978     ETHR_MEMBAR(ETHR_LoadLoad);
8979 #else
8980 #error "Missing implementation of ethr_atomic32_read_bor_rb()!"
8981 #endif
8982     return res;
8983 }
8984 
ETHR_ATMC32_FUNC__(read_bor_wb)8985 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
8986 {
8987     ethr_sint32_t res;
8988 #if defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
8989     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
8990 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
8991     ETHR_MEMBAR(ETHR_StoreStore);
8992     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
8993 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
8994     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
8995 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
8996     ETHR_MEMBAR(ETHR_StoreStore);
8997     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
8998 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
8999     ETHR_MEMBAR(ETHR_StoreStore);
9000     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
9001 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
9002     ETHR_MEMBAR(ETHR_StoreStore);
9003     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
9004 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
9005     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
9006 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
9007     ETHR_MEMBAR(ETHR_StoreStore);
9008     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
9009 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
9010     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
9011 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
9012     ETHR_MEMBAR(ETHR_StoreStore);
9013     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
9014 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
9015     ETHR_MEMBAR(ETHR_StoreStore);
9016     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
9017 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
9018     ETHR_MEMBAR(ETHR_StoreStore);
9019     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
9020 #else
9021 #error "Missing implementation of ethr_atomic32_read_bor_wb()!"
9022 #endif
9023     return res;
9024 }
9025 
ETHR_ATMC32_FUNC__(read_bor_acqb)9026 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
9027 {
9028     ethr_sint32_t res;
9029 #if defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
9030     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
9031 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
9032     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
9033     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9034 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
9035     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
9036     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9037 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
9038     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
9039 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
9040     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
9041     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9042 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
9043     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
9044     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9045 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
9046     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
9047 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
9048     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
9049     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9050 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
9051     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
9052     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9053 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
9054     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
9055 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
9056     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
9057     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9058 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
9059     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
9060     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9061 #else
9062 #error "Missing implementation of ethr_atomic32_read_bor_acqb()!"
9063 #endif
9064     return res;
9065 }
9066 
ETHR_ATMC32_FUNC__(read_bor_relb)9067 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
9068 {
9069     ethr_sint32_t res;
9070 #if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
9071     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
9072 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
9073     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9074     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
9075 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
9076     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9077     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
9078 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
9079     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
9080 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
9081     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9082     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
9083 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
9084     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9085     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
9086 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
9087     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
9088 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
9089     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9090     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
9091 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
9092     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9093     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
9094 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
9095     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
9096 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
9097     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9098     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
9099 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
9100     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9101     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
9102 #else
9103 #error "Missing implementation of ethr_atomic32_read_bor_relb()!"
9104 #endif
9105     return res;
9106 }
9107 
ETHR_ATMC32_FUNC__(read_bor_mb)9108 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
9109 {
9110     ethr_sint32_t res;
9111 #if defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
9112     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
9113 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
9114     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
9115     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9116 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
9117     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9118     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
9119 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
9120     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9121     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
9122     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9123 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
9124     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9125     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
9126     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9127 #elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
9128     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9129     res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
9130     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9131 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
9132     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
9133 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
9134     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
9135     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9136 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
9137     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9138     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
9139 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
9140     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9141     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
9142     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9143 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
9144     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9145     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
9146     ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9147 #elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
9148     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
9149     ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
9150     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
9151 #else
9152 #error "Missing implementation of ethr_atomic32_read_bor_mb()!"
9153 #endif
9154     return res;
9155 }
9156 
ETHR_ATMC32_FUNC__(read_bor_ddrb)9157 static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
9158 {
9159 #ifdef ETHR_ORDERED_READ_DEPEND
9160     return ETHR_ATMC32_FUNC__(read_bor)(var, val);
9161 #else
9162     return ETHR_ATMC32_FUNC__(read_bor_rb)(var, val);
9163 #endif
9164 }
9165 
9166 #endif /* ETHR_ATMC32_INLINE__ */
9167 
9168 #endif /* ETHR_ATOMICS_H__ */
9169