1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2010-2016. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 /*
22 * Description: Native atomics ethread support using libatomic_ops
23 * Author: Rickard Green
24 */
25
26 #ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__
27 #define ETHR_LIBATOMIC_OPS_ATOMIC_H__
28
29 /*
30 * libatomic_ops can be downloaded from:
31 * http://www.hpl.hp.com/research/linux/atomic_ops/
32 *
33 * These operations need to be defined by libatomic_ops;
34 * otherwise, we won't compile:
35 * - AO_nop_full()
36 * - AO_load() || AO_load_aquire()
37 * - AO_store() || AO_store_release()
38 * - AO_compare_and_swap() || AO_compare_and_swap_acquire()
39 * || AO_compare_and_swap_release() || AO_compare_and_swap_full()
40 *
41 */
42
43 #if ETHR_SIZEOF_AO_T == 4
44 #define ETHR_HAVE_NATIVE_ATOMIC32 1
45 #define ETHR_NATIVE_ATOMIC32_IMPL ETHR_NATIVE_IMPL__
46 #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
47 #define ETHR_ATMC_T__ ethr_native_atomic32_t
48 #define ETHR_AINT_T__ ethr_sint32_t
49 #define ETHR_AINT_SUFFIX__ "l"
50 #elif ETHR_SIZEOF_AO_T == 8
51 #define ETHR_HAVE_NATIVE_ATOMIC64 1
52 #define ETHR_NATIVE_ATOMIC64_IMPL ETHR_NATIVE_IMPL__
53 #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
54 #define ETHR_ATMC_T__ ethr_native_atomic64_t
55 #define ETHR_AINT_T__ ethr_sint64_t
56 #define ETHR_AINT_SUFFIX__ "q"
57 #else
58 #error "Unsupported integer size"
59 #endif
60
61 typedef struct {
62 volatile AO_t counter;
63 } ETHR_ATMC_T__;
64
65 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
66
67 #if ETHR_SIZEOF_AO_T == 4
68 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
69 #else
70 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
71 #endif
72
73 static ETHR_INLINE ETHR_AINT_T__ *
ETHR_NATMC_FUNC__(addr)74 ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
75 {
76 return (ETHR_AINT_T__ *) &var->counter;
77 }
78
79 #ifdef AO_HAVE_store
80
81 #if ETHR_SIZEOF_AO_T == 4
82 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
83 #else
84 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
85 #endif
86
87 static ETHR_INLINE void
ETHR_NATMC_FUNC__(set)88 ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
89 {
90 AO_store(&var->counter, (AO_t) value);
91 }
92
93 #endif
94
95 #ifdef AO_HAVE_store_write
96
97 #if ETHR_SIZEOF_AO_T == 4
98 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB 1
99 #else
100 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB 1
101 #endif
102
103 static ETHR_INLINE void
ETHR_NATMC_FUNC__(set_wb)104 ETHR_NATMC_FUNC__(set_wb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
105 {
106 AO_store_write(&var->counter, (AO_t) value);
107 }
108
109 #endif
110
111 #ifdef AO_HAVE_store_release
112
113 #if ETHR_SIZEOF_AO_T == 4
114 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
115 #else
116 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
117 #endif
118
119 static ETHR_INLINE void
ETHR_NATMC_FUNC__(set_relb)120 ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
121 {
122 AO_store_release(&var->counter, (AO_t) value);
123 }
124
125 #endif
126
127 #ifdef AO_HAVE_store_full
128
129 #if ETHR_SIZEOF_AO_T == 4
130 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1
131 #else
132 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1
133 #endif
134
135 static ETHR_INLINE void
ETHR_NATMC_FUNC__(set_mb)136 ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
137 {
138 AO_store_full(&var->counter, (AO_t) value);
139 }
140
141 #endif
142
143 #ifdef AO_HAVE_load
144
145 #if ETHR_SIZEOF_AO_T == 4
146 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
147 #else
148 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
149 #endif
150
151 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read)152 ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
153 {
154 return (ETHR_AINT_T__) AO_load(&var->counter);
155 }
156
157 #endif
158
159 #ifdef AO_HAVE_load_read
160
161 #if ETHR_SIZEOF_AO_T == 4
162 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB 1
163 #else
164 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB 1
165 #endif
166
167 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read_rb)168 ETHR_NATMC_FUNC__(read_rb)(ETHR_ATMC_T__ *var)
169 {
170 return (ETHR_AINT_T__) AO_load_read(&var->counter);
171 }
172
173 #endif
174
175 #ifdef AO_HAVE_load_acquire
176
177 #if ETHR_SIZEOF_AO_T == 4
178 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
179 #else
180 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1
181 #endif
182
183 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read_acqb)184 ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
185 {
186 return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
187 }
188
189 #endif
190
191 #ifdef AO_HAVE_load_full
192
193 #if ETHR_SIZEOF_AO_T == 4
194 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB 1
195 #else
196 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB 1
197 #endif
198
199 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read_mb)200 ETHR_NATMC_FUNC__(read_mb)(ETHR_ATMC_T__ *var)
201 {
202 return (ETHR_AINT_T__) AO_load_full(&var->counter);
203 }
204
205 #endif
206
207 #ifdef AO_HAVE_fetch_and_add
208
209 #if ETHR_SIZEOF_AO_T == 4
210 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
211 #else
212 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN 1
213 #endif
214
215 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return)216 ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
217 {
218 return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr;
219 }
220
221 #endif
222
223 #ifdef AO_HAVE_fetch_and_add_acquire
224
225 #if ETHR_SIZEOF_AO_T == 4
226 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1
227 #else
228 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB 1
229 #endif
230
231 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return_acqb)232 ETHR_NATMC_FUNC__(add_return_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
233 {
234 return ((ETHR_AINT_T__) AO_fetch_and_add_acquire(&var->counter, (AO_t) incr)) + incr;
235 }
236
237 #endif
238
239 #ifdef AO_HAVE_fetch_and_add_release
240
241 #if ETHR_SIZEOF_AO_T == 4
242 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB 1
243 #else
244 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB 1
245 #endif
246
247 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return_relb)248 ETHR_NATMC_FUNC__(add_return_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
249 {
250 return ((ETHR_AINT_T__) AO_fetch_and_add_release(&var->counter, (AO_t) incr)) + incr;
251 }
252
253 #endif
254
255 #ifdef AO_HAVE_fetch_and_add_full
256
257 #if ETHR_SIZEOF_AO_T == 4
258 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
259 #else
260 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
261 #endif
262
263 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return_mb)264 ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
265 {
266 return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr;
267 }
268
269 #endif
270
271 #ifdef AO_HAVE_fetch_and_add1
272
273 #if ETHR_SIZEOF_AO_T == 4
274 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
275 #else
276 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN 1
277 #endif
278
279 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(inc_return)280 ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
281 {
282 return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1;
283 }
284
285 #endif
286
287 #ifdef AO_HAVE_fetch_and_add1_acquire
288
289 #if ETHR_SIZEOF_AO_T == 4
290 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
291 #else
292 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
293 #endif
294
295 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(inc_return_acqb)296 ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
297 {
298 return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1;
299 }
300
301 #endif
302
303 #ifdef AO_HAVE_fetch_and_add1_release
304
305 #if ETHR_SIZEOF_AO_T == 4
306 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB 1
307 #else
308 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB 1
309 #endif
310
311 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(inc_return_relb)312 ETHR_NATMC_FUNC__(inc_return_relb)(ETHR_ATMC_T__ *var)
313 {
314 return ((ETHR_AINT_T__) AO_fetch_and_add1_release(&var->counter)) + 1;
315 }
316
317 #endif
318
319 #ifdef AO_HAVE_fetch_and_add1_full
320
321 #if ETHR_SIZEOF_AO_T == 4
322 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1
323 #else
324 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1
325 #endif
326
327 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(inc_return_mb)328 ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var)
329 {
330 return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1;
331 }
332
333 #endif
334
335 #ifdef AO_HAVE_fetch_and_sub1
336
337 #if ETHR_SIZEOF_AO_T == 4
338 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
339 #else
340 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN 1
341 #endif
342
343 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(dec_return)344 ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
345 {
346 return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1;
347 }
348
349 #endif
350
351 #ifdef AO_HAVE_fetch_and_sub1_acquire
352
353 #if ETHR_SIZEOF_AO_T == 4
354 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1
355 #else
356 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB 1
357 #endif
358
359 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(dec_return_acqb)360 ETHR_NATMC_FUNC__(dec_return_acqb)(ETHR_ATMC_T__ *var)
361 {
362 return ((ETHR_AINT_T__) AO_fetch_and_sub1_acquire(&var->counter)) - 1;
363 }
364
365 #endif
366
367 #ifdef AO_HAVE_fetch_and_sub1_release
368
369 #if ETHR_SIZEOF_AO_T == 4
370 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
371 #else
372 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
373 #endif
374
375 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(dec_return_relb)376 ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
377 {
378 return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1;
379 }
380
381 #endif
382
383 #ifdef AO_HAVE_fetch_and_sub1_full
384
385 #if ETHR_SIZEOF_AO_T == 4
386 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1
387 #else
388 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1
389 #endif
390
391 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(dec_return_mb)392 ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var)
393 {
394 return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1;
395 }
396
397 #endif
398
399 #if defined(AO_HAVE_compare_and_swap_full) || defined(AO_HAVE_fetch_compare_and_swap_full)
400
401 #if ETHR_SIZEOF_AO_T == 4
402 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
403 #else
404 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
405 #endif
406
407 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_mb)408 ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
409 ETHR_AINT_T__ new,
410 ETHR_AINT_T__ exp)
411 {
412 #if defined(AO_HAVE_fetch_compare_and_swap_full)
413 return (ETHR_AINT_T__) AO_fetch_compare_and_swap_full(&var->counter,
414 (AO_t) exp,
415 (AO_t) new);
416 #else
417 ETHR_AINT_T__ act;
418 do {
419 if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new))
420 return exp;
421 #ifdef AO_HAVE_load_acquire
422 act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
423 #else
424 act = (ETHR_AINT_T__) AO_load(&var->counter);
425 #endif
426 } while (act == exp);
427 #ifndef AO_HAVE_load_acquire
428 AO_nop_full();
429 #endif
430 return act;
431 #endif
432 }
433
434 #endif
435
436 #if defined(AO_HAVE_compare_and_swap) || defined(AO_HAVE_fetch_compare_and_swap)
437
438 #if ETHR_SIZEOF_AO_T == 4
439 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
440 #else
441 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1
442 #endif
443
444 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg)445 ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
446 ETHR_AINT_T__ new,
447 ETHR_AINT_T__ exp)
448 {
449 #if defined(AO_HAVE_fetch_compare_and_swap)
450 return (ETHR_AINT_T__) AO_fetch_compare_and_swap(&var->counter,
451 (AO_t) exp,
452 (AO_t) new);
453 #else
454 ETHR_AINT_T__ act;
455 do {
456 if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new))
457 return exp;
458 #ifdef AO_HAVE_load
459 act = (ETHR_AINT_T__) AO_load(&var->counter);
460 #else
461 act = (ETHR_AINT_T__) AO_load_aquire(&var->counter);
462 #endif
463 } while (act == exp);
464 return act;
465 #endif
466 }
467
468 #endif
469
470 #if defined(AO_HAVE_compare_and_swap_acquire) || defined(AO_HAVE_fetch_compare_and_swap_acquire)
471
472 #if ETHR_SIZEOF_AO_T == 4
473 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
474 #else
475 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
476 #endif
477
478 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_acqb)479 ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
480 ETHR_AINT_T__ new,
481 ETHR_AINT_T__ exp)
482 {
483 #if defined(AO_HAVE_fetch_compare_and_swap_acquire)
484 return (ETHR_AINT_T__) AO_fetch_compare_and_swap_acquire(&var->counter,
485 (AO_t) exp,
486 (AO_t) new);
487 #else
488 ETHR_AINT_T__ act;
489 do {
490 if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new))
491 return exp;
492 #ifdef AO_HAVE_load_acquire
493 act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
494 #else
495 act = (ETHR_AINT_T__) AO_load(&var->counter);
496 #endif
497 } while (act == exp);
498 #ifndef AO_HAVE_load_acquire
499 AO_nop_full();
500 #endif
501 return act;
502 #endif
503 }
504
505 #endif
506
507 #if defined(AO_HAVE_compare_and_swap_read) || defined(AO_HAVE_fetch_compare_and_swap_read)
508
509 #if ETHR_SIZEOF_AO_T == 4
510 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB 1
511 #else
512 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB 1
513 #endif
514
515 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_rb)516 ETHR_NATMC_FUNC__(cmpxchg_rb)(ETHR_ATMC_T__ *var,
517 ETHR_AINT_T__ new,
518 ETHR_AINT_T__ exp)
519 {
520 #if defined(AO_HAVE_fetch_compare_and_swap_read)
521 return (ETHR_AINT_T__) AO_fetch_compare_and_swap_read(&var->counter,
522 (AO_t) exp,
523 (AO_t) new);
524 #else
525 ETHR_AINT_T__ act;
526 do {
527 if (AO_compare_and_swap_read(&var->counter, (AO_t) exp, (AO_t) new))
528 return exp;
529 #if defined(AO_HAVE_load_read)
530 act = (ETHR_AINT_T__) AO_load_read(&var->counter);
531 #elif defined(AO_HAVE_load)
532 act = (ETHR_AINT_T__) AO_load(&var->counter);
533 #else
534 act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
535 #endif
536 } while (act == exp);
537 #ifndef AO_HAVE_load_read
538 #ifdef AO_HAVE_nop_read
539 AO_nop_read();
540 #else
541 AO_nop_full();
542 #endif
543 #endif
544 return act;
545 #endif
546 }
547
548 #endif
549
550 #if defined(AO_HAVE_compare_and_swap_release) || defined(AO_HAVE_fetch_compare_and_swap_release)
551
552 #if ETHR_SIZEOF_AO_T == 4
553 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
554 #else
555 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
556 #endif
557
558 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_relb)559 ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
560 ETHR_AINT_T__ new,
561 ETHR_AINT_T__ exp)
562 {
563 #if defined(AO_HAVE_fetch_compare_and_swap_release)
564 return (ETHR_AINT_T__) AO_fetch_compare_and_swap_release(&var->counter,
565 (AO_t) exp,
566 (AO_t) new);
567 #else
568 ETHR_AINT_T__ act;
569 do {
570 if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new))
571 return exp;
572 #ifdef AO_HAVE_load
573 act = (ETHR_AINT_T__) AO_load(&var->counter);
574 #else
575 act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
576 #endif
577 } while (act == exp);
578 return act;
579 #endif
580 }
581
582 #endif
583
584 #if defined(AO_HAVE_compare_and_swap_write) || defined(AO_HAVE_fetch_compare_and_swap_write)
585
586 #if ETHR_SIZEOF_AO_T == 4
587 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB 1
588 #else
589 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB 1
590 #endif
591
592 static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_wb)593 ETHR_NATMC_FUNC__(cmpxchg_wb)(ETHR_ATMC_T__ *var,
594 ETHR_AINT_T__ new,
595 ETHR_AINT_T__ exp)
596 {
597 #if defined(AO_HAVE_fetch_compare_and_swap_write)
598 return (ETHR_AINT_T__) AO_fetch_compare_and_swap_write(&var->counter,
599 (AO_t) exp,
600 (AO_t) new);
601 #else
602 ETHR_AINT_T__ act;
603 do {
604 if (AO_compare_and_swap_write(&var->counter, (AO_t) exp, (AO_t) new))
605 return exp;
606 #ifdef AO_HAVE_load
607 act = (ETHR_AINT_T__) AO_load(&var->counter);
608 #else
609 act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
610 #endif
611 } while (act == exp);
612 return act;
613 #endif
614 }
615
616 #endif
617
618 #endif /* ETHR_TRY_INLINE_FUNCS */
619
620 #undef ETHR_NATMC_FUNC__
621 #undef ETHR_ATMC_T__
622 #undef ETHR_AINT_T__
623
624 #endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */
625