1 /*
2 * Copyright (c) 2003-2004 Hewlett-Packard Development Company, L.P.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 * SOFTWARE.
21 */
22
23 /*
24 * Generalize atomic operations for atomic_ops.h.
25 * Should not be included directly.
26 *
27 * We make no attempt to define useless operations, such as
28 * AO_nop_acquire
29 * AO_nop_release
30 *
31 * We have also so far neglected to define some others, which
32 * do not appear likely to be useful, e.g. stores with acquire
33 * or read barriers.
34 *
35 * This file is sometimes included twice by atomic_ops.h.
36 * All definitions include explicit checks that we are not replacing
37 * an earlier definition. In general, more desirable expansions
38 * appear earlier so that we are more likely to use them.
39 *
40 * We only make safe generalizations, except that by default we define
41 * the ...dd_acquire_read operations to be equivalent to those without
42 * a barrier. On platforms for which this is unsafe, the platform-specific
43 * file must define AO_NO_DD_ORDERING.
44 */
45
46 #ifndef ATOMIC_OPS_H
47 # error Atomic_ops_generalize.h should not be included directly.
48 #endif
49
50 #if AO_CHAR_TS_T
51 # define AO_TS_COMPARE_AND_SWAP_FULL(a,o,n) \
52 AO_char_compare_and_swap_full(a,o,n)
53 # define AO_TS_COMPARE_AND_SWAP_ACQUIRE(a,o,n) \
54 AO_char_compare_and_swap_acquire(a,o,n)
55 # define AO_TS_COMPARE_AND_SWAP_RELEASE(a,o,n) \
56 AO_char_compare_and_swap_release(a,o,n)
57 # define AO_TS_COMPARE_AND_SWAP(a,o,n) \
58 AO_char_compare_and_swap(a,o,n)
59 #endif
60
61 #if AO_AO_TS_T
62 # define AO_TS_COMPARE_AND_SWAP_FULL(a,o,n) \
63 AO_compare_and_swap_full(a,o,n)
64 # define AO_TS_COMPARE_AND_SWAP_ACQUIRE(a,o,n) \
65 AO_compare_and_swap_acquire(a,o,n)
66 # define AO_TS_COMPARE_AND_SWAP_RELEASE(a,o,n) \
67 AO_compare_and_swap_release(a,o,n)
68 # define AO_TS_COMPARE_AND_SWAP(a,o,n) \
69 AO_compare_and_swap(a,o,n)
70 #endif
71
72 /* Generate test_and_set_full, if necessary and possible. */
73 #if !defined(AO_HAVE_test_and_set) && \
74 !defined(AO_HAVE_test_and_set_release) && \
75 !defined(AO_HAVE_test_and_set_acquire) && \
76 !defined(AO_HAVE_test_and_set_read) && \
77 !defined(AO_HAVE_test_and_set_full)
78 # if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_full) || \
79 AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_full)
80 AO_INLINE AO_TS_VAL_t
AO_test_and_set_full(volatile AO_TS_t * addr)81 AO_test_and_set_full(volatile AO_TS_t *addr)
82 {
83 if (AO_TS_COMPARE_AND_SWAP_FULL(addr, AO_TS_CLEAR, AO_TS_SET))
84 return AO_TS_CLEAR;
85 else
86 return AO_TS_SET;
87 }
88 # define AO_HAVE_test_and_set_full
89 # endif /* AO_HAVE_compare_and_swap_full */
90
91 # if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_acquire) || \
92 AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_acquire)
93 AO_INLINE AO_TS_VAL_t
AO_test_and_set_acquire(volatile AO_TS_t * addr)94 AO_test_and_set_acquire(volatile AO_TS_t *addr)
95 {
96 if (AO_TS_COMPARE_AND_SWAP_ACQUIRE(addr, AO_TS_CLEAR, AO_TS_SET))
97 return AO_TS_CLEAR;
98 else
99 return AO_TS_SET;
100 }
101 # define AO_HAVE_test_and_set_acquire
102 # endif /* AO_HAVE_compare_and_swap_acquire */
103
104 # if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_release) || \
105 AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_release)
106 AO_INLINE AO_TS_VAL_t
AO_test_and_set_release(volatile AO_TS_t * addr)107 AO_test_and_set_release(volatile AO_TS_t *addr)
108 {
109 if (AO_TS_COMPARE_AND_SWAP_RELEASE(addr, AO_TS_CLEAR, AO_TS_SET))
110 return AO_TS_CLEAR;
111 else
112 return AO_TS_SET;
113 }
114 # define AO_HAVE_test_and_set_release
115 # endif /* AO_HAVE_compare_and_swap_release */
116
117 # if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap) || \
118 AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap)
119 AO_INLINE AO_TS_VAL_t
AO_test_and_set(volatile AO_TS_t * addr)120 AO_test_and_set(volatile AO_TS_t *addr)
121 {
122 if (AO_TS_COMPARE_AND_SWAP(addr, AO_TS_CLEAR, AO_TS_SET))
123 return AO_TS_CLEAR;
124 else
125 return AO_TS_SET;
126 }
127 # define AO_HAVE_test_and_set
128 # endif /* AO_HAVE_compare_and_swap */
129
130 # if defined(AO_HAVE_test_and_set) && defined(AO_HAVE_nop_full) \
131 && !defined(AO_HAVE_test_and_set_acquire)
132 AO_INLINE AO_TS_VAL_t
AO_test_and_set_acquire(volatile AO_TS_t * addr)133 AO_test_and_set_acquire(volatile AO_TS_t *addr)
134 {
135 AO_TS_VAL_t result = AO_test_and_set(addr);
136 AO_nop_full();
137 return result;
138 }
139 # define AO_HAVE_test_and_set_acquire
140 # endif
141
142 #endif /* No prior test and set */
143
144 /* Nop */
145 #if !defined(AO_HAVE_nop)
AO_nop(void)146 AO_INLINE void AO_nop(void) {}
147 # define AO_HAVE_nop
148 #endif
149
150 #if defined(AO_HAVE_test_and_set_full) && !defined(AO_HAVE_nop_full)
151 AO_INLINE void
AO_nop_full(void)152 AO_nop_full(void)
153 {
154 AO_TS_t dummy = AO_TS_INITIALIZER;
155 AO_test_and_set_full(&dummy);
156 }
157 # define AO_HAVE_nop_full
158 #endif
159
160 #if defined(AO_HAVE_nop_acquire)
161 # error AO_nop_acquire is useless: dont define.
162 #endif
163 #if defined(AO_HAVE_nop_release)
164 # error AO_nop_release is useless: dont define.
165 #endif
166
167 #if defined(AO_HAVE_nop_full) && !defined(AO_HAVE_nop_read)
168 # define AO_nop_read() AO_nop_full()
169 # define AO_HAVE_nop_read
170 #endif
171
172 #if defined(AO_HAVE_nop_full) && !defined(AO_HAVE_nop_write)
173 # define AO_nop_write() AO_nop_full()
174 # define AO_HAVE_nop_write
175 #endif
176
177 /* Load */
178 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
179 # define AO_load_acquire(addr) AO_load_full(addr)
180 # define AO_HAVE_load_acquire
181 #endif
182
183 #if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
184 # define AO_load(addr) AO_load_acquire(addr)
185 # define AO_HAVE_load
186 #endif
187
188 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
189 # define AO_load_read(addr) AO_load_full(addr)
190 # define AO_HAVE_load_read
191 #endif
192
193 #if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_acquire)
194 # define AO_load_acquire_read(addr) AO_load_acquire(addr)
195 # define AO_HAVE_load_acquire_read
196 #endif
197
198 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) && \
199 !defined(AO_HAVE_load_acquire)
200 AO_INLINE AO_t
AO_load_acquire(const volatile AO_t * addr)201 AO_load_acquire(const volatile AO_t *addr)
202 {
203 AO_t result = AO_load(addr);
204 /* Acquire barrier would be useless, since the load could be delayed */
205 /* beyond it. */
206 AO_nop_full();
207 return result;
208 }
209 # define AO_HAVE_load_acquire
210 #endif
211
212 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) && \
213 !defined(AO_HAVE_load_read)
214 AO_INLINE AO_t
AO_load_read(const volatile AO_t * addr)215 AO_load_read(const volatile AO_t *addr)
216 {
217 AO_t result = AO_load(addr);
218 /* Acquire barrier would be useless, since the load could be delayed */
219 /* beyond it. */
220 AO_nop_read();
221 return result;
222 }
223 # define AO_HAVE_load_read
224 #endif
225
226 #if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) && \
227 !defined(AO_HAVE_load_full)
228 # define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
229 # define AO_HAVE_load_full
230 #endif
231
232 #if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_read)
233 # define AO_load_acquire_read(addr) AO_load_read(addr)
234 # define AO_HAVE_load_acquire_read
235 #endif
236
237 #if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load)
238 # define AO_load(addr) AO_load_acquire_read(addr)
239 # define AO_HAVE_load
240 #endif
241
242 #ifdef AO_NO_DD_ORDERING
243 # if defined(AO_HAVE_load_acquire_read)
244 # define AO_load_dd_acquire_read(addr) AO_load_acquire_read(addr)
245 # define AO_HAVE_load_dd_acquire_read
246 # endif
247 #else
248 # if defined(AO_HAVE_load)
249 # define AO_load_dd_acquire_read(addr) AO_load(addr)
250 # define AO_HAVE_load_dd_acquire_read
251 # endif
252 #endif
253
254
255 /* Store */
256
257 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
258 # define AO_store_release(addr,val) AO_store_full(addr,val)
259 # define AO_HAVE_store_release
260 #endif
261
262 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
263 # define AO_store(addr, val) AO_store_release(addr,val)
264 # define AO_HAVE_store
265 #endif
266
267 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
268 # define AO_store_write(addr,val) AO_store_full(addr,val)
269 # define AO_HAVE_store_write
270 #endif
271
272 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store_release_write)
273 # define AO_store_release_write(addr, val) AO_store_release(addr,val)
274 # define AO_HAVE_store_release_write
275 #endif
276
277 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
278 # define AO_store(addr, val) AO_store_write(addr,val)
279 # define AO_HAVE_store
280 #endif
281
282 #if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) && \
283 !defined(AO_HAVE_store_release)
284 # define AO_store_release(addr,val) (AO_nop_full(), AO_store(addr,val))
285 # define AO_HAVE_store_release
286 #endif
287
288 #if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) && \
289 !defined(AO_HAVE_store_write)
290 # define AO_store_write(addr, val) (AO_nop_write(), AO_store(addr,val))
291 # define AO_HAVE_store_write
292 #endif
293
294 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store_release_write)
295 # define AO_store_release_write(addr, val) AO_store_write(addr,val)
296 # define AO_HAVE_store_release_write
297 #endif
298
299 #if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) && \
300 !defined(AO_HAVE_store_full)
301 # define AO_store_full(addr, val) (AO_store_release(addr, val), AO_nop_full())
302 # define AO_HAVE_store_full
303 #endif
304
305 /* NEC LE-IT: Test and set */
306 #if defined(AO_HAVE_test_and_set) && \
307 defined(AO_HAVE_nop_full) && \
308 !defined(AO_HAVE_test_and_set_release)
309 # define AO_test_and_set_release(addr) \
310 (AO_nop_full(), AO_test_and_set(addr))
311 # define AO_HAVE_test_and_set_release
312 #endif
313
314 #if defined(AO_HAVE_test_and_set) && \
315 defined(AO_HAVE_nop_full) && \
316 !defined(AO_HAVE_test_and_set_acquire)
317 AO_INLINE AO_TS_t
AO_test_and_set_acquire(volatile AO_TS_t * addr)318 AO_test_and_set_acquire(volatile AO_TS_t *addr)
319 {
320 AO_TS_t res = AO_test_and_set(addr);
321 AO_nop_full();
322 return res;
323 }
324 # define AO_HAVE_test_and_set_acquire
325 #endif
326
327
328 /* Fetch_and_add */
329 /* We first try to implement fetch_and_add variants in terms */
330 /* of the corresponding compare_and_swap variants to minimize */
331 /* adding barriers. */
332 #if defined(AO_HAVE_compare_and_swap_full) && \
333 !defined(AO_HAVE_fetch_and_add_full)
334 AO_INLINE AO_t
AO_fetch_and_add_full(volatile AO_t * addr,AO_t incr)335 AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
336 {
337 AO_t old;
338 do
339 {
340 old = *addr;
341 }
342 while (!AO_compare_and_swap_full(addr, old, old+incr));
343 return old;
344 }
345 # define AO_HAVE_fetch_and_add_full
346 #endif
347
348 #if defined(AO_HAVE_compare_and_swap_acquire) && \
349 !defined(AO_HAVE_fetch_and_add_acquire)
350 AO_INLINE AO_t
AO_fetch_and_add_acquire(volatile AO_t * addr,AO_t incr)351 AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
352 {
353 AO_t old;
354 do
355 {
356 old = *addr;
357 }
358 while (!AO_compare_and_swap_acquire(addr, old, old+incr));
359 return old;
360 }
361 # define AO_HAVE_fetch_and_add_acquire
362 #endif
363
364 #if defined(AO_HAVE_compare_and_swap_release) && \
365 !defined(AO_HAVE_fetch_and_add_release)
366 AO_INLINE AO_t
AO_fetch_and_add_release(volatile AO_t * addr,AO_t incr)367 AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
368 {
369 AO_t old;
370 do
371 {
372 old = *addr;
373 }
374 while (!AO_compare_and_swap_release(addr, old, old+incr));
375 return old;
376 }
377 # define AO_HAVE_fetch_and_add_release
378 #endif
379
380 #if defined(AO_HAVE_compare_and_swap) && \
381 !defined(AO_HAVE_fetch_and_add)
382 AO_INLINE AO_t
AO_fetch_and_add(volatile AO_t * addr,AO_t incr)383 AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
384 {
385 AO_t old;
386 do
387 {
388 old = *addr;
389 }
390 while (!AO_compare_and_swap(addr, old, old+incr));
391 return old;
392 }
393 # define AO_HAVE_fetch_and_add
394 #endif
395
396 #if defined(AO_HAVE_fetch_and_add_full)
397 # if !defined(AO_HAVE_fetch_and_add_release)
398 # define AO_fetch_and_add_release(addr, val) \
399 AO_fetch_and_add_full(addr, val)
400 # define AO_HAVE_fetch_and_add_release
401 # endif
402 # if !defined(AO_HAVE_fetch_and_add_acquire)
403 # define AO_fetch_and_add_acquire(addr, val) \
404 AO_fetch_and_add_full(addr, val)
405 # define AO_HAVE_fetch_and_add_acquire
406 # endif
407 # if !defined(AO_HAVE_fetch_and_add_write)
408 # define AO_fetch_and_add_write(addr, val) \
409 AO_fetch_and_add_full(addr, val)
410 # define AO_HAVE_fetch_and_add_write
411 # endif
412 # if !defined(AO_HAVE_fetch_and_add_read)
413 # define AO_fetch_and_add_read(addr, val) \
414 AO_fetch_and_add_full(addr, val)
415 # define AO_HAVE_fetch_and_add_read
416 # endif
417 #endif /* AO_HAVE_fetch_and_add_full */
418
419 #if !defined(AO_HAVE_fetch_and_add) && \
420 defined(AO_HAVE_fetch_and_add_release)
421 # define AO_fetch_and_add(addr, val) \
422 AO_fetch_and_add_release(addr, val)
423 # define AO_HAVE_fetch_and_add
424 #endif
425 #if !defined(AO_HAVE_fetch_and_add) && \
426 defined(AO_HAVE_fetch_and_add_acquire)
427 # define AO_fetch_and_add(addr, val) \
428 AO_fetch_and_add_acquire(addr, val)
429 # define AO_HAVE_fetch_and_add
430 #endif
431 #if !defined(AO_HAVE_fetch_and_add) && \
432 defined(AO_HAVE_fetch_and_add_write)
433 # define AO_fetch_and_add(addr, val) \
434 AO_fetch_and_add_write(addr, val)
435 # define AO_HAVE_fetch_and_add
436 #endif
437 #if !defined(AO_HAVE_fetch_and_add) && \
438 defined(AO_HAVE_fetch_and_add_read)
439 # define AO_fetch_and_add(addr, val) \
440 AO_fetch_and_add_read(addr, val)
441 # define AO_HAVE_fetch_and_add
442 #endif
443
444 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
445 defined(AO_HAVE_nop_full) && \
446 !defined(AO_HAVE_fetch_and_add_full)
447 # define AO_fetch_and_add_full(addr, val) \
448 (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
449 # define AO_HAVE_fetch_and_add_full
450 #endif
451
452 #if !defined(AO_HAVE_fetch_and_add_release_write) && \
453 defined(AO_HAVE_fetch_and_add_write)
454 # define AO_fetch_and_add_release_write(addr, val) \
455 AO_fetch_and_add_write(addr, val)
456 # define AO_HAVE_fetch_and_add_release_write
457 #endif
458 #if !defined(AO_HAVE_fetch_and_add_release_write) && \
459 defined(AO_HAVE_fetch_and_add_release)
460 # define AO_fetch_and_add_release_write(addr, val) \
461 AO_fetch_and_add_release(addr, val)
462 # define AO_HAVE_fetch_and_add_release_write
463 #endif
464 #if !defined(AO_HAVE_fetch_and_add_acquire_read) && \
465 defined(AO_HAVE_fetch_and_add_read)
466 # define AO_fetch_and_add_acquire_read(addr, val) \
467 AO_fetch_and_add_read(addr, val)
468 # define AO_HAVE_fetch_and_add_acquire_read
469 #endif
470 #if !defined(AO_HAVE_fetch_and_add_acquire_read) && \
471 defined(AO_HAVE_fetch_and_add_acquire)
472 # define AO_fetch_and_add_acquire_read(addr, val) \
473 AO_fetch_and_add_acquire(addr, val)
474 # define AO_HAVE_fetch_and_add_acquire_read
475 #endif
476
477 #ifdef AO_NO_DD_ORDERING
478 # if defined(AO_HAVE_fetch_and_add_acquire_read)
479 # define AO_fetch_and_add_dd_acquire_read(addr, val) \
480 AO_fetch_and_add_acquire_read(addr, val)
481 # define AO_HAVE_fetch_and_add_dd_acquire_read
482 # endif
483 #else
484 # if defined(AO_HAVE_fetch_and_add)
485 # define AO_fetch_and_add_dd_acquire_read(addr, val) \
486 AO_fetch_and_add(addr, val)
487 # define AO_HAVE_fetch_and_add_dd_acquire_read
488 # endif
489 #endif
490
491 /* Fetch_and_add1 */
492
493 #if defined(AO_HAVE_fetch_and_add_full) &&\
494 !defined(AO_HAVE_fetch_and_add1_full)
495 # define AO_fetch_and_add1_full(addr) AO_fetch_and_add_full(addr,1)
496 # define AO_HAVE_fetch_and_add1_full
497 #endif
498 #if defined(AO_HAVE_fetch_and_add_release) &&\
499 !defined(AO_HAVE_fetch_and_add1_release)
500 # define AO_fetch_and_add1_release(addr) AO_fetch_and_add_release(addr,1)
501 # define AO_HAVE_fetch_and_add1_release
502 #endif
503 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
504 !defined(AO_HAVE_fetch_and_add1_acquire)
505 # define AO_fetch_and_add1_acquire(addr) AO_fetch_and_add_acquire(addr,1)
506 # define AO_HAVE_fetch_and_add1_acquire
507 #endif
508 #if defined(AO_HAVE_fetch_and_add_write) &&\
509 !defined(AO_HAVE_fetch_and_add1_write)
510 # define AO_fetch_and_add1_write(addr) AO_fetch_and_add_write(addr,1)
511 # define AO_HAVE_fetch_and_add1_write
512 #endif
513 #if defined(AO_HAVE_fetch_and_add_read) &&\
514 !defined(AO_HAVE_fetch_and_add1_read)
515 # define AO_fetch_and_add1_read(addr) AO_fetch_and_add_read(addr,1)
516 # define AO_HAVE_fetch_and_add1_read
517 #endif
518 #if defined(AO_HAVE_fetch_and_add_release_write) &&\
519 !defined(AO_HAVE_fetch_and_add1_release_write)
520 # define AO_fetch_and_add1_release_write(addr) \
521 AO_fetch_and_add_release_write(addr,1)
522 # define AO_HAVE_fetch_and_add1_release_write
523 #endif
524 #if defined(AO_HAVE_fetch_and_add_acquire_read) &&\
525 !defined(AO_HAVE_fetch_and_add1_acquire_read)
526 # define AO_fetch_and_add1_acquire_read(addr) \
527 AO_fetch_and_add_acquire_read(addr,1)
528 # define AO_HAVE_fetch_and_add1_acquire_read
529 #endif
530 #if defined(AO_HAVE_fetch_and_add) &&\
531 !defined(AO_HAVE_fetch_and_add1)
532 # define AO_fetch_and_add1(addr) \
533 AO_fetch_and_add(addr,1)
534 # define AO_HAVE_fetch_and_add1
535 #endif
536
537 #if defined(AO_HAVE_fetch_and_add1_full)
538 # if !defined(AO_HAVE_fetch_and_add1_release)
539 # define AO_fetch_and_add1_release(addr) \
540 AO_fetch_and_add1_full(addr)
541 # define AO_HAVE_fetch_and_add1_release
542 # endif
543 # if !defined(AO_HAVE_fetch_and_add1_acquire)
544 # define AO_fetch_and_add1_acquire(addr) \
545 AO_fetch_and_add1_full(addr)
546 # define AO_HAVE_fetch_and_add1_acquire
547 # endif
548 # if !defined(AO_HAVE_fetch_and_add1_write)
549 # define AO_fetch_and_add1_write(addr) \
550 AO_fetch_and_add1_full(addr)
551 # define AO_HAVE_fetch_and_add1_write
552 # endif
553 # if !defined(AO_HAVE_fetch_and_add1_read)
554 # define AO_fetch_and_add1_read(addr) \
555 AO_fetch_and_add1_full(addr)
556 # define AO_HAVE_fetch_and_add1_read
557 # endif
558 #endif /* AO_HAVE_fetch_and_add1_full */
559
560 #if !defined(AO_HAVE_fetch_and_add1) && \
561 defined(AO_HAVE_fetch_and_add1_release)
562 # define AO_fetch_and_add1(addr) \
563 AO_fetch_and_add1_release(addr)
564 # define AO_HAVE_fetch_and_add1
565 #endif
566 #if !defined(AO_HAVE_fetch_and_add1) && \
567 defined(AO_HAVE_fetch_and_add1_acquire)
568 # define AO_fetch_and_add1(addr) \
569 AO_fetch_and_add1_acquire(addr)
570 # define AO_HAVE_fetch_and_add1
571 #endif
572 #if !defined(AO_HAVE_fetch_and_add1) && \
573 defined(AO_HAVE_fetch_and_add1_write)
574 # define AO_fetch_and_add1(addr) \
575 AO_fetch_and_add1_write(addr)
576 # define AO_HAVE_fetch_and_add1
577 #endif
578 #if !defined(AO_HAVE_fetch_and_add1) && \
579 defined(AO_HAVE_fetch_and_add1_read)
580 # define AO_fetch_and_add1(addr) \
581 AO_fetch_and_add1_read(addr)
582 # define AO_HAVE_fetch_and_add1
583 #endif
584
585 #if defined(AO_HAVE_fetch_and_add1_acquire) &&\
586 defined(AO_HAVE_nop_full) && \
587 !defined(AO_HAVE_fetch_and_add1_full)
588 # define AO_fetch_and_add1_full(addr) \
589 (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
590 # define AO_HAVE_fetch_and_add1_full
591 #endif
592
593 #if !defined(AO_HAVE_fetch_and_add1_release_write) && \
594 defined(AO_HAVE_fetch_and_add1_write)
595 # define AO_fetch_and_add1_release_write(addr) \
596 AO_fetch_and_add1_write(addr)
597 # define AO_HAVE_fetch_and_add1_release_write
598 #endif
599 #if !defined(AO_HAVE_fetch_and_add1_release_write) && \
600 defined(AO_HAVE_fetch_and_add1_release)
601 # define AO_fetch_and_add1_release_write(addr) \
602 AO_fetch_and_add1_release(addr)
603 # define AO_HAVE_fetch_and_add1_release_write
604 #endif
605 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) && \
606 defined(AO_HAVE_fetch_and_add1_read)
607 # define AO_fetch_and_add1_acquire_read(addr) \
608 AO_fetch_and_add1_read(addr)
609 # define AO_HAVE_fetch_and_add1_acquire_read
610 #endif
611 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) && \
612 defined(AO_HAVE_fetch_and_add1_acquire)
613 # define AO_fetch_and_add1_acquire_read(addr) \
614 AO_fetch_and_add1_acquire(addr)
615 # define AO_HAVE_fetch_and_add1_acquire_read
616 #endif
617
618 #ifdef AO_NO_DD_ORDERING
619 # if defined(AO_HAVE_fetch_and_add1_acquire_read)
620 # define AO_fetch_and_add1_dd_acquire_read(addr) \
621 AO_fetch_and_add1_acquire_read(addr)
622 # define AO_HAVE_fetch_and_add1_dd_acquire_read
623 # endif
624 #else
625 # if defined(AO_HAVE_fetch_and_add1)
626 # define AO_fetch_and_add1_dd_acquire_read(addr) AO_fetch_and_add1(addr)
627 # define AO_HAVE_fetch_and_add1_dd_acquire_read
628 # endif
629 #endif
630
631 /* Fetch_and_sub1 */
632
633 #if defined(AO_HAVE_fetch_and_add_full) &&\
634 !defined(AO_HAVE_fetch_and_sub1_full)
635 # define AO_fetch_and_sub1_full(addr) AO_fetch_and_add_full(addr,(AO_t)(-1))
636 # define AO_HAVE_fetch_and_sub1_full
637 #endif
638 #if defined(AO_HAVE_fetch_and_add_release) &&\
639 !defined(AO_HAVE_fetch_and_sub1_release)
640 # define AO_fetch_and_sub1_release(addr) \
641 AO_fetch_and_add_release(addr,(AO_t)(-1))
642 # define AO_HAVE_fetch_and_sub1_release
643 #endif
644 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
645 !defined(AO_HAVE_fetch_and_sub1_acquire)
646 # define AO_fetch_and_sub1_acquire(addr) \
647 AO_fetch_and_add_acquire(addr,(AO_t)(-1))
648 # define AO_HAVE_fetch_and_sub1_acquire
649 #endif
650 #if defined(AO_HAVE_fetch_and_add_write) &&\
651 !defined(AO_HAVE_fetch_and_sub1_write)
652 # define AO_fetch_and_sub1_write(addr) \
653 AO_fetch_and_add_write(addr,(AO_t)(-1))
654 # define AO_HAVE_fetch_and_sub1_write
655 #endif
656 #if defined(AO_HAVE_fetch_and_add_read) &&\
657 !defined(AO_HAVE_fetch_and_sub1_read)
658 # define AO_fetch_and_sub1_read(addr) \
659 AO_fetch_and_add_read(addr,(AO_t)(-1))
660 # define AO_HAVE_fetch_and_sub1_read
661 #endif
662 #if defined(AO_HAVE_fetch_and_add_release_write) &&\
663 !defined(AO_HAVE_fetch_and_sub1_release_write)
664 # define AO_fetch_and_sub1_release_write(addr) \
665 AO_fetch_and_add_release_write(addr,(AO_t)(-1))
666 # define AO_HAVE_fetch_and_sub1_release_write
667 #endif
668 #if defined(AO_HAVE_fetch_and_add_acquire_read) &&\
669 !defined(AO_HAVE_fetch_and_sub1_acquire_read)
670 # define AO_fetch_and_sub1_acquire_read(addr) \
671 AO_fetch_and_add_acquire_read(addr,(AO_t)(-1))
672 # define AO_HAVE_fetch_and_sub1_acquire_read
673 #endif
674 #if defined(AO_HAVE_fetch_and_add) &&\
675 !defined(AO_HAVE_fetch_and_sub1)
676 # define AO_fetch_and_sub1(addr) \
677 AO_fetch_and_add(addr,(AO_t)(-1))
678 # define AO_HAVE_fetch_and_sub1
679 #endif
680
681 #if defined(AO_HAVE_fetch_and_sub1_full)
682 # if !defined(AO_HAVE_fetch_and_sub1_release)
683 # define AO_fetch_and_sub1_release(addr) \
684 AO_fetch_and_sub1_full(addr)
685 # define AO_HAVE_fetch_and_sub1_release
686 # endif
687 # if !defined(AO_HAVE_fetch_and_sub1_acquire)
688 # define AO_fetch_and_sub1_acquire(addr) \
689 AO_fetch_and_sub1_full(addr)
690 # define AO_HAVE_fetch_and_sub1_acquire
691 # endif
692 # if !defined(AO_HAVE_fetch_and_sub1_write)
693 # define AO_fetch_and_sub1_write(addr) \
694 AO_fetch_and_sub1_full(addr)
695 # define AO_HAVE_fetch_and_sub1_write
696 # endif
697 # if !defined(AO_HAVE_fetch_and_sub1_read)
698 # define AO_fetch_and_sub1_read(addr) \
699 AO_fetch_and_sub1_full(addr)
700 # define AO_HAVE_fetch_and_sub1_read
701 # endif
702 #endif /* AO_HAVE_fetch_and_sub1_full */
703
704 #if !defined(AO_HAVE_fetch_and_sub1) && \
705 defined(AO_HAVE_fetch_and_sub1_release)
706 # define AO_fetch_and_sub1(addr) \
707 AO_fetch_and_sub1_release(addr)
708 # define AO_HAVE_fetch_and_sub1
709 #endif
710 #if !defined(AO_HAVE_fetch_and_sub1) && \
711 defined(AO_HAVE_fetch_and_sub1_acquire)
712 # define AO_fetch_and_sub1(addr) \
713 AO_fetch_and_sub1_acquire(addr)
714 # define AO_HAVE_fetch_and_sub1
715 #endif
716 #if !defined(AO_HAVE_fetch_and_sub1) && \
717 defined(AO_HAVE_fetch_and_sub1_write)
718 # define AO_fetch_and_sub1(addr) \
719 AO_fetch_and_sub1_write(addr)
720 # define AO_HAVE_fetch_and_sub1
721 #endif
722 #if !defined(AO_HAVE_fetch_and_sub1) && \
723 defined(AO_HAVE_fetch_and_sub1_read)
724 # define AO_fetch_and_sub1(addr) \
725 AO_fetch_and_sub1_read(addr)
726 # define AO_HAVE_fetch_and_sub1
727 #endif
728
729 #if defined(AO_HAVE_fetch_and_sub1_acquire) &&\
730 defined(AO_HAVE_nop_full) && \
731 !defined(AO_HAVE_fetch_and_sub1_full)
732 # define AO_fetch_and_sub1_full(addr) \
733 (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
734 # define AO_HAVE_fetch_and_sub1_full
735 #endif
736
737 #if !defined(AO_HAVE_fetch_and_sub1_release_write) && \
738 defined(AO_HAVE_fetch_and_sub1_write)
739 # define AO_fetch_and_sub1_release_write(addr) \
740 AO_fetch_and_sub1_write(addr)
741 # define AO_HAVE_fetch_and_sub1_release_write
742 #endif
743 #if !defined(AO_HAVE_fetch_and_sub1_release_write) && \
744 defined(AO_HAVE_fetch_and_sub1_release)
745 # define AO_fetch_and_sub1_release_write(addr) \
746 AO_fetch_and_sub1_release(addr)
747 # define AO_HAVE_fetch_and_sub1_release_write
748 #endif
749 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) && \
750 defined(AO_HAVE_fetch_and_sub1_read)
751 # define AO_fetch_and_sub1_acquire_read(addr) \
752 AO_fetch_and_sub1_read(addr)
753 # define AO_HAVE_fetch_and_sub1_acquire_read
754 #endif
755 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) && \
756 defined(AO_HAVE_fetch_and_sub1_acquire)
757 # define AO_fetch_and_sub1_acquire_read(addr) \
758 AO_fetch_and_sub1_acquire(addr)
759 # define AO_HAVE_fetch_and_sub1_acquire_read
760 #endif
761
762 #ifdef AO_NO_DD_ORDERING
763 # if defined(AO_HAVE_fetch_and_sub1_acquire_read)
764 # define AO_fetch_and_sub1_dd_acquire_read(addr) \
765 AO_fetch_and_sub1_acquire_read(addr)
766 # define AO_HAVE_fetch_and_sub1_dd_acquire_read
767 # endif
768 #else
769 # if defined(AO_HAVE_fetch_and_sub1)
770 # define AO_fetch_and_sub1_dd_acquire_read(addr) AO_fetch_and_sub1(addr)
771 # define AO_HAVE_fetch_and_sub1_dd_acquire_read
772 # endif
773 #endif
774
775 /* Atomic or */
776 #if defined(AO_HAVE_compare_and_swap_full) && \
777 !defined(AO_HAVE_or_full)
778 AO_INLINE void
AO_or_full(volatile AO_t * addr,AO_t incr)779 AO_or_full(volatile AO_t *addr, AO_t incr)
780 {
781 AO_t old;
782 do
783 {
784 old = *addr;
785 }
786 while (!AO_compare_and_swap_full(addr, old, (old | incr)));
787 }
788 # define AO_HAVE_or_full
789 #endif
790
791 #if defined(AO_HAVE_or_full)
792 # if !defined(AO_HAVE_or_release)
793 # define AO_or_release(addr, val) \
794 AO_or_full(addr, val)
795 # define AO_HAVE_or_release
796 # endif
797 # if !defined(AO_HAVE_or_acquire)
798 # define AO_or_acquire(addr, val) \
799 AO_or_full(addr, val)
800 # define AO_HAVE_or_acquire
801 # endif
802 # if !defined(AO_HAVE_or_write)
803 # define AO_or_write(addr, val) \
804 AO_or_full(addr, val)
805 # define AO_HAVE_or_write
806 # endif
807 # if !defined(AO_HAVE_or_read)
808 # define AO_or_read(addr, val) \
809 AO_or_full(addr, val)
810 # define AO_HAVE_or_read
811 # endif
812 #endif /* AO_HAVE_or_full */
813
814 #if !defined(AO_HAVE_or) && \
815 defined(AO_HAVE_or_release)
816 # define AO_or(addr, val) \
817 AO_or_release(addr, val)
818 # define AO_HAVE_or
819 #endif
820 #if !defined(AO_HAVE_or) && \
821 defined(AO_HAVE_or_acquire)
822 # define AO_or(addr, val) \
823 AO_or_acquire(addr, val)
824 # define AO_HAVE_or
825 #endif
826 #if !defined(AO_HAVE_or) && \
827 defined(AO_HAVE_or_write)
828 # define AO_or(addr, val) \
829 AO_or_write(addr, val)
830 # define AO_HAVE_or
831 #endif
832 #if !defined(AO_HAVE_or) && \
833 defined(AO_HAVE_or_read)
834 # define AO_or(addr, val) \
835 AO_or_read(addr, val)
836 # define AO_HAVE_or
837 #endif
838
839 #if defined(AO_HAVE_or_acquire) &&\
840 defined(AO_HAVE_nop_full) && \
841 !defined(AO_HAVE_or_full)
842 # define AO_or_full(addr, val) \
843 (AO_nop_full(), AO_or_acquire(addr, val))
844 #endif
845
846 #if !defined(AO_HAVE_or_release_write) && \
847 defined(AO_HAVE_or_write)
848 # define AO_or_release_write(addr, val) \
849 AO_or_write(addr, val)
850 # define AO_HAVE_or_release_write
851 #endif
852 #if !defined(AO_HAVE_or_release_write) && \
853 defined(AO_HAVE_or_release)
854 # define AO_or_release_write(addr, val) \
855 AO_or_release(addr, val)
856 # define AO_HAVE_or_release_write
857 #endif
858 #if !defined(AO_HAVE_or_acquire_read) && \
859 defined(AO_HAVE_or_read)
860 # define AO_or_acquire_read(addr, val) \
861 AO_or_read(addr, val)
862 # define AO_HAVE_or_acquire_read
863 #endif
864 #if !defined(AO_HAVE_or_acquire_read) && \
865 defined(AO_HAVE_or_acquire)
866 # define AO_or_acquire_read(addr, val) \
867 AO_or_acquire(addr, val)
868 # define AO_HAVE_or_acquire_read
869 #endif
870
871 /* dd_aquire_read is meaningless. */
872
873 /* Test_and_set */
874
875 #if defined(AO_HAVE_test_and_set_full)
876 # if !defined(AO_HAVE_test_and_set_release)
877 # define AO_test_and_set_release(addr) \
878 AO_test_and_set_full(addr)
879 # define AO_HAVE_test_and_set_release
880 # endif
881 # if !defined(AO_HAVE_test_and_set_acquire)
882 # define AO_test_and_set_acquire(addr) \
883 AO_test_and_set_full(addr)
884 # define AO_HAVE_test_and_set_acquire
885 # endif
886 # if !defined(AO_HAVE_test_and_set_write)
887 # define AO_test_and_set_write(addr) \
888 AO_test_and_set_full(addr)
889 # define AO_HAVE_test_and_set_write
890 # endif
891 # if !defined(AO_HAVE_test_and_set_read)
892 # define AO_test_and_set_read(addr) \
893 AO_test_and_set_full(addr)
894 # define AO_HAVE_test_and_set_read
895 # endif
896 #endif /* AO_HAVE_test_and_set_full */
897
898 #if !defined(AO_HAVE_test_and_set) && \
899 defined(AO_HAVE_test_and_set_release)
900 # define AO_test_and_set(addr) \
901 AO_test_and_set_release(addr)
902 # define AO_HAVE_test_and_set
903 #endif
904 #if !defined(AO_HAVE_test_and_set) && \
905 defined(AO_HAVE_test_and_set_acquire)
906 # define AO_test_and_set(addr) \
907 AO_test_and_set_acquire(addr)
908 # define AO_HAVE_test_and_set
909 #endif
910 #if !defined(AO_HAVE_test_and_set) && \
911 defined(AO_HAVE_test_and_set_write)
912 # define AO_test_and_set(addr) \
913 AO_test_and_set_write(addr)
914 # define AO_HAVE_test_and_set
915 #endif
916 #if !defined(AO_HAVE_test_and_set) && \
917 defined(AO_HAVE_test_and_set_read)
918 # define AO_test_and_set(addr) \
919 AO_test_and_set_read(addr)
920 # define AO_HAVE_test_and_set
921 #endif
922
923 #if defined(AO_HAVE_test_and_set_acquire) &&\
924 defined(AO_HAVE_nop_full) && \
925 !defined(AO_HAVE_test_and_set_full)
926 # define AO_test_and_set_full(addr) \
927 (AO_nop_full(), AO_test_and_set_acquire(addr))
928 # define AO_HAVE_test_and_set_full
929 #endif
930
931 #if !defined(AO_HAVE_test_and_set_release_write) && \
932 defined(AO_HAVE_test_and_set_write)
933 # define AO_test_and_set_release_write(addr) \
934 AO_test_and_set_write(addr)
935 # define AO_HAVE_test_and_set_release_write
936 #endif
937 #if !defined(AO_HAVE_test_and_set_release_write) && \
938 defined(AO_HAVE_test_and_set_release)
939 # define AO_test_and_set_release_write(addr) \
940 AO_test_and_set_release(addr)
941 # define AO_HAVE_test_and_set_release_write
942 #endif
943 #if !defined(AO_HAVE_test_and_set_acquire_read) && \
944 defined(AO_HAVE_test_and_set_read)
945 # define AO_test_and_set_acquire_read(addr) \
946 AO_test_and_set_read(addr)
947 # define AO_HAVE_test_and_set_acquire_read
948 #endif
949 #if !defined(AO_HAVE_test_and_set_acquire_read) && \
950 defined(AO_HAVE_test_and_set_acquire)
951 # define AO_test_and_set_acquire_read(addr) \
952 AO_test_and_set_acquire(addr)
953 # define AO_HAVE_test_and_set_acquire_read
954 #endif
955
956 #ifdef AO_NO_DD_ORDERING
957 # if defined(AO_HAVE_test_and_set_acquire_read)
958 # define AO_test_and_set_dd_acquire_read(addr) \
959 AO_test_and_set_acquire_read(addr)
960 # define AO_HAVE_test_and_set_dd_acquire_read
961 # endif
962 #else
963 # if defined(AO_HAVE_test_and_set)
964 # define AO_test_and_set_dd_acquire_read(addr) AO_test_and_set(addr)
965 # define AO_HAVE_test_and_set_dd_acquire_read
966 # endif
967 #endif
968
969 /* Compare_and_swap */
970 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full)\
971 && !defined(AO_HAVE_compare_and_swap_acquire)
972 AO_INLINE int
AO_compare_and_swap_acquire(volatile AO_t * addr,AO_t old,AO_t new_val)973 AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val)
974 {
975 int result = AO_compare_and_swap(addr, old, new_val);
976 AO_nop_full();
977 return result;
978 }
979 # define AO_HAVE_compare_and_swap_acquire
980 #endif
981 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full)\
982 && !defined(AO_HAVE_compare_and_swap_release)
983 # define AO_compare_and_swap_release(addr, old, new_val) \
984 (AO_nop_full(), AO_compare_and_swap(addr, old, new_val))
985 # define AO_HAVE_compare_and_swap_release
986 #endif
987 #if defined(AO_HAVE_compare_and_swap_full)
988 # if !defined(AO_HAVE_compare_and_swap_release)
989 # define AO_compare_and_swap_release(addr, old, new_val) \
990 AO_compare_and_swap_full(addr, old, new_val)
991 # define AO_HAVE_compare_and_swap_release
992 # endif
993 # if !defined(AO_HAVE_compare_and_swap_acquire)
994 # define AO_compare_and_swap_acquire(addr, old, new_val) \
995 AO_compare_and_swap_full(addr, old, new_val)
996 # define AO_HAVE_compare_and_swap_acquire
997 # endif
998 # if !defined(AO_HAVE_compare_and_swap_write)
999 # define AO_compare_and_swap_write(addr, old, new_val) \
1000 AO_compare_and_swap_full(addr, old, new_val)
1001 # define AO_HAVE_compare_and_swap_write
1002 # endif
1003 # if !defined(AO_HAVE_compare_and_swap_read)
1004 # define AO_compare_and_swap_read(addr, old, new_val) \
1005 AO_compare_and_swap_full(addr, old, new_val)
1006 # define AO_HAVE_compare_and_swap_read
1007 # endif
1008 #endif /* AO_HAVE_compare_and_swap_full */
1009
1010 #if !defined(AO_HAVE_compare_and_swap) && \
1011 defined(AO_HAVE_compare_and_swap_release)
1012 # define AO_compare_and_swap(addr, old, new_val) \
1013 AO_compare_and_swap_release(addr, old, new_val)
1014 # define AO_HAVE_compare_and_swap
1015 #endif
1016 #if !defined(AO_HAVE_compare_and_swap) && \
1017 defined(AO_HAVE_compare_and_swap_acquire)
1018 # define AO_compare_and_swap(addr, old, new_val) \
1019 AO_compare_and_swap_acquire(addr, old, new_val)
1020 # define AO_HAVE_compare_and_swap
1021 #endif
1022 #if !defined(AO_HAVE_compare_and_swap) && \
1023 defined(AO_HAVE_compare_and_swap_write)
1024 # define AO_compare_and_swap(addr, old, new_val) \
1025 AO_compare_and_swap_write(addr, old, new_val)
1026 # define AO_HAVE_compare_and_swap
1027 #endif
1028 #if !defined(AO_HAVE_compare_and_swap) && \
1029 defined(AO_HAVE_compare_and_swap_read)
1030 # define AO_compare_and_swap(addr, old, new_val) \
1031 AO_compare_and_swap_read(addr, old, new_val)
1032 # define AO_HAVE_compare_and_swap
1033 #endif
1034
1035 #if defined(AO_HAVE_compare_and_swap_acquire) &&\
1036 defined(AO_HAVE_nop_full) && \
1037 !defined(AO_HAVE_compare_and_swap_full)
1038 # define AO_compare_and_swap_full(addr, old, new_val) \
1039 (AO_nop_full(), AO_compare_and_swap_acquire(addr, old, new_val))
1040 # define AO_HAVE_compare_and_swap_full
1041 #endif
1042
1043 #if !defined(AO_HAVE_compare_and_swap_release_write) && \
1044 defined(AO_HAVE_compare_and_swap_write)
1045 # define AO_compare_and_swap_release_write(addr, old, new_val) \
1046 AO_compare_and_swap_write(addr, old, new_val)
1047 # define AO_HAVE_compare_and_swap_release_write
1048 #endif
1049 #if !defined(AO_HAVE_compare_and_swap_release_write) && \
1050 defined(AO_HAVE_compare_and_swap_release)
1051 # define AO_compare_and_swap_release_write(addr, old, new_val) \
1052 AO_compare_and_swap_release(addr, old, new_val)
1053 # define AO_HAVE_compare_and_swap_release_write
1054 #endif
1055 #if !defined(AO_HAVE_compare_and_swap_acquire_read) && \
1056 defined(AO_HAVE_compare_and_swap_read)
1057 # define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1058 AO_compare_and_swap_read(addr, old, new_val)
1059 # define AO_HAVE_compare_and_swap_acquire_read
1060 #endif
1061 #if !defined(AO_HAVE_compare_and_swap_acquire_read) && \
1062 defined(AO_HAVE_compare_and_swap_acquire)
1063 # define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1064 AO_compare_and_swap_acquire(addr, old, new_val)
1065 # define AO_HAVE_compare_and_swap_acquire_read
1066 #endif
1067
1068 #ifdef AO_NO_DD_ORDERING
1069 # if defined(AO_HAVE_compare_and_swap_acquire_read)
1070 # define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1071 AO_compare_and_swap_acquire_read(addr, old, new_val)
1072 # define AO_HAVE_compare_and_swap_dd_acquire_read
1073 # endif
1074 #else
1075 # if defined(AO_HAVE_compare_and_swap)
1076 # define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1077 AO_compare_and_swap(addr, old, new_val)
1078 # define AO_HAVE_compare_and_swap_dd_acquire_read
1079 # endif
1080 #endif
1081
1082 #include "generalize-small.h"
1083
1084 /* Compare_double_and_swap_double */
1085 #if defined(AO_HAVE_compare_double_and_swap_double) && defined(AO_HAVE_nop_full)\
1086 && !defined(AO_HAVE_compare_double_and_swap_double_acquire)
1087 AO_INLINE int
AO_compare_double_and_swap_double_acquire(volatile AO_double_t * addr,AO_t o1,AO_t o2,AO_t n1,AO_t n2)1088 AO_compare_double_and_swap_double_acquire(volatile AO_double_t *addr,
1089 AO_t o1, AO_t o2,
1090 AO_t n1, AO_t n2)
1091 {
1092 int result = AO_compare_double_and_swap_double(addr, o1, o2, n1, n2);
1093 AO_nop_full();
1094 return result;
1095 }
1096 # define AO_HAVE_compare_double_and_swap_double_acquire
1097 #endif
1098 #if defined(AO_HAVE_compare_double_and_swap_double) \
1099 && defined(AO_HAVE_nop_full)\
1100 && !defined(AO_HAVE_compare_double_and_swap_double_release)
1101 # define AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2) \
1102 (AO_nop_full(), AO_compare_double_and_swap_double(addr, o1, o2, n1, n2))
1103 # define AO_HAVE_compare_double_and_swap_double_release
1104 #endif
1105 #if defined(AO_HAVE_compare_double_and_swap_double_full)
1106 # if !defined(AO_HAVE_compare_double_and_swap_double_release)
1107 # define AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2) \
1108 AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1109 # define AO_HAVE_compare_double_and_swap_double_release
1110 # endif
1111 # if !defined(AO_HAVE_compare_double_and_swap_double_acquire)
1112 # define AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2) \
1113 AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1114 # define AO_HAVE_compare_double_and_swap_double_acquire
1115 # endif
1116 # if !defined(AO_HAVE_compare_double_and_swap_double_write)
1117 # define AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2) \
1118 AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1119 # define AO_HAVE_compare_double_and_swap_double_write
1120 # endif
1121 # if !defined(AO_HAVE_compare_double_and_swap_double_read)
1122 # define AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2) \
1123 AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1124 # define AO_HAVE_compare_double_and_swap_double_read
1125 # endif
1126 #endif /* AO_HAVE_compare_double_and_swap_double_full */
1127
1128 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1129 defined(AO_HAVE_compare_double_and_swap_double_release)
1130 # define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1131 AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2)
1132 # define AO_HAVE_compare_double_and_swap_double
1133 #endif
1134 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1135 defined(AO_HAVE_compare_double_and_swap_double_acquire)
1136 # define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1137 AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2)
1138 # define AO_HAVE_compare_double_and_swap_double
1139 #endif
1140 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1141 defined(AO_HAVE_compare_double_and_swap_double_write)
1142 # define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1143 AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2)
1144 # define AO_HAVE_compare_double_and_swap_double
1145 #endif
1146 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1147 defined(AO_HAVE_compare_double_and_swap_double_read)
1148 # define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1149 AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2)
1150 # define AO_HAVE_compare_double_and_swap_double
1151 #endif
1152
1153 #if defined(AO_HAVE_compare_double_and_swap_double_acquire) &&\
1154 defined(AO_HAVE_nop_full) && \
1155 !defined(AO_HAVE_compare_double_and_swap_double_full)
1156 # define AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2) \
1157 (AO_nop_full(), AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2))
1158 # define AO_HAVE_compare_double_and_swap_double_full
1159 #endif
1160
1161 #if !defined(AO_HAVE_compare_double_and_swap_double_release_write) && \
1162 defined(AO_HAVE_compare_double_and_swap_double_write)
1163 # define AO_compare_double_and_swap_double_release_write(addr, o1, o2, n1, n2) \
1164 AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2)
1165 # define AO_HAVE_compare_double_and_swap_double_release_write
1166 #endif
1167 #if !defined(AO_HAVE_compare_double_and_swap_double_release_write) && \
1168 defined(AO_HAVE_compare_double_and_swap_double_release)
1169 # define AO_compare_double_and_swap_double_release_write(addr, o1, o2, n1, n2) \
1170 AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2)
1171 # define AO_HAVE_compare_double_and_swap_double_release_write
1172 #endif
1173 #if !defined(AO_HAVE_compare_double_and_swap_double_acquire_read) && \
1174 defined(AO_HAVE_compare_double_and_swap_double_read)
1175 # define AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2) \
1176 AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2)
1177 # define AO_HAVE_compare_double_and_swap_double_acquire_read
1178 #endif
1179 #if !defined(AO_HAVE_compare_double_and_swap_double_acquire_read) && \
1180 defined(AO_HAVE_compare_double_and_swap_double_acquire)
1181 # define AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2) \
1182 AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2)
1183 # define AO_HAVE_compare_double_and_swap_double_acquire_read
1184 #endif
1185
1186 #ifdef AO_NO_DD_ORDERING
1187 # if defined(AO_HAVE_compare_double_and_swap_double_acquire_read)
1188 # define AO_compare_double_and_swap_double_dd_acquire_read(addr, o1, o2, n1, n2) \
1189 AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2)
1190 # define AO_HAVE_compare_double_and_swap_double_dd_acquire_read
1191 # endif
1192 #else
1193 # if defined(AO_HAVE_compare_double_and_swap_double)
1194 # define AO_compare_double_and_swap_double_dd_acquire_read(addr, o1, o2, n1, n2) \
1195 AO_compare_double_and_swap_double(addr, o1, o2, n1, n2)
1196 # define AO_HAVE_compare_double_and_swap_double_dd_acquire_read
1197 # endif
1198 #endif
1199
1200 /* Compare_and_swap_double */
1201 #if defined(AO_HAVE_compare_and_swap_double) && defined(AO_HAVE_nop_full)\
1202 && !defined(AO_HAVE_compare_and_swap_double_acquire)
1203 AO_INLINE int
AO_compare_and_swap_double_acquire(volatile AO_double_t * addr,AO_t o1,AO_t n1,AO_t n2)1204 AO_compare_and_swap_double_acquire(volatile AO_double_t *addr,
1205 AO_t o1,
1206 AO_t n1, AO_t n2)
1207 {
1208 int result = AO_compare_and_swap_double(addr, o1, n1, n2);
1209 AO_nop_full();
1210 return result;
1211 }
1212 # define AO_HAVE_compare_and_swap_double_acquire
1213 #endif
1214 #if defined(AO_HAVE_compare_and_swap_double) \
1215 && defined(AO_HAVE_nop_full)\
1216 && !defined(AO_HAVE_compare_and_swap_double_release)
1217 # define AO_compare_and_swap_double_release(addr, o1, n1, n2) \
1218 (AO_nop_full(), AO_compare_and_swap_double(addr, o1, n1, n2))
1219 # define AO_HAVE_compare_and_swap_double_release
1220 #endif
1221 #if defined(AO_HAVE_compare_and_swap_double_full)
1222 # if !defined(AO_HAVE_compare_and_swap_double_release)
1223 # define AO_compare_and_swap_double_release(addr, o1, n1, n2) \
1224 AO_compare_and_swap_double_full(addr, o1, n1, n2)
1225 # define AO_HAVE_compare_and_swap_double_release
1226 # endif
1227 # if !defined(AO_HAVE_compare_and_swap_double_acquire)
1228 # define AO_compare_and_swap_double_acquire(addr, o1, n1, n2) \
1229 AO_compare_and_swap_double_full(addr, o1, n1, n2)
1230 # define AO_HAVE_compare_and_swap_double_acquire
1231 # endif
1232 # if !defined(AO_HAVE_compare_and_swap_double_write)
1233 # define AO_compare_and_swap_double_write(addr, o1, n1, n2) \
1234 AO_compare_and_swap_double_full(addr, o1, n1, n2)
1235 # define AO_HAVE_compare_and_swap_double_write
1236 # endif
1237 # if !defined(AO_HAVE_compare_and_swap_double_read)
1238 # define AO_compare_and_swap_double_read(addr, o1, n1, n2) \
1239 AO_compare_and_swap_double_full(addr, o1, n1, n2)
1240 # define AO_HAVE_compare_and_swap_double_read
1241 # endif
1242 #endif /* AO_HAVE_compare_and_swap_double_full */
1243
1244 #if !defined(AO_HAVE_compare_and_swap_double) && \
1245 defined(AO_HAVE_compare_and_swap_double_release)
1246 # define AO_compare_and_swap_double(addr, o1, n1, n2) \
1247 AO_compare_and_swap_double_release(addr, o1, n1, n2)
1248 # define AO_HAVE_compare_and_swap_double
1249 #endif
1250 #if !defined(AO_HAVE_compare_and_swap_double) && \
1251 defined(AO_HAVE_compare_and_swap_double_acquire)
1252 # define AO_compare_and_swap_double(addr, o1, n1, n2) \
1253 AO_compare_and_swap_double_acquire(addr, o1, n1, n2)
1254 # define AO_HAVE_compare_and_swap_double
1255 #endif
1256 #if !defined(AO_HAVE_compare_and_swap_double) && \
1257 defined(AO_HAVE_compare_and_swap_double_write)
1258 # define AO_compare_and_swap_double(addr, o1, n1, n2) \
1259 AO_compare_and_swap_double_write(addr, o1, n1, n2)
1260 # define AO_HAVE_compare_and_swap_double
1261 #endif
1262 #if !defined(AO_HAVE_compare_and_swap_double) && \
1263 defined(AO_HAVE_compare_and_swap_double_read)
1264 # define AO_compare_and_swap_double(addr, o1, n1, n2) \
1265 AO_compare_and_swap_double_read(addr, o1, n1, n2)
1266 # define AO_HAVE_compare_and_swap_double
1267 #endif
1268
1269 #if defined(AO_HAVE_compare_and_swap_double_acquire) &&\
1270 defined(AO_HAVE_nop_full) && \
1271 !defined(AO_HAVE_compare_and_swap_double_full)
1272 # define AO_compare_and_swap_double_full(addr, o1, n1, n2) \
1273 (AO_nop_full(), AO_compare_and_swap_double_acquire(addr, o1, n1, n2))
1274 # define AO_HAVE_compare_and_swap_double_full
1275 #endif
1276
1277 #if !defined(AO_HAVE_compare_and_swap_double_release_write) && \
1278 defined(AO_HAVE_compare_and_swap_double_write)
1279 # define AO_compare_and_swap_double_release_write(addr, o1, n1, n2) \
1280 AO_compare_and_swap_double_write(addr, o1, n1, n2)
1281 # define AO_HAVE_compare_and_swap_double_release_write
1282 #endif
1283 #if !defined(AO_HAVE_compare_and_swap_double_release_write) && \
1284 defined(AO_HAVE_compare_and_swap_double_release)
1285 # define AO_compare_and_swap_double_release_write(addr, o1, n1, n2) \
1286 AO_compare_and_swap_double_release(addr, o1, n1, n2)
1287 # define AO_HAVE_compare_and_swap_double_release_write
1288 #endif
1289 #if !defined(AO_HAVE_compare_and_swap_double_acquire_read) && \
1290 defined(AO_HAVE_compare_and_swap_double_read)
1291 # define AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2) \
1292 AO_compare_and_swap_double_read(addr, o1, n1, n2)
1293 # define AO_HAVE_compare_and_swap_double_acquire_read
1294 #endif
1295 #if !defined(AO_HAVE_compare_and_swap_double_acquire_read) && \
1296 defined(AO_HAVE_compare_and_swap_double_acquire)
1297 # define AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2) \
1298 AO_compare_and_swap_double_acquire(addr, o1, n1, n2)
1299 # define AO_HAVE_compare_and_swap_double_acquire_read
1300 #endif
1301
1302 #ifdef AO_NO_DD_ORDERING
1303 # if defined(AO_HAVE_compare_and_swap_double_acquire_read)
1304 # define AO_compare_and_swap_double_dd_acquire_read(addr, o1, n1, n2) \
1305 AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2)
1306 # define AO_HAVE_compare_and_swap_double_dd_acquire_read
1307 # endif
1308 #else
1309 # if defined(AO_HAVE_compare_and_swap_double)
1310 # define AO_compare_and_swap_double_dd_acquire_read(addr, o1, n1, n2) \
1311 AO_compare_and_swap_double(addr, o1, n1, n2)
1312 # define AO_HAVE_compare_and_swap_double_dd_acquire_read
1313 # endif
1314 #endif
1315
1316 /* NEC LE-IT: Convenience functions for AO_double compare and swap which */
1317 /* types and reads easier in code */
1318 #if defined(AO_HAVE_compare_double_and_swap_double_release) && \
1319 !defined(AO_HAVE_double_compare_and_swap_release)
1320 AO_INLINE int
AO_double_compare_and_swap_release(volatile AO_double_t * addr,AO_double_t old_val,AO_double_t new_val)1321 AO_double_compare_and_swap_release(volatile AO_double_t *addr,
1322 AO_double_t old_val, AO_double_t new_val)
1323 {
1324 return AO_compare_double_and_swap_double_release(addr,
1325 old_val.AO_val1, old_val.AO_val2,
1326 new_val.AO_val1, new_val.AO_val2);
1327 }
1328 #define AO_HAVE_double_compare_and_swap_release
1329 #endif
1330
1331 #if defined(AO_HAVE_compare_double_and_swap_double_acquire) && \
1332 !defined(AO_HAVE_double_compare_and_swap_acquire)
1333 AO_INLINE int
AO_double_compare_and_swap_acquire(volatile AO_double_t * addr,AO_double_t old_val,AO_double_t new_val)1334 AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
1335 AO_double_t old_val, AO_double_t new_val)
1336 {
1337 return AO_compare_double_and_swap_double_acquire(addr,
1338 old_val.AO_val1, old_val.AO_val2,
1339 new_val.AO_val1, new_val.AO_val2);
1340 }
1341 #define AO_HAVE_double_compare_and_swap_acquire
1342 #endif
1343
1344 #if defined(AO_HAVE_compare_double_and_swap_double_full) && \
1345 !defined(AO_HAVE_double_compare_and_swap_full)
1346 AO_INLINE int
AO_double_compare_and_swap_full(volatile AO_double_t * addr,AO_double_t old_val,AO_double_t new_val)1347 AO_double_compare_and_swap_full(volatile AO_double_t *addr,
1348 AO_double_t old_val, AO_double_t new_val)
1349 {
1350 return AO_compare_double_and_swap_double_full(addr,
1351 old_val.AO_val1, old_val.AO_val2,
1352 new_val.AO_val1, new_val.AO_val2);
1353 }
1354 #define AO_HAVE_double_compare_and_swap_full
1355 #endif
1356