1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2006 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
14 * Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
15 * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
16 * reserved.
17 * Copyright (c) 2017 Research Organization for Information Science
18 * and Technology (RIST). All rights reserved.
19 * Copyright (c) 2018-2020 Intel, Inc. All rights reserved.
20 * $COPYRIGHT$
21 *
22 * Additional copyrights may follow
23 *
24 * $HEADER$
25 */
26
27 /** @file
28 *
29 * Atomic operations.
30 *
31 * This API is patterned after the FreeBSD kernel atomic interface
32 * (which is influenced by Intel's ia64 architecture). The
33 * FreeBSD interface is documented at
34 *
35 * http://www.freebsd.org/cgi/man.cgi?query=atomic&sektion=9
36 *
37 * Only the necessary subset of functions are implemented here.
38 *
39 * The following #defines will be true / false based on
40 * assembly support:
41 *
42 * - \c PMIX_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
43 * - \c PMIX_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
44 * - \c PMIX_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/compare-exchange can be done "atomicly"
45 * - \c PMIX_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/compare-exchange can be done "atomicly"
46 *
47 * Note that for the Atomic math, atomic add/sub may be implemented as
48 * C code using pmix_atomic_compare_exchange. The appearance of atomic
49 * operation will be upheld in these cases.
50 */
51
52 #ifndef PMIX_SYS_ATOMIC_H
53 #define PMIX_SYS_ATOMIC_H 1
54
55 #include "src/include/pmix_config.h"
56
57 #include <stdbool.h>
58
59 #include "src/atomics/sys/architecture.h"
60 #include "src/include/pmix_stdatomic.h"
61
62 #if PMIX_ASSEMBLY_BUILTIN == PMIX_BUILTIN_C11
63
64 #include "atomic_stdc.h"
65
66 #else /* !PMIX_C_HAVE__ATOMIC */
67
68 /* do some quick #define cleanup in cases where we are doing
69 testing... */
70 #ifdef PMIX_DISABLE_INLINE_ASM
71 #undef PMIX_C_GCC_INLINE_ASSEMBLY
72 #define PMIX_C_GCC_INLINE_ASSEMBLY 0
73 #endif
74
75 /* define PMIX_{GCC,DEC,XLC}_INLINE_ASSEMBLY based on the
76 PMIX_C_{GCC,DEC,XLC}_INLINE_ASSEMBLY defines and whether we
77 are in C or C++ */
78 #if defined(c_plusplus) || defined(__cplusplus)
79 /* We no longer support inline assembly for C++ as PMIX is a C-only interface */
80 #define PMIX_GCC_INLINE_ASSEMBLY 0
81 #else
82 #define PMIX_GCC_INLINE_ASSEMBLY PMIX_C_GCC_INLINE_ASSEMBLY
83 #endif
84
85
86 BEGIN_C_DECLS
87 /**********************************************************************
88 *
89 * Data structures for atomic ops
90 *
91 *********************************************************************/
92 /**
93 * Volatile lock object (with optional padding).
94 *
95 * \note The internals of the lock are included here, but should be
96 * considered private. The implementation currently in use may choose
97 * to use an int or unsigned char as the lock value - the user is not
98 * informed either way.
99 */
100 struct pmix_atomic_lock_t {
101 union {
102 pmix_atomic_int32_t lock; /**< The lock address (an integer) */
103 volatile unsigned char sparc_lock; /**< The lock address on sparc */
104 char padding[sizeof(int)]; /**< Array for optional padding */
105 } u;
106 };
107 typedef struct pmix_atomic_lock_t pmix_atomic_lock_t;
108
109 /**********************************************************************
110 *
111 * Set or unset these macros in the architecture-specific atomic.h
112 * files if we need to specify them as inline or non-inline
113 *
114 *********************************************************************/
115 #if !PMIX_GCC_INLINE_ASSEMBLY
116 #define PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
117 #define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 0
118 #define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 0
119 #define PMIX_HAVE_INLINE_ATOMIC_ADD_32 0
120 #define PMIX_HAVE_INLINE_ATOMIC_AND_32 0
121 #define PMIX_HAVE_INLINE_ATOMIC_OR_32 0
122 #define PMIX_HAVE_INLINE_ATOMIC_XOR_32 0
123 #define PMIX_HAVE_INLINE_ATOMIC_SUB_32 0
124 #define PMIX_HAVE_INLINE_ATOMIC_ADD_64 0
125 #define PMIX_HAVE_INLINE_ATOMIC_AND_64 0
126 #define PMIX_HAVE_INLINE_ATOMIC_OR_64 0
127 #define PMIX_HAVE_INLINE_ATOMIC_XOR_64 0
128 #define PMIX_HAVE_INLINE_ATOMIC_SUB_64 0
129 #define PMIX_HAVE_INLINE_ATOMIC_SWAP_32 0
130 #define PMIX_HAVE_INLINE_ATOMIC_SWAP_64 0
131 #else
132 #define PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
133 #define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 1
134 #define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 1
135 #define PMIX_HAVE_INLINE_ATOMIC_ADD_32 1
136 #define PMIX_HAVE_INLINE_ATOMIC_AND_32 1
137 #define PMIX_HAVE_INLINE_ATOMIC_OR_32 1
138 #define PMIX_HAVE_INLINE_ATOMIC_XOR_32 1
139 #define PMIX_HAVE_INLINE_ATOMIC_SUB_32 1
140 #define PMIX_HAVE_INLINE_ATOMIC_ADD_64 1
141 #define PMIX_HAVE_INLINE_ATOMIC_AND_64 1
142 #define PMIX_HAVE_INLINE_ATOMIC_OR_64 1
143 #define PMIX_HAVE_INLINE_ATOMIC_XOR_64 1
144 #define PMIX_HAVE_INLINE_ATOMIC_SUB_64 1
145 #define PMIX_HAVE_INLINE_ATOMIC_SWAP_32 1
146 #define PMIX_HAVE_INLINE_ATOMIC_SWAP_64 1
147 #endif
148
149 /**
150 * Enumeration of lock states
151 */
152 enum {
153 PMIX_ATOMIC_LOCK_UNLOCKED = 0,
154 PMIX_ATOMIC_LOCK_LOCKED = 1
155 };
156
157 #define PMIX_ATOMIC_LOCK_INIT {.u = {.lock = PMIX_ATOMIC_LOCK_UNLOCKED}}
158
159 /**********************************************************************
160 *
161 * Load the appropriate architecture files and set some reasonable
162 * default values for our support
163 *
164 *********************************************************************/
165 #if defined(DOXYGEN)
166 /* don't include system-level gorp when generating doxygen files */
167 #elif PMIX_ASSEMBLY_BUILTIN == PMIX_BUILTIN_GCC
168 #include "src/atomics/sys/gcc_builtin/atomic.h"
169 #elif PMIX_ASSEMBLY_ARCH == PMIX_X86_64
170 #include "src/atomics/sys/x86_64/atomic.h"
171 #elif PMIX_ASSEMBLY_ARCH == PMIX_ARM
172 #include "src/atomics/sys/arm/atomic.h"
173 #elif PMIX_ASSEMBLY_ARCH == PMIX_ARM64
174 #include "src/atomics/sys/arm64/atomic.h"
175 #elif PMIX_ASSEMBLY_ARCH == PMIX_IA32
176 #include "src/atomics/sys/ia32/atomic.h"
177 #elif PMIX_ASSEMBLY_ARCH == PMIX_POWERPC32
178 #include "src/atomics/sys/powerpc/atomic.h"
179 #elif PMIX_ASSEMBLY_ARCH == PMIX_POWERPC64
180 #include "src/atomics/sys/powerpc/atomic.h"
181 #endif
182
183 #ifndef DOXYGEN
184 /* compare and set operations can't really be emulated from software,
185 so if these defines aren't already set, they should be set to 0
186 now */
187 #ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
188 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 0
189 #endif
190 #ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
191 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
192 #endif
193 #ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128
194 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
195 #endif
196 #ifndef PMIX_HAVE_ATOMIC_LLSC_32
197 #define PMIX_HAVE_ATOMIC_LLSC_32 0
198 #endif
199 #ifndef PMIX_HAVE_ATOMIC_LLSC_64
200 #define PMIX_HAVE_ATOMIC_LLSC_64 0
201 #endif
202 #endif /* DOXYGEN */
203
204 /**********************************************************************
205 *
206 * Memory Barriers - defined here if running doxygen or have barriers
207 * but can't inline
208 *
209 *********************************************************************/
210 #if !defined(PMIX_HAVE_ATOMIC_MEM_BARRIER) && !defined(DOXYGEN)
211 /* no way to emulate in C code */
212 #define PMIX_HAVE_ATOMIC_MEM_BARRIER 0
213 #endif
214
215 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MEM_BARRIER
216 /**
217 * Memory barrier
218 *
219 * Will use system-specific features to instruct the processor and
220 * memory controller that all writes and reads that have been posted
221 * before the call to \c pmix_atomic_mb() must appear to have
222 * completed before the next read or write.
223 *
224 * \note This can have some expensive side effects, including flushing
225 * the pipeline, preventing the cpu from reordering instructions, and
226 * generally grinding the memory controller's performance. Use only
227 * if you need *both* read and write barriers.
228 */
229
230 #if PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER
231 static inline
232 #endif
233 void pmix_atomic_mb(void);
234
235 /**
236 * Read memory barrier
237 *
238 * Use system-specific features to instruct the processor and memory
239 * conrtoller that all reads that have been posted before the call to
240 * \c pmix_atomic_rmb() must appear to have been completed before the
241 * next read. Nothing is said about the ordering of writes when using
242 * \c pmix_atomic_rmb().
243 */
244
245 #if PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER
246 static inline
247 #endif
248 void pmix_atomic_rmb(void);
249
250 /**
251 * Write memory barrier.
252 *
253 * Use system-specific features to instruct the processor and memory
254 * conrtoller that all writes that have been posted before the call to
255 * \c pmix_atomic_wmb() must appear to have been completed before the
256 * next write. Nothing is said about the ordering of reads when using
257 * \c pmix_atomic_wmb().
258 */
259
260 #if PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER
261 static inline
262 #endif
263 void pmix_atomic_wmb(void);
264
265 #endif /* defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MEM_BARRIER */
266
267
268 /**********************************************************************
269 *
270 * Atomic spinlocks - always inlined, if have atomic compare-and-swap
271 *
272 *********************************************************************/
273
274 #if !defined(PMIX_HAVE_ATOMIC_SPINLOCKS) && !defined(DOXYGEN)
275 /* 0 is more like "pending" - we'll fix up at the end after all
276 the static inline functions are declared */
277 #define PMIX_HAVE_ATOMIC_SPINLOCKS 0
278 #endif
279
280 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_SPINLOCKS || (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
281
282 /**
283 * Initialize a lock to value
284 *
285 * @param lock Address of the lock
286 * @param value Initial value to set lock to
287 */
288 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
289 static inline
290 #endif
291 void pmix_atomic_lock_init(pmix_atomic_lock_t* lock, int32_t value);
292
293
294 /**
295 * Try to acquire a lock.
296 *
297 * @param lock Address of the lock.
298 * @return 0 if the lock was acquired, 1 otherwise.
299 */
300 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
301 static inline
302 #endif
303 int pmix_atomic_trylock(pmix_atomic_lock_t *lock);
304
305
306 /**
307 * Acquire a lock by spinning.
308 *
309 * @param lock Address of the lock.
310 */
311 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
312 static inline
313 #endif
314 void pmix_atomic_lock(pmix_atomic_lock_t *lock);
315
316
317 /**
318 * Release a lock.
319 *
320 * @param lock Address of the lock.
321 */
322 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
323 static inline
324 #endif
325 void pmix_atomic_unlock(pmix_atomic_lock_t *lock);
326
327
328 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
329 #undef PMIX_HAVE_ATOMIC_SPINLOCKS
330 #define PMIX_HAVE_ATOMIC_SPINLOCKS (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
331 #define PMIX_NEED_INLINE_ATOMIC_SPINLOCKS 1
332 #endif
333
334 #endif /* PMIX_HAVE_ATOMIC_SPINLOCKS */
335
336
337 /**********************************************************************
338 *
339 * Atomic math operations
340 *
341 *********************************************************************/
342 #if !defined(PMIX_HAVE_ATOMIC_CMPSET_32) && !defined(DOXYGEN)
343 #define PMIX_HAVE_ATOMIC_CMPSET_32 0
344 #endif
345 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_CMPSET_32
346
347 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
348 static inline
349 #endif
350 bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval,
351 int32_t newval);
352
353 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
354 static inline
355 #endif
356 bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval,
357 int32_t newval);
358
359 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
360 static inline
361 #endif
362 bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval,
363 int32_t newval);
364 #endif
365
366
367 #if !defined(PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64) && !defined(DOXYGEN)
368 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
369 #endif
370 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
371
372 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
373 static inline
374 #endif
375 bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval,
376 int64_t newval);
377
378 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
379 static inline
380 #endif
381 bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval,
382 int64_t newval);
383
384 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
385 static inline
386 #endif
387 bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval,
388 int64_t newval);
389
390 #endif
391
392 #if !defined(PMIX_HAVE_ATOMIC_MATH_32) && !defined(DOXYGEN)
393 /* define to 0 for these tests. WIll fix up later. */
394 #define PMIX_HAVE_ATOMIC_MATH_32 0
395 #endif
396
397 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
398
399 static inline int32_t pmix_atomic_add_fetch_32(pmix_atomic_int32_t *addr, int delta);
400 static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int delta);
401 static inline int32_t pmix_atomic_and_fetch_32(pmix_atomic_int32_t *addr, int32_t value);
402 static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value);
403 static inline int32_t pmix_atomic_or_fetch_32(pmix_atomic_int32_t *addr, int32_t value);
404 static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value);
405 static inline int32_t pmix_atomic_xor_fetch_32(pmix_atomic_int32_t *addr, int32_t value);
406 static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value);
407 static inline int32_t pmix_atomic_sub_fetch_32(pmix_atomic_int32_t *addr, int delta);
408 static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int delta);
409 static inline int32_t pmix_atomic_min_fetch_32 (pmix_atomic_int32_t *addr, int32_t value);
410 static inline int32_t pmix_atomic_fetch_min_32 (pmix_atomic_int32_t *addr, int32_t value);
411 static inline int32_t pmix_atomic_max_fetch_32 (pmix_atomic_int32_t *addr, int32_t value);
412 static inline int32_t pmix_atomic_fetch_max_32 (pmix_atomic_int32_t *addr, int32_t value);
413
414 #endif /* PMIX_HAVE_ATOMIC_MATH_32 */
415
416 #if ! PMIX_HAVE_ATOMIC_MATH_32
417 /* fix up the value of pmix_have_atomic_math_32 to allow for C versions */
418 #undef PMIX_HAVE_ATOMIC_MATH_32
419 #define PMIX_HAVE_ATOMIC_MATH_32 PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
420 #endif
421
422 #ifndef PMIX_HAVE_ATOMIC_MATH_64
423 /* define to 0 for these tests. WIll fix up later. */
424 #define PMIX_HAVE_ATOMIC_MATH_64 0
425 #endif
426
427 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_64 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
428
429 static inline int64_t pmix_atomic_add_fetch_64(pmix_atomic_int64_t *addr, int64_t delta);
430 static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta);
431 static inline int64_t pmix_atomic_and_fetch_64(pmix_atomic_int64_t *addr, int64_t value);
432 static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value);
433 static inline int64_t pmix_atomic_or_fetch_64(pmix_atomic_int64_t *addr, int64_t value);
434 static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value);
435 static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value);
436 static inline int64_t pmix_atomic_sub_fetch_64(pmix_atomic_int64_t *addr, int64_t delta);
437 static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta);
438 static inline int64_t pmix_atomic_min_fetch_64 (pmix_atomic_int64_t *addr, int64_t value);
439 static inline int64_t pmix_atomic_fetch_min_64 (pmix_atomic_int64_t *addr, int64_t value);
440 static inline int64_t pmix_atomic_max_fetch_64 (pmix_atomic_int64_t *addr, int64_t value);
441 static inline int64_t pmix_atomic_fetch_max_64 (pmix_atomic_int64_t *addr, int64_t value);
442
443 #endif /* PMIX_HAVE_ATOMIC_MATH_64 */
444
445 #if ! PMIX_HAVE_ATOMIC_MATH_64
446 /* fix up the value of pmix_have_atomic_math_64 to allow for C versions */
447 #undef PMIX_HAVE_ATOMIC_MATH_64
448 #define PMIX_HAVE_ATOMIC_MATH_64 PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
449 #endif
450
451 /* provide a size_t add/subtract. When in debug mode, make it an
452 * inline function so that we don't have any casts in the
453 * interface and can catch type errors. When not in debug mode,
454 * just make it a macro, so that there's no performance penalty
455 */
456 #if defined(DOXYGEN) || PMIX_ENABLE_DEBUG
457 static inline size_t
pmix_atomic_add_fetch_size_t(pmix_atomic_size_t * addr,size_t delta)458 pmix_atomic_add_fetch_size_t(pmix_atomic_size_t *addr, size_t delta)
459 {
460 #if SIZEOF_SIZE_T == 4
461 return (size_t) pmix_atomic_add_fetch_32((int32_t*) addr, delta);
462 #elif SIZEOF_SIZE_T == 8
463 return (size_t) pmix_atomic_add_fetch_64((int64_t*) addr, delta);
464 #else
465 #error "Unknown size_t size"
466 #endif
467 }
468
469 static inline size_t
pmix_atomic_fetch_add_size_t(pmix_atomic_size_t * addr,size_t delta)470 pmix_atomic_fetch_add_size_t(pmix_atomic_size_t *addr, size_t delta)
471 {
472 #if SIZEOF_SIZE_T == 4
473 return (size_t) pmix_atomic_fetch_add_32((int32_t*) addr, delta);
474 #elif SIZEOF_SIZE_T == 8
475 return (size_t) pmix_atomic_fetch_add_64((int64_t*) addr, delta);
476 #else
477 #error "Unknown size_t size"
478 #endif
479 }
480
481 static inline size_t
pmix_atomic_sub_fetch_size_t(pmix_atomic_size_t * addr,size_t delta)482 pmix_atomic_sub_fetch_size_t(pmix_atomic_size_t *addr, size_t delta)
483 {
484 #if SIZEOF_SIZE_T == 4
485 return (size_t) pmix_atomic_sub_fetch_32((int32_t*) addr, delta);
486 #elif SIZEOF_SIZE_T == 8
487 return (size_t) pmix_atomic_sub_fetch_64((int64_t*) addr, delta);
488 #else
489 #error "Unknown size_t size"
490 #endif
491 }
492
493 static inline size_t
pmix_atomic_fetch_sub_size_t(pmix_atomic_size_t * addr,size_t delta)494 pmix_atomic_fetch_sub_size_t(pmix_atomic_size_t *addr, size_t delta)
495 {
496 #if SIZEOF_SIZE_T == 4
497 return (size_t) pmix_atomic_fetch_sub_32((int32_t*) addr, delta);
498 #elif SIZEOF_SIZE_T == 8
499 return (size_t) pmix_atomic_fetch_sub_64((int64_t*) addr, delta);
500 #else
501 #error "Unknown size_t size"
502 #endif
503 }
504
505 #else
506 #if SIZEOF_SIZE_T == 4
507 #define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_32((pmix_atomic_int32_t *) addr, delta))
508 #define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_32((pmix_atomic_int32_t *) addr, delta))
509 #define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_32((pmix_atomic_int32_t *) addr, delta))
510 #define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_32((pmix_atomic_int32_t *) addr, delta))
511 #elif SIZEOF_SIZE_T == 8
512 #define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_64((pmix_atomic_int64_t *) addr, delta))
513 #define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_64((pmix_atomic_int64_t *) addr, delta))
514 #define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_64((pmix_atomic_int64_t *) addr, delta))
515 #define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_64((pmix_atomic_int64_t *) addr, delta))
516 #else
517 #error "Unknown size_t size"
518 #endif
519 #endif
520
521 #if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
522 /* these are always done with inline functions, so always mark as
523 static inline */
524
525 static inline bool pmix_atomic_compare_exchange_strong_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval,
526 int64_t newval, size_t length);
527 static inline bool pmix_atomic_compare_exchange_strong_acq_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval,
528 int64_t newval, size_t length);
529 static inline bool pmix_atomic_compare_exchange_strong_rel_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval,
530 int64_t newval, size_t length);
531
532
533 static inline bool pmix_atomic_compare_exchange_strong_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval,
534 intptr_t newval);
535 static inline bool pmix_atomic_compare_exchange_strong_acq_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval,
536 intptr_t newval);
537 static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval,
538 intptr_t newval);
539
540 /**
541 * Atomic compare and set of generic type with relaxed semantics. This
542 * macro detect at compile time the type of the first argument and
543 * choose the correct function to be called.
544 *
545 * \note This macro should only be used for integer types.
546 *
547 * @param addr Address of <TYPE>.
548 * @param oldval Comparison value address of <TYPE>.
549 * @param newval New value to set if comparision is true <TYPE>.
550 *
551 * See pmix_atomic_compare_exchange_* for pseudo-code.
552 */
553 #define pmix_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \
554 pmix_atomic_compare_exchange_strong_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
555 (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
556
557 /**
558 * Atomic compare and set of generic type with acquire semantics. This
559 * macro detect at compile time the type of the first argument and
560 * choose the correct function to be called.
561 *
562 * \note This macro should only be used for integer types.
563 *
564 * @param addr Address of <TYPE>.
565 * @param oldval Comparison value address of <TYPE>.
566 * @param newval New value to set if comparision is true <TYPE>.
567 *
568 * See pmix_atomic_compare_exchange_acq_* for pseudo-code.
569 */
570 #define pmix_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \
571 pmix_atomic_compare_exchange_strong_acq_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
572 (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
573
574 /**
575 * Atomic compare and set of generic type with release semantics. This
576 * macro detect at compile time the type of the first argument and
577 * choose the correct function to be called.
578 *
579 * \note This macro should only be used for integer types.
580 *
581 * @param addr Address of <TYPE>.
582 * @param oldval Comparison value address of <TYPE>.
583 * @param newval New value to set if comparision is true <TYPE>.
584 *
585 * See pmix_atomic_compare_exchange_rel_* for pseudo-code.
586 */
587 #define pmix_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
588 pmix_atomic_compare_exchange_strong_rel_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
589 (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
590
591
592 #endif /* (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
593
594 #if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64)
595
596 static inline void pmix_atomic_add_xx(pmix_atomic_intptr_t* addr,
597 int32_t value, size_t length);
598 static inline void pmix_atomic_sub_xx(pmix_atomic_intptr_t* addr,
599 int32_t value, size_t length);
600
601 static inline intptr_t pmix_atomic_add_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta );
602 static inline intptr_t pmix_atomic_fetch_add_ptr( pmix_atomic_intptr_t* addr, void* delta );
603 static inline intptr_t pmix_atomic_sub_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta );
604 static inline intptr_t pmix_atomic_fetch_sub_ptr( pmix_atomic_intptr_t* addr, void* delta );
605
606 /**
607 * Atomically increment the content depending on the type. This
608 * macro detect at compile time the type of the first argument
609 * and choose the correct function to be called.
610 *
611 * \note This macro should only be used for integer types.
612 *
613 * @param addr Address of <TYPE>
614 * @param delta Value to add (converted to <TYPE>).
615 */
616 #define pmix_atomic_add( ADDR, VALUE ) \
617 pmix_atomic_add_xx( (pmix_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
618 sizeof(*(ADDR)) )
619
620 /**
621 * Atomically decrement the content depending on the type. This
622 * macro detect at compile time the type of the first argument
623 * and choose the correct function to be called.
624 *
625 * \note This macro should only be used for integer types.
626 *
627 * @param addr Address of <TYPE>
628 * @param delta Value to substract (converted to <TYPE>).
629 */
630 #define pmix_atomic_sub( ADDR, VALUE ) \
631 pmix_atomic_sub_xx( (pmix_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
632 sizeof(*(ADDR)) )
633
634 #endif /* PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64 */
635
636
637 /*
638 * Include inline implementations of everything not defined directly
639 * in assembly
640 */
641 #include "src/atomics/sys/atomic_impl.h"
642
643 #endif /* !PMIX_C_HAVE__ATOMIC */
644
645 END_C_DECLS
646
647 #endif /* PMIX_SYS_ATOMIC_H */
648