1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2013 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
14 * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
15 * reserved.
16 * Copyright (c) 2016 Research Organization for Information Science
17 * and Technology (RIST). All rights reserved.
18 * Copyright (c) 2018 Triad National Security, LLC. All rights
19 * reserved.
20 * $COPYRIGHT$
21 *
22 * Additional copyrights may follow
23 *
24 * $HEADER$
25 */
26
27 #ifndef OPAL_SYS_ARCH_ATOMIC_H
28 #define OPAL_SYS_ARCH_ATOMIC_H 1
29
30 #include <stdbool.h>
31
32 /**********************************************************************
33 *
34 * Memory Barriers
35 *
36 *********************************************************************/
37 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
38
39 #define OPAL_HAVE_ATOMIC_MATH_32 1
40 #define OPAL_HAVE_ATOMIC_CMPSET_32 1
41 #define OPAL_HAVE_ATOMIC_ADD_32 1
42 #define OPAL_HAVE_ATOMIC_SUB_32 1
43 #define OPAL_HAVE_ATOMIC_SWAP_32 1
44 #define OPAL_HAVE_ATOMIC_MATH_64 1
45 #define OPAL_HAVE_ATOMIC_CMPSET_64 1
46 #define OPAL_HAVE_ATOMIC_ADD_64 1
47 #define OPAL_HAVE_ATOMIC_SUB_64 1
48 #define OPAL_HAVE_ATOMIC_SWAP_64 1
49
50
opal_atomic_mb(void)51 static inline void opal_atomic_mb(void)
52 {
53 __atomic_thread_fence (__ATOMIC_SEQ_CST);
54 }
55
opal_atomic_rmb(void)56 static inline void opal_atomic_rmb(void)
57 {
58 #if OPAL_ASSEMBLY_ARCH == OPAL_X86_64
59 /* work around a bug in older gcc versions where ACQUIRE seems to get
60 * treated as a no-op instead of being equivalent to
61 * __asm__ __volatile__("": : :"memory") */
62 __atomic_thread_fence (__ATOMIC_SEQ_CST);
63 #else
64 __atomic_thread_fence (__ATOMIC_ACQUIRE);
65 #endif
66 }
67
opal_atomic_wmb(void)68 static inline void opal_atomic_wmb(void)
69 {
70 __atomic_thread_fence (__ATOMIC_RELEASE);
71 }
72
73 #define MB() opal_atomic_mb()
74
75 /**********************************************************************
76 *
77 * Atomic math operations
78 *
79 *********************************************************************/
80
81 /*
82 * Suppress numerous (spurious ?) warnings from Oracle Studio compilers
83 * see https://community.oracle.com/thread/3968347
84 */
85 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
86 #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
87 #endif
88
opal_atomic_cmpset_acq_32(volatile int32_t * addr,int32_t oldval,int32_t newval)89 static inline int opal_atomic_cmpset_acq_32( volatile int32_t *addr,
90 int32_t oldval, int32_t newval)
91 {
92 return __atomic_compare_exchange_n (addr, &oldval, newval, false,
93 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
94 }
95
96
opal_atomic_cmpset_rel_32(volatile int32_t * addr,int32_t oldval,int32_t newval)97 static inline int opal_atomic_cmpset_rel_32( volatile int32_t *addr,
98 int32_t oldval, int32_t newval)
99 {
100 return __atomic_compare_exchange_n (addr, &oldval, newval, false,
101 __ATOMIC_RELEASE, __ATOMIC_RELAXED);
102 }
103
opal_atomic_cmpset_32(volatile int32_t * addr,int32_t oldval,int32_t newval)104 static inline int opal_atomic_cmpset_32( volatile int32_t *addr,
105 int32_t oldval, int32_t newval)
106 {
107 return __atomic_compare_exchange_n (addr, &oldval, newval, false,
108 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
109 }
110
opal_atomic_swap_32(volatile int32_t * addr,int32_t newval)111 static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval)
112 {
113 int32_t oldval;
114 __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
115 return oldval;
116 }
117
opal_atomic_add_32(volatile int32_t * addr,int32_t delta)118 static inline int32_t opal_atomic_add_32(volatile int32_t *addr, int32_t delta)
119 {
120 return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
121 }
122
opal_atomic_sub_32(volatile int32_t * addr,int32_t delta)123 static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, int32_t delta)
124 {
125 return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
126 }
127
opal_atomic_cmpset_acq_64(volatile int64_t * addr,int64_t oldval,int64_t newval)128 static inline int opal_atomic_cmpset_acq_64( volatile int64_t *addr,
129 int64_t oldval, int64_t newval)
130 {
131 return __atomic_compare_exchange_n (addr, &oldval, newval, false,
132 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
133 }
134
opal_atomic_cmpset_rel_64(volatile int64_t * addr,int64_t oldval,int64_t newval)135 static inline int opal_atomic_cmpset_rel_64( volatile int64_t *addr,
136 int64_t oldval, int64_t newval)
137 {
138 return __atomic_compare_exchange_n (addr, &oldval, newval, false,
139 __ATOMIC_RELEASE, __ATOMIC_RELAXED);
140 }
141
142
opal_atomic_cmpset_64(volatile int64_t * addr,int64_t oldval,int64_t newval)143 static inline int opal_atomic_cmpset_64( volatile int64_t *addr,
144 int64_t oldval, int64_t newval)
145 {
146 return __atomic_compare_exchange_n (addr, &oldval, newval, false,
147 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
148 }
149
opal_atomic_swap_64(volatile int64_t * addr,int64_t newval)150 static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
151 {
152 int64_t oldval;
153 __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
154 return oldval;
155 }
156
opal_atomic_add_64(volatile int64_t * addr,int64_t delta)157 static inline int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta)
158 {
159 return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
160 }
161
opal_atomic_sub_64(volatile int64_t * addr,int64_t delta)162 static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
163 {
164 return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
165 }
166
167 #if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
168
169 #define OPAL_HAVE_ATOMIC_CMPSET_128 1
170
opal_atomic_cmpset_128(volatile opal_int128_t * addr,opal_int128_t oldval,opal_int128_t newval)171 static inline int opal_atomic_cmpset_128 (volatile opal_int128_t *addr,
172 opal_int128_t oldval, opal_int128_t newval)
173 {
174 return __atomic_compare_exchange_n (addr, &oldval, newval, false,
175 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
176 }
177
178 #elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
179
180 #define OPAL_HAVE_ATOMIC_CMPSET_128 1
181
182 /* __atomic version is not lock-free so use legacy __sync version */
183
opal_atomic_cmpset_128(volatile opal_int128_t * addr,opal_int128_t oldval,opal_int128_t newval)184 static inline int opal_atomic_cmpset_128 (volatile opal_int128_t *addr,
185 opal_int128_t oldval, opal_int128_t newval)
186 {
187 return __sync_bool_compare_and_swap (addr, oldval, newval);
188 }
189
190 #endif
191
192 #if defined(__HLE__)
193
194 #include <immintrin.h>
195
196 #define OPAL_HAVE_ATOMIC_SPINLOCKS 1
197
opal_atomic_lock_init(opal_atomic_lock_t * lock,int32_t value)198 static inline void opal_atomic_lock_init (opal_atomic_lock_t* lock, int32_t value)
199 {
200 lock->u.lock = value;
201 }
202
opal_atomic_trylock(opal_atomic_lock_t * lock)203 static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
204 {
205 int ret = __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
206 __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
207 if (OPAL_ATOMIC_LOCK_LOCKED == ret) {
208 /* abort the transaction */
209 _mm_pause ();
210 return 1;
211 }
212
213 return 0;
214 }
215
opal_atomic_lock(opal_atomic_lock_t * lock)216 static inline void opal_atomic_lock (opal_atomic_lock_t *lock)
217 {
218 while (OPAL_ATOMIC_LOCK_LOCKED == __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
219 __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
220 /* abort the transaction */
221 _mm_pause ();
222 }
223 }
224
opal_atomic_unlock(opal_atomic_lock_t * lock)225 static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
226 {
227 __atomic_store_n (&lock->u.lock, OPAL_ATOMIC_LOCK_UNLOCKED,
228 __ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
229 }
230
231 #endif
232
233 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
234 #pragma error_messages(default, E_ARG_INCOMPATIBLE_WITH_ARG_L)
235 #endif
236
237 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
238