1 /*-------------------------------------------------------------------------
2 *
3 * generic-gcc.h
4 * Atomic operations, implemented using gcc (or compatible) intrinsics.
5 *
6 * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
7 * Portions Copyright (c) 1994, Regents of the University of California
8 *
9 * NOTES:
10 *
11 * Documentation:
12 * * Legacy __sync Built-in Functions for Atomic Memory Access
13 * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html
14 * * Built-in functions for memory model aware atomic operations
15 * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html
16 *
17 * src/include/port/atomics/generic-gcc.h
18 *
19 *-------------------------------------------------------------------------
20 */
21
22 /* intentionally no include guards, should only be included by atomics.h */
23 #ifndef INSIDE_ATOMICS_H
24 #error "should be included via atomics.h"
25 #endif
26
27 /*
28 * An empty asm block should be a sufficient compiler barrier.
29 */
30 #define pg_compiler_barrier_impl() __asm__ __volatile__("" ::: "memory")
31
32 /*
33 * If we're on GCC 4.1.0 or higher, we should be able to get a memory barrier
34 * out of this compiler built-in. But we prefer to rely on platform specific
35 * definitions where possible, and use this only as a fallback.
36 */
37 #if !defined(pg_memory_barrier_impl)
38 # if defined(HAVE_GCC__ATOMIC_INT32_CAS)
39 # define pg_memory_barrier_impl() __atomic_thread_fence(__ATOMIC_SEQ_CST)
40 # elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))
41 # define pg_memory_barrier_impl() __sync_synchronize()
42 # endif
43 #endif /* !defined(pg_memory_barrier_impl) */
44
45 #if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
46 /* acquire semantics include read barrier semantics */
47 # define pg_read_barrier_impl() __atomic_thread_fence(__ATOMIC_ACQUIRE)
48 #endif
49
50 #if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
51 /* release semantics include write barrier semantics */
52 # define pg_write_barrier_impl() __atomic_thread_fence(__ATOMIC_RELEASE)
53 #endif
54
55
56 #ifdef HAVE_ATOMICS
57
58 /* generic gcc based atomic flag implementation */
59 #if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \
60 && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS))
61
62 #define PG_HAVE_ATOMIC_FLAG_SUPPORT
63 typedef struct pg_atomic_flag
64 {
65 /*
66 * If we have a choice, use int-width TAS, because that is more efficient
67 * and/or more reliably implemented on most non-Intel platforms. (Note
68 * that this code isn't used on x86[_64]; see arch-x86.h for that.)
69 */
70 #ifdef HAVE_GCC__SYNC_INT32_TAS
71 volatile int value;
72 #else
73 volatile char value;
74 #endif
75 } pg_atomic_flag;
76
77 #endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */
78
79 /* generic gcc based atomic uint32 implementation */
80 #if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \
81 && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS))
82
83 #define PG_HAVE_ATOMIC_U32_SUPPORT
84 typedef struct pg_atomic_uint32
85 {
86 volatile uint32 value;
87 } pg_atomic_uint32;
88
89 #endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */
90
91 /* generic gcc based atomic uint64 implementation */
92 #if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \
93 && !defined(PG_DISABLE_64_BIT_ATOMICS) \
94 && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS))
95
96 #define PG_HAVE_ATOMIC_U64_SUPPORT
97
98 typedef struct pg_atomic_uint64
99 {
100 volatile uint64 value pg_attribute_aligned(8);
101 } pg_atomic_uint64;
102
103 #endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */
104
105 #ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT
106
107 #if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS)
108
109 #ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG
110 #define PG_HAVE_ATOMIC_TEST_SET_FLAG
111 static inline bool
pg_atomic_test_set_flag_impl(volatile pg_atomic_flag * ptr)112 pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
113 {
114 /* NB: only an acquire barrier, not a full one */
115 /* some platform only support a 1 here */
116 return __sync_lock_test_and_set(&ptr->value, 1) == 0;
117 }
118 #endif
119
120 #endif /* defined(HAVE_GCC__SYNC_*_TAS) */
121
122 #ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
123 #define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
124 static inline bool
pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag * ptr)125 pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
126 {
127 return ptr->value == 0;
128 }
129 #endif
130
131 #ifndef PG_HAVE_ATOMIC_CLEAR_FLAG
132 #define PG_HAVE_ATOMIC_CLEAR_FLAG
133 static inline void
pg_atomic_clear_flag_impl(volatile pg_atomic_flag * ptr)134 pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
135 {
136 __sync_lock_release(&ptr->value);
137 }
138 #endif
139
140 #ifndef PG_HAVE_ATOMIC_INIT_FLAG
141 #define PG_HAVE_ATOMIC_INIT_FLAG
142 static inline void
pg_atomic_init_flag_impl(volatile pg_atomic_flag * ptr)143 pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
144 {
145 pg_atomic_clear_flag_impl(ptr);
146 }
147 #endif
148
149 #endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */
150
151 /* prefer __atomic, it has a better API */
152 #if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
153 #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32
154 static inline bool
pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 * ptr,uint32 * expected,uint32 newval)155 pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
156 uint32 *expected, uint32 newval)
157 {
158 /* FIXME: we can probably use a lower consistency model */
159 return __atomic_compare_exchange_n(&ptr->value, expected, newval, false,
160 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
161 }
162 #endif
163
164 #if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
165 #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32
166 static inline bool
pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 * ptr,uint32 * expected,uint32 newval)167 pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
168 uint32 *expected, uint32 newval)
169 {
170 bool ret;
171 uint32 current;
172 current = __sync_val_compare_and_swap(&ptr->value, *expected, newval);
173 ret = current == *expected;
174 *expected = current;
175 return ret;
176 }
177 #endif
178
179 /* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */
180
181 #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
182 #define PG_HAVE_ATOMIC_FETCH_ADD_U32
183 static inline uint32
pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 * ptr,int32 add_)184 pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
185 {
186 return __sync_fetch_and_add(&ptr->value, add_);
187 }
188 #endif
189
190 #if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
191 #define PG_HAVE_ATOMIC_FETCH_SUB_U32
192 static inline uint32
pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 * ptr,int32 sub_)193 pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
194 {
195 return __sync_fetch_and_sub(&ptr->value, sub_);
196 }
197 #endif
198
199 #if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
200 #define PG_HAVE_ATOMIC_FETCH_AND_U32
201 static inline uint32
pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 * ptr,uint32 and_)202 pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_)
203 {
204 return __sync_fetch_and_and(&ptr->value, and_);
205 }
206 #endif
207
208 #if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
209 #define PG_HAVE_ATOMIC_FETCH_OR_U32
210 static inline uint32
pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 * ptr,uint32 or_)211 pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_)
212 {
213 return __sync_fetch_and_or(&ptr->value, or_);
214 }
215 #endif
216
217
218 #if !defined(PG_DISABLE_64_BIT_ATOMICS)
219
220 #if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS)
221 #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
222 static inline bool
pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 * ptr,uint64 * expected,uint64 newval)223 pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
224 uint64 *expected, uint64 newval)
225 {
226 return __atomic_compare_exchange_n(&ptr->value, expected, newval, false,
227 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
228 }
229 #endif
230
231 #if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
232 #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
233 static inline bool
pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 * ptr,uint64 * expected,uint64 newval)234 pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
235 uint64 *expected, uint64 newval)
236 {
237 bool ret;
238 uint64 current;
239 current = __sync_val_compare_and_swap(&ptr->value, *expected, newval);
240 ret = current == *expected;
241 *expected = current;
242 return ret;
243 }
244 #endif
245
246 /* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */
247
248 #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
249 #define PG_HAVE_ATOMIC_FETCH_ADD_U64
250 static inline uint64
pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 * ptr,int64 add_)251 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
252 {
253 return __sync_fetch_and_add(&ptr->value, add_);
254 }
255 #endif
256
257 #if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
258 #define PG_HAVE_ATOMIC_FETCH_SUB_U64
259 static inline uint64
pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 * ptr,int64 sub_)260 pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
261 {
262 return __sync_fetch_and_sub(&ptr->value, sub_);
263 }
264 #endif
265
266 #if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
267 #define PG_HAVE_ATOMIC_FETCH_AND_U64
268 static inline uint64
pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 * ptr,uint64 and_)269 pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_)
270 {
271 return __sync_fetch_and_and(&ptr->value, and_);
272 }
273 #endif
274
275 #if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
276 #define PG_HAVE_ATOMIC_FETCH_OR_U64
277 static inline uint64
pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 * ptr,uint64 or_)278 pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_)
279 {
280 return __sync_fetch_and_or(&ptr->value, or_);
281 }
282 #endif
283
284 #endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */
285
286 #endif /* defined(HAVE_ATOMICS) */
287