1 /*****************************************************************************
2 Copyright (c) 1995, 2021, Oracle and/or its affiliates.
3 Copyright (c) 2008, Google Inc.
4
5 Portions of this file contain modifications contributed and copyrighted by
6 Google, Inc. Those modifications are gratefully acknowledged and are described
7 briefly in the InnoDB documentation. The contributions by Google are
8 incorporated with their permission, and subject to the conditions contained in
9 the file COPYING.Google.
10
11 This program is free software; you can redistribute it and/or modify
12 it under the terms of the GNU General Public License, version 2.0,
13 as published by the Free Software Foundation.
14
15 This program is also distributed with certain software (including
16 but not limited to OpenSSL) that is licensed under separate terms,
17 as designated in a particular file or component or in included license
18 documentation. The authors of MySQL hereby grant you an additional
19 permission to link the program and your derivative works with the
20 separately licensed software that they have included with MySQL.
21
22 This program is distributed in the hope that it will be useful,
23 but WITHOUT ANY WARRANTY; without even the implied warranty of
24 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 GNU General Public License, version 2.0, for more details.
26
27 You should have received a copy of the GNU General Public License along with
28 this program; if not, write to the Free Software Foundation, Inc.,
29 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
30
31 *****************************************************************************/
32
33 /**************************************************//**
34 @file include/os0atomic.h
35 Macros for using atomics
36
37 Created 2012-09-23 Sunny Bains (Split from os0sync.h)
38 *******************************************************/
39
40 #ifndef os0atomic_h
41 #define os0atomic_h
42
43 #include "univ.i"
44
45 #ifdef _WIN32
46
47 /** On Windows, InterlockedExchange operates on LONG variable */
48 typedef LONG lock_word_t;
49
50 #elif defined(MUTEX_FUTEX)
51
52 typedef int lock_word_t;
53
54 # else
55
56 typedef ulint lock_word_t;
57
58 #endif /* _WIN32 */
59
60 #if defined __i386__ || defined __x86_64__ || defined _M_IX86 \
61 || defined _M_X64 || defined __WIN__
62
63 #define IB_STRONG_MEMORY_MODEL
64
65 #endif /* __i386__ || __x86_64__ || _M_IX86 || _M_X64 || __WIN__ */
66
67 /**********************************************************//**
68 Atomic compare-and-swap and increment for InnoDB. */
69
70 /** Do an atomic test and set.
71 @param[in/out] ptr Memory location to set
72 @param[in] new_val new value
73 @return old value of memory location. */
74 UNIV_INLINE
75 lock_word_t
76 os_atomic_test_and_set(
77 volatile lock_word_t* ptr,
78 lock_word_t new_val);
79
80
81 /** Do an atomic compare and set
82 @param[in/out] ptr Memory location to set
83 @param[in] old_val old value to compare
84 @param[in] new_val new value to set
85 @return the value of ptr before the operation. */
86 UNIV_INLINE
87 lock_word_t
88 os_atomic_val_compare_and_swap(
89 volatile lock_word_t* ptr,
90 lock_word_t old_val,
91 lock_word_t new_val);
92
93 #ifdef _WIN32
94
95 /**********************************************************//**
96 Atomic compare and exchange of signed integers (both 32 and 64 bit).
97 @return value found before the exchange.
98 If it is not equal to old_value the exchange did not happen. */
99 UNIV_INLINE
100 lint
101 win_cmp_and_xchg_lint(
102 /*==================*/
103 volatile lint* ptr, /*!< in/out: source/destination */
104 lint new_val, /*!< in: exchange value */
105 lint old_val); /*!< in: value to compare to */
106
107 /**********************************************************//**
108 Atomic addition of signed integers.
109 @return Initial value of the variable pointed to by ptr */
110 UNIV_INLINE
111 lint
112 win_xchg_and_add(
113 /*=============*/
114 volatile lint* ptr, /*!< in/out: address of destination */
115 lint val); /*!< in: number to be added */
116
117 /**********************************************************//**
118 Atomic compare and exchange of unsigned integers.
119 @return value found before the exchange.
120 If it is not equal to old_value the exchange did not happen. */
121 UNIV_INLINE
122 ulint
123 win_cmp_and_xchg_ulint(
124 /*===================*/
125 volatile ulint* ptr, /*!< in/out: source/destination */
126 ulint new_val, /*!< in: exchange value */
127 ulint old_val); /*!< in: value to compare to */
128
129 /**********************************************************//**
130 Atomic compare and exchange of 32 bit unsigned integers.
131 @return value found before the exchange.
132 If it is not equal to old_value the exchange did not happen. */
133 UNIV_INLINE
134 DWORD
135 win_cmp_and_xchg_dword(
136 /*===================*/
137 volatile DWORD* ptr, /*!< in/out: source/destination */
138 DWORD new_val, /*!< in: exchange value */
139 DWORD old_val); /*!< in: value to compare to */
140
141 /**********************************************************//**
142 Returns true if swapped, ptr is pointer to target, old_val is value to
143 compare to, new_val is the value to swap in. */
144
145 # define os_compare_and_swap_lint(ptr, old_val, new_val) \
146 (win_cmp_and_xchg_lint(ptr, new_val, old_val) == old_val)
147
148 # define os_compare_and_swap_ulint(ptr, old_val, new_val) \
149 (win_cmp_and_xchg_ulint(ptr, new_val, old_val) == old_val)
150
151 # define os_compare_and_swap_uint32(ptr, old_val, new_val) \
152 (InterlockedCompareExchange(ptr, new_val, old_val) == old_val)
153
154 /* windows thread objects can always be passed to windows atomic functions */
155 # define os_compare_and_swap_thread_id(ptr, old_val, new_val) \
156 (win_cmp_and_xchg_dword(ptr, new_val, old_val) == old_val)
157
158 # define INNODB_RW_LOCKS_USE_ATOMICS
159 # define IB_ATOMICS_STARTUP_MSG \
160 "Mutexes and rw_locks use Windows interlocked functions"
161
162 /**********************************************************//**
163 Returns the resulting value, ptr is pointer to target, amount is the
164 amount of increment. */
165
166 # define os_atomic_increment_lint(ptr, amount) \
167 (win_xchg_and_add(ptr, amount) + amount)
168
169 # define os_atomic_increment_ulint(ptr, amount) \
170 (static_cast<ulint>(win_xchg_and_add( \
171 reinterpret_cast<volatile lint*>(ptr), \
172 static_cast<lint>(amount))) \
173 + static_cast<ulint>(amount))
174
175 # define os_atomic_increment_uint32(ptr, amount) \
176 (static_cast<ulint>(InterlockedExchangeAdd( \
177 reinterpret_cast<long*>(ptr), \
178 static_cast<long>(amount))) \
179 + static_cast<ulint>(amount))
180
181 # define os_atomic_increment_uint64(ptr, amount) \
182 (static_cast<ib_uint64_t>(InterlockedExchangeAdd64( \
183 reinterpret_cast<LONGLONG*>(ptr), \
184 static_cast<LONGLONG>(amount))) \
185 + static_cast<ib_uint64_t>(amount))
186
187 /**********************************************************//**
188 Returns the resulting value, ptr is pointer to target, amount is the
189 amount to decrement. There is no atomic substract function on Windows */
190
191 # define os_atomic_decrement_lint(ptr, amount) \
192 (win_xchg_and_add(ptr, -(static_cast<lint>(amount))) - amount)
193
194 # define os_atomic_decrement_ulint(ptr, amount) \
195 (static_cast<ulint>(win_xchg_and_add( \
196 reinterpret_cast<volatile lint*>(ptr), \
197 -(static_cast<lint>(amount)))) \
198 - static_cast<ulint>(amount))
199
200 # define os_atomic_decrement_uint32(ptr, amount) \
201 (static_cast<ib_uint32_t>(InterlockedExchangeAdd( \
202 reinterpret_cast<long*>(ptr), \
203 -(static_cast<long>(amount)))) \
204 - static_cast<ib_uint32_t>(amount))
205
206 # define os_atomic_decrement_uint64(ptr, amount) \
207 (static_cast<ib_uint64_t>(InterlockedExchangeAdd64( \
208 reinterpret_cast<LONGLONG*>(ptr), \
209 -(static_cast<LONGLONG>(amount)))) \
210 - static_cast<ib_uint64_t>(amount))
211
212 #else
213 /* Fall back to GCC-style atomic builtins. */
214
215 /**********************************************************//**
216 Returns true if swapped, ptr is pointer to target, old_val is value to
217 compare to, new_val is the value to swap in. */
218
219 #if defined(HAVE_GCC_SYNC_BUILTINS)
220
221 # define os_compare_and_swap(ptr, old_val, new_val) \
222 __sync_bool_compare_and_swap(ptr, old_val, new_val)
223
224 # define os_compare_and_swap_ulint(ptr, old_val, new_val) \
225 os_compare_and_swap(ptr, old_val, new_val)
226
227 # define os_compare_and_swap_lint(ptr, old_val, new_val) \
228 os_compare_and_swap(ptr, old_val, new_val)
229
230 # define os_compare_and_swap_uint32(ptr, old_val, new_val) \
231 os_compare_and_swap(ptr, old_val, new_val)
232
233 #else
234
235 UNIV_INLINE
236 bool
os_compare_and_swap_ulint(volatile ulint * ptr,ulint old_val,ulint new_val)237 os_compare_and_swap_ulint(volatile ulint* ptr, ulint old_val, ulint new_val)
238 {
239 return __atomic_compare_exchange_n(ptr, &old_val, new_val, 0,
240 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
241 }
242
243 UNIV_INLINE
244 bool
os_compare_and_swap_lint(volatile lint * ptr,lint old_val,lint new_val)245 os_compare_and_swap_lint(volatile lint* ptr, lint old_val, lint new_val)
246 {
247 return __atomic_compare_exchange_n(ptr, &old_val, new_val, 0,
248 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
249 }
250
251 UNIV_INLINE
252 bool
os_compare_and_swap_uint32(volatile ib_uint32_t * ptr,ib_uint32_t old_val,ib_uint32_t new_val)253 os_compare_and_swap_uint32(volatile ib_uint32_t* ptr, ib_uint32_t old_val, ib_uint32_t new_val)
254 {
255 return __atomic_compare_exchange_n(ptr, &old_val, new_val, 0,
256 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
257 }
258
259 #endif /* HAVE_GCC_SYNC_BUILTINS */
260
261 # ifdef HAVE_IB_ATOMIC_PTHREAD_T_GCC
262 #if defined(HAVE_GCC_SYNC_BUILTINS)
263 # define os_compare_and_swap_thread_id(ptr, old_val, new_val) \
264 os_compare_and_swap(ptr, old_val, new_val)
265 #else
266 UNIV_INLINE
267 bool
os_compare_and_swap_thread_id(volatile os_thread_id_t * ptr,os_thread_id_t old_val,os_thread_id_t new_val)268 os_compare_and_swap_thread_id(volatile os_thread_id_t* ptr, os_thread_id_t old_val, os_thread_id_t new_val)
269 {
270 return __atomic_compare_exchange_n(ptr, &old_val, new_val, 0,
271 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
272 }
273 #endif /* HAVE_GCC_SYNC_BUILTINS */
274 # define INNODB_RW_LOCKS_USE_ATOMICS
275 # define IB_ATOMICS_STARTUP_MSG \
276 "Mutexes and rw_locks use GCC atomic builtins"
277 # else /* HAVE_IB_ATOMIC_PTHREAD_T_GCC */
278 # define IB_ATOMICS_STARTUP_MSG \
279 "Mutexes use GCC atomic builtins, rw_locks do not"
280 # endif /* HAVE_IB_ATOMIC_PTHREAD_T_GCC */
281
282 /**********************************************************//**
283 Returns the resulting value, ptr is pointer to target, amount is the
284 amount of increment. */
285
286 #if defined(HAVE_GCC_SYNC_BUILTINS)
287 # define os_atomic_increment(ptr, amount) \
288 __sync_add_and_fetch(ptr, amount)
289 #else
290 # define os_atomic_increment(ptr, amount) \
291 __atomic_add_fetch(ptr, amount, __ATOMIC_SEQ_CST)
292 #endif /* HAVE_GCC_SYNC_BUILTINS */
293
294 # define os_atomic_increment_lint(ptr, amount) \
295 os_atomic_increment(ptr, amount)
296
297 # define os_atomic_increment_ulint(ptr, amount) \
298 os_atomic_increment(ptr, amount)
299
300 # define os_atomic_increment_uint32(ptr, amount ) \
301 os_atomic_increment(ptr, amount)
302
303 # define os_atomic_increment_uint64(ptr, amount) \
304 os_atomic_increment(ptr, amount)
305
306 /* Returns the resulting value, ptr is pointer to target, amount is the
307 amount to decrement. */
308
309 #if defined(HAVE_GCC_SYNC_BUILTINS)
310 # define os_atomic_decrement(ptr, amount) \
311 __sync_sub_and_fetch(ptr, amount)
312 #else
313 # define os_atomic_decrement(ptr, amount) \
314 __atomic_sub_fetch(ptr, amount, __ATOMIC_SEQ_CST)
315 #endif /* HAVE_GCC_SYNC_BUILTINS */
316
317 # define os_atomic_decrement_lint(ptr, amount) \
318 os_atomic_decrement(ptr, amount)
319
320 # define os_atomic_decrement_ulint(ptr, amount) \
321 os_atomic_decrement(ptr, amount)
322
323 # define os_atomic_decrement_uint32(ptr, amount) \
324 os_atomic_decrement(ptr, amount)
325
326 # define os_atomic_decrement_uint64(ptr, amount) \
327 os_atomic_decrement(ptr, amount)
328
329 #endif
330
331 #define os_atomic_inc_ulint(m,v,d) os_atomic_increment_ulint(v, d)
332 #define os_atomic_dec_ulint(m,v,d) os_atomic_decrement_ulint(v, d)
333 #define TAS(l, n) os_atomic_test_and_set((l), (n))
334 #define CAS(l, o, n) os_atomic_val_compare_and_swap((l), (o), (n))
335
336 /** barrier definitions for memory ordering */
337 #if defined(HAVE_IB_MACHINE_BARRIER_SOLARIS)
338 # define HAVE_MEMORY_BARRIER
339 # include <mbarrier.h>
340 # define os_rmb __machine_r_barrier()
341 # define os_wmb __machine_w_barrier()
342 # define IB_MEMORY_BARRIER_STARTUP_MSG \
343 "Solaris memory ordering functions are used for memory barrier"
344
345 #elif defined HAVE_IB_GCC_ATOMIC_THREAD_FENCE
346 # define HAVE_MEMORY_BARRIER
347 # define os_rmb __atomic_thread_fence(__ATOMIC_ACQUIRE)
348 # define os_wmb __atomic_thread_fence(__ATOMIC_RELEASE)
349 # define IB_MEMORY_BARRIER_STARTUP_MSG \
350 "GCC builtin __atomic_thread_fence() is used for memory barrier"
351
352 #elif defined(HAVE_IB_GCC_SYNC_SYNCHRONISE)
353 # define HAVE_MEMORY_BARRIER
354 # define os_rmb __sync_synchronize()
355 # define os_wmb __sync_synchronize()
356 # define IB_MEMORY_BARRIER_STARTUP_MSG \
357 "GCC builtin __sync_synchronize() is used for memory barrier"
358
359 #elif defined(HAVE_WINDOWS_MM_FENCE) && defined(_WIN64)
360 # define HAVE_MEMORY_BARRIER
361 # include <mmintrin.h>
362 # define os_rmb _mm_lfence()
363 # define os_wmb _mm_sfence()
364 # define IB_MEMORY_BARRIER_STARTUP_MSG \
365 "_mm_lfence() and _mm_sfence() are used for memory barrier"
366
367 #else
368 # define os_rmb
369 # define os_wmb
370 # define IB_MEMORY_BARRIER_STARTUP_MSG \
371 "Memory barrier is not used"
372 #endif
373
374 #ifndef UNIV_NONINL
375 #include "os0atomic.ic"
376 #endif /* UNIV_NOINL */
377
378 #endif /* !os0atomic_h */
379