1 /**
2 * \file
3 * Atomic operations
4 *
5 * Author:
6 * Dick Porter (dick@ximian.com)
7 *
8 * (C) 2002 Ximian, Inc.
9 * Copyright 2012 Xamarin Inc
10 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
11 */
12
13 #ifndef _WAPI_ATOMIC_H_
14 #define _WAPI_ATOMIC_H_
15
16 #include "config.h"
17 #include <glib.h>
18 #include <mono/utils/mono-membar.h>
19
20 /*
21 The current Nexus 7 arm-v7a fails with:
22 F/MonoDroid( 1568): shared runtime initialization error: Cannot load library: reloc_library[1285]: 37 cannot locate '__sync_val_compare_and_swap_8'
23
24 Apple targets have historically being problematic, xcode 4.6 would miscompile the intrinsic.
25 */
26
27 #if defined(HOST_WIN32)
28
29 #ifndef WIN32_LEAN_AND_MEAN
30 #define WIN32_LEAN_AND_MEAN
31 #endif
32 #include <windows.h>
33
34 static inline gint32
mono_atomic_cas_i32(volatile gint32 * dest,gint32 exch,gint32 comp)35 mono_atomic_cas_i32 (volatile gint32 *dest, gint32 exch, gint32 comp)
36 {
37 return InterlockedCompareExchange ((LONG volatile *)dest, (LONG)exch, (LONG)comp);
38 }
39
40 static inline gint64
mono_atomic_cas_i64(volatile gint64 * dest,gint64 exch,gint64 comp)41 mono_atomic_cas_i64 (volatile gint64 *dest, gint64 exch, gint64 comp)
42 {
43 return InterlockedCompareExchange64 ((LONG64 volatile *)dest, (LONG64)exch, (LONG64)comp);
44 }
45
46 static inline gpointer
mono_atomic_cas_ptr(volatile gpointer * dest,gpointer exch,gpointer comp)47 mono_atomic_cas_ptr (volatile gpointer *dest, gpointer exch, gpointer comp)
48 {
49 return InterlockedCompareExchangePointer ((PVOID volatile *)dest, (PVOID)exch, (PVOID)comp);
50 }
51
52 static inline gint32
mono_atomic_add_i32(volatile gint32 * dest,gint32 add)53 mono_atomic_add_i32 (volatile gint32 *dest, gint32 add)
54 {
55 return InterlockedAdd ((LONG volatile *)dest, (LONG)add);
56 }
57
58 static inline gint64
mono_atomic_add_i64(volatile gint64 * dest,gint64 add)59 mono_atomic_add_i64 (volatile gint64 *dest, gint64 add)
60 {
61 return InterlockedAdd64 ((LONG64 volatile *)dest, (LONG64)add);
62 }
63
64 static inline gint32
mono_atomic_inc_i32(volatile gint32 * dest)65 mono_atomic_inc_i32 (volatile gint32 *dest)
66 {
67 return InterlockedIncrement ((LONG volatile *)dest);
68 }
69
70 static inline gint64
mono_atomic_inc_i64(volatile gint64 * dest)71 mono_atomic_inc_i64 (volatile gint64 *dest)
72 {
73 return InterlockedIncrement64 ((LONG64 volatile *)dest);
74 }
75
76 static inline gint32
mono_atomic_dec_i32(volatile gint32 * dest)77 mono_atomic_dec_i32 (volatile gint32 *dest)
78 {
79 return InterlockedDecrement ((LONG volatile *)dest);
80 }
81
82 static inline gint64
mono_atomic_dec_i64(volatile gint64 * dest)83 mono_atomic_dec_i64 (volatile gint64 *dest)
84 {
85 return InterlockedDecrement64 ((LONG64 volatile *)dest);
86 }
87
88 static inline gint32
mono_atomic_xchg_i32(volatile gint32 * dest,gint32 exch)89 mono_atomic_xchg_i32 (volatile gint32 *dest, gint32 exch)
90 {
91 return InterlockedExchange ((LONG volatile *)dest, (LONG)exch);
92 }
93
94 static inline gint64
mono_atomic_xchg_i64(volatile gint64 * dest,gint64 exch)95 mono_atomic_xchg_i64 (volatile gint64 *dest, gint64 exch)
96 {
97 return InterlockedExchange64 ((LONG64 volatile *)dest, (LONG64)exch);
98 }
99
100 static inline gpointer
mono_atomic_xchg_ptr(volatile gpointer * dest,gpointer exch)101 mono_atomic_xchg_ptr (volatile gpointer *dest, gpointer exch)
102 {
103 return InterlockedExchangePointer ((PVOID volatile *)dest, (PVOID)exch);
104 }
105
106 static inline gint32
mono_atomic_fetch_add_i32(volatile gint32 * dest,gint32 add)107 mono_atomic_fetch_add_i32 (volatile gint32 *dest, gint32 add)
108 {
109 return InterlockedExchangeAdd ((LONG volatile *)dest, (LONG)add);
110 }
111
112 static inline gint64
mono_atomic_fetch_add_i64(volatile gint64 * dest,gint64 add)113 mono_atomic_fetch_add_i64 (volatile gint64 *dest, gint64 add)
114 {
115 return InterlockedExchangeAdd64 ((LONG64 volatile *)dest, (LONG)add);
116 }
117
118 static inline gint8
mono_atomic_load_i8(volatile gint8 * src)119 mono_atomic_load_i8 (volatile gint8 *src)
120 {
121 gint8 loaded_value = *src;
122 _ReadWriteBarrier ();
123
124 return loaded_value;
125 }
126
127 static inline gint16
mono_atomic_load_i16(volatile gint16 * src)128 mono_atomic_load_i16 (volatile gint16 *src)
129 {
130 gint16 loaded_value = *src;
131 _ReadWriteBarrier ();
132
133 return loaded_value;
134 }
135
mono_atomic_load_i32(volatile gint32 * src)136 static inline gint32 mono_atomic_load_i32 (volatile gint32 *src)
137 {
138 gint32 loaded_value = *src;
139 _ReadWriteBarrier ();
140
141 return loaded_value;
142 }
143
144 static inline gint64
mono_atomic_load_i64(volatile gint64 * src)145 mono_atomic_load_i64 (volatile gint64 *src)
146 {
147 #if defined(TARGET_AMD64)
148 gint64 loaded_value = *src;
149 _ReadWriteBarrier ();
150
151 return loaded_value;
152 #else
153 return InterlockedCompareExchange64 ((LONG64 volatile *)src, 0, 0);
154 #endif
155 }
156
157 static inline gpointer
mono_atomic_load_ptr(volatile gpointer * src)158 mono_atomic_load_ptr (volatile gpointer *src)
159 {
160 gpointer loaded_value = *src;
161 _ReadWriteBarrier ();
162
163 return loaded_value;
164 }
165
166 static inline void
mono_atomic_store_i8(volatile gint8 * dst,gint8 val)167 mono_atomic_store_i8 (volatile gint8 *dst, gint8 val)
168 {
169 #if (_MSC_VER >= 1600)
170 InterlockedExchange8 ((CHAR volatile *)dst, (CHAR)val);
171 #else
172 *dst = val;
173 mono_memory_barrier ();
174 #endif
175 }
176
177 static inline void
mono_atomic_store_i16(volatile gint16 * dst,gint16 val)178 mono_atomic_store_i16 (volatile gint16 *dst, gint16 val)
179 {
180 #if (_MSC_VER >= 1600)
181 InterlockedExchange16 ((SHORT volatile *)dst, (SHORT)val);
182 #else
183 *dst = val;
184 mono_memory_barrier ();
185 #endif
186 }
187
188 static inline void
mono_atomic_store_i32(volatile gint32 * dst,gint32 val)189 mono_atomic_store_i32 (volatile gint32 *dst, gint32 val)
190 {
191 InterlockedExchange ((LONG volatile *)dst, (LONG)val);
192 }
193
194 static inline void
mono_atomic_store_i64(volatile gint64 * dst,gint64 val)195 mono_atomic_store_i64 (volatile gint64 *dst, gint64 val)
196 {
197 InterlockedExchange64 ((LONG64 volatile *)dst, (LONG64)val);
198 }
199
200 static inline void
mono_atomic_store_ptr(volatile gpointer * dst,gpointer val)201 mono_atomic_store_ptr (volatile gpointer *dst, gpointer val)
202 {
203 InterlockedExchangePointer ((PVOID volatile *)dst, (PVOID)val);
204 }
205
206 /* Prefer GCC atomic ops if the target supports it (see configure.ac). */
207 #elif defined(USE_GCC_ATOMIC_OPS)
208
209 /*
210 * As of this comment (August 2016), all current Clang versions get atomic
211 * intrinsics on ARM64 wrong. All GCC versions prior to 5.3.0 do, too. The bug
212 * is the same: The compiler developers thought that the acq + rel barriers
213 * that ARM64 load/store instructions can impose are sufficient to provide
214 * sequential consistency semantics. This is not the case:
215 *
216 * http://lists.infradead.org/pipermail/linux-arm-kernel/2014-February/229588.html
217 *
218 * We work around this bug by inserting full barriers around each atomic
219 * intrinsic if we detect that we're built with a buggy compiler.
220 */
221
222 #if defined (HOST_ARM64) && (defined (__clang__) || MONO_GNUC_VERSION < 50300)
223 #define WRAP_ATOMIC_INTRINSIC(INTRIN) \
224 ({ \
225 mono_memory_barrier (); \
226 __typeof__ (INTRIN) atomic_ret__ = (INTRIN); \
227 mono_memory_barrier (); \
228 atomic_ret__; \
229 })
230
231 #define gcc_sync_val_compare_and_swap(a, b, c) WRAP_ATOMIC_INTRINSIC (__sync_val_compare_and_swap (a, b, c))
232 #define gcc_sync_add_and_fetch(a, b) WRAP_ATOMIC_INTRINSIC (__sync_add_and_fetch (a, b))
233 #define gcc_sync_sub_and_fetch(a, b) WRAP_ATOMIC_INTRINSIC (__sync_sub_and_fetch (a, b))
234 #define gcc_sync_fetch_and_add(a, b) WRAP_ATOMIC_INTRINSIC (__sync_fetch_and_add (a, b))
235 #else
236 #define gcc_sync_val_compare_and_swap(a, b, c) __sync_val_compare_and_swap (a, b, c)
237 #define gcc_sync_add_and_fetch(a, b) __sync_add_and_fetch (a, b)
238 #define gcc_sync_sub_and_fetch(a, b) __sync_sub_and_fetch (a, b)
239 #define gcc_sync_fetch_and_add(a, b) __sync_fetch_and_add (a, b)
240 #endif
241
mono_atomic_cas_i32(volatile gint32 * dest,gint32 exch,gint32 comp)242 static inline gint32 mono_atomic_cas_i32(volatile gint32 *dest,
243 gint32 exch, gint32 comp)
244 {
245 return gcc_sync_val_compare_and_swap (dest, comp, exch);
246 }
247
mono_atomic_cas_ptr(volatile gpointer * dest,gpointer exch,gpointer comp)248 static inline gpointer mono_atomic_cas_ptr(volatile gpointer *dest, gpointer exch, gpointer comp)
249 {
250 return gcc_sync_val_compare_and_swap (dest, comp, exch);
251 }
252
mono_atomic_add_i32(volatile gint32 * dest,gint32 add)253 static inline gint32 mono_atomic_add_i32(volatile gint32 *dest, gint32 add)
254 {
255 return gcc_sync_add_and_fetch (dest, add);
256 }
257
mono_atomic_inc_i32(volatile gint32 * val)258 static inline gint32 mono_atomic_inc_i32(volatile gint32 *val)
259 {
260 return gcc_sync_add_and_fetch (val, 1);
261 }
262
mono_atomic_dec_i32(volatile gint32 * val)263 static inline gint32 mono_atomic_dec_i32(volatile gint32 *val)
264 {
265 return gcc_sync_sub_and_fetch (val, 1);
266 }
267
mono_atomic_xchg_i32(volatile gint32 * val,gint32 new_val)268 static inline gint32 mono_atomic_xchg_i32(volatile gint32 *val, gint32 new_val)
269 {
270 gint32 old_val;
271 do {
272 old_val = *val;
273 } while (gcc_sync_val_compare_and_swap (val, old_val, new_val) != old_val);
274 return old_val;
275 }
276
mono_atomic_xchg_ptr(volatile gpointer * val,gpointer new_val)277 static inline gpointer mono_atomic_xchg_ptr(volatile gpointer *val,
278 gpointer new_val)
279 {
280 gpointer old_val;
281 do {
282 old_val = *val;
283 } while (gcc_sync_val_compare_and_swap (val, old_val, new_val) != old_val);
284 return old_val;
285 }
286
mono_atomic_fetch_add_i32(volatile gint32 * val,gint32 add)287 static inline gint32 mono_atomic_fetch_add_i32(volatile gint32 *val, gint32 add)
288 {
289 return gcc_sync_fetch_and_add (val, add);
290 }
291
mono_atomic_load_i8(volatile gint8 * src)292 static inline gint8 mono_atomic_load_i8(volatile gint8 *src)
293 {
294 /* Kind of a hack, but GCC doesn't give us anything better, and it's
295 * certainly not as bad as using a CAS loop. */
296 return gcc_sync_fetch_and_add (src, 0);
297 }
298
mono_atomic_load_i16(volatile gint16 * src)299 static inline gint16 mono_atomic_load_i16(volatile gint16 *src)
300 {
301 return gcc_sync_fetch_and_add (src, 0);
302 }
303
mono_atomic_load_i32(volatile gint32 * src)304 static inline gint32 mono_atomic_load_i32(volatile gint32 *src)
305 {
306 return gcc_sync_fetch_and_add (src, 0);
307 }
308
mono_atomic_store_i8(volatile gint8 * dst,gint8 val)309 static inline void mono_atomic_store_i8(volatile gint8 *dst, gint8 val)
310 {
311 /* Nothing useful from GCC at all, so fall back to CAS. */
312 gint8 old_val;
313 do {
314 old_val = *dst;
315 } while (gcc_sync_val_compare_and_swap (dst, old_val, val) != old_val);
316 }
317
mono_atomic_store_i16(volatile gint16 * dst,gint16 val)318 static inline void mono_atomic_store_i16(volatile gint16 *dst, gint16 val)
319 {
320 gint16 old_val;
321 do {
322 old_val = *dst;
323 } while (gcc_sync_val_compare_and_swap (dst, old_val, val) != old_val);
324 }
325
mono_atomic_store_i32(volatile gint32 * dst,gint32 val)326 static inline void mono_atomic_store_i32(volatile gint32 *dst, gint32 val)
327 {
328 /* Nothing useful from GCC at all, so fall back to CAS. */
329 gint32 old_val;
330 do {
331 old_val = *dst;
332 } while (gcc_sync_val_compare_and_swap (dst, old_val, val) != old_val);
333 }
334
335 #if defined (TARGET_OSX) || defined (__arm__) || (defined (__mips__) && !defined (__mips64)) || (defined (__powerpc__) && !defined (__powerpc64__)) || (defined (__sparc__) && !defined (__arch64__))
336 #define BROKEN_64BIT_ATOMICS_INTRINSIC 1
337 #endif
338
339 #if !defined (BROKEN_64BIT_ATOMICS_INTRINSIC)
340
mono_atomic_cas_i64(volatile gint64 * dest,gint64 exch,gint64 comp)341 static inline gint64 mono_atomic_cas_i64(volatile gint64 *dest, gint64 exch, gint64 comp)
342 {
343 return gcc_sync_val_compare_and_swap (dest, comp, exch);
344 }
345
mono_atomic_add_i64(volatile gint64 * dest,gint64 add)346 static inline gint64 mono_atomic_add_i64(volatile gint64 *dest, gint64 add)
347 {
348 return gcc_sync_add_and_fetch (dest, add);
349 }
350
mono_atomic_inc_i64(volatile gint64 * val)351 static inline gint64 mono_atomic_inc_i64(volatile gint64 *val)
352 {
353 return gcc_sync_add_and_fetch (val, 1);
354 }
355
mono_atomic_dec_i64(volatile gint64 * val)356 static inline gint64 mono_atomic_dec_i64(volatile gint64 *val)
357 {
358 return gcc_sync_sub_and_fetch (val, 1);
359 }
360
mono_atomic_fetch_add_i64(volatile gint64 * val,gint64 add)361 static inline gint64 mono_atomic_fetch_add_i64(volatile gint64 *val, gint64 add)
362 {
363 return gcc_sync_fetch_and_add (val, add);
364 }
365
mono_atomic_load_i64(volatile gint64 * src)366 static inline gint64 mono_atomic_load_i64(volatile gint64 *src)
367 {
368 /* Kind of a hack, but GCC doesn't give us anything better. */
369 return gcc_sync_fetch_and_add (src, 0);
370 }
371
372 #else
373
374 /* Implement 64-bit cas by hand or emulate it. */
375 extern gint64 mono_atomic_cas_i64(volatile gint64 *dest, gint64 exch, gint64 comp);
376
377 /* Implement all other 64-bit atomics in terms of a specialized CAS
378 * in this case, since chances are that the other 64-bit atomic
379 * intrinsics are broken too.
380 */
381
mono_atomic_fetch_add_i64(volatile gint64 * dest,gint64 add)382 static inline gint64 mono_atomic_fetch_add_i64(volatile gint64 *dest, gint64 add)
383 {
384 gint64 old_val;
385 do {
386 old_val = *dest;
387 } while (mono_atomic_cas_i64 (dest, old_val + add, old_val) != old_val);
388 return old_val;
389 }
390
mono_atomic_inc_i64(volatile gint64 * val)391 static inline gint64 mono_atomic_inc_i64(volatile gint64 *val)
392 {
393 gint64 get, set;
394 do {
395 get = *val;
396 set = get + 1;
397 } while (mono_atomic_cas_i64 (val, set, get) != get);
398 return set;
399 }
400
mono_atomic_dec_i64(volatile gint64 * val)401 static inline gint64 mono_atomic_dec_i64(volatile gint64 *val)
402 {
403 gint64 get, set;
404 do {
405 get = *val;
406 set = get - 1;
407 } while (mono_atomic_cas_i64 (val, set, get) != get);
408 return set;
409 }
410
mono_atomic_add_i64(volatile gint64 * dest,gint64 add)411 static inline gint64 mono_atomic_add_i64(volatile gint64 *dest, gint64 add)
412 {
413 gint64 get, set;
414 do {
415 get = *dest;
416 set = get + add;
417 } while (mono_atomic_cas_i64 (dest, set, get) != get);
418 return set;
419 }
420
mono_atomic_load_i64(volatile gint64 * src)421 static inline gint64 mono_atomic_load_i64(volatile gint64 *src)
422 {
423 return mono_atomic_cas_i64 (src, 0, 0);
424 }
425
426 #endif
427
mono_atomic_load_ptr(volatile gpointer * src)428 static inline gpointer mono_atomic_load_ptr(volatile gpointer *src)
429 {
430 return mono_atomic_cas_ptr (src, NULL, NULL);
431 }
432
mono_atomic_store_ptr(volatile gpointer * dst,gpointer val)433 static inline void mono_atomic_store_ptr(volatile gpointer *dst, gpointer val)
434 {
435 mono_atomic_xchg_ptr (dst, val);
436 }
437
438 /* We always implement this in terms of a 64-bit cas since
439 * GCC doesn't have an intrisic to model it anyway. */
mono_atomic_xchg_i64(volatile gint64 * val,gint64 new_val)440 static inline gint64 mono_atomic_xchg_i64(volatile gint64 *val, gint64 new_val)
441 {
442 gint64 old_val;
443 do {
444 old_val = *val;
445 } while (mono_atomic_cas_i64 (val, new_val, old_val) != old_val);
446 return old_val;
447 }
448
mono_atomic_store_i64(volatile gint64 * dst,gint64 val)449 static inline void mono_atomic_store_i64(volatile gint64 *dst, gint64 val)
450 {
451 /* Nothing useful from GCC at all, so fall back to CAS. */
452 mono_atomic_xchg_i64 (dst, val);
453 }
454
455 #else
456
457 #define WAPI_NO_ATOMIC_ASM
458
459 extern gint32 mono_atomic_cas_i32(volatile gint32 *dest, gint32 exch, gint32 comp);
460 extern gint64 mono_atomic_cas_i64(volatile gint64 *dest, gint64 exch, gint64 comp);
461 extern gpointer mono_atomic_cas_ptr(volatile gpointer *dest, gpointer exch, gpointer comp);
462 extern gint32 mono_atomic_add_i32(volatile gint32 *dest, gint32 add);
463 extern gint64 mono_atomic_add_i64(volatile gint64 *dest, gint64 add);
464 extern gint32 mono_atomic_inc_i32(volatile gint32 *dest);
465 extern gint64 mono_atomic_inc_i64(volatile gint64 *dest);
466 extern gint32 mono_atomic_dec_i32(volatile gint32 *dest);
467 extern gint64 mono_atomic_dec_i64(volatile gint64 *dest);
468 extern gint32 mono_atomic_xchg_i32(volatile gint32 *dest, gint32 exch);
469 extern gint64 mono_atomic_xchg_i64(volatile gint64 *dest, gint64 exch);
470 extern gpointer mono_atomic_xchg_ptr(volatile gpointer *dest, gpointer exch);
471 extern gint32 mono_atomic_fetch_add_i32(volatile gint32 *dest, gint32 add);
472 extern gint64 mono_atomic_fetch_add_i64(volatile gint64 *dest, gint64 add);
473 extern gint8 mono_atomic_load_i8(volatile gint8 *src);
474 extern gint16 mono_atomic_load_i16(volatile gint16 *src);
475 extern gint32 mono_atomic_load_i32(volatile gint32 *src);
476 extern gint64 mono_atomic_load_i64(volatile gint64 *src);
477 extern gpointer mono_atomic_load_ptr(volatile gpointer *src);
478 extern void mono_atomic_store_i8(volatile gint8 *dst, gint8 val);
479 extern void mono_atomic_store_i16(volatile gint16 *dst, gint16 val);
480 extern void mono_atomic_store_i32(volatile gint32 *dst, gint32 val);
481 extern void mono_atomic_store_i64(volatile gint64 *dst, gint64 val);
482 extern void mono_atomic_store_ptr(volatile gpointer *dst, gpointer val);
483
484 #endif
485
486 #if SIZEOF_VOID_P == 4
487 #define mono_atomic_fetch_add_word(p,add) mono_atomic_fetch_add_i32 ((volatile gint32*)p, (gint32)add)
488 #else
489 #define mono_atomic_fetch_add_word(p,add) mono_atomic_fetch_add_i64 ((volatile gint64*)p, (gint64)add)
490 #endif
491
492 /* The following functions cannot be found on any platform, and thus they can be declared without further existence checks */
493
494 static inline void
mono_atomic_store_bool(volatile gboolean * dest,gboolean val)495 mono_atomic_store_bool (volatile gboolean *dest, gboolean val)
496 {
497 /* both, gboolean and gint32, are int32_t; the purpose of these casts is to make things explicit */
498 mono_atomic_store_i32 ((volatile gint32 *)dest, (gint32)val);
499 }
500
501 #endif /* _WAPI_ATOMIC_H_ */
502