1 //===-- tsan_interface_atomic.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 //
11 //===----------------------------------------------------------------------===//
12
13 // ThreadSanitizer atomic operations are based on C++11/C1x standards.
14 // For background see C++11 standard. A slightly older, publicly
15 // available draft of the standard (not entirely up-to-date, but close enough
16 // for casual browsing) is available here:
17 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
18 // The following page contains more background information:
19 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
20
21 #include "sanitizer_common/sanitizer_placement_new.h"
22 #include "sanitizer_common/sanitizer_stacktrace.h"
23 #include "sanitizer_common/sanitizer_mutex.h"
24 #include "tsan_flags.h"
25 #include "tsan_interface.h"
26 #include "tsan_rtl.h"
27
28 using namespace __tsan;
29
30 #if !SANITIZER_GO && __TSAN_HAS_INT128
31 // Protects emulation of 128-bit atomic operations.
32 static StaticSpinMutex mutex128;
33 #endif
34
35 #if SANITIZER_DEBUG
IsLoadOrder(morder mo)36 static bool IsLoadOrder(morder mo) {
37 return mo == mo_relaxed || mo == mo_consume
38 || mo == mo_acquire || mo == mo_seq_cst;
39 }
40
IsStoreOrder(morder mo)41 static bool IsStoreOrder(morder mo) {
42 return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
43 }
44 #endif
45
IsReleaseOrder(morder mo)46 static bool IsReleaseOrder(morder mo) {
47 return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
48 }
49
IsAcquireOrder(morder mo)50 static bool IsAcquireOrder(morder mo) {
51 return mo == mo_consume || mo == mo_acquire
52 || mo == mo_acq_rel || mo == mo_seq_cst;
53 }
54
IsAcqRelOrder(morder mo)55 static bool IsAcqRelOrder(morder mo) {
56 return mo == mo_acq_rel || mo == mo_seq_cst;
57 }
58
func_xchg(volatile T * v,T op)59 template<typename T> T func_xchg(volatile T *v, T op) {
60 T res = __sync_lock_test_and_set(v, op);
61 // __sync_lock_test_and_set does not contain full barrier.
62 __sync_synchronize();
63 return res;
64 }
65
func_add(volatile T * v,T op)66 template<typename T> T func_add(volatile T *v, T op) {
67 return __sync_fetch_and_add(v, op);
68 }
69
func_sub(volatile T * v,T op)70 template<typename T> T func_sub(volatile T *v, T op) {
71 return __sync_fetch_and_sub(v, op);
72 }
73
func_and(volatile T * v,T op)74 template<typename T> T func_and(volatile T *v, T op) {
75 return __sync_fetch_and_and(v, op);
76 }
77
func_or(volatile T * v,T op)78 template<typename T> T func_or(volatile T *v, T op) {
79 return __sync_fetch_and_or(v, op);
80 }
81
func_xor(volatile T * v,T op)82 template<typename T> T func_xor(volatile T *v, T op) {
83 return __sync_fetch_and_xor(v, op);
84 }
85
func_nand(volatile T * v,T op)86 template<typename T> T func_nand(volatile T *v, T op) {
87 // clang does not support __sync_fetch_and_nand.
88 T cmp = *v;
89 for (;;) {
90 T newv = ~(cmp & op);
91 T cur = __sync_val_compare_and_swap(v, cmp, newv);
92 if (cmp == cur)
93 return cmp;
94 cmp = cur;
95 }
96 }
97
func_cas(volatile T * v,T cmp,T xch)98 template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
99 return __sync_val_compare_and_swap(v, cmp, xch);
100 }
101
102 // clang does not support 128-bit atomic ops.
103 // Atomic ops are executed under tsan internal mutex,
104 // here we assume that the atomic variables are not accessed
105 // from non-instrumented code.
106 #if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \
107 && __TSAN_HAS_INT128
func_xchg(volatile a128 * v,a128 op)108 a128 func_xchg(volatile a128 *v, a128 op) {
109 SpinMutexLock lock(&mutex128);
110 a128 cmp = *v;
111 *v = op;
112 return cmp;
113 }
114
func_add(volatile a128 * v,a128 op)115 a128 func_add(volatile a128 *v, a128 op) {
116 SpinMutexLock lock(&mutex128);
117 a128 cmp = *v;
118 *v = cmp + op;
119 return cmp;
120 }
121
func_sub(volatile a128 * v,a128 op)122 a128 func_sub(volatile a128 *v, a128 op) {
123 SpinMutexLock lock(&mutex128);
124 a128 cmp = *v;
125 *v = cmp - op;
126 return cmp;
127 }
128
func_and(volatile a128 * v,a128 op)129 a128 func_and(volatile a128 *v, a128 op) {
130 SpinMutexLock lock(&mutex128);
131 a128 cmp = *v;
132 *v = cmp & op;
133 return cmp;
134 }
135
func_or(volatile a128 * v,a128 op)136 a128 func_or(volatile a128 *v, a128 op) {
137 SpinMutexLock lock(&mutex128);
138 a128 cmp = *v;
139 *v = cmp | op;
140 return cmp;
141 }
142
func_xor(volatile a128 * v,a128 op)143 a128 func_xor(volatile a128 *v, a128 op) {
144 SpinMutexLock lock(&mutex128);
145 a128 cmp = *v;
146 *v = cmp ^ op;
147 return cmp;
148 }
149
func_nand(volatile a128 * v,a128 op)150 a128 func_nand(volatile a128 *v, a128 op) {
151 SpinMutexLock lock(&mutex128);
152 a128 cmp = *v;
153 *v = ~(cmp & op);
154 return cmp;
155 }
156
func_cas(volatile a128 * v,a128 cmp,a128 xch)157 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
158 SpinMutexLock lock(&mutex128);
159 a128 cur = *v;
160 if (cur == cmp)
161 *v = xch;
162 return cur;
163 }
164 #endif
165
166 template <typename T>
AccessSize()167 static int AccessSize() {
168 if (sizeof(T) <= 1)
169 return 1;
170 else if (sizeof(T) <= 2)
171 return 2;
172 else if (sizeof(T) <= 4)
173 return 4;
174 else
175 return 8;
176 // For 16-byte atomics we also use 8-byte memory access,
177 // this leads to false negatives only in very obscure cases.
178 }
179
180 #if !SANITIZER_GO
to_atomic(const volatile a8 * a)181 static atomic_uint8_t *to_atomic(const volatile a8 *a) {
182 return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
183 }
184
to_atomic(const volatile a16 * a)185 static atomic_uint16_t *to_atomic(const volatile a16 *a) {
186 return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
187 }
188 #endif
189
to_atomic(const volatile a32 * a)190 static atomic_uint32_t *to_atomic(const volatile a32 *a) {
191 return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
192 }
193
to_atomic(const volatile a64 * a)194 static atomic_uint64_t *to_atomic(const volatile a64 *a) {
195 return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
196 }
197
to_mo(morder mo)198 static memory_order to_mo(morder mo) {
199 switch (mo) {
200 case mo_relaxed: return memory_order_relaxed;
201 case mo_consume: return memory_order_consume;
202 case mo_acquire: return memory_order_acquire;
203 case mo_release: return memory_order_release;
204 case mo_acq_rel: return memory_order_acq_rel;
205 case mo_seq_cst: return memory_order_seq_cst;
206 }
207 DCHECK(0);
208 return memory_order_seq_cst;
209 }
210
211 template<typename T>
NoTsanAtomicLoad(const volatile T * a,morder mo)212 static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
213 return atomic_load(to_atomic(a), to_mo(mo));
214 }
215
216 #if __TSAN_HAS_INT128 && !SANITIZER_GO
NoTsanAtomicLoad(const volatile a128 * a,morder mo)217 static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
218 SpinMutexLock lock(&mutex128);
219 return *a;
220 }
221 #endif
222
223 template <typename T>
AtomicLoad(ThreadState * thr,uptr pc,const volatile T * a,morder mo)224 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
225 DCHECK(IsLoadOrder(mo));
226 // This fast-path is critical for performance.
227 // Assume the access is atomic.
228 if (!IsAcquireOrder(mo)) {
229 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
230 kAccessRead | kAccessAtomic);
231 return NoTsanAtomicLoad(a, mo);
232 }
233 // Don't create sync object if it does not exist yet. For example, an atomic
234 // pointer is initialized to nullptr and then periodically acquire-loaded.
235 T v = NoTsanAtomicLoad(a, mo);
236 SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
237 if (s) {
238 SlotLocker locker(thr);
239 ReadLock lock(&s->mtx);
240 thr->clock.Acquire(s->clock);
241 // Re-read under sync mutex because we need a consistent snapshot
242 // of the value and the clock we acquire.
243 v = NoTsanAtomicLoad(a, mo);
244 }
245 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead | kAccessAtomic);
246 return v;
247 }
248
249 template<typename T>
NoTsanAtomicStore(volatile T * a,T v,morder mo)250 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
251 atomic_store(to_atomic(a), v, to_mo(mo));
252 }
253
254 #if __TSAN_HAS_INT128 && !SANITIZER_GO
NoTsanAtomicStore(volatile a128 * a,a128 v,morder mo)255 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
256 SpinMutexLock lock(&mutex128);
257 *a = v;
258 }
259 #endif
260
261 template <typename T>
AtomicStore(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)262 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
263 morder mo) {
264 DCHECK(IsStoreOrder(mo));
265 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
266 // This fast-path is critical for performance.
267 // Assume the access is atomic.
268 // Strictly saying even relaxed store cuts off release sequence,
269 // so must reset the clock.
270 if (!IsReleaseOrder(mo)) {
271 NoTsanAtomicStore(a, v, mo);
272 return;
273 }
274 SlotLocker locker(thr);
275 {
276 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
277 Lock lock(&s->mtx);
278 thr->clock.ReleaseStore(&s->clock);
279 NoTsanAtomicStore(a, v, mo);
280 }
281 IncrementEpoch(thr);
282 }
283
284 template <typename T, T (*F)(volatile T *v, T op)>
AtomicRMW(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)285 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
286 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
287 if (LIKELY(mo == mo_relaxed))
288 return F(a, v);
289 SlotLocker locker(thr);
290 {
291 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
292 RWLock lock(&s->mtx, IsReleaseOrder(mo));
293 if (IsAcqRelOrder(mo))
294 thr->clock.ReleaseAcquire(&s->clock);
295 else if (IsReleaseOrder(mo))
296 thr->clock.Release(&s->clock);
297 else if (IsAcquireOrder(mo))
298 thr->clock.Acquire(s->clock);
299 v = F(a, v);
300 }
301 if (IsReleaseOrder(mo))
302 IncrementEpoch(thr);
303 return v;
304 }
305
306 template<typename T>
NoTsanAtomicExchange(volatile T * a,T v,morder mo)307 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
308 return func_xchg(a, v);
309 }
310
311 template<typename T>
NoTsanAtomicFetchAdd(volatile T * a,T v,morder mo)312 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
313 return func_add(a, v);
314 }
315
316 template<typename T>
NoTsanAtomicFetchSub(volatile T * a,T v,morder mo)317 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
318 return func_sub(a, v);
319 }
320
321 template<typename T>
NoTsanAtomicFetchAnd(volatile T * a,T v,morder mo)322 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
323 return func_and(a, v);
324 }
325
326 template<typename T>
NoTsanAtomicFetchOr(volatile T * a,T v,morder mo)327 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
328 return func_or(a, v);
329 }
330
331 template<typename T>
NoTsanAtomicFetchXor(volatile T * a,T v,morder mo)332 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
333 return func_xor(a, v);
334 }
335
336 template<typename T>
NoTsanAtomicFetchNand(volatile T * a,T v,morder mo)337 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
338 return func_nand(a, v);
339 }
340
341 template<typename T>
AtomicExchange(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)342 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
343 morder mo) {
344 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
345 }
346
347 template<typename T>
AtomicFetchAdd(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)348 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
349 morder mo) {
350 return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
351 }
352
353 template<typename T>
AtomicFetchSub(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)354 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
355 morder mo) {
356 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
357 }
358
359 template<typename T>
AtomicFetchAnd(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)360 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
361 morder mo) {
362 return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
363 }
364
365 template<typename T>
AtomicFetchOr(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)366 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
367 morder mo) {
368 return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
369 }
370
371 template<typename T>
AtomicFetchXor(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)372 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
373 morder mo) {
374 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
375 }
376
377 template<typename T>
AtomicFetchNand(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)378 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
379 morder mo) {
380 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
381 }
382
383 template<typename T>
NoTsanAtomicCAS(volatile T * a,T * c,T v,morder mo,morder fmo)384 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
385 return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
386 }
387
388 #if __TSAN_HAS_INT128
NoTsanAtomicCAS(volatile a128 * a,a128 * c,a128 v,morder mo,morder fmo)389 static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,
390 morder mo, morder fmo) {
391 a128 old = *c;
392 a128 cur = func_cas(a, old, v);
393 if (cur == old)
394 return true;
395 *c = cur;
396 return false;
397 }
398 #endif
399
400 template<typename T>
NoTsanAtomicCAS(volatile T * a,T c,T v,morder mo,morder fmo)401 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
402 NoTsanAtomicCAS(a, &c, v, mo, fmo);
403 return c;
404 }
405
406 template <typename T>
AtomicCAS(ThreadState * thr,uptr pc,volatile T * a,T * c,T v,morder mo,morder fmo)407 static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v,
408 morder mo, morder fmo) {
409 // 31.7.2.18: "The failure argument shall not be memory_order_release
410 // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
411 // (mo_relaxed) when those are used.
412 DCHECK(IsLoadOrder(fmo));
413
414 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
415 if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
416 T cc = *c;
417 T pr = func_cas(a, cc, v);
418 if (pr == cc)
419 return true;
420 *c = pr;
421 return false;
422 }
423 SlotLocker locker(thr);
424 bool release = IsReleaseOrder(mo);
425 bool success;
426 {
427 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
428 RWLock lock(&s->mtx, release);
429 T cc = *c;
430 T pr = func_cas(a, cc, v);
431 success = pr == cc;
432 if (!success) {
433 *c = pr;
434 mo = fmo;
435 }
436 if (success && IsAcqRelOrder(mo))
437 thr->clock.ReleaseAcquire(&s->clock);
438 else if (success && IsReleaseOrder(mo))
439 thr->clock.Release(&s->clock);
440 else if (IsAcquireOrder(mo))
441 thr->clock.Acquire(s->clock);
442 }
443 if (success && release)
444 IncrementEpoch(thr);
445 return success;
446 }
447
448 template<typename T>
AtomicCAS(ThreadState * thr,uptr pc,volatile T * a,T c,T v,morder mo,morder fmo)449 static T AtomicCAS(ThreadState *thr, uptr pc,
450 volatile T *a, T c, T v, morder mo, morder fmo) {
451 AtomicCAS(thr, pc, a, &c, v, mo, fmo);
452 return c;
453 }
454
455 #if !SANITIZER_GO
NoTsanAtomicFence(morder mo)456 static void NoTsanAtomicFence(morder mo) {
457 __sync_synchronize();
458 }
459
AtomicFence(ThreadState * thr,uptr pc,morder mo)460 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
461 // FIXME(dvyukov): not implemented.
462 __sync_synchronize();
463 }
464 #endif
465
466 // Interface functions follow.
467 #if !SANITIZER_GO
468
469 // C/C++
470
convert_morder(morder mo)471 static morder convert_morder(morder mo) {
472 if (flags()->force_seq_cst_atomics)
473 return (morder)mo_seq_cst;
474
475 // Filter out additional memory order flags:
476 // MEMMODEL_SYNC = 1 << 15
477 // __ATOMIC_HLE_ACQUIRE = 1 << 16
478 // __ATOMIC_HLE_RELEASE = 1 << 17
479 //
480 // HLE is an optimization, and we pretend that elision always fails.
481 // MEMMODEL_SYNC is used when lowering __sync_ atomics,
482 // since we use __sync_ atomics for actual atomic operations,
483 // we can safely ignore it as well. It also subtly affects semantics,
484 // but we don't model the difference.
485 return (morder)(mo & 0x7fff);
486 }
487
488 # define ATOMIC_IMPL(func, ...) \
489 ThreadState *const thr = cur_thread(); \
490 ProcessPendingSignals(thr); \
491 if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) \
492 return NoTsanAtomic##func(__VA_ARGS__); \
493 mo = convert_morder(mo); \
494 return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__);
495
496 extern "C" {
497 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_load(const volatile a8 * a,morder mo)498 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
499 ATOMIC_IMPL(Load, a, mo);
500 }
501
502 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_load(const volatile a16 * a,morder mo)503 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
504 ATOMIC_IMPL(Load, a, mo);
505 }
506
507 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_load(const volatile a32 * a,morder mo)508 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
509 ATOMIC_IMPL(Load, a, mo);
510 }
511
512 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_load(const volatile a64 * a,morder mo)513 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
514 ATOMIC_IMPL(Load, a, mo);
515 }
516
517 #if __TSAN_HAS_INT128
518 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_load(const volatile a128 * a,morder mo)519 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
520 ATOMIC_IMPL(Load, a, mo);
521 }
522 #endif
523
524 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_store(volatile a8 * a,a8 v,morder mo)525 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
526 ATOMIC_IMPL(Store, a, v, mo);
527 }
528
529 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_store(volatile a16 * a,a16 v,morder mo)530 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
531 ATOMIC_IMPL(Store, a, v, mo);
532 }
533
534 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_store(volatile a32 * a,a32 v,morder mo)535 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
536 ATOMIC_IMPL(Store, a, v, mo);
537 }
538
539 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_store(volatile a64 * a,a64 v,morder mo)540 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
541 ATOMIC_IMPL(Store, a, v, mo);
542 }
543
544 #if __TSAN_HAS_INT128
545 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_store(volatile a128 * a,a128 v,morder mo)546 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
547 ATOMIC_IMPL(Store, a, v, mo);
548 }
549 #endif
550
551 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_exchange(volatile a8 * a,a8 v,morder mo)552 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
553 ATOMIC_IMPL(Exchange, a, v, mo);
554 }
555
556 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_exchange(volatile a16 * a,a16 v,morder mo)557 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
558 ATOMIC_IMPL(Exchange, a, v, mo);
559 }
560
561 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_exchange(volatile a32 * a,a32 v,morder mo)562 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
563 ATOMIC_IMPL(Exchange, a, v, mo);
564 }
565
566 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_exchange(volatile a64 * a,a64 v,morder mo)567 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
568 ATOMIC_IMPL(Exchange, a, v, mo);
569 }
570
571 #if __TSAN_HAS_INT128
572 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_exchange(volatile a128 * a,a128 v,morder mo)573 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
574 ATOMIC_IMPL(Exchange, a, v, mo);
575 }
576 #endif
577
578 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_add(volatile a8 * a,a8 v,morder mo)579 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
580 ATOMIC_IMPL(FetchAdd, a, v, mo);
581 }
582
583 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_add(volatile a16 * a,a16 v,morder mo)584 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
585 ATOMIC_IMPL(FetchAdd, a, v, mo);
586 }
587
588 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_add(volatile a32 * a,a32 v,morder mo)589 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
590 ATOMIC_IMPL(FetchAdd, a, v, mo);
591 }
592
593 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_add(volatile a64 * a,a64 v,morder mo)594 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
595 ATOMIC_IMPL(FetchAdd, a, v, mo);
596 }
597
598 #if __TSAN_HAS_INT128
599 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_add(volatile a128 * a,a128 v,morder mo)600 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
601 ATOMIC_IMPL(FetchAdd, a, v, mo);
602 }
603 #endif
604
605 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_sub(volatile a8 * a,a8 v,morder mo)606 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
607 ATOMIC_IMPL(FetchSub, a, v, mo);
608 }
609
610 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_sub(volatile a16 * a,a16 v,morder mo)611 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
612 ATOMIC_IMPL(FetchSub, a, v, mo);
613 }
614
615 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_sub(volatile a32 * a,a32 v,morder mo)616 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
617 ATOMIC_IMPL(FetchSub, a, v, mo);
618 }
619
620 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_sub(volatile a64 * a,a64 v,morder mo)621 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
622 ATOMIC_IMPL(FetchSub, a, v, mo);
623 }
624
625 #if __TSAN_HAS_INT128
626 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_sub(volatile a128 * a,a128 v,morder mo)627 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
628 ATOMIC_IMPL(FetchSub, a, v, mo);
629 }
630 #endif
631
632 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_and(volatile a8 * a,a8 v,morder mo)633 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
634 ATOMIC_IMPL(FetchAnd, a, v, mo);
635 }
636
637 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_and(volatile a16 * a,a16 v,morder mo)638 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
639 ATOMIC_IMPL(FetchAnd, a, v, mo);
640 }
641
642 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_and(volatile a32 * a,a32 v,morder mo)643 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
644 ATOMIC_IMPL(FetchAnd, a, v, mo);
645 }
646
647 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_and(volatile a64 * a,a64 v,morder mo)648 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
649 ATOMIC_IMPL(FetchAnd, a, v, mo);
650 }
651
652 #if __TSAN_HAS_INT128
653 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_and(volatile a128 * a,a128 v,morder mo)654 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
655 ATOMIC_IMPL(FetchAnd, a, v, mo);
656 }
657 #endif
658
659 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_or(volatile a8 * a,a8 v,morder mo)660 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
661 ATOMIC_IMPL(FetchOr, a, v, mo);
662 }
663
664 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_or(volatile a16 * a,a16 v,morder mo)665 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
666 ATOMIC_IMPL(FetchOr, a, v, mo);
667 }
668
669 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_or(volatile a32 * a,a32 v,morder mo)670 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
671 ATOMIC_IMPL(FetchOr, a, v, mo);
672 }
673
674 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_or(volatile a64 * a,a64 v,morder mo)675 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
676 ATOMIC_IMPL(FetchOr, a, v, mo);
677 }
678
679 #if __TSAN_HAS_INT128
680 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_or(volatile a128 * a,a128 v,morder mo)681 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
682 ATOMIC_IMPL(FetchOr, a, v, mo);
683 }
684 #endif
685
686 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_xor(volatile a8 * a,a8 v,morder mo)687 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
688 ATOMIC_IMPL(FetchXor, a, v, mo);
689 }
690
691 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_xor(volatile a16 * a,a16 v,morder mo)692 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
693 ATOMIC_IMPL(FetchXor, a, v, mo);
694 }
695
696 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_xor(volatile a32 * a,a32 v,morder mo)697 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
698 ATOMIC_IMPL(FetchXor, a, v, mo);
699 }
700
701 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_xor(volatile a64 * a,a64 v,morder mo)702 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
703 ATOMIC_IMPL(FetchXor, a, v, mo);
704 }
705
706 #if __TSAN_HAS_INT128
707 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_xor(volatile a128 * a,a128 v,morder mo)708 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
709 ATOMIC_IMPL(FetchXor, a, v, mo);
710 }
711 #endif
712
713 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_nand(volatile a8 * a,a8 v,morder mo)714 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
715 ATOMIC_IMPL(FetchNand, a, v, mo);
716 }
717
718 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_nand(volatile a16 * a,a16 v,morder mo)719 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
720 ATOMIC_IMPL(FetchNand, a, v, mo);
721 }
722
723 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_nand(volatile a32 * a,a32 v,morder mo)724 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
725 ATOMIC_IMPL(FetchNand, a, v, mo);
726 }
727
728 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_nand(volatile a64 * a,a64 v,morder mo)729 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
730 ATOMIC_IMPL(FetchNand, a, v, mo);
731 }
732
733 #if __TSAN_HAS_INT128
734 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_nand(volatile a128 * a,a128 v,morder mo)735 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
736 ATOMIC_IMPL(FetchNand, a, v, mo);
737 }
738 #endif
739
740 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_compare_exchange_strong(volatile a8 * a,a8 * c,a8 v,morder mo,morder fmo)741 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
742 morder mo, morder fmo) {
743 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
744 }
745
746 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_compare_exchange_strong(volatile a16 * a,a16 * c,a16 v,morder mo,morder fmo)747 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
748 morder mo, morder fmo) {
749 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
750 }
751
752 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_compare_exchange_strong(volatile a32 * a,a32 * c,a32 v,morder mo,morder fmo)753 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
754 morder mo, morder fmo) {
755 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
756 }
757
758 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_compare_exchange_strong(volatile a64 * a,a64 * c,a64 v,morder mo,morder fmo)759 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
760 morder mo, morder fmo) {
761 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
762 }
763
764 #if __TSAN_HAS_INT128
765 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_compare_exchange_strong(volatile a128 * a,a128 * c,a128 v,morder mo,morder fmo)766 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
767 morder mo, morder fmo) {
768 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
769 }
770 #endif
771
772 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_compare_exchange_weak(volatile a8 * a,a8 * c,a8 v,morder mo,morder fmo)773 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
774 morder mo, morder fmo) {
775 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
776 }
777
778 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_compare_exchange_weak(volatile a16 * a,a16 * c,a16 v,morder mo,morder fmo)779 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
780 morder mo, morder fmo) {
781 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
782 }
783
784 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_compare_exchange_weak(volatile a32 * a,a32 * c,a32 v,morder mo,morder fmo)785 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
786 morder mo, morder fmo) {
787 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
788 }
789
790 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_compare_exchange_weak(volatile a64 * a,a64 * c,a64 v,morder mo,morder fmo)791 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
792 morder mo, morder fmo) {
793 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
794 }
795
796 #if __TSAN_HAS_INT128
797 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_compare_exchange_weak(volatile a128 * a,a128 * c,a128 v,morder mo,morder fmo)798 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
799 morder mo, morder fmo) {
800 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
801 }
802 #endif
803
804 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_compare_exchange_val(volatile a8 * a,a8 c,a8 v,morder mo,morder fmo)805 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
806 morder mo, morder fmo) {
807 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
808 }
809
810 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_compare_exchange_val(volatile a16 * a,a16 c,a16 v,morder mo,morder fmo)811 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
812 morder mo, morder fmo) {
813 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
814 }
815
816 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_compare_exchange_val(volatile a32 * a,a32 c,a32 v,morder mo,morder fmo)817 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
818 morder mo, morder fmo) {
819 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
820 }
821
822 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_compare_exchange_val(volatile a64 * a,a64 c,a64 v,morder mo,morder fmo)823 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
824 morder mo, morder fmo) {
825 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
826 }
827
828 #if __TSAN_HAS_INT128
829 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_compare_exchange_val(volatile a128 * a,a128 c,a128 v,morder mo,morder fmo)830 a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
831 morder mo, morder fmo) {
832 ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
833 }
834 #endif
835
836 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic_thread_fence(morder mo)837 void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); }
838
839 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic_signal_fence(morder mo)840 void __tsan_atomic_signal_fence(morder mo) {
841 }
842 } // extern "C"
843
844 #else // #if !SANITIZER_GO
845
846 // Go
847
848 # define ATOMIC(func, ...) \
849 if (thr->ignore_sync) { \
850 NoTsanAtomic##func(__VA_ARGS__); \
851 } else { \
852 FuncEntry(thr, cpc); \
853 Atomic##func(thr, pc, __VA_ARGS__); \
854 FuncExit(thr); \
855 }
856
857 # define ATOMIC_RET(func, ret, ...) \
858 if (thr->ignore_sync) { \
859 (ret) = NoTsanAtomic##func(__VA_ARGS__); \
860 } else { \
861 FuncEntry(thr, cpc); \
862 (ret) = Atomic##func(thr, pc, __VA_ARGS__); \
863 FuncExit(thr); \
864 }
865
866 extern "C" {
867 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_load(ThreadState * thr,uptr cpc,uptr pc,u8 * a)868 void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
869 ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire);
870 }
871
872 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_load(ThreadState * thr,uptr cpc,uptr pc,u8 * a)873 void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
874 ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire);
875 }
876
877 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_store(ThreadState * thr,uptr cpc,uptr pc,u8 * a)878 void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
879 ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release);
880 }
881
882 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_store(ThreadState * thr,uptr cpc,uptr pc,u8 * a)883 void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
884 ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release);
885 }
886
887 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_fetch_add(ThreadState * thr,uptr cpc,uptr pc,u8 * a)888 void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
889 ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
890 }
891
892 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_fetch_add(ThreadState * thr,uptr cpc,uptr pc,u8 * a)893 void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
894 ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
895 }
896
897 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_exchange(ThreadState * thr,uptr cpc,uptr pc,u8 * a)898 void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
899 ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
900 }
901
902 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_exchange(ThreadState * thr,uptr cpc,uptr pc,u8 * a)903 void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
904 ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
905 }
906
907 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_compare_exchange(ThreadState * thr,uptr cpc,uptr pc,u8 * a)908 void __tsan_go_atomic32_compare_exchange(
909 ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
910 a32 cur = 0;
911 a32 cmp = *(a32*)(a+8);
912 ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire);
913 *(bool*)(a+16) = (cur == cmp);
914 }
915
916 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_compare_exchange(ThreadState * thr,uptr cpc,uptr pc,u8 * a)917 void __tsan_go_atomic64_compare_exchange(
918 ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
919 a64 cur = 0;
920 a64 cmp = *(a64*)(a+8);
921 ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire);
922 *(bool*)(a+24) = (cur == cmp);
923 }
924 } // extern "C"
925 #endif // #if !SANITIZER_GO
926