1 //===-- tsan_interceptors_mac.cpp -----------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Mac-specific interceptors. 12 //===----------------------------------------------------------------------===// 13 14 #include "sanitizer_common/sanitizer_platform.h" 15 #if SANITIZER_MAC 16 17 #include "interception/interception.h" 18 #include "tsan_interceptors.h" 19 #include "tsan_interface.h" 20 #include "tsan_interface_ann.h" 21 #include "sanitizer_common/sanitizer_addrhashmap.h" 22 23 #include <errno.h> 24 #include <libkern/OSAtomic.h> 25 #include <objc/objc-sync.h> 26 #include <os/lock.h> 27 #include <sys/ucontext.h> 28 29 #if defined(__has_include) && __has_include(<xpc/xpc.h>) 30 #include <xpc/xpc.h> 31 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 32 33 typedef long long_t; 34 35 extern "C" { 36 int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); 37 int setcontext(const ucontext_t *ucp); 38 } 39 40 namespace __tsan { 41 42 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, 43 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are 44 // actually aliases of each other, and we cannot have different interceptors for 45 // them, because they're actually the same function. Thus, we have to stay 46 // conservative and treat the non-barrier versions as mo_acq_rel. 47 static constexpr morder kMacOrderBarrier = mo_acq_rel; 48 static constexpr morder kMacOrderNonBarrier = mo_acq_rel; 49 static constexpr morder kMacFailureOrder = mo_relaxed; 50 51 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 52 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 53 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 54 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ 55 } 56 57 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 58 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 59 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 60 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ 61 } 62 63 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 64 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 65 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 66 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ 67 } 68 69 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ 70 mo) \ 71 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 72 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 73 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ 74 } 75 76 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ 77 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 78 kMacOrderNonBarrier) \ 79 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 80 kMacOrderBarrier) \ 81 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ 82 kMacOrderNonBarrier) \ 83 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \ 84 kMacOrderBarrier) 85 86 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ 87 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 88 kMacOrderNonBarrier) \ 89 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 90 kMacOrderBarrier) \ 91 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \ 92 kMacOrderNonBarrier) \ 93 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ 94 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) 95 96 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, 97 OSATOMIC_INTERCEPTOR_PLUS_X) 98 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, 99 OSATOMIC_INTERCEPTOR_PLUS_1) 100 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub, 101 OSATOMIC_INTERCEPTOR_MINUS_1) 102 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X, 103 OSATOMIC_INTERCEPTOR) 104 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, 105 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 106 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, 107 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 108 109 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ 110 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ 111 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ 112 return tsan_atomic_f##_compare_exchange_strong( \ 113 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 114 kMacOrderNonBarrier, kMacFailureOrder); \ 115 } \ 116 \ 117 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ 118 t volatile *ptr) { \ 119 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ 120 return tsan_atomic_f##_compare_exchange_strong( \ 121 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 122 kMacOrderBarrier, kMacFailureOrder); \ 123 } 124 125 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) 126 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, 127 long_t) 128 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64, 129 void *) 130 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, 131 int32_t) 132 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, 133 int64_t) 134 135 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ 136 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ 137 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ 138 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ 139 char bit = 0x80u >> (n & 7); \ 140 char mask = clear ? ~bit : bit; \ 141 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ 142 return orig_byte & bit; \ 143 } 144 145 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ 146 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ 147 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) 148 149 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false) 150 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, 151 true) 152 153 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, 154 size_t offset) { 155 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset); 156 __tsan_release(item); 157 REAL(OSAtomicEnqueue)(list, item, offset); 158 } 159 160 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { 161 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); 162 void *item = REAL(OSAtomicDequeue)(list, offset); 163 if (item) __tsan_acquire(item); 164 return item; 165 } 166 167 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. 168 #if !SANITIZER_IOS 169 170 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, 171 size_t offset) { 172 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset); 173 __tsan_release(item); 174 REAL(OSAtomicFifoEnqueue)(list, item, offset); 175 } 176 177 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, 178 size_t offset) { 179 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); 180 void *item = REAL(OSAtomicFifoDequeue)(list, offset); 181 if (item) __tsan_acquire(item); 182 return item; 183 } 184 185 #endif 186 187 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { 188 CHECK(!cur_thread()->is_dead); 189 if (!cur_thread()->is_inited) { 190 return REAL(OSSpinLockLock)(lock); 191 } 192 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); 193 REAL(OSSpinLockLock)(lock); 194 Acquire(thr, pc, (uptr)lock); 195 } 196 197 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { 198 CHECK(!cur_thread()->is_dead); 199 if (!cur_thread()->is_inited) { 200 return REAL(OSSpinLockTry)(lock); 201 } 202 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); 203 bool result = REAL(OSSpinLockTry)(lock); 204 if (result) 205 Acquire(thr, pc, (uptr)lock); 206 return result; 207 } 208 209 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { 210 CHECK(!cur_thread()->is_dead); 211 if (!cur_thread()->is_inited) { 212 return REAL(OSSpinLockUnlock)(lock); 213 } 214 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); 215 Release(thr, pc, (uptr)lock); 216 REAL(OSSpinLockUnlock)(lock); 217 } 218 219 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { 220 CHECK(!cur_thread()->is_dead); 221 if (!cur_thread()->is_inited) { 222 return REAL(os_lock_lock)(lock); 223 } 224 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); 225 REAL(os_lock_lock)(lock); 226 Acquire(thr, pc, (uptr)lock); 227 } 228 229 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { 230 CHECK(!cur_thread()->is_dead); 231 if (!cur_thread()->is_inited) { 232 return REAL(os_lock_trylock)(lock); 233 } 234 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); 235 bool result = REAL(os_lock_trylock)(lock); 236 if (result) 237 Acquire(thr, pc, (uptr)lock); 238 return result; 239 } 240 241 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { 242 CHECK(!cur_thread()->is_dead); 243 if (!cur_thread()->is_inited) { 244 return REAL(os_lock_unlock)(lock); 245 } 246 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); 247 Release(thr, pc, (uptr)lock); 248 REAL(os_lock_unlock)(lock); 249 } 250 251 TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) { 252 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 253 return REAL(os_unfair_lock_lock)(lock); 254 } 255 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock); 256 REAL(os_unfair_lock_lock)(lock); 257 Acquire(thr, pc, (uptr)lock); 258 } 259 260 TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock, 261 u32 options) { 262 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 263 return REAL(os_unfair_lock_lock_with_options)(lock, options); 264 } 265 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options); 266 REAL(os_unfair_lock_lock_with_options)(lock, options); 267 Acquire(thr, pc, (uptr)lock); 268 } 269 270 TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) { 271 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 272 return REAL(os_unfair_lock_trylock)(lock); 273 } 274 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock); 275 bool result = REAL(os_unfair_lock_trylock)(lock); 276 if (result) 277 Acquire(thr, pc, (uptr)lock); 278 return result; 279 } 280 281 TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) { 282 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 283 return REAL(os_unfair_lock_unlock)(lock); 284 } 285 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock); 286 Release(thr, pc, (uptr)lock); 287 REAL(os_unfair_lock_unlock)(lock); 288 } 289 290 #if defined(__has_include) && __has_include(<xpc/xpc.h>) 291 292 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, 293 xpc_connection_t connection, xpc_handler_t handler) { 294 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, 295 handler); 296 Release(thr, pc, (uptr)connection); 297 xpc_handler_t new_handler = ^(xpc_object_t object) { 298 { 299 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler); 300 Acquire(thr, pc, (uptr)connection); 301 } 302 handler(object); 303 }; 304 REAL(xpc_connection_set_event_handler)(connection, new_handler); 305 } 306 307 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection, 308 dispatch_block_t barrier) { 309 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier); 310 Release(thr, pc, (uptr)connection); 311 dispatch_block_t new_barrier = ^() { 312 { 313 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier); 314 Acquire(thr, pc, (uptr)connection); 315 } 316 barrier(); 317 }; 318 REAL(xpc_connection_send_barrier)(connection, new_barrier); 319 } 320 321 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply, 322 xpc_connection_t connection, xpc_object_t message, 323 dispatch_queue_t replyq, xpc_handler_t handler) { 324 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection, 325 message, replyq, handler); 326 Release(thr, pc, (uptr)connection); 327 xpc_handler_t new_handler = ^(xpc_object_t object) { 328 { 329 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply); 330 Acquire(thr, pc, (uptr)connection); 331 } 332 handler(object); 333 }; 334 REAL(xpc_connection_send_message_with_reply) 335 (connection, message, replyq, new_handler); 336 } 337 338 TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { 339 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection); 340 Release(thr, pc, (uptr)connection); 341 REAL(xpc_connection_cancel)(connection); 342 } 343 344 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 345 346 // Determines whether the Obj-C object pointer is a tagged pointer. Tagged 347 // pointers encode the object data directly in their pointer bits and do not 348 // have an associated memory allocation. The Obj-C runtime uses tagged pointers 349 // to transparently optimize small objects. 350 static bool IsTaggedObjCPointer(id obj) { 351 const uptr kPossibleTaggedBits = 0x8000000000000001ull; 352 return ((uptr)obj & kPossibleTaggedBits) != 0; 353 } 354 355 // Returns an address which can be used to inform TSan about synchronization 356 // points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid 357 // address in the process space. We do a small allocation here to obtain a 358 // stable address (the array backing the hash map can change). The memory is 359 // never free'd (leaked) and allocation and locking are slow, but this code only 360 // runs for @synchronized with tagged pointers, which is very rare. 361 static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) { 362 typedef AddrHashMap<uptr, 5> Map; 363 static Map Addresses; 364 Map::Handle h(&Addresses, addr); 365 if (h.created()) { 366 ThreadIgnoreBegin(thr, pc); 367 *h = (uptr) user_alloc(thr, pc, /*size=*/1); 368 ThreadIgnoreEnd(thr); 369 } 370 return *h; 371 } 372 373 // Returns an address on which we can synchronize given an Obj-C object pointer. 374 // For normal object pointers, this is just the address of the object in memory. 375 // Tagged pointers are not backed by an actual memory allocation, so we need to 376 // synthesize a valid address. 377 static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) { 378 if (IsTaggedObjCPointer(obj)) 379 return GetOrCreateSyncAddress((uptr)obj, thr, pc); 380 return (uptr)obj; 381 } 382 383 TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { 384 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); 385 if (!obj) return REAL(objc_sync_enter)(obj); 386 uptr addr = SyncAddressForObjCObject(obj, thr, pc); 387 MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant); 388 int result = REAL(objc_sync_enter)(obj); 389 CHECK_EQ(result, OBJC_SYNC_SUCCESS); 390 MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant); 391 return result; 392 } 393 394 TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { 395 SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj); 396 if (!obj) return REAL(objc_sync_exit)(obj); 397 uptr addr = SyncAddressForObjCObject(obj, thr, pc); 398 MutexUnlock(thr, pc, addr); 399 int result = REAL(objc_sync_exit)(obj); 400 if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr); 401 return result; 402 } 403 404 TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { 405 { 406 SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); 407 } 408 // Because of swapcontext() semantics we have no option but to copy its 409 // implementation here 410 if (!oucp || !ucp) { 411 errno = EINVAL; 412 return -1; 413 } 414 ThreadState *thr = cur_thread(); 415 const int UCF_SWAPPED = 0x80000000; 416 oucp->uc_onstack &= ~UCF_SWAPPED; 417 thr->ignore_interceptors++; 418 int ret = getcontext(oucp); 419 if (!(oucp->uc_onstack & UCF_SWAPPED)) { 420 thr->ignore_interceptors--; 421 if (!ret) { 422 oucp->uc_onstack |= UCF_SWAPPED; 423 ret = setcontext(ucp); 424 } 425 } 426 return ret; 427 } 428 429 // On macOS, libc++ is always linked dynamically, so intercepting works the 430 // usual way. 431 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR 432 433 namespace { 434 struct fake_shared_weak_count { 435 volatile a64 shared_owners; 436 volatile a64 shared_weak_owners; 437 virtual void _unused_0x0() = 0; 438 virtual void _unused_0x8() = 0; 439 virtual void on_zero_shared() = 0; 440 virtual void _unused_0x18() = 0; 441 virtual void on_zero_shared_weak() = 0; 442 virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor 443 }; 444 } // namespace 445 446 // The following code adds libc++ interceptors for: 447 // void __shared_weak_count::__release_shared() _NOEXCEPT; 448 // bool __shared_count::__release_shared() _NOEXCEPT; 449 // Shared and weak pointers in C++ maintain reference counts via atomics in 450 // libc++.dylib, which are TSan-invisible, and this leads to false positives in 451 // destructor code. These interceptors re-implements the whole functions so that 452 // the mo_acq_rel semantics of the atomic decrement are visible. 453 // 454 // Unfortunately, the interceptors cannot simply Acquire/Release some sync 455 // object and call the original function, because it would have a race between 456 // the sync and the destruction of the object. Calling both under a lock will 457 // not work because the destructor can invoke this interceptor again (and even 458 // in a different thread, so recursive locks don't help). 459 460 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv, 461 fake_shared_weak_count *o) { 462 if (!flags()->shared_ptr_interceptor) 463 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o); 464 465 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv, 466 o); 467 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 468 Acquire(thr, pc, (uptr)&o->shared_owners); 469 o->on_zero_shared(); 470 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) == 471 0) { 472 Acquire(thr, pc, (uptr)&o->shared_weak_owners); 473 o->on_zero_shared_weak(); 474 } 475 } 476 } 477 478 STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv, 479 fake_shared_weak_count *o) { 480 if (!flags()->shared_ptr_interceptor) 481 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o); 482 483 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o); 484 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 485 Acquire(thr, pc, (uptr)&o->shared_owners); 486 o->on_zero_shared(); 487 return true; 488 } 489 return false; 490 } 491 492 namespace { 493 struct call_once_callback_args { 494 void (*orig_func)(void *arg); 495 void *orig_arg; 496 void *flag; 497 }; 498 499 void call_once_callback_wrapper(void *arg) { 500 call_once_callback_args *new_args = (call_once_callback_args *)arg; 501 new_args->orig_func(new_args->orig_arg); 502 __tsan_release(new_args->flag); 503 } 504 } // namespace 505 506 // This adds a libc++ interceptor for: 507 // void __call_once(volatile unsigned long&, void*, void(*)(void*)); 508 // C++11 call_once is implemented via an internal function __call_once which is 509 // inside libc++.dylib, and the atomic release store inside it is thus 510 // TSan-invisible. To avoid false positives, this interceptor wraps the callback 511 // function and performs an explicit Release after the user code has run. 512 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag, 513 void *arg, void (*func)(void *arg)) { 514 call_once_callback_args new_args = {func, arg, flag}; 515 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args, 516 call_once_callback_wrapper); 517 } 518 519 } // namespace __tsan 520 521 #endif // SANITIZER_MAC 522