1 //===-- tsan_interceptors_mac.cpp -----------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Mac-specific interceptors. 12 //===----------------------------------------------------------------------===// 13 14 #include "sanitizer_common/sanitizer_platform.h" 15 #if SANITIZER_MAC 16 17 #include "interception/interception.h" 18 #include "tsan_interceptors.h" 19 #include "tsan_interface.h" 20 #include "tsan_interface_ann.h" 21 #include "sanitizer_common/sanitizer_addrhashmap.h" 22 23 #include <errno.h> 24 #include <libkern/OSAtomic.h> 25 #include <objc/objc-sync.h> 26 #include <os/lock.h> 27 #include <sys/ucontext.h> 28 29 #if defined(__has_include) && __has_include(<xpc/xpc.h>) 30 #include <xpc/xpc.h> 31 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 32 33 typedef long long_t; 34 35 extern "C" { 36 int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); 37 int setcontext(const ucontext_t *ucp); 38 } 39 40 namespace __tsan { 41 42 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, 43 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are 44 // actually aliases of each other, and we cannot have different interceptors for 45 // them, because they're actually the same function. Thus, we have to stay 46 // conservative and treat the non-barrier versions as mo_acq_rel. 47 static const morder kMacOrderBarrier = mo_acq_rel; 48 static const morder kMacOrderNonBarrier = mo_acq_rel; 49 50 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 51 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 52 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 53 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ 54 } 55 56 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 57 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 58 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 59 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ 60 } 61 62 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 63 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 64 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 65 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ 66 } 67 68 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ 69 mo) \ 70 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 71 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 72 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ 73 } 74 75 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ 76 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 77 kMacOrderNonBarrier) \ 78 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 79 kMacOrderBarrier) \ 80 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ 81 kMacOrderNonBarrier) \ 82 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \ 83 kMacOrderBarrier) 84 85 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ 86 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 87 kMacOrderNonBarrier) \ 88 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 89 kMacOrderBarrier) \ 90 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \ 91 kMacOrderNonBarrier) \ 92 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ 93 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) 94 95 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, 96 OSATOMIC_INTERCEPTOR_PLUS_X) 97 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, 98 OSATOMIC_INTERCEPTOR_PLUS_1) 99 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub, 100 OSATOMIC_INTERCEPTOR_MINUS_1) 101 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X, 102 OSATOMIC_INTERCEPTOR) 103 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, 104 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 105 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, 106 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 107 108 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ 109 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ 110 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ 111 return tsan_atomic_f##_compare_exchange_strong( \ 112 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 113 kMacOrderNonBarrier, kMacOrderNonBarrier); \ 114 } \ 115 \ 116 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ 117 t volatile *ptr) { \ 118 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ 119 return tsan_atomic_f##_compare_exchange_strong( \ 120 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 121 kMacOrderBarrier, kMacOrderNonBarrier); \ 122 } 123 124 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) 125 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, 126 long_t) 127 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64, 128 void *) 129 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, 130 int32_t) 131 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, 132 int64_t) 133 134 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ 135 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ 136 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ 137 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ 138 char bit = 0x80u >> (n & 7); \ 139 char mask = clear ? ~bit : bit; \ 140 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ 141 return orig_byte & bit; \ 142 } 143 144 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ 145 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ 146 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) 147 148 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false) 149 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, 150 true) 151 152 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, 153 size_t offset) { 154 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset); 155 __tsan_release(item); 156 REAL(OSAtomicEnqueue)(list, item, offset); 157 } 158 159 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { 160 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); 161 void *item = REAL(OSAtomicDequeue)(list, offset); 162 if (item) __tsan_acquire(item); 163 return item; 164 } 165 166 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. 167 #if !SANITIZER_IOS 168 169 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, 170 size_t offset) { 171 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset); 172 __tsan_release(item); 173 REAL(OSAtomicFifoEnqueue)(list, item, offset); 174 } 175 176 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, 177 size_t offset) { 178 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); 179 void *item = REAL(OSAtomicFifoDequeue)(list, offset); 180 if (item) __tsan_acquire(item); 181 return item; 182 } 183 184 #endif 185 186 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { 187 CHECK(!cur_thread()->is_dead); 188 if (!cur_thread()->is_inited) { 189 return REAL(OSSpinLockLock)(lock); 190 } 191 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); 192 REAL(OSSpinLockLock)(lock); 193 Acquire(thr, pc, (uptr)lock); 194 } 195 196 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { 197 CHECK(!cur_thread()->is_dead); 198 if (!cur_thread()->is_inited) { 199 return REAL(OSSpinLockTry)(lock); 200 } 201 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); 202 bool result = REAL(OSSpinLockTry)(lock); 203 if (result) 204 Acquire(thr, pc, (uptr)lock); 205 return result; 206 } 207 208 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { 209 CHECK(!cur_thread()->is_dead); 210 if (!cur_thread()->is_inited) { 211 return REAL(OSSpinLockUnlock)(lock); 212 } 213 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); 214 Release(thr, pc, (uptr)lock); 215 REAL(OSSpinLockUnlock)(lock); 216 } 217 218 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { 219 CHECK(!cur_thread()->is_dead); 220 if (!cur_thread()->is_inited) { 221 return REAL(os_lock_lock)(lock); 222 } 223 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); 224 REAL(os_lock_lock)(lock); 225 Acquire(thr, pc, (uptr)lock); 226 } 227 228 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { 229 CHECK(!cur_thread()->is_dead); 230 if (!cur_thread()->is_inited) { 231 return REAL(os_lock_trylock)(lock); 232 } 233 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); 234 bool result = REAL(os_lock_trylock)(lock); 235 if (result) 236 Acquire(thr, pc, (uptr)lock); 237 return result; 238 } 239 240 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { 241 CHECK(!cur_thread()->is_dead); 242 if (!cur_thread()->is_inited) { 243 return REAL(os_lock_unlock)(lock); 244 } 245 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); 246 Release(thr, pc, (uptr)lock); 247 REAL(os_lock_unlock)(lock); 248 } 249 250 TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) { 251 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 252 return REAL(os_unfair_lock_lock)(lock); 253 } 254 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock); 255 REAL(os_unfair_lock_lock)(lock); 256 Acquire(thr, pc, (uptr)lock); 257 } 258 259 TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock, 260 u32 options) { 261 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 262 return REAL(os_unfair_lock_lock_with_options)(lock, options); 263 } 264 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options); 265 REAL(os_unfair_lock_lock_with_options)(lock, options); 266 Acquire(thr, pc, (uptr)lock); 267 } 268 269 TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) { 270 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 271 return REAL(os_unfair_lock_trylock)(lock); 272 } 273 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock); 274 bool result = REAL(os_unfair_lock_trylock)(lock); 275 if (result) 276 Acquire(thr, pc, (uptr)lock); 277 return result; 278 } 279 280 TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) { 281 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 282 return REAL(os_unfair_lock_unlock)(lock); 283 } 284 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock); 285 Release(thr, pc, (uptr)lock); 286 REAL(os_unfair_lock_unlock)(lock); 287 } 288 289 #if defined(__has_include) && __has_include(<xpc/xpc.h>) 290 291 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, 292 xpc_connection_t connection, xpc_handler_t handler) { 293 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, 294 handler); 295 Release(thr, pc, (uptr)connection); 296 xpc_handler_t new_handler = ^(xpc_object_t object) { 297 { 298 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler); 299 Acquire(thr, pc, (uptr)connection); 300 } 301 handler(object); 302 }; 303 REAL(xpc_connection_set_event_handler)(connection, new_handler); 304 } 305 306 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection, 307 dispatch_block_t barrier) { 308 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier); 309 Release(thr, pc, (uptr)connection); 310 dispatch_block_t new_barrier = ^() { 311 { 312 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier); 313 Acquire(thr, pc, (uptr)connection); 314 } 315 barrier(); 316 }; 317 REAL(xpc_connection_send_barrier)(connection, new_barrier); 318 } 319 320 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply, 321 xpc_connection_t connection, xpc_object_t message, 322 dispatch_queue_t replyq, xpc_handler_t handler) { 323 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection, 324 message, replyq, handler); 325 Release(thr, pc, (uptr)connection); 326 xpc_handler_t new_handler = ^(xpc_object_t object) { 327 { 328 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply); 329 Acquire(thr, pc, (uptr)connection); 330 } 331 handler(object); 332 }; 333 REAL(xpc_connection_send_message_with_reply) 334 (connection, message, replyq, new_handler); 335 } 336 337 TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { 338 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection); 339 Release(thr, pc, (uptr)connection); 340 REAL(xpc_connection_cancel)(connection); 341 } 342 343 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 344 345 // Determines whether the Obj-C object pointer is a tagged pointer. Tagged 346 // pointers encode the object data directly in their pointer bits and do not 347 // have an associated memory allocation. The Obj-C runtime uses tagged pointers 348 // to transparently optimize small objects. 349 static bool IsTaggedObjCPointer(id obj) { 350 const uptr kPossibleTaggedBits = 0x8000000000000001ull; 351 return ((uptr)obj & kPossibleTaggedBits) != 0; 352 } 353 354 // Returns an address which can be used to inform TSan about synchronization 355 // points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid 356 // address in the process space. We do a small allocation here to obtain a 357 // stable address (the array backing the hash map can change). The memory is 358 // never free'd (leaked) and allocation and locking are slow, but this code only 359 // runs for @synchronized with tagged pointers, which is very rare. 360 static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) { 361 typedef AddrHashMap<uptr, 5> Map; 362 static Map Addresses; 363 Map::Handle h(&Addresses, addr); 364 if (h.created()) { 365 ThreadIgnoreBegin(thr, pc); 366 *h = (uptr) user_alloc(thr, pc, /*size=*/1); 367 ThreadIgnoreEnd(thr, pc); 368 } 369 return *h; 370 } 371 372 // Returns an address on which we can synchronize given an Obj-C object pointer. 373 // For normal object pointers, this is just the address of the object in memory. 374 // Tagged pointers are not backed by an actual memory allocation, so we need to 375 // synthesize a valid address. 376 static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) { 377 if (IsTaggedObjCPointer(obj)) 378 return GetOrCreateSyncAddress((uptr)obj, thr, pc); 379 return (uptr)obj; 380 } 381 382 TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { 383 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); 384 if (!obj) return REAL(objc_sync_enter)(obj); 385 uptr addr = SyncAddressForObjCObject(obj, thr, pc); 386 MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant); 387 int result = REAL(objc_sync_enter)(obj); 388 CHECK_EQ(result, OBJC_SYNC_SUCCESS); 389 MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant); 390 return result; 391 } 392 393 TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { 394 SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj); 395 if (!obj) return REAL(objc_sync_exit)(obj); 396 uptr addr = SyncAddressForObjCObject(obj, thr, pc); 397 MutexUnlock(thr, pc, addr); 398 int result = REAL(objc_sync_exit)(obj); 399 if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr); 400 return result; 401 } 402 403 TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { 404 { 405 SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); 406 } 407 // Bacause of swapcontext() semantics we have no option but to copy its 408 // impementation here 409 if (!oucp || !ucp) { 410 errno = EINVAL; 411 return -1; 412 } 413 ThreadState *thr = cur_thread(); 414 const int UCF_SWAPPED = 0x80000000; 415 oucp->uc_onstack &= ~UCF_SWAPPED; 416 thr->ignore_interceptors++; 417 int ret = getcontext(oucp); 418 if (!(oucp->uc_onstack & UCF_SWAPPED)) { 419 thr->ignore_interceptors--; 420 if (!ret) { 421 oucp->uc_onstack |= UCF_SWAPPED; 422 ret = setcontext(ucp); 423 } 424 } 425 return ret; 426 } 427 428 // On macOS, libc++ is always linked dynamically, so intercepting works the 429 // usual way. 430 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR 431 432 namespace { 433 struct fake_shared_weak_count { 434 volatile a64 shared_owners; 435 volatile a64 shared_weak_owners; 436 virtual void _unused_0x0() = 0; 437 virtual void _unused_0x8() = 0; 438 virtual void on_zero_shared() = 0; 439 virtual void _unused_0x18() = 0; 440 virtual void on_zero_shared_weak() = 0; 441 virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor 442 }; 443 } // namespace 444 445 // The following code adds libc++ interceptors for: 446 // void __shared_weak_count::__release_shared() _NOEXCEPT; 447 // bool __shared_count::__release_shared() _NOEXCEPT; 448 // Shared and weak pointers in C++ maintain reference counts via atomics in 449 // libc++.dylib, which are TSan-invisible, and this leads to false positives in 450 // destructor code. These interceptors re-implements the whole functions so that 451 // the mo_acq_rel semantics of the atomic decrement are visible. 452 // 453 // Unfortunately, the interceptors cannot simply Acquire/Release some sync 454 // object and call the original function, because it would have a race between 455 // the sync and the destruction of the object. Calling both under a lock will 456 // not work because the destructor can invoke this interceptor again (and even 457 // in a different thread, so recursive locks don't help). 458 459 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv, 460 fake_shared_weak_count *o) { 461 if (!flags()->shared_ptr_interceptor) 462 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o); 463 464 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv, 465 o); 466 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 467 Acquire(thr, pc, (uptr)&o->shared_owners); 468 o->on_zero_shared(); 469 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) == 470 0) { 471 Acquire(thr, pc, (uptr)&o->shared_weak_owners); 472 o->on_zero_shared_weak(); 473 } 474 } 475 } 476 477 STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv, 478 fake_shared_weak_count *o) { 479 if (!flags()->shared_ptr_interceptor) 480 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o); 481 482 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o); 483 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 484 Acquire(thr, pc, (uptr)&o->shared_owners); 485 o->on_zero_shared(); 486 return true; 487 } 488 return false; 489 } 490 491 namespace { 492 struct call_once_callback_args { 493 void (*orig_func)(void *arg); 494 void *orig_arg; 495 void *flag; 496 }; 497 498 void call_once_callback_wrapper(void *arg) { 499 call_once_callback_args *new_args = (call_once_callback_args *)arg; 500 new_args->orig_func(new_args->orig_arg); 501 __tsan_release(new_args->flag); 502 } 503 } // namespace 504 505 // This adds a libc++ interceptor for: 506 // void __call_once(volatile unsigned long&, void*, void(*)(void*)); 507 // C++11 call_once is implemented via an internal function __call_once which is 508 // inside libc++.dylib, and the atomic release store inside it is thus 509 // TSan-invisible. To avoid false positives, this interceptor wraps the callback 510 // function and performs an explicit Release after the user code has run. 511 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag, 512 void *arg, void (*func)(void *arg)) { 513 call_once_callback_args new_args = {func, arg, flag}; 514 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args, 515 call_once_callback_wrapper); 516 } 517 518 } // namespace __tsan 519 520 #endif // SANITIZER_MAC 521