1 //===-- tsan_interceptors_posix.cpp ---------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // FIXME: move as many interceptors as possible into 12 // sanitizer_common/sanitizer_common_interceptors.inc 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_common/sanitizer_atomic.h" 16 #include "sanitizer_common/sanitizer_errno.h" 17 #include "sanitizer_common/sanitizer_libc.h" 18 #include "sanitizer_common/sanitizer_linux.h" 19 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h" 20 #include "sanitizer_common/sanitizer_platform_limits_posix.h" 21 #include "sanitizer_common/sanitizer_placement_new.h" 22 #include "sanitizer_common/sanitizer_posix.h" 23 #include "sanitizer_common/sanitizer_stacktrace.h" 24 #include "sanitizer_common/sanitizer_tls_get_addr.h" 25 #include "interception/interception.h" 26 #include "tsan_interceptors.h" 27 #include "tsan_interface.h" 28 #include "tsan_platform.h" 29 #include "tsan_suppressions.h" 30 #include "tsan_rtl.h" 31 #include "tsan_mman.h" 32 #include "tsan_fd.h" 33 34 using namespace __tsan; 35 36 #if SANITIZER_FREEBSD || SANITIZER_MAC 37 #define stdout __stdoutp 38 #define stderr __stderrp 39 #endif 40 41 #if SANITIZER_NETBSD 42 #define dirfd(dirp) (*(int *)(dirp)) 43 #define fileno_unlocked(fp) \ 44 (((__sanitizer_FILE *)fp)->_file == -1 \ 45 ? -1 \ 46 : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file)) 47 48 #define stdout ((__sanitizer_FILE*)&__sF[1]) 49 #define stderr ((__sanitizer_FILE*)&__sF[2]) 50 51 #define nanosleep __nanosleep50 52 #define vfork __vfork14 53 #endif 54 55 #if SANITIZER_ANDROID 56 #define mallopt(a, b) 57 #endif 58 59 #ifdef __mips__ 60 const int kSigCount = 129; 61 #else 62 const int kSigCount = 65; 63 #endif 64 65 #ifdef __mips__ 66 struct ucontext_t { 67 u64 opaque[768 / sizeof(u64) + 1]; 68 }; 69 #else 70 struct ucontext_t { 71 // The size is determined by looking at sizeof of real ucontext_t on linux. 72 u64 opaque[936 / sizeof(u64) + 1]; 73 }; 74 #endif 75 76 #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 77 #define PTHREAD_ABI_BASE "GLIBC_2.3.2" 78 #elif defined(__aarch64__) || SANITIZER_PPC64V2 79 #define PTHREAD_ABI_BASE "GLIBC_2.17" 80 #endif 81 82 extern "C" int pthread_attr_init(void *attr); 83 extern "C" int pthread_attr_destroy(void *attr); 84 DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *) 85 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); 86 extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); 87 extern "C" int pthread_setspecific(unsigned key, const void *v); 88 DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) 89 DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) 90 DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) 91 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) 92 extern "C" void *pthread_self(); 93 extern "C" void _exit(int status); 94 #if !SANITIZER_NETBSD 95 extern "C" int fileno_unlocked(void *stream); 96 extern "C" int dirfd(void *dirp); 97 #endif 98 #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD 99 extern "C" int mallopt(int param, int value); 100 #endif 101 #if SANITIZER_NETBSD 102 extern __sanitizer_FILE __sF[]; 103 #else 104 extern __sanitizer_FILE *stdout, *stderr; 105 #endif 106 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD 107 const int PTHREAD_MUTEX_RECURSIVE = 1; 108 const int PTHREAD_MUTEX_RECURSIVE_NP = 1; 109 #else 110 const int PTHREAD_MUTEX_RECURSIVE = 2; 111 const int PTHREAD_MUTEX_RECURSIVE_NP = 2; 112 #endif 113 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD 114 const int EPOLL_CTL_ADD = 1; 115 #endif 116 const int SIGILL = 4; 117 const int SIGTRAP = 5; 118 const int SIGABRT = 6; 119 const int SIGFPE = 8; 120 const int SIGSEGV = 11; 121 const int SIGPIPE = 13; 122 const int SIGTERM = 15; 123 #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD 124 const int SIGBUS = 10; 125 const int SIGSYS = 12; 126 #else 127 const int SIGBUS = 7; 128 const int SIGSYS = 31; 129 #endif 130 void *const MAP_FAILED = (void*)-1; 131 #if SANITIZER_NETBSD 132 const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; 133 #elif !SANITIZER_MAC 134 const int PTHREAD_BARRIER_SERIAL_THREAD = -1; 135 #endif 136 const int MAP_FIXED = 0x10; 137 typedef long long_t; 138 139 // From /usr/include/unistd.h 140 # define F_ULOCK 0 /* Unlock a previously locked region. */ 141 # define F_LOCK 1 /* Lock a region for exclusive use. */ 142 # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ 143 # define F_TEST 3 /* Test a region for other processes locks. */ 144 145 #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD 146 const int SA_SIGINFO = 0x40; 147 const int SIG_SETMASK = 3; 148 #elif defined(__mips__) 149 const int SA_SIGINFO = 8; 150 const int SIG_SETMASK = 3; 151 #else 152 const int SA_SIGINFO = 4; 153 const int SIG_SETMASK = 2; 154 #endif 155 156 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ 157 (cur_thread_init(), !cur_thread()->is_inited) 158 159 namespace __tsan { 160 struct SignalDesc { 161 bool armed; 162 bool sigaction; 163 __sanitizer_siginfo siginfo; 164 ucontext_t ctx; 165 }; 166 167 struct ThreadSignalContext { 168 int int_signal_send; 169 atomic_uintptr_t in_blocking_func; 170 atomic_uintptr_t have_pending_signals; 171 SignalDesc pending_signals[kSigCount]; 172 // emptyset and oldset are too big for stack. 173 __sanitizer_sigset_t emptyset; 174 __sanitizer_sigset_t oldset; 175 }; 176 177 // The sole reason tsan wraps atexit callbacks is to establish synchronization 178 // between callback setup and callback execution. 179 struct AtExitCtx { 180 void (*f)(); 181 void *arg; 182 }; 183 184 // InterceptorContext holds all global data required for interceptors. 185 // It's explicitly constructed in InitializeInterceptors with placement new 186 // and is never destroyed. This allows usage of members with non-trivial 187 // constructors and destructors. 188 struct InterceptorContext { 189 // The object is 64-byte aligned, because we want hot data to be located 190 // in a single cache line if possible (it's accessed in every interceptor). 191 ALIGNED(64) LibIgnore libignore; 192 __sanitizer_sigaction sigactions[kSigCount]; 193 #if !SANITIZER_MAC && !SANITIZER_NETBSD 194 unsigned finalize_key; 195 #endif 196 197 BlockingMutex atexit_mu; 198 Vector<struct AtExitCtx *> AtExitStack; 199 200 InterceptorContext() 201 : libignore(LINKER_INITIALIZED), AtExitStack() { 202 } 203 }; 204 205 static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)]; 206 InterceptorContext *interceptor_ctx() { 207 return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]); 208 } 209 210 LibIgnore *libignore() { 211 return &interceptor_ctx()->libignore; 212 } 213 214 void InitializeLibIgnore() { 215 const SuppressionContext &supp = *Suppressions(); 216 const uptr n = supp.SuppressionCount(); 217 for (uptr i = 0; i < n; i++) { 218 const Suppression *s = supp.SuppressionAt(i); 219 if (0 == internal_strcmp(s->type, kSuppressionLib)) 220 libignore()->AddIgnoredLibrary(s->templ); 221 } 222 if (flags()->ignore_noninstrumented_modules) 223 libignore()->IgnoreNoninstrumentedModules(true); 224 libignore()->OnLibraryLoaded(0); 225 } 226 227 // The following two hooks can be used by for cooperative scheduling when 228 // locking. 229 #ifdef TSAN_EXTERNAL_HOOKS 230 void OnPotentiallyBlockingRegionBegin(); 231 void OnPotentiallyBlockingRegionEnd(); 232 #else 233 SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {} 234 SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {} 235 #endif 236 237 } // namespace __tsan 238 239 static ThreadSignalContext *SigCtx(ThreadState *thr) { 240 ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx; 241 if (ctx == 0 && !thr->is_dead) { 242 ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext"); 243 MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); 244 thr->signal_ctx = ctx; 245 } 246 return ctx; 247 } 248 249 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, 250 uptr pc) 251 : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) { 252 Initialize(thr); 253 if (!thr_->is_inited) return; 254 if (!thr_->ignore_interceptors) FuncEntry(thr, pc); 255 DPrintf("#%d: intercept %s()\n", thr_->tid, fname); 256 ignoring_ = 257 !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || 258 libignore()->IsIgnored(pc, &in_ignored_lib_)); 259 EnableIgnores(); 260 } 261 262 ScopedInterceptor::~ScopedInterceptor() { 263 if (!thr_->is_inited) return; 264 DisableIgnores(); 265 if (!thr_->ignore_interceptors) { 266 ProcessPendingSignals(thr_); 267 FuncExit(thr_); 268 CheckNoLocks(thr_); 269 } 270 } 271 272 void ScopedInterceptor::EnableIgnores() { 273 if (ignoring_) { 274 ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false); 275 if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++; 276 if (in_ignored_lib_) { 277 DCHECK(!thr_->in_ignored_lib); 278 thr_->in_ignored_lib = true; 279 } 280 } 281 } 282 283 void ScopedInterceptor::DisableIgnores() { 284 if (ignoring_) { 285 ThreadIgnoreEnd(thr_, pc_); 286 if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--; 287 if (in_ignored_lib_) { 288 DCHECK(thr_->in_ignored_lib); 289 thr_->in_ignored_lib = false; 290 } 291 } 292 } 293 294 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) 295 #if SANITIZER_FREEBSD 296 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) 297 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) 298 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) 299 #elif SANITIZER_NETBSD 300 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) 301 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ 302 INTERCEPT_FUNCTION(__libc_##func) 303 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ 304 INTERCEPT_FUNCTION(__libc_thr_##func) 305 #else 306 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) 307 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) 308 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) 309 #endif 310 311 #define READ_STRING_OF_LEN(thr, pc, s, len, n) \ 312 MemoryAccessRange((thr), (pc), (uptr)(s), \ 313 common_flags()->strict_string_checks ? (len) + 1 : (n), false) 314 315 #define READ_STRING(thr, pc, s, n) \ 316 READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n)) 317 318 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) 319 320 struct BlockingCall { 321 explicit BlockingCall(ThreadState *thr) 322 : thr(thr) 323 , ctx(SigCtx(thr)) { 324 for (;;) { 325 atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed); 326 if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0) 327 break; 328 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 329 ProcessPendingSignals(thr); 330 } 331 // When we are in a "blocking call", we process signals asynchronously 332 // (right when they arrive). In this context we do not expect to be 333 // executing any user/runtime code. The known interceptor sequence when 334 // this is not true is: pthread_join -> munmap(stack). It's fine 335 // to ignore munmap in this case -- we handle stack shadow separately. 336 thr->ignore_interceptors++; 337 } 338 339 ~BlockingCall() { 340 thr->ignore_interceptors--; 341 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 342 } 343 344 ThreadState *thr; 345 ThreadSignalContext *ctx; 346 }; 347 348 TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { 349 SCOPED_TSAN_INTERCEPTOR(sleep, sec); 350 unsigned res = BLOCK_REAL(sleep)(sec); 351 AfterSleep(thr, pc); 352 return res; 353 } 354 355 TSAN_INTERCEPTOR(int, usleep, long_t usec) { 356 SCOPED_TSAN_INTERCEPTOR(usleep, usec); 357 int res = BLOCK_REAL(usleep)(usec); 358 AfterSleep(thr, pc); 359 return res; 360 } 361 362 TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { 363 SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); 364 int res = BLOCK_REAL(nanosleep)(req, rem); 365 AfterSleep(thr, pc); 366 return res; 367 } 368 369 TSAN_INTERCEPTOR(int, pause, int fake) { 370 SCOPED_TSAN_INTERCEPTOR(pause, fake); 371 return BLOCK_REAL(pause)(fake); 372 } 373 374 static void at_exit_wrapper() { 375 AtExitCtx *ctx; 376 { 377 // Ensure thread-safety. 378 BlockingMutexLock l(&interceptor_ctx()->atexit_mu); 379 380 // Pop AtExitCtx from the top of the stack of callback functions 381 uptr element = interceptor_ctx()->AtExitStack.Size() - 1; 382 ctx = interceptor_ctx()->AtExitStack[element]; 383 interceptor_ctx()->AtExitStack.PopBack(); 384 } 385 386 Acquire(cur_thread(), (uptr)0, (uptr)ctx); 387 ((void(*)())ctx->f)(); 388 InternalFree(ctx); 389 } 390 391 static void cxa_at_exit_wrapper(void *arg) { 392 Acquire(cur_thread(), 0, (uptr)arg); 393 AtExitCtx *ctx = (AtExitCtx*)arg; 394 ((void(*)(void *arg))ctx->f)(ctx->arg); 395 InternalFree(ctx); 396 } 397 398 static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), 399 void *arg, void *dso); 400 401 #if !SANITIZER_ANDROID 402 TSAN_INTERCEPTOR(int, atexit, void (*f)()) { 403 if (in_symbolizer()) 404 return 0; 405 // We want to setup the atexit callback even if we are in ignored lib 406 // or after fork. 407 SCOPED_INTERCEPTOR_RAW(atexit, f); 408 return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0); 409 } 410 #endif 411 412 TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { 413 if (in_symbolizer()) 414 return 0; 415 SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); 416 return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); 417 } 418 419 static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), 420 void *arg, void *dso) { 421 AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); 422 ctx->f = f; 423 ctx->arg = arg; 424 Release(thr, pc, (uptr)ctx); 425 // Memory allocation in __cxa_atexit will race with free during exit, 426 // because we do not see synchronization around atexit callback list. 427 ThreadIgnoreBegin(thr, pc); 428 int res; 429 if (!dso) { 430 // NetBSD does not preserve the 2nd argument if dso is equal to 0 431 // Store ctx in a local stack-like structure 432 433 // Ensure thread-safety. 434 BlockingMutexLock l(&interceptor_ctx()->atexit_mu); 435 436 res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0); 437 // Push AtExitCtx on the top of the stack of callback functions 438 if (!res) { 439 interceptor_ctx()->AtExitStack.PushBack(ctx); 440 } 441 } else { 442 res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso); 443 } 444 ThreadIgnoreEnd(thr, pc); 445 return res; 446 } 447 448 #if !SANITIZER_MAC && !SANITIZER_NETBSD 449 static void on_exit_wrapper(int status, void *arg) { 450 ThreadState *thr = cur_thread(); 451 uptr pc = 0; 452 Acquire(thr, pc, (uptr)arg); 453 AtExitCtx *ctx = (AtExitCtx*)arg; 454 ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); 455 InternalFree(ctx); 456 } 457 458 TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { 459 if (in_symbolizer()) 460 return 0; 461 SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); 462 AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); 463 ctx->f = (void(*)())f; 464 ctx->arg = arg; 465 Release(thr, pc, (uptr)ctx); 466 // Memory allocation in __cxa_atexit will race with free during exit, 467 // because we do not see synchronization around atexit callback list. 468 ThreadIgnoreBegin(thr, pc); 469 int res = REAL(on_exit)(on_exit_wrapper, ctx); 470 ThreadIgnoreEnd(thr, pc); 471 return res; 472 } 473 #define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit) 474 #else 475 #define TSAN_MAYBE_INTERCEPT_ON_EXIT 476 #endif 477 478 // Cleanup old bufs. 479 static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { 480 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 481 JmpBuf *buf = &thr->jmp_bufs[i]; 482 if (buf->sp <= sp) { 483 uptr sz = thr->jmp_bufs.Size(); 484 internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf)); 485 thr->jmp_bufs.PopBack(); 486 i--; 487 } 488 } 489 } 490 491 static void SetJmp(ThreadState *thr, uptr sp) { 492 if (!thr->is_inited) // called from libc guts during bootstrap 493 return; 494 // Cleanup old bufs. 495 JmpBufGarbageCollect(thr, sp); 496 // Remember the buf. 497 JmpBuf *buf = thr->jmp_bufs.PushBack(); 498 buf->sp = sp; 499 buf->shadow_stack_pos = thr->shadow_stack_pos; 500 ThreadSignalContext *sctx = SigCtx(thr); 501 buf->int_signal_send = sctx ? sctx->int_signal_send : 0; 502 buf->in_blocking_func = sctx ? 503 atomic_load(&sctx->in_blocking_func, memory_order_relaxed) : 504 false; 505 buf->in_signal_handler = atomic_load(&thr->in_signal_handler, 506 memory_order_relaxed); 507 } 508 509 static void LongJmp(ThreadState *thr, uptr *env) { 510 uptr sp = ExtractLongJmpSp(env); 511 // Find the saved buf with matching sp. 512 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 513 JmpBuf *buf = &thr->jmp_bufs[i]; 514 if (buf->sp == sp) { 515 CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); 516 // Unwind the stack. 517 while (thr->shadow_stack_pos > buf->shadow_stack_pos) 518 FuncExit(thr); 519 ThreadSignalContext *sctx = SigCtx(thr); 520 if (sctx) { 521 sctx->int_signal_send = buf->int_signal_send; 522 atomic_store(&sctx->in_blocking_func, buf->in_blocking_func, 523 memory_order_relaxed); 524 } 525 atomic_store(&thr->in_signal_handler, buf->in_signal_handler, 526 memory_order_relaxed); 527 JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp 528 return; 529 } 530 } 531 Printf("ThreadSanitizer: can't find longjmp buf\n"); 532 CHECK(0); 533 } 534 535 // FIXME: put everything below into a common extern "C" block? 536 extern "C" void __tsan_setjmp(uptr sp) { 537 cur_thread_init(); 538 SetJmp(cur_thread(), sp); 539 } 540 541 #if SANITIZER_MAC 542 TSAN_INTERCEPTOR(int, setjmp, void *env); 543 TSAN_INTERCEPTOR(int, _setjmp, void *env); 544 TSAN_INTERCEPTOR(int, sigsetjmp, void *env); 545 #else // SANITIZER_MAC 546 547 #if SANITIZER_NETBSD 548 #define setjmp_symname __setjmp14 549 #define sigsetjmp_symname __sigsetjmp14 550 #else 551 #define setjmp_symname setjmp 552 #define sigsetjmp_symname sigsetjmp 553 #endif 554 555 #define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x 556 #define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x) 557 #define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname) 558 #define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname) 559 560 #define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname) 561 #define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname) 562 563 // Not called. Merely to satisfy TSAN_INTERCEPT(). 564 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 565 int TSAN_INTERCEPTOR_SETJMP(void *env); 566 extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) { 567 CHECK(0); 568 return 0; 569 } 570 571 // FIXME: any reason to have a separate declaration? 572 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 573 int __interceptor__setjmp(void *env); 574 extern "C" int __interceptor__setjmp(void *env) { 575 CHECK(0); 576 return 0; 577 } 578 579 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 580 int TSAN_INTERCEPTOR_SIGSETJMP(void *env); 581 extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) { 582 CHECK(0); 583 return 0; 584 } 585 586 #if !SANITIZER_NETBSD 587 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 588 int __interceptor___sigsetjmp(void *env); 589 extern "C" int __interceptor___sigsetjmp(void *env) { 590 CHECK(0); 591 return 0; 592 } 593 #endif 594 595 extern "C" int setjmp_symname(void *env); 596 extern "C" int _setjmp(void *env); 597 extern "C" int sigsetjmp_symname(void *env); 598 #if !SANITIZER_NETBSD 599 extern "C" int __sigsetjmp(void *env); 600 #endif 601 DEFINE_REAL(int, setjmp_symname, void *env) 602 DEFINE_REAL(int, _setjmp, void *env) 603 DEFINE_REAL(int, sigsetjmp_symname, void *env) 604 #if !SANITIZER_NETBSD 605 DEFINE_REAL(int, __sigsetjmp, void *env) 606 #endif 607 #endif // SANITIZER_MAC 608 609 #if SANITIZER_NETBSD 610 #define longjmp_symname __longjmp14 611 #define siglongjmp_symname __siglongjmp14 612 #else 613 #define longjmp_symname longjmp 614 #define siglongjmp_symname siglongjmp 615 #endif 616 617 TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) { 618 // Note: if we call REAL(longjmp) in the context of ScopedInterceptor, 619 // bad things will happen. We will jump over ScopedInterceptor dtor and can 620 // leave thr->in_ignored_lib set. 621 { 622 SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val); 623 } 624 LongJmp(cur_thread(), env); 625 REAL(longjmp_symname)(env, val); 626 } 627 628 TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) { 629 { 630 SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val); 631 } 632 LongJmp(cur_thread(), env); 633 REAL(siglongjmp_symname)(env, val); 634 } 635 636 #if SANITIZER_NETBSD 637 TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { 638 { 639 SCOPED_INTERCEPTOR_RAW(_longjmp, env, val); 640 } 641 LongJmp(cur_thread(), env); 642 REAL(_longjmp)(env, val); 643 } 644 #endif 645 646 #if !SANITIZER_MAC 647 TSAN_INTERCEPTOR(void*, malloc, uptr size) { 648 if (in_symbolizer()) 649 return InternalAlloc(size); 650 void *p = 0; 651 { 652 SCOPED_INTERCEPTOR_RAW(malloc, size); 653 p = user_alloc(thr, pc, size); 654 } 655 invoke_malloc_hook(p, size); 656 return p; 657 } 658 659 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { 660 SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); 661 return user_memalign(thr, pc, align, sz); 662 } 663 664 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { 665 if (in_symbolizer()) 666 return InternalCalloc(size, n); 667 void *p = 0; 668 { 669 SCOPED_INTERCEPTOR_RAW(calloc, size, n); 670 p = user_calloc(thr, pc, size, n); 671 } 672 invoke_malloc_hook(p, n * size); 673 return p; 674 } 675 676 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { 677 if (in_symbolizer()) 678 return InternalRealloc(p, size); 679 if (p) 680 invoke_free_hook(p); 681 { 682 SCOPED_INTERCEPTOR_RAW(realloc, p, size); 683 p = user_realloc(thr, pc, p, size); 684 } 685 invoke_malloc_hook(p, size); 686 return p; 687 } 688 689 TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { 690 if (in_symbolizer()) 691 return InternalReallocArray(p, size, n); 692 if (p) 693 invoke_free_hook(p); 694 { 695 SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); 696 p = user_reallocarray(thr, pc, p, size, n); 697 } 698 invoke_malloc_hook(p, size); 699 return p; 700 } 701 702 TSAN_INTERCEPTOR(void, free, void *p) { 703 if (p == 0) 704 return; 705 if (in_symbolizer()) 706 return InternalFree(p); 707 invoke_free_hook(p); 708 SCOPED_INTERCEPTOR_RAW(free, p); 709 user_free(thr, pc, p); 710 } 711 712 TSAN_INTERCEPTOR(void, cfree, void *p) { 713 if (p == 0) 714 return; 715 if (in_symbolizer()) 716 return InternalFree(p); 717 invoke_free_hook(p); 718 SCOPED_INTERCEPTOR_RAW(cfree, p); 719 user_free(thr, pc, p); 720 } 721 722 TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { 723 SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); 724 return user_alloc_usable_size(p); 725 } 726 #endif 727 728 TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) { 729 SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); 730 uptr srclen = internal_strlen(src); 731 MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); 732 MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); 733 return REAL(strcpy)(dst, src); 734 } 735 736 TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { 737 SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); 738 uptr srclen = internal_strnlen(src, n); 739 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 740 MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); 741 return REAL(strncpy)(dst, src, n); 742 } 743 744 TSAN_INTERCEPTOR(char*, strdup, const char *str) { 745 SCOPED_TSAN_INTERCEPTOR(strdup, str); 746 // strdup will call malloc, so no instrumentation is required here. 747 return REAL(strdup)(str); 748 } 749 750 // Zero out addr if it points into shadow memory and was provided as a hint 751 // only, i.e., MAP_FIXED is not set. 752 static bool fix_mmap_addr(void **addr, long_t sz, int flags) { 753 if (*addr) { 754 if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { 755 if (flags & MAP_FIXED) { 756 errno = errno_EINVAL; 757 return false; 758 } else { 759 *addr = 0; 760 } 761 } 762 } 763 return true; 764 } 765 766 template <class Mmap> 767 static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap, 768 void *addr, SIZE_T sz, int prot, int flags, 769 int fd, OFF64_T off) { 770 if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED; 771 void *res = real_mmap(addr, sz, prot, flags, fd, off); 772 if (res != MAP_FAILED) { 773 if (fd > 0) FdAccess(thr, pc, fd); 774 MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz); 775 } 776 return res; 777 } 778 779 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { 780 SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); 781 UnmapShadow(thr, (uptr)addr, sz); 782 int res = REAL(munmap)(addr, sz); 783 return res; 784 } 785 786 #if SANITIZER_LINUX 787 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { 788 SCOPED_INTERCEPTOR_RAW(memalign, align, sz); 789 return user_memalign(thr, pc, align, sz); 790 } 791 #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) 792 #else 793 #define TSAN_MAYBE_INTERCEPT_MEMALIGN 794 #endif 795 796 #if !SANITIZER_MAC 797 TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { 798 if (in_symbolizer()) 799 return InternalAlloc(sz, nullptr, align); 800 SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); 801 return user_aligned_alloc(thr, pc, align, sz); 802 } 803 804 TSAN_INTERCEPTOR(void*, valloc, uptr sz) { 805 if (in_symbolizer()) 806 return InternalAlloc(sz, nullptr, GetPageSizeCached()); 807 SCOPED_INTERCEPTOR_RAW(valloc, sz); 808 return user_valloc(thr, pc, sz); 809 } 810 #endif 811 812 #if SANITIZER_LINUX 813 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { 814 if (in_symbolizer()) { 815 uptr PageSize = GetPageSizeCached(); 816 sz = sz ? RoundUpTo(sz, PageSize) : PageSize; 817 return InternalAlloc(sz, nullptr, PageSize); 818 } 819 SCOPED_INTERCEPTOR_RAW(pvalloc, sz); 820 return user_pvalloc(thr, pc, sz); 821 } 822 #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) 823 #else 824 #define TSAN_MAYBE_INTERCEPT_PVALLOC 825 #endif 826 827 #if !SANITIZER_MAC 828 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { 829 if (in_symbolizer()) { 830 void *p = InternalAlloc(sz, nullptr, align); 831 if (!p) 832 return errno_ENOMEM; 833 *memptr = p; 834 return 0; 835 } 836 SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); 837 return user_posix_memalign(thr, pc, memptr, align, sz); 838 } 839 #endif 840 841 // __cxa_guard_acquire and friends need to be intercepted in a special way - 842 // regular interceptors will break statically-linked libstdc++. Linux 843 // interceptors are especially defined as weak functions (so that they don't 844 // cause link errors when user defines them as well). So they silently 845 // auto-disable themselves when such symbol is already present in the binary. If 846 // we link libstdc++ statically, it will bring own __cxa_guard_acquire which 847 // will silently replace our interceptor. That's why on Linux we simply export 848 // these interceptors with INTERFACE_ATTRIBUTE. 849 // On OS X, we don't support statically linking, so we just use a regular 850 // interceptor. 851 #if SANITIZER_MAC 852 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR 853 #else 854 #define STDCXX_INTERCEPTOR(rettype, name, ...) \ 855 extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__) 856 #endif 857 858 // Used in thread-safe function static initialization. 859 STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { 860 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); 861 OnPotentiallyBlockingRegionBegin(); 862 auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd); 863 for (;;) { 864 u32 cmp = atomic_load(g, memory_order_acquire); 865 if (cmp == 0) { 866 if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed)) 867 return 1; 868 } else if (cmp == 1) { 869 Acquire(thr, pc, (uptr)g); 870 return 0; 871 } else { 872 internal_sched_yield(); 873 } 874 } 875 } 876 877 STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) { 878 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); 879 Release(thr, pc, (uptr)g); 880 atomic_store(g, 1, memory_order_release); 881 } 882 883 STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) { 884 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); 885 atomic_store(g, 0, memory_order_relaxed); 886 } 887 888 namespace __tsan { 889 void DestroyThreadState() { 890 ThreadState *thr = cur_thread(); 891 Processor *proc = thr->proc(); 892 ThreadFinish(thr); 893 ProcUnwire(proc, thr); 894 ProcDestroy(proc); 895 DTLS_Destroy(); 896 cur_thread_finalize(); 897 } 898 899 void PlatformCleanUpThreadState(ThreadState *thr) { 900 ThreadSignalContext *sctx = thr->signal_ctx; 901 if (sctx) { 902 thr->signal_ctx = 0; 903 UnmapOrDie(sctx, sizeof(*sctx)); 904 } 905 } 906 } // namespace __tsan 907 908 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD 909 static void thread_finalize(void *v) { 910 uptr iter = (uptr)v; 911 if (iter > 1) { 912 if (pthread_setspecific(interceptor_ctx()->finalize_key, 913 (void*)(iter - 1))) { 914 Printf("ThreadSanitizer: failed to set thread key\n"); 915 Die(); 916 } 917 return; 918 } 919 DestroyThreadState(); 920 } 921 #endif 922 923 924 struct ThreadParam { 925 void* (*callback)(void *arg); 926 void *param; 927 atomic_uintptr_t tid; 928 }; 929 930 extern "C" void *__tsan_thread_start_func(void *arg) { 931 ThreadParam *p = (ThreadParam*)arg; 932 void* (*callback)(void *arg) = p->callback; 933 void *param = p->param; 934 int tid = 0; 935 { 936 cur_thread_init(); 937 ThreadState *thr = cur_thread(); 938 // Thread-local state is not initialized yet. 939 ScopedIgnoreInterceptors ignore; 940 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD 941 ThreadIgnoreBegin(thr, 0); 942 if (pthread_setspecific(interceptor_ctx()->finalize_key, 943 (void *)GetPthreadDestructorIterations())) { 944 Printf("ThreadSanitizer: failed to set thread key\n"); 945 Die(); 946 } 947 ThreadIgnoreEnd(thr, 0); 948 #endif 949 while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) 950 internal_sched_yield(); 951 Processor *proc = ProcCreate(); 952 ProcWire(proc, thr); 953 ThreadStart(thr, tid, GetTid(), ThreadType::Regular); 954 atomic_store(&p->tid, 0, memory_order_release); 955 } 956 void *res = callback(param); 957 // Prevent the callback from being tail called, 958 // it mixes up stack traces. 959 volatile int foo = 42; 960 foo++; 961 return res; 962 } 963 964 TSAN_INTERCEPTOR(int, pthread_create, 965 void *th, void *attr, void *(*callback)(void*), void * param) { 966 SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); 967 968 MaybeSpawnBackgroundThread(); 969 970 if (ctx->after_multithreaded_fork) { 971 if (flags()->die_after_fork) { 972 Report("ThreadSanitizer: starting new threads after multi-threaded " 973 "fork is not supported. Dying (set die_after_fork=0 to override)\n"); 974 Die(); 975 } else { 976 VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded " 977 "fork is not supported (pid %d). Continuing because of " 978 "die_after_fork=0, but you are on your own\n", internal_getpid()); 979 } 980 } 981 __sanitizer_pthread_attr_t myattr; 982 if (attr == 0) { 983 pthread_attr_init(&myattr); 984 attr = &myattr; 985 } 986 int detached = 0; 987 REAL(pthread_attr_getdetachstate)(attr, &detached); 988 AdjustStackSize(attr); 989 990 ThreadParam p; 991 p.callback = callback; 992 p.param = param; 993 atomic_store(&p.tid, 0, memory_order_relaxed); 994 int res = -1; 995 { 996 // Otherwise we see false positives in pthread stack manipulation. 997 ScopedIgnoreInterceptors ignore; 998 ThreadIgnoreBegin(thr, pc); 999 res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); 1000 ThreadIgnoreEnd(thr, pc); 1001 } 1002 if (res == 0) { 1003 int tid = ThreadCreate(thr, pc, *(uptr*)th, IsStateDetached(detached)); 1004 CHECK_NE(tid, 0); 1005 // Synchronization on p.tid serves two purposes: 1006 // 1. ThreadCreate must finish before the new thread starts. 1007 // Otherwise the new thread can call pthread_detach, but the pthread_t 1008 // identifier is not yet registered in ThreadRegistry by ThreadCreate. 1009 // 2. ThreadStart must finish before this thread continues. 1010 // Otherwise, this thread can call pthread_detach and reset thr->sync 1011 // before the new thread got a chance to acquire from it in ThreadStart. 1012 atomic_store(&p.tid, tid, memory_order_release); 1013 while (atomic_load(&p.tid, memory_order_acquire) != 0) 1014 internal_sched_yield(); 1015 } 1016 if (attr == &myattr) 1017 pthread_attr_destroy(&myattr); 1018 return res; 1019 } 1020 1021 TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { 1022 SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); 1023 int tid = ThreadConsumeTid(thr, pc, (uptr)th); 1024 ThreadIgnoreBegin(thr, pc); 1025 int res = BLOCK_REAL(pthread_join)(th, ret); 1026 ThreadIgnoreEnd(thr, pc); 1027 if (res == 0) { 1028 ThreadJoin(thr, pc, tid); 1029 } 1030 return res; 1031 } 1032 1033 DEFINE_REAL_PTHREAD_FUNCTIONS 1034 1035 TSAN_INTERCEPTOR(int, pthread_detach, void *th) { 1036 SCOPED_INTERCEPTOR_RAW(pthread_detach, th); 1037 int tid = ThreadConsumeTid(thr, pc, (uptr)th); 1038 int res = REAL(pthread_detach)(th); 1039 if (res == 0) { 1040 ThreadDetach(thr, pc, tid); 1041 } 1042 return res; 1043 } 1044 1045 TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { 1046 { 1047 SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); 1048 #if !SANITIZER_MAC && !SANITIZER_ANDROID 1049 CHECK_EQ(thr, &cur_thread_placeholder); 1050 #endif 1051 } 1052 REAL(pthread_exit)(retval); 1053 } 1054 1055 #if SANITIZER_LINUX 1056 TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { 1057 SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret); 1058 int tid = ThreadConsumeTid(thr, pc, (uptr)th); 1059 ThreadIgnoreBegin(thr, pc); 1060 int res = REAL(pthread_tryjoin_np)(th, ret); 1061 ThreadIgnoreEnd(thr, pc); 1062 if (res == 0) 1063 ThreadJoin(thr, pc, tid); 1064 else 1065 ThreadNotJoined(thr, pc, tid, (uptr)th); 1066 return res; 1067 } 1068 1069 TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret, 1070 const struct timespec *abstime) { 1071 SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime); 1072 int tid = ThreadConsumeTid(thr, pc, (uptr)th); 1073 ThreadIgnoreBegin(thr, pc); 1074 int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime); 1075 ThreadIgnoreEnd(thr, pc); 1076 if (res == 0) 1077 ThreadJoin(thr, pc, tid); 1078 else 1079 ThreadNotJoined(thr, pc, tid, (uptr)th); 1080 return res; 1081 } 1082 #endif 1083 1084 // Problem: 1085 // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). 1086 // pthread_cond_t has different size in the different versions. 1087 // If call new REAL functions for old pthread_cond_t, they will corrupt memory 1088 // after pthread_cond_t (old cond is smaller). 1089 // If we call old REAL functions for new pthread_cond_t, we will lose some 1090 // functionality (e.g. old functions do not support waiting against 1091 // CLOCK_REALTIME). 1092 // Proper handling would require to have 2 versions of interceptors as well. 1093 // But this is messy, in particular requires linker scripts when sanitizer 1094 // runtime is linked into a shared library. 1095 // Instead we assume we don't have dynamic libraries built against old 1096 // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag 1097 // that allows to work with old libraries (but this mode does not support 1098 // some features, e.g. pthread_condattr_getpshared). 1099 static void *init_cond(void *c, bool force = false) { 1100 // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions. 1101 // So we allocate additional memory on the side large enough to hold 1102 // any pthread_cond_t object. Always call new REAL functions, but pass 1103 // the aux object to them. 1104 // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes 1105 // first word of pthread_cond_t to zero. 1106 // It's all relevant only for linux. 1107 if (!common_flags()->legacy_pthread_cond) 1108 return c; 1109 atomic_uintptr_t *p = (atomic_uintptr_t*)c; 1110 uptr cond = atomic_load(p, memory_order_acquire); 1111 if (!force && cond != 0) 1112 return (void*)cond; 1113 void *newcond = WRAP(malloc)(pthread_cond_t_sz); 1114 internal_memset(newcond, 0, pthread_cond_t_sz); 1115 if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond, 1116 memory_order_acq_rel)) 1117 return newcond; 1118 WRAP(free)(newcond); 1119 return (void*)cond; 1120 } 1121 1122 struct CondMutexUnlockCtx { 1123 ScopedInterceptor *si; 1124 ThreadState *thr; 1125 uptr pc; 1126 void *m; 1127 }; 1128 1129 static void cond_mutex_unlock(CondMutexUnlockCtx *arg) { 1130 // pthread_cond_wait interceptor has enabled async signal delivery 1131 // (see BlockingCall below). Disable async signals since we are running 1132 // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run 1133 // since the thread is cancelled, so we have to manually execute them 1134 // (the thread still can run some user code due to pthread_cleanup_push). 1135 ThreadSignalContext *ctx = SigCtx(arg->thr); 1136 CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1); 1137 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 1138 MutexPostLock(arg->thr, arg->pc, (uptr)arg->m, MutexFlagDoPreLockOnPostLock); 1139 // Undo BlockingCall ctor effects. 1140 arg->thr->ignore_interceptors--; 1141 arg->si->~ScopedInterceptor(); 1142 } 1143 1144 INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { 1145 void *cond = init_cond(c, true); 1146 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); 1147 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); 1148 return REAL(pthread_cond_init)(cond, a); 1149 } 1150 1151 static int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, 1152 int (*fn)(void *c, void *m, void *abstime), void *c, 1153 void *m, void *t) { 1154 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1155 MutexUnlock(thr, pc, (uptr)m); 1156 CondMutexUnlockCtx arg = {si, thr, pc, m}; 1157 int res = 0; 1158 // This ensures that we handle mutex lock even in case of pthread_cancel. 1159 // See test/tsan/cond_cancel.cpp. 1160 { 1161 // Enable signal delivery while the thread is blocked. 1162 BlockingCall bc(thr); 1163 res = call_pthread_cancel_with_cleanup( 1164 fn, c, m, t, (void (*)(void *arg))cond_mutex_unlock, &arg); 1165 } 1166 if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); 1167 MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock); 1168 return res; 1169 } 1170 1171 INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { 1172 void *cond = init_cond(c); 1173 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); 1174 return cond_wait(thr, pc, &si, (int (*)(void *c, void *m, void *abstime))REAL( 1175 pthread_cond_wait), 1176 cond, m, 0); 1177 } 1178 1179 INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { 1180 void *cond = init_cond(c); 1181 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); 1182 return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait), cond, m, 1183 abstime); 1184 } 1185 1186 #if SANITIZER_MAC 1187 INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m, 1188 void *reltime) { 1189 void *cond = init_cond(c); 1190 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime); 1191 return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait_relative_np), cond, 1192 m, reltime); 1193 } 1194 #endif 1195 1196 INTERCEPTOR(int, pthread_cond_signal, void *c) { 1197 void *cond = init_cond(c); 1198 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); 1199 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1200 return REAL(pthread_cond_signal)(cond); 1201 } 1202 1203 INTERCEPTOR(int, pthread_cond_broadcast, void *c) { 1204 void *cond = init_cond(c); 1205 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); 1206 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1207 return REAL(pthread_cond_broadcast)(cond); 1208 } 1209 1210 INTERCEPTOR(int, pthread_cond_destroy, void *c) { 1211 void *cond = init_cond(c); 1212 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); 1213 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); 1214 int res = REAL(pthread_cond_destroy)(cond); 1215 if (common_flags()->legacy_pthread_cond) { 1216 // Free our aux cond and zero the pointer to not leave dangling pointers. 1217 WRAP(free)(cond); 1218 atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed); 1219 } 1220 return res; 1221 } 1222 1223 TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { 1224 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); 1225 int res = REAL(pthread_mutex_init)(m, a); 1226 if (res == 0) { 1227 u32 flagz = 0; 1228 if (a) { 1229 int type = 0; 1230 if (REAL(pthread_mutexattr_gettype)(a, &type) == 0) 1231 if (type == PTHREAD_MUTEX_RECURSIVE || 1232 type == PTHREAD_MUTEX_RECURSIVE_NP) 1233 flagz |= MutexFlagWriteReentrant; 1234 } 1235 MutexCreate(thr, pc, (uptr)m, flagz); 1236 } 1237 return res; 1238 } 1239 1240 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { 1241 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); 1242 int res = REAL(pthread_mutex_destroy)(m); 1243 if (res == 0 || res == errno_EBUSY) { 1244 MutexDestroy(thr, pc, (uptr)m); 1245 } 1246 return res; 1247 } 1248 1249 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { 1250 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); 1251 int res = REAL(pthread_mutex_trylock)(m); 1252 if (res == errno_EOWNERDEAD) 1253 MutexRepair(thr, pc, (uptr)m); 1254 if (res == 0 || res == errno_EOWNERDEAD) 1255 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1256 return res; 1257 } 1258 1259 #if !SANITIZER_MAC 1260 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { 1261 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); 1262 int res = REAL(pthread_mutex_timedlock)(m, abstime); 1263 if (res == 0) { 1264 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1265 } 1266 return res; 1267 } 1268 #endif 1269 1270 #if !SANITIZER_MAC 1271 TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { 1272 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); 1273 int res = REAL(pthread_spin_init)(m, pshared); 1274 if (res == 0) { 1275 MutexCreate(thr, pc, (uptr)m); 1276 } 1277 return res; 1278 } 1279 1280 TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { 1281 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); 1282 int res = REAL(pthread_spin_destroy)(m); 1283 if (res == 0) { 1284 MutexDestroy(thr, pc, (uptr)m); 1285 } 1286 return res; 1287 } 1288 1289 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { 1290 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); 1291 MutexPreLock(thr, pc, (uptr)m); 1292 int res = REAL(pthread_spin_lock)(m); 1293 if (res == 0) { 1294 MutexPostLock(thr, pc, (uptr)m); 1295 } 1296 return res; 1297 } 1298 1299 TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { 1300 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); 1301 int res = REAL(pthread_spin_trylock)(m); 1302 if (res == 0) { 1303 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1304 } 1305 return res; 1306 } 1307 1308 TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { 1309 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); 1310 MutexUnlock(thr, pc, (uptr)m); 1311 int res = REAL(pthread_spin_unlock)(m); 1312 return res; 1313 } 1314 #endif 1315 1316 TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { 1317 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); 1318 int res = REAL(pthread_rwlock_init)(m, a); 1319 if (res == 0) { 1320 MutexCreate(thr, pc, (uptr)m); 1321 } 1322 return res; 1323 } 1324 1325 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { 1326 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); 1327 int res = REAL(pthread_rwlock_destroy)(m); 1328 if (res == 0) { 1329 MutexDestroy(thr, pc, (uptr)m); 1330 } 1331 return res; 1332 } 1333 1334 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { 1335 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); 1336 MutexPreReadLock(thr, pc, (uptr)m); 1337 int res = REAL(pthread_rwlock_rdlock)(m); 1338 if (res == 0) { 1339 MutexPostReadLock(thr, pc, (uptr)m); 1340 } 1341 return res; 1342 } 1343 1344 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { 1345 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); 1346 int res = REAL(pthread_rwlock_tryrdlock)(m); 1347 if (res == 0) { 1348 MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock); 1349 } 1350 return res; 1351 } 1352 1353 #if !SANITIZER_MAC 1354 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { 1355 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); 1356 int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); 1357 if (res == 0) { 1358 MutexPostReadLock(thr, pc, (uptr)m); 1359 } 1360 return res; 1361 } 1362 #endif 1363 1364 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { 1365 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); 1366 MutexPreLock(thr, pc, (uptr)m); 1367 int res = REAL(pthread_rwlock_wrlock)(m); 1368 if (res == 0) { 1369 MutexPostLock(thr, pc, (uptr)m); 1370 } 1371 return res; 1372 } 1373 1374 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { 1375 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); 1376 int res = REAL(pthread_rwlock_trywrlock)(m); 1377 if (res == 0) { 1378 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1379 } 1380 return res; 1381 } 1382 1383 #if !SANITIZER_MAC 1384 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { 1385 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); 1386 int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); 1387 if (res == 0) { 1388 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1389 } 1390 return res; 1391 } 1392 #endif 1393 1394 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { 1395 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); 1396 MutexReadOrWriteUnlock(thr, pc, (uptr)m); 1397 int res = REAL(pthread_rwlock_unlock)(m); 1398 return res; 1399 } 1400 1401 #if !SANITIZER_MAC 1402 TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { 1403 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); 1404 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); 1405 int res = REAL(pthread_barrier_init)(b, a, count); 1406 return res; 1407 } 1408 1409 TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { 1410 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); 1411 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); 1412 int res = REAL(pthread_barrier_destroy)(b); 1413 return res; 1414 } 1415 1416 TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { 1417 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); 1418 Release(thr, pc, (uptr)b); 1419 MemoryRead(thr, pc, (uptr)b, kSizeLog1); 1420 int res = REAL(pthread_barrier_wait)(b); 1421 MemoryRead(thr, pc, (uptr)b, kSizeLog1); 1422 if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { 1423 Acquire(thr, pc, (uptr)b); 1424 } 1425 return res; 1426 } 1427 #endif 1428 1429 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { 1430 SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); 1431 if (o == 0 || f == 0) 1432 return errno_EINVAL; 1433 atomic_uint32_t *a; 1434 1435 if (SANITIZER_MAC) 1436 a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t))); 1437 else if (SANITIZER_NETBSD) 1438 a = static_cast<atomic_uint32_t*> 1439 ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz)); 1440 else 1441 a = static_cast<atomic_uint32_t*>(o); 1442 1443 u32 v = atomic_load(a, memory_order_acquire); 1444 if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, 1445 memory_order_relaxed)) { 1446 (*f)(); 1447 if (!thr->in_ignored_lib) 1448 Release(thr, pc, (uptr)o); 1449 atomic_store(a, 2, memory_order_release); 1450 } else { 1451 while (v != 2) { 1452 internal_sched_yield(); 1453 v = atomic_load(a, memory_order_acquire); 1454 } 1455 if (!thr->in_ignored_lib) 1456 Acquire(thr, pc, (uptr)o); 1457 } 1458 return 0; 1459 } 1460 1461 #if SANITIZER_LINUX && !SANITIZER_ANDROID 1462 TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { 1463 SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); 1464 if (fd > 0) 1465 FdAccess(thr, pc, fd); 1466 return REAL(__fxstat)(version, fd, buf); 1467 } 1468 #define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat) 1469 #else 1470 #define TSAN_MAYBE_INTERCEPT___FXSTAT 1471 #endif 1472 1473 TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { 1474 #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD 1475 SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); 1476 if (fd > 0) 1477 FdAccess(thr, pc, fd); 1478 return REAL(fstat)(fd, buf); 1479 #else 1480 SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); 1481 if (fd > 0) 1482 FdAccess(thr, pc, fd); 1483 return REAL(__fxstat)(0, fd, buf); 1484 #endif 1485 } 1486 1487 #if SANITIZER_LINUX && !SANITIZER_ANDROID 1488 TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { 1489 SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); 1490 if (fd > 0) 1491 FdAccess(thr, pc, fd); 1492 return REAL(__fxstat64)(version, fd, buf); 1493 } 1494 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64) 1495 #else 1496 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 1497 #endif 1498 1499 #if SANITIZER_LINUX && !SANITIZER_ANDROID 1500 TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { 1501 SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); 1502 if (fd > 0) 1503 FdAccess(thr, pc, fd); 1504 return REAL(__fxstat64)(0, fd, buf); 1505 } 1506 #define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64) 1507 #else 1508 #define TSAN_MAYBE_INTERCEPT_FSTAT64 1509 #endif 1510 1511 TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) { 1512 SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode); 1513 READ_STRING(thr, pc, name, 0); 1514 int fd = REAL(open)(name, flags, mode); 1515 if (fd >= 0) 1516 FdFileCreate(thr, pc, fd); 1517 return fd; 1518 } 1519 1520 #if SANITIZER_LINUX 1521 TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) { 1522 SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode); 1523 READ_STRING(thr, pc, name, 0); 1524 int fd = REAL(open64)(name, flags, mode); 1525 if (fd >= 0) 1526 FdFileCreate(thr, pc, fd); 1527 return fd; 1528 } 1529 #define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64) 1530 #else 1531 #define TSAN_MAYBE_INTERCEPT_OPEN64 1532 #endif 1533 1534 TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { 1535 SCOPED_TSAN_INTERCEPTOR(creat, name, mode); 1536 READ_STRING(thr, pc, name, 0); 1537 int fd = REAL(creat)(name, mode); 1538 if (fd >= 0) 1539 FdFileCreate(thr, pc, fd); 1540 return fd; 1541 } 1542 1543 #if SANITIZER_LINUX 1544 TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { 1545 SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); 1546 READ_STRING(thr, pc, name, 0); 1547 int fd = REAL(creat64)(name, mode); 1548 if (fd >= 0) 1549 FdFileCreate(thr, pc, fd); 1550 return fd; 1551 } 1552 #define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64) 1553 #else 1554 #define TSAN_MAYBE_INTERCEPT_CREAT64 1555 #endif 1556 1557 TSAN_INTERCEPTOR(int, dup, int oldfd) { 1558 SCOPED_TSAN_INTERCEPTOR(dup, oldfd); 1559 int newfd = REAL(dup)(oldfd); 1560 if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) 1561 FdDup(thr, pc, oldfd, newfd, true); 1562 return newfd; 1563 } 1564 1565 TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { 1566 SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); 1567 int newfd2 = REAL(dup2)(oldfd, newfd); 1568 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1569 FdDup(thr, pc, oldfd, newfd2, false); 1570 return newfd2; 1571 } 1572 1573 #if !SANITIZER_MAC 1574 TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { 1575 SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); 1576 int newfd2 = REAL(dup3)(oldfd, newfd, flags); 1577 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1578 FdDup(thr, pc, oldfd, newfd2, false); 1579 return newfd2; 1580 } 1581 #endif 1582 1583 #if SANITIZER_LINUX 1584 TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { 1585 SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); 1586 int fd = REAL(eventfd)(initval, flags); 1587 if (fd >= 0) 1588 FdEventCreate(thr, pc, fd); 1589 return fd; 1590 } 1591 #define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd) 1592 #else 1593 #define TSAN_MAYBE_INTERCEPT_EVENTFD 1594 #endif 1595 1596 #if SANITIZER_LINUX 1597 TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { 1598 SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); 1599 if (fd >= 0) 1600 FdClose(thr, pc, fd); 1601 fd = REAL(signalfd)(fd, mask, flags); 1602 if (fd >= 0) 1603 FdSignalCreate(thr, pc, fd); 1604 return fd; 1605 } 1606 #define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd) 1607 #else 1608 #define TSAN_MAYBE_INTERCEPT_SIGNALFD 1609 #endif 1610 1611 #if SANITIZER_LINUX 1612 TSAN_INTERCEPTOR(int, inotify_init, int fake) { 1613 SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); 1614 int fd = REAL(inotify_init)(fake); 1615 if (fd >= 0) 1616 FdInotifyCreate(thr, pc, fd); 1617 return fd; 1618 } 1619 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init) 1620 #else 1621 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT 1622 #endif 1623 1624 #if SANITIZER_LINUX 1625 TSAN_INTERCEPTOR(int, inotify_init1, int flags) { 1626 SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); 1627 int fd = REAL(inotify_init1)(flags); 1628 if (fd >= 0) 1629 FdInotifyCreate(thr, pc, fd); 1630 return fd; 1631 } 1632 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1) 1633 #else 1634 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 1635 #endif 1636 1637 TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { 1638 SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); 1639 int fd = REAL(socket)(domain, type, protocol); 1640 if (fd >= 0) 1641 FdSocketCreate(thr, pc, fd); 1642 return fd; 1643 } 1644 1645 TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { 1646 SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); 1647 int res = REAL(socketpair)(domain, type, protocol, fd); 1648 if (res == 0 && fd[0] >= 0 && fd[1] >= 0) 1649 FdPipeCreate(thr, pc, fd[0], fd[1]); 1650 return res; 1651 } 1652 1653 TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { 1654 SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); 1655 FdSocketConnecting(thr, pc, fd); 1656 int res = REAL(connect)(fd, addr, addrlen); 1657 if (res == 0 && fd >= 0) 1658 FdSocketConnect(thr, pc, fd); 1659 return res; 1660 } 1661 1662 TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { 1663 SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen); 1664 int res = REAL(bind)(fd, addr, addrlen); 1665 if (fd > 0 && res == 0) 1666 FdAccess(thr, pc, fd); 1667 return res; 1668 } 1669 1670 TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { 1671 SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog); 1672 int res = REAL(listen)(fd, backlog); 1673 if (fd > 0 && res == 0) 1674 FdAccess(thr, pc, fd); 1675 return res; 1676 } 1677 1678 TSAN_INTERCEPTOR(int, close, int fd) { 1679 SCOPED_TSAN_INTERCEPTOR(close, fd); 1680 if (fd >= 0) 1681 FdClose(thr, pc, fd); 1682 return REAL(close)(fd); 1683 } 1684 1685 #if SANITIZER_LINUX 1686 TSAN_INTERCEPTOR(int, __close, int fd) { 1687 SCOPED_TSAN_INTERCEPTOR(__close, fd); 1688 if (fd >= 0) 1689 FdClose(thr, pc, fd); 1690 return REAL(__close)(fd); 1691 } 1692 #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close) 1693 #else 1694 #define TSAN_MAYBE_INTERCEPT___CLOSE 1695 #endif 1696 1697 // glibc guts 1698 #if SANITIZER_LINUX && !SANITIZER_ANDROID 1699 TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { 1700 SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); 1701 int fds[64]; 1702 int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); 1703 for (int i = 0; i < cnt; i++) { 1704 if (fds[i] > 0) 1705 FdClose(thr, pc, fds[i]); 1706 } 1707 REAL(__res_iclose)(state, free_addr); 1708 } 1709 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose) 1710 #else 1711 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE 1712 #endif 1713 1714 TSAN_INTERCEPTOR(int, pipe, int *pipefd) { 1715 SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); 1716 int res = REAL(pipe)(pipefd); 1717 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1718 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1719 return res; 1720 } 1721 1722 #if !SANITIZER_MAC 1723 TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { 1724 SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); 1725 int res = REAL(pipe2)(pipefd, flags); 1726 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1727 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1728 return res; 1729 } 1730 #endif 1731 1732 TSAN_INTERCEPTOR(int, unlink, char *path) { 1733 SCOPED_TSAN_INTERCEPTOR(unlink, path); 1734 Release(thr, pc, File2addr(path)); 1735 int res = REAL(unlink)(path); 1736 return res; 1737 } 1738 1739 TSAN_INTERCEPTOR(void*, tmpfile, int fake) { 1740 SCOPED_TSAN_INTERCEPTOR(tmpfile, fake); 1741 void *res = REAL(tmpfile)(fake); 1742 if (res) { 1743 int fd = fileno_unlocked(res); 1744 if (fd >= 0) 1745 FdFileCreate(thr, pc, fd); 1746 } 1747 return res; 1748 } 1749 1750 #if SANITIZER_LINUX 1751 TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { 1752 SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); 1753 void *res = REAL(tmpfile64)(fake); 1754 if (res) { 1755 int fd = fileno_unlocked(res); 1756 if (fd >= 0) 1757 FdFileCreate(thr, pc, fd); 1758 } 1759 return res; 1760 } 1761 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64) 1762 #else 1763 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 1764 #endif 1765 1766 static void FlushStreams() { 1767 // Flushing all the streams here may freeze the process if a child thread is 1768 // performing file stream operations at the same time. 1769 REAL(fflush)(stdout); 1770 REAL(fflush)(stderr); 1771 } 1772 1773 TSAN_INTERCEPTOR(void, abort, int fake) { 1774 SCOPED_TSAN_INTERCEPTOR(abort, fake); 1775 FlushStreams(); 1776 REAL(abort)(fake); 1777 } 1778 1779 TSAN_INTERCEPTOR(int, rmdir, char *path) { 1780 SCOPED_TSAN_INTERCEPTOR(rmdir, path); 1781 Release(thr, pc, Dir2addr(path)); 1782 int res = REAL(rmdir)(path); 1783 return res; 1784 } 1785 1786 TSAN_INTERCEPTOR(int, closedir, void *dirp) { 1787 SCOPED_TSAN_INTERCEPTOR(closedir, dirp); 1788 if (dirp) { 1789 int fd = dirfd(dirp); 1790 FdClose(thr, pc, fd); 1791 } 1792 return REAL(closedir)(dirp); 1793 } 1794 1795 #if SANITIZER_LINUX 1796 TSAN_INTERCEPTOR(int, epoll_create, int size) { 1797 SCOPED_TSAN_INTERCEPTOR(epoll_create, size); 1798 int fd = REAL(epoll_create)(size); 1799 if (fd >= 0) 1800 FdPollCreate(thr, pc, fd); 1801 return fd; 1802 } 1803 1804 TSAN_INTERCEPTOR(int, epoll_create1, int flags) { 1805 SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); 1806 int fd = REAL(epoll_create1)(flags); 1807 if (fd >= 0) 1808 FdPollCreate(thr, pc, fd); 1809 return fd; 1810 } 1811 1812 TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { 1813 SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); 1814 if (epfd >= 0) 1815 FdAccess(thr, pc, epfd); 1816 if (epfd >= 0 && fd >= 0) 1817 FdAccess(thr, pc, fd); 1818 if (op == EPOLL_CTL_ADD && epfd >= 0) 1819 FdRelease(thr, pc, epfd); 1820 int res = REAL(epoll_ctl)(epfd, op, fd, ev); 1821 return res; 1822 } 1823 1824 TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { 1825 SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); 1826 if (epfd >= 0) 1827 FdAccess(thr, pc, epfd); 1828 int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); 1829 if (res > 0 && epfd >= 0) 1830 FdAcquire(thr, pc, epfd); 1831 return res; 1832 } 1833 1834 TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout, 1835 void *sigmask) { 1836 SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask); 1837 if (epfd >= 0) 1838 FdAccess(thr, pc, epfd); 1839 int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask); 1840 if (res > 0 && epfd >= 0) 1841 FdAcquire(thr, pc, epfd); 1842 return res; 1843 } 1844 1845 #define TSAN_MAYBE_INTERCEPT_EPOLL \ 1846 TSAN_INTERCEPT(epoll_create); \ 1847 TSAN_INTERCEPT(epoll_create1); \ 1848 TSAN_INTERCEPT(epoll_ctl); \ 1849 TSAN_INTERCEPT(epoll_wait); \ 1850 TSAN_INTERCEPT(epoll_pwait) 1851 #else 1852 #define TSAN_MAYBE_INTERCEPT_EPOLL 1853 #endif 1854 1855 // The following functions are intercepted merely to process pending signals. 1856 // If program blocks signal X, we must deliver the signal before the function 1857 // returns. Similarly, if program unblocks a signal (or returns from sigsuspend) 1858 // it's better to deliver the signal straight away. 1859 TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { 1860 SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); 1861 return REAL(sigsuspend)(mask); 1862 } 1863 1864 TSAN_INTERCEPTOR(int, sigblock, int mask) { 1865 SCOPED_TSAN_INTERCEPTOR(sigblock, mask); 1866 return REAL(sigblock)(mask); 1867 } 1868 1869 TSAN_INTERCEPTOR(int, sigsetmask, int mask) { 1870 SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask); 1871 return REAL(sigsetmask)(mask); 1872 } 1873 1874 TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, 1875 __sanitizer_sigset_t *oldset) { 1876 SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset); 1877 return REAL(pthread_sigmask)(how, set, oldset); 1878 } 1879 1880 namespace __tsan { 1881 1882 static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, 1883 bool sigact, int sig, 1884 __sanitizer_siginfo *info, void *uctx) { 1885 __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; 1886 if (acquire) 1887 Acquire(thr, 0, (uptr)&sigactions[sig]); 1888 // Signals are generally asynchronous, so if we receive a signals when 1889 // ignores are enabled we should disable ignores. This is critical for sync 1890 // and interceptors, because otherwise we can miss syncronization and report 1891 // false races. 1892 int ignore_reads_and_writes = thr->ignore_reads_and_writes; 1893 int ignore_interceptors = thr->ignore_interceptors; 1894 int ignore_sync = thr->ignore_sync; 1895 if (!ctx->after_multithreaded_fork) { 1896 thr->ignore_reads_and_writes = 0; 1897 thr->fast_state.ClearIgnoreBit(); 1898 thr->ignore_interceptors = 0; 1899 thr->ignore_sync = 0; 1900 } 1901 // Ensure that the handler does not spoil errno. 1902 const int saved_errno = errno; 1903 errno = 99; 1904 // This code races with sigaction. Be careful to not read sa_sigaction twice. 1905 // Also need to remember pc for reporting before the call, 1906 // because the handler can reset it. 1907 volatile uptr pc = 1908 sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler; 1909 if (pc != sig_dfl && pc != sig_ign) { 1910 if (sigact) 1911 ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx); 1912 else 1913 ((__sanitizer_sighandler_ptr)pc)(sig); 1914 } 1915 if (!ctx->after_multithreaded_fork) { 1916 thr->ignore_reads_and_writes = ignore_reads_and_writes; 1917 if (ignore_reads_and_writes) 1918 thr->fast_state.SetIgnoreBit(); 1919 thr->ignore_interceptors = ignore_interceptors; 1920 thr->ignore_sync = ignore_sync; 1921 } 1922 // We do not detect errno spoiling for SIGTERM, 1923 // because some SIGTERM handlers do spoil errno but reraise SIGTERM, 1924 // tsan reports false positive in such case. 1925 // It's difficult to properly detect this situation (reraise), 1926 // because in async signal processing case (when handler is called directly 1927 // from rtl_generic_sighandler) we have not yet received the reraised 1928 // signal; and it looks too fragile to intercept all ways to reraise a signal. 1929 if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) { 1930 VarSizeStackTrace stack; 1931 // StackTrace::GetNestInstructionPc(pc) is used because return address is 1932 // expected, OutputReport() will undo this. 1933 ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); 1934 ThreadRegistryLock l(ctx->thread_registry); 1935 ScopedReport rep(ReportTypeErrnoInSignal); 1936 if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) { 1937 rep.AddStack(stack, true); 1938 OutputReport(thr, rep); 1939 } 1940 } 1941 errno = saved_errno; 1942 } 1943 1944 void ProcessPendingSignals(ThreadState *thr) { 1945 ThreadSignalContext *sctx = SigCtx(thr); 1946 if (sctx == 0 || 1947 atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0) 1948 return; 1949 atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed); 1950 atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); 1951 internal_sigfillset(&sctx->emptyset); 1952 int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset); 1953 CHECK_EQ(res, 0); 1954 for (int sig = 0; sig < kSigCount; sig++) { 1955 SignalDesc *signal = &sctx->pending_signals[sig]; 1956 if (signal->armed) { 1957 signal->armed = false; 1958 CallUserSignalHandler(thr, false, true, signal->sigaction, sig, 1959 &signal->siginfo, &signal->ctx); 1960 } 1961 } 1962 res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0); 1963 CHECK_EQ(res, 0); 1964 atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); 1965 } 1966 1967 } // namespace __tsan 1968 1969 static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { 1970 return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP || 1971 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || 1972 // If we are sending signal to ourselves, we must process it now. 1973 (sctx && sig == sctx->int_signal_send); 1974 } 1975 1976 void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, 1977 __sanitizer_siginfo *info, 1978 void *ctx) { 1979 cur_thread_init(); 1980 ThreadState *thr = cur_thread(); 1981 ThreadSignalContext *sctx = SigCtx(thr); 1982 if (sig < 0 || sig >= kSigCount) { 1983 VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig); 1984 return; 1985 } 1986 // Don't mess with synchronous signals. 1987 const bool sync = is_sync_signal(sctx, sig); 1988 if (sync || 1989 // If we are in blocking function, we can safely process it now 1990 // (but check if we are in a recursive interceptor, 1991 // i.e. pthread_join()->munmap()). 1992 (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) { 1993 atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); 1994 if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) { 1995 atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed); 1996 CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx); 1997 atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed); 1998 } else { 1999 // Be very conservative with when we do acquire in this case. 2000 // It's unsafe to do acquire in async handlers, because ThreadState 2001 // can be in inconsistent state. 2002 // SIGSYS looks relatively safe -- it's synchronous and can actually 2003 // need some global state. 2004 bool acq = (sig == SIGSYS); 2005 CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx); 2006 } 2007 atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); 2008 return; 2009 } 2010 2011 if (sctx == 0) 2012 return; 2013 SignalDesc *signal = &sctx->pending_signals[sig]; 2014 if (signal->armed == false) { 2015 signal->armed = true; 2016 signal->sigaction = sigact; 2017 if (info) 2018 internal_memcpy(&signal->siginfo, info, sizeof(*info)); 2019 if (ctx) 2020 internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); 2021 atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed); 2022 } 2023 } 2024 2025 static void rtl_sighandler(int sig) { 2026 rtl_generic_sighandler(false, sig, 0, 0); 2027 } 2028 2029 static void rtl_sigaction(int sig, __sanitizer_siginfo *info, void *ctx) { 2030 rtl_generic_sighandler(true, sig, info, ctx); 2031 } 2032 2033 TSAN_INTERCEPTOR(int, raise, int sig) { 2034 SCOPED_TSAN_INTERCEPTOR(raise, sig); 2035 ThreadSignalContext *sctx = SigCtx(thr); 2036 CHECK_NE(sctx, 0); 2037 int prev = sctx->int_signal_send; 2038 sctx->int_signal_send = sig; 2039 int res = REAL(raise)(sig); 2040 CHECK_EQ(sctx->int_signal_send, sig); 2041 sctx->int_signal_send = prev; 2042 return res; 2043 } 2044 2045 TSAN_INTERCEPTOR(int, kill, int pid, int sig) { 2046 SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); 2047 ThreadSignalContext *sctx = SigCtx(thr); 2048 CHECK_NE(sctx, 0); 2049 int prev = sctx->int_signal_send; 2050 if (pid == (int)internal_getpid()) { 2051 sctx->int_signal_send = sig; 2052 } 2053 int res = REAL(kill)(pid, sig); 2054 if (pid == (int)internal_getpid()) { 2055 CHECK_EQ(sctx->int_signal_send, sig); 2056 sctx->int_signal_send = prev; 2057 } 2058 return res; 2059 } 2060 2061 TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { 2062 SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); 2063 ThreadSignalContext *sctx = SigCtx(thr); 2064 CHECK_NE(sctx, 0); 2065 int prev = sctx->int_signal_send; 2066 if (tid == pthread_self()) { 2067 sctx->int_signal_send = sig; 2068 } 2069 int res = REAL(pthread_kill)(tid, sig); 2070 if (tid == pthread_self()) { 2071 CHECK_EQ(sctx->int_signal_send, sig); 2072 sctx->int_signal_send = prev; 2073 } 2074 return res; 2075 } 2076 2077 TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { 2078 SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); 2079 // It's intercepted merely to process pending signals. 2080 return REAL(gettimeofday)(tv, tz); 2081 } 2082 2083 TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, 2084 void *hints, void *rv) { 2085 SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); 2086 // We miss atomic synchronization in getaddrinfo, 2087 // and can report false race between malloc and free 2088 // inside of getaddrinfo. So ignore memory accesses. 2089 ThreadIgnoreBegin(thr, pc); 2090 int res = REAL(getaddrinfo)(node, service, hints, rv); 2091 ThreadIgnoreEnd(thr, pc); 2092 return res; 2093 } 2094 2095 TSAN_INTERCEPTOR(int, fork, int fake) { 2096 if (in_symbolizer()) 2097 return REAL(fork)(fake); 2098 SCOPED_INTERCEPTOR_RAW(fork, fake); 2099 ForkBefore(thr, pc); 2100 int pid; 2101 { 2102 // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and 2103 // we'll assert in CheckNoLocks() unless we ignore interceptors. 2104 ScopedIgnoreInterceptors ignore; 2105 pid = REAL(fork)(fake); 2106 } 2107 if (pid == 0) { 2108 // child 2109 ForkChildAfter(thr, pc); 2110 FdOnFork(thr, pc); 2111 } else if (pid > 0) { 2112 // parent 2113 ForkParentAfter(thr, pc); 2114 } else { 2115 // error 2116 ForkParentAfter(thr, pc); 2117 } 2118 return pid; 2119 } 2120 2121 TSAN_INTERCEPTOR(int, vfork, int fake) { 2122 // Some programs (e.g. openjdk) call close for all file descriptors 2123 // in the child process. Under tsan it leads to false positives, because 2124 // address space is shared, so the parent process also thinks that 2125 // the descriptors are closed (while they are actually not). 2126 // This leads to false positives due to missed synchronization. 2127 // Strictly saying this is undefined behavior, because vfork child is not 2128 // allowed to call any functions other than exec/exit. But this is what 2129 // openjdk does, so we want to handle it. 2130 // We could disable interceptors in the child process. But it's not possible 2131 // to simply intercept and wrap vfork, because vfork child is not allowed 2132 // to return from the function that calls vfork, and that's exactly what 2133 // we would do. So this would require some assembly trickery as well. 2134 // Instead we simply turn vfork into fork. 2135 return WRAP(fork)(fake); 2136 } 2137 2138 #if !SANITIZER_MAC && !SANITIZER_ANDROID 2139 typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size, 2140 void *data); 2141 struct dl_iterate_phdr_data { 2142 ThreadState *thr; 2143 uptr pc; 2144 dl_iterate_phdr_cb_t cb; 2145 void *data; 2146 }; 2147 2148 static bool IsAppNotRodata(uptr addr) { 2149 return IsAppMem(addr) && *(u64*)MemToShadow(addr) != kShadowRodata; 2150 } 2151 2152 static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, 2153 void *data) { 2154 dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data; 2155 // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later 2156 // accessible in dl_iterate_phdr callback. But we don't see synchronization 2157 // inside of dynamic linker, so we "unpoison" it here in order to not 2158 // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough 2159 // because some libc functions call __libc_dlopen. 2160 if (info && IsAppNotRodata((uptr)info->dlpi_name)) 2161 MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, 2162 internal_strlen(info->dlpi_name)); 2163 int res = cbdata->cb(info, size, cbdata->data); 2164 // Perform the check one more time in case info->dlpi_name was overwritten 2165 // by user callback. 2166 if (info && IsAppNotRodata((uptr)info->dlpi_name)) 2167 MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, 2168 internal_strlen(info->dlpi_name)); 2169 return res; 2170 } 2171 2172 TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) { 2173 SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr, cb, data); 2174 dl_iterate_phdr_data cbdata; 2175 cbdata.thr = thr; 2176 cbdata.pc = pc; 2177 cbdata.cb = cb; 2178 cbdata.data = data; 2179 int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata); 2180 return res; 2181 } 2182 #endif 2183 2184 static int OnExit(ThreadState *thr) { 2185 int status = Finalize(thr); 2186 FlushStreams(); 2187 return status; 2188 } 2189 2190 struct TsanInterceptorContext { 2191 ThreadState *thr; 2192 const uptr caller_pc; 2193 const uptr pc; 2194 }; 2195 2196 #if !SANITIZER_MAC 2197 static void HandleRecvmsg(ThreadState *thr, uptr pc, 2198 __sanitizer_msghdr *msg) { 2199 int fds[64]; 2200 int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds)); 2201 for (int i = 0; i < cnt; i++) 2202 FdEventCreate(thr, pc, fds[i]); 2203 } 2204 #endif 2205 2206 #include "sanitizer_common/sanitizer_platform_interceptors.h" 2207 // Causes interceptor recursion (getaddrinfo() and fopen()) 2208 #undef SANITIZER_INTERCEPT_GETADDRINFO 2209 // We define our own. 2210 #if SANITIZER_INTERCEPT_TLS_GET_ADDR 2211 #define NEED_TLS_GET_ADDR 2212 #endif 2213 #undef SANITIZER_INTERCEPT_TLS_GET_ADDR 2214 #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK 2215 2216 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) 2217 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ 2218 INTERCEPT_FUNCTION_VER(name, ver) 2219 2220 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ 2221 MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ 2222 ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ 2223 true) 2224 2225 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ 2226 MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ 2227 ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ 2228 false) 2229 2230 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ 2231 SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ 2232 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ 2233 ctx = (void *)&_ctx; \ 2234 (void) ctx; 2235 2236 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ 2237 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ 2238 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ 2239 ctx = (void *)&_ctx; \ 2240 (void) ctx; 2241 2242 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ 2243 if (path) \ 2244 Acquire(thr, pc, File2addr(path)); \ 2245 if (file) { \ 2246 int fd = fileno_unlocked(file); \ 2247 if (fd >= 0) FdFileCreate(thr, pc, fd); \ 2248 } 2249 2250 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ 2251 if (file) { \ 2252 int fd = fileno_unlocked(file); \ 2253 if (fd >= 0) FdClose(thr, pc, fd); \ 2254 } 2255 2256 #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ 2257 libignore()->OnLibraryLoaded(filename) 2258 2259 #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ 2260 libignore()->OnLibraryUnloaded() 2261 2262 #define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \ 2263 Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u) 2264 2265 #define COMMON_INTERCEPTOR_RELEASE(ctx, u) \ 2266 Release(((TsanInterceptorContext *) ctx)->thr, pc, u) 2267 2268 #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ 2269 Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path)) 2270 2271 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ 2272 FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2273 2274 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ 2275 FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2276 2277 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ 2278 FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2279 2280 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ 2281 FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) 2282 2283 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ 2284 ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) 2285 2286 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ 2287 __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name) 2288 2289 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) 2290 2291 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \ 2292 OnExit(((TsanInterceptorContext *) ctx)->thr) 2293 2294 #define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \ 2295 MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \ 2296 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2297 2298 #define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \ 2299 MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \ 2300 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2301 2302 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ 2303 MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \ 2304 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2305 2306 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ 2307 MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ 2308 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2309 2310 #define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \ 2311 MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \ 2312 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2313 2314 #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ 2315 off) \ 2316 do { \ 2317 return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \ 2318 off); \ 2319 } while (false) 2320 2321 #if !SANITIZER_MAC 2322 #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ 2323 HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ 2324 ((TsanInterceptorContext *)ctx)->pc, msg) 2325 #endif 2326 2327 #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ 2328 if (TsanThread *t = GetCurrentThread()) { \ 2329 *begin = t->tls_begin(); \ 2330 *end = t->tls_end(); \ 2331 } else { \ 2332 *begin = *end = 0; \ 2333 } 2334 2335 #define COMMON_INTERCEPTOR_USER_CALLBACK_START() \ 2336 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() 2337 2338 #define COMMON_INTERCEPTOR_USER_CALLBACK_END() \ 2339 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() 2340 2341 #include "sanitizer_common/sanitizer_common_interceptors.inc" 2342 2343 static int sigaction_impl(int sig, const __sanitizer_sigaction *act, 2344 __sanitizer_sigaction *old); 2345 static __sanitizer_sighandler_ptr signal_impl(int sig, 2346 __sanitizer_sighandler_ptr h); 2347 2348 #define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \ 2349 { return sigaction_impl(signo, act, oldact); } 2350 2351 #define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ 2352 { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); } 2353 2354 #include "sanitizer_common/sanitizer_signal_interceptors.inc" 2355 2356 int sigaction_impl(int sig, const __sanitizer_sigaction *act, 2357 __sanitizer_sigaction *old) { 2358 // Note: if we call REAL(sigaction) directly for any reason without proxying 2359 // the signal handler through rtl_sigaction, very bad things will happen. 2360 // The handler will run synchronously and corrupt tsan per-thread state. 2361 SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); 2362 __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; 2363 __sanitizer_sigaction old_stored; 2364 if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored)); 2365 __sanitizer_sigaction newact; 2366 if (act) { 2367 // Copy act into sigactions[sig]. 2368 // Can't use struct copy, because compiler can emit call to memcpy. 2369 // Can't use internal_memcpy, because it copies byte-by-byte, 2370 // and signal handler reads the handler concurrently. It it can read 2371 // some bytes from old value and some bytes from new value. 2372 // Use volatile to prevent insertion of memcpy. 2373 sigactions[sig].handler = 2374 *(volatile __sanitizer_sighandler_ptr const *)&act->handler; 2375 sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags; 2376 internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, 2377 sizeof(sigactions[sig].sa_mask)); 2378 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD 2379 sigactions[sig].sa_restorer = act->sa_restorer; 2380 #endif 2381 internal_memcpy(&newact, act, sizeof(newact)); 2382 internal_sigfillset(&newact.sa_mask); 2383 if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) { 2384 if (newact.sa_flags & SA_SIGINFO) 2385 newact.sigaction = rtl_sigaction; 2386 else 2387 newact.handler = rtl_sighandler; 2388 } 2389 ReleaseStore(thr, pc, (uptr)&sigactions[sig]); 2390 act = &newact; 2391 } 2392 int res = REAL(sigaction)(sig, act, old); 2393 if (res == 0 && old) { 2394 uptr cb = (uptr)old->sigaction; 2395 if (cb == (uptr)rtl_sigaction || cb == (uptr)rtl_sighandler) { 2396 internal_memcpy(old, &old_stored, sizeof(*old)); 2397 } 2398 } 2399 return res; 2400 } 2401 2402 static __sanitizer_sighandler_ptr signal_impl(int sig, 2403 __sanitizer_sighandler_ptr h) { 2404 __sanitizer_sigaction act; 2405 act.handler = h; 2406 internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); 2407 act.sa_flags = 0; 2408 __sanitizer_sigaction old; 2409 int res = sigaction_symname(sig, &act, &old); 2410 if (res) return (__sanitizer_sighandler_ptr)sig_err; 2411 return old.handler; 2412 } 2413 2414 #define TSAN_SYSCALL() \ 2415 ThreadState *thr = cur_thread(); \ 2416 if (thr->ignore_interceptors) \ 2417 return; \ 2418 ScopedSyscall scoped_syscall(thr) \ 2419 /**/ 2420 2421 struct ScopedSyscall { 2422 ThreadState *thr; 2423 2424 explicit ScopedSyscall(ThreadState *thr) 2425 : thr(thr) { 2426 Initialize(thr); 2427 } 2428 2429 ~ScopedSyscall() { 2430 ProcessPendingSignals(thr); 2431 } 2432 }; 2433 2434 #if !SANITIZER_FREEBSD && !SANITIZER_MAC 2435 static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { 2436 TSAN_SYSCALL(); 2437 MemoryAccessRange(thr, pc, p, s, write); 2438 } 2439 2440 static void syscall_acquire(uptr pc, uptr addr) { 2441 TSAN_SYSCALL(); 2442 Acquire(thr, pc, addr); 2443 DPrintf("syscall_acquire(%p)\n", addr); 2444 } 2445 2446 static void syscall_release(uptr pc, uptr addr) { 2447 TSAN_SYSCALL(); 2448 DPrintf("syscall_release(%p)\n", addr); 2449 Release(thr, pc, addr); 2450 } 2451 2452 static void syscall_fd_close(uptr pc, int fd) { 2453 TSAN_SYSCALL(); 2454 FdClose(thr, pc, fd); 2455 } 2456 2457 static USED void syscall_fd_acquire(uptr pc, int fd) { 2458 TSAN_SYSCALL(); 2459 FdAcquire(thr, pc, fd); 2460 DPrintf("syscall_fd_acquire(%p)\n", fd); 2461 } 2462 2463 static USED void syscall_fd_release(uptr pc, int fd) { 2464 TSAN_SYSCALL(); 2465 DPrintf("syscall_fd_release(%p)\n", fd); 2466 FdRelease(thr, pc, fd); 2467 } 2468 2469 static void syscall_pre_fork(uptr pc) { 2470 TSAN_SYSCALL(); 2471 ForkBefore(thr, pc); 2472 } 2473 2474 static void syscall_post_fork(uptr pc, int pid) { 2475 TSAN_SYSCALL(); 2476 if (pid == 0) { 2477 // child 2478 ForkChildAfter(thr, pc); 2479 FdOnFork(thr, pc); 2480 } else if (pid > 0) { 2481 // parent 2482 ForkParentAfter(thr, pc); 2483 } else { 2484 // error 2485 ForkParentAfter(thr, pc); 2486 } 2487 } 2488 #endif 2489 2490 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ 2491 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) 2492 2493 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ 2494 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) 2495 2496 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ 2497 do { \ 2498 (void)(p); \ 2499 (void)(s); \ 2500 } while (false) 2501 2502 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ 2503 do { \ 2504 (void)(p); \ 2505 (void)(s); \ 2506 } while (false) 2507 2508 #define COMMON_SYSCALL_ACQUIRE(addr) \ 2509 syscall_acquire(GET_CALLER_PC(), (uptr)(addr)) 2510 2511 #define COMMON_SYSCALL_RELEASE(addr) \ 2512 syscall_release(GET_CALLER_PC(), (uptr)(addr)) 2513 2514 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd) 2515 2516 #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd) 2517 2518 #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd) 2519 2520 #define COMMON_SYSCALL_PRE_FORK() \ 2521 syscall_pre_fork(GET_CALLER_PC()) 2522 2523 #define COMMON_SYSCALL_POST_FORK(res) \ 2524 syscall_post_fork(GET_CALLER_PC(), res) 2525 2526 #include "sanitizer_common/sanitizer_common_syscalls.inc" 2527 #include "sanitizer_common/sanitizer_syscalls_netbsd.inc" 2528 2529 #ifdef NEED_TLS_GET_ADDR 2530 // Define own interceptor instead of sanitizer_common's for three reasons: 2531 // 1. It must not process pending signals. 2532 // Signal handlers may contain MOVDQA instruction (see below). 2533 // 2. It must be as simple as possible to not contain MOVDQA. 2534 // 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which 2535 // is empty for tsan (meant only for msan). 2536 // Note: __tls_get_addr can be called with mis-aligned stack due to: 2537 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 2538 // So the interceptor must work with mis-aligned stack, in particular, does not 2539 // execute MOVDQA with stack addresses. 2540 TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { 2541 void *res = REAL(__tls_get_addr)(arg); 2542 ThreadState *thr = cur_thread(); 2543 if (!thr) 2544 return res; 2545 DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, 2546 thr->tls_addr + thr->tls_size); 2547 if (!dtv) 2548 return res; 2549 // New DTLS block has been allocated. 2550 MemoryResetRange(thr, 0, dtv->beg, dtv->size); 2551 return res; 2552 } 2553 #endif 2554 2555 #if SANITIZER_NETBSD 2556 TSAN_INTERCEPTOR(void, _lwp_exit) { 2557 SCOPED_TSAN_INTERCEPTOR(_lwp_exit); 2558 DestroyThreadState(); 2559 REAL(_lwp_exit)(); 2560 } 2561 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit) 2562 #else 2563 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT 2564 #endif 2565 2566 #if SANITIZER_FREEBSD 2567 TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) { 2568 SCOPED_TSAN_INTERCEPTOR(thr_exit, state); 2569 DestroyThreadState(); 2570 REAL(thr_exit(state)); 2571 } 2572 #define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit) 2573 #else 2574 #define TSAN_MAYBE_INTERCEPT_THR_EXIT 2575 #endif 2576 2577 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a) 2578 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c) 2579 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c) 2580 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m) 2581 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c) 2582 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a) 2583 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m) 2584 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m) 2585 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a) 2586 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m) 2587 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m) 2588 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m) 2589 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m) 2590 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m) 2591 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m) 2592 TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)()) 2593 TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b, 2594 void *c) 2595 2596 namespace __tsan { 2597 2598 static void finalize(void *arg) { 2599 ThreadState *thr = cur_thread(); 2600 int status = Finalize(thr); 2601 // Make sure the output is not lost. 2602 FlushStreams(); 2603 if (status) 2604 Die(); 2605 } 2606 2607 #if !SANITIZER_MAC && !SANITIZER_ANDROID 2608 static void unreachable() { 2609 Report("FATAL: ThreadSanitizer: unreachable called\n"); 2610 Die(); 2611 } 2612 #endif 2613 2614 // Define default implementation since interception of libdispatch is optional. 2615 SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} 2616 2617 void InitializeInterceptors() { 2618 #if !SANITIZER_MAC 2619 // We need to setup it early, because functions like dlsym() can call it. 2620 REAL(memset) = internal_memset; 2621 REAL(memcpy) = internal_memcpy; 2622 #endif 2623 2624 // Instruct libc malloc to consume less memory. 2625 #if SANITIZER_LINUX 2626 mallopt(1, 0); // M_MXFAST 2627 mallopt(-3, 32*1024); // M_MMAP_THRESHOLD 2628 #endif 2629 2630 new(interceptor_ctx()) InterceptorContext(); 2631 2632 InitializeCommonInterceptors(); 2633 InitializeSignalInterceptors(); 2634 InitializeLibdispatchInterceptors(); 2635 2636 #if !SANITIZER_MAC 2637 // We can not use TSAN_INTERCEPT to get setjmp addr, 2638 // because it does &setjmp and setjmp is not present in some versions of libc. 2639 using __interception::InterceptFunction; 2640 InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); 2641 InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); 2642 InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, 2643 0); 2644 #if !SANITIZER_NETBSD 2645 InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); 2646 #endif 2647 #endif 2648 2649 TSAN_INTERCEPT(longjmp_symname); 2650 TSAN_INTERCEPT(siglongjmp_symname); 2651 #if SANITIZER_NETBSD 2652 TSAN_INTERCEPT(_longjmp); 2653 #endif 2654 2655 TSAN_INTERCEPT(malloc); 2656 TSAN_INTERCEPT(__libc_memalign); 2657 TSAN_INTERCEPT(calloc); 2658 TSAN_INTERCEPT(realloc); 2659 TSAN_INTERCEPT(reallocarray); 2660 TSAN_INTERCEPT(free); 2661 TSAN_INTERCEPT(cfree); 2662 TSAN_INTERCEPT(munmap); 2663 TSAN_MAYBE_INTERCEPT_MEMALIGN; 2664 TSAN_INTERCEPT(valloc); 2665 TSAN_MAYBE_INTERCEPT_PVALLOC; 2666 TSAN_INTERCEPT(posix_memalign); 2667 2668 TSAN_INTERCEPT(strcpy); 2669 TSAN_INTERCEPT(strncpy); 2670 TSAN_INTERCEPT(strdup); 2671 2672 TSAN_INTERCEPT(pthread_create); 2673 TSAN_INTERCEPT(pthread_join); 2674 TSAN_INTERCEPT(pthread_detach); 2675 TSAN_INTERCEPT(pthread_exit); 2676 #if SANITIZER_LINUX 2677 TSAN_INTERCEPT(pthread_tryjoin_np); 2678 TSAN_INTERCEPT(pthread_timedjoin_np); 2679 #endif 2680 2681 TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE); 2682 TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE); 2683 TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE); 2684 TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE); 2685 TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE); 2686 TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE); 2687 2688 TSAN_INTERCEPT(pthread_mutex_init); 2689 TSAN_INTERCEPT(pthread_mutex_destroy); 2690 TSAN_INTERCEPT(pthread_mutex_trylock); 2691 TSAN_INTERCEPT(pthread_mutex_timedlock); 2692 2693 TSAN_INTERCEPT(pthread_spin_init); 2694 TSAN_INTERCEPT(pthread_spin_destroy); 2695 TSAN_INTERCEPT(pthread_spin_lock); 2696 TSAN_INTERCEPT(pthread_spin_trylock); 2697 TSAN_INTERCEPT(pthread_spin_unlock); 2698 2699 TSAN_INTERCEPT(pthread_rwlock_init); 2700 TSAN_INTERCEPT(pthread_rwlock_destroy); 2701 TSAN_INTERCEPT(pthread_rwlock_rdlock); 2702 TSAN_INTERCEPT(pthread_rwlock_tryrdlock); 2703 TSAN_INTERCEPT(pthread_rwlock_timedrdlock); 2704 TSAN_INTERCEPT(pthread_rwlock_wrlock); 2705 TSAN_INTERCEPT(pthread_rwlock_trywrlock); 2706 TSAN_INTERCEPT(pthread_rwlock_timedwrlock); 2707 TSAN_INTERCEPT(pthread_rwlock_unlock); 2708 2709 TSAN_INTERCEPT(pthread_barrier_init); 2710 TSAN_INTERCEPT(pthread_barrier_destroy); 2711 TSAN_INTERCEPT(pthread_barrier_wait); 2712 2713 TSAN_INTERCEPT(pthread_once); 2714 2715 TSAN_INTERCEPT(fstat); 2716 TSAN_MAYBE_INTERCEPT___FXSTAT; 2717 TSAN_MAYBE_INTERCEPT_FSTAT64; 2718 TSAN_MAYBE_INTERCEPT___FXSTAT64; 2719 TSAN_INTERCEPT(open); 2720 TSAN_MAYBE_INTERCEPT_OPEN64; 2721 TSAN_INTERCEPT(creat); 2722 TSAN_MAYBE_INTERCEPT_CREAT64; 2723 TSAN_INTERCEPT(dup); 2724 TSAN_INTERCEPT(dup2); 2725 TSAN_INTERCEPT(dup3); 2726 TSAN_MAYBE_INTERCEPT_EVENTFD; 2727 TSAN_MAYBE_INTERCEPT_SIGNALFD; 2728 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT; 2729 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1; 2730 TSAN_INTERCEPT(socket); 2731 TSAN_INTERCEPT(socketpair); 2732 TSAN_INTERCEPT(connect); 2733 TSAN_INTERCEPT(bind); 2734 TSAN_INTERCEPT(listen); 2735 TSAN_MAYBE_INTERCEPT_EPOLL; 2736 TSAN_INTERCEPT(close); 2737 TSAN_MAYBE_INTERCEPT___CLOSE; 2738 TSAN_MAYBE_INTERCEPT___RES_ICLOSE; 2739 TSAN_INTERCEPT(pipe); 2740 TSAN_INTERCEPT(pipe2); 2741 2742 TSAN_INTERCEPT(unlink); 2743 TSAN_INTERCEPT(tmpfile); 2744 TSAN_MAYBE_INTERCEPT_TMPFILE64; 2745 TSAN_INTERCEPT(abort); 2746 TSAN_INTERCEPT(rmdir); 2747 TSAN_INTERCEPT(closedir); 2748 2749 TSAN_INTERCEPT(sigsuspend); 2750 TSAN_INTERCEPT(sigblock); 2751 TSAN_INTERCEPT(sigsetmask); 2752 TSAN_INTERCEPT(pthread_sigmask); 2753 TSAN_INTERCEPT(raise); 2754 TSAN_INTERCEPT(kill); 2755 TSAN_INTERCEPT(pthread_kill); 2756 TSAN_INTERCEPT(sleep); 2757 TSAN_INTERCEPT(usleep); 2758 TSAN_INTERCEPT(nanosleep); 2759 TSAN_INTERCEPT(pause); 2760 TSAN_INTERCEPT(gettimeofday); 2761 TSAN_INTERCEPT(getaddrinfo); 2762 2763 TSAN_INTERCEPT(fork); 2764 TSAN_INTERCEPT(vfork); 2765 #if !SANITIZER_ANDROID 2766 TSAN_INTERCEPT(dl_iterate_phdr); 2767 #endif 2768 TSAN_MAYBE_INTERCEPT_ON_EXIT; 2769 TSAN_INTERCEPT(__cxa_atexit); 2770 TSAN_INTERCEPT(_exit); 2771 2772 #ifdef NEED_TLS_GET_ADDR 2773 TSAN_INTERCEPT(__tls_get_addr); 2774 #endif 2775 2776 TSAN_MAYBE_INTERCEPT__LWP_EXIT; 2777 TSAN_MAYBE_INTERCEPT_THR_EXIT; 2778 2779 #if !SANITIZER_MAC && !SANITIZER_ANDROID 2780 // Need to setup it, because interceptors check that the function is resolved. 2781 // But atexit is emitted directly into the module, so can't be resolved. 2782 REAL(atexit) = (int(*)(void(*)()))unreachable; 2783 #endif 2784 2785 if (REAL(__cxa_atexit)(&finalize, 0, 0)) { 2786 Printf("ThreadSanitizer: failed to setup atexit callback\n"); 2787 Die(); 2788 } 2789 2790 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD 2791 if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) { 2792 Printf("ThreadSanitizer: failed to create thread key\n"); 2793 Die(); 2794 } 2795 #endif 2796 2797 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init); 2798 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal); 2799 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast); 2800 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait); 2801 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy); 2802 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init); 2803 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy); 2804 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock); 2805 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init); 2806 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy); 2807 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock); 2808 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock); 2809 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock); 2810 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock); 2811 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock); 2812 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once); 2813 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask); 2814 2815 FdInit(); 2816 } 2817 2818 } // namespace __tsan 2819 2820 // Invisible barrier for tests. 2821 // There were several unsuccessful iterations for this functionality: 2822 // 1. Initially it was implemented in user code using 2823 // REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on 2824 // MacOS. Futexes are linux-specific for this matter. 2825 // 2. Then we switched to atomics+usleep(10). But usleep produced parasitic 2826 // "as-if synchronized via sleep" messages in reports which failed some 2827 // output tests. 2828 // 3. Then we switched to atomics+sched_yield. But this produced tons of tsan- 2829 // visible events, which lead to "failed to restore stack trace" failures. 2830 // Note that no_sanitize_thread attribute does not turn off atomic interception 2831 // so attaching it to the function defined in user code does not help. 2832 // That's why we now have what we have. 2833 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 2834 void __tsan_testonly_barrier_init(u64 *barrier, u32 count) { 2835 if (count >= (1 << 8)) { 2836 Printf("barrier_init: count is too large (%d)\n", count); 2837 Die(); 2838 } 2839 // 8 lsb is thread count, the remaining are count of entered threads. 2840 *barrier = count; 2841 } 2842 2843 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 2844 void __tsan_testonly_barrier_wait(u64 *barrier) { 2845 unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED); 2846 unsigned old_epoch = (old >> 8) / (old & 0xff); 2847 for (;;) { 2848 unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED); 2849 unsigned cur_epoch = (cur >> 8) / (cur & 0xff); 2850 if (cur_epoch != old_epoch) 2851 return; 2852 internal_sched_yield(); 2853 } 2854 } 2855