1 //===-- tsan_interceptors_posix.cpp ---------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // FIXME: move as many interceptors as possible into 12 // sanitizer_common/sanitizer_common_interceptors.inc 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_common/sanitizer_atomic.h" 16 #include "sanitizer_common/sanitizer_errno.h" 17 #include "sanitizer_common/sanitizer_libc.h" 18 #include "sanitizer_common/sanitizer_linux.h" 19 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h" 20 #include "sanitizer_common/sanitizer_platform_limits_posix.h" 21 #include "sanitizer_common/sanitizer_placement_new.h" 22 #include "sanitizer_common/sanitizer_posix.h" 23 #include "sanitizer_common/sanitizer_stacktrace.h" 24 #include "sanitizer_common/sanitizer_tls_get_addr.h" 25 #include "interception/interception.h" 26 #include "tsan_interceptors.h" 27 #include "tsan_interface.h" 28 #include "tsan_platform.h" 29 #include "tsan_suppressions.h" 30 #include "tsan_rtl.h" 31 #include "tsan_mman.h" 32 #include "tsan_fd.h" 33 34 #include <stdarg.h> 35 36 using namespace __tsan; 37 38 #if SANITIZER_FREEBSD || SANITIZER_APPLE 39 #define stdout __stdoutp 40 #define stderr __stderrp 41 #endif 42 43 #if SANITIZER_NETBSD 44 #define dirfd(dirp) (*(int *)(dirp)) 45 #define fileno_unlocked(fp) \ 46 (((__sanitizer_FILE *)fp)->_file == -1 \ 47 ? -1 \ 48 : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file)) 49 50 #define stdout ((__sanitizer_FILE*)&__sF[1]) 51 #define stderr ((__sanitizer_FILE*)&__sF[2]) 52 53 #define nanosleep __nanosleep50 54 #define vfork __vfork14 55 #endif 56 57 #ifdef __mips__ 58 const int kSigCount = 129; 59 #else 60 const int kSigCount = 65; 61 #endif 62 63 #ifdef __mips__ 64 struct ucontext_t { 65 u64 opaque[768 / sizeof(u64) + 1]; 66 }; 67 #else 68 struct ucontext_t { 69 // The size is determined by looking at sizeof of real ucontext_t on linux. 70 u64 opaque[936 / sizeof(u64) + 1]; 71 }; 72 #endif 73 74 #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \ 75 defined(__s390x__) 76 #define PTHREAD_ABI_BASE "GLIBC_2.3.2" 77 #elif defined(__aarch64__) || SANITIZER_PPC64V2 78 #define PTHREAD_ABI_BASE "GLIBC_2.17" 79 #endif 80 81 extern "C" int pthread_attr_init(void *attr); 82 extern "C" int pthread_attr_destroy(void *attr); 83 DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *) 84 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); 85 extern "C" int pthread_atfork(void (*prepare)(void), void (*parent)(void), 86 void (*child)(void)); 87 extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); 88 extern "C" int pthread_setspecific(unsigned key, const void *v); 89 DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) 90 DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) 91 DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) 92 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) 93 extern "C" int pthread_equal(void *t1, void *t2); 94 extern "C" void *pthread_self(); 95 extern "C" void _exit(int status); 96 #if !SANITIZER_NETBSD 97 extern "C" int fileno_unlocked(void *stream); 98 extern "C" int dirfd(void *dirp); 99 #endif 100 #if SANITIZER_NETBSD 101 extern __sanitizer_FILE __sF[]; 102 #else 103 extern __sanitizer_FILE *stdout, *stderr; 104 #endif 105 #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD 106 const int PTHREAD_MUTEX_RECURSIVE = 1; 107 const int PTHREAD_MUTEX_RECURSIVE_NP = 1; 108 #else 109 const int PTHREAD_MUTEX_RECURSIVE = 2; 110 const int PTHREAD_MUTEX_RECURSIVE_NP = 2; 111 #endif 112 #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD 113 const int EPOLL_CTL_ADD = 1; 114 #endif 115 const int SIGILL = 4; 116 const int SIGTRAP = 5; 117 const int SIGABRT = 6; 118 const int SIGFPE = 8; 119 const int SIGSEGV = 11; 120 const int SIGPIPE = 13; 121 const int SIGTERM = 15; 122 #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD 123 const int SIGBUS = 10; 124 const int SIGSYS = 12; 125 #else 126 const int SIGBUS = 7; 127 const int SIGSYS = 31; 128 #endif 129 void *const MAP_FAILED = (void*)-1; 130 #if SANITIZER_NETBSD 131 const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; 132 #elif !SANITIZER_APPLE 133 const int PTHREAD_BARRIER_SERIAL_THREAD = -1; 134 #endif 135 const int MAP_FIXED = 0x10; 136 typedef long long_t; 137 typedef __sanitizer::u16 mode_t; 138 139 // From /usr/include/unistd.h 140 # define F_ULOCK 0 /* Unlock a previously locked region. */ 141 # define F_LOCK 1 /* Lock a region for exclusive use. */ 142 # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ 143 # define F_TEST 3 /* Test a region for other processes locks. */ 144 145 #if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD 146 const int SA_SIGINFO = 0x40; 147 const int SIG_SETMASK = 3; 148 #elif defined(__mips__) 149 const int SA_SIGINFO = 8; 150 const int SIG_SETMASK = 3; 151 #else 152 const int SA_SIGINFO = 4; 153 const int SIG_SETMASK = 2; 154 #endif 155 156 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ 157 (!cur_thread_init()->is_inited) 158 159 namespace __tsan { 160 struct SignalDesc { 161 bool armed; 162 __sanitizer_siginfo siginfo; 163 ucontext_t ctx; 164 }; 165 166 struct ThreadSignalContext { 167 int int_signal_send; 168 atomic_uintptr_t in_blocking_func; 169 SignalDesc pending_signals[kSigCount]; 170 // emptyset and oldset are too big for stack. 171 __sanitizer_sigset_t emptyset; 172 __sanitizer_sigset_t oldset; 173 }; 174 175 // The sole reason tsan wraps atexit callbacks is to establish synchronization 176 // between callback setup and callback execution. 177 struct AtExitCtx { 178 void (*f)(); 179 void *arg; 180 uptr pc; 181 }; 182 183 // InterceptorContext holds all global data required for interceptors. 184 // It's explicitly constructed in InitializeInterceptors with placement new 185 // and is never destroyed. This allows usage of members with non-trivial 186 // constructors and destructors. 187 struct InterceptorContext { 188 // The object is 64-byte aligned, because we want hot data to be located 189 // in a single cache line if possible (it's accessed in every interceptor). 190 ALIGNED(64) LibIgnore libignore; 191 __sanitizer_sigaction sigactions[kSigCount]; 192 #if !SANITIZER_APPLE && !SANITIZER_NETBSD 193 unsigned finalize_key; 194 #endif 195 196 Mutex atexit_mu; 197 Vector<struct AtExitCtx *> AtExitStack; 198 199 InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {} 200 }; 201 202 static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)]; 203 InterceptorContext *interceptor_ctx() { 204 return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]); 205 } 206 207 LibIgnore *libignore() { 208 return &interceptor_ctx()->libignore; 209 } 210 211 void InitializeLibIgnore() { 212 const SuppressionContext &supp = *Suppressions(); 213 const uptr n = supp.SuppressionCount(); 214 for (uptr i = 0; i < n; i++) { 215 const Suppression *s = supp.SuppressionAt(i); 216 if (0 == internal_strcmp(s->type, kSuppressionLib)) 217 libignore()->AddIgnoredLibrary(s->templ); 218 } 219 if (flags()->ignore_noninstrumented_modules) 220 libignore()->IgnoreNoninstrumentedModules(true); 221 libignore()->OnLibraryLoaded(0); 222 } 223 224 // The following two hooks can be used by for cooperative scheduling when 225 // locking. 226 #ifdef TSAN_EXTERNAL_HOOKS 227 void OnPotentiallyBlockingRegionBegin(); 228 void OnPotentiallyBlockingRegionEnd(); 229 #else 230 SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {} 231 SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {} 232 #endif 233 234 } // namespace __tsan 235 236 static ThreadSignalContext *SigCtx(ThreadState *thr) { 237 ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx; 238 if (ctx == 0 && !thr->is_dead) { 239 ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext"); 240 MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); 241 thr->signal_ctx = ctx; 242 } 243 return ctx; 244 } 245 246 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, 247 uptr pc) 248 : thr_(thr), in_ignored_lib_(false), ignoring_(false) { 249 LazyInitialize(thr); 250 if (!thr_->is_inited) return; 251 if (!thr_->ignore_interceptors) FuncEntry(thr, pc); 252 DPrintf("#%d: intercept %s()\n", thr_->tid, fname); 253 ignoring_ = 254 !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || 255 libignore()->IsIgnored(pc, &in_ignored_lib_)); 256 EnableIgnores(); 257 } 258 259 ScopedInterceptor::~ScopedInterceptor() { 260 if (!thr_->is_inited) return; 261 DisableIgnores(); 262 if (!thr_->ignore_interceptors) { 263 ProcessPendingSignals(thr_); 264 FuncExit(thr_); 265 CheckedMutex::CheckNoLocks(); 266 } 267 } 268 269 NOINLINE 270 void ScopedInterceptor::EnableIgnoresImpl() { 271 ThreadIgnoreBegin(thr_, 0); 272 if (flags()->ignore_noninstrumented_modules) 273 thr_->suppress_reports++; 274 if (in_ignored_lib_) { 275 DCHECK(!thr_->in_ignored_lib); 276 thr_->in_ignored_lib = true; 277 } 278 } 279 280 NOINLINE 281 void ScopedInterceptor::DisableIgnoresImpl() { 282 ThreadIgnoreEnd(thr_); 283 if (flags()->ignore_noninstrumented_modules) 284 thr_->suppress_reports--; 285 if (in_ignored_lib_) { 286 DCHECK(thr_->in_ignored_lib); 287 thr_->in_ignored_lib = false; 288 } 289 } 290 291 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) 292 #if SANITIZER_FREEBSD || SANITIZER_NETBSD 293 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) 294 #else 295 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) 296 #endif 297 #if SANITIZER_FREEBSD 298 # define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \ 299 INTERCEPT_FUNCTION(_pthread_##func) 300 #else 301 # define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) 302 #endif 303 #if SANITIZER_NETBSD 304 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ 305 INTERCEPT_FUNCTION(__libc_##func) 306 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ 307 INTERCEPT_FUNCTION(__libc_thr_##func) 308 #else 309 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) 310 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) 311 #endif 312 313 #define READ_STRING_OF_LEN(thr, pc, s, len, n) \ 314 MemoryAccessRange((thr), (pc), (uptr)(s), \ 315 common_flags()->strict_string_checks ? (len) + 1 : (n), false) 316 317 #define READ_STRING(thr, pc, s, n) \ 318 READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n)) 319 320 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) 321 322 struct BlockingCall { 323 explicit BlockingCall(ThreadState *thr) 324 : thr(thr) 325 , ctx(SigCtx(thr)) { 326 for (;;) { 327 atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed); 328 if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0) 329 break; 330 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 331 ProcessPendingSignals(thr); 332 } 333 // When we are in a "blocking call", we process signals asynchronously 334 // (right when they arrive). In this context we do not expect to be 335 // executing any user/runtime code. The known interceptor sequence when 336 // this is not true is: pthread_join -> munmap(stack). It's fine 337 // to ignore munmap in this case -- we handle stack shadow separately. 338 thr->ignore_interceptors++; 339 } 340 341 ~BlockingCall() { 342 thr->ignore_interceptors--; 343 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 344 } 345 346 ThreadState *thr; 347 ThreadSignalContext *ctx; 348 }; 349 350 TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { 351 SCOPED_TSAN_INTERCEPTOR(sleep, sec); 352 unsigned res = BLOCK_REAL(sleep)(sec); 353 AfterSleep(thr, pc); 354 return res; 355 } 356 357 TSAN_INTERCEPTOR(int, usleep, long_t usec) { 358 SCOPED_TSAN_INTERCEPTOR(usleep, usec); 359 int res = BLOCK_REAL(usleep)(usec); 360 AfterSleep(thr, pc); 361 return res; 362 } 363 364 TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { 365 SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); 366 int res = BLOCK_REAL(nanosleep)(req, rem); 367 AfterSleep(thr, pc); 368 return res; 369 } 370 371 TSAN_INTERCEPTOR(int, pause, int fake) { 372 SCOPED_TSAN_INTERCEPTOR(pause, fake); 373 return BLOCK_REAL(pause)(fake); 374 } 375 376 // Note: we specifically call the function in such strange way 377 // with "installed_at" because in reports it will appear between 378 // callback frames and the frame that installed the callback. 379 static void at_exit_callback_installed_at() { 380 AtExitCtx *ctx; 381 { 382 // Ensure thread-safety. 383 Lock l(&interceptor_ctx()->atexit_mu); 384 385 // Pop AtExitCtx from the top of the stack of callback functions 386 uptr element = interceptor_ctx()->AtExitStack.Size() - 1; 387 ctx = interceptor_ctx()->AtExitStack[element]; 388 interceptor_ctx()->AtExitStack.PopBack(); 389 } 390 391 ThreadState *thr = cur_thread(); 392 Acquire(thr, ctx->pc, (uptr)ctx); 393 FuncEntry(thr, ctx->pc); 394 ((void(*)())ctx->f)(); 395 FuncExit(thr); 396 Free(ctx); 397 } 398 399 static void cxa_at_exit_callback_installed_at(void *arg) { 400 ThreadState *thr = cur_thread(); 401 AtExitCtx *ctx = (AtExitCtx*)arg; 402 Acquire(thr, ctx->pc, (uptr)arg); 403 FuncEntry(thr, ctx->pc); 404 ((void(*)(void *arg))ctx->f)(ctx->arg); 405 FuncExit(thr); 406 Free(ctx); 407 } 408 409 static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), 410 void *arg, void *dso); 411 412 #if !SANITIZER_ANDROID 413 TSAN_INTERCEPTOR(int, atexit, void (*f)()) { 414 if (in_symbolizer()) 415 return 0; 416 // We want to setup the atexit callback even if we are in ignored lib 417 // or after fork. 418 SCOPED_INTERCEPTOR_RAW(atexit, f); 419 return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0); 420 } 421 #endif 422 423 TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { 424 if (in_symbolizer()) 425 return 0; 426 SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); 427 return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso); 428 } 429 430 static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), 431 void *arg, void *dso) { 432 auto *ctx = New<AtExitCtx>(); 433 ctx->f = f; 434 ctx->arg = arg; 435 ctx->pc = pc; 436 Release(thr, pc, (uptr)ctx); 437 // Memory allocation in __cxa_atexit will race with free during exit, 438 // because we do not see synchronization around atexit callback list. 439 ThreadIgnoreBegin(thr, pc); 440 int res; 441 if (!dso) { 442 // NetBSD does not preserve the 2nd argument if dso is equal to 0 443 // Store ctx in a local stack-like structure 444 445 // Ensure thread-safety. 446 Lock l(&interceptor_ctx()->atexit_mu); 447 // __cxa_atexit calls calloc. If we don't ignore interceptors, we will fail 448 // due to atexit_mu held on exit from the calloc interceptor. 449 ScopedIgnoreInterceptors ignore; 450 451 res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_callback_installed_at, 452 0, 0); 453 // Push AtExitCtx on the top of the stack of callback functions 454 if (!res) { 455 interceptor_ctx()->AtExitStack.PushBack(ctx); 456 } 457 } else { 458 res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso); 459 } 460 ThreadIgnoreEnd(thr); 461 return res; 462 } 463 464 #if !SANITIZER_APPLE && !SANITIZER_NETBSD 465 static void on_exit_callback_installed_at(int status, void *arg) { 466 ThreadState *thr = cur_thread(); 467 AtExitCtx *ctx = (AtExitCtx*)arg; 468 Acquire(thr, ctx->pc, (uptr)arg); 469 FuncEntry(thr, ctx->pc); 470 ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); 471 FuncExit(thr); 472 Free(ctx); 473 } 474 475 TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { 476 if (in_symbolizer()) 477 return 0; 478 SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); 479 auto *ctx = New<AtExitCtx>(); 480 ctx->f = (void(*)())f; 481 ctx->arg = arg; 482 ctx->pc = GET_CALLER_PC(); 483 Release(thr, pc, (uptr)ctx); 484 // Memory allocation in __cxa_atexit will race with free during exit, 485 // because we do not see synchronization around atexit callback list. 486 ThreadIgnoreBegin(thr, pc); 487 int res = REAL(on_exit)(on_exit_callback_installed_at, ctx); 488 ThreadIgnoreEnd(thr); 489 return res; 490 } 491 #define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit) 492 #else 493 #define TSAN_MAYBE_INTERCEPT_ON_EXIT 494 #endif 495 496 // Cleanup old bufs. 497 static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { 498 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 499 JmpBuf *buf = &thr->jmp_bufs[i]; 500 if (buf->sp <= sp) { 501 uptr sz = thr->jmp_bufs.Size(); 502 internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf)); 503 thr->jmp_bufs.PopBack(); 504 i--; 505 } 506 } 507 } 508 509 static void SetJmp(ThreadState *thr, uptr sp) { 510 if (!thr->is_inited) // called from libc guts during bootstrap 511 return; 512 // Cleanup old bufs. 513 JmpBufGarbageCollect(thr, sp); 514 // Remember the buf. 515 JmpBuf *buf = thr->jmp_bufs.PushBack(); 516 buf->sp = sp; 517 buf->shadow_stack_pos = thr->shadow_stack_pos; 518 ThreadSignalContext *sctx = SigCtx(thr); 519 buf->int_signal_send = sctx ? sctx->int_signal_send : 0; 520 buf->in_blocking_func = sctx ? 521 atomic_load(&sctx->in_blocking_func, memory_order_relaxed) : 522 false; 523 buf->in_signal_handler = atomic_load(&thr->in_signal_handler, 524 memory_order_relaxed); 525 } 526 527 static void LongJmp(ThreadState *thr, uptr *env) { 528 uptr sp = ExtractLongJmpSp(env); 529 // Find the saved buf with matching sp. 530 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 531 JmpBuf *buf = &thr->jmp_bufs[i]; 532 if (buf->sp == sp) { 533 CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); 534 // Unwind the stack. 535 while (thr->shadow_stack_pos > buf->shadow_stack_pos) 536 FuncExit(thr); 537 ThreadSignalContext *sctx = SigCtx(thr); 538 if (sctx) { 539 sctx->int_signal_send = buf->int_signal_send; 540 atomic_store(&sctx->in_blocking_func, buf->in_blocking_func, 541 memory_order_relaxed); 542 } 543 atomic_store(&thr->in_signal_handler, buf->in_signal_handler, 544 memory_order_relaxed); 545 JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp 546 return; 547 } 548 } 549 Printf("ThreadSanitizer: can't find longjmp buf\n"); 550 CHECK(0); 551 } 552 553 // FIXME: put everything below into a common extern "C" block? 554 extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); } 555 556 #if SANITIZER_APPLE 557 TSAN_INTERCEPTOR(int, setjmp, void *env); 558 TSAN_INTERCEPTOR(int, _setjmp, void *env); 559 TSAN_INTERCEPTOR(int, sigsetjmp, void *env); 560 #else // SANITIZER_APPLE 561 562 #if SANITIZER_NETBSD 563 #define setjmp_symname __setjmp14 564 #define sigsetjmp_symname __sigsetjmp14 565 #else 566 #define setjmp_symname setjmp 567 #define sigsetjmp_symname sigsetjmp 568 #endif 569 570 #define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x 571 #define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x) 572 #define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname) 573 #define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname) 574 575 #define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname) 576 #define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname) 577 578 // Not called. Merely to satisfy TSAN_INTERCEPT(). 579 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 580 int TSAN_INTERCEPTOR_SETJMP(void *env); 581 extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) { 582 CHECK(0); 583 return 0; 584 } 585 586 // FIXME: any reason to have a separate declaration? 587 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 588 int __interceptor__setjmp(void *env); 589 extern "C" int __interceptor__setjmp(void *env) { 590 CHECK(0); 591 return 0; 592 } 593 594 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 595 int TSAN_INTERCEPTOR_SIGSETJMP(void *env); 596 extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) { 597 CHECK(0); 598 return 0; 599 } 600 601 #if !SANITIZER_NETBSD 602 extern "C" SANITIZER_INTERFACE_ATTRIBUTE 603 int __interceptor___sigsetjmp(void *env); 604 extern "C" int __interceptor___sigsetjmp(void *env) { 605 CHECK(0); 606 return 0; 607 } 608 #endif 609 610 extern "C" int setjmp_symname(void *env); 611 extern "C" int _setjmp(void *env); 612 extern "C" int sigsetjmp_symname(void *env); 613 #if !SANITIZER_NETBSD 614 extern "C" int __sigsetjmp(void *env); 615 #endif 616 DEFINE_REAL(int, setjmp_symname, void *env) 617 DEFINE_REAL(int, _setjmp, void *env) 618 DEFINE_REAL(int, sigsetjmp_symname, void *env) 619 #if !SANITIZER_NETBSD 620 DEFINE_REAL(int, __sigsetjmp, void *env) 621 #endif 622 #endif // SANITIZER_APPLE 623 624 #if SANITIZER_NETBSD 625 #define longjmp_symname __longjmp14 626 #define siglongjmp_symname __siglongjmp14 627 #else 628 #define longjmp_symname longjmp 629 #define siglongjmp_symname siglongjmp 630 #endif 631 632 TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) { 633 // Note: if we call REAL(longjmp) in the context of ScopedInterceptor, 634 // bad things will happen. We will jump over ScopedInterceptor dtor and can 635 // leave thr->in_ignored_lib set. 636 { 637 SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val); 638 } 639 LongJmp(cur_thread(), env); 640 REAL(longjmp_symname)(env, val); 641 } 642 643 TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) { 644 { 645 SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val); 646 } 647 LongJmp(cur_thread(), env); 648 REAL(siglongjmp_symname)(env, val); 649 } 650 651 #if SANITIZER_NETBSD 652 TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { 653 { 654 SCOPED_INTERCEPTOR_RAW(_longjmp, env, val); 655 } 656 LongJmp(cur_thread(), env); 657 REAL(_longjmp)(env, val); 658 } 659 #endif 660 661 #if !SANITIZER_APPLE 662 TSAN_INTERCEPTOR(void*, malloc, uptr size) { 663 if (in_symbolizer()) 664 return InternalAlloc(size); 665 void *p = 0; 666 { 667 SCOPED_INTERCEPTOR_RAW(malloc, size); 668 p = user_alloc(thr, pc, size); 669 } 670 invoke_malloc_hook(p, size); 671 return p; 672 } 673 674 // In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept 675 // __libc_memalign so that (1) we can detect races (2) free will not be called 676 // on libc internally allocated blocks. 677 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { 678 SCOPED_INTERCEPTOR_RAW(__libc_memalign, align, sz); 679 return user_memalign(thr, pc, align, sz); 680 } 681 682 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { 683 if (in_symbolizer()) 684 return InternalCalloc(size, n); 685 void *p = 0; 686 { 687 SCOPED_INTERCEPTOR_RAW(calloc, size, n); 688 p = user_calloc(thr, pc, size, n); 689 } 690 invoke_malloc_hook(p, n * size); 691 return p; 692 } 693 694 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { 695 if (in_symbolizer()) 696 return InternalRealloc(p, size); 697 if (p) 698 invoke_free_hook(p); 699 { 700 SCOPED_INTERCEPTOR_RAW(realloc, p, size); 701 p = user_realloc(thr, pc, p, size); 702 } 703 invoke_malloc_hook(p, size); 704 return p; 705 } 706 707 TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { 708 if (in_symbolizer()) 709 return InternalReallocArray(p, size, n); 710 if (p) 711 invoke_free_hook(p); 712 { 713 SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); 714 p = user_reallocarray(thr, pc, p, size, n); 715 } 716 invoke_malloc_hook(p, size); 717 return p; 718 } 719 720 TSAN_INTERCEPTOR(void, free, void *p) { 721 if (p == 0) 722 return; 723 if (in_symbolizer()) 724 return InternalFree(p); 725 invoke_free_hook(p); 726 SCOPED_INTERCEPTOR_RAW(free, p); 727 user_free(thr, pc, p); 728 } 729 730 TSAN_INTERCEPTOR(void, cfree, void *p) { 731 if (p == 0) 732 return; 733 if (in_symbolizer()) 734 return InternalFree(p); 735 invoke_free_hook(p); 736 SCOPED_INTERCEPTOR_RAW(cfree, p); 737 user_free(thr, pc, p); 738 } 739 740 TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { 741 SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); 742 return user_alloc_usable_size(p); 743 } 744 #endif 745 746 TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) { 747 SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); 748 uptr srclen = internal_strlen(src); 749 MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); 750 MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); 751 return REAL(strcpy)(dst, src); 752 } 753 754 TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { 755 SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); 756 uptr srclen = internal_strnlen(src, n); 757 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 758 MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); 759 return REAL(strncpy)(dst, src, n); 760 } 761 762 TSAN_INTERCEPTOR(char*, strdup, const char *str) { 763 SCOPED_TSAN_INTERCEPTOR(strdup, str); 764 // strdup will call malloc, so no instrumentation is required here. 765 return REAL(strdup)(str); 766 } 767 768 // Zero out addr if it points into shadow memory and was provided as a hint 769 // only, i.e., MAP_FIXED is not set. 770 static bool fix_mmap_addr(void **addr, long_t sz, int flags) { 771 if (*addr) { 772 if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { 773 if (flags & MAP_FIXED) { 774 errno = errno_EINVAL; 775 return false; 776 } else { 777 *addr = 0; 778 } 779 } 780 } 781 return true; 782 } 783 784 template <class Mmap> 785 static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap, 786 void *addr, SIZE_T sz, int prot, int flags, 787 int fd, OFF64_T off) { 788 if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED; 789 void *res = real_mmap(addr, sz, prot, flags, fd, off); 790 if (res != MAP_FAILED) { 791 if (!IsAppMem((uptr)res) || !IsAppMem((uptr)res + sz - 1)) { 792 Report("ThreadSanitizer: mmap at bad address: addr=%p size=%p res=%p\n", 793 addr, (void*)sz, res); 794 Die(); 795 } 796 if (fd > 0) FdAccess(thr, pc, fd); 797 MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz); 798 } 799 return res; 800 } 801 802 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { 803 SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); 804 UnmapShadow(thr, (uptr)addr, sz); 805 int res = REAL(munmap)(addr, sz); 806 return res; 807 } 808 809 #if SANITIZER_LINUX 810 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { 811 SCOPED_INTERCEPTOR_RAW(memalign, align, sz); 812 return user_memalign(thr, pc, align, sz); 813 } 814 #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) 815 #else 816 #define TSAN_MAYBE_INTERCEPT_MEMALIGN 817 #endif 818 819 #if !SANITIZER_APPLE 820 TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { 821 if (in_symbolizer()) 822 return InternalAlloc(sz, nullptr, align); 823 SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); 824 return user_aligned_alloc(thr, pc, align, sz); 825 } 826 827 TSAN_INTERCEPTOR(void*, valloc, uptr sz) { 828 if (in_symbolizer()) 829 return InternalAlloc(sz, nullptr, GetPageSizeCached()); 830 SCOPED_INTERCEPTOR_RAW(valloc, sz); 831 return user_valloc(thr, pc, sz); 832 } 833 #endif 834 835 #if SANITIZER_LINUX 836 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { 837 if (in_symbolizer()) { 838 uptr PageSize = GetPageSizeCached(); 839 sz = sz ? RoundUpTo(sz, PageSize) : PageSize; 840 return InternalAlloc(sz, nullptr, PageSize); 841 } 842 SCOPED_INTERCEPTOR_RAW(pvalloc, sz); 843 return user_pvalloc(thr, pc, sz); 844 } 845 #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) 846 #else 847 #define TSAN_MAYBE_INTERCEPT_PVALLOC 848 #endif 849 850 #if !SANITIZER_APPLE 851 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { 852 if (in_symbolizer()) { 853 void *p = InternalAlloc(sz, nullptr, align); 854 if (!p) 855 return errno_ENOMEM; 856 *memptr = p; 857 return 0; 858 } 859 SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); 860 return user_posix_memalign(thr, pc, memptr, align, sz); 861 } 862 #endif 863 864 // Both __cxa_guard_acquire and pthread_once 0-initialize 865 // the object initially. pthread_once does not have any 866 // other ABI requirements. __cxa_guard_acquire assumes 867 // that any non-0 value in the first byte means that 868 // initialization is completed. Contents of the remaining 869 // bytes are up to us. 870 constexpr u32 kGuardInit = 0; 871 constexpr u32 kGuardDone = 1; 872 constexpr u32 kGuardRunning = 1 << 16; 873 constexpr u32 kGuardWaiter = 1 << 17; 874 875 static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g, 876 bool blocking_hooks = true) { 877 if (blocking_hooks) 878 OnPotentiallyBlockingRegionBegin(); 879 auto on_exit = at_scope_exit([blocking_hooks] { 880 if (blocking_hooks) 881 OnPotentiallyBlockingRegionEnd(); 882 }); 883 884 for (;;) { 885 u32 cmp = atomic_load(g, memory_order_acquire); 886 if (cmp == kGuardInit) { 887 if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning, 888 memory_order_relaxed)) 889 return 1; 890 } else if (cmp == kGuardDone) { 891 if (!thr->in_ignored_lib) 892 Acquire(thr, pc, (uptr)g); 893 return 0; 894 } else { 895 if ((cmp & kGuardWaiter) || 896 atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter, 897 memory_order_relaxed)) 898 FutexWait(g, cmp | kGuardWaiter); 899 } 900 } 901 } 902 903 static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g, 904 u32 v) { 905 if (!thr->in_ignored_lib) 906 Release(thr, pc, (uptr)g); 907 u32 old = atomic_exchange(g, v, memory_order_release); 908 if (old & kGuardWaiter) 909 FutexWake(g, 1 << 30); 910 } 911 912 // __cxa_guard_acquire and friends need to be intercepted in a special way - 913 // regular interceptors will break statically-linked libstdc++. Linux 914 // interceptors are especially defined as weak functions (so that they don't 915 // cause link errors when user defines them as well). So they silently 916 // auto-disable themselves when such symbol is already present in the binary. If 917 // we link libstdc++ statically, it will bring own __cxa_guard_acquire which 918 // will silently replace our interceptor. That's why on Linux we simply export 919 // these interceptors with INTERFACE_ATTRIBUTE. 920 // On OS X, we don't support statically linking, so we just use a regular 921 // interceptor. 922 #if SANITIZER_APPLE 923 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR 924 #else 925 #define STDCXX_INTERCEPTOR(rettype, name, ...) \ 926 extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__) 927 #endif 928 929 // Used in thread-safe function static initialization. 930 STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { 931 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); 932 return guard_acquire(thr, pc, g); 933 } 934 935 STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) { 936 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); 937 guard_release(thr, pc, g, kGuardDone); 938 } 939 940 STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) { 941 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); 942 guard_release(thr, pc, g, kGuardInit); 943 } 944 945 namespace __tsan { 946 void DestroyThreadState() { 947 ThreadState *thr = cur_thread(); 948 Processor *proc = thr->proc(); 949 ThreadFinish(thr); 950 ProcUnwire(proc, thr); 951 ProcDestroy(proc); 952 DTLS_Destroy(); 953 cur_thread_finalize(); 954 } 955 956 void PlatformCleanUpThreadState(ThreadState *thr) { 957 ThreadSignalContext *sctx = thr->signal_ctx; 958 if (sctx) { 959 thr->signal_ctx = 0; 960 UnmapOrDie(sctx, sizeof(*sctx)); 961 } 962 } 963 } // namespace __tsan 964 965 #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD 966 static void thread_finalize(void *v) { 967 uptr iter = (uptr)v; 968 if (iter > 1) { 969 if (pthread_setspecific(interceptor_ctx()->finalize_key, 970 (void*)(iter - 1))) { 971 Printf("ThreadSanitizer: failed to set thread key\n"); 972 Die(); 973 } 974 return; 975 } 976 DestroyThreadState(); 977 } 978 #endif 979 980 981 struct ThreadParam { 982 void* (*callback)(void *arg); 983 void *param; 984 Tid tid; 985 Semaphore created; 986 Semaphore started; 987 }; 988 989 extern "C" void *__tsan_thread_start_func(void *arg) { 990 ThreadParam *p = (ThreadParam*)arg; 991 void* (*callback)(void *arg) = p->callback; 992 void *param = p->param; 993 { 994 ThreadState *thr = cur_thread_init(); 995 // Thread-local state is not initialized yet. 996 ScopedIgnoreInterceptors ignore; 997 #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD 998 ThreadIgnoreBegin(thr, 0); 999 if (pthread_setspecific(interceptor_ctx()->finalize_key, 1000 (void *)GetPthreadDestructorIterations())) { 1001 Printf("ThreadSanitizer: failed to set thread key\n"); 1002 Die(); 1003 } 1004 ThreadIgnoreEnd(thr); 1005 #endif 1006 p->created.Wait(); 1007 Processor *proc = ProcCreate(); 1008 ProcWire(proc, thr); 1009 ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular); 1010 p->started.Post(); 1011 } 1012 void *res = callback(param); 1013 // Prevent the callback from being tail called, 1014 // it mixes up stack traces. 1015 volatile int foo = 42; 1016 foo++; 1017 return res; 1018 } 1019 1020 TSAN_INTERCEPTOR(int, pthread_create, 1021 void *th, void *attr, void *(*callback)(void*), void * param) { 1022 SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); 1023 1024 MaybeSpawnBackgroundThread(); 1025 1026 if (ctx->after_multithreaded_fork) { 1027 if (flags()->die_after_fork) { 1028 Report("ThreadSanitizer: starting new threads after multi-threaded " 1029 "fork is not supported. Dying (set die_after_fork=0 to override)\n"); 1030 Die(); 1031 } else { 1032 VPrintf(1, 1033 "ThreadSanitizer: starting new threads after multi-threaded " 1034 "fork is not supported (pid %lu). Continuing because of " 1035 "die_after_fork=0, but you are on your own\n", 1036 internal_getpid()); 1037 } 1038 } 1039 __sanitizer_pthread_attr_t myattr; 1040 if (attr == 0) { 1041 pthread_attr_init(&myattr); 1042 attr = &myattr; 1043 } 1044 int detached = 0; 1045 REAL(pthread_attr_getdetachstate)(attr, &detached); 1046 AdjustStackSize(attr); 1047 1048 ThreadParam p; 1049 p.callback = callback; 1050 p.param = param; 1051 p.tid = kMainTid; 1052 int res = -1; 1053 { 1054 // Otherwise we see false positives in pthread stack manipulation. 1055 ScopedIgnoreInterceptors ignore; 1056 ThreadIgnoreBegin(thr, pc); 1057 res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); 1058 ThreadIgnoreEnd(thr); 1059 } 1060 if (res == 0) { 1061 p.tid = ThreadCreate(thr, pc, *(uptr *)th, IsStateDetached(detached)); 1062 CHECK_NE(p.tid, kMainTid); 1063 // Synchronization on p.tid serves two purposes: 1064 // 1. ThreadCreate must finish before the new thread starts. 1065 // Otherwise the new thread can call pthread_detach, but the pthread_t 1066 // identifier is not yet registered in ThreadRegistry by ThreadCreate. 1067 // 2. ThreadStart must finish before this thread continues. 1068 // Otherwise, this thread can call pthread_detach and reset thr->sync 1069 // before the new thread got a chance to acquire from it in ThreadStart. 1070 p.created.Post(); 1071 p.started.Wait(); 1072 } 1073 if (attr == &myattr) 1074 pthread_attr_destroy(&myattr); 1075 return res; 1076 } 1077 1078 TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { 1079 SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); 1080 Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); 1081 ThreadIgnoreBegin(thr, pc); 1082 int res = BLOCK_REAL(pthread_join)(th, ret); 1083 ThreadIgnoreEnd(thr); 1084 if (res == 0) { 1085 ThreadJoin(thr, pc, tid); 1086 } 1087 return res; 1088 } 1089 1090 DEFINE_REAL_PTHREAD_FUNCTIONS 1091 1092 TSAN_INTERCEPTOR(int, pthread_detach, void *th) { 1093 SCOPED_INTERCEPTOR_RAW(pthread_detach, th); 1094 Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); 1095 int res = REAL(pthread_detach)(th); 1096 if (res == 0) { 1097 ThreadDetach(thr, pc, tid); 1098 } 1099 return res; 1100 } 1101 1102 TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { 1103 { 1104 SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); 1105 #if !SANITIZER_APPLE && !SANITIZER_ANDROID 1106 CHECK_EQ(thr, &cur_thread_placeholder); 1107 #endif 1108 } 1109 REAL(pthread_exit)(retval); 1110 } 1111 1112 #if SANITIZER_LINUX 1113 TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { 1114 SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret); 1115 Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); 1116 ThreadIgnoreBegin(thr, pc); 1117 int res = REAL(pthread_tryjoin_np)(th, ret); 1118 ThreadIgnoreEnd(thr); 1119 if (res == 0) 1120 ThreadJoin(thr, pc, tid); 1121 else 1122 ThreadNotJoined(thr, pc, tid, (uptr)th); 1123 return res; 1124 } 1125 1126 TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret, 1127 const struct timespec *abstime) { 1128 SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime); 1129 Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); 1130 ThreadIgnoreBegin(thr, pc); 1131 int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime); 1132 ThreadIgnoreEnd(thr); 1133 if (res == 0) 1134 ThreadJoin(thr, pc, tid); 1135 else 1136 ThreadNotJoined(thr, pc, tid, (uptr)th); 1137 return res; 1138 } 1139 #endif 1140 1141 // Problem: 1142 // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). 1143 // pthread_cond_t has different size in the different versions. 1144 // If call new REAL functions for old pthread_cond_t, they will corrupt memory 1145 // after pthread_cond_t (old cond is smaller). 1146 // If we call old REAL functions for new pthread_cond_t, we will lose some 1147 // functionality (e.g. old functions do not support waiting against 1148 // CLOCK_REALTIME). 1149 // Proper handling would require to have 2 versions of interceptors as well. 1150 // But this is messy, in particular requires linker scripts when sanitizer 1151 // runtime is linked into a shared library. 1152 // Instead we assume we don't have dynamic libraries built against old 1153 // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag 1154 // that allows to work with old libraries (but this mode does not support 1155 // some features, e.g. pthread_condattr_getpshared). 1156 static void *init_cond(void *c, bool force = false) { 1157 // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions. 1158 // So we allocate additional memory on the side large enough to hold 1159 // any pthread_cond_t object. Always call new REAL functions, but pass 1160 // the aux object to them. 1161 // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes 1162 // first word of pthread_cond_t to zero. 1163 // It's all relevant only for linux. 1164 if (!common_flags()->legacy_pthread_cond) 1165 return c; 1166 atomic_uintptr_t *p = (atomic_uintptr_t*)c; 1167 uptr cond = atomic_load(p, memory_order_acquire); 1168 if (!force && cond != 0) 1169 return (void*)cond; 1170 void *newcond = WRAP(malloc)(pthread_cond_t_sz); 1171 internal_memset(newcond, 0, pthread_cond_t_sz); 1172 if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond, 1173 memory_order_acq_rel)) 1174 return newcond; 1175 WRAP(free)(newcond); 1176 return (void*)cond; 1177 } 1178 1179 namespace { 1180 1181 template <class Fn> 1182 struct CondMutexUnlockCtx { 1183 ScopedInterceptor *si; 1184 ThreadState *thr; 1185 uptr pc; 1186 void *m; 1187 void *c; 1188 const Fn &fn; 1189 1190 int Cancel() const { return fn(); } 1191 void Unlock() const; 1192 }; 1193 1194 template <class Fn> 1195 void CondMutexUnlockCtx<Fn>::Unlock() const { 1196 // pthread_cond_wait interceptor has enabled async signal delivery 1197 // (see BlockingCall below). Disable async signals since we are running 1198 // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run 1199 // since the thread is cancelled, so we have to manually execute them 1200 // (the thread still can run some user code due to pthread_cleanup_push). 1201 ThreadSignalContext *ctx = SigCtx(thr); 1202 CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1); 1203 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 1204 MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock); 1205 // Undo BlockingCall ctor effects. 1206 thr->ignore_interceptors--; 1207 si->~ScopedInterceptor(); 1208 } 1209 } // namespace 1210 1211 INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { 1212 void *cond = init_cond(c, true); 1213 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); 1214 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); 1215 return REAL(pthread_cond_init)(cond, a); 1216 } 1217 1218 template <class Fn> 1219 int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, const Fn &fn, 1220 void *c, void *m) { 1221 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1222 MutexUnlock(thr, pc, (uptr)m); 1223 int res = 0; 1224 // This ensures that we handle mutex lock even in case of pthread_cancel. 1225 // See test/tsan/cond_cancel.cpp. 1226 { 1227 // Enable signal delivery while the thread is blocked. 1228 BlockingCall bc(thr); 1229 CondMutexUnlockCtx<Fn> arg = {si, thr, pc, m, c, fn}; 1230 res = call_pthread_cancel_with_cleanup( 1231 [](void *arg) -> int { 1232 return ((const CondMutexUnlockCtx<Fn> *)arg)->Cancel(); 1233 }, 1234 [](void *arg) { ((const CondMutexUnlockCtx<Fn> *)arg)->Unlock(); }, 1235 &arg); 1236 } 1237 if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); 1238 MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock); 1239 return res; 1240 } 1241 1242 INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { 1243 void *cond = init_cond(c); 1244 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); 1245 return cond_wait( 1246 thr, pc, &si, [=]() { return REAL(pthread_cond_wait)(cond, m); }, cond, 1247 m); 1248 } 1249 1250 INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { 1251 void *cond = init_cond(c); 1252 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); 1253 return cond_wait( 1254 thr, pc, &si, 1255 [=]() { return REAL(pthread_cond_timedwait)(cond, m, abstime); }, cond, 1256 m); 1257 } 1258 1259 #if SANITIZER_LINUX 1260 INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m, 1261 __sanitizer_clockid_t clock, void *abstime) { 1262 void *cond = init_cond(c); 1263 SCOPED_TSAN_INTERCEPTOR(pthread_cond_clockwait, cond, m, clock, abstime); 1264 return cond_wait( 1265 thr, pc, &si, 1266 [=]() { return REAL(pthread_cond_clockwait)(cond, m, clock, abstime); }, 1267 cond, m); 1268 } 1269 #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT TSAN_INTERCEPT(pthread_cond_clockwait) 1270 #else 1271 #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT 1272 #endif 1273 1274 #if SANITIZER_APPLE 1275 INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m, 1276 void *reltime) { 1277 void *cond = init_cond(c); 1278 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime); 1279 return cond_wait( 1280 thr, pc, &si, 1281 [=]() { 1282 return REAL(pthread_cond_timedwait_relative_np)(cond, m, reltime); 1283 }, 1284 cond, m); 1285 } 1286 #endif 1287 1288 INTERCEPTOR(int, pthread_cond_signal, void *c) { 1289 void *cond = init_cond(c); 1290 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); 1291 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1292 return REAL(pthread_cond_signal)(cond); 1293 } 1294 1295 INTERCEPTOR(int, pthread_cond_broadcast, void *c) { 1296 void *cond = init_cond(c); 1297 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); 1298 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1299 return REAL(pthread_cond_broadcast)(cond); 1300 } 1301 1302 INTERCEPTOR(int, pthread_cond_destroy, void *c) { 1303 void *cond = init_cond(c); 1304 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); 1305 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); 1306 int res = REAL(pthread_cond_destroy)(cond); 1307 if (common_flags()->legacy_pthread_cond) { 1308 // Free our aux cond and zero the pointer to not leave dangling pointers. 1309 WRAP(free)(cond); 1310 atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed); 1311 } 1312 return res; 1313 } 1314 1315 TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { 1316 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); 1317 int res = REAL(pthread_mutex_init)(m, a); 1318 if (res == 0) { 1319 u32 flagz = 0; 1320 if (a) { 1321 int type = 0; 1322 if (REAL(pthread_mutexattr_gettype)(a, &type) == 0) 1323 if (type == PTHREAD_MUTEX_RECURSIVE || 1324 type == PTHREAD_MUTEX_RECURSIVE_NP) 1325 flagz |= MutexFlagWriteReentrant; 1326 } 1327 MutexCreate(thr, pc, (uptr)m, flagz); 1328 } 1329 return res; 1330 } 1331 1332 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { 1333 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); 1334 int res = REAL(pthread_mutex_destroy)(m); 1335 if (res == 0 || res == errno_EBUSY) { 1336 MutexDestroy(thr, pc, (uptr)m); 1337 } 1338 return res; 1339 } 1340 1341 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { 1342 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); 1343 int res = REAL(pthread_mutex_trylock)(m); 1344 if (res == errno_EOWNERDEAD) 1345 MutexRepair(thr, pc, (uptr)m); 1346 if (res == 0 || res == errno_EOWNERDEAD) 1347 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1348 return res; 1349 } 1350 1351 #if !SANITIZER_APPLE 1352 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { 1353 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); 1354 int res = REAL(pthread_mutex_timedlock)(m, abstime); 1355 if (res == 0) { 1356 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1357 } 1358 return res; 1359 } 1360 #endif 1361 1362 #if !SANITIZER_APPLE 1363 TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { 1364 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); 1365 int res = REAL(pthread_spin_init)(m, pshared); 1366 if (res == 0) { 1367 MutexCreate(thr, pc, (uptr)m); 1368 } 1369 return res; 1370 } 1371 1372 TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { 1373 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); 1374 int res = REAL(pthread_spin_destroy)(m); 1375 if (res == 0) { 1376 MutexDestroy(thr, pc, (uptr)m); 1377 } 1378 return res; 1379 } 1380 1381 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { 1382 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); 1383 MutexPreLock(thr, pc, (uptr)m); 1384 int res = REAL(pthread_spin_lock)(m); 1385 if (res == 0) { 1386 MutexPostLock(thr, pc, (uptr)m); 1387 } 1388 return res; 1389 } 1390 1391 TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { 1392 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); 1393 int res = REAL(pthread_spin_trylock)(m); 1394 if (res == 0) { 1395 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1396 } 1397 return res; 1398 } 1399 1400 TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { 1401 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); 1402 MutexUnlock(thr, pc, (uptr)m); 1403 int res = REAL(pthread_spin_unlock)(m); 1404 return res; 1405 } 1406 #endif 1407 1408 TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { 1409 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); 1410 int res = REAL(pthread_rwlock_init)(m, a); 1411 if (res == 0) { 1412 MutexCreate(thr, pc, (uptr)m); 1413 } 1414 return res; 1415 } 1416 1417 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { 1418 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); 1419 int res = REAL(pthread_rwlock_destroy)(m); 1420 if (res == 0) { 1421 MutexDestroy(thr, pc, (uptr)m); 1422 } 1423 return res; 1424 } 1425 1426 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { 1427 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); 1428 MutexPreReadLock(thr, pc, (uptr)m); 1429 int res = REAL(pthread_rwlock_rdlock)(m); 1430 if (res == 0) { 1431 MutexPostReadLock(thr, pc, (uptr)m); 1432 } 1433 return res; 1434 } 1435 1436 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { 1437 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); 1438 int res = REAL(pthread_rwlock_tryrdlock)(m); 1439 if (res == 0) { 1440 MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock); 1441 } 1442 return res; 1443 } 1444 1445 #if !SANITIZER_APPLE 1446 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { 1447 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); 1448 int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); 1449 if (res == 0) { 1450 MutexPostReadLock(thr, pc, (uptr)m); 1451 } 1452 return res; 1453 } 1454 #endif 1455 1456 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { 1457 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); 1458 MutexPreLock(thr, pc, (uptr)m); 1459 int res = REAL(pthread_rwlock_wrlock)(m); 1460 if (res == 0) { 1461 MutexPostLock(thr, pc, (uptr)m); 1462 } 1463 return res; 1464 } 1465 1466 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { 1467 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); 1468 int res = REAL(pthread_rwlock_trywrlock)(m); 1469 if (res == 0) { 1470 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1471 } 1472 return res; 1473 } 1474 1475 #if !SANITIZER_APPLE 1476 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { 1477 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); 1478 int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); 1479 if (res == 0) { 1480 MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); 1481 } 1482 return res; 1483 } 1484 #endif 1485 1486 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { 1487 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); 1488 MutexReadOrWriteUnlock(thr, pc, (uptr)m); 1489 int res = REAL(pthread_rwlock_unlock)(m); 1490 return res; 1491 } 1492 1493 #if !SANITIZER_APPLE 1494 TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { 1495 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); 1496 MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); 1497 int res = REAL(pthread_barrier_init)(b, a, count); 1498 return res; 1499 } 1500 1501 TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { 1502 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); 1503 MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); 1504 int res = REAL(pthread_barrier_destroy)(b); 1505 return res; 1506 } 1507 1508 TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { 1509 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); 1510 Release(thr, pc, (uptr)b); 1511 MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead); 1512 int res = REAL(pthread_barrier_wait)(b); 1513 MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead); 1514 if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { 1515 Acquire(thr, pc, (uptr)b); 1516 } 1517 return res; 1518 } 1519 #endif 1520 1521 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { 1522 SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); 1523 if (o == 0 || f == 0) 1524 return errno_EINVAL; 1525 atomic_uint32_t *a; 1526 1527 if (SANITIZER_APPLE) 1528 a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t))); 1529 else if (SANITIZER_NETBSD) 1530 a = static_cast<atomic_uint32_t*> 1531 ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz)); 1532 else 1533 a = static_cast<atomic_uint32_t*>(o); 1534 1535 // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks 1536 // result in crashes due to too little stack space. 1537 if (guard_acquire(thr, pc, a, !SANITIZER_APPLE)) { 1538 (*f)(); 1539 guard_release(thr, pc, a, kGuardDone); 1540 } 1541 return 0; 1542 } 1543 1544 #if SANITIZER_GLIBC 1545 TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { 1546 SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); 1547 if (fd > 0) 1548 FdAccess(thr, pc, fd); 1549 return REAL(__fxstat)(version, fd, buf); 1550 } 1551 #define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat) 1552 #else 1553 #define TSAN_MAYBE_INTERCEPT___FXSTAT 1554 #endif 1555 1556 TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { 1557 #if SANITIZER_GLIBC 1558 SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); 1559 if (fd > 0) 1560 FdAccess(thr, pc, fd); 1561 return REAL(__fxstat)(0, fd, buf); 1562 #else 1563 SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); 1564 if (fd > 0) 1565 FdAccess(thr, pc, fd); 1566 return REAL(fstat)(fd, buf); 1567 #endif 1568 } 1569 1570 #if SANITIZER_GLIBC 1571 TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { 1572 SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); 1573 if (fd > 0) 1574 FdAccess(thr, pc, fd); 1575 return REAL(__fxstat64)(version, fd, buf); 1576 } 1577 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64) 1578 #else 1579 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 1580 #endif 1581 1582 #if SANITIZER_GLIBC 1583 TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { 1584 SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); 1585 if (fd > 0) 1586 FdAccess(thr, pc, fd); 1587 return REAL(__fxstat64)(0, fd, buf); 1588 } 1589 #define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64) 1590 #else 1591 #define TSAN_MAYBE_INTERCEPT_FSTAT64 1592 #endif 1593 1594 TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) { 1595 va_list ap; 1596 va_start(ap, oflag); 1597 mode_t mode = va_arg(ap, int); 1598 va_end(ap); 1599 SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode); 1600 READ_STRING(thr, pc, name, 0); 1601 int fd = REAL(open)(name, oflag, mode); 1602 if (fd >= 0) 1603 FdFileCreate(thr, pc, fd); 1604 return fd; 1605 } 1606 1607 #if SANITIZER_LINUX 1608 TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) { 1609 va_list ap; 1610 va_start(ap, oflag); 1611 mode_t mode = va_arg(ap, int); 1612 va_end(ap); 1613 SCOPED_TSAN_INTERCEPTOR(open64, name, oflag, mode); 1614 READ_STRING(thr, pc, name, 0); 1615 int fd = REAL(open64)(name, oflag, mode); 1616 if (fd >= 0) 1617 FdFileCreate(thr, pc, fd); 1618 return fd; 1619 } 1620 #define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64) 1621 #else 1622 #define TSAN_MAYBE_INTERCEPT_OPEN64 1623 #endif 1624 1625 TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { 1626 SCOPED_TSAN_INTERCEPTOR(creat, name, mode); 1627 READ_STRING(thr, pc, name, 0); 1628 int fd = REAL(creat)(name, mode); 1629 if (fd >= 0) 1630 FdFileCreate(thr, pc, fd); 1631 return fd; 1632 } 1633 1634 #if SANITIZER_LINUX 1635 TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { 1636 SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); 1637 READ_STRING(thr, pc, name, 0); 1638 int fd = REAL(creat64)(name, mode); 1639 if (fd >= 0) 1640 FdFileCreate(thr, pc, fd); 1641 return fd; 1642 } 1643 #define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64) 1644 #else 1645 #define TSAN_MAYBE_INTERCEPT_CREAT64 1646 #endif 1647 1648 TSAN_INTERCEPTOR(int, dup, int oldfd) { 1649 SCOPED_TSAN_INTERCEPTOR(dup, oldfd); 1650 int newfd = REAL(dup)(oldfd); 1651 if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) 1652 FdDup(thr, pc, oldfd, newfd, true); 1653 return newfd; 1654 } 1655 1656 TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { 1657 SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); 1658 int newfd2 = REAL(dup2)(oldfd, newfd); 1659 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1660 FdDup(thr, pc, oldfd, newfd2, false); 1661 return newfd2; 1662 } 1663 1664 #if !SANITIZER_APPLE 1665 TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { 1666 SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); 1667 int newfd2 = REAL(dup3)(oldfd, newfd, flags); 1668 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1669 FdDup(thr, pc, oldfd, newfd2, false); 1670 return newfd2; 1671 } 1672 #endif 1673 1674 #if SANITIZER_LINUX 1675 TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { 1676 SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); 1677 int fd = REAL(eventfd)(initval, flags); 1678 if (fd >= 0) 1679 FdEventCreate(thr, pc, fd); 1680 return fd; 1681 } 1682 #define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd) 1683 #else 1684 #define TSAN_MAYBE_INTERCEPT_EVENTFD 1685 #endif 1686 1687 #if SANITIZER_LINUX 1688 TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { 1689 SCOPED_INTERCEPTOR_RAW(signalfd, fd, mask, flags); 1690 FdClose(thr, pc, fd); 1691 fd = REAL(signalfd)(fd, mask, flags); 1692 if (!MustIgnoreInterceptor(thr)) 1693 FdSignalCreate(thr, pc, fd); 1694 return fd; 1695 } 1696 #define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd) 1697 #else 1698 #define TSAN_MAYBE_INTERCEPT_SIGNALFD 1699 #endif 1700 1701 #if SANITIZER_LINUX 1702 TSAN_INTERCEPTOR(int, inotify_init, int fake) { 1703 SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); 1704 int fd = REAL(inotify_init)(fake); 1705 if (fd >= 0) 1706 FdInotifyCreate(thr, pc, fd); 1707 return fd; 1708 } 1709 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init) 1710 #else 1711 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT 1712 #endif 1713 1714 #if SANITIZER_LINUX 1715 TSAN_INTERCEPTOR(int, inotify_init1, int flags) { 1716 SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); 1717 int fd = REAL(inotify_init1)(flags); 1718 if (fd >= 0) 1719 FdInotifyCreate(thr, pc, fd); 1720 return fd; 1721 } 1722 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1) 1723 #else 1724 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 1725 #endif 1726 1727 TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { 1728 SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); 1729 int fd = REAL(socket)(domain, type, protocol); 1730 if (fd >= 0) 1731 FdSocketCreate(thr, pc, fd); 1732 return fd; 1733 } 1734 1735 TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { 1736 SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); 1737 int res = REAL(socketpair)(domain, type, protocol, fd); 1738 if (res == 0 && fd[0] >= 0 && fd[1] >= 0) 1739 FdPipeCreate(thr, pc, fd[0], fd[1]); 1740 return res; 1741 } 1742 1743 TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { 1744 SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); 1745 FdSocketConnecting(thr, pc, fd); 1746 int res = REAL(connect)(fd, addr, addrlen); 1747 if (res == 0 && fd >= 0) 1748 FdSocketConnect(thr, pc, fd); 1749 return res; 1750 } 1751 1752 TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { 1753 SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen); 1754 int res = REAL(bind)(fd, addr, addrlen); 1755 if (fd > 0 && res == 0) 1756 FdAccess(thr, pc, fd); 1757 return res; 1758 } 1759 1760 TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { 1761 SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog); 1762 int res = REAL(listen)(fd, backlog); 1763 if (fd > 0 && res == 0) 1764 FdAccess(thr, pc, fd); 1765 return res; 1766 } 1767 1768 TSAN_INTERCEPTOR(int, close, int fd) { 1769 SCOPED_INTERCEPTOR_RAW(close, fd); 1770 if (!in_symbolizer()) 1771 FdClose(thr, pc, fd); 1772 return REAL(close)(fd); 1773 } 1774 1775 #if SANITIZER_LINUX 1776 TSAN_INTERCEPTOR(int, __close, int fd) { 1777 SCOPED_INTERCEPTOR_RAW(__close, fd); 1778 FdClose(thr, pc, fd); 1779 return REAL(__close)(fd); 1780 } 1781 #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close) 1782 #else 1783 #define TSAN_MAYBE_INTERCEPT___CLOSE 1784 #endif 1785 1786 // glibc guts 1787 #if SANITIZER_LINUX && !SANITIZER_ANDROID 1788 TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { 1789 SCOPED_INTERCEPTOR_RAW(__res_iclose, state, free_addr); 1790 int fds[64]; 1791 int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); 1792 for (int i = 0; i < cnt; i++) FdClose(thr, pc, fds[i]); 1793 REAL(__res_iclose)(state, free_addr); 1794 } 1795 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose) 1796 #else 1797 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE 1798 #endif 1799 1800 TSAN_INTERCEPTOR(int, pipe, int *pipefd) { 1801 SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); 1802 int res = REAL(pipe)(pipefd); 1803 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1804 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1805 return res; 1806 } 1807 1808 #if !SANITIZER_APPLE 1809 TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { 1810 SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); 1811 int res = REAL(pipe2)(pipefd, flags); 1812 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1813 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1814 return res; 1815 } 1816 #endif 1817 1818 TSAN_INTERCEPTOR(int, unlink, char *path) { 1819 SCOPED_TSAN_INTERCEPTOR(unlink, path); 1820 Release(thr, pc, File2addr(path)); 1821 int res = REAL(unlink)(path); 1822 return res; 1823 } 1824 1825 TSAN_INTERCEPTOR(void*, tmpfile, int fake) { 1826 SCOPED_TSAN_INTERCEPTOR(tmpfile, fake); 1827 void *res = REAL(tmpfile)(fake); 1828 if (res) { 1829 int fd = fileno_unlocked(res); 1830 if (fd >= 0) 1831 FdFileCreate(thr, pc, fd); 1832 } 1833 return res; 1834 } 1835 1836 #if SANITIZER_LINUX 1837 TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { 1838 SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); 1839 void *res = REAL(tmpfile64)(fake); 1840 if (res) { 1841 int fd = fileno_unlocked(res); 1842 if (fd >= 0) 1843 FdFileCreate(thr, pc, fd); 1844 } 1845 return res; 1846 } 1847 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64) 1848 #else 1849 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 1850 #endif 1851 1852 static void FlushStreams() { 1853 // Flushing all the streams here may freeze the process if a child thread is 1854 // performing file stream operations at the same time. 1855 REAL(fflush)(stdout); 1856 REAL(fflush)(stderr); 1857 } 1858 1859 TSAN_INTERCEPTOR(void, abort, int fake) { 1860 SCOPED_TSAN_INTERCEPTOR(abort, fake); 1861 FlushStreams(); 1862 REAL(abort)(fake); 1863 } 1864 1865 TSAN_INTERCEPTOR(int, rmdir, char *path) { 1866 SCOPED_TSAN_INTERCEPTOR(rmdir, path); 1867 Release(thr, pc, Dir2addr(path)); 1868 int res = REAL(rmdir)(path); 1869 return res; 1870 } 1871 1872 TSAN_INTERCEPTOR(int, closedir, void *dirp) { 1873 SCOPED_INTERCEPTOR_RAW(closedir, dirp); 1874 if (dirp) { 1875 int fd = dirfd(dirp); 1876 FdClose(thr, pc, fd); 1877 } 1878 return REAL(closedir)(dirp); 1879 } 1880 1881 #if SANITIZER_LINUX 1882 TSAN_INTERCEPTOR(int, epoll_create, int size) { 1883 SCOPED_TSAN_INTERCEPTOR(epoll_create, size); 1884 int fd = REAL(epoll_create)(size); 1885 if (fd >= 0) 1886 FdPollCreate(thr, pc, fd); 1887 return fd; 1888 } 1889 1890 TSAN_INTERCEPTOR(int, epoll_create1, int flags) { 1891 SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); 1892 int fd = REAL(epoll_create1)(flags); 1893 if (fd >= 0) 1894 FdPollCreate(thr, pc, fd); 1895 return fd; 1896 } 1897 1898 TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { 1899 SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); 1900 if (epfd >= 0) 1901 FdAccess(thr, pc, epfd); 1902 if (epfd >= 0 && fd >= 0) 1903 FdAccess(thr, pc, fd); 1904 if (op == EPOLL_CTL_ADD && epfd >= 0) { 1905 FdPollAdd(thr, pc, epfd, fd); 1906 FdRelease(thr, pc, epfd); 1907 } 1908 int res = REAL(epoll_ctl)(epfd, op, fd, ev); 1909 return res; 1910 } 1911 1912 TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { 1913 SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); 1914 if (epfd >= 0) 1915 FdAccess(thr, pc, epfd); 1916 int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); 1917 if (res > 0 && epfd >= 0) 1918 FdAcquire(thr, pc, epfd); 1919 return res; 1920 } 1921 1922 TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout, 1923 void *sigmask) { 1924 SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask); 1925 if (epfd >= 0) 1926 FdAccess(thr, pc, epfd); 1927 int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask); 1928 if (res > 0 && epfd >= 0) 1929 FdAcquire(thr, pc, epfd); 1930 return res; 1931 } 1932 1933 #define TSAN_MAYBE_INTERCEPT_EPOLL \ 1934 TSAN_INTERCEPT(epoll_create); \ 1935 TSAN_INTERCEPT(epoll_create1); \ 1936 TSAN_INTERCEPT(epoll_ctl); \ 1937 TSAN_INTERCEPT(epoll_wait); \ 1938 TSAN_INTERCEPT(epoll_pwait) 1939 #else 1940 #define TSAN_MAYBE_INTERCEPT_EPOLL 1941 #endif 1942 1943 // The following functions are intercepted merely to process pending signals. 1944 // If program blocks signal X, we must deliver the signal before the function 1945 // returns. Similarly, if program unblocks a signal (or returns from sigsuspend) 1946 // it's better to deliver the signal straight away. 1947 TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { 1948 SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); 1949 return REAL(sigsuspend)(mask); 1950 } 1951 1952 TSAN_INTERCEPTOR(int, sigblock, int mask) { 1953 SCOPED_TSAN_INTERCEPTOR(sigblock, mask); 1954 return REAL(sigblock)(mask); 1955 } 1956 1957 TSAN_INTERCEPTOR(int, sigsetmask, int mask) { 1958 SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask); 1959 return REAL(sigsetmask)(mask); 1960 } 1961 1962 TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, 1963 __sanitizer_sigset_t *oldset) { 1964 SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset); 1965 return REAL(pthread_sigmask)(how, set, oldset); 1966 } 1967 1968 namespace __tsan { 1969 1970 static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) { 1971 VarSizeStackTrace stack; 1972 // StackTrace::GetNestInstructionPc(pc) is used because return address is 1973 // expected, OutputReport() will undo this. 1974 ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); 1975 ThreadRegistryLock l(&ctx->thread_registry); 1976 ScopedReport rep(ReportTypeErrnoInSignal); 1977 rep.SetSigNum(sig); 1978 if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) { 1979 rep.AddStack(stack, true); 1980 OutputReport(thr, rep); 1981 } 1982 } 1983 1984 static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, 1985 int sig, __sanitizer_siginfo *info, 1986 void *uctx) { 1987 CHECK(thr->slot); 1988 __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; 1989 if (acquire) 1990 Acquire(thr, 0, (uptr)&sigactions[sig]); 1991 // Signals are generally asynchronous, so if we receive a signals when 1992 // ignores are enabled we should disable ignores. This is critical for sync 1993 // and interceptors, because otherwise we can miss synchronization and report 1994 // false races. 1995 int ignore_reads_and_writes = thr->ignore_reads_and_writes; 1996 int ignore_interceptors = thr->ignore_interceptors; 1997 int ignore_sync = thr->ignore_sync; 1998 // For symbolizer we only process SIGSEGVs synchronously 1999 // (bug in symbolizer or in tsan). But we want to reset 2000 // in_symbolizer to fail gracefully. Symbolizer and user code 2001 // use different memory allocators, so if we don't reset 2002 // in_symbolizer we can get memory allocated with one being 2003 // feed with another, which can cause more crashes. 2004 int in_symbolizer = thr->in_symbolizer; 2005 if (!ctx->after_multithreaded_fork) { 2006 thr->ignore_reads_and_writes = 0; 2007 thr->fast_state.ClearIgnoreBit(); 2008 thr->ignore_interceptors = 0; 2009 thr->ignore_sync = 0; 2010 thr->in_symbolizer = 0; 2011 } 2012 // Ensure that the handler does not spoil errno. 2013 const int saved_errno = errno; 2014 errno = 99; 2015 // This code races with sigaction. Be careful to not read sa_sigaction twice. 2016 // Also need to remember pc for reporting before the call, 2017 // because the handler can reset it. 2018 volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO) 2019 ? (uptr)sigactions[sig].sigaction 2020 : (uptr)sigactions[sig].handler; 2021 if (pc != sig_dfl && pc != sig_ign) { 2022 // The callback can be either sa_handler or sa_sigaction. 2023 // They have different signatures, but we assume that passing 2024 // additional arguments to sa_handler works and is harmless. 2025 ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx); 2026 } 2027 if (!ctx->after_multithreaded_fork) { 2028 thr->ignore_reads_and_writes = ignore_reads_and_writes; 2029 if (ignore_reads_and_writes) 2030 thr->fast_state.SetIgnoreBit(); 2031 thr->ignore_interceptors = ignore_interceptors; 2032 thr->ignore_sync = ignore_sync; 2033 thr->in_symbolizer = in_symbolizer; 2034 } 2035 // We do not detect errno spoiling for SIGTERM, 2036 // because some SIGTERM handlers do spoil errno but reraise SIGTERM, 2037 // tsan reports false positive in such case. 2038 // It's difficult to properly detect this situation (reraise), 2039 // because in async signal processing case (when handler is called directly 2040 // from rtl_generic_sighandler) we have not yet received the reraised 2041 // signal; and it looks too fragile to intercept all ways to reraise a signal. 2042 if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM && 2043 errno != 99) 2044 ReportErrnoSpoiling(thr, pc, sig); 2045 errno = saved_errno; 2046 } 2047 2048 void ProcessPendingSignalsImpl(ThreadState *thr) { 2049 atomic_store(&thr->pending_signals, 0, memory_order_relaxed); 2050 ThreadSignalContext *sctx = SigCtx(thr); 2051 if (sctx == 0) 2052 return; 2053 atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); 2054 internal_sigfillset(&sctx->emptyset); 2055 int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset); 2056 CHECK_EQ(res, 0); 2057 for (int sig = 0; sig < kSigCount; sig++) { 2058 SignalDesc *signal = &sctx->pending_signals[sig]; 2059 if (signal->armed) { 2060 signal->armed = false; 2061 CallUserSignalHandler(thr, false, true, sig, &signal->siginfo, 2062 &signal->ctx); 2063 } 2064 } 2065 res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0); 2066 CHECK_EQ(res, 0); 2067 atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); 2068 } 2069 2070 } // namespace __tsan 2071 2072 static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { 2073 return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP || 2074 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || 2075 // If we are sending signal to ourselves, we must process it now. 2076 (sctx && sig == sctx->int_signal_send); 2077 } 2078 2079 void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { 2080 ThreadState *thr = cur_thread_init(); 2081 ThreadSignalContext *sctx = SigCtx(thr); 2082 if (sig < 0 || sig >= kSigCount) { 2083 VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig); 2084 return; 2085 } 2086 // Don't mess with synchronous signals. 2087 const bool sync = is_sync_signal(sctx, sig); 2088 if (sync || 2089 // If we are in blocking function, we can safely process it now 2090 // (but check if we are in a recursive interceptor, 2091 // i.e. pthread_join()->munmap()). 2092 (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) { 2093 atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); 2094 if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) { 2095 atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed); 2096 CallUserSignalHandler(thr, sync, true, sig, info, ctx); 2097 atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed); 2098 } else { 2099 // Be very conservative with when we do acquire in this case. 2100 // It's unsafe to do acquire in async handlers, because ThreadState 2101 // can be in inconsistent state. 2102 // SIGSYS looks relatively safe -- it's synchronous and can actually 2103 // need some global state. 2104 bool acq = (sig == SIGSYS); 2105 CallUserSignalHandler(thr, sync, acq, sig, info, ctx); 2106 } 2107 atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); 2108 return; 2109 } 2110 2111 if (sctx == 0) 2112 return; 2113 SignalDesc *signal = &sctx->pending_signals[sig]; 2114 if (signal->armed == false) { 2115 signal->armed = true; 2116 internal_memcpy(&signal->siginfo, info, sizeof(*info)); 2117 internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); 2118 atomic_store(&thr->pending_signals, 1, memory_order_relaxed); 2119 } 2120 } 2121 2122 TSAN_INTERCEPTOR(int, raise, int sig) { 2123 SCOPED_TSAN_INTERCEPTOR(raise, sig); 2124 ThreadSignalContext *sctx = SigCtx(thr); 2125 CHECK_NE(sctx, 0); 2126 int prev = sctx->int_signal_send; 2127 sctx->int_signal_send = sig; 2128 int res = REAL(raise)(sig); 2129 CHECK_EQ(sctx->int_signal_send, sig); 2130 sctx->int_signal_send = prev; 2131 return res; 2132 } 2133 2134 TSAN_INTERCEPTOR(int, kill, int pid, int sig) { 2135 SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); 2136 ThreadSignalContext *sctx = SigCtx(thr); 2137 CHECK_NE(sctx, 0); 2138 int prev = sctx->int_signal_send; 2139 if (pid == (int)internal_getpid()) { 2140 sctx->int_signal_send = sig; 2141 } 2142 int res = REAL(kill)(pid, sig); 2143 if (pid == (int)internal_getpid()) { 2144 CHECK_EQ(sctx->int_signal_send, sig); 2145 sctx->int_signal_send = prev; 2146 } 2147 return res; 2148 } 2149 2150 TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { 2151 SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); 2152 ThreadSignalContext *sctx = SigCtx(thr); 2153 CHECK_NE(sctx, 0); 2154 int prev = sctx->int_signal_send; 2155 bool self = pthread_equal(tid, pthread_self()); 2156 if (self) 2157 sctx->int_signal_send = sig; 2158 int res = REAL(pthread_kill)(tid, sig); 2159 if (self) { 2160 CHECK_EQ(sctx->int_signal_send, sig); 2161 sctx->int_signal_send = prev; 2162 } 2163 return res; 2164 } 2165 2166 TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { 2167 SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); 2168 // It's intercepted merely to process pending signals. 2169 return REAL(gettimeofday)(tv, tz); 2170 } 2171 2172 TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, 2173 void *hints, void *rv) { 2174 SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); 2175 // We miss atomic synchronization in getaddrinfo, 2176 // and can report false race between malloc and free 2177 // inside of getaddrinfo. So ignore memory accesses. 2178 ThreadIgnoreBegin(thr, pc); 2179 int res = REAL(getaddrinfo)(node, service, hints, rv); 2180 ThreadIgnoreEnd(thr); 2181 return res; 2182 } 2183 2184 TSAN_INTERCEPTOR(int, fork, int fake) { 2185 if (in_symbolizer()) 2186 return REAL(fork)(fake); 2187 SCOPED_INTERCEPTOR_RAW(fork, fake); 2188 return REAL(fork)(fake); 2189 } 2190 2191 void atfork_prepare() { 2192 if (in_symbolizer()) 2193 return; 2194 ThreadState *thr = cur_thread(); 2195 const uptr pc = StackTrace::GetCurrentPc(); 2196 ForkBefore(thr, pc); 2197 } 2198 2199 void atfork_parent() { 2200 if (in_symbolizer()) 2201 return; 2202 ThreadState *thr = cur_thread(); 2203 const uptr pc = StackTrace::GetCurrentPc(); 2204 ForkParentAfter(thr, pc); 2205 } 2206 2207 void atfork_child() { 2208 if (in_symbolizer()) 2209 return; 2210 ThreadState *thr = cur_thread(); 2211 const uptr pc = StackTrace::GetCurrentPc(); 2212 ForkChildAfter(thr, pc, true); 2213 FdOnFork(thr, pc); 2214 } 2215 2216 #if !SANITIZER_IOS 2217 TSAN_INTERCEPTOR(int, vfork, int fake) { 2218 // Some programs (e.g. openjdk) call close for all file descriptors 2219 // in the child process. Under tsan it leads to false positives, because 2220 // address space is shared, so the parent process also thinks that 2221 // the descriptors are closed (while they are actually not). 2222 // This leads to false positives due to missed synchronization. 2223 // Strictly saying this is undefined behavior, because vfork child is not 2224 // allowed to call any functions other than exec/exit. But this is what 2225 // openjdk does, so we want to handle it. 2226 // We could disable interceptors in the child process. But it's not possible 2227 // to simply intercept and wrap vfork, because vfork child is not allowed 2228 // to return from the function that calls vfork, and that's exactly what 2229 // we would do. So this would require some assembly trickery as well. 2230 // Instead we simply turn vfork into fork. 2231 return WRAP(fork)(fake); 2232 } 2233 #endif 2234 2235 #if SANITIZER_LINUX 2236 TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags, 2237 void *arg, int *parent_tid, void *tls, pid_t *child_tid) { 2238 SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls, 2239 child_tid); 2240 struct Arg { 2241 int (*fn)(void *); 2242 void *arg; 2243 }; 2244 auto wrapper = +[](void *p) -> int { 2245 auto *thr = cur_thread(); 2246 uptr pc = GET_CURRENT_PC(); 2247 // Start the background thread for fork, but not for clone. 2248 // For fork we did this always and it's known to work (or user code has 2249 // adopted). But if we do this for the new clone interceptor some code 2250 // (sandbox2) fails. So model we used to do for years and don't start the 2251 // background thread after clone. 2252 ForkChildAfter(thr, pc, false); 2253 FdOnFork(thr, pc); 2254 auto *arg = static_cast<Arg *>(p); 2255 return arg->fn(arg->arg); 2256 }; 2257 ForkBefore(thr, pc); 2258 Arg arg_wrapper = {fn, arg}; 2259 int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls, 2260 child_tid); 2261 ForkParentAfter(thr, pc); 2262 return pid; 2263 } 2264 #endif 2265 2266 #if !SANITIZER_APPLE && !SANITIZER_ANDROID 2267 typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size, 2268 void *data); 2269 struct dl_iterate_phdr_data { 2270 ThreadState *thr; 2271 uptr pc; 2272 dl_iterate_phdr_cb_t cb; 2273 void *data; 2274 }; 2275 2276 static bool IsAppNotRodata(uptr addr) { 2277 return IsAppMem(addr) && *MemToShadow(addr) != Shadow::kRodata; 2278 } 2279 2280 static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, 2281 void *data) { 2282 dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data; 2283 // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later 2284 // accessible in dl_iterate_phdr callback. But we don't see synchronization 2285 // inside of dynamic linker, so we "unpoison" it here in order to not 2286 // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough 2287 // because some libc functions call __libc_dlopen. 2288 if (info && IsAppNotRodata((uptr)info->dlpi_name)) 2289 MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, 2290 internal_strlen(info->dlpi_name)); 2291 int res = cbdata->cb(info, size, cbdata->data); 2292 // Perform the check one more time in case info->dlpi_name was overwritten 2293 // by user callback. 2294 if (info && IsAppNotRodata((uptr)info->dlpi_name)) 2295 MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, 2296 internal_strlen(info->dlpi_name)); 2297 return res; 2298 } 2299 2300 TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) { 2301 SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr, cb, data); 2302 dl_iterate_phdr_data cbdata; 2303 cbdata.thr = thr; 2304 cbdata.pc = pc; 2305 cbdata.cb = cb; 2306 cbdata.data = data; 2307 int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata); 2308 return res; 2309 } 2310 #endif 2311 2312 static int OnExit(ThreadState *thr) { 2313 int status = Finalize(thr); 2314 FlushStreams(); 2315 return status; 2316 } 2317 2318 struct TsanInterceptorContext { 2319 ThreadState *thr; 2320 const uptr pc; 2321 }; 2322 2323 #if !SANITIZER_APPLE 2324 static void HandleRecvmsg(ThreadState *thr, uptr pc, 2325 __sanitizer_msghdr *msg) { 2326 int fds[64]; 2327 int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds)); 2328 for (int i = 0; i < cnt; i++) 2329 FdEventCreate(thr, pc, fds[i]); 2330 } 2331 #endif 2332 2333 #include "sanitizer_common/sanitizer_platform_interceptors.h" 2334 // Causes interceptor recursion (getaddrinfo() and fopen()) 2335 #undef SANITIZER_INTERCEPT_GETADDRINFO 2336 // We define our own. 2337 #if SANITIZER_INTERCEPT_TLS_GET_ADDR 2338 #define NEED_TLS_GET_ADDR 2339 #endif 2340 #undef SANITIZER_INTERCEPT_TLS_GET_ADDR 2341 #define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1 2342 #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK 2343 2344 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) 2345 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ 2346 INTERCEPT_FUNCTION_VER(name, ver) 2347 #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \ 2348 (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name)) 2349 2350 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ 2351 MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ 2352 ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ 2353 true) 2354 2355 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ 2356 MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ 2357 ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ 2358 false) 2359 2360 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ 2361 SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ 2362 TsanInterceptorContext _ctx = {thr, pc}; \ 2363 ctx = (void *)&_ctx; \ 2364 (void)ctx; 2365 2366 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ 2367 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ 2368 TsanInterceptorContext _ctx = {thr, pc}; \ 2369 ctx = (void *)&_ctx; \ 2370 (void)ctx; 2371 2372 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ 2373 if (path) \ 2374 Acquire(thr, pc, File2addr(path)); \ 2375 if (file) { \ 2376 int fd = fileno_unlocked(file); \ 2377 if (fd >= 0) FdFileCreate(thr, pc, fd); \ 2378 } 2379 2380 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ 2381 if (file) { \ 2382 int fd = fileno_unlocked(file); \ 2383 FdClose(thr, pc, fd); \ 2384 } 2385 2386 #define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ 2387 ({ \ 2388 CheckNoDeepBind(filename, flag); \ 2389 ThreadIgnoreBegin(thr, 0); \ 2390 void *res = REAL(dlopen)(filename, flag); \ 2391 ThreadIgnoreEnd(thr); \ 2392 res; \ 2393 }) 2394 2395 #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ 2396 libignore()->OnLibraryLoaded(filename) 2397 2398 #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ 2399 libignore()->OnLibraryUnloaded() 2400 2401 #define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \ 2402 Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u) 2403 2404 #define COMMON_INTERCEPTOR_RELEASE(ctx, u) \ 2405 Release(((TsanInterceptorContext *) ctx)->thr, pc, u) 2406 2407 #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ 2408 Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path)) 2409 2410 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ 2411 FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2412 2413 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ 2414 FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2415 2416 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ 2417 FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2418 2419 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ 2420 FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) 2421 2422 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ 2423 ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) 2424 2425 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ 2426 if (pthread_equal(pthread_self(), reinterpret_cast<void *>(thread))) \ 2427 COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name); \ 2428 else \ 2429 __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name) 2430 2431 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) 2432 2433 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \ 2434 OnExit(((TsanInterceptorContext *) ctx)->thr) 2435 2436 #define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \ 2437 MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \ 2438 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2439 2440 #define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \ 2441 MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \ 2442 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2443 2444 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ 2445 MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \ 2446 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2447 2448 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ 2449 MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ 2450 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2451 2452 #define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \ 2453 MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \ 2454 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2455 2456 #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ 2457 off) \ 2458 do { \ 2459 return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \ 2460 off); \ 2461 } while (false) 2462 2463 #if !SANITIZER_APPLE 2464 #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ 2465 HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ 2466 ((TsanInterceptorContext *)ctx)->pc, msg) 2467 #endif 2468 2469 #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ 2470 if (TsanThread *t = GetCurrentThread()) { \ 2471 *begin = t->tls_begin(); \ 2472 *end = t->tls_end(); \ 2473 } else { \ 2474 *begin = *end = 0; \ 2475 } 2476 2477 #define COMMON_INTERCEPTOR_USER_CALLBACK_START() \ 2478 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() 2479 2480 #define COMMON_INTERCEPTOR_USER_CALLBACK_END() \ 2481 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() 2482 2483 #include "sanitizer_common/sanitizer_common_interceptors.inc" 2484 2485 static int sigaction_impl(int sig, const __sanitizer_sigaction *act, 2486 __sanitizer_sigaction *old); 2487 static __sanitizer_sighandler_ptr signal_impl(int sig, 2488 __sanitizer_sighandler_ptr h); 2489 2490 #define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \ 2491 { return sigaction_impl(signo, act, oldact); } 2492 2493 #define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ 2494 { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); } 2495 2496 #include "sanitizer_common/sanitizer_signal_interceptors.inc" 2497 2498 int sigaction_impl(int sig, const __sanitizer_sigaction *act, 2499 __sanitizer_sigaction *old) { 2500 // Note: if we call REAL(sigaction) directly for any reason without proxying 2501 // the signal handler through sighandler, very bad things will happen. 2502 // The handler will run synchronously and corrupt tsan per-thread state. 2503 SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); 2504 if (sig <= 0 || sig >= kSigCount) { 2505 errno = errno_EINVAL; 2506 return -1; 2507 } 2508 __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; 2509 __sanitizer_sigaction old_stored; 2510 if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored)); 2511 __sanitizer_sigaction newact; 2512 if (act) { 2513 // Copy act into sigactions[sig]. 2514 // Can't use struct copy, because compiler can emit call to memcpy. 2515 // Can't use internal_memcpy, because it copies byte-by-byte, 2516 // and signal handler reads the handler concurrently. It it can read 2517 // some bytes from old value and some bytes from new value. 2518 // Use volatile to prevent insertion of memcpy. 2519 sigactions[sig].handler = 2520 *(volatile __sanitizer_sighandler_ptr const *)&act->handler; 2521 sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags; 2522 internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, 2523 sizeof(sigactions[sig].sa_mask)); 2524 #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD 2525 sigactions[sig].sa_restorer = act->sa_restorer; 2526 #endif 2527 internal_memcpy(&newact, act, sizeof(newact)); 2528 internal_sigfillset(&newact.sa_mask); 2529 if ((act->sa_flags & SA_SIGINFO) || 2530 ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) { 2531 newact.sa_flags |= SA_SIGINFO; 2532 newact.sigaction = sighandler; 2533 } 2534 ReleaseStore(thr, pc, (uptr)&sigactions[sig]); 2535 act = &newact; 2536 } 2537 int res = REAL(sigaction)(sig, act, old); 2538 if (res == 0 && old && old->sigaction == sighandler) 2539 internal_memcpy(old, &old_stored, sizeof(*old)); 2540 return res; 2541 } 2542 2543 static __sanitizer_sighandler_ptr signal_impl(int sig, 2544 __sanitizer_sighandler_ptr h) { 2545 __sanitizer_sigaction act; 2546 act.handler = h; 2547 internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); 2548 act.sa_flags = 0; 2549 __sanitizer_sigaction old; 2550 int res = sigaction_symname(sig, &act, &old); 2551 if (res) return (__sanitizer_sighandler_ptr)sig_err; 2552 return old.handler; 2553 } 2554 2555 #define TSAN_SYSCALL() \ 2556 ThreadState *thr = cur_thread(); \ 2557 if (thr->ignore_interceptors) \ 2558 return; \ 2559 ScopedSyscall scoped_syscall(thr) 2560 2561 struct ScopedSyscall { 2562 ThreadState *thr; 2563 2564 explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); } 2565 2566 ~ScopedSyscall() { 2567 ProcessPendingSignals(thr); 2568 } 2569 }; 2570 2571 #if !SANITIZER_FREEBSD && !SANITIZER_APPLE 2572 static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { 2573 TSAN_SYSCALL(); 2574 MemoryAccessRange(thr, pc, p, s, write); 2575 } 2576 2577 static USED void syscall_acquire(uptr pc, uptr addr) { 2578 TSAN_SYSCALL(); 2579 Acquire(thr, pc, addr); 2580 DPrintf("syscall_acquire(0x%zx))\n", addr); 2581 } 2582 2583 static USED void syscall_release(uptr pc, uptr addr) { 2584 TSAN_SYSCALL(); 2585 DPrintf("syscall_release(0x%zx)\n", addr); 2586 Release(thr, pc, addr); 2587 } 2588 2589 static void syscall_fd_close(uptr pc, int fd) { 2590 auto *thr = cur_thread(); 2591 FdClose(thr, pc, fd); 2592 } 2593 2594 static USED void syscall_fd_acquire(uptr pc, int fd) { 2595 TSAN_SYSCALL(); 2596 FdAcquire(thr, pc, fd); 2597 DPrintf("syscall_fd_acquire(%d)\n", fd); 2598 } 2599 2600 static USED void syscall_fd_release(uptr pc, int fd) { 2601 TSAN_SYSCALL(); 2602 DPrintf("syscall_fd_release(%d)\n", fd); 2603 FdRelease(thr, pc, fd); 2604 } 2605 2606 static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); } 2607 2608 static void syscall_post_fork(uptr pc, int pid) { 2609 ThreadState *thr = cur_thread(); 2610 if (pid == 0) { 2611 // child 2612 ForkChildAfter(thr, pc, true); 2613 FdOnFork(thr, pc); 2614 } else if (pid > 0) { 2615 // parent 2616 ForkParentAfter(thr, pc); 2617 } else { 2618 // error 2619 ForkParentAfter(thr, pc); 2620 } 2621 } 2622 #endif 2623 2624 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ 2625 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) 2626 2627 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ 2628 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) 2629 2630 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ 2631 do { \ 2632 (void)(p); \ 2633 (void)(s); \ 2634 } while (false) 2635 2636 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ 2637 do { \ 2638 (void)(p); \ 2639 (void)(s); \ 2640 } while (false) 2641 2642 #define COMMON_SYSCALL_ACQUIRE(addr) \ 2643 syscall_acquire(GET_CALLER_PC(), (uptr)(addr)) 2644 2645 #define COMMON_SYSCALL_RELEASE(addr) \ 2646 syscall_release(GET_CALLER_PC(), (uptr)(addr)) 2647 2648 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd) 2649 2650 #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd) 2651 2652 #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd) 2653 2654 #define COMMON_SYSCALL_PRE_FORK() \ 2655 syscall_pre_fork(GET_CALLER_PC()) 2656 2657 #define COMMON_SYSCALL_POST_FORK(res) \ 2658 syscall_post_fork(GET_CALLER_PC(), res) 2659 2660 #include "sanitizer_common/sanitizer_common_syscalls.inc" 2661 #include "sanitizer_common/sanitizer_syscalls_netbsd.inc" 2662 2663 #ifdef NEED_TLS_GET_ADDR 2664 2665 static void handle_tls_addr(void *arg, void *res) { 2666 ThreadState *thr = cur_thread(); 2667 if (!thr) 2668 return; 2669 DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, 2670 thr->tls_addr + thr->tls_size); 2671 if (!dtv) 2672 return; 2673 // New DTLS block has been allocated. 2674 MemoryResetRange(thr, 0, dtv->beg, dtv->size); 2675 } 2676 2677 #if !SANITIZER_S390 2678 // Define own interceptor instead of sanitizer_common's for three reasons: 2679 // 1. It must not process pending signals. 2680 // Signal handlers may contain MOVDQA instruction (see below). 2681 // 2. It must be as simple as possible to not contain MOVDQA. 2682 // 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which 2683 // is empty for tsan (meant only for msan). 2684 // Note: __tls_get_addr can be called with mis-aligned stack due to: 2685 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 2686 // So the interceptor must work with mis-aligned stack, in particular, does not 2687 // execute MOVDQA with stack addresses. 2688 TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { 2689 void *res = REAL(__tls_get_addr)(arg); 2690 handle_tls_addr(arg, res); 2691 return res; 2692 } 2693 #else // SANITIZER_S390 2694 TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { 2695 uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset)); 2696 char *tp = static_cast<char *>(__builtin_thread_pointer()); 2697 handle_tls_addr(arg, res + tp); 2698 return res; 2699 } 2700 #endif 2701 #endif 2702 2703 #if SANITIZER_NETBSD 2704 TSAN_INTERCEPTOR(void, _lwp_exit) { 2705 SCOPED_TSAN_INTERCEPTOR(_lwp_exit); 2706 DestroyThreadState(); 2707 REAL(_lwp_exit)(); 2708 } 2709 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit) 2710 #else 2711 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT 2712 #endif 2713 2714 #if SANITIZER_FREEBSD 2715 TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) { 2716 SCOPED_TSAN_INTERCEPTOR(thr_exit, state); 2717 DestroyThreadState(); 2718 REAL(thr_exit(state)); 2719 } 2720 #define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit) 2721 #else 2722 #define TSAN_MAYBE_INTERCEPT_THR_EXIT 2723 #endif 2724 2725 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_init, void *c, void *a) 2726 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_destroy, void *c) 2727 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_signal, void *c) 2728 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_broadcast, void *c) 2729 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_wait, void *c, void *m) 2730 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_init, void *m, void *a) 2731 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_destroy, void *m) 2732 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_lock, void *m) 2733 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_trylock, void *m) 2734 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_unlock, void *m) 2735 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_init, void *l, void *a) 2736 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_destroy, void *l) 2737 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_rdlock, void *l) 2738 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_tryrdlock, void *l) 2739 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_wrlock, void *l) 2740 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_trywrlock, void *l) 2741 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_unlock, void *l) 2742 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, once, void *o, void (*i)()) 2743 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, sigmask, int f, void *n, void *o) 2744 2745 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a) 2746 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c) 2747 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c) 2748 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m) 2749 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c) 2750 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a) 2751 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m) 2752 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m) 2753 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a) 2754 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m) 2755 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m) 2756 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m) 2757 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m) 2758 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m) 2759 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m) 2760 TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)()) 2761 TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b, 2762 void *c) 2763 2764 namespace __tsan { 2765 2766 static void finalize(void *arg) { 2767 ThreadState *thr = cur_thread(); 2768 int status = Finalize(thr); 2769 // Make sure the output is not lost. 2770 FlushStreams(); 2771 if (status) 2772 Die(); 2773 } 2774 2775 #if !SANITIZER_APPLE && !SANITIZER_ANDROID 2776 static void unreachable() { 2777 Report("FATAL: ThreadSanitizer: unreachable called\n"); 2778 Die(); 2779 } 2780 #endif 2781 2782 // Define default implementation since interception of libdispatch is optional. 2783 SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} 2784 2785 void InitializeInterceptors() { 2786 #if !SANITIZER_APPLE 2787 // We need to setup it early, because functions like dlsym() can call it. 2788 REAL(memset) = internal_memset; 2789 REAL(memcpy) = internal_memcpy; 2790 #endif 2791 2792 new(interceptor_ctx()) InterceptorContext(); 2793 2794 InitializeCommonInterceptors(); 2795 InitializeSignalInterceptors(); 2796 InitializeLibdispatchInterceptors(); 2797 2798 #if !SANITIZER_APPLE 2799 // We can not use TSAN_INTERCEPT to get setjmp addr, 2800 // because it does &setjmp and setjmp is not present in some versions of libc. 2801 using __interception::InterceptFunction; 2802 InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); 2803 InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); 2804 InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, 2805 0); 2806 #if !SANITIZER_NETBSD 2807 InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); 2808 #endif 2809 #endif 2810 2811 TSAN_INTERCEPT(longjmp_symname); 2812 TSAN_INTERCEPT(siglongjmp_symname); 2813 #if SANITIZER_NETBSD 2814 TSAN_INTERCEPT(_longjmp); 2815 #endif 2816 2817 TSAN_INTERCEPT(malloc); 2818 TSAN_INTERCEPT(__libc_memalign); 2819 TSAN_INTERCEPT(calloc); 2820 TSAN_INTERCEPT(realloc); 2821 TSAN_INTERCEPT(reallocarray); 2822 TSAN_INTERCEPT(free); 2823 TSAN_INTERCEPT(cfree); 2824 TSAN_INTERCEPT(munmap); 2825 TSAN_MAYBE_INTERCEPT_MEMALIGN; 2826 TSAN_INTERCEPT(valloc); 2827 TSAN_MAYBE_INTERCEPT_PVALLOC; 2828 TSAN_INTERCEPT(posix_memalign); 2829 2830 TSAN_INTERCEPT(strcpy); 2831 TSAN_INTERCEPT(strncpy); 2832 TSAN_INTERCEPT(strdup); 2833 2834 TSAN_INTERCEPT(pthread_create); 2835 TSAN_INTERCEPT(pthread_join); 2836 TSAN_INTERCEPT(pthread_detach); 2837 TSAN_INTERCEPT(pthread_exit); 2838 #if SANITIZER_LINUX 2839 TSAN_INTERCEPT(pthread_tryjoin_np); 2840 TSAN_INTERCEPT(pthread_timedjoin_np); 2841 #endif 2842 2843 TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE); 2844 TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE); 2845 TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE); 2846 TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE); 2847 TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE); 2848 TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE); 2849 2850 TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT; 2851 2852 TSAN_INTERCEPT(pthread_mutex_init); 2853 TSAN_INTERCEPT(pthread_mutex_destroy); 2854 TSAN_INTERCEPT(pthread_mutex_trylock); 2855 TSAN_INTERCEPT(pthread_mutex_timedlock); 2856 2857 TSAN_INTERCEPT(pthread_spin_init); 2858 TSAN_INTERCEPT(pthread_spin_destroy); 2859 TSAN_INTERCEPT(pthread_spin_lock); 2860 TSAN_INTERCEPT(pthread_spin_trylock); 2861 TSAN_INTERCEPT(pthread_spin_unlock); 2862 2863 TSAN_INTERCEPT(pthread_rwlock_init); 2864 TSAN_INTERCEPT(pthread_rwlock_destroy); 2865 TSAN_INTERCEPT(pthread_rwlock_rdlock); 2866 TSAN_INTERCEPT(pthread_rwlock_tryrdlock); 2867 TSAN_INTERCEPT(pthread_rwlock_timedrdlock); 2868 TSAN_INTERCEPT(pthread_rwlock_wrlock); 2869 TSAN_INTERCEPT(pthread_rwlock_trywrlock); 2870 TSAN_INTERCEPT(pthread_rwlock_timedwrlock); 2871 TSAN_INTERCEPT(pthread_rwlock_unlock); 2872 2873 TSAN_INTERCEPT(pthread_barrier_init); 2874 TSAN_INTERCEPT(pthread_barrier_destroy); 2875 TSAN_INTERCEPT(pthread_barrier_wait); 2876 2877 TSAN_INTERCEPT(pthread_once); 2878 2879 TSAN_INTERCEPT(fstat); 2880 TSAN_MAYBE_INTERCEPT___FXSTAT; 2881 TSAN_MAYBE_INTERCEPT_FSTAT64; 2882 TSAN_MAYBE_INTERCEPT___FXSTAT64; 2883 TSAN_INTERCEPT(open); 2884 TSAN_MAYBE_INTERCEPT_OPEN64; 2885 TSAN_INTERCEPT(creat); 2886 TSAN_MAYBE_INTERCEPT_CREAT64; 2887 TSAN_INTERCEPT(dup); 2888 TSAN_INTERCEPT(dup2); 2889 TSAN_INTERCEPT(dup3); 2890 TSAN_MAYBE_INTERCEPT_EVENTFD; 2891 TSAN_MAYBE_INTERCEPT_SIGNALFD; 2892 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT; 2893 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1; 2894 TSAN_INTERCEPT(socket); 2895 TSAN_INTERCEPT(socketpair); 2896 TSAN_INTERCEPT(connect); 2897 TSAN_INTERCEPT(bind); 2898 TSAN_INTERCEPT(listen); 2899 TSAN_MAYBE_INTERCEPT_EPOLL; 2900 TSAN_INTERCEPT(close); 2901 TSAN_MAYBE_INTERCEPT___CLOSE; 2902 TSAN_MAYBE_INTERCEPT___RES_ICLOSE; 2903 TSAN_INTERCEPT(pipe); 2904 TSAN_INTERCEPT(pipe2); 2905 2906 TSAN_INTERCEPT(unlink); 2907 TSAN_INTERCEPT(tmpfile); 2908 TSAN_MAYBE_INTERCEPT_TMPFILE64; 2909 TSAN_INTERCEPT(abort); 2910 TSAN_INTERCEPT(rmdir); 2911 TSAN_INTERCEPT(closedir); 2912 2913 TSAN_INTERCEPT(sigsuspend); 2914 TSAN_INTERCEPT(sigblock); 2915 TSAN_INTERCEPT(sigsetmask); 2916 TSAN_INTERCEPT(pthread_sigmask); 2917 TSAN_INTERCEPT(raise); 2918 TSAN_INTERCEPT(kill); 2919 TSAN_INTERCEPT(pthread_kill); 2920 TSAN_INTERCEPT(sleep); 2921 TSAN_INTERCEPT(usleep); 2922 TSAN_INTERCEPT(nanosleep); 2923 TSAN_INTERCEPT(pause); 2924 TSAN_INTERCEPT(gettimeofday); 2925 TSAN_INTERCEPT(getaddrinfo); 2926 2927 TSAN_INTERCEPT(fork); 2928 TSAN_INTERCEPT(vfork); 2929 #if SANITIZER_LINUX 2930 TSAN_INTERCEPT(clone); 2931 #endif 2932 #if !SANITIZER_ANDROID 2933 TSAN_INTERCEPT(dl_iterate_phdr); 2934 #endif 2935 TSAN_MAYBE_INTERCEPT_ON_EXIT; 2936 TSAN_INTERCEPT(__cxa_atexit); 2937 TSAN_INTERCEPT(_exit); 2938 2939 #ifdef NEED_TLS_GET_ADDR 2940 #if !SANITIZER_S390 2941 TSAN_INTERCEPT(__tls_get_addr); 2942 #else 2943 TSAN_INTERCEPT(__tls_get_addr_internal); 2944 TSAN_INTERCEPT(__tls_get_offset); 2945 #endif 2946 #endif 2947 2948 TSAN_MAYBE_INTERCEPT__LWP_EXIT; 2949 TSAN_MAYBE_INTERCEPT_THR_EXIT; 2950 2951 #if !SANITIZER_APPLE && !SANITIZER_ANDROID 2952 // Need to setup it, because interceptors check that the function is resolved. 2953 // But atexit is emitted directly into the module, so can't be resolved. 2954 REAL(atexit) = (int(*)(void(*)()))unreachable; 2955 #endif 2956 2957 if (REAL(__cxa_atexit)(&finalize, 0, 0)) { 2958 Printf("ThreadSanitizer: failed to setup atexit callback\n"); 2959 Die(); 2960 } 2961 if (pthread_atfork(atfork_prepare, atfork_parent, atfork_child)) { 2962 Printf("ThreadSanitizer: failed to setup atfork callbacks\n"); 2963 Die(); 2964 } 2965 2966 #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD 2967 if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) { 2968 Printf("ThreadSanitizer: failed to create thread key\n"); 2969 Die(); 2970 } 2971 #endif 2972 2973 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_init); 2974 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_destroy); 2975 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_signal); 2976 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_broadcast); 2977 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_wait); 2978 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_init); 2979 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_destroy); 2980 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_lock); 2981 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_trylock); 2982 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_unlock); 2983 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_init); 2984 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_destroy); 2985 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_rdlock); 2986 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_tryrdlock); 2987 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_wrlock); 2988 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_trywrlock); 2989 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_unlock); 2990 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(once); 2991 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(sigmask); 2992 2993 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init); 2994 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal); 2995 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast); 2996 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait); 2997 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy); 2998 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init); 2999 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy); 3000 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock); 3001 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init); 3002 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy); 3003 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock); 3004 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock); 3005 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock); 3006 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock); 3007 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock); 3008 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once); 3009 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask); 3010 3011 FdInit(); 3012 } 3013 3014 } // namespace __tsan 3015 3016 // Invisible barrier for tests. 3017 // There were several unsuccessful iterations for this functionality: 3018 // 1. Initially it was implemented in user code using 3019 // REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on 3020 // MacOS. Futexes are linux-specific for this matter. 3021 // 2. Then we switched to atomics+usleep(10). But usleep produced parasitic 3022 // "as-if synchronized via sleep" messages in reports which failed some 3023 // output tests. 3024 // 3. Then we switched to atomics+sched_yield. But this produced tons of tsan- 3025 // visible events, which lead to "failed to restore stack trace" failures. 3026 // Note that no_sanitize_thread attribute does not turn off atomic interception 3027 // so attaching it to the function defined in user code does not help. 3028 // That's why we now have what we have. 3029 constexpr u32 kBarrierThreadBits = 10; 3030 constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits; 3031 3032 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init( 3033 atomic_uint32_t *barrier, u32 num_threads) { 3034 if (num_threads >= kBarrierThreads) { 3035 Printf("barrier_init: count is too large (%d)\n", num_threads); 3036 Die(); 3037 } 3038 // kBarrierThreadBits lsb is thread count, 3039 // the remaining are count of entered threads. 3040 atomic_store(barrier, num_threads, memory_order_relaxed); 3041 } 3042 3043 static u32 barrier_epoch(u32 value) { 3044 return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1)); 3045 } 3046 3047 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait( 3048 atomic_uint32_t *barrier) { 3049 u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed); 3050 u32 old_epoch = barrier_epoch(old); 3051 if (barrier_epoch(old + kBarrierThreads) != old_epoch) { 3052 FutexWake(barrier, (1 << 30)); 3053 return; 3054 } 3055 for (;;) { 3056 u32 cur = atomic_load(barrier, memory_order_relaxed); 3057 if (barrier_epoch(cur) != old_epoch) 3058 return; 3059 FutexWait(barrier, cur); 3060 } 3061 } 3062