110d565efSmrg //===-- tsan_rtl.cc -------------------------------------------------------===//
210d565efSmrg //
310d565efSmrg // This file is distributed under the University of Illinois Open Source
410d565efSmrg // License. See LICENSE.TXT for details.
510d565efSmrg //
610d565efSmrg //===----------------------------------------------------------------------===//
710d565efSmrg //
810d565efSmrg // This file is a part of ThreadSanitizer (TSan), a race detector.
910d565efSmrg //
1010d565efSmrg // Main file (entry points) for the TSan run-time.
1110d565efSmrg //===----------------------------------------------------------------------===//
1210d565efSmrg
1310d565efSmrg #include "sanitizer_common/sanitizer_atomic.h"
1410d565efSmrg #include "sanitizer_common/sanitizer_common.h"
15c7a68eb7Smrg #include "sanitizer_common/sanitizer_file.h"
1610d565efSmrg #include "sanitizer_common/sanitizer_libc.h"
1710d565efSmrg #include "sanitizer_common/sanitizer_stackdepot.h"
1810d565efSmrg #include "sanitizer_common/sanitizer_placement_new.h"
1910d565efSmrg #include "sanitizer_common/sanitizer_symbolizer.h"
2010d565efSmrg #include "tsan_defs.h"
2110d565efSmrg #include "tsan_platform.h"
2210d565efSmrg #include "tsan_rtl.h"
2310d565efSmrg #include "tsan_mman.h"
2410d565efSmrg #include "tsan_suppressions.h"
2510d565efSmrg #include "tsan_symbolize.h"
2610d565efSmrg #include "ubsan/ubsan_init.h"
2710d565efSmrg
2810d565efSmrg #ifdef __SSE3__
2910d565efSmrg // <emmintrin.h> transitively includes <stdlib.h>,
3010d565efSmrg // and it's prohibited to include std headers into tsan runtime.
3110d565efSmrg // So we do this dirty trick.
3210d565efSmrg #define _MM_MALLOC_H_INCLUDED
3310d565efSmrg #define __MM_MALLOC_H
3410d565efSmrg #include <emmintrin.h>
3510d565efSmrg typedef __m128i m128;
3610d565efSmrg #endif
3710d565efSmrg
3810d565efSmrg volatile int __tsan_resumed = 0;
3910d565efSmrg
__tsan_resume()4010d565efSmrg extern "C" void __tsan_resume() {
4110d565efSmrg __tsan_resumed = 1;
4210d565efSmrg }
4310d565efSmrg
4410d565efSmrg namespace __tsan {
4510d565efSmrg
4610d565efSmrg #if !SANITIZER_GO && !SANITIZER_MAC
4710d565efSmrg __attribute__((tls_model("initial-exec")))
4810d565efSmrg THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
4910d565efSmrg #endif
5010d565efSmrg static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
5110d565efSmrg Context *ctx;
5210d565efSmrg
5310d565efSmrg // Can be overriden by a front-end.
5410d565efSmrg #ifdef TSAN_EXTERNAL_HOOKS
5510d565efSmrg bool OnFinalize(bool failed);
5610d565efSmrg void OnInitialize();
5710d565efSmrg #else
5810d565efSmrg SANITIZER_WEAK_CXX_DEFAULT_IMPL
OnFinalize(bool failed)5910d565efSmrg bool OnFinalize(bool failed) {
6010d565efSmrg return failed;
6110d565efSmrg }
6210d565efSmrg SANITIZER_WEAK_CXX_DEFAULT_IMPL
OnInitialize()6310d565efSmrg void OnInitialize() {}
6410d565efSmrg #endif
6510d565efSmrg
6610d565efSmrg static char thread_registry_placeholder[sizeof(ThreadRegistry)];
6710d565efSmrg
CreateThreadContext(u32 tid)6810d565efSmrg static ThreadContextBase *CreateThreadContext(u32 tid) {
6910d565efSmrg // Map thread trace when context is created.
7010d565efSmrg char name[50];
7110d565efSmrg internal_snprintf(name, sizeof(name), "trace %u", tid);
7210d565efSmrg MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name);
7310d565efSmrg const uptr hdr = GetThreadTraceHeader(tid);
7410d565efSmrg internal_snprintf(name, sizeof(name), "trace header %u", tid);
7510d565efSmrg MapThreadTrace(hdr, sizeof(Trace), name);
7610d565efSmrg new((void*)hdr) Trace();
7710d565efSmrg // We are going to use only a small part of the trace with the default
7810d565efSmrg // value of history_size. However, the constructor writes to the whole trace.
7910d565efSmrg // Unmap the unused part.
8010d565efSmrg uptr hdr_end = hdr + sizeof(Trace);
8110d565efSmrg hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts());
8210d565efSmrg hdr_end = RoundUp(hdr_end, GetPageSizeCached());
8310d565efSmrg if (hdr_end < hdr + sizeof(Trace))
8410d565efSmrg UnmapOrDie((void*)hdr_end, hdr + sizeof(Trace) - hdr_end);
8510d565efSmrg void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext));
8610d565efSmrg return new(mem) ThreadContext(tid);
8710d565efSmrg }
8810d565efSmrg
8910d565efSmrg #if !SANITIZER_GO
9010d565efSmrg static const u32 kThreadQuarantineSize = 16;
9110d565efSmrg #else
9210d565efSmrg static const u32 kThreadQuarantineSize = 64;
9310d565efSmrg #endif
9410d565efSmrg
Context()9510d565efSmrg Context::Context()
9610d565efSmrg : initialized()
9710d565efSmrg , report_mtx(MutexTypeReport, StatMtxReport)
9810d565efSmrg , nreported()
9910d565efSmrg , nmissed_expected()
10010d565efSmrg , thread_registry(new(thread_registry_placeholder) ThreadRegistry(
10110d565efSmrg CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse))
10210d565efSmrg , racy_mtx(MutexTypeRacy, StatMtxRacy)
103*0fc04c29Smrg , racy_stacks()
104*0fc04c29Smrg , racy_addresses()
10510d565efSmrg , fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
106c7a68eb7Smrg , clock_alloc("clock allocator") {
107*0fc04c29Smrg fired_suppressions.reserve(8);
10810d565efSmrg }
10910d565efSmrg
11010d565efSmrg // The objects are allocated in TLS, so one may rely on zero-initialization.
ThreadState(Context * ctx,int tid,int unique_id,u64 epoch,unsigned reuse_count,uptr stk_addr,uptr stk_size,uptr tls_addr,uptr tls_size)11110d565efSmrg ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
11210d565efSmrg unsigned reuse_count,
11310d565efSmrg uptr stk_addr, uptr stk_size,
11410d565efSmrg uptr tls_addr, uptr tls_size)
11510d565efSmrg : fast_state(tid, epoch)
11610d565efSmrg // Do not touch these, rely on zero initialization,
11710d565efSmrg // they may be accessed before the ctor.
11810d565efSmrg // , ignore_reads_and_writes()
11910d565efSmrg // , ignore_interceptors()
12010d565efSmrg , clock(tid, reuse_count)
12110d565efSmrg #if !SANITIZER_GO
122*0fc04c29Smrg , jmp_bufs()
12310d565efSmrg #endif
12410d565efSmrg , tid(tid)
12510d565efSmrg , unique_id(unique_id)
12610d565efSmrg , stk_addr(stk_addr)
12710d565efSmrg , stk_size(stk_size)
12810d565efSmrg , tls_addr(tls_addr)
12910d565efSmrg , tls_size(tls_size)
13010d565efSmrg #if !SANITIZER_GO
13110d565efSmrg , last_sleep_clock(tid)
13210d565efSmrg #endif
13310d565efSmrg {
13410d565efSmrg }
13510d565efSmrg
13610d565efSmrg #if !SANITIZER_GO
MemoryProfiler(Context * ctx,fd_t fd,int i)13710d565efSmrg static void MemoryProfiler(Context *ctx, fd_t fd, int i) {
13810d565efSmrg uptr n_threads;
13910d565efSmrg uptr n_running_threads;
14010d565efSmrg ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads);
141*0fc04c29Smrg InternalMmapVector<char> buf(4096);
14210d565efSmrg WriteMemoryProfile(buf.data(), buf.size(), n_threads, n_running_threads);
14310d565efSmrg WriteToFile(fd, buf.data(), internal_strlen(buf.data()));
14410d565efSmrg }
14510d565efSmrg
BackgroundThread(void * arg)14610d565efSmrg static void BackgroundThread(void *arg) {
14710d565efSmrg // This is a non-initialized non-user thread, nothing to see here.
14810d565efSmrg // We don't use ScopedIgnoreInterceptors, because we want ignores to be
14910d565efSmrg // enabled even when the thread function exits (e.g. during pthread thread
15010d565efSmrg // shutdown code).
15110d565efSmrg cur_thread()->ignore_interceptors++;
15210d565efSmrg const u64 kMs2Ns = 1000 * 1000;
15310d565efSmrg
15410d565efSmrg fd_t mprof_fd = kInvalidFd;
15510d565efSmrg if (flags()->profile_memory && flags()->profile_memory[0]) {
15610d565efSmrg if (internal_strcmp(flags()->profile_memory, "stdout") == 0) {
15710d565efSmrg mprof_fd = 1;
15810d565efSmrg } else if (internal_strcmp(flags()->profile_memory, "stderr") == 0) {
15910d565efSmrg mprof_fd = 2;
16010d565efSmrg } else {
16110d565efSmrg InternalScopedString filename(kMaxPathLength);
16210d565efSmrg filename.append("%s.%d", flags()->profile_memory, (int)internal_getpid());
16310d565efSmrg fd_t fd = OpenFile(filename.data(), WrOnly);
16410d565efSmrg if (fd == kInvalidFd) {
16510d565efSmrg Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
16610d565efSmrg &filename[0]);
16710d565efSmrg } else {
16810d565efSmrg mprof_fd = fd;
16910d565efSmrg }
17010d565efSmrg }
17110d565efSmrg }
17210d565efSmrg
17310d565efSmrg u64 last_flush = NanoTime();
17410d565efSmrg uptr last_rss = 0;
17510d565efSmrg for (int i = 0;
17610d565efSmrg atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0;
17710d565efSmrg i++) {
17810d565efSmrg SleepForMillis(100);
17910d565efSmrg u64 now = NanoTime();
18010d565efSmrg
18110d565efSmrg // Flush memory if requested.
18210d565efSmrg if (flags()->flush_memory_ms > 0) {
18310d565efSmrg if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
18410d565efSmrg VPrintf(1, "ThreadSanitizer: periodic memory flush\n");
18510d565efSmrg FlushShadowMemory();
18610d565efSmrg last_flush = NanoTime();
18710d565efSmrg }
18810d565efSmrg }
18910d565efSmrg // GetRSS can be expensive on huge programs, so don't do it every 100ms.
19010d565efSmrg if (flags()->memory_limit_mb > 0) {
19110d565efSmrg uptr rss = GetRSS();
19210d565efSmrg uptr limit = uptr(flags()->memory_limit_mb) << 20;
19310d565efSmrg VPrintf(1, "ThreadSanitizer: memory flush check"
19410d565efSmrg " RSS=%llu LAST=%llu LIMIT=%llu\n",
19510d565efSmrg (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
19610d565efSmrg if (2 * rss > limit + last_rss) {
19710d565efSmrg VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n");
19810d565efSmrg FlushShadowMemory();
19910d565efSmrg rss = GetRSS();
20010d565efSmrg VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20);
20110d565efSmrg }
20210d565efSmrg last_rss = rss;
20310d565efSmrg }
20410d565efSmrg
20510d565efSmrg // Write memory profile if requested.
20610d565efSmrg if (mprof_fd != kInvalidFd)
20710d565efSmrg MemoryProfiler(ctx, mprof_fd, i);
20810d565efSmrg
20910d565efSmrg // Flush symbolizer cache if requested.
21010d565efSmrg if (flags()->flush_symbolizer_ms > 0) {
21110d565efSmrg u64 last = atomic_load(&ctx->last_symbolize_time_ns,
21210d565efSmrg memory_order_relaxed);
21310d565efSmrg if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) {
21410d565efSmrg Lock l(&ctx->report_mtx);
215*0fc04c29Smrg ScopedErrorReportLock l2;
21610d565efSmrg SymbolizeFlush();
21710d565efSmrg atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed);
21810d565efSmrg }
21910d565efSmrg }
22010d565efSmrg }
22110d565efSmrg }
22210d565efSmrg
StartBackgroundThread()22310d565efSmrg static void StartBackgroundThread() {
22410d565efSmrg ctx->background_thread = internal_start_thread(&BackgroundThread, 0);
22510d565efSmrg }
22610d565efSmrg
22710d565efSmrg #ifndef __mips__
StopBackgroundThread()22810d565efSmrg static void StopBackgroundThread() {
22910d565efSmrg atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed);
23010d565efSmrg internal_join_thread(ctx->background_thread);
23110d565efSmrg ctx->background_thread = 0;
23210d565efSmrg }
23310d565efSmrg #endif
23410d565efSmrg #endif
23510d565efSmrg
DontNeedShadowFor(uptr addr,uptr size)23610d565efSmrg void DontNeedShadowFor(uptr addr, uptr size) {
237c7a68eb7Smrg ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));
23810d565efSmrg }
23910d565efSmrg
MapShadow(uptr addr,uptr size)24010d565efSmrg void MapShadow(uptr addr, uptr size) {
24110d565efSmrg // Global data is not 64K aligned, but there are no adjacent mappings,
24210d565efSmrg // so we can get away with unaligned mapping.
24310d565efSmrg // CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
24410d565efSmrg const uptr kPageSize = GetPageSizeCached();
24510d565efSmrg uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
24610d565efSmrg uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
247*0fc04c29Smrg if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
248*0fc04c29Smrg Die();
24910d565efSmrg
25010d565efSmrg // Meta shadow is 2:1, so tread carefully.
25110d565efSmrg static bool data_mapped = false;
25210d565efSmrg static uptr mapped_meta_end = 0;
25310d565efSmrg uptr meta_begin = (uptr)MemToMeta(addr);
25410d565efSmrg uptr meta_end = (uptr)MemToMeta(addr + size);
25510d565efSmrg meta_begin = RoundDownTo(meta_begin, 64 << 10);
25610d565efSmrg meta_end = RoundUpTo(meta_end, 64 << 10);
25710d565efSmrg if (!data_mapped) {
25810d565efSmrg // First call maps data+bss.
25910d565efSmrg data_mapped = true;
260*0fc04c29Smrg if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow"))
261*0fc04c29Smrg Die();
26210d565efSmrg } else {
26310d565efSmrg // Mapping continous heap.
26410d565efSmrg // Windows wants 64K alignment.
26510d565efSmrg meta_begin = RoundDownTo(meta_begin, 64 << 10);
26610d565efSmrg meta_end = RoundUpTo(meta_end, 64 << 10);
26710d565efSmrg if (meta_end <= mapped_meta_end)
26810d565efSmrg return;
26910d565efSmrg if (meta_begin < mapped_meta_end)
27010d565efSmrg meta_begin = mapped_meta_end;
271*0fc04c29Smrg if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow"))
272*0fc04c29Smrg Die();
27310d565efSmrg mapped_meta_end = meta_end;
27410d565efSmrg }
27510d565efSmrg VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n",
27610d565efSmrg addr, addr+size, meta_begin, meta_end);
27710d565efSmrg }
27810d565efSmrg
MapThreadTrace(uptr addr,uptr size,const char * name)27910d565efSmrg void MapThreadTrace(uptr addr, uptr size, const char *name) {
28010d565efSmrg DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size);
28110d565efSmrg CHECK_GE(addr, TraceMemBeg());
28210d565efSmrg CHECK_LE(addr + size, TraceMemEnd());
28310d565efSmrg CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
284*0fc04c29Smrg if (!MmapFixedNoReserve(addr, size, name)) {
285*0fc04c29Smrg Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n",
286*0fc04c29Smrg addr, size);
28710d565efSmrg Die();
28810d565efSmrg }
28910d565efSmrg }
29010d565efSmrg
CheckShadowMapping()29110d565efSmrg static void CheckShadowMapping() {
29210d565efSmrg uptr beg, end;
29310d565efSmrg for (int i = 0; GetUserRegion(i, &beg, &end); i++) {
29410d565efSmrg // Skip cases for empty regions (heap definition for architectures that
29510d565efSmrg // do not use 64-bit allocator).
29610d565efSmrg if (beg == end)
29710d565efSmrg continue;
29810d565efSmrg VPrintf(3, "checking shadow region %p-%p\n", beg, end);
29910d565efSmrg uptr prev = 0;
30010d565efSmrg for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) {
30110d565efSmrg for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) {
30210d565efSmrg const uptr p = RoundDown(p0 + x, kShadowCell);
30310d565efSmrg if (p < beg || p >= end)
30410d565efSmrg continue;
30510d565efSmrg const uptr s = MemToShadow(p);
30610d565efSmrg const uptr m = (uptr)MemToMeta(p);
30710d565efSmrg VPrintf(3, " checking pointer %p: shadow=%p meta=%p\n", p, s, m);
30810d565efSmrg CHECK(IsAppMem(p));
30910d565efSmrg CHECK(IsShadowMem(s));
31010d565efSmrg CHECK_EQ(p, ShadowToMem(s));
31110d565efSmrg CHECK(IsMetaMem(m));
31210d565efSmrg if (prev) {
31310d565efSmrg // Ensure that shadow and meta mappings are linear within a single
31410d565efSmrg // user range. Lots of code that processes memory ranges assumes it.
31510d565efSmrg const uptr prev_s = MemToShadow(prev);
31610d565efSmrg const uptr prev_m = (uptr)MemToMeta(prev);
31710d565efSmrg CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier);
31810d565efSmrg CHECK_EQ((m - prev_m) / kMetaShadowSize,
31910d565efSmrg (p - prev) / kMetaShadowCell);
32010d565efSmrg }
32110d565efSmrg prev = p;
32210d565efSmrg }
32310d565efSmrg }
32410d565efSmrg }
32510d565efSmrg }
32610d565efSmrg
327*0fc04c29Smrg #if !SANITIZER_GO
OnStackUnwind(const SignalContext & sig,const void *,BufferedStackTrace * stack)328*0fc04c29Smrg static void OnStackUnwind(const SignalContext &sig, const void *,
329*0fc04c29Smrg BufferedStackTrace *stack) {
330*0fc04c29Smrg uptr top = 0;
331*0fc04c29Smrg uptr bottom = 0;
332*0fc04c29Smrg bool fast = common_flags()->fast_unwind_on_fatal;
333*0fc04c29Smrg if (fast) GetThreadStackTopAndBottom(false, &top, &bottom);
334*0fc04c29Smrg stack->Unwind(kStackTraceMax, sig.pc, sig.bp, sig.context, top, bottom, fast);
335*0fc04c29Smrg }
336*0fc04c29Smrg
TsanOnDeadlySignal(int signo,void * siginfo,void * context)337*0fc04c29Smrg static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) {
338*0fc04c29Smrg HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
339*0fc04c29Smrg }
340*0fc04c29Smrg #endif
341*0fc04c29Smrg
Initialize(ThreadState * thr)34210d565efSmrg void Initialize(ThreadState *thr) {
34310d565efSmrg // Thread safe because done before all threads exist.
34410d565efSmrg static bool is_initialized = false;
34510d565efSmrg if (is_initialized)
34610d565efSmrg return;
34710d565efSmrg is_initialized = true;
34810d565efSmrg // We are not ready to handle interceptors yet.
34910d565efSmrg ScopedIgnoreInterceptors ignore;
35010d565efSmrg SanitizerToolName = "ThreadSanitizer";
35110d565efSmrg // Install tool-specific callbacks in sanitizer_common.
35210d565efSmrg SetCheckFailedCallback(TsanCheckFailed);
35310d565efSmrg
35410d565efSmrg ctx = new(ctx_placeholder) Context;
35510d565efSmrg const char *options = GetEnv(SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS");
35610d565efSmrg CacheBinaryName();
357*0fc04c29Smrg CheckASLR();
35810d565efSmrg InitializeFlags(&ctx->flags, options);
35910d565efSmrg AvoidCVE_2016_2143();
36010d565efSmrg InitializePlatformEarly();
36110d565efSmrg #if !SANITIZER_GO
36210d565efSmrg // Re-exec ourselves if we need to set additional env or command line args.
36310d565efSmrg MaybeReexec();
36410d565efSmrg
36510d565efSmrg InitializeAllocator();
36610d565efSmrg ReplaceSystemMalloc();
36710d565efSmrg #endif
36810d565efSmrg if (common_flags()->detect_deadlocks)
36910d565efSmrg ctx->dd = DDetector::Create(flags());
37010d565efSmrg Processor *proc = ProcCreate();
37110d565efSmrg ProcWire(proc, thr);
37210d565efSmrg InitializeInterceptors();
37310d565efSmrg CheckShadowMapping();
37410d565efSmrg InitializePlatform();
37510d565efSmrg InitializeMutex();
37610d565efSmrg InitializeDynamicAnnotations();
37710d565efSmrg #if !SANITIZER_GO
37810d565efSmrg InitializeShadowMemory();
37910d565efSmrg InitializeAllocatorLate();
380*0fc04c29Smrg InstallDeadlySignalHandlers(TsanOnDeadlySignal);
38110d565efSmrg #endif
38210d565efSmrg // Setup correct file descriptor for error reports.
38310d565efSmrg __sanitizer_set_report_path(common_flags()->log_path);
38410d565efSmrg InitializeSuppressions();
38510d565efSmrg #if !SANITIZER_GO
38610d565efSmrg InitializeLibIgnore();
38710d565efSmrg Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
38810d565efSmrg #endif
38910d565efSmrg
39010d565efSmrg VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
39110d565efSmrg (int)internal_getpid());
39210d565efSmrg
39310d565efSmrg // Initialize thread 0.
39410d565efSmrg int tid = ThreadCreate(thr, 0, 0, true);
39510d565efSmrg CHECK_EQ(tid, 0);
396c7a68eb7Smrg ThreadStart(thr, tid, GetTid(), /*workerthread*/ false);
39710d565efSmrg #if TSAN_CONTAINS_UBSAN
39810d565efSmrg __ubsan::InitAsPlugin();
39910d565efSmrg #endif
40010d565efSmrg ctx->initialized = true;
40110d565efSmrg
40210d565efSmrg #if !SANITIZER_GO
40310d565efSmrg Symbolizer::LateInitialize();
40410d565efSmrg #endif
40510d565efSmrg
40610d565efSmrg if (flags()->stop_on_start) {
40710d565efSmrg Printf("ThreadSanitizer is suspended at startup (pid %d)."
40810d565efSmrg " Call __tsan_resume().\n",
40910d565efSmrg (int)internal_getpid());
41010d565efSmrg while (__tsan_resumed == 0) {}
41110d565efSmrg }
41210d565efSmrg
41310d565efSmrg OnInitialize();
41410d565efSmrg }
41510d565efSmrg
MaybeSpawnBackgroundThread()416*0fc04c29Smrg void MaybeSpawnBackgroundThread() {
417*0fc04c29Smrg // On MIPS, TSan initialization is run before
418*0fc04c29Smrg // __pthread_initialize_minimal_internal() is finished, so we can not spawn
419*0fc04c29Smrg // new threads.
420*0fc04c29Smrg #if !SANITIZER_GO && !defined(__mips__)
421*0fc04c29Smrg static atomic_uint32_t bg_thread = {};
422*0fc04c29Smrg if (atomic_load(&bg_thread, memory_order_relaxed) == 0 &&
423*0fc04c29Smrg atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) {
424*0fc04c29Smrg StartBackgroundThread();
425*0fc04c29Smrg SetSandboxingCallback(StopBackgroundThread);
426*0fc04c29Smrg }
427*0fc04c29Smrg #endif
428*0fc04c29Smrg }
429*0fc04c29Smrg
430*0fc04c29Smrg
Finalize(ThreadState * thr)43110d565efSmrg int Finalize(ThreadState *thr) {
43210d565efSmrg bool failed = false;
43310d565efSmrg
434c7a68eb7Smrg if (common_flags()->print_module_map == 1) PrintModuleMap();
435c7a68eb7Smrg
43610d565efSmrg if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
43710d565efSmrg SleepForMillis(flags()->atexit_sleep_ms);
43810d565efSmrg
43910d565efSmrg // Wait for pending reports.
44010d565efSmrg ctx->report_mtx.Lock();
441*0fc04c29Smrg { ScopedErrorReportLock l; }
44210d565efSmrg ctx->report_mtx.Unlock();
44310d565efSmrg
44410d565efSmrg #if !SANITIZER_GO
44510d565efSmrg if (Verbosity()) AllocatorPrintStats();
44610d565efSmrg #endif
44710d565efSmrg
44810d565efSmrg ThreadFinalize(thr);
44910d565efSmrg
45010d565efSmrg if (ctx->nreported) {
45110d565efSmrg failed = true;
45210d565efSmrg #if !SANITIZER_GO
45310d565efSmrg Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported);
45410d565efSmrg #else
45510d565efSmrg Printf("Found %d data race(s)\n", ctx->nreported);
45610d565efSmrg #endif
45710d565efSmrg }
45810d565efSmrg
45910d565efSmrg if (ctx->nmissed_expected) {
46010d565efSmrg failed = true;
46110d565efSmrg Printf("ThreadSanitizer: missed %d expected races\n",
46210d565efSmrg ctx->nmissed_expected);
46310d565efSmrg }
46410d565efSmrg
46510d565efSmrg if (common_flags()->print_suppressions)
46610d565efSmrg PrintMatchedSuppressions();
46710d565efSmrg #if !SANITIZER_GO
46810d565efSmrg if (flags()->print_benign)
46910d565efSmrg PrintMatchedBenignRaces();
47010d565efSmrg #endif
47110d565efSmrg
47210d565efSmrg failed = OnFinalize(failed);
47310d565efSmrg
47410d565efSmrg #if TSAN_COLLECT_STATS
47510d565efSmrg StatAggregate(ctx->stat, thr->stat);
47610d565efSmrg StatOutput(ctx->stat);
47710d565efSmrg #endif
47810d565efSmrg
47910d565efSmrg return failed ? common_flags()->exitcode : 0;
48010d565efSmrg }
48110d565efSmrg
48210d565efSmrg #if !SANITIZER_GO
ForkBefore(ThreadState * thr,uptr pc)48310d565efSmrg void ForkBefore(ThreadState *thr, uptr pc) {
48410d565efSmrg ctx->thread_registry->Lock();
48510d565efSmrg ctx->report_mtx.Lock();
48610d565efSmrg }
48710d565efSmrg
ForkParentAfter(ThreadState * thr,uptr pc)48810d565efSmrg void ForkParentAfter(ThreadState *thr, uptr pc) {
48910d565efSmrg ctx->report_mtx.Unlock();
49010d565efSmrg ctx->thread_registry->Unlock();
49110d565efSmrg }
49210d565efSmrg
ForkChildAfter(ThreadState * thr,uptr pc)49310d565efSmrg void ForkChildAfter(ThreadState *thr, uptr pc) {
49410d565efSmrg ctx->report_mtx.Unlock();
49510d565efSmrg ctx->thread_registry->Unlock();
49610d565efSmrg
49710d565efSmrg uptr nthread = 0;
49810d565efSmrg ctx->thread_registry->GetNumberOfThreads(0, 0, &nthread /* alive threads */);
49910d565efSmrg VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"
50010d565efSmrg " parent had %d threads\n", (int)internal_getpid(), (int)nthread);
50110d565efSmrg if (nthread == 1) {
50210d565efSmrg StartBackgroundThread();
50310d565efSmrg } else {
50410d565efSmrg // We've just forked a multi-threaded process. We cannot reasonably function
50510d565efSmrg // after that (some mutexes may be locked before fork). So just enable
50610d565efSmrg // ignores for everything in the hope that we will exec soon.
50710d565efSmrg ctx->after_multithreaded_fork = true;
50810d565efSmrg thr->ignore_interceptors++;
50910d565efSmrg ThreadIgnoreBegin(thr, pc);
51010d565efSmrg ThreadIgnoreSyncBegin(thr, pc);
51110d565efSmrg }
51210d565efSmrg }
51310d565efSmrg #endif
51410d565efSmrg
51510d565efSmrg #if SANITIZER_GO
51610d565efSmrg NOINLINE
GrowShadowStack(ThreadState * thr)51710d565efSmrg void GrowShadowStack(ThreadState *thr) {
51810d565efSmrg const int sz = thr->shadow_stack_end - thr->shadow_stack;
51910d565efSmrg const int newsz = 2 * sz;
52010d565efSmrg uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack,
52110d565efSmrg newsz * sizeof(uptr));
52210d565efSmrg internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
52310d565efSmrg internal_free(thr->shadow_stack);
52410d565efSmrg thr->shadow_stack = newstack;
52510d565efSmrg thr->shadow_stack_pos = newstack + sz;
52610d565efSmrg thr->shadow_stack_end = newstack + newsz;
52710d565efSmrg }
52810d565efSmrg #endif
52910d565efSmrg
CurrentStackId(ThreadState * thr,uptr pc)53010d565efSmrg u32 CurrentStackId(ThreadState *thr, uptr pc) {
53110d565efSmrg if (!thr->is_inited) // May happen during bootstrap.
53210d565efSmrg return 0;
53310d565efSmrg if (pc != 0) {
53410d565efSmrg #if !SANITIZER_GO
53510d565efSmrg DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
53610d565efSmrg #else
53710d565efSmrg if (thr->shadow_stack_pos == thr->shadow_stack_end)
53810d565efSmrg GrowShadowStack(thr);
53910d565efSmrg #endif
54010d565efSmrg thr->shadow_stack_pos[0] = pc;
54110d565efSmrg thr->shadow_stack_pos++;
54210d565efSmrg }
54310d565efSmrg u32 id = StackDepotPut(
54410d565efSmrg StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));
54510d565efSmrg if (pc != 0)
54610d565efSmrg thr->shadow_stack_pos--;
54710d565efSmrg return id;
54810d565efSmrg }
54910d565efSmrg
TraceSwitch(ThreadState * thr)55010d565efSmrg void TraceSwitch(ThreadState *thr) {
551*0fc04c29Smrg #if !SANITIZER_GO
552*0fc04c29Smrg if (ctx->after_multithreaded_fork)
553*0fc04c29Smrg return;
554*0fc04c29Smrg #endif
55510d565efSmrg thr->nomalloc++;
55610d565efSmrg Trace *thr_trace = ThreadTrace(thr->tid);
55710d565efSmrg Lock l(&thr_trace->mtx);
55810d565efSmrg unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts();
55910d565efSmrg TraceHeader *hdr = &thr_trace->headers[trace];
56010d565efSmrg hdr->epoch0 = thr->fast_state.epoch();
56110d565efSmrg ObtainCurrentStack(thr, 0, &hdr->stack0);
56210d565efSmrg hdr->mset0 = thr->mset;
56310d565efSmrg thr->nomalloc--;
56410d565efSmrg }
56510d565efSmrg
ThreadTrace(int tid)56610d565efSmrg Trace *ThreadTrace(int tid) {
56710d565efSmrg return (Trace*)GetThreadTraceHeader(tid);
56810d565efSmrg }
56910d565efSmrg
TraceTopPC(ThreadState * thr)57010d565efSmrg uptr TraceTopPC(ThreadState *thr) {
57110d565efSmrg Event *events = (Event*)GetThreadTrace(thr->tid);
57210d565efSmrg uptr pc = events[thr->fast_state.GetTracePos()];
57310d565efSmrg return pc;
57410d565efSmrg }
57510d565efSmrg
TraceSize()57610d565efSmrg uptr TraceSize() {
57710d565efSmrg return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1));
57810d565efSmrg }
57910d565efSmrg
TraceParts()58010d565efSmrg uptr TraceParts() {
58110d565efSmrg return TraceSize() / kTracePartSize;
58210d565efSmrg }
58310d565efSmrg
58410d565efSmrg #if !SANITIZER_GO
__tsan_trace_switch()58510d565efSmrg extern "C" void __tsan_trace_switch() {
58610d565efSmrg TraceSwitch(cur_thread());
58710d565efSmrg }
58810d565efSmrg
__tsan_report_race()58910d565efSmrg extern "C" void __tsan_report_race() {
59010d565efSmrg ReportRace(cur_thread());
59110d565efSmrg }
59210d565efSmrg #endif
59310d565efSmrg
59410d565efSmrg ALWAYS_INLINE
LoadShadow(u64 * p)59510d565efSmrg Shadow LoadShadow(u64 *p) {
59610d565efSmrg u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed);
59710d565efSmrg return Shadow(raw);
59810d565efSmrg }
59910d565efSmrg
60010d565efSmrg ALWAYS_INLINE
StoreShadow(u64 * sp,u64 s)60110d565efSmrg void StoreShadow(u64 *sp, u64 s) {
60210d565efSmrg atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed);
60310d565efSmrg }
60410d565efSmrg
60510d565efSmrg ALWAYS_INLINE
StoreIfNotYetStored(u64 * sp,u64 * s)60610d565efSmrg void StoreIfNotYetStored(u64 *sp, u64 *s) {
60710d565efSmrg StoreShadow(sp, *s);
60810d565efSmrg *s = 0;
60910d565efSmrg }
61010d565efSmrg
61110d565efSmrg ALWAYS_INLINE
HandleRace(ThreadState * thr,u64 * shadow_mem,Shadow cur,Shadow old)61210d565efSmrg void HandleRace(ThreadState *thr, u64 *shadow_mem,
61310d565efSmrg Shadow cur, Shadow old) {
61410d565efSmrg thr->racy_state[0] = cur.raw();
61510d565efSmrg thr->racy_state[1] = old.raw();
61610d565efSmrg thr->racy_shadow_addr = shadow_mem;
61710d565efSmrg #if !SANITIZER_GO
61810d565efSmrg HACKY_CALL(__tsan_report_race);
61910d565efSmrg #else
62010d565efSmrg ReportRace(thr);
62110d565efSmrg #endif
62210d565efSmrg }
62310d565efSmrg
HappensBefore(Shadow old,ThreadState * thr)62410d565efSmrg static inline bool HappensBefore(Shadow old, ThreadState *thr) {
62510d565efSmrg return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
62610d565efSmrg }
62710d565efSmrg
62810d565efSmrg ALWAYS_INLINE
MemoryAccessImpl1(ThreadState * thr,uptr addr,int kAccessSizeLog,bool kAccessIsWrite,bool kIsAtomic,u64 * shadow_mem,Shadow cur)62910d565efSmrg void MemoryAccessImpl1(ThreadState *thr, uptr addr,
63010d565efSmrg int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
63110d565efSmrg u64 *shadow_mem, Shadow cur) {
63210d565efSmrg StatInc(thr, StatMop);
63310d565efSmrg StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
63410d565efSmrg StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
63510d565efSmrg
63610d565efSmrg // This potentially can live in an MMX/SSE scratch register.
63710d565efSmrg // The required intrinsics are:
63810d565efSmrg // __m128i _mm_move_epi64(__m128i*);
63910d565efSmrg // _mm_storel_epi64(u64*, __m128i);
64010d565efSmrg u64 store_word = cur.raw();
64110d565efSmrg
64210d565efSmrg // scan all the shadow values and dispatch to 4 categories:
64310d565efSmrg // same, replace, candidate and race (see comments below).
64410d565efSmrg // we consider only 3 cases regarding access sizes:
64510d565efSmrg // equal, intersect and not intersect. initially I considered
64610d565efSmrg // larger and smaller as well, it allowed to replace some
64710d565efSmrg // 'candidates' with 'same' or 'replace', but I think
64810d565efSmrg // it's just not worth it (performance- and complexity-wise).
64910d565efSmrg
65010d565efSmrg Shadow old(0);
65110d565efSmrg
65210d565efSmrg // It release mode we manually unroll the loop,
65310d565efSmrg // because empirically gcc generates better code this way.
65410d565efSmrg // However, we can't afford unrolling in debug mode, because the function
65510d565efSmrg // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
65610d565efSmrg // threads, which is not enough for the unrolled loop.
65710d565efSmrg #if SANITIZER_DEBUG
65810d565efSmrg for (int idx = 0; idx < 4; idx++) {
65910d565efSmrg #include "tsan_update_shadow_word_inl.h"
66010d565efSmrg }
66110d565efSmrg #else
66210d565efSmrg int idx = 0;
66310d565efSmrg #include "tsan_update_shadow_word_inl.h"
66410d565efSmrg idx = 1;
66510d565efSmrg #include "tsan_update_shadow_word_inl.h"
66610d565efSmrg idx = 2;
66710d565efSmrg #include "tsan_update_shadow_word_inl.h"
66810d565efSmrg idx = 3;
66910d565efSmrg #include "tsan_update_shadow_word_inl.h"
67010d565efSmrg #endif
67110d565efSmrg
67210d565efSmrg // we did not find any races and had already stored
67310d565efSmrg // the current access info, so we are done
67410d565efSmrg if (LIKELY(store_word == 0))
67510d565efSmrg return;
67610d565efSmrg // choose a random candidate slot and replace it
67710d565efSmrg StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
67810d565efSmrg StatInc(thr, StatShadowReplace);
67910d565efSmrg return;
68010d565efSmrg RACE:
68110d565efSmrg HandleRace(thr, shadow_mem, cur, old);
68210d565efSmrg return;
68310d565efSmrg }
68410d565efSmrg
UnalignedMemoryAccess(ThreadState * thr,uptr pc,uptr addr,int size,bool kAccessIsWrite,bool kIsAtomic)68510d565efSmrg void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
68610d565efSmrg int size, bool kAccessIsWrite, bool kIsAtomic) {
68710d565efSmrg while (size) {
68810d565efSmrg int size1 = 1;
68910d565efSmrg int kAccessSizeLog = kSizeLog1;
69010d565efSmrg if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
69110d565efSmrg size1 = 8;
69210d565efSmrg kAccessSizeLog = kSizeLog8;
69310d565efSmrg } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
69410d565efSmrg size1 = 4;
69510d565efSmrg kAccessSizeLog = kSizeLog4;
69610d565efSmrg } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
69710d565efSmrg size1 = 2;
69810d565efSmrg kAccessSizeLog = kSizeLog2;
69910d565efSmrg }
70010d565efSmrg MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
70110d565efSmrg addr += size1;
70210d565efSmrg size -= size1;
70310d565efSmrg }
70410d565efSmrg }
70510d565efSmrg
70610d565efSmrg ALWAYS_INLINE
ContainsSameAccessSlow(u64 * s,u64 a,u64 sync_epoch,bool is_write)70710d565efSmrg bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
70810d565efSmrg Shadow cur(a);
70910d565efSmrg for (uptr i = 0; i < kShadowCnt; i++) {
71010d565efSmrg Shadow old(LoadShadow(&s[i]));
71110d565efSmrg if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
71210d565efSmrg old.TidWithIgnore() == cur.TidWithIgnore() &&
71310d565efSmrg old.epoch() > sync_epoch &&
71410d565efSmrg old.IsAtomic() == cur.IsAtomic() &&
71510d565efSmrg old.IsRead() <= cur.IsRead())
71610d565efSmrg return true;
71710d565efSmrg }
71810d565efSmrg return false;
71910d565efSmrg }
72010d565efSmrg
72110d565efSmrg #if defined(__SSE3__)
72210d565efSmrg #define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \
72310d565efSmrg _mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \
72410d565efSmrg (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
72510d565efSmrg ALWAYS_INLINE
ContainsSameAccessFast(u64 * s,u64 a,u64 sync_epoch,bool is_write)72610d565efSmrg bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
72710d565efSmrg // This is an optimized version of ContainsSameAccessSlow.
72810d565efSmrg // load current access into access[0:63]
72910d565efSmrg const m128 access = _mm_cvtsi64_si128(a);
73010d565efSmrg // duplicate high part of access in addr0:
73110d565efSmrg // addr0[0:31] = access[32:63]
73210d565efSmrg // addr0[32:63] = access[32:63]
73310d565efSmrg // addr0[64:95] = access[32:63]
73410d565efSmrg // addr0[96:127] = access[32:63]
73510d565efSmrg const m128 addr0 = SHUF(access, access, 1, 1, 1, 1);
73610d565efSmrg // load 4 shadow slots
73710d565efSmrg const m128 shadow0 = _mm_load_si128((__m128i*)s);
73810d565efSmrg const m128 shadow1 = _mm_load_si128((__m128i*)s + 1);
73910d565efSmrg // load high parts of 4 shadow slots into addr_vect:
74010d565efSmrg // addr_vect[0:31] = shadow0[32:63]
74110d565efSmrg // addr_vect[32:63] = shadow0[96:127]
74210d565efSmrg // addr_vect[64:95] = shadow1[32:63]
74310d565efSmrg // addr_vect[96:127] = shadow1[96:127]
74410d565efSmrg m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3);
74510d565efSmrg if (!is_write) {
74610d565efSmrg // set IsRead bit in addr_vect
74710d565efSmrg const m128 rw_mask1 = _mm_cvtsi64_si128(1<<15);
74810d565efSmrg const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
74910d565efSmrg addr_vect = _mm_or_si128(addr_vect, rw_mask);
75010d565efSmrg }
75110d565efSmrg // addr0 == addr_vect?
75210d565efSmrg const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect);
75310d565efSmrg // epoch1[0:63] = sync_epoch
75410d565efSmrg const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch);
75510d565efSmrg // epoch[0:31] = sync_epoch[0:31]
75610d565efSmrg // epoch[32:63] = sync_epoch[0:31]
75710d565efSmrg // epoch[64:95] = sync_epoch[0:31]
75810d565efSmrg // epoch[96:127] = sync_epoch[0:31]
75910d565efSmrg const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0);
76010d565efSmrg // load low parts of shadow cell epochs into epoch_vect:
76110d565efSmrg // epoch_vect[0:31] = shadow0[0:31]
76210d565efSmrg // epoch_vect[32:63] = shadow0[64:95]
76310d565efSmrg // epoch_vect[64:95] = shadow1[0:31]
76410d565efSmrg // epoch_vect[96:127] = shadow1[64:95]
76510d565efSmrg const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
76610d565efSmrg // epoch_vect >= sync_epoch?
76710d565efSmrg const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch);
76810d565efSmrg // addr_res & epoch_res
76910d565efSmrg const m128 res = _mm_and_si128(addr_res, epoch_res);
77010d565efSmrg // mask[0] = res[7]
77110d565efSmrg // mask[1] = res[15]
77210d565efSmrg // ...
77310d565efSmrg // mask[15] = res[127]
77410d565efSmrg const int mask = _mm_movemask_epi8(res);
77510d565efSmrg return mask != 0;
77610d565efSmrg }
77710d565efSmrg #endif
77810d565efSmrg
77910d565efSmrg ALWAYS_INLINE
ContainsSameAccess(u64 * s,u64 a,u64 sync_epoch,bool is_write)78010d565efSmrg bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
78110d565efSmrg #if defined(__SSE3__)
78210d565efSmrg bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
78310d565efSmrg // NOTE: this check can fail if the shadow is concurrently mutated
78410d565efSmrg // by other threads. But it still can be useful if you modify
78510d565efSmrg // ContainsSameAccessFast and want to ensure that it's not completely broken.
78610d565efSmrg // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
78710d565efSmrg return res;
78810d565efSmrg #else
78910d565efSmrg return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
79010d565efSmrg #endif
79110d565efSmrg }
79210d565efSmrg
79310d565efSmrg ALWAYS_INLINE USED
MemoryAccess(ThreadState * thr,uptr pc,uptr addr,int kAccessSizeLog,bool kAccessIsWrite,bool kIsAtomic)79410d565efSmrg void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
79510d565efSmrg int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) {
79610d565efSmrg u64 *shadow_mem = (u64*)MemToShadow(addr);
79710d565efSmrg DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
79810d565efSmrg " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
79910d565efSmrg (int)thr->fast_state.tid(), (void*)pc, (void*)addr,
80010d565efSmrg (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
80110d565efSmrg (uptr)shadow_mem[0], (uptr)shadow_mem[1],
80210d565efSmrg (uptr)shadow_mem[2], (uptr)shadow_mem[3]);
80310d565efSmrg #if SANITIZER_DEBUG
80410d565efSmrg if (!IsAppMem(addr)) {
80510d565efSmrg Printf("Access to non app mem %zx\n", addr);
80610d565efSmrg DCHECK(IsAppMem(addr));
80710d565efSmrg }
80810d565efSmrg if (!IsShadowMem((uptr)shadow_mem)) {
80910d565efSmrg Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
81010d565efSmrg DCHECK(IsShadowMem((uptr)shadow_mem));
81110d565efSmrg }
81210d565efSmrg #endif
81310d565efSmrg
81410d565efSmrg if (!SANITIZER_GO && *shadow_mem == kShadowRodata) {
81510d565efSmrg // Access to .rodata section, no races here.
81610d565efSmrg // Measurements show that it can be 10-20% of all memory accesses.
81710d565efSmrg StatInc(thr, StatMop);
81810d565efSmrg StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
81910d565efSmrg StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
82010d565efSmrg StatInc(thr, StatMopRodata);
82110d565efSmrg return;
82210d565efSmrg }
82310d565efSmrg
82410d565efSmrg FastState fast_state = thr->fast_state;
82510d565efSmrg if (fast_state.GetIgnoreBit()) {
82610d565efSmrg StatInc(thr, StatMop);
82710d565efSmrg StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
82810d565efSmrg StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
82910d565efSmrg StatInc(thr, StatMopIgnored);
83010d565efSmrg return;
83110d565efSmrg }
83210d565efSmrg
83310d565efSmrg Shadow cur(fast_state);
83410d565efSmrg cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
83510d565efSmrg cur.SetWrite(kAccessIsWrite);
83610d565efSmrg cur.SetAtomic(kIsAtomic);
83710d565efSmrg
83810d565efSmrg if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
83910d565efSmrg thr->fast_synch_epoch, kAccessIsWrite))) {
84010d565efSmrg StatInc(thr, StatMop);
84110d565efSmrg StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
84210d565efSmrg StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
84310d565efSmrg StatInc(thr, StatMopSame);
84410d565efSmrg return;
84510d565efSmrg }
84610d565efSmrg
84710d565efSmrg if (kCollectHistory) {
84810d565efSmrg fast_state.IncrementEpoch();
84910d565efSmrg thr->fast_state = fast_state;
85010d565efSmrg TraceAddEvent(thr, fast_state, EventTypeMop, pc);
85110d565efSmrg cur.IncrementEpoch();
85210d565efSmrg }
85310d565efSmrg
85410d565efSmrg MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
85510d565efSmrg shadow_mem, cur);
85610d565efSmrg }
85710d565efSmrg
85810d565efSmrg // Called by MemoryAccessRange in tsan_rtl_thread.cc
85910d565efSmrg ALWAYS_INLINE USED
MemoryAccessImpl(ThreadState * thr,uptr addr,int kAccessSizeLog,bool kAccessIsWrite,bool kIsAtomic,u64 * shadow_mem,Shadow cur)86010d565efSmrg void MemoryAccessImpl(ThreadState *thr, uptr addr,
86110d565efSmrg int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
86210d565efSmrg u64 *shadow_mem, Shadow cur) {
86310d565efSmrg if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
86410d565efSmrg thr->fast_synch_epoch, kAccessIsWrite))) {
86510d565efSmrg StatInc(thr, StatMop);
86610d565efSmrg StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
86710d565efSmrg StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
86810d565efSmrg StatInc(thr, StatMopSame);
86910d565efSmrg return;
87010d565efSmrg }
87110d565efSmrg
87210d565efSmrg MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
87310d565efSmrg shadow_mem, cur);
87410d565efSmrg }
87510d565efSmrg
MemoryRangeSet(ThreadState * thr,uptr pc,uptr addr,uptr size,u64 val)87610d565efSmrg static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
87710d565efSmrg u64 val) {
87810d565efSmrg (void)thr;
87910d565efSmrg (void)pc;
88010d565efSmrg if (size == 0)
88110d565efSmrg return;
88210d565efSmrg // FIXME: fix me.
88310d565efSmrg uptr offset = addr % kShadowCell;
88410d565efSmrg if (offset) {
88510d565efSmrg offset = kShadowCell - offset;
88610d565efSmrg if (size <= offset)
88710d565efSmrg return;
88810d565efSmrg addr += offset;
88910d565efSmrg size -= offset;
89010d565efSmrg }
89110d565efSmrg DCHECK_EQ(addr % 8, 0);
89210d565efSmrg // If a user passes some insane arguments (memset(0)),
89310d565efSmrg // let it just crash as usual.
89410d565efSmrg if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
89510d565efSmrg return;
89610d565efSmrg // Don't want to touch lots of shadow memory.
89710d565efSmrg // If a program maps 10MB stack, there is no need reset the whole range.
89810d565efSmrg size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
899c7a68eb7Smrg // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
900c7a68eb7Smrg if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
90110d565efSmrg u64 *p = (u64*)MemToShadow(addr);
90210d565efSmrg CHECK(IsShadowMem((uptr)p));
90310d565efSmrg CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
90410d565efSmrg // FIXME: may overwrite a part outside the region
90510d565efSmrg for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
90610d565efSmrg p[i++] = val;
90710d565efSmrg for (uptr j = 1; j < kShadowCnt; j++)
90810d565efSmrg p[i++] = 0;
90910d565efSmrg }
91010d565efSmrg } else {
91110d565efSmrg // The region is big, reset only beginning and end.
91210d565efSmrg const uptr kPageSize = GetPageSizeCached();
91310d565efSmrg u64 *begin = (u64*)MemToShadow(addr);
91410d565efSmrg u64 *end = begin + size / kShadowCell * kShadowCnt;
91510d565efSmrg u64 *p = begin;
91610d565efSmrg // Set at least first kPageSize/2 to page boundary.
91710d565efSmrg while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
91810d565efSmrg *p++ = val;
91910d565efSmrg for (uptr j = 1; j < kShadowCnt; j++)
92010d565efSmrg *p++ = 0;
92110d565efSmrg }
92210d565efSmrg // Reset middle part.
92310d565efSmrg u64 *p1 = p;
92410d565efSmrg p = RoundDown(end, kPageSize);
92510d565efSmrg UnmapOrDie((void*)p1, (uptr)p - (uptr)p1);
926*0fc04c29Smrg if (!MmapFixedNoReserve((uptr)p1, (uptr)p - (uptr)p1))
927*0fc04c29Smrg Die();
92810d565efSmrg // Set the ending.
92910d565efSmrg while (p < end) {
93010d565efSmrg *p++ = val;
93110d565efSmrg for (uptr j = 1; j < kShadowCnt; j++)
93210d565efSmrg *p++ = 0;
93310d565efSmrg }
93410d565efSmrg }
93510d565efSmrg }
93610d565efSmrg
MemoryResetRange(ThreadState * thr,uptr pc,uptr addr,uptr size)93710d565efSmrg void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
93810d565efSmrg MemoryRangeSet(thr, pc, addr, size, 0);
93910d565efSmrg }
94010d565efSmrg
MemoryRangeFreed(ThreadState * thr,uptr pc,uptr addr,uptr size)94110d565efSmrg void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
94210d565efSmrg // Processing more than 1k (4k of shadow) is expensive,
94310d565efSmrg // can cause excessive memory consumption (user does not necessary touch
94410d565efSmrg // the whole range) and most likely unnecessary.
94510d565efSmrg if (size > 1024)
94610d565efSmrg size = 1024;
94710d565efSmrg CHECK_EQ(thr->is_freeing, false);
94810d565efSmrg thr->is_freeing = true;
94910d565efSmrg MemoryAccessRange(thr, pc, addr, size, true);
95010d565efSmrg thr->is_freeing = false;
95110d565efSmrg if (kCollectHistory) {
95210d565efSmrg thr->fast_state.IncrementEpoch();
95310d565efSmrg TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
95410d565efSmrg }
95510d565efSmrg Shadow s(thr->fast_state);
95610d565efSmrg s.ClearIgnoreBit();
95710d565efSmrg s.MarkAsFreed();
95810d565efSmrg s.SetWrite(true);
95910d565efSmrg s.SetAddr0AndSizeLog(0, 3);
96010d565efSmrg MemoryRangeSet(thr, pc, addr, size, s.raw());
96110d565efSmrg }
96210d565efSmrg
MemoryRangeImitateWrite(ThreadState * thr,uptr pc,uptr addr,uptr size)96310d565efSmrg void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
96410d565efSmrg if (kCollectHistory) {
96510d565efSmrg thr->fast_state.IncrementEpoch();
96610d565efSmrg TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
96710d565efSmrg }
96810d565efSmrg Shadow s(thr->fast_state);
96910d565efSmrg s.ClearIgnoreBit();
97010d565efSmrg s.SetWrite(true);
97110d565efSmrg s.SetAddr0AndSizeLog(0, 3);
97210d565efSmrg MemoryRangeSet(thr, pc, addr, size, s.raw());
97310d565efSmrg }
97410d565efSmrg
97510d565efSmrg ALWAYS_INLINE USED
FuncEntry(ThreadState * thr,uptr pc)97610d565efSmrg void FuncEntry(ThreadState *thr, uptr pc) {
97710d565efSmrg StatInc(thr, StatFuncEnter);
97810d565efSmrg DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
97910d565efSmrg if (kCollectHistory) {
98010d565efSmrg thr->fast_state.IncrementEpoch();
98110d565efSmrg TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
98210d565efSmrg }
98310d565efSmrg
98410d565efSmrg // Shadow stack maintenance can be replaced with
98510d565efSmrg // stack unwinding during trace switch (which presumably must be faster).
98610d565efSmrg DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
98710d565efSmrg #if !SANITIZER_GO
98810d565efSmrg DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
98910d565efSmrg #else
99010d565efSmrg if (thr->shadow_stack_pos == thr->shadow_stack_end)
99110d565efSmrg GrowShadowStack(thr);
99210d565efSmrg #endif
99310d565efSmrg thr->shadow_stack_pos[0] = pc;
99410d565efSmrg thr->shadow_stack_pos++;
99510d565efSmrg }
99610d565efSmrg
99710d565efSmrg ALWAYS_INLINE USED
FuncExit(ThreadState * thr)99810d565efSmrg void FuncExit(ThreadState *thr) {
99910d565efSmrg StatInc(thr, StatFuncExit);
100010d565efSmrg DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
100110d565efSmrg if (kCollectHistory) {
100210d565efSmrg thr->fast_state.IncrementEpoch();
100310d565efSmrg TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
100410d565efSmrg }
100510d565efSmrg
100610d565efSmrg DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
100710d565efSmrg #if !SANITIZER_GO
100810d565efSmrg DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
100910d565efSmrg #endif
101010d565efSmrg thr->shadow_stack_pos--;
101110d565efSmrg }
101210d565efSmrg
ThreadIgnoreBegin(ThreadState * thr,uptr pc,bool save_stack)1013c7a68eb7Smrg void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack) {
101410d565efSmrg DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
101510d565efSmrg thr->ignore_reads_and_writes++;
101610d565efSmrg CHECK_GT(thr->ignore_reads_and_writes, 0);
101710d565efSmrg thr->fast_state.SetIgnoreBit();
101810d565efSmrg #if !SANITIZER_GO
1019c7a68eb7Smrg if (save_stack && !ctx->after_multithreaded_fork)
102010d565efSmrg thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
102110d565efSmrg #endif
102210d565efSmrg }
102310d565efSmrg
ThreadIgnoreEnd(ThreadState * thr,uptr pc)102410d565efSmrg void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {
102510d565efSmrg DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
1026c7a68eb7Smrg CHECK_GT(thr->ignore_reads_and_writes, 0);
102710d565efSmrg thr->ignore_reads_and_writes--;
102810d565efSmrg if (thr->ignore_reads_and_writes == 0) {
102910d565efSmrg thr->fast_state.ClearIgnoreBit();
103010d565efSmrg #if !SANITIZER_GO
103110d565efSmrg thr->mop_ignore_set.Reset();
103210d565efSmrg #endif
103310d565efSmrg }
103410d565efSmrg }
103510d565efSmrg
1036c7a68eb7Smrg #if !SANITIZER_GO
1037c7a68eb7Smrg extern "C" SANITIZER_INTERFACE_ATTRIBUTE
__tsan_testonly_shadow_stack_current_size()1038c7a68eb7Smrg uptr __tsan_testonly_shadow_stack_current_size() {
1039c7a68eb7Smrg ThreadState *thr = cur_thread();
1040c7a68eb7Smrg return thr->shadow_stack_pos - thr->shadow_stack;
1041c7a68eb7Smrg }
1042c7a68eb7Smrg #endif
1043c7a68eb7Smrg
ThreadIgnoreSyncBegin(ThreadState * thr,uptr pc,bool save_stack)1044c7a68eb7Smrg void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack) {
104510d565efSmrg DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
104610d565efSmrg thr->ignore_sync++;
104710d565efSmrg CHECK_GT(thr->ignore_sync, 0);
104810d565efSmrg #if !SANITIZER_GO
1049c7a68eb7Smrg if (save_stack && !ctx->after_multithreaded_fork)
105010d565efSmrg thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
105110d565efSmrg #endif
105210d565efSmrg }
105310d565efSmrg
ThreadIgnoreSyncEnd(ThreadState * thr,uptr pc)105410d565efSmrg void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) {
105510d565efSmrg DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
1056c7a68eb7Smrg CHECK_GT(thr->ignore_sync, 0);
105710d565efSmrg thr->ignore_sync--;
105810d565efSmrg #if !SANITIZER_GO
105910d565efSmrg if (thr->ignore_sync == 0)
106010d565efSmrg thr->sync_ignore_set.Reset();
106110d565efSmrg #endif
106210d565efSmrg }
106310d565efSmrg
operator ==(const MD5Hash & other) const106410d565efSmrg bool MD5Hash::operator==(const MD5Hash &other) const {
106510d565efSmrg return hash[0] == other.hash[0] && hash[1] == other.hash[1];
106610d565efSmrg }
106710d565efSmrg
106810d565efSmrg #if SANITIZER_DEBUG
build_consistency_debug()106910d565efSmrg void build_consistency_debug() {}
107010d565efSmrg #else
build_consistency_release()107110d565efSmrg void build_consistency_release() {}
107210d565efSmrg #endif
107310d565efSmrg
107410d565efSmrg #if TSAN_COLLECT_STATS
build_consistency_stats()107510d565efSmrg void build_consistency_stats() {}
107610d565efSmrg #else
build_consistency_nostats()107710d565efSmrg void build_consistency_nostats() {}
107810d565efSmrg #endif
107910d565efSmrg
108010d565efSmrg } // namespace __tsan
108110d565efSmrg
108210d565efSmrg #if !SANITIZER_GO
108310d565efSmrg // Must be included in this file to make sure everything is inlined.
108410d565efSmrg #include "tsan_interface_inl.h"
108510d565efSmrg #endif
1086