13cab2bb3Spatrick //===-- tsan_rtl_thread.cpp -----------------------------------------------===//
23cab2bb3Spatrick //
33cab2bb3Spatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
43cab2bb3Spatrick // See https://llvm.org/LICENSE.txt for license information.
53cab2bb3Spatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
63cab2bb3Spatrick //
73cab2bb3Spatrick //===----------------------------------------------------------------------===//
83cab2bb3Spatrick //
93cab2bb3Spatrick // This file is a part of ThreadSanitizer (TSan), a race detector.
103cab2bb3Spatrick //
113cab2bb3Spatrick //===----------------------------------------------------------------------===//
123cab2bb3Spatrick
133cab2bb3Spatrick #include "sanitizer_common/sanitizer_placement_new.h"
143cab2bb3Spatrick #include "tsan_rtl.h"
153cab2bb3Spatrick #include "tsan_mman.h"
163cab2bb3Spatrick #include "tsan_platform.h"
173cab2bb3Spatrick #include "tsan_report.h"
183cab2bb3Spatrick #include "tsan_sync.h"
193cab2bb3Spatrick
203cab2bb3Spatrick namespace __tsan {
213cab2bb3Spatrick
223cab2bb3Spatrick // ThreadContext implementation.
233cab2bb3Spatrick
ThreadContext(Tid tid)24*810390e3Srobert ThreadContext::ThreadContext(Tid tid) : ThreadContextBase(tid), thr(), sync() {}
253cab2bb3Spatrick
263cab2bb3Spatrick #if !SANITIZER_GO
~ThreadContext()273cab2bb3Spatrick ThreadContext::~ThreadContext() {
283cab2bb3Spatrick }
293cab2bb3Spatrick #endif
303cab2bb3Spatrick
OnReset()31*810390e3Srobert void ThreadContext::OnReset() { CHECK(!sync); }
323cab2bb3Spatrick
333cab2bb3Spatrick #if !SANITIZER_GO
343cab2bb3Spatrick struct ThreadLeak {
353cab2bb3Spatrick ThreadContext *tctx;
363cab2bb3Spatrick int count;
373cab2bb3Spatrick };
383cab2bb3Spatrick
CollectThreadLeaks(ThreadContextBase * tctx_base,void * arg)39*810390e3Srobert static void CollectThreadLeaks(ThreadContextBase *tctx_base, void *arg) {
40*810390e3Srobert auto &leaks = *static_cast<Vector<ThreadLeak> *>(arg);
41*810390e3Srobert auto *tctx = static_cast<ThreadContext *>(tctx_base);
423cab2bb3Spatrick if (tctx->detached || tctx->status != ThreadStatusFinished)
433cab2bb3Spatrick return;
443cab2bb3Spatrick for (uptr i = 0; i < leaks.Size(); i++) {
453cab2bb3Spatrick if (leaks[i].tctx->creation_stack_id == tctx->creation_stack_id) {
463cab2bb3Spatrick leaks[i].count++;
473cab2bb3Spatrick return;
483cab2bb3Spatrick }
493cab2bb3Spatrick }
50*810390e3Srobert leaks.PushBack({tctx, 1});
513cab2bb3Spatrick }
523cab2bb3Spatrick #endif
533cab2bb3Spatrick
54*810390e3Srobert // Disabled on Mac because lldb test TestTsanBasic fails:
55*810390e3Srobert // https://reviews.llvm.org/D112603#3163158
56*810390e3Srobert #if !SANITIZER_GO && !SANITIZER_APPLE
ReportIgnoresEnabled(ThreadContext * tctx,IgnoreSet * set)573cab2bb3Spatrick static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) {
58d89ec533Spatrick if (tctx->tid == kMainTid) {
593cab2bb3Spatrick Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
603cab2bb3Spatrick } else {
613cab2bb3Spatrick Printf("ThreadSanitizer: thread T%d %s finished with ignores enabled,"
623cab2bb3Spatrick " created at:\n", tctx->tid, tctx->name);
633cab2bb3Spatrick PrintStack(SymbolizeStackId(tctx->creation_stack_id));
643cab2bb3Spatrick }
653cab2bb3Spatrick Printf(" One of the following ignores was not ended"
663cab2bb3Spatrick " (in order of probability)\n");
673cab2bb3Spatrick for (uptr i = 0; i < set->Size(); i++) {
683cab2bb3Spatrick Printf(" Ignore was enabled at:\n");
693cab2bb3Spatrick PrintStack(SymbolizeStackId(set->At(i)));
703cab2bb3Spatrick }
713cab2bb3Spatrick Die();
723cab2bb3Spatrick }
733cab2bb3Spatrick
ThreadCheckIgnore(ThreadState * thr)743cab2bb3Spatrick static void ThreadCheckIgnore(ThreadState *thr) {
753cab2bb3Spatrick if (ctx->after_multithreaded_fork)
763cab2bb3Spatrick return;
773cab2bb3Spatrick if (thr->ignore_reads_and_writes)
783cab2bb3Spatrick ReportIgnoresEnabled(thr->tctx, &thr->mop_ignore_set);
793cab2bb3Spatrick if (thr->ignore_sync)
803cab2bb3Spatrick ReportIgnoresEnabled(thr->tctx, &thr->sync_ignore_set);
813cab2bb3Spatrick }
823cab2bb3Spatrick #else
ThreadCheckIgnore(ThreadState * thr)833cab2bb3Spatrick static void ThreadCheckIgnore(ThreadState *thr) {}
843cab2bb3Spatrick #endif
853cab2bb3Spatrick
ThreadFinalize(ThreadState * thr)863cab2bb3Spatrick void ThreadFinalize(ThreadState *thr) {
873cab2bb3Spatrick ThreadCheckIgnore(thr);
883cab2bb3Spatrick #if !SANITIZER_GO
89d89ec533Spatrick if (!ShouldReport(thr, ReportTypeThreadLeak))
903cab2bb3Spatrick return;
91*810390e3Srobert ThreadRegistryLock l(&ctx->thread_registry);
923cab2bb3Spatrick Vector<ThreadLeak> leaks;
93*810390e3Srobert ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
94*810390e3Srobert &leaks);
953cab2bb3Spatrick for (uptr i = 0; i < leaks.Size(); i++) {
963cab2bb3Spatrick ScopedReport rep(ReportTypeThreadLeak);
973cab2bb3Spatrick rep.AddThread(leaks[i].tctx, true);
983cab2bb3Spatrick rep.SetCount(leaks[i].count);
993cab2bb3Spatrick OutputReport(thr, rep);
1003cab2bb3Spatrick }
1013cab2bb3Spatrick #endif
1023cab2bb3Spatrick }
1033cab2bb3Spatrick
ThreadCount(ThreadState * thr)1043cab2bb3Spatrick int ThreadCount(ThreadState *thr) {
1053cab2bb3Spatrick uptr result;
106*810390e3Srobert ctx->thread_registry.GetNumberOfThreads(0, 0, &result);
1073cab2bb3Spatrick return (int)result;
1083cab2bb3Spatrick }
1093cab2bb3Spatrick
110*810390e3Srobert struct OnCreatedArgs {
111*810390e3Srobert VectorClock *sync;
112*810390e3Srobert uptr sync_epoch;
113*810390e3Srobert StackID stack;
114*810390e3Srobert };
115*810390e3Srobert
ThreadCreate(ThreadState * thr,uptr pc,uptr uid,bool detached)116*810390e3Srobert Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
117*810390e3Srobert // The main thread and GCD workers don't have a parent thread.
118*810390e3Srobert Tid parent = kInvalidTid;
119*810390e3Srobert OnCreatedArgs arg = {nullptr, 0, kInvalidStackID};
120*810390e3Srobert if (thr) {
121*810390e3Srobert parent = thr->tid;
122*810390e3Srobert arg.stack = CurrentStackId(thr, pc);
123*810390e3Srobert if (!thr->ignore_sync) {
124*810390e3Srobert SlotLocker locker(thr);
125*810390e3Srobert thr->clock.ReleaseStore(&arg.sync);
126*810390e3Srobert arg.sync_epoch = ctx->global_epoch;
127*810390e3Srobert IncrementEpoch(thr);
128*810390e3Srobert }
129*810390e3Srobert }
130*810390e3Srobert Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent, &arg);
131*810390e3Srobert DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent, tid, uid);
1323cab2bb3Spatrick return tid;
1333cab2bb3Spatrick }
1343cab2bb3Spatrick
OnCreated(void * arg)135*810390e3Srobert void ThreadContext::OnCreated(void *arg) {
136*810390e3Srobert OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg);
137*810390e3Srobert sync = args->sync;
138*810390e3Srobert sync_epoch = args->sync_epoch;
139*810390e3Srobert creation_stack_id = args->stack;
140*810390e3Srobert }
141*810390e3Srobert
__tsan_stack_initialization()142*810390e3Srobert extern "C" void __tsan_stack_initialization() {}
143*810390e3Srobert
144*810390e3Srobert struct OnStartedArgs {
145*810390e3Srobert ThreadState *thr;
146*810390e3Srobert uptr stk_addr;
147*810390e3Srobert uptr stk_size;
148*810390e3Srobert uptr tls_addr;
149*810390e3Srobert uptr tls_size;
150*810390e3Srobert };
151*810390e3Srobert
ThreadStart(ThreadState * thr,Tid tid,tid_t os_id,ThreadType thread_type)152*810390e3Srobert void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
1533cab2bb3Spatrick ThreadType thread_type) {
154*810390e3Srobert ctx->thread_registry.StartThread(tid, os_id, thread_type, thr);
155*810390e3Srobert if (!thr->ignore_sync) {
156*810390e3Srobert SlotAttachAndLock(thr);
157*810390e3Srobert if (thr->tctx->sync_epoch == ctx->global_epoch)
158*810390e3Srobert thr->clock.Acquire(thr->tctx->sync);
159*810390e3Srobert SlotUnlock(thr);
160*810390e3Srobert }
161*810390e3Srobert Free(thr->tctx->sync);
162*810390e3Srobert
1633cab2bb3Spatrick uptr stk_addr = 0;
1643cab2bb3Spatrick uptr stk_size = 0;
1653cab2bb3Spatrick uptr tls_addr = 0;
1663cab2bb3Spatrick uptr tls_size = 0;
1673cab2bb3Spatrick #if !SANITIZER_GO
1683cab2bb3Spatrick if (thread_type != ThreadType::Fiber)
169d89ec533Spatrick GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
170d89ec533Spatrick &tls_size);
1713cab2bb3Spatrick #endif
172*810390e3Srobert thr->stk_addr = stk_addr;
173*810390e3Srobert thr->stk_size = stk_size;
174*810390e3Srobert thr->tls_addr = tls_addr;
175*810390e3Srobert thr->tls_size = tls_size;
1763cab2bb3Spatrick
1773cab2bb3Spatrick #if !SANITIZER_GO
1783cab2bb3Spatrick if (ctx->after_multithreaded_fork) {
1793cab2bb3Spatrick thr->ignore_interceptors++;
1803cab2bb3Spatrick ThreadIgnoreBegin(thr, 0);
1813cab2bb3Spatrick ThreadIgnoreSyncBegin(thr, 0);
1823cab2bb3Spatrick }
1833cab2bb3Spatrick #endif
184*810390e3Srobert
185*810390e3Srobert #if !SANITIZER_GO
186*810390e3Srobert // Don't imitate stack/TLS writes for the main thread,
187*810390e3Srobert // because its initialization is synchronized with all
188*810390e3Srobert // subsequent threads anyway.
189*810390e3Srobert if (tid != kMainTid) {
190*810390e3Srobert if (stk_addr && stk_size) {
191*810390e3Srobert const uptr pc = StackTrace::GetNextInstructionPc(
192*810390e3Srobert reinterpret_cast<uptr>(__tsan_stack_initialization));
193*810390e3Srobert MemoryRangeImitateWrite(thr, pc, stk_addr, stk_size);
194*810390e3Srobert }
195*810390e3Srobert
196*810390e3Srobert if (tls_addr && tls_size)
197*810390e3Srobert ImitateTlsWrite(thr, tls_addr, tls_size);
198*810390e3Srobert }
199*810390e3Srobert #endif
200*810390e3Srobert }
201*810390e3Srobert
OnStarted(void * arg)202*810390e3Srobert void ThreadContext::OnStarted(void *arg) {
203*810390e3Srobert thr = static_cast<ThreadState *>(arg);
204*810390e3Srobert DPrintf("#%d: ThreadStart\n", tid);
205*810390e3Srobert new (thr) ThreadState(tid);
206*810390e3Srobert if (common_flags()->detect_deadlocks)
207*810390e3Srobert thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
208*810390e3Srobert thr->tctx = this;
209*810390e3Srobert #if !SANITIZER_GO
210*810390e3Srobert thr->is_inited = true;
211*810390e3Srobert #endif
2123cab2bb3Spatrick }
2133cab2bb3Spatrick
ThreadFinish(ThreadState * thr)2143cab2bb3Spatrick void ThreadFinish(ThreadState *thr) {
215*810390e3Srobert DPrintf("#%d: ThreadFinish\n", thr->tid);
2163cab2bb3Spatrick ThreadCheckIgnore(thr);
2173cab2bb3Spatrick if (thr->stk_addr && thr->stk_size)
2183cab2bb3Spatrick DontNeedShadowFor(thr->stk_addr, thr->stk_size);
2193cab2bb3Spatrick if (thr->tls_addr && thr->tls_size)
2203cab2bb3Spatrick DontNeedShadowFor(thr->tls_addr, thr->tls_size);
2213cab2bb3Spatrick thr->is_dead = true;
222*810390e3Srobert #if !SANITIZER_GO
223*810390e3Srobert thr->is_inited = false;
224*810390e3Srobert thr->ignore_interceptors++;
225*810390e3Srobert PlatformCleanUpThreadState(thr);
226*810390e3Srobert #endif
227*810390e3Srobert if (!thr->ignore_sync) {
228*810390e3Srobert SlotLocker locker(thr);
229*810390e3Srobert ThreadRegistryLock lock(&ctx->thread_registry);
230*810390e3Srobert // Note: detached is protected by the thread registry mutex,
231*810390e3Srobert // the thread may be detaching concurrently in another thread.
232*810390e3Srobert if (!thr->tctx->detached) {
233*810390e3Srobert thr->clock.ReleaseStore(&thr->tctx->sync);
234*810390e3Srobert thr->tctx->sync_epoch = ctx->global_epoch;
235*810390e3Srobert IncrementEpoch(thr);
236*810390e3Srobert }
237*810390e3Srobert }
238*810390e3Srobert #if !SANITIZER_GO
239*810390e3Srobert UnmapOrDie(thr->shadow_stack, kShadowStackSize * sizeof(uptr));
240*810390e3Srobert #else
241*810390e3Srobert Free(thr->shadow_stack);
242*810390e3Srobert #endif
243*810390e3Srobert thr->shadow_stack = nullptr;
244*810390e3Srobert thr->shadow_stack_pos = nullptr;
245*810390e3Srobert thr->shadow_stack_end = nullptr;
246*810390e3Srobert if (common_flags()->detect_deadlocks)
247*810390e3Srobert ctx->dd->DestroyLogicalThread(thr->dd_lt);
248*810390e3Srobert SlotDetach(thr);
249*810390e3Srobert ctx->thread_registry.FinishThread(thr->tid);
250*810390e3Srobert thr->~ThreadState();
251*810390e3Srobert }
252*810390e3Srobert
OnFinished()253*810390e3Srobert void ThreadContext::OnFinished() {
254*810390e3Srobert Lock lock(&ctx->slot_mtx);
255*810390e3Srobert Lock lock1(&trace.mtx);
256*810390e3Srobert // Queue all trace parts into the global recycle queue.
257*810390e3Srobert auto parts = &trace.parts;
258*810390e3Srobert while (trace.local_head) {
259*810390e3Srobert CHECK(parts->Queued(trace.local_head));
260*810390e3Srobert ctx->trace_part_recycle.PushBack(trace.local_head);
261*810390e3Srobert trace.local_head = parts->Next(trace.local_head);
262*810390e3Srobert }
263*810390e3Srobert ctx->trace_part_recycle_finished += parts->Size();
264*810390e3Srobert if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadHi) {
265*810390e3Srobert ctx->trace_part_finished_excess += parts->Size();
266*810390e3Srobert trace.parts_allocated = 0;
267*810390e3Srobert } else if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadLo &&
268*810390e3Srobert parts->Size() > 1) {
269*810390e3Srobert ctx->trace_part_finished_excess += parts->Size() - 1;
270*810390e3Srobert trace.parts_allocated = 1;
271*810390e3Srobert }
272*810390e3Srobert // From now on replay will use trace->final_pos.
273*810390e3Srobert trace.final_pos = (Event *)atomic_load_relaxed(&thr->trace_pos);
274*810390e3Srobert atomic_store_relaxed(&thr->trace_pos, 0);
275*810390e3Srobert thr->tctx = nullptr;
276*810390e3Srobert thr = nullptr;
2773cab2bb3Spatrick }
2783cab2bb3Spatrick
2791f9cb04fSpatrick struct ConsumeThreadContext {
2801f9cb04fSpatrick uptr uid;
2811f9cb04fSpatrick ThreadContextBase *tctx;
2821f9cb04fSpatrick };
2831f9cb04fSpatrick
ThreadConsumeTid(ThreadState * thr,uptr pc,uptr uid)284*810390e3Srobert Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
285*810390e3Srobert return ctx->thread_registry.ConsumeThreadUserId(uid);
2863cab2bb3Spatrick }
2873cab2bb3Spatrick
288*810390e3Srobert struct JoinArg {
289*810390e3Srobert VectorClock *sync;
290*810390e3Srobert uptr sync_epoch;
291*810390e3Srobert };
2923cab2bb3Spatrick
ThreadJoin(ThreadState * thr,uptr pc,Tid tid)293*810390e3Srobert void ThreadJoin(ThreadState *thr, uptr pc, Tid tid) {
2943cab2bb3Spatrick CHECK_GT(tid, 0);
2953cab2bb3Spatrick DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid);
296*810390e3Srobert JoinArg arg = {};
297*810390e3Srobert ctx->thread_registry.JoinThread(tid, &arg);
298*810390e3Srobert if (!thr->ignore_sync) {
299*810390e3Srobert SlotLocker locker(thr);
300*810390e3Srobert if (arg.sync_epoch == ctx->global_epoch)
301*810390e3Srobert thr->clock.Acquire(arg.sync);
302*810390e3Srobert }
303*810390e3Srobert Free(arg.sync);
3043cab2bb3Spatrick }
3053cab2bb3Spatrick
OnJoined(void * ptr)306*810390e3Srobert void ThreadContext::OnJoined(void *ptr) {
307*810390e3Srobert auto arg = static_cast<JoinArg *>(ptr);
308*810390e3Srobert arg->sync = sync;
309*810390e3Srobert arg->sync_epoch = sync_epoch;
310*810390e3Srobert sync = nullptr;
311*810390e3Srobert sync_epoch = 0;
3123cab2bb3Spatrick }
3133cab2bb3Spatrick
OnDead()314*810390e3Srobert void ThreadContext::OnDead() { CHECK_EQ(sync, nullptr); }
315*810390e3Srobert
ThreadDetach(ThreadState * thr,uptr pc,Tid tid)316*810390e3Srobert void ThreadDetach(ThreadState *thr, uptr pc, Tid tid) {
3173cab2bb3Spatrick CHECK_GT(tid, 0);
318*810390e3Srobert ctx->thread_registry.DetachThread(tid, thr);
319*810390e3Srobert }
320*810390e3Srobert
OnDetached(void * arg)321*810390e3Srobert void ThreadContext::OnDetached(void *arg) { Free(sync); }
322*810390e3Srobert
ThreadNotJoined(ThreadState * thr,uptr pc,Tid tid,uptr uid)323*810390e3Srobert void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid) {
324*810390e3Srobert CHECK_GT(tid, 0);
325*810390e3Srobert ctx->thread_registry.SetThreadUserId(tid, uid);
3263cab2bb3Spatrick }
3273cab2bb3Spatrick
ThreadSetName(ThreadState * thr,const char * name)3283cab2bb3Spatrick void ThreadSetName(ThreadState *thr, const char *name) {
329*810390e3Srobert ctx->thread_registry.SetThreadName(thr->tid, name);
3303cab2bb3Spatrick }
3313cab2bb3Spatrick
3323cab2bb3Spatrick #if !SANITIZER_GO
FiberSwitchImpl(ThreadState * from,ThreadState * to)3333cab2bb3Spatrick void FiberSwitchImpl(ThreadState *from, ThreadState *to) {
3343cab2bb3Spatrick Processor *proc = from->proc();
3353cab2bb3Spatrick ProcUnwire(proc, from);
3363cab2bb3Spatrick ProcWire(proc, to);
3373cab2bb3Spatrick set_cur_thread(to);
3383cab2bb3Spatrick }
3393cab2bb3Spatrick
FiberCreate(ThreadState * thr,uptr pc,unsigned flags)3403cab2bb3Spatrick ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) {
341*810390e3Srobert void *mem = Alloc(sizeof(ThreadState));
3423cab2bb3Spatrick ThreadState *fiber = static_cast<ThreadState *>(mem);
3433cab2bb3Spatrick internal_memset(fiber, 0, sizeof(*fiber));
344*810390e3Srobert Tid tid = ThreadCreate(thr, pc, 0, true);
3453cab2bb3Spatrick FiberSwitchImpl(thr, fiber);
3463cab2bb3Spatrick ThreadStart(fiber, tid, 0, ThreadType::Fiber);
3473cab2bb3Spatrick FiberSwitchImpl(fiber, thr);
3483cab2bb3Spatrick return fiber;
3493cab2bb3Spatrick }
3503cab2bb3Spatrick
FiberDestroy(ThreadState * thr,uptr pc,ThreadState * fiber)3513cab2bb3Spatrick void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) {
3523cab2bb3Spatrick FiberSwitchImpl(thr, fiber);
3533cab2bb3Spatrick ThreadFinish(fiber);
3543cab2bb3Spatrick FiberSwitchImpl(fiber, thr);
355*810390e3Srobert Free(fiber);
3563cab2bb3Spatrick }
3573cab2bb3Spatrick
FiberSwitch(ThreadState * thr,uptr pc,ThreadState * fiber,unsigned flags)3583cab2bb3Spatrick void FiberSwitch(ThreadState *thr, uptr pc,
3593cab2bb3Spatrick ThreadState *fiber, unsigned flags) {
3603cab2bb3Spatrick if (!(flags & FiberSwitchFlagNoSync))
3613cab2bb3Spatrick Release(thr, pc, (uptr)fiber);
3623cab2bb3Spatrick FiberSwitchImpl(thr, fiber);
3633cab2bb3Spatrick if (!(flags & FiberSwitchFlagNoSync))
3643cab2bb3Spatrick Acquire(fiber, pc, (uptr)fiber);
3653cab2bb3Spatrick }
3663cab2bb3Spatrick #endif
3673cab2bb3Spatrick
3683cab2bb3Spatrick } // namespace __tsan
369