168d75effSDimitry Andric //===-- tsan_rtl_mutex.cpp ------------------------------------------------===//
268d75effSDimitry Andric //
368d75effSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
468d75effSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
568d75effSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
668d75effSDimitry Andric //
768d75effSDimitry Andric //===----------------------------------------------------------------------===//
868d75effSDimitry Andric //
968d75effSDimitry Andric // This file is a part of ThreadSanitizer (TSan), a race detector.
1068d75effSDimitry Andric //
1168d75effSDimitry Andric //===----------------------------------------------------------------------===//
1268d75effSDimitry Andric
1368d75effSDimitry Andric #include <sanitizer_common/sanitizer_deadlock_detector_interface.h>
1468d75effSDimitry Andric #include <sanitizer_common/sanitizer_stackdepot.h>
1568d75effSDimitry Andric
1668d75effSDimitry Andric #include "tsan_rtl.h"
1768d75effSDimitry Andric #include "tsan_flags.h"
1868d75effSDimitry Andric #include "tsan_sync.h"
1968d75effSDimitry Andric #include "tsan_report.h"
2068d75effSDimitry Andric #include "tsan_symbolize.h"
2168d75effSDimitry Andric #include "tsan_platform.h"
2268d75effSDimitry Andric
2368d75effSDimitry Andric namespace __tsan {
2468d75effSDimitry Andric
2568d75effSDimitry Andric void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r);
260eae32dcSDimitry Andric void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
270eae32dcSDimitry Andric FastState last_lock, StackID creation_stack_id);
2868d75effSDimitry Andric
29e8d8bef9SDimitry Andric struct Callback final : public DDCallback {
3068d75effSDimitry Andric ThreadState *thr;
3168d75effSDimitry Andric uptr pc;
3268d75effSDimitry Andric
Callback__tsan::Callback3368d75effSDimitry Andric Callback(ThreadState *thr, uptr pc)
3468d75effSDimitry Andric : thr(thr)
3568d75effSDimitry Andric , pc(pc) {
3668d75effSDimitry Andric DDCallback::pt = thr->proc()->dd_pt;
3768d75effSDimitry Andric DDCallback::lt = thr->dd_lt;
3868d75effSDimitry Andric }
3968d75effSDimitry Andric
Unwind__tsan::Callback40349cc55cSDimitry Andric StackID Unwind() override { return CurrentStackId(thr, pc); }
UniqueTid__tsan::Callback410eae32dcSDimitry Andric int UniqueTid() override { return thr->tid; }
4268d75effSDimitry Andric };
4368d75effSDimitry Andric
DDMutexInit(ThreadState * thr,uptr pc,SyncVar * s)4468d75effSDimitry Andric void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) {
4568d75effSDimitry Andric Callback cb(thr, pc);
4668d75effSDimitry Andric ctx->dd->MutexInit(&cb, &s->dd);
470eae32dcSDimitry Andric s->dd.ctx = s->addr;
4868d75effSDimitry Andric }
4968d75effSDimitry Andric
ReportMutexMisuse(ThreadState * thr,uptr pc,ReportType typ,uptr addr,StackID creation_stack_id)5068d75effSDimitry Andric static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
510eae32dcSDimitry Andric uptr addr, StackID creation_stack_id) {
5268d75effSDimitry Andric // In Go, these misuses are either impossible, or detected by std lib,
5368d75effSDimitry Andric // or false positives (e.g. unlock in a different thread).
5468d75effSDimitry Andric if (SANITIZER_GO)
5568d75effSDimitry Andric return;
56fe6060f1SDimitry Andric if (!ShouldReport(thr, typ))
57fe6060f1SDimitry Andric return;
58349cc55cSDimitry Andric ThreadRegistryLock l(&ctx->thread_registry);
5968d75effSDimitry Andric ScopedReport rep(typ);
600eae32dcSDimitry Andric rep.AddMutex(addr, creation_stack_id);
6168d75effSDimitry Andric VarSizeStackTrace trace;
6268d75effSDimitry Andric ObtainCurrentStack(thr, pc, &trace);
6368d75effSDimitry Andric rep.AddStack(trace, true);
6468d75effSDimitry Andric rep.AddLocation(addr, 1);
6568d75effSDimitry Andric OutputReport(thr, rep);
6668d75effSDimitry Andric }
6768d75effSDimitry Andric
RecordMutexLock(ThreadState * thr,uptr pc,uptr addr,StackID stack_id,bool write)680eae32dcSDimitry Andric static void RecordMutexLock(ThreadState *thr, uptr pc, uptr addr,
690eae32dcSDimitry Andric StackID stack_id, bool write) {
700eae32dcSDimitry Andric auto typ = write ? EventType::kLock : EventType::kRLock;
710eae32dcSDimitry Andric // Note: it's important to trace before modifying mutex set
720eae32dcSDimitry Andric // because tracing can switch trace part and we write the current
730eae32dcSDimitry Andric // mutex set in the beginning of each part.
740eae32dcSDimitry Andric // If we do it in the opposite order, we will write already reduced
750eae32dcSDimitry Andric // mutex set in the beginning of the part and then trace unlock again.
760eae32dcSDimitry Andric TraceMutexLock(thr, typ, pc, addr, stack_id);
770eae32dcSDimitry Andric thr->mset.AddAddr(addr, stack_id, write);
780eae32dcSDimitry Andric }
790eae32dcSDimitry Andric
RecordMutexUnlock(ThreadState * thr,uptr addr)800eae32dcSDimitry Andric static void RecordMutexUnlock(ThreadState *thr, uptr addr) {
810eae32dcSDimitry Andric // See the comment in RecordMutexLock re order of operations.
820eae32dcSDimitry Andric TraceMutexUnlock(thr, addr);
830eae32dcSDimitry Andric thr->mset.DelAddr(addr);
840eae32dcSDimitry Andric }
850eae32dcSDimitry Andric
MutexCreate(ThreadState * thr,uptr pc,uptr addr,u32 flagz)86349cc55cSDimitry Andric void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
8768d75effSDimitry Andric DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
880eae32dcSDimitry Andric if (!(flagz & MutexFlagLinkerInit) && pc && IsAppMem(addr))
89349cc55cSDimitry Andric MemoryAccess(thr, pc, addr, 1, kAccessWrite);
900eae32dcSDimitry Andric SlotLocker locker(thr);
910eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
9268d75effSDimitry Andric s->SetFlags(flagz & MutexCreationFlagMask);
93349cc55cSDimitry Andric // Save stack in the case the sync object was created before as atomic.
940eae32dcSDimitry Andric if (!SANITIZER_GO && s->creation_stack_id == kInvalidStackID)
9568d75effSDimitry Andric s->creation_stack_id = CurrentStackId(thr, pc);
9668d75effSDimitry Andric }
9768d75effSDimitry Andric
MutexDestroy(ThreadState * thr,uptr pc,uptr addr,u32 flagz)98349cc55cSDimitry Andric void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
9968d75effSDimitry Andric DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
100349cc55cSDimitry Andric bool unlock_locked = false;
1010eae32dcSDimitry Andric StackID creation_stack_id;
1020eae32dcSDimitry Andric FastState last_lock;
103349cc55cSDimitry Andric {
1040eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncIfExists(addr);
1050eae32dcSDimitry Andric if (!s)
10668d75effSDimitry Andric return;
1070eae32dcSDimitry Andric SlotLocker locker(thr);
1080eae32dcSDimitry Andric {
1090eae32dcSDimitry Andric Lock lock(&s->mtx);
1100eae32dcSDimitry Andric creation_stack_id = s->creation_stack_id;
1110eae32dcSDimitry Andric last_lock = s->last_lock;
112349cc55cSDimitry Andric if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit) ||
113349cc55cSDimitry Andric ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
11468d75effSDimitry Andric // Destroy is no-op for linker-initialized mutexes.
11568d75effSDimitry Andric return;
11668d75effSDimitry Andric }
11768d75effSDimitry Andric if (common_flags()->detect_deadlocks) {
11868d75effSDimitry Andric Callback cb(thr, pc);
11968d75effSDimitry Andric ctx->dd->MutexDestroy(&cb, &s->dd);
12068d75effSDimitry Andric ctx->dd->MutexInit(&cb, &s->dd);
12168d75effSDimitry Andric }
122fe6060f1SDimitry Andric if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
123fe6060f1SDimitry Andric !s->IsFlagSet(MutexFlagBroken)) {
12468d75effSDimitry Andric s->SetFlags(MutexFlagBroken);
12568d75effSDimitry Andric unlock_locked = true;
12668d75effSDimitry Andric }
1270eae32dcSDimitry Andric s->Reset();
128349cc55cSDimitry Andric }
12968d75effSDimitry Andric // Imitate a memory write to catch unlock-destroy races.
1300eae32dcSDimitry Andric if (pc && IsAppMem(addr))
131*81ad6265SDimitry Andric MemoryAccess(thr, pc, addr, 1,
132*81ad6265SDimitry Andric kAccessWrite | kAccessFree | kAccessSlotLocked);
1330eae32dcSDimitry Andric }
1340eae32dcSDimitry Andric if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked))
1350eae32dcSDimitry Andric ReportDestroyLocked(thr, pc, addr, last_lock, creation_stack_id);
1360eae32dcSDimitry Andric thr->mset.DelAddr(addr, true);
13768d75effSDimitry Andric // s will be destroyed and freed in MetaMap::FreeBlock.
13868d75effSDimitry Andric }
13968d75effSDimitry Andric
MutexPreLock(ThreadState * thr,uptr pc,uptr addr,u32 flagz)140349cc55cSDimitry Andric void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
14168d75effSDimitry Andric DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
1420eae32dcSDimitry Andric if (flagz & MutexFlagTryLock)
1430eae32dcSDimitry Andric return;
1440eae32dcSDimitry Andric if (!common_flags()->detect_deadlocks)
1450eae32dcSDimitry Andric return;
14668d75effSDimitry Andric Callback cb(thr, pc);
1470eae32dcSDimitry Andric {
1480eae32dcSDimitry Andric SlotLocker locker(thr);
1490eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
1500eae32dcSDimitry Andric ReadLock lock(&s->mtx);
1510eae32dcSDimitry Andric s->UpdateFlags(flagz);
1520eae32dcSDimitry Andric if (s->owner_tid != thr->tid)
15368d75effSDimitry Andric ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
15468d75effSDimitry Andric }
155349cc55cSDimitry Andric ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
156349cc55cSDimitry Andric }
15768d75effSDimitry Andric
MutexPostLock(ThreadState * thr,uptr pc,uptr addr,u32 flagz,int rec)158349cc55cSDimitry Andric void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
15968d75effSDimitry Andric DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",
16068d75effSDimitry Andric thr->tid, addr, flagz, rec);
16168d75effSDimitry Andric if (flagz & MutexFlagRecursiveLock)
16268d75effSDimitry Andric CHECK_GT(rec, 0);
16368d75effSDimitry Andric else
16468d75effSDimitry Andric rec = 1;
1650eae32dcSDimitry Andric if (pc && IsAppMem(addr))
166349cc55cSDimitry Andric MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
1670eae32dcSDimitry Andric bool report_double_lock = false;
168349cc55cSDimitry Andric bool pre_lock = false;
169349cc55cSDimitry Andric bool first = false;
1700eae32dcSDimitry Andric StackID creation_stack_id = kInvalidStackID;
171349cc55cSDimitry Andric {
1720eae32dcSDimitry Andric SlotLocker locker(thr);
1730eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
1740eae32dcSDimitry Andric creation_stack_id = s->creation_stack_id;
1750eae32dcSDimitry Andric RecordMutexLock(thr, pc, addr, creation_stack_id, true);
1760eae32dcSDimitry Andric {
1770eae32dcSDimitry Andric Lock lock(&s->mtx);
1780eae32dcSDimitry Andric first = s->recursion == 0;
17968d75effSDimitry Andric s->UpdateFlags(flagz);
180fe6060f1SDimitry Andric if (s->owner_tid == kInvalidTid) {
18168d75effSDimitry Andric CHECK_EQ(s->recursion, 0);
18268d75effSDimitry Andric s->owner_tid = thr->tid;
1830eae32dcSDimitry Andric s->last_lock = thr->fast_state;
18468d75effSDimitry Andric } else if (s->owner_tid == thr->tid) {
18568d75effSDimitry Andric CHECK_GT(s->recursion, 0);
18668d75effSDimitry Andric } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
18768d75effSDimitry Andric s->SetFlags(MutexFlagBroken);
18868d75effSDimitry Andric report_double_lock = true;
18968d75effSDimitry Andric }
19068d75effSDimitry Andric s->recursion += rec;
19168d75effSDimitry Andric if (first) {
1920eae32dcSDimitry Andric if (!thr->ignore_sync) {
1930eae32dcSDimitry Andric thr->clock.Acquire(s->clock);
1940eae32dcSDimitry Andric thr->clock.Acquire(s->read_clock);
19568d75effSDimitry Andric }
1960eae32dcSDimitry Andric }
19768d75effSDimitry Andric if (first && common_flags()->detect_deadlocks) {
1980eae32dcSDimitry Andric pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
1990eae32dcSDimitry Andric !(flagz & MutexFlagTryLock);
20068d75effSDimitry Andric Callback cb(thr, pc);
20168d75effSDimitry Andric if (pre_lock)
20268d75effSDimitry Andric ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
20368d75effSDimitry Andric ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
20468d75effSDimitry Andric }
2050eae32dcSDimitry Andric }
206349cc55cSDimitry Andric }
20768d75effSDimitry Andric if (report_double_lock)
2080eae32dcSDimitry Andric ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr,
2090eae32dcSDimitry Andric creation_stack_id);
21068d75effSDimitry Andric if (first && pre_lock && common_flags()->detect_deadlocks) {
21168d75effSDimitry Andric Callback cb(thr, pc);
21268d75effSDimitry Andric ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
21368d75effSDimitry Andric }
21468d75effSDimitry Andric }
21568d75effSDimitry Andric
MutexUnlock(ThreadState * thr,uptr pc,uptr addr,u32 flagz)216349cc55cSDimitry Andric int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
21768d75effSDimitry Andric DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
2180eae32dcSDimitry Andric if (pc && IsAppMem(addr))
219349cc55cSDimitry Andric MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
2200eae32dcSDimitry Andric StackID creation_stack_id;
2210eae32dcSDimitry Andric RecordMutexUnlock(thr, addr);
222349cc55cSDimitry Andric bool report_bad_unlock = false;
223349cc55cSDimitry Andric int rec = 0;
224349cc55cSDimitry Andric {
2250eae32dcSDimitry Andric SlotLocker locker(thr);
2260eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
2270eae32dcSDimitry Andric bool released = false;
2280eae32dcSDimitry Andric {
2290eae32dcSDimitry Andric Lock lock(&s->mtx);
2300eae32dcSDimitry Andric creation_stack_id = s->creation_stack_id;
23168d75effSDimitry Andric if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
23268d75effSDimitry Andric if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
23368d75effSDimitry Andric s->SetFlags(MutexFlagBroken);
23468d75effSDimitry Andric report_bad_unlock = true;
23568d75effSDimitry Andric }
23668d75effSDimitry Andric } else {
23768d75effSDimitry Andric rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
23868d75effSDimitry Andric s->recursion -= rec;
23968d75effSDimitry Andric if (s->recursion == 0) {
240fe6060f1SDimitry Andric s->owner_tid = kInvalidTid;
2410eae32dcSDimitry Andric if (!thr->ignore_sync) {
2420eae32dcSDimitry Andric thr->clock.ReleaseStore(&s->clock);
2430eae32dcSDimitry Andric released = true;
24468d75effSDimitry Andric }
24568d75effSDimitry Andric }
2460eae32dcSDimitry Andric }
24768d75effSDimitry Andric if (common_flags()->detect_deadlocks && s->recursion == 0 &&
24868d75effSDimitry Andric !report_bad_unlock) {
24968d75effSDimitry Andric Callback cb(thr, pc);
25068d75effSDimitry Andric ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
25168d75effSDimitry Andric }
2520eae32dcSDimitry Andric }
2530eae32dcSDimitry Andric if (released)
2540eae32dcSDimitry Andric IncrementEpoch(thr);
255349cc55cSDimitry Andric }
25668d75effSDimitry Andric if (report_bad_unlock)
2570eae32dcSDimitry Andric ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
2580eae32dcSDimitry Andric creation_stack_id);
25968d75effSDimitry Andric if (common_flags()->detect_deadlocks && !report_bad_unlock) {
26068d75effSDimitry Andric Callback cb(thr, pc);
26168d75effSDimitry Andric ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
26268d75effSDimitry Andric }
26368d75effSDimitry Andric return rec;
26468d75effSDimitry Andric }
26568d75effSDimitry Andric
MutexPreReadLock(ThreadState * thr,uptr pc,uptr addr,u32 flagz)266349cc55cSDimitry Andric void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
26768d75effSDimitry Andric DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
2680eae32dcSDimitry Andric if ((flagz & MutexFlagTryLock) || !common_flags()->detect_deadlocks)
2690eae32dcSDimitry Andric return;
27068d75effSDimitry Andric Callback cb(thr, pc);
2710eae32dcSDimitry Andric {
2720eae32dcSDimitry Andric SlotLocker locker(thr);
2730eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
2740eae32dcSDimitry Andric ReadLock lock(&s->mtx);
2750eae32dcSDimitry Andric s->UpdateFlags(flagz);
27668d75effSDimitry Andric ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
277349cc55cSDimitry Andric }
27868d75effSDimitry Andric ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
27968d75effSDimitry Andric }
28068d75effSDimitry Andric
MutexPostReadLock(ThreadState * thr,uptr pc,uptr addr,u32 flagz)281349cc55cSDimitry Andric void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
28268d75effSDimitry Andric DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
2830eae32dcSDimitry Andric if (pc && IsAppMem(addr))
284349cc55cSDimitry Andric MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
285349cc55cSDimitry Andric bool report_bad_lock = false;
286349cc55cSDimitry Andric bool pre_lock = false;
2870eae32dcSDimitry Andric StackID creation_stack_id = kInvalidStackID;
288349cc55cSDimitry Andric {
2890eae32dcSDimitry Andric SlotLocker locker(thr);
2900eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
2910eae32dcSDimitry Andric creation_stack_id = s->creation_stack_id;
2920eae32dcSDimitry Andric RecordMutexLock(thr, pc, addr, creation_stack_id, false);
2930eae32dcSDimitry Andric {
2940eae32dcSDimitry Andric ReadLock lock(&s->mtx);
29568d75effSDimitry Andric s->UpdateFlags(flagz);
296fe6060f1SDimitry Andric if (s->owner_tid != kInvalidTid) {
29768d75effSDimitry Andric if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
29868d75effSDimitry Andric s->SetFlags(MutexFlagBroken);
29968d75effSDimitry Andric report_bad_lock = true;
30068d75effSDimitry Andric }
30168d75effSDimitry Andric }
3020eae32dcSDimitry Andric if (!thr->ignore_sync)
3030eae32dcSDimitry Andric thr->clock.Acquire(s->clock);
3040eae32dcSDimitry Andric s->last_lock = thr->fast_state;
30568d75effSDimitry Andric if (common_flags()->detect_deadlocks) {
3060eae32dcSDimitry Andric pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
3070eae32dcSDimitry Andric !(flagz & MutexFlagTryLock);
30868d75effSDimitry Andric Callback cb(thr, pc);
30968d75effSDimitry Andric if (pre_lock)
31068d75effSDimitry Andric ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
31168d75effSDimitry Andric ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
31268d75effSDimitry Andric }
3130eae32dcSDimitry Andric }
314349cc55cSDimitry Andric }
31568d75effSDimitry Andric if (report_bad_lock)
3160eae32dcSDimitry Andric ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr,
3170eae32dcSDimitry Andric creation_stack_id);
31868d75effSDimitry Andric if (pre_lock && common_flags()->detect_deadlocks) {
31968d75effSDimitry Andric Callback cb(thr, pc);
32068d75effSDimitry Andric ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
32168d75effSDimitry Andric }
32268d75effSDimitry Andric }
32368d75effSDimitry Andric
MutexReadUnlock(ThreadState * thr,uptr pc,uptr addr)324349cc55cSDimitry Andric void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
32568d75effSDimitry Andric DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
3260eae32dcSDimitry Andric if (pc && IsAppMem(addr))
327349cc55cSDimitry Andric MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
3280eae32dcSDimitry Andric RecordMutexUnlock(thr, addr);
3290eae32dcSDimitry Andric StackID creation_stack_id;
330349cc55cSDimitry Andric bool report_bad_unlock = false;
331349cc55cSDimitry Andric {
3320eae32dcSDimitry Andric SlotLocker locker(thr);
3330eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
3340eae32dcSDimitry Andric bool released = false;
3350eae32dcSDimitry Andric {
3360eae32dcSDimitry Andric Lock lock(&s->mtx);
3370eae32dcSDimitry Andric creation_stack_id = s->creation_stack_id;
338fe6060f1SDimitry Andric if (s->owner_tid != kInvalidTid) {
33968d75effSDimitry Andric if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
34068d75effSDimitry Andric s->SetFlags(MutexFlagBroken);
34168d75effSDimitry Andric report_bad_unlock = true;
34268d75effSDimitry Andric }
34368d75effSDimitry Andric }
3440eae32dcSDimitry Andric if (!thr->ignore_sync) {
3450eae32dcSDimitry Andric thr->clock.Release(&s->read_clock);
3460eae32dcSDimitry Andric released = true;
3470eae32dcSDimitry Andric }
34868d75effSDimitry Andric if (common_flags()->detect_deadlocks && s->recursion == 0) {
34968d75effSDimitry Andric Callback cb(thr, pc);
35068d75effSDimitry Andric ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
35168d75effSDimitry Andric }
352349cc55cSDimitry Andric }
3530eae32dcSDimitry Andric if (released)
3540eae32dcSDimitry Andric IncrementEpoch(thr);
3550eae32dcSDimitry Andric }
35668d75effSDimitry Andric if (report_bad_unlock)
3570eae32dcSDimitry Andric ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr,
3580eae32dcSDimitry Andric creation_stack_id);
35968d75effSDimitry Andric if (common_flags()->detect_deadlocks) {
36068d75effSDimitry Andric Callback cb(thr, pc);
36168d75effSDimitry Andric ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
36268d75effSDimitry Andric }
36368d75effSDimitry Andric }
36468d75effSDimitry Andric
MutexReadOrWriteUnlock(ThreadState * thr,uptr pc,uptr addr)365349cc55cSDimitry Andric void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
36668d75effSDimitry Andric DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
3670eae32dcSDimitry Andric if (pc && IsAppMem(addr))
368349cc55cSDimitry Andric MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
3690eae32dcSDimitry Andric RecordMutexUnlock(thr, addr);
3700eae32dcSDimitry Andric StackID creation_stack_id;
37168d75effSDimitry Andric bool report_bad_unlock = false;
372349cc55cSDimitry Andric bool write = true;
3730eae32dcSDimitry Andric {
3740eae32dcSDimitry Andric SlotLocker locker(thr);
3750eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
3760eae32dcSDimitry Andric bool released = false;
3770eae32dcSDimitry Andric {
3780eae32dcSDimitry Andric Lock lock(&s->mtx);
3790eae32dcSDimitry Andric creation_stack_id = s->creation_stack_id;
380fe6060f1SDimitry Andric if (s->owner_tid == kInvalidTid) {
38168d75effSDimitry Andric // Seems to be read unlock.
38268d75effSDimitry Andric write = false;
3830eae32dcSDimitry Andric if (!thr->ignore_sync) {
3840eae32dcSDimitry Andric thr->clock.Release(&s->read_clock);
3850eae32dcSDimitry Andric released = true;
3860eae32dcSDimitry Andric }
38768d75effSDimitry Andric } else if (s->owner_tid == thr->tid) {
38868d75effSDimitry Andric // Seems to be write unlock.
38968d75effSDimitry Andric CHECK_GT(s->recursion, 0);
39068d75effSDimitry Andric s->recursion--;
39168d75effSDimitry Andric if (s->recursion == 0) {
392fe6060f1SDimitry Andric s->owner_tid = kInvalidTid;
3930eae32dcSDimitry Andric if (!thr->ignore_sync) {
3940eae32dcSDimitry Andric thr->clock.ReleaseStore(&s->clock);
3950eae32dcSDimitry Andric released = true;
3960eae32dcSDimitry Andric }
39768d75effSDimitry Andric }
39868d75effSDimitry Andric } else if (!s->IsFlagSet(MutexFlagBroken)) {
39968d75effSDimitry Andric s->SetFlags(MutexFlagBroken);
40068d75effSDimitry Andric report_bad_unlock = true;
40168d75effSDimitry Andric }
40268d75effSDimitry Andric if (common_flags()->detect_deadlocks && s->recursion == 0) {
40368d75effSDimitry Andric Callback cb(thr, pc);
40468d75effSDimitry Andric ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
40568d75effSDimitry Andric }
4060eae32dcSDimitry Andric }
4070eae32dcSDimitry Andric if (released)
4080eae32dcSDimitry Andric IncrementEpoch(thr);
409349cc55cSDimitry Andric }
41068d75effSDimitry Andric if (report_bad_unlock)
4110eae32dcSDimitry Andric ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
4120eae32dcSDimitry Andric creation_stack_id);
41368d75effSDimitry Andric if (common_flags()->detect_deadlocks) {
41468d75effSDimitry Andric Callback cb(thr, pc);
41568d75effSDimitry Andric ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
41668d75effSDimitry Andric }
41768d75effSDimitry Andric }
41868d75effSDimitry Andric
MutexRepair(ThreadState * thr,uptr pc,uptr addr)419349cc55cSDimitry Andric void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
42068d75effSDimitry Andric DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
4210eae32dcSDimitry Andric SlotLocker locker(thr);
4220eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
4230eae32dcSDimitry Andric Lock lock(&s->mtx);
424fe6060f1SDimitry Andric s->owner_tid = kInvalidTid;
42568d75effSDimitry Andric s->recursion = 0;
42668d75effSDimitry Andric }
42768d75effSDimitry Andric
MutexInvalidAccess(ThreadState * thr,uptr pc,uptr addr)428349cc55cSDimitry Andric void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
42968d75effSDimitry Andric DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
4300eae32dcSDimitry Andric StackID creation_stack_id = kInvalidStackID;
4310eae32dcSDimitry Andric {
4320eae32dcSDimitry Andric SlotLocker locker(thr);
4330eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
4340eae32dcSDimitry Andric if (s)
4350eae32dcSDimitry Andric creation_stack_id = s->creation_stack_id;
4360eae32dcSDimitry Andric }
4370eae32dcSDimitry Andric ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr,
4380eae32dcSDimitry Andric creation_stack_id);
43968d75effSDimitry Andric }
44068d75effSDimitry Andric
Acquire(ThreadState * thr,uptr pc,uptr addr)441349cc55cSDimitry Andric void Acquire(ThreadState *thr, uptr pc, uptr addr) {
44268d75effSDimitry Andric DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
44368d75effSDimitry Andric if (thr->ignore_sync)
44468d75effSDimitry Andric return;
4450eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncIfExists(addr);
44668d75effSDimitry Andric if (!s)
44768d75effSDimitry Andric return;
4480eae32dcSDimitry Andric SlotLocker locker(thr);
4490eae32dcSDimitry Andric if (!s->clock)
4500eae32dcSDimitry Andric return;
4510eae32dcSDimitry Andric ReadLock lock(&s->mtx);
4520eae32dcSDimitry Andric thr->clock.Acquire(s->clock);
45368d75effSDimitry Andric }
45468d75effSDimitry Andric
AcquireGlobal(ThreadState * thr)455349cc55cSDimitry Andric void AcquireGlobal(ThreadState *thr) {
45668d75effSDimitry Andric DPrintf("#%d: AcquireGlobal\n", thr->tid);
45768d75effSDimitry Andric if (thr->ignore_sync)
45868d75effSDimitry Andric return;
4590eae32dcSDimitry Andric SlotLocker locker(thr);
4600eae32dcSDimitry Andric for (auto &slot : ctx->slots) thr->clock.Set(slot.sid, slot.epoch());
4615ffd83dbSDimitry Andric }
4625ffd83dbSDimitry Andric
Release(ThreadState * thr,uptr pc,uptr addr)463349cc55cSDimitry Andric void Release(ThreadState *thr, uptr pc, uptr addr) {
46468d75effSDimitry Andric DPrintf("#%d: Release %zx\n", thr->tid, addr);
46568d75effSDimitry Andric if (thr->ignore_sync)
46668d75effSDimitry Andric return;
4670eae32dcSDimitry Andric SlotLocker locker(thr);
4680eae32dcSDimitry Andric {
4690eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
4700eae32dcSDimitry Andric Lock lock(&s->mtx);
4710eae32dcSDimitry Andric thr->clock.Release(&s->clock);
4720eae32dcSDimitry Andric }
4730eae32dcSDimitry Andric IncrementEpoch(thr);
47468d75effSDimitry Andric }
47568d75effSDimitry Andric
ReleaseStore(ThreadState * thr,uptr pc,uptr addr)476349cc55cSDimitry Andric void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
47768d75effSDimitry Andric DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
47868d75effSDimitry Andric if (thr->ignore_sync)
47968d75effSDimitry Andric return;
4800eae32dcSDimitry Andric SlotLocker locker(thr);
4810eae32dcSDimitry Andric {
4820eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
4830eae32dcSDimitry Andric Lock lock(&s->mtx);
4840eae32dcSDimitry Andric thr->clock.ReleaseStore(&s->clock);
4850eae32dcSDimitry Andric }
4860eae32dcSDimitry Andric IncrementEpoch(thr);
4870eae32dcSDimitry Andric }
4880eae32dcSDimitry Andric
ReleaseStoreAcquire(ThreadState * thr,uptr pc,uptr addr)4890eae32dcSDimitry Andric void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
4900eae32dcSDimitry Andric DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
4910eae32dcSDimitry Andric if (thr->ignore_sync)
4920eae32dcSDimitry Andric return;
4930eae32dcSDimitry Andric SlotLocker locker(thr);
4940eae32dcSDimitry Andric {
4950eae32dcSDimitry Andric auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
4960eae32dcSDimitry Andric Lock lock(&s->mtx);
4970eae32dcSDimitry Andric thr->clock.ReleaseStoreAcquire(&s->clock);
4980eae32dcSDimitry Andric }
4990eae32dcSDimitry Andric IncrementEpoch(thr);
5000eae32dcSDimitry Andric }
5010eae32dcSDimitry Andric
IncrementEpoch(ThreadState * thr)5020eae32dcSDimitry Andric void IncrementEpoch(ThreadState *thr) {
5030eae32dcSDimitry Andric DCHECK(!thr->ignore_sync);
5040eae32dcSDimitry Andric DCHECK(thr->slot_locked);
5050eae32dcSDimitry Andric Epoch epoch = EpochInc(thr->fast_state.epoch());
5060eae32dcSDimitry Andric if (!EpochOverflow(epoch)) {
5070eae32dcSDimitry Andric Sid sid = thr->fast_state.sid();
5080eae32dcSDimitry Andric thr->clock.Set(sid, epoch);
5090eae32dcSDimitry Andric thr->fast_state.SetEpoch(epoch);
5100eae32dcSDimitry Andric thr->slot->SetEpoch(epoch);
5110eae32dcSDimitry Andric TraceTime(thr);
5120eae32dcSDimitry Andric }
51368d75effSDimitry Andric }
51468d75effSDimitry Andric
51568d75effSDimitry Andric #if !SANITIZER_GO
AfterSleep(ThreadState * thr,uptr pc)51668d75effSDimitry Andric void AfterSleep(ThreadState *thr, uptr pc) {
517349cc55cSDimitry Andric DPrintf("#%d: AfterSleep\n", thr->tid);
51868d75effSDimitry Andric if (thr->ignore_sync)
51968d75effSDimitry Andric return;
52068d75effSDimitry Andric thr->last_sleep_stack_id = CurrentStackId(thr, pc);
5210eae32dcSDimitry Andric thr->last_sleep_clock.Reset();
5220eae32dcSDimitry Andric SlotLocker locker(thr);
5230eae32dcSDimitry Andric for (auto &slot : ctx->slots)
5240eae32dcSDimitry Andric thr->last_sleep_clock.Set(slot.sid, slot.epoch());
52568d75effSDimitry Andric }
52668d75effSDimitry Andric #endif
52768d75effSDimitry Andric
ReportDeadlock(ThreadState * thr,uptr pc,DDReport * r)52868d75effSDimitry Andric void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
529fe6060f1SDimitry Andric if (r == 0 || !ShouldReport(thr, ReportTypeDeadlock))
53068d75effSDimitry Andric return;
531349cc55cSDimitry Andric ThreadRegistryLock l(&ctx->thread_registry);
53268d75effSDimitry Andric ScopedReport rep(ReportTypeDeadlock);
53368d75effSDimitry Andric for (int i = 0; i < r->n; i++) {
5340eae32dcSDimitry Andric rep.AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
53568d75effSDimitry Andric rep.AddUniqueTid((int)r->loop[i].thr_ctx);
53668d75effSDimitry Andric rep.AddThread((int)r->loop[i].thr_ctx);
53768d75effSDimitry Andric }
53868d75effSDimitry Andric uptr dummy_pc = 0x42;
53968d75effSDimitry Andric for (int i = 0; i < r->n; i++) {
54068d75effSDimitry Andric for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
54168d75effSDimitry Andric u32 stk = r->loop[i].stk[j];
5420eae32dcSDimitry Andric if (stk && stk != kInvalidStackID) {
54368d75effSDimitry Andric rep.AddStack(StackDepotGet(stk), true);
54468d75effSDimitry Andric } else {
54568d75effSDimitry Andric // Sometimes we fail to extract the stack trace (FIXME: investigate),
54668d75effSDimitry Andric // but we should still produce some stack trace in the report.
54768d75effSDimitry Andric rep.AddStack(StackTrace(&dummy_pc, 1), true);
54868d75effSDimitry Andric }
54968d75effSDimitry Andric }
55068d75effSDimitry Andric }
55168d75effSDimitry Andric OutputReport(thr, rep);
55268d75effSDimitry Andric }
55368d75effSDimitry Andric
ReportDestroyLocked(ThreadState * thr,uptr pc,uptr addr,FastState last_lock,StackID creation_stack_id)5540eae32dcSDimitry Andric void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
5550eae32dcSDimitry Andric FastState last_lock, StackID creation_stack_id) {
5560eae32dcSDimitry Andric // We need to lock the slot during RestoreStack because it protects
5570eae32dcSDimitry Andric // the slot journal.
5580eae32dcSDimitry Andric Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
5590eae32dcSDimitry Andric ThreadRegistryLock l0(&ctx->thread_registry);
5600eae32dcSDimitry Andric Lock slots_lock(&ctx->slot_mtx);
5610eae32dcSDimitry Andric ScopedReport rep(ReportTypeMutexDestroyLocked);
5620eae32dcSDimitry Andric rep.AddMutex(addr, creation_stack_id);
5630eae32dcSDimitry Andric VarSizeStackTrace trace;
5640eae32dcSDimitry Andric ObtainCurrentStack(thr, pc, &trace);
5650eae32dcSDimitry Andric rep.AddStack(trace, true);
5660eae32dcSDimitry Andric
5670eae32dcSDimitry Andric Tid tid;
5680eae32dcSDimitry Andric DynamicMutexSet mset;
5690eae32dcSDimitry Andric uptr tag;
5700eae32dcSDimitry Andric if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(), addr,
5710eae32dcSDimitry Andric 0, kAccessWrite, &tid, &trace, mset, &tag))
5720eae32dcSDimitry Andric return;
5730eae32dcSDimitry Andric rep.AddStack(trace, true);
5740eae32dcSDimitry Andric rep.AddLocation(addr, 1);
5750eae32dcSDimitry Andric OutputReport(thr, rep);
5760eae32dcSDimitry Andric }
5770eae32dcSDimitry Andric
57868d75effSDimitry Andric } // namespace __tsan
579