1 //===-- sanitizer_deadlock_detector2.cpp ----------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Deadlock detector implementation based on adjacency lists.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "sanitizer_deadlock_detector_interface.h"
14 #include "sanitizer_common.h"
15 #include "sanitizer_allocator_internal.h"
16 #include "sanitizer_placement_new.h"
17 #include "sanitizer_mutex.h"
18 
19 #if SANITIZER_DEADLOCK_DETECTOR_VERSION == 2
20 
21 namespace __sanitizer {
22 
23 const int kMaxNesting = 64;
24 const u32 kNoId = -1;
25 const u32 kEndId = -2;
26 const int kMaxLink = 8;
27 const int kL1Size = 1024;
28 const int kL2Size = 1024;
29 const int kMaxMutex = kL1Size * kL2Size;
30 
31 struct Id {
32   u32 id;
33   u32 seq;
34 
35   explicit Id(u32 id = 0, u32 seq = 0)
36       : id(id)
37       , seq(seq) {
38   }
39 };
40 
41 struct Link {
42   u32 id;
43   u32 seq;
44   u32 tid;
45   u32 stk0;
46   u32 stk1;
47 
48   explicit Link(u32 id = 0, u32 seq = 0, u32 tid = 0, u32 s0 = 0, u32 s1 = 0)
49       : id(id)
50       , seq(seq)
51       , tid(tid)
52       , stk0(s0)
53       , stk1(s1) {
54   }
55 };
56 
57 struct DDPhysicalThread {
58   DDReport rep;
59   bool report_pending;
60   bool visited[kMaxMutex];
61   Link pending[kMaxMutex];
62   Link path[kMaxMutex];
63 };
64 
65 struct ThreadMutex {
66   u32 id;
67   u32 stk;
68 };
69 
70 struct DDLogicalThread {
71   u64         ctx;
72   ThreadMutex locked[kMaxNesting];
73   int         nlocked;
74 };
75 
76 struct MutexState {
77   StaticSpinMutex mtx;
78   u32 seq;
79   int nlink;
80   Link link[kMaxLink];
81 };
82 
83 struct DD final : public DDetector {
84   explicit DD(const DDFlags *flags);
85 
86   DDPhysicalThread* CreatePhysicalThread();
87   void DestroyPhysicalThread(DDPhysicalThread *pt);
88 
89   DDLogicalThread* CreateLogicalThread(u64 ctx);
90   void DestroyLogicalThread(DDLogicalThread *lt);
91 
92   void MutexInit(DDCallback *cb, DDMutex *m);
93   void MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock);
94   void MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock,
95       bool trylock);
96   void MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock);
97   void MutexDestroy(DDCallback *cb, DDMutex *m);
98 
99   DDReport *GetReport(DDCallback *cb);
100 
101   void CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, DDMutex *mtx);
102   void Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath);
103   u32 allocateId(DDCallback *cb);
104   MutexState *getMutex(u32 id);
105   u32 getMutexId(MutexState *m);
106 
107   DDFlags flags;
108 
109   MutexState *mutex[kL1Size];
110 
111   SpinMutex mtx;
112   InternalMmapVector<u32> free_id;
113   int id_gen = 0;
114 };
115 
116 DDetector *DDetector::Create(const DDFlags *flags) {
117   (void)flags;
118   void *mem = MmapOrDie(sizeof(DD), "deadlock detector");
119   return new(mem) DD(flags);
120 }
121 
122 DD::DD(const DDFlags *flags) : flags(*flags) { free_id.reserve(1024); }
123 
124 DDPhysicalThread* DD::CreatePhysicalThread() {
125   DDPhysicalThread *pt = (DDPhysicalThread*)MmapOrDie(sizeof(DDPhysicalThread),
126       "deadlock detector (physical thread)");
127   return pt;
128 }
129 
130 void DD::DestroyPhysicalThread(DDPhysicalThread *pt) {
131   pt->~DDPhysicalThread();
132   UnmapOrDie(pt, sizeof(DDPhysicalThread));
133 }
134 
135 DDLogicalThread* DD::CreateLogicalThread(u64 ctx) {
136   DDLogicalThread *lt = (DDLogicalThread*)InternalAlloc(
137       sizeof(DDLogicalThread));
138   lt->ctx = ctx;
139   lt->nlocked = 0;
140   return lt;
141 }
142 
143 void DD::DestroyLogicalThread(DDLogicalThread *lt) {
144   lt->~DDLogicalThread();
145   InternalFree(lt);
146 }
147 
148 void DD::MutexInit(DDCallback *cb, DDMutex *m) {
149   VPrintf(2, "#%llu: DD::MutexInit(%p)\n", cb->lt->ctx, m);
150   m->id = kNoId;
151   m->recursion = 0;
152   atomic_store(&m->owner, 0, memory_order_relaxed);
153 }
154 
155 MutexState *DD::getMutex(u32 id) { return &mutex[id / kL2Size][id % kL2Size]; }
156 
157 u32 DD::getMutexId(MutexState *m) {
158   for (int i = 0; i < kL1Size; i++) {
159     MutexState *tab = mutex[i];
160     if (tab == 0)
161       break;
162     if (m >= tab && m < tab + kL2Size)
163       return i * kL2Size + (m - tab);
164   }
165   return -1;
166 }
167 
168 u32 DD::allocateId(DDCallback *cb) {
169   u32 id = -1;
170   SpinMutexLock l(&mtx);
171   if (free_id.size() > 0) {
172     id = free_id.back();
173     free_id.pop_back();
174   } else {
175     CHECK_LT(id_gen, kMaxMutex);
176     if ((id_gen % kL2Size) == 0) {
177       mutex[id_gen / kL2Size] = (MutexState *)MmapOrDie(
178           kL2Size * sizeof(MutexState), "deadlock detector (mutex table)");
179     }
180     id = id_gen++;
181   }
182   CHECK_LE(id, kMaxMutex);
183   VPrintf(3, "#%llu: DD::allocateId assign id %d\n", cb->lt->ctx, id);
184   return id;
185 }
186 
187 void DD::MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock) {
188   VPrintf(2, "#%llu: DD::MutexBeforeLock(%p, wlock=%d) nlocked=%d\n",
189       cb->lt->ctx, m, wlock, cb->lt->nlocked);
190   DDPhysicalThread *pt = cb->pt;
191   DDLogicalThread *lt = cb->lt;
192 
193   uptr owner = atomic_load(&m->owner, memory_order_relaxed);
194   if (owner == (uptr)cb->lt) {
195     VPrintf(3, "#%llu: DD::MutexBeforeLock recursive\n",
196         cb->lt->ctx);
197     return;
198   }
199 
200   CHECK_LE(lt->nlocked, kMaxNesting);
201 
202   // FIXME(dvyukov): don't allocate id if lt->nlocked == 0?
203   if (m->id == kNoId)
204     m->id = allocateId(cb);
205 
206   ThreadMutex *tm = &lt->locked[lt->nlocked++];
207   tm->id = m->id;
208   if (flags.second_deadlock_stack)
209     tm->stk = cb->Unwind();
210   if (lt->nlocked == 1) {
211     VPrintf(3, "#%llu: DD::MutexBeforeLock first mutex\n",
212         cb->lt->ctx);
213     return;
214   }
215 
216   bool added = false;
217   MutexState *mtx = getMutex(m->id);
218   for (int i = 0; i < lt->nlocked - 1; i++) {
219     u32 id1 = lt->locked[i].id;
220     u32 stk1 = lt->locked[i].stk;
221     MutexState *mtx1 = getMutex(id1);
222     SpinMutexLock l(&mtx1->mtx);
223     if (mtx1->nlink == kMaxLink) {
224       // FIXME(dvyukov): check stale links
225       continue;
226     }
227     int li = 0;
228     for (; li < mtx1->nlink; li++) {
229       Link *link = &mtx1->link[li];
230       if (link->id == m->id) {
231         if (link->seq != mtx->seq) {
232           link->seq = mtx->seq;
233           link->tid = lt->ctx;
234           link->stk0 = stk1;
235           link->stk1 = cb->Unwind();
236           added = true;
237           VPrintf(3, "#%llu: DD::MutexBeforeLock added %d->%d link\n",
238               cb->lt->ctx, getMutexId(mtx1), m->id);
239         }
240         break;
241       }
242     }
243     if (li == mtx1->nlink) {
244       // FIXME(dvyukov): check stale links
245       Link *link = &mtx1->link[mtx1->nlink++];
246       link->id = m->id;
247       link->seq = mtx->seq;
248       link->tid = lt->ctx;
249       link->stk0 = stk1;
250       link->stk1 = cb->Unwind();
251       added = true;
252       VPrintf(3, "#%llu: DD::MutexBeforeLock added %d->%d link\n",
253           cb->lt->ctx, getMutexId(mtx1), m->id);
254     }
255   }
256 
257   if (!added || mtx->nlink == 0) {
258     VPrintf(3, "#%llu: DD::MutexBeforeLock don't check\n",
259         cb->lt->ctx);
260     return;
261   }
262 
263   CycleCheck(pt, lt, m);
264 }
265 
266 void DD::MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock,
267     bool trylock) {
268   VPrintf(2, "#%llu: DD::MutexAfterLock(%p, wlock=%d, try=%d) nlocked=%d\n",
269       cb->lt->ctx, m, wlock, trylock, cb->lt->nlocked);
270   DDLogicalThread *lt = cb->lt;
271 
272   uptr owner = atomic_load(&m->owner, memory_order_relaxed);
273   if (owner == (uptr)cb->lt) {
274     VPrintf(3, "#%llu: DD::MutexAfterLock recursive\n", cb->lt->ctx);
275     CHECK(wlock);
276     m->recursion++;
277     return;
278   }
279   CHECK_EQ(owner, 0);
280   if (wlock) {
281     VPrintf(3, "#%llu: DD::MutexAfterLock set owner\n", cb->lt->ctx);
282     CHECK_EQ(m->recursion, 0);
283     m->recursion = 1;
284     atomic_store(&m->owner, (uptr)cb->lt, memory_order_relaxed);
285   }
286 
287   if (!trylock)
288     return;
289 
290   CHECK_LE(lt->nlocked, kMaxNesting);
291   if (m->id == kNoId)
292     m->id = allocateId(cb);
293   ThreadMutex *tm = &lt->locked[lt->nlocked++];
294   tm->id = m->id;
295   if (flags.second_deadlock_stack)
296     tm->stk = cb->Unwind();
297 }
298 
299 void DD::MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock) {
300   VPrintf(2, "#%llu: DD::MutexBeforeUnlock(%p, wlock=%d) nlocked=%d\n",
301       cb->lt->ctx, m, wlock, cb->lt->nlocked);
302   DDLogicalThread *lt = cb->lt;
303 
304   uptr owner = atomic_load(&m->owner, memory_order_relaxed);
305   if (owner == (uptr)cb->lt) {
306     VPrintf(3, "#%llu: DD::MutexBeforeUnlock recursive\n", cb->lt->ctx);
307     if (--m->recursion > 0)
308       return;
309     VPrintf(3, "#%llu: DD::MutexBeforeUnlock reset owner\n", cb->lt->ctx);
310     atomic_store(&m->owner, 0, memory_order_relaxed);
311   }
312   CHECK_NE(m->id, kNoId);
313   int last = lt->nlocked - 1;
314   for (int i = last; i >= 0; i--) {
315     if (cb->lt->locked[i].id == m->id) {
316       lt->locked[i] = lt->locked[last];
317       lt->nlocked--;
318       break;
319     }
320   }
321 }
322 
323 void DD::MutexDestroy(DDCallback *cb, DDMutex *m) {
324   VPrintf(2, "#%llu: DD::MutexDestroy(%p)\n",
325       cb->lt->ctx, m);
326   DDLogicalThread *lt = cb->lt;
327 
328   if (m->id == kNoId)
329     return;
330 
331   // Remove the mutex from lt->locked if there.
332   int last = lt->nlocked - 1;
333   for (int i = last; i >= 0; i--) {
334     if (lt->locked[i].id == m->id) {
335       lt->locked[i] = lt->locked[last];
336       lt->nlocked--;
337       break;
338     }
339   }
340 
341   // Clear and invalidate the mutex descriptor.
342   {
343     MutexState *mtx = getMutex(m->id);
344     SpinMutexLock l(&mtx->mtx);
345     mtx->seq++;
346     mtx->nlink = 0;
347   }
348 
349   // Return id to cache.
350   {
351     SpinMutexLock l(&mtx);
352     free_id.push_back(m->id);
353   }
354 }
355 
356 void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt,
357     DDMutex *m) {
358   internal_memset(pt->visited, 0, sizeof(pt->visited));
359   int npath = 0;
360   int npending = 0;
361   {
362     MutexState *mtx = getMutex(m->id);
363     SpinMutexLock l(&mtx->mtx);
364     for (int li = 0; li < mtx->nlink; li++)
365       pt->pending[npending++] = mtx->link[li];
366   }
367   while (npending > 0) {
368     Link link = pt->pending[--npending];
369     if (link.id == kEndId) {
370       npath--;
371       continue;
372     }
373     if (pt->visited[link.id])
374       continue;
375     MutexState *mtx1 = getMutex(link.id);
376     SpinMutexLock l(&mtx1->mtx);
377     if (mtx1->seq != link.seq)
378       continue;
379     pt->visited[link.id] = true;
380     if (mtx1->nlink == 0)
381       continue;
382     pt->path[npath++] = link;
383     pt->pending[npending++] = Link(kEndId);
384     if (link.id == m->id)
385       return Report(pt, lt, npath);  // Bingo!
386     for (int li = 0; li < mtx1->nlink; li++) {
387       Link *link1 = &mtx1->link[li];
388       // MutexState *mtx2 = getMutex(link->id);
389       // FIXME(dvyukov): fast seq check
390       // FIXME(dvyukov): fast nlink != 0 check
391       // FIXME(dvyukov): fast pending check?
392       // FIXME(dvyukov): npending can be larger than kMaxMutex
393       pt->pending[npending++] = *link1;
394     }
395   }
396 }
397 
398 void DD::Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath) {
399   DDReport *rep = &pt->rep;
400   rep->n = npath;
401   for (int i = 0; i < npath; i++) {
402     Link *link = &pt->path[i];
403     Link *link0 = &pt->path[i ? i - 1 : npath - 1];
404     rep->loop[i].thr_ctx = link->tid;
405     rep->loop[i].mtx_ctx0 = link0->id;
406     rep->loop[i].mtx_ctx1 = link->id;
407     rep->loop[i].stk[0] = flags.second_deadlock_stack ? link->stk0 : 0;
408     rep->loop[i].stk[1] = link->stk1;
409   }
410   pt->report_pending = true;
411 }
412 
413 DDReport *DD::GetReport(DDCallback *cb) {
414   if (!cb->pt->report_pending)
415     return 0;
416   cb->pt->report_pending = false;
417   return &cb->pt->rep;
418 }
419 
420 }  // namespace __sanitizer
421 #endif  // #if SANITIZER_DEADLOCK_DETECTOR_VERSION == 2
422