1 //===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef SCUDO_TSD_EXCLUSIVE_H_
10 #define SCUDO_TSD_EXCLUSIVE_H_
11 
12 #include "tsd.h"
13 
14 #include "string_utils.h"
15 
16 namespace scudo {
17 
18 struct ThreadState {
19   bool DisableMemInit : 1;
20   enum : unsigned {
21     NotInitialized = 0,
22     Initialized,
23     TornDown,
24   } InitState : 2;
25 };
26 
27 template <class Allocator> void teardownThread(void *Ptr);
28 
29 template <class Allocator> struct TSDRegistryExT {
30   void init(Allocator *Instance) REQUIRES(Mutex) {
31     DCHECK(!Initialized);
32     Instance->init();
33     CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0);
34     FallbackTSD.init(Instance);
35     Initialized = true;
36   }
37 
38   void initOnceMaybe(Allocator *Instance) EXCLUDES(Mutex) {
39     ScopedLock L(Mutex);
40     if (LIKELY(Initialized))
41       return;
42     init(Instance); // Sets Initialized.
43   }
44 
45   void unmapTestOnly(Allocator *Instance) EXCLUDES(Mutex) {
46     DCHECK(Instance);
47     if (reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey))) {
48       DCHECK_EQ(reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey)),
49                 Instance);
50       ThreadTSD.commitBack(Instance);
51       ThreadTSD = {};
52     }
53     CHECK_EQ(pthread_key_delete(PThreadKey), 0);
54     PThreadKey = {};
55     FallbackTSD.commitBack(Instance);
56     FallbackTSD = {};
57     State = {};
58     ScopedLock L(Mutex);
59     Initialized = false;
60   }
61 
62   void drainCaches(Allocator *Instance) {
63     // We don't have a way to iterate all thread local `ThreadTSD`s. Simply
64     // drain the `ThreadTSD` of current thread and `FallbackTSD`.
65     Instance->drainCache(&ThreadTSD);
66     FallbackTSD.lock();
67     Instance->drainCache(&FallbackTSD);
68     FallbackTSD.unlock();
69   }
70 
71   ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) {
72     if (LIKELY(State.InitState != ThreadState::NotInitialized))
73       return;
74     initThread(Instance, MinimalInit);
75   }
76 
77   // TODO(chiahungduan): Consider removing the argument `UnlockRequired` by
78   // embedding the logic into TSD or always locking the TSD. It will enable us
79   // to properly mark thread annotation here and adding proper runtime
80   // assertions in the member functions of TSD. For example, assert the lock is
81   // acquired before calling TSD::commitBack().
82   ALWAYS_INLINE TSD<Allocator> *
83   getTSDAndLock(bool *UnlockRequired) NO_THREAD_SAFETY_ANALYSIS {
84     if (LIKELY(State.InitState == ThreadState::Initialized &&
85                !atomic_load(&Disabled, memory_order_acquire))) {
86       *UnlockRequired = false;
87       return &ThreadTSD;
88     }
89     FallbackTSD.lock();
90     *UnlockRequired = true;
91     return &FallbackTSD;
92   }
93 
94   // To disable the exclusive TSD registry, we effectively lock the fallback TSD
95   // and force all threads to attempt to use it instead of their local one.
96   void disable() NO_THREAD_SAFETY_ANALYSIS {
97     Mutex.lock();
98     FallbackTSD.lock();
99     atomic_store(&Disabled, 1U, memory_order_release);
100   }
101 
102   void enable() NO_THREAD_SAFETY_ANALYSIS {
103     atomic_store(&Disabled, 0U, memory_order_release);
104     FallbackTSD.unlock();
105     Mutex.unlock();
106   }
107 
108   bool setOption(Option O, sptr Value) {
109     if (O == Option::ThreadDisableMemInit)
110       State.DisableMemInit = Value;
111     if (O == Option::MaxTSDsCount)
112       return false;
113     return true;
114   }
115 
116   bool getDisableMemInit() { return State.DisableMemInit; }
117 
118   void getStats(ScopedString *Str) {
119     // We don't have a way to iterate all thread local `ThreadTSD`s. Instead of
120     // printing only self `ThreadTSD` which may mislead the usage, we just skip
121     // it.
122     Str->append("Exclusive TSD don't support iterating each TSD\n");
123   }
124 
125 private:
126   // Using minimal initialization allows for global initialization while keeping
127   // the thread specific structure untouched. The fallback structure will be
128   // used instead.
129   NOINLINE void initThread(Allocator *Instance, bool MinimalInit) {
130     initOnceMaybe(Instance);
131     if (UNLIKELY(MinimalInit))
132       return;
133     CHECK_EQ(
134         pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0);
135     ThreadTSD.init(Instance);
136     State.InitState = ThreadState::Initialized;
137     Instance->callPostInitCallback();
138   }
139 
140   pthread_key_t PThreadKey = {};
141   bool Initialized GUARDED_BY(Mutex) = false;
142   atomic_u8 Disabled = {};
143   TSD<Allocator> FallbackTSD;
144   HybridMutex Mutex;
145   static thread_local ThreadState State;
146   static thread_local TSD<Allocator> ThreadTSD;
147 
148   friend void teardownThread<Allocator>(void *Ptr);
149 };
150 
151 template <class Allocator>
152 thread_local TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD;
153 template <class Allocator>
154 thread_local ThreadState TSDRegistryExT<Allocator>::State;
155 
156 template <class Allocator>
157 void teardownThread(void *Ptr) NO_THREAD_SAFETY_ANALYSIS {
158   typedef TSDRegistryExT<Allocator> TSDRegistryT;
159   Allocator *Instance = reinterpret_cast<Allocator *>(Ptr);
160   // The glibc POSIX thread-local-storage deallocation routine calls user
161   // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS.
162   // We want to be called last since other destructors might call free and the
163   // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the
164   // quarantine and swallowing the cache.
165   if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) {
166     TSDRegistryT::ThreadTSD.DestructorIterations--;
167     // If pthread_setspecific fails, we will go ahead with the teardown.
168     if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey,
169                                    Ptr) == 0))
170       return;
171   }
172   TSDRegistryT::ThreadTSD.commitBack(Instance);
173   TSDRegistryT::State.InitState = ThreadState::TornDown;
174 }
175 
176 } // namespace scudo
177 
178 #endif // SCUDO_TSD_EXCLUSIVE_H_
179