1 //===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef SCUDO_TSD_EXCLUSIVE_H_
10 #define SCUDO_TSD_EXCLUSIVE_H_
11 
12 #include "tsd.h"
13 
14 namespace scudo {
15 
16 enum class ThreadState : u8 {
17   NotInitialized = 0,
18   Initialized,
19   TornDown,
20 };
21 
22 template <class Allocator> void teardownThread(void *Ptr);
23 
24 template <class Allocator> struct TSDRegistryExT {
25   void initLinkerInitialized(Allocator *Instance) {
26     Instance->initLinkerInitialized();
27     CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0);
28     FallbackTSD.initLinkerInitialized(Instance);
29     Initialized = true;
30   }
31   void init(Allocator *Instance) {
32     memset(this, 0, sizeof(*this));
33     initLinkerInitialized(Instance);
34   }
35 
36   void unmapTestOnly() {}
37 
38   ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) {
39     if (LIKELY(State != ThreadState::NotInitialized))
40       return;
41     initThread(Instance, MinimalInit);
42   }
43 
44   ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) {
45     if (LIKELY(State == ThreadState::Initialized &&
46                !atomic_load(&Disabled, memory_order_acquire))) {
47       *UnlockRequired = false;
48       return &ThreadTSD;
49     }
50     FallbackTSD.lock();
51     *UnlockRequired = true;
52     return &FallbackTSD;
53   }
54 
55   // To disable the exclusive TSD registry, we effectively lock the fallback TSD
56   // and force all threads to attempt to use it instead of their local one.
57   void disable() {
58     Mutex.lock();
59     FallbackTSD.lock();
60     atomic_store(&Disabled, 1U, memory_order_release);
61   }
62 
63   void enable() {
64     atomic_store(&Disabled, 0U, memory_order_release);
65     FallbackTSD.unlock();
66     Mutex.unlock();
67   }
68 
69 private:
70   void initOnceMaybe(Allocator *Instance) {
71     ScopedLock L(Mutex);
72     if (LIKELY(Initialized))
73       return;
74     initLinkerInitialized(Instance); // Sets Initialized.
75   }
76 
77   // Using minimal initialization allows for global initialization while keeping
78   // the thread specific structure untouched. The fallback structure will be
79   // used instead.
80   NOINLINE void initThread(Allocator *Instance, bool MinimalInit) {
81     initOnceMaybe(Instance);
82     if (UNLIKELY(MinimalInit))
83       return;
84     CHECK_EQ(
85         pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0);
86     ThreadTSD.initLinkerInitialized(Instance);
87     State = ThreadState::Initialized;
88     Instance->callPostInitCallback();
89   }
90 
91   pthread_key_t PThreadKey;
92   bool Initialized;
93   atomic_u8 Disabled;
94   TSD<Allocator> FallbackTSD;
95   HybridMutex Mutex;
96   static THREADLOCAL ThreadState State;
97   static THREADLOCAL TSD<Allocator> ThreadTSD;
98 
99   friend void teardownThread<Allocator>(void *Ptr);
100 };
101 
102 template <class Allocator>
103 THREADLOCAL TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD;
104 template <class Allocator>
105 THREADLOCAL ThreadState TSDRegistryExT<Allocator>::State;
106 
107 template <class Allocator> void teardownThread(void *Ptr) {
108   typedef TSDRegistryExT<Allocator> TSDRegistryT;
109   Allocator *Instance = reinterpret_cast<Allocator *>(Ptr);
110   // The glibc POSIX thread-local-storage deallocation routine calls user
111   // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS.
112   // We want to be called last since other destructors might call free and the
113   // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the
114   // quarantine and swallowing the cache.
115   if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) {
116     TSDRegistryT::ThreadTSD.DestructorIterations--;
117     // If pthread_setspecific fails, we will go ahead with the teardown.
118     if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey,
119                                    Ptr) == 0))
120       return;
121   }
122   TSDRegistryT::ThreadTSD.commitBack(Instance);
123   TSDRegistryT::State = ThreadState::TornDown;
124 }
125 
126 } // namespace scudo
127 
128 #endif // SCUDO_TSD_EXCLUSIVE_H_
129