1 #ifndef JEMALLOC_INTERNAL_MUTEX_H 2 #define JEMALLOC_INTERNAL_MUTEX_H 3 4 #include "jemalloc/internal/atomic.h" 5 #include "jemalloc/internal/mutex_prof.h" 6 #include "jemalloc/internal/tsd.h" 7 #include "jemalloc/internal/witness.h" 8 9 typedef enum { 10 /* Can only acquire one mutex of a given witness rank at a time. */ 11 malloc_mutex_rank_exclusive, 12 /* 13 * Can acquire multiple mutexes of the same witness rank, but in 14 * address-ascending order only. 15 */ 16 malloc_mutex_address_ordered 17 } malloc_mutex_lock_order_t; 18 19 typedef struct malloc_mutex_s malloc_mutex_t; 20 struct malloc_mutex_s { 21 union { 22 struct { 23 /* 24 * prof_data is defined first to reduce cacheline 25 * bouncing: the data is not touched by the mutex holder 26 * during unlocking, while might be modified by 27 * contenders. Having it before the mutex itself could 28 * avoid prefetching a modified cacheline (for the 29 * unlocking thread). 30 */ 31 mutex_prof_data_t prof_data; 32 #ifdef _WIN32 33 # if _WIN32_WINNT >= 0x0600 34 SRWLOCK lock; 35 # else 36 CRITICAL_SECTION lock; 37 # endif 38 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) 39 os_unfair_lock lock; 40 #elif (defined(JEMALLOC_MUTEX_INIT_CB)) 41 pthread_mutex_t lock; 42 malloc_mutex_t *postponed_next; 43 #else 44 pthread_mutex_t lock; 45 #endif 46 /* 47 * Hint flag to avoid exclusive cache line contention 48 * during spin waiting 49 */ 50 atomic_b_t locked; 51 }; 52 /* 53 * We only touch witness when configured w/ debug. However we 54 * keep the field in a union when !debug so that we don't have 55 * to pollute the code base with #ifdefs, while avoid paying the 56 * memory cost. 57 */ 58 #if !defined(JEMALLOC_DEBUG) 59 witness_t witness; 60 malloc_mutex_lock_order_t lock_order; 61 #endif 62 }; 63 64 #if defined(JEMALLOC_DEBUG) 65 witness_t witness; 66 malloc_mutex_lock_order_t lock_order; 67 #endif 68 }; 69 70 /* 71 * Based on benchmark results, a fixed spin with this amount of retries works 72 * well for our critical sections. 73 */ 74 #define MALLOC_MUTEX_MAX_SPIN 250 75 76 #ifdef _WIN32 77 # if _WIN32_WINNT >= 0x0600 78 # define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) 79 # define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock) 80 # define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock)) 81 # else 82 # define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock) 83 # define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock) 84 # define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock)) 85 # endif 86 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) 87 # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) 88 # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) 89 # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) 90 #else 91 # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) 92 # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) 93 # define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0) 94 #endif 95 96 #define LOCK_PROF_DATA_INITIALIZER \ 97 {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \ 98 ATOMIC_INIT(0), 0, NULL, 0} 99 100 #ifdef _WIN32 101 # define MALLOC_MUTEX_INITIALIZER 102 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) 103 # if defined(JEMALLOC_DEBUG) 104 # define MALLOC_MUTEX_INITIALIZER \ 105 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ 106 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} 107 # else 108 # define MALLOC_MUTEX_INITIALIZER \ 109 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ 110 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} 111 # endif 112 #elif (defined(JEMALLOC_MUTEX_INIT_CB)) 113 # if (defined(JEMALLOC_DEBUG)) 114 # define MALLOC_MUTEX_INITIALIZER \ 115 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ 116 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} 117 # else 118 # define MALLOC_MUTEX_INITIALIZER \ 119 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ 120 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} 121 # endif 122 123 #else 124 # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT 125 # if defined(JEMALLOC_DEBUG) 126 # define MALLOC_MUTEX_INITIALIZER \ 127 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ 128 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} 129 # else 130 # define MALLOC_MUTEX_INITIALIZER \ 131 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ 132 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} 133 # endif 134 #endif 135 136 #ifdef JEMALLOC_LAZY_LOCK 137 extern bool isthreaded; 138 #endif 139 140 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 141 witness_rank_t rank, malloc_mutex_lock_order_t lock_order); 142 void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); 143 void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); 144 void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); 145 bool malloc_mutex_first_thread(void); 146 bool malloc_mutex_boot(void); 147 void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); 148 149 void malloc_mutex_lock_slow(malloc_mutex_t *mutex); 150 151 static inline void 152 malloc_mutex_lock_final(malloc_mutex_t *mutex) { 153 MALLOC_MUTEX_LOCK(mutex); 154 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); 155 } 156 157 static inline bool 158 malloc_mutex_trylock_final(malloc_mutex_t *mutex) { 159 return MALLOC_MUTEX_TRYLOCK(mutex); 160 } 161 162 static inline void 163 mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) { 164 if (config_stats) { 165 mutex_prof_data_t *data = &mutex->prof_data; 166 data->n_lock_ops++; 167 if (data->prev_owner != tsdn) { 168 data->prev_owner = tsdn; 169 data->n_owner_switches++; 170 } 171 } 172 } 173 174 /* Trylock: return false if the lock is successfully acquired. */ 175 static inline bool 176 malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { 177 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 178 if (isthreaded) { 179 if (malloc_mutex_trylock_final(mutex)) { 180 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); 181 return true; 182 } 183 mutex_owner_stats_update(tsdn, mutex); 184 } 185 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 186 187 return false; 188 } 189 190 /* Aggregate lock prof data. */ 191 static inline void 192 malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { 193 nstime_add(&sum->tot_wait_time, &data->tot_wait_time); 194 if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) { 195 nstime_copy(&sum->max_wait_time, &data->max_wait_time); 196 } 197 198 sum->n_wait_times += data->n_wait_times; 199 sum->n_spin_acquired += data->n_spin_acquired; 200 201 if (sum->max_n_thds < data->max_n_thds) { 202 sum->max_n_thds = data->max_n_thds; 203 } 204 uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds, 205 ATOMIC_RELAXED); 206 uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32( 207 &data->n_waiting_thds, ATOMIC_RELAXED); 208 atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds, 209 ATOMIC_RELAXED); 210 sum->n_owner_switches += data->n_owner_switches; 211 sum->n_lock_ops += data->n_lock_ops; 212 } 213 214 static inline void 215 malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { 216 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 217 if (isthreaded) { 218 if (malloc_mutex_trylock_final(mutex)) { 219 malloc_mutex_lock_slow(mutex); 220 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); 221 } 222 mutex_owner_stats_update(tsdn, mutex); 223 } 224 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 225 } 226 227 static inline void 228 malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { 229 atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); 230 witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 231 if (isthreaded) { 232 MALLOC_MUTEX_UNLOCK(mutex); 233 } 234 } 235 236 static inline void 237 malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { 238 witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 239 } 240 241 static inline void 242 malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { 243 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 244 } 245 246 /* Copy the prof data from mutex for processing. */ 247 static inline void 248 malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, 249 malloc_mutex_t *mutex) { 250 mutex_prof_data_t *source = &mutex->prof_data; 251 /* Can only read holding the mutex. */ 252 malloc_mutex_assert_owner(tsdn, mutex); 253 254 /* 255 * Not *really* allowed (we shouldn't be doing non-atomic loads of 256 * atomic data), but the mutex protection makes this safe, and writing 257 * a member-for-member copy is tedious for this situation. 258 */ 259 *data = *source; 260 /* n_wait_thds is not reported (modified w/o locking). */ 261 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); 262 } 263 264 static inline void 265 malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, 266 malloc_mutex_t *mutex) { 267 mutex_prof_data_t *source = &mutex->prof_data; 268 /* Can only read holding the mutex. */ 269 malloc_mutex_assert_owner(tsdn, mutex); 270 271 nstime_add(&data->tot_wait_time, &source->tot_wait_time); 272 if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) { 273 nstime_copy(&data->max_wait_time, &source->max_wait_time); 274 } 275 data->n_wait_times += source->n_wait_times; 276 data->n_spin_acquired += source->n_spin_acquired; 277 if (data->max_n_thds < source->max_n_thds) { 278 data->max_n_thds = source->max_n_thds; 279 } 280 /* n_wait_thds is not reported. */ 281 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); 282 data->n_owner_switches += source->n_owner_switches; 283 data->n_lock_ops += source->n_lock_ops; 284 } 285 286 #endif /* JEMALLOC_INTERNAL_MUTEX_H */ 287