1 #define JEMALLOC_TSD_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/assert.h" 6 #include "jemalloc/internal/mutex.h" 7 #include "jemalloc/internal/rtree.h" 8 9 /******************************************************************************/ 10 /* Data. */ 11 12 static unsigned ncleanups; 13 static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; 14 15 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 16 __thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; 17 __thread bool JEMALLOC_TLS_MODEL tsd_initialized = false; 18 bool tsd_booted = false; 19 #elif (defined(JEMALLOC_TLS)) 20 __thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; 21 pthread_key_t tsd_tsd; 22 bool tsd_booted = false; 23 #elif (defined(_WIN32)) 24 DWORD tsd_tsd; 25 tsd_wrapper_t tsd_boot_wrapper = {false, TSD_INITIALIZER}; 26 bool tsd_booted = false; 27 #else 28 29 /* 30 * This contains a mutex, but it's pretty convenient to allow the mutex code to 31 * have a dependency on tsd. So we define the struct here, and only refer to it 32 * by pointer in the header. 33 */ 34 struct tsd_init_head_s { 35 ql_head(tsd_init_block_t) blocks; 36 malloc_mutex_t lock; 37 }; 38 39 pthread_key_t tsd_tsd; 40 tsd_init_head_t tsd_init_head = { 41 ql_head_initializer(blocks), 42 MALLOC_MUTEX_INITIALIZER 43 }; 44 tsd_wrapper_t tsd_boot_wrapper = { 45 false, 46 TSD_INITIALIZER 47 }; 48 bool tsd_booted = false; 49 #endif 50 51 52 /******************************************************************************/ 53 54 void 55 tsd_slow_update(tsd_t *tsd) { 56 if (tsd_nominal(tsd)) { 57 if (malloc_slow || !tsd_tcache_enabled_get(tsd) || 58 tsd_reentrancy_level_get(tsd) > 0) { 59 tsd->state = tsd_state_nominal_slow; 60 } else { 61 tsd->state = tsd_state_nominal; 62 } 63 } 64 } 65 66 static bool 67 tsd_data_init(tsd_t *tsd) { 68 /* 69 * We initialize the rtree context first (before the tcache), since the 70 * tcache initialization depends on it. 71 */ 72 rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); 73 74 return tsd_tcache_enabled_data_init(tsd); 75 } 76 77 static void 78 assert_tsd_data_cleanup_done(tsd_t *tsd) { 79 assert(!tsd_nominal(tsd)); 80 assert(*tsd_arenap_get_unsafe(tsd) == NULL); 81 assert(*tsd_iarenap_get_unsafe(tsd) == NULL); 82 assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true); 83 assert(*tsd_arenas_tdatap_get_unsafe(tsd) == NULL); 84 assert(*tsd_tcache_enabledp_get_unsafe(tsd) == false); 85 assert(*tsd_prof_tdatap_get_unsafe(tsd) == NULL); 86 } 87 88 static bool 89 tsd_data_init_nocleanup(tsd_t *tsd) { 90 assert(tsd->state == tsd_state_reincarnated || 91 tsd->state == tsd_state_minimal_initialized); 92 /* 93 * During reincarnation, there is no guarantee that the cleanup function 94 * will be called (deallocation may happen after all tsd destructors). 95 * We set up tsd in a way that no cleanup is needed. 96 */ 97 rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); 98 *tsd_arenas_tdata_bypassp_get(tsd) = true; 99 *tsd_tcache_enabledp_get_unsafe(tsd) = false; 100 *tsd_reentrancy_levelp_get(tsd) = 1; 101 assert_tsd_data_cleanup_done(tsd); 102 103 return false; 104 } 105 106 tsd_t * 107 tsd_fetch_slow(tsd_t *tsd, bool minimal) { 108 assert(!tsd_fast(tsd)); 109 110 if (tsd->state == tsd_state_nominal_slow) { 111 /* On slow path but no work needed. */ 112 assert(malloc_slow || !tsd_tcache_enabled_get(tsd) || 113 tsd_reentrancy_level_get(tsd) > 0 || 114 *tsd_arenas_tdata_bypassp_get(tsd)); 115 } else if (tsd->state == tsd_state_uninitialized) { 116 if (!minimal) { 117 tsd->state = tsd_state_nominal; 118 tsd_slow_update(tsd); 119 /* Trigger cleanup handler registration. */ 120 tsd_set(tsd); 121 tsd_data_init(tsd); 122 } else { 123 tsd->state = tsd_state_minimal_initialized; 124 tsd_set(tsd); 125 tsd_data_init_nocleanup(tsd); 126 } 127 } else if (tsd->state == tsd_state_minimal_initialized) { 128 if (!minimal) { 129 /* Switch to fully initialized. */ 130 tsd->state = tsd_state_nominal; 131 assert(*tsd_reentrancy_levelp_get(tsd) >= 1); 132 (*tsd_reentrancy_levelp_get(tsd))--; 133 tsd_slow_update(tsd); 134 tsd_data_init(tsd); 135 } else { 136 assert_tsd_data_cleanup_done(tsd); 137 } 138 } else if (tsd->state == tsd_state_purgatory) { 139 tsd->state = tsd_state_reincarnated; 140 tsd_set(tsd); 141 tsd_data_init_nocleanup(tsd); 142 } else { 143 assert(tsd->state == tsd_state_reincarnated); 144 } 145 146 return tsd; 147 } 148 149 void * 150 malloc_tsd_malloc(size_t size) { 151 return a0malloc(CACHELINE_CEILING(size)); 152 } 153 154 void 155 malloc_tsd_dalloc(void *wrapper) { 156 a0dalloc(wrapper); 157 } 158 159 #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) 160 #ifndef _WIN32 161 JEMALLOC_EXPORT 162 #endif 163 void 164 _malloc_thread_cleanup(void) { 165 bool pending[MALLOC_TSD_CLEANUPS_MAX], again; 166 unsigned i; 167 168 for (i = 0; i < ncleanups; i++) { 169 pending[i] = true; 170 } 171 172 do { 173 again = false; 174 for (i = 0; i < ncleanups; i++) { 175 if (pending[i]) { 176 pending[i] = cleanups[i](); 177 if (pending[i]) { 178 again = true; 179 } 180 } 181 } 182 } while (again); 183 } 184 #endif 185 186 void 187 malloc_tsd_cleanup_register(bool (*f)(void)) { 188 assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX); 189 cleanups[ncleanups] = f; 190 ncleanups++; 191 } 192 193 static void 194 tsd_do_data_cleanup(tsd_t *tsd) { 195 prof_tdata_cleanup(tsd); 196 iarena_cleanup(tsd); 197 arena_cleanup(tsd); 198 arenas_tdata_cleanup(tsd); 199 tcache_cleanup(tsd); 200 witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd)); 201 } 202 203 void 204 tsd_cleanup(void *arg) { 205 tsd_t *tsd = (tsd_t *)arg; 206 207 switch (tsd->state) { 208 case tsd_state_uninitialized: 209 /* Do nothing. */ 210 break; 211 case tsd_state_minimal_initialized: 212 /* This implies the thread only did free() in its life time. */ 213 /* Fall through. */ 214 case tsd_state_reincarnated: 215 /* 216 * Reincarnated means another destructor deallocated memory 217 * after the destructor was called. Cleanup isn't required but 218 * is still called for testing and completeness. 219 */ 220 assert_tsd_data_cleanup_done(tsd); 221 /* Fall through. */ 222 case tsd_state_nominal: 223 case tsd_state_nominal_slow: 224 tsd_do_data_cleanup(tsd); 225 tsd->state = tsd_state_purgatory; 226 tsd_set(tsd); 227 break; 228 case tsd_state_purgatory: 229 /* 230 * The previous time this destructor was called, we set the 231 * state to tsd_state_purgatory so that other destructors 232 * wouldn't cause re-creation of the tsd. This time, do 233 * nothing, and do not request another callback. 234 */ 235 break; 236 default: 237 not_reached(); 238 } 239 #ifdef JEMALLOC_JET 240 test_callback_t test_callback = *tsd_test_callbackp_get_unsafe(tsd); 241 int *data = tsd_test_datap_get_unsafe(tsd); 242 if (test_callback != NULL) { 243 test_callback(data); 244 } 245 #endif 246 } 247 248 tsd_t * 249 malloc_tsd_boot0(void) { 250 tsd_t *tsd; 251 252 ncleanups = 0; 253 if (tsd_boot0()) { 254 return NULL; 255 } 256 tsd = tsd_fetch(); 257 *tsd_arenas_tdata_bypassp_get(tsd) = true; 258 return tsd; 259 } 260 261 void 262 malloc_tsd_boot1(void) { 263 tsd_boot1(); 264 tsd_t *tsd = tsd_fetch(); 265 /* malloc_slow has been set properly. Update tsd_slow. */ 266 tsd_slow_update(tsd); 267 *tsd_arenas_tdata_bypassp_get(tsd) = false; 268 } 269 270 #ifdef _WIN32 271 static BOOL WINAPI 272 _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { 273 switch (fdwReason) { 274 #ifdef JEMALLOC_LAZY_LOCK 275 case DLL_THREAD_ATTACH: 276 isthreaded = true; 277 break; 278 #endif 279 case DLL_THREAD_DETACH: 280 _malloc_thread_cleanup(); 281 break; 282 default: 283 break; 284 } 285 return true; 286 } 287 288 /* 289 * We need to be able to say "read" here (in the "pragma section"), but have 290 * hooked "read". We won't read for the rest of the file, so we can get away 291 * with unhooking. 292 */ 293 #ifdef read 294 # undef read 295 #endif 296 297 #ifdef _MSC_VER 298 # ifdef _M_IX86 299 # pragma comment(linker, "/INCLUDE:__tls_used") 300 # pragma comment(linker, "/INCLUDE:_tls_callback") 301 # else 302 # pragma comment(linker, "/INCLUDE:_tls_used") 303 # pragma comment(linker, "/INCLUDE:tls_callback") 304 # endif 305 # pragma section(".CRT$XLY",long,read) 306 #endif 307 JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used) 308 BOOL (WINAPI *const tls_callback)(HINSTANCE hinstDLL, 309 DWORD fdwReason, LPVOID lpvReserved) = _tls_callback; 310 #endif 311 312 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 313 !defined(_WIN32)) 314 void * 315 tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block) { 316 pthread_t self = pthread_self(); 317 tsd_init_block_t *iter; 318 319 /* Check whether this thread has already inserted into the list. */ 320 malloc_mutex_lock(TSDN_NULL, &head->lock); 321 ql_foreach(iter, &head->blocks, link) { 322 if (iter->thread == self) { 323 malloc_mutex_unlock(TSDN_NULL, &head->lock); 324 return iter->data; 325 } 326 } 327 /* Insert block into list. */ 328 ql_elm_new(block, link); 329 block->thread = self; 330 ql_tail_insert(&head->blocks, block, link); 331 malloc_mutex_unlock(TSDN_NULL, &head->lock); 332 return NULL; 333 } 334 335 void 336 tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) { 337 malloc_mutex_lock(TSDN_NULL, &head->lock); 338 ql_remove(&head->blocks, block, link); 339 malloc_mutex_unlock(TSDN_NULL, &head->lock); 340 } 341 #endif 342