1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 /* Maximum number of malloc_tsd users with cleanup functions. */ 5 #define MALLOC_TSD_CLEANUPS_MAX 2 6 7 typedef bool (*malloc_tsd_cleanup_t)(void); 8 9 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 10 !defined(_WIN32)) 11 typedef struct tsd_init_block_s tsd_init_block_t; 12 typedef struct tsd_init_head_s tsd_init_head_t; 13 #endif 14 15 typedef struct tsd_s tsd_t; 16 typedef struct tsdn_s tsdn_t; 17 18 #define TSDN_NULL ((tsdn_t *)0) 19 20 typedef enum { 21 tsd_state_uninitialized, 22 tsd_state_nominal, 23 tsd_state_purgatory, 24 tsd_state_reincarnated 25 } tsd_state_t; 26 27 /* 28 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There 29 * are five macros that support (at least) three use cases: file-private, 30 * library-private, and library-private inlined. Following is an example 31 * library-private tsd variable: 32 * 33 * In example.h: 34 * typedef struct { 35 * int x; 36 * int y; 37 * } example_t; 38 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) 39 * malloc_tsd_types(example_, example_t) 40 * malloc_tsd_protos(, example_, example_t) 41 * malloc_tsd_externs(example_, example_t) 42 * In example.c: 43 * malloc_tsd_data(, example_, example_t, EX_INITIALIZER) 44 * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER, 45 * example_tsd_cleanup) 46 * 47 * The result is a set of generated functions, e.g.: 48 * 49 * bool example_tsd_boot(void) {...} 50 * bool example_tsd_booted_get(void) {...} 51 * example_t *example_tsd_get(bool init) {...} 52 * void example_tsd_set(example_t *val) {...} 53 * 54 * Note that all of the functions deal in terms of (a_type *) rather than 55 * (a_type) so that it is possible to support non-pointer types (unlike 56 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is 57 * cast to (void *). This means that the cleanup function needs to cast the 58 * function argument to (a_type *), then dereference the resulting pointer to 59 * access fields, e.g. 60 * 61 * void 62 * example_tsd_cleanup(void *arg) 63 * { 64 * example_t *example = (example_t *)arg; 65 * 66 * example->x = 42; 67 * [...] 68 * if ([want the cleanup function to be called again]) 69 * example_tsd_set(example); 70 * } 71 * 72 * If example_tsd_set() is called within example_tsd_cleanup(), it will be 73 * called again. This is similar to how pthreads TSD destruction works, except 74 * that pthreads only calls the cleanup function again if the value was set to 75 * non-NULL. 76 */ 77 78 /* malloc_tsd_types(). */ 79 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 80 #define malloc_tsd_types(a_name, a_type) 81 #elif (defined(JEMALLOC_TLS)) 82 #define malloc_tsd_types(a_name, a_type) 83 #elif (defined(_WIN32)) 84 #define malloc_tsd_types(a_name, a_type) \ 85 typedef struct { \ 86 bool initialized; \ 87 a_type val; \ 88 } a_name##tsd_wrapper_t; 89 #else 90 #define malloc_tsd_types(a_name, a_type) \ 91 typedef struct { \ 92 bool initialized; \ 93 a_type val; \ 94 } a_name##tsd_wrapper_t; 95 #endif 96 97 /* malloc_tsd_protos(). */ 98 #define malloc_tsd_protos(a_attr, a_name, a_type) \ 99 a_attr bool \ 100 a_name##tsd_boot0(void); \ 101 a_attr void \ 102 a_name##tsd_boot1(void); \ 103 a_attr bool \ 104 a_name##tsd_boot(void); \ 105 a_attr bool \ 106 a_name##tsd_booted_get(void); \ 107 a_attr a_type * \ 108 a_name##tsd_get(bool init); \ 109 a_attr void \ 110 a_name##tsd_set(a_type *val); 111 112 /* malloc_tsd_externs(). */ 113 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 114 #define malloc_tsd_externs(a_name, a_type) \ 115 extern __thread a_type a_name##tsd_tls; \ 116 extern __thread bool a_name##tsd_initialized; \ 117 extern bool a_name##tsd_booted; 118 #elif (defined(JEMALLOC_TLS)) 119 #define malloc_tsd_externs(a_name, a_type) \ 120 extern __thread a_type a_name##tsd_tls; \ 121 extern pthread_key_t a_name##tsd_tsd; \ 122 extern bool a_name##tsd_booted; 123 #elif (defined(_WIN32)) 124 #define malloc_tsd_externs(a_name, a_type) \ 125 extern DWORD a_name##tsd_tsd; \ 126 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 127 extern bool a_name##tsd_booted; 128 #else 129 #define malloc_tsd_externs(a_name, a_type) \ 130 extern pthread_key_t a_name##tsd_tsd; \ 131 extern tsd_init_head_t a_name##tsd_init_head; \ 132 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 133 extern bool a_name##tsd_booted; 134 #endif 135 136 /* malloc_tsd_data(). */ 137 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 138 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 139 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 140 a_name##tsd_tls = a_initializer; \ 141 a_attr __thread bool JEMALLOC_TLS_MODEL \ 142 a_name##tsd_initialized = false; \ 143 a_attr bool a_name##tsd_booted = false; 144 #elif (defined(JEMALLOC_TLS)) 145 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 146 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 147 a_name##tsd_tls = a_initializer; \ 148 a_attr pthread_key_t a_name##tsd_tsd; \ 149 a_attr bool a_name##tsd_booted = false; 150 #elif (defined(_WIN32)) 151 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 152 a_attr DWORD a_name##tsd_tsd; \ 153 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 154 false, \ 155 a_initializer \ 156 }; \ 157 a_attr bool a_name##tsd_booted = false; 158 #else 159 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 160 a_attr pthread_key_t a_name##tsd_tsd; \ 161 a_attr tsd_init_head_t a_name##tsd_init_head = { \ 162 ql_head_initializer(blocks), \ 163 MALLOC_MUTEX_INITIALIZER \ 164 }; \ 165 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 166 false, \ 167 a_initializer \ 168 }; \ 169 a_attr bool a_name##tsd_booted = false; 170 #endif 171 172 /* malloc_tsd_funcs(). */ 173 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 174 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 175 a_cleanup) \ 176 /* Initialization/cleanup. */ \ 177 a_attr bool \ 178 a_name##tsd_cleanup_wrapper(void) \ 179 { \ 180 \ 181 if (a_name##tsd_initialized) { \ 182 a_name##tsd_initialized = false; \ 183 a_cleanup(&a_name##tsd_tls); \ 184 } \ 185 return (a_name##tsd_initialized); \ 186 } \ 187 a_attr bool \ 188 a_name##tsd_boot0(void) \ 189 { \ 190 \ 191 if (a_cleanup != malloc_tsd_no_cleanup) { \ 192 malloc_tsd_cleanup_register( \ 193 &a_name##tsd_cleanup_wrapper); \ 194 } \ 195 a_name##tsd_booted = true; \ 196 return (false); \ 197 } \ 198 a_attr void \ 199 a_name##tsd_boot1(void) \ 200 { \ 201 \ 202 /* Do nothing. */ \ 203 } \ 204 a_attr bool \ 205 a_name##tsd_boot(void) \ 206 { \ 207 \ 208 return (a_name##tsd_boot0()); \ 209 } \ 210 a_attr bool \ 211 a_name##tsd_booted_get(void) \ 212 { \ 213 \ 214 return (a_name##tsd_booted); \ 215 } \ 216 a_attr bool \ 217 a_name##tsd_get_allocates(void) \ 218 { \ 219 \ 220 return (false); \ 221 } \ 222 /* Get/set. */ \ 223 a_attr a_type * \ 224 a_name##tsd_get(bool init) \ 225 { \ 226 \ 227 assert(a_name##tsd_booted); \ 228 return (&a_name##tsd_tls); \ 229 } \ 230 a_attr void \ 231 a_name##tsd_set(a_type *val) \ 232 { \ 233 \ 234 assert(a_name##tsd_booted); \ 235 a_name##tsd_tls = (*val); \ 236 if (a_cleanup != malloc_tsd_no_cleanup) \ 237 a_name##tsd_initialized = true; \ 238 } 239 #elif (defined(JEMALLOC_TLS)) 240 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 241 a_cleanup) \ 242 /* Initialization/cleanup. */ \ 243 a_attr bool \ 244 a_name##tsd_boot0(void) \ 245 { \ 246 \ 247 if (a_cleanup != malloc_tsd_no_cleanup) { \ 248 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \ 249 0) \ 250 return (true); \ 251 } \ 252 a_name##tsd_booted = true; \ 253 return (false); \ 254 } \ 255 a_attr void \ 256 a_name##tsd_boot1(void) \ 257 { \ 258 \ 259 /* Do nothing. */ \ 260 } \ 261 a_attr bool \ 262 a_name##tsd_boot(void) \ 263 { \ 264 \ 265 return (a_name##tsd_boot0()); \ 266 } \ 267 a_attr bool \ 268 a_name##tsd_booted_get(void) \ 269 { \ 270 \ 271 return (a_name##tsd_booted); \ 272 } \ 273 a_attr bool \ 274 a_name##tsd_get_allocates(void) \ 275 { \ 276 \ 277 return (false); \ 278 } \ 279 /* Get/set. */ \ 280 a_attr a_type * \ 281 a_name##tsd_get(bool init) \ 282 { \ 283 \ 284 assert(a_name##tsd_booted); \ 285 return (&a_name##tsd_tls); \ 286 } \ 287 a_attr void \ 288 a_name##tsd_set(a_type *val) \ 289 { \ 290 \ 291 assert(a_name##tsd_booted); \ 292 a_name##tsd_tls = (*val); \ 293 if (a_cleanup != malloc_tsd_no_cleanup) { \ 294 if (pthread_setspecific(a_name##tsd_tsd, \ 295 (void *)(&a_name##tsd_tls))) { \ 296 malloc_write("<jemalloc>: Error" \ 297 " setting TSD for "#a_name"\n"); \ 298 if (opt_abort) \ 299 abort(); \ 300 } \ 301 } \ 302 } 303 #elif (defined(_WIN32)) 304 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 305 a_cleanup) \ 306 /* Initialization/cleanup. */ \ 307 a_attr bool \ 308 a_name##tsd_cleanup_wrapper(void) \ 309 { \ 310 DWORD error = GetLastError(); \ 311 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 312 TlsGetValue(a_name##tsd_tsd); \ 313 SetLastError(error); \ 314 \ 315 if (wrapper == NULL) \ 316 return (false); \ 317 if (a_cleanup != malloc_tsd_no_cleanup && \ 318 wrapper->initialized) { \ 319 wrapper->initialized = false; \ 320 a_cleanup(&wrapper->val); \ 321 if (wrapper->initialized) { \ 322 /* Trigger another cleanup round. */ \ 323 return (true); \ 324 } \ 325 } \ 326 malloc_tsd_dalloc(wrapper); \ 327 return (false); \ 328 } \ 329 a_attr void \ 330 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 331 { \ 332 \ 333 if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ 334 malloc_write("<jemalloc>: Error setting" \ 335 " TSD for "#a_name"\n"); \ 336 abort(); \ 337 } \ 338 } \ 339 a_attr a_name##tsd_wrapper_t * \ 340 a_name##tsd_wrapper_get(bool init) \ 341 { \ 342 DWORD error = GetLastError(); \ 343 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 344 TlsGetValue(a_name##tsd_tsd); \ 345 SetLastError(error); \ 346 \ 347 if (init && unlikely(wrapper == NULL)) { \ 348 wrapper = (a_name##tsd_wrapper_t *) \ 349 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 350 if (wrapper == NULL) { \ 351 malloc_write("<jemalloc>: Error allocating" \ 352 " TSD for "#a_name"\n"); \ 353 abort(); \ 354 } else { \ 355 wrapper->initialized = false; \ 356 wrapper->val = a_initializer; \ 357 } \ 358 a_name##tsd_wrapper_set(wrapper); \ 359 } \ 360 return (wrapper); \ 361 } \ 362 a_attr bool \ 363 a_name##tsd_boot0(void) \ 364 { \ 365 \ 366 a_name##tsd_tsd = TlsAlloc(); \ 367 if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ 368 return (true); \ 369 if (a_cleanup != malloc_tsd_no_cleanup) { \ 370 malloc_tsd_cleanup_register( \ 371 &a_name##tsd_cleanup_wrapper); \ 372 } \ 373 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 374 a_name##tsd_booted = true; \ 375 return (false); \ 376 } \ 377 a_attr void \ 378 a_name##tsd_boot1(void) \ 379 { \ 380 a_name##tsd_wrapper_t *wrapper; \ 381 wrapper = (a_name##tsd_wrapper_t *) \ 382 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 383 if (wrapper == NULL) { \ 384 malloc_write("<jemalloc>: Error allocating" \ 385 " TSD for "#a_name"\n"); \ 386 abort(); \ 387 } \ 388 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 389 sizeof(a_name##tsd_wrapper_t)); \ 390 a_name##tsd_wrapper_set(wrapper); \ 391 } \ 392 a_attr bool \ 393 a_name##tsd_boot(void) \ 394 { \ 395 \ 396 if (a_name##tsd_boot0()) \ 397 return (true); \ 398 a_name##tsd_boot1(); \ 399 return (false); \ 400 } \ 401 a_attr bool \ 402 a_name##tsd_booted_get(void) \ 403 { \ 404 \ 405 return (a_name##tsd_booted); \ 406 } \ 407 a_attr bool \ 408 a_name##tsd_get_allocates(void) \ 409 { \ 410 \ 411 return (true); \ 412 } \ 413 /* Get/set. */ \ 414 a_attr a_type * \ 415 a_name##tsd_get(bool init) \ 416 { \ 417 a_name##tsd_wrapper_t *wrapper; \ 418 \ 419 assert(a_name##tsd_booted); \ 420 wrapper = a_name##tsd_wrapper_get(init); \ 421 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \ 422 return (NULL); \ 423 return (&wrapper->val); \ 424 } \ 425 a_attr void \ 426 a_name##tsd_set(a_type *val) \ 427 { \ 428 a_name##tsd_wrapper_t *wrapper; \ 429 \ 430 assert(a_name##tsd_booted); \ 431 wrapper = a_name##tsd_wrapper_get(true); \ 432 wrapper->val = *(val); \ 433 if (a_cleanup != malloc_tsd_no_cleanup) \ 434 wrapper->initialized = true; \ 435 } 436 #else 437 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 438 a_cleanup) \ 439 /* Initialization/cleanup. */ \ 440 a_attr void \ 441 a_name##tsd_cleanup_wrapper(void *arg) \ 442 { \ 443 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \ 444 \ 445 if (a_cleanup != malloc_tsd_no_cleanup && \ 446 wrapper->initialized) { \ 447 wrapper->initialized = false; \ 448 a_cleanup(&wrapper->val); \ 449 if (wrapper->initialized) { \ 450 /* Trigger another cleanup round. */ \ 451 if (pthread_setspecific(a_name##tsd_tsd, \ 452 (void *)wrapper)) { \ 453 malloc_write("<jemalloc>: Error" \ 454 " setting TSD for "#a_name"\n"); \ 455 if (opt_abort) \ 456 abort(); \ 457 } \ 458 return; \ 459 } \ 460 } \ 461 malloc_tsd_dalloc(wrapper); \ 462 } \ 463 a_attr void \ 464 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 465 { \ 466 \ 467 if (pthread_setspecific(a_name##tsd_tsd, \ 468 (void *)wrapper)) { \ 469 malloc_write("<jemalloc>: Error setting" \ 470 " TSD for "#a_name"\n"); \ 471 abort(); \ 472 } \ 473 } \ 474 a_attr a_name##tsd_wrapper_t * \ 475 a_name##tsd_wrapper_get(bool init) \ 476 { \ 477 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 478 pthread_getspecific(a_name##tsd_tsd); \ 479 \ 480 if (init && unlikely(wrapper == NULL)) { \ 481 tsd_init_block_t block; \ 482 wrapper = (a_name##tsd_wrapper_t *) \ 483 tsd_init_check_recursion(&a_name##tsd_init_head, \ 484 &block); \ 485 if (wrapper) \ 486 return (wrapper); \ 487 wrapper = (a_name##tsd_wrapper_t *) \ 488 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 489 block.data = (void *)wrapper; \ 490 if (wrapper == NULL) { \ 491 malloc_write("<jemalloc>: Error allocating" \ 492 " TSD for "#a_name"\n"); \ 493 abort(); \ 494 } else { \ 495 wrapper->initialized = false; \ 496 wrapper->val = a_initializer; \ 497 } \ 498 a_name##tsd_wrapper_set(wrapper); \ 499 tsd_init_finish(&a_name##tsd_init_head, &block); \ 500 } \ 501 return (wrapper); \ 502 } \ 503 a_attr bool \ 504 a_name##tsd_boot0(void) \ 505 { \ 506 \ 507 if (pthread_key_create(&a_name##tsd_tsd, \ 508 a_name##tsd_cleanup_wrapper) != 0) \ 509 return (true); \ 510 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 511 a_name##tsd_booted = true; \ 512 return (false); \ 513 } \ 514 a_attr void \ 515 a_name##tsd_boot1(void) \ 516 { \ 517 a_name##tsd_wrapper_t *wrapper; \ 518 wrapper = (a_name##tsd_wrapper_t *) \ 519 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 520 if (wrapper == NULL) { \ 521 malloc_write("<jemalloc>: Error allocating" \ 522 " TSD for "#a_name"\n"); \ 523 abort(); \ 524 } \ 525 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 526 sizeof(a_name##tsd_wrapper_t)); \ 527 a_name##tsd_wrapper_set(wrapper); \ 528 } \ 529 a_attr bool \ 530 a_name##tsd_boot(void) \ 531 { \ 532 \ 533 if (a_name##tsd_boot0()) \ 534 return (true); \ 535 a_name##tsd_boot1(); \ 536 return (false); \ 537 } \ 538 a_attr bool \ 539 a_name##tsd_booted_get(void) \ 540 { \ 541 \ 542 return (a_name##tsd_booted); \ 543 } \ 544 a_attr bool \ 545 a_name##tsd_get_allocates(void) \ 546 { \ 547 \ 548 return (true); \ 549 } \ 550 /* Get/set. */ \ 551 a_attr a_type * \ 552 a_name##tsd_get(bool init) \ 553 { \ 554 a_name##tsd_wrapper_t *wrapper; \ 555 \ 556 assert(a_name##tsd_booted); \ 557 wrapper = a_name##tsd_wrapper_get(init); \ 558 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \ 559 return (NULL); \ 560 return (&wrapper->val); \ 561 } \ 562 a_attr void \ 563 a_name##tsd_set(a_type *val) \ 564 { \ 565 a_name##tsd_wrapper_t *wrapper; \ 566 \ 567 assert(a_name##tsd_booted); \ 568 wrapper = a_name##tsd_wrapper_get(true); \ 569 wrapper->val = *(val); \ 570 if (a_cleanup != malloc_tsd_no_cleanup) \ 571 wrapper->initialized = true; \ 572 } 573 #endif 574 575 #endif /* JEMALLOC_H_TYPES */ 576 /******************************************************************************/ 577 #ifdef JEMALLOC_H_STRUCTS 578 579 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 580 !defined(_WIN32)) 581 struct tsd_init_block_s { 582 ql_elm(tsd_init_block_t) link; 583 pthread_t thread; 584 void *data; 585 }; 586 struct tsd_init_head_s { 587 ql_head(tsd_init_block_t) blocks; 588 malloc_mutex_t lock; 589 }; 590 #endif 591 592 #define MALLOC_TSD \ 593 /* O(name, type) */ \ 594 O(tcache, tcache_t *) \ 595 O(thread_allocated, uint64_t) \ 596 O(thread_deallocated, uint64_t) \ 597 O(prof_tdata, prof_tdata_t *) \ 598 O(iarena, arena_t *) \ 599 O(arena, arena_t *) \ 600 O(arenas_tdata, arena_tdata_t *) \ 601 O(narenas_tdata, unsigned) \ 602 O(arenas_tdata_bypass, bool) \ 603 O(tcache_enabled, tcache_enabled_t) \ 604 O(quarantine, quarantine_t *) \ 605 O(witnesses, witness_list_t) \ 606 O(witness_fork, bool) \ 607 608 #define TSD_INITIALIZER { \ 609 tsd_state_uninitialized, \ 610 NULL, \ 611 0, \ 612 0, \ 613 NULL, \ 614 NULL, \ 615 NULL, \ 616 NULL, \ 617 0, \ 618 false, \ 619 tcache_enabled_default, \ 620 NULL, \ 621 ql_head_initializer(witnesses), \ 622 false \ 623 } 624 625 struct tsd_s { 626 tsd_state_t state; 627 #define O(n, t) \ 628 t n; 629 MALLOC_TSD 630 #undef O 631 }; 632 633 /* 634 * Wrapper around tsd_t that makes it possible to avoid implicit conversion 635 * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be 636 * explicitly converted to tsd_t, which is non-nullable. 637 */ 638 struct tsdn_s { 639 tsd_t tsd; 640 }; 641 642 static const tsd_t tsd_initializer = TSD_INITIALIZER; 643 644 malloc_tsd_types(, tsd_t) 645 646 #endif /* JEMALLOC_H_STRUCTS */ 647 /******************************************************************************/ 648 #ifdef JEMALLOC_H_EXTERNS 649 650 void *malloc_tsd_malloc(size_t size); 651 void malloc_tsd_dalloc(void *wrapper); 652 void malloc_tsd_no_cleanup(void *arg); 653 void malloc_tsd_cleanup_register(bool (*f)(void)); 654 tsd_t *malloc_tsd_boot0(void); 655 void malloc_tsd_boot1(void); 656 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 657 !defined(_WIN32)) 658 void *tsd_init_check_recursion(tsd_init_head_t *head, 659 tsd_init_block_t *block); 660 void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); 661 #endif 662 void tsd_cleanup(void *arg); 663 664 #endif /* JEMALLOC_H_EXTERNS */ 665 /******************************************************************************/ 666 #ifdef JEMALLOC_H_INLINES 667 668 #ifndef JEMALLOC_ENABLE_INLINE 669 malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) 670 671 tsd_t *tsd_fetch_impl(bool init); 672 tsd_t *tsd_fetch(void); 673 tsdn_t *tsd_tsdn(tsd_t *tsd); 674 bool tsd_nominal(tsd_t *tsd); 675 #define O(n, t) \ 676 t *tsd_##n##p_get(tsd_t *tsd); \ 677 t tsd_##n##_get(tsd_t *tsd); \ 678 void tsd_##n##_set(tsd_t *tsd, t n); 679 MALLOC_TSD 680 #undef O 681 tsdn_t *tsdn_fetch(void); 682 bool tsdn_null(const tsdn_t *tsdn); 683 tsd_t *tsdn_tsd(tsdn_t *tsdn); 684 #endif 685 686 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_)) 687 malloc_tsd_externs(, tsd_t) 688 malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) 689 690 JEMALLOC_ALWAYS_INLINE tsd_t * 691 tsd_fetch_impl(bool init) 692 { 693 tsd_t *tsd = tsd_get(init); 694 695 if (!init && tsd_get_allocates() && tsd == NULL) 696 return (NULL); 697 assert(tsd != NULL); 698 699 if (unlikely(tsd->state != tsd_state_nominal)) { 700 if (tsd->state == tsd_state_uninitialized) { 701 tsd->state = tsd_state_nominal; 702 /* Trigger cleanup handler registration. */ 703 tsd_set(tsd); 704 } else if (tsd->state == tsd_state_purgatory) { 705 tsd->state = tsd_state_reincarnated; 706 tsd_set(tsd); 707 } else 708 assert(tsd->state == tsd_state_reincarnated); 709 } 710 711 return (tsd); 712 } 713 714 JEMALLOC_ALWAYS_INLINE tsd_t * 715 tsd_fetch(void) 716 { 717 718 return (tsd_fetch_impl(true)); 719 } 720 721 JEMALLOC_ALWAYS_INLINE tsdn_t * 722 tsd_tsdn(tsd_t *tsd) 723 { 724 725 return ((tsdn_t *)tsd); 726 } 727 728 JEMALLOC_INLINE bool 729 tsd_nominal(tsd_t *tsd) 730 { 731 732 return (tsd->state == tsd_state_nominal); 733 } 734 735 #define O(n, t) \ 736 JEMALLOC_ALWAYS_INLINE t * \ 737 tsd_##n##p_get(tsd_t *tsd) \ 738 { \ 739 \ 740 return (&tsd->n); \ 741 } \ 742 \ 743 JEMALLOC_ALWAYS_INLINE t \ 744 tsd_##n##_get(tsd_t *tsd) \ 745 { \ 746 \ 747 return (*tsd_##n##p_get(tsd)); \ 748 } \ 749 \ 750 JEMALLOC_ALWAYS_INLINE void \ 751 tsd_##n##_set(tsd_t *tsd, t n) \ 752 { \ 753 \ 754 assert(tsd->state == tsd_state_nominal); \ 755 tsd->n = n; \ 756 } 757 MALLOC_TSD 758 #undef O 759 760 JEMALLOC_ALWAYS_INLINE tsdn_t * 761 tsdn_fetch(void) 762 { 763 764 if (!tsd_booted_get()) 765 return (NULL); 766 767 return (tsd_tsdn(tsd_fetch_impl(false))); 768 } 769 770 JEMALLOC_ALWAYS_INLINE bool 771 tsdn_null(const tsdn_t *tsdn) 772 { 773 774 return (tsdn == NULL); 775 } 776 777 JEMALLOC_ALWAYS_INLINE tsd_t * 778 tsdn_tsd(tsdn_t *tsdn) 779 { 780 781 assert(!tsdn_null(tsdn)); 782 783 return (&tsdn->tsd); 784 } 785 #endif 786 787 #endif /* JEMALLOC_H_INLINES */ 788 /******************************************************************************/ 789