1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 8 const char *__malloc_options_1_0 = NULL; 9 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 10 11 /* Runtime configuration options. */ 12 const char *je_malloc_conf JEMALLOC_ATTR(weak); 13 bool opt_abort = 14 #ifdef JEMALLOC_DEBUG 15 true 16 #else 17 false 18 #endif 19 ; 20 const char *opt_junk = 21 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 22 "true" 23 #else 24 "false" 25 #endif 26 ; 27 bool opt_junk_alloc = 28 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 29 true 30 #else 31 false 32 #endif 33 ; 34 bool opt_junk_free = 35 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 36 true 37 #else 38 false 39 #endif 40 ; 41 42 size_t opt_quarantine = ZU(0); 43 bool opt_redzone = false; 44 bool opt_utrace = false; 45 bool opt_xmalloc = false; 46 bool opt_zero = false; 47 size_t opt_narenas = 0; 48 49 /* Initialized to true if the process is running inside Valgrind. */ 50 bool in_valgrind; 51 52 unsigned ncpus; 53 54 /* Protects arenas initialization (arenas, narenas_total). */ 55 static malloc_mutex_t arenas_lock; 56 /* 57 * Arenas that are used to service external requests. Not all elements of the 58 * arenas array are necessarily used; arenas are created lazily as needed. 59 * 60 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 61 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 62 * takes some action to create them and allocate from them. 63 */ 64 static arena_t **arenas; 65 static unsigned narenas_total; 66 static arena_t *a0; /* arenas[0]; read-only after initialization. */ 67 static unsigned narenas_auto; /* Read-only after initialization. */ 68 69 typedef enum { 70 malloc_init_uninitialized = 3, 71 malloc_init_a0_initialized = 2, 72 malloc_init_recursible = 1, 73 malloc_init_initialized = 0 /* Common case --> jnz. */ 74 } malloc_init_t; 75 static malloc_init_t malloc_init_state = malloc_init_uninitialized; 76 77 JEMALLOC_ALIGNED(CACHELINE) 78 const size_t index2size_tab[NSIZES] = { 79 #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 80 ((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)), 81 SIZE_CLASSES 82 #undef SC 83 }; 84 85 JEMALLOC_ALIGNED(CACHELINE) 86 const uint8_t size2index_tab[] = { 87 #if LG_TINY_MIN == 0 88 #warning "Dangerous LG_TINY_MIN" 89 #define S2B_0(i) i, 90 #elif LG_TINY_MIN == 1 91 #warning "Dangerous LG_TINY_MIN" 92 #define S2B_1(i) i, 93 #elif LG_TINY_MIN == 2 94 #warning "Dangerous LG_TINY_MIN" 95 #define S2B_2(i) i, 96 #elif LG_TINY_MIN == 3 97 #define S2B_3(i) i, 98 #elif LG_TINY_MIN == 4 99 #define S2B_4(i) i, 100 #elif LG_TINY_MIN == 5 101 #define S2B_5(i) i, 102 #elif LG_TINY_MIN == 6 103 #define S2B_6(i) i, 104 #elif LG_TINY_MIN == 7 105 #define S2B_7(i) i, 106 #elif LG_TINY_MIN == 8 107 #define S2B_8(i) i, 108 #elif LG_TINY_MIN == 9 109 #define S2B_9(i) i, 110 #elif LG_TINY_MIN == 10 111 #define S2B_10(i) i, 112 #elif LG_TINY_MIN == 11 113 #define S2B_11(i) i, 114 #else 115 #error "Unsupported LG_TINY_MIN" 116 #endif 117 #if LG_TINY_MIN < 1 118 #define S2B_1(i) S2B_0(i) S2B_0(i) 119 #endif 120 #if LG_TINY_MIN < 2 121 #define S2B_2(i) S2B_1(i) S2B_1(i) 122 #endif 123 #if LG_TINY_MIN < 3 124 #define S2B_3(i) S2B_2(i) S2B_2(i) 125 #endif 126 #if LG_TINY_MIN < 4 127 #define S2B_4(i) S2B_3(i) S2B_3(i) 128 #endif 129 #if LG_TINY_MIN < 5 130 #define S2B_5(i) S2B_4(i) S2B_4(i) 131 #endif 132 #if LG_TINY_MIN < 6 133 #define S2B_6(i) S2B_5(i) S2B_5(i) 134 #endif 135 #if LG_TINY_MIN < 7 136 #define S2B_7(i) S2B_6(i) S2B_6(i) 137 #endif 138 #if LG_TINY_MIN < 8 139 #define S2B_8(i) S2B_7(i) S2B_7(i) 140 #endif 141 #if LG_TINY_MIN < 9 142 #define S2B_9(i) S2B_8(i) S2B_8(i) 143 #endif 144 #if LG_TINY_MIN < 10 145 #define S2B_10(i) S2B_9(i) S2B_9(i) 146 #endif 147 #if LG_TINY_MIN < 11 148 #define S2B_11(i) S2B_10(i) S2B_10(i) 149 #endif 150 #define S2B_no(i) 151 #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 152 S2B_##lg_delta_lookup(index) 153 SIZE_CLASSES 154 #undef S2B_3 155 #undef S2B_4 156 #undef S2B_5 157 #undef S2B_6 158 #undef S2B_7 159 #undef S2B_8 160 #undef S2B_9 161 #undef S2B_10 162 #undef S2B_11 163 #undef S2B_no 164 #undef SC 165 }; 166 167 #ifdef JEMALLOC_THREADED_INIT 168 /* Used to let the initializing thread recursively allocate. */ 169 # define NO_INITIALIZER ((unsigned long)0) 170 # define INITIALIZER pthread_self() 171 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 172 static pthread_t malloc_initializer = NO_INITIALIZER; 173 #else 174 # define NO_INITIALIZER false 175 # define INITIALIZER true 176 # define IS_INITIALIZER malloc_initializer 177 static bool malloc_initializer = NO_INITIALIZER; 178 #endif 179 180 /* Used to avoid initialization races. */ 181 #ifdef _WIN32 182 #if _WIN32_WINNT >= 0x0600 183 static malloc_mutex_t init_lock = SRWLOCK_INIT; 184 #else 185 static malloc_mutex_t init_lock; 186 static bool init_lock_initialized = false; 187 188 JEMALLOC_ATTR(constructor) 189 static void WINAPI 190 _init_init_lock(void) 191 { 192 193 /* If another constructor in the same binary is using mallctl to 194 * e.g. setup chunk hooks, it may end up running before this one, 195 * and malloc_init_hard will crash trying to lock the uninitialized 196 * lock. So we force an initialization of the lock in 197 * malloc_init_hard as well. We don't try to care about atomicity 198 * of the accessed to the init_lock_initialized boolean, since it 199 * really only matters early in the process creation, before any 200 * separate thread normally starts doing anything. */ 201 if (!init_lock_initialized) 202 malloc_mutex_init(&init_lock); 203 init_lock_initialized = true; 204 } 205 206 #ifdef _MSC_VER 207 # pragma section(".CRT$XCU", read) 208 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 209 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 210 #endif 211 #endif 212 #else 213 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 214 #endif 215 216 typedef struct { 217 void *p; /* Input pointer (as in realloc(p, s)). */ 218 size_t s; /* Request size. */ 219 void *r; /* Result pointer. */ 220 } malloc_utrace_t; 221 222 #ifdef JEMALLOC_UTRACE 223 # define UTRACE(a, b, c) do { \ 224 if (unlikely(opt_utrace)) { \ 225 int utrace_serrno = errno; \ 226 malloc_utrace_t ut; \ 227 ut.p = (a); \ 228 ut.s = (b); \ 229 ut.r = (c); \ 230 utrace(&ut, sizeof(ut)); \ 231 errno = utrace_serrno; \ 232 } \ 233 } while (0) 234 #else 235 # define UTRACE(a, b, c) 236 #endif 237 238 /******************************************************************************/ 239 /* 240 * Function prototypes for static functions that are referenced prior to 241 * definition. 242 */ 243 244 static bool malloc_init_hard_a0(void); 245 static bool malloc_init_hard(void); 246 247 /******************************************************************************/ 248 /* 249 * Begin miscellaneous support functions. 250 */ 251 252 JEMALLOC_ALWAYS_INLINE_C bool 253 malloc_initialized(void) 254 { 255 256 return (malloc_init_state == malloc_init_initialized); 257 } 258 259 JEMALLOC_ALWAYS_INLINE_C void 260 malloc_thread_init(void) 261 { 262 263 /* 264 * TSD initialization can't be safely done as a side effect of 265 * deallocation, because it is possible for a thread to do nothing but 266 * deallocate its TLS data via free(), in which case writing to TLS 267 * would cause write-after-free memory corruption. The quarantine 268 * facility *only* gets used as a side effect of deallocation, so make 269 * a best effort attempt at initializing its TSD by hooking all 270 * allocation events. 271 */ 272 if (config_fill && unlikely(opt_quarantine)) 273 quarantine_alloc_hook(); 274 } 275 276 JEMALLOC_ALWAYS_INLINE_C bool 277 malloc_init_a0(void) 278 { 279 280 if (unlikely(malloc_init_state == malloc_init_uninitialized)) 281 return (malloc_init_hard_a0()); 282 return (false); 283 } 284 285 JEMALLOC_ALWAYS_INLINE_C bool 286 malloc_init(void) 287 { 288 289 if (unlikely(!malloc_initialized()) && malloc_init_hard()) 290 return (true); 291 malloc_thread_init(); 292 293 return (false); 294 } 295 296 /* 297 * The a0*() functions are used instead of i[mcd]alloc() in situations that 298 * cannot tolerate TLS variable access. 299 */ 300 301 arena_t * 302 a0get(void) 303 { 304 305 assert(a0 != NULL); 306 return (a0); 307 } 308 309 static void * 310 a0ialloc(size_t size, bool zero, bool is_metadata) 311 { 312 313 if (unlikely(malloc_init_a0())) 314 return (NULL); 315 316 return (iallocztm(NULL, size, zero, false, is_metadata, a0get())); 317 } 318 319 static void 320 a0idalloc(void *ptr, bool is_metadata) 321 { 322 323 idalloctm(NULL, ptr, false, is_metadata); 324 } 325 326 void * 327 a0malloc(size_t size) 328 { 329 330 return (a0ialloc(size, false, true)); 331 } 332 333 void 334 a0dalloc(void *ptr) 335 { 336 337 a0idalloc(ptr, true); 338 } 339 340 /* 341 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive 342 * situations that cannot tolerate TLS variable access (TLS allocation and very 343 * early internal data structure initialization). 344 */ 345 346 void * 347 bootstrap_malloc(size_t size) 348 { 349 350 if (unlikely(size == 0)) 351 size = 1; 352 353 return (a0ialloc(size, false, false)); 354 } 355 356 void * 357 bootstrap_calloc(size_t num, size_t size) 358 { 359 size_t num_size; 360 361 num_size = num * size; 362 if (unlikely(num_size == 0)) { 363 assert(num == 0 || size == 0); 364 num_size = 1; 365 } 366 367 return (a0ialloc(num_size, true, false)); 368 } 369 370 void 371 bootstrap_free(void *ptr) 372 { 373 374 if (unlikely(ptr == NULL)) 375 return; 376 377 a0idalloc(ptr, false); 378 } 379 380 /* Create a new arena and insert it into the arenas array at index ind. */ 381 static arena_t * 382 arena_init_locked(unsigned ind) 383 { 384 arena_t *arena; 385 386 /* Expand arenas if necessary. */ 387 assert(ind <= narenas_total); 388 if (ind > MALLOCX_ARENA_MAX) 389 return (NULL); 390 if (ind == narenas_total) { 391 unsigned narenas_new = narenas_total + 1; 392 arena_t **arenas_new = 393 (arena_t **)a0malloc(CACHELINE_CEILING(narenas_new * 394 sizeof(arena_t *))); 395 if (arenas_new == NULL) 396 return (NULL); 397 memcpy(arenas_new, arenas, narenas_total * sizeof(arena_t *)); 398 arenas_new[ind] = NULL; 399 /* 400 * Deallocate only if arenas came from a0malloc() (not 401 * base_alloc()). 402 */ 403 if (narenas_total != narenas_auto) 404 a0dalloc(arenas); 405 arenas = arenas_new; 406 narenas_total = narenas_new; 407 } 408 409 /* 410 * Another thread may have already initialized arenas[ind] if it's an 411 * auto arena. 412 */ 413 arena = arenas[ind]; 414 if (arena != NULL) { 415 assert(ind < narenas_auto); 416 return (arena); 417 } 418 419 /* Actually initialize the arena. */ 420 arena = arenas[ind] = arena_new(ind); 421 return (arena); 422 } 423 424 arena_t * 425 arena_init(unsigned ind) 426 { 427 arena_t *arena; 428 429 malloc_mutex_lock(&arenas_lock); 430 arena = arena_init_locked(ind); 431 malloc_mutex_unlock(&arenas_lock); 432 return (arena); 433 } 434 435 unsigned 436 narenas_total_get(void) 437 { 438 unsigned narenas; 439 440 malloc_mutex_lock(&arenas_lock); 441 narenas = narenas_total; 442 malloc_mutex_unlock(&arenas_lock); 443 444 return (narenas); 445 } 446 447 static void 448 arena_bind_locked(tsd_t *tsd, unsigned ind) 449 { 450 arena_t *arena; 451 452 arena = arenas[ind]; 453 arena->nthreads++; 454 455 if (tsd_nominal(tsd)) 456 tsd_arena_set(tsd, arena); 457 } 458 459 static void 460 arena_bind(tsd_t *tsd, unsigned ind) 461 { 462 463 malloc_mutex_lock(&arenas_lock); 464 arena_bind_locked(tsd, ind); 465 malloc_mutex_unlock(&arenas_lock); 466 } 467 468 void 469 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) 470 { 471 arena_t *oldarena, *newarena; 472 473 malloc_mutex_lock(&arenas_lock); 474 oldarena = arenas[oldind]; 475 newarena = arenas[newind]; 476 oldarena->nthreads--; 477 newarena->nthreads++; 478 malloc_mutex_unlock(&arenas_lock); 479 tsd_arena_set(tsd, newarena); 480 } 481 482 unsigned 483 arena_nbound(unsigned ind) 484 { 485 unsigned nthreads; 486 487 malloc_mutex_lock(&arenas_lock); 488 nthreads = arenas[ind]->nthreads; 489 malloc_mutex_unlock(&arenas_lock); 490 return (nthreads); 491 } 492 493 static void 494 arena_unbind(tsd_t *tsd, unsigned ind) 495 { 496 arena_t *arena; 497 498 malloc_mutex_lock(&arenas_lock); 499 arena = arenas[ind]; 500 arena->nthreads--; 501 malloc_mutex_unlock(&arenas_lock); 502 tsd_arena_set(tsd, NULL); 503 } 504 505 arena_t * 506 arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) 507 { 508 arena_t *arena; 509 arena_t **arenas_cache = tsd_arenas_cache_get(tsd); 510 unsigned narenas_cache = tsd_narenas_cache_get(tsd); 511 unsigned narenas_actual = narenas_total_get(); 512 513 /* Deallocate old cache if it's too small. */ 514 if (arenas_cache != NULL && narenas_cache < narenas_actual) { 515 a0dalloc(arenas_cache); 516 arenas_cache = NULL; 517 narenas_cache = 0; 518 tsd_arenas_cache_set(tsd, arenas_cache); 519 tsd_narenas_cache_set(tsd, narenas_cache); 520 } 521 522 /* Allocate cache if it's missing. */ 523 if (arenas_cache == NULL) { 524 bool *arenas_cache_bypassp = tsd_arenas_cache_bypassp_get(tsd); 525 assert(ind < narenas_actual || !init_if_missing); 526 narenas_cache = (ind < narenas_actual) ? narenas_actual : ind+1; 527 528 if (tsd_nominal(tsd) && !*arenas_cache_bypassp) { 529 *arenas_cache_bypassp = true; 530 arenas_cache = (arena_t **)a0malloc(sizeof(arena_t *) * 531 narenas_cache); 532 *arenas_cache_bypassp = false; 533 } 534 if (arenas_cache == NULL) { 535 /* 536 * This function must always tell the truth, even if 537 * it's slow, so don't let OOM, thread cleanup (note 538 * tsd_nominal check), nor recursive allocation 539 * avoidance (note arenas_cache_bypass check) get in the 540 * way. 541 */ 542 if (ind >= narenas_actual) 543 return (NULL); 544 malloc_mutex_lock(&arenas_lock); 545 arena = arenas[ind]; 546 malloc_mutex_unlock(&arenas_lock); 547 return (arena); 548 } 549 assert(tsd_nominal(tsd) && !*arenas_cache_bypassp); 550 tsd_arenas_cache_set(tsd, arenas_cache); 551 tsd_narenas_cache_set(tsd, narenas_cache); 552 } 553 554 /* 555 * Copy to cache. It's possible that the actual number of arenas has 556 * increased since narenas_total_get() was called above, but that causes 557 * no correctness issues unless two threads concurrently execute the 558 * arenas.extend mallctl, which we trust mallctl synchronization to 559 * prevent. 560 */ 561 malloc_mutex_lock(&arenas_lock); 562 memcpy(arenas_cache, arenas, sizeof(arena_t *) * narenas_actual); 563 malloc_mutex_unlock(&arenas_lock); 564 if (narenas_cache > narenas_actual) { 565 memset(&arenas_cache[narenas_actual], 0, sizeof(arena_t *) * 566 (narenas_cache - narenas_actual)); 567 } 568 569 /* Read the refreshed cache, and init the arena if necessary. */ 570 arena = arenas_cache[ind]; 571 if (init_if_missing && arena == NULL) 572 arena = arenas_cache[ind] = arena_init(ind); 573 return (arena); 574 } 575 576 /* Slow path, called only by arena_choose(). */ 577 arena_t * 578 arena_choose_hard(tsd_t *tsd) 579 { 580 arena_t *ret; 581 582 if (narenas_auto > 1) { 583 unsigned i, choose, first_null; 584 585 choose = 0; 586 first_null = narenas_auto; 587 malloc_mutex_lock(&arenas_lock); 588 assert(a0get() != NULL); 589 for (i = 1; i < narenas_auto; i++) { 590 if (arenas[i] != NULL) { 591 /* 592 * Choose the first arena that has the lowest 593 * number of threads assigned to it. 594 */ 595 if (arenas[i]->nthreads < 596 arenas[choose]->nthreads) 597 choose = i; 598 } else if (first_null == narenas_auto) { 599 /* 600 * Record the index of the first uninitialized 601 * arena, in case all extant arenas are in use. 602 * 603 * NB: It is possible for there to be 604 * discontinuities in terms of initialized 605 * versus uninitialized arenas, due to the 606 * "thread.arena" mallctl. 607 */ 608 first_null = i; 609 } 610 } 611 612 if (arenas[choose]->nthreads == 0 613 || first_null == narenas_auto) { 614 /* 615 * Use an unloaded arena, or the least loaded arena if 616 * all arenas are already initialized. 617 */ 618 ret = arenas[choose]; 619 } else { 620 /* Initialize a new arena. */ 621 choose = first_null; 622 ret = arena_init_locked(choose); 623 if (ret == NULL) { 624 malloc_mutex_unlock(&arenas_lock); 625 return (NULL); 626 } 627 } 628 arena_bind_locked(tsd, choose); 629 malloc_mutex_unlock(&arenas_lock); 630 } else { 631 ret = a0get(); 632 arena_bind(tsd, 0); 633 } 634 635 return (ret); 636 } 637 638 void 639 thread_allocated_cleanup(tsd_t *tsd) 640 { 641 642 /* Do nothing. */ 643 } 644 645 void 646 thread_deallocated_cleanup(tsd_t *tsd) 647 { 648 649 /* Do nothing. */ 650 } 651 652 void 653 arena_cleanup(tsd_t *tsd) 654 { 655 arena_t *arena; 656 657 arena = tsd_arena_get(tsd); 658 if (arena != NULL) 659 arena_unbind(tsd, arena->ind); 660 } 661 662 void 663 arenas_cache_cleanup(tsd_t *tsd) 664 { 665 arena_t **arenas_cache; 666 667 arenas_cache = tsd_arenas_cache_get(tsd); 668 if (arenas_cache != NULL) { 669 tsd_arenas_cache_set(tsd, NULL); 670 a0dalloc(arenas_cache); 671 } 672 } 673 674 void 675 narenas_cache_cleanup(tsd_t *tsd) 676 { 677 678 /* Do nothing. */ 679 } 680 681 void 682 arenas_cache_bypass_cleanup(tsd_t *tsd) 683 { 684 685 /* Do nothing. */ 686 } 687 688 static void 689 stats_print_atexit(void) 690 { 691 692 if (config_tcache && config_stats) { 693 unsigned narenas, i; 694 695 /* 696 * Merge stats from extant threads. This is racy, since 697 * individual threads do not lock when recording tcache stats 698 * events. As a consequence, the final stats may be slightly 699 * out of date by the time they are reported, if other threads 700 * continue to allocate. 701 */ 702 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 703 arena_t *arena = arenas[i]; 704 if (arena != NULL) { 705 tcache_t *tcache; 706 707 /* 708 * tcache_stats_merge() locks bins, so if any 709 * code is introduced that acquires both arena 710 * and bin locks in the opposite order, 711 * deadlocks may result. 712 */ 713 malloc_mutex_lock(&arena->lock); 714 ql_foreach(tcache, &arena->tcache_ql, link) { 715 tcache_stats_merge(tcache, arena); 716 } 717 malloc_mutex_unlock(&arena->lock); 718 } 719 } 720 } 721 je_malloc_stats_print(NULL, NULL, NULL); 722 } 723 724 /* 725 * End miscellaneous support functions. 726 */ 727 /******************************************************************************/ 728 /* 729 * Begin initialization functions. 730 */ 731 732 #ifndef JEMALLOC_HAVE_SECURE_GETENV 733 static char * 734 secure_getenv(const char *name) 735 { 736 737 # ifdef JEMALLOC_HAVE_ISSETUGID 738 if (issetugid() != 0) 739 return (NULL); 740 # endif 741 return (getenv(name)); 742 } 743 #endif 744 745 static unsigned 746 malloc_ncpus(void) 747 { 748 long result; 749 750 #ifdef _WIN32 751 SYSTEM_INFO si; 752 GetSystemInfo(&si); 753 result = si.dwNumberOfProcessors; 754 #else 755 result = sysconf(_SC_NPROCESSORS_ONLN); 756 #endif 757 return ((result == -1) ? 1 : (unsigned)result); 758 } 759 760 static bool 761 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 762 char const **v_p, size_t *vlen_p) 763 { 764 bool accept; 765 const char *opts = *opts_p; 766 767 *k_p = opts; 768 769 for (accept = false; !accept;) { 770 switch (*opts) { 771 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 772 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 773 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 774 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 775 case 'Y': case 'Z': 776 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 777 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 778 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 779 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 780 case 'y': case 'z': 781 case '0': case '1': case '2': case '3': case '4': case '5': 782 case '6': case '7': case '8': case '9': 783 case '_': 784 opts++; 785 break; 786 case ':': 787 opts++; 788 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 789 *v_p = opts; 790 accept = true; 791 break; 792 case '\0': 793 if (opts != *opts_p) { 794 malloc_write("<jemalloc>: Conf string ends " 795 "with key\n"); 796 } 797 return (true); 798 default: 799 malloc_write("<jemalloc>: Malformed conf string\n"); 800 return (true); 801 } 802 } 803 804 for (accept = false; !accept;) { 805 switch (*opts) { 806 case ',': 807 opts++; 808 /* 809 * Look ahead one character here, because the next time 810 * this function is called, it will assume that end of 811 * input has been cleanly reached if no input remains, 812 * but we have optimistically already consumed the 813 * comma if one exists. 814 */ 815 if (*opts == '\0') { 816 malloc_write("<jemalloc>: Conf string ends " 817 "with comma\n"); 818 } 819 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 820 accept = true; 821 break; 822 case '\0': 823 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 824 accept = true; 825 break; 826 default: 827 opts++; 828 break; 829 } 830 } 831 832 *opts_p = opts; 833 return (false); 834 } 835 836 static void 837 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 838 size_t vlen) 839 { 840 841 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 842 (int)vlen, v); 843 } 844 845 static void 846 malloc_conf_init(void) 847 { 848 unsigned i; 849 char buf[PATH_MAX + 1]; 850 const char *opts, *k, *v; 851 size_t klen, vlen; 852 853 /* 854 * Automatically configure valgrind before processing options. The 855 * valgrind option remains in jemalloc 3.x for compatibility reasons. 856 */ 857 if (config_valgrind) { 858 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 859 if (config_fill && unlikely(in_valgrind)) { 860 opt_junk = "false"; 861 opt_junk_alloc = false; 862 opt_junk_free = false; 863 assert(!opt_zero); 864 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 865 opt_redzone = true; 866 } 867 if (config_tcache && unlikely(in_valgrind)) 868 opt_tcache = false; 869 } 870 871 for (i = 0; i < 3; i++) { 872 /* Get runtime configuration. */ 873 switch (i) { 874 case 0: 875 if (je_malloc_conf != NULL) { 876 /* 877 * Use options that were compiled into the 878 * program. 879 */ 880 opts = je_malloc_conf; 881 } else { 882 /* No configuration specified. */ 883 buf[0] = '\0'; 884 opts = buf; 885 } 886 break; 887 case 1: { 888 int linklen = 0; 889 #ifndef _WIN32 890 int saved_errno = errno; 891 const char *linkname = 892 # ifdef JEMALLOC_PREFIX 893 "/etc/"JEMALLOC_PREFIX"malloc.conf" 894 # else 895 "/etc/malloc.conf" 896 # endif 897 ; 898 899 /* 900 * Try to use the contents of the "/etc/malloc.conf" 901 * symbolic link's name. 902 */ 903 linklen = readlink(linkname, buf, sizeof(buf) - 1); 904 if (linklen == -1) { 905 /* No configuration specified. */ 906 linklen = 0; 907 /* Restore errno. */ 908 set_errno(saved_errno); 909 } 910 #endif 911 buf[linklen] = '\0'; 912 opts = buf; 913 break; 914 } case 2: { 915 const char *envname = 916 #ifdef JEMALLOC_PREFIX 917 JEMALLOC_CPREFIX"MALLOC_CONF" 918 #else 919 "MALLOC_CONF" 920 #endif 921 ; 922 923 if ((opts = secure_getenv(envname)) != NULL) { 924 /* 925 * Do nothing; opts is already initialized to 926 * the value of the MALLOC_CONF environment 927 * variable. 928 */ 929 } else { 930 /* No configuration specified. */ 931 buf[0] = '\0'; 932 opts = buf; 933 } 934 break; 935 } default: 936 not_reached(); 937 buf[0] = '\0'; 938 opts = buf; 939 } 940 941 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, 942 &vlen)) { 943 #define CONF_MATCH(n) \ 944 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 945 #define CONF_MATCH_VALUE(n) \ 946 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) 947 #define CONF_HANDLE_BOOL(o, n, cont) \ 948 if (CONF_MATCH(n)) { \ 949 if (CONF_MATCH_VALUE("true")) \ 950 o = true; \ 951 else if (CONF_MATCH_VALUE("false")) \ 952 o = false; \ 953 else { \ 954 malloc_conf_error( \ 955 "Invalid conf value", \ 956 k, klen, v, vlen); \ 957 } \ 958 if (cont) \ 959 continue; \ 960 } 961 #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ 962 if (CONF_MATCH(n)) { \ 963 uintmax_t um; \ 964 char *end; \ 965 \ 966 set_errno(0); \ 967 um = malloc_strtoumax(v, &end, 0); \ 968 if (get_errno() != 0 || (uintptr_t)end -\ 969 (uintptr_t)v != vlen) { \ 970 malloc_conf_error( \ 971 "Invalid conf value", \ 972 k, klen, v, vlen); \ 973 } else if (clip) { \ 974 if ((min) != 0 && um < (min)) \ 975 o = (min); \ 976 else if (um > (max)) \ 977 o = (max); \ 978 else \ 979 o = um; \ 980 } else { \ 981 if (((min) != 0 && um < (min)) \ 982 || um > (max)) { \ 983 malloc_conf_error( \ 984 "Out-of-range " \ 985 "conf value", \ 986 k, klen, v, vlen); \ 987 } else \ 988 o = um; \ 989 } \ 990 continue; \ 991 } 992 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 993 if (CONF_MATCH(n)) { \ 994 long l; \ 995 char *end; \ 996 \ 997 set_errno(0); \ 998 l = strtol(v, &end, 0); \ 999 if (get_errno() != 0 || (uintptr_t)end -\ 1000 (uintptr_t)v != vlen) { \ 1001 malloc_conf_error( \ 1002 "Invalid conf value", \ 1003 k, klen, v, vlen); \ 1004 } else if (l < (ssize_t)(min) || l > \ 1005 (ssize_t)(max)) { \ 1006 malloc_conf_error( \ 1007 "Out-of-range conf value", \ 1008 k, klen, v, vlen); \ 1009 } else \ 1010 o = l; \ 1011 continue; \ 1012 } 1013 #define CONF_HANDLE_CHAR_P(o, n, d) \ 1014 if (CONF_MATCH(n)) { \ 1015 size_t cpylen = (vlen <= \ 1016 sizeof(o)-1) ? vlen : \ 1017 sizeof(o)-1; \ 1018 strncpy(o, v, cpylen); \ 1019 o[cpylen] = '\0'; \ 1020 continue; \ 1021 } 1022 1023 CONF_HANDLE_BOOL(opt_abort, "abort", true) 1024 /* 1025 * Chunks always require at least one header page, 1026 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and 1027 * possibly an additional page in the presence of 1028 * redzones. In order to simplify options processing, 1029 * use a conservative bound that accommodates all these 1030 * constraints. 1031 */ 1032 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 1033 LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), 1034 (sizeof(size_t) << 3) - 1, true) 1035 if (strncmp("dss", k, klen) == 0) { 1036 int i; 1037 bool match = false; 1038 for (i = 0; i < dss_prec_limit; i++) { 1039 if (strncmp(dss_prec_names[i], v, vlen) 1040 == 0) { 1041 if (chunk_dss_prec_set(i)) { 1042 malloc_conf_error( 1043 "Error setting dss", 1044 k, klen, v, vlen); 1045 } else { 1046 opt_dss = 1047 dss_prec_names[i]; 1048 match = true; 1049 break; 1050 } 1051 } 1052 } 1053 if (!match) { 1054 malloc_conf_error("Invalid conf value", 1055 k, klen, v, vlen); 1056 } 1057 continue; 1058 } 1059 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 1060 SIZE_T_MAX, false) 1061 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 1062 -1, (sizeof(size_t) << 3) - 1) 1063 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) 1064 if (config_fill) { 1065 if (CONF_MATCH("junk")) { 1066 if (CONF_MATCH_VALUE("true")) { 1067 opt_junk = "true"; 1068 opt_junk_alloc = opt_junk_free = 1069 true; 1070 } else if (CONF_MATCH_VALUE("false")) { 1071 opt_junk = "false"; 1072 opt_junk_alloc = opt_junk_free = 1073 false; 1074 } else if (CONF_MATCH_VALUE("alloc")) { 1075 opt_junk = "alloc"; 1076 opt_junk_alloc = true; 1077 opt_junk_free = false; 1078 } else if (CONF_MATCH_VALUE("free")) { 1079 opt_junk = "free"; 1080 opt_junk_alloc = false; 1081 opt_junk_free = true; 1082 } else { 1083 malloc_conf_error( 1084 "Invalid conf value", k, 1085 klen, v, vlen); 1086 } 1087 continue; 1088 } 1089 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 1090 0, SIZE_T_MAX, false) 1091 CONF_HANDLE_BOOL(opt_redzone, "redzone", true) 1092 CONF_HANDLE_BOOL(opt_zero, "zero", true) 1093 } 1094 if (config_utrace) { 1095 CONF_HANDLE_BOOL(opt_utrace, "utrace", true) 1096 } 1097 if (config_xmalloc) { 1098 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) 1099 } 1100 if (config_tcache) { 1101 CONF_HANDLE_BOOL(opt_tcache, "tcache", 1102 !config_valgrind || !in_valgrind) 1103 if (CONF_MATCH("tcache")) { 1104 assert(config_valgrind && in_valgrind); 1105 if (opt_tcache) { 1106 opt_tcache = false; 1107 malloc_conf_error( 1108 "tcache cannot be enabled " 1109 "while running inside Valgrind", 1110 k, klen, v, vlen); 1111 } 1112 continue; 1113 } 1114 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 1115 "lg_tcache_max", -1, 1116 (sizeof(size_t) << 3) - 1) 1117 } 1118 if (config_prof) { 1119 CONF_HANDLE_BOOL(opt_prof, "prof", true) 1120 CONF_HANDLE_CHAR_P(opt_prof_prefix, 1121 "prof_prefix", "jeprof") 1122 CONF_HANDLE_BOOL(opt_prof_active, "prof_active", 1123 true) 1124 CONF_HANDLE_BOOL(opt_prof_thread_active_init, 1125 "prof_thread_active_init", true) 1126 CONF_HANDLE_SIZE_T(opt_lg_prof_sample, 1127 "lg_prof_sample", 0, 1128 (sizeof(uint64_t) << 3) - 1, true) 1129 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", 1130 true) 1131 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 1132 "lg_prof_interval", -1, 1133 (sizeof(uint64_t) << 3) - 1) 1134 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", 1135 true) 1136 CONF_HANDLE_BOOL(opt_prof_final, "prof_final", 1137 true) 1138 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", 1139 true) 1140 } 1141 malloc_conf_error("Invalid conf pair", k, klen, v, 1142 vlen); 1143 #undef CONF_MATCH 1144 #undef CONF_HANDLE_BOOL 1145 #undef CONF_HANDLE_SIZE_T 1146 #undef CONF_HANDLE_SSIZE_T 1147 #undef CONF_HANDLE_CHAR_P 1148 } 1149 } 1150 } 1151 1152 /* init_lock must be held. */ 1153 static bool 1154 malloc_init_hard_needed(void) 1155 { 1156 1157 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == 1158 malloc_init_recursible)) { 1159 /* 1160 * Another thread initialized the allocator before this one 1161 * acquired init_lock, or this thread is the initializing 1162 * thread, and it is recursively allocating. 1163 */ 1164 return (false); 1165 } 1166 #ifdef JEMALLOC_THREADED_INIT 1167 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { 1168 /* Busy-wait until the initializing thread completes. */ 1169 do { 1170 malloc_mutex_unlock(&init_lock); 1171 CPU_SPINWAIT; 1172 malloc_mutex_lock(&init_lock); 1173 } while (!malloc_initialized()); 1174 return (false); 1175 } 1176 #endif 1177 return (true); 1178 } 1179 1180 /* init_lock must be held. */ 1181 static bool 1182 malloc_init_hard_a0_locked(void) 1183 { 1184 1185 malloc_initializer = INITIALIZER; 1186 1187 if (config_prof) 1188 prof_boot0(); 1189 malloc_conf_init(); 1190 if (opt_stats_print) { 1191 /* Print statistics at exit. */ 1192 if (atexit(stats_print_atexit) != 0) { 1193 malloc_write("<jemalloc>: Error in atexit()\n"); 1194 if (opt_abort) 1195 abort(); 1196 } 1197 } 1198 if (base_boot()) 1199 return (true); 1200 if (chunk_boot()) 1201 return (true); 1202 if (ctl_boot()) 1203 return (true); 1204 if (config_prof) 1205 prof_boot1(); 1206 if (arena_boot()) 1207 return (true); 1208 if (config_tcache && tcache_boot()) 1209 return (true); 1210 if (malloc_mutex_init(&arenas_lock)) 1211 return (true); 1212 /* 1213 * Create enough scaffolding to allow recursive allocation in 1214 * malloc_ncpus(). 1215 */ 1216 narenas_total = narenas_auto = 1; 1217 arenas = &a0; 1218 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 1219 /* 1220 * Initialize one arena here. The rest are lazily created in 1221 * arena_choose_hard(). 1222 */ 1223 if (arena_init(0) == NULL) 1224 return (true); 1225 malloc_init_state = malloc_init_a0_initialized; 1226 return (false); 1227 } 1228 1229 static bool 1230 malloc_init_hard_a0(void) 1231 { 1232 bool ret; 1233 1234 malloc_mutex_lock(&init_lock); 1235 ret = malloc_init_hard_a0_locked(); 1236 malloc_mutex_unlock(&init_lock); 1237 return (ret); 1238 } 1239 1240 /* 1241 * Initialize data structures which may trigger recursive allocation. 1242 * 1243 * init_lock must be held. 1244 */ 1245 static void 1246 malloc_init_hard_recursible(void) 1247 { 1248 1249 malloc_init_state = malloc_init_recursible; 1250 malloc_mutex_unlock(&init_lock); 1251 1252 ncpus = malloc_ncpus(); 1253 1254 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 1255 && !defined(_WIN32) && !defined(__native_client__)) 1256 /* LinuxThreads's pthread_atfork() allocates. */ 1257 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 1258 jemalloc_postfork_child) != 0) { 1259 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 1260 if (opt_abort) 1261 abort(); 1262 } 1263 #endif 1264 malloc_mutex_lock(&init_lock); 1265 } 1266 1267 /* init_lock must be held. */ 1268 static bool 1269 malloc_init_hard_finish(void) 1270 { 1271 1272 if (mutex_boot()) 1273 return (true); 1274 1275 if (opt_narenas == 0) { 1276 /* 1277 * For SMP systems, create more than one arena per CPU by 1278 * default. 1279 */ 1280 if (ncpus > 1) 1281 opt_narenas = ncpus << 2; 1282 else 1283 opt_narenas = 1; 1284 } 1285 narenas_auto = opt_narenas; 1286 /* 1287 * Make sure that the arenas array can be allocated. In practice, this 1288 * limit is enough to allow the allocator to function, but the ctl 1289 * machinery will fail to allocate memory at far lower limits. 1290 */ 1291 if (narenas_auto > chunksize / sizeof(arena_t *)) { 1292 narenas_auto = chunksize / sizeof(arena_t *); 1293 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 1294 narenas_auto); 1295 } 1296 narenas_total = narenas_auto; 1297 1298 /* Allocate and initialize arenas. */ 1299 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); 1300 if (arenas == NULL) 1301 return (true); 1302 /* 1303 * Zero the array. In practice, this should always be pre-zeroed, 1304 * since it was just mmap()ed, but let's be sure. 1305 */ 1306 memset(arenas, 0, sizeof(arena_t *) * narenas_total); 1307 /* Copy the pointer to the one arena that was already initialized. */ 1308 arenas[0] = a0; 1309 1310 malloc_init_state = malloc_init_initialized; 1311 return (false); 1312 } 1313 1314 static bool 1315 malloc_init_hard(void) 1316 { 1317 1318 #if defined(_WIN32) && _WIN32_WINNT < 0x0600 1319 _init_init_lock(); 1320 #endif 1321 malloc_mutex_lock(&init_lock); 1322 if (!malloc_init_hard_needed()) { 1323 malloc_mutex_unlock(&init_lock); 1324 return (false); 1325 } 1326 1327 if (malloc_init_state != malloc_init_a0_initialized && 1328 malloc_init_hard_a0_locked()) { 1329 malloc_mutex_unlock(&init_lock); 1330 return (true); 1331 } 1332 if (malloc_tsd_boot0()) { 1333 malloc_mutex_unlock(&init_lock); 1334 return (true); 1335 } 1336 if (config_prof && prof_boot2()) { 1337 malloc_mutex_unlock(&init_lock); 1338 return (true); 1339 } 1340 1341 malloc_init_hard_recursible(); 1342 1343 if (malloc_init_hard_finish()) { 1344 malloc_mutex_unlock(&init_lock); 1345 return (true); 1346 } 1347 1348 malloc_mutex_unlock(&init_lock); 1349 malloc_tsd_boot1(); 1350 return (false); 1351 } 1352 1353 /* 1354 * End initialization functions. 1355 */ 1356 /******************************************************************************/ 1357 /* 1358 * Begin malloc(3)-compatible functions. 1359 */ 1360 1361 static void * 1362 imalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 1363 { 1364 void *p; 1365 1366 if (tctx == NULL) 1367 return (NULL); 1368 if (usize <= SMALL_MAXCLASS) { 1369 p = imalloc(tsd, LARGE_MINCLASS); 1370 if (p == NULL) 1371 return (NULL); 1372 arena_prof_promoted(p, usize); 1373 } else 1374 p = imalloc(tsd, usize); 1375 1376 return (p); 1377 } 1378 1379 JEMALLOC_ALWAYS_INLINE_C void * 1380 imalloc_prof(tsd_t *tsd, size_t usize) 1381 { 1382 void *p; 1383 prof_tctx_t *tctx; 1384 1385 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1386 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1387 p = imalloc_prof_sample(tsd, usize, tctx); 1388 else 1389 p = imalloc(tsd, usize); 1390 if (unlikely(p == NULL)) { 1391 prof_alloc_rollback(tsd, tctx, true); 1392 return (NULL); 1393 } 1394 prof_malloc(p, usize, tctx); 1395 1396 return (p); 1397 } 1398 1399 JEMALLOC_ALWAYS_INLINE_C void * 1400 imalloc_body(size_t size, tsd_t **tsd, size_t *usize) 1401 { 1402 1403 if (unlikely(malloc_init())) 1404 return (NULL); 1405 *tsd = tsd_fetch(); 1406 1407 if (config_prof && opt_prof) { 1408 *usize = s2u(size); 1409 if (unlikely(*usize == 0)) 1410 return (NULL); 1411 return (imalloc_prof(*tsd, *usize)); 1412 } 1413 1414 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 1415 *usize = s2u(size); 1416 return (imalloc(*tsd, size)); 1417 } 1418 1419 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1420 void JEMALLOC_NOTHROW * 1421 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 1422 je_malloc(size_t size) 1423 { 1424 void *ret; 1425 tsd_t *tsd; 1426 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1427 1428 if (size == 0) 1429 size = 1; 1430 1431 ret = imalloc_body(size, &tsd, &usize); 1432 if (unlikely(ret == NULL)) { 1433 if (config_xmalloc && unlikely(opt_xmalloc)) { 1434 malloc_write("<jemalloc>: Error in malloc(): " 1435 "out of memory\n"); 1436 abort(); 1437 } 1438 set_errno(ENOMEM); 1439 } 1440 if (config_stats && likely(ret != NULL)) { 1441 assert(usize == isalloc(ret, config_prof)); 1442 *tsd_thread_allocatedp_get(tsd) += usize; 1443 } 1444 UTRACE(0, size, ret); 1445 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 1446 return (ret); 1447 } 1448 1449 static void * 1450 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize, 1451 prof_tctx_t *tctx) 1452 { 1453 void *p; 1454 1455 if (tctx == NULL) 1456 return (NULL); 1457 if (usize <= SMALL_MAXCLASS) { 1458 assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS); 1459 p = ipalloc(tsd, LARGE_MINCLASS, alignment, false); 1460 if (p == NULL) 1461 return (NULL); 1462 arena_prof_promoted(p, usize); 1463 } else 1464 p = ipalloc(tsd, usize, alignment, false); 1465 1466 return (p); 1467 } 1468 1469 JEMALLOC_ALWAYS_INLINE_C void * 1470 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) 1471 { 1472 void *p; 1473 prof_tctx_t *tctx; 1474 1475 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1476 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1477 p = imemalign_prof_sample(tsd, alignment, usize, tctx); 1478 else 1479 p = ipalloc(tsd, usize, alignment, false); 1480 if (unlikely(p == NULL)) { 1481 prof_alloc_rollback(tsd, tctx, true); 1482 return (NULL); 1483 } 1484 prof_malloc(p, usize, tctx); 1485 1486 return (p); 1487 } 1488 1489 JEMALLOC_ATTR(nonnull(1)) 1490 static int 1491 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 1492 { 1493 int ret; 1494 tsd_t *tsd; 1495 size_t usize; 1496 void *result; 1497 1498 assert(min_alignment != 0); 1499 1500 if (unlikely(malloc_init())) { 1501 result = NULL; 1502 goto label_oom; 1503 } 1504 tsd = tsd_fetch(); 1505 if (size == 0) 1506 size = 1; 1507 1508 /* Make sure that alignment is a large enough power of 2. */ 1509 if (unlikely(((alignment - 1) & alignment) != 0 1510 || (alignment < min_alignment))) { 1511 if (config_xmalloc && unlikely(opt_xmalloc)) { 1512 malloc_write("<jemalloc>: Error allocating " 1513 "aligned memory: invalid alignment\n"); 1514 abort(); 1515 } 1516 result = NULL; 1517 ret = EINVAL; 1518 goto label_return; 1519 } 1520 1521 usize = sa2u(size, alignment); 1522 if (unlikely(usize == 0)) { 1523 result = NULL; 1524 goto label_oom; 1525 } 1526 1527 if (config_prof && opt_prof) 1528 result = imemalign_prof(tsd, alignment, usize); 1529 else 1530 result = ipalloc(tsd, usize, alignment, false); 1531 if (unlikely(result == NULL)) 1532 goto label_oom; 1533 assert(((uintptr_t)result & (alignment - 1)) == ZU(0)); 1534 1535 *memptr = result; 1536 ret = 0; 1537 label_return: 1538 if (config_stats && likely(result != NULL)) { 1539 assert(usize == isalloc(result, config_prof)); 1540 *tsd_thread_allocatedp_get(tsd) += usize; 1541 } 1542 UTRACE(0, size, result); 1543 return (ret); 1544 label_oom: 1545 assert(result == NULL); 1546 if (config_xmalloc && unlikely(opt_xmalloc)) { 1547 malloc_write("<jemalloc>: Error allocating aligned memory: " 1548 "out of memory\n"); 1549 abort(); 1550 } 1551 ret = ENOMEM; 1552 goto label_return; 1553 } 1554 1555 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 1556 JEMALLOC_ATTR(nonnull(1)) 1557 je_posix_memalign(void **memptr, size_t alignment, size_t size) 1558 { 1559 int ret = imemalign(memptr, alignment, size, sizeof(void *)); 1560 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 1561 config_prof), false); 1562 return (ret); 1563 } 1564 1565 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1566 void JEMALLOC_NOTHROW * 1567 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2) 1568 je_aligned_alloc(size_t alignment, size_t size) 1569 { 1570 void *ret; 1571 int err; 1572 1573 if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) { 1574 ret = NULL; 1575 set_errno(err); 1576 } 1577 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 1578 false); 1579 return (ret); 1580 } 1581 1582 static void * 1583 icalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 1584 { 1585 void *p; 1586 1587 if (tctx == NULL) 1588 return (NULL); 1589 if (usize <= SMALL_MAXCLASS) { 1590 p = icalloc(tsd, LARGE_MINCLASS); 1591 if (p == NULL) 1592 return (NULL); 1593 arena_prof_promoted(p, usize); 1594 } else 1595 p = icalloc(tsd, usize); 1596 1597 return (p); 1598 } 1599 1600 JEMALLOC_ALWAYS_INLINE_C void * 1601 icalloc_prof(tsd_t *tsd, size_t usize) 1602 { 1603 void *p; 1604 prof_tctx_t *tctx; 1605 1606 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1607 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1608 p = icalloc_prof_sample(tsd, usize, tctx); 1609 else 1610 p = icalloc(tsd, usize); 1611 if (unlikely(p == NULL)) { 1612 prof_alloc_rollback(tsd, tctx, true); 1613 return (NULL); 1614 } 1615 prof_malloc(p, usize, tctx); 1616 1617 return (p); 1618 } 1619 1620 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1621 void JEMALLOC_NOTHROW * 1622 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2) 1623 je_calloc(size_t num, size_t size) 1624 { 1625 void *ret; 1626 tsd_t *tsd; 1627 size_t num_size; 1628 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1629 1630 if (unlikely(malloc_init())) { 1631 num_size = 0; 1632 ret = NULL; 1633 goto label_return; 1634 } 1635 tsd = tsd_fetch(); 1636 1637 num_size = num * size; 1638 if (unlikely(num_size == 0)) { 1639 if (num == 0 || size == 0) 1640 num_size = 1; 1641 else { 1642 ret = NULL; 1643 goto label_return; 1644 } 1645 /* 1646 * Try to avoid division here. We know that it isn't possible to 1647 * overflow during multiplication if neither operand uses any of the 1648 * most significant half of the bits in a size_t. 1649 */ 1650 } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 1651 2))) && (num_size / size != num))) { 1652 /* size_t overflow. */ 1653 ret = NULL; 1654 goto label_return; 1655 } 1656 1657 if (config_prof && opt_prof) { 1658 usize = s2u(num_size); 1659 if (unlikely(usize == 0)) { 1660 ret = NULL; 1661 goto label_return; 1662 } 1663 ret = icalloc_prof(tsd, usize); 1664 } else { 1665 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 1666 usize = s2u(num_size); 1667 ret = icalloc(tsd, num_size); 1668 } 1669 1670 label_return: 1671 if (unlikely(ret == NULL)) { 1672 if (config_xmalloc && unlikely(opt_xmalloc)) { 1673 malloc_write("<jemalloc>: Error in calloc(): out of " 1674 "memory\n"); 1675 abort(); 1676 } 1677 set_errno(ENOMEM); 1678 } 1679 if (config_stats && likely(ret != NULL)) { 1680 assert(usize == isalloc(ret, config_prof)); 1681 *tsd_thread_allocatedp_get(tsd) += usize; 1682 } 1683 UTRACE(0, num_size, ret); 1684 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1685 return (ret); 1686 } 1687 1688 static void * 1689 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, 1690 prof_tctx_t *tctx) 1691 { 1692 void *p; 1693 1694 if (tctx == NULL) 1695 return (NULL); 1696 if (usize <= SMALL_MAXCLASS) { 1697 p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); 1698 if (p == NULL) 1699 return (NULL); 1700 arena_prof_promoted(p, usize); 1701 } else 1702 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 1703 1704 return (p); 1705 } 1706 1707 JEMALLOC_ALWAYS_INLINE_C void * 1708 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize) 1709 { 1710 void *p; 1711 bool prof_active; 1712 prof_tctx_t *old_tctx, *tctx; 1713 1714 prof_active = prof_active_get_unlocked(); 1715 old_tctx = prof_tctx_get(old_ptr); 1716 tctx = prof_alloc_prep(tsd, usize, prof_active, true); 1717 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1718 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); 1719 else 1720 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 1721 if (unlikely(p == NULL)) { 1722 prof_alloc_rollback(tsd, tctx, true); 1723 return (NULL); 1724 } 1725 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize, 1726 old_tctx); 1727 1728 return (p); 1729 } 1730 1731 JEMALLOC_INLINE_C void 1732 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache) 1733 { 1734 size_t usize; 1735 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1736 1737 assert(ptr != NULL); 1738 assert(malloc_initialized() || IS_INITIALIZER); 1739 1740 if (config_prof && opt_prof) { 1741 usize = isalloc(ptr, config_prof); 1742 prof_free(tsd, ptr, usize); 1743 } else if (config_stats || config_valgrind) 1744 usize = isalloc(ptr, config_prof); 1745 if (config_stats) 1746 *tsd_thread_deallocatedp_get(tsd) += usize; 1747 if (config_valgrind && unlikely(in_valgrind)) 1748 rzsize = p2rz(ptr); 1749 iqalloc(tsd, ptr, tcache); 1750 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1751 } 1752 1753 JEMALLOC_INLINE_C void 1754 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache) 1755 { 1756 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1757 1758 assert(ptr != NULL); 1759 assert(malloc_initialized() || IS_INITIALIZER); 1760 1761 if (config_prof && opt_prof) 1762 prof_free(tsd, ptr, usize); 1763 if (config_stats) 1764 *tsd_thread_deallocatedp_get(tsd) += usize; 1765 if (config_valgrind && unlikely(in_valgrind)) 1766 rzsize = p2rz(ptr); 1767 isqalloc(tsd, ptr, usize, tcache); 1768 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1769 } 1770 1771 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1772 void JEMALLOC_NOTHROW * 1773 JEMALLOC_ALLOC_SIZE(2) 1774 je_realloc(void *ptr, size_t size) 1775 { 1776 void *ret; 1777 tsd_t *tsd JEMALLOC_CC_SILENCE_INIT(NULL); 1778 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1779 size_t old_usize = 0; 1780 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1781 1782 if (unlikely(size == 0)) { 1783 if (ptr != NULL) { 1784 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1785 UTRACE(ptr, 0, 0); 1786 tsd = tsd_fetch(); 1787 ifree(tsd, ptr, tcache_get(tsd, false)); 1788 return (NULL); 1789 } 1790 size = 1; 1791 } 1792 1793 if (likely(ptr != NULL)) { 1794 assert(malloc_initialized() || IS_INITIALIZER); 1795 malloc_thread_init(); 1796 tsd = tsd_fetch(); 1797 1798 old_usize = isalloc(ptr, config_prof); 1799 if (config_valgrind && unlikely(in_valgrind)) 1800 old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); 1801 1802 if (config_prof && opt_prof) { 1803 usize = s2u(size); 1804 ret = unlikely(usize == 0) ? NULL : irealloc_prof(tsd, 1805 ptr, old_usize, usize); 1806 } else { 1807 if (config_stats || (config_valgrind && 1808 unlikely(in_valgrind))) 1809 usize = s2u(size); 1810 ret = iralloc(tsd, ptr, old_usize, size, 0, false); 1811 } 1812 } else { 1813 /* realloc(NULL, size) is equivalent to malloc(size). */ 1814 ret = imalloc_body(size, &tsd, &usize); 1815 } 1816 1817 if (unlikely(ret == NULL)) { 1818 if (config_xmalloc && unlikely(opt_xmalloc)) { 1819 malloc_write("<jemalloc>: Error in realloc(): " 1820 "out of memory\n"); 1821 abort(); 1822 } 1823 set_errno(ENOMEM); 1824 } 1825 if (config_stats && likely(ret != NULL)) { 1826 assert(usize == isalloc(ret, config_prof)); 1827 *tsd_thread_allocatedp_get(tsd) += usize; 1828 *tsd_thread_deallocatedp_get(tsd) += old_usize; 1829 } 1830 UTRACE(ptr, size, ret); 1831 JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize, 1832 old_rzsize, true, false); 1833 return (ret); 1834 } 1835 1836 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 1837 je_free(void *ptr) 1838 { 1839 1840 UTRACE(ptr, 0, 0); 1841 if (likely(ptr != NULL)) { 1842 tsd_t *tsd = tsd_fetch(); 1843 ifree(tsd, ptr, tcache_get(tsd, false)); 1844 } 1845 } 1846 1847 /* 1848 * End malloc(3)-compatible functions. 1849 */ 1850 /******************************************************************************/ 1851 /* 1852 * Begin non-standard override functions. 1853 */ 1854 1855 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 1856 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1857 void JEMALLOC_NOTHROW * 1858 JEMALLOC_ATTR(malloc) 1859 je_memalign(size_t alignment, size_t size) 1860 { 1861 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1862 if (unlikely(imemalign(&ret, alignment, size, 1) != 0)) 1863 ret = NULL; 1864 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1865 return (ret); 1866 } 1867 #endif 1868 1869 #ifdef JEMALLOC_OVERRIDE_VALLOC 1870 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1871 void JEMALLOC_NOTHROW * 1872 JEMALLOC_ATTR(malloc) 1873 je_valloc(size_t size) 1874 { 1875 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1876 if (unlikely(imemalign(&ret, PAGE, size, 1) != 0)) 1877 ret = NULL; 1878 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1879 return (ret); 1880 } 1881 #endif 1882 1883 /* 1884 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1885 * #define je_malloc malloc 1886 */ 1887 #define malloc_is_malloc 1 1888 #define is_malloc_(a) malloc_is_ ## a 1889 #define is_malloc(a) is_malloc_(a) 1890 1891 #if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)) 1892 /* 1893 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1894 * to inconsistently reference libc's malloc(3)-compatible functions 1895 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1896 * 1897 * These definitions interpose hooks in glibc. The functions are actually 1898 * passed an extra argument for the caller return address, which will be 1899 * ignored. 1900 */ 1901 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 1902 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 1903 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 1904 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK 1905 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 1906 je_memalign; 1907 # endif 1908 #endif 1909 1910 /* 1911 * End non-standard override functions. 1912 */ 1913 /******************************************************************************/ 1914 /* 1915 * Begin non-standard functions. 1916 */ 1917 1918 JEMALLOC_ALWAYS_INLINE_C bool 1919 imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize, 1920 size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) 1921 { 1922 1923 if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) { 1924 *alignment = 0; 1925 *usize = s2u(size); 1926 } else { 1927 *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 1928 *usize = sa2u(size, *alignment); 1929 } 1930 assert(*usize != 0); 1931 *zero = MALLOCX_ZERO_GET(flags); 1932 if ((flags & MALLOCX_TCACHE_MASK) != 0) { 1933 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 1934 *tcache = NULL; 1935 else 1936 *tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 1937 } else 1938 *tcache = tcache_get(tsd, true); 1939 if ((flags & MALLOCX_ARENA_MASK) != 0) { 1940 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 1941 *arena = arena_get(tsd, arena_ind, true, true); 1942 if (unlikely(*arena == NULL)) 1943 return (true); 1944 } else 1945 *arena = NULL; 1946 return (false); 1947 } 1948 1949 JEMALLOC_ALWAYS_INLINE_C bool 1950 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, 1951 size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) 1952 { 1953 1954 if (likely(flags == 0)) { 1955 *usize = s2u(size); 1956 assert(*usize != 0); 1957 *alignment = 0; 1958 *zero = false; 1959 *tcache = tcache_get(tsd, true); 1960 *arena = NULL; 1961 return (false); 1962 } else { 1963 return (imallocx_flags_decode_hard(tsd, size, flags, usize, 1964 alignment, zero, tcache, arena)); 1965 } 1966 } 1967 1968 JEMALLOC_ALWAYS_INLINE_C void * 1969 imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero, 1970 tcache_t *tcache, arena_t *arena) 1971 { 1972 1973 if (unlikely(alignment != 0)) 1974 return (ipalloct(tsd, usize, alignment, zero, tcache, arena)); 1975 if (unlikely(zero)) 1976 return (icalloct(tsd, usize, tcache, arena)); 1977 return (imalloct(tsd, usize, tcache, arena)); 1978 } 1979 1980 static void * 1981 imallocx_prof_sample(tsd_t *tsd, size_t usize, size_t alignment, bool zero, 1982 tcache_t *tcache, arena_t *arena) 1983 { 1984 void *p; 1985 1986 if (usize <= SMALL_MAXCLASS) { 1987 assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : 1988 sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); 1989 p = imallocx_flags(tsd, LARGE_MINCLASS, alignment, zero, tcache, 1990 arena); 1991 if (p == NULL) 1992 return (NULL); 1993 arena_prof_promoted(p, usize); 1994 } else 1995 p = imallocx_flags(tsd, usize, alignment, zero, tcache, arena); 1996 1997 return (p); 1998 } 1999 2000 JEMALLOC_ALWAYS_INLINE_C void * 2001 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) 2002 { 2003 void *p; 2004 size_t alignment; 2005 bool zero; 2006 tcache_t *tcache; 2007 arena_t *arena; 2008 prof_tctx_t *tctx; 2009 2010 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 2011 &zero, &tcache, &arena))) 2012 return (NULL); 2013 tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true); 2014 if (likely((uintptr_t)tctx == (uintptr_t)1U)) 2015 p = imallocx_flags(tsd, *usize, alignment, zero, tcache, arena); 2016 else if ((uintptr_t)tctx > (uintptr_t)1U) { 2017 p = imallocx_prof_sample(tsd, *usize, alignment, zero, tcache, 2018 arena); 2019 } else 2020 p = NULL; 2021 if (unlikely(p == NULL)) { 2022 prof_alloc_rollback(tsd, tctx, true); 2023 return (NULL); 2024 } 2025 prof_malloc(p, *usize, tctx); 2026 2027 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2028 return (p); 2029 } 2030 2031 JEMALLOC_ALWAYS_INLINE_C void * 2032 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) 2033 { 2034 void *p; 2035 size_t alignment; 2036 bool zero; 2037 tcache_t *tcache; 2038 arena_t *arena; 2039 2040 if (likely(flags == 0)) { 2041 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 2042 *usize = s2u(size); 2043 return (imalloc(tsd, size)); 2044 } 2045 2046 if (unlikely(imallocx_flags_decode_hard(tsd, size, flags, usize, 2047 &alignment, &zero, &tcache, &arena))) 2048 return (NULL); 2049 p = imallocx_flags(tsd, *usize, alignment, zero, tcache, arena); 2050 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2051 return (p); 2052 } 2053 2054 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2055 void JEMALLOC_NOTHROW * 2056 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 2057 je_mallocx(size_t size, int flags) 2058 { 2059 tsd_t *tsd; 2060 void *p; 2061 size_t usize; 2062 2063 assert(size != 0); 2064 2065 if (unlikely(malloc_init())) 2066 goto label_oom; 2067 tsd = tsd_fetch(); 2068 2069 if (config_prof && opt_prof) 2070 p = imallocx_prof(tsd, size, flags, &usize); 2071 else 2072 p = imallocx_no_prof(tsd, size, flags, &usize); 2073 if (unlikely(p == NULL)) 2074 goto label_oom; 2075 2076 if (config_stats) { 2077 assert(usize == isalloc(p, config_prof)); 2078 *tsd_thread_allocatedp_get(tsd) += usize; 2079 } 2080 UTRACE(0, size, p); 2081 JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags)); 2082 return (p); 2083 label_oom: 2084 if (config_xmalloc && unlikely(opt_xmalloc)) { 2085 malloc_write("<jemalloc>: Error in mallocx(): out of memory\n"); 2086 abort(); 2087 } 2088 UTRACE(0, size, 0); 2089 return (NULL); 2090 } 2091 2092 static void * 2093 irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, 2094 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, 2095 prof_tctx_t *tctx) 2096 { 2097 void *p; 2098 2099 if (tctx == NULL) 2100 return (NULL); 2101 if (usize <= SMALL_MAXCLASS) { 2102 p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment, 2103 zero, tcache, arena); 2104 if (p == NULL) 2105 return (NULL); 2106 arena_prof_promoted(p, usize); 2107 } else { 2108 p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero, 2109 tcache, arena); 2110 } 2111 2112 return (p); 2113 } 2114 2115 JEMALLOC_ALWAYS_INLINE_C void * 2116 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, 2117 size_t alignment, size_t *usize, bool zero, tcache_t *tcache, 2118 arena_t *arena) 2119 { 2120 void *p; 2121 bool prof_active; 2122 prof_tctx_t *old_tctx, *tctx; 2123 2124 prof_active = prof_active_get_unlocked(); 2125 old_tctx = prof_tctx_get(old_ptr); 2126 tctx = prof_alloc_prep(tsd, *usize, prof_active, true); 2127 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2128 p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize, 2129 alignment, zero, tcache, arena, tctx); 2130 } else { 2131 p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero, 2132 tcache, arena); 2133 } 2134 if (unlikely(p == NULL)) { 2135 prof_alloc_rollback(tsd, tctx, true); 2136 return (NULL); 2137 } 2138 2139 if (p == old_ptr && alignment != 0) { 2140 /* 2141 * The allocation did not move, so it is possible that the size 2142 * class is smaller than would guarantee the requested 2143 * alignment, and that the alignment constraint was 2144 * serendipitously satisfied. Additionally, old_usize may not 2145 * be the same as the current usize because of in-place large 2146 * reallocation. Therefore, query the actual value of usize. 2147 */ 2148 *usize = isalloc(p, config_prof); 2149 } 2150 prof_realloc(tsd, p, *usize, tctx, prof_active, true, old_ptr, 2151 old_usize, old_tctx); 2152 2153 return (p); 2154 } 2155 2156 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2157 void JEMALLOC_NOTHROW * 2158 JEMALLOC_ALLOC_SIZE(2) 2159 je_rallocx(void *ptr, size_t size, int flags) 2160 { 2161 void *p; 2162 tsd_t *tsd; 2163 size_t usize; 2164 size_t old_usize; 2165 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2166 size_t alignment = MALLOCX_ALIGN_GET(flags); 2167 bool zero = flags & MALLOCX_ZERO; 2168 arena_t *arena; 2169 tcache_t *tcache; 2170 2171 assert(ptr != NULL); 2172 assert(size != 0); 2173 assert(malloc_initialized() || IS_INITIALIZER); 2174 malloc_thread_init(); 2175 tsd = tsd_fetch(); 2176 2177 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2178 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2179 arena = arena_get(tsd, arena_ind, true, true); 2180 if (unlikely(arena == NULL)) 2181 goto label_oom; 2182 } else 2183 arena = NULL; 2184 2185 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2186 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2187 tcache = NULL; 2188 else 2189 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2190 } else 2191 tcache = tcache_get(tsd, true); 2192 2193 old_usize = isalloc(ptr, config_prof); 2194 if (config_valgrind && unlikely(in_valgrind)) 2195 old_rzsize = u2rz(old_usize); 2196 2197 if (config_prof && opt_prof) { 2198 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 2199 assert(usize != 0); 2200 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, 2201 zero, tcache, arena); 2202 if (unlikely(p == NULL)) 2203 goto label_oom; 2204 } else { 2205 p = iralloct(tsd, ptr, old_usize, size, alignment, zero, 2206 tcache, arena); 2207 if (unlikely(p == NULL)) 2208 goto label_oom; 2209 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 2210 usize = isalloc(p, config_prof); 2211 } 2212 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2213 2214 if (config_stats) { 2215 *tsd_thread_allocatedp_get(tsd) += usize; 2216 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2217 } 2218 UTRACE(ptr, size, p); 2219 JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize, 2220 old_rzsize, false, zero); 2221 return (p); 2222 label_oom: 2223 if (config_xmalloc && unlikely(opt_xmalloc)) { 2224 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 2225 abort(); 2226 } 2227 UTRACE(ptr, size, 0); 2228 return (NULL); 2229 } 2230 2231 JEMALLOC_ALWAYS_INLINE_C size_t 2232 ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, 2233 size_t alignment, bool zero) 2234 { 2235 size_t usize; 2236 2237 if (ixalloc(ptr, old_usize, size, extra, alignment, zero)) 2238 return (old_usize); 2239 usize = isalloc(ptr, config_prof); 2240 2241 return (usize); 2242 } 2243 2244 static size_t 2245 ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, 2246 size_t alignment, bool zero, prof_tctx_t *tctx) 2247 { 2248 size_t usize; 2249 2250 if (tctx == NULL) 2251 return (old_usize); 2252 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero); 2253 2254 return (usize); 2255 } 2256 2257 JEMALLOC_ALWAYS_INLINE_C size_t 2258 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, 2259 size_t extra, size_t alignment, bool zero) 2260 { 2261 size_t usize_max, usize; 2262 bool prof_active; 2263 prof_tctx_t *old_tctx, *tctx; 2264 2265 prof_active = prof_active_get_unlocked(); 2266 old_tctx = prof_tctx_get(ptr); 2267 /* 2268 * usize isn't knowable before ixalloc() returns when extra is non-zero. 2269 * Therefore, compute its maximum possible value and use that in 2270 * prof_alloc_prep() to decide whether to capture a backtrace. 2271 * prof_realloc() will use the actual usize to decide whether to sample. 2272 */ 2273 usize_max = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, 2274 alignment); 2275 assert(usize_max != 0); 2276 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); 2277 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2278 usize = ixallocx_prof_sample(ptr, old_usize, size, extra, 2279 alignment, zero, tctx); 2280 } else { 2281 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 2282 zero); 2283 } 2284 if (usize == old_usize) { 2285 prof_alloc_rollback(tsd, tctx, false); 2286 return (usize); 2287 } 2288 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize, 2289 old_tctx); 2290 2291 return (usize); 2292 } 2293 2294 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2295 je_xallocx(void *ptr, size_t size, size_t extra, int flags) 2296 { 2297 tsd_t *tsd; 2298 size_t usize, old_usize; 2299 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2300 size_t alignment = MALLOCX_ALIGN_GET(flags); 2301 bool zero = flags & MALLOCX_ZERO; 2302 2303 assert(ptr != NULL); 2304 assert(size != 0); 2305 assert(SIZE_T_MAX - size >= extra); 2306 assert(malloc_initialized() || IS_INITIALIZER); 2307 malloc_thread_init(); 2308 tsd = tsd_fetch(); 2309 2310 old_usize = isalloc(ptr, config_prof); 2311 2312 /* Clamp extra if necessary to avoid (size + extra) overflow. */ 2313 if (unlikely(size + extra > HUGE_MAXCLASS)) { 2314 /* Check for size overflow. */ 2315 if (unlikely(size > HUGE_MAXCLASS)) { 2316 usize = old_usize; 2317 goto label_not_resized; 2318 } 2319 extra = HUGE_MAXCLASS - size; 2320 } 2321 2322 if (config_valgrind && unlikely(in_valgrind)) 2323 old_rzsize = u2rz(old_usize); 2324 2325 if (config_prof && opt_prof) { 2326 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, 2327 alignment, zero); 2328 } else { 2329 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 2330 zero); 2331 } 2332 if (unlikely(usize == old_usize)) 2333 goto label_not_resized; 2334 2335 if (config_stats) { 2336 *tsd_thread_allocatedp_get(tsd) += usize; 2337 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2338 } 2339 JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize, 2340 old_rzsize, false, zero); 2341 label_not_resized: 2342 UTRACE(ptr, size, ptr); 2343 return (usize); 2344 } 2345 2346 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2347 JEMALLOC_ATTR(pure) 2348 je_sallocx(const void *ptr, int flags) 2349 { 2350 size_t usize; 2351 2352 assert(malloc_initialized() || IS_INITIALIZER); 2353 malloc_thread_init(); 2354 2355 if (config_ivsalloc) 2356 usize = ivsalloc(ptr, config_prof); 2357 else 2358 usize = isalloc(ptr, config_prof); 2359 2360 return (usize); 2361 } 2362 2363 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2364 je_dallocx(void *ptr, int flags) 2365 { 2366 tsd_t *tsd; 2367 tcache_t *tcache; 2368 2369 assert(ptr != NULL); 2370 assert(malloc_initialized() || IS_INITIALIZER); 2371 2372 tsd = tsd_fetch(); 2373 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2374 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2375 tcache = NULL; 2376 else 2377 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2378 } else 2379 tcache = tcache_get(tsd, false); 2380 2381 UTRACE(ptr, 0, 0); 2382 ifree(tsd_fetch(), ptr, tcache); 2383 } 2384 2385 JEMALLOC_ALWAYS_INLINE_C size_t 2386 inallocx(size_t size, int flags) 2387 { 2388 size_t usize; 2389 2390 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) 2391 usize = s2u(size); 2392 else 2393 usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); 2394 assert(usize != 0); 2395 return (usize); 2396 } 2397 2398 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2399 je_sdallocx(void *ptr, size_t size, int flags) 2400 { 2401 tsd_t *tsd; 2402 tcache_t *tcache; 2403 size_t usize; 2404 2405 assert(ptr != NULL); 2406 assert(malloc_initialized() || IS_INITIALIZER); 2407 usize = inallocx(size, flags); 2408 assert(usize == isalloc(ptr, config_prof)); 2409 2410 tsd = tsd_fetch(); 2411 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2412 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2413 tcache = NULL; 2414 else 2415 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2416 } else 2417 tcache = tcache_get(tsd, false); 2418 2419 UTRACE(ptr, 0, 0); 2420 isfree(tsd, ptr, usize, tcache); 2421 } 2422 2423 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2424 JEMALLOC_ATTR(pure) 2425 je_nallocx(size_t size, int flags) 2426 { 2427 2428 assert(size != 0); 2429 2430 if (unlikely(malloc_init())) 2431 return (0); 2432 2433 return (inallocx(size, flags)); 2434 } 2435 2436 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2437 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 2438 size_t newlen) 2439 { 2440 2441 if (unlikely(malloc_init())) 2442 return (EAGAIN); 2443 2444 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 2445 } 2446 2447 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2448 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 2449 { 2450 2451 if (unlikely(malloc_init())) 2452 return (EAGAIN); 2453 2454 return (ctl_nametomib(name, mibp, miblenp)); 2455 } 2456 2457 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2458 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 2459 void *newp, size_t newlen) 2460 { 2461 2462 if (unlikely(malloc_init())) 2463 return (EAGAIN); 2464 2465 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 2466 } 2467 2468 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2469 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 2470 const char *opts) 2471 { 2472 2473 stats_print(write_cb, cbopaque, opts); 2474 } 2475 2476 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2477 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 2478 { 2479 size_t ret; 2480 2481 assert(malloc_initialized() || IS_INITIALIZER); 2482 malloc_thread_init(); 2483 2484 if (config_ivsalloc) 2485 ret = ivsalloc(ptr, config_prof); 2486 else 2487 ret = (ptr == NULL) ? 0 : isalloc(ptr, config_prof); 2488 2489 return (ret); 2490 } 2491 2492 /* 2493 * End non-standard functions. 2494 */ 2495 /******************************************************************************/ 2496 /* 2497 * Begin compatibility functions. 2498 */ 2499 2500 #define ALLOCM_LG_ALIGN(la) (la) 2501 #define ALLOCM_ALIGN(a) (ffsl(a)-1) 2502 #define ALLOCM_ZERO ((int)0x40) 2503 #define ALLOCM_NO_MOVE ((int)0x80) 2504 2505 #define ALLOCM_SUCCESS 0 2506 #define ALLOCM_ERR_OOM 1 2507 #define ALLOCM_ERR_NOT_MOVED 2 2508 2509 int 2510 je_allocm(void **ptr, size_t *rsize, size_t size, int flags) 2511 { 2512 void *p; 2513 2514 assert(ptr != NULL); 2515 2516 p = je_mallocx(size, flags); 2517 if (p == NULL) 2518 return (ALLOCM_ERR_OOM); 2519 if (rsize != NULL) 2520 *rsize = isalloc(p, config_prof); 2521 *ptr = p; 2522 return (ALLOCM_SUCCESS); 2523 } 2524 2525 int 2526 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) 2527 { 2528 int ret; 2529 bool no_move = flags & ALLOCM_NO_MOVE; 2530 2531 assert(ptr != NULL); 2532 assert(*ptr != NULL); 2533 assert(size != 0); 2534 assert(SIZE_T_MAX - size >= extra); 2535 2536 if (no_move) { 2537 size_t usize = je_xallocx(*ptr, size, extra, flags); 2538 ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; 2539 if (rsize != NULL) 2540 *rsize = usize; 2541 } else { 2542 void *p = je_rallocx(*ptr, size+extra, flags); 2543 if (p != NULL) { 2544 *ptr = p; 2545 ret = ALLOCM_SUCCESS; 2546 } else 2547 ret = ALLOCM_ERR_OOM; 2548 if (rsize != NULL) 2549 *rsize = isalloc(*ptr, config_prof); 2550 } 2551 return (ret); 2552 } 2553 2554 int 2555 je_sallocm(const void *ptr, size_t *rsize, int flags) 2556 { 2557 2558 assert(rsize != NULL); 2559 *rsize = je_sallocx(ptr, flags); 2560 return (ALLOCM_SUCCESS); 2561 } 2562 2563 int 2564 je_dallocm(void *ptr, int flags) 2565 { 2566 2567 je_dallocx(ptr, flags); 2568 return (ALLOCM_SUCCESS); 2569 } 2570 2571 int 2572 je_nallocm(size_t *rsize, size_t size, int flags) 2573 { 2574 size_t usize; 2575 2576 usize = je_nallocx(size, flags); 2577 if (usize == 0) 2578 return (ALLOCM_ERR_OOM); 2579 if (rsize != NULL) 2580 *rsize = usize; 2581 return (ALLOCM_SUCCESS); 2582 } 2583 2584 #undef ALLOCM_LG_ALIGN 2585 #undef ALLOCM_ALIGN 2586 #undef ALLOCM_ZERO 2587 #undef ALLOCM_NO_MOVE 2588 2589 #undef ALLOCM_SUCCESS 2590 #undef ALLOCM_ERR_OOM 2591 #undef ALLOCM_ERR_NOT_MOVED 2592 2593 /* 2594 * End compatibility functions. 2595 */ 2596 /******************************************************************************/ 2597 /* 2598 * The following functions are used by threading libraries for protection of 2599 * malloc during fork(). 2600 */ 2601 2602 /* 2603 * If an application creates a thread before doing any allocation in the main 2604 * thread, then calls fork(2) in the main thread followed by memory allocation 2605 * in the child process, a race can occur that results in deadlock within the 2606 * child: the main thread may have forked while the created thread had 2607 * partially initialized the allocator. Ordinarily jemalloc prevents 2608 * fork/malloc races via the following functions it registers during 2609 * initialization using pthread_atfork(), but of course that does no good if 2610 * the allocator isn't fully initialized at fork time. The following library 2611 * constructor is a partial solution to this problem. It may still be possible 2612 * to trigger the deadlock described above, but doing so would involve forking 2613 * via a library constructor that runs before jemalloc's runs. 2614 */ 2615 JEMALLOC_ATTR(constructor) 2616 static void 2617 jemalloc_constructor(void) 2618 { 2619 2620 malloc_init(); 2621 } 2622 2623 #ifndef JEMALLOC_MUTEX_INIT_CB 2624 void 2625 jemalloc_prefork(void) 2626 #else 2627 JEMALLOC_EXPORT void 2628 _malloc_prefork(void) 2629 #endif 2630 { 2631 unsigned i; 2632 2633 #ifdef JEMALLOC_MUTEX_INIT_CB 2634 if (!malloc_initialized()) 2635 return; 2636 #endif 2637 assert(malloc_initialized()); 2638 2639 /* Acquire all mutexes in a safe order. */ 2640 ctl_prefork(); 2641 prof_prefork(); 2642 malloc_mutex_prefork(&arenas_lock); 2643 for (i = 0; i < narenas_total; i++) { 2644 if (arenas[i] != NULL) 2645 arena_prefork(arenas[i]); 2646 } 2647 chunk_prefork(); 2648 base_prefork(); 2649 } 2650 2651 #ifndef JEMALLOC_MUTEX_INIT_CB 2652 void 2653 jemalloc_postfork_parent(void) 2654 #else 2655 JEMALLOC_EXPORT void 2656 _malloc_postfork(void) 2657 #endif 2658 { 2659 unsigned i; 2660 2661 #ifdef JEMALLOC_MUTEX_INIT_CB 2662 if (!malloc_initialized()) 2663 return; 2664 #endif 2665 assert(malloc_initialized()); 2666 2667 /* Release all mutexes, now that fork() has completed. */ 2668 base_postfork_parent(); 2669 chunk_postfork_parent(); 2670 for (i = 0; i < narenas_total; i++) { 2671 if (arenas[i] != NULL) 2672 arena_postfork_parent(arenas[i]); 2673 } 2674 malloc_mutex_postfork_parent(&arenas_lock); 2675 prof_postfork_parent(); 2676 ctl_postfork_parent(); 2677 } 2678 2679 void 2680 jemalloc_postfork_child(void) 2681 { 2682 unsigned i; 2683 2684 assert(malloc_initialized()); 2685 2686 /* Release all mutexes, now that fork() has completed. */ 2687 base_postfork_child(); 2688 chunk_postfork_child(); 2689 for (i = 0; i < narenas_total; i++) { 2690 if (arenas[i] != NULL) 2691 arena_postfork_child(arenas[i]); 2692 } 2693 malloc_mutex_postfork_child(&arenas_lock); 2694 prof_postfork_child(); 2695 ctl_postfork_child(); 2696 } 2697 2698 void 2699 _malloc_first_thread(void) 2700 { 2701 2702 (void)malloc_mutex_first_thread(); 2703 } 2704 2705 /******************************************************************************/ 2706