1 #define JEMALLOC_C_
2 #include "jemalloc/internal/jemalloc_preamble.h"
3 #include "jemalloc/internal/jemalloc_internal_includes.h"
4
5 #include "jemalloc/internal/assert.h"
6 #include "jemalloc/internal/atomic.h"
7 #include "jemalloc/internal/ctl.h"
8 #include "jemalloc/internal/extent_dss.h"
9 #include "jemalloc/internal/extent_mmap.h"
10 #include "jemalloc/internal/hook.h"
11 #include "jemalloc/internal/jemalloc_internal_types.h"
12 #include "jemalloc/internal/log.h"
13 #include "jemalloc/internal/malloc_io.h"
14 #include "jemalloc/internal/mutex.h"
15 #include "jemalloc/internal/rtree.h"
16 #include "jemalloc/internal/safety_check.h"
17 #include "jemalloc/internal/sc.h"
18 #include "jemalloc/internal/spin.h"
19 #include "jemalloc/internal/sz.h"
20 #include "jemalloc/internal/ticker.h"
21 #include "jemalloc/internal/util.h"
22
23 /******************************************************************************/
24 /* Data. */
25
26 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
27 const char *__malloc_options_1_0 = NULL;
28 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
29
30 /* Runtime configuration options. */
31 const char *je_malloc_conf
32 #ifndef _WIN32
33 JEMALLOC_ATTR(weak)
34 #endif
35 ;
36 bool opt_abort =
37 #ifdef JEMALLOC_DEBUG
38 true
39 #else
40 false
41 #endif
42 ;
43 bool opt_abort_conf =
44 #ifdef JEMALLOC_DEBUG
45 true
46 #else
47 false
48 #endif
49 ;
50 /* Intentionally default off, even with debug builds. */
51 bool opt_confirm_conf = false;
52 const char *opt_junk =
53 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
54 "true"
55 #else
56 "false"
57 #endif
58 ;
59 bool opt_junk_alloc =
60 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
61 true
62 #else
63 false
64 #endif
65 ;
66 bool opt_junk_free =
67 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
68 true
69 #else
70 false
71 #endif
72 ;
73
74 bool opt_utrace = false;
75 bool opt_xmalloc = false;
76 bool opt_zero = false;
77 unsigned opt_narenas = 0;
78
79 unsigned ncpus;
80
81 /* Protects arenas initialization. */
82 malloc_mutex_t arenas_lock;
83 /*
84 * Arenas that are used to service external requests. Not all elements of the
85 * arenas array are necessarily used; arenas are created lazily as needed.
86 *
87 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
88 * arenas. arenas[narenas_auto..narenas_total) are only used if the application
89 * takes some action to create them and allocate from them.
90 *
91 * Points to an arena_t.
92 */
93 JEMALLOC_ALIGNED(CACHELINE)
94 atomic_p_t arenas[MALLOCX_ARENA_LIMIT];
95 static atomic_u_t narenas_total; /* Use narenas_total_*(). */
96 /* Below three are read-only after initialization. */
97 static arena_t *a0; /* arenas[0]. */
98 unsigned narenas_auto;
99 unsigned manual_arena_base;
100
101 typedef enum {
102 malloc_init_uninitialized = 3,
103 malloc_init_a0_initialized = 2,
104 malloc_init_recursible = 1,
105 malloc_init_initialized = 0 /* Common case --> jnz. */
106 } malloc_init_t;
107 static malloc_init_t malloc_init_state = malloc_init_uninitialized;
108
109 /* False should be the common case. Set to true to trigger initialization. */
110 bool malloc_slow = true;
111
112 /* When malloc_slow is true, set the corresponding bits for sanity check. */
113 enum {
114 flag_opt_junk_alloc = (1U),
115 flag_opt_junk_free = (1U << 1),
116 flag_opt_zero = (1U << 2),
117 flag_opt_utrace = (1U << 3),
118 flag_opt_xmalloc = (1U << 4)
119 };
120 static uint8_t malloc_slow_flags;
121
122 #ifdef JEMALLOC_THREADED_INIT
123 /* Used to let the initializing thread recursively allocate. */
124 # define NO_INITIALIZER ((unsigned long)0)
125 # define INITIALIZER pthread_self()
126 # define IS_INITIALIZER (malloc_initializer == pthread_self())
127 static pthread_t malloc_initializer = NO_INITIALIZER;
128 #else
129 # define NO_INITIALIZER false
130 # define INITIALIZER true
131 # define IS_INITIALIZER malloc_initializer
132 static bool malloc_initializer = NO_INITIALIZER;
133 #endif
134
135 /* Used to avoid initialization races. */
136 #ifdef _WIN32
137 #if _WIN32_WINNT >= 0x0600
138 static malloc_mutex_t init_lock = SRWLOCK_INIT;
139 #else
140 static malloc_mutex_t init_lock;
141 static bool init_lock_initialized = false;
142
JEMALLOC_ATTR(constructor)143 JEMALLOC_ATTR(constructor)
144 static void WINAPI
145 _init_init_lock(void) {
146 /*
147 * If another constructor in the same binary is using mallctl to e.g.
148 * set up extent hooks, it may end up running before this one, and
149 * malloc_init_hard will crash trying to lock the uninitialized lock. So
150 * we force an initialization of the lock in malloc_init_hard as well.
151 * We don't try to care about atomicity of the accessed to the
152 * init_lock_initialized boolean, since it really only matters early in
153 * the process creation, before any separate thread normally starts
154 * doing anything.
155 */
156 if (!init_lock_initialized) {
157 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT,
158 malloc_mutex_rank_exclusive);
159 }
160 init_lock_initialized = true;
161 }
162
163 #ifdef _MSC_VER
164 # pragma section(".CRT$XCU", read)
165 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
166 static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
167 #endif
168 #endif
169 #else
170 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
171 #endif
172
173 typedef struct {
174 void *p; /* Input pointer (as in realloc(p, s)). */
175 size_t s; /* Request size. */
176 void *r; /* Result pointer. */
177 } malloc_utrace_t;
178
179 #ifdef JEMALLOC_UTRACE
180 # define UTRACE(a, b, c) do { \
181 if (unlikely(opt_utrace)) { \
182 int utrace_serrno = errno; \
183 malloc_utrace_t ut; \
184 ut.p = (a); \
185 ut.s = (b); \
186 ut.r = (c); \
187 utrace(&ut, sizeof(ut)); \
188 errno = utrace_serrno; \
189 } \
190 } while (0)
191 #else
192 # define UTRACE(a, b, c)
193 #endif
194
195 /* Whether encountered any invalid config options. */
196 static bool had_conf_error = false;
197
198 /******************************************************************************/
199 /*
200 * Function prototypes for static functions that are referenced prior to
201 * definition.
202 */
203
204 static bool malloc_init_hard_a0(void);
205 static bool malloc_init_hard(void);
206
207 /******************************************************************************/
208 /*
209 * Begin miscellaneous support functions.
210 */
211
212 bool
malloc_initialized(void)213 malloc_initialized(void) {
214 return (malloc_init_state == malloc_init_initialized);
215 }
216
217 JEMALLOC_ALWAYS_INLINE bool
malloc_init_a0(void)218 malloc_init_a0(void) {
219 if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
220 return malloc_init_hard_a0();
221 }
222 return false;
223 }
224
225 JEMALLOC_ALWAYS_INLINE bool
malloc_init(void)226 malloc_init(void) {
227 if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
228 return true;
229 }
230 return false;
231 }
232
233 /*
234 * The a0*() functions are used instead of i{d,}alloc() in situations that
235 * cannot tolerate TLS variable access.
236 */
237
238 static void *
a0ialloc(size_t size,bool zero,bool is_internal)239 a0ialloc(size_t size, bool zero, bool is_internal) {
240 if (unlikely(malloc_init_a0())) {
241 return NULL;
242 }
243
244 return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL,
245 is_internal, arena_get(TSDN_NULL, 0, true), true);
246 }
247
248 static void
a0idalloc(void * ptr,bool is_internal)249 a0idalloc(void *ptr, bool is_internal) {
250 idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true);
251 }
252
253 void *
a0malloc(size_t size)254 a0malloc(size_t size) {
255 return a0ialloc(size, false, true);
256 }
257
258 void
a0dalloc(void * ptr)259 a0dalloc(void *ptr) {
260 a0idalloc(ptr, true);
261 }
262
263 /*
264 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive
265 * situations that cannot tolerate TLS variable access (TLS allocation and very
266 * early internal data structure initialization).
267 */
268
269 void *
bootstrap_malloc(size_t size)270 bootstrap_malloc(size_t size) {
271 if (unlikely(size == 0)) {
272 size = 1;
273 }
274
275 return a0ialloc(size, false, false);
276 }
277
278 void *
bootstrap_calloc(size_t num,size_t size)279 bootstrap_calloc(size_t num, size_t size) {
280 size_t num_size;
281
282 num_size = num * size;
283 if (unlikely(num_size == 0)) {
284 assert(num == 0 || size == 0);
285 num_size = 1;
286 }
287
288 return a0ialloc(num_size, true, false);
289 }
290
291 void
bootstrap_free(void * ptr)292 bootstrap_free(void *ptr) {
293 if (unlikely(ptr == NULL)) {
294 return;
295 }
296
297 a0idalloc(ptr, false);
298 }
299
300 void
arena_set(unsigned ind,arena_t * arena)301 arena_set(unsigned ind, arena_t *arena) {
302 atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE);
303 }
304
305 static void
narenas_total_set(unsigned narenas)306 narenas_total_set(unsigned narenas) {
307 atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE);
308 }
309
310 static void
narenas_total_inc(void)311 narenas_total_inc(void) {
312 atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE);
313 }
314
315 unsigned
narenas_total_get(void)316 narenas_total_get(void) {
317 return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE);
318 }
319
320 /* Create a new arena and insert it into the arenas array at index ind. */
321 static arena_t *
arena_init_locked(tsdn_t * tsdn,unsigned ind,extent_hooks_t * extent_hooks)322 arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
323 arena_t *arena;
324
325 assert(ind <= narenas_total_get());
326 if (ind >= MALLOCX_ARENA_LIMIT) {
327 return NULL;
328 }
329 if (ind == narenas_total_get()) {
330 narenas_total_inc();
331 }
332
333 /*
334 * Another thread may have already initialized arenas[ind] if it's an
335 * auto arena.
336 */
337 arena = arena_get(tsdn, ind, false);
338 if (arena != NULL) {
339 assert(arena_is_auto(arena));
340 return arena;
341 }
342
343 /* Actually initialize the arena. */
344 arena = arena_new(tsdn, ind, extent_hooks);
345
346 return arena;
347 }
348
349 static void
arena_new_create_background_thread(tsdn_t * tsdn,unsigned ind)350 arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
351 if (ind == 0) {
352 return;
353 }
354 /*
355 * Avoid creating a new background thread just for the huge arena, which
356 * purges eagerly by default.
357 */
358 if (have_background_thread && !arena_is_huge(ind)) {
359 if (background_thread_create(tsdn_tsd(tsdn), ind)) {
360 malloc_printf("<jemalloc>: error in background thread "
361 "creation for arena %u. Abort.\n", ind);
362 abort();
363 }
364 }
365 }
366
367 arena_t *
arena_init(tsdn_t * tsdn,unsigned ind,extent_hooks_t * extent_hooks)368 arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
369 arena_t *arena;
370
371 malloc_mutex_lock(tsdn, &arenas_lock);
372 arena = arena_init_locked(tsdn, ind, extent_hooks);
373 malloc_mutex_unlock(tsdn, &arenas_lock);
374
375 arena_new_create_background_thread(tsdn, ind);
376
377 return arena;
378 }
379
380 static void
arena_bind(tsd_t * tsd,unsigned ind,bool internal)381 arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
382 arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false);
383 arena_nthreads_inc(arena, internal);
384
385 if (internal) {
386 tsd_iarena_set(tsd, arena);
387 } else {
388 tsd_arena_set(tsd, arena);
389 unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1,
390 ATOMIC_RELAXED);
391 tsd_binshards_t *bins = tsd_binshardsp_get(tsd);
392 for (unsigned i = 0; i < SC_NBINS; i++) {
393 assert(bin_infos[i].n_shards > 0 &&
394 bin_infos[i].n_shards <= BIN_SHARDS_MAX);
395 bins->binshard[i] = shard % bin_infos[i].n_shards;
396 }
397 }
398 }
399
400 void
arena_migrate(tsd_t * tsd,unsigned oldind,unsigned newind)401 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) {
402 arena_t *oldarena, *newarena;
403
404 oldarena = arena_get(tsd_tsdn(tsd), oldind, false);
405 newarena = arena_get(tsd_tsdn(tsd), newind, false);
406 arena_nthreads_dec(oldarena, false);
407 arena_nthreads_inc(newarena, false);
408 tsd_arena_set(tsd, newarena);
409 }
410
411 static void
arena_unbind(tsd_t * tsd,unsigned ind,bool internal)412 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
413 arena_t *arena;
414
415 arena = arena_get(tsd_tsdn(tsd), ind, false);
416 arena_nthreads_dec(arena, internal);
417
418 if (internal) {
419 tsd_iarena_set(tsd, NULL);
420 } else {
421 tsd_arena_set(tsd, NULL);
422 }
423 }
424
425 arena_tdata_t *
arena_tdata_get_hard(tsd_t * tsd,unsigned ind)426 arena_tdata_get_hard(tsd_t *tsd, unsigned ind) {
427 arena_tdata_t *tdata, *arenas_tdata_old;
428 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
429 unsigned narenas_tdata_old, i;
430 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd);
431 unsigned narenas_actual = narenas_total_get();
432
433 /*
434 * Dissociate old tdata array (and set up for deallocation upon return)
435 * if it's too small.
436 */
437 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) {
438 arenas_tdata_old = arenas_tdata;
439 narenas_tdata_old = narenas_tdata;
440 arenas_tdata = NULL;
441 narenas_tdata = 0;
442 tsd_arenas_tdata_set(tsd, arenas_tdata);
443 tsd_narenas_tdata_set(tsd, narenas_tdata);
444 } else {
445 arenas_tdata_old = NULL;
446 narenas_tdata_old = 0;
447 }
448
449 /* Allocate tdata array if it's missing. */
450 if (arenas_tdata == NULL) {
451 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd);
452 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1;
453
454 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) {
455 *arenas_tdata_bypassp = true;
456 arenas_tdata = (arena_tdata_t *)a0malloc(
457 sizeof(arena_tdata_t) * narenas_tdata);
458 *arenas_tdata_bypassp = false;
459 }
460 if (arenas_tdata == NULL) {
461 tdata = NULL;
462 goto label_return;
463 }
464 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp);
465 tsd_arenas_tdata_set(tsd, arenas_tdata);
466 tsd_narenas_tdata_set(tsd, narenas_tdata);
467 }
468
469 /*
470 * Copy to tdata array. It's possible that the actual number of arenas
471 * has increased since narenas_total_get() was called above, but that
472 * causes no correctness issues unless two threads concurrently execute
473 * the arenas.create mallctl, which we trust mallctl synchronization to
474 * prevent.
475 */
476
477 /* Copy/initialize tickers. */
478 for (i = 0; i < narenas_actual; i++) {
479 if (i < narenas_tdata_old) {
480 ticker_copy(&arenas_tdata[i].decay_ticker,
481 &arenas_tdata_old[i].decay_ticker);
482 } else {
483 ticker_init(&arenas_tdata[i].decay_ticker,
484 DECAY_NTICKS_PER_UPDATE);
485 }
486 }
487 if (narenas_tdata > narenas_actual) {
488 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t)
489 * (narenas_tdata - narenas_actual));
490 }
491
492 /* Read the refreshed tdata array. */
493 tdata = &arenas_tdata[ind];
494 label_return:
495 if (arenas_tdata_old != NULL) {
496 a0dalloc(arenas_tdata_old);
497 }
498 return tdata;
499 }
500
501 /* Slow path, called only by arena_choose(). */
502 arena_t *
arena_choose_hard(tsd_t * tsd,bool internal)503 arena_choose_hard(tsd_t *tsd, bool internal) {
504 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
505
506 if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
507 unsigned choose = percpu_arena_choose();
508 ret = arena_get(tsd_tsdn(tsd), choose, true);
509 assert(ret != NULL);
510 arena_bind(tsd, arena_ind_get(ret), false);
511 arena_bind(tsd, arena_ind_get(ret), true);
512
513 return ret;
514 }
515
516 if (narenas_auto > 1) {
517 unsigned i, j, choose[2], first_null;
518 bool is_new_arena[2];
519
520 /*
521 * Determine binding for both non-internal and internal
522 * allocation.
523 *
524 * choose[0]: For application allocation.
525 * choose[1]: For internal metadata allocation.
526 */
527
528 for (j = 0; j < 2; j++) {
529 choose[j] = 0;
530 is_new_arena[j] = false;
531 }
532
533 first_null = narenas_auto;
534 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
535 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
536 for (i = 1; i < narenas_auto; i++) {
537 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
538 /*
539 * Choose the first arena that has the lowest
540 * number of threads assigned to it.
541 */
542 for (j = 0; j < 2; j++) {
543 if (arena_nthreads_get(arena_get(
544 tsd_tsdn(tsd), i, false), !!j) <
545 arena_nthreads_get(arena_get(
546 tsd_tsdn(tsd), choose[j], false),
547 !!j)) {
548 choose[j] = i;
549 }
550 }
551 } else if (first_null == narenas_auto) {
552 /*
553 * Record the index of the first uninitialized
554 * arena, in case all extant arenas are in use.
555 *
556 * NB: It is possible for there to be
557 * discontinuities in terms of initialized
558 * versus uninitialized arenas, due to the
559 * "thread.arena" mallctl.
560 */
561 first_null = i;
562 }
563 }
564
565 for (j = 0; j < 2; j++) {
566 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd),
567 choose[j], false), !!j) == 0 || first_null ==
568 narenas_auto) {
569 /*
570 * Use an unloaded arena, or the least loaded
571 * arena if all arenas are already initialized.
572 */
573 if (!!j == internal) {
574 ret = arena_get(tsd_tsdn(tsd),
575 choose[j], false);
576 }
577 } else {
578 arena_t *arena;
579
580 /* Initialize a new arena. */
581 choose[j] = first_null;
582 arena = arena_init_locked(tsd_tsdn(tsd),
583 choose[j],
584 (extent_hooks_t *)&extent_hooks_default);
585 if (arena == NULL) {
586 malloc_mutex_unlock(tsd_tsdn(tsd),
587 &arenas_lock);
588 return NULL;
589 }
590 is_new_arena[j] = true;
591 if (!!j == internal) {
592 ret = arena;
593 }
594 }
595 arena_bind(tsd, choose[j], !!j);
596 }
597 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
598
599 for (j = 0; j < 2; j++) {
600 if (is_new_arena[j]) {
601 assert(choose[j] > 0);
602 arena_new_create_background_thread(
603 tsd_tsdn(tsd), choose[j]);
604 }
605 }
606
607 } else {
608 ret = arena_get(tsd_tsdn(tsd), 0, false);
609 arena_bind(tsd, 0, false);
610 arena_bind(tsd, 0, true);
611 }
612
613 return ret;
614 }
615
616 void
iarena_cleanup(tsd_t * tsd)617 iarena_cleanup(tsd_t *tsd) {
618 arena_t *iarena;
619
620 iarena = tsd_iarena_get(tsd);
621 if (iarena != NULL) {
622 arena_unbind(tsd, arena_ind_get(iarena), true);
623 }
624 }
625
626 void
arena_cleanup(tsd_t * tsd)627 arena_cleanup(tsd_t *tsd) {
628 arena_t *arena;
629
630 arena = tsd_arena_get(tsd);
631 if (arena != NULL) {
632 arena_unbind(tsd, arena_ind_get(arena), false);
633 }
634 }
635
636 void
arenas_tdata_cleanup(tsd_t * tsd)637 arenas_tdata_cleanup(tsd_t *tsd) {
638 arena_tdata_t *arenas_tdata;
639
640 /* Prevent tsd->arenas_tdata from being (re)created. */
641 *tsd_arenas_tdata_bypassp_get(tsd) = true;
642
643 arenas_tdata = tsd_arenas_tdata_get(tsd);
644 if (arenas_tdata != NULL) {
645 tsd_arenas_tdata_set(tsd, NULL);
646 a0dalloc(arenas_tdata);
647 }
648 }
649
650 static void
stats_print_atexit(void)651 stats_print_atexit(void) {
652 if (config_stats) {
653 tsdn_t *tsdn;
654 unsigned narenas, i;
655
656 tsdn = tsdn_fetch();
657
658 /*
659 * Merge stats from extant threads. This is racy, since
660 * individual threads do not lock when recording tcache stats
661 * events. As a consequence, the final stats may be slightly
662 * out of date by the time they are reported, if other threads
663 * continue to allocate.
664 */
665 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
666 arena_t *arena = arena_get(tsdn, i, false);
667 if (arena != NULL) {
668 tcache_t *tcache;
669
670 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
671 ql_foreach(tcache, &arena->tcache_ql, link) {
672 tcache_stats_merge(tsdn, tcache, arena);
673 }
674 malloc_mutex_unlock(tsdn,
675 &arena->tcache_ql_mtx);
676 }
677 }
678 }
679 je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
680 }
681
682 /*
683 * Ensure that we don't hold any locks upon entry to or exit from allocator
684 * code (in a "broad" sense that doesn't count a reentrant allocation as an
685 * entrance or exit).
686 */
687 JEMALLOC_ALWAYS_INLINE void
check_entry_exit_locking(tsdn_t * tsdn)688 check_entry_exit_locking(tsdn_t *tsdn) {
689 if (!config_debug) {
690 return;
691 }
692 if (tsdn_null(tsdn)) {
693 return;
694 }
695 tsd_t *tsd = tsdn_tsd(tsdn);
696 /*
697 * It's possible we hold locks at entry/exit if we're in a nested
698 * allocation.
699 */
700 int8_t reentrancy_level = tsd_reentrancy_level_get(tsd);
701 if (reentrancy_level != 0) {
702 return;
703 }
704 witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
705 }
706
707 /*
708 * End miscellaneous support functions.
709 */
710 /******************************************************************************/
711 /*
712 * Begin initialization functions.
713 */
714
715 static char *
jemalloc_secure_getenv(const char * name)716 jemalloc_secure_getenv(const char *name) {
717 #ifdef JEMALLOC_HAVE_SECURE_GETENV
718 return secure_getenv(name);
719 #else
720 # ifdef JEMALLOC_HAVE_ISSETUGID
721 if (issetugid() != 0) {
722 return NULL;
723 }
724 # endif
725 return getenv(name);
726 #endif
727 }
728
729 static unsigned
malloc_ncpus(void)730 malloc_ncpus(void) {
731 long result;
732
733 #ifdef _WIN32
734 SYSTEM_INFO si;
735 GetSystemInfo(&si);
736 result = si.dwNumberOfProcessors;
737 #elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT)
738 /*
739 * glibc >= 2.6 has the CPU_COUNT macro.
740 *
741 * glibc's sysconf() uses isspace(). glibc allocates for the first time
742 * *before* setting up the isspace tables. Therefore we need a
743 * different method to get the number of CPUs.
744 */
745 {
746 cpu_set_t set;
747
748 pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
749 result = CPU_COUNT(&set);
750 }
751 #else
752 result = sysconf(_SC_NPROCESSORS_ONLN);
753 #endif
754 return ((result == -1) ? 1 : (unsigned)result);
755 }
756
757 static void
init_opt_stats_print_opts(const char * v,size_t vlen)758 init_opt_stats_print_opts(const char *v, size_t vlen) {
759 size_t opts_len = strlen(opt_stats_print_opts);
760 assert(opts_len <= stats_print_tot_num_options);
761
762 for (size_t i = 0; i < vlen; i++) {
763 switch (v[i]) {
764 #define OPTION(o, v, d, s) case o: break;
765 STATS_PRINT_OPTIONS
766 #undef OPTION
767 default: continue;
768 }
769
770 if (strchr(opt_stats_print_opts, v[i]) != NULL) {
771 /* Ignore repeated. */
772 continue;
773 }
774
775 opt_stats_print_opts[opts_len++] = v[i];
776 opt_stats_print_opts[opts_len] = '\0';
777 assert(opts_len <= stats_print_tot_num_options);
778 }
779 assert(opts_len == strlen(opt_stats_print_opts));
780 }
781
782 /* Reads the next size pair in a multi-sized option. */
783 static bool
malloc_conf_multi_sizes_next(const char ** slab_size_segment_cur,size_t * vlen_left,size_t * slab_start,size_t * slab_end,size_t * new_size)784 malloc_conf_multi_sizes_next(const char **slab_size_segment_cur,
785 size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) {
786 const char *cur = *slab_size_segment_cur;
787 char *end;
788 uintmax_t um;
789
790 set_errno(0);
791
792 /* First number, then '-' */
793 um = malloc_strtoumax(cur, &end, 0);
794 if (get_errno() != 0 || *end != '-') {
795 return true;
796 }
797 *slab_start = (size_t)um;
798 cur = end + 1;
799
800 /* Second number, then ':' */
801 um = malloc_strtoumax(cur, &end, 0);
802 if (get_errno() != 0 || *end != ':') {
803 return true;
804 }
805 *slab_end = (size_t)um;
806 cur = end + 1;
807
808 /* Last number */
809 um = malloc_strtoumax(cur, &end, 0);
810 if (get_errno() != 0) {
811 return true;
812 }
813 *new_size = (size_t)um;
814
815 /* Consume the separator if there is one. */
816 if (*end == '|') {
817 end++;
818 }
819
820 *vlen_left -= end - *slab_size_segment_cur;
821 *slab_size_segment_cur = end;
822
823 return false;
824 }
825
826 static bool
malloc_conf_next(char const ** opts_p,char const ** k_p,size_t * klen_p,char const ** v_p,size_t * vlen_p)827 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
828 char const **v_p, size_t *vlen_p) {
829 bool accept;
830 const char *opts = *opts_p;
831
832 *k_p = opts;
833
834 for (accept = false; !accept;) {
835 switch (*opts) {
836 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
837 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
838 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
839 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
840 case 'Y': case 'Z':
841 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
842 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
843 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
844 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
845 case 'y': case 'z':
846 case '0': case '1': case '2': case '3': case '4': case '5':
847 case '6': case '7': case '8': case '9':
848 case '_':
849 opts++;
850 break;
851 case ':':
852 opts++;
853 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
854 *v_p = opts;
855 accept = true;
856 break;
857 case '\0':
858 if (opts != *opts_p) {
859 malloc_write("<jemalloc>: Conf string ends "
860 "with key\n");
861 }
862 return true;
863 default:
864 malloc_write("<jemalloc>: Malformed conf string\n");
865 return true;
866 }
867 }
868
869 for (accept = false; !accept;) {
870 switch (*opts) {
871 case ',':
872 opts++;
873 /*
874 * Look ahead one character here, because the next time
875 * this function is called, it will assume that end of
876 * input has been cleanly reached if no input remains,
877 * but we have optimistically already consumed the
878 * comma if one exists.
879 */
880 if (*opts == '\0') {
881 malloc_write("<jemalloc>: Conf string ends "
882 "with comma\n");
883 }
884 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
885 accept = true;
886 break;
887 case '\0':
888 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
889 accept = true;
890 break;
891 default:
892 opts++;
893 break;
894 }
895 }
896
897 *opts_p = opts;
898 return false;
899 }
900
901 static void
malloc_abort_invalid_conf(void)902 malloc_abort_invalid_conf(void) {
903 assert(opt_abort_conf);
904 malloc_printf("<jemalloc>: Abort (abort_conf:true) on invalid conf "
905 "value (see above).\n");
906 abort();
907 }
908
909 static void
malloc_conf_error(const char * msg,const char * k,size_t klen,const char * v,size_t vlen)910 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
911 size_t vlen) {
912 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
913 (int)vlen, v);
914 /* If abort_conf is set, error out after processing all options. */
915 const char *experimental = "experimental_";
916 if (strncmp(k, experimental, strlen(experimental)) == 0) {
917 /* However, tolerate experimental features. */
918 return;
919 }
920 had_conf_error = true;
921 }
922
923 static void
malloc_slow_flag_init(void)924 malloc_slow_flag_init(void) {
925 /*
926 * Combine the runtime options into malloc_slow for fast path. Called
927 * after processing all the options.
928 */
929 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
930 | (opt_junk_free ? flag_opt_junk_free : 0)
931 | (opt_zero ? flag_opt_zero : 0)
932 | (opt_utrace ? flag_opt_utrace : 0)
933 | (opt_xmalloc ? flag_opt_xmalloc : 0);
934
935 malloc_slow = (malloc_slow_flags != 0);
936 }
937
938 /* Number of sources for initializing malloc_conf */
939 #define MALLOC_CONF_NSOURCES 4
940
941 static const char *
obtain_malloc_conf(unsigned which_source,char buf[PATH_MAX+1])942 obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) {
943 if (config_debug) {
944 static unsigned read_source = 0;
945 /*
946 * Each source should only be read once, to minimize # of
947 * syscalls on init.
948 */
949 assert(read_source++ == which_source);
950 }
951 assert(which_source < MALLOC_CONF_NSOURCES);
952
953 const char *ret;
954 switch (which_source) {
955 case 0:
956 ret = config_malloc_conf;
957 break;
958 case 1:
959 if (je_malloc_conf != NULL) {
960 /* Use options that were compiled into the program. */
961 ret = je_malloc_conf;
962 } else {
963 /* No configuration specified. */
964 ret = NULL;
965 }
966 break;
967 case 2: {
968 ssize_t linklen = 0;
969 #ifndef _WIN32
970 int saved_errno = errno;
971 const char *linkname =
972 # ifdef JEMALLOC_PREFIX
973 "/etc/"JEMALLOC_PREFIX"malloc.conf"
974 # else
975 "/etc/malloc.conf"
976 # endif
977 ;
978
979 /*
980 * Try to use the contents of the "/etc/malloc.conf" symbolic
981 * link's name.
982 */
983 #ifndef JEMALLOC_READLINKAT
984 linklen = readlink(linkname, buf, PATH_MAX);
985 #else
986 linklen = readlinkat(AT_FDCWD, linkname, buf, PATH_MAX);
987 #endif
988 if (linklen == -1) {
989 /* No configuration specified. */
990 linklen = 0;
991 /* Restore errno. */
992 set_errno(saved_errno);
993 }
994 #endif
995 buf[linklen] = '\0';
996 ret = buf;
997 break;
998 } case 3: {
999 const char *envname =
1000 #ifdef JEMALLOC_PREFIX
1001 JEMALLOC_CPREFIX"MALLOC_CONF"
1002 #else
1003 "MALLOC_CONF"
1004 #endif
1005 ;
1006
1007 if ((ret = jemalloc_secure_getenv(envname)) != NULL) {
1008 /*
1009 * Do nothing; opts is already initialized to the value
1010 * of the MALLOC_CONF environment variable.
1011 */
1012 } else {
1013 /* No configuration specified. */
1014 ret = NULL;
1015 }
1016 break;
1017 } default:
1018 not_reached();
1019 ret = NULL;
1020 }
1021 return ret;
1022 }
1023
1024 static void
malloc_conf_init_helper(sc_data_t * sc_data,unsigned bin_shard_sizes[SC_NBINS],bool initial_call,const char * opts_cache[MALLOC_CONF_NSOURCES],char buf[PATH_MAX+1])1025 malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
1026 bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES],
1027 char buf[PATH_MAX + 1]) {
1028 static const char *opts_explain[MALLOC_CONF_NSOURCES] = {
1029 "string specified via --with-malloc-conf",
1030 "string pointed to by the global variable malloc_conf",
1031 "\"name\" of the file referenced by the symbolic link named "
1032 "/etc/malloc.conf",
1033 "value of the environment variable MALLOC_CONF"
1034 };
1035 unsigned i;
1036 const char *opts, *k, *v;
1037 size_t klen, vlen;
1038
1039 for (i = 0; i < MALLOC_CONF_NSOURCES; i++) {
1040 /* Get runtime configuration. */
1041 if (initial_call) {
1042 opts_cache[i] = obtain_malloc_conf(i, buf);
1043 }
1044 opts = opts_cache[i];
1045 if (!initial_call && opt_confirm_conf) {
1046 malloc_printf(
1047 "<jemalloc>: malloc_conf #%u (%s): \"%s\"\n",
1048 i + 1, opts_explain[i], opts != NULL ? opts : "");
1049 }
1050 if (opts == NULL) {
1051 continue;
1052 }
1053
1054 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v,
1055 &vlen)) {
1056
1057 #define CONF_ERROR(msg, k, klen, v, vlen) \
1058 if (!initial_call) { \
1059 malloc_conf_error( \
1060 msg, k, klen, v, vlen); \
1061 cur_opt_valid = false; \
1062 }
1063 #define CONF_CONTINUE { \
1064 if (!initial_call && opt_confirm_conf \
1065 && cur_opt_valid) { \
1066 malloc_printf("<jemalloc>: -- " \
1067 "Set conf value: %.*s:%.*s" \
1068 "\n", (int)klen, k, \
1069 (int)vlen, v); \
1070 } \
1071 continue; \
1072 }
1073 #define CONF_MATCH(n) \
1074 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
1075 #define CONF_MATCH_VALUE(n) \
1076 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0)
1077 #define CONF_HANDLE_BOOL(o, n) \
1078 if (CONF_MATCH(n)) { \
1079 if (CONF_MATCH_VALUE("true")) { \
1080 o = true; \
1081 } else if (CONF_MATCH_VALUE("false")) { \
1082 o = false; \
1083 } else { \
1084 CONF_ERROR("Invalid conf value",\
1085 k, klen, v, vlen); \
1086 } \
1087 CONF_CONTINUE; \
1088 }
1089 /*
1090 * One of the CONF_MIN macros below expands, in one of the use points,
1091 * to "unsigned integer < 0", which is always false, triggering the
1092 * GCC -Wtype-limits warning, which we disable here and re-enable below.
1093 */
1094 JEMALLOC_DIAGNOSTIC_PUSH
1095 JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
1096
1097 #define CONF_DONT_CHECK_MIN(um, min) false
1098 #define CONF_CHECK_MIN(um, min) ((um) < (min))
1099 #define CONF_DONT_CHECK_MAX(um, max) false
1100 #define CONF_CHECK_MAX(um, max) ((um) > (max))
1101 #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \
1102 if (CONF_MATCH(n)) { \
1103 uintmax_t um; \
1104 char *end; \
1105 \
1106 set_errno(0); \
1107 um = malloc_strtoumax(v, &end, 0); \
1108 if (get_errno() != 0 || (uintptr_t)end -\
1109 (uintptr_t)v != vlen) { \
1110 CONF_ERROR("Invalid conf value",\
1111 k, klen, v, vlen); \
1112 } else if (clip) { \
1113 if (check_min(um, (t)(min))) { \
1114 o = (t)(min); \
1115 } else if ( \
1116 check_max(um, (t)(max))) { \
1117 o = (t)(max); \
1118 } else { \
1119 o = (t)um; \
1120 } \
1121 } else { \
1122 if (check_min(um, (t)(min)) || \
1123 check_max(um, (t)(max))) { \
1124 CONF_ERROR( \
1125 "Out-of-range " \
1126 "conf value", \
1127 k, klen, v, vlen); \
1128 } else { \
1129 o = (t)um; \
1130 } \
1131 } \
1132 CONF_CONTINUE; \
1133 }
1134 #define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \
1135 clip) \
1136 CONF_HANDLE_T_U(unsigned, o, n, min, max, \
1137 check_min, check_max, clip)
1138 #define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \
1139 CONF_HANDLE_T_U(size_t, o, n, min, max, \
1140 check_min, check_max, clip)
1141 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \
1142 if (CONF_MATCH(n)) { \
1143 long l; \
1144 char *end; \
1145 \
1146 set_errno(0); \
1147 l = strtol(v, &end, 0); \
1148 if (get_errno() != 0 || (uintptr_t)end -\
1149 (uintptr_t)v != vlen) { \
1150 CONF_ERROR("Invalid conf value",\
1151 k, klen, v, vlen); \
1152 } else if (l < (ssize_t)(min) || l > \
1153 (ssize_t)(max)) { \
1154 CONF_ERROR( \
1155 "Out-of-range conf value", \
1156 k, klen, v, vlen); \
1157 } else { \
1158 o = l; \
1159 } \
1160 CONF_CONTINUE; \
1161 }
1162 #define CONF_HANDLE_CHAR_P(o, n, d) \
1163 if (CONF_MATCH(n)) { \
1164 size_t cpylen = (vlen <= \
1165 sizeof(o)-1) ? vlen : \
1166 sizeof(o)-1; \
1167 strncpy(o, v, cpylen); \
1168 o[cpylen] = '\0'; \
1169 CONF_CONTINUE; \
1170 }
1171
1172 bool cur_opt_valid = true;
1173
1174 CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf")
1175 if (initial_call) {
1176 continue;
1177 }
1178
1179 CONF_HANDLE_BOOL(opt_abort, "abort")
1180 CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf")
1181 if (strncmp("metadata_thp", k, klen) == 0) {
1182 int i;
1183 bool match = false;
1184 for (i = 0; i < metadata_thp_mode_limit; i++) {
1185 if (strncmp(metadata_thp_mode_names[i],
1186 v, vlen) == 0) {
1187 opt_metadata_thp = i;
1188 match = true;
1189 break;
1190 }
1191 }
1192 if (!match) {
1193 CONF_ERROR("Invalid conf value",
1194 k, klen, v, vlen);
1195 }
1196 CONF_CONTINUE;
1197 }
1198 CONF_HANDLE_BOOL(opt_retain, "retain")
1199 if (strncmp("dss", k, klen) == 0) {
1200 int i;
1201 bool match = false;
1202 for (i = 0; i < dss_prec_limit; i++) {
1203 if (strncmp(dss_prec_names[i], v, vlen)
1204 == 0) {
1205 if (extent_dss_prec_set(i)) {
1206 CONF_ERROR(
1207 "Error setting dss",
1208 k, klen, v, vlen);
1209 } else {
1210 opt_dss =
1211 dss_prec_names[i];
1212 match = true;
1213 break;
1214 }
1215 }
1216 }
1217 if (!match) {
1218 CONF_ERROR("Invalid conf value",
1219 k, klen, v, vlen);
1220 }
1221 CONF_CONTINUE;
1222 }
1223 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
1224 UINT_MAX, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
1225 false)
1226 if (CONF_MATCH("bin_shards")) {
1227 const char *bin_shards_segment_cur = v;
1228 size_t vlen_left = vlen;
1229 do {
1230 size_t size_start;
1231 size_t size_end;
1232 size_t nshards;
1233 bool err = malloc_conf_multi_sizes_next(
1234 &bin_shards_segment_cur, &vlen_left,
1235 &size_start, &size_end, &nshards);
1236 if (err || bin_update_shard_size(
1237 bin_shard_sizes, size_start,
1238 size_end, nshards)) {
1239 CONF_ERROR(
1240 "Invalid settings for "
1241 "bin_shards", k, klen, v,
1242 vlen);
1243 break;
1244 }
1245 } while (vlen_left > 0);
1246 CONF_CONTINUE;
1247 }
1248 CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms,
1249 "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
1250 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
1251 SSIZE_MAX);
1252 CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms,
1253 "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
1254 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
1255 SSIZE_MAX);
1256 CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
1257 if (CONF_MATCH("stats_print_opts")) {
1258 init_opt_stats_print_opts(v, vlen);
1259 CONF_CONTINUE;
1260 }
1261 if (config_fill) {
1262 if (CONF_MATCH("junk")) {
1263 if (CONF_MATCH_VALUE("true")) {
1264 opt_junk = "true";
1265 opt_junk_alloc = opt_junk_free =
1266 true;
1267 } else if (CONF_MATCH_VALUE("false")) {
1268 opt_junk = "false";
1269 opt_junk_alloc = opt_junk_free =
1270 false;
1271 } else if (CONF_MATCH_VALUE("alloc")) {
1272 opt_junk = "alloc";
1273 opt_junk_alloc = true;
1274 opt_junk_free = false;
1275 } else if (CONF_MATCH_VALUE("free")) {
1276 opt_junk = "free";
1277 opt_junk_alloc = false;
1278 opt_junk_free = true;
1279 } else {
1280 CONF_ERROR(
1281 "Invalid conf value",
1282 k, klen, v, vlen);
1283 }
1284 CONF_CONTINUE;
1285 }
1286 CONF_HANDLE_BOOL(opt_zero, "zero")
1287 }
1288 if (config_utrace) {
1289 CONF_HANDLE_BOOL(opt_utrace, "utrace")
1290 }
1291 if (config_xmalloc) {
1292 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
1293 }
1294 CONF_HANDLE_BOOL(opt_tcache, "tcache")
1295 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max",
1296 -1, (sizeof(size_t) << 3) - 1)
1297
1298 /*
1299 * The runtime option of oversize_threshold remains
1300 * undocumented. It may be tweaked in the next major
1301 * release (6.0). The default value 8M is rather
1302 * conservative / safe. Tuning it further down may
1303 * improve fragmentation a bit more, but may also cause
1304 * contention on the huge arena.
1305 */
1306 CONF_HANDLE_SIZE_T(opt_oversize_threshold,
1307 "oversize_threshold", 0, SC_LARGE_MAXCLASS,
1308 CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false)
1309 CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit,
1310 "lg_extent_max_active_fit", 0,
1311 (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN,
1312 CONF_CHECK_MAX, false)
1313
1314 if (strncmp("percpu_arena", k, klen) == 0) {
1315 bool match = false;
1316 for (int i = percpu_arena_mode_names_base; i <
1317 percpu_arena_mode_names_limit; i++) {
1318 if (strncmp(percpu_arena_mode_names[i],
1319 v, vlen) == 0) {
1320 if (!have_percpu_arena) {
1321 CONF_ERROR(
1322 "No getcpu support",
1323 k, klen, v, vlen);
1324 }
1325 opt_percpu_arena = i;
1326 match = true;
1327 break;
1328 }
1329 }
1330 if (!match) {
1331 CONF_ERROR("Invalid conf value",
1332 k, klen, v, vlen);
1333 }
1334 CONF_CONTINUE;
1335 }
1336 CONF_HANDLE_BOOL(opt_background_thread,
1337 "background_thread");
1338 CONF_HANDLE_SIZE_T(opt_max_background_threads,
1339 "max_background_threads", 1,
1340 opt_max_background_threads,
1341 CONF_CHECK_MIN, CONF_CHECK_MAX,
1342 true);
1343 if (CONF_MATCH("slab_sizes")) {
1344 bool err;
1345 const char *slab_size_segment_cur = v;
1346 size_t vlen_left = vlen;
1347 do {
1348 size_t slab_start;
1349 size_t slab_end;
1350 size_t pgs;
1351 err = malloc_conf_multi_sizes_next(
1352 &slab_size_segment_cur,
1353 &vlen_left, &slab_start, &slab_end,
1354 &pgs);
1355 if (!err) {
1356 sc_data_update_slab_size(
1357 sc_data, slab_start,
1358 slab_end, (int)pgs);
1359 } else {
1360 CONF_ERROR("Invalid settings "
1361 "for slab_sizes",
1362 k, klen, v, vlen);
1363 }
1364 } while (!err && vlen_left > 0);
1365 CONF_CONTINUE;
1366 }
1367 if (config_prof) {
1368 CONF_HANDLE_BOOL(opt_prof, "prof")
1369 CONF_HANDLE_CHAR_P(opt_prof_prefix,
1370 "prof_prefix", "jeprof")
1371 CONF_HANDLE_BOOL(opt_prof_active, "prof_active")
1372 CONF_HANDLE_BOOL(opt_prof_thread_active_init,
1373 "prof_thread_active_init")
1374 CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
1375 "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
1376 - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
1377 true)
1378 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
1379 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
1380 "lg_prof_interval", -1,
1381 (sizeof(uint64_t) << 3) - 1)
1382 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
1383 CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
1384 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
1385 CONF_HANDLE_BOOL(opt_prof_log, "prof_log")
1386 }
1387 if (config_log) {
1388 if (CONF_MATCH("log")) {
1389 size_t cpylen = (
1390 vlen <= sizeof(log_var_names) ?
1391 vlen : sizeof(log_var_names) - 1);
1392 strncpy(log_var_names, v, cpylen);
1393 log_var_names[cpylen] = '\0';
1394 CONF_CONTINUE;
1395 }
1396 }
1397 if (CONF_MATCH("thp")) {
1398 bool match = false;
1399 for (int i = 0; i < thp_mode_names_limit; i++) {
1400 if (strncmp(thp_mode_names[i],v, vlen)
1401 == 0) {
1402 if (!have_madvise_huge) {
1403 CONF_ERROR(
1404 "No THP support",
1405 k, klen, v, vlen);
1406 }
1407 opt_thp = i;
1408 match = true;
1409 break;
1410 }
1411 }
1412 if (!match) {
1413 CONF_ERROR("Invalid conf value",
1414 k, klen, v, vlen);
1415 }
1416 CONF_CONTINUE;
1417 }
1418 CONF_ERROR("Invalid conf pair", k, klen, v, vlen);
1419 #undef CONF_ERROR
1420 #undef CONF_CONTINUE
1421 #undef CONF_MATCH
1422 #undef CONF_MATCH_VALUE
1423 #undef CONF_HANDLE_BOOL
1424 #undef CONF_DONT_CHECK_MIN
1425 #undef CONF_CHECK_MIN
1426 #undef CONF_DONT_CHECK_MAX
1427 #undef CONF_CHECK_MAX
1428 #undef CONF_HANDLE_T_U
1429 #undef CONF_HANDLE_UNSIGNED
1430 #undef CONF_HANDLE_SIZE_T
1431 #undef CONF_HANDLE_SSIZE_T
1432 #undef CONF_HANDLE_CHAR_P
1433 /* Re-enable diagnostic "-Wtype-limits" */
1434 JEMALLOC_DIAGNOSTIC_POP
1435 }
1436 if (opt_abort_conf && had_conf_error) {
1437 malloc_abort_invalid_conf();
1438 }
1439 }
1440 atomic_store_b(&log_init_done, true, ATOMIC_RELEASE);
1441 }
1442
1443 static void
malloc_conf_init(sc_data_t * sc_data,unsigned bin_shard_sizes[SC_NBINS])1444 malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
1445 const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL};
1446 char buf[PATH_MAX + 1];
1447
1448 /* The first call only set the confirm_conf option and opts_cache */
1449 malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf);
1450 malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache,
1451 NULL);
1452 }
1453
1454 #undef MALLOC_CONF_NSOURCES
1455
1456 static bool
malloc_init_hard_needed(void)1457 malloc_init_hard_needed(void) {
1458 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state ==
1459 malloc_init_recursible)) {
1460 /*
1461 * Another thread initialized the allocator before this one
1462 * acquired init_lock, or this thread is the initializing
1463 * thread, and it is recursively allocating.
1464 */
1465 return false;
1466 }
1467 #ifdef JEMALLOC_THREADED_INIT
1468 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
1469 /* Busy-wait until the initializing thread completes. */
1470 spin_t spinner = SPIN_INITIALIZER;
1471 do {
1472 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1473 spin_adaptive(&spinner);
1474 malloc_mutex_lock(TSDN_NULL, &init_lock);
1475 } while (!malloc_initialized());
1476 return false;
1477 }
1478 #endif
1479 return true;
1480 }
1481
1482 static bool
malloc_init_hard_a0_locked()1483 malloc_init_hard_a0_locked() {
1484 malloc_initializer = INITIALIZER;
1485
1486 JEMALLOC_DIAGNOSTIC_PUSH
1487 JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
1488 sc_data_t sc_data = {0};
1489 JEMALLOC_DIAGNOSTIC_POP
1490
1491 /*
1492 * Ordering here is somewhat tricky; we need sc_boot() first, since that
1493 * determines what the size classes will be, and then
1494 * malloc_conf_init(), since any slab size tweaking will need to be done
1495 * before sz_boot and bin_boot, which assume that the values they read
1496 * out of sc_data_global are final.
1497 */
1498 sc_boot(&sc_data);
1499 unsigned bin_shard_sizes[SC_NBINS];
1500 bin_shard_sizes_boot(bin_shard_sizes);
1501 /*
1502 * prof_boot0 only initializes opt_prof_prefix. We need to do it before
1503 * we parse malloc_conf options, in case malloc_conf parsing overwrites
1504 * it.
1505 */
1506 if (config_prof) {
1507 prof_boot0();
1508 }
1509 malloc_conf_init(&sc_data, bin_shard_sizes);
1510 sz_boot(&sc_data);
1511 bin_boot(&sc_data, bin_shard_sizes);
1512
1513 if (opt_stats_print) {
1514 /* Print statistics at exit. */
1515 if (atexit(stats_print_atexit) != 0) {
1516 malloc_write("<jemalloc>: Error in atexit()\n");
1517 if (opt_abort) {
1518 abort();
1519 }
1520 }
1521 }
1522 if (pages_boot()) {
1523 return true;
1524 }
1525 if (base_boot(TSDN_NULL)) {
1526 return true;
1527 }
1528 if (extent_boot()) {
1529 return true;
1530 }
1531 if (ctl_boot()) {
1532 return true;
1533 }
1534 if (config_prof) {
1535 prof_boot1();
1536 }
1537 arena_boot(&sc_data);
1538 if (tcache_boot(TSDN_NULL)) {
1539 return true;
1540 }
1541 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
1542 malloc_mutex_rank_exclusive)) {
1543 return true;
1544 }
1545 hook_boot();
1546 /*
1547 * Create enough scaffolding to allow recursive allocation in
1548 * malloc_ncpus().
1549 */
1550 narenas_auto = 1;
1551 manual_arena_base = narenas_auto + 1;
1552 memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
1553 /*
1554 * Initialize one arena here. The rest are lazily created in
1555 * arena_choose_hard().
1556 */
1557 if (arena_init(TSDN_NULL, 0, (extent_hooks_t *)&extent_hooks_default)
1558 == NULL) {
1559 return true;
1560 }
1561 a0 = arena_get(TSDN_NULL, 0, false);
1562 malloc_init_state = malloc_init_a0_initialized;
1563
1564 return false;
1565 }
1566
1567 static bool
malloc_init_hard_a0(void)1568 malloc_init_hard_a0(void) {
1569 bool ret;
1570
1571 malloc_mutex_lock(TSDN_NULL, &init_lock);
1572 ret = malloc_init_hard_a0_locked();
1573 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1574 return ret;
1575 }
1576
1577 /* Initialize data structures which may trigger recursive allocation. */
1578 static bool
malloc_init_hard_recursible(void)1579 malloc_init_hard_recursible(void) {
1580 malloc_init_state = malloc_init_recursible;
1581
1582 ncpus = malloc_ncpus();
1583
1584 #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
1585 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
1586 !defined(__native_client__))
1587 /* LinuxThreads' pthread_atfork() allocates. */
1588 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
1589 jemalloc_postfork_child) != 0) {
1590 malloc_write("<jemalloc>: Error in pthread_atfork()\n");
1591 if (opt_abort) {
1592 abort();
1593 }
1594 return true;
1595 }
1596 #endif
1597
1598 if (background_thread_boot0()) {
1599 return true;
1600 }
1601
1602 return false;
1603 }
1604
1605 static unsigned
malloc_narenas_default(void)1606 malloc_narenas_default(void) {
1607 assert(ncpus > 0);
1608 /*
1609 * For SMP systems, create more than one arena per CPU by
1610 * default.
1611 */
1612 if (ncpus > 1) {
1613 return ncpus << 2;
1614 } else {
1615 return 1;
1616 }
1617 }
1618
1619 static percpu_arena_mode_t
percpu_arena_as_initialized(percpu_arena_mode_t mode)1620 percpu_arena_as_initialized(percpu_arena_mode_t mode) {
1621 assert(!malloc_initialized());
1622 assert(mode <= percpu_arena_disabled);
1623
1624 if (mode != percpu_arena_disabled) {
1625 mode += percpu_arena_mode_enabled_base;
1626 }
1627
1628 return mode;
1629 }
1630
1631 static bool
malloc_init_narenas(void)1632 malloc_init_narenas(void) {
1633 assert(ncpus > 0);
1634
1635 if (opt_percpu_arena != percpu_arena_disabled) {
1636 if (!have_percpu_arena || malloc_getcpu() < 0) {
1637 opt_percpu_arena = percpu_arena_disabled;
1638 malloc_printf("<jemalloc>: perCPU arena getcpu() not "
1639 "available. Setting narenas to %u.\n", opt_narenas ?
1640 opt_narenas : malloc_narenas_default());
1641 if (opt_abort) {
1642 abort();
1643 }
1644 } else {
1645 if (ncpus >= MALLOCX_ARENA_LIMIT) {
1646 malloc_printf("<jemalloc>: narenas w/ percpu"
1647 "arena beyond limit (%d)\n", ncpus);
1648 if (opt_abort) {
1649 abort();
1650 }
1651 return true;
1652 }
1653 /* NB: opt_percpu_arena isn't fully initialized yet. */
1654 if (percpu_arena_as_initialized(opt_percpu_arena) ==
1655 per_phycpu_arena && ncpus % 2 != 0) {
1656 malloc_printf("<jemalloc>: invalid "
1657 "configuration -- per physical CPU arena "
1658 "with odd number (%u) of CPUs (no hyper "
1659 "threading?).\n", ncpus);
1660 if (opt_abort)
1661 abort();
1662 }
1663 unsigned n = percpu_arena_ind_limit(
1664 percpu_arena_as_initialized(opt_percpu_arena));
1665 if (opt_narenas < n) {
1666 /*
1667 * If narenas is specified with percpu_arena
1668 * enabled, actual narenas is set as the greater
1669 * of the two. percpu_arena_choose will be free
1670 * to use any of the arenas based on CPU
1671 * id. This is conservative (at a small cost)
1672 * but ensures correctness.
1673 *
1674 * If for some reason the ncpus determined at
1675 * boot is not the actual number (e.g. because
1676 * of affinity setting from numactl), reserving
1677 * narenas this way provides a workaround for
1678 * percpu_arena.
1679 */
1680 opt_narenas = n;
1681 }
1682 }
1683 }
1684 if (opt_narenas == 0) {
1685 opt_narenas = malloc_narenas_default();
1686 }
1687 assert(opt_narenas > 0);
1688
1689 narenas_auto = opt_narenas;
1690 /*
1691 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
1692 */
1693 if (narenas_auto >= MALLOCX_ARENA_LIMIT) {
1694 narenas_auto = MALLOCX_ARENA_LIMIT - 1;
1695 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
1696 narenas_auto);
1697 }
1698 narenas_total_set(narenas_auto);
1699 if (arena_init_huge()) {
1700 narenas_total_inc();
1701 }
1702 manual_arena_base = narenas_total_get();
1703
1704 return false;
1705 }
1706
1707 static void
malloc_init_percpu(void)1708 malloc_init_percpu(void) {
1709 opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena);
1710 }
1711
1712 static bool
malloc_init_hard_finish(void)1713 malloc_init_hard_finish(void) {
1714 if (malloc_mutex_boot()) {
1715 return true;
1716 }
1717
1718 malloc_init_state = malloc_init_initialized;
1719 malloc_slow_flag_init();
1720
1721 return false;
1722 }
1723
1724 static void
malloc_init_hard_cleanup(tsdn_t * tsdn,bool reentrancy_set)1725 malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) {
1726 malloc_mutex_assert_owner(tsdn, &init_lock);
1727 malloc_mutex_unlock(tsdn, &init_lock);
1728 if (reentrancy_set) {
1729 assert(!tsdn_null(tsdn));
1730 tsd_t *tsd = tsdn_tsd(tsdn);
1731 assert(tsd_reentrancy_level_get(tsd) > 0);
1732 post_reentrancy(tsd);
1733 }
1734 }
1735
1736 static bool
malloc_init_hard(void)1737 malloc_init_hard(void) {
1738 tsd_t *tsd;
1739
1740 #if defined(_WIN32) && _WIN32_WINNT < 0x0600
1741 _init_init_lock();
1742 #endif
1743 malloc_mutex_lock(TSDN_NULL, &init_lock);
1744
1745 #define UNLOCK_RETURN(tsdn, ret, reentrancy) \
1746 malloc_init_hard_cleanup(tsdn, reentrancy); \
1747 return ret;
1748
1749 if (!malloc_init_hard_needed()) {
1750 UNLOCK_RETURN(TSDN_NULL, false, false)
1751 }
1752
1753 if (malloc_init_state != malloc_init_a0_initialized &&
1754 malloc_init_hard_a0_locked()) {
1755 UNLOCK_RETURN(TSDN_NULL, true, false)
1756 }
1757
1758 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1759 /* Recursive allocation relies on functional tsd. */
1760 tsd = malloc_tsd_boot0();
1761 if (tsd == NULL) {
1762 return true;
1763 }
1764 if (malloc_init_hard_recursible()) {
1765 return true;
1766 }
1767
1768 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
1769 /* Set reentrancy level to 1 during init. */
1770 pre_reentrancy(tsd, NULL);
1771 /* Initialize narenas before prof_boot2 (for allocation). */
1772 if (malloc_init_narenas() || background_thread_boot1(tsd_tsdn(tsd))) {
1773 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1774 }
1775 if (config_prof && prof_boot2(tsd)) {
1776 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1777 }
1778
1779 malloc_init_percpu();
1780
1781 if (malloc_init_hard_finish()) {
1782 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1783 }
1784 post_reentrancy(tsd);
1785 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1786
1787 witness_assert_lockless(witness_tsd_tsdn(
1788 tsd_witness_tsdp_get_unsafe(tsd)));
1789 malloc_tsd_boot1();
1790 /* Update TSD after tsd_boot1. */
1791 tsd = tsd_fetch();
1792 if (opt_background_thread) {
1793 assert(have_background_thread);
1794 /*
1795 * Need to finish init & unlock first before creating background
1796 * threads (pthread_create depends on malloc). ctl_init (which
1797 * sets isthreaded) needs to be called without holding any lock.
1798 */
1799 background_thread_ctl_init(tsd_tsdn(tsd));
1800 if (background_thread_create(tsd, 0)) {
1801 return true;
1802 }
1803 }
1804 #undef UNLOCK_RETURN
1805 return false;
1806 }
1807
1808 /*
1809 * End initialization functions.
1810 */
1811 /******************************************************************************/
1812 /*
1813 * Begin allocation-path internal functions and data structures.
1814 */
1815
1816 /*
1817 * Settings determined by the documented behavior of the allocation functions.
1818 */
1819 typedef struct static_opts_s static_opts_t;
1820 struct static_opts_s {
1821 /* Whether or not allocation size may overflow. */
1822 bool may_overflow;
1823
1824 /*
1825 * Whether or not allocations (with alignment) of size 0 should be
1826 * treated as size 1.
1827 */
1828 bool bump_empty_aligned_alloc;
1829 /*
1830 * Whether to assert that allocations are not of size 0 (after any
1831 * bumping).
1832 */
1833 bool assert_nonempty_alloc;
1834
1835 /*
1836 * Whether or not to modify the 'result' argument to malloc in case of
1837 * error.
1838 */
1839 bool null_out_result_on_error;
1840 /* Whether to set errno when we encounter an error condition. */
1841 bool set_errno_on_error;
1842
1843 /*
1844 * The minimum valid alignment for functions requesting aligned storage.
1845 */
1846 size_t min_alignment;
1847
1848 /* The error string to use if we oom. */
1849 const char *oom_string;
1850 /* The error string to use if the passed-in alignment is invalid. */
1851 const char *invalid_alignment_string;
1852
1853 /*
1854 * False if we're configured to skip some time-consuming operations.
1855 *
1856 * This isn't really a malloc "behavior", but it acts as a useful
1857 * summary of several other static (or at least, static after program
1858 * initialization) options.
1859 */
1860 bool slow;
1861 /*
1862 * Return size.
1863 */
1864 bool usize;
1865 };
1866
1867 JEMALLOC_ALWAYS_INLINE void
static_opts_init(static_opts_t * static_opts)1868 static_opts_init(static_opts_t *static_opts) {
1869 static_opts->may_overflow = false;
1870 static_opts->bump_empty_aligned_alloc = false;
1871 static_opts->assert_nonempty_alloc = false;
1872 static_opts->null_out_result_on_error = false;
1873 static_opts->set_errno_on_error = false;
1874 static_opts->min_alignment = 0;
1875 static_opts->oom_string = "";
1876 static_opts->invalid_alignment_string = "";
1877 static_opts->slow = false;
1878 static_opts->usize = false;
1879 }
1880
1881 /*
1882 * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we
1883 * should have one constant here per magic value there. Note however that the
1884 * representations need not be related.
1885 */
1886 #define TCACHE_IND_NONE ((unsigned)-1)
1887 #define TCACHE_IND_AUTOMATIC ((unsigned)-2)
1888 #define ARENA_IND_AUTOMATIC ((unsigned)-1)
1889
1890 typedef struct dynamic_opts_s dynamic_opts_t;
1891 struct dynamic_opts_s {
1892 void **result;
1893 size_t usize;
1894 size_t num_items;
1895 size_t item_size;
1896 size_t alignment;
1897 bool zero;
1898 unsigned tcache_ind;
1899 unsigned arena_ind;
1900 };
1901
1902 JEMALLOC_ALWAYS_INLINE void
dynamic_opts_init(dynamic_opts_t * dynamic_opts)1903 dynamic_opts_init(dynamic_opts_t *dynamic_opts) {
1904 dynamic_opts->result = NULL;
1905 dynamic_opts->usize = 0;
1906 dynamic_opts->num_items = 0;
1907 dynamic_opts->item_size = 0;
1908 dynamic_opts->alignment = 0;
1909 dynamic_opts->zero = false;
1910 dynamic_opts->tcache_ind = TCACHE_IND_AUTOMATIC;
1911 dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC;
1912 }
1913
1914 /* ind is ignored if dopts->alignment > 0. */
1915 JEMALLOC_ALWAYS_INLINE void *
imalloc_no_sample(static_opts_t * sopts,dynamic_opts_t * dopts,tsd_t * tsd,size_t size,size_t usize,szind_t ind)1916 imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
1917 size_t size, size_t usize, szind_t ind) {
1918 tcache_t *tcache;
1919 arena_t *arena;
1920
1921 /* Fill in the tcache. */
1922 if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) {
1923 if (likely(!sopts->slow)) {
1924 /* Getting tcache ptr unconditionally. */
1925 tcache = tsd_tcachep_get(tsd);
1926 assert(tcache == tcache_get(tsd));
1927 } else {
1928 tcache = tcache_get(tsd);
1929 }
1930 } else if (dopts->tcache_ind == TCACHE_IND_NONE) {
1931 tcache = NULL;
1932 } else {
1933 tcache = tcaches_get(tsd, dopts->tcache_ind);
1934 }
1935
1936 /* Fill in the arena. */
1937 if (dopts->arena_ind == ARENA_IND_AUTOMATIC) {
1938 /*
1939 * In case of automatic arena management, we defer arena
1940 * computation until as late as we can, hoping to fill the
1941 * allocation out of the tcache.
1942 */
1943 arena = NULL;
1944 } else {
1945 arena = arena_get(tsd_tsdn(tsd), dopts->arena_ind, true);
1946 }
1947
1948 if (unlikely(dopts->alignment != 0)) {
1949 return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment,
1950 dopts->zero, tcache, arena);
1951 }
1952
1953 return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false,
1954 arena, sopts->slow);
1955 }
1956
1957 JEMALLOC_ALWAYS_INLINE void *
imalloc_sample(static_opts_t * sopts,dynamic_opts_t * dopts,tsd_t * tsd,size_t usize,szind_t ind)1958 imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
1959 size_t usize, szind_t ind) {
1960 void *ret;
1961
1962 /*
1963 * For small allocations, sampling bumps the usize. If so, we allocate
1964 * from the ind_large bucket.
1965 */
1966 szind_t ind_large;
1967 size_t bumped_usize = usize;
1968
1969 if (usize <= SC_SMALL_MAXCLASS) {
1970 assert(((dopts->alignment == 0) ?
1971 sz_s2u(SC_LARGE_MINCLASS) :
1972 sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment))
1973 == SC_LARGE_MINCLASS);
1974 ind_large = sz_size2index(SC_LARGE_MINCLASS);
1975 bumped_usize = sz_s2u(SC_LARGE_MINCLASS);
1976 ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize,
1977 bumped_usize, ind_large);
1978 if (unlikely(ret == NULL)) {
1979 return NULL;
1980 }
1981 arena_prof_promote(tsd_tsdn(tsd), ret, usize);
1982 } else {
1983 ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind);
1984 }
1985
1986 return ret;
1987 }
1988
1989 /*
1990 * Returns true if the allocation will overflow, and false otherwise. Sets
1991 * *size to the product either way.
1992 */
1993 JEMALLOC_ALWAYS_INLINE bool
compute_size_with_overflow(bool may_overflow,dynamic_opts_t * dopts,size_t * size)1994 compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts,
1995 size_t *size) {
1996 /*
1997 * This function is just num_items * item_size, except that we may have
1998 * to check for overflow.
1999 */
2000
2001 if (!may_overflow) {
2002 assert(dopts->num_items == 1);
2003 *size = dopts->item_size;
2004 return false;
2005 }
2006
2007 /* A size_t with its high-half bits all set to 1. */
2008 static const size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2);
2009
2010 *size = dopts->item_size * dopts->num_items;
2011
2012 if (unlikely(*size == 0)) {
2013 return (dopts->num_items != 0 && dopts->item_size != 0);
2014 }
2015
2016 /*
2017 * We got a non-zero size, but we don't know if we overflowed to get
2018 * there. To avoid having to do a divide, we'll be clever and note that
2019 * if both A and B can be represented in N/2 bits, then their product
2020 * can be represented in N bits (without the possibility of overflow).
2021 */
2022 if (likely((high_bits & (dopts->num_items | dopts->item_size)) == 0)) {
2023 return false;
2024 }
2025 if (likely(*size / dopts->item_size == dopts->num_items)) {
2026 return false;
2027 }
2028 return true;
2029 }
2030
2031 JEMALLOC_ALWAYS_INLINE int
imalloc_body(static_opts_t * sopts,dynamic_opts_t * dopts,tsd_t * tsd)2032 imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
2033 /* Where the actual allocated memory will live. */
2034 void *allocation = NULL;
2035 /* Filled in by compute_size_with_overflow below. */
2036 size_t size = 0;
2037 /*
2038 * For unaligned allocations, we need only ind. For aligned
2039 * allocations, or in case of stats or profiling we need usize.
2040 *
2041 * These are actually dead stores, in that their values are reset before
2042 * any branch on their value is taken. Sometimes though, it's
2043 * convenient to pass them as arguments before this point. To avoid
2044 * undefined behavior then, we initialize them with dummy stores.
2045 */
2046 szind_t ind = 0;
2047 size_t usize = 0;
2048
2049 /* Reentrancy is only checked on slow path. */
2050 int8_t reentrancy_level;
2051
2052 /* Compute the amount of memory the user wants. */
2053 if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts,
2054 &size))) {
2055 goto label_oom;
2056 }
2057
2058 if (unlikely(dopts->alignment < sopts->min_alignment
2059 || (dopts->alignment & (dopts->alignment - 1)) != 0)) {
2060 goto label_invalid_alignment;
2061 }
2062
2063 /* This is the beginning of the "core" algorithm. */
2064
2065 if (dopts->alignment == 0) {
2066 ind = sz_size2index(size);
2067 if (unlikely(ind >= SC_NSIZES)) {
2068 goto label_oom;
2069 }
2070 if (config_stats || (config_prof && opt_prof) || sopts->usize) {
2071 usize = sz_index2size(ind);
2072 dopts->usize = usize;
2073 assert(usize > 0 && usize
2074 <= SC_LARGE_MAXCLASS);
2075 }
2076 } else {
2077 if (sopts->bump_empty_aligned_alloc) {
2078 if (unlikely(size == 0)) {
2079 size = 1;
2080 }
2081 }
2082 usize = sz_sa2u(size, dopts->alignment);
2083 dopts->usize = usize;
2084 if (unlikely(usize == 0
2085 || usize > SC_LARGE_MAXCLASS)) {
2086 goto label_oom;
2087 }
2088 }
2089 /* Validate the user input. */
2090 if (sopts->assert_nonempty_alloc) {
2091 assert (size != 0);
2092 }
2093
2094 check_entry_exit_locking(tsd_tsdn(tsd));
2095
2096 /*
2097 * If we need to handle reentrancy, we can do it out of a
2098 * known-initialized arena (i.e. arena 0).
2099 */
2100 reentrancy_level = tsd_reentrancy_level_get(tsd);
2101 if (sopts->slow && unlikely(reentrancy_level > 0)) {
2102 /*
2103 * We should never specify particular arenas or tcaches from
2104 * within our internal allocations.
2105 */
2106 assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC ||
2107 dopts->tcache_ind == TCACHE_IND_NONE);
2108 assert(dopts->arena_ind == ARENA_IND_AUTOMATIC);
2109 dopts->tcache_ind = TCACHE_IND_NONE;
2110 /* We know that arena 0 has already been initialized. */
2111 dopts->arena_ind = 0;
2112 }
2113
2114 /* If profiling is on, get our profiling context. */
2115 if (config_prof && opt_prof) {
2116 /*
2117 * Note that if we're going down this path, usize must have been
2118 * initialized in the previous if statement.
2119 */
2120 prof_tctx_t *tctx = prof_alloc_prep(
2121 tsd, usize, prof_active_get_unlocked(), true);
2122
2123 alloc_ctx_t alloc_ctx;
2124 if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
2125 alloc_ctx.slab = (usize
2126 <= SC_SMALL_MAXCLASS);
2127 allocation = imalloc_no_sample(
2128 sopts, dopts, tsd, usize, usize, ind);
2129 } else if ((uintptr_t)tctx > (uintptr_t)1U) {
2130 /*
2131 * Note that ind might still be 0 here. This is fine;
2132 * imalloc_sample ignores ind if dopts->alignment > 0.
2133 */
2134 allocation = imalloc_sample(
2135 sopts, dopts, tsd, usize, ind);
2136 alloc_ctx.slab = false;
2137 } else {
2138 allocation = NULL;
2139 }
2140
2141 if (unlikely(allocation == NULL)) {
2142 prof_alloc_rollback(tsd, tctx, true);
2143 goto label_oom;
2144 }
2145 prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx);
2146 } else {
2147 /*
2148 * If dopts->alignment > 0, then ind is still 0, but usize was
2149 * computed in the previous if statement. Down the positive
2150 * alignment path, imalloc_no_sample ignores ind and size
2151 * (relying only on usize).
2152 */
2153 allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
2154 ind);
2155 if (unlikely(allocation == NULL)) {
2156 goto label_oom;
2157 }
2158 }
2159
2160 /*
2161 * Allocation has been done at this point. We still have some
2162 * post-allocation work to do though.
2163 */
2164 assert(dopts->alignment == 0
2165 || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));
2166
2167 if (config_stats) {
2168 assert(usize == isalloc(tsd_tsdn(tsd), allocation));
2169 *tsd_thread_allocatedp_get(tsd) += usize;
2170 }
2171
2172 if (sopts->slow) {
2173 UTRACE(0, size, allocation);
2174 }
2175
2176 /* Success! */
2177 check_entry_exit_locking(tsd_tsdn(tsd));
2178 *dopts->result = allocation;
2179 return 0;
2180
2181 label_oom:
2182 if (unlikely(sopts->slow) && config_xmalloc && unlikely(opt_xmalloc)) {
2183 malloc_write(sopts->oom_string);
2184 abort();
2185 }
2186
2187 if (sopts->slow) {
2188 UTRACE(NULL, size, NULL);
2189 }
2190
2191 check_entry_exit_locking(tsd_tsdn(tsd));
2192
2193 if (sopts->set_errno_on_error) {
2194 set_errno(ENOMEM);
2195 }
2196
2197 if (sopts->null_out_result_on_error) {
2198 *dopts->result = NULL;
2199 }
2200
2201 return ENOMEM;
2202
2203 /*
2204 * This label is only jumped to by one goto; we move it out of line
2205 * anyways to avoid obscuring the non-error paths, and for symmetry with
2206 * the oom case.
2207 */
2208 label_invalid_alignment:
2209 if (config_xmalloc && unlikely(opt_xmalloc)) {
2210 malloc_write(sopts->invalid_alignment_string);
2211 abort();
2212 }
2213
2214 if (sopts->set_errno_on_error) {
2215 set_errno(EINVAL);
2216 }
2217
2218 if (sopts->slow) {
2219 UTRACE(NULL, size, NULL);
2220 }
2221
2222 check_entry_exit_locking(tsd_tsdn(tsd));
2223
2224 if (sopts->null_out_result_on_error) {
2225 *dopts->result = NULL;
2226 }
2227
2228 return EINVAL;
2229 }
2230
2231 JEMALLOC_ALWAYS_INLINE bool
imalloc_init_check(static_opts_t * sopts,dynamic_opts_t * dopts)2232 imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) {
2233 if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) {
2234 if (config_xmalloc && unlikely(opt_xmalloc)) {
2235 malloc_write(sopts->oom_string);
2236 abort();
2237 }
2238 UTRACE(NULL, dopts->num_items * dopts->item_size, NULL);
2239 set_errno(ENOMEM);
2240 *dopts->result = NULL;
2241
2242 return false;
2243 }
2244
2245 return true;
2246 }
2247
2248 /* Returns the errno-style error code of the allocation. */
2249 JEMALLOC_ALWAYS_INLINE int
imalloc(static_opts_t * sopts,dynamic_opts_t * dopts)2250 imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) {
2251 if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
2252 return ENOMEM;
2253 }
2254
2255 /* We always need the tsd. Let's grab it right away. */
2256 tsd_t *tsd = tsd_fetch();
2257 assert(tsd);
2258 if (likely(tsd_fast(tsd))) {
2259 /* Fast and common path. */
2260 tsd_assert_fast(tsd);
2261 sopts->slow = false;
2262 return imalloc_body(sopts, dopts, tsd);
2263 } else {
2264 if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
2265 return ENOMEM;
2266 }
2267
2268 sopts->slow = true;
2269 return imalloc_body(sopts, dopts, tsd);
2270 }
2271 }
2272
2273 JEMALLOC_NOINLINE
2274 void *
malloc_default(size_t size)2275 malloc_default(size_t size) {
2276 void *ret;
2277 static_opts_t sopts;
2278 dynamic_opts_t dopts;
2279
2280 LOG("core.malloc.entry", "size: %zu", size);
2281
2282 static_opts_init(&sopts);
2283 dynamic_opts_init(&dopts);
2284
2285 sopts.null_out_result_on_error = true;
2286 sopts.set_errno_on_error = true;
2287 sopts.oom_string = "<jemalloc>: Error in malloc(): out of memory\n";
2288
2289 dopts.result = &ret;
2290 dopts.num_items = 1;
2291 dopts.item_size = size;
2292
2293 imalloc(&sopts, &dopts);
2294 /*
2295 * Note that this branch gets optimized away -- it immediately follows
2296 * the check on tsd_fast that sets sopts.slow.
2297 */
2298 if (sopts.slow) {
2299 uintptr_t args[3] = {size};
2300 hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args);
2301 }
2302
2303 LOG("core.malloc.exit", "result: %p", ret);
2304
2305 return ret;
2306 }
2307
2308 /******************************************************************************/
2309 /*
2310 * Begin malloc(3)-compatible functions.
2311 */
2312
2313 /*
2314 * malloc() fastpath.
2315 *
2316 * Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit
2317 * tcache. If either of these is false, we tail-call to the slowpath,
2318 * malloc_default(). Tail-calling is used to avoid any caller-saved
2319 * registers.
2320 *
2321 * fastpath supports ticker and profiling, both of which will also
2322 * tail-call to the slowpath if they fire.
2323 */
2324 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2325 void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)2326 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
2327 je_malloc(size_t size) {
2328 LOG("core.malloc.entry", "size: %zu", size);
2329
2330 if (tsd_get_allocates() && unlikely(!malloc_initialized())) {
2331 return malloc_default(size);
2332 }
2333
2334 tsd_t *tsd = tsd_get(false);
2335 if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) {
2336 return malloc_default(size);
2337 }
2338
2339 tcache_t *tcache = tsd_tcachep_get(tsd);
2340
2341 if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
2342 return malloc_default(size);
2343 }
2344
2345 szind_t ind = sz_size2index_lookup(size);
2346 size_t usize;
2347 if (config_stats || config_prof) {
2348 usize = sz_index2size(ind);
2349 }
2350 /* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */
2351 assert(ind < SC_NBINS);
2352 assert(size <= SC_SMALL_MAXCLASS);
2353
2354 if (config_prof) {
2355 int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd);
2356 bytes_until_sample -= usize;
2357 tsd_bytes_until_sample_set(tsd, bytes_until_sample);
2358
2359 if (unlikely(bytes_until_sample < 0)) {
2360 /*
2361 * Avoid a prof_active check on the fastpath.
2362 * If prof_active is false, set bytes_until_sample to
2363 * a large value. If prof_active is set to true,
2364 * bytes_until_sample will be reset.
2365 */
2366 if (!prof_active) {
2367 tsd_bytes_until_sample_set(tsd, SSIZE_MAX);
2368 }
2369 return malloc_default(size);
2370 }
2371 }
2372
2373 cache_bin_t *bin = tcache_small_bin_get(tcache, ind);
2374 bool tcache_success;
2375 void* ret = cache_bin_alloc_easy(bin, &tcache_success);
2376
2377 if (tcache_success) {
2378 if (config_stats) {
2379 *tsd_thread_allocatedp_get(tsd) += usize;
2380 bin->tstats.nrequests++;
2381 }
2382 if (config_prof) {
2383 tcache->prof_accumbytes += usize;
2384 }
2385
2386 LOG("core.malloc.exit", "result: %p", ret);
2387
2388 /* Fastpath success */
2389 return ret;
2390 }
2391
2392 return malloc_default(size);
2393 }
2394
2395 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2396 JEMALLOC_ATTR(nonnull(1))
je_posix_memalign(void ** memptr,size_t alignment,size_t size)2397 je_posix_memalign(void **memptr, size_t alignment, size_t size) {
2398 int ret;
2399 static_opts_t sopts;
2400 dynamic_opts_t dopts;
2401
2402 LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, "
2403 "size: %zu", memptr, alignment, size);
2404
2405 static_opts_init(&sopts);
2406 dynamic_opts_init(&dopts);
2407
2408 sopts.bump_empty_aligned_alloc = true;
2409 sopts.min_alignment = sizeof(void *);
2410 sopts.oom_string =
2411 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2412 sopts.invalid_alignment_string =
2413 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2414
2415 dopts.result = memptr;
2416 dopts.num_items = 1;
2417 dopts.item_size = size;
2418 dopts.alignment = alignment;
2419
2420 ret = imalloc(&sopts, &dopts);
2421 if (sopts.slow) {
2422 uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment,
2423 (uintptr_t)size};
2424 hook_invoke_alloc(hook_alloc_posix_memalign, *memptr,
2425 (uintptr_t)ret, args);
2426 }
2427
2428 LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret,
2429 *memptr);
2430
2431 return ret;
2432 }
2433
2434 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2435 void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)2436 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2)
2437 je_aligned_alloc(size_t alignment, size_t size) {
2438 void *ret;
2439
2440 static_opts_t sopts;
2441 dynamic_opts_t dopts;
2442
2443 LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n",
2444 alignment, size);
2445
2446 static_opts_init(&sopts);
2447 dynamic_opts_init(&dopts);
2448
2449 sopts.bump_empty_aligned_alloc = true;
2450 sopts.null_out_result_on_error = true;
2451 sopts.set_errno_on_error = true;
2452 sopts.min_alignment = 1;
2453 sopts.oom_string =
2454 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2455 sopts.invalid_alignment_string =
2456 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2457
2458 dopts.result = &ret;
2459 dopts.num_items = 1;
2460 dopts.item_size = size;
2461 dopts.alignment = alignment;
2462
2463 imalloc(&sopts, &dopts);
2464 if (sopts.slow) {
2465 uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size};
2466 hook_invoke_alloc(hook_alloc_aligned_alloc, ret,
2467 (uintptr_t)ret, args);
2468 }
2469
2470 LOG("core.aligned_alloc.exit", "result: %p", ret);
2471
2472 return ret;
2473 }
2474
2475 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2476 void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)2477 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
2478 je_calloc(size_t num, size_t size) {
2479 void *ret;
2480 static_opts_t sopts;
2481 dynamic_opts_t dopts;
2482
2483 LOG("core.calloc.entry", "num: %zu, size: %zu\n", num, size);
2484
2485 static_opts_init(&sopts);
2486 dynamic_opts_init(&dopts);
2487
2488 sopts.may_overflow = true;
2489 sopts.null_out_result_on_error = true;
2490 sopts.set_errno_on_error = true;
2491 sopts.oom_string = "<jemalloc>: Error in calloc(): out of memory\n";
2492
2493 dopts.result = &ret;
2494 dopts.num_items = num;
2495 dopts.item_size = size;
2496 dopts.zero = true;
2497
2498 imalloc(&sopts, &dopts);
2499 if (sopts.slow) {
2500 uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size};
2501 hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args);
2502 }
2503
2504 LOG("core.calloc.exit", "result: %p", ret);
2505
2506 return ret;
2507 }
2508
2509 static void *
irealloc_prof_sample(tsd_t * tsd,void * old_ptr,size_t old_usize,size_t usize,prof_tctx_t * tctx,hook_ralloc_args_t * hook_args)2510 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
2511 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
2512 void *p;
2513
2514 if (tctx == NULL) {
2515 return NULL;
2516 }
2517 if (usize <= SC_SMALL_MAXCLASS) {
2518 p = iralloc(tsd, old_ptr, old_usize,
2519 SC_LARGE_MINCLASS, 0, false, hook_args);
2520 if (p == NULL) {
2521 return NULL;
2522 }
2523 arena_prof_promote(tsd_tsdn(tsd), p, usize);
2524 } else {
2525 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
2526 hook_args);
2527 }
2528
2529 return p;
2530 }
2531
2532 JEMALLOC_ALWAYS_INLINE void *
irealloc_prof(tsd_t * tsd,void * old_ptr,size_t old_usize,size_t usize,alloc_ctx_t * alloc_ctx,hook_ralloc_args_t * hook_args)2533 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
2534 alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
2535 void *p;
2536 bool prof_active;
2537 prof_tctx_t *old_tctx, *tctx;
2538
2539 prof_active = prof_active_get_unlocked();
2540 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
2541 tctx = prof_alloc_prep(tsd, usize, prof_active, true);
2542 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
2543 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx,
2544 hook_args);
2545 } else {
2546 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
2547 hook_args);
2548 }
2549 if (unlikely(p == NULL)) {
2550 prof_alloc_rollback(tsd, tctx, true);
2551 return NULL;
2552 }
2553 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
2554 old_tctx);
2555
2556 return p;
2557 }
2558
2559 JEMALLOC_ALWAYS_INLINE void
ifree(tsd_t * tsd,void * ptr,tcache_t * tcache,bool slow_path)2560 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
2561 if (!slow_path) {
2562 tsd_assert_fast(tsd);
2563 }
2564 check_entry_exit_locking(tsd_tsdn(tsd));
2565 if (tsd_reentrancy_level_get(tsd) != 0) {
2566 assert(slow_path);
2567 }
2568
2569 assert(ptr != NULL);
2570 assert(malloc_initialized() || IS_INITIALIZER);
2571
2572 alloc_ctx_t alloc_ctx;
2573 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2574 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2575 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2576 assert(alloc_ctx.szind != SC_NSIZES);
2577
2578 size_t usize;
2579 if (config_prof && opt_prof) {
2580 usize = sz_index2size(alloc_ctx.szind);
2581 prof_free(tsd, ptr, usize, &alloc_ctx);
2582 } else if (config_stats) {
2583 usize = sz_index2size(alloc_ctx.szind);
2584 }
2585 if (config_stats) {
2586 *tsd_thread_deallocatedp_get(tsd) += usize;
2587 }
2588
2589 if (likely(!slow_path)) {
2590 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
2591 false);
2592 } else {
2593 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
2594 true);
2595 }
2596 }
2597
2598 JEMALLOC_ALWAYS_INLINE void
isfree(tsd_t * tsd,void * ptr,size_t usize,tcache_t * tcache,bool slow_path)2599 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
2600 if (!slow_path) {
2601 tsd_assert_fast(tsd);
2602 }
2603 check_entry_exit_locking(tsd_tsdn(tsd));
2604 if (tsd_reentrancy_level_get(tsd) != 0) {
2605 assert(slow_path);
2606 }
2607
2608 assert(ptr != NULL);
2609 assert(malloc_initialized() || IS_INITIALIZER);
2610
2611 alloc_ctx_t alloc_ctx, *ctx;
2612 if (!config_cache_oblivious && ((uintptr_t)ptr & PAGE_MASK) != 0) {
2613 /*
2614 * When cache_oblivious is disabled and ptr is not page aligned,
2615 * the allocation was not sampled -- usize can be used to
2616 * determine szind directly.
2617 */
2618 alloc_ctx.szind = sz_size2index(usize);
2619 alloc_ctx.slab = true;
2620 ctx = &alloc_ctx;
2621 if (config_debug) {
2622 alloc_ctx_t dbg_ctx;
2623 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2624 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree,
2625 rtree_ctx, (uintptr_t)ptr, true, &dbg_ctx.szind,
2626 &dbg_ctx.slab);
2627 assert(dbg_ctx.szind == alloc_ctx.szind);
2628 assert(dbg_ctx.slab == alloc_ctx.slab);
2629 }
2630 } else if (config_prof && opt_prof) {
2631 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2632 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2633 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2634 assert(alloc_ctx.szind == sz_size2index(usize));
2635 ctx = &alloc_ctx;
2636 } else {
2637 ctx = NULL;
2638 }
2639
2640 if (config_prof && opt_prof) {
2641 prof_free(tsd, ptr, usize, ctx);
2642 }
2643 if (config_stats) {
2644 *tsd_thread_deallocatedp_get(tsd) += usize;
2645 }
2646
2647 if (likely(!slow_path)) {
2648 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false);
2649 } else {
2650 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true);
2651 }
2652 }
2653
2654 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2655 void JEMALLOC_NOTHROW *
2656 JEMALLOC_ALLOC_SIZE(2)
je_realloc(void * ptr,size_t arg_size)2657 je_realloc(void *ptr, size_t arg_size) {
2658 void *ret;
2659 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL);
2660 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
2661 size_t old_usize = 0;
2662 size_t size = arg_size;
2663
2664 LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
2665
2666 if (unlikely(size == 0)) {
2667 size = 1;
2668 }
2669
2670 if (likely(ptr != NULL)) {
2671 assert(malloc_initialized() || IS_INITIALIZER);
2672 tsd_t *tsd = tsd_fetch();
2673
2674 check_entry_exit_locking(tsd_tsdn(tsd));
2675
2676
2677 hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr,
2678 (uintptr_t)arg_size, 0, 0}};
2679
2680 alloc_ctx_t alloc_ctx;
2681 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2682 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2683 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2684 assert(alloc_ctx.szind != SC_NSIZES);
2685 old_usize = sz_index2size(alloc_ctx.szind);
2686 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
2687 if (config_prof && opt_prof) {
2688 usize = sz_s2u(size);
2689 if (unlikely(usize == 0
2690 || usize > SC_LARGE_MAXCLASS)) {
2691 ret = NULL;
2692 } else {
2693 ret = irealloc_prof(tsd, ptr, old_usize, usize,
2694 &alloc_ctx, &hook_args);
2695 }
2696 } else {
2697 if (config_stats) {
2698 usize = sz_s2u(size);
2699 }
2700 ret = iralloc(tsd, ptr, old_usize, size, 0, false,
2701 &hook_args);
2702 }
2703 tsdn = tsd_tsdn(tsd);
2704 } else {
2705 /* realloc(NULL, size) is equivalent to malloc(size). */
2706 static_opts_t sopts;
2707 dynamic_opts_t dopts;
2708
2709 static_opts_init(&sopts);
2710 dynamic_opts_init(&dopts);
2711
2712 sopts.null_out_result_on_error = true;
2713 sopts.set_errno_on_error = true;
2714 sopts.oom_string =
2715 "<jemalloc>: Error in realloc(): out of memory\n";
2716
2717 dopts.result = &ret;
2718 dopts.num_items = 1;
2719 dopts.item_size = size;
2720
2721 imalloc(&sopts, &dopts);
2722 if (sopts.slow) {
2723 uintptr_t args[3] = {(uintptr_t)ptr, arg_size};
2724 hook_invoke_alloc(hook_alloc_realloc, ret,
2725 (uintptr_t)ret, args);
2726 }
2727
2728 return ret;
2729 }
2730
2731 if (unlikely(ret == NULL)) {
2732 if (config_xmalloc && unlikely(opt_xmalloc)) {
2733 malloc_write("<jemalloc>: Error in realloc(): "
2734 "out of memory\n");
2735 abort();
2736 }
2737 set_errno(ENOMEM);
2738 }
2739 if (config_stats && likely(ret != NULL)) {
2740 tsd_t *tsd;
2741
2742 assert(usize == isalloc(tsdn, ret));
2743 tsd = tsdn_tsd(tsdn);
2744 *tsd_thread_allocatedp_get(tsd) += usize;
2745 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2746 }
2747 UTRACE(ptr, size, ret);
2748 check_entry_exit_locking(tsdn);
2749
2750 LOG("core.realloc.exit", "result: %p", ret);
2751 return ret;
2752 }
2753
2754 JEMALLOC_NOINLINE
2755 void
free_default(void * ptr)2756 free_default(void *ptr) {
2757 UTRACE(ptr, 0, 0);
2758 if (likely(ptr != NULL)) {
2759 /*
2760 * We avoid setting up tsd fully (e.g. tcache, arena binding)
2761 * based on only free() calls -- other activities trigger the
2762 * minimal to full transition. This is because free() may
2763 * happen during thread shutdown after tls deallocation: if a
2764 * thread never had any malloc activities until then, a
2765 * fully-setup tsd won't be destructed properly.
2766 */
2767 tsd_t *tsd = tsd_fetch_min();
2768 check_entry_exit_locking(tsd_tsdn(tsd));
2769
2770 tcache_t *tcache;
2771 if (likely(tsd_fast(tsd))) {
2772 tsd_assert_fast(tsd);
2773 /* Unconditionally get tcache ptr on fast path. */
2774 tcache = tsd_tcachep_get(tsd);
2775 ifree(tsd, ptr, tcache, false);
2776 } else {
2777 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
2778 tcache = tcache_get(tsd);
2779 } else {
2780 tcache = NULL;
2781 }
2782 uintptr_t args_raw[3] = {(uintptr_t)ptr};
2783 hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw);
2784 ifree(tsd, ptr, tcache, true);
2785 }
2786 check_entry_exit_locking(tsd_tsdn(tsd));
2787 }
2788 }
2789
2790 JEMALLOC_ALWAYS_INLINE
free_fastpath(void * ptr,size_t size,bool size_hint)2791 bool free_fastpath(void *ptr, size_t size, bool size_hint) {
2792 tsd_t *tsd = tsd_get(false);
2793 if (unlikely(!tsd || !tsd_fast(tsd))) {
2794 return false;
2795 }
2796
2797 tcache_t *tcache = tsd_tcachep_get(tsd);
2798
2799 alloc_ctx_t alloc_ctx;
2800 /*
2801 * If !config_cache_oblivious, we can check PAGE alignment to
2802 * detect sampled objects. Otherwise addresses are
2803 * randomized, and we have to look it up in the rtree anyway.
2804 * See also isfree().
2805 */
2806 if (!size_hint || config_cache_oblivious) {
2807 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2808 bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree,
2809 rtree_ctx, (uintptr_t)ptr,
2810 &alloc_ctx.szind, &alloc_ctx.slab);
2811
2812 /* Note: profiled objects will have alloc_ctx.slab set */
2813 if (!res || !alloc_ctx.slab) {
2814 return false;
2815 }
2816 assert(alloc_ctx.szind != SC_NSIZES);
2817 } else {
2818 /*
2819 * Check for both sizes that are too large, and for sampled objects.
2820 * Sampled objects are always page-aligned. The sampled object check
2821 * will also check for null ptr.
2822 */
2823 if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) {
2824 return false;
2825 }
2826 alloc_ctx.szind = sz_size2index_lookup(size);
2827 }
2828
2829 if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
2830 return false;
2831 }
2832
2833 cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind);
2834 cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind];
2835 if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) {
2836 return false;
2837 }
2838
2839 if (config_stats) {
2840 size_t usize = sz_index2size(alloc_ctx.szind);
2841 *tsd_thread_deallocatedp_get(tsd) += usize;
2842 }
2843
2844 return true;
2845 }
2846
2847 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_free(void * ptr)2848 je_free(void *ptr) {
2849 LOG("core.free.entry", "ptr: %p", ptr);
2850
2851 if (!free_fastpath(ptr, 0, false)) {
2852 free_default(ptr);
2853 }
2854
2855 LOG("core.free.exit", "");
2856 }
2857
2858 /*
2859 * End malloc(3)-compatible functions.
2860 */
2861 /******************************************************************************/
2862 /*
2863 * Begin non-standard override functions.
2864 */
2865
2866 #ifdef JEMALLOC_OVERRIDE_MEMALIGN
2867 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2868 void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)2869 JEMALLOC_ATTR(malloc)
2870 je_memalign(size_t alignment, size_t size) {
2871 void *ret;
2872 static_opts_t sopts;
2873 dynamic_opts_t dopts;
2874
2875 LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment,
2876 size);
2877
2878 static_opts_init(&sopts);
2879 dynamic_opts_init(&dopts);
2880
2881 sopts.min_alignment = 1;
2882 sopts.oom_string =
2883 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2884 sopts.invalid_alignment_string =
2885 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2886 sopts.null_out_result_on_error = true;
2887
2888 dopts.result = &ret;
2889 dopts.num_items = 1;
2890 dopts.item_size = size;
2891 dopts.alignment = alignment;
2892
2893 imalloc(&sopts, &dopts);
2894 if (sopts.slow) {
2895 uintptr_t args[3] = {alignment, size};
2896 hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret,
2897 args);
2898 }
2899
2900 LOG("core.memalign.exit", "result: %p", ret);
2901 return ret;
2902 }
2903 #endif
2904
2905 #ifdef JEMALLOC_OVERRIDE_VALLOC
2906 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2907 void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)2908 JEMALLOC_ATTR(malloc)
2909 je_valloc(size_t size) {
2910 void *ret;
2911
2912 static_opts_t sopts;
2913 dynamic_opts_t dopts;
2914
2915 LOG("core.valloc.entry", "size: %zu\n", size);
2916
2917 static_opts_init(&sopts);
2918 dynamic_opts_init(&dopts);
2919
2920 sopts.null_out_result_on_error = true;
2921 sopts.min_alignment = PAGE;
2922 sopts.oom_string =
2923 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2924 sopts.invalid_alignment_string =
2925 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2926
2927 dopts.result = &ret;
2928 dopts.num_items = 1;
2929 dopts.item_size = size;
2930 dopts.alignment = PAGE;
2931
2932 imalloc(&sopts, &dopts);
2933 if (sopts.slow) {
2934 uintptr_t args[3] = {size};
2935 hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args);
2936 }
2937
2938 LOG("core.valloc.exit", "result: %p\n", ret);
2939 return ret;
2940 }
2941 #endif
2942
2943 #if defined(JEMALLOC_IS_MALLOC) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)
2944 /*
2945 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
2946 * to inconsistently reference libc's malloc(3)-compatible functions
2947 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
2948 *
2949 * These definitions interpose hooks in glibc. The functions are actually
2950 * passed an extra argument for the caller return address, which will be
2951 * ignored.
2952 */
2953 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
2954 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
2955 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
2956 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK
2957 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
2958 je_memalign;
2959 # endif
2960
2961 # ifdef CPU_COUNT
2962 /*
2963 * To enable static linking with glibc, the libc specific malloc interface must
2964 * be implemented also, so none of glibc's malloc.o functions are added to the
2965 * link.
2966 */
2967 # define ALIAS(je_fn) __attribute__((alias (#je_fn), used))
2968 /* To force macro expansion of je_ prefix before stringification. */
2969 # define PREALIAS(je_fn) ALIAS(je_fn)
2970 # ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC
2971 void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc);
2972 # endif
2973 # ifdef JEMALLOC_OVERRIDE___LIBC_FREE
2974 void __libc_free(void* ptr) PREALIAS(je_free);
2975 # endif
2976 # ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC
2977 void *__libc_malloc(size_t size) PREALIAS(je_malloc);
2978 # endif
2979 # ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN
2980 void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign);
2981 # endif
2982 # ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC
2983 void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc);
2984 # endif
2985 # ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC
2986 void *__libc_valloc(size_t size) PREALIAS(je_valloc);
2987 # endif
2988 # ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN
2989 int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign);
2990 # endif
2991 # undef PREALIAS
2992 # undef ALIAS
2993 # endif
2994 #endif
2995
2996 /*
2997 * End non-standard override functions.
2998 */
2999 /******************************************************************************/
3000 /*
3001 * Begin non-standard functions.
3002 */
3003
3004 #ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API
3005
3006 #define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y
3007 #define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \
3008 JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y)
3009
3010 typedef struct {
3011 void *ptr;
3012 size_t size;
3013 } smallocx_return_t;
3014
3015 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3016 smallocx_return_t JEMALLOC_NOTHROW
3017 /*
3018 * The attribute JEMALLOC_ATTR(malloc) cannot be used due to:
3019 * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488
3020 */
JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_,JEMALLOC_VERSION_GID_IDENT)3021 JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT)
3022 (size_t size, int flags) {
3023 /*
3024 * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be
3025 * used here because it makes writing beyond the `size`
3026 * of the `ptr` undefined behavior, but the objective
3027 * of this function is to allow writing beyond `size`
3028 * up to `smallocx_return_t::size`.
3029 */
3030 smallocx_return_t ret;
3031 static_opts_t sopts;
3032 dynamic_opts_t dopts;
3033
3034 LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags);
3035
3036 static_opts_init(&sopts);
3037 dynamic_opts_init(&dopts);
3038
3039 sopts.assert_nonempty_alloc = true;
3040 sopts.null_out_result_on_error = true;
3041 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
3042 sopts.usize = true;
3043
3044 dopts.result = &ret.ptr;
3045 dopts.num_items = 1;
3046 dopts.item_size = size;
3047 if (unlikely(flags != 0)) {
3048 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) {
3049 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
3050 }
3051
3052 dopts.zero = MALLOCX_ZERO_GET(flags);
3053
3054 if ((flags & MALLOCX_TCACHE_MASK) != 0) {
3055 if ((flags & MALLOCX_TCACHE_MASK)
3056 == MALLOCX_TCACHE_NONE) {
3057 dopts.tcache_ind = TCACHE_IND_NONE;
3058 } else {
3059 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags);
3060 }
3061 } else {
3062 dopts.tcache_ind = TCACHE_IND_AUTOMATIC;
3063 }
3064
3065 if ((flags & MALLOCX_ARENA_MASK) != 0)
3066 dopts.arena_ind = MALLOCX_ARENA_GET(flags);
3067 }
3068
3069 imalloc(&sopts, &dopts);
3070 assert(dopts.usize == je_nallocx(size, flags));
3071 ret.size = dopts.usize;
3072
3073 LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size);
3074 return ret;
3075 }
3076 #undef JEMALLOC_SMALLOCX_CONCAT_HELPER
3077 #undef JEMALLOC_SMALLOCX_CONCAT_HELPER2
3078 #endif
3079
3080 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3081 void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)3082 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
3083 je_mallocx(size_t size, int flags) {
3084 void *ret;
3085 static_opts_t sopts;
3086 dynamic_opts_t dopts;
3087
3088 LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags);
3089
3090 static_opts_init(&sopts);
3091 dynamic_opts_init(&dopts);
3092
3093 sopts.assert_nonempty_alloc = true;
3094 sopts.null_out_result_on_error = true;
3095 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
3096
3097 dopts.result = &ret;
3098 dopts.num_items = 1;
3099 dopts.item_size = size;
3100 if (unlikely(flags != 0)) {
3101 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) {
3102 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
3103 }
3104
3105 dopts.zero = MALLOCX_ZERO_GET(flags);
3106
3107 if ((flags & MALLOCX_TCACHE_MASK) != 0) {
3108 if ((flags & MALLOCX_TCACHE_MASK)
3109 == MALLOCX_TCACHE_NONE) {
3110 dopts.tcache_ind = TCACHE_IND_NONE;
3111 } else {
3112 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags);
3113 }
3114 } else {
3115 dopts.tcache_ind = TCACHE_IND_AUTOMATIC;
3116 }
3117
3118 if ((flags & MALLOCX_ARENA_MASK) != 0)
3119 dopts.arena_ind = MALLOCX_ARENA_GET(flags);
3120 }
3121
3122 imalloc(&sopts, &dopts);
3123 if (sopts.slow) {
3124 uintptr_t args[3] = {size, flags};
3125 hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret,
3126 args);
3127 }
3128
3129 LOG("core.mallocx.exit", "result: %p", ret);
3130 return ret;
3131 }
3132
3133 static void *
irallocx_prof_sample(tsdn_t * tsdn,void * old_ptr,size_t old_usize,size_t usize,size_t alignment,bool zero,tcache_t * tcache,arena_t * arena,prof_tctx_t * tctx,hook_ralloc_args_t * hook_args)3134 irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
3135 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
3136 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
3137 void *p;
3138
3139 if (tctx == NULL) {
3140 return NULL;
3141 }
3142 if (usize <= SC_SMALL_MAXCLASS) {
3143 p = iralloct(tsdn, old_ptr, old_usize,
3144 SC_LARGE_MINCLASS, alignment, zero, tcache,
3145 arena, hook_args);
3146 if (p == NULL) {
3147 return NULL;
3148 }
3149 arena_prof_promote(tsdn, p, usize);
3150 } else {
3151 p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero,
3152 tcache, arena, hook_args);
3153 }
3154
3155 return p;
3156 }
3157
3158 JEMALLOC_ALWAYS_INLINE void *
irallocx_prof(tsd_t * tsd,void * old_ptr,size_t old_usize,size_t size,size_t alignment,size_t * usize,bool zero,tcache_t * tcache,arena_t * arena,alloc_ctx_t * alloc_ctx,hook_ralloc_args_t * hook_args)3159 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
3160 size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
3161 arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
3162 void *p;
3163 bool prof_active;
3164 prof_tctx_t *old_tctx, *tctx;
3165
3166 prof_active = prof_active_get_unlocked();
3167 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
3168 tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
3169 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
3170 p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize,
3171 *usize, alignment, zero, tcache, arena, tctx, hook_args);
3172 } else {
3173 p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment,
3174 zero, tcache, arena, hook_args);
3175 }
3176 if (unlikely(p == NULL)) {
3177 prof_alloc_rollback(tsd, tctx, false);
3178 return NULL;
3179 }
3180
3181 if (p == old_ptr && alignment != 0) {
3182 /*
3183 * The allocation did not move, so it is possible that the size
3184 * class is smaller than would guarantee the requested
3185 * alignment, and that the alignment constraint was
3186 * serendipitously satisfied. Additionally, old_usize may not
3187 * be the same as the current usize because of in-place large
3188 * reallocation. Therefore, query the actual value of usize.
3189 */
3190 *usize = isalloc(tsd_tsdn(tsd), p);
3191 }
3192 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
3193 old_usize, old_tctx);
3194
3195 return p;
3196 }
3197
3198 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3199 void JEMALLOC_NOTHROW *
3200 JEMALLOC_ALLOC_SIZE(2)
je_rallocx(void * ptr,size_t size,int flags)3201 je_rallocx(void *ptr, size_t size, int flags) {
3202 void *p;
3203 tsd_t *tsd;
3204 size_t usize;
3205 size_t old_usize;
3206 size_t alignment = MALLOCX_ALIGN_GET(flags);
3207 bool zero = flags & MALLOCX_ZERO;
3208 arena_t *arena;
3209 tcache_t *tcache;
3210
3211 LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
3212 size, flags);
3213
3214
3215 assert(ptr != NULL);
3216 assert(size != 0);
3217 assert(malloc_initialized() || IS_INITIALIZER);
3218 tsd = tsd_fetch();
3219 check_entry_exit_locking(tsd_tsdn(tsd));
3220
3221 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
3222 unsigned arena_ind = MALLOCX_ARENA_GET(flags);
3223 arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
3224 if (unlikely(arena == NULL)) {
3225 goto label_oom;
3226 }
3227 } else {
3228 arena = NULL;
3229 }
3230
3231 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3232 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3233 tcache = NULL;
3234 } else {
3235 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3236 }
3237 } else {
3238 tcache = tcache_get(tsd);
3239 }
3240
3241 alloc_ctx_t alloc_ctx;
3242 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
3243 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
3244 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
3245 assert(alloc_ctx.szind != SC_NSIZES);
3246 old_usize = sz_index2size(alloc_ctx.szind);
3247 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
3248
3249 hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags,
3250 0}};
3251 if (config_prof && opt_prof) {
3252 usize = (alignment == 0) ?
3253 sz_s2u(size) : sz_sa2u(size, alignment);
3254 if (unlikely(usize == 0
3255 || usize > SC_LARGE_MAXCLASS)) {
3256 goto label_oom;
3257 }
3258 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
3259 zero, tcache, arena, &alloc_ctx, &hook_args);
3260 if (unlikely(p == NULL)) {
3261 goto label_oom;
3262 }
3263 } else {
3264 p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment,
3265 zero, tcache, arena, &hook_args);
3266 if (unlikely(p == NULL)) {
3267 goto label_oom;
3268 }
3269 if (config_stats) {
3270 usize = isalloc(tsd_tsdn(tsd), p);
3271 }
3272 }
3273 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
3274
3275 if (config_stats) {
3276 *tsd_thread_allocatedp_get(tsd) += usize;
3277 *tsd_thread_deallocatedp_get(tsd) += old_usize;
3278 }
3279 UTRACE(ptr, size, p);
3280 check_entry_exit_locking(tsd_tsdn(tsd));
3281
3282 LOG("core.rallocx.exit", "result: %p", p);
3283 return p;
3284 label_oom:
3285 if (config_xmalloc && unlikely(opt_xmalloc)) {
3286 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
3287 abort();
3288 }
3289 UTRACE(ptr, size, 0);
3290 check_entry_exit_locking(tsd_tsdn(tsd));
3291
3292 LOG("core.rallocx.exit", "result: %p", NULL);
3293 return NULL;
3294 }
3295
3296 JEMALLOC_ALWAYS_INLINE size_t
ixallocx_helper(tsdn_t * tsdn,void * ptr,size_t old_usize,size_t size,size_t extra,size_t alignment,bool zero)3297 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
3298 size_t extra, size_t alignment, bool zero) {
3299 size_t newsize;
3300
3301 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero,
3302 &newsize)) {
3303 return old_usize;
3304 }
3305
3306 return newsize;
3307 }
3308
3309 static size_t
ixallocx_prof_sample(tsdn_t * tsdn,void * ptr,size_t old_usize,size_t size,size_t extra,size_t alignment,bool zero,prof_tctx_t * tctx)3310 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
3311 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) {
3312 size_t usize;
3313
3314 if (tctx == NULL) {
3315 return old_usize;
3316 }
3317 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
3318 zero);
3319
3320 return usize;
3321 }
3322
3323 JEMALLOC_ALWAYS_INLINE size_t
ixallocx_prof(tsd_t * tsd,void * ptr,size_t old_usize,size_t size,size_t extra,size_t alignment,bool zero,alloc_ctx_t * alloc_ctx)3324 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
3325 size_t extra, size_t alignment, bool zero, alloc_ctx_t *alloc_ctx) {
3326 size_t usize_max, usize;
3327 bool prof_active;
3328 prof_tctx_t *old_tctx, *tctx;
3329
3330 prof_active = prof_active_get_unlocked();
3331 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr, alloc_ctx);
3332 /*
3333 * usize isn't knowable before ixalloc() returns when extra is non-zero.
3334 * Therefore, compute its maximum possible value and use that in
3335 * prof_alloc_prep() to decide whether to capture a backtrace.
3336 * prof_realloc() will use the actual usize to decide whether to sample.
3337 */
3338 if (alignment == 0) {
3339 usize_max = sz_s2u(size+extra);
3340 assert(usize_max > 0
3341 && usize_max <= SC_LARGE_MAXCLASS);
3342 } else {
3343 usize_max = sz_sa2u(size+extra, alignment);
3344 if (unlikely(usize_max == 0
3345 || usize_max > SC_LARGE_MAXCLASS)) {
3346 /*
3347 * usize_max is out of range, and chances are that
3348 * allocation will fail, but use the maximum possible
3349 * value and carry on with prof_alloc_prep(), just in
3350 * case allocation succeeds.
3351 */
3352 usize_max = SC_LARGE_MAXCLASS;
3353 }
3354 }
3355 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
3356
3357 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
3358 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
3359 size, extra, alignment, zero, tctx);
3360 } else {
3361 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
3362 extra, alignment, zero);
3363 }
3364 if (usize == old_usize) {
3365 prof_alloc_rollback(tsd, tctx, false);
3366 return usize;
3367 }
3368 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
3369 old_tctx);
3370
3371 return usize;
3372 }
3373
3374 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_xallocx(void * ptr,size_t size,size_t extra,int flags)3375 je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
3376 tsd_t *tsd;
3377 size_t usize, old_usize;
3378 size_t alignment = MALLOCX_ALIGN_GET(flags);
3379 bool zero = flags & MALLOCX_ZERO;
3380
3381 LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, "
3382 "flags: %d", ptr, size, extra, flags);
3383
3384 assert(ptr != NULL);
3385 assert(size != 0);
3386 assert(SIZE_T_MAX - size >= extra);
3387 assert(malloc_initialized() || IS_INITIALIZER);
3388 tsd = tsd_fetch();
3389 check_entry_exit_locking(tsd_tsdn(tsd));
3390
3391 alloc_ctx_t alloc_ctx;
3392 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
3393 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
3394 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
3395 assert(alloc_ctx.szind != SC_NSIZES);
3396 old_usize = sz_index2size(alloc_ctx.szind);
3397 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
3398 /*
3399 * The API explicitly absolves itself of protecting against (size +
3400 * extra) numerical overflow, but we may need to clamp extra to avoid
3401 * exceeding SC_LARGE_MAXCLASS.
3402 *
3403 * Ordinarily, size limit checking is handled deeper down, but here we
3404 * have to check as part of (size + extra) clamping, since we need the
3405 * clamped value in the above helper functions.
3406 */
3407 if (unlikely(size > SC_LARGE_MAXCLASS)) {
3408 usize = old_usize;
3409 goto label_not_resized;
3410 }
3411 if (unlikely(SC_LARGE_MAXCLASS - size < extra)) {
3412 extra = SC_LARGE_MAXCLASS - size;
3413 }
3414
3415 if (config_prof && opt_prof) {
3416 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
3417 alignment, zero, &alloc_ctx);
3418 } else {
3419 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
3420 extra, alignment, zero);
3421 }
3422 if (unlikely(usize == old_usize)) {
3423 goto label_not_resized;
3424 }
3425
3426 if (config_stats) {
3427 *tsd_thread_allocatedp_get(tsd) += usize;
3428 *tsd_thread_deallocatedp_get(tsd) += old_usize;
3429 }
3430 label_not_resized:
3431 if (unlikely(!tsd_fast(tsd))) {
3432 uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};
3433 hook_invoke_expand(hook_expand_xallocx, ptr, old_usize,
3434 usize, (uintptr_t)usize, args);
3435 }
3436
3437 UTRACE(ptr, size, ptr);
3438 check_entry_exit_locking(tsd_tsdn(tsd));
3439
3440 LOG("core.xallocx.exit", "result: %zu", usize);
3441 return usize;
3442 }
3443
3444 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
JEMALLOC_ATTR(pure)3445 JEMALLOC_ATTR(pure)
3446 je_sallocx(const void *ptr, int flags) {
3447 size_t usize;
3448 tsdn_t *tsdn;
3449
3450 LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags);
3451
3452 assert(malloc_initialized() || IS_INITIALIZER);
3453 assert(ptr != NULL);
3454
3455 tsdn = tsdn_fetch();
3456 check_entry_exit_locking(tsdn);
3457
3458 if (config_debug || force_ivsalloc) {
3459 usize = ivsalloc(tsdn, ptr);
3460 assert(force_ivsalloc || usize != 0);
3461 } else {
3462 usize = isalloc(tsdn, ptr);
3463 }
3464
3465 check_entry_exit_locking(tsdn);
3466
3467 LOG("core.sallocx.exit", "result: %zu", usize);
3468 return usize;
3469 }
3470
3471 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_dallocx(void * ptr,int flags)3472 je_dallocx(void *ptr, int flags) {
3473 LOG("core.dallocx.entry", "ptr: %p, flags: %d", ptr, flags);
3474
3475 assert(ptr != NULL);
3476 assert(malloc_initialized() || IS_INITIALIZER);
3477
3478 tsd_t *tsd = tsd_fetch();
3479 bool fast = tsd_fast(tsd);
3480 check_entry_exit_locking(tsd_tsdn(tsd));
3481
3482 tcache_t *tcache;
3483 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3484 /* Not allowed to be reentrant and specify a custom tcache. */
3485 assert(tsd_reentrancy_level_get(tsd) == 0);
3486 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3487 tcache = NULL;
3488 } else {
3489 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3490 }
3491 } else {
3492 if (likely(fast)) {
3493 tcache = tsd_tcachep_get(tsd);
3494 assert(tcache == tcache_get(tsd));
3495 } else {
3496 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
3497 tcache = tcache_get(tsd);
3498 } else {
3499 tcache = NULL;
3500 }
3501 }
3502 }
3503
3504 UTRACE(ptr, 0, 0);
3505 if (likely(fast)) {
3506 tsd_assert_fast(tsd);
3507 ifree(tsd, ptr, tcache, false);
3508 } else {
3509 uintptr_t args_raw[3] = {(uintptr_t)ptr, flags};
3510 hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw);
3511 ifree(tsd, ptr, tcache, true);
3512 }
3513 check_entry_exit_locking(tsd_tsdn(tsd));
3514
3515 LOG("core.dallocx.exit", "");
3516 }
3517
3518 JEMALLOC_ALWAYS_INLINE size_t
inallocx(tsdn_t * tsdn,size_t size,int flags)3519 inallocx(tsdn_t *tsdn, size_t size, int flags) {
3520 check_entry_exit_locking(tsdn);
3521
3522 size_t usize;
3523 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) {
3524 usize = sz_s2u(size);
3525 } else {
3526 usize = sz_sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
3527 }
3528 check_entry_exit_locking(tsdn);
3529 return usize;
3530 }
3531
3532 JEMALLOC_NOINLINE void
sdallocx_default(void * ptr,size_t size,int flags)3533 sdallocx_default(void *ptr, size_t size, int flags) {
3534 assert(ptr != NULL);
3535 assert(malloc_initialized() || IS_INITIALIZER);
3536
3537 tsd_t *tsd = tsd_fetch();
3538 bool fast = tsd_fast(tsd);
3539 size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
3540 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
3541 check_entry_exit_locking(tsd_tsdn(tsd));
3542
3543 tcache_t *tcache;
3544 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3545 /* Not allowed to be reentrant and specify a custom tcache. */
3546 assert(tsd_reentrancy_level_get(tsd) == 0);
3547 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3548 tcache = NULL;
3549 } else {
3550 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3551 }
3552 } else {
3553 if (likely(fast)) {
3554 tcache = tsd_tcachep_get(tsd);
3555 assert(tcache == tcache_get(tsd));
3556 } else {
3557 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
3558 tcache = tcache_get(tsd);
3559 } else {
3560 tcache = NULL;
3561 }
3562 }
3563 }
3564
3565 UTRACE(ptr, 0, 0);
3566 if (likely(fast)) {
3567 tsd_assert_fast(tsd);
3568 isfree(tsd, ptr, usize, tcache, false);
3569 } else {
3570 uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags};
3571 hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw);
3572 isfree(tsd, ptr, usize, tcache, true);
3573 }
3574 check_entry_exit_locking(tsd_tsdn(tsd));
3575
3576 }
3577
3578 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_sdallocx(void * ptr,size_t size,int flags)3579 je_sdallocx(void *ptr, size_t size, int flags) {
3580 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
3581 size, flags);
3582
3583 if (flags !=0 || !free_fastpath(ptr, size, true)) {
3584 sdallocx_default(ptr, size, flags);
3585 }
3586
3587 LOG("core.sdallocx.exit", "");
3588 }
3589
3590 void JEMALLOC_NOTHROW
je_sdallocx_noflags(void * ptr,size_t size)3591 je_sdallocx_noflags(void *ptr, size_t size) {
3592 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr,
3593 size);
3594
3595 if (!free_fastpath(ptr, size, true)) {
3596 sdallocx_default(ptr, size, 0);
3597 }
3598
3599 LOG("core.sdallocx.exit", "");
3600 }
3601
3602 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
JEMALLOC_ATTR(pure)3603 JEMALLOC_ATTR(pure)
3604 je_nallocx(size_t size, int flags) {
3605 size_t usize;
3606 tsdn_t *tsdn;
3607
3608 assert(size != 0);
3609
3610 if (unlikely(malloc_init())) {
3611 LOG("core.nallocx.exit", "result: %zu", ZU(0));
3612 return 0;
3613 }
3614
3615 tsdn = tsdn_fetch();
3616 check_entry_exit_locking(tsdn);
3617
3618 usize = inallocx(tsdn, size, flags);
3619 if (unlikely(usize > SC_LARGE_MAXCLASS)) {
3620 LOG("core.nallocx.exit", "result: %zu", ZU(0));
3621 return 0;
3622 }
3623
3624 check_entry_exit_locking(tsdn);
3625 LOG("core.nallocx.exit", "result: %zu", usize);
3626 return usize;
3627 }
3628
3629 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctl(const char * name,void * oldp,size_t * oldlenp,void * newp,size_t newlen)3630 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
3631 size_t newlen) {
3632 int ret;
3633 tsd_t *tsd;
3634
3635 LOG("core.mallctl.entry", "name: %s", name);
3636
3637 if (unlikely(malloc_init())) {
3638 LOG("core.mallctl.exit", "result: %d", EAGAIN);
3639 return EAGAIN;
3640 }
3641
3642 tsd = tsd_fetch();
3643 check_entry_exit_locking(tsd_tsdn(tsd));
3644 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
3645 check_entry_exit_locking(tsd_tsdn(tsd));
3646
3647 LOG("core.mallctl.exit", "result: %d", ret);
3648 return ret;
3649 }
3650
3651 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctlnametomib(const char * name,size_t * mibp,size_t * miblenp)3652 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) {
3653 int ret;
3654
3655 LOG("core.mallctlnametomib.entry", "name: %s", name);
3656
3657 if (unlikely(malloc_init())) {
3658 LOG("core.mallctlnametomib.exit", "result: %d", EAGAIN);
3659 return EAGAIN;
3660 }
3661
3662 tsd_t *tsd = tsd_fetch();
3663 check_entry_exit_locking(tsd_tsdn(tsd));
3664 ret = ctl_nametomib(tsd, name, mibp, miblenp);
3665 check_entry_exit_locking(tsd_tsdn(tsd));
3666
3667 LOG("core.mallctlnametomib.exit", "result: %d", ret);
3668 return ret;
3669 }
3670
3671 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctlbymib(const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)3672 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
3673 void *newp, size_t newlen) {
3674 int ret;
3675 tsd_t *tsd;
3676
3677 LOG("core.mallctlbymib.entry", "");
3678
3679 if (unlikely(malloc_init())) {
3680 LOG("core.mallctlbymib.exit", "result: %d", EAGAIN);
3681 return EAGAIN;
3682 }
3683
3684 tsd = tsd_fetch();
3685 check_entry_exit_locking(tsd_tsdn(tsd));
3686 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
3687 check_entry_exit_locking(tsd_tsdn(tsd));
3688 LOG("core.mallctlbymib.exit", "result: %d", ret);
3689 return ret;
3690 }
3691
3692 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_malloc_stats_print(void (* write_cb)(void *,const char *),void * cbopaque,const char * opts)3693 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
3694 const char *opts) {
3695 tsdn_t *tsdn;
3696
3697 LOG("core.malloc_stats_print.entry", "");
3698
3699 tsdn = tsdn_fetch();
3700 check_entry_exit_locking(tsdn);
3701 stats_print(write_cb, cbopaque, opts);
3702 check_entry_exit_locking(tsdn);
3703 LOG("core.malloc_stats_print.exit", "");
3704 }
3705
3706 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void * ptr)3707 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
3708 size_t ret;
3709 tsdn_t *tsdn;
3710
3711 LOG("core.malloc_usable_size.entry", "ptr: %p", ptr);
3712
3713 assert(malloc_initialized() || IS_INITIALIZER);
3714
3715 tsdn = tsdn_fetch();
3716 check_entry_exit_locking(tsdn);
3717
3718 if (unlikely(ptr == NULL)) {
3719 ret = 0;
3720 } else {
3721 if (config_debug || force_ivsalloc) {
3722 ret = ivsalloc(tsdn, ptr);
3723 assert(force_ivsalloc || ret != 0);
3724 } else {
3725 ret = isalloc(tsdn, ptr);
3726 }
3727 }
3728
3729 check_entry_exit_locking(tsdn);
3730 LOG("core.malloc_usable_size.exit", "result: %zu", ret);
3731 return ret;
3732 }
3733
3734 /*
3735 * End non-standard functions.
3736 */
3737 /******************************************************************************/
3738 /*
3739 * Begin compatibility functions.
3740 */
3741
3742 #define ALLOCM_LG_ALIGN(la) (la)
3743 #define ALLOCM_ALIGN(a) (ffsl(a)-1)
3744 #define ALLOCM_ZERO ((int)0x40)
3745 #define ALLOCM_NO_MOVE ((int)0x80)
3746
3747 #define ALLOCM_SUCCESS 0
3748 #define ALLOCM_ERR_OOM 1
3749 #define ALLOCM_ERR_NOT_MOVED 2
3750
3751 int
je_allocm(void ** ptr,size_t * rsize,size_t size,int flags)3752 je_allocm(void **ptr, size_t *rsize, size_t size, int flags) {
3753 assert(ptr != NULL);
3754
3755 void *p = je_mallocx(size, flags);
3756 if (p == NULL) {
3757 return (ALLOCM_ERR_OOM);
3758 }
3759 if (rsize != NULL) {
3760 *rsize = isalloc(tsdn_fetch(), p);
3761 }
3762 *ptr = p;
3763 return ALLOCM_SUCCESS;
3764 }
3765
3766 int
je_rallocm(void ** ptr,size_t * rsize,size_t size,size_t extra,int flags)3767 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) {
3768 assert(ptr != NULL);
3769 assert(*ptr != NULL);
3770 assert(size != 0);
3771 assert(SIZE_T_MAX - size >= extra);
3772
3773 int ret;
3774 bool no_move = flags & ALLOCM_NO_MOVE;
3775
3776 if (no_move) {
3777 size_t usize = je_xallocx(*ptr, size, extra, flags);
3778 ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
3779 if (rsize != NULL) {
3780 *rsize = usize;
3781 }
3782 } else {
3783 void *p = je_rallocx(*ptr, size+extra, flags);
3784 if (p != NULL) {
3785 *ptr = p;
3786 ret = ALLOCM_SUCCESS;
3787 } else {
3788 ret = ALLOCM_ERR_OOM;
3789 }
3790 if (rsize != NULL) {
3791 *rsize = isalloc(tsdn_fetch(), *ptr);
3792 }
3793 }
3794 return ret;
3795 }
3796
3797 int
je_sallocm(const void * ptr,size_t * rsize,int flags)3798 je_sallocm(const void *ptr, size_t *rsize, int flags) {
3799 assert(rsize != NULL);
3800 *rsize = je_sallocx(ptr, flags);
3801 return ALLOCM_SUCCESS;
3802 }
3803
3804 int
je_dallocm(void * ptr,int flags)3805 je_dallocm(void *ptr, int flags) {
3806 je_dallocx(ptr, flags);
3807 return ALLOCM_SUCCESS;
3808 }
3809
3810 int
je_nallocm(size_t * rsize,size_t size,int flags)3811 je_nallocm(size_t *rsize, size_t size, int flags) {
3812 size_t usize = je_nallocx(size, flags);
3813 if (usize == 0) {
3814 return ALLOCM_ERR_OOM;
3815 }
3816 if (rsize != NULL) {
3817 *rsize = usize;
3818 }
3819 return ALLOCM_SUCCESS;
3820 }
3821
3822 #undef ALLOCM_LG_ALIGN
3823 #undef ALLOCM_ALIGN
3824 #undef ALLOCM_ZERO
3825 #undef ALLOCM_NO_MOVE
3826
3827 #undef ALLOCM_SUCCESS
3828 #undef ALLOCM_ERR_OOM
3829 #undef ALLOCM_ERR_NOT_MOVED
3830
3831 /*
3832 * End compatibility functions.
3833 */
3834 /******************************************************************************/
3835 /*
3836 * The following functions are used by threading libraries for protection of
3837 * malloc during fork().
3838 */
3839
3840 /*
3841 * If an application creates a thread before doing any allocation in the main
3842 * thread, then calls fork(2) in the main thread followed by memory allocation
3843 * in the child process, a race can occur that results in deadlock within the
3844 * child: the main thread may have forked while the created thread had
3845 * partially initialized the allocator. Ordinarily jemalloc prevents
3846 * fork/malloc races via the following functions it registers during
3847 * initialization using pthread_atfork(), but of course that does no good if
3848 * the allocator isn't fully initialized at fork time. The following library
3849 * constructor is a partial solution to this problem. It may still be possible
3850 * to trigger the deadlock described above, but doing so would involve forking
3851 * via a library constructor that runs before jemalloc's runs.
3852 */
3853 #ifndef JEMALLOC_JET
JEMALLOC_ATTR(constructor)3854 JEMALLOC_ATTR(constructor)
3855 static void
3856 jemalloc_constructor(void) {
3857 malloc_init();
3858 }
3859 #endif
3860
3861 #ifndef JEMALLOC_MUTEX_INIT_CB
3862 void
jemalloc_prefork(void)3863 jemalloc_prefork(void)
3864 #else
3865 JEMALLOC_EXPORT void
3866 _malloc_prefork(void)
3867 #endif
3868 {
3869 tsd_t *tsd;
3870 unsigned i, j, narenas;
3871 arena_t *arena;
3872
3873 #ifdef JEMALLOC_MUTEX_INIT_CB
3874 if (!malloc_initialized()) {
3875 return;
3876 }
3877 #endif
3878 assert(malloc_initialized());
3879
3880 tsd = tsd_fetch();
3881
3882 narenas = narenas_total_get();
3883
3884 witness_prefork(tsd_witness_tsdp_get(tsd));
3885 /* Acquire all mutexes in a safe order. */
3886 ctl_prefork(tsd_tsdn(tsd));
3887 tcache_prefork(tsd_tsdn(tsd));
3888 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
3889 if (have_background_thread) {
3890 background_thread_prefork0(tsd_tsdn(tsd));
3891 }
3892 prof_prefork0(tsd_tsdn(tsd));
3893 if (have_background_thread) {
3894 background_thread_prefork1(tsd_tsdn(tsd));
3895 }
3896 /* Break arena prefork into stages to preserve lock order. */
3897 for (i = 0; i < 8; i++) {
3898 for (j = 0; j < narenas; j++) {
3899 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
3900 NULL) {
3901 switch (i) {
3902 case 0:
3903 arena_prefork0(tsd_tsdn(tsd), arena);
3904 break;
3905 case 1:
3906 arena_prefork1(tsd_tsdn(tsd), arena);
3907 break;
3908 case 2:
3909 arena_prefork2(tsd_tsdn(tsd), arena);
3910 break;
3911 case 3:
3912 arena_prefork3(tsd_tsdn(tsd), arena);
3913 break;
3914 case 4:
3915 arena_prefork4(tsd_tsdn(tsd), arena);
3916 break;
3917 case 5:
3918 arena_prefork5(tsd_tsdn(tsd), arena);
3919 break;
3920 case 6:
3921 arena_prefork6(tsd_tsdn(tsd), arena);
3922 break;
3923 case 7:
3924 arena_prefork7(tsd_tsdn(tsd), arena);
3925 break;
3926 default: not_reached();
3927 }
3928 }
3929 }
3930 }
3931 prof_prefork1(tsd_tsdn(tsd));
3932 tsd_prefork(tsd);
3933 }
3934
3935 #ifndef JEMALLOC_MUTEX_INIT_CB
3936 void
jemalloc_postfork_parent(void)3937 jemalloc_postfork_parent(void)
3938 #else
3939 JEMALLOC_EXPORT void
3940 _malloc_postfork(void)
3941 #endif
3942 {
3943 tsd_t *tsd;
3944 unsigned i, narenas;
3945
3946 #ifdef JEMALLOC_MUTEX_INIT_CB
3947 if (!malloc_initialized()) {
3948 return;
3949 }
3950 #endif
3951 assert(malloc_initialized());
3952
3953 tsd = tsd_fetch();
3954
3955 tsd_postfork_parent(tsd);
3956
3957 witness_postfork_parent(tsd_witness_tsdp_get(tsd));
3958 /* Release all mutexes, now that fork() has completed. */
3959 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
3960 arena_t *arena;
3961
3962 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
3963 arena_postfork_parent(tsd_tsdn(tsd), arena);
3964 }
3965 }
3966 prof_postfork_parent(tsd_tsdn(tsd));
3967 if (have_background_thread) {
3968 background_thread_postfork_parent(tsd_tsdn(tsd));
3969 }
3970 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
3971 tcache_postfork_parent(tsd_tsdn(tsd));
3972 ctl_postfork_parent(tsd_tsdn(tsd));
3973 }
3974
3975 void
jemalloc_postfork_child(void)3976 jemalloc_postfork_child(void) {
3977 tsd_t *tsd;
3978 unsigned i, narenas;
3979
3980 assert(malloc_initialized());
3981
3982 tsd = tsd_fetch();
3983
3984 tsd_postfork_child(tsd);
3985
3986 witness_postfork_child(tsd_witness_tsdp_get(tsd));
3987 /* Release all mutexes, now that fork() has completed. */
3988 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
3989 arena_t *arena;
3990
3991 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
3992 arena_postfork_child(tsd_tsdn(tsd), arena);
3993 }
3994 }
3995 prof_postfork_child(tsd_tsdn(tsd));
3996 if (have_background_thread) {
3997 background_thread_postfork_child(tsd_tsdn(tsd));
3998 }
3999 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
4000 tcache_postfork_child(tsd_tsdn(tsd));
4001 ctl_postfork_child(tsd_tsdn(tsd));
4002 }
4003
4004 void
_malloc_first_thread(void)4005 _malloc_first_thread(void)
4006 {
4007
4008 (void)malloc_mutex_first_thread();
4009 }
4010
4011 /******************************************************************************/
4012