1 /* -*- Mode: C; tab-width: 8; c-basic-offset: 8; indent-tabs-mode: t -*- */
2 /* vim:set softtabstop=8 shiftwidth=8 noet: */
3 /*-
4 * Copyright (C) 2006-2008 Jason Evans <jasone@FreeBSD.org>.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice(s), this list of conditions and the following disclaimer as
12 * the first lines of this file unmodified other than the possible
13 * addition of one or more copyright notices.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice(s), this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
17 * distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
26 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
28 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 *******************************************************************************
32 *
33 * This allocator implementation is designed to provide scalable performance
34 * for multi-threaded programs on multi-processor systems. The following
35 * features are included for this purpose:
36 *
37 * + Multiple arenas are used if there are multiple CPUs, which reduces lock
38 * contention and cache sloshing.
39 *
40 * + Cache line sharing between arenas is avoided for internal data
41 * structures.
42 *
43 * + Memory is managed in chunks and runs (chunks can be split into runs),
44 * rather than as individual pages. This provides a constant-time
45 * mechanism for associating allocations with particular arenas.
46 *
47 * Allocation requests are rounded up to the nearest size class, and no record
48 * of the original request size is maintained. Allocations are broken into
49 * categories according to size class. Assuming runtime defaults, 4 kB pages
50 * and a 16 byte quantum on a 32-bit system, the size classes in each category
51 * are as follows:
52 *
53 * |=====================================|
54 * | Category | Subcategory | Size |
55 * |=====================================|
56 * | Small | Tiny | 2 |
57 * | | | 4 |
58 * | | | 8 |
59 * | |----------------+---------|
60 * | | Quantum-spaced | 16 |
61 * | | | 32 |
62 * | | | 48 |
63 * | | | ... |
64 * | | | 480 |
65 * | | | 496 |
66 * | | | 512 |
67 * | |----------------+---------|
68 * | | Sub-page | 1 kB |
69 * | | | 2 kB |
70 * |=====================================|
71 * | Large | 4 kB |
72 * | | 8 kB |
73 * | | 12 kB |
74 * | | ... |
75 * | | 1012 kB |
76 * | | 1016 kB |
77 * | | 1020 kB |
78 * |=====================================|
79 * | Huge | 1 MB |
80 * | | 2 MB |
81 * | | 3 MB |
82 * | | ... |
83 * |=====================================|
84 *
85 * NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an
86 * allocation on Linux or Mac. So on 32-bit *nix, the smallest bucket size is
87 * 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes.
88 *
89 * A different mechanism is used for each category:
90 *
91 * Small : Each size class is segregated into its own set of runs. Each run
92 * maintains a bitmap of which regions are free/allocated.
93 *
94 * Large : Each allocation is backed by a dedicated run. Metadata are stored
95 * in the associated arena chunk header maps.
96 *
97 * Huge : Each allocation is backed by a dedicated contiguous set of chunks.
98 * Metadata are stored in a separate red-black tree.
99 *
100 *******************************************************************************
101 */
102
103 #ifdef MOZ_MEMORY_ANDROID
104 #define NO_TLS
105 #define _pthread_self() pthread_self()
106 #endif
107
108 /*
109 * On Linux, we use madvise(MADV_DONTNEED) to release memory back to the
110 * operating system. If we release 1MB of live pages with MADV_DONTNEED, our
111 * RSS will decrease by 1MB (almost) immediately.
112 *
113 * On Mac, we use madvise(MADV_FREE). Unlike MADV_DONTNEED on Linux, MADV_FREE
114 * on Mac doesn't cause the OS to release the specified pages immediately; the
115 * OS keeps them in our process until the machine comes under memory pressure.
116 *
117 * It's therefore difficult to measure the process's RSS on Mac, since, in the
118 * absence of memory pressure, the contribution from the heap to RSS will not
119 * decrease due to our madvise calls.
120 *
121 * We therefore define MALLOC_DOUBLE_PURGE on Mac. This causes jemalloc to
122 * track which pages have been MADV_FREE'd. You can then call
123 * jemalloc_purge_freed_pages(), which will force the OS to release those
124 * MADV_FREE'd pages, making the process's RSS reflect its true memory usage.
125 *
126 * The jemalloc_purge_freed_pages definition in memory/build/mozmemory.h needs
127 * to be adjusted if MALLOC_DOUBLE_PURGE is ever enabled on Linux.
128 */
129 #ifdef MOZ_MEMORY_DARWIN
130 #define MALLOC_DOUBLE_PURGE
131 #endif
132
133 /*
134 * MALLOC_PRODUCTION disables assertions and statistics gathering. It also
135 * defaults the A and J runtime options to off. These settings are appropriate
136 * for production systems.
137 */
138 #ifndef MOZ_MEMORY_DEBUG
139 # define MALLOC_PRODUCTION
140 #endif
141
142 /*
143 * Use only one arena by default. Mozilla does not currently make extensive
144 * use of concurrent allocation, so the increased fragmentation associated with
145 * multiple arenas is not warranted.
146 *
147 * When using the Servo style system, we do indeed make use of significant
148 * concurrent allocation, and the overhead matters. Bug 1291355 tracks
149 * investigating the fragmentation overhead of turning this on for users.
150 */
151 #ifndef MOZ_STYLO
152 #define MOZ_MEMORY_NARENAS_DEFAULT_ONE
153 #endif
154
155 /*
156 * Pass this set of options to jemalloc as its default. It does not override
157 * the options passed via the MALLOC_OPTIONS environment variable but is
158 * applied in addition to them.
159 */
160 #ifdef MOZ_WIDGET_GONK
161 /* Reduce the amount of unused dirty pages to 1MiB on B2G */
162 # define MOZ_MALLOC_OPTIONS "ff"
163 #else
164 # define MOZ_MALLOC_OPTIONS ""
165 #endif
166
167 /*
168 * MALLOC_STATS enables statistics calculation, and is required for
169 * jemalloc_stats().
170 */
171 #define MALLOC_STATS
172
173 /* Memory filling (junk/poison/zero). */
174 #define MALLOC_FILL
175
176 #ifndef MALLOC_PRODUCTION
177 /*
178 * MALLOC_DEBUG enables assertions and other sanity checks, and disables
179 * inline functions.
180 */
181 # define MALLOC_DEBUG
182
183 /* Allocation tracing. */
184 # ifndef MOZ_MEMORY_WINDOWS
185 # define MALLOC_UTRACE
186 # endif
187
188 /* Support optional abort() on OOM. */
189 # define MALLOC_XMALLOC
190
191 /* Support SYSV semantics. */
192 # define MALLOC_SYSV
193 #endif
194
195 /*
196 * MALLOC_VALIDATE causes malloc_usable_size() to perform some pointer
197 * validation. There are many possible errors that validation does not even
198 * attempt to detect.
199 */
200 #define MALLOC_VALIDATE
201
202 /*
203 * MALLOC_BALANCE enables monitoring of arena lock contention and dynamically
204 * re-balances arena load if exponentially averaged contention exceeds a
205 * certain threshold.
206 */
207 /* #define MALLOC_BALANCE */
208
209 #if defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
210 #define _GNU_SOURCE /* For mremap(2). */
211 #if 0 /* Enable in order to test decommit code on Linux. */
212 # define MALLOC_DECOMMIT
213 #endif
214 #endif
215
216 #include <sys/types.h>
217
218 #include <errno.h>
219 #include <stdlib.h>
220 #include <limits.h>
221 #include <stdarg.h>
222 #include <stdio.h>
223 #include <string.h>
224
225 #ifdef MOZ_MEMORY_WINDOWS
226
227 /* Some defines from the CRT internal headers that we need here. */
228 #define _CRT_SPINCOUNT 5000
229 #define __crtInitCritSecAndSpinCount InitializeCriticalSectionAndSpinCount
230 #include <io.h>
231 #include <windows.h>
232 #include <intrin.h>
233
234 #pragma warning( disable: 4267 4996 4146 )
235
236 #define bool BOOL
237 #define false FALSE
238 #define true TRUE
239 #define inline __inline
240 #define SIZE_T_MAX SIZE_MAX
241 #define STDERR_FILENO 2
242 #define PATH_MAX MAX_PATH
243 #define vsnprintf _vsnprintf
244
245 #ifndef NO_TLS
246 static unsigned long tlsIndex = 0xffffffff;
247 #endif
248
249 #define __thread
250 #define _pthread_self() __threadid()
251
252 /* use MSVC intrinsics */
253 #pragma intrinsic(_BitScanForward)
254 static __forceinline int
ffs(int x)255 ffs(int x)
256 {
257 unsigned long i;
258
259 if (_BitScanForward(&i, x) != 0)
260 return (i + 1);
261
262 return (0);
263 }
264
265 /* Implement getenv without using malloc */
266 static char mozillaMallocOptionsBuf[64];
267
268 #define getenv xgetenv
269 static char *
getenv(const char * name)270 getenv(const char *name)
271 {
272
273 if (GetEnvironmentVariableA(name, (LPSTR)&mozillaMallocOptionsBuf,
274 sizeof(mozillaMallocOptionsBuf)) > 0)
275 return (mozillaMallocOptionsBuf);
276
277 return (NULL);
278 }
279
280 typedef unsigned char uint8_t;
281 typedef unsigned uint32_t;
282 typedef unsigned long long uint64_t;
283 typedef unsigned long long uintmax_t;
284 #if defined(_WIN64)
285 typedef long long ssize_t;
286 #else
287 typedef long ssize_t;
288 #endif
289
290 #define MALLOC_DECOMMIT
291 #endif
292
293 /*
294 * Allow unmapping pages on all platforms. Note that if this is disabled,
295 * jemalloc will never unmap anything, instead recycling pages for later use.
296 */
297 #define JEMALLOC_MUNMAP
298
299 /*
300 * Enable limited chunk recycling on all platforms. Note that when
301 * JEMALLOC_MUNMAP is not defined, all chunks will be recycled unconditionally.
302 */
303 #define JEMALLOC_RECYCLE
304
305 #ifndef MOZ_MEMORY_WINDOWS
306 #ifndef MOZ_MEMORY_SOLARIS
307 #include <sys/cdefs.h>
308 #endif
309 #ifndef __DECONST
310 # define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
311 #endif
312 #ifndef MOZ_MEMORY
313 __FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z jasone $");
314 #include "libc_private.h"
315 #ifdef MALLOC_DEBUG
316 # define _LOCK_DEBUG
317 #endif
318 #include "spinlock.h"
319 #include "namespace.h"
320 #endif
321 #include <sys/mman.h>
322 #ifndef MADV_FREE
323 # define MADV_FREE MADV_DONTNEED
324 #endif
325 #ifndef MAP_NOSYNC
326 # define MAP_NOSYNC 0
327 #endif
328 #include <sys/param.h>
329 #ifndef MOZ_MEMORY
330 #include <sys/stddef.h>
331 #endif
332 #include <sys/time.h>
333 #include <sys/types.h>
334 #if !defined(MOZ_MEMORY_SOLARIS) && !defined(MOZ_MEMORY_ANDROID)
335 #include <sys/sysctl.h>
336 #endif
337 #include <sys/uio.h>
338 #ifndef MOZ_MEMORY
339 #include <sys/ktrace.h> /* Must come after several other sys/ includes. */
340
341 #include <machine/atomic.h>
342 #include <machine/cpufunc.h>
343 #include <machine/vmparam.h>
344 #endif
345
346 #include <errno.h>
347 #include <limits.h>
348 #ifndef SIZE_T_MAX
349 # define SIZE_T_MAX SIZE_MAX
350 #endif
351 #include <pthread.h>
352 #ifdef MOZ_MEMORY_DARWIN
353 #define _pthread_self pthread_self
354 #define _pthread_mutex_init pthread_mutex_init
355 #define _pthread_mutex_trylock pthread_mutex_trylock
356 #define _pthread_mutex_lock pthread_mutex_lock
357 #define _pthread_mutex_unlock pthread_mutex_unlock
358 #endif
359 #include <sched.h>
360 #include <stdarg.h>
361 #include <stdio.h>
362 #include <stdbool.h>
363 #include <stdint.h>
364 #include <stdlib.h>
365 #include <string.h>
366 #ifndef MOZ_MEMORY_DARWIN
367 #include <strings.h>
368 #endif
369 #include <unistd.h>
370
371 #ifdef MOZ_MEMORY_DARWIN
372 #include <libkern/OSAtomic.h>
373 #include <mach/mach_error.h>
374 #include <mach/mach_init.h>
375 #include <mach/vm_map.h>
376 #include <malloc/malloc.h>
377 #endif
378
379 #ifndef MOZ_MEMORY
380 #include "un-namespace.h"
381 #endif
382
383 #endif
384
385 #include "jemalloc_types.h"
386 #include "linkedlist.h"
387 #include "mozmemory_wrap.h"
388
389 /* Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that
390 * happen to override mmap() and call dlsym() from their overridden
391 * mmap(). The problem is that dlsym() calls malloc(), and this ends
392 * up in a dead lock in jemalloc.
393 * On these systems, we prefer to directly use the system call.
394 * We do that for Linux systems and kfreebsd with GNU userland.
395 * Note sanity checks are not done (alignment of offset, ...) because
396 * the uses of mmap are pretty limited, in jemalloc.
397 *
398 * On Alpha, glibc has a bug that prevents syscall() to work for system
399 * calls with 6 arguments
400 */
401 #if (defined(MOZ_MEMORY_LINUX) && !defined(__alpha__)) || \
402 (defined(MOZ_MEMORY_BSD) && defined(__GLIBC__))
403 #include <sys/syscall.h>
404 #if defined(SYS_mmap) || defined(SYS_mmap2)
405 static inline
_mmap(void * addr,size_t length,int prot,int flags,int fd,off_t offset)406 void *_mmap(void *addr, size_t length, int prot, int flags,
407 int fd, off_t offset)
408 {
409 /* S390 only passes one argument to the mmap system call, which is a
410 * pointer to a structure containing the arguments */
411 #ifdef __s390__
412 struct {
413 void *addr;
414 size_t length;
415 long prot;
416 long flags;
417 long fd;
418 off_t offset;
419 } args = { addr, length, prot, flags, fd, offset };
420 return (void *) syscall(SYS_mmap, &args);
421 #else
422 #ifdef SYS_mmap2
423 return (void *) syscall(SYS_mmap2, addr, length, prot, flags,
424 fd, offset >> 12);
425 #else
426 return (void *) syscall(SYS_mmap, addr, length, prot, flags,
427 fd, offset);
428 #endif
429 #endif
430 }
431 #define mmap _mmap
432 #define munmap(a, l) syscall(SYS_munmap, a, l)
433 #endif
434 #endif
435
436 #ifdef MOZ_MEMORY_DARWIN
437 static const bool isthreaded = true;
438 #endif
439
440 #if defined(MOZ_MEMORY_SOLARIS) && defined(MAP_ALIGN) && !defined(JEMALLOC_NEVER_USES_MAP_ALIGN)
441 #define JEMALLOC_USES_MAP_ALIGN /* Required on Solaris 10. Might improve performance elsewhere. */
442 #endif
443
444 #ifndef __DECONST
445 #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
446 #endif
447
448 #ifdef MOZ_MEMORY_WINDOWS
449 /* MSVC++ does not support C99 variable-length arrays. */
450 # define RB_NO_C99_VARARRAYS
451 #endif
452 #include "rb.h"
453
454 #ifdef MALLOC_DEBUG
455 /* Disable inlining to make debugging easier. */
456 #ifdef inline
457 #undef inline
458 #endif
459
460 # define inline
461 #endif
462
463 /* Size of stack-allocated buffer passed to strerror_r(). */
464 #define STRERROR_BUF 64
465
466 /* Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. */
467 # define QUANTUM_2POW_MIN 4
468 #if defined(_WIN64) || defined(__LP64__)
469 # define SIZEOF_PTR_2POW 3
470 #else
471 # define SIZEOF_PTR_2POW 2
472 #endif
473 #define PIC
474 #ifndef MOZ_MEMORY_DARWIN
475 static const bool isthreaded = true;
476 #else
477 # define NO_TLS
478 #endif
479 #if 0
480 #ifdef __i386__
481 # define QUANTUM_2POW_MIN 4
482 # define SIZEOF_PTR_2POW 2
483 # define CPU_SPINWAIT __asm__ volatile("pause")
484 #endif
485 #ifdef __ia64__
486 # define QUANTUM_2POW_MIN 4
487 # define SIZEOF_PTR_2POW 3
488 #endif
489 #ifdef __alpha__
490 # define QUANTUM_2POW_MIN 4
491 # define SIZEOF_PTR_2POW 3
492 # define NO_TLS
493 #endif
494 #ifdef __sparc64__
495 # define QUANTUM_2POW_MIN 4
496 # define SIZEOF_PTR_2POW 3
497 # define NO_TLS
498 #endif
499 #ifdef __amd64__
500 # define QUANTUM_2POW_MIN 4
501 # define SIZEOF_PTR_2POW 3
502 # define CPU_SPINWAIT __asm__ volatile("pause")
503 #endif
504 #ifdef __arm__
505 # define QUANTUM_2POW_MIN 3
506 # define SIZEOF_PTR_2POW 2
507 # define NO_TLS
508 #endif
509 #ifdef __mips__
510 # define QUANTUM_2POW_MIN 3
511 # define SIZEOF_PTR_2POW 2
512 # define NO_TLS
513 #endif
514 #ifdef __powerpc__
515 # define QUANTUM_2POW_MIN 4
516 # define SIZEOF_PTR_2POW 2
517 #endif
518 #endif
519
520 #define SIZEOF_PTR (1U << SIZEOF_PTR_2POW)
521
522 /* sizeof(int) == (1U << SIZEOF_INT_2POW). */
523 #ifndef SIZEOF_INT_2POW
524 # define SIZEOF_INT_2POW 2
525 #endif
526
527 /* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
528 #if (!defined(PIC) && !defined(NO_TLS))
529 # define NO_TLS
530 #endif
531
532 #ifdef NO_TLS
533 /* MALLOC_BALANCE requires TLS. */
534 # ifdef MALLOC_BALANCE
535 # undef MALLOC_BALANCE
536 # endif
537 #endif
538
539 /*
540 * Size and alignment of memory chunks that are allocated by the OS's virtual
541 * memory system.
542 */
543 #define CHUNK_2POW_DEFAULT 20
544 /* Maximum number of dirty pages per arena. */
545 #define DIRTY_MAX_DEFAULT (1U << 8)
546
547 /*
548 * Maximum size of L1 cache line. This is used to avoid cache line aliasing,
549 * so over-estimates are okay (up to a point), but under-estimates will
550 * negatively affect performance.
551 */
552 #define CACHELINE_2POW 6
553 #define CACHELINE ((size_t)(1U << CACHELINE_2POW))
554
555 /*
556 * Smallest size class to support. On Windows the smallest allocation size
557 * must be 8 bytes on 32-bit, 16 bytes on 64-bit. On Linux and Mac, even
558 * malloc(1) must reserve a word's worth of memory (see Mozilla bug 691003).
559 */
560 #ifdef MOZ_MEMORY_WINDOWS
561 #define TINY_MIN_2POW (sizeof(void*) == 8 ? 4 : 3)
562 #else
563 #define TINY_MIN_2POW (sizeof(void*) == 8 ? 3 : 2)
564 #endif
565
566 /*
567 * Maximum size class that is a multiple of the quantum, but not (necessarily)
568 * a power of 2. Above this size, allocations are rounded up to the nearest
569 * power of 2.
570 */
571 #define SMALL_MAX_2POW_DEFAULT 9
572 #define SMALL_MAX_DEFAULT (1U << SMALL_MAX_2POW_DEFAULT)
573
574 /*
575 * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized
576 * as small as possible such that this setting is still honored, without
577 * violating other constraints. The goal is to make runs as small as possible
578 * without exceeding a per run external fragmentation threshold.
579 *
580 * We use binary fixed point math for overhead computations, where the binary
581 * point is implicitly RUN_BFP bits to the left.
582 *
583 * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
584 * honored for some/all object sizes, since there is one bit of header overhead
585 * per object (plus a constant). This constraint is relaxed (ignored) for runs
586 * that are so small that the per-region overhead is greater than:
587 *
588 * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
589 */
590 #define RUN_BFP 12
591 /* \/ Implicit binary fixed point. */
592 #define RUN_MAX_OVRHD 0x0000003dU
593 #define RUN_MAX_OVRHD_RELAX 0x00001800U
594
595 /*
596 * Hyper-threaded CPUs may need a special instruction inside spin loops in
597 * order to yield to another virtual CPU. If no such instruction is defined
598 * above, make CPU_SPINWAIT a no-op.
599 */
600 #ifndef CPU_SPINWAIT
601 # define CPU_SPINWAIT
602 #endif
603
604 /*
605 * Adaptive spinning must eventually switch to blocking, in order to avoid the
606 * potential for priority inversion deadlock. Backing off past a certain point
607 * can actually waste time.
608 */
609 #define SPIN_LIMIT_2POW 11
610
611 /*
612 * Conversion from spinning to blocking is expensive; we use (1U <<
613 * BLOCK_COST_2POW) to estimate how many more times costly blocking is than
614 * worst-case spinning.
615 */
616 #define BLOCK_COST_2POW 4
617
618 #ifdef MALLOC_BALANCE
619 /*
620 * We use an exponential moving average to track recent lock contention,
621 * where the size of the history window is N, and alpha=2/(N+1).
622 *
623 * Due to integer math rounding, very small values here can cause
624 * substantial degradation in accuracy, thus making the moving average decay
625 * faster than it would with precise calculation.
626 */
627 # define BALANCE_ALPHA_INV_2POW 9
628
629 /*
630 * Threshold value for the exponential moving contention average at which to
631 * re-assign a thread.
632 */
633 # define BALANCE_THRESHOLD_DEFAULT (1U << (SPIN_LIMIT_2POW-4))
634 #endif
635
636 /******************************************************************************/
637
638 /* MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. */
639 #if defined(MALLOC_DECOMMIT) && defined(MALLOC_DOUBLE_PURGE)
640 #error MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive.
641 #endif
642
643 /*
644 * Mutexes based on spinlocks. We can't use normal pthread spinlocks in all
645 * places, because they require malloc()ed memory, which causes bootstrapping
646 * issues in some cases.
647 */
648 #if defined(MOZ_MEMORY_WINDOWS)
649 #define malloc_mutex_t CRITICAL_SECTION
650 #define malloc_spinlock_t CRITICAL_SECTION
651 #elif defined(MOZ_MEMORY_DARWIN)
652 typedef struct {
653 OSSpinLock lock;
654 } malloc_mutex_t;
655 typedef struct {
656 OSSpinLock lock;
657 } malloc_spinlock_t;
658 #elif defined(MOZ_MEMORY)
659 typedef pthread_mutex_t malloc_mutex_t;
660 typedef pthread_mutex_t malloc_spinlock_t;
661 #else
662 /* XXX these should #ifdef these for freebsd (and linux?) only */
663 typedef struct {
664 spinlock_t lock;
665 } malloc_mutex_t;
666 typedef malloc_spinlock_t malloc_mutex_t;
667 #endif
668
669 /* Set to true once the allocator has been initialized. */
670 static bool malloc_initialized = false;
671
672 #if defined(MOZ_MEMORY_WINDOWS)
673 /* No init lock for Windows. */
674 #elif defined(MOZ_MEMORY_DARWIN)
675 static malloc_mutex_t init_lock = {OS_SPINLOCK_INIT};
676 #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
677 static malloc_mutex_t init_lock = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
678 #elif defined(MOZ_MEMORY)
679 static malloc_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
680 #else
681 static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
682 #endif
683
684 /******************************************************************************/
685 /*
686 * Statistics data structures.
687 */
688
689 #ifdef MALLOC_STATS
690
691 typedef struct malloc_bin_stats_s malloc_bin_stats_t;
692 struct malloc_bin_stats_s {
693 /*
694 * Number of allocation requests that corresponded to the size of this
695 * bin.
696 */
697 uint64_t nrequests;
698
699 /* Total number of runs created for this bin's size class. */
700 uint64_t nruns;
701
702 /*
703 * Total number of runs reused by extracting them from the runs tree for
704 * this bin's size class.
705 */
706 uint64_t reruns;
707
708 /* High-water mark for this bin. */
709 unsigned long highruns;
710
711 /* Current number of runs in this bin. */
712 unsigned long curruns;
713 };
714
715 typedef struct arena_stats_s arena_stats_t;
716 struct arena_stats_s {
717 /* Number of bytes currently mapped. */
718 size_t mapped;
719
720 /*
721 * Total number of purge sweeps, total number of madvise calls made,
722 * and total pages purged in order to keep dirty unused memory under
723 * control.
724 */
725 uint64_t npurge;
726 uint64_t nmadvise;
727 uint64_t purged;
728 #ifdef MALLOC_DECOMMIT
729 /*
730 * Total number of decommit/commit operations, and total number of
731 * pages decommitted.
732 */
733 uint64_t ndecommit;
734 uint64_t ncommit;
735 uint64_t decommitted;
736 #endif
737
738 /* Current number of committed pages. */
739 size_t committed;
740
741 /* Per-size-category statistics. */
742 size_t allocated_small;
743 uint64_t nmalloc_small;
744 uint64_t ndalloc_small;
745
746 size_t allocated_large;
747 uint64_t nmalloc_large;
748 uint64_t ndalloc_large;
749
750 #ifdef MALLOC_BALANCE
751 /* Number of times this arena reassigned a thread due to contention. */
752 uint64_t nbalance;
753 #endif
754 };
755
756 #endif /* #ifdef MALLOC_STATS */
757
758 /******************************************************************************/
759 /*
760 * Extent data structures.
761 */
762
763 /* Tree of extents. */
764 typedef struct extent_node_s extent_node_t;
765 struct extent_node_s {
766 /* Linkage for the size/address-ordered tree. */
767 rb_node(extent_node_t) link_szad;
768
769 /* Linkage for the address-ordered tree. */
770 rb_node(extent_node_t) link_ad;
771
772 /* Pointer to the extent that this tree node is responsible for. */
773 void *addr;
774
775 /* Total region size. */
776 size_t size;
777
778 /* True if zero-filled; used by chunk recycling code. */
779 bool zeroed;
780 };
781 typedef rb_tree(extent_node_t) extent_tree_t;
782
783 /******************************************************************************/
784 /*
785 * Radix tree data structures.
786 */
787
788 #ifdef MALLOC_VALIDATE
789 /*
790 * Size of each radix tree node (must be a power of 2). This impacts tree
791 * depth.
792 */
793 # if (SIZEOF_PTR == 4)
794 # define MALLOC_RTREE_NODESIZE (1U << 14)
795 # else
796 # define MALLOC_RTREE_NODESIZE CACHELINE
797 # endif
798
799 typedef struct malloc_rtree_s malloc_rtree_t;
800 struct malloc_rtree_s {
801 malloc_spinlock_t lock;
802 void **root;
803 unsigned height;
804 unsigned level2bits[1]; /* Dynamically sized. */
805 };
806 #endif
807
808 /******************************************************************************/
809 /*
810 * Arena data structures.
811 */
812
813 typedef struct arena_s arena_t;
814 typedef struct arena_bin_s arena_bin_t;
815
816 /* Each element of the chunk map corresponds to one page within the chunk. */
817 typedef struct arena_chunk_map_s arena_chunk_map_t;
818 struct arena_chunk_map_s {
819 /*
820 * Linkage for run trees. There are two disjoint uses:
821 *
822 * 1) arena_t's runs_avail tree.
823 * 2) arena_run_t conceptually uses this linkage for in-use non-full
824 * runs, rather than directly embedding linkage.
825 */
826 rb_node(arena_chunk_map_t) link;
827
828 /*
829 * Run address (or size) and various flags are stored together. The bit
830 * layout looks like (assuming 32-bit system):
831 *
832 * ???????? ???????? ????---- -mckdzla
833 *
834 * ? : Unallocated: Run address for first/last pages, unset for internal
835 * pages.
836 * Small: Run address.
837 * Large: Run size for first page, unset for trailing pages.
838 * - : Unused.
839 * m : MADV_FREE/MADV_DONTNEED'ed?
840 * c : decommitted?
841 * k : key?
842 * d : dirty?
843 * z : zeroed?
844 * l : large?
845 * a : allocated?
846 *
847 * Following are example bit patterns for the three types of runs.
848 *
849 * r : run address
850 * s : run size
851 * x : don't care
852 * - : 0
853 * [cdzla] : bit set
854 *
855 * Unallocated:
856 * ssssssss ssssssss ssss---- --c-----
857 * xxxxxxxx xxxxxxxx xxxx---- ----d---
858 * ssssssss ssssssss ssss---- -----z--
859 *
860 * Small:
861 * rrrrrrrr rrrrrrrr rrrr---- -------a
862 * rrrrrrrr rrrrrrrr rrrr---- -------a
863 * rrrrrrrr rrrrrrrr rrrr---- -------a
864 *
865 * Large:
866 * ssssssss ssssssss ssss---- ------la
867 * -------- -------- -------- ------la
868 * -------- -------- -------- ------la
869 */
870 size_t bits;
871
872 /* Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether
873 * MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined.
874 *
875 * If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be
876 * re-committed with pages_commit() before it may be touched. If
877 * MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined.
878 *
879 * If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which
880 * are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with
881 * CHUNK_MAP_MADVISED.
882 *
883 * Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is
884 * defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED.
885 * When it's finally freed with jemalloc_purge_freed_pages, the page is marked
886 * as CHUNK_MAP_DECOMMITTED.
887 */
888 #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
889 #define CHUNK_MAP_MADVISED ((size_t)0x40U)
890 #define CHUNK_MAP_DECOMMITTED ((size_t)0x20U)
891 #define CHUNK_MAP_MADVISED_OR_DECOMMITTED (CHUNK_MAP_MADVISED | CHUNK_MAP_DECOMMITTED)
892 #endif
893 #define CHUNK_MAP_KEY ((size_t)0x10U)
894 #define CHUNK_MAP_DIRTY ((size_t)0x08U)
895 #define CHUNK_MAP_ZEROED ((size_t)0x04U)
896 #define CHUNK_MAP_LARGE ((size_t)0x02U)
897 #define CHUNK_MAP_ALLOCATED ((size_t)0x01U)
898 };
899 typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
900 typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
901
902 /* Arena chunk header. */
903 typedef struct arena_chunk_s arena_chunk_t;
904 struct arena_chunk_s {
905 /* Arena that owns the chunk. */
906 arena_t *arena;
907
908 /* Linkage for the arena's chunks_dirty tree. */
909 rb_node(arena_chunk_t) link_dirty;
910
911 #ifdef MALLOC_DOUBLE_PURGE
912 /* If we're double-purging, we maintain a linked list of chunks which
913 * have pages which have been madvise(MADV_FREE)'d but not explicitly
914 * purged.
915 *
916 * We're currently lazy and don't remove a chunk from this list when
917 * all its madvised pages are recommitted. */
918 LinkedList chunks_madvised_elem;
919 #endif
920
921 /* Number of dirty pages. */
922 size_t ndirty;
923
924 /* Map of pages within chunk that keeps track of free/large/small. */
925 arena_chunk_map_t map[1]; /* Dynamically sized. */
926 };
927 typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
928
929 typedef struct arena_run_s arena_run_t;
930 struct arena_run_s {
931 #if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
932 uint32_t magic;
933 # define ARENA_RUN_MAGIC 0x384adf93
934 #endif
935
936 /* Bin this run is associated with. */
937 arena_bin_t *bin;
938
939 /* Index of first element that might have a free region. */
940 unsigned regs_minelm;
941
942 /* Number of free regions in run. */
943 unsigned nfree;
944
945 /* Bitmask of in-use regions (0: in use, 1: free). */
946 unsigned regs_mask[1]; /* Dynamically sized. */
947 };
948
949 struct arena_bin_s {
950 /*
951 * Current run being used to service allocations of this bin's size
952 * class.
953 */
954 arena_run_t *runcur;
955
956 /*
957 * Tree of non-full runs. This tree is used when looking for an
958 * existing run when runcur is no longer usable. We choose the
959 * non-full run that is lowest in memory; this policy tends to keep
960 * objects packed well, and it can also help reduce the number of
961 * almost-empty chunks.
962 */
963 arena_run_tree_t runs;
964
965 /* Size of regions in a run for this bin's size class. */
966 size_t reg_size;
967
968 /* Total size of a run for this bin's size class. */
969 size_t run_size;
970
971 /* Total number of regions in a run for this bin's size class. */
972 uint32_t nregs;
973
974 /* Number of elements in a run's regs_mask for this bin's size class. */
975 uint32_t regs_mask_nelms;
976
977 /* Offset of first region in a run for this bin's size class. */
978 uint32_t reg0_offset;
979
980 #ifdef MALLOC_STATS
981 /* Bin statistics. */
982 malloc_bin_stats_t stats;
983 #endif
984 };
985
986 struct arena_s {
987 #if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
988 uint32_t magic;
989 # define ARENA_MAGIC 0x947d3d24
990 #endif
991
992 /* All operations on this arena require that lock be locked. */
993 #ifdef MOZ_MEMORY
994 malloc_spinlock_t lock;
995 #else
996 pthread_mutex_t lock;
997 #endif
998
999 #ifdef MALLOC_STATS
1000 arena_stats_t stats;
1001 #endif
1002
1003 /* Tree of dirty-page-containing chunks this arena manages. */
1004 arena_chunk_tree_t chunks_dirty;
1005
1006 #ifdef MALLOC_DOUBLE_PURGE
1007 /* Head of a linked list of MADV_FREE'd-page-containing chunks this
1008 * arena manages. */
1009 LinkedList chunks_madvised;
1010 #endif
1011
1012 /*
1013 * In order to avoid rapid chunk allocation/deallocation when an arena
1014 * oscillates right on the cusp of needing a new chunk, cache the most
1015 * recently freed chunk. The spare is left in the arena's chunk trees
1016 * until it is deleted.
1017 *
1018 * There is one spare chunk per arena, rather than one spare total, in
1019 * order to avoid interactions between multiple threads that could make
1020 * a single spare inadequate.
1021 */
1022 arena_chunk_t *spare;
1023
1024 /*
1025 * Current count of pages within unused runs that are potentially
1026 * dirty, and for which madvise(... MADV_FREE) has not been called. By
1027 * tracking this, we can institute a limit on how much dirty unused
1028 * memory is mapped for each arena.
1029 */
1030 size_t ndirty;
1031
1032 /*
1033 * Size/address-ordered tree of this arena's available runs. This tree
1034 * is used for first-best-fit run allocation.
1035 */
1036 arena_avail_tree_t runs_avail;
1037
1038 #ifdef MALLOC_BALANCE
1039 /*
1040 * The arena load balancing machinery needs to keep track of how much
1041 * lock contention there is. This value is exponentially averaged.
1042 */
1043 uint32_t contention;
1044 #endif
1045
1046 /*
1047 * bins is used to store rings of free regions of the following sizes,
1048 * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS.
1049 *
1050 * bins[i] | size |
1051 * --------+------+
1052 * 0 | 2 |
1053 * 1 | 4 |
1054 * 2 | 8 |
1055 * --------+------+
1056 * 3 | 16 |
1057 * 4 | 32 |
1058 * 5 | 48 |
1059 * 6 | 64 |
1060 * : :
1061 * : :
1062 * 33 | 496 |
1063 * 34 | 512 |
1064 * --------+------+
1065 * 35 | 1024 |
1066 * 36 | 2048 |
1067 * --------+------+
1068 */
1069 arena_bin_t bins[1]; /* Dynamically sized. */
1070 };
1071
1072 /******************************************************************************/
1073 /*
1074 * Data.
1075 */
1076
1077 #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1078 /* Number of CPUs. */
1079 static unsigned ncpus;
1080 #endif
1081
1082 #ifdef JEMALLOC_MUNMAP
1083 static const bool config_munmap = true;
1084 #else
1085 static const bool config_munmap = false;
1086 #endif
1087
1088 #ifdef JEMALLOC_RECYCLE
1089 static const bool config_recycle = true;
1090 #else
1091 static const bool config_recycle = false;
1092 #endif
1093
1094 /*
1095 * When MALLOC_STATIC_SIZES is defined most of the parameters
1096 * controlling the malloc behavior are defined as compile-time constants
1097 * for best performance and cannot be altered at runtime.
1098 */
1099 #if !defined(__ia64__) && !defined(__sparc__) && !defined(__mips__) && !defined(__aarch64__)
1100 #define MALLOC_STATIC_SIZES 1
1101 #endif
1102
1103 #ifdef MALLOC_STATIC_SIZES
1104
1105 /*
1106 * VM page size. It must divide the runtime CPU page size or the code
1107 * will abort.
1108 * Platform specific page size conditions copied from js/public/HeapAPI.h
1109 */
1110 #if (defined(SOLARIS) || defined(__FreeBSD__)) && \
1111 (defined(__sparc) || defined(__sparcv9) || defined(__ia64))
1112 #define pagesize_2pow ((size_t) 13)
1113 #elif defined(__powerpc64__)
1114 #define pagesize_2pow ((size_t) 16)
1115 #else
1116 #define pagesize_2pow ((size_t) 12)
1117 #endif
1118 #define pagesize ((size_t) 1 << pagesize_2pow)
1119 #define pagesize_mask (pagesize - 1)
1120
1121 /* Various quantum-related settings. */
1122
1123 #define QUANTUM_DEFAULT ((size_t) 1 << QUANTUM_2POW_MIN)
1124 static const size_t quantum = QUANTUM_DEFAULT;
1125 static const size_t quantum_mask = QUANTUM_DEFAULT - 1;
1126
1127 /* Various bin-related settings. */
1128
1129 static const size_t small_min = (QUANTUM_DEFAULT >> 1) + 1;
1130 static const size_t small_max = (size_t) SMALL_MAX_DEFAULT;
1131
1132 /* Max size class for bins. */
1133 static const size_t bin_maxclass = pagesize >> 1;
1134
1135 /* Number of (2^n)-spaced tiny bins. */
1136 static const unsigned ntbins = (unsigned)
1137 (QUANTUM_2POW_MIN - TINY_MIN_2POW);
1138
1139 /* Number of quantum-spaced bins. */
1140 static const unsigned nqbins = (unsigned)
1141 (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
1142
1143 /* Number of (2^n)-spaced sub-page bins. */
1144 static const unsigned nsbins = (unsigned)
1145 (pagesize_2pow -
1146 SMALL_MAX_2POW_DEFAULT - 1);
1147
1148 #else /* !MALLOC_STATIC_SIZES */
1149
1150 /* VM page size. */
1151 static size_t pagesize;
1152 static size_t pagesize_mask;
1153 static size_t pagesize_2pow;
1154
1155 /* Various bin-related settings. */
1156 static size_t bin_maxclass; /* Max size class for bins. */
1157 static unsigned ntbins; /* Number of (2^n)-spaced tiny bins. */
1158 static unsigned nqbins; /* Number of quantum-spaced bins. */
1159 static unsigned nsbins; /* Number of (2^n)-spaced sub-page bins. */
1160 static size_t small_min;
1161 static size_t small_max;
1162
1163 /* Various quantum-related settings. */
1164 static size_t quantum;
1165 static size_t quantum_mask; /* (quantum - 1). */
1166
1167 #endif
1168
1169 /* Various chunk-related settings. */
1170
1171 /*
1172 * Compute the header size such that it is large enough to contain the page map
1173 * and enough nodes for the worst case: one node per non-header page plus one
1174 * extra for situations where we briefly have one more node allocated than we
1175 * will need.
1176 */
1177 #define calculate_arena_header_size() \
1178 (sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1))
1179
1180 #define calculate_arena_header_pages() \
1181 ((calculate_arena_header_size() >> pagesize_2pow) + \
1182 ((calculate_arena_header_size() & pagesize_mask) ? 1 : 0))
1183
1184 /* Max size class for arenas. */
1185 #define calculate_arena_maxclass() \
1186 (chunksize - (arena_chunk_header_npages << pagesize_2pow))
1187
1188 /*
1189 * Recycle at most 128 chunks. With 1 MiB chunks, this means we retain at most
1190 * 6.25% of the process address space on a 32-bit OS for later use.
1191 */
1192 #define CHUNK_RECYCLE_LIMIT 128
1193
1194 #ifdef MALLOC_STATIC_SIZES
1195 #define CHUNKSIZE_DEFAULT ((size_t) 1 << CHUNK_2POW_DEFAULT)
1196 static const size_t chunksize = CHUNKSIZE_DEFAULT;
1197 static const size_t chunksize_mask =CHUNKSIZE_DEFAULT - 1;
1198 static const size_t chunk_npages = CHUNKSIZE_DEFAULT >> pagesize_2pow;
1199 #define arena_chunk_header_npages calculate_arena_header_pages()
1200 #define arena_maxclass calculate_arena_maxclass()
1201 static const size_t recycle_limit = CHUNK_RECYCLE_LIMIT * CHUNKSIZE_DEFAULT;
1202 #else
1203 static size_t chunksize;
1204 static size_t chunksize_mask; /* (chunksize - 1). */
1205 static size_t chunk_npages;
1206 static size_t arena_chunk_header_npages;
1207 static size_t arena_maxclass; /* Max size class for arenas. */
1208 static size_t recycle_limit;
1209 #endif
1210
1211 /* The current amount of recycled bytes, updated atomically. */
1212 static size_t recycled_size;
1213
1214 /********/
1215 /*
1216 * Chunks.
1217 */
1218
1219 #ifdef MALLOC_VALIDATE
1220 static malloc_rtree_t *chunk_rtree;
1221 #endif
1222
1223 /* Protects chunk-related data structures. */
1224 static malloc_mutex_t chunks_mtx;
1225
1226 /*
1227 * Trees of chunks that were previously allocated (trees differ only in node
1228 * ordering). These are used when allocating chunks, in an attempt to re-use
1229 * address space. Depending on function, different tree orderings are needed,
1230 * which is why there are two trees with the same contents.
1231 */
1232 static extent_tree_t chunks_szad_mmap;
1233 static extent_tree_t chunks_ad_mmap;
1234
1235 /* Protects huge allocation-related data structures. */
1236 static malloc_mutex_t huge_mtx;
1237
1238 /* Tree of chunks that are stand-alone huge allocations. */
1239 static extent_tree_t huge;
1240
1241 #ifdef MALLOC_STATS
1242 /* Huge allocation statistics. */
1243 static uint64_t huge_nmalloc;
1244 static uint64_t huge_ndalloc;
1245 static size_t huge_allocated;
1246 static size_t huge_mapped;
1247 #endif
1248
1249 /****************************/
1250 /*
1251 * base (internal allocation).
1252 */
1253
1254 /*
1255 * Current pages that are being used for internal memory allocations. These
1256 * pages are carved up in cacheline-size quanta, so that there is no chance of
1257 * false cache line sharing.
1258 */
1259 static void *base_pages;
1260 static void *base_next_addr;
1261 #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1262 static void *base_next_decommitted;
1263 #endif
1264 static void *base_past_addr; /* Addr immediately past base_pages. */
1265 static extent_node_t *base_nodes;
1266 static malloc_mutex_t base_mtx;
1267 #ifdef MALLOC_STATS
1268 static size_t base_mapped;
1269 static size_t base_committed;
1270 #endif
1271
1272 /********/
1273 /*
1274 * Arenas.
1275 */
1276
1277 /*
1278 * Arenas that are used to service external requests. Not all elements of the
1279 * arenas array are necessarily used; arenas are created lazily as needed.
1280 */
1281 static arena_t **arenas;
1282 static unsigned narenas;
1283 #ifndef NO_TLS
1284 # ifdef MALLOC_BALANCE
1285 static unsigned narenas_2pow;
1286 # else
1287 static unsigned next_arena;
1288 # endif
1289 #endif
1290 #ifdef MOZ_MEMORY
1291 static malloc_spinlock_t arenas_lock; /* Protects arenas initialization. */
1292 #else
1293 static pthread_mutex_t arenas_lock; /* Protects arenas initialization. */
1294 #endif
1295
1296 #ifndef NO_TLS
1297 /*
1298 * Map of pthread_self() --> arenas[???], used for selecting an arena to use
1299 * for allocations.
1300 */
1301 #ifndef MOZ_MEMORY_WINDOWS
1302 static __thread arena_t *arenas_map;
1303 #endif
1304 #endif
1305
1306 /*******************************/
1307 /*
1308 * Runtime configuration options.
1309 */
1310 MOZ_JEMALLOC_API
1311 const char *_malloc_options = MOZ_MALLOC_OPTIONS;
1312
1313 #ifndef MALLOC_PRODUCTION
1314 static bool opt_abort = true;
1315 #ifdef MALLOC_FILL
1316 static bool opt_junk = true;
1317 static bool opt_poison = true;
1318 static bool opt_zero = false;
1319 #endif
1320 #else
1321 static bool opt_abort = false;
1322 #ifdef MALLOC_FILL
1323 static const bool opt_junk = false;
1324 static const bool opt_poison = true;
1325 static const bool opt_zero = false;
1326 #endif
1327 #endif
1328
1329 static size_t opt_dirty_max = DIRTY_MAX_DEFAULT;
1330 #ifdef MALLOC_BALANCE
1331 static uint64_t opt_balance_threshold = BALANCE_THRESHOLD_DEFAULT;
1332 #endif
1333 static bool opt_print_stats = false;
1334 #ifdef MALLOC_STATIC_SIZES
1335 #define opt_quantum_2pow QUANTUM_2POW_MIN
1336 #define opt_small_max_2pow SMALL_MAX_2POW_DEFAULT
1337 #define opt_chunk_2pow CHUNK_2POW_DEFAULT
1338 #else
1339 static size_t opt_quantum_2pow = QUANTUM_2POW_MIN;
1340 static size_t opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
1341 static size_t opt_chunk_2pow = CHUNK_2POW_DEFAULT;
1342 #endif
1343 #ifdef MALLOC_UTRACE
1344 static bool opt_utrace = false;
1345 #endif
1346 #ifdef MALLOC_SYSV
1347 static bool opt_sysv = false;
1348 #endif
1349 #ifdef MALLOC_XMALLOC
1350 static bool opt_xmalloc = false;
1351 #endif
1352 static int opt_narenas_lshift = 0;
1353
1354 #ifdef MALLOC_UTRACE
1355 typedef struct {
1356 void *p;
1357 size_t s;
1358 void *r;
1359 } malloc_utrace_t;
1360
1361 #define UTRACE(a, b, c) \
1362 if (opt_utrace) { \
1363 malloc_utrace_t ut; \
1364 ut.p = (a); \
1365 ut.s = (b); \
1366 ut.r = (c); \
1367 utrace(&ut, sizeof(ut)); \
1368 }
1369 #else
1370 #define UTRACE(a, b, c)
1371 #endif
1372
1373 /******************************************************************************/
1374 /*
1375 * Begin function prototypes for non-inline static functions.
1376 */
1377
1378 static char *umax2s(uintmax_t x, unsigned base, char *s);
1379 static bool malloc_mutex_init(malloc_mutex_t *mutex);
1380 static bool malloc_spin_init(malloc_spinlock_t *lock);
1381 static void wrtmessage(const char *p1, const char *p2, const char *p3,
1382 const char *p4);
1383 #ifdef MALLOC_STATS
1384 #ifdef MOZ_MEMORY_DARWIN
1385 /* Avoid namespace collision with OS X's malloc APIs. */
1386 #define malloc_printf moz_malloc_printf
1387 #endif
1388 static void malloc_printf(const char *format, ...);
1389 #endif
1390 static bool base_pages_alloc(size_t minsize);
1391 static void *base_alloc(size_t size);
1392 static void *base_calloc(size_t number, size_t size);
1393 static extent_node_t *base_node_alloc(void);
1394 static void base_node_dealloc(extent_node_t *node);
1395 #ifdef MALLOC_STATS
1396 static void stats_print(arena_t *arena);
1397 #endif
1398 static void *pages_map(void *addr, size_t size);
1399 static void pages_unmap(void *addr, size_t size);
1400 static void *chunk_alloc_mmap(size_t size, size_t alignment);
1401 static void *chunk_recycle(extent_tree_t *chunks_szad,
1402 extent_tree_t *chunks_ad, size_t size,
1403 size_t alignment, bool base, bool *zero);
1404 static void *chunk_alloc(size_t size, size_t alignment, bool base, bool zero);
1405 static void chunk_record(extent_tree_t *chunks_szad,
1406 extent_tree_t *chunks_ad, void *chunk, size_t size);
1407 static bool chunk_dalloc_mmap(void *chunk, size_t size);
1408 static void chunk_dealloc(void *chunk, size_t size);
1409 #ifndef NO_TLS
1410 static arena_t *choose_arena_hard(void);
1411 #endif
1412 static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
1413 bool large, bool zero);
1414 static void arena_chunk_init(arena_t *arena, arena_chunk_t *chunk);
1415 static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
1416 static arena_run_t *arena_run_alloc(arena_t *arena, arena_bin_t *bin,
1417 size_t size, bool large, bool zero);
1418 static void arena_purge(arena_t *arena, bool all);
1419 static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
1420 static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
1421 arena_run_t *run, size_t oldsize, size_t newsize);
1422 static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
1423 arena_run_t *run, size_t oldsize, size_t newsize, bool dirty);
1424 static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
1425 static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
1426 static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
1427 #ifdef MALLOC_BALANCE
1428 static void arena_lock_balance_hard(arena_t *arena);
1429 #endif
1430 static void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
1431 static void *arena_palloc(arena_t *arena, size_t alignment, size_t size,
1432 size_t alloc_size);
1433 static size_t arena_salloc(const void *ptr);
1434 static void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk,
1435 void *ptr);
1436 static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
1437 void *ptr, size_t size, size_t oldsize);
1438 static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk,
1439 void *ptr, size_t size, size_t oldsize);
1440 static bool arena_ralloc_large(void *ptr, size_t size, size_t oldsize);
1441 static void *arena_ralloc(void *ptr, size_t size, size_t oldsize);
1442 static bool arena_new(arena_t *arena);
1443 static arena_t *arenas_extend(unsigned ind);
1444 static void *huge_malloc(size_t size, bool zero);
1445 static void *huge_palloc(size_t size, size_t alignment, bool zero);
1446 static void *huge_ralloc(void *ptr, size_t size, size_t oldsize);
1447 static void huge_dalloc(void *ptr);
1448 static void malloc_print_stats(void);
1449 #ifndef MOZ_MEMORY_WINDOWS
1450 static
1451 #endif
1452 bool malloc_init_hard(void);
1453
1454 static void _malloc_prefork(void);
1455 static void _malloc_postfork(void);
1456
1457 #ifdef MOZ_MEMORY_DARWIN
1458 /*
1459 * MALLOC_ZONE_T_NOTE
1460 *
1461 * On Darwin, we hook into the memory allocator using a malloc_zone_t struct.
1462 * We must be very careful around this struct because of different behaviour on
1463 * different versions of OSX.
1464 *
1465 * Each of OSX 10.5, 10.6 and 10.7 use different versions of the struct
1466 * (with version numbers 3, 6 and 8 respectively). The binary we use on each of
1467 * these platforms will not necessarily be built using the correct SDK [1].
1468 * This means we need to statically know the correct struct size to use on all
1469 * OSX releases, and have a fallback for unknown future versions. The struct
1470 * sizes defined in osx_zone_types.h.
1471 *
1472 * For OSX 10.8 and later, we may expect the malloc_zone_t struct to change
1473 * again, and need to dynamically account for this. By simply leaving
1474 * malloc_zone_t alone, we don't quite deal with the problem, because there
1475 * remain calls to jemalloc through the mozalloc interface. We check this
1476 * dynamically on each allocation, using the CHECK_DARWIN macro and
1477 * osx_use_jemalloc.
1478 *
1479 *
1480 * [1] Mozilla is built as a universal binary on Mac, supporting i386 and
1481 * x86_64. The i386 target is built using the 10.5 SDK, even if it runs on
1482 * 10.6. The x86_64 target is built using the 10.6 SDK, even if it runs on
1483 * 10.7 or later, or 10.5.
1484 *
1485 * FIXME:
1486 * When later versions of OSX come out (10.8 and up), we need to check their
1487 * malloc_zone_t versions. If they're greater than 8, we need a new version
1488 * of malloc_zone_t adapted into osx_zone_types.h.
1489 */
1490
1491 #ifndef MOZ_REPLACE_MALLOC
1492 #include "osx_zone_types.h"
1493
1494 #define LEOPARD_MALLOC_ZONE_T_VERSION 3
1495 #define SNOW_LEOPARD_MALLOC_ZONE_T_VERSION 6
1496 #define LION_MALLOC_ZONE_T_VERSION 8
1497
1498 static bool osx_use_jemalloc = false;
1499
1500
1501 static lion_malloc_zone l_szone;
1502 static malloc_zone_t * szone = (malloc_zone_t*)(&l_szone);
1503
1504 static lion_malloc_introspection l_ozone_introspect;
1505 static malloc_introspection_t * const ozone_introspect =
1506 (malloc_introspection_t*)(&l_ozone_introspect);
1507 static void szone2ozone(malloc_zone_t *zone, size_t size);
1508 static size_t zone_version_size(int version);
1509 #else
1510 static const bool osx_use_jemalloc = true;
1511 #endif
1512
1513 #endif
1514
1515 /*
1516 * End function prototypes.
1517 */
1518 /******************************************************************************/
1519
1520 static inline size_t
load_acquire_z(size_t * p)1521 load_acquire_z(size_t *p)
1522 {
1523 volatile size_t result = *p;
1524 # ifdef MOZ_MEMORY_WINDOWS
1525 /*
1526 * We use InterlockedExchange with a dummy value to insert a memory
1527 * barrier. This has been confirmed to generate the right instruction
1528 * and is also used by MinGW.
1529 */
1530 volatile long dummy = 0;
1531 InterlockedExchange(&dummy, 1);
1532 # else
1533 __sync_synchronize();
1534 # endif
1535 return result;
1536 }
1537
1538 /*
1539 * umax2s() provides minimal integer printing functionality, which is
1540 * especially useful for situations where allocation in vsnprintf() calls would
1541 * potentially cause deadlock.
1542 */
1543 #define UMAX2S_BUFSIZE 65
1544 char *
umax2s(uintmax_t x,unsigned base,char * s)1545 umax2s(uintmax_t x, unsigned base, char *s)
1546 {
1547 unsigned i;
1548
1549 i = UMAX2S_BUFSIZE - 1;
1550 s[i] = '\0';
1551 switch (base) {
1552 case 10:
1553 do {
1554 i--;
1555 s[i] = "0123456789"[x % 10];
1556 x /= 10;
1557 } while (x > 0);
1558 break;
1559 case 16:
1560 do {
1561 i--;
1562 s[i] = "0123456789abcdef"[x & 0xf];
1563 x >>= 4;
1564 } while (x > 0);
1565 break;
1566 default:
1567 do {
1568 i--;
1569 s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % base];
1570 x /= base;
1571 } while (x > 0);
1572 }
1573
1574 return (&s[i]);
1575 }
1576
1577 static void
wrtmessage(const char * p1,const char * p2,const char * p3,const char * p4)1578 wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
1579 {
1580 #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_WINDOWS)
1581 #define _write write
1582 #endif
1583 // Pretend to check _write() errors to suppress gcc warnings about
1584 // warn_unused_result annotations in some versions of glibc headers.
1585 if (_write(STDERR_FILENO, p1, (unsigned int) strlen(p1)) < 0)
1586 return;
1587 if (_write(STDERR_FILENO, p2, (unsigned int) strlen(p2)) < 0)
1588 return;
1589 if (_write(STDERR_FILENO, p3, (unsigned int) strlen(p3)) < 0)
1590 return;
1591 if (_write(STDERR_FILENO, p4, (unsigned int) strlen(p4)) < 0)
1592 return;
1593 }
1594
1595 MOZ_JEMALLOC_API
1596 void (*_malloc_message)(const char *p1, const char *p2, const char *p3,
1597 const char *p4) = wrtmessage;
1598
1599 #include "mozilla/Assertions.h"
1600 #include "mozilla/Attributes.h"
1601 #include "mozilla/TaggedAnonymousMemory.h"
1602 // Note: MozTaggedAnonymousMmap() could call an LD_PRELOADed mmap
1603 // instead of the one defined here; use only MozTagAnonymousMemory().
1604
1605 #ifdef MALLOC_DEBUG
1606 # define assert(e) MOZ_ASSERT(e)
1607 #else
1608 # define assert(e)
1609 #endif
1610
1611 #ifdef MOZ_MEMORY_ANDROID
1612 // Android's pthread.h does not declare pthread_atfork() until SDK 21.
1613 extern MOZ_EXPORT
1614 int pthread_atfork(void (*)(void), void (*)(void), void(*)(void));
1615 #endif
1616
1617 #if defined(MOZ_JEMALLOC_HARD_ASSERTS)
1618 # define RELEASE_ASSERT(assertion) do { \
1619 if (!(assertion)) { \
1620 MOZ_CRASH_UNSAFE_OOL(#assertion); \
1621 } \
1622 } while (0)
1623 #else
1624 # define RELEASE_ASSERT(assertion) assert(assertion)
1625 #endif
1626
1627 /******************************************************************************/
1628 /*
1629 * Begin mutex. We can't use normal pthread mutexes in all places, because
1630 * they require malloc()ed memory, which causes bootstrapping issues in some
1631 * cases.
1632 */
1633
1634 static bool
malloc_mutex_init(malloc_mutex_t * mutex)1635 malloc_mutex_init(malloc_mutex_t *mutex)
1636 {
1637 #if defined(MOZ_MEMORY_WINDOWS)
1638 if (isthreaded)
1639 if (! __crtInitCritSecAndSpinCount(mutex, _CRT_SPINCOUNT))
1640 return (true);
1641 #elif defined(MOZ_MEMORY_DARWIN)
1642 mutex->lock = OS_SPINLOCK_INIT;
1643 #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1644 pthread_mutexattr_t attr;
1645 if (pthread_mutexattr_init(&attr) != 0)
1646 return (true);
1647 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
1648 if (pthread_mutex_init(mutex, &attr) != 0) {
1649 pthread_mutexattr_destroy(&attr);
1650 return (true);
1651 }
1652 pthread_mutexattr_destroy(&attr);
1653 #elif defined(MOZ_MEMORY)
1654 if (pthread_mutex_init(mutex, NULL) != 0)
1655 return (true);
1656 #else
1657 static const spinlock_t lock = _SPINLOCK_INITIALIZER;
1658
1659 mutex->lock = lock;
1660 #endif
1661 return (false);
1662 }
1663
1664 static inline void
malloc_mutex_lock(malloc_mutex_t * mutex)1665 malloc_mutex_lock(malloc_mutex_t *mutex)
1666 {
1667
1668 #if defined(MOZ_MEMORY_WINDOWS)
1669 EnterCriticalSection(mutex);
1670 #elif defined(MOZ_MEMORY_DARWIN)
1671 OSSpinLockLock(&mutex->lock);
1672 #elif defined(MOZ_MEMORY)
1673 pthread_mutex_lock(mutex);
1674 #else
1675 if (isthreaded)
1676 _SPINLOCK(&mutex->lock);
1677 #endif
1678 }
1679
1680 static inline void
malloc_mutex_unlock(malloc_mutex_t * mutex)1681 malloc_mutex_unlock(malloc_mutex_t *mutex)
1682 {
1683
1684 #if defined(MOZ_MEMORY_WINDOWS)
1685 LeaveCriticalSection(mutex);
1686 #elif defined(MOZ_MEMORY_DARWIN)
1687 OSSpinLockUnlock(&mutex->lock);
1688 #elif defined(MOZ_MEMORY)
1689 pthread_mutex_unlock(mutex);
1690 #else
1691 if (isthreaded)
1692 _SPINUNLOCK(&mutex->lock);
1693 #endif
1694 }
1695
1696 #if (defined(__GNUC__))
1697 __attribute__((unused))
1698 # endif
1699 static bool
malloc_spin_init(malloc_spinlock_t * lock)1700 malloc_spin_init(malloc_spinlock_t *lock)
1701 {
1702 #if defined(MOZ_MEMORY_WINDOWS)
1703 if (isthreaded)
1704 if (! __crtInitCritSecAndSpinCount(lock, _CRT_SPINCOUNT))
1705 return (true);
1706 #elif defined(MOZ_MEMORY_DARWIN)
1707 lock->lock = OS_SPINLOCK_INIT;
1708 #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1709 pthread_mutexattr_t attr;
1710 if (pthread_mutexattr_init(&attr) != 0)
1711 return (true);
1712 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
1713 if (pthread_mutex_init(lock, &attr) != 0) {
1714 pthread_mutexattr_destroy(&attr);
1715 return (true);
1716 }
1717 pthread_mutexattr_destroy(&attr);
1718 #elif defined(MOZ_MEMORY)
1719 if (pthread_mutex_init(lock, NULL) != 0)
1720 return (true);
1721 #else
1722 lock->lock = _SPINLOCK_INITIALIZER;
1723 #endif
1724 return (false);
1725 }
1726
1727 static inline void
malloc_spin_lock(malloc_spinlock_t * lock)1728 malloc_spin_lock(malloc_spinlock_t *lock)
1729 {
1730
1731 #if defined(MOZ_MEMORY_WINDOWS)
1732 EnterCriticalSection(lock);
1733 #elif defined(MOZ_MEMORY_DARWIN)
1734 OSSpinLockLock(&lock->lock);
1735 #elif defined(MOZ_MEMORY)
1736 pthread_mutex_lock(lock);
1737 #else
1738 if (isthreaded)
1739 _SPINLOCK(&lock->lock);
1740 #endif
1741 }
1742
1743 static inline void
malloc_spin_unlock(malloc_spinlock_t * lock)1744 malloc_spin_unlock(malloc_spinlock_t *lock)
1745 {
1746 #if defined(MOZ_MEMORY_WINDOWS)
1747 LeaveCriticalSection(lock);
1748 #elif defined(MOZ_MEMORY_DARWIN)
1749 OSSpinLockUnlock(&lock->lock);
1750 #elif defined(MOZ_MEMORY)
1751 pthread_mutex_unlock(lock);
1752 #else
1753 if (isthreaded)
1754 _SPINUNLOCK(&lock->lock);
1755 #endif
1756 }
1757
1758 /*
1759 * End mutex.
1760 */
1761 /******************************************************************************/
1762 /*
1763 * Begin spin lock. Spin locks here are actually adaptive mutexes that block
1764 * after a period of spinning, because unbounded spinning would allow for
1765 * priority inversion.
1766 */
1767
1768 #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_DARWIN)
1769 # define malloc_spin_init malloc_mutex_init
1770 # define malloc_spin_lock malloc_mutex_lock
1771 # define malloc_spin_unlock malloc_mutex_unlock
1772 #endif
1773
1774 #ifndef MOZ_MEMORY
1775 /*
1776 * We use an unpublished interface to initialize pthread mutexes with an
1777 * allocation callback, in order to avoid infinite recursion.
1778 */
1779 int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
1780 void *(calloc_cb)(size_t, size_t));
1781
1782 __weak_reference(_pthread_mutex_init_calloc_cb_stub,
1783 _pthread_mutex_init_calloc_cb);
1784
1785 int
_pthread_mutex_init_calloc_cb_stub(pthread_mutex_t * mutex,void * (calloc_cb)(size_t,size_t))1786 _pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
1787 void *(calloc_cb)(size_t, size_t))
1788 {
1789
1790 return (0);
1791 }
1792
1793 static bool
malloc_spin_init(pthread_mutex_t * lock)1794 malloc_spin_init(pthread_mutex_t *lock)
1795 {
1796
1797 if (_pthread_mutex_init_calloc_cb(lock, base_calloc) != 0)
1798 return (true);
1799
1800 return (false);
1801 }
1802
1803 static inline unsigned
malloc_spin_lock(pthread_mutex_t * lock)1804 malloc_spin_lock(pthread_mutex_t *lock)
1805 {
1806 unsigned ret = 0;
1807
1808 if (isthreaded) {
1809 if (_pthread_mutex_trylock(lock) != 0) {
1810 unsigned i;
1811 volatile unsigned j;
1812
1813 /* Exponentially back off. */
1814 for (i = 1; i <= SPIN_LIMIT_2POW; i++) {
1815 for (j = 0; j < (1U << i); j++)
1816 ret++;
1817
1818 CPU_SPINWAIT;
1819 if (_pthread_mutex_trylock(lock) == 0)
1820 return (ret);
1821 }
1822
1823 /*
1824 * Spinning failed. Block until the lock becomes
1825 * available, in order to avoid indefinite priority
1826 * inversion.
1827 */
1828 _pthread_mutex_lock(lock);
1829 assert((ret << BLOCK_COST_2POW) != 0);
1830 return (ret << BLOCK_COST_2POW);
1831 }
1832 }
1833
1834 return (ret);
1835 }
1836
1837 static inline void
malloc_spin_unlock(pthread_mutex_t * lock)1838 malloc_spin_unlock(pthread_mutex_t *lock)
1839 {
1840
1841 if (isthreaded)
1842 _pthread_mutex_unlock(lock);
1843 }
1844 #endif
1845
1846 /*
1847 * End spin lock.
1848 */
1849 /******************************************************************************/
1850 /*
1851 * Begin Utility functions/macros.
1852 */
1853
1854 /* Return the chunk address for allocation address a. */
1855 #define CHUNK_ADDR2BASE(a) \
1856 ((void *)((uintptr_t)(a) & ~chunksize_mask))
1857
1858 /* Return the chunk offset of address a. */
1859 #define CHUNK_ADDR2OFFSET(a) \
1860 ((size_t)((uintptr_t)(a) & chunksize_mask))
1861
1862 /* Return the smallest chunk multiple that is >= s. */
1863 #define CHUNK_CEILING(s) \
1864 (((s) + chunksize_mask) & ~chunksize_mask)
1865
1866 /* Return the smallest cacheline multiple that is >= s. */
1867 #define CACHELINE_CEILING(s) \
1868 (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1))
1869
1870 /* Return the smallest quantum multiple that is >= a. */
1871 #define QUANTUM_CEILING(a) \
1872 (((a) + quantum_mask) & ~quantum_mask)
1873
1874 /* Return the smallest pagesize multiple that is >= s. */
1875 #define PAGE_CEILING(s) \
1876 (((s) + pagesize_mask) & ~pagesize_mask)
1877
1878 /* Compute the smallest power of 2 that is >= x. */
1879 static inline size_t
pow2_ceil(size_t x)1880 pow2_ceil(size_t x)
1881 {
1882
1883 x--;
1884 x |= x >> 1;
1885 x |= x >> 2;
1886 x |= x >> 4;
1887 x |= x >> 8;
1888 x |= x >> 16;
1889 #if (SIZEOF_PTR == 8)
1890 x |= x >> 32;
1891 #endif
1892 x++;
1893 return (x);
1894 }
1895
1896 #ifdef MALLOC_BALANCE
1897 /*
1898 * Use a simple linear congruential pseudo-random number generator:
1899 *
1900 * prn(y) = (a*x + c) % m
1901 *
1902 * where the following constants ensure maximal period:
1903 *
1904 * a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4.
1905 * c == Odd number (relatively prime to 2^n).
1906 * m == 2^32
1907 *
1908 * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
1909 *
1910 * This choice of m has the disadvantage that the quality of the bits is
1911 * proportional to bit position. For example. the lowest bit has a cycle of 2,
1912 * the next has a cycle of 4, etc. For this reason, we prefer to use the upper
1913 * bits.
1914 */
1915 # define PRN_DEFINE(suffix, var, a, c) \
1916 static inline void \
1917 sprn_##suffix(uint32_t seed) \
1918 { \
1919 var = seed; \
1920 } \
1921 \
1922 static inline uint32_t \
1923 prn_##suffix(uint32_t lg_range) \
1924 { \
1925 uint32_t ret, x; \
1926 \
1927 assert(lg_range > 0); \
1928 assert(lg_range <= 32); \
1929 \
1930 x = (var * (a)) + (c); \
1931 var = x; \
1932 ret = x >> (32 - lg_range); \
1933 \
1934 return (ret); \
1935 }
1936 # define SPRN(suffix, seed) sprn_##suffix(seed)
1937 # define PRN(suffix, lg_range) prn_##suffix(lg_range)
1938 #endif
1939
1940 #ifdef MALLOC_BALANCE
1941 /* Define the PRNG used for arena assignment. */
1942 static __thread uint32_t balance_x;
1943 PRN_DEFINE(balance, balance_x, 1297, 1301)
1944 #endif
1945
1946 #ifdef MALLOC_UTRACE
1947 static int
utrace(const void * addr,size_t len)1948 utrace(const void *addr, size_t len)
1949 {
1950 malloc_utrace_t *ut = (malloc_utrace_t *)addr;
1951 char buf_a[UMAX2S_BUFSIZE];
1952 char buf_b[UMAX2S_BUFSIZE];
1953
1954 assert(len == sizeof(malloc_utrace_t));
1955
1956 if (ut->p == NULL && ut->s == 0 && ut->r == NULL) {
1957 _malloc_message(
1958 umax2s(getpid(), 10, buf_a),
1959 " x USER malloc_init()\n", "", "");
1960 } else if (ut->p == NULL && ut->r != NULL) {
1961 _malloc_message(
1962 umax2s(getpid(), 10, buf_a),
1963 " x USER 0x",
1964 umax2s((uintptr_t)ut->r, 16, buf_b),
1965 " = malloc(");
1966 _malloc_message(
1967 umax2s(ut->s, 10, buf_a),
1968 ")\n", "", "");
1969 } else if (ut->p != NULL && ut->r != NULL) {
1970 _malloc_message(
1971 umax2s(getpid(), 10, buf_a),
1972 " x USER 0x",
1973 umax2s((uintptr_t)ut->r, 16, buf_b),
1974 " = realloc(0x");
1975 _malloc_message(
1976 umax2s((uintptr_t)ut->p, 16, buf_a),
1977 ", ",
1978 umax2s(ut->s, 10, buf_b),
1979 ")\n");
1980 } else {
1981 _malloc_message(
1982 umax2s(getpid(), 10, buf_a),
1983 " x USER free(0x",
1984 umax2s((uintptr_t)ut->p, 16, buf_b),
1985 ")\n");
1986 }
1987
1988 return (0);
1989 }
1990 #endif
1991
1992 static inline const char *
_getprogname(void)1993 _getprogname(void)
1994 {
1995
1996 return ("<jemalloc>");
1997 }
1998
1999 #ifdef MALLOC_STATS
2000 /*
2001 * Print to stderr in such a way as to (hopefully) avoid memory allocation.
2002 */
2003 static void
malloc_printf(const char * format,...)2004 malloc_printf(const char *format, ...)
2005 {
2006 char buf[4096];
2007 va_list ap;
2008
2009 va_start(ap, format);
2010 vsnprintf(buf, sizeof(buf), format, ap);
2011 va_end(ap);
2012 _malloc_message(buf, "", "", "");
2013 }
2014 #endif
2015
2016 /******************************************************************************/
2017
2018 static inline void
pages_decommit(void * addr,size_t size)2019 pages_decommit(void *addr, size_t size)
2020 {
2021
2022 #ifdef MOZ_MEMORY_WINDOWS
2023 /*
2024 * The region starting at addr may have been allocated in multiple calls
2025 * to VirtualAlloc and recycled, so decommitting the entire region in one
2026 * go may not be valid. However, since we allocate at least a chunk at a
2027 * time, we may touch any region in chunksized increments.
2028 */
2029 size_t pages_size = min(size, chunksize -
2030 CHUNK_ADDR2OFFSET((uintptr_t)addr));
2031 while (size > 0) {
2032 if (!VirtualFree(addr, pages_size, MEM_DECOMMIT))
2033 abort();
2034 addr = (void *)((uintptr_t)addr + pages_size);
2035 size -= pages_size;
2036 pages_size = min(size, chunksize);
2037 }
2038 #else
2039 if (mmap(addr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1,
2040 0) == MAP_FAILED)
2041 abort();
2042 MozTagAnonymousMemory(addr, size, "jemalloc-decommitted");
2043 #endif
2044 }
2045
2046 static inline void
pages_commit(void * addr,size_t size)2047 pages_commit(void *addr, size_t size)
2048 {
2049
2050 # ifdef MOZ_MEMORY_WINDOWS
2051 /*
2052 * The region starting at addr may have been allocated in multiple calls
2053 * to VirtualAlloc and recycled, so committing the entire region in one
2054 * go may not be valid. However, since we allocate at least a chunk at a
2055 * time, we may touch any region in chunksized increments.
2056 */
2057 size_t pages_size = min(size, chunksize -
2058 CHUNK_ADDR2OFFSET((uintptr_t)addr));
2059 while (size > 0) {
2060 if (!VirtualAlloc(addr, pages_size, MEM_COMMIT, PAGE_READWRITE))
2061 abort();
2062 addr = (void *)((uintptr_t)addr + pages_size);
2063 size -= pages_size;
2064 pages_size = min(size, chunksize);
2065 }
2066 # else
2067 if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE |
2068 MAP_ANON, -1, 0) == MAP_FAILED)
2069 abort();
2070 MozTagAnonymousMemory(addr, size, "jemalloc");
2071 # endif
2072 }
2073
2074 static bool
base_pages_alloc(size_t minsize)2075 base_pages_alloc(size_t minsize)
2076 {
2077 size_t csize;
2078 #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
2079 size_t pminsize;
2080 #endif
2081
2082 assert(minsize != 0);
2083 csize = CHUNK_CEILING(minsize);
2084 base_pages = chunk_alloc(csize, chunksize, true, false);
2085 if (base_pages == NULL)
2086 return (true);
2087 base_next_addr = base_pages;
2088 base_past_addr = (void *)((uintptr_t)base_pages + csize);
2089 #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
2090 /*
2091 * Leave enough pages for minsize committed, since otherwise they would
2092 * have to be immediately recommitted.
2093 */
2094 pminsize = PAGE_CEILING(minsize);
2095 base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize);
2096 # if defined(MALLOC_DECOMMIT)
2097 if (pminsize < csize)
2098 pages_decommit(base_next_decommitted, csize - pminsize);
2099 # endif
2100 # ifdef MALLOC_STATS
2101 base_mapped += csize;
2102 base_committed += pminsize;
2103 # endif
2104 #endif
2105
2106 return (false);
2107 }
2108
2109 static void *
base_alloc(size_t size)2110 base_alloc(size_t size)
2111 {
2112 void *ret;
2113 size_t csize;
2114
2115 /* Round size up to nearest multiple of the cacheline size. */
2116 csize = CACHELINE_CEILING(size);
2117
2118 malloc_mutex_lock(&base_mtx);
2119 /* Make sure there's enough space for the allocation. */
2120 if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
2121 if (base_pages_alloc(csize)) {
2122 malloc_mutex_unlock(&base_mtx);
2123 return (NULL);
2124 }
2125 }
2126 /* Allocate. */
2127 ret = base_next_addr;
2128 base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
2129 #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
2130 /* Make sure enough pages are committed for the new allocation. */
2131 if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) {
2132 void *pbase_next_addr =
2133 (void *)(PAGE_CEILING((uintptr_t)base_next_addr));
2134
2135 # ifdef MALLOC_DECOMMIT
2136 pages_commit(base_next_decommitted, (uintptr_t)pbase_next_addr -
2137 (uintptr_t)base_next_decommitted);
2138 # endif
2139 base_next_decommitted = pbase_next_addr;
2140 # ifdef MALLOC_STATS
2141 base_committed += (uintptr_t)pbase_next_addr -
2142 (uintptr_t)base_next_decommitted;
2143 # endif
2144 }
2145 #endif
2146 malloc_mutex_unlock(&base_mtx);
2147
2148 return (ret);
2149 }
2150
2151 static void *
base_calloc(size_t number,size_t size)2152 base_calloc(size_t number, size_t size)
2153 {
2154 void *ret;
2155
2156 ret = base_alloc(number * size);
2157 memset(ret, 0, number * size);
2158
2159 return (ret);
2160 }
2161
2162 static extent_node_t *
base_node_alloc(void)2163 base_node_alloc(void)
2164 {
2165 extent_node_t *ret;
2166
2167 malloc_mutex_lock(&base_mtx);
2168 if (base_nodes != NULL) {
2169 ret = base_nodes;
2170 base_nodes = *(extent_node_t **)ret;
2171 malloc_mutex_unlock(&base_mtx);
2172 } else {
2173 malloc_mutex_unlock(&base_mtx);
2174 ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
2175 }
2176
2177 return (ret);
2178 }
2179
2180 static void
base_node_dealloc(extent_node_t * node)2181 base_node_dealloc(extent_node_t *node)
2182 {
2183
2184 malloc_mutex_lock(&base_mtx);
2185 *(extent_node_t **)node = base_nodes;
2186 base_nodes = node;
2187 malloc_mutex_unlock(&base_mtx);
2188 }
2189
2190 /******************************************************************************/
2191
2192 #ifdef MALLOC_STATS
2193 static void
stats_print(arena_t * arena)2194 stats_print(arena_t *arena)
2195 {
2196 unsigned i, gap_start;
2197
2198 #ifdef MOZ_MEMORY_WINDOWS
2199 malloc_printf("dirty: %Iu page%s dirty, %I64u sweep%s,"
2200 " %I64u madvise%s, %I64u page%s purged\n",
2201 arena->ndirty, arena->ndirty == 1 ? "" : "s",
2202 arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
2203 arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
2204 arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
2205 # ifdef MALLOC_DECOMMIT
2206 malloc_printf("decommit: %I64u decommit%s, %I64u commit%s,"
2207 " %I64u page%s decommitted\n",
2208 arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
2209 arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
2210 arena->stats.decommitted,
2211 (arena->stats.decommitted == 1) ? "" : "s");
2212 # endif
2213
2214 malloc_printf(" allocated nmalloc ndalloc\n");
2215 malloc_printf("small: %12Iu %12I64u %12I64u\n",
2216 arena->stats.allocated_small, arena->stats.nmalloc_small,
2217 arena->stats.ndalloc_small);
2218 malloc_printf("large: %12Iu %12I64u %12I64u\n",
2219 arena->stats.allocated_large, arena->stats.nmalloc_large,
2220 arena->stats.ndalloc_large);
2221 malloc_printf("total: %12Iu %12I64u %12I64u\n",
2222 arena->stats.allocated_small + arena->stats.allocated_large,
2223 arena->stats.nmalloc_small + arena->stats.nmalloc_large,
2224 arena->stats.ndalloc_small + arena->stats.ndalloc_large);
2225 malloc_printf("mapped: %12Iu\n", arena->stats.mapped);
2226 #else
2227 malloc_printf("dirty: %zu page%s dirty, %llu sweep%s,"
2228 " %llu madvise%s, %llu page%s purged\n",
2229 arena->ndirty, arena->ndirty == 1 ? "" : "s",
2230 arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
2231 arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
2232 arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
2233 # ifdef MALLOC_DECOMMIT
2234 malloc_printf("decommit: %llu decommit%s, %llu commit%s,"
2235 " %llu page%s decommitted\n",
2236 arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
2237 arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
2238 arena->stats.decommitted,
2239 (arena->stats.decommitted == 1) ? "" : "s");
2240 # endif
2241
2242 malloc_printf(" allocated nmalloc ndalloc\n");
2243 malloc_printf("small: %12zu %12llu %12llu\n",
2244 arena->stats.allocated_small, arena->stats.nmalloc_small,
2245 arena->stats.ndalloc_small);
2246 malloc_printf("large: %12zu %12llu %12llu\n",
2247 arena->stats.allocated_large, arena->stats.nmalloc_large,
2248 arena->stats.ndalloc_large);
2249 malloc_printf("total: %12zu %12llu %12llu\n",
2250 arena->stats.allocated_small + arena->stats.allocated_large,
2251 arena->stats.nmalloc_small + arena->stats.nmalloc_large,
2252 arena->stats.ndalloc_small + arena->stats.ndalloc_large);
2253 malloc_printf("mapped: %12zu\n", arena->stats.mapped);
2254 #endif
2255 malloc_printf("bins: bin size regs pgs requests newruns"
2256 " reruns maxruns curruns\n");
2257 for (i = 0, gap_start = UINT_MAX; i < ntbins + nqbins + nsbins; i++) {
2258 if (arena->bins[i].stats.nrequests == 0) {
2259 if (gap_start == UINT_MAX)
2260 gap_start = i;
2261 } else {
2262 if (gap_start != UINT_MAX) {
2263 if (i > gap_start + 1) {
2264 /* Gap of more than one size class. */
2265 malloc_printf("[%u..%u]\n",
2266 gap_start, i - 1);
2267 } else {
2268 /* Gap of one size class. */
2269 malloc_printf("[%u]\n", gap_start);
2270 }
2271 gap_start = UINT_MAX;
2272 }
2273 malloc_printf(
2274 #if defined(MOZ_MEMORY_WINDOWS)
2275 "%13u %1s %4u %4u %3u %9I64u %9I64u"
2276 " %9I64u %7u %7u\n",
2277 #else
2278 "%13u %1s %4u %4u %3u %9llu %9llu"
2279 " %9llu %7lu %7lu\n",
2280 #endif
2281 i,
2282 i < ntbins ? "T" : i < ntbins + nqbins ? "Q" : "S",
2283 arena->bins[i].reg_size,
2284 arena->bins[i].nregs,
2285 arena->bins[i].run_size >> pagesize_2pow,
2286 arena->bins[i].stats.nrequests,
2287 arena->bins[i].stats.nruns,
2288 arena->bins[i].stats.reruns,
2289 arena->bins[i].stats.highruns,
2290 arena->bins[i].stats.curruns);
2291 }
2292 }
2293 if (gap_start != UINT_MAX) {
2294 if (i > gap_start + 1) {
2295 /* Gap of more than one size class. */
2296 malloc_printf("[%u..%u]\n", gap_start, i - 1);
2297 } else {
2298 /* Gap of one size class. */
2299 malloc_printf("[%u]\n", gap_start);
2300 }
2301 }
2302 }
2303 #endif
2304
2305 /*
2306 * End Utility functions/macros.
2307 */
2308 /******************************************************************************/
2309 /*
2310 * Begin extent tree code.
2311 */
2312
2313 static inline int
extent_szad_comp(extent_node_t * a,extent_node_t * b)2314 extent_szad_comp(extent_node_t *a, extent_node_t *b)
2315 {
2316 int ret;
2317 size_t a_size = a->size;
2318 size_t b_size = b->size;
2319
2320 ret = (a_size > b_size) - (a_size < b_size);
2321 if (ret == 0) {
2322 uintptr_t a_addr = (uintptr_t)a->addr;
2323 uintptr_t b_addr = (uintptr_t)b->addr;
2324
2325 ret = (a_addr > b_addr) - (a_addr < b_addr);
2326 }
2327
2328 return (ret);
2329 }
2330
2331 /* Wrap red-black tree macros in functions. */
rb_wrap(static,extent_tree_szad_,extent_tree_t,extent_node_t,link_szad,extent_szad_comp)2332 rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
2333 link_szad, extent_szad_comp)
2334
2335 static inline int
2336 extent_ad_comp(extent_node_t *a, extent_node_t *b)
2337 {
2338 uintptr_t a_addr = (uintptr_t)a->addr;
2339 uintptr_t b_addr = (uintptr_t)b->addr;
2340
2341 return ((a_addr > b_addr) - (a_addr < b_addr));
2342 }
2343
2344 /* Wrap red-black tree macros in functions. */
rb_wrap(static,extent_tree_ad_,extent_tree_t,extent_node_t,link_ad,extent_ad_comp)2345 rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
2346 extent_ad_comp)
2347
2348 /*
2349 * End extent tree code.
2350 */
2351 /******************************************************************************/
2352 /*
2353 * Begin chunk management functions.
2354 */
2355
2356 #ifdef MOZ_MEMORY_WINDOWS
2357
2358 static void *
2359 pages_map(void *addr, size_t size)
2360 {
2361 void *ret = NULL;
2362 ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
2363 PAGE_READWRITE);
2364 return (ret);
2365 }
2366
2367 static void
pages_unmap(void * addr,size_t size)2368 pages_unmap(void *addr, size_t size)
2369 {
2370 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
2371 _malloc_message(_getprogname(),
2372 ": (malloc) Error in VirtualFree()\n", "", "");
2373 if (opt_abort)
2374 abort();
2375 }
2376 }
2377 #else
2378 #ifdef JEMALLOC_USES_MAP_ALIGN
2379 static void *
2380 pages_map_align(size_t size, size_t alignment)
2381 {
2382 void *ret;
2383
2384 /*
2385 * We don't use MAP_FIXED here, because it can cause the *replacement*
2386 * of existing mappings, and we only want to create new mappings.
2387 */
2388 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE,
2389 MAP_PRIVATE | MAP_NOSYNC | MAP_ALIGN | MAP_ANON, -1, 0);
2390 assert(ret != NULL);
2391
2392 if (ret == MAP_FAILED)
2393 ret = NULL;
2394 else
2395 MozTagAnonymousMemory(ret, size, "jemalloc");
2396 return (ret);
2397 }
2398 #endif
2399
2400 static void *
2401 pages_map(void *addr, size_t size)
2402 {
2403 void *ret;
2404 #if defined(__ia64__)
2405 /*
2406 * The JS engine assumes that all allocated pointers have their high 17 bits clear,
2407 * which ia64's mmap doesn't support directly. However, we can emulate it by passing
2408 * mmap an "addr" parameter with those bits clear. The mmap will return that address,
2409 * or the nearest available memory above that address, providing a near-guarantee
2410 * that those bits are clear. If they are not, we return NULL below to indicate
2411 * out-of-memory.
2412 *
2413 * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual
2414 * address space.
2415 *
2416 * See Bug 589735 for more information.
2417 */
2418 bool check_placement = true;
2419 if (addr == NULL) {
2420 addr = (void*)0x0000070000000000;
2421 check_placement = false;
2422 }
2423 #endif
2424
2425 /*
2426 * We don't use MAP_FIXED here, because it can cause the *replacement*
2427 * of existing mappings, and we only want to create new mappings.
2428 */
2429 ret = mmap(addr, size, PROT_READ | PROT_WRITE,
2430 MAP_PRIVATE | MAP_ANON, -1, 0);
2431 assert(ret != NULL);
2432
2433 if (ret == MAP_FAILED) {
2434 ret = NULL;
2435 }
2436 #if defined(__ia64__)
2437 /*
2438 * If the allocated memory doesn't have its upper 17 bits clear, consider it
2439 * as out of memory.
2440 */
2441 else if ((long long)ret & 0xffff800000000000) {
2442 munmap(ret, size);
2443 ret = NULL;
2444 }
2445 /* If the caller requested a specific memory location, verify that's what mmap returned. */
2446 else if (check_placement && ret != addr) {
2447 #else
2448 else if (addr != NULL && ret != addr) {
2449 #endif
2450 /*
2451 * We succeeded in mapping memory, but not in the right place.
2452 */
2453 if (munmap(ret, size) == -1) {
2454 char buf[STRERROR_BUF];
2455
2456 if (strerror_r(errno, buf, sizeof(buf)) == 0) {
2457 _malloc_message(_getprogname(),
2458 ": (malloc) Error in munmap(): ", buf, "\n");
2459 }
2460 if (opt_abort)
2461 abort();
2462 }
2463 ret = NULL;
2464 }
2465 if (ret != NULL) {
2466 MozTagAnonymousMemory(ret, size, "jemalloc");
2467 }
2468
2469 #if defined(__ia64__)
2470 assert(ret == NULL || (!check_placement && ret != NULL)
2471 || (check_placement && ret == addr));
2472 #else
2473 assert(ret == NULL || (addr == NULL && ret != addr)
2474 || (addr != NULL && ret == addr));
2475 #endif
2476 return (ret);
2477 }
2478
2479 static void
2480 pages_unmap(void *addr, size_t size)
2481 {
2482
2483 if (munmap(addr, size) == -1) {
2484 char buf[STRERROR_BUF];
2485
2486 if (strerror_r(errno, buf, sizeof(buf)) == 0) {
2487 _malloc_message(_getprogname(),
2488 ": (malloc) Error in munmap(): ", buf, "\n");
2489 }
2490 if (opt_abort)
2491 abort();
2492 }
2493 }
2494 #endif
2495
2496 #ifdef MOZ_MEMORY_DARWIN
2497 #define VM_COPY_MIN (pagesize << 5)
2498 static inline void
pages_copy(void * dest,const void * src,size_t n)2499 pages_copy(void *dest, const void *src, size_t n)
2500 {
2501
2502 assert((void *)((uintptr_t)dest & ~pagesize_mask) == dest);
2503 assert(n >= VM_COPY_MIN);
2504 assert((void *)((uintptr_t)src & ~pagesize_mask) == src);
2505
2506 vm_copy(mach_task_self(), (vm_address_t)src, (vm_size_t)n,
2507 (vm_address_t)dest);
2508 }
2509 #endif
2510
2511 #ifdef MALLOC_VALIDATE
2512 static inline malloc_rtree_t *
malloc_rtree_new(unsigned bits)2513 malloc_rtree_new(unsigned bits)
2514 {
2515 malloc_rtree_t *ret;
2516 unsigned bits_per_level, height, i;
2517
2518 bits_per_level = ffs(pow2_ceil((MALLOC_RTREE_NODESIZE /
2519 sizeof(void *)))) - 1;
2520 height = bits / bits_per_level;
2521 if (height * bits_per_level != bits)
2522 height++;
2523 RELEASE_ASSERT(height * bits_per_level >= bits);
2524
2525 ret = (malloc_rtree_t*)base_calloc(1, sizeof(malloc_rtree_t) +
2526 (sizeof(unsigned) * (height - 1)));
2527 if (ret == NULL)
2528 return (NULL);
2529
2530 malloc_spin_init(&ret->lock);
2531 ret->height = height;
2532 if (bits_per_level * height > bits)
2533 ret->level2bits[0] = bits % bits_per_level;
2534 else
2535 ret->level2bits[0] = bits_per_level;
2536 for (i = 1; i < height; i++)
2537 ret->level2bits[i] = bits_per_level;
2538
2539 ret->root = (void**)base_calloc(1, sizeof(void *) << ret->level2bits[0]);
2540 if (ret->root == NULL) {
2541 /*
2542 * We leak the rtree here, since there's no generic base
2543 * deallocation.
2544 */
2545 return (NULL);
2546 }
2547
2548 return (ret);
2549 }
2550
2551 #define MALLOC_RTREE_GET_GENERATE(f) \
2552 /* The least significant bits of the key are ignored. */ \
2553 static inline void * \
2554 f(malloc_rtree_t *rtree, uintptr_t key) \
2555 { \
2556 void *ret; \
2557 uintptr_t subkey; \
2558 unsigned i, lshift, height, bits; \
2559 void **node, **child; \
2560 \
2561 MALLOC_RTREE_LOCK(&rtree->lock); \
2562 for (i = lshift = 0, height = rtree->height, node = rtree->root;\
2563 i < height - 1; \
2564 i++, lshift += bits, node = child) { \
2565 bits = rtree->level2bits[i]; \
2566 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); \
2567 child = (void**)node[subkey]; \
2568 if (child == NULL) { \
2569 MALLOC_RTREE_UNLOCK(&rtree->lock); \
2570 return (NULL); \
2571 } \
2572 } \
2573 \
2574 /* \
2575 * node is a leaf, so it contains values rather than node \
2576 * pointers. \
2577 */ \
2578 bits = rtree->level2bits[i]; \
2579 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); \
2580 ret = node[subkey]; \
2581 MALLOC_RTREE_UNLOCK(&rtree->lock); \
2582 \
2583 MALLOC_RTREE_GET_VALIDATE \
2584 return (ret); \
2585 }
2586
2587 #ifdef MALLOC_DEBUG
2588 # define MALLOC_RTREE_LOCK(l) malloc_spin_lock(l)
2589 # define MALLOC_RTREE_UNLOCK(l) malloc_spin_unlock(l)
2590 # define MALLOC_RTREE_GET_VALIDATE
2591 MALLOC_RTREE_GET_GENERATE(malloc_rtree_get_locked)
2592 # undef MALLOC_RTREE_LOCK
2593 # undef MALLOC_RTREE_UNLOCK
2594 # undef MALLOC_RTREE_GET_VALIDATE
2595 #endif
2596
2597 #define MALLOC_RTREE_LOCK(l)
2598 #define MALLOC_RTREE_UNLOCK(l)
2599 #ifdef MALLOC_DEBUG
2600 /*
2601 * Suppose that it were possible for a jemalloc-allocated chunk to be
2602 * munmap()ped, followed by a different allocator in another thread re-using
2603 * overlapping virtual memory, all without invalidating the cached rtree
2604 * value. The result would be a false positive (the rtree would claim that
2605 * jemalloc owns memory that it had actually discarded). I don't think this
2606 * scenario is possible, but the following assertion is a prudent sanity
2607 * check.
2608 */
2609 # define MALLOC_RTREE_GET_VALIDATE \
2610 assert(malloc_rtree_get_locked(rtree, key) == ret);
2611 #else
2612 # define MALLOC_RTREE_GET_VALIDATE
2613 #endif
MALLOC_RTREE_GET_GENERATE(malloc_rtree_get)2614 MALLOC_RTREE_GET_GENERATE(malloc_rtree_get)
2615 #undef MALLOC_RTREE_LOCK
2616 #undef MALLOC_RTREE_UNLOCK
2617 #undef MALLOC_RTREE_GET_VALIDATE
2618
2619 static inline bool
2620 malloc_rtree_set(malloc_rtree_t *rtree, uintptr_t key, void *val)
2621 {
2622 uintptr_t subkey;
2623 unsigned i, lshift, height, bits;
2624 void **node, **child;
2625
2626 malloc_spin_lock(&rtree->lock);
2627 for (i = lshift = 0, height = rtree->height, node = rtree->root;
2628 i < height - 1;
2629 i++, lshift += bits, node = child) {
2630 bits = rtree->level2bits[i];
2631 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
2632 child = (void**)node[subkey];
2633 if (child == NULL) {
2634 child = (void**)base_calloc(1, sizeof(void *) <<
2635 rtree->level2bits[i+1]);
2636 if (child == NULL) {
2637 malloc_spin_unlock(&rtree->lock);
2638 return (true);
2639 }
2640 node[subkey] = child;
2641 }
2642 }
2643
2644 /* node is a leaf, so it contains values rather than node pointers. */
2645 bits = rtree->level2bits[i];
2646 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
2647 node[subkey] = val;
2648 malloc_spin_unlock(&rtree->lock);
2649
2650 return (false);
2651 }
2652 #endif
2653
2654 /* pages_trim, chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked
2655 * from upstream jemalloc 3.4.1 to fix Mozilla bug 956501. */
2656
2657 /* Return the offset between a and the nearest aligned address at or below a. */
2658 #define ALIGNMENT_ADDR2OFFSET(a, alignment) \
2659 ((size_t)((uintptr_t)(a) & (alignment - 1)))
2660
2661 /* Return the smallest alignment multiple that is >= s. */
2662 #define ALIGNMENT_CEILING(s, alignment) \
2663 (((s) + (alignment - 1)) & (-(alignment)))
2664
2665 static void *
pages_trim(void * addr,size_t alloc_size,size_t leadsize,size_t size)2666 pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
2667 {
2668 void *ret = (void *)((uintptr_t)addr + leadsize);
2669
2670 assert(alloc_size >= leadsize + size);
2671 #ifdef MOZ_MEMORY_WINDOWS
2672 {
2673 void *new_addr;
2674
2675 pages_unmap(addr, alloc_size);
2676 new_addr = pages_map(ret, size);
2677 if (new_addr == ret)
2678 return (ret);
2679 if (new_addr)
2680 pages_unmap(new_addr, size);
2681 return (NULL);
2682 }
2683 #else
2684 {
2685 size_t trailsize = alloc_size - leadsize - size;
2686
2687 if (leadsize != 0)
2688 pages_unmap(addr, leadsize);
2689 if (trailsize != 0)
2690 pages_unmap((void *)((uintptr_t)ret + size), trailsize);
2691 return (ret);
2692 }
2693 #endif
2694 }
2695
2696 static void *
chunk_alloc_mmap_slow(size_t size,size_t alignment)2697 chunk_alloc_mmap_slow(size_t size, size_t alignment)
2698 {
2699 void *ret, *pages;
2700 size_t alloc_size, leadsize;
2701
2702 alloc_size = size + alignment - pagesize;
2703 /* Beware size_t wrap-around. */
2704 if (alloc_size < size)
2705 return (NULL);
2706 do {
2707 pages = pages_map(NULL, alloc_size);
2708 if (pages == NULL)
2709 return (NULL);
2710 leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
2711 (uintptr_t)pages;
2712 ret = pages_trim(pages, alloc_size, leadsize, size);
2713 } while (ret == NULL);
2714
2715 assert(ret != NULL);
2716 return (ret);
2717 }
2718
2719 static void *
chunk_alloc_mmap(size_t size,size_t alignment)2720 chunk_alloc_mmap(size_t size, size_t alignment)
2721 {
2722 #ifdef JEMALLOC_USES_MAP_ALIGN
2723 return pages_map_align(size, alignment);
2724 #else
2725 void *ret;
2726 size_t offset;
2727
2728 /*
2729 * Ideally, there would be a way to specify alignment to mmap() (like
2730 * NetBSD has), but in the absence of such a feature, we have to work
2731 * hard to efficiently create aligned mappings. The reliable, but
2732 * slow method is to create a mapping that is over-sized, then trim the
2733 * excess. However, that always results in one or two calls to
2734 * pages_unmap().
2735 *
2736 * Optimistically try mapping precisely the right amount before falling
2737 * back to the slow method, with the expectation that the optimistic
2738 * approach works most of the time.
2739 */
2740
2741 ret = pages_map(NULL, size);
2742 if (ret == NULL)
2743 return (NULL);
2744 offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
2745 if (offset != 0) {
2746 pages_unmap(ret, size);
2747 return (chunk_alloc_mmap_slow(size, alignment));
2748 }
2749
2750 assert(ret != NULL);
2751 return (ret);
2752 #endif
2753 }
2754
2755 bool
pages_purge(void * addr,size_t length)2756 pages_purge(void *addr, size_t length)
2757 {
2758 bool unzeroed;
2759
2760 #ifdef MALLOC_DECOMMIT
2761 pages_decommit(addr, length);
2762 unzeroed = false;
2763 #else
2764 # ifdef MOZ_MEMORY_WINDOWS
2765 /*
2766 * The region starting at addr may have been allocated in multiple calls
2767 * to VirtualAlloc and recycled, so resetting the entire region in one
2768 * go may not be valid. However, since we allocate at least a chunk at a
2769 * time, we may touch any region in chunksized increments.
2770 */
2771 size_t pages_size = min(length, chunksize -
2772 CHUNK_ADDR2OFFSET((uintptr_t)addr));
2773 while (length > 0) {
2774 VirtualAlloc(addr, pages_size, MEM_RESET, PAGE_READWRITE);
2775 addr = (void *)((uintptr_t)addr + pages_size);
2776 length -= pages_size;
2777 pages_size = min(length, chunksize);
2778 }
2779 unzeroed = true;
2780 # else
2781 # ifdef MOZ_MEMORY_LINUX
2782 # define JEMALLOC_MADV_PURGE MADV_DONTNEED
2783 # define JEMALLOC_MADV_ZEROS true
2784 # else /* FreeBSD and Darwin. */
2785 # define JEMALLOC_MADV_PURGE MADV_FREE
2786 # define JEMALLOC_MADV_ZEROS false
2787 # endif
2788 int err = madvise(addr, length, JEMALLOC_MADV_PURGE);
2789 unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0);
2790 # undef JEMALLOC_MADV_PURGE
2791 # undef JEMALLOC_MADV_ZEROS
2792 # endif
2793 #endif
2794 return (unzeroed);
2795 }
2796
2797 static void *
chunk_recycle(extent_tree_t * chunks_szad,extent_tree_t * chunks_ad,size_t size,size_t alignment,bool base,bool * zero)2798 chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size,
2799 size_t alignment, bool base, bool *zero)
2800 {
2801 void *ret;
2802 extent_node_t *node;
2803 extent_node_t key;
2804 size_t alloc_size, leadsize, trailsize;
2805 bool zeroed;
2806
2807 if (base) {
2808 /*
2809 * This function may need to call base_node_{,de}alloc(), but
2810 * the current chunk allocation request is on behalf of the
2811 * base allocator. Avoid deadlock (and if that weren't an
2812 * issue, potential for infinite recursion) by returning NULL.
2813 */
2814 return (NULL);
2815 }
2816
2817 alloc_size = size + alignment - chunksize;
2818 /* Beware size_t wrap-around. */
2819 if (alloc_size < size)
2820 return (NULL);
2821 key.addr = NULL;
2822 key.size = alloc_size;
2823 malloc_mutex_lock(&chunks_mtx);
2824 node = extent_tree_szad_nsearch(chunks_szad, &key);
2825 if (node == NULL) {
2826 malloc_mutex_unlock(&chunks_mtx);
2827 return (NULL);
2828 }
2829 leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) -
2830 (uintptr_t)node->addr;
2831 assert(node->size >= leadsize + size);
2832 trailsize = node->size - leadsize - size;
2833 ret = (void *)((uintptr_t)node->addr + leadsize);
2834 zeroed = node->zeroed;
2835 if (zeroed)
2836 *zero = true;
2837 /* Remove node from the tree. */
2838 extent_tree_szad_remove(chunks_szad, node);
2839 extent_tree_ad_remove(chunks_ad, node);
2840 if (leadsize != 0) {
2841 /* Insert the leading space as a smaller chunk. */
2842 node->size = leadsize;
2843 extent_tree_szad_insert(chunks_szad, node);
2844 extent_tree_ad_insert(chunks_ad, node);
2845 node = NULL;
2846 }
2847 if (trailsize != 0) {
2848 /* Insert the trailing space as a smaller chunk. */
2849 if (node == NULL) {
2850 /*
2851 * An additional node is required, but
2852 * base_node_alloc() can cause a new base chunk to be
2853 * allocated. Drop chunks_mtx in order to avoid
2854 * deadlock, and if node allocation fails, deallocate
2855 * the result before returning an error.
2856 */
2857 malloc_mutex_unlock(&chunks_mtx);
2858 node = base_node_alloc();
2859 if (node == NULL) {
2860 chunk_dealloc(ret, size);
2861 return (NULL);
2862 }
2863 malloc_mutex_lock(&chunks_mtx);
2864 }
2865 node->addr = (void *)((uintptr_t)(ret) + size);
2866 node->size = trailsize;
2867 node->zeroed = zeroed;
2868 extent_tree_szad_insert(chunks_szad, node);
2869 extent_tree_ad_insert(chunks_ad, node);
2870 node = NULL;
2871 }
2872
2873 if (config_munmap && config_recycle)
2874 recycled_size -= size;
2875
2876 malloc_mutex_unlock(&chunks_mtx);
2877
2878 if (node != NULL)
2879 base_node_dealloc(node);
2880 #ifdef MALLOC_DECOMMIT
2881 pages_commit(ret, size);
2882 #endif
2883 if (*zero) {
2884 if (zeroed == false)
2885 memset(ret, 0, size);
2886 #ifdef DEBUG
2887 else {
2888 size_t i;
2889 size_t *p = (size_t *)(uintptr_t)ret;
2890
2891 for (i = 0; i < size / sizeof(size_t); i++)
2892 assert(p[i] == 0);
2893 }
2894 #endif
2895 }
2896 return (ret);
2897 }
2898
2899 #ifdef MOZ_MEMORY_WINDOWS
2900 /*
2901 * On Windows, calls to VirtualAlloc and VirtualFree must be matched, making it
2902 * awkward to recycle allocations of varying sizes. Therefore we only allow
2903 * recycling when the size equals the chunksize, unless deallocation is entirely
2904 * disabled.
2905 */
2906 #define CAN_RECYCLE(size) (size == chunksize)
2907 #else
2908 #define CAN_RECYCLE(size) true
2909 #endif
2910
2911 static void *
chunk_alloc(size_t size,size_t alignment,bool base,bool zero)2912 chunk_alloc(size_t size, size_t alignment, bool base, bool zero)
2913 {
2914 void *ret;
2915
2916 assert(size != 0);
2917 assert((size & chunksize_mask) == 0);
2918 assert(alignment != 0);
2919 assert((alignment & chunksize_mask) == 0);
2920
2921 if (!config_munmap || (config_recycle && CAN_RECYCLE(size))) {
2922 ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap,
2923 size, alignment, base, &zero);
2924 if (ret != NULL)
2925 goto RETURN;
2926 }
2927 ret = chunk_alloc_mmap(size, alignment);
2928 if (ret != NULL) {
2929 goto RETURN;
2930 }
2931
2932 /* All strategies for allocation failed. */
2933 ret = NULL;
2934 RETURN:
2935
2936 #ifdef MALLOC_VALIDATE
2937 if (ret != NULL && base == false) {
2938 if (malloc_rtree_set(chunk_rtree, (uintptr_t)ret, ret)) {
2939 chunk_dealloc(ret, size);
2940 return (NULL);
2941 }
2942 }
2943 #endif
2944
2945 assert(CHUNK_ADDR2BASE(ret) == ret);
2946 return (ret);
2947 }
2948
2949 static void
chunk_record(extent_tree_t * chunks_szad,extent_tree_t * chunks_ad,void * chunk,size_t size)2950 chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
2951 size_t size)
2952 {
2953 bool unzeroed;
2954 extent_node_t *xnode, *node, *prev, *xprev, key;
2955
2956 unzeroed = pages_purge(chunk, size);
2957
2958 /*
2959 * Allocate a node before acquiring chunks_mtx even though it might not
2960 * be needed, because base_node_alloc() may cause a new base chunk to
2961 * be allocated, which could cause deadlock if chunks_mtx were already
2962 * held.
2963 */
2964 xnode = base_node_alloc();
2965 /* Use xprev to implement conditional deferred deallocation of prev. */
2966 xprev = NULL;
2967
2968 malloc_mutex_lock(&chunks_mtx);
2969 key.addr = (void *)((uintptr_t)chunk + size);
2970 node = extent_tree_ad_nsearch(chunks_ad, &key);
2971 /* Try to coalesce forward. */
2972 if (node != NULL && node->addr == key.addr) {
2973 /*
2974 * Coalesce chunk with the following address range. This does
2975 * not change the position within chunks_ad, so only
2976 * remove/insert from/into chunks_szad.
2977 */
2978 extent_tree_szad_remove(chunks_szad, node);
2979 node->addr = chunk;
2980 node->size += size;
2981 node->zeroed = (node->zeroed && (unzeroed == false));
2982 extent_tree_szad_insert(chunks_szad, node);
2983 } else {
2984 /* Coalescing forward failed, so insert a new node. */
2985 if (xnode == NULL) {
2986 /*
2987 * base_node_alloc() failed, which is an exceedingly
2988 * unlikely failure. Leak chunk; its pages have
2989 * already been purged, so this is only a virtual
2990 * memory leak.
2991 */
2992 goto label_return;
2993 }
2994 node = xnode;
2995 xnode = NULL; /* Prevent deallocation below. */
2996 node->addr = chunk;
2997 node->size = size;
2998 node->zeroed = (unzeroed == false);
2999 extent_tree_ad_insert(chunks_ad, node);
3000 extent_tree_szad_insert(chunks_szad, node);
3001 }
3002
3003 /* Try to coalesce backward. */
3004 prev = extent_tree_ad_prev(chunks_ad, node);
3005 if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
3006 chunk) {
3007 /*
3008 * Coalesce chunk with the previous address range. This does
3009 * not change the position within chunks_ad, so only
3010 * remove/insert node from/into chunks_szad.
3011 */
3012 extent_tree_szad_remove(chunks_szad, prev);
3013 extent_tree_ad_remove(chunks_ad, prev);
3014
3015 extent_tree_szad_remove(chunks_szad, node);
3016 node->addr = prev->addr;
3017 node->size += prev->size;
3018 node->zeroed = (node->zeroed && prev->zeroed);
3019 extent_tree_szad_insert(chunks_szad, node);
3020
3021 xprev = prev;
3022 }
3023
3024 if (config_munmap && config_recycle)
3025 recycled_size += size;
3026
3027 label_return:
3028 malloc_mutex_unlock(&chunks_mtx);
3029 /*
3030 * Deallocate xnode and/or xprev after unlocking chunks_mtx in order to
3031 * avoid potential deadlock.
3032 */
3033 if (xnode != NULL)
3034 base_node_dealloc(xnode);
3035 if (xprev != NULL)
3036 base_node_dealloc(xprev);
3037 }
3038
3039 static bool
chunk_dalloc_mmap(void * chunk,size_t size)3040 chunk_dalloc_mmap(void *chunk, size_t size)
3041 {
3042 if (!config_munmap || (config_recycle && CAN_RECYCLE(size) &&
3043 load_acquire_z(&recycled_size) < recycle_limit))
3044 return true;
3045
3046 pages_unmap(chunk, size);
3047 return false;
3048 }
3049
3050 #undef CAN_RECYCLE
3051
3052 static void
chunk_dealloc(void * chunk,size_t size)3053 chunk_dealloc(void *chunk, size_t size)
3054 {
3055
3056 assert(chunk != NULL);
3057 assert(CHUNK_ADDR2BASE(chunk) == chunk);
3058 assert(size != 0);
3059 assert((size & chunksize_mask) == 0);
3060
3061 #ifdef MALLOC_VALIDATE
3062 malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL);
3063 #endif
3064
3065 if (chunk_dalloc_mmap(chunk, size))
3066 chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size);
3067 }
3068
3069 /*
3070 * End chunk management functions.
3071 */
3072 /******************************************************************************/
3073 /*
3074 * Begin arena.
3075 */
3076
3077 /*
3078 * Choose an arena based on a per-thread value (fast-path code, calls slow-path
3079 * code if necessary).
3080 */
3081 static inline arena_t *
choose_arena(void)3082 choose_arena(void)
3083 {
3084 arena_t *ret;
3085
3086 /*
3087 * We can only use TLS if this is a PIC library, since for the static
3088 * library version, libc's malloc is used by TLS allocation, which
3089 * introduces a bootstrapping issue.
3090 */
3091 #ifndef NO_TLS
3092 if (isthreaded == false) {
3093 /* Avoid the overhead of TLS for single-threaded operation. */
3094 return (arenas[0]);
3095 }
3096
3097 # ifdef MOZ_MEMORY_WINDOWS
3098 ret = (arena_t*)TlsGetValue(tlsIndex);
3099 # else
3100 ret = arenas_map;
3101 # endif
3102
3103 if (ret == NULL) {
3104 ret = choose_arena_hard();
3105 RELEASE_ASSERT(ret != NULL);
3106 }
3107 #else
3108 if (isthreaded && narenas > 1) {
3109 unsigned long ind;
3110
3111 /*
3112 * Hash _pthread_self() to one of the arenas. There is a prime
3113 * number of arenas, so this has a reasonable chance of
3114 * working. Even so, the hashing can be easily thwarted by
3115 * inconvenient _pthread_self() values. Without specific
3116 * knowledge of how _pthread_self() calculates values, we can't
3117 * easily do much better than this.
3118 */
3119 ind = (unsigned long) _pthread_self() % narenas;
3120
3121 /*
3122 * Optimistially assume that arenas[ind] has been initialized.
3123 * At worst, we find out that some other thread has already
3124 * done so, after acquiring the lock in preparation. Note that
3125 * this lazy locking also has the effect of lazily forcing
3126 * cache coherency; without the lock acquisition, there's no
3127 * guarantee that modification of arenas[ind] by another thread
3128 * would be seen on this CPU for an arbitrary amount of time.
3129 *
3130 * In general, this approach to modifying a synchronized value
3131 * isn't a good idea, but in this case we only ever modify the
3132 * value once, so things work out well.
3133 */
3134 ret = arenas[ind];
3135 if (ret == NULL) {
3136 /*
3137 * Avoid races with another thread that may have already
3138 * initialized arenas[ind].
3139 */
3140 malloc_spin_lock(&arenas_lock);
3141 if (arenas[ind] == NULL)
3142 ret = arenas_extend((unsigned)ind);
3143 else
3144 ret = arenas[ind];
3145 malloc_spin_unlock(&arenas_lock);
3146 }
3147 } else
3148 ret = arenas[0];
3149 #endif
3150
3151 RELEASE_ASSERT(ret != NULL);
3152 return (ret);
3153 }
3154
3155 #ifndef NO_TLS
3156 /*
3157 * Choose an arena based on a per-thread value (slow-path code only, called
3158 * only by choose_arena()).
3159 */
3160 static arena_t *
choose_arena_hard(void)3161 choose_arena_hard(void)
3162 {
3163 arena_t *ret;
3164
3165 assert(isthreaded);
3166
3167 #ifdef MALLOC_BALANCE
3168 /* Seed the PRNG used for arena load balancing. */
3169 SPRN(balance, (uint32_t)(uintptr_t)(_pthread_self()));
3170 #endif
3171
3172 if (narenas > 1) {
3173 #ifdef MALLOC_BALANCE
3174 unsigned ind;
3175
3176 ind = PRN(balance, narenas_2pow);
3177 if ((ret = arenas[ind]) == NULL) {
3178 malloc_spin_lock(&arenas_lock);
3179 if ((ret = arenas[ind]) == NULL)
3180 ret = arenas_extend(ind);
3181 malloc_spin_unlock(&arenas_lock);
3182 }
3183 #else
3184 malloc_spin_lock(&arenas_lock);
3185 if ((ret = arenas[next_arena]) == NULL)
3186 ret = arenas_extend(next_arena);
3187 next_arena = (next_arena + 1) % narenas;
3188 malloc_spin_unlock(&arenas_lock);
3189 #endif
3190 } else
3191 ret = arenas[0];
3192
3193 #ifdef MOZ_MEMORY_WINDOWS
3194 TlsSetValue(tlsIndex, ret);
3195 #else
3196 arenas_map = ret;
3197 #endif
3198
3199 return (ret);
3200 }
3201 #endif
3202
3203 static inline int
arena_chunk_comp(arena_chunk_t * a,arena_chunk_t * b)3204 arena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b)
3205 {
3206 uintptr_t a_chunk = (uintptr_t)a;
3207 uintptr_t b_chunk = (uintptr_t)b;
3208
3209 assert(a != NULL);
3210 assert(b != NULL);
3211
3212 return ((a_chunk > b_chunk) - (a_chunk < b_chunk));
3213 }
3214
3215 /* Wrap red-black tree macros in functions. */
rb_wrap(static,arena_chunk_tree_dirty_,arena_chunk_tree_t,arena_chunk_t,link_dirty,arena_chunk_comp)3216 rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
3217 arena_chunk_t, link_dirty, arena_chunk_comp)
3218
3219 static inline int
3220 arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
3221 {
3222 uintptr_t a_mapelm = (uintptr_t)a;
3223 uintptr_t b_mapelm = (uintptr_t)b;
3224
3225 assert(a != NULL);
3226 assert(b != NULL);
3227
3228 return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm));
3229 }
3230
3231 /* Wrap red-black tree macros in functions. */
rb_wrap(static,arena_run_tree_,arena_run_tree_t,arena_chunk_map_t,link,arena_run_comp)3232 rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, link,
3233 arena_run_comp)
3234
3235 static inline int
3236 arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
3237 {
3238 int ret;
3239 size_t a_size = a->bits & ~pagesize_mask;
3240 size_t b_size = b->bits & ~pagesize_mask;
3241
3242 ret = (a_size > b_size) - (a_size < b_size);
3243 if (ret == 0) {
3244 uintptr_t a_mapelm, b_mapelm;
3245
3246 if ((a->bits & CHUNK_MAP_KEY) == 0)
3247 a_mapelm = (uintptr_t)a;
3248 else {
3249 /*
3250 * Treat keys as though they are lower than anything
3251 * else.
3252 */
3253 a_mapelm = 0;
3254 }
3255 b_mapelm = (uintptr_t)b;
3256
3257 ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm);
3258 }
3259
3260 return (ret);
3261 }
3262
3263 /* Wrap red-black tree macros in functions. */
rb_wrap(static,arena_avail_tree_,arena_avail_tree_t,arena_chunk_map_t,link,arena_avail_comp)3264 rb_wrap(static, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, link,
3265 arena_avail_comp)
3266
3267 static inline void *
3268 arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
3269 {
3270 void *ret;
3271 unsigned i, mask, bit, regind;
3272
3273 assert(run->magic == ARENA_RUN_MAGIC);
3274 assert(run->regs_minelm < bin->regs_mask_nelms);
3275
3276 /*
3277 * Move the first check outside the loop, so that run->regs_minelm can
3278 * be updated unconditionally, without the possibility of updating it
3279 * multiple times.
3280 */
3281 i = run->regs_minelm;
3282 mask = run->regs_mask[i];
3283 if (mask != 0) {
3284 /* Usable allocation found. */
3285 bit = ffs((int)mask) - 1;
3286
3287 regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
3288 assert(regind < bin->nregs);
3289 ret = (void *)(((uintptr_t)run) + bin->reg0_offset
3290 + (bin->reg_size * regind));
3291
3292 /* Clear bit. */
3293 mask ^= (1U << bit);
3294 run->regs_mask[i] = mask;
3295
3296 return (ret);
3297 }
3298
3299 for (i++; i < bin->regs_mask_nelms; i++) {
3300 mask = run->regs_mask[i];
3301 if (mask != 0) {
3302 /* Usable allocation found. */
3303 bit = ffs((int)mask) - 1;
3304
3305 regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
3306 assert(regind < bin->nregs);
3307 ret = (void *)(((uintptr_t)run) + bin->reg0_offset
3308 + (bin->reg_size * regind));
3309
3310 /* Clear bit. */
3311 mask ^= (1U << bit);
3312 run->regs_mask[i] = mask;
3313
3314 /*
3315 * Make a note that nothing before this element
3316 * contains a free region.
3317 */
3318 run->regs_minelm = i; /* Low payoff: + (mask == 0); */
3319
3320 return (ret);
3321 }
3322 }
3323 /* Not reached. */
3324 RELEASE_ASSERT(0);
3325 return (NULL);
3326 }
3327
3328 static inline void
arena_run_reg_dalloc(arena_run_t * run,arena_bin_t * bin,void * ptr,size_t size)3329 arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size)
3330 {
3331 /*
3332 * To divide by a number D that is not a power of two we multiply
3333 * by (2^21 / D) and then right shift by 21 positions.
3334 *
3335 * X / D
3336 *
3337 * becomes
3338 *
3339 * (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT
3340 */
3341 #define SIZE_INV_SHIFT 21
3342 #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1)
3343 static const unsigned size_invs[] = {
3344 SIZE_INV(3),
3345 SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
3346 SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
3347 SIZE_INV(12),SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
3348 SIZE_INV(16),SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
3349 SIZE_INV(20),SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
3350 SIZE_INV(24),SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
3351 SIZE_INV(28),SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
3352 #if (QUANTUM_2POW_MIN < 4)
3353 ,
3354 SIZE_INV(32), SIZE_INV(33), SIZE_INV(34), SIZE_INV(35),
3355 SIZE_INV(36), SIZE_INV(37), SIZE_INV(38), SIZE_INV(39),
3356 SIZE_INV(40), SIZE_INV(41), SIZE_INV(42), SIZE_INV(43),
3357 SIZE_INV(44), SIZE_INV(45), SIZE_INV(46), SIZE_INV(47),
3358 SIZE_INV(48), SIZE_INV(49), SIZE_INV(50), SIZE_INV(51),
3359 SIZE_INV(52), SIZE_INV(53), SIZE_INV(54), SIZE_INV(55),
3360 SIZE_INV(56), SIZE_INV(57), SIZE_INV(58), SIZE_INV(59),
3361 SIZE_INV(60), SIZE_INV(61), SIZE_INV(62), SIZE_INV(63)
3362 #endif
3363 };
3364 unsigned diff, regind, elm, bit;
3365
3366 assert(run->magic == ARENA_RUN_MAGIC);
3367 assert(((sizeof(size_invs)) / sizeof(unsigned)) + 3
3368 >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN));
3369
3370 /*
3371 * Avoid doing division with a variable divisor if possible. Using
3372 * actual division here can reduce allocator throughput by over 20%!
3373 */
3374 diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
3375 if ((size & (size - 1)) == 0) {
3376 /*
3377 * log2_table allows fast division of a power of two in the
3378 * [1..128] range.
3379 *
3380 * (x / divisor) becomes (x >> log2_table[divisor - 1]).
3381 */
3382 static const unsigned char log2_table[] = {
3383 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
3384 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
3385 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3386 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
3387 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3388 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3389 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3390 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
3391 };
3392
3393 if (size <= 128)
3394 regind = (diff >> log2_table[size - 1]);
3395 else if (size <= 32768)
3396 regind = diff >> (8 + log2_table[(size >> 8) - 1]);
3397 else {
3398 /*
3399 * The run size is too large for us to use the lookup
3400 * table. Use real division.
3401 */
3402 regind = diff / size;
3403 }
3404 } else if (size <= ((sizeof(size_invs) / sizeof(unsigned))
3405 << QUANTUM_2POW_MIN) + 2) {
3406 regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff;
3407 regind >>= SIZE_INV_SHIFT;
3408 } else {
3409 /*
3410 * size_invs isn't large enough to handle this size class, so
3411 * calculate regind using actual division. This only happens
3412 * if the user increases small_max via the 'S' runtime
3413 * configuration option.
3414 */
3415 regind = diff / size;
3416 };
3417 RELEASE_ASSERT(diff == regind * size);
3418 RELEASE_ASSERT(regind < bin->nregs);
3419
3420 elm = regind >> (SIZEOF_INT_2POW + 3);
3421 if (elm < run->regs_minelm)
3422 run->regs_minelm = elm;
3423 bit = regind - (elm << (SIZEOF_INT_2POW + 3));
3424 RELEASE_ASSERT((run->regs_mask[elm] & (1U << bit)) == 0);
3425 run->regs_mask[elm] |= (1U << bit);
3426 #undef SIZE_INV
3427 #undef SIZE_INV_SHIFT
3428 }
3429
3430 static void
arena_run_split(arena_t * arena,arena_run_t * run,size_t size,bool large,bool zero)3431 arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
3432 bool zero)
3433 {
3434 arena_chunk_t *chunk;
3435 size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i;
3436
3437 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
3438 old_ndirty = chunk->ndirty;
3439 run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk)
3440 >> pagesize_2pow);
3441 total_pages = (chunk->map[run_ind].bits & ~pagesize_mask) >>
3442 pagesize_2pow;
3443 need_pages = (size >> pagesize_2pow);
3444 assert(need_pages > 0);
3445 assert(need_pages <= total_pages);
3446 rem_pages = total_pages - need_pages;
3447
3448 arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]);
3449
3450 /* Keep track of trailing unused pages for later use. */
3451 if (rem_pages > 0) {
3452 chunk->map[run_ind+need_pages].bits = (rem_pages <<
3453 pagesize_2pow) | (chunk->map[run_ind+need_pages].bits &
3454 pagesize_mask);
3455 chunk->map[run_ind+total_pages-1].bits = (rem_pages <<
3456 pagesize_2pow) | (chunk->map[run_ind+total_pages-1].bits &
3457 pagesize_mask);
3458 arena_avail_tree_insert(&arena->runs_avail,
3459 &chunk->map[run_ind+need_pages]);
3460 }
3461
3462 for (i = 0; i < need_pages; i++) {
3463 #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
3464 /*
3465 * Commit decommitted pages if necessary. If a decommitted
3466 * page is encountered, commit all needed adjacent decommitted
3467 * pages in one operation, in order to reduce system call
3468 * overhead.
3469 */
3470 if (chunk->map[run_ind + i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) {
3471 size_t j;
3472
3473 /*
3474 * Advance i+j to just past the index of the last page
3475 * to commit. Clear CHUNK_MAP_DECOMMITTED and
3476 * CHUNK_MAP_MADVISED along the way.
3477 */
3478 for (j = 0; i + j < need_pages && (chunk->map[run_ind +
3479 i + j].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED); j++) {
3480 /* DECOMMITTED and MADVISED are mutually exclusive. */
3481 assert(!(chunk->map[run_ind + i + j].bits & CHUNK_MAP_DECOMMITTED &&
3482 chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED));
3483
3484 chunk->map[run_ind + i + j].bits &=
3485 ~CHUNK_MAP_MADVISED_OR_DECOMMITTED;
3486 }
3487
3488 # ifdef MALLOC_DECOMMIT
3489 pages_commit((void *)((uintptr_t)chunk + ((run_ind + i)
3490 << pagesize_2pow)), (j << pagesize_2pow));
3491 # ifdef MALLOC_STATS
3492 arena->stats.ncommit++;
3493 # endif
3494 # endif
3495
3496 # ifdef MALLOC_STATS
3497 arena->stats.committed += j;
3498 # endif
3499
3500 # ifndef MALLOC_DECOMMIT
3501 }
3502 # else
3503 } else /* No need to zero since commit zeros. */
3504 # endif
3505
3506 #endif
3507
3508 /* Zero if necessary. */
3509 if (zero) {
3510 if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED)
3511 == 0) {
3512 memset((void *)((uintptr_t)chunk + ((run_ind
3513 + i) << pagesize_2pow)), 0, pagesize);
3514 /* CHUNK_MAP_ZEROED is cleared below. */
3515 }
3516 }
3517
3518 /* Update dirty page accounting. */
3519 if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) {
3520 chunk->ndirty--;
3521 arena->ndirty--;
3522 /* CHUNK_MAP_DIRTY is cleared below. */
3523 }
3524
3525 /* Initialize the chunk map. */
3526 if (large) {
3527 chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE
3528 | CHUNK_MAP_ALLOCATED;
3529 } else {
3530 chunk->map[run_ind + i].bits = (size_t)run
3531 | CHUNK_MAP_ALLOCATED;
3532 }
3533 }
3534
3535 /*
3536 * Set the run size only in the first element for large runs. This is
3537 * primarily a debugging aid, since the lack of size info for trailing
3538 * pages only matters if the application tries to operate on an
3539 * interior pointer.
3540 */
3541 if (large)
3542 chunk->map[run_ind].bits |= size;
3543
3544 if (chunk->ndirty == 0 && old_ndirty > 0)
3545 arena_chunk_tree_dirty_remove(&arena->chunks_dirty, chunk);
3546 }
3547
3548 static void
arena_chunk_init(arena_t * arena,arena_chunk_t * chunk)3549 arena_chunk_init(arena_t *arena, arena_chunk_t *chunk)
3550 {
3551 arena_run_t *run;
3552 size_t i;
3553
3554 #ifdef MALLOC_STATS
3555 arena->stats.mapped += chunksize;
3556 #endif
3557
3558 chunk->arena = arena;
3559
3560 /*
3561 * Claim that no pages are in use, since the header is merely overhead.
3562 */
3563 chunk->ndirty = 0;
3564
3565 /* Initialize the map to contain one maximal free untouched run. */
3566 run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages <<
3567 pagesize_2pow));
3568 for (i = 0; i < arena_chunk_header_npages; i++)
3569 chunk->map[i].bits = 0;
3570 chunk->map[i].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3571 for (i++; i < chunk_npages-1; i++) {
3572 chunk->map[i].bits = CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3573 }
3574 chunk->map[chunk_npages-1].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3575
3576 #ifdef MALLOC_DECOMMIT
3577 /*
3578 * Start out decommitted, in order to force a closer correspondence
3579 * between dirty pages and committed untouched pages.
3580 */
3581 pages_decommit(run, arena_maxclass);
3582 # ifdef MALLOC_STATS
3583 arena->stats.ndecommit++;
3584 arena->stats.decommitted += (chunk_npages - arena_chunk_header_npages);
3585 # endif
3586 #endif
3587 #ifdef MALLOC_STATS
3588 arena->stats.committed += arena_chunk_header_npages;
3589 #endif
3590
3591 /* Insert the run into the runs_avail tree. */
3592 arena_avail_tree_insert(&arena->runs_avail,
3593 &chunk->map[arena_chunk_header_npages]);
3594
3595 #ifdef MALLOC_DOUBLE_PURGE
3596 LinkedList_Init(&chunk->chunks_madvised_elem);
3597 #endif
3598 }
3599
3600 static void
arena_chunk_dealloc(arena_t * arena,arena_chunk_t * chunk)3601 arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
3602 {
3603
3604 if (arena->spare != NULL) {
3605 if (arena->spare->ndirty > 0) {
3606 arena_chunk_tree_dirty_remove(
3607 &chunk->arena->chunks_dirty, arena->spare);
3608 arena->ndirty -= arena->spare->ndirty;
3609 #ifdef MALLOC_STATS
3610 arena->stats.committed -= arena->spare->ndirty;
3611 #endif
3612 }
3613
3614 #ifdef MALLOC_DOUBLE_PURGE
3615 /* This is safe to do even if arena->spare is not in the list. */
3616 LinkedList_Remove(&arena->spare->chunks_madvised_elem);
3617 #endif
3618
3619 chunk_dealloc((void *)arena->spare, chunksize);
3620 #ifdef MALLOC_STATS
3621 arena->stats.mapped -= chunksize;
3622 arena->stats.committed -= arena_chunk_header_npages;
3623 #endif
3624 }
3625
3626 /*
3627 * Remove run from runs_avail, so that the arena does not use it.
3628 * Dirty page flushing only uses the chunks_dirty tree, so leaving this
3629 * chunk in the chunks_* trees is sufficient for that purpose.
3630 */
3631 arena_avail_tree_remove(&arena->runs_avail,
3632 &chunk->map[arena_chunk_header_npages]);
3633
3634 arena->spare = chunk;
3635 }
3636
3637 static arena_run_t *
arena_run_alloc(arena_t * arena,arena_bin_t * bin,size_t size,bool large,bool zero)3638 arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large,
3639 bool zero)
3640 {
3641 arena_run_t *run;
3642 arena_chunk_map_t *mapelm, key;
3643
3644 assert(size <= arena_maxclass);
3645 assert((size & pagesize_mask) == 0);
3646
3647 /* Search the arena's chunks for the lowest best fit. */
3648 key.bits = size | CHUNK_MAP_KEY;
3649 mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
3650 if (mapelm != NULL) {
3651 arena_chunk_t *chunk =
3652 (arena_chunk_t*)CHUNK_ADDR2BASE(mapelm);
3653 size_t pageind = ((uintptr_t)mapelm -
3654 (uintptr_t)chunk->map) /
3655 sizeof(arena_chunk_map_t);
3656
3657 run = (arena_run_t *)((uintptr_t)chunk + (pageind
3658 << pagesize_2pow));
3659 arena_run_split(arena, run, size, large, zero);
3660 return (run);
3661 }
3662
3663 if (arena->spare != NULL) {
3664 /* Use the spare. */
3665 arena_chunk_t *chunk = arena->spare;
3666 arena->spare = NULL;
3667 run = (arena_run_t *)((uintptr_t)chunk +
3668 (arena_chunk_header_npages << pagesize_2pow));
3669 /* Insert the run into the runs_avail tree. */
3670 arena_avail_tree_insert(&arena->runs_avail,
3671 &chunk->map[arena_chunk_header_npages]);
3672 arena_run_split(arena, run, size, large, zero);
3673 return (run);
3674 }
3675
3676 /*
3677 * No usable runs. Create a new chunk from which to allocate
3678 * the run.
3679 */
3680 {
3681 arena_chunk_t *chunk = (arena_chunk_t *)
3682 chunk_alloc(chunksize, chunksize, false, true);
3683 if (chunk == NULL)
3684 return (NULL);
3685
3686 arena_chunk_init(arena, chunk);
3687 run = (arena_run_t *)((uintptr_t)chunk +
3688 (arena_chunk_header_npages << pagesize_2pow));
3689 }
3690 /* Update page map. */
3691 arena_run_split(arena, run, size, large, zero);
3692 return (run);
3693 }
3694
3695 static void
arena_purge(arena_t * arena,bool all)3696 arena_purge(arena_t *arena, bool all)
3697 {
3698 arena_chunk_t *chunk;
3699 size_t i, npages;
3700 /* If all is set purge all dirty pages. */
3701 size_t dirty_max = all ? 1 : opt_dirty_max;
3702 #ifdef MALLOC_DEBUG
3703 size_t ndirty = 0;
3704 rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty,
3705 chunk) {
3706 ndirty += chunk->ndirty;
3707 } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk)
3708 assert(ndirty == arena->ndirty);
3709 #endif
3710 RELEASE_ASSERT(all || (arena->ndirty > opt_dirty_max));
3711
3712 #ifdef MALLOC_STATS
3713 arena->stats.npurge++;
3714 #endif
3715
3716 /*
3717 * Iterate downward through chunks until enough dirty memory has been
3718 * purged. Terminate as soon as possible in order to minimize the
3719 * number of system calls, even if a chunk has only been partially
3720 * purged.
3721 */
3722 while (arena->ndirty > (dirty_max >> 1)) {
3723 #ifdef MALLOC_DOUBLE_PURGE
3724 bool madvised = false;
3725 #endif
3726 chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty);
3727 RELEASE_ASSERT(chunk != NULL);
3728
3729 for (i = chunk_npages - 1; chunk->ndirty > 0; i--) {
3730 RELEASE_ASSERT(i >= arena_chunk_header_npages);
3731
3732 if (chunk->map[i].bits & CHUNK_MAP_DIRTY) {
3733 #ifdef MALLOC_DECOMMIT
3734 const size_t free_operation = CHUNK_MAP_DECOMMITTED;
3735 #else
3736 const size_t free_operation = CHUNK_MAP_MADVISED;
3737 #endif
3738 assert((chunk->map[i].bits &
3739 CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
3740 chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
3741 /* Find adjacent dirty run(s). */
3742 for (npages = 1;
3743 i > arena_chunk_header_npages &&
3744 (chunk->map[i - 1].bits & CHUNK_MAP_DIRTY);
3745 npages++) {
3746 i--;
3747 assert((chunk->map[i].bits &
3748 CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
3749 chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
3750 }
3751 chunk->ndirty -= npages;
3752 arena->ndirty -= npages;
3753
3754 #ifdef MALLOC_DECOMMIT
3755 pages_decommit((void *)((uintptr_t)
3756 chunk + (i << pagesize_2pow)),
3757 (npages << pagesize_2pow));
3758 # ifdef MALLOC_STATS
3759 arena->stats.ndecommit++;
3760 arena->stats.decommitted += npages;
3761 # endif
3762 #endif
3763 #ifdef MALLOC_STATS
3764 arena->stats.committed -= npages;
3765 #endif
3766
3767 #ifndef MALLOC_DECOMMIT
3768 madvise((void *)((uintptr_t)chunk + (i <<
3769 pagesize_2pow)), (npages << pagesize_2pow),
3770 MADV_FREE);
3771 # ifdef MALLOC_DOUBLE_PURGE
3772 madvised = true;
3773 # endif
3774 #endif
3775 #ifdef MALLOC_STATS
3776 arena->stats.nmadvise++;
3777 arena->stats.purged += npages;
3778 #endif
3779 if (arena->ndirty <= (dirty_max >> 1))
3780 break;
3781 }
3782 }
3783
3784 if (chunk->ndirty == 0) {
3785 arena_chunk_tree_dirty_remove(&arena->chunks_dirty,
3786 chunk);
3787 }
3788 #ifdef MALLOC_DOUBLE_PURGE
3789 if (madvised) {
3790 /* The chunk might already be in the list, but this
3791 * makes sure it's at the front. */
3792 LinkedList_Remove(&chunk->chunks_madvised_elem);
3793 LinkedList_InsertHead(&arena->chunks_madvised, &chunk->chunks_madvised_elem);
3794 }
3795 #endif
3796 }
3797 }
3798
3799 static void
arena_run_dalloc(arena_t * arena,arena_run_t * run,bool dirty)3800 arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
3801 {
3802 arena_chunk_t *chunk;
3803 size_t size, run_ind, run_pages;
3804
3805 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
3806 run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk)
3807 >> pagesize_2pow);
3808 RELEASE_ASSERT(run_ind >= arena_chunk_header_npages);
3809 RELEASE_ASSERT(run_ind < chunk_npages);
3810 if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0)
3811 size = chunk->map[run_ind].bits & ~pagesize_mask;
3812 else
3813 size = run->bin->run_size;
3814 run_pages = (size >> pagesize_2pow);
3815
3816 /* Mark pages as unallocated in the chunk map. */
3817 if (dirty) {
3818 size_t i;
3819
3820 for (i = 0; i < run_pages; i++) {
3821 RELEASE_ASSERT((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY)
3822 == 0);
3823 chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY;
3824 }
3825
3826 if (chunk->ndirty == 0) {
3827 arena_chunk_tree_dirty_insert(&arena->chunks_dirty,
3828 chunk);
3829 }
3830 chunk->ndirty += run_pages;
3831 arena->ndirty += run_pages;
3832 } else {
3833 size_t i;
3834
3835 for (i = 0; i < run_pages; i++) {
3836 chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE |
3837 CHUNK_MAP_ALLOCATED);
3838 }
3839 }
3840 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3841 pagesize_mask);
3842 chunk->map[run_ind+run_pages-1].bits = size |
3843 (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3844
3845 /* Try to coalesce forward. */
3846 if (run_ind + run_pages < chunk_npages &&
3847 (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) {
3848 size_t nrun_size = chunk->map[run_ind+run_pages].bits &
3849 ~pagesize_mask;
3850
3851 /*
3852 * Remove successor from runs_avail; the coalesced run is
3853 * inserted later.
3854 */
3855 arena_avail_tree_remove(&arena->runs_avail,
3856 &chunk->map[run_ind+run_pages]);
3857
3858 size += nrun_size;
3859 run_pages = size >> pagesize_2pow;
3860
3861 RELEASE_ASSERT((chunk->map[run_ind+run_pages-1].bits & ~pagesize_mask)
3862 == nrun_size);
3863 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3864 pagesize_mask);
3865 chunk->map[run_ind+run_pages-1].bits = size |
3866 (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3867 }
3868
3869 /* Try to coalesce backward. */
3870 if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits &
3871 CHUNK_MAP_ALLOCATED) == 0) {
3872 size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask;
3873
3874 run_ind -= prun_size >> pagesize_2pow;
3875
3876 /*
3877 * Remove predecessor from runs_avail; the coalesced run is
3878 * inserted later.
3879 */
3880 arena_avail_tree_remove(&arena->runs_avail,
3881 &chunk->map[run_ind]);
3882
3883 size += prun_size;
3884 run_pages = size >> pagesize_2pow;
3885
3886 RELEASE_ASSERT((chunk->map[run_ind].bits & ~pagesize_mask) ==
3887 prun_size);
3888 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3889 pagesize_mask);
3890 chunk->map[run_ind+run_pages-1].bits = size |
3891 (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3892 }
3893
3894 /* Insert into runs_avail, now that coalescing is complete. */
3895 arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]);
3896
3897 /* Deallocate chunk if it is now completely unused. */
3898 if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask |
3899 CHUNK_MAP_ALLOCATED)) == arena_maxclass)
3900 arena_chunk_dealloc(arena, chunk);
3901
3902 /* Enforce opt_dirty_max. */
3903 if (arena->ndirty > opt_dirty_max)
3904 arena_purge(arena, false);
3905 }
3906
3907 static void
arena_run_trim_head(arena_t * arena,arena_chunk_t * chunk,arena_run_t * run,size_t oldsize,size_t newsize)3908 arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
3909 size_t oldsize, size_t newsize)
3910 {
3911 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
3912 size_t head_npages = (oldsize - newsize) >> pagesize_2pow;
3913
3914 assert(oldsize > newsize);
3915
3916 /*
3917 * Update the chunk map so that arena_run_dalloc() can treat the
3918 * leading run as separately allocated.
3919 */
3920 chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE |
3921 CHUNK_MAP_ALLOCATED;
3922 chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE |
3923 CHUNK_MAP_ALLOCATED;
3924
3925 arena_run_dalloc(arena, run, false);
3926 }
3927
3928 static void
arena_run_trim_tail(arena_t * arena,arena_chunk_t * chunk,arena_run_t * run,size_t oldsize,size_t newsize,bool dirty)3929 arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
3930 size_t oldsize, size_t newsize, bool dirty)
3931 {
3932 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
3933 size_t npages = newsize >> pagesize_2pow;
3934
3935 assert(oldsize > newsize);
3936
3937 /*
3938 * Update the chunk map so that arena_run_dalloc() can treat the
3939 * trailing run as separately allocated.
3940 */
3941 chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE |
3942 CHUNK_MAP_ALLOCATED;
3943 chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE
3944 | CHUNK_MAP_ALLOCATED;
3945
3946 arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
3947 dirty);
3948 }
3949
3950 static arena_run_t *
arena_bin_nonfull_run_get(arena_t * arena,arena_bin_t * bin)3951 arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
3952 {
3953 arena_chunk_map_t *mapelm;
3954 arena_run_t *run;
3955 unsigned i, remainder;
3956
3957 /* Look for a usable run. */
3958 mapelm = arena_run_tree_first(&bin->runs);
3959 if (mapelm != NULL) {
3960 /* run is guaranteed to have available space. */
3961 arena_run_tree_remove(&bin->runs, mapelm);
3962 run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
3963 #ifdef MALLOC_STATS
3964 bin->stats.reruns++;
3965 #endif
3966 return (run);
3967 }
3968 /* No existing runs have any space available. */
3969
3970 /* Allocate a new run. */
3971 run = arena_run_alloc(arena, bin, bin->run_size, false, false);
3972 if (run == NULL)
3973 return (NULL);
3974 /*
3975 * Don't initialize if a race in arena_run_alloc() allowed an existing
3976 * run to become usable.
3977 */
3978 if (run == bin->runcur)
3979 return (run);
3980
3981 /* Initialize run internals. */
3982 run->bin = bin;
3983
3984 for (i = 0; i < bin->regs_mask_nelms - 1; i++)
3985 run->regs_mask[i] = UINT_MAX;
3986 remainder = bin->nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1);
3987 if (remainder == 0)
3988 run->regs_mask[i] = UINT_MAX;
3989 else {
3990 /* The last element has spare bits that need to be unset. */
3991 run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3))
3992 - remainder));
3993 }
3994
3995 run->regs_minelm = 0;
3996
3997 run->nfree = bin->nregs;
3998 #if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
3999 run->magic = ARENA_RUN_MAGIC;
4000 #endif
4001
4002 #ifdef MALLOC_STATS
4003 bin->stats.nruns++;
4004 bin->stats.curruns++;
4005 if (bin->stats.curruns > bin->stats.highruns)
4006 bin->stats.highruns = bin->stats.curruns;
4007 #endif
4008 return (run);
4009 }
4010
4011 /* bin->runcur must have space available before this function is called. */
4012 static inline void *
arena_bin_malloc_easy(arena_t * arena,arena_bin_t * bin,arena_run_t * run)4013 arena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run)
4014 {
4015 void *ret;
4016
4017 RELEASE_ASSERT(run->magic == ARENA_RUN_MAGIC);
4018 RELEASE_ASSERT(run->nfree > 0);
4019
4020 ret = arena_run_reg_alloc(run, bin);
4021 RELEASE_ASSERT(ret != NULL);
4022 run->nfree--;
4023
4024 return (ret);
4025 }
4026
4027 /* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */
4028 static void *
arena_bin_malloc_hard(arena_t * arena,arena_bin_t * bin)4029 arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
4030 {
4031
4032 bin->runcur = arena_bin_nonfull_run_get(arena, bin);
4033 if (bin->runcur == NULL)
4034 return (NULL);
4035 RELEASE_ASSERT(bin->runcur->magic == ARENA_RUN_MAGIC);
4036 RELEASE_ASSERT(bin->runcur->nfree > 0);
4037
4038 return (arena_bin_malloc_easy(arena, bin, bin->runcur));
4039 }
4040
4041 /*
4042 * Calculate bin->run_size such that it meets the following constraints:
4043 *
4044 * *) bin->run_size >= min_run_size
4045 * *) bin->run_size <= arena_maxclass
4046 * *) bin->run_size <= RUN_MAX_SMALL
4047 * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
4048 *
4049 * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are
4050 * also calculated here, since these settings are all interdependent.
4051 */
4052 static size_t
arena_bin_run_size_calc(arena_bin_t * bin,size_t min_run_size)4053 arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
4054 {
4055 size_t try_run_size, good_run_size;
4056 unsigned good_nregs, good_mask_nelms, good_reg0_offset;
4057 unsigned try_nregs, try_mask_nelms, try_reg0_offset;
4058
4059 assert(min_run_size >= pagesize);
4060 assert(min_run_size <= arena_maxclass);
4061
4062 /*
4063 * Calculate known-valid settings before entering the run_size
4064 * expansion loop, so that the first part of the loop always copies
4065 * valid settings.
4066 *
4067 * The do..while loop iteratively reduces the number of regions until
4068 * the run header and the regions no longer overlap. A closed formula
4069 * would be quite messy, since there is an interdependency between the
4070 * header's mask length and the number of regions.
4071 */
4072 try_run_size = min_run_size;
4073 try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size)
4074 + 1; /* Counter-act try_nregs-- in loop. */
4075 do {
4076 try_nregs--;
4077 try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
4078 ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0);
4079 try_reg0_offset = try_run_size - (try_nregs * bin->reg_size);
4080 } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))
4081 > try_reg0_offset);
4082
4083 /* run_size expansion loop. */
4084 do {
4085 /*
4086 * Copy valid settings before trying more aggressive settings.
4087 */
4088 good_run_size = try_run_size;
4089 good_nregs = try_nregs;
4090 good_mask_nelms = try_mask_nelms;
4091 good_reg0_offset = try_reg0_offset;
4092
4093 /* Try more aggressive settings. */
4094 try_run_size += pagesize;
4095 try_nregs = ((try_run_size - sizeof(arena_run_t)) /
4096 bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */
4097 do {
4098 try_nregs--;
4099 try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
4100 ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ?
4101 1 : 0);
4102 try_reg0_offset = try_run_size - (try_nregs *
4103 bin->reg_size);
4104 } while (sizeof(arena_run_t) + (sizeof(unsigned) *
4105 (try_mask_nelms - 1)) > try_reg0_offset);
4106 } while (try_run_size <= arena_maxclass
4107 && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
4108 && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
4109
4110 assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1))
4111 <= good_reg0_offset);
4112 assert((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs);
4113
4114 /* Copy final settings. */
4115 bin->run_size = good_run_size;
4116 bin->nregs = good_nregs;
4117 bin->regs_mask_nelms = good_mask_nelms;
4118 bin->reg0_offset = good_reg0_offset;
4119
4120 return (good_run_size);
4121 }
4122
4123 #ifdef MALLOC_BALANCE
4124 static inline void
arena_lock_balance(arena_t * arena)4125 arena_lock_balance(arena_t *arena)
4126 {
4127 unsigned contention;
4128
4129 contention = malloc_spin_lock(&arena->lock);
4130 if (narenas > 1) {
4131 /*
4132 * Calculate the exponentially averaged contention for this
4133 * arena. Due to integer math always rounding down, this value
4134 * decays somewhat faster then normal.
4135 */
4136 arena->contention = (((uint64_t)arena->contention
4137 * (uint64_t)((1U << BALANCE_ALPHA_INV_2POW)-1))
4138 + (uint64_t)contention) >> BALANCE_ALPHA_INV_2POW;
4139 if (arena->contention >= opt_balance_threshold)
4140 arena_lock_balance_hard(arena);
4141 }
4142 }
4143
4144 static void
arena_lock_balance_hard(arena_t * arena)4145 arena_lock_balance_hard(arena_t *arena)
4146 {
4147 uint32_t ind;
4148
4149 arena->contention = 0;
4150 #ifdef MALLOC_STATS
4151 arena->stats.nbalance++;
4152 #endif
4153 ind = PRN(balance, narenas_2pow);
4154 if (arenas[ind] != NULL) {
4155 #ifdef MOZ_MEMORY_WINDOWS
4156 TlsSetValue(tlsIndex, arenas[ind]);
4157 #else
4158 arenas_map = arenas[ind];
4159 #endif
4160 } else {
4161 malloc_spin_lock(&arenas_lock);
4162 if (arenas[ind] != NULL) {
4163 #ifdef MOZ_MEMORY_WINDOWS
4164 TlsSetValue(tlsIndex, arenas[ind]);
4165 #else
4166 arenas_map = arenas[ind];
4167 #endif
4168 } else {
4169 #ifdef MOZ_MEMORY_WINDOWS
4170 TlsSetValue(tlsIndex, arenas_extend(ind));
4171 #else
4172 arenas_map = arenas_extend(ind);
4173 #endif
4174 }
4175 malloc_spin_unlock(&arenas_lock);
4176 }
4177 }
4178 #endif
4179
4180 static inline void *
arena_malloc_small(arena_t * arena,size_t size,bool zero)4181 arena_malloc_small(arena_t *arena, size_t size, bool zero)
4182 {
4183 void *ret;
4184 arena_bin_t *bin;
4185 arena_run_t *run;
4186
4187 if (size < small_min) {
4188 /* Tiny. */
4189 size = pow2_ceil(size);
4190 bin = &arena->bins[ffs((int)(size >> (TINY_MIN_2POW +
4191 1)))];
4192 #if (!defined(NDEBUG) || defined(MALLOC_STATS))
4193 /*
4194 * Bin calculation is always correct, but we may need
4195 * to fix size for the purposes of assertions and/or
4196 * stats accuracy.
4197 */
4198 if (size < (1U << TINY_MIN_2POW))
4199 size = (1U << TINY_MIN_2POW);
4200 #endif
4201 } else if (size <= small_max) {
4202 /* Quantum-spaced. */
4203 size = QUANTUM_CEILING(size);
4204 bin = &arena->bins[ntbins + (size >> opt_quantum_2pow)
4205 - 1];
4206 } else {
4207 /* Sub-page. */
4208 size = pow2_ceil(size);
4209 bin = &arena->bins[ntbins + nqbins
4210 + (ffs((int)(size >> opt_small_max_2pow)) - 2)];
4211 }
4212 RELEASE_ASSERT(size == bin->reg_size);
4213
4214 #ifdef MALLOC_BALANCE
4215 arena_lock_balance(arena);
4216 #else
4217 malloc_spin_lock(&arena->lock);
4218 #endif
4219 if ((run = bin->runcur) != NULL && run->nfree > 0)
4220 ret = arena_bin_malloc_easy(arena, bin, run);
4221 else
4222 ret = arena_bin_malloc_hard(arena, bin);
4223
4224 if (ret == NULL) {
4225 malloc_spin_unlock(&arena->lock);
4226 return (NULL);
4227 }
4228
4229 #ifdef MALLOC_STATS
4230 bin->stats.nrequests++;
4231 arena->stats.nmalloc_small++;
4232 arena->stats.allocated_small += size;
4233 #endif
4234 malloc_spin_unlock(&arena->lock);
4235
4236 if (zero == false) {
4237 #ifdef MALLOC_FILL
4238 if (opt_junk)
4239 memset(ret, 0xe4, size);
4240 else if (opt_zero)
4241 memset(ret, 0, size);
4242 #endif
4243 } else
4244 memset(ret, 0, size);
4245
4246 return (ret);
4247 }
4248
4249 static void *
arena_malloc_large(arena_t * arena,size_t size,bool zero)4250 arena_malloc_large(arena_t *arena, size_t size, bool zero)
4251 {
4252 void *ret;
4253
4254 /* Large allocation. */
4255 size = PAGE_CEILING(size);
4256 #ifdef MALLOC_BALANCE
4257 arena_lock_balance(arena);
4258 #else
4259 malloc_spin_lock(&arena->lock);
4260 #endif
4261 ret = (void *)arena_run_alloc(arena, NULL, size, true, zero);
4262 if (ret == NULL) {
4263 malloc_spin_unlock(&arena->lock);
4264 return (NULL);
4265 }
4266 #ifdef MALLOC_STATS
4267 arena->stats.nmalloc_large++;
4268 arena->stats.allocated_large += size;
4269 #endif
4270 malloc_spin_unlock(&arena->lock);
4271
4272 if (zero == false) {
4273 #ifdef MALLOC_FILL
4274 if (opt_junk)
4275 memset(ret, 0xe4, size);
4276 else if (opt_zero)
4277 memset(ret, 0, size);
4278 #endif
4279 }
4280
4281 return (ret);
4282 }
4283
4284 static inline void *
arena_malloc(arena_t * arena,size_t size,bool zero)4285 arena_malloc(arena_t *arena, size_t size, bool zero)
4286 {
4287
4288 assert(arena != NULL);
4289 RELEASE_ASSERT(arena->magic == ARENA_MAGIC);
4290 assert(size != 0);
4291 assert(QUANTUM_CEILING(size) <= arena_maxclass);
4292
4293 if (size <= bin_maxclass) {
4294 return (arena_malloc_small(arena, size, zero));
4295 } else
4296 return (arena_malloc_large(arena, size, zero));
4297 }
4298
4299 static inline void *
imalloc(size_t size)4300 imalloc(size_t size)
4301 {
4302
4303 assert(size != 0);
4304
4305 if (size <= arena_maxclass)
4306 return (arena_malloc(choose_arena(), size, false));
4307 else
4308 return (huge_malloc(size, false));
4309 }
4310
4311 static inline void *
icalloc(size_t size)4312 icalloc(size_t size)
4313 {
4314
4315 if (size <= arena_maxclass)
4316 return (arena_malloc(choose_arena(), size, true));
4317 else
4318 return (huge_malloc(size, true));
4319 }
4320
4321 /* Only handles large allocations that require more than page alignment. */
4322 static void *
arena_palloc(arena_t * arena,size_t alignment,size_t size,size_t alloc_size)4323 arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
4324 {
4325 void *ret;
4326 size_t offset;
4327 arena_chunk_t *chunk;
4328
4329 assert((size & pagesize_mask) == 0);
4330 assert((alignment & pagesize_mask) == 0);
4331
4332 #ifdef MALLOC_BALANCE
4333 arena_lock_balance(arena);
4334 #else
4335 malloc_spin_lock(&arena->lock);
4336 #endif
4337 ret = (void *)arena_run_alloc(arena, NULL, alloc_size, true, false);
4338 if (ret == NULL) {
4339 malloc_spin_unlock(&arena->lock);
4340 return (NULL);
4341 }
4342
4343 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
4344
4345 offset = (uintptr_t)ret & (alignment - 1);
4346 assert((offset & pagesize_mask) == 0);
4347 assert(offset < alloc_size);
4348 if (offset == 0)
4349 arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, alloc_size, size, false);
4350 else {
4351 size_t leadsize, trailsize;
4352
4353 leadsize = alignment - offset;
4354 if (leadsize > 0) {
4355 arena_run_trim_head(arena, chunk, (arena_run_t*)ret, alloc_size,
4356 alloc_size - leadsize);
4357 ret = (void *)((uintptr_t)ret + leadsize);
4358 }
4359
4360 trailsize = alloc_size - leadsize - size;
4361 if (trailsize != 0) {
4362 /* Trim trailing space. */
4363 assert(trailsize < alloc_size);
4364 arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, size + trailsize,
4365 size, false);
4366 }
4367 }
4368
4369 #ifdef MALLOC_STATS
4370 arena->stats.nmalloc_large++;
4371 arena->stats.allocated_large += size;
4372 #endif
4373 malloc_spin_unlock(&arena->lock);
4374
4375 #ifdef MALLOC_FILL
4376 if (opt_junk)
4377 memset(ret, 0xe4, size);
4378 else if (opt_zero)
4379 memset(ret, 0, size);
4380 #endif
4381 return (ret);
4382 }
4383
4384 static inline void *
ipalloc(size_t alignment,size_t size)4385 ipalloc(size_t alignment, size_t size)
4386 {
4387 void *ret;
4388 size_t ceil_size;
4389
4390 /*
4391 * Round size up to the nearest multiple of alignment.
4392 *
4393 * This done, we can take advantage of the fact that for each small
4394 * size class, every object is aligned at the smallest power of two
4395 * that is non-zero in the base two representation of the size. For
4396 * example:
4397 *
4398 * Size | Base 2 | Minimum alignment
4399 * -----+----------+------------------
4400 * 96 | 1100000 | 32
4401 * 144 | 10100000 | 32
4402 * 192 | 11000000 | 64
4403 *
4404 * Depending on runtime settings, it is possible that arena_malloc()
4405 * will further round up to a power of two, but that never causes
4406 * correctness issues.
4407 */
4408 ceil_size = (size + (alignment - 1)) & (-alignment);
4409 /*
4410 * (ceil_size < size) protects against the combination of maximal
4411 * alignment and size greater than maximal alignment.
4412 */
4413 if (ceil_size < size) {
4414 /* size_t overflow. */
4415 return (NULL);
4416 }
4417
4418 if (ceil_size <= pagesize || (alignment <= pagesize
4419 && ceil_size <= arena_maxclass))
4420 ret = arena_malloc(choose_arena(), ceil_size, false);
4421 else {
4422 size_t run_size;
4423
4424 /*
4425 * We can't achieve sub-page alignment, so round up alignment
4426 * permanently; it makes later calculations simpler.
4427 */
4428 alignment = PAGE_CEILING(alignment);
4429 ceil_size = PAGE_CEILING(size);
4430 /*
4431 * (ceil_size < size) protects against very large sizes within
4432 * pagesize of SIZE_T_MAX.
4433 *
4434 * (ceil_size + alignment < ceil_size) protects against the
4435 * combination of maximal alignment and ceil_size large enough
4436 * to cause overflow. This is similar to the first overflow
4437 * check above, but it needs to be repeated due to the new
4438 * ceil_size value, which may now be *equal* to maximal
4439 * alignment, whereas before we only detected overflow if the
4440 * original size was *greater* than maximal alignment.
4441 */
4442 if (ceil_size < size || ceil_size + alignment < ceil_size) {
4443 /* size_t overflow. */
4444 return (NULL);
4445 }
4446
4447 /*
4448 * Calculate the size of the over-size run that arena_palloc()
4449 * would need to allocate in order to guarantee the alignment.
4450 */
4451 if (ceil_size >= alignment)
4452 run_size = ceil_size + alignment - pagesize;
4453 else {
4454 /*
4455 * It is possible that (alignment << 1) will cause
4456 * overflow, but it doesn't matter because we also
4457 * subtract pagesize, which in the case of overflow
4458 * leaves us with a very large run_size. That causes
4459 * the first conditional below to fail, which means
4460 * that the bogus run_size value never gets used for
4461 * anything important.
4462 */
4463 run_size = (alignment << 1) - pagesize;
4464 }
4465
4466 if (run_size <= arena_maxclass) {
4467 ret = arena_palloc(choose_arena(), alignment, ceil_size,
4468 run_size);
4469 } else if (alignment <= chunksize)
4470 ret = huge_malloc(ceil_size, false);
4471 else
4472 ret = huge_palloc(ceil_size, alignment, false);
4473 }
4474
4475 assert(((uintptr_t)ret & (alignment - 1)) == 0);
4476 return (ret);
4477 }
4478
4479 /* Return the size of the allocation pointed to by ptr. */
4480 static size_t
arena_salloc(const void * ptr)4481 arena_salloc(const void *ptr)
4482 {
4483 size_t ret;
4484 arena_chunk_t *chunk;
4485 size_t pageind, mapbits;
4486
4487 assert(ptr != NULL);
4488 assert(CHUNK_ADDR2BASE(ptr) != ptr);
4489
4490 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4491 pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
4492 mapbits = chunk->map[pageind].bits;
4493 RELEASE_ASSERT((mapbits & CHUNK_MAP_ALLOCATED) != 0);
4494 if ((mapbits & CHUNK_MAP_LARGE) == 0) {
4495 arena_run_t *run = (arena_run_t *)(mapbits & ~pagesize_mask);
4496 RELEASE_ASSERT(run->magic == ARENA_RUN_MAGIC);
4497 ret = run->bin->reg_size;
4498 } else {
4499 ret = mapbits & ~pagesize_mask;
4500 RELEASE_ASSERT(ret != 0);
4501 }
4502
4503 return (ret);
4504 }
4505
4506 #if (defined(MALLOC_VALIDATE) || defined(MOZ_MEMORY_DARWIN))
4507 /*
4508 * Validate ptr before assuming that it points to an allocation. Currently,
4509 * the following validation is performed:
4510 *
4511 * + Check that ptr is not NULL.
4512 *
4513 * + Check that ptr lies within a mapped chunk.
4514 */
4515 static inline size_t
isalloc_validate(const void * ptr)4516 isalloc_validate(const void *ptr)
4517 {
4518 arena_chunk_t *chunk;
4519
4520 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4521 if (chunk == NULL)
4522 return (0);
4523
4524 if (malloc_rtree_get(chunk_rtree, (uintptr_t)chunk) == NULL)
4525 return (0);
4526
4527 if (chunk != ptr) {
4528 RELEASE_ASSERT(chunk->arena->magic == ARENA_MAGIC);
4529 return (arena_salloc(ptr));
4530 } else {
4531 size_t ret;
4532 extent_node_t *node;
4533 extent_node_t key;
4534
4535 /* Chunk. */
4536 key.addr = (void *)chunk;
4537 malloc_mutex_lock(&huge_mtx);
4538 node = extent_tree_ad_search(&huge, &key);
4539 if (node != NULL)
4540 ret = node->size;
4541 else
4542 ret = 0;
4543 malloc_mutex_unlock(&huge_mtx);
4544 return (ret);
4545 }
4546 }
4547 #endif
4548
4549 static inline size_t
isalloc(const void * ptr)4550 isalloc(const void *ptr)
4551 {
4552 size_t ret;
4553 arena_chunk_t *chunk;
4554
4555 assert(ptr != NULL);
4556
4557 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4558 if (chunk != ptr) {
4559 /* Region. */
4560 assert(chunk->arena->magic == ARENA_MAGIC);
4561
4562 ret = arena_salloc(ptr);
4563 } else {
4564 extent_node_t *node, key;
4565
4566 /* Chunk (huge allocation). */
4567
4568 malloc_mutex_lock(&huge_mtx);
4569
4570 /* Extract from tree of huge allocations. */
4571 key.addr = __DECONST(void *, ptr);
4572 node = extent_tree_ad_search(&huge, &key);
4573 RELEASE_ASSERT(node != NULL);
4574
4575 ret = node->size;
4576
4577 malloc_mutex_unlock(&huge_mtx);
4578 }
4579
4580 return (ret);
4581 }
4582
4583 static inline void
arena_dalloc_small(arena_t * arena,arena_chunk_t * chunk,void * ptr,arena_chunk_map_t * mapelm)4584 arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4585 arena_chunk_map_t *mapelm)
4586 {
4587 arena_run_t *run;
4588 arena_bin_t *bin;
4589 size_t size;
4590
4591 run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
4592 RELEASE_ASSERT(run->magic == ARENA_RUN_MAGIC);
4593 bin = run->bin;
4594 size = bin->reg_size;
4595
4596 #ifdef MALLOC_FILL
4597 if (opt_poison)
4598 memset(ptr, 0xe5, size);
4599 #endif
4600
4601 arena_run_reg_dalloc(run, bin, ptr, size);
4602 run->nfree++;
4603
4604 if (run->nfree == bin->nregs) {
4605 /* Deallocate run. */
4606 if (run == bin->runcur)
4607 bin->runcur = NULL;
4608 else if (bin->nregs != 1) {
4609 size_t run_pageind = (((uintptr_t)run -
4610 (uintptr_t)chunk)) >> pagesize_2pow;
4611 arena_chunk_map_t *run_mapelm =
4612 &chunk->map[run_pageind];
4613 /*
4614 * This block's conditional is necessary because if the
4615 * run only contains one region, then it never gets
4616 * inserted into the non-full runs tree.
4617 */
4618 RELEASE_ASSERT(arena_run_tree_search(&bin->runs, run_mapelm) ==
4619 run_mapelm);
4620 arena_run_tree_remove(&bin->runs, run_mapelm);
4621 }
4622 #if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
4623 run->magic = 0;
4624 #endif
4625 arena_run_dalloc(arena, run, true);
4626 #ifdef MALLOC_STATS
4627 bin->stats.curruns--;
4628 #endif
4629 } else if (run->nfree == 1 && run != bin->runcur) {
4630 /*
4631 * Make sure that bin->runcur always refers to the lowest
4632 * non-full run, if one exists.
4633 */
4634 if (bin->runcur == NULL)
4635 bin->runcur = run;
4636 else if ((uintptr_t)run < (uintptr_t)bin->runcur) {
4637 /* Switch runcur. */
4638 if (bin->runcur->nfree > 0) {
4639 arena_chunk_t *runcur_chunk =
4640 (arena_chunk_t*)CHUNK_ADDR2BASE(bin->runcur);
4641 size_t runcur_pageind =
4642 (((uintptr_t)bin->runcur -
4643 (uintptr_t)runcur_chunk)) >> pagesize_2pow;
4644 arena_chunk_map_t *runcur_mapelm =
4645 &runcur_chunk->map[runcur_pageind];
4646
4647 /* Insert runcur. */
4648 RELEASE_ASSERT(arena_run_tree_search(&bin->runs,
4649 runcur_mapelm) == NULL);
4650 arena_run_tree_insert(&bin->runs,
4651 runcur_mapelm);
4652 }
4653 bin->runcur = run;
4654 } else {
4655 size_t run_pageind = (((uintptr_t)run -
4656 (uintptr_t)chunk)) >> pagesize_2pow;
4657 arena_chunk_map_t *run_mapelm =
4658 &chunk->map[run_pageind];
4659
4660 RELEASE_ASSERT(arena_run_tree_search(&bin->runs, run_mapelm) ==
4661 NULL);
4662 arena_run_tree_insert(&bin->runs, run_mapelm);
4663 }
4664 }
4665 #ifdef MALLOC_STATS
4666 arena->stats.allocated_small -= size;
4667 arena->stats.ndalloc_small++;
4668 #endif
4669 }
4670
4671 static void
arena_dalloc_large(arena_t * arena,arena_chunk_t * chunk,void * ptr)4672 arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
4673 {
4674 /* Large allocation. */
4675 malloc_spin_lock(&arena->lock);
4676
4677 #ifdef MALLOC_FILL
4678 #ifndef MALLOC_STATS
4679 if (opt_poison)
4680 #endif
4681 #endif
4682 {
4683 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
4684 pagesize_2pow;
4685 size_t size = chunk->map[pageind].bits & ~pagesize_mask;
4686
4687 #ifdef MALLOC_FILL
4688 #ifdef MALLOC_STATS
4689 if (opt_poison)
4690 #endif
4691 memset(ptr, 0xe5, size);
4692 #endif
4693 #ifdef MALLOC_STATS
4694 arena->stats.allocated_large -= size;
4695 #endif
4696 }
4697 #ifdef MALLOC_STATS
4698 arena->stats.ndalloc_large++;
4699 #endif
4700
4701 arena_run_dalloc(arena, (arena_run_t *)ptr, true);
4702 malloc_spin_unlock(&arena->lock);
4703 }
4704
4705 static inline void
arena_dalloc(void * ptr,size_t offset)4706 arena_dalloc(void *ptr, size_t offset)
4707 {
4708 arena_chunk_t *chunk;
4709 arena_t *arena;
4710 size_t pageind;
4711 arena_chunk_map_t *mapelm;
4712
4713 assert(ptr != NULL);
4714 assert(offset != 0);
4715 assert(CHUNK_ADDR2OFFSET(ptr) == offset);
4716
4717 chunk = (arena_chunk_t *) ((uintptr_t)ptr - offset);
4718 arena = chunk->arena;
4719 assert(arena != NULL);
4720 RELEASE_ASSERT(arena->magic == ARENA_MAGIC);
4721
4722 pageind = offset >> pagesize_2pow;
4723 mapelm = &chunk->map[pageind];
4724 RELEASE_ASSERT((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
4725 if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
4726 /* Small allocation. */
4727 malloc_spin_lock(&arena->lock);
4728 arena_dalloc_small(arena, chunk, ptr, mapelm);
4729 malloc_spin_unlock(&arena->lock);
4730 } else
4731 arena_dalloc_large(arena, chunk, ptr);
4732 }
4733
4734 static inline void
idalloc(void * ptr)4735 idalloc(void *ptr)
4736 {
4737 size_t offset;
4738
4739 assert(ptr != NULL);
4740
4741 offset = CHUNK_ADDR2OFFSET(ptr);
4742 if (offset != 0)
4743 arena_dalloc(ptr, offset);
4744 else
4745 huge_dalloc(ptr);
4746 }
4747
4748 static void
arena_ralloc_large_shrink(arena_t * arena,arena_chunk_t * chunk,void * ptr,size_t size,size_t oldsize)4749 arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4750 size_t size, size_t oldsize)
4751 {
4752
4753 assert(size < oldsize);
4754
4755 /*
4756 * Shrink the run, and make trailing pages available for other
4757 * allocations.
4758 */
4759 #ifdef MALLOC_BALANCE
4760 arena_lock_balance(arena);
4761 #else
4762 malloc_spin_lock(&arena->lock);
4763 #endif
4764 arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size,
4765 true);
4766 #ifdef MALLOC_STATS
4767 arena->stats.allocated_large -= oldsize - size;
4768 #endif
4769 malloc_spin_unlock(&arena->lock);
4770 }
4771
4772 static bool
arena_ralloc_large_grow(arena_t * arena,arena_chunk_t * chunk,void * ptr,size_t size,size_t oldsize)4773 arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4774 size_t size, size_t oldsize)
4775 {
4776 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow;
4777 size_t npages = oldsize >> pagesize_2pow;
4778
4779 RELEASE_ASSERT(oldsize == (chunk->map[pageind].bits & ~pagesize_mask));
4780
4781 /* Try to extend the run. */
4782 assert(size > oldsize);
4783 #ifdef MALLOC_BALANCE
4784 arena_lock_balance(arena);
4785 #else
4786 malloc_spin_lock(&arena->lock);
4787 #endif
4788 if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits
4789 & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits &
4790 ~pagesize_mask) >= size - oldsize) {
4791 /*
4792 * The next run is available and sufficiently large. Split the
4793 * following run, then merge the first part with the existing
4794 * allocation.
4795 */
4796 arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
4797 ((pageind+npages) << pagesize_2pow)), size - oldsize, true,
4798 false);
4799
4800 chunk->map[pageind].bits = size | CHUNK_MAP_LARGE |
4801 CHUNK_MAP_ALLOCATED;
4802 chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE |
4803 CHUNK_MAP_ALLOCATED;
4804
4805 #ifdef MALLOC_STATS
4806 arena->stats.allocated_large += size - oldsize;
4807 #endif
4808 malloc_spin_unlock(&arena->lock);
4809 return (false);
4810 }
4811 malloc_spin_unlock(&arena->lock);
4812
4813 return (true);
4814 }
4815
4816 /*
4817 * Try to resize a large allocation, in order to avoid copying. This will
4818 * always fail if growing an object, and the following run is already in use.
4819 */
4820 static bool
arena_ralloc_large(void * ptr,size_t size,size_t oldsize)4821 arena_ralloc_large(void *ptr, size_t size, size_t oldsize)
4822 {
4823 size_t psize;
4824
4825 psize = PAGE_CEILING(size);
4826 if (psize == oldsize) {
4827 /* Same size class. */
4828 #ifdef MALLOC_FILL
4829 if (opt_poison && size < oldsize) {
4830 memset((void *)((uintptr_t)ptr + size), 0xe5, oldsize -
4831 size);
4832 }
4833 #endif
4834 return (false);
4835 } else {
4836 arena_chunk_t *chunk;
4837 arena_t *arena;
4838
4839 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4840 arena = chunk->arena;
4841 RELEASE_ASSERT(arena->magic == ARENA_MAGIC);
4842
4843 if (psize < oldsize) {
4844 #ifdef MALLOC_FILL
4845 /* Fill before shrinking in order avoid a race. */
4846 if (opt_poison) {
4847 memset((void *)((uintptr_t)ptr + size), 0xe5,
4848 oldsize - size);
4849 }
4850 #endif
4851 arena_ralloc_large_shrink(arena, chunk, ptr, psize,
4852 oldsize);
4853 return (false);
4854 } else {
4855 bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
4856 psize, oldsize);
4857 #ifdef MALLOC_FILL
4858 if (ret == false && opt_zero) {
4859 memset((void *)((uintptr_t)ptr + oldsize), 0,
4860 size - oldsize);
4861 }
4862 #endif
4863 return (ret);
4864 }
4865 }
4866 }
4867
4868 static void *
arena_ralloc(void * ptr,size_t size,size_t oldsize)4869 arena_ralloc(void *ptr, size_t size, size_t oldsize)
4870 {
4871 void *ret;
4872 size_t copysize;
4873
4874 /* Try to avoid moving the allocation. */
4875 if (size < small_min) {
4876 if (oldsize < small_min &&
4877 ffs((int)(pow2_ceil(size) >> (TINY_MIN_2POW + 1)))
4878 == ffs((int)(pow2_ceil(oldsize) >> (TINY_MIN_2POW + 1))))
4879 goto IN_PLACE; /* Same size class. */
4880 } else if (size <= small_max) {
4881 if (oldsize >= small_min && oldsize <= small_max &&
4882 (QUANTUM_CEILING(size) >> opt_quantum_2pow)
4883 == (QUANTUM_CEILING(oldsize) >> opt_quantum_2pow))
4884 goto IN_PLACE; /* Same size class. */
4885 } else if (size <= bin_maxclass) {
4886 if (oldsize > small_max && oldsize <= bin_maxclass &&
4887 pow2_ceil(size) == pow2_ceil(oldsize))
4888 goto IN_PLACE; /* Same size class. */
4889 } else if (oldsize > bin_maxclass && oldsize <= arena_maxclass) {
4890 assert(size > bin_maxclass);
4891 if (arena_ralloc_large(ptr, size, oldsize) == false)
4892 return (ptr);
4893 }
4894
4895 /*
4896 * If we get here, then size and oldsize are different enough that we
4897 * need to move the object. In that case, fall back to allocating new
4898 * space and copying.
4899 */
4900 ret = arena_malloc(choose_arena(), size, false);
4901 if (ret == NULL)
4902 return (NULL);
4903
4904 /* Junk/zero-filling were already done by arena_malloc(). */
4905 copysize = (size < oldsize) ? size : oldsize;
4906 #ifdef VM_COPY_MIN
4907 if (copysize >= VM_COPY_MIN)
4908 pages_copy(ret, ptr, copysize);
4909 else
4910 #endif
4911 memcpy(ret, ptr, copysize);
4912 idalloc(ptr);
4913 return (ret);
4914 IN_PLACE:
4915 #ifdef MALLOC_FILL
4916 if (opt_poison && size < oldsize)
4917 memset((void *)((uintptr_t)ptr + size), 0xe5, oldsize - size);
4918 else if (opt_zero && size > oldsize)
4919 memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
4920 #endif
4921 return (ptr);
4922 }
4923
4924 static inline void *
iralloc(void * ptr,size_t size)4925 iralloc(void *ptr, size_t size)
4926 {
4927 size_t oldsize;
4928
4929 assert(ptr != NULL);
4930 assert(size != 0);
4931
4932 oldsize = isalloc(ptr);
4933
4934 if (size <= arena_maxclass)
4935 return (arena_ralloc(ptr, size, oldsize));
4936 else
4937 return (huge_ralloc(ptr, size, oldsize));
4938 }
4939
4940 static bool
arena_new(arena_t * arena)4941 arena_new(arena_t *arena)
4942 {
4943 unsigned i;
4944 arena_bin_t *bin;
4945 size_t pow2_size, prev_run_size;
4946
4947 if (malloc_spin_init(&arena->lock))
4948 return (true);
4949
4950 #ifdef MALLOC_STATS
4951 memset(&arena->stats, 0, sizeof(arena_stats_t));
4952 #endif
4953
4954 /* Initialize chunks. */
4955 arena_chunk_tree_dirty_new(&arena->chunks_dirty);
4956 #ifdef MALLOC_DOUBLE_PURGE
4957 LinkedList_Init(&arena->chunks_madvised);
4958 #endif
4959 arena->spare = NULL;
4960
4961 arena->ndirty = 0;
4962
4963 arena_avail_tree_new(&arena->runs_avail);
4964
4965 #ifdef MALLOC_BALANCE
4966 arena->contention = 0;
4967 #endif
4968
4969 /* Initialize bins. */
4970 prev_run_size = pagesize;
4971
4972 /* (2^n)-spaced tiny bins. */
4973 for (i = 0; i < ntbins; i++) {
4974 bin = &arena->bins[i];
4975 bin->runcur = NULL;
4976 arena_run_tree_new(&bin->runs);
4977
4978 bin->reg_size = (1ULL << (TINY_MIN_2POW + i));
4979
4980 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
4981
4982 #ifdef MALLOC_STATS
4983 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
4984 #endif
4985 }
4986
4987 /* Quantum-spaced bins. */
4988 for (; i < ntbins + nqbins; i++) {
4989 bin = &arena->bins[i];
4990 bin->runcur = NULL;
4991 arena_run_tree_new(&bin->runs);
4992
4993 bin->reg_size = quantum * (i - ntbins + 1);
4994
4995 pow2_size = pow2_ceil(quantum * (i - ntbins + 1));
4996 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
4997
4998 #ifdef MALLOC_STATS
4999 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
5000 #endif
5001 }
5002
5003 /* (2^n)-spaced sub-page bins. */
5004 for (; i < ntbins + nqbins + nsbins; i++) {
5005 bin = &arena->bins[i];
5006 bin->runcur = NULL;
5007 arena_run_tree_new(&bin->runs);
5008
5009 bin->reg_size = (small_max << (i - (ntbins + nqbins) + 1));
5010
5011 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
5012
5013 #ifdef MALLOC_STATS
5014 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
5015 #endif
5016 }
5017
5018 #if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
5019 arena->magic = ARENA_MAGIC;
5020 #endif
5021
5022 return (false);
5023 }
5024
5025 /* Create a new arena and insert it into the arenas array at index ind. */
5026 static arena_t *
arenas_extend(unsigned ind)5027 arenas_extend(unsigned ind)
5028 {
5029 arena_t *ret;
5030
5031 /* Allocate enough space for trailing bins. */
5032 ret = (arena_t *)base_alloc(sizeof(arena_t)
5033 + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
5034 if (ret != NULL && arena_new(ret) == false) {
5035 arenas[ind] = ret;
5036 return (ret);
5037 }
5038 /* Only reached if there is an OOM error. */
5039
5040 /*
5041 * OOM here is quite inconvenient to propagate, since dealing with it
5042 * would require a check for failure in the fast path. Instead, punt
5043 * by using arenas[0]. In practice, this is an extremely unlikely
5044 * failure.
5045 */
5046 _malloc_message(_getprogname(),
5047 ": (malloc) Error initializing arena\n", "", "");
5048 if (opt_abort)
5049 abort();
5050
5051 return (arenas[0]);
5052 }
5053
5054 /*
5055 * End arena.
5056 */
5057 /******************************************************************************/
5058 /*
5059 * Begin general internal functions.
5060 */
5061
5062 static void *
huge_malloc(size_t size,bool zero)5063 huge_malloc(size_t size, bool zero)
5064 {
5065 return huge_palloc(size, chunksize, zero);
5066 }
5067
5068 static void *
huge_palloc(size_t size,size_t alignment,bool zero)5069 huge_palloc(size_t size, size_t alignment, bool zero)
5070 {
5071 void *ret;
5072 size_t csize;
5073 size_t psize;
5074 extent_node_t *node;
5075
5076 /* Allocate one or more contiguous chunks for this request. */
5077
5078 csize = CHUNK_CEILING(size);
5079 if (csize == 0) {
5080 /* size is large enough to cause size_t wrap-around. */
5081 return (NULL);
5082 }
5083
5084 /* Allocate an extent node with which to track the chunk. */
5085 node = base_node_alloc();
5086 if (node == NULL)
5087 return (NULL);
5088
5089 ret = chunk_alloc(csize, alignment, false, zero);
5090 if (ret == NULL) {
5091 base_node_dealloc(node);
5092 return (NULL);
5093 }
5094
5095 /* Insert node into huge. */
5096 node->addr = ret;
5097 psize = PAGE_CEILING(size);
5098 node->size = psize;
5099
5100 malloc_mutex_lock(&huge_mtx);
5101 extent_tree_ad_insert(&huge, node);
5102 #ifdef MALLOC_STATS
5103 huge_nmalloc++;
5104
5105 /* Although we allocated space for csize bytes, we indicate that we've
5106 * allocated only psize bytes.
5107 *
5108 * If DECOMMIT is defined, this is a reasonable thing to do, since
5109 * we'll explicitly decommit the bytes in excess of psize.
5110 *
5111 * If DECOMMIT is not defined, then we're relying on the OS to be lazy
5112 * about how it allocates physical pages to mappings. If we never
5113 * touch the pages in excess of psize, the OS won't allocate a physical
5114 * page, and we won't use more than psize bytes of physical memory.
5115 *
5116 * A correct program will only touch memory in excess of how much it
5117 * requested if it first calls malloc_usable_size and finds out how
5118 * much space it has to play with. But because we set node->size =
5119 * psize above, malloc_usable_size will return psize, not csize, and
5120 * the program will (hopefully) never touch bytes in excess of psize.
5121 * Thus those bytes won't take up space in physical memory, and we can
5122 * reasonably claim we never "allocated" them in the first place. */
5123 huge_allocated += psize;
5124 huge_mapped += csize;
5125 #endif
5126 malloc_mutex_unlock(&huge_mtx);
5127
5128 #ifdef MALLOC_DECOMMIT
5129 if (csize - psize > 0)
5130 pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
5131 #endif
5132
5133 #ifdef MALLOC_FILL
5134 if (zero == false) {
5135 if (opt_junk)
5136 # ifdef MALLOC_DECOMMIT
5137 memset(ret, 0xe4, psize);
5138 # else
5139 memset(ret, 0xe4, csize);
5140 # endif
5141 else if (opt_zero)
5142 # ifdef MALLOC_DECOMMIT
5143 memset(ret, 0, psize);
5144 # else
5145 memset(ret, 0, csize);
5146 # endif
5147 }
5148 #endif
5149
5150 return (ret);
5151 }
5152
5153 static void *
huge_ralloc(void * ptr,size_t size,size_t oldsize)5154 huge_ralloc(void *ptr, size_t size, size_t oldsize)
5155 {
5156 void *ret;
5157 size_t copysize;
5158
5159 /* Avoid moving the allocation if the size class would not change. */
5160
5161 if (oldsize > arena_maxclass &&
5162 CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
5163 size_t psize = PAGE_CEILING(size);
5164 #ifdef MALLOC_FILL
5165 if (opt_poison && size < oldsize) {
5166 memset((void *)((uintptr_t)ptr + size), 0xe5, oldsize
5167 - size);
5168 }
5169 #endif
5170 #ifdef MALLOC_DECOMMIT
5171 if (psize < oldsize) {
5172 extent_node_t *node, key;
5173
5174 pages_decommit((void *)((uintptr_t)ptr + psize),
5175 oldsize - psize);
5176
5177 /* Update recorded size. */
5178 malloc_mutex_lock(&huge_mtx);
5179 key.addr = __DECONST(void *, ptr);
5180 node = extent_tree_ad_search(&huge, &key);
5181 assert(node != NULL);
5182 assert(node->size == oldsize);
5183 # ifdef MALLOC_STATS
5184 huge_allocated -= oldsize - psize;
5185 /* No need to change huge_mapped, because we didn't
5186 * (un)map anything. */
5187 # endif
5188 node->size = psize;
5189 malloc_mutex_unlock(&huge_mtx);
5190 } else if (psize > oldsize) {
5191 pages_commit((void *)((uintptr_t)ptr + oldsize),
5192 psize - oldsize);
5193 }
5194 #endif
5195
5196 /* Although we don't have to commit or decommit anything if
5197 * DECOMMIT is not defined and the size class didn't change, we
5198 * do need to update the recorded size if the size increased,
5199 * so malloc_usable_size doesn't return a value smaller than
5200 * what was requested via realloc(). */
5201
5202 if (psize > oldsize) {
5203 /* Update recorded size. */
5204 extent_node_t *node, key;
5205 malloc_mutex_lock(&huge_mtx);
5206 key.addr = __DECONST(void *, ptr);
5207 node = extent_tree_ad_search(&huge, &key);
5208 assert(node != NULL);
5209 assert(node->size == oldsize);
5210 # ifdef MALLOC_STATS
5211 huge_allocated += psize - oldsize;
5212 /* No need to change huge_mapped, because we didn't
5213 * (un)map anything. */
5214 # endif
5215 node->size = psize;
5216 malloc_mutex_unlock(&huge_mtx);
5217 }
5218
5219 #ifdef MALLOC_FILL
5220 if (opt_zero && size > oldsize) {
5221 memset((void *)((uintptr_t)ptr + oldsize), 0, size
5222 - oldsize);
5223 }
5224 #endif
5225 return (ptr);
5226 }
5227
5228 /*
5229 * If we get here, then size and oldsize are different enough that we
5230 * need to use a different size class. In that case, fall back to
5231 * allocating new space and copying.
5232 */
5233 ret = huge_malloc(size, false);
5234 if (ret == NULL)
5235 return (NULL);
5236
5237 copysize = (size < oldsize) ? size : oldsize;
5238 #ifdef VM_COPY_MIN
5239 if (copysize >= VM_COPY_MIN)
5240 pages_copy(ret, ptr, copysize);
5241 else
5242 #endif
5243 memcpy(ret, ptr, copysize);
5244 idalloc(ptr);
5245 return (ret);
5246 }
5247
5248 static void
huge_dalloc(void * ptr)5249 huge_dalloc(void *ptr)
5250 {
5251 extent_node_t *node, key;
5252
5253 malloc_mutex_lock(&huge_mtx);
5254
5255 /* Extract from tree of huge allocations. */
5256 key.addr = ptr;
5257 node = extent_tree_ad_search(&huge, &key);
5258 assert(node != NULL);
5259 assert(node->addr == ptr);
5260 extent_tree_ad_remove(&huge, node);
5261
5262 #ifdef MALLOC_STATS
5263 huge_ndalloc++;
5264 huge_allocated -= node->size;
5265 huge_mapped -= CHUNK_CEILING(node->size);
5266 #endif
5267
5268 malloc_mutex_unlock(&huge_mtx);
5269
5270 /* Unmap chunk. */
5271 chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
5272
5273 base_node_dealloc(node);
5274 }
5275
5276 #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5277 #ifdef MOZ_MEMORY_BSD
5278 static inline unsigned
malloc_ncpus(void)5279 malloc_ncpus(void)
5280 {
5281 unsigned ret;
5282 int mib[2];
5283 size_t len;
5284
5285 mib[0] = CTL_HW;
5286 mib[1] = HW_NCPU;
5287 len = sizeof(ret);
5288 if (sysctl(mib, 2, &ret, &len, (void *) 0, 0) == -1) {
5289 /* Error. */
5290 return (1);
5291 }
5292
5293 return (ret);
5294 }
5295 #elif (defined(MOZ_MEMORY_LINUX))
5296 #include <fcntl.h>
5297
5298 static inline unsigned
malloc_ncpus(void)5299 malloc_ncpus(void)
5300 {
5301 unsigned ret;
5302 int fd, nread, column;
5303 char buf[1024];
5304 static const char matchstr[] = "processor\t:";
5305 int i;
5306
5307 /*
5308 * sysconf(3) would be the preferred method for determining the number
5309 * of CPUs, but it uses malloc internally, which causes untennable
5310 * recursion during malloc initialization.
5311 */
5312 fd = open("/proc/cpuinfo", O_RDONLY);
5313 if (fd == -1)
5314 return (1); /* Error. */
5315 /*
5316 * Count the number of occurrences of matchstr at the beginnings of
5317 * lines. This treats hyperthreaded CPUs as multiple processors.
5318 */
5319 column = 0;
5320 ret = 0;
5321 while (true) {
5322 nread = read(fd, &buf, sizeof(buf));
5323 if (nread <= 0)
5324 break; /* EOF or error. */
5325 for (i = 0;i < nread;i++) {
5326 char c = buf[i];
5327 if (c == '\n')
5328 column = 0;
5329 else if (column != -1) {
5330 if (c == matchstr[column]) {
5331 column++;
5332 if (column == sizeof(matchstr) - 1) {
5333 column = -1;
5334 ret++;
5335 }
5336 } else
5337 column = -1;
5338 }
5339 }
5340 }
5341
5342 if (ret == 0)
5343 ret = 1; /* Something went wrong in the parser. */
5344 close(fd);
5345
5346 return (ret);
5347 }
5348 #elif (defined(MOZ_MEMORY_DARWIN))
5349 #include <mach/mach_init.h>
5350 #include <mach/mach_host.h>
5351
5352 static inline unsigned
malloc_ncpus(void)5353 malloc_ncpus(void)
5354 {
5355 kern_return_t error;
5356 natural_t n;
5357 processor_info_array_t pinfo;
5358 mach_msg_type_number_t pinfocnt;
5359
5360 error = host_processor_info(mach_host_self(), PROCESSOR_BASIC_INFO,
5361 &n, &pinfo, &pinfocnt);
5362 if (error != KERN_SUCCESS)
5363 return (1); /* Error. */
5364 else
5365 return (n);
5366 }
5367 #elif (defined(MOZ_MEMORY_SOLARIS))
5368
5369 static inline unsigned
malloc_ncpus(void)5370 malloc_ncpus(void)
5371 {
5372 return sysconf(_SC_NPROCESSORS_ONLN);
5373 }
5374 #else
5375 static inline unsigned
malloc_ncpus(void)5376 malloc_ncpus(void)
5377 {
5378
5379 /*
5380 * We lack a way to determine the number of CPUs on this platform, so
5381 * assume 1 CPU.
5382 */
5383 return (1);
5384 }
5385 #endif
5386 #endif
5387
5388 static void
malloc_print_stats(void)5389 malloc_print_stats(void)
5390 {
5391
5392 if (opt_print_stats) {
5393 char s[UMAX2S_BUFSIZE];
5394 _malloc_message("___ Begin malloc statistics ___\n", "", "",
5395 "");
5396 _malloc_message("Assertions ",
5397 #ifdef NDEBUG
5398 "disabled",
5399 #else
5400 "enabled",
5401 #endif
5402 "\n", "");
5403 _malloc_message("Boolean MALLOC_OPTIONS: ",
5404 opt_abort ? "A" : "a", "", "");
5405 #ifdef MALLOC_FILL
5406 _malloc_message(opt_poison ? "C" : "c", "", "", "");
5407 _malloc_message(opt_junk ? "J" : "j", "", "", "");
5408 #endif
5409 _malloc_message("P", "", "", "");
5410 #ifdef MALLOC_UTRACE
5411 _malloc_message(opt_utrace ? "U" : "u", "", "", "");
5412 #endif
5413 #ifdef MALLOC_SYSV
5414 _malloc_message(opt_sysv ? "V" : "v", "", "", "");
5415 #endif
5416 #ifdef MALLOC_XMALLOC
5417 _malloc_message(opt_xmalloc ? "X" : "x", "", "", "");
5418 #endif
5419 #ifdef MALLOC_FILL
5420 _malloc_message(opt_zero ? "Z" : "z", "", "", "");
5421 #endif
5422 _malloc_message("\n", "", "", "");
5423
5424 #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5425 _malloc_message("CPUs: ", umax2s(ncpus, 10, s), "\n", "");
5426 #endif
5427 _malloc_message("Max arenas: ", umax2s(narenas, 10, s), "\n",
5428 "");
5429 #ifdef MALLOC_BALANCE
5430 _malloc_message("Arena balance threshold: ",
5431 umax2s(opt_balance_threshold, 10, s), "\n", "");
5432 #endif
5433 _malloc_message("Pointer size: ", umax2s(sizeof(void *), 10, s),
5434 "\n", "");
5435 _malloc_message("Quantum size: ", umax2s(quantum, 10, s), "\n",
5436 "");
5437 _malloc_message("Max small size: ", umax2s(small_max, 10, s),
5438 "\n", "");
5439 _malloc_message("Max dirty pages per arena: ",
5440 umax2s(opt_dirty_max, 10, s), "\n", "");
5441
5442 _malloc_message("Chunk size: ", umax2s(chunksize, 10, s), "",
5443 "");
5444 _malloc_message(" (2^", umax2s(opt_chunk_2pow, 10, s), ")\n",
5445 "");
5446
5447 #ifdef MALLOC_STATS
5448 {
5449 size_t allocated, mapped = 0;
5450 #ifdef MALLOC_BALANCE
5451 uint64_t nbalance = 0;
5452 #endif
5453 unsigned i;
5454 arena_t *arena;
5455
5456 /* Calculate and print allocated/mapped stats. */
5457
5458 /* arenas. */
5459 for (i = 0, allocated = 0; i < narenas; i++) {
5460 if (arenas[i] != NULL) {
5461 malloc_spin_lock(&arenas[i]->lock);
5462 allocated +=
5463 arenas[i]->stats.allocated_small;
5464 allocated +=
5465 arenas[i]->stats.allocated_large;
5466 mapped += arenas[i]->stats.mapped;
5467 #ifdef MALLOC_BALANCE
5468 nbalance += arenas[i]->stats.nbalance;
5469 #endif
5470 malloc_spin_unlock(&arenas[i]->lock);
5471 }
5472 }
5473
5474 /* huge/base. */
5475 malloc_mutex_lock(&huge_mtx);
5476 allocated += huge_allocated;
5477 mapped += huge_mapped;
5478 malloc_mutex_unlock(&huge_mtx);
5479
5480 malloc_mutex_lock(&base_mtx);
5481 mapped += base_mapped;
5482 malloc_mutex_unlock(&base_mtx);
5483
5484 #ifdef MOZ_MEMORY_WINDOWS
5485 malloc_printf("Allocated: %lu, mapped: %lu\n",
5486 allocated, mapped);
5487 #else
5488 malloc_printf("Allocated: %zu, mapped: %zu\n",
5489 allocated, mapped);
5490 #endif
5491
5492 #ifdef MALLOC_BALANCE
5493 malloc_printf("Arena balance reassignments: %llu\n",
5494 nbalance);
5495 #endif
5496
5497 /* Print chunk stats. */
5498 malloc_printf(
5499 "huge: nmalloc ndalloc allocated\n");
5500 #ifdef MOZ_MEMORY_WINDOWS
5501 malloc_printf(" %12llu %12llu %12lu\n",
5502 huge_nmalloc, huge_ndalloc, huge_allocated);
5503 #else
5504 malloc_printf(" %12llu %12llu %12zu\n",
5505 huge_nmalloc, huge_ndalloc, huge_allocated);
5506 #endif
5507 /* Print stats for each arena. */
5508 for (i = 0; i < narenas; i++) {
5509 arena = arenas[i];
5510 if (arena != NULL) {
5511 malloc_printf(
5512 "\narenas[%u]:\n", i);
5513 malloc_spin_lock(&arena->lock);
5514 stats_print(arena);
5515 malloc_spin_unlock(&arena->lock);
5516 }
5517 }
5518 }
5519 #endif /* #ifdef MALLOC_STATS */
5520 _malloc_message("--- End malloc statistics ---\n", "", "", "");
5521 }
5522 }
5523
5524 /*
5525 * FreeBSD's pthreads implementation calls malloc(3), so the malloc
5526 * implementation has to take pains to avoid infinite recursion during
5527 * initialization.
5528 */
5529 #if (defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_DARWIN))
5530 #define malloc_init() false
5531 #else
5532 static inline bool
malloc_init(void)5533 malloc_init(void)
5534 {
5535
5536 if (malloc_initialized == false)
5537 return (malloc_init_hard());
5538
5539 return (false);
5540 }
5541 #endif
5542
5543 #if !defined(MOZ_MEMORY_WINDOWS)
5544 static
5545 #endif
5546 bool
malloc_init_hard(void)5547 malloc_init_hard(void)
5548 {
5549 unsigned i;
5550 char buf[PATH_MAX + 1];
5551 const char *opts;
5552 long result;
5553 #ifndef MOZ_MEMORY_WINDOWS
5554 int linklen;
5555 #endif
5556 #ifdef MOZ_MEMORY_DARWIN
5557 malloc_zone_t* default_zone;
5558 #endif
5559
5560 #ifndef MOZ_MEMORY_WINDOWS
5561 malloc_mutex_lock(&init_lock);
5562 #endif
5563
5564 if (malloc_initialized) {
5565 /*
5566 * Another thread initialized the allocator before this one
5567 * acquired init_lock.
5568 */
5569 #ifndef MOZ_MEMORY_WINDOWS
5570 malloc_mutex_unlock(&init_lock);
5571 #endif
5572 return (false);
5573 }
5574
5575 #ifdef MOZ_MEMORY_WINDOWS
5576 /* get a thread local storage index */
5577 tlsIndex = TlsAlloc();
5578 #endif
5579
5580 /* Get page size and number of CPUs */
5581 #ifdef MOZ_MEMORY_WINDOWS
5582 {
5583 SYSTEM_INFO info;
5584
5585 GetSystemInfo(&info);
5586 result = info.dwPageSize;
5587
5588 #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5589 ncpus = info.dwNumberOfProcessors;
5590 #endif
5591 }
5592 #else
5593 #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5594 ncpus = malloc_ncpus();
5595 #endif
5596
5597 result = sysconf(_SC_PAGESIZE);
5598 assert(result != -1);
5599 #endif
5600
5601 /* We assume that the page size is a power of 2. */
5602 assert(((result - 1) & result) == 0);
5603 #ifdef MALLOC_STATIC_SIZES
5604 if (pagesize % (size_t) result) {
5605 _malloc_message(_getprogname(),
5606 "Compile-time page size does not divide the runtime one.\n",
5607 "", "");
5608 abort();
5609 }
5610 #else
5611 pagesize = (size_t) result;
5612 pagesize_mask = (size_t) result - 1;
5613 pagesize_2pow = ffs((int)result) - 1;
5614 #endif
5615
5616 for (i = 0; i < 3; i++) {
5617 unsigned j;
5618
5619 /* Get runtime configuration. */
5620 switch (i) {
5621 case 0:
5622 #ifndef MOZ_MEMORY_WINDOWS
5623 if ((linklen = readlink("/etc/malloc.conf", buf,
5624 sizeof(buf) - 1)) != -1) {
5625 /*
5626 * Use the contents of the "/etc/malloc.conf"
5627 * symbolic link's name.
5628 */
5629 buf[linklen] = '\0';
5630 opts = buf;
5631 } else
5632 #endif
5633 {
5634 /* No configuration specified. */
5635 buf[0] = '\0';
5636 opts = buf;
5637 }
5638 break;
5639 case 1:
5640 if ((opts = getenv("MALLOC_OPTIONS")) != NULL) {
5641 /*
5642 * Do nothing; opts is already initialized to
5643 * the value of the MALLOC_OPTIONS environment
5644 * variable.
5645 */
5646 } else {
5647 /* No configuration specified. */
5648 buf[0] = '\0';
5649 opts = buf;
5650 }
5651 break;
5652 case 2:
5653 if (_malloc_options != NULL) {
5654 /*
5655 * Use options that were compiled into the
5656 * program.
5657 */
5658 opts = _malloc_options;
5659 } else {
5660 /* No configuration specified. */
5661 buf[0] = '\0';
5662 opts = buf;
5663 }
5664 break;
5665 default:
5666 /* NOTREACHED */
5667 buf[0] = '\0';
5668 opts = buf;
5669 assert(false);
5670 }
5671
5672 for (j = 0; opts[j] != '\0'; j++) {
5673 unsigned k, nreps;
5674 bool nseen;
5675
5676 /* Parse repetition count, if any. */
5677 for (nreps = 0, nseen = false;; j++, nseen = true) {
5678 switch (opts[j]) {
5679 case '0': case '1': case '2': case '3':
5680 case '4': case '5': case '6': case '7':
5681 case '8': case '9':
5682 nreps *= 10;
5683 nreps += opts[j] - '0';
5684 break;
5685 default:
5686 goto MALLOC_OUT;
5687 }
5688 }
5689 MALLOC_OUT:
5690 if (nseen == false)
5691 nreps = 1;
5692
5693 for (k = 0; k < nreps; k++) {
5694 switch (opts[j]) {
5695 case 'a':
5696 opt_abort = false;
5697 break;
5698 case 'A':
5699 opt_abort = true;
5700 break;
5701 case 'b':
5702 #ifdef MALLOC_BALANCE
5703 opt_balance_threshold >>= 1;
5704 #endif
5705 break;
5706 case 'B':
5707 #ifdef MALLOC_BALANCE
5708 if (opt_balance_threshold == 0)
5709 opt_balance_threshold = 1;
5710 else if ((opt_balance_threshold << 1)
5711 > opt_balance_threshold)
5712 opt_balance_threshold <<= 1;
5713 #endif
5714 break;
5715 #ifdef MALLOC_FILL
5716 #ifndef MALLOC_PRODUCTION
5717 case 'c':
5718 opt_poison = false;
5719 break;
5720 case 'C':
5721 opt_poison = true;
5722 break;
5723 #endif
5724 #endif
5725 case 'f':
5726 opt_dirty_max >>= 1;
5727 break;
5728 case 'F':
5729 if (opt_dirty_max == 0)
5730 opt_dirty_max = 1;
5731 else if ((opt_dirty_max << 1) != 0)
5732 opt_dirty_max <<= 1;
5733 break;
5734 #ifdef MALLOC_FILL
5735 #ifndef MALLOC_PRODUCTION
5736 case 'j':
5737 opt_junk = false;
5738 break;
5739 case 'J':
5740 opt_junk = true;
5741 break;
5742 #endif
5743 #endif
5744 #ifndef MALLOC_STATIC_SIZES
5745 case 'k':
5746 /*
5747 * Chunks always require at least one
5748 * header page, so chunks can never be
5749 * smaller than two pages.
5750 */
5751 if (opt_chunk_2pow > pagesize_2pow + 1)
5752 opt_chunk_2pow--;
5753 break;
5754 case 'K':
5755 if (opt_chunk_2pow + 1 <
5756 (sizeof(size_t) << 3))
5757 opt_chunk_2pow++;
5758 break;
5759 #endif
5760 case 'n':
5761 opt_narenas_lshift--;
5762 break;
5763 case 'N':
5764 opt_narenas_lshift++;
5765 break;
5766 case 'p':
5767 opt_print_stats = false;
5768 break;
5769 case 'P':
5770 opt_print_stats = true;
5771 break;
5772 #ifndef MALLOC_STATIC_SIZES
5773 case 'q':
5774 if (opt_quantum_2pow > QUANTUM_2POW_MIN)
5775 opt_quantum_2pow--;
5776 break;
5777 case 'Q':
5778 if (opt_quantum_2pow < pagesize_2pow -
5779 1)
5780 opt_quantum_2pow++;
5781 break;
5782 case 's':
5783 if (opt_small_max_2pow >
5784 QUANTUM_2POW_MIN)
5785 opt_small_max_2pow--;
5786 break;
5787 case 'S':
5788 if (opt_small_max_2pow < pagesize_2pow
5789 - 1)
5790 opt_small_max_2pow++;
5791 break;
5792 #endif
5793 #ifdef MALLOC_UTRACE
5794 case 'u':
5795 opt_utrace = false;
5796 break;
5797 case 'U':
5798 opt_utrace = true;
5799 break;
5800 #endif
5801 #ifdef MALLOC_SYSV
5802 case 'v':
5803 opt_sysv = false;
5804 break;
5805 case 'V':
5806 opt_sysv = true;
5807 break;
5808 #endif
5809 #ifdef MALLOC_XMALLOC
5810 case 'x':
5811 opt_xmalloc = false;
5812 break;
5813 case 'X':
5814 opt_xmalloc = true;
5815 break;
5816 #endif
5817 #ifdef MALLOC_FILL
5818 #ifndef MALLOC_PRODUCTION
5819 case 'z':
5820 opt_zero = false;
5821 break;
5822 case 'Z':
5823 opt_zero = true;
5824 break;
5825 #endif
5826 #endif
5827 default: {
5828 char cbuf[2];
5829
5830 cbuf[0] = opts[j];
5831 cbuf[1] = '\0';
5832 _malloc_message(_getprogname(),
5833 ": (malloc) Unsupported character "
5834 "in malloc options: '", cbuf,
5835 "'\n");
5836 }
5837 }
5838 }
5839 }
5840 }
5841
5842 /* Take care to call atexit() only once. */
5843 if (opt_print_stats) {
5844 #ifndef MOZ_MEMORY_WINDOWS
5845 /* Print statistics at exit. */
5846 atexit(malloc_print_stats);
5847 #endif
5848 }
5849
5850 #ifndef MALLOC_STATIC_SIZES
5851 /* Set variables according to the value of opt_small_max_2pow. */
5852 if (opt_small_max_2pow < opt_quantum_2pow)
5853 opt_small_max_2pow = opt_quantum_2pow;
5854 small_max = (1U << opt_small_max_2pow);
5855
5856 /* Set bin-related variables. */
5857 bin_maxclass = (pagesize >> 1);
5858 assert(opt_quantum_2pow >= TINY_MIN_2POW);
5859 ntbins = opt_quantum_2pow - TINY_MIN_2POW;
5860 assert(ntbins <= opt_quantum_2pow);
5861 nqbins = (small_max >> opt_quantum_2pow);
5862 nsbins = pagesize_2pow - opt_small_max_2pow - 1;
5863
5864 /* Set variables according to the value of opt_quantum_2pow. */
5865 quantum = (1U << opt_quantum_2pow);
5866 quantum_mask = quantum - 1;
5867 if (ntbins > 0)
5868 small_min = (quantum >> 1) + 1;
5869 else
5870 small_min = 1;
5871 assert(small_min <= quantum);
5872
5873 /* Set variables according to the value of opt_chunk_2pow. */
5874 chunksize = (1LU << opt_chunk_2pow);
5875 chunksize_mask = chunksize - 1;
5876 chunk_npages = (chunksize >> pagesize_2pow);
5877
5878 arena_chunk_header_npages = calculate_arena_header_pages();
5879 arena_maxclass = calculate_arena_maxclass();
5880
5881 recycle_limit = CHUNK_RECYCLE_LIMIT * chunksize;
5882 #endif
5883
5884 recycled_size = 0;
5885
5886 #ifdef JEMALLOC_USES_MAP_ALIGN
5887 /*
5888 * When using MAP_ALIGN, the alignment parameter must be a power of two
5889 * multiple of the system pagesize, or mmap will fail.
5890 */
5891 assert((chunksize % pagesize) == 0);
5892 assert((1 << (ffs(chunksize / pagesize) - 1)) == (chunksize/pagesize));
5893 #endif
5894
5895 UTRACE(0, 0, 0);
5896
5897 /* Various sanity checks that regard configuration. */
5898 assert(quantum >= sizeof(void *));
5899 assert(quantum <= pagesize);
5900 assert(chunksize >= pagesize);
5901 assert(quantum * 4 <= chunksize);
5902
5903 /* Initialize chunks data. */
5904 malloc_mutex_init(&chunks_mtx);
5905 extent_tree_szad_new(&chunks_szad_mmap);
5906 extent_tree_ad_new(&chunks_ad_mmap);
5907
5908 /* Initialize huge allocation data. */
5909 malloc_mutex_init(&huge_mtx);
5910 extent_tree_ad_new(&huge);
5911 #ifdef MALLOC_STATS
5912 huge_nmalloc = 0;
5913 huge_ndalloc = 0;
5914 huge_allocated = 0;
5915 huge_mapped = 0;
5916 #endif
5917
5918 /* Initialize base allocation data structures. */
5919 #ifdef MALLOC_STATS
5920 base_mapped = 0;
5921 base_committed = 0;
5922 #endif
5923 base_nodes = NULL;
5924 malloc_mutex_init(&base_mtx);
5925
5926 #ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5927 narenas = 1;
5928 #else
5929 if (ncpus > 1) {
5930 /*
5931 * For SMP systems, create four times as many arenas as there
5932 * are CPUs by default.
5933 */
5934 opt_narenas_lshift += 2;
5935 }
5936
5937 /* Determine how many arenas to use. */
5938 narenas = ncpus;
5939 #endif
5940 if (opt_narenas_lshift > 0) {
5941 if ((narenas << opt_narenas_lshift) > narenas)
5942 narenas <<= opt_narenas_lshift;
5943 /*
5944 * Make sure not to exceed the limits of what base_alloc() can
5945 * handle.
5946 */
5947 if (narenas * sizeof(arena_t *) > chunksize)
5948 narenas = chunksize / sizeof(arena_t *);
5949 } else if (opt_narenas_lshift < 0) {
5950 if ((narenas >> -opt_narenas_lshift) < narenas)
5951 narenas >>= -opt_narenas_lshift;
5952 /* Make sure there is at least one arena. */
5953 if (narenas == 0)
5954 narenas = 1;
5955 }
5956 #ifdef MALLOC_BALANCE
5957 assert(narenas != 0);
5958 for (narenas_2pow = 0;
5959 (narenas >> (narenas_2pow + 1)) != 0;
5960 narenas_2pow++);
5961 #endif
5962
5963 #ifdef NO_TLS
5964 if (narenas > 1) {
5965 static const unsigned primes[] = {1, 3, 5, 7, 11, 13, 17, 19,
5966 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
5967 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149,
5968 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
5969 223, 227, 229, 233, 239, 241, 251, 257, 263};
5970 unsigned nprimes, parenas;
5971
5972 /*
5973 * Pick a prime number of hash arenas that is more than narenas
5974 * so that direct hashing of pthread_self() pointers tends to
5975 * spread allocations evenly among the arenas.
5976 */
5977 assert((narenas & 1) == 0); /* narenas must be even. */
5978 nprimes = (sizeof(primes) >> SIZEOF_INT_2POW);
5979 parenas = primes[nprimes - 1]; /* In case not enough primes. */
5980 for (i = 1; i < nprimes; i++) {
5981 if (primes[i] > narenas) {
5982 parenas = primes[i];
5983 break;
5984 }
5985 }
5986 narenas = parenas;
5987 }
5988 #endif
5989
5990 #ifndef NO_TLS
5991 # ifndef MALLOC_BALANCE
5992 next_arena = 0;
5993 # endif
5994 #endif
5995
5996 /* Allocate and initialize arenas. */
5997 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
5998 if (arenas == NULL) {
5999 #ifndef MOZ_MEMORY_WINDOWS
6000 malloc_mutex_unlock(&init_lock);
6001 #endif
6002 return (true);
6003 }
6004 /*
6005 * Zero the array. In practice, this should always be pre-zeroed,
6006 * since it was just mmap()ed, but let's be sure.
6007 */
6008 memset(arenas, 0, sizeof(arena_t *) * narenas);
6009
6010 /*
6011 * Initialize one arena here. The rest are lazily created in
6012 * choose_arena_hard().
6013 */
6014 arenas_extend(0);
6015 if (arenas[0] == NULL) {
6016 #ifndef MOZ_MEMORY_WINDOWS
6017 malloc_mutex_unlock(&init_lock);
6018 #endif
6019 return (true);
6020 }
6021 #ifndef NO_TLS
6022 /*
6023 * Assign the initial arena to the initial thread, in order to avoid
6024 * spurious creation of an extra arena if the application switches to
6025 * threaded mode.
6026 */
6027 #ifdef MOZ_MEMORY_WINDOWS
6028 TlsSetValue(tlsIndex, arenas[0]);
6029 #else
6030 arenas_map = arenas[0];
6031 #endif
6032 #endif
6033
6034 /*
6035 * Seed here for the initial thread, since choose_arena_hard() is only
6036 * called for other threads. The seed value doesn't really matter.
6037 */
6038 #ifdef MALLOC_BALANCE
6039 SPRN(balance, 42);
6040 #endif
6041
6042 malloc_spin_init(&arenas_lock);
6043
6044 #ifdef MALLOC_VALIDATE
6045 chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
6046 if (chunk_rtree == NULL)
6047 return (true);
6048 #endif
6049
6050 malloc_initialized = true;
6051
6052 #if !defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN)
6053 /* Prevent potential deadlock on malloc locks after fork. */
6054 pthread_atfork(_malloc_prefork, _malloc_postfork, _malloc_postfork);
6055 #endif
6056
6057 #if defined(NEEDS_PTHREAD_MMAP_UNALIGNED_TSD)
6058 if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
6059 malloc_printf("<jemalloc>: Error in pthread_key_create()\n");
6060 }
6061 #endif
6062
6063 #if defined(MOZ_MEMORY_DARWIN) && !defined(MOZ_REPLACE_MALLOC)
6064 /*
6065 * Overwrite the default memory allocator to use jemalloc everywhere.
6066 */
6067 default_zone = malloc_default_zone();
6068
6069 /*
6070 * We only use jemalloc with MacOS 10.6 and 10.7. jemalloc is disabled
6071 * on 32-bit builds (10.5 and 32-bit 10.6) due to bug 702250, an
6072 * apparent MacOS bug. In fact, this code isn't even compiled on
6073 * 32-bit builds.
6074 *
6075 * We'll have to update our code to work with newer versions, because
6076 * the malloc zone layout is likely to change.
6077 */
6078
6079 osx_use_jemalloc = (default_zone->version == SNOW_LEOPARD_MALLOC_ZONE_T_VERSION ||
6080 default_zone->version == LION_MALLOC_ZONE_T_VERSION);
6081
6082 /* Allow us dynamically turn off jemalloc for testing. */
6083 if (getenv("NO_MAC_JEMALLOC")) {
6084 osx_use_jemalloc = false;
6085 #ifdef __i386__
6086 malloc_printf("Warning: NO_MAC_JEMALLOC has no effect on "
6087 "i386 machines (such as this one).\n");
6088 #endif
6089 }
6090
6091 if (osx_use_jemalloc) {
6092 /*
6093 * Convert the default szone to an "overlay zone" that is capable
6094 * of deallocating szone-allocated objects, but allocating new
6095 * objects from jemalloc.
6096 */
6097 size_t size = zone_version_size(default_zone->version);
6098 szone2ozone(default_zone, size);
6099 }
6100 else {
6101 szone = default_zone;
6102 }
6103 #endif
6104
6105 #ifndef MOZ_MEMORY_WINDOWS
6106 malloc_mutex_unlock(&init_lock);
6107 #endif
6108 return (false);
6109 }
6110
6111 /* XXX Why not just expose malloc_print_stats()? */
6112 #ifdef MOZ_MEMORY_WINDOWS
6113 void
malloc_shutdown()6114 malloc_shutdown()
6115 {
6116
6117 malloc_print_stats();
6118 }
6119 #endif
6120
6121 /*
6122 * End general internal functions.
6123 */
6124 /******************************************************************************/
6125 /*
6126 * Begin malloc(3)-compatible functions.
6127 */
6128
6129 /*
6130 * Even though we compile with MOZ_MEMORY, we may have to dynamically decide
6131 * not to use jemalloc, as discussed above. However, we call jemalloc
6132 * functions directly from mozalloc. Since it's pretty dangerous to mix the
6133 * allocators, we need to call the OSX allocators from the functions below,
6134 * when osx_use_jemalloc is not (dynamically) set.
6135 *
6136 * Note that we assume jemalloc is enabled on i386. This is safe because the
6137 * only i386 versions of MacOS are 10.5 and 10.6, which we support. We have to
6138 * do this because madvise isn't in the malloc zone struct for 10.5.
6139 *
6140 * This means that NO_MAC_JEMALLOC doesn't work on i386.
6141 */
6142 #if defined(MOZ_MEMORY_DARWIN) && !defined(__i386__) && !defined(MOZ_REPLACE_MALLOC)
6143 #define DARWIN_ONLY(A) if (!osx_use_jemalloc) { A; }
6144 #else
6145 #define DARWIN_ONLY(A)
6146 #endif
6147
6148 MOZ_MEMORY_API void *
malloc_impl(size_t size)6149 malloc_impl(size_t size)
6150 {
6151 void *ret;
6152
6153 DARWIN_ONLY(return (szone->malloc)(szone, size));
6154
6155 if (malloc_init()) {
6156 ret = NULL;
6157 goto RETURN;
6158 }
6159
6160 if (size == 0) {
6161 #ifdef MALLOC_SYSV
6162 if (opt_sysv == false)
6163 #endif
6164 size = 1;
6165 #ifdef MALLOC_SYSV
6166 else {
6167 ret = NULL;
6168 goto RETURN;
6169 }
6170 #endif
6171 }
6172
6173 ret = imalloc(size);
6174
6175 RETURN:
6176 if (ret == NULL) {
6177 #ifdef MALLOC_XMALLOC
6178 if (opt_xmalloc) {
6179 _malloc_message(_getprogname(),
6180 ": (malloc) Error in malloc(): out of memory\n", "",
6181 "");
6182 abort();
6183 }
6184 #endif
6185 errno = ENOMEM;
6186 }
6187
6188 UTRACE(0, size, ret);
6189 return (ret);
6190 }
6191
6192 /*
6193 * In ELF systems the default visibility allows symbols to be preempted at
6194 * runtime. This in turn prevents the uses of memalign in this file from being
6195 * optimized. What we do in here is define two aliasing symbols (they point to
6196 * the same code): memalign and memalign_internal. The internal version has
6197 * hidden visibility and is used in every reference from this file.
6198 *
6199 * For more information on this technique, see section 2.2.7 (Avoid Using
6200 * Exported Symbols) in http://www.akkadia.org/drepper/dsohowto.pdf.
6201 */
6202
6203 #ifndef MOZ_REPLACE_MALLOC
6204 #if defined(__GNUC__) && !defined(MOZ_MEMORY_DARWIN)
6205 #define MOZ_MEMORY_ELF
6206 #endif
6207
6208 #ifdef MOZ_MEMORY_SOLARIS
6209 # ifdef __SUNPRO_C
6210 void *
6211 memalign_impl(size_t alignment, size_t size);
6212 #pragma no_inline(memalign_impl)
6213 # elif (defined(__GNUC__))
6214 __attribute__((noinline))
6215 # endif
6216 #else
6217 #if (defined(MOZ_MEMORY_ELF))
6218 __attribute__((visibility ("hidden")))
6219 #endif
6220 #endif
6221 #endif /* MOZ_REPLACE_MALLOC */
6222
6223 #ifdef MOZ_MEMORY_ELF
6224 #define MEMALIGN memalign_internal
6225 #else
6226 #define MEMALIGN memalign_impl
6227 #endif
6228
6229 #ifndef MOZ_MEMORY_ELF
6230 MOZ_MEMORY_API
6231 #endif
6232 void *
MEMALIGN(size_t alignment,size_t size)6233 MEMALIGN(size_t alignment, size_t size)
6234 {
6235 void *ret;
6236
6237 DARWIN_ONLY(return (szone->memalign)(szone, alignment, size));
6238
6239 assert(((alignment - 1) & alignment) == 0);
6240
6241 if (malloc_init()) {
6242 ret = NULL;
6243 goto RETURN;
6244 }
6245
6246 if (size == 0) {
6247 #ifdef MALLOC_SYSV
6248 if (opt_sysv == false)
6249 #endif
6250 size = 1;
6251 #ifdef MALLOC_SYSV
6252 else {
6253 ret = NULL;
6254 goto RETURN;
6255 }
6256 #endif
6257 }
6258
6259 alignment = alignment < sizeof(void*) ? sizeof(void*) : alignment;
6260 ret = ipalloc(alignment, size);
6261
6262 RETURN:
6263 #ifdef MALLOC_XMALLOC
6264 if (opt_xmalloc && ret == NULL) {
6265 _malloc_message(_getprogname(),
6266 ": (malloc) Error in memalign(): out of memory\n", "", "");
6267 abort();
6268 }
6269 #endif
6270 UTRACE(0, size, ret);
6271 return (ret);
6272 }
6273
6274 #ifdef MOZ_MEMORY_ELF
6275 extern void *
6276 memalign_impl(size_t alignment, size_t size) __attribute__((alias ("memalign_internal"), visibility ("default")));
6277 #endif
6278
6279 MOZ_MEMORY_API int
posix_memalign_impl(void ** memptr,size_t alignment,size_t size)6280 posix_memalign_impl(void **memptr, size_t alignment, size_t size)
6281 {
6282 void *result;
6283
6284 /* Make sure that alignment is a large enough power of 2. */
6285 if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
6286 #ifdef MALLOC_XMALLOC
6287 if (opt_xmalloc) {
6288 _malloc_message(_getprogname(),
6289 ": (malloc) Error in posix_memalign(): "
6290 "invalid alignment\n", "", "");
6291 abort();
6292 }
6293 #endif
6294 return (EINVAL);
6295 }
6296
6297 /* The 0-->1 size promotion is done in the memalign() call below */
6298
6299 result = MEMALIGN(alignment, size);
6300
6301 if (result == NULL)
6302 return (ENOMEM);
6303
6304 *memptr = result;
6305 return (0);
6306 }
6307
6308 MOZ_MEMORY_API void *
aligned_alloc_impl(size_t alignment,size_t size)6309 aligned_alloc_impl(size_t alignment, size_t size)
6310 {
6311 if (size % alignment) {
6312 #ifdef MALLOC_XMALLOC
6313 if (opt_xmalloc) {
6314 _malloc_message(_getprogname(),
6315 ": (malloc) Error in aligned_alloc(): "
6316 "size is not multiple of alignment\n", "", "");
6317 abort();
6318 }
6319 #endif
6320 return (NULL);
6321 }
6322 return MEMALIGN(alignment, size);
6323 }
6324
6325 MOZ_MEMORY_API void *
valloc_impl(size_t size)6326 valloc_impl(size_t size)
6327 {
6328 return (MEMALIGN(pagesize, size));
6329 }
6330
6331 MOZ_MEMORY_API void *
calloc_impl(size_t num,size_t size)6332 calloc_impl(size_t num, size_t size)
6333 {
6334 void *ret;
6335 size_t num_size;
6336
6337 DARWIN_ONLY(return (szone->calloc)(szone, num, size));
6338
6339 if (malloc_init()) {
6340 num_size = 0;
6341 ret = NULL;
6342 goto RETURN;
6343 }
6344
6345 num_size = num * size;
6346 if (num_size == 0) {
6347 #ifdef MALLOC_SYSV
6348 if ((opt_sysv == false) && ((num == 0) || (size == 0)))
6349 #endif
6350 num_size = 1;
6351 #ifdef MALLOC_SYSV
6352 else {
6353 ret = NULL;
6354 goto RETURN;
6355 }
6356 #endif
6357 /*
6358 * Try to avoid division here. We know that it isn't possible to
6359 * overflow during multiplication if neither operand uses any of the
6360 * most significant half of the bits in a size_t.
6361 */
6362 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
6363 && (num_size / size != num)) {
6364 /* size_t overflow. */
6365 ret = NULL;
6366 goto RETURN;
6367 }
6368
6369 ret = icalloc(num_size);
6370
6371 RETURN:
6372 if (ret == NULL) {
6373 #ifdef MALLOC_XMALLOC
6374 if (opt_xmalloc) {
6375 _malloc_message(_getprogname(),
6376 ": (malloc) Error in calloc(): out of memory\n", "",
6377 "");
6378 abort();
6379 }
6380 #endif
6381 errno = ENOMEM;
6382 }
6383
6384 UTRACE(0, num_size, ret);
6385 return (ret);
6386 }
6387
6388 MOZ_MEMORY_API void *
realloc_impl(void * ptr,size_t size)6389 realloc_impl(void *ptr, size_t size)
6390 {
6391 void *ret;
6392
6393 DARWIN_ONLY(return (szone->realloc)(szone, ptr, size));
6394
6395 if (size == 0) {
6396 #ifdef MALLOC_SYSV
6397 if (opt_sysv == false)
6398 #endif
6399 size = 1;
6400 #ifdef MALLOC_SYSV
6401 else {
6402 if (ptr != NULL)
6403 idalloc(ptr);
6404 ret = NULL;
6405 goto RETURN;
6406 }
6407 #endif
6408 }
6409
6410 if (ptr != NULL) {
6411 assert(malloc_initialized);
6412
6413 ret = iralloc(ptr, size);
6414
6415 if (ret == NULL) {
6416 #ifdef MALLOC_XMALLOC
6417 if (opt_xmalloc) {
6418 _malloc_message(_getprogname(),
6419 ": (malloc) Error in realloc(): out of "
6420 "memory\n", "", "");
6421 abort();
6422 }
6423 #endif
6424 errno = ENOMEM;
6425 }
6426 } else {
6427 if (malloc_init())
6428 ret = NULL;
6429 else
6430 ret = imalloc(size);
6431
6432 if (ret == NULL) {
6433 #ifdef MALLOC_XMALLOC
6434 if (opt_xmalloc) {
6435 _malloc_message(_getprogname(),
6436 ": (malloc) Error in realloc(): out of "
6437 "memory\n", "", "");
6438 abort();
6439 }
6440 #endif
6441 errno = ENOMEM;
6442 }
6443 }
6444
6445 #ifdef MALLOC_SYSV
6446 RETURN:
6447 #endif
6448 UTRACE(ptr, size, ret);
6449 return (ret);
6450 }
6451
6452 MOZ_MEMORY_API void
free_impl(void * ptr)6453 free_impl(void *ptr)
6454 {
6455 size_t offset;
6456
6457 DARWIN_ONLY((szone->free)(szone, ptr); return);
6458
6459 UTRACE(ptr, 0, 0);
6460
6461 /*
6462 * A version of idalloc that checks for NULL pointer but only for
6463 * huge allocations assuming that CHUNK_ADDR2OFFSET(NULL) == 0.
6464 */
6465 assert(CHUNK_ADDR2OFFSET(NULL) == 0);
6466 offset = CHUNK_ADDR2OFFSET(ptr);
6467 if (offset != 0)
6468 arena_dalloc(ptr, offset);
6469 else if (ptr != NULL)
6470 huge_dalloc(ptr);
6471 }
6472
6473 /*
6474 * End malloc(3)-compatible functions.
6475 */
6476 /******************************************************************************/
6477 /*
6478 * Begin non-standard functions.
6479 */
6480
6481 /* This was added by Mozilla for use by SQLite. */
6482 #if defined(MOZ_MEMORY_DARWIN) && !defined(MOZ_REPLACE_MALLOC)
6483 static
6484 #else
6485 MOZ_MEMORY_API
6486 #endif
6487 size_t
malloc_good_size_impl(size_t size)6488 malloc_good_size_impl(size_t size)
6489 {
6490 /*
6491 * This duplicates the logic in imalloc(), arena_malloc() and
6492 * arena_malloc_small().
6493 */
6494 if (size < small_min) {
6495 /* Small (tiny). */
6496 size = pow2_ceil(size);
6497 /*
6498 * We omit the #ifdefs from arena_malloc_small() --
6499 * it can be inaccurate with its size in some cases, but this
6500 * function must be accurate.
6501 */
6502 if (size < (1U << TINY_MIN_2POW))
6503 size = (1U << TINY_MIN_2POW);
6504 } else if (size <= small_max) {
6505 /* Small (quantum-spaced). */
6506 size = QUANTUM_CEILING(size);
6507 } else if (size <= bin_maxclass) {
6508 /* Small (sub-page). */
6509 size = pow2_ceil(size);
6510 } else if (size <= arena_maxclass) {
6511 /* Large. */
6512 size = PAGE_CEILING(size);
6513 } else {
6514 /*
6515 * Huge. We use PAGE_CEILING to get psize, instead of using
6516 * CHUNK_CEILING to get csize. This ensures that this
6517 * malloc_usable_size(malloc(n)) always matches
6518 * malloc_good_size(n).
6519 */
6520 size = PAGE_CEILING(size);
6521 }
6522 return size;
6523 }
6524
6525
6526 MOZ_MEMORY_API size_t
malloc_usable_size_impl(MALLOC_USABLE_SIZE_CONST_PTR void * ptr)6527 malloc_usable_size_impl(MALLOC_USABLE_SIZE_CONST_PTR void *ptr)
6528 {
6529 DARWIN_ONLY(return (szone->size)(szone, ptr));
6530
6531 #ifdef MALLOC_VALIDATE
6532 return (isalloc_validate(ptr));
6533 #else
6534 assert(ptr != NULL);
6535
6536 return (isalloc(ptr));
6537 #endif
6538 }
6539
6540 MOZ_JEMALLOC_API void
jemalloc_stats_impl(jemalloc_stats_t * stats)6541 jemalloc_stats_impl(jemalloc_stats_t *stats)
6542 {
6543 size_t i, non_arena_mapped, chunk_header_size;
6544
6545 assert(stats != NULL);
6546
6547 /*
6548 * Gather runtime settings.
6549 */
6550 stats->opt_abort = opt_abort;
6551 stats->opt_junk =
6552 #ifdef MALLOC_FILL
6553 opt_junk ? true :
6554 #endif
6555 false;
6556 stats->opt_poison =
6557 #ifdef MALLOC_FILL
6558 opt_poison ? true :
6559 #endif
6560 false;
6561 stats->opt_utrace =
6562 #ifdef MALLOC_UTRACE
6563 opt_utrace ? true :
6564 #endif
6565 false;
6566 stats->opt_sysv =
6567 #ifdef MALLOC_SYSV
6568 opt_sysv ? true :
6569 #endif
6570 false;
6571 stats->opt_xmalloc =
6572 #ifdef MALLOC_XMALLOC
6573 opt_xmalloc ? true :
6574 #endif
6575 false;
6576 stats->opt_zero =
6577 #ifdef MALLOC_FILL
6578 opt_zero ? true :
6579 #endif
6580 false;
6581 stats->narenas = narenas;
6582 stats->balance_threshold =
6583 #ifdef MALLOC_BALANCE
6584 opt_balance_threshold
6585 #else
6586 SIZE_T_MAX
6587 #endif
6588 ;
6589 stats->quantum = quantum;
6590 stats->small_max = small_max;
6591 stats->large_max = arena_maxclass;
6592 stats->chunksize = chunksize;
6593 stats->dirty_max = opt_dirty_max;
6594
6595 /*
6596 * Gather current memory usage statistics.
6597 */
6598 stats->mapped = 0;
6599 stats->allocated = 0;
6600 stats->waste = 0;
6601 stats->page_cache = 0;
6602 stats->bookkeeping = 0;
6603 stats->bin_unused = 0;
6604
6605 non_arena_mapped = 0;
6606
6607 /* Get huge mapped/allocated. */
6608 malloc_mutex_lock(&huge_mtx);
6609 non_arena_mapped += huge_mapped;
6610 stats->allocated += huge_allocated;
6611 assert(huge_mapped >= huge_allocated);
6612 malloc_mutex_unlock(&huge_mtx);
6613
6614 /* Get base mapped/allocated. */
6615 malloc_mutex_lock(&base_mtx);
6616 non_arena_mapped += base_mapped;
6617 stats->bookkeeping += base_committed;
6618 assert(base_mapped >= base_committed);
6619 malloc_mutex_unlock(&base_mtx);
6620
6621 /* Iterate over arenas. */
6622 for (i = 0; i < narenas; i++) {
6623 arena_t *arena = arenas[i];
6624 size_t arena_mapped, arena_allocated, arena_committed, arena_dirty, j,
6625 arena_unused, arena_headers;
6626 arena_run_t* run;
6627 arena_chunk_map_t* mapelm;
6628
6629 if (arena == NULL) {
6630 continue;
6631 }
6632
6633 arena_headers = 0;
6634 arena_unused = 0;
6635
6636 malloc_spin_lock(&arena->lock);
6637
6638 arena_mapped = arena->stats.mapped;
6639
6640 /* "committed" counts dirty and allocated memory. */
6641 arena_committed = arena->stats.committed << pagesize_2pow;
6642
6643 arena_allocated = arena->stats.allocated_small +
6644 arena->stats.allocated_large;
6645
6646 arena_dirty = arena->ndirty << pagesize_2pow;
6647
6648 for (j = 0; j < ntbins + nqbins + nsbins; j++) {
6649 arena_bin_t* bin = &arena->bins[j];
6650 size_t bin_unused = 0;
6651
6652 rb_foreach_begin(arena_chunk_map_t, link, &bin->runs, mapelm) {
6653 run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
6654 bin_unused += run->nfree * bin->reg_size;
6655 } rb_foreach_end(arena_chunk_map_t, link, &bin->runs, mapelm)
6656
6657 if (bin->runcur) {
6658 bin_unused += bin->runcur->nfree * bin->reg_size;
6659 }
6660
6661 arena_unused += bin_unused;
6662 arena_headers += bin->stats.curruns * bin->reg0_offset;
6663 }
6664
6665 malloc_spin_unlock(&arena->lock);
6666
6667 assert(arena_mapped >= arena_committed);
6668 assert(arena_committed >= arena_allocated + arena_dirty);
6669
6670 /* "waste" is committed memory that is neither dirty nor
6671 * allocated. */
6672 stats->mapped += arena_mapped;
6673 stats->allocated += arena_allocated;
6674 stats->page_cache += arena_dirty;
6675 stats->waste += arena_committed -
6676 arena_allocated - arena_dirty - arena_unused - arena_headers;
6677 stats->bin_unused += arena_unused;
6678 stats->bookkeeping += arena_headers;
6679 }
6680
6681 /* Account for arena chunk headers in bookkeeping rather than waste. */
6682 chunk_header_size =
6683 ((stats->mapped / stats->chunksize) * arena_chunk_header_npages) <<
6684 pagesize_2pow;
6685
6686 stats->mapped += non_arena_mapped;
6687 stats->bookkeeping += chunk_header_size;
6688 stats->waste -= chunk_header_size;
6689
6690 assert(stats->mapped >= stats->allocated + stats->waste +
6691 stats->page_cache + stats->bookkeeping);
6692 }
6693
6694 #ifdef MALLOC_DOUBLE_PURGE
6695
6696 /* Explicitly remove all of this chunk's MADV_FREE'd pages from memory. */
6697 static void
hard_purge_chunk(arena_chunk_t * chunk)6698 hard_purge_chunk(arena_chunk_t *chunk)
6699 {
6700 /* See similar logic in arena_purge(). */
6701
6702 size_t i;
6703 for (i = arena_chunk_header_npages; i < chunk_npages; i++) {
6704 /* Find all adjacent pages with CHUNK_MAP_MADVISED set. */
6705 size_t npages;
6706 for (npages = 0;
6707 chunk->map[i + npages].bits & CHUNK_MAP_MADVISED && i + npages < chunk_npages;
6708 npages++) {
6709 /* Turn off the chunk's MADV_FREED bit and turn on its
6710 * DECOMMITTED bit. */
6711 RELEASE_ASSERT(!(chunk->map[i + npages].bits & CHUNK_MAP_DECOMMITTED));
6712 chunk->map[i + npages].bits ^= CHUNK_MAP_MADVISED_OR_DECOMMITTED;
6713 }
6714
6715 /* We could use mincore to find out which pages are actually
6716 * present, but it's not clear that's better. */
6717 if (npages > 0) {
6718 pages_decommit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
6719 pages_commit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
6720 }
6721 i += npages;
6722 }
6723 }
6724
6725 /* Explicitly remove all of this arena's MADV_FREE'd pages from memory. */
6726 static void
hard_purge_arena(arena_t * arena)6727 hard_purge_arena(arena_t *arena)
6728 {
6729 malloc_spin_lock(&arena->lock);
6730
6731 while (!LinkedList_IsEmpty(&arena->chunks_madvised)) {
6732 LinkedList* next = arena->chunks_madvised.next;
6733 arena_chunk_t *chunk =
6734 LinkedList_Get(arena->chunks_madvised.next,
6735 arena_chunk_t, chunks_madvised_elem);
6736 hard_purge_chunk(chunk);
6737 LinkedList_Remove(&chunk->chunks_madvised_elem);
6738 }
6739
6740 malloc_spin_unlock(&arena->lock);
6741 }
6742
6743 MOZ_JEMALLOC_API void
jemalloc_purge_freed_pages_impl()6744 jemalloc_purge_freed_pages_impl()
6745 {
6746 size_t i;
6747 for (i = 0; i < narenas; i++) {
6748 arena_t *arena = arenas[i];
6749 if (arena != NULL)
6750 hard_purge_arena(arena);
6751 }
6752 if (!config_munmap || config_recycle) {
6753 malloc_mutex_lock(&chunks_mtx);
6754 extent_node_t *node = extent_tree_szad_first(&chunks_szad_mmap);
6755 while (node) {
6756 pages_decommit(node->addr, node->size);
6757 pages_commit(node->addr, node->size);
6758 node->zeroed = true;
6759 node = extent_tree_szad_next(&chunks_szad_mmap, node);
6760 }
6761 malloc_mutex_unlock(&chunks_mtx);
6762 }
6763 }
6764
6765 #else /* !defined MALLOC_DOUBLE_PURGE */
6766
6767 MOZ_JEMALLOC_API void
jemalloc_purge_freed_pages_impl()6768 jemalloc_purge_freed_pages_impl()
6769 {
6770 /* Do nothing. */
6771 }
6772
6773 #endif /* defined MALLOC_DOUBLE_PURGE */
6774
6775
6776
6777 #ifdef MOZ_MEMORY_WINDOWS
6778 void*
_recalloc(void * ptr,size_t count,size_t size)6779 _recalloc(void *ptr, size_t count, size_t size)
6780 {
6781 size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
6782 size_t newsize = count * size;
6783
6784 /*
6785 * In order for all trailing bytes to be zeroed, the caller needs to
6786 * use calloc(), followed by recalloc(). However, the current calloc()
6787 * implementation only zeros the bytes requested, so if recalloc() is
6788 * to work 100% correctly, calloc() will need to change to zero
6789 * trailing bytes.
6790 */
6791
6792 ptr = realloc_impl(ptr, newsize);
6793 if (ptr != NULL && oldsize < newsize) {
6794 memset((void *)((uintptr_t)ptr + oldsize), 0, newsize -
6795 oldsize);
6796 }
6797
6798 return ptr;
6799 }
6800
6801 /*
6802 * This impl of _expand doesn't ever actually expand or shrink blocks: it
6803 * simply replies that you may continue using a shrunk block.
6804 */
6805 void*
_expand(void * ptr,size_t newsize)6806 _expand(void *ptr, size_t newsize)
6807 {
6808 if (isalloc(ptr) >= newsize)
6809 return ptr;
6810
6811 return NULL;
6812 }
6813
6814 size_t
_msize(void * ptr)6815 _msize(void *ptr)
6816 {
6817
6818 return malloc_usable_size_impl(ptr);
6819 }
6820 #endif
6821
6822 MOZ_JEMALLOC_API void
jemalloc_free_dirty_pages_impl(void)6823 jemalloc_free_dirty_pages_impl(void)
6824 {
6825 size_t i;
6826 for (i = 0; i < narenas; i++) {
6827 arena_t *arena = arenas[i];
6828
6829 if (arena != NULL) {
6830 malloc_spin_lock(&arena->lock);
6831 arena_purge(arena, true);
6832 malloc_spin_unlock(&arena->lock);
6833 }
6834 }
6835 }
6836
6837 /*
6838 * End non-standard functions.
6839 */
6840 /******************************************************************************/
6841 /*
6842 * Begin library-private functions, used by threading libraries for protection
6843 * of malloc during fork(). These functions are only called if the program is
6844 * running in threaded mode, so there is no need to check whether the program
6845 * is threaded here.
6846 */
6847
6848 static void
_malloc_prefork(void)6849 _malloc_prefork(void)
6850 {
6851 unsigned i;
6852
6853 /* Acquire all mutexes in a safe order. */
6854
6855 malloc_spin_lock(&arenas_lock);
6856 for (i = 0; i < narenas; i++) {
6857 if (arenas[i] != NULL)
6858 malloc_spin_lock(&arenas[i]->lock);
6859 }
6860
6861 malloc_mutex_lock(&base_mtx);
6862
6863 malloc_mutex_lock(&huge_mtx);
6864 }
6865
6866 static void
_malloc_postfork(void)6867 _malloc_postfork(void)
6868 {
6869 unsigned i;
6870
6871 /* Release all mutexes, now that fork() has completed. */
6872
6873 malloc_mutex_unlock(&huge_mtx);
6874
6875 malloc_mutex_unlock(&base_mtx);
6876
6877 for (i = 0; i < narenas; i++) {
6878 if (arenas[i] != NULL)
6879 malloc_spin_unlock(&arenas[i]->lock);
6880 }
6881 malloc_spin_unlock(&arenas_lock);
6882 }
6883
6884 /*
6885 * End library-private functions.
6886 */
6887 /******************************************************************************/
6888
6889 #ifdef HAVE_DLOPEN
6890 # include <dlfcn.h>
6891 #endif
6892
6893 #if defined(MOZ_MEMORY_DARWIN)
6894
6895 #if !defined(MOZ_REPLACE_MALLOC)
6896 static void *
zone_malloc(malloc_zone_t * zone,size_t size)6897 zone_malloc(malloc_zone_t *zone, size_t size)
6898 {
6899
6900 return (malloc_impl(size));
6901 }
6902
6903 static void *
zone_calloc(malloc_zone_t * zone,size_t num,size_t size)6904 zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
6905 {
6906
6907 return (calloc_impl(num, size));
6908 }
6909
6910 static void *
zone_valloc(malloc_zone_t * zone,size_t size)6911 zone_valloc(malloc_zone_t *zone, size_t size)
6912 {
6913 void *ret = NULL; /* Assignment avoids useless compiler warning. */
6914
6915 posix_memalign_impl(&ret, pagesize, size);
6916
6917 return (ret);
6918 }
6919
6920 static void *
zone_memalign(malloc_zone_t * zone,size_t alignment,size_t size)6921 zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
6922 {
6923 return (memalign_impl(alignment, size));
6924 }
6925
6926 static void *
zone_destroy(malloc_zone_t * zone)6927 zone_destroy(malloc_zone_t *zone)
6928 {
6929
6930 /* This function should never be called. */
6931 assert(false);
6932 return (NULL);
6933 }
6934
6935 static size_t
zone_good_size(malloc_zone_t * zone,size_t size)6936 zone_good_size(malloc_zone_t *zone, size_t size)
6937 {
6938 return malloc_good_size_impl(size);
6939 }
6940
6941 static size_t
ozone_size(malloc_zone_t * zone,void * ptr)6942 ozone_size(malloc_zone_t *zone, void *ptr)
6943 {
6944 size_t ret = isalloc_validate(ptr);
6945 if (ret == 0)
6946 ret = szone->size(zone, ptr);
6947
6948 return ret;
6949 }
6950
6951 static void
ozone_free(malloc_zone_t * zone,void * ptr)6952 ozone_free(malloc_zone_t *zone, void *ptr)
6953 {
6954 if (isalloc_validate(ptr) != 0)
6955 free_impl(ptr);
6956 else {
6957 size_t size = szone->size(zone, ptr);
6958 if (size != 0)
6959 (szone->free)(zone, ptr);
6960 /* Otherwise we leak. */
6961 }
6962 }
6963
6964 static void *
ozone_realloc(malloc_zone_t * zone,void * ptr,size_t size)6965 ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
6966 {
6967 size_t oldsize;
6968 if (ptr == NULL)
6969 return (malloc_impl(size));
6970
6971 oldsize = isalloc_validate(ptr);
6972 if (oldsize != 0)
6973 return (realloc_impl(ptr, size));
6974 else {
6975 oldsize = szone->size(zone, ptr);
6976 if (oldsize == 0)
6977 return (malloc_impl(size));
6978 else {
6979 void *ret = malloc_impl(size);
6980 if (ret != NULL) {
6981 memcpy(ret, ptr, (oldsize < size) ? oldsize :
6982 size);
6983 (szone->free)(zone, ptr);
6984 }
6985 return (ret);
6986 }
6987 }
6988 }
6989
6990 static unsigned
ozone_batch_malloc(malloc_zone_t * zone,size_t size,void ** results,unsigned num_requested)6991 ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
6992 unsigned num_requested)
6993 {
6994 /* Don't bother implementing this interface, since it isn't required. */
6995 return 0;
6996 }
6997
6998 static void
ozone_batch_free(malloc_zone_t * zone,void ** to_be_freed,unsigned num)6999 ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
7000 {
7001 unsigned i;
7002
7003 for (i = 0; i < num; i++)
7004 ozone_free(zone, to_be_freed[i]);
7005 }
7006
7007 static void
ozone_free_definite_size(malloc_zone_t * zone,void * ptr,size_t size)7008 ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
7009 {
7010 if (isalloc_validate(ptr) != 0) {
7011 assert(isalloc_validate(ptr) == size);
7012 free_impl(ptr);
7013 } else {
7014 assert(size == szone->size(zone, ptr));
7015 l_szone.m16(zone, ptr, size);
7016 }
7017 }
7018
7019 static void
ozone_force_lock(malloc_zone_t * zone)7020 ozone_force_lock(malloc_zone_t *zone)
7021 {
7022 _malloc_prefork();
7023 szone->introspect->force_lock(zone);
7024 }
7025
7026 static void
ozone_force_unlock(malloc_zone_t * zone)7027 ozone_force_unlock(malloc_zone_t *zone)
7028 {
7029 szone->introspect->force_unlock(zone);
7030 _malloc_postfork();
7031 }
7032
7033 static size_t
zone_version_size(int version)7034 zone_version_size(int version)
7035 {
7036 switch (version)
7037 {
7038 case SNOW_LEOPARD_MALLOC_ZONE_T_VERSION:
7039 return sizeof(snow_leopard_malloc_zone);
7040 case LEOPARD_MALLOC_ZONE_T_VERSION:
7041 return sizeof(leopard_malloc_zone);
7042 default:
7043 case LION_MALLOC_ZONE_T_VERSION:
7044 return sizeof(lion_malloc_zone);
7045 }
7046 }
7047
7048 /*
7049 * Overlay the default scalable zone (szone) such that existing allocations are
7050 * drained, and further allocations come from jemalloc. This is necessary
7051 * because Core Foundation directly accesses and uses the szone before the
7052 * jemalloc library is even loaded.
7053 */
7054 static void
szone2ozone(malloc_zone_t * default_zone,size_t size)7055 szone2ozone(malloc_zone_t *default_zone, size_t size)
7056 {
7057 lion_malloc_zone *l_zone;
7058 assert(malloc_initialized);
7059
7060 /*
7061 * Stash a copy of the original szone so that we can call its
7062 * functions as needed. Note that internally, the szone stores its
7063 * bookkeeping data structures immediately following the malloc_zone_t
7064 * header, so when calling szone functions, we need to pass a pointer to
7065 * the original zone structure.
7066 */
7067 memcpy(szone, default_zone, size);
7068
7069 /* OSX 10.7 allocates the default zone in protected memory. */
7070 if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
7071 void* start_of_page = (void*)((size_t)(default_zone) & ~pagesize_mask);
7072 mprotect (start_of_page, size, PROT_READ | PROT_WRITE);
7073 }
7074
7075 default_zone->size = (void *)ozone_size;
7076 default_zone->malloc = (void *)zone_malloc;
7077 default_zone->calloc = (void *)zone_calloc;
7078 default_zone->valloc = (void *)zone_valloc;
7079 default_zone->free = (void *)ozone_free;
7080 default_zone->realloc = (void *)ozone_realloc;
7081 default_zone->destroy = (void *)zone_destroy;
7082 default_zone->batch_malloc = NULL;
7083 default_zone->batch_free = ozone_batch_free;
7084 default_zone->introspect = ozone_introspect;
7085
7086 /* Don't modify default_zone->zone_name; Mac libc may rely on the name
7087 * being unchanged. See Mozilla bug 694896. */
7088
7089 ozone_introspect->enumerator = NULL;
7090 ozone_introspect->good_size = (void *)zone_good_size;
7091 ozone_introspect->check = NULL;
7092 ozone_introspect->print = NULL;
7093 ozone_introspect->log = NULL;
7094 ozone_introspect->force_lock = (void *)ozone_force_lock;
7095 ozone_introspect->force_unlock = (void *)ozone_force_unlock;
7096 ozone_introspect->statistics = NULL;
7097
7098 /* Platform-dependent structs */
7099 l_zone = (lion_malloc_zone*)(default_zone);
7100
7101 if (default_zone->version >= SNOW_LEOPARD_MALLOC_ZONE_T_VERSION) {
7102 l_zone->m15 = (void (*)())zone_memalign;
7103 l_zone->m16 = (void (*)())ozone_free_definite_size;
7104 l_ozone_introspect.m9 = NULL;
7105 }
7106
7107 if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
7108 l_zone->m17 = NULL;
7109 l_ozone_introspect.m10 = NULL;
7110 l_ozone_introspect.m11 = NULL;
7111 l_ozone_introspect.m12 = NULL;
7112 l_ozone_introspect.m13 = NULL;
7113 }
7114 }
7115 #endif
7116
7117 __attribute__((constructor))
7118 void
jemalloc_darwin_init(void)7119 jemalloc_darwin_init(void)
7120 {
7121 if (malloc_init_hard())
7122 abort();
7123 }
7124
7125 #endif
7126
7127 /*
7128 * is_malloc(malloc_impl) is some macro magic to detect if malloc_impl is
7129 * defined as "malloc" in mozmemory_wrap.h
7130 */
7131 #define malloc_is_malloc 1
7132 #define is_malloc_(a) malloc_is_ ## a
7133 #define is_malloc(a) is_malloc_(a)
7134
7135 #if !defined(MOZ_MEMORY_DARWIN) && (is_malloc(malloc_impl) == 1)
7136 # if defined(__GLIBC__) && !defined(__UCLIBC__)
7137 /*
7138 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
7139 * to inconsistently reference libc's malloc(3)-compatible functions
7140 * (bug 493541).
7141 *
7142 * These definitions interpose hooks in glibc. The functions are actually
7143 * passed an extra argument for the caller return address, which will be
7144 * ignored.
7145 */
7146 MOZ_MEMORY_API void (*__free_hook)(void *ptr) = free_impl;
7147 MOZ_MEMORY_API void *(*__malloc_hook)(size_t size) = malloc_impl;
7148 MOZ_MEMORY_API void *(*__realloc_hook)(void *ptr, size_t size) = realloc_impl;
7149 MOZ_MEMORY_API void *(*__memalign_hook)(size_t alignment, size_t size) = MEMALIGN;
7150
7151 # elif defined(RTLD_DEEPBIND)
7152 /*
7153 * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
7154 * implementations permit similar inconsistencies? Should STV_SINGLETON
7155 * visibility be used for interposition where available?
7156 */
7157 # error "Interposing malloc is unsafe on this system without libc malloc hooks."
7158 # endif
7159 #endif
7160
7161 #ifdef MOZ_MEMORY_WINDOWS
7162 /*
7163 * In the new style jemalloc integration jemalloc is built as a separate
7164 * shared library. Since we're no longer hooking into the CRT binary,
7165 * we need to initialize the heap at the first opportunity we get.
7166 * DLL_PROCESS_ATTACH in DllMain is that opportunity.
7167 */
DllMain(HINSTANCE hModule,DWORD reason,LPVOID lpReserved)7168 BOOL APIENTRY DllMain(HINSTANCE hModule,
7169 DWORD reason,
7170 LPVOID lpReserved)
7171 {
7172 switch (reason) {
7173 case DLL_PROCESS_ATTACH:
7174 /* Don't force the system to page DllMain back in every time
7175 * we create/destroy a thread */
7176 DisableThreadLibraryCalls(hModule);
7177 /* Initialize the heap */
7178 malloc_init_hard();
7179 break;
7180
7181 case DLL_PROCESS_DETACH:
7182 break;
7183
7184 }
7185
7186 return TRUE;
7187 }
7188 #endif
7189