1 #include "test/jemalloc_test.h"
2
3 #include "jemalloc/internal/spin.h"
4
5 static unsigned arena_ind;
6 static size_t sz;
7 static size_t esz;
8 #define NEPOCHS 8
9 #define PER_THD_NALLOCS 1
10 static atomic_u_t epoch;
11 static atomic_u_t nfinished;
12
13 static unsigned
do_arena_create(extent_hooks_t * h)14 do_arena_create(extent_hooks_t *h) {
15 unsigned arena_ind;
16 size_t sz = sizeof(unsigned);
17 assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz,
18 (void *)(h != NULL ? &h : NULL), (h != NULL ? sizeof(h) : 0)), 0,
19 "Unexpected mallctl() failure");
20 return arena_ind;
21 }
22
23 static void
do_arena_destroy(unsigned arena_ind)24 do_arena_destroy(unsigned arena_ind) {
25 size_t mib[3];
26 size_t miblen;
27
28 miblen = sizeof(mib)/sizeof(size_t);
29 assert_d_eq(mallctlnametomib("arena.0.destroy", mib, &miblen), 0,
30 "Unexpected mallctlnametomib() failure");
31 mib[1] = (size_t)arena_ind;
32 assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
33 "Unexpected mallctlbymib() failure");
34 }
35
36 static void
do_refresh(void)37 do_refresh(void) {
38 uint64_t epoch = 1;
39 assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch,
40 sizeof(epoch)), 0, "Unexpected mallctl() failure");
41 }
42
43 static size_t
do_get_size_impl(const char * cmd,unsigned arena_ind)44 do_get_size_impl(const char *cmd, unsigned arena_ind) {
45 size_t mib[4];
46 size_t miblen = sizeof(mib) / sizeof(size_t);
47 size_t z = sizeof(size_t);
48
49 assert_d_eq(mallctlnametomib(cmd, mib, &miblen),
50 0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd);
51 mib[2] = arena_ind;
52 size_t size;
53 assert_d_eq(mallctlbymib(mib, miblen, (void *)&size, &z, NULL, 0),
54 0, "Unexpected mallctlbymib([\"%s\"], ...) failure", cmd);
55
56 return size;
57 }
58
59 static size_t
do_get_active(unsigned arena_ind)60 do_get_active(unsigned arena_ind) {
61 return do_get_size_impl("stats.arenas.0.pactive", arena_ind) * PAGE;
62 }
63
64 static size_t
do_get_mapped(unsigned arena_ind)65 do_get_mapped(unsigned arena_ind) {
66 return do_get_size_impl("stats.arenas.0.mapped", arena_ind);
67 }
68
69 static void *
thd_start(void * arg)70 thd_start(void *arg) {
71 for (unsigned next_epoch = 1; next_epoch < NEPOCHS; next_epoch++) {
72 /* Busy-wait for next epoch. */
73 unsigned cur_epoch;
74 spin_t spinner = SPIN_INITIALIZER;
75 while ((cur_epoch = atomic_load_u(&epoch, ATOMIC_ACQUIRE)) !=
76 next_epoch) {
77 spin_adaptive(&spinner);
78 }
79 assert_u_eq(cur_epoch, next_epoch, "Unexpected epoch");
80
81 /*
82 * Allocate. The main thread will reset the arena, so there's
83 * no need to deallocate.
84 */
85 for (unsigned i = 0; i < PER_THD_NALLOCS; i++) {
86 void *p = mallocx(sz, MALLOCX_ARENA(arena_ind) |
87 MALLOCX_TCACHE_NONE
88 );
89 assert_ptr_not_null(p,
90 "Unexpected mallocx() failure\n");
91 }
92
93 /* Let the main thread know we've finished this iteration. */
94 atomic_fetch_add_u(&nfinished, 1, ATOMIC_RELEASE);
95 }
96
97 return NULL;
98 }
99
TEST_BEGIN(test_retained)100 TEST_BEGIN(test_retained) {
101 test_skip_if(!config_stats);
102
103 arena_ind = do_arena_create(NULL);
104 sz = nallocx(HUGEPAGE, 0);
105 esz = sz + sz_large_pad;
106
107 atomic_store_u(&epoch, 0, ATOMIC_RELAXED);
108
109 unsigned nthreads = ncpus * 2;
110 if (LG_SIZEOF_PTR < 3 && nthreads > 16) {
111 nthreads = 16; /* 32-bit platform could run out of vaddr. */
112 }
113 VARIABLE_ARRAY(thd_t, threads, nthreads);
114 for (unsigned i = 0; i < nthreads; i++) {
115 thd_create(&threads[i], thd_start, NULL);
116 }
117
118 for (unsigned e = 1; e < NEPOCHS; e++) {
119 atomic_store_u(&nfinished, 0, ATOMIC_RELEASE);
120 atomic_store_u(&epoch, e, ATOMIC_RELEASE);
121
122 /* Wait for threads to finish allocating. */
123 spin_t spinner = SPIN_INITIALIZER;
124 while (atomic_load_u(&nfinished, ATOMIC_ACQUIRE) < nthreads) {
125 spin_adaptive(&spinner);
126 }
127
128 /*
129 * Assert that retained is no more than the sum of size classes
130 * that should have been used to satisfy the worker threads'
131 * requests, discounting per growth fragmentation.
132 */
133 do_refresh();
134
135 size_t allocated = esz * nthreads * PER_THD_NALLOCS;
136 size_t active = do_get_active(arena_ind);
137 assert_zu_le(allocated, active, "Unexpected active memory");
138 size_t mapped = do_get_mapped(arena_ind);
139 assert_zu_le(active, mapped, "Unexpected mapped memory");
140
141 arena_t *arena = arena_get(tsdn_fetch(), arena_ind, false);
142 size_t usable = 0;
143 size_t fragmented = 0;
144 for (pszind_t pind = sz_psz2ind(HUGEPAGE); pind <
145 arena->extent_grow_next; pind++) {
146 size_t psz = sz_pind2sz(pind);
147 size_t psz_fragmented = psz % esz;
148 size_t psz_usable = psz - psz_fragmented;
149 /*
150 * Only consider size classes that wouldn't be skipped.
151 */
152 if (psz_usable > 0) {
153 assert_zu_lt(usable, allocated,
154 "Excessive retained memory "
155 "(%#zx[+%#zx] > %#zx)", usable, psz_usable,
156 allocated);
157 fragmented += psz_fragmented;
158 usable += psz_usable;
159 }
160 }
161
162 /*
163 * Clean up arena. Destroying and recreating the arena
164 * is simpler that specifying extent hooks that deallocate
165 * (rather than retaining) during reset.
166 */
167 do_arena_destroy(arena_ind);
168 assert_u_eq(do_arena_create(NULL), arena_ind,
169 "Unexpected arena index");
170 }
171
172 for (unsigned i = 0; i < nthreads; i++) {
173 thd_join(threads[i], NULL);
174 }
175
176 do_arena_destroy(arena_ind);
177 }
178 TEST_END
179
180 int
main(void)181 main(void) {
182 return test(
183 test_retained);
184 }
185