1 #ifndef JEMALLOC_INTERNAL_ARENA_STATS_H
2 #define JEMALLOC_INTERNAL_ARENA_STATS_H
3
4 #include "jemalloc/internal/atomic.h"
5 #include "jemalloc/internal/mutex.h"
6 #include "jemalloc/internal/mutex_prof.h"
7 #include "jemalloc/internal/sc.h"
8
9 JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
10
11 /*
12 * In those architectures that support 64-bit atomics, we use atomic updates for
13 * our 64-bit values. Otherwise, we use a plain uint64_t and synchronize
14 * externally.
15 */
16 #ifdef JEMALLOC_ATOMIC_U64
17 typedef atomic_u64_t arena_stats_u64_t;
18 #else
19 /* Must hold the arena stats mutex while reading atomically. */
20 typedef uint64_t arena_stats_u64_t;
21 #endif
22
23 typedef struct arena_stats_large_s arena_stats_large_t;
24 struct arena_stats_large_s {
25 /*
26 * Total number of allocation/deallocation requests served directly by
27 * the arena.
28 */
29 arena_stats_u64_t nmalloc;
30 arena_stats_u64_t ndalloc;
31
32 /*
33 * Number of allocation requests that correspond to this size class.
34 * This includes requests served by tcache, though tcache only
35 * periodically merges into this counter.
36 */
37 arena_stats_u64_t nrequests; /* Partially derived. */
38 /*
39 * Number of tcache fills / flushes for large (similarly, periodically
40 * merged). Note that there is no large tcache batch-fill currently
41 * (i.e. only fill 1 at a time); however flush may be batched.
42 */
43 arena_stats_u64_t nfills; /* Partially derived. */
44 arena_stats_u64_t nflushes; /* Partially derived. */
45
46 /* Current number of allocations of this size class. */
47 size_t curlextents; /* Derived. */
48 };
49
50 typedef struct arena_stats_decay_s arena_stats_decay_t;
51 struct arena_stats_decay_s {
52 /* Total number of purge sweeps. */
53 arena_stats_u64_t npurge;
54 /* Total number of madvise calls made. */
55 arena_stats_u64_t nmadvise;
56 /* Total number of pages purged. */
57 arena_stats_u64_t purged;
58 };
59
60 typedef struct arena_stats_extents_s arena_stats_extents_t;
61 struct arena_stats_extents_s {
62 /*
63 * Stats for a given index in the range [0, SC_NPSIZES] in an extents_t.
64 * We track both bytes and # of extents: two extents in the same bucket
65 * may have different sizes if adjacent size classes differ by more than
66 * a page, so bytes cannot always be derived from # of extents.
67 */
68 atomic_zu_t ndirty;
69 atomic_zu_t dirty_bytes;
70 atomic_zu_t nmuzzy;
71 atomic_zu_t muzzy_bytes;
72 atomic_zu_t nretained;
73 atomic_zu_t retained_bytes;
74 };
75
76 /*
77 * Arena stats. Note that fields marked "derived" are not directly maintained
78 * within the arena code; rather their values are derived during stats merge
79 * requests.
80 */
81 typedef struct arena_stats_s arena_stats_t;
82 struct arena_stats_s {
83 #ifndef JEMALLOC_ATOMIC_U64
84 malloc_mutex_t mtx;
85 #endif
86
87 /* Number of bytes currently mapped, excluding retained memory. */
88 atomic_zu_t mapped; /* Partially derived. */
89
90 /*
91 * Number of unused virtual memory bytes currently retained. Retained
92 * bytes are technically mapped (though always decommitted or purged),
93 * but they are excluded from the mapped statistic (above).
94 */
95 atomic_zu_t retained; /* Derived. */
96
97 /* Number of extent_t structs allocated by base, but not being used. */
98 atomic_zu_t extent_avail;
99
100 arena_stats_decay_t decay_dirty;
101 arena_stats_decay_t decay_muzzy;
102
103 atomic_zu_t base; /* Derived. */
104 atomic_zu_t internal;
105 atomic_zu_t resident; /* Derived. */
106 atomic_zu_t metadata_thp;
107
108 atomic_zu_t allocated_large; /* Derived. */
109 arena_stats_u64_t nmalloc_large; /* Derived. */
110 arena_stats_u64_t ndalloc_large; /* Derived. */
111 arena_stats_u64_t nfills_large; /* Derived. */
112 arena_stats_u64_t nflushes_large; /* Derived. */
113 arena_stats_u64_t nrequests_large; /* Derived. */
114
115 /* VM space had to be leaked (undocumented). Normally 0. */
116 atomic_zu_t abandoned_vm;
117
118 /* Number of bytes cached in tcache associated with this arena. */
119 atomic_zu_t tcache_bytes; /* Derived. */
120
121 mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes];
122
123 /* One element for each large size class. */
124 arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
125
126 /* Arena uptime. */
127 nstime_t uptime;
128 };
129
130 static inline bool
arena_stats_init(tsdn_t * tsdn,arena_stats_t * arena_stats)131 arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) {
132 if (config_debug) {
133 for (size_t i = 0; i < sizeof(arena_stats_t); i++) {
134 assert(((char *)arena_stats)[i] == 0);
135 }
136 }
137 #ifndef JEMALLOC_ATOMIC_U64
138 if (malloc_mutex_init(&arena_stats->mtx, "arena_stats",
139 WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) {
140 return true;
141 }
142 #endif
143 /* Memory is zeroed, so there is no need to clear stats. */
144 return false;
145 }
146
147 static inline void
arena_stats_lock(tsdn_t * tsdn,arena_stats_t * arena_stats)148 arena_stats_lock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
149 #ifndef JEMALLOC_ATOMIC_U64
150 malloc_mutex_lock(tsdn, &arena_stats->mtx);
151 #endif
152 }
153
154 static inline void
arena_stats_unlock(tsdn_t * tsdn,arena_stats_t * arena_stats)155 arena_stats_unlock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
156 #ifndef JEMALLOC_ATOMIC_U64
157 malloc_mutex_unlock(tsdn, &arena_stats->mtx);
158 #endif
159 }
160
161 static inline uint64_t
arena_stats_read_u64(tsdn_t * tsdn,arena_stats_t * arena_stats,arena_stats_u64_t * p)162 arena_stats_read_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
163 arena_stats_u64_t *p) {
164 #ifdef JEMALLOC_ATOMIC_U64
165 return atomic_load_u64(p, ATOMIC_RELAXED);
166 #else
167 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
168 return *p;
169 #endif
170 }
171
172 static inline void
arena_stats_add_u64(tsdn_t * tsdn,arena_stats_t * arena_stats,arena_stats_u64_t * p,uint64_t x)173 arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
174 arena_stats_u64_t *p, uint64_t x) {
175 #ifdef JEMALLOC_ATOMIC_U64
176 atomic_fetch_add_u64(p, x, ATOMIC_RELAXED);
177 #else
178 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
179 *p += x;
180 #endif
181 }
182
183 static inline void
arena_stats_sub_u64(tsdn_t * tsdn,arena_stats_t * arena_stats,arena_stats_u64_t * p,uint64_t x)184 arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
185 arena_stats_u64_t *p, uint64_t x) {
186 #ifdef JEMALLOC_ATOMIC_U64
187 uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED);
188 assert(r - x <= r);
189 #else
190 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
191 *p -= x;
192 assert(*p + x >= *p);
193 #endif
194 }
195
196 /*
197 * Non-atomically sets *dst += src. *dst needs external synchronization.
198 * This lets us avoid the cost of a fetch_add when its unnecessary (note that
199 * the types here are atomic).
200 */
201 static inline void
arena_stats_accum_u64(arena_stats_u64_t * dst,uint64_t src)202 arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) {
203 #ifdef JEMALLOC_ATOMIC_U64
204 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED);
205 atomic_store_u64(dst, src + cur_dst, ATOMIC_RELAXED);
206 #else
207 *dst += src;
208 #endif
209 }
210
211 static inline size_t
arena_stats_read_zu(tsdn_t * tsdn,arena_stats_t * arena_stats,atomic_zu_t * p)212 arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
213 atomic_zu_t *p) {
214 #ifdef JEMALLOC_ATOMIC_U64
215 return atomic_load_zu(p, ATOMIC_RELAXED);
216 #else
217 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
218 return atomic_load_zu(p, ATOMIC_RELAXED);
219 #endif
220 }
221
222 static inline void
arena_stats_add_zu(tsdn_t * tsdn,arena_stats_t * arena_stats,atomic_zu_t * p,size_t x)223 arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
224 atomic_zu_t *p, size_t x) {
225 #ifdef JEMALLOC_ATOMIC_U64
226 atomic_fetch_add_zu(p, x, ATOMIC_RELAXED);
227 #else
228 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
229 size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
230 atomic_store_zu(p, cur + x, ATOMIC_RELAXED);
231 #endif
232 }
233
234 static inline void
arena_stats_sub_zu(tsdn_t * tsdn,arena_stats_t * arena_stats,atomic_zu_t * p,size_t x)235 arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
236 atomic_zu_t *p, size_t x) {
237 #ifdef JEMALLOC_ATOMIC_U64
238 size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED);
239 assert(r - x <= r);
240 #else
241 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
242 size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
243 atomic_store_zu(p, cur - x, ATOMIC_RELAXED);
244 #endif
245 }
246
247 /* Like the _u64 variant, needs an externally synchronized *dst. */
248 static inline void
arena_stats_accum_zu(atomic_zu_t * dst,size_t src)249 arena_stats_accum_zu(atomic_zu_t *dst, size_t src) {
250 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);
251 atomic_store_zu(dst, src + cur_dst, ATOMIC_RELAXED);
252 }
253
254 static inline void
arena_stats_large_flush_nrequests_add(tsdn_t * tsdn,arena_stats_t * arena_stats,szind_t szind,uint64_t nrequests)255 arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
256 szind_t szind, uint64_t nrequests) {
257 arena_stats_lock(tsdn, arena_stats);
258 arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS];
259 arena_stats_add_u64(tsdn, arena_stats, &lstats->nrequests, nrequests);
260 arena_stats_add_u64(tsdn, arena_stats, &lstats->nflushes, 1);
261 arena_stats_unlock(tsdn, arena_stats);
262 }
263
264 static inline void
arena_stats_mapped_add(tsdn_t * tsdn,arena_stats_t * arena_stats,size_t size)265 arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) {
266 arena_stats_lock(tsdn, arena_stats);
267 arena_stats_add_zu(tsdn, arena_stats, &arena_stats->mapped, size);
268 arena_stats_unlock(tsdn, arena_stats);
269 }
270
271 #endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */
272