1 #define JEMALLOC_CTL_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/assert.h" 6 #include "jemalloc/internal/ctl.h" 7 #include "jemalloc/internal/extent_dss.h" 8 #include "jemalloc/internal/extent_mmap.h" 9 #include "jemalloc/internal/mutex.h" 10 #include "jemalloc/internal/nstime.h" 11 #include "jemalloc/internal/size_classes.h" 12 #include "jemalloc/internal/util.h" 13 14 /******************************************************************************/ 15 /* Data. */ 16 17 /* 18 * ctl_mtx protects the following: 19 * - ctl_stats->* 20 */ 21 static malloc_mutex_t ctl_mtx; 22 static bool ctl_initialized; 23 static ctl_stats_t *ctl_stats; 24 static ctl_arenas_t *ctl_arenas; 25 26 /******************************************************************************/ 27 /* Helpers for named and indexed nodes. */ 28 29 static const ctl_named_node_t * 30 ctl_named_node(const ctl_node_t *node) { 31 return ((node->named) ? (const ctl_named_node_t *)node : NULL); 32 } 33 34 static const ctl_named_node_t * 35 ctl_named_children(const ctl_named_node_t *node, size_t index) { 36 const ctl_named_node_t *children = ctl_named_node(node->children); 37 38 return (children ? &children[index] : NULL); 39 } 40 41 static const ctl_indexed_node_t * 42 ctl_indexed_node(const ctl_node_t *node) { 43 return (!node->named ? (const ctl_indexed_node_t *)node : NULL); 44 } 45 46 /******************************************************************************/ 47 /* Function prototypes for non-inline static functions. */ 48 49 #define CTL_PROTO(n) \ 50 static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 51 void *oldp, size_t *oldlenp, void *newp, size_t newlen); 52 53 #define INDEX_PROTO(n) \ 54 static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \ 55 const size_t *mib, size_t miblen, size_t i); 56 57 CTL_PROTO(version) 58 CTL_PROTO(epoch) 59 CTL_PROTO(background_thread) 60 CTL_PROTO(thread_tcache_enabled) 61 CTL_PROTO(thread_tcache_flush) 62 CTL_PROTO(thread_prof_name) 63 CTL_PROTO(thread_prof_active) 64 CTL_PROTO(thread_arena) 65 CTL_PROTO(thread_allocated) 66 CTL_PROTO(thread_allocatedp) 67 CTL_PROTO(thread_deallocated) 68 CTL_PROTO(thread_deallocatedp) 69 CTL_PROTO(config_cache_oblivious) 70 CTL_PROTO(config_debug) 71 CTL_PROTO(config_fill) 72 CTL_PROTO(config_lazy_lock) 73 CTL_PROTO(config_malloc_conf) 74 CTL_PROTO(config_prof) 75 CTL_PROTO(config_prof_libgcc) 76 CTL_PROTO(config_prof_libunwind) 77 CTL_PROTO(config_stats) 78 CTL_PROTO(config_thp) 79 CTL_PROTO(config_utrace) 80 CTL_PROTO(config_xmalloc) 81 CTL_PROTO(opt_abort) 82 CTL_PROTO(opt_abort_conf) 83 CTL_PROTO(opt_retain) 84 CTL_PROTO(opt_dss) 85 CTL_PROTO(opt_narenas) 86 CTL_PROTO(opt_percpu_arena) 87 CTL_PROTO(opt_background_thread) 88 CTL_PROTO(opt_dirty_decay_ms) 89 CTL_PROTO(opt_muzzy_decay_ms) 90 CTL_PROTO(opt_stats_print) 91 CTL_PROTO(opt_stats_print_opts) 92 CTL_PROTO(opt_junk) 93 CTL_PROTO(opt_zero) 94 CTL_PROTO(opt_utrace) 95 CTL_PROTO(opt_xmalloc) 96 CTL_PROTO(opt_tcache) 97 CTL_PROTO(opt_lg_tcache_max) 98 CTL_PROTO(opt_prof) 99 CTL_PROTO(opt_prof_prefix) 100 CTL_PROTO(opt_prof_active) 101 CTL_PROTO(opt_prof_thread_active_init) 102 CTL_PROTO(opt_lg_prof_sample) 103 CTL_PROTO(opt_lg_prof_interval) 104 CTL_PROTO(opt_prof_gdump) 105 CTL_PROTO(opt_prof_final) 106 CTL_PROTO(opt_prof_leak) 107 CTL_PROTO(opt_prof_accum) 108 CTL_PROTO(tcache_create) 109 CTL_PROTO(tcache_flush) 110 CTL_PROTO(tcache_destroy) 111 CTL_PROTO(arena_i_initialized) 112 CTL_PROTO(arena_i_decay) 113 CTL_PROTO(arena_i_purge) 114 CTL_PROTO(arena_i_reset) 115 CTL_PROTO(arena_i_destroy) 116 CTL_PROTO(arena_i_dss) 117 CTL_PROTO(arena_i_dirty_decay_ms) 118 CTL_PROTO(arena_i_muzzy_decay_ms) 119 CTL_PROTO(arena_i_extent_hooks) 120 INDEX_PROTO(arena_i) 121 CTL_PROTO(arenas_bin_i_size) 122 CTL_PROTO(arenas_bin_i_nregs) 123 CTL_PROTO(arenas_bin_i_slab_size) 124 INDEX_PROTO(arenas_bin_i) 125 CTL_PROTO(arenas_lextent_i_size) 126 INDEX_PROTO(arenas_lextent_i) 127 CTL_PROTO(arenas_narenas) 128 CTL_PROTO(arenas_dirty_decay_ms) 129 CTL_PROTO(arenas_muzzy_decay_ms) 130 CTL_PROTO(arenas_quantum) 131 CTL_PROTO(arenas_page) 132 CTL_PROTO(arenas_tcache_max) 133 CTL_PROTO(arenas_nbins) 134 CTL_PROTO(arenas_nhbins) 135 CTL_PROTO(arenas_nlextents) 136 CTL_PROTO(arenas_create) 137 CTL_PROTO(prof_thread_active_init) 138 CTL_PROTO(prof_active) 139 CTL_PROTO(prof_dump) 140 CTL_PROTO(prof_gdump) 141 CTL_PROTO(prof_reset) 142 CTL_PROTO(prof_interval) 143 CTL_PROTO(lg_prof_sample) 144 CTL_PROTO(stats_arenas_i_small_allocated) 145 CTL_PROTO(stats_arenas_i_small_nmalloc) 146 CTL_PROTO(stats_arenas_i_small_ndalloc) 147 CTL_PROTO(stats_arenas_i_small_nrequests) 148 CTL_PROTO(stats_arenas_i_large_allocated) 149 CTL_PROTO(stats_arenas_i_large_nmalloc) 150 CTL_PROTO(stats_arenas_i_large_ndalloc) 151 CTL_PROTO(stats_arenas_i_large_nrequests) 152 CTL_PROTO(stats_arenas_i_bins_j_nmalloc) 153 CTL_PROTO(stats_arenas_i_bins_j_ndalloc) 154 CTL_PROTO(stats_arenas_i_bins_j_nrequests) 155 CTL_PROTO(stats_arenas_i_bins_j_curregs) 156 CTL_PROTO(stats_arenas_i_bins_j_nfills) 157 CTL_PROTO(stats_arenas_i_bins_j_nflushes) 158 CTL_PROTO(stats_arenas_i_bins_j_nslabs) 159 CTL_PROTO(stats_arenas_i_bins_j_nreslabs) 160 CTL_PROTO(stats_arenas_i_bins_j_curslabs) 161 INDEX_PROTO(stats_arenas_i_bins_j) 162 CTL_PROTO(stats_arenas_i_lextents_j_nmalloc) 163 CTL_PROTO(stats_arenas_i_lextents_j_ndalloc) 164 CTL_PROTO(stats_arenas_i_lextents_j_nrequests) 165 CTL_PROTO(stats_arenas_i_lextents_j_curlextents) 166 INDEX_PROTO(stats_arenas_i_lextents_j) 167 CTL_PROTO(stats_arenas_i_nthreads) 168 CTL_PROTO(stats_arenas_i_uptime) 169 CTL_PROTO(stats_arenas_i_dss) 170 CTL_PROTO(stats_arenas_i_dirty_decay_ms) 171 CTL_PROTO(stats_arenas_i_muzzy_decay_ms) 172 CTL_PROTO(stats_arenas_i_pactive) 173 CTL_PROTO(stats_arenas_i_pdirty) 174 CTL_PROTO(stats_arenas_i_pmuzzy) 175 CTL_PROTO(stats_arenas_i_mapped) 176 CTL_PROTO(stats_arenas_i_retained) 177 CTL_PROTO(stats_arenas_i_dirty_npurge) 178 CTL_PROTO(stats_arenas_i_dirty_nmadvise) 179 CTL_PROTO(stats_arenas_i_dirty_purged) 180 CTL_PROTO(stats_arenas_i_muzzy_npurge) 181 CTL_PROTO(stats_arenas_i_muzzy_nmadvise) 182 CTL_PROTO(stats_arenas_i_muzzy_purged) 183 CTL_PROTO(stats_arenas_i_base) 184 CTL_PROTO(stats_arenas_i_internal) 185 CTL_PROTO(stats_arenas_i_tcache_bytes) 186 CTL_PROTO(stats_arenas_i_resident) 187 INDEX_PROTO(stats_arenas_i) 188 CTL_PROTO(stats_allocated) 189 CTL_PROTO(stats_active) 190 CTL_PROTO(stats_background_thread_num_threads) 191 CTL_PROTO(stats_background_thread_num_runs) 192 CTL_PROTO(stats_background_thread_run_interval) 193 CTL_PROTO(stats_metadata) 194 CTL_PROTO(stats_resident) 195 CTL_PROTO(stats_mapped) 196 CTL_PROTO(stats_retained) 197 198 #define MUTEX_STATS_CTL_PROTO_GEN(n) \ 199 CTL_PROTO(stats_##n##_num_ops) \ 200 CTL_PROTO(stats_##n##_num_wait) \ 201 CTL_PROTO(stats_##n##_num_spin_acq) \ 202 CTL_PROTO(stats_##n##_num_owner_switch) \ 203 CTL_PROTO(stats_##n##_total_wait_time) \ 204 CTL_PROTO(stats_##n##_max_wait_time) \ 205 CTL_PROTO(stats_##n##_max_num_thds) 206 207 /* Global mutexes. */ 208 #define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx) 209 MUTEX_PROF_GLOBAL_MUTEXES 210 #undef OP 211 212 /* Per arena mutexes. */ 213 #define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx) 214 MUTEX_PROF_ARENA_MUTEXES 215 #undef OP 216 217 /* Arena bin mutexes. */ 218 MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex) 219 #undef MUTEX_STATS_CTL_PROTO_GEN 220 221 CTL_PROTO(stats_mutexes_reset) 222 223 /******************************************************************************/ 224 /* mallctl tree. */ 225 226 #define NAME(n) {true}, n 227 #define CHILD(t, c) \ 228 sizeof(c##_node) / sizeof(ctl_##t##_node_t), \ 229 (ctl_node_t *)c##_node, \ 230 NULL 231 #define CTL(c) 0, NULL, c##_ctl 232 233 /* 234 * Only handles internal indexed nodes, since there are currently no external 235 * ones. 236 */ 237 #define INDEX(i) {false}, i##_index 238 239 static const ctl_named_node_t thread_tcache_node[] = { 240 {NAME("enabled"), CTL(thread_tcache_enabled)}, 241 {NAME("flush"), CTL(thread_tcache_flush)} 242 }; 243 244 static const ctl_named_node_t thread_prof_node[] = { 245 {NAME("name"), CTL(thread_prof_name)}, 246 {NAME("active"), CTL(thread_prof_active)} 247 }; 248 249 static const ctl_named_node_t thread_node[] = { 250 {NAME("arena"), CTL(thread_arena)}, 251 {NAME("allocated"), CTL(thread_allocated)}, 252 {NAME("allocatedp"), CTL(thread_allocatedp)}, 253 {NAME("deallocated"), CTL(thread_deallocated)}, 254 {NAME("deallocatedp"), CTL(thread_deallocatedp)}, 255 {NAME("tcache"), CHILD(named, thread_tcache)}, 256 {NAME("prof"), CHILD(named, thread_prof)} 257 }; 258 259 static const ctl_named_node_t config_node[] = { 260 {NAME("cache_oblivious"), CTL(config_cache_oblivious)}, 261 {NAME("debug"), CTL(config_debug)}, 262 {NAME("fill"), CTL(config_fill)}, 263 {NAME("lazy_lock"), CTL(config_lazy_lock)}, 264 {NAME("malloc_conf"), CTL(config_malloc_conf)}, 265 {NAME("prof"), CTL(config_prof)}, 266 {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, 267 {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, 268 {NAME("stats"), CTL(config_stats)}, 269 {NAME("thp"), CTL(config_thp)}, 270 {NAME("utrace"), CTL(config_utrace)}, 271 {NAME("xmalloc"), CTL(config_xmalloc)} 272 }; 273 274 static const ctl_named_node_t opt_node[] = { 275 {NAME("abort"), CTL(opt_abort)}, 276 {NAME("abort_conf"), CTL(opt_abort_conf)}, 277 {NAME("retain"), CTL(opt_retain)}, 278 {NAME("dss"), CTL(opt_dss)}, 279 {NAME("narenas"), CTL(opt_narenas)}, 280 {NAME("percpu_arena"), CTL(opt_percpu_arena)}, 281 {NAME("background_thread"), CTL(opt_background_thread)}, 282 {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, 283 {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)}, 284 {NAME("stats_print"), CTL(opt_stats_print)}, 285 {NAME("stats_print_opts"), CTL(opt_stats_print_opts)}, 286 {NAME("junk"), CTL(opt_junk)}, 287 {NAME("zero"), CTL(opt_zero)}, 288 {NAME("utrace"), CTL(opt_utrace)}, 289 {NAME("xmalloc"), CTL(opt_xmalloc)}, 290 {NAME("tcache"), CTL(opt_tcache)}, 291 {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, 292 {NAME("prof"), CTL(opt_prof)}, 293 {NAME("prof_prefix"), CTL(opt_prof_prefix)}, 294 {NAME("prof_active"), CTL(opt_prof_active)}, 295 {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, 296 {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, 297 {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, 298 {NAME("prof_gdump"), CTL(opt_prof_gdump)}, 299 {NAME("prof_final"), CTL(opt_prof_final)}, 300 {NAME("prof_leak"), CTL(opt_prof_leak)}, 301 {NAME("prof_accum"), CTL(opt_prof_accum)} 302 }; 303 304 static const ctl_named_node_t tcache_node[] = { 305 {NAME("create"), CTL(tcache_create)}, 306 {NAME("flush"), CTL(tcache_flush)}, 307 {NAME("destroy"), CTL(tcache_destroy)} 308 }; 309 310 static const ctl_named_node_t arena_i_node[] = { 311 {NAME("initialized"), CTL(arena_i_initialized)}, 312 {NAME("decay"), CTL(arena_i_decay)}, 313 {NAME("purge"), CTL(arena_i_purge)}, 314 {NAME("reset"), CTL(arena_i_reset)}, 315 {NAME("destroy"), CTL(arena_i_destroy)}, 316 {NAME("dss"), CTL(arena_i_dss)}, 317 {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, 318 {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, 319 {NAME("extent_hooks"), CTL(arena_i_extent_hooks)} 320 }; 321 static const ctl_named_node_t super_arena_i_node[] = { 322 {NAME(""), CHILD(named, arena_i)} 323 }; 324 325 static const ctl_indexed_node_t arena_node[] = { 326 {INDEX(arena_i)} 327 }; 328 329 static const ctl_named_node_t arenas_bin_i_node[] = { 330 {NAME("size"), CTL(arenas_bin_i_size)}, 331 {NAME("nregs"), CTL(arenas_bin_i_nregs)}, 332 {NAME("slab_size"), CTL(arenas_bin_i_slab_size)} 333 }; 334 static const ctl_named_node_t super_arenas_bin_i_node[] = { 335 {NAME(""), CHILD(named, arenas_bin_i)} 336 }; 337 338 static const ctl_indexed_node_t arenas_bin_node[] = { 339 {INDEX(arenas_bin_i)} 340 }; 341 342 static const ctl_named_node_t arenas_lextent_i_node[] = { 343 {NAME("size"), CTL(arenas_lextent_i_size)} 344 }; 345 static const ctl_named_node_t super_arenas_lextent_i_node[] = { 346 {NAME(""), CHILD(named, arenas_lextent_i)} 347 }; 348 349 static const ctl_indexed_node_t arenas_lextent_node[] = { 350 {INDEX(arenas_lextent_i)} 351 }; 352 353 static const ctl_named_node_t arenas_node[] = { 354 {NAME("narenas"), CTL(arenas_narenas)}, 355 {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)}, 356 {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)}, 357 {NAME("quantum"), CTL(arenas_quantum)}, 358 {NAME("page"), CTL(arenas_page)}, 359 {NAME("tcache_max"), CTL(arenas_tcache_max)}, 360 {NAME("nbins"), CTL(arenas_nbins)}, 361 {NAME("nhbins"), CTL(arenas_nhbins)}, 362 {NAME("bin"), CHILD(indexed, arenas_bin)}, 363 {NAME("nlextents"), CTL(arenas_nlextents)}, 364 {NAME("lextent"), CHILD(indexed, arenas_lextent)}, 365 {NAME("create"), CTL(arenas_create)} 366 }; 367 368 static const ctl_named_node_t prof_node[] = { 369 {NAME("thread_active_init"), CTL(prof_thread_active_init)}, 370 {NAME("active"), CTL(prof_active)}, 371 {NAME("dump"), CTL(prof_dump)}, 372 {NAME("gdump"), CTL(prof_gdump)}, 373 {NAME("reset"), CTL(prof_reset)}, 374 {NAME("interval"), CTL(prof_interval)}, 375 {NAME("lg_sample"), CTL(lg_prof_sample)} 376 }; 377 378 static const ctl_named_node_t stats_arenas_i_small_node[] = { 379 {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, 380 {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, 381 {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, 382 {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} 383 }; 384 385 static const ctl_named_node_t stats_arenas_i_large_node[] = { 386 {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, 387 {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, 388 {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, 389 {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} 390 }; 391 392 #define MUTEX_PROF_DATA_NODE(prefix) \ 393 static const ctl_named_node_t stats_##prefix##_node[] = { \ 394 {NAME("num_ops"), \ 395 CTL(stats_##prefix##_num_ops)}, \ 396 {NAME("num_wait"), \ 397 CTL(stats_##prefix##_num_wait)}, \ 398 {NAME("num_spin_acq"), \ 399 CTL(stats_##prefix##_num_spin_acq)}, \ 400 {NAME("num_owner_switch"), \ 401 CTL(stats_##prefix##_num_owner_switch)}, \ 402 {NAME("total_wait_time"), \ 403 CTL(stats_##prefix##_total_wait_time)}, \ 404 {NAME("max_wait_time"), \ 405 CTL(stats_##prefix##_max_wait_time)}, \ 406 {NAME("max_num_thds"), \ 407 CTL(stats_##prefix##_max_num_thds)} \ 408 /* Note that # of current waiting thread not provided. */ \ 409 }; 410 411 MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex) 412 413 static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { 414 {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, 415 {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, 416 {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, 417 {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, 418 {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, 419 {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, 420 {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, 421 {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, 422 {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, 423 {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)} 424 }; 425 426 static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { 427 {NAME(""), CHILD(named, stats_arenas_i_bins_j)} 428 }; 429 430 static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { 431 {INDEX(stats_arenas_i_bins_j)} 432 }; 433 434 static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = { 435 {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)}, 436 {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)}, 437 {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)}, 438 {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)} 439 }; 440 static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = { 441 {NAME(""), CHILD(named, stats_arenas_i_lextents_j)} 442 }; 443 444 static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { 445 {INDEX(stats_arenas_i_lextents_j)} 446 }; 447 448 #define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) 449 MUTEX_PROF_ARENA_MUTEXES 450 #undef OP 451 452 static const ctl_named_node_t stats_arenas_i_mutexes_node[] = { 453 #define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)}, 454 MUTEX_PROF_ARENA_MUTEXES 455 #undef OP 456 }; 457 458 static const ctl_named_node_t stats_arenas_i_node[] = { 459 {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, 460 {NAME("uptime"), CTL(stats_arenas_i_uptime)}, 461 {NAME("dss"), CTL(stats_arenas_i_dss)}, 462 {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)}, 463 {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)}, 464 {NAME("pactive"), CTL(stats_arenas_i_pactive)}, 465 {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, 466 {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, 467 {NAME("mapped"), CTL(stats_arenas_i_mapped)}, 468 {NAME("retained"), CTL(stats_arenas_i_retained)}, 469 {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, 470 {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, 471 {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, 472 {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)}, 473 {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)}, 474 {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)}, 475 {NAME("base"), CTL(stats_arenas_i_base)}, 476 {NAME("internal"), CTL(stats_arenas_i_internal)}, 477 {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, 478 {NAME("resident"), CTL(stats_arenas_i_resident)}, 479 {NAME("small"), CHILD(named, stats_arenas_i_small)}, 480 {NAME("large"), CHILD(named, stats_arenas_i_large)}, 481 {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, 482 {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, 483 {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} 484 }; 485 static const ctl_named_node_t super_stats_arenas_i_node[] = { 486 {NAME(""), CHILD(named, stats_arenas_i)} 487 }; 488 489 static const ctl_indexed_node_t stats_arenas_node[] = { 490 {INDEX(stats_arenas_i)} 491 }; 492 493 static const ctl_named_node_t stats_background_thread_node[] = { 494 {NAME("num_threads"), CTL(stats_background_thread_num_threads)}, 495 {NAME("num_runs"), CTL(stats_background_thread_num_runs)}, 496 {NAME("run_interval"), CTL(stats_background_thread_run_interval)} 497 }; 498 499 #define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx) 500 MUTEX_PROF_GLOBAL_MUTEXES 501 #undef OP 502 503 static const ctl_named_node_t stats_mutexes_node[] = { 504 #define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)}, 505 MUTEX_PROF_GLOBAL_MUTEXES 506 #undef OP 507 {NAME("reset"), CTL(stats_mutexes_reset)} 508 }; 509 #undef MUTEX_PROF_DATA_NODE 510 511 static const ctl_named_node_t stats_node[] = { 512 {NAME("allocated"), CTL(stats_allocated)}, 513 {NAME("active"), CTL(stats_active)}, 514 {NAME("metadata"), CTL(stats_metadata)}, 515 {NAME("resident"), CTL(stats_resident)}, 516 {NAME("mapped"), CTL(stats_mapped)}, 517 {NAME("retained"), CTL(stats_retained)}, 518 {NAME("background_thread"), 519 CHILD(named, stats_background_thread)}, 520 {NAME("mutexes"), CHILD(named, stats_mutexes)}, 521 {NAME("arenas"), CHILD(indexed, stats_arenas)} 522 }; 523 524 static const ctl_named_node_t root_node[] = { 525 {NAME("version"), CTL(version)}, 526 {NAME("epoch"), CTL(epoch)}, 527 {NAME("background_thread"), CTL(background_thread)}, 528 {NAME("thread"), CHILD(named, thread)}, 529 {NAME("config"), CHILD(named, config)}, 530 {NAME("opt"), CHILD(named, opt)}, 531 {NAME("tcache"), CHILD(named, tcache)}, 532 {NAME("arena"), CHILD(indexed, arena)}, 533 {NAME("arenas"), CHILD(named, arenas)}, 534 {NAME("prof"), CHILD(named, prof)}, 535 {NAME("stats"), CHILD(named, stats)} 536 }; 537 static const ctl_named_node_t super_root_node[] = { 538 {NAME(""), CHILD(named, root)} 539 }; 540 541 #undef NAME 542 #undef CHILD 543 #undef CTL 544 #undef INDEX 545 546 /******************************************************************************/ 547 548 /* 549 * Sets *dst + *src non-atomically. This is safe, since everything is 550 * synchronized by the ctl mutex. 551 */ 552 static void 553 accum_arena_stats_u64(arena_stats_u64_t *dst, arena_stats_u64_t *src) { 554 #ifdef JEMALLOC_ATOMIC_U64 555 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED); 556 uint64_t cur_src = atomic_load_u64(src, ATOMIC_RELAXED); 557 atomic_store_u64(dst, cur_dst + cur_src, ATOMIC_RELAXED); 558 #else 559 *dst += *src; 560 #endif 561 } 562 563 /* Likewise: with ctl mutex synchronization, reading is simple. */ 564 static uint64_t 565 arena_stats_read_u64(arena_stats_u64_t *p) { 566 #ifdef JEMALLOC_ATOMIC_U64 567 return atomic_load_u64(p, ATOMIC_RELAXED); 568 #else 569 return *p; 570 #endif 571 } 572 573 static void accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) { 574 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED); 575 size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED); 576 atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED); 577 } 578 579 /******************************************************************************/ 580 581 static unsigned 582 arenas_i2a_impl(size_t i, bool compat, bool validate) { 583 unsigned a; 584 585 switch (i) { 586 case MALLCTL_ARENAS_ALL: 587 a = 0; 588 break; 589 case MALLCTL_ARENAS_DESTROYED: 590 a = 1; 591 break; 592 default: 593 if (compat && i == ctl_arenas->narenas) { 594 /* 595 * Provide deprecated backward compatibility for 596 * accessing the merged stats at index narenas rather 597 * than via MALLCTL_ARENAS_ALL. This is scheduled for 598 * removal in 6.0.0. 599 */ 600 a = 0; 601 } else if (validate && i >= ctl_arenas->narenas) { 602 a = UINT_MAX; 603 } else { 604 /* 605 * This function should never be called for an index 606 * more than one past the range of indices that have 607 * initialized ctl data. 608 */ 609 assert(i < ctl_arenas->narenas || (!validate && i == 610 ctl_arenas->narenas)); 611 a = (unsigned)i + 2; 612 } 613 break; 614 } 615 616 return a; 617 } 618 619 static unsigned 620 arenas_i2a(size_t i) { 621 return arenas_i2a_impl(i, true, false); 622 } 623 624 static ctl_arena_t * 625 arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) { 626 ctl_arena_t *ret; 627 628 assert(!compat || !init); 629 630 ret = ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)]; 631 if (init && ret == NULL) { 632 if (config_stats) { 633 struct container_s { 634 ctl_arena_t ctl_arena; 635 ctl_arena_stats_t astats; 636 }; 637 struct container_s *cont = 638 (struct container_s *)base_alloc(tsd_tsdn(tsd), 639 b0get(), sizeof(struct container_s), QUANTUM); 640 if (cont == NULL) { 641 return NULL; 642 } 643 ret = &cont->ctl_arena; 644 ret->astats = &cont->astats; 645 } else { 646 ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(), 647 sizeof(ctl_arena_t), QUANTUM); 648 if (ret == NULL) { 649 return NULL; 650 } 651 } 652 ret->arena_ind = (unsigned)i; 653 ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)] = ret; 654 } 655 656 assert(ret == NULL || arenas_i2a(ret->arena_ind) == arenas_i2a(i)); 657 return ret; 658 } 659 660 static ctl_arena_t * 661 arenas_i(size_t i) { 662 ctl_arena_t *ret = arenas_i_impl(tsd_fetch(), i, true, false); 663 assert(ret != NULL); 664 return ret; 665 } 666 667 static void 668 ctl_arena_clear(ctl_arena_t *ctl_arena) { 669 ctl_arena->nthreads = 0; 670 ctl_arena->dss = dss_prec_names[dss_prec_limit]; 671 ctl_arena->dirty_decay_ms = -1; 672 ctl_arena->muzzy_decay_ms = -1; 673 ctl_arena->pactive = 0; 674 ctl_arena->pdirty = 0; 675 ctl_arena->pmuzzy = 0; 676 if (config_stats) { 677 memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t)); 678 ctl_arena->astats->allocated_small = 0; 679 ctl_arena->astats->nmalloc_small = 0; 680 ctl_arena->astats->ndalloc_small = 0; 681 ctl_arena->astats->nrequests_small = 0; 682 memset(ctl_arena->astats->bstats, 0, NBINS * 683 sizeof(malloc_bin_stats_t)); 684 memset(ctl_arena->astats->lstats, 0, (NSIZES - NBINS) * 685 sizeof(malloc_large_stats_t)); 686 } 687 } 688 689 static void 690 ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { 691 unsigned i; 692 693 if (config_stats) { 694 arena_stats_merge(tsdn, arena, &ctl_arena->nthreads, 695 &ctl_arena->dss, &ctl_arena->dirty_decay_ms, 696 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, 697 &ctl_arena->pdirty, &ctl_arena->pmuzzy, 698 &ctl_arena->astats->astats, ctl_arena->astats->bstats, 699 ctl_arena->astats->lstats); 700 701 for (i = 0; i < NBINS; i++) { 702 ctl_arena->astats->allocated_small += 703 ctl_arena->astats->bstats[i].curregs * 704 sz_index2size(i); 705 ctl_arena->astats->nmalloc_small += 706 ctl_arena->astats->bstats[i].nmalloc; 707 ctl_arena->astats->ndalloc_small += 708 ctl_arena->astats->bstats[i].ndalloc; 709 ctl_arena->astats->nrequests_small += 710 ctl_arena->astats->bstats[i].nrequests; 711 } 712 } else { 713 arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads, 714 &ctl_arena->dss, &ctl_arena->dirty_decay_ms, 715 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, 716 &ctl_arena->pdirty, &ctl_arena->pmuzzy); 717 } 718 } 719 720 static void 721 ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, 722 bool destroyed) { 723 unsigned i; 724 725 if (!destroyed) { 726 ctl_sdarena->nthreads += ctl_arena->nthreads; 727 ctl_sdarena->pactive += ctl_arena->pactive; 728 ctl_sdarena->pdirty += ctl_arena->pdirty; 729 ctl_sdarena->pmuzzy += ctl_arena->pmuzzy; 730 } else { 731 assert(ctl_arena->nthreads == 0); 732 assert(ctl_arena->pactive == 0); 733 assert(ctl_arena->pdirty == 0); 734 assert(ctl_arena->pmuzzy == 0); 735 } 736 737 if (config_stats) { 738 ctl_arena_stats_t *sdstats = ctl_sdarena->astats; 739 ctl_arena_stats_t *astats = ctl_arena->astats; 740 741 if (!destroyed) { 742 accum_atomic_zu(&sdstats->astats.mapped, 743 &astats->astats.mapped); 744 accum_atomic_zu(&sdstats->astats.retained, 745 &astats->astats.retained); 746 } 747 748 accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, 749 &astats->astats.decay_dirty.npurge); 750 accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise, 751 &astats->astats.decay_dirty.nmadvise); 752 accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged, 753 &astats->astats.decay_dirty.purged); 754 755 accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge, 756 &astats->astats.decay_muzzy.npurge); 757 accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise, 758 &astats->astats.decay_muzzy.nmadvise); 759 accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, 760 &astats->astats.decay_muzzy.purged); 761 762 #define OP(mtx) malloc_mutex_prof_merge( \ 763 &(sdstats->astats.mutex_prof_data[ \ 764 arena_prof_mutex_##mtx]), \ 765 &(astats->astats.mutex_prof_data[ \ 766 arena_prof_mutex_##mtx])); 767 MUTEX_PROF_ARENA_MUTEXES 768 #undef OP 769 if (!destroyed) { 770 accum_atomic_zu(&sdstats->astats.base, 771 &astats->astats.base); 772 accum_atomic_zu(&sdstats->astats.internal, 773 &astats->astats.internal); 774 accum_atomic_zu(&sdstats->astats.resident, 775 &astats->astats.resident); 776 } else { 777 assert(atomic_load_zu( 778 &astats->astats.internal, ATOMIC_RELAXED) == 0); 779 } 780 781 if (!destroyed) { 782 sdstats->allocated_small += astats->allocated_small; 783 } else { 784 assert(astats->allocated_small == 0); 785 } 786 sdstats->nmalloc_small += astats->nmalloc_small; 787 sdstats->ndalloc_small += astats->ndalloc_small; 788 sdstats->nrequests_small += astats->nrequests_small; 789 790 if (!destroyed) { 791 accum_atomic_zu(&sdstats->astats.allocated_large, 792 &astats->astats.allocated_large); 793 } else { 794 assert(atomic_load_zu(&astats->astats.allocated_large, 795 ATOMIC_RELAXED) == 0); 796 } 797 accum_arena_stats_u64(&sdstats->astats.nmalloc_large, 798 &astats->astats.nmalloc_large); 799 accum_arena_stats_u64(&sdstats->astats.ndalloc_large, 800 &astats->astats.ndalloc_large); 801 accum_arena_stats_u64(&sdstats->astats.nrequests_large, 802 &astats->astats.nrequests_large); 803 804 accum_atomic_zu(&sdstats->astats.tcache_bytes, 805 &astats->astats.tcache_bytes); 806 807 if (ctl_arena->arena_ind == 0) { 808 sdstats->astats.uptime = astats->astats.uptime; 809 } 810 811 for (i = 0; i < NBINS; i++) { 812 sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; 813 sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; 814 sdstats->bstats[i].nrequests += 815 astats->bstats[i].nrequests; 816 if (!destroyed) { 817 sdstats->bstats[i].curregs += 818 astats->bstats[i].curregs; 819 } else { 820 assert(astats->bstats[i].curregs == 0); 821 } 822 sdstats->bstats[i].nfills += astats->bstats[i].nfills; 823 sdstats->bstats[i].nflushes += 824 astats->bstats[i].nflushes; 825 sdstats->bstats[i].nslabs += astats->bstats[i].nslabs; 826 sdstats->bstats[i].reslabs += astats->bstats[i].reslabs; 827 if (!destroyed) { 828 sdstats->bstats[i].curslabs += 829 astats->bstats[i].curslabs; 830 } else { 831 assert(astats->bstats[i].curslabs == 0); 832 } 833 malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data, 834 &astats->bstats[i].mutex_data); 835 } 836 837 for (i = 0; i < NSIZES - NBINS; i++) { 838 accum_arena_stats_u64(&sdstats->lstats[i].nmalloc, 839 &astats->lstats[i].nmalloc); 840 accum_arena_stats_u64(&sdstats->lstats[i].ndalloc, 841 &astats->lstats[i].ndalloc); 842 accum_arena_stats_u64(&sdstats->lstats[i].nrequests, 843 &astats->lstats[i].nrequests); 844 if (!destroyed) { 845 sdstats->lstats[i].curlextents += 846 astats->lstats[i].curlextents; 847 } else { 848 assert(astats->lstats[i].curlextents == 0); 849 } 850 } 851 } 852 } 853 854 static void 855 ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena, 856 unsigned i, bool destroyed) { 857 ctl_arena_t *ctl_arena = arenas_i(i); 858 859 ctl_arena_clear(ctl_arena); 860 ctl_arena_stats_amerge(tsdn, ctl_arena, arena); 861 /* Merge into sum stats as well. */ 862 ctl_arena_stats_sdmerge(ctl_sdarena, ctl_arena, destroyed); 863 } 864 865 static unsigned 866 ctl_arena_init(tsd_t *tsd, extent_hooks_t *extent_hooks) { 867 unsigned arena_ind; 868 ctl_arena_t *ctl_arena; 869 870 if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) != 871 NULL) { 872 ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link); 873 arena_ind = ctl_arena->arena_ind; 874 } else { 875 arena_ind = ctl_arenas->narenas; 876 } 877 878 /* Trigger stats allocation. */ 879 if (arenas_i_impl(tsd, arena_ind, false, true) == NULL) { 880 return UINT_MAX; 881 } 882 883 /* Initialize new arena. */ 884 if (arena_init(tsd_tsdn(tsd), arena_ind, extent_hooks) == NULL) { 885 return UINT_MAX; 886 } 887 888 if (arena_ind == ctl_arenas->narenas) { 889 ctl_arenas->narenas++; 890 } 891 892 return arena_ind; 893 } 894 895 static void 896 ctl_background_thread_stats_read(tsdn_t *tsdn) { 897 background_thread_stats_t *stats = &ctl_stats->background_thread; 898 if (!have_background_thread || 899 background_thread_stats_read(tsdn, stats)) { 900 memset(stats, 0, sizeof(background_thread_stats_t)); 901 nstime_init(&stats->run_interval, 0); 902 } 903 } 904 905 static void 906 ctl_refresh(tsdn_t *tsdn) { 907 unsigned i; 908 ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL); 909 VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas); 910 911 /* 912 * Clear sum stats, since they will be merged into by 913 * ctl_arena_refresh(). 914 */ 915 ctl_arena_clear(ctl_sarena); 916 917 for (i = 0; i < ctl_arenas->narenas; i++) { 918 tarenas[i] = arena_get(tsdn, i, false); 919 } 920 921 for (i = 0; i < ctl_arenas->narenas; i++) { 922 ctl_arena_t *ctl_arena = arenas_i(i); 923 bool initialized = (tarenas[i] != NULL); 924 925 ctl_arena->initialized = initialized; 926 if (initialized) { 927 ctl_arena_refresh(tsdn, tarenas[i], ctl_sarena, i, 928 false); 929 } 930 } 931 932 if (config_stats) { 933 ctl_stats->allocated = ctl_sarena->astats->allocated_small + 934 atomic_load_zu(&ctl_sarena->astats->astats.allocated_large, 935 ATOMIC_RELAXED); 936 ctl_stats->active = (ctl_sarena->pactive << LG_PAGE); 937 ctl_stats->metadata = atomic_load_zu( 938 &ctl_sarena->astats->astats.base, ATOMIC_RELAXED) + 939 atomic_load_zu(&ctl_sarena->astats->astats.internal, 940 ATOMIC_RELAXED); 941 ctl_stats->resident = atomic_load_zu( 942 &ctl_sarena->astats->astats.resident, ATOMIC_RELAXED); 943 ctl_stats->mapped = atomic_load_zu( 944 &ctl_sarena->astats->astats.mapped, ATOMIC_RELAXED); 945 ctl_stats->retained = atomic_load_zu( 946 &ctl_sarena->astats->astats.retained, ATOMIC_RELAXED); 947 948 ctl_background_thread_stats_read(tsdn); 949 950 #define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \ 951 malloc_mutex_lock(tsdn, &mtx); \ 952 malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \ 953 malloc_mutex_unlock(tsdn, &mtx); 954 955 if (config_prof && opt_prof) { 956 READ_GLOBAL_MUTEX_PROF_DATA(global_prof_mutex_prof, 957 bt2gctx_mtx); 958 } 959 if (have_background_thread) { 960 READ_GLOBAL_MUTEX_PROF_DATA( 961 global_prof_mutex_background_thread, 962 background_thread_lock); 963 } else { 964 memset(&ctl_stats->mutex_prof_data[ 965 global_prof_mutex_background_thread], 0, 966 sizeof(mutex_prof_data_t)); 967 } 968 /* We own ctl mutex already. */ 969 malloc_mutex_prof_read(tsdn, 970 &ctl_stats->mutex_prof_data[global_prof_mutex_ctl], 971 &ctl_mtx); 972 #undef READ_GLOBAL_MUTEX_PROF_DATA 973 } 974 ctl_arenas->epoch++; 975 } 976 977 static bool 978 ctl_init(tsd_t *tsd) { 979 bool ret; 980 tsdn_t *tsdn = tsd_tsdn(tsd); 981 982 malloc_mutex_lock(tsdn, &ctl_mtx); 983 if (!ctl_initialized) { 984 ctl_arena_t *ctl_sarena, *ctl_darena; 985 unsigned i; 986 987 /* 988 * Allocate demand-zeroed space for pointers to the full 989 * range of supported arena indices. 990 */ 991 if (ctl_arenas == NULL) { 992 ctl_arenas = (ctl_arenas_t *)base_alloc(tsdn, 993 b0get(), sizeof(ctl_arenas_t), QUANTUM); 994 if (ctl_arenas == NULL) { 995 ret = true; 996 goto label_return; 997 } 998 } 999 1000 if (config_stats && ctl_stats == NULL) { 1001 ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(), 1002 sizeof(ctl_stats_t), QUANTUM); 1003 if (ctl_stats == NULL) { 1004 ret = true; 1005 goto label_return; 1006 } 1007 } 1008 1009 /* 1010 * Allocate space for the current full range of arenas 1011 * here rather than doing it lazily elsewhere, in order 1012 * to limit when OOM-caused errors can occur. 1013 */ 1014 if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false, 1015 true)) == NULL) { 1016 ret = true; 1017 goto label_return; 1018 } 1019 ctl_sarena->initialized = true; 1020 1021 if ((ctl_darena = arenas_i_impl(tsd, MALLCTL_ARENAS_DESTROYED, 1022 false, true)) == NULL) { 1023 ret = true; 1024 goto label_return; 1025 } 1026 ctl_arena_clear(ctl_darena); 1027 /* 1028 * Don't toggle ctl_darena to initialized until an arena is 1029 * actually destroyed, so that arena.<i>.initialized can be used 1030 * to query whether the stats are relevant. 1031 */ 1032 1033 ctl_arenas->narenas = narenas_total_get(); 1034 for (i = 0; i < ctl_arenas->narenas; i++) { 1035 if (arenas_i_impl(tsd, i, false, true) == NULL) { 1036 ret = true; 1037 goto label_return; 1038 } 1039 } 1040 1041 ql_new(&ctl_arenas->destroyed); 1042 ctl_refresh(tsdn); 1043 1044 ctl_initialized = true; 1045 } 1046 1047 ret = false; 1048 label_return: 1049 malloc_mutex_unlock(tsdn, &ctl_mtx); 1050 return ret; 1051 } 1052 1053 static int 1054 ctl_lookup(tsdn_t *tsdn, const char *name, ctl_node_t const **nodesp, 1055 size_t *mibp, size_t *depthp) { 1056 int ret; 1057 const char *elm, *tdot, *dot; 1058 size_t elen, i, j; 1059 const ctl_named_node_t *node; 1060 1061 elm = name; 1062 /* Equivalent to strchrnul(). */ 1063 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0'); 1064 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 1065 if (elen == 0) { 1066 ret = ENOENT; 1067 goto label_return; 1068 } 1069 node = super_root_node; 1070 for (i = 0; i < *depthp; i++) { 1071 assert(node); 1072 assert(node->nchildren > 0); 1073 if (ctl_named_node(node->children) != NULL) { 1074 const ctl_named_node_t *pnode = node; 1075 1076 /* Children are named. */ 1077 for (j = 0; j < node->nchildren; j++) { 1078 const ctl_named_node_t *child = 1079 ctl_named_children(node, j); 1080 if (strlen(child->name) == elen && 1081 strncmp(elm, child->name, elen) == 0) { 1082 node = child; 1083 if (nodesp != NULL) { 1084 nodesp[i] = 1085 (const ctl_node_t *)node; 1086 } 1087 mibp[i] = j; 1088 break; 1089 } 1090 } 1091 if (node == pnode) { 1092 ret = ENOENT; 1093 goto label_return; 1094 } 1095 } else { 1096 uintmax_t index; 1097 const ctl_indexed_node_t *inode; 1098 1099 /* Children are indexed. */ 1100 index = malloc_strtoumax(elm, NULL, 10); 1101 if (index == UINTMAX_MAX || index > SIZE_T_MAX) { 1102 ret = ENOENT; 1103 goto label_return; 1104 } 1105 1106 inode = ctl_indexed_node(node->children); 1107 node = inode->index(tsdn, mibp, *depthp, (size_t)index); 1108 if (node == NULL) { 1109 ret = ENOENT; 1110 goto label_return; 1111 } 1112 1113 if (nodesp != NULL) { 1114 nodesp[i] = (const ctl_node_t *)node; 1115 } 1116 mibp[i] = (size_t)index; 1117 } 1118 1119 if (node->ctl != NULL) { 1120 /* Terminal node. */ 1121 if (*dot != '\0') { 1122 /* 1123 * The name contains more elements than are 1124 * in this path through the tree. 1125 */ 1126 ret = ENOENT; 1127 goto label_return; 1128 } 1129 /* Complete lookup successful. */ 1130 *depthp = i + 1; 1131 break; 1132 } 1133 1134 /* Update elm. */ 1135 if (*dot == '\0') { 1136 /* No more elements. */ 1137 ret = ENOENT; 1138 goto label_return; 1139 } 1140 elm = &dot[1]; 1141 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : 1142 strchr(elm, '\0'); 1143 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 1144 } 1145 1146 ret = 0; 1147 label_return: 1148 return ret; 1149 } 1150 1151 int 1152 ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, 1153 void *newp, size_t newlen) { 1154 int ret; 1155 size_t depth; 1156 ctl_node_t const *nodes[CTL_MAX_DEPTH]; 1157 size_t mib[CTL_MAX_DEPTH]; 1158 const ctl_named_node_t *node; 1159 1160 if (!ctl_initialized && ctl_init(tsd)) { 1161 ret = EAGAIN; 1162 goto label_return; 1163 } 1164 1165 depth = CTL_MAX_DEPTH; 1166 ret = ctl_lookup(tsd_tsdn(tsd), name, nodes, mib, &depth); 1167 if (ret != 0) { 1168 goto label_return; 1169 } 1170 1171 node = ctl_named_node(nodes[depth-1]); 1172 if (node != NULL && node->ctl) { 1173 ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen); 1174 } else { 1175 /* The name refers to a partial path through the ctl tree. */ 1176 ret = ENOENT; 1177 } 1178 1179 label_return: 1180 return(ret); 1181 } 1182 1183 int 1184 ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) { 1185 int ret; 1186 1187 if (!ctl_initialized && ctl_init(tsd)) { 1188 ret = EAGAIN; 1189 goto label_return; 1190 } 1191 1192 ret = ctl_lookup(tsd_tsdn(tsd), name, NULL, mibp, miblenp); 1193 label_return: 1194 return(ret); 1195 } 1196 1197 int 1198 ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1199 size_t *oldlenp, void *newp, size_t newlen) { 1200 int ret; 1201 const ctl_named_node_t *node; 1202 size_t i; 1203 1204 if (!ctl_initialized && ctl_init(tsd)) { 1205 ret = EAGAIN; 1206 goto label_return; 1207 } 1208 1209 /* Iterate down the tree. */ 1210 node = super_root_node; 1211 for (i = 0; i < miblen; i++) { 1212 assert(node); 1213 assert(node->nchildren > 0); 1214 if (ctl_named_node(node->children) != NULL) { 1215 /* Children are named. */ 1216 if (node->nchildren <= mib[i]) { 1217 ret = ENOENT; 1218 goto label_return; 1219 } 1220 node = ctl_named_children(node, mib[i]); 1221 } else { 1222 const ctl_indexed_node_t *inode; 1223 1224 /* Indexed element. */ 1225 inode = ctl_indexed_node(node->children); 1226 node = inode->index(tsd_tsdn(tsd), mib, miblen, mib[i]); 1227 if (node == NULL) { 1228 ret = ENOENT; 1229 goto label_return; 1230 } 1231 } 1232 } 1233 1234 /* Call the ctl function. */ 1235 if (node && node->ctl) { 1236 ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 1237 } else { 1238 /* Partial MIB. */ 1239 ret = ENOENT; 1240 } 1241 1242 label_return: 1243 return(ret); 1244 } 1245 1246 bool 1247 ctl_boot(void) { 1248 if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL, 1249 malloc_mutex_rank_exclusive)) { 1250 return true; 1251 } 1252 1253 ctl_initialized = false; 1254 1255 return false; 1256 } 1257 1258 void 1259 ctl_prefork(tsdn_t *tsdn) { 1260 malloc_mutex_prefork(tsdn, &ctl_mtx); 1261 } 1262 1263 void 1264 ctl_postfork_parent(tsdn_t *tsdn) { 1265 malloc_mutex_postfork_parent(tsdn, &ctl_mtx); 1266 } 1267 1268 void 1269 ctl_postfork_child(tsdn_t *tsdn) { 1270 malloc_mutex_postfork_child(tsdn, &ctl_mtx); 1271 } 1272 1273 /******************************************************************************/ 1274 /* *_ctl() functions. */ 1275 1276 #define READONLY() do { \ 1277 if (newp != NULL || newlen != 0) { \ 1278 ret = EPERM; \ 1279 goto label_return; \ 1280 } \ 1281 } while (0) 1282 1283 #define WRITEONLY() do { \ 1284 if (oldp != NULL || oldlenp != NULL) { \ 1285 ret = EPERM; \ 1286 goto label_return; \ 1287 } \ 1288 } while (0) 1289 1290 #define READ_XOR_WRITE() do { \ 1291 if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \ 1292 newlen != 0)) { \ 1293 ret = EPERM; \ 1294 goto label_return; \ 1295 } \ 1296 } while (0) 1297 1298 #define READ(v, t) do { \ 1299 if (oldp != NULL && oldlenp != NULL) { \ 1300 if (*oldlenp != sizeof(t)) { \ 1301 size_t copylen = (sizeof(t) <= *oldlenp) \ 1302 ? sizeof(t) : *oldlenp; \ 1303 memcpy(oldp, (void *)&(v), copylen); \ 1304 ret = EINVAL; \ 1305 goto label_return; \ 1306 } \ 1307 *(t *)oldp = (v); \ 1308 } \ 1309 } while (0) 1310 1311 #define WRITE(v, t) do { \ 1312 if (newp != NULL) { \ 1313 if (newlen != sizeof(t)) { \ 1314 ret = EINVAL; \ 1315 goto label_return; \ 1316 } \ 1317 (v) = *(t *)newp; \ 1318 } \ 1319 } while (0) 1320 1321 #define MIB_UNSIGNED(v, i) do { \ 1322 if (mib[i] > UINT_MAX) { \ 1323 ret = EFAULT; \ 1324 goto label_return; \ 1325 } \ 1326 v = (unsigned)mib[i]; \ 1327 } while (0) 1328 1329 /* 1330 * There's a lot of code duplication in the following macros due to limitations 1331 * in how nested cpp macros are expanded. 1332 */ 1333 #define CTL_RO_CLGEN(c, l, n, v, t) \ 1334 static int \ 1335 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1336 size_t *oldlenp, void *newp, size_t newlen) { \ 1337 int ret; \ 1338 t oldval; \ 1339 \ 1340 if (!(c)) { \ 1341 return ENOENT; \ 1342 } \ 1343 if (l) { \ 1344 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1345 } \ 1346 READONLY(); \ 1347 oldval = (v); \ 1348 READ(oldval, t); \ 1349 \ 1350 ret = 0; \ 1351 label_return: \ 1352 if (l) { \ 1353 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1354 } \ 1355 return ret; \ 1356 } 1357 1358 #define CTL_RO_CGEN(c, n, v, t) \ 1359 static int \ 1360 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1361 size_t *oldlenp, void *newp, size_t newlen) { \ 1362 int ret; \ 1363 t oldval; \ 1364 \ 1365 if (!(c)) { \ 1366 return ENOENT; \ 1367 } \ 1368 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1369 READONLY(); \ 1370 oldval = (v); \ 1371 READ(oldval, t); \ 1372 \ 1373 ret = 0; \ 1374 label_return: \ 1375 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1376 return ret; \ 1377 } 1378 1379 #define CTL_RO_GEN(n, v, t) \ 1380 static int \ 1381 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1382 size_t *oldlenp, void *newp, size_t newlen) { \ 1383 int ret; \ 1384 t oldval; \ 1385 \ 1386 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1387 READONLY(); \ 1388 oldval = (v); \ 1389 READ(oldval, t); \ 1390 \ 1391 ret = 0; \ 1392 label_return: \ 1393 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1394 return ret; \ 1395 } 1396 1397 /* 1398 * ctl_mtx is not acquired, under the assumption that no pertinent data will 1399 * mutate during the call. 1400 */ 1401 #define CTL_RO_NL_CGEN(c, n, v, t) \ 1402 static int \ 1403 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1404 size_t *oldlenp, void *newp, size_t newlen) { \ 1405 int ret; \ 1406 t oldval; \ 1407 \ 1408 if (!(c)) { \ 1409 return ENOENT; \ 1410 } \ 1411 READONLY(); \ 1412 oldval = (v); \ 1413 READ(oldval, t); \ 1414 \ 1415 ret = 0; \ 1416 label_return: \ 1417 return ret; \ 1418 } 1419 1420 #define CTL_RO_NL_GEN(n, v, t) \ 1421 static int \ 1422 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1423 size_t *oldlenp, void *newp, size_t newlen) { \ 1424 int ret; \ 1425 t oldval; \ 1426 \ 1427 READONLY(); \ 1428 oldval = (v); \ 1429 READ(oldval, t); \ 1430 \ 1431 ret = 0; \ 1432 label_return: \ 1433 return ret; \ 1434 } 1435 1436 #define CTL_TSD_RO_NL_CGEN(c, n, m, t) \ 1437 static int \ 1438 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1439 size_t *oldlenp, void *newp, size_t newlen) { \ 1440 int ret; \ 1441 t oldval; \ 1442 \ 1443 if (!(c)) { \ 1444 return ENOENT; \ 1445 } \ 1446 READONLY(); \ 1447 oldval = (m(tsd)); \ 1448 READ(oldval, t); \ 1449 \ 1450 ret = 0; \ 1451 label_return: \ 1452 return ret; \ 1453 } 1454 1455 #define CTL_RO_CONFIG_GEN(n, t) \ 1456 static int \ 1457 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1458 size_t *oldlenp, void *newp, size_t newlen) { \ 1459 int ret; \ 1460 t oldval; \ 1461 \ 1462 READONLY(); \ 1463 oldval = n; \ 1464 READ(oldval, t); \ 1465 \ 1466 ret = 0; \ 1467 label_return: \ 1468 return ret; \ 1469 } 1470 1471 /******************************************************************************/ 1472 1473 CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) 1474 1475 static int 1476 epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1477 size_t *oldlenp, void *newp, size_t newlen) { 1478 int ret; 1479 UNUSED uint64_t newval; 1480 1481 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1482 WRITE(newval, uint64_t); 1483 if (newp != NULL) { 1484 ctl_refresh(tsd_tsdn(tsd)); 1485 } 1486 READ(ctl_arenas->epoch, uint64_t); 1487 1488 ret = 0; 1489 label_return: 1490 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1491 return ret; 1492 } 1493 1494 static int 1495 background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1496 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1497 int ret; 1498 bool oldval; 1499 1500 if (!have_background_thread) { 1501 return ENOENT; 1502 } 1503 background_thread_ctl_init(tsd_tsdn(tsd)); 1504 1505 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1506 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 1507 if (newp == NULL) { 1508 oldval = background_thread_enabled(); 1509 READ(oldval, bool); 1510 } else { 1511 if (newlen != sizeof(bool)) { 1512 ret = EINVAL; 1513 goto label_return; 1514 } 1515 oldval = background_thread_enabled(); 1516 READ(oldval, bool); 1517 1518 bool newval = *(bool *)newp; 1519 if (newval == oldval) { 1520 ret = 0; 1521 goto label_return; 1522 } 1523 1524 background_thread_enabled_set(tsd_tsdn(tsd), newval); 1525 if (newval) { 1526 if (!can_enable_background_thread) { 1527 malloc_printf("<jemalloc>: Error in dlsym(" 1528 "RTLD_NEXT, \"pthread_create\"). Cannot " 1529 "enable background_thread\n"); 1530 ret = EFAULT; 1531 goto label_return; 1532 } 1533 if (background_threads_enable(tsd)) { 1534 ret = EFAULT; 1535 goto label_return; 1536 } 1537 } else { 1538 if (background_threads_disable(tsd)) { 1539 ret = EFAULT; 1540 goto label_return; 1541 } 1542 } 1543 } 1544 ret = 0; 1545 label_return: 1546 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 1547 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1548 1549 return ret; 1550 } 1551 1552 /******************************************************************************/ 1553 1554 CTL_RO_CONFIG_GEN(config_cache_oblivious, bool) 1555 CTL_RO_CONFIG_GEN(config_debug, bool) 1556 CTL_RO_CONFIG_GEN(config_fill, bool) 1557 CTL_RO_CONFIG_GEN(config_lazy_lock, bool) 1558 CTL_RO_CONFIG_GEN(config_malloc_conf, const char *) 1559 CTL_RO_CONFIG_GEN(config_prof, bool) 1560 CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) 1561 CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) 1562 CTL_RO_CONFIG_GEN(config_stats, bool) 1563 CTL_RO_CONFIG_GEN(config_thp, bool) 1564 CTL_RO_CONFIG_GEN(config_utrace, bool) 1565 CTL_RO_CONFIG_GEN(config_xmalloc, bool) 1566 1567 /******************************************************************************/ 1568 1569 CTL_RO_NL_GEN(opt_abort, opt_abort, bool) 1570 CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) 1571 CTL_RO_NL_GEN(opt_retain, opt_retain, bool) 1572 CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) 1573 CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) 1574 CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], 1575 const char *) 1576 CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) 1577 CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t) 1578 CTL_RO_NL_GEN(opt_muzzy_decay_ms, opt_muzzy_decay_ms, ssize_t) 1579 CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) 1580 CTL_RO_NL_GEN(opt_stats_print_opts, opt_stats_print_opts, const char *) 1581 CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) 1582 CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) 1583 CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) 1584 CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) 1585 CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) 1586 CTL_RO_NL_GEN(opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) 1587 CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) 1588 CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) 1589 CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) 1590 CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init, 1591 opt_prof_thread_active_init, bool) 1592 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) 1593 CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) 1594 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) 1595 CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) 1596 CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) 1597 CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) 1598 1599 /******************************************************************************/ 1600 1601 static int 1602 thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1603 size_t *oldlenp, void *newp, size_t newlen) { 1604 int ret; 1605 arena_t *oldarena; 1606 unsigned newind, oldind; 1607 1608 oldarena = arena_choose(tsd, NULL); 1609 if (oldarena == NULL) { 1610 return EAGAIN; 1611 } 1612 newind = oldind = arena_ind_get(oldarena); 1613 WRITE(newind, unsigned); 1614 READ(oldind, unsigned); 1615 1616 if (newind != oldind) { 1617 arena_t *newarena; 1618 1619 if (newind >= narenas_total_get()) { 1620 /* New arena index is out of range. */ 1621 ret = EFAULT; 1622 goto label_return; 1623 } 1624 1625 if (have_percpu_arena && 1626 PERCPU_ARENA_ENABLED(opt_percpu_arena)) { 1627 if (newind < percpu_arena_ind_limit(opt_percpu_arena)) { 1628 /* 1629 * If perCPU arena is enabled, thread_arena 1630 * control is not allowed for the auto arena 1631 * range. 1632 */ 1633 ret = EPERM; 1634 goto label_return; 1635 } 1636 } 1637 1638 /* Initialize arena if necessary. */ 1639 newarena = arena_get(tsd_tsdn(tsd), newind, true); 1640 if (newarena == NULL) { 1641 ret = EAGAIN; 1642 goto label_return; 1643 } 1644 /* Set new arena/tcache associations. */ 1645 arena_migrate(tsd, oldind, newind); 1646 if (tcache_available(tsd)) { 1647 tcache_arena_reassociate(tsd_tsdn(tsd), 1648 tsd_tcachep_get(tsd), newarena); 1649 } 1650 } 1651 1652 ret = 0; 1653 label_return: 1654 return ret; 1655 } 1656 1657 CTL_TSD_RO_NL_CGEN(config_stats, thread_allocated, tsd_thread_allocated_get, 1658 uint64_t) 1659 CTL_TSD_RO_NL_CGEN(config_stats, thread_allocatedp, tsd_thread_allocatedp_get, 1660 uint64_t *) 1661 CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocated, tsd_thread_deallocated_get, 1662 uint64_t) 1663 CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, 1664 tsd_thread_deallocatedp_get, uint64_t *) 1665 1666 static int 1667 thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1668 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1669 int ret; 1670 bool oldval; 1671 1672 oldval = tcache_enabled_get(tsd); 1673 if (newp != NULL) { 1674 if (newlen != sizeof(bool)) { 1675 ret = EINVAL; 1676 goto label_return; 1677 } 1678 tcache_enabled_set(tsd, *(bool *)newp); 1679 } 1680 READ(oldval, bool); 1681 1682 ret = 0; 1683 label_return: 1684 return ret; 1685 } 1686 1687 static int 1688 thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1689 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1690 int ret; 1691 1692 if (!tcache_available(tsd)) { 1693 ret = EFAULT; 1694 goto label_return; 1695 } 1696 1697 READONLY(); 1698 WRITEONLY(); 1699 1700 tcache_flush(tsd); 1701 1702 ret = 0; 1703 label_return: 1704 return ret; 1705 } 1706 1707 static int 1708 thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1709 size_t *oldlenp, void *newp, size_t newlen) { 1710 int ret; 1711 1712 if (!config_prof) { 1713 return ENOENT; 1714 } 1715 1716 READ_XOR_WRITE(); 1717 1718 if (newp != NULL) { 1719 if (newlen != sizeof(const char *)) { 1720 ret = EINVAL; 1721 goto label_return; 1722 } 1723 1724 if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) != 1725 0) { 1726 goto label_return; 1727 } 1728 } else { 1729 const char *oldname = prof_thread_name_get(tsd); 1730 READ(oldname, const char *); 1731 } 1732 1733 ret = 0; 1734 label_return: 1735 return ret; 1736 } 1737 1738 static int 1739 thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1740 size_t *oldlenp, void *newp, size_t newlen) { 1741 int ret; 1742 bool oldval; 1743 1744 if (!config_prof) { 1745 return ENOENT; 1746 } 1747 1748 oldval = prof_thread_active_get(tsd); 1749 if (newp != NULL) { 1750 if (newlen != sizeof(bool)) { 1751 ret = EINVAL; 1752 goto label_return; 1753 } 1754 if (prof_thread_active_set(tsd, *(bool *)newp)) { 1755 ret = EAGAIN; 1756 goto label_return; 1757 } 1758 } 1759 READ(oldval, bool); 1760 1761 ret = 0; 1762 label_return: 1763 return ret; 1764 } 1765 1766 /******************************************************************************/ 1767 1768 static int 1769 tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1770 size_t *oldlenp, void *newp, size_t newlen) { 1771 int ret; 1772 unsigned tcache_ind; 1773 1774 READONLY(); 1775 if (tcaches_create(tsd, &tcache_ind)) { 1776 ret = EFAULT; 1777 goto label_return; 1778 } 1779 READ(tcache_ind, unsigned); 1780 1781 ret = 0; 1782 label_return: 1783 return ret; 1784 } 1785 1786 static int 1787 tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1788 size_t *oldlenp, void *newp, size_t newlen) { 1789 int ret; 1790 unsigned tcache_ind; 1791 1792 WRITEONLY(); 1793 tcache_ind = UINT_MAX; 1794 WRITE(tcache_ind, unsigned); 1795 if (tcache_ind == UINT_MAX) { 1796 ret = EFAULT; 1797 goto label_return; 1798 } 1799 tcaches_flush(tsd, tcache_ind); 1800 1801 ret = 0; 1802 label_return: 1803 return ret; 1804 } 1805 1806 static int 1807 tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1808 size_t *oldlenp, void *newp, size_t newlen) { 1809 int ret; 1810 unsigned tcache_ind; 1811 1812 WRITEONLY(); 1813 tcache_ind = UINT_MAX; 1814 WRITE(tcache_ind, unsigned); 1815 if (tcache_ind == UINT_MAX) { 1816 ret = EFAULT; 1817 goto label_return; 1818 } 1819 tcaches_destroy(tsd, tcache_ind); 1820 1821 ret = 0; 1822 label_return: 1823 return ret; 1824 } 1825 1826 /******************************************************************************/ 1827 1828 static int 1829 arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1830 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1831 int ret; 1832 tsdn_t *tsdn = tsd_tsdn(tsd); 1833 unsigned arena_ind; 1834 bool initialized; 1835 1836 READONLY(); 1837 MIB_UNSIGNED(arena_ind, 1); 1838 1839 malloc_mutex_lock(tsdn, &ctl_mtx); 1840 initialized = arenas_i(arena_ind)->initialized; 1841 malloc_mutex_unlock(tsdn, &ctl_mtx); 1842 1843 READ(initialized, bool); 1844 1845 ret = 0; 1846 label_return: 1847 return ret; 1848 } 1849 1850 static void 1851 arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { 1852 malloc_mutex_lock(tsdn, &ctl_mtx); 1853 { 1854 unsigned narenas = ctl_arenas->narenas; 1855 1856 /* 1857 * Access via index narenas is deprecated, and scheduled for 1858 * removal in 6.0.0. 1859 */ 1860 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == narenas) { 1861 unsigned i; 1862 VARIABLE_ARRAY(arena_t *, tarenas, narenas); 1863 1864 for (i = 0; i < narenas; i++) { 1865 tarenas[i] = arena_get(tsdn, i, false); 1866 } 1867 1868 /* 1869 * No further need to hold ctl_mtx, since narenas and 1870 * tarenas contain everything needed below. 1871 */ 1872 malloc_mutex_unlock(tsdn, &ctl_mtx); 1873 1874 for (i = 0; i < narenas; i++) { 1875 if (tarenas[i] != NULL) { 1876 arena_decay(tsdn, tarenas[i], false, 1877 all); 1878 } 1879 } 1880 } else { 1881 arena_t *tarena; 1882 1883 assert(arena_ind < narenas); 1884 1885 tarena = arena_get(tsdn, arena_ind, false); 1886 1887 /* No further need to hold ctl_mtx. */ 1888 malloc_mutex_unlock(tsdn, &ctl_mtx); 1889 1890 if (tarena != NULL) { 1891 arena_decay(tsdn, tarena, false, all); 1892 } 1893 } 1894 } 1895 } 1896 1897 static int 1898 arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1899 size_t *oldlenp, void *newp, size_t newlen) { 1900 int ret; 1901 unsigned arena_ind; 1902 1903 READONLY(); 1904 WRITEONLY(); 1905 MIB_UNSIGNED(arena_ind, 1); 1906 arena_i_decay(tsd_tsdn(tsd), arena_ind, false); 1907 1908 ret = 0; 1909 label_return: 1910 return ret; 1911 } 1912 1913 static int 1914 arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1915 size_t *oldlenp, void *newp, size_t newlen) { 1916 int ret; 1917 unsigned arena_ind; 1918 1919 READONLY(); 1920 WRITEONLY(); 1921 MIB_UNSIGNED(arena_ind, 1); 1922 arena_i_decay(tsd_tsdn(tsd), arena_ind, true); 1923 1924 ret = 0; 1925 label_return: 1926 return ret; 1927 } 1928 1929 static int 1930 arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen, 1931 void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind, 1932 arena_t **arena) { 1933 int ret; 1934 1935 READONLY(); 1936 WRITEONLY(); 1937 MIB_UNSIGNED(*arena_ind, 1); 1938 1939 *arena = arena_get(tsd_tsdn(tsd), *arena_ind, false); 1940 if (*arena == NULL || arena_is_auto(*arena)) { 1941 ret = EFAULT; 1942 goto label_return; 1943 } 1944 1945 ret = 0; 1946 label_return: 1947 return ret; 1948 } 1949 1950 static void 1951 arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) { 1952 /* Temporarily disable the background thread during arena reset. */ 1953 if (have_background_thread) { 1954 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 1955 if (background_thread_enabled()) { 1956 unsigned ind = arena_ind % ncpus; 1957 background_thread_info_t *info = 1958 &background_thread_info[ind]; 1959 assert(info->state == background_thread_started); 1960 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 1961 info->state = background_thread_paused; 1962 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 1963 } 1964 } 1965 } 1966 1967 static void 1968 arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { 1969 if (have_background_thread) { 1970 if (background_thread_enabled()) { 1971 unsigned ind = arena_ind % ncpus; 1972 background_thread_info_t *info = 1973 &background_thread_info[ind]; 1974 assert(info->state == background_thread_paused); 1975 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 1976 info->state = background_thread_started; 1977 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 1978 } 1979 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 1980 } 1981 } 1982 1983 static int 1984 arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1985 size_t *oldlenp, void *newp, size_t newlen) { 1986 int ret; 1987 unsigned arena_ind; 1988 arena_t *arena; 1989 1990 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, 1991 newp, newlen, &arena_ind, &arena); 1992 if (ret != 0) { 1993 return ret; 1994 } 1995 1996 arena_reset_prepare_background_thread(tsd, arena_ind); 1997 arena_reset(tsd, arena); 1998 arena_reset_finish_background_thread(tsd, arena_ind); 1999 2000 return ret; 2001 } 2002 2003 static int 2004 arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2005 size_t *oldlenp, void *newp, size_t newlen) { 2006 int ret; 2007 unsigned arena_ind; 2008 arena_t *arena; 2009 ctl_arena_t *ctl_darena, *ctl_arena; 2010 2011 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, 2012 newp, newlen, &arena_ind, &arena); 2013 if (ret != 0) { 2014 goto label_return; 2015 } 2016 2017 if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena, 2018 true) != 0) { 2019 ret = EFAULT; 2020 goto label_return; 2021 } 2022 2023 arena_reset_prepare_background_thread(tsd, arena_ind); 2024 /* Merge stats after resetting and purging arena. */ 2025 arena_reset(tsd, arena); 2026 arena_decay(tsd_tsdn(tsd), arena, false, true); 2027 ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED); 2028 ctl_darena->initialized = true; 2029 ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true); 2030 /* Destroy arena. */ 2031 arena_destroy(tsd, arena); 2032 ctl_arena = arenas_i(arena_ind); 2033 ctl_arena->initialized = false; 2034 /* Record arena index for later recycling via arenas.create. */ 2035 ql_elm_new(ctl_arena, destroyed_link); 2036 ql_tail_insert(&ctl_arenas->destroyed, ctl_arena, destroyed_link); 2037 arena_reset_finish_background_thread(tsd, arena_ind); 2038 2039 assert(ret == 0); 2040 label_return: 2041 return ret; 2042 } 2043 2044 static int 2045 arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2046 size_t *oldlenp, void *newp, size_t newlen) { 2047 int ret; 2048 const char *dss = NULL; 2049 unsigned arena_ind; 2050 dss_prec_t dss_prec_old = dss_prec_limit; 2051 dss_prec_t dss_prec = dss_prec_limit; 2052 2053 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2054 WRITE(dss, const char *); 2055 MIB_UNSIGNED(arena_ind, 1); 2056 if (dss != NULL) { 2057 int i; 2058 bool match = false; 2059 2060 for (i = 0; i < dss_prec_limit; i++) { 2061 if (strcmp(dss_prec_names[i], dss) == 0) { 2062 dss_prec = i; 2063 match = true; 2064 break; 2065 } 2066 } 2067 2068 if (!match) { 2069 ret = EINVAL; 2070 goto label_return; 2071 } 2072 } 2073 2074 /* 2075 * Access via index narenas is deprecated, and scheduled for removal in 2076 * 6.0.0. 2077 */ 2078 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == 2079 ctl_arenas->narenas) { 2080 if (dss_prec != dss_prec_limit && 2081 extent_dss_prec_set(dss_prec)) { 2082 ret = EFAULT; 2083 goto label_return; 2084 } 2085 dss_prec_old = extent_dss_prec_get(); 2086 } else { 2087 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2088 if (arena == NULL || (dss_prec != dss_prec_limit && 2089 arena_dss_prec_set(arena, dss_prec))) { 2090 ret = EFAULT; 2091 goto label_return; 2092 } 2093 dss_prec_old = arena_dss_prec_get(arena); 2094 } 2095 2096 dss = dss_prec_names[dss_prec_old]; 2097 READ(dss, const char *); 2098 2099 ret = 0; 2100 label_return: 2101 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2102 return ret; 2103 } 2104 2105 static int 2106 arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, 2107 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { 2108 int ret; 2109 unsigned arena_ind; 2110 arena_t *arena; 2111 2112 MIB_UNSIGNED(arena_ind, 1); 2113 arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2114 if (arena == NULL) { 2115 ret = EFAULT; 2116 goto label_return; 2117 } 2118 2119 if (oldp != NULL && oldlenp != NULL) { 2120 size_t oldval = dirty ? arena_dirty_decay_ms_get(arena) : 2121 arena_muzzy_decay_ms_get(arena); 2122 READ(oldval, ssize_t); 2123 } 2124 if (newp != NULL) { 2125 if (newlen != sizeof(ssize_t)) { 2126 ret = EINVAL; 2127 goto label_return; 2128 } 2129 if (dirty ? arena_dirty_decay_ms_set(tsd_tsdn(tsd), arena, 2130 *(ssize_t *)newp) : arena_muzzy_decay_ms_set(tsd_tsdn(tsd), 2131 arena, *(ssize_t *)newp)) { 2132 ret = EFAULT; 2133 goto label_return; 2134 } 2135 } 2136 2137 ret = 0; 2138 label_return: 2139 return ret; 2140 } 2141 2142 static int 2143 arena_i_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2144 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2145 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2146 newlen, true); 2147 } 2148 2149 static int 2150 arena_i_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2151 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2152 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2153 newlen, false); 2154 } 2155 2156 static int 2157 arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2158 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2159 int ret; 2160 unsigned arena_ind; 2161 arena_t *arena; 2162 2163 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2164 MIB_UNSIGNED(arena_ind, 1); 2165 if (arena_ind < narenas_total_get() && (arena = 2166 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { 2167 if (newp != NULL) { 2168 extent_hooks_t *old_extent_hooks; 2169 extent_hooks_t *new_extent_hooks 2170 JEMALLOC_CC_SILENCE_INIT(NULL); 2171 WRITE(new_extent_hooks, extent_hooks_t *); 2172 old_extent_hooks = extent_hooks_set(tsd, arena, 2173 new_extent_hooks); 2174 READ(old_extent_hooks, extent_hooks_t *); 2175 } else { 2176 extent_hooks_t *old_extent_hooks = 2177 extent_hooks_get(arena); 2178 READ(old_extent_hooks, extent_hooks_t *); 2179 } 2180 } else { 2181 ret = EFAULT; 2182 goto label_return; 2183 } 2184 ret = 0; 2185 label_return: 2186 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2187 return ret; 2188 } 2189 2190 static const ctl_named_node_t * 2191 arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { 2192 const ctl_named_node_t *ret; 2193 2194 malloc_mutex_lock(tsdn, &ctl_mtx); 2195 switch (i) { 2196 case MALLCTL_ARENAS_ALL: 2197 case MALLCTL_ARENAS_DESTROYED: 2198 break; 2199 default: 2200 if (i > ctl_arenas->narenas) { 2201 ret = NULL; 2202 goto label_return; 2203 } 2204 break; 2205 } 2206 2207 ret = super_arena_i_node; 2208 label_return: 2209 malloc_mutex_unlock(tsdn, &ctl_mtx); 2210 return ret; 2211 } 2212 2213 /******************************************************************************/ 2214 2215 static int 2216 arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2217 size_t *oldlenp, void *newp, size_t newlen) { 2218 int ret; 2219 unsigned narenas; 2220 2221 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2222 READONLY(); 2223 if (*oldlenp != sizeof(unsigned)) { 2224 ret = EINVAL; 2225 goto label_return; 2226 } 2227 narenas = ctl_arenas->narenas; 2228 READ(narenas, unsigned); 2229 2230 ret = 0; 2231 label_return: 2232 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2233 return ret; 2234 } 2235 2236 static int 2237 arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, 2238 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { 2239 int ret; 2240 2241 if (oldp != NULL && oldlenp != NULL) { 2242 size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() : 2243 arena_muzzy_decay_ms_default_get()); 2244 READ(oldval, ssize_t); 2245 } 2246 if (newp != NULL) { 2247 if (newlen != sizeof(ssize_t)) { 2248 ret = EINVAL; 2249 goto label_return; 2250 } 2251 if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp) 2252 : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) { 2253 ret = EFAULT; 2254 goto label_return; 2255 } 2256 } 2257 2258 ret = 0; 2259 label_return: 2260 return ret; 2261 } 2262 2263 static int 2264 arenas_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2265 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2266 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2267 newlen, true); 2268 } 2269 2270 static int 2271 arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2272 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2273 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2274 newlen, false); 2275 } 2276 2277 CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) 2278 CTL_RO_NL_GEN(arenas_page, PAGE, size_t) 2279 CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) 2280 CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) 2281 CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) 2282 CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) 2283 CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) 2284 CTL_RO_NL_GEN(arenas_bin_i_slab_size, arena_bin_info[mib[2]].slab_size, size_t) 2285 static const ctl_named_node_t * 2286 arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { 2287 if (i > NBINS) { 2288 return NULL; 2289 } 2290 return super_arenas_bin_i_node; 2291 } 2292 2293 CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned) 2294 CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(NBINS+(szind_t)mib[2]), 2295 size_t) 2296 static const ctl_named_node_t * 2297 arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 2298 size_t i) { 2299 if (i > NSIZES - NBINS) { 2300 return NULL; 2301 } 2302 return super_arenas_lextent_i_node; 2303 } 2304 2305 static int 2306 arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2307 size_t *oldlenp, void *newp, size_t newlen) { 2308 int ret; 2309 extent_hooks_t *extent_hooks; 2310 unsigned arena_ind; 2311 2312 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2313 2314 extent_hooks = (extent_hooks_t *)&extent_hooks_default; 2315 WRITE(extent_hooks, extent_hooks_t *); 2316 if ((arena_ind = ctl_arena_init(tsd, extent_hooks)) == UINT_MAX) { 2317 ret = EAGAIN; 2318 goto label_return; 2319 } 2320 READ(arena_ind, unsigned); 2321 2322 ret = 0; 2323 label_return: 2324 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2325 return ret; 2326 } 2327 2328 /******************************************************************************/ 2329 2330 static int 2331 prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2332 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2333 int ret; 2334 bool oldval; 2335 2336 if (!config_prof) { 2337 return ENOENT; 2338 } 2339 2340 if (newp != NULL) { 2341 if (newlen != sizeof(bool)) { 2342 ret = EINVAL; 2343 goto label_return; 2344 } 2345 oldval = prof_thread_active_init_set(tsd_tsdn(tsd), 2346 *(bool *)newp); 2347 } else { 2348 oldval = prof_thread_active_init_get(tsd_tsdn(tsd)); 2349 } 2350 READ(oldval, bool); 2351 2352 ret = 0; 2353 label_return: 2354 return ret; 2355 } 2356 2357 static int 2358 prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2359 size_t *oldlenp, void *newp, size_t newlen) { 2360 int ret; 2361 bool oldval; 2362 2363 if (!config_prof) { 2364 return ENOENT; 2365 } 2366 2367 if (newp != NULL) { 2368 if (newlen != sizeof(bool)) { 2369 ret = EINVAL; 2370 goto label_return; 2371 } 2372 oldval = prof_active_set(tsd_tsdn(tsd), *(bool *)newp); 2373 } else { 2374 oldval = prof_active_get(tsd_tsdn(tsd)); 2375 } 2376 READ(oldval, bool); 2377 2378 ret = 0; 2379 label_return: 2380 return ret; 2381 } 2382 2383 static int 2384 prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2385 size_t *oldlenp, void *newp, size_t newlen) { 2386 int ret; 2387 const char *filename = NULL; 2388 2389 if (!config_prof) { 2390 return ENOENT; 2391 } 2392 2393 WRITEONLY(); 2394 WRITE(filename, const char *); 2395 2396 if (prof_mdump(tsd, filename)) { 2397 ret = EFAULT; 2398 goto label_return; 2399 } 2400 2401 ret = 0; 2402 label_return: 2403 return ret; 2404 } 2405 2406 static int 2407 prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2408 size_t *oldlenp, void *newp, size_t newlen) { 2409 int ret; 2410 bool oldval; 2411 2412 if (!config_prof) { 2413 return ENOENT; 2414 } 2415 2416 if (newp != NULL) { 2417 if (newlen != sizeof(bool)) { 2418 ret = EINVAL; 2419 goto label_return; 2420 } 2421 oldval = prof_gdump_set(tsd_tsdn(tsd), *(bool *)newp); 2422 } else { 2423 oldval = prof_gdump_get(tsd_tsdn(tsd)); 2424 } 2425 READ(oldval, bool); 2426 2427 ret = 0; 2428 label_return: 2429 return ret; 2430 } 2431 2432 static int 2433 prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2434 size_t *oldlenp, void *newp, size_t newlen) { 2435 int ret; 2436 size_t lg_sample = lg_prof_sample; 2437 2438 if (!config_prof) { 2439 return ENOENT; 2440 } 2441 2442 WRITEONLY(); 2443 WRITE(lg_sample, size_t); 2444 if (lg_sample >= (sizeof(uint64_t) << 3)) { 2445 lg_sample = (sizeof(uint64_t) << 3) - 1; 2446 } 2447 2448 prof_reset(tsd, lg_sample); 2449 2450 ret = 0; 2451 label_return: 2452 return ret; 2453 } 2454 2455 CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) 2456 CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) 2457 2458 /******************************************************************************/ 2459 2460 CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) 2461 CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t) 2462 CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t) 2463 CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t) 2464 CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t) 2465 CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t) 2466 2467 CTL_RO_CGEN(config_stats, stats_background_thread_num_threads, 2468 ctl_stats->background_thread.num_threads, size_t) 2469 CTL_RO_CGEN(config_stats, stats_background_thread_num_runs, 2470 ctl_stats->background_thread.num_runs, uint64_t) 2471 CTL_RO_CGEN(config_stats, stats_background_thread_run_interval, 2472 nstime_ns(&ctl_stats->background_thread.run_interval), uint64_t) 2473 2474 CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *) 2475 CTL_RO_GEN(stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms, 2476 ssize_t) 2477 CTL_RO_GEN(stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms, 2478 ssize_t) 2479 CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned) 2480 CTL_RO_GEN(stats_arenas_i_uptime, 2481 nstime_ns(&arenas_i(mib[2])->astats->astats.uptime), uint64_t) 2482 CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t) 2483 CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t) 2484 CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t) 2485 CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, 2486 atomic_load_zu(&arenas_i(mib[2])->astats->astats.mapped, ATOMIC_RELAXED), 2487 size_t) 2488 CTL_RO_CGEN(config_stats, stats_arenas_i_retained, 2489 atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), 2490 size_t) 2491 2492 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, 2493 arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_dirty.npurge), 2494 uint64_t) 2495 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise, 2496 arena_stats_read_u64( 2497 &arenas_i(mib[2])->astats->astats.decay_dirty.nmadvise), uint64_t) 2498 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged, 2499 arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_dirty.purged), 2500 uint64_t) 2501 2502 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge, 2503 arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_muzzy.npurge), 2504 uint64_t) 2505 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise, 2506 arena_stats_read_u64( 2507 &arenas_i(mib[2])->astats->astats.decay_muzzy.nmadvise), uint64_t) 2508 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged, 2509 arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_muzzy.purged), 2510 uint64_t) 2511 2512 CTL_RO_CGEN(config_stats, stats_arenas_i_base, 2513 atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED), 2514 size_t) 2515 CTL_RO_CGEN(config_stats, stats_arenas_i_internal, 2516 atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED), 2517 size_t) 2518 CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, 2519 atomic_load_zu(&arenas_i(mib[2])->astats->astats.tcache_bytes, 2520 ATOMIC_RELAXED), size_t) 2521 CTL_RO_CGEN(config_stats, stats_arenas_i_resident, 2522 atomic_load_zu(&arenas_i(mib[2])->astats->astats.resident, ATOMIC_RELAXED), 2523 size_t) 2524 2525 CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, 2526 arenas_i(mib[2])->astats->allocated_small, size_t) 2527 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, 2528 arenas_i(mib[2])->astats->nmalloc_small, uint64_t) 2529 CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, 2530 arenas_i(mib[2])->astats->ndalloc_small, uint64_t) 2531 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, 2532 arenas_i(mib[2])->astats->nrequests_small, uint64_t) 2533 CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, 2534 atomic_load_zu(&arenas_i(mib[2])->astats->astats.allocated_large, 2535 ATOMIC_RELAXED), size_t) 2536 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, 2537 arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.nmalloc_large), 2538 uint64_t) 2539 CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, 2540 arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.ndalloc_large), 2541 uint64_t) 2542 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, 2543 arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.nmalloc_large), 2544 uint64_t) /* Intentional. */ 2545 2546 /* Lock profiling related APIs below. */ 2547 #define RO_MUTEX_CTL_GEN(n, l) \ 2548 CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \ 2549 l.n_lock_ops, uint64_t) \ 2550 CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \ 2551 l.n_wait_times, uint64_t) \ 2552 CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \ 2553 l.n_spin_acquired, uint64_t) \ 2554 CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \ 2555 l.n_owner_switches, uint64_t) \ 2556 CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \ 2557 nstime_ns(&l.tot_wait_time), uint64_t) \ 2558 CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ 2559 nstime_ns(&l.max_wait_time), uint64_t) \ 2560 CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ 2561 l.max_n_thds, uint32_t) 2562 2563 /* Global mutexes. */ 2564 #define OP(mtx) \ 2565 RO_MUTEX_CTL_GEN(mutexes_##mtx, \ 2566 ctl_stats->mutex_prof_data[global_prof_mutex_##mtx]) 2567 MUTEX_PROF_GLOBAL_MUTEXES 2568 #undef OP 2569 2570 /* Per arena mutexes */ 2571 #define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \ 2572 arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx]) 2573 MUTEX_PROF_ARENA_MUTEXES 2574 #undef OP 2575 2576 /* tcache bin mutex */ 2577 RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, 2578 arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) 2579 #undef RO_MUTEX_CTL_GEN 2580 2581 /* Resets all mutex stats, including global, arena and bin mutexes. */ 2582 static int 2583 stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2584 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2585 if (!config_stats) { 2586 return ENOENT; 2587 } 2588 2589 tsdn_t *tsdn = tsd_tsdn(tsd); 2590 2591 #define MUTEX_PROF_RESET(mtx) \ 2592 malloc_mutex_lock(tsdn, &mtx); \ 2593 malloc_mutex_prof_data_reset(tsdn, &mtx); \ 2594 malloc_mutex_unlock(tsdn, &mtx); 2595 2596 /* Global mutexes: ctl and prof. */ 2597 MUTEX_PROF_RESET(ctl_mtx); 2598 if (have_background_thread) { 2599 MUTEX_PROF_RESET(background_thread_lock); 2600 } 2601 if (config_prof && opt_prof) { 2602 MUTEX_PROF_RESET(bt2gctx_mtx); 2603 } 2604 2605 2606 /* Per arena mutexes. */ 2607 unsigned n = narenas_total_get(); 2608 2609 for (unsigned i = 0; i < n; i++) { 2610 arena_t *arena = arena_get(tsdn, i, false); 2611 if (!arena) { 2612 continue; 2613 } 2614 MUTEX_PROF_RESET(arena->large_mtx); 2615 MUTEX_PROF_RESET(arena->extent_avail_mtx); 2616 MUTEX_PROF_RESET(arena->extents_dirty.mtx); 2617 MUTEX_PROF_RESET(arena->extents_muzzy.mtx); 2618 MUTEX_PROF_RESET(arena->extents_retained.mtx); 2619 MUTEX_PROF_RESET(arena->decay_dirty.mtx); 2620 MUTEX_PROF_RESET(arena->decay_muzzy.mtx); 2621 MUTEX_PROF_RESET(arena->tcache_ql_mtx); 2622 MUTEX_PROF_RESET(arena->base->mtx); 2623 2624 for (szind_t i = 0; i < NBINS; i++) { 2625 arena_bin_t *bin = &arena->bins[i]; 2626 MUTEX_PROF_RESET(bin->lock); 2627 } 2628 } 2629 #undef MUTEX_PROF_RESET 2630 return 0; 2631 } 2632 2633 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, 2634 arenas_i(mib[2])->astats->bstats[mib[4]].nmalloc, uint64_t) 2635 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc, 2636 arenas_i(mib[2])->astats->bstats[mib[4]].ndalloc, uint64_t) 2637 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, 2638 arenas_i(mib[2])->astats->bstats[mib[4]].nrequests, uint64_t) 2639 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs, 2640 arenas_i(mib[2])->astats->bstats[mib[4]].curregs, size_t) 2641 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills, 2642 arenas_i(mib[2])->astats->bstats[mib[4]].nfills, uint64_t) 2643 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes, 2644 arenas_i(mib[2])->astats->bstats[mib[4]].nflushes, uint64_t) 2645 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs, 2646 arenas_i(mib[2])->astats->bstats[mib[4]].nslabs, uint64_t) 2647 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs, 2648 arenas_i(mib[2])->astats->bstats[mib[4]].reslabs, uint64_t) 2649 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, 2650 arenas_i(mib[2])->astats->bstats[mib[4]].curslabs, size_t) 2651 2652 static const ctl_named_node_t * 2653 stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 2654 size_t j) { 2655 if (j > NBINS) { 2656 return NULL; 2657 } 2658 return super_stats_arenas_i_bins_j_node; 2659 } 2660 2661 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc, 2662 arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), 2663 uint64_t) 2664 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc, 2665 arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), 2666 uint64_t) 2667 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests, 2668 arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), 2669 uint64_t) 2670 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, 2671 arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) 2672 2673 static const ctl_named_node_t * 2674 stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 2675 size_t j) { 2676 if (j > NSIZES - NBINS) { 2677 return NULL; 2678 } 2679 return super_stats_arenas_i_lextents_j_node; 2680 } 2681 2682 static const ctl_named_node_t * 2683 stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { 2684 const ctl_named_node_t *ret; 2685 size_t a; 2686 2687 malloc_mutex_lock(tsdn, &ctl_mtx); 2688 a = arenas_i2a_impl(i, true, true); 2689 if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) { 2690 ret = NULL; 2691 goto label_return; 2692 } 2693 2694 ret = super_stats_arenas_i_node; 2695 label_return: 2696 malloc_mutex_unlock(tsdn, &ctl_mtx); 2697 return ret; 2698 } 2699