/netbsd/external/bsd/jemalloc/dist/include/jemalloc/internal/ |
H A D | arena_inlines_b.h | 24 return large_prof_tctx_get(tsdn, iealloc(tsdn, ptr)); in arena_prof_tctx_get() 44 large_prof_tctx_set(tsdn, iealloc(tsdn, ptr), tctx); in arena_prof_tctx_set() 65 if (unlikely(tsdn_null(tsdn))) { in arena_decay_ticks() 68 tsd = tsdn_tsd(tsdn); in arena_decay_ticks() 83 arena_decay_ticks(tsdn, arena, 1); in arena_decay_tick() 182 arena_dalloc_small(tsdn, ptr); in arena_dalloc_no_tcache() 185 large_dalloc(tsdn, extent); in arena_dalloc_no_tcache() 237 large_dalloc(tsdn, extent); in arena_dalloc() 279 arena_dalloc_small(tsdn, ptr); in arena_sdalloc_no_tcache() 282 large_dalloc(tsdn, extent); in arena_sdalloc_no_tcache() [all …]
|
H A D | jemalloc_internal_inlines_c.h | 27 iaalloc(tsdn_t *tsdn, const void *ptr) { in iaalloc() argument 30 return arena_aalloc(tsdn, ptr); in iaalloc() 34 isalloc(tsdn_t *tsdn, const void *ptr) { in isalloc() argument 37 return arena_salloc(tsdn, ptr); in isalloc() 48 if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) { in iallocztm() 55 arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); in iallocztm() 81 arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); in ipallocztm() 99 ivsalloc(tsdn_t *tsdn, const void *ptr) { in ivsalloc() argument 100 return arena_vsalloc(tsdn, ptr); in ivsalloc() 112 arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); in idalloctm() [all …]
|
H A D | mutex_pool.h | 42 mutex_pool_assert_not_held(tsdn, pool); in mutex_pool_lock() 45 malloc_mutex_lock(tsdn, mutex); in mutex_pool_lock() 51 malloc_mutex_unlock(tsdn, mutex); in mutex_pool_unlock() 64 malloc_mutex_lock(tsdn, mutex1); in mutex_pool_lock2() 65 malloc_mutex_lock(tsdn, mutex2); in mutex_pool_lock2() 67 malloc_mutex_lock(tsdn, mutex1); in mutex_pool_lock2() 69 malloc_mutex_lock(tsdn, mutex2); in mutex_pool_lock2() 70 malloc_mutex_lock(tsdn, mutex1); in mutex_pool_lock2() 80 malloc_mutex_unlock(tsdn, mutex1); in mutex_pool_unlock2() 82 malloc_mutex_unlock(tsdn, mutex1); in mutex_pool_unlock2() [all …]
|
H A D | arena_externs.h | 19 void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, 64 void arena_dalloc_small(tsdn_t *tsdn, void *ptr); 83 void arena_prefork0(tsdn_t *tsdn, arena_t *arena); 84 void arena_prefork1(tsdn_t *tsdn, arena_t *arena); 85 void arena_prefork2(tsdn_t *tsdn, arena_t *arena); 86 void arena_prefork3(tsdn_t *tsdn, arena_t *arena); 87 void arena_prefork4(tsdn_t *tsdn, arena_t *arena); 88 void arena_prefork5(tsdn_t *tsdn, arena_t *arena); 89 void arena_prefork6(tsdn_t *tsdn, arena_t *arena); 90 void arena_prefork7(tsdn_t *tsdn, arena_t *arena); [all …]
|
H A D | rtree.h | 126 void rtree_delete(tsdn_t *tsdn, rtree_t *rtree); 303 rtree_leaf_elm_slab_write(tsdn, rtree, elm, slab); in rtree_leaf_elm_write() 304 rtree_leaf_elm_szind_write(tsdn, rtree, elm, szind); in rtree_leaf_elm_write() 309 rtree_leaf_elm_extent_write(tsdn, rtree, elm, extent); in rtree_leaf_elm_write() 314 rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, in rtree_leaf_elm_szind_slab_update() argument 322 rtree_leaf_elm_slab_write(tsdn, rtree, elm, slab); in rtree_leaf_elm_szind_slab_update() 323 rtree_leaf_elm_szind_write(tsdn, rtree, elm, szind); in rtree_leaf_elm_szind_slab_update() 381 return rtree_leaf_elm_lookup_hard(tsdn, rtree, rtree_ctx, key, in rtree_leaf_elm_lookup() 398 rtree_leaf_elm_write(tsdn, rtree, elm, extent, szind, slab); in rtree_write() 487 assert(rtree_leaf_elm_extent_read(tsdn, rtree, elm, false) != in rtree_clear() [all …]
|
H A D | arena_stats.h | 117 malloc_mutex_lock(tsdn, &arena_stats->mtx); in arena_stats_lock() 124 malloc_mutex_unlock(tsdn, &arena_stats->mtx); in arena_stats_unlock() 134 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_read_u64() 145 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_add_u64() 157 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_sub_u64() 183 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_read_zu() 194 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_add_zu() 223 arena_stats_lock(tsdn, arena_stats); in arena_stats_large_nrequests_add() 226 arena_stats_unlock(tsdn, arena_stats); in arena_stats_large_nrequests_add() 231 arena_stats_lock(tsdn, arena_stats); in arena_stats_mapped_add() [all …]
|
H A D | extent_externs.h | 15 extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); 34 extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, 38 void extents_dalloc(tsdn_t *tsdn, arena_t *arena, 40 extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, 42 void extents_prefork(tsdn_t *tsdn, extents_t *extents); 45 extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, 49 void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, 51 void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, 53 bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, 56 bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, [all …]
|
H A D | mutex.h | 166 if (data->prev_owner != tsdn) { in mutex_owner_stats_update() 167 data->prev_owner = tsdn; in mutex_owner_stats_update() 175 malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { in malloc_mutex_trylock() argument 181 mutex_owner_stats_update(tsdn, mutex); in malloc_mutex_trylock() 183 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); in malloc_mutex_trylock() 213 malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { in malloc_mutex_lock() argument 219 mutex_owner_stats_update(tsdn, mutex); in malloc_mutex_lock() 221 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); in malloc_mutex_lock() 225 malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { in malloc_mutex_unlock() argument 244 malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, in malloc_mutex_prof_read() argument [all …]
|
/netbsd/external/bsd/jemalloc/include/jemalloc/internal/ |
H A D | arena_inlines_b.h | 24 return large_prof_tctx_get(tsdn, iealloc(tsdn, ptr)); in arena_prof_tctx_get() 44 large_prof_tctx_set(tsdn, iealloc(tsdn, ptr), tctx); in arena_prof_tctx_set() 65 if (unlikely(tsdn_null(tsdn))) { in arena_decay_ticks() 68 tsd = tsdn_tsd(tsdn); in arena_decay_ticks() 83 arena_decay_ticks(tsdn, arena, 1); in arena_decay_tick() 182 arena_dalloc_small(tsdn, ptr); in arena_dalloc_no_tcache() 185 large_dalloc(tsdn, extent); in arena_dalloc_no_tcache() 237 large_dalloc(tsdn, extent); in arena_dalloc() 279 arena_dalloc_small(tsdn, ptr); in arena_sdalloc_no_tcache() 282 large_dalloc(tsdn, extent); in arena_sdalloc_no_tcache() [all …]
|
H A D | jemalloc_internal_inlines_c.h | 27 iaalloc(tsdn_t *tsdn, const void *ptr) { in iaalloc() argument 30 return arena_aalloc(tsdn, ptr); in iaalloc() 34 isalloc(tsdn_t *tsdn, const void *ptr) { in isalloc() argument 37 return arena_salloc(tsdn, ptr); in isalloc() 48 if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) { in iallocztm() 55 arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); in iallocztm() 81 arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); in ipallocztm() 99 ivsalloc(tsdn_t *tsdn, const void *ptr) { in ivsalloc() argument 100 return arena_vsalloc(tsdn, ptr); in ivsalloc() 112 arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); in idalloctm() [all …]
|
H A D | mutex_pool.h | 42 mutex_pool_assert_not_held(tsdn, pool); in mutex_pool_lock() 45 malloc_mutex_lock(tsdn, mutex); in mutex_pool_lock() 51 malloc_mutex_unlock(tsdn, mutex); in mutex_pool_unlock() 64 malloc_mutex_lock(tsdn, mutex1); in mutex_pool_lock2() 65 malloc_mutex_lock(tsdn, mutex2); in mutex_pool_lock2() 67 malloc_mutex_lock(tsdn, mutex1); in mutex_pool_lock2() 69 malloc_mutex_lock(tsdn, mutex2); in mutex_pool_lock2() 70 malloc_mutex_lock(tsdn, mutex1); in mutex_pool_lock2() 80 malloc_mutex_unlock(tsdn, mutex1); in mutex_pool_unlock2() 82 malloc_mutex_unlock(tsdn, mutex1); in mutex_pool_unlock2() [all …]
|
H A D | arena_externs.h | 19 void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, 64 void arena_dalloc_small(tsdn_t *tsdn, void *ptr); 83 void arena_prefork0(tsdn_t *tsdn, arena_t *arena); 84 void arena_prefork1(tsdn_t *tsdn, arena_t *arena); 85 void arena_prefork2(tsdn_t *tsdn, arena_t *arena); 86 void arena_prefork3(tsdn_t *tsdn, arena_t *arena); 87 void arena_prefork4(tsdn_t *tsdn, arena_t *arena); 88 void arena_prefork5(tsdn_t *tsdn, arena_t *arena); 89 void arena_prefork6(tsdn_t *tsdn, arena_t *arena); 90 void arena_prefork7(tsdn_t *tsdn, arena_t *arena); [all …]
|
H A D | rtree.h | 126 void rtree_delete(tsdn_t *tsdn, rtree_t *rtree); 303 rtree_leaf_elm_slab_write(tsdn, rtree, elm, slab); in rtree_leaf_elm_write() 304 rtree_leaf_elm_szind_write(tsdn, rtree, elm, szind); in rtree_leaf_elm_write() 309 rtree_leaf_elm_extent_write(tsdn, rtree, elm, extent); in rtree_leaf_elm_write() 314 rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, in rtree_leaf_elm_szind_slab_update() argument 322 rtree_leaf_elm_slab_write(tsdn, rtree, elm, slab); in rtree_leaf_elm_szind_slab_update() 323 rtree_leaf_elm_szind_write(tsdn, rtree, elm, szind); in rtree_leaf_elm_szind_slab_update() 381 return rtree_leaf_elm_lookup_hard(tsdn, rtree, rtree_ctx, key, in rtree_leaf_elm_lookup() 398 rtree_leaf_elm_write(tsdn, rtree, elm, extent, szind, slab); in rtree_write() 487 assert(rtree_leaf_elm_extent_read(tsdn, rtree, elm, false) != in rtree_clear() [all …]
|
H A D | arena_stats.h | 117 malloc_mutex_lock(tsdn, &arena_stats->mtx); in arena_stats_lock() 124 malloc_mutex_unlock(tsdn, &arena_stats->mtx); in arena_stats_unlock() 134 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_read_u64() 145 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_add_u64() 157 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_sub_u64() 183 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_read_zu() 194 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); in arena_stats_add_zu() 223 arena_stats_lock(tsdn, arena_stats); in arena_stats_large_nrequests_add() 226 arena_stats_unlock(tsdn, arena_stats); in arena_stats_large_nrequests_add() 231 arena_stats_lock(tsdn, arena_stats); in arena_stats_mapped_add() [all …]
|
H A D | extent_externs.h | 15 extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); 34 extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, 38 void extents_dalloc(tsdn_t *tsdn, arena_t *arena, 40 extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, 42 void extents_prefork(tsdn_t *tsdn, extents_t *extents); 45 extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, 49 void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, 51 void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, 53 bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, 56 bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, [all …]
|
H A D | mutex.h | 166 if (data->prev_owner != tsdn) { in mutex_owner_stats_update() 167 data->prev_owner = tsdn; in mutex_owner_stats_update() 175 malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { in malloc_mutex_trylock() argument 181 mutex_owner_stats_update(tsdn, mutex); in malloc_mutex_trylock() 183 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); in malloc_mutex_trylock() 213 malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { in malloc_mutex_lock() argument 219 mutex_owner_stats_update(tsdn, mutex); in malloc_mutex_lock() 221 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); in malloc_mutex_lock() 225 malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { in malloc_mutex_unlock() argument 244 malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, in malloc_mutex_prof_read() argument [all …]
|
/netbsd/external/bsd/jemalloc/dist/src/ |
H A D | extent.c | 144 extent_lock(tsdn, extent1); in extent_rtree_leaf_elm_try_lock() 152 extent_unlock(tsdn, extent1); in extent_rtree_leaf_elm_try_lock() 721 prof_gdump(tsdn); in extent_gdump_add() 747 extent_lock(tsdn, extent); in extent_register_impl() 762 extent_unlock(tsdn, extent); in extent_register_impl() 819 extent_lock(tsdn, extent); in extent_deregister_impl() 1183 tsdn_t *tsdn; in extent_alloc_default() local 1186 tsdn = tsdn_fetch(); in extent_alloc_default() 1200 tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); in extent_hook_pre_reentrancy() 1217 tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); in extent_hook_post_reentrancy() [all …]
|
H A D | large.c | 28 assert(!tsdn_null(tsdn) || arena != NULL); in large_palloc() 44 if (likely(!tsdn_null(tsdn))) { in large_palloc() 45 arena = arena_choose(tsdn_tsd(tsdn), arena); in large_palloc() 55 malloc_mutex_lock(tsdn, &arena->large_mtx); in large_palloc() 57 malloc_mutex_unlock(tsdn, &arena->large_mtx); in large_palloc() 60 prof_idump(tsdn); in large_palloc() 70 arena_decay_tick(tsdn, arena); in large_palloc() 314 malloc_mutex_lock(tsdn, &arena->large_mtx); in large_dalloc_prep_impl() 346 large_dalloc(tsdn_t *tsdn, extent_t *extent) { in large_dalloc() argument 349 large_dalloc_finish_impl(tsdn, arena, extent); in large_dalloc() [all …]
|
H A D | arena.c | 170 malloc_mutex_prof_read(tsdn, in arena_stats_merge() 1261 prof_idump(tsdn); in arena_tcache_fill_small() 1302 arena_decay_tick(tsdn, arena); in arena_tcache_fill_small() 1349 prof_idump(tsdn); in arena_malloc_small() 1369 arena_decay_tick(tsdn, arena); in arena_malloc_small() 1378 if (likely(!tsdn_null(tsdn))) { in arena_malloc_hard() 1463 large_dalloc(tsdn, extent); in arena_dalloc_promoted() 1581 arena_decay_tick(tsdn, arena); in arena_dalloc_small() 1650 return large_ralloc(tsdn, arena, iealloc(tsdn, ptr), usize, in arena_ralloc() 1904 assert(!tsdn_null(tsdn)); in arena_new() [all …]
|
H A D | background_thread.c | 197 malloc_mutex_unlock(tsdn, &decay->mtx); 276 malloc_mutex_unlock(tsdn, &info->mtx); 280 malloc_mutex_lock(tsdn, &info->mtx); 297 arena_decay(tsdn, arena, true, false); 393 tsdn_t *tsdn = tsd_tsdn(tsd); 400 malloc_mutex_lock(tsdn, &info->mtx); 743 malloc_mutex_postfork_parent(tsdn, 752 malloc_mutex_postfork_child(tsdn, 766 malloc_mutex_lock(tsdn, &info->mtx); 790 malloc_mutex_lock(tsdn, &info->mtx); [all …]
|
H A D | rtree.c | 82 rtree_leaf_dalloc(tsdn, rtree, leaf); in rtree_delete_subtree() 88 rtree_node_dalloc(tsdn, rtree, subtree); in rtree_delete_subtree() 94 rtree_delete(tsdn_t *tsdn, rtree_t *rtree) { in rtree_delete() argument 104 malloc_mutex_lock(tsdn, &rtree->init_lock); in rtree_node_init() 111 node = rtree_node_alloc(tsdn, rtree, ZU(1) << in rtree_node_init() 123 malloc_mutex_unlock(tsdn, &rtree->init_lock); in rtree_node_init() 130 malloc_mutex_lock(tsdn, &rtree->init_lock); in rtree_leaf_init() 137 leaf = rtree_leaf_alloc(tsdn, rtree, ZU(1) << in rtree_leaf_init() 149 malloc_mutex_unlock(tsdn, &rtree->init_lock); in rtree_leaf_init() 252 rtree_child_node_read(tsdn, rtree, \ in rtree_leaf_elm_lookup_hard() [all …]
|
H A D | prof.c | 941 tsdn_t *tsdn; in prof_tdata_count() local 943 tsdn = tsdn_fetch(); in prof_tdata_count() 1138 tsdn_t *tsdn; member 1215 tsdn_t *tsdn; member 1226 (void *)arg->tsdn); in prof_gctx_merge_iter() 1282 tsdn_t *tsdn; member 1407 prof_tctx_dump_iter_arg.tsdn = tsdn; in prof_dump_gctx() 1544 tsdn_t *tsdn; member 1799 prof_idump(tsdn_t *tsdn) { in prof_idump() argument 1808 tsd = tsdn_tsd(tsdn); in prof_idump() [all …]
|
H A D | base.c | 44 tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); in base_map() 83 tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); in base_unmap() 285 malloc_mutex_lock(tsdn, &base->mtx); in base_block_alloc() 286 base_auto_thp_switch(tsdn, base); in base_block_alloc() 316 malloc_mutex_unlock(tsdn, &base->mtx); in base_extent_alloc() 320 malloc_mutex_lock(tsdn, &base->mtx); in base_extent_alloc() 428 malloc_mutex_lock(tsdn, &base->mtx); in base_alloc_impl() 451 malloc_mutex_unlock(tsdn, &base->mtx); in base_alloc_impl() 485 malloc_mutex_lock(tsdn, &base->mtx); in base_stats_get() 492 malloc_mutex_unlock(tsdn, &base->mtx); in base_stats_get() [all …]
|
/netbsd/external/bsd/jemalloc/dist/test/unit/ |
H A D | rtree.c | 24 malloc_mutex_lock(tsdn, &rtree->init_lock); in rtree_node_alloc_intercept() 33 rtree_node_dalloc_orig(tsdn, rtree, node); in rtree_node_dalloc_intercept() 68 tsdn_t *tsdn; in TEST_BEGIN() local 70 tsdn = tsdn_fetch(); in TEST_BEGIN() 78 rtree_delete(tsdn, rtree); in TEST_BEGIN() 94 tsdn_t *tsdn = tsdn_fetch(); in TEST_BEGIN() local 117 rtree_delete(tsdn, rtree); in TEST_BEGIN() 122 tsdn_t *tsdn = tsdn_fetch(); in TEST_BEGIN() local 154 rtree_delete(tsdn, rtree); in TEST_BEGIN() 162 tsdn_t *tsdn = tsdn_fetch(); in TEST_BEGIN() local [all …]
|
H A D | base.c | 33 tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); in TEST_BEGIN() local 57 base_delete(tsdn, base); in TEST_BEGIN() 75 tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); in TEST_BEGIN() local 76 base = base_new(tsdn, 0, &hooks); in TEST_BEGIN() 100 base_delete(tsdn, base); in TEST_BEGIN() 120 tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); in TEST_BEGIN() local 122 base = base_new(tsdn, 0, &hooks); in TEST_BEGIN() 144 p = base_alloc(tsdn, base, 1, alignment); in TEST_BEGIN() 156 r = base_alloc(tsdn, base, 1, alignment); in TEST_BEGIN() 183 r = base_alloc(tsdn, base, QUANTUM, QUANTUM); in TEST_BEGIN() [all …]
|