142 static bool	prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
144 static void	prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
212 	prof_tdata_t *tdata;  in rb_gen()  local
223 		tdata = prof_tdata_get(tsd, true);  in rb_gen()
224 		if (tdata != NULL) {  in rb_gen()
225 			prof_sample_threshold_update(tdata);  in rb_gen()
230 		malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);  in rb_gen()
235 			malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);  in rb_gen()
245 	malloc_mutex_lock(tsdn, tctx->tdata->lock);  in prof_malloc_sample_object()
253 	malloc_mutex_unlock(tsdn, tctx->tdata->lock);  in prof_malloc_sample_object()
258 	malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_free_sampled_object()
267 		malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_free_sampled_object()
280 prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_enter()  argument
282 	assert(tdata == prof_tdata_get(tsd, false));  in prof_enter()
284 	if (tdata != NULL) {  in prof_enter()
285 		assert(!tdata->enq);  in prof_enter()
286 		tdata->enq = true;  in prof_enter()
293 prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_leave()  argument
295 	assert(tdata == prof_tdata_get(tsd, false));  in prof_leave()
299 	if (tdata != NULL) {  in prof_leave()
302 		assert(tdata->enq);  in prof_leave()
303 		tdata->enq = false;  in prof_leave()
304 		idump = tdata->enq_idump;  in prof_leave()
305 		tdata->enq_idump = false;  in prof_leave()
306 		gdump = tdata->enq_gdump;  in prof_leave()
307 		tdata->enq_gdump = false;  in prof_leave()
580     prof_tdata_t *tdata) {  in prof_gctx_try_destroy()
615 	malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);  in prof_tctx_should_destroy()
645 	prof_tdata_t *tdata = tctx->tdata;  in prof_tctx_destroy()  local
649 	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_tctx_destroy()
657 	ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);  in prof_tctx_destroy()
658 	destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false);  in prof_tctx_destroy()
659 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_tctx_destroy()
705 		    tdata);  in prof_tctx_destroy()
708 	malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);  in prof_tctx_destroy()
711 		prof_tdata_destroy(tsd, tdata, false);  in prof_tctx_destroy()
720 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,  in prof_lookup_global()  argument
732 	prof_enter(tsd, tdata);  in prof_lookup_global()
735 		prof_leave(tsd, tdata);  in prof_lookup_global()
740 		prof_enter(tsd, tdata);  in prof_lookup_global()
746 				prof_leave(tsd, tdata);  in prof_lookup_global()
776 	prof_leave(tsd, tdata);  in prof_lookup_global()
790 	prof_tdata_t *tdata;  in prof_lookup()  local
795 	tdata = prof_tdata_get(tsd, false);  in prof_lookup()
796 	if (tdata == NULL) {  in prof_lookup()
800 	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
801 	not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);  in prof_lookup()
805 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
815 		if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,  in prof_lookup()
826 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_lookup()
830 		ret.p->tdata = tdata;  in prof_lookup()
831 		ret.p->thr_uid = tdata->thr_uid;  in prof_lookup()
832 		ret.p->thr_discrim = tdata->thr_discrim;  in prof_lookup()
835 		ret.p->tctx_uid = tdata->tctx_uid_next++;  in prof_lookup()
838 		malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
839 		error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);  in prof_lookup()
840 		malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_lookup()
843 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_lookup()
872 prof_sample_threshold_update(prof_tdata_t *tdata) {  in prof_sample_threshold_update()  argument
882 		tdata->bytes_until_sample = 0;  in prof_sample_threshold_update()
904 	r = prng_lg_range_u64(&tdata->prng_state, 53);  in prof_sample_threshold_update()
906 	tdata->bytes_until_sample = (uint64_t)(log(u) /  in prof_sample_threshold_update()
914 prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,  in prof_tdata_count_iter()  argument
941 	prof_tdata_t *tdata;  in prof_bt_count()  local
944 	tdata = prof_tdata_get(tsd, false);  in prof_bt_count()
945 	if (tdata == NULL) {  in prof_bt_count()
1056 prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {  in prof_tctx_merge_tdata()  argument
1057 	malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);  in prof_tctx_merge_tdata()
1071 		tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;  in prof_tctx_merge_tdata()
1072 		tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;  in prof_tctx_merge_tdata()
1074 			tdata->cnt_summed.accumobjs +=  in prof_tctx_merge_tdata()
1076 			tdata->cnt_summed.accumbytes +=  in prof_tctx_merge_tdata()
1220 	prof_tdata_t *tdata = prof_tdata_get(tsd, false);  in prof_gctx_finish()  local
1257 			prof_gctx_try_destroy(tsd, tdata, gctx, tdata);  in prof_gctx_finish()
1270 prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,  in prof_tdata_merge_iter()  argument
1275 	malloc_mutex_lock(arg->tsdn, tdata->lock);  in prof_tdata_merge_iter()
1276 	if (!tdata->expired) {  in prof_tdata_merge_iter()
1283 		tdata->dumping = true;  in prof_tdata_merge_iter()
1284 		memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));  in prof_tdata_merge_iter()
1285 		for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,  in prof_tdata_merge_iter()
1287 			prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);  in prof_tdata_merge_iter()
1290 		arg->cnt_all.curobjs += tdata->cnt_summed.curobjs;  in prof_tdata_merge_iter()
1291 		arg->cnt_all.curbytes += tdata->cnt_summed.curbytes;  in prof_tdata_merge_iter()
1293 			arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs;  in prof_tdata_merge_iter()
1294 			arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes;  in prof_tdata_merge_iter()
1297 		tdata->dumping = false;  in prof_tdata_merge_iter()
1299 	malloc_mutex_unlock(arg->tsdn, tdata->lock);  in prof_tdata_merge_iter()
1305 prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,  in prof_tdata_dump_iter()  argument
1309 	if (!tdata->dumping) {  in prof_tdata_dump_iter()
1315 	    tdata->thr_uid, tdata->cnt_summed.curobjs,  in prof_tdata_dump_iter()
1316 	    tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,  in prof_tdata_dump_iter()
1317 	    tdata->cnt_summed.accumbytes,  in prof_tdata_dump_iter()
1318 	    (tdata->thread_name != NULL) ? " " : "",  in prof_tdata_dump_iter()
1319 	    (tdata->thread_name != NULL) ? tdata->thread_name : "")) {  in prof_tdata_dump_iter()
1320 		return tdata;  in prof_tdata_dump_iter()
1552 prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata,  in prof_dump_prep()  argument
1562 	prof_enter(tsd, tdata);  in prof_dump_prep()
1590 	prof_leave(tsd, tdata);  in prof_dump_prep()
1595     bool leakcheck, prof_tdata_t *tdata,  in prof_dump_file()  argument
1640 	prof_tdata_t * tdata = prof_tdata_get(tsd, true);  in prof_dump()  local
1641 	if (tdata == NULL) {  in prof_dump()
1652 	prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,  in prof_dump()
1654 	bool err = prof_dump_file(tsd, propagate_err, filename, leakcheck, tdata,  in prof_dump()
1678 	prof_tdata_t *tdata;  in prof_cnt_all()  local
1684 	tdata = prof_tdata_get(tsd, false);  in prof_cnt_all()
1685 	if (tdata == NULL) {  in prof_cnt_all()
1701 	prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,  in prof_cnt_all()
1780 	prof_tdata_t *tdata;  in prof_idump()  local
1792 	tdata = prof_tdata_get(tsd, false);  in prof_idump()
1793 	if (tdata == NULL) {  in prof_idump()
1796 	if (tdata->enq) {  in prof_idump()
1797 		tdata->enq_idump = true;  in prof_idump()
1837 	prof_tdata_t *tdata;  in prof_gdump()  local
1849 	tdata = prof_tdata_get(tsd, false);  in prof_gdump()
1850 	if (tdata == NULL) {  in prof_gdump()
1853 	if (tdata->enq) {  in prof_gdump()
1854 		tdata->enq_gdump = true;  in prof_gdump()
1905 	prof_tdata_t *tdata;  in prof_tdata_init_impl()  local
1910 	tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),  in prof_tdata_init_impl()
1913 	if (tdata == NULL) {  in prof_tdata_init_impl()
1917 	tdata->lock = prof_tdata_mutex_choose(thr_uid);  in prof_tdata_init_impl()
1918 	tdata->thr_uid = thr_uid;  in prof_tdata_init_impl()
1919 	tdata->thr_discrim = thr_discrim;  in prof_tdata_init_impl()
1920 	tdata->thread_name = thread_name;  in prof_tdata_init_impl()
1921 	tdata->attached = true;  in prof_tdata_init_impl()
1922 	tdata->expired = false;  in prof_tdata_init_impl()
1923 	tdata->tctx_uid_next = 0;  in prof_tdata_init_impl()
1925 	if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,  in prof_tdata_init_impl()
1927 		idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);  in prof_tdata_init_impl()
1931 	tdata->prng_state = (uint64_t)(uintptr_t)tdata;  in prof_tdata_init_impl()
1932 	prof_sample_threshold_update(tdata);  in prof_tdata_init_impl()
1934 	tdata->enq = false;  in prof_tdata_init_impl()
1935 	tdata->enq_idump = false;  in prof_tdata_init_impl()
1936 	tdata->enq_gdump = false;  in prof_tdata_init_impl()
1938 	tdata->dumping = false;  in prof_tdata_init_impl()
1939 	tdata->active = active;  in prof_tdata_init_impl()
1942 	tdata_tree_insert(&tdatas, tdata);  in prof_tdata_init_impl()
1945 	return tdata;  in prof_tdata_init_impl()
1955 prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {  in prof_tdata_should_destroy_unlocked()  argument
1956 	if (tdata->attached && !even_if_attached) {  in prof_tdata_should_destroy_unlocked()
1959 	if (ckh_count(&tdata->bt2tctx) != 0) {  in prof_tdata_should_destroy_unlocked()
1966 prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,  in prof_tdata_should_destroy()  argument
1968 	malloc_mutex_assert_owner(tsdn, tdata->lock);  in prof_tdata_should_destroy()
1970 	return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);  in prof_tdata_should_destroy()
1974 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,  in prof_tdata_destroy_locked()  argument
1978 	tdata_tree_remove(&tdatas, tdata);  in prof_tdata_destroy_locked()
1980 	assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));  in prof_tdata_destroy_locked()
1982 	if (tdata->thread_name != NULL) {  in prof_tdata_destroy_locked()
1983 		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,  in prof_tdata_destroy_locked()
1986 	ckh_delete(tsd, &tdata->bt2tctx);  in prof_tdata_destroy_locked()
1987 	idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);  in prof_tdata_destroy_locked()
1991 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {  in prof_tdata_destroy()  argument
1993 	prof_tdata_destroy_locked(tsd, tdata, even_if_attached);  in prof_tdata_destroy()
1998 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_tdata_detach()  argument
2001 	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);  in prof_tdata_detach()
2002 	if (tdata->attached) {  in prof_tdata_detach()
2003 		destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,  in prof_tdata_detach()
2010 			tdata->attached = false;  in prof_tdata_detach()
2016 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);  in prof_tdata_detach()
2018 		prof_tdata_destroy(tsd, tdata, true);  in prof_tdata_detach()
2023 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) {  in prof_tdata_reinit()  argument
2024 	uint64_t thr_uid = tdata->thr_uid;  in prof_tdata_reinit()
2025 	uint64_t thr_discrim = tdata->thr_discrim + 1;  in prof_tdata_reinit()
2026 	char *thread_name = (tdata->thread_name != NULL) ?  in prof_tdata_reinit()
2027 	    prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL;  in prof_tdata_reinit()
2028 	bool active = tdata->active;  in prof_tdata_reinit()
2030 	prof_tdata_detach(tsd, tdata);  in prof_tdata_reinit()
2036 prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {  in prof_tdata_expire()  argument
2039 	malloc_mutex_lock(tsdn, tdata->lock);  in prof_tdata_expire()
2040 	if (!tdata->expired) {  in prof_tdata_expire()
2041 		tdata->expired = true;  in prof_tdata_expire()
2042 		destroy_tdata = tdata->attached ? false :  in prof_tdata_expire()
2043 		    prof_tdata_should_destroy(tsdn, tdata, false);  in prof_tdata_expire()
2047 	malloc_mutex_unlock(tsdn, tdata->lock);  in prof_tdata_expire()
2053 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,  in prof_tdata_reset_iter()  argument
2057 	return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);  in prof_tdata_reset_iter()
2089 	prof_tdata_t *tdata;  in prof_tdata_cleanup()  local
2095 	tdata = tsd_prof_tdata_get(tsd);  in prof_tdata_cleanup()
2096 	if (tdata != NULL) {  in prof_tdata_cleanup()
2097 		prof_tdata_detach(tsd, tdata);  in prof_tdata_cleanup()
2124 	prof_tdata_t *tdata;  in prof_thread_name_get()  local
2126 	tdata = prof_tdata_get(tsd, true);  in prof_thread_name_get()
2127 	if (tdata == NULL) {  in prof_thread_name_get()
2130 	return (tdata->thread_name != NULL ? tdata->thread_name : "");  in prof_thread_name_get()
2158 	prof_tdata_t *tdata;  in prof_thread_name_set()  local
2162 	tdata = prof_tdata_get(tsd, true);  in prof_thread_name_set()
2163 	if (tdata == NULL) {  in prof_thread_name_set()
2183 	if (tdata->thread_name != NULL) {  in prof_thread_name_set()
2184 		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,  in prof_thread_name_set()
2186 		tdata->thread_name = NULL;  in prof_thread_name_set()
2189 		tdata->thread_name = s;  in prof_thread_name_set()
2196 	prof_tdata_t *tdata;  in prof_thread_active_get()  local
2198 	tdata = prof_tdata_get(tsd, true);  in prof_thread_active_get()
2199 	if (tdata == NULL) {  in prof_thread_active_get()
2202 	return tdata->active;  in prof_thread_active_get()
2207 	prof_tdata_t *tdata;  in prof_thread_active_set()  local
2209 	tdata = prof_tdata_get(tsd, true);  in prof_thread_active_set()
2210 	if (tdata == NULL) {  in prof_thread_active_set()
2213 	tdata->active = active;  in prof_thread_active_set()