84 extern inline zap_phys_t *zap_f_phys(zap_t *zap);
86 static uint64_t zap_allocate_blocks(zap_t *zap, int nblocks);
102 fzap_upgrade(zap_t *zap, dmu_tx_t *tx, zap_flags_t flags)  in fzap_upgrade()  argument
104 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));  in fzap_upgrade()
105 	zap->zap_ismicro = FALSE;  in fzap_upgrade()
107 	zap->zap_dbu.dbu_evict_func_sync = zap_evict_sync;  in fzap_upgrade()
108 	zap->zap_dbu.dbu_evict_func_async = NULL;  in fzap_upgrade()
110 	mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, 0, 0);  in fzap_upgrade()
111 	zap->zap_f.zap_block_shift = highbit64(zap->zap_dbuf->db_size) - 1;  in fzap_upgrade()
113 	zap_phys_t *zp = zap_f_phys(zap);  in fzap_upgrade()
118 	bzero(zap->zap_dbuf->db_data, zap->zap_dbuf->db_size);  in fzap_upgrade()
122 	zp->zap_ptrtbl.zt_shift = ZAP_EMBEDDED_PTRTBL_SHIFT(zap);  in fzap_upgrade()
127 	zp->zap_salt = zap->zap_salt;  in fzap_upgrade()
128 	zp->zap_normflags = zap->zap_normflags;  in fzap_upgrade()
133 		ZAP_EMBEDDED_PTRTBL_ENT(zap, i) = 1;  in fzap_upgrade()
139 	VERIFY0(dmu_buf_hold(zap->zap_objset, zap->zap_object,  in fzap_upgrade()
140 	    1<<FZAP_BLOCK_SHIFT(zap), FTAG, &db, DMU_READ_NO_PREFETCH));  in fzap_upgrade()
153 zap_tryupgradedir(zap_t *zap, dmu_tx_t *tx)  in zap_tryupgradedir()  argument
155 	if (RW_WRITE_HELD(&zap->zap_rwlock))  in zap_tryupgradedir()
157 	if (rw_tryupgrade(&zap->zap_rwlock)) {  in zap_tryupgradedir()
158 		dmu_buf_will_dirty(zap->zap_dbuf, tx);  in zap_tryupgradedir()
169 zap_table_grow(zap_t *zap, zap_table_phys_t *tbl,  in zap_table_grow()  argument
174 	int bs = FZAP_BLOCK_SHIFT(zap);  in zap_table_grow()
178 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));  in zap_table_grow()
185 		newblk = zap_allocate_blocks(zap, tbl->zt_numblks * 2);  in zap_table_grow()
188 		dmu_prefetch(zap->zap_objset, zap->zap_object, 0,  in zap_table_grow()
199 	int err = dmu_buf_hold(zap->zap_objset, zap->zap_object,  in zap_table_grow()
206 	VERIFY0(dmu_buf_hold(zap->zap_objset, zap->zap_object,  in zap_table_grow()
213 	VERIFY0(dmu_buf_hold(zap->zap_objset, zap->zap_object,  in zap_table_grow()
228 		(void) dmu_free_range(zap->zap_objset, zap->zap_object,  in zap_table_grow()
245 zap_table_store(zap_t *zap, zap_table_phys_t *tbl, uint64_t idx, uint64_t val,  in zap_table_store()  argument
248 	int bs = FZAP_BLOCK_SHIFT(zap);  in zap_table_store()
250 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));  in zap_table_store()
259 	int err = dmu_buf_hold(zap->zap_objset, zap->zap_object,  in zap_table_store()
271 		err = dmu_buf_hold(zap->zap_objset, zap->zap_object,  in zap_table_store()
291 zap_table_load(zap_t *zap, zap_table_phys_t *tbl, uint64_t idx, uint64_t *valp)  in zap_table_load()  argument
293 	int bs = FZAP_BLOCK_SHIFT(zap);  in zap_table_load()
295 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));  in zap_table_load()
305 	dnode_t *dn = dmu_buf_dnode_enter(zap->zap_dbuf);  in zap_table_load()
309 	dmu_buf_dnode_exit(zap->zap_dbuf);  in zap_table_load()
323 		dn = dmu_buf_dnode_enter(zap->zap_dbuf);  in zap_table_load()
327 		dmu_buf_dnode_exit(zap->zap_dbuf);  in zap_table_load()
349 zap_grow_ptrtbl(zap_t *zap, dmu_tx_t *tx)  in zap_grow_ptrtbl()  argument
357 	if (zap_f_phys(zap)->zap_ptrtbl.zt_shift >= zap_hashbits(zap) - 2)  in zap_grow_ptrtbl()
360 	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {  in zap_grow_ptrtbl()
366 		ASSERT3U(zap_f_phys(zap)->zap_ptrtbl.zt_shift, ==,  in zap_grow_ptrtbl()
367 		    ZAP_EMBEDDED_PTRTBL_SHIFT(zap));  in zap_grow_ptrtbl()
368 		ASSERT0(zap_f_phys(zap)->zap_ptrtbl.zt_blk);  in zap_grow_ptrtbl()
370 		uint64_t newblk = zap_allocate_blocks(zap, 1);  in zap_grow_ptrtbl()
372 		int err = dmu_buf_hold(zap->zap_objset, zap->zap_object,  in zap_grow_ptrtbl()
373 		    newblk << FZAP_BLOCK_SHIFT(zap), FTAG, &db_new,  in zap_grow_ptrtbl()
378 		zap_ptrtbl_transfer(&ZAP_EMBEDDED_PTRTBL_ENT(zap, 0),  in zap_grow_ptrtbl()
379 		    db_new->db_data, 1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap));  in zap_grow_ptrtbl()
382 		zap_f_phys(zap)->zap_ptrtbl.zt_blk = newblk;  in zap_grow_ptrtbl()
383 		zap_f_phys(zap)->zap_ptrtbl.zt_numblks = 1;  in zap_grow_ptrtbl()
384 		zap_f_phys(zap)->zap_ptrtbl.zt_shift++;  in zap_grow_ptrtbl()
386 		ASSERT3U(1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift, ==,  in zap_grow_ptrtbl()
387 		    zap_f_phys(zap)->zap_ptrtbl.zt_numblks <<  in zap_grow_ptrtbl()
388 		    (FZAP_BLOCK_SHIFT(zap)-3));  in zap_grow_ptrtbl()
392 		return (zap_table_grow(zap, &zap_f_phys(zap)->zap_ptrtbl,  in zap_grow_ptrtbl()
398 zap_increment_num_entries(zap_t *zap, int delta, dmu_tx_t *tx)  in zap_increment_num_entries()  argument
400 	dmu_buf_will_dirty(zap->zap_dbuf, tx);  in zap_increment_num_entries()
401 	mutex_enter(&zap->zap_f.zap_num_entries_mtx);  in zap_increment_num_entries()
402 	ASSERT(delta > 0 || zap_f_phys(zap)->zap_num_entries >= -delta);  in zap_increment_num_entries()
403 	zap_f_phys(zap)->zap_num_entries += delta;  in zap_increment_num_entries()
404 	mutex_exit(&zap->zap_f.zap_num_entries_mtx);  in zap_increment_num_entries()
408 zap_allocate_blocks(zap_t *zap, int nblocks)  in zap_allocate_blocks()  argument
410 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));  in zap_allocate_blocks()
411 	uint64_t newblk = zap_f_phys(zap)->zap_freeblk;  in zap_allocate_blocks()
412 	zap_f_phys(zap)->zap_freeblk += nblocks;  in zap_allocate_blocks()
426 zap_create_leaf(zap_t *zap, dmu_tx_t *tx)  in zap_create_leaf()  argument
430 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));  in zap_create_leaf()
434 	l->l_blkid = zap_allocate_blocks(zap, 1);  in zap_create_leaf()
437 	VERIFY0(dmu_buf_hold(zap->zap_objset, zap->zap_object,  in zap_create_leaf()
438 	    l->l_blkid << FZAP_BLOCK_SHIFT(zap), NULL, &l->l_dbuf,  in zap_create_leaf()
444 	zap_leaf_init(l, zap->zap_normflags != 0);  in zap_create_leaf()
446 	zap_f_phys(zap)->zap_num_leafs++;  in zap_create_leaf()
452 fzap_count(zap_t *zap, uint64_t *count)  in fzap_count()  argument
454 	ASSERT(!zap->zap_ismicro);  in fzap_count()
455 	mutex_enter(&zap->zap_f.zap_num_entries_mtx); /* unnecessary */  in fzap_count()
456 	*count = zap_f_phys(zap)->zap_num_entries;  in fzap_count()
457 	mutex_exit(&zap->zap_f.zap_num_entries_mtx);  in fzap_count()
519 zap_get_leaf_byblk(zap_t *zap, uint64_t blkid, dmu_tx_t *tx, krw_t lt,  in zap_get_leaf_byblk()  argument
524 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));  in zap_get_leaf_byblk()
526 	int bs = FZAP_BLOCK_SHIFT(zap);  in zap_get_leaf_byblk()
527 	dnode_t *dn = dmu_buf_dnode_enter(zap->zap_dbuf);  in zap_get_leaf_byblk()
530 	dmu_buf_dnode_exit(zap->zap_dbuf);  in zap_get_leaf_byblk()
534 	ASSERT3U(db->db_object, ==, zap->zap_object);  in zap_get_leaf_byblk()
561 zap_idx_to_blk(zap_t *zap, uint64_t idx, uint64_t *valp)  in zap_idx_to_blk()  argument
563 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));  in zap_idx_to_blk()
565 	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {  in zap_idx_to_blk()
567 		    (1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift));  in zap_idx_to_blk()
568 		*valp = ZAP_EMBEDDED_PTRTBL_ENT(zap, idx);  in zap_idx_to_blk()
571 		return (zap_table_load(zap, &zap_f_phys(zap)->zap_ptrtbl,  in zap_idx_to_blk()
577 zap_set_idx_to_blk(zap_t *zap, uint64_t idx, uint64_t blk, dmu_tx_t *tx)  in zap_set_idx_to_blk()  argument
580 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));  in zap_set_idx_to_blk()
582 	if (zap_f_phys(zap)->zap_ptrtbl.zt_blk == 0) {  in zap_set_idx_to_blk()
583 		ZAP_EMBEDDED_PTRTBL_ENT(zap, idx) = blk;  in zap_set_idx_to_blk()
586 		return (zap_table_store(zap, &zap_f_phys(zap)->zap_ptrtbl,  in zap_set_idx_to_blk()
592 zap_deref_leaf(zap_t *zap, uint64_t h, dmu_tx_t *tx, krw_t lt, zap_leaf_t **lp)  in zap_deref_leaf()  argument
596 	ASSERT(zap->zap_dbuf == NULL ||  in zap_deref_leaf()
597 	    zap_f_phys(zap) == zap->zap_dbuf->db_data);  in zap_deref_leaf()
600 	if ((zap_f_phys(zap)->zap_block_type != ZBT_LEAF &&  in zap_deref_leaf()
601 	    zap_f_phys(zap)->zap_block_type != ZBT_HEADER) ||  in zap_deref_leaf()
602 	    zap_f_phys(zap)->zap_magic != ZAP_MAGIC) {  in zap_deref_leaf()
606 	uint64_t idx = ZAP_HASH_IDX(h, zap_f_phys(zap)->zap_ptrtbl.zt_shift);  in zap_deref_leaf()
607 	int err = zap_idx_to_blk(zap, idx, &blk);  in zap_deref_leaf()
610 	err = zap_get_leaf_byblk(zap, blk, tx, lt, lp);  in zap_deref_leaf()
622 	zap_t *zap = zn->zn_zap;  in zap_expand_leaf()  local
627 	ASSERT3U(old_prefix_len, <=, zap_f_phys(zap)->zap_ptrtbl.zt_shift);  in zap_expand_leaf()
628 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));  in zap_expand_leaf()
633 	if (zap_tryupgradedir(zap, tx) == 0 ||  in zap_expand_leaf()
634 	    old_prefix_len == zap_f_phys(zap)->zap_ptrtbl.zt_shift) {  in zap_expand_leaf()
636 		objset_t *os = zap->zap_objset;  in zap_expand_leaf()
637 		uint64_t object = zap->zap_object;  in zap_expand_leaf()
640 		zap_unlockdir(zap, tag);  in zap_expand_leaf()
643 		zap = zn->zn_zap;  in zap_expand_leaf()
646 		ASSERT(!zap->zap_ismicro);  in zap_expand_leaf()
649 		    zap_f_phys(zap)->zap_ptrtbl.zt_shift) {  in zap_expand_leaf()
650 			err = zap_grow_ptrtbl(zap, tx);  in zap_expand_leaf()
655 		err = zap_deref_leaf(zap, hash, tx, RW_WRITER, &l);  in zap_expand_leaf()
665 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));  in zap_expand_leaf()
666 	ASSERT3U(old_prefix_len, <, zap_f_phys(zap)->zap_ptrtbl.zt_shift);  in zap_expand_leaf()
670 	int prefix_diff = zap_f_phys(zap)->zap_ptrtbl.zt_shift -  in zap_expand_leaf()
678 		err = zap_idx_to_blk(zap, sibling + i, &blk);  in zap_expand_leaf()
684 	zap_leaf_t *nl = zap_create_leaf(zap, tx);  in zap_expand_leaf()
685 	zap_leaf_split(l, nl, zap->zap_normflags != 0);  in zap_expand_leaf()
689 		err = zap_set_idx_to_blk(zap, sibling + i, nl->l_blkid, tx);  in zap_expand_leaf()
709 	zap_t *zap = zn->zn_zap;  in zap_put_leaf_maybe_grow_ptrtbl()  local
710 	int shift = zap_f_phys(zap)->zap_ptrtbl.zt_shift;  in zap_put_leaf_maybe_grow_ptrtbl()
716 	if (leaffull || zap_f_phys(zap)->zap_ptrtbl.zt_nextblk) {  in zap_put_leaf_maybe_grow_ptrtbl()
721 		if (zap_tryupgradedir(zap, tx) == 0) {  in zap_put_leaf_maybe_grow_ptrtbl()
722 			objset_t *os = zap->zap_objset;  in zap_put_leaf_maybe_grow_ptrtbl()
723 			uint64_t zapobj = zap->zap_object;  in zap_put_leaf_maybe_grow_ptrtbl()
725 			zap_unlockdir(zap, tag);  in zap_put_leaf_maybe_grow_ptrtbl()
728 			zap = zn->zn_zap;  in zap_put_leaf_maybe_grow_ptrtbl()
734 		if (zap_f_phys(zap)->zap_ptrtbl.zt_shift == shift)  in zap_put_leaf_maybe_grow_ptrtbl()
735 			(void) zap_grow_ptrtbl(zap, tx);  in zap_put_leaf_maybe_grow_ptrtbl()
821 	zap_t *zap = zn->zn_zap;  in fzap_add_cd()  local
823 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));  in fzap_add_cd()
824 	ASSERT(!zap->zap_ismicro);  in fzap_add_cd()
827 	err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l);  in fzap_add_cd()
843 		zap_increment_num_entries(zap, 1, tx);  in fzap_add_cd()
846 		zap = zn->zn_zap;	/* zap_expand_leaf() may change zap */  in fzap_add_cd()
852 	if (zap != NULL)  in fzap_add_cd()
879 	zap_t *zap = zn->zn_zap;  in fzap_update()  local
881 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));  in fzap_update()
886 	err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l);  in fzap_update()
898 			zap_increment_num_entries(zap, 1, tx);  in fzap_update()
905 		zap = zn->zn_zap;	/* zap_expand_leaf() may change zap */  in fzap_update()
910 	if (zap != NULL)  in fzap_update()
962 	zap_t *zap = zn->zn_zap;  in fzap_prefetch()  local
965 	    zap_f_phys(zap)->zap_ptrtbl.zt_shift);  in fzap_prefetch()
966 	if (zap_idx_to_blk(zap, idx, &blk) != 0)  in fzap_prefetch()
968 	int bs = FZAP_BLOCK_SHIFT(zap);  in fzap_prefetch()
969 	dmu_prefetch(zap->zap_objset, zap->zap_object, 0, blk << bs, 1 << bs,  in fzap_prefetch()
1193 fzap_cursor_retrieve(zap_t *zap, zap_cursor_t *zc, zap_attribute_t *za)  in fzap_cursor_retrieve()  argument
1210 	    zc->zc_prefetch && zap_f_phys(zap)->zap_freeblk > 2) {  in fzap_cursor_retrieve()
1212 		    zap_f_phys(zap)->zap_freeblk << FZAP_BLOCK_SHIFT(zap),  in fzap_cursor_retrieve()
1227 		err = zap_deref_leaf(zap, zc->zc_hash, NULL, RW_READER,  in fzap_cursor_retrieve()
1264 		err = zap_entry_read_name(zap, &zeh,  in fzap_cursor_retrieve()
1270 		    NULL, za->za_name, zap);  in fzap_cursor_retrieve()
1277 zap_stats_ptrtbl(zap_t *zap, uint64_t *tbl, int len, zap_stats_t *zs)  in zap_stats_ptrtbl()  argument
1293 		int err = zap_get_leaf_byblk(zap, tbl[i], NULL, RW_READER, &l);  in zap_stats_ptrtbl()
1295 			zap_leaf_stats(zap, l, zs);  in zap_stats_ptrtbl()
1302 fzap_get_stats(zap_t *zap, zap_stats_t *zs)  in fzap_get_stats()  argument
1304 	int bs = FZAP_BLOCK_SHIFT(zap);  in fzap_get_stats()
1310 	zs->zs_num_leafs = zap_f_phys(zap)->zap_num_leafs;  in fzap_get_stats()
1311 	zs->zs_num_entries = zap_f_phys(zap)->zap_num_entries;  in fzap_get_stats()
1312 	zs->zs_num_blocks = zap_f_phys(zap)->zap_freeblk;  in fzap_get_stats()
1313 	zs->zs_block_type = zap_f_phys(zap)->zap_block_type;  in fzap_get_stats()
1314 	zs->zs_magic = zap_f_phys(zap)->zap_magic;  in fzap_get_stats()
1315 	zs->zs_salt = zap_f_phys(zap)->zap_salt;  in fzap_get_stats()
1320 	zs->zs_ptrtbl_len = 1ULL << zap_f_phys(zap)->zap_ptrtbl.zt_shift;  in fzap_get_stats()
1321 	zs->zs_ptrtbl_nextblk = zap_f_phys(zap)->zap_ptrtbl.zt_nextblk;  in fzap_get_stats()
1323 	    zap_f_phys(zap)->zap_ptrtbl.zt_blks_copied;  in fzap_get_stats()
1324 	zs->zs_ptrtbl_zt_blk = zap_f_phys(zap)->zap_ptrtbl.zt_blk;  in fzap_get_stats()
1325 	zs->zs_ptrtbl_zt_numblks = zap_f_phys(zap)->zap_ptrtbl.zt_numblks;  in fzap_get_stats()
1326 	zs->zs_ptrtbl_zt_shift = zap_f_phys(zap)->zap_ptrtbl.zt_shift;  in fzap_get_stats()
1328 	if (zap_f_phys(zap)->zap_ptrtbl.zt_numblks == 0) {  in fzap_get_stats()
1330 		zap_stats_ptrtbl(zap, &ZAP_EMBEDDED_PTRTBL_ENT(zap, 0),  in fzap_get_stats()
1331 		    1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap), zs);  in fzap_get_stats()
1333 		dmu_prefetch(zap->zap_objset, zap->zap_object, 0,  in fzap_get_stats()
1334 		    zap_f_phys(zap)->zap_ptrtbl.zt_blk << bs,  in fzap_get_stats()
1335 		    zap_f_phys(zap)->zap_ptrtbl.zt_numblks << bs,  in fzap_get_stats()
1338 		for (int b = 0; b < zap_f_phys(zap)->zap_ptrtbl.zt_numblks;  in fzap_get_stats()
1343 			err = dmu_buf_hold(zap->zap_objset, zap->zap_object,  in fzap_get_stats()
1344 			    (zap_f_phys(zap)->zap_ptrtbl.zt_blk + b) << bs,  in fzap_get_stats()
1347 				zap_stats_ptrtbl(zap, db->db_data,  in fzap_get_stats()