1eda14cbcSMatt Macy /*
2eda14cbcSMatt Macy * CDDL HEADER START
3eda14cbcSMatt Macy *
4eda14cbcSMatt Macy * The contents of this file are subject to the terms of the
5eda14cbcSMatt Macy * Common Development and Distribution License (the "License").
6eda14cbcSMatt Macy * You may not use this file except in compliance with the License.
7eda14cbcSMatt Macy *
8eda14cbcSMatt Macy * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9271171e0SMartin Matuska * or https://opensource.org/licenses/CDDL-1.0.
10eda14cbcSMatt Macy * See the License for the specific language governing permissions
11eda14cbcSMatt Macy * and limitations under the License.
12eda14cbcSMatt Macy *
13eda14cbcSMatt Macy * When distributing Covered Code, include this CDDL HEADER in each
14eda14cbcSMatt Macy * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15eda14cbcSMatt Macy * If applicable, add the following below this CDDL HEADER, with the
16eda14cbcSMatt Macy * fields enclosed by brackets "[]" replaced with your own identifying
17eda14cbcSMatt Macy * information: Portions Copyright [yyyy] [name of copyright owner]
18eda14cbcSMatt Macy *
19eda14cbcSMatt Macy * CDDL HEADER END
20eda14cbcSMatt Macy */
21eda14cbcSMatt Macy /*
22eda14cbcSMatt Macy * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
237877fdebSMatt Macy * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
24eda14cbcSMatt Macy * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
25eda14cbcSMatt Macy */
26eda14cbcSMatt Macy
27eda14cbcSMatt Macy #include <sys/zfs_context.h>
28eda14cbcSMatt Macy #include <sys/dbuf.h>
29eda14cbcSMatt Macy #include <sys/dnode.h>
30eda14cbcSMatt Macy #include <sys/dmu.h>
31eda14cbcSMatt Macy #include <sys/dmu_impl.h>
32eda14cbcSMatt Macy #include <sys/dmu_tx.h>
33eda14cbcSMatt Macy #include <sys/dmu_objset.h>
34eda14cbcSMatt Macy #include <sys/dsl_dir.h>
35eda14cbcSMatt Macy #include <sys/dsl_dataset.h>
36eda14cbcSMatt Macy #include <sys/spa.h>
37eda14cbcSMatt Macy #include <sys/zio.h>
38eda14cbcSMatt Macy #include <sys/dmu_zfetch.h>
39eda14cbcSMatt Macy #include <sys/range_tree.h>
40eda14cbcSMatt Macy #include <sys/trace_zfs.h>
41eda14cbcSMatt Macy #include <sys/zfs_project.h>
42eda14cbcSMatt Macy
43eda14cbcSMatt Macy dnode_stats_t dnode_stats = {
44eda14cbcSMatt Macy { "dnode_hold_dbuf_hold", KSTAT_DATA_UINT64 },
45eda14cbcSMatt Macy { "dnode_hold_dbuf_read", KSTAT_DATA_UINT64 },
46eda14cbcSMatt Macy { "dnode_hold_alloc_hits", KSTAT_DATA_UINT64 },
47eda14cbcSMatt Macy { "dnode_hold_alloc_misses", KSTAT_DATA_UINT64 },
48eda14cbcSMatt Macy { "dnode_hold_alloc_interior", KSTAT_DATA_UINT64 },
49eda14cbcSMatt Macy { "dnode_hold_alloc_lock_retry", KSTAT_DATA_UINT64 },
50eda14cbcSMatt Macy { "dnode_hold_alloc_lock_misses", KSTAT_DATA_UINT64 },
51eda14cbcSMatt Macy { "dnode_hold_alloc_type_none", KSTAT_DATA_UINT64 },
52eda14cbcSMatt Macy { "dnode_hold_free_hits", KSTAT_DATA_UINT64 },
53eda14cbcSMatt Macy { "dnode_hold_free_misses", KSTAT_DATA_UINT64 },
54eda14cbcSMatt Macy { "dnode_hold_free_lock_misses", KSTAT_DATA_UINT64 },
55eda14cbcSMatt Macy { "dnode_hold_free_lock_retry", KSTAT_DATA_UINT64 },
56eda14cbcSMatt Macy { "dnode_hold_free_overflow", KSTAT_DATA_UINT64 },
57eda14cbcSMatt Macy { "dnode_hold_free_refcount", KSTAT_DATA_UINT64 },
58eda14cbcSMatt Macy { "dnode_free_interior_lock_retry", KSTAT_DATA_UINT64 },
59eda14cbcSMatt Macy { "dnode_allocate", KSTAT_DATA_UINT64 },
60eda14cbcSMatt Macy { "dnode_reallocate", KSTAT_DATA_UINT64 },
61eda14cbcSMatt Macy { "dnode_buf_evict", KSTAT_DATA_UINT64 },
62eda14cbcSMatt Macy { "dnode_alloc_next_chunk", KSTAT_DATA_UINT64 },
63eda14cbcSMatt Macy { "dnode_alloc_race", KSTAT_DATA_UINT64 },
64eda14cbcSMatt Macy { "dnode_alloc_next_block", KSTAT_DATA_UINT64 },
65eda14cbcSMatt Macy { "dnode_move_invalid", KSTAT_DATA_UINT64 },
66eda14cbcSMatt Macy { "dnode_move_recheck1", KSTAT_DATA_UINT64 },
67eda14cbcSMatt Macy { "dnode_move_recheck2", KSTAT_DATA_UINT64 },
68eda14cbcSMatt Macy { "dnode_move_special", KSTAT_DATA_UINT64 },
69eda14cbcSMatt Macy { "dnode_move_handle", KSTAT_DATA_UINT64 },
70eda14cbcSMatt Macy { "dnode_move_rwlock", KSTAT_DATA_UINT64 },
71eda14cbcSMatt Macy { "dnode_move_active", KSTAT_DATA_UINT64 },
72eda14cbcSMatt Macy };
73eda14cbcSMatt Macy
74bb2d13b6SMartin Matuska dnode_sums_t dnode_sums;
75bb2d13b6SMartin Matuska
76eda14cbcSMatt Macy static kstat_t *dnode_ksp;
77eda14cbcSMatt Macy static kmem_cache_t *dnode_cache;
78eda14cbcSMatt Macy
79eda14cbcSMatt Macy static dnode_phys_t dnode_phys_zero __maybe_unused;
80eda14cbcSMatt Macy
81eda14cbcSMatt Macy int zfs_default_bs = SPA_MINBLOCKSHIFT;
82eda14cbcSMatt Macy int zfs_default_ibs = DN_MAX_INDBLKSHIFT;
83eda14cbcSMatt Macy
84eda14cbcSMatt Macy #ifdef _KERNEL
85eda14cbcSMatt Macy static kmem_cbrc_t dnode_move(void *, void *, size_t, void *);
86eda14cbcSMatt Macy #endif /* _KERNEL */
87eda14cbcSMatt Macy
88eda14cbcSMatt Macy static int
dbuf_compare(const void * x1,const void * x2)89eda14cbcSMatt Macy dbuf_compare(const void *x1, const void *x2)
90eda14cbcSMatt Macy {
91eda14cbcSMatt Macy const dmu_buf_impl_t *d1 = x1;
92eda14cbcSMatt Macy const dmu_buf_impl_t *d2 = x2;
93eda14cbcSMatt Macy
94eda14cbcSMatt Macy int cmp = TREE_CMP(d1->db_level, d2->db_level);
95eda14cbcSMatt Macy if (likely(cmp))
96eda14cbcSMatt Macy return (cmp);
97eda14cbcSMatt Macy
98eda14cbcSMatt Macy cmp = TREE_CMP(d1->db_blkid, d2->db_blkid);
99eda14cbcSMatt Macy if (likely(cmp))
100eda14cbcSMatt Macy return (cmp);
101eda14cbcSMatt Macy
1025fb307d2SMartin Matuska if (d1->db_state == DB_MARKER) {
1035fb307d2SMartin Matuska ASSERT3S(d2->db_state, !=, DB_MARKER);
1045fb307d2SMartin Matuska return (TREE_PCMP(d1->db_parent, d2));
1055fb307d2SMartin Matuska } else if (d2->db_state == DB_MARKER) {
1065fb307d2SMartin Matuska ASSERT3S(d1->db_state, !=, DB_MARKER);
1075fb307d2SMartin Matuska return (TREE_PCMP(d1, d2->db_parent));
1085fb307d2SMartin Matuska }
1095fb307d2SMartin Matuska
110eda14cbcSMatt Macy if (d1->db_state == DB_SEARCH) {
111eda14cbcSMatt Macy ASSERT3S(d2->db_state, !=, DB_SEARCH);
112eda14cbcSMatt Macy return (-1);
113eda14cbcSMatt Macy } else if (d2->db_state == DB_SEARCH) {
114eda14cbcSMatt Macy ASSERT3S(d1->db_state, !=, DB_SEARCH);
115eda14cbcSMatt Macy return (1);
116eda14cbcSMatt Macy }
117eda14cbcSMatt Macy
118eda14cbcSMatt Macy return (TREE_PCMP(d1, d2));
119eda14cbcSMatt Macy }
120eda14cbcSMatt Macy
121eda14cbcSMatt Macy static int
dnode_cons(void * arg,void * unused,int kmflag)122eda14cbcSMatt Macy dnode_cons(void *arg, void *unused, int kmflag)
123eda14cbcSMatt Macy {
124e92ffd9bSMartin Matuska (void) unused, (void) kmflag;
125eda14cbcSMatt Macy dnode_t *dn = arg;
126eda14cbcSMatt Macy
127eda14cbcSMatt Macy rw_init(&dn->dn_struct_rwlock, NULL, RW_NOLOCKDEP, NULL);
128eda14cbcSMatt Macy mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL);
129eda14cbcSMatt Macy mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL);
130eda14cbcSMatt Macy cv_init(&dn->dn_notxholds, NULL, CV_DEFAULT, NULL);
131eda14cbcSMatt Macy cv_init(&dn->dn_nodnholds, NULL, CV_DEFAULT, NULL);
132eda14cbcSMatt Macy
133eda14cbcSMatt Macy /*
134eda14cbcSMatt Macy * Every dbuf has a reference, and dropping a tracked reference is
135eda14cbcSMatt Macy * O(number of references), so don't track dn_holds.
136eda14cbcSMatt Macy */
137eda14cbcSMatt Macy zfs_refcount_create_untracked(&dn->dn_holds);
138eda14cbcSMatt Macy zfs_refcount_create(&dn->dn_tx_holds);
139eda14cbcSMatt Macy list_link_init(&dn->dn_link);
140eda14cbcSMatt Macy
141da5137abSMartin Matuska memset(dn->dn_next_type, 0, sizeof (dn->dn_next_type));
142da5137abSMartin Matuska memset(dn->dn_next_nblkptr, 0, sizeof (dn->dn_next_nblkptr));
143da5137abSMartin Matuska memset(dn->dn_next_nlevels, 0, sizeof (dn->dn_next_nlevels));
144da5137abSMartin Matuska memset(dn->dn_next_indblkshift, 0, sizeof (dn->dn_next_indblkshift));
145da5137abSMartin Matuska memset(dn->dn_next_bonustype, 0, sizeof (dn->dn_next_bonustype));
146da5137abSMartin Matuska memset(dn->dn_rm_spillblk, 0, sizeof (dn->dn_rm_spillblk));
147da5137abSMartin Matuska memset(dn->dn_next_bonuslen, 0, sizeof (dn->dn_next_bonuslen));
148da5137abSMartin Matuska memset(dn->dn_next_blksz, 0, sizeof (dn->dn_next_blksz));
149da5137abSMartin Matuska memset(dn->dn_next_maxblkid, 0, sizeof (dn->dn_next_maxblkid));
150eda14cbcSMatt Macy
151e92ffd9bSMartin Matuska for (int i = 0; i < TXG_SIZE; i++) {
152eda14cbcSMatt Macy multilist_link_init(&dn->dn_dirty_link[i]);
153eda14cbcSMatt Macy dn->dn_free_ranges[i] = NULL;
154eda14cbcSMatt Macy list_create(&dn->dn_dirty_records[i],
155eda14cbcSMatt Macy sizeof (dbuf_dirty_record_t),
156eda14cbcSMatt Macy offsetof(dbuf_dirty_record_t, dr_dirty_node));
157eda14cbcSMatt Macy }
158eda14cbcSMatt Macy
159eda14cbcSMatt Macy dn->dn_allocated_txg = 0;
160eda14cbcSMatt Macy dn->dn_free_txg = 0;
161eda14cbcSMatt Macy dn->dn_assigned_txg = 0;
162eda14cbcSMatt Macy dn->dn_dirty_txg = 0;
163eda14cbcSMatt Macy dn->dn_dirtyctx = 0;
164eda14cbcSMatt Macy dn->dn_dirtyctx_firstset = NULL;
165eda14cbcSMatt Macy dn->dn_bonus = NULL;
166eda14cbcSMatt Macy dn->dn_have_spill = B_FALSE;
167eda14cbcSMatt Macy dn->dn_zio = NULL;
168eda14cbcSMatt Macy dn->dn_oldused = 0;
169eda14cbcSMatt Macy dn->dn_oldflags = 0;
170eda14cbcSMatt Macy dn->dn_olduid = 0;
171eda14cbcSMatt Macy dn->dn_oldgid = 0;
172eda14cbcSMatt Macy dn->dn_oldprojid = ZFS_DEFAULT_PROJID;
173eda14cbcSMatt Macy dn->dn_newuid = 0;
174eda14cbcSMatt Macy dn->dn_newgid = 0;
175eda14cbcSMatt Macy dn->dn_newprojid = ZFS_DEFAULT_PROJID;
176eda14cbcSMatt Macy dn->dn_id_flags = 0;
177eda14cbcSMatt Macy
178eda14cbcSMatt Macy dn->dn_dbufs_count = 0;
179eda14cbcSMatt Macy avl_create(&dn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t),
180eda14cbcSMatt Macy offsetof(dmu_buf_impl_t, db_link));
181eda14cbcSMatt Macy
182eda14cbcSMatt Macy dn->dn_moved = 0;
183eda14cbcSMatt Macy return (0);
184eda14cbcSMatt Macy }
185eda14cbcSMatt Macy
186eda14cbcSMatt Macy static void
dnode_dest(void * arg,void * unused)187eda14cbcSMatt Macy dnode_dest(void *arg, void *unused)
188eda14cbcSMatt Macy {
189e92ffd9bSMartin Matuska (void) unused;
190eda14cbcSMatt Macy dnode_t *dn = arg;
191eda14cbcSMatt Macy
192eda14cbcSMatt Macy rw_destroy(&dn->dn_struct_rwlock);
193eda14cbcSMatt Macy mutex_destroy(&dn->dn_mtx);
194eda14cbcSMatt Macy mutex_destroy(&dn->dn_dbufs_mtx);
195eda14cbcSMatt Macy cv_destroy(&dn->dn_notxholds);
196eda14cbcSMatt Macy cv_destroy(&dn->dn_nodnholds);
197eda14cbcSMatt Macy zfs_refcount_destroy(&dn->dn_holds);
198eda14cbcSMatt Macy zfs_refcount_destroy(&dn->dn_tx_holds);
199eda14cbcSMatt Macy ASSERT(!list_link_active(&dn->dn_link));
200eda14cbcSMatt Macy
201e92ffd9bSMartin Matuska for (int i = 0; i < TXG_SIZE; i++) {
202eda14cbcSMatt Macy ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
203eda14cbcSMatt Macy ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
204eda14cbcSMatt Macy list_destroy(&dn->dn_dirty_records[i]);
205eda14cbcSMatt Macy ASSERT0(dn->dn_next_nblkptr[i]);
206eda14cbcSMatt Macy ASSERT0(dn->dn_next_nlevels[i]);
207eda14cbcSMatt Macy ASSERT0(dn->dn_next_indblkshift[i]);
208eda14cbcSMatt Macy ASSERT0(dn->dn_next_bonustype[i]);
209eda14cbcSMatt Macy ASSERT0(dn->dn_rm_spillblk[i]);
210eda14cbcSMatt Macy ASSERT0(dn->dn_next_bonuslen[i]);
211eda14cbcSMatt Macy ASSERT0(dn->dn_next_blksz[i]);
212eda14cbcSMatt Macy ASSERT0(dn->dn_next_maxblkid[i]);
213eda14cbcSMatt Macy }
214eda14cbcSMatt Macy
215eda14cbcSMatt Macy ASSERT0(dn->dn_allocated_txg);
216eda14cbcSMatt Macy ASSERT0(dn->dn_free_txg);
217eda14cbcSMatt Macy ASSERT0(dn->dn_assigned_txg);
218eda14cbcSMatt Macy ASSERT0(dn->dn_dirty_txg);
219eda14cbcSMatt Macy ASSERT0(dn->dn_dirtyctx);
220eda14cbcSMatt Macy ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL);
221eda14cbcSMatt Macy ASSERT3P(dn->dn_bonus, ==, NULL);
222eda14cbcSMatt Macy ASSERT(!dn->dn_have_spill);
223eda14cbcSMatt Macy ASSERT3P(dn->dn_zio, ==, NULL);
224eda14cbcSMatt Macy ASSERT0(dn->dn_oldused);
225eda14cbcSMatt Macy ASSERT0(dn->dn_oldflags);
226eda14cbcSMatt Macy ASSERT0(dn->dn_olduid);
227eda14cbcSMatt Macy ASSERT0(dn->dn_oldgid);
228eda14cbcSMatt Macy ASSERT0(dn->dn_oldprojid);
229eda14cbcSMatt Macy ASSERT0(dn->dn_newuid);
230eda14cbcSMatt Macy ASSERT0(dn->dn_newgid);
231eda14cbcSMatt Macy ASSERT0(dn->dn_newprojid);
232eda14cbcSMatt Macy ASSERT0(dn->dn_id_flags);
233eda14cbcSMatt Macy
234eda14cbcSMatt Macy ASSERT0(dn->dn_dbufs_count);
235eda14cbcSMatt Macy avl_destroy(&dn->dn_dbufs);
236eda14cbcSMatt Macy }
237eda14cbcSMatt Macy
238bb2d13b6SMartin Matuska static int
dnode_kstats_update(kstat_t * ksp,int rw)239bb2d13b6SMartin Matuska dnode_kstats_update(kstat_t *ksp, int rw)
240bb2d13b6SMartin Matuska {
241bb2d13b6SMartin Matuska dnode_stats_t *ds = ksp->ks_data;
242bb2d13b6SMartin Matuska
243bb2d13b6SMartin Matuska if (rw == KSTAT_WRITE)
244bb2d13b6SMartin Matuska return (EACCES);
245bb2d13b6SMartin Matuska ds->dnode_hold_dbuf_hold.value.ui64 =
246bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_dbuf_hold);
247bb2d13b6SMartin Matuska ds->dnode_hold_dbuf_read.value.ui64 =
248bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_dbuf_read);
249bb2d13b6SMartin Matuska ds->dnode_hold_alloc_hits.value.ui64 =
250bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_alloc_hits);
251bb2d13b6SMartin Matuska ds->dnode_hold_alloc_misses.value.ui64 =
252bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_alloc_misses);
253bb2d13b6SMartin Matuska ds->dnode_hold_alloc_interior.value.ui64 =
254bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_alloc_interior);
255bb2d13b6SMartin Matuska ds->dnode_hold_alloc_lock_retry.value.ui64 =
256bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_alloc_lock_retry);
257bb2d13b6SMartin Matuska ds->dnode_hold_alloc_lock_misses.value.ui64 =
258bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_alloc_lock_misses);
259bb2d13b6SMartin Matuska ds->dnode_hold_alloc_type_none.value.ui64 =
260bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_alloc_type_none);
261bb2d13b6SMartin Matuska ds->dnode_hold_free_hits.value.ui64 =
262bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_free_hits);
263bb2d13b6SMartin Matuska ds->dnode_hold_free_misses.value.ui64 =
264bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_free_misses);
265bb2d13b6SMartin Matuska ds->dnode_hold_free_lock_misses.value.ui64 =
266bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_free_lock_misses);
267bb2d13b6SMartin Matuska ds->dnode_hold_free_lock_retry.value.ui64 =
268bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_free_lock_retry);
269bb2d13b6SMartin Matuska ds->dnode_hold_free_refcount.value.ui64 =
270bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_free_refcount);
271bb2d13b6SMartin Matuska ds->dnode_hold_free_overflow.value.ui64 =
272bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_hold_free_overflow);
273bb2d13b6SMartin Matuska ds->dnode_free_interior_lock_retry.value.ui64 =
274bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_free_interior_lock_retry);
275bb2d13b6SMartin Matuska ds->dnode_allocate.value.ui64 =
276bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_allocate);
277bb2d13b6SMartin Matuska ds->dnode_reallocate.value.ui64 =
278bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_reallocate);
279bb2d13b6SMartin Matuska ds->dnode_buf_evict.value.ui64 =
280bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_buf_evict);
281bb2d13b6SMartin Matuska ds->dnode_alloc_next_chunk.value.ui64 =
282bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_alloc_next_chunk);
283bb2d13b6SMartin Matuska ds->dnode_alloc_race.value.ui64 =
284bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_alloc_race);
285bb2d13b6SMartin Matuska ds->dnode_alloc_next_block.value.ui64 =
286bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_alloc_next_block);
287bb2d13b6SMartin Matuska ds->dnode_move_invalid.value.ui64 =
288bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_move_invalid);
289bb2d13b6SMartin Matuska ds->dnode_move_recheck1.value.ui64 =
290bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_move_recheck1);
291bb2d13b6SMartin Matuska ds->dnode_move_recheck2.value.ui64 =
292bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_move_recheck2);
293bb2d13b6SMartin Matuska ds->dnode_move_special.value.ui64 =
294bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_move_special);
295bb2d13b6SMartin Matuska ds->dnode_move_handle.value.ui64 =
296bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_move_handle);
297bb2d13b6SMartin Matuska ds->dnode_move_rwlock.value.ui64 =
298bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_move_rwlock);
299bb2d13b6SMartin Matuska ds->dnode_move_active.value.ui64 =
300bb2d13b6SMartin Matuska wmsum_value(&dnode_sums.dnode_move_active);
301bb2d13b6SMartin Matuska return (0);
302bb2d13b6SMartin Matuska }
303bb2d13b6SMartin Matuska
304eda14cbcSMatt Macy void
dnode_init(void)305eda14cbcSMatt Macy dnode_init(void)
306eda14cbcSMatt Macy {
307eda14cbcSMatt Macy ASSERT(dnode_cache == NULL);
308eda14cbcSMatt Macy dnode_cache = kmem_cache_create("dnode_t", sizeof (dnode_t),
309eda14cbcSMatt Macy 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0);
310eda14cbcSMatt Macy kmem_cache_set_move(dnode_cache, dnode_move);
311eda14cbcSMatt Macy
312bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_dbuf_hold, 0);
313bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_dbuf_read, 0);
314bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_alloc_hits, 0);
315bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_alloc_misses, 0);
316bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_alloc_interior, 0);
317bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_alloc_lock_retry, 0);
318bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_alloc_lock_misses, 0);
319bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_alloc_type_none, 0);
320bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_free_hits, 0);
321bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_free_misses, 0);
322bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_free_lock_misses, 0);
323bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_free_lock_retry, 0);
324bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_free_refcount, 0);
325bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_hold_free_overflow, 0);
326bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_free_interior_lock_retry, 0);
327bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_allocate, 0);
328bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_reallocate, 0);
329bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_buf_evict, 0);
330bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_alloc_next_chunk, 0);
331bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_alloc_race, 0);
332bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_alloc_next_block, 0);
333bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_move_invalid, 0);
334bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_move_recheck1, 0);
335bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_move_recheck2, 0);
336bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_move_special, 0);
337bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_move_handle, 0);
338bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_move_rwlock, 0);
339bb2d13b6SMartin Matuska wmsum_init(&dnode_sums.dnode_move_active, 0);
340bb2d13b6SMartin Matuska
341eda14cbcSMatt Macy dnode_ksp = kstat_create("zfs", 0, "dnodestats", "misc",
342eda14cbcSMatt Macy KSTAT_TYPE_NAMED, sizeof (dnode_stats) / sizeof (kstat_named_t),
343eda14cbcSMatt Macy KSTAT_FLAG_VIRTUAL);
344eda14cbcSMatt Macy if (dnode_ksp != NULL) {
345eda14cbcSMatt Macy dnode_ksp->ks_data = &dnode_stats;
346bb2d13b6SMartin Matuska dnode_ksp->ks_update = dnode_kstats_update;
347eda14cbcSMatt Macy kstat_install(dnode_ksp);
348eda14cbcSMatt Macy }
349eda14cbcSMatt Macy }
350eda14cbcSMatt Macy
351eda14cbcSMatt Macy void
dnode_fini(void)352eda14cbcSMatt Macy dnode_fini(void)
353eda14cbcSMatt Macy {
354eda14cbcSMatt Macy if (dnode_ksp != NULL) {
355eda14cbcSMatt Macy kstat_delete(dnode_ksp);
356eda14cbcSMatt Macy dnode_ksp = NULL;
357eda14cbcSMatt Macy }
358eda14cbcSMatt Macy
359bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_dbuf_hold);
360bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_dbuf_read);
361bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_alloc_hits);
362bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_alloc_misses);
363bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_alloc_interior);
364bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_retry);
365bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_misses);
366bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_alloc_type_none);
367bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_free_hits);
368bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_free_misses);
369bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_free_lock_misses);
370bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_free_lock_retry);
371bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_free_refcount);
372bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_hold_free_overflow);
373bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_free_interior_lock_retry);
374bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_allocate);
375bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_reallocate);
376bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_buf_evict);
377bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_alloc_next_chunk);
378bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_alloc_race);
379bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_alloc_next_block);
380bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_move_invalid);
381bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_move_recheck1);
382bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_move_recheck2);
383bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_move_special);
384bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_move_handle);
385bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_move_rwlock);
386bb2d13b6SMartin Matuska wmsum_fini(&dnode_sums.dnode_move_active);
387bb2d13b6SMartin Matuska
388eda14cbcSMatt Macy kmem_cache_destroy(dnode_cache);
389eda14cbcSMatt Macy dnode_cache = NULL;
390eda14cbcSMatt Macy }
391eda14cbcSMatt Macy
392eda14cbcSMatt Macy
393eda14cbcSMatt Macy #ifdef ZFS_DEBUG
394eda14cbcSMatt Macy void
dnode_verify(dnode_t * dn)395eda14cbcSMatt Macy dnode_verify(dnode_t *dn)
396eda14cbcSMatt Macy {
397eda14cbcSMatt Macy int drop_struct_lock = FALSE;
398eda14cbcSMatt Macy
399eda14cbcSMatt Macy ASSERT(dn->dn_phys);
400eda14cbcSMatt Macy ASSERT(dn->dn_objset);
401eda14cbcSMatt Macy ASSERT(dn->dn_handle->dnh_dnode == dn);
402eda14cbcSMatt Macy
403eda14cbcSMatt Macy ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
404eda14cbcSMatt Macy
405eda14cbcSMatt Macy if (!(zfs_flags & ZFS_DEBUG_DNODE_VERIFY))
406eda14cbcSMatt Macy return;
407eda14cbcSMatt Macy
408eda14cbcSMatt Macy if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
409eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER);
410eda14cbcSMatt Macy drop_struct_lock = TRUE;
411eda14cbcSMatt Macy }
412eda14cbcSMatt Macy if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) {
413eda14cbcSMatt Macy int i;
414eda14cbcSMatt Macy int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
415eda14cbcSMatt Macy ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT);
416eda14cbcSMatt Macy if (dn->dn_datablkshift) {
417eda14cbcSMatt Macy ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT);
418eda14cbcSMatt Macy ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT);
419eda14cbcSMatt Macy ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz);
420eda14cbcSMatt Macy }
421eda14cbcSMatt Macy ASSERT3U(dn->dn_nlevels, <=, 30);
422eda14cbcSMatt Macy ASSERT(DMU_OT_IS_VALID(dn->dn_type));
423eda14cbcSMatt Macy ASSERT3U(dn->dn_nblkptr, >=, 1);
424eda14cbcSMatt Macy ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR);
425eda14cbcSMatt Macy ASSERT3U(dn->dn_bonuslen, <=, max_bonuslen);
426eda14cbcSMatt Macy ASSERT3U(dn->dn_datablksz, ==,
427eda14cbcSMatt Macy dn->dn_datablkszsec << SPA_MINBLOCKSHIFT);
428eda14cbcSMatt Macy ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0);
429eda14cbcSMatt Macy ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) +
430eda14cbcSMatt Macy dn->dn_bonuslen, <=, max_bonuslen);
431eda14cbcSMatt Macy for (i = 0; i < TXG_SIZE; i++) {
432eda14cbcSMatt Macy ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels);
433eda14cbcSMatt Macy }
434eda14cbcSMatt Macy }
435eda14cbcSMatt Macy if (dn->dn_phys->dn_type != DMU_OT_NONE)
436eda14cbcSMatt Macy ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels);
437eda14cbcSMatt Macy ASSERT(DMU_OBJECT_IS_SPECIAL(dn->dn_object) || dn->dn_dbuf != NULL);
438eda14cbcSMatt Macy if (dn->dn_dbuf != NULL) {
439eda14cbcSMatt Macy ASSERT3P(dn->dn_phys, ==,
440eda14cbcSMatt Macy (dnode_phys_t *)dn->dn_dbuf->db.db_data +
441eda14cbcSMatt Macy (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT)));
442eda14cbcSMatt Macy }
443eda14cbcSMatt Macy if (drop_struct_lock)
444eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
445eda14cbcSMatt Macy }
446eda14cbcSMatt Macy #endif
447eda14cbcSMatt Macy
448eda14cbcSMatt Macy void
dnode_byteswap(dnode_phys_t * dnp)449eda14cbcSMatt Macy dnode_byteswap(dnode_phys_t *dnp)
450eda14cbcSMatt Macy {
451eda14cbcSMatt Macy uint64_t *buf64 = (void*)&dnp->dn_blkptr;
452eda14cbcSMatt Macy int i;
453eda14cbcSMatt Macy
454eda14cbcSMatt Macy if (dnp->dn_type == DMU_OT_NONE) {
455da5137abSMartin Matuska memset(dnp, 0, sizeof (dnode_phys_t));
456eda14cbcSMatt Macy return;
457eda14cbcSMatt Macy }
458eda14cbcSMatt Macy
459eda14cbcSMatt Macy dnp->dn_datablkszsec = BSWAP_16(dnp->dn_datablkszsec);
460eda14cbcSMatt Macy dnp->dn_bonuslen = BSWAP_16(dnp->dn_bonuslen);
461eda14cbcSMatt Macy dnp->dn_extra_slots = BSWAP_8(dnp->dn_extra_slots);
462eda14cbcSMatt Macy dnp->dn_maxblkid = BSWAP_64(dnp->dn_maxblkid);
463eda14cbcSMatt Macy dnp->dn_used = BSWAP_64(dnp->dn_used);
464eda14cbcSMatt Macy
465eda14cbcSMatt Macy /*
466eda14cbcSMatt Macy * dn_nblkptr is only one byte, so it's OK to read it in either
467eda14cbcSMatt Macy * byte order. We can't read dn_bouslen.
468eda14cbcSMatt Macy */
469eda14cbcSMatt Macy ASSERT(dnp->dn_indblkshift <= SPA_MAXBLOCKSHIFT);
470eda14cbcSMatt Macy ASSERT(dnp->dn_nblkptr <= DN_MAX_NBLKPTR);
471eda14cbcSMatt Macy for (i = 0; i < dnp->dn_nblkptr * sizeof (blkptr_t)/8; i++)
472eda14cbcSMatt Macy buf64[i] = BSWAP_64(buf64[i]);
473eda14cbcSMatt Macy
474eda14cbcSMatt Macy /*
475eda14cbcSMatt Macy * OK to check dn_bonuslen for zero, because it won't matter if
476eda14cbcSMatt Macy * we have the wrong byte order. This is necessary because the
477eda14cbcSMatt Macy * dnode dnode is smaller than a regular dnode.
478eda14cbcSMatt Macy */
479eda14cbcSMatt Macy if (dnp->dn_bonuslen != 0) {
480eda14cbcSMatt Macy dmu_object_byteswap_t byteswap;
481eda14cbcSMatt Macy ASSERT(DMU_OT_IS_VALID(dnp->dn_bonustype));
482eda14cbcSMatt Macy byteswap = DMU_OT_BYTESWAP(dnp->dn_bonustype);
483a0b956f5SMartin Matuska dmu_ot_byteswap[byteswap].ob_func(DN_BONUS(dnp),
484a0b956f5SMartin Matuska DN_MAX_BONUS_LEN(dnp));
485eda14cbcSMatt Macy }
486eda14cbcSMatt Macy
487eda14cbcSMatt Macy /* Swap SPILL block if we have one */
488eda14cbcSMatt Macy if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)
489eda14cbcSMatt Macy byteswap_uint64_array(DN_SPILL_BLKPTR(dnp), sizeof (blkptr_t));
490eda14cbcSMatt Macy }
491eda14cbcSMatt Macy
492eda14cbcSMatt Macy void
dnode_buf_byteswap(void * vbuf,size_t size)493eda14cbcSMatt Macy dnode_buf_byteswap(void *vbuf, size_t size)
494eda14cbcSMatt Macy {
495eda14cbcSMatt Macy int i = 0;
496eda14cbcSMatt Macy
497eda14cbcSMatt Macy ASSERT3U(sizeof (dnode_phys_t), ==, (1<<DNODE_SHIFT));
498eda14cbcSMatt Macy ASSERT((size & (sizeof (dnode_phys_t)-1)) == 0);
499eda14cbcSMatt Macy
500eda14cbcSMatt Macy while (i < size) {
501eda14cbcSMatt Macy dnode_phys_t *dnp = (void *)(((char *)vbuf) + i);
502eda14cbcSMatt Macy dnode_byteswap(dnp);
503eda14cbcSMatt Macy
504eda14cbcSMatt Macy i += DNODE_MIN_SIZE;
505eda14cbcSMatt Macy if (dnp->dn_type != DMU_OT_NONE)
506eda14cbcSMatt Macy i += dnp->dn_extra_slots * DNODE_MIN_SIZE;
507eda14cbcSMatt Macy }
508eda14cbcSMatt Macy }
509eda14cbcSMatt Macy
510eda14cbcSMatt Macy void
dnode_setbonuslen(dnode_t * dn,int newsize,dmu_tx_t * tx)511eda14cbcSMatt Macy dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx)
512eda14cbcSMatt Macy {
513eda14cbcSMatt Macy ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
514eda14cbcSMatt Macy
515eda14cbcSMatt Macy dnode_setdirty(dn, tx);
516eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
517eda14cbcSMatt Macy ASSERT3U(newsize, <=, DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
518eda14cbcSMatt Macy (dn->dn_nblkptr-1) * sizeof (blkptr_t));
519eda14cbcSMatt Macy
520eda14cbcSMatt Macy if (newsize < dn->dn_bonuslen) {
521eda14cbcSMatt Macy /* clear any data after the end of the new size */
522eda14cbcSMatt Macy size_t diff = dn->dn_bonuslen - newsize;
523eda14cbcSMatt Macy char *data_end = ((char *)dn->dn_bonus->db.db_data) + newsize;
524da5137abSMartin Matuska memset(data_end, 0, diff);
525eda14cbcSMatt Macy }
526eda14cbcSMatt Macy
527eda14cbcSMatt Macy dn->dn_bonuslen = newsize;
528eda14cbcSMatt Macy if (newsize == 0)
529eda14cbcSMatt Macy dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN;
530eda14cbcSMatt Macy else
531eda14cbcSMatt Macy dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen;
532eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
533eda14cbcSMatt Macy }
534eda14cbcSMatt Macy
535eda14cbcSMatt Macy void
dnode_setbonus_type(dnode_t * dn,dmu_object_type_t newtype,dmu_tx_t * tx)536eda14cbcSMatt Macy dnode_setbonus_type(dnode_t *dn, dmu_object_type_t newtype, dmu_tx_t *tx)
537eda14cbcSMatt Macy {
538eda14cbcSMatt Macy ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
539eda14cbcSMatt Macy dnode_setdirty(dn, tx);
540eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
541eda14cbcSMatt Macy dn->dn_bonustype = newtype;
542eda14cbcSMatt Macy dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype;
543eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
544eda14cbcSMatt Macy }
545eda14cbcSMatt Macy
546eda14cbcSMatt Macy void
dnode_rm_spill(dnode_t * dn,dmu_tx_t * tx)547eda14cbcSMatt Macy dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx)
548eda14cbcSMatt Macy {
549eda14cbcSMatt Macy ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
550eda14cbcSMatt Macy ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
551eda14cbcSMatt Macy dnode_setdirty(dn, tx);
552eda14cbcSMatt Macy dn->dn_rm_spillblk[tx->tx_txg & TXG_MASK] = DN_KILL_SPILLBLK;
553eda14cbcSMatt Macy dn->dn_have_spill = B_FALSE;
554eda14cbcSMatt Macy }
555eda14cbcSMatt Macy
556eda14cbcSMatt Macy static void
dnode_setdblksz(dnode_t * dn,int size)557eda14cbcSMatt Macy dnode_setdblksz(dnode_t *dn, int size)
558eda14cbcSMatt Macy {
559eda14cbcSMatt Macy ASSERT0(P2PHASE(size, SPA_MINBLOCKSIZE));
560eda14cbcSMatt Macy ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
561eda14cbcSMatt Macy ASSERT3U(size, >=, SPA_MINBLOCKSIZE);
562eda14cbcSMatt Macy ASSERT3U(size >> SPA_MINBLOCKSHIFT, <,
563eda14cbcSMatt Macy 1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8));
564eda14cbcSMatt Macy dn->dn_datablksz = size;
565eda14cbcSMatt Macy dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT;
566eda14cbcSMatt Macy dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0;
567eda14cbcSMatt Macy }
568eda14cbcSMatt Macy
569eda14cbcSMatt Macy static dnode_t *
dnode_create(objset_t * os,dnode_phys_t * dnp,dmu_buf_impl_t * db,uint64_t object,dnode_handle_t * dnh)570eda14cbcSMatt Macy dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
571eda14cbcSMatt Macy uint64_t object, dnode_handle_t *dnh)
572eda14cbcSMatt Macy {
573eda14cbcSMatt Macy dnode_t *dn;
574eda14cbcSMatt Macy
575eda14cbcSMatt Macy dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
576eda14cbcSMatt Macy dn->dn_moved = 0;
577eda14cbcSMatt Macy
578eda14cbcSMatt Macy /*
579eda14cbcSMatt Macy * Defer setting dn_objset until the dnode is ready to be a candidate
580eda14cbcSMatt Macy * for the dnode_move() callback.
581eda14cbcSMatt Macy */
582eda14cbcSMatt Macy dn->dn_object = object;
583eda14cbcSMatt Macy dn->dn_dbuf = db;
584eda14cbcSMatt Macy dn->dn_handle = dnh;
585eda14cbcSMatt Macy dn->dn_phys = dnp;
586eda14cbcSMatt Macy
587eda14cbcSMatt Macy if (dnp->dn_datablkszsec) {
588eda14cbcSMatt Macy dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
589eda14cbcSMatt Macy } else {
590eda14cbcSMatt Macy dn->dn_datablksz = 0;
591eda14cbcSMatt Macy dn->dn_datablkszsec = 0;
592eda14cbcSMatt Macy dn->dn_datablkshift = 0;
593eda14cbcSMatt Macy }
594eda14cbcSMatt Macy dn->dn_indblkshift = dnp->dn_indblkshift;
595eda14cbcSMatt Macy dn->dn_nlevels = dnp->dn_nlevels;
596eda14cbcSMatt Macy dn->dn_type = dnp->dn_type;
597eda14cbcSMatt Macy dn->dn_nblkptr = dnp->dn_nblkptr;
598eda14cbcSMatt Macy dn->dn_checksum = dnp->dn_checksum;
599eda14cbcSMatt Macy dn->dn_compress = dnp->dn_compress;
600eda14cbcSMatt Macy dn->dn_bonustype = dnp->dn_bonustype;
601eda14cbcSMatt Macy dn->dn_bonuslen = dnp->dn_bonuslen;
602eda14cbcSMatt Macy dn->dn_num_slots = dnp->dn_extra_slots + 1;
603eda14cbcSMatt Macy dn->dn_maxblkid = dnp->dn_maxblkid;
604eda14cbcSMatt Macy dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0);
605eda14cbcSMatt Macy dn->dn_id_flags = 0;
606eda14cbcSMatt Macy
607eda14cbcSMatt Macy dmu_zfetch_init(&dn->dn_zfetch, dn);
608eda14cbcSMatt Macy
609eda14cbcSMatt Macy ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
610eda14cbcSMatt Macy ASSERT(zrl_is_locked(&dnh->dnh_zrlock));
611eda14cbcSMatt Macy ASSERT(!DN_SLOT_IS_PTR(dnh->dnh_dnode));
612eda14cbcSMatt Macy
613eda14cbcSMatt Macy mutex_enter(&os->os_lock);
614eda14cbcSMatt Macy
615eda14cbcSMatt Macy /*
616eda14cbcSMatt Macy * Exclude special dnodes from os_dnodes so an empty os_dnodes
617eda14cbcSMatt Macy * signifies that the special dnodes have no references from
618eda14cbcSMatt Macy * their children (the entries in os_dnodes). This allows
619eda14cbcSMatt Macy * dnode_destroy() to easily determine if the last child has
620eda14cbcSMatt Macy * been removed and then complete eviction of the objset.
621eda14cbcSMatt Macy */
622eda14cbcSMatt Macy if (!DMU_OBJECT_IS_SPECIAL(object))
623eda14cbcSMatt Macy list_insert_head(&os->os_dnodes, dn);
624eda14cbcSMatt Macy membar_producer();
625eda14cbcSMatt Macy
626eda14cbcSMatt Macy /*
627eda14cbcSMatt Macy * Everything else must be valid before assigning dn_objset
628eda14cbcSMatt Macy * makes the dnode eligible for dnode_move().
629eda14cbcSMatt Macy */
630eda14cbcSMatt Macy dn->dn_objset = os;
631eda14cbcSMatt Macy
632eda14cbcSMatt Macy dnh->dnh_dnode = dn;
633eda14cbcSMatt Macy mutex_exit(&os->os_lock);
634eda14cbcSMatt Macy
635eda14cbcSMatt Macy arc_space_consume(sizeof (dnode_t), ARC_SPACE_DNODE);
636eda14cbcSMatt Macy
637eda14cbcSMatt Macy return (dn);
638eda14cbcSMatt Macy }
639eda14cbcSMatt Macy
640eda14cbcSMatt Macy /*
641eda14cbcSMatt Macy * Caller must be holding the dnode handle, which is released upon return.
642eda14cbcSMatt Macy */
643eda14cbcSMatt Macy static void
dnode_destroy(dnode_t * dn)644eda14cbcSMatt Macy dnode_destroy(dnode_t *dn)
645eda14cbcSMatt Macy {
646eda14cbcSMatt Macy objset_t *os = dn->dn_objset;
647eda14cbcSMatt Macy boolean_t complete_os_eviction = B_FALSE;
648eda14cbcSMatt Macy
649eda14cbcSMatt Macy ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0);
650eda14cbcSMatt Macy
651eda14cbcSMatt Macy mutex_enter(&os->os_lock);
652eda14cbcSMatt Macy POINTER_INVALIDATE(&dn->dn_objset);
653eda14cbcSMatt Macy if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
654eda14cbcSMatt Macy list_remove(&os->os_dnodes, dn);
655eda14cbcSMatt Macy complete_os_eviction =
656eda14cbcSMatt Macy list_is_empty(&os->os_dnodes) &&
657eda14cbcSMatt Macy list_link_active(&os->os_evicting_node);
658eda14cbcSMatt Macy }
659eda14cbcSMatt Macy mutex_exit(&os->os_lock);
660eda14cbcSMatt Macy
661eda14cbcSMatt Macy /* the dnode can no longer move, so we can release the handle */
662eda14cbcSMatt Macy if (!zrl_is_locked(&dn->dn_handle->dnh_zrlock))
663eda14cbcSMatt Macy zrl_remove(&dn->dn_handle->dnh_zrlock);
664eda14cbcSMatt Macy
665eda14cbcSMatt Macy dn->dn_allocated_txg = 0;
666eda14cbcSMatt Macy dn->dn_free_txg = 0;
667eda14cbcSMatt Macy dn->dn_assigned_txg = 0;
668eda14cbcSMatt Macy dn->dn_dirty_txg = 0;
669eda14cbcSMatt Macy
670eda14cbcSMatt Macy dn->dn_dirtyctx = 0;
671eda14cbcSMatt Macy dn->dn_dirtyctx_firstset = NULL;
672eda14cbcSMatt Macy if (dn->dn_bonus != NULL) {
673eda14cbcSMatt Macy mutex_enter(&dn->dn_bonus->db_mtx);
674eda14cbcSMatt Macy dbuf_destroy(dn->dn_bonus);
675eda14cbcSMatt Macy dn->dn_bonus = NULL;
676eda14cbcSMatt Macy }
677eda14cbcSMatt Macy dn->dn_zio = NULL;
678eda14cbcSMatt Macy
679eda14cbcSMatt Macy dn->dn_have_spill = B_FALSE;
680eda14cbcSMatt Macy dn->dn_oldused = 0;
681eda14cbcSMatt Macy dn->dn_oldflags = 0;
682eda14cbcSMatt Macy dn->dn_olduid = 0;
683eda14cbcSMatt Macy dn->dn_oldgid = 0;
684eda14cbcSMatt Macy dn->dn_oldprojid = ZFS_DEFAULT_PROJID;
685eda14cbcSMatt Macy dn->dn_newuid = 0;
686eda14cbcSMatt Macy dn->dn_newgid = 0;
687eda14cbcSMatt Macy dn->dn_newprojid = ZFS_DEFAULT_PROJID;
688eda14cbcSMatt Macy dn->dn_id_flags = 0;
689eda14cbcSMatt Macy
690eda14cbcSMatt Macy dmu_zfetch_fini(&dn->dn_zfetch);
691eda14cbcSMatt Macy kmem_cache_free(dnode_cache, dn);
692eda14cbcSMatt Macy arc_space_return(sizeof (dnode_t), ARC_SPACE_DNODE);
693eda14cbcSMatt Macy
694eda14cbcSMatt Macy if (complete_os_eviction)
695eda14cbcSMatt Macy dmu_objset_evict_done(os);
696eda14cbcSMatt Macy }
697eda14cbcSMatt Macy
698eda14cbcSMatt Macy void
dnode_allocate(dnode_t * dn,dmu_object_type_t ot,int blocksize,int ibs,dmu_object_type_t bonustype,int bonuslen,int dn_slots,dmu_tx_t * tx)699eda14cbcSMatt Macy dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
700eda14cbcSMatt Macy dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx)
701eda14cbcSMatt Macy {
702eda14cbcSMatt Macy int i;
703eda14cbcSMatt Macy
704eda14cbcSMatt Macy ASSERT3U(dn_slots, >, 0);
705eda14cbcSMatt Macy ASSERT3U(dn_slots << DNODE_SHIFT, <=,
706eda14cbcSMatt Macy spa_maxdnodesize(dmu_objset_spa(dn->dn_objset)));
707eda14cbcSMatt Macy ASSERT3U(blocksize, <=,
708eda14cbcSMatt Macy spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
709eda14cbcSMatt Macy if (blocksize == 0)
710eda14cbcSMatt Macy blocksize = 1 << zfs_default_bs;
711eda14cbcSMatt Macy else
712eda14cbcSMatt Macy blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE);
713eda14cbcSMatt Macy
714eda14cbcSMatt Macy if (ibs == 0)
715eda14cbcSMatt Macy ibs = zfs_default_ibs;
716eda14cbcSMatt Macy
717eda14cbcSMatt Macy ibs = MIN(MAX(ibs, DN_MIN_INDBLKSHIFT), DN_MAX_INDBLKSHIFT);
718eda14cbcSMatt Macy
719eda14cbcSMatt Macy dprintf("os=%p obj=%llu txg=%llu blocksize=%d ibs=%d dn_slots=%d\n",
72033b8c039SMartin Matuska dn->dn_objset, (u_longlong_t)dn->dn_object,
72133b8c039SMartin Matuska (u_longlong_t)tx->tx_txg, blocksize, ibs, dn_slots);
722eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_allocate);
723eda14cbcSMatt Macy
724eda14cbcSMatt Macy ASSERT(dn->dn_type == DMU_OT_NONE);
725da5137abSMartin Matuska ASSERT0(memcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)));
726eda14cbcSMatt Macy ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE);
727eda14cbcSMatt Macy ASSERT(ot != DMU_OT_NONE);
728eda14cbcSMatt Macy ASSERT(DMU_OT_IS_VALID(ot));
729eda14cbcSMatt Macy ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) ||
730eda14cbcSMatt Macy (bonustype == DMU_OT_SA && bonuslen == 0) ||
7312ad756a6SMartin Matuska (bonustype == DMU_OTN_UINT64_METADATA && bonuslen == 0) ||
732eda14cbcSMatt Macy (bonustype != DMU_OT_NONE && bonuslen != 0));
733eda14cbcSMatt Macy ASSERT(DMU_OT_IS_VALID(bonustype));
734eda14cbcSMatt Macy ASSERT3U(bonuslen, <=, DN_SLOTS_TO_BONUSLEN(dn_slots));
735eda14cbcSMatt Macy ASSERT(dn->dn_type == DMU_OT_NONE);
736eda14cbcSMatt Macy ASSERT0(dn->dn_maxblkid);
737eda14cbcSMatt Macy ASSERT0(dn->dn_allocated_txg);
738eda14cbcSMatt Macy ASSERT0(dn->dn_assigned_txg);
739eda14cbcSMatt Macy ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds));
740eda14cbcSMatt Macy ASSERT3U(zfs_refcount_count(&dn->dn_holds), <=, 1);
741eda14cbcSMatt Macy ASSERT(avl_is_empty(&dn->dn_dbufs));
742eda14cbcSMatt Macy
743eda14cbcSMatt Macy for (i = 0; i < TXG_SIZE; i++) {
744eda14cbcSMatt Macy ASSERT0(dn->dn_next_nblkptr[i]);
745eda14cbcSMatt Macy ASSERT0(dn->dn_next_nlevels[i]);
746eda14cbcSMatt Macy ASSERT0(dn->dn_next_indblkshift[i]);
747eda14cbcSMatt Macy ASSERT0(dn->dn_next_bonuslen[i]);
748eda14cbcSMatt Macy ASSERT0(dn->dn_next_bonustype[i]);
749eda14cbcSMatt Macy ASSERT0(dn->dn_rm_spillblk[i]);
750eda14cbcSMatt Macy ASSERT0(dn->dn_next_blksz[i]);
751eda14cbcSMatt Macy ASSERT0(dn->dn_next_maxblkid[i]);
752eda14cbcSMatt Macy ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
753eda14cbcSMatt Macy ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL);
754eda14cbcSMatt Macy ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
755eda14cbcSMatt Macy }
756eda14cbcSMatt Macy
757eda14cbcSMatt Macy dn->dn_type = ot;
758eda14cbcSMatt Macy dnode_setdblksz(dn, blocksize);
759eda14cbcSMatt Macy dn->dn_indblkshift = ibs;
760eda14cbcSMatt Macy dn->dn_nlevels = 1;
761eda14cbcSMatt Macy dn->dn_num_slots = dn_slots;
762eda14cbcSMatt Macy if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */
763eda14cbcSMatt Macy dn->dn_nblkptr = 1;
764eda14cbcSMatt Macy else {
765eda14cbcSMatt Macy dn->dn_nblkptr = MIN(DN_MAX_NBLKPTR,
766eda14cbcSMatt Macy 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >>
767eda14cbcSMatt Macy SPA_BLKPTRSHIFT));
768eda14cbcSMatt Macy }
769eda14cbcSMatt Macy
770eda14cbcSMatt Macy dn->dn_bonustype = bonustype;
771eda14cbcSMatt Macy dn->dn_bonuslen = bonuslen;
772eda14cbcSMatt Macy dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
773eda14cbcSMatt Macy dn->dn_compress = ZIO_COMPRESS_INHERIT;
774eda14cbcSMatt Macy dn->dn_dirtyctx = 0;
775eda14cbcSMatt Macy
776eda14cbcSMatt Macy dn->dn_free_txg = 0;
777eda14cbcSMatt Macy dn->dn_dirtyctx_firstset = NULL;
7787877fdebSMatt Macy dn->dn_dirty_txg = 0;
779eda14cbcSMatt Macy
780eda14cbcSMatt Macy dn->dn_allocated_txg = tx->tx_txg;
781eda14cbcSMatt Macy dn->dn_id_flags = 0;
782eda14cbcSMatt Macy
783eda14cbcSMatt Macy dnode_setdirty(dn, tx);
784eda14cbcSMatt Macy dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs;
785eda14cbcSMatt Macy dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen;
786eda14cbcSMatt Macy dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype;
787eda14cbcSMatt Macy dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz;
788eda14cbcSMatt Macy }
789eda14cbcSMatt Macy
790eda14cbcSMatt Macy void
dnode_reallocate(dnode_t * dn,dmu_object_type_t ot,int blocksize,dmu_object_type_t bonustype,int bonuslen,int dn_slots,boolean_t keep_spill,dmu_tx_t * tx)791eda14cbcSMatt Macy dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
792eda14cbcSMatt Macy dmu_object_type_t bonustype, int bonuslen, int dn_slots,
793eda14cbcSMatt Macy boolean_t keep_spill, dmu_tx_t *tx)
794eda14cbcSMatt Macy {
795eda14cbcSMatt Macy int nblkptr;
796eda14cbcSMatt Macy
797eda14cbcSMatt Macy ASSERT3U(blocksize, >=, SPA_MINBLOCKSIZE);
798eda14cbcSMatt Macy ASSERT3U(blocksize, <=,
799eda14cbcSMatt Macy spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
800eda14cbcSMatt Macy ASSERT0(blocksize % SPA_MINBLOCKSIZE);
801eda14cbcSMatt Macy ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx));
802eda14cbcSMatt Macy ASSERT(tx->tx_txg != 0);
803eda14cbcSMatt Macy ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) ||
804eda14cbcSMatt Macy (bonustype != DMU_OT_NONE && bonuslen != 0) ||
805eda14cbcSMatt Macy (bonustype == DMU_OT_SA && bonuslen == 0));
806eda14cbcSMatt Macy ASSERT(DMU_OT_IS_VALID(bonustype));
807eda14cbcSMatt Macy ASSERT3U(bonuslen, <=,
808eda14cbcSMatt Macy DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))));
809eda14cbcSMatt Macy ASSERT3U(bonuslen, <=, DN_BONUS_SIZE(dn_slots << DNODE_SHIFT));
810eda14cbcSMatt Macy
811eda14cbcSMatt Macy dnode_free_interior_slots(dn);
812eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_reallocate);
813eda14cbcSMatt Macy
814eda14cbcSMatt Macy /* clean up any unreferenced dbufs */
815eda14cbcSMatt Macy dnode_evict_dbufs(dn);
816eda14cbcSMatt Macy
817eda14cbcSMatt Macy dn->dn_id_flags = 0;
818eda14cbcSMatt Macy
819eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
820eda14cbcSMatt Macy dnode_setdirty(dn, tx);
821eda14cbcSMatt Macy if (dn->dn_datablksz != blocksize) {
822eda14cbcSMatt Macy /* change blocksize */
823eda14cbcSMatt Macy ASSERT0(dn->dn_maxblkid);
824eda14cbcSMatt Macy ASSERT(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) ||
825eda14cbcSMatt Macy dnode_block_freed(dn, 0));
826eda14cbcSMatt Macy
827eda14cbcSMatt Macy dnode_setdblksz(dn, blocksize);
828eda14cbcSMatt Macy dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = blocksize;
829eda14cbcSMatt Macy }
830eda14cbcSMatt Macy if (dn->dn_bonuslen != bonuslen)
831eda14cbcSMatt Macy dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = bonuslen;
832eda14cbcSMatt Macy
833eda14cbcSMatt Macy if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */
834eda14cbcSMatt Macy nblkptr = 1;
835eda14cbcSMatt Macy else
836eda14cbcSMatt Macy nblkptr = MIN(DN_MAX_NBLKPTR,
837eda14cbcSMatt Macy 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >>
838eda14cbcSMatt Macy SPA_BLKPTRSHIFT));
839eda14cbcSMatt Macy if (dn->dn_bonustype != bonustype)
840eda14cbcSMatt Macy dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = bonustype;
841eda14cbcSMatt Macy if (dn->dn_nblkptr != nblkptr)
842eda14cbcSMatt Macy dn->dn_next_nblkptr[tx->tx_txg & TXG_MASK] = nblkptr;
843eda14cbcSMatt Macy if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR && !keep_spill) {
844eda14cbcSMatt Macy dbuf_rm_spill(dn, tx);
845eda14cbcSMatt Macy dnode_rm_spill(dn, tx);
846eda14cbcSMatt Macy }
847eda14cbcSMatt Macy
848eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
849eda14cbcSMatt Macy
850eda14cbcSMatt Macy /* change type */
851eda14cbcSMatt Macy dn->dn_type = ot;
852eda14cbcSMatt Macy
853eda14cbcSMatt Macy /* change bonus size and type */
854eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
855eda14cbcSMatt Macy dn->dn_bonustype = bonustype;
856eda14cbcSMatt Macy dn->dn_bonuslen = bonuslen;
857eda14cbcSMatt Macy dn->dn_num_slots = dn_slots;
858eda14cbcSMatt Macy dn->dn_nblkptr = nblkptr;
859eda14cbcSMatt Macy dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
860eda14cbcSMatt Macy dn->dn_compress = ZIO_COMPRESS_INHERIT;
861eda14cbcSMatt Macy ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR);
862eda14cbcSMatt Macy
863eda14cbcSMatt Macy /* fix up the bonus db_size */
864eda14cbcSMatt Macy if (dn->dn_bonus) {
865eda14cbcSMatt Macy dn->dn_bonus->db.db_size =
866eda14cbcSMatt Macy DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
867eda14cbcSMatt Macy (dn->dn_nblkptr-1) * sizeof (blkptr_t);
868eda14cbcSMatt Macy ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size);
869eda14cbcSMatt Macy }
870eda14cbcSMatt Macy
871eda14cbcSMatt Macy dn->dn_allocated_txg = tx->tx_txg;
872eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
873eda14cbcSMatt Macy }
874eda14cbcSMatt Macy
875eda14cbcSMatt Macy #ifdef _KERNEL
876eda14cbcSMatt Macy static void
dnode_move_impl(dnode_t * odn,dnode_t * ndn)877eda14cbcSMatt Macy dnode_move_impl(dnode_t *odn, dnode_t *ndn)
878eda14cbcSMatt Macy {
879eda14cbcSMatt Macy ASSERT(!RW_LOCK_HELD(&odn->dn_struct_rwlock));
880eda14cbcSMatt Macy ASSERT(MUTEX_NOT_HELD(&odn->dn_mtx));
881eda14cbcSMatt Macy ASSERT(MUTEX_NOT_HELD(&odn->dn_dbufs_mtx));
882eda14cbcSMatt Macy
883eda14cbcSMatt Macy /* Copy fields. */
884eda14cbcSMatt Macy ndn->dn_objset = odn->dn_objset;
885eda14cbcSMatt Macy ndn->dn_object = odn->dn_object;
886eda14cbcSMatt Macy ndn->dn_dbuf = odn->dn_dbuf;
887eda14cbcSMatt Macy ndn->dn_handle = odn->dn_handle;
888eda14cbcSMatt Macy ndn->dn_phys = odn->dn_phys;
889eda14cbcSMatt Macy ndn->dn_type = odn->dn_type;
890eda14cbcSMatt Macy ndn->dn_bonuslen = odn->dn_bonuslen;
891eda14cbcSMatt Macy ndn->dn_bonustype = odn->dn_bonustype;
892eda14cbcSMatt Macy ndn->dn_nblkptr = odn->dn_nblkptr;
893eda14cbcSMatt Macy ndn->dn_checksum = odn->dn_checksum;
894eda14cbcSMatt Macy ndn->dn_compress = odn->dn_compress;
895eda14cbcSMatt Macy ndn->dn_nlevels = odn->dn_nlevels;
896eda14cbcSMatt Macy ndn->dn_indblkshift = odn->dn_indblkshift;
897eda14cbcSMatt Macy ndn->dn_datablkshift = odn->dn_datablkshift;
898eda14cbcSMatt Macy ndn->dn_datablkszsec = odn->dn_datablkszsec;
899eda14cbcSMatt Macy ndn->dn_datablksz = odn->dn_datablksz;
900eda14cbcSMatt Macy ndn->dn_maxblkid = odn->dn_maxblkid;
901eda14cbcSMatt Macy ndn->dn_num_slots = odn->dn_num_slots;
902da5137abSMartin Matuska memcpy(ndn->dn_next_type, odn->dn_next_type,
903eda14cbcSMatt Macy sizeof (odn->dn_next_type));
904da5137abSMartin Matuska memcpy(ndn->dn_next_nblkptr, odn->dn_next_nblkptr,
905eda14cbcSMatt Macy sizeof (odn->dn_next_nblkptr));
906da5137abSMartin Matuska memcpy(ndn->dn_next_nlevels, odn->dn_next_nlevels,
907eda14cbcSMatt Macy sizeof (odn->dn_next_nlevels));
908da5137abSMartin Matuska memcpy(ndn->dn_next_indblkshift, odn->dn_next_indblkshift,
909eda14cbcSMatt Macy sizeof (odn->dn_next_indblkshift));
910da5137abSMartin Matuska memcpy(ndn->dn_next_bonustype, odn->dn_next_bonustype,
911eda14cbcSMatt Macy sizeof (odn->dn_next_bonustype));
912da5137abSMartin Matuska memcpy(ndn->dn_rm_spillblk, odn->dn_rm_spillblk,
913eda14cbcSMatt Macy sizeof (odn->dn_rm_spillblk));
914da5137abSMartin Matuska memcpy(ndn->dn_next_bonuslen, odn->dn_next_bonuslen,
915eda14cbcSMatt Macy sizeof (odn->dn_next_bonuslen));
916da5137abSMartin Matuska memcpy(ndn->dn_next_blksz, odn->dn_next_blksz,
917eda14cbcSMatt Macy sizeof (odn->dn_next_blksz));
918da5137abSMartin Matuska memcpy(ndn->dn_next_maxblkid, odn->dn_next_maxblkid,
919eda14cbcSMatt Macy sizeof (odn->dn_next_maxblkid));
920da5137abSMartin Matuska for (int i = 0; i < TXG_SIZE; i++) {
921eda14cbcSMatt Macy list_move_tail(&ndn->dn_dirty_records[i],
922eda14cbcSMatt Macy &odn->dn_dirty_records[i]);
923eda14cbcSMatt Macy }
924da5137abSMartin Matuska memcpy(ndn->dn_free_ranges, odn->dn_free_ranges,
925eda14cbcSMatt Macy sizeof (odn->dn_free_ranges));
926eda14cbcSMatt Macy ndn->dn_allocated_txg = odn->dn_allocated_txg;
927eda14cbcSMatt Macy ndn->dn_free_txg = odn->dn_free_txg;
928eda14cbcSMatt Macy ndn->dn_assigned_txg = odn->dn_assigned_txg;
929eda14cbcSMatt Macy ndn->dn_dirty_txg = odn->dn_dirty_txg;
930eda14cbcSMatt Macy ndn->dn_dirtyctx = odn->dn_dirtyctx;
931eda14cbcSMatt Macy ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset;
932eda14cbcSMatt Macy ASSERT(zfs_refcount_count(&odn->dn_tx_holds) == 0);
933eda14cbcSMatt Macy zfs_refcount_transfer(&ndn->dn_holds, &odn->dn_holds);
934eda14cbcSMatt Macy ASSERT(avl_is_empty(&ndn->dn_dbufs));
935eda14cbcSMatt Macy avl_swap(&ndn->dn_dbufs, &odn->dn_dbufs);
936eda14cbcSMatt Macy ndn->dn_dbufs_count = odn->dn_dbufs_count;
937eda14cbcSMatt Macy ndn->dn_bonus = odn->dn_bonus;
938eda14cbcSMatt Macy ndn->dn_have_spill = odn->dn_have_spill;
939eda14cbcSMatt Macy ndn->dn_zio = odn->dn_zio;
940eda14cbcSMatt Macy ndn->dn_oldused = odn->dn_oldused;
941eda14cbcSMatt Macy ndn->dn_oldflags = odn->dn_oldflags;
942eda14cbcSMatt Macy ndn->dn_olduid = odn->dn_olduid;
943eda14cbcSMatt Macy ndn->dn_oldgid = odn->dn_oldgid;
944eda14cbcSMatt Macy ndn->dn_oldprojid = odn->dn_oldprojid;
945eda14cbcSMatt Macy ndn->dn_newuid = odn->dn_newuid;
946eda14cbcSMatt Macy ndn->dn_newgid = odn->dn_newgid;
947eda14cbcSMatt Macy ndn->dn_newprojid = odn->dn_newprojid;
948eda14cbcSMatt Macy ndn->dn_id_flags = odn->dn_id_flags;
94916038816SMartin Matuska dmu_zfetch_init(&ndn->dn_zfetch, ndn);
950eda14cbcSMatt Macy
951eda14cbcSMatt Macy /*
952eda14cbcSMatt Macy * Update back pointers. Updating the handle fixes the back pointer of
953eda14cbcSMatt Macy * every descendant dbuf as well as the bonus dbuf.
954eda14cbcSMatt Macy */
955eda14cbcSMatt Macy ASSERT(ndn->dn_handle->dnh_dnode == odn);
956eda14cbcSMatt Macy ndn->dn_handle->dnh_dnode = ndn;
957eda14cbcSMatt Macy
958eda14cbcSMatt Macy /*
959eda14cbcSMatt Macy * Invalidate the original dnode by clearing all of its back pointers.
960eda14cbcSMatt Macy */
961eda14cbcSMatt Macy odn->dn_dbuf = NULL;
962eda14cbcSMatt Macy odn->dn_handle = NULL;
963eda14cbcSMatt Macy avl_create(&odn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t),
964eda14cbcSMatt Macy offsetof(dmu_buf_impl_t, db_link));
965eda14cbcSMatt Macy odn->dn_dbufs_count = 0;
966eda14cbcSMatt Macy odn->dn_bonus = NULL;
967eda14cbcSMatt Macy dmu_zfetch_fini(&odn->dn_zfetch);
968eda14cbcSMatt Macy
969eda14cbcSMatt Macy /*
970eda14cbcSMatt Macy * Set the low bit of the objset pointer to ensure that dnode_move()
971eda14cbcSMatt Macy * recognizes the dnode as invalid in any subsequent callback.
972eda14cbcSMatt Macy */
973eda14cbcSMatt Macy POINTER_INVALIDATE(&odn->dn_objset);
974eda14cbcSMatt Macy
975eda14cbcSMatt Macy /*
976eda14cbcSMatt Macy * Satisfy the destructor.
977eda14cbcSMatt Macy */
978da5137abSMartin Matuska for (int i = 0; i < TXG_SIZE; i++) {
979eda14cbcSMatt Macy list_create(&odn->dn_dirty_records[i],
980eda14cbcSMatt Macy sizeof (dbuf_dirty_record_t),
981eda14cbcSMatt Macy offsetof(dbuf_dirty_record_t, dr_dirty_node));
982eda14cbcSMatt Macy odn->dn_free_ranges[i] = NULL;
983eda14cbcSMatt Macy odn->dn_next_nlevels[i] = 0;
984eda14cbcSMatt Macy odn->dn_next_indblkshift[i] = 0;
985eda14cbcSMatt Macy odn->dn_next_bonustype[i] = 0;
986eda14cbcSMatt Macy odn->dn_rm_spillblk[i] = 0;
987eda14cbcSMatt Macy odn->dn_next_bonuslen[i] = 0;
988eda14cbcSMatt Macy odn->dn_next_blksz[i] = 0;
989eda14cbcSMatt Macy }
990eda14cbcSMatt Macy odn->dn_allocated_txg = 0;
991eda14cbcSMatt Macy odn->dn_free_txg = 0;
992eda14cbcSMatt Macy odn->dn_assigned_txg = 0;
993eda14cbcSMatt Macy odn->dn_dirty_txg = 0;
994eda14cbcSMatt Macy odn->dn_dirtyctx = 0;
995eda14cbcSMatt Macy odn->dn_dirtyctx_firstset = NULL;
996eda14cbcSMatt Macy odn->dn_have_spill = B_FALSE;
997eda14cbcSMatt Macy odn->dn_zio = NULL;
998eda14cbcSMatt Macy odn->dn_oldused = 0;
999eda14cbcSMatt Macy odn->dn_oldflags = 0;
1000eda14cbcSMatt Macy odn->dn_olduid = 0;
1001eda14cbcSMatt Macy odn->dn_oldgid = 0;
1002eda14cbcSMatt Macy odn->dn_oldprojid = ZFS_DEFAULT_PROJID;
1003eda14cbcSMatt Macy odn->dn_newuid = 0;
1004eda14cbcSMatt Macy odn->dn_newgid = 0;
1005eda14cbcSMatt Macy odn->dn_newprojid = ZFS_DEFAULT_PROJID;
1006eda14cbcSMatt Macy odn->dn_id_flags = 0;
1007eda14cbcSMatt Macy
1008eda14cbcSMatt Macy /*
1009eda14cbcSMatt Macy * Mark the dnode.
1010eda14cbcSMatt Macy */
1011eda14cbcSMatt Macy ndn->dn_moved = 1;
1012eda14cbcSMatt Macy odn->dn_moved = (uint8_t)-1;
1013eda14cbcSMatt Macy }
1014eda14cbcSMatt Macy
1015eda14cbcSMatt Macy static kmem_cbrc_t
dnode_move(void * buf,void * newbuf,size_t size,void * arg)1016eda14cbcSMatt Macy dnode_move(void *buf, void *newbuf, size_t size, void *arg)
1017eda14cbcSMatt Macy {
1018eda14cbcSMatt Macy dnode_t *odn = buf, *ndn = newbuf;
1019eda14cbcSMatt Macy objset_t *os;
1020eda14cbcSMatt Macy int64_t refcount;
1021eda14cbcSMatt Macy uint32_t dbufs;
1022eda14cbcSMatt Macy
1023eda14cbcSMatt Macy /*
1024eda14cbcSMatt Macy * The dnode is on the objset's list of known dnodes if the objset
1025eda14cbcSMatt Macy * pointer is valid. We set the low bit of the objset pointer when
1026eda14cbcSMatt Macy * freeing the dnode to invalidate it, and the memory patterns written
1027eda14cbcSMatt Macy * by kmem (baddcafe and deadbeef) set at least one of the two low bits.
1028eda14cbcSMatt Macy * A newly created dnode sets the objset pointer last of all to indicate
1029eda14cbcSMatt Macy * that the dnode is known and in a valid state to be moved by this
1030eda14cbcSMatt Macy * function.
1031eda14cbcSMatt Macy */
1032eda14cbcSMatt Macy os = odn->dn_objset;
1033eda14cbcSMatt Macy if (!POINTER_IS_VALID(os)) {
1034eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_move_invalid);
1035eda14cbcSMatt Macy return (KMEM_CBRC_DONT_KNOW);
1036eda14cbcSMatt Macy }
1037eda14cbcSMatt Macy
1038eda14cbcSMatt Macy /*
1039eda14cbcSMatt Macy * Ensure that the objset does not go away during the move.
1040eda14cbcSMatt Macy */
1041eda14cbcSMatt Macy rw_enter(&os_lock, RW_WRITER);
1042eda14cbcSMatt Macy if (os != odn->dn_objset) {
1043eda14cbcSMatt Macy rw_exit(&os_lock);
1044eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_move_recheck1);
1045eda14cbcSMatt Macy return (KMEM_CBRC_DONT_KNOW);
1046eda14cbcSMatt Macy }
1047eda14cbcSMatt Macy
1048eda14cbcSMatt Macy /*
1049eda14cbcSMatt Macy * If the dnode is still valid, then so is the objset. We know that no
1050eda14cbcSMatt Macy * valid objset can be freed while we hold os_lock, so we can safely
1051eda14cbcSMatt Macy * ensure that the objset remains in use.
1052eda14cbcSMatt Macy */
1053eda14cbcSMatt Macy mutex_enter(&os->os_lock);
1054eda14cbcSMatt Macy
1055eda14cbcSMatt Macy /*
1056eda14cbcSMatt Macy * Recheck the objset pointer in case the dnode was removed just before
1057eda14cbcSMatt Macy * acquiring the lock.
1058eda14cbcSMatt Macy */
1059eda14cbcSMatt Macy if (os != odn->dn_objset) {
1060eda14cbcSMatt Macy mutex_exit(&os->os_lock);
1061eda14cbcSMatt Macy rw_exit(&os_lock);
1062eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_move_recheck2);
1063eda14cbcSMatt Macy return (KMEM_CBRC_DONT_KNOW);
1064eda14cbcSMatt Macy }
1065eda14cbcSMatt Macy
1066eda14cbcSMatt Macy /*
1067eda14cbcSMatt Macy * At this point we know that as long as we hold os->os_lock, the dnode
1068eda14cbcSMatt Macy * cannot be freed and fields within the dnode can be safely accessed.
1069eda14cbcSMatt Macy * The objset listing this dnode cannot go away as long as this dnode is
1070eda14cbcSMatt Macy * on its list.
1071eda14cbcSMatt Macy */
1072eda14cbcSMatt Macy rw_exit(&os_lock);
1073eda14cbcSMatt Macy if (DMU_OBJECT_IS_SPECIAL(odn->dn_object)) {
1074eda14cbcSMatt Macy mutex_exit(&os->os_lock);
1075eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_move_special);
1076eda14cbcSMatt Macy return (KMEM_CBRC_NO);
1077eda14cbcSMatt Macy }
1078eda14cbcSMatt Macy ASSERT(odn->dn_dbuf != NULL); /* only "special" dnodes have no parent */
1079eda14cbcSMatt Macy
1080eda14cbcSMatt Macy /*
1081eda14cbcSMatt Macy * Lock the dnode handle to prevent the dnode from obtaining any new
1082eda14cbcSMatt Macy * holds. This also prevents the descendant dbufs and the bonus dbuf
1083eda14cbcSMatt Macy * from accessing the dnode, so that we can discount their holds. The
1084eda14cbcSMatt Macy * handle is safe to access because we know that while the dnode cannot
1085eda14cbcSMatt Macy * go away, neither can its handle. Once we hold dnh_zrlock, we can
1086eda14cbcSMatt Macy * safely move any dnode referenced only by dbufs.
1087eda14cbcSMatt Macy */
1088eda14cbcSMatt Macy if (!zrl_tryenter(&odn->dn_handle->dnh_zrlock)) {
1089eda14cbcSMatt Macy mutex_exit(&os->os_lock);
1090eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_move_handle);
1091eda14cbcSMatt Macy return (KMEM_CBRC_LATER);
1092eda14cbcSMatt Macy }
1093eda14cbcSMatt Macy
1094eda14cbcSMatt Macy /*
1095eda14cbcSMatt Macy * Ensure a consistent view of the dnode's holds and the dnode's dbufs.
1096eda14cbcSMatt Macy * We need to guarantee that there is a hold for every dbuf in order to
1097eda14cbcSMatt Macy * determine whether the dnode is actively referenced. Falsely matching
1098eda14cbcSMatt Macy * a dbuf to an active hold would lead to an unsafe move. It's possible
1099eda14cbcSMatt Macy * that a thread already having an active dnode hold is about to add a
1100eda14cbcSMatt Macy * dbuf, and we can't compare hold and dbuf counts while the add is in
1101eda14cbcSMatt Macy * progress.
1102eda14cbcSMatt Macy */
1103eda14cbcSMatt Macy if (!rw_tryenter(&odn->dn_struct_rwlock, RW_WRITER)) {
1104eda14cbcSMatt Macy zrl_exit(&odn->dn_handle->dnh_zrlock);
1105eda14cbcSMatt Macy mutex_exit(&os->os_lock);
1106eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_move_rwlock);
1107eda14cbcSMatt Macy return (KMEM_CBRC_LATER);
1108eda14cbcSMatt Macy }
1109eda14cbcSMatt Macy
1110eda14cbcSMatt Macy /*
1111eda14cbcSMatt Macy * A dbuf may be removed (evicted) without an active dnode hold. In that
1112eda14cbcSMatt Macy * case, the dbuf count is decremented under the handle lock before the
1113eda14cbcSMatt Macy * dbuf's hold is released. This order ensures that if we count the hold
1114eda14cbcSMatt Macy * after the dbuf is removed but before its hold is released, we will
1115eda14cbcSMatt Macy * treat the unmatched hold as active and exit safely. If we count the
1116eda14cbcSMatt Macy * hold before the dbuf is removed, the hold is discounted, and the
1117eda14cbcSMatt Macy * removal is blocked until the move completes.
1118eda14cbcSMatt Macy */
1119eda14cbcSMatt Macy refcount = zfs_refcount_count(&odn->dn_holds);
1120eda14cbcSMatt Macy ASSERT(refcount >= 0);
1121eda14cbcSMatt Macy dbufs = DN_DBUFS_COUNT(odn);
1122eda14cbcSMatt Macy
1123eda14cbcSMatt Macy /* We can't have more dbufs than dnode holds. */
1124eda14cbcSMatt Macy ASSERT3U(dbufs, <=, refcount);
1125eda14cbcSMatt Macy DTRACE_PROBE3(dnode__move, dnode_t *, odn, int64_t, refcount,
1126eda14cbcSMatt Macy uint32_t, dbufs);
1127eda14cbcSMatt Macy
1128eda14cbcSMatt Macy if (refcount > dbufs) {
1129eda14cbcSMatt Macy rw_exit(&odn->dn_struct_rwlock);
1130eda14cbcSMatt Macy zrl_exit(&odn->dn_handle->dnh_zrlock);
1131eda14cbcSMatt Macy mutex_exit(&os->os_lock);
1132eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_move_active);
1133eda14cbcSMatt Macy return (KMEM_CBRC_LATER);
1134eda14cbcSMatt Macy }
1135eda14cbcSMatt Macy
1136eda14cbcSMatt Macy rw_exit(&odn->dn_struct_rwlock);
1137eda14cbcSMatt Macy
1138eda14cbcSMatt Macy /*
1139eda14cbcSMatt Macy * At this point we know that anyone with a hold on the dnode is not
1140eda14cbcSMatt Macy * actively referencing it. The dnode is known and in a valid state to
1141eda14cbcSMatt Macy * move. We're holding the locks needed to execute the critical section.
1142eda14cbcSMatt Macy */
1143eda14cbcSMatt Macy dnode_move_impl(odn, ndn);
1144eda14cbcSMatt Macy
1145eda14cbcSMatt Macy list_link_replace(&odn->dn_link, &ndn->dn_link);
1146eda14cbcSMatt Macy /* If the dnode was safe to move, the refcount cannot have changed. */
1147eda14cbcSMatt Macy ASSERT(refcount == zfs_refcount_count(&ndn->dn_holds));
1148eda14cbcSMatt Macy ASSERT(dbufs == DN_DBUFS_COUNT(ndn));
1149eda14cbcSMatt Macy zrl_exit(&ndn->dn_handle->dnh_zrlock); /* handle has moved */
1150eda14cbcSMatt Macy mutex_exit(&os->os_lock);
1151eda14cbcSMatt Macy
1152eda14cbcSMatt Macy return (KMEM_CBRC_YES);
1153eda14cbcSMatt Macy }
1154eda14cbcSMatt Macy #endif /* _KERNEL */
1155eda14cbcSMatt Macy
1156eda14cbcSMatt Macy static void
dnode_slots_hold(dnode_children_t * children,int idx,int slots)1157eda14cbcSMatt Macy dnode_slots_hold(dnode_children_t *children, int idx, int slots)
1158eda14cbcSMatt Macy {
1159eda14cbcSMatt Macy ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1160eda14cbcSMatt Macy
1161eda14cbcSMatt Macy for (int i = idx; i < idx + slots; i++) {
1162eda14cbcSMatt Macy dnode_handle_t *dnh = &children->dnc_children[i];
1163eda14cbcSMatt Macy zrl_add(&dnh->dnh_zrlock);
1164eda14cbcSMatt Macy }
1165eda14cbcSMatt Macy }
1166eda14cbcSMatt Macy
1167eda14cbcSMatt Macy static void
dnode_slots_rele(dnode_children_t * children,int idx,int slots)1168eda14cbcSMatt Macy dnode_slots_rele(dnode_children_t *children, int idx, int slots)
1169eda14cbcSMatt Macy {
1170eda14cbcSMatt Macy ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1171eda14cbcSMatt Macy
1172eda14cbcSMatt Macy for (int i = idx; i < idx + slots; i++) {
1173eda14cbcSMatt Macy dnode_handle_t *dnh = &children->dnc_children[i];
1174eda14cbcSMatt Macy
1175eda14cbcSMatt Macy if (zrl_is_locked(&dnh->dnh_zrlock))
1176eda14cbcSMatt Macy zrl_exit(&dnh->dnh_zrlock);
1177eda14cbcSMatt Macy else
1178eda14cbcSMatt Macy zrl_remove(&dnh->dnh_zrlock);
1179eda14cbcSMatt Macy }
1180eda14cbcSMatt Macy }
1181eda14cbcSMatt Macy
1182eda14cbcSMatt Macy static int
dnode_slots_tryenter(dnode_children_t * children,int idx,int slots)1183eda14cbcSMatt Macy dnode_slots_tryenter(dnode_children_t *children, int idx, int slots)
1184eda14cbcSMatt Macy {
1185eda14cbcSMatt Macy ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1186eda14cbcSMatt Macy
1187eda14cbcSMatt Macy for (int i = idx; i < idx + slots; i++) {
1188eda14cbcSMatt Macy dnode_handle_t *dnh = &children->dnc_children[i];
1189eda14cbcSMatt Macy
1190eda14cbcSMatt Macy if (!zrl_tryenter(&dnh->dnh_zrlock)) {
1191eda14cbcSMatt Macy for (int j = idx; j < i; j++) {
1192eda14cbcSMatt Macy dnh = &children->dnc_children[j];
1193eda14cbcSMatt Macy zrl_exit(&dnh->dnh_zrlock);
1194eda14cbcSMatt Macy }
1195eda14cbcSMatt Macy
1196eda14cbcSMatt Macy return (0);
1197eda14cbcSMatt Macy }
1198eda14cbcSMatt Macy }
1199eda14cbcSMatt Macy
1200eda14cbcSMatt Macy return (1);
1201eda14cbcSMatt Macy }
1202eda14cbcSMatt Macy
1203eda14cbcSMatt Macy static void
dnode_set_slots(dnode_children_t * children,int idx,int slots,void * ptr)1204eda14cbcSMatt Macy dnode_set_slots(dnode_children_t *children, int idx, int slots, void *ptr)
1205eda14cbcSMatt Macy {
1206eda14cbcSMatt Macy ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1207eda14cbcSMatt Macy
1208eda14cbcSMatt Macy for (int i = idx; i < idx + slots; i++) {
1209eda14cbcSMatt Macy dnode_handle_t *dnh = &children->dnc_children[i];
1210eda14cbcSMatt Macy dnh->dnh_dnode = ptr;
1211eda14cbcSMatt Macy }
1212eda14cbcSMatt Macy }
1213eda14cbcSMatt Macy
1214eda14cbcSMatt Macy static boolean_t
dnode_check_slots_free(dnode_children_t * children,int idx,int slots)1215eda14cbcSMatt Macy dnode_check_slots_free(dnode_children_t *children, int idx, int slots)
1216eda14cbcSMatt Macy {
1217eda14cbcSMatt Macy ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1218eda14cbcSMatt Macy
1219eda14cbcSMatt Macy /*
1220eda14cbcSMatt Macy * If all dnode slots are either already free or
1221eda14cbcSMatt Macy * evictable return B_TRUE.
1222eda14cbcSMatt Macy */
1223eda14cbcSMatt Macy for (int i = idx; i < idx + slots; i++) {
1224eda14cbcSMatt Macy dnode_handle_t *dnh = &children->dnc_children[i];
1225eda14cbcSMatt Macy dnode_t *dn = dnh->dnh_dnode;
1226eda14cbcSMatt Macy
1227eda14cbcSMatt Macy if (dn == DN_SLOT_FREE) {
1228eda14cbcSMatt Macy continue;
1229eda14cbcSMatt Macy } else if (DN_SLOT_IS_PTR(dn)) {
1230eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1231eda14cbcSMatt Macy boolean_t can_free = (dn->dn_type == DMU_OT_NONE &&
1232eda14cbcSMatt Macy zfs_refcount_is_zero(&dn->dn_holds) &&
1233eda14cbcSMatt Macy !DNODE_IS_DIRTY(dn));
1234eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1235eda14cbcSMatt Macy
1236eda14cbcSMatt Macy if (!can_free)
1237eda14cbcSMatt Macy return (B_FALSE);
1238eda14cbcSMatt Macy else
1239eda14cbcSMatt Macy continue;
1240eda14cbcSMatt Macy } else {
1241eda14cbcSMatt Macy return (B_FALSE);
1242eda14cbcSMatt Macy }
1243eda14cbcSMatt Macy }
1244eda14cbcSMatt Macy
1245eda14cbcSMatt Macy return (B_TRUE);
1246eda14cbcSMatt Macy }
1247eda14cbcSMatt Macy
1248a2b560ccSMartin Matuska static uint_t
dnode_reclaim_slots(dnode_children_t * children,int idx,int slots)1249eda14cbcSMatt Macy dnode_reclaim_slots(dnode_children_t *children, int idx, int slots)
1250eda14cbcSMatt Macy {
1251a2b560ccSMartin Matuska uint_t reclaimed = 0;
1252a2b560ccSMartin Matuska
1253eda14cbcSMatt Macy ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1254eda14cbcSMatt Macy
1255eda14cbcSMatt Macy for (int i = idx; i < idx + slots; i++) {
1256eda14cbcSMatt Macy dnode_handle_t *dnh = &children->dnc_children[i];
1257eda14cbcSMatt Macy
1258eda14cbcSMatt Macy ASSERT(zrl_is_locked(&dnh->dnh_zrlock));
1259eda14cbcSMatt Macy
1260eda14cbcSMatt Macy if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1261eda14cbcSMatt Macy ASSERT3S(dnh->dnh_dnode->dn_type, ==, DMU_OT_NONE);
1262eda14cbcSMatt Macy dnode_destroy(dnh->dnh_dnode);
1263eda14cbcSMatt Macy dnh->dnh_dnode = DN_SLOT_FREE;
1264a2b560ccSMartin Matuska reclaimed++;
1265eda14cbcSMatt Macy }
1266eda14cbcSMatt Macy }
1267a2b560ccSMartin Matuska
1268a2b560ccSMartin Matuska return (reclaimed);
1269eda14cbcSMatt Macy }
1270eda14cbcSMatt Macy
1271eda14cbcSMatt Macy void
dnode_free_interior_slots(dnode_t * dn)1272eda14cbcSMatt Macy dnode_free_interior_slots(dnode_t *dn)
1273eda14cbcSMatt Macy {
1274eda14cbcSMatt Macy dnode_children_t *children = dmu_buf_get_user(&dn->dn_dbuf->db);
1275eda14cbcSMatt Macy int epb = dn->dn_dbuf->db.db_size >> DNODE_SHIFT;
1276eda14cbcSMatt Macy int idx = (dn->dn_object & (epb - 1)) + 1;
1277eda14cbcSMatt Macy int slots = dn->dn_num_slots - 1;
1278eda14cbcSMatt Macy
1279eda14cbcSMatt Macy if (slots == 0)
1280eda14cbcSMatt Macy return;
1281eda14cbcSMatt Macy
1282eda14cbcSMatt Macy ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1283eda14cbcSMatt Macy
1284eda14cbcSMatt Macy while (!dnode_slots_tryenter(children, idx, slots)) {
1285eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_free_interior_lock_retry);
1286c7046f76SMartin Matuska kpreempt(KPREEMPT_SYNC);
1287eda14cbcSMatt Macy }
1288eda14cbcSMatt Macy
1289eda14cbcSMatt Macy dnode_set_slots(children, idx, slots, DN_SLOT_FREE);
1290eda14cbcSMatt Macy dnode_slots_rele(children, idx, slots);
1291eda14cbcSMatt Macy }
1292eda14cbcSMatt Macy
1293eda14cbcSMatt Macy void
dnode_special_close(dnode_handle_t * dnh)1294eda14cbcSMatt Macy dnode_special_close(dnode_handle_t *dnh)
1295eda14cbcSMatt Macy {
1296eda14cbcSMatt Macy dnode_t *dn = dnh->dnh_dnode;
1297eda14cbcSMatt Macy
1298eda14cbcSMatt Macy /*
1299eda14cbcSMatt Macy * Ensure dnode_rele_and_unlock() has released dn_mtx, after final
1300eda14cbcSMatt Macy * zfs_refcount_remove()
1301eda14cbcSMatt Macy */
1302eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1303eda14cbcSMatt Macy if (zfs_refcount_count(&dn->dn_holds) > 0)
1304eda14cbcSMatt Macy cv_wait(&dn->dn_nodnholds, &dn->dn_mtx);
1305eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1306eda14cbcSMatt Macy ASSERT3U(zfs_refcount_count(&dn->dn_holds), ==, 0);
1307eda14cbcSMatt Macy
1308eda14cbcSMatt Macy ASSERT(dn->dn_dbuf == NULL ||
1309eda14cbcSMatt Macy dmu_buf_get_user(&dn->dn_dbuf->db) == NULL);
1310eda14cbcSMatt Macy zrl_add(&dnh->dnh_zrlock);
1311eda14cbcSMatt Macy dnode_destroy(dn); /* implicit zrl_remove() */
1312eda14cbcSMatt Macy zrl_destroy(&dnh->dnh_zrlock);
1313eda14cbcSMatt Macy dnh->dnh_dnode = NULL;
1314eda14cbcSMatt Macy }
1315eda14cbcSMatt Macy
1316eda14cbcSMatt Macy void
dnode_special_open(objset_t * os,dnode_phys_t * dnp,uint64_t object,dnode_handle_t * dnh)1317eda14cbcSMatt Macy dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object,
1318eda14cbcSMatt Macy dnode_handle_t *dnh)
1319eda14cbcSMatt Macy {
1320eda14cbcSMatt Macy dnode_t *dn;
1321eda14cbcSMatt Macy
1322eda14cbcSMatt Macy zrl_init(&dnh->dnh_zrlock);
13232c48331dSMatt Macy VERIFY3U(1, ==, zrl_tryenter(&dnh->dnh_zrlock));
1324eda14cbcSMatt Macy
1325eda14cbcSMatt Macy dn = dnode_create(os, dnp, NULL, object, dnh);
1326eda14cbcSMatt Macy DNODE_VERIFY(dn);
1327eda14cbcSMatt Macy
1328eda14cbcSMatt Macy zrl_exit(&dnh->dnh_zrlock);
1329eda14cbcSMatt Macy }
1330eda14cbcSMatt Macy
1331eda14cbcSMatt Macy static void
dnode_buf_evict_async(void * dbu)1332eda14cbcSMatt Macy dnode_buf_evict_async(void *dbu)
1333eda14cbcSMatt Macy {
1334eda14cbcSMatt Macy dnode_children_t *dnc = dbu;
1335eda14cbcSMatt Macy
1336eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_buf_evict);
1337eda14cbcSMatt Macy
1338eda14cbcSMatt Macy for (int i = 0; i < dnc->dnc_count; i++) {
1339eda14cbcSMatt Macy dnode_handle_t *dnh = &dnc->dnc_children[i];
1340eda14cbcSMatt Macy dnode_t *dn;
1341eda14cbcSMatt Macy
1342eda14cbcSMatt Macy /*
1343eda14cbcSMatt Macy * The dnode handle lock guards against the dnode moving to
1344eda14cbcSMatt Macy * another valid address, so there is no need here to guard
1345eda14cbcSMatt Macy * against changes to or from NULL.
1346eda14cbcSMatt Macy */
1347eda14cbcSMatt Macy if (!DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1348eda14cbcSMatt Macy zrl_destroy(&dnh->dnh_zrlock);
1349eda14cbcSMatt Macy dnh->dnh_dnode = DN_SLOT_UNINIT;
1350eda14cbcSMatt Macy continue;
1351eda14cbcSMatt Macy }
1352eda14cbcSMatt Macy
1353eda14cbcSMatt Macy zrl_add(&dnh->dnh_zrlock);
1354eda14cbcSMatt Macy dn = dnh->dnh_dnode;
1355eda14cbcSMatt Macy /*
1356eda14cbcSMatt Macy * If there are holds on this dnode, then there should
1357eda14cbcSMatt Macy * be holds on the dnode's containing dbuf as well; thus
1358eda14cbcSMatt Macy * it wouldn't be eligible for eviction and this function
1359eda14cbcSMatt Macy * would not have been called.
1360eda14cbcSMatt Macy */
1361eda14cbcSMatt Macy ASSERT(zfs_refcount_is_zero(&dn->dn_holds));
1362eda14cbcSMatt Macy ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds));
1363eda14cbcSMatt Macy
1364eda14cbcSMatt Macy dnode_destroy(dn); /* implicit zrl_remove() for first slot */
1365eda14cbcSMatt Macy zrl_destroy(&dnh->dnh_zrlock);
1366eda14cbcSMatt Macy dnh->dnh_dnode = DN_SLOT_UNINIT;
1367eda14cbcSMatt Macy }
1368eda14cbcSMatt Macy kmem_free(dnc, sizeof (dnode_children_t) +
1369eda14cbcSMatt Macy dnc->dnc_count * sizeof (dnode_handle_t));
1370eda14cbcSMatt Macy }
1371eda14cbcSMatt Macy
1372eda14cbcSMatt Macy /*
1373eda14cbcSMatt Macy * When the DNODE_MUST_BE_FREE flag is set, the "slots" parameter is used
1374eda14cbcSMatt Macy * to ensure the hole at the specified object offset is large enough to
1375eda14cbcSMatt Macy * hold the dnode being created. The slots parameter is also used to ensure
1376eda14cbcSMatt Macy * a dnode does not span multiple dnode blocks. In both of these cases, if
1377eda14cbcSMatt Macy * a failure occurs, ENOSPC is returned. Keep in mind, these failure cases
1378eda14cbcSMatt Macy * are only possible when using DNODE_MUST_BE_FREE.
1379eda14cbcSMatt Macy *
1380eda14cbcSMatt Macy * If the DNODE_MUST_BE_ALLOCATED flag is set, "slots" must be 0.
1381eda14cbcSMatt Macy * dnode_hold_impl() will check if the requested dnode is already consumed
1382eda14cbcSMatt Macy * as an extra dnode slot by an large dnode, in which case it returns
1383eda14cbcSMatt Macy * ENOENT.
1384eda14cbcSMatt Macy *
1385eda14cbcSMatt Macy * If the DNODE_DRY_RUN flag is set, we don't actually hold the dnode, just
1386eda14cbcSMatt Macy * return whether the hold would succeed or not. tag and dnp should set to
1387eda14cbcSMatt Macy * NULL in this case.
1388eda14cbcSMatt Macy *
1389eda14cbcSMatt Macy * errors:
1390eda14cbcSMatt Macy * EINVAL - Invalid object number or flags.
1391eda14cbcSMatt Macy * ENOSPC - Hole too small to fulfill "slots" request (DNODE_MUST_BE_FREE)
1392eda14cbcSMatt Macy * EEXIST - Refers to an allocated dnode (DNODE_MUST_BE_FREE)
1393eda14cbcSMatt Macy * - Refers to a freeing dnode (DNODE_MUST_BE_FREE)
1394eda14cbcSMatt Macy * - Refers to an interior dnode slot (DNODE_MUST_BE_ALLOCATED)
1395eda14cbcSMatt Macy * ENOENT - The requested dnode is not allocated (DNODE_MUST_BE_ALLOCATED)
1396eda14cbcSMatt Macy * - The requested dnode is being freed (DNODE_MUST_BE_ALLOCATED)
1397eda14cbcSMatt Macy * EIO - I/O error when reading the meta dnode dbuf.
1398eda14cbcSMatt Macy *
1399eda14cbcSMatt Macy * succeeds even for free dnodes.
1400eda14cbcSMatt Macy */
1401eda14cbcSMatt Macy int
dnode_hold_impl(objset_t * os,uint64_t object,int flag,int slots,const void * tag,dnode_t ** dnp)1402eda14cbcSMatt Macy dnode_hold_impl(objset_t *os, uint64_t object, int flag, int slots,
1403a0b956f5SMartin Matuska const void *tag, dnode_t **dnp)
1404eda14cbcSMatt Macy {
1405eda14cbcSMatt Macy int epb, idx, err;
1406eda14cbcSMatt Macy int drop_struct_lock = FALSE;
1407eda14cbcSMatt Macy int type;
1408eda14cbcSMatt Macy uint64_t blk;
1409eda14cbcSMatt Macy dnode_t *mdn, *dn;
1410eda14cbcSMatt Macy dmu_buf_impl_t *db;
1411eda14cbcSMatt Macy dnode_children_t *dnc;
1412eda14cbcSMatt Macy dnode_phys_t *dn_block;
1413eda14cbcSMatt Macy dnode_handle_t *dnh;
1414eda14cbcSMatt Macy
1415eda14cbcSMatt Macy ASSERT(!(flag & DNODE_MUST_BE_ALLOCATED) || (slots == 0));
1416eda14cbcSMatt Macy ASSERT(!(flag & DNODE_MUST_BE_FREE) || (slots > 0));
1417eda14cbcSMatt Macy IMPLY(flag & DNODE_DRY_RUN, (tag == NULL) && (dnp == NULL));
1418eda14cbcSMatt Macy
1419eda14cbcSMatt Macy /*
1420eda14cbcSMatt Macy * If you are holding the spa config lock as writer, you shouldn't
1421eda14cbcSMatt Macy * be asking the DMU to do *anything* unless it's the root pool
1422eda14cbcSMatt Macy * which may require us to read from the root filesystem while
1423eda14cbcSMatt Macy * holding some (not all) of the locks as writer.
1424eda14cbcSMatt Macy */
1425eda14cbcSMatt Macy ASSERT(spa_config_held(os->os_spa, SCL_ALL, RW_WRITER) == 0 ||
1426eda14cbcSMatt Macy (spa_is_root(os->os_spa) &&
1427eda14cbcSMatt Macy spa_config_held(os->os_spa, SCL_STATE, RW_WRITER)));
1428eda14cbcSMatt Macy
1429eda14cbcSMatt Macy ASSERT((flag & DNODE_MUST_BE_ALLOCATED) || (flag & DNODE_MUST_BE_FREE));
1430eda14cbcSMatt Macy
1431eda14cbcSMatt Macy if (object == DMU_USERUSED_OBJECT || object == DMU_GROUPUSED_OBJECT ||
1432eda14cbcSMatt Macy object == DMU_PROJECTUSED_OBJECT) {
1433eda14cbcSMatt Macy if (object == DMU_USERUSED_OBJECT)
1434eda14cbcSMatt Macy dn = DMU_USERUSED_DNODE(os);
1435eda14cbcSMatt Macy else if (object == DMU_GROUPUSED_OBJECT)
1436eda14cbcSMatt Macy dn = DMU_GROUPUSED_DNODE(os);
1437eda14cbcSMatt Macy else
1438eda14cbcSMatt Macy dn = DMU_PROJECTUSED_DNODE(os);
1439eda14cbcSMatt Macy if (dn == NULL)
1440eda14cbcSMatt Macy return (SET_ERROR(ENOENT));
1441eda14cbcSMatt Macy type = dn->dn_type;
1442eda14cbcSMatt Macy if ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE)
1443eda14cbcSMatt Macy return (SET_ERROR(ENOENT));
1444eda14cbcSMatt Macy if ((flag & DNODE_MUST_BE_FREE) && type != DMU_OT_NONE)
1445eda14cbcSMatt Macy return (SET_ERROR(EEXIST));
1446eda14cbcSMatt Macy DNODE_VERIFY(dn);
1447eda14cbcSMatt Macy /* Don't actually hold if dry run, just return 0 */
1448eda14cbcSMatt Macy if (!(flag & DNODE_DRY_RUN)) {
1449eda14cbcSMatt Macy (void) zfs_refcount_add(&dn->dn_holds, tag);
1450eda14cbcSMatt Macy *dnp = dn;
1451eda14cbcSMatt Macy }
1452eda14cbcSMatt Macy return (0);
1453eda14cbcSMatt Macy }
1454eda14cbcSMatt Macy
1455eda14cbcSMatt Macy if (object == 0 || object >= DN_MAX_OBJECT)
1456eda14cbcSMatt Macy return (SET_ERROR(EINVAL));
1457eda14cbcSMatt Macy
1458eda14cbcSMatt Macy mdn = DMU_META_DNODE(os);
1459eda14cbcSMatt Macy ASSERT(mdn->dn_object == DMU_META_DNODE_OBJECT);
1460eda14cbcSMatt Macy
1461eda14cbcSMatt Macy DNODE_VERIFY(mdn);
1462eda14cbcSMatt Macy
1463eda14cbcSMatt Macy if (!RW_WRITE_HELD(&mdn->dn_struct_rwlock)) {
1464eda14cbcSMatt Macy rw_enter(&mdn->dn_struct_rwlock, RW_READER);
1465eda14cbcSMatt Macy drop_struct_lock = TRUE;
1466eda14cbcSMatt Macy }
1467eda14cbcSMatt Macy
1468eda14cbcSMatt Macy blk = dbuf_whichblock(mdn, 0, object * sizeof (dnode_phys_t));
1469eda14cbcSMatt Macy db = dbuf_hold(mdn, blk, FTAG);
1470eda14cbcSMatt Macy if (drop_struct_lock)
1471eda14cbcSMatt Macy rw_exit(&mdn->dn_struct_rwlock);
1472eda14cbcSMatt Macy if (db == NULL) {
1473eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_dbuf_hold);
1474eda14cbcSMatt Macy return (SET_ERROR(EIO));
1475eda14cbcSMatt Macy }
1476eda14cbcSMatt Macy
1477eda14cbcSMatt Macy /*
1478eda14cbcSMatt Macy * We do not need to decrypt to read the dnode so it doesn't matter
1479eda14cbcSMatt Macy * if we get the encrypted or decrypted version.
1480eda14cbcSMatt Macy */
1481c40487d4SMatt Macy err = dbuf_read(db, NULL, DB_RF_CANFAIL |
1482c40487d4SMatt Macy DB_RF_NO_DECRYPT | DB_RF_NOPREFETCH);
1483eda14cbcSMatt Macy if (err) {
1484eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_dbuf_read);
1485eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1486eda14cbcSMatt Macy return (err);
1487eda14cbcSMatt Macy }
1488eda14cbcSMatt Macy
1489eda14cbcSMatt Macy ASSERT3U(db->db.db_size, >=, 1<<DNODE_SHIFT);
1490eda14cbcSMatt Macy epb = db->db.db_size >> DNODE_SHIFT;
1491eda14cbcSMatt Macy
1492eda14cbcSMatt Macy idx = object & (epb - 1);
1493eda14cbcSMatt Macy dn_block = (dnode_phys_t *)db->db.db_data;
1494eda14cbcSMatt Macy
1495eda14cbcSMatt Macy ASSERT(DB_DNODE(db)->dn_type == DMU_OT_DNODE);
1496eda14cbcSMatt Macy dnc = dmu_buf_get_user(&db->db);
1497eda14cbcSMatt Macy dnh = NULL;
1498eda14cbcSMatt Macy if (dnc == NULL) {
1499eda14cbcSMatt Macy dnode_children_t *winner;
1500eda14cbcSMatt Macy int skip = 0;
1501eda14cbcSMatt Macy
1502eda14cbcSMatt Macy dnc = kmem_zalloc(sizeof (dnode_children_t) +
1503eda14cbcSMatt Macy epb * sizeof (dnode_handle_t), KM_SLEEP);
1504eda14cbcSMatt Macy dnc->dnc_count = epb;
1505eda14cbcSMatt Macy dnh = &dnc->dnc_children[0];
1506eda14cbcSMatt Macy
1507eda14cbcSMatt Macy /* Initialize dnode slot status from dnode_phys_t */
1508eda14cbcSMatt Macy for (int i = 0; i < epb; i++) {
1509eda14cbcSMatt Macy zrl_init(&dnh[i].dnh_zrlock);
1510eda14cbcSMatt Macy
1511eda14cbcSMatt Macy if (skip) {
1512eda14cbcSMatt Macy skip--;
1513eda14cbcSMatt Macy continue;
1514eda14cbcSMatt Macy }
1515eda14cbcSMatt Macy
1516eda14cbcSMatt Macy if (dn_block[i].dn_type != DMU_OT_NONE) {
1517eda14cbcSMatt Macy int interior = dn_block[i].dn_extra_slots;
1518eda14cbcSMatt Macy
1519eda14cbcSMatt Macy dnode_set_slots(dnc, i, 1, DN_SLOT_ALLOCATED);
1520eda14cbcSMatt Macy dnode_set_slots(dnc, i + 1, interior,
1521eda14cbcSMatt Macy DN_SLOT_INTERIOR);
1522eda14cbcSMatt Macy skip = interior;
1523eda14cbcSMatt Macy } else {
1524eda14cbcSMatt Macy dnh[i].dnh_dnode = DN_SLOT_FREE;
1525eda14cbcSMatt Macy skip = 0;
1526eda14cbcSMatt Macy }
1527eda14cbcSMatt Macy }
1528eda14cbcSMatt Macy
1529eda14cbcSMatt Macy dmu_buf_init_user(&dnc->dnc_dbu, NULL,
1530eda14cbcSMatt Macy dnode_buf_evict_async, NULL);
1531eda14cbcSMatt Macy winner = dmu_buf_set_user(&db->db, &dnc->dnc_dbu);
1532eda14cbcSMatt Macy if (winner != NULL) {
1533eda14cbcSMatt Macy
1534eda14cbcSMatt Macy for (int i = 0; i < epb; i++)
1535eda14cbcSMatt Macy zrl_destroy(&dnh[i].dnh_zrlock);
1536eda14cbcSMatt Macy
1537eda14cbcSMatt Macy kmem_free(dnc, sizeof (dnode_children_t) +
1538eda14cbcSMatt Macy epb * sizeof (dnode_handle_t));
1539eda14cbcSMatt Macy dnc = winner;
1540eda14cbcSMatt Macy }
1541eda14cbcSMatt Macy }
1542eda14cbcSMatt Macy
1543eda14cbcSMatt Macy ASSERT(dnc->dnc_count == epb);
1544eda14cbcSMatt Macy
1545eda14cbcSMatt Macy if (flag & DNODE_MUST_BE_ALLOCATED) {
1546eda14cbcSMatt Macy slots = 1;
1547eda14cbcSMatt Macy
1548eda14cbcSMatt Macy dnode_slots_hold(dnc, idx, slots);
1549eda14cbcSMatt Macy dnh = &dnc->dnc_children[idx];
1550eda14cbcSMatt Macy
1551eda14cbcSMatt Macy if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1552eda14cbcSMatt Macy dn = dnh->dnh_dnode;
1553eda14cbcSMatt Macy } else if (dnh->dnh_dnode == DN_SLOT_INTERIOR) {
1554eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_alloc_interior);
1555eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1556eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1557eda14cbcSMatt Macy return (SET_ERROR(EEXIST));
1558eda14cbcSMatt Macy } else if (dnh->dnh_dnode != DN_SLOT_ALLOCATED) {
1559eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_alloc_misses);
1560eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1561eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1562eda14cbcSMatt Macy return (SET_ERROR(ENOENT));
1563eda14cbcSMatt Macy } else {
1564eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1565eda14cbcSMatt Macy while (!dnode_slots_tryenter(dnc, idx, slots)) {
1566eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_alloc_lock_retry);
1567c7046f76SMartin Matuska kpreempt(KPREEMPT_SYNC);
1568eda14cbcSMatt Macy }
1569eda14cbcSMatt Macy
1570eda14cbcSMatt Macy /*
1571eda14cbcSMatt Macy * Someone else won the race and called dnode_create()
1572eda14cbcSMatt Macy * after we checked DN_SLOT_IS_PTR() above but before
1573eda14cbcSMatt Macy * we acquired the lock.
1574eda14cbcSMatt Macy */
1575eda14cbcSMatt Macy if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1576eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_alloc_lock_misses);
1577eda14cbcSMatt Macy dn = dnh->dnh_dnode;
1578eda14cbcSMatt Macy } else {
1579eda14cbcSMatt Macy dn = dnode_create(os, dn_block + idx, db,
1580eda14cbcSMatt Macy object, dnh);
1581a2b560ccSMartin Matuska dmu_buf_add_user_size(&db->db,
1582a2b560ccSMartin Matuska sizeof (dnode_t));
1583eda14cbcSMatt Macy }
1584eda14cbcSMatt Macy }
1585eda14cbcSMatt Macy
1586eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1587eda14cbcSMatt Macy if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg != 0) {
1588eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_alloc_type_none);
1589eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1590eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1591eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1592eda14cbcSMatt Macy return (SET_ERROR(ENOENT));
1593eda14cbcSMatt Macy }
1594eda14cbcSMatt Macy
1595eda14cbcSMatt Macy /* Don't actually hold if dry run, just return 0 */
1596eda14cbcSMatt Macy if (flag & DNODE_DRY_RUN) {
1597eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1598eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1599eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1600eda14cbcSMatt Macy return (0);
1601eda14cbcSMatt Macy }
1602eda14cbcSMatt Macy
1603eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_alloc_hits);
1604eda14cbcSMatt Macy } else if (flag & DNODE_MUST_BE_FREE) {
1605eda14cbcSMatt Macy
1606eda14cbcSMatt Macy if (idx + slots - 1 >= DNODES_PER_BLOCK) {
1607eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_free_overflow);
1608eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1609eda14cbcSMatt Macy return (SET_ERROR(ENOSPC));
1610eda14cbcSMatt Macy }
1611eda14cbcSMatt Macy
1612eda14cbcSMatt Macy dnode_slots_hold(dnc, idx, slots);
1613eda14cbcSMatt Macy
1614eda14cbcSMatt Macy if (!dnode_check_slots_free(dnc, idx, slots)) {
1615eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_free_misses);
1616eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1617eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1618eda14cbcSMatt Macy return (SET_ERROR(ENOSPC));
1619eda14cbcSMatt Macy }
1620eda14cbcSMatt Macy
1621eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1622eda14cbcSMatt Macy while (!dnode_slots_tryenter(dnc, idx, slots)) {
1623eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_free_lock_retry);
1624c7046f76SMartin Matuska kpreempt(KPREEMPT_SYNC);
1625eda14cbcSMatt Macy }
1626eda14cbcSMatt Macy
1627eda14cbcSMatt Macy if (!dnode_check_slots_free(dnc, idx, slots)) {
1628eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_free_lock_misses);
1629eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1630eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1631eda14cbcSMatt Macy return (SET_ERROR(ENOSPC));
1632eda14cbcSMatt Macy }
1633eda14cbcSMatt Macy
1634eda14cbcSMatt Macy /*
1635eda14cbcSMatt Macy * Allocated but otherwise free dnodes which would
1636eda14cbcSMatt Macy * be in the interior of a multi-slot dnodes need
1637eda14cbcSMatt Macy * to be freed. Single slot dnodes can be safely
1638eda14cbcSMatt Macy * re-purposed as a performance optimization.
1639eda14cbcSMatt Macy */
1640a2b560ccSMartin Matuska if (slots > 1) {
1641a2b560ccSMartin Matuska uint_t reclaimed =
1642eda14cbcSMatt Macy dnode_reclaim_slots(dnc, idx + 1, slots - 1);
1643a2b560ccSMartin Matuska if (reclaimed > 0)
1644a2b560ccSMartin Matuska dmu_buf_sub_user_size(&db->db,
1645a2b560ccSMartin Matuska reclaimed * sizeof (dnode_t));
1646a2b560ccSMartin Matuska }
1647eda14cbcSMatt Macy
1648eda14cbcSMatt Macy dnh = &dnc->dnc_children[idx];
1649eda14cbcSMatt Macy if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1650eda14cbcSMatt Macy dn = dnh->dnh_dnode;
1651eda14cbcSMatt Macy } else {
1652eda14cbcSMatt Macy dn = dnode_create(os, dn_block + idx, db,
1653eda14cbcSMatt Macy object, dnh);
1654a2b560ccSMartin Matuska dmu_buf_add_user_size(&db->db, sizeof (dnode_t));
1655eda14cbcSMatt Macy }
1656eda14cbcSMatt Macy
1657eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1658eda14cbcSMatt Macy if (!zfs_refcount_is_zero(&dn->dn_holds) || dn->dn_free_txg) {
1659eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_free_refcount);
1660eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1661eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1662eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1663eda14cbcSMatt Macy return (SET_ERROR(EEXIST));
1664eda14cbcSMatt Macy }
1665eda14cbcSMatt Macy
1666eda14cbcSMatt Macy /* Don't actually hold if dry run, just return 0 */
1667eda14cbcSMatt Macy if (flag & DNODE_DRY_RUN) {
1668eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1669eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1670eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1671eda14cbcSMatt Macy return (0);
1672eda14cbcSMatt Macy }
1673eda14cbcSMatt Macy
1674eda14cbcSMatt Macy dnode_set_slots(dnc, idx + 1, slots - 1, DN_SLOT_INTERIOR);
1675eda14cbcSMatt Macy DNODE_STAT_BUMP(dnode_hold_free_hits);
1676eda14cbcSMatt Macy } else {
1677eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1678eda14cbcSMatt Macy return (SET_ERROR(EINVAL));
1679eda14cbcSMatt Macy }
1680eda14cbcSMatt Macy
1681eda14cbcSMatt Macy ASSERT0(dn->dn_free_txg);
1682eda14cbcSMatt Macy
1683eda14cbcSMatt Macy if (zfs_refcount_add(&dn->dn_holds, tag) == 1)
1684eda14cbcSMatt Macy dbuf_add_ref(db, dnh);
1685eda14cbcSMatt Macy
1686eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1687eda14cbcSMatt Macy
1688eda14cbcSMatt Macy /* Now we can rely on the hold to prevent the dnode from moving. */
1689eda14cbcSMatt Macy dnode_slots_rele(dnc, idx, slots);
1690eda14cbcSMatt Macy
1691eda14cbcSMatt Macy DNODE_VERIFY(dn);
1692eda14cbcSMatt Macy ASSERT3P(dnp, !=, NULL);
1693eda14cbcSMatt Macy ASSERT3P(dn->dn_dbuf, ==, db);
1694eda14cbcSMatt Macy ASSERT3U(dn->dn_object, ==, object);
1695eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1696eda14cbcSMatt Macy
1697eda14cbcSMatt Macy *dnp = dn;
1698eda14cbcSMatt Macy return (0);
1699eda14cbcSMatt Macy }
1700eda14cbcSMatt Macy
1701eda14cbcSMatt Macy /*
1702eda14cbcSMatt Macy * Return held dnode if the object is allocated, NULL if not.
1703eda14cbcSMatt Macy */
1704eda14cbcSMatt Macy int
dnode_hold(objset_t * os,uint64_t object,const void * tag,dnode_t ** dnp)1705a0b956f5SMartin Matuska dnode_hold(objset_t *os, uint64_t object, const void *tag, dnode_t **dnp)
1706eda14cbcSMatt Macy {
1707eda14cbcSMatt Macy return (dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, 0, tag,
1708eda14cbcSMatt Macy dnp));
1709eda14cbcSMatt Macy }
1710eda14cbcSMatt Macy
1711eda14cbcSMatt Macy /*
1712eda14cbcSMatt Macy * Can only add a reference if there is already at least one
1713eda14cbcSMatt Macy * reference on the dnode. Returns FALSE if unable to add a
1714eda14cbcSMatt Macy * new reference.
1715eda14cbcSMatt Macy */
1716eda14cbcSMatt Macy boolean_t
dnode_add_ref(dnode_t * dn,const void * tag)1717a0b956f5SMartin Matuska dnode_add_ref(dnode_t *dn, const void *tag)
1718eda14cbcSMatt Macy {
1719eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1720eda14cbcSMatt Macy if (zfs_refcount_is_zero(&dn->dn_holds)) {
1721eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1722eda14cbcSMatt Macy return (FALSE);
1723eda14cbcSMatt Macy }
1724eda14cbcSMatt Macy VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag));
1725eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1726eda14cbcSMatt Macy return (TRUE);
1727eda14cbcSMatt Macy }
1728eda14cbcSMatt Macy
1729eda14cbcSMatt Macy void
dnode_rele(dnode_t * dn,const void * tag)1730a0b956f5SMartin Matuska dnode_rele(dnode_t *dn, const void *tag)
1731eda14cbcSMatt Macy {
1732eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1733eda14cbcSMatt Macy dnode_rele_and_unlock(dn, tag, B_FALSE);
1734eda14cbcSMatt Macy }
1735eda14cbcSMatt Macy
1736eda14cbcSMatt Macy void
dnode_rele_and_unlock(dnode_t * dn,const void * tag,boolean_t evicting)1737a0b956f5SMartin Matuska dnode_rele_and_unlock(dnode_t *dn, const void *tag, boolean_t evicting)
1738eda14cbcSMatt Macy {
1739eda14cbcSMatt Macy uint64_t refs;
1740eda14cbcSMatt Macy /* Get while the hold prevents the dnode from moving. */
1741eda14cbcSMatt Macy dmu_buf_impl_t *db = dn->dn_dbuf;
1742eda14cbcSMatt Macy dnode_handle_t *dnh = dn->dn_handle;
1743eda14cbcSMatt Macy
1744eda14cbcSMatt Macy refs = zfs_refcount_remove(&dn->dn_holds, tag);
1745eda14cbcSMatt Macy if (refs == 0)
1746eda14cbcSMatt Macy cv_broadcast(&dn->dn_nodnholds);
1747eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1748eda14cbcSMatt Macy /* dnode could get destroyed at this point, so don't use it anymore */
1749eda14cbcSMatt Macy
1750eda14cbcSMatt Macy /*
1751eda14cbcSMatt Macy * It's unsafe to release the last hold on a dnode by dnode_rele() or
1752eda14cbcSMatt Macy * indirectly by dbuf_rele() while relying on the dnode handle to
1753eda14cbcSMatt Macy * prevent the dnode from moving, since releasing the last hold could
1754eda14cbcSMatt Macy * result in the dnode's parent dbuf evicting its dnode handles. For
1755eda14cbcSMatt Macy * that reason anyone calling dnode_rele() or dbuf_rele() without some
1756eda14cbcSMatt Macy * other direct or indirect hold on the dnode must first drop the dnode
1757eda14cbcSMatt Macy * handle.
1758eda14cbcSMatt Macy */
1759e92ffd9bSMartin Matuska #ifdef ZFS_DEBUG
1760eda14cbcSMatt Macy ASSERT(refs > 0 || dnh->dnh_zrlock.zr_owner != curthread);
1761e92ffd9bSMartin Matuska #endif
1762eda14cbcSMatt Macy
1763eda14cbcSMatt Macy /* NOTE: the DNODE_DNODE does not have a dn_dbuf */
1764eda14cbcSMatt Macy if (refs == 0 && db != NULL) {
1765eda14cbcSMatt Macy /*
1766eda14cbcSMatt Macy * Another thread could add a hold to the dnode handle in
1767eda14cbcSMatt Macy * dnode_hold_impl() while holding the parent dbuf. Since the
1768eda14cbcSMatt Macy * hold on the parent dbuf prevents the handle from being
1769eda14cbcSMatt Macy * destroyed, the hold on the handle is OK. We can't yet assert
1770eda14cbcSMatt Macy * that the handle has zero references, but that will be
1771eda14cbcSMatt Macy * asserted anyway when the handle gets destroyed.
1772eda14cbcSMatt Macy */
1773eda14cbcSMatt Macy mutex_enter(&db->db_mtx);
1774eda14cbcSMatt Macy dbuf_rele_and_unlock(db, dnh, evicting);
1775eda14cbcSMatt Macy }
1776eda14cbcSMatt Macy }
1777eda14cbcSMatt Macy
1778eda14cbcSMatt Macy /*
1779eda14cbcSMatt Macy * Test whether we can create a dnode at the specified location.
1780eda14cbcSMatt Macy */
1781eda14cbcSMatt Macy int
dnode_try_claim(objset_t * os,uint64_t object,int slots)1782eda14cbcSMatt Macy dnode_try_claim(objset_t *os, uint64_t object, int slots)
1783eda14cbcSMatt Macy {
1784eda14cbcSMatt Macy return (dnode_hold_impl(os, object, DNODE_MUST_BE_FREE | DNODE_DRY_RUN,
1785eda14cbcSMatt Macy slots, NULL, NULL));
1786eda14cbcSMatt Macy }
1787eda14cbcSMatt Macy
178881b22a98SMartin Matuska /*
17892276e539SMartin Matuska * Checks if the dnode itself is dirty, or is carrying any uncommitted records.
17902276e539SMartin Matuska * It is important to check both conditions, as some operations (eg appending
17912276e539SMartin Matuska * to a file) can dirty both as a single logical unit, but they are not synced
17922276e539SMartin Matuska * out atomically, so checking one and not the other can result in an object
17932276e539SMartin Matuska * appearing to be clean mid-way through a commit.
17942276e539SMartin Matuska *
17952276e539SMartin Matuska * Do not change this lightly! If you get it wrong, dmu_offset_next() can
17962276e539SMartin Matuska * detect a hole where there is really data, leading to silent corruption.
179781b22a98SMartin Matuska */
179881b22a98SMartin Matuska boolean_t
dnode_is_dirty(dnode_t * dn)179981b22a98SMartin Matuska dnode_is_dirty(dnode_t *dn)
180081b22a98SMartin Matuska {
180181b22a98SMartin Matuska mutex_enter(&dn->dn_mtx);
180263ee747fSMateusz Guzik
180381b22a98SMartin Matuska for (int i = 0; i < TXG_SIZE; i++) {
18042276e539SMartin Matuska if (multilist_link_active(&dn->dn_dirty_link[i]) ||
18052276e539SMartin Matuska !list_is_empty(&dn->dn_dirty_records[i])) {
180681b22a98SMartin Matuska mutex_exit(&dn->dn_mtx);
180781b22a98SMartin Matuska return (B_TRUE);
180881b22a98SMartin Matuska }
180981b22a98SMartin Matuska }
181063ee747fSMateusz Guzik
181181b22a98SMartin Matuska mutex_exit(&dn->dn_mtx);
181281b22a98SMartin Matuska
181381b22a98SMartin Matuska return (B_FALSE);
181481b22a98SMartin Matuska }
181581b22a98SMartin Matuska
1816eda14cbcSMatt Macy void
dnode_setdirty(dnode_t * dn,dmu_tx_t * tx)1817eda14cbcSMatt Macy dnode_setdirty(dnode_t *dn, dmu_tx_t *tx)
1818eda14cbcSMatt Macy {
1819eda14cbcSMatt Macy objset_t *os = dn->dn_objset;
1820eda14cbcSMatt Macy uint64_t txg = tx->tx_txg;
1821eda14cbcSMatt Macy
1822eda14cbcSMatt Macy if (DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
1823eda14cbcSMatt Macy dsl_dataset_dirty(os->os_dsl_dataset, tx);
1824eda14cbcSMatt Macy return;
1825eda14cbcSMatt Macy }
1826eda14cbcSMatt Macy
1827eda14cbcSMatt Macy DNODE_VERIFY(dn);
1828eda14cbcSMatt Macy
1829eda14cbcSMatt Macy #ifdef ZFS_DEBUG
1830eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1831eda14cbcSMatt Macy ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg);
1832eda14cbcSMatt Macy ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg);
1833eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1834eda14cbcSMatt Macy #endif
1835eda14cbcSMatt Macy
1836eda14cbcSMatt Macy /*
1837eda14cbcSMatt Macy * Determine old uid/gid when necessary
1838eda14cbcSMatt Macy */
1839eda14cbcSMatt Macy dmu_objset_userquota_get_ids(dn, B_TRUE, tx);
1840eda14cbcSMatt Macy
18413ff01b23SMartin Matuska multilist_t *dirtylist = &os->os_dirty_dnodes[txg & TXG_MASK];
1842eda14cbcSMatt Macy multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn);
1843eda14cbcSMatt Macy
1844eda14cbcSMatt Macy /*
1845eda14cbcSMatt Macy * If we are already marked dirty, we're done.
1846eda14cbcSMatt Macy */
1847eda14cbcSMatt Macy if (multilist_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) {
1848eda14cbcSMatt Macy multilist_sublist_unlock(mls);
1849eda14cbcSMatt Macy return;
1850eda14cbcSMatt Macy }
1851eda14cbcSMatt Macy
1852eda14cbcSMatt Macy ASSERT(!zfs_refcount_is_zero(&dn->dn_holds) ||
1853eda14cbcSMatt Macy !avl_is_empty(&dn->dn_dbufs));
1854eda14cbcSMatt Macy ASSERT(dn->dn_datablksz != 0);
1855eda14cbcSMatt Macy ASSERT0(dn->dn_next_bonuslen[txg & TXG_MASK]);
1856eda14cbcSMatt Macy ASSERT0(dn->dn_next_blksz[txg & TXG_MASK]);
1857eda14cbcSMatt Macy ASSERT0(dn->dn_next_bonustype[txg & TXG_MASK]);
1858eda14cbcSMatt Macy
1859eda14cbcSMatt Macy dprintf_ds(os->os_dsl_dataset, "obj=%llu txg=%llu\n",
186033b8c039SMartin Matuska (u_longlong_t)dn->dn_object, (u_longlong_t)txg);
1861eda14cbcSMatt Macy
1862eda14cbcSMatt Macy multilist_sublist_insert_head(mls, dn);
1863eda14cbcSMatt Macy
1864eda14cbcSMatt Macy multilist_sublist_unlock(mls);
1865eda14cbcSMatt Macy
1866eda14cbcSMatt Macy /*
1867eda14cbcSMatt Macy * The dnode maintains a hold on its containing dbuf as
1868eda14cbcSMatt Macy * long as there are holds on it. Each instantiated child
1869eda14cbcSMatt Macy * dbuf maintains a hold on the dnode. When the last child
1870eda14cbcSMatt Macy * drops its hold, the dnode will drop its hold on the
1871eda14cbcSMatt Macy * containing dbuf. We add a "dirty hold" here so that the
1872eda14cbcSMatt Macy * dnode will hang around after we finish processing its
1873eda14cbcSMatt Macy * children.
1874eda14cbcSMatt Macy */
1875eda14cbcSMatt Macy VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg));
1876eda14cbcSMatt Macy
1877eda14cbcSMatt Macy (void) dbuf_dirty(dn->dn_dbuf, tx);
1878eda14cbcSMatt Macy
1879eda14cbcSMatt Macy dsl_dataset_dirty(os->os_dsl_dataset, tx);
1880eda14cbcSMatt Macy }
1881eda14cbcSMatt Macy
1882eda14cbcSMatt Macy void
dnode_free(dnode_t * dn,dmu_tx_t * tx)1883eda14cbcSMatt Macy dnode_free(dnode_t *dn, dmu_tx_t *tx)
1884eda14cbcSMatt Macy {
1885eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1886eda14cbcSMatt Macy if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) {
1887eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1888eda14cbcSMatt Macy return;
1889eda14cbcSMatt Macy }
1890eda14cbcSMatt Macy dn->dn_free_txg = tx->tx_txg;
1891eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
1892eda14cbcSMatt Macy
1893eda14cbcSMatt Macy dnode_setdirty(dn, tx);
1894eda14cbcSMatt Macy }
1895eda14cbcSMatt Macy
1896eda14cbcSMatt Macy /*
1897eda14cbcSMatt Macy * Try to change the block size for the indicated dnode. This can only
1898eda14cbcSMatt Macy * succeed if there are no blocks allocated or dirty beyond first block
1899eda14cbcSMatt Macy */
1900eda14cbcSMatt Macy int
dnode_set_blksz(dnode_t * dn,uint64_t size,int ibs,dmu_tx_t * tx)1901eda14cbcSMatt Macy dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx)
1902eda14cbcSMatt Macy {
1903eda14cbcSMatt Macy dmu_buf_impl_t *db;
1904eda14cbcSMatt Macy int err;
1905eda14cbcSMatt Macy
1906eda14cbcSMatt Macy ASSERT3U(size, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
1907eda14cbcSMatt Macy if (size == 0)
1908eda14cbcSMatt Macy size = SPA_MINBLOCKSIZE;
1909eda14cbcSMatt Macy else
1910eda14cbcSMatt Macy size = P2ROUNDUP(size, SPA_MINBLOCKSIZE);
1911eda14cbcSMatt Macy
1912eda14cbcSMatt Macy if (ibs == dn->dn_indblkshift)
1913eda14cbcSMatt Macy ibs = 0;
1914eda14cbcSMatt Macy
1915315ee00fSMartin Matuska if (size == dn->dn_datablksz && ibs == 0)
1916eda14cbcSMatt Macy return (0);
1917eda14cbcSMatt Macy
1918eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
1919eda14cbcSMatt Macy
1920eda14cbcSMatt Macy /* Check for any allocated blocks beyond the first */
1921eda14cbcSMatt Macy if (dn->dn_maxblkid != 0)
1922eda14cbcSMatt Macy goto fail;
1923eda14cbcSMatt Macy
1924eda14cbcSMatt Macy mutex_enter(&dn->dn_dbufs_mtx);
1925eda14cbcSMatt Macy for (db = avl_first(&dn->dn_dbufs); db != NULL;
1926eda14cbcSMatt Macy db = AVL_NEXT(&dn->dn_dbufs, db)) {
1927eda14cbcSMatt Macy if (db->db_blkid != 0 && db->db_blkid != DMU_BONUS_BLKID &&
1928eda14cbcSMatt Macy db->db_blkid != DMU_SPILL_BLKID) {
1929eda14cbcSMatt Macy mutex_exit(&dn->dn_dbufs_mtx);
1930eda14cbcSMatt Macy goto fail;
1931eda14cbcSMatt Macy }
1932eda14cbcSMatt Macy }
1933eda14cbcSMatt Macy mutex_exit(&dn->dn_dbufs_mtx);
1934eda14cbcSMatt Macy
1935eda14cbcSMatt Macy if (ibs && dn->dn_nlevels != 1)
1936eda14cbcSMatt Macy goto fail;
1937eda14cbcSMatt Macy
1938315ee00fSMartin Matuska dnode_setdirty(dn, tx);
1939315ee00fSMartin Matuska if (size != dn->dn_datablksz) {
1940eda14cbcSMatt Macy /* resize the old block */
1941eda14cbcSMatt Macy err = dbuf_hold_impl(dn, 0, 0, TRUE, FALSE, FTAG, &db);
1942eda14cbcSMatt Macy if (err == 0) {
1943eda14cbcSMatt Macy dbuf_new_size(db, size, tx);
1944eda14cbcSMatt Macy } else if (err != ENOENT) {
1945eda14cbcSMatt Macy goto fail;
1946eda14cbcSMatt Macy }
1947eda14cbcSMatt Macy
1948eda14cbcSMatt Macy dnode_setdblksz(dn, size);
1949eda14cbcSMatt Macy dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = size;
1950315ee00fSMartin Matuska if (db)
1951315ee00fSMartin Matuska dbuf_rele(db, FTAG);
1952315ee00fSMartin Matuska }
1953eda14cbcSMatt Macy if (ibs) {
1954eda14cbcSMatt Macy dn->dn_indblkshift = ibs;
1955eda14cbcSMatt Macy dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs;
1956eda14cbcSMatt Macy }
1957eda14cbcSMatt Macy
1958eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
1959eda14cbcSMatt Macy return (0);
1960eda14cbcSMatt Macy
1961eda14cbcSMatt Macy fail:
1962eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
1963eda14cbcSMatt Macy return (SET_ERROR(ENOTSUP));
1964eda14cbcSMatt Macy }
1965eda14cbcSMatt Macy
1966eda14cbcSMatt Macy static void
dnode_set_nlevels_impl(dnode_t * dn,int new_nlevels,dmu_tx_t * tx)1967eda14cbcSMatt Macy dnode_set_nlevels_impl(dnode_t *dn, int new_nlevels, dmu_tx_t *tx)
1968eda14cbcSMatt Macy {
1969eda14cbcSMatt Macy uint64_t txgoff = tx->tx_txg & TXG_MASK;
1970eda14cbcSMatt Macy int old_nlevels = dn->dn_nlevels;
1971eda14cbcSMatt Macy dmu_buf_impl_t *db;
1972eda14cbcSMatt Macy list_t *list;
1973eda14cbcSMatt Macy dbuf_dirty_record_t *new, *dr, *dr_next;
1974eda14cbcSMatt Macy
1975eda14cbcSMatt Macy ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1976eda14cbcSMatt Macy
19777877fdebSMatt Macy ASSERT3U(new_nlevels, >, dn->dn_nlevels);
1978eda14cbcSMatt Macy dn->dn_nlevels = new_nlevels;
1979eda14cbcSMatt Macy
1980eda14cbcSMatt Macy ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]);
1981eda14cbcSMatt Macy dn->dn_next_nlevels[txgoff] = new_nlevels;
1982eda14cbcSMatt Macy
1983eda14cbcSMatt Macy /* dirty the left indirects */
1984eda14cbcSMatt Macy db = dbuf_hold_level(dn, old_nlevels, 0, FTAG);
1985eda14cbcSMatt Macy ASSERT(db != NULL);
1986eda14cbcSMatt Macy new = dbuf_dirty(db, tx);
1987eda14cbcSMatt Macy dbuf_rele(db, FTAG);
1988eda14cbcSMatt Macy
1989eda14cbcSMatt Macy /* transfer the dirty records to the new indirect */
1990eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
1991eda14cbcSMatt Macy mutex_enter(&new->dt.di.dr_mtx);
1992eda14cbcSMatt Macy list = &dn->dn_dirty_records[txgoff];
1993eda14cbcSMatt Macy for (dr = list_head(list); dr; dr = dr_next) {
1994eda14cbcSMatt Macy dr_next = list_next(&dn->dn_dirty_records[txgoff], dr);
19957877fdebSMatt Macy
19967877fdebSMatt Macy IMPLY(dr->dr_dbuf == NULL, old_nlevels == 1);
19977877fdebSMatt Macy if (dr->dr_dbuf == NULL ||
19987877fdebSMatt Macy (dr->dr_dbuf->db_level == old_nlevels - 1 &&
1999eda14cbcSMatt Macy dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID &&
20007877fdebSMatt Macy dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID)) {
2001eda14cbcSMatt Macy list_remove(&dn->dn_dirty_records[txgoff], dr);
2002eda14cbcSMatt Macy list_insert_tail(&new->dt.di.dr_children, dr);
2003eda14cbcSMatt Macy dr->dr_parent = new;
2004eda14cbcSMatt Macy }
2005eda14cbcSMatt Macy }
2006eda14cbcSMatt Macy mutex_exit(&new->dt.di.dr_mtx);
2007eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
2008eda14cbcSMatt Macy }
2009eda14cbcSMatt Macy
2010eda14cbcSMatt Macy int
dnode_set_nlevels(dnode_t * dn,int nlevels,dmu_tx_t * tx)2011eda14cbcSMatt Macy dnode_set_nlevels(dnode_t *dn, int nlevels, dmu_tx_t *tx)
2012eda14cbcSMatt Macy {
2013eda14cbcSMatt Macy int ret = 0;
2014eda14cbcSMatt Macy
2015eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2016eda14cbcSMatt Macy
2017eda14cbcSMatt Macy if (dn->dn_nlevels == nlevels) {
2018eda14cbcSMatt Macy ret = 0;
2019eda14cbcSMatt Macy goto out;
2020eda14cbcSMatt Macy } else if (nlevels < dn->dn_nlevels) {
2021eda14cbcSMatt Macy ret = SET_ERROR(EINVAL);
2022eda14cbcSMatt Macy goto out;
2023eda14cbcSMatt Macy }
2024eda14cbcSMatt Macy
2025eda14cbcSMatt Macy dnode_set_nlevels_impl(dn, nlevels, tx);
2026eda14cbcSMatt Macy
2027eda14cbcSMatt Macy out:
2028eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
2029eda14cbcSMatt Macy return (ret);
2030eda14cbcSMatt Macy }
2031eda14cbcSMatt Macy
2032eda14cbcSMatt Macy /* read-holding callers must not rely on the lock being continuously held */
2033eda14cbcSMatt Macy void
dnode_new_blkid(dnode_t * dn,uint64_t blkid,dmu_tx_t * tx,boolean_t have_read,boolean_t force)2034eda14cbcSMatt Macy dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read,
2035eda14cbcSMatt Macy boolean_t force)
2036eda14cbcSMatt Macy {
2037eda14cbcSMatt Macy int epbs, new_nlevels;
2038eda14cbcSMatt Macy uint64_t sz;
2039eda14cbcSMatt Macy
2040eda14cbcSMatt Macy ASSERT(blkid != DMU_BONUS_BLKID);
2041eda14cbcSMatt Macy
2042eda14cbcSMatt Macy ASSERT(have_read ?
2043eda14cbcSMatt Macy RW_READ_HELD(&dn->dn_struct_rwlock) :
2044eda14cbcSMatt Macy RW_WRITE_HELD(&dn->dn_struct_rwlock));
2045eda14cbcSMatt Macy
2046eda14cbcSMatt Macy /*
2047eda14cbcSMatt Macy * if we have a read-lock, check to see if we need to do any work
2048eda14cbcSMatt Macy * before upgrading to a write-lock.
2049eda14cbcSMatt Macy */
2050eda14cbcSMatt Macy if (have_read) {
2051eda14cbcSMatt Macy if (blkid <= dn->dn_maxblkid)
2052eda14cbcSMatt Macy return;
2053eda14cbcSMatt Macy
2054eda14cbcSMatt Macy if (!rw_tryupgrade(&dn->dn_struct_rwlock)) {
2055eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
2056eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2057eda14cbcSMatt Macy }
2058eda14cbcSMatt Macy }
2059eda14cbcSMatt Macy
2060eda14cbcSMatt Macy /*
2061eda14cbcSMatt Macy * Raw sends (indicated by the force flag) require that we take the
2062eda14cbcSMatt Macy * given blkid even if the value is lower than the current value.
2063eda14cbcSMatt Macy */
2064eda14cbcSMatt Macy if (!force && blkid <= dn->dn_maxblkid)
2065eda14cbcSMatt Macy goto out;
2066eda14cbcSMatt Macy
2067eda14cbcSMatt Macy /*
2068eda14cbcSMatt Macy * We use the (otherwise unused) top bit of dn_next_maxblkid[txgoff]
2069eda14cbcSMatt Macy * to indicate that this field is set. This allows us to set the
2070eda14cbcSMatt Macy * maxblkid to 0 on an existing object in dnode_sync().
2071eda14cbcSMatt Macy */
2072eda14cbcSMatt Macy dn->dn_maxblkid = blkid;
2073eda14cbcSMatt Macy dn->dn_next_maxblkid[tx->tx_txg & TXG_MASK] =
2074eda14cbcSMatt Macy blkid | DMU_NEXT_MAXBLKID_SET;
2075eda14cbcSMatt Macy
2076eda14cbcSMatt Macy /*
2077eda14cbcSMatt Macy * Compute the number of levels necessary to support the new maxblkid.
2078eda14cbcSMatt Macy * Raw sends will ensure nlevels is set correctly for us.
2079eda14cbcSMatt Macy */
2080eda14cbcSMatt Macy new_nlevels = 1;
2081eda14cbcSMatt Macy epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
2082eda14cbcSMatt Macy for (sz = dn->dn_nblkptr;
2083eda14cbcSMatt Macy sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs)
2084eda14cbcSMatt Macy new_nlevels++;
2085eda14cbcSMatt Macy
2086eda14cbcSMatt Macy ASSERT3U(new_nlevels, <=, DN_MAX_LEVELS);
2087eda14cbcSMatt Macy
2088eda14cbcSMatt Macy if (!force) {
2089eda14cbcSMatt Macy if (new_nlevels > dn->dn_nlevels)
2090eda14cbcSMatt Macy dnode_set_nlevels_impl(dn, new_nlevels, tx);
2091eda14cbcSMatt Macy } else {
2092eda14cbcSMatt Macy ASSERT3U(dn->dn_nlevels, >=, new_nlevels);
2093eda14cbcSMatt Macy }
2094eda14cbcSMatt Macy
2095eda14cbcSMatt Macy out:
2096eda14cbcSMatt Macy if (have_read)
2097eda14cbcSMatt Macy rw_downgrade(&dn->dn_struct_rwlock);
2098eda14cbcSMatt Macy }
2099eda14cbcSMatt Macy
2100eda14cbcSMatt Macy static void
dnode_dirty_l1(dnode_t * dn,uint64_t l1blkid,dmu_tx_t * tx)2101eda14cbcSMatt Macy dnode_dirty_l1(dnode_t *dn, uint64_t l1blkid, dmu_tx_t *tx)
2102eda14cbcSMatt Macy {
2103eda14cbcSMatt Macy dmu_buf_impl_t *db = dbuf_hold_level(dn, 1, l1blkid, FTAG);
2104eda14cbcSMatt Macy if (db != NULL) {
2105eda14cbcSMatt Macy dmu_buf_will_dirty(&db->db, tx);
2106eda14cbcSMatt Macy dbuf_rele(db, FTAG);
2107eda14cbcSMatt Macy }
2108eda14cbcSMatt Macy }
2109eda14cbcSMatt Macy
2110eda14cbcSMatt Macy /*
2111eda14cbcSMatt Macy * Dirty all the in-core level-1 dbufs in the range specified by start_blkid
2112eda14cbcSMatt Macy * and end_blkid.
2113eda14cbcSMatt Macy */
2114eda14cbcSMatt Macy static void
dnode_dirty_l1range(dnode_t * dn,uint64_t start_blkid,uint64_t end_blkid,dmu_tx_t * tx)2115eda14cbcSMatt Macy dnode_dirty_l1range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
2116eda14cbcSMatt Macy dmu_tx_t *tx)
2117eda14cbcSMatt Macy {
21182c48331dSMatt Macy dmu_buf_impl_t *db_search;
2119eda14cbcSMatt Macy dmu_buf_impl_t *db;
2120eda14cbcSMatt Macy avl_index_t where;
2121eda14cbcSMatt Macy
21222c48331dSMatt Macy db_search = kmem_zalloc(sizeof (dmu_buf_impl_t), KM_SLEEP);
21232c48331dSMatt Macy
2124eda14cbcSMatt Macy mutex_enter(&dn->dn_dbufs_mtx);
2125eda14cbcSMatt Macy
21262c48331dSMatt Macy db_search->db_level = 1;
21272c48331dSMatt Macy db_search->db_blkid = start_blkid + 1;
21282c48331dSMatt Macy db_search->db_state = DB_SEARCH;
2129eda14cbcSMatt Macy for (;;) {
2130eda14cbcSMatt Macy
21312c48331dSMatt Macy db = avl_find(&dn->dn_dbufs, db_search, &where);
2132eda14cbcSMatt Macy if (db == NULL)
2133eda14cbcSMatt Macy db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
2134eda14cbcSMatt Macy
2135eda14cbcSMatt Macy if (db == NULL || db->db_level != 1 ||
2136eda14cbcSMatt Macy db->db_blkid >= end_blkid) {
2137eda14cbcSMatt Macy break;
2138eda14cbcSMatt Macy }
2139eda14cbcSMatt Macy
2140eda14cbcSMatt Macy /*
2141eda14cbcSMatt Macy * Setup the next blkid we want to search for.
2142eda14cbcSMatt Macy */
21432c48331dSMatt Macy db_search->db_blkid = db->db_blkid + 1;
2144eda14cbcSMatt Macy ASSERT3U(db->db_blkid, >=, start_blkid);
2145eda14cbcSMatt Macy
2146eda14cbcSMatt Macy /*
2147eda14cbcSMatt Macy * If the dbuf transitions to DB_EVICTING while we're trying
2148eda14cbcSMatt Macy * to dirty it, then we will be unable to discover it in
2149eda14cbcSMatt Macy * the dbuf hash table. This will result in a call to
2150eda14cbcSMatt Macy * dbuf_create() which needs to acquire the dn_dbufs_mtx
2151eda14cbcSMatt Macy * lock. To avoid a deadlock, we drop the lock before
2152eda14cbcSMatt Macy * dirtying the level-1 dbuf.
2153eda14cbcSMatt Macy */
2154eda14cbcSMatt Macy mutex_exit(&dn->dn_dbufs_mtx);
2155eda14cbcSMatt Macy dnode_dirty_l1(dn, db->db_blkid, tx);
2156eda14cbcSMatt Macy mutex_enter(&dn->dn_dbufs_mtx);
2157eda14cbcSMatt Macy }
2158eda14cbcSMatt Macy
2159eda14cbcSMatt Macy #ifdef ZFS_DEBUG
2160eda14cbcSMatt Macy /*
2161eda14cbcSMatt Macy * Walk all the in-core level-1 dbufs and verify they have been dirtied.
2162eda14cbcSMatt Macy */
21632c48331dSMatt Macy db_search->db_level = 1;
21642c48331dSMatt Macy db_search->db_blkid = start_blkid + 1;
21652c48331dSMatt Macy db_search->db_state = DB_SEARCH;
21662c48331dSMatt Macy db = avl_find(&dn->dn_dbufs, db_search, &where);
2167eda14cbcSMatt Macy if (db == NULL)
2168eda14cbcSMatt Macy db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
2169eda14cbcSMatt Macy for (; db != NULL; db = AVL_NEXT(&dn->dn_dbufs, db)) {
2170eda14cbcSMatt Macy if (db->db_level != 1 || db->db_blkid >= end_blkid)
2171eda14cbcSMatt Macy break;
2172eda14cbcSMatt Macy if (db->db_state != DB_EVICTING)
2173eda14cbcSMatt Macy ASSERT(db->db_dirtycnt > 0);
2174eda14cbcSMatt Macy }
2175eda14cbcSMatt Macy #endif
21762c48331dSMatt Macy kmem_free(db_search, sizeof (dmu_buf_impl_t));
2177eda14cbcSMatt Macy mutex_exit(&dn->dn_dbufs_mtx);
2178eda14cbcSMatt Macy }
2179eda14cbcSMatt Macy
2180eda14cbcSMatt Macy void
dnode_set_dirtyctx(dnode_t * dn,dmu_tx_t * tx,const void * tag)2181a0b956f5SMartin Matuska dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, const void *tag)
2182eda14cbcSMatt Macy {
2183eda14cbcSMatt Macy /*
2184eda14cbcSMatt Macy * Don't set dirtyctx to SYNC if we're just modifying this as we
2185eda14cbcSMatt Macy * initialize the objset.
2186eda14cbcSMatt Macy */
2187eda14cbcSMatt Macy if (dn->dn_dirtyctx == DN_UNDIRTIED) {
2188eda14cbcSMatt Macy dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
2189eda14cbcSMatt Macy
2190eda14cbcSMatt Macy if (ds != NULL) {
2191eda14cbcSMatt Macy rrw_enter(&ds->ds_bp_rwlock, RW_READER, tag);
2192eda14cbcSMatt Macy }
2193eda14cbcSMatt Macy if (!BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
2194eda14cbcSMatt Macy if (dmu_tx_is_syncing(tx))
2195eda14cbcSMatt Macy dn->dn_dirtyctx = DN_DIRTY_SYNC;
2196eda14cbcSMatt Macy else
2197eda14cbcSMatt Macy dn->dn_dirtyctx = DN_DIRTY_OPEN;
2198eda14cbcSMatt Macy dn->dn_dirtyctx_firstset = tag;
2199eda14cbcSMatt Macy }
2200eda14cbcSMatt Macy if (ds != NULL) {
2201eda14cbcSMatt Macy rrw_exit(&ds->ds_bp_rwlock, tag);
2202eda14cbcSMatt Macy }
2203eda14cbcSMatt Macy }
2204eda14cbcSMatt Macy }
2205eda14cbcSMatt Macy
22066ba2210eSMartin Matuska static void
dnode_partial_zero(dnode_t * dn,uint64_t off,uint64_t blkoff,uint64_t len,dmu_tx_t * tx)22076ba2210eSMartin Matuska dnode_partial_zero(dnode_t *dn, uint64_t off, uint64_t blkoff, uint64_t len,
22086ba2210eSMartin Matuska dmu_tx_t *tx)
22096ba2210eSMartin Matuska {
22106ba2210eSMartin Matuska dmu_buf_impl_t *db;
22116ba2210eSMartin Matuska int res;
22126ba2210eSMartin Matuska
22136ba2210eSMartin Matuska rw_enter(&dn->dn_struct_rwlock, RW_READER);
22146ba2210eSMartin Matuska res = dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off), TRUE, FALSE,
22156ba2210eSMartin Matuska FTAG, &db);
22166ba2210eSMartin Matuska rw_exit(&dn->dn_struct_rwlock);
22176ba2210eSMartin Matuska if (res == 0) {
22186ba2210eSMartin Matuska db_lock_type_t dblt;
22196ba2210eSMartin Matuska boolean_t dirty;
22206ba2210eSMartin Matuska
22216ba2210eSMartin Matuska dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);
22226ba2210eSMartin Matuska /* don't dirty if not on disk and not dirty */
22236ba2210eSMartin Matuska dirty = !list_is_empty(&db->db_dirty_records) ||
22246ba2210eSMartin Matuska (db->db_blkptr && !BP_IS_HOLE(db->db_blkptr));
22256ba2210eSMartin Matuska dmu_buf_unlock_parent(db, dblt, FTAG);
22266ba2210eSMartin Matuska if (dirty) {
22276ba2210eSMartin Matuska caddr_t data;
22286ba2210eSMartin Matuska
22296ba2210eSMartin Matuska dmu_buf_will_dirty(&db->db, tx);
22306ba2210eSMartin Matuska data = db->db.db_data;
2231da5137abSMartin Matuska memset(data + blkoff, 0, len);
22326ba2210eSMartin Matuska }
22336ba2210eSMartin Matuska dbuf_rele(db, FTAG);
22346ba2210eSMartin Matuska }
22356ba2210eSMartin Matuska }
22366ba2210eSMartin Matuska
2237eda14cbcSMatt Macy void
dnode_free_range(dnode_t * dn,uint64_t off,uint64_t len,dmu_tx_t * tx)2238eda14cbcSMatt Macy dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
2239eda14cbcSMatt Macy {
2240eda14cbcSMatt Macy uint64_t blkoff, blkid, nblks;
2241eda14cbcSMatt Macy int blksz, blkshift, head, tail;
2242eda14cbcSMatt Macy int trunc = FALSE;
2243eda14cbcSMatt Macy int epbs;
2244eda14cbcSMatt Macy
2245eda14cbcSMatt Macy blksz = dn->dn_datablksz;
2246eda14cbcSMatt Macy blkshift = dn->dn_datablkshift;
2247eda14cbcSMatt Macy epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
2248eda14cbcSMatt Macy
2249eda14cbcSMatt Macy if (len == DMU_OBJECT_END) {
2250eda14cbcSMatt Macy len = UINT64_MAX - off;
2251eda14cbcSMatt Macy trunc = TRUE;
2252eda14cbcSMatt Macy }
2253eda14cbcSMatt Macy
2254eda14cbcSMatt Macy /*
2255eda14cbcSMatt Macy * First, block align the region to free:
2256eda14cbcSMatt Macy */
2257eda14cbcSMatt Macy if (ISP2(blksz)) {
2258eda14cbcSMatt Macy head = P2NPHASE(off, blksz);
2259eda14cbcSMatt Macy blkoff = P2PHASE(off, blksz);
2260eda14cbcSMatt Macy if ((off >> blkshift) > dn->dn_maxblkid)
2261eda14cbcSMatt Macy return;
2262eda14cbcSMatt Macy } else {
2263eda14cbcSMatt Macy ASSERT(dn->dn_maxblkid == 0);
2264eda14cbcSMatt Macy if (off == 0 && len >= blksz) {
2265eda14cbcSMatt Macy /*
2266eda14cbcSMatt Macy * Freeing the whole block; fast-track this request.
2267eda14cbcSMatt Macy */
2268eda14cbcSMatt Macy blkid = 0;
2269eda14cbcSMatt Macy nblks = 1;
2270eda14cbcSMatt Macy if (dn->dn_nlevels > 1) {
2271eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2272eda14cbcSMatt Macy dnode_dirty_l1(dn, 0, tx);
2273eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
2274eda14cbcSMatt Macy }
2275eda14cbcSMatt Macy goto done;
2276eda14cbcSMatt Macy } else if (off >= blksz) {
2277eda14cbcSMatt Macy /* Freeing past end-of-data */
2278eda14cbcSMatt Macy return;
2279eda14cbcSMatt Macy } else {
2280eda14cbcSMatt Macy /* Freeing part of the block. */
2281eda14cbcSMatt Macy head = blksz - off;
2282eda14cbcSMatt Macy ASSERT3U(head, >, 0);
2283eda14cbcSMatt Macy }
2284eda14cbcSMatt Macy blkoff = off;
2285eda14cbcSMatt Macy }
2286eda14cbcSMatt Macy /* zero out any partial block data at the start of the range */
2287eda14cbcSMatt Macy if (head) {
2288eda14cbcSMatt Macy ASSERT3U(blkoff + head, ==, blksz);
2289eda14cbcSMatt Macy if (len < head)
2290eda14cbcSMatt Macy head = len;
22916ba2210eSMartin Matuska dnode_partial_zero(dn, off, blkoff, head, tx);
2292eda14cbcSMatt Macy off += head;
2293eda14cbcSMatt Macy len -= head;
2294eda14cbcSMatt Macy }
2295eda14cbcSMatt Macy
2296eda14cbcSMatt Macy /* If the range was less than one block, we're done */
2297eda14cbcSMatt Macy if (len == 0)
2298eda14cbcSMatt Macy return;
2299eda14cbcSMatt Macy
2300eda14cbcSMatt Macy /* If the remaining range is past end of file, we're done */
2301eda14cbcSMatt Macy if ((off >> blkshift) > dn->dn_maxblkid)
2302eda14cbcSMatt Macy return;
2303eda14cbcSMatt Macy
2304eda14cbcSMatt Macy ASSERT(ISP2(blksz));
2305eda14cbcSMatt Macy if (trunc)
2306eda14cbcSMatt Macy tail = 0;
2307eda14cbcSMatt Macy else
2308eda14cbcSMatt Macy tail = P2PHASE(len, blksz);
2309eda14cbcSMatt Macy
2310eda14cbcSMatt Macy ASSERT0(P2PHASE(off, blksz));
2311eda14cbcSMatt Macy /* zero out any partial block data at the end of the range */
2312eda14cbcSMatt Macy if (tail) {
2313eda14cbcSMatt Macy if (len < tail)
2314eda14cbcSMatt Macy tail = len;
23156ba2210eSMartin Matuska dnode_partial_zero(dn, off + len, 0, tail, tx);
2316eda14cbcSMatt Macy len -= tail;
2317eda14cbcSMatt Macy }
2318eda14cbcSMatt Macy
2319eda14cbcSMatt Macy /* If the range did not include a full block, we are done */
2320eda14cbcSMatt Macy if (len == 0)
2321eda14cbcSMatt Macy return;
2322eda14cbcSMatt Macy
2323eda14cbcSMatt Macy ASSERT(IS_P2ALIGNED(off, blksz));
2324eda14cbcSMatt Macy ASSERT(trunc || IS_P2ALIGNED(len, blksz));
2325eda14cbcSMatt Macy blkid = off >> blkshift;
2326eda14cbcSMatt Macy nblks = len >> blkshift;
2327eda14cbcSMatt Macy if (trunc)
2328eda14cbcSMatt Macy nblks += 1;
2329eda14cbcSMatt Macy
2330eda14cbcSMatt Macy /*
2331eda14cbcSMatt Macy * Dirty all the indirect blocks in this range. Note that only
2332eda14cbcSMatt Macy * the first and last indirect blocks can actually be written
2333eda14cbcSMatt Macy * (if they were partially freed) -- they must be dirtied, even if
2334eda14cbcSMatt Macy * they do not exist on disk yet. The interior blocks will
2335eda14cbcSMatt Macy * be freed by free_children(), so they will not actually be written.
2336eda14cbcSMatt Macy * Even though these interior blocks will not be written, we
2337eda14cbcSMatt Macy * dirty them for two reasons:
2338eda14cbcSMatt Macy *
2339eda14cbcSMatt Macy * - It ensures that the indirect blocks remain in memory until
2340eda14cbcSMatt Macy * syncing context. (They have already been prefetched by
2341eda14cbcSMatt Macy * dmu_tx_hold_free(), so we don't have to worry about reading
2342eda14cbcSMatt Macy * them serially here.)
2343eda14cbcSMatt Macy *
2344eda14cbcSMatt Macy * - The dirty space accounting will put pressure on the txg sync
2345eda14cbcSMatt Macy * mechanism to begin syncing, and to delay transactions if there
2346eda14cbcSMatt Macy * is a large amount of freeing. Even though these indirect
2347eda14cbcSMatt Macy * blocks will not be written, we could need to write the same
2348eda14cbcSMatt Macy * amount of space if we copy the freed BPs into deadlists.
2349eda14cbcSMatt Macy */
2350eda14cbcSMatt Macy if (dn->dn_nlevels > 1) {
2351eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2352eda14cbcSMatt Macy uint64_t first, last;
2353eda14cbcSMatt Macy
2354eda14cbcSMatt Macy first = blkid >> epbs;
2355eda14cbcSMatt Macy dnode_dirty_l1(dn, first, tx);
2356eda14cbcSMatt Macy if (trunc)
2357eda14cbcSMatt Macy last = dn->dn_maxblkid >> epbs;
2358eda14cbcSMatt Macy else
2359eda14cbcSMatt Macy last = (blkid + nblks - 1) >> epbs;
2360eda14cbcSMatt Macy if (last != first)
2361eda14cbcSMatt Macy dnode_dirty_l1(dn, last, tx);
2362eda14cbcSMatt Macy
2363eda14cbcSMatt Macy dnode_dirty_l1range(dn, first, last, tx);
2364eda14cbcSMatt Macy
2365eda14cbcSMatt Macy int shift = dn->dn_datablkshift + dn->dn_indblkshift -
2366eda14cbcSMatt Macy SPA_BLKPTRSHIFT;
2367eda14cbcSMatt Macy for (uint64_t i = first + 1; i < last; i++) {
2368eda14cbcSMatt Macy /*
2369eda14cbcSMatt Macy * Set i to the blockid of the next non-hole
2370eda14cbcSMatt Macy * level-1 indirect block at or after i. Note
2371eda14cbcSMatt Macy * that dnode_next_offset() operates in terms of
2372eda14cbcSMatt Macy * level-0-equivalent bytes.
2373eda14cbcSMatt Macy */
2374eda14cbcSMatt Macy uint64_t ibyte = i << shift;
2375eda14cbcSMatt Macy int err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK,
2376eda14cbcSMatt Macy &ibyte, 2, 1, 0);
2377eda14cbcSMatt Macy i = ibyte >> shift;
2378eda14cbcSMatt Macy if (i >= last)
2379eda14cbcSMatt Macy break;
2380eda14cbcSMatt Macy
2381eda14cbcSMatt Macy /*
2382eda14cbcSMatt Macy * Normally we should not see an error, either
2383eda14cbcSMatt Macy * from dnode_next_offset() or dbuf_hold_level()
2384eda14cbcSMatt Macy * (except for ESRCH from dnode_next_offset).
2385eda14cbcSMatt Macy * If there is an i/o error, then when we read
2386eda14cbcSMatt Macy * this block in syncing context, it will use
2387eda14cbcSMatt Macy * ZIO_FLAG_MUSTSUCCEED, and thus hang/panic according
2388eda14cbcSMatt Macy * to the "failmode" property. dnode_next_offset()
2389eda14cbcSMatt Macy * doesn't have a flag to indicate MUSTSUCCEED.
2390eda14cbcSMatt Macy */
2391eda14cbcSMatt Macy if (err != 0)
2392eda14cbcSMatt Macy break;
2393eda14cbcSMatt Macy
2394eda14cbcSMatt Macy dnode_dirty_l1(dn, i, tx);
2395eda14cbcSMatt Macy }
2396eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
2397eda14cbcSMatt Macy }
2398eda14cbcSMatt Macy
2399eda14cbcSMatt Macy done:
2400eda14cbcSMatt Macy /*
2401eda14cbcSMatt Macy * Add this range to the dnode range list.
2402eda14cbcSMatt Macy * We will finish up this free operation in the syncing phase.
2403eda14cbcSMatt Macy */
2404eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
2405eda14cbcSMatt Macy {
2406eda14cbcSMatt Macy int txgoff = tx->tx_txg & TXG_MASK;
2407eda14cbcSMatt Macy if (dn->dn_free_ranges[txgoff] == NULL) {
2408eda14cbcSMatt Macy dn->dn_free_ranges[txgoff] = range_tree_create(NULL,
2409eda14cbcSMatt Macy RANGE_SEG64, NULL, 0, 0);
2410eda14cbcSMatt Macy }
2411eda14cbcSMatt Macy range_tree_clear(dn->dn_free_ranges[txgoff], blkid, nblks);
2412eda14cbcSMatt Macy range_tree_add(dn->dn_free_ranges[txgoff], blkid, nblks);
2413eda14cbcSMatt Macy }
2414eda14cbcSMatt Macy dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n",
241533b8c039SMartin Matuska (u_longlong_t)blkid, (u_longlong_t)nblks,
241633b8c039SMartin Matuska (u_longlong_t)tx->tx_txg);
2417eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
2418eda14cbcSMatt Macy
2419eda14cbcSMatt Macy dbuf_free_range(dn, blkid, blkid + nblks - 1, tx);
2420eda14cbcSMatt Macy dnode_setdirty(dn, tx);
2421eda14cbcSMatt Macy }
2422eda14cbcSMatt Macy
2423eda14cbcSMatt Macy static boolean_t
dnode_spill_freed(dnode_t * dn)2424eda14cbcSMatt Macy dnode_spill_freed(dnode_t *dn)
2425eda14cbcSMatt Macy {
2426eda14cbcSMatt Macy int i;
2427eda14cbcSMatt Macy
2428eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
2429eda14cbcSMatt Macy for (i = 0; i < TXG_SIZE; i++) {
2430eda14cbcSMatt Macy if (dn->dn_rm_spillblk[i] == DN_KILL_SPILLBLK)
2431eda14cbcSMatt Macy break;
2432eda14cbcSMatt Macy }
2433eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
2434eda14cbcSMatt Macy return (i < TXG_SIZE);
2435eda14cbcSMatt Macy }
2436eda14cbcSMatt Macy
2437eda14cbcSMatt Macy /* return TRUE if this blkid was freed in a recent txg, or FALSE if it wasn't */
2438eda14cbcSMatt Macy uint64_t
dnode_block_freed(dnode_t * dn,uint64_t blkid)2439eda14cbcSMatt Macy dnode_block_freed(dnode_t *dn, uint64_t blkid)
2440eda14cbcSMatt Macy {
2441eda14cbcSMatt Macy int i;
2442eda14cbcSMatt Macy
2443eda14cbcSMatt Macy if (blkid == DMU_BONUS_BLKID)
2444eda14cbcSMatt Macy return (FALSE);
2445eda14cbcSMatt Macy
2446eda14cbcSMatt Macy if (dn->dn_free_txg)
2447eda14cbcSMatt Macy return (TRUE);
2448eda14cbcSMatt Macy
2449eda14cbcSMatt Macy if (blkid == DMU_SPILL_BLKID)
2450eda14cbcSMatt Macy return (dnode_spill_freed(dn));
2451eda14cbcSMatt Macy
2452eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
2453eda14cbcSMatt Macy for (i = 0; i < TXG_SIZE; i++) {
2454eda14cbcSMatt Macy if (dn->dn_free_ranges[i] != NULL &&
2455eda14cbcSMatt Macy range_tree_contains(dn->dn_free_ranges[i], blkid, 1))
2456eda14cbcSMatt Macy break;
2457eda14cbcSMatt Macy }
2458eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
2459eda14cbcSMatt Macy return (i < TXG_SIZE);
2460eda14cbcSMatt Macy }
2461eda14cbcSMatt Macy
2462eda14cbcSMatt Macy /* call from syncing context when we actually write/free space for this dnode */
2463eda14cbcSMatt Macy void
dnode_diduse_space(dnode_t * dn,int64_t delta)2464eda14cbcSMatt Macy dnode_diduse_space(dnode_t *dn, int64_t delta)
2465eda14cbcSMatt Macy {
2466eda14cbcSMatt Macy uint64_t space;
2467eda14cbcSMatt Macy dprintf_dnode(dn, "dn=%p dnp=%p used=%llu delta=%lld\n",
2468eda14cbcSMatt Macy dn, dn->dn_phys,
2469eda14cbcSMatt Macy (u_longlong_t)dn->dn_phys->dn_used,
2470eda14cbcSMatt Macy (longlong_t)delta);
2471eda14cbcSMatt Macy
2472eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx);
2473eda14cbcSMatt Macy space = DN_USED_BYTES(dn->dn_phys);
2474eda14cbcSMatt Macy if (delta > 0) {
2475eda14cbcSMatt Macy ASSERT3U(space + delta, >=, space); /* no overflow */
2476eda14cbcSMatt Macy } else {
2477eda14cbcSMatt Macy ASSERT3U(space, >=, -delta); /* no underflow */
2478eda14cbcSMatt Macy }
2479eda14cbcSMatt Macy space += delta;
2480eda14cbcSMatt Macy if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) {
2481eda14cbcSMatt Macy ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0);
2482eda14cbcSMatt Macy ASSERT0(P2PHASE(space, 1<<DEV_BSHIFT));
2483eda14cbcSMatt Macy dn->dn_phys->dn_used = space >> DEV_BSHIFT;
2484eda14cbcSMatt Macy } else {
2485eda14cbcSMatt Macy dn->dn_phys->dn_used = space;
2486eda14cbcSMatt Macy dn->dn_phys->dn_flags |= DNODE_FLAG_USED_BYTES;
2487eda14cbcSMatt Macy }
2488eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx);
2489eda14cbcSMatt Macy }
2490eda14cbcSMatt Macy
2491eda14cbcSMatt Macy /*
2492eda14cbcSMatt Macy * Scans a block at the indicated "level" looking for a hole or data,
2493eda14cbcSMatt Macy * depending on 'flags'.
2494eda14cbcSMatt Macy *
2495eda14cbcSMatt Macy * If level > 0, then we are scanning an indirect block looking at its
2496eda14cbcSMatt Macy * pointers. If level == 0, then we are looking at a block of dnodes.
2497eda14cbcSMatt Macy *
2498eda14cbcSMatt Macy * If we don't find what we are looking for in the block, we return ESRCH.
2499eda14cbcSMatt Macy * Otherwise, return with *offset pointing to the beginning (if searching
2500eda14cbcSMatt Macy * forwards) or end (if searching backwards) of the range covered by the
2501eda14cbcSMatt Macy * block pointer we matched on (or dnode).
2502eda14cbcSMatt Macy *
2503eda14cbcSMatt Macy * The basic search algorithm used below by dnode_next_offset() is to
2504eda14cbcSMatt Macy * use this function to search up the block tree (widen the search) until
2505eda14cbcSMatt Macy * we find something (i.e., we don't return ESRCH) and then search back
2506eda14cbcSMatt Macy * down the tree (narrow the search) until we reach our original search
2507eda14cbcSMatt Macy * level.
2508eda14cbcSMatt Macy */
2509eda14cbcSMatt Macy static int
dnode_next_offset_level(dnode_t * dn,int flags,uint64_t * offset,int lvl,uint64_t blkfill,uint64_t txg)2510eda14cbcSMatt Macy dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset,
2511eda14cbcSMatt Macy int lvl, uint64_t blkfill, uint64_t txg)
2512eda14cbcSMatt Macy {
2513eda14cbcSMatt Macy dmu_buf_impl_t *db = NULL;
2514eda14cbcSMatt Macy void *data = NULL;
2515eda14cbcSMatt Macy uint64_t epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2516eda14cbcSMatt Macy uint64_t epb = 1ULL << epbs;
2517eda14cbcSMatt Macy uint64_t minfill, maxfill;
2518eda14cbcSMatt Macy boolean_t hole;
2519eda14cbcSMatt Macy int i, inc, error, span;
2520eda14cbcSMatt Macy
2521eda14cbcSMatt Macy ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
2522eda14cbcSMatt Macy
2523eda14cbcSMatt Macy hole = ((flags & DNODE_FIND_HOLE) != 0);
2524eda14cbcSMatt Macy inc = (flags & DNODE_FIND_BACKWARDS) ? -1 : 1;
2525eda14cbcSMatt Macy ASSERT(txg == 0 || !hole);
2526eda14cbcSMatt Macy
2527eda14cbcSMatt Macy if (lvl == dn->dn_phys->dn_nlevels) {
2528eda14cbcSMatt Macy error = 0;
2529eda14cbcSMatt Macy epb = dn->dn_phys->dn_nblkptr;
2530eda14cbcSMatt Macy data = dn->dn_phys->dn_blkptr;
2531eda14cbcSMatt Macy } else {
2532eda14cbcSMatt Macy uint64_t blkid = dbuf_whichblock(dn, lvl, *offset);
2533eda14cbcSMatt Macy error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FALSE, FTAG, &db);
2534eda14cbcSMatt Macy if (error) {
2535eda14cbcSMatt Macy if (error != ENOENT)
2536eda14cbcSMatt Macy return (error);
2537eda14cbcSMatt Macy if (hole)
2538eda14cbcSMatt Macy return (0);
2539eda14cbcSMatt Macy /*
2540eda14cbcSMatt Macy * This can only happen when we are searching up
2541eda14cbcSMatt Macy * the block tree for data. We don't really need to
2542eda14cbcSMatt Macy * adjust the offset, as we will just end up looking
2543eda14cbcSMatt Macy * at the pointer to this block in its parent, and its
2544eda14cbcSMatt Macy * going to be unallocated, so we will skip over it.
2545eda14cbcSMatt Macy */
2546eda14cbcSMatt Macy return (SET_ERROR(ESRCH));
2547eda14cbcSMatt Macy }
2548eda14cbcSMatt Macy error = dbuf_read(db, NULL,
2549c40487d4SMatt Macy DB_RF_CANFAIL | DB_RF_HAVESTRUCT |
2550c40487d4SMatt Macy DB_RF_NO_DECRYPT | DB_RF_NOPREFETCH);
2551eda14cbcSMatt Macy if (error) {
2552eda14cbcSMatt Macy dbuf_rele(db, FTAG);
2553eda14cbcSMatt Macy return (error);
2554eda14cbcSMatt Macy }
2555eda14cbcSMatt Macy data = db->db.db_data;
2556eda14cbcSMatt Macy rw_enter(&db->db_rwlock, RW_READER);
2557eda14cbcSMatt Macy }
2558eda14cbcSMatt Macy
2559eda14cbcSMatt Macy if (db != NULL && txg != 0 && (db->db_blkptr == NULL ||
2560783d3ff6SMartin Matuska BP_GET_LOGICAL_BIRTH(db->db_blkptr) <= txg ||
2561eda14cbcSMatt Macy BP_IS_HOLE(db->db_blkptr))) {
2562eda14cbcSMatt Macy /*
2563eda14cbcSMatt Macy * This can only happen when we are searching up the tree
2564eda14cbcSMatt Macy * and these conditions mean that we need to keep climbing.
2565eda14cbcSMatt Macy */
2566eda14cbcSMatt Macy error = SET_ERROR(ESRCH);
2567eda14cbcSMatt Macy } else if (lvl == 0) {
2568eda14cbcSMatt Macy dnode_phys_t *dnp = data;
2569eda14cbcSMatt Macy
2570eda14cbcSMatt Macy ASSERT(dn->dn_type == DMU_OT_DNODE);
2571eda14cbcSMatt Macy ASSERT(!(flags & DNODE_FIND_BACKWARDS));
2572eda14cbcSMatt Macy
2573eda14cbcSMatt Macy for (i = (*offset >> DNODE_SHIFT) & (blkfill - 1);
2574eda14cbcSMatt Macy i < blkfill; i += dnp[i].dn_extra_slots + 1) {
2575eda14cbcSMatt Macy if ((dnp[i].dn_type == DMU_OT_NONE) == hole)
2576eda14cbcSMatt Macy break;
2577eda14cbcSMatt Macy }
2578eda14cbcSMatt Macy
2579eda14cbcSMatt Macy if (i == blkfill)
2580eda14cbcSMatt Macy error = SET_ERROR(ESRCH);
2581eda14cbcSMatt Macy
2582eda14cbcSMatt Macy *offset = (*offset & ~(DNODE_BLOCK_SIZE - 1)) +
2583eda14cbcSMatt Macy (i << DNODE_SHIFT);
2584eda14cbcSMatt Macy } else {
2585eda14cbcSMatt Macy blkptr_t *bp = data;
2586eda14cbcSMatt Macy uint64_t start = *offset;
2587eda14cbcSMatt Macy span = (lvl - 1) * epbs + dn->dn_datablkshift;
2588eda14cbcSMatt Macy minfill = 0;
2589eda14cbcSMatt Macy maxfill = blkfill << ((lvl - 1) * epbs);
2590eda14cbcSMatt Macy
2591eda14cbcSMatt Macy if (hole)
2592eda14cbcSMatt Macy maxfill--;
2593eda14cbcSMatt Macy else
2594eda14cbcSMatt Macy minfill++;
2595eda14cbcSMatt Macy
2596eda14cbcSMatt Macy if (span >= 8 * sizeof (*offset)) {
2597eda14cbcSMatt Macy /* This only happens on the highest indirection level */
2598eda14cbcSMatt Macy ASSERT3U((lvl - 1), ==, dn->dn_phys->dn_nlevels - 1);
2599eda14cbcSMatt Macy *offset = 0;
2600eda14cbcSMatt Macy } else {
2601eda14cbcSMatt Macy *offset = *offset >> span;
2602eda14cbcSMatt Macy }
2603eda14cbcSMatt Macy
2604eda14cbcSMatt Macy for (i = BF64_GET(*offset, 0, epbs);
2605eda14cbcSMatt Macy i >= 0 && i < epb; i += inc) {
2606eda14cbcSMatt Macy if (BP_GET_FILL(&bp[i]) >= minfill &&
2607eda14cbcSMatt Macy BP_GET_FILL(&bp[i]) <= maxfill &&
2608783d3ff6SMartin Matuska (hole || BP_GET_LOGICAL_BIRTH(&bp[i]) > txg))
2609eda14cbcSMatt Macy break;
2610eda14cbcSMatt Macy if (inc > 0 || *offset > 0)
2611eda14cbcSMatt Macy *offset += inc;
2612eda14cbcSMatt Macy }
2613eda14cbcSMatt Macy
2614eda14cbcSMatt Macy if (span >= 8 * sizeof (*offset)) {
2615eda14cbcSMatt Macy *offset = start;
2616eda14cbcSMatt Macy } else {
2617eda14cbcSMatt Macy *offset = *offset << span;
2618eda14cbcSMatt Macy }
2619eda14cbcSMatt Macy
2620eda14cbcSMatt Macy if (inc < 0) {
2621eda14cbcSMatt Macy /* traversing backwards; position offset at the end */
26222a58b312SMartin Matuska if (span < 8 * sizeof (*offset))
26232a58b312SMartin Matuska *offset = MIN(*offset + (1ULL << span) - 1,
26242a58b312SMartin Matuska start);
2625eda14cbcSMatt Macy } else if (*offset < start) {
2626eda14cbcSMatt Macy *offset = start;
2627eda14cbcSMatt Macy }
2628eda14cbcSMatt Macy if (i < 0 || i >= epb)
2629eda14cbcSMatt Macy error = SET_ERROR(ESRCH);
2630eda14cbcSMatt Macy }
2631eda14cbcSMatt Macy
2632eda14cbcSMatt Macy if (db != NULL) {
2633eda14cbcSMatt Macy rw_exit(&db->db_rwlock);
2634eda14cbcSMatt Macy dbuf_rele(db, FTAG);
2635eda14cbcSMatt Macy }
2636eda14cbcSMatt Macy
2637eda14cbcSMatt Macy return (error);
2638eda14cbcSMatt Macy }
2639eda14cbcSMatt Macy
2640eda14cbcSMatt Macy /*
2641eda14cbcSMatt Macy * Find the next hole, data, or sparse region at or after *offset.
2642eda14cbcSMatt Macy * The value 'blkfill' tells us how many items we expect to find
2643eda14cbcSMatt Macy * in an L0 data block; this value is 1 for normal objects,
2644eda14cbcSMatt Macy * DNODES_PER_BLOCK for the meta dnode, and some fraction of
2645eda14cbcSMatt Macy * DNODES_PER_BLOCK when searching for sparse regions thereof.
2646eda14cbcSMatt Macy *
2647eda14cbcSMatt Macy * Examples:
2648eda14cbcSMatt Macy *
2649eda14cbcSMatt Macy * dnode_next_offset(dn, flags, offset, 1, 1, 0);
2650eda14cbcSMatt Macy * Finds the next/previous hole/data in a file.
2651eda14cbcSMatt Macy * Used in dmu_offset_next().
2652eda14cbcSMatt Macy *
2653eda14cbcSMatt Macy * dnode_next_offset(mdn, flags, offset, 0, DNODES_PER_BLOCK, txg);
2654eda14cbcSMatt Macy * Finds the next free/allocated dnode an objset's meta-dnode.
2655eda14cbcSMatt Macy * Only finds objects that have new contents since txg (ie.
2656eda14cbcSMatt Macy * bonus buffer changes and content removal are ignored).
2657eda14cbcSMatt Macy * Used in dmu_object_next().
2658eda14cbcSMatt Macy *
2659eda14cbcSMatt Macy * dnode_next_offset(mdn, DNODE_FIND_HOLE, offset, 2, DNODES_PER_BLOCK >> 2, 0);
2660eda14cbcSMatt Macy * Finds the next L2 meta-dnode bp that's at most 1/4 full.
2661eda14cbcSMatt Macy * Used in dmu_object_alloc().
2662eda14cbcSMatt Macy */
2663eda14cbcSMatt Macy int
dnode_next_offset(dnode_t * dn,int flags,uint64_t * offset,int minlvl,uint64_t blkfill,uint64_t txg)2664eda14cbcSMatt Macy dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset,
2665eda14cbcSMatt Macy int minlvl, uint64_t blkfill, uint64_t txg)
2666eda14cbcSMatt Macy {
2667eda14cbcSMatt Macy uint64_t initial_offset = *offset;
2668eda14cbcSMatt Macy int lvl, maxlvl;
2669eda14cbcSMatt Macy int error = 0;
2670eda14cbcSMatt Macy
2671eda14cbcSMatt Macy if (!(flags & DNODE_FIND_HAVELOCK))
2672eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER);
2673eda14cbcSMatt Macy
2674eda14cbcSMatt Macy if (dn->dn_phys->dn_nlevels == 0) {
2675eda14cbcSMatt Macy error = SET_ERROR(ESRCH);
2676eda14cbcSMatt Macy goto out;
2677eda14cbcSMatt Macy }
2678eda14cbcSMatt Macy
2679eda14cbcSMatt Macy if (dn->dn_datablkshift == 0) {
2680eda14cbcSMatt Macy if (*offset < dn->dn_datablksz) {
2681eda14cbcSMatt Macy if (flags & DNODE_FIND_HOLE)
2682eda14cbcSMatt Macy *offset = dn->dn_datablksz;
2683eda14cbcSMatt Macy } else {
2684eda14cbcSMatt Macy error = SET_ERROR(ESRCH);
2685eda14cbcSMatt Macy }
2686eda14cbcSMatt Macy goto out;
2687eda14cbcSMatt Macy }
2688eda14cbcSMatt Macy
2689eda14cbcSMatt Macy maxlvl = dn->dn_phys->dn_nlevels;
2690eda14cbcSMatt Macy
2691eda14cbcSMatt Macy for (lvl = minlvl; lvl <= maxlvl; lvl++) {
2692eda14cbcSMatt Macy error = dnode_next_offset_level(dn,
2693eda14cbcSMatt Macy flags, offset, lvl, blkfill, txg);
2694eda14cbcSMatt Macy if (error != ESRCH)
2695eda14cbcSMatt Macy break;
2696eda14cbcSMatt Macy }
2697eda14cbcSMatt Macy
2698eda14cbcSMatt Macy while (error == 0 && --lvl >= minlvl) {
2699eda14cbcSMatt Macy error = dnode_next_offset_level(dn,
2700eda14cbcSMatt Macy flags, offset, lvl, blkfill, txg);
2701eda14cbcSMatt Macy }
2702eda14cbcSMatt Macy
2703eda14cbcSMatt Macy /*
2704eda14cbcSMatt Macy * There's always a "virtual hole" at the end of the object, even
2705eda14cbcSMatt Macy * if all BP's which physically exist are non-holes.
2706eda14cbcSMatt Macy */
2707eda14cbcSMatt Macy if ((flags & DNODE_FIND_HOLE) && error == ESRCH && txg == 0 &&
2708eda14cbcSMatt Macy minlvl == 1 && blkfill == 1 && !(flags & DNODE_FIND_BACKWARDS)) {
2709eda14cbcSMatt Macy error = 0;
2710eda14cbcSMatt Macy }
2711eda14cbcSMatt Macy
2712eda14cbcSMatt Macy if (error == 0 && (flags & DNODE_FIND_BACKWARDS ?
2713eda14cbcSMatt Macy initial_offset < *offset : initial_offset > *offset))
2714eda14cbcSMatt Macy error = SET_ERROR(ESRCH);
2715eda14cbcSMatt Macy out:
2716eda14cbcSMatt Macy if (!(flags & DNODE_FIND_HAVELOCK))
2717eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock);
2718eda14cbcSMatt Macy
2719eda14cbcSMatt Macy return (error);
2720eda14cbcSMatt Macy }
2721eda14cbcSMatt Macy
2722eda14cbcSMatt Macy #if defined(_KERNEL)
2723eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_hold);
2724eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_rele);
2725eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_set_nlevels);
2726eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_set_blksz);
2727eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_free_range);
2728eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_evict_dbufs);
2729eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_evict_bonus);
2730eda14cbcSMatt Macy #endif
273115f0b8c3SMartin Matuska
273215f0b8c3SMartin Matuska ZFS_MODULE_PARAM(zfs, zfs_, default_bs, INT, ZMOD_RW,
273315f0b8c3SMartin Matuska "Default dnode block shift");
273415f0b8c3SMartin Matuska ZFS_MODULE_PARAM(zfs, zfs_, default_ibs, INT, ZMOD_RW,
273515f0b8c3SMartin Matuska "Default dnode indirect block shift");
2736