1eda14cbcSMatt Macy /* 2eda14cbcSMatt Macy * CDDL HEADER START 3eda14cbcSMatt Macy * 4eda14cbcSMatt Macy * The contents of this file are subject to the terms of the 5eda14cbcSMatt Macy * Common Development and Distribution License (the "License"). 6eda14cbcSMatt Macy * You may not use this file except in compliance with the License. 7eda14cbcSMatt Macy * 8eda14cbcSMatt Macy * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9eda14cbcSMatt Macy * or http://www.opensolaris.org/os/licensing. 10eda14cbcSMatt Macy * See the License for the specific language governing permissions 11eda14cbcSMatt Macy * and limitations under the License. 12eda14cbcSMatt Macy * 13eda14cbcSMatt Macy * When distributing Covered Code, include this CDDL HEADER in each 14eda14cbcSMatt Macy * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15eda14cbcSMatt Macy * If applicable, add the following below this CDDL HEADER, with the 16eda14cbcSMatt Macy * fields enclosed by brackets "[]" replaced with your own identifying 17eda14cbcSMatt Macy * information: Portions Copyright [yyyy] [name of copyright owner] 18eda14cbcSMatt Macy * 19eda14cbcSMatt Macy * CDDL HEADER END 20eda14cbcSMatt Macy */ 21eda14cbcSMatt Macy /* 22eda14cbcSMatt Macy * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23f9693befSMartin Matuska * Copyright (c) 2012, 2021 by Delphix. All rights reserved. 24eda14cbcSMatt Macy */ 25eda14cbcSMatt Macy 26eda14cbcSMatt Macy #include <sys/zfs_context.h> 27eda14cbcSMatt Macy #include <sys/zfs_refcount.h> 28eda14cbcSMatt Macy 29eda14cbcSMatt Macy /* 30eda14cbcSMatt Macy * Reference count tracking is disabled by default. It's memory requirements 31eda14cbcSMatt Macy * are reasonable, however as implemented it consumes a significant amount of 32eda14cbcSMatt Macy * cpu time. Until its performance is improved it should be manually enabled. 33eda14cbcSMatt Macy */ 34eda14cbcSMatt Macy int reference_tracking_enable = FALSE; 35eda14cbcSMatt Macy int reference_history = 3; /* tunable */ 36eda14cbcSMatt Macy 37eda14cbcSMatt Macy #ifdef ZFS_DEBUG 38eda14cbcSMatt Macy static kmem_cache_t *reference_cache; 39eda14cbcSMatt Macy static kmem_cache_t *reference_history_cache; 40eda14cbcSMatt Macy 41eda14cbcSMatt Macy void 42eda14cbcSMatt Macy zfs_refcount_init(void) 43eda14cbcSMatt Macy { 44eda14cbcSMatt Macy reference_cache = kmem_cache_create("reference_cache", 45eda14cbcSMatt Macy sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 46eda14cbcSMatt Macy 47eda14cbcSMatt Macy reference_history_cache = kmem_cache_create("reference_history_cache", 48eda14cbcSMatt Macy sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 49eda14cbcSMatt Macy } 50eda14cbcSMatt Macy 51eda14cbcSMatt Macy void 52eda14cbcSMatt Macy zfs_refcount_fini(void) 53eda14cbcSMatt Macy { 54eda14cbcSMatt Macy kmem_cache_destroy(reference_cache); 55eda14cbcSMatt Macy kmem_cache_destroy(reference_history_cache); 56eda14cbcSMatt Macy } 57eda14cbcSMatt Macy 58eda14cbcSMatt Macy void 59eda14cbcSMatt Macy zfs_refcount_create(zfs_refcount_t *rc) 60eda14cbcSMatt Macy { 61eda14cbcSMatt Macy mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL); 62eda14cbcSMatt Macy list_create(&rc->rc_list, sizeof (reference_t), 63eda14cbcSMatt Macy offsetof(reference_t, ref_link)); 64eda14cbcSMatt Macy list_create(&rc->rc_removed, sizeof (reference_t), 65eda14cbcSMatt Macy offsetof(reference_t, ref_link)); 66eda14cbcSMatt Macy rc->rc_count = 0; 67eda14cbcSMatt Macy rc->rc_removed_count = 0; 68eda14cbcSMatt Macy rc->rc_tracked = reference_tracking_enable; 69eda14cbcSMatt Macy } 70eda14cbcSMatt Macy 71eda14cbcSMatt Macy void 72eda14cbcSMatt Macy zfs_refcount_create_tracked(zfs_refcount_t *rc) 73eda14cbcSMatt Macy { 74eda14cbcSMatt Macy zfs_refcount_create(rc); 75eda14cbcSMatt Macy rc->rc_tracked = B_TRUE; 76eda14cbcSMatt Macy } 77eda14cbcSMatt Macy 78eda14cbcSMatt Macy void 79eda14cbcSMatt Macy zfs_refcount_create_untracked(zfs_refcount_t *rc) 80eda14cbcSMatt Macy { 81eda14cbcSMatt Macy zfs_refcount_create(rc); 82eda14cbcSMatt Macy rc->rc_tracked = B_FALSE; 83eda14cbcSMatt Macy } 84eda14cbcSMatt Macy 85eda14cbcSMatt Macy void 86eda14cbcSMatt Macy zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number) 87eda14cbcSMatt Macy { 88eda14cbcSMatt Macy reference_t *ref; 89eda14cbcSMatt Macy 90eda14cbcSMatt Macy ASSERT3U(rc->rc_count, ==, number); 91eda14cbcSMatt Macy while ((ref = list_head(&rc->rc_list))) { 92eda14cbcSMatt Macy list_remove(&rc->rc_list, ref); 93eda14cbcSMatt Macy kmem_cache_free(reference_cache, ref); 94eda14cbcSMatt Macy } 95eda14cbcSMatt Macy list_destroy(&rc->rc_list); 96eda14cbcSMatt Macy 97eda14cbcSMatt Macy while ((ref = list_head(&rc->rc_removed))) { 98eda14cbcSMatt Macy list_remove(&rc->rc_removed, ref); 99eda14cbcSMatt Macy kmem_cache_free(reference_history_cache, ref->ref_removed); 100eda14cbcSMatt Macy kmem_cache_free(reference_cache, ref); 101eda14cbcSMatt Macy } 102eda14cbcSMatt Macy list_destroy(&rc->rc_removed); 103eda14cbcSMatt Macy mutex_destroy(&rc->rc_mtx); 104eda14cbcSMatt Macy } 105eda14cbcSMatt Macy 106eda14cbcSMatt Macy void 107eda14cbcSMatt Macy zfs_refcount_destroy(zfs_refcount_t *rc) 108eda14cbcSMatt Macy { 109eda14cbcSMatt Macy zfs_refcount_destroy_many(rc, 0); 110eda14cbcSMatt Macy } 111eda14cbcSMatt Macy 112eda14cbcSMatt Macy int 113eda14cbcSMatt Macy zfs_refcount_is_zero(zfs_refcount_t *rc) 114eda14cbcSMatt Macy { 1152faf504dSMartin Matuska return (zfs_refcount_count(rc) == 0); 116eda14cbcSMatt Macy } 117eda14cbcSMatt Macy 118eda14cbcSMatt Macy int64_t 119eda14cbcSMatt Macy zfs_refcount_count(zfs_refcount_t *rc) 120eda14cbcSMatt Macy { 1212faf504dSMartin Matuska return (atomic_load_64(&rc->rc_count)); 122eda14cbcSMatt Macy } 123eda14cbcSMatt Macy 124eda14cbcSMatt Macy int64_t 125eda14cbcSMatt Macy zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder) 126eda14cbcSMatt Macy { 127eda14cbcSMatt Macy reference_t *ref = NULL; 128eda14cbcSMatt Macy int64_t count; 129eda14cbcSMatt Macy 1302faf504dSMartin Matuska if (!rc->rc_tracked) { 1312faf504dSMartin Matuska count = atomic_add_64_nv(&(rc)->rc_count, number); 1322faf504dSMartin Matuska ASSERT3U(count, >=, number); 1332faf504dSMartin Matuska return (count); 1342faf504dSMartin Matuska } 1352faf504dSMartin Matuska 136eda14cbcSMatt Macy ref = kmem_cache_alloc(reference_cache, KM_SLEEP); 137eda14cbcSMatt Macy ref->ref_holder = holder; 138eda14cbcSMatt Macy ref->ref_number = number; 139eda14cbcSMatt Macy mutex_enter(&rc->rc_mtx); 140eda14cbcSMatt Macy ASSERT3U(rc->rc_count, >=, 0); 141eda14cbcSMatt Macy list_insert_head(&rc->rc_list, ref); 142eda14cbcSMatt Macy rc->rc_count += number; 143eda14cbcSMatt Macy count = rc->rc_count; 144eda14cbcSMatt Macy mutex_exit(&rc->rc_mtx); 145eda14cbcSMatt Macy 146eda14cbcSMatt Macy return (count); 147eda14cbcSMatt Macy } 148eda14cbcSMatt Macy 149eda14cbcSMatt Macy int64_t 150eda14cbcSMatt Macy zfs_refcount_add(zfs_refcount_t *rc, const void *holder) 151eda14cbcSMatt Macy { 152eda14cbcSMatt Macy return (zfs_refcount_add_many(rc, 1, holder)); 153eda14cbcSMatt Macy } 154eda14cbcSMatt Macy 155eda14cbcSMatt Macy int64_t 156eda14cbcSMatt Macy zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number, 157eda14cbcSMatt Macy const void *holder) 158eda14cbcSMatt Macy { 159eda14cbcSMatt Macy reference_t *ref; 160eda14cbcSMatt Macy int64_t count; 161eda14cbcSMatt Macy 162eda14cbcSMatt Macy if (!rc->rc_tracked) { 1632faf504dSMartin Matuska count = atomic_add_64_nv(&(rc)->rc_count, -number); 1642faf504dSMartin Matuska ASSERT3S(count, >=, 0); 165eda14cbcSMatt Macy return (count); 166eda14cbcSMatt Macy } 167eda14cbcSMatt Macy 1682faf504dSMartin Matuska mutex_enter(&rc->rc_mtx); 1692faf504dSMartin Matuska ASSERT3U(rc->rc_count, >=, number); 170eda14cbcSMatt Macy for (ref = list_head(&rc->rc_list); ref; 171eda14cbcSMatt Macy ref = list_next(&rc->rc_list, ref)) { 172eda14cbcSMatt Macy if (ref->ref_holder == holder && ref->ref_number == number) { 173eda14cbcSMatt Macy list_remove(&rc->rc_list, ref); 174eda14cbcSMatt Macy if (reference_history > 0) { 175eda14cbcSMatt Macy ref->ref_removed = 176eda14cbcSMatt Macy kmem_cache_alloc(reference_history_cache, 177eda14cbcSMatt Macy KM_SLEEP); 178eda14cbcSMatt Macy list_insert_head(&rc->rc_removed, ref); 179eda14cbcSMatt Macy rc->rc_removed_count++; 180eda14cbcSMatt Macy if (rc->rc_removed_count > reference_history) { 181eda14cbcSMatt Macy ref = list_tail(&rc->rc_removed); 182eda14cbcSMatt Macy list_remove(&rc->rc_removed, ref); 183eda14cbcSMatt Macy kmem_cache_free(reference_history_cache, 184eda14cbcSMatt Macy ref->ref_removed); 185eda14cbcSMatt Macy kmem_cache_free(reference_cache, ref); 186eda14cbcSMatt Macy rc->rc_removed_count--; 187eda14cbcSMatt Macy } 188eda14cbcSMatt Macy } else { 189eda14cbcSMatt Macy kmem_cache_free(reference_cache, ref); 190eda14cbcSMatt Macy } 191eda14cbcSMatt Macy rc->rc_count -= number; 192eda14cbcSMatt Macy count = rc->rc_count; 193eda14cbcSMatt Macy mutex_exit(&rc->rc_mtx); 194eda14cbcSMatt Macy return (count); 195eda14cbcSMatt Macy } 196eda14cbcSMatt Macy } 197eda14cbcSMatt Macy panic("No such hold %p on refcount %llx", holder, 198eda14cbcSMatt Macy (u_longlong_t)(uintptr_t)rc); 199eda14cbcSMatt Macy return (-1); 200eda14cbcSMatt Macy } 201eda14cbcSMatt Macy 202eda14cbcSMatt Macy int64_t 203eda14cbcSMatt Macy zfs_refcount_remove(zfs_refcount_t *rc, const void *holder) 204eda14cbcSMatt Macy { 205eda14cbcSMatt Macy return (zfs_refcount_remove_many(rc, 1, holder)); 206eda14cbcSMatt Macy } 207eda14cbcSMatt Macy 208eda14cbcSMatt Macy void 209eda14cbcSMatt Macy zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src) 210eda14cbcSMatt Macy { 211eda14cbcSMatt Macy int64_t count, removed_count; 212eda14cbcSMatt Macy list_t list, removed; 213eda14cbcSMatt Macy 214eda14cbcSMatt Macy list_create(&list, sizeof (reference_t), 215eda14cbcSMatt Macy offsetof(reference_t, ref_link)); 216eda14cbcSMatt Macy list_create(&removed, sizeof (reference_t), 217eda14cbcSMatt Macy offsetof(reference_t, ref_link)); 218eda14cbcSMatt Macy 219eda14cbcSMatt Macy mutex_enter(&src->rc_mtx); 220eda14cbcSMatt Macy count = src->rc_count; 221eda14cbcSMatt Macy removed_count = src->rc_removed_count; 222eda14cbcSMatt Macy src->rc_count = 0; 223eda14cbcSMatt Macy src->rc_removed_count = 0; 224eda14cbcSMatt Macy list_move_tail(&list, &src->rc_list); 225eda14cbcSMatt Macy list_move_tail(&removed, &src->rc_removed); 226eda14cbcSMatt Macy mutex_exit(&src->rc_mtx); 227eda14cbcSMatt Macy 228eda14cbcSMatt Macy mutex_enter(&dst->rc_mtx); 229eda14cbcSMatt Macy dst->rc_count += count; 230eda14cbcSMatt Macy dst->rc_removed_count += removed_count; 231eda14cbcSMatt Macy list_move_tail(&dst->rc_list, &list); 232eda14cbcSMatt Macy list_move_tail(&dst->rc_removed, &removed); 233eda14cbcSMatt Macy mutex_exit(&dst->rc_mtx); 234eda14cbcSMatt Macy 235eda14cbcSMatt Macy list_destroy(&list); 236eda14cbcSMatt Macy list_destroy(&removed); 237eda14cbcSMatt Macy } 238eda14cbcSMatt Macy 239eda14cbcSMatt Macy void 240eda14cbcSMatt Macy zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number, 241eda14cbcSMatt Macy const void *current_holder, const void *new_holder) 242eda14cbcSMatt Macy { 243eda14cbcSMatt Macy reference_t *ref; 244eda14cbcSMatt Macy boolean_t found = B_FALSE; 245eda14cbcSMatt Macy 2462faf504dSMartin Matuska if (!rc->rc_tracked) 247eda14cbcSMatt Macy return; 248eda14cbcSMatt Macy 2492faf504dSMartin Matuska mutex_enter(&rc->rc_mtx); 250eda14cbcSMatt Macy for (ref = list_head(&rc->rc_list); ref; 251eda14cbcSMatt Macy ref = list_next(&rc->rc_list, ref)) { 252eda14cbcSMatt Macy if (ref->ref_holder == current_holder && 253eda14cbcSMatt Macy ref->ref_number == number) { 254eda14cbcSMatt Macy ref->ref_holder = new_holder; 255eda14cbcSMatt Macy found = B_TRUE; 256eda14cbcSMatt Macy break; 257eda14cbcSMatt Macy } 258eda14cbcSMatt Macy } 259eda14cbcSMatt Macy ASSERT(found); 260eda14cbcSMatt Macy mutex_exit(&rc->rc_mtx); 261eda14cbcSMatt Macy } 262eda14cbcSMatt Macy 263eda14cbcSMatt Macy void 264eda14cbcSMatt Macy zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder, 265eda14cbcSMatt Macy const void *new_holder) 266eda14cbcSMatt Macy { 267eda14cbcSMatt Macy return (zfs_refcount_transfer_ownership_many(rc, 1, current_holder, 268eda14cbcSMatt Macy new_holder)); 269eda14cbcSMatt Macy } 270eda14cbcSMatt Macy 271eda14cbcSMatt Macy /* 272eda14cbcSMatt Macy * If tracking is enabled, return true if a reference exists that matches 273eda14cbcSMatt Macy * the "holder" tag. If tracking is disabled, then return true if a reference 274eda14cbcSMatt Macy * might be held. 275eda14cbcSMatt Macy */ 276eda14cbcSMatt Macy boolean_t 277eda14cbcSMatt Macy zfs_refcount_held(zfs_refcount_t *rc, const void *holder) 278eda14cbcSMatt Macy { 279eda14cbcSMatt Macy reference_t *ref; 280eda14cbcSMatt Macy 2812faf504dSMartin Matuska if (!rc->rc_tracked) 2822faf504dSMartin Matuska return (zfs_refcount_count(rc) > 0); 2832faf504dSMartin Matuska 284eda14cbcSMatt Macy mutex_enter(&rc->rc_mtx); 285eda14cbcSMatt Macy for (ref = list_head(&rc->rc_list); ref; 286eda14cbcSMatt Macy ref = list_next(&rc->rc_list, ref)) { 287eda14cbcSMatt Macy if (ref->ref_holder == holder) { 288eda14cbcSMatt Macy mutex_exit(&rc->rc_mtx); 289eda14cbcSMatt Macy return (B_TRUE); 290eda14cbcSMatt Macy } 291eda14cbcSMatt Macy } 292eda14cbcSMatt Macy mutex_exit(&rc->rc_mtx); 293eda14cbcSMatt Macy return (B_FALSE); 294eda14cbcSMatt Macy } 295eda14cbcSMatt Macy 296eda14cbcSMatt Macy /* 297eda14cbcSMatt Macy * If tracking is enabled, return true if a reference does not exist that 298eda14cbcSMatt Macy * matches the "holder" tag. If tracking is disabled, always return true 299eda14cbcSMatt Macy * since the reference might not be held. 300eda14cbcSMatt Macy */ 301eda14cbcSMatt Macy boolean_t 302eda14cbcSMatt Macy zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder) 303eda14cbcSMatt Macy { 304eda14cbcSMatt Macy reference_t *ref; 305eda14cbcSMatt Macy 3062faf504dSMartin Matuska if (!rc->rc_tracked) 307eda14cbcSMatt Macy return (B_TRUE); 308eda14cbcSMatt Macy 3092faf504dSMartin Matuska mutex_enter(&rc->rc_mtx); 310eda14cbcSMatt Macy for (ref = list_head(&rc->rc_list); ref; 311eda14cbcSMatt Macy ref = list_next(&rc->rc_list, ref)) { 312eda14cbcSMatt Macy if (ref->ref_holder == holder) { 313eda14cbcSMatt Macy mutex_exit(&rc->rc_mtx); 314eda14cbcSMatt Macy return (B_FALSE); 315eda14cbcSMatt Macy } 316eda14cbcSMatt Macy } 317eda14cbcSMatt Macy mutex_exit(&rc->rc_mtx); 318eda14cbcSMatt Macy return (B_TRUE); 319eda14cbcSMatt Macy } 320f9693befSMartin Matuska 321*6ba2210eSMartin Matuska EXPORT_SYMBOL(zfs_refcount_create); 322*6ba2210eSMartin Matuska EXPORT_SYMBOL(zfs_refcount_destroy); 323*6ba2210eSMartin Matuska EXPORT_SYMBOL(zfs_refcount_is_zero); 324*6ba2210eSMartin Matuska EXPORT_SYMBOL(zfs_refcount_count); 325*6ba2210eSMartin Matuska EXPORT_SYMBOL(zfs_refcount_add); 326*6ba2210eSMartin Matuska EXPORT_SYMBOL(zfs_refcount_remove); 327*6ba2210eSMartin Matuska EXPORT_SYMBOL(zfs_refcount_held); 328*6ba2210eSMartin Matuska 329f9693befSMartin Matuska /* BEGIN CSTYLED */ 330f9693befSMartin Matuska ZFS_MODULE_PARAM(zfs, ,reference_tracking_enable, INT, ZMOD_RW, 331f9693befSMartin Matuska "Track reference holders to refcount_t objects"); 332f9693befSMartin Matuska 333f9693befSMartin Matuska ZFS_MODULE_PARAM(zfs, ,reference_history, INT, ZMOD_RW, 334f9693befSMartin Matuska "Maximum reference holders being tracked"); 335f9693befSMartin Matuska /* END CSTYLED */ 336eda14cbcSMatt Macy #endif /* ZFS_DEBUG */ 337