1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2005 John Baldwin <jhb@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30 #ifndef __SYS_REFCOUNT_H__ 31 #define __SYS_REFCOUNT_H__ 32 33 #include <machine/atomic.h> 34 35 #ifdef _KERNEL 36 #include <sys/systm.h> 37 #else 38 #include <stdbool.h> 39 #define KASSERT(exp, msg) /* */ 40 #endif 41 42 #define REFCOUNT_WAITER (1 << 31) /* Refcount has waiter. */ 43 #define REFCOUNT_SATURATION_VALUE (3U << 29) 44 45 #define REFCOUNT_SATURATED(val) (((val) & (1U << 30)) != 0) 46 #define REFCOUNT_COUNT(x) ((x) & ~REFCOUNT_WAITER) 47 48 bool refcount_release_last(volatile u_int *count, u_int n, u_int old); 49 void refcount_sleep(volatile u_int *count, const char *wmesg, int prio); 50 51 /* 52 * Attempt to handle reference count overflow and underflow. Force the counter 53 * to stay at the saturation value so that a counter overflow cannot trigger 54 * destruction of the containing object and instead leads to a less harmful 55 * memory leak. 56 */ 57 static __inline void 58 _refcount_update_saturated(volatile u_int *count) 59 { 60 #ifdef INVARIANTS 61 panic("refcount %p wraparound", count); 62 #else 63 atomic_store_int(count, REFCOUNT_SATURATION_VALUE); 64 #endif 65 } 66 67 static __inline void 68 refcount_init(volatile u_int *count, u_int value) 69 { 70 KASSERT(!REFCOUNT_SATURATED(value), 71 ("invalid initial refcount value %u", value)); 72 *count = value; 73 } 74 75 static __inline void 76 refcount_acquire(volatile u_int *count) 77 { 78 u_int old; 79 80 old = atomic_fetchadd_int(count, 1); 81 if (__predict_false(REFCOUNT_SATURATED(old))) 82 _refcount_update_saturated(count); 83 } 84 85 static __inline void 86 refcount_acquiren(volatile u_int *count, u_int n) 87 { 88 89 u_int old; 90 91 KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, 92 ("refcount_acquiren: n %d too large", n)); 93 old = atomic_fetchadd_int(count, n); 94 if (__predict_false(REFCOUNT_SATURATED(old))) 95 _refcount_update_saturated(count); 96 } 97 98 static __inline __result_use_check bool 99 refcount_acquire_checked(volatile u_int *count) 100 { 101 u_int lcount; 102 103 for (lcount = *count;;) { 104 if (__predict_false(REFCOUNT_SATURATED(lcount + 1))) 105 return (false); 106 if (__predict_true(atomic_fcmpset_int(count, &lcount, 107 lcount + 1) == 1)) 108 return (true); 109 } 110 } 111 112 static __inline bool 113 refcount_releasen(volatile u_int *count, u_int n) 114 { 115 u_int old; 116 117 KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, 118 ("refcount_releasen: n %d too large", n)); 119 atomic_thread_fence_rel(); 120 old = atomic_fetchadd_int(count, -n); 121 if (__predict_false(n >= REFCOUNT_COUNT(old) || 122 REFCOUNT_SATURATED(old))) 123 return (refcount_release_last(count, n, old)); 124 return (false); 125 } 126 127 static __inline bool 128 refcount_release(volatile u_int *count) 129 { 130 131 return (refcount_releasen(count, 1)); 132 } 133 134 static __inline void 135 refcount_wait(volatile u_int *count, const char *wmesg, int prio) 136 { 137 138 while (*count != 0) 139 refcount_sleep(count, wmesg, prio); 140 } 141 142 /* 143 * This functions returns non-zero if the refcount was 144 * incremented. Else zero is returned. 145 */ 146 static __inline __result_use_check bool 147 refcount_acquire_if_not_zero(volatile u_int *count) 148 { 149 u_int old; 150 151 old = *count; 152 for (;;) { 153 if (REFCOUNT_COUNT(old) == 0) 154 return (false); 155 if (__predict_false(REFCOUNT_SATURATED(old))) 156 return (true); 157 if (atomic_fcmpset_int(count, &old, old + 1)) 158 return (true); 159 } 160 } 161 162 static __inline __result_use_check bool 163 refcount_release_if_not_last(volatile u_int *count) 164 { 165 u_int old; 166 167 old = *count; 168 for (;;) { 169 if (REFCOUNT_COUNT(old) == 1) 170 return (false); 171 if (__predict_false(REFCOUNT_SATURATED(old))) 172 return (true); 173 if (atomic_fcmpset_int(count, &old, old - 1)) 174 return (true); 175 } 176 } 177 178 #endif /* ! __SYS_REFCOUNT_H__ */ 179