1d6fe50b6SJohn Baldwin /*-
24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause
3c4e20cadSPedro F. Giffuni *
4d6fe50b6SJohn Baldwin * Copyright (c) 2005 John Baldwin <jhb@FreeBSD.org>
5d6fe50b6SJohn Baldwin *
6d6fe50b6SJohn Baldwin * Redistribution and use in source and binary forms, with or without
7d6fe50b6SJohn Baldwin * modification, are permitted provided that the following conditions
8d6fe50b6SJohn Baldwin * are met:
9d6fe50b6SJohn Baldwin * 1. Redistributions of source code must retain the above copyright
10d6fe50b6SJohn Baldwin * notice, this list of conditions and the following disclaimer.
11d6fe50b6SJohn Baldwin * 2. Redistributions in binary form must reproduce the above copyright
12d6fe50b6SJohn Baldwin * notice, this list of conditions and the following disclaimer in the
13d6fe50b6SJohn Baldwin * documentation and/or other materials provided with the distribution.
14d6fe50b6SJohn Baldwin *
15d6fe50b6SJohn Baldwin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16d6fe50b6SJohn Baldwin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17d6fe50b6SJohn Baldwin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18d6fe50b6SJohn Baldwin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19d6fe50b6SJohn Baldwin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20d6fe50b6SJohn Baldwin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21d6fe50b6SJohn Baldwin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22d6fe50b6SJohn Baldwin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23d6fe50b6SJohn Baldwin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24d6fe50b6SJohn Baldwin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25d6fe50b6SJohn Baldwin * SUCH DAMAGE.
26d6fe50b6SJohn Baldwin */
27d6fe50b6SJohn Baldwin
28d6fe50b6SJohn Baldwin #ifndef __SYS_REFCOUNT_H__
29d6fe50b6SJohn Baldwin #define __SYS_REFCOUNT_H__
30d6fe50b6SJohn Baldwin
3175f31a5fSDag-Erling Smørgrav #include <machine/atomic.h>
3275f31a5fSDag-Erling Smørgrav
33ea37efb7SHans Petter Selasky #if defined(_KERNEL) || defined(_STANDALONE)
3475f31a5fSDag-Erling Smørgrav #include <sys/systm.h>
3575f31a5fSDag-Erling Smørgrav #else
3610040398SKonstantin Belousov #include <stdbool.h>
3775f31a5fSDag-Erling Smørgrav #define KASSERT(exp, msg) /* */
3875f31a5fSDag-Erling Smørgrav #endif
3975f31a5fSDag-Erling Smørgrav
40c99d0c58SMark Johnston #define REFCOUNT_SATURATED(val) (((val) & (1U << 31)) != 0)
41c99d0c58SMark Johnston #define REFCOUNT_SATURATION_VALUE (3U << 30)
420b21d894SMark Johnston
430b21d894SMark Johnston /*
440b21d894SMark Johnston * Attempt to handle reference count overflow and underflow. Force the counter
450b21d894SMark Johnston * to stay at the saturation value so that a counter overflow cannot trigger
460b21d894SMark Johnston * destruction of the containing object and instead leads to a less harmful
470b21d894SMark Johnston * memory leak.
480b21d894SMark Johnston */
490b21d894SMark Johnston static __inline void
_refcount_update_saturated(volatile u_int * count)500b21d894SMark Johnston _refcount_update_saturated(volatile u_int *count)
510b21d894SMark Johnston {
520b21d894SMark Johnston #ifdef INVARIANTS
530b21d894SMark Johnston panic("refcount %p wraparound", count);
540b21d894SMark Johnston #else
550b21d894SMark Johnston atomic_store_int(count, REFCOUNT_SATURATION_VALUE);
560b21d894SMark Johnston #endif
570b21d894SMark Johnston }
580b21d894SMark Johnston
59d6fe50b6SJohn Baldwin static __inline void
refcount_init(volatile u_int * count,u_int value)60d6fe50b6SJohn Baldwin refcount_init(volatile u_int *count, u_int value)
61d6fe50b6SJohn Baldwin {
620b21d894SMark Johnston KASSERT(!REFCOUNT_SATURATED(value),
630b21d894SMark Johnston ("invalid initial refcount value %u", value));
641a297ee5SMateusz Guzik atomic_store_int(count, value);
65d6fe50b6SJohn Baldwin }
66d6fe50b6SJohn Baldwin
67a67d5408SJeff Roberson static __inline u_int
refcount_load(volatile u_int * count)68e8900461SMark Johnston refcount_load(volatile u_int *count)
69e8900461SMark Johnston {
70e8900461SMark Johnston return (atomic_load_int(count));
71e8900461SMark Johnston }
72e8900461SMark Johnston
73e8900461SMark Johnston static __inline u_int
refcount_acquire(volatile u_int * count)74d6fe50b6SJohn Baldwin refcount_acquire(volatile u_int *count)
75d6fe50b6SJohn Baldwin {
760b21d894SMark Johnston u_int old;
77d6fe50b6SJohn Baldwin
780b21d894SMark Johnston old = atomic_fetchadd_int(count, 1);
790b21d894SMark Johnston if (__predict_false(REFCOUNT_SATURATED(old)))
800b21d894SMark Johnston _refcount_update_saturated(count);
81a67d5408SJeff Roberson
82a67d5408SJeff Roberson return (old);
83d6fe50b6SJohn Baldwin }
84d6fe50b6SJohn Baldwin
85a67d5408SJeff Roberson static __inline u_int
refcount_acquiren(volatile u_int * count,u_int n)8633205c60SJeff Roberson refcount_acquiren(volatile u_int *count, u_int n)
8733205c60SJeff Roberson {
8833205c60SJeff Roberson u_int old;
8933205c60SJeff Roberson
9033205c60SJeff Roberson KASSERT(n < REFCOUNT_SATURATION_VALUE / 2,
9140617291SHans Petter Selasky ("refcount_acquiren: n=%u too large", n));
9233205c60SJeff Roberson old = atomic_fetchadd_int(count, n);
9333205c60SJeff Roberson if (__predict_false(REFCOUNT_SATURATED(old)))
9433205c60SJeff Roberson _refcount_update_saturated(count);
95a67d5408SJeff Roberson
96a67d5408SJeff Roberson return (old);
9733205c60SJeff Roberson }
9833205c60SJeff Roberson
99f1cf2b9dSKonstantin Belousov static __inline __result_use_check bool
refcount_acquire_checked(volatile u_int * count)100f1cf2b9dSKonstantin Belousov refcount_acquire_checked(volatile u_int *count)
101f1cf2b9dSKonstantin Belousov {
1021a297ee5SMateusz Guzik u_int old;
103f1cf2b9dSKonstantin Belousov
1041a297ee5SMateusz Guzik old = atomic_load_int(count);
1051a297ee5SMateusz Guzik for (;;) {
1061a297ee5SMateusz Guzik if (__predict_false(REFCOUNT_SATURATED(old + 1)))
107f1cf2b9dSKonstantin Belousov return (false);
1081a297ee5SMateusz Guzik if (__predict_true(atomic_fcmpset_int(count, &old,
1091a297ee5SMateusz Guzik old + 1) == 1))
110f1cf2b9dSKonstantin Belousov return (true);
111f1cf2b9dSKonstantin Belousov }
112f1cf2b9dSKonstantin Belousov }
113f1cf2b9dSKonstantin Belousov
114f4043145SAndriy Gapon /*
115127a9d73SHans Petter Selasky * This functions returns non-zero if the refcount was
116127a9d73SHans Petter Selasky * incremented. Else zero is returned.
117f4043145SAndriy Gapon */
11813ff4eb1SKonstantin Belousov static __inline __result_use_check bool
refcount_acquire_if_gt(volatile u_int * count,u_int n)119a67d5408SJeff Roberson refcount_acquire_if_gt(volatile u_int *count, u_int n)
120f4043145SAndriy Gapon {
121f4043145SAndriy Gapon u_int old;
122f4043145SAndriy Gapon
1231a297ee5SMateusz Guzik old = atomic_load_int(count);
124f4043145SAndriy Gapon for (;;) {
125c99d0c58SMark Johnston if (old <= n)
12613ff4eb1SKonstantin Belousov return (false);
1270b21d894SMark Johnston if (__predict_false(REFCOUNT_SATURATED(old)))
1280b21d894SMark Johnston return (true);
129f4043145SAndriy Gapon if (atomic_fcmpset_int(count, &old, old + 1))
13013ff4eb1SKonstantin Belousov return (true);
131f4043145SAndriy Gapon }
132f4043145SAndriy Gapon }
133f4043145SAndriy Gapon
13413ff4eb1SKonstantin Belousov static __inline __result_use_check bool
refcount_acquire_if_not_zero(volatile u_int * count)135a67d5408SJeff Roberson refcount_acquire_if_not_zero(volatile u_int *count)
136f4043145SAndriy Gapon {
137f4043145SAndriy Gapon
138c99d0c58SMark Johnston return (refcount_acquire_if_gt(count, 0));
139c99d0c58SMark Johnston }
140c99d0c58SMark Johnston
141c99d0c58SMark Johnston static __inline bool
refcount_releasen(volatile u_int * count,u_int n)142c99d0c58SMark Johnston refcount_releasen(volatile u_int *count, u_int n)
143c99d0c58SMark Johnston {
144c99d0c58SMark Johnston u_int old;
145c99d0c58SMark Johnston
146c99d0c58SMark Johnston KASSERT(n < REFCOUNT_SATURATION_VALUE / 2,
147c99d0c58SMark Johnston ("refcount_releasen: n=%u too large", n));
148c99d0c58SMark Johnston
149c99d0c58SMark Johnston atomic_thread_fence_rel();
150c99d0c58SMark Johnston old = atomic_fetchadd_int(count, -n);
151c99d0c58SMark Johnston if (__predict_false(old < n || REFCOUNT_SATURATED(old))) {
152c99d0c58SMark Johnston _refcount_update_saturated(count);
153c99d0c58SMark Johnston return (false);
154c99d0c58SMark Johnston }
155c99d0c58SMark Johnston if (old > n)
156c99d0c58SMark Johnston return (false);
157c99d0c58SMark Johnston
158c99d0c58SMark Johnston /*
159c99d0c58SMark Johnston * Last reference. Signal the user to call the destructor.
160c99d0c58SMark Johnston *
161c99d0c58SMark Johnston * Ensure that the destructor sees all updates. This synchronizes with
162c99d0c58SMark Johnston * release fences from all routines which drop the count.
163c99d0c58SMark Johnston */
164c99d0c58SMark Johnston atomic_thread_fence_acq();
165c99d0c58SMark Johnston return (true);
166c99d0c58SMark Johnston }
167c99d0c58SMark Johnston
168c99d0c58SMark Johnston static __inline bool
refcount_release(volatile u_int * count)169c99d0c58SMark Johnston refcount_release(volatile u_int *count)
170c99d0c58SMark Johnston {
171c99d0c58SMark Johnston
172c99d0c58SMark Johnston return (refcount_releasen(count, 1));
173f4043145SAndriy Gapon }
174f4043145SAndriy Gapon
175e8900461SMark Johnston #define _refcount_release_if_cond(cond, name) \
176e8900461SMark Johnston static __inline __result_use_check bool \
177e8900461SMark Johnston _refcount_release_if_##name(volatile u_int *count, u_int n) \
178e8900461SMark Johnston { \
179e8900461SMark Johnston u_int old; \
180e8900461SMark Johnston \
181e8900461SMark Johnston KASSERT(n > 0, ("%s: zero increment", __func__)); \
182e8900461SMark Johnston old = atomic_load_int(count); \
183e8900461SMark Johnston for (;;) { \
184e8900461SMark Johnston if (!(cond)) \
185e8900461SMark Johnston return (false); \
186e8900461SMark Johnston if (__predict_false(REFCOUNT_SATURATED(old))) \
187e8900461SMark Johnston return (false); \
188e8900461SMark Johnston if (atomic_fcmpset_rel_int(count, &old, old - 1)) \
189e8900461SMark Johnston return (true); \
190e8900461SMark Johnston } \
191e8900461SMark Johnston }
192e8900461SMark Johnston _refcount_release_if_cond(old > n, gt)
193e8900461SMark Johnston _refcount_release_if_cond(old == n, eq)
194e8900461SMark Johnston
19551df5321SJeff Roberson static __inline __result_use_check bool
refcount_release_if_gt(volatile u_int * count,u_int n)19651df5321SJeff Roberson refcount_release_if_gt(volatile u_int *count, u_int n)
19751df5321SJeff Roberson {
19851df5321SJeff Roberson
199e8900461SMark Johnston return (_refcount_release_if_gt(count, n));
200e8900461SMark Johnston }
201e8900461SMark Johnston
202e8900461SMark Johnston static __inline __result_use_check bool
refcount_release_if_last(volatile u_int * count)203e8900461SMark Johnston refcount_release_if_last(volatile u_int *count)
204e8900461SMark Johnston {
205e8900461SMark Johnston
206e8900461SMark Johnston if (_refcount_release_if_eq(count, 1)) {
207e8900461SMark Johnston /* See the comment in refcount_releasen(). */
208e8900461SMark Johnston atomic_thread_fence_acq();
20951df5321SJeff Roberson return (true);
21051df5321SJeff Roberson }
211e8900461SMark Johnston return (false);
21251df5321SJeff Roberson }
21351df5321SJeff Roberson
214a67d5408SJeff Roberson static __inline __result_use_check bool
refcount_release_if_not_last(volatile u_int * count)215a67d5408SJeff Roberson refcount_release_if_not_last(volatile u_int *count)
216a67d5408SJeff Roberson {
217a67d5408SJeff Roberson
218e8900461SMark Johnston return (_refcount_release_if_gt(count, 1));
219a67d5408SJeff Roberson }
220c99d0c58SMark Johnston
221d6fe50b6SJohn Baldwin #endif /* !__SYS_REFCOUNT_H__ */
222