1c6d74635Srmind /*-
2*d6939920Srmind * Copyright (c) 2010-2020 The NetBSD Foundation, Inc.
3c6d74635Srmind * All rights reserved.
4c6d74635Srmind *
5c6d74635Srmind * This material is based upon work partially supported by The
6c6d74635Srmind * NetBSD Foundation under a contract with Mindaugas Rasiukevicius.
7c6d74635Srmind *
8c6d74635Srmind * Redistribution and use in source and binary forms, with or without
9c6d74635Srmind * modification, are permitted provided that the following conditions
10c6d74635Srmind * are met:
11c6d74635Srmind * 1. Redistributions of source code must retain the above copyright
12c6d74635Srmind * notice, this list of conditions and the following disclaimer.
13c6d74635Srmind * 2. Redistributions in binary form must reproduce the above copyright
14c6d74635Srmind * notice, this list of conditions and the following disclaimer in the
15c6d74635Srmind * documentation and/or other materials provided with the distribution.
16c6d74635Srmind *
17c6d74635Srmind * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
18c6d74635Srmind * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19c6d74635Srmind * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20c6d74635Srmind * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
21c6d74635Srmind * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22c6d74635Srmind * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23c6d74635Srmind * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24c6d74635Srmind * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25c6d74635Srmind * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26c6d74635Srmind * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27c6d74635Srmind * POSSIBILITY OF SUCH DAMAGE.
28c6d74635Srmind */
29c6d74635Srmind
30c6d74635Srmind /*
31c6d74635Srmind * NPF connection storage.
327e3fb338Srmind *
33*d6939920Srmind * Lock-free connection lookups are protected by EBR with an atomic
34*d6939920Srmind * reference acquisition before exiting the critical path. The caller
35*d6939920Srmind * is responsible for re-checking the connection state.
36*d6939920Srmind *
377e3fb338Srmind * Warning (not applicable for the userspace npfkern):
387e3fb338Srmind *
397e3fb338Srmind * thmap is partially lock-free data structure that uses its own
407e3fb338Srmind * spin-locks on the writer side (insert/delete operations).
417e3fb338Srmind *
427e3fb338Srmind * The relevant interrupt priority level (IPL) must be set and the
437e3fb338Srmind * kernel preemption disabled across the critical paths to prevent
447e3fb338Srmind * deadlocks and priority inversion problems. These are essentially
457e3fb338Srmind * the same guarantees as a spinning mutex(9) would provide.
467e3fb338Srmind *
477e3fb338Srmind * This is achieved with SPL routines splsoftnet() and splx() around
48*d6939920Srmind * the thmap_del() and thmap_put() calls. Note: we assume that the
49*d6939920Srmind * network stack invokes NPF at IPL_SOFTNET or lower, but not higher.
50c6d74635Srmind */
51c6d74635Srmind
520473fe8bSchristos #ifdef _KERNEL
53c6d74635Srmind #include <sys/cdefs.h>
54*d6939920Srmind __KERNEL_RCSID(0, "$NetBSD: npf_conndb.c,v 1.9 2020/05/30 14:16:56 rmind Exp $");
55c6d74635Srmind
56c6d74635Srmind #include <sys/param.h>
57c6d74635Srmind #include <sys/types.h>
58c6d74635Srmind
59c6d74635Srmind #include <sys/atomic.h>
60c6d74635Srmind #include <sys/kmem.h>
616f4ca96cSrmind #include <sys/thmap.h>
620473fe8bSchristos #endif
63c6d74635Srmind
64c6d74635Srmind #define __NPF_CONN_PRIVATE
65c6d74635Srmind #include "npf_conn.h"
66c6d74635Srmind #include "npf_impl.h"
67c6d74635Srmind
68c6d74635Srmind struct npf_conndb {
696f4ca96cSrmind thmap_t * cd_map;
70c6d74635Srmind
71c6d74635Srmind /*
726f4ca96cSrmind * There are three lists for connections: new, all and G/C.
736f4ca96cSrmind *
746f4ca96cSrmind * New connections are atomically inserted into the "new-list".
756f4ca96cSrmind * The G/C worker will move them to the doubly-linked list of all
766f4ca96cSrmind * active connections.
77c6d74635Srmind */
786f4ca96cSrmind npf_conn_t * cd_new;
796f4ca96cSrmind LIST_HEAD(, npf_conn) cd_list;
806f4ca96cSrmind LIST_HEAD(, npf_conn) cd_gclist;
81c6d74635Srmind
826f4ca96cSrmind /* The last inspected connection (for circular iteration). */
836f4ca96cSrmind npf_conn_t * cd_marker;
84c6d74635Srmind };
85c6d74635Srmind
867e3fb338Srmind typedef struct {
87*d6939920Srmind int step;
88*d6939920Srmind int interval_min;
89*d6939920Srmind int interval_max;
907e3fb338Srmind } npf_conndb_params_t;
917e3fb338Srmind
927e3fb338Srmind /*
937e3fb338Srmind * Pointer tag for connection keys which represent the "forwards" entry.
947e3fb338Srmind */
957e3fb338Srmind #define CONNDB_FORW_BIT ((uintptr_t)0x1)
967e3fb338Srmind #define CONNDB_ISFORW_P(p) (((uintptr_t)(p) & CONNDB_FORW_BIT) != 0)
977e3fb338Srmind #define CONNDB_GET_PTR(p) ((void *)((uintptr_t)(p) & ~CONNDB_FORW_BIT))
987e3fb338Srmind
997e3fb338Srmind void
npf_conndb_sysinit(npf_t * npf)1007e3fb338Srmind npf_conndb_sysinit(npf_t *npf)
1017e3fb338Srmind {
1027e3fb338Srmind npf_conndb_params_t *params = npf_param_allocgroup(npf,
1037e3fb338Srmind NPF_PARAMS_CONNDB, sizeof(npf_conndb_params_t));
1047e3fb338Srmind npf_param_t param_map[] = {
1057e3fb338Srmind {
1067e3fb338Srmind "gc.step",
107*d6939920Srmind ¶ms->step,
1087e3fb338Srmind .default_val = 256,
1097e3fb338Srmind .min = 1, .max = INT_MAX
110*d6939920Srmind },
111*d6939920Srmind {
112*d6939920Srmind "gc.interval_min",
113*d6939920Srmind ¶ms->interval_min,
114*d6939920Srmind .default_val = 50, // ms
115*d6939920Srmind .min = 10, .max = 10000
116*d6939920Srmind },
117*d6939920Srmind {
118*d6939920Srmind "gc.interval_max",
119*d6939920Srmind ¶ms->interval_max,
120*d6939920Srmind .default_val = 5000, // ms
121*d6939920Srmind .min = 10, .max = 10000
122*d6939920Srmind },
1237e3fb338Srmind };
1247e3fb338Srmind npf_param_register(npf, param_map, __arraycount(param_map));
1257e3fb338Srmind }
1267e3fb338Srmind
1277e3fb338Srmind void
npf_conndb_sysfini(npf_t * npf)1287e3fb338Srmind npf_conndb_sysfini(npf_t *npf)
1297e3fb338Srmind {
1307e3fb338Srmind const size_t len = sizeof(npf_conndb_params_t);
1317e3fb338Srmind npf_param_freegroup(npf, NPF_PARAMS_CONNDB, len);
1327e3fb338Srmind }
1337e3fb338Srmind
134c6d74635Srmind npf_conndb_t *
npf_conndb_create(void)135c6d74635Srmind npf_conndb_create(void)
136c6d74635Srmind {
137c6d74635Srmind npf_conndb_t *cd;
138c6d74635Srmind
1396f4ca96cSrmind cd = kmem_zalloc(sizeof(npf_conndb_t), KM_SLEEP);
1406f4ca96cSrmind cd->cd_map = thmap_create(0, NULL, THMAP_NOCOPY);
1416f4ca96cSrmind KASSERT(cd->cd_map != NULL);
142c6d74635Srmind
1436f4ca96cSrmind LIST_INIT(&cd->cd_list);
1446f4ca96cSrmind LIST_INIT(&cd->cd_gclist);
145c6d74635Srmind return cd;
146c6d74635Srmind }
147c6d74635Srmind
148c6d74635Srmind void
npf_conndb_destroy(npf_conndb_t * cd)149c6d74635Srmind npf_conndb_destroy(npf_conndb_t *cd)
150c6d74635Srmind {
1516f4ca96cSrmind KASSERT(cd->cd_new == NULL);
1526f4ca96cSrmind KASSERT(cd->cd_marker == NULL);
1536f4ca96cSrmind KASSERT(LIST_EMPTY(&cd->cd_list));
1546f4ca96cSrmind KASSERT(LIST_EMPTY(&cd->cd_gclist));
155c6d74635Srmind
1566f4ca96cSrmind thmap_destroy(cd->cd_map);
1576f4ca96cSrmind kmem_free(cd, sizeof(npf_conndb_t));
158c6d74635Srmind }
159c6d74635Srmind
160c6d74635Srmind /*
161c6d74635Srmind * npf_conndb_lookup: find a connection given the key.
162c6d74635Srmind */
163c6d74635Srmind npf_conn_t *
npf_conndb_lookup(npf_t * npf,const npf_connkey_t * ck,npf_flow_t * flow)164*d6939920Srmind npf_conndb_lookup(npf_t *npf, const npf_connkey_t *ck, npf_flow_t *flow)
165c6d74635Srmind {
1665c63a085Srmind npf_conndb_t *cd = atomic_load_relaxed(&npf->conn_db);
1677e3fb338Srmind const unsigned keylen = NPF_CONNKEY_LEN(ck);
168c6d74635Srmind npf_conn_t *con;
1697e3fb338Srmind void *val;
170c6d74635Srmind
1716f4ca96cSrmind /*
1726f4ca96cSrmind * Lookup the connection key in the key-value map.
1736f4ca96cSrmind */
1745c63a085Srmind int s = npf_config_read_enter(npf);
1757e3fb338Srmind val = thmap_get(cd->cd_map, ck->ck_key, keylen);
1767e3fb338Srmind if (!val) {
1775c63a085Srmind npf_config_read_exit(npf, s);
178c6d74635Srmind return NULL;
179c6d74635Srmind }
1807e3fb338Srmind
1817e3fb338Srmind /*
1827e3fb338Srmind * Determine whether this is the "forwards" or "backwards" key
1837e3fb338Srmind * and clear the pointer tag.
1847e3fb338Srmind */
185*d6939920Srmind *flow = CONNDB_ISFORW_P(val) ? NPF_FLOW_FORW : NPF_FLOW_BACK;
1867e3fb338Srmind con = CONNDB_GET_PTR(val);
1876f4ca96cSrmind KASSERT(con != NULL);
188c6d74635Srmind
1896f4ca96cSrmind /*
1906f4ca96cSrmind * Acquire a reference and return the connection.
1916f4ca96cSrmind */
192c6d74635Srmind atomic_inc_uint(&con->c_refcnt);
1935c63a085Srmind npf_config_read_exit(npf, s);
194c6d74635Srmind return con;
195c6d74635Srmind }
196c6d74635Srmind
197c6d74635Srmind /*
198c6d74635Srmind * npf_conndb_insert: insert the key representing the connection.
1996f4ca96cSrmind *
2006f4ca96cSrmind * => Returns true on success and false on failure.
201c6d74635Srmind */
202c6d74635Srmind bool
npf_conndb_insert(npf_conndb_t * cd,const npf_connkey_t * ck,npf_conn_t * con,npf_flow_t flow)2037e3fb338Srmind npf_conndb_insert(npf_conndb_t *cd, const npf_connkey_t *ck,
204*d6939920Srmind npf_conn_t *con, npf_flow_t flow)
205c6d74635Srmind {
2067e3fb338Srmind const unsigned keylen = NPF_CONNKEY_LEN(ck);
207*d6939920Srmind const uintptr_t tag = (CONNDB_FORW_BIT * !flow);
2087e3fb338Srmind void *val;
2097e3fb338Srmind bool ok;
2107e3fb338Srmind
2117e3fb338Srmind /*
2127e3fb338Srmind * Tag the connection pointer if this is the "forwards" key.
2137e3fb338Srmind */
2147e3fb338Srmind KASSERT(!CONNDB_ISFORW_P(con));
2157e3fb338Srmind val = (void *)((uintptr_t)(void *)con | tag);
2167e3fb338Srmind
2177e3fb338Srmind int s = splsoftnet();
2187e3fb338Srmind ok = thmap_put(cd->cd_map, ck->ck_key, keylen, val) == val;
2197e3fb338Srmind splx(s);
2207e3fb338Srmind
2217e3fb338Srmind return ok;
222c6d74635Srmind }
223c6d74635Srmind
224c6d74635Srmind /*
2256f4ca96cSrmind * npf_conndb_remove: find and delete connection key, returning the
2266f4ca96cSrmind * connection it represents.
227c6d74635Srmind */
228c6d74635Srmind npf_conn_t *
npf_conndb_remove(npf_conndb_t * cd,npf_connkey_t * ck)2296f4ca96cSrmind npf_conndb_remove(npf_conndb_t *cd, npf_connkey_t *ck)
230c6d74635Srmind {
2317e3fb338Srmind const unsigned keylen = NPF_CONNKEY_LEN(ck);
2327e3fb338Srmind void *val;
233c6d74635Srmind
2347e3fb338Srmind int s = splsoftnet();
2357e3fb338Srmind val = thmap_del(cd->cd_map, ck->ck_key, keylen);
2367e3fb338Srmind splx(s);
2377e3fb338Srmind
2387e3fb338Srmind return CONNDB_GET_PTR(val);
239c6d74635Srmind }
240c6d74635Srmind
241c6d74635Srmind /*
242c6d74635Srmind * npf_conndb_enqueue: atomically insert the connection into the
2436f4ca96cSrmind * singly-linked list of the "new" connections.
244c6d74635Srmind */
245c6d74635Srmind void
npf_conndb_enqueue(npf_conndb_t * cd,npf_conn_t * con)246c6d74635Srmind npf_conndb_enqueue(npf_conndb_t *cd, npf_conn_t *con)
247c6d74635Srmind {
248c6d74635Srmind npf_conn_t *head;
249c6d74635Srmind
250c6d74635Srmind do {
251*d6939920Srmind head = atomic_load_relaxed(&cd->cd_new);
252*d6939920Srmind atomic_store_relaxed(&con->c_next, head);
2536f4ca96cSrmind } while (atomic_cas_ptr(&cd->cd_new, head, con) != head);
254c6d74635Srmind }
255c6d74635Srmind
256c6d74635Srmind /*
2576f4ca96cSrmind * npf_conndb_update: migrate all new connections to the list of all
2586f4ca96cSrmind * connections; this must also be performed on npf_conndb_getlist()
2596f4ca96cSrmind * to provide a complete list of connections.
260c6d74635Srmind */
2616f4ca96cSrmind static void
npf_conndb_update(npf_conndb_t * cd)2626f4ca96cSrmind npf_conndb_update(npf_conndb_t *cd)
263c6d74635Srmind {
2646f4ca96cSrmind npf_conn_t *con;
2656f4ca96cSrmind
2666f4ca96cSrmind con = atomic_swap_ptr(&cd->cd_new, NULL);
2676f4ca96cSrmind while (con) {
268*d6939920Srmind npf_conn_t *next = atomic_load_relaxed(&con->c_next); // union
2696f4ca96cSrmind LIST_INSERT_HEAD(&cd->cd_list, con, c_entry);
2706f4ca96cSrmind con = next;
271c6d74635Srmind }
272c6d74635Srmind }
273c6d74635Srmind
274c6d74635Srmind /*
2756f4ca96cSrmind * npf_conndb_getlist: return the list of all connections.
276c6d74635Srmind */
277c6d74635Srmind npf_conn_t *
npf_conndb_getlist(npf_conndb_t * cd)278c6d74635Srmind npf_conndb_getlist(npf_conndb_t *cd)
279c6d74635Srmind {
2806f4ca96cSrmind npf_conndb_update(cd);
2816f4ca96cSrmind return LIST_FIRST(&cd->cd_list);
282c6d74635Srmind }
283c6d74635Srmind
284c6d74635Srmind /*
2856f4ca96cSrmind * npf_conndb_getnext: return the next connection, implementing
2866f4ca96cSrmind * the circular iteration.
2876f4ca96cSrmind */
2886f4ca96cSrmind npf_conn_t *
npf_conndb_getnext(npf_conndb_t * cd,npf_conn_t * con)2896f4ca96cSrmind npf_conndb_getnext(npf_conndb_t *cd, npf_conn_t *con)
2906f4ca96cSrmind {
2916f4ca96cSrmind /* Next.. */
2926f4ca96cSrmind if (con == NULL || (con = LIST_NEXT(con, c_entry)) == NULL) {
2936f4ca96cSrmind con = LIST_FIRST(&cd->cd_list);
2946f4ca96cSrmind }
2956f4ca96cSrmind return con;
2966f4ca96cSrmind }
2976f4ca96cSrmind
2986f4ca96cSrmind /*
2996f4ca96cSrmind * npf_conndb_gc_incr: incremental G/C of the expired connections.
3006f4ca96cSrmind */
301*d6939920Srmind static unsigned
npf_conndb_gc_incr(npf_t * npf,npf_conndb_t * cd,const time_t now)3027e3fb338Srmind npf_conndb_gc_incr(npf_t *npf, npf_conndb_t *cd, const time_t now)
3036f4ca96cSrmind {
3047e3fb338Srmind const npf_conndb_params_t *params = npf->params[NPF_PARAMS_CONNDB];
305*d6939920Srmind unsigned target = params->step;
306*d6939920Srmind unsigned gc_conns = 0;
3076f4ca96cSrmind npf_conn_t *con;
3086f4ca96cSrmind
309*d6939920Srmind KASSERT(mutex_owned(&npf->conn_lock));
310*d6939920Srmind
3116f4ca96cSrmind /*
3126f4ca96cSrmind * Second, start from the "last" (marker) connection.
3136f4ca96cSrmind * We must initialise the marker if it is not set yet.
3146f4ca96cSrmind */
3156f4ca96cSrmind if ((con = cd->cd_marker) == NULL) {
3166f4ca96cSrmind con = npf_conndb_getnext(cd, NULL);
3176f4ca96cSrmind cd->cd_marker = con;
3186f4ca96cSrmind }
3196f4ca96cSrmind
3206f4ca96cSrmind /*
3216f4ca96cSrmind * Scan the connections:
3226f4ca96cSrmind * - Limit the scan to the G/C step size.
3236f4ca96cSrmind * - Stop if we scanned all of them.
3246f4ca96cSrmind * - Update the marker connection.
3256f4ca96cSrmind */
3266f4ca96cSrmind while (con && target--) {
3276f4ca96cSrmind npf_conn_t *next = npf_conndb_getnext(cd, con);
3286f4ca96cSrmind
3296f4ca96cSrmind /*
3306f4ca96cSrmind * Can we G/C this connection?
3316f4ca96cSrmind */
3327e3fb338Srmind if (npf_conn_expired(npf, con, now)) {
3336f4ca96cSrmind /* Yes: move to the G/C list. */
3346f4ca96cSrmind LIST_REMOVE(con, c_entry);
3356f4ca96cSrmind LIST_INSERT_HEAD(&cd->cd_gclist, con, c_entry);
3366f4ca96cSrmind npf_conn_remove(cd, con);
337*d6939920Srmind gc_conns++;
3386f4ca96cSrmind
3396f4ca96cSrmind /* This connection cannot be a new marker anymore. */
3406f4ca96cSrmind if (con == next) {
3416f4ca96cSrmind next = NULL;
3426f4ca96cSrmind }
3436f4ca96cSrmind if (con == cd->cd_marker) {
3446f4ca96cSrmind cd->cd_marker = next;
3456f4ca96cSrmind con = next;
3466f4ca96cSrmind continue;
3476f4ca96cSrmind }
3486f4ca96cSrmind }
3496f4ca96cSrmind con = next;
3506f4ca96cSrmind
3516f4ca96cSrmind /*
3526f4ca96cSrmind * Circular iteration: if we returned back to the
3536f4ca96cSrmind * marker connection, then stop.
3546f4ca96cSrmind */
3556f4ca96cSrmind if (con == cd->cd_marker) {
3566f4ca96cSrmind break;
3576f4ca96cSrmind }
3586f4ca96cSrmind }
3596f4ca96cSrmind cd->cd_marker = con;
360*d6939920Srmind return gc_conns;
361*d6939920Srmind }
362*d6939920Srmind
363*d6939920Srmind /*
364*d6939920Srmind * gc_freq_tune: G/C frequency self-tuning.
365*d6939920Srmind *
366*d6939920Srmind * If there is something to G/C, then exponentially increase the wake
367*d6939920Srmind * up frequency. Otherwise, reduce the frequency. Enforce the lower
368*d6939920Srmind * and upper bounds.
369*d6939920Srmind *
370*d6939920Srmind * => Returns the number milliseconds until next G/C.
371*d6939920Srmind */
372*d6939920Srmind static unsigned
gc_freq_tune(const npf_t * npf,const npf_conndb_t * cd,const unsigned n)373*d6939920Srmind gc_freq_tune(const npf_t *npf, const npf_conndb_t *cd, const unsigned n)
374*d6939920Srmind {
375*d6939920Srmind const npf_conndb_params_t *params = npf->params[NPF_PARAMS_CONNDB];
376*d6939920Srmind int wtime = npf->worker_wait_time;
377*d6939920Srmind wtime = n ? (wtime >> 1) : (wtime << 1);
378*d6939920Srmind return MAX(MIN(wtime, params->interval_max), params->interval_min);
3796f4ca96cSrmind }
3806f4ca96cSrmind
3816f4ca96cSrmind /*
3826f4ca96cSrmind * npf_conndb_gc: garbage collect the expired connections.
3836f4ca96cSrmind *
3846f4ca96cSrmind * => Must run in a single-threaded manner.
3856f4ca96cSrmind * => If 'flush' is true, then destroy all connections.
3866f4ca96cSrmind * => If 'sync' is true, then perform passive serialisation.
387c6d74635Srmind */
388c6d74635Srmind void
npf_conndb_gc(npf_t * npf,npf_conndb_t * cd,bool flush,bool sync)3896f4ca96cSrmind npf_conndb_gc(npf_t *npf, npf_conndb_t *cd, bool flush, bool sync)
390c6d74635Srmind {
3916f4ca96cSrmind struct timespec tsnow;
392*d6939920Srmind unsigned gc_conns = 0;
3936f4ca96cSrmind npf_conn_t *con;
3946f4ca96cSrmind void *gcref;
3956f4ca96cSrmind
3966f4ca96cSrmind getnanouptime(&tsnow);
3976f4ca96cSrmind
3986f4ca96cSrmind /* First, migrate all new connections. */
3996f4ca96cSrmind mutex_enter(&npf->conn_lock);
4006f4ca96cSrmind npf_conndb_update(cd);
4016f4ca96cSrmind if (flush) {
4026f4ca96cSrmind /* Just unlink and move all connections to the G/C list. */
4036f4ca96cSrmind while ((con = LIST_FIRST(&cd->cd_list)) != NULL) {
4046f4ca96cSrmind LIST_REMOVE(con, c_entry);
4056f4ca96cSrmind LIST_INSERT_HEAD(&cd->cd_gclist, con, c_entry);
4066f4ca96cSrmind npf_conn_remove(cd, con);
4076f4ca96cSrmind }
4086f4ca96cSrmind cd->cd_marker = NULL;
4096f4ca96cSrmind } else {
4106f4ca96cSrmind /* Incremental G/C of the expired connections. */
411*d6939920Srmind gc_conns = npf_conndb_gc_incr(npf, cd, tsnow.tv_sec);
4126f4ca96cSrmind }
4136f4ca96cSrmind mutex_exit(&npf->conn_lock);
4146f4ca96cSrmind
4156f4ca96cSrmind /*
4166f4ca96cSrmind * Ensure it is safe to destroy the connections.
4176f4ca96cSrmind * Note: drop the conn_lock (see the lock order).
4186f4ca96cSrmind */
4196f4ca96cSrmind gcref = thmap_stage_gc(cd->cd_map);
420d43aee49Sriastradh if (sync && (gcref || !LIST_EMPTY(&cd->cd_gclist))) {
4216f4ca96cSrmind npf_config_enter(npf);
4226f4ca96cSrmind npf_config_sync(npf);
4236f4ca96cSrmind npf_config_exit(npf);
4246f4ca96cSrmind }
4256f4ca96cSrmind thmap_gc(cd->cd_map, gcref);
4266f4ca96cSrmind
427*d6939920Srmind /* Self-tune the G/C frequency. */
428*d6939920Srmind npf->worker_wait_time = gc_freq_tune(npf, cd, gc_conns);
429*d6939920Srmind
4306f4ca96cSrmind if (LIST_EMPTY(&cd->cd_gclist)) {
4316f4ca96cSrmind return;
4326f4ca96cSrmind }
4336f4ca96cSrmind
4346f4ca96cSrmind /*
4356f4ca96cSrmind * Garbage collect all expired connections.
4366f4ca96cSrmind * May need to wait for the references to drain.
4376f4ca96cSrmind */
4386f4ca96cSrmind while ((con = LIST_FIRST(&cd->cd_gclist)) != NULL) {
4396f4ca96cSrmind /*
4406f4ca96cSrmind * Destroy only if removed and no references. Otherwise,
4416f4ca96cSrmind * just do it next time, unless we are destroying all.
4426f4ca96cSrmind */
443*d6939920Srmind const unsigned refcnt = atomic_load_relaxed(&con->c_refcnt);
444*d6939920Srmind
445*d6939920Srmind if (__predict_false(refcnt)) {
446*d6939920Srmind if (flush) {
4476f4ca96cSrmind kpause("npfcongc", false, 1, NULL);
4486f4ca96cSrmind continue;
4496f4ca96cSrmind }
450*d6939920Srmind break; // exit the loop
451*d6939920Srmind }
4526f4ca96cSrmind LIST_REMOVE(con, c_entry);
4536f4ca96cSrmind npf_conn_destroy(npf, con);
4546f4ca96cSrmind }
455c6d74635Srmind }
456