1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1991, 1993, 1995
5 * The Regents of the University of California.
6 * Copyright (c) 2007-2009 Robert N. M. Watson
7 * Copyright (c) 2010-2011 Juniper Networks, Inc.
8 * Copyright (c) 2021-2022 Gleb Smirnoff <glebius@FreeBSD.org>
9 * All rights reserved.
10 *
11 * Portions of this software were developed by Robert N. M. Watson under
12 * contract to Juniper Networks, Inc.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
16 * are met:
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 * 3. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 */
38
39 #include <sys/cdefs.h>
40 #include "opt_ddb.h"
41 #include "opt_ipsec.h"
42 #include "opt_inet.h"
43 #include "opt_inet6.h"
44 #include "opt_ratelimit.h"
45 #include "opt_route.h"
46 #include "opt_rss.h"
47
48 #include <sys/param.h>
49 #include <sys/hash.h>
50 #include <sys/systm.h>
51 #include <sys/libkern.h>
52 #include <sys/lock.h>
53 #include <sys/malloc.h>
54 #include <sys/mbuf.h>
55 #include <sys/eventhandler.h>
56 #include <sys/domain.h>
57 #include <sys/proc.h>
58 #include <sys/protosw.h>
59 #include <sys/smp.h>
60 #include <sys/smr.h>
61 #include <sys/socket.h>
62 #include <sys/socketvar.h>
63 #include <sys/sockio.h>
64 #include <sys/priv.h>
65 #include <sys/proc.h>
66 #include <sys/refcount.h>
67 #include <sys/jail.h>
68 #include <sys/kernel.h>
69 #include <sys/sysctl.h>
70
71 #ifdef DDB
72 #include <ddb/ddb.h>
73 #endif
74
75 #include <vm/uma.h>
76 #include <vm/vm.h>
77
78 #include <net/if.h>
79 #include <net/if_var.h>
80 #include <net/if_private.h>
81 #include <net/if_types.h>
82 #include <net/if_llatbl.h>
83 #include <net/route.h>
84 #include <net/rss_config.h>
85 #include <net/vnet.h>
86
87 #if defined(INET) || defined(INET6)
88 #include <netinet/in.h>
89 #include <netinet/in_pcb.h>
90 #include <netinet/in_pcb_var.h>
91 #include <netinet/tcp.h>
92 #ifdef INET
93 #include <netinet/in_var.h>
94 #include <netinet/in_fib.h>
95 #endif
96 #include <netinet/ip_var.h>
97 #ifdef INET6
98 #include <netinet/ip6.h>
99 #include <netinet6/in6_pcb.h>
100 #include <netinet6/in6_var.h>
101 #include <netinet6/ip6_var.h>
102 #endif /* INET6 */
103 #include <net/route/nhop.h>
104 #endif
105
106 #include <netipsec/ipsec_support.h>
107
108 #include <security/mac/mac_framework.h>
109
110 #define INPCBLBGROUP_SIZMIN 8
111 #define INPCBLBGROUP_SIZMAX 256
112
113 #define INP_FREED 0x00000200 /* Went through in_pcbfree(). */
114 #define INP_INLBGROUP 0x01000000 /* Inserted into inpcblbgroup. */
115
116 /*
117 * These configure the range of local port addresses assigned to
118 * "unspecified" outgoing connections/packets/whatever.
119 */
120 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1; /* 1023 */
121 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART; /* 600 */
122 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST; /* 10000 */
123 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST; /* 65535 */
124 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO; /* 49152 */
125 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO; /* 65535 */
126
127 /*
128 * Reserved ports accessible only to root. There are significant
129 * security considerations that must be accounted for when changing these,
130 * but the security benefits can be great. Please be careful.
131 */
132 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */
133 VNET_DEFINE(int, ipport_reservedlow);
134
135 /* Enable random ephemeral port allocation by default. */
136 VNET_DEFINE(int, ipport_randomized) = 1;
137
138 #ifdef INET
139 static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
140 struct in_addr faddr, u_int fport_arg,
141 struct in_addr laddr, u_int lport_arg,
142 int lookupflags, uint8_t numa_domain);
143
144 #define RANGECHK(var, min, max) \
145 if ((var) < (min)) { (var) = (min); } \
146 else if ((var) > (max)) { (var) = (max); }
147
148 static int
sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)149 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
150 {
151 int error;
152
153 error = sysctl_handle_int(oidp, arg1, arg2, req);
154 if (error == 0) {
155 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
156 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
157 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
158 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
159 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
160 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
161 }
162 return (error);
163 }
164
165 #undef RANGECHK
166
167 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange,
168 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
169 "IP Ports");
170
171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
172 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
173 &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I",
174 "");
175 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
176 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
177 &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I",
178 "");
179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
180 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
181 &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I",
182 "");
183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
184 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
185 &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I",
186 "");
187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
188 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
189 &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I",
190 "");
191 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
192 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
193 &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I",
194 "");
195 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
196 CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
197 &VNET_NAME(ipport_reservedhigh), 0, "");
198 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
199 CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
200 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
201 CTLFLAG_VNET | CTLFLAG_RW,
202 &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
203
204 #ifdef RATELIMIT
205 counter_u64_t rate_limit_new;
206 counter_u64_t rate_limit_chg;
207 counter_u64_t rate_limit_active;
208 counter_u64_t rate_limit_alloc_fail;
209 counter_u64_t rate_limit_set_ok;
210
211 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
212 "IP Rate Limiting");
213 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD,
214 &rate_limit_active, "Active rate limited connections");
215 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD,
216 &rate_limit_alloc_fail, "Rate limited connection failures");
217 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
218 &rate_limit_set_ok, "Rate limited setting succeeded");
219 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD,
220 &rate_limit_new, "Total Rate limit new attempts");
221 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD,
222 &rate_limit_chg, "Total Rate limited change attempts");
223 #endif /* RATELIMIT */
224
225 #endif /* INET */
226
227 VNET_DEFINE(uint32_t, in_pcbhashseed);
228 static void
in_pcbhashseed_init(void)229 in_pcbhashseed_init(void)
230 {
231
232 V_in_pcbhashseed = arc4random();
233 }
234 VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST,
235 in_pcbhashseed_init, 0);
236
237 static void in_pcbremhash(struct inpcb *);
238
239 /*
240 * in_pcb.c: manage the Protocol Control Blocks.
241 *
242 * NOTE: It is assumed that most of these functions will be called with
243 * the pcbinfo lock held, and often, the inpcb lock held, as these utility
244 * functions often modify hash chains or addresses in pcbs.
245 */
246
247 static struct inpcblbgroup *
in_pcblbgroup_alloc(struct inpcblbgrouphead * hdr,struct ucred * cred,u_char vflag,uint16_t port,const union in_dependaddr * addr,int size,uint8_t numa_domain)248 in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, struct ucred *cred,
249 u_char vflag, uint16_t port, const union in_dependaddr *addr, int size,
250 uint8_t numa_domain)
251 {
252 struct inpcblbgroup *grp;
253 size_t bytes;
254
255 bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
256 grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
257 if (grp == NULL)
258 return (NULL);
259 grp->il_cred = crhold(cred);
260 grp->il_vflag = vflag;
261 grp->il_lport = port;
262 grp->il_numa_domain = numa_domain;
263 grp->il_dependladdr = *addr;
264 grp->il_inpsiz = size;
265 CK_LIST_INSERT_HEAD(hdr, grp, il_list);
266 return (grp);
267 }
268
269 static void
in_pcblbgroup_free_deferred(epoch_context_t ctx)270 in_pcblbgroup_free_deferred(epoch_context_t ctx)
271 {
272 struct inpcblbgroup *grp;
273
274 grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
275 crfree(grp->il_cred);
276 free(grp, M_PCB);
277 }
278
279 static void
in_pcblbgroup_free(struct inpcblbgroup * grp)280 in_pcblbgroup_free(struct inpcblbgroup *grp)
281 {
282
283 CK_LIST_REMOVE(grp, il_list);
284 NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
285 }
286
287 static struct inpcblbgroup *
in_pcblbgroup_resize(struct inpcblbgrouphead * hdr,struct inpcblbgroup * old_grp,int size)288 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
289 struct inpcblbgroup *old_grp, int size)
290 {
291 struct inpcblbgroup *grp;
292 int i;
293
294 grp = in_pcblbgroup_alloc(hdr, old_grp->il_cred, old_grp->il_vflag,
295 old_grp->il_lport, &old_grp->il_dependladdr, size,
296 old_grp->il_numa_domain);
297 if (grp == NULL)
298 return (NULL);
299
300 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
301 ("invalid new local group size %d and old local group count %d",
302 grp->il_inpsiz, old_grp->il_inpcnt));
303
304 for (i = 0; i < old_grp->il_inpcnt; ++i)
305 grp->il_inp[i] = old_grp->il_inp[i];
306 grp->il_inpcnt = old_grp->il_inpcnt;
307 in_pcblbgroup_free(old_grp);
308 return (grp);
309 }
310
311 /*
312 * PCB at index 'i' is removed from the group. Pull up the ones below il_inp[i]
313 * and shrink group if possible.
314 */
315 static void
in_pcblbgroup_reorder(struct inpcblbgrouphead * hdr,struct inpcblbgroup ** grpp,int i)316 in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp,
317 int i)
318 {
319 struct inpcblbgroup *grp, *new_grp;
320
321 grp = *grpp;
322 for (; i + 1 < grp->il_inpcnt; ++i)
323 grp->il_inp[i] = grp->il_inp[i + 1];
324 grp->il_inpcnt--;
325
326 if (grp->il_inpsiz > INPCBLBGROUP_SIZMIN &&
327 grp->il_inpcnt <= grp->il_inpsiz / 4) {
328 /* Shrink this group. */
329 new_grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz / 2);
330 if (new_grp != NULL)
331 *grpp = new_grp;
332 }
333 }
334
335 /*
336 * Add PCB to load balance group for SO_REUSEPORT_LB option.
337 */
338 static int
in_pcbinslbgrouphash(struct inpcb * inp,uint8_t numa_domain)339 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
340 {
341 const static struct timeval interval = { 60, 0 };
342 static struct timeval lastprint;
343 struct inpcbinfo *pcbinfo;
344 struct inpcblbgrouphead *hdr;
345 struct inpcblbgroup *grp;
346 uint32_t idx;
347
348 pcbinfo = inp->inp_pcbinfo;
349
350 INP_WLOCK_ASSERT(inp);
351 INP_HASH_WLOCK_ASSERT(pcbinfo);
352
353 #ifdef INET6
354 /*
355 * Don't allow IPv4 mapped INET6 wild socket.
356 */
357 if ((inp->inp_vflag & INP_IPV4) &&
358 inp->inp_laddr.s_addr == INADDR_ANY &&
359 INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
360 return (0);
361 }
362 #endif
363
364 idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
365 hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
366 CK_LIST_FOREACH(grp, hdr, il_list) {
367 if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison &&
368 grp->il_vflag == inp->inp_vflag &&
369 grp->il_lport == inp->inp_lport &&
370 grp->il_numa_domain == numa_domain &&
371 memcmp(&grp->il_dependladdr,
372 &inp->inp_inc.inc_ie.ie_dependladdr,
373 sizeof(grp->il_dependladdr)) == 0) {
374 break;
375 }
376 }
377 if (grp == NULL) {
378 /* Create new load balance group. */
379 grp = in_pcblbgroup_alloc(hdr, inp->inp_cred, inp->inp_vflag,
380 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
381 INPCBLBGROUP_SIZMIN, numa_domain);
382 if (grp == NULL)
383 return (ENOBUFS);
384 } else if (grp->il_inpcnt == grp->il_inpsiz) {
385 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
386 if (ratecheck(&lastprint, &interval))
387 printf("lb group port %d, limit reached\n",
388 ntohs(grp->il_lport));
389 return (0);
390 }
391
392 /* Expand this local group. */
393 grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
394 if (grp == NULL)
395 return (ENOBUFS);
396 }
397
398 KASSERT(grp->il_inpcnt < grp->il_inpsiz,
399 ("invalid local group size %d and count %d", grp->il_inpsiz,
400 grp->il_inpcnt));
401
402 grp->il_inp[grp->il_inpcnt] = inp;
403 grp->il_inpcnt++;
404 inp->inp_flags |= INP_INLBGROUP;
405 return (0);
406 }
407
408 /*
409 * Remove PCB from load balance group.
410 */
411 static void
in_pcbremlbgrouphash(struct inpcb * inp)412 in_pcbremlbgrouphash(struct inpcb *inp)
413 {
414 struct inpcbinfo *pcbinfo;
415 struct inpcblbgrouphead *hdr;
416 struct inpcblbgroup *grp;
417 int i;
418
419 pcbinfo = inp->inp_pcbinfo;
420
421 INP_WLOCK_ASSERT(inp);
422 MPASS(inp->inp_flags & INP_INLBGROUP);
423 INP_HASH_WLOCK_ASSERT(pcbinfo);
424
425 hdr = &pcbinfo->ipi_lbgrouphashbase[
426 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
427 CK_LIST_FOREACH(grp, hdr, il_list) {
428 for (i = 0; i < grp->il_inpcnt; ++i) {
429 if (grp->il_inp[i] != inp)
430 continue;
431
432 if (grp->il_inpcnt == 1) {
433 /* We are the last, free this local group. */
434 in_pcblbgroup_free(grp);
435 } else {
436 /* Pull up inpcbs, shrink group if possible. */
437 in_pcblbgroup_reorder(hdr, &grp, i);
438 }
439 inp->inp_flags &= ~INP_INLBGROUP;
440 return;
441 }
442 }
443 KASSERT(0, ("%s: did not find %p", __func__, inp));
444 }
445
446 int
in_pcblbgroup_numa(struct inpcb * inp,int arg)447 in_pcblbgroup_numa(struct inpcb *inp, int arg)
448 {
449 struct inpcbinfo *pcbinfo;
450 struct inpcblbgrouphead *hdr;
451 struct inpcblbgroup *grp;
452 int err, i;
453 uint8_t numa_domain;
454
455 switch (arg) {
456 case TCP_REUSPORT_LB_NUMA_NODOM:
457 numa_domain = M_NODOM;
458 break;
459 case TCP_REUSPORT_LB_NUMA_CURDOM:
460 numa_domain = PCPU_GET(domain);
461 break;
462 default:
463 if (arg < 0 || arg >= vm_ndomains)
464 return (EINVAL);
465 numa_domain = arg;
466 }
467
468 err = 0;
469 pcbinfo = inp->inp_pcbinfo;
470 INP_WLOCK_ASSERT(inp);
471 INP_HASH_WLOCK(pcbinfo);
472 hdr = &pcbinfo->ipi_lbgrouphashbase[
473 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
474 CK_LIST_FOREACH(grp, hdr, il_list) {
475 for (i = 0; i < grp->il_inpcnt; ++i) {
476 if (grp->il_inp[i] != inp)
477 continue;
478
479 if (grp->il_numa_domain == numa_domain) {
480 goto abort_with_hash_wlock;
481 }
482
483 /* Remove it from the old group. */
484 in_pcbremlbgrouphash(inp);
485
486 /* Add it to the new group based on numa domain. */
487 in_pcbinslbgrouphash(inp, numa_domain);
488 goto abort_with_hash_wlock;
489 }
490 }
491 err = ENOENT;
492 abort_with_hash_wlock:
493 INP_HASH_WUNLOCK(pcbinfo);
494 return (err);
495 }
496
497 /* Make sure it is safe to use hashinit(9) on CK_LIST. */
498 CTASSERT(sizeof(struct inpcbhead) == sizeof(LIST_HEAD(, inpcb)));
499
500 /*
501 * Initialize an inpcbinfo - a per-VNET instance of connections db.
502 */
503 void
in_pcbinfo_init(struct inpcbinfo * pcbinfo,struct inpcbstorage * pcbstor,u_int hash_nelements,u_int porthash_nelements)504 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor,
505 u_int hash_nelements, u_int porthash_nelements)
506 {
507
508 mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF);
509 mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name,
510 NULL, MTX_DEF);
511 #ifdef VIMAGE
512 pcbinfo->ipi_vnet = curvnet;
513 #endif
514 CK_LIST_INIT(&pcbinfo->ipi_listhead);
515 pcbinfo->ipi_count = 0;
516 pcbinfo->ipi_hash_exact = hashinit(hash_nelements, M_PCB,
517 &pcbinfo->ipi_hashmask);
518 pcbinfo->ipi_hash_wild = hashinit(hash_nelements, M_PCB,
519 &pcbinfo->ipi_hashmask);
520 porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
521 pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
522 &pcbinfo->ipi_porthashmask);
523 pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,
524 &pcbinfo->ipi_lbgrouphashmask);
525 pcbinfo->ipi_zone = pcbstor->ips_zone;
526 pcbinfo->ipi_portzone = pcbstor->ips_portzone;
527 pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone);
528 }
529
530 /*
531 * Destroy an inpcbinfo.
532 */
533 void
in_pcbinfo_destroy(struct inpcbinfo * pcbinfo)534 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
535 {
536
537 KASSERT(pcbinfo->ipi_count == 0,
538 ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
539
540 hashdestroy(pcbinfo->ipi_hash_exact, M_PCB, pcbinfo->ipi_hashmask);
541 hashdestroy(pcbinfo->ipi_hash_wild, M_PCB, pcbinfo->ipi_hashmask);
542 hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
543 pcbinfo->ipi_porthashmask);
544 hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,
545 pcbinfo->ipi_lbgrouphashmask);
546 mtx_destroy(&pcbinfo->ipi_hash_lock);
547 mtx_destroy(&pcbinfo->ipi_lock);
548 }
549
550 /*
551 * Initialize a pcbstorage - per protocol zones to allocate inpcbs.
552 */
553 static void inpcb_fini(void *, int);
554 void
in_pcbstorage_init(void * arg)555 in_pcbstorage_init(void *arg)
556 {
557 struct inpcbstorage *pcbstor = arg;
558
559 pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name,
560 pcbstor->ips_size, NULL, NULL, pcbstor->ips_pcbinit,
561 inpcb_fini, UMA_ALIGN_CACHE, UMA_ZONE_SMR);
562 pcbstor->ips_portzone = uma_zcreate(pcbstor->ips_portzone_name,
563 sizeof(struct inpcbport), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
564 uma_zone_set_smr(pcbstor->ips_portzone,
565 uma_zone_get_smr(pcbstor->ips_zone));
566 }
567
568 /*
569 * Destroy a pcbstorage - used by unloadable protocols.
570 */
571 void
in_pcbstorage_destroy(void * arg)572 in_pcbstorage_destroy(void *arg)
573 {
574 struct inpcbstorage *pcbstor = arg;
575
576 uma_zdestroy(pcbstor->ips_zone);
577 uma_zdestroy(pcbstor->ips_portzone);
578 }
579
580 /*
581 * Allocate a PCB and associate it with the socket.
582 * On success return with the PCB locked.
583 */
584 int
in_pcballoc(struct socket * so,struct inpcbinfo * pcbinfo)585 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
586 {
587 struct inpcb *inp;
588 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
589 int error;
590 #endif
591
592 inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);
593 if (inp == NULL)
594 return (ENOBUFS);
595 bzero(&inp->inp_start_zero, inp_zero_size);
596 #ifdef NUMA
597 inp->inp_numa_domain = M_NODOM;
598 #endif
599 inp->inp_pcbinfo = pcbinfo;
600 inp->inp_socket = so;
601 inp->inp_cred = crhold(so->so_cred);
602 inp->inp_inc.inc_fibnum = so->so_fibnum;
603 #ifdef MAC
604 error = mac_inpcb_init(inp, M_NOWAIT);
605 if (error != 0)
606 goto out;
607 mac_inpcb_create(so, inp);
608 #endif
609 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
610 error = ipsec_init_pcbpolicy(inp);
611 if (error != 0) {
612 #ifdef MAC
613 mac_inpcb_destroy(inp);
614 #endif
615 goto out;
616 }
617 #endif /*IPSEC*/
618 #ifdef INET6
619 if (INP_SOCKAF(so) == AF_INET6) {
620 inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6;
621 if (V_ip6_v6only)
622 inp->inp_flags |= IN6P_IPV6_V6ONLY;
623 #ifdef INET
624 else
625 inp->inp_vflag |= INP_IPV4;
626 #endif
627 if (V_ip6_auto_flowlabel)
628 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
629 inp->in6p_hops = -1; /* use kernel default */
630 }
631 #endif
632 #if defined(INET) && defined(INET6)
633 else
634 #endif
635 #ifdef INET
636 inp->inp_vflag |= INP_IPV4;
637 #endif
638 inp->inp_smr = SMR_SEQ_INVALID;
639
640 /*
641 * Routes in inpcb's can cache L2 as well; they are guaranteed
642 * to be cleaned up.
643 */
644 inp->inp_route.ro_flags = RT_LLE_CACHE;
645 refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */
646 INP_WLOCK(inp);
647 INP_INFO_WLOCK(pcbinfo);
648 pcbinfo->ipi_count++;
649 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
650 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list);
651 INP_INFO_WUNLOCK(pcbinfo);
652 so->so_pcb = inp;
653
654 return (0);
655
656 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
657 out:
658 crfree(inp->inp_cred);
659 #ifdef INVARIANTS
660 inp->inp_cred = NULL;
661 #endif
662 uma_zfree_smr(pcbinfo->ipi_zone, inp);
663 return (error);
664 #endif
665 }
666
667 #ifdef INET
668 int
in_pcbbind(struct inpcb * inp,struct sockaddr_in * sin,struct ucred * cred)669 in_pcbbind(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
670 {
671 int anonport, error;
672
673 KASSERT(sin == NULL || sin->sin_family == AF_INET,
674 ("%s: invalid address family for %p", __func__, sin));
675 KASSERT(sin == NULL || sin->sin_len == sizeof(struct sockaddr_in),
676 ("%s: invalid address length for %p", __func__, sin));
677 INP_WLOCK_ASSERT(inp);
678 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
679
680 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
681 return (EINVAL);
682 anonport = sin == NULL || sin->sin_port == 0;
683 error = in_pcbbind_setup(inp, sin, &inp->inp_laddr.s_addr,
684 &inp->inp_lport, cred);
685 if (error)
686 return (error);
687 if (in_pcbinshash(inp) != 0) {
688 inp->inp_laddr.s_addr = INADDR_ANY;
689 inp->inp_lport = 0;
690 return (EAGAIN);
691 }
692 if (anonport)
693 inp->inp_flags |= INP_ANONPORT;
694 return (0);
695 }
696 #endif
697
698 #if defined(INET) || defined(INET6)
699 /*
700 * Assign a local port like in_pcb_lport(), but also used with connect()
701 * and a foreign address and port. If fsa is non-NULL, choose a local port
702 * that is unused with those, otherwise one that is completely unused.
703 * lsa can be NULL for IPv6.
704 */
705 int
in_pcb_lport_dest(struct inpcb * inp,struct sockaddr * lsa,u_short * lportp,struct sockaddr * fsa,u_short fport,struct ucred * cred,int lookupflags)706 in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp,
707 struct sockaddr *fsa, u_short fport, struct ucred *cred, int lookupflags)
708 {
709 struct inpcbinfo *pcbinfo;
710 struct inpcb *tmpinp;
711 unsigned short *lastport;
712 int count, error;
713 u_short aux, first, last, lport;
714 #ifdef INET
715 struct in_addr laddr, faddr;
716 #endif
717 #ifdef INET6
718 struct in6_addr *laddr6, *faddr6;
719 #endif
720
721 pcbinfo = inp->inp_pcbinfo;
722
723 /*
724 * Because no actual state changes occur here, a global write lock on
725 * the pcbinfo isn't required.
726 */
727 INP_LOCK_ASSERT(inp);
728 INP_HASH_LOCK_ASSERT(pcbinfo);
729
730 if (inp->inp_flags & INP_HIGHPORT) {
731 first = V_ipport_hifirstauto; /* sysctl */
732 last = V_ipport_hilastauto;
733 lastport = &pcbinfo->ipi_lasthi;
734 } else if (inp->inp_flags & INP_LOWPORT) {
735 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
736 if (error)
737 return (error);
738 first = V_ipport_lowfirstauto; /* 1023 */
739 last = V_ipport_lowlastauto; /* 600 */
740 lastport = &pcbinfo->ipi_lastlow;
741 } else {
742 first = V_ipport_firstauto; /* sysctl */
743 last = V_ipport_lastauto;
744 lastport = &pcbinfo->ipi_lastport;
745 }
746
747 /*
748 * Instead of having two loops further down counting up or down
749 * make sure that first is always <= last and go with only one
750 * code path implementing all logic.
751 */
752 if (first > last) {
753 aux = first;
754 first = last;
755 last = aux;
756 }
757
758 #ifdef INET
759 laddr.s_addr = INADDR_ANY; /* used by INET6+INET below too */
760 if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
761 if (lsa != NULL)
762 laddr = ((struct sockaddr_in *)lsa)->sin_addr;
763 if (fsa != NULL)
764 faddr = ((struct sockaddr_in *)fsa)->sin_addr;
765 }
766 #endif
767 #ifdef INET6
768 laddr6 = NULL;
769 if ((inp->inp_vflag & INP_IPV6) != 0) {
770 if (lsa != NULL)
771 laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr;
772 if (fsa != NULL)
773 faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
774 }
775 #endif
776
777 tmpinp = NULL;
778 lport = *lportp;
779
780 if (V_ipport_randomized)
781 *lastport = first + (arc4random() % (last - first));
782
783 count = last - first;
784
785 do {
786 if (count-- < 0) /* completely used? */
787 return (EADDRNOTAVAIL);
788 ++*lastport;
789 if (*lastport < first || *lastport > last)
790 *lastport = first;
791 lport = htons(*lastport);
792
793 if (fsa != NULL) {
794 #ifdef INET
795 if (lsa->sa_family == AF_INET) {
796 tmpinp = in_pcblookup_hash_locked(pcbinfo,
797 faddr, fport, laddr, lport, lookupflags,
798 M_NODOM);
799 }
800 #endif
801 #ifdef INET6
802 if (lsa->sa_family == AF_INET6) {
803 tmpinp = in6_pcblookup_hash_locked(pcbinfo,
804 faddr6, fport, laddr6, lport, lookupflags,
805 M_NODOM);
806 }
807 #endif
808 } else {
809 #ifdef INET6
810 if ((inp->inp_vflag & INP_IPV6) != 0) {
811 tmpinp = in6_pcblookup_local(pcbinfo,
812 &inp->in6p_laddr, lport, lookupflags, cred);
813 #ifdef INET
814 if (tmpinp == NULL &&
815 (inp->inp_vflag & INP_IPV4))
816 tmpinp = in_pcblookup_local(pcbinfo,
817 laddr, lport, lookupflags, cred);
818 #endif
819 }
820 #endif
821 #if defined(INET) && defined(INET6)
822 else
823 #endif
824 #ifdef INET
825 tmpinp = in_pcblookup_local(pcbinfo, laddr,
826 lport, lookupflags, cred);
827 #endif
828 }
829 } while (tmpinp != NULL);
830
831 *lportp = lport;
832
833 return (0);
834 }
835
836 /*
837 * Select a local port (number) to use.
838 */
839 int
in_pcb_lport(struct inpcb * inp,struct in_addr * laddrp,u_short * lportp,struct ucred * cred,int lookupflags)840 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
841 struct ucred *cred, int lookupflags)
842 {
843 struct sockaddr_in laddr;
844
845 if (laddrp) {
846 bzero(&laddr, sizeof(laddr));
847 laddr.sin_family = AF_INET;
848 laddr.sin_addr = *laddrp;
849 }
850 return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr :
851 NULL, lportp, NULL, 0, cred, lookupflags));
852 }
853 #endif /* INET || INET6 */
854
855 #ifdef INET
856 /*
857 * Set up a bind operation on a PCB, performing port allocation
858 * as required, but do not actually modify the PCB. Callers can
859 * either complete the bind by setting inp_laddr/inp_lport and
860 * calling in_pcbinshash(), or they can just use the resulting
861 * port and address to authorise the sending of a once-off packet.
862 *
863 * On error, the values of *laddrp and *lportp are not changed.
864 */
865 int
in_pcbbind_setup(struct inpcb * inp,struct sockaddr_in * sin,in_addr_t * laddrp,u_short * lportp,struct ucred * cred)866 in_pcbbind_setup(struct inpcb *inp, struct sockaddr_in *sin, in_addr_t *laddrp,
867 u_short *lportp, struct ucred *cred)
868 {
869 struct socket *so = inp->inp_socket;
870 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
871 struct in_addr laddr;
872 u_short lport = 0;
873 int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
874 int error;
875
876 /*
877 * XXX: Maybe we could let SO_REUSEPORT_LB set SO_REUSEPORT bit here
878 * so that we don't have to add to the (already messy) code below.
879 */
880 int reuseport_lb = (so->so_options & SO_REUSEPORT_LB);
881
882 /*
883 * No state changes, so read locks are sufficient here.
884 */
885 INP_LOCK_ASSERT(inp);
886 INP_HASH_LOCK_ASSERT(pcbinfo);
887
888 laddr.s_addr = *laddrp;
889 if (sin != NULL && laddr.s_addr != INADDR_ANY)
890 return (EINVAL);
891 if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
892 lookupflags = INPLOOKUP_WILDCARD;
893 if (sin == NULL) {
894 if ((error = prison_local_ip4(cred, &laddr)) != 0)
895 return (error);
896 } else {
897 KASSERT(sin->sin_family == AF_INET,
898 ("%s: invalid family for address %p", __func__, sin));
899 KASSERT(sin->sin_len == sizeof(*sin),
900 ("%s: invalid length for address %p", __func__, sin));
901
902 error = prison_local_ip4(cred, &sin->sin_addr);
903 if (error)
904 return (error);
905 if (sin->sin_port != *lportp) {
906 /* Don't allow the port to change. */
907 if (*lportp != 0)
908 return (EINVAL);
909 lport = sin->sin_port;
910 }
911 /* NB: lport is left as 0 if the port isn't being changed. */
912 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
913 /*
914 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
915 * allow complete duplication of binding if
916 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
917 * and a multicast address is bound on both
918 * new and duplicated sockets.
919 */
920 if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
921 reuseport = SO_REUSEADDR|SO_REUSEPORT;
922 /*
923 * XXX: How to deal with SO_REUSEPORT_LB here?
924 * Treat same as SO_REUSEPORT for now.
925 */
926 if ((so->so_options &
927 (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0)
928 reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB;
929 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
930 sin->sin_port = 0; /* yech... */
931 bzero(&sin->sin_zero, sizeof(sin->sin_zero));
932 /*
933 * Is the address a local IP address?
934 * If INP_BINDANY is set, then the socket may be bound
935 * to any endpoint address, local or not.
936 */
937 if ((inp->inp_flags & INP_BINDANY) == 0 &&
938 ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
939 return (EADDRNOTAVAIL);
940 }
941 laddr = sin->sin_addr;
942 if (lport) {
943 struct inpcb *t;
944
945 /* GROSS */
946 if (ntohs(lport) <= V_ipport_reservedhigh &&
947 ntohs(lport) >= V_ipport_reservedlow &&
948 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
949 return (EACCES);
950 if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
951 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
952 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
953 lport, INPLOOKUP_WILDCARD, cred);
954 /*
955 * XXX
956 * This entire block sorely needs a rewrite.
957 */
958 if (t != NULL &&
959 (so->so_type != SOCK_STREAM ||
960 ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
961 (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
962 ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
963 (t->inp_socket->so_options & SO_REUSEPORT) ||
964 (t->inp_socket->so_options & SO_REUSEPORT_LB) == 0) &&
965 (inp->inp_cred->cr_uid !=
966 t->inp_cred->cr_uid))
967 return (EADDRINUSE);
968 }
969 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
970 lport, lookupflags, cred);
971 if (t != NULL && (reuseport & t->inp_socket->so_options) == 0 &&
972 (reuseport_lb & t->inp_socket->so_options) == 0) {
973 #ifdef INET6
974 if (ntohl(sin->sin_addr.s_addr) !=
975 INADDR_ANY ||
976 ntohl(t->inp_laddr.s_addr) !=
977 INADDR_ANY ||
978 (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
979 (t->inp_vflag & INP_IPV6PROTO) == 0)
980 #endif
981 return (EADDRINUSE);
982 }
983 }
984 }
985 if (*lportp != 0)
986 lport = *lportp;
987 if (lport == 0) {
988 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
989 if (error != 0)
990 return (error);
991 }
992 *laddrp = laddr.s_addr;
993 *lportp = lport;
994 return (0);
995 }
996
997 /*
998 * Connect from a socket to a specified address.
999 * Both address and port must be specified in argument sin.
1000 * If don't have a local address for this socket yet,
1001 * then pick one.
1002 */
1003 int
in_pcbconnect(struct inpcb * inp,struct sockaddr_in * sin,struct ucred * cred,bool rehash __unused)1004 in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred,
1005 bool rehash __unused)
1006 {
1007 u_short lport, fport;
1008 in_addr_t laddr, faddr;
1009 int anonport, error;
1010
1011 INP_WLOCK_ASSERT(inp);
1012 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1013 KASSERT(in_nullhost(inp->inp_faddr),
1014 ("%s: inp is already connected", __func__));
1015
1016 lport = inp->inp_lport;
1017 laddr = inp->inp_laddr.s_addr;
1018 anonport = (lport == 0);
1019 error = in_pcbconnect_setup(inp, sin, &laddr, &lport, &faddr, &fport,
1020 cred);
1021 if (error)
1022 return (error);
1023
1024 inp->inp_faddr.s_addr = faddr;
1025 inp->inp_fport = fport;
1026
1027 /* Do the initial binding of the local address if required. */
1028 if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
1029 inp->inp_lport = lport;
1030 inp->inp_laddr.s_addr = laddr;
1031 if (in_pcbinshash(inp) != 0) {
1032 inp->inp_laddr.s_addr = inp->inp_faddr.s_addr =
1033 INADDR_ANY;
1034 inp->inp_lport = inp->inp_fport = 0;
1035 return (EAGAIN);
1036 }
1037 } else {
1038 inp->inp_lport = lport;
1039 inp->inp_laddr.s_addr = laddr;
1040 if ((inp->inp_flags & INP_INHASHLIST) != 0)
1041 in_pcbrehash(inp);
1042 else
1043 in_pcbinshash(inp);
1044 }
1045
1046 if (anonport)
1047 inp->inp_flags |= INP_ANONPORT;
1048 return (0);
1049 }
1050
1051 /*
1052 * Do proper source address selection on an unbound socket in case
1053 * of connect. Take jails into account as well.
1054 */
1055 int
in_pcbladdr(struct inpcb * inp,struct in_addr * faddr,struct in_addr * laddr,struct ucred * cred)1056 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
1057 struct ucred *cred)
1058 {
1059 struct ifaddr *ifa;
1060 struct sockaddr *sa;
1061 struct sockaddr_in *sin, dst;
1062 struct nhop_object *nh;
1063 int error;
1064
1065 NET_EPOCH_ASSERT();
1066 KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
1067
1068 /*
1069 * Bypass source address selection and use the primary jail IP
1070 * if requested.
1071 */
1072 if (!prison_saddrsel_ip4(cred, laddr))
1073 return (0);
1074
1075 error = 0;
1076
1077 nh = NULL;
1078 bzero(&dst, sizeof(dst));
1079 sin = &dst;
1080 sin->sin_family = AF_INET;
1081 sin->sin_len = sizeof(struct sockaddr_in);
1082 sin->sin_addr.s_addr = faddr->s_addr;
1083
1084 /*
1085 * If route is known our src addr is taken from the i/f,
1086 * else punt.
1087 *
1088 * Find out route to destination.
1089 */
1090 if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1091 nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr,
1092 0, NHR_NONE, 0);
1093
1094 /*
1095 * If we found a route, use the address corresponding to
1096 * the outgoing interface.
1097 *
1098 * Otherwise assume faddr is reachable on a directly connected
1099 * network and try to find a corresponding interface to take
1100 * the source address from.
1101 */
1102 if (nh == NULL || nh->nh_ifp == NULL) {
1103 struct in_ifaddr *ia;
1104 struct ifnet *ifp;
1105
1106 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
1107 inp->inp_socket->so_fibnum));
1108 if (ia == NULL) {
1109 ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
1110 inp->inp_socket->so_fibnum));
1111 }
1112 if (ia == NULL) {
1113 error = ENETUNREACH;
1114 goto done;
1115 }
1116
1117 if (!prison_flag(cred, PR_IP4)) {
1118 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1119 goto done;
1120 }
1121
1122 ifp = ia->ia_ifp;
1123 ia = NULL;
1124 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1125 sa = ifa->ifa_addr;
1126 if (sa->sa_family != AF_INET)
1127 continue;
1128 sin = (struct sockaddr_in *)sa;
1129 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1130 ia = (struct in_ifaddr *)ifa;
1131 break;
1132 }
1133 }
1134 if (ia != NULL) {
1135 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1136 goto done;
1137 }
1138
1139 /* 3. As a last resort return the 'default' jail address. */
1140 error = prison_get_ip4(cred, laddr);
1141 goto done;
1142 }
1143
1144 /*
1145 * If the outgoing interface on the route found is not
1146 * a loopback interface, use the address from that interface.
1147 * In case of jails do those three steps:
1148 * 1. check if the interface address belongs to the jail. If so use it.
1149 * 2. check if we have any address on the outgoing interface
1150 * belonging to this jail. If so use it.
1151 * 3. as a last resort return the 'default' jail address.
1152 */
1153 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) {
1154 struct in_ifaddr *ia;
1155 struct ifnet *ifp;
1156
1157 /* If not jailed, use the default returned. */
1158 if (!prison_flag(cred, PR_IP4)) {
1159 ia = (struct in_ifaddr *)nh->nh_ifa;
1160 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1161 goto done;
1162 }
1163
1164 /* Jailed. */
1165 /* 1. Check if the iface address belongs to the jail. */
1166 sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr;
1167 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1168 ia = (struct in_ifaddr *)nh->nh_ifa;
1169 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1170 goto done;
1171 }
1172
1173 /*
1174 * 2. Check if we have any address on the outgoing interface
1175 * belonging to this jail.
1176 */
1177 ia = NULL;
1178 ifp = nh->nh_ifp;
1179 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1180 sa = ifa->ifa_addr;
1181 if (sa->sa_family != AF_INET)
1182 continue;
1183 sin = (struct sockaddr_in *)sa;
1184 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1185 ia = (struct in_ifaddr *)ifa;
1186 break;
1187 }
1188 }
1189 if (ia != NULL) {
1190 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1191 goto done;
1192 }
1193
1194 /* 3. As a last resort return the 'default' jail address. */
1195 error = prison_get_ip4(cred, laddr);
1196 goto done;
1197 }
1198
1199 /*
1200 * The outgoing interface is marked with 'loopback net', so a route
1201 * to ourselves is here.
1202 * Try to find the interface of the destination address and then
1203 * take the address from there. That interface is not necessarily
1204 * a loopback interface.
1205 * In case of jails, check that it is an address of the jail
1206 * and if we cannot find, fall back to the 'default' jail address.
1207 */
1208 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) {
1209 struct in_ifaddr *ia;
1210
1211 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst),
1212 inp->inp_socket->so_fibnum));
1213 if (ia == NULL)
1214 ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0,
1215 inp->inp_socket->so_fibnum));
1216 if (ia == NULL)
1217 ia = ifatoia(ifa_ifwithaddr(sintosa(&dst)));
1218
1219 if (!prison_flag(cred, PR_IP4)) {
1220 if (ia == NULL) {
1221 error = ENETUNREACH;
1222 goto done;
1223 }
1224 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1225 goto done;
1226 }
1227
1228 /* Jailed. */
1229 if (ia != NULL) {
1230 struct ifnet *ifp;
1231
1232 ifp = ia->ia_ifp;
1233 ia = NULL;
1234 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1235 sa = ifa->ifa_addr;
1236 if (sa->sa_family != AF_INET)
1237 continue;
1238 sin = (struct sockaddr_in *)sa;
1239 if (prison_check_ip4(cred,
1240 &sin->sin_addr) == 0) {
1241 ia = (struct in_ifaddr *)ifa;
1242 break;
1243 }
1244 }
1245 if (ia != NULL) {
1246 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1247 goto done;
1248 }
1249 }
1250
1251 /* 3. As a last resort return the 'default' jail address. */
1252 error = prison_get_ip4(cred, laddr);
1253 goto done;
1254 }
1255
1256 done:
1257 if (error == 0 && laddr->s_addr == INADDR_ANY)
1258 return (EHOSTUNREACH);
1259 return (error);
1260 }
1261
1262 /*
1263 * Set up for a connect from a socket to the specified address.
1264 * On entry, *laddrp and *lportp should contain the current local
1265 * address and port for the PCB; these are updated to the values
1266 * that should be placed in inp_laddr and inp_lport to complete
1267 * the connect.
1268 *
1269 * On success, *faddrp and *fportp will be set to the remote address
1270 * and port. These are not updated in the error case.
1271 */
1272 int
in_pcbconnect_setup(struct inpcb * inp,struct sockaddr_in * sin,in_addr_t * laddrp,u_short * lportp,in_addr_t * faddrp,u_short * fportp,struct ucred * cred)1273 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr_in *sin,
1274 in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1275 struct ucred *cred)
1276 {
1277 struct in_ifaddr *ia;
1278 struct in_addr laddr, faddr;
1279 u_short lport, fport;
1280 int error;
1281
1282 KASSERT(sin->sin_family == AF_INET,
1283 ("%s: invalid address family for %p", __func__, sin));
1284 KASSERT(sin->sin_len == sizeof(*sin),
1285 ("%s: invalid address length for %p", __func__, sin));
1286
1287 /*
1288 * Because a global state change doesn't actually occur here, a read
1289 * lock is sufficient.
1290 */
1291 NET_EPOCH_ASSERT();
1292 INP_LOCK_ASSERT(inp);
1293 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1294
1295 if (sin->sin_port == 0)
1296 return (EADDRNOTAVAIL);
1297 laddr.s_addr = *laddrp;
1298 lport = *lportp;
1299 faddr = sin->sin_addr;
1300 fport = sin->sin_port;
1301 #ifdef ROUTE_MPATH
1302 if (CALC_FLOWID_OUTBOUND) {
1303 uint32_t hash_val, hash_type;
1304
1305 hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport,
1306 inp->inp_socket->so_proto->pr_protocol, &hash_type);
1307
1308 inp->inp_flowid = hash_val;
1309 inp->inp_flowtype = hash_type;
1310 }
1311 #endif
1312 if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
1313 /*
1314 * If the destination address is INADDR_ANY,
1315 * use the primary local address.
1316 * If the supplied address is INADDR_BROADCAST,
1317 * and the primary interface supports broadcast,
1318 * choose the broadcast address for that interface.
1319 */
1320 if (faddr.s_addr == INADDR_ANY) {
1321 faddr =
1322 IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1323 if ((error = prison_get_ip4(cred, &faddr)) != 0)
1324 return (error);
1325 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1326 if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1327 IFF_BROADCAST)
1328 faddr = satosin(&CK_STAILQ_FIRST(
1329 &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1330 }
1331 }
1332 if (laddr.s_addr == INADDR_ANY) {
1333 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1334 /*
1335 * If the destination address is multicast and an outgoing
1336 * interface has been set as a multicast option, prefer the
1337 * address of that interface as our source address.
1338 */
1339 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1340 inp->inp_moptions != NULL) {
1341 struct ip_moptions *imo;
1342 struct ifnet *ifp;
1343
1344 imo = inp->inp_moptions;
1345 if (imo->imo_multicast_ifp != NULL) {
1346 ifp = imo->imo_multicast_ifp;
1347 CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1348 if (ia->ia_ifp == ifp &&
1349 prison_check_ip4(cred,
1350 &ia->ia_addr.sin_addr) == 0)
1351 break;
1352 }
1353 if (ia == NULL)
1354 error = EADDRNOTAVAIL;
1355 else {
1356 laddr = ia->ia_addr.sin_addr;
1357 error = 0;
1358 }
1359 }
1360 }
1361 if (error)
1362 return (error);
1363 }
1364
1365 if (lport != 0) {
1366 if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
1367 fport, laddr, lport, 0, M_NODOM) != NULL)
1368 return (EADDRINUSE);
1369 } else {
1370 struct sockaddr_in lsin, fsin;
1371
1372 bzero(&lsin, sizeof(lsin));
1373 bzero(&fsin, sizeof(fsin));
1374 lsin.sin_family = AF_INET;
1375 lsin.sin_addr = laddr;
1376 fsin.sin_family = AF_INET;
1377 fsin.sin_addr = faddr;
1378 error = in_pcb_lport_dest(inp, (struct sockaddr *) &lsin,
1379 &lport, (struct sockaddr *)& fsin, fport, cred,
1380 INPLOOKUP_WILDCARD);
1381 if (error)
1382 return (error);
1383 }
1384 *laddrp = laddr.s_addr;
1385 *lportp = lport;
1386 *faddrp = faddr.s_addr;
1387 *fportp = fport;
1388 return (0);
1389 }
1390
1391 void
in_pcbdisconnect(struct inpcb * inp)1392 in_pcbdisconnect(struct inpcb *inp)
1393 {
1394
1395 INP_WLOCK_ASSERT(inp);
1396 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1397 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
1398 ("%s: inp %p was already disconnected", __func__, inp));
1399
1400 in_pcbremhash_locked(inp);
1401
1402 /* See the comment in in_pcbinshash(). */
1403 inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr);
1404 inp->inp_laddr.s_addr = INADDR_ANY;
1405 inp->inp_faddr.s_addr = INADDR_ANY;
1406 inp->inp_fport = 0;
1407 }
1408 #endif /* INET */
1409
1410 /*
1411 * inpcb hash lookups are protected by SMR section.
1412 *
1413 * Once desired pcb has been found, switching from SMR section to a pcb
1414 * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK
1415 * here because SMR is a critical section.
1416 * In 99%+ cases inp_smr_lock() would obtain the lock immediately.
1417 */
1418 void
inp_lock(struct inpcb * inp,const inp_lookup_t lock)1419 inp_lock(struct inpcb *inp, const inp_lookup_t lock)
1420 {
1421
1422 lock == INPLOOKUP_RLOCKPCB ?
1423 rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock);
1424 }
1425
1426 void
inp_unlock(struct inpcb * inp,const inp_lookup_t lock)1427 inp_unlock(struct inpcb *inp, const inp_lookup_t lock)
1428 {
1429
1430 lock == INPLOOKUP_RLOCKPCB ?
1431 rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock);
1432 }
1433
1434 int
inp_trylock(struct inpcb * inp,const inp_lookup_t lock)1435 inp_trylock(struct inpcb *inp, const inp_lookup_t lock)
1436 {
1437
1438 return (lock == INPLOOKUP_RLOCKPCB ?
1439 rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock));
1440 }
1441
1442 static inline bool
_inp_smr_lock(struct inpcb * inp,const inp_lookup_t lock,const int ignflags)1443 _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags)
1444 {
1445
1446 MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB);
1447 SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr);
1448
1449 if (__predict_true(inp_trylock(inp, lock))) {
1450 if (__predict_false(inp->inp_flags & ignflags)) {
1451 smr_exit(inp->inp_pcbinfo->ipi_smr);
1452 inp_unlock(inp, lock);
1453 return (false);
1454 }
1455 smr_exit(inp->inp_pcbinfo->ipi_smr);
1456 return (true);
1457 }
1458
1459 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1460 smr_exit(inp->inp_pcbinfo->ipi_smr);
1461 inp_lock(inp, lock);
1462 if (__predict_false(in_pcbrele(inp, lock)))
1463 return (false);
1464 /*
1465 * inp acquired through refcount & lock for sure didn't went
1466 * through uma_zfree(). However, it may have already went
1467 * through in_pcbfree() and has another reference, that
1468 * prevented its release by our in_pcbrele().
1469 */
1470 if (__predict_false(inp->inp_flags & ignflags)) {
1471 inp_unlock(inp, lock);
1472 return (false);
1473 }
1474 return (true);
1475 } else {
1476 smr_exit(inp->inp_pcbinfo->ipi_smr);
1477 return (false);
1478 }
1479 }
1480
1481 bool
inp_smr_lock(struct inpcb * inp,const inp_lookup_t lock)1482 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock)
1483 {
1484
1485 /*
1486 * in_pcblookup() family of functions ignore not only freed entries,
1487 * that may be found due to lockless access to the hash, but dropped
1488 * entries, too.
1489 */
1490 return (_inp_smr_lock(inp, lock, INP_FREED | INP_DROPPED));
1491 }
1492
1493 /*
1494 * inp_next() - inpcb hash/list traversal iterator
1495 *
1496 * Requires initialized struct inpcb_iterator for context.
1497 * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR().
1498 *
1499 * - Iterator can have either write-lock or read-lock semantics, that can not
1500 * be changed later.
1501 * - Iterator can iterate either over all pcbs list (INP_ALL_LIST), or through
1502 * a single hash slot. Note: only rip_input() does the latter.
1503 * - Iterator may have optional bool matching function. The matching function
1504 * will be executed for each inpcb in the SMR context, so it can not acquire
1505 * locks and can safely access only immutable fields of inpcb.
1506 *
1507 * A fresh initialized iterator has NULL inpcb in its context and that
1508 * means that inp_next() call would return the very first inpcb on the list
1509 * locked with desired semantic. In all following calls the context pointer
1510 * shall hold the current inpcb pointer. The KPI user is not supposed to
1511 * unlock the current inpcb! Upon end of traversal inp_next() will return NULL
1512 * and write NULL to its context. After end of traversal an iterator can be
1513 * reused.
1514 *
1515 * List traversals have the following features/constraints:
1516 * - New entries won't be seen, as they are always added to the head of a list.
1517 * - Removed entries won't stop traversal as long as they are not added to
1518 * a different list. This is violated by in_pcbrehash().
1519 */
1520 #define II_LIST_FIRST(ipi, hash) \
1521 (((hash) == INP_ALL_LIST) ? \
1522 CK_LIST_FIRST(&(ipi)->ipi_listhead) : \
1523 CK_LIST_FIRST(&(ipi)->ipi_hash_exact[(hash)]))
1524 #define II_LIST_NEXT(inp, hash) \
1525 (((hash) == INP_ALL_LIST) ? \
1526 CK_LIST_NEXT((inp), inp_list) : \
1527 CK_LIST_NEXT((inp), inp_hash_exact))
1528 #define II_LOCK_ASSERT(inp, lock) \
1529 rw_assert(&(inp)->inp_lock, \
1530 (lock) == INPLOOKUP_RLOCKPCB ? RA_RLOCKED : RA_WLOCKED )
1531 struct inpcb *
inp_next(struct inpcb_iterator * ii)1532 inp_next(struct inpcb_iterator *ii)
1533 {
1534 const struct inpcbinfo *ipi = ii->ipi;
1535 inp_match_t *match = ii->match;
1536 void *ctx = ii->ctx;
1537 inp_lookup_t lock = ii->lock;
1538 int hash = ii->hash;
1539 struct inpcb *inp;
1540
1541 if (ii->inp == NULL) { /* First call. */
1542 smr_enter(ipi->ipi_smr);
1543 /* This is unrolled CK_LIST_FOREACH(). */
1544 for (inp = II_LIST_FIRST(ipi, hash);
1545 inp != NULL;
1546 inp = II_LIST_NEXT(inp, hash)) {
1547 if (match != NULL && (match)(inp, ctx) == false)
1548 continue;
1549 if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED)))
1550 break;
1551 else {
1552 smr_enter(ipi->ipi_smr);
1553 MPASS(inp != II_LIST_FIRST(ipi, hash));
1554 inp = II_LIST_FIRST(ipi, hash);
1555 if (inp == NULL)
1556 break;
1557 }
1558 }
1559
1560 if (inp == NULL)
1561 smr_exit(ipi->ipi_smr);
1562 else
1563 ii->inp = inp;
1564
1565 return (inp);
1566 }
1567
1568 /* Not a first call. */
1569 smr_enter(ipi->ipi_smr);
1570 restart:
1571 inp = ii->inp;
1572 II_LOCK_ASSERT(inp, lock);
1573 next:
1574 inp = II_LIST_NEXT(inp, hash);
1575 if (inp == NULL) {
1576 smr_exit(ipi->ipi_smr);
1577 goto found;
1578 }
1579
1580 if (match != NULL && (match)(inp, ctx) == false)
1581 goto next;
1582
1583 if (__predict_true(inp_trylock(inp, lock))) {
1584 if (__predict_false(inp->inp_flags & INP_FREED)) {
1585 /*
1586 * Entries are never inserted in middle of a list, thus
1587 * as long as we are in SMR, we can continue traversal.
1588 * Jump to 'restart' should yield in the same result,
1589 * but could produce unnecessary looping. Could this
1590 * looping be unbound?
1591 */
1592 inp_unlock(inp, lock);
1593 goto next;
1594 } else {
1595 smr_exit(ipi->ipi_smr);
1596 goto found;
1597 }
1598 }
1599
1600 /*
1601 * Can't obtain lock immediately, thus going hard. Once we exit the
1602 * SMR section we can no longer jump to 'next', and our only stable
1603 * anchoring point is ii->inp, which we keep locked for this case, so
1604 * we jump to 'restart'.
1605 */
1606 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1607 smr_exit(ipi->ipi_smr);
1608 inp_lock(inp, lock);
1609 if (__predict_false(in_pcbrele(inp, lock))) {
1610 smr_enter(ipi->ipi_smr);
1611 goto restart;
1612 }
1613 /*
1614 * See comment in inp_smr_lock().
1615 */
1616 if (__predict_false(inp->inp_flags & INP_FREED)) {
1617 inp_unlock(inp, lock);
1618 smr_enter(ipi->ipi_smr);
1619 goto restart;
1620 }
1621 } else
1622 goto next;
1623
1624 found:
1625 inp_unlock(ii->inp, lock);
1626 ii->inp = inp;
1627
1628 return (ii->inp);
1629 }
1630
1631 /*
1632 * in_pcbref() bumps the reference count on an inpcb in order to maintain
1633 * stability of an inpcb pointer despite the inpcb lock being released or
1634 * SMR section exited.
1635 *
1636 * To free a reference later in_pcbrele_(r|w)locked() must be performed.
1637 */
1638 void
in_pcbref(struct inpcb * inp)1639 in_pcbref(struct inpcb *inp)
1640 {
1641 u_int old __diagused;
1642
1643 old = refcount_acquire(&inp->inp_refcount);
1644 KASSERT(old > 0, ("%s: refcount 0", __func__));
1645 }
1646
1647 /*
1648 * Drop a refcount on an inpcb elevated using in_pcbref(), potentially
1649 * freeing the pcb, if the reference was very last.
1650 */
1651 bool
in_pcbrele_rlocked(struct inpcb * inp)1652 in_pcbrele_rlocked(struct inpcb *inp)
1653 {
1654
1655 INP_RLOCK_ASSERT(inp);
1656
1657 if (!refcount_release(&inp->inp_refcount))
1658 return (false);
1659
1660 MPASS(inp->inp_flags & INP_FREED);
1661 MPASS(inp->inp_socket == NULL);
1662 crfree(inp->inp_cred);
1663 #ifdef INVARIANTS
1664 inp->inp_cred = NULL;
1665 #endif
1666 INP_RUNLOCK(inp);
1667 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1668 return (true);
1669 }
1670
1671 bool
in_pcbrele_wlocked(struct inpcb * inp)1672 in_pcbrele_wlocked(struct inpcb *inp)
1673 {
1674
1675 INP_WLOCK_ASSERT(inp);
1676
1677 if (!refcount_release(&inp->inp_refcount))
1678 return (false);
1679
1680 MPASS(inp->inp_flags & INP_FREED);
1681 MPASS(inp->inp_socket == NULL);
1682 crfree(inp->inp_cred);
1683 #ifdef INVARIANTS
1684 inp->inp_cred = NULL;
1685 #endif
1686 INP_WUNLOCK(inp);
1687 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1688 return (true);
1689 }
1690
1691 bool
in_pcbrele(struct inpcb * inp,const inp_lookup_t lock)1692 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock)
1693 {
1694
1695 return (lock == INPLOOKUP_RLOCKPCB ?
1696 in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp));
1697 }
1698
1699 /*
1700 * Unconditionally schedule an inpcb to be freed by decrementing its
1701 * reference count, which should occur only after the inpcb has been detached
1702 * from its socket. If another thread holds a temporary reference (acquired
1703 * using in_pcbref()) then the free is deferred until that reference is
1704 * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked.
1705 * Almost all work, including removal from global lists, is done in this
1706 * context, where the pcbinfo lock is held.
1707 */
1708 void
in_pcbfree(struct inpcb * inp)1709 in_pcbfree(struct inpcb *inp)
1710 {
1711 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1712 #ifdef INET
1713 struct ip_moptions *imo;
1714 #endif
1715 #ifdef INET6
1716 struct ip6_moptions *im6o;
1717 #endif
1718
1719 INP_WLOCK_ASSERT(inp);
1720 KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1721 KASSERT((inp->inp_flags & INP_FREED) == 0,
1722 ("%s: called twice for pcb %p", __func__, inp));
1723
1724 /*
1725 * in_pcblookup_local() and in6_pcblookup_local() may return an inpcb
1726 * from the hash without acquiring inpcb lock, they rely on the hash
1727 * lock, thus in_pcbremhash() should be the first action.
1728 */
1729 if (inp->inp_flags & INP_INHASHLIST)
1730 in_pcbremhash(inp);
1731 INP_INFO_WLOCK(pcbinfo);
1732 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1733 pcbinfo->ipi_count--;
1734 CK_LIST_REMOVE(inp, inp_list);
1735 INP_INFO_WUNLOCK(pcbinfo);
1736
1737 #ifdef RATELIMIT
1738 if (inp->inp_snd_tag != NULL)
1739 in_pcbdetach_txrtlmt(inp);
1740 #endif
1741 inp->inp_flags |= INP_FREED;
1742 inp->inp_socket->so_pcb = NULL;
1743 inp->inp_socket = NULL;
1744
1745 RO_INVALIDATE_CACHE(&inp->inp_route);
1746 #ifdef MAC
1747 mac_inpcb_destroy(inp);
1748 #endif
1749 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1750 if (inp->inp_sp != NULL)
1751 ipsec_delete_pcbpolicy(inp);
1752 #endif
1753 #ifdef INET
1754 if (inp->inp_options)
1755 (void)m_free(inp->inp_options);
1756 DEBUG_POISON_POINTER(inp->inp_options);
1757 imo = inp->inp_moptions;
1758 DEBUG_POISON_POINTER(inp->inp_moptions);
1759 #endif
1760 #ifdef INET6
1761 if (inp->inp_vflag & INP_IPV6PROTO) {
1762 ip6_freepcbopts(inp->in6p_outputopts);
1763 DEBUG_POISON_POINTER(inp->in6p_outputopts);
1764 im6o = inp->in6p_moptions;
1765 DEBUG_POISON_POINTER(inp->in6p_moptions);
1766 } else
1767 im6o = NULL;
1768 #endif
1769
1770 if (__predict_false(in_pcbrele_wlocked(inp) == false)) {
1771 INP_WUNLOCK(inp);
1772 }
1773 #ifdef INET6
1774 ip6_freemoptions(im6o);
1775 #endif
1776 #ifdef INET
1777 inp_freemoptions(imo);
1778 #endif
1779 }
1780
1781 /*
1782 * Different protocols initialize their inpcbs differently - giving
1783 * different name to the lock. But they all are disposed the same.
1784 */
1785 static void
inpcb_fini(void * mem,int size)1786 inpcb_fini(void *mem, int size)
1787 {
1788 struct inpcb *inp = mem;
1789
1790 INP_LOCK_DESTROY(inp);
1791 }
1792
1793 /*
1794 * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1795 * port reservation, and preventing it from being returned by inpcb lookups.
1796 *
1797 * It is used by TCP to mark an inpcb as unused and avoid future packet
1798 * delivery or event notification when a socket remains open but TCP has
1799 * closed. This might occur as a result of a shutdown()-initiated TCP close
1800 * or a RST on the wire, and allows the port binding to be reused while still
1801 * maintaining the invariant that so_pcb always points to a valid inpcb until
1802 * in_pcbdetach().
1803 *
1804 * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1805 * in_pcbpurgeif0()?
1806 */
1807 void
in_pcbdrop(struct inpcb * inp)1808 in_pcbdrop(struct inpcb *inp)
1809 {
1810
1811 INP_WLOCK_ASSERT(inp);
1812
1813 inp->inp_flags |= INP_DROPPED;
1814 if (inp->inp_flags & INP_INHASHLIST)
1815 in_pcbremhash(inp);
1816 }
1817
1818 #ifdef INET
1819 /*
1820 * Common routines to return the socket addresses associated with inpcbs.
1821 */
1822 int
in_getsockaddr(struct socket * so,struct sockaddr * sa)1823 in_getsockaddr(struct socket *so, struct sockaddr *sa)
1824 {
1825 struct inpcb *inp;
1826
1827 inp = sotoinpcb(so);
1828 KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1829
1830 *(struct sockaddr_in *)sa = (struct sockaddr_in ){
1831 .sin_len = sizeof(struct sockaddr_in),
1832 .sin_family = AF_INET,
1833 .sin_port = inp->inp_lport,
1834 .sin_addr = inp->inp_laddr,
1835 };
1836
1837 return (0);
1838 }
1839
1840 int
in_getpeeraddr(struct socket * so,struct sockaddr * sa)1841 in_getpeeraddr(struct socket *so, struct sockaddr *sa)
1842 {
1843 struct inpcb *inp;
1844
1845 inp = sotoinpcb(so);
1846 KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1847
1848 *(struct sockaddr_in *)sa = (struct sockaddr_in ){
1849 .sin_len = sizeof(struct sockaddr_in),
1850 .sin_family = AF_INET,
1851 .sin_port = inp->inp_fport,
1852 .sin_addr = inp->inp_faddr,
1853 };
1854
1855 return (0);
1856 }
1857
1858 static bool
inp_v4_multi_match(const struct inpcb * inp,void * v __unused)1859 inp_v4_multi_match(const struct inpcb *inp, void *v __unused)
1860 {
1861
1862 if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL)
1863 return (true);
1864 else
1865 return (false);
1866 }
1867
1868 void
in_pcbpurgeif0(struct inpcbinfo * pcbinfo,struct ifnet * ifp)1869 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1870 {
1871 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,
1872 inp_v4_multi_match, NULL);
1873 struct inpcb *inp;
1874 struct in_multi *inm;
1875 struct in_mfilter *imf;
1876 struct ip_moptions *imo;
1877
1878 IN_MULTI_LOCK_ASSERT();
1879
1880 while ((inp = inp_next(&inpi)) != NULL) {
1881 INP_WLOCK_ASSERT(inp);
1882
1883 imo = inp->inp_moptions;
1884 /*
1885 * Unselect the outgoing interface if it is being
1886 * detached.
1887 */
1888 if (imo->imo_multicast_ifp == ifp)
1889 imo->imo_multicast_ifp = NULL;
1890
1891 /*
1892 * Drop multicast group membership if we joined
1893 * through the interface being detached.
1894 *
1895 * XXX This can all be deferred to an epoch_call
1896 */
1897 restart:
1898 IP_MFILTER_FOREACH(imf, &imo->imo_head) {
1899 if ((inm = imf->imf_inm) == NULL)
1900 continue;
1901 if (inm->inm_ifp != ifp)
1902 continue;
1903 ip_mfilter_remove(&imo->imo_head, imf);
1904 in_leavegroup_locked(inm, NULL);
1905 ip_mfilter_free(imf);
1906 goto restart;
1907 }
1908 }
1909 }
1910
1911 /*
1912 * Lookup a PCB based on the local address and port. Caller must hold the
1913 * hash lock. No inpcb locks or references are acquired.
1914 */
1915 #define INP_LOOKUP_MAPPED_PCB_COST 3
1916 struct inpcb *
in_pcblookup_local(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport,int lookupflags,struct ucred * cred)1917 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1918 u_short lport, int lookupflags, struct ucred *cred)
1919 {
1920 struct inpcb *inp;
1921 #ifdef INET6
1922 int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1923 #else
1924 int matchwild = 3;
1925 #endif
1926 int wildcard;
1927
1928 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1929 ("%s: invalid lookup flags %d", __func__, lookupflags));
1930 INP_HASH_LOCK_ASSERT(pcbinfo);
1931
1932 if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1933 struct inpcbhead *head;
1934 /*
1935 * Look for an unconnected (wildcard foreign addr) PCB that
1936 * matches the local address and port we're looking for.
1937 */
1938 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
1939 pcbinfo->ipi_hashmask)];
1940 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
1941 #ifdef INET6
1942 /* XXX inp locking */
1943 if ((inp->inp_vflag & INP_IPV4) == 0)
1944 continue;
1945 #endif
1946 if (inp->inp_faddr.s_addr == INADDR_ANY &&
1947 inp->inp_laddr.s_addr == laddr.s_addr &&
1948 inp->inp_lport == lport) {
1949 /*
1950 * Found?
1951 */
1952 if (prison_equal_ip4(cred->cr_prison,
1953 inp->inp_cred->cr_prison))
1954 return (inp);
1955 }
1956 }
1957 /*
1958 * Not found.
1959 */
1960 return (NULL);
1961 } else {
1962 struct inpcbporthead *porthash;
1963 struct inpcbport *phd;
1964 struct inpcb *match = NULL;
1965 /*
1966 * Best fit PCB lookup.
1967 *
1968 * First see if this local port is in use by looking on the
1969 * port hash list.
1970 */
1971 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1972 pcbinfo->ipi_porthashmask)];
1973 CK_LIST_FOREACH(phd, porthash, phd_hash) {
1974 if (phd->phd_port == lport)
1975 break;
1976 }
1977 if (phd != NULL) {
1978 /*
1979 * Port is in use by one or more PCBs. Look for best
1980 * fit.
1981 */
1982 CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1983 wildcard = 0;
1984 if (!prison_equal_ip4(inp->inp_cred->cr_prison,
1985 cred->cr_prison))
1986 continue;
1987 #ifdef INET6
1988 /* XXX inp locking */
1989 if ((inp->inp_vflag & INP_IPV4) == 0)
1990 continue;
1991 /*
1992 * We never select the PCB that has
1993 * INP_IPV6 flag and is bound to :: if
1994 * we have another PCB which is bound
1995 * to 0.0.0.0. If a PCB has the
1996 * INP_IPV6 flag, then we set its cost
1997 * higher than IPv4 only PCBs.
1998 *
1999 * Note that the case only happens
2000 * when a socket is bound to ::, under
2001 * the condition that the use of the
2002 * mapped address is allowed.
2003 */
2004 if ((inp->inp_vflag & INP_IPV6) != 0)
2005 wildcard += INP_LOOKUP_MAPPED_PCB_COST;
2006 #endif
2007 if (inp->inp_faddr.s_addr != INADDR_ANY)
2008 wildcard++;
2009 if (inp->inp_laddr.s_addr != INADDR_ANY) {
2010 if (laddr.s_addr == INADDR_ANY)
2011 wildcard++;
2012 else if (inp->inp_laddr.s_addr != laddr.s_addr)
2013 continue;
2014 } else {
2015 if (laddr.s_addr != INADDR_ANY)
2016 wildcard++;
2017 }
2018 if (wildcard < matchwild) {
2019 match = inp;
2020 matchwild = wildcard;
2021 if (matchwild == 0)
2022 break;
2023 }
2024 }
2025 }
2026 return (match);
2027 }
2028 }
2029 #undef INP_LOOKUP_MAPPED_PCB_COST
2030
2031 static bool
in_pcblookup_lb_numa_match(const struct inpcblbgroup * grp,int domain)2032 in_pcblookup_lb_numa_match(const struct inpcblbgroup *grp, int domain)
2033 {
2034 return (domain == M_NODOM || domain == grp->il_numa_domain);
2035 }
2036
2037 static struct inpcb *
in_pcblookup_lbgroup(const struct inpcbinfo * pcbinfo,const struct in_addr * faddr,uint16_t fport,const struct in_addr * laddr,uint16_t lport,int domain)2038 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
2039 const struct in_addr *faddr, uint16_t fport, const struct in_addr *laddr,
2040 uint16_t lport, int domain)
2041 {
2042 const struct inpcblbgrouphead *hdr;
2043 struct inpcblbgroup *grp;
2044 struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild;
2045
2046 INP_HASH_LOCK_ASSERT(pcbinfo);
2047
2048 hdr = &pcbinfo->ipi_lbgrouphashbase[
2049 INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
2050
2051 /*
2052 * Search for an LB group match based on the following criteria:
2053 * - prefer jailed groups to non-jailed groups
2054 * - prefer exact source address matches to wildcard matches
2055 * - prefer groups bound to the specified NUMA domain
2056 */
2057 jail_exact = jail_wild = local_exact = local_wild = NULL;
2058 CK_LIST_FOREACH(grp, hdr, il_list) {
2059 bool injail;
2060
2061 #ifdef INET6
2062 if (!(grp->il_vflag & INP_IPV4))
2063 continue;
2064 #endif
2065 if (grp->il_lport != lport)
2066 continue;
2067
2068 injail = prison_flag(grp->il_cred, PR_IP4) != 0;
2069 if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison,
2070 laddr) != 0)
2071 continue;
2072
2073 if (grp->il_laddr.s_addr == laddr->s_addr) {
2074 if (injail) {
2075 jail_exact = grp;
2076 if (in_pcblookup_lb_numa_match(grp, domain))
2077 /* This is a perfect match. */
2078 goto out;
2079 } else if (local_exact == NULL ||
2080 in_pcblookup_lb_numa_match(grp, domain)) {
2081 local_exact = grp;
2082 }
2083 } else if (grp->il_laddr.s_addr == INADDR_ANY) {
2084 if (injail) {
2085 if (jail_wild == NULL ||
2086 in_pcblookup_lb_numa_match(grp, domain))
2087 jail_wild = grp;
2088 } else if (local_wild == NULL ||
2089 in_pcblookup_lb_numa_match(grp, domain)) {
2090 local_wild = grp;
2091 }
2092 }
2093 }
2094
2095 if (jail_exact != NULL)
2096 grp = jail_exact;
2097 else if (jail_wild != NULL)
2098 grp = jail_wild;
2099 else if (local_exact != NULL)
2100 grp = local_exact;
2101 else
2102 grp = local_wild;
2103 if (grp == NULL)
2104 return (NULL);
2105 out:
2106 return (grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) %
2107 grp->il_inpcnt]);
2108 }
2109
2110 static bool
in_pcblookup_exact_match(const struct inpcb * inp,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport)2111 in_pcblookup_exact_match(const struct inpcb *inp, struct in_addr faddr,
2112 u_short fport, struct in_addr laddr, u_short lport)
2113 {
2114 #ifdef INET6
2115 /* XXX inp locking */
2116 if ((inp->inp_vflag & INP_IPV4) == 0)
2117 return (false);
2118 #endif
2119 if (inp->inp_faddr.s_addr == faddr.s_addr &&
2120 inp->inp_laddr.s_addr == laddr.s_addr &&
2121 inp->inp_fport == fport &&
2122 inp->inp_lport == lport)
2123 return (true);
2124 return (false);
2125 }
2126
2127 static struct inpcb *
in_pcblookup_hash_exact(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport)2128 in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2129 u_short fport, struct in_addr laddr, u_short lport)
2130 {
2131 struct inpcbhead *head;
2132 struct inpcb *inp;
2133
2134 INP_HASH_LOCK_ASSERT(pcbinfo);
2135
2136 head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,
2137 pcbinfo->ipi_hashmask)];
2138 CK_LIST_FOREACH(inp, head, inp_hash_exact) {
2139 if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
2140 return (inp);
2141 }
2142 return (NULL);
2143 }
2144
2145 typedef enum {
2146 INPLOOKUP_MATCH_NONE = 0,
2147 INPLOOKUP_MATCH_WILD = 1,
2148 INPLOOKUP_MATCH_LADDR = 2,
2149 } inp_lookup_match_t;
2150
2151 static inp_lookup_match_t
in_pcblookup_wild_match(const struct inpcb * inp,struct in_addr laddr,u_short lport)2152 in_pcblookup_wild_match(const struct inpcb *inp, struct in_addr laddr,
2153 u_short lport)
2154 {
2155 #ifdef INET6
2156 /* XXX inp locking */
2157 if ((inp->inp_vflag & INP_IPV4) == 0)
2158 return (INPLOOKUP_MATCH_NONE);
2159 #endif
2160 if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport)
2161 return (INPLOOKUP_MATCH_NONE);
2162 if (inp->inp_laddr.s_addr == INADDR_ANY)
2163 return (INPLOOKUP_MATCH_WILD);
2164 if (inp->inp_laddr.s_addr == laddr.s_addr)
2165 return (INPLOOKUP_MATCH_LADDR);
2166 return (INPLOOKUP_MATCH_NONE);
2167 }
2168
2169 #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1)
2170
2171 static struct inpcb *
in_pcblookup_hash_wild_smr(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport,const inp_lookup_t lockflags)2172 in_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2173 u_short fport, struct in_addr laddr, u_short lport,
2174 const inp_lookup_t lockflags)
2175 {
2176 struct inpcbhead *head;
2177 struct inpcb *inp;
2178
2179 KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
2180 ("%s: not in SMR read section", __func__));
2181
2182 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2183 pcbinfo->ipi_hashmask)];
2184 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2185 inp_lookup_match_t match;
2186
2187 match = in_pcblookup_wild_match(inp, laddr, lport);
2188 if (match == INPLOOKUP_MATCH_NONE)
2189 continue;
2190
2191 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2192 match = in_pcblookup_wild_match(inp, laddr, lport);
2193 if (match != INPLOOKUP_MATCH_NONE &&
2194 prison_check_ip4_locked(inp->inp_cred->cr_prison,
2195 &laddr) == 0)
2196 return (inp);
2197 inp_unlock(inp, lockflags);
2198 }
2199
2200 /*
2201 * The matching socket disappeared out from under us. Fall back
2202 * to a serialized lookup.
2203 */
2204 return (INP_LOOKUP_AGAIN);
2205 }
2206 return (NULL);
2207 }
2208
2209 static struct inpcb *
in_pcblookup_hash_wild_locked(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport)2210 in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2211 u_short fport, struct in_addr laddr, u_short lport)
2212 {
2213 struct inpcbhead *head;
2214 struct inpcb *inp, *local_wild, *local_exact, *jail_wild;
2215 #ifdef INET6
2216 struct inpcb *local_wild_mapped;
2217 #endif
2218
2219 INP_HASH_LOCK_ASSERT(pcbinfo);
2220
2221 /*
2222 * Order of socket selection - we always prefer jails.
2223 * 1. jailed, non-wild.
2224 * 2. jailed, wild.
2225 * 3. non-jailed, non-wild.
2226 * 4. non-jailed, wild.
2227 */
2228 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2229 pcbinfo->ipi_hashmask)];
2230 local_wild = local_exact = jail_wild = NULL;
2231 #ifdef INET6
2232 local_wild_mapped = NULL;
2233 #endif
2234 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2235 inp_lookup_match_t match;
2236 bool injail;
2237
2238 match = in_pcblookup_wild_match(inp, laddr, lport);
2239 if (match == INPLOOKUP_MATCH_NONE)
2240 continue;
2241
2242 injail = prison_flag(inp->inp_cred, PR_IP4) != 0;
2243 if (injail) {
2244 if (prison_check_ip4_locked(inp->inp_cred->cr_prison,
2245 &laddr) != 0)
2246 continue;
2247 } else {
2248 if (local_exact != NULL)
2249 continue;
2250 }
2251
2252 if (match == INPLOOKUP_MATCH_LADDR) {
2253 if (injail)
2254 return (inp);
2255 local_exact = inp;
2256 } else {
2257 #ifdef INET6
2258 /* XXX inp locking, NULL check */
2259 if (inp->inp_vflag & INP_IPV6PROTO)
2260 local_wild_mapped = inp;
2261 else
2262 #endif
2263 if (injail)
2264 jail_wild = inp;
2265 else
2266 local_wild = inp;
2267 }
2268 }
2269 if (jail_wild != NULL)
2270 return (jail_wild);
2271 if (local_exact != NULL)
2272 return (local_exact);
2273 if (local_wild != NULL)
2274 return (local_wild);
2275 #ifdef INET6
2276 if (local_wild_mapped != NULL)
2277 return (local_wild_mapped);
2278 #endif
2279 return (NULL);
2280 }
2281
2282 /*
2283 * Lookup PCB in hash list, using pcbinfo tables. This variation assumes
2284 * that the caller has either locked the hash list, which usually happens
2285 * for bind(2) operations, or is in SMR section, which happens when sorting
2286 * out incoming packets.
2287 */
2288 static struct inpcb *
in_pcblookup_hash_locked(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport_arg,struct in_addr laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain)2289 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2290 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2291 uint8_t numa_domain)
2292 {
2293 struct inpcb *inp;
2294 const u_short fport = fport_arg, lport = lport_arg;
2295
2296 KASSERT((lookupflags & ~INPLOOKUP_WILDCARD) == 0,
2297 ("%s: invalid lookup flags %d", __func__, lookupflags));
2298 KASSERT(faddr.s_addr != INADDR_ANY,
2299 ("%s: invalid foreign address", __func__));
2300 KASSERT(laddr.s_addr != INADDR_ANY,
2301 ("%s: invalid local address", __func__));
2302 INP_HASH_WLOCK_ASSERT(pcbinfo);
2303
2304 inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2305 if (inp != NULL)
2306 return (inp);
2307
2308 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2309 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2310 &laddr, lport, numa_domain);
2311 if (inp == NULL) {
2312 inp = in_pcblookup_hash_wild_locked(pcbinfo, faddr,
2313 fport, laddr, lport);
2314 }
2315 }
2316
2317 return (inp);
2318 }
2319
2320 static struct inpcb *
in_pcblookup_hash(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,uint8_t numa_domain)2321 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2322 u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2323 uint8_t numa_domain)
2324 {
2325 struct inpcb *inp;
2326 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2327
2328 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2329 ("%s: LOCKPCB not set", __func__));
2330
2331 INP_HASH_WLOCK(pcbinfo);
2332 inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2333 lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain);
2334 if (inp != NULL && !inp_trylock(inp, lockflags)) {
2335 in_pcbref(inp);
2336 INP_HASH_WUNLOCK(pcbinfo);
2337 inp_lock(inp, lockflags);
2338 if (in_pcbrele(inp, lockflags))
2339 /* XXX-MJ or retry until we get a negative match? */
2340 inp = NULL;
2341 } else {
2342 INP_HASH_WUNLOCK(pcbinfo);
2343 }
2344 return (inp);
2345 }
2346
2347 static struct inpcb *
in_pcblookup_hash_smr(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport_arg,struct in_addr laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain)2348 in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2349 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2350 uint8_t numa_domain)
2351 {
2352 struct inpcb *inp;
2353 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2354 const u_short fport = fport_arg, lport = lport_arg;
2355
2356 KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2357 ("%s: invalid lookup flags %d", __func__, lookupflags));
2358 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2359 ("%s: LOCKPCB not set", __func__));
2360
2361 smr_enter(pcbinfo->ipi_smr);
2362 inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2363 if (inp != NULL) {
2364 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2365 /*
2366 * Revalidate the 4-tuple, the socket could have been
2367 * disconnected.
2368 */
2369 if (__predict_true(in_pcblookup_exact_match(inp,
2370 faddr, fport, laddr, lport)))
2371 return (inp);
2372 inp_unlock(inp, lockflags);
2373 }
2374
2375 /*
2376 * We failed to lock the inpcb, or its connection state changed
2377 * out from under us. Fall back to a precise search.
2378 */
2379 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2380 lookupflags, numa_domain));
2381 }
2382
2383 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2384 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2385 &laddr, lport, numa_domain);
2386 if (inp != NULL) {
2387 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2388 if (__predict_true(in_pcblookup_wild_match(inp,
2389 laddr, lport) != INPLOOKUP_MATCH_NONE))
2390 return (inp);
2391 inp_unlock(inp, lockflags);
2392 }
2393 inp = INP_LOOKUP_AGAIN;
2394 } else {
2395 inp = in_pcblookup_hash_wild_smr(pcbinfo, faddr, fport,
2396 laddr, lport, lockflags);
2397 }
2398 if (inp == INP_LOOKUP_AGAIN) {
2399 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr,
2400 lport, lookupflags, numa_domain));
2401 }
2402 }
2403
2404 if (inp == NULL)
2405 smr_exit(pcbinfo->ipi_smr);
2406
2407 return (inp);
2408 }
2409
2410 /*
2411 * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2412 * from which a pre-calculated hash value may be extracted.
2413 */
2414 struct inpcb *
in_pcblookup(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,struct ifnet * ifp __unused)2415 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2416 struct in_addr laddr, u_int lport, int lookupflags,
2417 struct ifnet *ifp __unused)
2418 {
2419 return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
2420 lookupflags, M_NODOM));
2421 }
2422
2423 struct inpcb *
in_pcblookup_mbuf(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,struct ifnet * ifp __unused,struct mbuf * m)2424 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2425 u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2426 struct ifnet *ifp __unused, struct mbuf *m)
2427 {
2428 return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
2429 lookupflags, m->m_pkthdr.numa_domain));
2430 }
2431 #endif /* INET */
2432
2433 static bool
in_pcbjailed(const struct inpcb * inp,unsigned int flag)2434 in_pcbjailed(const struct inpcb *inp, unsigned int flag)
2435 {
2436 return (prison_flag(inp->inp_cred, flag) != 0);
2437 }
2438
2439 /*
2440 * Insert the PCB into a hash chain using ordering rules which ensure that
2441 * in_pcblookup_hash_wild_*() always encounter the highest-ranking PCB first.
2442 *
2443 * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs
2444 * with exact local addresses ahead of wildcard PCBs. Unbound v4-mapped v6 PCBs
2445 * always appear last no matter whether they are jailed.
2446 */
2447 static void
_in_pcbinshash_wild(struct inpcbhead * pcbhash,struct inpcb * inp)2448 _in_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2449 {
2450 struct inpcb *last;
2451 bool bound, injail;
2452
2453 INP_LOCK_ASSERT(inp);
2454 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2455
2456 last = NULL;
2457 bound = inp->inp_laddr.s_addr != INADDR_ANY;
2458 if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) {
2459 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2460 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2461 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2462 return;
2463 }
2464 }
2465 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2466 return;
2467 }
2468
2469 injail = in_pcbjailed(inp, PR_IP4);
2470 if (!injail) {
2471 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2472 if (!in_pcbjailed(last, PR_IP4))
2473 break;
2474 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2475 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2476 return;
2477 }
2478 }
2479 } else if (!CK_LIST_EMPTY(pcbhash) &&
2480 !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP4)) {
2481 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2482 return;
2483 }
2484 if (!bound) {
2485 CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
2486 if (last->inp_laddr.s_addr == INADDR_ANY)
2487 break;
2488 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2489 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2490 return;
2491 }
2492 }
2493 }
2494 if (last == NULL)
2495 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2496 else
2497 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2498 }
2499
2500 #ifdef INET6
2501 /*
2502 * See the comment above _in_pcbinshash_wild().
2503 */
2504 static void
_in6_pcbinshash_wild(struct inpcbhead * pcbhash,struct inpcb * inp)2505 _in6_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2506 {
2507 struct inpcb *last;
2508 bool bound, injail;
2509
2510 INP_LOCK_ASSERT(inp);
2511 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2512
2513 last = NULL;
2514 bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr);
2515 injail = in_pcbjailed(inp, PR_IP6);
2516 if (!injail) {
2517 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2518 if (!in_pcbjailed(last, PR_IP6))
2519 break;
2520 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2521 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2522 return;
2523 }
2524 }
2525 } else if (!CK_LIST_EMPTY(pcbhash) &&
2526 !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP6)) {
2527 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2528 return;
2529 }
2530 if (!bound) {
2531 CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
2532 if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr))
2533 break;
2534 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2535 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2536 return;
2537 }
2538 }
2539 }
2540 if (last == NULL)
2541 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2542 else
2543 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2544 }
2545 #endif
2546
2547 /*
2548 * Insert PCB onto various hash lists.
2549 */
2550 int
in_pcbinshash(struct inpcb * inp)2551 in_pcbinshash(struct inpcb *inp)
2552 {
2553 struct inpcbhead *pcbhash;
2554 struct inpcbporthead *pcbporthash;
2555 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2556 struct inpcbport *phd;
2557 uint32_t hash;
2558 bool connected;
2559
2560 INP_WLOCK_ASSERT(inp);
2561 INP_HASH_WLOCK_ASSERT(pcbinfo);
2562 KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2563 ("in_pcbinshash: INP_INHASHLIST"));
2564
2565 #ifdef INET6
2566 if (inp->inp_vflag & INP_IPV6) {
2567 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2568 inp->inp_fport, pcbinfo->ipi_hashmask);
2569 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2570 } else
2571 #endif
2572 {
2573 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2574 inp->inp_fport, pcbinfo->ipi_hashmask);
2575 connected = !in_nullhost(inp->inp_faddr);
2576 }
2577
2578 if (connected)
2579 pcbhash = &pcbinfo->ipi_hash_exact[hash];
2580 else
2581 pcbhash = &pcbinfo->ipi_hash_wild[hash];
2582
2583 pcbporthash = &pcbinfo->ipi_porthashbase[
2584 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2585
2586 /*
2587 * Add entry to load balance group.
2588 * Only do this if SO_REUSEPORT_LB is set.
2589 */
2590 if ((inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) {
2591 int error = in_pcbinslbgrouphash(inp, M_NODOM);
2592 if (error != 0)
2593 return (error);
2594 }
2595
2596 /*
2597 * Go through port list and look for a head for this lport.
2598 */
2599 CK_LIST_FOREACH(phd, pcbporthash, phd_hash) {
2600 if (phd->phd_port == inp->inp_lport)
2601 break;
2602 }
2603
2604 /*
2605 * If none exists, malloc one and tack it on.
2606 */
2607 if (phd == NULL) {
2608 phd = uma_zalloc_smr(pcbinfo->ipi_portzone, M_NOWAIT);
2609 if (phd == NULL) {
2610 if ((inp->inp_flags & INP_INLBGROUP) != 0)
2611 in_pcbremlbgrouphash(inp);
2612 return (ENOMEM);
2613 }
2614 phd->phd_port = inp->inp_lport;
2615 CK_LIST_INIT(&phd->phd_pcblist);
2616 CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2617 }
2618 inp->inp_phd = phd;
2619 CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2620
2621 /*
2622 * The PCB may have been disconnected in the past. Before we can safely
2623 * make it visible in the hash table, we must wait for all readers which
2624 * may be traversing this PCB to finish.
2625 */
2626 if (inp->inp_smr != SMR_SEQ_INVALID) {
2627 smr_wait(pcbinfo->ipi_smr, inp->inp_smr);
2628 inp->inp_smr = SMR_SEQ_INVALID;
2629 }
2630
2631 if (connected)
2632 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_exact);
2633 else {
2634 #ifdef INET6
2635 if ((inp->inp_vflag & INP_IPV6) != 0)
2636 _in6_pcbinshash_wild(pcbhash, inp);
2637 else
2638 #endif
2639 _in_pcbinshash_wild(pcbhash, inp);
2640 }
2641 inp->inp_flags |= INP_INHASHLIST;
2642
2643 return (0);
2644 }
2645
2646 void
in_pcbremhash_locked(struct inpcb * inp)2647 in_pcbremhash_locked(struct inpcb *inp)
2648 {
2649 struct inpcbport *phd = inp->inp_phd;
2650
2651 INP_WLOCK_ASSERT(inp);
2652 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2653 MPASS(inp->inp_flags & INP_INHASHLIST);
2654
2655 if ((inp->inp_flags & INP_INLBGROUP) != 0)
2656 in_pcbremlbgrouphash(inp);
2657 #ifdef INET6
2658 if (inp->inp_vflag & INP_IPV6) {
2659 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
2660 CK_LIST_REMOVE(inp, inp_hash_wild);
2661 else
2662 CK_LIST_REMOVE(inp, inp_hash_exact);
2663 } else
2664 #endif
2665 {
2666 if (in_nullhost(inp->inp_faddr))
2667 CK_LIST_REMOVE(inp, inp_hash_wild);
2668 else
2669 CK_LIST_REMOVE(inp, inp_hash_exact);
2670 }
2671 CK_LIST_REMOVE(inp, inp_portlist);
2672 if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
2673 CK_LIST_REMOVE(phd, phd_hash);
2674 uma_zfree_smr(inp->inp_pcbinfo->ipi_portzone, phd);
2675 }
2676 inp->inp_flags &= ~INP_INHASHLIST;
2677 }
2678
2679 static void
in_pcbremhash(struct inpcb * inp)2680 in_pcbremhash(struct inpcb *inp)
2681 {
2682 INP_HASH_WLOCK(inp->inp_pcbinfo);
2683 in_pcbremhash_locked(inp);
2684 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
2685 }
2686
2687 /*
2688 * Move PCB to the proper hash bucket when { faddr, fport } have been
2689 * changed. NOTE: This does not handle the case of the lport changing (the
2690 * hashed port list would have to be updated as well), so the lport must
2691 * not change after in_pcbinshash() has been called.
2692 */
2693 void
in_pcbrehash(struct inpcb * inp)2694 in_pcbrehash(struct inpcb *inp)
2695 {
2696 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2697 struct inpcbhead *head;
2698 uint32_t hash;
2699 bool connected;
2700
2701 INP_WLOCK_ASSERT(inp);
2702 INP_HASH_WLOCK_ASSERT(pcbinfo);
2703 KASSERT(inp->inp_flags & INP_INHASHLIST,
2704 ("%s: !INP_INHASHLIST", __func__));
2705 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
2706 ("%s: inp was disconnected", __func__));
2707
2708 #ifdef INET6
2709 if (inp->inp_vflag & INP_IPV6) {
2710 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2711 inp->inp_fport, pcbinfo->ipi_hashmask);
2712 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2713 } else
2714 #endif
2715 {
2716 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2717 inp->inp_fport, pcbinfo->ipi_hashmask);
2718 connected = !in_nullhost(inp->inp_faddr);
2719 }
2720
2721 /*
2722 * When rehashing, the caller must ensure that either the new or the old
2723 * foreign address was unspecified.
2724 */
2725 if (connected)
2726 CK_LIST_REMOVE(inp, inp_hash_wild);
2727 else
2728 CK_LIST_REMOVE(inp, inp_hash_exact);
2729
2730 if (connected) {
2731 head = &pcbinfo->ipi_hash_exact[hash];
2732 CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact);
2733 } else {
2734 head = &pcbinfo->ipi_hash_wild[hash];
2735 CK_LIST_INSERT_HEAD(head, inp, inp_hash_wild);
2736 }
2737 }
2738
2739 /*
2740 * Check for alternatives when higher level complains
2741 * about service problems. For now, invalidate cached
2742 * routing information. If the route was created dynamically
2743 * (by a redirect), time to try a default gateway again.
2744 */
2745 void
in_losing(struct inpcb * inp)2746 in_losing(struct inpcb *inp)
2747 {
2748
2749 RO_INVALIDATE_CACHE(&inp->inp_route);
2750 return;
2751 }
2752
2753 /*
2754 * A set label operation has occurred at the socket layer, propagate the
2755 * label change into the in_pcb for the socket.
2756 */
2757 void
in_pcbsosetlabel(struct socket * so)2758 in_pcbsosetlabel(struct socket *so)
2759 {
2760 #ifdef MAC
2761 struct inpcb *inp;
2762
2763 inp = sotoinpcb(so);
2764 KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2765
2766 INP_WLOCK(inp);
2767 SOCK_LOCK(so);
2768 mac_inpcb_sosetlabel(so, inp);
2769 SOCK_UNLOCK(so);
2770 INP_WUNLOCK(inp);
2771 #endif
2772 }
2773
2774 void
inp_wlock(struct inpcb * inp)2775 inp_wlock(struct inpcb *inp)
2776 {
2777
2778 INP_WLOCK(inp);
2779 }
2780
2781 void
inp_wunlock(struct inpcb * inp)2782 inp_wunlock(struct inpcb *inp)
2783 {
2784
2785 INP_WUNLOCK(inp);
2786 }
2787
2788 void
inp_rlock(struct inpcb * inp)2789 inp_rlock(struct inpcb *inp)
2790 {
2791
2792 INP_RLOCK(inp);
2793 }
2794
2795 void
inp_runlock(struct inpcb * inp)2796 inp_runlock(struct inpcb *inp)
2797 {
2798
2799 INP_RUNLOCK(inp);
2800 }
2801
2802 #ifdef INVARIANT_SUPPORT
2803 void
inp_lock_assert(struct inpcb * inp)2804 inp_lock_assert(struct inpcb *inp)
2805 {
2806
2807 INP_WLOCK_ASSERT(inp);
2808 }
2809
2810 void
inp_unlock_assert(struct inpcb * inp)2811 inp_unlock_assert(struct inpcb *inp)
2812 {
2813
2814 INP_UNLOCK_ASSERT(inp);
2815 }
2816 #endif
2817
2818 void
inp_apply_all(struct inpcbinfo * pcbinfo,void (* func)(struct inpcb *,void *),void * arg)2819 inp_apply_all(struct inpcbinfo *pcbinfo,
2820 void (*func)(struct inpcb *, void *), void *arg)
2821 {
2822 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2823 INPLOOKUP_WLOCKPCB);
2824 struct inpcb *inp;
2825
2826 while ((inp = inp_next(&inpi)) != NULL)
2827 func(inp, arg);
2828 }
2829
2830 struct socket *
inp_inpcbtosocket(struct inpcb * inp)2831 inp_inpcbtosocket(struct inpcb *inp)
2832 {
2833
2834 INP_WLOCK_ASSERT(inp);
2835 return (inp->inp_socket);
2836 }
2837
2838 void
inp_4tuple_get(struct inpcb * inp,uint32_t * laddr,uint16_t * lp,uint32_t * faddr,uint16_t * fp)2839 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2840 uint32_t *faddr, uint16_t *fp)
2841 {
2842
2843 INP_LOCK_ASSERT(inp);
2844 *laddr = inp->inp_laddr.s_addr;
2845 *faddr = inp->inp_faddr.s_addr;
2846 *lp = inp->inp_lport;
2847 *fp = inp->inp_fport;
2848 }
2849
2850 /*
2851 * Create an external-format (``xinpcb'') structure using the information in
2852 * the kernel-format in_pcb structure pointed to by inp. This is done to
2853 * reduce the spew of irrelevant information over this interface, to isolate
2854 * user code from changes in the kernel structure, and potentially to provide
2855 * information-hiding if we decide that some of this information should be
2856 * hidden from users.
2857 */
2858 void
in_pcbtoxinpcb(const struct inpcb * inp,struct xinpcb * xi)2859 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2860 {
2861
2862 bzero(xi, sizeof(*xi));
2863 xi->xi_len = sizeof(struct xinpcb);
2864 if (inp->inp_socket)
2865 sotoxsocket(inp->inp_socket, &xi->xi_socket);
2866 bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2867 xi->inp_gencnt = inp->inp_gencnt;
2868 xi->inp_flow = inp->inp_flow;
2869 xi->inp_flowid = inp->inp_flowid;
2870 xi->inp_flowtype = inp->inp_flowtype;
2871 xi->inp_flags = inp->inp_flags;
2872 xi->inp_flags2 = inp->inp_flags2;
2873 xi->in6p_cksum = inp->in6p_cksum;
2874 xi->in6p_hops = inp->in6p_hops;
2875 xi->inp_ip_tos = inp->inp_ip_tos;
2876 xi->inp_vflag = inp->inp_vflag;
2877 xi->inp_ip_ttl = inp->inp_ip_ttl;
2878 xi->inp_ip_p = inp->inp_ip_p;
2879 xi->inp_ip_minttl = inp->inp_ip_minttl;
2880 }
2881
2882 int
sysctl_setsockopt(SYSCTL_HANDLER_ARGS,struct inpcbinfo * pcbinfo,int (* ctloutput_set)(struct inpcb *,struct sockopt *))2883 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo,
2884 int (*ctloutput_set)(struct inpcb *, struct sockopt *))
2885 {
2886 struct sockopt sopt;
2887 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2888 INPLOOKUP_WLOCKPCB);
2889 struct inpcb *inp;
2890 struct sockopt_parameters *params;
2891 struct socket *so;
2892 int error;
2893 char buf[1024];
2894
2895 if (req->oldptr != NULL || req->oldlen != 0)
2896 return (EINVAL);
2897 if (req->newptr == NULL)
2898 return (EPERM);
2899 if (req->newlen > sizeof(buf))
2900 return (ENOMEM);
2901 error = SYSCTL_IN(req, buf, req->newlen);
2902 if (error != 0)
2903 return (error);
2904 if (req->newlen < sizeof(struct sockopt_parameters))
2905 return (EINVAL);
2906 params = (struct sockopt_parameters *)buf;
2907 sopt.sopt_level = params->sop_level;
2908 sopt.sopt_name = params->sop_optname;
2909 sopt.sopt_dir = SOPT_SET;
2910 sopt.sopt_val = params->sop_optval;
2911 sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters);
2912 sopt.sopt_td = NULL;
2913 #ifdef INET6
2914 if (params->sop_inc.inc_flags & INC_ISIPV6) {
2915 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_laddr))
2916 params->sop_inc.inc6_laddr.s6_addr16[1] =
2917 htons(params->sop_inc.inc6_zoneid & 0xffff);
2918 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_faddr))
2919 params->sop_inc.inc6_faddr.s6_addr16[1] =
2920 htons(params->sop_inc.inc6_zoneid & 0xffff);
2921 }
2922 #endif
2923 if (params->sop_inc.inc_lport != htons(0) &&
2924 params->sop_inc.inc_fport != htons(0)) {
2925 #ifdef INET6
2926 if (params->sop_inc.inc_flags & INC_ISIPV6)
2927 inpi.hash = INP6_PCBHASH(
2928 ¶ms->sop_inc.inc6_faddr,
2929 params->sop_inc.inc_lport,
2930 params->sop_inc.inc_fport,
2931 pcbinfo->ipi_hashmask);
2932 else
2933 #endif
2934 inpi.hash = INP_PCBHASH(
2935 ¶ms->sop_inc.inc_faddr,
2936 params->sop_inc.inc_lport,
2937 params->sop_inc.inc_fport,
2938 pcbinfo->ipi_hashmask);
2939 }
2940 while ((inp = inp_next(&inpi)) != NULL)
2941 if (inp->inp_gencnt == params->sop_id) {
2942 if (inp->inp_flags & INP_DROPPED) {
2943 INP_WUNLOCK(inp);
2944 return (ECONNRESET);
2945 }
2946 so = inp->inp_socket;
2947 KASSERT(so != NULL, ("inp_socket == NULL"));
2948 soref(so);
2949 if (params->sop_level == SOL_SOCKET) {
2950 INP_WUNLOCK(inp);
2951 error = sosetopt(so, &sopt);
2952 } else
2953 error = (*ctloutput_set)(inp, &sopt);
2954 sorele(so);
2955 break;
2956 }
2957 if (inp == NULL)
2958 error = ESRCH;
2959 return (error);
2960 }
2961
2962 #ifdef DDB
2963 static void
db_print_indent(int indent)2964 db_print_indent(int indent)
2965 {
2966 int i;
2967
2968 for (i = 0; i < indent; i++)
2969 db_printf(" ");
2970 }
2971
2972 static void
db_print_inconninfo(struct in_conninfo * inc,const char * name,int indent)2973 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
2974 {
2975 char faddr_str[48], laddr_str[48];
2976
2977 db_print_indent(indent);
2978 db_printf("%s at %p\n", name, inc);
2979
2980 indent += 2;
2981
2982 #ifdef INET6
2983 if (inc->inc_flags & INC_ISIPV6) {
2984 /* IPv6. */
2985 ip6_sprintf(laddr_str, &inc->inc6_laddr);
2986 ip6_sprintf(faddr_str, &inc->inc6_faddr);
2987 } else
2988 #endif
2989 {
2990 /* IPv4. */
2991 inet_ntoa_r(inc->inc_laddr, laddr_str);
2992 inet_ntoa_r(inc->inc_faddr, faddr_str);
2993 }
2994 db_print_indent(indent);
2995 db_printf("inc_laddr %s inc_lport %u\n", laddr_str,
2996 ntohs(inc->inc_lport));
2997 db_print_indent(indent);
2998 db_printf("inc_faddr %s inc_fport %u\n", faddr_str,
2999 ntohs(inc->inc_fport));
3000 }
3001
3002 static void
db_print_inpflags(int inp_flags)3003 db_print_inpflags(int inp_flags)
3004 {
3005 int comma;
3006
3007 comma = 0;
3008 if (inp_flags & INP_RECVOPTS) {
3009 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
3010 comma = 1;
3011 }
3012 if (inp_flags & INP_RECVRETOPTS) {
3013 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
3014 comma = 1;
3015 }
3016 if (inp_flags & INP_RECVDSTADDR) {
3017 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
3018 comma = 1;
3019 }
3020 if (inp_flags & INP_ORIGDSTADDR) {
3021 db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
3022 comma = 1;
3023 }
3024 if (inp_flags & INP_HDRINCL) {
3025 db_printf("%sINP_HDRINCL", comma ? ", " : "");
3026 comma = 1;
3027 }
3028 if (inp_flags & INP_HIGHPORT) {
3029 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
3030 comma = 1;
3031 }
3032 if (inp_flags & INP_LOWPORT) {
3033 db_printf("%sINP_LOWPORT", comma ? ", " : "");
3034 comma = 1;
3035 }
3036 if (inp_flags & INP_ANONPORT) {
3037 db_printf("%sINP_ANONPORT", comma ? ", " : "");
3038 comma = 1;
3039 }
3040 if (inp_flags & INP_RECVIF) {
3041 db_printf("%sINP_RECVIF", comma ? ", " : "");
3042 comma = 1;
3043 }
3044 if (inp_flags & INP_MTUDISC) {
3045 db_printf("%sINP_MTUDISC", comma ? ", " : "");
3046 comma = 1;
3047 }
3048 if (inp_flags & INP_RECVTTL) {
3049 db_printf("%sINP_RECVTTL", comma ? ", " : "");
3050 comma = 1;
3051 }
3052 if (inp_flags & INP_DONTFRAG) {
3053 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
3054 comma = 1;
3055 }
3056 if (inp_flags & INP_RECVTOS) {
3057 db_printf("%sINP_RECVTOS", comma ? ", " : "");
3058 comma = 1;
3059 }
3060 if (inp_flags & IN6P_IPV6_V6ONLY) {
3061 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
3062 comma = 1;
3063 }
3064 if (inp_flags & IN6P_PKTINFO) {
3065 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
3066 comma = 1;
3067 }
3068 if (inp_flags & IN6P_HOPLIMIT) {
3069 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
3070 comma = 1;
3071 }
3072 if (inp_flags & IN6P_HOPOPTS) {
3073 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
3074 comma = 1;
3075 }
3076 if (inp_flags & IN6P_DSTOPTS) {
3077 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
3078 comma = 1;
3079 }
3080 if (inp_flags & IN6P_RTHDR) {
3081 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
3082 comma = 1;
3083 }
3084 if (inp_flags & IN6P_RTHDRDSTOPTS) {
3085 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
3086 comma = 1;
3087 }
3088 if (inp_flags & IN6P_TCLASS) {
3089 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
3090 comma = 1;
3091 }
3092 if (inp_flags & IN6P_AUTOFLOWLABEL) {
3093 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
3094 comma = 1;
3095 }
3096 if (inp_flags & INP_ONESBCAST) {
3097 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
3098 comma = 1;
3099 }
3100 if (inp_flags & INP_DROPPED) {
3101 db_printf("%sINP_DROPPED", comma ? ", " : "");
3102 comma = 1;
3103 }
3104 if (inp_flags & INP_SOCKREF) {
3105 db_printf("%sINP_SOCKREF", comma ? ", " : "");
3106 comma = 1;
3107 }
3108 if (inp_flags & IN6P_RFC2292) {
3109 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
3110 comma = 1;
3111 }
3112 if (inp_flags & IN6P_MTU) {
3113 db_printf("IN6P_MTU%s", comma ? ", " : "");
3114 comma = 1;
3115 }
3116 }
3117
3118 static void
db_print_inpvflag(u_char inp_vflag)3119 db_print_inpvflag(u_char inp_vflag)
3120 {
3121 int comma;
3122
3123 comma = 0;
3124 if (inp_vflag & INP_IPV4) {
3125 db_printf("%sINP_IPV4", comma ? ", " : "");
3126 comma = 1;
3127 }
3128 if (inp_vflag & INP_IPV6) {
3129 db_printf("%sINP_IPV6", comma ? ", " : "");
3130 comma = 1;
3131 }
3132 if (inp_vflag & INP_IPV6PROTO) {
3133 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
3134 comma = 1;
3135 }
3136 }
3137
3138 static void
db_print_inpcb(struct inpcb * inp,const char * name,int indent)3139 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3140 {
3141
3142 db_print_indent(indent);
3143 db_printf("%s at %p\n", name, inp);
3144
3145 indent += 2;
3146
3147 db_print_indent(indent);
3148 db_printf("inp_flow: 0x%x\n", inp->inp_flow);
3149
3150 db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3151
3152 db_print_indent(indent);
3153 db_printf("inp_label: %p inp_flags: 0x%x (",
3154 inp->inp_label, inp->inp_flags);
3155 db_print_inpflags(inp->inp_flags);
3156 db_printf(")\n");
3157
3158 db_print_indent(indent);
3159 db_printf("inp_sp: %p inp_vflag: 0x%x (", inp->inp_sp,
3160 inp->inp_vflag);
3161 db_print_inpvflag(inp->inp_vflag);
3162 db_printf(")\n");
3163
3164 db_print_indent(indent);
3165 db_printf("inp_ip_ttl: %d inp_ip_p: %d inp_ip_minttl: %d\n",
3166 inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3167
3168 db_print_indent(indent);
3169 #ifdef INET6
3170 if (inp->inp_vflag & INP_IPV6) {
3171 db_printf("in6p_options: %p in6p_outputopts: %p "
3172 "in6p_moptions: %p\n", inp->in6p_options,
3173 inp->in6p_outputopts, inp->in6p_moptions);
3174 db_printf("in6p_icmp6filt: %p in6p_cksum %d "
3175 "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3176 inp->in6p_hops);
3177 } else
3178 #endif
3179 {
3180 db_printf("inp_ip_tos: %d inp_ip_options: %p "
3181 "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3182 inp->inp_options, inp->inp_moptions);
3183 }
3184
3185 db_print_indent(indent);
3186 db_printf("inp_phd: %p inp_gencnt: %ju\n", inp->inp_phd,
3187 (uintmax_t)inp->inp_gencnt);
3188 }
3189
DB_SHOW_COMMAND(inpcb,db_show_inpcb)3190 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
3191 {
3192 struct inpcb *inp;
3193
3194 if (!have_addr) {
3195 db_printf("usage: show inpcb <addr>\n");
3196 return;
3197 }
3198 inp = (struct inpcb *)addr;
3199
3200 db_print_inpcb(inp, "inpcb", 0);
3201 }
3202 #endif /* DDB */
3203
3204 #ifdef RATELIMIT
3205 /*
3206 * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3207 * if any.
3208 */
3209 int
in_pcbmodify_txrtlmt(struct inpcb * inp,uint32_t max_pacing_rate)3210 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3211 {
3212 union if_snd_tag_modify_params params = {
3213 .rate_limit.max_rate = max_pacing_rate,
3214 .rate_limit.flags = M_NOWAIT,
3215 };
3216 struct m_snd_tag *mst;
3217 int error;
3218
3219 mst = inp->inp_snd_tag;
3220 if (mst == NULL)
3221 return (EINVAL);
3222
3223 if (mst->sw->snd_tag_modify == NULL) {
3224 error = EOPNOTSUPP;
3225 } else {
3226 error = mst->sw->snd_tag_modify(mst, ¶ms);
3227 }
3228 return (error);
3229 }
3230
3231 /*
3232 * Query existing TX rate limit based on the existing
3233 * "inp->inp_snd_tag", if any.
3234 */
3235 int
in_pcbquery_txrtlmt(struct inpcb * inp,uint32_t * p_max_pacing_rate)3236 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3237 {
3238 union if_snd_tag_query_params params = { };
3239 struct m_snd_tag *mst;
3240 int error;
3241
3242 mst = inp->inp_snd_tag;
3243 if (mst == NULL)
3244 return (EINVAL);
3245
3246 if (mst->sw->snd_tag_query == NULL) {
3247 error = EOPNOTSUPP;
3248 } else {
3249 error = mst->sw->snd_tag_query(mst, ¶ms);
3250 if (error == 0 && p_max_pacing_rate != NULL)
3251 *p_max_pacing_rate = params.rate_limit.max_rate;
3252 }
3253 return (error);
3254 }
3255
3256 /*
3257 * Query existing TX queue level based on the existing
3258 * "inp->inp_snd_tag", if any.
3259 */
3260 int
in_pcbquery_txrlevel(struct inpcb * inp,uint32_t * p_txqueue_level)3261 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3262 {
3263 union if_snd_tag_query_params params = { };
3264 struct m_snd_tag *mst;
3265 int error;
3266
3267 mst = inp->inp_snd_tag;
3268 if (mst == NULL)
3269 return (EINVAL);
3270
3271 if (mst->sw->snd_tag_query == NULL)
3272 return (EOPNOTSUPP);
3273
3274 error = mst->sw->snd_tag_query(mst, ¶ms);
3275 if (error == 0 && p_txqueue_level != NULL)
3276 *p_txqueue_level = params.rate_limit.queue_level;
3277 return (error);
3278 }
3279
3280 /*
3281 * Allocate a new TX rate limit send tag from the network interface
3282 * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3283 */
3284 int
in_pcbattach_txrtlmt(struct inpcb * inp,struct ifnet * ifp,uint32_t flowtype,uint32_t flowid,uint32_t max_pacing_rate,struct m_snd_tag ** st)3285 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3286 uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st)
3287
3288 {
3289 union if_snd_tag_alloc_params params = {
3290 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3291 IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3292 .rate_limit.hdr.flowid = flowid,
3293 .rate_limit.hdr.flowtype = flowtype,
3294 .rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3295 .rate_limit.max_rate = max_pacing_rate,
3296 .rate_limit.flags = M_NOWAIT,
3297 };
3298 int error;
3299
3300 INP_WLOCK_ASSERT(inp);
3301
3302 /*
3303 * If there is already a send tag, or the INP is being torn
3304 * down, allocating a new send tag is not allowed. Else send
3305 * tags may leak.
3306 */
3307 if (*st != NULL || (inp->inp_flags & INP_DROPPED) != 0)
3308 return (EINVAL);
3309
3310 error = m_snd_tag_alloc(ifp, ¶ms, st);
3311 #ifdef INET
3312 if (error == 0) {
3313 counter_u64_add(rate_limit_set_ok, 1);
3314 counter_u64_add(rate_limit_active, 1);
3315 } else if (error != EOPNOTSUPP)
3316 counter_u64_add(rate_limit_alloc_fail, 1);
3317 #endif
3318 return (error);
3319 }
3320
3321 void
in_pcbdetach_tag(struct m_snd_tag * mst)3322 in_pcbdetach_tag(struct m_snd_tag *mst)
3323 {
3324
3325 m_snd_tag_rele(mst);
3326 #ifdef INET
3327 counter_u64_add(rate_limit_active, -1);
3328 #endif
3329 }
3330
3331 /*
3332 * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3333 * if any:
3334 */
3335 void
in_pcbdetach_txrtlmt(struct inpcb * inp)3336 in_pcbdetach_txrtlmt(struct inpcb *inp)
3337 {
3338 struct m_snd_tag *mst;
3339
3340 INP_WLOCK_ASSERT(inp);
3341
3342 mst = inp->inp_snd_tag;
3343 inp->inp_snd_tag = NULL;
3344
3345 if (mst == NULL)
3346 return;
3347
3348 m_snd_tag_rele(mst);
3349 #ifdef INET
3350 counter_u64_add(rate_limit_active, -1);
3351 #endif
3352 }
3353
3354 int
in_pcboutput_txrtlmt_locked(struct inpcb * inp,struct ifnet * ifp,struct mbuf * mb,uint32_t max_pacing_rate)3355 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3356 {
3357 int error;
3358
3359 /*
3360 * If the existing send tag is for the wrong interface due to
3361 * a route change, first drop the existing tag. Set the
3362 * CHANGED flag so that we will keep trying to allocate a new
3363 * tag if we fail to allocate one this time.
3364 */
3365 if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3366 in_pcbdetach_txrtlmt(inp);
3367 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3368 }
3369
3370 /*
3371 * NOTE: When attaching to a network interface a reference is
3372 * made to ensure the network interface doesn't go away until
3373 * all ratelimit connections are gone. The network interface
3374 * pointers compared below represent valid network interfaces,
3375 * except when comparing towards NULL.
3376 */
3377 if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3378 error = 0;
3379 } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3380 if (inp->inp_snd_tag != NULL)
3381 in_pcbdetach_txrtlmt(inp);
3382 error = 0;
3383 } else if (inp->inp_snd_tag == NULL) {
3384 /*
3385 * In order to utilize packet pacing with RSS, we need
3386 * to wait until there is a valid RSS hash before we
3387 * can proceed:
3388 */
3389 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3390 error = EAGAIN;
3391 } else {
3392 error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3393 mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3394 }
3395 } else {
3396 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3397 }
3398 if (error == 0 || error == EOPNOTSUPP)
3399 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3400
3401 return (error);
3402 }
3403
3404 /*
3405 * This function should be called when the INP_RATE_LIMIT_CHANGED flag
3406 * is set in the fast path and will attach/detach/modify the TX rate
3407 * limit send tag based on the socket's so_max_pacing_rate value.
3408 */
3409 void
in_pcboutput_txrtlmt(struct inpcb * inp,struct ifnet * ifp,struct mbuf * mb)3410 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3411 {
3412 struct socket *socket;
3413 uint32_t max_pacing_rate;
3414 bool did_upgrade;
3415
3416 if (inp == NULL)
3417 return;
3418
3419 socket = inp->inp_socket;
3420 if (socket == NULL)
3421 return;
3422
3423 if (!INP_WLOCKED(inp)) {
3424 /*
3425 * NOTE: If the write locking fails, we need to bail
3426 * out and use the non-ratelimited ring for the
3427 * transmit until there is a new chance to get the
3428 * write lock.
3429 */
3430 if (!INP_TRY_UPGRADE(inp))
3431 return;
3432 did_upgrade = 1;
3433 } else {
3434 did_upgrade = 0;
3435 }
3436
3437 /*
3438 * NOTE: The so_max_pacing_rate value is read unlocked,
3439 * because atomic updates are not required since the variable
3440 * is checked at every mbuf we send. It is assumed that the
3441 * variable read itself will be atomic.
3442 */
3443 max_pacing_rate = socket->so_max_pacing_rate;
3444
3445 in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3446
3447 if (did_upgrade)
3448 INP_DOWNGRADE(inp);
3449 }
3450
3451 /*
3452 * Track route changes for TX rate limiting.
3453 */
3454 void
in_pcboutput_eagain(struct inpcb * inp)3455 in_pcboutput_eagain(struct inpcb *inp)
3456 {
3457 bool did_upgrade;
3458
3459 if (inp == NULL)
3460 return;
3461
3462 if (inp->inp_snd_tag == NULL)
3463 return;
3464
3465 if (!INP_WLOCKED(inp)) {
3466 /*
3467 * NOTE: If the write locking fails, we need to bail
3468 * out and use the non-ratelimited ring for the
3469 * transmit until there is a new chance to get the
3470 * write lock.
3471 */
3472 if (!INP_TRY_UPGRADE(inp))
3473 return;
3474 did_upgrade = 1;
3475 } else {
3476 did_upgrade = 0;
3477 }
3478
3479 /* detach rate limiting */
3480 in_pcbdetach_txrtlmt(inp);
3481
3482 /* make sure new mbuf send tag allocation is made */
3483 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3484
3485 if (did_upgrade)
3486 INP_DOWNGRADE(inp);
3487 }
3488
3489 #ifdef INET
3490 static void
rl_init(void * st)3491 rl_init(void *st)
3492 {
3493 rate_limit_new = counter_u64_alloc(M_WAITOK);
3494 rate_limit_chg = counter_u64_alloc(M_WAITOK);
3495 rate_limit_active = counter_u64_alloc(M_WAITOK);
3496 rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3497 rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3498 }
3499
3500 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
3501 #endif
3502 #endif /* RATELIMIT */
3503