1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1991, 1993, 1995 5 * The Regents of the University of California. 6 * Copyright (c) 2007-2009 Robert N. M. Watson 7 * Copyright (c) 2010-2011 Juniper Networks, Inc. 8 * All rights reserved. 9 * 10 * Portions of this software were developed by Robert N. M. Watson under 11 * contract to Juniper Networks, Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 38 */ 39 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include "opt_ddb.h" 44 #include "opt_ipsec.h" 45 #include "opt_inet.h" 46 #include "opt_inet6.h" 47 #include "opt_ratelimit.h" 48 #include "opt_route.h" 49 #include "opt_rss.h" 50 51 #include <sys/param.h> 52 #include <sys/hash.h> 53 #include <sys/systm.h> 54 #include <sys/libkern.h> 55 #include <sys/lock.h> 56 #include <sys/malloc.h> 57 #include <sys/mbuf.h> 58 #include <sys/eventhandler.h> 59 #include <sys/domain.h> 60 #include <sys/protosw.h> 61 #include <sys/smp.h> 62 #include <sys/socket.h> 63 #include <sys/socketvar.h> 64 #include <sys/sockio.h> 65 #include <sys/priv.h> 66 #include <sys/proc.h> 67 #include <sys/refcount.h> 68 #include <sys/jail.h> 69 #include <sys/kernel.h> 70 #include <sys/sysctl.h> 71 72 #ifdef DDB 73 #include <ddb/ddb.h> 74 #endif 75 76 #include <vm/uma.h> 77 #include <vm/vm.h> 78 79 #include <net/if.h> 80 #include <net/if_var.h> 81 #include <net/if_types.h> 82 #include <net/if_llatbl.h> 83 #include <net/route.h> 84 #include <net/rss_config.h> 85 #include <net/vnet.h> 86 87 #if defined(INET) || defined(INET6) 88 #include <netinet/in.h> 89 #include <netinet/in_pcb.h> 90 #include <netinet/in_pcb_var.h> 91 #include <netinet/tcp.h> 92 #ifdef INET 93 #include <netinet/in_var.h> 94 #include <netinet/in_fib.h> 95 #endif 96 #include <netinet/ip_var.h> 97 #ifdef INET6 98 #include <netinet/ip6.h> 99 #include <netinet6/in6_pcb.h> 100 #include <netinet6/in6_var.h> 101 #include <netinet6/ip6_var.h> 102 #endif /* INET6 */ 103 #include <net/route/nhop.h> 104 #endif 105 106 #include <netipsec/ipsec_support.h> 107 108 #include <security/mac/mac_framework.h> 109 110 #define INPCBLBGROUP_SIZMIN 8 111 #define INPCBLBGROUP_SIZMAX 256 112 #define INP_FREED 0x00000200 /* See in_pcb.h. */ 113 114 /* 115 * These configure the range of local port addresses assigned to 116 * "unspecified" outgoing connections/packets/whatever. 117 */ 118 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1; /* 1023 */ 119 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART; /* 600 */ 120 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST; /* 10000 */ 121 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST; /* 65535 */ 122 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO; /* 49152 */ 123 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO; /* 65535 */ 124 125 /* 126 * Reserved ports accessible only to root. There are significant 127 * security considerations that must be accounted for when changing these, 128 * but the security benefits can be great. Please be careful. 129 */ 130 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */ 131 VNET_DEFINE(int, ipport_reservedlow); 132 133 /* Enable random ephemeral port allocation by default. */ 134 VNET_DEFINE(int, ipport_randomized) = 1; 135 136 #ifdef INET 137 static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, 138 struct in_addr faddr, u_int fport_arg, 139 struct in_addr laddr, u_int lport_arg, 140 int lookupflags, struct ifnet *ifp, 141 uint8_t numa_domain); 142 143 #define RANGECHK(var, min, max) \ 144 if ((var) < (min)) { (var) = (min); } \ 145 else if ((var) > (max)) { (var) = (max); } 146 147 static int 148 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 149 { 150 int error; 151 152 error = sysctl_handle_int(oidp, arg1, arg2, req); 153 if (error == 0) { 154 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 155 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 156 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX); 157 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX); 158 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX); 159 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX); 160 } 161 return (error); 162 } 163 164 #undef RANGECHK 165 166 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, 167 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 168 "IP Ports"); 169 170 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, 171 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 172 &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I", 173 ""); 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, 175 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 176 &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I", 177 ""); 178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, 179 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 180 &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I", 181 ""); 182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, 183 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 184 &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I", 185 ""); 186 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, 187 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 188 &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I", 189 ""); 190 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, 191 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 192 &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I", 193 ""); 194 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh, 195 CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE, 196 &VNET_NAME(ipport_reservedhigh), 0, ""); 197 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow, 198 CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, ""); 199 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized, 200 CTLFLAG_VNET | CTLFLAG_RW, 201 &VNET_NAME(ipport_randomized), 0, "Enable random port allocation"); 202 203 #ifdef RATELIMIT 204 counter_u64_t rate_limit_new; 205 counter_u64_t rate_limit_chg; 206 counter_u64_t rate_limit_active; 207 counter_u64_t rate_limit_alloc_fail; 208 counter_u64_t rate_limit_set_ok; 209 210 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 211 "IP Rate Limiting"); 212 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD, 213 &rate_limit_active, "Active rate limited connections"); 214 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD, 215 &rate_limit_alloc_fail, "Rate limited connection failures"); 216 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD, 217 &rate_limit_set_ok, "Rate limited setting succeeded"); 218 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD, 219 &rate_limit_new, "Total Rate limit new attempts"); 220 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD, 221 &rate_limit_chg, "Total Rate limited change attempts"); 222 223 #endif /* RATELIMIT */ 224 225 #endif /* INET */ 226 227 VNET_DEFINE(uint32_t, in_pcbhashseed); 228 static void 229 in_pcbhashseed_init(void) 230 { 231 232 V_in_pcbhashseed = arc4random(); 233 } 234 VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, 235 in_pcbhashseed_init, 0); 236 237 static void in_pcbremhash(struct inpcb *); 238 239 /* 240 * in_pcb.c: manage the Protocol Control Blocks. 241 * 242 * NOTE: It is assumed that most of these functions will be called with 243 * the pcbinfo lock held, and often, the inpcb lock held, as these utility 244 * functions often modify hash chains or addresses in pcbs. 245 */ 246 247 static struct inpcblbgroup * 248 in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, struct ucred *cred, 249 u_char vflag, uint16_t port, const union in_dependaddr *addr, int size, 250 uint8_t numa_domain) 251 { 252 struct inpcblbgroup *grp; 253 size_t bytes; 254 255 bytes = __offsetof(struct inpcblbgroup, il_inp[size]); 256 grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT); 257 if (grp == NULL) 258 return (NULL); 259 grp->il_cred = crhold(cred); 260 grp->il_vflag = vflag; 261 grp->il_lport = port; 262 grp->il_numa_domain = numa_domain; 263 grp->il_dependladdr = *addr; 264 grp->il_inpsiz = size; 265 CK_LIST_INSERT_HEAD(hdr, grp, il_list); 266 return (grp); 267 } 268 269 static void 270 in_pcblbgroup_free_deferred(epoch_context_t ctx) 271 { 272 struct inpcblbgroup *grp; 273 274 grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx); 275 crfree(grp->il_cred); 276 free(grp, M_PCB); 277 } 278 279 static void 280 in_pcblbgroup_free(struct inpcblbgroup *grp) 281 { 282 283 CK_LIST_REMOVE(grp, il_list); 284 NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx); 285 } 286 287 static struct inpcblbgroup * 288 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr, 289 struct inpcblbgroup *old_grp, int size) 290 { 291 struct inpcblbgroup *grp; 292 int i; 293 294 grp = in_pcblbgroup_alloc(hdr, old_grp->il_cred, old_grp->il_vflag, 295 old_grp->il_lport, &old_grp->il_dependladdr, size, 296 old_grp->il_numa_domain); 297 if (grp == NULL) 298 return (NULL); 299 300 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 301 ("invalid new local group size %d and old local group count %d", 302 grp->il_inpsiz, old_grp->il_inpcnt)); 303 304 for (i = 0; i < old_grp->il_inpcnt; ++i) 305 grp->il_inp[i] = old_grp->il_inp[i]; 306 grp->il_inpcnt = old_grp->il_inpcnt; 307 in_pcblbgroup_free(old_grp); 308 return (grp); 309 } 310 311 /* 312 * PCB at index 'i' is removed from the group. Pull up the ones below il_inp[i] 313 * and shrink group if possible. 314 */ 315 static void 316 in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp, 317 int i) 318 { 319 struct inpcblbgroup *grp, *new_grp; 320 321 grp = *grpp; 322 for (; i + 1 < grp->il_inpcnt; ++i) 323 grp->il_inp[i] = grp->il_inp[i + 1]; 324 grp->il_inpcnt--; 325 326 if (grp->il_inpsiz > INPCBLBGROUP_SIZMIN && 327 grp->il_inpcnt <= grp->il_inpsiz / 4) { 328 /* Shrink this group. */ 329 new_grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz / 2); 330 if (new_grp != NULL) 331 *grpp = new_grp; 332 } 333 } 334 335 /* 336 * Add PCB to load balance group for SO_REUSEPORT_LB option. 337 */ 338 static int 339 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain) 340 { 341 const static struct timeval interval = { 60, 0 }; 342 static struct timeval lastprint; 343 struct inpcbinfo *pcbinfo; 344 struct inpcblbgrouphead *hdr; 345 struct inpcblbgroup *grp; 346 uint32_t idx; 347 348 pcbinfo = inp->inp_pcbinfo; 349 350 INP_WLOCK_ASSERT(inp); 351 INP_HASH_WLOCK_ASSERT(pcbinfo); 352 353 #ifdef INET6 354 /* 355 * Don't allow IPv4 mapped INET6 wild socket. 356 */ 357 if ((inp->inp_vflag & INP_IPV4) && 358 inp->inp_laddr.s_addr == INADDR_ANY && 359 INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) { 360 return (0); 361 } 362 #endif 363 364 idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask); 365 hdr = &pcbinfo->ipi_lbgrouphashbase[idx]; 366 CK_LIST_FOREACH(grp, hdr, il_list) { 367 if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison && 368 grp->il_vflag == inp->inp_vflag && 369 grp->il_lport == inp->inp_lport && 370 grp->il_numa_domain == numa_domain && 371 memcmp(&grp->il_dependladdr, 372 &inp->inp_inc.inc_ie.ie_dependladdr, 373 sizeof(grp->il_dependladdr)) == 0) { 374 break; 375 } 376 } 377 if (grp == NULL) { 378 /* Create new load balance group. */ 379 grp = in_pcblbgroup_alloc(hdr, inp->inp_cred, inp->inp_vflag, 380 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 381 INPCBLBGROUP_SIZMIN, numa_domain); 382 if (grp == NULL) 383 return (ENOBUFS); 384 } else if (grp->il_inpcnt == grp->il_inpsiz) { 385 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) { 386 if (ratecheck(&lastprint, &interval)) 387 printf("lb group port %d, limit reached\n", 388 ntohs(grp->il_lport)); 389 return (0); 390 } 391 392 /* Expand this local group. */ 393 grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2); 394 if (grp == NULL) 395 return (ENOBUFS); 396 } 397 398 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 399 ("invalid local group size %d and count %d", grp->il_inpsiz, 400 grp->il_inpcnt)); 401 402 grp->il_inp[grp->il_inpcnt] = inp; 403 grp->il_inpcnt++; 404 return (0); 405 } 406 407 /* 408 * Remove PCB from load balance group. 409 */ 410 static void 411 in_pcbremlbgrouphash(struct inpcb *inp) 412 { 413 struct inpcbinfo *pcbinfo; 414 struct inpcblbgrouphead *hdr; 415 struct inpcblbgroup *grp; 416 int i; 417 418 pcbinfo = inp->inp_pcbinfo; 419 420 INP_WLOCK_ASSERT(inp); 421 INP_HASH_WLOCK_ASSERT(pcbinfo); 422 423 hdr = &pcbinfo->ipi_lbgrouphashbase[ 424 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)]; 425 CK_LIST_FOREACH(grp, hdr, il_list) { 426 for (i = 0; i < grp->il_inpcnt; ++i) { 427 if (grp->il_inp[i] != inp) 428 continue; 429 430 if (grp->il_inpcnt == 1) { 431 /* We are the last, free this local group. */ 432 in_pcblbgroup_free(grp); 433 } else { 434 /* Pull up inpcbs, shrink group if possible. */ 435 in_pcblbgroup_reorder(hdr, &grp, i); 436 } 437 return; 438 } 439 } 440 } 441 442 int 443 in_pcblbgroup_numa(struct inpcb *inp, int arg) 444 { 445 struct inpcbinfo *pcbinfo; 446 struct inpcblbgrouphead *hdr; 447 struct inpcblbgroup *grp; 448 int err, i; 449 uint8_t numa_domain; 450 451 switch (arg) { 452 case TCP_REUSPORT_LB_NUMA_NODOM: 453 numa_domain = M_NODOM; 454 break; 455 case TCP_REUSPORT_LB_NUMA_CURDOM: 456 numa_domain = PCPU_GET(domain); 457 break; 458 default: 459 if (arg < 0 || arg >= vm_ndomains) 460 return (EINVAL); 461 numa_domain = arg; 462 } 463 464 err = 0; 465 pcbinfo = inp->inp_pcbinfo; 466 INP_WLOCK_ASSERT(inp); 467 INP_HASH_WLOCK(pcbinfo); 468 hdr = &pcbinfo->ipi_lbgrouphashbase[ 469 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)]; 470 CK_LIST_FOREACH(grp, hdr, il_list) { 471 for (i = 0; i < grp->il_inpcnt; ++i) { 472 if (grp->il_inp[i] != inp) 473 continue; 474 475 if (grp->il_numa_domain == numa_domain) { 476 goto abort_with_hash_wlock; 477 } 478 479 /* Remove it from the old group. */ 480 in_pcbremlbgrouphash(inp); 481 482 /* Add it to the new group based on numa domain. */ 483 in_pcbinslbgrouphash(inp, numa_domain); 484 goto abort_with_hash_wlock; 485 } 486 } 487 err = ENOENT; 488 abort_with_hash_wlock: 489 INP_HASH_WUNLOCK(pcbinfo); 490 return (err); 491 } 492 493 /* Make sure it is safe to use hashinit(9) on CK_LIST. */ 494 CTASSERT(sizeof(struct inpcbhead) == sizeof(LIST_HEAD(, inpcb))); 495 496 /* 497 * Initialize an inpcbinfo - a per-VNET instance of connections db. 498 */ 499 void 500 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor, 501 u_int hash_nelements, u_int porthash_nelements) 502 { 503 504 mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF); 505 mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name, 506 NULL, MTX_DEF); 507 #ifdef VIMAGE 508 pcbinfo->ipi_vnet = curvnet; 509 #endif 510 CK_LIST_INIT(&pcbinfo->ipi_listhead); 511 pcbinfo->ipi_count = 0; 512 pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB, 513 &pcbinfo->ipi_hashmask); 514 porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1); 515 pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB, 516 &pcbinfo->ipi_porthashmask); 517 pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB, 518 &pcbinfo->ipi_lbgrouphashmask); 519 pcbinfo->ipi_zone = pcbstor->ips_zone; 520 pcbinfo->ipi_portzone = pcbstor->ips_portzone; 521 pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone); 522 } 523 524 /* 525 * Destroy an inpcbinfo. 526 */ 527 void 528 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo) 529 { 530 531 KASSERT(pcbinfo->ipi_count == 0, 532 ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count)); 533 534 hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask); 535 hashdestroy(pcbinfo->ipi_porthashbase, M_PCB, 536 pcbinfo->ipi_porthashmask); 537 hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB, 538 pcbinfo->ipi_lbgrouphashmask); 539 mtx_destroy(&pcbinfo->ipi_hash_lock); 540 mtx_destroy(&pcbinfo->ipi_lock); 541 } 542 543 /* 544 * Initialize a pcbstorage - per protocol zones to allocate inpcbs. 545 */ 546 static void inpcb_dtor(void *, int, void *); 547 static void inpcb_fini(void *, int); 548 void 549 in_pcbstorage_init(void *arg) 550 { 551 struct inpcbstorage *pcbstor = arg; 552 553 pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name, 554 sizeof(struct inpcb), NULL, inpcb_dtor, pcbstor->ips_pcbinit, 555 inpcb_fini, UMA_ALIGN_PTR, UMA_ZONE_SMR); 556 pcbstor->ips_portzone = uma_zcreate(pcbstor->ips_portzone_name, 557 sizeof(struct inpcbport), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 558 uma_zone_set_smr(pcbstor->ips_portzone, 559 uma_zone_get_smr(pcbstor->ips_zone)); 560 } 561 562 /* 563 * Destroy a pcbstorage - used by unloadable protocols. 564 */ 565 void 566 in_pcbstorage_destroy(void *arg) 567 { 568 struct inpcbstorage *pcbstor = arg; 569 570 uma_zdestroy(pcbstor->ips_zone); 571 uma_zdestroy(pcbstor->ips_portzone); 572 } 573 574 /* 575 * Allocate a PCB and associate it with the socket. 576 * On success return with the PCB locked. 577 */ 578 int 579 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 580 { 581 struct inpcb *inp; 582 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC) 583 int error; 584 #endif 585 586 inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT); 587 if (inp == NULL) 588 return (ENOBUFS); 589 bzero(&inp->inp_start_zero, inp_zero_size); 590 #ifdef NUMA 591 inp->inp_numa_domain = M_NODOM; 592 #endif 593 inp->inp_pcbinfo = pcbinfo; 594 inp->inp_socket = so; 595 inp->inp_cred = crhold(so->so_cred); 596 inp->inp_inc.inc_fibnum = so->so_fibnum; 597 #ifdef MAC 598 error = mac_inpcb_init(inp, M_NOWAIT); 599 if (error != 0) 600 goto out; 601 mac_inpcb_create(so, inp); 602 #endif 603 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 604 error = ipsec_init_pcbpolicy(inp); 605 if (error != 0) { 606 #ifdef MAC 607 mac_inpcb_destroy(inp); 608 #endif 609 goto out; 610 } 611 #endif /*IPSEC*/ 612 #ifdef INET6 613 if (INP_SOCKAF(so) == AF_INET6) { 614 inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6; 615 if (V_ip6_v6only) 616 inp->inp_flags |= IN6P_IPV6_V6ONLY; 617 #ifdef INET 618 else 619 inp->inp_vflag |= INP_IPV4; 620 #endif 621 if (V_ip6_auto_flowlabel) 622 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 623 inp->in6p_hops = -1; /* use kernel default */ 624 } 625 #endif 626 #if defined(INET) && defined(INET6) 627 else 628 #endif 629 #ifdef INET 630 inp->inp_vflag |= INP_IPV4; 631 #endif 632 /* 633 * Routes in inpcb's can cache L2 as well; they are guaranteed 634 * to be cleaned up. 635 */ 636 inp->inp_route.ro_flags = RT_LLE_CACHE; 637 refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */ 638 INP_WLOCK(inp); 639 INP_INFO_WLOCK(pcbinfo); 640 pcbinfo->ipi_count++; 641 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 642 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list); 643 INP_INFO_WUNLOCK(pcbinfo); 644 so->so_pcb = inp; 645 646 return (0); 647 648 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC) 649 out: 650 uma_zfree_smr(pcbinfo->ipi_zone, inp); 651 return (error); 652 #endif 653 } 654 655 #ifdef INET 656 int 657 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred) 658 { 659 int anonport, error; 660 661 KASSERT(nam == NULL || nam->sa_family == AF_INET, 662 ("%s: invalid address family for %p", __func__, nam)); 663 KASSERT(nam == NULL || nam->sa_len == sizeof(struct sockaddr_in), 664 ("%s: invalid address length for %p", __func__, nam)); 665 INP_WLOCK_ASSERT(inp); 666 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 667 668 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 669 return (EINVAL); 670 anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0; 671 error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr, 672 &inp->inp_lport, cred); 673 if (error) 674 return (error); 675 if (in_pcbinshash(inp) != 0) { 676 inp->inp_laddr.s_addr = INADDR_ANY; 677 inp->inp_lport = 0; 678 return (EAGAIN); 679 } 680 if (anonport) 681 inp->inp_flags |= INP_ANONPORT; 682 return (0); 683 } 684 #endif 685 686 #if defined(INET) || defined(INET6) 687 /* 688 * Assign a local port like in_pcb_lport(), but also used with connect() 689 * and a foreign address and port. If fsa is non-NULL, choose a local port 690 * that is unused with those, otherwise one that is completely unused. 691 * lsa can be NULL for IPv6. 692 */ 693 int 694 in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp, 695 struct sockaddr *fsa, u_short fport, struct ucred *cred, int lookupflags) 696 { 697 struct inpcbinfo *pcbinfo; 698 struct inpcb *tmpinp; 699 unsigned short *lastport; 700 int count, error; 701 u_short aux, first, last, lport; 702 #ifdef INET 703 struct in_addr laddr, faddr; 704 #endif 705 #ifdef INET6 706 struct in6_addr *laddr6, *faddr6; 707 #endif 708 709 pcbinfo = inp->inp_pcbinfo; 710 711 /* 712 * Because no actual state changes occur here, a global write lock on 713 * the pcbinfo isn't required. 714 */ 715 INP_LOCK_ASSERT(inp); 716 INP_HASH_LOCK_ASSERT(pcbinfo); 717 718 if (inp->inp_flags & INP_HIGHPORT) { 719 first = V_ipport_hifirstauto; /* sysctl */ 720 last = V_ipport_hilastauto; 721 lastport = &pcbinfo->ipi_lasthi; 722 } else if (inp->inp_flags & INP_LOWPORT) { 723 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT); 724 if (error) 725 return (error); 726 first = V_ipport_lowfirstauto; /* 1023 */ 727 last = V_ipport_lowlastauto; /* 600 */ 728 lastport = &pcbinfo->ipi_lastlow; 729 } else { 730 first = V_ipport_firstauto; /* sysctl */ 731 last = V_ipport_lastauto; 732 lastport = &pcbinfo->ipi_lastport; 733 } 734 735 /* 736 * Instead of having two loops further down counting up or down 737 * make sure that first is always <= last and go with only one 738 * code path implementing all logic. 739 */ 740 if (first > last) { 741 aux = first; 742 first = last; 743 last = aux; 744 } 745 746 #ifdef INET 747 laddr.s_addr = INADDR_ANY; /* used by INET6+INET below too */ 748 if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) { 749 if (lsa != NULL) 750 laddr = ((struct sockaddr_in *)lsa)->sin_addr; 751 if (fsa != NULL) 752 faddr = ((struct sockaddr_in *)fsa)->sin_addr; 753 } 754 #endif 755 #ifdef INET6 756 laddr6 = NULL; 757 if ((inp->inp_vflag & INP_IPV6) != 0) { 758 if (lsa != NULL) 759 laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr; 760 if (fsa != NULL) 761 faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr; 762 } 763 #endif 764 765 tmpinp = NULL; 766 lport = *lportp; 767 768 if (V_ipport_randomized) 769 *lastport = first + (arc4random() % (last - first)); 770 771 count = last - first; 772 773 do { 774 if (count-- < 0) /* completely used? */ 775 return (EADDRNOTAVAIL); 776 ++*lastport; 777 if (*lastport < first || *lastport > last) 778 *lastport = first; 779 lport = htons(*lastport); 780 781 if (fsa != NULL) { 782 #ifdef INET 783 if (lsa->sa_family == AF_INET) { 784 tmpinp = in_pcblookup_hash_locked(pcbinfo, 785 faddr, fport, laddr, lport, lookupflags, 786 NULL, M_NODOM); 787 } 788 #endif 789 #ifdef INET6 790 if (lsa->sa_family == AF_INET6) { 791 tmpinp = in6_pcblookup_hash_locked(pcbinfo, 792 faddr6, fport, laddr6, lport, lookupflags, 793 NULL, M_NODOM); 794 } 795 #endif 796 } else { 797 #ifdef INET6 798 if ((inp->inp_vflag & INP_IPV6) != 0) { 799 tmpinp = in6_pcblookup_local(pcbinfo, 800 &inp->in6p_laddr, lport, lookupflags, cred); 801 #ifdef INET 802 if (tmpinp == NULL && 803 (inp->inp_vflag & INP_IPV4)) 804 tmpinp = in_pcblookup_local(pcbinfo, 805 laddr, lport, lookupflags, cred); 806 #endif 807 } 808 #endif 809 #if defined(INET) && defined(INET6) 810 else 811 #endif 812 #ifdef INET 813 tmpinp = in_pcblookup_local(pcbinfo, laddr, 814 lport, lookupflags, cred); 815 #endif 816 } 817 } while (tmpinp != NULL); 818 819 *lportp = lport; 820 821 return (0); 822 } 823 824 /* 825 * Select a local port (number) to use. 826 */ 827 int 828 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp, 829 struct ucred *cred, int lookupflags) 830 { 831 struct sockaddr_in laddr; 832 833 if (laddrp) { 834 bzero(&laddr, sizeof(laddr)); 835 laddr.sin_family = AF_INET; 836 laddr.sin_addr = *laddrp; 837 } 838 return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr : 839 NULL, lportp, NULL, 0, cred, lookupflags)); 840 } 841 842 /* 843 * Return cached socket options. 844 */ 845 int 846 inp_so_options(const struct inpcb *inp) 847 { 848 int so_options; 849 850 so_options = 0; 851 852 if ((inp->inp_flags2 & INP_REUSEPORT_LB) != 0) 853 so_options |= SO_REUSEPORT_LB; 854 if ((inp->inp_flags2 & INP_REUSEPORT) != 0) 855 so_options |= SO_REUSEPORT; 856 if ((inp->inp_flags2 & INP_REUSEADDR) != 0) 857 so_options |= SO_REUSEADDR; 858 return (so_options); 859 } 860 #endif /* INET || INET6 */ 861 862 /* 863 * Check if a new BINDMULTI socket is allowed to be created. 864 * 865 * ni points to the new inp. 866 * oi points to the existing inp. 867 * 868 * This checks whether the existing inp also has BINDMULTI and 869 * whether the credentials match. 870 */ 871 int 872 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi) 873 { 874 /* Check permissions match */ 875 if ((ni->inp_flags2 & INP_BINDMULTI) && 876 (ni->inp_cred->cr_uid != 877 oi->inp_cred->cr_uid)) 878 return (0); 879 880 /* Check the existing inp has BINDMULTI set */ 881 if ((ni->inp_flags2 & INP_BINDMULTI) && 882 ((oi->inp_flags2 & INP_BINDMULTI) == 0)) 883 return (0); 884 885 /* 886 * We're okay - either INP_BINDMULTI isn't set on ni, or 887 * it is and it matches the checks. 888 */ 889 return (1); 890 } 891 892 #ifdef INET 893 /* 894 * Set up a bind operation on a PCB, performing port allocation 895 * as required, but do not actually modify the PCB. Callers can 896 * either complete the bind by setting inp_laddr/inp_lport and 897 * calling in_pcbinshash(), or they can just use the resulting 898 * port and address to authorise the sending of a once-off packet. 899 * 900 * On error, the values of *laddrp and *lportp are not changed. 901 */ 902 int 903 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp, 904 u_short *lportp, struct ucred *cred) 905 { 906 struct socket *so = inp->inp_socket; 907 struct sockaddr_in *sin; 908 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 909 struct in_addr laddr; 910 u_short lport = 0; 911 int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT); 912 int error; 913 914 /* 915 * XXX: Maybe we could let SO_REUSEPORT_LB set SO_REUSEPORT bit here 916 * so that we don't have to add to the (already messy) code below. 917 */ 918 int reuseport_lb = (so->so_options & SO_REUSEPORT_LB); 919 920 /* 921 * No state changes, so read locks are sufficient here. 922 */ 923 INP_LOCK_ASSERT(inp); 924 INP_HASH_LOCK_ASSERT(pcbinfo); 925 926 laddr.s_addr = *laddrp; 927 if (nam != NULL && laddr.s_addr != INADDR_ANY) 928 return (EINVAL); 929 if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0) 930 lookupflags = INPLOOKUP_WILDCARD; 931 if (nam == NULL) { 932 if ((error = prison_local_ip4(cred, &laddr)) != 0) 933 return (error); 934 } else { 935 sin = (struct sockaddr_in *)nam; 936 KASSERT(sin->sin_family == AF_INET, 937 ("%s: invalid family for address %p", __func__, sin)); 938 KASSERT(sin->sin_len == sizeof(*sin), 939 ("%s: invalid length for address %p", __func__, sin)); 940 941 error = prison_local_ip4(cred, &sin->sin_addr); 942 if (error) 943 return (error); 944 if (sin->sin_port != *lportp) { 945 /* Don't allow the port to change. */ 946 if (*lportp != 0) 947 return (EINVAL); 948 lport = sin->sin_port; 949 } 950 /* NB: lport is left as 0 if the port isn't being changed. */ 951 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 952 /* 953 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 954 * allow complete duplication of binding if 955 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 956 * and a multicast address is bound on both 957 * new and duplicated sockets. 958 */ 959 if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0) 960 reuseport = SO_REUSEADDR|SO_REUSEPORT; 961 /* 962 * XXX: How to deal with SO_REUSEPORT_LB here? 963 * Treat same as SO_REUSEPORT for now. 964 */ 965 if ((so->so_options & 966 (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0) 967 reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB; 968 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 969 sin->sin_port = 0; /* yech... */ 970 bzero(&sin->sin_zero, sizeof(sin->sin_zero)); 971 /* 972 * Is the address a local IP address? 973 * If INP_BINDANY is set, then the socket may be bound 974 * to any endpoint address, local or not. 975 */ 976 if ((inp->inp_flags & INP_BINDANY) == 0 && 977 ifa_ifwithaddr_check((struct sockaddr *)sin) == 0) 978 return (EADDRNOTAVAIL); 979 } 980 laddr = sin->sin_addr; 981 if (lport) { 982 struct inpcb *t; 983 984 /* GROSS */ 985 if (ntohs(lport) <= V_ipport_reservedhigh && 986 ntohs(lport) >= V_ipport_reservedlow && 987 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT)) 988 return (EACCES); 989 if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 990 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) { 991 t = in_pcblookup_local(pcbinfo, sin->sin_addr, 992 lport, INPLOOKUP_WILDCARD, cred); 993 /* 994 * XXX 995 * This entire block sorely needs a rewrite. 996 */ 997 if (t && 998 ((inp->inp_flags2 & INP_BINDMULTI) == 0) && 999 (so->so_type != SOCK_STREAM || 1000 ntohl(t->inp_faddr.s_addr) == INADDR_ANY) && 1001 (ntohl(sin->sin_addr.s_addr) != INADDR_ANY || 1002 ntohl(t->inp_laddr.s_addr) != INADDR_ANY || 1003 (t->inp_flags2 & INP_REUSEPORT) || 1004 (t->inp_flags2 & INP_REUSEPORT_LB) == 0) && 1005 (inp->inp_cred->cr_uid != 1006 t->inp_cred->cr_uid)) 1007 return (EADDRINUSE); 1008 1009 /* 1010 * If the socket is a BINDMULTI socket, then 1011 * the credentials need to match and the 1012 * original socket also has to have been bound 1013 * with BINDMULTI. 1014 */ 1015 if (t && (! in_pcbbind_check_bindmulti(inp, t))) 1016 return (EADDRINUSE); 1017 } 1018 t = in_pcblookup_local(pcbinfo, sin->sin_addr, 1019 lport, lookupflags, cred); 1020 if (t && ((inp->inp_flags2 & INP_BINDMULTI) == 0) && 1021 (reuseport & inp_so_options(t)) == 0 && 1022 (reuseport_lb & inp_so_options(t)) == 0) { 1023 #ifdef INET6 1024 if (ntohl(sin->sin_addr.s_addr) != 1025 INADDR_ANY || 1026 ntohl(t->inp_laddr.s_addr) != 1027 INADDR_ANY || 1028 (inp->inp_vflag & INP_IPV6PROTO) == 0 || 1029 (t->inp_vflag & INP_IPV6PROTO) == 0) 1030 #endif 1031 return (EADDRINUSE); 1032 if (t && (! in_pcbbind_check_bindmulti(inp, t))) 1033 return (EADDRINUSE); 1034 } 1035 } 1036 } 1037 if (*lportp != 0) 1038 lport = *lportp; 1039 if (lport == 0) { 1040 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags); 1041 if (error != 0) 1042 return (error); 1043 } 1044 *laddrp = laddr.s_addr; 1045 *lportp = lport; 1046 return (0); 1047 } 1048 1049 /* 1050 * Connect from a socket to a specified address. 1051 * Both address and port must be specified in argument sin. 1052 * If don't have a local address for this socket yet, 1053 * then pick one. 1054 */ 1055 int 1056 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred, 1057 bool rehash) 1058 { 1059 u_short lport, fport; 1060 in_addr_t laddr, faddr; 1061 int anonport, error; 1062 1063 INP_WLOCK_ASSERT(inp); 1064 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 1065 1066 lport = inp->inp_lport; 1067 laddr = inp->inp_laddr.s_addr; 1068 anonport = (lport == 0); 1069 error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport, 1070 NULL, cred); 1071 if (error) 1072 return (error); 1073 1074 /* Do the initial binding of the local address if required. */ 1075 if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) { 1076 KASSERT(rehash == true, 1077 ("Rehashing required for unbound inps")); 1078 inp->inp_lport = lport; 1079 inp->inp_laddr.s_addr = laddr; 1080 if (in_pcbinshash(inp) != 0) { 1081 inp->inp_laddr.s_addr = INADDR_ANY; 1082 inp->inp_lport = 0; 1083 return (EAGAIN); 1084 } 1085 } 1086 1087 /* Commit the remaining changes. */ 1088 inp->inp_lport = lport; 1089 inp->inp_laddr.s_addr = laddr; 1090 inp->inp_faddr.s_addr = faddr; 1091 inp->inp_fport = fport; 1092 if (rehash) { 1093 in_pcbrehash(inp); 1094 } else { 1095 in_pcbinshash(inp); 1096 } 1097 1098 if (anonport) 1099 inp->inp_flags |= INP_ANONPORT; 1100 return (0); 1101 } 1102 1103 /* 1104 * Do proper source address selection on an unbound socket in case 1105 * of connect. Take jails into account as well. 1106 */ 1107 int 1108 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, 1109 struct ucred *cred) 1110 { 1111 struct ifaddr *ifa; 1112 struct sockaddr *sa; 1113 struct sockaddr_in *sin, dst; 1114 struct nhop_object *nh; 1115 int error; 1116 1117 NET_EPOCH_ASSERT(); 1118 KASSERT(laddr != NULL, ("%s: laddr NULL", __func__)); 1119 1120 /* 1121 * Bypass source address selection and use the primary jail IP 1122 * if requested. 1123 */ 1124 if (!prison_saddrsel_ip4(cred, laddr)) 1125 return (0); 1126 1127 error = 0; 1128 1129 nh = NULL; 1130 bzero(&dst, sizeof(dst)); 1131 sin = &dst; 1132 sin->sin_family = AF_INET; 1133 sin->sin_len = sizeof(struct sockaddr_in); 1134 sin->sin_addr.s_addr = faddr->s_addr; 1135 1136 /* 1137 * If route is known our src addr is taken from the i/f, 1138 * else punt. 1139 * 1140 * Find out route to destination. 1141 */ 1142 if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) 1143 nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr, 1144 0, NHR_NONE, 0); 1145 1146 /* 1147 * If we found a route, use the address corresponding to 1148 * the outgoing interface. 1149 * 1150 * Otherwise assume faddr is reachable on a directly connected 1151 * network and try to find a corresponding interface to take 1152 * the source address from. 1153 */ 1154 if (nh == NULL || nh->nh_ifp == NULL) { 1155 struct in_ifaddr *ia; 1156 struct ifnet *ifp; 1157 1158 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin, 1159 inp->inp_socket->so_fibnum)); 1160 if (ia == NULL) { 1161 ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0, 1162 inp->inp_socket->so_fibnum)); 1163 } 1164 if (ia == NULL) { 1165 error = ENETUNREACH; 1166 goto done; 1167 } 1168 1169 if (!prison_flag(cred, PR_IP4)) { 1170 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1171 goto done; 1172 } 1173 1174 ifp = ia->ia_ifp; 1175 ia = NULL; 1176 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 1177 sa = ifa->ifa_addr; 1178 if (sa->sa_family != AF_INET) 1179 continue; 1180 sin = (struct sockaddr_in *)sa; 1181 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 1182 ia = (struct in_ifaddr *)ifa; 1183 break; 1184 } 1185 } 1186 if (ia != NULL) { 1187 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1188 goto done; 1189 } 1190 1191 /* 3. As a last resort return the 'default' jail address. */ 1192 error = prison_get_ip4(cred, laddr); 1193 goto done; 1194 } 1195 1196 /* 1197 * If the outgoing interface on the route found is not 1198 * a loopback interface, use the address from that interface. 1199 * In case of jails do those three steps: 1200 * 1. check if the interface address belongs to the jail. If so use it. 1201 * 2. check if we have any address on the outgoing interface 1202 * belonging to this jail. If so use it. 1203 * 3. as a last resort return the 'default' jail address. 1204 */ 1205 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) { 1206 struct in_ifaddr *ia; 1207 struct ifnet *ifp; 1208 1209 /* If not jailed, use the default returned. */ 1210 if (!prison_flag(cred, PR_IP4)) { 1211 ia = (struct in_ifaddr *)nh->nh_ifa; 1212 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1213 goto done; 1214 } 1215 1216 /* Jailed. */ 1217 /* 1. Check if the iface address belongs to the jail. */ 1218 sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr; 1219 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 1220 ia = (struct in_ifaddr *)nh->nh_ifa; 1221 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1222 goto done; 1223 } 1224 1225 /* 1226 * 2. Check if we have any address on the outgoing interface 1227 * belonging to this jail. 1228 */ 1229 ia = NULL; 1230 ifp = nh->nh_ifp; 1231 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 1232 sa = ifa->ifa_addr; 1233 if (sa->sa_family != AF_INET) 1234 continue; 1235 sin = (struct sockaddr_in *)sa; 1236 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 1237 ia = (struct in_ifaddr *)ifa; 1238 break; 1239 } 1240 } 1241 if (ia != NULL) { 1242 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1243 goto done; 1244 } 1245 1246 /* 3. As a last resort return the 'default' jail address. */ 1247 error = prison_get_ip4(cred, laddr); 1248 goto done; 1249 } 1250 1251 /* 1252 * The outgoing interface is marked with 'loopback net', so a route 1253 * to ourselves is here. 1254 * Try to find the interface of the destination address and then 1255 * take the address from there. That interface is not necessarily 1256 * a loopback interface. 1257 * In case of jails, check that it is an address of the jail 1258 * and if we cannot find, fall back to the 'default' jail address. 1259 */ 1260 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) { 1261 struct in_ifaddr *ia; 1262 1263 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst), 1264 inp->inp_socket->so_fibnum)); 1265 if (ia == NULL) 1266 ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0, 1267 inp->inp_socket->so_fibnum)); 1268 if (ia == NULL) 1269 ia = ifatoia(ifa_ifwithaddr(sintosa(&dst))); 1270 1271 if (!prison_flag(cred, PR_IP4)) { 1272 if (ia == NULL) { 1273 error = ENETUNREACH; 1274 goto done; 1275 } 1276 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1277 goto done; 1278 } 1279 1280 /* Jailed. */ 1281 if (ia != NULL) { 1282 struct ifnet *ifp; 1283 1284 ifp = ia->ia_ifp; 1285 ia = NULL; 1286 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 1287 sa = ifa->ifa_addr; 1288 if (sa->sa_family != AF_INET) 1289 continue; 1290 sin = (struct sockaddr_in *)sa; 1291 if (prison_check_ip4(cred, 1292 &sin->sin_addr) == 0) { 1293 ia = (struct in_ifaddr *)ifa; 1294 break; 1295 } 1296 } 1297 if (ia != NULL) { 1298 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1299 goto done; 1300 } 1301 } 1302 1303 /* 3. As a last resort return the 'default' jail address. */ 1304 error = prison_get_ip4(cred, laddr); 1305 goto done; 1306 } 1307 1308 done: 1309 return (error); 1310 } 1311 1312 /* 1313 * Set up for a connect from a socket to the specified address. 1314 * On entry, *laddrp and *lportp should contain the current local 1315 * address and port for the PCB; these are updated to the values 1316 * that should be placed in inp_laddr and inp_lport to complete 1317 * the connect. 1318 * 1319 * On success, *faddrp and *fportp will be set to the remote address 1320 * and port. These are not updated in the error case. 1321 * 1322 * If the operation fails because the connection already exists, 1323 * *oinpp will be set to the PCB of that connection so that the 1324 * caller can decide to override it. In all other cases, *oinpp 1325 * is set to NULL. 1326 */ 1327 int 1328 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam, 1329 in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp, 1330 struct inpcb **oinpp, struct ucred *cred) 1331 { 1332 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1333 struct in_ifaddr *ia; 1334 struct inpcb *oinp; 1335 struct in_addr laddr, faddr; 1336 u_short lport, fport; 1337 int error; 1338 1339 KASSERT(sin->sin_family == AF_INET, 1340 ("%s: invalid address family for %p", __func__, sin)); 1341 KASSERT(sin->sin_len == sizeof(*sin), 1342 ("%s: invalid address length for %p", __func__, sin)); 1343 1344 /* 1345 * Because a global state change doesn't actually occur here, a read 1346 * lock is sufficient. 1347 */ 1348 NET_EPOCH_ASSERT(); 1349 INP_LOCK_ASSERT(inp); 1350 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo); 1351 1352 if (oinpp != NULL) 1353 *oinpp = NULL; 1354 if (sin->sin_port == 0) 1355 return (EADDRNOTAVAIL); 1356 laddr.s_addr = *laddrp; 1357 lport = *lportp; 1358 faddr = sin->sin_addr; 1359 fport = sin->sin_port; 1360 #ifdef ROUTE_MPATH 1361 if (CALC_FLOWID_OUTBOUND) { 1362 uint32_t hash_val, hash_type; 1363 1364 hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport, 1365 inp->inp_socket->so_proto->pr_protocol, &hash_type); 1366 1367 inp->inp_flowid = hash_val; 1368 inp->inp_flowtype = hash_type; 1369 } 1370 #endif 1371 if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) { 1372 /* 1373 * If the destination address is INADDR_ANY, 1374 * use the primary local address. 1375 * If the supplied address is INADDR_BROADCAST, 1376 * and the primary interface supports broadcast, 1377 * choose the broadcast address for that interface. 1378 */ 1379 if (faddr.s_addr == INADDR_ANY) { 1380 faddr = 1381 IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr; 1382 if ((error = prison_get_ip4(cred, &faddr)) != 0) 1383 return (error); 1384 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) { 1385 if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags & 1386 IFF_BROADCAST) 1387 faddr = satosin(&CK_STAILQ_FIRST( 1388 &V_in_ifaddrhead)->ia_broadaddr)->sin_addr; 1389 } 1390 } 1391 if (laddr.s_addr == INADDR_ANY) { 1392 error = in_pcbladdr(inp, &faddr, &laddr, cred); 1393 /* 1394 * If the destination address is multicast and an outgoing 1395 * interface has been set as a multicast option, prefer the 1396 * address of that interface as our source address. 1397 */ 1398 if (IN_MULTICAST(ntohl(faddr.s_addr)) && 1399 inp->inp_moptions != NULL) { 1400 struct ip_moptions *imo; 1401 struct ifnet *ifp; 1402 1403 imo = inp->inp_moptions; 1404 if (imo->imo_multicast_ifp != NULL) { 1405 ifp = imo->imo_multicast_ifp; 1406 CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { 1407 if (ia->ia_ifp == ifp && 1408 prison_check_ip4(cred, 1409 &ia->ia_addr.sin_addr) == 0) 1410 break; 1411 } 1412 if (ia == NULL) 1413 error = EADDRNOTAVAIL; 1414 else { 1415 laddr = ia->ia_addr.sin_addr; 1416 error = 0; 1417 } 1418 } 1419 } 1420 if (error) 1421 return (error); 1422 } 1423 1424 if (lport != 0) { 1425 oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, 1426 fport, laddr, lport, 0, NULL, M_NODOM); 1427 if (oinp != NULL) { 1428 if (oinpp != NULL) 1429 *oinpp = oinp; 1430 return (EADDRINUSE); 1431 } 1432 } else { 1433 struct sockaddr_in lsin, fsin; 1434 1435 bzero(&lsin, sizeof(lsin)); 1436 bzero(&fsin, sizeof(fsin)); 1437 lsin.sin_family = AF_INET; 1438 lsin.sin_addr = laddr; 1439 fsin.sin_family = AF_INET; 1440 fsin.sin_addr = faddr; 1441 error = in_pcb_lport_dest(inp, (struct sockaddr *) &lsin, 1442 &lport, (struct sockaddr *)& fsin, fport, cred, 1443 INPLOOKUP_WILDCARD); 1444 if (error) 1445 return (error); 1446 } 1447 *laddrp = laddr.s_addr; 1448 *lportp = lport; 1449 *faddrp = faddr.s_addr; 1450 *fportp = fport; 1451 return (0); 1452 } 1453 1454 void 1455 in_pcbdisconnect(struct inpcb *inp) 1456 { 1457 1458 INP_WLOCK_ASSERT(inp); 1459 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 1460 1461 inp->inp_faddr.s_addr = INADDR_ANY; 1462 inp->inp_fport = 0; 1463 in_pcbrehash(inp); 1464 } 1465 #endif /* INET */ 1466 1467 /* 1468 * in_pcbdetach() is responsibe for disassociating a socket from an inpcb. 1469 * For most protocols, this will be invoked immediately prior to calling 1470 * in_pcbfree(). However, with TCP the inpcb may significantly outlive the 1471 * socket, in which case in_pcbfree() is deferred. 1472 */ 1473 void 1474 in_pcbdetach(struct inpcb *inp) 1475 { 1476 1477 KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__)); 1478 1479 #ifdef RATELIMIT 1480 if (inp->inp_snd_tag != NULL) 1481 in_pcbdetach_txrtlmt(inp); 1482 #endif 1483 inp->inp_socket->so_pcb = NULL; 1484 inp->inp_socket = NULL; 1485 } 1486 1487 /* 1488 * inpcb hash lookups are protected by SMR section. 1489 * 1490 * Once desired pcb has been found, switching from SMR section to a pcb 1491 * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK 1492 * here because SMR is a critical section. 1493 * In 99%+ cases inp_smr_lock() would obtain the lock immediately. 1494 */ 1495 static inline void 1496 inp_lock(struct inpcb *inp, const inp_lookup_t lock) 1497 { 1498 1499 lock == INPLOOKUP_RLOCKPCB ? 1500 rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock); 1501 } 1502 1503 static inline void 1504 inp_unlock(struct inpcb *inp, const inp_lookup_t lock) 1505 { 1506 1507 lock == INPLOOKUP_RLOCKPCB ? 1508 rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock); 1509 } 1510 1511 static inline int 1512 inp_trylock(struct inpcb *inp, const inp_lookup_t lock) 1513 { 1514 1515 return (lock == INPLOOKUP_RLOCKPCB ? 1516 rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock)); 1517 } 1518 1519 static inline bool 1520 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock) 1521 { 1522 1523 return (lock == INPLOOKUP_RLOCKPCB ? 1524 in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp)); 1525 } 1526 1527 static inline bool 1528 _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags) 1529 { 1530 1531 MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB); 1532 SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr); 1533 1534 if (__predict_true(inp_trylock(inp, lock))) { 1535 if (__predict_false(inp->inp_flags & ignflags)) { 1536 smr_exit(inp->inp_pcbinfo->ipi_smr); 1537 inp_unlock(inp, lock); 1538 return (false); 1539 } 1540 smr_exit(inp->inp_pcbinfo->ipi_smr); 1541 return (true); 1542 } 1543 1544 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) { 1545 smr_exit(inp->inp_pcbinfo->ipi_smr); 1546 inp_lock(inp, lock); 1547 if (__predict_false(in_pcbrele(inp, lock))) 1548 return (false); 1549 /* 1550 * inp acquired through refcount & lock for sure didn't went 1551 * through uma_zfree(). However, it may have already went 1552 * through in_pcbfree() and has another reference, that 1553 * prevented its release by our in_pcbrele(). 1554 */ 1555 if (__predict_false(inp->inp_flags & ignflags)) { 1556 inp_unlock(inp, lock); 1557 return (false); 1558 } 1559 return (true); 1560 } else { 1561 smr_exit(inp->inp_pcbinfo->ipi_smr); 1562 return (false); 1563 } 1564 } 1565 1566 bool 1567 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock) 1568 { 1569 1570 /* 1571 * in_pcblookup() family of functions ignore not only freed entries, 1572 * that may be found due to lockless access to the hash, but dropped 1573 * entries, too. 1574 */ 1575 return (_inp_smr_lock(inp, lock, INP_FREED | INP_DROPPED)); 1576 } 1577 1578 /* 1579 * inp_next() - inpcb hash/list traversal iterator 1580 * 1581 * Requires initialized struct inpcb_iterator for context. 1582 * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR(). 1583 * 1584 * - Iterator can have either write-lock or read-lock semantics, that can not 1585 * be changed later. 1586 * - Iterator can iterate either over all pcbs list (INP_ALL_LIST), or through 1587 * a single hash slot. Note: only rip_input() does the latter. 1588 * - Iterator may have optional bool matching function. The matching function 1589 * will be executed for each inpcb in the SMR context, so it can not acquire 1590 * locks and can safely access only immutable fields of inpcb. 1591 * 1592 * A fresh initialized iterator has NULL inpcb in its context and that 1593 * means that inp_next() call would return the very first inpcb on the list 1594 * locked with desired semantic. In all following calls the context pointer 1595 * shall hold the current inpcb pointer. The KPI user is not supposed to 1596 * unlock the current inpcb! Upon end of traversal inp_next() will return NULL 1597 * and write NULL to its context. After end of traversal an iterator can be 1598 * reused. 1599 * 1600 * List traversals have the following features/constraints: 1601 * - New entries won't be seen, as they are always added to the head of a list. 1602 * - Removed entries won't stop traversal as long as they are not added to 1603 * a different list. This is violated by in_pcbrehash(). 1604 */ 1605 #define II_LIST_FIRST(ipi, hash) \ 1606 (((hash) == INP_ALL_LIST) ? \ 1607 CK_LIST_FIRST(&(ipi)->ipi_listhead) : \ 1608 CK_LIST_FIRST(&(ipi)->ipi_hashbase[(hash)])) 1609 #define II_LIST_NEXT(inp, hash) \ 1610 (((hash) == INP_ALL_LIST) ? \ 1611 CK_LIST_NEXT((inp), inp_list) : \ 1612 CK_LIST_NEXT((inp), inp_hash)) 1613 #define II_LOCK_ASSERT(inp, lock) \ 1614 rw_assert(&(inp)->inp_lock, \ 1615 (lock) == INPLOOKUP_RLOCKPCB ? RA_RLOCKED : RA_WLOCKED ) 1616 struct inpcb * 1617 inp_next(struct inpcb_iterator *ii) 1618 { 1619 const struct inpcbinfo *ipi = ii->ipi; 1620 inp_match_t *match = ii->match; 1621 void *ctx = ii->ctx; 1622 inp_lookup_t lock = ii->lock; 1623 int hash = ii->hash; 1624 struct inpcb *inp; 1625 1626 if (ii->inp == NULL) { /* First call. */ 1627 smr_enter(ipi->ipi_smr); 1628 /* This is unrolled CK_LIST_FOREACH(). */ 1629 for (inp = II_LIST_FIRST(ipi, hash); 1630 inp != NULL; 1631 inp = II_LIST_NEXT(inp, hash)) { 1632 if (match != NULL && (match)(inp, ctx) == false) 1633 continue; 1634 if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED))) 1635 break; 1636 else { 1637 smr_enter(ipi->ipi_smr); 1638 MPASS(inp != II_LIST_FIRST(ipi, hash)); 1639 inp = II_LIST_FIRST(ipi, hash); 1640 if (inp == NULL) 1641 break; 1642 } 1643 } 1644 1645 if (inp == NULL) 1646 smr_exit(ipi->ipi_smr); 1647 else 1648 ii->inp = inp; 1649 1650 return (inp); 1651 } 1652 1653 /* Not a first call. */ 1654 smr_enter(ipi->ipi_smr); 1655 restart: 1656 inp = ii->inp; 1657 II_LOCK_ASSERT(inp, lock); 1658 next: 1659 inp = II_LIST_NEXT(inp, hash); 1660 if (inp == NULL) { 1661 smr_exit(ipi->ipi_smr); 1662 goto found; 1663 } 1664 1665 if (match != NULL && (match)(inp, ctx) == false) 1666 goto next; 1667 1668 if (__predict_true(inp_trylock(inp, lock))) { 1669 if (__predict_false(inp->inp_flags & INP_FREED)) { 1670 /* 1671 * Entries are never inserted in middle of a list, thus 1672 * as long as we are in SMR, we can continue traversal. 1673 * Jump to 'restart' should yield in the same result, 1674 * but could produce unnecessary looping. Could this 1675 * looping be unbound? 1676 */ 1677 inp_unlock(inp, lock); 1678 goto next; 1679 } else { 1680 smr_exit(ipi->ipi_smr); 1681 goto found; 1682 } 1683 } 1684 1685 /* 1686 * Can't obtain lock immediately, thus going hard. Once we exit the 1687 * SMR section we can no longer jump to 'next', and our only stable 1688 * anchoring point is ii->inp, which we keep locked for this case, so 1689 * we jump to 'restart'. 1690 */ 1691 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) { 1692 smr_exit(ipi->ipi_smr); 1693 inp_lock(inp, lock); 1694 if (__predict_false(in_pcbrele(inp, lock))) { 1695 smr_enter(ipi->ipi_smr); 1696 goto restart; 1697 } 1698 /* 1699 * See comment in inp_smr_lock(). 1700 */ 1701 if (__predict_false(inp->inp_flags & INP_FREED)) { 1702 inp_unlock(inp, lock); 1703 smr_enter(ipi->ipi_smr); 1704 goto restart; 1705 } 1706 } else 1707 goto next; 1708 1709 found: 1710 inp_unlock(ii->inp, lock); 1711 ii->inp = inp; 1712 1713 return (ii->inp); 1714 } 1715 1716 /* 1717 * in_pcbref() bumps the reference count on an inpcb in order to maintain 1718 * stability of an inpcb pointer despite the inpcb lock being released or 1719 * SMR section exited. 1720 * 1721 * To free a reference later in_pcbrele_(r|w)locked() must be performed. 1722 */ 1723 void 1724 in_pcbref(struct inpcb *inp) 1725 { 1726 u_int old __diagused; 1727 1728 old = refcount_acquire(&inp->inp_refcount); 1729 KASSERT(old > 0, ("%s: refcount 0", __func__)); 1730 } 1731 1732 /* 1733 * Drop a refcount on an inpcb elevated using in_pcbref(), potentially 1734 * freeing the pcb, if the reference was very last. 1735 */ 1736 bool 1737 in_pcbrele_rlocked(struct inpcb *inp) 1738 { 1739 1740 INP_RLOCK_ASSERT(inp); 1741 1742 if (refcount_release(&inp->inp_refcount) == 0) 1743 return (false); 1744 1745 MPASS(inp->inp_flags & INP_FREED); 1746 MPASS(inp->inp_socket == NULL); 1747 MPASS(inp->inp_in_hpts == 0); 1748 INP_RUNLOCK(inp); 1749 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp); 1750 return (true); 1751 } 1752 1753 bool 1754 in_pcbrele_wlocked(struct inpcb *inp) 1755 { 1756 1757 INP_WLOCK_ASSERT(inp); 1758 1759 if (refcount_release(&inp->inp_refcount) == 0) 1760 return (false); 1761 1762 MPASS(inp->inp_flags & INP_FREED); 1763 MPASS(inp->inp_socket == NULL); 1764 MPASS(inp->inp_in_hpts == 0); 1765 INP_WUNLOCK(inp); 1766 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp); 1767 return (true); 1768 } 1769 1770 /* 1771 * Unconditionally schedule an inpcb to be freed by decrementing its 1772 * reference count, which should occur only after the inpcb has been detached 1773 * from its socket. If another thread holds a temporary reference (acquired 1774 * using in_pcbref()) then the free is deferred until that reference is 1775 * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked. 1776 * Almost all work, including removal from global lists, is done in this 1777 * context, where the pcbinfo lock is held. 1778 */ 1779 void 1780 in_pcbfree(struct inpcb *inp) 1781 { 1782 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1783 #ifdef INET 1784 struct ip_moptions *imo; 1785 #endif 1786 #ifdef INET6 1787 struct ip6_moptions *im6o; 1788 #endif 1789 1790 INP_WLOCK_ASSERT(inp); 1791 KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__)); 1792 KASSERT((inp->inp_flags & INP_FREED) == 0, 1793 ("%s: called twice for pcb %p", __func__, inp)); 1794 1795 inp->inp_flags |= INP_FREED; 1796 INP_INFO_WLOCK(pcbinfo); 1797 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 1798 pcbinfo->ipi_count--; 1799 CK_LIST_REMOVE(inp, inp_list); 1800 INP_INFO_WUNLOCK(pcbinfo); 1801 1802 if (inp->inp_flags & INP_INHASHLIST) 1803 in_pcbremhash(inp); 1804 1805 RO_INVALIDATE_CACHE(&inp->inp_route); 1806 #ifdef MAC 1807 mac_inpcb_destroy(inp); 1808 #endif 1809 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 1810 if (inp->inp_sp != NULL) 1811 ipsec_delete_pcbpolicy(inp); 1812 #endif 1813 #ifdef INET 1814 if (inp->inp_options) 1815 (void)m_free(inp->inp_options); 1816 imo = inp->inp_moptions; 1817 #endif 1818 #ifdef INET6 1819 if (inp->inp_vflag & INP_IPV6PROTO) { 1820 ip6_freepcbopts(inp->in6p_outputopts); 1821 im6o = inp->in6p_moptions; 1822 } else 1823 im6o = NULL; 1824 #endif 1825 1826 if (__predict_false(in_pcbrele_wlocked(inp) == false)) { 1827 INP_WUNLOCK(inp); 1828 } 1829 #ifdef INET6 1830 ip6_freemoptions(im6o); 1831 #endif 1832 #ifdef INET 1833 inp_freemoptions(imo); 1834 #endif 1835 /* Destruction is finalized in inpcb_dtor(). */ 1836 } 1837 1838 static void 1839 inpcb_dtor(void *mem, int size, void *arg) 1840 { 1841 struct inpcb *inp = mem; 1842 1843 crfree(inp->inp_cred); 1844 #ifdef INVARIANTS 1845 inp->inp_cred = NULL; 1846 #endif 1847 } 1848 1849 /* 1850 * Different protocols initialize their inpcbs differently - giving 1851 * different name to the lock. But they all are disposed the same. 1852 */ 1853 static void 1854 inpcb_fini(void *mem, int size) 1855 { 1856 struct inpcb *inp = mem; 1857 1858 INP_LOCK_DESTROY(inp); 1859 } 1860 1861 /* 1862 * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and 1863 * port reservation, and preventing it from being returned by inpcb lookups. 1864 * 1865 * It is used by TCP to mark an inpcb as unused and avoid future packet 1866 * delivery or event notification when a socket remains open but TCP has 1867 * closed. This might occur as a result of a shutdown()-initiated TCP close 1868 * or a RST on the wire, and allows the port binding to be reused while still 1869 * maintaining the invariant that so_pcb always points to a valid inpcb until 1870 * in_pcbdetach(). 1871 * 1872 * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by 1873 * in_pcbnotifyall() and in_pcbpurgeif0()? 1874 */ 1875 void 1876 in_pcbdrop(struct inpcb *inp) 1877 { 1878 1879 INP_WLOCK_ASSERT(inp); 1880 #ifdef INVARIANTS 1881 if (inp->inp_socket != NULL && inp->inp_ppcb != NULL) 1882 MPASS(inp->inp_refcount > 1); 1883 #endif 1884 1885 inp->inp_flags |= INP_DROPPED; 1886 if (inp->inp_flags & INP_INHASHLIST) 1887 in_pcbremhash(inp); 1888 } 1889 1890 #ifdef INET 1891 /* 1892 * Common routines to return the socket addresses associated with inpcbs. 1893 */ 1894 struct sockaddr * 1895 in_sockaddr(in_port_t port, struct in_addr *addr_p) 1896 { 1897 struct sockaddr_in *sin; 1898 1899 sin = malloc(sizeof *sin, M_SONAME, 1900 M_WAITOK | M_ZERO); 1901 sin->sin_family = AF_INET; 1902 sin->sin_len = sizeof(*sin); 1903 sin->sin_addr = *addr_p; 1904 sin->sin_port = port; 1905 1906 return (struct sockaddr *)sin; 1907 } 1908 1909 int 1910 in_getsockaddr(struct socket *so, struct sockaddr **nam) 1911 { 1912 struct inpcb *inp; 1913 struct in_addr addr; 1914 in_port_t port; 1915 1916 inp = sotoinpcb(so); 1917 KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL")); 1918 1919 INP_RLOCK(inp); 1920 port = inp->inp_lport; 1921 addr = inp->inp_laddr; 1922 INP_RUNLOCK(inp); 1923 1924 *nam = in_sockaddr(port, &addr); 1925 return 0; 1926 } 1927 1928 int 1929 in_getpeeraddr(struct socket *so, struct sockaddr **nam) 1930 { 1931 struct inpcb *inp; 1932 struct in_addr addr; 1933 in_port_t port; 1934 1935 inp = sotoinpcb(so); 1936 KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL")); 1937 1938 INP_RLOCK(inp); 1939 port = inp->inp_fport; 1940 addr = inp->inp_faddr; 1941 INP_RUNLOCK(inp); 1942 1943 *nam = in_sockaddr(port, &addr); 1944 return 0; 1945 } 1946 1947 void 1948 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno, 1949 struct inpcb *(*notify)(struct inpcb *, int)) 1950 { 1951 struct inpcb *inp, *inp_temp; 1952 1953 INP_INFO_WLOCK(pcbinfo); 1954 CK_LIST_FOREACH_SAFE(inp, &pcbinfo->ipi_listhead, inp_list, inp_temp) { 1955 INP_WLOCK(inp); 1956 #ifdef INET6 1957 if ((inp->inp_vflag & INP_IPV4) == 0) { 1958 INP_WUNLOCK(inp); 1959 continue; 1960 } 1961 #endif 1962 if (inp->inp_faddr.s_addr != faddr.s_addr || 1963 inp->inp_socket == NULL) { 1964 INP_WUNLOCK(inp); 1965 continue; 1966 } 1967 if ((*notify)(inp, errno)) 1968 INP_WUNLOCK(inp); 1969 } 1970 INP_INFO_WUNLOCK(pcbinfo); 1971 } 1972 1973 static bool 1974 inp_v4_multi_match(const struct inpcb *inp, void *v __unused) 1975 { 1976 1977 if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL) 1978 return (true); 1979 else 1980 return (false); 1981 } 1982 1983 void 1984 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1985 { 1986 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB, 1987 inp_v4_multi_match, NULL); 1988 struct inpcb *inp; 1989 struct in_multi *inm; 1990 struct in_mfilter *imf; 1991 struct ip_moptions *imo; 1992 1993 IN_MULTI_LOCK_ASSERT(); 1994 1995 while ((inp = inp_next(&inpi)) != NULL) { 1996 INP_WLOCK_ASSERT(inp); 1997 1998 imo = inp->inp_moptions; 1999 /* 2000 * Unselect the outgoing interface if it is being 2001 * detached. 2002 */ 2003 if (imo->imo_multicast_ifp == ifp) 2004 imo->imo_multicast_ifp = NULL; 2005 2006 /* 2007 * Drop multicast group membership if we joined 2008 * through the interface being detached. 2009 * 2010 * XXX This can all be deferred to an epoch_call 2011 */ 2012 restart: 2013 IP_MFILTER_FOREACH(imf, &imo->imo_head) { 2014 if ((inm = imf->imf_inm) == NULL) 2015 continue; 2016 if (inm->inm_ifp != ifp) 2017 continue; 2018 ip_mfilter_remove(&imo->imo_head, imf); 2019 in_leavegroup_locked(inm, NULL); 2020 ip_mfilter_free(imf); 2021 goto restart; 2022 } 2023 } 2024 } 2025 2026 /* 2027 * Lookup a PCB based on the local address and port. Caller must hold the 2028 * hash lock. No inpcb locks or references are acquired. 2029 */ 2030 #define INP_LOOKUP_MAPPED_PCB_COST 3 2031 struct inpcb * 2032 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr, 2033 u_short lport, int lookupflags, struct ucred *cred) 2034 { 2035 struct inpcb *inp; 2036 #ifdef INET6 2037 int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST; 2038 #else 2039 int matchwild = 3; 2040 #endif 2041 int wildcard; 2042 2043 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, 2044 ("%s: invalid lookup flags %d", __func__, lookupflags)); 2045 INP_HASH_LOCK_ASSERT(pcbinfo); 2046 2047 if ((lookupflags & INPLOOKUP_WILDCARD) == 0) { 2048 struct inpcbhead *head; 2049 /* 2050 * Look for an unconnected (wildcard foreign addr) PCB that 2051 * matches the local address and port we're looking for. 2052 */ 2053 head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, 2054 pcbinfo->ipi_hashmask)]; 2055 CK_LIST_FOREACH(inp, head, inp_hash) { 2056 #ifdef INET6 2057 /* XXX inp locking */ 2058 if ((inp->inp_vflag & INP_IPV4) == 0) 2059 continue; 2060 #endif 2061 if (inp->inp_faddr.s_addr == INADDR_ANY && 2062 inp->inp_laddr.s_addr == laddr.s_addr && 2063 inp->inp_lport == lport) { 2064 /* 2065 * Found? 2066 */ 2067 if (prison_equal_ip4(cred->cr_prison, 2068 inp->inp_cred->cr_prison)) 2069 return (inp); 2070 } 2071 } 2072 /* 2073 * Not found. 2074 */ 2075 return (NULL); 2076 } else { 2077 struct inpcbporthead *porthash; 2078 struct inpcbport *phd; 2079 struct inpcb *match = NULL; 2080 /* 2081 * Best fit PCB lookup. 2082 * 2083 * First see if this local port is in use by looking on the 2084 * port hash list. 2085 */ 2086 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport, 2087 pcbinfo->ipi_porthashmask)]; 2088 CK_LIST_FOREACH(phd, porthash, phd_hash) { 2089 if (phd->phd_port == lport) 2090 break; 2091 } 2092 if (phd != NULL) { 2093 /* 2094 * Port is in use by one or more PCBs. Look for best 2095 * fit. 2096 */ 2097 CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 2098 wildcard = 0; 2099 if (!prison_equal_ip4(inp->inp_cred->cr_prison, 2100 cred->cr_prison)) 2101 continue; 2102 #ifdef INET6 2103 /* XXX inp locking */ 2104 if ((inp->inp_vflag & INP_IPV4) == 0) 2105 continue; 2106 /* 2107 * We never select the PCB that has 2108 * INP_IPV6 flag and is bound to :: if 2109 * we have another PCB which is bound 2110 * to 0.0.0.0. If a PCB has the 2111 * INP_IPV6 flag, then we set its cost 2112 * higher than IPv4 only PCBs. 2113 * 2114 * Note that the case only happens 2115 * when a socket is bound to ::, under 2116 * the condition that the use of the 2117 * mapped address is allowed. 2118 */ 2119 if ((inp->inp_vflag & INP_IPV6) != 0) 2120 wildcard += INP_LOOKUP_MAPPED_PCB_COST; 2121 #endif 2122 if (inp->inp_faddr.s_addr != INADDR_ANY) 2123 wildcard++; 2124 if (inp->inp_laddr.s_addr != INADDR_ANY) { 2125 if (laddr.s_addr == INADDR_ANY) 2126 wildcard++; 2127 else if (inp->inp_laddr.s_addr != laddr.s_addr) 2128 continue; 2129 } else { 2130 if (laddr.s_addr != INADDR_ANY) 2131 wildcard++; 2132 } 2133 if (wildcard < matchwild) { 2134 match = inp; 2135 matchwild = wildcard; 2136 if (matchwild == 0) 2137 break; 2138 } 2139 } 2140 } 2141 return (match); 2142 } 2143 } 2144 #undef INP_LOOKUP_MAPPED_PCB_COST 2145 2146 static bool 2147 in_pcblookup_lb_numa_match(const struct inpcblbgroup *grp, int domain) 2148 { 2149 return (domain == M_NODOM || domain == grp->il_numa_domain); 2150 } 2151 2152 static struct inpcb * 2153 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, 2154 const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr, 2155 uint16_t fport, int lookupflags, int domain) 2156 { 2157 const struct inpcblbgrouphead *hdr; 2158 struct inpcblbgroup *grp; 2159 struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild; 2160 2161 INP_HASH_LOCK_ASSERT(pcbinfo); 2162 2163 hdr = &pcbinfo->ipi_lbgrouphashbase[ 2164 INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)]; 2165 2166 /* 2167 * Search for an LB group match based on the following criteria: 2168 * - prefer jailed groups to non-jailed groups 2169 * - prefer exact source address matches to wildcard matches 2170 * - prefer groups bound to the specified NUMA domain 2171 */ 2172 jail_exact = jail_wild = local_exact = local_wild = NULL; 2173 CK_LIST_FOREACH(grp, hdr, il_list) { 2174 bool injail; 2175 2176 #ifdef INET6 2177 if (!(grp->il_vflag & INP_IPV4)) 2178 continue; 2179 #endif 2180 if (grp->il_lport != lport) 2181 continue; 2182 2183 injail = prison_flag(grp->il_cred, PR_IP4) != 0; 2184 if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison, 2185 laddr) != 0) 2186 continue; 2187 2188 if (grp->il_laddr.s_addr == laddr->s_addr) { 2189 if (injail) { 2190 jail_exact = grp; 2191 if (in_pcblookup_lb_numa_match(grp, domain)) 2192 /* This is a perfect match. */ 2193 goto out; 2194 } else if (local_exact == NULL || 2195 in_pcblookup_lb_numa_match(grp, domain)) { 2196 local_exact = grp; 2197 } 2198 } else if (grp->il_laddr.s_addr == INADDR_ANY && 2199 (lookupflags & INPLOOKUP_WILDCARD) != 0) { 2200 if (injail) { 2201 if (jail_wild == NULL || 2202 in_pcblookup_lb_numa_match(grp, domain)) 2203 jail_wild = grp; 2204 } else if (local_wild == NULL || 2205 in_pcblookup_lb_numa_match(grp, domain)) { 2206 local_wild = grp; 2207 } 2208 } 2209 } 2210 2211 if (jail_exact != NULL) 2212 grp = jail_exact; 2213 else if (jail_wild != NULL) 2214 grp = jail_wild; 2215 else if (local_exact != NULL) 2216 grp = local_exact; 2217 else 2218 grp = local_wild; 2219 if (grp == NULL) 2220 return (NULL); 2221 out: 2222 return (grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % 2223 grp->il_inpcnt]); 2224 } 2225 2226 /* 2227 * Lookup PCB in hash list, using pcbinfo tables. This variation assumes 2228 * that the caller has either locked the hash list, which usually happens 2229 * for bind(2) operations, or is in SMR section, which happens when sorting 2230 * out incoming packets. 2231 */ 2232 static struct inpcb * 2233 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2234 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, 2235 struct ifnet *ifp, uint8_t numa_domain) 2236 { 2237 struct inpcbhead *head; 2238 struct inpcb *inp, *tmpinp; 2239 u_short fport = fport_arg, lport = lport_arg; 2240 2241 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, 2242 ("%s: invalid lookup flags %d", __func__, lookupflags)); 2243 INP_HASH_LOCK_ASSERT(pcbinfo); 2244 2245 /* 2246 * First look for an exact match. 2247 */ 2248 tmpinp = NULL; 2249 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(&faddr, lport, fport, 2250 pcbinfo->ipi_hashmask)]; 2251 CK_LIST_FOREACH(inp, head, inp_hash) { 2252 #ifdef INET6 2253 /* XXX inp locking */ 2254 if ((inp->inp_vflag & INP_IPV4) == 0) 2255 continue; 2256 #endif 2257 if (inp->inp_faddr.s_addr == faddr.s_addr && 2258 inp->inp_laddr.s_addr == laddr.s_addr && 2259 inp->inp_fport == fport && 2260 inp->inp_lport == lport) { 2261 /* 2262 * XXX We should be able to directly return 2263 * the inp here, without any checks. 2264 * Well unless both bound with SO_REUSEPORT? 2265 */ 2266 if (prison_flag(inp->inp_cred, PR_IP4)) 2267 return (inp); 2268 if (tmpinp == NULL) 2269 tmpinp = inp; 2270 } 2271 } 2272 if (tmpinp != NULL) 2273 return (tmpinp); 2274 2275 /* 2276 * Then look for a wildcard match, if requested. 2277 */ 2278 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { 2279 struct inpcb *local_wild = NULL, *local_exact = NULL; 2280 #ifdef INET6 2281 struct inpcb *local_wild_mapped = NULL; 2282 #endif 2283 struct inpcb *jail_wild = NULL; 2284 int injail; 2285 2286 /* 2287 * First see if an LB group matches the request before scanning 2288 * all sockets on this port. 2289 */ 2290 inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr, 2291 fport, lookupflags, numa_domain); 2292 if (inp != NULL) 2293 return (inp); 2294 2295 /* 2296 * Order of socket selection - we always prefer jails. 2297 * 1. jailed, non-wild. 2298 * 2. jailed, wild. 2299 * 3. non-jailed, non-wild. 2300 * 4. non-jailed, wild. 2301 */ 2302 2303 head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, 2304 pcbinfo->ipi_hashmask)]; 2305 CK_LIST_FOREACH(inp, head, inp_hash) { 2306 #ifdef INET6 2307 /* XXX inp locking */ 2308 if ((inp->inp_vflag & INP_IPV4) == 0) 2309 continue; 2310 #endif 2311 if (inp->inp_faddr.s_addr != INADDR_ANY || 2312 inp->inp_lport != lport) 2313 continue; 2314 2315 injail = prison_flag(inp->inp_cred, PR_IP4); 2316 if (injail) { 2317 if (prison_check_ip4_locked( 2318 inp->inp_cred->cr_prison, &laddr) != 0) 2319 continue; 2320 } else { 2321 if (local_exact != NULL) 2322 continue; 2323 } 2324 2325 if (inp->inp_laddr.s_addr == laddr.s_addr) { 2326 if (injail) 2327 return (inp); 2328 else 2329 local_exact = inp; 2330 } else if (inp->inp_laddr.s_addr == INADDR_ANY) { 2331 #ifdef INET6 2332 /* XXX inp locking, NULL check */ 2333 if (inp->inp_vflag & INP_IPV6PROTO) 2334 local_wild_mapped = inp; 2335 else 2336 #endif 2337 if (injail) 2338 jail_wild = inp; 2339 else 2340 local_wild = inp; 2341 } 2342 } /* LIST_FOREACH */ 2343 if (jail_wild != NULL) 2344 return (jail_wild); 2345 if (local_exact != NULL) 2346 return (local_exact); 2347 if (local_wild != NULL) 2348 return (local_wild); 2349 #ifdef INET6 2350 if (local_wild_mapped != NULL) 2351 return (local_wild_mapped); 2352 #endif 2353 } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */ 2354 2355 return (NULL); 2356 } 2357 2358 /* 2359 * Lookup PCB in hash list, using pcbinfo tables. This variation locks the 2360 * hash list lock, and will return the inpcb locked (i.e., requires 2361 * INPLOOKUP_LOCKPCB). 2362 */ 2363 static struct inpcb * 2364 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2365 u_int fport, struct in_addr laddr, u_int lport, int lookupflags, 2366 struct ifnet *ifp, uint8_t numa_domain) 2367 { 2368 struct inpcb *inp; 2369 2370 smr_enter(pcbinfo->ipi_smr); 2371 inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, 2372 lookupflags & INPLOOKUP_WILDCARD, ifp, numa_domain); 2373 if (inp != NULL) { 2374 if (__predict_false(inp_smr_lock(inp, 2375 (lookupflags & INPLOOKUP_LOCKMASK)) == false)) 2376 inp = NULL; 2377 } else 2378 smr_exit(pcbinfo->ipi_smr); 2379 2380 return (inp); 2381 } 2382 2383 /* 2384 * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf 2385 * from which a pre-calculated hash value may be extracted. 2386 */ 2387 struct inpcb * 2388 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport, 2389 struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp) 2390 { 2391 2392 KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0, 2393 ("%s: invalid lookup flags %d", __func__, lookupflags)); 2394 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0, 2395 ("%s: LOCKPCB not set", __func__)); 2396 2397 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, 2398 lookupflags, ifp, M_NODOM)); 2399 } 2400 2401 struct inpcb * 2402 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2403 u_int fport, struct in_addr laddr, u_int lport, int lookupflags, 2404 struct ifnet *ifp, struct mbuf *m) 2405 { 2406 2407 KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0, 2408 ("%s: invalid lookup flags %d", __func__, lookupflags)); 2409 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0, 2410 ("%s: LOCKPCB not set", __func__)); 2411 2412 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, 2413 lookupflags, ifp, m->m_pkthdr.numa_domain)); 2414 } 2415 #endif /* INET */ 2416 2417 /* 2418 * Insert PCB onto various hash lists. 2419 */ 2420 int 2421 in_pcbinshash(struct inpcb *inp) 2422 { 2423 struct inpcbhead *pcbhash; 2424 struct inpcbporthead *pcbporthash; 2425 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2426 struct inpcbport *phd; 2427 2428 INP_WLOCK_ASSERT(inp); 2429 INP_HASH_WLOCK_ASSERT(pcbinfo); 2430 2431 KASSERT((inp->inp_flags & INP_INHASHLIST) == 0, 2432 ("in_pcbinshash: INP_INHASHLIST")); 2433 2434 #ifdef INET6 2435 if (inp->inp_vflag & INP_IPV6) 2436 pcbhash = &pcbinfo->ipi_hashbase[INP6_PCBHASH(&inp->in6p_faddr, 2437 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; 2438 else 2439 #endif 2440 pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(&inp->inp_faddr, 2441 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; 2442 2443 pcbporthash = &pcbinfo->ipi_porthashbase[ 2444 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)]; 2445 2446 /* 2447 * Add entry to load balance group. 2448 * Only do this if SO_REUSEPORT_LB is set. 2449 */ 2450 if ((inp->inp_flags2 & INP_REUSEPORT_LB) != 0) { 2451 int error = in_pcbinslbgrouphash(inp, M_NODOM); 2452 if (error != 0) 2453 return (error); 2454 } 2455 2456 /* 2457 * Go through port list and look for a head for this lport. 2458 */ 2459 CK_LIST_FOREACH(phd, pcbporthash, phd_hash) { 2460 if (phd->phd_port == inp->inp_lport) 2461 break; 2462 } 2463 2464 /* 2465 * If none exists, malloc one and tack it on. 2466 */ 2467 if (phd == NULL) { 2468 phd = uma_zalloc_smr(pcbinfo->ipi_portzone, M_NOWAIT); 2469 if (phd == NULL) { 2470 if ((inp->inp_flags2 & INP_REUSEPORT_LB) != 0) 2471 in_pcbremlbgrouphash(inp); 2472 return (ENOMEM); 2473 } 2474 phd->phd_port = inp->inp_lport; 2475 CK_LIST_INIT(&phd->phd_pcblist); 2476 CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 2477 } 2478 inp->inp_phd = phd; 2479 CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 2480 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash); 2481 inp->inp_flags |= INP_INHASHLIST; 2482 2483 return (0); 2484 } 2485 2486 static void 2487 in_pcbremhash(struct inpcb *inp) 2488 { 2489 struct inpcbport *phd = inp->inp_phd; 2490 2491 INP_WLOCK_ASSERT(inp); 2492 MPASS(inp->inp_flags & INP_INHASHLIST); 2493 2494 INP_HASH_WLOCK(inp->inp_pcbinfo); 2495 if ((inp->inp_flags2 & INP_REUSEPORT_LB) != 0) 2496 in_pcbremlbgrouphash(inp); 2497 CK_LIST_REMOVE(inp, inp_hash); 2498 CK_LIST_REMOVE(inp, inp_portlist); 2499 if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) { 2500 CK_LIST_REMOVE(phd, phd_hash); 2501 uma_zfree_smr(inp->inp_pcbinfo->ipi_portzone, phd); 2502 } 2503 INP_HASH_WUNLOCK(inp->inp_pcbinfo); 2504 inp->inp_flags &= ~INP_INHASHLIST; 2505 } 2506 2507 /* 2508 * Move PCB to the proper hash bucket when { faddr, fport } have been 2509 * changed. NOTE: This does not handle the case of the lport changing (the 2510 * hashed port list would have to be updated as well), so the lport must 2511 * not change after in_pcbinshash() has been called. 2512 * 2513 * XXXGL: a race between this function and SMR-protected hash iterator 2514 * will lead to iterator traversing a possibly wrong hash list. However, 2515 * this race should have been here since change from rwlock to epoch. 2516 */ 2517 void 2518 in_pcbrehash(struct inpcb *inp) 2519 { 2520 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2521 struct inpcbhead *head; 2522 2523 INP_WLOCK_ASSERT(inp); 2524 INP_HASH_WLOCK_ASSERT(pcbinfo); 2525 2526 KASSERT(inp->inp_flags & INP_INHASHLIST, 2527 ("in_pcbrehash: !INP_INHASHLIST")); 2528 2529 #ifdef INET6 2530 if (inp->inp_vflag & INP_IPV6) 2531 head = &pcbinfo->ipi_hashbase[INP6_PCBHASH(&inp->in6p_faddr, 2532 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; 2533 else 2534 #endif 2535 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(&inp->inp_faddr, 2536 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; 2537 2538 CK_LIST_REMOVE(inp, inp_hash); 2539 CK_LIST_INSERT_HEAD(head, inp, inp_hash); 2540 } 2541 2542 /* 2543 * Check for alternatives when higher level complains 2544 * about service problems. For now, invalidate cached 2545 * routing information. If the route was created dynamically 2546 * (by a redirect), time to try a default gateway again. 2547 */ 2548 void 2549 in_losing(struct inpcb *inp) 2550 { 2551 2552 RO_INVALIDATE_CACHE(&inp->inp_route); 2553 return; 2554 } 2555 2556 /* 2557 * A set label operation has occurred at the socket layer, propagate the 2558 * label change into the in_pcb for the socket. 2559 */ 2560 void 2561 in_pcbsosetlabel(struct socket *so) 2562 { 2563 #ifdef MAC 2564 struct inpcb *inp; 2565 2566 inp = sotoinpcb(so); 2567 KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL")); 2568 2569 INP_WLOCK(inp); 2570 SOCK_LOCK(so); 2571 mac_inpcb_sosetlabel(so, inp); 2572 SOCK_UNLOCK(so); 2573 INP_WUNLOCK(inp); 2574 #endif 2575 } 2576 2577 void 2578 inp_wlock(struct inpcb *inp) 2579 { 2580 2581 INP_WLOCK(inp); 2582 } 2583 2584 void 2585 inp_wunlock(struct inpcb *inp) 2586 { 2587 2588 INP_WUNLOCK(inp); 2589 } 2590 2591 void 2592 inp_rlock(struct inpcb *inp) 2593 { 2594 2595 INP_RLOCK(inp); 2596 } 2597 2598 void 2599 inp_runlock(struct inpcb *inp) 2600 { 2601 2602 INP_RUNLOCK(inp); 2603 } 2604 2605 #ifdef INVARIANT_SUPPORT 2606 void 2607 inp_lock_assert(struct inpcb *inp) 2608 { 2609 2610 INP_WLOCK_ASSERT(inp); 2611 } 2612 2613 void 2614 inp_unlock_assert(struct inpcb *inp) 2615 { 2616 2617 INP_UNLOCK_ASSERT(inp); 2618 } 2619 #endif 2620 2621 void 2622 inp_apply_all(struct inpcbinfo *pcbinfo, 2623 void (*func)(struct inpcb *, void *), void *arg) 2624 { 2625 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo, 2626 INPLOOKUP_WLOCKPCB); 2627 struct inpcb *inp; 2628 2629 while ((inp = inp_next(&inpi)) != NULL) 2630 func(inp, arg); 2631 } 2632 2633 struct socket * 2634 inp_inpcbtosocket(struct inpcb *inp) 2635 { 2636 2637 INP_WLOCK_ASSERT(inp); 2638 return (inp->inp_socket); 2639 } 2640 2641 struct tcpcb * 2642 inp_inpcbtotcpcb(struct inpcb *inp) 2643 { 2644 2645 INP_WLOCK_ASSERT(inp); 2646 return ((struct tcpcb *)inp->inp_ppcb); 2647 } 2648 2649 int 2650 inp_ip_tos_get(const struct inpcb *inp) 2651 { 2652 2653 return (inp->inp_ip_tos); 2654 } 2655 2656 void 2657 inp_ip_tos_set(struct inpcb *inp, int val) 2658 { 2659 2660 inp->inp_ip_tos = val; 2661 } 2662 2663 void 2664 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp, 2665 uint32_t *faddr, uint16_t *fp) 2666 { 2667 2668 INP_LOCK_ASSERT(inp); 2669 *laddr = inp->inp_laddr.s_addr; 2670 *faddr = inp->inp_faddr.s_addr; 2671 *lp = inp->inp_lport; 2672 *fp = inp->inp_fport; 2673 } 2674 2675 struct inpcb * 2676 so_sotoinpcb(struct socket *so) 2677 { 2678 2679 return (sotoinpcb(so)); 2680 } 2681 2682 /* 2683 * Create an external-format (``xinpcb'') structure using the information in 2684 * the kernel-format in_pcb structure pointed to by inp. This is done to 2685 * reduce the spew of irrelevant information over this interface, to isolate 2686 * user code from changes in the kernel structure, and potentially to provide 2687 * information-hiding if we decide that some of this information should be 2688 * hidden from users. 2689 */ 2690 void 2691 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi) 2692 { 2693 2694 bzero(xi, sizeof(*xi)); 2695 xi->xi_len = sizeof(struct xinpcb); 2696 if (inp->inp_socket) 2697 sotoxsocket(inp->inp_socket, &xi->xi_socket); 2698 bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo)); 2699 xi->inp_gencnt = inp->inp_gencnt; 2700 xi->inp_ppcb = (uintptr_t)inp->inp_ppcb; 2701 xi->inp_flow = inp->inp_flow; 2702 xi->inp_flowid = inp->inp_flowid; 2703 xi->inp_flowtype = inp->inp_flowtype; 2704 xi->inp_flags = inp->inp_flags; 2705 xi->inp_flags2 = inp->inp_flags2; 2706 xi->inp_rss_listen_bucket = inp->inp_rss_listen_bucket; 2707 xi->in6p_cksum = inp->in6p_cksum; 2708 xi->in6p_hops = inp->in6p_hops; 2709 xi->inp_ip_tos = inp->inp_ip_tos; 2710 xi->inp_vflag = inp->inp_vflag; 2711 xi->inp_ip_ttl = inp->inp_ip_ttl; 2712 xi->inp_ip_p = inp->inp_ip_p; 2713 xi->inp_ip_minttl = inp->inp_ip_minttl; 2714 } 2715 2716 int 2717 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo, 2718 int (*ctloutput_set)(struct inpcb *, struct sockopt *)) 2719 { 2720 struct sockopt sopt; 2721 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo, 2722 INPLOOKUP_WLOCKPCB); 2723 struct inpcb *inp; 2724 struct sockopt_parameters *params; 2725 struct socket *so; 2726 int error; 2727 char buf[1024]; 2728 2729 if (req->oldptr != NULL || req->oldlen != 0) 2730 return (EINVAL); 2731 if (req->newptr == NULL) 2732 return (EPERM); 2733 if (req->newlen > sizeof(buf)) 2734 return (ENOMEM); 2735 error = SYSCTL_IN(req, buf, req->newlen); 2736 if (error != 0) 2737 return (error); 2738 if (req->newlen < sizeof(struct sockopt_parameters)) 2739 return (EINVAL); 2740 params = (struct sockopt_parameters *)buf; 2741 sopt.sopt_level = params->sop_level; 2742 sopt.sopt_name = params->sop_optname; 2743 sopt.sopt_dir = SOPT_SET; 2744 sopt.sopt_val = params->sop_optval; 2745 sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters); 2746 sopt.sopt_td = NULL; 2747 #ifdef INET6 2748 if (params->sop_inc.inc_flags & INC_ISIPV6) { 2749 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_laddr)) 2750 params->sop_inc.inc6_laddr.s6_addr16[1] = 2751 htons(params->sop_inc.inc6_zoneid & 0xffff); 2752 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_faddr)) 2753 params->sop_inc.inc6_faddr.s6_addr16[1] = 2754 htons(params->sop_inc.inc6_zoneid & 0xffff); 2755 } 2756 #endif 2757 if (params->sop_inc.inc_lport != htons(0)) { 2758 if (params->sop_inc.inc_fport == htons(0)) 2759 inpi.hash = INP_PCBHASH_WILD(params->sop_inc.inc_lport, 2760 pcbinfo->ipi_hashmask); 2761 else 2762 #ifdef INET6 2763 if (params->sop_inc.inc_flags & INC_ISIPV6) 2764 inpi.hash = INP6_PCBHASH( 2765 ¶ms->sop_inc.inc6_faddr, 2766 params->sop_inc.inc_lport, 2767 params->sop_inc.inc_fport, 2768 pcbinfo->ipi_hashmask); 2769 else 2770 #endif 2771 inpi.hash = INP_PCBHASH( 2772 ¶ms->sop_inc.inc_faddr, 2773 params->sop_inc.inc_lport, 2774 params->sop_inc.inc_fport, 2775 pcbinfo->ipi_hashmask); 2776 } 2777 while ((inp = inp_next(&inpi)) != NULL) 2778 if (inp->inp_gencnt == params->sop_id) { 2779 if (inp->inp_flags & INP_DROPPED) { 2780 INP_WUNLOCK(inp); 2781 return (ECONNRESET); 2782 } 2783 so = inp->inp_socket; 2784 KASSERT(so != NULL, ("inp_socket == NULL")); 2785 soref(so); 2786 error = (*ctloutput_set)(inp, &sopt); 2787 sorele(so); 2788 break; 2789 } 2790 if (inp == NULL) 2791 error = ESRCH; 2792 return (error); 2793 } 2794 2795 #ifdef DDB 2796 static void 2797 db_print_indent(int indent) 2798 { 2799 int i; 2800 2801 for (i = 0; i < indent; i++) 2802 db_printf(" "); 2803 } 2804 2805 static void 2806 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent) 2807 { 2808 char faddr_str[48], laddr_str[48]; 2809 2810 db_print_indent(indent); 2811 db_printf("%s at %p\n", name, inc); 2812 2813 indent += 2; 2814 2815 #ifdef INET6 2816 if (inc->inc_flags & INC_ISIPV6) { 2817 /* IPv6. */ 2818 ip6_sprintf(laddr_str, &inc->inc6_laddr); 2819 ip6_sprintf(faddr_str, &inc->inc6_faddr); 2820 } else 2821 #endif 2822 { 2823 /* IPv4. */ 2824 inet_ntoa_r(inc->inc_laddr, laddr_str); 2825 inet_ntoa_r(inc->inc_faddr, faddr_str); 2826 } 2827 db_print_indent(indent); 2828 db_printf("inc_laddr %s inc_lport %u\n", laddr_str, 2829 ntohs(inc->inc_lport)); 2830 db_print_indent(indent); 2831 db_printf("inc_faddr %s inc_fport %u\n", faddr_str, 2832 ntohs(inc->inc_fport)); 2833 } 2834 2835 static void 2836 db_print_inpflags(int inp_flags) 2837 { 2838 int comma; 2839 2840 comma = 0; 2841 if (inp_flags & INP_RECVOPTS) { 2842 db_printf("%sINP_RECVOPTS", comma ? ", " : ""); 2843 comma = 1; 2844 } 2845 if (inp_flags & INP_RECVRETOPTS) { 2846 db_printf("%sINP_RECVRETOPTS", comma ? ", " : ""); 2847 comma = 1; 2848 } 2849 if (inp_flags & INP_RECVDSTADDR) { 2850 db_printf("%sINP_RECVDSTADDR", comma ? ", " : ""); 2851 comma = 1; 2852 } 2853 if (inp_flags & INP_ORIGDSTADDR) { 2854 db_printf("%sINP_ORIGDSTADDR", comma ? ", " : ""); 2855 comma = 1; 2856 } 2857 if (inp_flags & INP_HDRINCL) { 2858 db_printf("%sINP_HDRINCL", comma ? ", " : ""); 2859 comma = 1; 2860 } 2861 if (inp_flags & INP_HIGHPORT) { 2862 db_printf("%sINP_HIGHPORT", comma ? ", " : ""); 2863 comma = 1; 2864 } 2865 if (inp_flags & INP_LOWPORT) { 2866 db_printf("%sINP_LOWPORT", comma ? ", " : ""); 2867 comma = 1; 2868 } 2869 if (inp_flags & INP_ANONPORT) { 2870 db_printf("%sINP_ANONPORT", comma ? ", " : ""); 2871 comma = 1; 2872 } 2873 if (inp_flags & INP_RECVIF) { 2874 db_printf("%sINP_RECVIF", comma ? ", " : ""); 2875 comma = 1; 2876 } 2877 if (inp_flags & INP_MTUDISC) { 2878 db_printf("%sINP_MTUDISC", comma ? ", " : ""); 2879 comma = 1; 2880 } 2881 if (inp_flags & INP_RECVTTL) { 2882 db_printf("%sINP_RECVTTL", comma ? ", " : ""); 2883 comma = 1; 2884 } 2885 if (inp_flags & INP_DONTFRAG) { 2886 db_printf("%sINP_DONTFRAG", comma ? ", " : ""); 2887 comma = 1; 2888 } 2889 if (inp_flags & INP_RECVTOS) { 2890 db_printf("%sINP_RECVTOS", comma ? ", " : ""); 2891 comma = 1; 2892 } 2893 if (inp_flags & IN6P_IPV6_V6ONLY) { 2894 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : ""); 2895 comma = 1; 2896 } 2897 if (inp_flags & IN6P_PKTINFO) { 2898 db_printf("%sIN6P_PKTINFO", comma ? ", " : ""); 2899 comma = 1; 2900 } 2901 if (inp_flags & IN6P_HOPLIMIT) { 2902 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : ""); 2903 comma = 1; 2904 } 2905 if (inp_flags & IN6P_HOPOPTS) { 2906 db_printf("%sIN6P_HOPOPTS", comma ? ", " : ""); 2907 comma = 1; 2908 } 2909 if (inp_flags & IN6P_DSTOPTS) { 2910 db_printf("%sIN6P_DSTOPTS", comma ? ", " : ""); 2911 comma = 1; 2912 } 2913 if (inp_flags & IN6P_RTHDR) { 2914 db_printf("%sIN6P_RTHDR", comma ? ", " : ""); 2915 comma = 1; 2916 } 2917 if (inp_flags & IN6P_RTHDRDSTOPTS) { 2918 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : ""); 2919 comma = 1; 2920 } 2921 if (inp_flags & IN6P_TCLASS) { 2922 db_printf("%sIN6P_TCLASS", comma ? ", " : ""); 2923 comma = 1; 2924 } 2925 if (inp_flags & IN6P_AUTOFLOWLABEL) { 2926 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : ""); 2927 comma = 1; 2928 } 2929 if (inp_flags & INP_ONESBCAST) { 2930 db_printf("%sINP_ONESBCAST", comma ? ", " : ""); 2931 comma = 1; 2932 } 2933 if (inp_flags & INP_DROPPED) { 2934 db_printf("%sINP_DROPPED", comma ? ", " : ""); 2935 comma = 1; 2936 } 2937 if (inp_flags & INP_SOCKREF) { 2938 db_printf("%sINP_SOCKREF", comma ? ", " : ""); 2939 comma = 1; 2940 } 2941 if (inp_flags & IN6P_RFC2292) { 2942 db_printf("%sIN6P_RFC2292", comma ? ", " : ""); 2943 comma = 1; 2944 } 2945 if (inp_flags & IN6P_MTU) { 2946 db_printf("IN6P_MTU%s", comma ? ", " : ""); 2947 comma = 1; 2948 } 2949 } 2950 2951 static void 2952 db_print_inpvflag(u_char inp_vflag) 2953 { 2954 int comma; 2955 2956 comma = 0; 2957 if (inp_vflag & INP_IPV4) { 2958 db_printf("%sINP_IPV4", comma ? ", " : ""); 2959 comma = 1; 2960 } 2961 if (inp_vflag & INP_IPV6) { 2962 db_printf("%sINP_IPV6", comma ? ", " : ""); 2963 comma = 1; 2964 } 2965 if (inp_vflag & INP_IPV6PROTO) { 2966 db_printf("%sINP_IPV6PROTO", comma ? ", " : ""); 2967 comma = 1; 2968 } 2969 } 2970 2971 static void 2972 db_print_inpcb(struct inpcb *inp, const char *name, int indent) 2973 { 2974 2975 db_print_indent(indent); 2976 db_printf("%s at %p\n", name, inp); 2977 2978 indent += 2; 2979 2980 db_print_indent(indent); 2981 db_printf("inp_flow: 0x%x\n", inp->inp_flow); 2982 2983 db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent); 2984 2985 db_print_indent(indent); 2986 db_printf("inp_ppcb: %p inp_pcbinfo: %p inp_socket: %p\n", 2987 inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket); 2988 2989 db_print_indent(indent); 2990 db_printf("inp_label: %p inp_flags: 0x%x (", 2991 inp->inp_label, inp->inp_flags); 2992 db_print_inpflags(inp->inp_flags); 2993 db_printf(")\n"); 2994 2995 db_print_indent(indent); 2996 db_printf("inp_sp: %p inp_vflag: 0x%x (", inp->inp_sp, 2997 inp->inp_vflag); 2998 db_print_inpvflag(inp->inp_vflag); 2999 db_printf(")\n"); 3000 3001 db_print_indent(indent); 3002 db_printf("inp_ip_ttl: %d inp_ip_p: %d inp_ip_minttl: %d\n", 3003 inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl); 3004 3005 db_print_indent(indent); 3006 #ifdef INET6 3007 if (inp->inp_vflag & INP_IPV6) { 3008 db_printf("in6p_options: %p in6p_outputopts: %p " 3009 "in6p_moptions: %p\n", inp->in6p_options, 3010 inp->in6p_outputopts, inp->in6p_moptions); 3011 db_printf("in6p_icmp6filt: %p in6p_cksum %d " 3012 "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum, 3013 inp->in6p_hops); 3014 } else 3015 #endif 3016 { 3017 db_printf("inp_ip_tos: %d inp_ip_options: %p " 3018 "inp_ip_moptions: %p\n", inp->inp_ip_tos, 3019 inp->inp_options, inp->inp_moptions); 3020 } 3021 3022 db_print_indent(indent); 3023 db_printf("inp_phd: %p inp_gencnt: %ju\n", inp->inp_phd, 3024 (uintmax_t)inp->inp_gencnt); 3025 } 3026 3027 DB_SHOW_COMMAND(inpcb, db_show_inpcb) 3028 { 3029 struct inpcb *inp; 3030 3031 if (!have_addr) { 3032 db_printf("usage: show inpcb <addr>\n"); 3033 return; 3034 } 3035 inp = (struct inpcb *)addr; 3036 3037 db_print_inpcb(inp, "inpcb", 0); 3038 } 3039 #endif /* DDB */ 3040 3041 #ifdef RATELIMIT 3042 /* 3043 * Modify TX rate limit based on the existing "inp->inp_snd_tag", 3044 * if any. 3045 */ 3046 int 3047 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate) 3048 { 3049 union if_snd_tag_modify_params params = { 3050 .rate_limit.max_rate = max_pacing_rate, 3051 .rate_limit.flags = M_NOWAIT, 3052 }; 3053 struct m_snd_tag *mst; 3054 int error; 3055 3056 mst = inp->inp_snd_tag; 3057 if (mst == NULL) 3058 return (EINVAL); 3059 3060 if (mst->sw->snd_tag_modify == NULL) { 3061 error = EOPNOTSUPP; 3062 } else { 3063 error = mst->sw->snd_tag_modify(mst, ¶ms); 3064 } 3065 return (error); 3066 } 3067 3068 /* 3069 * Query existing TX rate limit based on the existing 3070 * "inp->inp_snd_tag", if any. 3071 */ 3072 int 3073 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate) 3074 { 3075 union if_snd_tag_query_params params = { }; 3076 struct m_snd_tag *mst; 3077 int error; 3078 3079 mst = inp->inp_snd_tag; 3080 if (mst == NULL) 3081 return (EINVAL); 3082 3083 if (mst->sw->snd_tag_query == NULL) { 3084 error = EOPNOTSUPP; 3085 } else { 3086 error = mst->sw->snd_tag_query(mst, ¶ms); 3087 if (error == 0 && p_max_pacing_rate != NULL) 3088 *p_max_pacing_rate = params.rate_limit.max_rate; 3089 } 3090 return (error); 3091 } 3092 3093 /* 3094 * Query existing TX queue level based on the existing 3095 * "inp->inp_snd_tag", if any. 3096 */ 3097 int 3098 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level) 3099 { 3100 union if_snd_tag_query_params params = { }; 3101 struct m_snd_tag *mst; 3102 int error; 3103 3104 mst = inp->inp_snd_tag; 3105 if (mst == NULL) 3106 return (EINVAL); 3107 3108 if (mst->sw->snd_tag_query == NULL) 3109 return (EOPNOTSUPP); 3110 3111 error = mst->sw->snd_tag_query(mst, ¶ms); 3112 if (error == 0 && p_txqueue_level != NULL) 3113 *p_txqueue_level = params.rate_limit.queue_level; 3114 return (error); 3115 } 3116 3117 /* 3118 * Allocate a new TX rate limit send tag from the network interface 3119 * given by the "ifp" argument and save it in "inp->inp_snd_tag": 3120 */ 3121 int 3122 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp, 3123 uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st) 3124 3125 { 3126 union if_snd_tag_alloc_params params = { 3127 .rate_limit.hdr.type = (max_pacing_rate == -1U) ? 3128 IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT, 3129 .rate_limit.hdr.flowid = flowid, 3130 .rate_limit.hdr.flowtype = flowtype, 3131 .rate_limit.hdr.numa_domain = inp->inp_numa_domain, 3132 .rate_limit.max_rate = max_pacing_rate, 3133 .rate_limit.flags = M_NOWAIT, 3134 }; 3135 int error; 3136 3137 INP_WLOCK_ASSERT(inp); 3138 3139 /* 3140 * If there is already a send tag, or the INP is being torn 3141 * down, allocating a new send tag is not allowed. Else send 3142 * tags may leak. 3143 */ 3144 if (*st != NULL || (inp->inp_flags & INP_DROPPED) != 0) 3145 return (EINVAL); 3146 3147 error = m_snd_tag_alloc(ifp, ¶ms, st); 3148 #ifdef INET 3149 if (error == 0) { 3150 counter_u64_add(rate_limit_set_ok, 1); 3151 counter_u64_add(rate_limit_active, 1); 3152 } else if (error != EOPNOTSUPP) 3153 counter_u64_add(rate_limit_alloc_fail, 1); 3154 #endif 3155 return (error); 3156 } 3157 3158 void 3159 in_pcbdetach_tag(struct m_snd_tag *mst) 3160 { 3161 3162 m_snd_tag_rele(mst); 3163 #ifdef INET 3164 counter_u64_add(rate_limit_active, -1); 3165 #endif 3166 } 3167 3168 /* 3169 * Free an existing TX rate limit tag based on the "inp->inp_snd_tag", 3170 * if any: 3171 */ 3172 void 3173 in_pcbdetach_txrtlmt(struct inpcb *inp) 3174 { 3175 struct m_snd_tag *mst; 3176 3177 INP_WLOCK_ASSERT(inp); 3178 3179 mst = inp->inp_snd_tag; 3180 inp->inp_snd_tag = NULL; 3181 3182 if (mst == NULL) 3183 return; 3184 3185 m_snd_tag_rele(mst); 3186 #ifdef INET 3187 counter_u64_add(rate_limit_active, -1); 3188 #endif 3189 } 3190 3191 int 3192 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate) 3193 { 3194 int error; 3195 3196 /* 3197 * If the existing send tag is for the wrong interface due to 3198 * a route change, first drop the existing tag. Set the 3199 * CHANGED flag so that we will keep trying to allocate a new 3200 * tag if we fail to allocate one this time. 3201 */ 3202 if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) { 3203 in_pcbdetach_txrtlmt(inp); 3204 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED; 3205 } 3206 3207 /* 3208 * NOTE: When attaching to a network interface a reference is 3209 * made to ensure the network interface doesn't go away until 3210 * all ratelimit connections are gone. The network interface 3211 * pointers compared below represent valid network interfaces, 3212 * except when comparing towards NULL. 3213 */ 3214 if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) { 3215 error = 0; 3216 } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) { 3217 if (inp->inp_snd_tag != NULL) 3218 in_pcbdetach_txrtlmt(inp); 3219 error = 0; 3220 } else if (inp->inp_snd_tag == NULL) { 3221 /* 3222 * In order to utilize packet pacing with RSS, we need 3223 * to wait until there is a valid RSS hash before we 3224 * can proceed: 3225 */ 3226 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) { 3227 error = EAGAIN; 3228 } else { 3229 error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb), 3230 mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag); 3231 } 3232 } else { 3233 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate); 3234 } 3235 if (error == 0 || error == EOPNOTSUPP) 3236 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 3237 3238 return (error); 3239 } 3240 3241 /* 3242 * This function should be called when the INP_RATE_LIMIT_CHANGED flag 3243 * is set in the fast path and will attach/detach/modify the TX rate 3244 * limit send tag based on the socket's so_max_pacing_rate value. 3245 */ 3246 void 3247 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb) 3248 { 3249 struct socket *socket; 3250 uint32_t max_pacing_rate; 3251 bool did_upgrade; 3252 3253 if (inp == NULL) 3254 return; 3255 3256 socket = inp->inp_socket; 3257 if (socket == NULL) 3258 return; 3259 3260 if (!INP_WLOCKED(inp)) { 3261 /* 3262 * NOTE: If the write locking fails, we need to bail 3263 * out and use the non-ratelimited ring for the 3264 * transmit until there is a new chance to get the 3265 * write lock. 3266 */ 3267 if (!INP_TRY_UPGRADE(inp)) 3268 return; 3269 did_upgrade = 1; 3270 } else { 3271 did_upgrade = 0; 3272 } 3273 3274 /* 3275 * NOTE: The so_max_pacing_rate value is read unlocked, 3276 * because atomic updates are not required since the variable 3277 * is checked at every mbuf we send. It is assumed that the 3278 * variable read itself will be atomic. 3279 */ 3280 max_pacing_rate = socket->so_max_pacing_rate; 3281 3282 in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate); 3283 3284 if (did_upgrade) 3285 INP_DOWNGRADE(inp); 3286 } 3287 3288 /* 3289 * Track route changes for TX rate limiting. 3290 */ 3291 void 3292 in_pcboutput_eagain(struct inpcb *inp) 3293 { 3294 bool did_upgrade; 3295 3296 if (inp == NULL) 3297 return; 3298 3299 if (inp->inp_snd_tag == NULL) 3300 return; 3301 3302 if (!INP_WLOCKED(inp)) { 3303 /* 3304 * NOTE: If the write locking fails, we need to bail 3305 * out and use the non-ratelimited ring for the 3306 * transmit until there is a new chance to get the 3307 * write lock. 3308 */ 3309 if (!INP_TRY_UPGRADE(inp)) 3310 return; 3311 did_upgrade = 1; 3312 } else { 3313 did_upgrade = 0; 3314 } 3315 3316 /* detach rate limiting */ 3317 in_pcbdetach_txrtlmt(inp); 3318 3319 /* make sure new mbuf send tag allocation is made */ 3320 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED; 3321 3322 if (did_upgrade) 3323 INP_DOWNGRADE(inp); 3324 } 3325 3326 #ifdef INET 3327 static void 3328 rl_init(void *st) 3329 { 3330 rate_limit_new = counter_u64_alloc(M_WAITOK); 3331 rate_limit_chg = counter_u64_alloc(M_WAITOK); 3332 rate_limit_active = counter_u64_alloc(M_WAITOK); 3333 rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK); 3334 rate_limit_set_ok = counter_u64_alloc(M_WAITOK); 3335 } 3336 3337 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL); 3338 #endif 3339 #endif /* RATELIMIT */ 3340