1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/domain.h> 74 #include <sys/protosw.h> 75 #include <sys/socket.h> 76 #include <sys/socketvar.h> 77 #include <sys/proc.h> 78 #include <sys/priv.h> 79 #include <sys/jail.h> 80 #include <sys/kernel.h> 81 #include <sys/sysctl.h> 82 83 #include <sys/thread2.h> 84 #include <sys/socketvar2.h> 85 #include <sys/msgport2.h> 86 87 #include <machine/limits.h> 88 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/route.h> 92 #include <net/netisr2.h> 93 #include <net/toeplitz2.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip_var.h> 99 #ifdef INET6 100 #include <netinet/ip6.h> 101 #include <netinet6/ip6_var.h> 102 #endif /* INET6 */ 103 104 #ifdef IPSEC 105 #include <netinet6/ipsec.h> 106 #include <netproto/key/key.h> 107 #include <netproto/ipsec/esp_var.h> 108 #endif 109 110 #ifdef FAST_IPSEC 111 #if defined(IPSEC) || defined(IPSEC_ESP) 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!" 113 #endif 114 115 #include <netproto/ipsec/ipsec.h> 116 #include <netproto/ipsec/key.h> 117 #define IPSEC 118 #endif /* FAST_IPSEC */ 119 120 #define INP_LOCALGROUP_SIZMIN 8 121 #define INP_LOCALGROUP_SIZMAX 256 122 123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash, 124 struct in_addr laddr, u_int lport_arg, int wild_okay, 125 struct ucred *cred); 126 127 struct in_addr zeroin_addr; 128 129 /* 130 * These configure the range of local port addresses assigned to 131 * "unspecified" outgoing connections/packets/whatever. 132 */ 133 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 134 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 135 136 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 137 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 138 139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 140 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 141 142 #define RANGECHK(var, min, max) \ 143 if ((var) < (min)) { (var) = (min); } \ 144 else if ((var) > (max)) { (var) = (max); } 145 146 int udpencap_enable = 1; /* enabled by default */ 147 int udpencap_port = 4500; /* triggers decapsulation */ 148 149 /* 150 * Per-netisr inpcb markers. 151 * NOTE: they should only be used in netisrs. 152 */ 153 static struct inpcb *in_pcbmarkers; 154 static struct inpcontainer *in_pcbcontainer_markers; 155 156 static int 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 158 { 159 int error; 160 161 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 162 if (!error) { 163 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 164 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 165 166 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 167 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 168 169 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 170 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 171 } 172 return (error); 173 } 174 175 #undef RANGECHK 176 177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 178 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 180 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 182 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 184 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 186 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 188 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 190 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 191 192 static int ip_porthash_trycount = 15; 193 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW, 194 &ip_porthash_trycount, 0, 195 "Number of tries to find local port matching hash of 4-tuple"); 196 197 /* 198 * in_pcb.c: manage the Protocol Control Blocks. 199 * 200 * NOTE: It is assumed that most of these functions will be called from 201 * a critical section. XXX - There are, unfortunately, a few exceptions 202 * to this rule that should be fixed. 203 * 204 * NOTE: The caller should initialize the cpu field to the cpu running the 205 * protocol stack associated with this inpcbinfo. 206 */ 207 208 void 209 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 210 { 211 KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu)); 212 pcbinfo->cpu = cpu; 213 214 LIST_INIT(&pcbinfo->pcblisthead); 215 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 216 M_WAITOK | M_ZERO); 217 218 if (shared) { 219 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 220 M_PCB, M_WAITOK); 221 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 222 } else { 223 pcbinfo->infotoken = NULL; 224 } 225 } 226 227 void 228 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo, 229 int portinfo_cnt) 230 { 231 232 KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt)); 233 pcbinfo->portinfo = portinfo; 234 pcbinfo->portinfo_cnt = portinfo_cnt; 235 } 236 237 struct baddynamicports baddynamicports; 238 239 /* 240 * Check if the specified port is invalid for dynamic allocation. 241 */ 242 int 243 in_baddynamic(u_int16_t port, u_int16_t proto) 244 { 245 switch (proto) { 246 case IPPROTO_TCP: 247 return (DP_ISSET(baddynamicports.tcp, port)); 248 case IPPROTO_UDP: 249 #ifdef IPSEC 250 /* Cannot preset this as it is a sysctl */ 251 if (port == udpencap_port) 252 return (1); 253 #endif 254 return (DP_ISSET(baddynamicports.udp, port)); 255 default: 256 return (0); 257 } 258 } 259 260 void 261 in_pcbonlist(struct inpcb *inp) 262 { 263 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 264 265 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 266 ("not in the correct netisr")); 267 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 268 inp->inp_flags |= INP_ONLIST; 269 270 GET_PCBINFO_TOKEN(pcbinfo); 271 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 272 pcbinfo->ipi_count++; 273 REL_PCBINFO_TOKEN(pcbinfo); 274 } 275 276 void 277 in_pcbofflist(struct inpcb *inp) 278 { 279 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 280 281 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 282 ("not in the correct netisr")); 283 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 284 inp->inp_flags &= ~INP_ONLIST; 285 286 GET_PCBINFO_TOKEN(pcbinfo); 287 LIST_REMOVE(inp, inp_list); 288 KASSERT(pcbinfo->ipi_count > 0, 289 ("invalid inpcb count %d", pcbinfo->ipi_count)); 290 pcbinfo->ipi_count--; 291 REL_PCBINFO_TOKEN(pcbinfo); 292 } 293 294 /* 295 * Allocate a PCB and associate it with the socket. 296 */ 297 int 298 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 299 { 300 struct inpcb *inp; 301 #ifdef IPSEC 302 int error; 303 #endif 304 305 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 306 if (inp == NULL) 307 return (ENOMEM); 308 inp->inp_lgrpindex = -1; 309 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 310 inp->inp_pcbinfo = pcbinfo; 311 inp->inp_socket = so; 312 #ifdef IPSEC 313 error = ipsec_init_policy(so, &inp->inp_sp); 314 if (error != 0) { 315 kfree(inp, M_PCB); 316 return (error); 317 } 318 #endif 319 #ifdef INET6 320 if (INP_CHECK_SOCKAF(so, AF_INET6)) { 321 if (ip6_auto_flowlabel) 322 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 323 inp->inp_af = AF_INET6; 324 } else 325 #endif 326 inp->inp_af = AF_INET; 327 soreference(so); 328 so->so_pcb = inp; 329 330 in_pcbonlist(inp); 331 return (0); 332 } 333 334 /* 335 * Unlink a pcb with the intention of moving it to another cpu with a 336 * different pcbinfo. While unlinked nothing should attempt to dereference 337 * inp_pcbinfo, NULL it out so we assert if it does. 338 */ 339 void 340 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 341 { 342 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 343 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 344 ("already linked")); 345 346 in_pcbofflist(inp); 347 inp->inp_pcbinfo = NULL; 348 } 349 350 void 351 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 352 { 353 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 354 } 355 356 /* 357 * Relink a pcb into a new pcbinfo. 358 */ 359 void 360 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 361 { 362 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 363 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 364 ("already linked")); 365 366 inp->inp_pcbinfo = pcbinfo; 367 in_pcbonlist(inp); 368 } 369 370 void 371 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 372 { 373 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 374 } 375 376 static boolean_t 377 in_pcbporthash_update(struct inpcbportinfo *portinfo, 378 struct inpcb *inp, u_short lport, struct ucred *cred, int wild) 379 { 380 struct inpcbporthead *porthash; 381 382 /* 383 * This has to be atomic. If the porthash is shared across multiple 384 * protocol threads, e.g. tcp and udp, then the token must be held. 385 */ 386 porthash = in_pcbporthash_head(portinfo, lport); 387 GET_PORTHASH_TOKEN(porthash); 388 389 if (in_pcblookup_local(porthash, inp->inp_laddr, lport, 390 wild, cred) != NULL) { 391 REL_PORTHASH_TOKEN(porthash); 392 return FALSE; 393 } 394 inp->inp_lport = lport; 395 in_pcbinsporthash(porthash, inp); 396 397 REL_PORTHASH_TOKEN(porthash); 398 return TRUE; 399 } 400 401 static int 402 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 403 { 404 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 405 struct inpcbportinfo *portinfo; 406 u_short first, last, lport, step; 407 u_short *lastport; 408 int count, error; 409 int portinfo_first, portinfo_idx; 410 411 inp->inp_flags |= INP_ANONPORT; 412 413 step = pcbinfo->portinfo_cnt; 414 portinfo_first = mycpuid % pcbinfo->portinfo_cnt; 415 portinfo_idx = portinfo_first; 416 loop: 417 portinfo = &pcbinfo->portinfo[portinfo_idx]; 418 419 if (inp->inp_flags & INP_HIGHPORT) { 420 first = ipport_hifirstauto; /* sysctl */ 421 last = ipport_hilastauto; 422 lastport = &portinfo->lasthi; 423 } else if (inp->inp_flags & INP_LOWPORT) { 424 if (cred && 425 (error = 426 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 427 inp->inp_laddr.s_addr = INADDR_ANY; 428 return error; 429 } 430 first = ipport_lowfirstauto; /* 1023 */ 431 last = ipport_lowlastauto; /* 600 */ 432 lastport = &portinfo->lastlow; 433 } else { 434 first = ipport_firstauto; /* sysctl */ 435 last = ipport_lastauto; 436 lastport = &portinfo->lastport; 437 } 438 439 /* 440 * Simple check to ensure all ports are not used up causing 441 * a deadlock here. 442 * 443 * We split the two cases (up and down) so that the direction 444 * is not being tested on each round of the loop. 445 */ 446 if (first > last) { 447 /* 448 * counting down 449 */ 450 in_pcbportrange(&first, &last, portinfo->offset, step); 451 count = (first - last) / step; 452 453 for (;;) { 454 if (count-- < 0) { /* completely used? */ 455 error = EADDRNOTAVAIL; 456 break; 457 } 458 lport = in_pcblastport_down(lastport, 459 first, last, step); 460 KKASSERT((lport % pcbinfo->portinfo_cnt) == 461 portinfo->offset); 462 lport = htons(lport); 463 464 if (in_pcbporthash_update(portinfo, inp, lport, 465 cred, wild)) { 466 error = 0; 467 break; 468 } 469 } 470 } else { 471 /* 472 * counting up 473 */ 474 in_pcbportrange(&last, &first, portinfo->offset, step); 475 count = (last - first) / step; 476 477 for (;;) { 478 if (count-- < 0) { /* completely used? */ 479 error = EADDRNOTAVAIL; 480 break; 481 } 482 lport = in_pcblastport_up(lastport, first, last, step); 483 KKASSERT((lport % pcbinfo->portinfo_cnt) == 484 portinfo->offset); 485 lport = htons(lport); 486 487 if (in_pcbporthash_update(portinfo, inp, lport, 488 cred, wild)) { 489 error = 0; 490 break; 491 } 492 } 493 } 494 495 if (error) { 496 /* Try next portinfo */ 497 portinfo_idx++; 498 portinfo_idx %= pcbinfo->portinfo_cnt; 499 if (portinfo_idx != portinfo_first) 500 goto loop; 501 inp->inp_laddr.s_addr = INADDR_ANY; 502 } 503 return error; 504 } 505 506 int 507 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 508 { 509 struct socket *so = inp->inp_socket; 510 struct sockaddr_in jsin; 511 struct ucred *cred = NULL; 512 int wild = 0; 513 514 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 515 return (EADDRNOTAVAIL); 516 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 517 return (EINVAL); /* already bound */ 518 519 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 520 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 521 if (td->td_proc) 522 cred = td->td_proc->p_ucred; 523 524 if (nam != NULL) { 525 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 526 struct inpcbinfo *pcbinfo; 527 struct inpcbportinfo *portinfo; 528 struct inpcbporthead *porthash; 529 struct inpcb *t; 530 u_short lport, lport_ho; 531 int reuseport = (so->so_options & SO_REUSEPORT); 532 int error; 533 534 if (nam->sa_len != sizeof *sin) 535 return (EINVAL); 536 #ifdef notdef 537 /* 538 * We should check the family, but old programs 539 * incorrectly fail to initialize it. 540 */ 541 if (sin->sin_family != AF_INET) 542 return (EAFNOSUPPORT); 543 #endif 544 if (!prison_replace_wildcards(td, nam)) 545 return (EINVAL); 546 547 lport = sin->sin_port; 548 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 549 /* 550 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 551 * allow complete duplication of binding if 552 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 553 * and a multicast address is bound on both 554 * new and duplicated sockets. 555 */ 556 if (so->so_options & SO_REUSEADDR) 557 reuseport = SO_REUSEADDR | SO_REUSEPORT; 558 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 559 sin->sin_port = 0; /* yech... */ 560 bzero(&sin->sin_zero, sizeof sin->sin_zero); 561 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 562 return (EADDRNOTAVAIL); 563 } 564 565 inp->inp_laddr = sin->sin_addr; 566 567 jsin.sin_family = AF_INET; 568 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 569 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 570 inp->inp_laddr.s_addr = INADDR_ANY; 571 return (EINVAL); 572 } 573 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 574 575 if (lport == 0) { 576 /* Auto-select local port */ 577 return in_pcbsetlport(inp, wild, cred); 578 } 579 lport_ho = ntohs(lport); 580 581 /* GROSS */ 582 if (lport_ho < IPPORT_RESERVED && cred && 583 (error = 584 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 585 inp->inp_laddr.s_addr = INADDR_ANY; 586 return (error); 587 } 588 589 /* 590 * Locate the proper portinfo based on lport 591 */ 592 pcbinfo = inp->inp_pcbinfo; 593 portinfo = 594 &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 595 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == 596 portinfo->offset); 597 598 /* 599 * This has to be atomic. If the porthash is shared across 600 * multiple protocol threads, e.g. tcp and udp then the token 601 * must be held. 602 */ 603 porthash = in_pcbporthash_head(portinfo, lport); 604 GET_PORTHASH_TOKEN(porthash); 605 606 if (so->so_cred->cr_uid != 0 && 607 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 608 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 609 INPLOOKUP_WILDCARD, cred); 610 if (t && 611 (so->so_cred->cr_uid != 612 t->inp_socket->so_cred->cr_uid)) { 613 inp->inp_laddr.s_addr = INADDR_ANY; 614 error = EADDRINUSE; 615 goto done; 616 } 617 } 618 if (cred && !prison_replace_wildcards(td, nam)) { 619 inp->inp_laddr.s_addr = INADDR_ANY; 620 error = EADDRNOTAVAIL; 621 goto done; 622 } 623 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 624 wild, cred); 625 if (t && !(reuseport & t->inp_socket->so_options)) { 626 inp->inp_laddr.s_addr = INADDR_ANY; 627 error = EADDRINUSE; 628 goto done; 629 } 630 inp->inp_lport = lport; 631 in_pcbinsporthash(porthash, inp); 632 error = 0; 633 done: 634 REL_PORTHASH_TOKEN(porthash); 635 return (error); 636 } else { 637 jsin.sin_family = AF_INET; 638 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 639 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 640 inp->inp_laddr.s_addr = INADDR_ANY; 641 return (EINVAL); 642 } 643 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 644 645 return in_pcbsetlport(inp, wild, cred); 646 } 647 } 648 649 static struct inpcb * 650 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr, 651 u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred) 652 { 653 struct inpcb *inp; 654 struct inpcbport *phd; 655 struct inpcb *match = NULL; 656 657 /* 658 * If the porthashbase is shared across several cpus, it must 659 * have been locked. 660 */ 661 ASSERT_PORTHASH_TOKEN_HELD(porthash); 662 663 /* 664 * Best fit PCB lookup. 665 * 666 * First see if this local port is in use by looking on the 667 * port hash list. 668 */ 669 LIST_FOREACH(phd, porthash, phd_hash) { 670 if (phd->phd_port == lport) 671 break; 672 } 673 if (phd != NULL) { 674 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 675 #ifdef INET6 676 if (!INP_ISIPV4(inp)) 677 continue; 678 #endif 679 if (inp->inp_laddr.s_addr != INADDR_ANY && 680 inp->inp_laddr.s_addr != laddr.s_addr) 681 continue; 682 683 if (inp->inp_faddr.s_addr != INADDR_ANY && 684 inp->inp_faddr.s_addr != faddr.s_addr) 685 continue; 686 687 if (inp->inp_fport != 0 && inp->inp_fport != fport) 688 continue; 689 690 if (cred == NULL || 691 cred->cr_prison == 692 inp->inp_socket->so_cred->cr_prison) { 693 match = inp; 694 break; 695 } 696 } 697 } 698 return (match); 699 } 700 701 static boolean_t 702 in_pcbporthash_update4(struct inpcbportinfo *portinfo, 703 struct inpcb *inp, u_short lport, const struct sockaddr_in *sin, 704 struct ucred *cred) 705 { 706 struct inpcbporthead *porthash; 707 708 /* 709 * This has to be atomic. If the porthash is shared across multiple 710 * protocol threads, e.g. tcp and udp, then the token must be held. 711 */ 712 porthash = in_pcbporthash_head(portinfo, lport); 713 GET_PORTHASH_TOKEN(porthash); 714 715 if (in_pcblookup_localremote(porthash, inp->inp_laddr, 716 lport, sin->sin_addr, sin->sin_port, cred) != NULL) { 717 REL_PORTHASH_TOKEN(porthash); 718 return FALSE; 719 } 720 inp->inp_lport = lport; 721 in_pcbinsporthash(porthash, inp); 722 723 REL_PORTHASH_TOKEN(porthash); 724 return TRUE; 725 } 726 727 int 728 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 729 struct thread *td) 730 { 731 struct proc *p = td->td_proc; 732 u_short *lastport; 733 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 734 struct sockaddr_in jsin; 735 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 736 struct inpcbportinfo *portinfo; 737 struct ucred *cred = NULL; 738 u_short first, last, lport, step; 739 int count, error, selfconn; 740 int portinfo_first, portinfo_idx; 741 int hash_cpu, hash_count, hash_count0; 742 uint32_t hash_base = 0, hash; 743 744 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 745 return (EADDRNOTAVAIL); 746 747 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 748 if (inp->inp_lport != 0) 749 return (EINVAL); /* already bound */ 750 751 KKASSERT(p); 752 cred = p->p_ucred; 753 754 jsin.sin_family = AF_INET; 755 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 756 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 757 inp->inp_laddr.s_addr = INADDR_ANY; 758 return (EINVAL); 759 } 760 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 761 762 hash_count0 = ip_porthash_trycount; 763 if (hash_count0 > 0) { 764 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^ 765 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^ 766 toeplitz_piecemeal_port(sin->sin_port); 767 } else { 768 hash_count0 = 0; 769 } 770 771 inp->inp_flags |= INP_ANONPORT; 772 773 step = pcbinfo->portinfo_cnt; 774 portinfo_first = mycpuid % pcbinfo->portinfo_cnt; 775 portinfo_idx = portinfo_first; 776 loop: 777 hash_cpu = portinfo_idx % netisr_ncpus; 778 portinfo = &pcbinfo->portinfo[portinfo_idx]; 779 selfconn = 0; 780 781 if (inp->inp_flags & INP_HIGHPORT) { 782 first = ipport_hifirstauto; /* sysctl */ 783 last = ipport_hilastauto; 784 lastport = &portinfo->lasthi; 785 } else if (inp->inp_flags & INP_LOWPORT) { 786 if (cred && 787 (error = 788 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 789 inp->inp_laddr.s_addr = INADDR_ANY; 790 return (error); 791 } 792 first = ipport_lowfirstauto; /* 1023 */ 793 last = ipport_lowlastauto; /* 600 */ 794 lastport = &portinfo->lastlow; 795 } else { 796 first = ipport_firstauto; /* sysctl */ 797 last = ipport_lastauto; 798 lastport = &portinfo->lastport; 799 } 800 801 /* This could happen on loopback interface */ 802 #define IS_SELFCONNECT(inp, lport, sin) \ 803 (__predict_false((sin)->sin_port == (lport) && \ 804 (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr)) 805 806 /* 807 * Simple check to ensure all ports are not used up causing 808 * a deadlock here. 809 * 810 * We split the two cases (up and down) so that the direction 811 * is not being tested on each round of the loop. 812 */ 813 hash_count = hash_count0; 814 if (first > last) { 815 /* 816 * counting down 817 */ 818 in_pcbportrange(&first, &last, portinfo->offset, step); 819 count = ((first - last) / step) + hash_count; 820 821 for (;;) { 822 if (count-- < 0) { /* completely used? */ 823 error = EADDRNOTAVAIL; 824 break; 825 } 826 827 lport = in_pcblastport_down(lastport, first, last, 828 step); 829 KKASSERT((lport % pcbinfo->portinfo_cnt) == 830 portinfo->offset); 831 lport = htons(lport); 832 if (IS_SELFCONNECT(inp, lport, sin)) { 833 if (!selfconn) { 834 ++count; /* don't count this try */ 835 selfconn = 1; 836 } 837 continue; 838 } 839 840 if (hash_count) { 841 --hash_count; 842 hash = hash_base ^ 843 toeplitz_piecemeal_port(lport); 844 if (netisr_hashcpu(hash) != hash_cpu && 845 hash_count) 846 continue; 847 } 848 849 if (in_pcbporthash_update4(portinfo, 850 inp, lport, sin, cred)) { 851 error = 0; 852 break; 853 } 854 } 855 } else { 856 /* 857 * counting up 858 */ 859 in_pcbportrange(&last, &first, portinfo->offset, step); 860 count = ((last - first) / step) + hash_count; 861 862 for (;;) { 863 if (count-- < 0) { /* completely used? */ 864 error = EADDRNOTAVAIL; 865 break; 866 } 867 868 lport = in_pcblastport_up(lastport, first, last, step); 869 KKASSERT((lport % pcbinfo->portinfo_cnt) == 870 portinfo->offset); 871 lport = htons(lport); 872 if (IS_SELFCONNECT(inp, lport, sin)) { 873 if (!selfconn) { 874 ++count; /* don't count this try */ 875 selfconn = 1; 876 } 877 continue; 878 } 879 880 if (hash_count) { 881 --hash_count; 882 hash = hash_base ^ 883 toeplitz_piecemeal_port(lport); 884 if (netisr_hashcpu(hash) != hash_cpu && 885 hash_count) 886 continue; 887 } 888 889 if (in_pcbporthash_update4(portinfo, 890 inp, lport, sin, cred)) { 891 error = 0; 892 break; 893 } 894 } 895 } 896 897 #undef IS_SELFCONNECT 898 899 if (error) { 900 /* Try next portinfo */ 901 portinfo_idx++; 902 portinfo_idx %= pcbinfo->portinfo_cnt; 903 if (portinfo_idx != portinfo_first) 904 goto loop; 905 inp->inp_laddr.s_addr = INADDR_ANY; 906 } 907 return error; 908 } 909 910 /* 911 * Transform old in_pcbconnect() into an inner subroutine for new 912 * in_pcbconnect(): Do some validity-checking on the remote 913 * address (in mbuf 'nam') and then determine local host address 914 * (i.e., which interface) to use to access that remote host. 915 * 916 * This preserves definition of in_pcbconnect(), while supporting a 917 * slightly different version for T/TCP. (This is more than 918 * a bit of a kludge, but cleaning up the internal interfaces would 919 * have forced minor changes in every protocol). 920 */ 921 int 922 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 923 struct sockaddr_in **plocal_sin, struct thread *td, int find) 924 { 925 struct in_ifaddr *ia; 926 struct ucred *cred = NULL; 927 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 928 struct sockaddr *jsin; 929 int jailed = 0, alloc_route = 0; 930 931 if (nam->sa_len != sizeof *sin) 932 return (EINVAL); 933 if (sin->sin_family != AF_INET) 934 return (EAFNOSUPPORT); 935 if (sin->sin_port == 0) 936 return (EADDRNOTAVAIL); 937 if (td && td->td_proc && td->td_proc->p_ucred) 938 cred = td->td_proc->p_ucred; 939 if (cred && cred->cr_prison) 940 jailed = 1; 941 if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) { 942 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 943 /* 944 * If the destination address is INADDR_ANY, 945 * use the primary local address. 946 * If the supplied address is INADDR_BROADCAST, 947 * and the primary interface supports broadcast, 948 * choose the broadcast address for that interface. 949 */ 950 if (sin->sin_addr.s_addr == INADDR_ANY) 951 sin->sin_addr = IA_SIN(ia)->sin_addr; 952 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST && 953 (ia->ia_ifp->if_flags & IFF_BROADCAST)) 954 sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr; 955 } 956 if (find) { 957 struct route *ro; 958 959 ia = NULL; 960 /* 961 * If route is known or can be allocated now, 962 * our src addr is taken from the i/f, else punt. 963 * Note that we should check the address family of the cached 964 * destination, in case of sharing the cache with IPv6. 965 */ 966 ro = &inp->inp_route; 967 if (ro->ro_rt && 968 (!(ro->ro_rt->rt_flags & RTF_UP) || 969 ro->ro_dst.sa_family != AF_INET || 970 satosin(&ro->ro_dst)->sin_addr.s_addr != 971 sin->sin_addr.s_addr || 972 inp->inp_socket->so_options & SO_DONTROUTE)) { 973 RTFREE(ro->ro_rt); 974 ro->ro_rt = NULL; 975 } 976 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 977 (ro->ro_rt == NULL || 978 ro->ro_rt->rt_ifp == NULL)) { 979 /* No route yet, so try to acquire one */ 980 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 981 ro->ro_dst.sa_family = AF_INET; 982 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 983 ((struct sockaddr_in *) &ro->ro_dst)->sin_addr = 984 sin->sin_addr; 985 rtalloc(ro); 986 alloc_route = 1; 987 } 988 /* 989 * If we found a route, use the address 990 * corresponding to the outgoing interface 991 * unless it is the loopback (in case a route 992 * to our address on another net goes to loopback). 993 */ 994 if (ro->ro_rt && 995 !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) { 996 if (jailed) { 997 if (jailed_ip(cred->cr_prison, 998 ro->ro_rt->rt_ifa->ifa_addr)) { 999 ia = ifatoia(ro->ro_rt->rt_ifa); 1000 } 1001 } else { 1002 ia = ifatoia(ro->ro_rt->rt_ifa); 1003 } 1004 } 1005 if (ia == NULL) { 1006 u_short fport = sin->sin_port; 1007 1008 sin->sin_port = 0; 1009 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 1010 if (ia && jailed && !jailed_ip(cred->cr_prison, 1011 sintosa(&ia->ia_addr))) 1012 ia = NULL; 1013 if (ia == NULL) 1014 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 1015 if (ia && jailed && !jailed_ip(cred->cr_prison, 1016 sintosa(&ia->ia_addr))) 1017 ia = NULL; 1018 sin->sin_port = fport; 1019 if (ia == NULL && 1020 !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 1021 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 1022 if (ia && jailed && !jailed_ip(cred->cr_prison, 1023 sintosa(&ia->ia_addr))) 1024 ia = NULL; 1025 1026 if (!jailed && ia == NULL) 1027 goto fail; 1028 } 1029 /* 1030 * If the destination address is multicast and an outgoing 1031 * interface has been set as a multicast option, use the 1032 * address of that interface as our source address. 1033 */ 1034 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 1035 inp->inp_moptions != NULL) { 1036 struct ip_moptions *imo; 1037 struct ifnet *ifp; 1038 1039 imo = inp->inp_moptions; 1040 if ((ifp = imo->imo_multicast_ifp) != NULL) { 1041 struct in_ifaddr_container *iac; 1042 1043 ia = NULL; 1044 TAILQ_FOREACH(iac, 1045 &in_ifaddrheads[mycpuid], ia_link) { 1046 if (iac->ia->ia_ifp == ifp) { 1047 ia = iac->ia; 1048 break; 1049 } 1050 } 1051 if (ia == NULL) 1052 goto fail; 1053 } 1054 } 1055 /* 1056 * Don't do pcblookup call here; return interface in plocal_sin 1057 * and exit to caller, that will do the lookup. 1058 */ 1059 if (ia == NULL && jailed) { 1060 if ((jsin = prison_get_nonlocal( 1061 cred->cr_prison, AF_INET, NULL)) != NULL || 1062 (jsin = prison_get_local( 1063 cred->cr_prison, AF_INET, NULL)) != NULL) { 1064 *plocal_sin = satosin(jsin); 1065 } else { 1066 /* IPv6 only Jail */ 1067 goto fail; 1068 } 1069 } else { 1070 *plocal_sin = &ia->ia_addr; 1071 } 1072 } 1073 return (0); 1074 fail: 1075 if (alloc_route) 1076 in_pcbresetroute(inp); 1077 return (EADDRNOTAVAIL); 1078 } 1079 1080 int 1081 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 1082 struct sockaddr_in **plocal_sin, struct thread *td) 1083 { 1084 return in_pcbladdr_find(inp, nam, plocal_sin, td, 1085 (inp->inp_laddr.s_addr == INADDR_ANY)); 1086 } 1087 1088 /* 1089 * Outer subroutine: 1090 * Connect from a socket to a specified address. 1091 * Both address and port must be specified in argument sin. 1092 * If don't have a local address for this socket yet, 1093 * then pick one. 1094 */ 1095 int 1096 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 1097 { 1098 struct sockaddr_in *if_sin; 1099 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1100 int error; 1101 1102 if_sin = NULL; /* avoid gcc warnings */ 1103 1104 /* Call inner routine to assign local interface address. */ 1105 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1106 return (error); 1107 1108 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1109 inp->inp_laddr.s_addr ? 1110 inp->inp_laddr : if_sin->sin_addr, 1111 inp->inp_lport, FALSE, NULL) != NULL) { 1112 return (EADDRINUSE); 1113 } 1114 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1115 if (inp->inp_lport == 0) { 1116 error = in_pcbbind(inp, NULL, td); 1117 if (error) 1118 return (error); 1119 } 1120 inp->inp_laddr = if_sin->sin_addr; 1121 } 1122 inp->inp_faddr = sin->sin_addr; 1123 inp->inp_fport = sin->sin_port; 1124 in_pcbinsconnhash(inp); 1125 return (0); 1126 } 1127 1128 void 1129 in_pcbdisconnect(struct inpcb *inp) 1130 { 1131 1132 in_pcbremconnhash(inp); 1133 inp->inp_faddr.s_addr = INADDR_ANY; 1134 inp->inp_fport = 0; 1135 } 1136 1137 void 1138 in_pcbdetach(struct inpcb *inp) 1139 { 1140 struct socket *so = inp->inp_socket; 1141 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1142 1143 #ifdef IPSEC 1144 ipsec4_delete_pcbpolicy(inp); 1145 #endif /*IPSEC*/ 1146 inp->inp_gencnt = ++ipi->ipi_gencnt; 1147 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1148 in_pcbremlists(inp); 1149 so->so_pcb = NULL; 1150 sofree(so); /* remove pcb ref */ 1151 if (inp->inp_options) 1152 m_free(inp->inp_options); 1153 if (inp->inp_route.ro_rt) 1154 rtfree(inp->inp_route.ro_rt); 1155 ip_freemoptions(inp->inp_moptions); 1156 kfree(inp, M_PCB); 1157 } 1158 1159 /* 1160 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1161 * socket received RST. 1162 */ 1163 static int 1164 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1165 { 1166 struct inpcb *inp; 1167 struct sockaddr_in *sin; 1168 1169 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1170 inp = so->so_pcb; 1171 if (!inp) 1172 return (ECONNRESET); 1173 1174 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1175 sin->sin_family = AF_INET; 1176 sin->sin_len = sizeof *sin; 1177 sin->sin_port = inp->inp_lport; 1178 sin->sin_addr = inp->inp_laddr; 1179 1180 *nam = (struct sockaddr *)sin; 1181 return (0); 1182 } 1183 1184 void 1185 in_setsockaddr_dispatch(netmsg_t msg) 1186 { 1187 int error; 1188 1189 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1190 lwkt_replymsg(&msg->lmsg, error); 1191 } 1192 1193 /* 1194 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1195 * socket received RST. 1196 */ 1197 int 1198 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1199 { 1200 struct inpcb *inp; 1201 struct sockaddr_in *sin; 1202 1203 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1204 inp = so->so_pcb; 1205 if (!inp) 1206 return (ECONNRESET); 1207 1208 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1209 sin->sin_family = AF_INET; 1210 sin->sin_len = sizeof *sin; 1211 sin->sin_port = inp->inp_fport; 1212 sin->sin_addr = inp->inp_faddr; 1213 1214 *nam = (struct sockaddr *)sin; 1215 return (0); 1216 } 1217 1218 void 1219 in_setpeeraddr_dispatch(netmsg_t msg) 1220 { 1221 int error; 1222 1223 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1224 lwkt_replymsg(&msg->lmsg, error); 1225 } 1226 1227 void 1228 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1229 inp_notify_t notify) 1230 { 1231 struct inpcb *inp, *marker; 1232 1233 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1234 ("not in the correct netisr")); 1235 marker = &in_pcbmarkers[mycpuid]; 1236 1237 /* 1238 * NOTE: 1239 * - If INP_PLACEMARKER is set we must ignore the rest of the 1240 * structure and skip it. 1241 * - It is safe to nuke inpcbs here, since we are in their own 1242 * netisr. 1243 */ 1244 GET_PCBINFO_TOKEN(pcbinfo); 1245 1246 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1247 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1248 LIST_REMOVE(marker, inp_list); 1249 LIST_INSERT_AFTER(inp, marker, inp_list); 1250 1251 if (inp->inp_flags & INP_PLACEMARKER) 1252 continue; 1253 #ifdef INET6 1254 if (!INP_ISIPV4(inp)) 1255 continue; 1256 #endif 1257 if (inp->inp_faddr.s_addr != faddr.s_addr || 1258 inp->inp_socket == NULL) 1259 continue; 1260 (*notify)(inp, err); /* can remove inp from list! */ 1261 } 1262 LIST_REMOVE(marker, inp_list); 1263 1264 REL_PCBINFO_TOKEN(pcbinfo); 1265 } 1266 1267 void 1268 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1269 { 1270 struct inpcb *inp, *marker; 1271 1272 /* 1273 * We only need to make sure that we are in netisr0, where all 1274 * multicast operation happen. We could check inpcbinfo which 1275 * does not belong to netisr0 by holding the inpcbinfo's token. 1276 * In this case, the pcbinfo must be able to be shared, i.e. 1277 * pcbinfo->infotoken is not NULL. 1278 */ 1279 ASSERT_IN_NETISR(0); 1280 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1281 ("pcbinfo could not be shared")); 1282 1283 /* 1284 * Get a marker for the current netisr (netisr0). 1285 * 1286 * It is possible that the multicast address deletion blocks, 1287 * which could cause temporary token releasing. So we use 1288 * inpcb marker here to get a coherent view of the inpcb list. 1289 * 1290 * While, on the other hand, moptions are only added and deleted 1291 * in netisr0, so we would not see staled moption or miss moption 1292 * even if the token was released due to the blocking multicast 1293 * address deletion. 1294 */ 1295 marker = &in_pcbmarkers[mycpuid]; 1296 1297 GET_PCBINFO_TOKEN(pcbinfo); 1298 1299 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1300 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1301 struct ip_moptions *imo; 1302 1303 LIST_REMOVE(marker, inp_list); 1304 LIST_INSERT_AFTER(inp, marker, inp_list); 1305 1306 if (inp->inp_flags & INP_PLACEMARKER) 1307 continue; 1308 imo = inp->inp_moptions; 1309 if (INP_ISIPV4(inp) && imo != NULL) { 1310 int i, gap; 1311 1312 /* 1313 * Unselect the outgoing interface if it is being 1314 * detached. 1315 */ 1316 if (imo->imo_multicast_ifp == ifp) 1317 imo->imo_multicast_ifp = NULL; 1318 1319 /* 1320 * Drop multicast group membership if we joined 1321 * through the interface being detached. 1322 */ 1323 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1324 i++) { 1325 if (imo->imo_membership[i]->inm_ifp == ifp) { 1326 /* 1327 * NOTE: 1328 * This could block and the pcbinfo 1329 * token could be passively released. 1330 */ 1331 in_delmulti(imo->imo_membership[i]); 1332 gap++; 1333 } else if (gap != 0) 1334 imo->imo_membership[i - gap] = 1335 imo->imo_membership[i]; 1336 } 1337 imo->imo_num_memberships -= gap; 1338 } 1339 } 1340 LIST_REMOVE(marker, inp_list); 1341 1342 REL_PCBINFO_TOKEN(pcbinfo); 1343 } 1344 1345 /* 1346 * Check for alternatives when higher level complains 1347 * about service problems. For now, invalidate cached 1348 * routing information. If the route was created dynamically 1349 * (by a redirect), time to try a default gateway again. 1350 */ 1351 void 1352 in_losing(struct inpcb *inp) 1353 { 1354 struct rtentry *rt; 1355 struct rt_addrinfo rtinfo; 1356 1357 if ((rt = inp->inp_route.ro_rt)) { 1358 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1359 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1360 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1361 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1362 rtinfo.rti_flags = rt->rt_flags; 1363 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1364 if (rt->rt_flags & RTF_DYNAMIC) { 1365 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1366 rt_mask(rt), rt->rt_flags, NULL); 1367 } 1368 inp->inp_route.ro_rt = NULL; 1369 rtfree(rt); 1370 /* 1371 * A new route can be allocated 1372 * the next time output is attempted. 1373 */ 1374 } 1375 } 1376 1377 /* 1378 * After a routing change, flush old routing 1379 * and allocate a (hopefully) better one. 1380 */ 1381 void 1382 in_rtchange(struct inpcb *inp, int err) 1383 { 1384 if (inp->inp_route.ro_rt) { 1385 rtfree(inp->inp_route.ro_rt); 1386 inp->inp_route.ro_rt = NULL; 1387 /* 1388 * A new route can be allocated the next time 1389 * output is attempted. 1390 */ 1391 } 1392 } 1393 1394 /* 1395 * Lookup a PCB based on the local address and port. 1396 */ 1397 static struct inpcb * 1398 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr, 1399 u_int lport_arg, int wild_okay, struct ucred *cred) 1400 { 1401 struct inpcb *inp; 1402 int matchwild = 3, wildcard; 1403 u_short lport = lport_arg; 1404 struct inpcbport *phd; 1405 struct inpcb *match = NULL; 1406 1407 /* 1408 * If the porthashbase is shared across several cpus, it must 1409 * have been locked. 1410 */ 1411 ASSERT_PORTHASH_TOKEN_HELD(porthash); 1412 1413 /* 1414 * Best fit PCB lookup. 1415 * 1416 * First see if this local port is in use by looking on the 1417 * port hash list. 1418 */ 1419 LIST_FOREACH(phd, porthash, phd_hash) { 1420 if (phd->phd_port == lport) 1421 break; 1422 } 1423 if (phd != NULL) { 1424 /* 1425 * Port is in use by one or more PCBs. Look for best 1426 * fit. 1427 */ 1428 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1429 wildcard = 0; 1430 #ifdef INET6 1431 if (!INP_ISIPV4(inp)) 1432 continue; 1433 #endif 1434 if (inp->inp_faddr.s_addr != INADDR_ANY) 1435 wildcard++; 1436 if (inp->inp_laddr.s_addr != INADDR_ANY) { 1437 if (laddr.s_addr == INADDR_ANY) 1438 wildcard++; 1439 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1440 continue; 1441 } else { 1442 if (laddr.s_addr != INADDR_ANY) 1443 wildcard++; 1444 } 1445 if (wildcard && !wild_okay) 1446 continue; 1447 if (wildcard < matchwild && 1448 (cred == NULL || 1449 cred->cr_prison == 1450 inp->inp_socket->so_cred->cr_prison)) { 1451 match = inp; 1452 matchwild = wildcard; 1453 if (matchwild == 0) { 1454 break; 1455 } 1456 } 1457 } 1458 } 1459 return (match); 1460 } 1461 1462 struct inpcb * 1463 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1464 const struct inpcb *inp) 1465 { 1466 const struct inp_localgrphead *hdr; 1467 const struct inp_localgroup *grp; 1468 int i; 1469 1470 if (pcbinfo->localgrphashbase == NULL) 1471 return NULL; 1472 1473 GET_PCBINFO_TOKEN(pcbinfo); 1474 1475 hdr = &pcbinfo->localgrphashbase[ 1476 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1477 1478 LIST_FOREACH(grp, hdr, il_list) { 1479 if (grp->il_af == inp->inp_af && 1480 grp->il_lport == inp->inp_lport && 1481 memcmp(&grp->il_dependladdr, 1482 &inp->inp_inc.inc_ie.ie_dependladdr, 1483 sizeof(grp->il_dependladdr)) == 0) { 1484 break; 1485 } 1486 } 1487 if (grp == NULL || grp->il_inpcnt == 1) { 1488 REL_PCBINFO_TOKEN(pcbinfo); 1489 return NULL; 1490 } 1491 1492 KASSERT(grp->il_inpcnt >= 2, 1493 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1494 for (i = 0; i < grp->il_inpcnt; ++i) { 1495 if (grp->il_inp[i] == inp) { 1496 int last = grp->il_inpcnt - 1; 1497 1498 if (i == last) 1499 last = grp->il_inpcnt - 2; 1500 REL_PCBINFO_TOKEN(pcbinfo); 1501 return grp->il_inp[last]; 1502 } 1503 } 1504 REL_PCBINFO_TOKEN(pcbinfo); 1505 return NULL; 1506 } 1507 1508 static struct inpcb * 1509 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1510 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1511 { 1512 struct inpcb *local_wild = NULL; 1513 const struct inp_localgrphead *hdr; 1514 const struct inp_localgroup *grp; 1515 1516 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1517 1518 hdr = &pcbinfo->localgrphashbase[ 1519 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1520 1521 /* 1522 * Order of socket selection: 1523 * 1. non-wild. 1524 * 2. wild. 1525 * 1526 * NOTE: Local group does not contain jailed sockets 1527 */ 1528 LIST_FOREACH(grp, hdr, il_list) { 1529 #ifdef INET6 1530 if (grp->il_af != AF_INET) 1531 continue; 1532 #endif 1533 if (grp->il_lport == lport) { 1534 int idx; 1535 1536 /* 1537 * Modulo-N is used here, which greatly reduces 1538 * completion queue token contention, thus more 1539 * cpu time is saved. 1540 */ 1541 idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt; 1542 if (grp->il_laddr.s_addr == laddr.s_addr) 1543 return grp->il_inp[idx]; 1544 else if (grp->il_laddr.s_addr == INADDR_ANY) 1545 local_wild = grp->il_inp[idx]; 1546 } 1547 } 1548 if (local_wild != NULL) 1549 return local_wild; 1550 return NULL; 1551 } 1552 1553 /* 1554 * Lookup PCB in hash list. 1555 */ 1556 struct inpcb * 1557 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1558 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1559 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1560 { 1561 struct inpcbhead *head; 1562 struct inpcb *inp, *jinp=NULL; 1563 u_short fport = fport_arg, lport = lport_arg; 1564 1565 /* 1566 * First look for an exact match. 1567 */ 1568 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1569 laddr.s_addr, lport, pcbinfo->hashmask)]; 1570 LIST_FOREACH(inp, head, inp_hash) { 1571 #ifdef INET6 1572 if (!INP_ISIPV4(inp)) 1573 continue; 1574 #endif 1575 if (in_hosteq(inp->inp_faddr, faddr) && 1576 in_hosteq(inp->inp_laddr, laddr) && 1577 inp->inp_fport == fport && inp->inp_lport == lport) { 1578 /* found */ 1579 if (inp->inp_socket == NULL || 1580 inp->inp_socket->so_cred->cr_prison == NULL) { 1581 return (inp); 1582 } else { 1583 if (jinp == NULL) 1584 jinp = inp; 1585 } 1586 } 1587 } 1588 if (jinp != NULL) 1589 return (jinp); 1590 1591 if (wildcard) { 1592 struct inpcb *local_wild = NULL; 1593 struct inpcb *jinp_wild = NULL; 1594 struct inpcontainer *ic; 1595 struct inpcontainerhead *chead; 1596 struct sockaddr_in jsin; 1597 struct ucred *cred; 1598 1599 GET_PCBINFO_TOKEN(pcbinfo); 1600 1601 /* 1602 * Check local group first 1603 */ 1604 if (pcbinfo->localgrphashbase != NULL && 1605 m != NULL && (m->m_flags & M_HASH) && 1606 !(ifp && ifp->if_type == IFT_FAITH)) { 1607 inp = inp_localgroup_lookup(pcbinfo, 1608 laddr, lport, m->m_pkthdr.hash); 1609 if (inp != NULL) { 1610 REL_PCBINFO_TOKEN(pcbinfo); 1611 return inp; 1612 } 1613 } 1614 1615 /* 1616 * Order of socket selection: 1617 * 1. non-jailed, non-wild. 1618 * 2. non-jailed, wild. 1619 * 3. jailed, non-wild. 1620 * 4. jailed, wild. 1621 */ 1622 jsin.sin_family = AF_INET; 1623 chead = &pcbinfo->wildcardhashbase[ 1624 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1625 LIST_FOREACH(ic, chead, ic_list) { 1626 inp = ic->ic_inp; 1627 if (inp->inp_flags & INP_PLACEMARKER) 1628 continue; 1629 1630 jsin.sin_addr.s_addr = laddr.s_addr; 1631 #ifdef INET6 1632 if (!INP_ISIPV4(inp)) 1633 continue; 1634 #endif 1635 if (inp->inp_socket != NULL) 1636 cred = inp->inp_socket->so_cred; 1637 else 1638 cred = NULL; 1639 if (cred != NULL && jailed(cred)) { 1640 if (jinp != NULL) 1641 continue; 1642 else 1643 if (!jailed_ip(cred->cr_prison, 1644 (struct sockaddr *)&jsin)) 1645 continue; 1646 } 1647 if (inp->inp_lport == lport) { 1648 if (ifp && ifp->if_type == IFT_FAITH && 1649 !(inp->inp_flags & INP_FAITH)) 1650 continue; 1651 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1652 if (cred != NULL && jailed(cred)) { 1653 jinp = inp; 1654 } else { 1655 REL_PCBINFO_TOKEN(pcbinfo); 1656 return (inp); 1657 } 1658 } 1659 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1660 if (cred != NULL && jailed(cred)) 1661 jinp_wild = inp; 1662 else 1663 local_wild = inp; 1664 } 1665 } 1666 } 1667 1668 REL_PCBINFO_TOKEN(pcbinfo); 1669 1670 if (local_wild != NULL) 1671 return (local_wild); 1672 if (jinp != NULL) 1673 return (jinp); 1674 return (jinp_wild); 1675 } 1676 1677 /* 1678 * Not found. 1679 */ 1680 return (NULL); 1681 } 1682 1683 struct inpcb * 1684 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1685 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1686 boolean_t wildcard, struct ifnet *ifp) 1687 { 1688 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1689 laddr, lport_arg, wildcard, ifp, NULL); 1690 } 1691 1692 /* 1693 * Insert PCB into connection hash table. 1694 */ 1695 void 1696 in_pcbinsconnhash(struct inpcb *inp) 1697 { 1698 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1699 struct inpcbhead *bucket; 1700 u_int32_t hashkey_faddr, hashkey_laddr; 1701 1702 #ifdef INET6 1703 if (INP_ISIPV6(inp)) { 1704 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1705 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1706 } else { 1707 #endif 1708 hashkey_faddr = inp->inp_faddr.s_addr; 1709 hashkey_laddr = inp->inp_laddr.s_addr; 1710 #ifdef INET6 1711 } 1712 #endif 1713 1714 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1715 ("not in the correct netisr")); 1716 ASSERT_INP_NOTINHASH(inp); 1717 inp->inp_flags |= INP_CONNECTED; 1718 1719 /* 1720 * Insert into the connection hash table. 1721 */ 1722 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1723 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1724 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1725 } 1726 1727 /* 1728 * Remove PCB from connection hash table. 1729 */ 1730 void 1731 in_pcbremconnhash(struct inpcb *inp) 1732 { 1733 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1734 1735 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1736 ("not in the correct netisr")); 1737 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1738 1739 LIST_REMOVE(inp, inp_hash); 1740 inp->inp_flags &= ~INP_CONNECTED; 1741 } 1742 1743 /* 1744 * Insert PCB into port hash table. 1745 */ 1746 void 1747 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp) 1748 { 1749 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1750 struct inpcbport *phd; 1751 1752 /* 1753 * If the porthashbase is shared across several cpus, it must 1754 * have been locked. 1755 */ 1756 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash); 1757 1758 /* 1759 * Insert into the port hash table. 1760 */ 1761 1762 /* Go through port list and look for a head for this lport. */ 1763 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1764 if (phd->phd_port == inp->inp_lport) 1765 break; 1766 } 1767 1768 /* If none exists, use saved one and tack it on. */ 1769 if (phd == NULL) { 1770 KKASSERT(pcbinfo->portsave != NULL); 1771 phd = pcbinfo->portsave; 1772 pcbinfo->portsave = NULL; 1773 phd->phd_port = inp->inp_lport; 1774 LIST_INIT(&phd->phd_pcblist); 1775 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1776 } 1777 1778 inp->inp_porthash = pcbporthash; 1779 inp->inp_phd = phd; 1780 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1781 1782 /* 1783 * Malloc one inpcbport for later use. It is safe to use 1784 * "wait" malloc here (port token would be released, if 1785 * malloc ever blocked), since all changes to the porthash 1786 * are done. 1787 */ 1788 if (pcbinfo->portsave == NULL) { 1789 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1790 M_PCB, M_INTWAIT | M_ZERO); 1791 } 1792 } 1793 1794 void 1795 in_pcbinsporthash_lport(struct inpcb *inp) 1796 { 1797 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1798 struct inpcbportinfo *portinfo; 1799 struct inpcbporthead *porthash; 1800 u_short lport_ho; 1801 1802 /* Locate the proper portinfo based on lport */ 1803 lport_ho = ntohs(inp->inp_lport); 1804 portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 1805 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset); 1806 1807 porthash = in_pcbporthash_head(portinfo, inp->inp_lport); 1808 GET_PORTHASH_TOKEN(porthash); 1809 in_pcbinsporthash(porthash, inp); 1810 REL_PORTHASH_TOKEN(porthash); 1811 } 1812 1813 void 1814 in_pcbremporthash(struct inpcb *inp) 1815 { 1816 struct inpcbporthead *porthash; 1817 struct inpcbport *phd; 1818 1819 if (inp->inp_phd == NULL) 1820 return; 1821 KASSERT(inp->inp_lport != 0, ("inpcb has no lport")); 1822 1823 porthash = inp->inp_porthash; 1824 KASSERT(porthash != NULL, ("no porthash")); 1825 1826 GET_PORTHASH_TOKEN(porthash); 1827 1828 phd = inp->inp_phd; 1829 LIST_REMOVE(inp, inp_portlist); 1830 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 1831 LIST_REMOVE(phd, phd_hash); 1832 kfree(phd, M_PCB); 1833 } 1834 1835 REL_PORTHASH_TOKEN(porthash); 1836 1837 inp->inp_phd = NULL; 1838 /* NOTE: Don't whack inp_lport, which may be used later */ 1839 } 1840 1841 static struct inp_localgroup * 1842 inp_localgroup_alloc(u_char af, uint16_t port, 1843 const union in_dependaddr *addr, int size) 1844 { 1845 struct inp_localgroup *grp; 1846 1847 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 1848 M_TEMP, M_INTWAIT | M_ZERO); 1849 grp->il_af = af; 1850 grp->il_lport = port; 1851 grp->il_dependladdr = *addr; 1852 grp->il_inpsiz = size; 1853 1854 return grp; 1855 } 1856 1857 static void 1858 inp_localgroup_free(struct inp_localgroup *grp) 1859 { 1860 kfree(grp, M_TEMP); 1861 } 1862 1863 static void 1864 inp_localgroup_destroy(struct inp_localgroup *grp) 1865 { 1866 LIST_REMOVE(grp, il_list); 1867 inp_localgroup_free(grp); 1868 } 1869 1870 static void 1871 inp_localgroup_copy(struct inp_localgroup *grp, 1872 const struct inp_localgroup *old_grp) 1873 { 1874 int i; 1875 1876 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 1877 ("invalid new local group size %d and old local group count %d", 1878 grp->il_inpsiz, old_grp->il_inpcnt)); 1879 for (i = 0; i < old_grp->il_inpcnt; ++i) 1880 grp->il_inp[i] = old_grp->il_inp[i]; 1881 grp->il_inpcnt = old_grp->il_inpcnt; 1882 } 1883 1884 static void 1885 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1886 { 1887 struct inp_localgrphead *hdr; 1888 struct inp_localgroup *grp, *grp_alloc = NULL; 1889 struct ucred *cred; 1890 int i, idx; 1891 1892 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1893 1894 if (pcbinfo->localgrphashbase == NULL) 1895 return; 1896 1897 /* 1898 * XXX don't allow jailed socket to join local group 1899 */ 1900 if (inp->inp_socket != NULL) 1901 cred = inp->inp_socket->so_cred; 1902 else 1903 cred = NULL; 1904 if (cred != NULL && jailed(cred)) 1905 return; 1906 1907 hdr = &pcbinfo->localgrphashbase[ 1908 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1909 1910 again: 1911 LIST_FOREACH(grp, hdr, il_list) { 1912 if (grp->il_af == inp->inp_af && 1913 grp->il_lport == inp->inp_lport && 1914 memcmp(&grp->il_dependladdr, 1915 &inp->inp_inc.inc_ie.ie_dependladdr, 1916 sizeof(grp->il_dependladdr)) == 0) { 1917 break; 1918 } 1919 } 1920 if (grp == NULL) { 1921 /* 1922 * Create a new local group 1923 */ 1924 if (grp_alloc == NULL) { 1925 grp_alloc = inp_localgroup_alloc(inp->inp_af, 1926 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 1927 INP_LOCALGROUP_SIZMIN); 1928 /* 1929 * Local group allocation could block and the 1930 * local group w/ the same property might have 1931 * been added by others when we were blocked; 1932 * check again. 1933 */ 1934 goto again; 1935 } else { 1936 /* Local group has been allocated; link it */ 1937 grp = grp_alloc; 1938 grp_alloc = NULL; 1939 LIST_INSERT_HEAD(hdr, grp, il_list); 1940 } 1941 } else if (grp->il_inpcnt == grp->il_inpsiz) { 1942 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 1943 static int limit_logged = 0; 1944 1945 if (!limit_logged) { 1946 limit_logged = 1; 1947 kprintf("local group port %d, " 1948 "limit reached\n", ntohs(grp->il_lport)); 1949 } 1950 if (grp_alloc != NULL) { 1951 /* 1952 * This would happen if the local group 1953 * w/ the same property was expanded when 1954 * our local group allocation blocked. 1955 */ 1956 inp_localgroup_free(grp_alloc); 1957 } 1958 return; 1959 } 1960 1961 /* 1962 * Expand this local group 1963 */ 1964 if (grp_alloc == NULL || 1965 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 1966 if (grp_alloc != NULL) 1967 inp_localgroup_free(grp_alloc); 1968 grp_alloc = inp_localgroup_alloc(grp->il_af, 1969 grp->il_lport, &grp->il_dependladdr, 1970 grp->il_inpsiz * 2); 1971 /* 1972 * Local group allocation could block and the 1973 * local group w/ the same property might have 1974 * been expanded by others when we were blocked; 1975 * check again. 1976 */ 1977 goto again; 1978 } 1979 1980 /* 1981 * Save the old local group, link the new one, and then 1982 * destroy the old local group 1983 */ 1984 inp_localgroup_copy(grp_alloc, grp); 1985 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 1986 inp_localgroup_destroy(grp); 1987 1988 grp = grp_alloc; 1989 grp_alloc = NULL; 1990 } else { 1991 /* 1992 * Found the local group 1993 */ 1994 if (grp_alloc != NULL) { 1995 /* 1996 * This would happen if the local group w/ the 1997 * same property was added or expanded when our 1998 * local group allocation blocked. 1999 */ 2000 inp_localgroup_free(grp_alloc); 2001 grp_alloc = NULL; 2002 } 2003 } 2004 2005 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 2006 ("invalid local group size %d and count %d", 2007 grp->il_inpsiz, grp->il_inpcnt)); 2008 2009 /* 2010 * Keep the local group sorted by the inpcb local group index 2011 * in ascending order. 2012 * 2013 * This eases the multi-process userland application which uses 2014 * SO_REUSEPORT sockets and binds process to the owner cpu of 2015 * the SO_REUSEPORT socket: 2016 * If we didn't sort the local group by the inpcb local group 2017 * index and one of the process owning an inpcb in this local 2018 * group restarted, e.g. crashed and restarted by watchdog, 2019 * other processes owning a inpcb in this local group would have 2020 * to detect that event, refetch its socket's owner cpu, and 2021 * re-bind. 2022 */ 2023 idx = grp->il_inpcnt; 2024 for (i = 0; i < idx; ++i) { 2025 struct inpcb *oinp = grp->il_inp[i]; 2026 2027 if (oinp->inp_lgrpindex > i) { 2028 if (inp->inp_lgrpindex < 0) { 2029 inp->inp_lgrpindex = i; 2030 } else if (inp->inp_lgrpindex != i) { 2031 if (bootverbose) { 2032 kprintf("inp %p: grpidx %d, " 2033 "assigned to %d, cpu%d\n", 2034 inp, inp->inp_lgrpindex, i, 2035 mycpuid); 2036 } 2037 } 2038 grp->il_inp[i] = inp; 2039 2040 /* Pull down inpcbs */ 2041 for (; i < grp->il_inpcnt; ++i) { 2042 struct inpcb *oinp1 = grp->il_inp[i + 1]; 2043 2044 grp->il_inp[i + 1] = oinp; 2045 oinp = oinp1; 2046 } 2047 grp->il_inpcnt++; 2048 return; 2049 } 2050 } 2051 2052 if (inp->inp_lgrpindex < 0) { 2053 inp->inp_lgrpindex = idx; 2054 } else if (inp->inp_lgrpindex != idx) { 2055 if (bootverbose) { 2056 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 2057 inp, inp->inp_lgrpindex, idx, mycpuid); 2058 } 2059 } 2060 grp->il_inp[idx] = inp; 2061 grp->il_inpcnt++; 2062 } 2063 2064 void 2065 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2066 { 2067 struct inpcontainer *ic; 2068 struct inpcontainerhead *bucket; 2069 2070 GET_PCBINFO_TOKEN(pcbinfo); 2071 2072 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 2073 2074 bucket = &pcbinfo->wildcardhashbase[ 2075 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2076 2077 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 2078 ic->ic_inp = inp; 2079 LIST_INSERT_HEAD(bucket, ic, ic_list); 2080 2081 REL_PCBINFO_TOKEN(pcbinfo); 2082 } 2083 2084 /* 2085 * Insert PCB into wildcard hash table. 2086 */ 2087 void 2088 in_pcbinswildcardhash(struct inpcb *inp) 2089 { 2090 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2091 2092 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2093 ("not in correct netisr")); 2094 ASSERT_INP_NOTINHASH(inp); 2095 inp->inp_flags |= INP_WILDCARD; 2096 2097 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 2098 } 2099 2100 static void 2101 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2102 { 2103 struct inp_localgrphead *hdr; 2104 struct inp_localgroup *grp; 2105 2106 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2107 2108 if (pcbinfo->localgrphashbase == NULL) 2109 return; 2110 2111 hdr = &pcbinfo->localgrphashbase[ 2112 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2113 2114 LIST_FOREACH(grp, hdr, il_list) { 2115 int i; 2116 2117 for (i = 0; i < grp->il_inpcnt; ++i) { 2118 if (grp->il_inp[i] != inp) 2119 continue; 2120 2121 if (grp->il_inpcnt == 1) { 2122 /* Destroy this local group */ 2123 inp_localgroup_destroy(grp); 2124 } else { 2125 /* Pull up inpcbs */ 2126 for (; i + 1 < grp->il_inpcnt; ++i) 2127 grp->il_inp[i] = grp->il_inp[i + 1]; 2128 grp->il_inpcnt--; 2129 } 2130 return; 2131 } 2132 } 2133 } 2134 2135 void 2136 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2137 { 2138 struct inpcontainer *ic; 2139 struct inpcontainerhead *head; 2140 2141 GET_PCBINFO_TOKEN(pcbinfo); 2142 2143 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2144 2145 /* find bucket */ 2146 head = &pcbinfo->wildcardhashbase[ 2147 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2148 2149 LIST_FOREACH(ic, head, ic_list) { 2150 if (ic->ic_inp == inp) 2151 goto found; 2152 } 2153 REL_PCBINFO_TOKEN(pcbinfo); 2154 return; /* not found! */ 2155 2156 found: 2157 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2158 REL_PCBINFO_TOKEN(pcbinfo); 2159 kfree(ic, M_TEMP); /* deallocate container */ 2160 } 2161 2162 /* 2163 * Remove PCB from wildcard hash table. 2164 */ 2165 void 2166 in_pcbremwildcardhash(struct inpcb *inp) 2167 { 2168 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2169 2170 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2171 ("not in correct netisr")); 2172 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2173 2174 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2175 inp->inp_lgrpindex = -1; 2176 inp->inp_flags &= ~INP_WILDCARD; 2177 } 2178 2179 /* 2180 * Remove PCB from various lists. 2181 */ 2182 void 2183 in_pcbremlists(struct inpcb *inp) 2184 { 2185 in_pcbremporthash(inp); 2186 if (inp->inp_flags & INP_WILDCARD) { 2187 in_pcbremwildcardhash(inp); 2188 } else if (inp->inp_flags & INP_CONNECTED) { 2189 in_pcbremconnhash(inp); 2190 } 2191 2192 if (inp->inp_flags & INP_ONLIST) 2193 in_pcbofflist(inp); 2194 } 2195 2196 int 2197 prison_xinpcb(struct thread *td, struct inpcb *inp) 2198 { 2199 struct ucred *cr; 2200 2201 if (td->td_proc == NULL) 2202 return (0); 2203 cr = td->td_proc->p_ucred; 2204 if (cr->cr_prison == NULL) 2205 return (0); 2206 if (inp->inp_socket && inp->inp_socket->so_cred && 2207 inp->inp_socket->so_cred->cr_prison && 2208 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2209 return (0); 2210 return (1); 2211 } 2212 2213 int 2214 in_pcblist_range(SYSCTL_HANDLER_ARGS) 2215 { 2216 struct inpcbinfo *pcbinfo_arr = arg1; 2217 int pcbinfo_arrlen = arg2; 2218 struct inpcb *marker; 2219 int cpu, origcpu; 2220 int error, n; 2221 2222 KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1, 2223 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2224 2225 /* 2226 * The process of preparing the TCB list is too time-consuming and 2227 * resource-intensive to repeat twice on every request. 2228 */ 2229 n = 0; 2230 if (req->oldptr == NULL) { 2231 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2232 n += pcbinfo_arr[cpu].ipi_count; 2233 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2234 return 0; 2235 } 2236 2237 if (req->newptr != NULL) 2238 return EPERM; 2239 2240 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2241 marker->inp_flags |= INP_PLACEMARKER; 2242 2243 /* 2244 * OK, now we're committed to doing something. Re-fetch ipi_count 2245 * after obtaining the generation count. 2246 */ 2247 error = 0; 2248 origcpu = mycpuid; 2249 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2250 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2251 struct inpcb *inp; 2252 struct xinpcb xi; 2253 int i; 2254 2255 lwkt_migratecpu(cpu); 2256 2257 GET_PCBINFO_TOKEN(pcbinfo); 2258 2259 n = pcbinfo->ipi_count; 2260 2261 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2262 i = 0; 2263 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2264 LIST_REMOVE(marker, inp_list); 2265 LIST_INSERT_AFTER(inp, marker, inp_list); 2266 2267 if (inp->inp_flags & INP_PLACEMARKER) 2268 continue; 2269 if (prison_xinpcb(req->td, inp)) 2270 continue; 2271 2272 bzero(&xi, sizeof xi); 2273 xi.xi_len = sizeof xi; 2274 bcopy(inp, &xi.xi_inp, sizeof *inp); 2275 if (inp->inp_socket) 2276 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2277 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2278 break; 2279 ++i; 2280 } 2281 LIST_REMOVE(marker, inp_list); 2282 2283 REL_PCBINFO_TOKEN(pcbinfo); 2284 2285 if (error == 0 && i < n) { 2286 bzero(&xi, sizeof xi); 2287 xi.xi_len = sizeof xi; 2288 while (i < n) { 2289 error = SYSCTL_OUT(req, &xi, sizeof xi); 2290 if (error) 2291 break; 2292 ++i; 2293 } 2294 } 2295 } 2296 2297 lwkt_migratecpu(origcpu); 2298 kfree(marker, M_TEMP); 2299 return error; 2300 } 2301 2302 int 2303 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS) 2304 { 2305 2306 return (in_pcblist_range(oidp, arg1, netisr_ncpus, req)); 2307 } 2308 2309 void 2310 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2311 { 2312 struct sockaddr_in *sin; 2313 2314 KASSERT(faddr->sa_family == AF_INET, 2315 ("not AF_INET faddr %d", faddr->sa_family)); 2316 2317 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2318 sin->sin_family = AF_INET; 2319 sin->sin_len = sizeof(*sin); 2320 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2321 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2322 2323 so->so_faddr = (struct sockaddr *)sin; 2324 } 2325 2326 void 2327 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2328 u_short offset) 2329 { 2330 memset(portinfo, 0, sizeof(*portinfo)); 2331 2332 portinfo->offset = offset; 2333 portinfo->lastport = offset; 2334 portinfo->lastlow = offset; 2335 portinfo->lasthi = offset; 2336 2337 portinfo->porthashbase = hashinit(hashsize, M_PCB, 2338 &portinfo->porthashmask); 2339 } 2340 2341 void 2342 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2343 { 2344 int hi, lo; 2345 2346 if (step == 1) 2347 return; 2348 2349 hi = *hi0; 2350 lo = *lo0; 2351 2352 hi = rounddown(hi, step); 2353 hi += ofs; 2354 if (hi > (int)*hi0) 2355 hi -= step; 2356 2357 lo = roundup(lo, step); 2358 lo -= (step - ofs); 2359 if (lo < (int)*lo0) 2360 lo += step; 2361 2362 *hi0 = hi; 2363 *lo0 = lo; 2364 } 2365 2366 void 2367 in_pcbglobalinit(void) 2368 { 2369 int cpu; 2370 2371 in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB, 2372 M_WAITOK | M_ZERO); 2373 in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer), 2374 M_PCB, M_WAITOK | M_ZERO); 2375 2376 for (cpu = 0; cpu < ncpus; ++cpu) { 2377 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2378 struct inpcb *marker = &in_pcbmarkers[cpu]; 2379 2380 marker->inp_flags |= INP_PLACEMARKER; 2381 ic->ic_inp = marker; 2382 } 2383 } 2384 2385 struct inpcb * 2386 in_pcbmarker(int cpuid) 2387 { 2388 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid)); 2389 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 2390 2391 return &in_pcbmarkers[cpuid]; 2392 } 2393 2394 struct inpcontainer * 2395 in_pcbcontainer_marker(int cpuid) 2396 { 2397 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid)); 2398 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 2399 2400 return &in_pcbcontainer_markers[cpuid]; 2401 } 2402 2403 void 2404 in_pcbresetroute(struct inpcb *inp) 2405 { 2406 struct route *ro = &inp->inp_route; 2407 2408 if (ro->ro_rt != NULL) 2409 RTFREE(ro->ro_rt); 2410 bzero(ro, sizeof(*ro)); 2411 } 2412 2413 u_short 2414 in_pcblastport_down(volatile u_short *lastport, u_short first, u_short last, 2415 u_short step) 2416 { 2417 u_short lport; 2418 2419 for (;;) { 2420 u_short olport; 2421 2422 olport = *lastport; 2423 lport = olport - step; 2424 if (__predict_false(lport > first || lport < last)) 2425 lport = first; 2426 if (atomic_cmpset_short(lastport, olport, lport)) 2427 break; 2428 } 2429 return lport; 2430 } 2431 2432 u_short 2433 in_pcblastport_up(volatile u_short *lastport, u_short first, u_short last, 2434 u_short step) 2435 { 2436 u_short lport; 2437 2438 for (;;) { 2439 u_short olport; 2440 2441 olport = *lastport; 2442 lport = olport + step; 2443 if (__predict_false(lport < first || lport > last)) 2444 lport = first; 2445 if (atomic_cmpset_short(lastport, olport, lport)) 2446 break; 2447 } 2448 return lport; 2449 } 2450