1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/domain.h> 74 #include <sys/protosw.h> 75 #include <sys/socket.h> 76 #include <sys/socketvar.h> 77 #include <sys/proc.h> 78 #include <sys/priv.h> 79 #include <sys/jail.h> 80 #include <sys/kernel.h> 81 #include <sys/sysctl.h> 82 83 #include <sys/thread2.h> 84 #include <sys/socketvar2.h> 85 #include <sys/msgport2.h> 86 87 #include <machine/limits.h> 88 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/route.h> 92 #include <net/netisr2.h> 93 #include <net/toeplitz2.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip_var.h> 99 #ifdef INET6 100 #include <netinet/ip6.h> 101 #include <netinet6/ip6_var.h> 102 #endif /* INET6 */ 103 104 #ifdef IPSEC 105 #include <netinet6/ipsec.h> 106 #include <netproto/key/key.h> 107 #include <netproto/ipsec/esp_var.h> 108 #endif 109 110 #ifdef FAST_IPSEC 111 #if defined(IPSEC) || defined(IPSEC_ESP) 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!" 113 #endif 114 115 #include <netproto/ipsec/ipsec.h> 116 #include <netproto/ipsec/key.h> 117 #define IPSEC 118 #endif /* FAST_IPSEC */ 119 120 #define INP_LOCALGROUP_SIZMIN 8 121 #define INP_LOCALGROUP_SIZMAX 256 122 123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash, 124 struct in_addr laddr, u_int lport_arg, int wild_okay, 125 struct ucred *cred); 126 127 struct in_addr zeroin_addr; 128 129 /* 130 * These configure the range of local port addresses assigned to 131 * "unspecified" outgoing connections/packets/whatever. 132 */ 133 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 134 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 135 136 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 137 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 138 139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 140 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 141 142 #define RANGECHK(var, min, max) \ 143 if ((var) < (min)) { (var) = (min); } \ 144 else if ((var) > (max)) { (var) = (max); } 145 146 int udpencap_enable = 1; /* enabled by default */ 147 int udpencap_port = 4500; /* triggers decapsulation */ 148 149 /* 150 * Per-netisr inpcb markers. 151 * NOTE: they should only be used in netisrs. 152 */ 153 static struct inpcb *in_pcbmarkers; 154 static struct inpcontainer *in_pcbcontainer_markers; 155 156 static int 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 158 { 159 int error; 160 161 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 162 if (!error) { 163 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 164 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 165 166 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 167 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 168 169 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 170 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 171 } 172 return (error); 173 } 174 175 #undef RANGECHK 176 177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 178 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 180 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 182 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 184 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 186 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 188 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 190 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 191 192 static int ip_porthash_trycount = 15; 193 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW, 194 &ip_porthash_trycount, 0, 195 "Number of tries to find local port matching hash of 4-tuple"); 196 197 /* 198 * in_pcb.c: manage the Protocol Control Blocks. 199 * 200 * NOTE: It is assumed that most of these functions will be called from 201 * a critical section. XXX - There are, unfortunately, a few exceptions 202 * to this rule that should be fixed. 203 * 204 * NOTE: The caller should initialize the cpu field to the cpu running the 205 * protocol stack associated with this inpcbinfo. 206 */ 207 208 void 209 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 210 { 211 KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu)); 212 pcbinfo->cpu = cpu; 213 214 LIST_INIT(&pcbinfo->pcblisthead); 215 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 216 M_WAITOK | M_ZERO); 217 218 if (shared) { 219 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 220 M_PCB, M_WAITOK); 221 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 222 } else { 223 pcbinfo->infotoken = NULL; 224 } 225 } 226 227 struct baddynamicports baddynamicports; 228 229 /* 230 * Check if the specified port is invalid for dynamic allocation. 231 */ 232 int 233 in_baddynamic(u_int16_t port, u_int16_t proto) 234 { 235 switch (proto) { 236 case IPPROTO_TCP: 237 return (DP_ISSET(baddynamicports.tcp, port)); 238 case IPPROTO_UDP: 239 #ifdef IPSEC 240 /* Cannot preset this as it is a sysctl */ 241 if (port == udpencap_port) 242 return (1); 243 #endif 244 return (DP_ISSET(baddynamicports.udp, port)); 245 default: 246 return (0); 247 } 248 } 249 250 void 251 in_pcbonlist(struct inpcb *inp) 252 { 253 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 254 255 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 256 ("not in the correct netisr")); 257 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 258 inp->inp_flags |= INP_ONLIST; 259 260 GET_PCBINFO_TOKEN(pcbinfo); 261 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 262 pcbinfo->ipi_count++; 263 REL_PCBINFO_TOKEN(pcbinfo); 264 } 265 266 void 267 in_pcbofflist(struct inpcb *inp) 268 { 269 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 270 271 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 272 ("not in the correct netisr")); 273 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 274 inp->inp_flags &= ~INP_ONLIST; 275 276 GET_PCBINFO_TOKEN(pcbinfo); 277 LIST_REMOVE(inp, inp_list); 278 KASSERT(pcbinfo->ipi_count > 0, 279 ("invalid inpcb count %d", pcbinfo->ipi_count)); 280 pcbinfo->ipi_count--; 281 REL_PCBINFO_TOKEN(pcbinfo); 282 } 283 284 /* 285 * Allocate a PCB and associate it with the socket. 286 */ 287 int 288 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 289 { 290 struct inpcb *inp; 291 #ifdef IPSEC 292 int error; 293 #endif 294 295 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 296 if (inp == NULL) 297 return (ENOMEM); 298 inp->inp_lgrpindex = -1; 299 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 300 inp->inp_pcbinfo = pcbinfo; 301 inp->inp_socket = so; 302 #ifdef IPSEC 303 error = ipsec_init_policy(so, &inp->inp_sp); 304 if (error != 0) { 305 kfree(inp, M_PCB); 306 return (error); 307 } 308 #endif 309 #ifdef INET6 310 if (INP_CHECK_SOCKAF(so, AF_INET6)) { 311 if (ip6_auto_flowlabel) 312 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 313 inp->inp_af = AF_INET6; 314 } else 315 #endif 316 inp->inp_af = AF_INET; 317 soreference(so); 318 so->so_pcb = inp; 319 320 in_pcbonlist(inp); 321 return (0); 322 } 323 324 /* 325 * Unlink a pcb with the intention of moving it to another cpu with a 326 * different pcbinfo. While unlinked nothing should attempt to dereference 327 * inp_pcbinfo, NULL it out so we assert if it does. 328 */ 329 void 330 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 331 { 332 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 333 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 334 ("already linked")); 335 336 in_pcbofflist(inp); 337 inp->inp_pcbinfo = NULL; 338 } 339 340 void 341 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 342 { 343 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 344 } 345 346 /* 347 * Relink a pcb into a new pcbinfo. 348 */ 349 void 350 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 351 { 352 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 353 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 354 ("already linked")); 355 356 inp->inp_pcbinfo = pcbinfo; 357 in_pcbonlist(inp); 358 } 359 360 void 361 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 362 { 363 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 364 } 365 366 static boolean_t 367 in_pcbporthash_update(struct inpcbportinfo *portinfo, 368 struct inpcb *inp, u_short lport, struct ucred *cred, int wild) 369 { 370 struct inpcbporthead *porthash; 371 372 /* 373 * This has to be atomic. If the porthash is shared across multiple 374 * protocol threads, e.g. tcp and udp, then the token must be held. 375 */ 376 porthash = in_pcbporthash_head(portinfo, lport); 377 GET_PORTHASH_TOKEN(porthash); 378 379 if (in_pcblookup_local(porthash, inp->inp_laddr, lport, 380 wild, cred) != NULL) { 381 REL_PORTHASH_TOKEN(porthash); 382 return FALSE; 383 } 384 inp->inp_lport = lport; 385 in_pcbinsporthash(porthash, inp); 386 387 REL_PORTHASH_TOKEN(porthash); 388 return TRUE; 389 } 390 391 static int 392 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 393 { 394 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 395 struct inpcbportinfo *portinfo; 396 u_short first, last, lport, step; 397 u_short *lastport; 398 int count, error; 399 int portinfo_first, portinfo_idx; 400 401 inp->inp_flags |= INP_ANONPORT; 402 403 step = pcbinfo->portinfo_mask + 1; 404 portinfo_first = mycpuid & pcbinfo->portinfo_mask; 405 portinfo_idx = portinfo_first; 406 loop: 407 portinfo = &pcbinfo->portinfo[portinfo_idx]; 408 409 if (inp->inp_flags & INP_HIGHPORT) { 410 first = ipport_hifirstauto; /* sysctl */ 411 last = ipport_hilastauto; 412 lastport = &portinfo->lasthi; 413 } else if (inp->inp_flags & INP_LOWPORT) { 414 if (cred && 415 (error = 416 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 417 inp->inp_laddr.s_addr = INADDR_ANY; 418 return error; 419 } 420 first = ipport_lowfirstauto; /* 1023 */ 421 last = ipport_lowlastauto; /* 600 */ 422 lastport = &portinfo->lastlow; 423 } else { 424 first = ipport_firstauto; /* sysctl */ 425 last = ipport_lastauto; 426 lastport = &portinfo->lastport; 427 } 428 429 /* 430 * Simple check to ensure all ports are not used up causing 431 * a deadlock here. 432 * 433 * We split the two cases (up and down) so that the direction 434 * is not being tested on each round of the loop. 435 */ 436 if (first > last) { 437 /* 438 * counting down 439 */ 440 in_pcbportrange(&first, &last, portinfo->offset, step); 441 count = (first - last) / step; 442 443 for (;;) { 444 if (count-- < 0) { /* completely used? */ 445 error = EADDRNOTAVAIL; 446 break; 447 } 448 lport = in_pcblastport_down(lastport, 449 first, last, step); 450 KKASSERT((lport & pcbinfo->portinfo_mask) == 451 portinfo->offset); 452 lport = htons(lport); 453 454 if (in_pcbporthash_update(portinfo, inp, lport, 455 cred, wild)) { 456 error = 0; 457 break; 458 } 459 } 460 } else { 461 /* 462 * counting up 463 */ 464 in_pcbportrange(&last, &first, portinfo->offset, step); 465 count = (last - first) / step; 466 467 for (;;) { 468 if (count-- < 0) { /* completely used? */ 469 error = EADDRNOTAVAIL; 470 break; 471 } 472 lport = in_pcblastport_up(lastport, first, last, step); 473 KKASSERT((lport & pcbinfo->portinfo_mask) == 474 portinfo->offset); 475 lport = htons(lport); 476 477 if (in_pcbporthash_update(portinfo, inp, lport, 478 cred, wild)) { 479 error = 0; 480 break; 481 } 482 } 483 } 484 485 if (error) { 486 /* Try next portinfo */ 487 portinfo_idx++; 488 portinfo_idx &= pcbinfo->portinfo_mask; 489 if (portinfo_idx != portinfo_first) 490 goto loop; 491 inp->inp_laddr.s_addr = INADDR_ANY; 492 } 493 return error; 494 } 495 496 int 497 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 498 { 499 struct socket *so = inp->inp_socket; 500 struct sockaddr_in jsin; 501 struct ucred *cred = NULL; 502 int wild = 0; 503 504 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 505 return (EADDRNOTAVAIL); 506 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 507 return (EINVAL); /* already bound */ 508 509 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 510 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 511 if (td->td_proc) 512 cred = td->td_proc->p_ucred; 513 514 if (nam != NULL) { 515 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 516 struct inpcbinfo *pcbinfo; 517 struct inpcbportinfo *portinfo; 518 struct inpcbporthead *porthash; 519 struct inpcb *t; 520 u_short lport, lport_ho; 521 int reuseport = (so->so_options & SO_REUSEPORT); 522 int error; 523 524 if (nam->sa_len != sizeof *sin) 525 return (EINVAL); 526 #ifdef notdef 527 /* 528 * We should check the family, but old programs 529 * incorrectly fail to initialize it. 530 */ 531 if (sin->sin_family != AF_INET) 532 return (EAFNOSUPPORT); 533 #endif 534 if (!prison_replace_wildcards(td, nam)) 535 return (EINVAL); 536 537 lport = sin->sin_port; 538 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 539 /* 540 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 541 * allow complete duplication of binding if 542 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 543 * and a multicast address is bound on both 544 * new and duplicated sockets. 545 */ 546 if (so->so_options & SO_REUSEADDR) 547 reuseport = SO_REUSEADDR | SO_REUSEPORT; 548 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 549 sin->sin_port = 0; /* yech... */ 550 bzero(&sin->sin_zero, sizeof sin->sin_zero); 551 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 552 return (EADDRNOTAVAIL); 553 } 554 555 inp->inp_laddr = sin->sin_addr; 556 557 jsin.sin_family = AF_INET; 558 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 559 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 560 inp->inp_laddr.s_addr = INADDR_ANY; 561 return (EINVAL); 562 } 563 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 564 565 if (lport == 0) { 566 /* Auto-select local port */ 567 return in_pcbsetlport(inp, wild, cred); 568 } 569 lport_ho = ntohs(lport); 570 571 /* GROSS */ 572 if (lport_ho < IPPORT_RESERVED && cred && 573 (error = 574 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 575 inp->inp_laddr.s_addr = INADDR_ANY; 576 return (error); 577 } 578 579 /* 580 * Locate the proper portinfo based on lport 581 */ 582 pcbinfo = inp->inp_pcbinfo; 583 portinfo = 584 &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask]; 585 KKASSERT((lport_ho & pcbinfo->portinfo_mask) == 586 portinfo->offset); 587 588 /* 589 * This has to be atomic. If the porthash is shared across 590 * multiple protocol threads, e.g. tcp and udp then the token 591 * must be held. 592 */ 593 porthash = in_pcbporthash_head(portinfo, lport); 594 GET_PORTHASH_TOKEN(porthash); 595 596 if (so->so_cred->cr_uid != 0 && 597 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 598 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 599 INPLOOKUP_WILDCARD, cred); 600 if (t && 601 (so->so_cred->cr_uid != 602 t->inp_socket->so_cred->cr_uid)) { 603 inp->inp_laddr.s_addr = INADDR_ANY; 604 error = EADDRINUSE; 605 goto done; 606 } 607 } 608 if (cred && !prison_replace_wildcards(td, nam)) { 609 inp->inp_laddr.s_addr = INADDR_ANY; 610 error = EADDRNOTAVAIL; 611 goto done; 612 } 613 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 614 wild, cred); 615 if (t && !(reuseport & t->inp_socket->so_options)) { 616 inp->inp_laddr.s_addr = INADDR_ANY; 617 error = EADDRINUSE; 618 goto done; 619 } 620 inp->inp_lport = lport; 621 in_pcbinsporthash(porthash, inp); 622 error = 0; 623 done: 624 REL_PORTHASH_TOKEN(porthash); 625 return (error); 626 } else { 627 jsin.sin_family = AF_INET; 628 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 629 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 630 inp->inp_laddr.s_addr = INADDR_ANY; 631 return (EINVAL); 632 } 633 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 634 635 return in_pcbsetlport(inp, wild, cred); 636 } 637 } 638 639 static struct inpcb * 640 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr, 641 u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred) 642 { 643 struct inpcb *inp; 644 struct inpcbport *phd; 645 struct inpcb *match = NULL; 646 647 /* 648 * If the porthashbase is shared across several cpus, it must 649 * have been locked. 650 */ 651 ASSERT_PORTHASH_TOKEN_HELD(porthash); 652 653 /* 654 * Best fit PCB lookup. 655 * 656 * First see if this local port is in use by looking on the 657 * port hash list. 658 */ 659 LIST_FOREACH(phd, porthash, phd_hash) { 660 if (phd->phd_port == lport) 661 break; 662 } 663 if (phd != NULL) { 664 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 665 #ifdef INET6 666 if (!INP_ISIPV4(inp)) 667 continue; 668 #endif 669 if (inp->inp_laddr.s_addr != INADDR_ANY && 670 inp->inp_laddr.s_addr != laddr.s_addr) 671 continue; 672 673 if (inp->inp_faddr.s_addr != INADDR_ANY && 674 inp->inp_faddr.s_addr != faddr.s_addr) 675 continue; 676 677 if (inp->inp_fport != 0 && inp->inp_fport != fport) 678 continue; 679 680 if (cred == NULL || 681 cred->cr_prison == 682 inp->inp_socket->so_cred->cr_prison) { 683 match = inp; 684 break; 685 } 686 } 687 } 688 return (match); 689 } 690 691 static boolean_t 692 in_pcbporthash_update4(struct inpcbportinfo *portinfo, 693 struct inpcb *inp, u_short lport, const struct sockaddr_in *sin, 694 struct ucred *cred) 695 { 696 struct inpcbporthead *porthash; 697 698 /* 699 * This has to be atomic. If the porthash is shared across multiple 700 * protocol threads, e.g. tcp and udp, then the token must be held. 701 */ 702 porthash = in_pcbporthash_head(portinfo, lport); 703 GET_PORTHASH_TOKEN(porthash); 704 705 if (in_pcblookup_localremote(porthash, inp->inp_laddr, 706 lport, sin->sin_addr, sin->sin_port, cred) != NULL) { 707 REL_PORTHASH_TOKEN(porthash); 708 return FALSE; 709 } 710 inp->inp_lport = lport; 711 in_pcbinsporthash(porthash, inp); 712 713 REL_PORTHASH_TOKEN(porthash); 714 return TRUE; 715 } 716 717 int 718 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 719 struct thread *td) 720 { 721 struct proc *p = td->td_proc; 722 u_short *lastport; 723 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 724 struct sockaddr_in jsin; 725 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 726 struct inpcbportinfo *portinfo; 727 struct ucred *cred = NULL; 728 u_short first, last, lport, step; 729 int count, error, selfconn; 730 int portinfo_first, portinfo_idx; 731 int hash_cpu, hash_count, hash_count0; 732 uint32_t hash_base = 0, hash; 733 734 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 735 return (EADDRNOTAVAIL); 736 737 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 738 if (inp->inp_lport != 0) 739 return (EINVAL); /* already bound */ 740 741 KKASSERT(p); 742 cred = p->p_ucred; 743 744 jsin.sin_family = AF_INET; 745 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 746 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 747 inp->inp_laddr.s_addr = INADDR_ANY; 748 return (EINVAL); 749 } 750 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 751 752 hash_count0 = ip_porthash_trycount; 753 if (hash_count0 > 0) { 754 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^ 755 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^ 756 toeplitz_piecemeal_port(sin->sin_port); 757 } else { 758 hash_count0 = 0; 759 } 760 761 inp->inp_flags |= INP_ANONPORT; 762 763 step = pcbinfo->portinfo_mask + 1; 764 portinfo_first = mycpuid & pcbinfo->portinfo_mask; 765 portinfo_idx = portinfo_first; 766 loop: 767 hash_cpu = portinfo_idx & ncpus2_mask; 768 portinfo = &pcbinfo->portinfo[portinfo_idx]; 769 selfconn = 0; 770 771 if (inp->inp_flags & INP_HIGHPORT) { 772 first = ipport_hifirstauto; /* sysctl */ 773 last = ipport_hilastauto; 774 lastport = &portinfo->lasthi; 775 } else if (inp->inp_flags & INP_LOWPORT) { 776 if (cred && 777 (error = 778 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 779 inp->inp_laddr.s_addr = INADDR_ANY; 780 return (error); 781 } 782 first = ipport_lowfirstauto; /* 1023 */ 783 last = ipport_lowlastauto; /* 600 */ 784 lastport = &portinfo->lastlow; 785 } else { 786 first = ipport_firstauto; /* sysctl */ 787 last = ipport_lastauto; 788 lastport = &portinfo->lastport; 789 } 790 791 /* This could happen on loopback interface */ 792 #define IS_SELFCONNECT(inp, lport, sin) \ 793 (__predict_false((sin)->sin_port == (lport) && \ 794 (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr)) 795 796 /* 797 * Simple check to ensure all ports are not used up causing 798 * a deadlock here. 799 * 800 * We split the two cases (up and down) so that the direction 801 * is not being tested on each round of the loop. 802 */ 803 hash_count = hash_count0; 804 if (first > last) { 805 /* 806 * counting down 807 */ 808 in_pcbportrange(&first, &last, portinfo->offset, step); 809 count = ((first - last) / step) + hash_count; 810 811 for (;;) { 812 if (count-- < 0) { /* completely used? */ 813 error = EADDRNOTAVAIL; 814 break; 815 } 816 817 lport = in_pcblastport_down(lastport, first, last, 818 step); 819 KKASSERT((lport & pcbinfo->portinfo_mask) == 820 portinfo->offset); 821 lport = htons(lport); 822 if (IS_SELFCONNECT(inp, lport, sin)) { 823 if (!selfconn) { 824 ++count; /* don't count this try */ 825 selfconn = 1; 826 } 827 continue; 828 } 829 830 if (hash_count) { 831 --hash_count; 832 hash = hash_base ^ 833 toeplitz_piecemeal_port(lport); 834 if ((hash & ncpus2_mask) != hash_cpu && hash_count) 835 continue; 836 } 837 838 if (in_pcbporthash_update4(portinfo, 839 inp, lport, sin, cred)) { 840 error = 0; 841 break; 842 } 843 } 844 } else { 845 /* 846 * counting up 847 */ 848 in_pcbportrange(&last, &first, portinfo->offset, step); 849 count = ((last - first) / step) + hash_count; 850 851 for (;;) { 852 if (count-- < 0) { /* completely used? */ 853 error = EADDRNOTAVAIL; 854 break; 855 } 856 857 lport = in_pcblastport_up(lastport, first, last, step); 858 KKASSERT((lport & pcbinfo->portinfo_mask) == 859 portinfo->offset); 860 lport = htons(lport); 861 if (IS_SELFCONNECT(inp, lport, sin)) { 862 if (!selfconn) { 863 ++count; /* don't count this try */ 864 selfconn = 1; 865 } 866 continue; 867 } 868 869 if (hash_count) { 870 --hash_count; 871 hash = hash_base ^ 872 toeplitz_piecemeal_port(lport); 873 if ((hash & ncpus2_mask) != hash_cpu && hash_count) 874 continue; 875 } 876 877 if (in_pcbporthash_update4(portinfo, 878 inp, lport, sin, cred)) { 879 error = 0; 880 break; 881 } 882 } 883 } 884 885 #undef IS_SELFCONNECT 886 887 if (error) { 888 /* Try next portinfo */ 889 portinfo_idx++; 890 portinfo_idx &= pcbinfo->portinfo_mask; 891 if (portinfo_idx != portinfo_first) 892 goto loop; 893 inp->inp_laddr.s_addr = INADDR_ANY; 894 } 895 return error; 896 } 897 898 /* 899 * Transform old in_pcbconnect() into an inner subroutine for new 900 * in_pcbconnect(): Do some validity-checking on the remote 901 * address (in mbuf 'nam') and then determine local host address 902 * (i.e., which interface) to use to access that remote host. 903 * 904 * This preserves definition of in_pcbconnect(), while supporting a 905 * slightly different version for T/TCP. (This is more than 906 * a bit of a kludge, but cleaning up the internal interfaces would 907 * have forced minor changes in every protocol). 908 */ 909 int 910 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 911 struct sockaddr_in **plocal_sin, struct thread *td, int find) 912 { 913 struct in_ifaddr *ia; 914 struct ucred *cred = NULL; 915 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 916 struct sockaddr *jsin; 917 int jailed = 0, alloc_route = 0; 918 919 if (nam->sa_len != sizeof *sin) 920 return (EINVAL); 921 if (sin->sin_family != AF_INET) 922 return (EAFNOSUPPORT); 923 if (sin->sin_port == 0) 924 return (EADDRNOTAVAIL); 925 if (td && td->td_proc && td->td_proc->p_ucred) 926 cred = td->td_proc->p_ucred; 927 if (cred && cred->cr_prison) 928 jailed = 1; 929 if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) { 930 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 931 /* 932 * If the destination address is INADDR_ANY, 933 * use the primary local address. 934 * If the supplied address is INADDR_BROADCAST, 935 * and the primary interface supports broadcast, 936 * choose the broadcast address for that interface. 937 */ 938 if (sin->sin_addr.s_addr == INADDR_ANY) 939 sin->sin_addr = IA_SIN(ia)->sin_addr; 940 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST && 941 (ia->ia_ifp->if_flags & IFF_BROADCAST)) 942 sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr; 943 } 944 if (find) { 945 struct route *ro; 946 947 ia = NULL; 948 /* 949 * If route is known or can be allocated now, 950 * our src addr is taken from the i/f, else punt. 951 * Note that we should check the address family of the cached 952 * destination, in case of sharing the cache with IPv6. 953 */ 954 ro = &inp->inp_route; 955 if (ro->ro_rt && 956 (!(ro->ro_rt->rt_flags & RTF_UP) || 957 ro->ro_dst.sa_family != AF_INET || 958 satosin(&ro->ro_dst)->sin_addr.s_addr != 959 sin->sin_addr.s_addr || 960 inp->inp_socket->so_options & SO_DONTROUTE)) { 961 RTFREE(ro->ro_rt); 962 ro->ro_rt = NULL; 963 } 964 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 965 (ro->ro_rt == NULL || 966 ro->ro_rt->rt_ifp == NULL)) { 967 /* No route yet, so try to acquire one */ 968 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 969 ro->ro_dst.sa_family = AF_INET; 970 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 971 ((struct sockaddr_in *) &ro->ro_dst)->sin_addr = 972 sin->sin_addr; 973 rtalloc(ro); 974 alloc_route = 1; 975 } 976 /* 977 * If we found a route, use the address 978 * corresponding to the outgoing interface 979 * unless it is the loopback (in case a route 980 * to our address on another net goes to loopback). 981 */ 982 if (ro->ro_rt && 983 !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) { 984 if (jailed) { 985 if (jailed_ip(cred->cr_prison, 986 ro->ro_rt->rt_ifa->ifa_addr)) { 987 ia = ifatoia(ro->ro_rt->rt_ifa); 988 } 989 } else { 990 ia = ifatoia(ro->ro_rt->rt_ifa); 991 } 992 } 993 if (ia == NULL) { 994 u_short fport = sin->sin_port; 995 996 sin->sin_port = 0; 997 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 998 if (ia && jailed && !jailed_ip(cred->cr_prison, 999 sintosa(&ia->ia_addr))) 1000 ia = NULL; 1001 if (ia == NULL) 1002 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 1003 if (ia && jailed && !jailed_ip(cred->cr_prison, 1004 sintosa(&ia->ia_addr))) 1005 ia = NULL; 1006 sin->sin_port = fport; 1007 if (ia == NULL && 1008 !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 1009 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 1010 if (ia && jailed && !jailed_ip(cred->cr_prison, 1011 sintosa(&ia->ia_addr))) 1012 ia = NULL; 1013 1014 if (!jailed && ia == NULL) 1015 goto fail; 1016 } 1017 /* 1018 * If the destination address is multicast and an outgoing 1019 * interface has been set as a multicast option, use the 1020 * address of that interface as our source address. 1021 */ 1022 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 1023 inp->inp_moptions != NULL) { 1024 struct ip_moptions *imo; 1025 struct ifnet *ifp; 1026 1027 imo = inp->inp_moptions; 1028 if ((ifp = imo->imo_multicast_ifp) != NULL) { 1029 struct in_ifaddr_container *iac; 1030 1031 ia = NULL; 1032 TAILQ_FOREACH(iac, 1033 &in_ifaddrheads[mycpuid], ia_link) { 1034 if (iac->ia->ia_ifp == ifp) { 1035 ia = iac->ia; 1036 break; 1037 } 1038 } 1039 if (ia == NULL) 1040 goto fail; 1041 } 1042 } 1043 /* 1044 * Don't do pcblookup call here; return interface in plocal_sin 1045 * and exit to caller, that will do the lookup. 1046 */ 1047 if (ia == NULL && jailed) { 1048 if ((jsin = prison_get_nonlocal( 1049 cred->cr_prison, AF_INET, NULL)) != NULL || 1050 (jsin = prison_get_local( 1051 cred->cr_prison, AF_INET, NULL)) != NULL) { 1052 *plocal_sin = satosin(jsin); 1053 } else { 1054 /* IPv6 only Jail */ 1055 goto fail; 1056 } 1057 } else { 1058 *plocal_sin = &ia->ia_addr; 1059 } 1060 } 1061 return (0); 1062 fail: 1063 if (alloc_route) 1064 in_pcbresetroute(inp); 1065 return (EADDRNOTAVAIL); 1066 } 1067 1068 int 1069 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 1070 struct sockaddr_in **plocal_sin, struct thread *td) 1071 { 1072 return in_pcbladdr_find(inp, nam, plocal_sin, td, 1073 (inp->inp_laddr.s_addr == INADDR_ANY)); 1074 } 1075 1076 /* 1077 * Outer subroutine: 1078 * Connect from a socket to a specified address. 1079 * Both address and port must be specified in argument sin. 1080 * If don't have a local address for this socket yet, 1081 * then pick one. 1082 */ 1083 int 1084 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 1085 { 1086 struct sockaddr_in *if_sin; 1087 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1088 int error; 1089 1090 if_sin = NULL; /* avoid gcc warnings */ 1091 1092 /* Call inner routine to assign local interface address. */ 1093 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1094 return (error); 1095 1096 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1097 inp->inp_laddr.s_addr ? 1098 inp->inp_laddr : if_sin->sin_addr, 1099 inp->inp_lport, FALSE, NULL) != NULL) { 1100 return (EADDRINUSE); 1101 } 1102 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1103 if (inp->inp_lport == 0) { 1104 error = in_pcbbind(inp, NULL, td); 1105 if (error) 1106 return (error); 1107 } 1108 inp->inp_laddr = if_sin->sin_addr; 1109 } 1110 inp->inp_faddr = sin->sin_addr; 1111 inp->inp_fport = sin->sin_port; 1112 in_pcbinsconnhash(inp); 1113 return (0); 1114 } 1115 1116 void 1117 in_pcbdisconnect(struct inpcb *inp) 1118 { 1119 1120 in_pcbremconnhash(inp); 1121 inp->inp_faddr.s_addr = INADDR_ANY; 1122 inp->inp_fport = 0; 1123 } 1124 1125 void 1126 in_pcbdetach(struct inpcb *inp) 1127 { 1128 struct socket *so = inp->inp_socket; 1129 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1130 1131 #ifdef IPSEC 1132 ipsec4_delete_pcbpolicy(inp); 1133 #endif /*IPSEC*/ 1134 inp->inp_gencnt = ++ipi->ipi_gencnt; 1135 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1136 in_pcbremlists(inp); 1137 so->so_pcb = NULL; 1138 sofree(so); /* remove pcb ref */ 1139 if (inp->inp_options) 1140 m_free(inp->inp_options); 1141 if (inp->inp_route.ro_rt) 1142 rtfree(inp->inp_route.ro_rt); 1143 ip_freemoptions(inp->inp_moptions); 1144 kfree(inp, M_PCB); 1145 } 1146 1147 /* 1148 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1149 * socket received RST. 1150 */ 1151 static int 1152 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1153 { 1154 struct inpcb *inp; 1155 struct sockaddr_in *sin; 1156 1157 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1158 inp = so->so_pcb; 1159 if (!inp) 1160 return (ECONNRESET); 1161 1162 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1163 sin->sin_family = AF_INET; 1164 sin->sin_len = sizeof *sin; 1165 sin->sin_port = inp->inp_lport; 1166 sin->sin_addr = inp->inp_laddr; 1167 1168 *nam = (struct sockaddr *)sin; 1169 return (0); 1170 } 1171 1172 void 1173 in_setsockaddr_dispatch(netmsg_t msg) 1174 { 1175 int error; 1176 1177 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1178 lwkt_replymsg(&msg->lmsg, error); 1179 } 1180 1181 /* 1182 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1183 * socket received RST. 1184 */ 1185 int 1186 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1187 { 1188 struct inpcb *inp; 1189 struct sockaddr_in *sin; 1190 1191 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1192 inp = so->so_pcb; 1193 if (!inp) 1194 return (ECONNRESET); 1195 1196 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1197 sin->sin_family = AF_INET; 1198 sin->sin_len = sizeof *sin; 1199 sin->sin_port = inp->inp_fport; 1200 sin->sin_addr = inp->inp_faddr; 1201 1202 *nam = (struct sockaddr *)sin; 1203 return (0); 1204 } 1205 1206 void 1207 in_setpeeraddr_dispatch(netmsg_t msg) 1208 { 1209 int error; 1210 1211 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1212 lwkt_replymsg(&msg->lmsg, error); 1213 } 1214 1215 void 1216 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1217 inp_notify_t notify) 1218 { 1219 struct inpcb *inp, *marker; 1220 1221 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1222 ("not in the correct netisr")); 1223 marker = &in_pcbmarkers[mycpuid]; 1224 1225 /* 1226 * NOTE: 1227 * - If INP_PLACEMARKER is set we must ignore the rest of the 1228 * structure and skip it. 1229 * - It is safe to nuke inpcbs here, since we are in their own 1230 * netisr. 1231 */ 1232 GET_PCBINFO_TOKEN(pcbinfo); 1233 1234 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1235 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1236 LIST_REMOVE(marker, inp_list); 1237 LIST_INSERT_AFTER(inp, marker, inp_list); 1238 1239 if (inp->inp_flags & INP_PLACEMARKER) 1240 continue; 1241 #ifdef INET6 1242 if (!INP_ISIPV4(inp)) 1243 continue; 1244 #endif 1245 if (inp->inp_faddr.s_addr != faddr.s_addr || 1246 inp->inp_socket == NULL) 1247 continue; 1248 (*notify)(inp, err); /* can remove inp from list! */ 1249 } 1250 LIST_REMOVE(marker, inp_list); 1251 1252 REL_PCBINFO_TOKEN(pcbinfo); 1253 } 1254 1255 void 1256 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1257 { 1258 struct inpcb *inp, *marker; 1259 1260 /* 1261 * We only need to make sure that we are in netisr0, where all 1262 * multicast operation happen. We could check inpcbinfo which 1263 * does not belong to netisr0 by holding the inpcbinfo's token. 1264 * In this case, the pcbinfo must be able to be shared, i.e. 1265 * pcbinfo->infotoken is not NULL. 1266 */ 1267 ASSERT_IN_NETISR(0); 1268 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1269 ("pcbinfo could not be shared")); 1270 1271 /* 1272 * Get a marker for the current netisr (netisr0). 1273 * 1274 * It is possible that the multicast address deletion blocks, 1275 * which could cause temporary token releasing. So we use 1276 * inpcb marker here to get a coherent view of the inpcb list. 1277 * 1278 * While, on the other hand, moptions are only added and deleted 1279 * in netisr0, so we would not see staled moption or miss moption 1280 * even if the token was released due to the blocking multicast 1281 * address deletion. 1282 */ 1283 marker = &in_pcbmarkers[mycpuid]; 1284 1285 GET_PCBINFO_TOKEN(pcbinfo); 1286 1287 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1288 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1289 struct ip_moptions *imo; 1290 1291 LIST_REMOVE(marker, inp_list); 1292 LIST_INSERT_AFTER(inp, marker, inp_list); 1293 1294 if (inp->inp_flags & INP_PLACEMARKER) 1295 continue; 1296 imo = inp->inp_moptions; 1297 if (INP_ISIPV4(inp) && imo != NULL) { 1298 int i, gap; 1299 1300 /* 1301 * Unselect the outgoing interface if it is being 1302 * detached. 1303 */ 1304 if (imo->imo_multicast_ifp == ifp) 1305 imo->imo_multicast_ifp = NULL; 1306 1307 /* 1308 * Drop multicast group membership if we joined 1309 * through the interface being detached. 1310 */ 1311 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1312 i++) { 1313 if (imo->imo_membership[i]->inm_ifp == ifp) { 1314 /* 1315 * NOTE: 1316 * This could block and the pcbinfo 1317 * token could be passively released. 1318 */ 1319 in_delmulti(imo->imo_membership[i]); 1320 gap++; 1321 } else if (gap != 0) 1322 imo->imo_membership[i - gap] = 1323 imo->imo_membership[i]; 1324 } 1325 imo->imo_num_memberships -= gap; 1326 } 1327 } 1328 LIST_REMOVE(marker, inp_list); 1329 1330 REL_PCBINFO_TOKEN(pcbinfo); 1331 } 1332 1333 /* 1334 * Check for alternatives when higher level complains 1335 * about service problems. For now, invalidate cached 1336 * routing information. If the route was created dynamically 1337 * (by a redirect), time to try a default gateway again. 1338 */ 1339 void 1340 in_losing(struct inpcb *inp) 1341 { 1342 struct rtentry *rt; 1343 struct rt_addrinfo rtinfo; 1344 1345 if ((rt = inp->inp_route.ro_rt)) { 1346 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1347 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1348 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1349 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1350 rtinfo.rti_flags = rt->rt_flags; 1351 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1352 if (rt->rt_flags & RTF_DYNAMIC) { 1353 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1354 rt_mask(rt), rt->rt_flags, NULL); 1355 } 1356 inp->inp_route.ro_rt = NULL; 1357 rtfree(rt); 1358 /* 1359 * A new route can be allocated 1360 * the next time output is attempted. 1361 */ 1362 } 1363 } 1364 1365 /* 1366 * After a routing change, flush old routing 1367 * and allocate a (hopefully) better one. 1368 */ 1369 void 1370 in_rtchange(struct inpcb *inp, int err) 1371 { 1372 if (inp->inp_route.ro_rt) { 1373 rtfree(inp->inp_route.ro_rt); 1374 inp->inp_route.ro_rt = NULL; 1375 /* 1376 * A new route can be allocated the next time 1377 * output is attempted. 1378 */ 1379 } 1380 } 1381 1382 /* 1383 * Lookup a PCB based on the local address and port. 1384 */ 1385 static struct inpcb * 1386 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr, 1387 u_int lport_arg, int wild_okay, struct ucred *cred) 1388 { 1389 struct inpcb *inp; 1390 int matchwild = 3, wildcard; 1391 u_short lport = lport_arg; 1392 struct inpcbport *phd; 1393 struct inpcb *match = NULL; 1394 1395 /* 1396 * If the porthashbase is shared across several cpus, it must 1397 * have been locked. 1398 */ 1399 ASSERT_PORTHASH_TOKEN_HELD(porthash); 1400 1401 /* 1402 * Best fit PCB lookup. 1403 * 1404 * First see if this local port is in use by looking on the 1405 * port hash list. 1406 */ 1407 LIST_FOREACH(phd, porthash, phd_hash) { 1408 if (phd->phd_port == lport) 1409 break; 1410 } 1411 if (phd != NULL) { 1412 /* 1413 * Port is in use by one or more PCBs. Look for best 1414 * fit. 1415 */ 1416 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1417 wildcard = 0; 1418 #ifdef INET6 1419 if (!INP_ISIPV4(inp)) 1420 continue; 1421 #endif 1422 if (inp->inp_faddr.s_addr != INADDR_ANY) 1423 wildcard++; 1424 if (inp->inp_laddr.s_addr != INADDR_ANY) { 1425 if (laddr.s_addr == INADDR_ANY) 1426 wildcard++; 1427 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1428 continue; 1429 } else { 1430 if (laddr.s_addr != INADDR_ANY) 1431 wildcard++; 1432 } 1433 if (wildcard && !wild_okay) 1434 continue; 1435 if (wildcard < matchwild && 1436 (cred == NULL || 1437 cred->cr_prison == 1438 inp->inp_socket->so_cred->cr_prison)) { 1439 match = inp; 1440 matchwild = wildcard; 1441 if (matchwild == 0) { 1442 break; 1443 } 1444 } 1445 } 1446 } 1447 return (match); 1448 } 1449 1450 struct inpcb * 1451 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1452 const struct inpcb *inp) 1453 { 1454 const struct inp_localgrphead *hdr; 1455 const struct inp_localgroup *grp; 1456 int i; 1457 1458 if (pcbinfo->localgrphashbase == NULL) 1459 return NULL; 1460 1461 GET_PCBINFO_TOKEN(pcbinfo); 1462 1463 hdr = &pcbinfo->localgrphashbase[ 1464 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1465 1466 LIST_FOREACH(grp, hdr, il_list) { 1467 if (grp->il_af == inp->inp_af && 1468 grp->il_lport == inp->inp_lport && 1469 memcmp(&grp->il_dependladdr, 1470 &inp->inp_inc.inc_ie.ie_dependladdr, 1471 sizeof(grp->il_dependladdr)) == 0) { 1472 break; 1473 } 1474 } 1475 if (grp == NULL || grp->il_inpcnt == 1) { 1476 REL_PCBINFO_TOKEN(pcbinfo); 1477 return NULL; 1478 } 1479 1480 KASSERT(grp->il_inpcnt >= 2, 1481 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1482 for (i = 0; i < grp->il_inpcnt; ++i) { 1483 if (grp->il_inp[i] == inp) { 1484 int last = grp->il_inpcnt - 1; 1485 1486 if (i == last) 1487 last = grp->il_inpcnt - 2; 1488 REL_PCBINFO_TOKEN(pcbinfo); 1489 return grp->il_inp[last]; 1490 } 1491 } 1492 REL_PCBINFO_TOKEN(pcbinfo); 1493 return NULL; 1494 } 1495 1496 static struct inpcb * 1497 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1498 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1499 { 1500 struct inpcb *local_wild = NULL; 1501 const struct inp_localgrphead *hdr; 1502 const struct inp_localgroup *grp; 1503 1504 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1505 1506 hdr = &pcbinfo->localgrphashbase[ 1507 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1508 1509 /* 1510 * Order of socket selection: 1511 * 1. non-wild. 1512 * 2. wild. 1513 * 1514 * NOTE: Local group does not contain jailed sockets 1515 */ 1516 LIST_FOREACH(grp, hdr, il_list) { 1517 #ifdef INET6 1518 if (grp->il_af != AF_INET) 1519 continue; 1520 #endif 1521 if (grp->il_lport == lport) { 1522 int idx; 1523 1524 /* 1525 * Modulo-N is used here, which greatly reduces 1526 * completion queue token contention, thus more 1527 * cpu time is saved. 1528 */ 1529 idx = pkt_hash % grp->il_inpcnt; 1530 if (grp->il_laddr.s_addr == laddr.s_addr) 1531 return grp->il_inp[idx]; 1532 else if (grp->il_laddr.s_addr == INADDR_ANY) 1533 local_wild = grp->il_inp[idx]; 1534 } 1535 } 1536 if (local_wild != NULL) 1537 return local_wild; 1538 return NULL; 1539 } 1540 1541 /* 1542 * Lookup PCB in hash list. 1543 */ 1544 struct inpcb * 1545 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1546 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1547 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1548 { 1549 struct inpcbhead *head; 1550 struct inpcb *inp, *jinp=NULL; 1551 u_short fport = fport_arg, lport = lport_arg; 1552 1553 /* 1554 * First look for an exact match. 1555 */ 1556 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1557 laddr.s_addr, lport, pcbinfo->hashmask)]; 1558 LIST_FOREACH(inp, head, inp_hash) { 1559 #ifdef INET6 1560 if (!INP_ISIPV4(inp)) 1561 continue; 1562 #endif 1563 if (in_hosteq(inp->inp_faddr, faddr) && 1564 in_hosteq(inp->inp_laddr, laddr) && 1565 inp->inp_fport == fport && inp->inp_lport == lport) { 1566 /* found */ 1567 if (inp->inp_socket == NULL || 1568 inp->inp_socket->so_cred->cr_prison == NULL) { 1569 return (inp); 1570 } else { 1571 if (jinp == NULL) 1572 jinp = inp; 1573 } 1574 } 1575 } 1576 if (jinp != NULL) 1577 return (jinp); 1578 1579 if (wildcard) { 1580 struct inpcb *local_wild = NULL; 1581 struct inpcb *jinp_wild = NULL; 1582 struct inpcontainer *ic; 1583 struct inpcontainerhead *chead; 1584 struct sockaddr_in jsin; 1585 struct ucred *cred; 1586 1587 GET_PCBINFO_TOKEN(pcbinfo); 1588 1589 /* 1590 * Check local group first 1591 */ 1592 if (pcbinfo->localgrphashbase != NULL && 1593 m != NULL && (m->m_flags & M_HASH) && 1594 !(ifp && ifp->if_type == IFT_FAITH)) { 1595 inp = inp_localgroup_lookup(pcbinfo, 1596 laddr, lport, m->m_pkthdr.hash); 1597 if (inp != NULL) { 1598 REL_PCBINFO_TOKEN(pcbinfo); 1599 return inp; 1600 } 1601 } 1602 1603 /* 1604 * Order of socket selection: 1605 * 1. non-jailed, non-wild. 1606 * 2. non-jailed, wild. 1607 * 3. jailed, non-wild. 1608 * 4. jailed, wild. 1609 */ 1610 jsin.sin_family = AF_INET; 1611 chead = &pcbinfo->wildcardhashbase[ 1612 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1613 LIST_FOREACH(ic, chead, ic_list) { 1614 inp = ic->ic_inp; 1615 if (inp->inp_flags & INP_PLACEMARKER) 1616 continue; 1617 1618 jsin.sin_addr.s_addr = laddr.s_addr; 1619 #ifdef INET6 1620 if (!INP_ISIPV4(inp)) 1621 continue; 1622 #endif 1623 if (inp->inp_socket != NULL) 1624 cred = inp->inp_socket->so_cred; 1625 else 1626 cred = NULL; 1627 if (cred != NULL && jailed(cred)) { 1628 if (jinp != NULL) 1629 continue; 1630 else 1631 if (!jailed_ip(cred->cr_prison, 1632 (struct sockaddr *)&jsin)) 1633 continue; 1634 } 1635 if (inp->inp_lport == lport) { 1636 if (ifp && ifp->if_type == IFT_FAITH && 1637 !(inp->inp_flags & INP_FAITH)) 1638 continue; 1639 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1640 if (cred != NULL && jailed(cred)) { 1641 jinp = inp; 1642 } else { 1643 REL_PCBINFO_TOKEN(pcbinfo); 1644 return (inp); 1645 } 1646 } 1647 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1648 if (cred != NULL && jailed(cred)) 1649 jinp_wild = inp; 1650 else 1651 local_wild = inp; 1652 } 1653 } 1654 } 1655 1656 REL_PCBINFO_TOKEN(pcbinfo); 1657 1658 if (local_wild != NULL) 1659 return (local_wild); 1660 if (jinp != NULL) 1661 return (jinp); 1662 return (jinp_wild); 1663 } 1664 1665 /* 1666 * Not found. 1667 */ 1668 return (NULL); 1669 } 1670 1671 struct inpcb * 1672 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1673 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1674 boolean_t wildcard, struct ifnet *ifp) 1675 { 1676 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1677 laddr, lport_arg, wildcard, ifp, NULL); 1678 } 1679 1680 /* 1681 * Insert PCB into connection hash table. 1682 */ 1683 void 1684 in_pcbinsconnhash(struct inpcb *inp) 1685 { 1686 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1687 struct inpcbhead *bucket; 1688 u_int32_t hashkey_faddr, hashkey_laddr; 1689 1690 #ifdef INET6 1691 if (INP_ISIPV6(inp)) { 1692 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1693 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1694 } else { 1695 #endif 1696 hashkey_faddr = inp->inp_faddr.s_addr; 1697 hashkey_laddr = inp->inp_laddr.s_addr; 1698 #ifdef INET6 1699 } 1700 #endif 1701 1702 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1703 ("not in the correct netisr")); 1704 ASSERT_INP_NOTINHASH(inp); 1705 inp->inp_flags |= INP_CONNECTED; 1706 1707 /* 1708 * Insert into the connection hash table. 1709 */ 1710 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1711 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1712 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1713 } 1714 1715 /* 1716 * Remove PCB from connection hash table. 1717 */ 1718 void 1719 in_pcbremconnhash(struct inpcb *inp) 1720 { 1721 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1722 1723 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1724 ("not in the correct netisr")); 1725 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1726 1727 LIST_REMOVE(inp, inp_hash); 1728 inp->inp_flags &= ~INP_CONNECTED; 1729 } 1730 1731 /* 1732 * Insert PCB into port hash table. 1733 */ 1734 void 1735 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp) 1736 { 1737 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1738 struct inpcbport *phd; 1739 1740 /* 1741 * If the porthashbase is shared across several cpus, it must 1742 * have been locked. 1743 */ 1744 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash); 1745 1746 /* 1747 * Insert into the port hash table. 1748 */ 1749 1750 /* Go through port list and look for a head for this lport. */ 1751 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1752 if (phd->phd_port == inp->inp_lport) 1753 break; 1754 } 1755 1756 /* If none exists, use saved one and tack it on. */ 1757 if (phd == NULL) { 1758 KKASSERT(pcbinfo->portsave != NULL); 1759 phd = pcbinfo->portsave; 1760 pcbinfo->portsave = NULL; 1761 phd->phd_port = inp->inp_lport; 1762 LIST_INIT(&phd->phd_pcblist); 1763 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1764 } 1765 1766 inp->inp_porthash = pcbporthash; 1767 inp->inp_phd = phd; 1768 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1769 1770 /* 1771 * Malloc one inpcbport for later use. It is safe to use 1772 * "wait" malloc here (port token would be released, if 1773 * malloc ever blocked), since all changes to the porthash 1774 * are done. 1775 */ 1776 if (pcbinfo->portsave == NULL) { 1777 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1778 M_PCB, M_INTWAIT | M_ZERO); 1779 } 1780 } 1781 1782 void 1783 in_pcbinsporthash_lport(struct inpcb *inp) 1784 { 1785 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1786 struct inpcbportinfo *portinfo; 1787 struct inpcbporthead *porthash; 1788 u_short lport_ho; 1789 1790 /* Locate the proper portinfo based on lport */ 1791 lport_ho = ntohs(inp->inp_lport); 1792 portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask]; 1793 KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset); 1794 1795 porthash = in_pcbporthash_head(portinfo, inp->inp_lport); 1796 GET_PORTHASH_TOKEN(porthash); 1797 in_pcbinsporthash(porthash, inp); 1798 REL_PORTHASH_TOKEN(porthash); 1799 } 1800 1801 void 1802 in_pcbremporthash(struct inpcb *inp) 1803 { 1804 struct inpcbporthead *porthash; 1805 struct inpcbport *phd; 1806 1807 if (inp->inp_phd == NULL) 1808 return; 1809 KASSERT(inp->inp_lport != 0, ("inpcb has no lport")); 1810 1811 porthash = inp->inp_porthash; 1812 KASSERT(porthash != NULL, ("no porthash")); 1813 1814 GET_PORTHASH_TOKEN(porthash); 1815 1816 phd = inp->inp_phd; 1817 LIST_REMOVE(inp, inp_portlist); 1818 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 1819 LIST_REMOVE(phd, phd_hash); 1820 kfree(phd, M_PCB); 1821 } 1822 1823 REL_PORTHASH_TOKEN(porthash); 1824 1825 inp->inp_phd = NULL; 1826 /* NOTE: Don't whack inp_lport, which may be used later */ 1827 } 1828 1829 static struct inp_localgroup * 1830 inp_localgroup_alloc(u_char af, uint16_t port, 1831 const union in_dependaddr *addr, int size) 1832 { 1833 struct inp_localgroup *grp; 1834 1835 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 1836 M_TEMP, M_INTWAIT | M_ZERO); 1837 grp->il_af = af; 1838 grp->il_lport = port; 1839 grp->il_dependladdr = *addr; 1840 grp->il_inpsiz = size; 1841 1842 return grp; 1843 } 1844 1845 static void 1846 inp_localgroup_free(struct inp_localgroup *grp) 1847 { 1848 kfree(grp, M_TEMP); 1849 } 1850 1851 static void 1852 inp_localgroup_destroy(struct inp_localgroup *grp) 1853 { 1854 LIST_REMOVE(grp, il_list); 1855 inp_localgroup_free(grp); 1856 } 1857 1858 static void 1859 inp_localgroup_copy(struct inp_localgroup *grp, 1860 const struct inp_localgroup *old_grp) 1861 { 1862 int i; 1863 1864 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 1865 ("invalid new local group size %d and old local group count %d", 1866 grp->il_inpsiz, old_grp->il_inpcnt)); 1867 for (i = 0; i < old_grp->il_inpcnt; ++i) 1868 grp->il_inp[i] = old_grp->il_inp[i]; 1869 grp->il_inpcnt = old_grp->il_inpcnt; 1870 } 1871 1872 static void 1873 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1874 { 1875 struct inp_localgrphead *hdr; 1876 struct inp_localgroup *grp, *grp_alloc = NULL; 1877 struct ucred *cred; 1878 int i, idx; 1879 1880 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1881 1882 if (pcbinfo->localgrphashbase == NULL) 1883 return; 1884 1885 /* 1886 * XXX don't allow jailed socket to join local group 1887 */ 1888 if (inp->inp_socket != NULL) 1889 cred = inp->inp_socket->so_cred; 1890 else 1891 cred = NULL; 1892 if (cred != NULL && jailed(cred)) 1893 return; 1894 1895 hdr = &pcbinfo->localgrphashbase[ 1896 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1897 1898 again: 1899 LIST_FOREACH(grp, hdr, il_list) { 1900 if (grp->il_af == inp->inp_af && 1901 grp->il_lport == inp->inp_lport && 1902 memcmp(&grp->il_dependladdr, 1903 &inp->inp_inc.inc_ie.ie_dependladdr, 1904 sizeof(grp->il_dependladdr)) == 0) { 1905 break; 1906 } 1907 } 1908 if (grp == NULL) { 1909 /* 1910 * Create a new local group 1911 */ 1912 if (grp_alloc == NULL) { 1913 grp_alloc = inp_localgroup_alloc(inp->inp_af, 1914 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 1915 INP_LOCALGROUP_SIZMIN); 1916 /* 1917 * Local group allocation could block and the 1918 * local group w/ the same property might have 1919 * been added by others when we were blocked; 1920 * check again. 1921 */ 1922 goto again; 1923 } else { 1924 /* Local group has been allocated; link it */ 1925 grp = grp_alloc; 1926 grp_alloc = NULL; 1927 LIST_INSERT_HEAD(hdr, grp, il_list); 1928 } 1929 } else if (grp->il_inpcnt == grp->il_inpsiz) { 1930 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 1931 static int limit_logged = 0; 1932 1933 if (!limit_logged) { 1934 limit_logged = 1; 1935 kprintf("local group port %d, " 1936 "limit reached\n", ntohs(grp->il_lport)); 1937 } 1938 if (grp_alloc != NULL) { 1939 /* 1940 * This would happen if the local group 1941 * w/ the same property was expanded when 1942 * our local group allocation blocked. 1943 */ 1944 inp_localgroup_free(grp_alloc); 1945 } 1946 return; 1947 } 1948 1949 /* 1950 * Expand this local group 1951 */ 1952 if (grp_alloc == NULL || 1953 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 1954 if (grp_alloc != NULL) 1955 inp_localgroup_free(grp_alloc); 1956 grp_alloc = inp_localgroup_alloc(grp->il_af, 1957 grp->il_lport, &grp->il_dependladdr, 1958 grp->il_inpsiz * 2); 1959 /* 1960 * Local group allocation could block and the 1961 * local group w/ the same property might have 1962 * been expanded by others when we were blocked; 1963 * check again. 1964 */ 1965 goto again; 1966 } 1967 1968 /* 1969 * Save the old local group, link the new one, and then 1970 * destroy the old local group 1971 */ 1972 inp_localgroup_copy(grp_alloc, grp); 1973 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 1974 inp_localgroup_destroy(grp); 1975 1976 grp = grp_alloc; 1977 grp_alloc = NULL; 1978 } else { 1979 /* 1980 * Found the local group 1981 */ 1982 if (grp_alloc != NULL) { 1983 /* 1984 * This would happen if the local group w/ the 1985 * same property was added or expanded when our 1986 * local group allocation blocked. 1987 */ 1988 inp_localgroup_free(grp_alloc); 1989 grp_alloc = NULL; 1990 } 1991 } 1992 1993 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 1994 ("invalid local group size %d and count %d", 1995 grp->il_inpsiz, grp->il_inpcnt)); 1996 1997 /* 1998 * Keep the local group sorted by the inpcb local group index 1999 * in ascending order. 2000 * 2001 * This eases the multi-process userland application which uses 2002 * SO_REUSEPORT sockets and binds process to the owner cpu of 2003 * the SO_REUSEPORT socket: 2004 * If we didn't sort the local group by the inpcb local group 2005 * index and one of the process owning an inpcb in this local 2006 * group restarted, e.g. crashed and restarted by watchdog, 2007 * other processes owning a inpcb in this local group would have 2008 * to detect that event, refetch its socket's owner cpu, and 2009 * re-bind. 2010 */ 2011 idx = grp->il_inpcnt; 2012 for (i = 0; i < idx; ++i) { 2013 struct inpcb *oinp = grp->il_inp[i]; 2014 2015 if (oinp->inp_lgrpindex > i) { 2016 if (inp->inp_lgrpindex < 0) { 2017 inp->inp_lgrpindex = i; 2018 } else if (inp->inp_lgrpindex != i) { 2019 if (bootverbose) { 2020 kprintf("inp %p: grpidx %d, " 2021 "assigned to %d, cpu%d\n", 2022 inp, inp->inp_lgrpindex, i, 2023 mycpuid); 2024 } 2025 } 2026 grp->il_inp[i] = inp; 2027 2028 /* Pull down inpcbs */ 2029 for (; i < grp->il_inpcnt; ++i) { 2030 struct inpcb *oinp1 = grp->il_inp[i + 1]; 2031 2032 grp->il_inp[i + 1] = oinp; 2033 oinp = oinp1; 2034 } 2035 grp->il_inpcnt++; 2036 return; 2037 } 2038 } 2039 2040 if (inp->inp_lgrpindex < 0) { 2041 inp->inp_lgrpindex = idx; 2042 } else if (inp->inp_lgrpindex != idx) { 2043 if (bootverbose) { 2044 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 2045 inp, inp->inp_lgrpindex, idx, mycpuid); 2046 } 2047 } 2048 grp->il_inp[idx] = inp; 2049 grp->il_inpcnt++; 2050 } 2051 2052 void 2053 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2054 { 2055 struct inpcontainer *ic; 2056 struct inpcontainerhead *bucket; 2057 2058 GET_PCBINFO_TOKEN(pcbinfo); 2059 2060 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 2061 2062 bucket = &pcbinfo->wildcardhashbase[ 2063 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2064 2065 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 2066 ic->ic_inp = inp; 2067 LIST_INSERT_HEAD(bucket, ic, ic_list); 2068 2069 REL_PCBINFO_TOKEN(pcbinfo); 2070 } 2071 2072 /* 2073 * Insert PCB into wildcard hash table. 2074 */ 2075 void 2076 in_pcbinswildcardhash(struct inpcb *inp) 2077 { 2078 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2079 2080 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2081 ("not in correct netisr")); 2082 ASSERT_INP_NOTINHASH(inp); 2083 inp->inp_flags |= INP_WILDCARD; 2084 2085 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 2086 } 2087 2088 static void 2089 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2090 { 2091 struct inp_localgrphead *hdr; 2092 struct inp_localgroup *grp; 2093 2094 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2095 2096 if (pcbinfo->localgrphashbase == NULL) 2097 return; 2098 2099 hdr = &pcbinfo->localgrphashbase[ 2100 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2101 2102 LIST_FOREACH(grp, hdr, il_list) { 2103 int i; 2104 2105 for (i = 0; i < grp->il_inpcnt; ++i) { 2106 if (grp->il_inp[i] != inp) 2107 continue; 2108 2109 if (grp->il_inpcnt == 1) { 2110 /* Destroy this local group */ 2111 inp_localgroup_destroy(grp); 2112 } else { 2113 /* Pull up inpcbs */ 2114 for (; i + 1 < grp->il_inpcnt; ++i) 2115 grp->il_inp[i] = grp->il_inp[i + 1]; 2116 grp->il_inpcnt--; 2117 } 2118 return; 2119 } 2120 } 2121 } 2122 2123 void 2124 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2125 { 2126 struct inpcontainer *ic; 2127 struct inpcontainerhead *head; 2128 2129 GET_PCBINFO_TOKEN(pcbinfo); 2130 2131 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2132 2133 /* find bucket */ 2134 head = &pcbinfo->wildcardhashbase[ 2135 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2136 2137 LIST_FOREACH(ic, head, ic_list) { 2138 if (ic->ic_inp == inp) 2139 goto found; 2140 } 2141 REL_PCBINFO_TOKEN(pcbinfo); 2142 return; /* not found! */ 2143 2144 found: 2145 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2146 REL_PCBINFO_TOKEN(pcbinfo); 2147 kfree(ic, M_TEMP); /* deallocate container */ 2148 } 2149 2150 /* 2151 * Remove PCB from wildcard hash table. 2152 */ 2153 void 2154 in_pcbremwildcardhash(struct inpcb *inp) 2155 { 2156 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2157 2158 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2159 ("not in correct netisr")); 2160 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2161 2162 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2163 inp->inp_lgrpindex = -1; 2164 inp->inp_flags &= ~INP_WILDCARD; 2165 } 2166 2167 /* 2168 * Remove PCB from various lists. 2169 */ 2170 void 2171 in_pcbremlists(struct inpcb *inp) 2172 { 2173 in_pcbremporthash(inp); 2174 if (inp->inp_flags & INP_WILDCARD) { 2175 in_pcbremwildcardhash(inp); 2176 } else if (inp->inp_flags & INP_CONNECTED) { 2177 in_pcbremconnhash(inp); 2178 } 2179 2180 if (inp->inp_flags & INP_ONLIST) 2181 in_pcbofflist(inp); 2182 } 2183 2184 int 2185 prison_xinpcb(struct thread *td, struct inpcb *inp) 2186 { 2187 struct ucred *cr; 2188 2189 if (td->td_proc == NULL) 2190 return (0); 2191 cr = td->td_proc->p_ucred; 2192 if (cr->cr_prison == NULL) 2193 return (0); 2194 if (inp->inp_socket && inp->inp_socket->so_cred && 2195 inp->inp_socket->so_cred->cr_prison && 2196 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2197 return (0); 2198 return (1); 2199 } 2200 2201 int 2202 in_pcblist_global(SYSCTL_HANDLER_ARGS) 2203 { 2204 struct inpcbinfo *pcbinfo_arr = arg1; 2205 int pcbinfo_arrlen = arg2; 2206 struct inpcb *marker; 2207 int cpu, origcpu; 2208 int error, n; 2209 2210 KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1, 2211 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2212 2213 /* 2214 * The process of preparing the TCB list is too time-consuming and 2215 * resource-intensive to repeat twice on every request. 2216 */ 2217 n = 0; 2218 if (req->oldptr == NULL) { 2219 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2220 n += pcbinfo_arr[cpu].ipi_count; 2221 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2222 return 0; 2223 } 2224 2225 if (req->newptr != NULL) 2226 return EPERM; 2227 2228 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2229 marker->inp_flags |= INP_PLACEMARKER; 2230 2231 /* 2232 * OK, now we're committed to doing something. Re-fetch ipi_count 2233 * after obtaining the generation count. 2234 */ 2235 error = 0; 2236 origcpu = mycpuid; 2237 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2238 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2239 struct inpcb *inp; 2240 struct xinpcb xi; 2241 int i; 2242 2243 lwkt_migratecpu(cpu); 2244 2245 GET_PCBINFO_TOKEN(pcbinfo); 2246 2247 n = pcbinfo->ipi_count; 2248 2249 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2250 i = 0; 2251 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2252 LIST_REMOVE(marker, inp_list); 2253 LIST_INSERT_AFTER(inp, marker, inp_list); 2254 2255 if (inp->inp_flags & INP_PLACEMARKER) 2256 continue; 2257 if (prison_xinpcb(req->td, inp)) 2258 continue; 2259 2260 bzero(&xi, sizeof xi); 2261 xi.xi_len = sizeof xi; 2262 bcopy(inp, &xi.xi_inp, sizeof *inp); 2263 if (inp->inp_socket) 2264 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2265 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2266 break; 2267 ++i; 2268 } 2269 LIST_REMOVE(marker, inp_list); 2270 2271 REL_PCBINFO_TOKEN(pcbinfo); 2272 2273 if (error == 0 && i < n) { 2274 bzero(&xi, sizeof xi); 2275 xi.xi_len = sizeof xi; 2276 while (i < n) { 2277 error = SYSCTL_OUT(req, &xi, sizeof xi); 2278 if (error) 2279 break; 2280 ++i; 2281 } 2282 } 2283 } 2284 2285 lwkt_migratecpu(origcpu); 2286 kfree(marker, M_TEMP); 2287 return error; 2288 } 2289 2290 int 2291 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS) 2292 { 2293 return in_pcblist_global(oidp, arg1, ncpus2, req); 2294 } 2295 2296 void 2297 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2298 { 2299 struct sockaddr_in *sin; 2300 2301 KASSERT(faddr->sa_family == AF_INET, 2302 ("not AF_INET faddr %d", faddr->sa_family)); 2303 2304 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2305 sin->sin_family = AF_INET; 2306 sin->sin_len = sizeof(*sin); 2307 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2308 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2309 2310 so->so_faddr = (struct sockaddr *)sin; 2311 } 2312 2313 void 2314 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2315 u_short offset) 2316 { 2317 memset(portinfo, 0, sizeof(*portinfo)); 2318 2319 portinfo->offset = offset; 2320 portinfo->lastport = offset; 2321 portinfo->lastlow = offset; 2322 portinfo->lasthi = offset; 2323 2324 portinfo->porthashbase = hashinit(hashsize, M_PCB, 2325 &portinfo->porthashmask); 2326 } 2327 2328 void 2329 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2330 { 2331 int hi, lo; 2332 2333 if (step == 1) 2334 return; 2335 2336 hi = *hi0; 2337 lo = *lo0; 2338 2339 hi = rounddown2(hi, step); 2340 hi += ofs; 2341 if (hi > (int)*hi0) 2342 hi -= step; 2343 2344 lo = roundup2(lo, step); 2345 lo -= (step - ofs); 2346 if (lo < (int)*lo0) 2347 lo += step; 2348 2349 *hi0 = hi; 2350 *lo0 = lo; 2351 } 2352 2353 void 2354 in_pcbglobalinit(void) 2355 { 2356 int cpu; 2357 2358 in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB, 2359 M_WAITOK | M_ZERO); 2360 in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer), 2361 M_PCB, M_WAITOK | M_ZERO); 2362 2363 for (cpu = 0; cpu < ncpus; ++cpu) { 2364 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2365 struct inpcb *marker = &in_pcbmarkers[cpu]; 2366 2367 marker->inp_flags |= INP_PLACEMARKER; 2368 ic->ic_inp = marker; 2369 } 2370 } 2371 2372 struct inpcb * 2373 in_pcbmarker(int cpuid) 2374 { 2375 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid)); 2376 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 2377 2378 return &in_pcbmarkers[cpuid]; 2379 } 2380 2381 struct inpcontainer * 2382 in_pcbcontainer_marker(int cpuid) 2383 { 2384 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid)); 2385 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 2386 2387 return &in_pcbcontainer_markers[cpuid]; 2388 } 2389 2390 void 2391 in_pcbresetroute(struct inpcb *inp) 2392 { 2393 struct route *ro = &inp->inp_route; 2394 2395 if (ro->ro_rt != NULL) 2396 RTFREE(ro->ro_rt); 2397 bzero(ro, sizeof(*ro)); 2398 } 2399 2400 u_short 2401 in_pcblastport_down(volatile u_short *lastport, u_short first, u_short last, 2402 u_short step) 2403 { 2404 u_short lport; 2405 2406 for (;;) { 2407 u_short olport; 2408 2409 olport = *lastport; 2410 lport = olport - step; 2411 if (__predict_false(lport > first || lport < last)) 2412 lport = first; 2413 if (atomic_cmpset_short(lastport, olport, lport)) 2414 break; 2415 } 2416 return lport; 2417 } 2418 2419 u_short 2420 in_pcblastport_up(volatile u_short *lastport, u_short first, u_short last, 2421 u_short step) 2422 { 2423 u_short lport; 2424 2425 for (;;) { 2426 u_short olport; 2427 2428 olport = *lastport; 2429 lport = olport + step; 2430 if (__predict_false(lport < first || lport > last)) 2431 lport = first; 2432 if (atomic_cmpset_short(lastport, olport, lport)) 2433 break; 2434 } 2435 return lport; 2436 } 2437