1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/domain.h> 74 #include <sys/protosw.h> 75 #include <sys/socket.h> 76 #include <sys/socketvar.h> 77 #include <sys/proc.h> 78 #include <sys/priv.h> 79 #include <sys/jail.h> 80 #include <sys/kernel.h> 81 #include <sys/sysctl.h> 82 83 #include <sys/thread2.h> 84 #include <sys/socketvar2.h> 85 #include <sys/msgport2.h> 86 87 #include <machine/limits.h> 88 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/route.h> 92 #include <net/netisr2.h> 93 94 #include <netinet/in.h> 95 #include <netinet/in_pcb.h> 96 #include <netinet/in_var.h> 97 #include <netinet/ip_var.h> 98 #ifdef INET6 99 #include <netinet/ip6.h> 100 #include <netinet6/ip6_var.h> 101 #endif /* INET6 */ 102 103 #ifdef IPSEC 104 #include <netinet6/ipsec.h> 105 #include <netproto/key/key.h> 106 #include <netproto/ipsec/esp_var.h> 107 #endif 108 109 #ifdef FAST_IPSEC 110 #if defined(IPSEC) || defined(IPSEC_ESP) 111 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!" 112 #endif 113 114 #include <netproto/ipsec/ipsec.h> 115 #include <netproto/ipsec/key.h> 116 #define IPSEC 117 #endif /* FAST_IPSEC */ 118 119 #define INP_LOCALGROUP_SIZMIN 8 120 #define INP_LOCALGROUP_SIZMAX 256 121 122 struct in_addr zeroin_addr; 123 124 /* 125 * These configure the range of local port addresses assigned to 126 * "unspecified" outgoing connections/packets/whatever. 127 */ 128 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 129 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 130 131 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 132 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 133 134 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 135 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 136 137 #define RANGECHK(var, min, max) \ 138 if ((var) < (min)) { (var) = (min); } \ 139 else if ((var) > (max)) { (var) = (max); } 140 141 int udpencap_enable = 1; /* enabled by default */ 142 int udpencap_port = 4500; /* triggers decapsulation */ 143 144 /* 145 * Per-netisr inpcb markers. 146 * NOTE: they should only be used in netisrs. 147 */ 148 static struct inpcb *in_pcbmarkers; 149 static struct inpcontainer *in_pcbcontainer_markers; 150 151 static int 152 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 153 { 154 int error; 155 156 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 157 if (!error) { 158 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 159 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 160 161 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 162 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 163 164 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 165 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 166 } 167 return (error); 168 } 169 170 #undef RANGECHK 171 172 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 173 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 175 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 176 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 177 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 179 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 180 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 181 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 183 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 184 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 185 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 186 187 /* 188 * in_pcb.c: manage the Protocol Control Blocks. 189 * 190 * NOTE: It is assumed that most of these functions will be called from 191 * a critical section. XXX - There are, unfortunately, a few exceptions 192 * to this rule that should be fixed. 193 * 194 * NOTE: The caller should initialize the cpu field to the cpu running the 195 * protocol stack associated with this inpcbinfo. 196 */ 197 198 void 199 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 200 { 201 KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu)); 202 pcbinfo->cpu = cpu; 203 204 LIST_INIT(&pcbinfo->pcblisthead); 205 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 206 M_WAITOK | M_ZERO); 207 208 if (shared) { 209 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 210 M_PCB, M_WAITOK); 211 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 212 } else { 213 pcbinfo->infotoken = NULL; 214 } 215 } 216 217 struct baddynamicports baddynamicports; 218 219 /* 220 * Check if the specified port is invalid for dynamic allocation. 221 */ 222 int 223 in_baddynamic(u_int16_t port, u_int16_t proto) 224 { 225 switch (proto) { 226 case IPPROTO_TCP: 227 return (DP_ISSET(baddynamicports.tcp, port)); 228 case IPPROTO_UDP: 229 #ifdef IPSEC 230 /* Cannot preset this as it is a sysctl */ 231 if (port == udpencap_port) 232 return (1); 233 #endif 234 return (DP_ISSET(baddynamicports.udp, port)); 235 default: 236 return (0); 237 } 238 } 239 240 void 241 in_pcbonlist(struct inpcb *inp) 242 { 243 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 244 245 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 246 ("not in the correct netisr")); 247 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 248 inp->inp_flags |= INP_ONLIST; 249 250 GET_PCBINFO_TOKEN(pcbinfo); 251 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 252 pcbinfo->ipi_count++; 253 REL_PCBINFO_TOKEN(pcbinfo); 254 } 255 256 void 257 in_pcbofflist(struct inpcb *inp) 258 { 259 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 260 261 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 262 ("not in the correct netisr")); 263 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 264 inp->inp_flags &= ~INP_ONLIST; 265 266 GET_PCBINFO_TOKEN(pcbinfo); 267 LIST_REMOVE(inp, inp_list); 268 KASSERT(pcbinfo->ipi_count > 0, 269 ("invalid inpcb count %d", pcbinfo->ipi_count)); 270 pcbinfo->ipi_count--; 271 REL_PCBINFO_TOKEN(pcbinfo); 272 } 273 274 /* 275 * Allocate a PCB and associate it with the socket. 276 */ 277 int 278 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 279 { 280 struct inpcb *inp; 281 #ifdef IPSEC 282 int error; 283 #endif 284 285 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 286 if (inp == NULL) 287 return (ENOMEM); 288 inp->inp_lgrpindex = -1; 289 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 290 inp->inp_pcbinfo = pcbinfo; 291 inp->inp_socket = so; 292 #ifdef IPSEC 293 error = ipsec_init_policy(so, &inp->inp_sp); 294 if (error != 0) { 295 kfree(inp, M_PCB); 296 return (error); 297 } 298 #endif 299 #ifdef INET6 300 if (INP_SOCKAF(so) == AF_INET6 && ip6_v6only) 301 inp->inp_flags |= IN6P_IPV6_V6ONLY; 302 if (ip6_auto_flowlabel) 303 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 304 #endif 305 soreference(so); 306 so->so_pcb = inp; 307 308 in_pcbonlist(inp); 309 return (0); 310 } 311 312 /* 313 * Unlink a pcb with the intention of moving it to another cpu with a 314 * different pcbinfo. While unlinked nothing should attempt to dereference 315 * inp_pcbinfo, NULL it out so we assert if it does. 316 */ 317 void 318 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 319 { 320 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 321 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 322 ("already linked")); 323 324 in_pcbofflist(inp); 325 inp->inp_pcbinfo = NULL; 326 } 327 328 void 329 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 330 { 331 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 332 } 333 334 /* 335 * Relink a pcb into a new pcbinfo. 336 */ 337 void 338 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 339 { 340 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 341 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 342 ("already linked")); 343 344 inp->inp_pcbinfo = pcbinfo; 345 in_pcbonlist(inp); 346 } 347 348 void 349 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 350 { 351 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 352 } 353 354 static int 355 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 356 { 357 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 358 struct inpcbportinfo *portinfo; 359 u_short first, last, lport, step; 360 u_short *lastport; 361 int count, error; 362 int portinfo_first, portinfo_idx; 363 364 inp->inp_flags |= INP_ANONPORT; 365 366 step = pcbinfo->portinfo_mask + 1; 367 portinfo_first = mycpuid & pcbinfo->portinfo_mask; 368 portinfo_idx = portinfo_first; 369 loop: 370 portinfo = &pcbinfo->portinfo[portinfo_idx]; 371 372 if (inp->inp_flags & INP_HIGHPORT) { 373 first = ipport_hifirstauto; /* sysctl */ 374 last = ipport_hilastauto; 375 lastport = &portinfo->lasthi; 376 } else if (inp->inp_flags & INP_LOWPORT) { 377 if (cred && 378 (error = 379 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 380 inp->inp_laddr.s_addr = INADDR_ANY; 381 return error; 382 } 383 first = ipport_lowfirstauto; /* 1023 */ 384 last = ipport_lowlastauto; /* 600 */ 385 lastport = &portinfo->lastlow; 386 } else { 387 first = ipport_firstauto; /* sysctl */ 388 last = ipport_lastauto; 389 lastport = &portinfo->lastport; 390 } 391 392 /* 393 * This has to be atomic. If the porthash is shared across multiple 394 * protocol threads (aka tcp) then the token must be held. 395 */ 396 GET_PORT_TOKEN(portinfo); 397 398 /* 399 * Simple check to ensure all ports are not used up causing 400 * a deadlock here. 401 * 402 * We split the two cases (up and down) so that the direction 403 * is not being tested on each round of the loop. 404 */ 405 if (first > last) { 406 /* 407 * counting down 408 */ 409 in_pcbportrange(&first, &last, portinfo->offset, step); 410 count = (first - last) / step; 411 412 do { 413 if (count-- < 0) { /* completely used? */ 414 error = EADDRNOTAVAIL; 415 goto done; 416 } 417 *lastport -= step; 418 if (*lastport > first || *lastport < last) 419 *lastport = first; 420 KKASSERT((*lastport & pcbinfo->portinfo_mask) == 421 portinfo->offset); 422 lport = htons(*lastport); 423 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport, 424 wild, cred)); 425 } else { 426 /* 427 * counting up 428 */ 429 in_pcbportrange(&last, &first, portinfo->offset, step); 430 count = (last - first) / step; 431 432 do { 433 if (count-- < 0) { /* completely used? */ 434 error = EADDRNOTAVAIL; 435 goto done; 436 } 437 *lastport += step; 438 if (*lastport < first || *lastport > last) 439 *lastport = first; 440 KKASSERT((*lastport & pcbinfo->portinfo_mask) == 441 portinfo->offset); 442 lport = htons(*lastport); 443 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport, 444 wild, cred)); 445 } 446 inp->inp_lport = lport; 447 in_pcbinsporthash(portinfo, inp); 448 error = 0; 449 done: 450 REL_PORT_TOKEN(portinfo); 451 452 if (error) { 453 /* Try next portinfo */ 454 portinfo_idx++; 455 portinfo_idx &= pcbinfo->portinfo_mask; 456 if (portinfo_idx != portinfo_first) 457 goto loop; 458 inp->inp_laddr.s_addr = INADDR_ANY; 459 } 460 return error; 461 } 462 463 int 464 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 465 { 466 struct socket *so = inp->inp_socket; 467 struct sockaddr_in jsin; 468 struct ucred *cred = NULL; 469 int wild = 0; 470 471 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 472 return (EADDRNOTAVAIL); 473 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 474 return (EINVAL); /* already bound */ 475 476 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 477 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 478 if (td->td_proc) 479 cred = td->td_proc->p_ucred; 480 481 if (nam != NULL) { 482 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 483 struct inpcbinfo *pcbinfo; 484 struct inpcbportinfo *portinfo; 485 struct inpcb *t; 486 u_short lport, lport_ho; 487 int reuseport = (so->so_options & SO_REUSEPORT); 488 int error; 489 490 if (nam->sa_len != sizeof *sin) 491 return (EINVAL); 492 #ifdef notdef 493 /* 494 * We should check the family, but old programs 495 * incorrectly fail to initialize it. 496 */ 497 if (sin->sin_family != AF_INET) 498 return (EAFNOSUPPORT); 499 #endif 500 if (!prison_replace_wildcards(td, nam)) 501 return (EINVAL); 502 503 lport = sin->sin_port; 504 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 505 /* 506 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 507 * allow complete duplication of binding if 508 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 509 * and a multicast address is bound on both 510 * new and duplicated sockets. 511 */ 512 if (so->so_options & SO_REUSEADDR) 513 reuseport = SO_REUSEADDR | SO_REUSEPORT; 514 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 515 sin->sin_port = 0; /* yech... */ 516 bzero(&sin->sin_zero, sizeof sin->sin_zero); 517 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 518 return (EADDRNOTAVAIL); 519 } 520 521 inp->inp_laddr = sin->sin_addr; 522 523 jsin.sin_family = AF_INET; 524 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 525 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 526 inp->inp_laddr.s_addr = INADDR_ANY; 527 return (EINVAL); 528 } 529 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 530 531 if (lport == 0) { 532 /* Auto-select local port */ 533 return in_pcbsetlport(inp, wild, cred); 534 } 535 lport_ho = ntohs(lport); 536 537 /* GROSS */ 538 if (lport_ho < IPPORT_RESERVED && cred && 539 (error = 540 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 541 inp->inp_laddr.s_addr = INADDR_ANY; 542 return (error); 543 } 544 545 /* 546 * Locate the proper portinfo based on lport 547 */ 548 pcbinfo = inp->inp_pcbinfo; 549 portinfo = 550 &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask]; 551 KKASSERT((lport_ho & pcbinfo->portinfo_mask) == 552 portinfo->offset); 553 554 /* 555 * This has to be atomic. If the porthash is shared across 556 * multiple protocol threads (aka tcp) then the token must 557 * be held. 558 */ 559 GET_PORT_TOKEN(portinfo); 560 561 if (so->so_cred->cr_uid != 0 && 562 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 563 t = in_pcblookup_local(portinfo, sin->sin_addr, lport, 564 INPLOOKUP_WILDCARD, cred); 565 if (t && 566 (!in_nullhost(sin->sin_addr) || 567 !in_nullhost(t->inp_laddr) || 568 (t->inp_socket->so_options & SO_REUSEPORT) == 0) && 569 (so->so_cred->cr_uid != 570 t->inp_socket->so_cred->cr_uid)) { 571 #ifdef INET6 572 if (!in_nullhost(sin->sin_addr) || 573 !in_nullhost(t->inp_laddr) || 574 INP_SOCKAF(so) == INP_SOCKAF(t->inp_socket)) 575 #endif 576 { 577 inp->inp_laddr.s_addr = INADDR_ANY; 578 error = EADDRINUSE; 579 goto done; 580 } 581 } 582 } 583 if (cred && !prison_replace_wildcards(td, nam)) { 584 inp->inp_laddr.s_addr = INADDR_ANY; 585 error = EADDRNOTAVAIL; 586 goto done; 587 } 588 t = in_pcblookup_local(portinfo, sin->sin_addr, lport, 589 wild, cred); 590 if (t && !(reuseport & t->inp_socket->so_options)) { 591 #ifdef INET6 592 if (!in_nullhost(sin->sin_addr) || 593 !in_nullhost(t->inp_laddr) || 594 INP_SOCKAF(so) == INP_SOCKAF(t->inp_socket)) 595 #endif 596 { 597 inp->inp_laddr.s_addr = INADDR_ANY; 598 error = EADDRINUSE; 599 goto done; 600 } 601 } 602 inp->inp_lport = lport; 603 in_pcbinsporthash(portinfo, inp); 604 error = 0; 605 done: 606 REL_PORT_TOKEN(portinfo); 607 return (error); 608 } else { 609 jsin.sin_family = AF_INET; 610 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 611 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 612 inp->inp_laddr.s_addr = INADDR_ANY; 613 return (EINVAL); 614 } 615 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 616 617 return in_pcbsetlport(inp, wild, cred); 618 } 619 } 620 621 static struct inpcb * 622 in_pcblookup_localremote(struct inpcbportinfo *portinfo, struct in_addr laddr, 623 u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred) 624 { 625 struct inpcb *inp; 626 struct inpcbporthead *porthash; 627 struct inpcbport *phd; 628 struct inpcb *match = NULL; 629 630 /* 631 * If the porthashbase is shared across several cpus, it must 632 * have been locked. 633 */ 634 ASSERT_PORT_TOKEN_HELD(portinfo); 635 636 /* 637 * Best fit PCB lookup. 638 * 639 * First see if this local port is in use by looking on the 640 * port hash list. 641 */ 642 porthash = &portinfo->porthashbase[ 643 INP_PCBPORTHASH(lport, portinfo->porthashmask)]; 644 LIST_FOREACH(phd, porthash, phd_hash) { 645 if (phd->phd_port == lport) 646 break; 647 } 648 if (phd != NULL) { 649 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 650 #ifdef INET6 651 if ((inp->inp_vflag & INP_IPV4) == 0) 652 continue; 653 #endif 654 if (inp->inp_laddr.s_addr != INADDR_ANY && 655 inp->inp_laddr.s_addr != laddr.s_addr) 656 continue; 657 658 if (inp->inp_faddr.s_addr != INADDR_ANY && 659 inp->inp_faddr.s_addr != faddr.s_addr) 660 continue; 661 662 if (inp->inp_fport != 0 && inp->inp_fport != fport) 663 continue; 664 665 if (cred == NULL || 666 cred->cr_prison == 667 inp->inp_socket->so_cred->cr_prison) { 668 match = inp; 669 break; 670 } 671 } 672 } 673 return (match); 674 } 675 676 int 677 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 678 struct thread *td) 679 { 680 struct proc *p = td->td_proc; 681 unsigned short *lastport; 682 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 683 struct sockaddr_in jsin; 684 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 685 struct inpcbportinfo *portinfo; 686 struct ucred *cred = NULL; 687 u_short first, last, lport, step; 688 int count, error, dup; 689 int portinfo_first, portinfo_idx; 690 691 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 692 return (EADDRNOTAVAIL); 693 694 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 695 if (inp->inp_lport != 0) 696 return (EINVAL); /* already bound */ 697 698 KKASSERT(p); 699 cred = p->p_ucred; 700 701 jsin.sin_family = AF_INET; 702 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 703 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 704 inp->inp_laddr.s_addr = INADDR_ANY; 705 return (EINVAL); 706 } 707 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 708 709 inp->inp_flags |= INP_ANONPORT; 710 711 step = pcbinfo->portinfo_mask + 1; 712 portinfo_first = mycpuid & pcbinfo->portinfo_mask; 713 portinfo_idx = portinfo_first; 714 loop: 715 portinfo = &pcbinfo->portinfo[portinfo_idx]; 716 dup = 0; 717 718 if (inp->inp_flags & INP_HIGHPORT) { 719 first = ipport_hifirstauto; /* sysctl */ 720 last = ipport_hilastauto; 721 lastport = &portinfo->lasthi; 722 } else if (inp->inp_flags & INP_LOWPORT) { 723 if (cred && 724 (error = 725 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 726 inp->inp_laddr.s_addr = INADDR_ANY; 727 return (error); 728 } 729 first = ipport_lowfirstauto; /* 1023 */ 730 last = ipport_lowlastauto; /* 600 */ 731 lastport = &portinfo->lastlow; 732 } else { 733 first = ipport_firstauto; /* sysctl */ 734 last = ipport_lastauto; 735 lastport = &portinfo->lastport; 736 } 737 738 /* 739 * This has to be atomic. If the porthash is shared across multiple 740 * protocol threads (aka tcp) then the token must be held. 741 */ 742 GET_PORT_TOKEN(portinfo); 743 744 again: 745 /* 746 * Simple check to ensure all ports are not used up causing 747 * a deadlock here. 748 * 749 * We split the two cases (up and down) so that the direction 750 * is not being tested on each round of the loop. 751 */ 752 if (first > last) { 753 /* 754 * counting down 755 */ 756 in_pcbportrange(&first, &last, portinfo->offset, step); 757 count = (first - last) / step; 758 759 do { 760 if (count-- < 0) { /* completely used? */ 761 error = EADDRNOTAVAIL; 762 goto done; 763 } 764 *lastport -= step; 765 if (*lastport > first || *lastport < last) 766 *lastport = first; 767 KKASSERT((*lastport & pcbinfo->portinfo_mask) == 768 portinfo->offset); 769 lport = htons(*lastport); 770 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr, 771 lport, sin->sin_addr, sin->sin_port, cred)); 772 } else { 773 /* 774 * counting up 775 */ 776 in_pcbportrange(&last, &first, portinfo->offset, step); 777 count = (last - first) / step; 778 779 do { 780 if (count-- < 0) { /* completely used? */ 781 error = EADDRNOTAVAIL; 782 goto done; 783 } 784 *lastport += step; 785 if (*lastport < first || *lastport > last) 786 *lastport = first; 787 KKASSERT((*lastport & pcbinfo->portinfo_mask) == 788 portinfo->offset); 789 lport = htons(*lastport); 790 } while (in_pcblookup_localremote(portinfo, inp->inp_laddr, 791 lport, sin->sin_addr, sin->sin_port, cred)); 792 } 793 794 /* This could happen on loopback interface */ 795 if (sin->sin_port == lport && 796 sin->sin_addr.s_addr == inp->inp_laddr.s_addr) { 797 if (dup) { 798 /* 799 * Duplicate again; give up 800 */ 801 error = EADDRNOTAVAIL; 802 goto done; 803 } 804 dup = 1; 805 goto again; 806 } 807 inp->inp_lport = lport; 808 in_pcbinsporthash(portinfo, inp); 809 error = 0; 810 done: 811 REL_PORT_TOKEN(portinfo); 812 813 if (error) { 814 /* Try next portinfo */ 815 portinfo_idx++; 816 portinfo_idx &= pcbinfo->portinfo_mask; 817 if (portinfo_idx != portinfo_first) 818 goto loop; 819 inp->inp_laddr.s_addr = INADDR_ANY; 820 } 821 return error; 822 } 823 824 /* 825 * Transform old in_pcbconnect() into an inner subroutine for new 826 * in_pcbconnect(): Do some validity-checking on the remote 827 * address (in mbuf 'nam') and then determine local host address 828 * (i.e., which interface) to use to access that remote host. 829 * 830 * This preserves definition of in_pcbconnect(), while supporting a 831 * slightly different version for T/TCP. (This is more than 832 * a bit of a kludge, but cleaning up the internal interfaces would 833 * have forced minor changes in every protocol). 834 */ 835 int 836 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 837 struct sockaddr_in **plocal_sin, struct thread *td, int find) 838 { 839 struct in_ifaddr *ia; 840 struct ucred *cred = NULL; 841 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 842 struct sockaddr *jsin; 843 int jailed = 0, alloc_route = 0; 844 845 if (nam->sa_len != sizeof *sin) 846 return (EINVAL); 847 if (sin->sin_family != AF_INET) 848 return (EAFNOSUPPORT); 849 if (sin->sin_port == 0) 850 return (EADDRNOTAVAIL); 851 if (td && td->td_proc && td->td_proc->p_ucred) 852 cred = td->td_proc->p_ucred; 853 if (cred && cred->cr_prison) 854 jailed = 1; 855 if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) { 856 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 857 /* 858 * If the destination address is INADDR_ANY, 859 * use the primary local address. 860 * If the supplied address is INADDR_BROADCAST, 861 * and the primary interface supports broadcast, 862 * choose the broadcast address for that interface. 863 */ 864 if (sin->sin_addr.s_addr == INADDR_ANY) 865 sin->sin_addr = IA_SIN(ia)->sin_addr; 866 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST && 867 (ia->ia_ifp->if_flags & IFF_BROADCAST)) 868 sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr; 869 } 870 if (find) { 871 struct route *ro; 872 873 ia = NULL; 874 /* 875 * If route is known or can be allocated now, 876 * our src addr is taken from the i/f, else punt. 877 * Note that we should check the address family of the cached 878 * destination, in case of sharing the cache with IPv6. 879 */ 880 ro = &inp->inp_route; 881 if (ro->ro_rt && 882 (!(ro->ro_rt->rt_flags & RTF_UP) || 883 ro->ro_dst.sa_family != AF_INET || 884 satosin(&ro->ro_dst)->sin_addr.s_addr != 885 sin->sin_addr.s_addr || 886 inp->inp_socket->so_options & SO_DONTROUTE)) { 887 RTFREE(ro->ro_rt); 888 ro->ro_rt = NULL; 889 } 890 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 891 (ro->ro_rt == NULL || 892 ro->ro_rt->rt_ifp == NULL)) { 893 /* No route yet, so try to acquire one */ 894 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 895 ro->ro_dst.sa_family = AF_INET; 896 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 897 ((struct sockaddr_in *) &ro->ro_dst)->sin_addr = 898 sin->sin_addr; 899 rtalloc(ro); 900 alloc_route = 1; 901 } 902 /* 903 * If we found a route, use the address 904 * corresponding to the outgoing interface 905 * unless it is the loopback (in case a route 906 * to our address on another net goes to loopback). 907 */ 908 if (ro->ro_rt && 909 !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) { 910 if (jailed) { 911 if (jailed_ip(cred->cr_prison, 912 ro->ro_rt->rt_ifa->ifa_addr)) { 913 ia = ifatoia(ro->ro_rt->rt_ifa); 914 } 915 } else { 916 ia = ifatoia(ro->ro_rt->rt_ifa); 917 } 918 } 919 if (ia == NULL) { 920 u_short fport = sin->sin_port; 921 922 sin->sin_port = 0; 923 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 924 if (ia && jailed && !jailed_ip(cred->cr_prison, 925 sintosa(&ia->ia_addr))) 926 ia = NULL; 927 if (ia == NULL) 928 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 929 if (ia && jailed && !jailed_ip(cred->cr_prison, 930 sintosa(&ia->ia_addr))) 931 ia = NULL; 932 sin->sin_port = fport; 933 if (ia == NULL && 934 !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 935 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 936 if (ia && jailed && !jailed_ip(cred->cr_prison, 937 sintosa(&ia->ia_addr))) 938 ia = NULL; 939 940 if (!jailed && ia == NULL) 941 goto fail; 942 } 943 /* 944 * If the destination address is multicast and an outgoing 945 * interface has been set as a multicast option, use the 946 * address of that interface as our source address. 947 */ 948 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 949 inp->inp_moptions != NULL) { 950 struct ip_moptions *imo; 951 struct ifnet *ifp; 952 953 imo = inp->inp_moptions; 954 if (imo->imo_multicast_ifp != NULL) { 955 struct in_ifaddr_container *iac; 956 957 ifp = imo->imo_multicast_ifp; 958 ia = NULL; 959 TAILQ_FOREACH(iac, 960 &in_ifaddrheads[mycpuid], ia_link) { 961 if (iac->ia->ia_ifp == ifp) { 962 ia = iac->ia; 963 break; 964 } 965 } 966 if (ia == NULL) 967 goto fail; 968 } 969 } 970 /* 971 * Don't do pcblookup call here; return interface in plocal_sin 972 * and exit to caller, that will do the lookup. 973 */ 974 if (ia == NULL && jailed) { 975 if ((jsin = prison_get_nonlocal( 976 cred->cr_prison, AF_INET, NULL)) != NULL || 977 (jsin = prison_get_local( 978 cred->cr_prison, AF_INET, NULL)) != NULL) { 979 *plocal_sin = satosin(jsin); 980 } else { 981 /* IPv6 only Jail */ 982 goto fail; 983 } 984 } else { 985 *plocal_sin = &ia->ia_addr; 986 } 987 } 988 return (0); 989 fail: 990 if (alloc_route) 991 in_pcbresetroute(inp); 992 return (EADDRNOTAVAIL); 993 } 994 995 int 996 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 997 struct sockaddr_in **plocal_sin, struct thread *td) 998 { 999 return in_pcbladdr_find(inp, nam, plocal_sin, td, 1000 (inp->inp_laddr.s_addr == INADDR_ANY)); 1001 } 1002 1003 /* 1004 * Outer subroutine: 1005 * Connect from a socket to a specified address. 1006 * Both address and port must be specified in argument sin. 1007 * If don't have a local address for this socket yet, 1008 * then pick one. 1009 */ 1010 int 1011 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 1012 { 1013 struct sockaddr_in *if_sin; 1014 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1015 int error; 1016 1017 /* Call inner routine to assign local interface address. */ 1018 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1019 return (error); 1020 1021 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1022 inp->inp_laddr.s_addr ? 1023 inp->inp_laddr : if_sin->sin_addr, 1024 inp->inp_lport, FALSE, NULL) != NULL) { 1025 return (EADDRINUSE); 1026 } 1027 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1028 if (inp->inp_lport == 0) { 1029 error = in_pcbbind(inp, NULL, td); 1030 if (error) 1031 return (error); 1032 } 1033 inp->inp_laddr = if_sin->sin_addr; 1034 } 1035 inp->inp_faddr = sin->sin_addr; 1036 inp->inp_fport = sin->sin_port; 1037 in_pcbinsconnhash(inp); 1038 return (0); 1039 } 1040 1041 void 1042 in_pcbdisconnect(struct inpcb *inp) 1043 { 1044 1045 in_pcbremconnhash(inp); 1046 inp->inp_faddr.s_addr = INADDR_ANY; 1047 inp->inp_fport = 0; 1048 } 1049 1050 void 1051 in_pcbdetach(struct inpcb *inp) 1052 { 1053 struct socket *so = inp->inp_socket; 1054 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1055 1056 #ifdef IPSEC 1057 ipsec4_delete_pcbpolicy(inp); 1058 #endif /*IPSEC*/ 1059 inp->inp_gencnt = ++ipi->ipi_gencnt; 1060 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1061 in_pcbremlists(inp); 1062 so->so_pcb = NULL; 1063 sofree(so); /* remove pcb ref */ 1064 if (inp->inp_options) 1065 m_free(inp->inp_options); 1066 if (inp->inp_route.ro_rt) 1067 rtfree(inp->inp_route.ro_rt); 1068 ip_freemoptions(inp->inp_moptions); 1069 inp->inp_vflag = 0; 1070 kfree(inp, M_PCB); 1071 } 1072 1073 /* 1074 * The calling convention of in_setsockaddr() and in_setpeeraddr() was 1075 * modified to match the pru_sockaddr() and pru_peeraddr() entry points 1076 * in struct pr_usrreqs, so that protocols can just reference then directly 1077 * without the need for a wrapper function. The socket must have a valid 1078 * (i.e., non-nil) PCB, but it should be impossible to get an invalid one 1079 * except through a kernel programming error, so it is acceptable to panic 1080 * (or in this case trap) if the PCB is invalid. (Actually, we don't trap 1081 * because there actually /is/ a programming error somewhere... XXX) 1082 */ 1083 int 1084 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1085 { 1086 struct inpcb *inp; 1087 struct sockaddr_in *sin; 1088 1089 /* 1090 * Do the malloc first in case it blocks. 1091 */ 1092 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1093 sin->sin_family = AF_INET; 1094 sin->sin_len = sizeof *sin; 1095 1096 crit_enter(); 1097 inp = so->so_pcb; 1098 if (!inp) { 1099 crit_exit(); 1100 kfree(sin, M_SONAME); 1101 return (ECONNRESET); 1102 } 1103 sin->sin_port = inp->inp_lport; 1104 sin->sin_addr = inp->inp_laddr; 1105 crit_exit(); 1106 1107 *nam = (struct sockaddr *)sin; 1108 return (0); 1109 } 1110 1111 void 1112 in_setsockaddr_dispatch(netmsg_t msg) 1113 { 1114 int error; 1115 1116 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1117 lwkt_replymsg(&msg->lmsg, error); 1118 } 1119 1120 int 1121 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1122 { 1123 struct inpcb *inp; 1124 struct sockaddr_in *sin; 1125 1126 /* 1127 * Do the malloc first in case it blocks. 1128 */ 1129 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1130 sin->sin_family = AF_INET; 1131 sin->sin_len = sizeof *sin; 1132 1133 crit_enter(); 1134 inp = so->so_pcb; 1135 if (!inp) { 1136 crit_exit(); 1137 kfree(sin, M_SONAME); 1138 return (ECONNRESET); 1139 } 1140 sin->sin_port = inp->inp_fport; 1141 sin->sin_addr = inp->inp_faddr; 1142 crit_exit(); 1143 1144 *nam = (struct sockaddr *)sin; 1145 return (0); 1146 } 1147 1148 void 1149 in_setpeeraddr_dispatch(netmsg_t msg) 1150 { 1151 int error; 1152 1153 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1154 lwkt_replymsg(&msg->lmsg, error); 1155 } 1156 1157 void 1158 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1159 inp_notify_t notify) 1160 { 1161 struct inpcb *inp, *marker; 1162 1163 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1164 ("not in the correct netisr")); 1165 marker = &in_pcbmarkers[mycpuid]; 1166 1167 /* 1168 * NOTE: 1169 * - If INP_PLACEMARKER is set we must ignore the rest of the 1170 * structure and skip it. 1171 * - It is safe to nuke inpcbs here, since we are in their own 1172 * netisr. 1173 */ 1174 GET_PCBINFO_TOKEN(pcbinfo); 1175 1176 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1177 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1178 LIST_REMOVE(marker, inp_list); 1179 LIST_INSERT_AFTER(inp, marker, inp_list); 1180 1181 if (inp->inp_flags & INP_PLACEMARKER) 1182 continue; 1183 #ifdef INET6 1184 if (!(inp->inp_vflag & INP_IPV4)) 1185 continue; 1186 #endif 1187 if (inp->inp_faddr.s_addr != faddr.s_addr || 1188 inp->inp_socket == NULL) 1189 continue; 1190 (*notify)(inp, err); /* can remove inp from list! */ 1191 } 1192 LIST_REMOVE(marker, inp_list); 1193 1194 REL_PCBINFO_TOKEN(pcbinfo); 1195 } 1196 1197 void 1198 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1199 { 1200 struct inpcb *inp, *marker; 1201 1202 /* 1203 * We only need to make sure that we are in netisr0, where all 1204 * multicast operation happen. We could check inpcbinfo which 1205 * does not belong to netisr0 by holding the inpcbinfo's token. 1206 * In this case, the pcbinfo must be able to be shared, i.e. 1207 * pcbinfo->infotoken is not NULL. 1208 */ 1209 KASSERT(&curthread->td_msgport == netisr_cpuport(0), 1210 ("not in netisr0")); 1211 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1212 ("pcbinfo could not be shared")); 1213 1214 /* 1215 * Get a marker for the current netisr (netisr0). 1216 * 1217 * It is possible that the multicast address deletion blocks, 1218 * which could cause temporary token releasing. So we use 1219 * inpcb marker here to get a coherent view of the inpcb list. 1220 * 1221 * While, on the other hand, moptions are only added and deleted 1222 * in netisr0, so we would not see staled moption or miss moption 1223 * even if the token was released due to the blocking multicast 1224 * address deletion. 1225 */ 1226 marker = &in_pcbmarkers[mycpuid]; 1227 1228 GET_PCBINFO_TOKEN(pcbinfo); 1229 1230 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1231 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1232 struct ip_moptions *imo; 1233 1234 LIST_REMOVE(marker, inp_list); 1235 LIST_INSERT_AFTER(inp, marker, inp_list); 1236 1237 if (inp->inp_flags & INP_PLACEMARKER) 1238 continue; 1239 imo = inp->inp_moptions; 1240 if ((inp->inp_vflag & INP_IPV4) && imo != NULL) { 1241 int i, gap; 1242 1243 /* 1244 * Unselect the outgoing interface if it is being 1245 * detached. 1246 */ 1247 if (imo->imo_multicast_ifp == ifp) 1248 imo->imo_multicast_ifp = NULL; 1249 1250 /* 1251 * Drop multicast group membership if we joined 1252 * through the interface being detached. 1253 */ 1254 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1255 i++) { 1256 if (imo->imo_membership[i]->inm_ifp == ifp) { 1257 /* 1258 * NOTE: 1259 * This could block and the pcbinfo 1260 * token could be passively released. 1261 */ 1262 in_delmulti(imo->imo_membership[i]); 1263 gap++; 1264 } else if (gap != 0) 1265 imo->imo_membership[i - gap] = 1266 imo->imo_membership[i]; 1267 } 1268 imo->imo_num_memberships -= gap; 1269 } 1270 } 1271 LIST_REMOVE(marker, inp_list); 1272 1273 REL_PCBINFO_TOKEN(pcbinfo); 1274 } 1275 1276 /* 1277 * Check for alternatives when higher level complains 1278 * about service problems. For now, invalidate cached 1279 * routing information. If the route was created dynamically 1280 * (by a redirect), time to try a default gateway again. 1281 */ 1282 void 1283 in_losing(struct inpcb *inp) 1284 { 1285 struct rtentry *rt; 1286 struct rt_addrinfo rtinfo; 1287 1288 if ((rt = inp->inp_route.ro_rt)) { 1289 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1290 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1291 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1292 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1293 rtinfo.rti_flags = rt->rt_flags; 1294 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1295 if (rt->rt_flags & RTF_DYNAMIC) { 1296 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1297 rt_mask(rt), rt->rt_flags, NULL); 1298 } 1299 inp->inp_route.ro_rt = NULL; 1300 rtfree(rt); 1301 /* 1302 * A new route can be allocated 1303 * the next time output is attempted. 1304 */ 1305 } 1306 } 1307 1308 /* 1309 * After a routing change, flush old routing 1310 * and allocate a (hopefully) better one. 1311 */ 1312 void 1313 in_rtchange(struct inpcb *inp, int err) 1314 { 1315 if (inp->inp_route.ro_rt) { 1316 rtfree(inp->inp_route.ro_rt); 1317 inp->inp_route.ro_rt = NULL; 1318 /* 1319 * A new route can be allocated the next time 1320 * output is attempted. 1321 */ 1322 } 1323 } 1324 1325 /* 1326 * Lookup a PCB based on the local address and port. 1327 */ 1328 struct inpcb * 1329 in_pcblookup_local(struct inpcbportinfo *portinfo, struct in_addr laddr, 1330 u_int lport_arg, int wild_okay, struct ucred *cred) 1331 { 1332 struct inpcb *inp; 1333 int matchwild = 3, wildcard; 1334 u_short lport = lport_arg; 1335 struct inpcbporthead *porthash; 1336 struct inpcbport *phd; 1337 struct inpcb *match = NULL; 1338 1339 /* 1340 * If the porthashbase is shared across several cpus, it must 1341 * have been locked. 1342 */ 1343 ASSERT_PORT_TOKEN_HELD(portinfo); 1344 1345 /* 1346 * Best fit PCB lookup. 1347 * 1348 * First see if this local port is in use by looking on the 1349 * port hash list. 1350 */ 1351 porthash = &portinfo->porthashbase[ 1352 INP_PCBPORTHASH(lport, portinfo->porthashmask)]; 1353 LIST_FOREACH(phd, porthash, phd_hash) { 1354 if (phd->phd_port == lport) 1355 break; 1356 } 1357 if (phd != NULL) { 1358 /* 1359 * Port is in use by one or more PCBs. Look for best 1360 * fit. 1361 */ 1362 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1363 wildcard = 0; 1364 #ifdef INET6 1365 if ((inp->inp_vflag & INP_IPV4) == 0) 1366 continue; 1367 #endif 1368 if (inp->inp_faddr.s_addr != INADDR_ANY) 1369 wildcard++; 1370 if (inp->inp_laddr.s_addr != INADDR_ANY) { 1371 if (laddr.s_addr == INADDR_ANY) 1372 wildcard++; 1373 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1374 continue; 1375 } else { 1376 if (laddr.s_addr != INADDR_ANY) 1377 wildcard++; 1378 } 1379 if (wildcard && !wild_okay) 1380 continue; 1381 if (wildcard < matchwild && 1382 (cred == NULL || 1383 cred->cr_prison == 1384 inp->inp_socket->so_cred->cr_prison)) { 1385 match = inp; 1386 matchwild = wildcard; 1387 if (matchwild == 0) { 1388 break; 1389 } 1390 } 1391 } 1392 } 1393 return (match); 1394 } 1395 1396 struct inpcb * 1397 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1398 const struct inpcb *inp) 1399 { 1400 const struct inp_localgrphead *hdr; 1401 const struct inp_localgroup *grp; 1402 int i; 1403 1404 if (pcbinfo->localgrphashbase == NULL) 1405 return NULL; 1406 1407 GET_PCBINFO_TOKEN(pcbinfo); 1408 1409 hdr = &pcbinfo->localgrphashbase[ 1410 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1411 1412 LIST_FOREACH(grp, hdr, il_list) { 1413 if (grp->il_vflag == inp->inp_vflag && 1414 grp->il_lport == inp->inp_lport && 1415 memcmp(&grp->il_dependladdr, 1416 &inp->inp_inc.inc_ie.ie_dependladdr, 1417 sizeof(grp->il_dependladdr)) == 0) { 1418 break; 1419 } 1420 } 1421 if (grp == NULL || grp->il_inpcnt == 1) { 1422 REL_PCBINFO_TOKEN(pcbinfo); 1423 return NULL; 1424 } 1425 1426 KASSERT(grp->il_inpcnt >= 2, 1427 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1428 for (i = 0; i < grp->il_inpcnt; ++i) { 1429 if (grp->il_inp[i] == inp) { 1430 int last = grp->il_inpcnt - 1; 1431 1432 if (i == last) 1433 last = grp->il_inpcnt - 2; 1434 REL_PCBINFO_TOKEN(pcbinfo); 1435 return grp->il_inp[last]; 1436 } 1437 } 1438 REL_PCBINFO_TOKEN(pcbinfo); 1439 return NULL; 1440 } 1441 1442 static struct inpcb * 1443 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1444 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1445 { 1446 struct inpcb *local_wild = NULL; 1447 const struct inp_localgrphead *hdr; 1448 const struct inp_localgroup *grp; 1449 1450 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1451 1452 hdr = &pcbinfo->localgrphashbase[ 1453 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1454 1455 /* 1456 * Order of socket selection: 1457 * 1. non-wild. 1458 * 2. wild. 1459 * 1460 * NOTE: 1461 * - Local group does not contain jailed sockets 1462 * - Local group does not contain IPv4 mapped INET6 wild sockets 1463 */ 1464 LIST_FOREACH(grp, hdr, il_list) { 1465 #ifdef INET6 1466 if (!(grp->il_vflag & INP_IPV4)) 1467 continue; 1468 #endif 1469 if (grp->il_lport == lport) { 1470 int idx; 1471 1472 /* 1473 * Modulo-N is used here, which greatly reduces 1474 * completion queue token contention, thus more 1475 * cpu time is saved. 1476 */ 1477 idx = pkt_hash % grp->il_inpcnt; 1478 if (grp->il_laddr.s_addr == laddr.s_addr) 1479 return grp->il_inp[idx]; 1480 else if (grp->il_laddr.s_addr == INADDR_ANY) 1481 local_wild = grp->il_inp[idx]; 1482 } 1483 } 1484 if (local_wild != NULL) 1485 return local_wild; 1486 return NULL; 1487 } 1488 1489 /* 1490 * Lookup PCB in hash list. 1491 */ 1492 struct inpcb * 1493 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1494 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1495 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1496 { 1497 struct inpcbhead *head; 1498 struct inpcb *inp, *jinp=NULL; 1499 u_short fport = fport_arg, lport = lport_arg; 1500 1501 /* 1502 * First look for an exact match. 1503 */ 1504 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1505 laddr.s_addr, lport, pcbinfo->hashmask)]; 1506 LIST_FOREACH(inp, head, inp_hash) { 1507 #ifdef INET6 1508 if (!(inp->inp_vflag & INP_IPV4)) 1509 continue; 1510 #endif 1511 if (in_hosteq(inp->inp_faddr, faddr) && 1512 in_hosteq(inp->inp_laddr, laddr) && 1513 inp->inp_fport == fport && inp->inp_lport == lport) { 1514 /* found */ 1515 if (inp->inp_socket == NULL || 1516 inp->inp_socket->so_cred->cr_prison == NULL) { 1517 return (inp); 1518 } else { 1519 if (jinp == NULL) 1520 jinp = inp; 1521 } 1522 } 1523 } 1524 if (jinp != NULL) 1525 return (jinp); 1526 1527 if (wildcard) { 1528 struct inpcb *local_wild = NULL; 1529 struct inpcb *jinp_wild = NULL; 1530 #ifdef INET6 1531 struct inpcb *local_wild_mapped = NULL; 1532 #endif 1533 struct inpcontainer *ic; 1534 struct inpcontainerhead *chead; 1535 struct sockaddr_in jsin; 1536 struct ucred *cred; 1537 1538 GET_PCBINFO_TOKEN(pcbinfo); 1539 1540 /* 1541 * Check local group first 1542 */ 1543 if (pcbinfo->localgrphashbase != NULL && 1544 m != NULL && (m->m_flags & M_HASH) && 1545 !(ifp && ifp->if_type == IFT_FAITH)) { 1546 inp = inp_localgroup_lookup(pcbinfo, 1547 laddr, lport, m->m_pkthdr.hash); 1548 if (inp != NULL) { 1549 REL_PCBINFO_TOKEN(pcbinfo); 1550 return inp; 1551 } 1552 } 1553 1554 /* 1555 * Order of socket selection: 1556 * 1. non-jailed, non-wild. 1557 * 2. non-jailed, wild. 1558 * 3. jailed, non-wild. 1559 * 4. jailed, wild. 1560 */ 1561 jsin.sin_family = AF_INET; 1562 chead = &pcbinfo->wildcardhashbase[ 1563 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1564 LIST_FOREACH(ic, chead, ic_list) { 1565 inp = ic->ic_inp; 1566 if (inp->inp_flags & INP_PLACEMARKER) 1567 continue; 1568 1569 jsin.sin_addr.s_addr = laddr.s_addr; 1570 #ifdef INET6 1571 if (!(inp->inp_vflag & INP_IPV4)) 1572 continue; 1573 #endif 1574 if (inp->inp_socket != NULL) 1575 cred = inp->inp_socket->so_cred; 1576 else 1577 cred = NULL; 1578 if (cred != NULL && jailed(cred)) { 1579 if (jinp != NULL) 1580 continue; 1581 else 1582 if (!jailed_ip(cred->cr_prison, 1583 (struct sockaddr *)&jsin)) 1584 continue; 1585 } 1586 if (inp->inp_lport == lport) { 1587 if (ifp && ifp->if_type == IFT_FAITH && 1588 !(inp->inp_flags & INP_FAITH)) 1589 continue; 1590 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1591 if (cred != NULL && jailed(cred)) { 1592 jinp = inp; 1593 } else { 1594 REL_PCBINFO_TOKEN(pcbinfo); 1595 return (inp); 1596 } 1597 } 1598 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1599 #ifdef INET6 1600 if (INP_CHECK_SOCKAF(inp->inp_socket, 1601 AF_INET6)) 1602 local_wild_mapped = inp; 1603 else 1604 #endif 1605 if (cred != NULL && 1606 jailed(cred)) 1607 jinp_wild = inp; 1608 else 1609 local_wild = inp; 1610 } 1611 } 1612 } 1613 1614 REL_PCBINFO_TOKEN(pcbinfo); 1615 1616 if (local_wild != NULL) 1617 return (local_wild); 1618 #ifdef INET6 1619 if (local_wild_mapped != NULL) 1620 return (local_wild_mapped); 1621 #endif 1622 if (jinp != NULL) 1623 return (jinp); 1624 return (jinp_wild); 1625 } 1626 1627 /* 1628 * Not found. 1629 */ 1630 return (NULL); 1631 } 1632 1633 struct inpcb * 1634 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1635 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1636 boolean_t wildcard, struct ifnet *ifp) 1637 { 1638 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1639 laddr, lport_arg, wildcard, ifp, NULL); 1640 } 1641 1642 /* 1643 * Insert PCB into connection hash table. 1644 */ 1645 void 1646 in_pcbinsconnhash(struct inpcb *inp) 1647 { 1648 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1649 struct inpcbhead *bucket; 1650 u_int32_t hashkey_faddr, hashkey_laddr; 1651 1652 #ifdef INET6 1653 if (inp->inp_vflag & INP_IPV6) { 1654 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1655 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1656 } else { 1657 #endif 1658 hashkey_faddr = inp->inp_faddr.s_addr; 1659 hashkey_laddr = inp->inp_laddr.s_addr; 1660 #ifdef INET6 1661 } 1662 #endif 1663 1664 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1665 ("not in the correct netisr")); 1666 ASSERT_INP_NOTINHASH(inp); 1667 inp->inp_flags |= INP_CONNECTED; 1668 1669 /* 1670 * Insert into the connection hash table. 1671 */ 1672 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1673 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1674 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1675 } 1676 1677 /* 1678 * Remove PCB from connection hash table. 1679 */ 1680 void 1681 in_pcbremconnhash(struct inpcb *inp) 1682 { 1683 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1684 1685 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1686 ("not in the correct netisr")); 1687 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1688 1689 LIST_REMOVE(inp, inp_hash); 1690 inp->inp_flags &= ~INP_CONNECTED; 1691 } 1692 1693 /* 1694 * Insert PCB into port hash table. 1695 */ 1696 void 1697 in_pcbinsporthash(struct inpcbportinfo *portinfo, struct inpcb *inp) 1698 { 1699 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1700 struct inpcbporthead *pcbporthash; 1701 struct inpcbport *phd; 1702 1703 /* 1704 * If the porthashbase is shared across several cpus, it must 1705 * have been locked. 1706 */ 1707 ASSERT_PORT_TOKEN_HELD(portinfo); 1708 1709 /* 1710 * Insert into the port hash table. 1711 */ 1712 pcbporthash = &portinfo->porthashbase[ 1713 INP_PCBPORTHASH(inp->inp_lport, portinfo->porthashmask)]; 1714 1715 /* Go through port list and look for a head for this lport. */ 1716 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1717 if (phd->phd_port == inp->inp_lport) 1718 break; 1719 } 1720 1721 /* If none exists, use saved one and tack it on. */ 1722 if (phd == NULL) { 1723 KKASSERT(pcbinfo->portsave != NULL); 1724 phd = pcbinfo->portsave; 1725 pcbinfo->portsave = NULL; 1726 phd->phd_port = inp->inp_lport; 1727 LIST_INIT(&phd->phd_pcblist); 1728 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1729 } 1730 1731 inp->inp_portinfo = portinfo; 1732 inp->inp_phd = phd; 1733 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1734 1735 /* 1736 * Malloc one inpcbport for later use. It is safe to use 1737 * "wait" malloc here (port token would be released, if 1738 * malloc ever blocked), since all changes to the porthash 1739 * are done. 1740 */ 1741 if (pcbinfo->portsave == NULL) { 1742 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1743 M_PCB, M_INTWAIT | M_ZERO); 1744 } 1745 } 1746 1747 void 1748 in_pcbinsporthash_lport(struct inpcb *inp) 1749 { 1750 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1751 struct inpcbportinfo *portinfo; 1752 u_short lport_ho; 1753 1754 /* Locate the proper portinfo based on lport */ 1755 lport_ho = ntohs(inp->inp_lport); 1756 portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask]; 1757 KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset); 1758 1759 GET_PORT_TOKEN(portinfo); 1760 in_pcbinsporthash(portinfo, inp); 1761 REL_PORT_TOKEN(portinfo); 1762 } 1763 1764 static struct inp_localgroup * 1765 inp_localgroup_alloc(u_char vflag, 1766 uint16_t port, const union in_dependaddr *addr, int size) 1767 { 1768 struct inp_localgroup *grp; 1769 1770 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 1771 M_TEMP, M_INTWAIT | M_ZERO); 1772 grp->il_vflag = vflag; 1773 grp->il_lport = port; 1774 grp->il_dependladdr = *addr; 1775 grp->il_inpsiz = size; 1776 1777 return grp; 1778 } 1779 1780 static void 1781 inp_localgroup_free(struct inp_localgroup *grp) 1782 { 1783 kfree(grp, M_TEMP); 1784 } 1785 1786 static void 1787 inp_localgroup_destroy(struct inp_localgroup *grp) 1788 { 1789 LIST_REMOVE(grp, il_list); 1790 inp_localgroup_free(grp); 1791 } 1792 1793 static void 1794 inp_localgroup_copy(struct inp_localgroup *grp, 1795 const struct inp_localgroup *old_grp) 1796 { 1797 int i; 1798 1799 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 1800 ("invalid new local group size %d and old local group count %d", 1801 grp->il_inpsiz, old_grp->il_inpcnt)); 1802 for (i = 0; i < old_grp->il_inpcnt; ++i) 1803 grp->il_inp[i] = old_grp->il_inp[i]; 1804 grp->il_inpcnt = old_grp->il_inpcnt; 1805 } 1806 1807 static void 1808 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1809 { 1810 struct inp_localgrphead *hdr; 1811 struct inp_localgroup *grp, *grp_alloc = NULL; 1812 struct ucred *cred; 1813 int i, idx; 1814 1815 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1816 1817 if (pcbinfo->localgrphashbase == NULL) 1818 return; 1819 1820 /* 1821 * XXX don't allow jailed socket to join local group 1822 */ 1823 if (inp->inp_socket != NULL) 1824 cred = inp->inp_socket->so_cred; 1825 else 1826 cred = NULL; 1827 if (cred != NULL && jailed(cred)) 1828 return; 1829 1830 #ifdef INET6 1831 /* 1832 * XXX don't allow IPv4 mapped INET6 wild socket 1833 */ 1834 if ((inp->inp_vflag & INP_IPV4) && 1835 inp->inp_laddr.s_addr == INADDR_ANY && 1836 INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) 1837 return; 1838 #endif 1839 1840 hdr = &pcbinfo->localgrphashbase[ 1841 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1842 1843 again: 1844 LIST_FOREACH(grp, hdr, il_list) { 1845 if (grp->il_vflag == inp->inp_vflag && 1846 grp->il_lport == inp->inp_lport && 1847 memcmp(&grp->il_dependladdr, 1848 &inp->inp_inc.inc_ie.ie_dependladdr, 1849 sizeof(grp->il_dependladdr)) == 0) { 1850 break; 1851 } 1852 } 1853 if (grp == NULL) { 1854 /* 1855 * Create a new local group 1856 */ 1857 if (grp_alloc == NULL) { 1858 grp_alloc = inp_localgroup_alloc(inp->inp_vflag, 1859 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 1860 INP_LOCALGROUP_SIZMIN); 1861 /* 1862 * Local group allocation could block and the 1863 * local group w/ the same property might have 1864 * been added by others when we were blocked; 1865 * check again. 1866 */ 1867 goto again; 1868 } else { 1869 /* Local group has been allocated; link it */ 1870 grp = grp_alloc; 1871 grp_alloc = NULL; 1872 LIST_INSERT_HEAD(hdr, grp, il_list); 1873 } 1874 } else if (grp->il_inpcnt == grp->il_inpsiz) { 1875 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 1876 static int limit_logged = 0; 1877 1878 if (!limit_logged) { 1879 limit_logged = 1; 1880 kprintf("local group port %d, " 1881 "limit reached\n", ntohs(grp->il_lport)); 1882 } 1883 if (grp_alloc != NULL) { 1884 /* 1885 * This would happen if the local group 1886 * w/ the same property was expanded when 1887 * our local group allocation blocked. 1888 */ 1889 inp_localgroup_free(grp_alloc); 1890 } 1891 return; 1892 } 1893 1894 /* 1895 * Expand this local group 1896 */ 1897 if (grp_alloc == NULL || 1898 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 1899 if (grp_alloc != NULL) 1900 inp_localgroup_free(grp_alloc); 1901 grp_alloc = inp_localgroup_alloc(grp->il_vflag, 1902 grp->il_lport, &grp->il_dependladdr, 1903 grp->il_inpsiz * 2); 1904 /* 1905 * Local group allocation could block and the 1906 * local group w/ the same property might have 1907 * been expanded by others when we were blocked; 1908 * check again. 1909 */ 1910 goto again; 1911 } 1912 1913 /* 1914 * Save the old local group, link the new one, and then 1915 * destroy the old local group 1916 */ 1917 inp_localgroup_copy(grp_alloc, grp); 1918 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 1919 inp_localgroup_destroy(grp); 1920 1921 grp = grp_alloc; 1922 grp_alloc = NULL; 1923 } else { 1924 /* 1925 * Found the local group 1926 */ 1927 if (grp_alloc != NULL) { 1928 /* 1929 * This would happen if the local group w/ the 1930 * same property was added or expanded when our 1931 * local group allocation blocked. 1932 */ 1933 inp_localgroup_free(grp_alloc); 1934 grp_alloc = NULL; 1935 } 1936 } 1937 1938 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 1939 ("invalid local group size %d and count %d", 1940 grp->il_inpsiz, grp->il_inpcnt)); 1941 1942 /* 1943 * Keep the local group sorted by the inpcb local group index 1944 * in ascending order. 1945 * 1946 * This eases the multi-process userland application which uses 1947 * SO_REUSEPORT sockets and binds process to the owner cpu of 1948 * the SO_REUSEPORT socket: 1949 * If we didn't sort the local group by the inpcb local group 1950 * index and one of the process owning an inpcb in this local 1951 * group restarted, e.g. crashed and restarted by watchdog, 1952 * other processes owning a inpcb in this local group would have 1953 * to detect that event, refetch its socket's owner cpu, and 1954 * re-bind. 1955 */ 1956 idx = grp->il_inpcnt; 1957 for (i = 0; i < idx; ++i) { 1958 struct inpcb *oinp = grp->il_inp[i]; 1959 1960 if (oinp->inp_lgrpindex > i) { 1961 if (inp->inp_lgrpindex < 0) { 1962 inp->inp_lgrpindex = i; 1963 } else if (inp->inp_lgrpindex != i) { 1964 if (bootverbose) { 1965 kprintf("inp %p: grpidx %d, " 1966 "assigned to %d, cpu%d\n", 1967 inp, inp->inp_lgrpindex, i, 1968 mycpuid); 1969 } 1970 } 1971 grp->il_inp[i] = inp; 1972 1973 /* Pull down inpcbs */ 1974 for (; i < grp->il_inpcnt; ++i) { 1975 struct inpcb *oinp1 = grp->il_inp[i + 1]; 1976 1977 grp->il_inp[i + 1] = oinp; 1978 oinp = oinp1; 1979 } 1980 grp->il_inpcnt++; 1981 return; 1982 } 1983 } 1984 1985 if (inp->inp_lgrpindex < 0) { 1986 inp->inp_lgrpindex = idx; 1987 } else if (inp->inp_lgrpindex != idx) { 1988 if (bootverbose) { 1989 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 1990 inp, inp->inp_lgrpindex, idx, mycpuid); 1991 } 1992 } 1993 grp->il_inp[idx] = inp; 1994 grp->il_inpcnt++; 1995 } 1996 1997 void 1998 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1999 { 2000 struct inpcontainer *ic; 2001 struct inpcontainerhead *bucket; 2002 2003 GET_PCBINFO_TOKEN(pcbinfo); 2004 2005 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 2006 2007 bucket = &pcbinfo->wildcardhashbase[ 2008 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2009 2010 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 2011 ic->ic_inp = inp; 2012 LIST_INSERT_HEAD(bucket, ic, ic_list); 2013 2014 REL_PCBINFO_TOKEN(pcbinfo); 2015 } 2016 2017 /* 2018 * Insert PCB into wildcard hash table. 2019 */ 2020 void 2021 in_pcbinswildcardhash(struct inpcb *inp) 2022 { 2023 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2024 2025 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2026 ("not in correct netisr")); 2027 ASSERT_INP_NOTINHASH(inp); 2028 inp->inp_flags |= INP_WILDCARD; 2029 2030 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 2031 } 2032 2033 static void 2034 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2035 { 2036 struct inp_localgrphead *hdr; 2037 struct inp_localgroup *grp; 2038 2039 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2040 2041 if (pcbinfo->localgrphashbase == NULL) 2042 return; 2043 2044 hdr = &pcbinfo->localgrphashbase[ 2045 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2046 2047 LIST_FOREACH(grp, hdr, il_list) { 2048 int i; 2049 2050 for (i = 0; i < grp->il_inpcnt; ++i) { 2051 if (grp->il_inp[i] != inp) 2052 continue; 2053 2054 if (grp->il_inpcnt == 1) { 2055 /* Destroy this local group */ 2056 inp_localgroup_destroy(grp); 2057 } else { 2058 /* Pull up inpcbs */ 2059 for (; i + 1 < grp->il_inpcnt; ++i) 2060 grp->il_inp[i] = grp->il_inp[i + 1]; 2061 grp->il_inpcnt--; 2062 } 2063 return; 2064 } 2065 } 2066 } 2067 2068 void 2069 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2070 { 2071 struct inpcontainer *ic; 2072 struct inpcontainerhead *head; 2073 2074 GET_PCBINFO_TOKEN(pcbinfo); 2075 2076 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2077 2078 /* find bucket */ 2079 head = &pcbinfo->wildcardhashbase[ 2080 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2081 2082 LIST_FOREACH(ic, head, ic_list) { 2083 if (ic->ic_inp == inp) 2084 goto found; 2085 } 2086 REL_PCBINFO_TOKEN(pcbinfo); 2087 return; /* not found! */ 2088 2089 found: 2090 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2091 REL_PCBINFO_TOKEN(pcbinfo); 2092 kfree(ic, M_TEMP); /* deallocate container */ 2093 } 2094 2095 /* 2096 * Remove PCB from wildcard hash table. 2097 */ 2098 void 2099 in_pcbremwildcardhash(struct inpcb *inp) 2100 { 2101 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2102 2103 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2104 ("not in correct netisr")); 2105 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2106 2107 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2108 inp->inp_lgrpindex = -1; 2109 inp->inp_flags &= ~INP_WILDCARD; 2110 } 2111 2112 /* 2113 * Remove PCB from various lists. 2114 */ 2115 void 2116 in_pcbremlists(struct inpcb *inp) 2117 { 2118 if (inp->inp_lport) { 2119 struct inpcbportinfo *portinfo; 2120 struct inpcbport *phd; 2121 2122 /* 2123 * NOTE: 2124 * inp->inp_portinfo is _not_ necessary same as 2125 * inp->inp_pcbinfo->portinfo. 2126 */ 2127 portinfo = inp->inp_portinfo; 2128 GET_PORT_TOKEN(portinfo); 2129 2130 phd = inp->inp_phd; 2131 LIST_REMOVE(inp, inp_portlist); 2132 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 2133 LIST_REMOVE(phd, phd_hash); 2134 kfree(phd, M_PCB); 2135 } 2136 2137 REL_PORT_TOKEN(portinfo); 2138 } 2139 if (inp->inp_flags & INP_WILDCARD) { 2140 in_pcbremwildcardhash(inp); 2141 } else if (inp->inp_flags & INP_CONNECTED) { 2142 in_pcbremconnhash(inp); 2143 } 2144 2145 if (inp->inp_flags & INP_ONLIST) 2146 in_pcbofflist(inp); 2147 } 2148 2149 int 2150 prison_xinpcb(struct thread *td, struct inpcb *inp) 2151 { 2152 struct ucred *cr; 2153 2154 if (td->td_proc == NULL) 2155 return (0); 2156 cr = td->td_proc->p_ucred; 2157 if (cr->cr_prison == NULL) 2158 return (0); 2159 if (inp->inp_socket && inp->inp_socket->so_cred && 2160 inp->inp_socket->so_cred->cr_prison && 2161 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2162 return (0); 2163 return (1); 2164 } 2165 2166 int 2167 in_pcblist_global(SYSCTL_HANDLER_ARGS) 2168 { 2169 struct inpcbinfo *pcbinfo_arr = arg1; 2170 int pcbinfo_arrlen = arg2; 2171 struct inpcb *marker; 2172 int cpu, origcpu; 2173 int error, n; 2174 2175 KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1, 2176 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2177 2178 /* 2179 * The process of preparing the TCB list is too time-consuming and 2180 * resource-intensive to repeat twice on every request. 2181 */ 2182 n = 0; 2183 if (req->oldptr == NULL) { 2184 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2185 n += pcbinfo_arr[cpu].ipi_count; 2186 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2187 return 0; 2188 } 2189 2190 if (req->newptr != NULL) 2191 return EPERM; 2192 2193 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2194 marker->inp_flags |= INP_PLACEMARKER; 2195 2196 /* 2197 * OK, now we're committed to doing something. Re-fetch ipi_count 2198 * after obtaining the generation count. 2199 */ 2200 error = 0; 2201 origcpu = mycpuid; 2202 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2203 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2204 struct inpcb *inp; 2205 struct xinpcb xi; 2206 int i; 2207 2208 lwkt_migratecpu(cpu); 2209 2210 GET_PCBINFO_TOKEN(pcbinfo); 2211 2212 n = pcbinfo->ipi_count; 2213 2214 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2215 i = 0; 2216 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2217 LIST_REMOVE(marker, inp_list); 2218 LIST_INSERT_AFTER(inp, marker, inp_list); 2219 2220 if (inp->inp_flags & INP_PLACEMARKER) 2221 continue; 2222 if (prison_xinpcb(req->td, inp)) 2223 continue; 2224 2225 bzero(&xi, sizeof xi); 2226 xi.xi_len = sizeof xi; 2227 bcopy(inp, &xi.xi_inp, sizeof *inp); 2228 if (inp->inp_socket) 2229 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2230 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2231 break; 2232 ++i; 2233 } 2234 LIST_REMOVE(marker, inp_list); 2235 2236 REL_PCBINFO_TOKEN(pcbinfo); 2237 2238 if (error == 0 && i < n) { 2239 bzero(&xi, sizeof xi); 2240 xi.xi_len = sizeof xi; 2241 while (i < n) { 2242 error = SYSCTL_OUT(req, &xi, sizeof xi); 2243 if (error) 2244 break; 2245 ++i; 2246 } 2247 } 2248 } 2249 2250 lwkt_migratecpu(origcpu); 2251 kfree(marker, M_TEMP); 2252 return error; 2253 } 2254 2255 int 2256 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS) 2257 { 2258 return in_pcblist_global(oidp, arg1, ncpus2, req); 2259 } 2260 2261 void 2262 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2263 { 2264 struct sockaddr_in *sin; 2265 2266 KASSERT(faddr->sa_family == AF_INET, 2267 ("not AF_INET faddr %d", faddr->sa_family)); 2268 2269 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2270 sin->sin_family = AF_INET; 2271 sin->sin_len = sizeof(*sin); 2272 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2273 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2274 2275 so->so_faddr = (struct sockaddr *)sin; 2276 } 2277 2278 void 2279 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2280 boolean_t shared, u_short offset) 2281 { 2282 memset(portinfo, 0, sizeof(*portinfo)); 2283 2284 portinfo->offset = offset; 2285 portinfo->lastport = offset; 2286 portinfo->lastlow = offset; 2287 portinfo->lasthi = offset; 2288 2289 portinfo->porthashbase = hashinit(hashsize, M_PCB, 2290 &portinfo->porthashmask); 2291 2292 if (shared) { 2293 portinfo->porttoken = kmalloc(sizeof(struct lwkt_token), 2294 M_PCB, M_WAITOK); 2295 lwkt_token_init(portinfo->porttoken, "porttoken"); 2296 } 2297 } 2298 2299 void 2300 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2301 { 2302 int hi, lo; 2303 2304 if (step == 1) 2305 return; 2306 2307 hi = *hi0; 2308 lo = *lo0; 2309 2310 hi = rounddown2(hi, step); 2311 hi += ofs; 2312 if (hi > (int)*hi0) 2313 hi -= step; 2314 2315 lo = roundup2(lo, step); 2316 lo -= (step - ofs); 2317 if (lo < (int)*lo0) 2318 lo += step; 2319 2320 *hi0 = hi; 2321 *lo0 = lo; 2322 } 2323 2324 void 2325 in_pcbglobalinit(void) 2326 { 2327 int cpu; 2328 2329 in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB, 2330 M_WAITOK | M_ZERO); 2331 in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer), 2332 M_PCB, M_WAITOK | M_ZERO); 2333 2334 for (cpu = 0; cpu < ncpus; ++cpu) { 2335 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2336 struct inpcb *marker = &in_pcbmarkers[cpu]; 2337 2338 marker->inp_flags |= INP_PLACEMARKER; 2339 ic->ic_inp = marker; 2340 } 2341 } 2342 2343 struct inpcb * 2344 in_pcbmarker(int cpuid) 2345 { 2346 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid)); 2347 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 2348 2349 return &in_pcbmarkers[cpuid]; 2350 } 2351 2352 struct inpcontainer * 2353 in_pcbcontainer_marker(int cpuid) 2354 { 2355 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid)); 2356 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 2357 2358 return &in_pcbcontainer_markers[cpuid]; 2359 } 2360 2361 void 2362 in_pcbresetroute(struct inpcb *inp) 2363 { 2364 struct route *ro = &inp->inp_route; 2365 2366 if (ro->ro_rt != NULL) 2367 RTFREE(ro->ro_rt); 2368 bzero(ro, sizeof(*ro)); 2369 } 2370