1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_inet6.h" 67 68 #include <sys/param.h> 69 #include <sys/systm.h> 70 #include <sys/malloc.h> 71 #include <sys/mbuf.h> 72 #include <sys/domain.h> 73 #include <sys/protosw.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/proc.h> 77 #include <sys/priv.h> 78 #include <sys/jail.h> 79 #include <sys/kernel.h> 80 #include <sys/sysctl.h> 81 82 #include <sys/socketvar2.h> 83 #include <sys/msgport2.h> 84 85 #include <machine/limits.h> 86 87 #include <net/if.h> 88 #include <net/if_types.h> 89 #include <net/route.h> 90 #include <net/netisr2.h> 91 #include <net/toeplitz2.h> 92 93 #include <netinet/in.h> 94 #include <netinet/in_pcb.h> 95 #include <netinet/in_var.h> 96 #include <netinet/ip_var.h> 97 #ifdef INET6 98 #include <netinet/ip6.h> 99 #include <netinet6/ip6_var.h> 100 #endif /* INET6 */ 101 102 #define INP_LOCALGROUP_SIZMIN 8 103 #define INP_LOCALGROUP_SIZMAX 256 104 105 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash, 106 struct in_addr laddr, u_int lport_arg, int wild_okay, 107 struct ucred *cred); 108 109 struct in_addr zeroin_addr; 110 111 /* 112 * These configure the range of local port addresses assigned to 113 * "unspecified" outgoing connections/packets/whatever. 114 */ 115 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 116 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 117 118 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 119 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 120 121 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 122 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 123 124 #define RANGECHK(var, min, max) \ 125 if ((var) < (min)) { (var) = (min); } \ 126 else if ((var) > (max)) { (var) = (max); } 127 128 int udpencap_enable = 1; /* enabled by default */ 129 int udpencap_port = 4500; /* triggers decapsulation */ 130 131 /* 132 * Per-netisr inpcb markers. 133 * NOTE: they should only be used in netisrs. 134 */ 135 static struct inpcb *in_pcbmarkers; 136 static struct inpcontainer *in_pcbcontainer_markers; 137 138 static int 139 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 140 { 141 int error; 142 143 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 144 if (!error) { 145 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 146 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 147 148 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 149 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 150 151 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 152 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 153 } 154 return (error); 155 } 156 157 #undef RANGECHK 158 159 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 160 161 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 162 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 163 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 164 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 165 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 166 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 167 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 168 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 169 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 170 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 172 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 173 174 /* Initialized by ip_init() */ 175 int ip_porthash_trycount; 176 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW, 177 &ip_porthash_trycount, 0, 178 "Number of tries to find local port matching hash of 4-tuple"); 179 180 /* 181 * in_pcb.c: manage the Protocol Control Blocks. 182 * 183 * NOTE: It is assumed that most of these functions will be called from 184 * a critical section. XXX - There are, unfortunately, a few exceptions 185 * to this rule that should be fixed. 186 * 187 * NOTE: The caller should initialize the cpu field to the cpu running the 188 * protocol stack associated with this inpcbinfo. 189 */ 190 191 void 192 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 193 { 194 KASSERT(cpu >= 0 && cpu < netisr_ncpus, ("invalid cpu%d", cpu)); 195 pcbinfo->cpu = cpu; 196 197 LIST_INIT(&pcbinfo->pcblisthead); 198 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 199 M_WAITOK | M_ZERO); 200 201 if (shared) { 202 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 203 M_PCB, M_WAITOK); 204 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 205 } else { 206 pcbinfo->infotoken = NULL; 207 } 208 } 209 210 void 211 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo, 212 int portinfo_cnt) 213 { 214 215 KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt)); 216 pcbinfo->portinfo = portinfo; 217 pcbinfo->portinfo_cnt = portinfo_cnt; 218 } 219 220 struct baddynamicports baddynamicports; 221 222 /* 223 * Check if the specified port is invalid for dynamic allocation. 224 */ 225 int 226 in_baddynamic(u_int16_t port, u_int16_t proto) 227 { 228 switch (proto) { 229 case IPPROTO_TCP: 230 return (DP_ISSET(baddynamicports.tcp, port)); 231 case IPPROTO_UDP: 232 return (DP_ISSET(baddynamicports.udp, port)); 233 default: 234 return (0); 235 } 236 } 237 238 void 239 in_pcbonlist(struct inpcb *inp) 240 { 241 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 242 243 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 244 ("not in the correct netisr")); 245 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 246 inp->inp_flags |= INP_ONLIST; 247 248 GET_PCBINFO_TOKEN(pcbinfo); 249 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 250 pcbinfo->ipi_count++; 251 REL_PCBINFO_TOKEN(pcbinfo); 252 } 253 254 void 255 in_pcbofflist(struct inpcb *inp) 256 { 257 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 258 259 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 260 ("not in the correct netisr")); 261 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 262 inp->inp_flags &= ~INP_ONLIST; 263 264 GET_PCBINFO_TOKEN(pcbinfo); 265 LIST_REMOVE(inp, inp_list); 266 KASSERT(pcbinfo->ipi_count > 0, 267 ("invalid inpcb count %d", pcbinfo->ipi_count)); 268 pcbinfo->ipi_count--; 269 REL_PCBINFO_TOKEN(pcbinfo); 270 } 271 272 /* 273 * Allocate a PCB and associate it with the socket. 274 */ 275 int 276 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 277 { 278 struct inpcb *inp; 279 280 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 281 if (inp == NULL) 282 return (ENOMEM); 283 inp->inp_lgrpindex = -1; 284 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 285 inp->inp_pcbinfo = pcbinfo; 286 inp->inp_socket = so; 287 #ifdef INET6 288 if (INP_CHECK_SOCKAF(so, AF_INET6)) { 289 if (ip6_auto_flowlabel) 290 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 291 inp->inp_af = AF_INET6; 292 } else 293 #endif 294 inp->inp_af = AF_INET; 295 soreference(so); 296 so->so_pcb = inp; 297 298 in_pcbonlist(inp); 299 return (0); 300 } 301 302 /* 303 * Unlink a pcb with the intention of moving it to another cpu with a 304 * different pcbinfo. While unlinked nothing should attempt to dereference 305 * inp_pcbinfo, NULL it out so we assert if it does. 306 */ 307 void 308 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 309 { 310 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 311 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 312 ("already linked")); 313 314 in_pcbofflist(inp); 315 inp->inp_pcbinfo = NULL; 316 } 317 318 void 319 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 320 { 321 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 322 } 323 324 /* 325 * Relink a pcb into a new pcbinfo. 326 */ 327 void 328 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 329 { 330 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 331 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 332 ("already linked")); 333 334 inp->inp_pcbinfo = pcbinfo; 335 in_pcbonlist(inp); 336 } 337 338 void 339 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 340 { 341 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 342 } 343 344 static boolean_t 345 in_pcbporthash_update(struct inpcbportinfo *portinfo, 346 struct inpcb *inp, u_short lport, struct ucred *cred, int wild) 347 { 348 struct inpcbporthead *porthash; 349 350 /* 351 * This has to be atomic. If the porthash is shared across multiple 352 * protocol threads, e.g. tcp and udp, then the token must be held. 353 */ 354 porthash = in_pcbporthash_head(portinfo, lport); 355 GET_PORTHASH_TOKEN(porthash); 356 357 if (in_pcblookup_local(porthash, inp->inp_laddr, lport, wild, cred)) { 358 REL_PORTHASH_TOKEN(porthash); 359 return FALSE; 360 } 361 inp->inp_lport = lport; 362 in_pcbinsporthash(porthash, inp); 363 364 REL_PORTHASH_TOKEN(porthash); 365 return TRUE; 366 } 367 368 static int 369 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 370 { 371 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 372 struct inpcbportinfo *portinfo; 373 u_short first, last, lport, step, first0, last0; 374 int count, error; 375 int portinfo_first, portinfo_idx; 376 uint32_t cut; 377 378 inp->inp_flags |= INP_ANONPORT; 379 380 step = pcbinfo->portinfo_cnt; 381 portinfo_first = mycpuid % pcbinfo->portinfo_cnt; 382 portinfo_idx = portinfo_first; 383 384 if (inp->inp_flags & INP_HIGHPORT) { 385 first0 = ipport_hifirstauto; /* sysctl */ 386 last0 = ipport_hilastauto; 387 } else if (inp->inp_flags & INP_LOWPORT) { 388 if (cred && 389 (error = 390 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 391 inp->inp_laddr.s_addr = INADDR_ANY; 392 return error; 393 } 394 first0 = ipport_lowfirstauto; /* 1023 */ 395 last0 = ipport_lowlastauto; /* 600 */ 396 } else { 397 first0 = ipport_firstauto; /* sysctl */ 398 last0 = ipport_lastauto; 399 } 400 if (first0 > last0) { 401 lport = last0; 402 last0 = first0; 403 first0 = lport; 404 } 405 KKASSERT(last0 >= first0); 406 407 cut = karc4random(); 408 loop: 409 portinfo = &pcbinfo->portinfo[portinfo_idx]; 410 first = first0; 411 last = last0; 412 413 /* 414 * Simple check to ensure all ports are not used up causing 415 * a deadlock here. 416 */ 417 in_pcbportrange(&last, &first, portinfo->offset, step); 418 lport = last - first; 419 count = lport / step; 420 421 lport = rounddown(cut % lport, step) + first; 422 KKASSERT(lport % step == portinfo->offset); 423 424 for (;;) { 425 if (count-- < 0) { /* completely used? */ 426 error = EADDRNOTAVAIL; 427 break; 428 } 429 430 if (__predict_false(lport < first || lport > last)) { 431 lport = first; 432 KKASSERT(lport % step == portinfo->offset); 433 } 434 435 if (in_pcbporthash_update(portinfo, inp, htons(lport), 436 cred, wild)) { 437 error = 0; 438 break; 439 } 440 441 lport += step; 442 KKASSERT(lport % step == portinfo->offset); 443 } 444 445 if (error) { 446 /* Try next portinfo */ 447 portinfo_idx++; 448 portinfo_idx %= pcbinfo->portinfo_cnt; 449 if (portinfo_idx != portinfo_first) 450 goto loop; 451 inp->inp_laddr.s_addr = INADDR_ANY; 452 } 453 return error; 454 } 455 456 static __inline struct inpcbporthead * 457 OBTAIN_LPORTHASH_TOKEN(struct inpcbinfo *pcbinfo, u_short lport) 458 { 459 struct inpcbportinfo *portinfo; 460 struct inpcbporthead *porthash; 461 u_short lport_ho = ntohs(lport); 462 463 /* 464 * Locate the proper portinfo based on lport 465 */ 466 portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 467 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset); 468 469 porthash = in_pcbporthash_head(portinfo, lport); 470 GET_PORTHASH_TOKEN(porthash); 471 472 return porthash; 473 } 474 475 int 476 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 477 { 478 struct socket *so = inp->inp_socket; 479 struct sockaddr_in jsin; 480 struct ucred *cred = NULL; 481 int wild = 0; 482 483 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 484 return (EADDRNOTAVAIL); 485 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 486 return (EINVAL); /* already bound */ 487 488 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 489 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 490 if (td->td_proc) 491 cred = td->td_proc->p_ucred; 492 493 if (nam != NULL) { 494 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 495 struct inpcbporthead *porthash; 496 struct inpcb *t; 497 u_short lport; 498 int reuseport = (so->so_options & SO_REUSEPORT); 499 int error; 500 501 if (nam->sa_len != sizeof *sin) 502 return (EINVAL); 503 #ifdef notdef 504 /* 505 * We should check the family, but old programs 506 * incorrectly fail to initialize it. 507 */ 508 if (sin->sin_family != AF_INET) 509 return (EAFNOSUPPORT); 510 #endif 511 if (!prison_replace_wildcards(td, nam)) 512 return (EINVAL); 513 514 lport = sin->sin_port; 515 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 516 /* 517 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 518 * allow complete duplication of binding if 519 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 520 * and a multicast address is bound on both 521 * new and duplicated sockets. 522 */ 523 if (so->so_options & SO_REUSEADDR) 524 reuseport = SO_REUSEADDR | SO_REUSEPORT; 525 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 526 sin->sin_port = 0; /* yech... */ 527 bzero(&sin->sin_zero, sizeof sin->sin_zero); 528 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 529 return (EADDRNOTAVAIL); 530 } 531 532 inp->inp_laddr = sin->sin_addr; 533 534 jsin.sin_family = AF_INET; 535 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 536 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 537 inp->inp_laddr.s_addr = INADDR_ANY; 538 return (EINVAL); 539 } 540 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 541 542 if (lport == 0) { 543 /* Auto-select local port */ 544 return in_pcbsetlport(inp, wild, cred); 545 } 546 547 /* GROSS */ 548 if (ntohs(lport) < IPPORT_RESERVED && cred && 549 (error = 550 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 551 inp->inp_laddr.s_addr = INADDR_ANY; 552 return (error); 553 } 554 555 /* 556 * This has to be atomic. If the porthash is shared across 557 * multiple protocol threads, e.g. tcp and udp then the token 558 * must be held. 559 */ 560 porthash = OBTAIN_LPORTHASH_TOKEN(inp->inp_pcbinfo, lport); 561 562 if (so->so_cred->cr_uid != 0 && 563 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 564 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 565 INPLOOKUP_WILDCARD, cred); 566 if (t && 567 (so->so_cred->cr_uid != 568 t->inp_socket->so_cred->cr_uid)) { 569 inp->inp_laddr.s_addr = INADDR_ANY; 570 error = EADDRINUSE; 571 goto done; 572 } 573 } 574 if (cred && !prison_replace_wildcards(td, nam)) { 575 inp->inp_laddr.s_addr = INADDR_ANY; 576 error = EADDRNOTAVAIL; 577 goto done; 578 } 579 580 /* 581 * When binding to a local port if the best match is against 582 * an accepted socket we generally want to allow the binding. 583 * This means that there is no longer any specific socket 584 * bound or bound for listening. 585 */ 586 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 587 wild, cred); 588 if (t && 589 (reuseport & t->inp_socket->so_options) == 0 && 590 (t->inp_socket->so_state & SS_ACCEPTMECH) == 0) { 591 inp->inp_laddr.s_addr = INADDR_ANY; 592 error = EADDRINUSE; 593 goto done; 594 } 595 inp->inp_lport = lport; 596 in_pcbinsporthash(porthash, inp); 597 error = 0; 598 done: 599 REL_PORTHASH_TOKEN(porthash); 600 return (error); 601 } else { 602 jsin.sin_family = AF_INET; 603 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 604 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 605 inp->inp_laddr.s_addr = INADDR_ANY; 606 return (EINVAL); 607 } 608 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 609 610 return in_pcbsetlport(inp, wild, cred); 611 } 612 } 613 614 /* 615 * Lookup a PCB based on the local and remote address and port. 616 * 617 * This function is only used when scanning for a free port. 618 */ 619 static struct inpcb * 620 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr, 621 u_short lport, struct in_addr faddr, u_short fport, 622 struct ucred *cred) 623 { 624 struct inpcb *inp; 625 struct inpcbport *phd; 626 struct inpcb *match = NULL; 627 struct prison *pscan; 628 struct prison *pr; 629 630 /* 631 * If the porthashbase is shared across several cpus, it must 632 * have been locked. 633 */ 634 ASSERT_PORTHASH_TOKEN_HELD(porthash); 635 636 /* 637 * Best fit PCB lookup. 638 * 639 * First see if this local port is in use by looking on the 640 * port hash list. 641 */ 642 LIST_FOREACH(phd, porthash, phd_hash) { 643 if (phd->phd_port == lport) 644 break; 645 } 646 if (phd != NULL) { 647 pr = cred ? cred->cr_prison : NULL; 648 649 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 650 #ifdef INET6 651 if (!INP_ISIPV4(inp)) 652 continue; 653 #endif 654 if (inp->inp_laddr.s_addr == INADDR_ANY) { 655 if (inp->inp_socket && inp->inp_socket->so_cred) 656 pscan = inp->inp_socket->so_cred->cr_prison; 657 else 658 pscan = NULL; 659 if (pr != pscan) 660 continue; 661 } else { 662 if (inp->inp_laddr.s_addr != laddr.s_addr) 663 continue; 664 } 665 666 if (inp->inp_faddr.s_addr != INADDR_ANY && 667 inp->inp_faddr.s_addr != faddr.s_addr) 668 continue; 669 670 if (inp->inp_fport != 0 && inp->inp_fport != fport) 671 continue; 672 673 match = inp; 674 break; 675 } 676 } 677 return (match); 678 } 679 680 static boolean_t 681 in_pcbporthash_update4(struct inpcbportinfo *portinfo, struct inpcb *inp, 682 u_short lport, const struct sockaddr_in *sin, 683 struct ucred *cred) 684 { 685 struct inpcbporthead *porthash; 686 687 /* 688 * This has to be atomic. If the porthash is shared across multiple 689 * protocol threads, e.g. tcp and udp, then the token must be held. 690 */ 691 porthash = in_pcbporthash_head(portinfo, lport); 692 GET_PORTHASH_TOKEN(porthash); 693 694 if (in_pcblookup_localremote(porthash, inp->inp_laddr, lport, 695 sin->sin_addr, sin->sin_port, cred)) { 696 REL_PORTHASH_TOKEN(porthash); 697 return FALSE; 698 } 699 inp->inp_lport = lport; 700 in_pcbinsporthash(porthash, inp); 701 702 REL_PORTHASH_TOKEN(porthash); 703 return TRUE; 704 } 705 706 int 707 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 708 struct thread *td) 709 { 710 struct proc *p = td->td_proc; 711 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 712 struct sockaddr_in jsin; 713 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 714 struct ucred *cred = NULL; 715 u_short first, last, lport; 716 int count, hash_count; 717 int error, selfconn = 0; 718 int cpuid = mycpuid; 719 uint32_t hash_base = 0, hash; 720 721 ASSERT_NETISR_NCPUS(cpuid); 722 723 if (TAILQ_EMPTY(&in_ifaddrheads[cpuid])) /* XXX broken! */ 724 return (EADDRNOTAVAIL); 725 726 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 727 if (inp->inp_lport != 0) 728 return (EINVAL); /* already bound */ 729 730 KKASSERT(p); 731 cred = p->p_ucred; 732 733 jsin.sin_family = AF_INET; 734 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 735 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 736 inp->inp_laddr.s_addr = INADDR_ANY; 737 return (EINVAL); 738 } 739 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 740 741 hash_count = ip_porthash_trycount; 742 if (hash_count > 0) { 743 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^ 744 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^ 745 toeplitz_piecemeal_port(sin->sin_port); 746 } else { 747 hash_count = 0; 748 } 749 750 inp->inp_flags |= INP_ANONPORT; 751 752 if (inp->inp_flags & INP_HIGHPORT) { 753 first = ipport_hifirstauto; /* sysctl */ 754 last = ipport_hilastauto; 755 } else if (inp->inp_flags & INP_LOWPORT) { 756 if (cred && 757 (error = 758 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 759 inp->inp_laddr.s_addr = INADDR_ANY; 760 return (error); 761 } 762 first = ipport_lowfirstauto; /* 1023 */ 763 last = ipport_lowlastauto; /* 600 */ 764 } else { 765 first = ipport_firstauto; /* sysctl */ 766 last = ipport_lastauto; 767 } 768 if (first > last) { 769 lport = last; 770 last = first; 771 first = lport; 772 } 773 KKASSERT(last >= first); 774 775 count = last - first; 776 lport = (karc4random() % count) + first; 777 count += hash_count; 778 779 /* 780 * Simple check to ensure all ports are not used up causing 781 * a deadlock here. 782 */ 783 for (;;) { 784 u_short lport_no; 785 786 if (count-- < 0) { /* completely used? */ 787 error = EADDRNOTAVAIL; 788 break; 789 } 790 791 if (__predict_false(lport < first || lport > last)) 792 lport = first; 793 lport_no = htons(lport); 794 795 /* This could happen on loopback interface */ 796 if (__predict_false(sin->sin_port == lport_no && 797 sin->sin_addr.s_addr == inp->inp_laddr.s_addr)) { 798 if (!selfconn) { 799 ++count; /* don't count this try */ 800 selfconn = 1; 801 } 802 goto next; 803 } 804 805 if (hash_count) { 806 --hash_count; 807 hash = hash_base ^ 808 toeplitz_piecemeal_port(lport_no); 809 if (netisr_hashcpu(hash) != cpuid && hash_count) 810 goto next; 811 } 812 813 if (in_pcbporthash_update4( 814 &pcbinfo->portinfo[lport % pcbinfo->portinfo_cnt], 815 inp, lport_no, sin, cred)) { 816 error = 0; 817 break; 818 } 819 next: 820 ++lport; 821 } 822 823 if (error) 824 inp->inp_laddr.s_addr = INADDR_ANY; 825 return (error); 826 } 827 828 /* 829 * Figure out the local interface address to pair against the requested 830 * target address, as well as validate the target address. 831 */ 832 int 833 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 834 struct sockaddr_in **plocal_sin, struct thread *td, int find) 835 { 836 struct in_ifaddr_container *iac; 837 struct in_ifaddr *ia; 838 struct ucred *cred = NULL; 839 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 840 struct sockaddr *jsin; 841 struct prison *pr; 842 struct route *ro; 843 int alloc_route = 0; 844 845 if (nam->sa_len != sizeof *sin) 846 return (EINVAL); 847 if (sin->sin_family != AF_INET) 848 return (EAFNOSUPPORT); 849 if (sin->sin_port == 0) 850 return (EADDRNOTAVAIL); 851 852 /* 853 * Are we in a jail? 854 */ 855 pr = NULL; 856 if (td && td->td_proc && td->td_proc->p_ucred) 857 cred = td->td_proc->p_ucred; 858 if (cred) 859 pr = cred->cr_prison; 860 861 /* 862 * If the destination address is INADDR_ANY then use the primary 863 * local address. 864 * 865 * If the supplied address is INADDR_BROADCAST, and the primary 866 * interface supports broadcast, choose the broadcast address for 867 * that interface. 868 * 869 * If jailed, locate an interface address acceptable to the jail. 870 */ 871 if (sin->sin_addr.s_addr == INADDR_ANY) { 872 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) { 873 ia = iac->ia; 874 if (pr == NULL || 875 jailed_ip(pr, sintosa(&ia->ia_addr))) { 876 sin->sin_addr = IA_SIN(ia)->sin_addr; 877 break; 878 } 879 } 880 } else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST) { 881 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) { 882 ia = iac->ia; 883 if ((pr == NULL || 884 jailed_ip(pr, sintosa(&ia->ia_addr))) && 885 (iac->ia->ia_ifp->if_flags & IFF_BROADCAST)) { 886 sin->sin_addr = 887 satosin(&ia->ia_broadaddr)->sin_addr; 888 break; 889 } 890 } 891 } 892 893 /* 894 * If asked to do a search, use the cached route or do a route table 895 * lookup to try to find an acceptable local interface IP. 896 */ 897 if (find == 0) 898 return 0; 899 900 ia = NULL; 901 902 /* 903 * If we have a cached route, check to see if it is acceptable. 904 * If not, free it. 905 */ 906 ro = &inp->inp_route; 907 if (ro->ro_rt && 908 (!(ro->ro_rt->rt_flags & RTF_UP) || 909 ro->ro_dst.sa_family != AF_INET || 910 satosin(&ro->ro_dst)->sin_addr.s_addr != 911 sin->sin_addr.s_addr || 912 inp->inp_socket->so_options & SO_DONTROUTE)) { 913 RTFREE(ro->ro_rt); 914 ro->ro_rt = NULL; 915 } 916 917 /* 918 * If we do not have a route, construct one and do a lookup, 919 * unless we are forbidden to do so. 920 * 921 * Note that we should check the address family of the cached 922 * destination, in case of sharing the cache with IPv6. 923 */ 924 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 925 (ro->ro_rt == NULL || ro->ro_rt->rt_ifp == NULL)) { 926 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 927 ro->ro_dst.sa_family = AF_INET; 928 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 929 ((struct sockaddr_in *)&ro->ro_dst)->sin_addr = sin->sin_addr; 930 rtalloc(ro); 931 alloc_route = 1; 932 } 933 934 /* 935 * If we found a route, use the address corresponding to the 936 * outgoing interface. 937 * 938 * If jailed, try to find a compatible address on the outgoing 939 * interface. 940 */ 941 if (ro->ro_rt) { 942 ia = ifatoia(ro->ro_rt->rt_ifa); 943 if (pr == NULL) 944 goto skip; 945 if (jailed_ip(pr, sintosa(&ia->ia_addr))) 946 goto skip; 947 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) { 948 if (iac->ia->ia_ifp != ia->ia_ifp) 949 continue; 950 ia = iac->ia; 951 if (jailed_ip(pr, sintosa(&ia->ia_addr))) 952 goto skip; 953 } 954 ia = NULL; 955 } 956 skip: 957 958 /* 959 * If the route didn't work or there was no route, 960 * fall-back to the first address in in_ifaddrheads[]. 961 * 962 * If jailed and this address is not available for 963 * the jail, leave ia set to NULL. 964 */ 965 if (ia == NULL) { 966 u_short fport = sin->sin_port; 967 968 sin->sin_port = 0; 969 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 970 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr))) 971 ia = NULL; 972 973 if (ia == NULL) 974 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 975 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr))) 976 ia = NULL; 977 978 sin->sin_port = fport; 979 if (ia == NULL && !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 980 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 981 982 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr))) 983 ia = NULL; 984 985 if (pr == NULL && ia == NULL) 986 goto fail; 987 } 988 989 /* 990 * If the destination address is multicast and an outgoing 991 * interface has been set as a multicast option, use the 992 * address of that interface as our source address. 993 */ 994 if (pr == NULL && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 995 inp->inp_moptions != NULL) { 996 struct ip_moptions *imo; 997 struct ifnet *ifp; 998 999 imo = inp->inp_moptions; 1000 if ((ifp = imo->imo_multicast_ifp) != NULL) { 1001 struct in_ifaddr_container *iac; 1002 1003 ia = NULL; 1004 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) { 1005 if (iac->ia->ia_ifp == ifp) { 1006 ia = iac->ia; 1007 break; 1008 } 1009 } 1010 if (ia == NULL) 1011 goto fail; 1012 } 1013 } 1014 1015 /* 1016 * If we still don't have a local address, and are jailed, 1017 * use the jail's first non-localhost IP. If there isn't 1018 * one, use the jail's first localhost IP. 1019 * 1020 * Don't do pcblookup call here; return interface in plocal_sin 1021 * and exit to caller, that will do the lookup. 1022 */ 1023 if (ia == NULL && pr) { 1024 jsin = prison_get_nonlocal(cred->cr_prison, AF_INET, NULL); 1025 if (jsin == NULL) 1026 jsin = prison_get_local(cred->cr_prison, AF_INET, NULL); 1027 if (jsin) 1028 *plocal_sin = satosin(jsin); 1029 else 1030 goto fail; 1031 } else if (ia) { 1032 *plocal_sin = &ia->ia_addr; 1033 } else { 1034 goto fail; 1035 } 1036 return (0); 1037 fail: 1038 if (alloc_route) 1039 in_pcbresetroute(inp); 1040 return (EADDRNOTAVAIL); 1041 } 1042 1043 int 1044 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 1045 struct sockaddr_in **plocal_sin, struct thread *td) 1046 { 1047 return in_pcbladdr_find(inp, nam, plocal_sin, td, 1048 (inp->inp_laddr.s_addr == INADDR_ANY)); 1049 } 1050 1051 /* 1052 * Outer subroutine: 1053 * Connect from a socket to a specified address. 1054 * Both address and port must be specified in argument sin. 1055 * If don't have a local address for this socket yet, 1056 * then pick one. 1057 */ 1058 int 1059 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 1060 { 1061 struct sockaddr_in *if_sin; 1062 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1063 int error; 1064 1065 if_sin = NULL; /* avoid gcc warnings */ 1066 1067 /* Call inner routine to assign local interface address. */ 1068 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1069 return (error); 1070 1071 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1072 inp->inp_laddr.s_addr ? 1073 inp->inp_laddr : if_sin->sin_addr, 1074 inp->inp_lport, FALSE, NULL) != NULL) { 1075 return (EADDRINUSE); 1076 } 1077 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1078 if (inp->inp_lport == 0) { 1079 error = in_pcbbind(inp, NULL, td); 1080 if (error) 1081 return (error); 1082 } 1083 inp->inp_laddr = if_sin->sin_addr; 1084 } 1085 inp->inp_faddr = sin->sin_addr; 1086 inp->inp_fport = sin->sin_port; 1087 in_pcbinsconnhash(inp); 1088 return (0); 1089 } 1090 1091 void 1092 in_pcbdisconnect(struct inpcb *inp) 1093 { 1094 1095 in_pcbremconnhash(inp); 1096 inp->inp_faddr.s_addr = INADDR_ANY; 1097 inp->inp_fport = 0; 1098 } 1099 1100 void 1101 in_pcbdetach(struct inpcb *inp) 1102 { 1103 struct socket *so = inp->inp_socket; 1104 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1105 1106 inp->inp_gencnt = ++ipi->ipi_gencnt; 1107 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1108 in_pcbremlists(inp); 1109 so->so_pcb = NULL; 1110 sofree(so); /* remove pcb ref */ 1111 if (inp->inp_options) 1112 m_free(inp->inp_options); 1113 if (inp->inp_route.ro_rt) 1114 rtfree(inp->inp_route.ro_rt); 1115 ip_freemoptions(inp->inp_moptions); 1116 kfree(inp, M_PCB); 1117 } 1118 1119 /* 1120 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1121 * socket received RST. 1122 */ 1123 static int 1124 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1125 { 1126 struct inpcb *inp; 1127 struct sockaddr_in *sin; 1128 1129 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1130 inp = so->so_pcb; 1131 if (!inp) 1132 return (ECONNRESET); 1133 1134 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1135 sin->sin_family = AF_INET; 1136 sin->sin_len = sizeof *sin; 1137 sin->sin_port = inp->inp_lport; 1138 sin->sin_addr = inp->inp_laddr; 1139 1140 *nam = (struct sockaddr *)sin; 1141 return (0); 1142 } 1143 1144 void 1145 in_setsockaddr_dispatch(netmsg_t msg) 1146 { 1147 int error; 1148 1149 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1150 lwkt_replymsg(&msg->lmsg, error); 1151 } 1152 1153 /* 1154 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1155 * socket received RST. 1156 */ 1157 int 1158 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1159 { 1160 struct inpcb *inp; 1161 struct sockaddr_in *sin; 1162 1163 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1164 inp = so->so_pcb; 1165 if (!inp) 1166 return (ECONNRESET); 1167 1168 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1169 sin->sin_family = AF_INET; 1170 sin->sin_len = sizeof *sin; 1171 sin->sin_port = inp->inp_fport; 1172 sin->sin_addr = inp->inp_faddr; 1173 1174 *nam = (struct sockaddr *)sin; 1175 return (0); 1176 } 1177 1178 void 1179 in_setpeeraddr_dispatch(netmsg_t msg) 1180 { 1181 int error; 1182 1183 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1184 lwkt_replymsg(&msg->lmsg, error); 1185 } 1186 1187 void 1188 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1189 inp_notify_t notify) 1190 { 1191 struct inpcb *inp, *marker; 1192 1193 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1194 ("not in the correct netisr")); 1195 marker = in_pcbmarker(); 1196 1197 /* 1198 * NOTE: 1199 * - If INP_PLACEMARKER is set we must ignore the rest of the 1200 * structure and skip it. 1201 * - It is safe to nuke inpcbs here, since we are in their own 1202 * netisr. 1203 */ 1204 GET_PCBINFO_TOKEN(pcbinfo); 1205 1206 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1207 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1208 LIST_REMOVE(marker, inp_list); 1209 LIST_INSERT_AFTER(inp, marker, inp_list); 1210 1211 if (inp->inp_flags & INP_PLACEMARKER) 1212 continue; 1213 #ifdef INET6 1214 if (!INP_ISIPV4(inp)) 1215 continue; 1216 #endif 1217 if (inp->inp_faddr.s_addr != faddr.s_addr || 1218 inp->inp_socket == NULL) 1219 continue; 1220 (*notify)(inp, err); /* can remove inp from list! */ 1221 } 1222 LIST_REMOVE(marker, inp_list); 1223 1224 REL_PCBINFO_TOKEN(pcbinfo); 1225 } 1226 1227 void 1228 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1229 { 1230 struct inpcb *inp, *marker; 1231 1232 /* 1233 * We only need to make sure that we are in netisr0, where all 1234 * multicast operation happen. We could check inpcbinfo which 1235 * does not belong to netisr0 by holding the inpcbinfo's token. 1236 * In this case, the pcbinfo must be able to be shared, i.e. 1237 * pcbinfo->infotoken is not NULL. 1238 */ 1239 ASSERT_NETISR0; 1240 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1241 ("pcbinfo could not be shared")); 1242 1243 /* 1244 * Get a marker for the current netisr (netisr0). 1245 * 1246 * It is possible that the multicast address deletion blocks, 1247 * which could cause temporary token releasing. So we use 1248 * inpcb marker here to get a coherent view of the inpcb list. 1249 * 1250 * While, on the other hand, moptions are only added and deleted 1251 * in netisr0, so we would not see staled moption or miss moption 1252 * even if the token was released due to the blocking multicast 1253 * address deletion. 1254 */ 1255 marker = in_pcbmarker(); 1256 1257 GET_PCBINFO_TOKEN(pcbinfo); 1258 1259 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1260 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1261 struct ip_moptions *imo; 1262 1263 LIST_REMOVE(marker, inp_list); 1264 LIST_INSERT_AFTER(inp, marker, inp_list); 1265 1266 if (inp->inp_flags & INP_PLACEMARKER) 1267 continue; 1268 imo = inp->inp_moptions; 1269 if (INP_ISIPV4(inp) && imo != NULL) { 1270 int i, gap; 1271 1272 /* 1273 * Unselect the outgoing interface if it is being 1274 * detached. 1275 */ 1276 if (imo->imo_multicast_ifp == ifp) 1277 imo->imo_multicast_ifp = NULL; 1278 1279 /* 1280 * Drop multicast group membership if we joined 1281 * through the interface being detached. 1282 */ 1283 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1284 i++) { 1285 if (imo->imo_membership[i]->inm_ifp == ifp) { 1286 /* 1287 * NOTE: 1288 * This could block and the pcbinfo 1289 * token could be passively released. 1290 */ 1291 in_delmulti(imo->imo_membership[i]); 1292 gap++; 1293 } else if (gap != 0) 1294 imo->imo_membership[i - gap] = 1295 imo->imo_membership[i]; 1296 } 1297 imo->imo_num_memberships -= gap; 1298 } 1299 } 1300 LIST_REMOVE(marker, inp_list); 1301 1302 REL_PCBINFO_TOKEN(pcbinfo); 1303 } 1304 1305 /* 1306 * Check for alternatives when higher level complains 1307 * about service problems. For now, invalidate cached 1308 * routing information. If the route was created dynamically 1309 * (by a redirect), time to try a default gateway again. 1310 */ 1311 void 1312 in_losing(struct inpcb *inp) 1313 { 1314 struct rtentry *rt; 1315 struct rt_addrinfo rtinfo; 1316 1317 if ((rt = inp->inp_route.ro_rt)) { 1318 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1319 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1320 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1321 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1322 rtinfo.rti_flags = rt->rt_flags; 1323 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1324 if (rt->rt_flags & RTF_DYNAMIC) { 1325 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1326 rt_mask(rt), rt->rt_flags, NULL); 1327 } 1328 inp->inp_route.ro_rt = NULL; 1329 rtfree(rt); 1330 /* 1331 * A new route can be allocated 1332 * the next time output is attempted. 1333 */ 1334 } 1335 } 1336 1337 /* 1338 * After a routing change, flush old routing 1339 * and allocate a (hopefully) better one. 1340 */ 1341 void 1342 in_rtchange(struct inpcb *inp, int err) 1343 { 1344 if (inp->inp_route.ro_rt) { 1345 rtfree(inp->inp_route.ro_rt); 1346 inp->inp_route.ro_rt = NULL; 1347 /* 1348 * A new route can be allocated the next time 1349 * output is attempted. 1350 */ 1351 } 1352 } 1353 1354 /* 1355 * Lookup a PCB based on the local address and port. 1356 * 1357 * This function is only used when scanning for a free port. 1358 */ 1359 static struct inpcb * 1360 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr, 1361 u_int lport_arg, int wild_okay, struct ucred *cred) 1362 { 1363 struct prison *pscan; 1364 struct prison *pr; 1365 struct inpcb *inp; 1366 int matchwild = 3, wildcard; 1367 u_short lport = lport_arg; 1368 struct inpcbport *phd; 1369 struct inpcb *match = NULL; 1370 1371 /* 1372 * If the porthashbase is shared across several cpus, it must 1373 * have been locked. 1374 */ 1375 ASSERT_PORTHASH_TOKEN_HELD(porthash); 1376 1377 /* 1378 * Best fit PCB lookup. 1379 * 1380 * First see if this local port is in use by looking on the 1381 * port hash list. 1382 */ 1383 LIST_FOREACH(phd, porthash, phd_hash) { 1384 if (phd->phd_port == lport) 1385 break; 1386 } 1387 if (phd != NULL) { 1388 pr = cred ? cred->cr_prison : NULL; 1389 1390 /* 1391 * Port is in use by one or more PCBs. Look for best 1392 * fit. 1393 * 1394 * If in a prison we may wish to allow the jail to override 1395 * a wildcard listen on the host. Since the jail forces its 1396 * own wildcard listens to a specific set of jail IPs, this 1397 * override allows most services on the host to remain as 1398 * they were and still be 'jail friendly'. 1399 */ 1400 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1401 wildcard = 0; 1402 #ifdef INET6 1403 if (!INP_ISIPV4(inp)) 1404 continue; 1405 #endif 1406 if (inp->inp_faddr.s_addr != INADDR_ANY) 1407 wildcard++; 1408 1409 /* 1410 * Prison are independent of each other in terms 1411 * of allowing bindings. This can result in multiple 1412 * overloaded bindings which in_pcblookup_pkthash() 1413 * will have to sort out. 1414 * 1415 * Allow wildcarded entries to co-exist with specific 1416 * entries. Specific entries override wildcarded 1417 * entries. 1418 */ 1419 if (inp->inp_socket && inp->inp_socket->so_cred) 1420 pscan = inp->inp_socket->so_cred->cr_prison; 1421 else 1422 pscan = NULL; 1423 if (pr != pscan) 1424 continue; 1425 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1426 if (laddr.s_addr != INADDR_ANY) 1427 wildcard++; 1428 } else { 1429 if (laddr.s_addr == INADDR_ANY) 1430 wildcard++; 1431 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1432 continue; 1433 } 1434 if (wildcard && !wild_okay) 1435 continue; 1436 if (wildcard < matchwild) { 1437 match = inp; 1438 matchwild = wildcard; 1439 if (matchwild == 0) 1440 break; 1441 } 1442 } 1443 } 1444 return (match); 1445 } 1446 1447 struct inpcb * 1448 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1449 const struct inpcb *inp) 1450 { 1451 const struct inp_localgrphead *hdr; 1452 const struct inp_localgroup *grp; 1453 int i; 1454 1455 if (pcbinfo->localgrphashbase == NULL) 1456 return NULL; 1457 1458 GET_PCBINFO_TOKEN(pcbinfo); 1459 1460 hdr = &pcbinfo->localgrphashbase[ 1461 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1462 1463 LIST_FOREACH(grp, hdr, il_list) { 1464 if (grp->il_af == inp->inp_af && 1465 grp->il_lport == inp->inp_lport && 1466 memcmp(&grp->il_dependladdr, 1467 &inp->inp_inc.inc_ie.ie_dependladdr, 1468 sizeof(grp->il_dependladdr)) == 0) { 1469 break; 1470 } 1471 } 1472 if (grp == NULL || grp->il_inpcnt == 1) { 1473 REL_PCBINFO_TOKEN(pcbinfo); 1474 return NULL; 1475 } 1476 1477 KASSERT(grp->il_inpcnt >= 2, 1478 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1479 for (i = 0; i < grp->il_inpcnt; ++i) { 1480 if (grp->il_inp[i] == inp) { 1481 int last = grp->il_inpcnt - 1; 1482 1483 if (i == last) 1484 last = grp->il_inpcnt - 2; 1485 REL_PCBINFO_TOKEN(pcbinfo); 1486 return grp->il_inp[last]; 1487 } 1488 } 1489 REL_PCBINFO_TOKEN(pcbinfo); 1490 return NULL; 1491 } 1492 1493 static struct inpcb * 1494 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1495 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1496 { 1497 struct inpcb *local_wild; 1498 struct inpcb *jinp; 1499 struct inpcb *jinp_wild; 1500 struct inpcb *inp; 1501 const struct inp_localgrphead *hdr; 1502 const struct inp_localgroup *grp; 1503 struct sockaddr_in jsin; 1504 struct prison *pr; 1505 struct ucred *cred; 1506 int idx; 1507 int net_listen_ov_local; 1508 int net_listen_ov_wild; 1509 1510 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1511 1512 hdr = &pcbinfo->localgrphashbase[ 1513 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1514 1515 /* 1516 * Order of socket selection: 1517 * 1. non-wild. 1518 * 2. wild. 1519 * 1520 * NOTE: Local group does not contain jailed sockets 1521 */ 1522 jsin.sin_family = AF_INET; 1523 jsin.sin_addr.s_addr = laddr.s_addr; 1524 1525 jinp = NULL; 1526 jinp_wild = NULL; 1527 local_wild = NULL; 1528 net_listen_ov_local = 0; 1529 net_listen_ov_wild = 0; 1530 1531 LIST_FOREACH(grp, hdr, il_list) { 1532 #ifdef INET6 1533 if (grp->il_af != AF_INET) 1534 continue; 1535 #endif 1536 if (grp->il_lport != lport) 1537 continue; 1538 1539 /* 1540 * look for a match 1541 */ 1542 idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt; 1543 inp = grp->il_inp[idx]; 1544 1545 /* 1546 * Modulo-N is used here, which greatly reduces 1547 * completion queue token contention, thus more 1548 * cpu time is saved. 1549 */ 1550 if (grp->il_jailed) { 1551 if (inp->inp_socket == NULL) 1552 continue; 1553 cred = inp->inp_socket->so_cred; 1554 if (cred == NULL) 1555 continue; 1556 pr = cred->cr_prison; 1557 if (pr == NULL) 1558 continue; 1559 if (!jailed_ip(pr, (struct sockaddr *)&jsin)) 1560 continue; 1561 if (grp->il_laddr.s_addr == laddr.s_addr) { 1562 jinp = inp; 1563 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE)) 1564 net_listen_ov_local = 1; 1565 1566 } else if (grp->il_laddr.s_addr == INADDR_ANY && 1567 jinp_wild == NULL) { 1568 jinp_wild = inp; 1569 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE)) 1570 net_listen_ov_wild = 1; 1571 } 1572 } else { 1573 if (grp->il_laddr.s_addr == laddr.s_addr) { 1574 return inp; 1575 } else if (grp->il_laddr.s_addr == INADDR_ANY) { 1576 local_wild = inp; 1577 } 1578 } 1579 } 1580 1581 if (net_listen_ov_local) 1582 return jinp; 1583 if (net_listen_ov_wild) 1584 return jinp_wild; 1585 if (local_wild) 1586 return (local_wild); 1587 if (jinp) 1588 return (jinp); 1589 return (jinp_wild); 1590 } 1591 1592 /* 1593 * Lookup PCB in hash list. 1594 * 1595 * This is used to match incoming packets to a pcb 1596 */ 1597 struct inpcb * 1598 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1599 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1600 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1601 { 1602 struct inpcbhead *head; 1603 struct inpcb *inp, *jinp=NULL; 1604 u_short fport = fport_arg, lport = lport_arg; 1605 1606 /* 1607 * First look for an exact match. 1608 */ 1609 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1610 laddr.s_addr, lport, 1611 pcbinfo->hashmask)]; 1612 LIST_FOREACH(inp, head, inp_hash) { 1613 #ifdef INET6 1614 if (!INP_ISIPV4(inp)) 1615 continue; 1616 #endif 1617 if (in_hosteq(inp->inp_faddr, faddr) && 1618 in_hosteq(inp->inp_laddr, laddr) && 1619 inp->inp_fport == fport && inp->inp_lport == lport) { 1620 /* 1621 * Found specific address, host overrides jailed 1622 * inpcb. 1623 */ 1624 if (inp->inp_socket == NULL || 1625 inp->inp_socket->so_cred->cr_prison == NULL) { 1626 return (inp); 1627 } 1628 if (jinp == NULL) 1629 jinp = inp; 1630 } 1631 } 1632 if (jinp != NULL) 1633 return (jinp); 1634 1635 /* 1636 * We generally get here for connections to wildcarded listeners. 1637 * Any wildcarded listeners in jails must be restricted to the 1638 * jailed IPs only. 1639 */ 1640 if (wildcard) { 1641 struct inpcb *local_wild = NULL; 1642 struct inpcb *jinp_wild = NULL; 1643 struct inpcontainer *ic; 1644 struct inpcontainerhead *chead; 1645 struct sockaddr_in jsin; 1646 struct ucred *cred; 1647 struct prison *pr; 1648 int net_listen_ov_local = 0; 1649 int net_listen_ov_wild = 0; 1650 1651 GET_PCBINFO_TOKEN(pcbinfo); 1652 1653 /* 1654 * Check local group first. When present, the localgroup 1655 * hash utilizes the same non-jailed-vs/jailed priortization 1656 * that the normal wildcardhash does. 1657 */ 1658 if (pcbinfo->localgrphashbase != NULL && 1659 m != NULL && (m->m_flags & M_HASH)) { 1660 inp = inp_localgroup_lookup(pcbinfo, laddr, lport, 1661 m->m_pkthdr.hash); 1662 if (inp != NULL) { 1663 REL_PCBINFO_TOKEN(pcbinfo); 1664 return inp; 1665 } 1666 } 1667 1668 /* 1669 * Order of socket selection: 1670 * 1671 * 1. non-jailed, non-wild. 1672 * 2. non-jailed, wild. (allow_listen_override on) 1673 * 3. jailed, non-wild. 1674 * 4. jailed, wild. 1675 * 5. non-jailed, wild. (allow_listen_override off) 1676 * 1677 * NOTE: jailed wildcards are still restricted to the jail 1678 * IPs. 1679 * 1680 * NOTE: (1) and (3) already handled above. 1681 */ 1682 jsin.sin_family = AF_INET; 1683 chead = &pcbinfo->wildcardhashbase[ 1684 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1685 1686 LIST_FOREACH(ic, chead, ic_list) { 1687 inp = ic->ic_inp; 1688 if (inp->inp_flags & INP_PLACEMARKER) 1689 continue; 1690 1691 /* 1692 * Basic validation 1693 */ 1694 #ifdef INET6 1695 if (!INP_ISIPV4(inp)) 1696 continue; 1697 #endif 1698 if (inp->inp_lport != lport) 1699 continue; 1700 1701 /* 1702 * Calculate prison, setup jsin for jailed_ip() 1703 * check. 1704 */ 1705 jsin.sin_addr.s_addr = laddr.s_addr; 1706 pr = NULL; 1707 cred = NULL; 1708 if (inp->inp_socket) { 1709 cred = inp->inp_socket->so_cred; 1710 if (cred) 1711 pr = cred->cr_prison; 1712 } 1713 1714 /* 1715 * Assign jinp, jinp_wild, and local_wild as 1716 * appropriate, track whether the jail supports 1717 * listen overrides. 1718 */ 1719 if (pr) { 1720 if (!jailed_ip(pr, (struct sockaddr *)&jsin)) 1721 continue; 1722 if (inp->inp_laddr.s_addr == laddr.s_addr && 1723 jinp == NULL) { 1724 jinp = inp; 1725 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE)) 1726 net_listen_ov_local = 1; 1727 } 1728 if (inp->inp_laddr.s_addr == INADDR_ANY && 1729 jinp_wild == NULL) { 1730 jinp_wild = inp; 1731 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE)) 1732 net_listen_ov_wild = 1; 1733 } 1734 } else { 1735 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1736 REL_PCBINFO_TOKEN(pcbinfo); 1737 return (inp); 1738 } 1739 if (inp->inp_laddr.s_addr == INADDR_ANY) 1740 local_wild = inp; 1741 } 1742 } 1743 1744 REL_PCBINFO_TOKEN(pcbinfo); 1745 1746 if (net_listen_ov_local) 1747 return jinp; 1748 if (net_listen_ov_wild) 1749 return jinp_wild; 1750 if (local_wild) 1751 return (local_wild); 1752 if (jinp) 1753 return (jinp); 1754 return (jinp_wild); 1755 } 1756 1757 /* 1758 * Not found. 1759 */ 1760 return (NULL); 1761 } 1762 1763 struct inpcb * 1764 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1765 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1766 boolean_t wildcard, struct ifnet *ifp) 1767 { 1768 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1769 laddr, lport_arg, wildcard, ifp, NULL); 1770 } 1771 1772 /* 1773 * Insert PCB into connection hash table. 1774 */ 1775 void 1776 in_pcbinsconnhash(struct inpcb *inp) 1777 { 1778 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1779 struct inpcbhead *bucket; 1780 u_int32_t hashkey_faddr, hashkey_laddr; 1781 1782 #ifdef INET6 1783 if (INP_ISIPV6(inp)) { 1784 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1785 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1786 } else { 1787 #endif 1788 hashkey_faddr = inp->inp_faddr.s_addr; 1789 hashkey_laddr = inp->inp_laddr.s_addr; 1790 #ifdef INET6 1791 } 1792 #endif 1793 1794 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1795 ("not in the correct netisr")); 1796 ASSERT_INP_NOTINHASH(inp); 1797 inp->inp_flags |= INP_CONNECTED; 1798 1799 /* 1800 * Insert into the connection hash table. 1801 */ 1802 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1803 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1804 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1805 } 1806 1807 /* 1808 * Remove PCB from connection hash table. 1809 */ 1810 void 1811 in_pcbremconnhash(struct inpcb *inp) 1812 { 1813 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1814 1815 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1816 ("not in the correct netisr")); 1817 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1818 1819 LIST_REMOVE(inp, inp_hash); 1820 inp->inp_flags &= ~INP_CONNECTED; 1821 } 1822 1823 /* 1824 * Insert PCB into port hash table. 1825 */ 1826 void 1827 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp) 1828 { 1829 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1830 struct inpcbport *phd; 1831 1832 /* 1833 * If the porthashbase is shared across several cpus, it must 1834 * have been locked. 1835 */ 1836 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash); 1837 1838 /* 1839 * Insert into the port hash table. 1840 */ 1841 1842 /* Go through port list and look for a head for this lport. */ 1843 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1844 if (phd->phd_port == inp->inp_lport) 1845 break; 1846 } 1847 1848 /* If none exists, use saved one and tack it on. */ 1849 if (phd == NULL) { 1850 KKASSERT(pcbinfo->portsave != NULL); 1851 phd = pcbinfo->portsave; 1852 pcbinfo->portsave = NULL; 1853 phd->phd_port = inp->inp_lport; 1854 LIST_INIT(&phd->phd_pcblist); 1855 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1856 } 1857 1858 inp->inp_porthash = pcbporthash; 1859 inp->inp_phd = phd; 1860 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1861 1862 /* 1863 * Malloc one inpcbport for later use. It is safe to use 1864 * "wait" malloc here (port token would be released, if 1865 * malloc ever blocked), since all changes to the porthash 1866 * are done. 1867 */ 1868 if (pcbinfo->portsave == NULL) { 1869 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1870 M_PCB, M_INTWAIT | M_ZERO); 1871 } 1872 } 1873 1874 void 1875 in_pcbinsporthash_lport(struct inpcb *inp) 1876 { 1877 struct inpcbporthead *porthash; 1878 1879 porthash = OBTAIN_LPORTHASH_TOKEN(inp->inp_pcbinfo, inp->inp_lport); 1880 in_pcbinsporthash(porthash, inp); 1881 REL_PORTHASH_TOKEN(porthash); 1882 } 1883 1884 void 1885 in_pcbremporthash(struct inpcb *inp) 1886 { 1887 struct inpcbporthead *porthash; 1888 struct inpcbport *phd; 1889 1890 if (inp->inp_phd == NULL) 1891 return; 1892 KASSERT(inp->inp_lport != 0, ("inpcb has no lport")); 1893 1894 porthash = inp->inp_porthash; 1895 KASSERT(porthash != NULL, ("no porthash")); 1896 1897 GET_PORTHASH_TOKEN(porthash); 1898 1899 phd = inp->inp_phd; 1900 LIST_REMOVE(inp, inp_portlist); 1901 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 1902 LIST_REMOVE(phd, phd_hash); 1903 kfree(phd, M_PCB); 1904 } 1905 1906 REL_PORTHASH_TOKEN(porthash); 1907 1908 inp->inp_phd = NULL; 1909 /* NOTE: Don't whack inp_lport, which may be used later */ 1910 } 1911 1912 static struct inp_localgroup * 1913 inp_localgroup_alloc(u_char af, uint16_t port, 1914 const union in_dependaddr *addr, int size) 1915 { 1916 struct inp_localgroup *grp; 1917 1918 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 1919 M_TEMP, M_INTWAIT | M_ZERO); 1920 grp->il_af = af; 1921 grp->il_lport = port; 1922 grp->il_dependladdr = *addr; 1923 grp->il_inpsiz = size; 1924 1925 return grp; 1926 } 1927 1928 static void 1929 inp_localgroup_free(struct inp_localgroup *grp) 1930 { 1931 kfree(grp, M_TEMP); 1932 } 1933 1934 static void 1935 inp_localgroup_destroy(struct inp_localgroup *grp) 1936 { 1937 LIST_REMOVE(grp, il_list); 1938 inp_localgroup_free(grp); 1939 } 1940 1941 static void 1942 inp_localgroup_copy(struct inp_localgroup *grp, 1943 const struct inp_localgroup *old_grp) 1944 { 1945 int i; 1946 1947 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 1948 ("invalid new local group size %d and old local group count %d", 1949 grp->il_inpsiz, old_grp->il_inpcnt)); 1950 for (i = 0; i < old_grp->il_inpcnt; ++i) 1951 grp->il_inp[i] = old_grp->il_inp[i]; 1952 grp->il_inpcnt = old_grp->il_inpcnt; 1953 } 1954 1955 static void 1956 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1957 { 1958 struct inp_localgrphead *hdr; 1959 struct inp_localgroup *grp, *grp_alloc = NULL; 1960 u_char isjailed; 1961 int i, idx; 1962 1963 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1964 1965 if (pcbinfo->localgrphashbase == NULL) 1966 return; 1967 1968 /* 1969 * Further separate groups by whether the inp is jailed or not. 1970 * This allows the inp_localgroup_lookup() code to manage port 1971 * overloading between jails and non-jails. 1972 * 1973 * XXX all jails are collected into one group, which works fine 1974 * as we expect the jails to be listening on different addresses. 1975 * If this changes in the future we may have to break the groups 1976 * up by prison pointer as well. 1977 */ 1978 if (inp->inp_socket && inp->inp_socket->so_cred) 1979 isjailed = jailed(inp->inp_socket->so_cred); 1980 else 1981 isjailed = 0; 1982 1983 hdr = &pcbinfo->localgrphashbase[ 1984 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1985 1986 again: 1987 LIST_FOREACH(grp, hdr, il_list) { 1988 if (grp->il_af == inp->inp_af && 1989 grp->il_lport == inp->inp_lport && 1990 grp->il_jailed == isjailed && 1991 memcmp(&grp->il_dependladdr, 1992 &inp->inp_inc.inc_ie.ie_dependladdr, 1993 sizeof(grp->il_dependladdr)) == 0) { 1994 break; 1995 } 1996 } 1997 if (grp == NULL) { 1998 /* 1999 * Create a new local group 2000 */ 2001 if (grp_alloc == NULL) { 2002 grp_alloc = inp_localgroup_alloc(inp->inp_af, 2003 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 2004 INP_LOCALGROUP_SIZMIN); 2005 /* 2006 * Local group allocation could block and the 2007 * local group w/ the same property might have 2008 * been added by others when we were blocked; 2009 * check again. 2010 */ 2011 goto again; 2012 } else { 2013 /* Local group has been allocated; link it */ 2014 grp = grp_alloc; 2015 grp->il_jailed = isjailed; 2016 grp_alloc = NULL; 2017 LIST_INSERT_HEAD(hdr, grp, il_list); 2018 } 2019 } else if (grp->il_inpcnt == grp->il_inpsiz) { 2020 #if 0 2021 /* 2022 * REMOVED - Ensure that all entries are placed in the 2023 * localgroup so jail operations can be 2024 * deterministic on a il_lport basis. 2025 */ 2026 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 2027 static int limit_logged = 0; 2028 2029 if (!limit_logged) { 2030 limit_logged = 1; 2031 kprintf("local group port %d, " 2032 "limit reached\n", ntohs(grp->il_lport)); 2033 } 2034 if (grp_alloc != NULL) { 2035 /* 2036 * This would happen if the local group 2037 * w/ the same property was expanded when 2038 * our local group allocation blocked. 2039 */ 2040 inp_localgroup_free(grp_alloc); 2041 } 2042 return; 2043 } 2044 #endif 2045 2046 /* 2047 * Expand this local group 2048 */ 2049 if (grp_alloc == NULL || 2050 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 2051 if (grp_alloc != NULL) 2052 inp_localgroup_free(grp_alloc); 2053 grp_alloc = inp_localgroup_alloc(grp->il_af, 2054 grp->il_lport, &grp->il_dependladdr, 2055 grp->il_inpsiz * 2); 2056 /* 2057 * Local group allocation could block and the 2058 * local group w/ the same property might have 2059 * been expanded by others when we were blocked; 2060 * check again. 2061 */ 2062 goto again; 2063 } 2064 2065 /* 2066 * Save the old local group, link the new one, and then 2067 * destroy the old local group 2068 */ 2069 inp_localgroup_copy(grp_alloc, grp); 2070 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 2071 inp_localgroup_destroy(grp); 2072 2073 grp = grp_alloc; 2074 grp->il_jailed = isjailed; 2075 grp_alloc = NULL; 2076 } else { 2077 /* 2078 * Found the local group 2079 */ 2080 if (grp_alloc != NULL) { 2081 /* 2082 * This would happen if the local group w/ the 2083 * same property was added or expanded when our 2084 * local group allocation blocked. 2085 */ 2086 inp_localgroup_free(grp_alloc); 2087 grp_alloc = NULL; 2088 } 2089 } 2090 2091 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 2092 ("invalid local group size %d and count %d", 2093 grp->il_inpsiz, grp->il_inpcnt)); 2094 2095 /* 2096 * Keep the local group sorted by the inpcb local group index 2097 * in ascending order. 2098 * 2099 * This eases the multi-process userland application which uses 2100 * SO_REUSEPORT sockets and binds process to the owner cpu of 2101 * the SO_REUSEPORT socket: 2102 * If we didn't sort the local group by the inpcb local group 2103 * index and one of the process owning an inpcb in this local 2104 * group restarted, e.g. crashed and restarted by watchdog, 2105 * other processes owning a inpcb in this local group would have 2106 * to detect that event, refetch its socket's owner cpu, and 2107 * re-bind. 2108 */ 2109 idx = grp->il_inpcnt; 2110 for (i = 0; i < idx; ++i) { 2111 struct inpcb *oinp = grp->il_inp[i]; 2112 2113 if (oinp->inp_lgrpindex > i) { 2114 if (inp->inp_lgrpindex < 0) { 2115 inp->inp_lgrpindex = i; 2116 } else if (inp->inp_lgrpindex != i) { 2117 if (bootverbose) { 2118 kprintf("inp %p: grpidx %d, " 2119 "assigned to %d, cpu%d\n", 2120 inp, inp->inp_lgrpindex, i, 2121 mycpuid); 2122 } 2123 } 2124 grp->il_inp[i] = inp; 2125 2126 /* Pull down inpcbs */ 2127 for (; i < grp->il_inpcnt; ++i) { 2128 struct inpcb *oinp1 = grp->il_inp[i + 1]; 2129 2130 grp->il_inp[i + 1] = oinp; 2131 oinp = oinp1; 2132 } 2133 grp->il_inpcnt++; 2134 return; 2135 } 2136 } 2137 2138 if (inp->inp_lgrpindex < 0) { 2139 inp->inp_lgrpindex = idx; 2140 } else if (inp->inp_lgrpindex != idx) { 2141 if (bootverbose) { 2142 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 2143 inp, inp->inp_lgrpindex, idx, mycpuid); 2144 } 2145 } 2146 grp->il_inp[idx] = inp; 2147 grp->il_inpcnt++; 2148 } 2149 2150 void 2151 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2152 { 2153 struct inpcontainer *ic; 2154 struct inpcontainerhead *bucket; 2155 2156 GET_PCBINFO_TOKEN(pcbinfo); 2157 2158 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 2159 2160 bucket = &pcbinfo->wildcardhashbase[ 2161 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2162 2163 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 2164 ic->ic_inp = inp; 2165 LIST_INSERT_HEAD(bucket, ic, ic_list); 2166 2167 REL_PCBINFO_TOKEN(pcbinfo); 2168 } 2169 2170 /* 2171 * Insert PCB into wildcard hash table. 2172 */ 2173 void 2174 in_pcbinswildcardhash(struct inpcb *inp) 2175 { 2176 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2177 2178 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2179 ("not in correct netisr")); 2180 ASSERT_INP_NOTINHASH(inp); 2181 inp->inp_flags |= INP_WILDCARD; 2182 2183 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 2184 } 2185 2186 static void 2187 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2188 { 2189 struct inp_localgrphead *hdr; 2190 struct inp_localgroup *grp; 2191 2192 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2193 2194 if (pcbinfo->localgrphashbase == NULL) 2195 return; 2196 2197 hdr = &pcbinfo->localgrphashbase[ 2198 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2199 2200 LIST_FOREACH(grp, hdr, il_list) { 2201 int i; 2202 2203 for (i = 0; i < grp->il_inpcnt; ++i) { 2204 if (grp->il_inp[i] != inp) 2205 continue; 2206 2207 if (grp->il_inpcnt == 1) { 2208 /* Destroy this local group */ 2209 inp_localgroup_destroy(grp); 2210 } else { 2211 /* Pull up inpcbs */ 2212 for (; i + 1 < grp->il_inpcnt; ++i) 2213 grp->il_inp[i] = grp->il_inp[i + 1]; 2214 grp->il_inpcnt--; 2215 } 2216 return; 2217 } 2218 } 2219 } 2220 2221 void 2222 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2223 { 2224 struct inpcontainer *ic; 2225 struct inpcontainerhead *head; 2226 2227 GET_PCBINFO_TOKEN(pcbinfo); 2228 2229 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2230 2231 /* find bucket */ 2232 head = &pcbinfo->wildcardhashbase[ 2233 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2234 2235 LIST_FOREACH(ic, head, ic_list) { 2236 if (ic->ic_inp == inp) 2237 goto found; 2238 } 2239 REL_PCBINFO_TOKEN(pcbinfo); 2240 return; /* not found! */ 2241 2242 found: 2243 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2244 REL_PCBINFO_TOKEN(pcbinfo); 2245 kfree(ic, M_TEMP); /* deallocate container */ 2246 } 2247 2248 /* 2249 * Remove PCB from wildcard hash table. 2250 */ 2251 void 2252 in_pcbremwildcardhash(struct inpcb *inp) 2253 { 2254 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2255 2256 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2257 ("not in correct netisr")); 2258 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2259 2260 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2261 inp->inp_lgrpindex = -1; 2262 inp->inp_flags &= ~INP_WILDCARD; 2263 } 2264 2265 /* 2266 * Remove PCB from various lists. 2267 */ 2268 void 2269 in_pcbremlists(struct inpcb *inp) 2270 { 2271 in_pcbremporthash(inp); 2272 if (inp->inp_flags & INP_WILDCARD) { 2273 in_pcbremwildcardhash(inp); 2274 } else if (inp->inp_flags & INP_CONNECTED) { 2275 in_pcbremconnhash(inp); 2276 } 2277 2278 if (inp->inp_flags & INP_ONLIST) 2279 in_pcbofflist(inp); 2280 } 2281 2282 int 2283 prison_xinpcb(struct thread *td, struct inpcb *inp) 2284 { 2285 struct ucred *cr; 2286 2287 if (td->td_proc == NULL) 2288 return (0); 2289 cr = td->td_proc->p_ucred; 2290 if (cr->cr_prison == NULL) 2291 return (0); 2292 if (inp->inp_socket && inp->inp_socket->so_cred && 2293 inp->inp_socket->so_cred->cr_prison && 2294 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2295 return (0); 2296 return (1); 2297 } 2298 2299 int 2300 in_pcblist_range(SYSCTL_HANDLER_ARGS) 2301 { 2302 struct inpcbinfo *pcbinfo_arr = arg1; 2303 int pcbinfo_arrlen = arg2; 2304 struct inpcb *marker; 2305 int cpu, origcpu; 2306 int error, n; 2307 2308 KASSERT(pcbinfo_arrlen <= netisr_ncpus && pcbinfo_arrlen >= 1, 2309 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2310 2311 /* 2312 * The process of preparing the TCB list is too time-consuming and 2313 * resource-intensive to repeat twice on every request. 2314 */ 2315 n = 0; 2316 if (req->oldptr == NULL) { 2317 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2318 n += pcbinfo_arr[cpu].ipi_count; 2319 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2320 return 0; 2321 } 2322 2323 if (req->newptr != NULL) 2324 return EPERM; 2325 2326 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2327 marker->inp_flags |= INP_PLACEMARKER; 2328 2329 /* 2330 * OK, now we're committed to doing something. Re-fetch ipi_count 2331 * after obtaining the generation count. 2332 */ 2333 error = 0; 2334 origcpu = mycpuid; 2335 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2336 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2337 struct inpcb *inp; 2338 struct xinpcb xi; 2339 int i; 2340 2341 lwkt_migratecpu(cpu); 2342 2343 GET_PCBINFO_TOKEN(pcbinfo); 2344 2345 n = pcbinfo->ipi_count; 2346 2347 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2348 i = 0; 2349 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2350 LIST_REMOVE(marker, inp_list); 2351 LIST_INSERT_AFTER(inp, marker, inp_list); 2352 2353 if (inp->inp_flags & INP_PLACEMARKER) 2354 continue; 2355 if (prison_xinpcb(req->td, inp)) 2356 continue; 2357 2358 bzero(&xi, sizeof xi); 2359 xi.xi_len = sizeof xi; 2360 bcopy(inp, &xi.xi_inp, sizeof *inp); 2361 if (inp->inp_socket) 2362 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2363 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2364 break; 2365 ++i; 2366 } 2367 LIST_REMOVE(marker, inp_list); 2368 2369 REL_PCBINFO_TOKEN(pcbinfo); 2370 2371 if (error == 0 && i < n) { 2372 bzero(&xi, sizeof xi); 2373 xi.xi_len = sizeof xi; 2374 while (i < n) { 2375 error = SYSCTL_OUT(req, &xi, sizeof xi); 2376 if (error) 2377 break; 2378 ++i; 2379 } 2380 } 2381 } 2382 2383 lwkt_migratecpu(origcpu); 2384 kfree(marker, M_TEMP); 2385 return error; 2386 } 2387 2388 int 2389 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS) 2390 { 2391 2392 return (in_pcblist_range(oidp, arg1, netisr_ncpus, req)); 2393 } 2394 2395 void 2396 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2397 { 2398 struct sockaddr_in *sin; 2399 2400 KASSERT(faddr->sa_family == AF_INET, 2401 ("not AF_INET faddr %d", faddr->sa_family)); 2402 2403 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2404 sin->sin_family = AF_INET; 2405 sin->sin_len = sizeof(*sin); 2406 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2407 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2408 2409 so->so_faddr = (struct sockaddr *)sin; 2410 } 2411 2412 void 2413 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2414 u_short offset) 2415 { 2416 memset(portinfo, 0, sizeof(*portinfo)); 2417 2418 portinfo->offset = offset; 2419 portinfo->porthashbase = phashinit(hashsize, M_PCB, 2420 &portinfo->porthashcnt); 2421 } 2422 2423 void 2424 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2425 { 2426 int hi, lo; 2427 2428 if (step == 1) 2429 return; 2430 2431 hi = *hi0; 2432 lo = *lo0; 2433 2434 hi = rounddown(hi, step); 2435 hi += ofs; 2436 if (hi > (int)*hi0) 2437 hi -= step; 2438 2439 lo = roundup(lo, step); 2440 lo -= (step - ofs); 2441 if (lo < (int)*lo0) 2442 lo += step; 2443 2444 *hi0 = hi; 2445 *lo0 = lo; 2446 } 2447 2448 void 2449 in_pcbglobalinit(void) 2450 { 2451 int cpu; 2452 2453 in_pcbmarkers = kmalloc(netisr_ncpus * sizeof(struct inpcb), M_PCB, 2454 M_WAITOK | M_ZERO); 2455 in_pcbcontainer_markers = 2456 kmalloc(netisr_ncpus * sizeof(struct inpcontainer), M_PCB, 2457 M_WAITOK | M_ZERO); 2458 2459 for (cpu = 0; cpu < netisr_ncpus; ++cpu) { 2460 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2461 struct inpcb *marker = &in_pcbmarkers[cpu]; 2462 2463 marker->inp_flags |= INP_PLACEMARKER; 2464 ic->ic_inp = marker; 2465 } 2466 } 2467 2468 struct inpcb * 2469 in_pcbmarker(void) 2470 { 2471 2472 ASSERT_NETISR_NCPUS(mycpuid); 2473 return &in_pcbmarkers[mycpuid]; 2474 } 2475 2476 struct inpcontainer * 2477 in_pcbcontainer_marker(void) 2478 { 2479 2480 ASSERT_NETISR_NCPUS(mycpuid); 2481 return &in_pcbcontainer_markers[mycpuid]; 2482 } 2483 2484 void 2485 in_pcbresetroute(struct inpcb *inp) 2486 { 2487 struct route *ro = &inp->inp_route; 2488 2489 if (ro->ro_rt != NULL) 2490 RTFREE(ro->ro_rt); 2491 bzero(ro, sizeof(*ro)); 2492 } 2493