1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_inet6.h" 67 68 #include <sys/param.h> 69 #include <sys/systm.h> 70 #include <sys/malloc.h> 71 #include <sys/mbuf.h> 72 #include <sys/domain.h> 73 #include <sys/protosw.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/proc.h> 77 #include <sys/priv.h> 78 #include <sys/jail.h> 79 #include <sys/kernel.h> 80 #include <sys/sysctl.h> 81 82 #include <sys/socketvar2.h> 83 #include <sys/msgport2.h> 84 85 #include <machine/limits.h> 86 87 #include <net/if.h> 88 #include <net/if_types.h> 89 #include <net/route.h> 90 #include <net/netisr2.h> 91 #include <net/toeplitz2.h> 92 93 #include <netinet/in.h> 94 #include <netinet/in_pcb.h> 95 #include <netinet/in_var.h> 96 #include <netinet/ip_var.h> 97 #ifdef INET6 98 #include <netinet/ip6.h> 99 #include <netinet6/ip6_var.h> 100 #endif /* INET6 */ 101 102 #define INP_LOCALGROUP_SIZMIN 8 103 #define INP_LOCALGROUP_SIZMAX 256 104 105 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash, 106 struct in_addr laddr, u_int lport_arg, int wild_okay, 107 struct ucred *cred); 108 109 struct in_addr zeroin_addr; 110 111 /* 112 * These configure the range of local port addresses assigned to 113 * "unspecified" outgoing connections/packets/whatever. 114 */ 115 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 116 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 117 118 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 119 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 120 121 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 122 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 123 124 #define RANGECHK(var, min, max) \ 125 if ((var) < (min)) { (var) = (min); } \ 126 else if ((var) > (max)) { (var) = (max); } 127 128 int udpencap_enable = 1; /* enabled by default */ 129 int udpencap_port = 4500; /* triggers decapsulation */ 130 131 /* 132 * Per-netisr inpcb markers. 133 * NOTE: they should only be used in netisrs. 134 */ 135 static struct inpcb *in_pcbmarkers; 136 static struct inpcontainer *in_pcbcontainer_markers; 137 138 static int 139 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 140 { 141 int error; 142 143 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 144 if (!error) { 145 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 146 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 147 148 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 149 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 150 151 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 152 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 153 } 154 return (error); 155 } 156 157 #undef RANGECHK 158 159 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 160 161 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 162 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 163 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 164 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 165 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 166 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 167 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 168 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 169 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 170 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 172 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 173 174 /* Initialized by ip_init() */ 175 int ip_porthash_trycount; 176 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW, 177 &ip_porthash_trycount, 0, 178 "Number of tries to find local port matching hash of 4-tuple"); 179 180 /* 181 * in_pcb.c: manage the Protocol Control Blocks. 182 * 183 * NOTE: It is assumed that most of these functions will be called from 184 * a critical section. XXX - There are, unfortunately, a few exceptions 185 * to this rule that should be fixed. 186 * 187 * NOTE: The caller should initialize the cpu field to the cpu running the 188 * protocol stack associated with this inpcbinfo. 189 */ 190 191 void 192 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 193 { 194 KASSERT(cpu >= 0 && cpu < netisr_ncpus, ("invalid cpu%d", cpu)); 195 pcbinfo->cpu = cpu; 196 197 LIST_INIT(&pcbinfo->pcblisthead); 198 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 199 M_WAITOK | M_ZERO); 200 201 if (shared) { 202 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 203 M_PCB, M_WAITOK); 204 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 205 } else { 206 pcbinfo->infotoken = NULL; 207 } 208 } 209 210 void 211 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo, 212 int portinfo_cnt) 213 { 214 215 KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt)); 216 pcbinfo->portinfo = portinfo; 217 pcbinfo->portinfo_cnt = portinfo_cnt; 218 } 219 220 struct baddynamicports baddynamicports; 221 222 /* 223 * Check if the specified port is invalid for dynamic allocation. 224 */ 225 int 226 in_baddynamic(u_int16_t port, u_int16_t proto) 227 { 228 switch (proto) { 229 case IPPROTO_TCP: 230 return (DP_ISSET(baddynamicports.tcp, port)); 231 case IPPROTO_UDP: 232 return (DP_ISSET(baddynamicports.udp, port)); 233 default: 234 return (0); 235 } 236 } 237 238 void 239 in_pcbonlist(struct inpcb *inp) 240 { 241 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 242 243 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 244 ("not in the correct netisr")); 245 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 246 inp->inp_flags |= INP_ONLIST; 247 248 GET_PCBINFO_TOKEN(pcbinfo); 249 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 250 pcbinfo->ipi_count++; 251 REL_PCBINFO_TOKEN(pcbinfo); 252 } 253 254 void 255 in_pcbofflist(struct inpcb *inp) 256 { 257 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 258 259 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 260 ("not in the correct netisr")); 261 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 262 inp->inp_flags &= ~INP_ONLIST; 263 264 GET_PCBINFO_TOKEN(pcbinfo); 265 LIST_REMOVE(inp, inp_list); 266 KASSERT(pcbinfo->ipi_count > 0, 267 ("invalid inpcb count %d", pcbinfo->ipi_count)); 268 pcbinfo->ipi_count--; 269 REL_PCBINFO_TOKEN(pcbinfo); 270 } 271 272 /* 273 * Allocate a PCB and associate it with the socket. 274 */ 275 int 276 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 277 { 278 struct inpcb *inp; 279 280 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 281 if (inp == NULL) 282 return (ENOMEM); 283 inp->inp_lgrpindex = -1; 284 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 285 inp->inp_pcbinfo = pcbinfo; 286 inp->inp_socket = so; 287 #ifdef INET6 288 if (INP_CHECK_SOCKAF(so, AF_INET6)) { 289 if (ip6_auto_flowlabel) 290 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 291 inp->inp_af = AF_INET6; 292 } else 293 #endif 294 inp->inp_af = AF_INET; 295 soreference(so); 296 so->so_pcb = inp; 297 298 in_pcbonlist(inp); 299 return (0); 300 } 301 302 /* 303 * Unlink a pcb with the intention of moving it to another cpu with a 304 * different pcbinfo. While unlinked nothing should attempt to dereference 305 * inp_pcbinfo, NULL it out so we assert if it does. 306 */ 307 void 308 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 309 { 310 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 311 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 312 ("already linked")); 313 314 in_pcbofflist(inp); 315 inp->inp_pcbinfo = NULL; 316 } 317 318 void 319 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 320 { 321 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 322 } 323 324 /* 325 * Relink a pcb into a new pcbinfo. 326 */ 327 void 328 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 329 { 330 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 331 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 332 ("already linked")); 333 334 inp->inp_pcbinfo = pcbinfo; 335 in_pcbonlist(inp); 336 } 337 338 void 339 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 340 { 341 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 342 } 343 344 static boolean_t 345 in_pcbporthash_update(struct inpcbportinfo *portinfo, 346 struct inpcb *inp, u_short lport, struct ucred *cred, int wild) 347 { 348 struct inpcbporthead *porthash; 349 350 /* 351 * This has to be atomic. If the porthash is shared across multiple 352 * protocol threads, e.g. tcp and udp, then the token must be held. 353 */ 354 porthash = in_pcbporthash_head(portinfo, lport); 355 GET_PORTHASH_TOKEN(porthash); 356 357 if (in_pcblookup_local(porthash, inp->inp_laddr, lport, 358 wild, cred) != NULL) { 359 REL_PORTHASH_TOKEN(porthash); 360 return FALSE; 361 } 362 inp->inp_lport = lport; 363 in_pcbinsporthash(porthash, inp); 364 365 REL_PORTHASH_TOKEN(porthash); 366 return TRUE; 367 } 368 369 static int 370 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 371 { 372 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 373 struct inpcbportinfo *portinfo; 374 u_short first, last, lport, step, first0, last0; 375 int count, error; 376 int portinfo_first, portinfo_idx; 377 uint32_t cut; 378 379 inp->inp_flags |= INP_ANONPORT; 380 381 step = pcbinfo->portinfo_cnt; 382 portinfo_first = mycpuid % pcbinfo->portinfo_cnt; 383 portinfo_idx = portinfo_first; 384 385 if (inp->inp_flags & INP_HIGHPORT) { 386 first0 = ipport_hifirstauto; /* sysctl */ 387 last0 = ipport_hilastauto; 388 } else if (inp->inp_flags & INP_LOWPORT) { 389 if (cred && 390 (error = 391 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 392 inp->inp_laddr.s_addr = INADDR_ANY; 393 return error; 394 } 395 first0 = ipport_lowfirstauto; /* 1023 */ 396 last0 = ipport_lowlastauto; /* 600 */ 397 } else { 398 first0 = ipport_firstauto; /* sysctl */ 399 last0 = ipport_lastauto; 400 } 401 if (first0 > last0) { 402 lport = last0; 403 last0 = first0; 404 first0 = lport; 405 } 406 KKASSERT(last0 >= first0); 407 408 cut = karc4random(); 409 loop: 410 portinfo = &pcbinfo->portinfo[portinfo_idx]; 411 first = first0; 412 last = last0; 413 414 /* 415 * Simple check to ensure all ports are not used up causing 416 * a deadlock here. 417 */ 418 in_pcbportrange(&last, &first, portinfo->offset, step); 419 lport = last - first; 420 count = lport / step; 421 422 lport = rounddown(cut % lport, step) + first; 423 KKASSERT(lport % step == portinfo->offset); 424 425 for (;;) { 426 if (count-- < 0) { /* completely used? */ 427 error = EADDRNOTAVAIL; 428 break; 429 } 430 431 if (__predict_false(lport < first || lport > last)) { 432 lport = first; 433 KKASSERT(lport % step == portinfo->offset); 434 } 435 436 if (in_pcbporthash_update(portinfo, inp, htons(lport), 437 cred, wild)) { 438 error = 0; 439 break; 440 } 441 442 lport += step; 443 KKASSERT(lport % step == portinfo->offset); 444 } 445 446 if (error) { 447 /* Try next portinfo */ 448 portinfo_idx++; 449 portinfo_idx %= pcbinfo->portinfo_cnt; 450 if (portinfo_idx != portinfo_first) 451 goto loop; 452 inp->inp_laddr.s_addr = INADDR_ANY; 453 } 454 return error; 455 } 456 457 int 458 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 459 { 460 struct socket *so = inp->inp_socket; 461 struct sockaddr_in jsin; 462 struct ucred *cred = NULL; 463 int wild = 0; 464 465 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 466 return (EADDRNOTAVAIL); 467 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 468 return (EINVAL); /* already bound */ 469 470 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 471 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 472 if (td->td_proc) 473 cred = td->td_proc->p_ucred; 474 475 if (nam != NULL) { 476 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 477 struct inpcbinfo *pcbinfo; 478 struct inpcbportinfo *portinfo; 479 struct inpcbporthead *porthash; 480 struct inpcb *t; 481 u_short lport, lport_ho; 482 int reuseport = (so->so_options & SO_REUSEPORT); 483 int error; 484 485 if (nam->sa_len != sizeof *sin) 486 return (EINVAL); 487 #ifdef notdef 488 /* 489 * We should check the family, but old programs 490 * incorrectly fail to initialize it. 491 */ 492 if (sin->sin_family != AF_INET) 493 return (EAFNOSUPPORT); 494 #endif 495 if (!prison_replace_wildcards(td, nam)) 496 return (EINVAL); 497 498 lport = sin->sin_port; 499 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 500 /* 501 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 502 * allow complete duplication of binding if 503 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 504 * and a multicast address is bound on both 505 * new and duplicated sockets. 506 */ 507 if (so->so_options & SO_REUSEADDR) 508 reuseport = SO_REUSEADDR | SO_REUSEPORT; 509 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 510 sin->sin_port = 0; /* yech... */ 511 bzero(&sin->sin_zero, sizeof sin->sin_zero); 512 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 513 return (EADDRNOTAVAIL); 514 } 515 516 inp->inp_laddr = sin->sin_addr; 517 518 jsin.sin_family = AF_INET; 519 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 520 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 521 inp->inp_laddr.s_addr = INADDR_ANY; 522 return (EINVAL); 523 } 524 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 525 526 if (lport == 0) { 527 /* Auto-select local port */ 528 return in_pcbsetlport(inp, wild, cred); 529 } 530 lport_ho = ntohs(lport); 531 532 /* GROSS */ 533 if (lport_ho < IPPORT_RESERVED && cred && 534 (error = 535 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 536 inp->inp_laddr.s_addr = INADDR_ANY; 537 return (error); 538 } 539 540 /* 541 * Locate the proper portinfo based on lport 542 */ 543 pcbinfo = inp->inp_pcbinfo; 544 portinfo = 545 &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 546 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == 547 portinfo->offset); 548 549 /* 550 * This has to be atomic. If the porthash is shared across 551 * multiple protocol threads, e.g. tcp and udp then the token 552 * must be held. 553 */ 554 porthash = in_pcbporthash_head(portinfo, lport); 555 GET_PORTHASH_TOKEN(porthash); 556 557 if (so->so_cred->cr_uid != 0 && 558 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 559 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 560 INPLOOKUP_WILDCARD, cred); 561 if (t && 562 (so->so_cred->cr_uid != 563 t->inp_socket->so_cred->cr_uid)) { 564 inp->inp_laddr.s_addr = INADDR_ANY; 565 error = EADDRINUSE; 566 goto done; 567 } 568 } 569 if (cred && !prison_replace_wildcards(td, nam)) { 570 inp->inp_laddr.s_addr = INADDR_ANY; 571 error = EADDRNOTAVAIL; 572 goto done; 573 } 574 575 /* 576 * When binding to a local port if the best match is against 577 * an accepted socket we generally want to allow the binding. 578 * This means that there is no longer any specific socket 579 * bound or bound for listening. 580 */ 581 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 582 wild, cred); 583 if (t && 584 (reuseport & t->inp_socket->so_options) == 0 && 585 (t->inp_socket->so_state & SS_ACCEPTMECH) == 0) { 586 inp->inp_laddr.s_addr = INADDR_ANY; 587 error = EADDRINUSE; 588 goto done; 589 } 590 inp->inp_lport = lport; 591 in_pcbinsporthash(porthash, inp); 592 error = 0; 593 done: 594 REL_PORTHASH_TOKEN(porthash); 595 return (error); 596 } else { 597 jsin.sin_family = AF_INET; 598 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 599 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 600 inp->inp_laddr.s_addr = INADDR_ANY; 601 return (EINVAL); 602 } 603 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 604 605 return in_pcbsetlport(inp, wild, cred); 606 } 607 } 608 609 static struct inpcb * 610 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr, 611 u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred) 612 { 613 struct inpcb *inp; 614 struct inpcbport *phd; 615 struct inpcb *match = NULL; 616 617 /* 618 * If the porthashbase is shared across several cpus, it must 619 * have been locked. 620 */ 621 ASSERT_PORTHASH_TOKEN_HELD(porthash); 622 623 /* 624 * Best fit PCB lookup. 625 * 626 * First see if this local port is in use by looking on the 627 * port hash list. 628 */ 629 LIST_FOREACH(phd, porthash, phd_hash) { 630 if (phd->phd_port == lport) 631 break; 632 } 633 if (phd != NULL) { 634 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 635 #ifdef INET6 636 if (!INP_ISIPV4(inp)) 637 continue; 638 #endif 639 if (inp->inp_laddr.s_addr != INADDR_ANY && 640 inp->inp_laddr.s_addr != laddr.s_addr) 641 continue; 642 643 if (inp->inp_faddr.s_addr != INADDR_ANY && 644 inp->inp_faddr.s_addr != faddr.s_addr) 645 continue; 646 647 if (inp->inp_fport != 0 && inp->inp_fport != fport) 648 continue; 649 650 if (cred == NULL || 651 cred->cr_prison == 652 inp->inp_socket->so_cred->cr_prison) { 653 match = inp; 654 break; 655 } 656 } 657 } 658 return (match); 659 } 660 661 static boolean_t 662 in_pcbporthash_update4(struct inpcbportinfo *portinfo, 663 struct inpcb *inp, u_short lport, const struct sockaddr_in *sin, 664 struct ucred *cred) 665 { 666 struct inpcbporthead *porthash; 667 668 /* 669 * This has to be atomic. If the porthash is shared across multiple 670 * protocol threads, e.g. tcp and udp, then the token must be held. 671 */ 672 porthash = in_pcbporthash_head(portinfo, lport); 673 GET_PORTHASH_TOKEN(porthash); 674 675 if (in_pcblookup_localremote(porthash, inp->inp_laddr, 676 lport, sin->sin_addr, sin->sin_port, cred) != NULL) { 677 REL_PORTHASH_TOKEN(porthash); 678 return FALSE; 679 } 680 inp->inp_lport = lport; 681 in_pcbinsporthash(porthash, inp); 682 683 REL_PORTHASH_TOKEN(porthash); 684 return TRUE; 685 } 686 687 int 688 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 689 struct thread *td) 690 { 691 struct proc *p = td->td_proc; 692 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 693 struct sockaddr_in jsin; 694 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 695 struct ucred *cred = NULL; 696 u_short first, last, lport; 697 int count, hash_count; 698 int error, selfconn = 0; 699 int cpuid = mycpuid; 700 uint32_t hash_base = 0, hash; 701 702 ASSERT_NETISR_NCPUS(cpuid); 703 704 if (TAILQ_EMPTY(&in_ifaddrheads[cpuid])) /* XXX broken! */ 705 return (EADDRNOTAVAIL); 706 707 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 708 if (inp->inp_lport != 0) 709 return (EINVAL); /* already bound */ 710 711 KKASSERT(p); 712 cred = p->p_ucred; 713 714 jsin.sin_family = AF_INET; 715 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 716 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 717 inp->inp_laddr.s_addr = INADDR_ANY; 718 return (EINVAL); 719 } 720 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 721 722 hash_count = ip_porthash_trycount; 723 if (hash_count > 0) { 724 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^ 725 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^ 726 toeplitz_piecemeal_port(sin->sin_port); 727 } else { 728 hash_count = 0; 729 } 730 731 inp->inp_flags |= INP_ANONPORT; 732 733 if (inp->inp_flags & INP_HIGHPORT) { 734 first = ipport_hifirstauto; /* sysctl */ 735 last = ipport_hilastauto; 736 } else if (inp->inp_flags & INP_LOWPORT) { 737 if (cred && 738 (error = 739 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 740 inp->inp_laddr.s_addr = INADDR_ANY; 741 return (error); 742 } 743 first = ipport_lowfirstauto; /* 1023 */ 744 last = ipport_lowlastauto; /* 600 */ 745 } else { 746 first = ipport_firstauto; /* sysctl */ 747 last = ipport_lastauto; 748 } 749 if (first > last) { 750 lport = last; 751 last = first; 752 first = lport; 753 } 754 KKASSERT(last >= first); 755 756 count = last - first; 757 lport = (karc4random() % count) + first; 758 count += hash_count; 759 760 /* 761 * Simple check to ensure all ports are not used up causing 762 * a deadlock here. 763 */ 764 for (;;) { 765 u_short lport_no; 766 767 if (count-- < 0) { /* completely used? */ 768 error = EADDRNOTAVAIL; 769 break; 770 } 771 772 if (__predict_false(lport < first || lport > last)) 773 lport = first; 774 lport_no = htons(lport); 775 776 /* This could happen on loopback interface */ 777 if (__predict_false(sin->sin_port == lport_no && 778 sin->sin_addr.s_addr == inp->inp_laddr.s_addr)) { 779 if (!selfconn) { 780 ++count; /* don't count this try */ 781 selfconn = 1; 782 } 783 goto next; 784 } 785 786 if (hash_count) { 787 --hash_count; 788 hash = hash_base ^ 789 toeplitz_piecemeal_port(lport_no); 790 if (netisr_hashcpu(hash) != cpuid && hash_count) 791 goto next; 792 } 793 794 if (in_pcbporthash_update4( 795 &pcbinfo->portinfo[lport % pcbinfo->portinfo_cnt], 796 inp, lport_no, sin, cred)) { 797 error = 0; 798 break; 799 } 800 next: 801 ++lport; 802 } 803 804 if (error) 805 inp->inp_laddr.s_addr = INADDR_ANY; 806 return (error); 807 } 808 809 /* 810 * Transform old in_pcbconnect() into an inner subroutine for new 811 * in_pcbconnect(): Do some validity-checking on the remote 812 * address (in mbuf 'nam') and then determine local host address 813 * (i.e., which interface) to use to access that remote host. 814 * 815 * This preserves definition of in_pcbconnect(), while supporting a 816 * slightly different version for T/TCP. (This is more than 817 * a bit of a kludge, but cleaning up the internal interfaces would 818 * have forced minor changes in every protocol). 819 */ 820 int 821 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 822 struct sockaddr_in **plocal_sin, struct thread *td, int find) 823 { 824 struct in_ifaddr *ia; 825 struct ucred *cred = NULL; 826 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 827 struct sockaddr *jsin; 828 int jailed = 0, alloc_route = 0; 829 830 if (nam->sa_len != sizeof *sin) 831 return (EINVAL); 832 if (sin->sin_family != AF_INET) 833 return (EAFNOSUPPORT); 834 if (sin->sin_port == 0) 835 return (EADDRNOTAVAIL); 836 if (td && td->td_proc && td->td_proc->p_ucred) 837 cred = td->td_proc->p_ucred; 838 if (cred && cred->cr_prison) 839 jailed = 1; 840 if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) { 841 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 842 /* 843 * If the destination address is INADDR_ANY, 844 * use the primary local address. 845 * If the supplied address is INADDR_BROADCAST, 846 * and the primary interface supports broadcast, 847 * choose the broadcast address for that interface. 848 */ 849 if (sin->sin_addr.s_addr == INADDR_ANY) 850 sin->sin_addr = IA_SIN(ia)->sin_addr; 851 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST && 852 (ia->ia_ifp->if_flags & IFF_BROADCAST)) 853 sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr; 854 } 855 if (find) { 856 struct route *ro; 857 858 ia = NULL; 859 /* 860 * If route is known or can be allocated now, 861 * our src addr is taken from the i/f, else punt. 862 * Note that we should check the address family of the cached 863 * destination, in case of sharing the cache with IPv6. 864 */ 865 ro = &inp->inp_route; 866 if (ro->ro_rt && 867 (!(ro->ro_rt->rt_flags & RTF_UP) || 868 ro->ro_dst.sa_family != AF_INET || 869 satosin(&ro->ro_dst)->sin_addr.s_addr != 870 sin->sin_addr.s_addr || 871 inp->inp_socket->so_options & SO_DONTROUTE)) { 872 RTFREE(ro->ro_rt); 873 ro->ro_rt = NULL; 874 } 875 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 876 (ro->ro_rt == NULL || 877 ro->ro_rt->rt_ifp == NULL)) { 878 /* No route yet, so try to acquire one */ 879 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 880 ro->ro_dst.sa_family = AF_INET; 881 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 882 ((struct sockaddr_in *) &ro->ro_dst)->sin_addr = 883 sin->sin_addr; 884 rtalloc(ro); 885 alloc_route = 1; 886 } 887 /* 888 * If we found a route, use the address 889 * corresponding to the outgoing interface 890 * unless it is the loopback (in case a route 891 * to our address on another net goes to loopback). 892 */ 893 if (ro->ro_rt && 894 !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) { 895 if (jailed) { 896 if (jailed_ip(cred->cr_prison, 897 ro->ro_rt->rt_ifa->ifa_addr)) { 898 ia = ifatoia(ro->ro_rt->rt_ifa); 899 } 900 } else { 901 ia = ifatoia(ro->ro_rt->rt_ifa); 902 } 903 } 904 if (ia == NULL) { 905 u_short fport = sin->sin_port; 906 907 sin->sin_port = 0; 908 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 909 if (ia && jailed && !jailed_ip(cred->cr_prison, 910 sintosa(&ia->ia_addr))) 911 ia = NULL; 912 if (ia == NULL) 913 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 914 if (ia && jailed && !jailed_ip(cred->cr_prison, 915 sintosa(&ia->ia_addr))) 916 ia = NULL; 917 sin->sin_port = fport; 918 if (ia == NULL && 919 !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 920 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 921 if (ia && jailed && !jailed_ip(cred->cr_prison, 922 sintosa(&ia->ia_addr))) 923 ia = NULL; 924 925 if (!jailed && ia == NULL) 926 goto fail; 927 } 928 /* 929 * If the destination address is multicast and an outgoing 930 * interface has been set as a multicast option, use the 931 * address of that interface as our source address. 932 */ 933 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 934 inp->inp_moptions != NULL) { 935 struct ip_moptions *imo; 936 struct ifnet *ifp; 937 938 imo = inp->inp_moptions; 939 if ((ifp = imo->imo_multicast_ifp) != NULL) { 940 struct in_ifaddr_container *iac; 941 942 ia = NULL; 943 TAILQ_FOREACH(iac, 944 &in_ifaddrheads[mycpuid], ia_link) { 945 if (iac->ia->ia_ifp == ifp) { 946 ia = iac->ia; 947 break; 948 } 949 } 950 if (ia == NULL) 951 goto fail; 952 } 953 } 954 /* 955 * Don't do pcblookup call here; return interface in plocal_sin 956 * and exit to caller, that will do the lookup. 957 */ 958 if (ia == NULL && jailed) { 959 if ((jsin = prison_get_nonlocal( 960 cred->cr_prison, AF_INET, NULL)) != NULL || 961 (jsin = prison_get_local( 962 cred->cr_prison, AF_INET, NULL)) != NULL) { 963 *plocal_sin = satosin(jsin); 964 } else { 965 /* IPv6 only Jail */ 966 goto fail; 967 } 968 } else { 969 *plocal_sin = &ia->ia_addr; 970 } 971 } 972 return (0); 973 fail: 974 if (alloc_route) 975 in_pcbresetroute(inp); 976 return (EADDRNOTAVAIL); 977 } 978 979 int 980 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 981 struct sockaddr_in **plocal_sin, struct thread *td) 982 { 983 return in_pcbladdr_find(inp, nam, plocal_sin, td, 984 (inp->inp_laddr.s_addr == INADDR_ANY)); 985 } 986 987 /* 988 * Outer subroutine: 989 * Connect from a socket to a specified address. 990 * Both address and port must be specified in argument sin. 991 * If don't have a local address for this socket yet, 992 * then pick one. 993 */ 994 int 995 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 996 { 997 struct sockaddr_in *if_sin; 998 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 999 int error; 1000 1001 if_sin = NULL; /* avoid gcc warnings */ 1002 1003 /* Call inner routine to assign local interface address. */ 1004 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1005 return (error); 1006 1007 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1008 inp->inp_laddr.s_addr ? 1009 inp->inp_laddr : if_sin->sin_addr, 1010 inp->inp_lport, FALSE, NULL) != NULL) { 1011 return (EADDRINUSE); 1012 } 1013 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1014 if (inp->inp_lport == 0) { 1015 error = in_pcbbind(inp, NULL, td); 1016 if (error) 1017 return (error); 1018 } 1019 inp->inp_laddr = if_sin->sin_addr; 1020 } 1021 inp->inp_faddr = sin->sin_addr; 1022 inp->inp_fport = sin->sin_port; 1023 in_pcbinsconnhash(inp); 1024 return (0); 1025 } 1026 1027 void 1028 in_pcbdisconnect(struct inpcb *inp) 1029 { 1030 1031 in_pcbremconnhash(inp); 1032 inp->inp_faddr.s_addr = INADDR_ANY; 1033 inp->inp_fport = 0; 1034 } 1035 1036 void 1037 in_pcbdetach(struct inpcb *inp) 1038 { 1039 struct socket *so = inp->inp_socket; 1040 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1041 1042 inp->inp_gencnt = ++ipi->ipi_gencnt; 1043 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1044 in_pcbremlists(inp); 1045 so->so_pcb = NULL; 1046 sofree(so); /* remove pcb ref */ 1047 if (inp->inp_options) 1048 m_free(inp->inp_options); 1049 if (inp->inp_route.ro_rt) 1050 rtfree(inp->inp_route.ro_rt); 1051 ip_freemoptions(inp->inp_moptions); 1052 kfree(inp, M_PCB); 1053 } 1054 1055 /* 1056 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1057 * socket received RST. 1058 */ 1059 static int 1060 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1061 { 1062 struct inpcb *inp; 1063 struct sockaddr_in *sin; 1064 1065 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1066 inp = so->so_pcb; 1067 if (!inp) 1068 return (ECONNRESET); 1069 1070 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1071 sin->sin_family = AF_INET; 1072 sin->sin_len = sizeof *sin; 1073 sin->sin_port = inp->inp_lport; 1074 sin->sin_addr = inp->inp_laddr; 1075 1076 *nam = (struct sockaddr *)sin; 1077 return (0); 1078 } 1079 1080 void 1081 in_setsockaddr_dispatch(netmsg_t msg) 1082 { 1083 int error; 1084 1085 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1086 lwkt_replymsg(&msg->lmsg, error); 1087 } 1088 1089 /* 1090 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1091 * socket received RST. 1092 */ 1093 int 1094 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1095 { 1096 struct inpcb *inp; 1097 struct sockaddr_in *sin; 1098 1099 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1100 inp = so->so_pcb; 1101 if (!inp) 1102 return (ECONNRESET); 1103 1104 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1105 sin->sin_family = AF_INET; 1106 sin->sin_len = sizeof *sin; 1107 sin->sin_port = inp->inp_fport; 1108 sin->sin_addr = inp->inp_faddr; 1109 1110 *nam = (struct sockaddr *)sin; 1111 return (0); 1112 } 1113 1114 void 1115 in_setpeeraddr_dispatch(netmsg_t msg) 1116 { 1117 int error; 1118 1119 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1120 lwkt_replymsg(&msg->lmsg, error); 1121 } 1122 1123 void 1124 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1125 inp_notify_t notify) 1126 { 1127 struct inpcb *inp, *marker; 1128 1129 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1130 ("not in the correct netisr")); 1131 marker = in_pcbmarker(); 1132 1133 /* 1134 * NOTE: 1135 * - If INP_PLACEMARKER is set we must ignore the rest of the 1136 * structure and skip it. 1137 * - It is safe to nuke inpcbs here, since we are in their own 1138 * netisr. 1139 */ 1140 GET_PCBINFO_TOKEN(pcbinfo); 1141 1142 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1143 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1144 LIST_REMOVE(marker, inp_list); 1145 LIST_INSERT_AFTER(inp, marker, inp_list); 1146 1147 if (inp->inp_flags & INP_PLACEMARKER) 1148 continue; 1149 #ifdef INET6 1150 if (!INP_ISIPV4(inp)) 1151 continue; 1152 #endif 1153 if (inp->inp_faddr.s_addr != faddr.s_addr || 1154 inp->inp_socket == NULL) 1155 continue; 1156 (*notify)(inp, err); /* can remove inp from list! */ 1157 } 1158 LIST_REMOVE(marker, inp_list); 1159 1160 REL_PCBINFO_TOKEN(pcbinfo); 1161 } 1162 1163 void 1164 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1165 { 1166 struct inpcb *inp, *marker; 1167 1168 /* 1169 * We only need to make sure that we are in netisr0, where all 1170 * multicast operation happen. We could check inpcbinfo which 1171 * does not belong to netisr0 by holding the inpcbinfo's token. 1172 * In this case, the pcbinfo must be able to be shared, i.e. 1173 * pcbinfo->infotoken is not NULL. 1174 */ 1175 ASSERT_NETISR0; 1176 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1177 ("pcbinfo could not be shared")); 1178 1179 /* 1180 * Get a marker for the current netisr (netisr0). 1181 * 1182 * It is possible that the multicast address deletion blocks, 1183 * which could cause temporary token releasing. So we use 1184 * inpcb marker here to get a coherent view of the inpcb list. 1185 * 1186 * While, on the other hand, moptions are only added and deleted 1187 * in netisr0, so we would not see staled moption or miss moption 1188 * even if the token was released due to the blocking multicast 1189 * address deletion. 1190 */ 1191 marker = in_pcbmarker(); 1192 1193 GET_PCBINFO_TOKEN(pcbinfo); 1194 1195 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1196 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1197 struct ip_moptions *imo; 1198 1199 LIST_REMOVE(marker, inp_list); 1200 LIST_INSERT_AFTER(inp, marker, inp_list); 1201 1202 if (inp->inp_flags & INP_PLACEMARKER) 1203 continue; 1204 imo = inp->inp_moptions; 1205 if (INP_ISIPV4(inp) && imo != NULL) { 1206 int i, gap; 1207 1208 /* 1209 * Unselect the outgoing interface if it is being 1210 * detached. 1211 */ 1212 if (imo->imo_multicast_ifp == ifp) 1213 imo->imo_multicast_ifp = NULL; 1214 1215 /* 1216 * Drop multicast group membership if we joined 1217 * through the interface being detached. 1218 */ 1219 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1220 i++) { 1221 if (imo->imo_membership[i]->inm_ifp == ifp) { 1222 /* 1223 * NOTE: 1224 * This could block and the pcbinfo 1225 * token could be passively released. 1226 */ 1227 in_delmulti(imo->imo_membership[i]); 1228 gap++; 1229 } else if (gap != 0) 1230 imo->imo_membership[i - gap] = 1231 imo->imo_membership[i]; 1232 } 1233 imo->imo_num_memberships -= gap; 1234 } 1235 } 1236 LIST_REMOVE(marker, inp_list); 1237 1238 REL_PCBINFO_TOKEN(pcbinfo); 1239 } 1240 1241 /* 1242 * Check for alternatives when higher level complains 1243 * about service problems. For now, invalidate cached 1244 * routing information. If the route was created dynamically 1245 * (by a redirect), time to try a default gateway again. 1246 */ 1247 void 1248 in_losing(struct inpcb *inp) 1249 { 1250 struct rtentry *rt; 1251 struct rt_addrinfo rtinfo; 1252 1253 if ((rt = inp->inp_route.ro_rt)) { 1254 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1255 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1256 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1257 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1258 rtinfo.rti_flags = rt->rt_flags; 1259 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1260 if (rt->rt_flags & RTF_DYNAMIC) { 1261 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1262 rt_mask(rt), rt->rt_flags, NULL); 1263 } 1264 inp->inp_route.ro_rt = NULL; 1265 rtfree(rt); 1266 /* 1267 * A new route can be allocated 1268 * the next time output is attempted. 1269 */ 1270 } 1271 } 1272 1273 /* 1274 * After a routing change, flush old routing 1275 * and allocate a (hopefully) better one. 1276 */ 1277 void 1278 in_rtchange(struct inpcb *inp, int err) 1279 { 1280 if (inp->inp_route.ro_rt) { 1281 rtfree(inp->inp_route.ro_rt); 1282 inp->inp_route.ro_rt = NULL; 1283 /* 1284 * A new route can be allocated the next time 1285 * output is attempted. 1286 */ 1287 } 1288 } 1289 1290 /* 1291 * Lookup a PCB based on the local address and port. 1292 */ 1293 static struct inpcb * 1294 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr, 1295 u_int lport_arg, int wild_okay, struct ucred *cred) 1296 { 1297 struct inpcb *inp; 1298 int matchwild = 3, wildcard; 1299 u_short lport = lport_arg; 1300 struct inpcbport *phd; 1301 struct inpcb *match = NULL; 1302 1303 /* 1304 * If the porthashbase is shared across several cpus, it must 1305 * have been locked. 1306 */ 1307 ASSERT_PORTHASH_TOKEN_HELD(porthash); 1308 1309 /* 1310 * Best fit PCB lookup. 1311 * 1312 * First see if this local port is in use by looking on the 1313 * port hash list. 1314 */ 1315 LIST_FOREACH(phd, porthash, phd_hash) { 1316 if (phd->phd_port == lport) 1317 break; 1318 } 1319 if (phd != NULL) { 1320 /* 1321 * Port is in use by one or more PCBs. Look for best 1322 * fit. 1323 */ 1324 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1325 wildcard = 0; 1326 #ifdef INET6 1327 if (!INP_ISIPV4(inp)) 1328 continue; 1329 #endif 1330 if (inp->inp_faddr.s_addr != INADDR_ANY) 1331 wildcard++; 1332 if (inp->inp_laddr.s_addr != INADDR_ANY) { 1333 if (laddr.s_addr == INADDR_ANY) 1334 wildcard++; 1335 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1336 continue; 1337 } else { 1338 if (laddr.s_addr != INADDR_ANY) 1339 wildcard++; 1340 } 1341 if (wildcard && !wild_okay) 1342 continue; 1343 if (wildcard < matchwild && 1344 (cred == NULL || 1345 cred->cr_prison == 1346 inp->inp_socket->so_cred->cr_prison)) { 1347 match = inp; 1348 matchwild = wildcard; 1349 if (matchwild == 0) { 1350 break; 1351 } 1352 } 1353 } 1354 } 1355 return (match); 1356 } 1357 1358 struct inpcb * 1359 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1360 const struct inpcb *inp) 1361 { 1362 const struct inp_localgrphead *hdr; 1363 const struct inp_localgroup *grp; 1364 int i; 1365 1366 if (pcbinfo->localgrphashbase == NULL) 1367 return NULL; 1368 1369 GET_PCBINFO_TOKEN(pcbinfo); 1370 1371 hdr = &pcbinfo->localgrphashbase[ 1372 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1373 1374 LIST_FOREACH(grp, hdr, il_list) { 1375 if (grp->il_af == inp->inp_af && 1376 grp->il_lport == inp->inp_lport && 1377 memcmp(&grp->il_dependladdr, 1378 &inp->inp_inc.inc_ie.ie_dependladdr, 1379 sizeof(grp->il_dependladdr)) == 0) { 1380 break; 1381 } 1382 } 1383 if (grp == NULL || grp->il_inpcnt == 1) { 1384 REL_PCBINFO_TOKEN(pcbinfo); 1385 return NULL; 1386 } 1387 1388 KASSERT(grp->il_inpcnt >= 2, 1389 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1390 for (i = 0; i < grp->il_inpcnt; ++i) { 1391 if (grp->il_inp[i] == inp) { 1392 int last = grp->il_inpcnt - 1; 1393 1394 if (i == last) 1395 last = grp->il_inpcnt - 2; 1396 REL_PCBINFO_TOKEN(pcbinfo); 1397 return grp->il_inp[last]; 1398 } 1399 } 1400 REL_PCBINFO_TOKEN(pcbinfo); 1401 return NULL; 1402 } 1403 1404 static struct inpcb * 1405 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1406 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1407 { 1408 struct inpcb *local_wild = NULL; 1409 const struct inp_localgrphead *hdr; 1410 const struct inp_localgroup *grp; 1411 1412 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1413 1414 hdr = &pcbinfo->localgrphashbase[ 1415 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1416 1417 /* 1418 * Order of socket selection: 1419 * 1. non-wild. 1420 * 2. wild. 1421 * 1422 * NOTE: Local group does not contain jailed sockets 1423 */ 1424 LIST_FOREACH(grp, hdr, il_list) { 1425 #ifdef INET6 1426 if (grp->il_af != AF_INET) 1427 continue; 1428 #endif 1429 if (grp->il_lport == lport) { 1430 int idx; 1431 1432 /* 1433 * Modulo-N is used here, which greatly reduces 1434 * completion queue token contention, thus more 1435 * cpu time is saved. 1436 */ 1437 idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt; 1438 if (grp->il_laddr.s_addr == laddr.s_addr) 1439 return grp->il_inp[idx]; 1440 else if (grp->il_laddr.s_addr == INADDR_ANY) 1441 local_wild = grp->il_inp[idx]; 1442 } 1443 } 1444 if (local_wild != NULL) 1445 return local_wild; 1446 return NULL; 1447 } 1448 1449 /* 1450 * Lookup PCB in hash list. 1451 */ 1452 struct inpcb * 1453 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1454 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1455 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1456 { 1457 struct inpcbhead *head; 1458 struct inpcb *inp, *jinp=NULL; 1459 u_short fport = fport_arg, lport = lport_arg; 1460 1461 /* 1462 * First look for an exact match. 1463 */ 1464 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1465 laddr.s_addr, lport, pcbinfo->hashmask)]; 1466 LIST_FOREACH(inp, head, inp_hash) { 1467 #ifdef INET6 1468 if (!INP_ISIPV4(inp)) 1469 continue; 1470 #endif 1471 if (in_hosteq(inp->inp_faddr, faddr) && 1472 in_hosteq(inp->inp_laddr, laddr) && 1473 inp->inp_fport == fport && inp->inp_lport == lport) { 1474 /* found */ 1475 if (inp->inp_socket == NULL || 1476 inp->inp_socket->so_cred->cr_prison == NULL) { 1477 return (inp); 1478 } else { 1479 if (jinp == NULL) 1480 jinp = inp; 1481 } 1482 } 1483 } 1484 if (jinp != NULL) 1485 return (jinp); 1486 1487 if (wildcard) { 1488 struct inpcb *local_wild = NULL; 1489 struct inpcb *jinp_wild = NULL; 1490 struct inpcontainer *ic; 1491 struct inpcontainerhead *chead; 1492 struct sockaddr_in jsin; 1493 struct ucred *cred; 1494 1495 GET_PCBINFO_TOKEN(pcbinfo); 1496 1497 /* 1498 * Check local group first 1499 */ 1500 if (pcbinfo->localgrphashbase != NULL && 1501 m != NULL && (m->m_flags & M_HASH)) { 1502 inp = inp_localgroup_lookup(pcbinfo, 1503 laddr, lport, m->m_pkthdr.hash); 1504 if (inp != NULL) { 1505 REL_PCBINFO_TOKEN(pcbinfo); 1506 return inp; 1507 } 1508 } 1509 1510 /* 1511 * Order of socket selection: 1512 * 1. non-jailed, non-wild. 1513 * 2. non-jailed, wild. 1514 * 3. jailed, non-wild. 1515 * 4. jailed, wild. 1516 */ 1517 jsin.sin_family = AF_INET; 1518 chead = &pcbinfo->wildcardhashbase[ 1519 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1520 LIST_FOREACH(ic, chead, ic_list) { 1521 inp = ic->ic_inp; 1522 if (inp->inp_flags & INP_PLACEMARKER) 1523 continue; 1524 1525 jsin.sin_addr.s_addr = laddr.s_addr; 1526 #ifdef INET6 1527 if (!INP_ISIPV4(inp)) 1528 continue; 1529 #endif 1530 if (inp->inp_socket != NULL) 1531 cred = inp->inp_socket->so_cred; 1532 else 1533 cred = NULL; 1534 if (cred != NULL && jailed(cred)) { 1535 if (jinp != NULL) 1536 continue; 1537 else 1538 if (!jailed_ip(cred->cr_prison, 1539 (struct sockaddr *)&jsin)) 1540 continue; 1541 } 1542 if (inp->inp_lport == lport) { 1543 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1544 if (cred != NULL && jailed(cred)) { 1545 jinp = inp; 1546 } else { 1547 REL_PCBINFO_TOKEN(pcbinfo); 1548 return (inp); 1549 } 1550 } 1551 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1552 if (cred != NULL && jailed(cred)) 1553 jinp_wild = inp; 1554 else 1555 local_wild = inp; 1556 } 1557 } 1558 } 1559 1560 REL_PCBINFO_TOKEN(pcbinfo); 1561 1562 if (local_wild != NULL) 1563 return (local_wild); 1564 if (jinp != NULL) 1565 return (jinp); 1566 return (jinp_wild); 1567 } 1568 1569 /* 1570 * Not found. 1571 */ 1572 return (NULL); 1573 } 1574 1575 struct inpcb * 1576 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1577 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1578 boolean_t wildcard, struct ifnet *ifp) 1579 { 1580 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1581 laddr, lport_arg, wildcard, ifp, NULL); 1582 } 1583 1584 /* 1585 * Insert PCB into connection hash table. 1586 */ 1587 void 1588 in_pcbinsconnhash(struct inpcb *inp) 1589 { 1590 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1591 struct inpcbhead *bucket; 1592 u_int32_t hashkey_faddr, hashkey_laddr; 1593 1594 #ifdef INET6 1595 if (INP_ISIPV6(inp)) { 1596 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1597 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1598 } else { 1599 #endif 1600 hashkey_faddr = inp->inp_faddr.s_addr; 1601 hashkey_laddr = inp->inp_laddr.s_addr; 1602 #ifdef INET6 1603 } 1604 #endif 1605 1606 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1607 ("not in the correct netisr")); 1608 ASSERT_INP_NOTINHASH(inp); 1609 inp->inp_flags |= INP_CONNECTED; 1610 1611 /* 1612 * Insert into the connection hash table. 1613 */ 1614 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1615 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1616 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1617 } 1618 1619 /* 1620 * Remove PCB from connection hash table. 1621 */ 1622 void 1623 in_pcbremconnhash(struct inpcb *inp) 1624 { 1625 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1626 1627 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1628 ("not in the correct netisr")); 1629 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1630 1631 LIST_REMOVE(inp, inp_hash); 1632 inp->inp_flags &= ~INP_CONNECTED; 1633 } 1634 1635 /* 1636 * Insert PCB into port hash table. 1637 */ 1638 void 1639 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp) 1640 { 1641 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1642 struct inpcbport *phd; 1643 1644 /* 1645 * If the porthashbase is shared across several cpus, it must 1646 * have been locked. 1647 */ 1648 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash); 1649 1650 /* 1651 * Insert into the port hash table. 1652 */ 1653 1654 /* Go through port list and look for a head for this lport. */ 1655 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1656 if (phd->phd_port == inp->inp_lport) 1657 break; 1658 } 1659 1660 /* If none exists, use saved one and tack it on. */ 1661 if (phd == NULL) { 1662 KKASSERT(pcbinfo->portsave != NULL); 1663 phd = pcbinfo->portsave; 1664 pcbinfo->portsave = NULL; 1665 phd->phd_port = inp->inp_lport; 1666 LIST_INIT(&phd->phd_pcblist); 1667 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1668 } 1669 1670 inp->inp_porthash = pcbporthash; 1671 inp->inp_phd = phd; 1672 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1673 1674 /* 1675 * Malloc one inpcbport for later use. It is safe to use 1676 * "wait" malloc here (port token would be released, if 1677 * malloc ever blocked), since all changes to the porthash 1678 * are done. 1679 */ 1680 if (pcbinfo->portsave == NULL) { 1681 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1682 M_PCB, M_INTWAIT | M_ZERO); 1683 } 1684 } 1685 1686 void 1687 in_pcbinsporthash_lport(struct inpcb *inp) 1688 { 1689 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1690 struct inpcbportinfo *portinfo; 1691 struct inpcbporthead *porthash; 1692 u_short lport_ho; 1693 1694 /* Locate the proper portinfo based on lport */ 1695 lport_ho = ntohs(inp->inp_lport); 1696 portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 1697 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset); 1698 1699 porthash = in_pcbporthash_head(portinfo, inp->inp_lport); 1700 GET_PORTHASH_TOKEN(porthash); 1701 in_pcbinsporthash(porthash, inp); 1702 REL_PORTHASH_TOKEN(porthash); 1703 } 1704 1705 void 1706 in_pcbremporthash(struct inpcb *inp) 1707 { 1708 struct inpcbporthead *porthash; 1709 struct inpcbport *phd; 1710 1711 if (inp->inp_phd == NULL) 1712 return; 1713 KASSERT(inp->inp_lport != 0, ("inpcb has no lport")); 1714 1715 porthash = inp->inp_porthash; 1716 KASSERT(porthash != NULL, ("no porthash")); 1717 1718 GET_PORTHASH_TOKEN(porthash); 1719 1720 phd = inp->inp_phd; 1721 LIST_REMOVE(inp, inp_portlist); 1722 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 1723 LIST_REMOVE(phd, phd_hash); 1724 kfree(phd, M_PCB); 1725 } 1726 1727 REL_PORTHASH_TOKEN(porthash); 1728 1729 inp->inp_phd = NULL; 1730 /* NOTE: Don't whack inp_lport, which may be used later */ 1731 } 1732 1733 static struct inp_localgroup * 1734 inp_localgroup_alloc(u_char af, uint16_t port, 1735 const union in_dependaddr *addr, int size) 1736 { 1737 struct inp_localgroup *grp; 1738 1739 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 1740 M_TEMP, M_INTWAIT | M_ZERO); 1741 grp->il_af = af; 1742 grp->il_lport = port; 1743 grp->il_dependladdr = *addr; 1744 grp->il_inpsiz = size; 1745 1746 return grp; 1747 } 1748 1749 static void 1750 inp_localgroup_free(struct inp_localgroup *grp) 1751 { 1752 kfree(grp, M_TEMP); 1753 } 1754 1755 static void 1756 inp_localgroup_destroy(struct inp_localgroup *grp) 1757 { 1758 LIST_REMOVE(grp, il_list); 1759 inp_localgroup_free(grp); 1760 } 1761 1762 static void 1763 inp_localgroup_copy(struct inp_localgroup *grp, 1764 const struct inp_localgroup *old_grp) 1765 { 1766 int i; 1767 1768 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 1769 ("invalid new local group size %d and old local group count %d", 1770 grp->il_inpsiz, old_grp->il_inpcnt)); 1771 for (i = 0; i < old_grp->il_inpcnt; ++i) 1772 grp->il_inp[i] = old_grp->il_inp[i]; 1773 grp->il_inpcnt = old_grp->il_inpcnt; 1774 } 1775 1776 static void 1777 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1778 { 1779 struct inp_localgrphead *hdr; 1780 struct inp_localgroup *grp, *grp_alloc = NULL; 1781 struct ucred *cred; 1782 int i, idx; 1783 1784 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1785 1786 if (pcbinfo->localgrphashbase == NULL) 1787 return; 1788 1789 /* 1790 * XXX don't allow jailed socket to join local group 1791 */ 1792 if (inp->inp_socket != NULL) 1793 cred = inp->inp_socket->so_cred; 1794 else 1795 cred = NULL; 1796 if (cred != NULL && jailed(cred)) 1797 return; 1798 1799 hdr = &pcbinfo->localgrphashbase[ 1800 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1801 1802 again: 1803 LIST_FOREACH(grp, hdr, il_list) { 1804 if (grp->il_af == inp->inp_af && 1805 grp->il_lport == inp->inp_lport && 1806 memcmp(&grp->il_dependladdr, 1807 &inp->inp_inc.inc_ie.ie_dependladdr, 1808 sizeof(grp->il_dependladdr)) == 0) { 1809 break; 1810 } 1811 } 1812 if (grp == NULL) { 1813 /* 1814 * Create a new local group 1815 */ 1816 if (grp_alloc == NULL) { 1817 grp_alloc = inp_localgroup_alloc(inp->inp_af, 1818 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 1819 INP_LOCALGROUP_SIZMIN); 1820 /* 1821 * Local group allocation could block and the 1822 * local group w/ the same property might have 1823 * been added by others when we were blocked; 1824 * check again. 1825 */ 1826 goto again; 1827 } else { 1828 /* Local group has been allocated; link it */ 1829 grp = grp_alloc; 1830 grp_alloc = NULL; 1831 LIST_INSERT_HEAD(hdr, grp, il_list); 1832 } 1833 } else if (grp->il_inpcnt == grp->il_inpsiz) { 1834 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 1835 static int limit_logged = 0; 1836 1837 if (!limit_logged) { 1838 limit_logged = 1; 1839 kprintf("local group port %d, " 1840 "limit reached\n", ntohs(grp->il_lport)); 1841 } 1842 if (grp_alloc != NULL) { 1843 /* 1844 * This would happen if the local group 1845 * w/ the same property was expanded when 1846 * our local group allocation blocked. 1847 */ 1848 inp_localgroup_free(grp_alloc); 1849 } 1850 return; 1851 } 1852 1853 /* 1854 * Expand this local group 1855 */ 1856 if (grp_alloc == NULL || 1857 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 1858 if (grp_alloc != NULL) 1859 inp_localgroup_free(grp_alloc); 1860 grp_alloc = inp_localgroup_alloc(grp->il_af, 1861 grp->il_lport, &grp->il_dependladdr, 1862 grp->il_inpsiz * 2); 1863 /* 1864 * Local group allocation could block and the 1865 * local group w/ the same property might have 1866 * been expanded by others when we were blocked; 1867 * check again. 1868 */ 1869 goto again; 1870 } 1871 1872 /* 1873 * Save the old local group, link the new one, and then 1874 * destroy the old local group 1875 */ 1876 inp_localgroup_copy(grp_alloc, grp); 1877 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 1878 inp_localgroup_destroy(grp); 1879 1880 grp = grp_alloc; 1881 grp_alloc = NULL; 1882 } else { 1883 /* 1884 * Found the local group 1885 */ 1886 if (grp_alloc != NULL) { 1887 /* 1888 * This would happen if the local group w/ the 1889 * same property was added or expanded when our 1890 * local group allocation blocked. 1891 */ 1892 inp_localgroup_free(grp_alloc); 1893 grp_alloc = NULL; 1894 } 1895 } 1896 1897 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 1898 ("invalid local group size %d and count %d", 1899 grp->il_inpsiz, grp->il_inpcnt)); 1900 1901 /* 1902 * Keep the local group sorted by the inpcb local group index 1903 * in ascending order. 1904 * 1905 * This eases the multi-process userland application which uses 1906 * SO_REUSEPORT sockets and binds process to the owner cpu of 1907 * the SO_REUSEPORT socket: 1908 * If we didn't sort the local group by the inpcb local group 1909 * index and one of the process owning an inpcb in this local 1910 * group restarted, e.g. crashed and restarted by watchdog, 1911 * other processes owning a inpcb in this local group would have 1912 * to detect that event, refetch its socket's owner cpu, and 1913 * re-bind. 1914 */ 1915 idx = grp->il_inpcnt; 1916 for (i = 0; i < idx; ++i) { 1917 struct inpcb *oinp = grp->il_inp[i]; 1918 1919 if (oinp->inp_lgrpindex > i) { 1920 if (inp->inp_lgrpindex < 0) { 1921 inp->inp_lgrpindex = i; 1922 } else if (inp->inp_lgrpindex != i) { 1923 if (bootverbose) { 1924 kprintf("inp %p: grpidx %d, " 1925 "assigned to %d, cpu%d\n", 1926 inp, inp->inp_lgrpindex, i, 1927 mycpuid); 1928 } 1929 } 1930 grp->il_inp[i] = inp; 1931 1932 /* Pull down inpcbs */ 1933 for (; i < grp->il_inpcnt; ++i) { 1934 struct inpcb *oinp1 = grp->il_inp[i + 1]; 1935 1936 grp->il_inp[i + 1] = oinp; 1937 oinp = oinp1; 1938 } 1939 grp->il_inpcnt++; 1940 return; 1941 } 1942 } 1943 1944 if (inp->inp_lgrpindex < 0) { 1945 inp->inp_lgrpindex = idx; 1946 } else if (inp->inp_lgrpindex != idx) { 1947 if (bootverbose) { 1948 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 1949 inp, inp->inp_lgrpindex, idx, mycpuid); 1950 } 1951 } 1952 grp->il_inp[idx] = inp; 1953 grp->il_inpcnt++; 1954 } 1955 1956 void 1957 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1958 { 1959 struct inpcontainer *ic; 1960 struct inpcontainerhead *bucket; 1961 1962 GET_PCBINFO_TOKEN(pcbinfo); 1963 1964 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 1965 1966 bucket = &pcbinfo->wildcardhashbase[ 1967 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 1968 1969 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 1970 ic->ic_inp = inp; 1971 LIST_INSERT_HEAD(bucket, ic, ic_list); 1972 1973 REL_PCBINFO_TOKEN(pcbinfo); 1974 } 1975 1976 /* 1977 * Insert PCB into wildcard hash table. 1978 */ 1979 void 1980 in_pcbinswildcardhash(struct inpcb *inp) 1981 { 1982 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1983 1984 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1985 ("not in correct netisr")); 1986 ASSERT_INP_NOTINHASH(inp); 1987 inp->inp_flags |= INP_WILDCARD; 1988 1989 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 1990 } 1991 1992 static void 1993 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1994 { 1995 struct inp_localgrphead *hdr; 1996 struct inp_localgroup *grp; 1997 1998 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1999 2000 if (pcbinfo->localgrphashbase == NULL) 2001 return; 2002 2003 hdr = &pcbinfo->localgrphashbase[ 2004 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2005 2006 LIST_FOREACH(grp, hdr, il_list) { 2007 int i; 2008 2009 for (i = 0; i < grp->il_inpcnt; ++i) { 2010 if (grp->il_inp[i] != inp) 2011 continue; 2012 2013 if (grp->il_inpcnt == 1) { 2014 /* Destroy this local group */ 2015 inp_localgroup_destroy(grp); 2016 } else { 2017 /* Pull up inpcbs */ 2018 for (; i + 1 < grp->il_inpcnt; ++i) 2019 grp->il_inp[i] = grp->il_inp[i + 1]; 2020 grp->il_inpcnt--; 2021 } 2022 return; 2023 } 2024 } 2025 } 2026 2027 void 2028 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2029 { 2030 struct inpcontainer *ic; 2031 struct inpcontainerhead *head; 2032 2033 GET_PCBINFO_TOKEN(pcbinfo); 2034 2035 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2036 2037 /* find bucket */ 2038 head = &pcbinfo->wildcardhashbase[ 2039 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2040 2041 LIST_FOREACH(ic, head, ic_list) { 2042 if (ic->ic_inp == inp) 2043 goto found; 2044 } 2045 REL_PCBINFO_TOKEN(pcbinfo); 2046 return; /* not found! */ 2047 2048 found: 2049 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2050 REL_PCBINFO_TOKEN(pcbinfo); 2051 kfree(ic, M_TEMP); /* deallocate container */ 2052 } 2053 2054 /* 2055 * Remove PCB from wildcard hash table. 2056 */ 2057 void 2058 in_pcbremwildcardhash(struct inpcb *inp) 2059 { 2060 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2061 2062 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2063 ("not in correct netisr")); 2064 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2065 2066 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2067 inp->inp_lgrpindex = -1; 2068 inp->inp_flags &= ~INP_WILDCARD; 2069 } 2070 2071 /* 2072 * Remove PCB from various lists. 2073 */ 2074 void 2075 in_pcbremlists(struct inpcb *inp) 2076 { 2077 in_pcbremporthash(inp); 2078 if (inp->inp_flags & INP_WILDCARD) { 2079 in_pcbremwildcardhash(inp); 2080 } else if (inp->inp_flags & INP_CONNECTED) { 2081 in_pcbremconnhash(inp); 2082 } 2083 2084 if (inp->inp_flags & INP_ONLIST) 2085 in_pcbofflist(inp); 2086 } 2087 2088 int 2089 prison_xinpcb(struct thread *td, struct inpcb *inp) 2090 { 2091 struct ucred *cr; 2092 2093 if (td->td_proc == NULL) 2094 return (0); 2095 cr = td->td_proc->p_ucred; 2096 if (cr->cr_prison == NULL) 2097 return (0); 2098 if (inp->inp_socket && inp->inp_socket->so_cred && 2099 inp->inp_socket->so_cred->cr_prison && 2100 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2101 return (0); 2102 return (1); 2103 } 2104 2105 int 2106 in_pcblist_range(SYSCTL_HANDLER_ARGS) 2107 { 2108 struct inpcbinfo *pcbinfo_arr = arg1; 2109 int pcbinfo_arrlen = arg2; 2110 struct inpcb *marker; 2111 int cpu, origcpu; 2112 int error, n; 2113 2114 KASSERT(pcbinfo_arrlen <= netisr_ncpus && pcbinfo_arrlen >= 1, 2115 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2116 2117 /* 2118 * The process of preparing the TCB list is too time-consuming and 2119 * resource-intensive to repeat twice on every request. 2120 */ 2121 n = 0; 2122 if (req->oldptr == NULL) { 2123 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2124 n += pcbinfo_arr[cpu].ipi_count; 2125 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2126 return 0; 2127 } 2128 2129 if (req->newptr != NULL) 2130 return EPERM; 2131 2132 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2133 marker->inp_flags |= INP_PLACEMARKER; 2134 2135 /* 2136 * OK, now we're committed to doing something. Re-fetch ipi_count 2137 * after obtaining the generation count. 2138 */ 2139 error = 0; 2140 origcpu = mycpuid; 2141 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2142 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2143 struct inpcb *inp; 2144 struct xinpcb xi; 2145 int i; 2146 2147 lwkt_migratecpu(cpu); 2148 2149 GET_PCBINFO_TOKEN(pcbinfo); 2150 2151 n = pcbinfo->ipi_count; 2152 2153 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2154 i = 0; 2155 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2156 LIST_REMOVE(marker, inp_list); 2157 LIST_INSERT_AFTER(inp, marker, inp_list); 2158 2159 if (inp->inp_flags & INP_PLACEMARKER) 2160 continue; 2161 if (prison_xinpcb(req->td, inp)) 2162 continue; 2163 2164 bzero(&xi, sizeof xi); 2165 xi.xi_len = sizeof xi; 2166 bcopy(inp, &xi.xi_inp, sizeof *inp); 2167 if (inp->inp_socket) 2168 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2169 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2170 break; 2171 ++i; 2172 } 2173 LIST_REMOVE(marker, inp_list); 2174 2175 REL_PCBINFO_TOKEN(pcbinfo); 2176 2177 if (error == 0 && i < n) { 2178 bzero(&xi, sizeof xi); 2179 xi.xi_len = sizeof xi; 2180 while (i < n) { 2181 error = SYSCTL_OUT(req, &xi, sizeof xi); 2182 if (error) 2183 break; 2184 ++i; 2185 } 2186 } 2187 } 2188 2189 lwkt_migratecpu(origcpu); 2190 kfree(marker, M_TEMP); 2191 return error; 2192 } 2193 2194 int 2195 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS) 2196 { 2197 2198 return (in_pcblist_range(oidp, arg1, netisr_ncpus, req)); 2199 } 2200 2201 void 2202 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2203 { 2204 struct sockaddr_in *sin; 2205 2206 KASSERT(faddr->sa_family == AF_INET, 2207 ("not AF_INET faddr %d", faddr->sa_family)); 2208 2209 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2210 sin->sin_family = AF_INET; 2211 sin->sin_len = sizeof(*sin); 2212 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2213 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2214 2215 so->so_faddr = (struct sockaddr *)sin; 2216 } 2217 2218 void 2219 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2220 u_short offset) 2221 { 2222 memset(portinfo, 0, sizeof(*portinfo)); 2223 2224 portinfo->offset = offset; 2225 portinfo->porthashbase = phashinit(hashsize, M_PCB, 2226 &portinfo->porthashcnt); 2227 } 2228 2229 void 2230 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2231 { 2232 int hi, lo; 2233 2234 if (step == 1) 2235 return; 2236 2237 hi = *hi0; 2238 lo = *lo0; 2239 2240 hi = rounddown(hi, step); 2241 hi += ofs; 2242 if (hi > (int)*hi0) 2243 hi -= step; 2244 2245 lo = roundup(lo, step); 2246 lo -= (step - ofs); 2247 if (lo < (int)*lo0) 2248 lo += step; 2249 2250 *hi0 = hi; 2251 *lo0 = lo; 2252 } 2253 2254 void 2255 in_pcbglobalinit(void) 2256 { 2257 int cpu; 2258 2259 in_pcbmarkers = kmalloc(netisr_ncpus * sizeof(struct inpcb), M_PCB, 2260 M_WAITOK | M_ZERO); 2261 in_pcbcontainer_markers = 2262 kmalloc(netisr_ncpus * sizeof(struct inpcontainer), M_PCB, 2263 M_WAITOK | M_ZERO); 2264 2265 for (cpu = 0; cpu < netisr_ncpus; ++cpu) { 2266 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2267 struct inpcb *marker = &in_pcbmarkers[cpu]; 2268 2269 marker->inp_flags |= INP_PLACEMARKER; 2270 ic->ic_inp = marker; 2271 } 2272 } 2273 2274 struct inpcb * 2275 in_pcbmarker(void) 2276 { 2277 2278 ASSERT_NETISR_NCPUS(mycpuid); 2279 return &in_pcbmarkers[mycpuid]; 2280 } 2281 2282 struct inpcontainer * 2283 in_pcbcontainer_marker(void) 2284 { 2285 2286 ASSERT_NETISR_NCPUS(mycpuid); 2287 return &in_pcbcontainer_markers[mycpuid]; 2288 } 2289 2290 void 2291 in_pcbresetroute(struct inpcb *inp) 2292 { 2293 struct route *ro = &inp->inp_route; 2294 2295 if (ro->ro_rt != NULL) 2296 RTFREE(ro->ro_rt); 2297 bzero(ro, sizeof(*ro)); 2298 } 2299