1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/domain.h> 74 #include <sys/protosw.h> 75 #include <sys/socket.h> 76 #include <sys/socketvar.h> 77 #include <sys/proc.h> 78 #include <sys/priv.h> 79 #include <sys/jail.h> 80 #include <sys/kernel.h> 81 #include <sys/sysctl.h> 82 83 #include <sys/thread2.h> 84 #include <sys/socketvar2.h> 85 #include <sys/msgport2.h> 86 87 #include <machine/limits.h> 88 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/route.h> 92 #include <net/netisr2.h> 93 #include <net/toeplitz2.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip_var.h> 99 #ifdef INET6 100 #include <netinet/ip6.h> 101 #include <netinet6/ip6_var.h> 102 #endif /* INET6 */ 103 104 #ifdef IPSEC 105 #include <netinet6/ipsec.h> 106 #include <netproto/key/key.h> 107 #include <netproto/ipsec/esp_var.h> 108 #endif 109 110 #ifdef FAST_IPSEC 111 #if defined(IPSEC) || defined(IPSEC_ESP) 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!" 113 #endif 114 115 #include <netproto/ipsec/ipsec.h> 116 #include <netproto/ipsec/key.h> 117 #define IPSEC 118 #endif /* FAST_IPSEC */ 119 120 #define INP_LOCALGROUP_SIZMIN 8 121 #define INP_LOCALGROUP_SIZMAX 256 122 123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash, 124 struct in_addr laddr, u_int lport_arg, int wild_okay, 125 struct ucred *cred); 126 127 struct in_addr zeroin_addr; 128 129 /* 130 * These configure the range of local port addresses assigned to 131 * "unspecified" outgoing connections/packets/whatever. 132 */ 133 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 134 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 135 136 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 137 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 138 139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 140 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 141 142 #define RANGECHK(var, min, max) \ 143 if ((var) < (min)) { (var) = (min); } \ 144 else if ((var) > (max)) { (var) = (max); } 145 146 int udpencap_enable = 1; /* enabled by default */ 147 int udpencap_port = 4500; /* triggers decapsulation */ 148 149 /* 150 * Per-netisr inpcb markers. 151 * NOTE: they should only be used in netisrs. 152 */ 153 static struct inpcb *in_pcbmarkers; 154 static struct inpcontainer *in_pcbcontainer_markers; 155 156 static int 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 158 { 159 int error; 160 161 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 162 if (!error) { 163 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 164 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 165 166 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 167 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 168 169 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 170 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 171 } 172 return (error); 173 } 174 175 #undef RANGECHK 176 177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 178 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 180 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 182 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 184 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 186 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 188 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 190 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 191 192 /* Initialized by ip_init() */ 193 int ip_porthash_trycount; 194 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW, 195 &ip_porthash_trycount, 0, 196 "Number of tries to find local port matching hash of 4-tuple"); 197 198 /* 199 * in_pcb.c: manage the Protocol Control Blocks. 200 * 201 * NOTE: It is assumed that most of these functions will be called from 202 * a critical section. XXX - There are, unfortunately, a few exceptions 203 * to this rule that should be fixed. 204 * 205 * NOTE: The caller should initialize the cpu field to the cpu running the 206 * protocol stack associated with this inpcbinfo. 207 */ 208 209 void 210 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 211 { 212 KASSERT(cpu >= 0 && cpu < netisr_ncpus, ("invalid cpu%d", cpu)); 213 pcbinfo->cpu = cpu; 214 215 LIST_INIT(&pcbinfo->pcblisthead); 216 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 217 M_WAITOK | M_ZERO); 218 219 if (shared) { 220 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 221 M_PCB, M_WAITOK); 222 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 223 } else { 224 pcbinfo->infotoken = NULL; 225 } 226 } 227 228 void 229 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo, 230 int portinfo_cnt) 231 { 232 233 KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt)); 234 pcbinfo->portinfo = portinfo; 235 pcbinfo->portinfo_cnt = portinfo_cnt; 236 } 237 238 struct baddynamicports baddynamicports; 239 240 /* 241 * Check if the specified port is invalid for dynamic allocation. 242 */ 243 int 244 in_baddynamic(u_int16_t port, u_int16_t proto) 245 { 246 switch (proto) { 247 case IPPROTO_TCP: 248 return (DP_ISSET(baddynamicports.tcp, port)); 249 case IPPROTO_UDP: 250 #ifdef IPSEC 251 /* Cannot preset this as it is a sysctl */ 252 if (port == udpencap_port) 253 return (1); 254 #endif 255 return (DP_ISSET(baddynamicports.udp, port)); 256 default: 257 return (0); 258 } 259 } 260 261 void 262 in_pcbonlist(struct inpcb *inp) 263 { 264 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 265 266 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 267 ("not in the correct netisr")); 268 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 269 inp->inp_flags |= INP_ONLIST; 270 271 GET_PCBINFO_TOKEN(pcbinfo); 272 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 273 pcbinfo->ipi_count++; 274 REL_PCBINFO_TOKEN(pcbinfo); 275 } 276 277 void 278 in_pcbofflist(struct inpcb *inp) 279 { 280 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 281 282 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 283 ("not in the correct netisr")); 284 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 285 inp->inp_flags &= ~INP_ONLIST; 286 287 GET_PCBINFO_TOKEN(pcbinfo); 288 LIST_REMOVE(inp, inp_list); 289 KASSERT(pcbinfo->ipi_count > 0, 290 ("invalid inpcb count %d", pcbinfo->ipi_count)); 291 pcbinfo->ipi_count--; 292 REL_PCBINFO_TOKEN(pcbinfo); 293 } 294 295 /* 296 * Allocate a PCB and associate it with the socket. 297 */ 298 int 299 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 300 { 301 struct inpcb *inp; 302 #ifdef IPSEC 303 int error; 304 #endif 305 306 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 307 if (inp == NULL) 308 return (ENOMEM); 309 inp->inp_lgrpindex = -1; 310 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 311 inp->inp_pcbinfo = pcbinfo; 312 inp->inp_socket = so; 313 #ifdef IPSEC 314 error = ipsec_init_policy(so, &inp->inp_sp); 315 if (error != 0) { 316 kfree(inp, M_PCB); 317 return (error); 318 } 319 #endif 320 #ifdef INET6 321 if (INP_CHECK_SOCKAF(so, AF_INET6)) { 322 if (ip6_auto_flowlabel) 323 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 324 inp->inp_af = AF_INET6; 325 } else 326 #endif 327 inp->inp_af = AF_INET; 328 soreference(so); 329 so->so_pcb = inp; 330 331 in_pcbonlist(inp); 332 return (0); 333 } 334 335 /* 336 * Unlink a pcb with the intention of moving it to another cpu with a 337 * different pcbinfo. While unlinked nothing should attempt to dereference 338 * inp_pcbinfo, NULL it out so we assert if it does. 339 */ 340 void 341 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 342 { 343 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 344 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 345 ("already linked")); 346 347 in_pcbofflist(inp); 348 inp->inp_pcbinfo = NULL; 349 } 350 351 void 352 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 353 { 354 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 355 } 356 357 /* 358 * Relink a pcb into a new pcbinfo. 359 */ 360 void 361 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 362 { 363 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 364 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 365 ("already linked")); 366 367 inp->inp_pcbinfo = pcbinfo; 368 in_pcbonlist(inp); 369 } 370 371 void 372 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 373 { 374 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 375 } 376 377 static boolean_t 378 in_pcbporthash_update(struct inpcbportinfo *portinfo, 379 struct inpcb *inp, u_short lport, struct ucred *cred, int wild) 380 { 381 struct inpcbporthead *porthash; 382 383 /* 384 * This has to be atomic. If the porthash is shared across multiple 385 * protocol threads, e.g. tcp and udp, then the token must be held. 386 */ 387 porthash = in_pcbporthash_head(portinfo, lport); 388 GET_PORTHASH_TOKEN(porthash); 389 390 if (in_pcblookup_local(porthash, inp->inp_laddr, lport, 391 wild, cred) != NULL) { 392 REL_PORTHASH_TOKEN(porthash); 393 return FALSE; 394 } 395 inp->inp_lport = lport; 396 in_pcbinsporthash(porthash, inp); 397 398 REL_PORTHASH_TOKEN(porthash); 399 return TRUE; 400 } 401 402 static int 403 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 404 { 405 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 406 struct inpcbportinfo *portinfo; 407 u_short first, last, lport, step, first0, last0; 408 int count, error; 409 int portinfo_first, portinfo_idx; 410 uint32_t cut; 411 412 inp->inp_flags |= INP_ANONPORT; 413 414 step = pcbinfo->portinfo_cnt; 415 portinfo_first = mycpuid % pcbinfo->portinfo_cnt; 416 portinfo_idx = portinfo_first; 417 418 if (inp->inp_flags & INP_HIGHPORT) { 419 first0 = ipport_hifirstauto; /* sysctl */ 420 last0 = ipport_hilastauto; 421 } else if (inp->inp_flags & INP_LOWPORT) { 422 if (cred && 423 (error = 424 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 425 inp->inp_laddr.s_addr = INADDR_ANY; 426 return error; 427 } 428 first0 = ipport_lowfirstauto; /* 1023 */ 429 last0 = ipport_lowlastauto; /* 600 */ 430 } else { 431 first0 = ipport_firstauto; /* sysctl */ 432 last0 = ipport_lastauto; 433 } 434 if (first0 > last0) { 435 lport = last0; 436 last0 = first0; 437 first0 = lport; 438 } 439 KKASSERT(last0 >= first0); 440 441 cut = karc4random(); 442 loop: 443 portinfo = &pcbinfo->portinfo[portinfo_idx]; 444 first = first0; 445 last = last0; 446 447 /* 448 * Simple check to ensure all ports are not used up causing 449 * a deadlock here. 450 */ 451 in_pcbportrange(&last, &first, portinfo->offset, step); 452 lport = last - first; 453 count = lport / step; 454 455 lport = rounddown(cut % lport, step) + first; 456 KKASSERT(lport % step == portinfo->offset); 457 458 for (;;) { 459 if (count-- < 0) { /* completely used? */ 460 error = EADDRNOTAVAIL; 461 break; 462 } 463 464 if (__predict_false(lport < first || lport > last)) { 465 lport = first; 466 KKASSERT(lport % step == portinfo->offset); 467 } 468 469 if (in_pcbporthash_update(portinfo, inp, htons(lport), 470 cred, wild)) { 471 error = 0; 472 break; 473 } 474 475 lport += step; 476 KKASSERT(lport % step == portinfo->offset); 477 } 478 479 if (error) { 480 /* Try next portinfo */ 481 portinfo_idx++; 482 portinfo_idx %= pcbinfo->portinfo_cnt; 483 if (portinfo_idx != portinfo_first) 484 goto loop; 485 inp->inp_laddr.s_addr = INADDR_ANY; 486 } 487 return error; 488 } 489 490 int 491 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 492 { 493 struct socket *so = inp->inp_socket; 494 struct sockaddr_in jsin; 495 struct ucred *cred = NULL; 496 int wild = 0; 497 498 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 499 return (EADDRNOTAVAIL); 500 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 501 return (EINVAL); /* already bound */ 502 503 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 504 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 505 if (td->td_proc) 506 cred = td->td_proc->p_ucred; 507 508 if (nam != NULL) { 509 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 510 struct inpcbinfo *pcbinfo; 511 struct inpcbportinfo *portinfo; 512 struct inpcbporthead *porthash; 513 struct inpcb *t; 514 u_short lport, lport_ho; 515 int reuseport = (so->so_options & SO_REUSEPORT); 516 int error; 517 518 if (nam->sa_len != sizeof *sin) 519 return (EINVAL); 520 #ifdef notdef 521 /* 522 * We should check the family, but old programs 523 * incorrectly fail to initialize it. 524 */ 525 if (sin->sin_family != AF_INET) 526 return (EAFNOSUPPORT); 527 #endif 528 if (!prison_replace_wildcards(td, nam)) 529 return (EINVAL); 530 531 lport = sin->sin_port; 532 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 533 /* 534 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 535 * allow complete duplication of binding if 536 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 537 * and a multicast address is bound on both 538 * new and duplicated sockets. 539 */ 540 if (so->so_options & SO_REUSEADDR) 541 reuseport = SO_REUSEADDR | SO_REUSEPORT; 542 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 543 sin->sin_port = 0; /* yech... */ 544 bzero(&sin->sin_zero, sizeof sin->sin_zero); 545 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 546 return (EADDRNOTAVAIL); 547 } 548 549 inp->inp_laddr = sin->sin_addr; 550 551 jsin.sin_family = AF_INET; 552 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 553 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 554 inp->inp_laddr.s_addr = INADDR_ANY; 555 return (EINVAL); 556 } 557 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 558 559 if (lport == 0) { 560 /* Auto-select local port */ 561 return in_pcbsetlport(inp, wild, cred); 562 } 563 lport_ho = ntohs(lport); 564 565 /* GROSS */ 566 if (lport_ho < IPPORT_RESERVED && cred && 567 (error = 568 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 569 inp->inp_laddr.s_addr = INADDR_ANY; 570 return (error); 571 } 572 573 /* 574 * Locate the proper portinfo based on lport 575 */ 576 pcbinfo = inp->inp_pcbinfo; 577 portinfo = 578 &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 579 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == 580 portinfo->offset); 581 582 /* 583 * This has to be atomic. If the porthash is shared across 584 * multiple protocol threads, e.g. tcp and udp then the token 585 * must be held. 586 */ 587 porthash = in_pcbporthash_head(portinfo, lport); 588 GET_PORTHASH_TOKEN(porthash); 589 590 if (so->so_cred->cr_uid != 0 && 591 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 592 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 593 INPLOOKUP_WILDCARD, cred); 594 if (t && 595 (so->so_cred->cr_uid != 596 t->inp_socket->so_cred->cr_uid)) { 597 inp->inp_laddr.s_addr = INADDR_ANY; 598 error = EADDRINUSE; 599 goto done; 600 } 601 } 602 if (cred && !prison_replace_wildcards(td, nam)) { 603 inp->inp_laddr.s_addr = INADDR_ANY; 604 error = EADDRNOTAVAIL; 605 goto done; 606 } 607 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 608 wild, cred); 609 if (t && !(reuseport & t->inp_socket->so_options)) { 610 inp->inp_laddr.s_addr = INADDR_ANY; 611 error = EADDRINUSE; 612 goto done; 613 } 614 inp->inp_lport = lport; 615 in_pcbinsporthash(porthash, inp); 616 error = 0; 617 done: 618 REL_PORTHASH_TOKEN(porthash); 619 return (error); 620 } else { 621 jsin.sin_family = AF_INET; 622 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 623 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 624 inp->inp_laddr.s_addr = INADDR_ANY; 625 return (EINVAL); 626 } 627 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 628 629 return in_pcbsetlport(inp, wild, cred); 630 } 631 } 632 633 static struct inpcb * 634 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr, 635 u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred) 636 { 637 struct inpcb *inp; 638 struct inpcbport *phd; 639 struct inpcb *match = NULL; 640 641 /* 642 * If the porthashbase is shared across several cpus, it must 643 * have been locked. 644 */ 645 ASSERT_PORTHASH_TOKEN_HELD(porthash); 646 647 /* 648 * Best fit PCB lookup. 649 * 650 * First see if this local port is in use by looking on the 651 * port hash list. 652 */ 653 LIST_FOREACH(phd, porthash, phd_hash) { 654 if (phd->phd_port == lport) 655 break; 656 } 657 if (phd != NULL) { 658 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 659 #ifdef INET6 660 if (!INP_ISIPV4(inp)) 661 continue; 662 #endif 663 if (inp->inp_laddr.s_addr != INADDR_ANY && 664 inp->inp_laddr.s_addr != laddr.s_addr) 665 continue; 666 667 if (inp->inp_faddr.s_addr != INADDR_ANY && 668 inp->inp_faddr.s_addr != faddr.s_addr) 669 continue; 670 671 if (inp->inp_fport != 0 && inp->inp_fport != fport) 672 continue; 673 674 if (cred == NULL || 675 cred->cr_prison == 676 inp->inp_socket->so_cred->cr_prison) { 677 match = inp; 678 break; 679 } 680 } 681 } 682 return (match); 683 } 684 685 static boolean_t 686 in_pcbporthash_update4(struct inpcbportinfo *portinfo, 687 struct inpcb *inp, u_short lport, const struct sockaddr_in *sin, 688 struct ucred *cred) 689 { 690 struct inpcbporthead *porthash; 691 692 /* 693 * This has to be atomic. If the porthash is shared across multiple 694 * protocol threads, e.g. tcp and udp, then the token must be held. 695 */ 696 porthash = in_pcbporthash_head(portinfo, lport); 697 GET_PORTHASH_TOKEN(porthash); 698 699 if (in_pcblookup_localremote(porthash, inp->inp_laddr, 700 lport, sin->sin_addr, sin->sin_port, cred) != NULL) { 701 REL_PORTHASH_TOKEN(porthash); 702 return FALSE; 703 } 704 inp->inp_lport = lport; 705 in_pcbinsporthash(porthash, inp); 706 707 REL_PORTHASH_TOKEN(porthash); 708 return TRUE; 709 } 710 711 int 712 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 713 struct thread *td) 714 { 715 struct proc *p = td->td_proc; 716 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 717 struct sockaddr_in jsin; 718 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 719 struct ucred *cred = NULL; 720 u_short first, last, lport; 721 int count, hash_count; 722 int error, selfconn = 0; 723 int cpuid = mycpuid; 724 uint32_t hash_base = 0, hash; 725 726 ASSERT_NETISR_NCPUS(cpuid); 727 728 if (TAILQ_EMPTY(&in_ifaddrheads[cpuid])) /* XXX broken! */ 729 return (EADDRNOTAVAIL); 730 731 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 732 if (inp->inp_lport != 0) 733 return (EINVAL); /* already bound */ 734 735 KKASSERT(p); 736 cred = p->p_ucred; 737 738 jsin.sin_family = AF_INET; 739 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 740 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 741 inp->inp_laddr.s_addr = INADDR_ANY; 742 return (EINVAL); 743 } 744 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 745 746 hash_count = ip_porthash_trycount; 747 if (hash_count > 0) { 748 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^ 749 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^ 750 toeplitz_piecemeal_port(sin->sin_port); 751 } else { 752 hash_count = 0; 753 } 754 755 inp->inp_flags |= INP_ANONPORT; 756 757 if (inp->inp_flags & INP_HIGHPORT) { 758 first = ipport_hifirstauto; /* sysctl */ 759 last = ipport_hilastauto; 760 } else if (inp->inp_flags & INP_LOWPORT) { 761 if (cred && 762 (error = 763 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 764 inp->inp_laddr.s_addr = INADDR_ANY; 765 return (error); 766 } 767 first = ipport_lowfirstauto; /* 1023 */ 768 last = ipport_lowlastauto; /* 600 */ 769 } else { 770 first = ipport_firstauto; /* sysctl */ 771 last = ipport_lastauto; 772 } 773 if (first > last) { 774 lport = last; 775 last = first; 776 first = lport; 777 } 778 KKASSERT(last >= first); 779 780 count = last - first; 781 lport = (karc4random() % count) + first; 782 count += hash_count; 783 784 /* 785 * Simple check to ensure all ports are not used up causing 786 * a deadlock here. 787 */ 788 for (;;) { 789 u_short lport_no; 790 791 if (count-- < 0) { /* completely used? */ 792 error = EADDRNOTAVAIL; 793 break; 794 } 795 796 if (__predict_false(lport < first || lport > last)) 797 lport = first; 798 lport_no = htons(lport); 799 800 /* This could happen on loopback interface */ 801 if (__predict_false(sin->sin_port == lport_no && 802 sin->sin_addr.s_addr == inp->inp_laddr.s_addr)) { 803 if (!selfconn) { 804 ++count; /* don't count this try */ 805 selfconn = 1; 806 } 807 goto next; 808 } 809 810 if (hash_count) { 811 --hash_count; 812 hash = hash_base ^ 813 toeplitz_piecemeal_port(lport_no); 814 if (netisr_hashcpu(hash) != cpuid && hash_count) 815 goto next; 816 } 817 818 if (in_pcbporthash_update4( 819 &pcbinfo->portinfo[lport % pcbinfo->portinfo_cnt], 820 inp, lport_no, sin, cred)) { 821 error = 0; 822 break; 823 } 824 next: 825 ++lport; 826 } 827 828 if (error) 829 inp->inp_laddr.s_addr = INADDR_ANY; 830 return (error); 831 } 832 833 /* 834 * Transform old in_pcbconnect() into an inner subroutine for new 835 * in_pcbconnect(): Do some validity-checking on the remote 836 * address (in mbuf 'nam') and then determine local host address 837 * (i.e., which interface) to use to access that remote host. 838 * 839 * This preserves definition of in_pcbconnect(), while supporting a 840 * slightly different version for T/TCP. (This is more than 841 * a bit of a kludge, but cleaning up the internal interfaces would 842 * have forced minor changes in every protocol). 843 */ 844 int 845 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 846 struct sockaddr_in **plocal_sin, struct thread *td, int find) 847 { 848 struct in_ifaddr *ia; 849 struct ucred *cred = NULL; 850 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 851 struct sockaddr *jsin; 852 int jailed = 0, alloc_route = 0; 853 854 if (nam->sa_len != sizeof *sin) 855 return (EINVAL); 856 if (sin->sin_family != AF_INET) 857 return (EAFNOSUPPORT); 858 if (sin->sin_port == 0) 859 return (EADDRNOTAVAIL); 860 if (td && td->td_proc && td->td_proc->p_ucred) 861 cred = td->td_proc->p_ucred; 862 if (cred && cred->cr_prison) 863 jailed = 1; 864 if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) { 865 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 866 /* 867 * If the destination address is INADDR_ANY, 868 * use the primary local address. 869 * If the supplied address is INADDR_BROADCAST, 870 * and the primary interface supports broadcast, 871 * choose the broadcast address for that interface. 872 */ 873 if (sin->sin_addr.s_addr == INADDR_ANY) 874 sin->sin_addr = IA_SIN(ia)->sin_addr; 875 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST && 876 (ia->ia_ifp->if_flags & IFF_BROADCAST)) 877 sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr; 878 } 879 if (find) { 880 struct route *ro; 881 882 ia = NULL; 883 /* 884 * If route is known or can be allocated now, 885 * our src addr is taken from the i/f, else punt. 886 * Note that we should check the address family of the cached 887 * destination, in case of sharing the cache with IPv6. 888 */ 889 ro = &inp->inp_route; 890 if (ro->ro_rt && 891 (!(ro->ro_rt->rt_flags & RTF_UP) || 892 ro->ro_dst.sa_family != AF_INET || 893 satosin(&ro->ro_dst)->sin_addr.s_addr != 894 sin->sin_addr.s_addr || 895 inp->inp_socket->so_options & SO_DONTROUTE)) { 896 RTFREE(ro->ro_rt); 897 ro->ro_rt = NULL; 898 } 899 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 900 (ro->ro_rt == NULL || 901 ro->ro_rt->rt_ifp == NULL)) { 902 /* No route yet, so try to acquire one */ 903 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 904 ro->ro_dst.sa_family = AF_INET; 905 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 906 ((struct sockaddr_in *) &ro->ro_dst)->sin_addr = 907 sin->sin_addr; 908 rtalloc(ro); 909 alloc_route = 1; 910 } 911 /* 912 * If we found a route, use the address 913 * corresponding to the outgoing interface 914 * unless it is the loopback (in case a route 915 * to our address on another net goes to loopback). 916 */ 917 if (ro->ro_rt && 918 !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) { 919 if (jailed) { 920 if (jailed_ip(cred->cr_prison, 921 ro->ro_rt->rt_ifa->ifa_addr)) { 922 ia = ifatoia(ro->ro_rt->rt_ifa); 923 } 924 } else { 925 ia = ifatoia(ro->ro_rt->rt_ifa); 926 } 927 } 928 if (ia == NULL) { 929 u_short fport = sin->sin_port; 930 931 sin->sin_port = 0; 932 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 933 if (ia && jailed && !jailed_ip(cred->cr_prison, 934 sintosa(&ia->ia_addr))) 935 ia = NULL; 936 if (ia == NULL) 937 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 938 if (ia && jailed && !jailed_ip(cred->cr_prison, 939 sintosa(&ia->ia_addr))) 940 ia = NULL; 941 sin->sin_port = fport; 942 if (ia == NULL && 943 !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 944 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 945 if (ia && jailed && !jailed_ip(cred->cr_prison, 946 sintosa(&ia->ia_addr))) 947 ia = NULL; 948 949 if (!jailed && ia == NULL) 950 goto fail; 951 } 952 /* 953 * If the destination address is multicast and an outgoing 954 * interface has been set as a multicast option, use the 955 * address of that interface as our source address. 956 */ 957 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 958 inp->inp_moptions != NULL) { 959 struct ip_moptions *imo; 960 struct ifnet *ifp; 961 962 imo = inp->inp_moptions; 963 if ((ifp = imo->imo_multicast_ifp) != NULL) { 964 struct in_ifaddr_container *iac; 965 966 ia = NULL; 967 TAILQ_FOREACH(iac, 968 &in_ifaddrheads[mycpuid], ia_link) { 969 if (iac->ia->ia_ifp == ifp) { 970 ia = iac->ia; 971 break; 972 } 973 } 974 if (ia == NULL) 975 goto fail; 976 } 977 } 978 /* 979 * Don't do pcblookup call here; return interface in plocal_sin 980 * and exit to caller, that will do the lookup. 981 */ 982 if (ia == NULL && jailed) { 983 if ((jsin = prison_get_nonlocal( 984 cred->cr_prison, AF_INET, NULL)) != NULL || 985 (jsin = prison_get_local( 986 cred->cr_prison, AF_INET, NULL)) != NULL) { 987 *plocal_sin = satosin(jsin); 988 } else { 989 /* IPv6 only Jail */ 990 goto fail; 991 } 992 } else { 993 *plocal_sin = &ia->ia_addr; 994 } 995 } 996 return (0); 997 fail: 998 if (alloc_route) 999 in_pcbresetroute(inp); 1000 return (EADDRNOTAVAIL); 1001 } 1002 1003 int 1004 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 1005 struct sockaddr_in **plocal_sin, struct thread *td) 1006 { 1007 return in_pcbladdr_find(inp, nam, plocal_sin, td, 1008 (inp->inp_laddr.s_addr == INADDR_ANY)); 1009 } 1010 1011 /* 1012 * Outer subroutine: 1013 * Connect from a socket to a specified address. 1014 * Both address and port must be specified in argument sin. 1015 * If don't have a local address for this socket yet, 1016 * then pick one. 1017 */ 1018 int 1019 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 1020 { 1021 struct sockaddr_in *if_sin; 1022 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1023 int error; 1024 1025 if_sin = NULL; /* avoid gcc warnings */ 1026 1027 /* Call inner routine to assign local interface address. */ 1028 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1029 return (error); 1030 1031 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1032 inp->inp_laddr.s_addr ? 1033 inp->inp_laddr : if_sin->sin_addr, 1034 inp->inp_lport, FALSE, NULL) != NULL) { 1035 return (EADDRINUSE); 1036 } 1037 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1038 if (inp->inp_lport == 0) { 1039 error = in_pcbbind(inp, NULL, td); 1040 if (error) 1041 return (error); 1042 } 1043 inp->inp_laddr = if_sin->sin_addr; 1044 } 1045 inp->inp_faddr = sin->sin_addr; 1046 inp->inp_fport = sin->sin_port; 1047 in_pcbinsconnhash(inp); 1048 return (0); 1049 } 1050 1051 void 1052 in_pcbdisconnect(struct inpcb *inp) 1053 { 1054 1055 in_pcbremconnhash(inp); 1056 inp->inp_faddr.s_addr = INADDR_ANY; 1057 inp->inp_fport = 0; 1058 } 1059 1060 void 1061 in_pcbdetach(struct inpcb *inp) 1062 { 1063 struct socket *so = inp->inp_socket; 1064 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1065 1066 #ifdef IPSEC 1067 ipsec4_delete_pcbpolicy(inp); 1068 #endif /*IPSEC*/ 1069 inp->inp_gencnt = ++ipi->ipi_gencnt; 1070 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1071 in_pcbremlists(inp); 1072 so->so_pcb = NULL; 1073 sofree(so); /* remove pcb ref */ 1074 if (inp->inp_options) 1075 m_free(inp->inp_options); 1076 if (inp->inp_route.ro_rt) 1077 rtfree(inp->inp_route.ro_rt); 1078 ip_freemoptions(inp->inp_moptions); 1079 kfree(inp, M_PCB); 1080 } 1081 1082 /* 1083 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1084 * socket received RST. 1085 */ 1086 static int 1087 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1088 { 1089 struct inpcb *inp; 1090 struct sockaddr_in *sin; 1091 1092 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1093 inp = so->so_pcb; 1094 if (!inp) 1095 return (ECONNRESET); 1096 1097 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1098 sin->sin_family = AF_INET; 1099 sin->sin_len = sizeof *sin; 1100 sin->sin_port = inp->inp_lport; 1101 sin->sin_addr = inp->inp_laddr; 1102 1103 *nam = (struct sockaddr *)sin; 1104 return (0); 1105 } 1106 1107 void 1108 in_setsockaddr_dispatch(netmsg_t msg) 1109 { 1110 int error; 1111 1112 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1113 lwkt_replymsg(&msg->lmsg, error); 1114 } 1115 1116 /* 1117 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1118 * socket received RST. 1119 */ 1120 int 1121 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1122 { 1123 struct inpcb *inp; 1124 struct sockaddr_in *sin; 1125 1126 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1127 inp = so->so_pcb; 1128 if (!inp) 1129 return (ECONNRESET); 1130 1131 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1132 sin->sin_family = AF_INET; 1133 sin->sin_len = sizeof *sin; 1134 sin->sin_port = inp->inp_fport; 1135 sin->sin_addr = inp->inp_faddr; 1136 1137 *nam = (struct sockaddr *)sin; 1138 return (0); 1139 } 1140 1141 void 1142 in_setpeeraddr_dispatch(netmsg_t msg) 1143 { 1144 int error; 1145 1146 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1147 lwkt_replymsg(&msg->lmsg, error); 1148 } 1149 1150 void 1151 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1152 inp_notify_t notify) 1153 { 1154 struct inpcb *inp, *marker; 1155 1156 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1157 ("not in the correct netisr")); 1158 marker = in_pcbmarker(); 1159 1160 /* 1161 * NOTE: 1162 * - If INP_PLACEMARKER is set we must ignore the rest of the 1163 * structure and skip it. 1164 * - It is safe to nuke inpcbs here, since we are in their own 1165 * netisr. 1166 */ 1167 GET_PCBINFO_TOKEN(pcbinfo); 1168 1169 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1170 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1171 LIST_REMOVE(marker, inp_list); 1172 LIST_INSERT_AFTER(inp, marker, inp_list); 1173 1174 if (inp->inp_flags & INP_PLACEMARKER) 1175 continue; 1176 #ifdef INET6 1177 if (!INP_ISIPV4(inp)) 1178 continue; 1179 #endif 1180 if (inp->inp_faddr.s_addr != faddr.s_addr || 1181 inp->inp_socket == NULL) 1182 continue; 1183 (*notify)(inp, err); /* can remove inp from list! */ 1184 } 1185 LIST_REMOVE(marker, inp_list); 1186 1187 REL_PCBINFO_TOKEN(pcbinfo); 1188 } 1189 1190 void 1191 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1192 { 1193 struct inpcb *inp, *marker; 1194 1195 /* 1196 * We only need to make sure that we are in netisr0, where all 1197 * multicast operation happen. We could check inpcbinfo which 1198 * does not belong to netisr0 by holding the inpcbinfo's token. 1199 * In this case, the pcbinfo must be able to be shared, i.e. 1200 * pcbinfo->infotoken is not NULL. 1201 */ 1202 ASSERT_NETISR0; 1203 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1204 ("pcbinfo could not be shared")); 1205 1206 /* 1207 * Get a marker for the current netisr (netisr0). 1208 * 1209 * It is possible that the multicast address deletion blocks, 1210 * which could cause temporary token releasing. So we use 1211 * inpcb marker here to get a coherent view of the inpcb list. 1212 * 1213 * While, on the other hand, moptions are only added and deleted 1214 * in netisr0, so we would not see staled moption or miss moption 1215 * even if the token was released due to the blocking multicast 1216 * address deletion. 1217 */ 1218 marker = in_pcbmarker(); 1219 1220 GET_PCBINFO_TOKEN(pcbinfo); 1221 1222 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1223 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1224 struct ip_moptions *imo; 1225 1226 LIST_REMOVE(marker, inp_list); 1227 LIST_INSERT_AFTER(inp, marker, inp_list); 1228 1229 if (inp->inp_flags & INP_PLACEMARKER) 1230 continue; 1231 imo = inp->inp_moptions; 1232 if (INP_ISIPV4(inp) && imo != NULL) { 1233 int i, gap; 1234 1235 /* 1236 * Unselect the outgoing interface if it is being 1237 * detached. 1238 */ 1239 if (imo->imo_multicast_ifp == ifp) 1240 imo->imo_multicast_ifp = NULL; 1241 1242 /* 1243 * Drop multicast group membership if we joined 1244 * through the interface being detached. 1245 */ 1246 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1247 i++) { 1248 if (imo->imo_membership[i]->inm_ifp == ifp) { 1249 /* 1250 * NOTE: 1251 * This could block and the pcbinfo 1252 * token could be passively released. 1253 */ 1254 in_delmulti(imo->imo_membership[i]); 1255 gap++; 1256 } else if (gap != 0) 1257 imo->imo_membership[i - gap] = 1258 imo->imo_membership[i]; 1259 } 1260 imo->imo_num_memberships -= gap; 1261 } 1262 } 1263 LIST_REMOVE(marker, inp_list); 1264 1265 REL_PCBINFO_TOKEN(pcbinfo); 1266 } 1267 1268 /* 1269 * Check for alternatives when higher level complains 1270 * about service problems. For now, invalidate cached 1271 * routing information. If the route was created dynamically 1272 * (by a redirect), time to try a default gateway again. 1273 */ 1274 void 1275 in_losing(struct inpcb *inp) 1276 { 1277 struct rtentry *rt; 1278 struct rt_addrinfo rtinfo; 1279 1280 if ((rt = inp->inp_route.ro_rt)) { 1281 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1282 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1283 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1284 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1285 rtinfo.rti_flags = rt->rt_flags; 1286 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1287 if (rt->rt_flags & RTF_DYNAMIC) { 1288 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1289 rt_mask(rt), rt->rt_flags, NULL); 1290 } 1291 inp->inp_route.ro_rt = NULL; 1292 rtfree(rt); 1293 /* 1294 * A new route can be allocated 1295 * the next time output is attempted. 1296 */ 1297 } 1298 } 1299 1300 /* 1301 * After a routing change, flush old routing 1302 * and allocate a (hopefully) better one. 1303 */ 1304 void 1305 in_rtchange(struct inpcb *inp, int err) 1306 { 1307 if (inp->inp_route.ro_rt) { 1308 rtfree(inp->inp_route.ro_rt); 1309 inp->inp_route.ro_rt = NULL; 1310 /* 1311 * A new route can be allocated the next time 1312 * output is attempted. 1313 */ 1314 } 1315 } 1316 1317 /* 1318 * Lookup a PCB based on the local address and port. 1319 */ 1320 static struct inpcb * 1321 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr, 1322 u_int lport_arg, int wild_okay, struct ucred *cred) 1323 { 1324 struct inpcb *inp; 1325 int matchwild = 3, wildcard; 1326 u_short lport = lport_arg; 1327 struct inpcbport *phd; 1328 struct inpcb *match = NULL; 1329 1330 /* 1331 * If the porthashbase is shared across several cpus, it must 1332 * have been locked. 1333 */ 1334 ASSERT_PORTHASH_TOKEN_HELD(porthash); 1335 1336 /* 1337 * Best fit PCB lookup. 1338 * 1339 * First see if this local port is in use by looking on the 1340 * port hash list. 1341 */ 1342 LIST_FOREACH(phd, porthash, phd_hash) { 1343 if (phd->phd_port == lport) 1344 break; 1345 } 1346 if (phd != NULL) { 1347 /* 1348 * Port is in use by one or more PCBs. Look for best 1349 * fit. 1350 */ 1351 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1352 wildcard = 0; 1353 #ifdef INET6 1354 if (!INP_ISIPV4(inp)) 1355 continue; 1356 #endif 1357 if (inp->inp_faddr.s_addr != INADDR_ANY) 1358 wildcard++; 1359 if (inp->inp_laddr.s_addr != INADDR_ANY) { 1360 if (laddr.s_addr == INADDR_ANY) 1361 wildcard++; 1362 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1363 continue; 1364 } else { 1365 if (laddr.s_addr != INADDR_ANY) 1366 wildcard++; 1367 } 1368 if (wildcard && !wild_okay) 1369 continue; 1370 if (wildcard < matchwild && 1371 (cred == NULL || 1372 cred->cr_prison == 1373 inp->inp_socket->so_cred->cr_prison)) { 1374 match = inp; 1375 matchwild = wildcard; 1376 if (matchwild == 0) { 1377 break; 1378 } 1379 } 1380 } 1381 } 1382 return (match); 1383 } 1384 1385 struct inpcb * 1386 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1387 const struct inpcb *inp) 1388 { 1389 const struct inp_localgrphead *hdr; 1390 const struct inp_localgroup *grp; 1391 int i; 1392 1393 if (pcbinfo->localgrphashbase == NULL) 1394 return NULL; 1395 1396 GET_PCBINFO_TOKEN(pcbinfo); 1397 1398 hdr = &pcbinfo->localgrphashbase[ 1399 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1400 1401 LIST_FOREACH(grp, hdr, il_list) { 1402 if (grp->il_af == inp->inp_af && 1403 grp->il_lport == inp->inp_lport && 1404 memcmp(&grp->il_dependladdr, 1405 &inp->inp_inc.inc_ie.ie_dependladdr, 1406 sizeof(grp->il_dependladdr)) == 0) { 1407 break; 1408 } 1409 } 1410 if (grp == NULL || grp->il_inpcnt == 1) { 1411 REL_PCBINFO_TOKEN(pcbinfo); 1412 return NULL; 1413 } 1414 1415 KASSERT(grp->il_inpcnt >= 2, 1416 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1417 for (i = 0; i < grp->il_inpcnt; ++i) { 1418 if (grp->il_inp[i] == inp) { 1419 int last = grp->il_inpcnt - 1; 1420 1421 if (i == last) 1422 last = grp->il_inpcnt - 2; 1423 REL_PCBINFO_TOKEN(pcbinfo); 1424 return grp->il_inp[last]; 1425 } 1426 } 1427 REL_PCBINFO_TOKEN(pcbinfo); 1428 return NULL; 1429 } 1430 1431 static struct inpcb * 1432 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1433 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1434 { 1435 struct inpcb *local_wild = NULL; 1436 const struct inp_localgrphead *hdr; 1437 const struct inp_localgroup *grp; 1438 1439 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1440 1441 hdr = &pcbinfo->localgrphashbase[ 1442 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1443 1444 /* 1445 * Order of socket selection: 1446 * 1. non-wild. 1447 * 2. wild. 1448 * 1449 * NOTE: Local group does not contain jailed sockets 1450 */ 1451 LIST_FOREACH(grp, hdr, il_list) { 1452 #ifdef INET6 1453 if (grp->il_af != AF_INET) 1454 continue; 1455 #endif 1456 if (grp->il_lport == lport) { 1457 int idx; 1458 1459 /* 1460 * Modulo-N is used here, which greatly reduces 1461 * completion queue token contention, thus more 1462 * cpu time is saved. 1463 */ 1464 idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt; 1465 if (grp->il_laddr.s_addr == laddr.s_addr) 1466 return grp->il_inp[idx]; 1467 else if (grp->il_laddr.s_addr == INADDR_ANY) 1468 local_wild = grp->il_inp[idx]; 1469 } 1470 } 1471 if (local_wild != NULL) 1472 return local_wild; 1473 return NULL; 1474 } 1475 1476 /* 1477 * Lookup PCB in hash list. 1478 */ 1479 struct inpcb * 1480 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1481 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1482 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1483 { 1484 struct inpcbhead *head; 1485 struct inpcb *inp, *jinp=NULL; 1486 u_short fport = fport_arg, lport = lport_arg; 1487 1488 /* 1489 * First look for an exact match. 1490 */ 1491 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1492 laddr.s_addr, lport, pcbinfo->hashmask)]; 1493 LIST_FOREACH(inp, head, inp_hash) { 1494 #ifdef INET6 1495 if (!INP_ISIPV4(inp)) 1496 continue; 1497 #endif 1498 if (in_hosteq(inp->inp_faddr, faddr) && 1499 in_hosteq(inp->inp_laddr, laddr) && 1500 inp->inp_fport == fport && inp->inp_lport == lport) { 1501 /* found */ 1502 if (inp->inp_socket == NULL || 1503 inp->inp_socket->so_cred->cr_prison == NULL) { 1504 return (inp); 1505 } else { 1506 if (jinp == NULL) 1507 jinp = inp; 1508 } 1509 } 1510 } 1511 if (jinp != NULL) 1512 return (jinp); 1513 1514 if (wildcard) { 1515 struct inpcb *local_wild = NULL; 1516 struct inpcb *jinp_wild = NULL; 1517 struct inpcontainer *ic; 1518 struct inpcontainerhead *chead; 1519 struct sockaddr_in jsin; 1520 struct ucred *cred; 1521 1522 GET_PCBINFO_TOKEN(pcbinfo); 1523 1524 /* 1525 * Check local group first 1526 */ 1527 if (pcbinfo->localgrphashbase != NULL && 1528 m != NULL && (m->m_flags & M_HASH)) { 1529 inp = inp_localgroup_lookup(pcbinfo, 1530 laddr, lport, m->m_pkthdr.hash); 1531 if (inp != NULL) { 1532 REL_PCBINFO_TOKEN(pcbinfo); 1533 return inp; 1534 } 1535 } 1536 1537 /* 1538 * Order of socket selection: 1539 * 1. non-jailed, non-wild. 1540 * 2. non-jailed, wild. 1541 * 3. jailed, non-wild. 1542 * 4. jailed, wild. 1543 */ 1544 jsin.sin_family = AF_INET; 1545 chead = &pcbinfo->wildcardhashbase[ 1546 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1547 LIST_FOREACH(ic, chead, ic_list) { 1548 inp = ic->ic_inp; 1549 if (inp->inp_flags & INP_PLACEMARKER) 1550 continue; 1551 1552 jsin.sin_addr.s_addr = laddr.s_addr; 1553 #ifdef INET6 1554 if (!INP_ISIPV4(inp)) 1555 continue; 1556 #endif 1557 if (inp->inp_socket != NULL) 1558 cred = inp->inp_socket->so_cred; 1559 else 1560 cred = NULL; 1561 if (cred != NULL && jailed(cred)) { 1562 if (jinp != NULL) 1563 continue; 1564 else 1565 if (!jailed_ip(cred->cr_prison, 1566 (struct sockaddr *)&jsin)) 1567 continue; 1568 } 1569 if (inp->inp_lport == lport) { 1570 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1571 if (cred != NULL && jailed(cred)) { 1572 jinp = inp; 1573 } else { 1574 REL_PCBINFO_TOKEN(pcbinfo); 1575 return (inp); 1576 } 1577 } 1578 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1579 if (cred != NULL && jailed(cred)) 1580 jinp_wild = inp; 1581 else 1582 local_wild = inp; 1583 } 1584 } 1585 } 1586 1587 REL_PCBINFO_TOKEN(pcbinfo); 1588 1589 if (local_wild != NULL) 1590 return (local_wild); 1591 if (jinp != NULL) 1592 return (jinp); 1593 return (jinp_wild); 1594 } 1595 1596 /* 1597 * Not found. 1598 */ 1599 return (NULL); 1600 } 1601 1602 struct inpcb * 1603 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1604 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1605 boolean_t wildcard, struct ifnet *ifp) 1606 { 1607 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1608 laddr, lport_arg, wildcard, ifp, NULL); 1609 } 1610 1611 /* 1612 * Insert PCB into connection hash table. 1613 */ 1614 void 1615 in_pcbinsconnhash(struct inpcb *inp) 1616 { 1617 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1618 struct inpcbhead *bucket; 1619 u_int32_t hashkey_faddr, hashkey_laddr; 1620 1621 #ifdef INET6 1622 if (INP_ISIPV6(inp)) { 1623 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1624 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1625 } else { 1626 #endif 1627 hashkey_faddr = inp->inp_faddr.s_addr; 1628 hashkey_laddr = inp->inp_laddr.s_addr; 1629 #ifdef INET6 1630 } 1631 #endif 1632 1633 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1634 ("not in the correct netisr")); 1635 ASSERT_INP_NOTINHASH(inp); 1636 inp->inp_flags |= INP_CONNECTED; 1637 1638 /* 1639 * Insert into the connection hash table. 1640 */ 1641 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1642 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1643 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1644 } 1645 1646 /* 1647 * Remove PCB from connection hash table. 1648 */ 1649 void 1650 in_pcbremconnhash(struct inpcb *inp) 1651 { 1652 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1653 1654 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1655 ("not in the correct netisr")); 1656 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1657 1658 LIST_REMOVE(inp, inp_hash); 1659 inp->inp_flags &= ~INP_CONNECTED; 1660 } 1661 1662 /* 1663 * Insert PCB into port hash table. 1664 */ 1665 void 1666 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp) 1667 { 1668 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1669 struct inpcbport *phd; 1670 1671 /* 1672 * If the porthashbase is shared across several cpus, it must 1673 * have been locked. 1674 */ 1675 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash); 1676 1677 /* 1678 * Insert into the port hash table. 1679 */ 1680 1681 /* Go through port list and look for a head for this lport. */ 1682 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1683 if (phd->phd_port == inp->inp_lport) 1684 break; 1685 } 1686 1687 /* If none exists, use saved one and tack it on. */ 1688 if (phd == NULL) { 1689 KKASSERT(pcbinfo->portsave != NULL); 1690 phd = pcbinfo->portsave; 1691 pcbinfo->portsave = NULL; 1692 phd->phd_port = inp->inp_lport; 1693 LIST_INIT(&phd->phd_pcblist); 1694 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1695 } 1696 1697 inp->inp_porthash = pcbporthash; 1698 inp->inp_phd = phd; 1699 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1700 1701 /* 1702 * Malloc one inpcbport for later use. It is safe to use 1703 * "wait" malloc here (port token would be released, if 1704 * malloc ever blocked), since all changes to the porthash 1705 * are done. 1706 */ 1707 if (pcbinfo->portsave == NULL) { 1708 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1709 M_PCB, M_INTWAIT | M_ZERO); 1710 } 1711 } 1712 1713 void 1714 in_pcbinsporthash_lport(struct inpcb *inp) 1715 { 1716 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1717 struct inpcbportinfo *portinfo; 1718 struct inpcbporthead *porthash; 1719 u_short lport_ho; 1720 1721 /* Locate the proper portinfo based on lport */ 1722 lport_ho = ntohs(inp->inp_lport); 1723 portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 1724 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset); 1725 1726 porthash = in_pcbporthash_head(portinfo, inp->inp_lport); 1727 GET_PORTHASH_TOKEN(porthash); 1728 in_pcbinsporthash(porthash, inp); 1729 REL_PORTHASH_TOKEN(porthash); 1730 } 1731 1732 void 1733 in_pcbremporthash(struct inpcb *inp) 1734 { 1735 struct inpcbporthead *porthash; 1736 struct inpcbport *phd; 1737 1738 if (inp->inp_phd == NULL) 1739 return; 1740 KASSERT(inp->inp_lport != 0, ("inpcb has no lport")); 1741 1742 porthash = inp->inp_porthash; 1743 KASSERT(porthash != NULL, ("no porthash")); 1744 1745 GET_PORTHASH_TOKEN(porthash); 1746 1747 phd = inp->inp_phd; 1748 LIST_REMOVE(inp, inp_portlist); 1749 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 1750 LIST_REMOVE(phd, phd_hash); 1751 kfree(phd, M_PCB); 1752 } 1753 1754 REL_PORTHASH_TOKEN(porthash); 1755 1756 inp->inp_phd = NULL; 1757 /* NOTE: Don't whack inp_lport, which may be used later */ 1758 } 1759 1760 static struct inp_localgroup * 1761 inp_localgroup_alloc(u_char af, uint16_t port, 1762 const union in_dependaddr *addr, int size) 1763 { 1764 struct inp_localgroup *grp; 1765 1766 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 1767 M_TEMP, M_INTWAIT | M_ZERO); 1768 grp->il_af = af; 1769 grp->il_lport = port; 1770 grp->il_dependladdr = *addr; 1771 grp->il_inpsiz = size; 1772 1773 return grp; 1774 } 1775 1776 static void 1777 inp_localgroup_free(struct inp_localgroup *grp) 1778 { 1779 kfree(grp, M_TEMP); 1780 } 1781 1782 static void 1783 inp_localgroup_destroy(struct inp_localgroup *grp) 1784 { 1785 LIST_REMOVE(grp, il_list); 1786 inp_localgroup_free(grp); 1787 } 1788 1789 static void 1790 inp_localgroup_copy(struct inp_localgroup *grp, 1791 const struct inp_localgroup *old_grp) 1792 { 1793 int i; 1794 1795 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 1796 ("invalid new local group size %d and old local group count %d", 1797 grp->il_inpsiz, old_grp->il_inpcnt)); 1798 for (i = 0; i < old_grp->il_inpcnt; ++i) 1799 grp->il_inp[i] = old_grp->il_inp[i]; 1800 grp->il_inpcnt = old_grp->il_inpcnt; 1801 } 1802 1803 static void 1804 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1805 { 1806 struct inp_localgrphead *hdr; 1807 struct inp_localgroup *grp, *grp_alloc = NULL; 1808 struct ucred *cred; 1809 int i, idx; 1810 1811 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1812 1813 if (pcbinfo->localgrphashbase == NULL) 1814 return; 1815 1816 /* 1817 * XXX don't allow jailed socket to join local group 1818 */ 1819 if (inp->inp_socket != NULL) 1820 cred = inp->inp_socket->so_cred; 1821 else 1822 cred = NULL; 1823 if (cred != NULL && jailed(cred)) 1824 return; 1825 1826 hdr = &pcbinfo->localgrphashbase[ 1827 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1828 1829 again: 1830 LIST_FOREACH(grp, hdr, il_list) { 1831 if (grp->il_af == inp->inp_af && 1832 grp->il_lport == inp->inp_lport && 1833 memcmp(&grp->il_dependladdr, 1834 &inp->inp_inc.inc_ie.ie_dependladdr, 1835 sizeof(grp->il_dependladdr)) == 0) { 1836 break; 1837 } 1838 } 1839 if (grp == NULL) { 1840 /* 1841 * Create a new local group 1842 */ 1843 if (grp_alloc == NULL) { 1844 grp_alloc = inp_localgroup_alloc(inp->inp_af, 1845 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 1846 INP_LOCALGROUP_SIZMIN); 1847 /* 1848 * Local group allocation could block and the 1849 * local group w/ the same property might have 1850 * been added by others when we were blocked; 1851 * check again. 1852 */ 1853 goto again; 1854 } else { 1855 /* Local group has been allocated; link it */ 1856 grp = grp_alloc; 1857 grp_alloc = NULL; 1858 LIST_INSERT_HEAD(hdr, grp, il_list); 1859 } 1860 } else if (grp->il_inpcnt == grp->il_inpsiz) { 1861 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 1862 static int limit_logged = 0; 1863 1864 if (!limit_logged) { 1865 limit_logged = 1; 1866 kprintf("local group port %d, " 1867 "limit reached\n", ntohs(grp->il_lport)); 1868 } 1869 if (grp_alloc != NULL) { 1870 /* 1871 * This would happen if the local group 1872 * w/ the same property was expanded when 1873 * our local group allocation blocked. 1874 */ 1875 inp_localgroup_free(grp_alloc); 1876 } 1877 return; 1878 } 1879 1880 /* 1881 * Expand this local group 1882 */ 1883 if (grp_alloc == NULL || 1884 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 1885 if (grp_alloc != NULL) 1886 inp_localgroup_free(grp_alloc); 1887 grp_alloc = inp_localgroup_alloc(grp->il_af, 1888 grp->il_lport, &grp->il_dependladdr, 1889 grp->il_inpsiz * 2); 1890 /* 1891 * Local group allocation could block and the 1892 * local group w/ the same property might have 1893 * been expanded by others when we were blocked; 1894 * check again. 1895 */ 1896 goto again; 1897 } 1898 1899 /* 1900 * Save the old local group, link the new one, and then 1901 * destroy the old local group 1902 */ 1903 inp_localgroup_copy(grp_alloc, grp); 1904 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 1905 inp_localgroup_destroy(grp); 1906 1907 grp = grp_alloc; 1908 grp_alloc = NULL; 1909 } else { 1910 /* 1911 * Found the local group 1912 */ 1913 if (grp_alloc != NULL) { 1914 /* 1915 * This would happen if the local group w/ the 1916 * same property was added or expanded when our 1917 * local group allocation blocked. 1918 */ 1919 inp_localgroup_free(grp_alloc); 1920 grp_alloc = NULL; 1921 } 1922 } 1923 1924 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 1925 ("invalid local group size %d and count %d", 1926 grp->il_inpsiz, grp->il_inpcnt)); 1927 1928 /* 1929 * Keep the local group sorted by the inpcb local group index 1930 * in ascending order. 1931 * 1932 * This eases the multi-process userland application which uses 1933 * SO_REUSEPORT sockets and binds process to the owner cpu of 1934 * the SO_REUSEPORT socket: 1935 * If we didn't sort the local group by the inpcb local group 1936 * index and one of the process owning an inpcb in this local 1937 * group restarted, e.g. crashed and restarted by watchdog, 1938 * other processes owning a inpcb in this local group would have 1939 * to detect that event, refetch its socket's owner cpu, and 1940 * re-bind. 1941 */ 1942 idx = grp->il_inpcnt; 1943 for (i = 0; i < idx; ++i) { 1944 struct inpcb *oinp = grp->il_inp[i]; 1945 1946 if (oinp->inp_lgrpindex > i) { 1947 if (inp->inp_lgrpindex < 0) { 1948 inp->inp_lgrpindex = i; 1949 } else if (inp->inp_lgrpindex != i) { 1950 if (bootverbose) { 1951 kprintf("inp %p: grpidx %d, " 1952 "assigned to %d, cpu%d\n", 1953 inp, inp->inp_lgrpindex, i, 1954 mycpuid); 1955 } 1956 } 1957 grp->il_inp[i] = inp; 1958 1959 /* Pull down inpcbs */ 1960 for (; i < grp->il_inpcnt; ++i) { 1961 struct inpcb *oinp1 = grp->il_inp[i + 1]; 1962 1963 grp->il_inp[i + 1] = oinp; 1964 oinp = oinp1; 1965 } 1966 grp->il_inpcnt++; 1967 return; 1968 } 1969 } 1970 1971 if (inp->inp_lgrpindex < 0) { 1972 inp->inp_lgrpindex = idx; 1973 } else if (inp->inp_lgrpindex != idx) { 1974 if (bootverbose) { 1975 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 1976 inp, inp->inp_lgrpindex, idx, mycpuid); 1977 } 1978 } 1979 grp->il_inp[idx] = inp; 1980 grp->il_inpcnt++; 1981 } 1982 1983 void 1984 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1985 { 1986 struct inpcontainer *ic; 1987 struct inpcontainerhead *bucket; 1988 1989 GET_PCBINFO_TOKEN(pcbinfo); 1990 1991 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 1992 1993 bucket = &pcbinfo->wildcardhashbase[ 1994 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 1995 1996 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 1997 ic->ic_inp = inp; 1998 LIST_INSERT_HEAD(bucket, ic, ic_list); 1999 2000 REL_PCBINFO_TOKEN(pcbinfo); 2001 } 2002 2003 /* 2004 * Insert PCB into wildcard hash table. 2005 */ 2006 void 2007 in_pcbinswildcardhash(struct inpcb *inp) 2008 { 2009 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2010 2011 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2012 ("not in correct netisr")); 2013 ASSERT_INP_NOTINHASH(inp); 2014 inp->inp_flags |= INP_WILDCARD; 2015 2016 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 2017 } 2018 2019 static void 2020 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2021 { 2022 struct inp_localgrphead *hdr; 2023 struct inp_localgroup *grp; 2024 2025 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2026 2027 if (pcbinfo->localgrphashbase == NULL) 2028 return; 2029 2030 hdr = &pcbinfo->localgrphashbase[ 2031 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2032 2033 LIST_FOREACH(grp, hdr, il_list) { 2034 int i; 2035 2036 for (i = 0; i < grp->il_inpcnt; ++i) { 2037 if (grp->il_inp[i] != inp) 2038 continue; 2039 2040 if (grp->il_inpcnt == 1) { 2041 /* Destroy this local group */ 2042 inp_localgroup_destroy(grp); 2043 } else { 2044 /* Pull up inpcbs */ 2045 for (; i + 1 < grp->il_inpcnt; ++i) 2046 grp->il_inp[i] = grp->il_inp[i + 1]; 2047 grp->il_inpcnt--; 2048 } 2049 return; 2050 } 2051 } 2052 } 2053 2054 void 2055 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2056 { 2057 struct inpcontainer *ic; 2058 struct inpcontainerhead *head; 2059 2060 GET_PCBINFO_TOKEN(pcbinfo); 2061 2062 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2063 2064 /* find bucket */ 2065 head = &pcbinfo->wildcardhashbase[ 2066 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2067 2068 LIST_FOREACH(ic, head, ic_list) { 2069 if (ic->ic_inp == inp) 2070 goto found; 2071 } 2072 REL_PCBINFO_TOKEN(pcbinfo); 2073 return; /* not found! */ 2074 2075 found: 2076 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2077 REL_PCBINFO_TOKEN(pcbinfo); 2078 kfree(ic, M_TEMP); /* deallocate container */ 2079 } 2080 2081 /* 2082 * Remove PCB from wildcard hash table. 2083 */ 2084 void 2085 in_pcbremwildcardhash(struct inpcb *inp) 2086 { 2087 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2088 2089 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2090 ("not in correct netisr")); 2091 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2092 2093 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2094 inp->inp_lgrpindex = -1; 2095 inp->inp_flags &= ~INP_WILDCARD; 2096 } 2097 2098 /* 2099 * Remove PCB from various lists. 2100 */ 2101 void 2102 in_pcbremlists(struct inpcb *inp) 2103 { 2104 in_pcbremporthash(inp); 2105 if (inp->inp_flags & INP_WILDCARD) { 2106 in_pcbremwildcardhash(inp); 2107 } else if (inp->inp_flags & INP_CONNECTED) { 2108 in_pcbremconnhash(inp); 2109 } 2110 2111 if (inp->inp_flags & INP_ONLIST) 2112 in_pcbofflist(inp); 2113 } 2114 2115 int 2116 prison_xinpcb(struct thread *td, struct inpcb *inp) 2117 { 2118 struct ucred *cr; 2119 2120 if (td->td_proc == NULL) 2121 return (0); 2122 cr = td->td_proc->p_ucred; 2123 if (cr->cr_prison == NULL) 2124 return (0); 2125 if (inp->inp_socket && inp->inp_socket->so_cred && 2126 inp->inp_socket->so_cred->cr_prison && 2127 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2128 return (0); 2129 return (1); 2130 } 2131 2132 int 2133 in_pcblist_range(SYSCTL_HANDLER_ARGS) 2134 { 2135 struct inpcbinfo *pcbinfo_arr = arg1; 2136 int pcbinfo_arrlen = arg2; 2137 struct inpcb *marker; 2138 int cpu, origcpu; 2139 int error, n; 2140 2141 KASSERT(pcbinfo_arrlen <= netisr_ncpus && pcbinfo_arrlen >= 1, 2142 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2143 2144 /* 2145 * The process of preparing the TCB list is too time-consuming and 2146 * resource-intensive to repeat twice on every request. 2147 */ 2148 n = 0; 2149 if (req->oldptr == NULL) { 2150 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2151 n += pcbinfo_arr[cpu].ipi_count; 2152 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2153 return 0; 2154 } 2155 2156 if (req->newptr != NULL) 2157 return EPERM; 2158 2159 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2160 marker->inp_flags |= INP_PLACEMARKER; 2161 2162 /* 2163 * OK, now we're committed to doing something. Re-fetch ipi_count 2164 * after obtaining the generation count. 2165 */ 2166 error = 0; 2167 origcpu = mycpuid; 2168 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2169 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2170 struct inpcb *inp; 2171 struct xinpcb xi; 2172 int i; 2173 2174 lwkt_migratecpu(cpu); 2175 2176 GET_PCBINFO_TOKEN(pcbinfo); 2177 2178 n = pcbinfo->ipi_count; 2179 2180 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2181 i = 0; 2182 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2183 LIST_REMOVE(marker, inp_list); 2184 LIST_INSERT_AFTER(inp, marker, inp_list); 2185 2186 if (inp->inp_flags & INP_PLACEMARKER) 2187 continue; 2188 if (prison_xinpcb(req->td, inp)) 2189 continue; 2190 2191 bzero(&xi, sizeof xi); 2192 xi.xi_len = sizeof xi; 2193 bcopy(inp, &xi.xi_inp, sizeof *inp); 2194 if (inp->inp_socket) 2195 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2196 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2197 break; 2198 ++i; 2199 } 2200 LIST_REMOVE(marker, inp_list); 2201 2202 REL_PCBINFO_TOKEN(pcbinfo); 2203 2204 if (error == 0 && i < n) { 2205 bzero(&xi, sizeof xi); 2206 xi.xi_len = sizeof xi; 2207 while (i < n) { 2208 error = SYSCTL_OUT(req, &xi, sizeof xi); 2209 if (error) 2210 break; 2211 ++i; 2212 } 2213 } 2214 } 2215 2216 lwkt_migratecpu(origcpu); 2217 kfree(marker, M_TEMP); 2218 return error; 2219 } 2220 2221 int 2222 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS) 2223 { 2224 2225 return (in_pcblist_range(oidp, arg1, netisr_ncpus, req)); 2226 } 2227 2228 void 2229 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2230 { 2231 struct sockaddr_in *sin; 2232 2233 KASSERT(faddr->sa_family == AF_INET, 2234 ("not AF_INET faddr %d", faddr->sa_family)); 2235 2236 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2237 sin->sin_family = AF_INET; 2238 sin->sin_len = sizeof(*sin); 2239 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2240 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2241 2242 so->so_faddr = (struct sockaddr *)sin; 2243 } 2244 2245 void 2246 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2247 u_short offset) 2248 { 2249 memset(portinfo, 0, sizeof(*portinfo)); 2250 2251 portinfo->offset = offset; 2252 portinfo->porthashbase = phashinit(hashsize, M_PCB, 2253 &portinfo->porthashcnt); 2254 } 2255 2256 void 2257 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2258 { 2259 int hi, lo; 2260 2261 if (step == 1) 2262 return; 2263 2264 hi = *hi0; 2265 lo = *lo0; 2266 2267 hi = rounddown(hi, step); 2268 hi += ofs; 2269 if (hi > (int)*hi0) 2270 hi -= step; 2271 2272 lo = roundup(lo, step); 2273 lo -= (step - ofs); 2274 if (lo < (int)*lo0) 2275 lo += step; 2276 2277 *hi0 = hi; 2278 *lo0 = lo; 2279 } 2280 2281 void 2282 in_pcbglobalinit(void) 2283 { 2284 int cpu; 2285 2286 in_pcbmarkers = kmalloc(netisr_ncpus * sizeof(struct inpcb), M_PCB, 2287 M_WAITOK | M_ZERO); 2288 in_pcbcontainer_markers = 2289 kmalloc(netisr_ncpus * sizeof(struct inpcontainer), M_PCB, 2290 M_WAITOK | M_ZERO); 2291 2292 for (cpu = 0; cpu < netisr_ncpus; ++cpu) { 2293 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2294 struct inpcb *marker = &in_pcbmarkers[cpu]; 2295 2296 marker->inp_flags |= INP_PLACEMARKER; 2297 ic->ic_inp = marker; 2298 } 2299 } 2300 2301 struct inpcb * 2302 in_pcbmarker(void) 2303 { 2304 2305 ASSERT_NETISR_NCPUS(mycpuid); 2306 return &in_pcbmarkers[mycpuid]; 2307 } 2308 2309 struct inpcontainer * 2310 in_pcbcontainer_marker(void) 2311 { 2312 2313 ASSERT_NETISR_NCPUS(mycpuid); 2314 return &in_pcbcontainer_markers[mycpuid]; 2315 } 2316 2317 void 2318 in_pcbresetroute(struct inpcb *inp) 2319 { 2320 struct route *ro = &inp->inp_route; 2321 2322 if (ro->ro_rt != NULL) 2323 RTFREE(ro->ro_rt); 2324 bzero(ro, sizeof(*ro)); 2325 } 2326