1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/domain.h> 74 #include <sys/protosw.h> 75 #include <sys/socket.h> 76 #include <sys/socketvar.h> 77 #include <sys/proc.h> 78 #include <sys/priv.h> 79 #include <sys/jail.h> 80 #include <sys/kernel.h> 81 #include <sys/sysctl.h> 82 83 #include <sys/thread2.h> 84 #include <sys/socketvar2.h> 85 #include <sys/msgport2.h> 86 87 #include <machine/limits.h> 88 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/route.h> 92 #include <net/netisr2.h> 93 #include <net/toeplitz2.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip_var.h> 99 #ifdef INET6 100 #include <netinet/ip6.h> 101 #include <netinet6/ip6_var.h> 102 #endif /* INET6 */ 103 104 #ifdef IPSEC 105 #include <netinet6/ipsec.h> 106 #include <netproto/key/key.h> 107 #include <netproto/ipsec/esp_var.h> 108 #endif 109 110 #ifdef FAST_IPSEC 111 #if defined(IPSEC) || defined(IPSEC_ESP) 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!" 113 #endif 114 115 #include <netproto/ipsec/ipsec.h> 116 #include <netproto/ipsec/key.h> 117 #define IPSEC 118 #endif /* FAST_IPSEC */ 119 120 #define INP_LOCALGROUP_SIZMIN 8 121 #define INP_LOCALGROUP_SIZMAX 256 122 123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash, 124 struct in_addr laddr, u_int lport_arg, int wild_okay, 125 struct ucred *cred); 126 127 struct in_addr zeroin_addr; 128 129 /* 130 * These configure the range of local port addresses assigned to 131 * "unspecified" outgoing connections/packets/whatever. 132 */ 133 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 134 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 135 136 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 137 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 138 139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 140 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 141 142 #define RANGECHK(var, min, max) \ 143 if ((var) < (min)) { (var) = (min); } \ 144 else if ((var) > (max)) { (var) = (max); } 145 146 int udpencap_enable = 1; /* enabled by default */ 147 int udpencap_port = 4500; /* triggers decapsulation */ 148 149 /* 150 * Per-netisr inpcb markers. 151 * NOTE: they should only be used in netisrs. 152 */ 153 static struct inpcb *in_pcbmarkers; 154 static struct inpcontainer *in_pcbcontainer_markers; 155 156 static int 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 158 { 159 int error; 160 161 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 162 if (!error) { 163 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 164 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 165 166 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 167 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 168 169 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 170 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 171 } 172 return (error); 173 } 174 175 #undef RANGECHK 176 177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 178 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 180 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 182 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 184 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 186 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 188 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 190 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 191 192 static int ip_porthash_trycount = 15; 193 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW, 194 &ip_porthash_trycount, 0, 195 "Number of tries to find local port matching hash of 4-tuple"); 196 197 /* 198 * in_pcb.c: manage the Protocol Control Blocks. 199 * 200 * NOTE: It is assumed that most of these functions will be called from 201 * a critical section. XXX - There are, unfortunately, a few exceptions 202 * to this rule that should be fixed. 203 * 204 * NOTE: The caller should initialize the cpu field to the cpu running the 205 * protocol stack associated with this inpcbinfo. 206 */ 207 208 void 209 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 210 { 211 KASSERT(cpu >= 0 && cpu < netisr_ncpus, ("invalid cpu%d", cpu)); 212 pcbinfo->cpu = cpu; 213 214 LIST_INIT(&pcbinfo->pcblisthead); 215 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 216 M_WAITOK | M_ZERO); 217 218 if (shared) { 219 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 220 M_PCB, M_WAITOK); 221 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 222 } else { 223 pcbinfo->infotoken = NULL; 224 } 225 } 226 227 void 228 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo, 229 int portinfo_cnt) 230 { 231 232 KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt)); 233 pcbinfo->portinfo = portinfo; 234 pcbinfo->portinfo_cnt = portinfo_cnt; 235 } 236 237 struct baddynamicports baddynamicports; 238 239 /* 240 * Check if the specified port is invalid for dynamic allocation. 241 */ 242 int 243 in_baddynamic(u_int16_t port, u_int16_t proto) 244 { 245 switch (proto) { 246 case IPPROTO_TCP: 247 return (DP_ISSET(baddynamicports.tcp, port)); 248 case IPPROTO_UDP: 249 #ifdef IPSEC 250 /* Cannot preset this as it is a sysctl */ 251 if (port == udpencap_port) 252 return (1); 253 #endif 254 return (DP_ISSET(baddynamicports.udp, port)); 255 default: 256 return (0); 257 } 258 } 259 260 void 261 in_pcbonlist(struct inpcb *inp) 262 { 263 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 264 265 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 266 ("not in the correct netisr")); 267 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 268 inp->inp_flags |= INP_ONLIST; 269 270 GET_PCBINFO_TOKEN(pcbinfo); 271 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 272 pcbinfo->ipi_count++; 273 REL_PCBINFO_TOKEN(pcbinfo); 274 } 275 276 void 277 in_pcbofflist(struct inpcb *inp) 278 { 279 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 280 281 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 282 ("not in the correct netisr")); 283 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 284 inp->inp_flags &= ~INP_ONLIST; 285 286 GET_PCBINFO_TOKEN(pcbinfo); 287 LIST_REMOVE(inp, inp_list); 288 KASSERT(pcbinfo->ipi_count > 0, 289 ("invalid inpcb count %d", pcbinfo->ipi_count)); 290 pcbinfo->ipi_count--; 291 REL_PCBINFO_TOKEN(pcbinfo); 292 } 293 294 /* 295 * Allocate a PCB and associate it with the socket. 296 */ 297 int 298 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 299 { 300 struct inpcb *inp; 301 #ifdef IPSEC 302 int error; 303 #endif 304 305 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 306 if (inp == NULL) 307 return (ENOMEM); 308 inp->inp_lgrpindex = -1; 309 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 310 inp->inp_pcbinfo = pcbinfo; 311 inp->inp_socket = so; 312 #ifdef IPSEC 313 error = ipsec_init_policy(so, &inp->inp_sp); 314 if (error != 0) { 315 kfree(inp, M_PCB); 316 return (error); 317 } 318 #endif 319 #ifdef INET6 320 if (INP_CHECK_SOCKAF(so, AF_INET6)) { 321 if (ip6_auto_flowlabel) 322 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 323 inp->inp_af = AF_INET6; 324 } else 325 #endif 326 inp->inp_af = AF_INET; 327 soreference(so); 328 so->so_pcb = inp; 329 330 in_pcbonlist(inp); 331 return (0); 332 } 333 334 /* 335 * Unlink a pcb with the intention of moving it to another cpu with a 336 * different pcbinfo. While unlinked nothing should attempt to dereference 337 * inp_pcbinfo, NULL it out so we assert if it does. 338 */ 339 void 340 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 341 { 342 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 343 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 344 ("already linked")); 345 346 in_pcbofflist(inp); 347 inp->inp_pcbinfo = NULL; 348 } 349 350 void 351 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 352 { 353 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 354 } 355 356 /* 357 * Relink a pcb into a new pcbinfo. 358 */ 359 void 360 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 361 { 362 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 363 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 364 ("already linked")); 365 366 inp->inp_pcbinfo = pcbinfo; 367 in_pcbonlist(inp); 368 } 369 370 void 371 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 372 { 373 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 374 } 375 376 static boolean_t 377 in_pcbporthash_update(struct inpcbportinfo *portinfo, 378 struct inpcb *inp, u_short lport, struct ucred *cred, int wild) 379 { 380 struct inpcbporthead *porthash; 381 382 /* 383 * This has to be atomic. If the porthash is shared across multiple 384 * protocol threads, e.g. tcp and udp, then the token must be held. 385 */ 386 porthash = in_pcbporthash_head(portinfo, lport); 387 GET_PORTHASH_TOKEN(porthash); 388 389 if (in_pcblookup_local(porthash, inp->inp_laddr, lport, 390 wild, cred) != NULL) { 391 REL_PORTHASH_TOKEN(porthash); 392 return FALSE; 393 } 394 inp->inp_lport = lport; 395 in_pcbinsporthash(porthash, inp); 396 397 REL_PORTHASH_TOKEN(porthash); 398 return TRUE; 399 } 400 401 static int 402 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 403 { 404 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 405 struct inpcbportinfo *portinfo; 406 u_short first, last, lport, step, first0, last0; 407 int count, error; 408 int portinfo_first, portinfo_idx; 409 uint32_t cut; 410 411 inp->inp_flags |= INP_ANONPORT; 412 413 step = pcbinfo->portinfo_cnt; 414 portinfo_first = mycpuid % pcbinfo->portinfo_cnt; 415 portinfo_idx = portinfo_first; 416 417 if (inp->inp_flags & INP_HIGHPORT) { 418 first0 = ipport_hifirstauto; /* sysctl */ 419 last0 = ipport_hilastauto; 420 } else if (inp->inp_flags & INP_LOWPORT) { 421 if (cred && 422 (error = 423 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 424 inp->inp_laddr.s_addr = INADDR_ANY; 425 return error; 426 } 427 first0 = ipport_lowfirstauto; /* 1023 */ 428 last0 = ipport_lowlastauto; /* 600 */ 429 } else { 430 first0 = ipport_firstauto; /* sysctl */ 431 last0 = ipport_lastauto; 432 } 433 if (first0 > last0) { 434 lport = last0; 435 last0 = first0; 436 first0 = lport; 437 } 438 KKASSERT(last0 >= first0); 439 440 cut = karc4random(); 441 loop: 442 portinfo = &pcbinfo->portinfo[portinfo_idx]; 443 first = first0; 444 last = last0; 445 446 /* 447 * Simple check to ensure all ports are not used up causing 448 * a deadlock here. 449 */ 450 in_pcbportrange(&last, &first, portinfo->offset, step); 451 lport = last - first; 452 count = lport / step; 453 454 lport = rounddown(cut % lport, step) + first; 455 KKASSERT(lport % step == portinfo->offset); 456 457 for (;;) { 458 if (count-- < 0) { /* completely used? */ 459 error = EADDRNOTAVAIL; 460 break; 461 } 462 463 if (__predict_false(lport < first || lport > last)) { 464 lport = first; 465 KKASSERT(lport % step == portinfo->offset); 466 } 467 468 if (in_pcbporthash_update(portinfo, inp, htons(lport), 469 cred, wild)) { 470 error = 0; 471 break; 472 } 473 474 lport += step; 475 KKASSERT(lport % step == portinfo->offset); 476 } 477 478 if (error) { 479 /* Try next portinfo */ 480 portinfo_idx++; 481 portinfo_idx %= pcbinfo->portinfo_cnt; 482 if (portinfo_idx != portinfo_first) 483 goto loop; 484 inp->inp_laddr.s_addr = INADDR_ANY; 485 } 486 return error; 487 } 488 489 int 490 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 491 { 492 struct socket *so = inp->inp_socket; 493 struct sockaddr_in jsin; 494 struct ucred *cred = NULL; 495 int wild = 0; 496 497 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 498 return (EADDRNOTAVAIL); 499 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 500 return (EINVAL); /* already bound */ 501 502 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 503 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 504 if (td->td_proc) 505 cred = td->td_proc->p_ucred; 506 507 if (nam != NULL) { 508 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 509 struct inpcbinfo *pcbinfo; 510 struct inpcbportinfo *portinfo; 511 struct inpcbporthead *porthash; 512 struct inpcb *t; 513 u_short lport, lport_ho; 514 int reuseport = (so->so_options & SO_REUSEPORT); 515 int error; 516 517 if (nam->sa_len != sizeof *sin) 518 return (EINVAL); 519 #ifdef notdef 520 /* 521 * We should check the family, but old programs 522 * incorrectly fail to initialize it. 523 */ 524 if (sin->sin_family != AF_INET) 525 return (EAFNOSUPPORT); 526 #endif 527 if (!prison_replace_wildcards(td, nam)) 528 return (EINVAL); 529 530 lport = sin->sin_port; 531 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 532 /* 533 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 534 * allow complete duplication of binding if 535 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 536 * and a multicast address is bound on both 537 * new and duplicated sockets. 538 */ 539 if (so->so_options & SO_REUSEADDR) 540 reuseport = SO_REUSEADDR | SO_REUSEPORT; 541 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 542 sin->sin_port = 0; /* yech... */ 543 bzero(&sin->sin_zero, sizeof sin->sin_zero); 544 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 545 return (EADDRNOTAVAIL); 546 } 547 548 inp->inp_laddr = sin->sin_addr; 549 550 jsin.sin_family = AF_INET; 551 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 552 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 553 inp->inp_laddr.s_addr = INADDR_ANY; 554 return (EINVAL); 555 } 556 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 557 558 if (lport == 0) { 559 /* Auto-select local port */ 560 return in_pcbsetlport(inp, wild, cred); 561 } 562 lport_ho = ntohs(lport); 563 564 /* GROSS */ 565 if (lport_ho < IPPORT_RESERVED && cred && 566 (error = 567 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 568 inp->inp_laddr.s_addr = INADDR_ANY; 569 return (error); 570 } 571 572 /* 573 * Locate the proper portinfo based on lport 574 */ 575 pcbinfo = inp->inp_pcbinfo; 576 portinfo = 577 &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 578 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == 579 portinfo->offset); 580 581 /* 582 * This has to be atomic. If the porthash is shared across 583 * multiple protocol threads, e.g. tcp and udp then the token 584 * must be held. 585 */ 586 porthash = in_pcbporthash_head(portinfo, lport); 587 GET_PORTHASH_TOKEN(porthash); 588 589 if (so->so_cred->cr_uid != 0 && 590 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 591 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 592 INPLOOKUP_WILDCARD, cred); 593 if (t && 594 (so->so_cred->cr_uid != 595 t->inp_socket->so_cred->cr_uid)) { 596 inp->inp_laddr.s_addr = INADDR_ANY; 597 error = EADDRINUSE; 598 goto done; 599 } 600 } 601 if (cred && !prison_replace_wildcards(td, nam)) { 602 inp->inp_laddr.s_addr = INADDR_ANY; 603 error = EADDRNOTAVAIL; 604 goto done; 605 } 606 t = in_pcblookup_local(porthash, sin->sin_addr, lport, 607 wild, cred); 608 if (t && !(reuseport & t->inp_socket->so_options)) { 609 inp->inp_laddr.s_addr = INADDR_ANY; 610 error = EADDRINUSE; 611 goto done; 612 } 613 inp->inp_lport = lport; 614 in_pcbinsporthash(porthash, inp); 615 error = 0; 616 done: 617 REL_PORTHASH_TOKEN(porthash); 618 return (error); 619 } else { 620 jsin.sin_family = AF_INET; 621 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 622 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 623 inp->inp_laddr.s_addr = INADDR_ANY; 624 return (EINVAL); 625 } 626 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 627 628 return in_pcbsetlport(inp, wild, cred); 629 } 630 } 631 632 static struct inpcb * 633 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr, 634 u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred) 635 { 636 struct inpcb *inp; 637 struct inpcbport *phd; 638 struct inpcb *match = NULL; 639 640 /* 641 * If the porthashbase is shared across several cpus, it must 642 * have been locked. 643 */ 644 ASSERT_PORTHASH_TOKEN_HELD(porthash); 645 646 /* 647 * Best fit PCB lookup. 648 * 649 * First see if this local port is in use by looking on the 650 * port hash list. 651 */ 652 LIST_FOREACH(phd, porthash, phd_hash) { 653 if (phd->phd_port == lport) 654 break; 655 } 656 if (phd != NULL) { 657 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 658 #ifdef INET6 659 if (!INP_ISIPV4(inp)) 660 continue; 661 #endif 662 if (inp->inp_laddr.s_addr != INADDR_ANY && 663 inp->inp_laddr.s_addr != laddr.s_addr) 664 continue; 665 666 if (inp->inp_faddr.s_addr != INADDR_ANY && 667 inp->inp_faddr.s_addr != faddr.s_addr) 668 continue; 669 670 if (inp->inp_fport != 0 && inp->inp_fport != fport) 671 continue; 672 673 if (cred == NULL || 674 cred->cr_prison == 675 inp->inp_socket->so_cred->cr_prison) { 676 match = inp; 677 break; 678 } 679 } 680 } 681 return (match); 682 } 683 684 static boolean_t 685 in_pcbporthash_update4(struct inpcbportinfo *portinfo, 686 struct inpcb *inp, u_short lport, const struct sockaddr_in *sin, 687 struct ucred *cred) 688 { 689 struct inpcbporthead *porthash; 690 691 /* 692 * This has to be atomic. If the porthash is shared across multiple 693 * protocol threads, e.g. tcp and udp, then the token must be held. 694 */ 695 porthash = in_pcbporthash_head(portinfo, lport); 696 GET_PORTHASH_TOKEN(porthash); 697 698 if (in_pcblookup_localremote(porthash, inp->inp_laddr, 699 lport, sin->sin_addr, sin->sin_port, cred) != NULL) { 700 REL_PORTHASH_TOKEN(porthash); 701 return FALSE; 702 } 703 inp->inp_lport = lport; 704 in_pcbinsporthash(porthash, inp); 705 706 REL_PORTHASH_TOKEN(porthash); 707 return TRUE; 708 } 709 710 int 711 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 712 struct thread *td) 713 { 714 struct proc *p = td->td_proc; 715 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 716 struct sockaddr_in jsin; 717 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 718 struct inpcbportinfo *portinfo; 719 struct ucred *cred = NULL; 720 u_short first, last, lport, step, first0, last0; 721 int count, error, selfconn; 722 int portinfo_first, portinfo_idx; 723 int hash_cpu, hash_count, hash_count0; 724 uint32_t hash_base = 0, hash, cut; 725 726 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 727 return (EADDRNOTAVAIL); 728 729 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 730 if (inp->inp_lport != 0) 731 return (EINVAL); /* already bound */ 732 733 KKASSERT(p); 734 cred = p->p_ucred; 735 736 jsin.sin_family = AF_INET; 737 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 738 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 739 inp->inp_laddr.s_addr = INADDR_ANY; 740 return (EINVAL); 741 } 742 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 743 744 hash_count0 = ip_porthash_trycount; 745 if (hash_count0 > 0) { 746 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^ 747 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^ 748 toeplitz_piecemeal_port(sin->sin_port); 749 } else { 750 hash_count0 = 0; 751 } 752 753 inp->inp_flags |= INP_ANONPORT; 754 755 step = pcbinfo->portinfo_cnt; 756 portinfo_first = mycpuid % pcbinfo->portinfo_cnt; 757 portinfo_idx = portinfo_first; 758 759 if (inp->inp_flags & INP_HIGHPORT) { 760 first0 = ipport_hifirstauto; /* sysctl */ 761 last0 = ipport_hilastauto; 762 } else if (inp->inp_flags & INP_LOWPORT) { 763 if (cred && 764 (error = 765 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 766 inp->inp_laddr.s_addr = INADDR_ANY; 767 return (error); 768 } 769 first0 = ipport_lowfirstauto; /* 1023 */ 770 last0 = ipport_lowlastauto; /* 600 */ 771 } else { 772 first0 = ipport_firstauto; /* sysctl */ 773 last0 = ipport_lastauto; 774 } 775 if (first0 > last0) { 776 lport = last0; 777 last0 = first0; 778 first0 = lport; 779 } 780 KKASSERT(last0 >= first0); 781 782 cut = karc4random(); 783 loop: 784 hash_cpu = portinfo_idx % netisr_ncpus; 785 portinfo = &pcbinfo->portinfo[portinfo_idx]; 786 selfconn = 0; 787 first = first0; 788 last = last0; 789 790 /* This could happen on loopback interface */ 791 #define IS_SELFCONNECT(inp, lport, sin) \ 792 (__predict_false((sin)->sin_port == (lport) && \ 793 (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr)) 794 795 /* 796 * Simple check to ensure all ports are not used up causing 797 * a deadlock here. 798 */ 799 hash_count = hash_count0; 800 801 in_pcbportrange(&last, &first, portinfo->offset, step); 802 lport = last - first; 803 count = (lport / step) + hash_count; 804 805 lport = rounddown(cut % lport, step) + first; 806 KKASSERT(lport % step == portinfo->offset); 807 808 for (;;) { 809 u_short lport_no; 810 811 if (count-- < 0) { /* completely used? */ 812 error = EADDRNOTAVAIL; 813 break; 814 } 815 816 if (__predict_false(lport < first || lport > last)) { 817 lport = first; 818 KKASSERT(lport % step == portinfo->offset); 819 } 820 821 lport_no = htons(lport); 822 if (IS_SELFCONNECT(inp, lport_no, sin)) { 823 if (!selfconn) { 824 ++count; /* don't count this try */ 825 selfconn = 1; 826 } 827 goto next; 828 } 829 830 if (hash_count) { 831 --hash_count; 832 hash = hash_base ^ 833 toeplitz_piecemeal_port(lport_no); 834 if (netisr_hashcpu(hash) != hash_cpu && 835 hash_count) 836 goto next; 837 } 838 839 if (in_pcbporthash_update4(portinfo, 840 inp, lport_no, sin, cred)) { 841 error = 0; 842 break; 843 } 844 next: 845 lport += step; 846 KKASSERT(lport % step == portinfo->offset); 847 } 848 849 #undef IS_SELFCONNECT 850 851 if (error) { 852 /* Try next portinfo */ 853 portinfo_idx++; 854 portinfo_idx %= pcbinfo->portinfo_cnt; 855 if (portinfo_idx != portinfo_first) 856 goto loop; 857 inp->inp_laddr.s_addr = INADDR_ANY; 858 } 859 return error; 860 } 861 862 /* 863 * Transform old in_pcbconnect() into an inner subroutine for new 864 * in_pcbconnect(): Do some validity-checking on the remote 865 * address (in mbuf 'nam') and then determine local host address 866 * (i.e., which interface) to use to access that remote host. 867 * 868 * This preserves definition of in_pcbconnect(), while supporting a 869 * slightly different version for T/TCP. (This is more than 870 * a bit of a kludge, but cleaning up the internal interfaces would 871 * have forced minor changes in every protocol). 872 */ 873 int 874 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 875 struct sockaddr_in **plocal_sin, struct thread *td, int find) 876 { 877 struct in_ifaddr *ia; 878 struct ucred *cred = NULL; 879 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 880 struct sockaddr *jsin; 881 int jailed = 0, alloc_route = 0; 882 883 if (nam->sa_len != sizeof *sin) 884 return (EINVAL); 885 if (sin->sin_family != AF_INET) 886 return (EAFNOSUPPORT); 887 if (sin->sin_port == 0) 888 return (EADDRNOTAVAIL); 889 if (td && td->td_proc && td->td_proc->p_ucred) 890 cred = td->td_proc->p_ucred; 891 if (cred && cred->cr_prison) 892 jailed = 1; 893 if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) { 894 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 895 /* 896 * If the destination address is INADDR_ANY, 897 * use the primary local address. 898 * If the supplied address is INADDR_BROADCAST, 899 * and the primary interface supports broadcast, 900 * choose the broadcast address for that interface. 901 */ 902 if (sin->sin_addr.s_addr == INADDR_ANY) 903 sin->sin_addr = IA_SIN(ia)->sin_addr; 904 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST && 905 (ia->ia_ifp->if_flags & IFF_BROADCAST)) 906 sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr; 907 } 908 if (find) { 909 struct route *ro; 910 911 ia = NULL; 912 /* 913 * If route is known or can be allocated now, 914 * our src addr is taken from the i/f, else punt. 915 * Note that we should check the address family of the cached 916 * destination, in case of sharing the cache with IPv6. 917 */ 918 ro = &inp->inp_route; 919 if (ro->ro_rt && 920 (!(ro->ro_rt->rt_flags & RTF_UP) || 921 ro->ro_dst.sa_family != AF_INET || 922 satosin(&ro->ro_dst)->sin_addr.s_addr != 923 sin->sin_addr.s_addr || 924 inp->inp_socket->so_options & SO_DONTROUTE)) { 925 RTFREE(ro->ro_rt); 926 ro->ro_rt = NULL; 927 } 928 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 929 (ro->ro_rt == NULL || 930 ro->ro_rt->rt_ifp == NULL)) { 931 /* No route yet, so try to acquire one */ 932 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 933 ro->ro_dst.sa_family = AF_INET; 934 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 935 ((struct sockaddr_in *) &ro->ro_dst)->sin_addr = 936 sin->sin_addr; 937 rtalloc(ro); 938 alloc_route = 1; 939 } 940 /* 941 * If we found a route, use the address 942 * corresponding to the outgoing interface 943 * unless it is the loopback (in case a route 944 * to our address on another net goes to loopback). 945 */ 946 if (ro->ro_rt && 947 !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) { 948 if (jailed) { 949 if (jailed_ip(cred->cr_prison, 950 ro->ro_rt->rt_ifa->ifa_addr)) { 951 ia = ifatoia(ro->ro_rt->rt_ifa); 952 } 953 } else { 954 ia = ifatoia(ro->ro_rt->rt_ifa); 955 } 956 } 957 if (ia == NULL) { 958 u_short fport = sin->sin_port; 959 960 sin->sin_port = 0; 961 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 962 if (ia && jailed && !jailed_ip(cred->cr_prison, 963 sintosa(&ia->ia_addr))) 964 ia = NULL; 965 if (ia == NULL) 966 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 967 if (ia && jailed && !jailed_ip(cred->cr_prison, 968 sintosa(&ia->ia_addr))) 969 ia = NULL; 970 sin->sin_port = fport; 971 if (ia == NULL && 972 !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 973 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 974 if (ia && jailed && !jailed_ip(cred->cr_prison, 975 sintosa(&ia->ia_addr))) 976 ia = NULL; 977 978 if (!jailed && ia == NULL) 979 goto fail; 980 } 981 /* 982 * If the destination address is multicast and an outgoing 983 * interface has been set as a multicast option, use the 984 * address of that interface as our source address. 985 */ 986 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 987 inp->inp_moptions != NULL) { 988 struct ip_moptions *imo; 989 struct ifnet *ifp; 990 991 imo = inp->inp_moptions; 992 if ((ifp = imo->imo_multicast_ifp) != NULL) { 993 struct in_ifaddr_container *iac; 994 995 ia = NULL; 996 TAILQ_FOREACH(iac, 997 &in_ifaddrheads[mycpuid], ia_link) { 998 if (iac->ia->ia_ifp == ifp) { 999 ia = iac->ia; 1000 break; 1001 } 1002 } 1003 if (ia == NULL) 1004 goto fail; 1005 } 1006 } 1007 /* 1008 * Don't do pcblookup call here; return interface in plocal_sin 1009 * and exit to caller, that will do the lookup. 1010 */ 1011 if (ia == NULL && jailed) { 1012 if ((jsin = prison_get_nonlocal( 1013 cred->cr_prison, AF_INET, NULL)) != NULL || 1014 (jsin = prison_get_local( 1015 cred->cr_prison, AF_INET, NULL)) != NULL) { 1016 *plocal_sin = satosin(jsin); 1017 } else { 1018 /* IPv6 only Jail */ 1019 goto fail; 1020 } 1021 } else { 1022 *plocal_sin = &ia->ia_addr; 1023 } 1024 } 1025 return (0); 1026 fail: 1027 if (alloc_route) 1028 in_pcbresetroute(inp); 1029 return (EADDRNOTAVAIL); 1030 } 1031 1032 int 1033 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 1034 struct sockaddr_in **plocal_sin, struct thread *td) 1035 { 1036 return in_pcbladdr_find(inp, nam, plocal_sin, td, 1037 (inp->inp_laddr.s_addr == INADDR_ANY)); 1038 } 1039 1040 /* 1041 * Outer subroutine: 1042 * Connect from a socket to a specified address. 1043 * Both address and port must be specified in argument sin. 1044 * If don't have a local address for this socket yet, 1045 * then pick one. 1046 */ 1047 int 1048 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 1049 { 1050 struct sockaddr_in *if_sin; 1051 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1052 int error; 1053 1054 if_sin = NULL; /* avoid gcc warnings */ 1055 1056 /* Call inner routine to assign local interface address. */ 1057 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1058 return (error); 1059 1060 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1061 inp->inp_laddr.s_addr ? 1062 inp->inp_laddr : if_sin->sin_addr, 1063 inp->inp_lport, FALSE, NULL) != NULL) { 1064 return (EADDRINUSE); 1065 } 1066 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1067 if (inp->inp_lport == 0) { 1068 error = in_pcbbind(inp, NULL, td); 1069 if (error) 1070 return (error); 1071 } 1072 inp->inp_laddr = if_sin->sin_addr; 1073 } 1074 inp->inp_faddr = sin->sin_addr; 1075 inp->inp_fport = sin->sin_port; 1076 in_pcbinsconnhash(inp); 1077 return (0); 1078 } 1079 1080 void 1081 in_pcbdisconnect(struct inpcb *inp) 1082 { 1083 1084 in_pcbremconnhash(inp); 1085 inp->inp_faddr.s_addr = INADDR_ANY; 1086 inp->inp_fport = 0; 1087 } 1088 1089 void 1090 in_pcbdetach(struct inpcb *inp) 1091 { 1092 struct socket *so = inp->inp_socket; 1093 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1094 1095 #ifdef IPSEC 1096 ipsec4_delete_pcbpolicy(inp); 1097 #endif /*IPSEC*/ 1098 inp->inp_gencnt = ++ipi->ipi_gencnt; 1099 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1100 in_pcbremlists(inp); 1101 so->so_pcb = NULL; 1102 sofree(so); /* remove pcb ref */ 1103 if (inp->inp_options) 1104 m_free(inp->inp_options); 1105 if (inp->inp_route.ro_rt) 1106 rtfree(inp->inp_route.ro_rt); 1107 ip_freemoptions(inp->inp_moptions); 1108 kfree(inp, M_PCB); 1109 } 1110 1111 /* 1112 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1113 * socket received RST. 1114 */ 1115 static int 1116 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1117 { 1118 struct inpcb *inp; 1119 struct sockaddr_in *sin; 1120 1121 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1122 inp = so->so_pcb; 1123 if (!inp) 1124 return (ECONNRESET); 1125 1126 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1127 sin->sin_family = AF_INET; 1128 sin->sin_len = sizeof *sin; 1129 sin->sin_port = inp->inp_lport; 1130 sin->sin_addr = inp->inp_laddr; 1131 1132 *nam = (struct sockaddr *)sin; 1133 return (0); 1134 } 1135 1136 void 1137 in_setsockaddr_dispatch(netmsg_t msg) 1138 { 1139 int error; 1140 1141 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1142 lwkt_replymsg(&msg->lmsg, error); 1143 } 1144 1145 /* 1146 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1147 * socket received RST. 1148 */ 1149 int 1150 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1151 { 1152 struct inpcb *inp; 1153 struct sockaddr_in *sin; 1154 1155 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1156 inp = so->so_pcb; 1157 if (!inp) 1158 return (ECONNRESET); 1159 1160 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1161 sin->sin_family = AF_INET; 1162 sin->sin_len = sizeof *sin; 1163 sin->sin_port = inp->inp_fport; 1164 sin->sin_addr = inp->inp_faddr; 1165 1166 *nam = (struct sockaddr *)sin; 1167 return (0); 1168 } 1169 1170 void 1171 in_setpeeraddr_dispatch(netmsg_t msg) 1172 { 1173 int error; 1174 1175 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1176 lwkt_replymsg(&msg->lmsg, error); 1177 } 1178 1179 void 1180 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1181 inp_notify_t notify) 1182 { 1183 struct inpcb *inp, *marker; 1184 1185 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1186 ("not in the correct netisr")); 1187 marker = in_pcbmarker(); 1188 1189 /* 1190 * NOTE: 1191 * - If INP_PLACEMARKER is set we must ignore the rest of the 1192 * structure and skip it. 1193 * - It is safe to nuke inpcbs here, since we are in their own 1194 * netisr. 1195 */ 1196 GET_PCBINFO_TOKEN(pcbinfo); 1197 1198 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1199 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1200 LIST_REMOVE(marker, inp_list); 1201 LIST_INSERT_AFTER(inp, marker, inp_list); 1202 1203 if (inp->inp_flags & INP_PLACEMARKER) 1204 continue; 1205 #ifdef INET6 1206 if (!INP_ISIPV4(inp)) 1207 continue; 1208 #endif 1209 if (inp->inp_faddr.s_addr != faddr.s_addr || 1210 inp->inp_socket == NULL) 1211 continue; 1212 (*notify)(inp, err); /* can remove inp from list! */ 1213 } 1214 LIST_REMOVE(marker, inp_list); 1215 1216 REL_PCBINFO_TOKEN(pcbinfo); 1217 } 1218 1219 void 1220 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1221 { 1222 struct inpcb *inp, *marker; 1223 1224 /* 1225 * We only need to make sure that we are in netisr0, where all 1226 * multicast operation happen. We could check inpcbinfo which 1227 * does not belong to netisr0 by holding the inpcbinfo's token. 1228 * In this case, the pcbinfo must be able to be shared, i.e. 1229 * pcbinfo->infotoken is not NULL. 1230 */ 1231 ASSERT_NETISR0; 1232 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1233 ("pcbinfo could not be shared")); 1234 1235 /* 1236 * Get a marker for the current netisr (netisr0). 1237 * 1238 * It is possible that the multicast address deletion blocks, 1239 * which could cause temporary token releasing. So we use 1240 * inpcb marker here to get a coherent view of the inpcb list. 1241 * 1242 * While, on the other hand, moptions are only added and deleted 1243 * in netisr0, so we would not see staled moption or miss moption 1244 * even if the token was released due to the blocking multicast 1245 * address deletion. 1246 */ 1247 marker = in_pcbmarker(); 1248 1249 GET_PCBINFO_TOKEN(pcbinfo); 1250 1251 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1252 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1253 struct ip_moptions *imo; 1254 1255 LIST_REMOVE(marker, inp_list); 1256 LIST_INSERT_AFTER(inp, marker, inp_list); 1257 1258 if (inp->inp_flags & INP_PLACEMARKER) 1259 continue; 1260 imo = inp->inp_moptions; 1261 if (INP_ISIPV4(inp) && imo != NULL) { 1262 int i, gap; 1263 1264 /* 1265 * Unselect the outgoing interface if it is being 1266 * detached. 1267 */ 1268 if (imo->imo_multicast_ifp == ifp) 1269 imo->imo_multicast_ifp = NULL; 1270 1271 /* 1272 * Drop multicast group membership if we joined 1273 * through the interface being detached. 1274 */ 1275 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1276 i++) { 1277 if (imo->imo_membership[i]->inm_ifp == ifp) { 1278 /* 1279 * NOTE: 1280 * This could block and the pcbinfo 1281 * token could be passively released. 1282 */ 1283 in_delmulti(imo->imo_membership[i]); 1284 gap++; 1285 } else if (gap != 0) 1286 imo->imo_membership[i - gap] = 1287 imo->imo_membership[i]; 1288 } 1289 imo->imo_num_memberships -= gap; 1290 } 1291 } 1292 LIST_REMOVE(marker, inp_list); 1293 1294 REL_PCBINFO_TOKEN(pcbinfo); 1295 } 1296 1297 /* 1298 * Check for alternatives when higher level complains 1299 * about service problems. For now, invalidate cached 1300 * routing information. If the route was created dynamically 1301 * (by a redirect), time to try a default gateway again. 1302 */ 1303 void 1304 in_losing(struct inpcb *inp) 1305 { 1306 struct rtentry *rt; 1307 struct rt_addrinfo rtinfo; 1308 1309 if ((rt = inp->inp_route.ro_rt)) { 1310 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1311 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1312 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1313 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1314 rtinfo.rti_flags = rt->rt_flags; 1315 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1316 if (rt->rt_flags & RTF_DYNAMIC) { 1317 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1318 rt_mask(rt), rt->rt_flags, NULL); 1319 } 1320 inp->inp_route.ro_rt = NULL; 1321 rtfree(rt); 1322 /* 1323 * A new route can be allocated 1324 * the next time output is attempted. 1325 */ 1326 } 1327 } 1328 1329 /* 1330 * After a routing change, flush old routing 1331 * and allocate a (hopefully) better one. 1332 */ 1333 void 1334 in_rtchange(struct inpcb *inp, int err) 1335 { 1336 if (inp->inp_route.ro_rt) { 1337 rtfree(inp->inp_route.ro_rt); 1338 inp->inp_route.ro_rt = NULL; 1339 /* 1340 * A new route can be allocated the next time 1341 * output is attempted. 1342 */ 1343 } 1344 } 1345 1346 /* 1347 * Lookup a PCB based on the local address and port. 1348 */ 1349 static struct inpcb * 1350 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr, 1351 u_int lport_arg, int wild_okay, struct ucred *cred) 1352 { 1353 struct inpcb *inp; 1354 int matchwild = 3, wildcard; 1355 u_short lport = lport_arg; 1356 struct inpcbport *phd; 1357 struct inpcb *match = NULL; 1358 1359 /* 1360 * If the porthashbase is shared across several cpus, it must 1361 * have been locked. 1362 */ 1363 ASSERT_PORTHASH_TOKEN_HELD(porthash); 1364 1365 /* 1366 * Best fit PCB lookup. 1367 * 1368 * First see if this local port is in use by looking on the 1369 * port hash list. 1370 */ 1371 LIST_FOREACH(phd, porthash, phd_hash) { 1372 if (phd->phd_port == lport) 1373 break; 1374 } 1375 if (phd != NULL) { 1376 /* 1377 * Port is in use by one or more PCBs. Look for best 1378 * fit. 1379 */ 1380 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1381 wildcard = 0; 1382 #ifdef INET6 1383 if (!INP_ISIPV4(inp)) 1384 continue; 1385 #endif 1386 if (inp->inp_faddr.s_addr != INADDR_ANY) 1387 wildcard++; 1388 if (inp->inp_laddr.s_addr != INADDR_ANY) { 1389 if (laddr.s_addr == INADDR_ANY) 1390 wildcard++; 1391 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1392 continue; 1393 } else { 1394 if (laddr.s_addr != INADDR_ANY) 1395 wildcard++; 1396 } 1397 if (wildcard && !wild_okay) 1398 continue; 1399 if (wildcard < matchwild && 1400 (cred == NULL || 1401 cred->cr_prison == 1402 inp->inp_socket->so_cred->cr_prison)) { 1403 match = inp; 1404 matchwild = wildcard; 1405 if (matchwild == 0) { 1406 break; 1407 } 1408 } 1409 } 1410 } 1411 return (match); 1412 } 1413 1414 struct inpcb * 1415 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1416 const struct inpcb *inp) 1417 { 1418 const struct inp_localgrphead *hdr; 1419 const struct inp_localgroup *grp; 1420 int i; 1421 1422 if (pcbinfo->localgrphashbase == NULL) 1423 return NULL; 1424 1425 GET_PCBINFO_TOKEN(pcbinfo); 1426 1427 hdr = &pcbinfo->localgrphashbase[ 1428 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1429 1430 LIST_FOREACH(grp, hdr, il_list) { 1431 if (grp->il_af == inp->inp_af && 1432 grp->il_lport == inp->inp_lport && 1433 memcmp(&grp->il_dependladdr, 1434 &inp->inp_inc.inc_ie.ie_dependladdr, 1435 sizeof(grp->il_dependladdr)) == 0) { 1436 break; 1437 } 1438 } 1439 if (grp == NULL || grp->il_inpcnt == 1) { 1440 REL_PCBINFO_TOKEN(pcbinfo); 1441 return NULL; 1442 } 1443 1444 KASSERT(grp->il_inpcnt >= 2, 1445 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1446 for (i = 0; i < grp->il_inpcnt; ++i) { 1447 if (grp->il_inp[i] == inp) { 1448 int last = grp->il_inpcnt - 1; 1449 1450 if (i == last) 1451 last = grp->il_inpcnt - 2; 1452 REL_PCBINFO_TOKEN(pcbinfo); 1453 return grp->il_inp[last]; 1454 } 1455 } 1456 REL_PCBINFO_TOKEN(pcbinfo); 1457 return NULL; 1458 } 1459 1460 static struct inpcb * 1461 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1462 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1463 { 1464 struct inpcb *local_wild = NULL; 1465 const struct inp_localgrphead *hdr; 1466 const struct inp_localgroup *grp; 1467 1468 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1469 1470 hdr = &pcbinfo->localgrphashbase[ 1471 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1472 1473 /* 1474 * Order of socket selection: 1475 * 1. non-wild. 1476 * 2. wild. 1477 * 1478 * NOTE: Local group does not contain jailed sockets 1479 */ 1480 LIST_FOREACH(grp, hdr, il_list) { 1481 #ifdef INET6 1482 if (grp->il_af != AF_INET) 1483 continue; 1484 #endif 1485 if (grp->il_lport == lport) { 1486 int idx; 1487 1488 /* 1489 * Modulo-N is used here, which greatly reduces 1490 * completion queue token contention, thus more 1491 * cpu time is saved. 1492 */ 1493 idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt; 1494 if (grp->il_laddr.s_addr == laddr.s_addr) 1495 return grp->il_inp[idx]; 1496 else if (grp->il_laddr.s_addr == INADDR_ANY) 1497 local_wild = grp->il_inp[idx]; 1498 } 1499 } 1500 if (local_wild != NULL) 1501 return local_wild; 1502 return NULL; 1503 } 1504 1505 /* 1506 * Lookup PCB in hash list. 1507 */ 1508 struct inpcb * 1509 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1510 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1511 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1512 { 1513 struct inpcbhead *head; 1514 struct inpcb *inp, *jinp=NULL; 1515 u_short fport = fport_arg, lport = lport_arg; 1516 1517 /* 1518 * First look for an exact match. 1519 */ 1520 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1521 laddr.s_addr, lport, pcbinfo->hashmask)]; 1522 LIST_FOREACH(inp, head, inp_hash) { 1523 #ifdef INET6 1524 if (!INP_ISIPV4(inp)) 1525 continue; 1526 #endif 1527 if (in_hosteq(inp->inp_faddr, faddr) && 1528 in_hosteq(inp->inp_laddr, laddr) && 1529 inp->inp_fport == fport && inp->inp_lport == lport) { 1530 /* found */ 1531 if (inp->inp_socket == NULL || 1532 inp->inp_socket->so_cred->cr_prison == NULL) { 1533 return (inp); 1534 } else { 1535 if (jinp == NULL) 1536 jinp = inp; 1537 } 1538 } 1539 } 1540 if (jinp != NULL) 1541 return (jinp); 1542 1543 if (wildcard) { 1544 struct inpcb *local_wild = NULL; 1545 struct inpcb *jinp_wild = NULL; 1546 struct inpcontainer *ic; 1547 struct inpcontainerhead *chead; 1548 struct sockaddr_in jsin; 1549 struct ucred *cred; 1550 1551 GET_PCBINFO_TOKEN(pcbinfo); 1552 1553 /* 1554 * Check local group first 1555 */ 1556 if (pcbinfo->localgrphashbase != NULL && 1557 m != NULL && (m->m_flags & M_HASH) && 1558 !(ifp && ifp->if_type == IFT_FAITH)) { 1559 inp = inp_localgroup_lookup(pcbinfo, 1560 laddr, lport, m->m_pkthdr.hash); 1561 if (inp != NULL) { 1562 REL_PCBINFO_TOKEN(pcbinfo); 1563 return inp; 1564 } 1565 } 1566 1567 /* 1568 * Order of socket selection: 1569 * 1. non-jailed, non-wild. 1570 * 2. non-jailed, wild. 1571 * 3. jailed, non-wild. 1572 * 4. jailed, wild. 1573 */ 1574 jsin.sin_family = AF_INET; 1575 chead = &pcbinfo->wildcardhashbase[ 1576 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1577 LIST_FOREACH(ic, chead, ic_list) { 1578 inp = ic->ic_inp; 1579 if (inp->inp_flags & INP_PLACEMARKER) 1580 continue; 1581 1582 jsin.sin_addr.s_addr = laddr.s_addr; 1583 #ifdef INET6 1584 if (!INP_ISIPV4(inp)) 1585 continue; 1586 #endif 1587 if (inp->inp_socket != NULL) 1588 cred = inp->inp_socket->so_cred; 1589 else 1590 cred = NULL; 1591 if (cred != NULL && jailed(cred)) { 1592 if (jinp != NULL) 1593 continue; 1594 else 1595 if (!jailed_ip(cred->cr_prison, 1596 (struct sockaddr *)&jsin)) 1597 continue; 1598 } 1599 if (inp->inp_lport == lport) { 1600 if (ifp && ifp->if_type == IFT_FAITH && 1601 !(inp->inp_flags & INP_FAITH)) 1602 continue; 1603 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1604 if (cred != NULL && jailed(cred)) { 1605 jinp = inp; 1606 } else { 1607 REL_PCBINFO_TOKEN(pcbinfo); 1608 return (inp); 1609 } 1610 } 1611 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1612 if (cred != NULL && jailed(cred)) 1613 jinp_wild = inp; 1614 else 1615 local_wild = inp; 1616 } 1617 } 1618 } 1619 1620 REL_PCBINFO_TOKEN(pcbinfo); 1621 1622 if (local_wild != NULL) 1623 return (local_wild); 1624 if (jinp != NULL) 1625 return (jinp); 1626 return (jinp_wild); 1627 } 1628 1629 /* 1630 * Not found. 1631 */ 1632 return (NULL); 1633 } 1634 1635 struct inpcb * 1636 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1637 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1638 boolean_t wildcard, struct ifnet *ifp) 1639 { 1640 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1641 laddr, lport_arg, wildcard, ifp, NULL); 1642 } 1643 1644 /* 1645 * Insert PCB into connection hash table. 1646 */ 1647 void 1648 in_pcbinsconnhash(struct inpcb *inp) 1649 { 1650 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1651 struct inpcbhead *bucket; 1652 u_int32_t hashkey_faddr, hashkey_laddr; 1653 1654 #ifdef INET6 1655 if (INP_ISIPV6(inp)) { 1656 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1657 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1658 } else { 1659 #endif 1660 hashkey_faddr = inp->inp_faddr.s_addr; 1661 hashkey_laddr = inp->inp_laddr.s_addr; 1662 #ifdef INET6 1663 } 1664 #endif 1665 1666 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1667 ("not in the correct netisr")); 1668 ASSERT_INP_NOTINHASH(inp); 1669 inp->inp_flags |= INP_CONNECTED; 1670 1671 /* 1672 * Insert into the connection hash table. 1673 */ 1674 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1675 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1676 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1677 } 1678 1679 /* 1680 * Remove PCB from connection hash table. 1681 */ 1682 void 1683 in_pcbremconnhash(struct inpcb *inp) 1684 { 1685 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1686 1687 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1688 ("not in the correct netisr")); 1689 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1690 1691 LIST_REMOVE(inp, inp_hash); 1692 inp->inp_flags &= ~INP_CONNECTED; 1693 } 1694 1695 /* 1696 * Insert PCB into port hash table. 1697 */ 1698 void 1699 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp) 1700 { 1701 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1702 struct inpcbport *phd; 1703 1704 /* 1705 * If the porthashbase is shared across several cpus, it must 1706 * have been locked. 1707 */ 1708 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash); 1709 1710 /* 1711 * Insert into the port hash table. 1712 */ 1713 1714 /* Go through port list and look for a head for this lport. */ 1715 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1716 if (phd->phd_port == inp->inp_lport) 1717 break; 1718 } 1719 1720 /* If none exists, use saved one and tack it on. */ 1721 if (phd == NULL) { 1722 KKASSERT(pcbinfo->portsave != NULL); 1723 phd = pcbinfo->portsave; 1724 pcbinfo->portsave = NULL; 1725 phd->phd_port = inp->inp_lport; 1726 LIST_INIT(&phd->phd_pcblist); 1727 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1728 } 1729 1730 inp->inp_porthash = pcbporthash; 1731 inp->inp_phd = phd; 1732 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1733 1734 /* 1735 * Malloc one inpcbport for later use. It is safe to use 1736 * "wait" malloc here (port token would be released, if 1737 * malloc ever blocked), since all changes to the porthash 1738 * are done. 1739 */ 1740 if (pcbinfo->portsave == NULL) { 1741 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1742 M_PCB, M_INTWAIT | M_ZERO); 1743 } 1744 } 1745 1746 void 1747 in_pcbinsporthash_lport(struct inpcb *inp) 1748 { 1749 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1750 struct inpcbportinfo *portinfo; 1751 struct inpcbporthead *porthash; 1752 u_short lport_ho; 1753 1754 /* Locate the proper portinfo based on lport */ 1755 lport_ho = ntohs(inp->inp_lport); 1756 portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 1757 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset); 1758 1759 porthash = in_pcbporthash_head(portinfo, inp->inp_lport); 1760 GET_PORTHASH_TOKEN(porthash); 1761 in_pcbinsporthash(porthash, inp); 1762 REL_PORTHASH_TOKEN(porthash); 1763 } 1764 1765 void 1766 in_pcbremporthash(struct inpcb *inp) 1767 { 1768 struct inpcbporthead *porthash; 1769 struct inpcbport *phd; 1770 1771 if (inp->inp_phd == NULL) 1772 return; 1773 KASSERT(inp->inp_lport != 0, ("inpcb has no lport")); 1774 1775 porthash = inp->inp_porthash; 1776 KASSERT(porthash != NULL, ("no porthash")); 1777 1778 GET_PORTHASH_TOKEN(porthash); 1779 1780 phd = inp->inp_phd; 1781 LIST_REMOVE(inp, inp_portlist); 1782 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 1783 LIST_REMOVE(phd, phd_hash); 1784 kfree(phd, M_PCB); 1785 } 1786 1787 REL_PORTHASH_TOKEN(porthash); 1788 1789 inp->inp_phd = NULL; 1790 /* NOTE: Don't whack inp_lport, which may be used later */ 1791 } 1792 1793 static struct inp_localgroup * 1794 inp_localgroup_alloc(u_char af, uint16_t port, 1795 const union in_dependaddr *addr, int size) 1796 { 1797 struct inp_localgroup *grp; 1798 1799 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 1800 M_TEMP, M_INTWAIT | M_ZERO); 1801 grp->il_af = af; 1802 grp->il_lport = port; 1803 grp->il_dependladdr = *addr; 1804 grp->il_inpsiz = size; 1805 1806 return grp; 1807 } 1808 1809 static void 1810 inp_localgroup_free(struct inp_localgroup *grp) 1811 { 1812 kfree(grp, M_TEMP); 1813 } 1814 1815 static void 1816 inp_localgroup_destroy(struct inp_localgroup *grp) 1817 { 1818 LIST_REMOVE(grp, il_list); 1819 inp_localgroup_free(grp); 1820 } 1821 1822 static void 1823 inp_localgroup_copy(struct inp_localgroup *grp, 1824 const struct inp_localgroup *old_grp) 1825 { 1826 int i; 1827 1828 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 1829 ("invalid new local group size %d and old local group count %d", 1830 grp->il_inpsiz, old_grp->il_inpcnt)); 1831 for (i = 0; i < old_grp->il_inpcnt; ++i) 1832 grp->il_inp[i] = old_grp->il_inp[i]; 1833 grp->il_inpcnt = old_grp->il_inpcnt; 1834 } 1835 1836 static void 1837 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1838 { 1839 struct inp_localgrphead *hdr; 1840 struct inp_localgroup *grp, *grp_alloc = NULL; 1841 struct ucred *cred; 1842 int i, idx; 1843 1844 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1845 1846 if (pcbinfo->localgrphashbase == NULL) 1847 return; 1848 1849 /* 1850 * XXX don't allow jailed socket to join local group 1851 */ 1852 if (inp->inp_socket != NULL) 1853 cred = inp->inp_socket->so_cred; 1854 else 1855 cred = NULL; 1856 if (cred != NULL && jailed(cred)) 1857 return; 1858 1859 hdr = &pcbinfo->localgrphashbase[ 1860 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1861 1862 again: 1863 LIST_FOREACH(grp, hdr, il_list) { 1864 if (grp->il_af == inp->inp_af && 1865 grp->il_lport == inp->inp_lport && 1866 memcmp(&grp->il_dependladdr, 1867 &inp->inp_inc.inc_ie.ie_dependladdr, 1868 sizeof(grp->il_dependladdr)) == 0) { 1869 break; 1870 } 1871 } 1872 if (grp == NULL) { 1873 /* 1874 * Create a new local group 1875 */ 1876 if (grp_alloc == NULL) { 1877 grp_alloc = inp_localgroup_alloc(inp->inp_af, 1878 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 1879 INP_LOCALGROUP_SIZMIN); 1880 /* 1881 * Local group allocation could block and the 1882 * local group w/ the same property might have 1883 * been added by others when we were blocked; 1884 * check again. 1885 */ 1886 goto again; 1887 } else { 1888 /* Local group has been allocated; link it */ 1889 grp = grp_alloc; 1890 grp_alloc = NULL; 1891 LIST_INSERT_HEAD(hdr, grp, il_list); 1892 } 1893 } else if (grp->il_inpcnt == grp->il_inpsiz) { 1894 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 1895 static int limit_logged = 0; 1896 1897 if (!limit_logged) { 1898 limit_logged = 1; 1899 kprintf("local group port %d, " 1900 "limit reached\n", ntohs(grp->il_lport)); 1901 } 1902 if (grp_alloc != NULL) { 1903 /* 1904 * This would happen if the local group 1905 * w/ the same property was expanded when 1906 * our local group allocation blocked. 1907 */ 1908 inp_localgroup_free(grp_alloc); 1909 } 1910 return; 1911 } 1912 1913 /* 1914 * Expand this local group 1915 */ 1916 if (grp_alloc == NULL || 1917 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 1918 if (grp_alloc != NULL) 1919 inp_localgroup_free(grp_alloc); 1920 grp_alloc = inp_localgroup_alloc(grp->il_af, 1921 grp->il_lport, &grp->il_dependladdr, 1922 grp->il_inpsiz * 2); 1923 /* 1924 * Local group allocation could block and the 1925 * local group w/ the same property might have 1926 * been expanded by others when we were blocked; 1927 * check again. 1928 */ 1929 goto again; 1930 } 1931 1932 /* 1933 * Save the old local group, link the new one, and then 1934 * destroy the old local group 1935 */ 1936 inp_localgroup_copy(grp_alloc, grp); 1937 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 1938 inp_localgroup_destroy(grp); 1939 1940 grp = grp_alloc; 1941 grp_alloc = NULL; 1942 } else { 1943 /* 1944 * Found the local group 1945 */ 1946 if (grp_alloc != NULL) { 1947 /* 1948 * This would happen if the local group w/ the 1949 * same property was added or expanded when our 1950 * local group allocation blocked. 1951 */ 1952 inp_localgroup_free(grp_alloc); 1953 grp_alloc = NULL; 1954 } 1955 } 1956 1957 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 1958 ("invalid local group size %d and count %d", 1959 grp->il_inpsiz, grp->il_inpcnt)); 1960 1961 /* 1962 * Keep the local group sorted by the inpcb local group index 1963 * in ascending order. 1964 * 1965 * This eases the multi-process userland application which uses 1966 * SO_REUSEPORT sockets and binds process to the owner cpu of 1967 * the SO_REUSEPORT socket: 1968 * If we didn't sort the local group by the inpcb local group 1969 * index and one of the process owning an inpcb in this local 1970 * group restarted, e.g. crashed and restarted by watchdog, 1971 * other processes owning a inpcb in this local group would have 1972 * to detect that event, refetch its socket's owner cpu, and 1973 * re-bind. 1974 */ 1975 idx = grp->il_inpcnt; 1976 for (i = 0; i < idx; ++i) { 1977 struct inpcb *oinp = grp->il_inp[i]; 1978 1979 if (oinp->inp_lgrpindex > i) { 1980 if (inp->inp_lgrpindex < 0) { 1981 inp->inp_lgrpindex = i; 1982 } else if (inp->inp_lgrpindex != i) { 1983 if (bootverbose) { 1984 kprintf("inp %p: grpidx %d, " 1985 "assigned to %d, cpu%d\n", 1986 inp, inp->inp_lgrpindex, i, 1987 mycpuid); 1988 } 1989 } 1990 grp->il_inp[i] = inp; 1991 1992 /* Pull down inpcbs */ 1993 for (; i < grp->il_inpcnt; ++i) { 1994 struct inpcb *oinp1 = grp->il_inp[i + 1]; 1995 1996 grp->il_inp[i + 1] = oinp; 1997 oinp = oinp1; 1998 } 1999 grp->il_inpcnt++; 2000 return; 2001 } 2002 } 2003 2004 if (inp->inp_lgrpindex < 0) { 2005 inp->inp_lgrpindex = idx; 2006 } else if (inp->inp_lgrpindex != idx) { 2007 if (bootverbose) { 2008 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 2009 inp, inp->inp_lgrpindex, idx, mycpuid); 2010 } 2011 } 2012 grp->il_inp[idx] = inp; 2013 grp->il_inpcnt++; 2014 } 2015 2016 void 2017 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2018 { 2019 struct inpcontainer *ic; 2020 struct inpcontainerhead *bucket; 2021 2022 GET_PCBINFO_TOKEN(pcbinfo); 2023 2024 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 2025 2026 bucket = &pcbinfo->wildcardhashbase[ 2027 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2028 2029 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 2030 ic->ic_inp = inp; 2031 LIST_INSERT_HEAD(bucket, ic, ic_list); 2032 2033 REL_PCBINFO_TOKEN(pcbinfo); 2034 } 2035 2036 /* 2037 * Insert PCB into wildcard hash table. 2038 */ 2039 void 2040 in_pcbinswildcardhash(struct inpcb *inp) 2041 { 2042 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2043 2044 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2045 ("not in correct netisr")); 2046 ASSERT_INP_NOTINHASH(inp); 2047 inp->inp_flags |= INP_WILDCARD; 2048 2049 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 2050 } 2051 2052 static void 2053 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2054 { 2055 struct inp_localgrphead *hdr; 2056 struct inp_localgroup *grp; 2057 2058 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2059 2060 if (pcbinfo->localgrphashbase == NULL) 2061 return; 2062 2063 hdr = &pcbinfo->localgrphashbase[ 2064 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2065 2066 LIST_FOREACH(grp, hdr, il_list) { 2067 int i; 2068 2069 for (i = 0; i < grp->il_inpcnt; ++i) { 2070 if (grp->il_inp[i] != inp) 2071 continue; 2072 2073 if (grp->il_inpcnt == 1) { 2074 /* Destroy this local group */ 2075 inp_localgroup_destroy(grp); 2076 } else { 2077 /* Pull up inpcbs */ 2078 for (; i + 1 < grp->il_inpcnt; ++i) 2079 grp->il_inp[i] = grp->il_inp[i + 1]; 2080 grp->il_inpcnt--; 2081 } 2082 return; 2083 } 2084 } 2085 } 2086 2087 void 2088 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2089 { 2090 struct inpcontainer *ic; 2091 struct inpcontainerhead *head; 2092 2093 GET_PCBINFO_TOKEN(pcbinfo); 2094 2095 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2096 2097 /* find bucket */ 2098 head = &pcbinfo->wildcardhashbase[ 2099 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2100 2101 LIST_FOREACH(ic, head, ic_list) { 2102 if (ic->ic_inp == inp) 2103 goto found; 2104 } 2105 REL_PCBINFO_TOKEN(pcbinfo); 2106 return; /* not found! */ 2107 2108 found: 2109 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2110 REL_PCBINFO_TOKEN(pcbinfo); 2111 kfree(ic, M_TEMP); /* deallocate container */ 2112 } 2113 2114 /* 2115 * Remove PCB from wildcard hash table. 2116 */ 2117 void 2118 in_pcbremwildcardhash(struct inpcb *inp) 2119 { 2120 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2121 2122 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2123 ("not in correct netisr")); 2124 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2125 2126 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2127 inp->inp_lgrpindex = -1; 2128 inp->inp_flags &= ~INP_WILDCARD; 2129 } 2130 2131 /* 2132 * Remove PCB from various lists. 2133 */ 2134 void 2135 in_pcbremlists(struct inpcb *inp) 2136 { 2137 in_pcbremporthash(inp); 2138 if (inp->inp_flags & INP_WILDCARD) { 2139 in_pcbremwildcardhash(inp); 2140 } else if (inp->inp_flags & INP_CONNECTED) { 2141 in_pcbremconnhash(inp); 2142 } 2143 2144 if (inp->inp_flags & INP_ONLIST) 2145 in_pcbofflist(inp); 2146 } 2147 2148 int 2149 prison_xinpcb(struct thread *td, struct inpcb *inp) 2150 { 2151 struct ucred *cr; 2152 2153 if (td->td_proc == NULL) 2154 return (0); 2155 cr = td->td_proc->p_ucred; 2156 if (cr->cr_prison == NULL) 2157 return (0); 2158 if (inp->inp_socket && inp->inp_socket->so_cred && 2159 inp->inp_socket->so_cred->cr_prison && 2160 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2161 return (0); 2162 return (1); 2163 } 2164 2165 int 2166 in_pcblist_range(SYSCTL_HANDLER_ARGS) 2167 { 2168 struct inpcbinfo *pcbinfo_arr = arg1; 2169 int pcbinfo_arrlen = arg2; 2170 struct inpcb *marker; 2171 int cpu, origcpu; 2172 int error, n; 2173 2174 KASSERT(pcbinfo_arrlen <= netisr_ncpus && pcbinfo_arrlen >= 1, 2175 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2176 2177 /* 2178 * The process of preparing the TCB list is too time-consuming and 2179 * resource-intensive to repeat twice on every request. 2180 */ 2181 n = 0; 2182 if (req->oldptr == NULL) { 2183 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2184 n += pcbinfo_arr[cpu].ipi_count; 2185 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2186 return 0; 2187 } 2188 2189 if (req->newptr != NULL) 2190 return EPERM; 2191 2192 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2193 marker->inp_flags |= INP_PLACEMARKER; 2194 2195 /* 2196 * OK, now we're committed to doing something. Re-fetch ipi_count 2197 * after obtaining the generation count. 2198 */ 2199 error = 0; 2200 origcpu = mycpuid; 2201 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2202 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2203 struct inpcb *inp; 2204 struct xinpcb xi; 2205 int i; 2206 2207 lwkt_migratecpu(cpu); 2208 2209 GET_PCBINFO_TOKEN(pcbinfo); 2210 2211 n = pcbinfo->ipi_count; 2212 2213 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2214 i = 0; 2215 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2216 LIST_REMOVE(marker, inp_list); 2217 LIST_INSERT_AFTER(inp, marker, inp_list); 2218 2219 if (inp->inp_flags & INP_PLACEMARKER) 2220 continue; 2221 if (prison_xinpcb(req->td, inp)) 2222 continue; 2223 2224 bzero(&xi, sizeof xi); 2225 xi.xi_len = sizeof xi; 2226 bcopy(inp, &xi.xi_inp, sizeof *inp); 2227 if (inp->inp_socket) 2228 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2229 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2230 break; 2231 ++i; 2232 } 2233 LIST_REMOVE(marker, inp_list); 2234 2235 REL_PCBINFO_TOKEN(pcbinfo); 2236 2237 if (error == 0 && i < n) { 2238 bzero(&xi, sizeof xi); 2239 xi.xi_len = sizeof xi; 2240 while (i < n) { 2241 error = SYSCTL_OUT(req, &xi, sizeof xi); 2242 if (error) 2243 break; 2244 ++i; 2245 } 2246 } 2247 } 2248 2249 lwkt_migratecpu(origcpu); 2250 kfree(marker, M_TEMP); 2251 return error; 2252 } 2253 2254 int 2255 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS) 2256 { 2257 2258 return (in_pcblist_range(oidp, arg1, netisr_ncpus, req)); 2259 } 2260 2261 void 2262 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2263 { 2264 struct sockaddr_in *sin; 2265 2266 KASSERT(faddr->sa_family == AF_INET, 2267 ("not AF_INET faddr %d", faddr->sa_family)); 2268 2269 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2270 sin->sin_family = AF_INET; 2271 sin->sin_len = sizeof(*sin); 2272 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2273 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2274 2275 so->so_faddr = (struct sockaddr *)sin; 2276 } 2277 2278 void 2279 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2280 u_short offset) 2281 { 2282 memset(portinfo, 0, sizeof(*portinfo)); 2283 2284 portinfo->offset = offset; 2285 portinfo->porthashbase = phashinit(hashsize, M_PCB, 2286 &portinfo->porthashcnt); 2287 } 2288 2289 void 2290 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2291 { 2292 int hi, lo; 2293 2294 if (step == 1) 2295 return; 2296 2297 hi = *hi0; 2298 lo = *lo0; 2299 2300 hi = rounddown(hi, step); 2301 hi += ofs; 2302 if (hi > (int)*hi0) 2303 hi -= step; 2304 2305 lo = roundup(lo, step); 2306 lo -= (step - ofs); 2307 if (lo < (int)*lo0) 2308 lo += step; 2309 2310 *hi0 = hi; 2311 *lo0 = lo; 2312 } 2313 2314 void 2315 in_pcbglobalinit(void) 2316 { 2317 int cpu; 2318 2319 in_pcbmarkers = kmalloc(netisr_ncpus * sizeof(struct inpcb), M_PCB, 2320 M_WAITOK | M_ZERO); 2321 in_pcbcontainer_markers = 2322 kmalloc(netisr_ncpus * sizeof(struct inpcontainer), M_PCB, 2323 M_WAITOK | M_ZERO); 2324 2325 for (cpu = 0; cpu < netisr_ncpus; ++cpu) { 2326 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2327 struct inpcb *marker = &in_pcbmarkers[cpu]; 2328 2329 marker->inp_flags |= INP_PLACEMARKER; 2330 ic->ic_inp = marker; 2331 } 2332 } 2333 2334 struct inpcb * 2335 in_pcbmarker(void) 2336 { 2337 2338 ASSERT_NETISR_NCPUS(mycpuid); 2339 return &in_pcbmarkers[mycpuid]; 2340 } 2341 2342 struct inpcontainer * 2343 in_pcbcontainer_marker(void) 2344 { 2345 2346 ASSERT_NETISR_NCPUS(mycpuid); 2347 return &in_pcbcontainer_markers[mycpuid]; 2348 } 2349 2350 void 2351 in_pcbresetroute(struct inpcb *inp) 2352 { 2353 struct route *ro = &inp->inp_route; 2354 2355 if (ro->ro_rt != NULL) 2356 RTFREE(ro->ro_rt); 2357 bzero(ro, sizeof(*ro)); 2358 } 2359