1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_inet6.h" 67 68 #include <sys/param.h> 69 #include <sys/systm.h> 70 #include <sys/malloc.h> 71 #include <sys/mbuf.h> 72 #include <sys/domain.h> 73 #include <sys/protosw.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/proc.h> 77 #include <sys/priv.h> 78 #include <sys/jail.h> 79 #include <sys/kernel.h> 80 #include <sys/sysctl.h> 81 82 #include <sys/socketvar2.h> 83 #include <sys/msgport2.h> 84 85 #include <machine/limits.h> 86 87 #include <net/if.h> 88 #include <net/if_types.h> 89 #include <net/route.h> 90 #include <net/netisr2.h> 91 #include <net/toeplitz2.h> 92 93 #include <netinet/in.h> 94 #include <netinet/in_pcb.h> 95 #include <netinet/in_var.h> 96 #include <netinet/ip_var.h> 97 #ifdef INET6 98 #include <netinet/ip6.h> 99 #include <netinet6/ip6_var.h> 100 #endif /* INET6 */ 101 102 #define INP_LOCALGROUP_SIZMIN 8 103 #define INP_LOCALGROUP_SIZMAX 256 104 105 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash, 106 struct in_addr laddr, u_int lport_arg, int wild_okay, 107 struct ucred *cred); 108 109 struct in_addr zeroin_addr; 110 111 /* 112 * These configure the range of local port addresses assigned to 113 * "unspecified" outgoing connections/packets/whatever. 114 */ 115 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 116 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 117 118 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 119 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 120 121 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 122 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 123 124 #define RANGECHK(var, min, max) \ 125 if ((var) < (min)) { (var) = (min); } \ 126 else if ((var) > (max)) { (var) = (max); } 127 128 int udpencap_enable = 1; /* enabled by default */ 129 int udpencap_port = 4500; /* triggers decapsulation */ 130 131 /* 132 * Per-netisr inpcb markers. 133 * NOTE: they should only be used in netisrs. 134 */ 135 static struct inpcb *in_pcbmarkers; 136 static struct inpcontainer *in_pcbcontainer_markers; 137 138 static int 139 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 140 { 141 int error; 142 143 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 144 if (!error) { 145 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 146 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 147 148 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 149 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 150 151 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 152 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 153 } 154 return (error); 155 } 156 157 #undef RANGECHK 158 159 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 160 161 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 162 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 163 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 164 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 165 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 166 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 167 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 168 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 169 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 170 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 172 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 173 174 /* Initialized by ip_init() */ 175 int ip_porthash_trycount; 176 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW, 177 &ip_porthash_trycount, 0, 178 "Number of tries to find local port matching hash of 4-tuple"); 179 180 /* 181 * in_pcb.c: manage the Protocol Control Blocks. 182 * 183 * NOTE: It is assumed that most of these functions will be called from 184 * a critical section. XXX - There are, unfortunately, a few exceptions 185 * to this rule that should be fixed. 186 * 187 * NOTE: The caller should initialize the cpu field to the cpu running the 188 * protocol stack associated with this inpcbinfo. 189 */ 190 191 void 192 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 193 { 194 KASSERT(cpu >= 0 && cpu < netisr_ncpus, ("invalid cpu%d", cpu)); 195 pcbinfo->cpu = cpu; 196 197 LIST_INIT(&pcbinfo->pcblisthead); 198 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 199 M_WAITOK | M_ZERO); 200 201 if (shared) { 202 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 203 M_PCB, M_WAITOK); 204 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 205 } else { 206 pcbinfo->infotoken = NULL; 207 } 208 } 209 210 void 211 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo, 212 int portinfo_cnt) 213 { 214 215 KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt)); 216 pcbinfo->portinfo = portinfo; 217 pcbinfo->portinfo_cnt = portinfo_cnt; 218 } 219 220 struct baddynamicports baddynamicports; 221 222 /* 223 * Check if the specified port is invalid for dynamic allocation. 224 */ 225 int 226 in_baddynamic(u_int16_t port, u_int16_t proto) 227 { 228 switch (proto) { 229 case IPPROTO_TCP: 230 return (DP_ISSET(baddynamicports.tcp, port)); 231 case IPPROTO_UDP: 232 return (DP_ISSET(baddynamicports.udp, port)); 233 default: 234 return (0); 235 } 236 } 237 238 void 239 in_pcbonlist(struct inpcb *inp) 240 { 241 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 242 243 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 244 ("not in the correct netisr")); 245 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 246 inp->inp_flags |= INP_ONLIST; 247 248 GET_PCBINFO_TOKEN(pcbinfo); 249 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 250 pcbinfo->ipi_count++; 251 REL_PCBINFO_TOKEN(pcbinfo); 252 } 253 254 void 255 in_pcbofflist(struct inpcb *inp) 256 { 257 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 258 259 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 260 ("not in the correct netisr")); 261 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 262 inp->inp_flags &= ~INP_ONLIST; 263 264 GET_PCBINFO_TOKEN(pcbinfo); 265 LIST_REMOVE(inp, inp_list); 266 KASSERT(pcbinfo->ipi_count > 0, 267 ("invalid inpcb count %d", pcbinfo->ipi_count)); 268 pcbinfo->ipi_count--; 269 REL_PCBINFO_TOKEN(pcbinfo); 270 } 271 272 /* 273 * Allocate a PCB and associate it with the socket. 274 */ 275 int 276 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 277 { 278 struct inpcb *inp; 279 280 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 281 if (inp == NULL) 282 return (ENOMEM); 283 inp->inp_lgrpindex = -1; 284 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 285 inp->inp_pcbinfo = pcbinfo; 286 inp->inp_socket = so; 287 #ifdef INET6 288 if (INP_CHECK_SOCKAF(so, AF_INET6)) { 289 if (ip6_auto_flowlabel) 290 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 291 inp->inp_af = AF_INET6; 292 } else 293 #endif 294 inp->inp_af = AF_INET; 295 soreference(so); 296 so->so_pcb = inp; 297 298 in_pcbonlist(inp); 299 return (0); 300 } 301 302 /* 303 * Unlink a pcb with the intention of moving it to another cpu with a 304 * different pcbinfo. While unlinked nothing should attempt to dereference 305 * inp_pcbinfo, NULL it out so we assert if it does. 306 */ 307 void 308 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 309 { 310 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 311 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 312 ("already linked")); 313 314 in_pcbofflist(inp); 315 inp->inp_pcbinfo = NULL; 316 } 317 318 void 319 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 320 { 321 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 322 } 323 324 /* 325 * Relink a pcb into a new pcbinfo. 326 */ 327 void 328 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 329 { 330 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 331 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 332 ("already linked")); 333 334 inp->inp_pcbinfo = pcbinfo; 335 in_pcbonlist(inp); 336 } 337 338 void 339 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 340 { 341 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 342 } 343 344 static boolean_t 345 in_pcbporthash_update(struct inpcbportinfo *portinfo, 346 struct inpcb *inp, u_short lport, struct ucred *cred, int wild) 347 { 348 struct inpcbporthead *porthash; 349 350 /* 351 * This has to be atomic. If the porthash is shared across multiple 352 * protocol threads, e.g. tcp and udp, then the token must be held. 353 */ 354 porthash = in_pcbporthash_head(portinfo, lport); 355 GET_PORTHASH_TOKEN(porthash); 356 357 if (in_pcblookup_local(porthash, inp->inp_laddr, lport, wild, cred)) { 358 REL_PORTHASH_TOKEN(porthash); 359 return FALSE; 360 } 361 inp->inp_lport = lport; 362 in_pcbinsporthash(porthash, inp); 363 364 REL_PORTHASH_TOKEN(porthash); 365 return TRUE; 366 } 367 368 static int 369 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 370 { 371 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 372 struct inpcbportinfo *portinfo; 373 u_short first, last, lport, step, first0, last0; 374 int count, error; 375 int portinfo_first, portinfo_idx; 376 uint32_t cut; 377 378 inp->inp_flags |= INP_ANONPORT; 379 380 step = pcbinfo->portinfo_cnt; 381 portinfo_first = mycpuid % pcbinfo->portinfo_cnt; 382 portinfo_idx = portinfo_first; 383 384 if (inp->inp_flags & INP_HIGHPORT) { 385 first0 = ipport_hifirstauto; /* sysctl */ 386 last0 = ipport_hilastauto; 387 } else if (inp->inp_flags & INP_LOWPORT) { 388 if (cred && 389 (error = 390 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 391 inp->inp_laddr.s_addr = INADDR_ANY; 392 return error; 393 } 394 first0 = ipport_lowfirstauto; /* 1023 */ 395 last0 = ipport_lowlastauto; /* 600 */ 396 } else { 397 first0 = ipport_firstauto; /* sysctl */ 398 last0 = ipport_lastauto; 399 } 400 if (first0 > last0) { 401 lport = last0; 402 last0 = first0; 403 first0 = lport; 404 } 405 KKASSERT(last0 >= first0); 406 407 cut = karc4random(); 408 loop: 409 portinfo = &pcbinfo->portinfo[portinfo_idx]; 410 first = first0; 411 last = last0; 412 413 /* 414 * Simple check to ensure all ports are not used up causing 415 * a deadlock here. 416 */ 417 in_pcbportrange(&last, &first, portinfo->offset, step); 418 lport = last - first; 419 count = lport / step; 420 421 lport = rounddown(cut % lport, step) + first; 422 KKASSERT(lport % step == portinfo->offset); 423 424 for (;;) { 425 if (count-- < 0) { /* completely used? */ 426 error = EADDRNOTAVAIL; 427 break; 428 } 429 430 if (__predict_false(lport < first || lport > last)) { 431 lport = first; 432 KKASSERT(lport % step == portinfo->offset); 433 } 434 435 if (in_pcbporthash_update(portinfo, inp, htons(lport), 436 cred, wild)) { 437 error = 0; 438 break; 439 } 440 441 lport += step; 442 KKASSERT(lport % step == portinfo->offset); 443 } 444 445 if (error) { 446 /* Try next portinfo */ 447 portinfo_idx++; 448 portinfo_idx %= pcbinfo->portinfo_cnt; 449 if (portinfo_idx != portinfo_first) 450 goto loop; 451 inp->inp_laddr.s_addr = INADDR_ANY; 452 } 453 return error; 454 } 455 456 static __inline struct inpcbporthead * 457 OBTAIN_LPORTHASH_TOKEN(struct inpcbinfo *pcbinfo, u_short lport) 458 { 459 struct inpcbportinfo *portinfo; 460 struct inpcbporthead *porthash; 461 u_short lport_ho = ntohs(lport); 462 463 /* 464 * Locate the proper portinfo based on lport 465 */ 466 portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt]; 467 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset); 468 469 porthash = in_pcbporthash_head(portinfo, lport); 470 GET_PORTHASH_TOKEN(porthash); 471 472 return porthash; 473 } 474 475 static int 476 in_pcbbind_laddr(struct sockaddr_in *sin, struct in_addr *laddr, 477 struct thread *td) 478 { 479 struct sockaddr_in jsin; 480 481 if (!prison_replace_wildcards(td, (struct sockaddr *)sin)) 482 return (EINVAL); 483 484 if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 485 sin->sin_addr.s_addr != INADDR_ANY) { 486 sin->sin_port = 0; /* yech... */ 487 bzero(&sin->sin_zero, sizeof sin->sin_zero); 488 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 489 return (EADDRNOTAVAIL); 490 } 491 492 *laddr = sin->sin_addr; 493 494 jsin.sin_family = AF_INET; 495 jsin.sin_addr.s_addr = laddr->s_addr; 496 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 497 laddr->s_addr = INADDR_ANY; 498 return (EINVAL); 499 } 500 laddr->s_addr = jsin.sin_addr.s_addr; 501 502 return (0); 503 } 504 505 static int 506 in_pcbbind_laddrport_check(const struct socket *so, struct sockaddr_in *sin, 507 struct inpcbporthead *porthash, int wild, struct ucred *cred, 508 struct thread *td) 509 { 510 int reuseport = (so->so_options & SO_REUSEPORT); 511 struct inpcb *t; 512 513 ASSERT_PORTHASH_TOKEN_HELD(porthash); 514 515 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 516 /* 517 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 518 * allow complete duplication of binding if 519 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 520 * and a multicast address is bound on both 521 * new and duplicated sockets. 522 */ 523 if (so->so_options & SO_REUSEADDR) 524 reuseport = SO_REUSEADDR | SO_REUSEPORT; 525 } 526 527 if (so->so_cred->cr_uid != 0 && 528 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 529 t = in_pcblookup_local(porthash, sin->sin_addr, sin->sin_port, 530 INPLOOKUP_WILDCARD, cred); 531 if (t && t->inp_socket != so && 532 (so->so_cred->cr_uid != t->inp_socket->so_cred->cr_uid)) 533 return (EADDRINUSE); 534 } 535 if (cred && !prison_replace_wildcards(td, (struct sockaddr *)sin)) 536 return (EADDRNOTAVAIL); 537 538 /* 539 * When binding to a local port if the best match is against 540 * an accepted socket we generally want to allow the binding. 541 * This means that there is no longer any specific socket 542 * bound or bound for listening. 543 */ 544 t = in_pcblookup_local(porthash, sin->sin_addr, sin->sin_port, 545 wild, cred); 546 if (t && t->inp_socket != so && 547 (reuseport & t->inp_socket->so_options) == 0 && 548 (t->inp_socket->so_state & SS_ACCEPTMECH) == 0) 549 return (EADDRINUSE); 550 551 return (0); 552 } 553 554 int 555 in_pcbsrcaddr_check(const struct inpcb *inp, struct sockaddr_in *sin, 556 struct in_addr *laddr, struct thread *td) 557 { 558 const struct socket *so = inp->inp_socket; 559 struct inpcbporthead *porthash; 560 struct ucred *cred = NULL; 561 int wild = 0; 562 int error; 563 564 /* inp must be bound beforehand. */ 565 KKASSERT(inp->inp_lport != 0); 566 KKASSERT(sin->sin_len == sizeof(*sin)); 567 568 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 569 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 570 if (td->td_proc) 571 cred = td->td_proc->p_ucred; 572 573 /* Always use inp_lport */ 574 sin->sin_port = inp->inp_lport; 575 576 error = in_pcbbind_laddr(sin, laddr, td); 577 if (error) 578 return (error); 579 580 if (IN_MULTICAST(ntohl(laddr->s_addr))) { 581 /* Unlike bind, multicast src address is not allowed. */ 582 return (EINVAL); 583 } 584 585 if (inp->inp_laddr.s_addr == laddr->s_addr) { 586 /* 587 * src address is same as what we bound to. 588 * 589 * inp_laddr == INADDR_ANY && srcaddr == INADDR_ANY 590 * is allowed, which does not really matter. 591 */ 592 return (0); 593 } else if (inp->inp_laddr.s_addr != INADDR_ANY && 594 !IN_MULTICAST(ntohl(inp->inp_laddr.s_addr))) { 595 /* Already bound to a specific address */ 596 return (EINVAL); 597 } 598 599 /* 600 * This has to be atomic. If the porthash is shared across 601 * multiple protocol threads, e.g. tcp and udp then the token 602 * must be held. 603 */ 604 porthash = OBTAIN_LPORTHASH_TOKEN(inp->inp_pcbinfo, inp->inp_lport); 605 606 /* 607 * Restore the sin_port whacked by in_pcbbind_ladddr(); 608 * sin->sin_port is checked by in_pcbbind_laddrport_check(). 609 */ 610 sin->sin_port = inp->inp_lport; 611 612 error = in_pcbbind_laddrport_check(so, sin, porthash, wild, cred, td); 613 if (error) 614 laddr->s_addr = INADDR_ANY; 615 616 REL_PORTHASH_TOKEN(porthash); 617 return (error); 618 } 619 620 int 621 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 622 { 623 const struct socket *so = inp->inp_socket; 624 struct ucred *cred = NULL; 625 int wild = 0; 626 627 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 628 return (EADDRNOTAVAIL); 629 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 630 return (EINVAL); /* already bound */ 631 632 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 633 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 634 if (td->td_proc) 635 cred = td->td_proc->p_ucred; 636 637 if (nam != NULL) { 638 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 639 struct inpcbporthead *porthash; 640 u_short lport; 641 int error; 642 643 if (nam->sa_len != sizeof *sin) 644 return (EINVAL); 645 #ifdef notdef 646 /* 647 * We should check the family, but old programs 648 * incorrectly fail to initialize it. 649 */ 650 if (sin->sin_family != AF_INET) 651 return (EAFNOSUPPORT); 652 #endif 653 654 /* 655 * Save sin_port for later use, since it will 656 * be whacked by in_pcbbind_laddr(). 657 */ 658 lport = sin->sin_port; 659 660 error = in_pcbbind_laddr(sin, &inp->inp_laddr, td); 661 if (error) 662 return (error); 663 664 if (lport == 0) { 665 /* Auto-select local port */ 666 return in_pcbsetlport(inp, wild, cred); 667 } 668 669 /* GROSS */ 670 if (ntohs(lport) < IPPORT_RESERVED && cred && 671 (error = 672 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 673 inp->inp_laddr.s_addr = INADDR_ANY; 674 return (error); 675 } 676 677 /* 678 * This has to be atomic. If the porthash is shared across 679 * multiple protocol threads, e.g. tcp and udp then the token 680 * must be held. 681 */ 682 porthash = OBTAIN_LPORTHASH_TOKEN(inp->inp_pcbinfo, lport); 683 684 /* 685 * Restore the sin_port whacked by in_pcbbind_ladddr(); 686 * sin->sin_port is checked by in_pcbbind_laddrport_check(). 687 */ 688 sin->sin_port = lport; 689 690 error = in_pcbbind_laddrport_check(so, sin, porthash, 691 wild, cred, td); 692 if (error) { 693 inp->inp_laddr.s_addr = INADDR_ANY; 694 goto done; 695 } 696 697 inp->inp_lport = lport; 698 in_pcbinsporthash(porthash, inp); 699 error = 0; 700 done: 701 REL_PORTHASH_TOKEN(porthash); 702 return (error); 703 } else { 704 struct sockaddr_in jsin; 705 706 jsin.sin_family = AF_INET; 707 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 708 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 709 inp->inp_laddr.s_addr = INADDR_ANY; 710 return (EINVAL); 711 } 712 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 713 714 return in_pcbsetlport(inp, wild, cred); 715 } 716 } 717 718 /* 719 * Lookup a PCB based on the local and remote address and port. 720 * 721 * This function is only used when scanning for a free port. 722 */ 723 static struct inpcb * 724 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr, 725 u_short lport, struct in_addr faddr, u_short fport, 726 struct ucred *cred) 727 { 728 struct inpcb *inp; 729 struct inpcbport *phd; 730 struct inpcb *match = NULL; 731 struct prison *pscan; 732 struct prison *pr; 733 734 /* 735 * If the porthashbase is shared across several cpus, it must 736 * have been locked. 737 */ 738 ASSERT_PORTHASH_TOKEN_HELD(porthash); 739 740 /* 741 * Best fit PCB lookup. 742 * 743 * First see if this local port is in use by looking on the 744 * port hash list. 745 */ 746 LIST_FOREACH(phd, porthash, phd_hash) { 747 if (phd->phd_port == lport) 748 break; 749 } 750 if (phd != NULL) { 751 pr = cred ? cred->cr_prison : NULL; 752 753 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 754 #ifdef INET6 755 if (!INP_ISIPV4(inp)) 756 continue; 757 #endif 758 if (inp->inp_laddr.s_addr == INADDR_ANY) { 759 if (inp->inp_socket && inp->inp_socket->so_cred) 760 pscan = inp->inp_socket->so_cred->cr_prison; 761 else 762 pscan = NULL; 763 if (pr != pscan) 764 continue; 765 } else { 766 if (inp->inp_laddr.s_addr != laddr.s_addr) 767 continue; 768 } 769 770 if (inp->inp_faddr.s_addr != INADDR_ANY && 771 inp->inp_faddr.s_addr != faddr.s_addr) 772 continue; 773 774 if (inp->inp_fport != 0 && inp->inp_fport != fport) 775 continue; 776 777 match = inp; 778 break; 779 } 780 } 781 return (match); 782 } 783 784 static boolean_t 785 in_pcbporthash_update4(struct inpcbportinfo *portinfo, struct inpcb *inp, 786 u_short lport, const struct sockaddr_in *sin, 787 struct ucred *cred) 788 { 789 struct inpcbporthead *porthash; 790 791 /* 792 * This has to be atomic. If the porthash is shared across multiple 793 * protocol threads, e.g. tcp and udp, then the token must be held. 794 */ 795 porthash = in_pcbporthash_head(portinfo, lport); 796 GET_PORTHASH_TOKEN(porthash); 797 798 if (in_pcblookup_localremote(porthash, inp->inp_laddr, lport, 799 sin->sin_addr, sin->sin_port, cred)) { 800 REL_PORTHASH_TOKEN(porthash); 801 return FALSE; 802 } 803 inp->inp_lport = lport; 804 in_pcbinsporthash(porthash, inp); 805 806 REL_PORTHASH_TOKEN(porthash); 807 return TRUE; 808 } 809 810 int 811 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 812 struct thread *td) 813 { 814 struct proc *p = td->td_proc; 815 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 816 struct sockaddr_in jsin; 817 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 818 struct ucred *cred = NULL; 819 u_short first, last, lport; 820 int count, hash_count; 821 int error, selfconn = 0; 822 int cpuid = mycpuid; 823 uint32_t hash_base = 0, hash; 824 825 ASSERT_NETISR_NCPUS(cpuid); 826 827 if (TAILQ_EMPTY(&in_ifaddrheads[cpuid])) /* XXX broken! */ 828 return (EADDRNOTAVAIL); 829 830 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 831 if (inp->inp_lport != 0) 832 return (EINVAL); /* already bound */ 833 834 KKASSERT(p); 835 cred = p->p_ucred; 836 837 jsin.sin_family = AF_INET; 838 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 839 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 840 inp->inp_laddr.s_addr = INADDR_ANY; 841 return (EINVAL); 842 } 843 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 844 845 hash_count = ip_porthash_trycount; 846 if (hash_count > 0) { 847 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^ 848 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^ 849 toeplitz_piecemeal_port(sin->sin_port); 850 } else { 851 hash_count = 0; 852 } 853 854 inp->inp_flags |= INP_ANONPORT; 855 856 if (inp->inp_flags & INP_HIGHPORT) { 857 first = ipport_hifirstauto; /* sysctl */ 858 last = ipport_hilastauto; 859 } else if (inp->inp_flags & INP_LOWPORT) { 860 if (cred && 861 (error = 862 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 863 inp->inp_laddr.s_addr = INADDR_ANY; 864 return (error); 865 } 866 first = ipport_lowfirstauto; /* 1023 */ 867 last = ipport_lowlastauto; /* 600 */ 868 } else { 869 first = ipport_firstauto; /* sysctl */ 870 last = ipport_lastauto; 871 } 872 if (first > last) { 873 lport = last; 874 last = first; 875 first = lport; 876 } 877 KKASSERT(last >= first); 878 879 count = last - first; 880 lport = (karc4random() % count) + first; 881 count += hash_count; 882 883 /* 884 * Simple check to ensure all ports are not used up causing 885 * a deadlock here. 886 */ 887 for (;;) { 888 u_short lport_no; 889 890 if (count-- < 0) { /* completely used? */ 891 error = EADDRNOTAVAIL; 892 break; 893 } 894 895 if (__predict_false(lport < first || lport > last)) 896 lport = first; 897 lport_no = htons(lport); 898 899 /* This could happen on loopback interface */ 900 if (__predict_false(sin->sin_port == lport_no && 901 sin->sin_addr.s_addr == inp->inp_laddr.s_addr)) { 902 if (!selfconn) { 903 ++count; /* don't count this try */ 904 selfconn = 1; 905 } 906 goto next; 907 } 908 909 if (hash_count) { 910 --hash_count; 911 hash = hash_base ^ 912 toeplitz_piecemeal_port(lport_no); 913 if (netisr_hashcpu(hash) != cpuid && hash_count) 914 goto next; 915 } 916 917 if (in_pcbporthash_update4( 918 &pcbinfo->portinfo[lport % pcbinfo->portinfo_cnt], 919 inp, lport_no, sin, cred)) { 920 error = 0; 921 break; 922 } 923 next: 924 ++lport; 925 } 926 927 if (error) 928 inp->inp_laddr.s_addr = INADDR_ANY; 929 return (error); 930 } 931 932 /* 933 * Figure out the local interface address to pair against the requested 934 * target address, as well as validate the target address. 935 */ 936 int 937 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 938 struct sockaddr_in **plocal_sin, struct thread *td, int find) 939 { 940 struct in_ifaddr_container *iac; 941 struct in_ifaddr *ia; 942 struct ucred *cred = NULL; 943 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 944 struct sockaddr *jsin; 945 struct prison *pr; 946 struct route *ro; 947 int alloc_route = 0; 948 949 if (nam->sa_len != sizeof *sin) 950 return (EINVAL); 951 if (sin->sin_family != AF_INET) 952 return (EAFNOSUPPORT); 953 if (sin->sin_port == 0) 954 return (EADDRNOTAVAIL); 955 956 /* 957 * Are we in a jail? 958 */ 959 pr = NULL; 960 if (td && td->td_proc && td->td_proc->p_ucred) 961 cred = td->td_proc->p_ucred; 962 if (cred) 963 pr = cred->cr_prison; 964 965 /* 966 * If the destination address is INADDR_ANY then use the primary 967 * local address. 968 * 969 * If the supplied address is INADDR_BROADCAST, and the primary 970 * interface supports broadcast, choose the broadcast address for 971 * that interface. 972 * 973 * If jailed, locate an interface address acceptable to the jail. 974 */ 975 if (sin->sin_addr.s_addr == INADDR_ANY) { 976 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) { 977 ia = iac->ia; 978 if (pr == NULL || 979 jailed_ip(pr, sintosa(&ia->ia_addr))) { 980 sin->sin_addr = IA_SIN(ia)->sin_addr; 981 break; 982 } 983 } 984 } else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST) { 985 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) { 986 ia = iac->ia; 987 if ((pr == NULL || 988 jailed_ip(pr, sintosa(&ia->ia_addr))) && 989 (iac->ia->ia_ifp->if_flags & IFF_BROADCAST)) { 990 sin->sin_addr = 991 satosin(&ia->ia_broadaddr)->sin_addr; 992 break; 993 } 994 } 995 } 996 997 /* 998 * If asked to do a search, use the cached route or do a route table 999 * lookup to try to find an acceptable local interface IP. 1000 */ 1001 if (find == 0) 1002 return 0; 1003 1004 ia = NULL; 1005 1006 /* 1007 * If we have a cached route, check to see if it is acceptable. 1008 * If not, free it. 1009 */ 1010 ro = &inp->inp_route; 1011 if (ro->ro_rt && 1012 (!(ro->ro_rt->rt_flags & RTF_UP) || 1013 ro->ro_dst.sa_family != AF_INET || 1014 satosin(&ro->ro_dst)->sin_addr.s_addr != 1015 sin->sin_addr.s_addr || 1016 inp->inp_socket->so_options & SO_DONTROUTE)) { 1017 RTFREE(ro->ro_rt); 1018 ro->ro_rt = NULL; 1019 } 1020 1021 /* 1022 * If we do not have a route, construct one and do a lookup, 1023 * unless we are forbidden to do so. 1024 * 1025 * Note that we should check the address family of the cached 1026 * destination, in case of sharing the cache with IPv6. 1027 */ 1028 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 1029 (ro->ro_rt == NULL || ro->ro_rt->rt_ifp == NULL)) { 1030 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 1031 ro->ro_dst.sa_family = AF_INET; 1032 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 1033 ((struct sockaddr_in *)&ro->ro_dst)->sin_addr = sin->sin_addr; 1034 rtalloc(ro); 1035 alloc_route = 1; 1036 } 1037 1038 /* 1039 * If we found a route, use the address corresponding to the 1040 * outgoing interface. 1041 * 1042 * If jailed, try to find a compatible address on the outgoing 1043 * interface. 1044 */ 1045 if (ro->ro_rt) { 1046 ia = ifatoia(ro->ro_rt->rt_ifa); 1047 if (pr == NULL) 1048 goto skip; 1049 if (jailed_ip(pr, sintosa(&ia->ia_addr))) 1050 goto skip; 1051 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) { 1052 if (iac->ia->ia_ifp != ia->ia_ifp) 1053 continue; 1054 ia = iac->ia; 1055 if (jailed_ip(pr, sintosa(&ia->ia_addr))) 1056 goto skip; 1057 } 1058 ia = NULL; 1059 } 1060 skip: 1061 1062 /* 1063 * If the route didn't work or there was no route, 1064 * fall-back to the first address in in_ifaddrheads[]. 1065 * 1066 * If jailed and this address is not available for 1067 * the jail, leave ia set to NULL. 1068 */ 1069 if (ia == NULL) { 1070 u_short fport = sin->sin_port; 1071 1072 sin->sin_port = 0; 1073 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 1074 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr))) 1075 ia = NULL; 1076 1077 if (ia == NULL) 1078 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 1079 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr))) 1080 ia = NULL; 1081 1082 sin->sin_port = fport; 1083 if (ia == NULL && !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 1084 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 1085 1086 if (ia && pr && !jailed_ip(pr, sintosa(&ia->ia_addr))) 1087 ia = NULL; 1088 1089 if (pr == NULL && ia == NULL) 1090 goto fail; 1091 } 1092 1093 /* 1094 * If the destination address is multicast and an outgoing 1095 * interface has been set as a multicast option, use the 1096 * address of that interface as our source address. 1097 */ 1098 if (pr == NULL && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 1099 inp->inp_moptions != NULL) { 1100 struct ip_moptions *imo; 1101 struct ifnet *ifp; 1102 1103 imo = inp->inp_moptions; 1104 if ((ifp = imo->imo_multicast_ifp) != NULL) { 1105 struct in_ifaddr_container *iac; 1106 1107 ia = NULL; 1108 TAILQ_FOREACH(iac, &in_ifaddrheads[mycpuid], ia_link) { 1109 if (iac->ia->ia_ifp == ifp) { 1110 ia = iac->ia; 1111 break; 1112 } 1113 } 1114 if (ia == NULL) 1115 goto fail; 1116 } 1117 } 1118 1119 /* 1120 * If we still don't have a local address, and are jailed, 1121 * use the jail's first non-localhost IP. If there isn't 1122 * one, use the jail's first localhost IP. 1123 * 1124 * Don't do pcblookup call here; return interface in plocal_sin 1125 * and exit to caller, that will do the lookup. 1126 */ 1127 if (ia == NULL && pr) { 1128 jsin = prison_get_nonlocal(cred->cr_prison, AF_INET, NULL); 1129 if (jsin == NULL) 1130 jsin = prison_get_local(cred->cr_prison, AF_INET, NULL); 1131 if (jsin) 1132 *plocal_sin = satosin(jsin); 1133 else 1134 goto fail; 1135 } else if (ia) { 1136 *plocal_sin = &ia->ia_addr; 1137 } else { 1138 goto fail; 1139 } 1140 return (0); 1141 fail: 1142 if (alloc_route) 1143 in_pcbresetroute(inp); 1144 return (EADDRNOTAVAIL); 1145 } 1146 1147 int 1148 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 1149 struct sockaddr_in **plocal_sin, struct thread *td) 1150 { 1151 return in_pcbladdr_find(inp, nam, plocal_sin, td, 1152 (inp->inp_laddr.s_addr == INADDR_ANY)); 1153 } 1154 1155 /* 1156 * Outer subroutine: 1157 * Connect from a socket to a specified address. 1158 * Both address and port must be specified in argument sin. 1159 * If don't have a local address for this socket yet, 1160 * then pick one. 1161 */ 1162 int 1163 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 1164 { 1165 struct sockaddr_in *if_sin; 1166 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1167 int error; 1168 1169 if_sin = NULL; /* avoid gcc warnings */ 1170 1171 /* Call inner routine to assign local interface address. */ 1172 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1173 return (error); 1174 1175 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1176 inp->inp_laddr.s_addr ? 1177 inp->inp_laddr : if_sin->sin_addr, 1178 inp->inp_lport, FALSE, NULL) != NULL) { 1179 return (EADDRINUSE); 1180 } 1181 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1182 if (inp->inp_lport == 0) { 1183 error = in_pcbbind(inp, NULL, td); 1184 if (error) 1185 return (error); 1186 } 1187 inp->inp_laddr = if_sin->sin_addr; 1188 } 1189 inp->inp_faddr = sin->sin_addr; 1190 inp->inp_fport = sin->sin_port; 1191 in_pcbinsconnhash(inp); 1192 return (0); 1193 } 1194 1195 void 1196 in_pcbdisconnect(struct inpcb *inp) 1197 { 1198 1199 in_pcbremconnhash(inp); 1200 inp->inp_faddr.s_addr = INADDR_ANY; 1201 inp->inp_fport = 0; 1202 } 1203 1204 void 1205 in_pcbdetach(struct inpcb *inp) 1206 { 1207 struct socket *so = inp->inp_socket; 1208 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1209 1210 inp->inp_gencnt = ++ipi->ipi_gencnt; 1211 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1212 in_pcbremlists(inp); 1213 so->so_pcb = NULL; 1214 sofree(so); /* remove pcb ref */ 1215 if (inp->inp_options) 1216 m_free(inp->inp_options); 1217 if (inp->inp_route.ro_rt) 1218 rtfree(inp->inp_route.ro_rt); 1219 ip_freemoptions(inp->inp_moptions); 1220 kfree(inp, M_PCB); 1221 } 1222 1223 /* 1224 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1225 * socket received RST. 1226 */ 1227 static int 1228 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1229 { 1230 struct inpcb *inp; 1231 struct sockaddr_in *sin; 1232 1233 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1234 inp = so->so_pcb; 1235 if (!inp) 1236 return (ECONNRESET); 1237 1238 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1239 sin->sin_family = AF_INET; 1240 sin->sin_len = sizeof *sin; 1241 sin->sin_port = inp->inp_lport; 1242 sin->sin_addr = inp->inp_laddr; 1243 1244 *nam = (struct sockaddr *)sin; 1245 return (0); 1246 } 1247 1248 void 1249 in_setsockaddr_dispatch(netmsg_t msg) 1250 { 1251 int error; 1252 1253 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1254 lwkt_replymsg(&msg->lmsg, error); 1255 } 1256 1257 /* 1258 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1259 * socket received RST. 1260 */ 1261 int 1262 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1263 { 1264 struct inpcb *inp; 1265 struct sockaddr_in *sin; 1266 1267 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1268 inp = so->so_pcb; 1269 if (!inp) 1270 return (ECONNRESET); 1271 1272 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1273 sin->sin_family = AF_INET; 1274 sin->sin_len = sizeof *sin; 1275 sin->sin_port = inp->inp_fport; 1276 sin->sin_addr = inp->inp_faddr; 1277 1278 *nam = (struct sockaddr *)sin; 1279 return (0); 1280 } 1281 1282 void 1283 in_setpeeraddr_dispatch(netmsg_t msg) 1284 { 1285 int error; 1286 1287 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1288 lwkt_replymsg(&msg->lmsg, error); 1289 } 1290 1291 void 1292 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1293 inp_notify_t notify) 1294 { 1295 struct inpcb *inp, *marker; 1296 1297 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1298 ("not in the correct netisr")); 1299 marker = in_pcbmarker(); 1300 1301 /* 1302 * NOTE: 1303 * - If INP_PLACEMARKER is set we must ignore the rest of the 1304 * structure and skip it. 1305 * - It is safe to nuke inpcbs here, since we are in their own 1306 * netisr. 1307 */ 1308 GET_PCBINFO_TOKEN(pcbinfo); 1309 1310 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1311 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1312 LIST_REMOVE(marker, inp_list); 1313 LIST_INSERT_AFTER(inp, marker, inp_list); 1314 1315 if (inp->inp_flags & INP_PLACEMARKER) 1316 continue; 1317 #ifdef INET6 1318 if (!INP_ISIPV4(inp)) 1319 continue; 1320 #endif 1321 if (inp->inp_faddr.s_addr != faddr.s_addr || 1322 inp->inp_socket == NULL) 1323 continue; 1324 (*notify)(inp, err); /* can remove inp from list! */ 1325 } 1326 LIST_REMOVE(marker, inp_list); 1327 1328 REL_PCBINFO_TOKEN(pcbinfo); 1329 } 1330 1331 void 1332 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1333 { 1334 struct inpcb *inp, *marker; 1335 1336 /* 1337 * We only need to make sure that we are in netisr0, where all 1338 * multicast operation happen. We could check inpcbinfo which 1339 * does not belong to netisr0 by holding the inpcbinfo's token. 1340 * In this case, the pcbinfo must be able to be shared, i.e. 1341 * pcbinfo->infotoken is not NULL. 1342 */ 1343 ASSERT_NETISR0; 1344 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1345 ("pcbinfo could not be shared")); 1346 1347 /* 1348 * Get a marker for the current netisr (netisr0). 1349 * 1350 * It is possible that the multicast address deletion blocks, 1351 * which could cause temporary token releasing. So we use 1352 * inpcb marker here to get a coherent view of the inpcb list. 1353 * 1354 * While, on the other hand, moptions are only added and deleted 1355 * in netisr0, so we would not see staled moption or miss moption 1356 * even if the token was released due to the blocking multicast 1357 * address deletion. 1358 */ 1359 marker = in_pcbmarker(); 1360 1361 GET_PCBINFO_TOKEN(pcbinfo); 1362 1363 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1364 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1365 struct ip_moptions *imo; 1366 1367 LIST_REMOVE(marker, inp_list); 1368 LIST_INSERT_AFTER(inp, marker, inp_list); 1369 1370 if (inp->inp_flags & INP_PLACEMARKER) 1371 continue; 1372 imo = inp->inp_moptions; 1373 if (INP_ISIPV4(inp) && imo != NULL) { 1374 int i, gap; 1375 1376 /* 1377 * Unselect the outgoing interface if it is being 1378 * detached. 1379 */ 1380 if (imo->imo_multicast_ifp == ifp) 1381 imo->imo_multicast_ifp = NULL; 1382 1383 /* 1384 * Drop multicast group membership if we joined 1385 * through the interface being detached. 1386 */ 1387 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1388 i++) { 1389 if (imo->imo_membership[i]->inm_ifp == ifp) { 1390 /* 1391 * NOTE: 1392 * This could block and the pcbinfo 1393 * token could be passively released. 1394 */ 1395 in_delmulti(imo->imo_membership[i]); 1396 gap++; 1397 } else if (gap != 0) 1398 imo->imo_membership[i - gap] = 1399 imo->imo_membership[i]; 1400 } 1401 imo->imo_num_memberships -= gap; 1402 } 1403 } 1404 LIST_REMOVE(marker, inp_list); 1405 1406 REL_PCBINFO_TOKEN(pcbinfo); 1407 } 1408 1409 /* 1410 * Check for alternatives when higher level complains 1411 * about service problems. For now, invalidate cached 1412 * routing information. If the route was created dynamically 1413 * (by a redirect), time to try a default gateway again. 1414 */ 1415 void 1416 in_losing(struct inpcb *inp) 1417 { 1418 struct rtentry *rt; 1419 struct rt_addrinfo rtinfo; 1420 1421 if ((rt = inp->inp_route.ro_rt)) { 1422 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1423 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1424 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1425 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1426 rtinfo.rti_flags = rt->rt_flags; 1427 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1428 if (rt->rt_flags & RTF_DYNAMIC) { 1429 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1430 rt_mask(rt), rt->rt_flags, NULL); 1431 } 1432 inp->inp_route.ro_rt = NULL; 1433 rtfree(rt); 1434 /* 1435 * A new route can be allocated 1436 * the next time output is attempted. 1437 */ 1438 } 1439 } 1440 1441 /* 1442 * After a routing change, flush old routing 1443 * and allocate a (hopefully) better one. 1444 */ 1445 void 1446 in_rtchange(struct inpcb *inp, int err) 1447 { 1448 if (inp->inp_route.ro_rt) { 1449 rtfree(inp->inp_route.ro_rt); 1450 inp->inp_route.ro_rt = NULL; 1451 /* 1452 * A new route can be allocated the next time 1453 * output is attempted. 1454 */ 1455 } 1456 } 1457 1458 /* 1459 * Lookup a PCB based on the local address and port. 1460 * 1461 * This function is only used when scanning for a free port. 1462 */ 1463 static struct inpcb * 1464 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr, 1465 u_int lport_arg, int wild_okay, struct ucred *cred) 1466 { 1467 struct prison *pscan; 1468 struct prison *pr; 1469 struct inpcb *inp; 1470 int matchwild = 3, wildcard; 1471 u_short lport = lport_arg; 1472 struct inpcbport *phd; 1473 struct inpcb *match = NULL; 1474 1475 /* 1476 * If the porthashbase is shared across several cpus, it must 1477 * have been locked. 1478 */ 1479 ASSERT_PORTHASH_TOKEN_HELD(porthash); 1480 1481 /* 1482 * Best fit PCB lookup. 1483 * 1484 * First see if this local port is in use by looking on the 1485 * port hash list. 1486 */ 1487 LIST_FOREACH(phd, porthash, phd_hash) { 1488 if (phd->phd_port == lport) 1489 break; 1490 } 1491 if (phd != NULL) { 1492 pr = cred ? cred->cr_prison : NULL; 1493 1494 /* 1495 * Port is in use by one or more PCBs. Look for best 1496 * fit. 1497 * 1498 * If in a prison we may wish to allow the jail to override 1499 * a wildcard listen on the host. Since the jail forces its 1500 * own wildcard listens to a specific set of jail IPs, this 1501 * override allows most services on the host to remain as 1502 * they were and still be 'jail friendly'. 1503 */ 1504 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1505 wildcard = 0; 1506 #ifdef INET6 1507 if (!INP_ISIPV4(inp)) 1508 continue; 1509 #endif 1510 if (inp->inp_faddr.s_addr != INADDR_ANY) 1511 wildcard++; 1512 1513 /* 1514 * Prison are independent of each other in terms 1515 * of allowing bindings. This can result in multiple 1516 * overloaded bindings which in_pcblookup_pkthash() 1517 * will have to sort out. 1518 * 1519 * Allow wildcarded entries to co-exist with specific 1520 * entries. Specific entries override wildcarded 1521 * entries. 1522 */ 1523 if (inp->inp_socket && inp->inp_socket->so_cred) 1524 pscan = inp->inp_socket->so_cred->cr_prison; 1525 else 1526 pscan = NULL; 1527 if (pr != pscan) 1528 continue; 1529 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1530 if (laddr.s_addr != INADDR_ANY) 1531 wildcard++; 1532 } else { 1533 if (laddr.s_addr == INADDR_ANY) 1534 wildcard++; 1535 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1536 continue; 1537 } 1538 if (wildcard && !wild_okay) 1539 continue; 1540 if (wildcard < matchwild) { 1541 match = inp; 1542 matchwild = wildcard; 1543 if (matchwild == 0) 1544 break; 1545 } 1546 } 1547 } 1548 return (match); 1549 } 1550 1551 struct inpcb * 1552 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1553 const struct inpcb *inp) 1554 { 1555 const struct inp_localgrphead *hdr; 1556 const struct inp_localgroup *grp; 1557 int i; 1558 1559 if (pcbinfo->localgrphashbase == NULL) 1560 return NULL; 1561 1562 GET_PCBINFO_TOKEN(pcbinfo); 1563 1564 hdr = &pcbinfo->localgrphashbase[ 1565 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1566 1567 LIST_FOREACH(grp, hdr, il_list) { 1568 if (grp->il_af == inp->inp_af && 1569 grp->il_lport == inp->inp_lport && 1570 memcmp(&grp->il_dependladdr, 1571 &inp->inp_inc.inc_ie.ie_dependladdr, 1572 sizeof(grp->il_dependladdr)) == 0) { 1573 break; 1574 } 1575 } 1576 if (grp == NULL || grp->il_inpcnt == 1) { 1577 REL_PCBINFO_TOKEN(pcbinfo); 1578 return NULL; 1579 } 1580 1581 KASSERT(grp->il_inpcnt >= 2, 1582 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1583 for (i = 0; i < grp->il_inpcnt; ++i) { 1584 if (grp->il_inp[i] == inp) { 1585 int last = grp->il_inpcnt - 1; 1586 1587 if (i == last) 1588 last = grp->il_inpcnt - 2; 1589 REL_PCBINFO_TOKEN(pcbinfo); 1590 return grp->il_inp[last]; 1591 } 1592 } 1593 REL_PCBINFO_TOKEN(pcbinfo); 1594 return NULL; 1595 } 1596 1597 static struct inpcb * 1598 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1599 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1600 { 1601 struct inpcb *local_wild; 1602 struct inpcb *jinp; 1603 struct inpcb *jinp_wild; 1604 struct inpcb *inp; 1605 const struct inp_localgrphead *hdr; 1606 const struct inp_localgroup *grp; 1607 struct sockaddr_in jsin; 1608 struct prison *pr; 1609 struct ucred *cred; 1610 int idx; 1611 int net_listen_ov_local; 1612 int net_listen_ov_wild; 1613 1614 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1615 1616 hdr = &pcbinfo->localgrphashbase[ 1617 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1618 1619 /* 1620 * Order of socket selection: 1621 * 1. non-wild. 1622 * 2. wild. 1623 * 1624 * NOTE: Local group does not contain jailed sockets 1625 */ 1626 jsin.sin_family = AF_INET; 1627 jsin.sin_addr.s_addr = laddr.s_addr; 1628 1629 jinp = NULL; 1630 jinp_wild = NULL; 1631 local_wild = NULL; 1632 net_listen_ov_local = 0; 1633 net_listen_ov_wild = 0; 1634 1635 LIST_FOREACH(grp, hdr, il_list) { 1636 #ifdef INET6 1637 if (grp->il_af != AF_INET) 1638 continue; 1639 #endif 1640 if (grp->il_lport != lport) 1641 continue; 1642 1643 /* 1644 * look for a match 1645 */ 1646 idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt; 1647 inp = grp->il_inp[idx]; 1648 1649 /* 1650 * Modulo-N is used here, which greatly reduces 1651 * completion queue token contention, thus more 1652 * cpu time is saved. 1653 */ 1654 if (grp->il_jailed) { 1655 if (inp->inp_socket == NULL) 1656 continue; 1657 cred = inp->inp_socket->so_cred; 1658 if (cred == NULL) 1659 continue; 1660 pr = cred->cr_prison; 1661 if (pr == NULL) 1662 continue; 1663 if (!jailed_ip(pr, (struct sockaddr *)&jsin)) 1664 continue; 1665 if (grp->il_laddr.s_addr == laddr.s_addr) { 1666 jinp = inp; 1667 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE)) 1668 net_listen_ov_local = 1; 1669 1670 } else if (grp->il_laddr.s_addr == INADDR_ANY && 1671 jinp_wild == NULL) { 1672 jinp_wild = inp; 1673 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE)) 1674 net_listen_ov_wild = 1; 1675 } 1676 } else { 1677 if (grp->il_laddr.s_addr == laddr.s_addr) { 1678 return inp; 1679 } else if (grp->il_laddr.s_addr == INADDR_ANY) { 1680 local_wild = inp; 1681 } 1682 } 1683 } 1684 1685 if (net_listen_ov_local) 1686 return jinp; 1687 if (net_listen_ov_wild) 1688 return jinp_wild; 1689 if (local_wild) 1690 return (local_wild); 1691 if (jinp) 1692 return (jinp); 1693 return (jinp_wild); 1694 } 1695 1696 /* 1697 * Lookup PCB in hash list. 1698 * 1699 * This is used to match incoming packets to a pcb 1700 */ 1701 struct inpcb * 1702 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1703 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1704 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1705 { 1706 struct inpcbhead *head; 1707 struct inpcb *inp, *jinp=NULL; 1708 u_short fport = fport_arg, lport = lport_arg; 1709 1710 /* 1711 * First look for an exact match. 1712 */ 1713 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1714 laddr.s_addr, lport, 1715 pcbinfo->hashmask)]; 1716 LIST_FOREACH(inp, head, inp_hash) { 1717 #ifdef INET6 1718 if (!INP_ISIPV4(inp)) 1719 continue; 1720 #endif 1721 if (in_hosteq(inp->inp_faddr, faddr) && 1722 in_hosteq(inp->inp_laddr, laddr) && 1723 inp->inp_fport == fport && inp->inp_lport == lport) { 1724 /* 1725 * Found specific address, host overrides jailed 1726 * inpcb. 1727 */ 1728 if (inp->inp_socket == NULL || 1729 inp->inp_socket->so_cred->cr_prison == NULL) { 1730 return (inp); 1731 } 1732 if (jinp == NULL) 1733 jinp = inp; 1734 } 1735 } 1736 if (jinp != NULL) 1737 return (jinp); 1738 1739 /* 1740 * We generally get here for connections to wildcarded listeners. 1741 * Any wildcarded listeners in jails must be restricted to the 1742 * jailed IPs only. 1743 */ 1744 if (wildcard) { 1745 struct inpcb *local_wild = NULL; 1746 struct inpcb *jinp_wild = NULL; 1747 struct inpcontainer *ic; 1748 struct inpcontainerhead *chead; 1749 struct sockaddr_in jsin; 1750 struct ucred *cred; 1751 struct prison *pr; 1752 int net_listen_ov_local = 0; 1753 int net_listen_ov_wild = 0; 1754 1755 GET_PCBINFO_TOKEN(pcbinfo); 1756 1757 /* 1758 * Check local group first. When present, the localgroup 1759 * hash utilizes the same non-jailed-vs/jailed priortization 1760 * that the normal wildcardhash does. 1761 */ 1762 if (pcbinfo->localgrphashbase != NULL && 1763 m != NULL && (m->m_flags & M_HASH)) { 1764 inp = inp_localgroup_lookup(pcbinfo, laddr, lport, 1765 m->m_pkthdr.hash); 1766 if (inp != NULL) { 1767 REL_PCBINFO_TOKEN(pcbinfo); 1768 return inp; 1769 } 1770 } 1771 1772 /* 1773 * Order of socket selection: 1774 * 1775 * 1. non-jailed, non-wild. 1776 * 2. non-jailed, wild. (allow_listen_override on) 1777 * 3. jailed, non-wild. 1778 * 4. jailed, wild. 1779 * 5. non-jailed, wild. (allow_listen_override off) 1780 * 1781 * NOTE: jailed wildcards are still restricted to the jail 1782 * IPs. 1783 * 1784 * NOTE: (1) and (3) already handled above. 1785 */ 1786 jsin.sin_family = AF_INET; 1787 chead = &pcbinfo->wildcardhashbase[ 1788 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1789 1790 LIST_FOREACH(ic, chead, ic_list) { 1791 inp = ic->ic_inp; 1792 if (inp->inp_flags & INP_PLACEMARKER) 1793 continue; 1794 1795 /* 1796 * Basic validation 1797 */ 1798 #ifdef INET6 1799 if (!INP_ISIPV4(inp)) 1800 continue; 1801 #endif 1802 if (inp->inp_lport != lport) 1803 continue; 1804 1805 /* 1806 * Calculate prison, setup jsin for jailed_ip() 1807 * check. 1808 */ 1809 jsin.sin_addr.s_addr = laddr.s_addr; 1810 pr = NULL; 1811 cred = NULL; 1812 if (inp->inp_socket) { 1813 cred = inp->inp_socket->so_cred; 1814 if (cred) 1815 pr = cred->cr_prison; 1816 } 1817 1818 /* 1819 * Assign jinp, jinp_wild, and local_wild as 1820 * appropriate, track whether the jail supports 1821 * listen overrides. 1822 */ 1823 if (pr) { 1824 if (!jailed_ip(pr, (struct sockaddr *)&jsin)) 1825 continue; 1826 if (inp->inp_laddr.s_addr == laddr.s_addr && 1827 jinp == NULL) { 1828 jinp = inp; 1829 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE)) 1830 net_listen_ov_local = 1; 1831 } 1832 if (inp->inp_laddr.s_addr == INADDR_ANY && 1833 jinp_wild == NULL) { 1834 jinp_wild = inp; 1835 if (PRISON_CAP_ISSET(pr->pr_caps, PRISON_CAP_NET_LISTEN_OVERRIDE)) 1836 net_listen_ov_wild = 1; 1837 } 1838 } else { 1839 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1840 REL_PCBINFO_TOKEN(pcbinfo); 1841 return (inp); 1842 } 1843 if (inp->inp_laddr.s_addr == INADDR_ANY) 1844 local_wild = inp; 1845 } 1846 } 1847 1848 REL_PCBINFO_TOKEN(pcbinfo); 1849 1850 if (net_listen_ov_local) 1851 return jinp; 1852 if (net_listen_ov_wild) 1853 return jinp_wild; 1854 if (local_wild) 1855 return (local_wild); 1856 if (jinp) 1857 return (jinp); 1858 return (jinp_wild); 1859 } 1860 1861 /* 1862 * Not found. 1863 */ 1864 return (NULL); 1865 } 1866 1867 struct inpcb * 1868 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1869 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1870 boolean_t wildcard, struct ifnet *ifp) 1871 { 1872 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1873 laddr, lport_arg, wildcard, ifp, NULL); 1874 } 1875 1876 /* 1877 * Insert PCB into connection hash table. 1878 */ 1879 void 1880 in_pcbinsconnhash(struct inpcb *inp) 1881 { 1882 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1883 struct inpcbhead *bucket; 1884 u_int32_t hashkey_faddr, hashkey_laddr; 1885 1886 #ifdef INET6 1887 if (INP_ISIPV6(inp)) { 1888 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1889 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1890 } else { 1891 #endif 1892 hashkey_faddr = inp->inp_faddr.s_addr; 1893 hashkey_laddr = inp->inp_laddr.s_addr; 1894 #ifdef INET6 1895 } 1896 #endif 1897 1898 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1899 ("not in the correct netisr")); 1900 ASSERT_INP_NOTINHASH(inp); 1901 inp->inp_flags |= INP_CONNECTED; 1902 1903 /* 1904 * Insert into the connection hash table. 1905 */ 1906 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1907 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1908 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1909 } 1910 1911 /* 1912 * Remove PCB from connection hash table. 1913 */ 1914 void 1915 in_pcbremconnhash(struct inpcb *inp) 1916 { 1917 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1918 1919 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1920 ("not in the correct netisr")); 1921 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1922 1923 LIST_REMOVE(inp, inp_hash); 1924 inp->inp_flags &= ~INP_CONNECTED; 1925 } 1926 1927 /* 1928 * Insert PCB into port hash table. 1929 */ 1930 void 1931 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp) 1932 { 1933 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1934 struct inpcbport *phd; 1935 1936 /* 1937 * If the porthashbase is shared across several cpus, it must 1938 * have been locked. 1939 */ 1940 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash); 1941 1942 /* 1943 * Insert into the port hash table. 1944 */ 1945 1946 /* Go through port list and look for a head for this lport. */ 1947 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1948 if (phd->phd_port == inp->inp_lport) 1949 break; 1950 } 1951 1952 /* If none exists, use saved one and tack it on. */ 1953 if (phd == NULL) { 1954 KKASSERT(pcbinfo->portsave != NULL); 1955 phd = pcbinfo->portsave; 1956 pcbinfo->portsave = NULL; 1957 phd->phd_port = inp->inp_lport; 1958 LIST_INIT(&phd->phd_pcblist); 1959 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1960 } 1961 1962 inp->inp_porthash = pcbporthash; 1963 inp->inp_phd = phd; 1964 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1965 1966 /* 1967 * Malloc one inpcbport for later use. It is safe to use 1968 * "wait" malloc here (port token would be released, if 1969 * malloc ever blocked), since all changes to the porthash 1970 * are done. 1971 */ 1972 if (pcbinfo->portsave == NULL) { 1973 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1974 M_PCB, M_INTWAIT | M_ZERO); 1975 } 1976 } 1977 1978 void 1979 in_pcbinsporthash_lport(struct inpcb *inp) 1980 { 1981 struct inpcbporthead *porthash; 1982 1983 porthash = OBTAIN_LPORTHASH_TOKEN(inp->inp_pcbinfo, inp->inp_lport); 1984 in_pcbinsporthash(porthash, inp); 1985 REL_PORTHASH_TOKEN(porthash); 1986 } 1987 1988 void 1989 in_pcbremporthash(struct inpcb *inp) 1990 { 1991 struct inpcbporthead *porthash; 1992 struct inpcbport *phd; 1993 1994 if (inp->inp_phd == NULL) 1995 return; 1996 KASSERT(inp->inp_lport != 0, ("inpcb has no lport")); 1997 1998 porthash = inp->inp_porthash; 1999 KASSERT(porthash != NULL, ("no porthash")); 2000 2001 GET_PORTHASH_TOKEN(porthash); 2002 2003 phd = inp->inp_phd; 2004 LIST_REMOVE(inp, inp_portlist); 2005 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 2006 LIST_REMOVE(phd, phd_hash); 2007 kfree(phd, M_PCB); 2008 } 2009 2010 REL_PORTHASH_TOKEN(porthash); 2011 2012 inp->inp_phd = NULL; 2013 /* NOTE: Don't whack inp_lport, which may be used later */ 2014 } 2015 2016 static struct inp_localgroup * 2017 inp_localgroup_alloc(u_char af, uint16_t port, 2018 const union in_dependaddr *addr, int size) 2019 { 2020 struct inp_localgroup *grp; 2021 2022 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 2023 M_TEMP, M_INTWAIT | M_ZERO); 2024 grp->il_af = af; 2025 grp->il_lport = port; 2026 grp->il_dependladdr = *addr; 2027 grp->il_inpsiz = size; 2028 2029 return grp; 2030 } 2031 2032 static void 2033 inp_localgroup_free(struct inp_localgroup *grp) 2034 { 2035 kfree(grp, M_TEMP); 2036 } 2037 2038 static void 2039 inp_localgroup_destroy(struct inp_localgroup *grp) 2040 { 2041 LIST_REMOVE(grp, il_list); 2042 inp_localgroup_free(grp); 2043 } 2044 2045 static void 2046 inp_localgroup_copy(struct inp_localgroup *grp, 2047 const struct inp_localgroup *old_grp) 2048 { 2049 int i; 2050 2051 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 2052 ("invalid new local group size %d and old local group count %d", 2053 grp->il_inpsiz, old_grp->il_inpcnt)); 2054 for (i = 0; i < old_grp->il_inpcnt; ++i) 2055 grp->il_inp[i] = old_grp->il_inp[i]; 2056 grp->il_inpcnt = old_grp->il_inpcnt; 2057 } 2058 2059 static void 2060 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2061 { 2062 struct inp_localgrphead *hdr; 2063 struct inp_localgroup *grp, *grp_alloc = NULL; 2064 u_char isjailed; 2065 int i, idx; 2066 2067 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2068 2069 if (pcbinfo->localgrphashbase == NULL) 2070 return; 2071 2072 /* 2073 * Further separate groups by whether the inp is jailed or not. 2074 * This allows the inp_localgroup_lookup() code to manage port 2075 * overloading between jails and non-jails. 2076 * 2077 * XXX all jails are collected into one group, which works fine 2078 * as we expect the jails to be listening on different addresses. 2079 * If this changes in the future we may have to break the groups 2080 * up by prison pointer as well. 2081 */ 2082 if (inp->inp_socket && inp->inp_socket->so_cred) 2083 isjailed = jailed(inp->inp_socket->so_cred); 2084 else 2085 isjailed = 0; 2086 2087 hdr = &pcbinfo->localgrphashbase[ 2088 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2089 2090 again: 2091 LIST_FOREACH(grp, hdr, il_list) { 2092 if (grp->il_af == inp->inp_af && 2093 grp->il_lport == inp->inp_lport && 2094 grp->il_jailed == isjailed && 2095 memcmp(&grp->il_dependladdr, 2096 &inp->inp_inc.inc_ie.ie_dependladdr, 2097 sizeof(grp->il_dependladdr)) == 0) { 2098 break; 2099 } 2100 } 2101 if (grp == NULL) { 2102 /* 2103 * Create a new local group 2104 */ 2105 if (grp_alloc == NULL) { 2106 grp_alloc = inp_localgroup_alloc(inp->inp_af, 2107 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 2108 INP_LOCALGROUP_SIZMIN); 2109 /* 2110 * Local group allocation could block and the 2111 * local group w/ the same property might have 2112 * been added by others when we were blocked; 2113 * check again. 2114 */ 2115 goto again; 2116 } else { 2117 /* Local group has been allocated; link it */ 2118 grp = grp_alloc; 2119 grp->il_jailed = isjailed; 2120 grp_alloc = NULL; 2121 LIST_INSERT_HEAD(hdr, grp, il_list); 2122 } 2123 } else if (grp->il_inpcnt == grp->il_inpsiz) { 2124 #if 0 2125 /* 2126 * REMOVED - Ensure that all entries are placed in the 2127 * localgroup so jail operations can be 2128 * deterministic on a il_lport basis. 2129 */ 2130 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 2131 static int limit_logged = 0; 2132 2133 if (!limit_logged) { 2134 limit_logged = 1; 2135 kprintf("local group port %d, " 2136 "limit reached\n", ntohs(grp->il_lport)); 2137 } 2138 if (grp_alloc != NULL) { 2139 /* 2140 * This would happen if the local group 2141 * w/ the same property was expanded when 2142 * our local group allocation blocked. 2143 */ 2144 inp_localgroup_free(grp_alloc); 2145 } 2146 return; 2147 } 2148 #endif 2149 2150 /* 2151 * Expand this local group 2152 */ 2153 if (grp_alloc == NULL || 2154 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 2155 if (grp_alloc != NULL) 2156 inp_localgroup_free(grp_alloc); 2157 grp_alloc = inp_localgroup_alloc(grp->il_af, 2158 grp->il_lport, &grp->il_dependladdr, 2159 grp->il_inpsiz * 2); 2160 /* 2161 * Local group allocation could block and the 2162 * local group w/ the same property might have 2163 * been expanded by others when we were blocked; 2164 * check again. 2165 */ 2166 goto again; 2167 } 2168 2169 /* 2170 * Save the old local group, link the new one, and then 2171 * destroy the old local group 2172 */ 2173 inp_localgroup_copy(grp_alloc, grp); 2174 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 2175 inp_localgroup_destroy(grp); 2176 2177 grp = grp_alloc; 2178 grp->il_jailed = isjailed; 2179 grp_alloc = NULL; 2180 } else { 2181 /* 2182 * Found the local group 2183 */ 2184 if (grp_alloc != NULL) { 2185 /* 2186 * This would happen if the local group w/ the 2187 * same property was added or expanded when our 2188 * local group allocation blocked. 2189 */ 2190 inp_localgroup_free(grp_alloc); 2191 grp_alloc = NULL; 2192 } 2193 } 2194 2195 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 2196 ("invalid local group size %d and count %d", 2197 grp->il_inpsiz, grp->il_inpcnt)); 2198 2199 /* 2200 * Keep the local group sorted by the inpcb local group index 2201 * in ascending order. 2202 * 2203 * This eases the multi-process userland application which uses 2204 * SO_REUSEPORT sockets and binds process to the owner cpu of 2205 * the SO_REUSEPORT socket: 2206 * If we didn't sort the local group by the inpcb local group 2207 * index and one of the process owning an inpcb in this local 2208 * group restarted, e.g. crashed and restarted by watchdog, 2209 * other processes owning a inpcb in this local group would have 2210 * to detect that event, refetch its socket's owner cpu, and 2211 * re-bind. 2212 */ 2213 idx = grp->il_inpcnt; 2214 for (i = 0; i < idx; ++i) { 2215 struct inpcb *oinp = grp->il_inp[i]; 2216 2217 if (oinp->inp_lgrpindex > i) { 2218 if (inp->inp_lgrpindex < 0) { 2219 inp->inp_lgrpindex = i; 2220 } else if (inp->inp_lgrpindex != i) { 2221 if (bootverbose) { 2222 kprintf("inp %p: grpidx %d, " 2223 "assigned to %d, cpu%d\n", 2224 inp, inp->inp_lgrpindex, i, 2225 mycpuid); 2226 } 2227 } 2228 grp->il_inp[i] = inp; 2229 2230 /* Pull down inpcbs */ 2231 for (; i < grp->il_inpcnt; ++i) { 2232 struct inpcb *oinp1 = grp->il_inp[i + 1]; 2233 2234 grp->il_inp[i + 1] = oinp; 2235 oinp = oinp1; 2236 } 2237 grp->il_inpcnt++; 2238 return; 2239 } 2240 } 2241 2242 if (inp->inp_lgrpindex < 0) { 2243 inp->inp_lgrpindex = idx; 2244 } else if (inp->inp_lgrpindex != idx) { 2245 if (bootverbose) { 2246 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 2247 inp, inp->inp_lgrpindex, idx, mycpuid); 2248 } 2249 } 2250 grp->il_inp[idx] = inp; 2251 grp->il_inpcnt++; 2252 } 2253 2254 void 2255 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2256 { 2257 struct inpcontainer *ic; 2258 struct inpcontainerhead *bucket; 2259 2260 GET_PCBINFO_TOKEN(pcbinfo); 2261 2262 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 2263 2264 bucket = &pcbinfo->wildcardhashbase[ 2265 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2266 2267 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 2268 ic->ic_inp = inp; 2269 LIST_INSERT_HEAD(bucket, ic, ic_list); 2270 2271 REL_PCBINFO_TOKEN(pcbinfo); 2272 } 2273 2274 /* 2275 * Insert PCB into wildcard hash table. 2276 */ 2277 void 2278 in_pcbinswildcardhash(struct inpcb *inp) 2279 { 2280 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2281 2282 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2283 ("not in correct netisr")); 2284 ASSERT_INP_NOTINHASH(inp); 2285 inp->inp_flags |= INP_WILDCARD; 2286 2287 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 2288 } 2289 2290 static void 2291 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2292 { 2293 struct inp_localgrphead *hdr; 2294 struct inp_localgroup *grp; 2295 2296 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2297 2298 if (pcbinfo->localgrphashbase == NULL) 2299 return; 2300 2301 hdr = &pcbinfo->localgrphashbase[ 2302 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2303 2304 LIST_FOREACH(grp, hdr, il_list) { 2305 int i; 2306 2307 for (i = 0; i < grp->il_inpcnt; ++i) { 2308 if (grp->il_inp[i] != inp) 2309 continue; 2310 2311 if (grp->il_inpcnt == 1) { 2312 /* Destroy this local group */ 2313 inp_localgroup_destroy(grp); 2314 } else { 2315 /* Pull up inpcbs */ 2316 for (; i + 1 < grp->il_inpcnt; ++i) 2317 grp->il_inp[i] = grp->il_inp[i + 1]; 2318 grp->il_inpcnt--; 2319 } 2320 return; 2321 } 2322 } 2323 } 2324 2325 void 2326 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2327 { 2328 struct inpcontainer *ic; 2329 struct inpcontainerhead *head; 2330 2331 GET_PCBINFO_TOKEN(pcbinfo); 2332 2333 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2334 2335 /* find bucket */ 2336 head = &pcbinfo->wildcardhashbase[ 2337 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2338 2339 LIST_FOREACH(ic, head, ic_list) { 2340 if (ic->ic_inp == inp) 2341 goto found; 2342 } 2343 REL_PCBINFO_TOKEN(pcbinfo); 2344 return; /* not found! */ 2345 2346 found: 2347 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2348 REL_PCBINFO_TOKEN(pcbinfo); 2349 kfree(ic, M_TEMP); /* deallocate container */ 2350 } 2351 2352 /* 2353 * Remove PCB from wildcard hash table. 2354 */ 2355 void 2356 in_pcbremwildcardhash(struct inpcb *inp) 2357 { 2358 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2359 2360 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2361 ("not in correct netisr")); 2362 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2363 2364 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2365 inp->inp_lgrpindex = -1; 2366 inp->inp_flags &= ~INP_WILDCARD; 2367 } 2368 2369 /* 2370 * Remove PCB from various lists. 2371 */ 2372 void 2373 in_pcbremlists(struct inpcb *inp) 2374 { 2375 in_pcbremporthash(inp); 2376 if (inp->inp_flags & INP_WILDCARD) { 2377 in_pcbremwildcardhash(inp); 2378 } else if (inp->inp_flags & INP_CONNECTED) { 2379 in_pcbremconnhash(inp); 2380 } 2381 2382 if (inp->inp_flags & INP_ONLIST) 2383 in_pcbofflist(inp); 2384 } 2385 2386 int 2387 prison_xinpcb(struct thread *td, struct inpcb *inp) 2388 { 2389 struct ucred *cr; 2390 2391 if (td->td_proc == NULL) 2392 return (0); 2393 cr = td->td_proc->p_ucred; 2394 if (cr->cr_prison == NULL) 2395 return (0); 2396 if (inp->inp_socket && inp->inp_socket->so_cred && 2397 inp->inp_socket->so_cred->cr_prison && 2398 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2399 return (0); 2400 return (1); 2401 } 2402 2403 int 2404 in_pcblist_range(SYSCTL_HANDLER_ARGS) 2405 { 2406 struct inpcbinfo *pcbinfo_arr = arg1; 2407 int pcbinfo_arrlen = arg2; 2408 struct inpcb *marker; 2409 int cpu, origcpu; 2410 int error, n; 2411 2412 KASSERT(pcbinfo_arrlen <= netisr_ncpus && pcbinfo_arrlen >= 1, 2413 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2414 2415 /* 2416 * The process of preparing the TCB list is too time-consuming and 2417 * resource-intensive to repeat twice on every request. 2418 */ 2419 n = 0; 2420 if (req->oldptr == NULL) { 2421 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2422 n += pcbinfo_arr[cpu].ipi_count; 2423 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2424 return 0; 2425 } 2426 2427 if (req->newptr != NULL) 2428 return EPERM; 2429 2430 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2431 marker->inp_flags |= INP_PLACEMARKER; 2432 2433 /* 2434 * OK, now we're committed to doing something. Re-fetch ipi_count 2435 * after obtaining the generation count. 2436 */ 2437 error = 0; 2438 origcpu = mycpuid; 2439 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2440 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2441 struct inpcb *inp; 2442 struct xinpcb xi; 2443 int i; 2444 2445 lwkt_migratecpu(cpu); 2446 2447 GET_PCBINFO_TOKEN(pcbinfo); 2448 2449 n = pcbinfo->ipi_count; 2450 2451 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2452 i = 0; 2453 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2454 LIST_REMOVE(marker, inp_list); 2455 LIST_INSERT_AFTER(inp, marker, inp_list); 2456 2457 if (inp->inp_flags & INP_PLACEMARKER) 2458 continue; 2459 if (prison_xinpcb(req->td, inp)) 2460 continue; 2461 2462 bzero(&xi, sizeof xi); 2463 xi.xi_len = sizeof xi; 2464 bcopy(inp, &xi.xi_inp, sizeof *inp); 2465 if (inp->inp_socket) 2466 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2467 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2468 break; 2469 ++i; 2470 } 2471 LIST_REMOVE(marker, inp_list); 2472 2473 REL_PCBINFO_TOKEN(pcbinfo); 2474 2475 if (error == 0 && i < n) { 2476 bzero(&xi, sizeof xi); 2477 xi.xi_len = sizeof xi; 2478 while (i < n) { 2479 error = SYSCTL_OUT(req, &xi, sizeof xi); 2480 if (error) 2481 break; 2482 ++i; 2483 } 2484 } 2485 } 2486 2487 lwkt_migratecpu(origcpu); 2488 kfree(marker, M_TEMP); 2489 return error; 2490 } 2491 2492 int 2493 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS) 2494 { 2495 2496 return (in_pcblist_range(oidp, arg1, netisr_ncpus, req)); 2497 } 2498 2499 void 2500 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2501 { 2502 struct sockaddr_in *sin; 2503 2504 KASSERT(faddr->sa_family == AF_INET, 2505 ("not AF_INET faddr %d", faddr->sa_family)); 2506 2507 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2508 sin->sin_family = AF_INET; 2509 sin->sin_len = sizeof(*sin); 2510 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2511 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2512 2513 so->so_faddr = (struct sockaddr *)sin; 2514 } 2515 2516 void 2517 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2518 u_short offset) 2519 { 2520 memset(portinfo, 0, sizeof(*portinfo)); 2521 2522 portinfo->offset = offset; 2523 portinfo->porthashbase = phashinit(hashsize, M_PCB, 2524 &portinfo->porthashcnt); 2525 } 2526 2527 void 2528 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2529 { 2530 int hi, lo; 2531 2532 if (step == 1) 2533 return; 2534 2535 hi = *hi0; 2536 lo = *lo0; 2537 2538 hi = rounddown(hi, step); 2539 hi += ofs; 2540 if (hi > (int)*hi0) 2541 hi -= step; 2542 2543 lo = roundup(lo, step); 2544 lo -= (step - ofs); 2545 if (lo < (int)*lo0) 2546 lo += step; 2547 2548 *hi0 = hi; 2549 *lo0 = lo; 2550 } 2551 2552 void 2553 in_pcbglobalinit(void) 2554 { 2555 int cpu; 2556 2557 in_pcbmarkers = kmalloc(netisr_ncpus * sizeof(struct inpcb), M_PCB, 2558 M_WAITOK | M_ZERO); 2559 in_pcbcontainer_markers = 2560 kmalloc(netisr_ncpus * sizeof(struct inpcontainer), M_PCB, 2561 M_WAITOK | M_ZERO); 2562 2563 for (cpu = 0; cpu < netisr_ncpus; ++cpu) { 2564 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2565 struct inpcb *marker = &in_pcbmarkers[cpu]; 2566 2567 marker->inp_flags |= INP_PLACEMARKER; 2568 ic->ic_inp = marker; 2569 } 2570 } 2571 2572 struct inpcb * 2573 in_pcbmarker(void) 2574 { 2575 2576 ASSERT_NETISR_NCPUS(mycpuid); 2577 return &in_pcbmarkers[mycpuid]; 2578 } 2579 2580 struct inpcontainer * 2581 in_pcbcontainer_marker(void) 2582 { 2583 2584 ASSERT_NETISR_NCPUS(mycpuid); 2585 return &in_pcbcontainer_markers[mycpuid]; 2586 } 2587 2588 void 2589 in_pcbresetroute(struct inpcb *inp) 2590 { 2591 struct route *ro = &inp->inp_route; 2592 2593 if (ro->ro_rt != NULL) 2594 RTFREE(ro->ro_rt); 2595 bzero(ro, sizeof(*ro)); 2596 } 2597