1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)udp_usrreq.c 8.6 (Berkeley) 5/23/95 63 * $FreeBSD: src/sys/netinet/udp_usrreq.c,v 1.64.2.18 2003/01/24 05:11:34 sam Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/kernel.h> 72 #include <sys/malloc.h> 73 #include <sys/mbuf.h> 74 #include <sys/domain.h> 75 #include <sys/proc.h> 76 #include <sys/priv.h> 77 #include <sys/protosw.h> 78 #include <sys/socket.h> 79 #include <sys/socketvar.h> 80 #include <sys/sysctl.h> 81 #include <sys/syslog.h> 82 #include <sys/in_cksum.h> 83 #include <sys/ktr.h> 84 85 #include <sys/thread2.h> 86 #include <sys/socketvar2.h> 87 #include <sys/serialize.h> 88 89 #include <machine/stdarg.h> 90 91 #include <net/if.h> 92 #include <net/route.h> 93 #include <net/netmsg2.h> 94 #include <net/netisr2.h> 95 96 #include <netinet/in.h> 97 #include <netinet/in_systm.h> 98 #include <netinet/ip.h> 99 #ifdef INET6 100 #include <netinet/ip6.h> 101 #endif 102 #include <netinet/in_pcb.h> 103 #include <netinet/in_var.h> 104 #include <netinet/ip_var.h> 105 #ifdef INET6 106 #include <netinet6/ip6_var.h> 107 #endif 108 #include <netinet/ip_icmp.h> 109 #include <netinet/icmp_var.h> 110 #include <netinet/udp.h> 111 #include <netinet/udp_var.h> 112 113 #ifdef FAST_IPSEC 114 #include <netproto/ipsec/ipsec.h> 115 #endif 116 117 #ifdef IPSEC 118 #include <netinet6/ipsec.h> 119 #endif 120 121 #define MSGF_UDP_SEND MSGF_PROTO1 122 123 #define INP_DIRECT_DETACH INP_FLAG_PROTO2 124 125 #define UDP_KTR_STRING "inp=%p" 126 #define UDP_KTR_ARGS struct inpcb *inp 127 128 #ifndef KTR_UDP 129 #define KTR_UDP KTR_ALL 130 #endif 131 132 KTR_INFO_MASTER(udp); 133 KTR_INFO(KTR_UDP, udp, send_beg, 0, UDP_KTR_STRING, UDP_KTR_ARGS); 134 KTR_INFO(KTR_UDP, udp, send_end, 1, UDP_KTR_STRING, UDP_KTR_ARGS); 135 KTR_INFO(KTR_UDP, udp, send_ipout, 2, UDP_KTR_STRING, UDP_KTR_ARGS); 136 KTR_INFO(KTR_UDP, udp, redisp_ipout_beg, 3, UDP_KTR_STRING, UDP_KTR_ARGS); 137 KTR_INFO(KTR_UDP, udp, redisp_ipout_end, 4, UDP_KTR_STRING, UDP_KTR_ARGS); 138 KTR_INFO(KTR_UDP, udp, send_redisp, 5, UDP_KTR_STRING, UDP_KTR_ARGS); 139 KTR_INFO(KTR_UDP, udp, send_inswildcard, 6, UDP_KTR_STRING, UDP_KTR_ARGS); 140 141 #define logudp(name, inp) KTR_LOG(udp_##name, inp) 142 143 /* 144 * UDP protocol implementation. 145 * Per RFC 768, August, 1980. 146 */ 147 #ifndef COMPAT_42 148 static int udpcksum = 1; 149 #else 150 static int udpcksum = 0; /* XXX */ 151 #endif 152 SYSCTL_INT(_net_inet_udp, UDPCTL_CHECKSUM, checksum, CTLFLAG_RW, 153 &udpcksum, 0, "Enable checksumming of UDP packets"); 154 155 int log_in_vain = 0; 156 SYSCTL_INT(_net_inet_udp, OID_AUTO, log_in_vain, CTLFLAG_RW, 157 &log_in_vain, 0, "Log all incoming UDP packets"); 158 159 static int blackhole = 0; 160 SYSCTL_INT(_net_inet_udp, OID_AUTO, blackhole, CTLFLAG_RW, 161 &blackhole, 0, "Do not send port unreachables for refused connects"); 162 163 static int strict_mcast_mship = 1; 164 SYSCTL_INT(_net_inet_udp, OID_AUTO, strict_mcast_mship, CTLFLAG_RW, 165 &strict_mcast_mship, 0, "Only send multicast to member sockets"); 166 167 int udp_sosend_async = 1; 168 SYSCTL_INT(_net_inet_udp, OID_AUTO, sosend_async, CTLFLAG_RW, 169 &udp_sosend_async, 0, "UDP asynchronized pru_send"); 170 171 int udp_sosend_prepend = 1; 172 SYSCTL_INT(_net_inet_udp, OID_AUTO, sosend_prepend, CTLFLAG_RW, 173 &udp_sosend_prepend, 0, 174 "Prepend enough space for proto and link header in pru_send"); 175 176 static int udp_reuseport_ext = 1; 177 SYSCTL_INT(_net_inet_udp, OID_AUTO, reuseport_ext, CTLFLAG_RW, 178 &udp_reuseport_ext, 0, "SO_REUSEPORT extension"); 179 180 struct inpcbinfo udbinfo[MAXCPU]; 181 182 #ifndef UDBHASHSIZE 183 #define UDBHASHSIZE 16 184 #endif 185 186 struct udpstat udpstat_percpu[MAXCPU] __cachealign; 187 188 static void udp_append(struct inpcb *last, struct ip *ip, 189 struct mbuf *n, int off, struct sockaddr_in *udp_in); 190 191 static int udp_connect_oncpu(struct inpcb *inp, struct sockaddr_in *sin, 192 struct sockaddr_in *if_sin); 193 194 static boolean_t udp_inswildcardhash(struct inpcb *inp, 195 struct netmsg_base *msg, int error); 196 static void udp_remwildcardhash(struct inpcb *inp); 197 198 void 199 udp_init(void) 200 { 201 struct inpcbportinfo *portinfo; 202 int cpu; 203 204 portinfo = kmalloc_cachealign(sizeof(*portinfo) * ncpus2, M_PCB, 205 M_WAITOK); 206 207 for (cpu = 0; cpu < ncpus2; cpu++) { 208 struct inpcbinfo *uicb = &udbinfo[cpu]; 209 210 /* 211 * NOTE: 212 * UDP pcb list, wildcard hash table and localgroup hash 213 * table are shared. 214 */ 215 in_pcbinfo_init(uicb, cpu, TRUE); 216 uicb->hashbase = hashinit(UDBHASHSIZE, M_PCB, &uicb->hashmask); 217 218 in_pcbportinfo_init(&portinfo[cpu], UDBHASHSIZE, TRUE, cpu); 219 uicb->portinfo = portinfo; 220 uicb->portinfo_mask = ncpus2_mask; 221 222 uicb->wildcardhashbase = hashinit(UDBHASHSIZE, M_PCB, 223 &uicb->wildcardhashmask); 224 uicb->localgrphashbase = hashinit(UDBHASHSIZE, M_PCB, 225 &uicb->localgrphashmask); 226 227 uicb->ipi_size = sizeof(struct inpcb); 228 } 229 230 /* 231 * Initialize UDP statistics counters for each CPU. 232 */ 233 for (cpu = 0; cpu < ncpus; ++cpu) 234 bzero(&udpstat_percpu[cpu], sizeof(struct udpstat)); 235 } 236 237 static int 238 sysctl_udpstat(SYSCTL_HANDLER_ARGS) 239 { 240 int cpu, error = 0; 241 242 for (cpu = 0; cpu < ncpus; ++cpu) { 243 if ((error = SYSCTL_OUT(req, &udpstat_percpu[cpu], 244 sizeof(struct udpstat)))) 245 break; 246 if ((error = SYSCTL_IN(req, &udpstat_percpu[cpu], 247 sizeof(struct udpstat)))) 248 break; 249 } 250 251 return (error); 252 } 253 SYSCTL_PROC(_net_inet_udp, UDPCTL_STATS, stats, (CTLTYPE_OPAQUE | CTLFLAG_RW), 254 0, 0, sysctl_udpstat, "S,udpstat", "UDP statistics"); 255 256 void 257 udp_ctloutput(netmsg_t msg) 258 { 259 struct socket *so = msg->base.nm_so; 260 struct sockopt *sopt = msg->ctloutput.nm_sopt; 261 struct inpcb *inp = so->so_pcb; 262 263 if (inp == NULL) { 264 lwkt_replymsg(&msg->lmsg, EINVAL); 265 return; 266 } 267 268 if (sopt->sopt_level == IPPROTO_IP && sopt->sopt_dir == SOPT_SET) { 269 switch (sopt->sopt_name) { 270 case IP_MULTICAST_IF: 271 case IP_MULTICAST_VIF: 272 case IP_MULTICAST_TTL: 273 case IP_MULTICAST_LOOP: 274 case IP_ADD_MEMBERSHIP: 275 case IP_DROP_MEMBERSHIP: 276 /* 277 * This pr_ctloutput msg will be forwarded 278 * to netisr0 to run; we can't do direct 279 * detaching anymore. 280 * 281 * NOTE: 282 * Don't optimize for the sockets whose 283 * current so_port is netisr0's msgport. 284 * These sockets could be connect(2)'ed 285 * later and the so_port will be changed. 286 */ 287 inp->inp_flags &= ~INP_DIRECT_DETACH; 288 break; 289 } 290 } 291 return ip_ctloutput(msg); 292 } 293 294 /* 295 * Check multicast packets to make sure they are only sent to sockets with 296 * multicast memberships for the packet's destination address and arrival 297 * interface. Multicast packets to multicast-unaware sockets are also 298 * disallowed. 299 * 300 * Returns 0 if the packet is acceptable, -1 if it is not. 301 */ 302 static __inline int 303 check_multicast_membership(const struct ip *ip, const struct inpcb *inp, 304 const struct mbuf *m) 305 { 306 const struct ip_moptions *mopt; 307 int mshipno; 308 309 if (strict_mcast_mship == 0 || 310 !IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) { 311 return (0); 312 } 313 314 ASSERT_IN_NETISR(0); 315 316 mopt = inp->inp_moptions; 317 if (mopt == NULL) 318 return (-1); 319 for (mshipno = 0; mshipno < mopt->imo_num_memberships; ++mshipno) { 320 const struct in_multi *maddr = mopt->imo_membership[mshipno]; 321 322 if (ip->ip_dst.s_addr == maddr->inm_addr.s_addr && 323 m->m_pkthdr.rcvif == maddr->inm_ifp) { 324 return (0); 325 } 326 } 327 return (-1); 328 } 329 330 struct udp_mcast_arg { 331 struct inpcb *inp; 332 struct inpcb *last; 333 struct ip *ip; 334 struct mbuf *m; 335 int iphlen; 336 struct sockaddr_in *udp_in; 337 }; 338 339 static int 340 udp_mcast_input(struct udp_mcast_arg *arg) 341 { 342 struct inpcb *inp = arg->inp; 343 struct inpcb *last = arg->last; 344 struct ip *ip = arg->ip; 345 struct mbuf *m = arg->m; 346 347 if (check_multicast_membership(ip, inp, m) < 0) 348 return ERESTART; /* caller continue */ 349 350 if (last != NULL) { 351 struct mbuf *n; 352 353 #ifdef IPSEC 354 /* check AH/ESP integrity. */ 355 if (ipsec4_in_reject_so(m, last->inp_socket)) 356 ipsecstat.in_polvio++; 357 /* do not inject data to pcb */ 358 else 359 #endif /*IPSEC*/ 360 #ifdef FAST_IPSEC 361 /* check AH/ESP integrity. */ 362 if (ipsec4_in_reject(m, last)) 363 ; 364 else 365 #endif /*FAST_IPSEC*/ 366 if ((n = m_copypacket(m, M_NOWAIT)) != NULL) 367 udp_append(last, ip, n, 368 arg->iphlen + sizeof(struct udphdr), 369 arg->udp_in); 370 } 371 arg->last = last = inp; 372 373 /* 374 * Don't look for additional matches if this one does 375 * not have either the SO_REUSEPORT or SO_REUSEADDR 376 * socket options set. This heuristic avoids searching 377 * through all pcbs in the common case of a non-shared 378 * port. It * assumes that an application will never 379 * clear these options after setting them. 380 */ 381 if (!(last->inp_socket->so_options & 382 (SO_REUSEPORT | SO_REUSEADDR))) 383 return EJUSTRETURN; /* caller stop */ 384 return 0; 385 } 386 387 int 388 udp_input(struct mbuf **mp, int *offp, int proto) 389 { 390 struct sockaddr_in udp_in = { sizeof udp_in, AF_INET }; 391 int iphlen; 392 struct ip *ip; 393 struct udphdr *uh; 394 struct inpcb *inp; 395 struct mbuf *m; 396 struct mbuf *opts = NULL; 397 int len, off; 398 struct ip save_ip; 399 struct inpcbinfo *pcbinfo = &udbinfo[mycpuid]; 400 401 off = *offp; 402 m = *mp; 403 *mp = NULL; 404 405 iphlen = off; 406 udp_stat.udps_ipackets++; 407 408 /* 409 * Strip IP options, if any; should skip this, 410 * make available to user, and use on returned packets, 411 * but we don't yet have a way to check the checksum 412 * with options still present. 413 */ 414 if (iphlen > sizeof(struct ip)) { 415 ip_stripoptions(m); 416 iphlen = sizeof(struct ip); 417 } 418 419 /* 420 * IP and UDP headers are together in first mbuf. 421 * Already checked and pulled up in ip_demux(). 422 */ 423 KASSERT(m->m_len >= iphlen + sizeof(struct udphdr), 424 ("UDP header not in one mbuf")); 425 426 ip = mtod(m, struct ip *); 427 uh = (struct udphdr *)((caddr_t)ip + iphlen); 428 429 /* destination port of 0 is illegal, based on RFC768. */ 430 if (uh->uh_dport == 0) 431 goto bad; 432 433 /* 434 * Make mbuf data length reflect UDP length. 435 * If not enough data to reflect UDP length, drop. 436 */ 437 len = ntohs((u_short)uh->uh_ulen); 438 if (ip->ip_len != len) { 439 if (len > ip->ip_len || len < sizeof(struct udphdr)) { 440 udp_stat.udps_badlen++; 441 goto bad; 442 } 443 m_adj(m, len - ip->ip_len); 444 /* ip->ip_len = len; */ 445 } 446 /* 447 * Save a copy of the IP header in case we want restore it 448 * for sending an ICMP error message in response. 449 */ 450 save_ip = *ip; 451 452 /* 453 * Checksum extended UDP header and data. 454 */ 455 if (uh->uh_sum) { 456 if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) { 457 if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) 458 uh->uh_sum = m->m_pkthdr.csum_data; 459 else 460 uh->uh_sum = in_pseudo(ip->ip_src.s_addr, 461 ip->ip_dst.s_addr, htonl((u_short)len + 462 m->m_pkthdr.csum_data + IPPROTO_UDP)); 463 uh->uh_sum ^= 0xffff; 464 } else { 465 char b[9]; 466 467 bcopy(((struct ipovly *)ip)->ih_x1, b, 9); 468 bzero(((struct ipovly *)ip)->ih_x1, 9); 469 ((struct ipovly *)ip)->ih_len = uh->uh_ulen; 470 uh->uh_sum = in_cksum(m, len + sizeof(struct ip)); 471 bcopy(b, ((struct ipovly *)ip)->ih_x1, 9); 472 } 473 if (uh->uh_sum) { 474 udp_stat.udps_badsum++; 475 m_freem(m); 476 return(IPPROTO_DONE); 477 } 478 } else 479 udp_stat.udps_nosum++; 480 481 if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) || 482 in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif)) { 483 struct inpcbhead *connhead; 484 struct inpcontainer *ic, *ic_marker; 485 struct inpcontainerhead *ichead; 486 struct udp_mcast_arg arg; 487 struct inpcb *last; 488 int error; 489 490 /* 491 * Deliver a multicast or broadcast datagram to *all* sockets 492 * for which the local and remote addresses and ports match 493 * those of the incoming datagram. This allows more than 494 * one process to receive multi/broadcasts on the same port. 495 * (This really ought to be done for unicast datagrams as 496 * well, but that would cause problems with existing 497 * applications that open both address-specific sockets and 498 * a wildcard socket listening to the same port -- they would 499 * end up receiving duplicates of every unicast datagram. 500 * Those applications open the multiple sockets to overcome an 501 * inadequacy of the UDP socket interface, but for backwards 502 * compatibility we avoid the problem here rather than 503 * fixing the interface. Maybe 4.5BSD will remedy this?) 504 */ 505 506 /* 507 * Construct sockaddr format source address. 508 */ 509 udp_in.sin_port = uh->uh_sport; 510 udp_in.sin_addr = ip->ip_src; 511 arg.udp_in = &udp_in; 512 /* 513 * Locate pcb(s) for datagram. 514 * (Algorithm copied from raw_intr().) 515 */ 516 last = NULL; 517 arg.iphlen = iphlen; 518 519 connhead = &pcbinfo->hashbase[ 520 INP_PCBCONNHASH(ip->ip_src.s_addr, uh->uh_sport, 521 ip->ip_dst.s_addr, uh->uh_dport, pcbinfo->hashmask)]; 522 LIST_FOREACH(inp, connhead, inp_hash) { 523 #ifdef INET6 524 if (!INP_ISIPV4(inp)) 525 continue; 526 #endif 527 if (!in_hosteq(inp->inp_faddr, ip->ip_src) || 528 !in_hosteq(inp->inp_laddr, ip->ip_dst) || 529 inp->inp_fport != uh->uh_sport || 530 inp->inp_lport != uh->uh_dport) 531 continue; 532 533 arg.inp = inp; 534 arg.last = last; 535 arg.ip = ip; 536 arg.m = m; 537 538 error = udp_mcast_input(&arg); 539 if (error == ERESTART) 540 continue; 541 last = arg.last; 542 543 if (error == EJUSTRETURN) 544 goto done; 545 } 546 547 ichead = &pcbinfo->wildcardhashbase[ 548 INP_PCBWILDCARDHASH(uh->uh_dport, 549 pcbinfo->wildcardhashmask)]; 550 ic_marker = in_pcbcontainer_marker(mycpuid); 551 552 GET_PCBINFO_TOKEN(pcbinfo); 553 LIST_INSERT_HEAD(ichead, ic_marker, ic_list); 554 while ((ic = LIST_NEXT(ic_marker, ic_list)) != NULL) { 555 LIST_REMOVE(ic_marker, ic_list); 556 LIST_INSERT_AFTER(ic, ic_marker, ic_list); 557 558 inp = ic->ic_inp; 559 if (inp->inp_flags & INP_PLACEMARKER) 560 continue; 561 #ifdef INET6 562 if (!INP_ISIPV4(inp)) 563 continue; 564 #endif 565 if (inp->inp_lport != uh->uh_dport) 566 continue; 567 if (inp->inp_laddr.s_addr != INADDR_ANY && 568 inp->inp_laddr.s_addr != ip->ip_dst.s_addr) 569 continue; 570 571 arg.inp = inp; 572 arg.last = last; 573 arg.ip = ip; 574 arg.m = m; 575 576 error = udp_mcast_input(&arg); 577 if (error == ERESTART) 578 continue; 579 last = arg.last; 580 581 if (error == EJUSTRETURN) 582 break; 583 } 584 LIST_REMOVE(ic_marker, ic_list); 585 REL_PCBINFO_TOKEN(pcbinfo); 586 done: 587 if (last == NULL) { 588 /* 589 * No matching pcb found; discard datagram. 590 * (No need to send an ICMP Port Unreachable 591 * for a broadcast or multicast datgram.) 592 */ 593 udp_stat.udps_noportbcast++; 594 goto bad; 595 } 596 #ifdef IPSEC 597 /* check AH/ESP integrity. */ 598 if (ipsec4_in_reject_so(m, last->inp_socket)) { 599 ipsecstat.in_polvio++; 600 goto bad; 601 } 602 #endif /*IPSEC*/ 603 #ifdef FAST_IPSEC 604 /* check AH/ESP integrity. */ 605 if (ipsec4_in_reject(m, last)) 606 goto bad; 607 #endif /*FAST_IPSEC*/ 608 udp_append(last, ip, m, iphlen + sizeof(struct udphdr), 609 &udp_in); 610 return(IPPROTO_DONE); 611 } 612 /* 613 * Locate pcb for datagram. 614 */ 615 inp = in_pcblookup_pkthash(pcbinfo, ip->ip_src, uh->uh_sport, 616 ip->ip_dst, uh->uh_dport, TRUE, m->m_pkthdr.rcvif, 617 udp_reuseport_ext ? m : NULL); 618 if (inp == NULL) { 619 if (log_in_vain) { 620 char buf[sizeof "aaa.bbb.ccc.ddd"]; 621 622 strcpy(buf, inet_ntoa(ip->ip_dst)); 623 log(LOG_INFO, 624 "Connection attempt to UDP %s:%d from %s:%d\n", 625 buf, ntohs(uh->uh_dport), inet_ntoa(ip->ip_src), 626 ntohs(uh->uh_sport)); 627 } 628 udp_stat.udps_noport++; 629 if (m->m_flags & (M_BCAST | M_MCAST)) { 630 udp_stat.udps_noportbcast++; 631 goto bad; 632 } 633 if (blackhole) 634 goto bad; 635 #ifdef ICMP_BANDLIM 636 if (badport_bandlim(BANDLIM_ICMP_UNREACH) < 0) 637 goto bad; 638 #endif 639 *ip = save_ip; 640 ip->ip_len += iphlen; 641 icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_PORT, 0, 0); 642 return(IPPROTO_DONE); 643 } 644 KASSERT(INP_ISIPV4(inp), ("not inet inpcb")); 645 #ifdef IPSEC 646 if (ipsec4_in_reject_so(m, inp->inp_socket)) { 647 ipsecstat.in_polvio++; 648 goto bad; 649 } 650 #endif /*IPSEC*/ 651 #ifdef FAST_IPSEC 652 if (ipsec4_in_reject(m, inp)) 653 goto bad; 654 #endif /*FAST_IPSEC*/ 655 /* 656 * Check the minimum TTL for socket. 657 */ 658 if (ip->ip_ttl < inp->inp_ip_minttl) 659 goto bad; 660 661 /* 662 * Construct sockaddr format source address. 663 * Stuff source address and datagram in user buffer. 664 */ 665 udp_in.sin_port = uh->uh_sport; 666 udp_in.sin_addr = ip->ip_src; 667 if ((inp->inp_flags & INP_CONTROLOPTS) || 668 (inp->inp_socket->so_options & SO_TIMESTAMP)) 669 ip_savecontrol(inp, &opts, ip, m); 670 m_adj(m, iphlen + sizeof(struct udphdr)); 671 672 lwkt_gettoken(&inp->inp_socket->so_rcv.ssb_token); 673 if (ssb_appendaddr(&inp->inp_socket->so_rcv, 674 (struct sockaddr *)&udp_in, m, opts) == 0) { 675 lwkt_reltoken(&inp->inp_socket->so_rcv.ssb_token); 676 udp_stat.udps_fullsock++; 677 goto bad; 678 } 679 lwkt_reltoken(&inp->inp_socket->so_rcv.ssb_token); 680 sorwakeup(inp->inp_socket); 681 return(IPPROTO_DONE); 682 bad: 683 m_freem(m); 684 if (opts) 685 m_freem(opts); 686 return(IPPROTO_DONE); 687 } 688 689 /* 690 * subroutine of udp_input(), mainly for source code readability. 691 * caller must properly init udp_ip6 and udp_in6 beforehand. 692 */ 693 static void 694 udp_append(struct inpcb *last, struct ip *ip, struct mbuf *n, int off, 695 struct sockaddr_in *udp_in) 696 { 697 struct mbuf *opts = NULL; 698 int ret; 699 700 KASSERT(INP_ISIPV4(last), ("not inet inpcb")); 701 702 if (last->inp_flags & INP_CONTROLOPTS || 703 last->inp_socket->so_options & SO_TIMESTAMP) 704 ip_savecontrol(last, &opts, ip, n); 705 m_adj(n, off); 706 707 lwkt_gettoken(&last->inp_socket->so_rcv.ssb_token); 708 ret = ssb_appendaddr(&last->inp_socket->so_rcv, 709 (struct sockaddr *)udp_in, n, opts); 710 lwkt_reltoken(&last->inp_socket->so_rcv.ssb_token); 711 if (ret == 0) { 712 m_freem(n); 713 if (opts) 714 m_freem(opts); 715 udp_stat.udps_fullsock++; 716 } else { 717 sorwakeup(last->inp_socket); 718 } 719 } 720 721 /* 722 * Notify a udp user of an asynchronous error; 723 * just wake up so that he can collect error status. 724 */ 725 void 726 udp_notify(struct inpcb *inp, int error) 727 { 728 inp->inp_socket->so_error = error; 729 sorwakeup(inp->inp_socket); 730 sowwakeup(inp->inp_socket); 731 } 732 733 struct netmsg_udp_notify { 734 struct netmsg_base base; 735 inp_notify_t nm_notify; 736 struct in_addr nm_faddr; 737 int nm_arg; 738 }; 739 740 static void 741 udp_notifyall_oncpu(netmsg_t msg) 742 { 743 struct netmsg_udp_notify *nm = (struct netmsg_udp_notify *)msg; 744 int nextcpu, cpu = mycpuid; 745 746 in_pcbnotifyall(&udbinfo[cpu], nm->nm_faddr, nm->nm_arg, nm->nm_notify); 747 748 nextcpu = cpu + 1; 749 if (nextcpu < ncpus2) 750 lwkt_forwardmsg(netisr_cpuport(nextcpu), &nm->base.lmsg); 751 else 752 lwkt_replymsg(&nm->base.lmsg, 0); 753 } 754 755 inp_notify_t 756 udp_get_inpnotify(int cmd, const struct sockaddr *sa, 757 struct ip **ip0, int *cpuid) 758 { 759 struct in_addr faddr; 760 struct ip *ip = *ip0; 761 inp_notify_t notify = udp_notify; 762 763 faddr = ((const struct sockaddr_in *)sa)->sin_addr; 764 if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY) 765 return NULL; 766 767 if (PRC_IS_REDIRECT(cmd)) { 768 ip = NULL; 769 notify = in_rtchange; 770 } else if (cmd == PRC_HOSTDEAD) { 771 ip = NULL; 772 } else if ((unsigned)cmd >= PRC_NCMDS || inetctlerrmap[cmd] == 0) { 773 return NULL; 774 } 775 776 if (cpuid != NULL) { 777 if (ip == NULL) { 778 /* Go through all CPUs */ 779 *cpuid = ncpus; 780 } else { 781 const struct udphdr *uh; 782 783 uh = (const struct udphdr *) 784 ((caddr_t)ip + (ip->ip_hl << 2)); 785 *cpuid = udp_addrcpu(faddr.s_addr, uh->uh_dport, 786 ip->ip_src.s_addr, uh->uh_sport); 787 } 788 } 789 790 *ip0 = ip; 791 return notify; 792 } 793 794 void 795 udp_ctlinput(netmsg_t msg) 796 { 797 struct sockaddr *sa = msg->ctlinput.nm_arg; 798 struct ip *ip = msg->ctlinput.nm_extra; 799 int cmd = msg->ctlinput.nm_cmd, cpuid; 800 inp_notify_t notify; 801 struct in_addr faddr; 802 803 notify = udp_get_inpnotify(cmd, sa, &ip, &cpuid); 804 if (notify == NULL) 805 goto done; 806 807 faddr = ((struct sockaddr_in *)sa)->sin_addr; 808 if (ip) { 809 const struct udphdr *uh; 810 struct inpcb *inp; 811 812 if (cpuid != mycpuid) 813 goto done; 814 815 uh = (const struct udphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 816 inp = in_pcblookup_hash(&udbinfo[mycpuid], faddr, uh->uh_dport, 817 ip->ip_src, uh->uh_sport, 0, NULL); 818 if (inp != NULL && inp->inp_socket != NULL) 819 notify(inp, inetctlerrmap[cmd]); 820 } else if (msg->ctlinput.nm_direct) { 821 if (cpuid != ncpus && cpuid != mycpuid) 822 goto done; 823 if (mycpuid >= ncpus2) 824 goto done; 825 826 in_pcbnotifyall(&udbinfo[mycpuid], faddr, inetctlerrmap[cmd], 827 notify); 828 } else { 829 struct netmsg_udp_notify *nm; 830 831 ASSERT_IN_NETISR(0); 832 nm = kmalloc(sizeof(*nm), M_LWKTMSG, M_INTWAIT); 833 netmsg_init(&nm->base, NULL, &netisr_afree_rport, 834 0, udp_notifyall_oncpu); 835 nm->nm_faddr = faddr; 836 nm->nm_arg = inetctlerrmap[cmd]; 837 nm->nm_notify = notify; 838 lwkt_sendmsg(netisr_cpuport(0), &nm->base.lmsg); 839 } 840 done: 841 lwkt_replymsg(&msg->lmsg, 0); 842 } 843 844 SYSCTL_PROC(_net_inet_udp, UDPCTL_PCBLIST, pcblist, CTLFLAG_RD, udbinfo, 0, 845 in_pcblist_global_ncpus2, "S,xinpcb", "List of active UDP sockets"); 846 847 static int 848 udp_getcred(SYSCTL_HANDLER_ARGS) 849 { 850 struct sockaddr_in addrs[2]; 851 struct ucred cred0, *cred = NULL; 852 struct inpcb *inp; 853 int error, cpu, origcpu; 854 855 error = priv_check(req->td, PRIV_ROOT); 856 if (error) 857 return (error); 858 error = SYSCTL_IN(req, addrs, sizeof addrs); 859 if (error) 860 return (error); 861 862 origcpu = mycpuid; 863 cpu = udp_addrcpu(addrs[1].sin_addr.s_addr, addrs[1].sin_port, 864 addrs[0].sin_addr.s_addr, addrs[0].sin_port); 865 866 lwkt_migratecpu(cpu); 867 868 inp = in_pcblookup_hash(&udbinfo[cpu], 869 addrs[1].sin_addr, addrs[1].sin_port, 870 addrs[0].sin_addr, addrs[0].sin_port, TRUE, NULL); 871 if (inp == NULL || inp->inp_socket == NULL) { 872 error = ENOENT; 873 } else if (inp->inp_socket->so_cred != NULL) { 874 cred0 = *(inp->inp_socket->so_cred); 875 cred = &cred0; 876 } 877 878 lwkt_migratecpu(origcpu); 879 880 if (error) 881 return error; 882 883 return SYSCTL_OUT(req, cred, sizeof(struct ucred)); 884 } 885 SYSCTL_PROC(_net_inet_udp, OID_AUTO, getcred, CTLTYPE_OPAQUE|CTLFLAG_RW, 886 0, 0, udp_getcred, "S,ucred", "Get the ucred of a UDP connection"); 887 888 static void 889 udp_send_redispatch(netmsg_t msg) 890 { 891 struct mbuf *m = msg->send.nm_m; 892 int pru_flags = msg->send.nm_flags; 893 struct inpcb *inp = msg->send.base.nm_so->so_pcb; 894 struct mbuf *m_opt = msg->send.nm_control; /* XXX save ipopt */ 895 int flags = msg->send.nm_priv; /* ip_output flags */ 896 int error; 897 898 logudp(redisp_ipout_beg, inp); 899 900 /* 901 * - Don't use inp route cache. It should only be used in the 902 * inp owner netisr. 903 * - Access to inp_moptions should be safe, since multicast UDP 904 * datagrams are redispatched to netisr0 and inp_moptions is 905 * changed only in netisr0. 906 */ 907 error = ip_output(m, m_opt, NULL, flags, inp->inp_moptions, inp); 908 if ((pru_flags & PRUS_NOREPLY) == 0) 909 lwkt_replymsg(&msg->send.base.lmsg, error); 910 911 if (m_opt != NULL) { 912 /* Free saved ip options, if any */ 913 m_freem(m_opt); 914 } 915 916 logudp(redisp_ipout_end, inp); 917 } 918 919 static void 920 udp_send(netmsg_t msg) 921 { 922 struct socket *so = msg->send.base.nm_so; 923 struct mbuf *m = msg->send.nm_m; 924 struct sockaddr *dstaddr = msg->send.nm_addr; 925 int pru_flags = msg->send.nm_flags; 926 struct inpcb *inp = so->so_pcb; 927 struct thread *td = msg->send.nm_td; 928 int flags; 929 930 struct udpiphdr *ui; 931 int len = m->m_pkthdr.len; 932 struct sockaddr_in *sin; /* really is initialized before use */ 933 int error = 0, cpu; 934 935 KKASSERT(msg->send.nm_control == NULL); 936 937 logudp(send_beg, inp); 938 939 if (inp == NULL) { 940 error = EINVAL; 941 goto release; 942 } 943 944 if (len + sizeof(struct udpiphdr) > IP_MAXPACKET) { 945 error = EMSGSIZE; 946 goto release; 947 } 948 949 if (inp->inp_lport == 0) { /* unbound socket */ 950 boolean_t forwarded; 951 952 error = in_pcbbind(inp, NULL, td); 953 if (error) 954 goto release; 955 956 /* 957 * Need to call udp_send again, after this inpcb is 958 * inserted into wildcard hash table. 959 */ 960 msg->send.base.lmsg.ms_flags |= MSGF_UDP_SEND; 961 forwarded = udp_inswildcardhash(inp, &msg->send.base, 0); 962 if (forwarded) { 963 /* 964 * The message is further forwarded, so we are 965 * done here. 966 */ 967 logudp(send_inswildcard, inp); 968 return; 969 } 970 } 971 972 if (dstaddr != NULL) { /* destination address specified */ 973 if (inp->inp_faddr.s_addr != INADDR_ANY) { 974 /* already connected */ 975 error = EISCONN; 976 goto release; 977 } 978 sin = (struct sockaddr_in *)dstaddr; 979 if (!prison_remote_ip(td, (struct sockaddr *)&sin)) { 980 error = EAFNOSUPPORT; /* IPv6 only jail */ 981 goto release; 982 } 983 } else { 984 if (inp->inp_faddr.s_addr == INADDR_ANY) { 985 /* no destination specified and not already connected */ 986 error = ENOTCONN; 987 goto release; 988 } 989 sin = NULL; 990 } 991 992 /* 993 * Calculate data length and get a mbuf 994 * for UDP and IP headers. 995 */ 996 M_PREPEND(m, sizeof(struct udpiphdr), M_NOWAIT); 997 if (m == NULL) { 998 error = ENOBUFS; 999 goto release; 1000 } 1001 1002 /* 1003 * Fill in mbuf with extended UDP header 1004 * and addresses and length put into network format. 1005 */ 1006 ui = mtod(m, struct udpiphdr *); 1007 bzero(ui->ui_x1, sizeof ui->ui_x1); /* XXX still needed? */ 1008 ui->ui_pr = IPPROTO_UDP; 1009 1010 /* 1011 * Set destination address. 1012 */ 1013 if (dstaddr != NULL) { /* use specified destination */ 1014 ui->ui_dst = sin->sin_addr; 1015 ui->ui_dport = sin->sin_port; 1016 } else { /* use connected destination */ 1017 ui->ui_dst = inp->inp_faddr; 1018 ui->ui_dport = inp->inp_fport; 1019 } 1020 1021 /* 1022 * Set source address. 1023 */ 1024 if (inp->inp_laddr.s_addr == INADDR_ANY || 1025 IN_MULTICAST(ntohl(inp->inp_laddr.s_addr))) { 1026 struct sockaddr_in *if_sin; 1027 1028 if (dstaddr == NULL) { 1029 /* 1030 * connect() had (or should have) failed because 1031 * the interface had no IP address, but the 1032 * application proceeded to call send() anyways. 1033 */ 1034 error = ENOTCONN; 1035 goto release; 1036 } 1037 1038 /* Look up outgoing interface. */ 1039 error = in_pcbladdr_find(inp, dstaddr, &if_sin, td, 1); 1040 if (error) 1041 goto release; 1042 ui->ui_src = if_sin->sin_addr; /* use address of interface */ 1043 } else { 1044 ui->ui_src = inp->inp_laddr; /* use non-null bound address */ 1045 } 1046 ui->ui_sport = inp->inp_lport; 1047 KASSERT(inp->inp_lport != 0, ("inp lport should have been bound")); 1048 1049 /* 1050 * Release the original thread, since it is no longer used 1051 */ 1052 if (pru_flags & PRUS_HELDTD) { 1053 lwkt_rele(td); 1054 pru_flags &= ~PRUS_HELDTD; 1055 } 1056 /* 1057 * Free the dest address, since it is no longer needed 1058 */ 1059 if (pru_flags & PRUS_FREEADDR) { 1060 kfree(dstaddr, M_SONAME); 1061 pru_flags &= ~PRUS_FREEADDR; 1062 } 1063 1064 ui->ui_ulen = htons((u_short)len + sizeof(struct udphdr)); 1065 1066 /* 1067 * Set up checksum and output datagram. 1068 */ 1069 if (udpcksum) { 1070 ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr, 1071 htons((u_short)len + sizeof(struct udphdr) + IPPROTO_UDP)); 1072 m->m_pkthdr.csum_flags = CSUM_UDP; 1073 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1074 m->m_pkthdr.csum_thlen = sizeof(struct udphdr); 1075 } else { 1076 ui->ui_sum = 0; 1077 } 1078 ((struct ip *)ui)->ip_len = sizeof(struct udpiphdr) + len; 1079 ((struct ip *)ui)->ip_ttl = inp->inp_ip_ttl; /* XXX */ 1080 ((struct ip *)ui)->ip_tos = inp->inp_ip_tos; /* XXX */ 1081 udp_stat.udps_opackets++; 1082 1083 flags = IP_DEBUGROUTE | 1084 (inp->inp_socket->so_options & (SO_DONTROUTE | SO_BROADCAST)); 1085 if (pru_flags & PRUS_DONTROUTE) 1086 flags |= SO_DONTROUTE; 1087 1088 if (inp->inp_flags & INP_CONNECTED) { 1089 /* 1090 * For connected socket, this datagram has already 1091 * been in the correct netisr; no need to rehash. 1092 */ 1093 goto sendit; 1094 } 1095 1096 cpu = udp_addrcpu(ui->ui_dst.s_addr, ui->ui_dport, 1097 ui->ui_src.s_addr, ui->ui_sport); 1098 if (cpu != mycpuid) { 1099 struct mbuf *m_opt = NULL; 1100 struct netmsg_pru_send *smsg; 1101 struct lwkt_port *port = netisr_cpuport(cpu); 1102 1103 /* 1104 * Not on the CPU that matches this UDP datagram hash; 1105 * redispatch to the correct CPU to do the ip_output(). 1106 */ 1107 if (inp->inp_options != NULL) { 1108 /* 1109 * If there are ip options, then save a copy, 1110 * since accessing inp_options on other CPUs' 1111 * is not safe. 1112 * 1113 * XXX optimize this? 1114 */ 1115 m_opt = m_copym(inp->inp_options, 0, M_COPYALL, 1116 M_WAITOK); 1117 } 1118 if ((pru_flags & PRUS_NOREPLY) == 0) { 1119 /* 1120 * Change some parts of the original netmsg and 1121 * forward it to the target netisr. 1122 * 1123 * NOTE: so_port MUST NOT be checked in the target 1124 * netisr. 1125 */ 1126 smsg = &msg->send; 1127 smsg->nm_priv = flags; /* ip_output flags */ 1128 smsg->nm_m = m; 1129 smsg->nm_control = m_opt; /* XXX save ipopt */ 1130 smsg->base.lmsg.ms_flags |= MSGF_IGNSOPORT; 1131 smsg->base.nm_dispatch = udp_send_redispatch; 1132 lwkt_forwardmsg(port, &smsg->base.lmsg); 1133 } else { 1134 /* 1135 * Recreate the netmsg, since the original mbuf 1136 * could have been changed. And send it to the 1137 * target netisr. 1138 * 1139 * NOTE: so_port MUST NOT be checked in the target 1140 * netisr. 1141 */ 1142 smsg = &m->m_hdr.mh_sndmsg; 1143 netmsg_init(&smsg->base, so, &netisr_apanic_rport, 1144 MSGF_IGNSOPORT, udp_send_redispatch); 1145 smsg->nm_priv = flags; /* ip_output flags */ 1146 smsg->nm_flags = pru_flags; 1147 smsg->nm_m = m; 1148 smsg->nm_control = m_opt; /* XXX save ipopt */ 1149 lwkt_sendmsg(port, &smsg->base.lmsg); 1150 } 1151 1152 /* This UDP datagram is redispatched; done */ 1153 logudp(send_redisp, inp); 1154 return; 1155 } 1156 1157 sendit: 1158 logudp(send_ipout, inp); 1159 error = ip_output(m, inp->inp_options, &inp->inp_route, flags, 1160 inp->inp_moptions, inp); 1161 m = NULL; 1162 1163 release: 1164 if (m != NULL) 1165 m_freem(m); 1166 1167 if (pru_flags & PRUS_HELDTD) 1168 lwkt_rele(td); 1169 if (pru_flags & PRUS_FREEADDR) 1170 kfree(dstaddr, M_SONAME); 1171 if ((pru_flags & PRUS_NOREPLY) == 0) 1172 lwkt_replymsg(&msg->send.base.lmsg, error); 1173 1174 logudp(send_end, inp); 1175 } 1176 1177 u_long udp_sendspace = 9216; /* really max datagram size */ 1178 /* 40 1K datagrams */ 1179 SYSCTL_INT(_net_inet_udp, UDPCTL_MAXDGRAM, maxdgram, CTLFLAG_RW, 1180 &udp_sendspace, 0, "Maximum outgoing UDP datagram size"); 1181 1182 u_long udp_recvspace = 40 * (1024 + 1183 #ifdef INET6 1184 sizeof(struct sockaddr_in6) 1185 #else 1186 sizeof(struct sockaddr_in) 1187 #endif 1188 ); 1189 SYSCTL_INT(_net_inet_udp, UDPCTL_RECVSPACE, recvspace, CTLFLAG_RW, 1190 &udp_recvspace, 0, "Maximum incoming UDP datagram size"); 1191 1192 /* 1193 * This should never happen, since UDP socket does not support 1194 * connection acception (SO_ACCEPTCONN, i.e. listen(2)). 1195 */ 1196 static void 1197 udp_abort(netmsg_t msg __unused) 1198 { 1199 panic("udp_abort is called"); 1200 } 1201 1202 static int 1203 udp_preattach(struct socket *so, int proto __unused, struct pru_attach_info *ai) 1204 { 1205 return soreserve(so, udp_sendspace, udp_recvspace, ai->sb_rlimit); 1206 } 1207 1208 static void 1209 udp_attach(netmsg_t msg) 1210 { 1211 struct socket *so = msg->attach.base.nm_so; 1212 struct pru_attach_info *ai = msg->attach.nm_ai; 1213 struct inpcb *inp; 1214 int error; 1215 1216 KASSERT(so->so_pcb == NULL, ("udp socket attached")); 1217 1218 if (ai != NULL) { 1219 error = udp_preattach(so, 0 /* don't care */, ai); 1220 if (error) 1221 goto out; 1222 } else { 1223 /* Post attach; do nothing */ 1224 } 1225 1226 error = in_pcballoc(so, &udbinfo[mycpuid]); 1227 if (error) 1228 goto out; 1229 1230 inp = so->so_pcb; 1231 inp->inp_flags |= INP_DIRECT_DETACH; 1232 inp->inp_ip_ttl = ip_defttl; 1233 error = 0; 1234 out: 1235 lwkt_replymsg(&msg->attach.base.lmsg, error); 1236 } 1237 1238 static void 1239 udp_inswildcard_replymsg(netmsg_t msg) 1240 { 1241 lwkt_msg_t lmsg = &msg->lmsg; 1242 1243 if (lmsg->ms_flags & MSGF_UDP_SEND) { 1244 udp_send(msg); 1245 /* msg is replied by udp_send() */ 1246 } else { 1247 lwkt_replymsg(lmsg, lmsg->ms_error); 1248 } 1249 } 1250 1251 static void 1252 udp_soreuseport_dispatch(netmsg_t msg) 1253 { 1254 /* This inpcb has already been in the wildcard hash. */ 1255 in_pcblink_flags(msg->base.nm_so->so_pcb, &udbinfo[mycpuid], 0); 1256 udp_inswildcard_replymsg(msg); 1257 } 1258 1259 static void 1260 udp_sosetport(struct lwkt_msg *msg, lwkt_port_t port) 1261 { 1262 sosetport(((struct netmsg_base *)msg)->nm_so, port); 1263 } 1264 1265 static boolean_t 1266 udp_inswildcardhash_oncpu(struct inpcb *inp, struct netmsg_base *msg) 1267 { 1268 int cpu; 1269 1270 KASSERT(inp->inp_pcbinfo == &udbinfo[mycpuid], 1271 ("not on owner cpu")); 1272 1273 in_pcbinswildcardhash(inp); 1274 for (cpu = 0; cpu < ncpus2; ++cpu) { 1275 if (cpu == mycpuid) { 1276 /* 1277 * This inpcb has been inserted by the above 1278 * in_pcbinswildcardhash(). 1279 */ 1280 continue; 1281 } 1282 in_pcbinswildcardhash_oncpu(inp, &udbinfo[cpu]); 1283 } 1284 1285 if (inp->inp_socket->so_options & SO_REUSEPORT) { 1286 /* 1287 * For SO_REUSEPORT socket, redistribute it based on its 1288 * local group index. 1289 */ 1290 cpu = inp->inp_lgrpindex & ncpus2_mask; 1291 if (cpu != mycpuid) { 1292 struct lwkt_port *port = netisr_cpuport(cpu); 1293 lwkt_msg_t lmsg = &msg->lmsg; 1294 1295 /* 1296 * We are moving the protocol processing port the 1297 * socket is on, we have to unlink here and re-link 1298 * on the target cpu (this inpcb is still left in 1299 * the wildcard hash). 1300 */ 1301 in_pcbunlink_flags(inp, &udbinfo[mycpuid], 0); 1302 msg->nm_dispatch = udp_soreuseport_dispatch; 1303 1304 /* 1305 * See the related comment in tcp_usrreq.c 1306 * tcp_connect() 1307 */ 1308 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1309 lwkt_forwardmsg(port, lmsg); 1310 return TRUE; /* forwarded */ 1311 } 1312 } 1313 return FALSE; 1314 } 1315 1316 static void 1317 udp_inswildcardhash_dispatch(netmsg_t msg) 1318 { 1319 struct inpcb *inp = msg->base.nm_so->so_pcb; 1320 boolean_t forwarded; 1321 1322 KASSERT(inp->inp_lport != 0, ("local port not set yet")); 1323 KASSERT((ntohs(inp->inp_lport) & ncpus2_mask) == mycpuid, 1324 ("not target cpu")); 1325 1326 in_pcblink(inp, &udbinfo[mycpuid]); 1327 1328 forwarded = udp_inswildcardhash_oncpu(inp, &msg->base); 1329 if (forwarded) { 1330 /* The message is further forwarded, so we are done here. */ 1331 return; 1332 } 1333 udp_inswildcard_replymsg(msg); 1334 } 1335 1336 static boolean_t 1337 udp_inswildcardhash(struct inpcb *inp, struct netmsg_base *msg, int error) 1338 { 1339 lwkt_msg_t lmsg = &msg->lmsg; 1340 int cpu; 1341 1342 ASSERT_INP_NOTINHASH(inp); 1343 1344 /* This inpcb could no longer be directly detached */ 1345 inp->inp_flags &= ~INP_DIRECT_DETACH; 1346 1347 /* 1348 * Always clear the route cache, so we don't need to 1349 * worry about any owner CPU changes later. 1350 */ 1351 in_pcbresetroute(inp); 1352 1353 KASSERT(inp->inp_lport != 0, ("local port not set yet")); 1354 cpu = ntohs(inp->inp_lport) & ncpus2_mask; 1355 1356 lmsg->ms_error = error; 1357 if (cpu != mycpuid) { 1358 struct lwkt_port *port = netisr_cpuport(cpu); 1359 1360 /* 1361 * We are moving the protocol processing port the socket 1362 * is on, we have to unlink here and re-link on the 1363 * target cpu. 1364 */ 1365 in_pcbunlink(inp, &udbinfo[mycpuid]); 1366 msg->nm_dispatch = udp_inswildcardhash_dispatch; 1367 1368 /* See the related comment in tcp_usrreq.c tcp_connect() */ 1369 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1370 lwkt_forwardmsg(port, lmsg); 1371 return TRUE; /* forwarded */ 1372 } 1373 1374 return udp_inswildcardhash_oncpu(inp, msg); 1375 } 1376 1377 static void 1378 udp_bind(netmsg_t msg) 1379 { 1380 struct socket *so = msg->bind.base.nm_so; 1381 struct inpcb *inp; 1382 int error; 1383 1384 inp = so->so_pcb; 1385 if (inp) { 1386 struct sockaddr *nam = msg->bind.nm_nam; 1387 struct thread *td = msg->bind.nm_td; 1388 1389 error = in_pcbbind(inp, nam, td); 1390 if (error == 0) { 1391 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1392 boolean_t forwarded; 1393 1394 if (sin->sin_addr.s_addr != INADDR_ANY) 1395 inp->inp_flags |= INP_WASBOUND_NOTANY; 1396 1397 forwarded = udp_inswildcardhash(inp, 1398 &msg->bind.base, 0); 1399 if (forwarded) { 1400 /* 1401 * The message is further forwarded, so 1402 * we are done here. 1403 */ 1404 return; 1405 } 1406 } 1407 } else { 1408 error = EINVAL; 1409 } 1410 lwkt_replymsg(&msg->bind.base.lmsg, error); 1411 } 1412 1413 static int 1414 udp_preconnect(struct socket *so, const struct sockaddr *nam __unused, 1415 struct thread *td __unused) 1416 { 1417 sosetstate(so, SS_ISCONNECTED); /* XXX */ 1418 return 0; 1419 } 1420 1421 static void 1422 udp_connect(netmsg_t msg) 1423 { 1424 struct socket *so = msg->connect.base.nm_so; 1425 struct sockaddr *nam = msg->connect.nm_nam; 1426 struct thread *td = msg->connect.nm_td; 1427 struct inpcb *inp; 1428 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1429 struct sockaddr_in *if_sin; 1430 struct lwkt_port *port; 1431 int error; 1432 1433 KKASSERT(msg->connect.nm_m == NULL); 1434 1435 inp = so->so_pcb; 1436 if (inp == NULL) { 1437 error = EINVAL; 1438 goto out; 1439 } 1440 1441 if (msg->connect.nm_flags & PRUC_RECONNECT) { 1442 msg->connect.nm_flags &= ~PRUC_RECONNECT; 1443 in_pcblink(inp, &udbinfo[mycpuid]); 1444 } 1445 1446 if (inp->inp_faddr.s_addr != INADDR_ANY) { 1447 error = EISCONN; 1448 goto out; 1449 } 1450 error = 0; 1451 1452 /* 1453 * Bind if we have to 1454 */ 1455 if (inp->inp_lport == 0) { 1456 error = in_pcbbind(inp, NULL, td); 1457 if (error) 1458 goto out; 1459 } 1460 1461 /* 1462 * Calculate the correct protocol processing thread. The connect 1463 * operation must run there. 1464 */ 1465 error = in_pcbladdr(inp, nam, &if_sin, td); 1466 if (error) 1467 goto out; 1468 if (!prison_remote_ip(td, nam)) { 1469 error = EAFNOSUPPORT; /* IPv6 only jail */ 1470 goto out; 1471 } 1472 1473 port = udp_addrport(sin->sin_addr.s_addr, sin->sin_port, 1474 inp->inp_laddr.s_addr != INADDR_ANY ? 1475 inp->inp_laddr.s_addr : if_sin->sin_addr.s_addr, inp->inp_lport); 1476 if (port != &curthread->td_msgport) { 1477 lwkt_msg_t lmsg = &msg->connect.base.lmsg; 1478 int nm_flags = PRUC_RECONNECT; 1479 1480 /* 1481 * in_pcbladdr() may have allocated a route entry for us 1482 * on the current CPU, but we need a route entry on the 1483 * inpcb's owner CPU, so free it here. 1484 */ 1485 in_pcbresetroute(inp); 1486 1487 if (inp->inp_flags & INP_WILDCARD) { 1488 /* 1489 * Remove this inpcb from the wildcard hash before 1490 * the socket's msgport changes. 1491 */ 1492 udp_remwildcardhash(inp); 1493 } 1494 1495 if (so->so_orig_port == NULL) { 1496 /* 1497 * First time change protocol processing port. 1498 * Save the current port for synchronization upon 1499 * udp_detach. 1500 */ 1501 so->so_orig_port = &curthread->td_msgport; 1502 } else { 1503 /* 1504 * We have changed protocol processing port more 1505 * than once. We could not do direct detach 1506 * anymore, because we lose the track of the 1507 * original protocol processing ports to perform 1508 * synchronization upon udp_detach. This should 1509 * be rare though. 1510 */ 1511 inp->inp_flags &= ~INP_DIRECT_DETACH; 1512 } 1513 1514 /* 1515 * We are moving the protocol processing port the socket 1516 * is on, we have to unlink here and re-link on the 1517 * target cpu. 1518 */ 1519 in_pcbunlink(inp, &udbinfo[mycpuid]); 1520 msg->connect.nm_flags |= nm_flags; 1521 1522 /* See the related comment in tcp_usrreq.c tcp_connect() */ 1523 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1524 lwkt_forwardmsg(port, lmsg); 1525 /* msg invalid now */ 1526 return; 1527 } 1528 error = udp_connect_oncpu(inp, sin, if_sin); 1529 out: 1530 if (msg->connect.nm_flags & PRUC_HELDTD) 1531 lwkt_rele(td); 1532 if (error && (msg->connect.nm_flags & PRUC_ASYNC)) { 1533 so->so_error = error; 1534 soclrstate(so, SS_ISCONNECTED); /* XXX */ 1535 } 1536 if (error && inp != NULL && inp->inp_lport != 0 && 1537 (inp->inp_flags & INP_WILDCARD) == 0) { 1538 boolean_t forwarded; 1539 1540 /* Connect failed; put it to wildcard hash. */ 1541 forwarded = udp_inswildcardhash(inp, &msg->connect.base, 1542 error); 1543 if (forwarded) { 1544 /* 1545 * The message is further forwarded, so we are done 1546 * here. 1547 */ 1548 return; 1549 } 1550 } 1551 lwkt_replymsg(&msg->connect.base.lmsg, error); 1552 } 1553 1554 static void 1555 udp_remwildcardhash(struct inpcb *inp) 1556 { 1557 int cpu; 1558 1559 KASSERT(inp->inp_pcbinfo == &udbinfo[mycpuid], 1560 ("not on owner cpu")); 1561 1562 for (cpu = 0; cpu < ncpus2; ++cpu) { 1563 if (cpu == mycpuid) { 1564 /* 1565 * This inpcb will be removed by the later 1566 * in_pcbremwildcardhash(). 1567 */ 1568 continue; 1569 } 1570 in_pcbremwildcardhash_oncpu(inp, &udbinfo[cpu]); 1571 } 1572 in_pcbremwildcardhash(inp); 1573 } 1574 1575 static int 1576 udp_connect_oncpu(struct inpcb *inp, struct sockaddr_in *sin, 1577 struct sockaddr_in *if_sin) 1578 { 1579 struct socket *so = inp->inp_socket; 1580 struct inpcb *oinp; 1581 1582 oinp = in_pcblookup_hash(inp->inp_pcbinfo, 1583 sin->sin_addr, sin->sin_port, 1584 inp->inp_laddr.s_addr != INADDR_ANY ? 1585 inp->inp_laddr : if_sin->sin_addr, inp->inp_lport, FALSE, NULL); 1586 if (oinp != NULL) 1587 return EADDRINUSE; 1588 1589 /* 1590 * No more errors can occur, finish adjusting the socket 1591 * and change the processing port to reflect the connected 1592 * socket. Once set we can no longer safely mess with the 1593 * socket. 1594 */ 1595 1596 if (inp->inp_flags & INP_WILDCARD) 1597 udp_remwildcardhash(inp); 1598 1599 if (inp->inp_laddr.s_addr == INADDR_ANY) 1600 inp->inp_laddr = if_sin->sin_addr; 1601 inp->inp_faddr = sin->sin_addr; 1602 inp->inp_fport = sin->sin_port; 1603 in_pcbinsconnhash(inp); 1604 1605 soisconnected(so); 1606 1607 return 0; 1608 } 1609 1610 static void 1611 udp_detach2(struct socket *so) 1612 { 1613 in_pcbdetach(so->so_pcb); 1614 sodiscard(so); 1615 sofree(so); 1616 } 1617 1618 static void 1619 udp_detach_final_dispatch(netmsg_t msg) 1620 { 1621 udp_detach2(msg->base.nm_so); 1622 } 1623 1624 static void 1625 udp_detach_oncpu_dispatch(netmsg_t msg) 1626 { 1627 struct netmsg_base *clomsg = &msg->base; 1628 struct socket *so = clomsg->nm_so; 1629 struct inpcb *inp = so->so_pcb; 1630 struct thread *td = curthread; 1631 int nextcpu, cpuid = mycpuid; 1632 1633 KASSERT(td->td_type == TD_TYPE_NETISR, ("not in netisr")); 1634 1635 if (inp->inp_flags & INP_WILDCARD) { 1636 /* 1637 * This inp will be removed on the inp's 1638 * owner CPU later, so don't do it now. 1639 */ 1640 if (&td->td_msgport != so->so_port) 1641 in_pcbremwildcardhash_oncpu(inp, &udbinfo[cpuid]); 1642 } 1643 1644 if (cpuid == 0) { 1645 /* 1646 * Free and clear multicast socket option, 1647 * which is only accessed in netisr0. 1648 */ 1649 ip_freemoptions(inp->inp_moptions); 1650 inp->inp_moptions = NULL; 1651 } 1652 1653 nextcpu = cpuid + 1; 1654 if (nextcpu < ncpus2) { 1655 lwkt_forwardmsg(netisr_cpuport(nextcpu), &clomsg->lmsg); 1656 } else { 1657 /* 1658 * No one could see this inpcb now; destroy this 1659 * inpcb in its owner netisr. 1660 */ 1661 netmsg_init(clomsg, so, &netisr_apanic_rport, 0, 1662 udp_detach_final_dispatch); 1663 lwkt_sendmsg(so->so_port, &clomsg->lmsg); 1664 } 1665 } 1666 1667 static void 1668 udp_detach_syncorig_dispatch(netmsg_t msg) 1669 { 1670 struct netmsg_base *clomsg = &msg->base; 1671 struct socket *so = clomsg->nm_so; 1672 1673 /* 1674 * Original protocol processing port is synchronized; 1675 * destroy this inpcb in its owner netisr. 1676 */ 1677 netmsg_init(clomsg, so, &netisr_apanic_rport, 0, 1678 udp_detach_final_dispatch); 1679 lwkt_sendmsg(so->so_port, &clomsg->lmsg); 1680 } 1681 1682 static void 1683 udp_detach(netmsg_t msg) 1684 { 1685 struct socket *so = msg->detach.base.nm_so; 1686 struct netmsg_base *clomsg; 1687 struct inpcb *inp; 1688 1689 inp = so->so_pcb; 1690 if (inp == NULL) { 1691 lwkt_replymsg(&msg->detach.base.lmsg, EINVAL); 1692 return; 1693 } 1694 1695 /* 1696 * Reply EJUSTRETURN ASAP, we will call sodiscard() and 1697 * sofree() later. 1698 */ 1699 lwkt_replymsg(&msg->detach.base.lmsg, EJUSTRETURN); 1700 1701 if (ncpus2 == 1) { 1702 /* Only one CPU, detach the inpcb directly. */ 1703 udp_detach2(so); 1704 return; 1705 } 1706 1707 /* 1708 * Remove this inpcb from the inpcb list first, so that 1709 * no one could find this inpcb from the inpcb list. 1710 */ 1711 in_pcbofflist(inp); 1712 1713 if (inp->inp_flags & INP_DIRECT_DETACH) { 1714 /* 1715 * Direct detaching is allowed 1716 */ 1717 KASSERT((inp->inp_flags & INP_WILDCARD) == 0, 1718 ("in the wildcardhash")); 1719 KASSERT(inp->inp_moptions == NULL, ("has mcast options")); 1720 if (so->so_orig_port == NULL) { 1721 udp_detach2(so); 1722 } else { 1723 /* 1724 * Protocol processing port changed once, so 1725 * we need to make sure that there are nothing 1726 * left on the original protocol processing 1727 * port before we destroy this socket and inpcb. 1728 * This is more lightweight than going through 1729 * all UDP processing netisrs. 1730 */ 1731 clomsg = &so->so_clomsg; 1732 netmsg_init(clomsg, so, &netisr_apanic_rport, 1733 MSGF_IGNSOPORT, udp_detach_syncorig_dispatch); 1734 lwkt_sendmsg(so->so_orig_port, &clomsg->lmsg); 1735 } 1736 return; 1737 } 1738 1739 /* 1740 * Go through netisrs which process UDP to make sure 1741 * no one could find this inpcb anymore. 1742 */ 1743 clomsg = &so->so_clomsg; 1744 netmsg_init(clomsg, so, &netisr_apanic_rport, MSGF_IGNSOPORT, 1745 udp_detach_oncpu_dispatch); 1746 lwkt_sendmsg(netisr_cpuport(0), &clomsg->lmsg); 1747 } 1748 1749 static void 1750 udp_disconnect(netmsg_t msg) 1751 { 1752 struct socket *so = msg->disconnect.base.nm_so; 1753 struct inpcb *inp; 1754 boolean_t forwarded; 1755 int error = 0; 1756 1757 inp = so->so_pcb; 1758 if (inp == NULL) { 1759 error = EINVAL; 1760 goto out; 1761 } 1762 if (inp->inp_faddr.s_addr == INADDR_ANY) { 1763 error = ENOTCONN; 1764 goto out; 1765 } 1766 1767 soclrstate(so, SS_ISCONNECTED); /* XXX */ 1768 1769 in_pcbdisconnect(inp); 1770 1771 /* 1772 * Follow traditional BSD behavior and retain the local port 1773 * binding. But, fix the old misbehavior of overwriting any 1774 * previously bound local address. 1775 */ 1776 if (!(inp->inp_flags & INP_WASBOUND_NOTANY)) 1777 inp->inp_laddr.s_addr = INADDR_ANY; 1778 1779 if (so->so_state & SS_ISCLOSING) { 1780 /* 1781 * If this socket is being closed, there is no need 1782 * to put this socket back into wildcard hash table. 1783 */ 1784 error = 0; 1785 goto out; 1786 } 1787 1788 forwarded = udp_inswildcardhash(inp, &msg->disconnect.base, 0); 1789 if (forwarded) { 1790 /* 1791 * The message is further forwarded, so we are done 1792 * here. 1793 */ 1794 return; 1795 } 1796 out: 1797 lwkt_replymsg(&msg->disconnect.base.lmsg, error); 1798 } 1799 1800 void 1801 udp_shutdown(netmsg_t msg) 1802 { 1803 struct socket *so = msg->shutdown.base.nm_so; 1804 struct inpcb *inp; 1805 int error; 1806 1807 inp = so->so_pcb; 1808 if (inp) { 1809 socantsendmore(so); 1810 error = 0; 1811 } else { 1812 error = EINVAL; 1813 } 1814 lwkt_replymsg(&msg->shutdown.base.lmsg, error); 1815 } 1816 1817 struct pr_usrreqs udp_usrreqs = { 1818 .pru_abort = udp_abort, 1819 .pru_accept = pr_generic_notsupp, 1820 .pru_attach = udp_attach, 1821 .pru_bind = udp_bind, 1822 .pru_connect = udp_connect, 1823 .pru_connect2 = pr_generic_notsupp, 1824 .pru_control = in_control_dispatch, 1825 .pru_detach = udp_detach, 1826 .pru_disconnect = udp_disconnect, 1827 .pru_listen = pr_generic_notsupp, 1828 .pru_peeraddr = in_setpeeraddr_dispatch, 1829 .pru_rcvd = pr_generic_notsupp, 1830 .pru_rcvoob = pr_generic_notsupp, 1831 .pru_send = udp_send, 1832 .pru_sense = pru_sense_null, 1833 .pru_shutdown = udp_shutdown, 1834 .pru_sockaddr = in_setsockaddr_dispatch, 1835 .pru_sosend = sosendudp, 1836 .pru_soreceive = soreceive, 1837 .pru_preconnect = udp_preconnect, 1838 .pru_preattach = udp_preattach 1839 }; 1840