1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)udp_usrreq.c 8.6 (Berkeley) 5/23/95 63 * $FreeBSD: src/sys/netinet/udp_usrreq.c,v 1.64.2.18 2003/01/24 05:11:34 sam Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/kernel.h> 72 #include <sys/malloc.h> 73 #include <sys/mbuf.h> 74 #include <sys/domain.h> 75 #include <sys/proc.h> 76 #include <sys/priv.h> 77 #include <sys/protosw.h> 78 #include <sys/socket.h> 79 #include <sys/socketvar.h> 80 #include <sys/sysctl.h> 81 #include <sys/syslog.h> 82 #include <sys/in_cksum.h> 83 #include <sys/ktr.h> 84 85 #include <sys/thread2.h> 86 #include <sys/socketvar2.h> 87 #include <sys/serialize.h> 88 89 #include <machine/stdarg.h> 90 91 #include <net/if.h> 92 #include <net/route.h> 93 #include <net/netmsg2.h> 94 #include <net/netisr2.h> 95 96 #include <netinet/in.h> 97 #include <netinet/in_systm.h> 98 #include <netinet/ip.h> 99 #ifdef INET6 100 #include <netinet/ip6.h> 101 #endif 102 #include <netinet/in_pcb.h> 103 #include <netinet/in_var.h> 104 #include <netinet/ip_var.h> 105 #ifdef INET6 106 #include <netinet6/ip6_var.h> 107 #endif 108 #include <netinet/ip_icmp.h> 109 #include <netinet/icmp_var.h> 110 #include <netinet/udp.h> 111 #include <netinet/udp_var.h> 112 113 #ifdef FAST_IPSEC 114 #include <netproto/ipsec/ipsec.h> 115 #endif 116 117 #ifdef IPSEC 118 #include <netinet6/ipsec.h> 119 #endif 120 121 #define MSGF_UDP_SEND MSGF_PROTO1 122 123 #define INP_DIRECT_DETACH INP_FLAG_PROTO2 124 125 #define UDP_KTR_STRING "inp=%p" 126 #define UDP_KTR_ARGS struct inpcb *inp 127 128 #ifndef KTR_UDP 129 #define KTR_UDP KTR_ALL 130 #endif 131 132 KTR_INFO_MASTER(udp); 133 KTR_INFO(KTR_UDP, udp, send_beg, 0, UDP_KTR_STRING, UDP_KTR_ARGS); 134 KTR_INFO(KTR_UDP, udp, send_end, 1, UDP_KTR_STRING, UDP_KTR_ARGS); 135 KTR_INFO(KTR_UDP, udp, send_ipout, 2, UDP_KTR_STRING, UDP_KTR_ARGS); 136 KTR_INFO(KTR_UDP, udp, redisp_ipout_beg, 3, UDP_KTR_STRING, UDP_KTR_ARGS); 137 KTR_INFO(KTR_UDP, udp, redisp_ipout_end, 4, UDP_KTR_STRING, UDP_KTR_ARGS); 138 KTR_INFO(KTR_UDP, udp, send_redisp, 5, UDP_KTR_STRING, UDP_KTR_ARGS); 139 KTR_INFO(KTR_UDP, udp, send_inswildcard, 6, UDP_KTR_STRING, UDP_KTR_ARGS); 140 141 #define logudp(name, inp) KTR_LOG(udp_##name, inp) 142 143 /* 144 * UDP protocol implementation. 145 * Per RFC 768, August, 1980. 146 */ 147 #ifndef COMPAT_42 148 static int udpcksum = 1; 149 #else 150 static int udpcksum = 0; /* XXX */ 151 #endif 152 SYSCTL_INT(_net_inet_udp, UDPCTL_CHECKSUM, checksum, CTLFLAG_RW, 153 &udpcksum, 0, "Enable checksumming of UDP packets"); 154 155 int log_in_vain = 0; 156 SYSCTL_INT(_net_inet_udp, OID_AUTO, log_in_vain, CTLFLAG_RW, 157 &log_in_vain, 0, "Log all incoming UDP packets"); 158 159 static int blackhole = 0; 160 SYSCTL_INT(_net_inet_udp, OID_AUTO, blackhole, CTLFLAG_RW, 161 &blackhole, 0, "Do not send port unreachables for refused connects"); 162 163 static int strict_mcast_mship = 1; 164 SYSCTL_INT(_net_inet_udp, OID_AUTO, strict_mcast_mship, CTLFLAG_RW, 165 &strict_mcast_mship, 0, "Only send multicast to member sockets"); 166 167 int udp_sosend_async = 1; 168 SYSCTL_INT(_net_inet_udp, OID_AUTO, sosend_async, CTLFLAG_RW, 169 &udp_sosend_async, 0, "UDP asynchronized pru_send"); 170 171 int udp_sosend_prepend = 1; 172 SYSCTL_INT(_net_inet_udp, OID_AUTO, sosend_prepend, CTLFLAG_RW, 173 &udp_sosend_prepend, 0, 174 "Prepend enough space for proto and link header in pru_send"); 175 176 static int udp_reuseport_ext = 1; 177 SYSCTL_INT(_net_inet_udp, OID_AUTO, reuseport_ext, CTLFLAG_RW, 178 &udp_reuseport_ext, 0, "SO_REUSEPORT extension"); 179 180 struct inpcbinfo udbinfo[MAXCPU]; 181 182 #ifndef UDBHASHSIZE 183 #define UDBHASHSIZE 16 184 #endif 185 186 struct udpstat udpstat_percpu[MAXCPU] __cachealign; 187 188 static void udp_append(struct inpcb *last, struct ip *ip, 189 struct mbuf *n, int off, struct sockaddr_in *udp_in); 190 191 static int udp_connect_oncpu(struct inpcb *inp, struct sockaddr_in *sin, 192 struct sockaddr_in *if_sin); 193 194 static boolean_t udp_inswildcardhash(struct inpcb *inp, 195 struct netmsg_base *msg, int error); 196 static void udp_remwildcardhash(struct inpcb *inp); 197 198 void 199 udp_init(void) 200 { 201 struct inpcbportinfo *portinfo; 202 int cpu; 203 204 portinfo = kmalloc_cachealign(sizeof(*portinfo) * ncpus2, M_PCB, 205 M_WAITOK); 206 207 for (cpu = 0; cpu < ncpus2; cpu++) { 208 struct inpcbinfo *uicb = &udbinfo[cpu]; 209 210 /* 211 * NOTE: 212 * UDP pcb list, wildcard hash table and localgroup hash 213 * table are shared. 214 */ 215 in_pcbinfo_init(uicb, cpu, TRUE); 216 uicb->hashbase = hashinit(UDBHASHSIZE, M_PCB, &uicb->hashmask); 217 218 in_pcbportinfo_init(&portinfo[cpu], UDBHASHSIZE, TRUE, cpu); 219 uicb->portinfo = portinfo; 220 uicb->portinfo_mask = ncpus2_mask; 221 222 uicb->wildcardhashbase = hashinit(UDBHASHSIZE, M_PCB, 223 &uicb->wildcardhashmask); 224 uicb->localgrphashbase = hashinit(UDBHASHSIZE, M_PCB, 225 &uicb->localgrphashmask); 226 227 uicb->ipi_size = sizeof(struct inpcb); 228 } 229 230 /* 231 * Initialize UDP statistics counters for each CPU. 232 */ 233 for (cpu = 0; cpu < ncpus; ++cpu) 234 bzero(&udpstat_percpu[cpu], sizeof(struct udpstat)); 235 } 236 237 static int 238 sysctl_udpstat(SYSCTL_HANDLER_ARGS) 239 { 240 int cpu, error = 0; 241 242 for (cpu = 0; cpu < ncpus; ++cpu) { 243 if ((error = SYSCTL_OUT(req, &udpstat_percpu[cpu], 244 sizeof(struct udpstat)))) 245 break; 246 if ((error = SYSCTL_IN(req, &udpstat_percpu[cpu], 247 sizeof(struct udpstat)))) 248 break; 249 } 250 251 return (error); 252 } 253 SYSCTL_PROC(_net_inet_udp, UDPCTL_STATS, stats, (CTLTYPE_OPAQUE | CTLFLAG_RW), 254 0, 0, sysctl_udpstat, "S,udpstat", "UDP statistics"); 255 256 void 257 udp_ctloutput(netmsg_t msg) 258 { 259 struct socket *so = msg->base.nm_so; 260 struct sockopt *sopt = msg->ctloutput.nm_sopt; 261 struct inpcb *inp = so->so_pcb; 262 263 if (sopt->sopt_level == IPPROTO_IP) { 264 switch (sopt->sopt_name) { 265 case IP_MULTICAST_IF: 266 case IP_MULTICAST_VIF: 267 case IP_MULTICAST_TTL: 268 case IP_MULTICAST_LOOP: 269 case IP_ADD_MEMBERSHIP: 270 case IP_DROP_MEMBERSHIP: 271 if (&curthread->td_msgport != netisr_cpuport(0)) { 272 /* 273 * This pr_ctloutput msg will be forwarded 274 * to netisr0 to run; we can't do direct 275 * detaching anymore. 276 */ 277 inp->inp_flags &= ~INP_DIRECT_DETACH; 278 } 279 break; 280 } 281 } 282 return ip_ctloutput(msg); 283 } 284 285 /* 286 * Check multicast packets to make sure they are only sent to sockets with 287 * multicast memberships for the packet's destination address and arrival 288 * interface. Multicast packets to multicast-unaware sockets are also 289 * disallowed. 290 * 291 * Returns 0 if the packet is acceptable, -1 if it is not. 292 */ 293 static __inline int 294 check_multicast_membership(const struct ip *ip, const struct inpcb *inp, 295 const struct mbuf *m) 296 { 297 const struct ip_moptions *mopt; 298 int mshipno; 299 300 if (strict_mcast_mship == 0 || 301 !IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) { 302 return (0); 303 } 304 305 KASSERT(&curthread->td_msgport == netisr_cpuport(0), 306 ("multicast input not in netisr0")); 307 308 mopt = inp->inp_moptions; 309 if (mopt == NULL) 310 return (-1); 311 for (mshipno = 0; mshipno < mopt->imo_num_memberships; ++mshipno) { 312 const struct in_multi *maddr = mopt->imo_membership[mshipno]; 313 314 if (ip->ip_dst.s_addr == maddr->inm_addr.s_addr && 315 m->m_pkthdr.rcvif == maddr->inm_ifp) { 316 return (0); 317 } 318 } 319 return (-1); 320 } 321 322 struct udp_mcast_arg { 323 struct inpcb *inp; 324 struct inpcb *last; 325 struct ip *ip; 326 struct mbuf *m; 327 int iphlen; 328 struct sockaddr_in *udp_in; 329 }; 330 331 static int 332 udp_mcast_input(struct udp_mcast_arg *arg) 333 { 334 struct inpcb *inp = arg->inp; 335 struct inpcb *last = arg->last; 336 struct ip *ip = arg->ip; 337 struct mbuf *m = arg->m; 338 339 if (check_multicast_membership(ip, inp, m) < 0) 340 return ERESTART; /* caller continue */ 341 342 if (last != NULL) { 343 struct mbuf *n; 344 345 #ifdef IPSEC 346 /* check AH/ESP integrity. */ 347 if (ipsec4_in_reject_so(m, last->inp_socket)) 348 ipsecstat.in_polvio++; 349 /* do not inject data to pcb */ 350 else 351 #endif /*IPSEC*/ 352 #ifdef FAST_IPSEC 353 /* check AH/ESP integrity. */ 354 if (ipsec4_in_reject(m, last)) 355 ; 356 else 357 #endif /*FAST_IPSEC*/ 358 if ((n = m_copypacket(m, MB_DONTWAIT)) != NULL) 359 udp_append(last, ip, n, 360 arg->iphlen + sizeof(struct udphdr), 361 arg->udp_in); 362 } 363 arg->last = last = inp; 364 365 /* 366 * Don't look for additional matches if this one does 367 * not have either the SO_REUSEPORT or SO_REUSEADDR 368 * socket options set. This heuristic avoids searching 369 * through all pcbs in the common case of a non-shared 370 * port. It * assumes that an application will never 371 * clear these options after setting them. 372 */ 373 if (!(last->inp_socket->so_options & 374 (SO_REUSEPORT | SO_REUSEADDR))) 375 return EJUSTRETURN; /* caller stop */ 376 return 0; 377 } 378 379 int 380 udp_input(struct mbuf **mp, int *offp, int proto) 381 { 382 struct sockaddr_in udp_in = { sizeof udp_in, AF_INET }; 383 int iphlen; 384 struct ip *ip; 385 struct udphdr *uh; 386 struct inpcb *inp; 387 struct mbuf *m; 388 struct mbuf *opts = NULL; 389 int len, off; 390 struct ip save_ip; 391 struct inpcbinfo *pcbinfo = &udbinfo[mycpuid]; 392 393 off = *offp; 394 m = *mp; 395 *mp = NULL; 396 397 iphlen = off; 398 udp_stat.udps_ipackets++; 399 400 /* 401 * Strip IP options, if any; should skip this, 402 * make available to user, and use on returned packets, 403 * but we don't yet have a way to check the checksum 404 * with options still present. 405 */ 406 if (iphlen > sizeof(struct ip)) { 407 ip_stripoptions(m); 408 iphlen = sizeof(struct ip); 409 } 410 411 /* 412 * IP and UDP headers are together in first mbuf. 413 * Already checked and pulled up in ip_demux(). 414 */ 415 KASSERT(m->m_len >= iphlen + sizeof(struct udphdr), 416 ("UDP header not in one mbuf")); 417 418 ip = mtod(m, struct ip *); 419 uh = (struct udphdr *)((caddr_t)ip + iphlen); 420 421 /* destination port of 0 is illegal, based on RFC768. */ 422 if (uh->uh_dport == 0) 423 goto bad; 424 425 /* 426 * Make mbuf data length reflect UDP length. 427 * If not enough data to reflect UDP length, drop. 428 */ 429 len = ntohs((u_short)uh->uh_ulen); 430 if (ip->ip_len != len) { 431 if (len > ip->ip_len || len < sizeof(struct udphdr)) { 432 udp_stat.udps_badlen++; 433 goto bad; 434 } 435 m_adj(m, len - ip->ip_len); 436 /* ip->ip_len = len; */ 437 } 438 /* 439 * Save a copy of the IP header in case we want restore it 440 * for sending an ICMP error message in response. 441 */ 442 save_ip = *ip; 443 444 /* 445 * Checksum extended UDP header and data. 446 */ 447 if (uh->uh_sum) { 448 if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) { 449 if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) 450 uh->uh_sum = m->m_pkthdr.csum_data; 451 else 452 uh->uh_sum = in_pseudo(ip->ip_src.s_addr, 453 ip->ip_dst.s_addr, htonl((u_short)len + 454 m->m_pkthdr.csum_data + IPPROTO_UDP)); 455 uh->uh_sum ^= 0xffff; 456 } else { 457 char b[9]; 458 459 bcopy(((struct ipovly *)ip)->ih_x1, b, 9); 460 bzero(((struct ipovly *)ip)->ih_x1, 9); 461 ((struct ipovly *)ip)->ih_len = uh->uh_ulen; 462 uh->uh_sum = in_cksum(m, len + sizeof(struct ip)); 463 bcopy(b, ((struct ipovly *)ip)->ih_x1, 9); 464 } 465 if (uh->uh_sum) { 466 udp_stat.udps_badsum++; 467 m_freem(m); 468 return(IPPROTO_DONE); 469 } 470 } else 471 udp_stat.udps_nosum++; 472 473 if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) || 474 in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif)) { 475 struct inpcbhead *connhead; 476 struct inpcontainer *ic, *ic_marker; 477 struct inpcontainerhead *ichead; 478 struct udp_mcast_arg arg; 479 struct inpcb *last; 480 int error; 481 482 /* 483 * Deliver a multicast or broadcast datagram to *all* sockets 484 * for which the local and remote addresses and ports match 485 * those of the incoming datagram. This allows more than 486 * one process to receive multi/broadcasts on the same port. 487 * (This really ought to be done for unicast datagrams as 488 * well, but that would cause problems with existing 489 * applications that open both address-specific sockets and 490 * a wildcard socket listening to the same port -- they would 491 * end up receiving duplicates of every unicast datagram. 492 * Those applications open the multiple sockets to overcome an 493 * inadequacy of the UDP socket interface, but for backwards 494 * compatibility we avoid the problem here rather than 495 * fixing the interface. Maybe 4.5BSD will remedy this?) 496 */ 497 498 /* 499 * Construct sockaddr format source address. 500 */ 501 udp_in.sin_port = uh->uh_sport; 502 udp_in.sin_addr = ip->ip_src; 503 arg.udp_in = &udp_in; 504 /* 505 * Locate pcb(s) for datagram. 506 * (Algorithm copied from raw_intr().) 507 */ 508 last = NULL; 509 arg.iphlen = iphlen; 510 511 connhead = &pcbinfo->hashbase[ 512 INP_PCBCONNHASH(ip->ip_src.s_addr, uh->uh_sport, 513 ip->ip_dst.s_addr, uh->uh_dport, pcbinfo->hashmask)]; 514 LIST_FOREACH(inp, connhead, inp_hash) { 515 #ifdef INET6 516 if (!INP_ISIPV4(inp)) 517 continue; 518 #endif 519 if (!in_hosteq(inp->inp_faddr, ip->ip_src) || 520 !in_hosteq(inp->inp_laddr, ip->ip_dst) || 521 inp->inp_fport != uh->uh_sport || 522 inp->inp_lport != uh->uh_dport) 523 continue; 524 525 arg.inp = inp; 526 arg.last = last; 527 arg.ip = ip; 528 arg.m = m; 529 530 error = udp_mcast_input(&arg); 531 if (error == ERESTART) 532 continue; 533 last = arg.last; 534 535 if (error == EJUSTRETURN) 536 goto done; 537 } 538 539 ichead = &pcbinfo->wildcardhashbase[ 540 INP_PCBWILDCARDHASH(uh->uh_dport, 541 pcbinfo->wildcardhashmask)]; 542 ic_marker = in_pcbcontainer_marker(mycpuid); 543 544 GET_PCBINFO_TOKEN(pcbinfo); 545 LIST_INSERT_HEAD(ichead, ic_marker, ic_list); 546 while ((ic = LIST_NEXT(ic_marker, ic_list)) != NULL) { 547 LIST_REMOVE(ic_marker, ic_list); 548 LIST_INSERT_AFTER(ic, ic_marker, ic_list); 549 550 inp = ic->ic_inp; 551 if (inp->inp_flags & INP_PLACEMARKER) 552 continue; 553 #ifdef INET6 554 if (!INP_ISIPV4(inp)) 555 continue; 556 #endif 557 if (inp->inp_lport != uh->uh_dport) 558 continue; 559 if (inp->inp_laddr.s_addr != INADDR_ANY && 560 inp->inp_laddr.s_addr != ip->ip_dst.s_addr) 561 continue; 562 563 arg.inp = inp; 564 arg.last = last; 565 arg.ip = ip; 566 arg.m = m; 567 568 error = udp_mcast_input(&arg); 569 if (error == ERESTART) 570 continue; 571 last = arg.last; 572 573 if (error == EJUSTRETURN) 574 break; 575 } 576 LIST_REMOVE(ic_marker, ic_list); 577 REL_PCBINFO_TOKEN(pcbinfo); 578 done: 579 if (last == NULL) { 580 /* 581 * No matching pcb found; discard datagram. 582 * (No need to send an ICMP Port Unreachable 583 * for a broadcast or multicast datgram.) 584 */ 585 udp_stat.udps_noportbcast++; 586 goto bad; 587 } 588 #ifdef IPSEC 589 /* check AH/ESP integrity. */ 590 if (ipsec4_in_reject_so(m, last->inp_socket)) { 591 ipsecstat.in_polvio++; 592 goto bad; 593 } 594 #endif /*IPSEC*/ 595 #ifdef FAST_IPSEC 596 /* check AH/ESP integrity. */ 597 if (ipsec4_in_reject(m, last)) 598 goto bad; 599 #endif /*FAST_IPSEC*/ 600 udp_append(last, ip, m, iphlen + sizeof(struct udphdr), 601 &udp_in); 602 return(IPPROTO_DONE); 603 } 604 /* 605 * Locate pcb for datagram. 606 */ 607 inp = in_pcblookup_pkthash(pcbinfo, ip->ip_src, uh->uh_sport, 608 ip->ip_dst, uh->uh_dport, TRUE, m->m_pkthdr.rcvif, 609 udp_reuseport_ext ? m : NULL); 610 if (inp == NULL) { 611 if (log_in_vain) { 612 char buf[sizeof "aaa.bbb.ccc.ddd"]; 613 614 strcpy(buf, inet_ntoa(ip->ip_dst)); 615 log(LOG_INFO, 616 "Connection attempt to UDP %s:%d from %s:%d\n", 617 buf, ntohs(uh->uh_dport), inet_ntoa(ip->ip_src), 618 ntohs(uh->uh_sport)); 619 } 620 udp_stat.udps_noport++; 621 if (m->m_flags & (M_BCAST | M_MCAST)) { 622 udp_stat.udps_noportbcast++; 623 goto bad; 624 } 625 if (blackhole) 626 goto bad; 627 #ifdef ICMP_BANDLIM 628 if (badport_bandlim(BANDLIM_ICMP_UNREACH) < 0) 629 goto bad; 630 #endif 631 *ip = save_ip; 632 ip->ip_len += iphlen; 633 icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_PORT, 0, 0); 634 return(IPPROTO_DONE); 635 } 636 KASSERT(INP_ISIPV4(inp), ("not inet inpcb")); 637 #ifdef IPSEC 638 if (ipsec4_in_reject_so(m, inp->inp_socket)) { 639 ipsecstat.in_polvio++; 640 goto bad; 641 } 642 #endif /*IPSEC*/ 643 #ifdef FAST_IPSEC 644 if (ipsec4_in_reject(m, inp)) 645 goto bad; 646 #endif /*FAST_IPSEC*/ 647 /* 648 * Check the minimum TTL for socket. 649 */ 650 if (ip->ip_ttl < inp->inp_ip_minttl) 651 goto bad; 652 653 /* 654 * Construct sockaddr format source address. 655 * Stuff source address and datagram in user buffer. 656 */ 657 udp_in.sin_port = uh->uh_sport; 658 udp_in.sin_addr = ip->ip_src; 659 if ((inp->inp_flags & INP_CONTROLOPTS) || 660 (inp->inp_socket->so_options & SO_TIMESTAMP)) 661 ip_savecontrol(inp, &opts, ip, m); 662 m_adj(m, iphlen + sizeof(struct udphdr)); 663 664 lwkt_gettoken(&inp->inp_socket->so_rcv.ssb_token); 665 if (ssb_appendaddr(&inp->inp_socket->so_rcv, 666 (struct sockaddr *)&udp_in, m, opts) == 0) { 667 lwkt_reltoken(&inp->inp_socket->so_rcv.ssb_token); 668 udp_stat.udps_fullsock++; 669 goto bad; 670 } 671 lwkt_reltoken(&inp->inp_socket->so_rcv.ssb_token); 672 sorwakeup(inp->inp_socket); 673 return(IPPROTO_DONE); 674 bad: 675 m_freem(m); 676 if (opts) 677 m_freem(opts); 678 return(IPPROTO_DONE); 679 } 680 681 /* 682 * subroutine of udp_input(), mainly for source code readability. 683 * caller must properly init udp_ip6 and udp_in6 beforehand. 684 */ 685 static void 686 udp_append(struct inpcb *last, struct ip *ip, struct mbuf *n, int off, 687 struct sockaddr_in *udp_in) 688 { 689 struct mbuf *opts = NULL; 690 int ret; 691 692 KASSERT(INP_ISIPV4(last), ("not inet inpcb")); 693 694 if (last->inp_flags & INP_CONTROLOPTS || 695 last->inp_socket->so_options & SO_TIMESTAMP) 696 ip_savecontrol(last, &opts, ip, n); 697 m_adj(n, off); 698 699 lwkt_gettoken(&last->inp_socket->so_rcv.ssb_token); 700 ret = ssb_appendaddr(&last->inp_socket->so_rcv, 701 (struct sockaddr *)udp_in, n, opts); 702 lwkt_reltoken(&last->inp_socket->so_rcv.ssb_token); 703 if (ret == 0) { 704 m_freem(n); 705 if (opts) 706 m_freem(opts); 707 udp_stat.udps_fullsock++; 708 } else { 709 sorwakeup(last->inp_socket); 710 } 711 } 712 713 /* 714 * Notify a udp user of an asynchronous error; 715 * just wake up so that he can collect error status. 716 */ 717 void 718 udp_notify(struct inpcb *inp, int error) 719 { 720 inp->inp_socket->so_error = error; 721 sorwakeup(inp->inp_socket); 722 sowwakeup(inp->inp_socket); 723 } 724 725 struct netmsg_udp_notify { 726 struct netmsg_base base; 727 inp_notify_t nm_notify; 728 struct in_addr nm_faddr; 729 int nm_arg; 730 }; 731 732 static void 733 udp_notifyall_oncpu(netmsg_t msg) 734 { 735 struct netmsg_udp_notify *nm = (struct netmsg_udp_notify *)msg; 736 int nextcpu, cpu = mycpuid; 737 738 in_pcbnotifyall(&udbinfo[cpu], nm->nm_faddr, nm->nm_arg, nm->nm_notify); 739 740 nextcpu = cpu + 1; 741 if (nextcpu < ncpus2) 742 lwkt_forwardmsg(netisr_cpuport(nextcpu), &nm->base.lmsg); 743 else 744 lwkt_replymsg(&nm->base.lmsg, 0); 745 } 746 747 inp_notify_t 748 udp_get_inpnotify(int cmd, const struct sockaddr *sa, 749 struct ip **ip0, int *cpuid) 750 { 751 struct in_addr faddr; 752 struct ip *ip = *ip0; 753 inp_notify_t notify = udp_notify; 754 755 faddr = ((const struct sockaddr_in *)sa)->sin_addr; 756 if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY) 757 return NULL; 758 759 if (PRC_IS_REDIRECT(cmd)) { 760 ip = NULL; 761 notify = in_rtchange; 762 } else if (cmd == PRC_HOSTDEAD) { 763 ip = NULL; 764 } else if ((unsigned)cmd >= PRC_NCMDS || inetctlerrmap[cmd] == 0) { 765 return NULL; 766 } 767 768 if (cpuid != NULL) { 769 if (ip == NULL) { 770 /* Go through all CPUs */ 771 *cpuid = ncpus; 772 } else { 773 const struct udphdr *uh; 774 775 uh = (const struct udphdr *) 776 ((caddr_t)ip + (ip->ip_hl << 2)); 777 *cpuid = udp_addrcpu(faddr.s_addr, uh->uh_dport, 778 ip->ip_src.s_addr, uh->uh_sport); 779 } 780 } 781 782 *ip0 = ip; 783 return notify; 784 } 785 786 void 787 udp_ctlinput(netmsg_t msg) 788 { 789 struct sockaddr *sa = msg->ctlinput.nm_arg; 790 struct ip *ip = msg->ctlinput.nm_extra; 791 int cmd = msg->ctlinput.nm_cmd, cpuid; 792 inp_notify_t notify; 793 struct in_addr faddr; 794 795 notify = udp_get_inpnotify(cmd, sa, &ip, &cpuid); 796 if (notify == NULL) 797 goto done; 798 799 faddr = ((struct sockaddr_in *)sa)->sin_addr; 800 if (ip) { 801 const struct udphdr *uh; 802 struct inpcb *inp; 803 804 if (cpuid != mycpuid) 805 goto done; 806 807 uh = (const struct udphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 808 inp = in_pcblookup_hash(&udbinfo[mycpuid], faddr, uh->uh_dport, 809 ip->ip_src, uh->uh_sport, 0, NULL); 810 if (inp != NULL && inp->inp_socket != NULL) 811 notify(inp, inetctlerrmap[cmd]); 812 } else if (msg->ctlinput.nm_direct) { 813 if (cpuid != ncpus && cpuid != mycpuid) 814 goto done; 815 if (mycpuid >= ncpus2) 816 goto done; 817 818 in_pcbnotifyall(&udbinfo[mycpuid], faddr, inetctlerrmap[cmd], 819 notify); 820 } else { 821 struct netmsg_udp_notify *nm; 822 823 KKASSERT(&curthread->td_msgport == netisr_cpuport(0)); 824 nm = kmalloc(sizeof(*nm), M_LWKTMSG, M_INTWAIT); 825 netmsg_init(&nm->base, NULL, &netisr_afree_rport, 826 0, udp_notifyall_oncpu); 827 nm->nm_faddr = faddr; 828 nm->nm_arg = inetctlerrmap[cmd]; 829 nm->nm_notify = notify; 830 lwkt_sendmsg(netisr_cpuport(0), &nm->base.lmsg); 831 } 832 done: 833 lwkt_replymsg(&msg->lmsg, 0); 834 } 835 836 SYSCTL_PROC(_net_inet_udp, UDPCTL_PCBLIST, pcblist, CTLFLAG_RD, udbinfo, 0, 837 in_pcblist_global_ncpus2, "S,xinpcb", "List of active UDP sockets"); 838 839 static int 840 udp_getcred(SYSCTL_HANDLER_ARGS) 841 { 842 struct sockaddr_in addrs[2]; 843 struct ucred cred0, *cred = NULL; 844 struct inpcb *inp; 845 int error, cpu, origcpu; 846 847 error = priv_check(req->td, PRIV_ROOT); 848 if (error) 849 return (error); 850 error = SYSCTL_IN(req, addrs, sizeof addrs); 851 if (error) 852 return (error); 853 854 origcpu = mycpuid; 855 cpu = udp_addrcpu(addrs[1].sin_addr.s_addr, addrs[1].sin_port, 856 addrs[0].sin_addr.s_addr, addrs[0].sin_port); 857 858 lwkt_migratecpu(cpu); 859 860 inp = in_pcblookup_hash(&udbinfo[cpu], 861 addrs[1].sin_addr, addrs[1].sin_port, 862 addrs[0].sin_addr, addrs[0].sin_port, TRUE, NULL); 863 if (inp == NULL || inp->inp_socket == NULL) { 864 error = ENOENT; 865 } else if (inp->inp_socket->so_cred != NULL) { 866 cred0 = *(inp->inp_socket->so_cred); 867 cred = &cred0; 868 } 869 870 lwkt_migratecpu(origcpu); 871 872 if (error) 873 return error; 874 875 return SYSCTL_OUT(req, cred, sizeof(struct ucred)); 876 } 877 SYSCTL_PROC(_net_inet_udp, OID_AUTO, getcred, CTLTYPE_OPAQUE|CTLFLAG_RW, 878 0, 0, udp_getcred, "S,ucred", "Get the ucred of a UDP connection"); 879 880 static void 881 udp_send_redispatch(netmsg_t msg) 882 { 883 struct mbuf *m = msg->send.nm_m; 884 int pru_flags = msg->send.nm_flags; 885 struct inpcb *inp = msg->send.base.nm_so->so_pcb; 886 struct mbuf *m_opt = msg->send.nm_control; /* XXX save ipopt */ 887 int flags = msg->send.nm_priv; /* ip_output flags */ 888 int error; 889 890 logudp(redisp_ipout_beg, inp); 891 892 /* 893 * - Don't use inp route cache. It should only be used in the 894 * inp owner netisr. 895 * - Access to inp_moptions should be safe, since multicast UDP 896 * datagrams are redispatched to netisr0 and inp_moptions is 897 * changed only in netisr0. 898 */ 899 error = ip_output(m, m_opt, NULL, flags, inp->inp_moptions, inp); 900 if ((pru_flags & PRUS_NOREPLY) == 0) 901 lwkt_replymsg(&msg->send.base.lmsg, error); 902 903 if (m_opt != NULL) { 904 /* Free saved ip options, if any */ 905 m_freem(m_opt); 906 } 907 908 logudp(redisp_ipout_end, inp); 909 } 910 911 static void 912 udp_send(netmsg_t msg) 913 { 914 struct socket *so = msg->send.base.nm_so; 915 struct mbuf *m = msg->send.nm_m; 916 struct sockaddr *dstaddr = msg->send.nm_addr; 917 int pru_flags = msg->send.nm_flags; 918 struct inpcb *inp = so->so_pcb; 919 struct thread *td = msg->send.nm_td; 920 int flags; 921 922 struct udpiphdr *ui; 923 int len = m->m_pkthdr.len; 924 struct sockaddr_in *sin; /* really is initialized before use */ 925 int error = 0, cpu; 926 927 KKASSERT(msg->send.nm_control == NULL); 928 929 logudp(send_beg, inp); 930 931 if (inp == NULL) { 932 error = EINVAL; 933 goto release; 934 } 935 936 if (len + sizeof(struct udpiphdr) > IP_MAXPACKET) { 937 error = EMSGSIZE; 938 goto release; 939 } 940 941 if (inp->inp_lport == 0) { /* unbound socket */ 942 boolean_t forwarded; 943 944 error = in_pcbbind(inp, NULL, td); 945 if (error) 946 goto release; 947 948 /* 949 * Need to call udp_send again, after this inpcb is 950 * inserted into wildcard hash table. 951 */ 952 msg->send.base.lmsg.ms_flags |= MSGF_UDP_SEND; 953 forwarded = udp_inswildcardhash(inp, &msg->send.base, 0); 954 if (forwarded) { 955 /* 956 * The message is further forwarded, so we are 957 * done here. 958 */ 959 logudp(send_inswildcard, inp); 960 return; 961 } 962 } 963 964 if (dstaddr != NULL) { /* destination address specified */ 965 if (inp->inp_faddr.s_addr != INADDR_ANY) { 966 /* already connected */ 967 error = EISCONN; 968 goto release; 969 } 970 sin = (struct sockaddr_in *)dstaddr; 971 if (!prison_remote_ip(td, (struct sockaddr *)&sin)) { 972 error = EAFNOSUPPORT; /* IPv6 only jail */ 973 goto release; 974 } 975 } else { 976 if (inp->inp_faddr.s_addr == INADDR_ANY) { 977 /* no destination specified and not already connected */ 978 error = ENOTCONN; 979 goto release; 980 } 981 sin = NULL; 982 } 983 984 /* 985 * Calculate data length and get a mbuf 986 * for UDP and IP headers. 987 */ 988 M_PREPEND(m, sizeof(struct udpiphdr), MB_DONTWAIT); 989 if (m == NULL) { 990 error = ENOBUFS; 991 goto release; 992 } 993 994 /* 995 * Fill in mbuf with extended UDP header 996 * and addresses and length put into network format. 997 */ 998 ui = mtod(m, struct udpiphdr *); 999 bzero(ui->ui_x1, sizeof ui->ui_x1); /* XXX still needed? */ 1000 ui->ui_pr = IPPROTO_UDP; 1001 1002 /* 1003 * Set destination address. 1004 */ 1005 if (dstaddr != NULL) { /* use specified destination */ 1006 ui->ui_dst = sin->sin_addr; 1007 ui->ui_dport = sin->sin_port; 1008 } else { /* use connected destination */ 1009 ui->ui_dst = inp->inp_faddr; 1010 ui->ui_dport = inp->inp_fport; 1011 } 1012 1013 /* 1014 * Set source address. 1015 */ 1016 if (inp->inp_laddr.s_addr == INADDR_ANY || 1017 IN_MULTICAST(ntohl(inp->inp_laddr.s_addr))) { 1018 struct sockaddr_in *if_sin; 1019 1020 if (dstaddr == NULL) { 1021 /* 1022 * connect() had (or should have) failed because 1023 * the interface had no IP address, but the 1024 * application proceeded to call send() anyways. 1025 */ 1026 error = ENOTCONN; 1027 goto release; 1028 } 1029 1030 /* Look up outgoing interface. */ 1031 error = in_pcbladdr_find(inp, dstaddr, &if_sin, td, 1); 1032 if (error) 1033 goto release; 1034 ui->ui_src = if_sin->sin_addr; /* use address of interface */ 1035 } else { 1036 ui->ui_src = inp->inp_laddr; /* use non-null bound address */ 1037 } 1038 ui->ui_sport = inp->inp_lport; 1039 KASSERT(inp->inp_lport != 0, ("inp lport should have been bound")); 1040 1041 /* 1042 * Release the original thread, since it is no longer used 1043 */ 1044 if (pru_flags & PRUS_HELDTD) { 1045 lwkt_rele(td); 1046 pru_flags &= ~PRUS_HELDTD; 1047 } 1048 /* 1049 * Free the dest address, since it is no longer needed 1050 */ 1051 if (pru_flags & PRUS_FREEADDR) { 1052 kfree(dstaddr, M_SONAME); 1053 pru_flags &= ~PRUS_FREEADDR; 1054 } 1055 1056 ui->ui_ulen = htons((u_short)len + sizeof(struct udphdr)); 1057 1058 /* 1059 * Set up checksum and output datagram. 1060 */ 1061 if (udpcksum) { 1062 ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr, 1063 htons((u_short)len + sizeof(struct udphdr) + IPPROTO_UDP)); 1064 m->m_pkthdr.csum_flags = CSUM_UDP; 1065 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1066 m->m_pkthdr.csum_thlen = sizeof(struct udphdr); 1067 } else { 1068 ui->ui_sum = 0; 1069 } 1070 ((struct ip *)ui)->ip_len = sizeof(struct udpiphdr) + len; 1071 ((struct ip *)ui)->ip_ttl = inp->inp_ip_ttl; /* XXX */ 1072 ((struct ip *)ui)->ip_tos = inp->inp_ip_tos; /* XXX */ 1073 udp_stat.udps_opackets++; 1074 1075 flags = IP_DEBUGROUTE | 1076 (inp->inp_socket->so_options & (SO_DONTROUTE | SO_BROADCAST)); 1077 if (pru_flags & PRUS_DONTROUTE) 1078 flags |= SO_DONTROUTE; 1079 1080 if (inp->inp_flags & INP_CONNECTED) { 1081 /* 1082 * For connected socket, this datagram has already 1083 * been in the correct netisr; no need to rehash. 1084 */ 1085 goto sendit; 1086 } 1087 1088 cpu = udp_addrcpu(ui->ui_dst.s_addr, ui->ui_dport, 1089 ui->ui_src.s_addr, ui->ui_sport); 1090 if (cpu != mycpuid) { 1091 struct mbuf *m_opt = NULL; 1092 struct netmsg_pru_send *smsg; 1093 struct lwkt_port *port = netisr_cpuport(cpu); 1094 1095 /* 1096 * Not on the CPU that matches this UDP datagram hash; 1097 * redispatch to the correct CPU to do the ip_output(). 1098 */ 1099 if (inp->inp_options != NULL) { 1100 /* 1101 * If there are ip options, then save a copy, 1102 * since accessing inp_options on other CPUs' 1103 * is not safe. 1104 * 1105 * XXX optimize this? 1106 */ 1107 m_opt = m_copym(inp->inp_options, 0, M_COPYALL, 1108 MB_WAIT); 1109 } 1110 if ((pru_flags & PRUS_NOREPLY) == 0) { 1111 /* 1112 * Change some parts of the original netmsg and 1113 * forward it to the target netisr. 1114 * 1115 * NOTE: so_port MUST NOT be checked in the target 1116 * netisr. 1117 */ 1118 smsg = &msg->send; 1119 smsg->nm_priv = flags; /* ip_output flags */ 1120 smsg->nm_m = m; 1121 smsg->nm_control = m_opt; /* XXX save ipopt */ 1122 smsg->base.lmsg.ms_flags |= MSGF_IGNSOPORT; 1123 smsg->base.nm_dispatch = udp_send_redispatch; 1124 lwkt_forwardmsg(port, &smsg->base.lmsg); 1125 } else { 1126 /* 1127 * Recreate the netmsg, since the original mbuf 1128 * could have been changed. And send it to the 1129 * target netisr. 1130 * 1131 * NOTE: so_port MUST NOT be checked in the target 1132 * netisr. 1133 */ 1134 smsg = &m->m_hdr.mh_sndmsg; 1135 netmsg_init(&smsg->base, so, &netisr_apanic_rport, 1136 MSGF_IGNSOPORT, udp_send_redispatch); 1137 smsg->nm_priv = flags; /* ip_output flags */ 1138 smsg->nm_flags = pru_flags; 1139 smsg->nm_m = m; 1140 smsg->nm_control = m_opt; /* XXX save ipopt */ 1141 lwkt_sendmsg(port, &smsg->base.lmsg); 1142 } 1143 1144 /* This UDP datagram is redispatched; done */ 1145 logudp(send_redisp, inp); 1146 return; 1147 } 1148 1149 sendit: 1150 logudp(send_ipout, inp); 1151 error = ip_output(m, inp->inp_options, &inp->inp_route, flags, 1152 inp->inp_moptions, inp); 1153 m = NULL; 1154 1155 release: 1156 if (m != NULL) 1157 m_freem(m); 1158 1159 if (pru_flags & PRUS_HELDTD) 1160 lwkt_rele(td); 1161 if (pru_flags & PRUS_FREEADDR) 1162 kfree(dstaddr, M_SONAME); 1163 if ((pru_flags & PRUS_NOREPLY) == 0) 1164 lwkt_replymsg(&msg->send.base.lmsg, error); 1165 1166 logudp(send_end, inp); 1167 } 1168 1169 u_long udp_sendspace = 9216; /* really max datagram size */ 1170 /* 40 1K datagrams */ 1171 SYSCTL_INT(_net_inet_udp, UDPCTL_MAXDGRAM, maxdgram, CTLFLAG_RW, 1172 &udp_sendspace, 0, "Maximum outgoing UDP datagram size"); 1173 1174 u_long udp_recvspace = 40 * (1024 + 1175 #ifdef INET6 1176 sizeof(struct sockaddr_in6) 1177 #else 1178 sizeof(struct sockaddr_in) 1179 #endif 1180 ); 1181 SYSCTL_INT(_net_inet_udp, UDPCTL_RECVSPACE, recvspace, CTLFLAG_RW, 1182 &udp_recvspace, 0, "Maximum incoming UDP datagram size"); 1183 1184 /* 1185 * This should never happen, since UDP socket does not support 1186 * connection acception (SO_ACCEPTCONN, i.e. listen(2)). 1187 */ 1188 static void 1189 udp_abort(netmsg_t msg __unused) 1190 { 1191 panic("udp_abort is called"); 1192 } 1193 1194 static void 1195 udp_attach(netmsg_t msg) 1196 { 1197 struct socket *so = msg->attach.base.nm_so; 1198 struct pru_attach_info *ai = msg->attach.nm_ai; 1199 struct inpcb *inp; 1200 int error; 1201 1202 inp = so->so_pcb; 1203 if (inp != NULL) { 1204 error = EINVAL; 1205 goto out; 1206 } 1207 error = soreserve(so, udp_sendspace, udp_recvspace, ai->sb_rlimit); 1208 if (error) 1209 goto out; 1210 1211 error = in_pcballoc(so, &udbinfo[mycpuid]); 1212 if (error) 1213 goto out; 1214 1215 inp = (struct inpcb *)so->so_pcb; 1216 inp->inp_flags |= INP_DIRECT_DETACH; 1217 inp->inp_ip_ttl = ip_defttl; 1218 error = 0; 1219 out: 1220 lwkt_replymsg(&msg->attach.base.lmsg, error); 1221 } 1222 1223 static void 1224 udp_inswildcard_replymsg(netmsg_t msg) 1225 { 1226 lwkt_msg_t lmsg = &msg->lmsg; 1227 1228 if (lmsg->ms_flags & MSGF_UDP_SEND) { 1229 udp_send(msg); 1230 /* msg is replied by udp_send() */ 1231 } else { 1232 lwkt_replymsg(lmsg, lmsg->ms_error); 1233 } 1234 } 1235 1236 static void 1237 udp_soreuseport_dispatch(netmsg_t msg) 1238 { 1239 /* This inpcb has already been in the wildcard hash. */ 1240 in_pcblink_flags(msg->base.nm_so->so_pcb, &udbinfo[mycpuid], 0); 1241 udp_inswildcard_replymsg(msg); 1242 } 1243 1244 static void 1245 udp_sosetport(struct lwkt_msg *msg, lwkt_port_t port) 1246 { 1247 sosetport(((struct netmsg_base *)msg)->nm_so, port); 1248 } 1249 1250 static boolean_t 1251 udp_inswildcardhash_oncpu(struct inpcb *inp, struct netmsg_base *msg) 1252 { 1253 int cpu; 1254 1255 KASSERT(inp->inp_pcbinfo == &udbinfo[mycpuid], 1256 ("not on owner cpu")); 1257 1258 in_pcbinswildcardhash(inp); 1259 for (cpu = 0; cpu < ncpus2; ++cpu) { 1260 if (cpu == mycpuid) { 1261 /* 1262 * This inpcb has been inserted by the above 1263 * in_pcbinswildcardhash(). 1264 */ 1265 continue; 1266 } 1267 in_pcbinswildcardhash_oncpu(inp, &udbinfo[cpu]); 1268 } 1269 1270 if (inp->inp_socket->so_options & SO_REUSEPORT) { 1271 /* 1272 * For SO_REUSEPORT socket, redistribute it based on its 1273 * local group index. 1274 */ 1275 cpu = inp->inp_lgrpindex & ncpus2_mask; 1276 if (cpu != mycpuid) { 1277 struct lwkt_port *port = netisr_cpuport(cpu); 1278 lwkt_msg_t lmsg = &msg->lmsg; 1279 1280 /* 1281 * We are moving the protocol processing port the 1282 * socket is on, we have to unlink here and re-link 1283 * on the target cpu (this inpcb is still left in 1284 * the wildcard hash). 1285 */ 1286 in_pcbunlink_flags(inp, &udbinfo[mycpuid], 0); 1287 msg->nm_dispatch = udp_soreuseport_dispatch; 1288 1289 /* 1290 * See the related comment in tcp_usrreq.c 1291 * tcp_connect() 1292 */ 1293 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1294 lwkt_forwardmsg(port, lmsg); 1295 return TRUE; /* forwarded */ 1296 } 1297 } 1298 return FALSE; 1299 } 1300 1301 static void 1302 udp_inswildcardhash_dispatch(netmsg_t msg) 1303 { 1304 struct inpcb *inp = msg->base.nm_so->so_pcb; 1305 boolean_t forwarded; 1306 1307 KASSERT(inp->inp_lport != 0, ("local port not set yet")); 1308 KASSERT((ntohs(inp->inp_lport) & ncpus2_mask) == mycpuid, 1309 ("not target cpu")); 1310 1311 in_pcblink(inp, &udbinfo[mycpuid]); 1312 1313 forwarded = udp_inswildcardhash_oncpu(inp, &msg->base); 1314 if (forwarded) { 1315 /* The message is further forwarded, so we are done here. */ 1316 return; 1317 } 1318 udp_inswildcard_replymsg(msg); 1319 } 1320 1321 static boolean_t 1322 udp_inswildcardhash(struct inpcb *inp, struct netmsg_base *msg, int error) 1323 { 1324 lwkt_msg_t lmsg = &msg->lmsg; 1325 int cpu; 1326 1327 ASSERT_INP_NOTINHASH(inp); 1328 1329 /* This inpcb could no longer be directly detached */ 1330 inp->inp_flags &= ~INP_DIRECT_DETACH; 1331 1332 /* 1333 * Always clear the route cache, so we don't need to 1334 * worry about any owner CPU changes later. 1335 */ 1336 in_pcbresetroute(inp); 1337 1338 KASSERT(inp->inp_lport != 0, ("local port not set yet")); 1339 cpu = ntohs(inp->inp_lport) & ncpus2_mask; 1340 1341 lmsg->ms_error = error; 1342 if (cpu != mycpuid) { 1343 struct lwkt_port *port = netisr_cpuport(cpu); 1344 1345 /* 1346 * We are moving the protocol processing port the socket 1347 * is on, we have to unlink here and re-link on the 1348 * target cpu. 1349 */ 1350 in_pcbunlink(inp, &udbinfo[mycpuid]); 1351 msg->nm_dispatch = udp_inswildcardhash_dispatch; 1352 1353 /* See the related comment in tcp_usrreq.c tcp_connect() */ 1354 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1355 lwkt_forwardmsg(port, lmsg); 1356 return TRUE; /* forwarded */ 1357 } 1358 1359 return udp_inswildcardhash_oncpu(inp, msg); 1360 } 1361 1362 static void 1363 udp_bind(netmsg_t msg) 1364 { 1365 struct socket *so = msg->bind.base.nm_so; 1366 struct inpcb *inp; 1367 int error; 1368 1369 inp = so->so_pcb; 1370 if (inp) { 1371 struct sockaddr *nam = msg->bind.nm_nam; 1372 struct thread *td = msg->bind.nm_td; 1373 1374 error = in_pcbbind(inp, nam, td); 1375 if (error == 0) { 1376 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1377 boolean_t forwarded; 1378 1379 if (sin->sin_addr.s_addr != INADDR_ANY) 1380 inp->inp_flags |= INP_WASBOUND_NOTANY; 1381 1382 forwarded = udp_inswildcardhash(inp, 1383 &msg->bind.base, 0); 1384 if (forwarded) { 1385 /* 1386 * The message is further forwarded, so 1387 * we are done here. 1388 */ 1389 return; 1390 } 1391 } 1392 } else { 1393 error = EINVAL; 1394 } 1395 lwkt_replymsg(&msg->bind.base.lmsg, error); 1396 } 1397 1398 static void 1399 udp_connect(netmsg_t msg) 1400 { 1401 struct socket *so = msg->connect.base.nm_so; 1402 struct sockaddr *nam = msg->connect.nm_nam; 1403 struct thread *td = msg->connect.nm_td; 1404 struct inpcb *inp; 1405 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1406 struct sockaddr_in *if_sin; 1407 struct lwkt_port *port; 1408 int error; 1409 1410 KKASSERT(msg->connect.nm_m == NULL); 1411 1412 inp = so->so_pcb; 1413 if (inp == NULL) { 1414 error = EINVAL; 1415 goto out; 1416 } 1417 1418 if (msg->connect.nm_flags & PRUC_RECONNECT) { 1419 msg->connect.nm_flags &= ~PRUC_RECONNECT; 1420 in_pcblink(inp, &udbinfo[mycpuid]); 1421 } 1422 1423 if (inp->inp_faddr.s_addr != INADDR_ANY) { 1424 error = EISCONN; 1425 goto out; 1426 } 1427 error = 0; 1428 1429 /* 1430 * Bind if we have to 1431 */ 1432 if (inp->inp_lport == 0) { 1433 error = in_pcbbind(inp, NULL, td); 1434 if (error) 1435 goto out; 1436 } 1437 1438 /* 1439 * Calculate the correct protocol processing thread. The connect 1440 * operation must run there. 1441 */ 1442 error = in_pcbladdr(inp, nam, &if_sin, td); 1443 if (error) 1444 goto out; 1445 if (!prison_remote_ip(td, nam)) { 1446 error = EAFNOSUPPORT; /* IPv6 only jail */ 1447 goto out; 1448 } 1449 1450 port = udp_addrport(sin->sin_addr.s_addr, sin->sin_port, 1451 inp->inp_laddr.s_addr != INADDR_ANY ? 1452 inp->inp_laddr.s_addr : if_sin->sin_addr.s_addr, inp->inp_lport); 1453 if (port != &curthread->td_msgport) { 1454 lwkt_msg_t lmsg = &msg->connect.base.lmsg; 1455 int nm_flags = PRUC_RECONNECT; 1456 1457 /* 1458 * in_pcbladdr() may have allocated a route entry for us 1459 * on the current CPU, but we need a route entry on the 1460 * inpcb's owner CPU, so free it here. 1461 */ 1462 in_pcbresetroute(inp); 1463 1464 if (inp->inp_flags & INP_WILDCARD) { 1465 /* 1466 * Remove this inpcb from the wildcard hash before 1467 * the socket's msgport changes. 1468 */ 1469 udp_remwildcardhash(inp); 1470 } 1471 1472 /* 1473 * We are moving the protocol processing port the socket 1474 * is on, we have to unlink here and re-link on the 1475 * target cpu. 1476 */ 1477 in_pcbunlink(inp, &udbinfo[mycpuid]); 1478 msg->connect.nm_flags |= nm_flags; 1479 1480 /* See the related comment in tcp_usrreq.c tcp_connect() */ 1481 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1482 lwkt_forwardmsg(port, lmsg); 1483 /* msg invalid now */ 1484 return; 1485 } 1486 error = udp_connect_oncpu(inp, sin, if_sin); 1487 out: 1488 if (error && inp != NULL && inp->inp_lport != 0 && 1489 (inp->inp_flags & INP_WILDCARD) == 0) { 1490 boolean_t forwarded; 1491 1492 /* Connect failed; put it to wildcard hash. */ 1493 forwarded = udp_inswildcardhash(inp, &msg->connect.base, 1494 error); 1495 if (forwarded) { 1496 /* 1497 * The message is further forwarded, so we are done 1498 * here. 1499 */ 1500 return; 1501 } 1502 } 1503 lwkt_replymsg(&msg->connect.base.lmsg, error); 1504 } 1505 1506 static void 1507 udp_remwildcardhash(struct inpcb *inp) 1508 { 1509 int cpu; 1510 1511 KASSERT(inp->inp_pcbinfo == &udbinfo[mycpuid], 1512 ("not on owner cpu")); 1513 1514 for (cpu = 0; cpu < ncpus2; ++cpu) { 1515 if (cpu == mycpuid) { 1516 /* 1517 * This inpcb will be removed by the later 1518 * in_pcbremwildcardhash(). 1519 */ 1520 continue; 1521 } 1522 in_pcbremwildcardhash_oncpu(inp, &udbinfo[cpu]); 1523 } 1524 in_pcbremwildcardhash(inp); 1525 } 1526 1527 static int 1528 udp_connect_oncpu(struct inpcb *inp, struct sockaddr_in *sin, 1529 struct sockaddr_in *if_sin) 1530 { 1531 struct socket *so = inp->inp_socket; 1532 struct inpcb *oinp; 1533 1534 oinp = in_pcblookup_hash(inp->inp_pcbinfo, 1535 sin->sin_addr, sin->sin_port, 1536 inp->inp_laddr.s_addr != INADDR_ANY ? 1537 inp->inp_laddr : if_sin->sin_addr, inp->inp_lport, FALSE, NULL); 1538 if (oinp != NULL) 1539 return EADDRINUSE; 1540 1541 /* 1542 * No more errors can occur, finish adjusting the socket 1543 * and change the processing port to reflect the connected 1544 * socket. Once set we can no longer safely mess with the 1545 * socket. 1546 */ 1547 1548 if (inp->inp_flags & INP_WILDCARD) 1549 udp_remwildcardhash(inp); 1550 1551 if (inp->inp_laddr.s_addr == INADDR_ANY) 1552 inp->inp_laddr = if_sin->sin_addr; 1553 inp->inp_faddr = sin->sin_addr; 1554 inp->inp_fport = sin->sin_port; 1555 in_pcbinsconnhash(inp); 1556 1557 soisconnected(so); 1558 1559 return 0; 1560 } 1561 1562 static void 1563 udp_detach2(struct socket *so) 1564 { 1565 in_pcbdetach(so->so_pcb); 1566 sodiscard(so); 1567 sofree(so); 1568 } 1569 1570 static void 1571 udp_detach_final_dispatch(netmsg_t msg) 1572 { 1573 udp_detach2(msg->base.nm_so); 1574 } 1575 1576 static void 1577 udp_detach_oncpu_dispatch(netmsg_t msg) 1578 { 1579 struct netmsg_base *clomsg = &msg->base; 1580 struct socket *so = clomsg->nm_so; 1581 struct inpcb *inp = so->so_pcb; 1582 struct thread *td = curthread; 1583 int nextcpu, cpuid = mycpuid; 1584 1585 KASSERT(td->td_type == TD_TYPE_NETISR, ("not in netisr")); 1586 1587 if (inp->inp_flags & INP_WILDCARD) { 1588 /* 1589 * This inp will be removed on the inp's 1590 * owner CPU later, so don't do it now. 1591 */ 1592 if (&td->td_msgport != so->so_port) 1593 in_pcbremwildcardhash_oncpu(inp, &udbinfo[cpuid]); 1594 } 1595 1596 if (cpuid == 0) { 1597 /* 1598 * Free and clear multicast socket option, 1599 * which is only accessed in netisr0. 1600 */ 1601 ip_freemoptions(inp->inp_moptions); 1602 inp->inp_moptions = NULL; 1603 } 1604 1605 nextcpu = cpuid + 1; 1606 if (nextcpu < ncpus2) { 1607 lwkt_forwardmsg(netisr_cpuport(nextcpu), &clomsg->lmsg); 1608 } else { 1609 /* 1610 * No one could see this inpcb now; destroy this 1611 * inpcb in its owner netisr. 1612 */ 1613 netmsg_init(clomsg, so, &netisr_apanic_rport, 0, 1614 udp_detach_final_dispatch); 1615 lwkt_sendmsg(so->so_port, &clomsg->lmsg); 1616 } 1617 } 1618 1619 static void 1620 udp_detach(netmsg_t msg) 1621 { 1622 struct socket *so = msg->detach.base.nm_so; 1623 struct netmsg_base *clomsg; 1624 struct inpcb *inp; 1625 1626 inp = so->so_pcb; 1627 if (inp == NULL) { 1628 lwkt_replymsg(&msg->detach.base.lmsg, EINVAL); 1629 return; 1630 } 1631 1632 /* 1633 * Reply EJUSTRETURN ASAP, we will call sodiscard() and 1634 * sofree() later. 1635 */ 1636 lwkt_replymsg(&msg->detach.base.lmsg, EJUSTRETURN); 1637 1638 if (ncpus2 == 1) { 1639 /* Only one CPU, detach the inpcb directly. */ 1640 udp_detach2(so); 1641 return; 1642 } 1643 1644 /* 1645 * Remove this inpcb from the inpcb list first, so that 1646 * no one could find this inpcb from the inpcb list. 1647 */ 1648 in_pcbofflist(inp); 1649 1650 if (inp->inp_flags & INP_DIRECT_DETACH) { 1651 /* 1652 * Direct detaching is allowed 1653 */ 1654 KASSERT((inp->inp_flags & INP_WILDCARD) == 0, 1655 ("in the wildcardhash")); 1656 KASSERT(inp->inp_moptions == NULL, ("has mcast options")); 1657 udp_detach2(so); 1658 return; 1659 } 1660 1661 /* 1662 * Go through netisrs which process UDP to make sure 1663 * no one could find this inpcb anymore. 1664 */ 1665 clomsg = &so->so_clomsg; 1666 netmsg_init(clomsg, so, &netisr_apanic_rport, MSGF_IGNSOPORT, 1667 udp_detach_oncpu_dispatch); 1668 lwkt_sendmsg(netisr_cpuport(0), &clomsg->lmsg); 1669 } 1670 1671 static void 1672 udp_disconnect(netmsg_t msg) 1673 { 1674 struct socket *so = msg->disconnect.base.nm_so; 1675 struct inpcb *inp; 1676 boolean_t forwarded; 1677 int error = 0; 1678 1679 inp = so->so_pcb; 1680 if (inp == NULL) { 1681 error = EINVAL; 1682 goto out; 1683 } 1684 if (inp->inp_faddr.s_addr == INADDR_ANY) { 1685 error = ENOTCONN; 1686 goto out; 1687 } 1688 1689 soclrstate(so, SS_ISCONNECTED); /* XXX */ 1690 1691 in_pcbdisconnect(inp); 1692 1693 /* 1694 * Follow traditional BSD behavior and retain the local port 1695 * binding. But, fix the old misbehavior of overwriting any 1696 * previously bound local address. 1697 */ 1698 if (!(inp->inp_flags & INP_WASBOUND_NOTANY)) 1699 inp->inp_laddr.s_addr = INADDR_ANY; 1700 1701 if (so->so_state & SS_ISCLOSING) { 1702 /* 1703 * If this socket is being closed, there is no need 1704 * to put this socket back into wildcard hash table. 1705 */ 1706 error = 0; 1707 goto out; 1708 } 1709 1710 forwarded = udp_inswildcardhash(inp, &msg->disconnect.base, 0); 1711 if (forwarded) { 1712 /* 1713 * The message is further forwarded, so we are done 1714 * here. 1715 */ 1716 return; 1717 } 1718 out: 1719 lwkt_replymsg(&msg->disconnect.base.lmsg, error); 1720 } 1721 1722 void 1723 udp_shutdown(netmsg_t msg) 1724 { 1725 struct socket *so = msg->shutdown.base.nm_so; 1726 struct inpcb *inp; 1727 int error; 1728 1729 inp = so->so_pcb; 1730 if (inp) { 1731 socantsendmore(so); 1732 error = 0; 1733 } else { 1734 error = EINVAL; 1735 } 1736 lwkt_replymsg(&msg->shutdown.base.lmsg, error); 1737 } 1738 1739 struct pr_usrreqs udp_usrreqs = { 1740 .pru_abort = udp_abort, 1741 .pru_accept = pr_generic_notsupp, 1742 .pru_attach = udp_attach, 1743 .pru_bind = udp_bind, 1744 .pru_connect = udp_connect, 1745 .pru_connect2 = pr_generic_notsupp, 1746 .pru_control = in_control_dispatch, 1747 .pru_detach = udp_detach, 1748 .pru_disconnect = udp_disconnect, 1749 .pru_listen = pr_generic_notsupp, 1750 .pru_peeraddr = in_setpeeraddr_dispatch, 1751 .pru_rcvd = pr_generic_notsupp, 1752 .pru_rcvoob = pr_generic_notsupp, 1753 .pru_send = udp_send, 1754 .pru_sense = pru_sense_null, 1755 .pru_shutdown = udp_shutdown, 1756 .pru_sockaddr = in_setsockaddr_dispatch, 1757 .pru_sosend = sosendudp, 1758 .pru_soreceive = soreceive 1759 }; 1760