1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)udp_usrreq.c 8.6 (Berkeley) 5/23/95 63 * $FreeBSD: src/sys/netinet/udp_usrreq.c,v 1.64.2.18 2003/01/24 05:11:34 sam Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/kernel.h> 72 #include <sys/malloc.h> 73 #include <sys/mbuf.h> 74 #include <sys/domain.h> 75 #include <sys/proc.h> 76 #include <sys/priv.h> 77 #include <sys/protosw.h> 78 #include <sys/socket.h> 79 #include <sys/socketvar.h> 80 #include <sys/sysctl.h> 81 #include <sys/syslog.h> 82 #include <sys/in_cksum.h> 83 #include <sys/ktr.h> 84 85 #include <sys/thread2.h> 86 #include <sys/socketvar2.h> 87 #include <sys/serialize.h> 88 89 #include <machine/stdarg.h> 90 91 #include <net/if.h> 92 #include <net/route.h> 93 #include <net/netmsg2.h> 94 #include <net/netisr2.h> 95 96 #include <netinet/in.h> 97 #include <netinet/in_systm.h> 98 #include <netinet/ip.h> 99 #ifdef INET6 100 #include <netinet/ip6.h> 101 #endif 102 #include <netinet/in_pcb.h> 103 #include <netinet/in_var.h> 104 #include <netinet/ip_var.h> 105 #ifdef INET6 106 #include <netinet6/ip6_var.h> 107 #endif 108 #include <netinet/ip_icmp.h> 109 #include <netinet/icmp_var.h> 110 #include <netinet/udp.h> 111 #include <netinet/udp_var.h> 112 113 #ifdef FAST_IPSEC 114 #include <netproto/ipsec/ipsec.h> 115 #endif 116 117 #ifdef IPSEC 118 #include <netinet6/ipsec.h> 119 #endif 120 121 #define MSGF_UDP_SEND MSGF_PROTO1 122 123 #define INP_DIRECT_DETACH INP_FLAG_PROTO2 124 125 #define UDP_KTR_STRING "inp=%p" 126 #define UDP_KTR_ARGS struct inpcb *inp 127 128 #ifndef KTR_UDP 129 #define KTR_UDP KTR_ALL 130 #endif 131 132 KTR_INFO_MASTER(udp); 133 KTR_INFO(KTR_UDP, udp, send_beg, 0, UDP_KTR_STRING, UDP_KTR_ARGS); 134 KTR_INFO(KTR_UDP, udp, send_end, 1, UDP_KTR_STRING, UDP_KTR_ARGS); 135 KTR_INFO(KTR_UDP, udp, send_ipout, 2, UDP_KTR_STRING, UDP_KTR_ARGS); 136 KTR_INFO(KTR_UDP, udp, redisp_ipout_beg, 3, UDP_KTR_STRING, UDP_KTR_ARGS); 137 KTR_INFO(KTR_UDP, udp, redisp_ipout_end, 4, UDP_KTR_STRING, UDP_KTR_ARGS); 138 KTR_INFO(KTR_UDP, udp, send_redisp, 5, UDP_KTR_STRING, UDP_KTR_ARGS); 139 KTR_INFO(KTR_UDP, udp, send_inswildcard, 6, UDP_KTR_STRING, UDP_KTR_ARGS); 140 141 #define logudp(name, inp) KTR_LOG(udp_##name, inp) 142 143 /* 144 * UDP protocol implementation. 145 * Per RFC 768, August, 1980. 146 */ 147 #ifndef COMPAT_42 148 static int udpcksum = 1; 149 #else 150 static int udpcksum = 0; /* XXX */ 151 #endif 152 SYSCTL_INT(_net_inet_udp, UDPCTL_CHECKSUM, checksum, CTLFLAG_RW, 153 &udpcksum, 0, "Enable checksumming of UDP packets"); 154 155 int log_in_vain = 0; 156 SYSCTL_INT(_net_inet_udp, OID_AUTO, log_in_vain, CTLFLAG_RW, 157 &log_in_vain, 0, "Log all incoming UDP packets"); 158 159 static int blackhole = 0; 160 SYSCTL_INT(_net_inet_udp, OID_AUTO, blackhole, CTLFLAG_RW, 161 &blackhole, 0, "Do not send port unreachables for refused connects"); 162 163 static int strict_mcast_mship = 1; 164 SYSCTL_INT(_net_inet_udp, OID_AUTO, strict_mcast_mship, CTLFLAG_RW, 165 &strict_mcast_mship, 0, "Only send multicast to member sockets"); 166 167 int udp_sosend_async = 1; 168 SYSCTL_INT(_net_inet_udp, OID_AUTO, sosend_async, CTLFLAG_RW, 169 &udp_sosend_async, 0, "UDP asynchronized pru_send"); 170 171 int udp_sosend_prepend = 1; 172 SYSCTL_INT(_net_inet_udp, OID_AUTO, sosend_prepend, CTLFLAG_RW, 173 &udp_sosend_prepend, 0, 174 "Prepend enough space for proto and link header in pru_send"); 175 176 static int udp_reuseport_ext = 1; 177 SYSCTL_INT(_net_inet_udp, OID_AUTO, reuseport_ext, CTLFLAG_RW, 178 &udp_reuseport_ext, 0, "SO_REUSEPORT extension"); 179 180 struct inpcbinfo udbinfo[MAXCPU]; 181 182 #ifndef UDBHASHSIZE 183 #define UDBHASHSIZE 16 184 #endif 185 186 struct udpstat udpstat_percpu[MAXCPU] __cachealign; 187 188 static void udp_append(struct inpcb *last, struct ip *ip, 189 struct mbuf *n, int off, struct sockaddr_in *udp_in); 190 191 static int udp_connect_oncpu(struct inpcb *inp, struct sockaddr_in *sin, 192 struct sockaddr_in *if_sin, uint16_t hash); 193 194 static boolean_t udp_inswildcardhash(struct inpcb *inp, 195 struct netmsg_base *msg, int error); 196 static void udp_remwildcardhash(struct inpcb *inp); 197 198 static __inline int 199 udp_lportcpu(short lport) 200 { 201 return (ntohs(lport) % netisr_ncpus); 202 } 203 204 void 205 udp_init(void) 206 { 207 struct inpcbportinfo *portinfo; 208 int cpu; 209 210 portinfo = kmalloc_cachealign(sizeof(*portinfo) * netisr_ncpus, M_PCB, 211 M_WAITOK); 212 213 for (cpu = 0; cpu < netisr_ncpus; cpu++) { 214 struct inpcbinfo *uicb = &udbinfo[cpu]; 215 216 /* 217 * NOTE: 218 * UDP pcb list, wildcard hash table and localgroup hash 219 * table are shared. 220 */ 221 in_pcbinfo_init(uicb, cpu, TRUE); 222 uicb->hashbase = hashinit(UDBHASHSIZE, M_PCB, &uicb->hashmask); 223 224 in_pcbportinfo_init(&portinfo[cpu], UDBHASHSIZE, cpu); 225 in_pcbportinfo_set(uicb, portinfo, netisr_ncpus); 226 227 uicb->wildcardhashbase = hashinit(UDBHASHSIZE, M_PCB, 228 &uicb->wildcardhashmask); 229 uicb->localgrphashbase = hashinit(UDBHASHSIZE, M_PCB, 230 &uicb->localgrphashmask); 231 232 uicb->ipi_size = sizeof(struct inpcb); 233 } 234 235 /* 236 * Initialize UDP statistics counters for each CPU. 237 */ 238 for (cpu = 0; cpu < ncpus; ++cpu) 239 bzero(&udpstat_percpu[cpu], sizeof(struct udpstat)); 240 } 241 242 static int 243 sysctl_udpstat(SYSCTL_HANDLER_ARGS) 244 { 245 int cpu, error = 0; 246 247 for (cpu = 0; cpu < ncpus; ++cpu) { 248 if ((error = SYSCTL_OUT(req, &udpstat_percpu[cpu], 249 sizeof(struct udpstat)))) 250 break; 251 if ((error = SYSCTL_IN(req, &udpstat_percpu[cpu], 252 sizeof(struct udpstat)))) 253 break; 254 } 255 256 return (error); 257 } 258 SYSCTL_PROC(_net_inet_udp, UDPCTL_STATS, stats, (CTLTYPE_OPAQUE | CTLFLAG_RW), 259 0, 0, sysctl_udpstat, "S,udpstat", "UDP statistics"); 260 261 void 262 udp_ctloutput(netmsg_t msg) 263 { 264 struct socket *so = msg->base.nm_so; 265 struct sockopt *sopt = msg->ctloutput.nm_sopt; 266 struct inpcb *inp = so->so_pcb; 267 268 if (inp == NULL) { 269 lwkt_replymsg(&msg->lmsg, EINVAL); 270 return; 271 } 272 273 if (sopt->sopt_level == IPPROTO_IP && sopt->sopt_dir == SOPT_SET) { 274 switch (sopt->sopt_name) { 275 case IP_MULTICAST_IF: 276 case IP_MULTICAST_VIF: 277 case IP_MULTICAST_TTL: 278 case IP_MULTICAST_LOOP: 279 case IP_ADD_MEMBERSHIP: 280 case IP_DROP_MEMBERSHIP: 281 /* 282 * This pr_ctloutput msg will be forwarded 283 * to netisr0 to run; we can't do direct 284 * detaching anymore. 285 * 286 * NOTE: 287 * Don't optimize for the sockets whose 288 * current so_port is netisr0's msgport. 289 * These sockets could be connect(2)'ed 290 * later and the so_port will be changed. 291 */ 292 inp->inp_flags &= ~INP_DIRECT_DETACH; 293 break; 294 } 295 } 296 return ip_ctloutput(msg); 297 } 298 299 /* 300 * Check multicast packets to make sure they are only sent to sockets with 301 * multicast memberships for the packet's destination address and arrival 302 * interface. Multicast packets to multicast-unaware sockets are also 303 * disallowed. 304 * 305 * Returns 0 if the packet is acceptable, -1 if it is not. 306 */ 307 static __inline int 308 check_multicast_membership(const struct ip *ip, const struct inpcb *inp, 309 const struct mbuf *m) 310 { 311 const struct ip_moptions *mopt; 312 int mshipno; 313 314 if (strict_mcast_mship == 0 || 315 !IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) { 316 return (0); 317 } 318 319 ASSERT_IN_NETISR(0); 320 321 mopt = inp->inp_moptions; 322 if (mopt == NULL) 323 return (-1); 324 for (mshipno = 0; mshipno < mopt->imo_num_memberships; ++mshipno) { 325 const struct in_multi *maddr = mopt->imo_membership[mshipno]; 326 327 if (ip->ip_dst.s_addr == maddr->inm_addr.s_addr && 328 m->m_pkthdr.rcvif == maddr->inm_ifp) { 329 return (0); 330 } 331 } 332 return (-1); 333 } 334 335 struct udp_mcast_arg { 336 struct inpcb *inp; 337 struct inpcb *last; 338 struct ip *ip; 339 struct mbuf *m; 340 int iphlen; 341 struct sockaddr_in *udp_in; 342 }; 343 344 static int 345 udp_mcast_input(struct udp_mcast_arg *arg) 346 { 347 struct inpcb *inp = arg->inp; 348 struct inpcb *last = arg->last; 349 struct ip *ip = arg->ip; 350 struct mbuf *m = arg->m; 351 352 if (check_multicast_membership(ip, inp, m) < 0) 353 return ERESTART; /* caller continue */ 354 355 if (last != NULL) { 356 struct mbuf *n; 357 358 #ifdef IPSEC 359 /* check AH/ESP integrity. */ 360 if (ipsec4_in_reject_so(m, last->inp_socket)) 361 ipsecstat.in_polvio++; 362 /* do not inject data to pcb */ 363 else 364 #endif /*IPSEC*/ 365 #ifdef FAST_IPSEC 366 /* check AH/ESP integrity. */ 367 if (ipsec4_in_reject(m, last)) 368 ; 369 else 370 #endif /*FAST_IPSEC*/ 371 if ((n = m_copypacket(m, M_NOWAIT)) != NULL) 372 udp_append(last, ip, n, 373 arg->iphlen + sizeof(struct udphdr), 374 arg->udp_in); 375 } 376 arg->last = last = inp; 377 378 /* 379 * Don't look for additional matches if this one does 380 * not have either the SO_REUSEPORT or SO_REUSEADDR 381 * socket options set. This heuristic avoids searching 382 * through all pcbs in the common case of a non-shared 383 * port. It * assumes that an application will never 384 * clear these options after setting them. 385 */ 386 if (!(last->inp_socket->so_options & 387 (SO_REUSEPORT | SO_REUSEADDR))) 388 return EJUSTRETURN; /* caller stop */ 389 return 0; 390 } 391 392 int 393 udp_input(struct mbuf **mp, int *offp, int proto) 394 { 395 struct sockaddr_in udp_in = { sizeof udp_in, AF_INET }; 396 int iphlen; 397 struct ip *ip; 398 struct udphdr *uh; 399 struct inpcb *inp; 400 struct mbuf *m; 401 struct mbuf *opts = NULL; 402 int len, off; 403 struct ip save_ip; 404 struct inpcbinfo *pcbinfo = &udbinfo[mycpuid]; 405 406 off = *offp; 407 m = *mp; 408 *mp = NULL; 409 410 iphlen = off; 411 udp_stat.udps_ipackets++; 412 413 /* 414 * Strip IP options, if any; should skip this, 415 * make available to user, and use on returned packets, 416 * but we don't yet have a way to check the checksum 417 * with options still present. 418 */ 419 if (iphlen > sizeof(struct ip)) { 420 ip_stripoptions(m); 421 iphlen = sizeof(struct ip); 422 } 423 424 /* 425 * IP and UDP headers are together in first mbuf. 426 * Already checked and pulled up in ip_demux(). 427 */ 428 KASSERT(m->m_len >= iphlen + sizeof(struct udphdr), 429 ("UDP header not in one mbuf")); 430 431 ip = mtod(m, struct ip *); 432 uh = (struct udphdr *)((caddr_t)ip + iphlen); 433 434 /* destination port of 0 is illegal, based on RFC768. */ 435 if (uh->uh_dport == 0) 436 goto bad; 437 438 /* 439 * Make mbuf data length reflect UDP length. 440 * If not enough data to reflect UDP length, drop. 441 */ 442 len = ntohs((u_short)uh->uh_ulen); 443 if (ip->ip_len != len) { 444 if (len > ip->ip_len || len < sizeof(struct udphdr)) { 445 udp_stat.udps_badlen++; 446 goto bad; 447 } 448 m_adj(m, len - ip->ip_len); 449 /* ip->ip_len = len; */ 450 } 451 /* 452 * Save a copy of the IP header in case we want restore it 453 * for sending an ICMP error message in response. 454 */ 455 save_ip = *ip; 456 457 /* 458 * Checksum extended UDP header and data. 459 */ 460 if (uh->uh_sum) { 461 if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) { 462 if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) 463 uh->uh_sum = m->m_pkthdr.csum_data; 464 else 465 uh->uh_sum = in_pseudo(ip->ip_src.s_addr, 466 ip->ip_dst.s_addr, htonl((u_short)len + 467 m->m_pkthdr.csum_data + IPPROTO_UDP)); 468 uh->uh_sum ^= 0xffff; 469 } else { 470 char b[9]; 471 472 bcopy(((struct ipovly *)ip)->ih_x1, b, 9); 473 bzero(((struct ipovly *)ip)->ih_x1, 9); 474 ((struct ipovly *)ip)->ih_len = uh->uh_ulen; 475 uh->uh_sum = in_cksum(m, len + sizeof(struct ip)); 476 bcopy(b, ((struct ipovly *)ip)->ih_x1, 9); 477 } 478 if (uh->uh_sum) { 479 udp_stat.udps_badsum++; 480 m_freem(m); 481 return(IPPROTO_DONE); 482 } 483 } else 484 udp_stat.udps_nosum++; 485 486 if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) || 487 in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif)) { 488 struct inpcbhead *connhead; 489 struct inpcontainer *ic, *ic_marker; 490 struct inpcontainerhead *ichead; 491 struct udp_mcast_arg arg; 492 struct inpcb *last; 493 int error; 494 495 /* 496 * Deliver a multicast or broadcast datagram to *all* sockets 497 * for which the local and remote addresses and ports match 498 * those of the incoming datagram. This allows more than 499 * one process to receive multi/broadcasts on the same port. 500 * (This really ought to be done for unicast datagrams as 501 * well, but that would cause problems with existing 502 * applications that open both address-specific sockets and 503 * a wildcard socket listening to the same port -- they would 504 * end up receiving duplicates of every unicast datagram. 505 * Those applications open the multiple sockets to overcome an 506 * inadequacy of the UDP socket interface, but for backwards 507 * compatibility we avoid the problem here rather than 508 * fixing the interface. Maybe 4.5BSD will remedy this?) 509 */ 510 511 /* 512 * Construct sockaddr format source address. 513 */ 514 udp_in.sin_port = uh->uh_sport; 515 udp_in.sin_addr = ip->ip_src; 516 arg.udp_in = &udp_in; 517 /* 518 * Locate pcb(s) for datagram. 519 * (Algorithm copied from raw_intr().) 520 */ 521 last = NULL; 522 arg.iphlen = iphlen; 523 524 connhead = &pcbinfo->hashbase[ 525 INP_PCBCONNHASH(ip->ip_src.s_addr, uh->uh_sport, 526 ip->ip_dst.s_addr, uh->uh_dport, pcbinfo->hashmask)]; 527 LIST_FOREACH(inp, connhead, inp_hash) { 528 #ifdef INET6 529 if (!INP_ISIPV4(inp)) 530 continue; 531 #endif 532 if (!in_hosteq(inp->inp_faddr, ip->ip_src) || 533 !in_hosteq(inp->inp_laddr, ip->ip_dst) || 534 inp->inp_fport != uh->uh_sport || 535 inp->inp_lport != uh->uh_dport) 536 continue; 537 538 arg.inp = inp; 539 arg.last = last; 540 arg.ip = ip; 541 arg.m = m; 542 543 error = udp_mcast_input(&arg); 544 if (error == ERESTART) 545 continue; 546 last = arg.last; 547 548 if (error == EJUSTRETURN) 549 goto done; 550 } 551 552 ichead = &pcbinfo->wildcardhashbase[ 553 INP_PCBWILDCARDHASH(uh->uh_dport, 554 pcbinfo->wildcardhashmask)]; 555 ic_marker = in_pcbcontainer_marker(mycpuid); 556 557 GET_PCBINFO_TOKEN(pcbinfo); 558 LIST_INSERT_HEAD(ichead, ic_marker, ic_list); 559 while ((ic = LIST_NEXT(ic_marker, ic_list)) != NULL) { 560 LIST_REMOVE(ic_marker, ic_list); 561 LIST_INSERT_AFTER(ic, ic_marker, ic_list); 562 563 inp = ic->ic_inp; 564 if (inp->inp_flags & INP_PLACEMARKER) 565 continue; 566 #ifdef INET6 567 if (!INP_ISIPV4(inp)) 568 continue; 569 #endif 570 if (inp->inp_lport != uh->uh_dport) 571 continue; 572 if (inp->inp_laddr.s_addr != INADDR_ANY && 573 inp->inp_laddr.s_addr != ip->ip_dst.s_addr) 574 continue; 575 576 arg.inp = inp; 577 arg.last = last; 578 arg.ip = ip; 579 arg.m = m; 580 581 error = udp_mcast_input(&arg); 582 if (error == ERESTART) 583 continue; 584 last = arg.last; 585 586 if (error == EJUSTRETURN) 587 break; 588 } 589 LIST_REMOVE(ic_marker, ic_list); 590 REL_PCBINFO_TOKEN(pcbinfo); 591 done: 592 if (last == NULL) { 593 /* 594 * No matching pcb found; discard datagram. 595 * (No need to send an ICMP Port Unreachable 596 * for a broadcast or multicast datgram.) 597 */ 598 udp_stat.udps_noportbcast++; 599 goto bad; 600 } 601 #ifdef IPSEC 602 /* check AH/ESP integrity. */ 603 if (ipsec4_in_reject_so(m, last->inp_socket)) { 604 ipsecstat.in_polvio++; 605 goto bad; 606 } 607 #endif /*IPSEC*/ 608 #ifdef FAST_IPSEC 609 /* check AH/ESP integrity. */ 610 if (ipsec4_in_reject(m, last)) 611 goto bad; 612 #endif /*FAST_IPSEC*/ 613 udp_append(last, ip, m, iphlen + sizeof(struct udphdr), 614 &udp_in); 615 return(IPPROTO_DONE); 616 } 617 /* 618 * Locate pcb for datagram. 619 */ 620 inp = in_pcblookup_pkthash(pcbinfo, ip->ip_src, uh->uh_sport, 621 ip->ip_dst, uh->uh_dport, TRUE, m->m_pkthdr.rcvif, 622 udp_reuseport_ext ? m : NULL); 623 if (inp == NULL) { 624 if (log_in_vain) { 625 char src[INET_ADDRSTRLEN], dst[INET_ADDRSTRLEN]; 626 627 log(LOG_INFO, 628 "Connection attempt to UDP %s:%d from %s:%d\n", 629 kinet_ntoa(ip->ip_dst, dst), ntohs(uh->uh_dport), 630 kinet_ntoa(ip->ip_src, src), ntohs(uh->uh_sport)); 631 } 632 udp_stat.udps_noport++; 633 if (m->m_flags & (M_BCAST | M_MCAST)) { 634 udp_stat.udps_noportbcast++; 635 goto bad; 636 } 637 if (blackhole) 638 goto bad; 639 #ifdef ICMP_BANDLIM 640 if (badport_bandlim(BANDLIM_ICMP_UNREACH) < 0) 641 goto bad; 642 #endif 643 *ip = save_ip; 644 ip->ip_len += iphlen; 645 icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_PORT, 0, 0); 646 return(IPPROTO_DONE); 647 } 648 KASSERT(INP_ISIPV4(inp), ("not inet inpcb")); 649 #ifdef IPSEC 650 if (ipsec4_in_reject_so(m, inp->inp_socket)) { 651 ipsecstat.in_polvio++; 652 goto bad; 653 } 654 #endif /*IPSEC*/ 655 #ifdef FAST_IPSEC 656 if (ipsec4_in_reject(m, inp)) 657 goto bad; 658 #endif /*FAST_IPSEC*/ 659 /* 660 * Check the minimum TTL for socket. 661 */ 662 if (ip->ip_ttl < inp->inp_ip_minttl) 663 goto bad; 664 665 /* 666 * Construct sockaddr format source address. 667 * Stuff source address and datagram in user buffer. 668 */ 669 udp_in.sin_port = uh->uh_sport; 670 udp_in.sin_addr = ip->ip_src; 671 if ((inp->inp_flags & INP_CONTROLOPTS) || 672 (inp->inp_socket->so_options & SO_TIMESTAMP)) 673 ip_savecontrol(inp, &opts, ip, m); 674 m_adj(m, iphlen + sizeof(struct udphdr)); 675 676 lwkt_gettoken(&inp->inp_socket->so_rcv.ssb_token); 677 if (ssb_appendaddr(&inp->inp_socket->so_rcv, 678 (struct sockaddr *)&udp_in, m, opts) == 0) { 679 lwkt_reltoken(&inp->inp_socket->so_rcv.ssb_token); 680 udp_stat.udps_fullsock++; 681 goto bad; 682 } 683 lwkt_reltoken(&inp->inp_socket->so_rcv.ssb_token); 684 sorwakeup(inp->inp_socket); 685 return(IPPROTO_DONE); 686 bad: 687 m_freem(m); 688 if (opts) 689 m_freem(opts); 690 return(IPPROTO_DONE); 691 } 692 693 /* 694 * subroutine of udp_input(), mainly for source code readability. 695 * caller must properly init udp_ip6 and udp_in6 beforehand. 696 */ 697 static void 698 udp_append(struct inpcb *last, struct ip *ip, struct mbuf *n, int off, 699 struct sockaddr_in *udp_in) 700 { 701 struct mbuf *opts = NULL; 702 int ret; 703 704 KASSERT(INP_ISIPV4(last), ("not inet inpcb")); 705 706 if (last->inp_flags & INP_CONTROLOPTS || 707 last->inp_socket->so_options & SO_TIMESTAMP) 708 ip_savecontrol(last, &opts, ip, n); 709 m_adj(n, off); 710 711 lwkt_gettoken(&last->inp_socket->so_rcv.ssb_token); 712 ret = ssb_appendaddr(&last->inp_socket->so_rcv, 713 (struct sockaddr *)udp_in, n, opts); 714 lwkt_reltoken(&last->inp_socket->so_rcv.ssb_token); 715 if (ret == 0) { 716 m_freem(n); 717 if (opts) 718 m_freem(opts); 719 udp_stat.udps_fullsock++; 720 } else { 721 sorwakeup(last->inp_socket); 722 } 723 } 724 725 /* 726 * Notify a udp user of an asynchronous error; 727 * just wake up so that he can collect error status. 728 */ 729 void 730 udp_notify(struct inpcb *inp, int error) 731 { 732 inp->inp_socket->so_error = error; 733 sorwakeup(inp->inp_socket); 734 sowwakeup(inp->inp_socket); 735 } 736 737 struct netmsg_udp_notify { 738 struct netmsg_base base; 739 inp_notify_t nm_notify; 740 struct in_addr nm_faddr; 741 int nm_arg; 742 }; 743 744 static void 745 udp_notifyall_oncpu(netmsg_t msg) 746 { 747 struct netmsg_udp_notify *nm = (struct netmsg_udp_notify *)msg; 748 int nextcpu, cpu = mycpuid; 749 750 in_pcbnotifyall(&udbinfo[cpu], nm->nm_faddr, nm->nm_arg, nm->nm_notify); 751 752 nextcpu = cpu + 1; 753 if (nextcpu < netisr_ncpus) 754 lwkt_forwardmsg(netisr_cpuport(nextcpu), &nm->base.lmsg); 755 else 756 lwkt_replymsg(&nm->base.lmsg, 0); 757 } 758 759 inp_notify_t 760 udp_get_inpnotify(int cmd, const struct sockaddr *sa, 761 struct ip **ip0, int *cpuid) 762 { 763 struct in_addr faddr; 764 struct ip *ip = *ip0; 765 inp_notify_t notify = udp_notify; 766 767 faddr = ((const struct sockaddr_in *)sa)->sin_addr; 768 if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY) 769 return NULL; 770 771 if (PRC_IS_REDIRECT(cmd)) { 772 ip = NULL; 773 notify = in_rtchange; 774 } else if (cmd == PRC_HOSTDEAD) { 775 ip = NULL; 776 } else if ((unsigned)cmd >= PRC_NCMDS || inetctlerrmap[cmd] == 0) { 777 return NULL; 778 } 779 780 if (cpuid != NULL) { 781 if (ip == NULL) { 782 /* Go through all CPUs */ 783 *cpuid = ncpus; 784 } else { 785 const struct udphdr *uh; 786 787 uh = (const struct udphdr *) 788 ((caddr_t)ip + (ip->ip_hl << 2)); 789 *cpuid = udp_addrcpu(faddr.s_addr, uh->uh_dport, 790 ip->ip_src.s_addr, uh->uh_sport); 791 } 792 } 793 794 *ip0 = ip; 795 return notify; 796 } 797 798 void 799 udp_ctlinput(netmsg_t msg) 800 { 801 struct sockaddr *sa = msg->ctlinput.nm_arg; 802 struct ip *ip = msg->ctlinput.nm_extra; 803 int cmd = msg->ctlinput.nm_cmd, cpuid; 804 inp_notify_t notify; 805 struct in_addr faddr; 806 807 notify = udp_get_inpnotify(cmd, sa, &ip, &cpuid); 808 if (notify == NULL) 809 goto done; 810 811 faddr = ((struct sockaddr_in *)sa)->sin_addr; 812 if (ip) { 813 const struct udphdr *uh; 814 struct inpcb *inp; 815 816 if (cpuid != mycpuid) 817 goto done; 818 819 uh = (const struct udphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 820 inp = in_pcblookup_hash(&udbinfo[mycpuid], faddr, uh->uh_dport, 821 ip->ip_src, uh->uh_sport, 0, NULL); 822 if (inp != NULL && inp->inp_socket != NULL) 823 notify(inp, inetctlerrmap[cmd]); 824 } else if (msg->ctlinput.nm_direct) { 825 if (cpuid != ncpus && cpuid != mycpuid) 826 goto done; 827 if (mycpuid >= netisr_ncpus) 828 goto done; 829 830 in_pcbnotifyall(&udbinfo[mycpuid], faddr, inetctlerrmap[cmd], 831 notify); 832 } else { 833 struct netmsg_udp_notify *nm; 834 835 ASSERT_IN_NETISR(0); 836 nm = kmalloc(sizeof(*nm), M_LWKTMSG, M_INTWAIT); 837 netmsg_init(&nm->base, NULL, &netisr_afree_rport, 838 0, udp_notifyall_oncpu); 839 nm->nm_faddr = faddr; 840 nm->nm_arg = inetctlerrmap[cmd]; 841 nm->nm_notify = notify; 842 lwkt_sendmsg(netisr_cpuport(0), &nm->base.lmsg); 843 } 844 done: 845 lwkt_replymsg(&msg->lmsg, 0); 846 } 847 848 SYSCTL_PROC(_net_inet_udp, UDPCTL_PCBLIST, pcblist, CTLFLAG_RD, udbinfo, 0, 849 in_pcblist_ncpus, "S,xinpcb", "List of active UDP sockets"); 850 851 static int 852 udp_getcred(SYSCTL_HANDLER_ARGS) 853 { 854 struct sockaddr_in addrs[2]; 855 struct ucred cred0, *cred = NULL; 856 struct inpcb *inp; 857 int error, cpu, origcpu; 858 859 error = priv_check(req->td, PRIV_ROOT); 860 if (error) 861 return (error); 862 error = SYSCTL_IN(req, addrs, sizeof addrs); 863 if (error) 864 return (error); 865 866 origcpu = mycpuid; 867 cpu = udp_addrcpu(addrs[1].sin_addr.s_addr, addrs[1].sin_port, 868 addrs[0].sin_addr.s_addr, addrs[0].sin_port); 869 870 lwkt_migratecpu(cpu); 871 872 inp = in_pcblookup_hash(&udbinfo[cpu], 873 addrs[1].sin_addr, addrs[1].sin_port, 874 addrs[0].sin_addr, addrs[0].sin_port, TRUE, NULL); 875 if (inp == NULL || inp->inp_socket == NULL) { 876 error = ENOENT; 877 } else if (inp->inp_socket->so_cred != NULL) { 878 cred0 = *(inp->inp_socket->so_cred); 879 cred = &cred0; 880 } 881 882 lwkt_migratecpu(origcpu); 883 884 if (error) 885 return error; 886 887 return SYSCTL_OUT(req, cred, sizeof(struct ucred)); 888 } 889 SYSCTL_PROC(_net_inet_udp, OID_AUTO, getcred, CTLTYPE_OPAQUE|CTLFLAG_RW, 890 0, 0, udp_getcred, "S,ucred", "Get the ucred of a UDP connection"); 891 892 static void 893 udp_send_redispatch(netmsg_t msg) 894 { 895 struct mbuf *m = msg->send.nm_m; 896 int pru_flags = msg->send.nm_flags; 897 struct inpcb *inp = msg->send.base.nm_so->so_pcb; 898 struct mbuf *m_opt = msg->send.nm_control; /* XXX save ipopt */ 899 int flags = msg->send.nm_priv; /* ip_output flags */ 900 int error; 901 902 logudp(redisp_ipout_beg, inp); 903 904 /* 905 * - Don't use inp route cache. It should only be used in the 906 * inp owner netisr. 907 * - Access to inp_moptions should be safe, since multicast UDP 908 * datagrams are redispatched to netisr0 and inp_moptions is 909 * changed only in netisr0. 910 */ 911 error = ip_output(m, m_opt, NULL, flags, inp->inp_moptions, inp); 912 if ((pru_flags & PRUS_NOREPLY) == 0) 913 lwkt_replymsg(&msg->send.base.lmsg, error); 914 915 if (m_opt != NULL) { 916 /* Free saved ip options, if any */ 917 m_freem(m_opt); 918 } 919 920 logudp(redisp_ipout_end, inp); 921 } 922 923 static void 924 udp_send(netmsg_t msg) 925 { 926 struct socket *so = msg->send.base.nm_so; 927 struct mbuf *m = msg->send.nm_m; 928 struct sockaddr *dstaddr = msg->send.nm_addr; 929 int pru_flags = msg->send.nm_flags; 930 struct inpcb *inp = so->so_pcb; 931 struct thread *td = msg->send.nm_td; 932 uint16_t hash; 933 int flags; 934 935 struct udpiphdr *ui; 936 int len = m->m_pkthdr.len; 937 struct sockaddr_in *sin; /* really is initialized before use */ 938 int error = 0, cpu; 939 940 KKASSERT(msg->send.nm_control == NULL); 941 942 logudp(send_beg, inp); 943 944 if (inp == NULL) { 945 error = EINVAL; 946 goto release; 947 } 948 949 if (len + sizeof(struct udpiphdr) > IP_MAXPACKET) { 950 error = EMSGSIZE; 951 goto release; 952 } 953 954 if (inp->inp_lport == 0) { /* unbound socket */ 955 boolean_t forwarded; 956 957 error = in_pcbbind(inp, NULL, td); 958 if (error) 959 goto release; 960 961 /* 962 * Need to call udp_send again, after this inpcb is 963 * inserted into wildcard hash table. 964 */ 965 msg->send.base.lmsg.ms_flags |= MSGF_UDP_SEND; 966 forwarded = udp_inswildcardhash(inp, &msg->send.base, 0); 967 if (forwarded) { 968 /* 969 * The message is further forwarded, so we are 970 * done here. 971 */ 972 logudp(send_inswildcard, inp); 973 return; 974 } 975 } 976 977 if (dstaddr != NULL) { /* destination address specified */ 978 if (inp->inp_faddr.s_addr != INADDR_ANY) { 979 /* already connected */ 980 error = EISCONN; 981 goto release; 982 } 983 sin = (struct sockaddr_in *)dstaddr; 984 if (!prison_remote_ip(td, (struct sockaddr *)&sin)) { 985 error = EAFNOSUPPORT; /* IPv6 only jail */ 986 goto release; 987 } 988 } else { 989 if (inp->inp_faddr.s_addr == INADDR_ANY) { 990 /* no destination specified and not already connected */ 991 error = ENOTCONN; 992 goto release; 993 } 994 sin = NULL; 995 } 996 997 /* 998 * Calculate data length and get a mbuf 999 * for UDP and IP headers. 1000 */ 1001 M_PREPEND(m, sizeof(struct udpiphdr), M_NOWAIT); 1002 if (m == NULL) { 1003 error = ENOBUFS; 1004 goto release; 1005 } 1006 1007 /* 1008 * Fill in mbuf with extended UDP header 1009 * and addresses and length put into network format. 1010 */ 1011 ui = mtod(m, struct udpiphdr *); 1012 bzero(ui->ui_x1, sizeof ui->ui_x1); /* XXX still needed? */ 1013 ui->ui_pr = IPPROTO_UDP; 1014 1015 /* 1016 * Set destination address. 1017 */ 1018 if (dstaddr != NULL) { /* use specified destination */ 1019 ui->ui_dst = sin->sin_addr; 1020 ui->ui_dport = sin->sin_port; 1021 } else { /* use connected destination */ 1022 ui->ui_dst = inp->inp_faddr; 1023 ui->ui_dport = inp->inp_fport; 1024 } 1025 1026 /* 1027 * Set source address. 1028 */ 1029 if (inp->inp_laddr.s_addr == INADDR_ANY || 1030 IN_MULTICAST(ntohl(inp->inp_laddr.s_addr))) { 1031 struct sockaddr_in *if_sin; 1032 1033 if (dstaddr == NULL) { 1034 /* 1035 * connect() had (or should have) failed because 1036 * the interface had no IP address, but the 1037 * application proceeded to call send() anyways. 1038 */ 1039 error = ENOTCONN; 1040 goto release; 1041 } 1042 1043 /* Look up outgoing interface. */ 1044 error = in_pcbladdr_find(inp, dstaddr, &if_sin, td, 1); 1045 if (error) 1046 goto release; 1047 ui->ui_src = if_sin->sin_addr; /* use address of interface */ 1048 } else { 1049 ui->ui_src = inp->inp_laddr; /* use non-null bound address */ 1050 } 1051 ui->ui_sport = inp->inp_lport; 1052 KASSERT(inp->inp_lport != 0, ("inp lport should have been bound")); 1053 1054 /* 1055 * Release the original thread, since it is no longer used 1056 */ 1057 if (pru_flags & PRUS_HELDTD) { 1058 lwkt_rele(td); 1059 pru_flags &= ~PRUS_HELDTD; 1060 } 1061 /* 1062 * Free the dest address, since it is no longer needed 1063 */ 1064 if (pru_flags & PRUS_FREEADDR) { 1065 kfree(dstaddr, M_SONAME); 1066 pru_flags &= ~PRUS_FREEADDR; 1067 } 1068 1069 ui->ui_ulen = htons((u_short)len + sizeof(struct udphdr)); 1070 1071 /* 1072 * Set up checksum and output datagram. 1073 */ 1074 if (udpcksum) { 1075 ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr, 1076 htons((u_short)len + sizeof(struct udphdr) + IPPROTO_UDP)); 1077 m->m_pkthdr.csum_flags = CSUM_UDP; 1078 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 1079 m->m_pkthdr.csum_thlen = sizeof(struct udphdr); 1080 } else { 1081 ui->ui_sum = 0; 1082 } 1083 ((struct ip *)ui)->ip_len = sizeof(struct udpiphdr) + len; 1084 ((struct ip *)ui)->ip_ttl = inp->inp_ip_ttl; /* XXX */ 1085 ((struct ip *)ui)->ip_tos = inp->inp_ip_tos; /* XXX */ 1086 udp_stat.udps_opackets++; 1087 1088 flags = IP_DEBUGROUTE | 1089 (inp->inp_socket->so_options & (SO_DONTROUTE | SO_BROADCAST)); 1090 if (pru_flags & PRUS_DONTROUTE) 1091 flags |= SO_DONTROUTE; 1092 1093 if (inp->inp_flags & INP_CONNECTED) { 1094 /* 1095 * For connected socket, this datagram has already 1096 * been in the correct netisr; no need to rehash. 1097 */ 1098 KASSERT(inp->inp_flags & INP_HASH, ("inpcb has no hash")); 1099 m_sethash(m, inp->inp_hashval); 1100 goto sendit; 1101 } 1102 1103 hash = udp_addrhash(ui->ui_dst.s_addr, ui->ui_dport, 1104 ui->ui_src.s_addr, ui->ui_sport); 1105 m_sethash(m, hash); 1106 1107 cpu = netisr_hashcpu(hash); 1108 if (cpu != mycpuid) { 1109 struct mbuf *m_opt = NULL; 1110 struct netmsg_pru_send *smsg; 1111 struct lwkt_port *port = netisr_cpuport(cpu); 1112 1113 /* 1114 * Not on the CPU that matches this UDP datagram hash; 1115 * redispatch to the correct CPU to do the ip_output(). 1116 */ 1117 if (inp->inp_options != NULL) { 1118 /* 1119 * If there are ip options, then save a copy, 1120 * since accessing inp_options on other CPUs' 1121 * is not safe. 1122 * 1123 * XXX optimize this? 1124 */ 1125 m_opt = m_copym(inp->inp_options, 0, M_COPYALL, 1126 M_WAITOK); 1127 } 1128 if ((pru_flags & PRUS_NOREPLY) == 0) { 1129 /* 1130 * Change some parts of the original netmsg and 1131 * forward it to the target netisr. 1132 * 1133 * NOTE: so_port MUST NOT be checked in the target 1134 * netisr. 1135 */ 1136 smsg = &msg->send; 1137 smsg->nm_priv = flags; /* ip_output flags */ 1138 smsg->nm_m = m; 1139 smsg->nm_control = m_opt; /* XXX save ipopt */ 1140 smsg->base.lmsg.ms_flags |= MSGF_IGNSOPORT; 1141 smsg->base.nm_dispatch = udp_send_redispatch; 1142 lwkt_forwardmsg(port, &smsg->base.lmsg); 1143 } else { 1144 /* 1145 * Recreate the netmsg, since the original mbuf 1146 * could have been changed. And send it to the 1147 * target netisr. 1148 * 1149 * NOTE: so_port MUST NOT be checked in the target 1150 * netisr. 1151 */ 1152 smsg = &m->m_hdr.mh_sndmsg; 1153 netmsg_init(&smsg->base, so, &netisr_apanic_rport, 1154 MSGF_IGNSOPORT, udp_send_redispatch); 1155 smsg->nm_priv = flags; /* ip_output flags */ 1156 smsg->nm_flags = pru_flags; 1157 smsg->nm_m = m; 1158 smsg->nm_control = m_opt; /* XXX save ipopt */ 1159 lwkt_sendmsg(port, &smsg->base.lmsg); 1160 } 1161 1162 /* This UDP datagram is redispatched; done */ 1163 logudp(send_redisp, inp); 1164 return; 1165 } 1166 1167 sendit: 1168 logudp(send_ipout, inp); 1169 error = ip_output(m, inp->inp_options, &inp->inp_route, flags, 1170 inp->inp_moptions, inp); 1171 m = NULL; 1172 1173 release: 1174 if (m != NULL) 1175 m_freem(m); 1176 1177 if (pru_flags & PRUS_HELDTD) 1178 lwkt_rele(td); 1179 if (pru_flags & PRUS_FREEADDR) 1180 kfree(dstaddr, M_SONAME); 1181 if ((pru_flags & PRUS_NOREPLY) == 0) 1182 lwkt_replymsg(&msg->send.base.lmsg, error); 1183 1184 logudp(send_end, inp); 1185 } 1186 1187 u_long udp_sendspace = 9216; /* really max datagram size */ 1188 /* 40 1K datagrams */ 1189 SYSCTL_INT(_net_inet_udp, UDPCTL_MAXDGRAM, maxdgram, CTLFLAG_RW, 1190 &udp_sendspace, 0, "Maximum outgoing UDP datagram size"); 1191 1192 u_long udp_recvspace = 40 * (1024 + 1193 #ifdef INET6 1194 sizeof(struct sockaddr_in6) 1195 #else 1196 sizeof(struct sockaddr_in) 1197 #endif 1198 ); 1199 SYSCTL_INT(_net_inet_udp, UDPCTL_RECVSPACE, recvspace, CTLFLAG_RW, 1200 &udp_recvspace, 0, "Maximum incoming UDP datagram size"); 1201 1202 /* 1203 * This should never happen, since UDP socket does not support 1204 * connection acception (SO_ACCEPTCONN, i.e. listen(2)). 1205 */ 1206 static void 1207 udp_abort(netmsg_t msg __unused) 1208 { 1209 panic("udp_abort is called"); 1210 } 1211 1212 static int 1213 udp_preattach(struct socket *so, int proto __unused, struct pru_attach_info *ai) 1214 { 1215 return soreserve(so, udp_sendspace, udp_recvspace, ai->sb_rlimit); 1216 } 1217 1218 static void 1219 udp_attach(netmsg_t msg) 1220 { 1221 struct socket *so = msg->attach.base.nm_so; 1222 struct pru_attach_info *ai = msg->attach.nm_ai; 1223 struct inpcb *inp; 1224 int error; 1225 1226 KASSERT(so->so_pcb == NULL, ("udp socket attached")); 1227 1228 if (ai != NULL) { 1229 error = udp_preattach(so, 0 /* don't care */, ai); 1230 if (error) 1231 goto out; 1232 } else { 1233 /* Post attach; do nothing */ 1234 } 1235 1236 error = in_pcballoc(so, &udbinfo[mycpuid]); 1237 if (error) 1238 goto out; 1239 1240 inp = so->so_pcb; 1241 inp->inp_flags |= INP_DIRECT_DETACH; 1242 inp->inp_ip_ttl = ip_defttl; 1243 error = 0; 1244 out: 1245 lwkt_replymsg(&msg->attach.base.lmsg, error); 1246 } 1247 1248 static void 1249 udp_inswildcard_replymsg(netmsg_t msg) 1250 { 1251 lwkt_msg_t lmsg = &msg->lmsg; 1252 1253 if (lmsg->ms_flags & MSGF_UDP_SEND) { 1254 udp_send(msg); 1255 /* msg is replied by udp_send() */ 1256 } else { 1257 lwkt_replymsg(lmsg, lmsg->ms_error); 1258 } 1259 } 1260 1261 static void 1262 udp_soreuseport_dispatch(netmsg_t msg) 1263 { 1264 /* This inpcb has already been in the wildcard hash. */ 1265 in_pcblink_flags(msg->base.nm_so->so_pcb, &udbinfo[mycpuid], 0); 1266 udp_inswildcard_replymsg(msg); 1267 } 1268 1269 static void 1270 udp_sosetport(struct lwkt_msg *msg, lwkt_port_t port) 1271 { 1272 sosetport(((struct netmsg_base *)msg)->nm_so, port); 1273 } 1274 1275 static boolean_t 1276 udp_inswildcardhash_oncpu(struct inpcb *inp, struct netmsg_base *msg) 1277 { 1278 int cpu; 1279 1280 KASSERT(inp->inp_pcbinfo == &udbinfo[mycpuid], 1281 ("not on owner cpu")); 1282 1283 in_pcbinswildcardhash(inp); 1284 for (cpu = 0; cpu < netisr_ncpus; ++cpu) { 1285 if (cpu == mycpuid) { 1286 /* 1287 * This inpcb has been inserted by the above 1288 * in_pcbinswildcardhash(). 1289 */ 1290 continue; 1291 } 1292 in_pcbinswildcardhash_oncpu(inp, &udbinfo[cpu]); 1293 } 1294 1295 if (inp->inp_socket->so_options & SO_REUSEPORT) { 1296 /* 1297 * For SO_REUSEPORT socket, redistribute it based on its 1298 * local group index. 1299 */ 1300 cpu = inp->inp_lgrpindex % netisr_ncpus; 1301 if (cpu != mycpuid) { 1302 struct lwkt_port *port = netisr_cpuport(cpu); 1303 lwkt_msg_t lmsg = &msg->lmsg; 1304 1305 /* 1306 * We are moving the protocol processing port the 1307 * socket is on, we have to unlink here and re-link 1308 * on the target cpu (this inpcb is still left in 1309 * the wildcard hash). 1310 */ 1311 in_pcbunlink_flags(inp, &udbinfo[mycpuid], 0); 1312 msg->nm_dispatch = udp_soreuseport_dispatch; 1313 1314 /* 1315 * See the related comment in tcp_usrreq.c 1316 * tcp_connect() 1317 */ 1318 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1319 lwkt_forwardmsg(port, lmsg); 1320 return TRUE; /* forwarded */ 1321 } 1322 } 1323 return FALSE; 1324 } 1325 1326 static void 1327 udp_inswildcardhash_dispatch(netmsg_t msg) 1328 { 1329 struct inpcb *inp = msg->base.nm_so->so_pcb; 1330 boolean_t forwarded; 1331 1332 KASSERT(inp->inp_lport != 0, ("local port not set yet")); 1333 KASSERT(udp_lportcpu(inp->inp_lport) == mycpuid, ("not target cpu")); 1334 1335 in_pcblink(inp, &udbinfo[mycpuid]); 1336 1337 forwarded = udp_inswildcardhash_oncpu(inp, &msg->base); 1338 if (forwarded) { 1339 /* The message is further forwarded, so we are done here. */ 1340 return; 1341 } 1342 udp_inswildcard_replymsg(msg); 1343 } 1344 1345 static boolean_t 1346 udp_inswildcardhash(struct inpcb *inp, struct netmsg_base *msg, int error) 1347 { 1348 lwkt_msg_t lmsg = &msg->lmsg; 1349 int cpu; 1350 1351 ASSERT_INP_NOTINHASH(inp); 1352 1353 /* This inpcb could no longer be directly detached */ 1354 inp->inp_flags &= ~INP_DIRECT_DETACH; 1355 1356 /* 1357 * Always clear the route cache, so we don't need to 1358 * worry about any owner CPU changes later. 1359 */ 1360 in_pcbresetroute(inp); 1361 1362 KASSERT(inp->inp_lport != 0, ("local port not set yet")); 1363 cpu = udp_lportcpu(inp->inp_lport); 1364 1365 lmsg->ms_error = error; 1366 if (cpu != mycpuid) { 1367 struct lwkt_port *port = netisr_cpuport(cpu); 1368 1369 /* 1370 * We are moving the protocol processing port the socket 1371 * is on, we have to unlink here and re-link on the 1372 * target cpu. 1373 */ 1374 in_pcbunlink(inp, &udbinfo[mycpuid]); 1375 msg->nm_dispatch = udp_inswildcardhash_dispatch; 1376 1377 /* See the related comment in tcp_usrreq.c tcp_connect() */ 1378 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1379 lwkt_forwardmsg(port, lmsg); 1380 return TRUE; /* forwarded */ 1381 } 1382 1383 return udp_inswildcardhash_oncpu(inp, msg); 1384 } 1385 1386 static void 1387 udp_bind(netmsg_t msg) 1388 { 1389 struct socket *so = msg->bind.base.nm_so; 1390 struct inpcb *inp; 1391 int error; 1392 1393 inp = so->so_pcb; 1394 if (inp) { 1395 struct sockaddr *nam = msg->bind.nm_nam; 1396 struct thread *td = msg->bind.nm_td; 1397 struct sockaddr_in *sin; 1398 lwkt_port_t port; 1399 int cpu; 1400 1401 /* 1402 * Check "already bound" here (in_pcbbind() does the same 1403 * check though), so we don't forward a connected/bound 1404 * socket randomly which would panic in the following 1405 * in_pcbunlink(). 1406 */ 1407 if (inp->inp_lport != 0 || 1408 inp->inp_laddr.s_addr != INADDR_ANY) { 1409 error = EINVAL; /* already bound */ 1410 goto done; 1411 } 1412 1413 if (nam->sa_len != sizeof(*sin)) { 1414 error = EINVAL; 1415 goto done; 1416 } 1417 sin = (struct sockaddr_in *)nam; 1418 1419 cpu = udp_lportcpu(sin->sin_port); 1420 port = netisr_cpuport(cpu); 1421 1422 /* 1423 * See the related comment in tcp_usrreq.c tcp_usr_bind(). 1424 * The exception is that we use local port based netisr 1425 * to serialize in_pcbbind(). 1426 */ 1427 if (&curthread->td_msgport != port) { 1428 lwkt_msg_t lmsg = &msg->bind.base.lmsg; 1429 1430 KASSERT((msg->bind.nm_flags & PRUB_RELINK) == 0, 1431 ("already asked to relink")); 1432 1433 in_pcbunlink(so->so_pcb, &udbinfo[mycpuid]); 1434 msg->bind.nm_flags |= PRUB_RELINK; 1435 1436 /* 1437 * See the related comment in tcp_usrreq.c 1438 * tcp_connect(). 1439 */ 1440 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1441 lwkt_forwardmsg(port, lmsg); 1442 /* msg invalid now */ 1443 return; 1444 } 1445 KASSERT(so->so_port == port, ("so_port is not netisr%d", cpu)); 1446 1447 if (msg->bind.nm_flags & PRUB_RELINK) { 1448 msg->bind.nm_flags &= ~PRUB_RELINK; 1449 in_pcblink(so->so_pcb, &udbinfo[mycpuid]); 1450 } 1451 KASSERT(inp->inp_pcbinfo == &udbinfo[cpu], 1452 ("pcbinfo is not udbinfo%d", cpu)); 1453 1454 error = in_pcbbind(inp, nam, td); 1455 if (error == 0) { 1456 boolean_t forwarded; 1457 1458 if (sin->sin_addr.s_addr != INADDR_ANY) 1459 inp->inp_flags |= INP_WASBOUND_NOTANY; 1460 1461 forwarded = udp_inswildcardhash(inp, 1462 &msg->bind.base, 0); 1463 if (forwarded) { 1464 /* 1465 * The message is further forwarded, so 1466 * we are done here. 1467 */ 1468 return; 1469 } 1470 } 1471 } else { 1472 error = EINVAL; 1473 } 1474 done: 1475 lwkt_replymsg(&msg->bind.base.lmsg, error); 1476 } 1477 1478 static int 1479 udp_preconnect(struct socket *so, const struct sockaddr *nam __unused, 1480 struct thread *td __unused) 1481 { 1482 sosetstate(so, SS_ISCONNECTED); /* XXX */ 1483 return 0; 1484 } 1485 1486 static void 1487 udp_connect(netmsg_t msg) 1488 { 1489 struct socket *so = msg->connect.base.nm_so; 1490 struct sockaddr *nam = msg->connect.nm_nam; 1491 struct thread *td = msg->connect.nm_td; 1492 struct inpcb *inp; 1493 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1494 struct sockaddr_in *if_sin; 1495 struct lwkt_port *port; 1496 uint16_t hash; 1497 int error; 1498 1499 KKASSERT(msg->connect.nm_m == NULL); 1500 1501 inp = so->so_pcb; 1502 if (inp == NULL) { 1503 error = EINVAL; 1504 goto out; 1505 } 1506 1507 if (msg->connect.nm_flags & PRUC_RECONNECT) { 1508 msg->connect.nm_flags &= ~PRUC_RECONNECT; 1509 in_pcblink(inp, &udbinfo[mycpuid]); 1510 } 1511 1512 if (inp->inp_faddr.s_addr != INADDR_ANY) { 1513 error = EISCONN; 1514 goto out; 1515 } 1516 error = 0; 1517 1518 /* 1519 * Bind if we have to 1520 */ 1521 if (inp->inp_lport == 0) { 1522 error = in_pcbbind(inp, NULL, td); 1523 if (error) 1524 goto out; 1525 } 1526 1527 /* 1528 * Calculate the correct protocol processing thread. The connect 1529 * operation must run there. 1530 */ 1531 error = in_pcbladdr(inp, nam, &if_sin, td); 1532 if (error) 1533 goto out; 1534 if (!prison_remote_ip(td, nam)) { 1535 error = EAFNOSUPPORT; /* IPv6 only jail */ 1536 goto out; 1537 } 1538 1539 hash = udp_addrhash(sin->sin_addr.s_addr, sin->sin_port, 1540 inp->inp_laddr.s_addr != INADDR_ANY ? 1541 inp->inp_laddr.s_addr : if_sin->sin_addr.s_addr, inp->inp_lport); 1542 port = netisr_hashport(hash); 1543 if (port != &curthread->td_msgport) { 1544 lwkt_msg_t lmsg = &msg->connect.base.lmsg; 1545 int nm_flags = PRUC_RECONNECT; 1546 1547 /* 1548 * in_pcbladdr() may have allocated a route entry for us 1549 * on the current CPU, but we need a route entry on the 1550 * inpcb's owner CPU, so free it here. 1551 */ 1552 in_pcbresetroute(inp); 1553 1554 if (inp->inp_flags & INP_WILDCARD) { 1555 /* 1556 * Remove this inpcb from the wildcard hash before 1557 * the socket's msgport changes. 1558 */ 1559 udp_remwildcardhash(inp); 1560 } 1561 1562 if (so->so_orig_port == NULL) { 1563 /* 1564 * First time change protocol processing port. 1565 * Save the current port for synchronization upon 1566 * udp_detach. 1567 */ 1568 so->so_orig_port = &curthread->td_msgport; 1569 } else { 1570 /* 1571 * We have changed protocol processing port more 1572 * than once. We could not do direct detach 1573 * anymore, because we lose the track of the 1574 * original protocol processing ports to perform 1575 * synchronization upon udp_detach. This should 1576 * be rare though. 1577 */ 1578 inp->inp_flags &= ~INP_DIRECT_DETACH; 1579 } 1580 1581 /* 1582 * We are moving the protocol processing port the socket 1583 * is on, we have to unlink here and re-link on the 1584 * target cpu. 1585 */ 1586 in_pcbunlink(inp, &udbinfo[mycpuid]); 1587 msg->connect.nm_flags |= nm_flags; 1588 1589 /* See the related comment in tcp_usrreq.c tcp_connect() */ 1590 lwkt_setmsg_receipt(lmsg, udp_sosetport); 1591 lwkt_forwardmsg(port, lmsg); 1592 /* msg invalid now */ 1593 return; 1594 } 1595 error = udp_connect_oncpu(inp, sin, if_sin, hash); 1596 out: 1597 if (msg->connect.nm_flags & PRUC_HELDTD) 1598 lwkt_rele(td); 1599 if (error && (msg->connect.nm_flags & PRUC_ASYNC)) { 1600 if (inp->inp_lport == 0) { 1601 /* 1602 * As long as we have the local port, it is fine 1603 * for connect to fail, e.g. disconnect. 1604 */ 1605 so->so_error = error; 1606 } 1607 soclrstate(so, SS_ISCONNECTED); 1608 /* 1609 * Wake up callers blocked on this socket to make sure 1610 * that they can see this error. 1611 * 1612 * NOTE: 1613 * sodisconnected() can't be used here, which bricks 1614 * sending and receiving. 1615 */ 1616 wakeup(&so->so_timeo); 1617 sowwakeup(so); 1618 sorwakeup(so); 1619 } 1620 if (error && inp != NULL && inp->inp_lport != 0 && 1621 (inp->inp_flags & INP_WILDCARD) == 0) { 1622 boolean_t forwarded; 1623 1624 /* Connect failed; put it to wildcard hash. */ 1625 forwarded = udp_inswildcardhash(inp, &msg->connect.base, 1626 error); 1627 if (forwarded) { 1628 /* 1629 * The message is further forwarded, so we are done 1630 * here. 1631 */ 1632 return; 1633 } 1634 } 1635 lwkt_replymsg(&msg->connect.base.lmsg, error); 1636 } 1637 1638 static void 1639 udp_remwildcardhash(struct inpcb *inp) 1640 { 1641 int cpu; 1642 1643 KASSERT(inp->inp_pcbinfo == &udbinfo[mycpuid], 1644 ("not on owner cpu")); 1645 1646 for (cpu = 0; cpu < netisr_ncpus; ++cpu) { 1647 if (cpu == mycpuid) { 1648 /* 1649 * This inpcb will be removed by the later 1650 * in_pcbremwildcardhash(). 1651 */ 1652 continue; 1653 } 1654 in_pcbremwildcardhash_oncpu(inp, &udbinfo[cpu]); 1655 } 1656 in_pcbremwildcardhash(inp); 1657 } 1658 1659 static int 1660 udp_connect_oncpu(struct inpcb *inp, struct sockaddr_in *sin, 1661 struct sockaddr_in *if_sin, uint16_t hash) 1662 { 1663 struct socket *so = inp->inp_socket; 1664 struct inpcb *oinp; 1665 1666 oinp = in_pcblookup_hash(inp->inp_pcbinfo, 1667 sin->sin_addr, sin->sin_port, 1668 inp->inp_laddr.s_addr != INADDR_ANY ? 1669 inp->inp_laddr : if_sin->sin_addr, inp->inp_lport, FALSE, NULL); 1670 if (oinp != NULL) 1671 return EADDRINUSE; 1672 1673 /* 1674 * No more errors can occur, finish adjusting the socket 1675 * and change the processing port to reflect the connected 1676 * socket. Once set we can no longer safely mess with the 1677 * socket. 1678 */ 1679 1680 if (inp->inp_flags & INP_WILDCARD) 1681 udp_remwildcardhash(inp); 1682 1683 if (inp->inp_laddr.s_addr == INADDR_ANY) 1684 inp->inp_laddr = if_sin->sin_addr; 1685 inp->inp_faddr = sin->sin_addr; 1686 inp->inp_fport = sin->sin_port; 1687 in_pcbinsconnhash(inp); 1688 1689 inp->inp_flags |= INP_HASH; 1690 inp->inp_hashval = hash; 1691 1692 soisconnected(so); 1693 1694 return 0; 1695 } 1696 1697 static void 1698 udp_detach2(struct socket *so) 1699 { 1700 in_pcbdetach(so->so_pcb); 1701 sodiscard(so); 1702 sofree(so); 1703 } 1704 1705 static void 1706 udp_detach_final_dispatch(netmsg_t msg) 1707 { 1708 udp_detach2(msg->base.nm_so); 1709 } 1710 1711 static void 1712 udp_detach_oncpu_dispatch(netmsg_t msg) 1713 { 1714 struct netmsg_base *clomsg = &msg->base; 1715 struct socket *so = clomsg->nm_so; 1716 struct inpcb *inp = so->so_pcb; 1717 struct thread *td = curthread; 1718 int nextcpu, cpuid = mycpuid; 1719 1720 KASSERT(td->td_type == TD_TYPE_NETISR, ("not in netisr")); 1721 1722 if (inp->inp_flags & INP_WILDCARD) { 1723 /* 1724 * This inp will be removed on the inp's 1725 * owner CPU later, so don't do it now. 1726 */ 1727 if (&td->td_msgport != so->so_port) 1728 in_pcbremwildcardhash_oncpu(inp, &udbinfo[cpuid]); 1729 } 1730 1731 if (cpuid == 0) { 1732 /* 1733 * Free and clear multicast socket option, 1734 * which is only accessed in netisr0. 1735 */ 1736 ip_freemoptions(inp->inp_moptions); 1737 inp->inp_moptions = NULL; 1738 } 1739 1740 nextcpu = cpuid + 1; 1741 if (nextcpu < netisr_ncpus) { 1742 lwkt_forwardmsg(netisr_cpuport(nextcpu), &clomsg->lmsg); 1743 } else { 1744 /* 1745 * No one could see this inpcb now; destroy this 1746 * inpcb in its owner netisr. 1747 */ 1748 netmsg_init(clomsg, so, &netisr_apanic_rport, 0, 1749 udp_detach_final_dispatch); 1750 lwkt_sendmsg(so->so_port, &clomsg->lmsg); 1751 } 1752 } 1753 1754 static void 1755 udp_detach_syncorig_dispatch(netmsg_t msg) 1756 { 1757 struct netmsg_base *clomsg = &msg->base; 1758 struct socket *so = clomsg->nm_so; 1759 1760 /* 1761 * Original protocol processing port is synchronized; 1762 * destroy this inpcb in its owner netisr. 1763 */ 1764 netmsg_init(clomsg, so, &netisr_apanic_rport, 0, 1765 udp_detach_final_dispatch); 1766 lwkt_sendmsg(so->so_port, &clomsg->lmsg); 1767 } 1768 1769 static void 1770 udp_detach(netmsg_t msg) 1771 { 1772 struct socket *so = msg->detach.base.nm_so; 1773 struct netmsg_base *clomsg; 1774 struct inpcb *inp; 1775 1776 inp = so->so_pcb; 1777 if (inp == NULL) { 1778 lwkt_replymsg(&msg->detach.base.lmsg, EINVAL); 1779 return; 1780 } 1781 1782 /* 1783 * Reply EJUSTRETURN ASAP, we will call sodiscard() and 1784 * sofree() later. 1785 */ 1786 lwkt_replymsg(&msg->detach.base.lmsg, EJUSTRETURN); 1787 1788 if (netisr_ncpus == 1) { 1789 /* Only one CPU, detach the inpcb directly. */ 1790 udp_detach2(so); 1791 return; 1792 } 1793 1794 /* 1795 * Remove this inpcb from the inpcb list first, so that 1796 * no one could find this inpcb from the inpcb list. 1797 */ 1798 in_pcbofflist(inp); 1799 1800 /* 1801 * Remove this inpcb from the local port hash directly 1802 * here, so that its bound local port could be recycled 1803 * timely. 1804 */ 1805 in_pcbremporthash(inp); 1806 1807 if (inp->inp_flags & INP_DIRECT_DETACH) { 1808 /* 1809 * Direct detaching is allowed 1810 */ 1811 KASSERT((inp->inp_flags & INP_WILDCARD) == 0, 1812 ("in the wildcardhash")); 1813 KASSERT(inp->inp_moptions == NULL, ("has mcast options")); 1814 if (so->so_orig_port == NULL) { 1815 udp_detach2(so); 1816 } else { 1817 /* 1818 * Protocol processing port changed once, so 1819 * we need to make sure that there are nothing 1820 * left on the original protocol processing 1821 * port before we destroy this socket and inpcb. 1822 * This is more lightweight than going through 1823 * all UDP processing netisrs. 1824 */ 1825 clomsg = &so->so_clomsg; 1826 netmsg_init(clomsg, so, &netisr_apanic_rport, 1827 MSGF_IGNSOPORT, udp_detach_syncorig_dispatch); 1828 lwkt_sendmsg(so->so_orig_port, &clomsg->lmsg); 1829 } 1830 return; 1831 } 1832 1833 /* 1834 * Go through netisrs which process UDP to make sure 1835 * no one could find this inpcb anymore. 1836 */ 1837 clomsg = &so->so_clomsg; 1838 netmsg_init(clomsg, so, &netisr_apanic_rport, MSGF_IGNSOPORT, 1839 udp_detach_oncpu_dispatch); 1840 lwkt_sendmsg(netisr_cpuport(0), &clomsg->lmsg); 1841 } 1842 1843 static void 1844 udp_disconnect(netmsg_t msg) 1845 { 1846 struct socket *so = msg->disconnect.base.nm_so; 1847 struct inpcb *inp; 1848 boolean_t forwarded; 1849 int error = 0; 1850 1851 inp = so->so_pcb; 1852 if (inp == NULL) { 1853 error = EINVAL; 1854 goto out; 1855 } 1856 if (inp->inp_faddr.s_addr == INADDR_ANY) { 1857 error = ENOTCONN; 1858 goto out; 1859 } 1860 1861 soclrstate(so, SS_ISCONNECTED); /* XXX */ 1862 1863 in_pcbdisconnect(inp); 1864 inp->inp_flags &= ~INP_HASH; 1865 1866 /* 1867 * Follow traditional BSD behavior and retain the local port 1868 * binding. But, fix the old misbehavior of overwriting any 1869 * previously bound local address. 1870 */ 1871 if (!(inp->inp_flags & INP_WASBOUND_NOTANY)) 1872 inp->inp_laddr.s_addr = INADDR_ANY; 1873 1874 if (so->so_state & SS_ISCLOSING) { 1875 /* 1876 * If this socket is being closed, there is no need 1877 * to put this socket back into wildcard hash table. 1878 */ 1879 error = 0; 1880 goto out; 1881 } 1882 1883 forwarded = udp_inswildcardhash(inp, &msg->disconnect.base, 0); 1884 if (forwarded) { 1885 /* 1886 * The message is further forwarded, so we are done 1887 * here. 1888 */ 1889 return; 1890 } 1891 out: 1892 lwkt_replymsg(&msg->disconnect.base.lmsg, error); 1893 } 1894 1895 void 1896 udp_shutdown(netmsg_t msg) 1897 { 1898 struct socket *so = msg->shutdown.base.nm_so; 1899 struct inpcb *inp; 1900 int error; 1901 1902 inp = so->so_pcb; 1903 if (inp) { 1904 socantsendmore(so); 1905 error = 0; 1906 } else { 1907 error = EINVAL; 1908 } 1909 lwkt_replymsg(&msg->shutdown.base.lmsg, error); 1910 } 1911 1912 struct pr_usrreqs udp_usrreqs = { 1913 .pru_abort = udp_abort, 1914 .pru_accept = pr_generic_notsupp, 1915 .pru_attach = udp_attach, 1916 .pru_bind = udp_bind, 1917 .pru_connect = udp_connect, 1918 .pru_connect2 = pr_generic_notsupp, 1919 .pru_control = in_control_dispatch, 1920 .pru_detach = udp_detach, 1921 .pru_disconnect = udp_disconnect, 1922 .pru_listen = pr_generic_notsupp, 1923 .pru_peeraddr = in_setpeeraddr_dispatch, 1924 .pru_rcvd = pr_generic_notsupp, 1925 .pru_rcvoob = pr_generic_notsupp, 1926 .pru_send = udp_send, 1927 .pru_sense = pru_sense_null, 1928 .pru_shutdown = udp_shutdown, 1929 .pru_sockaddr = in_setsockaddr_dispatch, 1930 .pru_sosend = sosendudp, 1931 .pru_soreceive = soreceive, 1932 .pru_preconnect = udp_preconnect, 1933 .pru_preattach = udp_preattach 1934 }; 1935