xref: /dragonfly/sys/netinet/ip_demux.c (revision 31524921)
1 /*
2  * Copyright (c) 2003, 2004 Jeffrey M. Hsu.  All rights reserved.
3  * Copyright (c) 2003, 2004 The DragonFly Project.  All rights reserved.
4  *
5  * This code is derived from software contributed to The DragonFly Project
6  * by Jeffrey M. Hsu.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. Neither the name of The DragonFly Project nor the names of its
17  *    contributors may be used to endorse or promote products derived
18  *    from this software without specific, prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 #include "opt_inet.h"
35 #include "opt_rss.h"
36 
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/socket.h>
41 #include <sys/socketvar.h>
42 #include <sys/thread.h>
43 #include <sys/sysctl.h>
44 #include <sys/globaldata.h>
45 
46 #include <net/if.h>
47 #include <net/netisr2.h>
48 #include <net/toeplitz2.h>
49 
50 #include <netinet/in_systm.h>
51 #include <netinet/in.h>
52 #include <netinet/in_var.h>
53 #include <netinet/in_pcb.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip_var.h>
56 #include <netinet/tcp.h>
57 #include <netinet/tcpip.h>
58 #include <netinet/tcp_var.h>
59 #include <netinet/udp.h>
60 #include <netinet/udp_var.h>
61 
62 struct initport_index {
63 	uint32_t	port_index;
64 } __cachealign;
65 static struct initport_index	initport_indices[MAXCPU];
66 
67 /*
68  * Toeplitz hash functions - the idea is to match the hardware.
69  */
70 static __inline int
71 INP_MPORT_HASH_UDP(in_addr_t faddr, in_addr_t laddr,
72 		   in_port_t fport, in_port_t lport)
73 {
74 	return toeplitz_hash(toeplitz_rawhash_addr(faddr, laddr));
75 }
76 
77 static __inline int
78 INP_MPORT_HASH_TCP(in_addr_t faddr, in_addr_t laddr,
79 		   in_port_t fport, in_port_t lport)
80 {
81 	return toeplitz_hash(
82 	       toeplitz_rawhash_addrport(faddr, laddr, fport, lport));
83 }
84 
85 /*
86  * Map a network address to a processor.
87  */
88 int
89 tcp_addrcpu(in_addr_t faddr, in_port_t fport, in_addr_t laddr, in_port_t lport)
90 {
91 	return (netisr_hashcpu(INP_MPORT_HASH_TCP(faddr, laddr, fport, lport)));
92 }
93 
94 int
95 udp_addrcpu(in_addr_t faddr, in_port_t fport, in_addr_t laddr, in_port_t lport)
96 {
97 	/*
98 	 * NOTE: laddr could be multicast, since UDP socket could be
99 	 * bound to multicast address.
100 	 */
101 	if (IN_MULTICAST(ntohl(faddr)) || IN_MULTICAST(ntohl(laddr))) {
102 		/* XXX handle multicast on CPU0 for now */
103 		return 0;
104 	}
105 	return (netisr_hashcpu(INP_MPORT_HASH_UDP(faddr, laddr, fport, lport)));
106 }
107 
108 /*
109  * If the packet is a valid IP datagram, upon returning of this function
110  * following things are promised:
111  *
112  * o  IP header (including any possible IP options) and any data preceding
113  *    IP header (usually linker layer header) are in one mbuf (m_len).
114  * o  IP header length is not less than the minimum (sizeof(struct ip)).
115  * o  IP total length is not less than IP header length.
116  * o  IP datagram resides completely in the mbuf chain,
117  *    i.e. pkthdr.len >= IP total length.
118  *
119  * If the packet is a UDP datagram,
120  * o  IP header (including any possible IP options) and UDP header are in
121  *    one mbuf (m_len).
122  * o  IP total length is not less than (IP header length + UDP header length).
123  *
124  * If the packet is a TCP segment,
125  * o  IP header (including any possible IP options) and TCP header (including
126  *    any possible TCP options) are in one mbuf (m_len).
127  * o  TCP header length is not less than the minimum (sizeof(struct tcphdr)).
128  * o  IP total length is not less than (IP header length + TCP header length).
129  */
130 boolean_t
131 ip_lengthcheck(struct mbuf **mp, int hoff)
132 {
133 	struct mbuf *m = *mp;
134 	struct ip *ip;
135 	int len, iphlen, iplen;
136 	struct tcphdr *th;
137 	int thoff;				/* TCP data offset */
138 
139 	len = hoff + sizeof(struct ip);
140 
141 	/* The packet must be at least the size of an IP header. */
142 	if (m->m_pkthdr.len < len) {
143 		ipstat.ips_tooshort++;
144 		goto fail;
145 	}
146 
147 	/* The fixed IP header must reside completely in the first mbuf. */
148 	if (m->m_len < len) {
149 		m = m_pullup(m, len);
150 		if (m == NULL) {
151 			ipstat.ips_toosmall++;
152 			goto fail;
153 		}
154 	}
155 
156 	ip = mtodoff(m, struct ip *, hoff);
157 
158 	/* Bound check the packet's stated IP header length. */
159 	iphlen = ip->ip_hl << 2;
160 	if (iphlen < sizeof(struct ip)) {	/* minimum header length */
161 		ipstat.ips_badhlen++;
162 		goto fail;
163 	}
164 
165 	/* The full IP header must reside completely in the one mbuf. */
166 	if (m->m_len < hoff + iphlen) {
167 		m = m_pullup(m, hoff + iphlen);
168 		if (m == NULL) {
169 			ipstat.ips_badhlen++;
170 			goto fail;
171 		}
172 		ip = mtodoff(m, struct ip *, hoff);
173 	}
174 
175 	iplen = ntohs(ip->ip_len);
176 
177 	/*
178 	 * Check that the amount of data in the buffers is as
179 	 * at least much as the IP header would have us expect.
180 	 */
181 	if (m->m_pkthdr.len < hoff + iplen) {
182 		ipstat.ips_tooshort++;
183 		goto fail;
184 	}
185 
186 	/*
187 	 * Fragments other than the first fragment don't have much
188 	 * length information.
189 	 */
190 	if (ntohs(ip->ip_off) & IP_OFFMASK)
191 		goto ipcheckonly;
192 
193 	/*
194 	 * The TCP/IP or UDP/IP header must be entirely contained within
195 	 * the first fragment of a packet.  Packet filters will break if they
196 	 * aren't.
197 	 *
198 	 * Since the packet will be trimmed to ip_len we must also make sure
199 	 * the potentially trimmed down length is still sufficient to hold
200 	 * the header(s).
201 	 */
202 	switch (ip->ip_p) {
203 	case IPPROTO_TCP:
204 		if (iplen < iphlen + sizeof(struct tcphdr)) {
205 			++tcpstat.tcps_rcvshort;
206 			goto fail;
207 		}
208 		if (m->m_len < hoff + iphlen + sizeof(struct tcphdr)) {
209 			m = m_pullup(m, hoff + iphlen + sizeof(struct tcphdr));
210 			if (m == NULL) {
211 				tcpstat.tcps_rcvshort++;
212 				goto fail;
213 			}
214 			ip = mtodoff(m, struct ip *, hoff);
215 		}
216 		th = (struct tcphdr *)((caddr_t)ip + iphlen);
217 		thoff = th->th_off << 2;
218 		if (thoff < sizeof(struct tcphdr) ||
219 		    thoff + iphlen > ntohs(ip->ip_len)) {
220 			tcpstat.tcps_rcvbadoff++;
221 			goto fail;
222 		}
223 		if (m->m_len < hoff + iphlen + thoff) {
224 			m = m_pullup(m, hoff + iphlen + thoff);
225 			if (m == NULL) {
226 				tcpstat.tcps_rcvshort++;
227 				goto fail;
228 			}
229 		}
230 		break;
231 	case IPPROTO_UDP:
232 		if (iplen < iphlen + sizeof(struct udphdr)) {
233 			++udp_stat.udps_hdrops;
234 			goto fail;
235 		}
236 		if (m->m_len < hoff + iphlen + sizeof(struct udphdr)) {
237 			m = m_pullup(m, hoff + iphlen + sizeof(struct udphdr));
238 			if (m == NULL) {
239 				udp_stat.udps_hdrops++;
240 				goto fail;
241 			}
242 		}
243 		break;
244 	default:
245 ipcheckonly:
246 		if (iplen < iphlen) {
247 			++ipstat.ips_badlen;
248 			goto fail;
249 		}
250 		break;
251 	}
252 
253 	m->m_flags |= M_LENCHECKED;
254 	*mp = m;
255 	return TRUE;
256 
257 fail:
258 	if (m != NULL)
259 		m_freem(m);
260 	*mp = NULL;
261 	return FALSE;
262 }
263 
264 /*
265  * Assign a protocol processing thread to a packet.  The IP header is at
266  * offset (hoff) in the packet (i.e. the mac header might still be intact).
267  *
268  * This function can blow away the mbuf if the packet is malformed.
269  */
270 void
271 ip_hashfn(struct mbuf **mptr, int hoff)
272 {
273 	struct ip *ip;
274 	int iphlen;
275 	struct tcphdr *th;
276 	struct udphdr *uh;
277 	struct mbuf *m;
278 	int hash;
279 
280 	if (!ip_lengthcheck(mptr, hoff))
281 		return;
282 
283 	m = *mptr;
284 	ip = mtodoff(m, struct ip *, hoff);
285 	iphlen = ip->ip_hl << 2;
286 
287 	if (ntohs(ip->ip_off) & (IP_MF | IP_OFFMASK)) {
288 		hash = toeplitz_hash(toeplitz_rawhash_addr(
289 		    ip->ip_src.s_addr, ip->ip_dst.s_addr));
290 		goto back;
291 	}
292 
293 	switch (ip->ip_p) {
294 	case IPPROTO_TCP:
295 		th = (struct tcphdr *)((caddr_t)ip + iphlen);
296 		hash = INP_MPORT_HASH_TCP(ip->ip_src.s_addr, ip->ip_dst.s_addr,
297 		    th->th_sport, th->th_dport);
298 		break;
299 
300 	case IPPROTO_UDP:
301 		if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) {
302 			/* XXX handle multicast on CPU0 for now */
303 			hash = 0;
304 			break;
305 		}
306 		uh = (struct udphdr *)((caddr_t)ip + iphlen);
307 		hash = INP_MPORT_HASH_UDP(ip->ip_src.s_addr, ip->ip_dst.s_addr,
308 		    uh->uh_sport, uh->uh_dport);
309 		break;
310 
311 	default:
312 		hash = 0;
313 		break;
314 	}
315 back:
316 	m->m_flags |= M_HASH;
317 	m->m_pkthdr.hash = hash;
318 }
319 
320 /*
321  * Verify and adjust the hash value of the packet.
322  *
323  * Unlike ip_hashfn(), the packet content is not accessed.  The packet info
324  * (pi) and the hash of the packet (m_pkthdr.hash) is used instead.
325  *
326  * Caller has already made sure that m_pkthdr.hash is valid, i.e. m_flags
327  * has M_HASH set.
328  */
329 void
330 ip_hashcheck(struct mbuf *m, const struct pktinfo *pi)
331 {
332 	KASSERT((m->m_flags & M_HASH), ("no valid packet hash"));
333 
334 	switch (pi->pi_l3proto) {
335 	case IPPROTO_TCP:
336 	case IPPROTO_UDP:
337 		break;
338 
339 	default:
340 		/* Let software calculate the hash */
341 		m->m_flags &= ~M_HASH;
342 		break;
343 	}
344 }
345 
346 /*
347  * This is used to map a socket to a message port for sendmsg() and friends.
348  * It is not called for any other purpose.  In the case of TCP we just return
349  * the port already installed in the socket.
350  */
351 lwkt_port_t
352 tcp_soport(struct socket *so, struct sockaddr *nam,
353 	   struct mbuf **dummy __unused)
354 {
355 	return(so->so_port);
356 }
357 
358 /*
359  * Used to route icmp messages to the proper protocol thread for ctlinput
360  * operation.
361  */
362 lwkt_port_t
363 tcp_ctlport(int cmd, struct sockaddr *sa, void *vip, int *cpuid)
364 {
365 	struct ip *ip = vip;
366 	inp_notify_t notify;
367 	int arg;
368 
369 	notify = tcp_get_inpnotify(cmd, sa, &arg, &ip, cpuid);
370 	if (notify == NULL)
371 		return NULL;
372 
373 	if (*cpuid == ncpus) {
374 		/*
375 		 * Go through all CPUs.
376 		 *
377 		 * A new message will be allocated later to save necessary
378 		 * information and will be forwarded to all network protocol
379 		 * threads in the following way:
380 		 *
381 		 * (the the thread owns the msgport that we return here)
382 		 * netisr0 <--+
383 		 *    |       |
384 		 *    |       |
385 		 *    |       |
386 		 *    +-------+
387 		 *     sendmsg
388 		 *     [msg is kmalloc()ed]
389 		 *
390 		 *
391 		 * Later on, when the msg is received by netisr0:
392 		 *
393 		 *         forwardmsg         forwardmsg
394 		 * netisr0 ---------> netisr1 ---------> netisrN
395 		 *                                       [msg is kfree()ed]
396 		 */
397 		return netisr_cpuport(0);
398 	} else {
399 		return netisr_cpuport(*cpuid);
400 	}
401 }
402 
403 lwkt_port_t
404 tcp_addrport(in_addr_t faddr, in_port_t fport, in_addr_t laddr, in_port_t lport)
405 {
406 	return(netisr_cpuport(tcp_addrcpu(faddr, fport, laddr, lport)));
407 }
408 
409 lwkt_port_t
410 tcp_addrport0(void)
411 {
412 	return(netisr_cpuport(0));
413 }
414 
415 lwkt_port_t
416 udp_addrport(in_addr_t faddr, in_port_t fport, in_addr_t laddr, in_port_t lport)
417 {
418 	return(netisr_cpuport(udp_addrcpu(faddr, fport, laddr, lport)));
419 }
420 
421 /*
422  * Used to route icmp messages to the proper protocol thread for ctlinput
423  * operation.
424  */
425 lwkt_port_t
426 udp_ctlport(int cmd, struct sockaddr *sa, void *vip, int *cpuid)
427 {
428 	struct ip *ip = vip;
429 	inp_notify_t notify;
430 
431 	notify = udp_get_inpnotify(cmd, sa, &ip, cpuid);
432 	if (notify == NULL)
433 		return NULL;
434 
435 	if (*cpuid == ncpus) {
436 		/*
437 		 * Go through all CPUs.
438 		 *
439 		 * See the comment in tcp_ctlport.
440 		 */
441 		return netisr_cpuport(0);
442 	} else {
443 		return netisr_cpuport(*cpuid);
444 	}
445 }
446 
447 static __inline struct lwkt_port *
448 initport_ncpus2(void)
449 {
450 	int cpu = mycpuid;
451 
452 	if (cpu < ncpus2) {
453 		return netisr_cpuport(cpu);
454 	} else {
455 		return netisr_cpuport(
456 		    ((initport_indices[cpu].port_index++) + (uint32_t)cpu) &
457 		    ncpus2_mask);
458 	}
459 }
460 
461 struct lwkt_port *
462 tcp_initport(void)
463 {
464 	return initport_ncpus2();
465 }
466 
467 struct lwkt_port *
468 udp_initport(void)
469 {
470 	return initport_ncpus2();
471 }
472