xref: /minix/sys/sys/mbuf.h (revision 0a6a1f1d)
1 /*	$NetBSD: mbuf.h,v 1.158 2015/06/04 09:19:59 ozaki-r Exp $	*/
2 
3 /*-
4  * Copyright (c) 1996, 1997, 1999, 2001, 2007 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9  * NASA Ames Research Center and Matt Thomas of 3am Software Foundry.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 /*
34  * Copyright (c) 1982, 1986, 1988, 1993
35  *	The Regents of the University of California.  All rights reserved.
36  *
37  * Redistribution and use in source and binary forms, with or without
38  * modification, are permitted provided that the following conditions
39  * are met:
40  * 1. Redistributions of source code must retain the above copyright
41  *    notice, this list of conditions and the following disclaimer.
42  * 2. Redistributions in binary form must reproduce the above copyright
43  *    notice, this list of conditions and the following disclaimer in the
44  *    documentation and/or other materials provided with the distribution.
45  * 3. Neither the name of the University nor the names of its contributors
46  *    may be used to endorse or promote products derived from this software
47  *    without specific prior written permission.
48  *
49  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59  * SUCH DAMAGE.
60  *
61  *	@(#)mbuf.h	8.5 (Berkeley) 2/19/95
62  */
63 
64 #ifndef _SYS_MBUF_H_
65 #define _SYS_MBUF_H_
66 
67 #ifdef _KERNEL_OPT
68 #include "opt_mbuftrace.h"
69 #endif
70 
71 #ifndef M_WAITOK
72 #include <sys/malloc.h>
73 #endif
74 #include <sys/pool.h>
75 #include <sys/queue.h>
76 #if defined(_KERNEL)
77 #include <sys/percpu_types.h>
78 #endif /* defined(_KERNEL) */
79 
80 /* For offsetof() */
81 #if defined(_KERNEL) || defined(_STANDALONE)
82 #include <sys/systm.h>
83 #else
84 #include <stddef.h>
85 #endif
86 
87 #include <uvm/uvm_param.h>	/* for MIN_PAGE_SIZE */
88 
89 /*
90  * Mbufs are of a single size, MSIZE (machine/param.h), which
91  * includes overhead.  An mbuf may add a single "mbuf cluster" of size
92  * MCLBYTES (also in machine/param.h), which has no additional overhead
93  * and is used instead of the internal data area; this is done when
94  * at least MINCLSIZE of data must be stored.
95  */
96 
97 /* Packet tags structure */
98 struct m_tag {
99 	SLIST_ENTRY(m_tag)	m_tag_link;	/* List of packet tags */
100 	uint16_t		m_tag_id;	/* Tag ID */
101 	uint16_t		m_tag_len;	/* Length of data */
102 };
103 
104 /* mbuf ownership structure */
105 struct mowner {
106 	char mo_name[16];		/* owner name (fxp0) */
107 	char mo_descr[16];		/* owner description (input) */
108 	LIST_ENTRY(mowner) mo_link;	/* */
109 	struct percpu *mo_counters;
110 };
111 
112 #define MOWNER_INIT(x, y) { .mo_name = x, .mo_descr = y }
113 
114 enum mowner_counter_index {
115 	MOWNER_COUNTER_CLAIMS,		/* # of small mbuf claimed */
116 	MOWNER_COUNTER_RELEASES,	/* # of small mbuf released */
117 	MOWNER_COUNTER_CLUSTER_CLAIMS,	/* # of M_CLUSTER mbuf claimed */
118 	MOWNER_COUNTER_CLUSTER_RELEASES,/* # of M_CLUSTER mbuf released */
119 	MOWNER_COUNTER_EXT_CLAIMS,	/* # of M_EXT mbuf claimed */
120 	MOWNER_COUNTER_EXT_RELEASES,	/* # of M_EXT mbuf released */
121 
122 	MOWNER_COUNTER_NCOUNTERS,
123 };
124 
125 #if defined(_KERNEL)
126 struct mowner_counter {
127 	u_long mc_counter[MOWNER_COUNTER_NCOUNTERS];
128 };
129 #endif /* defined(_KERNEL) */
130 
131 /* userland-exported version of struct mowner */
132 struct mowner_user {
133 	char mo_name[16];		/* owner name (fxp0) */
134 	char mo_descr[16];		/* owner description (input) */
135 	LIST_ENTRY(mowner) mo_link;	/* unused padding; for compatibility */
136 	u_long mo_counter[MOWNER_COUNTER_NCOUNTERS]; /* counters */
137 };
138 
139 /*
140  * Macros for type conversion
141  * mtod(m,t) -	convert mbuf pointer to data pointer of correct type
142  */
143 #define	mtod(m, t)	((t)((m)->m_data))
144 
145 /* header at beginning of each mbuf: */
146 struct m_hdr {
147 	struct	mbuf *mh_next;		/* next buffer in chain */
148 	struct	mbuf *mh_nextpkt;	/* next chain in queue/record */
149 	char   *mh_data;		/* location of data */
150 	struct	mowner *mh_owner;	/* mbuf owner */
151 	int	mh_len;			/* amount of data in this mbuf */
152 	int	mh_flags;		/* flags; see below */
153 	paddr_t	mh_paddr;		/* physical address of mbuf */
154 	short	mh_type;		/* type of data in this mbuf */
155 };
156 
157 /*
158  * record/packet header in first mbuf of chain; valid if M_PKTHDR set
159  *
160  * A note about csum_data: For the out-bound direction, the low 16 bits
161  * indicates the offset after the L4 header where the final L4 checksum value
162  * is to be stored and the high 16 bits is the length of the L3 header (the
163  * start of the data to be checksumed).  For the in-bound direction, it is only
164  * valid if the M_CSUM_DATA flag is set.  In this case, an L4 checksum has been
165  * calculated by hardware, but it is up to software to perform final
166  * verification.
167  *
168  * Note for in-bound TCP/UDP checksums, we expect the csum_data to NOT
169  * be bit-wise inverted (the final step in the calculation of an IP
170  * checksum) -- this is so we can accumulate the checksum for fragmented
171  * packets during reassembly.
172  */
173 struct	pkthdr {
174 	struct ifnet	*rcvif;			/* rcv interface */
175 	SLIST_HEAD(packet_tags, m_tag) tags;	/* list of packet tags */
176 	int		len;			/* total packet length */
177 	int		csum_flags;		/* checksum flags */
178 	uint32_t	csum_data;		/* checksum data */
179 	u_int		segsz;			/* segment size */
180 };
181 
182 /*
183  * Note: These bits are carefully arrange so that the compiler can have
184  * a prayer of generating a jump table.
185  */
186 #define	M_CSUM_TCPv4		0x00000001	/* TCP header/payload */
187 #define	M_CSUM_UDPv4		0x00000002	/* UDP header/payload */
188 #define	M_CSUM_TCP_UDP_BAD	0x00000004	/* TCP/UDP checksum bad */
189 #define	M_CSUM_DATA		0x00000008	/* consult csum_data */
190 #define	M_CSUM_TCPv6		0x00000010	/* IPv6 TCP header/payload */
191 #define	M_CSUM_UDPv6		0x00000020	/* IPv6 UDP header/payload */
192 #define	M_CSUM_IPv4		0x00000040	/* IPv4 header */
193 #define	M_CSUM_IPv4_BAD		0x00000080	/* IPv4 header checksum bad */
194 #define	M_CSUM_TSOv4		0x00000100	/* TCPv4 segmentation offload */
195 #define	M_CSUM_TSOv6		0x00000200	/* TCPv6 segmentation offload */
196 
197 /* Checksum-assist quirks: keep separate from jump-table bits. */
198 #define	M_CSUM_NO_PSEUDOHDR	0x80000000	/* Rx csum_data does not include
199 						 * the UDP/TCP pseudo-hdr, and
200 						 * is not yet 1s-complemented.
201 						 */
202 
203 #define M_CSUM_BITS \
204     "\20\1TCPv4\2UDPv4\3TCP_UDP_BAD\4DATA\5TCPv6\6UDPv6\7IPv4\10IPv4_BAD" \
205     "\11TSOv4\12TSOv6\40NO_PSEUDOHDR"
206 
207 /*
208  * Macros for manipulating csum_data on outgoing packets.  These are
209  * used to pass information down from the L4/L3 to the L2.
210  */
211 #define	M_CSUM_DATA_IPv4_IPHL(x)	((x) >> 16)
212 #define	M_CSUM_DATA_IPv4_OFFSET(x)	((x) & 0xffff)
213 
214 /*
215  * Macros for M_CSUM_TCPv6 and M_CSUM_UDPv6
216  *
217  * M_CSUM_DATA_IPv6_HL: length of ip6_hdr + ext header.
218  * ie. offset of UDP/TCP header in the packet.
219  *
220  * M_CSUM_DATA_IPv6_OFFSET: offset of the checksum field in UDP/TCP header.
221  */
222 
223 #define	M_CSUM_DATA_IPv6_HL(x)		((x) >> 16)
224 #define	M_CSUM_DATA_IPv6_HL_SET(x, v)	(x) = ((x) & 0xffff) | ((v) << 16)
225 #define	M_CSUM_DATA_IPv6_OFFSET(x)	((x) & 0xffff)
226 
227 /*
228  * Max # of pages we can attach to m_ext.  This is carefully chosen
229  * to be able to handle SOSEND_LOAN_CHUNK with our minimum sized page.
230  */
231 #ifdef MIN_PAGE_SIZE
232 #define	M_EXT_MAXPAGES		((65536 / MIN_PAGE_SIZE) + 1)
233 #endif
234 
235 /* description of external storage mapped into mbuf, valid if M_EXT set */
236 struct _m_ext_storage {
237 	unsigned int ext_refcnt;
238 	int ext_flags;
239 	char *ext_buf;			/* start of buffer */
240 	void (*ext_free)		/* free routine if not the usual */
241 		(struct mbuf *, void *, size_t, void *);
242 	void *ext_arg;			/* argument for ext_free */
243 	size_t ext_size;		/* size of buffer, for ext_free */
244 	union {
245 		paddr_t extun_paddr;	/* physical address (M_EXT_CLUSTER) */
246 					/* pages (M_EXT_PAGES) */
247 	/*
248 	 * XXX This is gross, but it doesn't really matter; this is
249 	 * XXX overlaid on top of the mbuf data area.
250 	 */
251 #ifdef M_EXT_MAXPAGES
252 		struct vm_page *extun_pgs[M_EXT_MAXPAGES];
253 #endif
254 	} ext_un;
255 #define	ext_paddr	ext_un.extun_paddr
256 #define	ext_pgs		ext_un.extun_pgs
257 #ifdef DEBUG
258 	const char *ext_ofile;
259 	const char *ext_nfile;
260 	int ext_oline;
261 	int ext_nline;
262 #endif
263 };
264 
265 struct _m_ext {
266 	struct mbuf *ext_ref;
267 	struct _m_ext_storage ext_storage;
268 };
269 
270 #define	M_PADDR_INVALID		POOL_PADDR_INVALID
271 
272 /*
273  * Definition of "struct mbuf".
274  * Don't change this without understanding how MHLEN/MLEN are defined.
275  */
276 #define	MBUF_DEFINE(name, mhlen, mlen)					\
277 	struct name {							\
278 		struct	m_hdr m_hdr;					\
279 		union {							\
280 			struct {					\
281 				struct	pkthdr MH_pkthdr;		\
282 				union {					\
283 					struct	_m_ext MH_ext;		\
284 					char MH_databuf[(mhlen)];	\
285 				} MH_dat;				\
286 			} MH;						\
287 			char M_databuf[(mlen)];				\
288 		} M_dat;						\
289 	}
290 #define	m_next		m_hdr.mh_next
291 #define	m_len		m_hdr.mh_len
292 #define	m_data		m_hdr.mh_data
293 #define	m_owner		m_hdr.mh_owner
294 #define	m_type		m_hdr.mh_type
295 #define	m_flags		m_hdr.mh_flags
296 #define	m_nextpkt	m_hdr.mh_nextpkt
297 #define	m_paddr		m_hdr.mh_paddr
298 #define	m_pkthdr	M_dat.MH.MH_pkthdr
299 #define	m_ext_storage	M_dat.MH.MH_dat.MH_ext.ext_storage
300 #define	m_ext_ref	M_dat.MH.MH_dat.MH_ext.ext_ref
301 #define	m_ext		m_ext_ref->m_ext_storage
302 #define	m_pktdat	M_dat.MH.MH_dat.MH_databuf
303 #define	m_dat		M_dat.M_databuf
304 
305 /*
306  * Dummy mbuf structure to calculate the right values for MLEN/MHLEN, taking
307  * into account inter-structure padding.
308  */
309 MBUF_DEFINE(_mbuf_dummy, 1, 1);
310 
311 /* normal data len */
312 #define	MLEN		(MSIZE - offsetof(struct _mbuf_dummy, m_dat))
313 /* data len w/pkthdr */
314 #define	MHLEN		(MSIZE - offsetof(struct _mbuf_dummy, m_pktdat))
315 
316 #define	MINCLSIZE	(MHLEN+MLEN+1)	/* smallest amount to put in cluster */
317 #define	M_MAXCOMPRESS	(MHLEN / 2)	/* max amount to copy for compression */
318 
319 /*
320  * The *real* struct mbuf
321  */
322 MBUF_DEFINE(mbuf, MHLEN, MLEN);
323 
324 /* mbuf flags */
325 #define	M_EXT		0x00000001	/* has associated external storage */
326 #define	M_PKTHDR	0x00000002	/* start of record */
327 #define	M_EOR		0x00000004	/* end of record */
328 #define	M_PROTO1	0x00000008	/* protocol-specific */
329 
330 /* mbuf pkthdr flags, also in m_flags */
331 #define	M_AUTHIPHDR	0x00000010	/* data origin authentication for
332 					 * IP header */
333 #define	M_DECRYPTED	0x00000020	/* confidentiality */
334 #define	M_LOOP		0x00000040	/* for Mbuf statistics */
335 #define	M_AUTHIPDGM     0x00000080	/* data origin authentication */
336 #define	M_BCAST		0x00000100	/* send/received as link-level
337 					 * broadcast */
338 #define	M_MCAST		0x00000200	/* send/received as link-level
339 					 * multicast */
340 #define	M_CANFASTFWD	0x00000400	/* used by filters to indicate
341 					 * packet can be fast-forwarded */
342 #define	M_ANYCAST6	0x00000800	/* received as IPv6 anycast */
343 
344 #define	M_LINK0		0x00001000	/* link layer specific flag */
345 #define	M_LINK1		0x00002000	/* link layer specific flag */
346 #define	M_LINK2		0x00004000	/* link layer specific flag */
347 
348 #define	M_LINK3		0x00008000	/* link layer specific flag */
349 #define	M_LINK4		0x00010000	/* link layer specific flag */
350 #define	M_LINK5		0x00020000	/* link layer specific flag */
351 #define	M_LINK6		0x00040000	/* link layer specific flag */
352 #define	M_LINK7		0x00080000	/* link layer specific flag */
353 
354 /* additional flags for M_EXT mbufs */
355 #define	M_EXT_FLAGS	0xff000000
356 #define	M_EXT_CLUSTER	0x01000000	/* ext is a cluster */
357 #define	M_EXT_PAGES	0x02000000	/* ext_pgs is valid */
358 #define	M_EXT_ROMAP	0x04000000	/* ext mapping is r-o at MMU */
359 #define	M_EXT_RW	0x08000000	/* ext storage is writable */
360 
361 /* for source-level compatibility */
362 #define	M_CLUSTER	M_EXT_CLUSTER
363 
364 #define M_FLAGS_BITS \
365     "\20\1EXT\2PKTHDR\3EOR\4PROTO1\5AUTHIPHDR\6DECRYPTED\7LOOP\10AUTHIPDGM" \
366     "\11BCAST\12MCAST\13CANFASTFWD\14ANYCAST6\15LINK0\16LINK1\17LINK2\20LINK3" \
367     "\21LINK4\22LINK5\23LINK6\24LINK7" \
368     "\31EXT_CLUSTER\32EXT_PAGES\33EXT_ROMAP\34EXT_RW"
369 
370 /* flags copied when copying m_pkthdr */
371 #define	M_COPYFLAGS	(M_PKTHDR|M_EOR|M_BCAST|M_MCAST|M_CANFASTFWD|M_ANYCAST6|M_LINK0|M_LINK1|M_LINK2|M_AUTHIPHDR|M_DECRYPTED|M_LOOP|M_AUTHIPDGM)
372 
373 /* flag copied when shallow-copying external storage */
374 #define	M_EXTCOPYFLAGS	(M_EXT|M_EXT_FLAGS)
375 
376 /* mbuf types */
377 #define	MT_FREE		0	/* should be on free list */
378 #define	MT_DATA		1	/* dynamic (data) allocation */
379 #define	MT_HEADER	2	/* packet header */
380 #define	MT_SONAME	3	/* socket name */
381 #define	MT_SOOPTS	4	/* socket options */
382 #define	MT_FTABLE	5	/* fragment reassembly header */
383 #define MT_CONTROL	6	/* extra-data protocol message */
384 #define MT_OOBDATA	7	/* expedited data  */
385 
386 #ifdef MBUFTYPES
387 static const char * const mbuftypes[] = {
388 	"mbfree",
389 	"mbdata",
390 	"mbheader",
391 	"mbsoname",
392 	"mbsopts",
393 	"mbftable",
394 	"mbcontrol",
395 	"mboobdata",
396 };
397 #endif
398 
399 /* flags to m_get/MGET */
400 #define	M_DONTWAIT	M_NOWAIT
401 #define	M_WAIT		M_WAITOK
402 
403 #ifdef MBUFTRACE
404 /*
405  * mbuf allocation tracing
406  */
407 void mowner_init(struct mbuf *, int);
408 void mowner_ref(struct mbuf *, int);
409 void m_claim(struct mbuf *, struct mowner *);
410 void mowner_revoke(struct mbuf *, bool, int);
411 void mowner_attach(struct mowner *);
412 void mowner_detach(struct mowner *);
413 void m_claimm(struct mbuf *, struct mowner *);
414 #else
415 #define mowner_init(m, type)		do { } while (/* CONSTCOND */ 0)
416 #define	mowner_ref(m, flags)		do { } while (/* CONSTCOND */ 0)
417 #define	mowner_revoke(m, all, flags)	do { } while (/* CONSTCOND */ 0)
418 #define	m_claim(m, mowner) 		do { } while (/* CONSTCOND */ 0)
419 #define	mowner_attach(mo)		do { } while (/* CONSTCOND */ 0)
420 #define	mowner_detach(mo)		do { } while (/* CONSTCOND */ 0)
421 #define	m_claimm(m, mo)			do { } while (/* CONSTCOND */ 0)
422 #endif
423 
424 #define	MCLAIM(m, mo)		m_claim((m), (mo))
425 #define	MOWNER_ATTACH(mo)	mowner_attach(mo)
426 #define	MOWNER_DETACH(mo)	mowner_detach(mo)
427 
428 /*
429  * mbuf allocation/deallocation macros:
430  *
431  *	MGET(struct mbuf *m, int how, int type)
432  * allocates an mbuf and initializes it to contain internal data.
433  *
434  *	MGETHDR(struct mbuf *m, int how, int type)
435  * allocates an mbuf and initializes it to contain a packet header
436  * and internal data.
437  *
438  * If 'how' is M_WAIT, these macros (and the corresponding functions)
439  * are guaranteed to return successfully.
440  */
441 #define	MGET(m, how, type)	m = m_get((how), (type))
442 #define	MGETHDR(m, how, type)	m = m_gethdr((how), (type))
443 
444 #if defined(_KERNEL)
445 #define	_M_
446 /*
447  * Macros for tracking external storage associated with an mbuf.
448  */
449 #ifdef DEBUG
450 #define MCLREFDEBUGN(m, file, line)					\
451 do {									\
452 	(m)->m_ext.ext_nfile = (file);					\
453 	(m)->m_ext.ext_nline = (line);					\
454 } while (/* CONSTCOND */ 0)
455 
456 #define MCLREFDEBUGO(m, file, line)					\
457 do {									\
458 	(m)->m_ext.ext_ofile = (file);					\
459 	(m)->m_ext.ext_oline = (line);					\
460 } while (/* CONSTCOND */ 0)
461 #else
462 #define MCLREFDEBUGN(m, file, line)
463 #define MCLREFDEBUGO(m, file, line)
464 #endif
465 
466 #define	MCLINITREFERENCE(m)						\
467 do {									\
468 	KDASSERT(((m)->m_flags & M_EXT) == 0);				\
469 	(m)->m_ext_ref = (m);						\
470 	(m)->m_ext.ext_refcnt = 1;					\
471 	MCLREFDEBUGO((m), __FILE__, __LINE__);				\
472 	MCLREFDEBUGN((m), NULL, 0);					\
473 } while (/* CONSTCOND */ 0)
474 
475 /*
476  * Macros for mbuf external storage.
477  *
478  * MCLGET allocates and adds an mbuf cluster to a normal mbuf;
479  * the flag M_EXT is set upon success.
480  *
481  * MEXTMALLOC allocates external storage and adds it to
482  * a normal mbuf; the flag M_EXT is set upon success.
483  *
484  * MEXTADD adds pre-allocated external storage to
485  * a normal mbuf; the flag M_EXT is set upon success.
486  */
487 
488 #define	_MCLGET(m, pool_cache, size, how)				\
489 do {									\
490 	(m)->m_ext_storage.ext_buf = (char *)				\
491 	    pool_cache_get_paddr((pool_cache),				\
492 		(how) == M_WAIT ? (PR_WAITOK|PR_LIMITFAIL) : 0,		\
493 		&(m)->m_ext_storage.ext_paddr);				\
494 	if ((m)->m_ext_storage.ext_buf != NULL) {			\
495 		MCLINITREFERENCE(m);					\
496 		(m)->m_data = (m)->m_ext.ext_buf;			\
497 		(m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) |	\
498 				M_EXT|M_CLUSTER|M_EXT_RW;		\
499 		(m)->m_ext.ext_flags = 0;				\
500 		(m)->m_ext.ext_size = (size);				\
501 		(m)->m_ext.ext_free = NULL;				\
502 		(m)->m_ext.ext_arg = (pool_cache);			\
503 		/* ext_paddr initialized above */			\
504 		mowner_ref((m), M_EXT|M_CLUSTER);			\
505 	}								\
506 } while (/* CONSTCOND */ 0)
507 
508 /*
509  * The standard mbuf cluster pool.
510  */
511 #define	MCLGET(m, how)	_MCLGET((m), mcl_cache, MCLBYTES, (how))
512 
513 #define	MEXTMALLOC(m, size, how)					\
514 do {									\
515 	(m)->m_ext_storage.ext_buf = (char *)				\
516 	    malloc((size), mbtypes[(m)->m_type], (how));		\
517 	if ((m)->m_ext_storage.ext_buf != NULL) {			\
518 		MCLINITREFERENCE(m);					\
519 		(m)->m_data = (m)->m_ext.ext_buf;			\
520 		(m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) |	\
521 				M_EXT|M_EXT_RW;				\
522 		(m)->m_ext.ext_flags = 0;				\
523 		(m)->m_ext.ext_size = (size);				\
524 		(m)->m_ext.ext_free = NULL;				\
525 		(m)->m_ext.ext_arg = NULL;				\
526 		mowner_ref((m), M_EXT);					\
527 	}								\
528 } while (/* CONSTCOND */ 0)
529 
530 #define	MEXTADD(m, buf, size, type, free, arg)				\
531 do {									\
532 	MCLINITREFERENCE(m);						\
533 	(m)->m_data = (m)->m_ext.ext_buf = (char *)(buf);		\
534 	(m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | M_EXT;	\
535 	(m)->m_ext.ext_flags = 0;					\
536 	(m)->m_ext.ext_size = (size);					\
537 	(m)->m_ext.ext_free = (free);					\
538 	(m)->m_ext.ext_arg = (arg);					\
539 	mowner_ref((m), M_EXT);						\
540 } while (/* CONSTCOND */ 0)
541 
542 /*
543  * Reset the data pointer on an mbuf.
544  */
545 #define	MRESETDATA(m)							\
546 do {									\
547 	if ((m)->m_flags & M_EXT)					\
548 		(m)->m_data = (m)->m_ext.ext_buf;			\
549 	else if ((m)->m_flags & M_PKTHDR)				\
550 		(m)->m_data = (m)->m_pktdat;				\
551 	else								\
552 		(m)->m_data = (m)->m_dat;				\
553 } while (/* CONSTCOND */ 0)
554 
555 /*
556  * MFREE(struct mbuf *m, struct mbuf *n)
557  * Free a single mbuf and associated external storage.
558  * Place the successor, if any, in n.
559  */
560 #define	MFREE(m, n)							\
561 	mowner_revoke((m), 1, (m)->m_flags);				\
562 	mbstat_type_add((m)->m_type, -1);				\
563 	if ((m)->m_flags & M_PKTHDR)					\
564 		m_tag_delete_chain((m), NULL);				\
565 	(n) = (m)->m_next;						\
566 	if ((m)->m_flags & M_EXT) {					\
567 		m_ext_free((m));						\
568 	} else {							\
569 		KASSERT((m)->m_type != MT_FREE);				\
570 		(m)->m_type = MT_FREE;					\
571 		pool_cache_put(mb_cache, (m));				\
572 	}								\
573 
574 /*
575  * Copy mbuf pkthdr from `from' to `to'.
576  * `from' must have M_PKTHDR set, and `to' must be empty.
577  */
578 #define	M_COPY_PKTHDR(to, from)						\
579 do {									\
580 	(to)->m_pkthdr = (from)->m_pkthdr;				\
581 	(to)->m_flags = (from)->m_flags & M_COPYFLAGS;			\
582 	SLIST_INIT(&(to)->m_pkthdr.tags);				\
583 	m_tag_copy_chain((to), (from));					\
584 	(to)->m_data = (to)->m_pktdat;					\
585 } while (/* CONSTCOND */ 0)
586 
587 /*
588  * Move mbuf pkthdr from `from' to `to'.
589  * `from' must have M_PKTHDR set, and `to' must be empty.
590  */
591 #define	M_MOVE_PKTHDR(to, from)	m_move_pkthdr(to, from)
592 
593 /*
594  * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place
595  * an object of the specified size at the end of the mbuf, longword aligned.
596  */
597 #define	M_ALIGN(m, len)							\
598 do {									\
599 	(m)->m_data += (MLEN - (len)) &~ (sizeof(long) - 1);		\
600 } while (/* CONSTCOND */ 0)
601 
602 /*
603  * As above, for mbufs allocated with m_gethdr/MGETHDR
604  * or initialized by M_COPY_PKTHDR.
605  */
606 #define	MH_ALIGN(m, len)						\
607 do {									\
608 	(m)->m_data += (MHLEN - (len)) &~ (sizeof(long) - 1);		\
609 } while (/* CONSTCOND */ 0)
610 
611 /*
612  * Determine if an mbuf's data area is read-only.  This is true
613  * if external storage is read-only mapped, or not marked as R/W,
614  * or referenced by more than one mbuf.
615  */
616 #define	M_READONLY(m)							\
617 	(((m)->m_flags & M_EXT) != 0 &&					\
618 	  (((m)->m_flags & (M_EXT_ROMAP|M_EXT_RW)) != M_EXT_RW ||	\
619 	  (m)->m_ext.ext_refcnt > 1))
620 
621 #define	M_UNWRITABLE(__m, __len)					\
622 	((__m)->m_len < (__len) || M_READONLY((__m)))
623 /*
624  * Determine if an mbuf's data area is read-only at the MMU.
625  */
626 #define	M_ROMAP(m)							\
627 	(((m)->m_flags & (M_EXT|M_EXT_ROMAP)) == (M_EXT|M_EXT_ROMAP))
628 
629 /*
630  * Compute the amount of space available
631  * before the current start of data in an mbuf.
632  */
633 #define	_M_LEADINGSPACE(m)						\
634 	((m)->m_flags & M_EXT ? (m)->m_data - (m)->m_ext.ext_buf :	\
635 	 (m)->m_flags & M_PKTHDR ? (m)->m_data - (m)->m_pktdat :	\
636 	 (m)->m_data - (m)->m_dat)
637 
638 #define	M_LEADINGSPACE(m)						\
639 	(M_READONLY((m)) ? 0 : _M_LEADINGSPACE((m)))
640 
641 /*
642  * Compute the amount of space available
643  * after the end of data in an mbuf.
644  */
645 #define	_M_TRAILINGSPACE(m)						\
646 	((m)->m_flags & M_EXT ? (m)->m_ext.ext_buf + (m)->m_ext.ext_size - \
647 	 ((m)->m_data + (m)->m_len) :					\
648 	 &(m)->m_dat[MLEN] - ((m)->m_data + (m)->m_len))
649 
650 #define	M_TRAILINGSPACE(m)						\
651 	(M_READONLY((m)) ? 0 : _M_TRAILINGSPACE((m)))
652 
653 /*
654  * Compute the address of an mbuf's data area.
655  */
656 #define	M_BUFADDR(m)							\
657 	(((m)->m_flags & M_PKTHDR) ? (m)->m_pktdat : (m)->m_dat)
658 
659 /*
660  * Compute the offset of the beginning of the data buffer of a non-ext
661  * mbuf.
662  */
663 #define	M_BUFOFFSET(m)							\
664 	(((m)->m_flags & M_PKTHDR) ?					\
665 	 offsetof(struct mbuf, m_pktdat) : offsetof(struct mbuf, m_dat))
666 
667 /*
668  * Arrange to prepend space of size plen to mbuf m.
669  * If a new mbuf must be allocated, how specifies whether to wait.
670  * If how is M_DONTWAIT and allocation fails, the original mbuf chain
671  * is freed and m is set to NULL.
672  */
673 #define	M_PREPEND(m, plen, how)						\
674 do {									\
675 	if (M_LEADINGSPACE(m) >= (plen)) {				\
676 		(m)->m_data -= (plen);					\
677 		(m)->m_len += (plen);					\
678 	} else								\
679 		(m) = m_prepend((m), (plen), (how));			\
680 	if ((m) && (m)->m_flags & M_PKTHDR)				\
681 		(m)->m_pkthdr.len += (plen);				\
682 } while (/* CONSTCOND */ 0)
683 
684 /* change mbuf to new type */
685 #define MCHTYPE(m, t)							\
686 do {									\
687 	KASSERT((t) != MT_FREE);					\
688 	mbstat_type_add((m)->m_type, -1);				\
689 	mbstat_type_add(t, 1);						\
690 	(m)->m_type = t;						\
691 } while (/* CONSTCOND */ 0)
692 
693 /* length to m_copy to copy all */
694 #define	M_COPYALL	-1
695 
696 /* compatibility with 4.3 */
697 #define  m_copy(m, o, l)	m_copym((m), (o), (l), M_DONTWAIT)
698 
699 /*
700  * Allow drivers and/or protocols to use the rcvif member of
701  * PKTHDR mbufs to store private context information.
702  */
703 #define	M_GETCTX(m, t)		((t)(m)->m_pkthdr.rcvif)
704 #define	M_SETCTX(m, c)		((void)((m)->m_pkthdr.rcvif = (void *)(c)))
705 
706 #endif /* defined(_KERNEL) */
707 
708 /*
709  * Simple mbuf queueing system
710  *
711  * this is basically a SIMPLEQ adapted to mbuf use (ie using
712  * m_nextpkt instead of field.sqe_next).
713  *
714  * m_next is ignored, so queueing chains of mbufs is possible
715  */
716 #define MBUFQ_HEAD(name)					\
717 struct name {							\
718 	struct mbuf *mq_first;					\
719 	struct mbuf **mq_last;					\
720 }
721 
722 #define MBUFQ_INIT(q)		do {				\
723 	(q)->mq_first = NULL;					\
724 	(q)->mq_last = &(q)->mq_first;				\
725 } while (/*CONSTCOND*/0)
726 
727 #define MBUFQ_ENQUEUE(q, m)	do {				\
728 	(m)->m_nextpkt = NULL;					\
729 	*(q)->mq_last = (m);					\
730 	(q)->mq_last = &(m)->m_nextpkt;				\
731 } while (/*CONSTCOND*/0)
732 
733 #define MBUFQ_PREPEND(q, m)	do {				\
734 	if (((m)->m_nextpkt = (q)->mq_first) == NULL)		\
735 		(q)->mq_last = &(m)->m_nextpkt;			\
736 	(q)->mq_first = (m);					\
737 } while (/*CONSTCOND*/0)
738 
739 #define MBUFQ_DEQUEUE(q, m)	do {				\
740 	if (((m) = (q)->mq_first) != NULL) { 			\
741 		if (((q)->mq_first = (m)->m_nextpkt) == NULL)	\
742 			(q)->mq_last = &(q)->mq_first;		\
743 		else						\
744 			(m)->m_nextpkt = NULL;			\
745 	}							\
746 } while (/*CONSTCOND*/0)
747 
748 #define MBUFQ_DRAIN(q)		do {				\
749 	struct mbuf *__m0;					\
750 	while ((__m0 = (q)->mq_first) != NULL) {		\
751 		(q)->mq_first = __m0->m_nextpkt;		\
752 		m_freem(__m0);					\
753 	}							\
754 	(q)->mq_last = &(q)->mq_first;				\
755 } while (/*CONSTCOND*/0)
756 
757 #define MBUFQ_FIRST(q)		((q)->mq_first)
758 #define MBUFQ_NEXT(m)		((m)->m_nextpkt)
759 #define MBUFQ_LAST(q)		(*(q)->mq_last)
760 
761 /*
762  * Mbuf statistics.
763  * For statistics related to mbuf and cluster allocations, see also the
764  * pool headers (mb_cache and mcl_cache).
765  */
766 struct mbstat {
767 	u_long	_m_spare;	/* formerly m_mbufs */
768 	u_long	_m_spare1;	/* formerly m_clusters */
769 	u_long	_m_spare2;	/* spare field */
770 	u_long	_m_spare3;	/* formely m_clfree - free clusters */
771 	u_long	m_drops;	/* times failed to find space */
772 	u_long	m_wait;		/* times waited for space */
773 	u_long	m_drain;	/* times drained protocols for space */
774 	u_short	m_mtypes[256];	/* type specific mbuf allocations */
775 };
776 
777 struct mbstat_cpu {
778 	u_int	m_mtypes[256];	/* type specific mbuf allocations */
779 };
780 
781 /*
782  * Mbuf sysctl variables.
783  */
784 #define	MBUF_MSIZE		1	/* int: mbuf base size */
785 #define	MBUF_MCLBYTES		2	/* int: mbuf cluster size */
786 #define	MBUF_NMBCLUSTERS	3	/* int: limit on the # of clusters */
787 #define	MBUF_MBLOWAT		4	/* int: mbuf low water mark */
788 #define	MBUF_MCLLOWAT		5	/* int: mbuf cluster low water mark */
789 #define	MBUF_STATS		6	/* struct: mbstat */
790 #define	MBUF_MOWNERS		7	/* struct: m_owner[] */
791 #define	MBUF_MAXID		8	/* number of valid MBUF ids */
792 
793 #define	CTL_MBUF_NAMES {						\
794 	{ 0, 0 },							\
795 	{ "msize", CTLTYPE_INT },					\
796 	{ "mclbytes", CTLTYPE_INT },					\
797 	{ "nmbclusters", CTLTYPE_INT },					\
798 	{ "mblowat", CTLTYPE_INT },					\
799 	{ "mcllowat", CTLTYPE_INT },					\
800 	{ 0 /* "stats" */, CTLTYPE_STRUCT },				\
801 	{ 0 /* "mowners" */, CTLTYPE_STRUCT },				\
802 }
803 
804 #ifdef	_KERNEL
805 extern struct mbstat mbstat;
806 extern int	nmbclusters;		/* limit on the # of clusters */
807 extern int	mblowat;		/* mbuf low water mark */
808 extern int	mcllowat;		/* mbuf cluster low water mark */
809 extern int	max_linkhdr;		/* largest link-level header */
810 extern int	max_protohdr;		/* largest protocol header */
811 extern int	max_hdr;		/* largest link+protocol header */
812 extern int	max_datalen;		/* MHLEN - max_hdr */
813 extern const int msize;			/* mbuf base size */
814 extern const int mclbytes;		/* mbuf cluster size */
815 extern pool_cache_t mb_cache;
816 extern pool_cache_t mcl_cache;
817 #ifdef MBUFTRACE
818 LIST_HEAD(mownerhead, mowner);
819 extern struct mownerhead mowners;
820 extern struct mowner unknown_mowners[];
821 extern struct mowner revoked_mowner;
822 #endif
823 
824 MALLOC_DECLARE(M_MBUF);
825 MALLOC_DECLARE(M_SONAME);
826 
827 struct	mbuf *m_copym(struct mbuf *, int, int, int);
828 struct	mbuf *m_copypacket(struct mbuf *, int);
829 struct	mbuf *m_devget(char *, int, int, struct ifnet *,
830 			    void (*copy)(const void *, void *, size_t));
831 struct	mbuf *m_dup(struct mbuf *, int, int, int);
832 struct	mbuf *m_free(struct mbuf *);
833 struct	mbuf *m_get(int, int);
834 struct	mbuf *m_getclr(int, int);
835 struct	mbuf *m_gethdr(int, int);
836 struct	mbuf *m_prepend(struct mbuf *,int, int);
837 struct	mbuf *m_pulldown(struct mbuf *, int, int, int *);
838 struct	mbuf *m_pullup(struct mbuf *, int);
839 struct	mbuf *m_copyup(struct mbuf *, int, int);
840 struct	mbuf *m_split(struct mbuf *,int, int);
841 struct	mbuf *m_getptr(struct mbuf *, int, int *);
842 void	m_adj(struct mbuf *, int);
843 struct	mbuf *m_defrag(struct mbuf *, int);
844 int	m_apply(struct mbuf *, int, int,
845 		int (*)(void *, void *, unsigned int), void *);
846 void	m_cat(struct mbuf *,struct mbuf *);
847 void	m_clget(struct mbuf *, int);
848 int	m_mballoc(int, int);
849 void	m_copyback(struct mbuf *, int, int, const void *);
850 struct	mbuf *m_copyback_cow(struct mbuf *, int, int, const void *, int);
851 int 	m_makewritable(struct mbuf **, int, int, int);
852 struct	mbuf *m_getcl(int, int, int);
853 void	m_copydata(struct mbuf *, int, int, void *);
854 void	m_freem(struct mbuf *);
855 void	m_reclaim(void *, int);
856 void	mbinit(void);
857 void	m_ext_free(struct mbuf *);
858 char *	m_mapin(struct mbuf *);
859 void	m_move_pkthdr(struct mbuf *, struct mbuf *);
860 
861 bool	m_ensure_contig(struct mbuf **, int);
862 struct mbuf *m_add(struct mbuf *, struct mbuf *);
863 void	m_align(struct mbuf *, int);
864 int	m_append(struct mbuf *, int, const void *);
865 
866 /* Inline routines. */
867 static __inline u_int m_length(const struct mbuf *) __unused;
868 
869 /* Statistics */
870 void mbstat_type_add(int, int);
871 
872 /* Packet tag routines */
873 struct	m_tag *m_tag_get(int, int, int);
874 void	m_tag_free(struct m_tag *);
875 void	m_tag_prepend(struct mbuf *, struct m_tag *);
876 void	m_tag_unlink(struct mbuf *, struct m_tag *);
877 void	m_tag_delete(struct mbuf *, struct m_tag *);
878 void	m_tag_delete_chain(struct mbuf *, struct m_tag *);
879 void	m_tag_delete_nonpersistent(struct mbuf *);
880 struct	m_tag *m_tag_find(const struct mbuf *, int, struct m_tag *);
881 struct	m_tag *m_tag_copy(struct m_tag *);
882 int	m_tag_copy_chain(struct mbuf *, struct mbuf *);
883 void	m_tag_init(struct mbuf *);
884 struct	m_tag *m_tag_first(struct mbuf *);
885 struct	m_tag *m_tag_next(struct mbuf *, struct m_tag *);
886 
887 /* Packet tag types */
888 #define PACKET_TAG_NONE				0  /* Nothing */
889 #define PACKET_TAG_VLAN				1  /* VLAN ID */
890 #define PACKET_TAG_ENCAP			2  /* encapsulation data */
891 #define PACKET_TAG_ESP				3  /* ESP information */
892 #define PACKET_TAG_PF				11 /* packet filter */
893 #define PACKET_TAG_ALTQ_QID			12 /* ALTQ queue id */
894 
895 #define PACKET_TAG_IPSEC_IN_CRYPTO_DONE		16
896 #define PACKET_TAG_IPSEC_IN_DONE		17
897 #define PACKET_TAG_IPSEC_OUT_DONE		18
898 #define	PACKET_TAG_IPSEC_OUT_CRYPTO_NEEDED	19  /* NIC IPsec crypto req'ed */
899 #define	PACKET_TAG_IPSEC_IN_COULD_DO_CRYPTO	20  /* NIC notifies IPsec */
900 #define	PACKET_TAG_IPSEC_PENDING_TDB		21  /* Reminder to do IPsec */
901 
902 #define	PACKET_TAG_IPSEC_SOCKET			22 /* IPSEC socket ref */
903 #define	PACKET_TAG_IPSEC_HISTORY		23 /* IPSEC history */
904 
905 #define	PACKET_TAG_IPSEC_NAT_T_PORTS		25 /* two uint16_t */
906 
907 #define	PACKET_TAG_INET6			26 /* IPv6 info */
908 
909 #define	PACKET_TAG_ECO_RETRYPARMS		27 /* Econet retry parameters */
910 
911 #define	PACKET_TAG_TUNNEL_INFO			28 /* tunnel identification and
912 						    * protocol callback, for
913 						    * loop detection/recovery
914 						    */
915 
916 #define	PACKET_TAG_MPLS				29 /* Indicate it's for MPLS */
917 
918 /*
919  * Return the number of bytes in the mbuf chain, m.
920  */
921 static __inline u_int
m_length(const struct mbuf * m)922 m_length(const struct mbuf *m)
923 {
924 	const struct mbuf *m0;
925 	u_int pktlen;
926 
927 	if ((m->m_flags & M_PKTHDR) != 0)
928 		return m->m_pkthdr.len;
929 
930 	pktlen = 0;
931 	for (m0 = m; m0 != NULL; m0 = m0->m_next)
932 		pktlen += m0->m_len;
933 	return pktlen;
934 }
935 
936 void m_print(const struct mbuf *, const char *, void (*)(const char *, ...)
937     __printflike(1, 2));
938 
939 #endif /* _KERNEL */
940 #endif /* !_SYS_MBUF_H_ */
941