1 /* $NetBSD: mbuf.h,v 1.158 2015/06/04 09:19:59 ozaki-r Exp $ */
2
3 /*-
4 * Copyright (c) 1996, 1997, 1999, 2001, 2007 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center and Matt Thomas of 3am Software Foundry.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*
34 * Copyright (c) 1982, 1986, 1988, 1993
35 * The Regents of the University of California. All rights reserved.
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions
39 * are met:
40 * 1. Redistributions of source code must retain the above copyright
41 * notice, this list of conditions and the following disclaimer.
42 * 2. Redistributions in binary form must reproduce the above copyright
43 * notice, this list of conditions and the following disclaimer in the
44 * documentation and/or other materials provided with the distribution.
45 * 3. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)mbuf.h 8.5 (Berkeley) 2/19/95
62 */
63
64 #ifndef _SYS_MBUF_H_
65 #define _SYS_MBUF_H_
66
67 #ifdef _KERNEL_OPT
68 #include "opt_mbuftrace.h"
69 #endif
70
71 #ifndef M_WAITOK
72 #include <sys/malloc.h>
73 #endif
74 #include <sys/pool.h>
75 #include <sys/queue.h>
76 #if defined(_KERNEL)
77 #include <sys/percpu_types.h>
78 #endif /* defined(_KERNEL) */
79
80 /* For offsetof() */
81 #if defined(_KERNEL) || defined(_STANDALONE)
82 #include <sys/systm.h>
83 #else
84 #include <stddef.h>
85 #endif
86
87 #include <uvm/uvm_param.h> /* for MIN_PAGE_SIZE */
88
89 /*
90 * Mbufs are of a single size, MSIZE (machine/param.h), which
91 * includes overhead. An mbuf may add a single "mbuf cluster" of size
92 * MCLBYTES (also in machine/param.h), which has no additional overhead
93 * and is used instead of the internal data area; this is done when
94 * at least MINCLSIZE of data must be stored.
95 */
96
97 /* Packet tags structure */
98 struct m_tag {
99 SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */
100 uint16_t m_tag_id; /* Tag ID */
101 uint16_t m_tag_len; /* Length of data */
102 };
103
104 /* mbuf ownership structure */
105 struct mowner {
106 char mo_name[16]; /* owner name (fxp0) */
107 char mo_descr[16]; /* owner description (input) */
108 LIST_ENTRY(mowner) mo_link; /* */
109 struct percpu *mo_counters;
110 };
111
112 #define MOWNER_INIT(x, y) { .mo_name = x, .mo_descr = y }
113
114 enum mowner_counter_index {
115 MOWNER_COUNTER_CLAIMS, /* # of small mbuf claimed */
116 MOWNER_COUNTER_RELEASES, /* # of small mbuf released */
117 MOWNER_COUNTER_CLUSTER_CLAIMS, /* # of M_CLUSTER mbuf claimed */
118 MOWNER_COUNTER_CLUSTER_RELEASES,/* # of M_CLUSTER mbuf released */
119 MOWNER_COUNTER_EXT_CLAIMS, /* # of M_EXT mbuf claimed */
120 MOWNER_COUNTER_EXT_RELEASES, /* # of M_EXT mbuf released */
121
122 MOWNER_COUNTER_NCOUNTERS,
123 };
124
125 #if defined(_KERNEL)
126 struct mowner_counter {
127 u_long mc_counter[MOWNER_COUNTER_NCOUNTERS];
128 };
129 #endif /* defined(_KERNEL) */
130
131 /* userland-exported version of struct mowner */
132 struct mowner_user {
133 char mo_name[16]; /* owner name (fxp0) */
134 char mo_descr[16]; /* owner description (input) */
135 LIST_ENTRY(mowner) mo_link; /* unused padding; for compatibility */
136 u_long mo_counter[MOWNER_COUNTER_NCOUNTERS]; /* counters */
137 };
138
139 /*
140 * Macros for type conversion
141 * mtod(m,t) - convert mbuf pointer to data pointer of correct type
142 */
143 #define mtod(m, t) ((t)((m)->m_data))
144
145 /* header at beginning of each mbuf: */
146 struct m_hdr {
147 struct mbuf *mh_next; /* next buffer in chain */
148 struct mbuf *mh_nextpkt; /* next chain in queue/record */
149 char *mh_data; /* location of data */
150 struct mowner *mh_owner; /* mbuf owner */
151 int mh_len; /* amount of data in this mbuf */
152 int mh_flags; /* flags; see below */
153 paddr_t mh_paddr; /* physical address of mbuf */
154 short mh_type; /* type of data in this mbuf */
155 };
156
157 /*
158 * record/packet header in first mbuf of chain; valid if M_PKTHDR set
159 *
160 * A note about csum_data: For the out-bound direction, the low 16 bits
161 * indicates the offset after the L4 header where the final L4 checksum value
162 * is to be stored and the high 16 bits is the length of the L3 header (the
163 * start of the data to be checksumed). For the in-bound direction, it is only
164 * valid if the M_CSUM_DATA flag is set. In this case, an L4 checksum has been
165 * calculated by hardware, but it is up to software to perform final
166 * verification.
167 *
168 * Note for in-bound TCP/UDP checksums, we expect the csum_data to NOT
169 * be bit-wise inverted (the final step in the calculation of an IP
170 * checksum) -- this is so we can accumulate the checksum for fragmented
171 * packets during reassembly.
172 */
173 struct pkthdr {
174 struct ifnet *rcvif; /* rcv interface */
175 SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */
176 int len; /* total packet length */
177 int csum_flags; /* checksum flags */
178 uint32_t csum_data; /* checksum data */
179 u_int segsz; /* segment size */
180 };
181
182 /*
183 * Note: These bits are carefully arrange so that the compiler can have
184 * a prayer of generating a jump table.
185 */
186 #define M_CSUM_TCPv4 0x00000001 /* TCP header/payload */
187 #define M_CSUM_UDPv4 0x00000002 /* UDP header/payload */
188 #define M_CSUM_TCP_UDP_BAD 0x00000004 /* TCP/UDP checksum bad */
189 #define M_CSUM_DATA 0x00000008 /* consult csum_data */
190 #define M_CSUM_TCPv6 0x00000010 /* IPv6 TCP header/payload */
191 #define M_CSUM_UDPv6 0x00000020 /* IPv6 UDP header/payload */
192 #define M_CSUM_IPv4 0x00000040 /* IPv4 header */
193 #define M_CSUM_IPv4_BAD 0x00000080 /* IPv4 header checksum bad */
194 #define M_CSUM_TSOv4 0x00000100 /* TCPv4 segmentation offload */
195 #define M_CSUM_TSOv6 0x00000200 /* TCPv6 segmentation offload */
196
197 /* Checksum-assist quirks: keep separate from jump-table bits. */
198 #define M_CSUM_NO_PSEUDOHDR 0x80000000 /* Rx csum_data does not include
199 * the UDP/TCP pseudo-hdr, and
200 * is not yet 1s-complemented.
201 */
202
203 #define M_CSUM_BITS \
204 "\20\1TCPv4\2UDPv4\3TCP_UDP_BAD\4DATA\5TCPv6\6UDPv6\7IPv4\10IPv4_BAD" \
205 "\11TSOv4\12TSOv6\40NO_PSEUDOHDR"
206
207 /*
208 * Macros for manipulating csum_data on outgoing packets. These are
209 * used to pass information down from the L4/L3 to the L2.
210 */
211 #define M_CSUM_DATA_IPv4_IPHL(x) ((x) >> 16)
212 #define M_CSUM_DATA_IPv4_OFFSET(x) ((x) & 0xffff)
213
214 /*
215 * Macros for M_CSUM_TCPv6 and M_CSUM_UDPv6
216 *
217 * M_CSUM_DATA_IPv6_HL: length of ip6_hdr + ext header.
218 * ie. offset of UDP/TCP header in the packet.
219 *
220 * M_CSUM_DATA_IPv6_OFFSET: offset of the checksum field in UDP/TCP header.
221 */
222
223 #define M_CSUM_DATA_IPv6_HL(x) ((x) >> 16)
224 #define M_CSUM_DATA_IPv6_HL_SET(x, v) (x) = ((x) & 0xffff) | ((v) << 16)
225 #define M_CSUM_DATA_IPv6_OFFSET(x) ((x) & 0xffff)
226
227 /*
228 * Max # of pages we can attach to m_ext. This is carefully chosen
229 * to be able to handle SOSEND_LOAN_CHUNK with our minimum sized page.
230 */
231 #ifdef MIN_PAGE_SIZE
232 #define M_EXT_MAXPAGES ((65536 / MIN_PAGE_SIZE) + 1)
233 #endif
234
235 /* description of external storage mapped into mbuf, valid if M_EXT set */
236 struct _m_ext_storage {
237 unsigned int ext_refcnt;
238 int ext_flags;
239 char *ext_buf; /* start of buffer */
240 void (*ext_free) /* free routine if not the usual */
241 (struct mbuf *, void *, size_t, void *);
242 void *ext_arg; /* argument for ext_free */
243 size_t ext_size; /* size of buffer, for ext_free */
244 union {
245 paddr_t extun_paddr; /* physical address (M_EXT_CLUSTER) */
246 /* pages (M_EXT_PAGES) */
247 /*
248 * XXX This is gross, but it doesn't really matter; this is
249 * XXX overlaid on top of the mbuf data area.
250 */
251 #ifdef M_EXT_MAXPAGES
252 struct vm_page *extun_pgs[M_EXT_MAXPAGES];
253 #endif
254 } ext_un;
255 #define ext_paddr ext_un.extun_paddr
256 #define ext_pgs ext_un.extun_pgs
257 #ifdef DEBUG
258 const char *ext_ofile;
259 const char *ext_nfile;
260 int ext_oline;
261 int ext_nline;
262 #endif
263 };
264
265 struct _m_ext {
266 struct mbuf *ext_ref;
267 struct _m_ext_storage ext_storage;
268 };
269
270 #define M_PADDR_INVALID POOL_PADDR_INVALID
271
272 /*
273 * Definition of "struct mbuf".
274 * Don't change this without understanding how MHLEN/MLEN are defined.
275 */
276 #define MBUF_DEFINE(name, mhlen, mlen) \
277 struct name { \
278 struct m_hdr m_hdr; \
279 union { \
280 struct { \
281 struct pkthdr MH_pkthdr; \
282 union { \
283 struct _m_ext MH_ext; \
284 char MH_databuf[(mhlen)]; \
285 } MH_dat; \
286 } MH; \
287 char M_databuf[(mlen)]; \
288 } M_dat; \
289 }
290 #define m_next m_hdr.mh_next
291 #define m_len m_hdr.mh_len
292 #define m_data m_hdr.mh_data
293 #define m_owner m_hdr.mh_owner
294 #define m_type m_hdr.mh_type
295 #define m_flags m_hdr.mh_flags
296 #define m_nextpkt m_hdr.mh_nextpkt
297 #define m_paddr m_hdr.mh_paddr
298 #define m_pkthdr M_dat.MH.MH_pkthdr
299 #define m_ext_storage M_dat.MH.MH_dat.MH_ext.ext_storage
300 #define m_ext_ref M_dat.MH.MH_dat.MH_ext.ext_ref
301 #define m_ext m_ext_ref->m_ext_storage
302 #define m_pktdat M_dat.MH.MH_dat.MH_databuf
303 #define m_dat M_dat.M_databuf
304
305 /*
306 * Dummy mbuf structure to calculate the right values for MLEN/MHLEN, taking
307 * into account inter-structure padding.
308 */
309 MBUF_DEFINE(_mbuf_dummy, 1, 1);
310
311 /* normal data len */
312 #define MLEN (MSIZE - offsetof(struct _mbuf_dummy, m_dat))
313 /* data len w/pkthdr */
314 #define MHLEN (MSIZE - offsetof(struct _mbuf_dummy, m_pktdat))
315
316 #define MINCLSIZE (MHLEN+MLEN+1) /* smallest amount to put in cluster */
317 #define M_MAXCOMPRESS (MHLEN / 2) /* max amount to copy for compression */
318
319 /*
320 * The *real* struct mbuf
321 */
322 MBUF_DEFINE(mbuf, MHLEN, MLEN);
323
324 /* mbuf flags */
325 #define M_EXT 0x00000001 /* has associated external storage */
326 #define M_PKTHDR 0x00000002 /* start of record */
327 #define M_EOR 0x00000004 /* end of record */
328 #define M_PROTO1 0x00000008 /* protocol-specific */
329
330 /* mbuf pkthdr flags, also in m_flags */
331 #define M_AUTHIPHDR 0x00000010 /* data origin authentication for
332 * IP header */
333 #define M_DECRYPTED 0x00000020 /* confidentiality */
334 #define M_LOOP 0x00000040 /* for Mbuf statistics */
335 #define M_AUTHIPDGM 0x00000080 /* data origin authentication */
336 #define M_BCAST 0x00000100 /* send/received as link-level
337 * broadcast */
338 #define M_MCAST 0x00000200 /* send/received as link-level
339 * multicast */
340 #define M_CANFASTFWD 0x00000400 /* used by filters to indicate
341 * packet can be fast-forwarded */
342 #define M_ANYCAST6 0x00000800 /* received as IPv6 anycast */
343
344 #define M_LINK0 0x00001000 /* link layer specific flag */
345 #define M_LINK1 0x00002000 /* link layer specific flag */
346 #define M_LINK2 0x00004000 /* link layer specific flag */
347
348 #define M_LINK3 0x00008000 /* link layer specific flag */
349 #define M_LINK4 0x00010000 /* link layer specific flag */
350 #define M_LINK5 0x00020000 /* link layer specific flag */
351 #define M_LINK6 0x00040000 /* link layer specific flag */
352 #define M_LINK7 0x00080000 /* link layer specific flag */
353
354 /* additional flags for M_EXT mbufs */
355 #define M_EXT_FLAGS 0xff000000
356 #define M_EXT_CLUSTER 0x01000000 /* ext is a cluster */
357 #define M_EXT_PAGES 0x02000000 /* ext_pgs is valid */
358 #define M_EXT_ROMAP 0x04000000 /* ext mapping is r-o at MMU */
359 #define M_EXT_RW 0x08000000 /* ext storage is writable */
360
361 /* for source-level compatibility */
362 #define M_CLUSTER M_EXT_CLUSTER
363
364 #define M_FLAGS_BITS \
365 "\20\1EXT\2PKTHDR\3EOR\4PROTO1\5AUTHIPHDR\6DECRYPTED\7LOOP\10AUTHIPDGM" \
366 "\11BCAST\12MCAST\13CANFASTFWD\14ANYCAST6\15LINK0\16LINK1\17LINK2\20LINK3" \
367 "\21LINK4\22LINK5\23LINK6\24LINK7" \
368 "\31EXT_CLUSTER\32EXT_PAGES\33EXT_ROMAP\34EXT_RW"
369
370 /* flags copied when copying m_pkthdr */
371 #define M_COPYFLAGS (M_PKTHDR|M_EOR|M_BCAST|M_MCAST|M_CANFASTFWD|M_ANYCAST6|M_LINK0|M_LINK1|M_LINK2|M_AUTHIPHDR|M_DECRYPTED|M_LOOP|M_AUTHIPDGM)
372
373 /* flag copied when shallow-copying external storage */
374 #define M_EXTCOPYFLAGS (M_EXT|M_EXT_FLAGS)
375
376 /* mbuf types */
377 #define MT_FREE 0 /* should be on free list */
378 #define MT_DATA 1 /* dynamic (data) allocation */
379 #define MT_HEADER 2 /* packet header */
380 #define MT_SONAME 3 /* socket name */
381 #define MT_SOOPTS 4 /* socket options */
382 #define MT_FTABLE 5 /* fragment reassembly header */
383 #define MT_CONTROL 6 /* extra-data protocol message */
384 #define MT_OOBDATA 7 /* expedited data */
385
386 #ifdef MBUFTYPES
387 static const char * const mbuftypes[] = {
388 "mbfree",
389 "mbdata",
390 "mbheader",
391 "mbsoname",
392 "mbsopts",
393 "mbftable",
394 "mbcontrol",
395 "mboobdata",
396 };
397 #endif
398
399 /* flags to m_get/MGET */
400 #define M_DONTWAIT M_NOWAIT
401 #define M_WAIT M_WAITOK
402
403 #ifdef MBUFTRACE
404 /*
405 * mbuf allocation tracing
406 */
407 void mowner_init(struct mbuf *, int);
408 void mowner_ref(struct mbuf *, int);
409 void m_claim(struct mbuf *, struct mowner *);
410 void mowner_revoke(struct mbuf *, bool, int);
411 void mowner_attach(struct mowner *);
412 void mowner_detach(struct mowner *);
413 void m_claimm(struct mbuf *, struct mowner *);
414 #else
415 #define mowner_init(m, type) do { } while (/* CONSTCOND */ 0)
416 #define mowner_ref(m, flags) do { } while (/* CONSTCOND */ 0)
417 #define mowner_revoke(m, all, flags) do { } while (/* CONSTCOND */ 0)
418 #define m_claim(m, mowner) do { } while (/* CONSTCOND */ 0)
419 #define mowner_attach(mo) do { } while (/* CONSTCOND */ 0)
420 #define mowner_detach(mo) do { } while (/* CONSTCOND */ 0)
421 #define m_claimm(m, mo) do { } while (/* CONSTCOND */ 0)
422 #endif
423
424 #define MCLAIM(m, mo) m_claim((m), (mo))
425 #define MOWNER_ATTACH(mo) mowner_attach(mo)
426 #define MOWNER_DETACH(mo) mowner_detach(mo)
427
428 /*
429 * mbuf allocation/deallocation macros:
430 *
431 * MGET(struct mbuf *m, int how, int type)
432 * allocates an mbuf and initializes it to contain internal data.
433 *
434 * MGETHDR(struct mbuf *m, int how, int type)
435 * allocates an mbuf and initializes it to contain a packet header
436 * and internal data.
437 *
438 * If 'how' is M_WAIT, these macros (and the corresponding functions)
439 * are guaranteed to return successfully.
440 */
441 #define MGET(m, how, type) m = m_get((how), (type))
442 #define MGETHDR(m, how, type) m = m_gethdr((how), (type))
443
444 #if defined(_KERNEL)
445 #define _M_
446 /*
447 * Macros for tracking external storage associated with an mbuf.
448 */
449 #ifdef DEBUG
450 #define MCLREFDEBUGN(m, file, line) \
451 do { \
452 (m)->m_ext.ext_nfile = (file); \
453 (m)->m_ext.ext_nline = (line); \
454 } while (/* CONSTCOND */ 0)
455
456 #define MCLREFDEBUGO(m, file, line) \
457 do { \
458 (m)->m_ext.ext_ofile = (file); \
459 (m)->m_ext.ext_oline = (line); \
460 } while (/* CONSTCOND */ 0)
461 #else
462 #define MCLREFDEBUGN(m, file, line)
463 #define MCLREFDEBUGO(m, file, line)
464 #endif
465
466 #define MCLINITREFERENCE(m) \
467 do { \
468 KDASSERT(((m)->m_flags & M_EXT) == 0); \
469 (m)->m_ext_ref = (m); \
470 (m)->m_ext.ext_refcnt = 1; \
471 MCLREFDEBUGO((m), __FILE__, __LINE__); \
472 MCLREFDEBUGN((m), NULL, 0); \
473 } while (/* CONSTCOND */ 0)
474
475 /*
476 * Macros for mbuf external storage.
477 *
478 * MCLGET allocates and adds an mbuf cluster to a normal mbuf;
479 * the flag M_EXT is set upon success.
480 *
481 * MEXTMALLOC allocates external storage and adds it to
482 * a normal mbuf; the flag M_EXT is set upon success.
483 *
484 * MEXTADD adds pre-allocated external storage to
485 * a normal mbuf; the flag M_EXT is set upon success.
486 */
487
488 #define _MCLGET(m, pool_cache, size, how) \
489 do { \
490 (m)->m_ext_storage.ext_buf = (char *) \
491 pool_cache_get_paddr((pool_cache), \
492 (how) == M_WAIT ? (PR_WAITOK|PR_LIMITFAIL) : 0, \
493 &(m)->m_ext_storage.ext_paddr); \
494 if ((m)->m_ext_storage.ext_buf != NULL) { \
495 MCLINITREFERENCE(m); \
496 (m)->m_data = (m)->m_ext.ext_buf; \
497 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | \
498 M_EXT|M_CLUSTER|M_EXT_RW; \
499 (m)->m_ext.ext_flags = 0; \
500 (m)->m_ext.ext_size = (size); \
501 (m)->m_ext.ext_free = NULL; \
502 (m)->m_ext.ext_arg = (pool_cache); \
503 /* ext_paddr initialized above */ \
504 mowner_ref((m), M_EXT|M_CLUSTER); \
505 } \
506 } while (/* CONSTCOND */ 0)
507
508 /*
509 * The standard mbuf cluster pool.
510 */
511 #define MCLGET(m, how) _MCLGET((m), mcl_cache, MCLBYTES, (how))
512
513 #define MEXTMALLOC(m, size, how) \
514 do { \
515 (m)->m_ext_storage.ext_buf = (char *) \
516 malloc((size), mbtypes[(m)->m_type], (how)); \
517 if ((m)->m_ext_storage.ext_buf != NULL) { \
518 MCLINITREFERENCE(m); \
519 (m)->m_data = (m)->m_ext.ext_buf; \
520 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | \
521 M_EXT|M_EXT_RW; \
522 (m)->m_ext.ext_flags = 0; \
523 (m)->m_ext.ext_size = (size); \
524 (m)->m_ext.ext_free = NULL; \
525 (m)->m_ext.ext_arg = NULL; \
526 mowner_ref((m), M_EXT); \
527 } \
528 } while (/* CONSTCOND */ 0)
529
530 #define MEXTADD(m, buf, size, type, free, arg) \
531 do { \
532 MCLINITREFERENCE(m); \
533 (m)->m_data = (m)->m_ext.ext_buf = (char *)(buf); \
534 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | M_EXT; \
535 (m)->m_ext.ext_flags = 0; \
536 (m)->m_ext.ext_size = (size); \
537 (m)->m_ext.ext_free = (free); \
538 (m)->m_ext.ext_arg = (arg); \
539 mowner_ref((m), M_EXT); \
540 } while (/* CONSTCOND */ 0)
541
542 /*
543 * Reset the data pointer on an mbuf.
544 */
545 #define MRESETDATA(m) \
546 do { \
547 if ((m)->m_flags & M_EXT) \
548 (m)->m_data = (m)->m_ext.ext_buf; \
549 else if ((m)->m_flags & M_PKTHDR) \
550 (m)->m_data = (m)->m_pktdat; \
551 else \
552 (m)->m_data = (m)->m_dat; \
553 } while (/* CONSTCOND */ 0)
554
555 /*
556 * MFREE(struct mbuf *m, struct mbuf *n)
557 * Free a single mbuf and associated external storage.
558 * Place the successor, if any, in n.
559 */
560 #define MFREE(m, n) \
561 mowner_revoke((m), 1, (m)->m_flags); \
562 mbstat_type_add((m)->m_type, -1); \
563 if ((m)->m_flags & M_PKTHDR) \
564 m_tag_delete_chain((m), NULL); \
565 (n) = (m)->m_next; \
566 if ((m)->m_flags & M_EXT) { \
567 m_ext_free((m)); \
568 } else { \
569 KASSERT((m)->m_type != MT_FREE); \
570 (m)->m_type = MT_FREE; \
571 pool_cache_put(mb_cache, (m)); \
572 } \
573
574 /*
575 * Copy mbuf pkthdr from `from' to `to'.
576 * `from' must have M_PKTHDR set, and `to' must be empty.
577 */
578 #define M_COPY_PKTHDR(to, from) \
579 do { \
580 (to)->m_pkthdr = (from)->m_pkthdr; \
581 (to)->m_flags = (from)->m_flags & M_COPYFLAGS; \
582 SLIST_INIT(&(to)->m_pkthdr.tags); \
583 m_tag_copy_chain((to), (from)); \
584 (to)->m_data = (to)->m_pktdat; \
585 } while (/* CONSTCOND */ 0)
586
587 /*
588 * Move mbuf pkthdr from `from' to `to'.
589 * `from' must have M_PKTHDR set, and `to' must be empty.
590 */
591 #define M_MOVE_PKTHDR(to, from) m_move_pkthdr(to, from)
592
593 /*
594 * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place
595 * an object of the specified size at the end of the mbuf, longword aligned.
596 */
597 #define M_ALIGN(m, len) \
598 do { \
599 (m)->m_data += (MLEN - (len)) &~ (sizeof(long) - 1); \
600 } while (/* CONSTCOND */ 0)
601
602 /*
603 * As above, for mbufs allocated with m_gethdr/MGETHDR
604 * or initialized by M_COPY_PKTHDR.
605 */
606 #define MH_ALIGN(m, len) \
607 do { \
608 (m)->m_data += (MHLEN - (len)) &~ (sizeof(long) - 1); \
609 } while (/* CONSTCOND */ 0)
610
611 /*
612 * Determine if an mbuf's data area is read-only. This is true
613 * if external storage is read-only mapped, or not marked as R/W,
614 * or referenced by more than one mbuf.
615 */
616 #define M_READONLY(m) \
617 (((m)->m_flags & M_EXT) != 0 && \
618 (((m)->m_flags & (M_EXT_ROMAP|M_EXT_RW)) != M_EXT_RW || \
619 (m)->m_ext.ext_refcnt > 1))
620
621 #define M_UNWRITABLE(__m, __len) \
622 ((__m)->m_len < (__len) || M_READONLY((__m)))
623 /*
624 * Determine if an mbuf's data area is read-only at the MMU.
625 */
626 #define M_ROMAP(m) \
627 (((m)->m_flags & (M_EXT|M_EXT_ROMAP)) == (M_EXT|M_EXT_ROMAP))
628
629 /*
630 * Compute the amount of space available
631 * before the current start of data in an mbuf.
632 */
633 #define _M_LEADINGSPACE(m) \
634 ((m)->m_flags & M_EXT ? (m)->m_data - (m)->m_ext.ext_buf : \
635 (m)->m_flags & M_PKTHDR ? (m)->m_data - (m)->m_pktdat : \
636 (m)->m_data - (m)->m_dat)
637
638 #define M_LEADINGSPACE(m) \
639 (M_READONLY((m)) ? 0 : _M_LEADINGSPACE((m)))
640
641 /*
642 * Compute the amount of space available
643 * after the end of data in an mbuf.
644 */
645 #define _M_TRAILINGSPACE(m) \
646 ((m)->m_flags & M_EXT ? (m)->m_ext.ext_buf + (m)->m_ext.ext_size - \
647 ((m)->m_data + (m)->m_len) : \
648 &(m)->m_dat[MLEN] - ((m)->m_data + (m)->m_len))
649
650 #define M_TRAILINGSPACE(m) \
651 (M_READONLY((m)) ? 0 : _M_TRAILINGSPACE((m)))
652
653 /*
654 * Compute the address of an mbuf's data area.
655 */
656 #define M_BUFADDR(m) \
657 (((m)->m_flags & M_PKTHDR) ? (m)->m_pktdat : (m)->m_dat)
658
659 /*
660 * Compute the offset of the beginning of the data buffer of a non-ext
661 * mbuf.
662 */
663 #define M_BUFOFFSET(m) \
664 (((m)->m_flags & M_PKTHDR) ? \
665 offsetof(struct mbuf, m_pktdat) : offsetof(struct mbuf, m_dat))
666
667 /*
668 * Arrange to prepend space of size plen to mbuf m.
669 * If a new mbuf must be allocated, how specifies whether to wait.
670 * If how is M_DONTWAIT and allocation fails, the original mbuf chain
671 * is freed and m is set to NULL.
672 */
673 #define M_PREPEND(m, plen, how) \
674 do { \
675 if (M_LEADINGSPACE(m) >= (plen)) { \
676 (m)->m_data -= (plen); \
677 (m)->m_len += (plen); \
678 } else \
679 (m) = m_prepend((m), (plen), (how)); \
680 if ((m) && (m)->m_flags & M_PKTHDR) \
681 (m)->m_pkthdr.len += (plen); \
682 } while (/* CONSTCOND */ 0)
683
684 /* change mbuf to new type */
685 #define MCHTYPE(m, t) \
686 do { \
687 KASSERT((t) != MT_FREE); \
688 mbstat_type_add((m)->m_type, -1); \
689 mbstat_type_add(t, 1); \
690 (m)->m_type = t; \
691 } while (/* CONSTCOND */ 0)
692
693 /* length to m_copy to copy all */
694 #define M_COPYALL -1
695
696 /* compatibility with 4.3 */
697 #define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT)
698
699 /*
700 * Allow drivers and/or protocols to use the rcvif member of
701 * PKTHDR mbufs to store private context information.
702 */
703 #define M_GETCTX(m, t) ((t)(m)->m_pkthdr.rcvif)
704 #define M_SETCTX(m, c) ((void)((m)->m_pkthdr.rcvif = (void *)(c)))
705
706 #endif /* defined(_KERNEL) */
707
708 /*
709 * Simple mbuf queueing system
710 *
711 * this is basically a SIMPLEQ adapted to mbuf use (ie using
712 * m_nextpkt instead of field.sqe_next).
713 *
714 * m_next is ignored, so queueing chains of mbufs is possible
715 */
716 #define MBUFQ_HEAD(name) \
717 struct name { \
718 struct mbuf *mq_first; \
719 struct mbuf **mq_last; \
720 }
721
722 #define MBUFQ_INIT(q) do { \
723 (q)->mq_first = NULL; \
724 (q)->mq_last = &(q)->mq_first; \
725 } while (/*CONSTCOND*/0)
726
727 #define MBUFQ_ENQUEUE(q, m) do { \
728 (m)->m_nextpkt = NULL; \
729 *(q)->mq_last = (m); \
730 (q)->mq_last = &(m)->m_nextpkt; \
731 } while (/*CONSTCOND*/0)
732
733 #define MBUFQ_PREPEND(q, m) do { \
734 if (((m)->m_nextpkt = (q)->mq_first) == NULL) \
735 (q)->mq_last = &(m)->m_nextpkt; \
736 (q)->mq_first = (m); \
737 } while (/*CONSTCOND*/0)
738
739 #define MBUFQ_DEQUEUE(q, m) do { \
740 if (((m) = (q)->mq_first) != NULL) { \
741 if (((q)->mq_first = (m)->m_nextpkt) == NULL) \
742 (q)->mq_last = &(q)->mq_first; \
743 else \
744 (m)->m_nextpkt = NULL; \
745 } \
746 } while (/*CONSTCOND*/0)
747
748 #define MBUFQ_DRAIN(q) do { \
749 struct mbuf *__m0; \
750 while ((__m0 = (q)->mq_first) != NULL) { \
751 (q)->mq_first = __m0->m_nextpkt; \
752 m_freem(__m0); \
753 } \
754 (q)->mq_last = &(q)->mq_first; \
755 } while (/*CONSTCOND*/0)
756
757 #define MBUFQ_FIRST(q) ((q)->mq_first)
758 #define MBUFQ_NEXT(m) ((m)->m_nextpkt)
759 #define MBUFQ_LAST(q) (*(q)->mq_last)
760
761 /*
762 * Mbuf statistics.
763 * For statistics related to mbuf and cluster allocations, see also the
764 * pool headers (mb_cache and mcl_cache).
765 */
766 struct mbstat {
767 u_long _m_spare; /* formerly m_mbufs */
768 u_long _m_spare1; /* formerly m_clusters */
769 u_long _m_spare2; /* spare field */
770 u_long _m_spare3; /* formely m_clfree - free clusters */
771 u_long m_drops; /* times failed to find space */
772 u_long m_wait; /* times waited for space */
773 u_long m_drain; /* times drained protocols for space */
774 u_short m_mtypes[256]; /* type specific mbuf allocations */
775 };
776
777 struct mbstat_cpu {
778 u_int m_mtypes[256]; /* type specific mbuf allocations */
779 };
780
781 /*
782 * Mbuf sysctl variables.
783 */
784 #define MBUF_MSIZE 1 /* int: mbuf base size */
785 #define MBUF_MCLBYTES 2 /* int: mbuf cluster size */
786 #define MBUF_NMBCLUSTERS 3 /* int: limit on the # of clusters */
787 #define MBUF_MBLOWAT 4 /* int: mbuf low water mark */
788 #define MBUF_MCLLOWAT 5 /* int: mbuf cluster low water mark */
789 #define MBUF_STATS 6 /* struct: mbstat */
790 #define MBUF_MOWNERS 7 /* struct: m_owner[] */
791 #define MBUF_MAXID 8 /* number of valid MBUF ids */
792
793 #define CTL_MBUF_NAMES { \
794 { 0, 0 }, \
795 { "msize", CTLTYPE_INT }, \
796 { "mclbytes", CTLTYPE_INT }, \
797 { "nmbclusters", CTLTYPE_INT }, \
798 { "mblowat", CTLTYPE_INT }, \
799 { "mcllowat", CTLTYPE_INT }, \
800 { 0 /* "stats" */, CTLTYPE_STRUCT }, \
801 { 0 /* "mowners" */, CTLTYPE_STRUCT }, \
802 }
803
804 #ifdef _KERNEL
805 extern struct mbstat mbstat;
806 extern int nmbclusters; /* limit on the # of clusters */
807 extern int mblowat; /* mbuf low water mark */
808 extern int mcllowat; /* mbuf cluster low water mark */
809 extern int max_linkhdr; /* largest link-level header */
810 extern int max_protohdr; /* largest protocol header */
811 extern int max_hdr; /* largest link+protocol header */
812 extern int max_datalen; /* MHLEN - max_hdr */
813 extern const int msize; /* mbuf base size */
814 extern const int mclbytes; /* mbuf cluster size */
815 extern pool_cache_t mb_cache;
816 extern pool_cache_t mcl_cache;
817 #ifdef MBUFTRACE
818 LIST_HEAD(mownerhead, mowner);
819 extern struct mownerhead mowners;
820 extern struct mowner unknown_mowners[];
821 extern struct mowner revoked_mowner;
822 #endif
823
824 MALLOC_DECLARE(M_MBUF);
825 MALLOC_DECLARE(M_SONAME);
826
827 struct mbuf *m_copym(struct mbuf *, int, int, int);
828 struct mbuf *m_copypacket(struct mbuf *, int);
829 struct mbuf *m_devget(char *, int, int, struct ifnet *,
830 void (*copy)(const void *, void *, size_t));
831 struct mbuf *m_dup(struct mbuf *, int, int, int);
832 struct mbuf *m_free(struct mbuf *);
833 struct mbuf *m_get(int, int);
834 struct mbuf *m_getclr(int, int);
835 struct mbuf *m_gethdr(int, int);
836 struct mbuf *m_prepend(struct mbuf *,int, int);
837 struct mbuf *m_pulldown(struct mbuf *, int, int, int *);
838 struct mbuf *m_pullup(struct mbuf *, int);
839 struct mbuf *m_copyup(struct mbuf *, int, int);
840 struct mbuf *m_split(struct mbuf *,int, int);
841 struct mbuf *m_getptr(struct mbuf *, int, int *);
842 void m_adj(struct mbuf *, int);
843 struct mbuf *m_defrag(struct mbuf *, int);
844 int m_apply(struct mbuf *, int, int,
845 int (*)(void *, void *, unsigned int), void *);
846 void m_cat(struct mbuf *,struct mbuf *);
847 void m_clget(struct mbuf *, int);
848 int m_mballoc(int, int);
849 void m_copyback(struct mbuf *, int, int, const void *);
850 struct mbuf *m_copyback_cow(struct mbuf *, int, int, const void *, int);
851 int m_makewritable(struct mbuf **, int, int, int);
852 struct mbuf *m_getcl(int, int, int);
853 void m_copydata(struct mbuf *, int, int, void *);
854 void m_freem(struct mbuf *);
855 void m_reclaim(void *, int);
856 void mbinit(void);
857 void m_ext_free(struct mbuf *);
858 char * m_mapin(struct mbuf *);
859 void m_move_pkthdr(struct mbuf *, struct mbuf *);
860
861 bool m_ensure_contig(struct mbuf **, int);
862 struct mbuf *m_add(struct mbuf *, struct mbuf *);
863 void m_align(struct mbuf *, int);
864 int m_append(struct mbuf *, int, const void *);
865
866 /* Inline routines. */
867 static __inline u_int m_length(const struct mbuf *) __unused;
868
869 /* Statistics */
870 void mbstat_type_add(int, int);
871
872 /* Packet tag routines */
873 struct m_tag *m_tag_get(int, int, int);
874 void m_tag_free(struct m_tag *);
875 void m_tag_prepend(struct mbuf *, struct m_tag *);
876 void m_tag_unlink(struct mbuf *, struct m_tag *);
877 void m_tag_delete(struct mbuf *, struct m_tag *);
878 void m_tag_delete_chain(struct mbuf *, struct m_tag *);
879 void m_tag_delete_nonpersistent(struct mbuf *);
880 struct m_tag *m_tag_find(const struct mbuf *, int, struct m_tag *);
881 struct m_tag *m_tag_copy(struct m_tag *);
882 int m_tag_copy_chain(struct mbuf *, struct mbuf *);
883 void m_tag_init(struct mbuf *);
884 struct m_tag *m_tag_first(struct mbuf *);
885 struct m_tag *m_tag_next(struct mbuf *, struct m_tag *);
886
887 /* Packet tag types */
888 #define PACKET_TAG_NONE 0 /* Nothing */
889 #define PACKET_TAG_VLAN 1 /* VLAN ID */
890 #define PACKET_TAG_ENCAP 2 /* encapsulation data */
891 #define PACKET_TAG_ESP 3 /* ESP information */
892 #define PACKET_TAG_PF 11 /* packet filter */
893 #define PACKET_TAG_ALTQ_QID 12 /* ALTQ queue id */
894
895 #define PACKET_TAG_IPSEC_IN_CRYPTO_DONE 16
896 #define PACKET_TAG_IPSEC_IN_DONE 17
897 #define PACKET_TAG_IPSEC_OUT_DONE 18
898 #define PACKET_TAG_IPSEC_OUT_CRYPTO_NEEDED 19 /* NIC IPsec crypto req'ed */
899 #define PACKET_TAG_IPSEC_IN_COULD_DO_CRYPTO 20 /* NIC notifies IPsec */
900 #define PACKET_TAG_IPSEC_PENDING_TDB 21 /* Reminder to do IPsec */
901
902 #define PACKET_TAG_IPSEC_SOCKET 22 /* IPSEC socket ref */
903 #define PACKET_TAG_IPSEC_HISTORY 23 /* IPSEC history */
904
905 #define PACKET_TAG_IPSEC_NAT_T_PORTS 25 /* two uint16_t */
906
907 #define PACKET_TAG_INET6 26 /* IPv6 info */
908
909 #define PACKET_TAG_ECO_RETRYPARMS 27 /* Econet retry parameters */
910
911 #define PACKET_TAG_TUNNEL_INFO 28 /* tunnel identification and
912 * protocol callback, for
913 * loop detection/recovery
914 */
915
916 #define PACKET_TAG_MPLS 29 /* Indicate it's for MPLS */
917
918 /*
919 * Return the number of bytes in the mbuf chain, m.
920 */
921 static __inline u_int
m_length(const struct mbuf * m)922 m_length(const struct mbuf *m)
923 {
924 const struct mbuf *m0;
925 u_int pktlen;
926
927 if ((m->m_flags & M_PKTHDR) != 0)
928 return m->m_pkthdr.len;
929
930 pktlen = 0;
931 for (m0 = m; m0 != NULL; m0 = m0->m_next)
932 pktlen += m0->m_len;
933 return pktlen;
934 }
935
936 void m_print(const struct mbuf *, const char *, void (*)(const char *, ...)
937 __printflike(1, 2));
938
939 #endif /* _KERNEL */
940 #endif /* !_SYS_MBUF_H_ */
941