1 /* $NetBSD: mbuf.h,v 1.153 2013/11/14 00:50:36 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 1996, 1997, 1999, 2001, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and Matt Thomas of 3am Software Foundry. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1982, 1986, 1988, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Neither the name of the University nor the names of its contributors 46 * may be used to endorse or promote products derived from this software 47 * without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 59 * SUCH DAMAGE. 60 * 61 * @(#)mbuf.h 8.5 (Berkeley) 2/19/95 62 */ 63 64 #ifndef _SYS_MBUF_H_ 65 #define _SYS_MBUF_H_ 66 67 #ifdef _KERNEL_OPT 68 #include "opt_mbuftrace.h" 69 #endif 70 71 #ifndef M_WAITOK 72 #include <sys/malloc.h> 73 #endif 74 #include <sys/pool.h> 75 #include <sys/queue.h> 76 #if defined(_KERNEL) 77 #include <sys/percpu_types.h> 78 #endif /* defined(_KERNEL) */ 79 80 /* For offsetof() */ 81 #if defined(_KERNEL) || defined(_STANDALONE) 82 #include <sys/systm.h> 83 #else 84 #include <stddef.h> 85 #endif 86 87 #include <uvm/uvm_param.h> /* for MIN_PAGE_SIZE */ 88 89 /* 90 * Mbufs are of a single size, MSIZE (machine/param.h), which 91 * includes overhead. An mbuf may add a single "mbuf cluster" of size 92 * MCLBYTES (also in machine/param.h), which has no additional overhead 93 * and is used instead of the internal data area; this is done when 94 * at least MINCLSIZE of data must be stored. 95 */ 96 97 /* Packet tags structure */ 98 struct m_tag { 99 SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ 100 uint16_t m_tag_id; /* Tag ID */ 101 uint16_t m_tag_len; /* Length of data */ 102 }; 103 104 /* mbuf ownership structure */ 105 struct mowner { 106 char mo_name[16]; /* owner name (fxp0) */ 107 char mo_descr[16]; /* owner description (input) */ 108 LIST_ENTRY(mowner) mo_link; /* */ 109 struct percpu *mo_counters; 110 }; 111 112 #define MOWNER_INIT(x, y) { .mo_name = x, .mo_descr = y } 113 114 enum mowner_counter_index { 115 MOWNER_COUNTER_CLAIMS, /* # of small mbuf claimed */ 116 MOWNER_COUNTER_RELEASES, /* # of small mbuf released */ 117 MOWNER_COUNTER_CLUSTER_CLAIMS, /* # of M_CLUSTER mbuf claimed */ 118 MOWNER_COUNTER_CLUSTER_RELEASES,/* # of M_CLUSTER mbuf released */ 119 MOWNER_COUNTER_EXT_CLAIMS, /* # of M_EXT mbuf claimed */ 120 MOWNER_COUNTER_EXT_RELEASES, /* # of M_EXT mbuf released */ 121 122 MOWNER_COUNTER_NCOUNTERS, 123 }; 124 125 #if defined(_KERNEL) 126 struct mowner_counter { 127 u_long mc_counter[MOWNER_COUNTER_NCOUNTERS]; 128 }; 129 #endif /* defined(_KERNEL) */ 130 131 /* userland-exported version of struct mowner */ 132 struct mowner_user { 133 char mo_name[16]; /* owner name (fxp0) */ 134 char mo_descr[16]; /* owner description (input) */ 135 LIST_ENTRY(mowner) mo_link; /* unused padding; for compatibility */ 136 u_long mo_counter[MOWNER_COUNTER_NCOUNTERS]; /* counters */ 137 }; 138 139 /* 140 * Macros for type conversion 141 * mtod(m,t) - convert mbuf pointer to data pointer of correct type 142 */ 143 #define mtod(m, t) ((t)((m)->m_data)) 144 145 /* header at beginning of each mbuf: */ 146 struct m_hdr { 147 struct mbuf *mh_next; /* next buffer in chain */ 148 struct mbuf *mh_nextpkt; /* next chain in queue/record */ 149 char *mh_data; /* location of data */ 150 struct mowner *mh_owner; /* mbuf owner */ 151 int mh_len; /* amount of data in this mbuf */ 152 int mh_flags; /* flags; see below */ 153 paddr_t mh_paddr; /* physical address of mbuf */ 154 short mh_type; /* type of data in this mbuf */ 155 }; 156 157 /* 158 * record/packet header in first mbuf of chain; valid if M_PKTHDR set 159 * 160 * A note about csum_data: For the out-bound direction, the low 16 bits 161 * indicates the offset after the L4 header where the final L4 checksum value 162 * is to be stored and the high 16 bits is the length of the L3 header (the 163 * start of the data to be checksumed). For the in-bound direction, it is only 164 * valid if the M_CSUM_DATA flag is set. In this case, an L4 checksum has been 165 * calculated by hardware, but it is up to software to perform final 166 * verification. 167 * 168 * Note for in-bound TCP/UDP checksums, we expect the csum_data to NOT 169 * be bit-wise inverted (the final step in the calculation of an IP 170 * checksum) -- this is so we can accumulate the checksum for fragmented 171 * packets during reassembly. 172 */ 173 struct pkthdr { 174 struct ifnet *rcvif; /* rcv interface */ 175 SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */ 176 int len; /* total packet length */ 177 int csum_flags; /* checksum flags */ 178 uint32_t csum_data; /* checksum data */ 179 u_int segsz; /* segment size */ 180 }; 181 182 /* 183 * Note: These bits are carefully arrange so that the compiler can have 184 * a prayer of generating a jump table. 185 */ 186 #define M_CSUM_TCPv4 0x00000001 /* TCP header/payload */ 187 #define M_CSUM_UDPv4 0x00000002 /* UDP header/payload */ 188 #define M_CSUM_TCP_UDP_BAD 0x00000004 /* TCP/UDP checksum bad */ 189 #define M_CSUM_DATA 0x00000008 /* consult csum_data */ 190 #define M_CSUM_TCPv6 0x00000010 /* IPv6 TCP header/payload */ 191 #define M_CSUM_UDPv6 0x00000020 /* IPv6 UDP header/payload */ 192 #define M_CSUM_IPv4 0x00000040 /* IPv4 header */ 193 #define M_CSUM_IPv4_BAD 0x00000080 /* IPv4 header checksum bad */ 194 #define M_CSUM_TSOv4 0x00000100 /* TCPv4 segmentation offload */ 195 #define M_CSUM_TSOv6 0x00000200 /* TCPv6 segmentation offload */ 196 197 /* Checksum-assist quirks: keep separate from jump-table bits. */ 198 #define M_CSUM_NO_PSEUDOHDR 0x80000000 /* Rx csum_data does not include 199 * the UDP/TCP pseudo-hdr, and 200 * is not yet 1s-complemented. 201 */ 202 203 #define M_CSUM_BITS \ 204 "\20\1TCPv4\2UDPv4\3TCP_UDP_BAD\4DATA\5TCPv6\6UDPv6\7IPv4\10IPv4_BAD" \ 205 "\11TSOv4\12TSOv6\40NO_PSEUDOHDR" 206 207 /* 208 * Macros for manipulating csum_data on outgoing packets. These are 209 * used to pass information down from the L4/L3 to the L2. 210 */ 211 #define M_CSUM_DATA_IPv4_IPHL(x) ((x) >> 16) 212 #define M_CSUM_DATA_IPv4_OFFSET(x) ((x) & 0xffff) 213 214 /* 215 * Macros for M_CSUM_TCPv6 and M_CSUM_UDPv6 216 * 217 * M_CSUM_DATA_IPv6_HL: length of ip6_hdr + ext header. 218 * ie. offset of UDP/TCP header in the packet. 219 * 220 * M_CSUM_DATA_IPv6_OFFSET: offset of the checksum field in UDP/TCP header. 221 */ 222 223 #define M_CSUM_DATA_IPv6_HL(x) ((x) >> 16) 224 #define M_CSUM_DATA_IPv6_HL_SET(x, v) (x) = ((x) & 0xffff) | ((v) << 16) 225 #define M_CSUM_DATA_IPv6_OFFSET(x) ((x) & 0xffff) 226 227 /* 228 * Max # of pages we can attach to m_ext. This is carefully chosen 229 * to be able to handle SOSEND_LOAN_CHUNK with our minimum sized page. 230 */ 231 #ifdef MIN_PAGE_SIZE 232 #define M_EXT_MAXPAGES ((65536 / MIN_PAGE_SIZE) + 1) 233 #endif 234 235 /* description of external storage mapped into mbuf, valid if M_EXT set */ 236 struct _m_ext_storage { 237 unsigned int ext_refcnt; 238 int ext_flags; 239 char *ext_buf; /* start of buffer */ 240 void (*ext_free) /* free routine if not the usual */ 241 (struct mbuf *, void *, size_t, void *); 242 void *ext_arg; /* argument for ext_free */ 243 size_t ext_size; /* size of buffer, for ext_free */ 244 union { 245 paddr_t extun_paddr; /* physical address (M_EXT_CLUSTER) */ 246 /* pages (M_EXT_PAGES) */ 247 /* 248 * XXX This is gross, but it doesn't really matter; this is 249 * XXX overlaid on top of the mbuf data area. 250 */ 251 #ifdef M_EXT_MAXPAGES 252 struct vm_page *extun_pgs[M_EXT_MAXPAGES]; 253 #endif 254 } ext_un; 255 #define ext_paddr ext_un.extun_paddr 256 #define ext_pgs ext_un.extun_pgs 257 #ifdef DEBUG 258 const char *ext_ofile; 259 const char *ext_nfile; 260 int ext_oline; 261 int ext_nline; 262 #endif 263 }; 264 265 struct _m_ext { 266 struct mbuf *ext_ref; 267 struct _m_ext_storage ext_storage; 268 }; 269 270 #define M_PADDR_INVALID POOL_PADDR_INVALID 271 272 /* 273 * Definition of "struct mbuf". 274 * Don't change this without understanding how MHLEN/MLEN are defined. 275 */ 276 #define MBUF_DEFINE(name, mhlen, mlen) \ 277 struct name { \ 278 struct m_hdr m_hdr; \ 279 union { \ 280 struct { \ 281 struct pkthdr MH_pkthdr; \ 282 union { \ 283 struct _m_ext MH_ext; \ 284 char MH_databuf[(mhlen)]; \ 285 } MH_dat; \ 286 } MH; \ 287 char M_databuf[(mlen)]; \ 288 } M_dat; \ 289 } 290 #define m_next m_hdr.mh_next 291 #define m_len m_hdr.mh_len 292 #define m_data m_hdr.mh_data 293 #define m_owner m_hdr.mh_owner 294 #define m_type m_hdr.mh_type 295 #define m_flags m_hdr.mh_flags 296 #define m_nextpkt m_hdr.mh_nextpkt 297 #define m_paddr m_hdr.mh_paddr 298 #define m_pkthdr M_dat.MH.MH_pkthdr 299 #define m_ext_storage M_dat.MH.MH_dat.MH_ext.ext_storage 300 #define m_ext_ref M_dat.MH.MH_dat.MH_ext.ext_ref 301 #define m_ext m_ext_ref->m_ext_storage 302 #define m_pktdat M_dat.MH.MH_dat.MH_databuf 303 #define m_dat M_dat.M_databuf 304 305 /* 306 * Dummy mbuf structure to calculate the right values for MLEN/MHLEN, taking 307 * into account inter-structure padding. 308 */ 309 MBUF_DEFINE(_mbuf_dummy, 1, 1); 310 311 /* normal data len */ 312 #define MLEN (MSIZE - offsetof(struct _mbuf_dummy, m_dat)) 313 /* data len w/pkthdr */ 314 #define MHLEN (MSIZE - offsetof(struct _mbuf_dummy, m_pktdat)) 315 316 #define MINCLSIZE (MHLEN+MLEN+1) /* smallest amount to put in cluster */ 317 #define M_MAXCOMPRESS (MHLEN / 2) /* max amount to copy for compression */ 318 319 /* 320 * The *real* struct mbuf 321 */ 322 MBUF_DEFINE(mbuf, MHLEN, MLEN); 323 324 /* mbuf flags */ 325 #define M_EXT 0x00001 /* has associated external storage */ 326 #define M_PKTHDR 0x00002 /* start of record */ 327 #define M_EOR 0x00004 /* end of record */ 328 #define M_PROTO1 0x00008 /* protocol-specific */ 329 330 /* mbuf pkthdr flags, also in m_flags */ 331 #define M_AUTHIPHDR 0x00010 /* data origin authentication for IP header */ 332 #define M_DECRYPTED 0x00020 /* confidentiality */ 333 #define M_LOOP 0x00040 /* for Mbuf statistics */ 334 #define M_AUTHIPDGM 0x00080 /* data origin authentication */ 335 #define M_BCAST 0x00100 /* send/received as link-level broadcast */ 336 #define M_MCAST 0x00200 /* send/received as link-level multicast */ 337 #define M_CANFASTFWD 0x00400 /* used by filters to indicate packet can 338 be fast-forwarded */ 339 #define M_ANYCAST6 0x00800 /* received as IPv6 anycast */ 340 #define M_LINK0 0x01000 /* link layer specific flag */ 341 #define M_LINK1 0x02000 /* link layer specific flag */ 342 #define M_LINK2 0x04000 /* link layer specific flag */ 343 #define M_LINK3 0x08000 /* link layer specific flag */ 344 #define M_LINK4 0x10000 /* link layer specific flag */ 345 #define M_LINK5 0x20000 /* link layer specific flag */ 346 #define M_LINK6 0x40000 /* link layer specific flag */ 347 #define M_LINK7 0x80000 /* link layer specific flag */ 348 349 /* additional flags for M_EXT mbufs */ 350 #define M_EXT_FLAGS 0xff000000 351 #define M_EXT_CLUSTER 0x01000000 /* ext is a cluster */ 352 #define M_EXT_PAGES 0x02000000 /* ext_pgs is valid */ 353 #define M_EXT_ROMAP 0x04000000 /* ext mapping is r-o at MMU */ 354 #define M_EXT_RW 0x08000000 /* ext storage is writable */ 355 356 /* for source-level compatibility */ 357 #define M_CLUSTER M_EXT_CLUSTER 358 359 #define M_FLAGS_BITS \ 360 "\20\1EXT\2PKTHDR\3EOR\4PROTO1\5AUTHIPHDR\6DECRYPTED\7LOOP\10AUTHIPDGM" \ 361 "\11BCAST\12MCAST\13CANFASTFWD\14ANYCAST6\15LINK0\16LINK1\17LINK2\20LINK3" \ 362 "\21LINK4\22LINK5\23LINK6\24LINK7" \ 363 "\31EXT_CLUSTER\32EXT_PAGES\33EXT_ROMAP\34EXT_RW" 364 365 /* flags copied when copying m_pkthdr */ 366 #define M_COPYFLAGS (M_PKTHDR|M_EOR|M_BCAST|M_MCAST|M_CANFASTFWD|M_ANYCAST6|M_LINK0|M_LINK1|M_LINK2|M_AUTHIPHDR|M_DECRYPTED|M_LOOP|M_AUTHIPDGM) 367 368 /* flag copied when shallow-copying external storage */ 369 #define M_EXTCOPYFLAGS (M_EXT|M_EXT_FLAGS) 370 371 /* mbuf types */ 372 #define MT_FREE 0 /* should be on free list */ 373 #define MT_DATA 1 /* dynamic (data) allocation */ 374 #define MT_HEADER 2 /* packet header */ 375 #define MT_SONAME 3 /* socket name */ 376 #define MT_SOOPTS 4 /* socket options */ 377 #define MT_FTABLE 5 /* fragment reassembly header */ 378 #define MT_CONTROL 6 /* extra-data protocol message */ 379 #define MT_OOBDATA 7 /* expedited data */ 380 381 #ifdef MBUFTYPES 382 static const char *mbuftypes[] = { 383 "mbfree", 384 "mbdata", 385 "mbheader", 386 "mbsoname", 387 "mbsopts", 388 "mbftable", 389 "mbcontrol", 390 "mboobdata", 391 }; 392 #endif 393 394 /* flags to m_get/MGET */ 395 #define M_DONTWAIT M_NOWAIT 396 #define M_WAIT M_WAITOK 397 398 #ifdef MBUFTRACE 399 /* 400 * mbuf allocation tracing 401 */ 402 void mowner_init(struct mbuf *, int); 403 void mowner_ref(struct mbuf *, int); 404 void m_claim(struct mbuf *, struct mowner *); 405 void mowner_revoke(struct mbuf *, bool, int); 406 void mowner_attach(struct mowner *); 407 void mowner_detach(struct mowner *); 408 void m_claimm(struct mbuf *, struct mowner *); 409 #else 410 #define mowner_init(m, type) do { } while (/* CONSTCOND */ 0) 411 #define mowner_ref(m, flags) do { } while (/* CONSTCOND */ 0) 412 #define mowner_revoke(m, all, flags) do { } while (/* CONSTCOND */ 0) 413 #define m_claim(m, mowner) do { } while (/* CONSTCOND */ 0) 414 #define mowner_attach(mo) do { } while (/* CONSTCOND */ 0) 415 #define mowner_detach(mo) do { } while (/* CONSTCOND */ 0) 416 #define m_claimm(m, mo) do { } while (/* CONSTCOND */ 0) 417 #endif 418 419 #define MCLAIM(m, mo) m_claim((m), (mo)) 420 #define MOWNER_ATTACH(mo) mowner_attach(mo) 421 #define MOWNER_DETACH(mo) mowner_detach(mo) 422 423 /* 424 * mbuf allocation/deallocation macros: 425 * 426 * MGET(struct mbuf *m, int how, int type) 427 * allocates an mbuf and initializes it to contain internal data. 428 * 429 * MGETHDR(struct mbuf *m, int how, int type) 430 * allocates an mbuf and initializes it to contain a packet header 431 * and internal data. 432 * 433 * If 'how' is M_WAIT, these macros (and the corresponding functions) 434 * are guaranteed to return successfully. 435 */ 436 #define MGET(m, how, type) m = m_get((how), (type)) 437 #define MGETHDR(m, how, type) m = m_gethdr((how), (type)) 438 439 #if defined(_KERNEL) 440 #define _M_ 441 /* 442 * Macros for tracking external storage associated with an mbuf. 443 */ 444 #ifdef DEBUG 445 #define MCLREFDEBUGN(m, file, line) \ 446 do { \ 447 (m)->m_ext.ext_nfile = (file); \ 448 (m)->m_ext.ext_nline = (line); \ 449 } while (/* CONSTCOND */ 0) 450 451 #define MCLREFDEBUGO(m, file, line) \ 452 do { \ 453 (m)->m_ext.ext_ofile = (file); \ 454 (m)->m_ext.ext_oline = (line); \ 455 } while (/* CONSTCOND */ 0) 456 #else 457 #define MCLREFDEBUGN(m, file, line) 458 #define MCLREFDEBUGO(m, file, line) 459 #endif 460 461 #define MCLINITREFERENCE(m) \ 462 do { \ 463 KDASSERT(((m)->m_flags & M_EXT) == 0); \ 464 (m)->m_ext_ref = (m); \ 465 (m)->m_ext.ext_refcnt = 1; \ 466 MCLREFDEBUGO((m), __FILE__, __LINE__); \ 467 MCLREFDEBUGN((m), NULL, 0); \ 468 } while (/* CONSTCOND */ 0) 469 470 /* 471 * Macros for mbuf external storage. 472 * 473 * MCLGET allocates and adds an mbuf cluster to a normal mbuf; 474 * the flag M_EXT is set upon success. 475 * 476 * MEXTMALLOC allocates external storage and adds it to 477 * a normal mbuf; the flag M_EXT is set upon success. 478 * 479 * MEXTADD adds pre-allocated external storage to 480 * a normal mbuf; the flag M_EXT is set upon success. 481 */ 482 483 #define _MCLGET(m, pool_cache, size, how) \ 484 do { \ 485 (m)->m_ext_storage.ext_buf = \ 486 pool_cache_get_paddr((pool_cache), \ 487 (how) == M_WAIT ? (PR_WAITOK|PR_LIMITFAIL) : 0, \ 488 &(m)->m_ext_storage.ext_paddr); \ 489 if ((m)->m_ext_storage.ext_buf != NULL) { \ 490 MCLINITREFERENCE(m); \ 491 (m)->m_data = (m)->m_ext.ext_buf; \ 492 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | \ 493 M_EXT|M_CLUSTER|M_EXT_RW; \ 494 (m)->m_ext.ext_flags = 0; \ 495 (m)->m_ext.ext_size = (size); \ 496 (m)->m_ext.ext_free = NULL; \ 497 (m)->m_ext.ext_arg = (pool_cache); \ 498 /* ext_paddr initialized above */ \ 499 mowner_ref((m), M_EXT|M_CLUSTER); \ 500 } \ 501 } while (/* CONSTCOND */ 0) 502 503 /* 504 * The standard mbuf cluster pool. 505 */ 506 #define MCLGET(m, how) _MCLGET((m), mcl_cache, MCLBYTES, (how)) 507 508 #define MEXTMALLOC(m, size, how) \ 509 do { \ 510 (m)->m_ext_storage.ext_buf = \ 511 malloc((size), mbtypes[(m)->m_type], (how)); \ 512 if ((m)->m_ext_storage.ext_buf != NULL) { \ 513 MCLINITREFERENCE(m); \ 514 (m)->m_data = (m)->m_ext.ext_buf; \ 515 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | \ 516 M_EXT|M_EXT_RW; \ 517 (m)->m_ext.ext_flags = 0; \ 518 (m)->m_ext.ext_size = (size); \ 519 (m)->m_ext.ext_free = NULL; \ 520 (m)->m_ext.ext_arg = NULL; \ 521 mowner_ref((m), M_EXT); \ 522 } \ 523 } while (/* CONSTCOND */ 0) 524 525 #define MEXTADD(m, buf, size, type, free, arg) \ 526 do { \ 527 MCLINITREFERENCE(m); \ 528 (m)->m_data = (m)->m_ext.ext_buf = (void *)(buf); \ 529 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | M_EXT; \ 530 (m)->m_ext.ext_flags = 0; \ 531 (m)->m_ext.ext_size = (size); \ 532 (m)->m_ext.ext_free = (free); \ 533 (m)->m_ext.ext_arg = (arg); \ 534 mowner_ref((m), M_EXT); \ 535 } while (/* CONSTCOND */ 0) 536 537 /* 538 * Reset the data pointer on an mbuf. 539 */ 540 #define MRESETDATA(m) \ 541 do { \ 542 if ((m)->m_flags & M_EXT) \ 543 (m)->m_data = (m)->m_ext.ext_buf; \ 544 else if ((m)->m_flags & M_PKTHDR) \ 545 (m)->m_data = (m)->m_pktdat; \ 546 else \ 547 (m)->m_data = (m)->m_dat; \ 548 } while (/* CONSTCOND */ 0) 549 550 /* 551 * MFREE(struct mbuf *m, struct mbuf *n) 552 * Free a single mbuf and associated external storage. 553 * Place the successor, if any, in n. 554 */ 555 #define MFREE(m, n) \ 556 mowner_revoke((m), 1, (m)->m_flags); \ 557 mbstat_type_add((m)->m_type, -1); \ 558 if ((m)->m_flags & M_PKTHDR) \ 559 m_tag_delete_chain((m), NULL); \ 560 (n) = (m)->m_next; \ 561 if ((m)->m_flags & M_EXT) { \ 562 m_ext_free((m)); \ 563 } else { \ 564 KASSERT((m)->m_type != MT_FREE); \ 565 (m)->m_type = MT_FREE; \ 566 pool_cache_put(mb_cache, (m)); \ 567 } \ 568 569 /* 570 * Copy mbuf pkthdr from `from' to `to'. 571 * `from' must have M_PKTHDR set, and `to' must be empty. 572 */ 573 #define M_COPY_PKTHDR(to, from) \ 574 do { \ 575 (to)->m_pkthdr = (from)->m_pkthdr; \ 576 (to)->m_flags = (from)->m_flags & M_COPYFLAGS; \ 577 SLIST_INIT(&(to)->m_pkthdr.tags); \ 578 m_tag_copy_chain((to), (from)); \ 579 (to)->m_data = (to)->m_pktdat; \ 580 } while (/* CONSTCOND */ 0) 581 582 /* 583 * Move mbuf pkthdr from `from' to `to'. 584 * `from' must have M_PKTHDR set, and `to' must be empty. 585 */ 586 #define M_MOVE_PKTHDR(to, from) m_move_pkthdr(to, from) 587 588 /* 589 * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place 590 * an object of the specified size at the end of the mbuf, longword aligned. 591 */ 592 #define M_ALIGN(m, len) \ 593 do { \ 594 (m)->m_data += (MLEN - (len)) &~ (sizeof(long) - 1); \ 595 } while (/* CONSTCOND */ 0) 596 597 /* 598 * As above, for mbufs allocated with m_gethdr/MGETHDR 599 * or initialized by M_COPY_PKTHDR. 600 */ 601 #define MH_ALIGN(m, len) \ 602 do { \ 603 (m)->m_data += (MHLEN - (len)) &~ (sizeof(long) - 1); \ 604 } while (/* CONSTCOND */ 0) 605 606 /* 607 * Determine if an mbuf's data area is read-only. This is true 608 * if external storage is read-only mapped, or not marked as R/W, 609 * or referenced by more than one mbuf. 610 */ 611 #define M_READONLY(m) \ 612 (((m)->m_flags & M_EXT) != 0 && \ 613 (((m)->m_flags & (M_EXT_ROMAP|M_EXT_RW)) != M_EXT_RW || \ 614 (m)->m_ext.ext_refcnt > 1)) 615 616 #define M_UNWRITABLE(__m, __len) \ 617 ((__m)->m_len < (__len) || M_READONLY((__m))) 618 /* 619 * Determine if an mbuf's data area is read-only at the MMU. 620 */ 621 #define M_ROMAP(m) \ 622 (((m)->m_flags & (M_EXT|M_EXT_ROMAP)) == (M_EXT|M_EXT_ROMAP)) 623 624 /* 625 * Compute the amount of space available 626 * before the current start of data in an mbuf. 627 */ 628 #define _M_LEADINGSPACE(m) \ 629 ((m)->m_flags & M_EXT ? (m)->m_data - (m)->m_ext.ext_buf : \ 630 (m)->m_flags & M_PKTHDR ? (m)->m_data - (m)->m_pktdat : \ 631 (m)->m_data - (m)->m_dat) 632 633 #define M_LEADINGSPACE(m) \ 634 (M_READONLY((m)) ? 0 : _M_LEADINGSPACE((m))) 635 636 /* 637 * Compute the amount of space available 638 * after the end of data in an mbuf. 639 */ 640 #define _M_TRAILINGSPACE(m) \ 641 ((m)->m_flags & M_EXT ? (m)->m_ext.ext_buf + (m)->m_ext.ext_size - \ 642 ((m)->m_data + (m)->m_len) : \ 643 &(m)->m_dat[MLEN] - ((m)->m_data + (m)->m_len)) 644 645 #define M_TRAILINGSPACE(m) \ 646 (M_READONLY((m)) ? 0 : _M_TRAILINGSPACE((m))) 647 648 /* 649 * Compute the address of an mbuf's data area. 650 */ 651 #define M_BUFADDR(m) \ 652 (((m)->m_flags & M_PKTHDR) ? (m)->m_pktdat : (m)->m_dat) 653 654 /* 655 * Compute the offset of the beginning of the data buffer of a non-ext 656 * mbuf. 657 */ 658 #define M_BUFOFFSET(m) \ 659 (((m)->m_flags & M_PKTHDR) ? \ 660 offsetof(struct mbuf, m_pktdat) : offsetof(struct mbuf, m_dat)) 661 662 /* 663 * Arrange to prepend space of size plen to mbuf m. 664 * If a new mbuf must be allocated, how specifies whether to wait. 665 * If how is M_DONTWAIT and allocation fails, the original mbuf chain 666 * is freed and m is set to NULL. 667 */ 668 #define M_PREPEND(m, plen, how) \ 669 do { \ 670 if (M_LEADINGSPACE(m) >= (plen)) { \ 671 (m)->m_data -= (plen); \ 672 (m)->m_len += (plen); \ 673 } else \ 674 (m) = m_prepend((m), (plen), (how)); \ 675 if ((m) && (m)->m_flags & M_PKTHDR) \ 676 (m)->m_pkthdr.len += (plen); \ 677 } while (/* CONSTCOND */ 0) 678 679 /* change mbuf to new type */ 680 #define MCHTYPE(m, t) \ 681 do { \ 682 KASSERT((t) != MT_FREE); \ 683 mbstat_type_add((m)->m_type, -1); \ 684 mbstat_type_add(t, 1); \ 685 (m)->m_type = t; \ 686 } while (/* CONSTCOND */ 0) 687 688 /* length to m_copy to copy all */ 689 #define M_COPYALL -1 690 691 /* compatibility with 4.3 */ 692 #define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT) 693 694 /* 695 * Allow drivers and/or protocols to use the rcvif member of 696 * PKTHDR mbufs to store private context information. 697 */ 698 #define M_GETCTX(m, t) ((t)(m)->m_pkthdr.rcvif) 699 #define M_SETCTX(m, c) ((void)((m)->m_pkthdr.rcvif = (void *)(c))) 700 701 #endif /* defined(_KERNEL) */ 702 703 /* 704 * Simple mbuf queueing system 705 * 706 * this is basically a SIMPLEQ adapted to mbuf use (ie using 707 * m_nextpkt instead of field.sqe_next). 708 * 709 * m_next is ignored, so queueing chains of mbufs is possible 710 */ 711 #define MBUFQ_HEAD(name) \ 712 struct name { \ 713 struct mbuf *mq_first; \ 714 struct mbuf **mq_last; \ 715 } 716 717 #define MBUFQ_INIT(q) do { \ 718 (q)->mq_first = NULL; \ 719 (q)->mq_last = &(q)->mq_first; \ 720 } while (/*CONSTCOND*/0) 721 722 #define MBUFQ_ENQUEUE(q, m) do { \ 723 (m)->m_nextpkt = NULL; \ 724 *(q)->mq_last = (m); \ 725 (q)->mq_last = &(m)->m_nextpkt; \ 726 } while (/*CONSTCOND*/0) 727 728 #define MBUFQ_PREPEND(q, m) do { \ 729 if (((m)->m_nextpkt = (q)->mq_first) == NULL) \ 730 (q)->mq_last = &(m)->m_nextpkt; \ 731 (q)->mq_first = (m); \ 732 } while (/*CONSTCOND*/0) 733 734 #define MBUFQ_DEQUEUE(q, m) do { \ 735 if (((m) = (q)->mq_first) != NULL) { \ 736 if (((q)->mq_first = (m)->m_nextpkt) == NULL) \ 737 (q)->mq_last = &(q)->mq_first; \ 738 else \ 739 (m)->m_nextpkt = NULL; \ 740 } \ 741 } while (/*CONSTCOND*/0) 742 743 #define MBUFQ_DRAIN(q) do { \ 744 struct mbuf *__m0; \ 745 while ((__m0 = (q)->mq_first) != NULL) { \ 746 (q)->mq_first = __m0->m_nextpkt; \ 747 m_freem(__m0); \ 748 } \ 749 (q)->mq_last = &(q)->mq_first; \ 750 } while (/*CONSTCOND*/0) 751 752 #define MBUFQ_FIRST(q) ((q)->mq_first) 753 #define MBUFQ_NEXT(m) ((m)->m_nextpkt) 754 #define MBUFQ_LAST(q) (*(q)->mq_last) 755 756 /* 757 * Mbuf statistics. 758 * For statistics related to mbuf and cluster allocations, see also the 759 * pool headers (mb_cache and mcl_cache). 760 */ 761 struct mbstat { 762 u_long _m_spare; /* formerly m_mbufs */ 763 u_long _m_spare1; /* formerly m_clusters */ 764 u_long _m_spare2; /* spare field */ 765 u_long _m_spare3; /* formely m_clfree - free clusters */ 766 u_long m_drops; /* times failed to find space */ 767 u_long m_wait; /* times waited for space */ 768 u_long m_drain; /* times drained protocols for space */ 769 u_short m_mtypes[256]; /* type specific mbuf allocations */ 770 }; 771 772 struct mbstat_cpu { 773 u_int m_mtypes[256]; /* type specific mbuf allocations */ 774 }; 775 776 /* 777 * Mbuf sysctl variables. 778 */ 779 #define MBUF_MSIZE 1 /* int: mbuf base size */ 780 #define MBUF_MCLBYTES 2 /* int: mbuf cluster size */ 781 #define MBUF_NMBCLUSTERS 3 /* int: limit on the # of clusters */ 782 #define MBUF_MBLOWAT 4 /* int: mbuf low water mark */ 783 #define MBUF_MCLLOWAT 5 /* int: mbuf cluster low water mark */ 784 #define MBUF_STATS 6 /* struct: mbstat */ 785 #define MBUF_MOWNERS 7 /* struct: m_owner[] */ 786 #define MBUF_MAXID 8 /* number of valid MBUF ids */ 787 788 #define CTL_MBUF_NAMES { \ 789 { 0, 0 }, \ 790 { "msize", CTLTYPE_INT }, \ 791 { "mclbytes", CTLTYPE_INT }, \ 792 { "nmbclusters", CTLTYPE_INT }, \ 793 { "mblowat", CTLTYPE_INT }, \ 794 { "mcllowat", CTLTYPE_INT }, \ 795 { 0 /* "stats" */, CTLTYPE_STRUCT }, \ 796 { 0 /* "mowners" */, CTLTYPE_STRUCT }, \ 797 } 798 799 #ifdef _KERNEL 800 extern struct mbstat mbstat; 801 extern int nmbclusters; /* limit on the # of clusters */ 802 extern int mblowat; /* mbuf low water mark */ 803 extern int mcllowat; /* mbuf cluster low water mark */ 804 extern int max_linkhdr; /* largest link-level header */ 805 extern int max_protohdr; /* largest protocol header */ 806 extern int max_hdr; /* largest link+protocol header */ 807 extern int max_datalen; /* MHLEN - max_hdr */ 808 extern const int msize; /* mbuf base size */ 809 extern const int mclbytes; /* mbuf cluster size */ 810 extern pool_cache_t mb_cache; 811 extern pool_cache_t mcl_cache; 812 #ifdef MBUFTRACE 813 LIST_HEAD(mownerhead, mowner); 814 extern struct mownerhead mowners; 815 extern struct mowner unknown_mowners[]; 816 extern struct mowner revoked_mowner; 817 #endif 818 819 MALLOC_DECLARE(M_MBUF); 820 MALLOC_DECLARE(M_SONAME); 821 MALLOC_DECLARE(M_SOOPTS); 822 823 struct mbuf *m_copym(struct mbuf *, int, int, int); 824 struct mbuf *m_copypacket(struct mbuf *, int); 825 struct mbuf *m_devget(char *, int, int, struct ifnet *, 826 void (*copy)(const void *, void *, size_t)); 827 struct mbuf *m_dup(struct mbuf *, int, int, int); 828 struct mbuf *m_free(struct mbuf *); 829 struct mbuf *m_get(int, int); 830 struct mbuf *m_getclr(int, int); 831 struct mbuf *m_gethdr(int, int); 832 struct mbuf *m_prepend(struct mbuf *,int, int); 833 struct mbuf *m_pulldown(struct mbuf *, int, int, int *); 834 struct mbuf *m_pullup(struct mbuf *, int); 835 struct mbuf *m_copyup(struct mbuf *, int, int); 836 struct mbuf *m_split(struct mbuf *,int, int); 837 struct mbuf *m_getptr(struct mbuf *, int, int *); 838 void m_adj(struct mbuf *, int); 839 struct mbuf *m_defrag(struct mbuf *, int); 840 int m_apply(struct mbuf *, int, int, 841 int (*)(void *, void *, unsigned int), void *); 842 void m_cat(struct mbuf *,struct mbuf *); 843 void m_clget(struct mbuf *, int); 844 int m_mballoc(int, int); 845 void m_copyback(struct mbuf *, int, int, const void *); 846 struct mbuf *m_copyback_cow(struct mbuf *, int, int, const void *, int); 847 int m_makewritable(struct mbuf **, int, int, int); 848 struct mbuf *m_getcl(int, int, int); 849 void m_copydata(struct mbuf *, int, int, void *); 850 void m_freem(struct mbuf *); 851 void m_reclaim(void *, int); 852 void mbinit(void); 853 void m_ext_free(struct mbuf *); 854 char * m_mapin(struct mbuf *); 855 void m_move_pkthdr(struct mbuf *, struct mbuf *); 856 857 bool m_ensure_contig(struct mbuf **, int); 858 struct mbuf *m_add(struct mbuf *, struct mbuf *); 859 void m_align(struct mbuf *, int); 860 int m_append(struct mbuf *, int, const void *); 861 862 /* Inline routines. */ 863 static __inline u_int m_length(const struct mbuf *) __unused; 864 865 /* Statistics */ 866 void mbstat_type_add(int, int); 867 868 /* Packet tag routines */ 869 struct m_tag *m_tag_get(int, int, int); 870 void m_tag_free(struct m_tag *); 871 void m_tag_prepend(struct mbuf *, struct m_tag *); 872 void m_tag_unlink(struct mbuf *, struct m_tag *); 873 void m_tag_delete(struct mbuf *, struct m_tag *); 874 void m_tag_delete_chain(struct mbuf *, struct m_tag *); 875 void m_tag_delete_nonpersistent(struct mbuf *); 876 struct m_tag *m_tag_find(const struct mbuf *, int, struct m_tag *); 877 struct m_tag *m_tag_copy(struct m_tag *); 878 int m_tag_copy_chain(struct mbuf *, struct mbuf *); 879 void m_tag_init(struct mbuf *); 880 struct m_tag *m_tag_first(struct mbuf *); 881 struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); 882 883 /* Packet tag types */ 884 #define PACKET_TAG_NONE 0 /* Nothing */ 885 #define PACKET_TAG_VLAN 1 /* VLAN ID */ 886 #define PACKET_TAG_ENCAP 2 /* encapsulation data */ 887 #define PACKET_TAG_ESP 3 /* ESP information */ 888 #define PACKET_TAG_PF 11 /* packet filter */ 889 #define PACKET_TAG_ALTQ_QID 12 /* ALTQ queue id */ 890 891 #define PACKET_TAG_IPSEC_IN_CRYPTO_DONE 16 892 #define PACKET_TAG_IPSEC_IN_DONE 17 893 #define PACKET_TAG_IPSEC_OUT_DONE 18 894 #define PACKET_TAG_IPSEC_OUT_CRYPTO_NEEDED 19 /* NIC IPsec crypto req'ed */ 895 #define PACKET_TAG_IPSEC_IN_COULD_DO_CRYPTO 20 /* NIC notifies IPsec */ 896 #define PACKET_TAG_IPSEC_PENDING_TDB 21 /* Reminder to do IPsec */ 897 898 #define PACKET_TAG_IPSEC_SOCKET 22 /* IPSEC socket ref */ 899 #define PACKET_TAG_IPSEC_HISTORY 23 /* IPSEC history */ 900 901 #define PACKET_TAG_IPSEC_NAT_T_PORTS 25 /* two uint16_t */ 902 903 #define PACKET_TAG_INET6 26 /* IPv6 info */ 904 905 #define PACKET_TAG_ECO_RETRYPARMS 27 /* Econet retry parameters */ 906 907 #define PACKET_TAG_TUNNEL_INFO 28 /* tunnel identification and 908 * protocol callback, for 909 * loop detection/recovery 910 */ 911 912 /* 913 * Return the number of bytes in the mbuf chain, m. 914 */ 915 static __inline u_int 916 m_length(const struct mbuf *m) 917 { 918 const struct mbuf *m0; 919 u_int pktlen; 920 921 if ((m->m_flags & M_PKTHDR) != 0) 922 return m->m_pkthdr.len; 923 924 pktlen = 0; 925 for (m0 = m; m0 != NULL; m0 = m0->m_next) 926 pktlen += m0->m_len; 927 return pktlen; 928 } 929 930 void m_print(const struct mbuf *, const char *, void (*)(const char *, ...) 931 __printflike(1, 2)); 932 933 #endif /* _KERNEL */ 934 #endif /* !_SYS_MBUF_H_ */ 935