1 /* $NetBSD: mbuf.h,v 1.158 2015/06/04 09:19:59 ozaki-r Exp $ */ 2 3 /*- 4 * Copyright (c) 1996, 1997, 1999, 2001, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and Matt Thomas of 3am Software Foundry. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1982, 1986, 1988, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Neither the name of the University nor the names of its contributors 46 * may be used to endorse or promote products derived from this software 47 * without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 59 * SUCH DAMAGE. 60 * 61 * @(#)mbuf.h 8.5 (Berkeley) 2/19/95 62 */ 63 64 #ifndef _SYS_MBUF_H_ 65 #define _SYS_MBUF_H_ 66 67 #ifdef _KERNEL_OPT 68 #include "opt_mbuftrace.h" 69 #endif 70 71 #ifndef M_WAITOK 72 #include <sys/malloc.h> 73 #endif 74 #include <sys/pool.h> 75 #include <sys/queue.h> 76 #if defined(_KERNEL) 77 #include <sys/percpu_types.h> 78 #endif /* defined(_KERNEL) */ 79 80 /* For offsetof() */ 81 #if defined(_KERNEL) || defined(_STANDALONE) 82 #include <sys/systm.h> 83 #else 84 #include <stddef.h> 85 #endif 86 87 #include <uvm/uvm_param.h> /* for MIN_PAGE_SIZE */ 88 89 /* 90 * Mbufs are of a single size, MSIZE (machine/param.h), which 91 * includes overhead. An mbuf may add a single "mbuf cluster" of size 92 * MCLBYTES (also in machine/param.h), which has no additional overhead 93 * and is used instead of the internal data area; this is done when 94 * at least MINCLSIZE of data must be stored. 95 */ 96 97 /* Packet tags structure */ 98 struct m_tag { 99 SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ 100 uint16_t m_tag_id; /* Tag ID */ 101 uint16_t m_tag_len; /* Length of data */ 102 }; 103 104 /* mbuf ownership structure */ 105 struct mowner { 106 char mo_name[16]; /* owner name (fxp0) */ 107 char mo_descr[16]; /* owner description (input) */ 108 LIST_ENTRY(mowner) mo_link; /* */ 109 struct percpu *mo_counters; 110 }; 111 112 #define MOWNER_INIT(x, y) { .mo_name = x, .mo_descr = y } 113 114 enum mowner_counter_index { 115 MOWNER_COUNTER_CLAIMS, /* # of small mbuf claimed */ 116 MOWNER_COUNTER_RELEASES, /* # of small mbuf released */ 117 MOWNER_COUNTER_CLUSTER_CLAIMS, /* # of M_CLUSTER mbuf claimed */ 118 MOWNER_COUNTER_CLUSTER_RELEASES,/* # of M_CLUSTER mbuf released */ 119 MOWNER_COUNTER_EXT_CLAIMS, /* # of M_EXT mbuf claimed */ 120 MOWNER_COUNTER_EXT_RELEASES, /* # of M_EXT mbuf released */ 121 122 MOWNER_COUNTER_NCOUNTERS, 123 }; 124 125 #if defined(_KERNEL) 126 struct mowner_counter { 127 u_long mc_counter[MOWNER_COUNTER_NCOUNTERS]; 128 }; 129 #endif /* defined(_KERNEL) */ 130 131 /* userland-exported version of struct mowner */ 132 struct mowner_user { 133 char mo_name[16]; /* owner name (fxp0) */ 134 char mo_descr[16]; /* owner description (input) */ 135 LIST_ENTRY(mowner) mo_link; /* unused padding; for compatibility */ 136 u_long mo_counter[MOWNER_COUNTER_NCOUNTERS]; /* counters */ 137 }; 138 139 /* 140 * Macros for type conversion 141 * mtod(m,t) - convert mbuf pointer to data pointer of correct type 142 */ 143 #define mtod(m, t) ((t)((m)->m_data)) 144 145 /* header at beginning of each mbuf: */ 146 struct m_hdr { 147 struct mbuf *mh_next; /* next buffer in chain */ 148 struct mbuf *mh_nextpkt; /* next chain in queue/record */ 149 char *mh_data; /* location of data */ 150 struct mowner *mh_owner; /* mbuf owner */ 151 int mh_len; /* amount of data in this mbuf */ 152 int mh_flags; /* flags; see below */ 153 paddr_t mh_paddr; /* physical address of mbuf */ 154 short mh_type; /* type of data in this mbuf */ 155 }; 156 157 /* 158 * record/packet header in first mbuf of chain; valid if M_PKTHDR set 159 * 160 * A note about csum_data: For the out-bound direction, the low 16 bits 161 * indicates the offset after the L4 header where the final L4 checksum value 162 * is to be stored and the high 16 bits is the length of the L3 header (the 163 * start of the data to be checksumed). For the in-bound direction, it is only 164 * valid if the M_CSUM_DATA flag is set. In this case, an L4 checksum has been 165 * calculated by hardware, but it is up to software to perform final 166 * verification. 167 * 168 * Note for in-bound TCP/UDP checksums, we expect the csum_data to NOT 169 * be bit-wise inverted (the final step in the calculation of an IP 170 * checksum) -- this is so we can accumulate the checksum for fragmented 171 * packets during reassembly. 172 */ 173 struct pkthdr { 174 struct ifnet *rcvif; /* rcv interface */ 175 SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */ 176 int len; /* total packet length */ 177 int csum_flags; /* checksum flags */ 178 uint32_t csum_data; /* checksum data */ 179 u_int segsz; /* segment size */ 180 }; 181 182 /* 183 * Note: These bits are carefully arrange so that the compiler can have 184 * a prayer of generating a jump table. 185 */ 186 #define M_CSUM_TCPv4 0x00000001 /* TCP header/payload */ 187 #define M_CSUM_UDPv4 0x00000002 /* UDP header/payload */ 188 #define M_CSUM_TCP_UDP_BAD 0x00000004 /* TCP/UDP checksum bad */ 189 #define M_CSUM_DATA 0x00000008 /* consult csum_data */ 190 #define M_CSUM_TCPv6 0x00000010 /* IPv6 TCP header/payload */ 191 #define M_CSUM_UDPv6 0x00000020 /* IPv6 UDP header/payload */ 192 #define M_CSUM_IPv4 0x00000040 /* IPv4 header */ 193 #define M_CSUM_IPv4_BAD 0x00000080 /* IPv4 header checksum bad */ 194 #define M_CSUM_TSOv4 0x00000100 /* TCPv4 segmentation offload */ 195 #define M_CSUM_TSOv6 0x00000200 /* TCPv6 segmentation offload */ 196 197 /* Checksum-assist quirks: keep separate from jump-table bits. */ 198 #define M_CSUM_NO_PSEUDOHDR 0x80000000 /* Rx csum_data does not include 199 * the UDP/TCP pseudo-hdr, and 200 * is not yet 1s-complemented. 201 */ 202 203 #define M_CSUM_BITS \ 204 "\20\1TCPv4\2UDPv4\3TCP_UDP_BAD\4DATA\5TCPv6\6UDPv6\7IPv4\10IPv4_BAD" \ 205 "\11TSOv4\12TSOv6\40NO_PSEUDOHDR" 206 207 /* 208 * Macros for manipulating csum_data on outgoing packets. These are 209 * used to pass information down from the L4/L3 to the L2. 210 */ 211 #define M_CSUM_DATA_IPv4_IPHL(x) ((x) >> 16) 212 #define M_CSUM_DATA_IPv4_OFFSET(x) ((x) & 0xffff) 213 214 /* 215 * Macros for M_CSUM_TCPv6 and M_CSUM_UDPv6 216 * 217 * M_CSUM_DATA_IPv6_HL: length of ip6_hdr + ext header. 218 * ie. offset of UDP/TCP header in the packet. 219 * 220 * M_CSUM_DATA_IPv6_OFFSET: offset of the checksum field in UDP/TCP header. 221 */ 222 223 #define M_CSUM_DATA_IPv6_HL(x) ((x) >> 16) 224 #define M_CSUM_DATA_IPv6_HL_SET(x, v) (x) = ((x) & 0xffff) | ((v) << 16) 225 #define M_CSUM_DATA_IPv6_OFFSET(x) ((x) & 0xffff) 226 227 /* 228 * Max # of pages we can attach to m_ext. This is carefully chosen 229 * to be able to handle SOSEND_LOAN_CHUNK with our minimum sized page. 230 */ 231 #ifdef MIN_PAGE_SIZE 232 #define M_EXT_MAXPAGES ((65536 / MIN_PAGE_SIZE) + 1) 233 #endif 234 235 /* description of external storage mapped into mbuf, valid if M_EXT set */ 236 struct _m_ext_storage { 237 unsigned int ext_refcnt; 238 int ext_flags; 239 char *ext_buf; /* start of buffer */ 240 void (*ext_free) /* free routine if not the usual */ 241 (struct mbuf *, void *, size_t, void *); 242 void *ext_arg; /* argument for ext_free */ 243 size_t ext_size; /* size of buffer, for ext_free */ 244 union { 245 paddr_t extun_paddr; /* physical address (M_EXT_CLUSTER) */ 246 /* pages (M_EXT_PAGES) */ 247 /* 248 * XXX This is gross, but it doesn't really matter; this is 249 * XXX overlaid on top of the mbuf data area. 250 */ 251 #ifdef M_EXT_MAXPAGES 252 struct vm_page *extun_pgs[M_EXT_MAXPAGES]; 253 #endif 254 } ext_un; 255 #define ext_paddr ext_un.extun_paddr 256 #define ext_pgs ext_un.extun_pgs 257 #ifdef DEBUG 258 const char *ext_ofile; 259 const char *ext_nfile; 260 int ext_oline; 261 int ext_nline; 262 #endif 263 }; 264 265 struct _m_ext { 266 struct mbuf *ext_ref; 267 struct _m_ext_storage ext_storage; 268 }; 269 270 #define M_PADDR_INVALID POOL_PADDR_INVALID 271 272 /* 273 * Definition of "struct mbuf". 274 * Don't change this without understanding how MHLEN/MLEN are defined. 275 */ 276 #define MBUF_DEFINE(name, mhlen, mlen) \ 277 struct name { \ 278 struct m_hdr m_hdr; \ 279 union { \ 280 struct { \ 281 struct pkthdr MH_pkthdr; \ 282 union { \ 283 struct _m_ext MH_ext; \ 284 char MH_databuf[(mhlen)]; \ 285 } MH_dat; \ 286 } MH; \ 287 char M_databuf[(mlen)]; \ 288 } M_dat; \ 289 } 290 #define m_next m_hdr.mh_next 291 #define m_len m_hdr.mh_len 292 #define m_data m_hdr.mh_data 293 #define m_owner m_hdr.mh_owner 294 #define m_type m_hdr.mh_type 295 #define m_flags m_hdr.mh_flags 296 #define m_nextpkt m_hdr.mh_nextpkt 297 #define m_paddr m_hdr.mh_paddr 298 #define m_pkthdr M_dat.MH.MH_pkthdr 299 #define m_ext_storage M_dat.MH.MH_dat.MH_ext.ext_storage 300 #define m_ext_ref M_dat.MH.MH_dat.MH_ext.ext_ref 301 #define m_ext m_ext_ref->m_ext_storage 302 #define m_pktdat M_dat.MH.MH_dat.MH_databuf 303 #define m_dat M_dat.M_databuf 304 305 /* 306 * Dummy mbuf structure to calculate the right values for MLEN/MHLEN, taking 307 * into account inter-structure padding. 308 */ 309 MBUF_DEFINE(_mbuf_dummy, 1, 1); 310 311 /* normal data len */ 312 #define MLEN (MSIZE - offsetof(struct _mbuf_dummy, m_dat)) 313 /* data len w/pkthdr */ 314 #define MHLEN (MSIZE - offsetof(struct _mbuf_dummy, m_pktdat)) 315 316 #define MINCLSIZE (MHLEN+MLEN+1) /* smallest amount to put in cluster */ 317 #define M_MAXCOMPRESS (MHLEN / 2) /* max amount to copy for compression */ 318 319 /* 320 * The *real* struct mbuf 321 */ 322 MBUF_DEFINE(mbuf, MHLEN, MLEN); 323 324 /* mbuf flags */ 325 #define M_EXT 0x00000001 /* has associated external storage */ 326 #define M_PKTHDR 0x00000002 /* start of record */ 327 #define M_EOR 0x00000004 /* end of record */ 328 #define M_PROTO1 0x00000008 /* protocol-specific */ 329 330 /* mbuf pkthdr flags, also in m_flags */ 331 #define M_AUTHIPHDR 0x00000010 /* data origin authentication for 332 * IP header */ 333 #define M_DECRYPTED 0x00000020 /* confidentiality */ 334 #define M_LOOP 0x00000040 /* for Mbuf statistics */ 335 #define M_AUTHIPDGM 0x00000080 /* data origin authentication */ 336 #define M_BCAST 0x00000100 /* send/received as link-level 337 * broadcast */ 338 #define M_MCAST 0x00000200 /* send/received as link-level 339 * multicast */ 340 #define M_CANFASTFWD 0x00000400 /* used by filters to indicate 341 * packet can be fast-forwarded */ 342 #define M_ANYCAST6 0x00000800 /* received as IPv6 anycast */ 343 344 #define M_LINK0 0x00001000 /* link layer specific flag */ 345 #define M_LINK1 0x00002000 /* link layer specific flag */ 346 #define M_LINK2 0x00004000 /* link layer specific flag */ 347 348 #define M_LINK3 0x00008000 /* link layer specific flag */ 349 #define M_LINK4 0x00010000 /* link layer specific flag */ 350 #define M_LINK5 0x00020000 /* link layer specific flag */ 351 #define M_LINK6 0x00040000 /* link layer specific flag */ 352 #define M_LINK7 0x00080000 /* link layer specific flag */ 353 354 /* additional flags for M_EXT mbufs */ 355 #define M_EXT_FLAGS 0xff000000 356 #define M_EXT_CLUSTER 0x01000000 /* ext is a cluster */ 357 #define M_EXT_PAGES 0x02000000 /* ext_pgs is valid */ 358 #define M_EXT_ROMAP 0x04000000 /* ext mapping is r-o at MMU */ 359 #define M_EXT_RW 0x08000000 /* ext storage is writable */ 360 361 /* for source-level compatibility */ 362 #define M_CLUSTER M_EXT_CLUSTER 363 364 #define M_FLAGS_BITS \ 365 "\20\1EXT\2PKTHDR\3EOR\4PROTO1\5AUTHIPHDR\6DECRYPTED\7LOOP\10AUTHIPDGM" \ 366 "\11BCAST\12MCAST\13CANFASTFWD\14ANYCAST6\15LINK0\16LINK1\17LINK2\20LINK3" \ 367 "\21LINK4\22LINK5\23LINK6\24LINK7" \ 368 "\31EXT_CLUSTER\32EXT_PAGES\33EXT_ROMAP\34EXT_RW" 369 370 /* flags copied when copying m_pkthdr */ 371 #define M_COPYFLAGS (M_PKTHDR|M_EOR|M_BCAST|M_MCAST|M_CANFASTFWD|M_ANYCAST6|M_LINK0|M_LINK1|M_LINK2|M_AUTHIPHDR|M_DECRYPTED|M_LOOP|M_AUTHIPDGM) 372 373 /* flag copied when shallow-copying external storage */ 374 #define M_EXTCOPYFLAGS (M_EXT|M_EXT_FLAGS) 375 376 /* mbuf types */ 377 #define MT_FREE 0 /* should be on free list */ 378 #define MT_DATA 1 /* dynamic (data) allocation */ 379 #define MT_HEADER 2 /* packet header */ 380 #define MT_SONAME 3 /* socket name */ 381 #define MT_SOOPTS 4 /* socket options */ 382 #define MT_FTABLE 5 /* fragment reassembly header */ 383 #define MT_CONTROL 6 /* extra-data protocol message */ 384 #define MT_OOBDATA 7 /* expedited data */ 385 386 #ifdef MBUFTYPES 387 static const char * const mbuftypes[] = { 388 "mbfree", 389 "mbdata", 390 "mbheader", 391 "mbsoname", 392 "mbsopts", 393 "mbftable", 394 "mbcontrol", 395 "mboobdata", 396 }; 397 #endif 398 399 /* flags to m_get/MGET */ 400 #define M_DONTWAIT M_NOWAIT 401 #define M_WAIT M_WAITOK 402 403 #ifdef MBUFTRACE 404 /* 405 * mbuf allocation tracing 406 */ 407 void mowner_init(struct mbuf *, int); 408 void mowner_ref(struct mbuf *, int); 409 void m_claim(struct mbuf *, struct mowner *); 410 void mowner_revoke(struct mbuf *, bool, int); 411 void mowner_attach(struct mowner *); 412 void mowner_detach(struct mowner *); 413 void m_claimm(struct mbuf *, struct mowner *); 414 #else 415 #define mowner_init(m, type) do { } while (/* CONSTCOND */ 0) 416 #define mowner_ref(m, flags) do { } while (/* CONSTCOND */ 0) 417 #define mowner_revoke(m, all, flags) do { } while (/* CONSTCOND */ 0) 418 #define m_claim(m, mowner) do { } while (/* CONSTCOND */ 0) 419 #define mowner_attach(mo) do { } while (/* CONSTCOND */ 0) 420 #define mowner_detach(mo) do { } while (/* CONSTCOND */ 0) 421 #define m_claimm(m, mo) do { } while (/* CONSTCOND */ 0) 422 #endif 423 424 #define MCLAIM(m, mo) m_claim((m), (mo)) 425 #define MOWNER_ATTACH(mo) mowner_attach(mo) 426 #define MOWNER_DETACH(mo) mowner_detach(mo) 427 428 /* 429 * mbuf allocation/deallocation macros: 430 * 431 * MGET(struct mbuf *m, int how, int type) 432 * allocates an mbuf and initializes it to contain internal data. 433 * 434 * MGETHDR(struct mbuf *m, int how, int type) 435 * allocates an mbuf and initializes it to contain a packet header 436 * and internal data. 437 * 438 * If 'how' is M_WAIT, these macros (and the corresponding functions) 439 * are guaranteed to return successfully. 440 */ 441 #define MGET(m, how, type) m = m_get((how), (type)) 442 #define MGETHDR(m, how, type) m = m_gethdr((how), (type)) 443 444 #if defined(_KERNEL) 445 #define _M_ 446 /* 447 * Macros for tracking external storage associated with an mbuf. 448 */ 449 #ifdef DEBUG 450 #define MCLREFDEBUGN(m, file, line) \ 451 do { \ 452 (m)->m_ext.ext_nfile = (file); \ 453 (m)->m_ext.ext_nline = (line); \ 454 } while (/* CONSTCOND */ 0) 455 456 #define MCLREFDEBUGO(m, file, line) \ 457 do { \ 458 (m)->m_ext.ext_ofile = (file); \ 459 (m)->m_ext.ext_oline = (line); \ 460 } while (/* CONSTCOND */ 0) 461 #else 462 #define MCLREFDEBUGN(m, file, line) 463 #define MCLREFDEBUGO(m, file, line) 464 #endif 465 466 #define MCLINITREFERENCE(m) \ 467 do { \ 468 KDASSERT(((m)->m_flags & M_EXT) == 0); \ 469 (m)->m_ext_ref = (m); \ 470 (m)->m_ext.ext_refcnt = 1; \ 471 MCLREFDEBUGO((m), __FILE__, __LINE__); \ 472 MCLREFDEBUGN((m), NULL, 0); \ 473 } while (/* CONSTCOND */ 0) 474 475 /* 476 * Macros for mbuf external storage. 477 * 478 * MCLGET allocates and adds an mbuf cluster to a normal mbuf; 479 * the flag M_EXT is set upon success. 480 * 481 * MEXTMALLOC allocates external storage and adds it to 482 * a normal mbuf; the flag M_EXT is set upon success. 483 * 484 * MEXTADD adds pre-allocated external storage to 485 * a normal mbuf; the flag M_EXT is set upon success. 486 */ 487 488 #define _MCLGET(m, pool_cache, size, how) \ 489 do { \ 490 (m)->m_ext_storage.ext_buf = (char *) \ 491 pool_cache_get_paddr((pool_cache), \ 492 (how) == M_WAIT ? (PR_WAITOK|PR_LIMITFAIL) : 0, \ 493 &(m)->m_ext_storage.ext_paddr); \ 494 if ((m)->m_ext_storage.ext_buf != NULL) { \ 495 MCLINITREFERENCE(m); \ 496 (m)->m_data = (m)->m_ext.ext_buf; \ 497 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | \ 498 M_EXT|M_CLUSTER|M_EXT_RW; \ 499 (m)->m_ext.ext_flags = 0; \ 500 (m)->m_ext.ext_size = (size); \ 501 (m)->m_ext.ext_free = NULL; \ 502 (m)->m_ext.ext_arg = (pool_cache); \ 503 /* ext_paddr initialized above */ \ 504 mowner_ref((m), M_EXT|M_CLUSTER); \ 505 } \ 506 } while (/* CONSTCOND */ 0) 507 508 /* 509 * The standard mbuf cluster pool. 510 */ 511 #define MCLGET(m, how) _MCLGET((m), mcl_cache, MCLBYTES, (how)) 512 513 #define MEXTMALLOC(m, size, how) \ 514 do { \ 515 (m)->m_ext_storage.ext_buf = (char *) \ 516 malloc((size), mbtypes[(m)->m_type], (how)); \ 517 if ((m)->m_ext_storage.ext_buf != NULL) { \ 518 MCLINITREFERENCE(m); \ 519 (m)->m_data = (m)->m_ext.ext_buf; \ 520 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | \ 521 M_EXT|M_EXT_RW; \ 522 (m)->m_ext.ext_flags = 0; \ 523 (m)->m_ext.ext_size = (size); \ 524 (m)->m_ext.ext_free = NULL; \ 525 (m)->m_ext.ext_arg = NULL; \ 526 mowner_ref((m), M_EXT); \ 527 } \ 528 } while (/* CONSTCOND */ 0) 529 530 #define MEXTADD(m, buf, size, type, free, arg) \ 531 do { \ 532 MCLINITREFERENCE(m); \ 533 (m)->m_data = (m)->m_ext.ext_buf = (char *)(buf); \ 534 (m)->m_flags = ((m)->m_flags & ~M_EXTCOPYFLAGS) | M_EXT; \ 535 (m)->m_ext.ext_flags = 0; \ 536 (m)->m_ext.ext_size = (size); \ 537 (m)->m_ext.ext_free = (free); \ 538 (m)->m_ext.ext_arg = (arg); \ 539 mowner_ref((m), M_EXT); \ 540 } while (/* CONSTCOND */ 0) 541 542 /* 543 * Reset the data pointer on an mbuf. 544 */ 545 #define MRESETDATA(m) \ 546 do { \ 547 if ((m)->m_flags & M_EXT) \ 548 (m)->m_data = (m)->m_ext.ext_buf; \ 549 else if ((m)->m_flags & M_PKTHDR) \ 550 (m)->m_data = (m)->m_pktdat; \ 551 else \ 552 (m)->m_data = (m)->m_dat; \ 553 } while (/* CONSTCOND */ 0) 554 555 /* 556 * MFREE(struct mbuf *m, struct mbuf *n) 557 * Free a single mbuf and associated external storage. 558 * Place the successor, if any, in n. 559 */ 560 #define MFREE(m, n) \ 561 mowner_revoke((m), 1, (m)->m_flags); \ 562 mbstat_type_add((m)->m_type, -1); \ 563 if ((m)->m_flags & M_PKTHDR) \ 564 m_tag_delete_chain((m), NULL); \ 565 (n) = (m)->m_next; \ 566 if ((m)->m_flags & M_EXT) { \ 567 m_ext_free((m)); \ 568 } else { \ 569 KASSERT((m)->m_type != MT_FREE); \ 570 (m)->m_type = MT_FREE; \ 571 pool_cache_put(mb_cache, (m)); \ 572 } \ 573 574 /* 575 * Copy mbuf pkthdr from `from' to `to'. 576 * `from' must have M_PKTHDR set, and `to' must be empty. 577 */ 578 #define M_COPY_PKTHDR(to, from) \ 579 do { \ 580 (to)->m_pkthdr = (from)->m_pkthdr; \ 581 (to)->m_flags = (from)->m_flags & M_COPYFLAGS; \ 582 SLIST_INIT(&(to)->m_pkthdr.tags); \ 583 m_tag_copy_chain((to), (from)); \ 584 (to)->m_data = (to)->m_pktdat; \ 585 } while (/* CONSTCOND */ 0) 586 587 /* 588 * Move mbuf pkthdr from `from' to `to'. 589 * `from' must have M_PKTHDR set, and `to' must be empty. 590 */ 591 #define M_MOVE_PKTHDR(to, from) m_move_pkthdr(to, from) 592 593 /* 594 * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place 595 * an object of the specified size at the end of the mbuf, longword aligned. 596 */ 597 #define M_ALIGN(m, len) \ 598 do { \ 599 (m)->m_data += (MLEN - (len)) &~ (sizeof(long) - 1); \ 600 } while (/* CONSTCOND */ 0) 601 602 /* 603 * As above, for mbufs allocated with m_gethdr/MGETHDR 604 * or initialized by M_COPY_PKTHDR. 605 */ 606 #define MH_ALIGN(m, len) \ 607 do { \ 608 (m)->m_data += (MHLEN - (len)) &~ (sizeof(long) - 1); \ 609 } while (/* CONSTCOND */ 0) 610 611 /* 612 * Determine if an mbuf's data area is read-only. This is true 613 * if external storage is read-only mapped, or not marked as R/W, 614 * or referenced by more than one mbuf. 615 */ 616 #define M_READONLY(m) \ 617 (((m)->m_flags & M_EXT) != 0 && \ 618 (((m)->m_flags & (M_EXT_ROMAP|M_EXT_RW)) != M_EXT_RW || \ 619 (m)->m_ext.ext_refcnt > 1)) 620 621 #define M_UNWRITABLE(__m, __len) \ 622 ((__m)->m_len < (__len) || M_READONLY((__m))) 623 /* 624 * Determine if an mbuf's data area is read-only at the MMU. 625 */ 626 #define M_ROMAP(m) \ 627 (((m)->m_flags & (M_EXT|M_EXT_ROMAP)) == (M_EXT|M_EXT_ROMAP)) 628 629 /* 630 * Compute the amount of space available 631 * before the current start of data in an mbuf. 632 */ 633 #define _M_LEADINGSPACE(m) \ 634 ((m)->m_flags & M_EXT ? (m)->m_data - (m)->m_ext.ext_buf : \ 635 (m)->m_flags & M_PKTHDR ? (m)->m_data - (m)->m_pktdat : \ 636 (m)->m_data - (m)->m_dat) 637 638 #define M_LEADINGSPACE(m) \ 639 (M_READONLY((m)) ? 0 : _M_LEADINGSPACE((m))) 640 641 /* 642 * Compute the amount of space available 643 * after the end of data in an mbuf. 644 */ 645 #define _M_TRAILINGSPACE(m) \ 646 ((m)->m_flags & M_EXT ? (m)->m_ext.ext_buf + (m)->m_ext.ext_size - \ 647 ((m)->m_data + (m)->m_len) : \ 648 &(m)->m_dat[MLEN] - ((m)->m_data + (m)->m_len)) 649 650 #define M_TRAILINGSPACE(m) \ 651 (M_READONLY((m)) ? 0 : _M_TRAILINGSPACE((m))) 652 653 /* 654 * Compute the address of an mbuf's data area. 655 */ 656 #define M_BUFADDR(m) \ 657 (((m)->m_flags & M_PKTHDR) ? (m)->m_pktdat : (m)->m_dat) 658 659 /* 660 * Compute the offset of the beginning of the data buffer of a non-ext 661 * mbuf. 662 */ 663 #define M_BUFOFFSET(m) \ 664 (((m)->m_flags & M_PKTHDR) ? \ 665 offsetof(struct mbuf, m_pktdat) : offsetof(struct mbuf, m_dat)) 666 667 /* 668 * Arrange to prepend space of size plen to mbuf m. 669 * If a new mbuf must be allocated, how specifies whether to wait. 670 * If how is M_DONTWAIT and allocation fails, the original mbuf chain 671 * is freed and m is set to NULL. 672 */ 673 #define M_PREPEND(m, plen, how) \ 674 do { \ 675 if (M_LEADINGSPACE(m) >= (plen)) { \ 676 (m)->m_data -= (plen); \ 677 (m)->m_len += (plen); \ 678 } else \ 679 (m) = m_prepend((m), (plen), (how)); \ 680 if ((m) && (m)->m_flags & M_PKTHDR) \ 681 (m)->m_pkthdr.len += (plen); \ 682 } while (/* CONSTCOND */ 0) 683 684 /* change mbuf to new type */ 685 #define MCHTYPE(m, t) \ 686 do { \ 687 KASSERT((t) != MT_FREE); \ 688 mbstat_type_add((m)->m_type, -1); \ 689 mbstat_type_add(t, 1); \ 690 (m)->m_type = t; \ 691 } while (/* CONSTCOND */ 0) 692 693 /* length to m_copy to copy all */ 694 #define M_COPYALL -1 695 696 /* compatibility with 4.3 */ 697 #define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT) 698 699 /* 700 * Allow drivers and/or protocols to use the rcvif member of 701 * PKTHDR mbufs to store private context information. 702 */ 703 #define M_GETCTX(m, t) ((t)(m)->m_pkthdr.rcvif) 704 #define M_SETCTX(m, c) ((void)((m)->m_pkthdr.rcvif = (void *)(c))) 705 706 #endif /* defined(_KERNEL) */ 707 708 /* 709 * Simple mbuf queueing system 710 * 711 * this is basically a SIMPLEQ adapted to mbuf use (ie using 712 * m_nextpkt instead of field.sqe_next). 713 * 714 * m_next is ignored, so queueing chains of mbufs is possible 715 */ 716 #define MBUFQ_HEAD(name) \ 717 struct name { \ 718 struct mbuf *mq_first; \ 719 struct mbuf **mq_last; \ 720 } 721 722 #define MBUFQ_INIT(q) do { \ 723 (q)->mq_first = NULL; \ 724 (q)->mq_last = &(q)->mq_first; \ 725 } while (/*CONSTCOND*/0) 726 727 #define MBUFQ_ENQUEUE(q, m) do { \ 728 (m)->m_nextpkt = NULL; \ 729 *(q)->mq_last = (m); \ 730 (q)->mq_last = &(m)->m_nextpkt; \ 731 } while (/*CONSTCOND*/0) 732 733 #define MBUFQ_PREPEND(q, m) do { \ 734 if (((m)->m_nextpkt = (q)->mq_first) == NULL) \ 735 (q)->mq_last = &(m)->m_nextpkt; \ 736 (q)->mq_first = (m); \ 737 } while (/*CONSTCOND*/0) 738 739 #define MBUFQ_DEQUEUE(q, m) do { \ 740 if (((m) = (q)->mq_first) != NULL) { \ 741 if (((q)->mq_first = (m)->m_nextpkt) == NULL) \ 742 (q)->mq_last = &(q)->mq_first; \ 743 else \ 744 (m)->m_nextpkt = NULL; \ 745 } \ 746 } while (/*CONSTCOND*/0) 747 748 #define MBUFQ_DRAIN(q) do { \ 749 struct mbuf *__m0; \ 750 while ((__m0 = (q)->mq_first) != NULL) { \ 751 (q)->mq_first = __m0->m_nextpkt; \ 752 m_freem(__m0); \ 753 } \ 754 (q)->mq_last = &(q)->mq_first; \ 755 } while (/*CONSTCOND*/0) 756 757 #define MBUFQ_FIRST(q) ((q)->mq_first) 758 #define MBUFQ_NEXT(m) ((m)->m_nextpkt) 759 #define MBUFQ_LAST(q) (*(q)->mq_last) 760 761 /* 762 * Mbuf statistics. 763 * For statistics related to mbuf and cluster allocations, see also the 764 * pool headers (mb_cache and mcl_cache). 765 */ 766 struct mbstat { 767 u_long _m_spare; /* formerly m_mbufs */ 768 u_long _m_spare1; /* formerly m_clusters */ 769 u_long _m_spare2; /* spare field */ 770 u_long _m_spare3; /* formely m_clfree - free clusters */ 771 u_long m_drops; /* times failed to find space */ 772 u_long m_wait; /* times waited for space */ 773 u_long m_drain; /* times drained protocols for space */ 774 u_short m_mtypes[256]; /* type specific mbuf allocations */ 775 }; 776 777 struct mbstat_cpu { 778 u_int m_mtypes[256]; /* type specific mbuf allocations */ 779 }; 780 781 /* 782 * Mbuf sysctl variables. 783 */ 784 #define MBUF_MSIZE 1 /* int: mbuf base size */ 785 #define MBUF_MCLBYTES 2 /* int: mbuf cluster size */ 786 #define MBUF_NMBCLUSTERS 3 /* int: limit on the # of clusters */ 787 #define MBUF_MBLOWAT 4 /* int: mbuf low water mark */ 788 #define MBUF_MCLLOWAT 5 /* int: mbuf cluster low water mark */ 789 #define MBUF_STATS 6 /* struct: mbstat */ 790 #define MBUF_MOWNERS 7 /* struct: m_owner[] */ 791 #define MBUF_MAXID 8 /* number of valid MBUF ids */ 792 793 #define CTL_MBUF_NAMES { \ 794 { 0, 0 }, \ 795 { "msize", CTLTYPE_INT }, \ 796 { "mclbytes", CTLTYPE_INT }, \ 797 { "nmbclusters", CTLTYPE_INT }, \ 798 { "mblowat", CTLTYPE_INT }, \ 799 { "mcllowat", CTLTYPE_INT }, \ 800 { 0 /* "stats" */, CTLTYPE_STRUCT }, \ 801 { 0 /* "mowners" */, CTLTYPE_STRUCT }, \ 802 } 803 804 #ifdef _KERNEL 805 extern struct mbstat mbstat; 806 extern int nmbclusters; /* limit on the # of clusters */ 807 extern int mblowat; /* mbuf low water mark */ 808 extern int mcllowat; /* mbuf cluster low water mark */ 809 extern int max_linkhdr; /* largest link-level header */ 810 extern int max_protohdr; /* largest protocol header */ 811 extern int max_hdr; /* largest link+protocol header */ 812 extern int max_datalen; /* MHLEN - max_hdr */ 813 extern const int msize; /* mbuf base size */ 814 extern const int mclbytes; /* mbuf cluster size */ 815 extern pool_cache_t mb_cache; 816 extern pool_cache_t mcl_cache; 817 #ifdef MBUFTRACE 818 LIST_HEAD(mownerhead, mowner); 819 extern struct mownerhead mowners; 820 extern struct mowner unknown_mowners[]; 821 extern struct mowner revoked_mowner; 822 #endif 823 824 MALLOC_DECLARE(M_MBUF); 825 MALLOC_DECLARE(M_SONAME); 826 827 struct mbuf *m_copym(struct mbuf *, int, int, int); 828 struct mbuf *m_copypacket(struct mbuf *, int); 829 struct mbuf *m_devget(char *, int, int, struct ifnet *, 830 void (*copy)(const void *, void *, size_t)); 831 struct mbuf *m_dup(struct mbuf *, int, int, int); 832 struct mbuf *m_free(struct mbuf *); 833 struct mbuf *m_get(int, int); 834 struct mbuf *m_getclr(int, int); 835 struct mbuf *m_gethdr(int, int); 836 struct mbuf *m_prepend(struct mbuf *,int, int); 837 struct mbuf *m_pulldown(struct mbuf *, int, int, int *); 838 struct mbuf *m_pullup(struct mbuf *, int); 839 struct mbuf *m_copyup(struct mbuf *, int, int); 840 struct mbuf *m_split(struct mbuf *,int, int); 841 struct mbuf *m_getptr(struct mbuf *, int, int *); 842 void m_adj(struct mbuf *, int); 843 struct mbuf *m_defrag(struct mbuf *, int); 844 int m_apply(struct mbuf *, int, int, 845 int (*)(void *, void *, unsigned int), void *); 846 void m_cat(struct mbuf *,struct mbuf *); 847 void m_clget(struct mbuf *, int); 848 int m_mballoc(int, int); 849 void m_copyback(struct mbuf *, int, int, const void *); 850 struct mbuf *m_copyback_cow(struct mbuf *, int, int, const void *, int); 851 int m_makewritable(struct mbuf **, int, int, int); 852 struct mbuf *m_getcl(int, int, int); 853 void m_copydata(struct mbuf *, int, int, void *); 854 void m_freem(struct mbuf *); 855 void m_reclaim(void *, int); 856 void mbinit(void); 857 void m_ext_free(struct mbuf *); 858 char * m_mapin(struct mbuf *); 859 void m_move_pkthdr(struct mbuf *, struct mbuf *); 860 861 bool m_ensure_contig(struct mbuf **, int); 862 struct mbuf *m_add(struct mbuf *, struct mbuf *); 863 void m_align(struct mbuf *, int); 864 int m_append(struct mbuf *, int, const void *); 865 866 /* Inline routines. */ 867 static __inline u_int m_length(const struct mbuf *) __unused; 868 869 /* Statistics */ 870 void mbstat_type_add(int, int); 871 872 /* Packet tag routines */ 873 struct m_tag *m_tag_get(int, int, int); 874 void m_tag_free(struct m_tag *); 875 void m_tag_prepend(struct mbuf *, struct m_tag *); 876 void m_tag_unlink(struct mbuf *, struct m_tag *); 877 void m_tag_delete(struct mbuf *, struct m_tag *); 878 void m_tag_delete_chain(struct mbuf *, struct m_tag *); 879 void m_tag_delete_nonpersistent(struct mbuf *); 880 struct m_tag *m_tag_find(const struct mbuf *, int, struct m_tag *); 881 struct m_tag *m_tag_copy(struct m_tag *); 882 int m_tag_copy_chain(struct mbuf *, struct mbuf *); 883 void m_tag_init(struct mbuf *); 884 struct m_tag *m_tag_first(struct mbuf *); 885 struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); 886 887 /* Packet tag types */ 888 #define PACKET_TAG_NONE 0 /* Nothing */ 889 #define PACKET_TAG_VLAN 1 /* VLAN ID */ 890 #define PACKET_TAG_ENCAP 2 /* encapsulation data */ 891 #define PACKET_TAG_ESP 3 /* ESP information */ 892 #define PACKET_TAG_PF 11 /* packet filter */ 893 #define PACKET_TAG_ALTQ_QID 12 /* ALTQ queue id */ 894 895 #define PACKET_TAG_IPSEC_IN_CRYPTO_DONE 16 896 #define PACKET_TAG_IPSEC_IN_DONE 17 897 #define PACKET_TAG_IPSEC_OUT_DONE 18 898 #define PACKET_TAG_IPSEC_OUT_CRYPTO_NEEDED 19 /* NIC IPsec crypto req'ed */ 899 #define PACKET_TAG_IPSEC_IN_COULD_DO_CRYPTO 20 /* NIC notifies IPsec */ 900 #define PACKET_TAG_IPSEC_PENDING_TDB 21 /* Reminder to do IPsec */ 901 902 #define PACKET_TAG_IPSEC_SOCKET 22 /* IPSEC socket ref */ 903 #define PACKET_TAG_IPSEC_HISTORY 23 /* IPSEC history */ 904 905 #define PACKET_TAG_IPSEC_NAT_T_PORTS 25 /* two uint16_t */ 906 907 #define PACKET_TAG_INET6 26 /* IPv6 info */ 908 909 #define PACKET_TAG_ECO_RETRYPARMS 27 /* Econet retry parameters */ 910 911 #define PACKET_TAG_TUNNEL_INFO 28 /* tunnel identification and 912 * protocol callback, for 913 * loop detection/recovery 914 */ 915 916 #define PACKET_TAG_MPLS 29 /* Indicate it's for MPLS */ 917 918 /* 919 * Return the number of bytes in the mbuf chain, m. 920 */ 921 static __inline u_int 922 m_length(const struct mbuf *m) 923 { 924 const struct mbuf *m0; 925 u_int pktlen; 926 927 if ((m->m_flags & M_PKTHDR) != 0) 928 return m->m_pkthdr.len; 929 930 pktlen = 0; 931 for (m0 = m; m0 != NULL; m0 = m0->m_next) 932 pktlen += m0->m_len; 933 return pktlen; 934 } 935 936 void m_print(const struct mbuf *, const char *, void (*)(const char *, ...) 937 __printflike(1, 2)); 938 939 #endif /* _KERNEL */ 940 #endif /* !_SYS_MBUF_H_ */ 941