1 /* $NetBSD: uipc_mbuf.c,v 1.59 2002/03/09 01:46:33 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 /* 41 * Copyright (c) 1982, 1986, 1988, 1991, 1993 42 * The Regents of the University of California. All rights reserved. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by the University of 55 * California, Berkeley and its contributors. 56 * 4. Neither the name of the University nor the names of its contributors 57 * may be used to endorse or promote products derived from this software 58 * without specific prior written permission. 59 * 60 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 61 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 62 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 63 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 64 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 65 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 66 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 67 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 68 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 69 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 70 * SUCH DAMAGE. 71 * 72 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 73 */ 74 75 #include <sys/cdefs.h> 76 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.59 2002/03/09 01:46:33 thorpej Exp $"); 77 78 #include <sys/param.h> 79 #include <sys/systm.h> 80 #include <sys/proc.h> 81 #include <sys/malloc.h> 82 #include <sys/map.h> 83 #define MBTYPES 84 #include <sys/mbuf.h> 85 #include <sys/kernel.h> 86 #include <sys/syslog.h> 87 #include <sys/domain.h> 88 #include <sys/protosw.h> 89 #include <sys/pool.h> 90 #include <sys/socket.h> 91 #include <sys/sysctl.h> 92 93 #include <net/if.h> 94 95 #include <uvm/uvm_extern.h> 96 97 98 struct pool mbpool; /* mbuf pool */ 99 struct pool mclpool; /* mbuf cluster pool */ 100 101 struct pool_cache mbpool_cache; 102 struct pool_cache mclpool_cache; 103 104 struct mbstat mbstat; 105 int max_linkhdr; 106 int max_protohdr; 107 int max_hdr; 108 int max_datalen; 109 110 void *mclpool_alloc __P((struct pool *, int)); 111 void mclpool_release __P((struct pool *, void *)); 112 113 struct pool_allocator mclpool_allocator = { 114 mclpool_alloc, mclpool_release, 0, 115 }; 116 117 static struct mbuf *m_copym0 __P((struct mbuf *, int, int, int, int)); 118 119 const char mclpool_warnmsg[] = 120 "WARNING: mclpool limit reached; increase NMBCLUSTERS"; 121 122 /* 123 * Initialize the mbuf allcator. 124 */ 125 void 126 mbinit() 127 { 128 129 pool_init(&mbpool, msize, 0, 0, 0, "mbpl", NULL); 130 pool_init(&mclpool, mclbytes, 0, 0, 0, "mclpl", &mclpool_allocator); 131 132 pool_set_drain_hook(&mbpool, m_reclaim, NULL); 133 pool_set_drain_hook(&mclpool, m_reclaim, NULL); 134 135 pool_cache_init(&mbpool_cache, &mbpool, NULL, NULL, NULL); 136 pool_cache_init(&mclpool_cache, &mclpool, NULL, NULL, NULL); 137 138 /* 139 * Set the hard limit on the mclpool to the number of 140 * mbuf clusters the kernel is to support. Log the limit 141 * reached message max once a minute. 142 */ 143 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60); 144 145 /* 146 * Set a low water mark for both mbufs and clusters. This should 147 * help ensure that they can be allocated in a memory starvation 148 * situation. This is important for e.g. diskless systems which 149 * must allocate mbufs in order for the pagedaemon to clean pages. 150 */ 151 pool_setlowat(&mbpool, mblowat); 152 pool_setlowat(&mclpool, mcllowat); 153 } 154 155 int 156 sysctl_dombuf(name, namelen, oldp, oldlenp, newp, newlen) 157 int *name; 158 u_int namelen; 159 void *oldp; 160 size_t *oldlenp; 161 void *newp; 162 size_t newlen; 163 { 164 int error, newval; 165 166 /* All sysctl names at this level are terminal. */ 167 if (namelen != 1) 168 return (ENOTDIR); /* overloaded */ 169 170 switch (name[0]) { 171 case MBUF_MSIZE: 172 return (sysctl_rdint(oldp, oldlenp, newp, msize)); 173 case MBUF_MCLBYTES: 174 return (sysctl_rdint(oldp, oldlenp, newp, mclbytes)); 175 case MBUF_NMBCLUSTERS: 176 /* 177 * If we have direct-mapped pool pages, we can adjust this 178 * number on the fly. If not, we're limited by the size 179 * of mb_map, and cannot change this value. 180 * 181 * Note: we only allow the value to be increased, never 182 * decreased. 183 */ 184 if (mb_map == NULL) { 185 newval = nmbclusters; 186 error = sysctl_int(oldp, oldlenp, newp, newlen, 187 &newval); 188 if (error != 0) 189 return (error); 190 if (newp != NULL) { 191 if (newval >= nmbclusters) { 192 nmbclusters = newval; 193 pool_sethardlimit(&mclpool, 194 nmbclusters, mclpool_warnmsg, 60); 195 } else 196 error = EINVAL; 197 } 198 return (error); 199 } else 200 return (sysctl_rdint(oldp, oldlenp, newp, nmbclusters)); 201 case MBUF_MBLOWAT: 202 case MBUF_MCLLOWAT: 203 /* New value must be >= 0. */ 204 newval = (name[0] == MBUF_MBLOWAT) ? mblowat : mcllowat; 205 error = sysctl_int(oldp, oldlenp, newp, newlen, &newval); 206 if (error != 0) 207 return (error); 208 if (newp != NULL) { 209 if (newval >= 0) { 210 if (name[0] == MBUF_MBLOWAT) { 211 mblowat = newval; 212 pool_setlowat(&mbpool, newval); 213 } else { 214 mcllowat = newval; 215 pool_setlowat(&mclpool, newval); 216 } 217 } else 218 error = EINVAL; 219 } 220 return (error); 221 default: 222 return (EOPNOTSUPP); 223 } 224 /* NOTREACHED */ 225 } 226 227 void * 228 mclpool_alloc(pp, flags) 229 struct pool *pp; 230 int flags; 231 { 232 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 233 234 return ((void *)uvm_km_alloc_poolpage1(mb_map, NULL, waitok)); 235 } 236 237 void 238 mclpool_release(pp, v) 239 struct pool *pp; 240 void *v; 241 { 242 243 uvm_km_free_poolpage1(mb_map, (vaddr_t)v); 244 } 245 246 void 247 m_reclaim(void *arg, int flags) 248 { 249 struct domain *dp; 250 struct protosw *pr; 251 struct ifnet *ifp; 252 int s = splvm(); 253 254 for (dp = domains; dp; dp = dp->dom_next) 255 for (pr = dp->dom_protosw; 256 pr < dp->dom_protoswNPROTOSW; pr++) 257 if (pr->pr_drain) 258 (*pr->pr_drain)(); 259 for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) 260 if (ifp->if_drain) 261 (*ifp->if_drain)(ifp); 262 splx(s); 263 mbstat.m_drain++; 264 } 265 266 /* 267 * Space allocation routines. 268 * These are also available as macros 269 * for critical paths. 270 */ 271 struct mbuf * 272 m_get(nowait, type) 273 int nowait, type; 274 { 275 struct mbuf *m; 276 277 MGET(m, nowait, type); 278 return (m); 279 } 280 281 struct mbuf * 282 m_gethdr(nowait, type) 283 int nowait, type; 284 { 285 struct mbuf *m; 286 287 MGETHDR(m, nowait, type); 288 return (m); 289 } 290 291 struct mbuf * 292 m_getclr(nowait, type) 293 int nowait, type; 294 { 295 struct mbuf *m; 296 297 MGET(m, nowait, type); 298 if (m == 0) 299 return (0); 300 memset(mtod(m, caddr_t), 0, MLEN); 301 return (m); 302 } 303 304 struct mbuf * 305 m_free(m) 306 struct mbuf *m; 307 { 308 struct mbuf *n; 309 310 MFREE(m, n); 311 return (n); 312 } 313 314 void 315 m_freem(m) 316 struct mbuf *m; 317 { 318 struct mbuf *n; 319 320 if (m == NULL) 321 return; 322 do { 323 MFREE(m, n); 324 m = n; 325 } while (m); 326 } 327 328 /* 329 * Mbuffer utility routines. 330 */ 331 332 /* 333 * Lesser-used path for M_PREPEND: 334 * allocate new mbuf to prepend to chain, 335 * copy junk along. 336 */ 337 struct mbuf * 338 m_prepend(m, len, how) 339 struct mbuf *m; 340 int len, how; 341 { 342 struct mbuf *mn; 343 344 MGET(mn, how, m->m_type); 345 if (mn == (struct mbuf *)NULL) { 346 m_freem(m); 347 return ((struct mbuf *)NULL); 348 } 349 if (m->m_flags & M_PKTHDR) { 350 M_COPY_PKTHDR(mn, m); 351 m->m_flags &= ~M_PKTHDR; 352 } 353 mn->m_next = m; 354 m = mn; 355 if (len < MHLEN) 356 MH_ALIGN(m, len); 357 m->m_len = len; 358 return (m); 359 } 360 361 /* 362 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 363 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 364 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 365 */ 366 int MCFail; 367 368 struct mbuf * 369 m_copym(m, off0, len, wait) 370 struct mbuf *m; 371 int off0, wait; 372 int len; 373 { 374 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */ 375 } 376 377 struct mbuf * 378 m_dup(m, off0, len, wait) 379 struct mbuf *m; 380 int off0, wait; 381 int len; 382 { 383 return m_copym0(m, off0, len, wait, 1); /* deep copy */ 384 } 385 386 static struct mbuf * 387 m_copym0(m, off0, len, wait, deep) 388 struct mbuf *m; 389 int off0, wait; 390 int len; 391 int deep; /* deep copy */ 392 { 393 struct mbuf *n, **np; 394 int off = off0; 395 struct mbuf *top; 396 int copyhdr = 0; 397 398 if (off < 0 || len < 0) 399 panic("m_copym: off %d, len %d", off, len); 400 if (off == 0 && m->m_flags & M_PKTHDR) 401 copyhdr = 1; 402 while (off > 0) { 403 if (m == 0) 404 panic("m_copym: m == 0"); 405 if (off < m->m_len) 406 break; 407 off -= m->m_len; 408 m = m->m_next; 409 } 410 np = ⊤ 411 top = 0; 412 while (len > 0) { 413 if (m == 0) { 414 if (len != M_COPYALL) 415 panic("m_copym: m == 0 and not COPYALL"); 416 break; 417 } 418 MGET(n, wait, m->m_type); 419 *np = n; 420 if (n == 0) 421 goto nospace; 422 if (copyhdr) { 423 M_COPY_PKTHDR(n, m); 424 if (len == M_COPYALL) 425 n->m_pkthdr.len -= off0; 426 else 427 n->m_pkthdr.len = len; 428 copyhdr = 0; 429 } 430 n->m_len = min(len, m->m_len - off); 431 if (m->m_flags & M_EXT) { 432 if (!deep) { 433 n->m_data = m->m_data + off; 434 n->m_ext = m->m_ext; 435 MCLADDREFERENCE(m, n); 436 } else { 437 /* 438 * we are unsure about the way m was allocated. 439 * copy into multiple MCLBYTES cluster mbufs. 440 */ 441 MCLGET(n, wait); 442 n->m_len = 0; 443 n->m_len = M_TRAILINGSPACE(n); 444 n->m_len = min(n->m_len, len); 445 n->m_len = min(n->m_len, m->m_len - off); 446 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, 447 (unsigned)n->m_len); 448 } 449 } else 450 memcpy(mtod(n, caddr_t), mtod(m, caddr_t)+off, 451 (unsigned)n->m_len); 452 if (len != M_COPYALL) 453 len -= n->m_len; 454 off += n->m_len; 455 #ifdef DIAGNOSTIC 456 if (off > m->m_len) 457 panic("m_copym0 overrun"); 458 #endif 459 if (off == m->m_len) { 460 m = m->m_next; 461 off = 0; 462 } 463 np = &n->m_next; 464 } 465 if (top == 0) 466 MCFail++; 467 return (top); 468 nospace: 469 m_freem(top); 470 MCFail++; 471 return (0); 472 } 473 474 /* 475 * Copy an entire packet, including header (which must be present). 476 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 477 */ 478 struct mbuf * 479 m_copypacket(m, how) 480 struct mbuf *m; 481 int how; 482 { 483 struct mbuf *top, *n, *o; 484 485 MGET(n, how, m->m_type); 486 top = n; 487 if (!n) 488 goto nospace; 489 490 M_COPY_PKTHDR(n, m); 491 n->m_len = m->m_len; 492 if (m->m_flags & M_EXT) { 493 n->m_data = m->m_data; 494 n->m_ext = m->m_ext; 495 MCLADDREFERENCE(m, n); 496 } else { 497 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 498 } 499 500 m = m->m_next; 501 while (m) { 502 MGET(o, how, m->m_type); 503 if (!o) 504 goto nospace; 505 506 n->m_next = o; 507 n = n->m_next; 508 509 n->m_len = m->m_len; 510 if (m->m_flags & M_EXT) { 511 n->m_data = m->m_data; 512 n->m_ext = m->m_ext; 513 MCLADDREFERENCE(m, n); 514 } else { 515 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 516 } 517 518 m = m->m_next; 519 } 520 return top; 521 nospace: 522 m_freem(top); 523 MCFail++; 524 return 0; 525 } 526 527 /* 528 * Copy data from an mbuf chain starting "off" bytes from the beginning, 529 * continuing for "len" bytes, into the indicated buffer. 530 */ 531 void 532 m_copydata(m, off, len, cp) 533 struct mbuf *m; 534 int off; 535 int len; 536 caddr_t cp; 537 { 538 unsigned count; 539 540 if (off < 0 || len < 0) 541 panic("m_copydata"); 542 while (off > 0) { 543 if (m == 0) 544 panic("m_copydata"); 545 if (off < m->m_len) 546 break; 547 off -= m->m_len; 548 m = m->m_next; 549 } 550 while (len > 0) { 551 if (m == 0) 552 panic("m_copydata"); 553 count = min(m->m_len - off, len); 554 memcpy(cp, mtod(m, caddr_t) + off, count); 555 len -= count; 556 cp += count; 557 off = 0; 558 m = m->m_next; 559 } 560 } 561 562 /* 563 * Concatenate mbuf chain n to m. 564 * Both chains must be of the same type (e.g. MT_DATA). 565 * Any m_pkthdr is not updated. 566 */ 567 void 568 m_cat(m, n) 569 struct mbuf *m, *n; 570 { 571 while (m->m_next) 572 m = m->m_next; 573 while (n) { 574 if (m->m_flags & M_EXT || 575 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 576 /* just join the two chains */ 577 m->m_next = n; 578 return; 579 } 580 /* splat the data from one into the other */ 581 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 582 (u_int)n->m_len); 583 m->m_len += n->m_len; 584 n = m_free(n); 585 } 586 } 587 588 void 589 m_adj(mp, req_len) 590 struct mbuf *mp; 591 int req_len; 592 { 593 int len = req_len; 594 struct mbuf *m; 595 int count; 596 597 if ((m = mp) == NULL) 598 return; 599 if (len >= 0) { 600 /* 601 * Trim from head. 602 */ 603 while (m != NULL && len > 0) { 604 if (m->m_len <= len) { 605 len -= m->m_len; 606 m->m_len = 0; 607 m = m->m_next; 608 } else { 609 m->m_len -= len; 610 m->m_data += len; 611 len = 0; 612 } 613 } 614 m = mp; 615 if (mp->m_flags & M_PKTHDR) 616 m->m_pkthdr.len -= (req_len - len); 617 } else { 618 /* 619 * Trim from tail. Scan the mbuf chain, 620 * calculating its length and finding the last mbuf. 621 * If the adjustment only affects this mbuf, then just 622 * adjust and return. Otherwise, rescan and truncate 623 * after the remaining size. 624 */ 625 len = -len; 626 count = 0; 627 for (;;) { 628 count += m->m_len; 629 if (m->m_next == (struct mbuf *)0) 630 break; 631 m = m->m_next; 632 } 633 if (m->m_len >= len) { 634 m->m_len -= len; 635 if (mp->m_flags & M_PKTHDR) 636 mp->m_pkthdr.len -= len; 637 return; 638 } 639 count -= len; 640 if (count < 0) 641 count = 0; 642 /* 643 * Correct length for chain is "count". 644 * Find the mbuf with last data, adjust its length, 645 * and toss data from remaining mbufs on chain. 646 */ 647 m = mp; 648 if (m->m_flags & M_PKTHDR) 649 m->m_pkthdr.len = count; 650 for (; m; m = m->m_next) { 651 if (m->m_len >= count) { 652 m->m_len = count; 653 break; 654 } 655 count -= m->m_len; 656 } 657 while (m->m_next) 658 (m = m->m_next) ->m_len = 0; 659 } 660 } 661 662 /* 663 * Rearange an mbuf chain so that len bytes are contiguous 664 * and in the data area of an mbuf (so that mtod and dtom 665 * will work for a structure of size len). Returns the resulting 666 * mbuf chain on success, frees it and returns null on failure. 667 * If there is room, it will add up to max_protohdr-len extra bytes to the 668 * contiguous region in an attempt to avoid being called next time. 669 */ 670 int MPFail; 671 672 struct mbuf * 673 m_pullup(n, len) 674 struct mbuf *n; 675 int len; 676 { 677 struct mbuf *m; 678 int count; 679 int space; 680 681 /* 682 * If first mbuf has no cluster, and has room for len bytes 683 * without shifting current data, pullup into it, 684 * otherwise allocate a new mbuf to prepend to the chain. 685 */ 686 if ((n->m_flags & M_EXT) == 0 && 687 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 688 if (n->m_len >= len) 689 return (n); 690 m = n; 691 n = n->m_next; 692 len -= m->m_len; 693 } else { 694 if (len > MHLEN) 695 goto bad; 696 MGET(m, M_DONTWAIT, n->m_type); 697 if (m == 0) 698 goto bad; 699 m->m_len = 0; 700 if (n->m_flags & M_PKTHDR) { 701 M_COPY_PKTHDR(m, n); 702 n->m_flags &= ~M_PKTHDR; 703 } 704 } 705 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 706 do { 707 count = min(min(max(len, max_protohdr), space), n->m_len); 708 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 709 (unsigned)count); 710 len -= count; 711 m->m_len += count; 712 n->m_len -= count; 713 space -= count; 714 if (n->m_len) 715 n->m_data += count; 716 else 717 n = m_free(n); 718 } while (len > 0 && n); 719 if (len > 0) { 720 (void) m_free(m); 721 goto bad; 722 } 723 m->m_next = n; 724 return (m); 725 bad: 726 m_freem(n); 727 MPFail++; 728 return (0); 729 } 730 731 /* 732 * Partition an mbuf chain in two pieces, returning the tail -- 733 * all but the first len0 bytes. In case of failure, it returns NULL and 734 * attempts to restore the chain to its original state. 735 */ 736 struct mbuf * 737 m_split(m0, len0, wait) 738 struct mbuf *m0; 739 int len0, wait; 740 { 741 struct mbuf *m, *n; 742 unsigned len = len0, remain, len_save; 743 744 for (m = m0; m && len > m->m_len; m = m->m_next) 745 len -= m->m_len; 746 if (m == 0) 747 return (0); 748 remain = m->m_len - len; 749 if (m0->m_flags & M_PKTHDR) { 750 MGETHDR(n, wait, m0->m_type); 751 if (n == 0) 752 return (0); 753 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 754 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 755 len_save = m0->m_pkthdr.len; 756 m0->m_pkthdr.len = len0; 757 if (m->m_flags & M_EXT) 758 goto extpacket; 759 if (remain > MHLEN) { 760 /* m can't be the lead packet */ 761 MH_ALIGN(n, 0); 762 n->m_next = m_split(m, len, wait); 763 if (n->m_next == 0) { 764 (void) m_free(n); 765 m0->m_pkthdr.len = len_save; 766 return (0); 767 } else 768 return (n); 769 } else 770 MH_ALIGN(n, remain); 771 } else if (remain == 0) { 772 n = m->m_next; 773 m->m_next = 0; 774 return (n); 775 } else { 776 MGET(n, wait, m->m_type); 777 if (n == 0) 778 return (0); 779 M_ALIGN(n, remain); 780 } 781 extpacket: 782 if (m->m_flags & M_EXT) { 783 n->m_ext = m->m_ext; 784 MCLADDREFERENCE(m, n); 785 n->m_data = m->m_data + len; 786 } else { 787 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain); 788 } 789 n->m_len = remain; 790 m->m_len = len; 791 n->m_next = m->m_next; 792 m->m_next = 0; 793 return (n); 794 } 795 /* 796 * Routine to copy from device local memory into mbufs. 797 */ 798 struct mbuf * 799 m_devget(buf, totlen, off0, ifp, copy) 800 char *buf; 801 int totlen, off0; 802 struct ifnet *ifp; 803 void (*copy) __P((const void *from, void *to, size_t len)); 804 { 805 struct mbuf *m; 806 struct mbuf *top = 0, **mp = ⊤ 807 int off = off0, len; 808 char *cp; 809 char *epkt; 810 811 cp = buf; 812 epkt = cp + totlen; 813 if (off) { 814 /* 815 * If 'off' is non-zero, packet is trailer-encapsulated, 816 * so we have to skip the type and length fields. 817 */ 818 cp += off + 2 * sizeof(u_int16_t); 819 totlen -= 2 * sizeof(u_int16_t); 820 } 821 MGETHDR(m, M_DONTWAIT, MT_DATA); 822 if (m == 0) 823 return (0); 824 m->m_pkthdr.rcvif = ifp; 825 m->m_pkthdr.len = totlen; 826 m->m_len = MHLEN; 827 828 while (totlen > 0) { 829 if (top) { 830 MGET(m, M_DONTWAIT, MT_DATA); 831 if (m == 0) { 832 m_freem(top); 833 return (0); 834 } 835 m->m_len = MLEN; 836 } 837 len = min(totlen, epkt - cp); 838 if (len >= MINCLSIZE) { 839 MCLGET(m, M_DONTWAIT); 840 if ((m->m_flags & M_EXT) == 0) { 841 m_free(m); 842 m_freem(top); 843 return (0); 844 } 845 m->m_len = len = min(len, MCLBYTES); 846 } else { 847 /* 848 * Place initial small packet/header at end of mbuf. 849 */ 850 if (len < m->m_len) { 851 if (top == 0 && len + max_linkhdr <= m->m_len) 852 m->m_data += max_linkhdr; 853 m->m_len = len; 854 } else 855 len = m->m_len; 856 } 857 if (copy) 858 copy(cp, mtod(m, caddr_t), (size_t)len); 859 else 860 memcpy(mtod(m, caddr_t), cp, (size_t)len); 861 cp += len; 862 *mp = m; 863 mp = &m->m_next; 864 totlen -= len; 865 if (cp == epkt) 866 cp = buf; 867 } 868 return (top); 869 } 870 871 /* 872 * Copy data from a buffer back into the indicated mbuf chain, 873 * starting "off" bytes from the beginning, extending the mbuf 874 * chain if necessary. 875 */ 876 void 877 m_copyback(m0, off, len, cp) 878 struct mbuf *m0; 879 int off; 880 int len; 881 caddr_t cp; 882 { 883 int mlen; 884 struct mbuf *m = m0, *n; 885 int totlen = 0; 886 887 if (m0 == 0) 888 return; 889 while (off > (mlen = m->m_len)) { 890 off -= mlen; 891 totlen += mlen; 892 if (m->m_next == 0) { 893 n = m_getclr(M_DONTWAIT, m->m_type); 894 if (n == 0) 895 goto out; 896 n->m_len = min(MLEN, len + off); 897 m->m_next = n; 898 } 899 m = m->m_next; 900 } 901 while (len > 0) { 902 mlen = min (m->m_len - off, len); 903 memcpy(mtod(m, caddr_t) + off, cp, (unsigned)mlen); 904 cp += mlen; 905 len -= mlen; 906 mlen += off; 907 off = 0; 908 totlen += mlen; 909 if (len == 0) 910 break; 911 if (m->m_next == 0) { 912 n = m_get(M_DONTWAIT, m->m_type); 913 if (n == 0) 914 break; 915 n->m_len = min(MLEN, len); 916 m->m_next = n; 917 } 918 m = m->m_next; 919 } 920 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 921 m->m_pkthdr.len = totlen; 922 } 923