1 /* $OpenBSD: uipc_mbuf.c,v 1.7 1998/02/03 19:06:27 deraadt Exp $ */ 2 /* $NetBSD: uipc_mbuf.c,v 1.15.4.1 1996/06/13 17:11:44 cgd Exp $ */ 3 4 /* 5 * Copyright (c) 1982, 1986, 1988, 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 37 */ 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/proc.h> 42 #include <sys/malloc.h> 43 #include <sys/map.h> 44 #define MBTYPES 45 #include <sys/mbuf.h> 46 #include <sys/kernel.h> 47 #include <sys/syslog.h> 48 #include <sys/domain.h> 49 #include <sys/protosw.h> 50 51 #include <machine/cpu.h> 52 53 #include <vm/vm.h> 54 55 extern vm_map_t mb_map; 56 struct mbuf *mbutl; 57 char *mclrefcnt; 58 int needqueuedrain; 59 60 void 61 mbinit() 62 { 63 int s; 64 65 s = splimp(); 66 if (m_clalloc(max(4096/CLBYTES, 1), M_DONTWAIT) == 0) 67 goto bad; 68 splx(s); 69 return; 70 bad: 71 panic("mbinit"); 72 } 73 74 /* 75 * Allocate some number of mbuf clusters 76 * and place on cluster free list. 77 * Must be called at splimp. 78 */ 79 /* ARGSUSED */ 80 int 81 m_clalloc(ncl, nowait) 82 register int ncl; 83 int nowait; 84 { 85 volatile static struct timeval lastlogged; 86 struct timeval curtime, logdiff; 87 register caddr_t p; 88 register int i; 89 int npg, s; 90 91 npg = ncl * CLSIZE; 92 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), !nowait); 93 if (p == NULL) { 94 s = splclock(); 95 curtime = time; 96 splx(s); 97 timersub(&curtime, &lastlogged, &logdiff); 98 if (logdiff.tv_sec >= 60) { 99 lastlogged = curtime; 100 log(LOG_ERR, "mb_map full\n"); 101 } 102 m_reclaim(); 103 return (mclfree != NULL); 104 } 105 ncl = ncl * CLBYTES / MCLBYTES; 106 for (i = 0; i < ncl; i++) { 107 ((union mcluster *)p)->mcl_next = mclfree; 108 mclfree = (union mcluster *)p; 109 p += MCLBYTES; 110 mbstat.m_clfree++; 111 } 112 mbstat.m_clusters += ncl; 113 return (1); 114 } 115 116 /* 117 * When MGET failes, ask protocols to free space when short of memory, 118 * then re-attempt to allocate an mbuf. 119 */ 120 struct mbuf * 121 m_retry(i, t) 122 int i, t; 123 { 124 register struct mbuf *m; 125 126 if (i & M_DONTWAIT) { 127 needqueuedrain = 1; 128 setsoftnet (); 129 return (NULL); 130 } 131 m_reclaim(); 132 #define m_retry(i, t) (struct mbuf *)0 133 MGET(m, i, t); 134 #undef m_retry 135 return (m); 136 } 137 138 /* 139 * As above; retry an MGETHDR. 140 */ 141 struct mbuf * 142 m_retryhdr(i, t) 143 int i, t; 144 { 145 register struct mbuf *m; 146 147 if (i & M_DONTWAIT) { 148 needqueuedrain = 1; 149 setsoftnet (); 150 return (NULL); 151 } 152 m_reclaim(); 153 #define m_retryhdr(i, t) (struct mbuf *)0 154 MGETHDR(m, i, t); 155 #undef m_retryhdr 156 return (m); 157 } 158 159 void 160 m_reclaim() 161 { 162 register struct domain *dp; 163 register struct protosw *pr; 164 int s = splimp(); 165 166 needqueuedrain = 0; 167 for (dp = domains; dp; dp = dp->dom_next) 168 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 169 if (pr->pr_drain) 170 (*pr->pr_drain)(); 171 splx(s); 172 mbstat.m_drain++; 173 } 174 175 /* 176 * Space allocation routines. 177 * These are also available as macros 178 * for critical paths. 179 */ 180 struct mbuf * 181 m_get(nowait, type) 182 int nowait, type; 183 { 184 register struct mbuf *m; 185 186 MGET(m, nowait, type); 187 return (m); 188 } 189 190 struct mbuf * 191 m_gethdr(nowait, type) 192 int nowait, type; 193 { 194 register struct mbuf *m; 195 196 MGETHDR(m, nowait, type); 197 return (m); 198 } 199 200 struct mbuf * 201 m_getclr(nowait, type) 202 int nowait, type; 203 { 204 register struct mbuf *m; 205 206 MGET(m, nowait, type); 207 if (m == 0) 208 return (0); 209 bzero(mtod(m, caddr_t), MLEN); 210 return (m); 211 } 212 213 struct mbuf * 214 m_free(m) 215 struct mbuf *m; 216 { 217 register struct mbuf *n; 218 219 MFREE(m, n); 220 return (n); 221 } 222 223 void 224 m_freem(m) 225 register struct mbuf *m; 226 { 227 register struct mbuf *n; 228 229 if (m == NULL) 230 return; 231 do { 232 MFREE(m, n); 233 } while ((m = n) != NULL); 234 } 235 236 /* 237 * Mbuffer utility routines. 238 */ 239 240 /* 241 * Lesser-used path for M_PREPEND: 242 * allocate new mbuf to prepend to chain, 243 * copy junk along. 244 */ 245 struct mbuf * 246 m_prepend(m, len, how) 247 register struct mbuf *m; 248 int len, how; 249 { 250 struct mbuf *mn; 251 252 MGET(mn, how, m->m_type); 253 if (mn == (struct mbuf *)NULL) { 254 m_freem(m); 255 return ((struct mbuf *)NULL); 256 } 257 if (m->m_flags & M_PKTHDR) { 258 M_COPY_PKTHDR(mn, m); 259 m->m_flags &= ~M_PKTHDR; 260 } 261 mn->m_next = m; 262 m = mn; 263 if (len < MHLEN) 264 MH_ALIGN(m, len); 265 m->m_len = len; 266 return (m); 267 } 268 269 /* 270 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 271 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 272 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 273 */ 274 int MCFail; 275 276 struct mbuf * 277 m_copym(m, off0, len, wait) 278 register struct mbuf *m; 279 int off0, wait; 280 register int len; 281 { 282 register struct mbuf *n, **np; 283 register int off = off0; 284 struct mbuf *top; 285 int copyhdr = 0; 286 287 if (off < 0 || len < 0) 288 panic("m_copym"); 289 if (off == 0 && m->m_flags & M_PKTHDR) 290 copyhdr = 1; 291 while (off > 0) { 292 if (m == 0) 293 panic("m_copym"); 294 if (off < m->m_len) 295 break; 296 off -= m->m_len; 297 m = m->m_next; 298 } 299 np = ⊤ 300 top = 0; 301 while (len > 0) { 302 if (m == 0) { 303 if (len != M_COPYALL) 304 panic("m_copym"); 305 break; 306 } 307 MGET(n, wait, m->m_type); 308 *np = n; 309 if (n == 0) 310 goto nospace; 311 if (copyhdr) { 312 M_COPY_PKTHDR(n, m); 313 if (len == M_COPYALL) 314 n->m_pkthdr.len -= off0; 315 else 316 n->m_pkthdr.len = len; 317 copyhdr = 0; 318 } 319 n->m_len = min(len, m->m_len - off); 320 if (m->m_flags & M_EXT) { 321 n->m_data = m->m_data + off; 322 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 323 n->m_ext = m->m_ext; 324 n->m_flags |= M_EXT; 325 } else 326 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 327 (unsigned)n->m_len); 328 if (len != M_COPYALL) 329 len -= n->m_len; 330 off = 0; 331 m = m->m_next; 332 np = &n->m_next; 333 } 334 if (top == 0) 335 MCFail++; 336 return (top); 337 nospace: 338 m_freem(top); 339 MCFail++; 340 return (0); 341 } 342 343 /* 344 * Copy data from an mbuf chain starting "off" bytes from the beginning, 345 * continuing for "len" bytes, into the indicated buffer. 346 */ 347 void 348 m_copydata(m, off, len, cp) 349 register struct mbuf *m; 350 register int off; 351 register int len; 352 caddr_t cp; 353 { 354 register unsigned count; 355 356 if (off < 0 || len < 0) 357 panic("m_copydata"); 358 while (off > 0) { 359 if (m == 0) 360 panic("m_copydata"); 361 if (off < m->m_len) 362 break; 363 off -= m->m_len; 364 m = m->m_next; 365 } 366 while (len > 0) { 367 if (m == 0) 368 panic("m_copydata"); 369 count = min(m->m_len - off, len); 370 bcopy(mtod(m, caddr_t) + off, cp, count); 371 len -= count; 372 cp += count; 373 off = 0; 374 m = m->m_next; 375 } 376 } 377 378 /* 379 * Concatenate mbuf chain n to m. 380 * Both chains must be of the same type (e.g. MT_DATA). 381 * Any m_pkthdr is not updated. 382 */ 383 void 384 m_cat(m, n) 385 register struct mbuf *m, *n; 386 { 387 while (m->m_next) 388 m = m->m_next; 389 while (n) { 390 if (m->m_flags & M_EXT || 391 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 392 /* just join the two chains */ 393 m->m_next = n; 394 return; 395 } 396 /* splat the data from one into the other */ 397 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 398 (u_int)n->m_len); 399 m->m_len += n->m_len; 400 n = m_free(n); 401 } 402 } 403 404 void 405 m_adj(mp, req_len) 406 struct mbuf *mp; 407 int req_len; 408 { 409 register int len = req_len; 410 register struct mbuf *m; 411 register int count; 412 413 if ((m = mp) == NULL) 414 return; 415 if (len >= 0) { 416 /* 417 * Trim from head. 418 */ 419 while (m != NULL && len > 0) { 420 if (m->m_len <= len) { 421 len -= m->m_len; 422 m->m_len = 0; 423 m = m->m_next; 424 } else { 425 m->m_len -= len; 426 m->m_data += len; 427 len = 0; 428 } 429 } 430 m = mp; 431 if (mp->m_flags & M_PKTHDR) 432 m->m_pkthdr.len -= (req_len - len); 433 } else { 434 /* 435 * Trim from tail. Scan the mbuf chain, 436 * calculating its length and finding the last mbuf. 437 * If the adjustment only affects this mbuf, then just 438 * adjust and return. Otherwise, rescan and truncate 439 * after the remaining size. 440 */ 441 len = -len; 442 count = 0; 443 for (;;) { 444 count += m->m_len; 445 if (m->m_next == (struct mbuf *)0) 446 break; 447 m = m->m_next; 448 } 449 if (m->m_len >= len) { 450 m->m_len -= len; 451 if (mp->m_flags & M_PKTHDR) 452 mp->m_pkthdr.len -= len; 453 return; 454 } 455 count -= len; 456 if (count < 0) 457 count = 0; 458 /* 459 * Correct length for chain is "count". 460 * Find the mbuf with last data, adjust its length, 461 * and toss data from remaining mbufs on chain. 462 */ 463 m = mp; 464 if (m->m_flags & M_PKTHDR) 465 m->m_pkthdr.len = count; 466 for (; m; m = m->m_next) { 467 if (m->m_len >= count) { 468 m->m_len = count; 469 break; 470 } 471 count -= m->m_len; 472 } 473 while ((m = m->m_next) != NULL) 474 m->m_len = 0; 475 } 476 } 477 478 /* 479 * Rearange an mbuf chain so that len bytes are contiguous 480 * and in the data area of an mbuf (so that mtod and dtom 481 * will work for a structure of size len). Returns the resulting 482 * mbuf chain on success, frees it and returns null on failure. 483 * If there is room, it will add up to max_protohdr-len extra bytes to the 484 * contiguous region in an attempt to avoid being called next time. 485 */ 486 int MPFail; 487 488 struct mbuf * 489 m_pullup(n, len) 490 register struct mbuf *n; 491 int len; 492 { 493 register struct mbuf *m; 494 register int count; 495 int space; 496 497 /* 498 * If first mbuf has no cluster, and has room for len bytes 499 * without shifting current data, pullup into it, 500 * otherwise allocate a new mbuf to prepend to the chain. 501 */ 502 if ((n->m_flags & M_EXT) == 0 && 503 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 504 if (n->m_len >= len) 505 return (n); 506 m = n; 507 n = n->m_next; 508 len -= m->m_len; 509 } else { 510 if (len > MHLEN) 511 goto bad; 512 MGET(m, M_DONTWAIT, n->m_type); 513 if (m == 0) 514 goto bad; 515 m->m_len = 0; 516 if (n->m_flags & M_PKTHDR) { 517 M_COPY_PKTHDR(m, n); 518 n->m_flags &= ~M_PKTHDR; 519 } 520 } 521 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 522 do { 523 count = min(min(max(len, max_protohdr), space), n->m_len); 524 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 525 (unsigned)count); 526 len -= count; 527 m->m_len += count; 528 n->m_len -= count; 529 space -= count; 530 if (n->m_len) 531 n->m_data += count; 532 else 533 n = m_free(n); 534 } while (len > 0 && n); 535 if (len > 0) { 536 (void) m_free(m); 537 goto bad; 538 } 539 m->m_next = n; 540 return (m); 541 bad: 542 m_freem(n); 543 MPFail++; 544 return (0); 545 } 546 547 /* 548 * Partition an mbuf chain in two pieces, returning the tail -- 549 * all but the first len0 bytes. In case of failure, it returns NULL and 550 * attempts to restore the chain to its original state. 551 */ 552 struct mbuf * 553 m_split(m0, len0, wait) 554 register struct mbuf *m0; 555 int len0, wait; 556 { 557 register struct mbuf *m, *n; 558 unsigned len = len0, remain, olen; 559 560 for (m = m0; m && len > m->m_len; m = m->m_next) 561 len -= m->m_len; 562 if (m == 0) 563 return (0); 564 remain = m->m_len - len; 565 if (m0->m_flags & M_PKTHDR) { 566 MGETHDR(n, wait, m0->m_type); 567 if (n == 0) 568 return (0); 569 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 570 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 571 m0->m_pkthdr.len = len0; 572 if (m->m_flags & M_EXT) 573 goto extpacket; 574 if (remain > MHLEN) { 575 /* m can't be the lead packet */ 576 MH_ALIGN(n, 0); 577 n->m_next = m_split(m, len, wait); 578 if (n->m_next == 0) { 579 (void) m_free(n); 580 m0->m_pkthdr.len = olen; 581 return (0); 582 } else 583 return (n); 584 } else 585 MH_ALIGN(n, remain); 586 } else if (remain == 0) { 587 n = m->m_next; 588 m->m_next = 0; 589 return (n); 590 } else { 591 MGET(n, wait, m->m_type); 592 if (n == 0) 593 return (0); 594 M_ALIGN(n, remain); 595 } 596 extpacket: 597 if (m->m_flags & M_EXT) { 598 n->m_flags |= M_EXT; 599 n->m_ext = m->m_ext; 600 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 601 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */ 602 n->m_data = m->m_data + len; 603 } else { 604 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 605 } 606 n->m_len = remain; 607 m->m_len = len; 608 n->m_next = m->m_next; 609 m->m_next = 0; 610 return (n); 611 } 612 /* 613 * Routine to copy from device local memory into mbufs. 614 */ 615 struct mbuf * 616 m_devget(buf, totlen, off0, ifp, copy) 617 char *buf; 618 int totlen, off0; 619 struct ifnet *ifp; 620 void (*copy) __P((const void *, void *, size_t)); 621 { 622 register struct mbuf *m; 623 struct mbuf *top = 0, **mp = ⊤ 624 register int off = off0, len; 625 register char *cp; 626 char *epkt; 627 628 cp = buf; 629 epkt = cp + totlen; 630 if (off) { 631 /* 632 * If 'off' is non-zero, packet is trailer-encapsulated, 633 * so we have to skip the type and length fields. 634 */ 635 cp += off + 2 * sizeof(u_int16_t); 636 totlen -= 2 * sizeof(u_int16_t); 637 } 638 MGETHDR(m, M_DONTWAIT, MT_DATA); 639 if (m == 0) 640 return (0); 641 m->m_pkthdr.rcvif = ifp; 642 m->m_pkthdr.len = totlen; 643 m->m_len = MHLEN; 644 645 while (totlen > 0) { 646 if (top) { 647 MGET(m, M_DONTWAIT, MT_DATA); 648 if (m == 0) { 649 m_freem(top); 650 return (0); 651 } 652 m->m_len = MLEN; 653 } 654 len = min(totlen, epkt - cp); 655 if (len >= MINCLSIZE) { 656 MCLGET(m, M_DONTWAIT); 657 if (m->m_flags & M_EXT) 658 m->m_len = len = min(len, MCLBYTES); 659 else 660 len = m->m_len; 661 } else { 662 /* 663 * Place initial small packet/header at end of mbuf. 664 */ 665 if (len < m->m_len) { 666 if (top == 0 && len + max_linkhdr <= m->m_len) 667 m->m_data += max_linkhdr; 668 m->m_len = len; 669 } else 670 len = m->m_len; 671 } 672 if (copy) 673 copy(cp, mtod(m, caddr_t), (size_t)len); 674 else 675 bcopy(cp, mtod(m, caddr_t), (size_t)len); 676 cp += len; 677 *mp = m; 678 mp = &m->m_next; 679 totlen -= len; 680 if (cp == epkt) 681 cp = buf; 682 } 683 return (top); 684 } 685