1 /* 2 * Copyright (c) 1982, 1986, 1988, 1991 Regents of the University of California. 3 * All rights reserved. 4 * 5 * %sccs.include.redist.c% 6 * 7 * @(#)uipc_mbuf.c 7.24 (Berkeley) 07/12/92 8 */ 9 10 #include "param.h" 11 #include "systm.h" 12 #include "proc.h" 13 #include "malloc.h" 14 #include "map.h" 15 #define MBTYPES 16 #include "mbuf.h" 17 #include "kernel.h" 18 #include "syslog.h" 19 #include "domain.h" 20 #include "protosw.h" 21 #include "vm/vm.h" 22 23 extern vm_map_t mb_map; 24 struct mbuf *mbutl; 25 char *mclrefcnt; 26 27 mbinit() 28 { 29 int s; 30 31 #if CLBYTES < 4096 32 #define NCL_INIT (4096/CLBYTES) 33 #else 34 #define NCL_INIT 1 35 #endif 36 s = splimp(); 37 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) 38 goto bad; 39 splx(s); 40 return; 41 bad: 42 panic("mbinit"); 43 } 44 45 /* 46 * Allocate some number of mbuf clusters 47 * and place on cluster free list. 48 * Must be called at splimp. 49 */ 50 /* ARGSUSED */ 51 m_clalloc(ncl, nowait) 52 register int ncl; 53 int nowait; 54 { 55 int npg, mbx; 56 register caddr_t p; 57 register int i; 58 static int logged; 59 60 npg = ncl * CLSIZE; 61 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), !nowait); 62 if (p == NULL) { 63 if (logged == 0) { 64 logged++; 65 log(LOG_ERR, "mb_map full\n"); 66 } 67 return (0); 68 } 69 ncl = ncl * CLBYTES / MCLBYTES; 70 for (i = 0; i < ncl; i++) { 71 ((union mcluster *)p)->mcl_next = mclfree; 72 mclfree = (union mcluster *)p; 73 p += MCLBYTES; 74 mbstat.m_clfree++; 75 } 76 mbstat.m_clusters += ncl; 77 return (1); 78 } 79 80 /* 81 * When MGET failes, ask protocols to free space when short of memory, 82 * then re-attempt to allocate an mbuf. 83 */ 84 struct mbuf * 85 m_retry(i, t) 86 int i, t; 87 { 88 register struct mbuf *m; 89 90 m_reclaim(); 91 #define m_retry(i, t) (struct mbuf *)0 92 MGET(m, i, t); 93 #undef m_retry 94 return (m); 95 } 96 97 /* 98 * As above; retry an MGETHDR. 99 */ 100 struct mbuf * 101 m_retryhdr(i, t) 102 int i, t; 103 { 104 register struct mbuf *m; 105 106 m_reclaim(); 107 #define m_retryhdr(i, t) (struct mbuf *)0 108 MGETHDR(m, i, t); 109 #undef m_retryhdr 110 return (m); 111 } 112 113 m_reclaim() 114 { 115 register struct domain *dp; 116 register struct protosw *pr; 117 int s = splimp(); 118 119 for (dp = domains; dp; dp = dp->dom_next) 120 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 121 if (pr->pr_drain) 122 (*pr->pr_drain)(); 123 splx(s); 124 mbstat.m_drain++; 125 } 126 127 /* 128 * Space allocation routines. 129 * These are also available as macros 130 * for critical paths. 131 */ 132 struct mbuf * 133 m_get(nowait, type) 134 int nowait, type; 135 { 136 register struct mbuf *m; 137 138 MGET(m, nowait, type); 139 return (m); 140 } 141 142 struct mbuf * 143 m_gethdr(nowait, type) 144 int nowait, type; 145 { 146 register struct mbuf *m; 147 148 MGETHDR(m, nowait, type); 149 return (m); 150 } 151 152 struct mbuf * 153 m_getclr(nowait, type) 154 int nowait, type; 155 { 156 register struct mbuf *m; 157 158 MGET(m, nowait, type); 159 if (m == 0) 160 return (0); 161 bzero(mtod(m, caddr_t), MLEN); 162 return (m); 163 } 164 165 struct mbuf * 166 m_free(m) 167 struct mbuf *m; 168 { 169 register struct mbuf *n; 170 171 MFREE(m, n); 172 return (n); 173 } 174 175 void 176 m_freem(m) 177 register struct mbuf *m; 178 { 179 register struct mbuf *n; 180 181 if (m == NULL) 182 return; 183 do { 184 MFREE(m, n); 185 } while (m = n); 186 } 187 188 /* 189 * Mbuffer utility routines. 190 */ 191 192 /* 193 * Lesser-used path for M_PREPEND: 194 * allocate new mbuf to prepend to chain, 195 * copy junk along. 196 */ 197 struct mbuf * 198 m_prepend(m, len, how) 199 register struct mbuf *m; 200 int len, how; 201 { 202 struct mbuf *mn; 203 204 MGET(mn, how, m->m_type); 205 if (mn == (struct mbuf *)NULL) { 206 m_freem(m); 207 return ((struct mbuf *)NULL); 208 } 209 if (m->m_flags & M_PKTHDR) { 210 M_COPY_PKTHDR(mn, m); 211 m->m_flags &= ~M_PKTHDR; 212 } 213 mn->m_next = m; 214 m = mn; 215 if (len < MHLEN) 216 MH_ALIGN(m, len); 217 m->m_len = len; 218 return (m); 219 } 220 221 /* 222 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 223 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 224 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 225 */ 226 int MCFail; 227 228 struct mbuf * 229 m_copym(m, off0, len, wait) 230 register struct mbuf *m; 231 int off0, wait; 232 register int len; 233 { 234 register struct mbuf *n, **np; 235 register int off = off0; 236 struct mbuf *top; 237 int copyhdr = 0; 238 239 if (off < 0 || len < 0) 240 panic("m_copym"); 241 if (off == 0 && m->m_flags & M_PKTHDR) 242 copyhdr = 1; 243 while (off > 0) { 244 if (m == 0) 245 panic("m_copym"); 246 if (off < m->m_len) 247 break; 248 off -= m->m_len; 249 m = m->m_next; 250 } 251 np = ⊤ 252 top = 0; 253 while (len > 0) { 254 if (m == 0) { 255 if (len != M_COPYALL) 256 panic("m_copym"); 257 break; 258 } 259 MGET(n, wait, m->m_type); 260 *np = n; 261 if (n == 0) 262 goto nospace; 263 if (copyhdr) { 264 M_COPY_PKTHDR(n, m); 265 if (len == M_COPYALL) 266 n->m_pkthdr.len -= off0; 267 else 268 n->m_pkthdr.len = len; 269 copyhdr = 0; 270 } 271 n->m_len = min(len, m->m_len - off); 272 if (m->m_flags & M_EXT) { 273 n->m_data = m->m_data + off; 274 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 275 n->m_ext = m->m_ext; 276 n->m_flags |= M_EXT; 277 } else 278 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 279 (unsigned)n->m_len); 280 if (len != M_COPYALL) 281 len -= n->m_len; 282 off = 0; 283 m = m->m_next; 284 np = &n->m_next; 285 } 286 if (top == 0) 287 MCFail++; 288 return (top); 289 nospace: 290 m_freem(top); 291 MCFail++; 292 return (0); 293 } 294 295 /* 296 * Copy data from an mbuf chain starting "off" bytes from the beginning, 297 * continuing for "len" bytes, into the indicated buffer. 298 */ 299 m_copydata(m, off, len, cp) 300 register struct mbuf *m; 301 register int off; 302 register int len; 303 caddr_t cp; 304 { 305 register unsigned count; 306 307 if (off < 0 || len < 0) 308 panic("m_copydata"); 309 while (off > 0) { 310 if (m == 0) 311 panic("m_copydata"); 312 if (off < m->m_len) 313 break; 314 off -= m->m_len; 315 m = m->m_next; 316 } 317 while (len > 0) { 318 if (m == 0) 319 panic("m_copydata"); 320 count = min(m->m_len - off, len); 321 bcopy(mtod(m, caddr_t) + off, cp, count); 322 len -= count; 323 cp += count; 324 off = 0; 325 m = m->m_next; 326 } 327 } 328 329 /* 330 * Concatenate mbuf chain n to m. 331 * Both chains must be of the same type (e.g. MT_DATA). 332 * Any m_pkthdr is not updated. 333 */ 334 m_cat(m, n) 335 register struct mbuf *m, *n; 336 { 337 while (m->m_next) 338 m = m->m_next; 339 while (n) { 340 if (m->m_flags & M_EXT || 341 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 342 /* just join the two chains */ 343 m->m_next = n; 344 return; 345 } 346 /* splat the data from one into the other */ 347 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 348 (u_int)n->m_len); 349 m->m_len += n->m_len; 350 n = m_free(n); 351 } 352 } 353 354 m_adj(mp, req_len) 355 struct mbuf *mp; 356 int req_len; 357 { 358 register int len = req_len; 359 register struct mbuf *m; 360 register count; 361 362 if ((m = mp) == NULL) 363 return; 364 if (len >= 0) { 365 /* 366 * Trim from head. 367 */ 368 while (m != NULL && len > 0) { 369 if (m->m_len <= len) { 370 len -= m->m_len; 371 m->m_len = 0; 372 m = m->m_next; 373 } else { 374 m->m_len -= len; 375 m->m_data += len; 376 len = 0; 377 } 378 } 379 m = mp; 380 if (mp->m_flags & M_PKTHDR) 381 m->m_pkthdr.len -= (req_len - len); 382 } else { 383 /* 384 * Trim from tail. Scan the mbuf chain, 385 * calculating its length and finding the last mbuf. 386 * If the adjustment only affects this mbuf, then just 387 * adjust and return. Otherwise, rescan and truncate 388 * after the remaining size. 389 */ 390 len = -len; 391 count = 0; 392 for (;;) { 393 count += m->m_len; 394 if (m->m_next == (struct mbuf *)0) 395 break; 396 m = m->m_next; 397 } 398 if (m->m_len >= len) { 399 m->m_len -= len; 400 if ((mp = m)->m_flags & M_PKTHDR) 401 m->m_pkthdr.len -= len; 402 return; 403 } 404 count -= len; 405 if (count < 0) 406 count = 0; 407 /* 408 * Correct length for chain is "count". 409 * Find the mbuf with last data, adjust its length, 410 * and toss data from remaining mbufs on chain. 411 */ 412 m = mp; 413 if (m->m_flags & M_PKTHDR) 414 m->m_pkthdr.len = count; 415 for (; m; m = m->m_next) { 416 if (m->m_len >= count) { 417 m->m_len = count; 418 break; 419 } 420 count -= m->m_len; 421 } 422 while (m = m->m_next) 423 m->m_len = 0; 424 } 425 } 426 427 /* 428 * Rearange an mbuf chain so that len bytes are contiguous 429 * and in the data area of an mbuf (so that mtod and dtom 430 * will work for a structure of size len). Returns the resulting 431 * mbuf chain on success, frees it and returns null on failure. 432 * If there is room, it will add up to max_protohdr-len extra bytes to the 433 * contiguous region in an attempt to avoid being called next time. 434 */ 435 int MPFail; 436 437 struct mbuf * 438 m_pullup(n, len) 439 register struct mbuf *n; 440 int len; 441 { 442 register struct mbuf *m; 443 register int count; 444 int space; 445 446 /* 447 * If first mbuf has no cluster, and has room for len bytes 448 * without shifting current data, pullup into it, 449 * otherwise allocate a new mbuf to prepend to the chain. 450 */ 451 if ((n->m_flags & M_EXT) == 0 && 452 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 453 if (n->m_len >= len) 454 return (n); 455 m = n; 456 n = n->m_next; 457 len -= m->m_len; 458 } else { 459 if (len > MHLEN) 460 goto bad; 461 MGET(m, M_DONTWAIT, n->m_type); 462 if (m == 0) 463 goto bad; 464 m->m_len = 0; 465 if (n->m_flags & M_PKTHDR) { 466 M_COPY_PKTHDR(m, n); 467 n->m_flags &= ~M_PKTHDR; 468 } 469 } 470 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 471 do { 472 count = min(min(max(len, max_protohdr), space), n->m_len); 473 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 474 (unsigned)count); 475 len -= count; 476 m->m_len += count; 477 n->m_len -= count; 478 space -= count; 479 if (n->m_len) 480 n->m_data += count; 481 else 482 n = m_free(n); 483 } while (len > 0 && n); 484 if (len > 0) { 485 (void) m_free(m); 486 goto bad; 487 } 488 m->m_next = n; 489 return (m); 490 bad: 491 m_freem(n); 492 MPFail++; 493 return (0); 494 } 495 496 /* 497 * Partition an mbuf chain in two pieces, returning the tail -- 498 * all but the first len0 bytes. In case of failure, it returns NULL and 499 * attempts to restore the chain to its original state. 500 */ 501 struct mbuf * 502 m_split(m0, len0, wait) 503 register struct mbuf *m0; 504 int len0, wait; 505 { 506 register struct mbuf *m, *n; 507 unsigned len = len0, remain; 508 509 for (m = m0; m && len > m->m_len; m = m->m_next) 510 len -= m->m_len; 511 if (m == 0) 512 return (0); 513 remain = m->m_len - len; 514 if (m0->m_flags & M_PKTHDR) { 515 MGETHDR(n, wait, m0->m_type); 516 if (n == 0) 517 return (0); 518 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 519 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 520 m0->m_pkthdr.len = len0; 521 if (m->m_flags & M_EXT) 522 goto extpacket; 523 if (remain > MHLEN) { 524 /* m can't be the lead packet */ 525 MH_ALIGN(n, 0); 526 n->m_next = m_split(m, len, wait); 527 if (n->m_next == 0) { 528 (void) m_free(n); 529 return (0); 530 } else 531 return (n); 532 } else 533 MH_ALIGN(n, remain); 534 } else if (remain == 0) { 535 n = m->m_next; 536 m->m_next = 0; 537 return (n); 538 } else { 539 MGET(n, wait, m->m_type); 540 if (n == 0) 541 return (0); 542 M_ALIGN(n, remain); 543 } 544 extpacket: 545 if (m->m_flags & M_EXT) { 546 n->m_flags |= M_EXT; 547 n->m_ext = m->m_ext; 548 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 549 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */ 550 n->m_data = m->m_data + len; 551 } else { 552 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 553 } 554 n->m_len = remain; 555 m->m_len = len; 556 n->m_next = m->m_next; 557 m->m_next = 0; 558 return (n); 559 } 560 /* 561 * Routine to copy from device local memory into mbufs. 562 */ 563 struct mbuf * 564 m_devget(buf, totlen, off0, ifp, copy) 565 char *buf; 566 int totlen, off0; 567 struct ifnet *ifp; 568 void (*copy)(); 569 { 570 register struct mbuf *m; 571 struct mbuf *top = 0, **mp = ⊤ 572 register int off = off0, len; 573 register char *cp; 574 char *epkt; 575 576 cp = buf; 577 epkt = cp + totlen; 578 if (off) { 579 cp += off + 2 * sizeof(u_short); 580 totlen -= 2 * sizeof(u_short); 581 } 582 MGETHDR(m, M_DONTWAIT, MT_DATA); 583 if (m == 0) 584 return (0); 585 m->m_pkthdr.rcvif = ifp; 586 m->m_pkthdr.len = totlen; 587 m->m_len = MHLEN; 588 589 while (totlen > 0) { 590 if (top) { 591 MGET(m, M_DONTWAIT, MT_DATA); 592 if (m == 0) { 593 m_freem(top); 594 return (0); 595 } 596 m->m_len = MLEN; 597 } 598 len = min(totlen, epkt - cp); 599 if (len >= MINCLSIZE) { 600 MCLGET(m, M_DONTWAIT); 601 if (m->m_flags & M_EXT) 602 m->m_len = len = min(len, MCLBYTES); 603 else 604 len = m->m_len; 605 } else { 606 /* 607 * Place initial small packet/header at end of mbuf. 608 */ 609 if (len < m->m_len) { 610 if (top == 0 && len + max_linkhdr <= m->m_len) 611 m->m_data += max_linkhdr; 612 m->m_len = len; 613 } else 614 len = m->m_len; 615 } 616 if (copy) 617 copy(cp, mtod(m, caddr_t), (unsigned)len); 618 else 619 bcopy(cp, mtod(m, caddr_t), (unsigned)len); 620 cp += len; 621 *mp = m; 622 mp = &m->m_next; 623 totlen -= len; 624 if (cp == epkt) 625 cp = buf; 626 } 627 return (top); 628 } 629