1 /* 2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 1982, 1986, 1988, 1990, 1993 4 * The Regents of the University of California. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93 31 * $FreeBSD: src/sys/kern/uipc_socket2.c,v 1.55.2.17 2002/08/31 19:04:55 dwmalone Exp $ 32 */ 33 34 #include "opt_param.h" 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/domain.h> 38 #include <sys/file.h> /* for maxfiles */ 39 #include <sys/kernel.h> 40 #include <sys/proc.h> 41 #include <sys/malloc.h> 42 #include <sys/mbuf.h> 43 #include <sys/protosw.h> 44 #include <sys/resourcevar.h> 45 #include <sys/stat.h> 46 #include <sys/socket.h> 47 #include <sys/socketvar.h> 48 49 #include <sys/thread2.h> 50 #include <sys/msgport2.h> 51 52 /* 53 * Routines to add and remove data from an mbuf queue. 54 * 55 * The routines sbappend() or sbappendrecord() are normally called to 56 * append new mbufs to a socket buffer. sbappendrecord() differs from 57 * sbappend() in that data supplied is treated as the beginning of a new 58 * record. sbappend() only begins a new record if the last mbuf in the 59 * sockbuf is marked M_EOR. 60 * 61 * To place a sender's address, optional access rights, and data in a 62 * socket receive buffer, sbappendaddr() or sbappendcontrol() should be 63 * used. These functions also begin a new record. 64 * 65 * Reliable protocols may use the socket send buffer to hold data 66 * awaiting acknowledgement. Data is normally copied from a socket 67 * send buffer in a protocol with m_copym for output to a peer, 68 * and then removing the data from the socket buffer with sbdrop() 69 * or sbdroprecord() when the data is acknowledged by the peer. 70 */ 71 72 /* 73 * Append mbuf chain m to the last record in the socket buffer sb. 74 * The additional space associated the mbuf chain is recorded in sb. 75 * Empty mbufs are discarded and mbufs are compacted where possible. 76 * 77 * If M_EOR is set in the first or last mbuf of the last record, the 78 * mbuf chain is appended as a new record. M_EOR is usually just set 79 * in the last mbuf of the last record's mbuf chain (see sbcompress()), 80 * but this may be changed in the future since there is no real need 81 * to propogate the flag any more. 82 */ 83 void 84 sbappend(struct sockbuf *sb, struct mbuf *m) 85 { 86 struct mbuf *n; 87 88 mbuftrackid(m, 16); 89 90 if (m) { 91 n = sb->sb_lastrecord; 92 if (n) { 93 if (n->m_flags & M_EOR) { 94 sbappendrecord(sb, m); 95 return; 96 } 97 } 98 n = sb->sb_lastmbuf; 99 if (n) { 100 if (n->m_flags & M_EOR) { 101 sbappendrecord(sb, m); 102 return; 103 } 104 } 105 sbcompress(sb, m, n); 106 } 107 } 108 109 /* 110 * sbappendstream() is an optimized form of sbappend() for protocols 111 * such as TCP that only have one record in the socket buffer, are 112 * not PR_ATOMIC, nor allow MT_CONTROL data. A protocol that uses 113 * sbappendstream() must use sbappendstream() exclusively. 114 */ 115 void 116 sbappendstream(struct sockbuf *sb, struct mbuf *m) 117 { 118 mbuftrackid(m, 17); 119 KKASSERT(m->m_nextpkt == NULL); 120 sbcompress(sb, m, sb->sb_lastmbuf); 121 } 122 123 #ifdef SOCKBUF_DEBUG 124 125 void 126 _sbcheck(struct sockbuf *sb) 127 { 128 struct mbuf *m; 129 struct mbuf *n = NULL; 130 u_long len = 0, mbcnt = 0; 131 132 for (m = sb->sb_mb; m; m = n) { 133 n = m->m_nextpkt; 134 if (n == NULL && sb->sb_lastrecord != m) { 135 kprintf("sockbuf %p mismatched lastrecord %p vs %p\n", sb, sb->sb_lastrecord, m); 136 panic("sbcheck1"); 137 } 138 for (; m; m = m->m_next) { 139 len += m->m_len; 140 mbcnt += MSIZE; 141 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ 142 mbcnt += m->m_ext.ext_size; 143 if (n == NULL && m->m_next == NULL) { 144 if (sb->sb_lastmbuf != m) { 145 kprintf("sockbuf %p mismatched lastmbuf %p vs %p\n", sb, sb->sb_lastmbuf, m); 146 panic("sbcheck2"); 147 } 148 } 149 } 150 } 151 if (sb->sb_mb == NULL) { 152 if (sb->sb_lastrecord != NULL) { 153 kprintf("sockbuf %p is empty, lastrecord not NULL: %p\n", 154 sb, sb->sb_lastrecord); 155 panic("sbcheck3"); 156 } 157 if (sb->sb_lastmbuf != NULL) { 158 kprintf("sockbuf %p is empty, lastmbuf not NULL: %p\n", 159 sb, sb->sb_lastmbuf); 160 panic("sbcheck4"); 161 } 162 } 163 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 164 kprintf("sockbuf %p cc %ld != %ld || mbcnt %ld != %ld\n", 165 sb, len, sb->sb_cc, mbcnt, sb->sb_mbcnt); 166 panic("sbcheck5"); 167 } 168 } 169 170 #endif 171 172 /* 173 * Same as sbappend(), except the mbuf chain begins a new record. 174 */ 175 void 176 sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 177 { 178 struct mbuf *firstmbuf; 179 struct mbuf *secondmbuf; 180 181 if (m0 == NULL) 182 return; 183 mbuftrackid(m0, 18); 184 185 sbcheck(sb); 186 187 /* 188 * Break the first mbuf off from the rest of the mbuf chain. 189 */ 190 firstmbuf = m0; 191 secondmbuf = m0->m_next; 192 m0->m_next = NULL; 193 194 /* 195 * Insert the first mbuf of the m0 mbuf chain as the last record of 196 * the sockbuf. Note this permits zero length records! Keep the 197 * sockbuf state consistent. 198 */ 199 if (sb->sb_mb == NULL) 200 sb->sb_mb = firstmbuf; 201 else 202 sb->sb_lastrecord->m_nextpkt = firstmbuf; 203 sb->sb_lastrecord = firstmbuf; /* update hint for new last record */ 204 sb->sb_lastmbuf = firstmbuf; /* update hint for new last mbuf */ 205 206 /* 207 * propagate the EOR flag so sbcompress() can pick it up 208 */ 209 if ((firstmbuf->m_flags & M_EOR) && (secondmbuf != NULL)) { 210 firstmbuf->m_flags &= ~M_EOR; 211 secondmbuf->m_flags |= M_EOR; 212 } 213 214 /* 215 * The succeeding call to sbcompress() omits accounting for 216 * the first mbuf, so do it here. 217 */ 218 sballoc(sb, firstmbuf); 219 220 /* Compact the rest of the mbuf chain in after the first mbuf. */ 221 sbcompress(sb, secondmbuf, firstmbuf); 222 } 223 224 /* 225 * Append address and data, and optionally, control (ancillary) data 226 * to the receive queue of a socket. If present, 227 * m0 must include a packet header with total length. 228 * Returns 0 if insufficient mbufs. 229 */ 230 int 231 sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, 232 struct mbuf *control) 233 { 234 struct mbuf *m, *n; 235 int eor; 236 237 mbuftrackid(m0, 19); 238 mbuftrackid(control, 20); 239 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 240 panic("sbappendaddr"); 241 sbcheck(sb); 242 243 for (n = control; n; n = n->m_next) { 244 if (n->m_next == NULL) /* keep pointer to last control buf */ 245 break; 246 } 247 if (asa->sa_len > MLEN) 248 return (0); 249 MGET(m, M_NOWAIT, MT_SONAME); 250 if (m == NULL) 251 return (0); 252 KKASSERT(m->m_nextpkt == NULL); 253 m->m_len = asa->sa_len; 254 bcopy(asa, mtod(m, caddr_t), asa->sa_len); 255 if (n) 256 n->m_next = m0; /* concatenate data to control */ 257 else 258 control = m0; 259 m->m_next = control; 260 for (n = m; n; n = n->m_next) 261 sballoc(sb, n); 262 263 if (sb->sb_mb == NULL) 264 sb->sb_mb = m; 265 else 266 sb->sb_lastrecord->m_nextpkt = m; 267 sb->sb_lastrecord = m; 268 269 /* 270 * Propogate M_EOR to the last mbuf and calculate sb_lastmbuf 271 * so sbappend() can find it. 272 */ 273 eor = m->m_flags; 274 while (m->m_next) { 275 m->m_flags &= ~M_EOR; 276 m = m->m_next; 277 eor |= m->m_flags; 278 } 279 m->m_flags |= eor & M_EOR; 280 sb->sb_lastmbuf = m; 281 282 return (1); 283 } 284 285 /* 286 * Append control information followed by data. Both the control and data 287 * must be non-null. 288 */ 289 int 290 sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 291 { 292 struct mbuf *n; 293 u_int length, cmbcnt, m0mbcnt; 294 int eor; 295 296 KASSERT(control != NULL, ("sbappendcontrol")); 297 KKASSERT(control->m_nextpkt == NULL); 298 sbcheck(sb); 299 300 mbuftrackid(m0, 21); 301 mbuftrackid(control, 22); 302 303 length = m_countm(control, &n, &cmbcnt) + m_countm(m0, NULL, &m0mbcnt); 304 305 KKASSERT(m0 != NULL); 306 307 n->m_next = m0; /* concatenate data to control */ 308 309 if (sb->sb_mb == NULL) 310 sb->sb_mb = control; 311 else 312 sb->sb_lastrecord->m_nextpkt = control; 313 sb->sb_lastrecord = control; 314 315 /* 316 * Propogate M_EOR to the last mbuf and calculate sb_lastmbuf 317 * so sbappend() can find it. 318 */ 319 eor = m0->m_flags; 320 while (m0->m_next) { 321 m0->m_flags &= ~M_EOR; 322 m0 = m0->m_next; 323 eor |= m0->m_flags; 324 } 325 m0->m_flags |= eor & M_EOR; 326 sb->sb_lastmbuf = m0; 327 328 sb->sb_cc += length; 329 sb->sb_mbcnt += cmbcnt + m0mbcnt; 330 331 return (1); 332 } 333 334 /* 335 * Compress mbuf chain m into the socket buffer sb following mbuf tailm. 336 * If tailm is null, the buffer is presumed empty. Also, as a side-effect, 337 * increment the sockbuf counts for each mbuf in the chain. 338 */ 339 void 340 sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *tailm) 341 { 342 int eor = 0; 343 struct mbuf *free_chain = NULL; 344 345 mbuftrackid(m, 23); 346 347 sbcheck(sb); 348 while (m) { 349 struct mbuf *o; 350 351 eor |= m->m_flags & M_EOR; 352 /* 353 * Disregard empty mbufs as long as we don't encounter 354 * an end-of-record or there is a trailing mbuf of 355 * the same type to propagate the EOR flag to. 356 * 357 * Defer the m_free() call because it can block and break 358 * the atomicy of the sockbuf. 359 */ 360 if (m->m_len == 0 && 361 (eor == 0 || 362 (((o = m->m_next) || (o = tailm)) && 363 o->m_type == m->m_type))) { 364 o = m->m_next; 365 m->m_next = free_chain; 366 free_chain = m; 367 m = o; 368 continue; 369 } 370 371 /* 372 * See if we can coalesce with preceding mbuf. Never try 373 * to coalesce a mbuf representing an end-of-record or 374 * a mbuf locked by userland for reading. 375 */ 376 if (tailm && !(tailm->m_flags & (M_EOR | M_SOLOCKED)) && 377 M_WRITABLE(tailm) && 378 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 379 m->m_len <= M_TRAILINGSPACE(tailm) && 380 tailm->m_type == m->m_type) { 381 u_long mbcnt_sz; 382 383 bcopy(mtod(m, caddr_t), 384 mtod(tailm, caddr_t) + tailm->m_len, 385 (unsigned)m->m_len); 386 tailm->m_len += m->m_len; 387 388 sb->sb_cc += m->m_len; /* update sb counter */ 389 390 /* 391 * Fix the wrongly updated mbcnt_prealloc 392 */ 393 mbcnt_sz = MSIZE; 394 if (m->m_flags & M_EXT) 395 mbcnt_sz += m->m_ext.ext_size; 396 atomic_subtract_long(&sb->sb_mbcnt_prealloc, mbcnt_sz); 397 398 o = m->m_next; 399 m->m_next = free_chain; 400 free_chain = m; 401 m = o; 402 continue; 403 } 404 405 /* Insert whole mbuf. */ 406 if (tailm == NULL) { 407 KASSERT(sb->sb_mb == NULL, 408 ("sbcompress: sb_mb not NULL")); 409 sb->sb_mb = m; /* only mbuf in sockbuf */ 410 sb->sb_lastrecord = m; /* new last record */ 411 } else { 412 tailm->m_next = m; /* tack m on following tailm */ 413 } 414 sb->sb_lastmbuf = m; /* update last mbuf hint */ 415 416 tailm = m; /* just inserted mbuf becomes the new tail */ 417 m = m->m_next; /* advance to next mbuf */ 418 tailm->m_next = NULL; /* split inserted mbuf off from chain */ 419 420 /* update sb counters for just added mbuf */ 421 sballoc(sb, tailm); 422 423 /* clear EOR on intermediate mbufs */ 424 tailm->m_flags &= ~M_EOR; 425 } 426 427 /* 428 * Propogate EOR to the last mbuf 429 */ 430 if (eor) { 431 if (tailm) 432 tailm->m_flags |= eor; 433 else 434 kprintf("semi-panic: sbcompress"); 435 } 436 437 /* 438 * Clean up any defered frees. 439 */ 440 while (free_chain) 441 free_chain = m_free(free_chain); 442 443 sbcheck(sb); 444 } 445 446 /* 447 * Free all mbufs in a sockbuf. 448 * Check that all resources are reclaimed. 449 */ 450 void 451 sbflush(struct sockbuf *sb) 452 { 453 while (sb->sb_mbcnt) { 454 /* 455 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty: 456 * we would loop forever. Panic instead. 457 */ 458 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 459 break; 460 sbdrop(sb, (int)sb->sb_cc); 461 } 462 KASSERT(!(sb->sb_cc || sb->sb_mb || sb->sb_mbcnt || sb->sb_lastmbuf), 463 ("sbflush: cc %ld || mb %p || mbcnt %ld || lastmbuf %p", 464 sb->sb_cc, sb->sb_mb, sb->sb_mbcnt, sb->sb_lastmbuf)); 465 } 466 467 /* 468 * Drop data from (the front of) a sockbuf. If the current record is 469 * exhausted this routine will move onto the next one and continue dropping 470 * data. 471 */ 472 void 473 sbdrop(struct sockbuf *sb, int len) 474 { 475 struct mbuf *m; 476 struct mbuf *free_chain = NULL; 477 478 sbcheck(sb); 479 crit_enter(); 480 481 m = sb->sb_mb; 482 while (m && len > 0) { 483 if (m->m_len > len) { 484 m->m_len -= len; 485 m->m_data += len; 486 sb->sb_cc -= len; 487 atomic_subtract_long(&sb->sb_cc_prealloc, len); 488 break; 489 } 490 len -= m->m_len; 491 m = sbunlinkmbuf(sb, m, &free_chain); 492 if (m == NULL && len) 493 m = sb->sb_mb; 494 } 495 496 /* 497 * Remove any trailing 0-length mbufs in the current record. If 498 * the last record for which data was removed is now empty, m will be 499 * NULL. 500 */ 501 while (m && m->m_len == 0) { 502 m = sbunlinkmbuf(sb, m, &free_chain); 503 } 504 crit_exit(); 505 if (free_chain) 506 m_freem(free_chain); 507 sbcheck(sb); 508 } 509 510 /* 511 * Drop a record off the front of a sockbuf and move the next record 512 * to the front. 513 * 514 * Must be called while holding a critical section. 515 */ 516 void 517 sbdroprecord(struct sockbuf *sb) 518 { 519 struct mbuf *m; 520 struct mbuf *n; 521 522 sbcheck(sb); 523 m = sb->sb_mb; 524 if (m) { 525 if ((sb->sb_mb = m->m_nextpkt) == NULL) { 526 sb->sb_lastrecord = NULL; 527 sb->sb_lastmbuf = NULL; 528 } 529 m->m_nextpkt = NULL; 530 for (n = m; n; n = n->m_next) 531 sbfree(sb, n); 532 m_freem(m); 533 sbcheck(sb); 534 } 535 } 536 537 /* 538 * Drop the first mbuf off the sockbuf and move the next mbuf to the front. 539 * Currently only the head mbuf of the sockbuf may be dropped this way. 540 * 541 * The next mbuf in the same record as the mbuf being removed is returned 542 * or NULL if the record is exhausted. Note that other records may remain 543 * in the sockbuf when NULL is returned. 544 * 545 * Must be called while holding a critical section. 546 */ 547 struct mbuf * 548 sbunlinkmbuf(struct sockbuf *sb, struct mbuf *m, struct mbuf **free_chain) 549 { 550 struct mbuf *n; 551 552 KKASSERT(sb->sb_mb == m); 553 sbfree(sb, m); 554 n = m->m_next; 555 if (n) { 556 sb->sb_mb = n; 557 if (sb->sb_lastrecord == m) 558 sb->sb_lastrecord = n; 559 KKASSERT(sb->sb_lastmbuf != m); 560 n->m_nextpkt = m->m_nextpkt; 561 } else { 562 sb->sb_mb = m->m_nextpkt; 563 if (sb->sb_lastrecord == m) { 564 KKASSERT(sb->sb_mb == NULL); 565 sb->sb_lastrecord = NULL; 566 } 567 if (sb->sb_mb == NULL) 568 sb->sb_lastmbuf = NULL; 569 } 570 m->m_nextpkt = NULL; 571 if (free_chain) { 572 m->m_next = *free_chain; 573 *free_chain = m; 574 } else { 575 m->m_next = NULL; 576 } 577 return(n); 578 } 579 580 /* 581 * Create a "control" mbuf containing the specified data 582 * with the specified type for presentation on a socket buffer. 583 */ 584 struct mbuf * 585 sbcreatecontrol(const void *p, size_t size, int type, int level) 586 { 587 struct cmsghdr *cp; 588 struct mbuf *m; 589 590 if (CMSG_SPACE(size) > MCLBYTES) 591 return (NULL); 592 m = m_getl(CMSG_SPACE(size), M_NOWAIT, MT_CONTROL, 0, NULL); 593 if (m == NULL) 594 return (NULL); 595 m->m_len = CMSG_SPACE(size); 596 cp = mtod(m, struct cmsghdr *); 597 if (p != NULL) 598 memcpy(CMSG_DATA(cp), p, size); 599 cp->cmsg_len = CMSG_LEN(size); 600 cp->cmsg_level = level; 601 cp->cmsg_type = type; 602 mbuftrackid(m, 24); 603 return (m); 604 } 605