1 /* $NetBSD: vfs_bio.c,v 1.85 2002/09/06 13:18:43 gehenna Exp $ */ 2 3 /*- 4 * Copyright (c) 1994 Christopher G. Demetriou 5 * Copyright (c) 1982, 1986, 1989, 1993 6 * The Regents of the University of California. All rights reserved. 7 * (c) UNIX System Laboratories, Inc. 8 * All or some portions of this file are derived from material licensed 9 * to the University of California by American Telephone and Telegraph 10 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 11 * the permission of UNIX System Laboratories, Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 42 */ 43 44 /* 45 * Some references: 46 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986) 47 * Leffler, et al.: The Design and Implementation of the 4.3BSD 48 * UNIX Operating System (Addison Welley, 1989) 49 */ 50 51 #include "opt_softdep.h" 52 53 #include <sys/cdefs.h> 54 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.85 2002/09/06 13:18:43 gehenna Exp $"); 55 56 #include <sys/param.h> 57 #include <sys/systm.h> 58 #include <sys/proc.h> 59 #include <sys/buf.h> 60 #include <sys/vnode.h> 61 #include <sys/mount.h> 62 #include <sys/malloc.h> 63 #include <sys/resourcevar.h> 64 #include <sys/conf.h> 65 66 #include <uvm/uvm.h> 67 68 #include <miscfs/specfs/specdev.h> 69 70 /* Macros to clear/set/test flags. */ 71 #define SET(t, f) (t) |= (f) 72 #define CLR(t, f) (t) &= ~(f) 73 #define ISSET(t, f) ((t) & (f)) 74 75 /* 76 * Definitions for the buffer hash lists. 77 */ 78 #define BUFHASH(dvp, lbn) \ 79 (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash]) 80 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash; 81 u_long bufhash; 82 #ifndef SOFTDEP 83 struct bio_ops bioops; /* I/O operation notification */ 84 #endif 85 86 /* 87 * Insq/Remq for the buffer hash lists. 88 */ 89 #define binshash(bp, dp) LIST_INSERT_HEAD(dp, bp, b_hash) 90 #define bremhash(bp) LIST_REMOVE(bp, b_hash) 91 92 /* 93 * Definitions for the buffer free lists. 94 */ 95 #define BQUEUES 4 /* number of free buffer queues */ 96 97 #define BQ_LOCKED 0 /* super-blocks &c */ 98 #define BQ_LRU 1 /* lru, useful buffers */ 99 #define BQ_AGE 2 /* rubbish */ 100 #define BQ_EMPTY 3 /* buffer headers with no memory */ 101 102 TAILQ_HEAD(bqueues, buf) bufqueues[BQUEUES]; 103 int needbuffer; 104 105 /* 106 * Buffer pool for I/O buffers. 107 */ 108 struct pool bufpool; 109 110 /* 111 * Insq/Remq for the buffer free lists. 112 */ 113 #define binsheadfree(bp, dp) TAILQ_INSERT_HEAD(dp, bp, b_freelist) 114 #define binstailfree(bp, dp) TAILQ_INSERT_TAIL(dp, bp, b_freelist) 115 116 static __inline struct buf *bio_doread __P((struct vnode *, daddr_t, int, 117 struct ucred *, int)); 118 int count_lock_queue __P((void)); 119 120 void 121 bremfree(bp) 122 struct buf *bp; 123 { 124 int s = splbio(); 125 126 struct bqueues *dp = NULL; 127 128 /* 129 * We only calculate the head of the freelist when removing 130 * the last element of the list as that is the only time that 131 * it is needed (e.g. to reset the tail pointer). 132 * 133 * NB: This makes an assumption about how tailq's are implemented. 134 */ 135 if (TAILQ_NEXT(bp, b_freelist) == NULL) { 136 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) 137 if (dp->tqh_last == &bp->b_freelist.tqe_next) 138 break; 139 if (dp == &bufqueues[BQUEUES]) 140 panic("bremfree: lost tail"); 141 } 142 TAILQ_REMOVE(dp, bp, b_freelist); 143 splx(s); 144 } 145 146 /* 147 * Initialize buffers and hash links for buffers. 148 */ 149 void 150 bufinit() 151 { 152 struct buf *bp; 153 struct bqueues *dp; 154 u_int i, base, residual; 155 156 /* 157 * Initialize the buffer pool. This pool is used for buffers 158 * which are strictly I/O control blocks, not buffer cache 159 * buffers. 160 */ 161 pool_init(&bufpool, sizeof(struct buf), 0, 0, 0, "bufpl", NULL); 162 163 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) 164 TAILQ_INIT(dp); 165 bufhashtbl = hashinit(nbuf, HASH_LIST, M_CACHE, M_WAITOK, &bufhash); 166 base = bufpages / nbuf; 167 residual = bufpages % nbuf; 168 for (i = 0; i < nbuf; i++) { 169 bp = &buf[i]; 170 memset((char *)bp, 0, sizeof(*bp)); 171 bp->b_dev = NODEV; 172 bp->b_vnbufs.le_next = NOLIST; 173 LIST_INIT(&bp->b_dep); 174 bp->b_data = buffers + i * MAXBSIZE; 175 if (i < residual) 176 bp->b_bufsize = (base + 1) * PAGE_SIZE; 177 else 178 bp->b_bufsize = base * PAGE_SIZE; 179 bp->b_flags = B_INVAL; 180 dp = bp->b_bufsize ? &bufqueues[BQ_AGE] : &bufqueues[BQ_EMPTY]; 181 binsheadfree(bp, dp); 182 binshash(bp, &invalhash); 183 } 184 } 185 186 static __inline struct buf * 187 bio_doread(vp, blkno, size, cred, async) 188 struct vnode *vp; 189 daddr_t blkno; 190 int size; 191 struct ucred *cred; 192 int async; 193 { 194 struct buf *bp; 195 struct proc *p = (curproc != NULL ? curproc : &proc0); /* XXX */ 196 197 bp = getblk(vp, blkno, size, 0, 0); 198 199 /* 200 * If buffer does not have data valid, start a read. 201 * Note that if buffer is B_INVAL, getblk() won't return it. 202 * Therefore, it's valid if it's I/O has completed or been delayed. 203 */ 204 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { 205 /* Start I/O for the buffer. */ 206 SET(bp->b_flags, B_READ | async); 207 VOP_STRATEGY(bp); 208 209 /* Pay for the read. */ 210 p->p_stats->p_ru.ru_inblock++; 211 } else if (async) { 212 brelse(bp); 213 } 214 215 return (bp); 216 } 217 218 /* 219 * Read a disk block. 220 * This algorithm described in Bach (p.54). 221 */ 222 int 223 bread(vp, blkno, size, cred, bpp) 224 struct vnode *vp; 225 daddr_t blkno; 226 int size; 227 struct ucred *cred; 228 struct buf **bpp; 229 { 230 struct buf *bp; 231 232 /* Get buffer for block. */ 233 bp = *bpp = bio_doread(vp, blkno, size, cred, 0); 234 235 /* Wait for the read to complete, and return result. */ 236 return (biowait(bp)); 237 } 238 239 /* 240 * Read-ahead multiple disk blocks. The first is sync, the rest async. 241 * Trivial modification to the breada algorithm presented in Bach (p.55). 242 */ 243 int 244 breadn(vp, blkno, size, rablks, rasizes, nrablks, cred, bpp) 245 struct vnode *vp; 246 daddr_t blkno; int size; 247 daddr_t rablks[]; int rasizes[]; 248 int nrablks; 249 struct ucred *cred; 250 struct buf **bpp; 251 { 252 struct buf *bp; 253 int i; 254 255 bp = *bpp = bio_doread(vp, blkno, size, cred, 0); 256 257 /* 258 * For each of the read-ahead blocks, start a read, if necessary. 259 */ 260 for (i = 0; i < nrablks; i++) { 261 /* If it's in the cache, just go on to next one. */ 262 if (incore(vp, rablks[i])) 263 continue; 264 265 /* Get a buffer for the read-ahead block */ 266 (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC); 267 } 268 269 /* Otherwise, we had to start a read for it; wait until it's valid. */ 270 return (biowait(bp)); 271 } 272 273 /* 274 * Read with single-block read-ahead. Defined in Bach (p.55), but 275 * implemented as a call to breadn(). 276 * XXX for compatibility with old file systems. 277 */ 278 int 279 breada(vp, blkno, size, rablkno, rabsize, cred, bpp) 280 struct vnode *vp; 281 daddr_t blkno; int size; 282 daddr_t rablkno; int rabsize; 283 struct ucred *cred; 284 struct buf **bpp; 285 { 286 287 return (breadn(vp, blkno, size, &rablkno, &rabsize, 1, cred, bpp)); 288 } 289 290 /* 291 * Block write. Described in Bach (p.56) 292 */ 293 int 294 bwrite(bp) 295 struct buf *bp; 296 { 297 int rv, sync, wasdelayed, s; 298 struct proc *p = (curproc != NULL ? curproc : &proc0); /* XXX */ 299 struct vnode *vp; 300 struct mount *mp; 301 302 vp = bp->b_vp; 303 if (vp != NULL) { 304 if (vp->v_type == VBLK) 305 mp = vp->v_specmountpoint; 306 else 307 mp = vp->v_mount; 308 } else { 309 mp = NULL; 310 } 311 312 /* 313 * Remember buffer type, to switch on it later. If the write was 314 * synchronous, but the file system was mounted with MNT_ASYNC, 315 * convert it to a delayed write. 316 * XXX note that this relies on delayed tape writes being converted 317 * to async, not sync writes (which is safe, but ugly). 318 */ 319 sync = !ISSET(bp->b_flags, B_ASYNC); 320 if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) { 321 bdwrite(bp); 322 return (0); 323 } 324 325 /* 326 * Collect statistics on synchronous and asynchronous writes. 327 * Writes to block devices are charged to their associated 328 * filesystem (if any). 329 */ 330 if (mp != NULL) { 331 if (sync) 332 mp->mnt_stat.f_syncwrites++; 333 else 334 mp->mnt_stat.f_asyncwrites++; 335 } 336 337 wasdelayed = ISSET(bp->b_flags, B_DELWRI); 338 339 s = splbio(); 340 341 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI)); 342 343 /* 344 * Pay for the I/O operation and make sure the buf is on the correct 345 * vnode queue. 346 */ 347 if (wasdelayed) 348 reassignbuf(bp, bp->b_vp); 349 else 350 p->p_stats->p_ru.ru_oublock++; 351 352 /* Initiate disk write. Make sure the appropriate party is charged. */ 353 bp->b_vp->v_numoutput++; 354 splx(s); 355 356 VOP_STRATEGY(bp); 357 358 if (sync) { 359 /* If I/O was synchronous, wait for it to complete. */ 360 rv = biowait(bp); 361 362 /* Release the buffer. */ 363 brelse(bp); 364 365 return (rv); 366 } else { 367 return (0); 368 } 369 } 370 371 int 372 vn_bwrite(v) 373 void *v; 374 { 375 struct vop_bwrite_args *ap = v; 376 377 return (bwrite(ap->a_bp)); 378 } 379 380 /* 381 * Delayed write. 382 * 383 * The buffer is marked dirty, but is not queued for I/O. 384 * This routine should be used when the buffer is expected 385 * to be modified again soon, typically a small write that 386 * partially fills a buffer. 387 * 388 * NB: magnetic tapes cannot be delayed; they must be 389 * written in the order that the writes are requested. 390 * 391 * Described in Leffler, et al. (pp. 208-213). 392 */ 393 void 394 bdwrite(bp) 395 struct buf *bp; 396 { 397 struct proc *p = (curproc != NULL ? curproc : &proc0); /* XXX */ 398 const struct bdevsw *bdev; 399 int s; 400 401 /* If this is a tape block, write the block now. */ 402 /* XXX NOTE: the memory filesystem usurpes major device */ 403 /* XXX number 4095, which is a bad idea. */ 404 if (bp->b_dev != NODEV && major(bp->b_dev) != 4095) { 405 bdev = bdevsw_lookup(bp->b_dev); 406 if (bdev != NULL && bdev->d_type == D_TAPE) { 407 bawrite(bp); 408 return; 409 } 410 } 411 412 /* 413 * If the block hasn't been seen before: 414 * (1) Mark it as having been seen, 415 * (2) Charge for the write, 416 * (3) Make sure it's on its vnode's correct block list. 417 */ 418 s = splbio(); 419 420 if (!ISSET(bp->b_flags, B_DELWRI)) { 421 SET(bp->b_flags, B_DELWRI); 422 p->p_stats->p_ru.ru_oublock++; 423 reassignbuf(bp, bp->b_vp); 424 } 425 426 /* Otherwise, the "write" is done, so mark and release the buffer. */ 427 CLR(bp->b_flags, B_NEEDCOMMIT|B_DONE); 428 splx(s); 429 430 brelse(bp); 431 } 432 433 /* 434 * Asynchronous block write; just an asynchronous bwrite(). 435 */ 436 void 437 bawrite(bp) 438 struct buf *bp; 439 { 440 441 SET(bp->b_flags, B_ASYNC); 442 VOP_BWRITE(bp); 443 } 444 445 /* 446 * Same as first half of bdwrite, mark buffer dirty, but do not release it. 447 */ 448 void 449 bdirty(bp) 450 struct buf *bp; 451 { 452 struct proc *p = (curproc != NULL ? curproc : &proc0); /* XXX */ 453 int s; 454 455 s = splbio(); 456 457 CLR(bp->b_flags, B_AGE); 458 459 if (!ISSET(bp->b_flags, B_DELWRI)) { 460 SET(bp->b_flags, B_DELWRI); 461 p->p_stats->p_ru.ru_oublock++; 462 reassignbuf(bp, bp->b_vp); 463 } 464 465 splx(s); 466 } 467 468 /* 469 * Release a buffer on to the free lists. 470 * Described in Bach (p. 46). 471 */ 472 void 473 brelse(bp) 474 struct buf *bp; 475 { 476 struct bqueues *bufq; 477 int s; 478 479 KASSERT(ISSET(bp->b_flags, B_BUSY)); 480 481 /* Wake up any processes waiting for any buffer to become free. */ 482 if (needbuffer) { 483 needbuffer = 0; 484 wakeup(&needbuffer); 485 } 486 487 /* Block disk interrupts. */ 488 s = splbio(); 489 490 /* Wake up any proceeses waiting for _this_ buffer to become free. */ 491 if (ISSET(bp->b_flags, B_WANTED)) { 492 CLR(bp->b_flags, B_WANTED|B_AGE); 493 wakeup(bp); 494 } 495 496 /* 497 * Determine which queue the buffer should be on, then put it there. 498 */ 499 500 /* If it's locked, don't report an error; try again later. */ 501 if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR)) 502 CLR(bp->b_flags, B_ERROR); 503 504 /* If it's not cacheable, or an error, mark it invalid. */ 505 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR))) 506 SET(bp->b_flags, B_INVAL); 507 508 if (ISSET(bp->b_flags, B_VFLUSH)) { 509 /* 510 * This is a delayed write buffer that was just flushed to 511 * disk. It is still on the LRU queue. If it's become 512 * invalid, then we need to move it to a different queue; 513 * otherwise leave it in its current position. 514 */ 515 CLR(bp->b_flags, B_VFLUSH); 516 if (!ISSET(bp->b_flags, B_ERROR|B_INVAL|B_LOCKED|B_AGE)) 517 goto already_queued; 518 else 519 bremfree(bp); 520 } 521 522 if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL)) { 523 /* 524 * If it's invalid or empty, dissociate it from its vnode 525 * and put on the head of the appropriate queue. 526 */ 527 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 528 (*bioops.io_deallocate)(bp); 529 CLR(bp->b_flags, B_DONE|B_DELWRI); 530 if (bp->b_vp) { 531 reassignbuf(bp, bp->b_vp); 532 brelvp(bp); 533 } 534 if (bp->b_bufsize <= 0) 535 /* no data */ 536 bufq = &bufqueues[BQ_EMPTY]; 537 else 538 /* invalid data */ 539 bufq = &bufqueues[BQ_AGE]; 540 binsheadfree(bp, bufq); 541 } else { 542 /* 543 * It has valid data. Put it on the end of the appropriate 544 * queue, so that it'll stick around for as long as possible. 545 * If buf is AGE, but has dependencies, must put it on last 546 * bufqueue to be scanned, ie LRU. This protects against the 547 * livelock where BQ_AGE only has buffers with dependencies, 548 * and we thus never get to the dependent buffers in BQ_LRU. 549 */ 550 if (ISSET(bp->b_flags, B_LOCKED)) 551 /* locked in core */ 552 bufq = &bufqueues[BQ_LOCKED]; 553 else if (!ISSET(bp->b_flags, B_AGE)) 554 /* valid data */ 555 bufq = &bufqueues[BQ_LRU]; 556 else { 557 /* stale but valid data */ 558 int has_deps; 559 560 if (LIST_FIRST(&bp->b_dep) != NULL && 561 bioops.io_countdeps) 562 has_deps = (*bioops.io_countdeps)(bp, 0); 563 else 564 has_deps = 0; 565 bufq = has_deps ? &bufqueues[BQ_LRU] : 566 &bufqueues[BQ_AGE]; 567 } 568 binstailfree(bp, bufq); 569 } 570 571 already_queued: 572 /* Unlock the buffer. */ 573 CLR(bp->b_flags, B_AGE|B_ASYNC|B_BUSY|B_NOCACHE); 574 SET(bp->b_flags, B_CACHE); 575 576 /* Allow disk interrupts. */ 577 splx(s); 578 } 579 580 /* 581 * Determine if a block is in the cache. 582 * Just look on what would be its hash chain. If it's there, return 583 * a pointer to it, unless it's marked invalid. If it's marked invalid, 584 * we normally don't return the buffer, unless the caller explicitly 585 * wants us to. 586 */ 587 struct buf * 588 incore(vp, blkno) 589 struct vnode *vp; 590 daddr_t blkno; 591 { 592 struct buf *bp; 593 594 /* Search hash chain */ 595 LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) { 596 if (bp->b_lblkno == blkno && bp->b_vp == vp && 597 !ISSET(bp->b_flags, B_INVAL)) 598 return (bp); 599 } 600 601 return (NULL); 602 } 603 604 /* 605 * Get a block of requested size that is associated with 606 * a given vnode and block offset. If it is found in the 607 * block cache, mark it as having been found, make it busy 608 * and return it. Otherwise, return an empty block of the 609 * correct size. It is up to the caller to insure that the 610 * cached blocks be of the correct size. 611 */ 612 struct buf * 613 getblk(vp, blkno, size, slpflag, slptimeo) 614 struct vnode *vp; 615 daddr_t blkno; 616 int size, slpflag, slptimeo; 617 { 618 struct buf *bp; 619 int s, err; 620 621 start: 622 bp = incore(vp, blkno); 623 if (bp != NULL) { 624 s = splbio(); 625 if (ISSET(bp->b_flags, B_BUSY)) { 626 if (curproc == uvm.pagedaemon_proc) { 627 splx(s); 628 return NULL; 629 } 630 SET(bp->b_flags, B_WANTED); 631 err = tsleep(bp, slpflag | (PRIBIO + 1), "getblk", 632 slptimeo); 633 splx(s); 634 if (err) 635 return (NULL); 636 goto start; 637 } 638 #ifdef DIAGNOSTIC 639 if (ISSET(bp->b_flags, B_DONE|B_DELWRI) && 640 bp->b_bcount < size && vp->v_type != VBLK) 641 panic("getblk: block size invariant failed"); 642 #endif 643 SET(bp->b_flags, B_BUSY); 644 bremfree(bp); 645 splx(s); 646 } else { 647 if ((bp = getnewbuf(slpflag, slptimeo)) == NULL) 648 goto start; 649 650 binshash(bp, BUFHASH(vp, blkno)); 651 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno; 652 s = splbio(); 653 bgetvp(vp, bp); 654 splx(s); 655 } 656 allocbuf(bp, size); 657 return (bp); 658 } 659 660 /* 661 * Get an empty, disassociated buffer of given size. 662 */ 663 struct buf * 664 geteblk(size) 665 int size; 666 { 667 struct buf *bp; 668 669 while ((bp = getnewbuf(0, 0)) == 0) 670 ; 671 SET(bp->b_flags, B_INVAL); 672 binshash(bp, &invalhash); 673 allocbuf(bp, size); 674 return (bp); 675 } 676 677 /* 678 * Expand or contract the actual memory allocated to a buffer. 679 * 680 * If the buffer shrinks, data is lost, so it's up to the 681 * caller to have written it out *first*; this routine will not 682 * start a write. If the buffer grows, it's the callers 683 * responsibility to fill out the buffer's additional contents. 684 */ 685 void 686 allocbuf(bp, size) 687 struct buf *bp; 688 int size; 689 { 690 struct buf *nbp; 691 vsize_t desired_size; 692 int s; 693 694 desired_size = round_page((vsize_t)size); 695 if (desired_size > MAXBSIZE) 696 panic("allocbuf: buffer larger than MAXBSIZE requested"); 697 698 if (bp->b_bufsize == desired_size) 699 goto out; 700 701 /* 702 * If the buffer is smaller than the desired size, we need to snarf 703 * it from other buffers. Get buffers (via getnewbuf()), and 704 * steal their pages. 705 */ 706 while (bp->b_bufsize < desired_size) { 707 int amt; 708 709 /* find a buffer */ 710 while ((nbp = getnewbuf(0, 0)) == NULL) 711 ; 712 713 SET(nbp->b_flags, B_INVAL); 714 binshash(nbp, &invalhash); 715 716 /* and steal its pages, up to the amount we need */ 717 amt = min(nbp->b_bufsize, (desired_size - bp->b_bufsize)); 718 pagemove((nbp->b_data + nbp->b_bufsize - amt), 719 bp->b_data + bp->b_bufsize, amt); 720 bp->b_bufsize += amt; 721 nbp->b_bufsize -= amt; 722 723 /* reduce transfer count if we stole some data */ 724 if (nbp->b_bcount > nbp->b_bufsize) 725 nbp->b_bcount = nbp->b_bufsize; 726 727 #ifdef DIAGNOSTIC 728 if (nbp->b_bufsize < 0) 729 panic("allocbuf: negative bufsize"); 730 #endif 731 732 brelse(nbp); 733 } 734 735 /* 736 * If we want a buffer smaller than the current size, 737 * shrink this buffer. Grab a buf head from the EMPTY queue, 738 * move a page onto it, and put it on front of the AGE queue. 739 * If there are no free buffer headers, leave the buffer alone. 740 */ 741 if (bp->b_bufsize > desired_size) { 742 s = splbio(); 743 if ((nbp = TAILQ_FIRST(&bufqueues[BQ_EMPTY])) == NULL) { 744 /* No free buffer head */ 745 splx(s); 746 goto out; 747 } 748 bremfree(nbp); 749 SET(nbp->b_flags, B_BUSY); 750 splx(s); 751 752 /* move the page to it and note this change */ 753 pagemove(bp->b_data + desired_size, 754 nbp->b_data, bp->b_bufsize - desired_size); 755 nbp->b_bufsize = bp->b_bufsize - desired_size; 756 bp->b_bufsize = desired_size; 757 nbp->b_bcount = 0; 758 SET(nbp->b_flags, B_INVAL); 759 760 /* release the newly-filled buffer and leave */ 761 brelse(nbp); 762 } 763 764 out: 765 bp->b_bcount = size; 766 } 767 768 /* 769 * Find a buffer which is available for use. 770 * Select something from a free list. 771 * Preference is to AGE list, then LRU list. 772 */ 773 struct buf * 774 getnewbuf(slpflag, slptimeo) 775 int slpflag, slptimeo; 776 { 777 struct buf *bp; 778 int s; 779 780 start: 781 s = splbio(); 782 if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE])) != NULL || 783 (bp = TAILQ_FIRST(&bufqueues[BQ_LRU])) != NULL) { 784 bremfree(bp); 785 } else { 786 /* wait for a free buffer of any kind */ 787 needbuffer = 1; 788 tsleep(&needbuffer, slpflag|(PRIBIO+1), "getnewbuf", slptimeo); 789 splx(s); 790 return (NULL); 791 } 792 793 if (ISSET(bp->b_flags, B_VFLUSH)) { 794 /* 795 * This is a delayed write buffer being flushed to disk. Make 796 * sure it gets aged out of the queue when it's finished, and 797 * leave it off the LRU queue. 798 */ 799 CLR(bp->b_flags, B_VFLUSH); 800 SET(bp->b_flags, B_AGE); 801 splx(s); 802 goto start; 803 } 804 805 /* Buffer is no longer on free lists. */ 806 SET(bp->b_flags, B_BUSY); 807 808 /* 809 * If buffer was a delayed write, start it and return NULL 810 * (since we might sleep while starting the write). 811 */ 812 if (ISSET(bp->b_flags, B_DELWRI)) { 813 splx(s); 814 /* 815 * This buffer has gone through the LRU, so make sure it gets 816 * reused ASAP. 817 */ 818 SET(bp->b_flags, B_AGE); 819 bawrite(bp); 820 return (NULL); 821 } 822 823 /* disassociate us from our vnode, if we had one... */ 824 if (bp->b_vp) 825 brelvp(bp); 826 splx(s); 827 828 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 829 (*bioops.io_deallocate)(bp); 830 831 /* clear out various other fields */ 832 bp->b_flags = B_BUSY; 833 bp->b_dev = NODEV; 834 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = 0; 835 bp->b_iodone = 0; 836 bp->b_error = 0; 837 bp->b_resid = 0; 838 bp->b_bcount = 0; 839 840 bremhash(bp); 841 return (bp); 842 } 843 844 /* 845 * Wait for operations on the buffer to complete. 846 * When they do, extract and return the I/O's error value. 847 */ 848 int 849 biowait(bp) 850 struct buf *bp; 851 { 852 int s; 853 854 s = splbio(); 855 while (!ISSET(bp->b_flags, B_DONE | B_DELWRI)) 856 tsleep(bp, PRIBIO + 1, "biowait", 0); 857 splx(s); 858 859 /* check for interruption of I/O (e.g. via NFS), then errors. */ 860 if (ISSET(bp->b_flags, B_EINTR)) { 861 CLR(bp->b_flags, B_EINTR); 862 return (EINTR); 863 } else if (ISSET(bp->b_flags, B_ERROR)) 864 return (bp->b_error ? bp->b_error : EIO); 865 else 866 return (0); 867 } 868 869 /* 870 * Mark I/O complete on a buffer. 871 * 872 * If a callback has been requested, e.g. the pageout 873 * daemon, do so. Otherwise, awaken waiting processes. 874 * 875 * [ Leffler, et al., says on p.247: 876 * "This routine wakes up the blocked process, frees the buffer 877 * for an asynchronous write, or, for a request by the pagedaemon 878 * process, invokes a procedure specified in the buffer structure" ] 879 * 880 * In real life, the pagedaemon (or other system processes) wants 881 * to do async stuff to, and doesn't want the buffer brelse()'d. 882 * (for swap pager, that puts swap buffers on the free lists (!!!), 883 * for the vn device, that puts malloc'd buffers on the free lists!) 884 */ 885 void 886 biodone(bp) 887 struct buf *bp; 888 { 889 int s = splbio(); 890 891 if (ISSET(bp->b_flags, B_DONE)) 892 panic("biodone already"); 893 SET(bp->b_flags, B_DONE); /* note that it's done */ 894 895 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete) 896 (*bioops.io_complete)(bp); 897 898 if (!ISSET(bp->b_flags, B_READ)) /* wake up reader */ 899 vwakeup(bp); 900 901 if (ISSET(bp->b_flags, B_CALL)) { /* if necessary, call out */ 902 CLR(bp->b_flags, B_CALL); /* but note callout done */ 903 (*bp->b_iodone)(bp); 904 } else { 905 if (ISSET(bp->b_flags, B_ASYNC)) /* if async, release */ 906 brelse(bp); 907 else { /* or just wakeup the buffer */ 908 CLR(bp->b_flags, B_WANTED); 909 wakeup(bp); 910 } 911 } 912 913 splx(s); 914 } 915 916 /* 917 * Return a count of buffers on the "locked" queue. 918 */ 919 int 920 count_lock_queue() 921 { 922 struct buf *bp; 923 int n = 0; 924 925 TAILQ_FOREACH(bp, &bufqueues[BQ_LOCKED], b_freelist) 926 n++; 927 return (n); 928 } 929 930 #ifdef DEBUG 931 /* 932 * Print out statistics on the current allocation of the buffer pool. 933 * Can be enabled to print out on every ``sync'' by setting "syncprt" 934 * in vfs_syscalls.c using sysctl. 935 */ 936 void 937 vfs_bufstats() 938 { 939 int s, i, j, count; 940 struct buf *bp; 941 struct bqueues *dp; 942 int counts[(MAXBSIZE / PAGE_SIZE) + 1]; 943 static char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE", "EMPTY" }; 944 945 for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) { 946 count = 0; 947 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++) 948 counts[j] = 0; 949 s = splbio(); 950 TAILQ_FOREACH(bp, dp, b_freelist) { 951 counts[bp->b_bufsize/PAGE_SIZE]++; 952 count++; 953 } 954 splx(s); 955 printf("%s: total-%d", bname[i], count); 956 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++) 957 if (counts[j] != 0) 958 printf(", %d-%d", j * PAGE_SIZE, counts[j]); 959 printf("\n"); 960 } 961 } 962 #endif /* DEBUG */ 963