1 /* $OpenBSD: vfs_bio.c,v 1.201 2020/07/14 06:02:50 beck Exp $ */ 2 /* $NetBSD: vfs_bio.c,v 1.44 1996/06/11 11:15:36 pk Exp $ */ 3 4 /* 5 * Copyright (c) 1994 Christopher G. Demetriou 6 * Copyright (c) 1982, 1986, 1989, 1993 7 * The Regents of the University of California. All rights reserved. 8 * (c) UNIX System Laboratories, Inc. 9 * All or some portions of this file are derived from material licensed 10 * to the University of California by American Telephone and Telegraph 11 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 12 * the permission of UNIX System Laboratories, Inc. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. Neither the name of the University nor the names of its contributors 23 * may be used to endorse or promote products derived from this software 24 * without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 * 38 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 39 */ 40 41 /* 42 * Some references: 43 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986) 44 * Leffler, et al.: The Design and Implementation of the 4.3BSD 45 * UNIX Operating System (Addison Welley, 1989) 46 */ 47 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/proc.h> 51 #include <sys/buf.h> 52 #include <sys/vnode.h> 53 #include <sys/mount.h> 54 #include <sys/malloc.h> 55 #include <sys/pool.h> 56 #include <sys/resourcevar.h> 57 #include <sys/conf.h> 58 #include <sys/kernel.h> 59 #include <sys/specdev.h> 60 #include <uvm/uvm_extern.h> 61 62 /* XXX Should really be in buf.h, but for uvm_constraint_range.. */ 63 int buf_realloc_pages(struct buf *, struct uvm_constraint_range *, int); 64 65 struct uvm_constraint_range high_constraint; 66 int fliphigh; 67 68 int nobuffers; 69 int needbuffer; 70 struct bio_ops bioops; 71 72 /* private bufcache functions */ 73 void bufcache_init(void); 74 void bufcache_adjust(void); 75 struct buf *bufcache_gethighcleanbuf(void); 76 struct buf *bufcache_getdmacleanbuf(void); 77 78 /* 79 * Buffer pool for I/O buffers. 80 */ 81 struct pool bufpool; 82 struct bufhead bufhead = LIST_HEAD_INITIALIZER(bufhead); 83 void buf_put(struct buf *); 84 85 struct buf *bio_doread(struct vnode *, daddr_t, int, int); 86 struct buf *buf_get(struct vnode *, daddr_t, size_t); 87 void bread_cluster_callback(struct buf *); 88 int64_t bufcache_recover_dmapages(int discard, int64_t howmany); 89 90 struct bcachestats bcstats; /* counters */ 91 long lodirtypages; /* dirty page count low water mark */ 92 long hidirtypages; /* dirty page count high water mark */ 93 long targetpages; /* target number of pages for cache size */ 94 long buflowpages; /* smallest size cache allowed */ 95 long bufhighpages; /* largest size cache allowed */ 96 long bufbackpages; /* minimum number of pages we shrink when asked to */ 97 98 vsize_t bufkvm; 99 100 struct proc *cleanerproc; 101 int bd_req; /* Sleep point for cleaner daemon. */ 102 103 #define NUM_CACHES 2 104 #define DMA_CACHE 0 105 struct bufcache cleancache[NUM_CACHES]; 106 struct bufqueue dirtyqueue; 107 108 void 109 buf_put(struct buf *bp) 110 { 111 splassert(IPL_BIO); 112 113 #ifdef DIAGNOSTIC 114 if (bp->b_pobj != NULL) 115 KASSERT(bp->b_bufsize > 0); 116 if (ISSET(bp->b_flags, B_DELWRI)) 117 panic("buf_put: releasing dirty buffer"); 118 if (bp->b_freelist.tqe_next != NOLIST && 119 bp->b_freelist.tqe_next != (void *)-1) 120 panic("buf_put: still on the free list"); 121 if (bp->b_vnbufs.le_next != NOLIST && 122 bp->b_vnbufs.le_next != (void *)-1) 123 panic("buf_put: still on the vnode list"); 124 if (!LIST_EMPTY(&bp->b_dep)) 125 panic("buf_put: b_dep is not empty"); 126 #endif 127 128 LIST_REMOVE(bp, b_list); 129 bcstats.numbufs--; 130 131 if (buf_dealloc_mem(bp) != 0) 132 return; 133 pool_put(&bufpool, bp); 134 } 135 136 /* 137 * Initialize buffers and hash links for buffers. 138 */ 139 void 140 bufinit(void) 141 { 142 u_int64_t dmapages; 143 u_int64_t highpages; 144 145 dmapages = uvm_pagecount(&dma_constraint); 146 /* take away a guess at how much of this the kernel will consume */ 147 dmapages -= (atop(physmem) - atop(uvmexp.free)); 148 149 /* See if we have memory above the dma accessible region. */ 150 high_constraint.ucr_low = dma_constraint.ucr_high; 151 high_constraint.ucr_high = no_constraint.ucr_high; 152 if (high_constraint.ucr_low != high_constraint.ucr_high) 153 high_constraint.ucr_low++; 154 highpages = uvm_pagecount(&high_constraint); 155 156 /* 157 * Do we have any significant amount of high memory above 158 * the DMA region? if so enable moving buffers there, if not, 159 * don't bother. 160 */ 161 if (highpages > dmapages / 4) 162 fliphigh = 1; 163 else 164 fliphigh = 0; 165 166 /* 167 * If MD code doesn't say otherwise, use up to 10% of DMA'able 168 * memory for buffers. 169 */ 170 if (bufcachepercent == 0) 171 bufcachepercent = 10; 172 173 /* 174 * XXX these values and their same use in kern_sysctl 175 * need to move into buf.h 176 */ 177 KASSERT(bufcachepercent <= 90); 178 KASSERT(bufcachepercent >= 5); 179 if (bufpages == 0) 180 bufpages = dmapages * bufcachepercent / 100; 181 if (bufpages < BCACHE_MIN) 182 bufpages = BCACHE_MIN; 183 KASSERT(bufpages < dmapages); 184 185 bufhighpages = bufpages; 186 187 /* 188 * Set the base backoff level for the buffer cache. We will 189 * not allow uvm to steal back more than this number of pages. 190 */ 191 buflowpages = dmapages * 5 / 100; 192 if (buflowpages < BCACHE_MIN) 193 buflowpages = BCACHE_MIN; 194 195 /* 196 * set bufbackpages to 100 pages, or 10 percent of the low water mark 197 * if we don't have that many pages. 198 */ 199 200 bufbackpages = buflowpages * 10 / 100; 201 if (bufbackpages > 100) 202 bufbackpages = 100; 203 204 /* 205 * If the MD code does not say otherwise, reserve 10% of kva 206 * space for mapping buffers. 207 */ 208 if (bufkvm == 0) 209 bufkvm = VM_KERNEL_SPACE_SIZE / 10; 210 211 /* 212 * Don't use more than twice the amount of bufpages for mappings. 213 * It's twice since we map things sparsely. 214 */ 215 if (bufkvm > bufpages * PAGE_SIZE) 216 bufkvm = bufpages * PAGE_SIZE; 217 /* 218 * Round bufkvm to MAXPHYS because we allocate chunks of va space 219 * in MAXPHYS chunks. 220 */ 221 bufkvm &= ~(MAXPHYS - 1); 222 223 pool_init(&bufpool, sizeof(struct buf), 0, IPL_BIO, 0, "bufpl", NULL); 224 225 bufcache_init(); 226 227 /* 228 * hmm - bufkvm is an argument because it's static, while 229 * bufpages is global because it can change while running. 230 */ 231 buf_mem_init(bufkvm); 232 233 /* 234 * Set the dirty page high water mark to be less than the low 235 * water mark for pages in the buffer cache. This ensures we 236 * can always back off by throwing away clean pages, and give 237 * ourselves a chance to write out the dirty pages eventually. 238 */ 239 hidirtypages = (buflowpages / 4) * 3; 240 lodirtypages = buflowpages / 2; 241 242 /* 243 * We are allowed to use up to the reserve. 244 */ 245 targetpages = bufpages - RESERVE_PAGES; 246 } 247 248 /* 249 * Change cachepct 250 */ 251 void 252 bufadjust(int newbufpages) 253 { 254 int s; 255 int64_t npages; 256 257 if (newbufpages < buflowpages) 258 newbufpages = buflowpages; 259 260 s = splbio(); 261 bufpages = newbufpages; 262 263 /* 264 * We are allowed to use up to the reserve 265 */ 266 targetpages = bufpages - RESERVE_PAGES; 267 268 npages = bcstats.dmapages - targetpages; 269 270 /* 271 * Shrinking the cache happens here only if someone has manually 272 * adjusted bufcachepercent - or the pagedaemon has told us 273 * to give back memory *now* - so we give it all back. 274 */ 275 if (bcstats.dmapages > targetpages) 276 (void) bufcache_recover_dmapages(0, bcstats.dmapages - targetpages); 277 bufcache_adjust(); 278 279 /* 280 * Wake up the cleaner if we have lots of dirty pages, 281 * or if we are getting low on buffer cache kva. 282 */ 283 if ((UNCLEAN_PAGES >= hidirtypages) || 284 bcstats.kvaslots_avail <= 2 * RESERVE_SLOTS) 285 wakeup(&bd_req); 286 287 splx(s); 288 } 289 290 /* 291 * Make the buffer cache back off from cachepct. 292 */ 293 int 294 bufbackoff(struct uvm_constraint_range *range, long size) 295 { 296 /* 297 * Back off "size" buffer cache pages. Called by the page 298 * daemon to consume buffer cache pages rather than scanning. 299 * 300 * It returns 0 to the pagedaemon to indicate that it has 301 * succeeded in freeing enough pages. It returns -1 to 302 * indicate that it could not and the pagedaemon should take 303 * other measures. 304 * 305 */ 306 long pdelta, oldbufpages; 307 308 /* 309 * If we will accept high memory for this backoff 310 * try to steal it from the high memory buffer cache. 311 */ 312 if (range->ucr_high > dma_constraint.ucr_high) { 313 struct buf *bp; 314 int64_t start = bcstats.numbufpages, recovered = 0; 315 int s = splbio(); 316 317 while ((recovered < size) && 318 (bp = bufcache_gethighcleanbuf())) { 319 bufcache_take(bp); 320 if (bp->b_vp) { 321 RBT_REMOVE(buf_rb_bufs, 322 &bp->b_vp->v_bufs_tree, bp); 323 brelvp(bp); 324 } 325 buf_put(bp); 326 recovered = start - bcstats.numbufpages; 327 } 328 bufcache_adjust(); 329 splx(s); 330 331 /* If we got enough, return success */ 332 if (recovered >= size) 333 return 0; 334 335 /* 336 * If we needed only memory above DMA, 337 * return failure 338 */ 339 if (range->ucr_low > dma_constraint.ucr_high) 340 return -1; 341 342 /* Otherwise get the rest from DMA */ 343 size -= recovered; 344 } 345 346 /* 347 * XXX Otherwise do the dma memory cache dance. this needs 348 * refactoring later to get rid of 'bufpages' 349 */ 350 351 /* 352 * Back off by at least bufbackpages. If the page daemon gave us 353 * a larger size, back off by that much. 354 */ 355 pdelta = (size > bufbackpages) ? size : bufbackpages; 356 357 if (bufpages <= buflowpages) 358 return(-1); 359 if (bufpages - pdelta < buflowpages) 360 pdelta = bufpages - buflowpages; 361 oldbufpages = bufpages; 362 bufadjust(bufpages - pdelta); 363 if (oldbufpages - bufpages < size) 364 return (-1); /* we did not free what we were asked */ 365 else 366 return(0); 367 } 368 369 370 /* 371 * Opportunistically flip a buffer into high memory. Will move the buffer 372 * if memory is available without sleeping, and return 0, otherwise will 373 * fail and return -1 with the buffer unchanged. 374 */ 375 376 int 377 buf_flip_high(struct buf *bp) 378 { 379 int s; 380 int ret = -1; 381 382 KASSERT(ISSET(bp->b_flags, B_BC)); 383 KASSERT(ISSET(bp->b_flags, B_DMA)); 384 KASSERT(bp->cache == DMA_CACHE); 385 KASSERT(fliphigh); 386 387 /* Attempt to move the buffer to high memory if we can */ 388 s = splbio(); 389 if (buf_realloc_pages(bp, &high_constraint, UVM_PLA_NOWAIT) == 0) { 390 KASSERT(!ISSET(bp->b_flags, B_DMA)); 391 bcstats.highflips++; 392 ret = 0; 393 } else 394 bcstats.highflops++; 395 splx(s); 396 397 return ret; 398 } 399 400 /* 401 * Flip a buffer to dma reachable memory, when we need it there for 402 * I/O. This can sleep since it will wait for memory allocation in the 403 * DMA reachable area since we have to have the buffer there to proceed. 404 */ 405 void 406 buf_flip_dma(struct buf *bp) 407 { 408 KASSERT(ISSET(bp->b_flags, B_BC)); 409 KASSERT(ISSET(bp->b_flags, B_BUSY)); 410 KASSERT(bp->cache < NUM_CACHES); 411 412 if (!ISSET(bp->b_flags, B_DMA)) { 413 int s = splbio(); 414 415 /* move buf to dma reachable memory */ 416 (void) buf_realloc_pages(bp, &dma_constraint, UVM_PLA_WAITOK); 417 KASSERT(ISSET(bp->b_flags, B_DMA)); 418 bcstats.dmaflips++; 419 splx(s); 420 } 421 422 if (bp->cache > DMA_CACHE) { 423 CLR(bp->b_flags, B_COLD); 424 CLR(bp->b_flags, B_WARM); 425 bp->cache = DMA_CACHE; 426 } 427 } 428 429 struct buf * 430 bio_doread(struct vnode *vp, daddr_t blkno, int size, int async) 431 { 432 struct buf *bp; 433 struct mount *mp; 434 435 bp = getblk(vp, blkno, size, 0, INFSLP); 436 437 /* 438 * If buffer does not have valid data, start a read. 439 * Note that if buffer is B_INVAL, getblk() won't return it. 440 * Therefore, it's valid if its I/O has completed or been delayed. 441 */ 442 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { 443 SET(bp->b_flags, B_READ | async); 444 bcstats.pendingreads++; 445 bcstats.numreads++; 446 VOP_STRATEGY(bp); 447 /* Pay for the read. */ 448 curproc->p_ru.ru_inblock++; /* XXX */ 449 } else if (async) { 450 brelse(bp); 451 } 452 453 mp = vp->v_type == VBLK ? vp->v_specmountpoint : vp->v_mount; 454 455 /* 456 * Collect statistics on synchronous and asynchronous reads. 457 * Reads from block devices are charged to their associated 458 * filesystem (if any). 459 */ 460 if (mp != NULL) { 461 if (async == 0) 462 mp->mnt_stat.f_syncreads++; 463 else 464 mp->mnt_stat.f_asyncreads++; 465 } 466 467 return (bp); 468 } 469 470 /* 471 * Read a disk block. 472 * This algorithm described in Bach (p.54). 473 */ 474 int 475 bread(struct vnode *vp, daddr_t blkno, int size, struct buf **bpp) 476 { 477 struct buf *bp; 478 479 /* Get buffer for block. */ 480 bp = *bpp = bio_doread(vp, blkno, size, 0); 481 482 /* Wait for the read to complete, and return result. */ 483 return (biowait(bp)); 484 } 485 486 /* 487 * Read-ahead multiple disk blocks. The first is sync, the rest async. 488 * Trivial modification to the breada algorithm presented in Bach (p.55). 489 */ 490 int 491 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t rablks[], 492 int rasizes[], int nrablks, struct buf **bpp) 493 { 494 struct buf *bp; 495 int i; 496 497 bp = *bpp = bio_doread(vp, blkno, size, 0); 498 499 /* 500 * For each of the read-ahead blocks, start a read, if necessary. 501 */ 502 for (i = 0; i < nrablks; i++) { 503 /* If it's in the cache, just go on to next one. */ 504 if (incore(vp, rablks[i])) 505 continue; 506 507 /* Get a buffer for the read-ahead block */ 508 (void) bio_doread(vp, rablks[i], rasizes[i], B_ASYNC); 509 } 510 511 /* Otherwise, we had to start a read for it; wait until it's valid. */ 512 return (biowait(bp)); 513 } 514 515 /* 516 * Called from interrupt context. 517 */ 518 void 519 bread_cluster_callback(struct buf *bp) 520 { 521 struct buf **xbpp = bp->b_saveaddr; 522 int i; 523 524 if (xbpp[1] != NULL) { 525 size_t newsize = xbpp[1]->b_bufsize; 526 527 /* 528 * Shrink this buffer's mapping to only cover its part of 529 * the total I/O. 530 */ 531 buf_fix_mapping(bp, newsize); 532 bp->b_bcount = newsize; 533 } 534 535 /* Invalidate read-ahead buffers if read short */ 536 if (bp->b_resid > 0) { 537 for (i = 1; xbpp[i] != NULL; i++) 538 continue; 539 for (i = i - 1; i != 0; i--) { 540 if (xbpp[i]->b_bufsize <= bp->b_resid) { 541 bp->b_resid -= xbpp[i]->b_bufsize; 542 SET(xbpp[i]->b_flags, B_INVAL); 543 } else if (bp->b_resid > 0) { 544 bp->b_resid = 0; 545 SET(xbpp[i]->b_flags, B_INVAL); 546 } else 547 break; 548 } 549 } 550 551 for (i = 1; xbpp[i] != NULL; i++) { 552 if (ISSET(bp->b_flags, B_ERROR)) 553 SET(xbpp[i]->b_flags, B_INVAL | B_ERROR); 554 /* 555 * Move the pages from the master buffer's uvm object 556 * into the individual buffer's uvm objects. 557 */ 558 struct uvm_object *newobj = &xbpp[i]->b_uobj; 559 struct uvm_object *oldobj = &bp->b_uobj; 560 int page; 561 562 uvm_objinit(newobj, NULL, 1); 563 for (page = 0; page < atop(xbpp[i]->b_bufsize); page++) { 564 struct vm_page *pg = uvm_pagelookup(oldobj, 565 xbpp[i]->b_poffs + ptoa(page)); 566 KASSERT(pg != NULL); 567 KASSERT(pg->wire_count == 1); 568 uvm_pagerealloc(pg, newobj, xbpp[i]->b_poffs + ptoa(page)); 569 } 570 xbpp[i]->b_pobj = newobj; 571 572 biodone(xbpp[i]); 573 } 574 575 free(xbpp, M_TEMP, (i + 1) * sizeof(*xbpp)); 576 577 if (ISSET(bp->b_flags, B_ASYNC)) { 578 brelse(bp); 579 } else { 580 CLR(bp->b_flags, B_WANTED); 581 wakeup(bp); 582 } 583 } 584 585 /* 586 * Read-ahead multiple disk blocks, but make sure only one (big) I/O 587 * request is sent to the disk. 588 * XXX This should probably be dropped and breadn should instead be optimized 589 * XXX to do fewer I/O requests. 590 */ 591 int 592 bread_cluster(struct vnode *vp, daddr_t blkno, int size, struct buf **rbpp) 593 { 594 struct buf *bp, **xbpp; 595 int howmany, maxra, i, inc; 596 daddr_t sblkno; 597 598 *rbpp = bio_doread(vp, blkno, size, 0); 599 600 /* 601 * If the buffer is in the cache skip any I/O operation. 602 */ 603 if (ISSET((*rbpp)->b_flags, B_CACHE)) 604 goto out; 605 606 if (size != round_page(size)) 607 goto out; 608 609 if (VOP_BMAP(vp, blkno + 1, NULL, &sblkno, &maxra)) 610 goto out; 611 612 maxra++; 613 if (sblkno == -1 || maxra < 2) 614 goto out; 615 616 howmany = MAXPHYS / size; 617 if (howmany > maxra) 618 howmany = maxra; 619 620 xbpp = mallocarray(howmany + 1, sizeof(*xbpp), M_TEMP, M_NOWAIT); 621 if (xbpp == NULL) 622 goto out; 623 624 for (i = howmany - 1; i >= 0; i--) { 625 size_t sz; 626 627 /* 628 * First buffer allocates big enough size to cover what 629 * all the other buffers need. 630 */ 631 sz = i == 0 ? howmany * size : 0; 632 633 xbpp[i] = buf_get(vp, blkno + i + 1, sz); 634 if (xbpp[i] == NULL) { 635 for (++i; i < howmany; i++) { 636 SET(xbpp[i]->b_flags, B_INVAL); 637 brelse(xbpp[i]); 638 } 639 free(xbpp, M_TEMP, (howmany + 1) * sizeof(*xbpp)); 640 goto out; 641 } 642 } 643 644 bp = xbpp[0]; 645 646 xbpp[howmany] = NULL; 647 648 inc = btodb(size); 649 650 for (i = 1; i < howmany; i++) { 651 bcstats.pendingreads++; 652 bcstats.numreads++; 653 /* 654 * We set B_DMA here because bp above will be B_DMA, 655 * and we are playing buffer slice-n-dice games from 656 * the memory allocated in bp. 657 */ 658 SET(xbpp[i]->b_flags, B_DMA | B_READ | B_ASYNC); 659 xbpp[i]->b_blkno = sblkno + (i * inc); 660 xbpp[i]->b_bufsize = xbpp[i]->b_bcount = size; 661 xbpp[i]->b_data = NULL; 662 xbpp[i]->b_pobj = bp->b_pobj; 663 xbpp[i]->b_poffs = bp->b_poffs + (i * size); 664 } 665 666 KASSERT(bp->b_lblkno == blkno + 1); 667 KASSERT(bp->b_vp == vp); 668 669 bp->b_blkno = sblkno; 670 SET(bp->b_flags, B_READ | B_ASYNC | B_CALL); 671 672 bp->b_saveaddr = (void *)xbpp; 673 bp->b_iodone = bread_cluster_callback; 674 675 bcstats.pendingreads++; 676 bcstats.numreads++; 677 VOP_STRATEGY(bp); 678 curproc->p_ru.ru_inblock++; 679 680 out: 681 return (biowait(*rbpp)); 682 } 683 684 /* 685 * Block write. Described in Bach (p.56) 686 */ 687 int 688 bwrite(struct buf *bp) 689 { 690 int rv, async, wasdelayed, s; 691 struct vnode *vp; 692 struct mount *mp; 693 694 vp = bp->b_vp; 695 if (vp != NULL) 696 mp = vp->v_type == VBLK? vp->v_specmountpoint : vp->v_mount; 697 else 698 mp = NULL; 699 700 /* 701 * Remember buffer type, to switch on it later. If the write was 702 * synchronous, but the file system was mounted with MNT_ASYNC, 703 * convert it to a delayed write. 704 * XXX note that this relies on delayed tape writes being converted 705 * to async, not sync writes (which is safe, but ugly). 706 */ 707 async = ISSET(bp->b_flags, B_ASYNC); 708 if (!async && mp && ISSET(mp->mnt_flag, MNT_ASYNC)) { 709 /* 710 * Don't convert writes from VND on async filesystems 711 * that already have delayed writes in the upper layer. 712 */ 713 if (!ISSET(bp->b_flags, B_NOCACHE)) { 714 bdwrite(bp); 715 return (0); 716 } 717 } 718 719 /* 720 * Collect statistics on synchronous and asynchronous writes. 721 * Writes to block devices are charged to their associated 722 * filesystem (if any). 723 */ 724 if (mp != NULL) { 725 if (async) 726 mp->mnt_stat.f_asyncwrites++; 727 else 728 mp->mnt_stat.f_syncwrites++; 729 } 730 bcstats.pendingwrites++; 731 bcstats.numwrites++; 732 733 wasdelayed = ISSET(bp->b_flags, B_DELWRI); 734 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI)); 735 736 s = splbio(); 737 738 /* 739 * If not synchronous, pay for the I/O operation and make 740 * sure the buf is on the correct vnode queue. We have 741 * to do this now, because if we don't, the vnode may not 742 * be properly notified that its I/O has completed. 743 */ 744 if (wasdelayed) { 745 reassignbuf(bp); 746 } else 747 curproc->p_ru.ru_oublock++; 748 749 750 /* Initiate disk write. Make sure the appropriate party is charged. */ 751 bp->b_vp->v_numoutput++; 752 splx(s); 753 buf_flip_dma(bp); 754 SET(bp->b_flags, B_WRITEINPROG); 755 VOP_STRATEGY(bp); 756 757 /* 758 * If the queue is above the high water mark, wait till 759 * the number of outstanding write bufs drops below the low 760 * water mark. 761 */ 762 if (bp->b_bq) 763 bufq_wait(bp->b_bq); 764 765 if (async) 766 return (0); 767 768 /* 769 * If I/O was synchronous, wait for it to complete. 770 */ 771 rv = biowait(bp); 772 773 /* Release the buffer. */ 774 brelse(bp); 775 776 return (rv); 777 } 778 779 780 /* 781 * Delayed write. 782 * 783 * The buffer is marked dirty, but is not queued for I/O. 784 * This routine should be used when the buffer is expected 785 * to be modified again soon, typically a small write that 786 * partially fills a buffer. 787 * 788 * NB: magnetic tapes cannot be delayed; they must be 789 * written in the order that the writes are requested. 790 * 791 * Described in Leffler, et al. (pp. 208-213). 792 */ 793 void 794 bdwrite(struct buf *bp) 795 { 796 int s; 797 798 /* 799 * If the block hasn't been seen before: 800 * (1) Mark it as having been seen, 801 * (2) Charge for the write. 802 * (3) Make sure it's on its vnode's correct block list, 803 * (4) If a buffer is rewritten, move it to end of dirty list 804 */ 805 if (!ISSET(bp->b_flags, B_DELWRI)) { 806 SET(bp->b_flags, B_DELWRI); 807 s = splbio(); 808 buf_flip_dma(bp); 809 reassignbuf(bp); 810 splx(s); 811 curproc->p_ru.ru_oublock++; /* XXX */ 812 } 813 814 /* The "write" is done, so mark and release the buffer. */ 815 CLR(bp->b_flags, B_NEEDCOMMIT); 816 CLR(bp->b_flags, B_NOCACHE); /* Must cache delayed writes */ 817 SET(bp->b_flags, B_DONE); 818 brelse(bp); 819 } 820 821 /* 822 * Asynchronous block write; just an asynchronous bwrite(). 823 */ 824 void 825 bawrite(struct buf *bp) 826 { 827 828 SET(bp->b_flags, B_ASYNC); 829 VOP_BWRITE(bp); 830 } 831 832 /* 833 * Must be called at splbio() 834 */ 835 void 836 buf_dirty(struct buf *bp) 837 { 838 splassert(IPL_BIO); 839 840 #ifdef DIAGNOSTIC 841 if (!ISSET(bp->b_flags, B_BUSY)) 842 panic("Trying to dirty buffer on freelist!"); 843 #endif 844 845 if (ISSET(bp->b_flags, B_DELWRI) == 0) { 846 SET(bp->b_flags, B_DELWRI); 847 buf_flip_dma(bp); 848 reassignbuf(bp); 849 } 850 } 851 852 /* 853 * Must be called at splbio() 854 */ 855 void 856 buf_undirty(struct buf *bp) 857 { 858 splassert(IPL_BIO); 859 860 #ifdef DIAGNOSTIC 861 if (!ISSET(bp->b_flags, B_BUSY)) 862 panic("Trying to undirty buffer on freelist!"); 863 #endif 864 if (ISSET(bp->b_flags, B_DELWRI)) { 865 CLR(bp->b_flags, B_DELWRI); 866 reassignbuf(bp); 867 } 868 } 869 870 /* 871 * Release a buffer on to the free lists. 872 * Described in Bach (p. 46). 873 */ 874 void 875 brelse(struct buf *bp) 876 { 877 int s; 878 879 s = splbio(); 880 881 if (bp->b_data != NULL) 882 KASSERT(bp->b_bufsize > 0); 883 884 /* 885 * softdep is basically incompatible with not cacheing buffers 886 * that have dependencies, so this buffer must be cached 887 */ 888 if (LIST_FIRST(&bp->b_dep) != NULL) 889 CLR(bp->b_flags, B_NOCACHE); 890 891 /* 892 * Determine which queue the buffer should be on, then put it there. 893 */ 894 895 /* If it's not cacheable, or an error, mark it invalid. */ 896 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR))) 897 SET(bp->b_flags, B_INVAL); 898 /* If it's a write error, also mark the vnode as damaged. */ 899 if (ISSET(bp->b_flags, B_ERROR) && !ISSET(bp->b_flags, B_READ)) { 900 if (bp->b_vp && bp->b_vp->v_type == VREG) 901 SET(bp->b_vp->v_bioflag, VBIOERROR); 902 } 903 904 if (ISSET(bp->b_flags, B_INVAL)) { 905 /* 906 * If the buffer is invalid, free it now rather than leaving 907 * it in a queue and wasting memory. 908 */ 909 if (LIST_FIRST(&bp->b_dep) != NULL) 910 buf_deallocate(bp); 911 912 if (ISSET(bp->b_flags, B_DELWRI)) { 913 CLR(bp->b_flags, B_DELWRI); 914 } 915 916 if (bp->b_vp) { 917 RBT_REMOVE(buf_rb_bufs, &bp->b_vp->v_bufs_tree, bp); 918 brelvp(bp); 919 } 920 bp->b_vp = NULL; 921 922 /* 923 * Wake up any processes waiting for _this_ buffer to 924 * become free. They are not allowed to grab it 925 * since it will be freed. But the only sleeper is 926 * getblk and it will restart the operation after 927 * sleep. 928 */ 929 if (ISSET(bp->b_flags, B_WANTED)) { 930 CLR(bp->b_flags, B_WANTED); 931 wakeup(bp); 932 } 933 buf_put(bp); 934 } else { 935 /* 936 * It has valid data. Put it on the end of the appropriate 937 * queue, so that it'll stick around for as long as possible. 938 */ 939 bufcache_release(bp); 940 941 /* Unlock the buffer. */ 942 CLR(bp->b_flags, (B_AGE | B_ASYNC | B_NOCACHE | B_DEFERRED)); 943 buf_release(bp); 944 945 /* Wake up any processes waiting for _this_ buffer to 946 * become free. */ 947 if (ISSET(bp->b_flags, B_WANTED)) { 948 CLR(bp->b_flags, B_WANTED); 949 wakeup(bp); 950 } 951 } 952 953 /* Wake up syncer and cleaner processes waiting for buffers. */ 954 if (nobuffers) { 955 nobuffers = 0; 956 wakeup(&nobuffers); 957 } 958 959 /* Wake up any processes waiting for any buffer to become free. */ 960 if (needbuffer && bcstats.dmapages < targetpages && 961 bcstats.kvaslots_avail > RESERVE_SLOTS) { 962 needbuffer = 0; 963 wakeup(&needbuffer); 964 } 965 966 splx(s); 967 } 968 969 /* 970 * Determine if a block is in the cache. Just look on what would be its hash 971 * chain. If it's there, return a pointer to it, unless it's marked invalid. 972 */ 973 struct buf * 974 incore(struct vnode *vp, daddr_t blkno) 975 { 976 struct buf *bp; 977 struct buf b; 978 int s; 979 980 s = splbio(); 981 982 /* Search buf lookup tree */ 983 b.b_lblkno = blkno; 984 bp = RBT_FIND(buf_rb_bufs, &vp->v_bufs_tree, &b); 985 if (bp != NULL && ISSET(bp->b_flags, B_INVAL)) 986 bp = NULL; 987 988 splx(s); 989 return (bp); 990 } 991 992 /* 993 * Get a block of requested size that is associated with 994 * a given vnode and block offset. If it is found in the 995 * block cache, mark it as having been found, make it busy 996 * and return it. Otherwise, return an empty block of the 997 * correct size. It is up to the caller to ensure that the 998 * cached blocks be of the correct size. 999 */ 1000 struct buf * 1001 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, 1002 uint64_t slptimeo) 1003 { 1004 struct buf *bp; 1005 struct buf b; 1006 int s, error; 1007 1008 /* 1009 * XXX 1010 * The following is an inlined version of 'incore()', but with 1011 * the 'invalid' test moved to after the 'busy' test. It's 1012 * necessary because there are some cases in which the NFS 1013 * code sets B_INVAL prior to writing data to the server, but 1014 * in which the buffers actually contain valid data. In this 1015 * case, we can't allow the system to allocate a new buffer for 1016 * the block until the write is finished. 1017 */ 1018 start: 1019 s = splbio(); 1020 b.b_lblkno = blkno; 1021 bp = RBT_FIND(buf_rb_bufs, &vp->v_bufs_tree, &b); 1022 if (bp != NULL) { 1023 if (ISSET(bp->b_flags, B_BUSY)) { 1024 SET(bp->b_flags, B_WANTED); 1025 error = tsleep_nsec(bp, slpflag | (PRIBIO + 1), 1026 "getblk", slptimeo); 1027 splx(s); 1028 if (error) 1029 return (NULL); 1030 goto start; 1031 } 1032 1033 if (!ISSET(bp->b_flags, B_INVAL)) { 1034 bcstats.cachehits++; 1035 SET(bp->b_flags, B_CACHE); 1036 bufcache_take(bp); 1037 buf_acquire(bp); 1038 splx(s); 1039 return (bp); 1040 } 1041 } 1042 splx(s); 1043 1044 if ((bp = buf_get(vp, blkno, size)) == NULL) 1045 goto start; 1046 1047 return (bp); 1048 } 1049 1050 /* 1051 * Get an empty, disassociated buffer of given size. 1052 */ 1053 struct buf * 1054 geteblk(size_t size) 1055 { 1056 struct buf *bp; 1057 1058 while ((bp = buf_get(NULL, 0, size)) == NULL) 1059 continue; 1060 1061 return (bp); 1062 } 1063 1064 /* 1065 * Allocate a buffer. 1066 * If vp is given, put it into the buffer cache for that vnode. 1067 * If size != 0, allocate memory and call buf_map(). 1068 * If there is already a buffer for the given vnode/blkno, return NULL. 1069 */ 1070 struct buf * 1071 buf_get(struct vnode *vp, daddr_t blkno, size_t size) 1072 { 1073 struct buf *bp; 1074 int poolwait = size == 0 ? PR_NOWAIT : PR_WAITOK; 1075 int npages; 1076 int s; 1077 1078 s = splbio(); 1079 if (size) { 1080 /* 1081 * Wake up the cleaner if we have lots of dirty pages, 1082 * or if we are getting low on buffer cache kva. 1083 */ 1084 if (UNCLEAN_PAGES >= hidirtypages || 1085 bcstats.kvaslots_avail <= 2 * RESERVE_SLOTS) 1086 wakeup(&bd_req); 1087 1088 npages = atop(round_page(size)); 1089 1090 /* 1091 * if our cache has been previously shrunk, 1092 * allow it to grow again with use up to 1093 * bufhighpages (cachepercent) 1094 */ 1095 if (bufpages < bufhighpages) 1096 bufadjust(bufhighpages); 1097 1098 /* 1099 * If we would go over the page target with our 1100 * new allocation, free enough buffers first 1101 * to stay at the target with our new allocation. 1102 */ 1103 if (bcstats.dmapages + npages > targetpages) { 1104 (void) bufcache_recover_dmapages(0, npages); 1105 bufcache_adjust(); 1106 } 1107 1108 /* 1109 * If we get here, we tried to free the world down 1110 * above, and couldn't get down - Wake the cleaner 1111 * and wait for it to push some buffers out. 1112 */ 1113 if ((bcstats.dmapages + npages > targetpages || 1114 bcstats.kvaslots_avail <= RESERVE_SLOTS) && 1115 curproc != syncerproc && curproc != cleanerproc) { 1116 wakeup(&bd_req); 1117 needbuffer++; 1118 tsleep_nsec(&needbuffer, PRIBIO, "needbuffer", INFSLP); 1119 splx(s); 1120 return (NULL); 1121 } 1122 if (bcstats.dmapages + npages > bufpages) { 1123 /* cleaner or syncer */ 1124 nobuffers = 1; 1125 tsleep_nsec(&nobuffers, PRIBIO, "nobuffers", INFSLP); 1126 splx(s); 1127 return (NULL); 1128 } 1129 } 1130 1131 bp = pool_get(&bufpool, poolwait|PR_ZERO); 1132 1133 if (bp == NULL) { 1134 splx(s); 1135 return (NULL); 1136 } 1137 1138 bp->b_freelist.tqe_next = NOLIST; 1139 bp->b_dev = NODEV; 1140 LIST_INIT(&bp->b_dep); 1141 bp->b_bcount = size; 1142 1143 buf_acquire_nomap(bp); 1144 1145 if (vp != NULL) { 1146 /* 1147 * We insert the buffer into the hash with B_BUSY set 1148 * while we allocate pages for it. This way any getblk 1149 * that happens while we allocate pages will wait for 1150 * this buffer instead of starting its own buf_get. 1151 * 1152 * But first, we check if someone beat us to it. 1153 */ 1154 if (incore(vp, blkno)) { 1155 pool_put(&bufpool, bp); 1156 splx(s); 1157 return (NULL); 1158 } 1159 1160 bp->b_blkno = bp->b_lblkno = blkno; 1161 bgetvp(vp, bp); 1162 if (RBT_INSERT(buf_rb_bufs, &vp->v_bufs_tree, bp)) 1163 panic("buf_get: dup lblk vp %p bp %p", vp, bp); 1164 } else { 1165 bp->b_vnbufs.le_next = NOLIST; 1166 SET(bp->b_flags, B_INVAL); 1167 bp->b_vp = NULL; 1168 } 1169 1170 LIST_INSERT_HEAD(&bufhead, bp, b_list); 1171 bcstats.numbufs++; 1172 1173 if (size) { 1174 buf_alloc_pages(bp, round_page(size)); 1175 KASSERT(ISSET(bp->b_flags, B_DMA)); 1176 buf_map(bp); 1177 } 1178 1179 SET(bp->b_flags, B_BC); 1180 splx(s); 1181 1182 return (bp); 1183 } 1184 1185 /* 1186 * Buffer cleaning daemon. 1187 */ 1188 void 1189 buf_daemon(void *arg) 1190 { 1191 struct buf *bp = NULL; 1192 int s, pushed = 0; 1193 1194 s = splbio(); 1195 for (;;) { 1196 if (bp == NULL || (pushed >= 16 && 1197 UNCLEAN_PAGES < hidirtypages && 1198 bcstats.kvaslots_avail > 2 * RESERVE_SLOTS)){ 1199 pushed = 0; 1200 /* 1201 * Wake up anyone who was waiting for buffers 1202 * to be released. 1203 */ 1204 if (needbuffer) { 1205 needbuffer = 0; 1206 wakeup(&needbuffer); 1207 } 1208 tsleep_nsec(&bd_req, PRIBIO - 7, "cleaner", INFSLP); 1209 } 1210 1211 while ((bp = bufcache_getdirtybuf())) { 1212 if (UNCLEAN_PAGES < lodirtypages && 1213 bcstats.kvaslots_avail > 2 * RESERVE_SLOTS && 1214 pushed >= 16) 1215 break; 1216 1217 bufcache_take(bp); 1218 buf_acquire(bp); 1219 splx(s); 1220 1221 if (ISSET(bp->b_flags, B_INVAL)) { 1222 brelse(bp); 1223 s = splbio(); 1224 continue; 1225 } 1226 #ifdef DIAGNOSTIC 1227 if (!ISSET(bp->b_flags, B_DELWRI)) 1228 panic("Clean buffer on dirty queue"); 1229 #endif 1230 if (LIST_FIRST(&bp->b_dep) != NULL && 1231 !ISSET(bp->b_flags, B_DEFERRED) && 1232 buf_countdeps(bp, 0, 0)) { 1233 SET(bp->b_flags, B_DEFERRED); 1234 s = splbio(); 1235 bufcache_release(bp); 1236 buf_release(bp); 1237 continue; 1238 } 1239 1240 bawrite(bp); 1241 pushed++; 1242 1243 sched_pause(yield); 1244 1245 s = splbio(); 1246 } 1247 } 1248 } 1249 1250 /* 1251 * Wait for operations on the buffer to complete. 1252 * When they do, extract and return the I/O's error value. 1253 */ 1254 int 1255 biowait(struct buf *bp) 1256 { 1257 int s; 1258 1259 KASSERT(!(bp->b_flags & B_ASYNC)); 1260 1261 s = splbio(); 1262 while (!ISSET(bp->b_flags, B_DONE)) 1263 tsleep_nsec(bp, PRIBIO + 1, "biowait", INFSLP); 1264 splx(s); 1265 1266 /* check for interruption of I/O (e.g. via NFS), then errors. */ 1267 if (ISSET(bp->b_flags, B_EINTR)) { 1268 CLR(bp->b_flags, B_EINTR); 1269 return (EINTR); 1270 } 1271 1272 if (ISSET(bp->b_flags, B_ERROR)) 1273 return (bp->b_error ? bp->b_error : EIO); 1274 else 1275 return (0); 1276 } 1277 1278 /* 1279 * Mark I/O complete on a buffer. 1280 * 1281 * If a callback has been requested, e.g. the pageout 1282 * daemon, do so. Otherwise, awaken waiting processes. 1283 * 1284 * [ Leffler, et al., says on p.247: 1285 * "This routine wakes up the blocked process, frees the buffer 1286 * for an asynchronous write, or, for a request by the pagedaemon 1287 * process, invokes a procedure specified in the buffer structure" ] 1288 * 1289 * In real life, the pagedaemon (or other system processes) wants 1290 * to do async stuff to, and doesn't want the buffer brelse()'d. 1291 * (for swap pager, that puts swap buffers on the free lists (!!!), 1292 * for the vn device, that puts malloc'd buffers on the free lists!) 1293 * 1294 * Must be called at splbio(). 1295 */ 1296 void 1297 biodone(struct buf *bp) 1298 { 1299 splassert(IPL_BIO); 1300 1301 if (ISSET(bp->b_flags, B_DONE)) 1302 panic("biodone already"); 1303 SET(bp->b_flags, B_DONE); /* note that it's done */ 1304 1305 if (bp->b_bq) 1306 bufq_done(bp->b_bq, bp); 1307 1308 if (LIST_FIRST(&bp->b_dep) != NULL) 1309 buf_complete(bp); 1310 1311 if (!ISSET(bp->b_flags, B_READ)) { 1312 CLR(bp->b_flags, B_WRITEINPROG); 1313 vwakeup(bp->b_vp); 1314 } 1315 if (bcstats.numbufs && 1316 (!(ISSET(bp->b_flags, B_RAW) || ISSET(bp->b_flags, B_PHYS)))) { 1317 if (!ISSET(bp->b_flags, B_READ)) { 1318 bcstats.pendingwrites--; 1319 } else 1320 bcstats.pendingreads--; 1321 } 1322 if (ISSET(bp->b_flags, B_CALL)) { /* if necessary, call out */ 1323 CLR(bp->b_flags, B_CALL); /* but note callout done */ 1324 (*bp->b_iodone)(bp); 1325 } else { 1326 if (ISSET(bp->b_flags, B_ASYNC)) {/* if async, release it */ 1327 brelse(bp); 1328 } else { /* or just wakeup the buffer */ 1329 CLR(bp->b_flags, B_WANTED); 1330 wakeup(bp); 1331 } 1332 } 1333 } 1334 1335 #ifdef DDB 1336 void bcstats_print(int (*)(const char *, ...) 1337 __attribute__((__format__(__kprintf__,1,2)))); 1338 /* 1339 * bcstats_print: ddb hook to print interesting buffer cache counters 1340 */ 1341 void 1342 bcstats_print( 1343 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2)))) 1344 { 1345 (*pr)("Current Buffer Cache status:\n"); 1346 (*pr)("numbufs %lld busymapped %lld, delwri %lld\n", 1347 bcstats.numbufs, bcstats.busymapped, bcstats.delwribufs); 1348 (*pr)("kvaslots %lld avail kva slots %lld\n", 1349 bcstats.kvaslots, bcstats.kvaslots_avail); 1350 (*pr)("bufpages %lld, dmapages %lld, dirtypages %lld\n", 1351 bcstats.numbufpages, bcstats.dmapages, bcstats.numdirtypages); 1352 (*pr)("pendingreads %lld, pendingwrites %lld\n", 1353 bcstats.pendingreads, bcstats.pendingwrites); 1354 (*pr)("highflips %lld, highflops %lld, dmaflips %lld\n", 1355 bcstats.highflips, bcstats.highflops, bcstats.dmaflips); 1356 } 1357 #endif 1358 1359 void 1360 buf_adjcnt(struct buf *bp, long ncount) 1361 { 1362 KASSERT(ncount <= bp->b_bufsize); 1363 bp->b_bcount = ncount; 1364 } 1365 1366 /* bufcache freelist code below */ 1367 /* 1368 * Copyright (c) 2014 Ted Unangst <tedu@openbsd.org> 1369 * 1370 * Permission to use, copy, modify, and distribute this software for any 1371 * purpose with or without fee is hereby granted, provided that the above 1372 * copyright notice and this permission notice appear in all copies. 1373 * 1374 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 1375 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 1376 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 1377 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 1378 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 1379 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 1380 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 1381 */ 1382 1383 /* 1384 * The code below implements a variant of the 2Q buffer cache algorithm by 1385 * Johnson and Shasha. 1386 * 1387 * General Outline 1388 * We divide the buffer cache into three working sets: current, previous, 1389 * and long term. Each list is itself LRU and buffers get promoted and moved 1390 * around between them. A buffer starts its life in the current working set. 1391 * As time passes and newer buffers push it out, it will turn into the previous 1392 * working set and is subject to recycling. But if it's accessed again from 1393 * the previous working set, that's an indication that it's actually in the 1394 * long term working set, so we promote it there. The separation of current 1395 * and previous working sets prevents us from promoting a buffer that's only 1396 * temporarily hot to the long term cache. 1397 * 1398 * The objective is to provide scan resistance by making the long term 1399 * working set ineligible for immediate recycling, even as the current 1400 * working set is rapidly turned over. 1401 * 1402 * Implementation 1403 * The code below identifies the current, previous, and long term sets as 1404 * hotqueue, coldqueue, and warmqueue. The hot and warm queues are capped at 1405 * 1/3 of the total clean pages, after which point they start pushing their 1406 * oldest buffers into coldqueue. 1407 * A buf always starts out with neither WARM or COLD flags set (implying HOT). 1408 * When released, it will be returned to the tail of the hotqueue list. 1409 * When the hotqueue gets too large, the oldest hot buf will be moved to the 1410 * coldqueue, with the B_COLD flag set. When a cold buf is released, we set 1411 * the B_WARM flag and put it onto the warmqueue. Warm bufs are also 1412 * directly returned to the end of the warmqueue. As with the hotqueue, when 1413 * the warmqueue grows too large, B_WARM bufs are moved onto the coldqueue. 1414 * 1415 * Note that this design does still support large working sets, greater 1416 * than the cap of hotqueue or warmqueue would imply. The coldqueue is still 1417 * cached and has no maximum length. The hot and warm queues form a Y feeding 1418 * into the coldqueue. Moving bufs between queues is constant time, so this 1419 * design decays to one long warm->cold queue. 1420 * 1421 * In the 2Q paper, hotqueue and coldqueue are A1in and A1out. The warmqueue 1422 * is Am. We always cache pages, as opposed to pointers to pages for A1. 1423 * 1424 * This implementation adds support for multiple 2q caches. 1425 * 1426 * If we have more than one 2q cache, as bufs fall off the cold queue 1427 * for recyclying, bufs that have been warm before (which retain the 1428 * B_WARM flag in addition to B_COLD) can be put into the hot queue of 1429 * a second level 2Q cache. buffers which are only B_COLD are 1430 * recycled. Bufs falling off the last cache's cold queue are always 1431 * recycled. 1432 * 1433 */ 1434 1435 /* 1436 * this function is called when a hot or warm queue may have exceeded its 1437 * size limit. it will move a buf to the coldqueue. 1438 */ 1439 int chillbufs(struct 1440 bufcache *cache, struct bufqueue *queue, int64_t *queuepages); 1441 1442 void 1443 bufcache_init(void) 1444 { 1445 int i; 1446 1447 for (i = 0; i < NUM_CACHES; i++) { 1448 TAILQ_INIT(&cleancache[i].hotqueue); 1449 TAILQ_INIT(&cleancache[i].coldqueue); 1450 TAILQ_INIT(&cleancache[i].warmqueue); 1451 } 1452 TAILQ_INIT(&dirtyqueue); 1453 } 1454 1455 /* 1456 * if the buffer caches have shrunk, we may need to rebalance our queues. 1457 */ 1458 void 1459 bufcache_adjust(void) 1460 { 1461 int i; 1462 1463 for (i = 0; i < NUM_CACHES; i++) { 1464 while (chillbufs(&cleancache[i], &cleancache[i].warmqueue, 1465 &cleancache[i].warmbufpages) || 1466 chillbufs(&cleancache[i], &cleancache[i].hotqueue, 1467 &cleancache[i].hotbufpages)) 1468 continue; 1469 } 1470 } 1471 1472 /* 1473 * Get a clean buffer from the cache. if "discard" is set do not promote 1474 * previously warm buffers as normal, because we are tossing everything 1475 * away such as in a hibernation 1476 */ 1477 struct buf * 1478 bufcache_getcleanbuf(int cachenum, int discard) 1479 { 1480 struct buf *bp = NULL; 1481 struct bufcache *cache = &cleancache[cachenum]; 1482 struct bufqueue * queue; 1483 1484 splassert(IPL_BIO); 1485 1486 /* try cold queue */ 1487 while ((bp = TAILQ_FIRST(&cache->coldqueue)) || 1488 (bp = TAILQ_FIRST(&cache->warmqueue)) || 1489 (bp = TAILQ_FIRST(&cache->hotqueue))) { 1490 int64_t pages = atop(bp->b_bufsize); 1491 struct bufcache *newcache; 1492 1493 if (discard || cachenum >= NUM_CACHES - 1) { 1494 /* Victim selected, give it up */ 1495 return bp; 1496 } 1497 KASSERT(bp->cache == cachenum); 1498 1499 /* 1500 * If this buffer was warm before, move it to 1501 * the hot queue in the next cache 1502 */ 1503 1504 if (fliphigh) { 1505 /* 1506 * If we are in the DMA cache, try to flip the 1507 * buffer up high to move it on to the other 1508 * caches. if we can't move the buffer to high 1509 * memory without sleeping, we give it up and 1510 * return it rather than fight for more memory 1511 * against non buffer cache competitors. 1512 */ 1513 SET(bp->b_flags, B_BUSY); 1514 if (bp->cache == 0 && buf_flip_high(bp) == -1) { 1515 CLR(bp->b_flags, B_BUSY); 1516 return bp; 1517 } 1518 CLR(bp->b_flags, B_BUSY); 1519 } 1520 1521 /* Move the buffer to the hot queue in the next cache */ 1522 if (ISSET(bp->b_flags, B_COLD)) { 1523 queue = &cache->coldqueue; 1524 } else if (ISSET(bp->b_flags, B_WARM)) { 1525 queue = &cache->warmqueue; 1526 cache->warmbufpages -= pages; 1527 } else { 1528 queue = &cache->hotqueue; 1529 cache->hotbufpages -= pages; 1530 } 1531 TAILQ_REMOVE(queue, bp, b_freelist); 1532 cache->cachepages -= pages; 1533 CLR(bp->b_flags, B_WARM); 1534 CLR(bp->b_flags, B_COLD); 1535 bp->cache++; 1536 newcache= &cleancache[bp->cache]; 1537 newcache->cachepages += pages; 1538 newcache->hotbufpages += pages; 1539 chillbufs(newcache, &newcache->hotqueue, 1540 &newcache->hotbufpages); 1541 TAILQ_INSERT_TAIL(&newcache->hotqueue, bp, b_freelist); 1542 } 1543 return bp; 1544 } 1545 1546 1547 void 1548 discard_buffer(struct buf *bp) { 1549 bufcache_take(bp); 1550 if (bp->b_vp) { 1551 RBT_REMOVE(buf_rb_bufs, 1552 &bp->b_vp->v_bufs_tree, bp); 1553 brelvp(bp); 1554 } 1555 buf_put(bp); 1556 } 1557 1558 int64_t 1559 bufcache_recover_dmapages(int discard, int64_t howmany) 1560 { 1561 struct buf *bp = NULL; 1562 struct bufcache *cache = &cleancache[DMA_CACHE]; 1563 struct bufqueue * queue; 1564 int64_t recovered = 0; 1565 1566 splassert(IPL_BIO); 1567 1568 while ((recovered < howmany) && 1569 ((bp = TAILQ_FIRST(&cache->coldqueue)) || 1570 (bp = TAILQ_FIRST(&cache->warmqueue)) || 1571 (bp = TAILQ_FIRST(&cache->hotqueue)))) { 1572 int64_t pages = atop(bp->b_bufsize); 1573 struct bufcache *newcache; 1574 1575 if (discard || DMA_CACHE >= NUM_CACHES - 1) { 1576 discard_buffer(bp); 1577 continue; 1578 } 1579 KASSERT(bp->cache == DMA_CACHE); 1580 1581 /* 1582 * If this buffer was warm before, move it to 1583 * the hot queue in the next cache 1584 */ 1585 1586 /* 1587 * One way or another, the pages for this 1588 * buffer are leaving DMA memory 1589 */ 1590 recovered += pages; 1591 1592 if (!fliphigh) { 1593 discard_buffer(bp); 1594 continue; 1595 } 1596 1597 /* 1598 * If we are in the DMA cache, try to flip the 1599 * buffer up high to move it on to the other 1600 * caches. if we can't move the buffer to high 1601 * memory without sleeping, we give it up 1602 * now rather than fight for more memory 1603 * against non buffer cache competitors. 1604 */ 1605 SET(bp->b_flags, B_BUSY); 1606 if (bp->cache == 0 && buf_flip_high(bp) == -1) { 1607 CLR(bp->b_flags, B_BUSY); 1608 discard_buffer(bp); 1609 continue; 1610 } 1611 CLR(bp->b_flags, B_BUSY); 1612 1613 /* 1614 * Move the buffer to the hot queue in the next cache 1615 */ 1616 if (ISSET(bp->b_flags, B_COLD)) { 1617 queue = &cache->coldqueue; 1618 } else if (ISSET(bp->b_flags, B_WARM)) { 1619 queue = &cache->warmqueue; 1620 cache->warmbufpages -= pages; 1621 } else { 1622 queue = &cache->hotqueue; 1623 cache->hotbufpages -= pages; 1624 } 1625 TAILQ_REMOVE(queue, bp, b_freelist); 1626 cache->cachepages -= pages; 1627 CLR(bp->b_flags, B_WARM); 1628 CLR(bp->b_flags, B_COLD); 1629 bp->cache++; 1630 newcache= &cleancache[bp->cache]; 1631 newcache->cachepages += pages; 1632 newcache->hotbufpages += pages; 1633 chillbufs(newcache, &newcache->hotqueue, 1634 &newcache->hotbufpages); 1635 TAILQ_INSERT_TAIL(&newcache->hotqueue, bp, b_freelist); 1636 } 1637 return recovered; 1638 } 1639 1640 struct buf * 1641 bufcache_getcleanbuf_range(int start, int end, int discard) 1642 { 1643 int i, j = start, q = end; 1644 struct buf *bp = NULL; 1645 1646 /* 1647 * XXX in theory we could promote warm buffers into a previous queue 1648 * so in the pathological case of where we go through all the caches 1649 * without getting a buffer we have to start at the beginning again. 1650 */ 1651 while (j <= q) { 1652 for (i = q; i >= j; i--) 1653 if ((bp = bufcache_getcleanbuf(i, discard))) 1654 return (bp); 1655 j++; 1656 } 1657 return bp; 1658 } 1659 1660 struct buf * 1661 bufcache_gethighcleanbuf(void) 1662 { 1663 if (!fliphigh) 1664 return NULL; 1665 return bufcache_getcleanbuf_range(DMA_CACHE + 1, NUM_CACHES - 1, 0); 1666 } 1667 1668 1669 struct buf * 1670 bufcache_getdmacleanbuf(void) 1671 { 1672 if (fliphigh) 1673 return bufcache_getcleanbuf_range(DMA_CACHE, DMA_CACHE, 0); 1674 return bufcache_getcleanbuf_range(DMA_CACHE, NUM_CACHES - 1, 0); 1675 } 1676 1677 1678 struct buf * 1679 bufcache_getdirtybuf(void) 1680 { 1681 return TAILQ_FIRST(&dirtyqueue); 1682 } 1683 1684 void 1685 bufcache_take(struct buf *bp) 1686 { 1687 struct bufqueue *queue; 1688 int64_t pages; 1689 1690 splassert(IPL_BIO); 1691 KASSERT(ISSET(bp->b_flags, B_BC)); 1692 KASSERT(bp->cache >= DMA_CACHE); 1693 KASSERT((bp->cache < NUM_CACHES)); 1694 1695 pages = atop(bp->b_bufsize); 1696 struct bufcache *cache = &cleancache[bp->cache]; 1697 if (!ISSET(bp->b_flags, B_DELWRI)) { 1698 if (ISSET(bp->b_flags, B_COLD)) { 1699 queue = &cache->coldqueue; 1700 } else if (ISSET(bp->b_flags, B_WARM)) { 1701 queue = &cache->warmqueue; 1702 cache->warmbufpages -= pages; 1703 } else { 1704 queue = &cache->hotqueue; 1705 cache->hotbufpages -= pages; 1706 } 1707 bcstats.numcleanpages -= pages; 1708 cache->cachepages -= pages; 1709 } else { 1710 queue = &dirtyqueue; 1711 bcstats.numdirtypages -= pages; 1712 bcstats.delwribufs--; 1713 } 1714 TAILQ_REMOVE(queue, bp, b_freelist); 1715 } 1716 1717 /* move buffers from a hot or warm queue to a cold queue in a cache */ 1718 int 1719 chillbufs(struct bufcache *cache, struct bufqueue *queue, int64_t *queuepages) 1720 { 1721 struct buf *bp; 1722 int64_t limit, pages; 1723 1724 /* 1725 * We limit the hot queue to be small, with a max of 4096 pages. 1726 * We limit the warm queue to half the cache size. 1727 * 1728 * We impose a minimum size of 96 to prevent too much "wobbling". 1729 */ 1730 if (queue == &cache->hotqueue) 1731 limit = min(cache->cachepages / 20, 4096); 1732 else if (queue == &cache->warmqueue) 1733 limit = (cache->cachepages / 2); 1734 else 1735 panic("chillbufs: invalid queue"); 1736 1737 if (*queuepages > 96 && *queuepages > limit) { 1738 bp = TAILQ_FIRST(queue); 1739 if (!bp) 1740 panic("inconsistent bufpage counts"); 1741 pages = atop(bp->b_bufsize); 1742 *queuepages -= pages; 1743 TAILQ_REMOVE(queue, bp, b_freelist); 1744 /* we do not clear B_WARM */ 1745 SET(bp->b_flags, B_COLD); 1746 TAILQ_INSERT_TAIL(&cache->coldqueue, bp, b_freelist); 1747 return 1; 1748 } 1749 return 0; 1750 } 1751 1752 void 1753 bufcache_release(struct buf *bp) 1754 { 1755 struct bufqueue *queue; 1756 int64_t pages; 1757 struct bufcache *cache = &cleancache[bp->cache]; 1758 1759 pages = atop(bp->b_bufsize); 1760 KASSERT(ISSET(bp->b_flags, B_BC)); 1761 if (fliphigh) { 1762 if (ISSET(bp->b_flags, B_DMA) && bp->cache > 0) 1763 panic("B_DMA buffer release from cache %d", 1764 bp->cache); 1765 else if ((!ISSET(bp->b_flags, B_DMA)) && bp->cache == 0) 1766 panic("Non B_DMA buffer release from cache %d", 1767 bp->cache); 1768 } 1769 1770 if (!ISSET(bp->b_flags, B_DELWRI)) { 1771 int64_t *queuepages; 1772 if (ISSET(bp->b_flags, B_WARM | B_COLD)) { 1773 SET(bp->b_flags, B_WARM); 1774 CLR(bp->b_flags, B_COLD); 1775 queue = &cache->warmqueue; 1776 queuepages = &cache->warmbufpages; 1777 } else { 1778 queue = &cache->hotqueue; 1779 queuepages = &cache->hotbufpages; 1780 } 1781 *queuepages += pages; 1782 bcstats.numcleanpages += pages; 1783 cache->cachepages += pages; 1784 chillbufs(cache, queue, queuepages); 1785 } else { 1786 queue = &dirtyqueue; 1787 bcstats.numdirtypages += pages; 1788 bcstats.delwribufs++; 1789 } 1790 TAILQ_INSERT_TAIL(queue, bp, b_freelist); 1791 } 1792 1793 #ifdef HIBERNATE 1794 /* 1795 * Nuke the buffer cache from orbit when hibernating. We do not want to save 1796 * any clean cache pages to swap and read them back. the original disk files 1797 * are just as good. 1798 */ 1799 void 1800 hibernate_suspend_bufcache(void) 1801 { 1802 struct buf *bp; 1803 int s; 1804 1805 s = splbio(); 1806 /* Chuck away all the cache pages.. discard bufs, do not promote */ 1807 while ((bp = bufcache_getcleanbuf_range(DMA_CACHE, NUM_CACHES - 1, 1))) { 1808 bufcache_take(bp); 1809 if (bp->b_vp) { 1810 RBT_REMOVE(buf_rb_bufs, &bp->b_vp->v_bufs_tree, bp); 1811 brelvp(bp); 1812 } 1813 buf_put(bp); 1814 } 1815 splx(s); 1816 } 1817 1818 void 1819 hibernate_resume_bufcache(void) 1820 { 1821 /* XXX Nothing needed here for now */ 1822 } 1823 #endif /* HIBERNATE */ 1824