1 /* 2 * (MPSAFE) 3 * 4 * Copyright (c) 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * The Mach Operating System project at Carnegie-Mellon University. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * from: @(#)vm_pager.c 8.6 (Berkeley) 1/12/94 35 * 36 * 37 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 38 * All rights reserved. 39 * 40 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 41 * 42 * Permission to use, copy, modify and distribute this software and 43 * its documentation is hereby granted, provided that both the copyright 44 * notice and this permission notice appear in all copies of the 45 * software, derivative works or modified versions, and any portions 46 * thereof, and that both notices appear in supporting documentation. 47 * 48 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 49 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 50 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 51 * 52 * Carnegie Mellon requests users of this software to return to 53 * 54 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 55 * School of Computer Science 56 * Carnegie Mellon University 57 * Pittsburgh PA 15213-3890 58 * 59 * any improvements or extensions that they make and grant Carnegie the 60 * rights to redistribute these changes. 61 * 62 * $FreeBSD: src/sys/vm/vm_pager.c,v 1.54.2.2 2001/11/18 07:11:00 dillon Exp $ 63 */ 64 65 /* 66 * Paging space routine stubs. Emulates a matchmaker-like interface 67 * for builtin pagers. 68 */ 69 70 #include <sys/param.h> 71 #include <sys/systm.h> 72 #include <sys/kernel.h> 73 #include <sys/vnode.h> 74 #include <sys/buf.h> 75 #include <sys/ucred.h> 76 #include <sys/dsched.h> 77 #include <sys/proc.h> 78 #include <sys/sysctl.h> 79 #include <sys/thread2.h> 80 81 #include <vm/vm.h> 82 #include <vm/vm_param.h> 83 #include <vm/vm_kern.h> 84 #include <vm/vm_object.h> 85 #include <vm/vm_page.h> 86 #include <vm/vm_pager.h> 87 #include <vm/vm_extern.h> 88 89 #include <sys/buf2.h> 90 #include <vm/vm_page2.h> 91 92 extern struct pagerops defaultpagerops; 93 extern struct pagerops swappagerops; 94 extern struct pagerops vnodepagerops; 95 extern struct pagerops devicepagerops; 96 extern struct pagerops physpagerops; 97 98 static int dead_pager_getpage (vm_object_t, vm_page_t *, int); 99 static void dead_pager_putpages (vm_object_t, vm_page_t *, int, int, int *); 100 static boolean_t dead_pager_haspage (vm_object_t, vm_pindex_t); 101 static void dead_pager_dealloc (vm_object_t); 102 103 /* 104 * No requirements. 105 */ 106 static int 107 dead_pager_getpage(vm_object_t obj, vm_page_t *mpp, int seqaccess) 108 { 109 return VM_PAGER_FAIL; 110 } 111 112 /* 113 * No requirements. 114 */ 115 static void 116 dead_pager_putpages(vm_object_t object, vm_page_t *m, int count, int flags, 117 int *rtvals) 118 { 119 int i; 120 121 for (i = 0; i < count; i++) { 122 rtvals[i] = VM_PAGER_AGAIN; 123 } 124 } 125 126 /* 127 * No requirements. 128 */ 129 static boolean_t 130 dead_pager_haspage(vm_object_t object, vm_pindex_t pindex) 131 { 132 return FALSE; 133 } 134 135 /* 136 * No requirements. 137 */ 138 static void 139 dead_pager_dealloc(vm_object_t object) 140 { 141 KKASSERT(object->swblock_count == 0); 142 return; 143 } 144 145 static struct pagerops deadpagerops = { 146 dead_pager_dealloc, 147 dead_pager_getpage, 148 dead_pager_putpages, 149 dead_pager_haspage 150 }; 151 152 struct pagerops *pagertab[] = { 153 &defaultpagerops, /* OBJT_DEFAULT */ 154 &swappagerops, /* OBJT_SWAP */ 155 &vnodepagerops, /* OBJT_VNODE */ 156 &devicepagerops, /* OBJT_DEVICE */ 157 &devicepagerops, /* OBJT_MGTDEVICE */ 158 &physpagerops, /* OBJT_PHYS */ 159 &deadpagerops /* OBJT_DEAD */ 160 }; 161 162 int npagers = NELEM(pagertab); 163 164 /* 165 * Kernel address space for mapping pages. 166 * Used by pagers where KVAs are needed for IO. 167 * 168 * XXX needs to be large enough to support the number of pending async 169 * cleaning requests (NPENDINGIO == 64) * the maximum swap cluster size 170 * (MAXPHYS == 64k) if you want to get the most efficiency. 171 */ 172 #define PAGER_MAP_SIZE (8 * 1024 * 1024) 173 174 #define BSWHSIZE 16 175 #define BSWHMASK (BSWHSIZE - 1) 176 177 TAILQ_HEAD(swqueue, buf); 178 179 int pager_map_size = PAGER_MAP_SIZE; 180 struct vm_map pager_map; 181 182 static vm_offset_t swapbkva_mem; /* swap buffers kva */ 183 static vm_offset_t swapbkva_kva; /* swap buffers kva */ 184 static struct swqueue bswlist_mem[BSWHSIZE]; /* with preallocated memory */ 185 static struct swqueue bswlist_kva[BSWHSIZE]; /* with kva */ 186 static struct swqueue bswlist_raw[BSWHSIZE]; /* without kva */ 187 static struct spinlock bswspin_mem[BSWHSIZE]; 188 static struct spinlock bswspin_kva[BSWHSIZE]; 189 static struct spinlock bswspin_raw[BSWHSIZE]; 190 static int pbuf_raw_count; 191 static int pbuf_kva_count; 192 static int pbuf_mem_count; 193 194 SYSCTL_INT(_vfs, OID_AUTO, pbuf_raw_count, CTLFLAG_RD, &pbuf_raw_count, 0, 195 "Kernel pbuf raw reservations"); 196 SYSCTL_INT(_vfs, OID_AUTO, pbuf_kva_count, CTLFLAG_RD, &pbuf_kva_count, 0, 197 "Kernel pbuf kva reservations"); 198 SYSCTL_INT(_vfs, OID_AUTO, pbuf_mem_count, CTLFLAG_RD, &pbuf_mem_count, 0, 199 "Kernel pbuf mem reservations"); 200 201 /* 202 * Initialize the swap buffer list. 203 * 204 * Called from the low level boot code only. 205 */ 206 static void 207 vm_pager_init(void *arg __unused) 208 { 209 int i; 210 211 for (i = 0; i < BSWHSIZE; ++i) { 212 TAILQ_INIT(&bswlist_mem[i]); 213 TAILQ_INIT(&bswlist_kva[i]); 214 TAILQ_INIT(&bswlist_raw[i]); 215 spin_init(&bswspin_mem[i], "bswmem"); 216 spin_init(&bswspin_kva[i], "bswkva"); 217 spin_init(&bswspin_raw[i], "bswraw"); 218 } 219 } 220 SYSINIT(vm_mem, SI_BOOT1_VM, SI_ORDER_SECOND, vm_pager_init, NULL); 221 222 /* 223 * Called from the low level boot code only. 224 */ 225 static 226 void 227 vm_pager_bufferinit(void *dummy __unused) 228 { 229 struct buf *bp; 230 long i; 231 232 /* 233 * Reserve KVM space for pbuf data. 234 */ 235 swapbkva_mem = kmem_alloc_pageable(&pager_map, nswbuf_mem * MAXPHYS, 236 VM_SUBSYS_BUFDATA); 237 if (!swapbkva_mem) 238 panic("Not enough pager_map VM space for physical buffers"); 239 swapbkva_kva = kmem_alloc_pageable(&pager_map, nswbuf_kva * MAXPHYS, 240 VM_SUBSYS_BUFDATA); 241 if (!swapbkva_kva) 242 panic("Not enough pager_map VM space for physical buffers"); 243 244 /* 245 * Initial pbuf setup. 246 * 247 * mem - These pbufs have permanently allocated memory 248 * kva - These pbufs have unallocated kva reservations 249 * raw - These pbufs have no kva reservations 250 */ 251 252 /* 253 * Buffers with pre-allocated kernel memory can be convenient for 254 * copyin/copyout because no SMP page invalidation or other pmap 255 * operations are needed. 256 */ 257 #if 1 258 bp = swbuf_mem; 259 for (i = 0; i < nswbuf_mem; ++i, ++bp) { 260 vm_page_t m; 261 vm_pindex_t pg; 262 int j; 263 264 bp->b_kvabase = (caddr_t)((intptr_t)i * MAXPHYS) + swapbkva_mem; 265 bp->b_kvasize = MAXPHYS; 266 bp->b_swindex = i & BSWHMASK; 267 BUF_LOCKINIT(bp); 268 buf_dep_init(bp); 269 TAILQ_INSERT_HEAD(&bswlist_mem[i & BSWHMASK], bp, b_freelist); 270 atomic_add_int(&pbuf_mem_count, 1); 271 bp->b_data = bp->b_kvabase; 272 bp->b_bcount = MAXPHYS; 273 bp->b_xio.xio_pages = bp->b_xio.xio_internal_pages; 274 275 pg = (vm_offset_t)bp->b_kvabase >> PAGE_SHIFT; 276 vm_object_hold(&kernel_object); 277 for (j = 0; j < MAXPHYS / PAGE_SIZE; ++j) { 278 m = vm_page_alloc(&kernel_object, pg, VM_ALLOC_NORMAL | 279 VM_ALLOC_SYSTEM); 280 KKASSERT(m != NULL); 281 bp->b_xio.xio_internal_pages[j] = m; 282 vm_page_wire(m); 283 /* early boot, no other cpus running yet */ 284 pmap_kenter_noinval(pg * PAGE_SIZE, VM_PAGE_TO_PHYS(m)); 285 cpu_invlpg((void *)(pg * PAGE_SIZE)); 286 vm_page_wakeup(m); 287 ++pg; 288 } 289 vm_object_drop(&kernel_object); 290 bp->b_xio.xio_npages = j; 291 } 292 #endif 293 294 /* 295 * Buffers with pre-assigned KVA bases. The KVA has no memory pages 296 * assigned to it. Saves the caller from having to reserve KVA for 297 * the page map. 298 */ 299 bp = swbuf_kva; 300 for (i = 0; i < nswbuf_kva; ++i, ++bp) { 301 bp->b_kvabase = (caddr_t)((intptr_t)i * MAXPHYS) + swapbkva_kva; 302 bp->b_kvasize = MAXPHYS; 303 bp->b_swindex = i & BSWHMASK; 304 BUF_LOCKINIT(bp); 305 buf_dep_init(bp); 306 TAILQ_INSERT_HEAD(&bswlist_kva[i & BSWHMASK], bp, b_freelist); 307 atomic_add_int(&pbuf_kva_count, 1); 308 } 309 310 /* 311 * RAW buffers with no KVA mappings. 312 * 313 * NOTE: We use KM_NOTLBSYNC here to reduce unnecessary IPIs 314 * during startup, which can really slow down emulated 315 * systems. 316 */ 317 nswbuf_raw = nbuf * 2; 318 swbuf_raw = (void *)kmem_alloc3(&kernel_map, 319 round_page(nswbuf_raw * sizeof(struct buf)), 320 VM_SUBSYS_BUFDATA, 321 KM_NOTLBSYNC); 322 smp_invltlb(); 323 bp = swbuf_raw; 324 for (i = 0; i < nswbuf_raw; ++i, ++bp) { 325 bp->b_swindex = i & BSWHMASK; 326 BUF_LOCKINIT(bp); 327 buf_dep_init(bp); 328 TAILQ_INSERT_HEAD(&bswlist_raw[i & BSWHMASK], bp, b_freelist); 329 atomic_add_int(&pbuf_raw_count, 1); 330 } 331 } 332 333 SYSINIT(do_vmpg, SI_BOOT2_MACHDEP, SI_ORDER_FIRST, vm_pager_bufferinit, NULL); 334 335 /* 336 * No requirements. 337 */ 338 void 339 vm_pager_deallocate(vm_object_t object) 340 { 341 (*pagertab[object->type]->pgo_dealloc) (object); 342 } 343 344 /* 345 * vm_pager_get_pages() - inline, see vm/vm_pager.h 346 * vm_pager_put_pages() - inline, see vm/vm_pager.h 347 * vm_pager_has_page() - inline, see vm/vm_pager.h 348 * vm_pager_page_inserted() - inline, see vm/vm_pager.h 349 * vm_pager_page_removed() - inline, see vm/vm_pager.h 350 */ 351 352 /* 353 * Search the specified pager object list for an object with the 354 * specified handle. If an object with the specified handle is found, 355 * increase its reference count and return it. Otherwise, return NULL. 356 * 357 * The pager object list must be locked. 358 */ 359 vm_object_t 360 vm_pager_object_lookup(struct pagerlst *pg_list, void *handle) 361 { 362 vm_object_t object; 363 364 TAILQ_FOREACH(object, pg_list, pager_object_list) { 365 if (object->handle == handle) { 366 VM_OBJECT_LOCK(object); 367 if ((object->flags & OBJ_DEAD) == 0) { 368 vm_object_reference_locked(object); 369 VM_OBJECT_UNLOCK(object); 370 break; 371 } 372 VM_OBJECT_UNLOCK(object); 373 } 374 } 375 return (object); 376 } 377 378 /* 379 * Initialize a physical buffer. 380 * 381 * No requirements. 382 */ 383 static void 384 initpbuf(struct buf *bp) 385 { 386 bp->b_qindex = 0; /* BQUEUE_NONE */ 387 bp->b_data = bp->b_kvabase; /* NULL if pbuf sans kva */ 388 bp->b_flags = B_PAGING; 389 bp->b_cmd = BUF_CMD_DONE; 390 bp->b_error = 0; 391 bp->b_bcount = 0; 392 bp->b_bufsize = MAXPHYS; 393 initbufbio(bp); 394 xio_init(&bp->b_xio); 395 BUF_LOCK(bp, LK_EXCLUSIVE); 396 } 397 398 /* 399 * Allocate a physical buffer 400 * 401 * If (pfreecnt != NULL) then *pfreecnt will be decremented on return and 402 * the function will block while it is <= 0. 403 * 404 * Physical buffers can be with or without KVA space reserved. There 405 * are severe limitations on the ones with KVA reserved, and fewer 406 * limitations on the ones without. getpbuf() gets one without, 407 * getpbuf_kva() gets one with. 408 * 409 * No requirements. 410 */ 411 struct buf * 412 getpbuf(int *pfreecnt) 413 { 414 struct buf *bp; 415 int iter; 416 int loops; 417 418 for (;;) { 419 while (pfreecnt && *pfreecnt <= 0) { 420 tsleep_interlock(pfreecnt, 0); 421 if ((int)atomic_fetchadd_int(pfreecnt, 0) <= 0) 422 tsleep(pfreecnt, PINTERLOCKED, "wswbuf0", 0); 423 } 424 if (pbuf_raw_count <= 0) { 425 tsleep_interlock(&pbuf_raw_count, 0); 426 if ((int)atomic_fetchadd_int(&pbuf_raw_count, 0) <= 0) 427 tsleep(&pbuf_raw_count, PINTERLOCKED, 428 "wswbuf1", 0); 429 continue; 430 } 431 iter = mycpuid & BSWHMASK; 432 for (loops = BSWHSIZE; loops; --loops) { 433 if (TAILQ_FIRST(&bswlist_raw[iter]) == NULL) { 434 iter = (iter + 1) & BSWHMASK; 435 continue; 436 } 437 spin_lock(&bswspin_raw[iter]); 438 if ((bp = TAILQ_FIRST(&bswlist_raw[iter])) == NULL) { 439 spin_unlock(&bswspin_raw[iter]); 440 iter = (iter + 1) & BSWHMASK; 441 continue; 442 } 443 TAILQ_REMOVE(&bswlist_raw[iter], bp, b_freelist); 444 atomic_add_int(&pbuf_raw_count, -1); 445 if (pfreecnt) 446 atomic_add_int(pfreecnt, -1); 447 spin_unlock(&bswspin_raw[iter]); 448 initpbuf(bp); 449 450 return bp; 451 } 452 } 453 /* not reached */ 454 } 455 456 struct buf * 457 getpbuf_kva(int *pfreecnt) 458 { 459 struct buf *bp; 460 int iter; 461 int loops; 462 463 for (;;) { 464 while (pfreecnt && *pfreecnt <= 0) { 465 tsleep_interlock(pfreecnt, 0); 466 if ((int)atomic_fetchadd_int(pfreecnt, 0) <= 0) 467 tsleep(pfreecnt, PINTERLOCKED, "wswbuf2", 0); 468 } 469 if (pbuf_kva_count <= 0) { 470 tsleep_interlock(&pbuf_kva_count, 0); 471 if ((int)atomic_fetchadd_int(&pbuf_kva_count, 0) <= 0) 472 tsleep(&pbuf_kva_count, PINTERLOCKED, 473 "wswbuf3", 0); 474 continue; 475 } 476 iter = mycpuid & BSWHMASK; 477 for (loops = BSWHSIZE; loops; --loops) { 478 if (TAILQ_FIRST(&bswlist_kva[iter]) == NULL) { 479 iter = (iter + 1) & BSWHMASK; 480 continue; 481 } 482 spin_lock(&bswspin_kva[iter]); 483 if ((bp = TAILQ_FIRST(&bswlist_kva[iter])) == NULL) { 484 spin_unlock(&bswspin_kva[iter]); 485 iter = (iter + 1) & BSWHMASK; 486 continue; 487 } 488 TAILQ_REMOVE(&bswlist_kva[iter], bp, b_freelist); 489 atomic_add_int(&pbuf_kva_count, -1); 490 if (pfreecnt) 491 atomic_add_int(pfreecnt, -1); 492 spin_unlock(&bswspin_kva[iter]); 493 initpbuf(bp); 494 495 return bp; 496 } 497 } 498 /* not reached */ 499 } 500 501 /* 502 * Allocate a pbuf with kernel memory already preallocated. Caller must 503 * not change the mapping. 504 */ 505 struct buf * 506 getpbuf_mem(int *pfreecnt) 507 { 508 struct buf *bp; 509 int iter; 510 int loops; 511 512 for (;;) { 513 while (pfreecnt && *pfreecnt <= 0) { 514 tsleep_interlock(pfreecnt, 0); 515 if ((int)atomic_fetchadd_int(pfreecnt, 0) <= 0) 516 tsleep(pfreecnt, PINTERLOCKED, "wswbuf4", 0); 517 } 518 if (pbuf_mem_count <= 0) { 519 tsleep_interlock(&pbuf_mem_count, 0); 520 if ((int)atomic_fetchadd_int(&pbuf_mem_count, 0) <= 0) 521 tsleep(&pbuf_mem_count, PINTERLOCKED, 522 "wswbuf5", 0); 523 continue; 524 } 525 iter = mycpuid & BSWHMASK; 526 for (loops = BSWHSIZE; loops; --loops) { 527 if (TAILQ_FIRST(&bswlist_mem[iter]) == NULL) { 528 iter = (iter + 1) & BSWHMASK; 529 continue; 530 } 531 spin_lock(&bswspin_mem[iter]); 532 if ((bp = TAILQ_FIRST(&bswlist_mem[iter])) == NULL) { 533 spin_unlock(&bswspin_mem[iter]); 534 iter = (iter + 1) & BSWHMASK; 535 continue; 536 } 537 TAILQ_REMOVE(&bswlist_mem[iter], bp, b_freelist); 538 atomic_add_int(&pbuf_mem_count, -1); 539 if (pfreecnt) 540 atomic_add_int(pfreecnt, -1); 541 spin_unlock(&bswspin_mem[iter]); 542 initpbuf(bp); 543 544 return bp; 545 } 546 } 547 /* not reached */ 548 } 549 550 /* 551 * Allocate a physical buffer, if one is available. 552 * 553 * Note that there is no NULL hack here - all subsystems using this 554 * call are required to use a non-NULL pfreecnt. 555 * 556 * No requirements. 557 */ 558 struct buf * 559 trypbuf(int *pfreecnt) 560 { 561 struct buf *bp; 562 int iter = mycpuid & BSWHMASK; 563 int loops; 564 565 for (loops = BSWHSIZE; loops; --loops) { 566 if (*pfreecnt <= 0 || TAILQ_FIRST(&bswlist_raw[iter]) == NULL) { 567 iter = (iter + 1) & BSWHMASK; 568 continue; 569 } 570 spin_lock(&bswspin_raw[iter]); 571 if (*pfreecnt <= 0 || 572 (bp = TAILQ_FIRST(&bswlist_raw[iter])) == NULL) { 573 spin_unlock(&bswspin_raw[iter]); 574 iter = (iter + 1) & BSWHMASK; 575 continue; 576 } 577 TAILQ_REMOVE(&bswlist_raw[iter], bp, b_freelist); 578 atomic_add_int(&pbuf_raw_count, -1); 579 atomic_add_int(pfreecnt, -1); 580 581 spin_unlock(&bswspin_raw[iter]); 582 583 initpbuf(bp); 584 585 return bp; 586 } 587 return NULL; 588 } 589 590 struct buf * 591 trypbuf_kva(int *pfreecnt) 592 { 593 struct buf *bp; 594 int iter = mycpuid & BSWHMASK; 595 int loops; 596 597 for (loops = BSWHSIZE; loops; --loops) { 598 if (*pfreecnt <= 0 || TAILQ_FIRST(&bswlist_kva[iter]) == NULL) { 599 iter = (iter + 1) & BSWHMASK; 600 continue; 601 } 602 spin_lock(&bswspin_kva[iter]); 603 if (*pfreecnt <= 0 || 604 (bp = TAILQ_FIRST(&bswlist_kva[iter])) == NULL) { 605 spin_unlock(&bswspin_kva[iter]); 606 iter = (iter + 1) & BSWHMASK; 607 continue; 608 } 609 TAILQ_REMOVE(&bswlist_kva[iter], bp, b_freelist); 610 atomic_add_int(&pbuf_kva_count, -1); 611 atomic_add_int(pfreecnt, -1); 612 613 spin_unlock(&bswspin_kva[iter]); 614 615 initpbuf(bp); 616 617 return bp; 618 } 619 return NULL; 620 } 621 622 /* 623 * Release a physical buffer 624 * 625 * NOTE: pfreecnt can be NULL, but this 'feature' will be removed 626 * relatively soon when the rest of the subsystems get smart about it. XXX 627 * 628 * No requirements. 629 */ 630 void 631 relpbuf(struct buf *bp, int *pfreecnt) 632 { 633 int wake = 0; 634 int wake_free = 0; 635 int iter = bp->b_swindex; 636 637 KKASSERT(bp->b_flags & B_PAGING); 638 dsched_buf_exit(bp); 639 640 BUF_UNLOCK(bp); 641 642 if (bp >= swbuf_mem && bp < &swbuf_mem[nswbuf_mem]) { 643 KKASSERT(bp->b_kvabase); 644 spin_lock(&bswspin_mem[iter]); 645 TAILQ_INSERT_HEAD(&bswlist_mem[iter], bp, b_freelist); 646 if (atomic_fetchadd_int(&pbuf_mem_count, 1) == nswbuf_mem / 4) 647 wake = 1; 648 if (pfreecnt) { 649 if (atomic_fetchadd_int(pfreecnt, 1) == 1) 650 wake_free = 1; 651 } 652 spin_unlock(&bswspin_mem[iter]); 653 if (wake) 654 wakeup(&pbuf_mem_count); 655 } else if (bp >= swbuf_kva && bp < &swbuf_kva[nswbuf_kva]) { 656 KKASSERT(bp->b_kvabase); 657 spin_lock(&bswspin_kva[iter]); 658 TAILQ_INSERT_HEAD(&bswlist_kva[iter], bp, b_freelist); 659 if (atomic_fetchadd_int(&pbuf_kva_count, 1) == nswbuf_kva / 4) 660 wake = 1; 661 if (pfreecnt) { 662 if (atomic_fetchadd_int(pfreecnt, 1) == 1) 663 wake_free = 1; 664 } 665 spin_unlock(&bswspin_kva[iter]); 666 if (wake) 667 wakeup(&pbuf_kva_count); 668 } else { 669 KKASSERT(bp->b_kvabase == NULL); 670 KKASSERT(bp >= swbuf_raw && bp < &swbuf_raw[nswbuf_raw]); 671 spin_lock(&bswspin_raw[iter]); 672 TAILQ_INSERT_HEAD(&bswlist_raw[iter], bp, b_freelist); 673 if (atomic_fetchadd_int(&pbuf_raw_count, 1) == nswbuf_raw / 4) 674 wake = 1; 675 if (pfreecnt) { 676 if (atomic_fetchadd_int(pfreecnt, 1) == 1) 677 wake_free = 1; 678 } 679 spin_unlock(&bswspin_raw[iter]); 680 if (wake) 681 wakeup(&pbuf_raw_count); 682 } 683 if (wake_free) 684 wakeup(pfreecnt); 685 } 686 687 void 688 pbuf_adjcount(int *pfreecnt, int n) 689 { 690 if (n) { 691 atomic_add_int(pfreecnt, n); 692 wakeup(pfreecnt); 693 } 694 } 695