1 /* 2 * Copyright (c) 1990 University of Utah. 3 * Copyright (c) 1991 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to Berkeley by 7 * the Systems Programming Group of the University of Utah Computer 8 * Science Department. 9 * 10 * %sccs.include.redist.c% 11 * 12 * @(#)swap_pager.c 7.3 (Berkeley) 04/20/91 13 */ 14 15 /* 16 * Quick hack to page to dedicated partition(s). 17 * TODO: 18 * Add multiprocessor locks 19 * Deal with async writes in a better fashion 20 */ 21 22 #include "swappager.h" 23 #if NSWAPPAGER > 0 24 25 #include "param.h" 26 #include "proc.h" 27 #include "buf.h" 28 #include "map.h" 29 #include "systm.h" 30 #include "specdev.h" 31 #include "vnode.h" 32 #include "malloc.h" 33 #include "queue.h" 34 35 #include "vm_param.h" 36 #include "queue.h" 37 #include "lock.h" 38 #include "vm_prot.h" 39 #include "vm_object.h" 40 #include "vm_page.h" 41 #include "vm_pageout.h" 42 #include "swap_pager.h" 43 44 #define NSWSIZES 16 /* size of swtab */ 45 #define NPENDINGIO 64 /* max # of pending cleans */ 46 #define MAXDADDRS 64 /* max # of disk addrs for fixed allocations */ 47 48 #ifdef DEBUG 49 int swpagerdebug = 0x100; 50 #define SDB_FOLLOW 0x001 51 #define SDB_INIT 0x002 52 #define SDB_ALLOC 0x004 53 #define SDB_IO 0x008 54 #define SDB_WRITE 0x010 55 #define SDB_FAIL 0x020 56 #define SDB_ALLOCBLK 0x040 57 #define SDB_FULL 0x080 58 #define SDB_ANOM 0x100 59 #define SDB_ANOMPANIC 0x200 60 #endif 61 62 struct swpagerclean { 63 queue_head_t spc_list; 64 int spc_flags; 65 struct buf *spc_bp; 66 sw_pager_t spc_swp; 67 vm_offset_t spc_kva; 68 vm_page_t spc_m; 69 } swcleanlist[NPENDINGIO]; 70 typedef struct swpagerclean *swp_clean_t; 71 72 /* spc_flags values */ 73 #define SPC_FREE 0x00 74 #define SPC_BUSY 0x01 75 #define SPC_DONE 0x02 76 #define SPC_ERROR 0x04 77 #define SPC_DIRTY 0x08 78 79 struct swtab { 80 vm_size_t st_osize; /* size of object (bytes) */ 81 int st_bsize; /* vs. size of swap block (DEV_BSIZE units) */ 82 #ifdef DEBUG 83 u_long st_inuse; /* number in this range in use */ 84 u_long st_usecnt; /* total used of this size */ 85 #endif 86 } swtab[NSWSIZES+1]; 87 88 #ifdef DEBUG 89 int swap_pager_pendingio; /* max pending async "clean" ops */ 90 int swap_pager_poip; /* pageouts in progress */ 91 int swap_pager_piip; /* pageins in progress */ 92 #endif 93 94 queue_head_t swap_pager_inuse; /* list of pending page cleans */ 95 queue_head_t swap_pager_free; /* list of free pager clean structs */ 96 queue_head_t swap_pager_list; /* list of "named" anon regions */ 97 98 void 99 swap_pager_init() 100 { 101 register swp_clean_t spc; 102 register int i, bsize; 103 extern int dmmin, dmmax; 104 int maxbsize; 105 106 #ifdef DEBUG 107 if (swpagerdebug & (SDB_FOLLOW|SDB_INIT)) 108 printf("swpg_init()\n"); 109 #endif 110 dfltpagerops = &swappagerops; 111 queue_init(&swap_pager_list); 112 113 /* 114 * Initialize clean lists 115 */ 116 queue_init(&swap_pager_inuse); 117 queue_init(&swap_pager_free); 118 for (i = 0, spc = swcleanlist; i < NPENDINGIO; i++, spc++) { 119 queue_enter(&swap_pager_free, spc, swp_clean_t, spc_list); 120 spc->spc_flags = SPC_FREE; 121 } 122 123 /* 124 * Calculate the swap allocation constants. 125 */ 126 if (dmmin == 0) { 127 dmmin = DMMIN; 128 if (dmmin < CLBYTES/DEV_BSIZE) 129 dmmin = CLBYTES/DEV_BSIZE; 130 } 131 if (dmmax == 0) 132 dmmax = DMMAX; 133 134 /* 135 * Fill in our table of object size vs. allocation size 136 */ 137 bsize = btodb(PAGE_SIZE); 138 if (bsize < dmmin) 139 bsize = dmmin; 140 maxbsize = btodb(sizeof(sw_bm_t) * NBBY * PAGE_SIZE); 141 if (maxbsize > dmmax) 142 maxbsize = dmmax; 143 for (i = 0; i < NSWSIZES; i++) { 144 swtab[i].st_osize = (vm_size_t) (MAXDADDRS * dbtob(bsize)); 145 swtab[i].st_bsize = bsize; 146 #ifdef DEBUG 147 if (swpagerdebug & SDB_INIT) 148 printf("swpg_init: ix %d, size %x, bsize %x\n", 149 i, swtab[i].st_osize, swtab[i].st_bsize); 150 #endif 151 if (bsize >= maxbsize) 152 break; 153 bsize *= 2; 154 } 155 swtab[i].st_osize = 0; 156 swtab[i].st_bsize = bsize; 157 } 158 159 /* 160 * Allocate a pager structure and associated resources. 161 * Note that if we are called from the pageout daemon (handle == NULL) 162 * we should not wait for memory as it could resulting in deadlock. 163 */ 164 vm_pager_t 165 swap_pager_alloc(handle, size, prot) 166 caddr_t handle; 167 register vm_size_t size; 168 vm_prot_t prot; 169 { 170 register vm_pager_t pager; 171 register sw_pager_t swp; 172 struct swtab *swt; 173 int waitok; 174 175 #ifdef DEBUG 176 if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOC)) 177 printf("swpg_alloc(%x, %x, %x)\n", handle, size, prot); 178 #endif 179 /* 180 * If this is a "named" anonymous region, look it up and 181 * return the appropriate pager if it exists. 182 */ 183 if (handle) { 184 pager = vm_pager_lookup(&swap_pager_list, handle); 185 if (pager != NULL) { 186 /* 187 * Use vm_object_lookup to gain a reference 188 * to the object and also to remove from the 189 * object cache. 190 */ 191 if (vm_object_lookup(pager) == NULL) 192 panic("swap_pager_alloc: bad object"); 193 return(pager); 194 } 195 } 196 /* 197 * Pager doesn't exist, allocate swap management resources 198 * and initialize. 199 */ 200 waitok = handle ? M_WAITOK : M_NOWAIT; 201 pager = (vm_pager_t)malloc(sizeof *pager, M_VMPAGER, waitok); 202 if (pager == NULL) 203 return(NULL); 204 swp = (sw_pager_t)malloc(sizeof *swp, M_VMPGDATA, waitok); 205 if (swp == NULL) { 206 #ifdef DEBUG 207 if (swpagerdebug & SDB_FAIL) 208 printf("swpg_alloc: swpager malloc failed\n"); 209 #endif 210 free((caddr_t)pager, M_VMPAGER); 211 return(NULL); 212 } 213 size = round_page(size); 214 for (swt = swtab; swt->st_osize; swt++) 215 if (size <= swt->st_osize) 216 break; 217 #ifdef DEBUG 218 swt->st_inuse++; 219 swt->st_usecnt++; 220 #endif 221 swp->sw_osize = size; 222 swp->sw_bsize = swt->st_bsize; 223 swp->sw_nblocks = (btodb(size) + swp->sw_bsize - 1) / swp->sw_bsize; 224 swp->sw_blocks = (sw_blk_t) 225 malloc(swp->sw_nblocks*sizeof(*swp->sw_blocks), 226 M_VMPGDATA, M_NOWAIT); 227 if (swp->sw_blocks == NULL) { 228 free((caddr_t)swp, M_VMPGDATA); 229 free((caddr_t)pager, M_VMPAGER); 230 #ifdef DEBUG 231 if (swpagerdebug & SDB_FAIL) 232 printf("swpg_alloc: sw_blocks malloc failed\n"); 233 swt->st_inuse--; 234 swt->st_usecnt--; 235 #endif 236 return(FALSE); 237 } 238 bzero((caddr_t)swp->sw_blocks, 239 swp->sw_nblocks * sizeof(*swp->sw_blocks)); 240 swp->sw_poip = 0; 241 if (handle) { 242 vm_object_t object; 243 244 swp->sw_flags = SW_NAMED; 245 queue_enter(&swap_pager_list, pager, vm_pager_t, pg_list); 246 /* 247 * Consistant with other pagers: return with object 248 * referenced. Can't do this with handle == NULL 249 * since it might be the pageout daemon calling. 250 */ 251 object = vm_object_allocate(size); 252 vm_object_enter(object, pager); 253 vm_object_setpager(object, pager, 0, FALSE); 254 } else { 255 swp->sw_flags = 0; 256 queue_init(&pager->pg_list); 257 } 258 pager->pg_handle = handle; 259 pager->pg_ops = &swappagerops; 260 pager->pg_type = PG_SWAP; 261 pager->pg_data = (caddr_t)swp; 262 263 #ifdef DEBUG 264 if (swpagerdebug & SDB_ALLOC) 265 printf("swpg_alloc: pg_data %x, %x of %x at %x\n", 266 swp, swp->sw_nblocks, swp->sw_bsize, swp->sw_blocks); 267 #endif 268 return(pager); 269 } 270 271 void 272 swap_pager_dealloc(pager) 273 vm_pager_t pager; 274 { 275 register int i; 276 register sw_blk_t bp; 277 register sw_pager_t swp; 278 struct swtab *swt; 279 int s; 280 281 #ifdef DEBUG 282 /* save panic time state */ 283 if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) 284 return; 285 if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOC)) 286 printf("swpg_dealloc(%x)\n", pager); 287 #endif 288 /* 289 * Remove from list right away so lookups will fail if we 290 * block for pageout completion. 291 */ 292 swp = (sw_pager_t) pager->pg_data; 293 if (swp->sw_flags & SW_NAMED) { 294 queue_remove(&swap_pager_list, pager, vm_pager_t, pg_list); 295 swp->sw_flags &= ~SW_NAMED; 296 } 297 #ifdef DEBUG 298 for (swt = swtab; swt->st_osize; swt++) 299 if (swp->sw_osize <= swt->st_osize) 300 break; 301 swt->st_inuse--; 302 #endif 303 304 /* 305 * Wait for all pageouts to finish and remove 306 * all entries from cleaning list. 307 */ 308 s = splbio(); 309 while (swp->sw_poip) { 310 swp->sw_flags |= SW_WANTED; 311 assert_wait((int)swp); 312 thread_block(); 313 } 314 splx(s); 315 (void) swap_pager_clean(NULL, B_WRITE); 316 317 /* 318 * Free left over swap blocks 319 */ 320 for (i = 0, bp = swp->sw_blocks; i < swp->sw_nblocks; i++, bp++) 321 if (bp->swb_block) { 322 #ifdef DEBUG 323 if (swpagerdebug & (SDB_ALLOCBLK|SDB_FULL)) 324 printf("swpg_dealloc: blk %x\n", 325 bp->swb_block); 326 #endif 327 rmfree(swapmap, swp->sw_bsize, bp->swb_block); 328 } 329 /* 330 * Free swap management resources 331 */ 332 free((caddr_t)swp->sw_blocks, M_VMPGDATA); 333 free((caddr_t)swp, M_VMPGDATA); 334 free((caddr_t)pager, M_VMPAGER); 335 } 336 337 swap_pager_getpage(pager, m, sync) 338 vm_pager_t pager; 339 vm_page_t m; 340 boolean_t sync; 341 { 342 #ifdef DEBUG 343 if (swpagerdebug & SDB_FOLLOW) 344 printf("swpg_getpage(%x, %x, %d)\n", pager, m, sync); 345 #endif 346 return(swap_pager_io((sw_pager_t)pager->pg_data, m, B_READ)); 347 } 348 349 swap_pager_putpage(pager, m, sync) 350 vm_pager_t pager; 351 vm_page_t m; 352 boolean_t sync; 353 { 354 int flags; 355 356 #ifdef DEBUG 357 if (swpagerdebug & SDB_FOLLOW) 358 printf("swpg_putpage(%x, %x, %d)\n", pager, m, sync); 359 #endif 360 if (pager == NULL) { 361 (void) swap_pager_clean(NULL, B_WRITE); 362 return; 363 } 364 flags = B_WRITE; 365 if (!sync) 366 flags |= B_ASYNC; 367 return(swap_pager_io((sw_pager_t)pager->pg_data, m, flags)); 368 } 369 370 boolean_t 371 swap_pager_haspage(pager, offset) 372 vm_pager_t pager; 373 vm_offset_t offset; 374 { 375 register sw_pager_t swp; 376 register sw_blk_t swb; 377 int ix; 378 379 #ifdef DEBUG 380 if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOCBLK)) 381 printf("swpg_haspage(%x, %x) ", pager, offset); 382 #endif 383 swp = (sw_pager_t) pager->pg_data; 384 ix = offset / dbtob(swp->sw_bsize); 385 if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 386 #ifdef DEBUG 387 if (swpagerdebug & (SDB_FAIL|SDB_FOLLOW|SDB_ALLOCBLK)) 388 printf("swpg_haspage: %x bad offset %x, ix %x\n", 389 swp->sw_blocks, offset, ix); 390 #endif 391 return(FALSE); 392 } 393 swb = &swp->sw_blocks[ix]; 394 if (swb->swb_block) 395 ix = atop(offset % dbtob(swp->sw_bsize)); 396 #ifdef DEBUG 397 if (swpagerdebug & SDB_ALLOCBLK) 398 printf("%x blk %x+%x ", swp->sw_blocks, swb->swb_block, ix); 399 if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOCBLK)) 400 printf("-> %c\n", 401 "FT"[swb->swb_block && (swb->swb_mask & (1 << ix))]); 402 #endif 403 if (swb->swb_block && (swb->swb_mask & (1 << ix))) 404 return(TRUE); 405 return(FALSE); 406 } 407 408 /* 409 * Scaled down version of swap(). 410 * Assumes that PAGE_SIZE < MAXPHYS; i.e. only one operation needed. 411 * BOGUS: lower level IO routines expect a KVA so we have to map our 412 * provided physical page into the KVA to keep them happy. 413 */ 414 swap_pager_io(swp, m, flags) 415 register sw_pager_t swp; 416 vm_page_t m; 417 int flags; 418 { 419 register struct buf *bp; 420 register sw_blk_t swb; 421 register int s; 422 int ix; 423 boolean_t rv; 424 vm_offset_t kva, off; 425 swp_clean_t spc; 426 427 #ifdef DEBUG 428 /* save panic time state */ 429 if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) 430 return; 431 if (swpagerdebug & (SDB_FOLLOW|SDB_IO)) 432 printf("swpg_io(%x, %x, %x)\n", swp, m, flags); 433 #endif 434 435 /* 436 * For reads (pageins) and synchronous writes, we clean up 437 * all completed async pageouts and check to see if this 438 * page is currently being cleaned. If it is, we just wait 439 * til the operation is done before continuing. 440 */ 441 if ((flags & B_ASYNC) == 0) { 442 s = splbio(); 443 while (swap_pager_clean(m, flags&B_READ)) { 444 swp->sw_flags |= SW_WANTED; 445 assert_wait((int)swp); 446 thread_block(); 447 } 448 splx(s); 449 } 450 /* 451 * For async writes (pageouts), we cleanup completed pageouts so 452 * that all available resources are freed. Also tells us if this 453 * page is already being cleaned. If it is, or no resources 454 * are available, we try again later. 455 */ 456 else if (swap_pager_clean(m, B_WRITE) || queue_empty(&swap_pager_free)) 457 return(VM_PAGER_FAIL); 458 459 /* 460 * Determine swap block and allocate as necessary. 461 */ 462 off = m->offset + m->object->paging_offset; 463 ix = off / dbtob(swp->sw_bsize); 464 if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 465 #ifdef DEBUG 466 if (swpagerdebug & SDB_FAIL) 467 printf("swpg_io: bad offset %x+%x(%d) in %x\n", 468 m->offset, m->object->paging_offset, 469 ix, swp->sw_blocks); 470 #endif 471 return(VM_PAGER_FAIL); 472 } 473 swb = &swp->sw_blocks[ix]; 474 off = off % dbtob(swp->sw_bsize); 475 if (flags & B_READ) { 476 if (swb->swb_block == 0 || 477 (swb->swb_mask & (1 << atop(off))) == 0) { 478 #ifdef DEBUG 479 if (swpagerdebug & (SDB_ALLOCBLK|SDB_FAIL)) 480 printf("swpg_io: %x bad read: blk %x+%x, mask %x, off %x+%x\n", 481 swp->sw_blocks, 482 swb->swb_block, atop(off), 483 swb->swb_mask, 484 m->offset, m->object->paging_offset); 485 #endif 486 /* XXX: should we zero page here?? */ 487 return(VM_PAGER_FAIL); 488 } 489 } else if (swb->swb_block == 0) { 490 swb->swb_block = rmalloc(swapmap, swp->sw_bsize); 491 if (swb->swb_block == 0) { 492 #ifdef DEBUG 493 if (swpagerdebug & SDB_FAIL) 494 printf("swpg_io: rmalloc of %x failed\n", 495 swp->sw_bsize); 496 #endif 497 return(VM_PAGER_FAIL); 498 } 499 #ifdef DEBUG 500 if (swpagerdebug & (SDB_FULL|SDB_ALLOCBLK)) 501 printf("swpg_io: %x alloc blk %x at ix %x\n", 502 swp->sw_blocks, swb->swb_block, ix); 503 #endif 504 } 505 506 /* 507 * Allocate a kernel virtual address and initialize so that PTE 508 * is available for lower level IO drivers. 509 */ 510 kva = vm_pager_map_page(m); 511 512 /* 513 * Get a swap buffer header and perform the IO 514 */ 515 s = splbio(); 516 while (bswlist.av_forw == NULL) { 517 #ifdef DEBUG 518 if (swpagerdebug & SDB_ANOM) 519 printf("swpg_io: wait on swbuf for %x (%d)\n", 520 m, flags); 521 #endif 522 bswlist.b_flags |= B_WANTED; 523 sleep((caddr_t)&bswlist, PSWP+1); 524 } 525 bp = bswlist.av_forw; 526 bswlist.av_forw = bp->av_forw; 527 splx(s); 528 bp->b_flags = B_BUSY | (flags & B_READ); 529 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 530 bp->b_un.b_addr = (caddr_t)kva; 531 bp->b_blkno = swb->swb_block + btodb(off); 532 VHOLD(swapdev_vp); 533 bp->b_vp = swapdev_vp; 534 if (swapdev_vp->v_type == VBLK) 535 bp->b_dev = swapdev_vp->v_rdev; 536 bp->b_bcount = PAGE_SIZE; 537 if ((bp->b_flags & B_READ) == 0) 538 swapdev_vp->v_numoutput++; 539 540 /* 541 * If this is an async write we set up additional buffer fields 542 * and place a "cleaning" entry on the inuse queue. 543 */ 544 if ((flags & (B_READ|B_ASYNC)) == B_ASYNC) { 545 #ifdef DEBUG 546 if (queue_empty(&swap_pager_free)) 547 panic("swpg_io: lost spc"); 548 #endif 549 queue_remove_first(&swap_pager_free, 550 spc, swp_clean_t, spc_list); 551 #ifdef DEBUG 552 if (spc->spc_flags != SPC_FREE) 553 panic("swpg_io: bad free spc"); 554 #endif 555 spc->spc_flags = SPC_BUSY; 556 spc->spc_bp = bp; 557 spc->spc_swp = swp; 558 spc->spc_kva = kva; 559 spc->spc_m = m; 560 #ifdef DEBUG 561 m->pagerowned = 1; 562 #endif 563 bp->b_flags |= B_CALL; 564 bp->b_iodone = swap_pager_iodone; 565 s = splbio(); 566 swp->sw_poip++; 567 queue_enter(&swap_pager_inuse, spc, swp_clean_t, spc_list); 568 569 #ifdef DEBUG 570 swap_pager_poip++; 571 if (swpagerdebug & SDB_WRITE) 572 printf("swpg_io: write: bp=%x swp=%x spc=%x poip=%d\n", 573 bp, swp, spc, swp->sw_poip); 574 if ((swpagerdebug & SDB_ALLOCBLK) && 575 (swb->swb_mask & (1 << atop(off))) == 0) 576 printf("swpg_io: %x write blk %x+%x\n", 577 swp->sw_blocks, swb->swb_block, atop(off)); 578 #endif 579 swb->swb_mask |= (1 << atop(off)); 580 /* 581 * XXX: Block write faults til we are done. 582 */ 583 m->page_lock = VM_PROT_WRITE; 584 m->unlock_request = VM_PROT_ALL; 585 pmap_copy_on_write(VM_PAGE_TO_PHYS(m)); 586 splx(s); 587 } 588 #ifdef DEBUG 589 if (swpagerdebug & SDB_IO) 590 printf("swpg_io: IO start: bp %x, db %x, va %x, pa %x\n", 591 bp, swb->swb_block+btodb(off), kva, VM_PAGE_TO_PHYS(m)); 592 #endif 593 VOP_STRATEGY(bp); 594 if ((flags & (B_READ|B_ASYNC)) == B_ASYNC) { 595 #ifdef DEBUG 596 if (swpagerdebug & SDB_IO) 597 printf("swpg_io: IO started: bp %x\n", bp); 598 #endif 599 return(VM_PAGER_PEND); 600 } 601 s = splbio(); 602 #ifdef DEBUG 603 if (flags & B_READ) 604 swap_pager_piip++; 605 else 606 swap_pager_poip++; 607 #endif 608 while ((bp->b_flags & B_DONE) == 0) { 609 assert_wait((int)bp); 610 thread_block(); 611 } 612 #ifdef DEBUG 613 if (flags & B_READ) 614 --swap_pager_piip; 615 else 616 --swap_pager_poip; 617 #endif 618 rv = (bp->b_flags & B_ERROR) ? VM_PAGER_FAIL : VM_PAGER_OK; 619 bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_PAGET|B_UAREA|B_DIRTY); 620 bp->av_forw = bswlist.av_forw; 621 bswlist.av_forw = bp; 622 if (bp->b_vp) 623 brelvp(bp); 624 if (bswlist.b_flags & B_WANTED) { 625 bswlist.b_flags &= ~B_WANTED; 626 thread_wakeup((int)&bswlist); 627 } 628 if ((flags & B_READ) == 0 && rv == VM_PAGER_OK) { 629 m->clean = 1; 630 pmap_clear_modify(VM_PAGE_TO_PHYS(m)); 631 } 632 splx(s); 633 #ifdef DEBUG 634 if (swpagerdebug & SDB_IO) 635 printf("swpg_io: IO done: bp %x, rv %d\n", bp, rv); 636 if ((swpagerdebug & SDB_FAIL) && rv == VM_PAGER_FAIL) 637 printf("swpg_io: IO error\n"); 638 #endif 639 vm_pager_unmap_page(kva); 640 return(rv); 641 } 642 643 boolean_t 644 swap_pager_clean(m, rw) 645 vm_page_t m; 646 int rw; 647 { 648 register swp_clean_t spc, tspc; 649 register int s; 650 651 #ifdef DEBUG 652 /* save panic time state */ 653 if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) 654 return; 655 if (swpagerdebug & SDB_FOLLOW) 656 printf("swpg_clean(%x, %d)\n", m, rw); 657 #endif 658 tspc = NULL; 659 for (;;) { 660 /* 661 * Look up and removal from inuse list must be done 662 * at splbio() to avoid conflicts with swap_pager_iodone. 663 */ 664 s = splbio(); 665 spc = (swp_clean_t) queue_first(&swap_pager_inuse); 666 while (!queue_end(&swap_pager_inuse, (queue_entry_t)spc)) { 667 if ((spc->spc_flags & SPC_DONE) && 668 swap_pager_finish(spc)) { 669 queue_remove(&swap_pager_inuse, spc, 670 swp_clean_t, spc_list); 671 break; 672 } 673 if (m && m == spc->spc_m) { 674 #ifdef DEBUG 675 if (swpagerdebug & SDB_ANOM) 676 printf("swpg_clean: %x on list, flags %x\n", 677 m, spc->spc_flags); 678 #endif 679 tspc = spc; 680 } 681 spc = (swp_clean_t) queue_next(&spc->spc_list); 682 } 683 684 /* 685 * No operations done, thats all we can do for now. 686 */ 687 if (queue_end(&swap_pager_inuse, (queue_entry_t)spc)) 688 break; 689 splx(s); 690 691 /* 692 * The desired page was found to be busy earlier in 693 * the scan but has since completed. 694 */ 695 if (tspc && tspc == spc) { 696 #ifdef DEBUG 697 if (swpagerdebug & SDB_ANOM) 698 printf("swpg_clean: %x done while looking\n", 699 m); 700 #endif 701 tspc = NULL; 702 } 703 spc->spc_flags = SPC_FREE; 704 vm_pager_unmap_page(spc->spc_kva); 705 queue_enter(&swap_pager_free, spc, swp_clean_t, spc_list); 706 #ifdef DEBUG 707 if (swpagerdebug & SDB_WRITE) 708 printf("swpg_clean: free spc %x\n", spc); 709 #endif 710 } 711 /* 712 * If we found that the desired page is already being cleaned 713 * mark it so that swap_pager_iodone() will not set the clean 714 * flag before the pageout daemon has another chance to clean it. 715 */ 716 if (tspc && rw == B_WRITE) { 717 #ifdef DEBUG 718 if (swpagerdebug & SDB_ANOM) 719 printf("swpg_clean: %x on clean list\n", tspc); 720 #endif 721 tspc->spc_flags |= SPC_DIRTY; 722 } 723 splx(s); 724 725 #ifdef DEBUG 726 if (swpagerdebug & SDB_WRITE) 727 printf("swpg_clean: return %d\n", tspc ? TRUE : FALSE); 728 if ((swpagerdebug & SDB_ANOM) && tspc) 729 printf("swpg_clean: %s of cleaning page %x\n", 730 rw == B_READ ? "get" : "put", m); 731 #endif 732 return(tspc ? TRUE : FALSE); 733 } 734 735 swap_pager_finish(spc) 736 register swp_clean_t spc; 737 { 738 vm_object_t object = spc->spc_m->object; 739 740 /* 741 * Mark the paging operation as done. 742 * (XXX) If we cannot get the lock, leave it til later. 743 * (XXX) Also we are assuming that an async write is a 744 * pageout operation that has incremented the counter. 745 */ 746 if (!vm_object_lock_try(object)) 747 return(0); 748 749 #ifdef DEBUG 750 spc->spc_m->pagerowned = 0; 751 #endif 752 753 if (--object->paging_in_progress == 0) 754 thread_wakeup((int) object); 755 756 /* 757 * XXX: this isn't even close to the right thing to do, 758 * introduces a variety of race conditions. 759 * 760 * If dirty, vm_pageout() has attempted to clean the page 761 * again. In this case we do not do anything as we will 762 * see the page again shortly. Otherwise, if no error mark 763 * as clean and inform the pmap system. If error, mark as 764 * dirty so we will try again (XXX: could get stuck doing 765 * this, should give up after awhile). 766 */ 767 if ((spc->spc_flags & SPC_DIRTY) == 0) { 768 if (spc->spc_flags & SPC_ERROR) { 769 printf("swap_pager: clean of %x failed\n", 770 VM_PAGE_TO_PHYS(spc->spc_m)); 771 spc->spc_m->laundry = TRUE; 772 } else { 773 spc->spc_m->clean = TRUE; 774 pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m)); 775 } 776 } 777 /* 778 * XXX: allow blocked write faults to continue 779 */ 780 spc->spc_m->page_lock = spc->spc_m->unlock_request = VM_PROT_NONE; 781 PAGE_WAKEUP(spc->spc_m); 782 783 vm_object_unlock(object); 784 return(1); 785 } 786 787 swap_pager_iodone(bp) 788 register struct buf *bp; 789 { 790 register swp_clean_t spc; 791 daddr_t blk; 792 int s; 793 794 #ifdef DEBUG 795 /* save panic time state */ 796 if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) 797 return; 798 if (swpagerdebug & SDB_FOLLOW) 799 printf("swpg_iodone(%x)\n", bp); 800 #endif 801 s = splbio(); 802 spc = (swp_clean_t) queue_first(&swap_pager_inuse); 803 while (!queue_end(&swap_pager_inuse, (queue_entry_t)spc)) { 804 if (spc->spc_bp == bp) 805 break; 806 spc = (swp_clean_t) queue_next(&spc->spc_list); 807 } 808 #ifdef DEBUG 809 if (queue_end(&swap_pager_inuse, (queue_entry_t)spc)) 810 panic("swpg_iodone: bp not found"); 811 #endif 812 813 spc->spc_flags &= ~SPC_BUSY; 814 spc->spc_flags |= SPC_DONE; 815 if (bp->b_flags & B_ERROR) 816 spc->spc_flags |= SPC_ERROR; 817 spc->spc_bp = NULL; 818 blk = bp->b_blkno; 819 820 #ifdef DEBUG 821 --swap_pager_poip; 822 if (swpagerdebug & SDB_WRITE) 823 printf("swpg_iodone: bp=%x swp=%x flags=%x spc=%x poip=%x\n", 824 bp, spc->spc_swp, spc->spc_swp->sw_flags, 825 spc, spc->spc_swp->sw_poip); 826 #endif 827 828 spc->spc_swp->sw_poip--; 829 if (spc->spc_swp->sw_flags & SW_WANTED) { 830 spc->spc_swp->sw_flags &= ~SW_WANTED; 831 thread_wakeup((int)spc->spc_swp); 832 } 833 834 bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_PAGET|B_UAREA|B_DIRTY); 835 bp->av_forw = bswlist.av_forw; 836 bswlist.av_forw = bp; 837 if (bp->b_vp) 838 brelvp(bp); 839 if (bswlist.b_flags & B_WANTED) { 840 bswlist.b_flags &= ~B_WANTED; 841 thread_wakeup((int)&bswlist); 842 } 843 #if 0 844 /* 845 * XXX: this isn't even close to the right thing to do, 846 * introduces a variety of race conditions. 847 * 848 * If dirty, vm_pageout() has attempted to clean the page 849 * again. In this case we do not do anything as we will 850 * see the page again shortly. Otherwise, if no error mark 851 * as clean and inform the pmap system. If error, mark as 852 * dirty so we will try again (XXX: could get stuck doing 853 * this, should give up after awhile). 854 */ 855 if ((spc->spc_flags & SPC_DIRTY) == 0) { 856 if (spc->spc_flags & SPC_ERROR) { 857 printf("swap_pager: clean of %x (block %x) failed\n", 858 VM_PAGE_TO_PHYS(spc->spc_m), blk); 859 spc->spc_m->laundry = TRUE; 860 } else { 861 spc->spc_m->clean = TRUE; 862 pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m)); 863 } 864 } 865 /* 866 * XXX: allow blocked write faults to continue 867 */ 868 spc->spc_m->page_lock = spc->spc_m->unlock_request = VM_PROT_NONE; 869 PAGE_WAKEUP(spc->spc_m); 870 #endif 871 872 thread_wakeup((int) &vm_pages_needed); 873 splx(s); 874 } 875 #endif 876