1 /* $NetBSD: uvm_swap.c,v 1.73 2002/11/02 07:40:49 perry Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 31 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.73 2002/11/02 07:40:49 perry Exp $"); 36 37 #include "fs_nfs.h" 38 #include "opt_uvmhist.h" 39 #include "opt_compat_netbsd.h" 40 #include "opt_ddb.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/buf.h> 45 #include <sys/conf.h> 46 #include <sys/proc.h> 47 #include <sys/namei.h> 48 #include <sys/disklabel.h> 49 #include <sys/errno.h> 50 #include <sys/kernel.h> 51 #include <sys/malloc.h> 52 #include <sys/vnode.h> 53 #include <sys/file.h> 54 #include <sys/extent.h> 55 #include <sys/mount.h> 56 #include <sys/pool.h> 57 #include <sys/syscallargs.h> 58 #include <sys/swap.h> 59 60 #include <uvm/uvm.h> 61 62 #include <miscfs/specfs/specdev.h> 63 64 /* 65 * uvm_swap.c: manage configuration and i/o to swap space. 66 */ 67 68 /* 69 * swap space is managed in the following way: 70 * 71 * each swap partition or file is described by a "swapdev" structure. 72 * each "swapdev" structure contains a "swapent" structure which contains 73 * information that is passed up to the user (via system calls). 74 * 75 * each swap partition is assigned a "priority" (int) which controls 76 * swap parition usage. 77 * 78 * the system maintains a global data structure describing all swap 79 * partitions/files. there is a sorted LIST of "swappri" structures 80 * which describe "swapdev"'s at that priority. this LIST is headed 81 * by the "swap_priority" global var. each "swappri" contains a 82 * CIRCLEQ of "swapdev" structures at that priority. 83 * 84 * locking: 85 * - swap_syscall_lock (sleep lock): this lock serializes the swapctl 86 * system call and prevents the swap priority list from changing 87 * while we are in the middle of a system call (e.g. SWAP_STATS). 88 * - uvm.swap_data_lock (simple_lock): this lock protects all swap data 89 * structures including the priority list, the swapdev structures, 90 * and the swapmap extent. 91 * 92 * each swap device has the following info: 93 * - swap device in use (could be disabled, preventing future use) 94 * - swap enabled (allows new allocations on swap) 95 * - map info in /dev/drum 96 * - vnode pointer 97 * for swap files only: 98 * - block size 99 * - max byte count in buffer 100 * - buffer 101 * 102 * userland controls and configures swap with the swapctl(2) system call. 103 * the sys_swapctl performs the following operations: 104 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 105 * [2] SWAP_STATS: given a pointer to an array of swapent structures 106 * (passed in via "arg") of a size passed in via "misc" ... we load 107 * the current swap config into the array. The actual work is done 108 * in the uvm_swap_stats(9) function. 109 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 110 * priority in "misc", start swapping on it. 111 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 112 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 113 * "misc") 114 */ 115 116 /* 117 * swapdev: describes a single swap partition/file 118 * 119 * note the following should be true: 120 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 121 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 122 */ 123 struct swapdev { 124 struct oswapent swd_ose; 125 #define swd_dev swd_ose.ose_dev /* device id */ 126 #define swd_flags swd_ose.ose_flags /* flags:inuse/enable/fake */ 127 #define swd_priority swd_ose.ose_priority /* our priority */ 128 /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */ 129 char *swd_path; /* saved pathname of device */ 130 int swd_pathlen; /* length of pathname */ 131 int swd_npages; /* #pages we can use */ 132 int swd_npginuse; /* #pages in use */ 133 int swd_npgbad; /* #pages bad */ 134 int swd_drumoffset; /* page0 offset in drum */ 135 int swd_drumsize; /* #pages in drum */ 136 struct extent *swd_ex; /* extent for this swapdev */ 137 char swd_exname[12]; /* name of extent above */ 138 struct vnode *swd_vp; /* backing vnode */ 139 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */ 140 141 int swd_bsize; /* blocksize (bytes) */ 142 int swd_maxactive; /* max active i/o reqs */ 143 struct bufq_state swd_tab; /* buffer list */ 144 int swd_active; /* number of active buffers */ 145 }; 146 147 /* 148 * swap device priority entry; the list is kept sorted on `spi_priority'. 149 */ 150 struct swappri { 151 int spi_priority; /* priority */ 152 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 153 /* circleq of swapdevs at this priority */ 154 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 155 }; 156 157 /* 158 * The following two structures are used to keep track of data transfers 159 * on swap devices associated with regular files. 160 * NOTE: this code is more or less a copy of vnd.c; we use the same 161 * structure names here to ease porting.. 162 */ 163 struct vndxfer { 164 struct buf *vx_bp; /* Pointer to parent buffer */ 165 struct swapdev *vx_sdp; 166 int vx_error; 167 int vx_pending; /* # of pending aux buffers */ 168 int vx_flags; 169 #define VX_BUSY 1 170 #define VX_DEAD 2 171 }; 172 173 struct vndbuf { 174 struct buf vb_buf; 175 struct vndxfer *vb_xfer; 176 }; 177 178 179 /* 180 * We keep a of pool vndbuf's and vndxfer structures. 181 */ 182 static struct pool vndxfer_pool; 183 static struct pool vndbuf_pool; 184 185 #define getvndxfer(vnx) do { \ 186 int s = splbio(); \ 187 vnx = pool_get(&vndxfer_pool, PR_WAITOK); \ 188 splx(s); \ 189 } while (/*CONSTCOND*/ 0) 190 191 #define putvndxfer(vnx) { \ 192 pool_put(&vndxfer_pool, (void *)(vnx)); \ 193 } 194 195 #define getvndbuf(vbp) do { \ 196 int s = splbio(); \ 197 vbp = pool_get(&vndbuf_pool, PR_WAITOK); \ 198 splx(s); \ 199 } while (/*CONSTCOND*/ 0) 200 201 #define putvndbuf(vbp) { \ 202 pool_put(&vndbuf_pool, (void *)(vbp)); \ 203 } 204 205 /* 206 * local variables 207 */ 208 static struct extent *swapmap; /* controls the mapping of /dev/drum */ 209 210 /* list of all active swap devices [by priority] */ 211 LIST_HEAD(swap_priority, swappri); 212 static struct swap_priority swap_priority; 213 214 /* locks */ 215 struct lock swap_syscall_lock; 216 217 /* 218 * prototypes 219 */ 220 static struct swapdev *swapdrum_getsdp __P((int)); 221 222 static struct swapdev *swaplist_find __P((struct vnode *, int)); 223 static void swaplist_insert __P((struct swapdev *, 224 struct swappri *, int)); 225 static void swaplist_trim __P((void)); 226 227 static int swap_on __P((struct proc *, struct swapdev *)); 228 static int swap_off __P((struct proc *, struct swapdev *)); 229 230 static void sw_reg_strategy __P((struct swapdev *, struct buf *, int)); 231 static void sw_reg_iodone __P((struct buf *)); 232 static void sw_reg_start __P((struct swapdev *)); 233 234 static int uvm_swap_io __P((struct vm_page **, int, int, int)); 235 236 dev_type_read(swread); 237 dev_type_write(swwrite); 238 dev_type_strategy(swstrategy); 239 240 const struct bdevsw swap_bdevsw = { 241 noopen, noclose, swstrategy, noioctl, nodump, nosize, 242 }; 243 244 const struct cdevsw swap_cdevsw = { 245 nullopen, nullclose, swread, swwrite, noioctl, 246 nostop, notty, nopoll, nommap, nokqfilter 247 }; 248 249 /* 250 * uvm_swap_init: init the swap system data structures and locks 251 * 252 * => called at boot time from init_main.c after the filesystems 253 * are brought up (which happens after uvm_init()) 254 */ 255 void 256 uvm_swap_init() 257 { 258 UVMHIST_FUNC("uvm_swap_init"); 259 260 UVMHIST_CALLED(pdhist); 261 /* 262 * first, init the swap list, its counter, and its lock. 263 * then get a handle on the vnode for /dev/drum by using 264 * the its dev_t number ("swapdev", from MD conf.c). 265 */ 266 267 LIST_INIT(&swap_priority); 268 uvmexp.nswapdev = 0; 269 lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0); 270 simple_lock_init(&uvm.swap_data_lock); 271 272 if (bdevvp(swapdev, &swapdev_vp)) 273 panic("uvm_swap_init: can't get vnode for swap device"); 274 275 /* 276 * create swap block resource map to map /dev/drum. the range 277 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 278 * that block 0 is reserved (used to indicate an allocation 279 * failure, or no allocation). 280 */ 281 swapmap = extent_create("swapmap", 1, INT_MAX, 282 M_VMSWAP, 0, 0, EX_NOWAIT); 283 if (swapmap == 0) 284 panic("uvm_swap_init: extent_create failed"); 285 286 /* 287 * allocate pools for structures used for swapping to files. 288 */ 289 290 pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, 291 "swp vnx", NULL); 292 293 pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, 294 "swp vnd", NULL); 295 296 /* 297 * done! 298 */ 299 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 300 } 301 302 /* 303 * swaplist functions: functions that operate on the list of swap 304 * devices on the system. 305 */ 306 307 /* 308 * swaplist_insert: insert swap device "sdp" into the global list 309 * 310 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 311 * => caller must provide a newly malloc'd swappri structure (we will 312 * FREE it if we don't need it... this it to prevent malloc blocking 313 * here while adding swap) 314 */ 315 static void 316 swaplist_insert(sdp, newspp, priority) 317 struct swapdev *sdp; 318 struct swappri *newspp; 319 int priority; 320 { 321 struct swappri *spp, *pspp; 322 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 323 324 /* 325 * find entry at or after which to insert the new device. 326 */ 327 pspp = NULL; 328 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 329 if (priority <= spp->spi_priority) 330 break; 331 pspp = spp; 332 } 333 334 /* 335 * new priority? 336 */ 337 if (spp == NULL || spp->spi_priority != priority) { 338 spp = newspp; /* use newspp! */ 339 UVMHIST_LOG(pdhist, "created new swappri = %d", 340 priority, 0, 0, 0); 341 342 spp->spi_priority = priority; 343 CIRCLEQ_INIT(&spp->spi_swapdev); 344 345 if (pspp) 346 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 347 else 348 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 349 } else { 350 /* we don't need a new priority structure, free it */ 351 FREE(newspp, M_VMSWAP); 352 } 353 354 /* 355 * priority found (or created). now insert on the priority's 356 * circleq list and bump the total number of swapdevs. 357 */ 358 sdp->swd_priority = priority; 359 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 360 uvmexp.nswapdev++; 361 } 362 363 /* 364 * swaplist_find: find and optionally remove a swap device from the 365 * global list. 366 * 367 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 368 * => we return the swapdev we found (and removed) 369 */ 370 static struct swapdev * 371 swaplist_find(vp, remove) 372 struct vnode *vp; 373 boolean_t remove; 374 { 375 struct swapdev *sdp; 376 struct swappri *spp; 377 378 /* 379 * search the lists for the requested vp 380 */ 381 382 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 383 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 384 if (sdp->swd_vp == vp) { 385 if (remove) { 386 CIRCLEQ_REMOVE(&spp->spi_swapdev, 387 sdp, swd_next); 388 uvmexp.nswapdev--; 389 } 390 return(sdp); 391 } 392 } 393 } 394 return (NULL); 395 } 396 397 398 /* 399 * swaplist_trim: scan priority list for empty priority entries and kill 400 * them. 401 * 402 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 403 */ 404 static void 405 swaplist_trim() 406 { 407 struct swappri *spp, *nextspp; 408 409 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) { 410 nextspp = LIST_NEXT(spp, spi_swappri); 411 if (CIRCLEQ_FIRST(&spp->spi_swapdev) != 412 (void *)&spp->spi_swapdev) 413 continue; 414 LIST_REMOVE(spp, spi_swappri); 415 free(spp, M_VMSWAP); 416 } 417 } 418 419 /* 420 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 421 * to the "swapdev" that maps that section of the drum. 422 * 423 * => each swapdev takes one big contig chunk of the drum 424 * => caller must hold uvm.swap_data_lock 425 */ 426 static struct swapdev * 427 swapdrum_getsdp(pgno) 428 int pgno; 429 { 430 struct swapdev *sdp; 431 struct swappri *spp; 432 433 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 434 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 435 if (sdp->swd_flags & SWF_FAKE) 436 continue; 437 if (pgno >= sdp->swd_drumoffset && 438 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 439 return sdp; 440 } 441 } 442 } 443 return NULL; 444 } 445 446 447 /* 448 * sys_swapctl: main entry point for swapctl(2) system call 449 * [with two helper functions: swap_on and swap_off] 450 */ 451 int 452 sys_swapctl(p, v, retval) 453 struct proc *p; 454 void *v; 455 register_t *retval; 456 { 457 struct sys_swapctl_args /* { 458 syscallarg(int) cmd; 459 syscallarg(void *) arg; 460 syscallarg(int) misc; 461 } */ *uap = (struct sys_swapctl_args *)v; 462 struct vnode *vp; 463 struct nameidata nd; 464 struct swappri *spp; 465 struct swapdev *sdp; 466 struct swapent *sep; 467 char userpath[PATH_MAX + 1]; 468 size_t len; 469 int error, misc; 470 int priority; 471 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 472 473 misc = SCARG(uap, misc); 474 475 /* 476 * ensure serialized syscall access by grabbing the swap_syscall_lock 477 */ 478 lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL); 479 480 /* 481 * we handle the non-priv NSWAP and STATS request first. 482 * 483 * SWAP_NSWAP: return number of config'd swap devices 484 * [can also be obtained with uvmexp sysctl] 485 */ 486 if (SCARG(uap, cmd) == SWAP_NSWAP) { 487 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev, 488 0, 0, 0); 489 *retval = uvmexp.nswapdev; 490 error = 0; 491 goto out; 492 } 493 494 /* 495 * SWAP_STATS: get stats on current # of configured swap devs 496 * 497 * note that the swap_priority list can't change as long 498 * as we are holding the swap_syscall_lock. we don't want 499 * to grab the uvm.swap_data_lock because we may fault&sleep during 500 * copyout() and we don't want to be holding that lock then! 501 */ 502 if (SCARG(uap, cmd) == SWAP_STATS 503 #if defined(COMPAT_13) 504 || SCARG(uap, cmd) == SWAP_OSTATS 505 #endif 506 ) { 507 misc = MIN(uvmexp.nswapdev, misc); 508 #if defined(COMPAT_13) 509 if (SCARG(uap, cmd) == SWAP_OSTATS) 510 len = sizeof(struct oswapent) * misc; 511 else 512 #endif 513 len = sizeof(struct swapent) * misc; 514 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK); 515 516 uvm_swap_stats(SCARG(uap, cmd), sep, misc, retval); 517 error = copyout(sep, (void *)SCARG(uap, arg), len); 518 519 free(sep, M_TEMP); 520 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 521 goto out; 522 } 523 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 524 dev_t *devp = (dev_t *)SCARG(uap, arg); 525 526 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 527 goto out; 528 } 529 530 /* 531 * all other requests require superuser privs. verify. 532 */ 533 if ((error = suser(p->p_ucred, &p->p_acflag))) 534 goto out; 535 536 /* 537 * at this point we expect a path name in arg. we will 538 * use namei() to gain a vnode reference (vref), and lock 539 * the vnode (VOP_LOCK). 540 * 541 * XXX: a NULL arg means use the root vnode pointer (e.g. for 542 * miniroot) 543 */ 544 if (SCARG(uap, arg) == NULL) { 545 vp = rootvp; /* miniroot */ 546 if (vget(vp, LK_EXCLUSIVE)) { 547 error = EBUSY; 548 goto out; 549 } 550 if (SCARG(uap, cmd) == SWAP_ON && 551 copystr("miniroot", userpath, sizeof userpath, &len)) 552 panic("swapctl: miniroot copy failed"); 553 } else { 554 int space; 555 char *where; 556 557 if (SCARG(uap, cmd) == SWAP_ON) { 558 if ((error = copyinstr(SCARG(uap, arg), userpath, 559 sizeof userpath, &len))) 560 goto out; 561 space = UIO_SYSSPACE; 562 where = userpath; 563 } else { 564 space = UIO_USERSPACE; 565 where = (char *)SCARG(uap, arg); 566 } 567 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, p); 568 if ((error = namei(&nd))) 569 goto out; 570 vp = nd.ni_vp; 571 } 572 /* note: "vp" is referenced and locked */ 573 574 error = 0; /* assume no error */ 575 switch(SCARG(uap, cmd)) { 576 577 case SWAP_DUMPDEV: 578 if (vp->v_type != VBLK) { 579 error = ENOTBLK; 580 break; 581 } 582 dumpdev = vp->v_rdev; 583 cpu_dumpconf(); 584 break; 585 586 case SWAP_CTL: 587 /* 588 * get new priority, remove old entry (if any) and then 589 * reinsert it in the correct place. finally, prune out 590 * any empty priority structures. 591 */ 592 priority = SCARG(uap, misc); 593 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 594 simple_lock(&uvm.swap_data_lock); 595 if ((sdp = swaplist_find(vp, 1)) == NULL) { 596 error = ENOENT; 597 } else { 598 swaplist_insert(sdp, spp, priority); 599 swaplist_trim(); 600 } 601 simple_unlock(&uvm.swap_data_lock); 602 if (error) 603 free(spp, M_VMSWAP); 604 break; 605 606 case SWAP_ON: 607 608 /* 609 * check for duplicates. if none found, then insert a 610 * dummy entry on the list to prevent someone else from 611 * trying to enable this device while we are working on 612 * it. 613 */ 614 615 priority = SCARG(uap, misc); 616 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK); 617 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 618 memset(sdp, 0, sizeof(*sdp)); 619 sdp->swd_flags = SWF_FAKE; 620 sdp->swd_vp = vp; 621 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 622 bufq_alloc(&sdp->swd_tab, BUFQ_DISKSORT|BUFQ_SORT_RAWBLOCK); 623 simple_lock(&uvm.swap_data_lock); 624 if (swaplist_find(vp, 0) != NULL) { 625 error = EBUSY; 626 simple_unlock(&uvm.swap_data_lock); 627 bufq_free(&sdp->swd_tab); 628 free(sdp, M_VMSWAP); 629 free(spp, M_VMSWAP); 630 break; 631 } 632 swaplist_insert(sdp, spp, priority); 633 simple_unlock(&uvm.swap_data_lock); 634 635 sdp->swd_pathlen = len; 636 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK); 637 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0) 638 panic("swapctl: copystr"); 639 640 /* 641 * we've now got a FAKE placeholder in the swap list. 642 * now attempt to enable swap on it. if we fail, undo 643 * what we've done and kill the fake entry we just inserted. 644 * if swap_on is a success, it will clear the SWF_FAKE flag 645 */ 646 647 if ((error = swap_on(p, sdp)) != 0) { 648 simple_lock(&uvm.swap_data_lock); 649 (void) swaplist_find(vp, 1); /* kill fake entry */ 650 swaplist_trim(); 651 simple_unlock(&uvm.swap_data_lock); 652 bufq_free(&sdp->swd_tab); 653 free(sdp->swd_path, M_VMSWAP); 654 free(sdp, M_VMSWAP); 655 break; 656 } 657 break; 658 659 case SWAP_OFF: 660 simple_lock(&uvm.swap_data_lock); 661 if ((sdp = swaplist_find(vp, 0)) == NULL) { 662 simple_unlock(&uvm.swap_data_lock); 663 error = ENXIO; 664 break; 665 } 666 667 /* 668 * If a device isn't in use or enabled, we 669 * can't stop swapping from it (again). 670 */ 671 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 672 simple_unlock(&uvm.swap_data_lock); 673 error = EBUSY; 674 break; 675 } 676 677 /* 678 * do the real work. 679 */ 680 error = swap_off(p, sdp); 681 break; 682 683 default: 684 error = EINVAL; 685 } 686 687 /* 688 * done! release the ref gained by namei() and unlock. 689 */ 690 vput(vp); 691 692 out: 693 lockmgr(&swap_syscall_lock, LK_RELEASE, NULL); 694 695 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 696 return (error); 697 } 698 699 /* 700 * swap_stats: implements swapctl(SWAP_STATS). The function is kept 701 * away from sys_swapctl() in order to allow COMPAT_* swapctl() 702 * emulation to use it directly without going through sys_swapctl(). 703 * The problem with using sys_swapctl() there is that it involves 704 * copying the swapent array to the stackgap, and this array's size 705 * is not known at build time. Hence it would not be possible to 706 * ensure it would fit in the stackgap in any case. 707 */ 708 void 709 uvm_swap_stats(cmd, sep, sec, retval) 710 int cmd; 711 struct swapent *sep; 712 int sec; 713 register_t *retval; 714 { 715 struct swappri *spp; 716 struct swapdev *sdp; 717 int count = 0; 718 719 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 720 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev); 721 sdp != (void *)&spp->spi_swapdev && sec-- > 0; 722 sdp = CIRCLEQ_NEXT(sdp, swd_next)) { 723 /* 724 * backwards compatibility for system call. 725 * note that we use 'struct oswapent' as an 726 * overlay into both 'struct swapdev' and 727 * the userland 'struct swapent', as we 728 * want to retain backwards compatibility 729 * with NetBSD 1.3. 730 */ 731 sdp->swd_ose.ose_inuse = 732 btodb((u_int64_t)sdp->swd_npginuse << 733 PAGE_SHIFT); 734 (void)memcpy(sep, &sdp->swd_ose, 735 sizeof(struct oswapent)); 736 737 /* now copy out the path if necessary */ 738 #if defined(COMPAT_13) 739 if (cmd == SWAP_STATS) 740 #endif 741 (void)memcpy(&sep->se_path, sdp->swd_path, 742 sdp->swd_pathlen); 743 744 count++; 745 #if defined(COMPAT_13) 746 if (cmd == SWAP_OSTATS) 747 sep = (struct swapent *) 748 ((struct oswapent *)sep + 1); 749 else 750 #endif 751 sep++; 752 } 753 } 754 755 *retval = count; 756 return; 757 } 758 759 /* 760 * swap_on: attempt to enable a swapdev for swapping. note that the 761 * swapdev is already on the global list, but disabled (marked 762 * SWF_FAKE). 763 * 764 * => we avoid the start of the disk (to protect disk labels) 765 * => we also avoid the miniroot, if we are swapping to root. 766 * => caller should leave uvm.swap_data_lock unlocked, we may lock it 767 * if needed. 768 */ 769 static int 770 swap_on(p, sdp) 771 struct proc *p; 772 struct swapdev *sdp; 773 { 774 static int count = 0; /* static */ 775 struct vnode *vp; 776 int error, npages, nblocks, size; 777 long addr; 778 u_long result; 779 struct vattr va; 780 #ifdef NFS 781 extern int (**nfsv2_vnodeop_p) __P((void *)); 782 #endif /* NFS */ 783 const struct bdevsw *bdev; 784 dev_t dev; 785 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 786 787 /* 788 * we want to enable swapping on sdp. the swd_vp contains 789 * the vnode we want (locked and ref'd), and the swd_dev 790 * contains the dev_t of the file, if it a block device. 791 */ 792 793 vp = sdp->swd_vp; 794 dev = sdp->swd_dev; 795 796 /* 797 * open the swap file (mostly useful for block device files to 798 * let device driver know what is up). 799 * 800 * we skip the open/close for root on swap because the root 801 * has already been opened when root was mounted (mountroot). 802 */ 803 if (vp != rootvp) { 804 if ((error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p))) 805 return (error); 806 } 807 808 /* XXX this only works for block devices */ 809 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 810 811 /* 812 * we now need to determine the size of the swap area. for 813 * block specials we can call the d_psize function. 814 * for normal files, we must stat [get attrs]. 815 * 816 * we put the result in nblks. 817 * for normal files, we also want the filesystem block size 818 * (which we get with statfs). 819 */ 820 switch (vp->v_type) { 821 case VBLK: 822 bdev = bdevsw_lookup(dev); 823 if (bdev == NULL || bdev->d_psize == NULL || 824 (nblocks = (*bdev->d_psize)(dev)) == -1) { 825 error = ENXIO; 826 goto bad; 827 } 828 break; 829 830 case VREG: 831 if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p))) 832 goto bad; 833 nblocks = (int)btodb(va.va_size); 834 if ((error = 835 VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0) 836 goto bad; 837 838 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; 839 /* 840 * limit the max # of outstanding I/O requests we issue 841 * at any one time. take it easy on NFS servers. 842 */ 843 #ifdef NFS 844 if (vp->v_op == nfsv2_vnodeop_p) 845 sdp->swd_maxactive = 2; /* XXX */ 846 else 847 #endif /* NFS */ 848 sdp->swd_maxactive = 8; /* XXX */ 849 break; 850 851 default: 852 error = ENXIO; 853 goto bad; 854 } 855 856 /* 857 * save nblocks in a safe place and convert to pages. 858 */ 859 860 sdp->swd_ose.ose_nblks = nblocks; 861 npages = dbtob((u_int64_t)nblocks) >> PAGE_SHIFT; 862 863 /* 864 * for block special files, we want to make sure that leave 865 * the disklabel and bootblocks alone, so we arrange to skip 866 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 867 * note that because of this the "size" can be less than the 868 * actual number of blocks on the device. 869 */ 870 if (vp->v_type == VBLK) { 871 /* we use pages 1 to (size - 1) [inclusive] */ 872 size = npages - 1; 873 addr = 1; 874 } else { 875 /* we use pages 0 to (size - 1) [inclusive] */ 876 size = npages; 877 addr = 0; 878 } 879 880 /* 881 * make sure we have enough blocks for a reasonable sized swap 882 * area. we want at least one page. 883 */ 884 885 if (size < 1) { 886 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 887 error = EINVAL; 888 goto bad; 889 } 890 891 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 892 893 /* 894 * now we need to allocate an extent to manage this swap device 895 */ 896 snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x", 897 count++); 898 899 /* note that extent_create's 3rd arg is inclusive, thus "- 1" */ 900 sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP, 901 0, 0, EX_WAITOK); 902 /* allocate the `saved' region from the extent so it won't be used */ 903 if (addr) { 904 if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK)) 905 panic("disklabel region"); 906 } 907 908 /* 909 * if the vnode we are swapping to is the root vnode 910 * (i.e. we are swapping to the miniroot) then we want 911 * to make sure we don't overwrite it. do a statfs to 912 * find its size and skip over it. 913 */ 914 if (vp == rootvp) { 915 struct mount *mp; 916 struct statfs *sp; 917 int rootblocks, rootpages; 918 919 mp = rootvnode->v_mount; 920 sp = &mp->mnt_stat; 921 rootblocks = sp->f_blocks * btodb(sp->f_bsize); 922 /* 923 * XXX: sp->f_blocks isn't the total number of 924 * blocks in the filesystem, it's the number of 925 * data blocks. so, our rootblocks almost 926 * definitely underestimates the total size 927 * of the filesystem - how badly depends on the 928 * details of the filesystem type. there isn't 929 * an obvious way to deal with this cleanly 930 * and perfectly, so for now we just pad our 931 * rootblocks estimate with an extra 5 percent. 932 */ 933 rootblocks += (rootblocks >> 5) + 934 (rootblocks >> 6) + 935 (rootblocks >> 7); 936 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 937 if (rootpages > size) 938 panic("swap_on: miniroot larger than swap?"); 939 940 if (extent_alloc_region(sdp->swd_ex, addr, 941 rootpages, EX_WAITOK)) 942 panic("swap_on: unable to preserve miniroot"); 943 944 size -= rootpages; 945 printf("Preserved %d pages of miniroot ", rootpages); 946 printf("leaving %d pages of swap\n", size); 947 } 948 949 /* 950 * try to add anons to reflect the new swap space. 951 */ 952 953 error = uvm_anon_add(size); 954 if (error) { 955 goto bad; 956 } 957 958 /* 959 * add a ref to vp to reflect usage as a swap device. 960 */ 961 vref(vp); 962 963 /* 964 * now add the new swapdev to the drum and enable. 965 */ 966 if (extent_alloc(swapmap, npages, EX_NOALIGN, EX_NOBOUNDARY, 967 EX_WAITOK, &result)) 968 panic("swapdrum_add"); 969 970 sdp->swd_drumoffset = (int)result; 971 sdp->swd_drumsize = npages; 972 sdp->swd_npages = size; 973 simple_lock(&uvm.swap_data_lock); 974 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 975 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 976 uvmexp.swpages += size; 977 simple_unlock(&uvm.swap_data_lock); 978 return (0); 979 980 /* 981 * failure: clean up and return error. 982 */ 983 984 bad: 985 if (sdp->swd_ex) { 986 extent_destroy(sdp->swd_ex); 987 } 988 if (vp != rootvp) { 989 (void)VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p); 990 } 991 return (error); 992 } 993 994 /* 995 * swap_off: stop swapping on swapdev 996 * 997 * => swap data should be locked, we will unlock. 998 */ 999 static int 1000 swap_off(p, sdp) 1001 struct proc *p; 1002 struct swapdev *sdp; 1003 { 1004 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 1005 UVMHIST_LOG(pdhist, " dev=%x", sdp->swd_dev,0,0,0); 1006 1007 /* disable the swap area being removed */ 1008 sdp->swd_flags &= ~SWF_ENABLE; 1009 simple_unlock(&uvm.swap_data_lock); 1010 1011 /* 1012 * the idea is to find all the pages that are paged out to this 1013 * device, and page them all in. in uvm, swap-backed pageable 1014 * memory can take two forms: aobjs and anons. call the 1015 * swapoff hook for each subsystem to bring in pages. 1016 */ 1017 1018 if (uao_swap_off(sdp->swd_drumoffset, 1019 sdp->swd_drumoffset + sdp->swd_drumsize) || 1020 anon_swap_off(sdp->swd_drumoffset, 1021 sdp->swd_drumoffset + sdp->swd_drumsize)) { 1022 1023 simple_lock(&uvm.swap_data_lock); 1024 sdp->swd_flags |= SWF_ENABLE; 1025 simple_unlock(&uvm.swap_data_lock); 1026 return ENOMEM; 1027 } 1028 KASSERT(sdp->swd_npginuse == sdp->swd_npgbad); 1029 1030 /* 1031 * done with the vnode. 1032 * drop our ref on the vnode before calling VOP_CLOSE() 1033 * so that spec_close() can tell if this is the last close. 1034 */ 1035 vrele(sdp->swd_vp); 1036 if (sdp->swd_vp != rootvp) { 1037 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p); 1038 } 1039 1040 /* remove anons from the system */ 1041 uvm_anon_remove(sdp->swd_npages); 1042 1043 simple_lock(&uvm.swap_data_lock); 1044 uvmexp.swpages -= sdp->swd_npages; 1045 1046 if (swaplist_find(sdp->swd_vp, 1) == NULL) 1047 panic("swap_off: swapdev not in list"); 1048 swaplist_trim(); 1049 simple_unlock(&uvm.swap_data_lock); 1050 1051 /* 1052 * free all resources! 1053 */ 1054 extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize, 1055 EX_WAITOK); 1056 extent_destroy(sdp->swd_ex); 1057 bufq_free(&sdp->swd_tab); 1058 free(sdp, M_VMSWAP); 1059 return (0); 1060 } 1061 1062 /* 1063 * /dev/drum interface and i/o functions 1064 */ 1065 1066 /* 1067 * swread: the read function for the drum (just a call to physio) 1068 */ 1069 /*ARGSUSED*/ 1070 int 1071 swread(dev, uio, ioflag) 1072 dev_t dev; 1073 struct uio *uio; 1074 int ioflag; 1075 { 1076 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1077 1078 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1079 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1080 } 1081 1082 /* 1083 * swwrite: the write function for the drum (just a call to physio) 1084 */ 1085 /*ARGSUSED*/ 1086 int 1087 swwrite(dev, uio, ioflag) 1088 dev_t dev; 1089 struct uio *uio; 1090 int ioflag; 1091 { 1092 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1093 1094 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1095 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1096 } 1097 1098 /* 1099 * swstrategy: perform I/O on the drum 1100 * 1101 * => we must map the i/o request from the drum to the correct swapdev. 1102 */ 1103 void 1104 swstrategy(bp) 1105 struct buf *bp; 1106 { 1107 struct swapdev *sdp; 1108 struct vnode *vp; 1109 int s, pageno, bn; 1110 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1111 1112 /* 1113 * convert block number to swapdev. note that swapdev can't 1114 * be yanked out from under us because we are holding resources 1115 * in it (i.e. the blocks we are doing I/O on). 1116 */ 1117 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1118 simple_lock(&uvm.swap_data_lock); 1119 sdp = swapdrum_getsdp(pageno); 1120 simple_unlock(&uvm.swap_data_lock); 1121 if (sdp == NULL) { 1122 bp->b_error = EINVAL; 1123 bp->b_flags |= B_ERROR; 1124 biodone(bp); 1125 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1126 return; 1127 } 1128 1129 /* 1130 * convert drum page number to block number on this swapdev. 1131 */ 1132 1133 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1134 bn = btodb((u_int64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1135 1136 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1137 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1138 sdp->swd_drumoffset, bn, bp->b_bcount); 1139 1140 /* 1141 * for block devices we finish up here. 1142 * for regular files we have to do more work which we delegate 1143 * to sw_reg_strategy(). 1144 */ 1145 1146 switch (sdp->swd_vp->v_type) { 1147 default: 1148 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type); 1149 1150 case VBLK: 1151 1152 /* 1153 * must convert "bp" from an I/O on /dev/drum to an I/O 1154 * on the swapdev (sdp). 1155 */ 1156 s = splbio(); 1157 bp->b_blkno = bn; /* swapdev block number */ 1158 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1159 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1160 1161 /* 1162 * if we are doing a write, we have to redirect the i/o on 1163 * drum's v_numoutput counter to the swapdevs. 1164 */ 1165 if ((bp->b_flags & B_READ) == 0) { 1166 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1167 vp->v_numoutput++; /* put it on swapdev */ 1168 } 1169 1170 /* 1171 * finally plug in swapdev vnode and start I/O 1172 */ 1173 bp->b_vp = vp; 1174 splx(s); 1175 VOP_STRATEGY(bp); 1176 return; 1177 1178 case VREG: 1179 /* 1180 * delegate to sw_reg_strategy function. 1181 */ 1182 sw_reg_strategy(sdp, bp, bn); 1183 return; 1184 } 1185 /* NOTREACHED */ 1186 } 1187 1188 /* 1189 * sw_reg_strategy: handle swap i/o to regular files 1190 */ 1191 static void 1192 sw_reg_strategy(sdp, bp, bn) 1193 struct swapdev *sdp; 1194 struct buf *bp; 1195 int bn; 1196 { 1197 struct vnode *vp; 1198 struct vndxfer *vnx; 1199 daddr_t nbn; 1200 caddr_t addr; 1201 off_t byteoff; 1202 int s, off, nra, error, sz, resid; 1203 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1204 1205 /* 1206 * allocate a vndxfer head for this transfer and point it to 1207 * our buffer. 1208 */ 1209 getvndxfer(vnx); 1210 vnx->vx_flags = VX_BUSY; 1211 vnx->vx_error = 0; 1212 vnx->vx_pending = 0; 1213 vnx->vx_bp = bp; 1214 vnx->vx_sdp = sdp; 1215 1216 /* 1217 * setup for main loop where we read filesystem blocks into 1218 * our buffer. 1219 */ 1220 error = 0; 1221 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1222 addr = bp->b_data; /* current position in buffer */ 1223 byteoff = dbtob((u_int64_t)bn); 1224 1225 for (resid = bp->b_resid; resid; resid -= sz) { 1226 struct vndbuf *nbp; 1227 1228 /* 1229 * translate byteoffset into block number. return values: 1230 * vp = vnode of underlying device 1231 * nbn = new block number (on underlying vnode dev) 1232 * nra = num blocks we can read-ahead (excludes requested 1233 * block) 1234 */ 1235 nra = 0; 1236 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1237 &vp, &nbn, &nra); 1238 1239 if (error == 0 && nbn == (daddr_t)-1) { 1240 /* 1241 * this used to just set error, but that doesn't 1242 * do the right thing. Instead, it causes random 1243 * memory errors. The panic() should remain until 1244 * this condition doesn't destabilize the system. 1245 */ 1246 #if 1 1247 panic("sw_reg_strategy: swap to sparse file"); 1248 #else 1249 error = EIO; /* failure */ 1250 #endif 1251 } 1252 1253 /* 1254 * punt if there was an error or a hole in the file. 1255 * we must wait for any i/o ops we have already started 1256 * to finish before returning. 1257 * 1258 * XXX we could deal with holes here but it would be 1259 * a hassle (in the write case). 1260 */ 1261 if (error) { 1262 s = splbio(); 1263 vnx->vx_error = error; /* pass error up */ 1264 goto out; 1265 } 1266 1267 /* 1268 * compute the size ("sz") of this transfer (in bytes). 1269 */ 1270 off = byteoff % sdp->swd_bsize; 1271 sz = (1 + nra) * sdp->swd_bsize - off; 1272 if (sz > resid) 1273 sz = resid; 1274 1275 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1276 "vp %p/%p offset 0x%x/0x%x", 1277 sdp->swd_vp, vp, byteoff, nbn); 1278 1279 /* 1280 * now get a buf structure. note that the vb_buf is 1281 * at the front of the nbp structure so that you can 1282 * cast pointers between the two structure easily. 1283 */ 1284 getvndbuf(nbp); 1285 nbp->vb_buf.b_flags = bp->b_flags | B_CALL; 1286 nbp->vb_buf.b_bcount = sz; 1287 nbp->vb_buf.b_bufsize = sz; 1288 nbp->vb_buf.b_error = 0; 1289 nbp->vb_buf.b_data = addr; 1290 nbp->vb_buf.b_lblkno = 0; 1291 nbp->vb_buf.b_blkno = nbn + btodb(off); 1292 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1293 nbp->vb_buf.b_iodone = sw_reg_iodone; 1294 nbp->vb_buf.b_vp = vp; 1295 if (vp->v_type == VBLK) { 1296 nbp->vb_buf.b_dev = vp->v_rdev; 1297 } 1298 LIST_INIT(&nbp->vb_buf.b_dep); 1299 1300 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1301 1302 /* 1303 * Just sort by block number 1304 */ 1305 s = splbio(); 1306 if (vnx->vx_error != 0) { 1307 putvndbuf(nbp); 1308 goto out; 1309 } 1310 vnx->vx_pending++; 1311 1312 /* sort it in and start I/O if we are not over our limit */ 1313 BUFQ_PUT(&sdp->swd_tab, &nbp->vb_buf); 1314 sw_reg_start(sdp); 1315 splx(s); 1316 1317 /* 1318 * advance to the next I/O 1319 */ 1320 byteoff += sz; 1321 addr += sz; 1322 } 1323 1324 s = splbio(); 1325 1326 out: /* Arrive here at splbio */ 1327 vnx->vx_flags &= ~VX_BUSY; 1328 if (vnx->vx_pending == 0) { 1329 if (vnx->vx_error != 0) { 1330 bp->b_error = vnx->vx_error; 1331 bp->b_flags |= B_ERROR; 1332 } 1333 putvndxfer(vnx); 1334 biodone(bp); 1335 } 1336 splx(s); 1337 } 1338 1339 /* 1340 * sw_reg_start: start an I/O request on the requested swapdev 1341 * 1342 * => reqs are sorted by b_rawblkno (above) 1343 */ 1344 static void 1345 sw_reg_start(sdp) 1346 struct swapdev *sdp; 1347 { 1348 struct buf *bp; 1349 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1350 1351 /* recursion control */ 1352 if ((sdp->swd_flags & SWF_BUSY) != 0) 1353 return; 1354 1355 sdp->swd_flags |= SWF_BUSY; 1356 1357 while (sdp->swd_active < sdp->swd_maxactive) { 1358 bp = BUFQ_GET(&sdp->swd_tab); 1359 if (bp == NULL) 1360 break; 1361 sdp->swd_active++; 1362 1363 UVMHIST_LOG(pdhist, 1364 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1365 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1366 if ((bp->b_flags & B_READ) == 0) 1367 bp->b_vp->v_numoutput++; 1368 1369 VOP_STRATEGY(bp); 1370 } 1371 sdp->swd_flags &= ~SWF_BUSY; 1372 } 1373 1374 /* 1375 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1376 * 1377 * => note that we can recover the vndbuf struct by casting the buf ptr 1378 */ 1379 static void 1380 sw_reg_iodone(bp) 1381 struct buf *bp; 1382 { 1383 struct vndbuf *vbp = (struct vndbuf *) bp; 1384 struct vndxfer *vnx = vbp->vb_xfer; 1385 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1386 struct swapdev *sdp = vnx->vx_sdp; 1387 int s, resid, error; 1388 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1389 1390 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1391 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1392 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1393 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1394 1395 /* 1396 * protect vbp at splbio and update. 1397 */ 1398 1399 s = splbio(); 1400 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1401 pbp->b_resid -= resid; 1402 vnx->vx_pending--; 1403 1404 if (vbp->vb_buf.b_flags & B_ERROR) { 1405 /* pass error upward */ 1406 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO; 1407 UVMHIST_LOG(pdhist, " got error=%d !", error, 0, 0, 0); 1408 vnx->vx_error = error; 1409 } 1410 1411 /* 1412 * kill vbp structure 1413 */ 1414 putvndbuf(vbp); 1415 1416 /* 1417 * wrap up this transaction if it has run to completion or, in 1418 * case of an error, when all auxiliary buffers have returned. 1419 */ 1420 if (vnx->vx_error != 0) { 1421 /* pass error upward */ 1422 pbp->b_flags |= B_ERROR; 1423 pbp->b_error = vnx->vx_error; 1424 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1425 putvndxfer(vnx); 1426 biodone(pbp); 1427 } 1428 } else if (pbp->b_resid == 0) { 1429 KASSERT(vnx->vx_pending == 0); 1430 if ((vnx->vx_flags & VX_BUSY) == 0) { 1431 UVMHIST_LOG(pdhist, " iodone error=%d !", 1432 pbp, vnx->vx_error, 0, 0); 1433 putvndxfer(vnx); 1434 biodone(pbp); 1435 } 1436 } 1437 1438 /* 1439 * done! start next swapdev I/O if one is pending 1440 */ 1441 sdp->swd_active--; 1442 sw_reg_start(sdp); 1443 splx(s); 1444 } 1445 1446 1447 /* 1448 * uvm_swap_alloc: allocate space on swap 1449 * 1450 * => allocation is done "round robin" down the priority list, as we 1451 * allocate in a priority we "rotate" the circle queue. 1452 * => space can be freed with uvm_swap_free 1453 * => we return the page slot number in /dev/drum (0 == invalid slot) 1454 * => we lock uvm.swap_data_lock 1455 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1456 */ 1457 int 1458 uvm_swap_alloc(nslots, lessok) 1459 int *nslots; /* IN/OUT */ 1460 boolean_t lessok; 1461 { 1462 struct swapdev *sdp; 1463 struct swappri *spp; 1464 u_long result; 1465 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1466 1467 /* 1468 * no swap devices configured yet? definite failure. 1469 */ 1470 if (uvmexp.nswapdev < 1) 1471 return 0; 1472 1473 /* 1474 * lock data lock, convert slots into blocks, and enter loop 1475 */ 1476 simple_lock(&uvm.swap_data_lock); 1477 1478 ReTry: /* XXXMRG */ 1479 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1480 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1481 /* if it's not enabled, then we can't swap from it */ 1482 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1483 continue; 1484 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1485 continue; 1486 if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN, 1487 EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT, 1488 &result) != 0) { 1489 continue; 1490 } 1491 1492 /* 1493 * successful allocation! now rotate the circleq. 1494 */ 1495 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1496 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1497 sdp->swd_npginuse += *nslots; 1498 uvmexp.swpginuse += *nslots; 1499 simple_unlock(&uvm.swap_data_lock); 1500 /* done! return drum slot number */ 1501 UVMHIST_LOG(pdhist, 1502 "success! returning %d slots starting at %d", 1503 *nslots, result + sdp->swd_drumoffset, 0, 0); 1504 return (result + sdp->swd_drumoffset); 1505 } 1506 } 1507 1508 /* XXXMRG: BEGIN HACK */ 1509 if (*nslots > 1 && lessok) { 1510 *nslots = 1; 1511 goto ReTry; /* XXXMRG: ugh! extent should support this for us */ 1512 } 1513 /* XXXMRG: END HACK */ 1514 1515 simple_unlock(&uvm.swap_data_lock); 1516 return 0; 1517 } 1518 1519 /* 1520 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1521 * 1522 * => we lock uvm.swap_data_lock 1523 */ 1524 void 1525 uvm_swap_markbad(startslot, nslots) 1526 int startslot; 1527 int nslots; 1528 { 1529 struct swapdev *sdp; 1530 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1531 1532 simple_lock(&uvm.swap_data_lock); 1533 sdp = swapdrum_getsdp(startslot); 1534 1535 /* 1536 * we just keep track of how many pages have been marked bad 1537 * in this device, to make everything add up in swap_off(). 1538 * we assume here that the range of slots will all be within 1539 * one swap device. 1540 */ 1541 1542 sdp->swd_npgbad += nslots; 1543 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1544 simple_unlock(&uvm.swap_data_lock); 1545 } 1546 1547 /* 1548 * uvm_swap_free: free swap slots 1549 * 1550 * => this can be all or part of an allocation made by uvm_swap_alloc 1551 * => we lock uvm.swap_data_lock 1552 */ 1553 void 1554 uvm_swap_free(startslot, nslots) 1555 int startslot; 1556 int nslots; 1557 { 1558 struct swapdev *sdp; 1559 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1560 1561 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1562 startslot, 0, 0); 1563 1564 /* 1565 * ignore attempts to free the "bad" slot. 1566 */ 1567 1568 if (startslot == SWSLOT_BAD) { 1569 return; 1570 } 1571 1572 /* 1573 * convert drum slot offset back to sdp, free the blocks 1574 * in the extent, and return. must hold pri lock to do 1575 * lookup and access the extent. 1576 */ 1577 1578 simple_lock(&uvm.swap_data_lock); 1579 sdp = swapdrum_getsdp(startslot); 1580 KASSERT(uvmexp.nswapdev >= 1); 1581 KASSERT(sdp != NULL); 1582 KASSERT(sdp->swd_npginuse >= nslots); 1583 if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots, 1584 EX_MALLOCOK|EX_NOWAIT) != 0) { 1585 printf("warning: resource shortage: %d pages of swap lost\n", 1586 nslots); 1587 } 1588 sdp->swd_npginuse -= nslots; 1589 uvmexp.swpginuse -= nslots; 1590 simple_unlock(&uvm.swap_data_lock); 1591 } 1592 1593 /* 1594 * uvm_swap_put: put any number of pages into a contig place on swap 1595 * 1596 * => can be sync or async 1597 */ 1598 1599 int 1600 uvm_swap_put(swslot, ppsp, npages, flags) 1601 int swslot; 1602 struct vm_page **ppsp; 1603 int npages; 1604 int flags; 1605 { 1606 int error; 1607 1608 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1609 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1610 return error; 1611 } 1612 1613 /* 1614 * uvm_swap_get: get a single page from swap 1615 * 1616 * => usually a sync op (from fault) 1617 */ 1618 1619 int 1620 uvm_swap_get(page, swslot, flags) 1621 struct vm_page *page; 1622 int swslot, flags; 1623 { 1624 int error; 1625 1626 uvmexp.nswget++; 1627 KASSERT(flags & PGO_SYNCIO); 1628 if (swslot == SWSLOT_BAD) { 1629 return EIO; 1630 } 1631 error = uvm_swap_io(&page, swslot, 1, B_READ | 1632 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1633 if (error == 0) { 1634 1635 /* 1636 * this page is no longer only in swap. 1637 */ 1638 1639 simple_lock(&uvm.swap_data_lock); 1640 KASSERT(uvmexp.swpgonly > 0); 1641 uvmexp.swpgonly--; 1642 simple_unlock(&uvm.swap_data_lock); 1643 } 1644 return error; 1645 } 1646 1647 /* 1648 * uvm_swap_io: do an i/o operation to swap 1649 */ 1650 1651 static int 1652 uvm_swap_io(pps, startslot, npages, flags) 1653 struct vm_page **pps; 1654 int startslot, npages, flags; 1655 { 1656 daddr_t startblk; 1657 struct buf *bp; 1658 vaddr_t kva; 1659 int error, s, mapinflags; 1660 boolean_t write, async; 1661 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1662 1663 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1664 startslot, npages, flags, 0); 1665 1666 write = (flags & B_READ) == 0; 1667 async = (flags & B_ASYNC) != 0; 1668 1669 /* 1670 * convert starting drum slot to block number 1671 */ 1672 1673 startblk = btodb((u_int64_t)startslot << PAGE_SHIFT); 1674 1675 /* 1676 * first, map the pages into the kernel. 1677 */ 1678 1679 mapinflags = !write ? 1680 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1681 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1682 kva = uvm_pagermapin(pps, npages, mapinflags); 1683 1684 /* 1685 * now allocate a buf for the i/o. 1686 */ 1687 1688 s = splbio(); 1689 bp = pool_get(&bufpool, PR_WAITOK); 1690 splx(s); 1691 1692 /* 1693 * fill in the bp/sbp. we currently route our i/o through 1694 * /dev/drum's vnode [swapdev_vp]. 1695 */ 1696 1697 bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC)); 1698 bp->b_proc = &proc0; /* XXX */ 1699 bp->b_vnbufs.le_next = NOLIST; 1700 bp->b_data = (caddr_t)kva; 1701 bp->b_blkno = startblk; 1702 bp->b_vp = swapdev_vp; 1703 bp->b_dev = swapdev_vp->v_rdev; 1704 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1705 LIST_INIT(&bp->b_dep); 1706 1707 /* 1708 * bump v_numoutput (counter of number of active outputs). 1709 */ 1710 1711 if (write) { 1712 s = splbio(); 1713 swapdev_vp->v_numoutput++; 1714 splx(s); 1715 } 1716 1717 /* 1718 * for async ops we must set up the iodone handler. 1719 */ 1720 1721 if (async) { 1722 bp->b_flags |= B_CALL; 1723 bp->b_iodone = uvm_aio_biodone; 1724 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1725 } 1726 UVMHIST_LOG(pdhist, 1727 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1728 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1729 1730 /* 1731 * now we start the I/O, and if async, return. 1732 */ 1733 1734 VOP_STRATEGY(bp); 1735 if (async) 1736 return 0; 1737 1738 /* 1739 * must be sync i/o. wait for it to finish 1740 */ 1741 1742 error = biowait(bp); 1743 1744 /* 1745 * kill the pager mapping 1746 */ 1747 1748 uvm_pagermapout(kva, npages); 1749 1750 /* 1751 * now dispose of the buf and we're done. 1752 */ 1753 1754 s = splbio(); 1755 if (write) 1756 vwakeup(bp); 1757 pool_put(&bufpool, bp); 1758 splx(s); 1759 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1760 return (error); 1761 } 1762