1 /* $NetBSD: uvm_pager.c,v 1.57 2002/05/15 06:57:50 matt Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp 35 */ 36 37 /* 38 * uvm_pager.c: generic functions used to assist the pagers. 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.57 2002/05/15 06:57:50 matt Exp $"); 43 44 #include "opt_uvmhist.h" 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/malloc.h> 50 #include <sys/pool.h> 51 #include <sys/vnode.h> 52 53 #define UVM_PAGER 54 #include <uvm/uvm.h> 55 56 struct pool *uvm_aiobuf_pool; 57 58 /* 59 * list of uvm pagers in the system 60 */ 61 62 struct uvm_pagerops * const uvmpagerops[] = { 63 &aobj_pager, 64 &uvm_deviceops, 65 &uvm_vnodeops, 66 &ubc_pager, 67 }; 68 69 /* 70 * the pager map: provides KVA for I/O 71 */ 72 73 struct vm_map *pager_map; /* XXX */ 74 struct simplelock pager_map_wanted_lock; 75 boolean_t pager_map_wanted; /* locked by pager map */ 76 static vaddr_t emergva; 77 static boolean_t emerginuse; 78 79 /* 80 * uvm_pager_init: init pagers (at boot time) 81 */ 82 83 void 84 uvm_pager_init() 85 { 86 int lcv; 87 vaddr_t sva, eva; 88 89 /* 90 * init pager map 91 */ 92 93 sva = 0; 94 pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, PAGER_MAP_SIZE, 0, 95 FALSE, NULL); 96 simple_lock_init(&pager_map_wanted_lock); 97 pager_map_wanted = FALSE; 98 emergva = uvm_km_valloc(kernel_map, MAXBSIZE); 99 emerginuse = FALSE; 100 101 /* 102 * init ASYNC I/O queue 103 */ 104 105 TAILQ_INIT(&uvm.aio_done); 106 107 /* 108 * call pager init functions 109 */ 110 for (lcv = 0 ; lcv < sizeof(uvmpagerops)/sizeof(struct uvm_pagerops *); 111 lcv++) { 112 if (uvmpagerops[lcv]->pgo_init) 113 uvmpagerops[lcv]->pgo_init(); 114 } 115 } 116 117 /* 118 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings 119 * 120 * we basically just map in a blank map entry to reserve the space in the 121 * map and then use pmap_enter() to put the mappings in by hand. 122 */ 123 124 vaddr_t 125 uvm_pagermapin(pps, npages, flags) 126 struct vm_page **pps; 127 int npages; 128 int flags; 129 { 130 vsize_t size; 131 vaddr_t kva; 132 vaddr_t cva; 133 struct vm_page *pp; 134 vm_prot_t prot; 135 UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist); 136 137 UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d)", pps, npages,0,0); 138 139 /* 140 * compute protection. outgoing I/O only needs read 141 * access to the page, whereas incoming needs read/write. 142 */ 143 144 prot = VM_PROT_READ; 145 if (flags & UVMPAGER_MAPIN_READ) 146 prot |= VM_PROT_WRITE; 147 148 ReStart: 149 size = npages << PAGE_SHIFT; 150 kva = 0; /* let system choose VA */ 151 152 if (uvm_map(pager_map, &kva, size, NULL, 153 UVM_UNKNOWN_OFFSET, 0, UVM_FLAG_NOMERGE) != 0) { 154 if (curproc == uvm.pagedaemon_proc) { 155 simple_lock(&pager_map_wanted_lock); 156 if (emerginuse) { 157 UVM_UNLOCK_AND_WAIT(&emergva, 158 &pager_map_wanted_lock, FALSE, 159 "emergva", 0); 160 goto ReStart; 161 } 162 emerginuse = TRUE; 163 simple_unlock(&pager_map_wanted_lock); 164 kva = emergva; 165 KASSERT(npages <= MAXBSIZE >> PAGE_SHIFT); 166 goto enter; 167 } 168 if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) { 169 UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0); 170 return(0); 171 } 172 simple_lock(&pager_map_wanted_lock); 173 pager_map_wanted = TRUE; 174 UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0); 175 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, FALSE, 176 "pager_map", 0); 177 goto ReStart; 178 } 179 180 enter: 181 /* got it */ 182 for (cva = kva ; size != 0 ; size -= PAGE_SIZE, cva += PAGE_SIZE) { 183 pp = *pps++; 184 KASSERT(pp); 185 KASSERT(pp->flags & PG_BUSY); 186 pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot); 187 } 188 pmap_update(vm_map_pmap(pager_map)); 189 190 UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0); 191 return(kva); 192 } 193 194 /* 195 * uvm_pagermapout: remove pager_map mapping 196 * 197 * we remove our mappings by hand and then remove the mapping (waking 198 * up anyone wanting space). 199 */ 200 201 void 202 uvm_pagermapout(kva, npages) 203 vaddr_t kva; 204 int npages; 205 { 206 vsize_t size = npages << PAGE_SHIFT; 207 struct vm_map_entry *entries; 208 UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist); 209 210 UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0); 211 212 /* 213 * duplicate uvm_unmap, but add in pager_map_wanted handling. 214 */ 215 216 pmap_kremove(kva, npages << PAGE_SHIFT); 217 if (kva == emergva) { 218 simple_lock(&pager_map_wanted_lock); 219 emerginuse = FALSE; 220 wakeup(&emergva); 221 simple_unlock(&pager_map_wanted_lock); 222 return; 223 } 224 225 vm_map_lock(pager_map); 226 uvm_unmap_remove(pager_map, kva, kva + size, &entries); 227 simple_lock(&pager_map_wanted_lock); 228 if (pager_map_wanted) { 229 pager_map_wanted = FALSE; 230 wakeup(pager_map); 231 } 232 simple_unlock(&pager_map_wanted_lock); 233 vm_map_unlock(pager_map); 234 if (entries) 235 uvm_unmap_detach(entries, 0); 236 pmap_update(pmap_kernel()); 237 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 238 } 239 240 /* 241 * interrupt-context iodone handler for nested i/o bufs. 242 * 243 * => must be at splbio(). 244 */ 245 246 void 247 uvm_aio_biodone1(bp) 248 struct buf *bp; 249 { 250 struct buf *mbp = bp->b_private; 251 252 KASSERT(mbp != bp); 253 if (bp->b_flags & B_ERROR) { 254 mbp->b_flags |= B_ERROR; 255 mbp->b_error = bp->b_error; 256 } 257 mbp->b_resid -= bp->b_bcount; 258 pool_put(&bufpool, bp); 259 if (mbp->b_resid == 0) { 260 biodone(mbp); 261 } 262 } 263 264 /* 265 * interrupt-context iodone handler for single-buf i/os 266 * or the top-level buf of a nested-buf i/o. 267 * 268 * => must be at splbio(). 269 */ 270 271 void 272 uvm_aio_biodone(bp) 273 struct buf *bp; 274 { 275 /* reset b_iodone for when this is a single-buf i/o. */ 276 bp->b_iodone = uvm_aio_aiodone; 277 278 simple_lock(&uvm.aiodoned_lock); /* locks uvm.aio_done */ 279 TAILQ_INSERT_TAIL(&uvm.aio_done, bp, b_freelist); 280 wakeup(&uvm.aiodoned); 281 simple_unlock(&uvm.aiodoned_lock); 282 } 283 284 /* 285 * uvm_aio_aiodone: do iodone processing for async i/os. 286 * this should be called in thread context, not interrupt context. 287 */ 288 289 void 290 uvm_aio_aiodone(bp) 291 struct buf *bp; 292 { 293 int npages = bp->b_bufsize >> PAGE_SHIFT; 294 struct vm_page *pg, *pgs[npages]; 295 struct uvm_object *uobj; 296 struct simplelock *slock; 297 int s, i, error, swslot; 298 boolean_t write, swap; 299 UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist); 300 UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0); 301 302 error = (bp->b_flags & B_ERROR) ? (bp->b_error ? bp->b_error : EIO) : 0; 303 write = (bp->b_flags & B_READ) == 0; 304 /* XXXUBC B_NOCACHE is for swap pager, should be done differently */ 305 if (write && !(bp->b_flags & B_NOCACHE) && bioops.io_pageiodone) { 306 (*bioops.io_pageiodone)(bp); 307 } 308 309 uobj = NULL; 310 for (i = 0; i < npages; i++) { 311 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); 312 UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0); 313 } 314 uvm_pagermapout((vaddr_t)bp->b_data, npages); 315 316 swslot = 0; 317 slock = NULL; 318 pg = pgs[0]; 319 swap = (pg->uanon != NULL && pg->uobject == NULL) || 320 (pg->pqflags & PQ_AOBJ) != 0; 321 if (!swap) { 322 uobj = pg->uobject; 323 slock = &uobj->vmobjlock; 324 simple_lock(slock); 325 uvm_lock_pageq(); 326 } else if (error) { 327 if (pg->uobject != NULL) { 328 swslot = uao_find_swslot(pg->uobject, pg->offset); 329 } else { 330 swslot = pg->uanon->an_swslot; 331 } 332 KASSERT(swslot); 333 } 334 for (i = 0; i < npages; i++) { 335 pg = pgs[i]; 336 KASSERT(swap || pg->uobject == uobj); 337 UVMHIST_LOG(ubchist, "pg %p", pg, 0,0,0); 338 339 /* 340 * for swap i/os, lock each page's object (or anon) 341 * individually since each page may need a different lock. 342 */ 343 344 if (swap) { 345 if (pg->uobject != NULL) { 346 slock = &pg->uobject->vmobjlock; 347 } else { 348 slock = &pg->uanon->an_lock; 349 } 350 simple_lock(slock); 351 uvm_lock_pageq(); 352 } 353 354 /* 355 * process errors. for reads, just mark the page to be freed. 356 * for writes, if the error was ENOMEM, we assume this was 357 * a transient failure so we mark the page dirty so that 358 * we'll try to write it again later. for all other write 359 * errors, we assume the error is permanent, thus the data 360 * in the page is lost. bummer. 361 */ 362 363 if (error) { 364 if (!write) { 365 pg->flags |= PG_RELEASED; 366 continue; 367 } else if (error == ENOMEM) { 368 if (pg->flags & PG_PAGEOUT) { 369 pg->flags &= ~PG_PAGEOUT; 370 uvmexp.paging--; 371 } 372 pg->flags &= ~PG_CLEAN; 373 uvm_pageactivate(pg); 374 } 375 } 376 377 /* 378 * if the page is PG_FAKE, this must have been a read to 379 * initialize the page. clear PG_FAKE and activate the page. 380 * we must also clear the pmap "modified" flag since it may 381 * still be set from the page's previous identity. 382 */ 383 384 if (pg->flags & PG_FAKE) { 385 KASSERT(!write); 386 pg->flags &= ~PG_FAKE; 387 uvm_pageactivate(pg); 388 pmap_clear_modify(pg); 389 } 390 391 /* 392 * do accounting for pagedaemon i/o and arrange to free 393 * the pages instead of just unbusying them. 394 */ 395 396 if (pg->flags & PG_PAGEOUT) { 397 pg->flags &= ~PG_PAGEOUT; 398 uvmexp.paging--; 399 pg->flags |= PG_RELEASED; 400 } 401 402 /* 403 * for swap pages, unlock everything for this page now. 404 */ 405 406 if (swap) { 407 uvm_page_unbusy(&pg, 1); 408 uvm_unlock_pageq(); 409 simple_unlock(slock); 410 } 411 } 412 if (!swap) { 413 uvm_page_unbusy(pgs, npages); 414 uvm_unlock_pageq(); 415 simple_unlock(slock); 416 } else { 417 KASSERT(write); 418 419 /* these pages are now only in swap. */ 420 simple_lock(&uvm.swap_data_lock); 421 KASSERT(uvmexp.swpgonly + npages <= uvmexp.swpginuse); 422 uvmexp.swpgonly += npages; 423 simple_unlock(&uvm.swap_data_lock); 424 if (error) { 425 uvm_swap_markbad(swslot, npages); 426 } 427 } 428 s = splbio(); 429 if (write && (bp->b_flags & B_AGE) != 0) { 430 vwakeup(bp); 431 } 432 pool_put(&bufpool, bp); 433 splx(s); 434 } 435