1 /* $OpenBSD: uvm_amap.c,v 1.7 2001/01/29 02:07:42 niklas Exp $ */ 2 /* $NetBSD: uvm_amap.c,v 1.20 1999/04/11 04:04:11 chs Exp $ */ 3 4 /* 5 * 6 * Copyright (c) 1997 Charles D. Cranor and Washington University. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by Charles D. Cranor and 20 * Washington University. 21 * 4. The name of the author may not be used to endorse or promote products 22 * derived from this software without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * uvm_amap.c: amap operations 38 */ 39 40 /* 41 * this file contains functions that perform operations on amaps. see 42 * uvm_amap.h for a brief explanation of the role of amaps in uvm. 43 */ 44 45 #undef UVM_AMAP_INLINE /* enable/disable amap inlines */ 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/proc.h> 50 #include <sys/malloc.h> 51 #include <sys/pool.h> 52 53 #include <vm/vm.h> 54 #include <vm/vm_page.h> 55 #include <vm/vm_kern.h> 56 57 #define UVM_AMAP_C /* ensure disabled inlines are in */ 58 #include <uvm/uvm.h> 59 #include <uvm/uvm_swap.h> 60 61 /* 62 * pool for allocation of vm_map structures. note that the pool has 63 * its own simplelock for its protection. also note that in order to 64 * avoid an endless loop, the amap pool's allocator cannot allocate 65 * memory from an amap (it currently goes through the kernel uobj, so 66 * we are ok). 67 */ 68 69 struct pool uvm_amap_pool; 70 71 /* 72 * local functions 73 */ 74 75 static struct vm_amap *amap_alloc1 __P((int, int, int)); 76 77 #ifdef UVM_AMAP_PPREF 78 /* 79 * what is ppref? ppref is an _optional_ amap feature which is used 80 * to keep track of reference counts on a per-page basis. it is enabled 81 * when UVM_AMAP_PPREF is defined. 82 * 83 * when enabled, an array of ints is allocated for the pprefs. this 84 * array is allocated only when a partial reference is added to the 85 * map (either by unmapping part of the amap, or gaining a reference 86 * to only a part of an amap). if the malloc of the array fails 87 * (M_NOWAIT), then we set the array pointer to PPREF_NONE to indicate 88 * that we tried to do ppref's but couldn't alloc the array so just 89 * give up (after all, this is an optional feature!). 90 * 91 * the array is divided into page sized "chunks." for chunks of length 1, 92 * the chunk reference count plus one is stored in that chunk's slot. 93 * for chunks of length > 1 the first slot contains (the reference count 94 * plus one) * -1. [the negative value indicates that the length is 95 * greater than one.] the second slot of the chunk contains the length 96 * of the chunk. here is an example: 97 * 98 * actual REFS: 2 2 2 2 3 1 1 0 0 0 4 4 0 1 1 1 99 * ppref: -3 4 x x 4 -2 2 -1 3 x -5 2 1 -2 3 x 100 * <----------><-><----><-------><----><-><-------> 101 * (x = don't care) 102 * 103 * this allows us to allow one int to contain the ref count for the whole 104 * chunk. note that the "plus one" part is needed because a reference 105 * count of zero is neither positive or negative (need a way to tell 106 * if we've got one zero or a bunch of them). 107 * 108 * here are some in-line functions to help us. 109 */ 110 111 static __inline void pp_getreflen __P((int *, int, int *, int *)); 112 static __inline void pp_setreflen __P((int *, int, int, int)); 113 114 /* 115 * pp_getreflen: get the reference and length for a specific offset 116 * 117 * => ppref's amap must be locked 118 */ 119 static __inline void 120 pp_getreflen(ppref, offset, refp, lenp) 121 int *ppref, offset, *refp, *lenp; 122 { 123 124 if (ppref[offset] > 0) { /* chunk size must be 1 */ 125 *refp = ppref[offset] - 1; /* don't forget to adjust */ 126 *lenp = 1; 127 } else { 128 *refp = (ppref[offset] * -1) - 1; 129 *lenp = ppref[offset+1]; 130 } 131 } 132 133 /* 134 * pp_setreflen: set the reference and length for a specific offset 135 * 136 * => ppref's amap must be locked 137 */ 138 static __inline void 139 pp_setreflen(ppref, offset, ref, len) 140 int *ppref, offset, ref, len; 141 { 142 if (len == 1) { 143 ppref[offset] = ref + 1; 144 } else { 145 ppref[offset] = (ref + 1) * -1; 146 ppref[offset+1] = len; 147 } 148 } 149 #endif 150 151 /* 152 * amap_init: called at boot time to init global amap data structures 153 */ 154 155 void 156 amap_init() 157 158 { 159 /* 160 * Initialize the vm_amap pool. 161 */ 162 pool_init(&uvm_amap_pool, sizeof(struct vm_amap), 0, 0, 0, 163 "amappl", 0, pool_page_alloc_nointr, pool_page_free_nointr, 164 M_UVMAMAP); 165 } 166 167 /* 168 * amap_alloc1: internal function that allocates an amap, but does not 169 * init the overlay. 170 * 171 * => lock on returned amap is init'd 172 */ 173 static inline struct vm_amap * 174 amap_alloc1(slots, padslots, waitf) 175 int slots, padslots, waitf; 176 { 177 struct vm_amap *amap; 178 int totalslots = slots + padslots; 179 180 amap = pool_get(&uvm_amap_pool, (waitf == M_WAITOK) ? PR_WAITOK : 0); 181 if (amap == NULL) 182 return(NULL); 183 184 simple_lock_init(&amap->am_l); 185 amap->am_ref = 1; 186 amap->am_flags = 0; 187 #ifdef UVM_AMAP_PPREF 188 amap->am_ppref = NULL; 189 #endif 190 amap->am_maxslot = totalslots; 191 amap->am_nslot = slots; 192 amap->am_nused = 0; 193 MALLOC(amap->am_slots, int *, totalslots * sizeof(int), M_UVMAMAP, waitf); 194 if (amap->am_slots) { 195 MALLOC(amap->am_bckptr, int *, totalslots * sizeof(int), M_UVMAMAP, waitf); 196 if (amap->am_bckptr) { 197 MALLOC(amap->am_anon, struct vm_anon **, 198 totalslots * sizeof(struct vm_anon *), M_UVMAMAP, waitf); 199 } 200 } 201 202 if (amap->am_anon) 203 return(amap); 204 205 if (amap->am_slots) { 206 FREE(amap->am_slots, M_UVMAMAP); 207 if (amap->am_bckptr) 208 FREE(amap->am_bckptr, M_UVMAMAP); 209 } 210 pool_put(&uvm_amap_pool, amap); 211 return (NULL); 212 } 213 214 /* 215 * amap_alloc: allocate an amap to manage "sz" bytes of anonymous VM 216 * 217 * => caller should ensure sz is a multiple of PAGE_SIZE 218 * => reference count to new amap is set to one 219 * => new amap is returned unlocked 220 */ 221 222 struct vm_amap * 223 amap_alloc(sz, padsz, waitf) 224 vaddr_t sz, padsz; 225 int waitf; 226 { 227 struct vm_amap *amap; 228 int slots, padslots; 229 UVMHIST_FUNC("amap_alloc"); UVMHIST_CALLED(maphist); 230 231 AMAP_B2SLOT(slots, sz); /* load slots */ 232 AMAP_B2SLOT(padslots, padsz); 233 234 amap = amap_alloc1(slots, padslots, waitf); 235 if (amap) 236 memset(amap->am_anon, 0, (slots + padslots) * sizeof(struct vm_anon *)); 237 238 UVMHIST_LOG(maphist,"<- done, amap = 0x%x, sz=%d", amap, sz, 0, 0); 239 return(amap); 240 } 241 242 243 /* 244 * amap_free: free an amap 245 * 246 * => the amap must be locked (mainly for simplelock accounting) 247 * => the amap should have a zero reference count and be empty 248 */ 249 void 250 amap_free(amap) 251 struct vm_amap *amap; 252 { 253 UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist); 254 255 #ifdef DIAGNOSTIC 256 if (amap->am_ref || amap->am_nused) 257 panic("amap_free"); 258 #endif 259 260 FREE(amap->am_slots, M_UVMAMAP); 261 FREE(amap->am_bckptr, M_UVMAMAP); 262 FREE(amap->am_anon, M_UVMAMAP); 263 #ifdef UVM_AMAP_PPREF 264 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) 265 FREE(amap->am_ppref, M_UVMAMAP); 266 #endif 267 amap_unlock(amap); /* mainly for lock debugging */ 268 pool_put(&uvm_amap_pool, amap); 269 270 UVMHIST_LOG(maphist,"<- done, freed amap = 0x%x", amap, 0, 0, 0); 271 } 272 273 /* 274 * amap_extend: extend the size of an amap (if needed) 275 * 276 * => called from uvm_map when we want to extend an amap to cover 277 * a new mapping (rather than allocate a new one) 278 * => amap should be unlocked (we will lock it) 279 * => to safely extend an amap it should have a reference count of 280 * one (thus it can't be shared) 281 * => XXXCDC: needs a waitflag or failure return value? 282 * => XXXCDC: support padding at this level? 283 */ 284 void 285 amap_extend(entry, addsize) 286 vm_map_entry_t entry; 287 vsize_t addsize; 288 { 289 struct vm_amap *amap = entry->aref.ar_amap; 290 int slotoff = entry->aref.ar_pageoff; 291 int slotmapped, slotadd, slotneed; 292 #ifdef UVM_AMAP_PPREF 293 int *newppref, *oldppref; 294 #endif 295 u_int *newsl, *newbck, *oldsl, *oldbck; 296 struct vm_anon **newover, **oldover; 297 int slotadded; 298 UVMHIST_FUNC("amap_extend"); UVMHIST_CALLED(maphist); 299 300 UVMHIST_LOG(maphist, " (entry=0x%x, addsize=0x%x)", entry,addsize,0,0); 301 302 /* 303 * first, determine how many slots we need in the amap. don't 304 * forget that ar_pageoff could be non-zero: this means that 305 * there are some unused slots before us in the amap. 306 */ 307 308 amap_lock(amap); /* lock! */ 309 310 AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */ 311 AMAP_B2SLOT(slotadd, addsize); /* slots to add */ 312 slotneed = slotoff + slotmapped + slotadd; 313 314 /* 315 * case 1: we already have enough slots in the map and thus 316 * only need to bump the reference counts on the slots we are 317 * adding. 318 */ 319 320 if (amap->am_nslot >= slotneed) { 321 #ifdef UVM_AMAP_PPREF 322 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 323 amap_pp_adjref(amap, slotoff + slotmapped, addsize, 1); 324 } 325 #endif 326 amap_unlock(amap); 327 UVMHIST_LOG(maphist,"<- done (case 1), amap = 0x%x, sltneed=%d", 328 amap, slotneed, 0, 0); 329 return; /* done! */ 330 } 331 332 /* 333 * case 2: we pre-allocated slots for use and we just need to 334 * bump nslot up to take account for these slots. 335 */ 336 if (amap->am_maxslot >= slotneed) { 337 #ifdef UVM_AMAP_PPREF 338 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 339 if ((slotoff + slotmapped) < amap->am_nslot) 340 amap_pp_adjref(amap, slotoff + slotmapped, 341 (amap->am_nslot - (slotoff + slotmapped)) << 342 PAGE_SHIFT, 1); 343 pp_setreflen(amap->am_ppref, amap->am_nslot, 1, 344 slotneed - amap->am_nslot); 345 } 346 #endif 347 amap->am_nslot = slotneed; 348 amap_unlock(amap); 349 /* 350 * no need to zero am_anon since that was done at 351 * alloc time and we never shrink an allocation. 352 */ 353 UVMHIST_LOG(maphist,"<- done (case 2), amap = 0x%x, slotneed=%d", 354 amap, slotneed, 0, 0); 355 return; 356 } 357 358 /* 359 * case 3: we need to malloc a new amap and copy all the amap 360 * data over from old amap to the new one. 361 * 362 * XXXCDC: could we take advantage of a kernel realloc()? 363 */ 364 365 amap_unlock(amap); /* unlock in case we sleep in malloc */ 366 #ifdef UVM_AMAP_PPREF 367 newppref = NULL; 368 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 369 MALLOC(newppref, int *, slotneed * sizeof(int), M_UVMAMAP, 370 M_NOWAIT); 371 if (newppref == NULL) { 372 /* give up if malloc fails */ 373 FREE(amap->am_ppref, M_UVMAMAP); 374 amap->am_ppref = PPREF_NONE; 375 } 376 } 377 #endif 378 MALLOC(newsl, int *, slotneed * sizeof(int), M_UVMAMAP, M_WAITOK); 379 MALLOC(newbck, int *, slotneed * sizeof(int), M_UVMAMAP, M_WAITOK); 380 MALLOC(newover, struct vm_anon **, slotneed * sizeof(struct vm_anon *), 381 M_UVMAMAP, M_WAITOK); 382 amap_lock(amap); /* re-lock! */ 383 384 #ifdef DIAGNOSTIC 385 if (amap->am_maxslot >= slotneed) 386 panic("amap_extend: amap changed during malloc"); 387 #endif 388 389 /* 390 * now copy everything over to new malloc'd areas... 391 */ 392 393 slotadded = slotneed - amap->am_nslot; 394 395 /* do am_slots */ 396 oldsl = amap->am_slots; 397 memcpy(newsl, oldsl, sizeof(int) * amap->am_nused); 398 amap->am_slots = newsl; 399 400 /* do am_anon */ 401 oldover = amap->am_anon; 402 memcpy(newover, oldover, sizeof(struct vm_anon *) * amap->am_nslot); 403 memset(newover + amap->am_nslot, 0, sizeof(struct vm_anon *) * slotadded); 404 amap->am_anon = newover; 405 406 /* do am_bckptr */ 407 oldbck = amap->am_bckptr; 408 memcpy(newbck, oldbck, sizeof(int) * amap->am_nslot); 409 memset(newbck + amap->am_nslot, 0, sizeof(int) * slotadded); /* XXX: needed? */ 410 amap->am_bckptr = newbck; 411 412 #ifdef UVM_AMAP_PPREF 413 /* do ppref */ 414 oldppref = amap->am_ppref; 415 if (newppref) { 416 memcpy(newppref, oldppref, sizeof(int) * amap->am_nslot); 417 memset(newppref + amap->am_nslot, 0, sizeof(int) * slotadded); 418 amap->am_ppref = newppref; 419 if ((slotoff + slotmapped) < amap->am_nslot) 420 amap_pp_adjref(amap, slotoff + slotmapped, 421 (amap->am_nslot - (slotoff + slotmapped)) << 422 PAGE_SHIFT, 1); 423 pp_setreflen(newppref, amap->am_nslot, 1, slotadded); 424 } 425 #endif 426 427 /* update master values */ 428 amap->am_nslot = slotneed; 429 amap->am_maxslot = slotneed; 430 431 /* unlock */ 432 amap_unlock(amap); 433 434 /* and free */ 435 FREE(oldsl, M_UVMAMAP); 436 FREE(oldbck, M_UVMAMAP); 437 FREE(oldover, M_UVMAMAP); 438 #ifdef UVM_AMAP_PPREF 439 if (oldppref && oldppref != PPREF_NONE) 440 FREE(oldppref, M_UVMAMAP); 441 #endif 442 UVMHIST_LOG(maphist,"<- done (case 3), amap = 0x%x, slotneed=%d", 443 amap, slotneed, 0, 0); 444 } 445 446 /* 447 * amap_share_protect: change protection of anons in a shared amap 448 * 449 * for shared amaps, given the current data structure layout, it is 450 * not possible for us to directly locate all maps referencing the 451 * shared anon (to change the protection). in order to protect data 452 * in shared maps we use pmap_page_protect(). [this is useful for IPC 453 * mechanisms like map entry passing that may want to write-protect 454 * all mappings of a shared amap.] we traverse am_anon or am_slots 455 * depending on the current state of the amap. 456 * 457 * => entry's map and amap must be locked by the caller 458 */ 459 void 460 amap_share_protect(entry, prot) 461 vm_map_entry_t entry; 462 vm_prot_t prot; 463 { 464 struct vm_amap *amap = entry->aref.ar_amap; 465 int slots, lcv, slot, stop; 466 467 AMAP_B2SLOT(slots, (entry->end - entry->start)); 468 stop = entry->aref.ar_pageoff + slots; 469 470 if (slots < amap->am_nused) { 471 /* cheaper to traverse am_anon */ 472 for (lcv = entry->aref.ar_pageoff ; lcv < stop ; lcv++) { 473 if (amap->am_anon[lcv] == NULL) 474 continue; 475 if (amap->am_anon[lcv]->u.an_page != NULL) 476 pmap_page_protect( 477 PMAP_PGARG(amap->am_anon[lcv]->u.an_page), 478 prot); 479 } 480 return; 481 } 482 483 /* cheaper to traverse am_slots */ 484 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 485 slot = amap->am_slots[lcv]; 486 if (slot < entry->aref.ar_pageoff || slot >= stop) 487 continue; 488 if (amap->am_anon[slot]->u.an_page != NULL) 489 pmap_page_protect( 490 PMAP_PGARG(amap->am_anon[slot]->u.an_page), prot); 491 } 492 return; 493 } 494 495 /* 496 * amap_wipeout: wipeout all anon's in an amap; then free the amap! 497 * 498 * => called from amap_unref when the final reference to an amap is 499 * discarded (i.e. when reference count == 1) 500 * => the amap should be locked (by the caller) 501 */ 502 503 void 504 amap_wipeout(amap) 505 struct vm_amap *amap; 506 { 507 int lcv, slot; 508 struct vm_anon *anon; 509 UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist); 510 UVMHIST_LOG(maphist,"(amap=0x%x)", amap, 0,0,0); 511 512 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 513 int refs; 514 515 slot = amap->am_slots[lcv]; 516 anon = amap->am_anon[slot]; 517 518 if (anon == NULL || anon->an_ref == 0) 519 panic("amap_wipeout: corrupt amap"); 520 521 simple_lock(&anon->an_lock); /* lock anon */ 522 523 UVMHIST_LOG(maphist," processing anon 0x%x, ref=%d", anon, 524 anon->an_ref, 0, 0); 525 526 refs = --anon->an_ref; 527 simple_unlock(&anon->an_lock); 528 if (refs == 0) { 529 /* 530 * we had the last reference to a vm_anon. free it. 531 */ 532 uvm_anfree(anon); 533 } 534 } 535 536 /* 537 * now we free the map 538 */ 539 540 amap->am_ref = 0; /* ... was one */ 541 amap->am_nused = 0; 542 amap_free(amap); /* will unlock and free amap */ 543 UVMHIST_LOG(maphist,"<- done!", 0,0,0,0); 544 } 545 546 /* 547 * amap_copy: ensure that a map entry's "needs_copy" flag is false 548 * by copying the amap if necessary. 549 * 550 * => an entry with a null amap pointer will get a new (blank) one. 551 * => the map that the map entry belongs to must be locked by caller. 552 * => the amap currently attached to "entry" (if any) must be unlocked. 553 * => if canchunk is true, then we may clip the entry into a chunk 554 * => "startva" and "endva" are used only if canchunk is true. they are 555 * used to limit chunking (e.g. if you have a large space that you 556 * know you are going to need to allocate amaps for, there is no point 557 * in allowing that to be chunked) 558 */ 559 560 void 561 amap_copy(map, entry, waitf, canchunk, startva, endva) 562 vm_map_t map; 563 vm_map_entry_t entry; 564 int waitf; 565 boolean_t canchunk; 566 vaddr_t startva, endva; 567 { 568 struct vm_amap *amap, *srcamap; 569 int slots, lcv; 570 vaddr_t chunksize; 571 UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist); 572 UVMHIST_LOG(maphist, " (map=%p, entry=%p, waitf=%d)", map, entry, waitf, 0); 573 574 /* 575 * is there a map to copy? if not, create one from scratch. 576 */ 577 578 if (entry->aref.ar_amap == NULL) { 579 580 /* 581 * check to see if we have a large amap that we can 582 * chunk. we align startva/endva to chunk-sized 583 * boundaries and then clip to them. 584 */ 585 586 if (canchunk && atop(entry->end - entry->start) >= 587 UVM_AMAP_LARGE) { 588 /* convert slots to bytes */ 589 chunksize = UVM_AMAP_CHUNK << PAGE_SHIFT; 590 startva = (startva / chunksize) * chunksize; 591 endva = roundup(endva, chunksize); 592 UVMHIST_LOG(maphist, " chunk amap ==> clip 0x%x->0x%x" 593 "to 0x%x->0x%x", entry->start, entry->end, startva, 594 endva); 595 UVM_MAP_CLIP_START(map, entry, startva); 596 /* watch out for endva wrap-around! */ 597 if (endva >= startva) 598 UVM_MAP_CLIP_END(map, entry, endva); 599 } 600 601 UVMHIST_LOG(maphist, "<- done [creating new amap 0x%x->0x%x]", 602 entry->start, entry->end, 0, 0); 603 entry->aref.ar_pageoff = 0; 604 entry->aref.ar_amap = amap_alloc(entry->end - entry->start, 0, 605 waitf); 606 if (entry->aref.ar_amap != NULL) 607 entry->etype &= ~UVM_ET_NEEDSCOPY; 608 return; 609 } 610 611 /* 612 * first check and see if we are the only map entry 613 * referencing the amap we currently have. if so, then we can 614 * just take it over rather than copying it. note that we are 615 * reading am_ref with the amap unlocked... the value can only 616 * be one if we have the only reference to the amap (via our 617 * locked map). if we are greater than one we fall through to 618 * the next case (where we double check the value). 619 */ 620 621 if (entry->aref.ar_amap->am_ref == 1) { 622 entry->etype &= ~UVM_ET_NEEDSCOPY; 623 UVMHIST_LOG(maphist, "<- done [ref cnt = 1, took it over]", 624 0, 0, 0, 0); 625 return; 626 } 627 628 /* 629 * looks like we need to copy the map. 630 */ 631 632 UVMHIST_LOG(maphist," amap=%p, ref=%d, must copy it", 633 entry->aref.ar_amap, entry->aref.ar_amap->am_ref, 0, 0); 634 AMAP_B2SLOT(slots, entry->end - entry->start); 635 amap = amap_alloc1(slots, 0, waitf); 636 if (amap == NULL) { 637 UVMHIST_LOG(maphist, " amap_alloc1 failed", 0,0,0,0); 638 return; 639 } 640 srcamap = entry->aref.ar_amap; 641 amap_lock(srcamap); 642 643 /* 644 * need to double check reference count now that we've got the 645 * src amap locked down. the reference count could have 646 * changed while we were in malloc. if the reference count 647 * dropped down to one we take over the old map rather than 648 * copying the amap. 649 */ 650 651 if (srcamap->am_ref == 1) { /* take it over? */ 652 entry->etype &= ~UVM_ET_NEEDSCOPY; 653 amap->am_ref--; /* drop final reference to map */ 654 amap_free(amap); /* dispose of new (unused) amap */ 655 amap_unlock(srcamap); 656 return; 657 } 658 659 /* 660 * we must copy it now. 661 */ 662 663 UVMHIST_LOG(maphist, " copying amap now",0, 0, 0, 0); 664 for (lcv = 0 ; lcv < slots; lcv++) { 665 amap->am_anon[lcv] = 666 srcamap->am_anon[entry->aref.ar_pageoff + lcv]; 667 if (amap->am_anon[lcv] == NULL) 668 continue; 669 simple_lock(&amap->am_anon[lcv]->an_lock); 670 amap->am_anon[lcv]->an_ref++; 671 simple_unlock(&amap->am_anon[lcv]->an_lock); 672 amap->am_bckptr[lcv] = amap->am_nused; 673 amap->am_slots[amap->am_nused] = lcv; 674 amap->am_nused++; 675 } 676 677 /* 678 * drop our reference to the old amap (srcamap) and unlock. 679 * we know that the reference count on srcamap is greater than 680 * one (we checked above), so there is no way we could drop 681 * the count to zero. [and no need to worry about freeing it] 682 */ 683 684 srcamap->am_ref--; 685 if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0) 686 srcamap->am_flags &= ~AMAP_SHARED; /* clear shared flag */ 687 #ifdef UVM_AMAP_PPREF 688 if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) { 689 amap_pp_adjref(srcamap, entry->aref.ar_pageoff, 690 entry->end - entry->start, -1); 691 } 692 #endif 693 694 amap_unlock(srcamap); 695 696 /* 697 * install new amap. 698 */ 699 700 entry->aref.ar_pageoff = 0; 701 entry->aref.ar_amap = amap; 702 entry->etype &= ~UVM_ET_NEEDSCOPY; 703 704 /* 705 * done! 706 */ 707 UVMHIST_LOG(maphist, "<- done",0, 0, 0, 0); 708 } 709 710 /* 711 * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2) 712 * 713 * called during fork(2) when the parent process has a wired map 714 * entry. in that case we want to avoid write-protecting pages 715 * in the parent's map (e.g. like what you'd do for a COW page) 716 * so we resolve the COW here. 717 * 718 * => assume parent's entry was wired, thus all pages are resident. 719 * => assume pages that are loaned out (loan_count) are already mapped 720 * read-only in all maps, and thus no need for us to worry about them 721 * => assume both parent and child vm_map's are locked 722 * => caller passes child's map/entry in to us 723 * => if we run out of memory we will unlock the amap and sleep _with_ the 724 * parent and child vm_map's locked(!). we have to do this since 725 * we are in the middle of a fork(2) and we can't let the parent 726 * map change until we are done copying all the map entrys. 727 * => XXXCDC: out of memory should cause fork to fail, but there is 728 * currently no easy way to do this (needs fix) 729 * => page queues must be unlocked (we may lock them) 730 */ 731 732 void 733 amap_cow_now(map, entry) 734 struct vm_map *map; 735 struct vm_map_entry *entry; 736 { 737 struct vm_amap *amap = entry->aref.ar_amap; 738 int lcv, slot; 739 struct vm_anon *anon, *nanon; 740 struct vm_page *pg, *npg; 741 742 /* 743 * note that if we unlock the amap then we must ReStart the "lcv" for 744 * loop because some other process could reorder the anon's in the 745 * am_anon[] array on us while the lock is dropped. 746 */ 747 ReStart: 748 amap_lock(amap); 749 750 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 751 752 /* 753 * get the page 754 */ 755 756 slot = amap->am_slots[lcv]; 757 anon = amap->am_anon[slot]; 758 simple_lock(&anon->an_lock); 759 pg = anon->u.an_page; 760 761 /* 762 * page must be resident since parent is wired 763 */ 764 765 if (pg == NULL) 766 panic("amap_cow_now: non-resident wired page in anon %p", 767 anon); 768 769 /* 770 * if the anon ref count is one and the page is not loaned, 771 * then we are safe (the child has exclusive access to the 772 * page). if the page is loaned, then it must already be 773 * mapped read-only. 774 * 775 * we only need to get involved when these are not true. 776 * [note: if loan_count == 0, then the anon must own the page] 777 */ 778 779 if (anon->an_ref > 1 && pg->loan_count == 0) { 780 781 /* 782 * if the page is busy then we have to unlock, wait for 783 * it and then restart. 784 */ 785 if (pg->flags & PG_BUSY) { 786 pg->flags |= PG_WANTED; 787 amap_unlock(amap); 788 UVM_UNLOCK_AND_WAIT(pg, &anon->an_lock, FALSE, 789 "cownow", 0); 790 goto ReStart; 791 } 792 793 /* 794 * ok, time to do a copy-on-write to a new anon 795 */ 796 nanon = uvm_analloc(); 797 if (nanon) 798 npg = uvm_pagealloc(NULL, 0, nanon, 0); 799 else 800 npg = NULL; /* XXX: quiet gcc warning */ 801 802 if (nanon == NULL || npg == NULL) { 803 /* out of memory */ 804 /* 805 * XXXCDC: we should cause fork to fail, but 806 * we can't ... 807 */ 808 if (nanon) 809 uvm_anfree(nanon); 810 simple_unlock(&anon->an_lock); 811 amap_unlock(amap); 812 uvm_wait("cownowpage"); 813 goto ReStart; 814 } 815 816 /* 817 * got it... now we can copy the data and replace anon 818 * with our new one... 819 */ 820 uvm_pagecopy(pg, npg); /* old -> new */ 821 anon->an_ref--; /* can't drop to zero */ 822 amap->am_anon[slot] = nanon; /* replace */ 823 824 /* 825 * drop PG_BUSY on new page ... since we have had it's 826 * owner locked the whole time it can't be 827 * PG_RELEASED | PG_WANTED. 828 */ 829 npg->flags &= ~(PG_BUSY|PG_FAKE); 830 UVM_PAGE_OWN(npg, NULL); 831 uvm_lock_pageq(); 832 uvm_pageactivate(npg); 833 uvm_unlock_pageq(); 834 } 835 836 simple_unlock(&anon->an_lock); 837 /* 838 * done with this anon, next ...! 839 */ 840 841 } /* end of 'for' loop */ 842 843 return; 844 } 845 846 /* 847 * amap_splitref: split a single reference into two separate references 848 * 849 * => called from uvm_map's clip routines 850 * => origref's map should be locked 851 * => origref->ar_amap should be unlocked (we will lock) 852 */ 853 void 854 amap_splitref(origref, splitref, offset) 855 struct vm_aref *origref, *splitref; 856 vaddr_t offset; 857 { 858 int leftslots; 859 UVMHIST_FUNC("amap_splitref"); UVMHIST_CALLED(maphist); 860 861 AMAP_B2SLOT(leftslots, offset); 862 if (leftslots == 0) 863 panic("amap_splitref: split at zero offset"); 864 865 /* 866 * lock the amap 867 */ 868 amap_lock(origref->ar_amap); 869 870 /* 871 * now: amap is locked and we have a valid am_mapped array. 872 */ 873 874 if (origref->ar_amap->am_nslot - origref->ar_pageoff - leftslots <= 0) 875 panic("amap_splitref: map size check failed"); 876 877 #ifdef UVM_AMAP_PPREF 878 /* 879 * establish ppref before we add a duplicate reference to the amap 880 */ 881 if (origref->ar_amap->am_ppref == NULL) 882 amap_pp_establish(origref->ar_amap); 883 #endif 884 885 splitref->ar_amap = origref->ar_amap; 886 splitref->ar_amap->am_ref++; /* not a share reference */ 887 splitref->ar_pageoff = origref->ar_pageoff + leftslots; 888 889 amap_unlock(origref->ar_amap); 890 } 891 892 #ifdef UVM_AMAP_PPREF 893 894 /* 895 * amap_pp_establish: add a ppref array to an amap, if possible 896 * 897 * => amap locked by caller 898 */ 899 void 900 amap_pp_establish(amap) 901 struct vm_amap *amap; 902 { 903 904 MALLOC(amap->am_ppref, int *, sizeof(int) * amap->am_maxslot, 905 M_UVMAMAP, M_NOWAIT); 906 907 /* 908 * if we fail then we just won't use ppref for this amap 909 */ 910 if (amap->am_ppref == NULL) { 911 amap->am_ppref = PPREF_NONE; /* not using it */ 912 return; 913 } 914 915 /* 916 * init ppref 917 */ 918 memset(amap->am_ppref, 0, sizeof(int) * amap->am_maxslot); 919 pp_setreflen(amap->am_ppref, 0, amap->am_ref, amap->am_nslot); 920 return; 921 } 922 923 /* 924 * amap_pp_adjref: adjust reference count to a part of an amap using the 925 * per-page reference count array. 926 * 927 * => map and amap locked by caller 928 * => caller must check that ppref != PPREF_NONE before calling 929 */ 930 void 931 amap_pp_adjref(amap, curslot, bytelen, adjval) 932 struct vm_amap *amap; 933 int curslot; 934 vsize_t bytelen; 935 int adjval; 936 { 937 int slots, stopslot, *ppref, lcv; 938 int ref, len; 939 940 /* 941 * get init values 942 */ 943 944 AMAP_B2SLOT(slots, bytelen); 945 stopslot = curslot + slots; 946 ppref = amap->am_ppref; 947 948 /* 949 * first advance to the correct place in the ppref array, fragment 950 * if needed. 951 */ 952 953 for (lcv = 0 ; lcv < curslot ; lcv += len) { 954 pp_getreflen(ppref, lcv, &ref, &len); 955 if (lcv + len > curslot) { /* goes past start? */ 956 pp_setreflen(ppref, lcv, ref, curslot - lcv); 957 pp_setreflen(ppref, curslot, ref, len - (curslot -lcv)); 958 len = curslot - lcv; /* new length of entry @ lcv */ 959 } 960 } 961 962 /* 963 * now adjust reference counts in range (make sure we dont overshoot) 964 */ 965 966 if (lcv != curslot) 967 panic("amap_pp_adjref: overshot target"); 968 969 for (/* lcv already set */; lcv < stopslot ; lcv += len) { 970 pp_getreflen(ppref, lcv, &ref, &len); 971 if (lcv + len > stopslot) { /* goes past end? */ 972 pp_setreflen(ppref, lcv, ref, stopslot - lcv); 973 pp_setreflen(ppref, stopslot, ref, 974 len - (stopslot - lcv)); 975 len = stopslot - lcv; 976 } 977 ref = ref + adjval; /* ADJUST! */ 978 if (ref < 0) 979 panic("amap_pp_adjref: negative reference count"); 980 pp_setreflen(ppref, lcv, ref, len); 981 if (ref == 0) 982 amap_wiperange(amap, lcv, len); 983 } 984 985 } 986 987 /* 988 * amap_wiperange: wipe out a range of an amap 989 * [different from amap_wipeout because the amap is kept intact] 990 * 991 * => both map and amap must be locked by caller. 992 */ 993 void 994 amap_wiperange(amap, slotoff, slots) 995 struct vm_amap *amap; 996 int slotoff, slots; 997 { 998 int byanon, lcv, stop, curslot, ptr; 999 struct vm_anon *anon; 1000 UVMHIST_FUNC("amap_wiperange"); UVMHIST_CALLED(maphist); 1001 1002 /* 1003 * we can either traverse the amap by am_anon or by am_slots depending 1004 * on which is cheaper. decide now. 1005 */ 1006 1007 if (slots < amap->am_nused) { 1008 byanon = TRUE; 1009 lcv = slotoff; 1010 stop = slotoff + slots; 1011 } else { 1012 byanon = FALSE; 1013 lcv = 0; 1014 stop = amap->am_nused; 1015 } 1016 1017 /* 1018 * ok, now do it! 1019 */ 1020 1021 for (; lcv < stop; lcv++) { 1022 int refs; 1023 1024 /* 1025 * verify the anon is ok. 1026 */ 1027 if (byanon) { 1028 if (amap->am_anon[lcv] == NULL) 1029 continue; 1030 curslot = lcv; 1031 } else { 1032 curslot = amap->am_slots[lcv]; 1033 if (curslot < slotoff || curslot >= stop) 1034 continue; 1035 } 1036 anon = amap->am_anon[curslot]; 1037 1038 /* 1039 * remove it from the amap 1040 */ 1041 amap->am_anon[curslot] = NULL; 1042 ptr = amap->am_bckptr[curslot]; 1043 if (ptr != (amap->am_nused - 1)) { 1044 amap->am_slots[ptr] = 1045 amap->am_slots[amap->am_nused - 1]; 1046 amap->am_bckptr[amap->am_slots[ptr]] = 1047 ptr; /* back ptr. */ 1048 } 1049 amap->am_nused--; 1050 1051 /* 1052 * drop anon reference count 1053 */ 1054 simple_lock(&anon->an_lock); 1055 refs = --anon->an_ref; 1056 simple_unlock(&anon->an_lock); 1057 if (refs == 0) { 1058 /* 1059 * we just eliminated the last reference to an anon. 1060 * free it. 1061 */ 1062 uvm_anfree(anon); 1063 } 1064 } 1065 } 1066 1067 #endif 1068