1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_fault.c 8.5 (Berkeley) 01/09/95 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Page fault handling module. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 46 #include <vm/vm.h> 47 #include <vm/vm_page.h> 48 #include <vm/vm_pageout.h> 49 50 /* 51 * vm_fault: 52 * 53 * Handle a page fault occuring at the given address, 54 * requiring the given permissions, in the map specified. 55 * If successful, the page is inserted into the 56 * associated physical map. 57 * 58 * NOTE: the given address should be truncated to the 59 * proper page address. 60 * 61 * KERN_SUCCESS is returned if the page fault is handled; otherwise, 62 * a standard error specifying why the fault is fatal is returned. 63 * 64 * 65 * The map in question must be referenced, and remains so. 66 * Caller may hold no locks. 67 */ 68 int 69 vm_fault(map, vaddr, fault_type, change_wiring) 70 vm_map_t map; 71 vm_offset_t vaddr; 72 vm_prot_t fault_type; 73 boolean_t change_wiring; 74 { 75 vm_object_t first_object; 76 vm_offset_t first_offset; 77 vm_map_entry_t entry; 78 register vm_object_t object; 79 register vm_offset_t offset; 80 register vm_page_t m; 81 vm_page_t first_m; 82 vm_prot_t prot; 83 int result; 84 boolean_t wired; 85 boolean_t su; 86 boolean_t lookup_still_valid; 87 boolean_t page_exists; 88 vm_page_t old_m; 89 vm_object_t next_object; 90 91 cnt.v_faults++; /* needs lock XXX */ 92 /* 93 * Recovery actions 94 */ 95 #define FREE_PAGE(m) { \ 96 PAGE_WAKEUP(m); \ 97 vm_page_lock_queues(); \ 98 vm_page_free(m); \ 99 vm_page_unlock_queues(); \ 100 } 101 102 #define RELEASE_PAGE(m) { \ 103 PAGE_WAKEUP(m); \ 104 vm_page_lock_queues(); \ 105 vm_page_activate(m); \ 106 vm_page_unlock_queues(); \ 107 } 108 109 #define UNLOCK_MAP { \ 110 if (lookup_still_valid) { \ 111 vm_map_lookup_done(map, entry); \ 112 lookup_still_valid = FALSE; \ 113 } \ 114 } 115 116 #define UNLOCK_THINGS { \ 117 object->paging_in_progress--; \ 118 vm_object_unlock(object); \ 119 if (object != first_object) { \ 120 vm_object_lock(first_object); \ 121 FREE_PAGE(first_m); \ 122 first_object->paging_in_progress--; \ 123 vm_object_unlock(first_object); \ 124 } \ 125 UNLOCK_MAP; \ 126 } 127 128 #define UNLOCK_AND_DEALLOCATE { \ 129 UNLOCK_THINGS; \ 130 vm_object_deallocate(first_object); \ 131 } 132 133 RetryFault: ; 134 135 /* 136 * Find the backing store object and offset into 137 * it to begin the search. 138 */ 139 140 if ((result = vm_map_lookup(&map, vaddr, fault_type, &entry, 141 &first_object, &first_offset, 142 &prot, &wired, &su)) != KERN_SUCCESS) { 143 return(result); 144 } 145 lookup_still_valid = TRUE; 146 147 if (wired) 148 fault_type = prot; 149 150 first_m = NULL; 151 152 /* 153 * Make a reference to this object to 154 * prevent its disposal while we are messing with 155 * it. Once we have the reference, the map is free 156 * to be diddled. Since objects reference their 157 * shadows (and copies), they will stay around as well. 158 */ 159 160 vm_object_lock(first_object); 161 162 first_object->ref_count++; 163 first_object->paging_in_progress++; 164 165 /* 166 * INVARIANTS (through entire routine): 167 * 168 * 1) At all times, we must either have the object 169 * lock or a busy page in some object to prevent 170 * some other thread from trying to bring in 171 * the same page. 172 * 173 * Note that we cannot hold any locks during the 174 * pager access or when waiting for memory, so 175 * we use a busy page then. 176 * 177 * Note also that we aren't as concerned about 178 * more than one thead attempting to pager_data_unlock 179 * the same page at once, so we don't hold the page 180 * as busy then, but do record the highest unlock 181 * value so far. [Unlock requests may also be delivered 182 * out of order.] 183 * 184 * 2) Once we have a busy page, we must remove it from 185 * the pageout queues, so that the pageout daemon 186 * will not grab it away. 187 * 188 * 3) To prevent another thread from racing us down the 189 * shadow chain and entering a new page in the top 190 * object before we do, we must keep a busy page in 191 * the top object while following the shadow chain. 192 * 193 * 4) We must increment paging_in_progress on any object 194 * for which we have a busy page, to prevent 195 * vm_object_collapse from removing the busy page 196 * without our noticing. 197 */ 198 199 /* 200 * Search for the page at object/offset. 201 */ 202 203 object = first_object; 204 offset = first_offset; 205 206 /* 207 * See whether this page is resident 208 */ 209 210 while (TRUE) { 211 m = vm_page_lookup(object, offset); 212 if (m != NULL) { 213 /* 214 * If the page is being brought in, 215 * wait for it and then retry. 216 */ 217 if (m->flags & PG_BUSY) { 218 #ifdef DOTHREADS 219 int wait_result; 220 221 PAGE_ASSERT_WAIT(m, !change_wiring); 222 UNLOCK_THINGS; 223 thread_block(); 224 wait_result = current_thread()->wait_result; 225 vm_object_deallocate(first_object); 226 if (wait_result != THREAD_AWAKENED) 227 return(KERN_SUCCESS); 228 goto RetryFault; 229 #else 230 PAGE_ASSERT_WAIT(m, !change_wiring); 231 UNLOCK_THINGS; 232 cnt.v_intrans++; 233 thread_block(); 234 vm_object_deallocate(first_object); 235 goto RetryFault; 236 #endif 237 } 238 239 /* 240 * Remove the page from the pageout daemon's 241 * reach while we play with it. 242 */ 243 244 vm_page_lock_queues(); 245 if (m->flags & PG_INACTIVE) { 246 TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq); 247 m->flags &= ~PG_INACTIVE; 248 cnt.v_inactive_count--; 249 cnt.v_reactivated++; 250 } 251 252 if (m->flags & PG_ACTIVE) { 253 TAILQ_REMOVE(&vm_page_queue_active, m, pageq); 254 m->flags &= ~PG_ACTIVE; 255 cnt.v_active_count--; 256 } 257 vm_page_unlock_queues(); 258 259 /* 260 * Mark page busy for other threads. 261 */ 262 m->flags |= PG_BUSY; 263 break; 264 } 265 266 if (((object->pager != NULL) && 267 (!change_wiring || wired)) 268 || (object == first_object)) { 269 270 /* 271 * Allocate a new page for this object/offset 272 * pair. 273 */ 274 275 m = vm_page_alloc(object, offset); 276 277 if (m == NULL) { 278 UNLOCK_AND_DEALLOCATE; 279 VM_WAIT; 280 goto RetryFault; 281 } 282 } 283 284 if (object->pager != NULL && (!change_wiring || wired)) { 285 int rv; 286 287 /* 288 * Now that we have a busy page, we can 289 * release the object lock. 290 */ 291 vm_object_unlock(object); 292 293 /* 294 * Call the pager to retrieve the data, if any, 295 * after releasing the lock on the map. 296 */ 297 UNLOCK_MAP; 298 cnt.v_pageins++; 299 rv = vm_pager_get(object->pager, m, TRUE); 300 301 /* 302 * Reaquire the object lock to preserve our 303 * invariant. 304 */ 305 vm_object_lock(object); 306 307 /* 308 * Found the page. 309 * Leave it busy while we play with it. 310 */ 311 if (rv == VM_PAGER_OK) { 312 /* 313 * Relookup in case pager changed page. 314 * Pager is responsible for disposition 315 * of old page if moved. 316 */ 317 m = vm_page_lookup(object, offset); 318 319 cnt.v_pgpgin++; 320 m->flags &= ~PG_FAKE; 321 m->flags |= PG_CLEAN; 322 pmap_clear_modify(VM_PAGE_TO_PHYS(m)); 323 break; 324 } 325 326 /* 327 * IO error or page outside the range of the pager: 328 * cleanup and return an error. 329 */ 330 if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) { 331 FREE_PAGE(m); 332 UNLOCK_AND_DEALLOCATE; 333 return(KERN_PROTECTION_FAILURE); /* XXX */ 334 } 335 /* 336 * rv == VM_PAGER_FAIL: 337 * 338 * Page does not exist at this object/offset. 339 * Free the bogus page (waking up anyone waiting 340 * for it) and continue on to the next object. 341 * 342 * If this is the top-level object, we must 343 * leave the busy page to prevent another 344 * thread from rushing past us, and inserting 345 * the page in that object at the same time 346 * that we are. 347 */ 348 if (object != first_object) { 349 FREE_PAGE(m); 350 /* note that `m' is not used after this */ 351 } 352 } 353 354 /* 355 * We get here if the object has no pager (or unwiring) 356 * or the pager doesn't have the page. 357 */ 358 if (object == first_object) 359 first_m = m; 360 361 /* 362 * Move on to the next object. Lock the next 363 * object before unlocking the current one. 364 */ 365 366 offset += object->shadow_offset; 367 next_object = object->shadow; 368 if (next_object == NULL) { 369 /* 370 * If there's no object left, fill the page 371 * in the top object with zeros. 372 */ 373 if (object != first_object) { 374 object->paging_in_progress--; 375 vm_object_unlock(object); 376 377 object = first_object; 378 offset = first_offset; 379 m = first_m; 380 vm_object_lock(object); 381 } 382 first_m = NULL; 383 384 vm_page_zero_fill(m); 385 cnt.v_zfod++; 386 m->flags &= ~PG_FAKE; 387 break; 388 } 389 else { 390 vm_object_lock(next_object); 391 if (object != first_object) 392 object->paging_in_progress--; 393 vm_object_unlock(object); 394 object = next_object; 395 object->paging_in_progress++; 396 } 397 } 398 399 if ((m->flags & (PG_ACTIVE | PG_INACTIVE | PG_BUSY)) != PG_BUSY) 400 panic("vm_fault: active, inactive or !busy after main loop"); 401 402 /* 403 * PAGE HAS BEEN FOUND. 404 * [Loop invariant still holds -- the object lock 405 * is held.] 406 */ 407 408 old_m = m; /* save page that would be copied */ 409 410 /* 411 * If the page is being written, but isn't 412 * already owned by the top-level object, 413 * we have to copy it into a new page owned 414 * by the top-level object. 415 */ 416 417 if (object != first_object) { 418 /* 419 * We only really need to copy if we 420 * want to write it. 421 */ 422 423 if (fault_type & VM_PROT_WRITE) { 424 425 /* 426 * If we try to collapse first_object at this 427 * point, we may deadlock when we try to get 428 * the lock on an intermediate object (since we 429 * have the bottom object locked). We can't 430 * unlock the bottom object, because the page 431 * we found may move (by collapse) if we do. 432 * 433 * Instead, we first copy the page. Then, when 434 * we have no more use for the bottom object, 435 * we unlock it and try to collapse. 436 * 437 * Note that we copy the page even if we didn't 438 * need to... that's the breaks. 439 */ 440 441 /* 442 * We already have an empty page in 443 * first_object - use it. 444 */ 445 446 vm_page_copy(m, first_m); 447 first_m->flags &= ~PG_FAKE; 448 449 /* 450 * If another map is truly sharing this 451 * page with us, we have to flush all 452 * uses of the original page, since we 453 * can't distinguish those which want the 454 * original from those which need the 455 * new copy. 456 * 457 * XXX If we know that only one map has 458 * access to this page, then we could 459 * avoid the pmap_page_protect() call. 460 */ 461 462 vm_page_lock_queues(); 463 vm_page_activate(m); 464 vm_page_deactivate(m); 465 pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); 466 vm_page_unlock_queues(); 467 468 /* 469 * We no longer need the old page or object. 470 */ 471 PAGE_WAKEUP(m); 472 object->paging_in_progress--; 473 vm_object_unlock(object); 474 475 /* 476 * Only use the new page below... 477 */ 478 479 cnt.v_cow_faults++; 480 m = first_m; 481 object = first_object; 482 offset = first_offset; 483 484 /* 485 * Now that we've gotten the copy out of the 486 * way, let's try to collapse the top object. 487 */ 488 vm_object_lock(object); 489 /* 490 * But we have to play ugly games with 491 * paging_in_progress to do that... 492 */ 493 object->paging_in_progress--; 494 vm_object_collapse(object); 495 object->paging_in_progress++; 496 } 497 else { 498 prot &= ~VM_PROT_WRITE; 499 m->flags |= PG_COPYONWRITE; 500 } 501 } 502 503 if (m->flags & (PG_ACTIVE|PG_INACTIVE)) 504 panic("vm_fault: active or inactive before copy object handling"); 505 506 /* 507 * If the page is being written, but hasn't been 508 * copied to the copy-object, we have to copy it there. 509 */ 510 RetryCopy: 511 if (first_object->copy != NULL) { 512 vm_object_t copy_object = first_object->copy; 513 vm_offset_t copy_offset; 514 vm_page_t copy_m; 515 516 /* 517 * We only need to copy if we want to write it. 518 */ 519 if ((fault_type & VM_PROT_WRITE) == 0) { 520 prot &= ~VM_PROT_WRITE; 521 m->flags |= PG_COPYONWRITE; 522 } 523 else { 524 /* 525 * Try to get the lock on the copy_object. 526 */ 527 if (!vm_object_lock_try(copy_object)) { 528 vm_object_unlock(object); 529 /* should spin a bit here... */ 530 vm_object_lock(object); 531 goto RetryCopy; 532 } 533 534 /* 535 * Make another reference to the copy-object, 536 * to keep it from disappearing during the 537 * copy. 538 */ 539 copy_object->ref_count++; 540 541 /* 542 * Does the page exist in the copy? 543 */ 544 copy_offset = first_offset 545 - copy_object->shadow_offset; 546 copy_m = vm_page_lookup(copy_object, copy_offset); 547 if (page_exists = (copy_m != NULL)) { 548 if (copy_m->flags & PG_BUSY) { 549 #ifdef DOTHREADS 550 int wait_result; 551 552 /* 553 * If the page is being brought 554 * in, wait for it and then retry. 555 */ 556 PAGE_ASSERT_WAIT(copy_m, !change_wiring); 557 RELEASE_PAGE(m); 558 copy_object->ref_count--; 559 vm_object_unlock(copy_object); 560 UNLOCK_THINGS; 561 thread_block(); 562 wait_result = current_thread()->wait_result; 563 vm_object_deallocate(first_object); 564 if (wait_result != THREAD_AWAKENED) 565 return(KERN_SUCCESS); 566 goto RetryFault; 567 #else 568 /* 569 * If the page is being brought 570 * in, wait for it and then retry. 571 */ 572 PAGE_ASSERT_WAIT(copy_m, !change_wiring); 573 RELEASE_PAGE(m); 574 copy_object->ref_count--; 575 vm_object_unlock(copy_object); 576 UNLOCK_THINGS; 577 thread_block(); 578 vm_object_deallocate(first_object); 579 goto RetryFault; 580 #endif 581 } 582 } 583 584 /* 585 * If the page is not in memory (in the object) 586 * and the object has a pager, we have to check 587 * if the pager has the data in secondary 588 * storage. 589 */ 590 if (!page_exists) { 591 592 /* 593 * If we don't allocate a (blank) page 594 * here... another thread could try 595 * to page it in, allocate a page, and 596 * then block on the busy page in its 597 * shadow (first_object). Then we'd 598 * trip over the busy page after we 599 * found that the copy_object's pager 600 * doesn't have the page... 601 */ 602 copy_m = vm_page_alloc(copy_object, 603 copy_offset); 604 if (copy_m == NULL) { 605 /* 606 * Wait for a page, then retry. 607 */ 608 RELEASE_PAGE(m); 609 copy_object->ref_count--; 610 vm_object_unlock(copy_object); 611 UNLOCK_AND_DEALLOCATE; 612 VM_WAIT; 613 goto RetryFault; 614 } 615 616 if (copy_object->pager != NULL) { 617 vm_object_unlock(object); 618 vm_object_unlock(copy_object); 619 UNLOCK_MAP; 620 621 page_exists = vm_pager_has_page( 622 copy_object->pager, 623 (copy_offset + copy_object->paging_offset)); 624 625 vm_object_lock(copy_object); 626 627 /* 628 * Since the map is unlocked, someone 629 * else could have copied this object 630 * and put a different copy_object 631 * between the two. Or, the last 632 * reference to the copy-object (other 633 * than the one we have) may have 634 * disappeared - if that has happened, 635 * we don't need to make the copy. 636 */ 637 if (copy_object->shadow != object || 638 copy_object->ref_count == 1) { 639 /* 640 * Gaah... start over! 641 */ 642 FREE_PAGE(copy_m); 643 vm_object_unlock(copy_object); 644 vm_object_deallocate(copy_object); 645 /* may block */ 646 vm_object_lock(object); 647 goto RetryCopy; 648 } 649 vm_object_lock(object); 650 651 if (page_exists) { 652 /* 653 * We didn't need the page 654 */ 655 FREE_PAGE(copy_m); 656 } 657 } 658 } 659 if (!page_exists) { 660 /* 661 * Must copy page into copy-object. 662 */ 663 vm_page_copy(m, copy_m); 664 copy_m->flags &= ~PG_FAKE; 665 666 /* 667 * Things to remember: 668 * 1. The copied page must be marked 'dirty' 669 * so it will be paged out to the copy 670 * object. 671 * 2. If the old page was in use by any users 672 * of the copy-object, it must be removed 673 * from all pmaps. (We can't know which 674 * pmaps use it.) 675 */ 676 vm_page_lock_queues(); 677 pmap_page_protect(VM_PAGE_TO_PHYS(old_m), 678 VM_PROT_NONE); 679 copy_m->flags &= ~PG_CLEAN; 680 vm_page_activate(copy_m); /* XXX */ 681 vm_page_unlock_queues(); 682 683 PAGE_WAKEUP(copy_m); 684 } 685 /* 686 * The reference count on copy_object must be 687 * at least 2: one for our extra reference, 688 * and at least one from the outside world 689 * (we checked that when we last locked 690 * copy_object). 691 */ 692 copy_object->ref_count--; 693 vm_object_unlock(copy_object); 694 m->flags &= ~PG_COPYONWRITE; 695 } 696 } 697 698 if (m->flags & (PG_ACTIVE | PG_INACTIVE)) 699 panic("vm_fault: active or inactive before retrying lookup"); 700 701 /* 702 * We must verify that the maps have not changed 703 * since our last lookup. 704 */ 705 706 if (!lookup_still_valid) { 707 vm_object_t retry_object; 708 vm_offset_t retry_offset; 709 vm_prot_t retry_prot; 710 711 /* 712 * Since map entries may be pageable, make sure we can 713 * take a page fault on them. 714 */ 715 vm_object_unlock(object); 716 717 /* 718 * To avoid trying to write_lock the map while another 719 * thread has it read_locked (in vm_map_pageable), we 720 * do not try for write permission. If the page is 721 * still writable, we will get write permission. If it 722 * is not, or has been marked needs_copy, we enter the 723 * mapping without write permission, and will merely 724 * take another fault. 725 */ 726 result = vm_map_lookup(&map, vaddr, 727 fault_type & ~VM_PROT_WRITE, &entry, 728 &retry_object, &retry_offset, &retry_prot, 729 &wired, &su); 730 731 vm_object_lock(object); 732 733 /* 734 * If we don't need the page any longer, put it on the 735 * active list (the easiest thing to do here). If no 736 * one needs it, pageout will grab it eventually. 737 */ 738 739 if (result != KERN_SUCCESS) { 740 RELEASE_PAGE(m); 741 UNLOCK_AND_DEALLOCATE; 742 return(result); 743 } 744 745 lookup_still_valid = TRUE; 746 747 if ((retry_object != first_object) || 748 (retry_offset != first_offset)) { 749 RELEASE_PAGE(m); 750 UNLOCK_AND_DEALLOCATE; 751 goto RetryFault; 752 } 753 754 /* 755 * Check whether the protection has changed or the object 756 * has been copied while we left the map unlocked. 757 * Changing from read to write permission is OK - we leave 758 * the page write-protected, and catch the write fault. 759 * Changing from write to read permission means that we 760 * can't mark the page write-enabled after all. 761 */ 762 prot &= retry_prot; 763 if (m->flags & PG_COPYONWRITE) 764 prot &= ~VM_PROT_WRITE; 765 } 766 767 /* 768 * (the various bits we're fiddling with here are locked by 769 * the object's lock) 770 */ 771 772 /* XXX This distorts the meaning of the copy_on_write bit */ 773 774 if (prot & VM_PROT_WRITE) 775 m->flags &= ~PG_COPYONWRITE; 776 777 /* 778 * It's critically important that a wired-down page be faulted 779 * only once in each map for which it is wired. 780 */ 781 782 if (m->flags & (PG_ACTIVE | PG_INACTIVE)) 783 panic("vm_fault: active or inactive before pmap_enter"); 784 785 vm_object_unlock(object); 786 787 /* 788 * Put this page into the physical map. 789 * We had to do the unlock above because pmap_enter 790 * may cause other faults. We don't put the 791 * page back on the active queue until later so 792 * that the page-out daemon won't find us (yet). 793 */ 794 795 pmap_enter(map->pmap, vaddr, VM_PAGE_TO_PHYS(m), prot, wired); 796 797 /* 798 * If the page is not wired down, then put it where the 799 * pageout daemon can find it. 800 */ 801 vm_object_lock(object); 802 vm_page_lock_queues(); 803 if (change_wiring) { 804 if (wired) 805 vm_page_wire(m); 806 else 807 vm_page_unwire(m); 808 } 809 else 810 vm_page_activate(m); 811 vm_page_unlock_queues(); 812 813 /* 814 * Unlock everything, and return 815 */ 816 817 PAGE_WAKEUP(m); 818 UNLOCK_AND_DEALLOCATE; 819 820 return(KERN_SUCCESS); 821 822 } 823 824 /* 825 * vm_fault_wire: 826 * 827 * Wire down a range of virtual addresses in a map. 828 */ 829 int 830 vm_fault_wire(map, start, end) 831 vm_map_t map; 832 vm_offset_t start, end; 833 { 834 register vm_offset_t va; 835 register pmap_t pmap; 836 int rv; 837 838 pmap = vm_map_pmap(map); 839 840 /* 841 * Inform the physical mapping system that the 842 * range of addresses may not fault, so that 843 * page tables and such can be locked down as well. 844 */ 845 846 pmap_pageable(pmap, start, end, FALSE); 847 848 /* 849 * We simulate a fault to get the page and enter it 850 * in the physical map. 851 */ 852 853 for (va = start; va < end; va += PAGE_SIZE) { 854 rv = vm_fault(map, va, VM_PROT_NONE, TRUE); 855 if (rv) { 856 if (va != start) 857 vm_fault_unwire(map, start, va); 858 return(rv); 859 } 860 } 861 return(KERN_SUCCESS); 862 } 863 864 865 /* 866 * vm_fault_unwire: 867 * 868 * Unwire a range of virtual addresses in a map. 869 */ 870 void 871 vm_fault_unwire(map, start, end) 872 vm_map_t map; 873 vm_offset_t start, end; 874 { 875 876 register vm_offset_t va, pa; 877 register pmap_t pmap; 878 879 pmap = vm_map_pmap(map); 880 881 /* 882 * Since the pages are wired down, we must be able to 883 * get their mappings from the physical map system. 884 */ 885 886 vm_page_lock_queues(); 887 888 for (va = start; va < end; va += PAGE_SIZE) { 889 pa = pmap_extract(pmap, va); 890 if (pa == (vm_offset_t) 0) { 891 panic("unwire: page not in pmap"); 892 } 893 pmap_change_wiring(pmap, va, FALSE); 894 vm_page_unwire(PHYS_TO_VM_PAGE(pa)); 895 } 896 vm_page_unlock_queues(); 897 898 /* 899 * Inform the physical mapping system that the range 900 * of addresses may fault, so that page tables and 901 * such may be unwired themselves. 902 */ 903 904 pmap_pageable(pmap, start, end, TRUE); 905 906 } 907 908 /* 909 * Routine: 910 * vm_fault_copy_entry 911 * Function: 912 * Copy all of the pages from a wired-down map entry to another. 913 * 914 * In/out conditions: 915 * The source and destination maps must be locked for write. 916 * The source map entry must be wired down (or be a sharing map 917 * entry corresponding to a main map entry that is wired down). 918 */ 919 920 void 921 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry) 922 vm_map_t dst_map; 923 vm_map_t src_map; 924 vm_map_entry_t dst_entry; 925 vm_map_entry_t src_entry; 926 { 927 928 vm_object_t dst_object; 929 vm_object_t src_object; 930 vm_offset_t dst_offset; 931 vm_offset_t src_offset; 932 vm_prot_t prot; 933 vm_offset_t vaddr; 934 vm_page_t dst_m; 935 vm_page_t src_m; 936 937 #ifdef lint 938 src_map++; 939 #endif 940 941 src_object = src_entry->object.vm_object; 942 src_offset = src_entry->offset; 943 944 /* 945 * Create the top-level object for the destination entry. 946 * (Doesn't actually shadow anything - we copy the pages 947 * directly.) 948 */ 949 dst_object = vm_object_allocate( 950 (vm_size_t) (dst_entry->end - dst_entry->start)); 951 952 dst_entry->object.vm_object = dst_object; 953 dst_entry->offset = 0; 954 955 prot = dst_entry->max_protection; 956 957 /* 958 * Loop through all of the pages in the entry's range, copying 959 * each one from the source object (it should be there) to the 960 * destination object. 961 */ 962 for (vaddr = dst_entry->start, dst_offset = 0; 963 vaddr < dst_entry->end; 964 vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) { 965 966 /* 967 * Allocate a page in the destination object 968 */ 969 vm_object_lock(dst_object); 970 do { 971 dst_m = vm_page_alloc(dst_object, dst_offset); 972 if (dst_m == NULL) { 973 vm_object_unlock(dst_object); 974 VM_WAIT; 975 vm_object_lock(dst_object); 976 } 977 } while (dst_m == NULL); 978 979 /* 980 * Find the page in the source object, and copy it in. 981 * (Because the source is wired down, the page will be 982 * in memory.) 983 */ 984 vm_object_lock(src_object); 985 src_m = vm_page_lookup(src_object, dst_offset + src_offset); 986 if (src_m == NULL) 987 panic("vm_fault_copy_wired: page missing"); 988 989 vm_page_copy(src_m, dst_m); 990 991 /* 992 * Enter it in the pmap... 993 */ 994 vm_object_unlock(src_object); 995 vm_object_unlock(dst_object); 996 997 pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m), 998 prot, FALSE); 999 1000 /* 1001 * Mark it no longer busy, and put it on the active list. 1002 */ 1003 vm_object_lock(dst_object); 1004 vm_page_lock_queues(); 1005 vm_page_activate(dst_m); 1006 vm_page_unlock_queues(); 1007 PAGE_WAKEUP(dst_m); 1008 vm_object_unlock(dst_object); 1009 } 1010 1011 } 1012