1 /* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_object.c 7.3 (Berkeley) 04/21/91 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Virtual memory object module. 41 */ 42 43 #include "param.h" 44 #include "malloc.h" 45 46 #include "vm.h" 47 #include "vm_page.h" 48 49 /* 50 * Virtual memory objects maintain the actual data 51 * associated with allocated virtual memory. A given 52 * page of memory exists within exactly one object. 53 * 54 * An object is only deallocated when all "references" 55 * are given up. Only one "reference" to a given 56 * region of an object should be writeable. 57 * 58 * Associated with each object is a list of all resident 59 * memory pages belonging to that object; this list is 60 * maintained by the "vm_page" module, and locked by the object's 61 * lock. 62 * 63 * Each object also records a "pager" routine which is 64 * used to retrieve (and store) pages to the proper backing 65 * storage. In addition, objects may be backed by other 66 * objects from which they were virtual-copied. 67 * 68 * The only items within the object structure which are 69 * modified after time of creation are: 70 * reference count locked by object's lock 71 * pager routine locked by object's lock 72 * 73 */ 74 75 struct vm_object kernel_object_store; 76 struct vm_object kmem_object_store; 77 78 #define VM_OBJECT_HASH_COUNT 157 79 80 int vm_cache_max = 100; /* can patch if necessary */ 81 queue_head_t vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 82 83 long object_collapses = 0; 84 long object_bypasses = 0; 85 86 /* 87 * vm_object_init: 88 * 89 * Initialize the VM objects module. 90 */ 91 void vm_object_init() 92 { 93 register int i; 94 95 queue_init(&vm_object_cached_list); 96 queue_init(&vm_object_list); 97 vm_object_count = 0; 98 simple_lock_init(&vm_cache_lock); 99 simple_lock_init(&vm_object_list_lock); 100 101 for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 102 queue_init(&vm_object_hashtable[i]); 103 104 kernel_object = &kernel_object_store; 105 _vm_object_allocate(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, 106 kernel_object); 107 108 kmem_object = &kmem_object_store; 109 _vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object); 110 } 111 112 /* 113 * vm_object_allocate: 114 * 115 * Returns a new object with the given size. 116 */ 117 118 vm_object_t vm_object_allocate(size) 119 vm_size_t size; 120 { 121 register vm_object_t result; 122 123 result = (vm_object_t) 124 malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK); 125 126 _vm_object_allocate(size, result); 127 128 return(result); 129 } 130 131 _vm_object_allocate(size, object) 132 vm_size_t size; 133 register vm_object_t object; 134 { 135 queue_init(&object->memq); 136 vm_object_lock_init(object); 137 object->ref_count = 1; 138 object->resident_page_count = 0; 139 object->size = size; 140 object->can_persist = FALSE; 141 object->paging_in_progress = 0; 142 object->copy = NULL; 143 144 /* 145 * Object starts out read-write, with no pager. 146 */ 147 148 object->pager = NULL; 149 object->pager_ready = FALSE; 150 object->internal = TRUE; /* vm_allocate_with_pager will reset */ 151 object->paging_offset = 0; 152 object->shadow = NULL; 153 object->shadow_offset = (vm_offset_t) 0; 154 155 simple_lock(&vm_object_list_lock); 156 queue_enter(&vm_object_list, object, vm_object_t, object_list); 157 vm_object_count++; 158 simple_unlock(&vm_object_list_lock); 159 } 160 161 /* 162 * vm_object_reference: 163 * 164 * Gets another reference to the given object. 165 */ 166 void vm_object_reference(object) 167 register vm_object_t object; 168 { 169 if (object == NULL) 170 return; 171 172 vm_object_lock(object); 173 object->ref_count++; 174 vm_object_unlock(object); 175 } 176 177 /* 178 * vm_object_deallocate: 179 * 180 * Release a reference to the specified object, 181 * gained either through a vm_object_allocate 182 * or a vm_object_reference call. When all references 183 * are gone, storage associated with this object 184 * may be relinquished. 185 * 186 * No object may be locked. 187 */ 188 void vm_object_deallocate(object) 189 register vm_object_t object; 190 { 191 vm_object_t temp; 192 193 while (object != NULL) { 194 195 /* 196 * The cache holds a reference (uncounted) to 197 * the object; we must lock it before removing 198 * the object. 199 */ 200 201 vm_object_cache_lock(); 202 203 /* 204 * Lose the reference 205 */ 206 vm_object_lock(object); 207 if (--(object->ref_count) != 0) { 208 209 /* 210 * If there are still references, then 211 * we are done. 212 */ 213 vm_object_unlock(object); 214 vm_object_cache_unlock(); 215 return; 216 } 217 218 /* 219 * See if this object can persist. If so, enter 220 * it in the cache, then deactivate all of its 221 * pages. 222 */ 223 224 if (object->can_persist) { 225 226 queue_enter(&vm_object_cached_list, object, 227 vm_object_t, cached_list); 228 vm_object_cached++; 229 vm_object_cache_unlock(); 230 231 vm_object_deactivate_pages(object); 232 vm_object_unlock(object); 233 234 vm_object_cache_trim(); 235 return; 236 } 237 238 /* 239 * Make sure no one can look us up now. 240 */ 241 vm_object_remove(object->pager); 242 vm_object_cache_unlock(); 243 244 temp = object->shadow; 245 vm_object_terminate(object); 246 /* unlocks and deallocates object */ 247 object = temp; 248 } 249 } 250 251 252 /* 253 * vm_object_terminate actually destroys the specified object, freeing 254 * up all previously used resources. 255 * 256 * The object must be locked. 257 */ 258 void vm_object_terminate(object) 259 register vm_object_t object; 260 { 261 register vm_page_t p; 262 vm_object_t shadow_object; 263 264 /* 265 * Detach the object from its shadow if we are the shadow's 266 * copy. 267 */ 268 if ((shadow_object = object->shadow) != NULL) { 269 vm_object_lock(shadow_object); 270 if (shadow_object->copy == object) 271 shadow_object->copy = NULL; 272 #if 0 273 else if (shadow_object->copy != NULL) 274 panic("vm_object_terminate: copy/shadow inconsistency"); 275 #endif 276 vm_object_unlock(shadow_object); 277 } 278 279 /* 280 * Wait until the pageout daemon is through 281 * with the object. 282 */ 283 284 while (object->paging_in_progress != 0) { 285 vm_object_sleep(object, object, FALSE); 286 vm_object_lock(object); 287 } 288 289 290 /* 291 * While the paging system is locked, 292 * pull the object's pages off the active 293 * and inactive queues. This keeps the 294 * pageout daemon from playing with them 295 * during vm_pager_deallocate. 296 * 297 * We can't free the pages yet, because the 298 * object's pager may have to write them out 299 * before deallocating the paging space. 300 */ 301 302 p = (vm_page_t) queue_first(&object->memq); 303 while (!queue_end(&object->memq, (queue_entry_t) p)) { 304 VM_PAGE_CHECK(p); 305 306 vm_page_lock_queues(); 307 if (p->active) { 308 queue_remove(&vm_page_queue_active, p, vm_page_t, 309 pageq); 310 p->active = FALSE; 311 vm_page_active_count--; 312 } 313 314 if (p->inactive) { 315 queue_remove(&vm_page_queue_inactive, p, vm_page_t, 316 pageq); 317 p->inactive = FALSE; 318 vm_page_inactive_count--; 319 } 320 vm_page_unlock_queues(); 321 p = (vm_page_t) queue_next(&p->listq); 322 } 323 324 vm_object_unlock(object); 325 326 if (object->paging_in_progress != 0) 327 panic("vm_object_deallocate: pageout in progress"); 328 329 /* 330 * Clean and free the pages, as appropriate. 331 * All references to the object are gone, 332 * so we don't need to lock it. 333 */ 334 335 if (!object->internal) { 336 vm_object_lock(object); 337 vm_object_page_clean(object, 0, 0); 338 vm_object_unlock(object); 339 } 340 while (!queue_empty(&object->memq)) { 341 p = (vm_page_t) queue_first(&object->memq); 342 343 VM_PAGE_CHECK(p); 344 345 vm_page_lock_queues(); 346 vm_page_free(p); 347 vm_page_unlock_queues(); 348 } 349 350 /* 351 * Let the pager know object is dead. 352 */ 353 354 if (object->pager != NULL) 355 vm_pager_deallocate(object->pager); 356 357 358 simple_lock(&vm_object_list_lock); 359 queue_remove(&vm_object_list, object, vm_object_t, object_list); 360 vm_object_count--; 361 simple_unlock(&vm_object_list_lock); 362 363 /* 364 * Free the space for the object. 365 */ 366 367 free((caddr_t)object, M_VMOBJ); 368 } 369 370 /* 371 * vm_object_page_clean 372 * 373 * Clean all dirty pages in the specified range of object. 374 * Leaves page on whatever queue it is currently on. 375 * 376 * Odd semantics: if start == end, we clean everything. 377 * 378 * The object must be locked. 379 */ 380 vm_object_page_clean(object, start, end) 381 register vm_object_t object; 382 register vm_offset_t start; 383 register vm_offset_t end; 384 { 385 register vm_page_t p; 386 387 if (object->pager == NULL) 388 return; 389 390 again: 391 p = (vm_page_t) queue_first(&object->memq); 392 while (!queue_end(&object->memq, (queue_entry_t) p)) { 393 if (start == end || 394 p->offset >= start && p->offset < end) { 395 if (p->clean && pmap_is_modified(VM_PAGE_TO_PHYS(p))) 396 p->clean = FALSE; 397 pmap_remove_all(VM_PAGE_TO_PHYS(p)); 398 if (!p->clean) { 399 p->busy = TRUE; 400 object->paging_in_progress++; 401 vm_object_unlock(object); 402 (void) vm_pager_put(object->pager, p, TRUE); 403 vm_object_lock(object); 404 object->paging_in_progress--; 405 p->busy = FALSE; 406 PAGE_WAKEUP(p); 407 goto again; 408 } 409 } 410 p = (vm_page_t) queue_next(&p->listq); 411 } 412 } 413 414 /* 415 * vm_object_deactivate_pages 416 * 417 * Deactivate all pages in the specified object. (Keep its pages 418 * in memory even though it is no longer referenced.) 419 * 420 * The object must be locked. 421 */ 422 vm_object_deactivate_pages(object) 423 register vm_object_t object; 424 { 425 register vm_page_t p, next; 426 427 p = (vm_page_t) queue_first(&object->memq); 428 while (!queue_end(&object->memq, (queue_entry_t) p)) { 429 next = (vm_page_t) queue_next(&p->listq); 430 vm_page_lock_queues(); 431 vm_page_deactivate(p); 432 vm_page_unlock_queues(); 433 p = next; 434 } 435 } 436 437 /* 438 * Trim the object cache to size. 439 */ 440 vm_object_cache_trim() 441 { 442 register vm_object_t object; 443 444 vm_object_cache_lock(); 445 while (vm_object_cached > vm_cache_max) { 446 object = (vm_object_t) queue_first(&vm_object_cached_list); 447 vm_object_cache_unlock(); 448 449 if (object != vm_object_lookup(object->pager)) 450 panic("vm_object_deactivate: I'm sooo confused."); 451 452 pager_cache(object, FALSE); 453 454 vm_object_cache_lock(); 455 } 456 vm_object_cache_unlock(); 457 } 458 459 460 /* 461 * vm_object_shutdown() 462 * 463 * Shut down the object system. Unfortunately, while we 464 * may be trying to do this, init is happily waiting for 465 * processes to exit, and therefore will be causing some objects 466 * to be deallocated. To handle this, we gain a fake reference 467 * to all objects we release paging areas for. This will prevent 468 * a duplicate deallocation. This routine is probably full of 469 * race conditions! 470 */ 471 472 void vm_object_shutdown() 473 { 474 register vm_object_t object; 475 476 /* 477 * Clean up the object cache *before* we screw up the reference 478 * counts on all of the objects. 479 */ 480 481 vm_object_cache_clear(); 482 483 printf("free paging spaces: "); 484 485 /* 486 * First we gain a reference to each object so that 487 * no one else will deallocate them. 488 */ 489 490 simple_lock(&vm_object_list_lock); 491 object = (vm_object_t) queue_first(&vm_object_list); 492 while (!queue_end(&vm_object_list, (queue_entry_t) object)) { 493 vm_object_reference(object); 494 object = (vm_object_t) queue_next(&object->object_list); 495 } 496 simple_unlock(&vm_object_list_lock); 497 498 /* 499 * Now we deallocate all the paging areas. We don't need 500 * to lock anything because we've reduced to a single 501 * processor while shutting down. This also assumes that 502 * no new objects are being created. 503 */ 504 505 object = (vm_object_t) queue_first(&vm_object_list); 506 while (!queue_end(&vm_object_list, (queue_entry_t) object)) { 507 if (object->pager != NULL) 508 vm_pager_deallocate(object->pager); 509 object = (vm_object_t) queue_next(&object->object_list); 510 printf("."); 511 } 512 printf("done.\n"); 513 } 514 515 /* 516 * vm_object_pmap_copy: 517 * 518 * Makes all physical pages in the specified 519 * object range copy-on-write. No writeable 520 * references to these pages should remain. 521 * 522 * The object must *not* be locked. 523 */ 524 void vm_object_pmap_copy(object, start, end) 525 register vm_object_t object; 526 register vm_offset_t start; 527 register vm_offset_t end; 528 { 529 register vm_page_t p; 530 531 if (object == NULL) 532 return; 533 534 vm_object_lock(object); 535 p = (vm_page_t) queue_first(&object->memq); 536 while (!queue_end(&object->memq, (queue_entry_t) p)) { 537 if ((start <= p->offset) && (p->offset < end)) { 538 if (!p->copy_on_write) { 539 pmap_copy_on_write(VM_PAGE_TO_PHYS(p)); 540 p->copy_on_write = TRUE; 541 } 542 } 543 p = (vm_page_t) queue_next(&p->listq); 544 } 545 vm_object_unlock(object); 546 } 547 548 /* 549 * vm_object_pmap_remove: 550 * 551 * Removes all physical pages in the specified 552 * object range from all physical maps. 553 * 554 * The object must *not* be locked. 555 */ 556 void vm_object_pmap_remove(object, start, end) 557 register vm_object_t object; 558 register vm_offset_t start; 559 register vm_offset_t end; 560 { 561 register vm_page_t p; 562 563 if (object == NULL) 564 return; 565 566 vm_object_lock(object); 567 p = (vm_page_t) queue_first(&object->memq); 568 while (!queue_end(&object->memq, (queue_entry_t) p)) { 569 if ((start <= p->offset) && (p->offset < end)) { 570 pmap_remove_all(VM_PAGE_TO_PHYS(p)); 571 } 572 p = (vm_page_t) queue_next(&p->listq); 573 } 574 vm_object_unlock(object); 575 } 576 577 /* 578 * vm_object_copy: 579 * 580 * Create a new object which is a copy of an existing 581 * object, and mark all of the pages in the existing 582 * object 'copy-on-write'. The new object has one reference. 583 * Returns the new object. 584 * 585 * May defer the copy until later if the object is not backed 586 * up by a non-default pager. 587 */ 588 void vm_object_copy(src_object, src_offset, size, 589 dst_object, dst_offset, src_needs_copy) 590 register vm_object_t src_object; 591 vm_offset_t src_offset; 592 vm_size_t size; 593 vm_object_t *dst_object; /* OUT */ 594 vm_offset_t *dst_offset; /* OUT */ 595 boolean_t *src_needs_copy; /* OUT */ 596 { 597 register vm_object_t new_copy; 598 register vm_object_t old_copy; 599 vm_offset_t new_start, new_end; 600 601 register vm_page_t p; 602 603 if (src_object == NULL) { 604 /* 605 * Nothing to copy 606 */ 607 *dst_object = NULL; 608 *dst_offset = 0; 609 *src_needs_copy = FALSE; 610 return; 611 } 612 613 /* 614 * If the object's pager is null_pager or the 615 * default pager, we don't have to make a copy 616 * of it. Instead, we set the needs copy flag and 617 * make a shadow later. 618 */ 619 620 vm_object_lock(src_object); 621 if (src_object->pager == NULL || 622 src_object->internal) { 623 624 /* 625 * Make another reference to the object 626 */ 627 src_object->ref_count++; 628 629 /* 630 * Mark all of the pages copy-on-write. 631 */ 632 for (p = (vm_page_t) queue_first(&src_object->memq); 633 !queue_end(&src_object->memq, (queue_entry_t)p); 634 p = (vm_page_t) queue_next(&p->listq)) { 635 if (src_offset <= p->offset && 636 p->offset < src_offset + size) 637 p->copy_on_write = TRUE; 638 } 639 vm_object_unlock(src_object); 640 641 *dst_object = src_object; 642 *dst_offset = src_offset; 643 644 /* 645 * Must make a shadow when write is desired 646 */ 647 *src_needs_copy = TRUE; 648 return; 649 } 650 651 /* 652 * Try to collapse the object before copying it. 653 */ 654 vm_object_collapse(src_object); 655 656 /* 657 * If the object has a pager, the pager wants to 658 * see all of the changes. We need a copy-object 659 * for the changed pages. 660 * 661 * If there is a copy-object, and it is empty, 662 * no changes have been made to the object since the 663 * copy-object was made. We can use the same copy- 664 * object. 665 */ 666 667 Retry1: 668 old_copy = src_object->copy; 669 if (old_copy != NULL) { 670 /* 671 * Try to get the locks (out of order) 672 */ 673 if (!vm_object_lock_try(old_copy)) { 674 vm_object_unlock(src_object); 675 676 /* should spin a bit here... */ 677 vm_object_lock(src_object); 678 goto Retry1; 679 } 680 681 if (old_copy->resident_page_count == 0 && 682 old_copy->pager == NULL) { 683 /* 684 * Return another reference to 685 * the existing copy-object. 686 */ 687 old_copy->ref_count++; 688 vm_object_unlock(old_copy); 689 vm_object_unlock(src_object); 690 *dst_object = old_copy; 691 *dst_offset = src_offset; 692 *src_needs_copy = FALSE; 693 return; 694 } 695 vm_object_unlock(old_copy); 696 } 697 vm_object_unlock(src_object); 698 699 /* 700 * If the object has a pager, the pager wants 701 * to see all of the changes. We must make 702 * a copy-object and put the changed pages there. 703 * 704 * The copy-object is always made large enough to 705 * completely shadow the original object, since 706 * it may have several users who want to shadow 707 * the original object at different points. 708 */ 709 710 new_copy = vm_object_allocate(src_object->size); 711 712 Retry2: 713 vm_object_lock(src_object); 714 /* 715 * Copy object may have changed while we were unlocked 716 */ 717 old_copy = src_object->copy; 718 if (old_copy != NULL) { 719 /* 720 * Try to get the locks (out of order) 721 */ 722 if (!vm_object_lock_try(old_copy)) { 723 vm_object_unlock(src_object); 724 goto Retry2; 725 } 726 727 /* 728 * Consistency check 729 */ 730 if (old_copy->shadow != src_object || 731 old_copy->shadow_offset != (vm_offset_t) 0) 732 panic("vm_object_copy: copy/shadow inconsistency"); 733 734 /* 735 * Make the old copy-object shadow the new one. 736 * It will receive no more pages from the original 737 * object. 738 */ 739 740 src_object->ref_count--; /* remove ref. from old_copy */ 741 old_copy->shadow = new_copy; 742 new_copy->ref_count++; /* locking not needed - we 743 have the only pointer */ 744 vm_object_unlock(old_copy); /* done with old_copy */ 745 } 746 747 new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 748 new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 749 750 /* 751 * Point the new copy at the existing object. 752 */ 753 754 new_copy->shadow = src_object; 755 new_copy->shadow_offset = new_start; 756 src_object->ref_count++; 757 src_object->copy = new_copy; 758 759 /* 760 * Mark all the affected pages of the existing object 761 * copy-on-write. 762 */ 763 p = (vm_page_t) queue_first(&src_object->memq); 764 while (!queue_end(&src_object->memq, (queue_entry_t) p)) { 765 if ((new_start <= p->offset) && (p->offset < new_end)) { 766 p->copy_on_write = TRUE; 767 } 768 p = (vm_page_t) queue_next(&p->listq); 769 } 770 771 vm_object_unlock(src_object); 772 773 *dst_object = new_copy; 774 *dst_offset = src_offset - new_start; 775 *src_needs_copy = FALSE; 776 } 777 778 /* 779 * vm_object_shadow: 780 * 781 * Create a new object which is backed by the 782 * specified existing object range. The source 783 * object reference is deallocated. 784 * 785 * The new object and offset into that object 786 * are returned in the source parameters. 787 */ 788 789 void vm_object_shadow(object, offset, length) 790 vm_object_t *object; /* IN/OUT */ 791 vm_offset_t *offset; /* IN/OUT */ 792 vm_size_t length; 793 { 794 register vm_object_t source; 795 register vm_object_t result; 796 797 source = *object; 798 799 /* 800 * Allocate a new object with the given length 801 */ 802 803 if ((result = vm_object_allocate(length)) == NULL) 804 panic("vm_object_shadow: no object for shadowing"); 805 806 /* 807 * The new object shadows the source object, adding 808 * a reference to it. Our caller changes his reference 809 * to point to the new object, removing a reference to 810 * the source object. Net result: no change of reference 811 * count. 812 */ 813 result->shadow = source; 814 815 /* 816 * Store the offset into the source object, 817 * and fix up the offset into the new object. 818 */ 819 820 result->shadow_offset = *offset; 821 822 /* 823 * Return the new things 824 */ 825 826 *offset = 0; 827 *object = result; 828 } 829 830 /* 831 * Set the specified object's pager to the specified pager. 832 */ 833 834 void vm_object_setpager(object, pager, paging_offset, 835 read_only) 836 vm_object_t object; 837 vm_pager_t pager; 838 vm_offset_t paging_offset; 839 boolean_t read_only; 840 { 841 #ifdef lint 842 read_only++; /* No longer used */ 843 #endif lint 844 845 vm_object_lock(object); /* XXX ? */ 846 object->pager = pager; 847 object->paging_offset = paging_offset; 848 vm_object_unlock(object); /* XXX ? */ 849 } 850 851 /* 852 * vm_object_hash hashes the pager/id pair. 853 */ 854 855 #define vm_object_hash(pager) \ 856 (((unsigned)pager)%VM_OBJECT_HASH_COUNT) 857 858 /* 859 * vm_object_lookup looks in the object cache for an object with the 860 * specified pager and paging id. 861 */ 862 863 vm_object_t vm_object_lookup(pager) 864 vm_pager_t pager; 865 { 866 register queue_t bucket; 867 register vm_object_hash_entry_t entry; 868 vm_object_t object; 869 870 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 871 872 vm_object_cache_lock(); 873 874 entry = (vm_object_hash_entry_t) queue_first(bucket); 875 while (!queue_end(bucket, (queue_entry_t) entry)) { 876 object = entry->object; 877 if (object->pager == pager) { 878 vm_object_lock(object); 879 if (object->ref_count == 0) { 880 queue_remove(&vm_object_cached_list, object, 881 vm_object_t, cached_list); 882 vm_object_cached--; 883 } 884 object->ref_count++; 885 vm_object_unlock(object); 886 vm_object_cache_unlock(); 887 return(object); 888 } 889 entry = (vm_object_hash_entry_t) queue_next(&entry->hash_links); 890 } 891 892 vm_object_cache_unlock(); 893 return(NULL); 894 } 895 896 /* 897 * vm_object_enter enters the specified object/pager/id into 898 * the hash table. 899 */ 900 901 void vm_object_enter(object, pager) 902 vm_object_t object; 903 vm_pager_t pager; 904 { 905 register queue_t bucket; 906 register vm_object_hash_entry_t entry; 907 908 /* 909 * We don't cache null objects, and we can't cache 910 * objects with the null pager. 911 */ 912 913 if (object == NULL) 914 return; 915 if (pager == NULL) 916 return; 917 918 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 919 entry = (vm_object_hash_entry_t) 920 malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK); 921 entry->object = object; 922 object->can_persist = TRUE; 923 924 vm_object_cache_lock(); 925 queue_enter(bucket, entry, vm_object_hash_entry_t, hash_links); 926 vm_object_cache_unlock(); 927 } 928 929 /* 930 * vm_object_remove: 931 * 932 * Remove the pager from the hash table. 933 * Note: This assumes that the object cache 934 * is locked. XXX this should be fixed 935 * by reorganizing vm_object_deallocate. 936 */ 937 vm_object_remove(pager) 938 register vm_pager_t pager; 939 { 940 register queue_t bucket; 941 register vm_object_hash_entry_t entry; 942 register vm_object_t object; 943 944 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 945 946 entry = (vm_object_hash_entry_t) queue_first(bucket); 947 while (!queue_end(bucket, (queue_entry_t) entry)) { 948 object = entry->object; 949 if (object->pager == pager) { 950 queue_remove(bucket, entry, vm_object_hash_entry_t, 951 hash_links); 952 free((caddr_t)entry, M_VMOBJHASH); 953 break; 954 } 955 entry = (vm_object_hash_entry_t) queue_next(&entry->hash_links); 956 } 957 } 958 959 /* 960 * vm_object_cache_clear removes all objects from the cache. 961 * 962 */ 963 964 void vm_object_cache_clear() 965 { 966 register vm_object_t object; 967 968 /* 969 * Remove each object in the cache by scanning down the 970 * list of cached objects. 971 */ 972 vm_object_cache_lock(); 973 while (!queue_empty(&vm_object_cached_list)) { 974 object = (vm_object_t) queue_first(&vm_object_cached_list); 975 vm_object_cache_unlock(); 976 977 /* 978 * Note: it is important that we use vm_object_lookup 979 * to gain a reference, and not vm_object_reference, because 980 * the logic for removing an object from the cache lies in 981 * lookup. 982 */ 983 if (object != vm_object_lookup(object->pager)) 984 panic("vm_object_cache_clear: I'm sooo confused."); 985 pager_cache(object, FALSE); 986 987 vm_object_cache_lock(); 988 } 989 vm_object_cache_unlock(); 990 } 991 992 boolean_t vm_object_collapse_allowed = TRUE; 993 /* 994 * vm_object_collapse: 995 * 996 * Collapse an object with the object backing it. 997 * Pages in the backing object are moved into the 998 * parent, and the backing object is deallocated. 999 * 1000 * Requires that the object be locked and the page 1001 * queues be unlocked. 1002 * 1003 */ 1004 void vm_object_collapse(object) 1005 register vm_object_t object; 1006 1007 { 1008 register vm_object_t backing_object; 1009 register vm_offset_t backing_offset; 1010 register vm_size_t size; 1011 register vm_offset_t new_offset; 1012 register vm_page_t p, pp; 1013 1014 if (!vm_object_collapse_allowed) 1015 return; 1016 1017 while (TRUE) { 1018 /* 1019 * Verify that the conditions are right for collapse: 1020 * 1021 * The object exists and no pages in it are currently 1022 * being paged out (or have ever been paged out). 1023 */ 1024 if (object == NULL || 1025 object->paging_in_progress != 0 || 1026 object->pager != NULL) 1027 return; 1028 1029 /* 1030 * There is a backing object, and 1031 */ 1032 1033 if ((backing_object = object->shadow) == NULL) 1034 return; 1035 1036 vm_object_lock(backing_object); 1037 /* 1038 * ... 1039 * The backing object is not read_only, 1040 * and no pages in the backing object are 1041 * currently being paged out. 1042 * The backing object is internal. 1043 */ 1044 1045 if (!backing_object->internal || 1046 backing_object->paging_in_progress != 0) { 1047 vm_object_unlock(backing_object); 1048 return; 1049 } 1050 1051 /* 1052 * The backing object can't be a copy-object: 1053 * the shadow_offset for the copy-object must stay 1054 * as 0. Furthermore (for the 'we have all the 1055 * pages' case), if we bypass backing_object and 1056 * just shadow the next object in the chain, old 1057 * pages from that object would then have to be copied 1058 * BOTH into the (former) backing_object and into the 1059 * parent object. 1060 */ 1061 if (backing_object->shadow != NULL && 1062 backing_object->shadow->copy != NULL) { 1063 vm_object_unlock(backing_object); 1064 return; 1065 } 1066 1067 /* 1068 * We know that we can either collapse the backing 1069 * object (if the parent is the only reference to 1070 * it) or (perhaps) remove the parent's reference 1071 * to it. 1072 */ 1073 1074 backing_offset = object->shadow_offset; 1075 size = object->size; 1076 1077 /* 1078 * If there is exactly one reference to the backing 1079 * object, we can collapse it into the parent. 1080 */ 1081 1082 if (backing_object->ref_count == 1) { 1083 1084 /* 1085 * We can collapse the backing object. 1086 * 1087 * Move all in-memory pages from backing_object 1088 * to the parent. Pages that have been paged out 1089 * will be overwritten by any of the parent's 1090 * pages that shadow them. 1091 */ 1092 1093 while (!queue_empty(&backing_object->memq)) { 1094 1095 p = (vm_page_t) 1096 queue_first(&backing_object->memq); 1097 1098 new_offset = (p->offset - backing_offset); 1099 1100 /* 1101 * If the parent has a page here, or if 1102 * this page falls outside the parent, 1103 * dispose of it. 1104 * 1105 * Otherwise, move it as planned. 1106 */ 1107 1108 if (p->offset < backing_offset || 1109 new_offset >= size) { 1110 vm_page_lock_queues(); 1111 vm_page_free(p); 1112 vm_page_unlock_queues(); 1113 } else { 1114 pp = vm_page_lookup(object, new_offset); 1115 if (pp != NULL && !pp->fake) { 1116 vm_page_lock_queues(); 1117 vm_page_free(p); 1118 vm_page_unlock_queues(); 1119 } 1120 else { 1121 if (pp) { 1122 /* may be someone waiting for it */ 1123 PAGE_WAKEUP(pp); 1124 vm_page_lock_queues(); 1125 vm_page_free(pp); 1126 vm_page_unlock_queues(); 1127 } 1128 vm_page_rename(p, object, new_offset); 1129 } 1130 } 1131 } 1132 1133 /* 1134 * Move the pager from backing_object to object. 1135 * 1136 * XXX We're only using part of the paging space 1137 * for keeps now... we ought to discard the 1138 * unused portion. 1139 */ 1140 1141 object->pager = backing_object->pager; 1142 object->paging_offset += backing_offset; 1143 1144 backing_object->pager = NULL; 1145 1146 /* 1147 * Object now shadows whatever backing_object did. 1148 * Note that the reference to backing_object->shadow 1149 * moves from within backing_object to within object. 1150 */ 1151 1152 object->shadow = backing_object->shadow; 1153 object->shadow_offset += backing_object->shadow_offset; 1154 if (object->shadow != NULL && 1155 object->shadow->copy != NULL) { 1156 panic("vm_object_collapse: we collapsed a copy-object!"); 1157 } 1158 /* 1159 * Discard backing_object. 1160 * 1161 * Since the backing object has no pages, no 1162 * pager left, and no object references within it, 1163 * all that is necessary is to dispose of it. 1164 */ 1165 1166 vm_object_unlock(backing_object); 1167 1168 simple_lock(&vm_object_list_lock); 1169 queue_remove(&vm_object_list, backing_object, 1170 vm_object_t, object_list); 1171 vm_object_count--; 1172 simple_unlock(&vm_object_list_lock); 1173 1174 free((caddr_t)backing_object, M_VMOBJ); 1175 1176 object_collapses++; 1177 } 1178 else { 1179 /* 1180 * If all of the pages in the backing object are 1181 * shadowed by the parent object, the parent 1182 * object no longer has to shadow the backing 1183 * object; it can shadow the next one in the 1184 * chain. 1185 * 1186 * The backing object must not be paged out - we'd 1187 * have to check all of the paged-out pages, as 1188 * well. 1189 */ 1190 1191 if (backing_object->pager != NULL) { 1192 vm_object_unlock(backing_object); 1193 return; 1194 } 1195 1196 /* 1197 * Should have a check for a 'small' number 1198 * of pages here. 1199 */ 1200 1201 p = (vm_page_t) queue_first(&backing_object->memq); 1202 while (!queue_end(&backing_object->memq, 1203 (queue_entry_t) p)) { 1204 1205 new_offset = (p->offset - backing_offset); 1206 1207 /* 1208 * If the parent has a page here, or if 1209 * this page falls outside the parent, 1210 * keep going. 1211 * 1212 * Otherwise, the backing_object must be 1213 * left in the chain. 1214 */ 1215 1216 if (p->offset >= backing_offset && 1217 new_offset <= size && 1218 ((pp = vm_page_lookup(object, new_offset)) 1219 == NULL || 1220 pp->fake)) { 1221 /* 1222 * Page still needed. 1223 * Can't go any further. 1224 */ 1225 vm_object_unlock(backing_object); 1226 return; 1227 } 1228 p = (vm_page_t) queue_next(&p->listq); 1229 } 1230 1231 /* 1232 * Make the parent shadow the next object 1233 * in the chain. Deallocating backing_object 1234 * will not remove it, since its reference 1235 * count is at least 2. 1236 */ 1237 1238 vm_object_reference(object->shadow = backing_object->shadow); 1239 object->shadow_offset += backing_object->shadow_offset; 1240 1241 /* Drop the reference count on backing_object. 1242 * Since its ref_count was at least 2, it 1243 * will not vanish; so we don't need to call 1244 * vm_object_deallocate. 1245 */ 1246 backing_object->ref_count--; 1247 vm_object_unlock(backing_object); 1248 1249 object_bypasses ++; 1250 1251 } 1252 1253 /* 1254 * Try again with this object's new backing object. 1255 */ 1256 } 1257 } 1258 1259 /* 1260 * vm_object_page_remove: [internal] 1261 * 1262 * Removes all physical pages in the specified 1263 * object range from the object's list of pages. 1264 * 1265 * The object must be locked. 1266 */ 1267 void vm_object_page_remove(object, start, end) 1268 register vm_object_t object; 1269 register vm_offset_t start; 1270 register vm_offset_t end; 1271 { 1272 register vm_page_t p, next; 1273 1274 if (object == NULL) 1275 return; 1276 1277 p = (vm_page_t) queue_first(&object->memq); 1278 while (!queue_end(&object->memq, (queue_entry_t) p)) { 1279 next = (vm_page_t) queue_next(&p->listq); 1280 if ((start <= p->offset) && (p->offset < end)) { 1281 pmap_remove_all(VM_PAGE_TO_PHYS(p)); 1282 vm_page_lock_queues(); 1283 vm_page_free(p); 1284 vm_page_unlock_queues(); 1285 } 1286 p = next; 1287 } 1288 } 1289 1290 /* 1291 * Routine: vm_object_coalesce 1292 * Function: Coalesces two objects backing up adjoining 1293 * regions of memory into a single object. 1294 * 1295 * returns TRUE if objects were combined. 1296 * 1297 * NOTE: Only works at the moment if the second object is NULL - 1298 * if it's not, which object do we lock first? 1299 * 1300 * Parameters: 1301 * prev_object First object to coalesce 1302 * prev_offset Offset into prev_object 1303 * next_object Second object into coalesce 1304 * next_offset Offset into next_object 1305 * 1306 * prev_size Size of reference to prev_object 1307 * next_size Size of reference to next_object 1308 * 1309 * Conditions: 1310 * The object must *not* be locked. 1311 */ 1312 boolean_t vm_object_coalesce(prev_object, next_object, 1313 prev_offset, next_offset, 1314 prev_size, next_size) 1315 1316 register vm_object_t prev_object; 1317 vm_object_t next_object; 1318 vm_offset_t prev_offset, next_offset; 1319 vm_size_t prev_size, next_size; 1320 { 1321 vm_size_t newsize; 1322 1323 #ifdef lint 1324 next_offset++; 1325 #endif lint 1326 1327 if (next_object != NULL) { 1328 return(FALSE); 1329 } 1330 1331 if (prev_object == NULL) { 1332 return(TRUE); 1333 } 1334 1335 vm_object_lock(prev_object); 1336 1337 /* 1338 * Try to collapse the object first 1339 */ 1340 vm_object_collapse(prev_object); 1341 1342 /* 1343 * Can't coalesce if: 1344 * . more than one reference 1345 * . paged out 1346 * . shadows another object 1347 * . has a copy elsewhere 1348 * (any of which mean that the pages not mapped to 1349 * prev_entry may be in use anyway) 1350 */ 1351 1352 if (prev_object->ref_count > 1 || 1353 prev_object->pager != NULL || 1354 prev_object->shadow != NULL || 1355 prev_object->copy != NULL) { 1356 vm_object_unlock(prev_object); 1357 return(FALSE); 1358 } 1359 1360 /* 1361 * Remove any pages that may still be in the object from 1362 * a previous deallocation. 1363 */ 1364 1365 vm_object_page_remove(prev_object, 1366 prev_offset + prev_size, 1367 prev_offset + prev_size + next_size); 1368 1369 /* 1370 * Extend the object if necessary. 1371 */ 1372 newsize = prev_offset + prev_size + next_size; 1373 if (newsize > prev_object->size) 1374 prev_object->size = newsize; 1375 1376 vm_object_unlock(prev_object); 1377 return(TRUE); 1378 } 1379 1380 /* 1381 * vm_object_print: [ debug ] 1382 */ 1383 void vm_object_print(object, full) 1384 vm_object_t object; 1385 boolean_t full; 1386 { 1387 register vm_page_t p; 1388 extern indent; 1389 1390 register int count; 1391 1392 if (object == NULL) 1393 return; 1394 1395 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1396 (int) object, (int) object->size, 1397 object->resident_page_count, object->ref_count); 1398 printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1399 (int) object->pager, (int) object->paging_offset, 1400 (int) object->shadow, (int) object->shadow_offset); 1401 printf("cache: next=0x%x, prev=0x%x\n", 1402 object->cached_list.next, object->cached_list.prev); 1403 1404 if (!full) 1405 return; 1406 1407 indent += 2; 1408 count = 0; 1409 p = (vm_page_t) queue_first(&object->memq); 1410 while (!queue_end(&object->memq, (queue_entry_t) p)) { 1411 if (count == 0) 1412 iprintf("memory:="); 1413 else if (count == 6) { 1414 printf("\n"); 1415 iprintf(" ..."); 1416 count = 0; 1417 } else 1418 printf(","); 1419 count++; 1420 1421 printf("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p)); 1422 p = (vm_page_t) queue_next(&p->listq); 1423 } 1424 if (count != 0) 1425 printf("\n"); 1426 indent -= 2; 1427 } 1428