1 /* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_object.c 7.9 (Berkeley) 11/20/91 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Virtual memory object module. 41 */ 42 43 #include "param.h" 44 #include "malloc.h" 45 46 #include "vm.h" 47 #include "vm_page.h" 48 49 /* 50 * Virtual memory objects maintain the actual data 51 * associated with allocated virtual memory. A given 52 * page of memory exists within exactly one object. 53 * 54 * An object is only deallocated when all "references" 55 * are given up. Only one "reference" to a given 56 * region of an object should be writeable. 57 * 58 * Associated with each object is a list of all resident 59 * memory pages belonging to that object; this list is 60 * maintained by the "vm_page" module, and locked by the object's 61 * lock. 62 * 63 * Each object also records a "pager" routine which is 64 * used to retrieve (and store) pages to the proper backing 65 * storage. In addition, objects may be backed by other 66 * objects from which they were virtual-copied. 67 * 68 * The only items within the object structure which are 69 * modified after time of creation are: 70 * reference count locked by object's lock 71 * pager routine locked by object's lock 72 * 73 */ 74 75 struct vm_object kernel_object_store; 76 struct vm_object kmem_object_store; 77 78 #define VM_OBJECT_HASH_COUNT 157 79 80 int vm_cache_max = 100; /* can patch if necessary */ 81 queue_head_t vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 82 83 long object_collapses = 0; 84 long object_bypasses = 0; 85 86 /* 87 * vm_object_init: 88 * 89 * Initialize the VM objects module. 90 */ 91 void vm_object_init(size) 92 vm_size_t size; 93 { 94 register int i; 95 96 queue_init(&vm_object_cached_list); 97 queue_init(&vm_object_list); 98 vm_object_count = 0; 99 simple_lock_init(&vm_cache_lock); 100 simple_lock_init(&vm_object_list_lock); 101 102 for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 103 queue_init(&vm_object_hashtable[i]); 104 105 kernel_object = &kernel_object_store; 106 _vm_object_allocate(size, kernel_object); 107 108 kmem_object = &kmem_object_store; 109 _vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object); 110 } 111 112 /* 113 * vm_object_allocate: 114 * 115 * Returns a new object with the given size. 116 */ 117 118 vm_object_t vm_object_allocate(size) 119 vm_size_t size; 120 { 121 register vm_object_t result; 122 123 result = (vm_object_t) 124 malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK); 125 126 _vm_object_allocate(size, result); 127 128 return(result); 129 } 130 131 _vm_object_allocate(size, object) 132 vm_size_t size; 133 register vm_object_t object; 134 { 135 queue_init(&object->memq); 136 vm_object_lock_init(object); 137 object->ref_count = 1; 138 object->resident_page_count = 0; 139 object->size = size; 140 object->flags = OBJ_INTERNAL; /* vm_allocate_with_pager will reset */ 141 object->paging_in_progress = 0; 142 object->copy = NULL; 143 144 /* 145 * Object starts out read-write, with no pager. 146 */ 147 148 object->pager = NULL; 149 object->paging_offset = 0; 150 object->shadow = NULL; 151 object->shadow_offset = (vm_offset_t) 0; 152 153 simple_lock(&vm_object_list_lock); 154 queue_enter(&vm_object_list, object, vm_object_t, object_list); 155 vm_object_count++; 156 simple_unlock(&vm_object_list_lock); 157 } 158 159 /* 160 * vm_object_reference: 161 * 162 * Gets another reference to the given object. 163 */ 164 void vm_object_reference(object) 165 register vm_object_t object; 166 { 167 if (object == NULL) 168 return; 169 170 vm_object_lock(object); 171 object->ref_count++; 172 vm_object_unlock(object); 173 } 174 175 /* 176 * vm_object_deallocate: 177 * 178 * Release a reference to the specified object, 179 * gained either through a vm_object_allocate 180 * or a vm_object_reference call. When all references 181 * are gone, storage associated with this object 182 * may be relinquished. 183 * 184 * No object may be locked. 185 */ 186 void vm_object_deallocate(object) 187 register vm_object_t object; 188 { 189 vm_object_t temp; 190 191 while (object != NULL) { 192 193 /* 194 * The cache holds a reference (uncounted) to 195 * the object; we must lock it before removing 196 * the object. 197 */ 198 199 vm_object_cache_lock(); 200 201 /* 202 * Lose the reference 203 */ 204 vm_object_lock(object); 205 if (--(object->ref_count) != 0) { 206 207 /* 208 * If there are still references, then 209 * we are done. 210 */ 211 vm_object_unlock(object); 212 vm_object_cache_unlock(); 213 return; 214 } 215 216 /* 217 * See if this object can persist. If so, enter 218 * it in the cache, then deactivate all of its 219 * pages. 220 */ 221 222 if (object->flags & OBJ_CANPERSIST) { 223 224 queue_enter(&vm_object_cached_list, object, 225 vm_object_t, cached_list); 226 vm_object_cached++; 227 vm_object_cache_unlock(); 228 229 vm_object_deactivate_pages(object); 230 vm_object_unlock(object); 231 232 vm_object_cache_trim(); 233 return; 234 } 235 236 /* 237 * Make sure no one can look us up now. 238 */ 239 vm_object_remove(object->pager); 240 vm_object_cache_unlock(); 241 242 temp = object->shadow; 243 vm_object_terminate(object); 244 /* unlocks and deallocates object */ 245 object = temp; 246 } 247 } 248 249 250 /* 251 * vm_object_terminate actually destroys the specified object, freeing 252 * up all previously used resources. 253 * 254 * The object must be locked. 255 */ 256 void vm_object_terminate(object) 257 register vm_object_t object; 258 { 259 register vm_page_t p; 260 vm_object_t shadow_object; 261 262 /* 263 * Detach the object from its shadow if we are the shadow's 264 * copy. 265 */ 266 if ((shadow_object = object->shadow) != NULL) { 267 vm_object_lock(shadow_object); 268 if (shadow_object->copy == object) 269 shadow_object->copy = NULL; 270 #if 0 271 else if (shadow_object->copy != NULL) 272 panic("vm_object_terminate: copy/shadow inconsistency"); 273 #endif 274 vm_object_unlock(shadow_object); 275 } 276 277 /* 278 * Wait until the pageout daemon is through with the object. 279 */ 280 while (object->paging_in_progress) { 281 vm_object_sleep((int)object, object, FALSE); 282 vm_object_lock(object); 283 } 284 285 /* 286 * If not an internal object clean all the pages, removing them 287 * from paging queues as we go. 288 */ 289 if ((object->flags & OBJ_INTERNAL) == 0) { 290 vm_object_page_clean(object, 0, 0, TRUE); 291 vm_object_unlock(object); 292 } 293 294 /* 295 * Now free the pages. 296 * For internal objects, this also removes them from paging queues. 297 */ 298 while (!queue_empty(&object->memq)) { 299 p = (vm_page_t) queue_first(&object->memq); 300 VM_PAGE_CHECK(p); 301 vm_page_lock_queues(); 302 vm_page_free(p); 303 vm_page_unlock_queues(); 304 } 305 if ((object->flags & OBJ_INTERNAL) == 0) 306 vm_object_unlock(object); 307 308 /* 309 * Let the pager know object is dead. 310 */ 311 if (object->pager != NULL) 312 vm_pager_deallocate(object->pager); 313 314 simple_lock(&vm_object_list_lock); 315 queue_remove(&vm_object_list, object, vm_object_t, object_list); 316 vm_object_count--; 317 simple_unlock(&vm_object_list_lock); 318 319 /* 320 * Free the space for the object. 321 */ 322 free((caddr_t)object, M_VMOBJ); 323 } 324 325 /* 326 * vm_object_page_clean 327 * 328 * Clean all dirty pages in the specified range of object. 329 * If dequeue is TRUE, pages are removed from any paging queue 330 * they were on, otherwise they are left on whatever queue they 331 * were on before the cleaning operation began. 332 * 333 * Odd semantics: if start == end, we clean everything. 334 * 335 * The object must be locked. 336 */ 337 vm_object_page_clean(object, start, end, dequeue) 338 register vm_object_t object; 339 register vm_offset_t start; 340 register vm_offset_t end; 341 boolean_t dequeue; 342 { 343 register vm_page_t p; 344 int onqueue; 345 346 if (object->pager == NULL) 347 return; 348 349 again: 350 /* 351 * Wait until the pageout daemon is through with the object. 352 */ 353 while (object->paging_in_progress) { 354 vm_object_sleep((int)object, object, FALSE); 355 vm_object_lock(object); 356 } 357 /* 358 * Loop through the object page list cleaning as necessary. 359 */ 360 p = (vm_page_t) queue_first(&object->memq); 361 while (!queue_end(&object->memq, (queue_entry_t) p)) { 362 if (start == end || 363 p->offset >= start && p->offset < end) { 364 if (p->clean && pmap_is_modified(VM_PAGE_TO_PHYS(p))) 365 p->clean = FALSE; 366 /* 367 * Remove the page from any paging queue. 368 * This needs to be done if either we have been 369 * explicitly asked to do so or it is about to 370 * be cleaned (see comment below). 371 */ 372 if (dequeue || !p->clean) { 373 vm_page_lock_queues(); 374 if (p->active) { 375 queue_remove(&vm_page_queue_active, 376 p, vm_page_t, pageq); 377 p->active = FALSE; 378 cnt.v_active_count--; 379 onqueue = 1; 380 } else if (p->inactive) { 381 queue_remove(&vm_page_queue_inactive, 382 p, vm_page_t, pageq); 383 p->inactive = FALSE; 384 cnt.v_inactive_count--; 385 onqueue = -1; 386 } else 387 onqueue = 0; 388 vm_page_unlock_queues(); 389 } 390 /* 391 * To ensure the state of the page doesn't change 392 * during the clean operation we do two things. 393 * First we set the busy bit and invalidate all 394 * mappings to ensure that thread accesses to the 395 * page block (in vm_fault). Second, we remove 396 * the page from any paging queue to foil the 397 * pageout daemon (vm_pageout_scan). 398 */ 399 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 400 if (!p->clean) { 401 p->busy = TRUE; 402 object->paging_in_progress++; 403 vm_object_unlock(object); 404 (void) vm_pager_put(object->pager, p, TRUE); 405 vm_object_lock(object); 406 object->paging_in_progress--; 407 if (!dequeue && onqueue) { 408 vm_page_lock_queues(); 409 if (onqueue > 0) 410 vm_page_activate(p); 411 else 412 vm_page_deactivate(p); 413 vm_page_unlock_queues(); 414 } 415 p->busy = FALSE; 416 PAGE_WAKEUP(p); 417 goto again; 418 } 419 } 420 p = (vm_page_t) queue_next(&p->listq); 421 } 422 } 423 424 /* 425 * vm_object_deactivate_pages 426 * 427 * Deactivate all pages in the specified object. (Keep its pages 428 * in memory even though it is no longer referenced.) 429 * 430 * The object must be locked. 431 */ 432 vm_object_deactivate_pages(object) 433 register vm_object_t object; 434 { 435 register vm_page_t p, next; 436 437 p = (vm_page_t) queue_first(&object->memq); 438 while (!queue_end(&object->memq, (queue_entry_t) p)) { 439 next = (vm_page_t) queue_next(&p->listq); 440 vm_page_lock_queues(); 441 vm_page_deactivate(p); 442 vm_page_unlock_queues(); 443 p = next; 444 } 445 } 446 447 /* 448 * Trim the object cache to size. 449 */ 450 vm_object_cache_trim() 451 { 452 register vm_object_t object; 453 454 vm_object_cache_lock(); 455 while (vm_object_cached > vm_cache_max) { 456 object = (vm_object_t) queue_first(&vm_object_cached_list); 457 vm_object_cache_unlock(); 458 459 if (object != vm_object_lookup(object->pager)) 460 panic("vm_object_deactivate: I'm sooo confused."); 461 462 pager_cache(object, FALSE); 463 464 vm_object_cache_lock(); 465 } 466 vm_object_cache_unlock(); 467 } 468 469 470 /* 471 * vm_object_shutdown() 472 * 473 * Shut down the object system. Unfortunately, while we 474 * may be trying to do this, init is happily waiting for 475 * processes to exit, and therefore will be causing some objects 476 * to be deallocated. To handle this, we gain a fake reference 477 * to all objects we release paging areas for. This will prevent 478 * a duplicate deallocation. This routine is probably full of 479 * race conditions! 480 */ 481 482 void vm_object_shutdown() 483 { 484 register vm_object_t object; 485 486 /* 487 * Clean up the object cache *before* we screw up the reference 488 * counts on all of the objects. 489 */ 490 491 vm_object_cache_clear(); 492 493 printf("free paging spaces: "); 494 495 /* 496 * First we gain a reference to each object so that 497 * no one else will deallocate them. 498 */ 499 500 simple_lock(&vm_object_list_lock); 501 object = (vm_object_t) queue_first(&vm_object_list); 502 while (!queue_end(&vm_object_list, (queue_entry_t) object)) { 503 vm_object_reference(object); 504 object = (vm_object_t) queue_next(&object->object_list); 505 } 506 simple_unlock(&vm_object_list_lock); 507 508 /* 509 * Now we deallocate all the paging areas. We don't need 510 * to lock anything because we've reduced to a single 511 * processor while shutting down. This also assumes that 512 * no new objects are being created. 513 */ 514 515 object = (vm_object_t) queue_first(&vm_object_list); 516 while (!queue_end(&vm_object_list, (queue_entry_t) object)) { 517 if (object->pager != NULL) 518 vm_pager_deallocate(object->pager); 519 object = (vm_object_t) queue_next(&object->object_list); 520 printf("."); 521 } 522 printf("done.\n"); 523 } 524 525 /* 526 * vm_object_pmap_copy: 527 * 528 * Makes all physical pages in the specified 529 * object range copy-on-write. No writeable 530 * references to these pages should remain. 531 * 532 * The object must *not* be locked. 533 */ 534 void vm_object_pmap_copy(object, start, end) 535 register vm_object_t object; 536 register vm_offset_t start; 537 register vm_offset_t end; 538 { 539 register vm_page_t p; 540 541 if (object == NULL) 542 return; 543 544 vm_object_lock(object); 545 p = (vm_page_t) queue_first(&object->memq); 546 while (!queue_end(&object->memq, (queue_entry_t) p)) { 547 if ((start <= p->offset) && (p->offset < end)) { 548 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 549 p->copy_on_write = TRUE; 550 } 551 p = (vm_page_t) queue_next(&p->listq); 552 } 553 vm_object_unlock(object); 554 } 555 556 /* 557 * vm_object_pmap_remove: 558 * 559 * Removes all physical pages in the specified 560 * object range from all physical maps. 561 * 562 * The object must *not* be locked. 563 */ 564 void vm_object_pmap_remove(object, start, end) 565 register vm_object_t object; 566 register vm_offset_t start; 567 register vm_offset_t end; 568 { 569 register vm_page_t p; 570 571 if (object == NULL) 572 return; 573 574 vm_object_lock(object); 575 p = (vm_page_t) queue_first(&object->memq); 576 while (!queue_end(&object->memq, (queue_entry_t) p)) { 577 if ((start <= p->offset) && (p->offset < end)) 578 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 579 p = (vm_page_t) queue_next(&p->listq); 580 } 581 vm_object_unlock(object); 582 } 583 584 /* 585 * vm_object_copy: 586 * 587 * Create a new object which is a copy of an existing 588 * object, and mark all of the pages in the existing 589 * object 'copy-on-write'. The new object has one reference. 590 * Returns the new object. 591 * 592 * May defer the copy until later if the object is not backed 593 * up by a non-default pager. 594 */ 595 void vm_object_copy(src_object, src_offset, size, 596 dst_object, dst_offset, src_needs_copy) 597 register vm_object_t src_object; 598 vm_offset_t src_offset; 599 vm_size_t size; 600 vm_object_t *dst_object; /* OUT */ 601 vm_offset_t *dst_offset; /* OUT */ 602 boolean_t *src_needs_copy; /* OUT */ 603 { 604 register vm_object_t new_copy; 605 register vm_object_t old_copy; 606 vm_offset_t new_start, new_end; 607 608 register vm_page_t p; 609 610 if (src_object == NULL) { 611 /* 612 * Nothing to copy 613 */ 614 *dst_object = NULL; 615 *dst_offset = 0; 616 *src_needs_copy = FALSE; 617 return; 618 } 619 620 /* 621 * If the object's pager is null_pager or the 622 * default pager, we don't have to make a copy 623 * of it. Instead, we set the needs copy flag and 624 * make a shadow later. 625 */ 626 627 vm_object_lock(src_object); 628 if (src_object->pager == NULL || 629 (src_object->flags & OBJ_INTERNAL)) { 630 631 /* 632 * Make another reference to the object 633 */ 634 src_object->ref_count++; 635 636 /* 637 * Mark all of the pages copy-on-write. 638 */ 639 for (p = (vm_page_t) queue_first(&src_object->memq); 640 !queue_end(&src_object->memq, (queue_entry_t)p); 641 p = (vm_page_t) queue_next(&p->listq)) { 642 if (src_offset <= p->offset && 643 p->offset < src_offset + size) 644 p->copy_on_write = TRUE; 645 } 646 vm_object_unlock(src_object); 647 648 *dst_object = src_object; 649 *dst_offset = src_offset; 650 651 /* 652 * Must make a shadow when write is desired 653 */ 654 *src_needs_copy = TRUE; 655 return; 656 } 657 658 /* 659 * Try to collapse the object before copying it. 660 */ 661 vm_object_collapse(src_object); 662 663 /* 664 * If the object has a pager, the pager wants to 665 * see all of the changes. We need a copy-object 666 * for the changed pages. 667 * 668 * If there is a copy-object, and it is empty, 669 * no changes have been made to the object since the 670 * copy-object was made. We can use the same copy- 671 * object. 672 */ 673 674 Retry1: 675 old_copy = src_object->copy; 676 if (old_copy != NULL) { 677 /* 678 * Try to get the locks (out of order) 679 */ 680 if (!vm_object_lock_try(old_copy)) { 681 vm_object_unlock(src_object); 682 683 /* should spin a bit here... */ 684 vm_object_lock(src_object); 685 goto Retry1; 686 } 687 688 if (old_copy->resident_page_count == 0 && 689 old_copy->pager == NULL) { 690 /* 691 * Return another reference to 692 * the existing copy-object. 693 */ 694 old_copy->ref_count++; 695 vm_object_unlock(old_copy); 696 vm_object_unlock(src_object); 697 *dst_object = old_copy; 698 *dst_offset = src_offset; 699 *src_needs_copy = FALSE; 700 return; 701 } 702 vm_object_unlock(old_copy); 703 } 704 vm_object_unlock(src_object); 705 706 /* 707 * If the object has a pager, the pager wants 708 * to see all of the changes. We must make 709 * a copy-object and put the changed pages there. 710 * 711 * The copy-object is always made large enough to 712 * completely shadow the original object, since 713 * it may have several users who want to shadow 714 * the original object at different points. 715 */ 716 717 new_copy = vm_object_allocate(src_object->size); 718 719 Retry2: 720 vm_object_lock(src_object); 721 /* 722 * Copy object may have changed while we were unlocked 723 */ 724 old_copy = src_object->copy; 725 if (old_copy != NULL) { 726 /* 727 * Try to get the locks (out of order) 728 */ 729 if (!vm_object_lock_try(old_copy)) { 730 vm_object_unlock(src_object); 731 goto Retry2; 732 } 733 734 /* 735 * Consistency check 736 */ 737 if (old_copy->shadow != src_object || 738 old_copy->shadow_offset != (vm_offset_t) 0) 739 panic("vm_object_copy: copy/shadow inconsistency"); 740 741 /* 742 * Make the old copy-object shadow the new one. 743 * It will receive no more pages from the original 744 * object. 745 */ 746 747 src_object->ref_count--; /* remove ref. from old_copy */ 748 old_copy->shadow = new_copy; 749 new_copy->ref_count++; /* locking not needed - we 750 have the only pointer */ 751 vm_object_unlock(old_copy); /* done with old_copy */ 752 } 753 754 new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 755 new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 756 757 /* 758 * Point the new copy at the existing object. 759 */ 760 761 new_copy->shadow = src_object; 762 new_copy->shadow_offset = new_start; 763 src_object->ref_count++; 764 src_object->copy = new_copy; 765 766 /* 767 * Mark all the affected pages of the existing object 768 * copy-on-write. 769 */ 770 p = (vm_page_t) queue_first(&src_object->memq); 771 while (!queue_end(&src_object->memq, (queue_entry_t) p)) { 772 if ((new_start <= p->offset) && (p->offset < new_end)) 773 p->copy_on_write = TRUE; 774 p = (vm_page_t) queue_next(&p->listq); 775 } 776 777 vm_object_unlock(src_object); 778 779 *dst_object = new_copy; 780 *dst_offset = src_offset - new_start; 781 *src_needs_copy = FALSE; 782 } 783 784 /* 785 * vm_object_shadow: 786 * 787 * Create a new object which is backed by the 788 * specified existing object range. The source 789 * object reference is deallocated. 790 * 791 * The new object and offset into that object 792 * are returned in the source parameters. 793 */ 794 795 void vm_object_shadow(object, offset, length) 796 vm_object_t *object; /* IN/OUT */ 797 vm_offset_t *offset; /* IN/OUT */ 798 vm_size_t length; 799 { 800 register vm_object_t source; 801 register vm_object_t result; 802 803 source = *object; 804 805 /* 806 * Allocate a new object with the given length 807 */ 808 809 if ((result = vm_object_allocate(length)) == NULL) 810 panic("vm_object_shadow: no object for shadowing"); 811 812 /* 813 * The new object shadows the source object, adding 814 * a reference to it. Our caller changes his reference 815 * to point to the new object, removing a reference to 816 * the source object. Net result: no change of reference 817 * count. 818 */ 819 result->shadow = source; 820 821 /* 822 * Store the offset into the source object, 823 * and fix up the offset into the new object. 824 */ 825 826 result->shadow_offset = *offset; 827 828 /* 829 * Return the new things 830 */ 831 832 *offset = 0; 833 *object = result; 834 } 835 836 /* 837 * Set the specified object's pager to the specified pager. 838 */ 839 840 void vm_object_setpager(object, pager, paging_offset, 841 read_only) 842 vm_object_t object; 843 vm_pager_t pager; 844 vm_offset_t paging_offset; 845 boolean_t read_only; 846 { 847 #ifdef lint 848 read_only++; /* No longer used */ 849 #endif lint 850 851 vm_object_lock(object); /* XXX ? */ 852 object->pager = pager; 853 object->paging_offset = paging_offset; 854 vm_object_unlock(object); /* XXX ? */ 855 } 856 857 /* 858 * vm_object_hash hashes the pager/id pair. 859 */ 860 861 #define vm_object_hash(pager) \ 862 (((unsigned)pager)%VM_OBJECT_HASH_COUNT) 863 864 /* 865 * vm_object_lookup looks in the object cache for an object with the 866 * specified pager and paging id. 867 */ 868 869 vm_object_t vm_object_lookup(pager) 870 vm_pager_t pager; 871 { 872 register queue_t bucket; 873 register vm_object_hash_entry_t entry; 874 vm_object_t object; 875 876 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 877 878 vm_object_cache_lock(); 879 880 entry = (vm_object_hash_entry_t) queue_first(bucket); 881 while (!queue_end(bucket, (queue_entry_t) entry)) { 882 object = entry->object; 883 if (object->pager == pager) { 884 vm_object_lock(object); 885 if (object->ref_count == 0) { 886 queue_remove(&vm_object_cached_list, object, 887 vm_object_t, cached_list); 888 vm_object_cached--; 889 } 890 object->ref_count++; 891 vm_object_unlock(object); 892 vm_object_cache_unlock(); 893 return(object); 894 } 895 entry = (vm_object_hash_entry_t) queue_next(&entry->hash_links); 896 } 897 898 vm_object_cache_unlock(); 899 return(NULL); 900 } 901 902 /* 903 * vm_object_enter enters the specified object/pager/id into 904 * the hash table. 905 */ 906 907 void vm_object_enter(object, pager) 908 vm_object_t object; 909 vm_pager_t pager; 910 { 911 register queue_t bucket; 912 register vm_object_hash_entry_t entry; 913 914 /* 915 * We don't cache null objects, and we can't cache 916 * objects with the null pager. 917 */ 918 919 if (object == NULL) 920 return; 921 if (pager == NULL) 922 return; 923 924 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 925 entry = (vm_object_hash_entry_t) 926 malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK); 927 entry->object = object; 928 object->flags |= OBJ_CANPERSIST; 929 930 vm_object_cache_lock(); 931 queue_enter(bucket, entry, vm_object_hash_entry_t, hash_links); 932 vm_object_cache_unlock(); 933 } 934 935 /* 936 * vm_object_remove: 937 * 938 * Remove the pager from the hash table. 939 * Note: This assumes that the object cache 940 * is locked. XXX this should be fixed 941 * by reorganizing vm_object_deallocate. 942 */ 943 vm_object_remove(pager) 944 register vm_pager_t pager; 945 { 946 register queue_t bucket; 947 register vm_object_hash_entry_t entry; 948 register vm_object_t object; 949 950 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 951 952 entry = (vm_object_hash_entry_t) queue_first(bucket); 953 while (!queue_end(bucket, (queue_entry_t) entry)) { 954 object = entry->object; 955 if (object->pager == pager) { 956 queue_remove(bucket, entry, vm_object_hash_entry_t, 957 hash_links); 958 free((caddr_t)entry, M_VMOBJHASH); 959 break; 960 } 961 entry = (vm_object_hash_entry_t) queue_next(&entry->hash_links); 962 } 963 } 964 965 /* 966 * vm_object_cache_clear removes all objects from the cache. 967 * 968 */ 969 970 void vm_object_cache_clear() 971 { 972 register vm_object_t object; 973 974 /* 975 * Remove each object in the cache by scanning down the 976 * list of cached objects. 977 */ 978 vm_object_cache_lock(); 979 while (!queue_empty(&vm_object_cached_list)) { 980 object = (vm_object_t) queue_first(&vm_object_cached_list); 981 vm_object_cache_unlock(); 982 983 /* 984 * Note: it is important that we use vm_object_lookup 985 * to gain a reference, and not vm_object_reference, because 986 * the logic for removing an object from the cache lies in 987 * lookup. 988 */ 989 if (object != vm_object_lookup(object->pager)) 990 panic("vm_object_cache_clear: I'm sooo confused."); 991 pager_cache(object, FALSE); 992 993 vm_object_cache_lock(); 994 } 995 vm_object_cache_unlock(); 996 } 997 998 boolean_t vm_object_collapse_allowed = TRUE; 999 /* 1000 * vm_object_collapse: 1001 * 1002 * Collapse an object with the object backing it. 1003 * Pages in the backing object are moved into the 1004 * parent, and the backing object is deallocated. 1005 * 1006 * Requires that the object be locked and the page 1007 * queues be unlocked. 1008 * 1009 */ 1010 void vm_object_collapse(object) 1011 register vm_object_t object; 1012 1013 { 1014 register vm_object_t backing_object; 1015 register vm_offset_t backing_offset; 1016 register vm_size_t size; 1017 register vm_offset_t new_offset; 1018 register vm_page_t p, pp; 1019 1020 if (!vm_object_collapse_allowed) 1021 return; 1022 1023 while (TRUE) { 1024 /* 1025 * Verify that the conditions are right for collapse: 1026 * 1027 * The object exists and no pages in it are currently 1028 * being paged out (or have ever been paged out). 1029 */ 1030 if (object == NULL || 1031 object->paging_in_progress != 0 || 1032 object->pager != NULL) 1033 return; 1034 1035 /* 1036 * There is a backing object, and 1037 */ 1038 1039 if ((backing_object = object->shadow) == NULL) 1040 return; 1041 1042 vm_object_lock(backing_object); 1043 /* 1044 * ... 1045 * The backing object is not read_only, 1046 * and no pages in the backing object are 1047 * currently being paged out. 1048 * The backing object is internal. 1049 */ 1050 1051 if ((backing_object->flags & OBJ_INTERNAL) == 0 || 1052 backing_object->paging_in_progress != 0) { 1053 vm_object_unlock(backing_object); 1054 return; 1055 } 1056 1057 /* 1058 * The backing object can't be a copy-object: 1059 * the shadow_offset for the copy-object must stay 1060 * as 0. Furthermore (for the 'we have all the 1061 * pages' case), if we bypass backing_object and 1062 * just shadow the next object in the chain, old 1063 * pages from that object would then have to be copied 1064 * BOTH into the (former) backing_object and into the 1065 * parent object. 1066 */ 1067 if (backing_object->shadow != NULL && 1068 backing_object->shadow->copy != NULL) { 1069 vm_object_unlock(backing_object); 1070 return; 1071 } 1072 1073 /* 1074 * We know that we can either collapse the backing 1075 * object (if the parent is the only reference to 1076 * it) or (perhaps) remove the parent's reference 1077 * to it. 1078 */ 1079 1080 backing_offset = object->shadow_offset; 1081 size = object->size; 1082 1083 /* 1084 * If there is exactly one reference to the backing 1085 * object, we can collapse it into the parent. 1086 */ 1087 1088 if (backing_object->ref_count == 1) { 1089 1090 /* 1091 * We can collapse the backing object. 1092 * 1093 * Move all in-memory pages from backing_object 1094 * to the parent. Pages that have been paged out 1095 * will be overwritten by any of the parent's 1096 * pages that shadow them. 1097 */ 1098 1099 while (!queue_empty(&backing_object->memq)) { 1100 1101 p = (vm_page_t) 1102 queue_first(&backing_object->memq); 1103 1104 new_offset = (p->offset - backing_offset); 1105 1106 /* 1107 * If the parent has a page here, or if 1108 * this page falls outside the parent, 1109 * dispose of it. 1110 * 1111 * Otherwise, move it as planned. 1112 */ 1113 1114 if (p->offset < backing_offset || 1115 new_offset >= size) { 1116 vm_page_lock_queues(); 1117 vm_page_free(p); 1118 vm_page_unlock_queues(); 1119 } else { 1120 pp = vm_page_lookup(object, new_offset); 1121 if (pp != NULL && !pp->fake) { 1122 vm_page_lock_queues(); 1123 vm_page_free(p); 1124 vm_page_unlock_queues(); 1125 } 1126 else { 1127 if (pp) { 1128 /* may be someone waiting for it */ 1129 PAGE_WAKEUP(pp); 1130 vm_page_lock_queues(); 1131 vm_page_free(pp); 1132 vm_page_unlock_queues(); 1133 } 1134 vm_page_rename(p, object, new_offset); 1135 } 1136 } 1137 } 1138 1139 /* 1140 * Move the pager from backing_object to object. 1141 * 1142 * XXX We're only using part of the paging space 1143 * for keeps now... we ought to discard the 1144 * unused portion. 1145 */ 1146 1147 object->pager = backing_object->pager; 1148 object->paging_offset += backing_offset; 1149 1150 backing_object->pager = NULL; 1151 1152 /* 1153 * Object now shadows whatever backing_object did. 1154 * Note that the reference to backing_object->shadow 1155 * moves from within backing_object to within object. 1156 */ 1157 1158 object->shadow = backing_object->shadow; 1159 object->shadow_offset += backing_object->shadow_offset; 1160 if (object->shadow != NULL && 1161 object->shadow->copy != NULL) { 1162 panic("vm_object_collapse: we collapsed a copy-object!"); 1163 } 1164 /* 1165 * Discard backing_object. 1166 * 1167 * Since the backing object has no pages, no 1168 * pager left, and no object references within it, 1169 * all that is necessary is to dispose of it. 1170 */ 1171 1172 vm_object_unlock(backing_object); 1173 1174 simple_lock(&vm_object_list_lock); 1175 queue_remove(&vm_object_list, backing_object, 1176 vm_object_t, object_list); 1177 vm_object_count--; 1178 simple_unlock(&vm_object_list_lock); 1179 1180 free((caddr_t)backing_object, M_VMOBJ); 1181 1182 object_collapses++; 1183 } 1184 else { 1185 /* 1186 * If all of the pages in the backing object are 1187 * shadowed by the parent object, the parent 1188 * object no longer has to shadow the backing 1189 * object; it can shadow the next one in the 1190 * chain. 1191 * 1192 * The backing object must not be paged out - we'd 1193 * have to check all of the paged-out pages, as 1194 * well. 1195 */ 1196 1197 if (backing_object->pager != NULL) { 1198 vm_object_unlock(backing_object); 1199 return; 1200 } 1201 1202 /* 1203 * Should have a check for a 'small' number 1204 * of pages here. 1205 */ 1206 1207 p = (vm_page_t) queue_first(&backing_object->memq); 1208 while (!queue_end(&backing_object->memq, 1209 (queue_entry_t) p)) { 1210 1211 new_offset = (p->offset - backing_offset); 1212 1213 /* 1214 * If the parent has a page here, or if 1215 * this page falls outside the parent, 1216 * keep going. 1217 * 1218 * Otherwise, the backing_object must be 1219 * left in the chain. 1220 */ 1221 1222 if (p->offset >= backing_offset && 1223 new_offset <= size && 1224 ((pp = vm_page_lookup(object, new_offset)) 1225 == NULL || 1226 pp->fake)) { 1227 /* 1228 * Page still needed. 1229 * Can't go any further. 1230 */ 1231 vm_object_unlock(backing_object); 1232 return; 1233 } 1234 p = (vm_page_t) queue_next(&p->listq); 1235 } 1236 1237 /* 1238 * Make the parent shadow the next object 1239 * in the chain. Deallocating backing_object 1240 * will not remove it, since its reference 1241 * count is at least 2. 1242 */ 1243 1244 vm_object_reference(object->shadow = backing_object->shadow); 1245 object->shadow_offset += backing_object->shadow_offset; 1246 1247 /* Drop the reference count on backing_object. 1248 * Since its ref_count was at least 2, it 1249 * will not vanish; so we don't need to call 1250 * vm_object_deallocate. 1251 */ 1252 backing_object->ref_count--; 1253 vm_object_unlock(backing_object); 1254 1255 object_bypasses ++; 1256 1257 } 1258 1259 /* 1260 * Try again with this object's new backing object. 1261 */ 1262 } 1263 } 1264 1265 /* 1266 * vm_object_page_remove: [internal] 1267 * 1268 * Removes all physical pages in the specified 1269 * object range from the object's list of pages. 1270 * 1271 * The object must be locked. 1272 */ 1273 void vm_object_page_remove(object, start, end) 1274 register vm_object_t object; 1275 register vm_offset_t start; 1276 register vm_offset_t end; 1277 { 1278 register vm_page_t p, next; 1279 1280 if (object == NULL) 1281 return; 1282 1283 p = (vm_page_t) queue_first(&object->memq); 1284 while (!queue_end(&object->memq, (queue_entry_t) p)) { 1285 next = (vm_page_t) queue_next(&p->listq); 1286 if ((start <= p->offset) && (p->offset < end)) { 1287 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1288 vm_page_lock_queues(); 1289 vm_page_free(p); 1290 vm_page_unlock_queues(); 1291 } 1292 p = next; 1293 } 1294 } 1295 1296 /* 1297 * Routine: vm_object_coalesce 1298 * Function: Coalesces two objects backing up adjoining 1299 * regions of memory into a single object. 1300 * 1301 * returns TRUE if objects were combined. 1302 * 1303 * NOTE: Only works at the moment if the second object is NULL - 1304 * if it's not, which object do we lock first? 1305 * 1306 * Parameters: 1307 * prev_object First object to coalesce 1308 * prev_offset Offset into prev_object 1309 * next_object Second object into coalesce 1310 * next_offset Offset into next_object 1311 * 1312 * prev_size Size of reference to prev_object 1313 * next_size Size of reference to next_object 1314 * 1315 * Conditions: 1316 * The object must *not* be locked. 1317 */ 1318 boolean_t vm_object_coalesce(prev_object, next_object, 1319 prev_offset, next_offset, 1320 prev_size, next_size) 1321 1322 register vm_object_t prev_object; 1323 vm_object_t next_object; 1324 vm_offset_t prev_offset, next_offset; 1325 vm_size_t prev_size, next_size; 1326 { 1327 vm_size_t newsize; 1328 1329 #ifdef lint 1330 next_offset++; 1331 #endif lint 1332 1333 if (next_object != NULL) { 1334 return(FALSE); 1335 } 1336 1337 if (prev_object == NULL) { 1338 return(TRUE); 1339 } 1340 1341 vm_object_lock(prev_object); 1342 1343 /* 1344 * Try to collapse the object first 1345 */ 1346 vm_object_collapse(prev_object); 1347 1348 /* 1349 * Can't coalesce if: 1350 * . more than one reference 1351 * . paged out 1352 * . shadows another object 1353 * . has a copy elsewhere 1354 * (any of which mean that the pages not mapped to 1355 * prev_entry may be in use anyway) 1356 */ 1357 1358 if (prev_object->ref_count > 1 || 1359 prev_object->pager != NULL || 1360 prev_object->shadow != NULL || 1361 prev_object->copy != NULL) { 1362 vm_object_unlock(prev_object); 1363 return(FALSE); 1364 } 1365 1366 /* 1367 * Remove any pages that may still be in the object from 1368 * a previous deallocation. 1369 */ 1370 1371 vm_object_page_remove(prev_object, 1372 prev_offset + prev_size, 1373 prev_offset + prev_size + next_size); 1374 1375 /* 1376 * Extend the object if necessary. 1377 */ 1378 newsize = prev_offset + prev_size + next_size; 1379 if (newsize > prev_object->size) 1380 prev_object->size = newsize; 1381 1382 vm_object_unlock(prev_object); 1383 return(TRUE); 1384 } 1385 1386 /* 1387 * vm_object_print: [ debug ] 1388 */ 1389 void vm_object_print(object, full) 1390 vm_object_t object; 1391 boolean_t full; 1392 { 1393 register vm_page_t p; 1394 extern indent; 1395 1396 register int count; 1397 1398 if (object == NULL) 1399 return; 1400 1401 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1402 (int) object, (int) object->size, 1403 object->resident_page_count, object->ref_count); 1404 printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1405 (int) object->pager, (int) object->paging_offset, 1406 (int) object->shadow, (int) object->shadow_offset); 1407 printf("cache: next=0x%x, prev=0x%x\n", 1408 object->cached_list.next, object->cached_list.prev); 1409 1410 if (!full) 1411 return; 1412 1413 indent += 2; 1414 count = 0; 1415 p = (vm_page_t) queue_first(&object->memq); 1416 while (!queue_end(&object->memq, (queue_entry_t) p)) { 1417 if (count == 0) 1418 iprintf("memory:="); 1419 else if (count == 6) { 1420 printf("\n"); 1421 iprintf(" ..."); 1422 count = 0; 1423 } else 1424 printf(","); 1425 count++; 1426 1427 printf("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p)); 1428 p = (vm_page_t) queue_next(&p->listq); 1429 } 1430 if (count != 0) 1431 printf("\n"); 1432 indent -= 2; 1433 } 1434