1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_object.c 8.2 (Berkeley) 12/30/93 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Virtual memory object module. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/malloc.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 50 /* 51 * Virtual memory objects maintain the actual data 52 * associated with allocated virtual memory. A given 53 * page of memory exists within exactly one object. 54 * 55 * An object is only deallocated when all "references" 56 * are given up. Only one "reference" to a given 57 * region of an object should be writeable. 58 * 59 * Associated with each object is a list of all resident 60 * memory pages belonging to that object; this list is 61 * maintained by the "vm_page" module, and locked by the object's 62 * lock. 63 * 64 * Each object also records a "pager" routine which is 65 * used to retrieve (and store) pages to the proper backing 66 * storage. In addition, objects may be backed by other 67 * objects from which they were virtual-copied. 68 * 69 * The only items within the object structure which are 70 * modified after time of creation are: 71 * reference count locked by object's lock 72 * pager routine locked by object's lock 73 * 74 */ 75 76 struct vm_object kernel_object_store; 77 struct vm_object kmem_object_store; 78 79 #define VM_OBJECT_HASH_COUNT 157 80 81 int vm_cache_max = 100; /* can patch if necessary */ 82 struct vm_object_hash_head vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 83 84 long object_collapses = 0; 85 long object_bypasses = 0; 86 87 static void _vm_object_allocate __P((vm_size_t, vm_object_t)); 88 89 /* 90 * vm_object_init: 91 * 92 * Initialize the VM objects module. 93 */ 94 void vm_object_init(size) 95 vm_size_t size; 96 { 97 register int i; 98 99 TAILQ_INIT(&vm_object_cached_list); 100 TAILQ_INIT(&vm_object_list); 101 vm_object_count = 0; 102 simple_lock_init(&vm_cache_lock); 103 simple_lock_init(&vm_object_list_lock); 104 105 for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 106 TAILQ_INIT(&vm_object_hashtable[i]); 107 108 kernel_object = &kernel_object_store; 109 _vm_object_allocate(size, kernel_object); 110 111 kmem_object = &kmem_object_store; 112 _vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object); 113 } 114 115 /* 116 * vm_object_allocate: 117 * 118 * Returns a new object with the given size. 119 */ 120 121 vm_object_t vm_object_allocate(size) 122 vm_size_t size; 123 { 124 register vm_object_t result; 125 126 result = (vm_object_t) 127 malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK); 128 129 _vm_object_allocate(size, result); 130 131 return(result); 132 } 133 134 static void 135 _vm_object_allocate(size, object) 136 vm_size_t size; 137 register vm_object_t object; 138 { 139 TAILQ_INIT(&object->memq); 140 vm_object_lock_init(object); 141 object->ref_count = 1; 142 object->resident_page_count = 0; 143 object->size = size; 144 object->flags = OBJ_INTERNAL; /* vm_allocate_with_pager will reset */ 145 object->paging_in_progress = 0; 146 object->copy = NULL; 147 148 /* 149 * Object starts out read-write, with no pager. 150 */ 151 152 object->pager = NULL; 153 object->paging_offset = 0; 154 object->shadow = NULL; 155 object->shadow_offset = (vm_offset_t) 0; 156 157 simple_lock(&vm_object_list_lock); 158 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 159 vm_object_count++; 160 simple_unlock(&vm_object_list_lock); 161 } 162 163 /* 164 * vm_object_reference: 165 * 166 * Gets another reference to the given object. 167 */ 168 void vm_object_reference(object) 169 register vm_object_t object; 170 { 171 if (object == NULL) 172 return; 173 174 vm_object_lock(object); 175 object->ref_count++; 176 vm_object_unlock(object); 177 } 178 179 /* 180 * vm_object_deallocate: 181 * 182 * Release a reference to the specified object, 183 * gained either through a vm_object_allocate 184 * or a vm_object_reference call. When all references 185 * are gone, storage associated with this object 186 * may be relinquished. 187 * 188 * No object may be locked. 189 */ 190 void vm_object_deallocate(object) 191 register vm_object_t object; 192 { 193 vm_object_t temp; 194 195 while (object != NULL) { 196 197 /* 198 * The cache holds a reference (uncounted) to 199 * the object; we must lock it before removing 200 * the object. 201 */ 202 203 vm_object_cache_lock(); 204 205 /* 206 * Lose the reference 207 */ 208 vm_object_lock(object); 209 if (--(object->ref_count) != 0) { 210 211 /* 212 * If there are still references, then 213 * we are done. 214 */ 215 vm_object_unlock(object); 216 vm_object_cache_unlock(); 217 return; 218 } 219 220 /* 221 * See if this object can persist. If so, enter 222 * it in the cache, then deactivate all of its 223 * pages. 224 */ 225 226 if (object->flags & OBJ_CANPERSIST) { 227 228 TAILQ_INSERT_TAIL(&vm_object_cached_list, object, 229 cached_list); 230 vm_object_cached++; 231 vm_object_cache_unlock(); 232 233 vm_object_deactivate_pages(object); 234 vm_object_unlock(object); 235 236 vm_object_cache_trim(); 237 return; 238 } 239 240 /* 241 * Make sure no one can look us up now. 242 */ 243 vm_object_remove(object->pager); 244 vm_object_cache_unlock(); 245 246 temp = object->shadow; 247 vm_object_terminate(object); 248 /* unlocks and deallocates object */ 249 object = temp; 250 } 251 } 252 253 254 /* 255 * vm_object_terminate actually destroys the specified object, freeing 256 * up all previously used resources. 257 * 258 * The object must be locked. 259 */ 260 void vm_object_terminate(object) 261 register vm_object_t object; 262 { 263 register vm_page_t p; 264 vm_object_t shadow_object; 265 266 /* 267 * Detach the object from its shadow if we are the shadow's 268 * copy. 269 */ 270 if ((shadow_object = object->shadow) != NULL) { 271 vm_object_lock(shadow_object); 272 if (shadow_object->copy == object) 273 shadow_object->copy = NULL; 274 #if 0 275 else if (shadow_object->copy != NULL) 276 panic("vm_object_terminate: copy/shadow inconsistency"); 277 #endif 278 vm_object_unlock(shadow_object); 279 } 280 281 /* 282 * Wait until the pageout daemon is through with the object. 283 */ 284 while (object->paging_in_progress) { 285 vm_object_sleep((int)object, object, FALSE); 286 vm_object_lock(object); 287 } 288 289 /* 290 * If not an internal object clean all the pages, removing them 291 * from paging queues as we go. 292 */ 293 if ((object->flags & OBJ_INTERNAL) == 0) { 294 vm_object_page_clean(object, 0, 0, TRUE); 295 vm_object_unlock(object); 296 } 297 298 /* 299 * Now free the pages. 300 * For internal objects, this also removes them from paging queues. 301 */ 302 while ((p = object->memq.tqh_first) != NULL) { 303 VM_PAGE_CHECK(p); 304 vm_page_lock_queues(); 305 vm_page_free(p); 306 vm_page_unlock_queues(); 307 } 308 if ((object->flags & OBJ_INTERNAL) == 0) 309 vm_object_unlock(object); 310 311 /* 312 * Let the pager know object is dead. 313 */ 314 if (object->pager != NULL) 315 vm_pager_deallocate(object->pager); 316 317 simple_lock(&vm_object_list_lock); 318 TAILQ_REMOVE(&vm_object_list, object, object_list); 319 vm_object_count--; 320 simple_unlock(&vm_object_list_lock); 321 322 /* 323 * Free the space for the object. 324 */ 325 free((caddr_t)object, M_VMOBJ); 326 } 327 328 /* 329 * vm_object_page_clean 330 * 331 * Clean all dirty pages in the specified range of object. 332 * If de_queue is TRUE, pages are removed from any paging queue 333 * they were on, otherwise they are left on whatever queue they 334 * were on before the cleaning operation began. 335 * 336 * Odd semantics: if start == end, we clean everything. 337 * 338 * The object must be locked. 339 */ 340 void 341 vm_object_page_clean(object, start, end, de_queue) 342 register vm_object_t object; 343 register vm_offset_t start; 344 register vm_offset_t end; 345 boolean_t de_queue; 346 { 347 register vm_page_t p; 348 int onqueue; 349 350 if (object->pager == NULL) 351 return; 352 353 again: 354 /* 355 * Wait until the pageout daemon is through with the object. 356 */ 357 while (object->paging_in_progress) { 358 vm_object_sleep((int)object, object, FALSE); 359 vm_object_lock(object); 360 } 361 /* 362 * Loop through the object page list cleaning as necessary. 363 */ 364 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 365 if (start == end || 366 p->offset >= start && p->offset < end) { 367 if ((p->flags & PG_CLEAN) && 368 pmap_is_modified(VM_PAGE_TO_PHYS(p))) 369 p->flags &= ~PG_CLEAN; 370 /* 371 * Remove the page from any paging queue. 372 * This needs to be done if either we have been 373 * explicitly asked to do so or it is about to 374 * be cleaned (see comment below). 375 */ 376 if (de_queue || !(p->flags & PG_CLEAN)) { 377 vm_page_lock_queues(); 378 if (p->flags & PG_ACTIVE) { 379 TAILQ_REMOVE(&vm_page_queue_active, 380 p, pageq); 381 p->flags &= ~PG_ACTIVE; 382 cnt.v_active_count--; 383 onqueue = 1; 384 } else if (p->flags & PG_INACTIVE) { 385 TAILQ_REMOVE(&vm_page_queue_inactive, 386 p, pageq); 387 p->flags &= ~PG_INACTIVE; 388 cnt.v_inactive_count--; 389 onqueue = -1; 390 } else 391 onqueue = 0; 392 vm_page_unlock_queues(); 393 } 394 /* 395 * To ensure the state of the page doesn't change 396 * during the clean operation we do two things. 397 * First we set the busy bit and invalidate all 398 * mappings to ensure that thread accesses to the 399 * page block (in vm_fault). Second, we remove 400 * the page from any paging queue to foil the 401 * pageout daemon (vm_pageout_scan). 402 */ 403 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 404 if (!(p->flags & PG_CLEAN)) { 405 p->flags |= PG_BUSY; 406 object->paging_in_progress++; 407 vm_object_unlock(object); 408 (void) vm_pager_put(object->pager, p, TRUE); 409 vm_object_lock(object); 410 object->paging_in_progress--; 411 if (!de_queue && onqueue) { 412 vm_page_lock_queues(); 413 if (onqueue > 0) 414 vm_page_activate(p); 415 else 416 vm_page_deactivate(p); 417 vm_page_unlock_queues(); 418 } 419 p->flags &= ~PG_BUSY; 420 PAGE_WAKEUP(p); 421 goto again; 422 } 423 } 424 } 425 } 426 427 /* 428 * vm_object_deactivate_pages 429 * 430 * Deactivate all pages in the specified object. (Keep its pages 431 * in memory even though it is no longer referenced.) 432 * 433 * The object must be locked. 434 */ 435 void 436 vm_object_deactivate_pages(object) 437 register vm_object_t object; 438 { 439 register vm_page_t p, next; 440 441 for (p = object->memq.tqh_first; p != NULL; p = next) { 442 next = p->listq.tqe_next; 443 vm_page_lock_queues(); 444 vm_page_deactivate(p); 445 vm_page_unlock_queues(); 446 } 447 } 448 449 /* 450 * Trim the object cache to size. 451 */ 452 void 453 vm_object_cache_trim() 454 { 455 register vm_object_t object; 456 457 vm_object_cache_lock(); 458 while (vm_object_cached > vm_cache_max) { 459 object = vm_object_cached_list.tqh_first; 460 vm_object_cache_unlock(); 461 462 if (object != vm_object_lookup(object->pager)) 463 panic("vm_object_deactivate: I'm sooo confused."); 464 465 pager_cache(object, FALSE); 466 467 vm_object_cache_lock(); 468 } 469 vm_object_cache_unlock(); 470 } 471 472 473 /* 474 * vm_object_shutdown() 475 * 476 * Shut down the object system. Unfortunately, while we 477 * may be trying to do this, init is happily waiting for 478 * processes to exit, and therefore will be causing some objects 479 * to be deallocated. To handle this, we gain a fake reference 480 * to all objects we release paging areas for. This will prevent 481 * a duplicate deallocation. This routine is probably full of 482 * race conditions! 483 */ 484 485 void vm_object_shutdown() 486 { 487 register vm_object_t object; 488 489 /* 490 * Clean up the object cache *before* we screw up the reference 491 * counts on all of the objects. 492 */ 493 494 vm_object_cache_clear(); 495 496 printf("free paging spaces: "); 497 498 /* 499 * First we gain a reference to each object so that 500 * no one else will deallocate them. 501 */ 502 503 simple_lock(&vm_object_list_lock); 504 for (object = vm_object_list.tqh_first; 505 object != NULL; 506 object = object->object_list.tqe_next) 507 vm_object_reference(object); 508 simple_unlock(&vm_object_list_lock); 509 510 /* 511 * Now we deallocate all the paging areas. We don't need 512 * to lock anything because we've reduced to a single 513 * processor while shutting down. This also assumes that 514 * no new objects are being created. 515 */ 516 517 for (object = vm_object_list.tqh_first; 518 object != NULL; 519 object = object->object_list.tqe_next) { 520 if (object->pager != NULL) 521 vm_pager_deallocate(object->pager); 522 printf("."); 523 } 524 printf("done.\n"); 525 } 526 527 /* 528 * vm_object_pmap_copy: 529 * 530 * Makes all physical pages in the specified 531 * object range copy-on-write. No writeable 532 * references to these pages should remain. 533 * 534 * The object must *not* be locked. 535 */ 536 void vm_object_pmap_copy(object, start, end) 537 register vm_object_t object; 538 register vm_offset_t start; 539 register vm_offset_t end; 540 { 541 register vm_page_t p; 542 543 if (object == NULL) 544 return; 545 546 vm_object_lock(object); 547 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 548 if ((start <= p->offset) && (p->offset < end)) { 549 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 550 p->flags |= PG_COPYONWRITE; 551 } 552 } 553 vm_object_unlock(object); 554 } 555 556 /* 557 * vm_object_pmap_remove: 558 * 559 * Removes all physical pages in the specified 560 * object range from all physical maps. 561 * 562 * The object must *not* be locked. 563 */ 564 void vm_object_pmap_remove(object, start, end) 565 register vm_object_t object; 566 register vm_offset_t start; 567 register vm_offset_t end; 568 { 569 register vm_page_t p; 570 571 if (object == NULL) 572 return; 573 574 vm_object_lock(object); 575 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 576 if ((start <= p->offset) && (p->offset < end)) 577 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 578 vm_object_unlock(object); 579 } 580 581 /* 582 * vm_object_copy: 583 * 584 * Create a new object which is a copy of an existing 585 * object, and mark all of the pages in the existing 586 * object 'copy-on-write'. The new object has one reference. 587 * Returns the new object. 588 * 589 * May defer the copy until later if the object is not backed 590 * up by a non-default pager. 591 */ 592 void vm_object_copy(src_object, src_offset, size, 593 dst_object, dst_offset, src_needs_copy) 594 register vm_object_t src_object; 595 vm_offset_t src_offset; 596 vm_size_t size; 597 vm_object_t *dst_object; /* OUT */ 598 vm_offset_t *dst_offset; /* OUT */ 599 boolean_t *src_needs_copy; /* OUT */ 600 { 601 register vm_object_t new_copy; 602 register vm_object_t old_copy; 603 vm_offset_t new_start, new_end; 604 605 register vm_page_t p; 606 607 if (src_object == NULL) { 608 /* 609 * Nothing to copy 610 */ 611 *dst_object = NULL; 612 *dst_offset = 0; 613 *src_needs_copy = FALSE; 614 return; 615 } 616 617 /* 618 * If the object's pager is null_pager or the 619 * default pager, we don't have to make a copy 620 * of it. Instead, we set the needs copy flag and 621 * make a shadow later. 622 */ 623 624 vm_object_lock(src_object); 625 if (src_object->pager == NULL || 626 (src_object->flags & OBJ_INTERNAL)) { 627 628 /* 629 * Make another reference to the object 630 */ 631 src_object->ref_count++; 632 633 /* 634 * Mark all of the pages copy-on-write. 635 */ 636 for (p = src_object->memq.tqh_first; p; p = p->listq.tqe_next) 637 if (src_offset <= p->offset && 638 p->offset < src_offset + size) 639 p->flags |= PG_COPYONWRITE; 640 vm_object_unlock(src_object); 641 642 *dst_object = src_object; 643 *dst_offset = src_offset; 644 645 /* 646 * Must make a shadow when write is desired 647 */ 648 *src_needs_copy = TRUE; 649 return; 650 } 651 652 /* 653 * Try to collapse the object before copying it. 654 */ 655 vm_object_collapse(src_object); 656 657 /* 658 * If the object has a pager, the pager wants to 659 * see all of the changes. We need a copy-object 660 * for the changed pages. 661 * 662 * If there is a copy-object, and it is empty, 663 * no changes have been made to the object since the 664 * copy-object was made. We can use the same copy- 665 * object. 666 */ 667 668 Retry1: 669 old_copy = src_object->copy; 670 if (old_copy != NULL) { 671 /* 672 * Try to get the locks (out of order) 673 */ 674 if (!vm_object_lock_try(old_copy)) { 675 vm_object_unlock(src_object); 676 677 /* should spin a bit here... */ 678 vm_object_lock(src_object); 679 goto Retry1; 680 } 681 682 if (old_copy->resident_page_count == 0 && 683 old_copy->pager == NULL) { 684 /* 685 * Return another reference to 686 * the existing copy-object. 687 */ 688 old_copy->ref_count++; 689 vm_object_unlock(old_copy); 690 vm_object_unlock(src_object); 691 *dst_object = old_copy; 692 *dst_offset = src_offset; 693 *src_needs_copy = FALSE; 694 return; 695 } 696 vm_object_unlock(old_copy); 697 } 698 vm_object_unlock(src_object); 699 700 /* 701 * If the object has a pager, the pager wants 702 * to see all of the changes. We must make 703 * a copy-object and put the changed pages there. 704 * 705 * The copy-object is always made large enough to 706 * completely shadow the original object, since 707 * it may have several users who want to shadow 708 * the original object at different points. 709 */ 710 711 new_copy = vm_object_allocate(src_object->size); 712 713 Retry2: 714 vm_object_lock(src_object); 715 /* 716 * Copy object may have changed while we were unlocked 717 */ 718 old_copy = src_object->copy; 719 if (old_copy != NULL) { 720 /* 721 * Try to get the locks (out of order) 722 */ 723 if (!vm_object_lock_try(old_copy)) { 724 vm_object_unlock(src_object); 725 goto Retry2; 726 } 727 728 /* 729 * Consistency check 730 */ 731 if (old_copy->shadow != src_object || 732 old_copy->shadow_offset != (vm_offset_t) 0) 733 panic("vm_object_copy: copy/shadow inconsistency"); 734 735 /* 736 * Make the old copy-object shadow the new one. 737 * It will receive no more pages from the original 738 * object. 739 */ 740 741 src_object->ref_count--; /* remove ref. from old_copy */ 742 old_copy->shadow = new_copy; 743 new_copy->ref_count++; /* locking not needed - we 744 have the only pointer */ 745 vm_object_unlock(old_copy); /* done with old_copy */ 746 } 747 748 new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 749 new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 750 751 /* 752 * Point the new copy at the existing object. 753 */ 754 755 new_copy->shadow = src_object; 756 new_copy->shadow_offset = new_start; 757 src_object->ref_count++; 758 src_object->copy = new_copy; 759 760 /* 761 * Mark all the affected pages of the existing object 762 * copy-on-write. 763 */ 764 for (p = src_object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 765 if ((new_start <= p->offset) && (p->offset < new_end)) 766 p->flags |= PG_COPYONWRITE; 767 768 vm_object_unlock(src_object); 769 770 *dst_object = new_copy; 771 *dst_offset = src_offset - new_start; 772 *src_needs_copy = FALSE; 773 } 774 775 /* 776 * vm_object_shadow: 777 * 778 * Create a new object which is backed by the 779 * specified existing object range. The source 780 * object reference is deallocated. 781 * 782 * The new object and offset into that object 783 * are returned in the source parameters. 784 */ 785 786 void vm_object_shadow(object, offset, length) 787 vm_object_t *object; /* IN/OUT */ 788 vm_offset_t *offset; /* IN/OUT */ 789 vm_size_t length; 790 { 791 register vm_object_t source; 792 register vm_object_t result; 793 794 source = *object; 795 796 /* 797 * Allocate a new object with the given length 798 */ 799 800 if ((result = vm_object_allocate(length)) == NULL) 801 panic("vm_object_shadow: no object for shadowing"); 802 803 /* 804 * The new object shadows the source object, adding 805 * a reference to it. Our caller changes his reference 806 * to point to the new object, removing a reference to 807 * the source object. Net result: no change of reference 808 * count. 809 */ 810 result->shadow = source; 811 812 /* 813 * Store the offset into the source object, 814 * and fix up the offset into the new object. 815 */ 816 817 result->shadow_offset = *offset; 818 819 /* 820 * Return the new things 821 */ 822 823 *offset = 0; 824 *object = result; 825 } 826 827 /* 828 * Set the specified object's pager to the specified pager. 829 */ 830 831 void vm_object_setpager(object, pager, paging_offset, 832 read_only) 833 vm_object_t object; 834 vm_pager_t pager; 835 vm_offset_t paging_offset; 836 boolean_t read_only; 837 { 838 #ifdef lint 839 read_only++; /* No longer used */ 840 #endif 841 842 vm_object_lock(object); /* XXX ? */ 843 object->pager = pager; 844 object->paging_offset = paging_offset; 845 vm_object_unlock(object); /* XXX ? */ 846 } 847 848 /* 849 * vm_object_hash hashes the pager/id pair. 850 */ 851 852 #define vm_object_hash(pager) \ 853 (((unsigned)pager)%VM_OBJECT_HASH_COUNT) 854 855 /* 856 * vm_object_lookup looks in the object cache for an object with the 857 * specified pager and paging id. 858 */ 859 860 vm_object_t vm_object_lookup(pager) 861 vm_pager_t pager; 862 { 863 register vm_object_hash_entry_t entry; 864 vm_object_t object; 865 866 vm_object_cache_lock(); 867 868 for (entry = vm_object_hashtable[vm_object_hash(pager)].tqh_first; 869 entry != NULL; 870 entry = entry->hash_links.tqe_next) { 871 object = entry->object; 872 if (object->pager == pager) { 873 vm_object_lock(object); 874 if (object->ref_count == 0) { 875 TAILQ_REMOVE(&vm_object_cached_list, object, 876 cached_list); 877 vm_object_cached--; 878 } 879 object->ref_count++; 880 vm_object_unlock(object); 881 vm_object_cache_unlock(); 882 return(object); 883 } 884 } 885 886 vm_object_cache_unlock(); 887 return(NULL); 888 } 889 890 /* 891 * vm_object_enter enters the specified object/pager/id into 892 * the hash table. 893 */ 894 895 void vm_object_enter(object, pager) 896 vm_object_t object; 897 vm_pager_t pager; 898 { 899 struct vm_object_hash_head *bucket; 900 register vm_object_hash_entry_t entry; 901 902 /* 903 * We don't cache null objects, and we can't cache 904 * objects with the null pager. 905 */ 906 907 if (object == NULL) 908 return; 909 if (pager == NULL) 910 return; 911 912 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 913 entry = (vm_object_hash_entry_t) 914 malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK); 915 entry->object = object; 916 object->flags |= OBJ_CANPERSIST; 917 918 vm_object_cache_lock(); 919 TAILQ_INSERT_TAIL(bucket, entry, hash_links); 920 vm_object_cache_unlock(); 921 } 922 923 /* 924 * vm_object_remove: 925 * 926 * Remove the pager from the hash table. 927 * Note: This assumes that the object cache 928 * is locked. XXX this should be fixed 929 * by reorganizing vm_object_deallocate. 930 */ 931 void 932 vm_object_remove(pager) 933 register vm_pager_t pager; 934 { 935 struct vm_object_hash_head *bucket; 936 register vm_object_hash_entry_t entry; 937 register vm_object_t object; 938 939 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 940 941 for (entry = bucket->tqh_first; 942 entry != NULL; 943 entry = entry->hash_links.tqe_next) { 944 object = entry->object; 945 if (object->pager == pager) { 946 TAILQ_REMOVE(bucket, entry, hash_links); 947 free((caddr_t)entry, M_VMOBJHASH); 948 break; 949 } 950 } 951 } 952 953 /* 954 * vm_object_cache_clear removes all objects from the cache. 955 * 956 */ 957 958 void vm_object_cache_clear() 959 { 960 register vm_object_t object; 961 962 /* 963 * Remove each object in the cache by scanning down the 964 * list of cached objects. 965 */ 966 vm_object_cache_lock(); 967 while ((object = vm_object_cached_list.tqh_first) != NULL) { 968 vm_object_cache_unlock(); 969 970 /* 971 * Note: it is important that we use vm_object_lookup 972 * to gain a reference, and not vm_object_reference, because 973 * the logic for removing an object from the cache lies in 974 * lookup. 975 */ 976 if (object != vm_object_lookup(object->pager)) 977 panic("vm_object_cache_clear: I'm sooo confused."); 978 pager_cache(object, FALSE); 979 980 vm_object_cache_lock(); 981 } 982 vm_object_cache_unlock(); 983 } 984 985 boolean_t vm_object_collapse_allowed = TRUE; 986 /* 987 * vm_object_collapse: 988 * 989 * Collapse an object with the object backing it. 990 * Pages in the backing object are moved into the 991 * parent, and the backing object is deallocated. 992 * 993 * Requires that the object be locked and the page 994 * queues be unlocked. 995 * 996 */ 997 void vm_object_collapse(object) 998 register vm_object_t object; 999 1000 { 1001 register vm_object_t backing_object; 1002 register vm_offset_t backing_offset; 1003 register vm_size_t size; 1004 register vm_offset_t new_offset; 1005 register vm_page_t p, pp; 1006 1007 if (!vm_object_collapse_allowed) 1008 return; 1009 1010 while (TRUE) { 1011 /* 1012 * Verify that the conditions are right for collapse: 1013 * 1014 * The object exists and no pages in it are currently 1015 * being paged out (or have ever been paged out). 1016 */ 1017 if (object == NULL || 1018 object->paging_in_progress != 0 || 1019 object->pager != NULL) 1020 return; 1021 1022 /* 1023 * There is a backing object, and 1024 */ 1025 1026 if ((backing_object = object->shadow) == NULL) 1027 return; 1028 1029 vm_object_lock(backing_object); 1030 /* 1031 * ... 1032 * The backing object is not read_only, 1033 * and no pages in the backing object are 1034 * currently being paged out. 1035 * The backing object is internal. 1036 */ 1037 1038 if ((backing_object->flags & OBJ_INTERNAL) == 0 || 1039 backing_object->paging_in_progress != 0) { 1040 vm_object_unlock(backing_object); 1041 return; 1042 } 1043 1044 /* 1045 * The backing object can't be a copy-object: 1046 * the shadow_offset for the copy-object must stay 1047 * as 0. Furthermore (for the 'we have all the 1048 * pages' case), if we bypass backing_object and 1049 * just shadow the next object in the chain, old 1050 * pages from that object would then have to be copied 1051 * BOTH into the (former) backing_object and into the 1052 * parent object. 1053 */ 1054 if (backing_object->shadow != NULL && 1055 backing_object->shadow->copy != NULL) { 1056 vm_object_unlock(backing_object); 1057 return; 1058 } 1059 1060 /* 1061 * We know that we can either collapse the backing 1062 * object (if the parent is the only reference to 1063 * it) or (perhaps) remove the parent's reference 1064 * to it. 1065 */ 1066 1067 backing_offset = object->shadow_offset; 1068 size = object->size; 1069 1070 /* 1071 * If there is exactly one reference to the backing 1072 * object, we can collapse it into the parent. 1073 */ 1074 1075 if (backing_object->ref_count == 1) { 1076 1077 /* 1078 * We can collapse the backing object. 1079 * 1080 * Move all in-memory pages from backing_object 1081 * to the parent. Pages that have been paged out 1082 * will be overwritten by any of the parent's 1083 * pages that shadow them. 1084 */ 1085 1086 while ((p = backing_object->memq.tqh_first) != NULL) { 1087 1088 new_offset = (p->offset - backing_offset); 1089 1090 /* 1091 * If the parent has a page here, or if 1092 * this page falls outside the parent, 1093 * dispose of it. 1094 * 1095 * Otherwise, move it as planned. 1096 */ 1097 1098 if (p->offset < backing_offset || 1099 new_offset >= size) { 1100 vm_page_lock_queues(); 1101 vm_page_free(p); 1102 vm_page_unlock_queues(); 1103 } else { 1104 pp = vm_page_lookup(object, new_offset); 1105 if (pp != NULL && !(pp->flags & PG_FAKE)) { 1106 vm_page_lock_queues(); 1107 vm_page_free(p); 1108 vm_page_unlock_queues(); 1109 } 1110 else { 1111 if (pp) { 1112 /* may be someone waiting for it */ 1113 PAGE_WAKEUP(pp); 1114 vm_page_lock_queues(); 1115 vm_page_free(pp); 1116 vm_page_unlock_queues(); 1117 } 1118 vm_page_rename(p, object, new_offset); 1119 } 1120 } 1121 } 1122 1123 /* 1124 * Move the pager from backing_object to object. 1125 * 1126 * XXX We're only using part of the paging space 1127 * for keeps now... we ought to discard the 1128 * unused portion. 1129 */ 1130 1131 object->pager = backing_object->pager; 1132 object->paging_offset += backing_offset; 1133 1134 backing_object->pager = NULL; 1135 1136 /* 1137 * Object now shadows whatever backing_object did. 1138 * Note that the reference to backing_object->shadow 1139 * moves from within backing_object to within object. 1140 */ 1141 1142 object->shadow = backing_object->shadow; 1143 object->shadow_offset += backing_object->shadow_offset; 1144 if (object->shadow != NULL && 1145 object->shadow->copy != NULL) { 1146 panic("vm_object_collapse: we collapsed a copy-object!"); 1147 } 1148 /* 1149 * Discard backing_object. 1150 * 1151 * Since the backing object has no pages, no 1152 * pager left, and no object references within it, 1153 * all that is necessary is to dispose of it. 1154 */ 1155 1156 vm_object_unlock(backing_object); 1157 1158 simple_lock(&vm_object_list_lock); 1159 TAILQ_REMOVE(&vm_object_list, backing_object, 1160 object_list); 1161 vm_object_count--; 1162 simple_unlock(&vm_object_list_lock); 1163 1164 free((caddr_t)backing_object, M_VMOBJ); 1165 1166 object_collapses++; 1167 } 1168 else { 1169 /* 1170 * If all of the pages in the backing object are 1171 * shadowed by the parent object, the parent 1172 * object no longer has to shadow the backing 1173 * object; it can shadow the next one in the 1174 * chain. 1175 * 1176 * The backing object must not be paged out - we'd 1177 * have to check all of the paged-out pages, as 1178 * well. 1179 */ 1180 1181 if (backing_object->pager != NULL) { 1182 vm_object_unlock(backing_object); 1183 return; 1184 } 1185 1186 /* 1187 * Should have a check for a 'small' number 1188 * of pages here. 1189 */ 1190 1191 for (p = backing_object->memq.tqh_first; 1192 p != NULL; 1193 p = p->listq.tqe_next) { 1194 new_offset = (p->offset - backing_offset); 1195 1196 /* 1197 * If the parent has a page here, or if 1198 * this page falls outside the parent, 1199 * keep going. 1200 * 1201 * Otherwise, the backing_object must be 1202 * left in the chain. 1203 */ 1204 1205 if (p->offset >= backing_offset && 1206 new_offset <= size && 1207 ((pp = vm_page_lookup(object, new_offset)) 1208 == NULL || 1209 (pp->flags & PG_FAKE))) { 1210 /* 1211 * Page still needed. 1212 * Can't go any further. 1213 */ 1214 vm_object_unlock(backing_object); 1215 return; 1216 } 1217 } 1218 1219 /* 1220 * Make the parent shadow the next object 1221 * in the chain. Deallocating backing_object 1222 * will not remove it, since its reference 1223 * count is at least 2. 1224 */ 1225 1226 vm_object_reference(object->shadow = backing_object->shadow); 1227 object->shadow_offset += backing_object->shadow_offset; 1228 1229 /* Drop the reference count on backing_object. 1230 * Since its ref_count was at least 2, it 1231 * will not vanish; so we don't need to call 1232 * vm_object_deallocate. 1233 */ 1234 backing_object->ref_count--; 1235 vm_object_unlock(backing_object); 1236 1237 object_bypasses ++; 1238 1239 } 1240 1241 /* 1242 * Try again with this object's new backing object. 1243 */ 1244 } 1245 } 1246 1247 /* 1248 * vm_object_page_remove: [internal] 1249 * 1250 * Removes all physical pages in the specified 1251 * object range from the object's list of pages. 1252 * 1253 * The object must be locked. 1254 */ 1255 void vm_object_page_remove(object, start, end) 1256 register vm_object_t object; 1257 register vm_offset_t start; 1258 register vm_offset_t end; 1259 { 1260 register vm_page_t p, next; 1261 1262 if (object == NULL) 1263 return; 1264 1265 for (p = object->memq.tqh_first; p != NULL; p = next) { 1266 next = p->listq.tqe_next; 1267 if ((start <= p->offset) && (p->offset < end)) { 1268 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1269 vm_page_lock_queues(); 1270 vm_page_free(p); 1271 vm_page_unlock_queues(); 1272 } 1273 } 1274 } 1275 1276 /* 1277 * Routine: vm_object_coalesce 1278 * Function: Coalesces two objects backing up adjoining 1279 * regions of memory into a single object. 1280 * 1281 * returns TRUE if objects were combined. 1282 * 1283 * NOTE: Only works at the moment if the second object is NULL - 1284 * if it's not, which object do we lock first? 1285 * 1286 * Parameters: 1287 * prev_object First object to coalesce 1288 * prev_offset Offset into prev_object 1289 * next_object Second object into coalesce 1290 * next_offset Offset into next_object 1291 * 1292 * prev_size Size of reference to prev_object 1293 * next_size Size of reference to next_object 1294 * 1295 * Conditions: 1296 * The object must *not* be locked. 1297 */ 1298 boolean_t vm_object_coalesce(prev_object, next_object, 1299 prev_offset, next_offset, 1300 prev_size, next_size) 1301 1302 register vm_object_t prev_object; 1303 vm_object_t next_object; 1304 vm_offset_t prev_offset, next_offset; 1305 vm_size_t prev_size, next_size; 1306 { 1307 vm_size_t newsize; 1308 1309 #ifdef lint 1310 next_offset++; 1311 #endif 1312 1313 if (next_object != NULL) { 1314 return(FALSE); 1315 } 1316 1317 if (prev_object == NULL) { 1318 return(TRUE); 1319 } 1320 1321 vm_object_lock(prev_object); 1322 1323 /* 1324 * Try to collapse the object first 1325 */ 1326 vm_object_collapse(prev_object); 1327 1328 /* 1329 * Can't coalesce if: 1330 * . more than one reference 1331 * . paged out 1332 * . shadows another object 1333 * . has a copy elsewhere 1334 * (any of which mean that the pages not mapped to 1335 * prev_entry may be in use anyway) 1336 */ 1337 1338 if (prev_object->ref_count > 1 || 1339 prev_object->pager != NULL || 1340 prev_object->shadow != NULL || 1341 prev_object->copy != NULL) { 1342 vm_object_unlock(prev_object); 1343 return(FALSE); 1344 } 1345 1346 /* 1347 * Remove any pages that may still be in the object from 1348 * a previous deallocation. 1349 */ 1350 1351 vm_object_page_remove(prev_object, 1352 prev_offset + prev_size, 1353 prev_offset + prev_size + next_size); 1354 1355 /* 1356 * Extend the object if necessary. 1357 */ 1358 newsize = prev_offset + prev_size + next_size; 1359 if (newsize > prev_object->size) 1360 prev_object->size = newsize; 1361 1362 vm_object_unlock(prev_object); 1363 return(TRUE); 1364 } 1365 1366 /* 1367 * vm_object_print: [ debug ] 1368 */ 1369 void vm_object_print(object, full) 1370 vm_object_t object; 1371 boolean_t full; 1372 { 1373 register vm_page_t p; 1374 extern indent; 1375 1376 register int count; 1377 1378 if (object == NULL) 1379 return; 1380 1381 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1382 (int) object, (int) object->size, 1383 object->resident_page_count, object->ref_count); 1384 printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1385 (int) object->pager, (int) object->paging_offset, 1386 (int) object->shadow, (int) object->shadow_offset); 1387 printf("cache: next=0x%x, prev=0x%x\n", 1388 object->cached_list.tqe_next, object->cached_list.tqe_prev); 1389 1390 if (!full) 1391 return; 1392 1393 indent += 2; 1394 count = 0; 1395 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 1396 if (count == 0) 1397 iprintf("memory:="); 1398 else if (count == 6) { 1399 printf("\n"); 1400 iprintf(" ..."); 1401 count = 0; 1402 } else 1403 printf(","); 1404 count++; 1405 1406 printf("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p)); 1407 } 1408 if (count != 0) 1409 printf("\n"); 1410 indent -= 2; 1411 } 1412