1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_object.c 8.1 (Berkeley) 06/11/93 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Virtual memory object module. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/malloc.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 50 /* 51 * Virtual memory objects maintain the actual data 52 * associated with allocated virtual memory. A given 53 * page of memory exists within exactly one object. 54 * 55 * An object is only deallocated when all "references" 56 * are given up. Only one "reference" to a given 57 * region of an object should be writeable. 58 * 59 * Associated with each object is a list of all resident 60 * memory pages belonging to that object; this list is 61 * maintained by the "vm_page" module, and locked by the object's 62 * lock. 63 * 64 * Each object also records a "pager" routine which is 65 * used to retrieve (and store) pages to the proper backing 66 * storage. In addition, objects may be backed by other 67 * objects from which they were virtual-copied. 68 * 69 * The only items within the object structure which are 70 * modified after time of creation are: 71 * reference count locked by object's lock 72 * pager routine locked by object's lock 73 * 74 */ 75 76 struct vm_object kernel_object_store; 77 struct vm_object kmem_object_store; 78 79 #define VM_OBJECT_HASH_COUNT 157 80 81 int vm_cache_max = 100; /* can patch if necessary */ 82 queue_head_t vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 83 84 long object_collapses = 0; 85 long object_bypasses = 0; 86 87 static void _vm_object_allocate __P((vm_size_t, vm_object_t)); 88 89 /* 90 * vm_object_init: 91 * 92 * Initialize the VM objects module. 93 */ 94 void vm_object_init(size) 95 vm_size_t size; 96 { 97 register int i; 98 99 queue_init(&vm_object_cached_list); 100 queue_init(&vm_object_list); 101 vm_object_count = 0; 102 simple_lock_init(&vm_cache_lock); 103 simple_lock_init(&vm_object_list_lock); 104 105 for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 106 queue_init(&vm_object_hashtable[i]); 107 108 kernel_object = &kernel_object_store; 109 _vm_object_allocate(size, kernel_object); 110 111 kmem_object = &kmem_object_store; 112 _vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object); 113 } 114 115 /* 116 * vm_object_allocate: 117 * 118 * Returns a new object with the given size. 119 */ 120 121 vm_object_t vm_object_allocate(size) 122 vm_size_t size; 123 { 124 register vm_object_t result; 125 126 result = (vm_object_t) 127 malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK); 128 129 _vm_object_allocate(size, result); 130 131 return(result); 132 } 133 134 static void 135 _vm_object_allocate(size, object) 136 vm_size_t size; 137 register vm_object_t object; 138 { 139 queue_init(&object->memq); 140 vm_object_lock_init(object); 141 object->ref_count = 1; 142 object->resident_page_count = 0; 143 object->size = size; 144 object->flags = OBJ_INTERNAL; /* vm_allocate_with_pager will reset */ 145 object->paging_in_progress = 0; 146 object->copy = NULL; 147 148 /* 149 * Object starts out read-write, with no pager. 150 */ 151 152 object->pager = NULL; 153 object->paging_offset = 0; 154 object->shadow = NULL; 155 object->shadow_offset = (vm_offset_t) 0; 156 157 simple_lock(&vm_object_list_lock); 158 queue_enter(&vm_object_list, object, vm_object_t, object_list); 159 vm_object_count++; 160 simple_unlock(&vm_object_list_lock); 161 } 162 163 /* 164 * vm_object_reference: 165 * 166 * Gets another reference to the given object. 167 */ 168 void vm_object_reference(object) 169 register vm_object_t object; 170 { 171 if (object == NULL) 172 return; 173 174 vm_object_lock(object); 175 object->ref_count++; 176 vm_object_unlock(object); 177 } 178 179 /* 180 * vm_object_deallocate: 181 * 182 * Release a reference to the specified object, 183 * gained either through a vm_object_allocate 184 * or a vm_object_reference call. When all references 185 * are gone, storage associated with this object 186 * may be relinquished. 187 * 188 * No object may be locked. 189 */ 190 void vm_object_deallocate(object) 191 register vm_object_t object; 192 { 193 vm_object_t temp; 194 195 while (object != NULL) { 196 197 /* 198 * The cache holds a reference (uncounted) to 199 * the object; we must lock it before removing 200 * the object. 201 */ 202 203 vm_object_cache_lock(); 204 205 /* 206 * Lose the reference 207 */ 208 vm_object_lock(object); 209 if (--(object->ref_count) != 0) { 210 211 /* 212 * If there are still references, then 213 * we are done. 214 */ 215 vm_object_unlock(object); 216 vm_object_cache_unlock(); 217 return; 218 } 219 220 /* 221 * See if this object can persist. If so, enter 222 * it in the cache, then deactivate all of its 223 * pages. 224 */ 225 226 if (object->flags & OBJ_CANPERSIST) { 227 228 queue_enter(&vm_object_cached_list, object, 229 vm_object_t, cached_list); 230 vm_object_cached++; 231 vm_object_cache_unlock(); 232 233 vm_object_deactivate_pages(object); 234 vm_object_unlock(object); 235 236 vm_object_cache_trim(); 237 return; 238 } 239 240 /* 241 * Make sure no one can look us up now. 242 */ 243 vm_object_remove(object->pager); 244 vm_object_cache_unlock(); 245 246 temp = object->shadow; 247 vm_object_terminate(object); 248 /* unlocks and deallocates object */ 249 object = temp; 250 } 251 } 252 253 254 /* 255 * vm_object_terminate actually destroys the specified object, freeing 256 * up all previously used resources. 257 * 258 * The object must be locked. 259 */ 260 void vm_object_terminate(object) 261 register vm_object_t object; 262 { 263 register vm_page_t p; 264 vm_object_t shadow_object; 265 266 /* 267 * Detach the object from its shadow if we are the shadow's 268 * copy. 269 */ 270 if ((shadow_object = object->shadow) != NULL) { 271 vm_object_lock(shadow_object); 272 if (shadow_object->copy == object) 273 shadow_object->copy = NULL; 274 #if 0 275 else if (shadow_object->copy != NULL) 276 panic("vm_object_terminate: copy/shadow inconsistency"); 277 #endif 278 vm_object_unlock(shadow_object); 279 } 280 281 /* 282 * Wait until the pageout daemon is through with the object. 283 */ 284 while (object->paging_in_progress) { 285 vm_object_sleep((int)object, object, FALSE); 286 vm_object_lock(object); 287 } 288 289 /* 290 * If not an internal object clean all the pages, removing them 291 * from paging queues as we go. 292 */ 293 if ((object->flags & OBJ_INTERNAL) == 0) { 294 vm_object_page_clean(object, 0, 0, TRUE); 295 vm_object_unlock(object); 296 } 297 298 /* 299 * Now free the pages. 300 * For internal objects, this also removes them from paging queues. 301 */ 302 while (!queue_empty(&object->memq)) { 303 p = (vm_page_t) queue_first(&object->memq); 304 VM_PAGE_CHECK(p); 305 vm_page_lock_queues(); 306 vm_page_free(p); 307 vm_page_unlock_queues(); 308 } 309 if ((object->flags & OBJ_INTERNAL) == 0) 310 vm_object_unlock(object); 311 312 /* 313 * Let the pager know object is dead. 314 */ 315 if (object->pager != NULL) 316 vm_pager_deallocate(object->pager); 317 318 simple_lock(&vm_object_list_lock); 319 queue_remove(&vm_object_list, object, vm_object_t, object_list); 320 vm_object_count--; 321 simple_unlock(&vm_object_list_lock); 322 323 /* 324 * Free the space for the object. 325 */ 326 free((caddr_t)object, M_VMOBJ); 327 } 328 329 /* 330 * vm_object_page_clean 331 * 332 * Clean all dirty pages in the specified range of object. 333 * If de_queue is TRUE, pages are removed from any paging queue 334 * they were on, otherwise they are left on whatever queue they 335 * were on before the cleaning operation began. 336 * 337 * Odd semantics: if start == end, we clean everything. 338 * 339 * The object must be locked. 340 */ 341 void 342 vm_object_page_clean(object, start, end, de_queue) 343 register vm_object_t object; 344 register vm_offset_t start; 345 register vm_offset_t end; 346 boolean_t de_queue; 347 { 348 register vm_page_t p; 349 int onqueue; 350 351 if (object->pager == NULL) 352 return; 353 354 again: 355 /* 356 * Wait until the pageout daemon is through with the object. 357 */ 358 while (object->paging_in_progress) { 359 vm_object_sleep((int)object, object, FALSE); 360 vm_object_lock(object); 361 } 362 /* 363 * Loop through the object page list cleaning as necessary. 364 */ 365 p = (vm_page_t) queue_first(&object->memq); 366 while (!queue_end(&object->memq, (queue_entry_t) p)) { 367 if (start == end || 368 p->offset >= start && p->offset < end) { 369 if ((p->flags & PG_CLEAN) && 370 pmap_is_modified(VM_PAGE_TO_PHYS(p))) 371 p->flags &= ~PG_CLEAN; 372 /* 373 * Remove the page from any paging queue. 374 * This needs to be done if either we have been 375 * explicitly asked to do so or it is about to 376 * be cleaned (see comment below). 377 */ 378 if (de_queue || !(p->flags & PG_CLEAN)) { 379 vm_page_lock_queues(); 380 if (p->flags & PG_ACTIVE) { 381 queue_remove(&vm_page_queue_active, 382 p, vm_page_t, pageq); 383 p->flags &= ~PG_ACTIVE; 384 cnt.v_active_count--; 385 onqueue = 1; 386 } else if (p->flags & PG_INACTIVE) { 387 queue_remove(&vm_page_queue_inactive, 388 p, vm_page_t, pageq); 389 p->flags &= ~PG_INACTIVE; 390 cnt.v_inactive_count--; 391 onqueue = -1; 392 } else 393 onqueue = 0; 394 vm_page_unlock_queues(); 395 } 396 /* 397 * To ensure the state of the page doesn't change 398 * during the clean operation we do two things. 399 * First we set the busy bit and invalidate all 400 * mappings to ensure that thread accesses to the 401 * page block (in vm_fault). Second, we remove 402 * the page from any paging queue to foil the 403 * pageout daemon (vm_pageout_scan). 404 */ 405 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 406 if (!(p->flags & PG_CLEAN)) { 407 p->flags |= PG_BUSY; 408 object->paging_in_progress++; 409 vm_object_unlock(object); 410 (void) vm_pager_put(object->pager, p, TRUE); 411 vm_object_lock(object); 412 object->paging_in_progress--; 413 if (!de_queue && onqueue) { 414 vm_page_lock_queues(); 415 if (onqueue > 0) 416 vm_page_activate(p); 417 else 418 vm_page_deactivate(p); 419 vm_page_unlock_queues(); 420 } 421 p->flags &= ~PG_BUSY; 422 PAGE_WAKEUP(p); 423 goto again; 424 } 425 } 426 p = (vm_page_t) queue_next(&p->listq); 427 } 428 } 429 430 /* 431 * vm_object_deactivate_pages 432 * 433 * Deactivate all pages in the specified object. (Keep its pages 434 * in memory even though it is no longer referenced.) 435 * 436 * The object must be locked. 437 */ 438 void 439 vm_object_deactivate_pages(object) 440 register vm_object_t object; 441 { 442 register vm_page_t p, next; 443 444 p = (vm_page_t) queue_first(&object->memq); 445 while (!queue_end(&object->memq, (queue_entry_t) p)) { 446 next = (vm_page_t) queue_next(&p->listq); 447 vm_page_lock_queues(); 448 vm_page_deactivate(p); 449 vm_page_unlock_queues(); 450 p = next; 451 } 452 } 453 454 /* 455 * Trim the object cache to size. 456 */ 457 void 458 vm_object_cache_trim() 459 { 460 register vm_object_t object; 461 462 vm_object_cache_lock(); 463 while (vm_object_cached > vm_cache_max) { 464 object = (vm_object_t) queue_first(&vm_object_cached_list); 465 vm_object_cache_unlock(); 466 467 if (object != vm_object_lookup(object->pager)) 468 panic("vm_object_deactivate: I'm sooo confused."); 469 470 pager_cache(object, FALSE); 471 472 vm_object_cache_lock(); 473 } 474 vm_object_cache_unlock(); 475 } 476 477 478 /* 479 * vm_object_shutdown() 480 * 481 * Shut down the object system. Unfortunately, while we 482 * may be trying to do this, init is happily waiting for 483 * processes to exit, and therefore will be causing some objects 484 * to be deallocated. To handle this, we gain a fake reference 485 * to all objects we release paging areas for. This will prevent 486 * a duplicate deallocation. This routine is probably full of 487 * race conditions! 488 */ 489 490 void vm_object_shutdown() 491 { 492 register vm_object_t object; 493 494 /* 495 * Clean up the object cache *before* we screw up the reference 496 * counts on all of the objects. 497 */ 498 499 vm_object_cache_clear(); 500 501 printf("free paging spaces: "); 502 503 /* 504 * First we gain a reference to each object so that 505 * no one else will deallocate them. 506 */ 507 508 simple_lock(&vm_object_list_lock); 509 object = (vm_object_t) queue_first(&vm_object_list); 510 while (!queue_end(&vm_object_list, (queue_entry_t) object)) { 511 vm_object_reference(object); 512 object = (vm_object_t) queue_next(&object->object_list); 513 } 514 simple_unlock(&vm_object_list_lock); 515 516 /* 517 * Now we deallocate all the paging areas. We don't need 518 * to lock anything because we've reduced to a single 519 * processor while shutting down. This also assumes that 520 * no new objects are being created. 521 */ 522 523 object = (vm_object_t) queue_first(&vm_object_list); 524 while (!queue_end(&vm_object_list, (queue_entry_t) object)) { 525 if (object->pager != NULL) 526 vm_pager_deallocate(object->pager); 527 object = (vm_object_t) queue_next(&object->object_list); 528 printf("."); 529 } 530 printf("done.\n"); 531 } 532 533 /* 534 * vm_object_pmap_copy: 535 * 536 * Makes all physical pages in the specified 537 * object range copy-on-write. No writeable 538 * references to these pages should remain. 539 * 540 * The object must *not* be locked. 541 */ 542 void vm_object_pmap_copy(object, start, end) 543 register vm_object_t object; 544 register vm_offset_t start; 545 register vm_offset_t end; 546 { 547 register vm_page_t p; 548 549 if (object == NULL) 550 return; 551 552 vm_object_lock(object); 553 p = (vm_page_t) queue_first(&object->memq); 554 while (!queue_end(&object->memq, (queue_entry_t) p)) { 555 if ((start <= p->offset) && (p->offset < end)) { 556 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 557 p->flags |= PG_COPYONWRITE; 558 } 559 p = (vm_page_t) queue_next(&p->listq); 560 } 561 vm_object_unlock(object); 562 } 563 564 /* 565 * vm_object_pmap_remove: 566 * 567 * Removes all physical pages in the specified 568 * object range from all physical maps. 569 * 570 * The object must *not* be locked. 571 */ 572 void vm_object_pmap_remove(object, start, end) 573 register vm_object_t object; 574 register vm_offset_t start; 575 register vm_offset_t end; 576 { 577 register vm_page_t p; 578 579 if (object == NULL) 580 return; 581 582 vm_object_lock(object); 583 p = (vm_page_t) queue_first(&object->memq); 584 while (!queue_end(&object->memq, (queue_entry_t) p)) { 585 if ((start <= p->offset) && (p->offset < end)) 586 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 587 p = (vm_page_t) queue_next(&p->listq); 588 } 589 vm_object_unlock(object); 590 } 591 592 /* 593 * vm_object_copy: 594 * 595 * Create a new object which is a copy of an existing 596 * object, and mark all of the pages in the existing 597 * object 'copy-on-write'. The new object has one reference. 598 * Returns the new object. 599 * 600 * May defer the copy until later if the object is not backed 601 * up by a non-default pager. 602 */ 603 void vm_object_copy(src_object, src_offset, size, 604 dst_object, dst_offset, src_needs_copy) 605 register vm_object_t src_object; 606 vm_offset_t src_offset; 607 vm_size_t size; 608 vm_object_t *dst_object; /* OUT */ 609 vm_offset_t *dst_offset; /* OUT */ 610 boolean_t *src_needs_copy; /* OUT */ 611 { 612 register vm_object_t new_copy; 613 register vm_object_t old_copy; 614 vm_offset_t new_start, new_end; 615 616 register vm_page_t p; 617 618 if (src_object == NULL) { 619 /* 620 * Nothing to copy 621 */ 622 *dst_object = NULL; 623 *dst_offset = 0; 624 *src_needs_copy = FALSE; 625 return; 626 } 627 628 /* 629 * If the object's pager is null_pager or the 630 * default pager, we don't have to make a copy 631 * of it. Instead, we set the needs copy flag and 632 * make a shadow later. 633 */ 634 635 vm_object_lock(src_object); 636 if (src_object->pager == NULL || 637 (src_object->flags & OBJ_INTERNAL)) { 638 639 /* 640 * Make another reference to the object 641 */ 642 src_object->ref_count++; 643 644 /* 645 * Mark all of the pages copy-on-write. 646 */ 647 for (p = (vm_page_t) queue_first(&src_object->memq); 648 !queue_end(&src_object->memq, (queue_entry_t)p); 649 p = (vm_page_t) queue_next(&p->listq)) { 650 if (src_offset <= p->offset && 651 p->offset < src_offset + size) 652 p->flags |= PG_COPYONWRITE; 653 } 654 vm_object_unlock(src_object); 655 656 *dst_object = src_object; 657 *dst_offset = src_offset; 658 659 /* 660 * Must make a shadow when write is desired 661 */ 662 *src_needs_copy = TRUE; 663 return; 664 } 665 666 /* 667 * Try to collapse the object before copying it. 668 */ 669 vm_object_collapse(src_object); 670 671 /* 672 * If the object has a pager, the pager wants to 673 * see all of the changes. We need a copy-object 674 * for the changed pages. 675 * 676 * If there is a copy-object, and it is empty, 677 * no changes have been made to the object since the 678 * copy-object was made. We can use the same copy- 679 * object. 680 */ 681 682 Retry1: 683 old_copy = src_object->copy; 684 if (old_copy != NULL) { 685 /* 686 * Try to get the locks (out of order) 687 */ 688 if (!vm_object_lock_try(old_copy)) { 689 vm_object_unlock(src_object); 690 691 /* should spin a bit here... */ 692 vm_object_lock(src_object); 693 goto Retry1; 694 } 695 696 if (old_copy->resident_page_count == 0 && 697 old_copy->pager == NULL) { 698 /* 699 * Return another reference to 700 * the existing copy-object. 701 */ 702 old_copy->ref_count++; 703 vm_object_unlock(old_copy); 704 vm_object_unlock(src_object); 705 *dst_object = old_copy; 706 *dst_offset = src_offset; 707 *src_needs_copy = FALSE; 708 return; 709 } 710 vm_object_unlock(old_copy); 711 } 712 vm_object_unlock(src_object); 713 714 /* 715 * If the object has a pager, the pager wants 716 * to see all of the changes. We must make 717 * a copy-object and put the changed pages there. 718 * 719 * The copy-object is always made large enough to 720 * completely shadow the original object, since 721 * it may have several users who want to shadow 722 * the original object at different points. 723 */ 724 725 new_copy = vm_object_allocate(src_object->size); 726 727 Retry2: 728 vm_object_lock(src_object); 729 /* 730 * Copy object may have changed while we were unlocked 731 */ 732 old_copy = src_object->copy; 733 if (old_copy != NULL) { 734 /* 735 * Try to get the locks (out of order) 736 */ 737 if (!vm_object_lock_try(old_copy)) { 738 vm_object_unlock(src_object); 739 goto Retry2; 740 } 741 742 /* 743 * Consistency check 744 */ 745 if (old_copy->shadow != src_object || 746 old_copy->shadow_offset != (vm_offset_t) 0) 747 panic("vm_object_copy: copy/shadow inconsistency"); 748 749 /* 750 * Make the old copy-object shadow the new one. 751 * It will receive no more pages from the original 752 * object. 753 */ 754 755 src_object->ref_count--; /* remove ref. from old_copy */ 756 old_copy->shadow = new_copy; 757 new_copy->ref_count++; /* locking not needed - we 758 have the only pointer */ 759 vm_object_unlock(old_copy); /* done with old_copy */ 760 } 761 762 new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 763 new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 764 765 /* 766 * Point the new copy at the existing object. 767 */ 768 769 new_copy->shadow = src_object; 770 new_copy->shadow_offset = new_start; 771 src_object->ref_count++; 772 src_object->copy = new_copy; 773 774 /* 775 * Mark all the affected pages of the existing object 776 * copy-on-write. 777 */ 778 p = (vm_page_t) queue_first(&src_object->memq); 779 while (!queue_end(&src_object->memq, (queue_entry_t) p)) { 780 if ((new_start <= p->offset) && (p->offset < new_end)) 781 p->flags |= PG_COPYONWRITE; 782 p = (vm_page_t) queue_next(&p->listq); 783 } 784 785 vm_object_unlock(src_object); 786 787 *dst_object = new_copy; 788 *dst_offset = src_offset - new_start; 789 *src_needs_copy = FALSE; 790 } 791 792 /* 793 * vm_object_shadow: 794 * 795 * Create a new object which is backed by the 796 * specified existing object range. The source 797 * object reference is deallocated. 798 * 799 * The new object and offset into that object 800 * are returned in the source parameters. 801 */ 802 803 void vm_object_shadow(object, offset, length) 804 vm_object_t *object; /* IN/OUT */ 805 vm_offset_t *offset; /* IN/OUT */ 806 vm_size_t length; 807 { 808 register vm_object_t source; 809 register vm_object_t result; 810 811 source = *object; 812 813 /* 814 * Allocate a new object with the given length 815 */ 816 817 if ((result = vm_object_allocate(length)) == NULL) 818 panic("vm_object_shadow: no object for shadowing"); 819 820 /* 821 * The new object shadows the source object, adding 822 * a reference to it. Our caller changes his reference 823 * to point to the new object, removing a reference to 824 * the source object. Net result: no change of reference 825 * count. 826 */ 827 result->shadow = source; 828 829 /* 830 * Store the offset into the source object, 831 * and fix up the offset into the new object. 832 */ 833 834 result->shadow_offset = *offset; 835 836 /* 837 * Return the new things 838 */ 839 840 *offset = 0; 841 *object = result; 842 } 843 844 /* 845 * Set the specified object's pager to the specified pager. 846 */ 847 848 void vm_object_setpager(object, pager, paging_offset, 849 read_only) 850 vm_object_t object; 851 vm_pager_t pager; 852 vm_offset_t paging_offset; 853 boolean_t read_only; 854 { 855 #ifdef lint 856 read_only++; /* No longer used */ 857 #endif 858 859 vm_object_lock(object); /* XXX ? */ 860 object->pager = pager; 861 object->paging_offset = paging_offset; 862 vm_object_unlock(object); /* XXX ? */ 863 } 864 865 /* 866 * vm_object_hash hashes the pager/id pair. 867 */ 868 869 #define vm_object_hash(pager) \ 870 (((unsigned)pager)%VM_OBJECT_HASH_COUNT) 871 872 /* 873 * vm_object_lookup looks in the object cache for an object with the 874 * specified pager and paging id. 875 */ 876 877 vm_object_t vm_object_lookup(pager) 878 vm_pager_t pager; 879 { 880 register queue_t bucket; 881 register vm_object_hash_entry_t entry; 882 vm_object_t object; 883 884 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 885 886 vm_object_cache_lock(); 887 888 entry = (vm_object_hash_entry_t) queue_first(bucket); 889 while (!queue_end(bucket, (queue_entry_t) entry)) { 890 object = entry->object; 891 if (object->pager == pager) { 892 vm_object_lock(object); 893 if (object->ref_count == 0) { 894 queue_remove(&vm_object_cached_list, object, 895 vm_object_t, cached_list); 896 vm_object_cached--; 897 } 898 object->ref_count++; 899 vm_object_unlock(object); 900 vm_object_cache_unlock(); 901 return(object); 902 } 903 entry = (vm_object_hash_entry_t) queue_next(&entry->hash_links); 904 } 905 906 vm_object_cache_unlock(); 907 return(NULL); 908 } 909 910 /* 911 * vm_object_enter enters the specified object/pager/id into 912 * the hash table. 913 */ 914 915 void vm_object_enter(object, pager) 916 vm_object_t object; 917 vm_pager_t pager; 918 { 919 register queue_t bucket; 920 register vm_object_hash_entry_t entry; 921 922 /* 923 * We don't cache null objects, and we can't cache 924 * objects with the null pager. 925 */ 926 927 if (object == NULL) 928 return; 929 if (pager == NULL) 930 return; 931 932 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 933 entry = (vm_object_hash_entry_t) 934 malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK); 935 entry->object = object; 936 object->flags |= OBJ_CANPERSIST; 937 938 vm_object_cache_lock(); 939 queue_enter(bucket, entry, vm_object_hash_entry_t, hash_links); 940 vm_object_cache_unlock(); 941 } 942 943 /* 944 * vm_object_remove: 945 * 946 * Remove the pager from the hash table. 947 * Note: This assumes that the object cache 948 * is locked. XXX this should be fixed 949 * by reorganizing vm_object_deallocate. 950 */ 951 void 952 vm_object_remove(pager) 953 register vm_pager_t pager; 954 { 955 register queue_t bucket; 956 register vm_object_hash_entry_t entry; 957 register vm_object_t object; 958 959 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 960 961 entry = (vm_object_hash_entry_t) queue_first(bucket); 962 while (!queue_end(bucket, (queue_entry_t) entry)) { 963 object = entry->object; 964 if (object->pager == pager) { 965 queue_remove(bucket, entry, vm_object_hash_entry_t, 966 hash_links); 967 free((caddr_t)entry, M_VMOBJHASH); 968 break; 969 } 970 entry = (vm_object_hash_entry_t) queue_next(&entry->hash_links); 971 } 972 } 973 974 /* 975 * vm_object_cache_clear removes all objects from the cache. 976 * 977 */ 978 979 void vm_object_cache_clear() 980 { 981 register vm_object_t object; 982 983 /* 984 * Remove each object in the cache by scanning down the 985 * list of cached objects. 986 */ 987 vm_object_cache_lock(); 988 while (!queue_empty(&vm_object_cached_list)) { 989 object = (vm_object_t) queue_first(&vm_object_cached_list); 990 vm_object_cache_unlock(); 991 992 /* 993 * Note: it is important that we use vm_object_lookup 994 * to gain a reference, and not vm_object_reference, because 995 * the logic for removing an object from the cache lies in 996 * lookup. 997 */ 998 if (object != vm_object_lookup(object->pager)) 999 panic("vm_object_cache_clear: I'm sooo confused."); 1000 pager_cache(object, FALSE); 1001 1002 vm_object_cache_lock(); 1003 } 1004 vm_object_cache_unlock(); 1005 } 1006 1007 boolean_t vm_object_collapse_allowed = TRUE; 1008 /* 1009 * vm_object_collapse: 1010 * 1011 * Collapse an object with the object backing it. 1012 * Pages in the backing object are moved into the 1013 * parent, and the backing object is deallocated. 1014 * 1015 * Requires that the object be locked and the page 1016 * queues be unlocked. 1017 * 1018 */ 1019 void vm_object_collapse(object) 1020 register vm_object_t object; 1021 1022 { 1023 register vm_object_t backing_object; 1024 register vm_offset_t backing_offset; 1025 register vm_size_t size; 1026 register vm_offset_t new_offset; 1027 register vm_page_t p, pp; 1028 1029 if (!vm_object_collapse_allowed) 1030 return; 1031 1032 while (TRUE) { 1033 /* 1034 * Verify that the conditions are right for collapse: 1035 * 1036 * The object exists and no pages in it are currently 1037 * being paged out (or have ever been paged out). 1038 */ 1039 if (object == NULL || 1040 object->paging_in_progress != 0 || 1041 object->pager != NULL) 1042 return; 1043 1044 /* 1045 * There is a backing object, and 1046 */ 1047 1048 if ((backing_object = object->shadow) == NULL) 1049 return; 1050 1051 vm_object_lock(backing_object); 1052 /* 1053 * ... 1054 * The backing object is not read_only, 1055 * and no pages in the backing object are 1056 * currently being paged out. 1057 * The backing object is internal. 1058 */ 1059 1060 if ((backing_object->flags & OBJ_INTERNAL) == 0 || 1061 backing_object->paging_in_progress != 0) { 1062 vm_object_unlock(backing_object); 1063 return; 1064 } 1065 1066 /* 1067 * The backing object can't be a copy-object: 1068 * the shadow_offset for the copy-object must stay 1069 * as 0. Furthermore (for the 'we have all the 1070 * pages' case), if we bypass backing_object and 1071 * just shadow the next object in the chain, old 1072 * pages from that object would then have to be copied 1073 * BOTH into the (former) backing_object and into the 1074 * parent object. 1075 */ 1076 if (backing_object->shadow != NULL && 1077 backing_object->shadow->copy != NULL) { 1078 vm_object_unlock(backing_object); 1079 return; 1080 } 1081 1082 /* 1083 * We know that we can either collapse the backing 1084 * object (if the parent is the only reference to 1085 * it) or (perhaps) remove the parent's reference 1086 * to it. 1087 */ 1088 1089 backing_offset = object->shadow_offset; 1090 size = object->size; 1091 1092 /* 1093 * If there is exactly one reference to the backing 1094 * object, we can collapse it into the parent. 1095 */ 1096 1097 if (backing_object->ref_count == 1) { 1098 1099 /* 1100 * We can collapse the backing object. 1101 * 1102 * Move all in-memory pages from backing_object 1103 * to the parent. Pages that have been paged out 1104 * will be overwritten by any of the parent's 1105 * pages that shadow them. 1106 */ 1107 1108 while (!queue_empty(&backing_object->memq)) { 1109 1110 p = (vm_page_t) 1111 queue_first(&backing_object->memq); 1112 1113 new_offset = (p->offset - backing_offset); 1114 1115 /* 1116 * If the parent has a page here, or if 1117 * this page falls outside the parent, 1118 * dispose of it. 1119 * 1120 * Otherwise, move it as planned. 1121 */ 1122 1123 if (p->offset < backing_offset || 1124 new_offset >= size) { 1125 vm_page_lock_queues(); 1126 vm_page_free(p); 1127 vm_page_unlock_queues(); 1128 } else { 1129 pp = vm_page_lookup(object, new_offset); 1130 if (pp != NULL && !(pp->flags & PG_FAKE)) { 1131 vm_page_lock_queues(); 1132 vm_page_free(p); 1133 vm_page_unlock_queues(); 1134 } 1135 else { 1136 if (pp) { 1137 /* may be someone waiting for it */ 1138 PAGE_WAKEUP(pp); 1139 vm_page_lock_queues(); 1140 vm_page_free(pp); 1141 vm_page_unlock_queues(); 1142 } 1143 vm_page_rename(p, object, new_offset); 1144 } 1145 } 1146 } 1147 1148 /* 1149 * Move the pager from backing_object to object. 1150 * 1151 * XXX We're only using part of the paging space 1152 * for keeps now... we ought to discard the 1153 * unused portion. 1154 */ 1155 1156 object->pager = backing_object->pager; 1157 object->paging_offset += backing_offset; 1158 1159 backing_object->pager = NULL; 1160 1161 /* 1162 * Object now shadows whatever backing_object did. 1163 * Note that the reference to backing_object->shadow 1164 * moves from within backing_object to within object. 1165 */ 1166 1167 object->shadow = backing_object->shadow; 1168 object->shadow_offset += backing_object->shadow_offset; 1169 if (object->shadow != NULL && 1170 object->shadow->copy != NULL) { 1171 panic("vm_object_collapse: we collapsed a copy-object!"); 1172 } 1173 /* 1174 * Discard backing_object. 1175 * 1176 * Since the backing object has no pages, no 1177 * pager left, and no object references within it, 1178 * all that is necessary is to dispose of it. 1179 */ 1180 1181 vm_object_unlock(backing_object); 1182 1183 simple_lock(&vm_object_list_lock); 1184 queue_remove(&vm_object_list, backing_object, 1185 vm_object_t, object_list); 1186 vm_object_count--; 1187 simple_unlock(&vm_object_list_lock); 1188 1189 free((caddr_t)backing_object, M_VMOBJ); 1190 1191 object_collapses++; 1192 } 1193 else { 1194 /* 1195 * If all of the pages in the backing object are 1196 * shadowed by the parent object, the parent 1197 * object no longer has to shadow the backing 1198 * object; it can shadow the next one in the 1199 * chain. 1200 * 1201 * The backing object must not be paged out - we'd 1202 * have to check all of the paged-out pages, as 1203 * well. 1204 */ 1205 1206 if (backing_object->pager != NULL) { 1207 vm_object_unlock(backing_object); 1208 return; 1209 } 1210 1211 /* 1212 * Should have a check for a 'small' number 1213 * of pages here. 1214 */ 1215 1216 p = (vm_page_t) queue_first(&backing_object->memq); 1217 while (!queue_end(&backing_object->memq, 1218 (queue_entry_t) p)) { 1219 1220 new_offset = (p->offset - backing_offset); 1221 1222 /* 1223 * If the parent has a page here, or if 1224 * this page falls outside the parent, 1225 * keep going. 1226 * 1227 * Otherwise, the backing_object must be 1228 * left in the chain. 1229 */ 1230 1231 if (p->offset >= backing_offset && 1232 new_offset <= size && 1233 ((pp = vm_page_lookup(object, new_offset)) 1234 == NULL || 1235 (pp->flags & PG_FAKE))) { 1236 /* 1237 * Page still needed. 1238 * Can't go any further. 1239 */ 1240 vm_object_unlock(backing_object); 1241 return; 1242 } 1243 p = (vm_page_t) queue_next(&p->listq); 1244 } 1245 1246 /* 1247 * Make the parent shadow the next object 1248 * in the chain. Deallocating backing_object 1249 * will not remove it, since its reference 1250 * count is at least 2. 1251 */ 1252 1253 vm_object_reference(object->shadow = backing_object->shadow); 1254 object->shadow_offset += backing_object->shadow_offset; 1255 1256 /* Drop the reference count on backing_object. 1257 * Since its ref_count was at least 2, it 1258 * will not vanish; so we don't need to call 1259 * vm_object_deallocate. 1260 */ 1261 backing_object->ref_count--; 1262 vm_object_unlock(backing_object); 1263 1264 object_bypasses ++; 1265 1266 } 1267 1268 /* 1269 * Try again with this object's new backing object. 1270 */ 1271 } 1272 } 1273 1274 /* 1275 * vm_object_page_remove: [internal] 1276 * 1277 * Removes all physical pages in the specified 1278 * object range from the object's list of pages. 1279 * 1280 * The object must be locked. 1281 */ 1282 void vm_object_page_remove(object, start, end) 1283 register vm_object_t object; 1284 register vm_offset_t start; 1285 register vm_offset_t end; 1286 { 1287 register vm_page_t p, next; 1288 1289 if (object == NULL) 1290 return; 1291 1292 p = (vm_page_t) queue_first(&object->memq); 1293 while (!queue_end(&object->memq, (queue_entry_t) p)) { 1294 next = (vm_page_t) queue_next(&p->listq); 1295 if ((start <= p->offset) && (p->offset < end)) { 1296 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1297 vm_page_lock_queues(); 1298 vm_page_free(p); 1299 vm_page_unlock_queues(); 1300 } 1301 p = next; 1302 } 1303 } 1304 1305 /* 1306 * Routine: vm_object_coalesce 1307 * Function: Coalesces two objects backing up adjoining 1308 * regions of memory into a single object. 1309 * 1310 * returns TRUE if objects were combined. 1311 * 1312 * NOTE: Only works at the moment if the second object is NULL - 1313 * if it's not, which object do we lock first? 1314 * 1315 * Parameters: 1316 * prev_object First object to coalesce 1317 * prev_offset Offset into prev_object 1318 * next_object Second object into coalesce 1319 * next_offset Offset into next_object 1320 * 1321 * prev_size Size of reference to prev_object 1322 * next_size Size of reference to next_object 1323 * 1324 * Conditions: 1325 * The object must *not* be locked. 1326 */ 1327 boolean_t vm_object_coalesce(prev_object, next_object, 1328 prev_offset, next_offset, 1329 prev_size, next_size) 1330 1331 register vm_object_t prev_object; 1332 vm_object_t next_object; 1333 vm_offset_t prev_offset, next_offset; 1334 vm_size_t prev_size, next_size; 1335 { 1336 vm_size_t newsize; 1337 1338 #ifdef lint 1339 next_offset++; 1340 #endif 1341 1342 if (next_object != NULL) { 1343 return(FALSE); 1344 } 1345 1346 if (prev_object == NULL) { 1347 return(TRUE); 1348 } 1349 1350 vm_object_lock(prev_object); 1351 1352 /* 1353 * Try to collapse the object first 1354 */ 1355 vm_object_collapse(prev_object); 1356 1357 /* 1358 * Can't coalesce if: 1359 * . more than one reference 1360 * . paged out 1361 * . shadows another object 1362 * . has a copy elsewhere 1363 * (any of which mean that the pages not mapped to 1364 * prev_entry may be in use anyway) 1365 */ 1366 1367 if (prev_object->ref_count > 1 || 1368 prev_object->pager != NULL || 1369 prev_object->shadow != NULL || 1370 prev_object->copy != NULL) { 1371 vm_object_unlock(prev_object); 1372 return(FALSE); 1373 } 1374 1375 /* 1376 * Remove any pages that may still be in the object from 1377 * a previous deallocation. 1378 */ 1379 1380 vm_object_page_remove(prev_object, 1381 prev_offset + prev_size, 1382 prev_offset + prev_size + next_size); 1383 1384 /* 1385 * Extend the object if necessary. 1386 */ 1387 newsize = prev_offset + prev_size + next_size; 1388 if (newsize > prev_object->size) 1389 prev_object->size = newsize; 1390 1391 vm_object_unlock(prev_object); 1392 return(TRUE); 1393 } 1394 1395 /* 1396 * vm_object_print: [ debug ] 1397 */ 1398 void vm_object_print(object, full) 1399 vm_object_t object; 1400 boolean_t full; 1401 { 1402 register vm_page_t p; 1403 extern indent; 1404 1405 register int count; 1406 1407 if (object == NULL) 1408 return; 1409 1410 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1411 (int) object, (int) object->size, 1412 object->resident_page_count, object->ref_count); 1413 printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1414 (int) object->pager, (int) object->paging_offset, 1415 (int) object->shadow, (int) object->shadow_offset); 1416 printf("cache: next=0x%x, prev=0x%x\n", 1417 object->cached_list.qe_next, object->cached_list.qe_prev); 1418 1419 if (!full) 1420 return; 1421 1422 indent += 2; 1423 count = 0; 1424 p = (vm_page_t) queue_first(&object->memq); 1425 while (!queue_end(&object->memq, (queue_entry_t) p)) { 1426 if (count == 0) 1427 iprintf("memory:="); 1428 else if (count == 6) { 1429 printf("\n"); 1430 iprintf(" ..."); 1431 count = 0; 1432 } else 1433 printf(","); 1434 count++; 1435 1436 printf("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p)); 1437 p = (vm_page_t) queue_next(&p->listq); 1438 } 1439 if (count != 0) 1440 printf("\n"); 1441 indent -= 2; 1442 } 1443