1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_object.c 8.7 (Berkeley) 05/11/95 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Virtual memory object module. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/malloc.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 50 /* 51 * Virtual memory objects maintain the actual data 52 * associated with allocated virtual memory. A given 53 * page of memory exists within exactly one object. 54 * 55 * An object is only deallocated when all "references" 56 * are given up. Only one "reference" to a given 57 * region of an object should be writeable. 58 * 59 * Associated with each object is a list of all resident 60 * memory pages belonging to that object; this list is 61 * maintained by the "vm_page" module, and locked by the object's 62 * lock. 63 * 64 * Each object also records a "pager" routine which is 65 * used to retrieve (and store) pages to the proper backing 66 * storage. In addition, objects may be backed by other 67 * objects from which they were virtual-copied. 68 * 69 * The only items within the object structure which are 70 * modified after time of creation are: 71 * reference count locked by object's lock 72 * pager routine locked by object's lock 73 * 74 */ 75 76 struct vm_object kernel_object_store; 77 struct vm_object kmem_object_store; 78 79 #define VM_OBJECT_HASH_COUNT 157 80 81 int vm_cache_max = 100; /* can patch if necessary */ 82 struct vm_object_hash_head vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 83 84 long object_collapses = 0; 85 long object_bypasses = 0; 86 87 static void _vm_object_allocate __P((vm_size_t, vm_object_t)); 88 89 /* 90 * vm_object_init: 91 * 92 * Initialize the VM objects module. 93 */ 94 void 95 vm_object_init(size) 96 vm_size_t size; 97 { 98 register int i; 99 100 TAILQ_INIT(&vm_object_cached_list); 101 TAILQ_INIT(&vm_object_list); 102 vm_object_count = 0; 103 simple_lock_init(&vm_cache_lock); 104 simple_lock_init(&vm_object_list_lock); 105 106 for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 107 TAILQ_INIT(&vm_object_hashtable[i]); 108 109 kernel_object = &kernel_object_store; 110 _vm_object_allocate(size, kernel_object); 111 112 kmem_object = &kmem_object_store; 113 _vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object); 114 } 115 116 /* 117 * vm_object_allocate: 118 * 119 * Returns a new object with the given size. 120 */ 121 122 vm_object_t 123 vm_object_allocate(size) 124 vm_size_t size; 125 { 126 register vm_object_t result; 127 128 result = (vm_object_t) 129 malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK); 130 131 _vm_object_allocate(size, result); 132 133 return(result); 134 } 135 136 static void 137 _vm_object_allocate(size, object) 138 vm_size_t size; 139 register vm_object_t object; 140 { 141 TAILQ_INIT(&object->memq); 142 vm_object_lock_init(object); 143 object->ref_count = 1; 144 object->resident_page_count = 0; 145 object->size = size; 146 object->flags = OBJ_INTERNAL; /* vm_allocate_with_pager will reset */ 147 object->paging_in_progress = 0; 148 object->copy = NULL; 149 150 /* 151 * Object starts out read-write, with no pager. 152 */ 153 154 object->pager = NULL; 155 object->paging_offset = 0; 156 object->shadow = NULL; 157 object->shadow_offset = (vm_offset_t) 0; 158 159 simple_lock(&vm_object_list_lock); 160 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 161 vm_object_count++; 162 cnt.v_nzfod += atop(size); 163 simple_unlock(&vm_object_list_lock); 164 } 165 166 /* 167 * vm_object_reference: 168 * 169 * Gets another reference to the given object. 170 */ 171 void 172 vm_object_reference(object) 173 register vm_object_t object; 174 { 175 if (object == NULL) 176 return; 177 178 vm_object_lock(object); 179 object->ref_count++; 180 vm_object_unlock(object); 181 } 182 183 /* 184 * vm_object_deallocate: 185 * 186 * Release a reference to the specified object, 187 * gained either through a vm_object_allocate 188 * or a vm_object_reference call. When all references 189 * are gone, storage associated with this object 190 * may be relinquished. 191 * 192 * No object may be locked. 193 */ 194 void 195 vm_object_deallocate(object) 196 register vm_object_t object; 197 { 198 vm_object_t temp; 199 200 while (object != NULL) { 201 202 /* 203 * The cache holds a reference (uncounted) to 204 * the object; we must lock it before removing 205 * the object. 206 */ 207 208 vm_object_cache_lock(); 209 210 /* 211 * Lose the reference 212 */ 213 vm_object_lock(object); 214 if (--(object->ref_count) != 0) { 215 216 /* 217 * If there are still references, then 218 * we are done. 219 */ 220 vm_object_unlock(object); 221 vm_object_cache_unlock(); 222 return; 223 } 224 225 /* 226 * See if this object can persist. If so, enter 227 * it in the cache, then deactivate all of its 228 * pages. 229 */ 230 231 if (object->flags & OBJ_CANPERSIST) { 232 233 TAILQ_INSERT_TAIL(&vm_object_cached_list, object, 234 cached_list); 235 vm_object_cached++; 236 vm_object_cache_unlock(); 237 238 vm_object_deactivate_pages(object); 239 vm_object_unlock(object); 240 241 vm_object_cache_trim(); 242 return; 243 } 244 245 /* 246 * Make sure no one can look us up now. 247 */ 248 vm_object_remove(object->pager); 249 vm_object_cache_unlock(); 250 251 temp = object->shadow; 252 vm_object_terminate(object); 253 /* unlocks and deallocates object */ 254 object = temp; 255 } 256 } 257 258 259 /* 260 * vm_object_terminate actually destroys the specified object, freeing 261 * up all previously used resources. 262 * 263 * The object must be locked. 264 */ 265 void 266 vm_object_terminate(object) 267 register vm_object_t object; 268 { 269 register vm_page_t p; 270 vm_object_t shadow_object; 271 272 /* 273 * Detach the object from its shadow if we are the shadow's 274 * copy. 275 */ 276 if ((shadow_object = object->shadow) != NULL) { 277 vm_object_lock(shadow_object); 278 if (shadow_object->copy == object) 279 shadow_object->copy = NULL; 280 #if 0 281 else if (shadow_object->copy != NULL) 282 panic("vm_object_terminate: copy/shadow inconsistency"); 283 #endif 284 vm_object_unlock(shadow_object); 285 } 286 287 /* 288 * Wait until the pageout daemon is through with the object. 289 */ 290 while (object->paging_in_progress) { 291 vm_object_sleep(object, object, FALSE); 292 vm_object_lock(object); 293 } 294 295 /* 296 * If not an internal object clean all the pages, removing them 297 * from paging queues as we go. 298 * 299 * XXX need to do something in the event of a cleaning error. 300 */ 301 if ((object->flags & OBJ_INTERNAL) == 0) 302 (void) vm_object_page_clean(object, 0, 0, TRUE, TRUE); 303 304 /* 305 * Now free the pages. 306 * For internal objects, this also removes them from paging queues. 307 */ 308 while ((p = object->memq.tqh_first) != NULL) { 309 VM_PAGE_CHECK(p); 310 vm_page_lock_queues(); 311 vm_page_free(p); 312 cnt.v_pfree++; 313 vm_page_unlock_queues(); 314 } 315 vm_object_unlock(object); 316 317 /* 318 * Let the pager know object is dead. 319 */ 320 if (object->pager != NULL) 321 vm_pager_deallocate(object->pager); 322 323 simple_lock(&vm_object_list_lock); 324 TAILQ_REMOVE(&vm_object_list, object, object_list); 325 vm_object_count--; 326 simple_unlock(&vm_object_list_lock); 327 328 /* 329 * Free the space for the object. 330 */ 331 free((caddr_t)object, M_VMOBJ); 332 } 333 334 /* 335 * vm_object_page_clean 336 * 337 * Clean all dirty pages in the specified range of object. 338 * If syncio is TRUE, page cleaning is done synchronously. 339 * If de_queue is TRUE, pages are removed from any paging queue 340 * they were on, otherwise they are left on whatever queue they 341 * were on before the cleaning operation began. 342 * 343 * Odd semantics: if start == end, we clean everything. 344 * 345 * The object must be locked. 346 * 347 * Returns TRUE if all was well, FALSE if there was a pager error 348 * somewhere. We attempt to clean (and dequeue) all pages regardless 349 * of where an error occurs. 350 */ 351 boolean_t 352 vm_object_page_clean(object, start, end, syncio, de_queue) 353 register vm_object_t object; 354 register vm_offset_t start; 355 register vm_offset_t end; 356 boolean_t syncio; 357 boolean_t de_queue; 358 { 359 register vm_page_t p; 360 int onqueue; 361 boolean_t noerror = TRUE; 362 363 if (object == NULL) 364 return (TRUE); 365 366 /* 367 * If it is an internal object and there is no pager, attempt to 368 * allocate one. Note that vm_object_collapse may relocate one 369 * from a collapsed object so we must recheck afterward. 370 */ 371 if ((object->flags & OBJ_INTERNAL) && object->pager == NULL) { 372 vm_object_collapse(object); 373 if (object->pager == NULL) { 374 vm_pager_t pager; 375 376 vm_object_unlock(object); 377 pager = vm_pager_allocate(PG_DFLT, (caddr_t)0, 378 object->size, VM_PROT_ALL, 379 (vm_offset_t)0); 380 if (pager) 381 vm_object_setpager(object, pager, 0, FALSE); 382 vm_object_lock(object); 383 } 384 } 385 if (object->pager == NULL) 386 return (FALSE); 387 388 again: 389 /* 390 * Wait until the pageout daemon is through with the object. 391 */ 392 while (object->paging_in_progress) { 393 vm_object_sleep(object, object, FALSE); 394 vm_object_lock(object); 395 } 396 /* 397 * Loop through the object page list cleaning as necessary. 398 */ 399 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 400 if ((start == end || p->offset >= start && p->offset < end) && 401 !(p->flags & PG_FICTITIOUS)) { 402 if ((p->flags & PG_CLEAN) && 403 pmap_is_modified(VM_PAGE_TO_PHYS(p))) 404 p->flags &= ~PG_CLEAN; 405 /* 406 * Remove the page from any paging queue. 407 * This needs to be done if either we have been 408 * explicitly asked to do so or it is about to 409 * be cleaned (see comment below). 410 */ 411 if (de_queue || !(p->flags & PG_CLEAN)) { 412 vm_page_lock_queues(); 413 if (p->flags & PG_ACTIVE) { 414 TAILQ_REMOVE(&vm_page_queue_active, 415 p, pageq); 416 p->flags &= ~PG_ACTIVE; 417 cnt.v_active_count--; 418 onqueue = 1; 419 } else if (p->flags & PG_INACTIVE) { 420 TAILQ_REMOVE(&vm_page_queue_inactive, 421 p, pageq); 422 p->flags &= ~PG_INACTIVE; 423 cnt.v_inactive_count--; 424 onqueue = -1; 425 } else 426 onqueue = 0; 427 vm_page_unlock_queues(); 428 } 429 /* 430 * To ensure the state of the page doesn't change 431 * during the clean operation we do two things. 432 * First we set the busy bit and write-protect all 433 * mappings to ensure that write accesses to the 434 * page block (in vm_fault). Second, we remove 435 * the page from any paging queue to foil the 436 * pageout daemon (vm_pageout_scan). 437 */ 438 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 439 if (!(p->flags & PG_CLEAN)) { 440 p->flags |= PG_BUSY; 441 object->paging_in_progress++; 442 vm_object_unlock(object); 443 /* 444 * XXX if put fails we mark the page as 445 * clean to avoid an infinite loop. 446 * Will loose changes to the page. 447 */ 448 if (vm_pager_put(object->pager, p, syncio)) { 449 printf("%s: pager_put error\n", 450 "vm_object_page_clean"); 451 p->flags |= PG_CLEAN; 452 noerror = FALSE; 453 } 454 vm_object_lock(object); 455 object->paging_in_progress--; 456 if (!de_queue && onqueue) { 457 vm_page_lock_queues(); 458 if (onqueue > 0) 459 vm_page_activate(p); 460 else 461 vm_page_deactivate(p); 462 vm_page_unlock_queues(); 463 } 464 p->flags &= ~PG_BUSY; 465 PAGE_WAKEUP(p); 466 goto again; 467 } 468 } 469 } 470 return (noerror); 471 } 472 473 /* 474 * vm_object_deactivate_pages 475 * 476 * Deactivate all pages in the specified object. (Keep its pages 477 * in memory even though it is no longer referenced.) 478 * 479 * The object must be locked. 480 */ 481 void 482 vm_object_deactivate_pages(object) 483 register vm_object_t object; 484 { 485 register vm_page_t p, next; 486 487 for (p = object->memq.tqh_first; p != NULL; p = next) { 488 next = p->listq.tqe_next; 489 vm_page_lock_queues(); 490 vm_page_deactivate(p); 491 vm_page_unlock_queues(); 492 } 493 } 494 495 /* 496 * Trim the object cache to size. 497 */ 498 void 499 vm_object_cache_trim() 500 { 501 register vm_object_t object; 502 503 vm_object_cache_lock(); 504 while (vm_object_cached > vm_cache_max) { 505 object = vm_object_cached_list.tqh_first; 506 vm_object_cache_unlock(); 507 508 if (object != vm_object_lookup(object->pager)) 509 panic("vm_object_deactivate: I'm sooo confused."); 510 511 pager_cache(object, FALSE); 512 513 vm_object_cache_lock(); 514 } 515 vm_object_cache_unlock(); 516 } 517 518 /* 519 * vm_object_pmap_copy: 520 * 521 * Makes all physical pages in the specified 522 * object range copy-on-write. No writeable 523 * references to these pages should remain. 524 * 525 * The object must *not* be locked. 526 */ 527 void 528 vm_object_pmap_copy(object, start, end) 529 register vm_object_t object; 530 register vm_offset_t start; 531 register vm_offset_t end; 532 { 533 register vm_page_t p; 534 535 if (object == NULL) 536 return; 537 538 vm_object_lock(object); 539 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 540 if ((start <= p->offset) && (p->offset < end)) { 541 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 542 p->flags |= PG_COPYONWRITE; 543 } 544 } 545 vm_object_unlock(object); 546 } 547 548 /* 549 * vm_object_pmap_remove: 550 * 551 * Removes all physical pages in the specified 552 * object range from all physical maps. 553 * 554 * The object must *not* be locked. 555 */ 556 void 557 vm_object_pmap_remove(object, start, end) 558 register vm_object_t object; 559 register vm_offset_t start; 560 register vm_offset_t end; 561 { 562 register vm_page_t p; 563 564 if (object == NULL) 565 return; 566 567 vm_object_lock(object); 568 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 569 if ((start <= p->offset) && (p->offset < end)) 570 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 571 vm_object_unlock(object); 572 } 573 574 /* 575 * vm_object_copy: 576 * 577 * Create a new object which is a copy of an existing 578 * object, and mark all of the pages in the existing 579 * object 'copy-on-write'. The new object has one reference. 580 * Returns the new object. 581 * 582 * May defer the copy until later if the object is not backed 583 * up by a non-default pager. 584 */ 585 void 586 vm_object_copy(src_object, src_offset, size, 587 dst_object, dst_offset, src_needs_copy) 588 register vm_object_t src_object; 589 vm_offset_t src_offset; 590 vm_size_t size; 591 vm_object_t *dst_object; /* OUT */ 592 vm_offset_t *dst_offset; /* OUT */ 593 boolean_t *src_needs_copy; /* OUT */ 594 { 595 register vm_object_t new_copy; 596 register vm_object_t old_copy; 597 vm_offset_t new_start, new_end; 598 599 register vm_page_t p; 600 601 if (src_object == NULL) { 602 /* 603 * Nothing to copy 604 */ 605 *dst_object = NULL; 606 *dst_offset = 0; 607 *src_needs_copy = FALSE; 608 return; 609 } 610 611 /* 612 * If the object's pager is null_pager or the 613 * default pager, we don't have to make a copy 614 * of it. Instead, we set the needs copy flag and 615 * make a shadow later. 616 */ 617 618 vm_object_lock(src_object); 619 if (src_object->pager == NULL || 620 (src_object->flags & OBJ_INTERNAL)) { 621 622 /* 623 * Make another reference to the object 624 */ 625 src_object->ref_count++; 626 627 /* 628 * Mark all of the pages copy-on-write. 629 */ 630 for (p = src_object->memq.tqh_first; p; p = p->listq.tqe_next) 631 if (src_offset <= p->offset && 632 p->offset < src_offset + size) 633 p->flags |= PG_COPYONWRITE; 634 vm_object_unlock(src_object); 635 636 *dst_object = src_object; 637 *dst_offset = src_offset; 638 639 /* 640 * Must make a shadow when write is desired 641 */ 642 *src_needs_copy = TRUE; 643 return; 644 } 645 646 /* 647 * Try to collapse the object before copying it. 648 */ 649 vm_object_collapse(src_object); 650 651 /* 652 * If the object has a pager, the pager wants to 653 * see all of the changes. We need a copy-object 654 * for the changed pages. 655 * 656 * If there is a copy-object, and it is empty, 657 * no changes have been made to the object since the 658 * copy-object was made. We can use the same copy- 659 * object. 660 */ 661 662 Retry1: 663 old_copy = src_object->copy; 664 if (old_copy != NULL) { 665 /* 666 * Try to get the locks (out of order) 667 */ 668 if (!vm_object_lock_try(old_copy)) { 669 vm_object_unlock(src_object); 670 671 /* should spin a bit here... */ 672 vm_object_lock(src_object); 673 goto Retry1; 674 } 675 676 if (old_copy->resident_page_count == 0 && 677 old_copy->pager == NULL) { 678 /* 679 * Return another reference to 680 * the existing copy-object. 681 */ 682 old_copy->ref_count++; 683 vm_object_unlock(old_copy); 684 vm_object_unlock(src_object); 685 *dst_object = old_copy; 686 *dst_offset = src_offset; 687 *src_needs_copy = FALSE; 688 return; 689 } 690 vm_object_unlock(old_copy); 691 } 692 vm_object_unlock(src_object); 693 694 /* 695 * If the object has a pager, the pager wants 696 * to see all of the changes. We must make 697 * a copy-object and put the changed pages there. 698 * 699 * The copy-object is always made large enough to 700 * completely shadow the original object, since 701 * it may have several users who want to shadow 702 * the original object at different points. 703 */ 704 705 new_copy = vm_object_allocate(src_object->size); 706 707 Retry2: 708 vm_object_lock(src_object); 709 /* 710 * Copy object may have changed while we were unlocked 711 */ 712 old_copy = src_object->copy; 713 if (old_copy != NULL) { 714 /* 715 * Try to get the locks (out of order) 716 */ 717 if (!vm_object_lock_try(old_copy)) { 718 vm_object_unlock(src_object); 719 goto Retry2; 720 } 721 722 /* 723 * Consistency check 724 */ 725 if (old_copy->shadow != src_object || 726 old_copy->shadow_offset != (vm_offset_t) 0) 727 panic("vm_object_copy: copy/shadow inconsistency"); 728 729 /* 730 * Make the old copy-object shadow the new one. 731 * It will receive no more pages from the original 732 * object. 733 */ 734 735 src_object->ref_count--; /* remove ref. from old_copy */ 736 old_copy->shadow = new_copy; 737 new_copy->ref_count++; /* locking not needed - we 738 have the only pointer */ 739 vm_object_unlock(old_copy); /* done with old_copy */ 740 } 741 742 new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 743 new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 744 745 /* 746 * Point the new copy at the existing object. 747 */ 748 749 new_copy->shadow = src_object; 750 new_copy->shadow_offset = new_start; 751 src_object->ref_count++; 752 src_object->copy = new_copy; 753 754 /* 755 * Mark all the affected pages of the existing object 756 * copy-on-write. 757 */ 758 for (p = src_object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 759 if ((new_start <= p->offset) && (p->offset < new_end)) 760 p->flags |= PG_COPYONWRITE; 761 762 vm_object_unlock(src_object); 763 764 *dst_object = new_copy; 765 *dst_offset = src_offset - new_start; 766 *src_needs_copy = FALSE; 767 } 768 769 /* 770 * vm_object_shadow: 771 * 772 * Create a new object which is backed by the 773 * specified existing object range. The source 774 * object reference is deallocated. 775 * 776 * The new object and offset into that object 777 * are returned in the source parameters. 778 */ 779 780 void 781 vm_object_shadow(object, offset, length) 782 vm_object_t *object; /* IN/OUT */ 783 vm_offset_t *offset; /* IN/OUT */ 784 vm_size_t length; 785 { 786 register vm_object_t source; 787 register vm_object_t result; 788 789 source = *object; 790 791 /* 792 * Allocate a new object with the given length 793 */ 794 795 if ((result = vm_object_allocate(length)) == NULL) 796 panic("vm_object_shadow: no object for shadowing"); 797 798 /* 799 * The new object shadows the source object, adding 800 * a reference to it. Our caller changes his reference 801 * to point to the new object, removing a reference to 802 * the source object. Net result: no change of reference 803 * count. 804 */ 805 result->shadow = source; 806 807 /* 808 * Store the offset into the source object, 809 * and fix up the offset into the new object. 810 */ 811 812 result->shadow_offset = *offset; 813 814 /* 815 * Return the new things 816 */ 817 818 *offset = 0; 819 *object = result; 820 } 821 822 /* 823 * Set the specified object's pager to the specified pager. 824 */ 825 826 void 827 vm_object_setpager(object, pager, paging_offset, 828 read_only) 829 vm_object_t object; 830 vm_pager_t pager; 831 vm_offset_t paging_offset; 832 boolean_t read_only; 833 { 834 #ifdef lint 835 read_only++; /* No longer used */ 836 #endif 837 838 vm_object_lock(object); /* XXX ? */ 839 object->pager = pager; 840 object->paging_offset = paging_offset; 841 vm_object_unlock(object); /* XXX ? */ 842 } 843 844 /* 845 * vm_object_hash hashes the pager/id pair. 846 */ 847 848 #define vm_object_hash(pager) \ 849 (((unsigned long)pager)%VM_OBJECT_HASH_COUNT) 850 851 /* 852 * vm_object_lookup looks in the object cache for an object with the 853 * specified pager and paging id. 854 */ 855 856 vm_object_t 857 vm_object_lookup(pager) 858 vm_pager_t pager; 859 { 860 register vm_object_hash_entry_t entry; 861 vm_object_t object; 862 863 vm_object_cache_lock(); 864 865 for (entry = vm_object_hashtable[vm_object_hash(pager)].tqh_first; 866 entry != NULL; 867 entry = entry->hash_links.tqe_next) { 868 object = entry->object; 869 if (object->pager == pager) { 870 vm_object_lock(object); 871 if (object->ref_count == 0) { 872 TAILQ_REMOVE(&vm_object_cached_list, object, 873 cached_list); 874 vm_object_cached--; 875 } 876 object->ref_count++; 877 vm_object_unlock(object); 878 vm_object_cache_unlock(); 879 return(object); 880 } 881 } 882 883 vm_object_cache_unlock(); 884 return(NULL); 885 } 886 887 /* 888 * vm_object_enter enters the specified object/pager/id into 889 * the hash table. 890 */ 891 892 void 893 vm_object_enter(object, pager) 894 vm_object_t object; 895 vm_pager_t pager; 896 { 897 struct vm_object_hash_head *bucket; 898 register vm_object_hash_entry_t entry; 899 900 /* 901 * We don't cache null objects, and we can't cache 902 * objects with the null pager. 903 */ 904 905 if (object == NULL) 906 return; 907 if (pager == NULL) 908 return; 909 910 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 911 entry = (vm_object_hash_entry_t) 912 malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK); 913 entry->object = object; 914 object->flags |= OBJ_CANPERSIST; 915 916 vm_object_cache_lock(); 917 TAILQ_INSERT_TAIL(bucket, entry, hash_links); 918 vm_object_cache_unlock(); 919 } 920 921 /* 922 * vm_object_remove: 923 * 924 * Remove the pager from the hash table. 925 * Note: This assumes that the object cache 926 * is locked. XXX this should be fixed 927 * by reorganizing vm_object_deallocate. 928 */ 929 void 930 vm_object_remove(pager) 931 register vm_pager_t pager; 932 { 933 struct vm_object_hash_head *bucket; 934 register vm_object_hash_entry_t entry; 935 register vm_object_t object; 936 937 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 938 939 for (entry = bucket->tqh_first; 940 entry != NULL; 941 entry = entry->hash_links.tqe_next) { 942 object = entry->object; 943 if (object->pager == pager) { 944 TAILQ_REMOVE(bucket, entry, hash_links); 945 free((caddr_t)entry, M_VMOBJHASH); 946 break; 947 } 948 } 949 } 950 951 /* 952 * vm_object_cache_clear removes all objects from the cache. 953 * 954 */ 955 void 956 vm_object_cache_clear() 957 { 958 register vm_object_t object; 959 960 /* 961 * Remove each object in the cache by scanning down the 962 * list of cached objects. 963 */ 964 vm_object_cache_lock(); 965 while ((object = vm_object_cached_list.tqh_first) != NULL) { 966 vm_object_cache_unlock(); 967 968 /* 969 * Note: it is important that we use vm_object_lookup 970 * to gain a reference, and not vm_object_reference, because 971 * the logic for removing an object from the cache lies in 972 * lookup. 973 */ 974 if (object != vm_object_lookup(object->pager)) 975 panic("vm_object_cache_clear: I'm sooo confused."); 976 pager_cache(object, FALSE); 977 978 vm_object_cache_lock(); 979 } 980 vm_object_cache_unlock(); 981 } 982 983 boolean_t vm_object_collapse_allowed = TRUE; 984 /* 985 * vm_object_collapse: 986 * 987 * Collapse an object with the object backing it. 988 * Pages in the backing object are moved into the 989 * parent, and the backing object is deallocated. 990 * 991 * Requires that the object be locked and the page 992 * queues be unlocked. 993 * 994 */ 995 void 996 vm_object_collapse(object) 997 register vm_object_t object; 998 999 { 1000 register vm_object_t backing_object; 1001 register vm_offset_t backing_offset; 1002 register vm_size_t size; 1003 register vm_offset_t new_offset; 1004 register vm_page_t p, pp; 1005 1006 if (!vm_object_collapse_allowed) 1007 return; 1008 1009 while (TRUE) { 1010 /* 1011 * Verify that the conditions are right for collapse: 1012 * 1013 * The object exists and no pages in it are currently 1014 * being paged out (or have ever been paged out). 1015 */ 1016 if (object == NULL || 1017 object->paging_in_progress != 0 || 1018 object->pager != NULL) 1019 return; 1020 1021 /* 1022 * There is a backing object, and 1023 */ 1024 1025 if ((backing_object = object->shadow) == NULL) 1026 return; 1027 1028 vm_object_lock(backing_object); 1029 /* 1030 * ... 1031 * The backing object is not read_only, 1032 * and no pages in the backing object are 1033 * currently being paged out. 1034 * The backing object is internal. 1035 */ 1036 1037 if ((backing_object->flags & OBJ_INTERNAL) == 0 || 1038 backing_object->paging_in_progress != 0) { 1039 vm_object_unlock(backing_object); 1040 return; 1041 } 1042 1043 /* 1044 * The backing object can't be a copy-object: 1045 * the shadow_offset for the copy-object must stay 1046 * as 0. Furthermore (for the 'we have all the 1047 * pages' case), if we bypass backing_object and 1048 * just shadow the next object in the chain, old 1049 * pages from that object would then have to be copied 1050 * BOTH into the (former) backing_object and into the 1051 * parent object. 1052 */ 1053 if (backing_object->shadow != NULL && 1054 backing_object->shadow->copy != NULL) { 1055 vm_object_unlock(backing_object); 1056 return; 1057 } 1058 1059 /* 1060 * We know that we can either collapse the backing 1061 * object (if the parent is the only reference to 1062 * it) or (perhaps) remove the parent's reference 1063 * to it. 1064 */ 1065 1066 backing_offset = object->shadow_offset; 1067 size = object->size; 1068 1069 /* 1070 * If there is exactly one reference to the backing 1071 * object, we can collapse it into the parent. 1072 */ 1073 1074 if (backing_object->ref_count == 1) { 1075 1076 /* 1077 * We can collapse the backing object. 1078 * 1079 * Move all in-memory pages from backing_object 1080 * to the parent. Pages that have been paged out 1081 * will be overwritten by any of the parent's 1082 * pages that shadow them. 1083 */ 1084 1085 while ((p = backing_object->memq.tqh_first) != NULL) { 1086 new_offset = (p->offset - backing_offset); 1087 1088 /* 1089 * If the parent has a page here, or if 1090 * this page falls outside the parent, 1091 * dispose of it. 1092 * 1093 * Otherwise, move it as planned. 1094 */ 1095 1096 if (p->offset < backing_offset || 1097 new_offset >= size) { 1098 vm_page_lock_queues(); 1099 vm_page_free(p); 1100 vm_page_unlock_queues(); 1101 } else { 1102 pp = vm_page_lookup(object, new_offset); 1103 if (pp != NULL && !(pp->flags & PG_FAKE)) { 1104 vm_page_lock_queues(); 1105 vm_page_free(p); 1106 vm_page_unlock_queues(); 1107 } 1108 else { 1109 if (pp) { 1110 /* may be someone waiting for it */ 1111 PAGE_WAKEUP(pp); 1112 vm_page_lock_queues(); 1113 vm_page_free(pp); 1114 vm_page_unlock_queues(); 1115 } 1116 vm_page_rename(p, object, new_offset); 1117 } 1118 } 1119 } 1120 1121 /* 1122 * Move the pager from backing_object to object. 1123 * 1124 * XXX We're only using part of the paging space 1125 * for keeps now... we ought to discard the 1126 * unused portion. 1127 */ 1128 1129 if (backing_object->pager) { 1130 object->pager = backing_object->pager; 1131 object->paging_offset = backing_offset + 1132 backing_object->paging_offset; 1133 backing_object->pager = NULL; 1134 } 1135 1136 /* 1137 * Object now shadows whatever backing_object did. 1138 * Note that the reference to backing_object->shadow 1139 * moves from within backing_object to within object. 1140 */ 1141 1142 object->shadow = backing_object->shadow; 1143 object->shadow_offset += backing_object->shadow_offset; 1144 if (object->shadow != NULL && 1145 object->shadow->copy != NULL) { 1146 panic("vm_object_collapse: we collapsed a copy-object!"); 1147 } 1148 /* 1149 * Discard backing_object. 1150 * 1151 * Since the backing object has no pages, no 1152 * pager left, and no object references within it, 1153 * all that is necessary is to dispose of it. 1154 */ 1155 1156 vm_object_unlock(backing_object); 1157 1158 simple_lock(&vm_object_list_lock); 1159 TAILQ_REMOVE(&vm_object_list, backing_object, 1160 object_list); 1161 vm_object_count--; 1162 simple_unlock(&vm_object_list_lock); 1163 1164 free((caddr_t)backing_object, M_VMOBJ); 1165 1166 object_collapses++; 1167 } 1168 else { 1169 /* 1170 * If all of the pages in the backing object are 1171 * shadowed by the parent object, the parent 1172 * object no longer has to shadow the backing 1173 * object; it can shadow the next one in the 1174 * chain. 1175 * 1176 * The backing object must not be paged out - we'd 1177 * have to check all of the paged-out pages, as 1178 * well. 1179 */ 1180 1181 if (backing_object->pager != NULL) { 1182 vm_object_unlock(backing_object); 1183 return; 1184 } 1185 1186 /* 1187 * Should have a check for a 'small' number 1188 * of pages here. 1189 */ 1190 1191 for (p = backing_object->memq.tqh_first; 1192 p != NULL; 1193 p = p->listq.tqe_next) { 1194 new_offset = (p->offset - backing_offset); 1195 1196 /* 1197 * If the parent has a page here, or if 1198 * this page falls outside the parent, 1199 * keep going. 1200 * 1201 * Otherwise, the backing_object must be 1202 * left in the chain. 1203 */ 1204 1205 if (p->offset >= backing_offset && 1206 new_offset < size && 1207 ((pp = vm_page_lookup(object, new_offset)) 1208 == NULL || 1209 (pp->flags & PG_FAKE))) { 1210 /* 1211 * Page still needed. 1212 * Can't go any further. 1213 */ 1214 vm_object_unlock(backing_object); 1215 return; 1216 } 1217 } 1218 1219 /* 1220 * Make the parent shadow the next object 1221 * in the chain. Deallocating backing_object 1222 * will not remove it, since its reference 1223 * count is at least 2. 1224 */ 1225 1226 object->shadow = backing_object->shadow; 1227 vm_object_reference(object->shadow); 1228 object->shadow_offset += backing_object->shadow_offset; 1229 1230 /* 1231 * Backing object might have had a copy pointer 1232 * to us. If it did, clear it. 1233 */ 1234 if (backing_object->copy == object) { 1235 backing_object->copy = NULL; 1236 } 1237 1238 /* Drop the reference count on backing_object. 1239 * Since its ref_count was at least 2, it 1240 * will not vanish; so we don't need to call 1241 * vm_object_deallocate. 1242 */ 1243 backing_object->ref_count--; 1244 vm_object_unlock(backing_object); 1245 1246 object_bypasses ++; 1247 1248 } 1249 1250 /* 1251 * Try again with this object's new backing object. 1252 */ 1253 } 1254 } 1255 1256 /* 1257 * vm_object_page_remove: [internal] 1258 * 1259 * Removes all physical pages in the specified 1260 * object range from the object's list of pages. 1261 * 1262 * The object must be locked. 1263 */ 1264 void 1265 vm_object_page_remove(object, start, end) 1266 register vm_object_t object; 1267 register vm_offset_t start; 1268 register vm_offset_t end; 1269 { 1270 register vm_page_t p, next; 1271 1272 if (object == NULL) 1273 return; 1274 1275 for (p = object->memq.tqh_first; p != NULL; p = next) { 1276 next = p->listq.tqe_next; 1277 if ((start <= p->offset) && (p->offset < end)) { 1278 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1279 vm_page_lock_queues(); 1280 vm_page_free(p); 1281 vm_page_unlock_queues(); 1282 } 1283 } 1284 } 1285 1286 /* 1287 * Routine: vm_object_coalesce 1288 * Function: Coalesces two objects backing up adjoining 1289 * regions of memory into a single object. 1290 * 1291 * returns TRUE if objects were combined. 1292 * 1293 * NOTE: Only works at the moment if the second object is NULL - 1294 * if it's not, which object do we lock first? 1295 * 1296 * Parameters: 1297 * prev_object First object to coalesce 1298 * prev_offset Offset into prev_object 1299 * next_object Second object into coalesce 1300 * next_offset Offset into next_object 1301 * 1302 * prev_size Size of reference to prev_object 1303 * next_size Size of reference to next_object 1304 * 1305 * Conditions: 1306 * The object must *not* be locked. 1307 */ 1308 boolean_t 1309 vm_object_coalesce(prev_object, next_object, 1310 prev_offset, next_offset, 1311 prev_size, next_size) 1312 1313 register vm_object_t prev_object; 1314 vm_object_t next_object; 1315 vm_offset_t prev_offset, next_offset; 1316 vm_size_t prev_size, next_size; 1317 { 1318 vm_size_t newsize; 1319 1320 #ifdef lint 1321 next_offset++; 1322 #endif 1323 1324 if (next_object != NULL) { 1325 return(FALSE); 1326 } 1327 1328 if (prev_object == NULL) { 1329 return(TRUE); 1330 } 1331 1332 vm_object_lock(prev_object); 1333 1334 /* 1335 * Try to collapse the object first 1336 */ 1337 vm_object_collapse(prev_object); 1338 1339 /* 1340 * Can't coalesce if: 1341 * . more than one reference 1342 * . paged out 1343 * . shadows another object 1344 * . has a copy elsewhere 1345 * (any of which mean that the pages not mapped to 1346 * prev_entry may be in use anyway) 1347 */ 1348 1349 if (prev_object->ref_count > 1 || 1350 prev_object->pager != NULL || 1351 prev_object->shadow != NULL || 1352 prev_object->copy != NULL) { 1353 vm_object_unlock(prev_object); 1354 return(FALSE); 1355 } 1356 1357 /* 1358 * Remove any pages that may still be in the object from 1359 * a previous deallocation. 1360 */ 1361 1362 vm_object_page_remove(prev_object, 1363 prev_offset + prev_size, 1364 prev_offset + prev_size + next_size); 1365 1366 /* 1367 * Extend the object if necessary. 1368 */ 1369 newsize = prev_offset + prev_size + next_size; 1370 if (newsize > prev_object->size) 1371 prev_object->size = newsize; 1372 1373 vm_object_unlock(prev_object); 1374 return(TRUE); 1375 } 1376 1377 /* 1378 * vm_object_print: [ debug ] 1379 */ 1380 void 1381 vm_object_print(object, full) 1382 vm_object_t object; 1383 boolean_t full; 1384 { 1385 register vm_page_t p; 1386 extern indent; 1387 1388 register int count; 1389 1390 if (object == NULL) 1391 return; 1392 1393 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1394 (int) object, (int) object->size, 1395 object->resident_page_count, object->ref_count); 1396 printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1397 (int) object->pager, (int) object->paging_offset, 1398 (int) object->shadow, (int) object->shadow_offset); 1399 printf("cache: next=0x%x, prev=0x%x\n", 1400 object->cached_list.tqe_next, object->cached_list.tqe_prev); 1401 1402 if (!full) 1403 return; 1404 1405 indent += 2; 1406 count = 0; 1407 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 1408 if (count == 0) 1409 iprintf("memory:="); 1410 else if (count == 6) { 1411 printf("\n"); 1412 iprintf(" ..."); 1413 count = 0; 1414 } else 1415 printf(","); 1416 count++; 1417 1418 printf("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p)); 1419 } 1420 if (count != 0) 1421 printf("\n"); 1422 indent -= 2; 1423 } 1424