1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_object.c 8.6 (Berkeley) 01/09/95 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Virtual memory object module. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/malloc.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 50 /* 51 * Virtual memory objects maintain the actual data 52 * associated with allocated virtual memory. A given 53 * page of memory exists within exactly one object. 54 * 55 * An object is only deallocated when all "references" 56 * are given up. Only one "reference" to a given 57 * region of an object should be writeable. 58 * 59 * Associated with each object is a list of all resident 60 * memory pages belonging to that object; this list is 61 * maintained by the "vm_page" module, and locked by the object's 62 * lock. 63 * 64 * Each object also records a "pager" routine which is 65 * used to retrieve (and store) pages to the proper backing 66 * storage. In addition, objects may be backed by other 67 * objects from which they were virtual-copied. 68 * 69 * The only items within the object structure which are 70 * modified after time of creation are: 71 * reference count locked by object's lock 72 * pager routine locked by object's lock 73 * 74 */ 75 76 struct vm_object kernel_object_store; 77 struct vm_object kmem_object_store; 78 79 #define VM_OBJECT_HASH_COUNT 157 80 81 int vm_cache_max = 100; /* can patch if necessary */ 82 struct vm_object_hash_head vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 83 84 long object_collapses = 0; 85 long object_bypasses = 0; 86 87 static void _vm_object_allocate __P((vm_size_t, vm_object_t)); 88 89 /* 90 * vm_object_init: 91 * 92 * Initialize the VM objects module. 93 */ 94 void 95 vm_object_init(size) 96 vm_size_t size; 97 { 98 register int i; 99 100 TAILQ_INIT(&vm_object_cached_list); 101 TAILQ_INIT(&vm_object_list); 102 vm_object_count = 0; 103 simple_lock_init(&vm_cache_lock); 104 simple_lock_init(&vm_object_list_lock); 105 106 for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 107 TAILQ_INIT(&vm_object_hashtable[i]); 108 109 kernel_object = &kernel_object_store; 110 _vm_object_allocate(size, kernel_object); 111 112 kmem_object = &kmem_object_store; 113 _vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object); 114 } 115 116 /* 117 * vm_object_allocate: 118 * 119 * Returns a new object with the given size. 120 */ 121 122 vm_object_t 123 vm_object_allocate(size) 124 vm_size_t size; 125 { 126 register vm_object_t result; 127 128 result = (vm_object_t) 129 malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK); 130 131 _vm_object_allocate(size, result); 132 133 return(result); 134 } 135 136 static void 137 _vm_object_allocate(size, object) 138 vm_size_t size; 139 register vm_object_t object; 140 { 141 TAILQ_INIT(&object->memq); 142 vm_object_lock_init(object); 143 object->ref_count = 1; 144 object->resident_page_count = 0; 145 object->size = size; 146 object->flags = OBJ_INTERNAL; /* vm_allocate_with_pager will reset */ 147 object->paging_in_progress = 0; 148 object->copy = NULL; 149 150 /* 151 * Object starts out read-write, with no pager. 152 */ 153 154 object->pager = NULL; 155 object->paging_offset = 0; 156 object->shadow = NULL; 157 object->shadow_offset = (vm_offset_t) 0; 158 159 simple_lock(&vm_object_list_lock); 160 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 161 vm_object_count++; 162 cnt.v_nzfod += atop(size); 163 simple_unlock(&vm_object_list_lock); 164 } 165 166 /* 167 * vm_object_reference: 168 * 169 * Gets another reference to the given object. 170 */ 171 void 172 vm_object_reference(object) 173 register vm_object_t object; 174 { 175 if (object == NULL) 176 return; 177 178 vm_object_lock(object); 179 object->ref_count++; 180 vm_object_unlock(object); 181 } 182 183 /* 184 * vm_object_deallocate: 185 * 186 * Release a reference to the specified object, 187 * gained either through a vm_object_allocate 188 * or a vm_object_reference call. When all references 189 * are gone, storage associated with this object 190 * may be relinquished. 191 * 192 * No object may be locked. 193 */ 194 void 195 vm_object_deallocate(object) 196 register vm_object_t object; 197 { 198 vm_object_t temp; 199 200 while (object != NULL) { 201 202 /* 203 * The cache holds a reference (uncounted) to 204 * the object; we must lock it before removing 205 * the object. 206 */ 207 208 vm_object_cache_lock(); 209 210 /* 211 * Lose the reference 212 */ 213 vm_object_lock(object); 214 if (--(object->ref_count) != 0) { 215 216 /* 217 * If there are still references, then 218 * we are done. 219 */ 220 vm_object_unlock(object); 221 vm_object_cache_unlock(); 222 return; 223 } 224 225 /* 226 * See if this object can persist. If so, enter 227 * it in the cache, then deactivate all of its 228 * pages. 229 */ 230 231 if (object->flags & OBJ_CANPERSIST) { 232 233 TAILQ_INSERT_TAIL(&vm_object_cached_list, object, 234 cached_list); 235 vm_object_cached++; 236 vm_object_cache_unlock(); 237 238 vm_object_deactivate_pages(object); 239 vm_object_unlock(object); 240 241 vm_object_cache_trim(); 242 return; 243 } 244 245 /* 246 * Make sure no one can look us up now. 247 */ 248 vm_object_remove(object->pager); 249 vm_object_cache_unlock(); 250 251 temp = object->shadow; 252 vm_object_terminate(object); 253 /* unlocks and deallocates object */ 254 object = temp; 255 } 256 } 257 258 259 /* 260 * vm_object_terminate actually destroys the specified object, freeing 261 * up all previously used resources. 262 * 263 * The object must be locked. 264 */ 265 void 266 vm_object_terminate(object) 267 register vm_object_t object; 268 { 269 register vm_page_t p; 270 vm_object_t shadow_object; 271 272 /* 273 * Detach the object from its shadow if we are the shadow's 274 * copy. 275 */ 276 if ((shadow_object = object->shadow) != NULL) { 277 vm_object_lock(shadow_object); 278 if (shadow_object->copy == object) 279 shadow_object->copy = NULL; 280 #if 0 281 else if (shadow_object->copy != NULL) 282 panic("vm_object_terminate: copy/shadow inconsistency"); 283 #endif 284 vm_object_unlock(shadow_object); 285 } 286 287 /* 288 * Wait until the pageout daemon is through with the object. 289 */ 290 while (object->paging_in_progress) { 291 vm_object_sleep(object, object, FALSE); 292 vm_object_lock(object); 293 } 294 295 /* 296 * If not an internal object clean all the pages, removing them 297 * from paging queues as we go. 298 * 299 * XXX need to do something in the event of a cleaning error. 300 */ 301 if ((object->flags & OBJ_INTERNAL) == 0) { 302 (void) vm_object_page_clean(object, 0, 0, TRUE, TRUE); 303 vm_object_unlock(object); 304 } 305 306 /* 307 * Now free the pages. 308 * For internal objects, this also removes them from paging queues. 309 */ 310 while ((p = object->memq.tqh_first) != NULL) { 311 VM_PAGE_CHECK(p); 312 vm_page_lock_queues(); 313 vm_page_free(p); 314 cnt.v_pfree++; 315 vm_page_unlock_queues(); 316 } 317 if ((object->flags & OBJ_INTERNAL) == 0) 318 vm_object_unlock(object); 319 320 /* 321 * Let the pager know object is dead. 322 */ 323 if (object->pager != NULL) 324 vm_pager_deallocate(object->pager); 325 326 simple_lock(&vm_object_list_lock); 327 TAILQ_REMOVE(&vm_object_list, object, object_list); 328 vm_object_count--; 329 simple_unlock(&vm_object_list_lock); 330 331 /* 332 * Free the space for the object. 333 */ 334 free((caddr_t)object, M_VMOBJ); 335 } 336 337 /* 338 * vm_object_page_clean 339 * 340 * Clean all dirty pages in the specified range of object. 341 * If syncio is TRUE, page cleaning is done synchronously. 342 * If de_queue is TRUE, pages are removed from any paging queue 343 * they were on, otherwise they are left on whatever queue they 344 * were on before the cleaning operation began. 345 * 346 * Odd semantics: if start == end, we clean everything. 347 * 348 * The object must be locked. 349 * 350 * Returns TRUE if all was well, FALSE if there was a pager error 351 * somewhere. We attempt to clean (and dequeue) all pages regardless 352 * of where an error occurs. 353 */ 354 boolean_t 355 vm_object_page_clean(object, start, end, syncio, de_queue) 356 register vm_object_t object; 357 register vm_offset_t start; 358 register vm_offset_t end; 359 boolean_t syncio; 360 boolean_t de_queue; 361 { 362 register vm_page_t p; 363 int onqueue; 364 boolean_t noerror = TRUE; 365 366 if (object == NULL) 367 return (TRUE); 368 369 /* 370 * If it is an internal object and there is no pager, attempt to 371 * allocate one. Note that vm_object_collapse may relocate one 372 * from a collapsed object so we must recheck afterward. 373 */ 374 if ((object->flags & OBJ_INTERNAL) && object->pager == NULL) { 375 vm_object_collapse(object); 376 if (object->pager == NULL) { 377 vm_pager_t pager; 378 379 vm_object_unlock(object); 380 pager = vm_pager_allocate(PG_DFLT, (caddr_t)0, 381 object->size, VM_PROT_ALL, 382 (vm_offset_t)0); 383 if (pager) 384 vm_object_setpager(object, pager, 0, FALSE); 385 vm_object_lock(object); 386 } 387 } 388 if (object->pager == NULL) 389 return (FALSE); 390 391 again: 392 /* 393 * Wait until the pageout daemon is through with the object. 394 */ 395 while (object->paging_in_progress) { 396 vm_object_sleep(object, object, FALSE); 397 vm_object_lock(object); 398 } 399 /* 400 * Loop through the object page list cleaning as necessary. 401 */ 402 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 403 if ((start == end || p->offset >= start && p->offset < end) && 404 !(p->flags & PG_FICTITIOUS)) { 405 if ((p->flags & PG_CLEAN) && 406 pmap_is_modified(VM_PAGE_TO_PHYS(p))) 407 p->flags &= ~PG_CLEAN; 408 /* 409 * Remove the page from any paging queue. 410 * This needs to be done if either we have been 411 * explicitly asked to do so or it is about to 412 * be cleaned (see comment below). 413 */ 414 if (de_queue || !(p->flags & PG_CLEAN)) { 415 vm_page_lock_queues(); 416 if (p->flags & PG_ACTIVE) { 417 TAILQ_REMOVE(&vm_page_queue_active, 418 p, pageq); 419 p->flags &= ~PG_ACTIVE; 420 cnt.v_active_count--; 421 onqueue = 1; 422 } else if (p->flags & PG_INACTIVE) { 423 TAILQ_REMOVE(&vm_page_queue_inactive, 424 p, pageq); 425 p->flags &= ~PG_INACTIVE; 426 cnt.v_inactive_count--; 427 onqueue = -1; 428 } else 429 onqueue = 0; 430 vm_page_unlock_queues(); 431 } 432 /* 433 * To ensure the state of the page doesn't change 434 * during the clean operation we do two things. 435 * First we set the busy bit and write-protect all 436 * mappings to ensure that write accesses to the 437 * page block (in vm_fault). Second, we remove 438 * the page from any paging queue to foil the 439 * pageout daemon (vm_pageout_scan). 440 */ 441 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 442 if (!(p->flags & PG_CLEAN)) { 443 p->flags |= PG_BUSY; 444 object->paging_in_progress++; 445 vm_object_unlock(object); 446 /* 447 * XXX if put fails we mark the page as 448 * clean to avoid an infinite loop. 449 * Will loose changes to the page. 450 */ 451 if (vm_pager_put(object->pager, p, syncio)) { 452 printf("%s: pager_put error\n", 453 "vm_object_page_clean"); 454 p->flags |= PG_CLEAN; 455 noerror = FALSE; 456 } 457 vm_object_lock(object); 458 object->paging_in_progress--; 459 if (!de_queue && onqueue) { 460 vm_page_lock_queues(); 461 if (onqueue > 0) 462 vm_page_activate(p); 463 else 464 vm_page_deactivate(p); 465 vm_page_unlock_queues(); 466 } 467 p->flags &= ~PG_BUSY; 468 PAGE_WAKEUP(p); 469 goto again; 470 } 471 } 472 } 473 return (noerror); 474 } 475 476 /* 477 * vm_object_deactivate_pages 478 * 479 * Deactivate all pages in the specified object. (Keep its pages 480 * in memory even though it is no longer referenced.) 481 * 482 * The object must be locked. 483 */ 484 void 485 vm_object_deactivate_pages(object) 486 register vm_object_t object; 487 { 488 register vm_page_t p, next; 489 490 for (p = object->memq.tqh_first; p != NULL; p = next) { 491 next = p->listq.tqe_next; 492 vm_page_lock_queues(); 493 vm_page_deactivate(p); 494 vm_page_unlock_queues(); 495 } 496 } 497 498 /* 499 * Trim the object cache to size. 500 */ 501 void 502 vm_object_cache_trim() 503 { 504 register vm_object_t object; 505 506 vm_object_cache_lock(); 507 while (vm_object_cached > vm_cache_max) { 508 object = vm_object_cached_list.tqh_first; 509 vm_object_cache_unlock(); 510 511 if (object != vm_object_lookup(object->pager)) 512 panic("vm_object_deactivate: I'm sooo confused."); 513 514 pager_cache(object, FALSE); 515 516 vm_object_cache_lock(); 517 } 518 vm_object_cache_unlock(); 519 } 520 521 /* 522 * vm_object_pmap_copy: 523 * 524 * Makes all physical pages in the specified 525 * object range copy-on-write. No writeable 526 * references to these pages should remain. 527 * 528 * The object must *not* be locked. 529 */ 530 void 531 vm_object_pmap_copy(object, start, end) 532 register vm_object_t object; 533 register vm_offset_t start; 534 register vm_offset_t end; 535 { 536 register vm_page_t p; 537 538 if (object == NULL) 539 return; 540 541 vm_object_lock(object); 542 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 543 if ((start <= p->offset) && (p->offset < end)) { 544 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 545 p->flags |= PG_COPYONWRITE; 546 } 547 } 548 vm_object_unlock(object); 549 } 550 551 /* 552 * vm_object_pmap_remove: 553 * 554 * Removes all physical pages in the specified 555 * object range from all physical maps. 556 * 557 * The object must *not* be locked. 558 */ 559 void 560 vm_object_pmap_remove(object, start, end) 561 register vm_object_t object; 562 register vm_offset_t start; 563 register vm_offset_t end; 564 { 565 register vm_page_t p; 566 567 if (object == NULL) 568 return; 569 570 vm_object_lock(object); 571 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 572 if ((start <= p->offset) && (p->offset < end)) 573 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 574 vm_object_unlock(object); 575 } 576 577 /* 578 * vm_object_copy: 579 * 580 * Create a new object which is a copy of an existing 581 * object, and mark all of the pages in the existing 582 * object 'copy-on-write'. The new object has one reference. 583 * Returns the new object. 584 * 585 * May defer the copy until later if the object is not backed 586 * up by a non-default pager. 587 */ 588 void 589 vm_object_copy(src_object, src_offset, size, 590 dst_object, dst_offset, src_needs_copy) 591 register vm_object_t src_object; 592 vm_offset_t src_offset; 593 vm_size_t size; 594 vm_object_t *dst_object; /* OUT */ 595 vm_offset_t *dst_offset; /* OUT */ 596 boolean_t *src_needs_copy; /* OUT */ 597 { 598 register vm_object_t new_copy; 599 register vm_object_t old_copy; 600 vm_offset_t new_start, new_end; 601 602 register vm_page_t p; 603 604 if (src_object == NULL) { 605 /* 606 * Nothing to copy 607 */ 608 *dst_object = NULL; 609 *dst_offset = 0; 610 *src_needs_copy = FALSE; 611 return; 612 } 613 614 /* 615 * If the object's pager is null_pager or the 616 * default pager, we don't have to make a copy 617 * of it. Instead, we set the needs copy flag and 618 * make a shadow later. 619 */ 620 621 vm_object_lock(src_object); 622 if (src_object->pager == NULL || 623 (src_object->flags & OBJ_INTERNAL)) { 624 625 /* 626 * Make another reference to the object 627 */ 628 src_object->ref_count++; 629 630 /* 631 * Mark all of the pages copy-on-write. 632 */ 633 for (p = src_object->memq.tqh_first; p; p = p->listq.tqe_next) 634 if (src_offset <= p->offset && 635 p->offset < src_offset + size) 636 p->flags |= PG_COPYONWRITE; 637 vm_object_unlock(src_object); 638 639 *dst_object = src_object; 640 *dst_offset = src_offset; 641 642 /* 643 * Must make a shadow when write is desired 644 */ 645 *src_needs_copy = TRUE; 646 return; 647 } 648 649 /* 650 * Try to collapse the object before copying it. 651 */ 652 vm_object_collapse(src_object); 653 654 /* 655 * If the object has a pager, the pager wants to 656 * see all of the changes. We need a copy-object 657 * for the changed pages. 658 * 659 * If there is a copy-object, and it is empty, 660 * no changes have been made to the object since the 661 * copy-object was made. We can use the same copy- 662 * object. 663 */ 664 665 Retry1: 666 old_copy = src_object->copy; 667 if (old_copy != NULL) { 668 /* 669 * Try to get the locks (out of order) 670 */ 671 if (!vm_object_lock_try(old_copy)) { 672 vm_object_unlock(src_object); 673 674 /* should spin a bit here... */ 675 vm_object_lock(src_object); 676 goto Retry1; 677 } 678 679 if (old_copy->resident_page_count == 0 && 680 old_copy->pager == NULL) { 681 /* 682 * Return another reference to 683 * the existing copy-object. 684 */ 685 old_copy->ref_count++; 686 vm_object_unlock(old_copy); 687 vm_object_unlock(src_object); 688 *dst_object = old_copy; 689 *dst_offset = src_offset; 690 *src_needs_copy = FALSE; 691 return; 692 } 693 vm_object_unlock(old_copy); 694 } 695 vm_object_unlock(src_object); 696 697 /* 698 * If the object has a pager, the pager wants 699 * to see all of the changes. We must make 700 * a copy-object and put the changed pages there. 701 * 702 * The copy-object is always made large enough to 703 * completely shadow the original object, since 704 * it may have several users who want to shadow 705 * the original object at different points. 706 */ 707 708 new_copy = vm_object_allocate(src_object->size); 709 710 Retry2: 711 vm_object_lock(src_object); 712 /* 713 * Copy object may have changed while we were unlocked 714 */ 715 old_copy = src_object->copy; 716 if (old_copy != NULL) { 717 /* 718 * Try to get the locks (out of order) 719 */ 720 if (!vm_object_lock_try(old_copy)) { 721 vm_object_unlock(src_object); 722 goto Retry2; 723 } 724 725 /* 726 * Consistency check 727 */ 728 if (old_copy->shadow != src_object || 729 old_copy->shadow_offset != (vm_offset_t) 0) 730 panic("vm_object_copy: copy/shadow inconsistency"); 731 732 /* 733 * Make the old copy-object shadow the new one. 734 * It will receive no more pages from the original 735 * object. 736 */ 737 738 src_object->ref_count--; /* remove ref. from old_copy */ 739 old_copy->shadow = new_copy; 740 new_copy->ref_count++; /* locking not needed - we 741 have the only pointer */ 742 vm_object_unlock(old_copy); /* done with old_copy */ 743 } 744 745 new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 746 new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 747 748 /* 749 * Point the new copy at the existing object. 750 */ 751 752 new_copy->shadow = src_object; 753 new_copy->shadow_offset = new_start; 754 src_object->ref_count++; 755 src_object->copy = new_copy; 756 757 /* 758 * Mark all the affected pages of the existing object 759 * copy-on-write. 760 */ 761 for (p = src_object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 762 if ((new_start <= p->offset) && (p->offset < new_end)) 763 p->flags |= PG_COPYONWRITE; 764 765 vm_object_unlock(src_object); 766 767 *dst_object = new_copy; 768 *dst_offset = src_offset - new_start; 769 *src_needs_copy = FALSE; 770 } 771 772 /* 773 * vm_object_shadow: 774 * 775 * Create a new object which is backed by the 776 * specified existing object range. The source 777 * object reference is deallocated. 778 * 779 * The new object and offset into that object 780 * are returned in the source parameters. 781 */ 782 783 void 784 vm_object_shadow(object, offset, length) 785 vm_object_t *object; /* IN/OUT */ 786 vm_offset_t *offset; /* IN/OUT */ 787 vm_size_t length; 788 { 789 register vm_object_t source; 790 register vm_object_t result; 791 792 source = *object; 793 794 /* 795 * Allocate a new object with the given length 796 */ 797 798 if ((result = vm_object_allocate(length)) == NULL) 799 panic("vm_object_shadow: no object for shadowing"); 800 801 /* 802 * The new object shadows the source object, adding 803 * a reference to it. Our caller changes his reference 804 * to point to the new object, removing a reference to 805 * the source object. Net result: no change of reference 806 * count. 807 */ 808 result->shadow = source; 809 810 /* 811 * Store the offset into the source object, 812 * and fix up the offset into the new object. 813 */ 814 815 result->shadow_offset = *offset; 816 817 /* 818 * Return the new things 819 */ 820 821 *offset = 0; 822 *object = result; 823 } 824 825 /* 826 * Set the specified object's pager to the specified pager. 827 */ 828 829 void 830 vm_object_setpager(object, pager, paging_offset, 831 read_only) 832 vm_object_t object; 833 vm_pager_t pager; 834 vm_offset_t paging_offset; 835 boolean_t read_only; 836 { 837 #ifdef lint 838 read_only++; /* No longer used */ 839 #endif 840 841 vm_object_lock(object); /* XXX ? */ 842 object->pager = pager; 843 object->paging_offset = paging_offset; 844 vm_object_unlock(object); /* XXX ? */ 845 } 846 847 /* 848 * vm_object_hash hashes the pager/id pair. 849 */ 850 851 #define vm_object_hash(pager) \ 852 (((unsigned long)pager)%VM_OBJECT_HASH_COUNT) 853 854 /* 855 * vm_object_lookup looks in the object cache for an object with the 856 * specified pager and paging id. 857 */ 858 859 vm_object_t 860 vm_object_lookup(pager) 861 vm_pager_t pager; 862 { 863 register vm_object_hash_entry_t entry; 864 vm_object_t object; 865 866 vm_object_cache_lock(); 867 868 for (entry = vm_object_hashtable[vm_object_hash(pager)].tqh_first; 869 entry != NULL; 870 entry = entry->hash_links.tqe_next) { 871 object = entry->object; 872 if (object->pager == pager) { 873 vm_object_lock(object); 874 if (object->ref_count == 0) { 875 TAILQ_REMOVE(&vm_object_cached_list, object, 876 cached_list); 877 vm_object_cached--; 878 } 879 object->ref_count++; 880 vm_object_unlock(object); 881 vm_object_cache_unlock(); 882 return(object); 883 } 884 } 885 886 vm_object_cache_unlock(); 887 return(NULL); 888 } 889 890 /* 891 * vm_object_enter enters the specified object/pager/id into 892 * the hash table. 893 */ 894 895 void 896 vm_object_enter(object, pager) 897 vm_object_t object; 898 vm_pager_t pager; 899 { 900 struct vm_object_hash_head *bucket; 901 register vm_object_hash_entry_t entry; 902 903 /* 904 * We don't cache null objects, and we can't cache 905 * objects with the null pager. 906 */ 907 908 if (object == NULL) 909 return; 910 if (pager == NULL) 911 return; 912 913 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 914 entry = (vm_object_hash_entry_t) 915 malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK); 916 entry->object = object; 917 object->flags |= OBJ_CANPERSIST; 918 919 vm_object_cache_lock(); 920 TAILQ_INSERT_TAIL(bucket, entry, hash_links); 921 vm_object_cache_unlock(); 922 } 923 924 /* 925 * vm_object_remove: 926 * 927 * Remove the pager from the hash table. 928 * Note: This assumes that the object cache 929 * is locked. XXX this should be fixed 930 * by reorganizing vm_object_deallocate. 931 */ 932 void 933 vm_object_remove(pager) 934 register vm_pager_t pager; 935 { 936 struct vm_object_hash_head *bucket; 937 register vm_object_hash_entry_t entry; 938 register vm_object_t object; 939 940 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 941 942 for (entry = bucket->tqh_first; 943 entry != NULL; 944 entry = entry->hash_links.tqe_next) { 945 object = entry->object; 946 if (object->pager == pager) { 947 TAILQ_REMOVE(bucket, entry, hash_links); 948 free((caddr_t)entry, M_VMOBJHASH); 949 break; 950 } 951 } 952 } 953 954 /* 955 * vm_object_cache_clear removes all objects from the cache. 956 * 957 */ 958 void 959 vm_object_cache_clear() 960 { 961 register vm_object_t object; 962 963 /* 964 * Remove each object in the cache by scanning down the 965 * list of cached objects. 966 */ 967 vm_object_cache_lock(); 968 while ((object = vm_object_cached_list.tqh_first) != NULL) { 969 vm_object_cache_unlock(); 970 971 /* 972 * Note: it is important that we use vm_object_lookup 973 * to gain a reference, and not vm_object_reference, because 974 * the logic for removing an object from the cache lies in 975 * lookup. 976 */ 977 if (object != vm_object_lookup(object->pager)) 978 panic("vm_object_cache_clear: I'm sooo confused."); 979 pager_cache(object, FALSE); 980 981 vm_object_cache_lock(); 982 } 983 vm_object_cache_unlock(); 984 } 985 986 boolean_t vm_object_collapse_allowed = TRUE; 987 /* 988 * vm_object_collapse: 989 * 990 * Collapse an object with the object backing it. 991 * Pages in the backing object are moved into the 992 * parent, and the backing object is deallocated. 993 * 994 * Requires that the object be locked and the page 995 * queues be unlocked. 996 * 997 */ 998 void 999 vm_object_collapse(object) 1000 register vm_object_t object; 1001 1002 { 1003 register vm_object_t backing_object; 1004 register vm_offset_t backing_offset; 1005 register vm_size_t size; 1006 register vm_offset_t new_offset; 1007 register vm_page_t p, pp; 1008 1009 if (!vm_object_collapse_allowed) 1010 return; 1011 1012 while (TRUE) { 1013 /* 1014 * Verify that the conditions are right for collapse: 1015 * 1016 * The object exists and no pages in it are currently 1017 * being paged out (or have ever been paged out). 1018 */ 1019 if (object == NULL || 1020 object->paging_in_progress != 0 || 1021 object->pager != NULL) 1022 return; 1023 1024 /* 1025 * There is a backing object, and 1026 */ 1027 1028 if ((backing_object = object->shadow) == NULL) 1029 return; 1030 1031 vm_object_lock(backing_object); 1032 /* 1033 * ... 1034 * The backing object is not read_only, 1035 * and no pages in the backing object are 1036 * currently being paged out. 1037 * The backing object is internal. 1038 */ 1039 1040 if ((backing_object->flags & OBJ_INTERNAL) == 0 || 1041 backing_object->paging_in_progress != 0) { 1042 vm_object_unlock(backing_object); 1043 return; 1044 } 1045 1046 /* 1047 * The backing object can't be a copy-object: 1048 * the shadow_offset for the copy-object must stay 1049 * as 0. Furthermore (for the 'we have all the 1050 * pages' case), if we bypass backing_object and 1051 * just shadow the next object in the chain, old 1052 * pages from that object would then have to be copied 1053 * BOTH into the (former) backing_object and into the 1054 * parent object. 1055 */ 1056 if (backing_object->shadow != NULL && 1057 backing_object->shadow->copy != NULL) { 1058 vm_object_unlock(backing_object); 1059 return; 1060 } 1061 1062 /* 1063 * We know that we can either collapse the backing 1064 * object (if the parent is the only reference to 1065 * it) or (perhaps) remove the parent's reference 1066 * to it. 1067 */ 1068 1069 backing_offset = object->shadow_offset; 1070 size = object->size; 1071 1072 /* 1073 * If there is exactly one reference to the backing 1074 * object, we can collapse it into the parent. 1075 */ 1076 1077 if (backing_object->ref_count == 1) { 1078 1079 /* 1080 * We can collapse the backing object. 1081 * 1082 * Move all in-memory pages from backing_object 1083 * to the parent. Pages that have been paged out 1084 * will be overwritten by any of the parent's 1085 * pages that shadow them. 1086 */ 1087 1088 while ((p = backing_object->memq.tqh_first) != NULL) { 1089 new_offset = (p->offset - backing_offset); 1090 1091 /* 1092 * If the parent has a page here, or if 1093 * this page falls outside the parent, 1094 * dispose of it. 1095 * 1096 * Otherwise, move it as planned. 1097 */ 1098 1099 if (p->offset < backing_offset || 1100 new_offset >= size) { 1101 vm_page_lock_queues(); 1102 vm_page_free(p); 1103 vm_page_unlock_queues(); 1104 } else { 1105 pp = vm_page_lookup(object, new_offset); 1106 if (pp != NULL && !(pp->flags & PG_FAKE)) { 1107 vm_page_lock_queues(); 1108 vm_page_free(p); 1109 vm_page_unlock_queues(); 1110 } 1111 else { 1112 if (pp) { 1113 /* may be someone waiting for it */ 1114 PAGE_WAKEUP(pp); 1115 vm_page_lock_queues(); 1116 vm_page_free(pp); 1117 vm_page_unlock_queues(); 1118 } 1119 vm_page_rename(p, object, new_offset); 1120 } 1121 } 1122 } 1123 1124 /* 1125 * Move the pager from backing_object to object. 1126 * 1127 * XXX We're only using part of the paging space 1128 * for keeps now... we ought to discard the 1129 * unused portion. 1130 */ 1131 1132 if (backing_object->pager) { 1133 object->pager = backing_object->pager; 1134 object->paging_offset = backing_offset + 1135 backing_object->paging_offset; 1136 backing_object->pager = NULL; 1137 } 1138 1139 /* 1140 * Object now shadows whatever backing_object did. 1141 * Note that the reference to backing_object->shadow 1142 * moves from within backing_object to within object. 1143 */ 1144 1145 object->shadow = backing_object->shadow; 1146 object->shadow_offset += backing_object->shadow_offset; 1147 if (object->shadow != NULL && 1148 object->shadow->copy != NULL) { 1149 panic("vm_object_collapse: we collapsed a copy-object!"); 1150 } 1151 /* 1152 * Discard backing_object. 1153 * 1154 * Since the backing object has no pages, no 1155 * pager left, and no object references within it, 1156 * all that is necessary is to dispose of it. 1157 */ 1158 1159 vm_object_unlock(backing_object); 1160 1161 simple_lock(&vm_object_list_lock); 1162 TAILQ_REMOVE(&vm_object_list, backing_object, 1163 object_list); 1164 vm_object_count--; 1165 simple_unlock(&vm_object_list_lock); 1166 1167 free((caddr_t)backing_object, M_VMOBJ); 1168 1169 object_collapses++; 1170 } 1171 else { 1172 /* 1173 * If all of the pages in the backing object are 1174 * shadowed by the parent object, the parent 1175 * object no longer has to shadow the backing 1176 * object; it can shadow the next one in the 1177 * chain. 1178 * 1179 * The backing object must not be paged out - we'd 1180 * have to check all of the paged-out pages, as 1181 * well. 1182 */ 1183 1184 if (backing_object->pager != NULL) { 1185 vm_object_unlock(backing_object); 1186 return; 1187 } 1188 1189 /* 1190 * Should have a check for a 'small' number 1191 * of pages here. 1192 */ 1193 1194 for (p = backing_object->memq.tqh_first; 1195 p != NULL; 1196 p = p->listq.tqe_next) { 1197 new_offset = (p->offset - backing_offset); 1198 1199 /* 1200 * If the parent has a page here, or if 1201 * this page falls outside the parent, 1202 * keep going. 1203 * 1204 * Otherwise, the backing_object must be 1205 * left in the chain. 1206 */ 1207 1208 if (p->offset >= backing_offset && 1209 new_offset < size && 1210 ((pp = vm_page_lookup(object, new_offset)) 1211 == NULL || 1212 (pp->flags & PG_FAKE))) { 1213 /* 1214 * Page still needed. 1215 * Can't go any further. 1216 */ 1217 vm_object_unlock(backing_object); 1218 return; 1219 } 1220 } 1221 1222 /* 1223 * Make the parent shadow the next object 1224 * in the chain. Deallocating backing_object 1225 * will not remove it, since its reference 1226 * count is at least 2. 1227 */ 1228 1229 object->shadow = backing_object->shadow; 1230 vm_object_reference(object->shadow); 1231 object->shadow_offset += backing_object->shadow_offset; 1232 1233 /* 1234 * Backing object might have had a copy pointer 1235 * to us. If it did, clear it. 1236 */ 1237 if (backing_object->copy == object) { 1238 backing_object->copy = NULL; 1239 } 1240 1241 /* Drop the reference count on backing_object. 1242 * Since its ref_count was at least 2, it 1243 * will not vanish; so we don't need to call 1244 * vm_object_deallocate. 1245 */ 1246 backing_object->ref_count--; 1247 vm_object_unlock(backing_object); 1248 1249 object_bypasses ++; 1250 1251 } 1252 1253 /* 1254 * Try again with this object's new backing object. 1255 */ 1256 } 1257 } 1258 1259 /* 1260 * vm_object_page_remove: [internal] 1261 * 1262 * Removes all physical pages in the specified 1263 * object range from the object's list of pages. 1264 * 1265 * The object must be locked. 1266 */ 1267 void 1268 vm_object_page_remove(object, start, end) 1269 register vm_object_t object; 1270 register vm_offset_t start; 1271 register vm_offset_t end; 1272 { 1273 register vm_page_t p, next; 1274 1275 if (object == NULL) 1276 return; 1277 1278 for (p = object->memq.tqh_first; p != NULL; p = next) { 1279 next = p->listq.tqe_next; 1280 if ((start <= p->offset) && (p->offset < end)) { 1281 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1282 vm_page_lock_queues(); 1283 vm_page_free(p); 1284 vm_page_unlock_queues(); 1285 } 1286 } 1287 } 1288 1289 /* 1290 * Routine: vm_object_coalesce 1291 * Function: Coalesces two objects backing up adjoining 1292 * regions of memory into a single object. 1293 * 1294 * returns TRUE if objects were combined. 1295 * 1296 * NOTE: Only works at the moment if the second object is NULL - 1297 * if it's not, which object do we lock first? 1298 * 1299 * Parameters: 1300 * prev_object First object to coalesce 1301 * prev_offset Offset into prev_object 1302 * next_object Second object into coalesce 1303 * next_offset Offset into next_object 1304 * 1305 * prev_size Size of reference to prev_object 1306 * next_size Size of reference to next_object 1307 * 1308 * Conditions: 1309 * The object must *not* be locked. 1310 */ 1311 boolean_t 1312 vm_object_coalesce(prev_object, next_object, 1313 prev_offset, next_offset, 1314 prev_size, next_size) 1315 1316 register vm_object_t prev_object; 1317 vm_object_t next_object; 1318 vm_offset_t prev_offset, next_offset; 1319 vm_size_t prev_size, next_size; 1320 { 1321 vm_size_t newsize; 1322 1323 #ifdef lint 1324 next_offset++; 1325 #endif 1326 1327 if (next_object != NULL) { 1328 return(FALSE); 1329 } 1330 1331 if (prev_object == NULL) { 1332 return(TRUE); 1333 } 1334 1335 vm_object_lock(prev_object); 1336 1337 /* 1338 * Try to collapse the object first 1339 */ 1340 vm_object_collapse(prev_object); 1341 1342 /* 1343 * Can't coalesce if: 1344 * . more than one reference 1345 * . paged out 1346 * . shadows another object 1347 * . has a copy elsewhere 1348 * (any of which mean that the pages not mapped to 1349 * prev_entry may be in use anyway) 1350 */ 1351 1352 if (prev_object->ref_count > 1 || 1353 prev_object->pager != NULL || 1354 prev_object->shadow != NULL || 1355 prev_object->copy != NULL) { 1356 vm_object_unlock(prev_object); 1357 return(FALSE); 1358 } 1359 1360 /* 1361 * Remove any pages that may still be in the object from 1362 * a previous deallocation. 1363 */ 1364 1365 vm_object_page_remove(prev_object, 1366 prev_offset + prev_size, 1367 prev_offset + prev_size + next_size); 1368 1369 /* 1370 * Extend the object if necessary. 1371 */ 1372 newsize = prev_offset + prev_size + next_size; 1373 if (newsize > prev_object->size) 1374 prev_object->size = newsize; 1375 1376 vm_object_unlock(prev_object); 1377 return(TRUE); 1378 } 1379 1380 /* 1381 * vm_object_print: [ debug ] 1382 */ 1383 void 1384 vm_object_print(object, full) 1385 vm_object_t object; 1386 boolean_t full; 1387 { 1388 register vm_page_t p; 1389 extern indent; 1390 1391 register int count; 1392 1393 if (object == NULL) 1394 return; 1395 1396 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1397 (int) object, (int) object->size, 1398 object->resident_page_count, object->ref_count); 1399 printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1400 (int) object->pager, (int) object->paging_offset, 1401 (int) object->shadow, (int) object->shadow_offset); 1402 printf("cache: next=0x%x, prev=0x%x\n", 1403 object->cached_list.tqe_next, object->cached_list.tqe_prev); 1404 1405 if (!full) 1406 return; 1407 1408 indent += 2; 1409 count = 0; 1410 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 1411 if (count == 0) 1412 iprintf("memory:="); 1413 else if (count == 6) { 1414 printf("\n"); 1415 iprintf(" ..."); 1416 count = 0; 1417 } else 1418 printf(","); 1419 count++; 1420 1421 printf("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p)); 1422 } 1423 if (count != 0) 1424 printf("\n"); 1425 indent -= 2; 1426 } 1427