1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_object.c 8.5 (Berkeley) 03/22/94 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Virtual memory object module. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/malloc.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 50 /* 51 * Virtual memory objects maintain the actual data 52 * associated with allocated virtual memory. A given 53 * page of memory exists within exactly one object. 54 * 55 * An object is only deallocated when all "references" 56 * are given up. Only one "reference" to a given 57 * region of an object should be writeable. 58 * 59 * Associated with each object is a list of all resident 60 * memory pages belonging to that object; this list is 61 * maintained by the "vm_page" module, and locked by the object's 62 * lock. 63 * 64 * Each object also records a "pager" routine which is 65 * used to retrieve (and store) pages to the proper backing 66 * storage. In addition, objects may be backed by other 67 * objects from which they were virtual-copied. 68 * 69 * The only items within the object structure which are 70 * modified after time of creation are: 71 * reference count locked by object's lock 72 * pager routine locked by object's lock 73 * 74 */ 75 76 struct vm_object kernel_object_store; 77 struct vm_object kmem_object_store; 78 79 #define VM_OBJECT_HASH_COUNT 157 80 81 int vm_cache_max = 100; /* can patch if necessary */ 82 struct vm_object_hash_head vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 83 84 long object_collapses = 0; 85 long object_bypasses = 0; 86 87 static void _vm_object_allocate __P((vm_size_t, vm_object_t)); 88 89 /* 90 * vm_object_init: 91 * 92 * Initialize the VM objects module. 93 */ 94 void vm_object_init(size) 95 vm_size_t size; 96 { 97 register int i; 98 99 TAILQ_INIT(&vm_object_cached_list); 100 TAILQ_INIT(&vm_object_list); 101 vm_object_count = 0; 102 simple_lock_init(&vm_cache_lock); 103 simple_lock_init(&vm_object_list_lock); 104 105 for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 106 TAILQ_INIT(&vm_object_hashtable[i]); 107 108 kernel_object = &kernel_object_store; 109 _vm_object_allocate(size, kernel_object); 110 111 kmem_object = &kmem_object_store; 112 _vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object); 113 } 114 115 /* 116 * vm_object_allocate: 117 * 118 * Returns a new object with the given size. 119 */ 120 121 vm_object_t vm_object_allocate(size) 122 vm_size_t size; 123 { 124 register vm_object_t result; 125 126 result = (vm_object_t) 127 malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK); 128 129 _vm_object_allocate(size, result); 130 131 return(result); 132 } 133 134 static void 135 _vm_object_allocate(size, object) 136 vm_size_t size; 137 register vm_object_t object; 138 { 139 TAILQ_INIT(&object->memq); 140 vm_object_lock_init(object); 141 object->ref_count = 1; 142 object->resident_page_count = 0; 143 object->size = size; 144 object->flags = OBJ_INTERNAL; /* vm_allocate_with_pager will reset */ 145 object->paging_in_progress = 0; 146 object->copy = NULL; 147 148 /* 149 * Object starts out read-write, with no pager. 150 */ 151 152 object->pager = NULL; 153 object->paging_offset = 0; 154 object->shadow = NULL; 155 object->shadow_offset = (vm_offset_t) 0; 156 157 simple_lock(&vm_object_list_lock); 158 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 159 vm_object_count++; 160 cnt.v_nzfod += atop(size); 161 simple_unlock(&vm_object_list_lock); 162 } 163 164 /* 165 * vm_object_reference: 166 * 167 * Gets another reference to the given object. 168 */ 169 void vm_object_reference(object) 170 register vm_object_t object; 171 { 172 if (object == NULL) 173 return; 174 175 vm_object_lock(object); 176 object->ref_count++; 177 vm_object_unlock(object); 178 } 179 180 /* 181 * vm_object_deallocate: 182 * 183 * Release a reference to the specified object, 184 * gained either through a vm_object_allocate 185 * or a vm_object_reference call. When all references 186 * are gone, storage associated with this object 187 * may be relinquished. 188 * 189 * No object may be locked. 190 */ 191 void vm_object_deallocate(object) 192 register vm_object_t object; 193 { 194 vm_object_t temp; 195 196 while (object != NULL) { 197 198 /* 199 * The cache holds a reference (uncounted) to 200 * the object; we must lock it before removing 201 * the object. 202 */ 203 204 vm_object_cache_lock(); 205 206 /* 207 * Lose the reference 208 */ 209 vm_object_lock(object); 210 if (--(object->ref_count) != 0) { 211 212 /* 213 * If there are still references, then 214 * we are done. 215 */ 216 vm_object_unlock(object); 217 vm_object_cache_unlock(); 218 return; 219 } 220 221 /* 222 * See if this object can persist. If so, enter 223 * it in the cache, then deactivate all of its 224 * pages. 225 */ 226 227 if (object->flags & OBJ_CANPERSIST) { 228 229 TAILQ_INSERT_TAIL(&vm_object_cached_list, object, 230 cached_list); 231 vm_object_cached++; 232 vm_object_cache_unlock(); 233 234 vm_object_deactivate_pages(object); 235 vm_object_unlock(object); 236 237 vm_object_cache_trim(); 238 return; 239 } 240 241 /* 242 * Make sure no one can look us up now. 243 */ 244 vm_object_remove(object->pager); 245 vm_object_cache_unlock(); 246 247 temp = object->shadow; 248 vm_object_terminate(object); 249 /* unlocks and deallocates object */ 250 object = temp; 251 } 252 } 253 254 255 /* 256 * vm_object_terminate actually destroys the specified object, freeing 257 * up all previously used resources. 258 * 259 * The object must be locked. 260 */ 261 void vm_object_terminate(object) 262 register vm_object_t object; 263 { 264 register vm_page_t p; 265 vm_object_t shadow_object; 266 267 /* 268 * Detach the object from its shadow if we are the shadow's 269 * copy. 270 */ 271 if ((shadow_object = object->shadow) != NULL) { 272 vm_object_lock(shadow_object); 273 if (shadow_object->copy == object) 274 shadow_object->copy = NULL; 275 #if 0 276 else if (shadow_object->copy != NULL) 277 panic("vm_object_terminate: copy/shadow inconsistency"); 278 #endif 279 vm_object_unlock(shadow_object); 280 } 281 282 /* 283 * Wait until the pageout daemon is through with the object. 284 */ 285 while (object->paging_in_progress) { 286 vm_object_sleep((int)object, object, FALSE); 287 vm_object_lock(object); 288 } 289 290 /* 291 * If not an internal object clean all the pages, removing them 292 * from paging queues as we go. 293 * 294 * XXX need to do something in the event of a cleaning error. 295 */ 296 if ((object->flags & OBJ_INTERNAL) == 0) { 297 (void) vm_object_page_clean(object, 0, 0, TRUE, TRUE); 298 vm_object_unlock(object); 299 } 300 301 /* 302 * Now free the pages. 303 * For internal objects, this also removes them from paging queues. 304 */ 305 while ((p = object->memq.tqh_first) != NULL) { 306 VM_PAGE_CHECK(p); 307 vm_page_lock_queues(); 308 vm_page_free(p); 309 cnt.v_pfree++; 310 vm_page_unlock_queues(); 311 } 312 if ((object->flags & OBJ_INTERNAL) == 0) 313 vm_object_unlock(object); 314 315 /* 316 * Let the pager know object is dead. 317 */ 318 if (object->pager != NULL) 319 vm_pager_deallocate(object->pager); 320 321 simple_lock(&vm_object_list_lock); 322 TAILQ_REMOVE(&vm_object_list, object, object_list); 323 vm_object_count--; 324 simple_unlock(&vm_object_list_lock); 325 326 /* 327 * Free the space for the object. 328 */ 329 free((caddr_t)object, M_VMOBJ); 330 } 331 332 /* 333 * vm_object_page_clean 334 * 335 * Clean all dirty pages in the specified range of object. 336 * If syncio is TRUE, page cleaning is done synchronously. 337 * If de_queue is TRUE, pages are removed from any paging queue 338 * they were on, otherwise they are left on whatever queue they 339 * were on before the cleaning operation began. 340 * 341 * Odd semantics: if start == end, we clean everything. 342 * 343 * The object must be locked. 344 * 345 * Returns TRUE if all was well, FALSE if there was a pager error 346 * somewhere. We attempt to clean (and dequeue) all pages regardless 347 * of where an error occurs. 348 */ 349 boolean_t 350 vm_object_page_clean(object, start, end, syncio, de_queue) 351 register vm_object_t object; 352 register vm_offset_t start; 353 register vm_offset_t end; 354 boolean_t syncio; 355 boolean_t de_queue; 356 { 357 register vm_page_t p; 358 int onqueue; 359 boolean_t noerror = TRUE; 360 361 if (object == NULL) 362 return (TRUE); 363 364 /* 365 * If it is an internal object and there is no pager, attempt to 366 * allocate one. Note that vm_object_collapse may relocate one 367 * from a collapsed object so we must recheck afterward. 368 */ 369 if ((object->flags & OBJ_INTERNAL) && object->pager == NULL) { 370 vm_object_collapse(object); 371 if (object->pager == NULL) { 372 vm_pager_t pager; 373 374 vm_object_unlock(object); 375 pager = vm_pager_allocate(PG_DFLT, (caddr_t)0, 376 object->size, VM_PROT_ALL, 377 (vm_offset_t)0); 378 if (pager) 379 vm_object_setpager(object, pager, 0, FALSE); 380 vm_object_lock(object); 381 } 382 } 383 if (object->pager == NULL) 384 return (FALSE); 385 386 again: 387 /* 388 * Wait until the pageout daemon is through with the object. 389 */ 390 while (object->paging_in_progress) { 391 vm_object_sleep((int)object, object, FALSE); 392 vm_object_lock(object); 393 } 394 /* 395 * Loop through the object page list cleaning as necessary. 396 */ 397 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 398 if ((start == end || p->offset >= start && p->offset < end) && 399 !(p->flags & PG_FICTITIOUS)) { 400 if ((p->flags & PG_CLEAN) && 401 pmap_is_modified(VM_PAGE_TO_PHYS(p))) 402 p->flags &= ~PG_CLEAN; 403 /* 404 * Remove the page from any paging queue. 405 * This needs to be done if either we have been 406 * explicitly asked to do so or it is about to 407 * be cleaned (see comment below). 408 */ 409 if (de_queue || !(p->flags & PG_CLEAN)) { 410 vm_page_lock_queues(); 411 if (p->flags & PG_ACTIVE) { 412 TAILQ_REMOVE(&vm_page_queue_active, 413 p, pageq); 414 p->flags &= ~PG_ACTIVE; 415 cnt.v_active_count--; 416 onqueue = 1; 417 } else if (p->flags & PG_INACTIVE) { 418 TAILQ_REMOVE(&vm_page_queue_inactive, 419 p, pageq); 420 p->flags &= ~PG_INACTIVE; 421 cnt.v_inactive_count--; 422 onqueue = -1; 423 } else 424 onqueue = 0; 425 vm_page_unlock_queues(); 426 } 427 /* 428 * To ensure the state of the page doesn't change 429 * during the clean operation we do two things. 430 * First we set the busy bit and write-protect all 431 * mappings to ensure that write accesses to the 432 * page block (in vm_fault). Second, we remove 433 * the page from any paging queue to foil the 434 * pageout daemon (vm_pageout_scan). 435 */ 436 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 437 if (!(p->flags & PG_CLEAN)) { 438 p->flags |= PG_BUSY; 439 object->paging_in_progress++; 440 vm_object_unlock(object); 441 /* 442 * XXX if put fails we mark the page as 443 * clean to avoid an infinite loop. 444 * Will loose changes to the page. 445 */ 446 if (vm_pager_put(object->pager, p, syncio)) { 447 printf("%s: pager_put error\n", 448 "vm_object_page_clean"); 449 p->flags |= PG_CLEAN; 450 noerror = FALSE; 451 } 452 vm_object_lock(object); 453 object->paging_in_progress--; 454 if (!de_queue && onqueue) { 455 vm_page_lock_queues(); 456 if (onqueue > 0) 457 vm_page_activate(p); 458 else 459 vm_page_deactivate(p); 460 vm_page_unlock_queues(); 461 } 462 p->flags &= ~PG_BUSY; 463 PAGE_WAKEUP(p); 464 goto again; 465 } 466 } 467 } 468 return (noerror); 469 } 470 471 /* 472 * vm_object_deactivate_pages 473 * 474 * Deactivate all pages in the specified object. (Keep its pages 475 * in memory even though it is no longer referenced.) 476 * 477 * The object must be locked. 478 */ 479 void 480 vm_object_deactivate_pages(object) 481 register vm_object_t object; 482 { 483 register vm_page_t p, next; 484 485 for (p = object->memq.tqh_first; p != NULL; p = next) { 486 next = p->listq.tqe_next; 487 vm_page_lock_queues(); 488 vm_page_deactivate(p); 489 vm_page_unlock_queues(); 490 } 491 } 492 493 /* 494 * Trim the object cache to size. 495 */ 496 void 497 vm_object_cache_trim() 498 { 499 register vm_object_t object; 500 501 vm_object_cache_lock(); 502 while (vm_object_cached > vm_cache_max) { 503 object = vm_object_cached_list.tqh_first; 504 vm_object_cache_unlock(); 505 506 if (object != vm_object_lookup(object->pager)) 507 panic("vm_object_deactivate: I'm sooo confused."); 508 509 pager_cache(object, FALSE); 510 511 vm_object_cache_lock(); 512 } 513 vm_object_cache_unlock(); 514 } 515 516 /* 517 * vm_object_pmap_copy: 518 * 519 * Makes all physical pages in the specified 520 * object range copy-on-write. No writeable 521 * references to these pages should remain. 522 * 523 * The object must *not* be locked. 524 */ 525 void vm_object_pmap_copy(object, start, end) 526 register vm_object_t object; 527 register vm_offset_t start; 528 register vm_offset_t end; 529 { 530 register vm_page_t p; 531 532 if (object == NULL) 533 return; 534 535 vm_object_lock(object); 536 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 537 if ((start <= p->offset) && (p->offset < end)) { 538 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 539 p->flags |= PG_COPYONWRITE; 540 } 541 } 542 vm_object_unlock(object); 543 } 544 545 /* 546 * vm_object_pmap_remove: 547 * 548 * Removes all physical pages in the specified 549 * object range from all physical maps. 550 * 551 * The object must *not* be locked. 552 */ 553 void vm_object_pmap_remove(object, start, end) 554 register vm_object_t object; 555 register vm_offset_t start; 556 register vm_offset_t end; 557 { 558 register vm_page_t p; 559 560 if (object == NULL) 561 return; 562 563 vm_object_lock(object); 564 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 565 if ((start <= p->offset) && (p->offset < end)) 566 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 567 vm_object_unlock(object); 568 } 569 570 /* 571 * vm_object_copy: 572 * 573 * Create a new object which is a copy of an existing 574 * object, and mark all of the pages in the existing 575 * object 'copy-on-write'. The new object has one reference. 576 * Returns the new object. 577 * 578 * May defer the copy until later if the object is not backed 579 * up by a non-default pager. 580 */ 581 void vm_object_copy(src_object, src_offset, size, 582 dst_object, dst_offset, src_needs_copy) 583 register vm_object_t src_object; 584 vm_offset_t src_offset; 585 vm_size_t size; 586 vm_object_t *dst_object; /* OUT */ 587 vm_offset_t *dst_offset; /* OUT */ 588 boolean_t *src_needs_copy; /* OUT */ 589 { 590 register vm_object_t new_copy; 591 register vm_object_t old_copy; 592 vm_offset_t new_start, new_end; 593 594 register vm_page_t p; 595 596 if (src_object == NULL) { 597 /* 598 * Nothing to copy 599 */ 600 *dst_object = NULL; 601 *dst_offset = 0; 602 *src_needs_copy = FALSE; 603 return; 604 } 605 606 /* 607 * If the object's pager is null_pager or the 608 * default pager, we don't have to make a copy 609 * of it. Instead, we set the needs copy flag and 610 * make a shadow later. 611 */ 612 613 vm_object_lock(src_object); 614 if (src_object->pager == NULL || 615 (src_object->flags & OBJ_INTERNAL)) { 616 617 /* 618 * Make another reference to the object 619 */ 620 src_object->ref_count++; 621 622 /* 623 * Mark all of the pages copy-on-write. 624 */ 625 for (p = src_object->memq.tqh_first; p; p = p->listq.tqe_next) 626 if (src_offset <= p->offset && 627 p->offset < src_offset + size) 628 p->flags |= PG_COPYONWRITE; 629 vm_object_unlock(src_object); 630 631 *dst_object = src_object; 632 *dst_offset = src_offset; 633 634 /* 635 * Must make a shadow when write is desired 636 */ 637 *src_needs_copy = TRUE; 638 return; 639 } 640 641 /* 642 * Try to collapse the object before copying it. 643 */ 644 vm_object_collapse(src_object); 645 646 /* 647 * If the object has a pager, the pager wants to 648 * see all of the changes. We need a copy-object 649 * for the changed pages. 650 * 651 * If there is a copy-object, and it is empty, 652 * no changes have been made to the object since the 653 * copy-object was made. We can use the same copy- 654 * object. 655 */ 656 657 Retry1: 658 old_copy = src_object->copy; 659 if (old_copy != NULL) { 660 /* 661 * Try to get the locks (out of order) 662 */ 663 if (!vm_object_lock_try(old_copy)) { 664 vm_object_unlock(src_object); 665 666 /* should spin a bit here... */ 667 vm_object_lock(src_object); 668 goto Retry1; 669 } 670 671 if (old_copy->resident_page_count == 0 && 672 old_copy->pager == NULL) { 673 /* 674 * Return another reference to 675 * the existing copy-object. 676 */ 677 old_copy->ref_count++; 678 vm_object_unlock(old_copy); 679 vm_object_unlock(src_object); 680 *dst_object = old_copy; 681 *dst_offset = src_offset; 682 *src_needs_copy = FALSE; 683 return; 684 } 685 vm_object_unlock(old_copy); 686 } 687 vm_object_unlock(src_object); 688 689 /* 690 * If the object has a pager, the pager wants 691 * to see all of the changes. We must make 692 * a copy-object and put the changed pages there. 693 * 694 * The copy-object is always made large enough to 695 * completely shadow the original object, since 696 * it may have several users who want to shadow 697 * the original object at different points. 698 */ 699 700 new_copy = vm_object_allocate(src_object->size); 701 702 Retry2: 703 vm_object_lock(src_object); 704 /* 705 * Copy object may have changed while we were unlocked 706 */ 707 old_copy = src_object->copy; 708 if (old_copy != NULL) { 709 /* 710 * Try to get the locks (out of order) 711 */ 712 if (!vm_object_lock_try(old_copy)) { 713 vm_object_unlock(src_object); 714 goto Retry2; 715 } 716 717 /* 718 * Consistency check 719 */ 720 if (old_copy->shadow != src_object || 721 old_copy->shadow_offset != (vm_offset_t) 0) 722 panic("vm_object_copy: copy/shadow inconsistency"); 723 724 /* 725 * Make the old copy-object shadow the new one. 726 * It will receive no more pages from the original 727 * object. 728 */ 729 730 src_object->ref_count--; /* remove ref. from old_copy */ 731 old_copy->shadow = new_copy; 732 new_copy->ref_count++; /* locking not needed - we 733 have the only pointer */ 734 vm_object_unlock(old_copy); /* done with old_copy */ 735 } 736 737 new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 738 new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 739 740 /* 741 * Point the new copy at the existing object. 742 */ 743 744 new_copy->shadow = src_object; 745 new_copy->shadow_offset = new_start; 746 src_object->ref_count++; 747 src_object->copy = new_copy; 748 749 /* 750 * Mark all the affected pages of the existing object 751 * copy-on-write. 752 */ 753 for (p = src_object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 754 if ((new_start <= p->offset) && (p->offset < new_end)) 755 p->flags |= PG_COPYONWRITE; 756 757 vm_object_unlock(src_object); 758 759 *dst_object = new_copy; 760 *dst_offset = src_offset - new_start; 761 *src_needs_copy = FALSE; 762 } 763 764 /* 765 * vm_object_shadow: 766 * 767 * Create a new object which is backed by the 768 * specified existing object range. The source 769 * object reference is deallocated. 770 * 771 * The new object and offset into that object 772 * are returned in the source parameters. 773 */ 774 775 void vm_object_shadow(object, offset, length) 776 vm_object_t *object; /* IN/OUT */ 777 vm_offset_t *offset; /* IN/OUT */ 778 vm_size_t length; 779 { 780 register vm_object_t source; 781 register vm_object_t result; 782 783 source = *object; 784 785 /* 786 * Allocate a new object with the given length 787 */ 788 789 if ((result = vm_object_allocate(length)) == NULL) 790 panic("vm_object_shadow: no object for shadowing"); 791 792 /* 793 * The new object shadows the source object, adding 794 * a reference to it. Our caller changes his reference 795 * to point to the new object, removing a reference to 796 * the source object. Net result: no change of reference 797 * count. 798 */ 799 result->shadow = source; 800 801 /* 802 * Store the offset into the source object, 803 * and fix up the offset into the new object. 804 */ 805 806 result->shadow_offset = *offset; 807 808 /* 809 * Return the new things 810 */ 811 812 *offset = 0; 813 *object = result; 814 } 815 816 /* 817 * Set the specified object's pager to the specified pager. 818 */ 819 820 void vm_object_setpager(object, pager, paging_offset, 821 read_only) 822 vm_object_t object; 823 vm_pager_t pager; 824 vm_offset_t paging_offset; 825 boolean_t read_only; 826 { 827 #ifdef lint 828 read_only++; /* No longer used */ 829 #endif 830 831 vm_object_lock(object); /* XXX ? */ 832 object->pager = pager; 833 object->paging_offset = paging_offset; 834 vm_object_unlock(object); /* XXX ? */ 835 } 836 837 /* 838 * vm_object_hash hashes the pager/id pair. 839 */ 840 841 #define vm_object_hash(pager) \ 842 (((unsigned)pager)%VM_OBJECT_HASH_COUNT) 843 844 /* 845 * vm_object_lookup looks in the object cache for an object with the 846 * specified pager and paging id. 847 */ 848 849 vm_object_t vm_object_lookup(pager) 850 vm_pager_t pager; 851 { 852 register vm_object_hash_entry_t entry; 853 vm_object_t object; 854 855 vm_object_cache_lock(); 856 857 for (entry = vm_object_hashtable[vm_object_hash(pager)].tqh_first; 858 entry != NULL; 859 entry = entry->hash_links.tqe_next) { 860 object = entry->object; 861 if (object->pager == pager) { 862 vm_object_lock(object); 863 if (object->ref_count == 0) { 864 TAILQ_REMOVE(&vm_object_cached_list, object, 865 cached_list); 866 vm_object_cached--; 867 } 868 object->ref_count++; 869 vm_object_unlock(object); 870 vm_object_cache_unlock(); 871 return(object); 872 } 873 } 874 875 vm_object_cache_unlock(); 876 return(NULL); 877 } 878 879 /* 880 * vm_object_enter enters the specified object/pager/id into 881 * the hash table. 882 */ 883 884 void vm_object_enter(object, pager) 885 vm_object_t object; 886 vm_pager_t pager; 887 { 888 struct vm_object_hash_head *bucket; 889 register vm_object_hash_entry_t entry; 890 891 /* 892 * We don't cache null objects, and we can't cache 893 * objects with the null pager. 894 */ 895 896 if (object == NULL) 897 return; 898 if (pager == NULL) 899 return; 900 901 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 902 entry = (vm_object_hash_entry_t) 903 malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK); 904 entry->object = object; 905 object->flags |= OBJ_CANPERSIST; 906 907 vm_object_cache_lock(); 908 TAILQ_INSERT_TAIL(bucket, entry, hash_links); 909 vm_object_cache_unlock(); 910 } 911 912 /* 913 * vm_object_remove: 914 * 915 * Remove the pager from the hash table. 916 * Note: This assumes that the object cache 917 * is locked. XXX this should be fixed 918 * by reorganizing vm_object_deallocate. 919 */ 920 void 921 vm_object_remove(pager) 922 register vm_pager_t pager; 923 { 924 struct vm_object_hash_head *bucket; 925 register vm_object_hash_entry_t entry; 926 register vm_object_t object; 927 928 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 929 930 for (entry = bucket->tqh_first; 931 entry != NULL; 932 entry = entry->hash_links.tqe_next) { 933 object = entry->object; 934 if (object->pager == pager) { 935 TAILQ_REMOVE(bucket, entry, hash_links); 936 free((caddr_t)entry, M_VMOBJHASH); 937 break; 938 } 939 } 940 } 941 942 /* 943 * vm_object_cache_clear removes all objects from the cache. 944 * 945 */ 946 947 void vm_object_cache_clear() 948 { 949 register vm_object_t object; 950 951 /* 952 * Remove each object in the cache by scanning down the 953 * list of cached objects. 954 */ 955 vm_object_cache_lock(); 956 while ((object = vm_object_cached_list.tqh_first) != NULL) { 957 vm_object_cache_unlock(); 958 959 /* 960 * Note: it is important that we use vm_object_lookup 961 * to gain a reference, and not vm_object_reference, because 962 * the logic for removing an object from the cache lies in 963 * lookup. 964 */ 965 if (object != vm_object_lookup(object->pager)) 966 panic("vm_object_cache_clear: I'm sooo confused."); 967 pager_cache(object, FALSE); 968 969 vm_object_cache_lock(); 970 } 971 vm_object_cache_unlock(); 972 } 973 974 boolean_t vm_object_collapse_allowed = TRUE; 975 /* 976 * vm_object_collapse: 977 * 978 * Collapse an object with the object backing it. 979 * Pages in the backing object are moved into the 980 * parent, and the backing object is deallocated. 981 * 982 * Requires that the object be locked and the page 983 * queues be unlocked. 984 * 985 */ 986 void vm_object_collapse(object) 987 register vm_object_t object; 988 989 { 990 register vm_object_t backing_object; 991 register vm_offset_t backing_offset; 992 register vm_size_t size; 993 register vm_offset_t new_offset; 994 register vm_page_t p, pp; 995 996 if (!vm_object_collapse_allowed) 997 return; 998 999 while (TRUE) { 1000 /* 1001 * Verify that the conditions are right for collapse: 1002 * 1003 * The object exists and no pages in it are currently 1004 * being paged out (or have ever been paged out). 1005 */ 1006 if (object == NULL || 1007 object->paging_in_progress != 0 || 1008 object->pager != NULL) 1009 return; 1010 1011 /* 1012 * There is a backing object, and 1013 */ 1014 1015 if ((backing_object = object->shadow) == NULL) 1016 return; 1017 1018 vm_object_lock(backing_object); 1019 /* 1020 * ... 1021 * The backing object is not read_only, 1022 * and no pages in the backing object are 1023 * currently being paged out. 1024 * The backing object is internal. 1025 */ 1026 1027 if ((backing_object->flags & OBJ_INTERNAL) == 0 || 1028 backing_object->paging_in_progress != 0) { 1029 vm_object_unlock(backing_object); 1030 return; 1031 } 1032 1033 /* 1034 * The backing object can't be a copy-object: 1035 * the shadow_offset for the copy-object must stay 1036 * as 0. Furthermore (for the 'we have all the 1037 * pages' case), if we bypass backing_object and 1038 * just shadow the next object in the chain, old 1039 * pages from that object would then have to be copied 1040 * BOTH into the (former) backing_object and into the 1041 * parent object. 1042 */ 1043 if (backing_object->shadow != NULL && 1044 backing_object->shadow->copy != NULL) { 1045 vm_object_unlock(backing_object); 1046 return; 1047 } 1048 1049 /* 1050 * We know that we can either collapse the backing 1051 * object (if the parent is the only reference to 1052 * it) or (perhaps) remove the parent's reference 1053 * to it. 1054 */ 1055 1056 backing_offset = object->shadow_offset; 1057 size = object->size; 1058 1059 /* 1060 * If there is exactly one reference to the backing 1061 * object, we can collapse it into the parent. 1062 */ 1063 1064 if (backing_object->ref_count == 1) { 1065 1066 /* 1067 * We can collapse the backing object. 1068 * 1069 * Move all in-memory pages from backing_object 1070 * to the parent. Pages that have been paged out 1071 * will be overwritten by any of the parent's 1072 * pages that shadow them. 1073 */ 1074 1075 while ((p = backing_object->memq.tqh_first) != NULL) { 1076 new_offset = (p->offset - backing_offset); 1077 1078 /* 1079 * If the parent has a page here, or if 1080 * this page falls outside the parent, 1081 * dispose of it. 1082 * 1083 * Otherwise, move it as planned. 1084 */ 1085 1086 if (p->offset < backing_offset || 1087 new_offset >= size) { 1088 vm_page_lock_queues(); 1089 vm_page_free(p); 1090 vm_page_unlock_queues(); 1091 } else { 1092 pp = vm_page_lookup(object, new_offset); 1093 if (pp != NULL && !(pp->flags & PG_FAKE)) { 1094 vm_page_lock_queues(); 1095 vm_page_free(p); 1096 vm_page_unlock_queues(); 1097 } 1098 else { 1099 if (pp) { 1100 /* may be someone waiting for it */ 1101 PAGE_WAKEUP(pp); 1102 vm_page_lock_queues(); 1103 vm_page_free(pp); 1104 vm_page_unlock_queues(); 1105 } 1106 vm_page_rename(p, object, new_offset); 1107 } 1108 } 1109 } 1110 1111 /* 1112 * Move the pager from backing_object to object. 1113 * 1114 * XXX We're only using part of the paging space 1115 * for keeps now... we ought to discard the 1116 * unused portion. 1117 */ 1118 1119 if (backing_object->pager) { 1120 object->pager = backing_object->pager; 1121 object->paging_offset = backing_offset + 1122 backing_object->paging_offset; 1123 backing_object->pager = NULL; 1124 } 1125 1126 /* 1127 * Object now shadows whatever backing_object did. 1128 * Note that the reference to backing_object->shadow 1129 * moves from within backing_object to within object. 1130 */ 1131 1132 object->shadow = backing_object->shadow; 1133 object->shadow_offset += backing_object->shadow_offset; 1134 if (object->shadow != NULL && 1135 object->shadow->copy != NULL) { 1136 panic("vm_object_collapse: we collapsed a copy-object!"); 1137 } 1138 /* 1139 * Discard backing_object. 1140 * 1141 * Since the backing object has no pages, no 1142 * pager left, and no object references within it, 1143 * all that is necessary is to dispose of it. 1144 */ 1145 1146 vm_object_unlock(backing_object); 1147 1148 simple_lock(&vm_object_list_lock); 1149 TAILQ_REMOVE(&vm_object_list, backing_object, 1150 object_list); 1151 vm_object_count--; 1152 simple_unlock(&vm_object_list_lock); 1153 1154 free((caddr_t)backing_object, M_VMOBJ); 1155 1156 object_collapses++; 1157 } 1158 else { 1159 /* 1160 * If all of the pages in the backing object are 1161 * shadowed by the parent object, the parent 1162 * object no longer has to shadow the backing 1163 * object; it can shadow the next one in the 1164 * chain. 1165 * 1166 * The backing object must not be paged out - we'd 1167 * have to check all of the paged-out pages, as 1168 * well. 1169 */ 1170 1171 if (backing_object->pager != NULL) { 1172 vm_object_unlock(backing_object); 1173 return; 1174 } 1175 1176 /* 1177 * Should have a check for a 'small' number 1178 * of pages here. 1179 */ 1180 1181 for (p = backing_object->memq.tqh_first; 1182 p != NULL; 1183 p = p->listq.tqe_next) { 1184 new_offset = (p->offset - backing_offset); 1185 1186 /* 1187 * If the parent has a page here, or if 1188 * this page falls outside the parent, 1189 * keep going. 1190 * 1191 * Otherwise, the backing_object must be 1192 * left in the chain. 1193 */ 1194 1195 if (p->offset >= backing_offset && 1196 new_offset < size && 1197 ((pp = vm_page_lookup(object, new_offset)) 1198 == NULL || 1199 (pp->flags & PG_FAKE))) { 1200 /* 1201 * Page still needed. 1202 * Can't go any further. 1203 */ 1204 vm_object_unlock(backing_object); 1205 return; 1206 } 1207 } 1208 1209 /* 1210 * Make the parent shadow the next object 1211 * in the chain. Deallocating backing_object 1212 * will not remove it, since its reference 1213 * count is at least 2. 1214 */ 1215 1216 object->shadow = backing_object->shadow; 1217 vm_object_reference(object->shadow); 1218 object->shadow_offset += backing_object->shadow_offset; 1219 1220 /* 1221 * Backing object might have had a copy pointer 1222 * to us. If it did, clear it. 1223 */ 1224 if (backing_object->copy == object) { 1225 backing_object->copy = NULL; 1226 } 1227 1228 /* Drop the reference count on backing_object. 1229 * Since its ref_count was at least 2, it 1230 * will not vanish; so we don't need to call 1231 * vm_object_deallocate. 1232 */ 1233 backing_object->ref_count--; 1234 vm_object_unlock(backing_object); 1235 1236 object_bypasses ++; 1237 1238 } 1239 1240 /* 1241 * Try again with this object's new backing object. 1242 */ 1243 } 1244 } 1245 1246 /* 1247 * vm_object_page_remove: [internal] 1248 * 1249 * Removes all physical pages in the specified 1250 * object range from the object's list of pages. 1251 * 1252 * The object must be locked. 1253 */ 1254 void vm_object_page_remove(object, start, end) 1255 register vm_object_t object; 1256 register vm_offset_t start; 1257 register vm_offset_t end; 1258 { 1259 register vm_page_t p, next; 1260 1261 if (object == NULL) 1262 return; 1263 1264 for (p = object->memq.tqh_first; p != NULL; p = next) { 1265 next = p->listq.tqe_next; 1266 if ((start <= p->offset) && (p->offset < end)) { 1267 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1268 vm_page_lock_queues(); 1269 vm_page_free(p); 1270 vm_page_unlock_queues(); 1271 } 1272 } 1273 } 1274 1275 /* 1276 * Routine: vm_object_coalesce 1277 * Function: Coalesces two objects backing up adjoining 1278 * regions of memory into a single object. 1279 * 1280 * returns TRUE if objects were combined. 1281 * 1282 * NOTE: Only works at the moment if the second object is NULL - 1283 * if it's not, which object do we lock first? 1284 * 1285 * Parameters: 1286 * prev_object First object to coalesce 1287 * prev_offset Offset into prev_object 1288 * next_object Second object into coalesce 1289 * next_offset Offset into next_object 1290 * 1291 * prev_size Size of reference to prev_object 1292 * next_size Size of reference to next_object 1293 * 1294 * Conditions: 1295 * The object must *not* be locked. 1296 */ 1297 boolean_t vm_object_coalesce(prev_object, next_object, 1298 prev_offset, next_offset, 1299 prev_size, next_size) 1300 1301 register vm_object_t prev_object; 1302 vm_object_t next_object; 1303 vm_offset_t prev_offset, next_offset; 1304 vm_size_t prev_size, next_size; 1305 { 1306 vm_size_t newsize; 1307 1308 #ifdef lint 1309 next_offset++; 1310 #endif 1311 1312 if (next_object != NULL) { 1313 return(FALSE); 1314 } 1315 1316 if (prev_object == NULL) { 1317 return(TRUE); 1318 } 1319 1320 vm_object_lock(prev_object); 1321 1322 /* 1323 * Try to collapse the object first 1324 */ 1325 vm_object_collapse(prev_object); 1326 1327 /* 1328 * Can't coalesce if: 1329 * . more than one reference 1330 * . paged out 1331 * . shadows another object 1332 * . has a copy elsewhere 1333 * (any of which mean that the pages not mapped to 1334 * prev_entry may be in use anyway) 1335 */ 1336 1337 if (prev_object->ref_count > 1 || 1338 prev_object->pager != NULL || 1339 prev_object->shadow != NULL || 1340 prev_object->copy != NULL) { 1341 vm_object_unlock(prev_object); 1342 return(FALSE); 1343 } 1344 1345 /* 1346 * Remove any pages that may still be in the object from 1347 * a previous deallocation. 1348 */ 1349 1350 vm_object_page_remove(prev_object, 1351 prev_offset + prev_size, 1352 prev_offset + prev_size + next_size); 1353 1354 /* 1355 * Extend the object if necessary. 1356 */ 1357 newsize = prev_offset + prev_size + next_size; 1358 if (newsize > prev_object->size) 1359 prev_object->size = newsize; 1360 1361 vm_object_unlock(prev_object); 1362 return(TRUE); 1363 } 1364 1365 /* 1366 * vm_object_print: [ debug ] 1367 */ 1368 void vm_object_print(object, full) 1369 vm_object_t object; 1370 boolean_t full; 1371 { 1372 register vm_page_t p; 1373 extern indent; 1374 1375 register int count; 1376 1377 if (object == NULL) 1378 return; 1379 1380 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1381 (int) object, (int) object->size, 1382 object->resident_page_count, object->ref_count); 1383 printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1384 (int) object->pager, (int) object->paging_offset, 1385 (int) object->shadow, (int) object->shadow_offset); 1386 printf("cache: next=0x%x, prev=0x%x\n", 1387 object->cached_list.tqe_next, object->cached_list.tqe_prev); 1388 1389 if (!full) 1390 return; 1391 1392 indent += 2; 1393 count = 0; 1394 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 1395 if (count == 0) 1396 iprintf("memory:="); 1397 else if (count == 6) { 1398 printf("\n"); 1399 iprintf(" ..."); 1400 count = 0; 1401 } else 1402 printf(","); 1403 count++; 1404 1405 printf("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p)); 1406 } 1407 if (count != 0) 1408 printf("\n"); 1409 indent -= 2; 1410 } 1411