1 /* 2 * Copyright (c) 1991, 1993, 2013 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 33 * 34 * 35 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 36 * All rights reserved. 37 * 38 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 39 * 40 * Permission to use, copy, modify and distribute this software and 41 * its documentation is hereby granted, provided that both the copyright 42 * notice and this permission notice appear in all copies of the 43 * software, derivative works or modified versions, and any portions 44 * thereof, and that both notices appear in supporting documentation. 45 * 46 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 47 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 48 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 49 * 50 * Carnegie Mellon requests users of this software to return to 51 * 52 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 53 * School of Computer Science 54 * Carnegie Mellon University 55 * Pittsburgh PA 15213-3890 56 * 57 * any improvements or extensions that they make and grant Carnegie the 58 * rights to redistribute these changes. 59 * 60 * $FreeBSD: src/sys/vm/vm_object.c,v 1.171.2.8 2003/05/26 19:17:56 alc Exp $ 61 */ 62 63 /* 64 * Virtual memory object module. 65 */ 66 67 #include <sys/param.h> 68 #include <sys/systm.h> 69 #include <sys/proc.h> /* for curproc, pageproc */ 70 #include <sys/thread.h> 71 #include <sys/vnode.h> 72 #include <sys/vmmeter.h> 73 #include <sys/mman.h> 74 #include <sys/mount.h> 75 #include <sys/kernel.h> 76 #include <sys/sysctl.h> 77 #include <sys/refcount.h> 78 79 #include <vm/vm.h> 80 #include <vm/vm_param.h> 81 #include <vm/pmap.h> 82 #include <vm/vm_map.h> 83 #include <vm/vm_object.h> 84 #include <vm/vm_page.h> 85 #include <vm/vm_pageout.h> 86 #include <vm/vm_pager.h> 87 #include <vm/swap_pager.h> 88 #include <vm/vm_kern.h> 89 #include <vm/vm_extern.h> 90 #include <vm/vm_zone.h> 91 92 #include <vm/vm_page2.h> 93 94 #include <machine/specialreg.h> 95 96 #define EASY_SCAN_FACTOR 8 97 98 static void vm_object_qcollapse(vm_object_t object, 99 vm_object_t backing_object); 100 static void vm_object_page_collect_flush(vm_object_t object, vm_page_t p, 101 int pagerflags); 102 static void vm_object_lock_init(vm_object_t); 103 104 105 /* 106 * Virtual memory objects maintain the actual data 107 * associated with allocated virtual memory. A given 108 * page of memory exists within exactly one object. 109 * 110 * An object is only deallocated when all "references" 111 * are given up. Only one "reference" to a given 112 * region of an object should be writeable. 113 * 114 * Associated with each object is a list of all resident 115 * memory pages belonging to that object; this list is 116 * maintained by the "vm_page" module, and locked by the object's 117 * lock. 118 * 119 * Each object also records a "pager" routine which is 120 * used to retrieve (and store) pages to the proper backing 121 * storage. In addition, objects may be backed by other 122 * objects from which they were virtual-copied. 123 * 124 * The only items within the object structure which are 125 * modified after time of creation are: 126 * reference count locked by object's lock 127 * pager routine locked by object's lock 128 * 129 */ 130 131 struct vm_object kernel_object; 132 133 static long vm_object_count; 134 135 static long object_collapses; 136 static long object_bypasses; 137 static int next_index; 138 static vm_zone_t obj_zone; 139 static struct vm_zone obj_zone_store; 140 #define VM_OBJECTS_INIT 256 141 static struct vm_object vm_objects_init[VM_OBJECTS_INIT]; 142 143 struct object_q vm_object_lists[VMOBJ_HSIZE]; 144 struct lwkt_token vmobj_tokens[VMOBJ_HSIZE]; 145 146 /* 147 * Misc low level routines 148 */ 149 static void 150 vm_object_lock_init(vm_object_t obj) 151 { 152 #if defined(DEBUG_LOCKS) 153 int i; 154 155 obj->debug_hold_bitmap = 0; 156 obj->debug_hold_ovfl = 0; 157 for (i = 0; i < VMOBJ_DEBUG_ARRAY_SIZE; i++) { 158 obj->debug_hold_thrs[i] = NULL; 159 obj->debug_hold_file[i] = NULL; 160 obj->debug_hold_line[i] = 0; 161 } 162 #endif 163 } 164 165 void 166 vm_object_lock_swap(void) 167 { 168 lwkt_token_swap(); 169 } 170 171 void 172 vm_object_lock(vm_object_t obj) 173 { 174 lwkt_gettoken(&obj->token); 175 } 176 177 /* 178 * Returns TRUE on sucesss 179 */ 180 static int 181 vm_object_lock_try(vm_object_t obj) 182 { 183 return(lwkt_trytoken(&obj->token)); 184 } 185 186 void 187 vm_object_lock_shared(vm_object_t obj) 188 { 189 lwkt_gettoken_shared(&obj->token); 190 } 191 192 void 193 vm_object_unlock(vm_object_t obj) 194 { 195 lwkt_reltoken(&obj->token); 196 } 197 198 void 199 vm_object_upgrade(vm_object_t obj) 200 { 201 lwkt_reltoken(&obj->token); 202 lwkt_gettoken(&obj->token); 203 } 204 205 void 206 vm_object_downgrade(vm_object_t obj) 207 { 208 lwkt_reltoken(&obj->token); 209 lwkt_gettoken_shared(&obj->token); 210 } 211 212 static __inline void 213 vm_object_assert_held(vm_object_t obj) 214 { 215 ASSERT_LWKT_TOKEN_HELD(&obj->token); 216 } 217 218 void 219 #ifndef DEBUG_LOCKS 220 vm_object_hold(vm_object_t obj) 221 #else 222 debugvm_object_hold(vm_object_t obj, char *file, int line) 223 #endif 224 { 225 KKASSERT(obj != NULL); 226 227 /* 228 * Object must be held (object allocation is stable due to callers 229 * context, typically already holding the token on a parent object) 230 * prior to potentially blocking on the lock, otherwise the object 231 * can get ripped away from us. 232 */ 233 refcount_acquire(&obj->hold_count); 234 vm_object_lock(obj); 235 236 #if defined(DEBUG_LOCKS) 237 int i; 238 u_int mask; 239 240 for (;;) { 241 mask = ~obj->debug_hold_bitmap; 242 cpu_ccfence(); 243 if (mask == 0xFFFFFFFFU) { 244 if (obj->debug_hold_ovfl == 0) 245 obj->debug_hold_ovfl = 1; 246 break; 247 } 248 i = ffs(mask) - 1; 249 if (atomic_cmpset_int(&obj->debug_hold_bitmap, ~mask, 250 ~mask | (1 << i))) { 251 obj->debug_hold_bitmap |= (1 << i); 252 obj->debug_hold_thrs[i] = curthread; 253 obj->debug_hold_file[i] = file; 254 obj->debug_hold_line[i] = line; 255 break; 256 } 257 } 258 #endif 259 } 260 261 int 262 #ifndef DEBUG_LOCKS 263 vm_object_hold_try(vm_object_t obj) 264 #else 265 debugvm_object_hold_try(vm_object_t obj, char *file, int line) 266 #endif 267 { 268 KKASSERT(obj != NULL); 269 270 /* 271 * Object must be held (object allocation is stable due to callers 272 * context, typically already holding the token on a parent object) 273 * prior to potentially blocking on the lock, otherwise the object 274 * can get ripped away from us. 275 */ 276 refcount_acquire(&obj->hold_count); 277 if (vm_object_lock_try(obj) == 0) { 278 if (refcount_release(&obj->hold_count)) { 279 if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD)) 280 zfree(obj_zone, obj); 281 } 282 return(0); 283 } 284 285 #if defined(DEBUG_LOCKS) 286 int i; 287 u_int mask; 288 289 for (;;) { 290 mask = ~obj->debug_hold_bitmap; 291 cpu_ccfence(); 292 if (mask == 0xFFFFFFFFU) { 293 if (obj->debug_hold_ovfl == 0) 294 obj->debug_hold_ovfl = 1; 295 break; 296 } 297 i = ffs(mask) - 1; 298 if (atomic_cmpset_int(&obj->debug_hold_bitmap, ~mask, 299 ~mask | (1 << i))) { 300 obj->debug_hold_bitmap |= (1 << i); 301 obj->debug_hold_thrs[i] = curthread; 302 obj->debug_hold_file[i] = file; 303 obj->debug_hold_line[i] = line; 304 break; 305 } 306 } 307 #endif 308 return(1); 309 } 310 311 void 312 #ifndef DEBUG_LOCKS 313 vm_object_hold_shared(vm_object_t obj) 314 #else 315 debugvm_object_hold_shared(vm_object_t obj, char *file, int line) 316 #endif 317 { 318 KKASSERT(obj != NULL); 319 320 /* 321 * Object must be held (object allocation is stable due to callers 322 * context, typically already holding the token on a parent object) 323 * prior to potentially blocking on the lock, otherwise the object 324 * can get ripped away from us. 325 */ 326 refcount_acquire(&obj->hold_count); 327 vm_object_lock_shared(obj); 328 329 #if defined(DEBUG_LOCKS) 330 int i; 331 u_int mask; 332 333 for (;;) { 334 mask = ~obj->debug_hold_bitmap; 335 cpu_ccfence(); 336 if (mask == 0xFFFFFFFFU) { 337 if (obj->debug_hold_ovfl == 0) 338 obj->debug_hold_ovfl = 1; 339 break; 340 } 341 i = ffs(mask) - 1; 342 if (atomic_cmpset_int(&obj->debug_hold_bitmap, ~mask, 343 ~mask | (1 << i))) { 344 obj->debug_hold_bitmap |= (1 << i); 345 obj->debug_hold_thrs[i] = curthread; 346 obj->debug_hold_file[i] = file; 347 obj->debug_hold_line[i] = line; 348 break; 349 } 350 } 351 #endif 352 } 353 354 /* 355 * Drop the token and hold_count on the object. 356 * 357 * WARNING! Token might be shared. 358 */ 359 void 360 vm_object_drop(vm_object_t obj) 361 { 362 if (obj == NULL) 363 return; 364 365 #if defined(DEBUG_LOCKS) 366 int found = 0; 367 int i; 368 369 for (i = 0; i < VMOBJ_DEBUG_ARRAY_SIZE; i++) { 370 if ((obj->debug_hold_bitmap & (1 << i)) && 371 (obj->debug_hold_thrs[i] == curthread)) { 372 obj->debug_hold_bitmap &= ~(1 << i); 373 obj->debug_hold_thrs[i] = NULL; 374 obj->debug_hold_file[i] = NULL; 375 obj->debug_hold_line[i] = 0; 376 found = 1; 377 break; 378 } 379 } 380 381 if (found == 0 && obj->debug_hold_ovfl == 0) 382 panic("vm_object: attempt to drop hold on non-self-held obj"); 383 #endif 384 385 /* 386 * No new holders should be possible once we drop hold_count 1->0 as 387 * there is no longer any way to reference the object. 388 */ 389 KKASSERT(obj->hold_count > 0); 390 if (refcount_release(&obj->hold_count)) { 391 if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD)) { 392 vm_object_unlock(obj); 393 zfree(obj_zone, obj); 394 } else { 395 vm_object_unlock(obj); 396 } 397 } else { 398 vm_object_unlock(obj); 399 } 400 } 401 402 /* 403 * Initialize a freshly allocated object, returning a held object. 404 * 405 * Used only by vm_object_allocate() and zinitna(). 406 * 407 * No requirements. 408 */ 409 void 410 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object) 411 { 412 int incr; 413 int n; 414 415 RB_INIT(&object->rb_memq); 416 LIST_INIT(&object->shadow_head); 417 lwkt_token_init(&object->token, "vmobj"); 418 419 object->type = type; 420 object->size = size; 421 object->ref_count = 1; 422 object->memattr = VM_MEMATTR_DEFAULT; 423 object->hold_count = 0; 424 object->flags = 0; 425 if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP)) 426 vm_object_set_flag(object, OBJ_ONEMAPPING); 427 object->paging_in_progress = 0; 428 object->resident_page_count = 0; 429 object->agg_pv_list_count = 0; 430 object->shadow_count = 0; 431 /* cpu localization twist */ 432 object->pg_color = (int)(intptr_t)curthread; 433 if ( size > (PQ_L2_SIZE / 3 + PQ_PRIME1)) 434 incr = PQ_L2_SIZE / 3 + PQ_PRIME1; 435 else 436 incr = size; 437 next_index = (next_index + incr) & PQ_L2_MASK; 438 object->handle = NULL; 439 object->backing_object = NULL; 440 object->backing_object_offset = (vm_ooffset_t)0; 441 442 object->generation++; 443 object->swblock_count = 0; 444 RB_INIT(&object->swblock_root); 445 vm_object_lock_init(object); 446 pmap_object_init(object); 447 448 vm_object_hold(object); 449 450 n = VMOBJ_HASH(object); 451 atomic_add_long(&vm_object_count, 1); 452 lwkt_gettoken(&vmobj_tokens[n]); 453 TAILQ_INSERT_TAIL(&vm_object_lists[n], object, object_list); 454 lwkt_reltoken(&vmobj_tokens[n]); 455 } 456 457 /* 458 * Initialize the VM objects module. 459 * 460 * Called from the low level boot code only. 461 */ 462 void 463 vm_object_init(void) 464 { 465 int i; 466 467 for (i = 0; i < VMOBJ_HSIZE; ++i) { 468 TAILQ_INIT(&vm_object_lists[i]); 469 lwkt_token_init(&vmobj_tokens[i], "vmobjlst"); 470 } 471 472 _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(KvaEnd), 473 &kernel_object); 474 vm_object_drop(&kernel_object); 475 476 obj_zone = &obj_zone_store; 477 zbootinit(obj_zone, "VM OBJECT", sizeof (struct vm_object), 478 vm_objects_init, VM_OBJECTS_INIT); 479 } 480 481 void 482 vm_object_init2(void) 483 { 484 zinitna(obj_zone, NULL, NULL, 0, 0, ZONE_PANICFAIL, 1); 485 } 486 487 /* 488 * Allocate and return a new object of the specified type and size. 489 * 490 * No requirements. 491 */ 492 vm_object_t 493 vm_object_allocate(objtype_t type, vm_pindex_t size) 494 { 495 vm_object_t result; 496 497 result = (vm_object_t) zalloc(obj_zone); 498 499 _vm_object_allocate(type, size, result); 500 vm_object_drop(result); 501 502 return (result); 503 } 504 505 /* 506 * This version returns a held object, allowing further atomic initialization 507 * of the object. 508 */ 509 vm_object_t 510 vm_object_allocate_hold(objtype_t type, vm_pindex_t size) 511 { 512 vm_object_t result; 513 514 result = (vm_object_t) zalloc(obj_zone); 515 516 _vm_object_allocate(type, size, result); 517 518 return (result); 519 } 520 521 /* 522 * Add an additional reference to a vm_object. The object must already be 523 * held. The original non-lock version is no longer supported. The object 524 * must NOT be chain locked by anyone at the time the reference is added. 525 * 526 * Referencing a chain-locked object can blow up the fairly sensitive 527 * ref_count and shadow_count tests in the deallocator. Most callers 528 * will call vm_object_chain_wait() prior to calling 529 * vm_object_reference_locked() to avoid the case. 530 * 531 * The object must be held, but may be held shared if desired (hence why 532 * we use an atomic op). 533 */ 534 void 535 vm_object_reference_locked(vm_object_t object) 536 { 537 KKASSERT(object != NULL); 538 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 539 KKASSERT((object->chainlk & (CHAINLK_EXCL | CHAINLK_MASK)) == 0); 540 atomic_add_int(&object->ref_count, 1); 541 if (object->type == OBJT_VNODE) { 542 vref(object->handle); 543 /* XXX what if the vnode is being destroyed? */ 544 } 545 } 546 547 /* 548 * This version is only allowed for vnode objects. 549 */ 550 void 551 vm_object_reference_quick(vm_object_t object) 552 { 553 KKASSERT(object->type == OBJT_VNODE); 554 atomic_add_int(&object->ref_count, 1); 555 vref(object->handle); 556 } 557 558 /* 559 * Object OBJ_CHAINLOCK lock handling. 560 * 561 * The caller can chain-lock backing objects recursively and then 562 * use vm_object_chain_release_all() to undo the whole chain. 563 * 564 * Chain locks are used to prevent collapses and are only applicable 565 * to OBJT_DEFAULT and OBJT_SWAP objects. Chain locking operations 566 * on other object types are ignored. This is also important because 567 * it allows e.g. the vnode underlying a memory mapping to take concurrent 568 * faults. 569 * 570 * The object must usually be held on entry, though intermediate 571 * objects need not be held on release. The object must be held exclusively, 572 * NOT shared. Note that the prefault path checks the shared state and 573 * avoids using the chain functions. 574 */ 575 void 576 vm_object_chain_wait(vm_object_t object, int shared) 577 { 578 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 579 for (;;) { 580 uint32_t chainlk = object->chainlk; 581 582 cpu_ccfence(); 583 if (shared) { 584 if (chainlk & (CHAINLK_EXCL | CHAINLK_EXCLREQ)) { 585 tsleep_interlock(object, 0); 586 if (atomic_cmpset_int(&object->chainlk, 587 chainlk, 588 chainlk | CHAINLK_WAIT)) { 589 tsleep(object, PINTERLOCKED, 590 "objchns", 0); 591 } 592 /* retry */ 593 } else { 594 break; 595 } 596 /* retry */ 597 } else { 598 if (chainlk & (CHAINLK_MASK | CHAINLK_EXCL)) { 599 tsleep_interlock(object, 0); 600 if (atomic_cmpset_int(&object->chainlk, 601 chainlk, 602 chainlk | CHAINLK_WAIT)) 603 { 604 tsleep(object, PINTERLOCKED, 605 "objchnx", 0); 606 } 607 /* retry */ 608 } else { 609 if (atomic_cmpset_int(&object->chainlk, 610 chainlk, 611 chainlk & ~CHAINLK_WAIT)) 612 { 613 if (chainlk & CHAINLK_WAIT) 614 wakeup(object); 615 break; 616 } 617 /* retry */ 618 } 619 } 620 /* retry */ 621 } 622 } 623 624 void 625 vm_object_chain_acquire(vm_object_t object, int shared) 626 { 627 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP) 628 return; 629 if (vm_shared_fault == 0) 630 shared = 0; 631 632 for (;;) { 633 uint32_t chainlk = object->chainlk; 634 635 cpu_ccfence(); 636 if (shared) { 637 if (chainlk & (CHAINLK_EXCL | CHAINLK_EXCLREQ)) { 638 tsleep_interlock(object, 0); 639 if (atomic_cmpset_int(&object->chainlk, 640 chainlk, 641 chainlk | CHAINLK_WAIT)) { 642 tsleep(object, PINTERLOCKED, 643 "objchns", 0); 644 } 645 /* retry */ 646 } else if (atomic_cmpset_int(&object->chainlk, 647 chainlk, chainlk + 1)) { 648 break; 649 } 650 /* retry */ 651 } else { 652 if (chainlk & (CHAINLK_MASK | CHAINLK_EXCL)) { 653 tsleep_interlock(object, 0); 654 if (atomic_cmpset_int(&object->chainlk, 655 chainlk, 656 chainlk | 657 CHAINLK_WAIT | 658 CHAINLK_EXCLREQ)) { 659 tsleep(object, PINTERLOCKED, 660 "objchnx", 0); 661 } 662 /* retry */ 663 } else { 664 if (atomic_cmpset_int(&object->chainlk, 665 chainlk, 666 (chainlk | CHAINLK_EXCL) & 667 ~(CHAINLK_EXCLREQ | 668 CHAINLK_WAIT))) { 669 if (chainlk & CHAINLK_WAIT) 670 wakeup(object); 671 break; 672 } 673 /* retry */ 674 } 675 } 676 /* retry */ 677 } 678 } 679 680 void 681 vm_object_chain_release(vm_object_t object) 682 { 683 /*ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));*/ 684 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP) 685 return; 686 KKASSERT(object->chainlk & (CHAINLK_MASK | CHAINLK_EXCL)); 687 for (;;) { 688 uint32_t chainlk = object->chainlk; 689 690 cpu_ccfence(); 691 if (chainlk & CHAINLK_MASK) { 692 if ((chainlk & CHAINLK_MASK) == 1 && 693 atomic_cmpset_int(&object->chainlk, 694 chainlk, 695 (chainlk - 1) & ~CHAINLK_WAIT)) { 696 if (chainlk & CHAINLK_WAIT) 697 wakeup(object); 698 break; 699 } 700 if ((chainlk & CHAINLK_MASK) > 1 && 701 atomic_cmpset_int(&object->chainlk, 702 chainlk, chainlk - 1)) { 703 break; 704 } 705 /* retry */ 706 } else { 707 KKASSERT(chainlk & CHAINLK_EXCL); 708 if (atomic_cmpset_int(&object->chainlk, 709 chainlk, 710 chainlk & ~(CHAINLK_EXCL | 711 CHAINLK_WAIT))) { 712 if (chainlk & CHAINLK_WAIT) 713 wakeup(object); 714 break; 715 } 716 } 717 } 718 } 719 720 /* 721 * Release the chain from first_object through and including stopobj. 722 * The caller is typically holding the first and last object locked 723 * (shared or exclusive) to prevent destruction races. 724 * 725 * We release stopobj first as an optimization as this object is most 726 * likely to be shared across multiple processes. 727 */ 728 void 729 vm_object_chain_release_all(vm_object_t first_object, vm_object_t stopobj) 730 { 731 vm_object_t backing_object; 732 vm_object_t object; 733 734 vm_object_chain_release(stopobj); 735 object = first_object; 736 737 while (object != stopobj) { 738 KKASSERT(object); 739 backing_object = object->backing_object; 740 vm_object_chain_release(object); 741 object = backing_object; 742 } 743 } 744 745 /* 746 * Dereference an object and its underlying vnode. The object may be 747 * held shared. On return the object will remain held. 748 * 749 * This function may return a vnode in *vpp which the caller must release 750 * after the caller drops its own lock. If vpp is NULL, we assume that 751 * the caller was holding an exclusive lock on the object and we vrele() 752 * the vp ourselves. 753 */ 754 static void 755 vm_object_vndeallocate(vm_object_t object, struct vnode **vpp) 756 { 757 struct vnode *vp = (struct vnode *) object->handle; 758 759 KASSERT(object->type == OBJT_VNODE, 760 ("vm_object_vndeallocate: not a vnode object")); 761 KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); 762 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 763 #ifdef INVARIANTS 764 if (object->ref_count == 0) { 765 vprint("vm_object_vndeallocate", vp); 766 panic("vm_object_vndeallocate: bad object reference count"); 767 } 768 #endif 769 for (;;) { 770 int count = object->ref_count; 771 cpu_ccfence(); 772 if (count == 1) { 773 vm_object_upgrade(object); 774 if (atomic_cmpset_int(&object->ref_count, count, 0)) { 775 vclrflags(vp, VTEXT); 776 break; 777 } 778 } else { 779 if (atomic_cmpset_int(&object->ref_count, 780 count, count - 1)) { 781 break; 782 } 783 } 784 /* retry */ 785 } 786 787 /* 788 * vrele or return the vp to vrele. We can only safely vrele(vp) 789 * if the object was locked exclusively. But there are two races 790 * here. 791 * 792 * We had to upgrade the object above to safely clear VTEXT 793 * but the alternative path where the shared lock is retained 794 * can STILL race to 0 in other paths and cause our own vrele() 795 * to terminate the vnode. We can't allow that if the VM object 796 * is still locked shared. 797 */ 798 if (vpp) 799 *vpp = vp; 800 else 801 vrele(vp); 802 } 803 804 /* 805 * Release a reference to the specified object, gained either through a 806 * vm_object_allocate or a vm_object_reference call. When all references 807 * are gone, storage associated with this object may be relinquished. 808 * 809 * The caller does not have to hold the object locked but must have control 810 * over the reference in question in order to guarantee that the object 811 * does not get ripped out from under us. 812 * 813 * XXX Currently all deallocations require an exclusive lock. 814 */ 815 void 816 vm_object_deallocate(vm_object_t object) 817 { 818 struct vnode *vp; 819 int count; 820 821 if (object == NULL) 822 return; 823 for (;;) { 824 count = object->ref_count; 825 cpu_ccfence(); 826 827 /* 828 * If decrementing the count enters into special handling 829 * territory (0, 1, or 2) we have to do it the hard way. 830 * Fortunate though, objects with only a few refs like this 831 * are not likely to be heavily contended anyway. 832 * 833 * For vnode objects we only care about 1->0 transitions. 834 */ 835 if (count <= 3 || (object->type == OBJT_VNODE && count <= 1)) { 836 vm_object_hold(object); 837 vm_object_deallocate_locked(object); 838 vm_object_drop(object); 839 break; 840 } 841 842 /* 843 * Try to decrement ref_count without acquiring a hold on 844 * the object. This is particularly important for the exec*() 845 * and exit*() code paths because the program binary may 846 * have a great deal of sharing and an exclusive lock will 847 * crowbar performance in those circumstances. 848 */ 849 if (object->type == OBJT_VNODE) { 850 vp = (struct vnode *)object->handle; 851 if (atomic_cmpset_int(&object->ref_count, 852 count, count - 1)) { 853 vrele(vp); 854 break; 855 } 856 /* retry */ 857 } else { 858 if (atomic_cmpset_int(&object->ref_count, 859 count, count - 1)) { 860 break; 861 } 862 /* retry */ 863 } 864 /* retry */ 865 } 866 } 867 868 void 869 vm_object_deallocate_locked(vm_object_t object) 870 { 871 struct vm_object_dealloc_list *dlist = NULL; 872 struct vm_object_dealloc_list *dtmp; 873 vm_object_t temp; 874 int must_drop = 0; 875 876 /* 877 * We may chain deallocate object, but additional objects may 878 * collect on the dlist which also have to be deallocated. We 879 * must avoid a recursion, vm_object chains can get deep. 880 */ 881 882 again: 883 while (object != NULL) { 884 /* 885 * vnode case, caller either locked the object exclusively 886 * or this is a recursion with must_drop != 0 and the vnode 887 * object will be locked shared. 888 * 889 * If locked shared we have to drop the object before we can 890 * call vrele() or risk a shared/exclusive livelock. 891 */ 892 if (object->type == OBJT_VNODE) { 893 ASSERT_LWKT_TOKEN_HELD(&object->token); 894 if (must_drop) { 895 struct vnode *tmp_vp; 896 897 vm_object_vndeallocate(object, &tmp_vp); 898 vm_object_drop(object); 899 must_drop = 0; 900 object = NULL; 901 vrele(tmp_vp); 902 } else { 903 vm_object_vndeallocate(object, NULL); 904 } 905 break; 906 } 907 ASSERT_LWKT_TOKEN_HELD_EXCL(&object->token); 908 909 /* 910 * Normal case (object is locked exclusively) 911 */ 912 if (object->ref_count == 0) { 913 panic("vm_object_deallocate: object deallocated " 914 "too many times: %d", object->type); 915 } 916 if (object->ref_count > 2) { 917 atomic_add_int(&object->ref_count, -1); 918 break; 919 } 920 921 /* 922 * Here on ref_count of one or two, which are special cases for 923 * objects. 924 * 925 * Nominal ref_count > 1 case if the second ref is not from 926 * a shadow. 927 * 928 * (ONEMAPPING only applies to DEFAULT AND SWAP objects) 929 */ 930 if (object->ref_count == 2 && object->shadow_count == 0) { 931 if (object->type == OBJT_DEFAULT || 932 object->type == OBJT_SWAP) { 933 vm_object_set_flag(object, OBJ_ONEMAPPING); 934 } 935 atomic_add_int(&object->ref_count, -1); 936 break; 937 } 938 939 /* 940 * If the second ref is from a shadow we chain along it 941 * upwards if object's handle is exhausted. 942 * 943 * We have to decrement object->ref_count before potentially 944 * collapsing the first shadow object or the collapse code 945 * will not be able to handle the degenerate case to remove 946 * object. However, if we do it too early the object can 947 * get ripped out from under us. 948 */ 949 if (object->ref_count == 2 && object->shadow_count == 1 && 950 object->handle == NULL && (object->type == OBJT_DEFAULT || 951 object->type == OBJT_SWAP)) { 952 temp = LIST_FIRST(&object->shadow_head); 953 KKASSERT(temp != NULL); 954 vm_object_hold(temp); 955 956 /* 957 * Wait for any paging to complete so the collapse 958 * doesn't (or isn't likely to) qcollapse. pip 959 * waiting must occur before we acquire the 960 * chainlock. 961 */ 962 while ( 963 temp->paging_in_progress || 964 object->paging_in_progress 965 ) { 966 vm_object_pip_wait(temp, "objde1"); 967 vm_object_pip_wait(object, "objde2"); 968 } 969 970 /* 971 * If the parent is locked we have to give up, as 972 * otherwise we would be acquiring locks in the 973 * wrong order and potentially deadlock. 974 */ 975 if (temp->chainlk & (CHAINLK_EXCL | CHAINLK_MASK)) { 976 vm_object_drop(temp); 977 goto skip; 978 } 979 vm_object_chain_acquire(temp, 0); 980 981 /* 982 * Recheck/retry after the hold and the paging 983 * wait, both of which can block us. 984 */ 985 if (object->ref_count != 2 || 986 object->shadow_count != 1 || 987 object->handle || 988 LIST_FIRST(&object->shadow_head) != temp || 989 (object->type != OBJT_DEFAULT && 990 object->type != OBJT_SWAP)) { 991 vm_object_chain_release(temp); 992 vm_object_drop(temp); 993 continue; 994 } 995 996 /* 997 * We can safely drop object's ref_count now. 998 */ 999 KKASSERT(object->ref_count == 2); 1000 atomic_add_int(&object->ref_count, -1); 1001 1002 /* 1003 * If our single parent is not collapseable just 1004 * decrement ref_count (2->1) and stop. 1005 */ 1006 if (temp->handle || (temp->type != OBJT_DEFAULT && 1007 temp->type != OBJT_SWAP)) { 1008 vm_object_chain_release(temp); 1009 vm_object_drop(temp); 1010 break; 1011 } 1012 1013 /* 1014 * At this point we have already dropped object's 1015 * ref_count so it is possible for a race to 1016 * deallocate obj out from under us. Any collapse 1017 * will re-check the situation. We must not block 1018 * until we are able to collapse. 1019 * 1020 * Bump temp's ref_count to avoid an unwanted 1021 * degenerate recursion (can't call 1022 * vm_object_reference_locked() because it asserts 1023 * that CHAINLOCK is not set). 1024 */ 1025 atomic_add_int(&temp->ref_count, 1); 1026 KKASSERT(temp->ref_count > 1); 1027 1028 /* 1029 * Collapse temp, then deallocate the extra ref 1030 * formally. 1031 */ 1032 vm_object_collapse(temp, &dlist); 1033 vm_object_chain_release(temp); 1034 if (must_drop) { 1035 vm_object_lock_swap(); 1036 vm_object_drop(object); 1037 } 1038 object = temp; 1039 must_drop = 1; 1040 continue; 1041 } 1042 1043 /* 1044 * Drop the ref and handle termination on the 1->0 transition. 1045 * We may have blocked above so we have to recheck. 1046 */ 1047 skip: 1048 KKASSERT(object->ref_count != 0); 1049 if (object->ref_count >= 2) { 1050 atomic_add_int(&object->ref_count, -1); 1051 break; 1052 } 1053 KKASSERT(object->ref_count == 1); 1054 1055 /* 1056 * 1->0 transition. Chain through the backing_object. 1057 * Maintain the ref until we've located the backing object, 1058 * then re-check. 1059 */ 1060 while ((temp = object->backing_object) != NULL) { 1061 if (temp->type == OBJT_VNODE) 1062 vm_object_hold_shared(temp); 1063 else 1064 vm_object_hold(temp); 1065 if (temp == object->backing_object) 1066 break; 1067 vm_object_drop(temp); 1068 } 1069 1070 /* 1071 * 1->0 transition verified, retry if ref_count is no longer 1072 * 1. Otherwise disconnect the backing_object (temp) and 1073 * clean up. 1074 */ 1075 if (object->ref_count != 1) { 1076 vm_object_drop(temp); 1077 continue; 1078 } 1079 1080 /* 1081 * It shouldn't be possible for the object to be chain locked 1082 * if we're removing the last ref on it. 1083 */ 1084 KKASSERT((object->chainlk & (CHAINLK_EXCL|CHAINLK_MASK)) == 0); 1085 1086 if (temp) { 1087 if (object->flags & OBJ_ONSHADOW) { 1088 LIST_REMOVE(object, shadow_list); 1089 temp->shadow_count--; 1090 temp->generation++; 1091 vm_object_clear_flag(object, OBJ_ONSHADOW); 1092 } 1093 object->backing_object = NULL; 1094 } 1095 1096 atomic_add_int(&object->ref_count, -1); 1097 if ((object->flags & OBJ_DEAD) == 0) 1098 vm_object_terminate(object); 1099 if (must_drop && temp) 1100 vm_object_lock_swap(); 1101 if (must_drop) 1102 vm_object_drop(object); 1103 object = temp; 1104 must_drop = 1; 1105 } 1106 1107 if (must_drop && object) 1108 vm_object_drop(object); 1109 1110 /* 1111 * Additional tail recursion on dlist. Avoid a recursion. Objects 1112 * on the dlist have a hold count but are not locked. 1113 */ 1114 if ((dtmp = dlist) != NULL) { 1115 dlist = dtmp->next; 1116 object = dtmp->object; 1117 kfree(dtmp, M_TEMP); 1118 1119 vm_object_lock(object); /* already held, add lock */ 1120 must_drop = 1; /* and we're responsible for it */ 1121 goto again; 1122 } 1123 } 1124 1125 /* 1126 * Destroy the specified object, freeing up related resources. 1127 * 1128 * The object must have zero references. 1129 * 1130 * The object must held. The caller is responsible for dropping the object 1131 * after terminate returns. Terminate does NOT drop the object. 1132 */ 1133 static int vm_object_terminate_callback(vm_page_t p, void *data); 1134 1135 void 1136 vm_object_terminate(vm_object_t object) 1137 { 1138 int n; 1139 1140 /* 1141 * Make sure no one uses us. Once we set OBJ_DEAD we should be 1142 * able to safely block. 1143 */ 1144 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 1145 KKASSERT((object->flags & OBJ_DEAD) == 0); 1146 vm_object_set_flag(object, OBJ_DEAD); 1147 1148 /* 1149 * Wait for the pageout daemon to be done with the object 1150 */ 1151 vm_object_pip_wait(object, "objtrm1"); 1152 1153 KASSERT(!object->paging_in_progress, 1154 ("vm_object_terminate: pageout in progress")); 1155 1156 /* 1157 * Clean and free the pages, as appropriate. All references to the 1158 * object are gone, so we don't need to lock it. 1159 */ 1160 if (object->type == OBJT_VNODE) { 1161 struct vnode *vp; 1162 1163 /* 1164 * Clean pages and flush buffers. 1165 * 1166 * NOTE! TMPFS buffer flushes do not typically flush the 1167 * actual page to swap as this would be highly 1168 * inefficient, and normal filesystems usually wrap 1169 * page flushes with buffer cache buffers. 1170 * 1171 * To deal with this we have to call vinvalbuf() both 1172 * before and after the vm_object_page_clean(). 1173 */ 1174 vp = (struct vnode *) object->handle; 1175 vinvalbuf(vp, V_SAVE, 0, 0); 1176 vm_object_page_clean(object, 0, 0, OBJPC_SYNC); 1177 vinvalbuf(vp, V_SAVE, 0, 0); 1178 } 1179 1180 /* 1181 * Wait for any I/O to complete, after which there had better not 1182 * be any references left on the object. 1183 */ 1184 vm_object_pip_wait(object, "objtrm2"); 1185 1186 if (object->ref_count != 0) { 1187 panic("vm_object_terminate: object with references, " 1188 "ref_count=%d", object->ref_count); 1189 } 1190 1191 /* 1192 * Cleanup any shared pmaps associated with this object. 1193 */ 1194 pmap_object_free(object); 1195 1196 /* 1197 * Now free any remaining pages. For internal objects, this also 1198 * removes them from paging queues. Don't free wired pages, just 1199 * remove them from the object. 1200 */ 1201 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL, 1202 vm_object_terminate_callback, NULL); 1203 1204 /* 1205 * Let the pager know object is dead. 1206 */ 1207 vm_pager_deallocate(object); 1208 1209 /* 1210 * Wait for the object hold count to hit 1, clean out pages as 1211 * we go. vmobj_token interlocks any race conditions that might 1212 * pick the object up from the vm_object_list after we have cleared 1213 * rb_memq. 1214 */ 1215 for (;;) { 1216 if (RB_ROOT(&object->rb_memq) == NULL) 1217 break; 1218 kprintf("vm_object_terminate: Warning, object %p " 1219 "still has %d pages\n", 1220 object, object->resident_page_count); 1221 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL, 1222 vm_object_terminate_callback, NULL); 1223 } 1224 1225 /* 1226 * There had better not be any pages left 1227 */ 1228 KKASSERT(object->resident_page_count == 0); 1229 1230 /* 1231 * Remove the object from the global object list. 1232 */ 1233 n = VMOBJ_HASH(object); 1234 lwkt_gettoken(&vmobj_tokens[n]); 1235 TAILQ_REMOVE(&vm_object_lists[n], object, object_list); 1236 lwkt_reltoken(&vmobj_tokens[n]); 1237 atomic_add_long(&vm_object_count, -1); 1238 1239 if (object->ref_count != 0) { 1240 panic("vm_object_terminate2: object with references, " 1241 "ref_count=%d", object->ref_count); 1242 } 1243 1244 /* 1245 * NOTE: The object hold_count is at least 1, so we cannot zfree() 1246 * the object here. See vm_object_drop(). 1247 */ 1248 } 1249 1250 /* 1251 * The caller must hold the object. 1252 */ 1253 static int 1254 vm_object_terminate_callback(vm_page_t p, void *data __unused) 1255 { 1256 vm_object_t object; 1257 1258 object = p->object; 1259 vm_page_busy_wait(p, TRUE, "vmpgtrm"); 1260 if (object != p->object) { 1261 kprintf("vm_object_terminate: Warning: Encountered " 1262 "busied page %p on queue %d\n", p, p->queue); 1263 vm_page_wakeup(p); 1264 } else if (p->wire_count == 0) { 1265 /* 1266 * NOTE: p->dirty and PG_NEED_COMMIT are ignored. 1267 */ 1268 vm_page_free(p); 1269 mycpu->gd_cnt.v_pfree++; 1270 } else { 1271 if (p->queue != PQ_NONE) 1272 kprintf("vm_object_terminate: Warning: Encountered " 1273 "wired page %p on queue %d\n", p, p->queue); 1274 vm_page_remove(p); 1275 vm_page_wakeup(p); 1276 } 1277 lwkt_yield(); 1278 return(0); 1279 } 1280 1281 /* 1282 * Clean all dirty pages in the specified range of object. Leaves page 1283 * on whatever queue it is currently on. If NOSYNC is set then do not 1284 * write out pages with PG_NOSYNC set (originally comes from MAP_NOSYNC), 1285 * leaving the object dirty. 1286 * 1287 * When stuffing pages asynchronously, allow clustering. XXX we need a 1288 * synchronous clustering mode implementation. 1289 * 1290 * Odd semantics: if start == end, we clean everything. 1291 * 1292 * The object must be locked? XXX 1293 */ 1294 static int vm_object_page_clean_pass1(struct vm_page *p, void *data); 1295 static int vm_object_page_clean_pass2(struct vm_page *p, void *data); 1296 1297 void 1298 vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 1299 int flags) 1300 { 1301 struct rb_vm_page_scan_info info; 1302 struct vnode *vp; 1303 int wholescan; 1304 int pagerflags; 1305 int generation; 1306 1307 vm_object_hold(object); 1308 if (object->type != OBJT_VNODE || 1309 (object->flags & OBJ_MIGHTBEDIRTY) == 0) { 1310 vm_object_drop(object); 1311 return; 1312 } 1313 1314 pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? 1315 VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; 1316 pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0; 1317 1318 vp = object->handle; 1319 1320 /* 1321 * Interlock other major object operations. This allows us to 1322 * temporarily clear OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY. 1323 */ 1324 vm_object_set_flag(object, OBJ_CLEANING); 1325 1326 /* 1327 * Handle 'entire object' case 1328 */ 1329 info.start_pindex = start; 1330 if (end == 0) { 1331 info.end_pindex = object->size - 1; 1332 } else { 1333 info.end_pindex = end - 1; 1334 } 1335 wholescan = (start == 0 && info.end_pindex == object->size - 1); 1336 info.limit = flags; 1337 info.pagerflags = pagerflags; 1338 info.object = object; 1339 1340 /* 1341 * If cleaning the entire object do a pass to mark the pages read-only. 1342 * If everything worked out ok, clear OBJ_WRITEABLE and 1343 * OBJ_MIGHTBEDIRTY. 1344 */ 1345 if (wholescan) { 1346 info.error = 0; 1347 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp, 1348 vm_object_page_clean_pass1, &info); 1349 if (info.error == 0) { 1350 vm_object_clear_flag(object, 1351 OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 1352 if (object->type == OBJT_VNODE && 1353 (vp = (struct vnode *)object->handle) != NULL) { 1354 /* 1355 * Use new-style interface to clear VISDIRTY 1356 * because the vnode is not necessarily removed 1357 * from the syncer list(s) as often as it was 1358 * under the old interface, which can leave 1359 * the vnode on the syncer list after reclaim. 1360 */ 1361 vclrobjdirty(vp); 1362 } 1363 } 1364 } 1365 1366 /* 1367 * Do a pass to clean all the dirty pages we find. 1368 */ 1369 do { 1370 info.error = 0; 1371 generation = object->generation; 1372 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp, 1373 vm_object_page_clean_pass2, &info); 1374 } while (info.error || generation != object->generation); 1375 1376 vm_object_clear_flag(object, OBJ_CLEANING); 1377 vm_object_drop(object); 1378 } 1379 1380 /* 1381 * The caller must hold the object. 1382 */ 1383 static 1384 int 1385 vm_object_page_clean_pass1(struct vm_page *p, void *data) 1386 { 1387 struct rb_vm_page_scan_info *info = data; 1388 1389 vm_page_flag_set(p, PG_CLEANCHK); 1390 if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { 1391 info->error = 1; 1392 } else if (vm_page_busy_try(p, FALSE) == 0) { 1393 vm_page_protect(p, VM_PROT_READ); /* must not block */ 1394 vm_page_wakeup(p); 1395 } else { 1396 info->error = 1; 1397 } 1398 lwkt_yield(); 1399 return(0); 1400 } 1401 1402 /* 1403 * The caller must hold the object 1404 */ 1405 static 1406 int 1407 vm_object_page_clean_pass2(struct vm_page *p, void *data) 1408 { 1409 struct rb_vm_page_scan_info *info = data; 1410 int generation; 1411 1412 /* 1413 * Do not mess with pages that were inserted after we started 1414 * the cleaning pass. 1415 */ 1416 if ((p->flags & PG_CLEANCHK) == 0) 1417 goto done; 1418 1419 generation = info->object->generation; 1420 vm_page_busy_wait(p, TRUE, "vpcwai"); 1421 if (p->object != info->object || 1422 info->object->generation != generation) { 1423 info->error = 1; 1424 vm_page_wakeup(p); 1425 goto done; 1426 } 1427 1428 /* 1429 * Before wasting time traversing the pmaps, check for trivial 1430 * cases where the page cannot be dirty. 1431 */ 1432 if (p->valid == 0 || (p->queue - p->pc) == PQ_CACHE) { 1433 KKASSERT((p->dirty & p->valid) == 0 && 1434 (p->flags & PG_NEED_COMMIT) == 0); 1435 vm_page_wakeup(p); 1436 goto done; 1437 } 1438 1439 /* 1440 * Check whether the page is dirty or not. The page has been set 1441 * to be read-only so the check will not race a user dirtying the 1442 * page. 1443 */ 1444 vm_page_test_dirty(p); 1445 if ((p->dirty & p->valid) == 0 && (p->flags & PG_NEED_COMMIT) == 0) { 1446 vm_page_flag_clear(p, PG_CLEANCHK); 1447 vm_page_wakeup(p); 1448 goto done; 1449 } 1450 1451 /* 1452 * If we have been asked to skip nosync pages and this is a 1453 * nosync page, skip it. Note that the object flags were 1454 * not cleared in this case (because pass1 will have returned an 1455 * error), so we do not have to set them. 1456 */ 1457 if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { 1458 vm_page_flag_clear(p, PG_CLEANCHK); 1459 vm_page_wakeup(p); 1460 goto done; 1461 } 1462 1463 /* 1464 * Flush as many pages as we can. PG_CLEANCHK will be cleared on 1465 * the pages that get successfully flushed. Set info->error if 1466 * we raced an object modification. 1467 */ 1468 vm_object_page_collect_flush(info->object, p, info->pagerflags); 1469 vm_wait_nominal(); 1470 done: 1471 lwkt_yield(); 1472 return(0); 1473 } 1474 1475 /* 1476 * Collect the specified page and nearby pages and flush them out. 1477 * The number of pages flushed is returned. The passed page is busied 1478 * by the caller and we are responsible for its disposition. 1479 * 1480 * The caller must hold the object. 1481 */ 1482 static void 1483 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags) 1484 { 1485 int error; 1486 int is; 1487 int ib; 1488 int i; 1489 int page_base; 1490 vm_pindex_t pi; 1491 vm_page_t ma[BLIST_MAX_ALLOC]; 1492 1493 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 1494 1495 pi = p->pindex; 1496 page_base = pi % BLIST_MAX_ALLOC; 1497 ma[page_base] = p; 1498 ib = page_base - 1; 1499 is = page_base + 1; 1500 1501 while (ib >= 0) { 1502 vm_page_t tp; 1503 1504 tp = vm_page_lookup_busy_try(object, pi - page_base + ib, 1505 TRUE, &error); 1506 if (error) 1507 break; 1508 if (tp == NULL) 1509 break; 1510 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 1511 (tp->flags & PG_CLEANCHK) == 0) { 1512 vm_page_wakeup(tp); 1513 break; 1514 } 1515 if ((tp->queue - tp->pc) == PQ_CACHE) { 1516 vm_page_flag_clear(tp, PG_CLEANCHK); 1517 vm_page_wakeup(tp); 1518 break; 1519 } 1520 vm_page_test_dirty(tp); 1521 if ((tp->dirty & tp->valid) == 0 && 1522 (tp->flags & PG_NEED_COMMIT) == 0) { 1523 vm_page_flag_clear(tp, PG_CLEANCHK); 1524 vm_page_wakeup(tp); 1525 break; 1526 } 1527 ma[ib] = tp; 1528 --ib; 1529 } 1530 ++ib; /* fixup */ 1531 1532 while (is < BLIST_MAX_ALLOC && 1533 pi - page_base + is < object->size) { 1534 vm_page_t tp; 1535 1536 tp = vm_page_lookup_busy_try(object, pi - page_base + is, 1537 TRUE, &error); 1538 if (error) 1539 break; 1540 if (tp == NULL) 1541 break; 1542 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 1543 (tp->flags & PG_CLEANCHK) == 0) { 1544 vm_page_wakeup(tp); 1545 break; 1546 } 1547 if ((tp->queue - tp->pc) == PQ_CACHE) { 1548 vm_page_flag_clear(tp, PG_CLEANCHK); 1549 vm_page_wakeup(tp); 1550 break; 1551 } 1552 vm_page_test_dirty(tp); 1553 if ((tp->dirty & tp->valid) == 0 && 1554 (tp->flags & PG_NEED_COMMIT) == 0) { 1555 vm_page_flag_clear(tp, PG_CLEANCHK); 1556 vm_page_wakeup(tp); 1557 break; 1558 } 1559 ma[is] = tp; 1560 ++is; 1561 } 1562 1563 /* 1564 * All pages in the ma[] array are busied now 1565 */ 1566 for (i = ib; i < is; ++i) { 1567 vm_page_flag_clear(ma[i], PG_CLEANCHK); 1568 vm_page_hold(ma[i]); /* XXX need this any more? */ 1569 } 1570 vm_pageout_flush(&ma[ib], is - ib, pagerflags); 1571 for (i = ib; i < is; ++i) /* XXX need this any more? */ 1572 vm_page_unhold(ma[i]); 1573 } 1574 1575 /* 1576 * Same as vm_object_pmap_copy, except range checking really 1577 * works, and is meant for small sections of an object. 1578 * 1579 * This code protects resident pages by making them read-only 1580 * and is typically called on a fork or split when a page 1581 * is converted to copy-on-write. 1582 * 1583 * NOTE: If the page is already at VM_PROT_NONE, calling 1584 * vm_page_protect will have no effect. 1585 */ 1586 void 1587 vm_object_pmap_copy_1(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1588 { 1589 vm_pindex_t idx; 1590 vm_page_t p; 1591 1592 if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0) 1593 return; 1594 1595 vm_object_hold(object); 1596 for (idx = start; idx < end; idx++) { 1597 p = vm_page_lookup(object, idx); 1598 if (p == NULL) 1599 continue; 1600 vm_page_protect(p, VM_PROT_READ); 1601 } 1602 vm_object_drop(object); 1603 } 1604 1605 /* 1606 * Removes all physical pages in the specified object range from all 1607 * physical maps. 1608 * 1609 * The object must *not* be locked. 1610 */ 1611 1612 static int vm_object_pmap_remove_callback(vm_page_t p, void *data); 1613 1614 void 1615 vm_object_pmap_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1616 { 1617 struct rb_vm_page_scan_info info; 1618 1619 if (object == NULL) 1620 return; 1621 info.start_pindex = start; 1622 info.end_pindex = end - 1; 1623 1624 vm_object_hold(object); 1625 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp, 1626 vm_object_pmap_remove_callback, &info); 1627 if (start == 0 && end == object->size) 1628 vm_object_clear_flag(object, OBJ_WRITEABLE); 1629 vm_object_drop(object); 1630 } 1631 1632 /* 1633 * The caller must hold the object 1634 */ 1635 static int 1636 vm_object_pmap_remove_callback(vm_page_t p, void *data __unused) 1637 { 1638 vm_page_protect(p, VM_PROT_NONE); 1639 return(0); 1640 } 1641 1642 /* 1643 * Implements the madvise function at the object/page level. 1644 * 1645 * MADV_WILLNEED (any object) 1646 * 1647 * Activate the specified pages if they are resident. 1648 * 1649 * MADV_DONTNEED (any object) 1650 * 1651 * Deactivate the specified pages if they are resident. 1652 * 1653 * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects, OBJ_ONEMAPPING only) 1654 * 1655 * Deactivate and clean the specified pages if they are 1656 * resident. This permits the process to reuse the pages 1657 * without faulting or the kernel to reclaim the pages 1658 * without I/O. 1659 * 1660 * No requirements. 1661 */ 1662 void 1663 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, int count, int advise) 1664 { 1665 vm_pindex_t end, tpindex; 1666 vm_object_t tobject; 1667 vm_object_t xobj; 1668 vm_page_t m; 1669 int error; 1670 1671 if (object == NULL) 1672 return; 1673 1674 end = pindex + count; 1675 1676 vm_object_hold(object); 1677 tobject = object; 1678 1679 /* 1680 * Locate and adjust resident pages 1681 */ 1682 for (; pindex < end; pindex += 1) { 1683 relookup: 1684 if (tobject != object) 1685 vm_object_drop(tobject); 1686 tobject = object; 1687 tpindex = pindex; 1688 shadowlookup: 1689 /* 1690 * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages 1691 * and those pages must be OBJ_ONEMAPPING. 1692 */ 1693 if (advise == MADV_FREE) { 1694 if ((tobject->type != OBJT_DEFAULT && 1695 tobject->type != OBJT_SWAP) || 1696 (tobject->flags & OBJ_ONEMAPPING) == 0) { 1697 continue; 1698 } 1699 } 1700 1701 m = vm_page_lookup_busy_try(tobject, tpindex, TRUE, &error); 1702 1703 if (error) { 1704 vm_page_sleep_busy(m, TRUE, "madvpo"); 1705 goto relookup; 1706 } 1707 if (m == NULL) { 1708 /* 1709 * There may be swap even if there is no backing page 1710 */ 1711 if (advise == MADV_FREE && tobject->type == OBJT_SWAP) 1712 swap_pager_freespace(tobject, tpindex, 1); 1713 1714 /* 1715 * next object 1716 */ 1717 while ((xobj = tobject->backing_object) != NULL) { 1718 KKASSERT(xobj != object); 1719 vm_object_hold(xobj); 1720 if (xobj == tobject->backing_object) 1721 break; 1722 vm_object_drop(xobj); 1723 } 1724 if (xobj == NULL) 1725 continue; 1726 tpindex += OFF_TO_IDX(tobject->backing_object_offset); 1727 if (tobject != object) { 1728 vm_object_lock_swap(); 1729 vm_object_drop(tobject); 1730 } 1731 tobject = xobj; 1732 goto shadowlookup; 1733 } 1734 1735 /* 1736 * If the page is not in a normal active state, we skip it. 1737 * If the page is not managed there are no page queues to 1738 * mess with. Things can break if we mess with pages in 1739 * any of the below states. 1740 */ 1741 if (m->wire_count || 1742 (m->flags & (PG_UNMANAGED | PG_NEED_COMMIT)) || 1743 m->valid != VM_PAGE_BITS_ALL 1744 ) { 1745 vm_page_wakeup(m); 1746 continue; 1747 } 1748 1749 /* 1750 * Theoretically once a page is known not to be busy, an 1751 * interrupt cannot come along and rip it out from under us. 1752 */ 1753 1754 if (advise == MADV_WILLNEED) { 1755 vm_page_activate(m); 1756 } else if (advise == MADV_DONTNEED) { 1757 vm_page_dontneed(m); 1758 } else if (advise == MADV_FREE) { 1759 /* 1760 * Mark the page clean. This will allow the page 1761 * to be freed up by the system. However, such pages 1762 * are often reused quickly by malloc()/free() 1763 * so we do not do anything that would cause 1764 * a page fault if we can help it. 1765 * 1766 * Specifically, we do not try to actually free 1767 * the page now nor do we try to put it in the 1768 * cache (which would cause a page fault on reuse). 1769 * 1770 * But we do make the page is freeable as we 1771 * can without actually taking the step of unmapping 1772 * it. 1773 */ 1774 pmap_clear_modify(m); 1775 m->dirty = 0; 1776 m->act_count = 0; 1777 vm_page_dontneed(m); 1778 if (tobject->type == OBJT_SWAP) 1779 swap_pager_freespace(tobject, tpindex, 1); 1780 } 1781 vm_page_wakeup(m); 1782 } 1783 if (tobject != object) 1784 vm_object_drop(tobject); 1785 vm_object_drop(object); 1786 } 1787 1788 /* 1789 * Create a new object which is backed by the specified existing object 1790 * range. Replace the pointer and offset that was pointing at the existing 1791 * object with the pointer/offset for the new object. 1792 * 1793 * No other requirements. 1794 */ 1795 void 1796 vm_object_shadow(vm_object_t *objectp, vm_ooffset_t *offset, vm_size_t length, 1797 int addref) 1798 { 1799 vm_object_t source; 1800 vm_object_t result; 1801 int useshadowlist; 1802 1803 source = *objectp; 1804 1805 /* 1806 * Don't create the new object if the old object isn't shared. 1807 * We have to chain wait before adding the reference to avoid 1808 * racing a collapse or deallocation. 1809 * 1810 * Add the additional ref to source here to avoid racing a later 1811 * collapse or deallocation. Clear the ONEMAPPING flag whether 1812 * addref is TRUE or not in this case because the original object 1813 * will be shadowed. 1814 */ 1815 useshadowlist = 0; 1816 if (source) { 1817 if (source->type != OBJT_VNODE) { 1818 useshadowlist = 1; 1819 vm_object_hold(source); 1820 vm_object_chain_wait(source, 0); 1821 if (source->ref_count == 1 && 1822 source->handle == NULL && 1823 (source->type == OBJT_DEFAULT || 1824 source->type == OBJT_SWAP)) { 1825 if (addref) { 1826 vm_object_reference_locked(source); 1827 vm_object_clear_flag(source, OBJ_ONEMAPPING); 1828 } 1829 vm_object_drop(source); 1830 return; 1831 } 1832 vm_object_reference_locked(source); 1833 vm_object_clear_flag(source, OBJ_ONEMAPPING); 1834 } else { 1835 vm_object_reference_quick(source); 1836 vm_object_clear_flag(source, OBJ_ONEMAPPING); 1837 } 1838 } 1839 1840 /* 1841 * Allocate a new object with the given length. The new object 1842 * is returned referenced but we may have to add another one. 1843 * If we are adding a second reference we must clear OBJ_ONEMAPPING. 1844 * (typically because the caller is about to clone a vm_map_entry). 1845 * 1846 * The source object currently has an extra reference to prevent 1847 * collapses into it while we mess with its shadow list, which 1848 * we will remove later in this routine. 1849 */ 1850 if ((result = vm_object_allocate(OBJT_DEFAULT, length)) == NULL) 1851 panic("vm_object_shadow: no object for shadowing"); 1852 vm_object_hold(result); 1853 if (addref) { 1854 vm_object_reference_locked(result); 1855 vm_object_clear_flag(result, OBJ_ONEMAPPING); 1856 } 1857 1858 /* 1859 * The new object shadows the source object. Chain wait before 1860 * adjusting shadow_count or the shadow list to avoid races. 1861 * 1862 * Try to optimize the result object's page color when shadowing 1863 * in order to maintain page coloring consistency in the combined 1864 * shadowed object. 1865 * 1866 * SHADOWING IS NOT APPLICABLE TO OBJT_VNODE OBJECTS 1867 */ 1868 KKASSERT(result->backing_object == NULL); 1869 result->backing_object = source; 1870 if (source) { 1871 if (useshadowlist) { 1872 vm_object_chain_wait(source, 0); 1873 LIST_INSERT_HEAD(&source->shadow_head, 1874 result, shadow_list); 1875 source->shadow_count++; 1876 source->generation++; 1877 vm_object_set_flag(result, OBJ_ONSHADOW); 1878 } 1879 /* cpu localization twist */ 1880 result->pg_color = (int)(intptr_t)curthread; 1881 } 1882 1883 /* 1884 * Adjust the return storage. Drop the ref on source before 1885 * returning. 1886 */ 1887 result->backing_object_offset = *offset; 1888 vm_object_drop(result); 1889 *offset = 0; 1890 if (source) { 1891 if (useshadowlist) { 1892 vm_object_deallocate_locked(source); 1893 vm_object_drop(source); 1894 } else { 1895 vm_object_deallocate(source); 1896 } 1897 } 1898 1899 /* 1900 * Return the new things 1901 */ 1902 *objectp = result; 1903 } 1904 1905 #define OBSC_TEST_ALL_SHADOWED 0x0001 1906 #define OBSC_COLLAPSE_NOWAIT 0x0002 1907 #define OBSC_COLLAPSE_WAIT 0x0004 1908 1909 static int vm_object_backing_scan_callback(vm_page_t p, void *data); 1910 1911 /* 1912 * The caller must hold the object. 1913 */ 1914 static __inline int 1915 vm_object_backing_scan(vm_object_t object, vm_object_t backing_object, int op) 1916 { 1917 struct rb_vm_page_scan_info info; 1918 int n; 1919 1920 vm_object_assert_held(object); 1921 vm_object_assert_held(backing_object); 1922 1923 KKASSERT(backing_object == object->backing_object); 1924 info.backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 1925 1926 /* 1927 * Initial conditions 1928 */ 1929 if (op & OBSC_TEST_ALL_SHADOWED) { 1930 /* 1931 * We do not want to have to test for the existence of 1932 * swap pages in the backing object. XXX but with the 1933 * new swapper this would be pretty easy to do. 1934 * 1935 * XXX what about anonymous MAP_SHARED memory that hasn't 1936 * been ZFOD faulted yet? If we do not test for this, the 1937 * shadow test may succeed! XXX 1938 */ 1939 if (backing_object->type != OBJT_DEFAULT) 1940 return(0); 1941 } 1942 if (op & OBSC_COLLAPSE_WAIT) { 1943 KKASSERT((backing_object->flags & OBJ_DEAD) == 0); 1944 vm_object_set_flag(backing_object, OBJ_DEAD); 1945 1946 n = VMOBJ_HASH(backing_object); 1947 lwkt_gettoken(&vmobj_tokens[n]); 1948 TAILQ_REMOVE(&vm_object_lists[n], backing_object, object_list); 1949 lwkt_reltoken(&vmobj_tokens[n]); 1950 atomic_add_long(&vm_object_count, -1); 1951 } 1952 1953 /* 1954 * Our scan. We have to retry if a negative error code is returned, 1955 * otherwise 0 or 1 will be returned in info.error. 0 Indicates that 1956 * the scan had to be stopped because the parent does not completely 1957 * shadow the child. 1958 */ 1959 info.object = object; 1960 info.backing_object = backing_object; 1961 info.limit = op; 1962 do { 1963 info.error = 1; 1964 vm_page_rb_tree_RB_SCAN(&backing_object->rb_memq, NULL, 1965 vm_object_backing_scan_callback, 1966 &info); 1967 } while (info.error < 0); 1968 1969 return(info.error); 1970 } 1971 1972 /* 1973 * The caller must hold the object. 1974 */ 1975 static int 1976 vm_object_backing_scan_callback(vm_page_t p, void *data) 1977 { 1978 struct rb_vm_page_scan_info *info = data; 1979 vm_object_t backing_object; 1980 vm_object_t object; 1981 vm_pindex_t pindex; 1982 vm_pindex_t new_pindex; 1983 vm_pindex_t backing_offset_index; 1984 int op; 1985 1986 pindex = p->pindex; 1987 new_pindex = pindex - info->backing_offset_index; 1988 op = info->limit; 1989 object = info->object; 1990 backing_object = info->backing_object; 1991 backing_offset_index = info->backing_offset_index; 1992 1993 if (op & OBSC_TEST_ALL_SHADOWED) { 1994 vm_page_t pp; 1995 1996 /* 1997 * Ignore pages outside the parent object's range 1998 * and outside the parent object's mapping of the 1999 * backing object. 2000 * 2001 * note that we do not busy the backing object's 2002 * page. 2003 */ 2004 if (pindex < backing_offset_index || 2005 new_pindex >= object->size 2006 ) { 2007 return(0); 2008 } 2009 2010 /* 2011 * See if the parent has the page or if the parent's 2012 * object pager has the page. If the parent has the 2013 * page but the page is not valid, the parent's 2014 * object pager must have the page. 2015 * 2016 * If this fails, the parent does not completely shadow 2017 * the object and we might as well give up now. 2018 */ 2019 pp = vm_page_lookup(object, new_pindex); 2020 if ((pp == NULL || pp->valid == 0) && 2021 !vm_pager_has_page(object, new_pindex) 2022 ) { 2023 info->error = 0; /* problemo */ 2024 return(-1); /* stop the scan */ 2025 } 2026 } 2027 2028 /* 2029 * Check for busy page. Note that we may have lost (p) when we 2030 * possibly blocked above. 2031 */ 2032 if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) { 2033 vm_page_t pp; 2034 2035 if (vm_page_busy_try(p, TRUE)) { 2036 if (op & OBSC_COLLAPSE_NOWAIT) { 2037 return(0); 2038 } else { 2039 /* 2040 * If we slept, anything could have 2041 * happened. Ask that the scan be restarted. 2042 * 2043 * Since the object is marked dead, the 2044 * backing offset should not have changed. 2045 */ 2046 vm_page_sleep_busy(p, TRUE, "vmocol"); 2047 info->error = -1; 2048 return(-1); 2049 } 2050 } 2051 2052 /* 2053 * If (p) is no longer valid restart the scan. 2054 */ 2055 if (p->object != backing_object || p->pindex != pindex) { 2056 kprintf("vm_object_backing_scan: Warning: page " 2057 "%p ripped out from under us\n", p); 2058 vm_page_wakeup(p); 2059 info->error = -1; 2060 return(-1); 2061 } 2062 2063 if (op & OBSC_COLLAPSE_NOWAIT) { 2064 if (p->valid == 0 || 2065 p->wire_count || 2066 (p->flags & PG_NEED_COMMIT)) { 2067 vm_page_wakeup(p); 2068 return(0); 2069 } 2070 } else { 2071 /* XXX what if p->valid == 0 , hold_count, etc? */ 2072 } 2073 2074 KASSERT( 2075 p->object == backing_object, 2076 ("vm_object_qcollapse(): object mismatch") 2077 ); 2078 2079 /* 2080 * Destroy any associated swap 2081 */ 2082 if (backing_object->type == OBJT_SWAP) 2083 swap_pager_freespace(backing_object, p->pindex, 1); 2084 2085 if ( 2086 p->pindex < backing_offset_index || 2087 new_pindex >= object->size 2088 ) { 2089 /* 2090 * Page is out of the parent object's range, we 2091 * can simply destroy it. 2092 */ 2093 vm_page_protect(p, VM_PROT_NONE); 2094 vm_page_free(p); 2095 return(0); 2096 } 2097 2098 pp = vm_page_lookup(object, new_pindex); 2099 if (pp != NULL || vm_pager_has_page(object, new_pindex)) { 2100 /* 2101 * page already exists in parent OR swap exists 2102 * for this location in the parent. Destroy 2103 * the original page from the backing object. 2104 * 2105 * Leave the parent's page alone 2106 */ 2107 vm_page_protect(p, VM_PROT_NONE); 2108 vm_page_free(p); 2109 return(0); 2110 } 2111 2112 /* 2113 * Page does not exist in parent, rename the 2114 * page from the backing object to the main object. 2115 * 2116 * If the page was mapped to a process, it can remain 2117 * mapped through the rename. 2118 */ 2119 if ((p->queue - p->pc) == PQ_CACHE) 2120 vm_page_deactivate(p); 2121 2122 vm_page_rename(p, object, new_pindex); 2123 vm_page_wakeup(p); 2124 /* page automatically made dirty by rename */ 2125 } 2126 return(0); 2127 } 2128 2129 /* 2130 * This version of collapse allows the operation to occur earlier and 2131 * when paging_in_progress is true for an object... This is not a complete 2132 * operation, but should plug 99.9% of the rest of the leaks. 2133 * 2134 * The caller must hold the object and backing_object and both must be 2135 * chainlocked. 2136 * 2137 * (only called from vm_object_collapse) 2138 */ 2139 static void 2140 vm_object_qcollapse(vm_object_t object, vm_object_t backing_object) 2141 { 2142 if (backing_object->ref_count == 1) { 2143 atomic_add_int(&backing_object->ref_count, 2); 2144 vm_object_backing_scan(object, backing_object, 2145 OBSC_COLLAPSE_NOWAIT); 2146 atomic_add_int(&backing_object->ref_count, -2); 2147 } 2148 } 2149 2150 /* 2151 * Collapse an object with the object backing it. Pages in the backing 2152 * object are moved into the parent, and the backing object is deallocated. 2153 * Any conflict is resolved in favor of the parent's existing pages. 2154 * 2155 * object must be held and chain-locked on call. 2156 * 2157 * The caller must have an extra ref on object to prevent a race from 2158 * destroying it during the collapse. 2159 */ 2160 void 2161 vm_object_collapse(vm_object_t object, struct vm_object_dealloc_list **dlistp) 2162 { 2163 struct vm_object_dealloc_list *dlist = NULL; 2164 vm_object_t backing_object; 2165 2166 /* 2167 * Only one thread is attempting a collapse at any given moment. 2168 * There are few restrictions for (object) that callers of this 2169 * function check so reentrancy is likely. 2170 */ 2171 KKASSERT(object != NULL); 2172 vm_object_assert_held(object); 2173 KKASSERT(object->chainlk & (CHAINLK_MASK | CHAINLK_EXCL)); 2174 2175 for (;;) { 2176 vm_object_t bbobj; 2177 int dodealloc; 2178 2179 /* 2180 * We can only collapse a DEFAULT/SWAP object with a 2181 * DEFAULT/SWAP object. 2182 */ 2183 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP) { 2184 backing_object = NULL; 2185 break; 2186 } 2187 2188 backing_object = object->backing_object; 2189 if (backing_object == NULL) 2190 break; 2191 if (backing_object->type != OBJT_DEFAULT && 2192 backing_object->type != OBJT_SWAP) { 2193 backing_object = NULL; 2194 break; 2195 } 2196 2197 /* 2198 * Hold the backing_object and check for races 2199 */ 2200 vm_object_hold(backing_object); 2201 if (backing_object != object->backing_object || 2202 (backing_object->type != OBJT_DEFAULT && 2203 backing_object->type != OBJT_SWAP)) { 2204 vm_object_drop(backing_object); 2205 continue; 2206 } 2207 2208 /* 2209 * Chain-lock the backing object too because if we 2210 * successfully merge its pages into the top object we 2211 * will collapse backing_object->backing_object as the 2212 * new backing_object. Re-check that it is still our 2213 * backing object. 2214 */ 2215 vm_object_chain_acquire(backing_object, 0); 2216 if (backing_object != object->backing_object) { 2217 vm_object_chain_release(backing_object); 2218 vm_object_drop(backing_object); 2219 continue; 2220 } 2221 2222 /* 2223 * we check the backing object first, because it is most likely 2224 * not collapsable. 2225 */ 2226 if (backing_object->handle != NULL || 2227 (backing_object->type != OBJT_DEFAULT && 2228 backing_object->type != OBJT_SWAP) || 2229 (backing_object->flags & OBJ_DEAD) || 2230 object->handle != NULL || 2231 (object->type != OBJT_DEFAULT && 2232 object->type != OBJT_SWAP) || 2233 (object->flags & OBJ_DEAD)) { 2234 break; 2235 } 2236 2237 /* 2238 * If paging is in progress we can't do a normal collapse. 2239 */ 2240 if ( 2241 object->paging_in_progress != 0 || 2242 backing_object->paging_in_progress != 0 2243 ) { 2244 vm_object_qcollapse(object, backing_object); 2245 break; 2246 } 2247 2248 /* 2249 * We know that we can either collapse the backing object (if 2250 * the parent is the only reference to it) or (perhaps) have 2251 * the parent bypass the object if the parent happens to shadow 2252 * all the resident pages in the entire backing object. 2253 * 2254 * This is ignoring pager-backed pages such as swap pages. 2255 * vm_object_backing_scan fails the shadowing test in this 2256 * case. 2257 */ 2258 if (backing_object->ref_count == 1) { 2259 /* 2260 * If there is exactly one reference to the backing 2261 * object, we can collapse it into the parent. 2262 */ 2263 KKASSERT(object->backing_object == backing_object); 2264 vm_object_backing_scan(object, backing_object, 2265 OBSC_COLLAPSE_WAIT); 2266 2267 /* 2268 * Move the pager from backing_object to object. 2269 */ 2270 if (backing_object->type == OBJT_SWAP) { 2271 vm_object_pip_add(backing_object, 1); 2272 2273 /* 2274 * scrap the paging_offset junk and do a 2275 * discrete copy. This also removes major 2276 * assumptions about how the swap-pager 2277 * works from where it doesn't belong. The 2278 * new swapper is able to optimize the 2279 * destroy-source case. 2280 */ 2281 vm_object_pip_add(object, 1); 2282 swap_pager_copy(backing_object, object, 2283 OFF_TO_IDX(object->backing_object_offset), 2284 TRUE); 2285 vm_object_pip_wakeup(object); 2286 vm_object_pip_wakeup(backing_object); 2287 } 2288 2289 /* 2290 * Object now shadows whatever backing_object did. 2291 * Remove object from backing_object's shadow_list. 2292 */ 2293 KKASSERT(object->backing_object == backing_object); 2294 if (object->flags & OBJ_ONSHADOW) { 2295 LIST_REMOVE(object, shadow_list); 2296 backing_object->shadow_count--; 2297 backing_object->generation++; 2298 vm_object_clear_flag(object, OBJ_ONSHADOW); 2299 } 2300 2301 /* 2302 * backing_object->backing_object moves from within 2303 * backing_object to within object. 2304 * 2305 * OBJT_VNODE bbobj's should have empty shadow lists. 2306 */ 2307 while ((bbobj = backing_object->backing_object) != NULL) { 2308 if (bbobj->type == OBJT_VNODE) 2309 vm_object_hold_shared(bbobj); 2310 else 2311 vm_object_hold(bbobj); 2312 if (bbobj == backing_object->backing_object) 2313 break; 2314 vm_object_drop(bbobj); 2315 } 2316 if (bbobj) { 2317 if (backing_object->flags & OBJ_ONSHADOW) { 2318 /* not locked exclusively if vnode */ 2319 KKASSERT(bbobj->type != OBJT_VNODE); 2320 LIST_REMOVE(backing_object, 2321 shadow_list); 2322 bbobj->shadow_count--; 2323 bbobj->generation++; 2324 vm_object_clear_flag(backing_object, 2325 OBJ_ONSHADOW); 2326 } 2327 backing_object->backing_object = NULL; 2328 } 2329 object->backing_object = bbobj; 2330 if (bbobj) { 2331 if (bbobj->type != OBJT_VNODE) { 2332 LIST_INSERT_HEAD(&bbobj->shadow_head, 2333 object, shadow_list); 2334 bbobj->shadow_count++; 2335 bbobj->generation++; 2336 vm_object_set_flag(object, 2337 OBJ_ONSHADOW); 2338 } 2339 } 2340 2341 object->backing_object_offset += 2342 backing_object->backing_object_offset; 2343 2344 vm_object_drop(bbobj); 2345 2346 /* 2347 * Discard the old backing_object. Nothing should be 2348 * able to ref it, other than a vm_map_split(), 2349 * and vm_map_split() will stall on our chain lock. 2350 * And we control the parent so it shouldn't be 2351 * possible for it to go away either. 2352 * 2353 * Since the backing object has no pages, no pager 2354 * left, and no object references within it, all 2355 * that is necessary is to dispose of it. 2356 */ 2357 KASSERT(backing_object->ref_count == 1, 2358 ("backing_object %p was somehow " 2359 "re-referenced during collapse!", 2360 backing_object)); 2361 KASSERT(RB_EMPTY(&backing_object->rb_memq), 2362 ("backing_object %p somehow has left " 2363 "over pages during collapse!", 2364 backing_object)); 2365 2366 /* 2367 * The object can be destroyed. 2368 * 2369 * XXX just fall through and dodealloc instead 2370 * of forcing destruction? 2371 */ 2372 atomic_add_int(&backing_object->ref_count, -1); 2373 if ((backing_object->flags & OBJ_DEAD) == 0) 2374 vm_object_terminate(backing_object); 2375 object_collapses++; 2376 dodealloc = 0; 2377 } else { 2378 /* 2379 * If we do not entirely shadow the backing object, 2380 * there is nothing we can do so we give up. 2381 */ 2382 if (vm_object_backing_scan(object, backing_object, 2383 OBSC_TEST_ALL_SHADOWED) == 0) { 2384 break; 2385 } 2386 2387 /* 2388 * bbobj is backing_object->backing_object. Since 2389 * object completely shadows backing_object we can 2390 * bypass it and become backed by bbobj instead. 2391 * 2392 * The shadow list for vnode backing objects is not 2393 * used and a shared hold is allowed. 2394 */ 2395 while ((bbobj = backing_object->backing_object) != NULL) { 2396 if (bbobj->type == OBJT_VNODE) 2397 vm_object_hold_shared(bbobj); 2398 else 2399 vm_object_hold(bbobj); 2400 if (bbobj == backing_object->backing_object) 2401 break; 2402 vm_object_drop(bbobj); 2403 } 2404 2405 /* 2406 * Make object shadow bbobj instead of backing_object. 2407 * Remove object from backing_object's shadow list. 2408 * 2409 * Deallocating backing_object will not remove 2410 * it, since its reference count is at least 2. 2411 */ 2412 KKASSERT(object->backing_object == backing_object); 2413 if (object->flags & OBJ_ONSHADOW) { 2414 LIST_REMOVE(object, shadow_list); 2415 backing_object->shadow_count--; 2416 backing_object->generation++; 2417 vm_object_clear_flag(object, OBJ_ONSHADOW); 2418 } 2419 2420 /* 2421 * Add a ref to bbobj, bbobj now shadows object. 2422 * 2423 * NOTE: backing_object->backing_object still points 2424 * to bbobj. That relationship remains intact 2425 * because backing_object has > 1 ref, so 2426 * someone else is pointing to it (hence why 2427 * we can't collapse it into object and can 2428 * only handle the all-shadowed bypass case). 2429 */ 2430 if (bbobj) { 2431 if (bbobj->type != OBJT_VNODE) { 2432 vm_object_chain_wait(bbobj, 0); 2433 vm_object_reference_locked(bbobj); 2434 LIST_INSERT_HEAD(&bbobj->shadow_head, 2435 object, shadow_list); 2436 bbobj->shadow_count++; 2437 bbobj->generation++; 2438 vm_object_set_flag(object, 2439 OBJ_ONSHADOW); 2440 } else { 2441 vm_object_reference_quick(bbobj); 2442 } 2443 object->backing_object_offset += 2444 backing_object->backing_object_offset; 2445 object->backing_object = bbobj; 2446 vm_object_drop(bbobj); 2447 } else { 2448 object->backing_object = NULL; 2449 } 2450 2451 /* 2452 * Drop the reference count on backing_object. To 2453 * handle ref_count races properly we can't assume 2454 * that the ref_count is still at least 2 so we 2455 * have to actually call vm_object_deallocate() 2456 * (after clearing the chainlock). 2457 */ 2458 object_bypasses++; 2459 dodealloc = 1; 2460 } 2461 2462 /* 2463 * Ok, we want to loop on the new object->bbobj association, 2464 * possibly collapsing it further. However if dodealloc is 2465 * non-zero we have to deallocate the backing_object which 2466 * itself can potentially undergo a collapse, creating a 2467 * recursion depth issue with the LWKT token subsystem. 2468 * 2469 * In the case where we must deallocate the backing_object 2470 * it is possible now that the backing_object has a single 2471 * shadow count on some other object (not represented here 2472 * as yet), since it no longer shadows us. Thus when we 2473 * call vm_object_deallocate() it may attempt to collapse 2474 * itself into its remaining parent. 2475 */ 2476 if (dodealloc) { 2477 struct vm_object_dealloc_list *dtmp; 2478 2479 vm_object_chain_release(backing_object); 2480 vm_object_unlock(backing_object); 2481 /* backing_object remains held */ 2482 2483 /* 2484 * Auto-deallocation list for caller convenience. 2485 */ 2486 if (dlistp == NULL) 2487 dlistp = &dlist; 2488 2489 dtmp = kmalloc(sizeof(*dtmp), M_TEMP, M_WAITOK); 2490 dtmp->object = backing_object; 2491 dtmp->next = *dlistp; 2492 *dlistp = dtmp; 2493 } else { 2494 vm_object_chain_release(backing_object); 2495 vm_object_drop(backing_object); 2496 } 2497 /* backing_object = NULL; not needed */ 2498 /* loop */ 2499 } 2500 2501 /* 2502 * Clean up any left over backing_object 2503 */ 2504 if (backing_object) { 2505 vm_object_chain_release(backing_object); 2506 vm_object_drop(backing_object); 2507 } 2508 2509 /* 2510 * Clean up any auto-deallocation list. This is a convenience 2511 * for top-level callers so they don't have to pass &dlist. 2512 * Do not clean up any caller-passed dlistp, the caller will 2513 * do that. 2514 */ 2515 if (dlist) 2516 vm_object_deallocate_list(&dlist); 2517 2518 } 2519 2520 /* 2521 * vm_object_collapse() may collect additional objects in need of 2522 * deallocation. This routine deallocates these objects. The 2523 * deallocation itself can trigger additional collapses (which the 2524 * deallocate function takes care of). This procedure is used to 2525 * reduce procedural recursion since these vm_object shadow chains 2526 * can become quite long. 2527 */ 2528 void 2529 vm_object_deallocate_list(struct vm_object_dealloc_list **dlistp) 2530 { 2531 struct vm_object_dealloc_list *dlist; 2532 2533 while ((dlist = *dlistp) != NULL) { 2534 *dlistp = dlist->next; 2535 vm_object_lock(dlist->object); 2536 vm_object_deallocate_locked(dlist->object); 2537 vm_object_drop(dlist->object); 2538 kfree(dlist, M_TEMP); 2539 } 2540 } 2541 2542 /* 2543 * Removes all physical pages in the specified object range from the 2544 * object's list of pages. 2545 * 2546 * No requirements. 2547 */ 2548 static int vm_object_page_remove_callback(vm_page_t p, void *data); 2549 2550 void 2551 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 2552 boolean_t clean_only) 2553 { 2554 struct rb_vm_page_scan_info info; 2555 int all; 2556 2557 /* 2558 * Degenerate cases and assertions 2559 */ 2560 vm_object_hold(object); 2561 if (object == NULL || 2562 (object->resident_page_count == 0 && object->swblock_count == 0)) { 2563 vm_object_drop(object); 2564 return; 2565 } 2566 KASSERT(object->type != OBJT_PHYS, 2567 ("attempt to remove pages from a physical object")); 2568 2569 /* 2570 * Indicate that paging is occuring on the object 2571 */ 2572 vm_object_pip_add(object, 1); 2573 2574 /* 2575 * Figure out the actual removal range and whether we are removing 2576 * the entire contents of the object or not. If removing the entire 2577 * contents, be sure to get all pages, even those that might be 2578 * beyond the end of the object. 2579 */ 2580 info.start_pindex = start; 2581 if (end == 0) 2582 info.end_pindex = (vm_pindex_t)-1; 2583 else 2584 info.end_pindex = end - 1; 2585 info.limit = clean_only; 2586 all = (start == 0 && info.end_pindex >= object->size - 1); 2587 2588 /* 2589 * Loop until we are sure we have gotten them all. 2590 */ 2591 do { 2592 info.error = 0; 2593 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp, 2594 vm_object_page_remove_callback, &info); 2595 } while (info.error); 2596 2597 /* 2598 * Remove any related swap if throwing away pages, or for 2599 * non-swap objects (the swap is a clean copy in that case). 2600 */ 2601 if (object->type != OBJT_SWAP || clean_only == FALSE) { 2602 if (all) 2603 swap_pager_freespace_all(object); 2604 else 2605 swap_pager_freespace(object, info.start_pindex, 2606 info.end_pindex - info.start_pindex + 1); 2607 } 2608 2609 /* 2610 * Cleanup 2611 */ 2612 vm_object_pip_wakeup(object); 2613 vm_object_drop(object); 2614 } 2615 2616 /* 2617 * The caller must hold the object 2618 */ 2619 static int 2620 vm_object_page_remove_callback(vm_page_t p, void *data) 2621 { 2622 struct rb_vm_page_scan_info *info = data; 2623 2624 if (vm_page_busy_try(p, TRUE)) { 2625 vm_page_sleep_busy(p, TRUE, "vmopar"); 2626 info->error = 1; 2627 return(0); 2628 } 2629 2630 /* 2631 * Wired pages cannot be destroyed, but they can be invalidated 2632 * and we do so if clean_only (limit) is not set. 2633 * 2634 * WARNING! The page may be wired due to being part of a buffer 2635 * cache buffer, and the buffer might be marked B_CACHE. 2636 * This is fine as part of a truncation but VFSs must be 2637 * sure to fix the buffer up when re-extending the file. 2638 * 2639 * NOTE! PG_NEED_COMMIT is ignored. 2640 */ 2641 if (p->wire_count != 0) { 2642 vm_page_protect(p, VM_PROT_NONE); 2643 if (info->limit == 0) 2644 p->valid = 0; 2645 vm_page_wakeup(p); 2646 return(0); 2647 } 2648 2649 /* 2650 * limit is our clean_only flag. If set and the page is dirty or 2651 * requires a commit, do not free it. If set and the page is being 2652 * held by someone, do not free it. 2653 */ 2654 if (info->limit && p->valid) { 2655 vm_page_test_dirty(p); 2656 if ((p->valid & p->dirty) || (p->flags & PG_NEED_COMMIT)) { 2657 vm_page_wakeup(p); 2658 return(0); 2659 } 2660 } 2661 2662 /* 2663 * Destroy the page 2664 */ 2665 vm_page_protect(p, VM_PROT_NONE); 2666 vm_page_free(p); 2667 return(0); 2668 } 2669 2670 /* 2671 * Coalesces two objects backing up adjoining regions of memory into a 2672 * single object. 2673 * 2674 * returns TRUE if objects were combined. 2675 * 2676 * NOTE: Only works at the moment if the second object is NULL - 2677 * if it's not, which object do we lock first? 2678 * 2679 * Parameters: 2680 * prev_object First object to coalesce 2681 * prev_offset Offset into prev_object 2682 * next_object Second object into coalesce 2683 * next_offset Offset into next_object 2684 * 2685 * prev_size Size of reference to prev_object 2686 * next_size Size of reference to next_object 2687 * 2688 * The caller does not need to hold (prev_object) but must have a stable 2689 * pointer to it (typically by holding the vm_map locked). 2690 */ 2691 boolean_t 2692 vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex, 2693 vm_size_t prev_size, vm_size_t next_size) 2694 { 2695 vm_pindex_t next_pindex; 2696 2697 if (prev_object == NULL) 2698 return (TRUE); 2699 2700 vm_object_hold(prev_object); 2701 2702 if (prev_object->type != OBJT_DEFAULT && 2703 prev_object->type != OBJT_SWAP) { 2704 vm_object_drop(prev_object); 2705 return (FALSE); 2706 } 2707 2708 /* 2709 * Try to collapse the object first 2710 */ 2711 vm_object_chain_acquire(prev_object, 0); 2712 vm_object_collapse(prev_object, NULL); 2713 2714 /* 2715 * Can't coalesce if: . more than one reference . paged out . shadows 2716 * another object . has a copy elsewhere (any of which mean that the 2717 * pages not mapped to prev_entry may be in use anyway) 2718 */ 2719 2720 if (prev_object->backing_object != NULL) { 2721 vm_object_chain_release(prev_object); 2722 vm_object_drop(prev_object); 2723 return (FALSE); 2724 } 2725 2726 prev_size >>= PAGE_SHIFT; 2727 next_size >>= PAGE_SHIFT; 2728 next_pindex = prev_pindex + prev_size; 2729 2730 if ((prev_object->ref_count > 1) && 2731 (prev_object->size != next_pindex)) { 2732 vm_object_chain_release(prev_object); 2733 vm_object_drop(prev_object); 2734 return (FALSE); 2735 } 2736 2737 /* 2738 * Remove any pages that may still be in the object from a previous 2739 * deallocation. 2740 */ 2741 if (next_pindex < prev_object->size) { 2742 vm_object_page_remove(prev_object, 2743 next_pindex, 2744 next_pindex + next_size, FALSE); 2745 if (prev_object->type == OBJT_SWAP) 2746 swap_pager_freespace(prev_object, 2747 next_pindex, next_size); 2748 } 2749 2750 /* 2751 * Extend the object if necessary. 2752 */ 2753 if (next_pindex + next_size > prev_object->size) 2754 prev_object->size = next_pindex + next_size; 2755 2756 vm_object_chain_release(prev_object); 2757 vm_object_drop(prev_object); 2758 return (TRUE); 2759 } 2760 2761 /* 2762 * Make the object writable and flag is being possibly dirty. 2763 * 2764 * The object might not be held (or might be held but held shared), 2765 * the related vnode is probably not held either. Object and vnode are 2766 * stable by virtue of the vm_page busied by the caller preventing 2767 * destruction. 2768 * 2769 * If the related mount is flagged MNTK_THR_SYNC we need to call 2770 * vsetobjdirty(). Filesystems using this option usually shortcut 2771 * synchronization by only scanning the syncer list. 2772 */ 2773 void 2774 vm_object_set_writeable_dirty(vm_object_t object) 2775 { 2776 struct vnode *vp; 2777 2778 /*vm_object_assert_held(object);*/ 2779 /* 2780 * Avoid contention in vm fault path by checking the state before 2781 * issuing an atomic op on it. 2782 */ 2783 if ((object->flags & (OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY)) != 2784 (OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY)) { 2785 vm_object_set_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 2786 } 2787 if (object->type == OBJT_VNODE && 2788 (vp = (struct vnode *)object->handle) != NULL) { 2789 if ((vp->v_flag & VOBJDIRTY) == 0) { 2790 if (vp->v_mount && 2791 (vp->v_mount->mnt_kern_flag & MNTK_THR_SYNC)) { 2792 /* 2793 * New style THR_SYNC places vnodes on the 2794 * syncer list more deterministically. 2795 */ 2796 vsetobjdirty(vp); 2797 } else { 2798 /* 2799 * Old style scan would not necessarily place 2800 * a vnode on the syncer list when possibly 2801 * modified via mmap. 2802 */ 2803 vsetflags(vp, VOBJDIRTY); 2804 } 2805 } 2806 } 2807 } 2808 2809 #include "opt_ddb.h" 2810 #ifdef DDB 2811 #include <sys/kernel.h> 2812 2813 #include <sys/cons.h> 2814 2815 #include <ddb/ddb.h> 2816 2817 static int _vm_object_in_map (vm_map_t map, vm_object_t object, 2818 vm_map_entry_t entry); 2819 static int vm_object_in_map (vm_object_t object); 2820 2821 /* 2822 * The caller must hold the object. 2823 */ 2824 static int 2825 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry) 2826 { 2827 vm_map_t tmpm; 2828 vm_map_entry_t tmpe; 2829 vm_object_t obj, nobj; 2830 int entcount; 2831 2832 if (map == 0) 2833 return 0; 2834 if (entry == 0) { 2835 tmpe = map->header.next; 2836 entcount = map->nentries; 2837 while (entcount-- && (tmpe != &map->header)) { 2838 if( _vm_object_in_map(map, object, tmpe)) { 2839 return 1; 2840 } 2841 tmpe = tmpe->next; 2842 } 2843 return (0); 2844 } 2845 switch(entry->maptype) { 2846 case VM_MAPTYPE_SUBMAP: 2847 tmpm = entry->object.sub_map; 2848 tmpe = tmpm->header.next; 2849 entcount = tmpm->nentries; 2850 while (entcount-- && tmpe != &tmpm->header) { 2851 if( _vm_object_in_map(tmpm, object, tmpe)) { 2852 return 1; 2853 } 2854 tmpe = tmpe->next; 2855 } 2856 break; 2857 case VM_MAPTYPE_NORMAL: 2858 case VM_MAPTYPE_VPAGETABLE: 2859 obj = entry->object.vm_object; 2860 while (obj) { 2861 if (obj == object) { 2862 if (obj != entry->object.vm_object) 2863 vm_object_drop(obj); 2864 return 1; 2865 } 2866 while ((nobj = obj->backing_object) != NULL) { 2867 vm_object_hold(nobj); 2868 if (nobj == obj->backing_object) 2869 break; 2870 vm_object_drop(nobj); 2871 } 2872 if (obj != entry->object.vm_object) { 2873 if (nobj) 2874 vm_object_lock_swap(); 2875 vm_object_drop(obj); 2876 } 2877 obj = nobj; 2878 } 2879 break; 2880 default: 2881 break; 2882 } 2883 return 0; 2884 } 2885 2886 static int vm_object_in_map_callback(struct proc *p, void *data); 2887 2888 struct vm_object_in_map_info { 2889 vm_object_t object; 2890 int rv; 2891 }; 2892 2893 /* 2894 * Debugging only 2895 */ 2896 static int 2897 vm_object_in_map(vm_object_t object) 2898 { 2899 struct vm_object_in_map_info info; 2900 2901 info.rv = 0; 2902 info.object = object; 2903 2904 allproc_scan(vm_object_in_map_callback, &info); 2905 if (info.rv) 2906 return 1; 2907 if( _vm_object_in_map(&kernel_map, object, 0)) 2908 return 1; 2909 if( _vm_object_in_map(&pager_map, object, 0)) 2910 return 1; 2911 if( _vm_object_in_map(&buffer_map, object, 0)) 2912 return 1; 2913 return 0; 2914 } 2915 2916 /* 2917 * Debugging only 2918 */ 2919 static int 2920 vm_object_in_map_callback(struct proc *p, void *data) 2921 { 2922 struct vm_object_in_map_info *info = data; 2923 2924 if (p->p_vmspace) { 2925 if (_vm_object_in_map(&p->p_vmspace->vm_map, info->object, 0)) { 2926 info->rv = 1; 2927 return -1; 2928 } 2929 } 2930 return (0); 2931 } 2932 2933 DB_SHOW_COMMAND(vmochk, vm_object_check) 2934 { 2935 vm_object_t object; 2936 int n; 2937 2938 /* 2939 * make sure that internal objs are in a map somewhere 2940 * and none have zero ref counts. 2941 */ 2942 for (n = 0; n < VMOBJ_HSIZE; ++n) { 2943 for (object = TAILQ_FIRST(&vm_object_lists[n]); 2944 object != NULL; 2945 object = TAILQ_NEXT(object, object_list)) { 2946 if (object->type == OBJT_MARKER) 2947 continue; 2948 if (object->handle != NULL || 2949 (object->type != OBJT_DEFAULT && 2950 object->type != OBJT_SWAP)) { 2951 continue; 2952 } 2953 if (object->ref_count == 0) { 2954 db_printf("vmochk: internal obj has " 2955 "zero ref count: %ld\n", 2956 (long)object->size); 2957 } 2958 if (vm_object_in_map(object)) 2959 continue; 2960 db_printf("vmochk: internal obj is not in a map: " 2961 "ref: %d, size: %lu: 0x%lx, " 2962 "backing_object: %p\n", 2963 object->ref_count, (u_long)object->size, 2964 (u_long)object->size, 2965 (void *)object->backing_object); 2966 } 2967 } 2968 } 2969 2970 /* 2971 * Debugging only 2972 */ 2973 DB_SHOW_COMMAND(object, vm_object_print_static) 2974 { 2975 /* XXX convert args. */ 2976 vm_object_t object = (vm_object_t)addr; 2977 boolean_t full = have_addr; 2978 2979 vm_page_t p; 2980 2981 /* XXX count is an (unused) arg. Avoid shadowing it. */ 2982 #define count was_count 2983 2984 int count; 2985 2986 if (object == NULL) 2987 return; 2988 2989 db_iprintf( 2990 "Object %p: type=%d, size=0x%lx, res=%d, ref=%d, flags=0x%x\n", 2991 object, (int)object->type, (u_long)object->size, 2992 object->resident_page_count, object->ref_count, object->flags); 2993 /* 2994 * XXX no %qd in kernel. Truncate object->backing_object_offset. 2995 */ 2996 db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%lx\n", 2997 object->shadow_count, 2998 object->backing_object ? object->backing_object->ref_count : 0, 2999 object->backing_object, (long)object->backing_object_offset); 3000 3001 if (!full) 3002 return; 3003 3004 db_indent += 2; 3005 count = 0; 3006 RB_FOREACH(p, vm_page_rb_tree, &object->rb_memq) { 3007 if (count == 0) 3008 db_iprintf("memory:="); 3009 else if (count == 6) { 3010 db_printf("\n"); 3011 db_iprintf(" ..."); 3012 count = 0; 3013 } else 3014 db_printf(","); 3015 count++; 3016 3017 db_printf("(off=0x%lx,page=0x%lx)", 3018 (u_long) p->pindex, (u_long) VM_PAGE_TO_PHYS(p)); 3019 } 3020 if (count != 0) 3021 db_printf("\n"); 3022 db_indent -= 2; 3023 } 3024 3025 /* XXX. */ 3026 #undef count 3027 3028 /* 3029 * XXX need this non-static entry for calling from vm_map_print. 3030 * 3031 * Debugging only 3032 */ 3033 void 3034 vm_object_print(/* db_expr_t */ long addr, 3035 boolean_t have_addr, 3036 /* db_expr_t */ long count, 3037 char *modif) 3038 { 3039 vm_object_print_static(addr, have_addr, count, modif); 3040 } 3041 3042 /* 3043 * Debugging only 3044 */ 3045 DB_SHOW_COMMAND(vmopag, vm_object_print_pages) 3046 { 3047 vm_object_t object; 3048 int nl = 0; 3049 int c; 3050 int n; 3051 3052 for (n = 0; n < VMOBJ_HSIZE; ++n) { 3053 for (object = TAILQ_FIRST(&vm_object_lists[n]); 3054 object != NULL; 3055 object = TAILQ_NEXT(object, object_list)) { 3056 vm_pindex_t idx, fidx; 3057 vm_pindex_t osize; 3058 vm_paddr_t pa = -1, padiff; 3059 int rcount; 3060 vm_page_t m; 3061 3062 if (object->type == OBJT_MARKER) 3063 continue; 3064 db_printf("new object: %p\n", (void *)object); 3065 if ( nl > 18) { 3066 c = cngetc(); 3067 if (c != ' ') 3068 return; 3069 nl = 0; 3070 } 3071 nl++; 3072 rcount = 0; 3073 fidx = 0; 3074 osize = object->size; 3075 if (osize > 128) 3076 osize = 128; 3077 for (idx = 0; idx < osize; idx++) { 3078 m = vm_page_lookup(object, idx); 3079 if (m == NULL) { 3080 if (rcount) { 3081 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 3082 (long)fidx, rcount, (long)pa); 3083 if ( nl > 18) { 3084 c = cngetc(); 3085 if (c != ' ') 3086 return; 3087 nl = 0; 3088 } 3089 nl++; 3090 rcount = 0; 3091 } 3092 continue; 3093 } 3094 3095 if (rcount && 3096 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) { 3097 ++rcount; 3098 continue; 3099 } 3100 if (rcount) { 3101 padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m); 3102 padiff >>= PAGE_SHIFT; 3103 padiff &= PQ_L2_MASK; 3104 if (padiff == 0) { 3105 pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE; 3106 ++rcount; 3107 continue; 3108 } 3109 db_printf(" index(%ld)run(%d)pa(0x%lx)", 3110 (long)fidx, rcount, (long)pa); 3111 db_printf("pd(%ld)\n", (long)padiff); 3112 if ( nl > 18) { 3113 c = cngetc(); 3114 if (c != ' ') 3115 return; 3116 nl = 0; 3117 } 3118 nl++; 3119 } 3120 fidx = idx; 3121 pa = VM_PAGE_TO_PHYS(m); 3122 rcount = 1; 3123 } 3124 if (rcount) { 3125 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 3126 (long)fidx, rcount, (long)pa); 3127 if ( nl > 18) { 3128 c = cngetc(); 3129 if (c != ' ') 3130 return; 3131 nl = 0; 3132 } 3133 nl++; 3134 } 3135 } 3136 } 3137 } 3138 #endif /* DDB */ 3139