1 /* 2 * Copyright (c) 1991, 1993, 2013 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 33 * 34 * 35 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 36 * All rights reserved. 37 * 38 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 39 * 40 * Permission to use, copy, modify and distribute this software and 41 * its documentation is hereby granted, provided that both the copyright 42 * notice and this permission notice appear in all copies of the 43 * software, derivative works or modified versions, and any portions 44 * thereof, and that both notices appear in supporting documentation. 45 * 46 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 47 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 48 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 49 * 50 * Carnegie Mellon requests users of this software to return to 51 * 52 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 53 * School of Computer Science 54 * Carnegie Mellon University 55 * Pittsburgh PA 15213-3890 56 * 57 * any improvements or extensions that they make and grant Carnegie the 58 * rights to redistribute these changes. 59 * 60 * $FreeBSD: src/sys/vm/vm_object.c,v 1.171.2.8 2003/05/26 19:17:56 alc Exp $ 61 */ 62 63 /* 64 * Virtual memory object module. 65 */ 66 67 #include <sys/param.h> 68 #include <sys/systm.h> 69 #include <sys/proc.h> /* for curproc, pageproc */ 70 #include <sys/thread.h> 71 #include <sys/vnode.h> 72 #include <sys/vmmeter.h> 73 #include <sys/mman.h> 74 #include <sys/mount.h> 75 #include <sys/kernel.h> 76 #include <sys/sysctl.h> 77 #include <sys/refcount.h> 78 79 #include <vm/vm.h> 80 #include <vm/vm_param.h> 81 #include <vm/pmap.h> 82 #include <vm/vm_map.h> 83 #include <vm/vm_object.h> 84 #include <vm/vm_page.h> 85 #include <vm/vm_pageout.h> 86 #include <vm/vm_pager.h> 87 #include <vm/swap_pager.h> 88 #include <vm/vm_kern.h> 89 #include <vm/vm_extern.h> 90 #include <vm/vm_zone.h> 91 92 #include <vm/vm_page2.h> 93 94 #include <machine/specialreg.h> 95 96 #define EASY_SCAN_FACTOR 8 97 98 static void vm_object_qcollapse(vm_object_t object, 99 vm_object_t backing_object); 100 static void vm_object_page_collect_flush(vm_object_t object, vm_page_t p, 101 int pagerflags); 102 static void vm_object_lock_init(vm_object_t); 103 104 105 /* 106 * Virtual memory objects maintain the actual data 107 * associated with allocated virtual memory. A given 108 * page of memory exists within exactly one object. 109 * 110 * An object is only deallocated when all "references" 111 * are given up. Only one "reference" to a given 112 * region of an object should be writeable. 113 * 114 * Associated with each object is a list of all resident 115 * memory pages belonging to that object; this list is 116 * maintained by the "vm_page" module, and locked by the object's 117 * lock. 118 * 119 * Each object also records a "pager" routine which is 120 * used to retrieve (and store) pages to the proper backing 121 * storage. In addition, objects may be backed by other 122 * objects from which they were virtual-copied. 123 * 124 * The only items within the object structure which are 125 * modified after time of creation are: 126 * reference count locked by object's lock 127 * pager routine locked by object's lock 128 * 129 */ 130 131 struct vm_object kernel_object; 132 133 static long object_collapses; 134 static long object_bypasses; 135 136 struct vm_object_hash vm_object_hash[VMOBJ_HSIZE]; 137 138 MALLOC_DEFINE(M_VM_OBJECT, "vm_object", "vm_object structures"); 139 140 #if defined(DEBUG_LOCKS) 141 142 #define vm_object_vndeallocate(obj, vpp) \ 143 debugvm_object_vndeallocate(obj, vpp, __FILE__, __LINE__) 144 145 /* 146 * Debug helper to track hold/drop/ref/deallocate calls. 147 */ 148 static void 149 debugvm_object_add(vm_object_t obj, char *file, int line, int addrem) 150 { 151 int i; 152 153 i = atomic_fetchadd_int(&obj->debug_index, 1); 154 i = i & (VMOBJ_DEBUG_ARRAY_SIZE - 1); 155 ksnprintf(obj->debug_hold_thrs[i], 156 sizeof(obj->debug_hold_thrs[i]), 157 "%c%d:(%d):%s", 158 (addrem == -1 ? '-' : (addrem == 1 ? '+' : '=')), 159 (curthread->td_proc ? curthread->td_proc->p_pid : -1), 160 obj->ref_count, 161 curthread->td_comm); 162 obj->debug_hold_file[i] = file; 163 obj->debug_hold_line[i] = line; 164 #if 0 165 /* Uncomment for debugging obj refs/derefs in reproducable cases */ 166 if (strcmp(curthread->td_comm, "sshd") == 0) { 167 kprintf("%d %p refs=%d ar=%d file: %s/%d\n", 168 (curthread->td_proc ? curthread->td_proc->p_pid : -1), 169 obj, obj->ref_count, addrem, file, line); 170 } 171 #endif 172 } 173 174 #endif 175 176 /* 177 * Misc low level routines 178 */ 179 static void 180 vm_object_lock_init(vm_object_t obj) 181 { 182 #if defined(DEBUG_LOCKS) 183 int i; 184 185 obj->debug_index = 0; 186 for (i = 0; i < VMOBJ_DEBUG_ARRAY_SIZE; i++) { 187 obj->debug_hold_thrs[i][0] = 0; 188 obj->debug_hold_file[i] = NULL; 189 obj->debug_hold_line[i] = 0; 190 } 191 #endif 192 } 193 194 void 195 vm_object_lock_swap(void) 196 { 197 lwkt_token_swap(); 198 } 199 200 void 201 vm_object_lock(vm_object_t obj) 202 { 203 lwkt_gettoken(&obj->token); 204 } 205 206 /* 207 * Returns TRUE on sucesss 208 */ 209 static int 210 vm_object_lock_try(vm_object_t obj) 211 { 212 return(lwkt_trytoken(&obj->token)); 213 } 214 215 void 216 vm_object_lock_shared(vm_object_t obj) 217 { 218 lwkt_gettoken_shared(&obj->token); 219 } 220 221 void 222 vm_object_unlock(vm_object_t obj) 223 { 224 lwkt_reltoken(&obj->token); 225 } 226 227 void 228 vm_object_upgrade(vm_object_t obj) 229 { 230 lwkt_reltoken(&obj->token); 231 lwkt_gettoken(&obj->token); 232 } 233 234 void 235 vm_object_downgrade(vm_object_t obj) 236 { 237 lwkt_reltoken(&obj->token); 238 lwkt_gettoken_shared(&obj->token); 239 } 240 241 static __inline void 242 vm_object_assert_held(vm_object_t obj) 243 { 244 ASSERT_LWKT_TOKEN_HELD(&obj->token); 245 } 246 247 static __inline int 248 vm_quickcolor(void) 249 { 250 globaldata_t gd = mycpu; 251 int pg_color; 252 253 pg_color = (int)(intptr_t)gd->gd_curthread >> 10; 254 pg_color += gd->gd_quick_color; 255 gd->gd_quick_color += PQ_PRIME2; 256 257 return pg_color; 258 } 259 260 void 261 VMOBJDEBUG(vm_object_hold)(vm_object_t obj VMOBJDBARGS) 262 { 263 KKASSERT(obj != NULL); 264 265 /* 266 * Object must be held (object allocation is stable due to callers 267 * context, typically already holding the token on a parent object) 268 * prior to potentially blocking on the lock, otherwise the object 269 * can get ripped away from us. 270 */ 271 refcount_acquire(&obj->hold_count); 272 vm_object_lock(obj); 273 274 #if defined(DEBUG_LOCKS) 275 debugvm_object_add(obj, file, line, 1); 276 #endif 277 } 278 279 int 280 VMOBJDEBUG(vm_object_hold_try)(vm_object_t obj VMOBJDBARGS) 281 { 282 KKASSERT(obj != NULL); 283 284 /* 285 * Object must be held (object allocation is stable due to callers 286 * context, typically already holding the token on a parent object) 287 * prior to potentially blocking on the lock, otherwise the object 288 * can get ripped away from us. 289 */ 290 refcount_acquire(&obj->hold_count); 291 if (vm_object_lock_try(obj) == 0) { 292 if (refcount_release(&obj->hold_count)) { 293 if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD)) 294 kfree(obj, M_VM_OBJECT); 295 } 296 return(0); 297 } 298 299 #if defined(DEBUG_LOCKS) 300 debugvm_object_add(obj, file, line, 1); 301 #endif 302 return(1); 303 } 304 305 void 306 VMOBJDEBUG(vm_object_hold_shared)(vm_object_t obj VMOBJDBARGS) 307 { 308 KKASSERT(obj != NULL); 309 310 /* 311 * Object must be held (object allocation is stable due to callers 312 * context, typically already holding the token on a parent object) 313 * prior to potentially blocking on the lock, otherwise the object 314 * can get ripped away from us. 315 */ 316 refcount_acquire(&obj->hold_count); 317 vm_object_lock_shared(obj); 318 319 #if defined(DEBUG_LOCKS) 320 debugvm_object_add(obj, file, line, 1); 321 #endif 322 } 323 324 /* 325 * Drop the token and hold_count on the object. 326 * 327 * WARNING! Token might be shared. 328 */ 329 void 330 VMOBJDEBUG(vm_object_drop)(vm_object_t obj VMOBJDBARGS) 331 { 332 if (obj == NULL) 333 return; 334 335 /* 336 * No new holders should be possible once we drop hold_count 1->0 as 337 * there is no longer any way to reference the object. 338 */ 339 KKASSERT(obj->hold_count > 0); 340 if (refcount_release(&obj->hold_count)) { 341 #if defined(DEBUG_LOCKS) 342 debugvm_object_add(obj, file, line, -1); 343 #endif 344 345 if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD)) { 346 vm_object_unlock(obj); 347 kfree(obj, M_VM_OBJECT); 348 } else { 349 vm_object_unlock(obj); 350 } 351 } else { 352 #if defined(DEBUG_LOCKS) 353 debugvm_object_add(obj, file, line, -1); 354 #endif 355 vm_object_unlock(obj); 356 } 357 } 358 359 /* 360 * Initialize a freshly allocated object, returning a held object. 361 * 362 * Used only by vm_object_allocate(), zinitna() and vm_object_init(). 363 * 364 * No requirements. 365 */ 366 void 367 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object) 368 { 369 struct vm_object_hash *hash; 370 371 RB_INIT(&object->rb_memq); 372 LIST_INIT(&object->shadow_head); 373 lwkt_token_init(&object->token, "vmobj"); 374 375 object->type = type; 376 object->size = size; 377 object->ref_count = 1; 378 object->memattr = VM_MEMATTR_DEFAULT; 379 object->hold_count = 0; 380 object->flags = 0; 381 if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP)) 382 vm_object_set_flag(object, OBJ_ONEMAPPING); 383 object->paging_in_progress = 0; 384 object->resident_page_count = 0; 385 object->shadow_count = 0; 386 /* cpu localization twist */ 387 object->pg_color = vm_quickcolor(); 388 object->handle = NULL; 389 object->backing_object = NULL; 390 object->backing_object_offset = (vm_ooffset_t)0; 391 392 atomic_add_int(&object->generation, 1); 393 object->swblock_count = 0; 394 RB_INIT(&object->swblock_root); 395 vm_object_lock_init(object); 396 pmap_object_init(object); 397 398 vm_object_hold(object); 399 400 hash = VMOBJ_HASH(object); 401 lwkt_gettoken(&hash->token); 402 TAILQ_INSERT_TAIL(&hash->list, object, object_list); 403 lwkt_reltoken(&hash->token); 404 } 405 406 /* 407 * Initialize a VM object. 408 */ 409 void 410 vm_object_init(vm_object_t object, vm_pindex_t size) 411 { 412 _vm_object_allocate(OBJT_DEFAULT, size, object); 413 vm_object_drop(object); 414 } 415 416 /* 417 * Initialize the VM objects module. 418 * 419 * Called from the low level boot code only. Note that this occurs before 420 * kmalloc is initialized so we cannot allocate any VM objects. 421 */ 422 void 423 vm_object_init1(void) 424 { 425 int i; 426 427 for (i = 0; i < VMOBJ_HSIZE; ++i) { 428 TAILQ_INIT(&vm_object_hash[i].list); 429 lwkt_token_init(&vm_object_hash[i].token, "vmobjlst"); 430 } 431 432 _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(KvaEnd), 433 &kernel_object); 434 vm_object_drop(&kernel_object); 435 } 436 437 void 438 vm_object_init2(void) 439 { 440 kmalloc_set_unlimited(M_VM_OBJECT); 441 } 442 443 /* 444 * Allocate and return a new object of the specified type and size. 445 * 446 * No requirements. 447 */ 448 vm_object_t 449 vm_object_allocate(objtype_t type, vm_pindex_t size) 450 { 451 vm_object_t obj; 452 453 obj = kmalloc(sizeof(*obj), M_VM_OBJECT, M_INTWAIT|M_ZERO); 454 _vm_object_allocate(type, size, obj); 455 vm_object_drop(obj); 456 457 return (obj); 458 } 459 460 /* 461 * This version returns a held object, allowing further atomic initialization 462 * of the object. 463 */ 464 vm_object_t 465 vm_object_allocate_hold(objtype_t type, vm_pindex_t size) 466 { 467 vm_object_t obj; 468 469 obj = kmalloc(sizeof(*obj), M_VM_OBJECT, M_INTWAIT|M_ZERO); 470 _vm_object_allocate(type, size, obj); 471 472 return (obj); 473 } 474 475 /* 476 * Add an additional reference to a vm_object. The object must already be 477 * held. The original non-lock version is no longer supported. The object 478 * must NOT be chain locked by anyone at the time the reference is added. 479 * 480 * Referencing a chain-locked object can blow up the fairly sensitive 481 * ref_count and shadow_count tests in the deallocator. Most callers 482 * will call vm_object_chain_wait() prior to calling 483 * vm_object_reference_locked() to avoid the case. 484 * 485 * The object must be held, but may be held shared if desired (hence why 486 * we use an atomic op). 487 */ 488 void 489 VMOBJDEBUG(vm_object_reference_locked)(vm_object_t object VMOBJDBARGS) 490 { 491 KKASSERT(object != NULL); 492 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 493 KKASSERT((object->chainlk & (CHAINLK_EXCL | CHAINLK_MASK)) == 0); 494 atomic_add_int(&object->ref_count, 1); 495 if (object->type == OBJT_VNODE) { 496 vref(object->handle); 497 /* XXX what if the vnode is being destroyed? */ 498 } 499 #if defined(DEBUG_LOCKS) 500 debugvm_object_add(object, file, line, 1); 501 #endif 502 } 503 504 /* 505 * This version is only allowed for vnode objects. 506 */ 507 void 508 VMOBJDEBUG(vm_object_reference_quick)(vm_object_t object VMOBJDBARGS) 509 { 510 KKASSERT(object->type == OBJT_VNODE); 511 atomic_add_int(&object->ref_count, 1); 512 vref(object->handle); 513 #if defined(DEBUG_LOCKS) 514 debugvm_object_add(object, file, line, 1); 515 #endif 516 } 517 518 /* 519 * Object OBJ_CHAINLOCK lock handling. 520 * 521 * The caller can chain-lock backing objects recursively and then 522 * use vm_object_chain_release_all() to undo the whole chain. 523 * 524 * Chain locks are used to prevent collapses and are only applicable 525 * to OBJT_DEFAULT and OBJT_SWAP objects. Chain locking operations 526 * on other object types are ignored. This is also important because 527 * it allows e.g. the vnode underlying a memory mapping to take concurrent 528 * faults. 529 * 530 * The object must usually be held on entry, though intermediate 531 * objects need not be held on release. The object must be held exclusively, 532 * NOT shared. Note that the prefault path checks the shared state and 533 * avoids using the chain functions. 534 */ 535 void 536 vm_object_chain_wait(vm_object_t object, int shared) 537 { 538 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 539 for (;;) { 540 uint32_t chainlk = object->chainlk; 541 542 cpu_ccfence(); 543 if (shared) { 544 if (chainlk & (CHAINLK_EXCL | CHAINLK_EXCLREQ)) { 545 tsleep_interlock(object, 0); 546 if (atomic_cmpset_int(&object->chainlk, 547 chainlk, 548 chainlk | CHAINLK_WAIT)) { 549 tsleep(object, PINTERLOCKED, 550 "objchns", 0); 551 } 552 /* retry */ 553 } else { 554 break; 555 } 556 /* retry */ 557 } else { 558 if (chainlk & (CHAINLK_MASK | CHAINLK_EXCL)) { 559 tsleep_interlock(object, 0); 560 if (atomic_cmpset_int(&object->chainlk, 561 chainlk, 562 chainlk | CHAINLK_WAIT)) 563 { 564 tsleep(object, PINTERLOCKED, 565 "objchnx", 0); 566 } 567 /* retry */ 568 } else { 569 if (atomic_cmpset_int(&object->chainlk, 570 chainlk, 571 chainlk & ~CHAINLK_WAIT)) 572 { 573 if (chainlk & CHAINLK_WAIT) 574 wakeup(object); 575 break; 576 } 577 /* retry */ 578 } 579 } 580 /* retry */ 581 } 582 } 583 584 void 585 vm_object_chain_acquire(vm_object_t object, int shared) 586 { 587 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP) 588 return; 589 if (vm_shared_fault == 0) 590 shared = 0; 591 592 for (;;) { 593 uint32_t chainlk = object->chainlk; 594 595 cpu_ccfence(); 596 if (shared) { 597 if (chainlk & (CHAINLK_EXCL | CHAINLK_EXCLREQ)) { 598 tsleep_interlock(object, 0); 599 if (atomic_cmpset_int(&object->chainlk, 600 chainlk, 601 chainlk | CHAINLK_WAIT)) { 602 tsleep(object, PINTERLOCKED, 603 "objchns", 0); 604 } 605 /* retry */ 606 } else if (atomic_cmpset_int(&object->chainlk, 607 chainlk, chainlk + 1)) { 608 break; 609 } 610 /* retry */ 611 } else { 612 if (chainlk & (CHAINLK_MASK | CHAINLK_EXCL)) { 613 tsleep_interlock(object, 0); 614 if (atomic_cmpset_int(&object->chainlk, 615 chainlk, 616 chainlk | 617 CHAINLK_WAIT | 618 CHAINLK_EXCLREQ)) { 619 tsleep(object, PINTERLOCKED, 620 "objchnx", 0); 621 } 622 /* retry */ 623 } else { 624 if (atomic_cmpset_int(&object->chainlk, 625 chainlk, 626 (chainlk | CHAINLK_EXCL) & 627 ~(CHAINLK_EXCLREQ | 628 CHAINLK_WAIT))) { 629 if (chainlk & CHAINLK_WAIT) 630 wakeup(object); 631 break; 632 } 633 /* retry */ 634 } 635 } 636 /* retry */ 637 } 638 } 639 640 void 641 vm_object_chain_release(vm_object_t object) 642 { 643 /*ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));*/ 644 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP) 645 return; 646 KKASSERT(object->chainlk & (CHAINLK_MASK | CHAINLK_EXCL)); 647 for (;;) { 648 uint32_t chainlk = object->chainlk; 649 650 cpu_ccfence(); 651 if (chainlk & CHAINLK_MASK) { 652 if ((chainlk & CHAINLK_MASK) == 1 && 653 atomic_cmpset_int(&object->chainlk, 654 chainlk, 655 (chainlk - 1) & ~CHAINLK_WAIT)) { 656 if (chainlk & CHAINLK_WAIT) 657 wakeup(object); 658 break; 659 } 660 if ((chainlk & CHAINLK_MASK) > 1 && 661 atomic_cmpset_int(&object->chainlk, 662 chainlk, chainlk - 1)) { 663 break; 664 } 665 /* retry */ 666 } else { 667 KKASSERT(chainlk & CHAINLK_EXCL); 668 if (atomic_cmpset_int(&object->chainlk, 669 chainlk, 670 chainlk & ~(CHAINLK_EXCL | 671 CHAINLK_WAIT))) { 672 if (chainlk & CHAINLK_WAIT) 673 wakeup(object); 674 break; 675 } 676 } 677 } 678 } 679 680 /* 681 * Release the chain from first_object through and including stopobj. 682 * The caller is typically holding the first and last object locked 683 * (shared or exclusive) to prevent destruction races. 684 * 685 * We release stopobj first as an optimization as this object is most 686 * likely to be shared across multiple processes. 687 */ 688 void 689 vm_object_chain_release_all(vm_object_t first_object, vm_object_t stopobj) 690 { 691 vm_object_t backing_object; 692 vm_object_t object; 693 694 vm_object_chain_release(stopobj); 695 object = first_object; 696 697 while (object != stopobj) { 698 KKASSERT(object); 699 backing_object = object->backing_object; 700 vm_object_chain_release(object); 701 object = backing_object; 702 } 703 } 704 705 /* 706 * Dereference an object and its underlying vnode. The object may be 707 * held shared. On return the object will remain held. 708 * 709 * This function may return a vnode in *vpp which the caller must release 710 * after the caller drops its own lock. If vpp is NULL, we assume that 711 * the caller was holding an exclusive lock on the object and we vrele() 712 * the vp ourselves. 713 */ 714 static void 715 VMOBJDEBUG(vm_object_vndeallocate)(vm_object_t object, struct vnode **vpp 716 VMOBJDBARGS) 717 { 718 struct vnode *vp = (struct vnode *) object->handle; 719 720 KASSERT(object->type == OBJT_VNODE, 721 ("vm_object_vndeallocate: not a vnode object")); 722 KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); 723 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 724 #ifdef INVARIANTS 725 if (object->ref_count == 0) { 726 vprint("vm_object_vndeallocate", vp); 727 panic("vm_object_vndeallocate: bad object reference count"); 728 } 729 #endif 730 for (;;) { 731 int count = object->ref_count; 732 cpu_ccfence(); 733 if (count == 1) { 734 vm_object_upgrade(object); 735 if (atomic_cmpset_int(&object->ref_count, count, 0)) { 736 vclrflags(vp, VTEXT); 737 break; 738 } 739 } else { 740 if (atomic_cmpset_int(&object->ref_count, 741 count, count - 1)) { 742 break; 743 } 744 } 745 /* retry */ 746 } 747 #if defined(DEBUG_LOCKS) 748 debugvm_object_add(object, file, line, -1); 749 #endif 750 751 /* 752 * vrele or return the vp to vrele. We can only safely vrele(vp) 753 * if the object was locked exclusively. But there are two races 754 * here. 755 * 756 * We had to upgrade the object above to safely clear VTEXT 757 * but the alternative path where the shared lock is retained 758 * can STILL race to 0 in other paths and cause our own vrele() 759 * to terminate the vnode. We can't allow that if the VM object 760 * is still locked shared. 761 */ 762 if (vpp) 763 *vpp = vp; 764 else 765 vrele(vp); 766 } 767 768 /* 769 * Release a reference to the specified object, gained either through a 770 * vm_object_allocate or a vm_object_reference call. When all references 771 * are gone, storage associated with this object may be relinquished. 772 * 773 * The caller does not have to hold the object locked but must have control 774 * over the reference in question in order to guarantee that the object 775 * does not get ripped out from under us. 776 * 777 * XXX Currently all deallocations require an exclusive lock. 778 */ 779 void 780 VMOBJDEBUG(vm_object_deallocate)(vm_object_t object VMOBJDBARGS) 781 { 782 struct vnode *vp; 783 int count; 784 785 if (object == NULL) 786 return; 787 788 for (;;) { 789 count = object->ref_count; 790 cpu_ccfence(); 791 792 /* 793 * If decrementing the count enters into special handling 794 * territory (0, 1, or 2) we have to do it the hard way. 795 * Fortunate though, objects with only a few refs like this 796 * are not likely to be heavily contended anyway. 797 * 798 * For vnode objects we only care about 1->0 transitions. 799 */ 800 if (count <= 3 || (object->type == OBJT_VNODE && count <= 1)) { 801 #if defined(DEBUG_LOCKS) 802 debugvm_object_add(object, file, line, 0); 803 #endif 804 vm_object_hold(object); 805 vm_object_deallocate_locked(object); 806 vm_object_drop(object); 807 break; 808 } 809 810 /* 811 * Try to decrement ref_count without acquiring a hold on 812 * the object. This is particularly important for the exec*() 813 * and exit*() code paths because the program binary may 814 * have a great deal of sharing and an exclusive lock will 815 * crowbar performance in those circumstances. 816 */ 817 if (object->type == OBJT_VNODE) { 818 vp = (struct vnode *)object->handle; 819 if (atomic_cmpset_int(&object->ref_count, 820 count, count - 1)) { 821 #if defined(DEBUG_LOCKS) 822 debugvm_object_add(object, file, line, -1); 823 #endif 824 825 vrele(vp); 826 break; 827 } 828 /* retry */ 829 } else { 830 if (atomic_cmpset_int(&object->ref_count, 831 count, count - 1)) { 832 #if defined(DEBUG_LOCKS) 833 debugvm_object_add(object, file, line, -1); 834 #endif 835 break; 836 } 837 /* retry */ 838 } 839 /* retry */ 840 } 841 } 842 843 void 844 VMOBJDEBUG(vm_object_deallocate_locked)(vm_object_t object VMOBJDBARGS) 845 { 846 struct vm_object_dealloc_list *dlist = NULL; 847 struct vm_object_dealloc_list *dtmp; 848 vm_object_t temp; 849 int must_drop = 0; 850 851 /* 852 * We may chain deallocate object, but additional objects may 853 * collect on the dlist which also have to be deallocated. We 854 * must avoid a recursion, vm_object chains can get deep. 855 */ 856 857 again: 858 while (object != NULL) { 859 /* 860 * vnode case, caller either locked the object exclusively 861 * or this is a recursion with must_drop != 0 and the vnode 862 * object will be locked shared. 863 * 864 * If locked shared we have to drop the object before we can 865 * call vrele() or risk a shared/exclusive livelock. 866 */ 867 if (object->type == OBJT_VNODE) { 868 ASSERT_LWKT_TOKEN_HELD(&object->token); 869 if (must_drop) { 870 struct vnode *tmp_vp; 871 872 vm_object_vndeallocate(object, &tmp_vp); 873 vm_object_drop(object); 874 must_drop = 0; 875 object = NULL; 876 vrele(tmp_vp); 877 } else { 878 vm_object_vndeallocate(object, NULL); 879 } 880 break; 881 } 882 ASSERT_LWKT_TOKEN_HELD_EXCL(&object->token); 883 884 /* 885 * Normal case (object is locked exclusively) 886 */ 887 if (object->ref_count == 0) { 888 panic("vm_object_deallocate: object deallocated " 889 "too many times: %d", object->type); 890 } 891 if (object->ref_count > 2) { 892 atomic_add_int(&object->ref_count, -1); 893 #if defined(DEBUG_LOCKS) 894 debugvm_object_add(object, file, line, -1); 895 #endif 896 break; 897 } 898 899 /* 900 * Here on ref_count of one or two, which are special cases for 901 * objects. 902 * 903 * Nominal ref_count > 1 case if the second ref is not from 904 * a shadow. 905 * 906 * (ONEMAPPING only applies to DEFAULT AND SWAP objects) 907 */ 908 if (object->ref_count == 2 && object->shadow_count == 0) { 909 if (object->type == OBJT_DEFAULT || 910 object->type == OBJT_SWAP) { 911 vm_object_set_flag(object, OBJ_ONEMAPPING); 912 } 913 atomic_add_int(&object->ref_count, -1); 914 #if defined(DEBUG_LOCKS) 915 debugvm_object_add(object, file, line, -1); 916 #endif 917 break; 918 } 919 920 /* 921 * If the second ref is from a shadow we chain along it 922 * upwards if object's handle is exhausted. 923 * 924 * We have to decrement object->ref_count before potentially 925 * collapsing the first shadow object or the collapse code 926 * will not be able to handle the degenerate case to remove 927 * object. However, if we do it too early the object can 928 * get ripped out from under us. 929 */ 930 if (object->ref_count == 2 && object->shadow_count == 1 && 931 object->handle == NULL && (object->type == OBJT_DEFAULT || 932 object->type == OBJT_SWAP)) { 933 temp = LIST_FIRST(&object->shadow_head); 934 KKASSERT(temp != NULL); 935 vm_object_hold(temp); 936 937 /* 938 * Wait for any paging to complete so the collapse 939 * doesn't (or isn't likely to) qcollapse. pip 940 * waiting must occur before we acquire the 941 * chainlock. 942 */ 943 while ( 944 temp->paging_in_progress || 945 object->paging_in_progress 946 ) { 947 vm_object_pip_wait(temp, "objde1"); 948 vm_object_pip_wait(object, "objde2"); 949 } 950 951 /* 952 * If the parent is locked we have to give up, as 953 * otherwise we would be acquiring locks in the 954 * wrong order and potentially deadlock. 955 */ 956 if (temp->chainlk & (CHAINLK_EXCL | CHAINLK_MASK)) { 957 vm_object_drop(temp); 958 goto skip; 959 } 960 vm_object_chain_acquire(temp, 0); 961 962 /* 963 * Recheck/retry after the hold and the paging 964 * wait, both of which can block us. 965 */ 966 if (object->ref_count != 2 || 967 object->shadow_count != 1 || 968 object->handle || 969 LIST_FIRST(&object->shadow_head) != temp || 970 (object->type != OBJT_DEFAULT && 971 object->type != OBJT_SWAP)) { 972 vm_object_chain_release(temp); 973 vm_object_drop(temp); 974 continue; 975 } 976 977 /* 978 * We can safely drop object's ref_count now. 979 */ 980 KKASSERT(object->ref_count == 2); 981 atomic_add_int(&object->ref_count, -1); 982 #if defined(DEBUG_LOCKS) 983 debugvm_object_add(object, file, line, -1); 984 #endif 985 986 /* 987 * If our single parent is not collapseable just 988 * decrement ref_count (2->1) and stop. 989 */ 990 if (temp->handle || (temp->type != OBJT_DEFAULT && 991 temp->type != OBJT_SWAP)) { 992 vm_object_chain_release(temp); 993 vm_object_drop(temp); 994 break; 995 } 996 997 /* 998 * At this point we have already dropped object's 999 * ref_count so it is possible for a race to 1000 * deallocate obj out from under us. Any collapse 1001 * will re-check the situation. We must not block 1002 * until we are able to collapse. 1003 * 1004 * Bump temp's ref_count to avoid an unwanted 1005 * degenerate recursion (can't call 1006 * vm_object_reference_locked() because it asserts 1007 * that CHAINLOCK is not set). 1008 */ 1009 atomic_add_int(&temp->ref_count, 1); 1010 KKASSERT(temp->ref_count > 1); 1011 1012 /* 1013 * Collapse temp, then deallocate the extra ref 1014 * formally. 1015 */ 1016 vm_object_collapse(temp, &dlist); 1017 vm_object_chain_release(temp); 1018 if (must_drop) { 1019 vm_object_lock_swap(); 1020 vm_object_drop(object); 1021 } 1022 object = temp; 1023 must_drop = 1; 1024 continue; 1025 } 1026 1027 /* 1028 * Drop the ref and handle termination on the 1->0 transition. 1029 * We may have blocked above so we have to recheck. 1030 */ 1031 skip: 1032 KKASSERT(object->ref_count != 0); 1033 if (object->ref_count >= 2) { 1034 atomic_add_int(&object->ref_count, -1); 1035 #if defined(DEBUG_LOCKS) 1036 debugvm_object_add(object, file, line, -1); 1037 #endif 1038 break; 1039 } 1040 KKASSERT(object->ref_count == 1); 1041 1042 /* 1043 * 1->0 transition. Chain through the backing_object. 1044 * Maintain the ref until we've located the backing object, 1045 * then re-check. 1046 */ 1047 while ((temp = object->backing_object) != NULL) { 1048 if (temp->type == OBJT_VNODE) 1049 vm_object_hold_shared(temp); 1050 else 1051 vm_object_hold(temp); 1052 if (temp == object->backing_object) 1053 break; 1054 vm_object_drop(temp); 1055 } 1056 1057 /* 1058 * 1->0 transition verified, retry if ref_count is no longer 1059 * 1. Otherwise disconnect the backing_object (temp) and 1060 * clean up. 1061 */ 1062 if (object->ref_count != 1) { 1063 vm_object_drop(temp); 1064 continue; 1065 } 1066 1067 /* 1068 * It shouldn't be possible for the object to be chain locked 1069 * if we're removing the last ref on it. 1070 * 1071 * Removing object from temp's shadow list requires dropping 1072 * temp, which we will do on loop. 1073 * 1074 * NOTE! vnodes do not use the shadow list, but still have 1075 * the backing_object reference. 1076 */ 1077 KKASSERT((object->chainlk & (CHAINLK_EXCL|CHAINLK_MASK)) == 0); 1078 1079 if (temp) { 1080 if (object->flags & OBJ_ONSHADOW) { 1081 LIST_REMOVE(object, shadow_list); 1082 temp->shadow_count--; 1083 atomic_add_int(&temp->generation, 1); 1084 vm_object_clear_flag(object, OBJ_ONSHADOW); 1085 } 1086 object->backing_object = NULL; 1087 } 1088 1089 atomic_add_int(&object->ref_count, -1); 1090 if ((object->flags & OBJ_DEAD) == 0) 1091 vm_object_terminate(object); 1092 if (must_drop && temp) 1093 vm_object_lock_swap(); 1094 if (must_drop) 1095 vm_object_drop(object); 1096 object = temp; 1097 must_drop = 1; 1098 } 1099 1100 if (must_drop && object) 1101 vm_object_drop(object); 1102 1103 /* 1104 * Additional tail recursion on dlist. Avoid a recursion. Objects 1105 * on the dlist have a hold count but are not locked. 1106 */ 1107 if ((dtmp = dlist) != NULL) { 1108 dlist = dtmp->next; 1109 object = dtmp->object; 1110 kfree(dtmp, M_TEMP); 1111 1112 vm_object_lock(object); /* already held, add lock */ 1113 must_drop = 1; /* and we're responsible for it */ 1114 goto again; 1115 } 1116 } 1117 1118 /* 1119 * Destroy the specified object, freeing up related resources. 1120 * 1121 * The object must have zero references. 1122 * 1123 * The object must held. The caller is responsible for dropping the object 1124 * after terminate returns. Terminate does NOT drop the object. 1125 */ 1126 static int vm_object_terminate_callback(vm_page_t p, void *data); 1127 1128 void 1129 vm_object_terminate(vm_object_t object) 1130 { 1131 struct rb_vm_page_scan_info info; 1132 struct vm_object_hash *hash; 1133 1134 /* 1135 * Make sure no one uses us. Once we set OBJ_DEAD we should be 1136 * able to safely block. 1137 */ 1138 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 1139 KKASSERT((object->flags & OBJ_DEAD) == 0); 1140 vm_object_set_flag(object, OBJ_DEAD); 1141 1142 /* 1143 * Wait for the pageout daemon to be done with the object 1144 */ 1145 vm_object_pip_wait(object, "objtrm1"); 1146 1147 KASSERT(!object->paging_in_progress, 1148 ("vm_object_terminate: pageout in progress")); 1149 1150 /* 1151 * Clean and free the pages, as appropriate. All references to the 1152 * object are gone, so we don't need to lock it. 1153 */ 1154 if (object->type == OBJT_VNODE) { 1155 struct vnode *vp; 1156 1157 /* 1158 * Clean pages and flush buffers. 1159 * 1160 * NOTE! TMPFS buffer flushes do not typically flush the 1161 * actual page to swap as this would be highly 1162 * inefficient, and normal filesystems usually wrap 1163 * page flushes with buffer cache buffers. 1164 * 1165 * To deal with this we have to call vinvalbuf() both 1166 * before and after the vm_object_page_clean(). 1167 */ 1168 vp = (struct vnode *) object->handle; 1169 vinvalbuf(vp, V_SAVE, 0, 0); 1170 vm_object_page_clean(object, 0, 0, OBJPC_SYNC); 1171 vinvalbuf(vp, V_SAVE, 0, 0); 1172 } 1173 1174 /* 1175 * Wait for any I/O to complete, after which there had better not 1176 * be any references left on the object. 1177 */ 1178 vm_object_pip_wait(object, "objtrm2"); 1179 1180 if (object->ref_count != 0) { 1181 panic("vm_object_terminate: object with references, " 1182 "ref_count=%d", object->ref_count); 1183 } 1184 1185 /* 1186 * Cleanup any shared pmaps associated with this object. 1187 */ 1188 pmap_object_free(object); 1189 1190 /* 1191 * Now free any remaining pages. For internal objects, this also 1192 * removes them from paging queues. Don't free wired pages, just 1193 * remove them from the object. 1194 */ 1195 info.count = 0; 1196 info.object = object; 1197 do { 1198 info.error = 0; 1199 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL, 1200 vm_object_terminate_callback, &info); 1201 } while (info.error); 1202 1203 /* 1204 * Let the pager know object is dead. 1205 */ 1206 vm_pager_deallocate(object); 1207 1208 /* 1209 * Wait for the object hold count to hit 1, clean out pages as 1210 * we go. vmobj_token interlocks any race conditions that might 1211 * pick the object up from the vm_object_list after we have cleared 1212 * rb_memq. 1213 */ 1214 for (;;) { 1215 if (RB_ROOT(&object->rb_memq) == NULL) 1216 break; 1217 kprintf("vm_object_terminate: Warning, object %p " 1218 "still has %ld pages\n", 1219 object, object->resident_page_count); 1220 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL, 1221 vm_object_terminate_callback, &info); 1222 } 1223 1224 /* 1225 * There had better not be any pages left 1226 */ 1227 KKASSERT(object->resident_page_count == 0); 1228 1229 /* 1230 * Remove the object from the global object list. 1231 */ 1232 hash = VMOBJ_HASH(object); 1233 lwkt_gettoken(&hash->token); 1234 TAILQ_REMOVE(&hash->list, object, object_list); 1235 lwkt_reltoken(&hash->token); 1236 1237 if (object->ref_count != 0) { 1238 panic("vm_object_terminate2: object with references, " 1239 "ref_count=%d", object->ref_count); 1240 } 1241 1242 /* 1243 * NOTE: The object hold_count is at least 1, so we cannot kfree() 1244 * the object here. See vm_object_drop(). 1245 */ 1246 } 1247 1248 /* 1249 * The caller must hold the object. 1250 */ 1251 static int 1252 vm_object_terminate_callback(vm_page_t p, void *data) 1253 { 1254 struct rb_vm_page_scan_info *info = data; 1255 vm_object_t object; 1256 1257 object = p->object; 1258 KKASSERT(object == info->object); 1259 if (vm_page_busy_try(p, TRUE)) { 1260 vm_page_sleep_busy(p, TRUE, "vmotrm"); 1261 info->error = 1; 1262 return 0; 1263 } 1264 if (object != p->object) { 1265 /* XXX remove once we determine it can't happen */ 1266 kprintf("vm_object_terminate: Warning: Encountered " 1267 "busied page %p on queue %d\n", p, p->queue); 1268 vm_page_wakeup(p); 1269 info->error = 1; 1270 } else if (p->wire_count == 0) { 1271 /* 1272 * NOTE: p->dirty and PG_NEED_COMMIT are ignored. 1273 */ 1274 vm_page_free(p); 1275 mycpu->gd_cnt.v_pfree++; 1276 } else { 1277 if (p->queue != PQ_NONE) 1278 kprintf("vm_object_terminate: Warning: Encountered " 1279 "wired page %p on queue %d\n", p, p->queue); 1280 vm_page_remove(p); 1281 vm_page_wakeup(p); 1282 } 1283 1284 /* 1285 * Must be at end to avoid SMP races, caller holds object token 1286 */ 1287 if ((++info->count & 63) == 0) 1288 lwkt_user_yield(); 1289 return(0); 1290 } 1291 1292 /* 1293 * Clean all dirty pages in the specified range of object. Leaves page 1294 * on whatever queue it is currently on. If NOSYNC is set then do not 1295 * write out pages with PG_NOSYNC set (originally comes from MAP_NOSYNC), 1296 * leaving the object dirty. 1297 * 1298 * When stuffing pages asynchronously, allow clustering. XXX we need a 1299 * synchronous clustering mode implementation. 1300 * 1301 * Odd semantics: if start == end, we clean everything. 1302 * 1303 * The object must be locked? XXX 1304 */ 1305 static int vm_object_page_clean_pass1(struct vm_page *p, void *data); 1306 static int vm_object_page_clean_pass2(struct vm_page *p, void *data); 1307 1308 void 1309 vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 1310 int flags) 1311 { 1312 struct rb_vm_page_scan_info info; 1313 struct vnode *vp; 1314 int wholescan; 1315 int pagerflags; 1316 int generation; 1317 1318 vm_object_hold(object); 1319 if (object->type != OBJT_VNODE || 1320 (object->flags & OBJ_MIGHTBEDIRTY) == 0) { 1321 vm_object_drop(object); 1322 return; 1323 } 1324 1325 pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? 1326 VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; 1327 pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0; 1328 1329 vp = object->handle; 1330 1331 /* 1332 * Interlock other major object operations. This allows us to 1333 * temporarily clear OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY. 1334 */ 1335 vm_object_set_flag(object, OBJ_CLEANING); 1336 1337 /* 1338 * Handle 'entire object' case 1339 */ 1340 info.start_pindex = start; 1341 if (end == 0) { 1342 info.end_pindex = object->size - 1; 1343 } else { 1344 info.end_pindex = end - 1; 1345 } 1346 wholescan = (start == 0 && info.end_pindex == object->size - 1); 1347 info.limit = flags; 1348 info.pagerflags = pagerflags; 1349 info.object = object; 1350 1351 /* 1352 * If cleaning the entire object do a pass to mark the pages read-only. 1353 * If everything worked out ok, clear OBJ_WRITEABLE and 1354 * OBJ_MIGHTBEDIRTY. 1355 */ 1356 if (wholescan) { 1357 info.error = 0; 1358 info.count = 0; 1359 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp, 1360 vm_object_page_clean_pass1, &info); 1361 if (info.error == 0) { 1362 vm_object_clear_flag(object, 1363 OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 1364 if (object->type == OBJT_VNODE && 1365 (vp = (struct vnode *)object->handle) != NULL) { 1366 /* 1367 * Use new-style interface to clear VISDIRTY 1368 * because the vnode is not necessarily removed 1369 * from the syncer list(s) as often as it was 1370 * under the old interface, which can leave 1371 * the vnode on the syncer list after reclaim. 1372 */ 1373 vclrobjdirty(vp); 1374 } 1375 } 1376 } 1377 1378 /* 1379 * Do a pass to clean all the dirty pages we find. 1380 */ 1381 do { 1382 info.error = 0; 1383 info.count = 0; 1384 generation = object->generation; 1385 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp, 1386 vm_object_page_clean_pass2, &info); 1387 } while (info.error || generation != object->generation); 1388 1389 vm_object_clear_flag(object, OBJ_CLEANING); 1390 vm_object_drop(object); 1391 } 1392 1393 /* 1394 * The caller must hold the object. 1395 */ 1396 static 1397 int 1398 vm_object_page_clean_pass1(struct vm_page *p, void *data) 1399 { 1400 struct rb_vm_page_scan_info *info = data; 1401 1402 KKASSERT(p->object == info->object); 1403 1404 vm_page_flag_set(p, PG_CLEANCHK); 1405 if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { 1406 info->error = 1; 1407 } else if (vm_page_busy_try(p, FALSE)) { 1408 info->error = 1; 1409 } else { 1410 KKASSERT(p->object == info->object); 1411 vm_page_protect(p, VM_PROT_READ); 1412 vm_page_wakeup(p); 1413 } 1414 1415 /* 1416 * Must be at end to avoid SMP races, caller holds object token 1417 */ 1418 if ((++info->count & 63) == 0) 1419 lwkt_user_yield(); 1420 return(0); 1421 } 1422 1423 /* 1424 * The caller must hold the object 1425 */ 1426 static 1427 int 1428 vm_object_page_clean_pass2(struct vm_page *p, void *data) 1429 { 1430 struct rb_vm_page_scan_info *info = data; 1431 int generation; 1432 1433 KKASSERT(p->object == info->object); 1434 1435 /* 1436 * Do not mess with pages that were inserted after we started 1437 * the cleaning pass. 1438 */ 1439 if ((p->flags & PG_CLEANCHK) == 0) 1440 goto done; 1441 1442 generation = info->object->generation; 1443 1444 if (vm_page_busy_try(p, TRUE)) { 1445 vm_page_sleep_busy(p, TRUE, "vpcwai"); 1446 info->error = 1; 1447 goto done; 1448 } 1449 1450 KKASSERT(p->object == info->object && 1451 info->object->generation == generation); 1452 1453 /* 1454 * Before wasting time traversing the pmaps, check for trivial 1455 * cases where the page cannot be dirty. 1456 */ 1457 if (p->valid == 0 || (p->queue - p->pc) == PQ_CACHE) { 1458 KKASSERT((p->dirty & p->valid) == 0 && 1459 (p->flags & PG_NEED_COMMIT) == 0); 1460 vm_page_wakeup(p); 1461 goto done; 1462 } 1463 1464 /* 1465 * Check whether the page is dirty or not. The page has been set 1466 * to be read-only so the check will not race a user dirtying the 1467 * page. 1468 */ 1469 vm_page_test_dirty(p); 1470 if ((p->dirty & p->valid) == 0 && (p->flags & PG_NEED_COMMIT) == 0) { 1471 vm_page_flag_clear(p, PG_CLEANCHK); 1472 vm_page_wakeup(p); 1473 goto done; 1474 } 1475 1476 /* 1477 * If we have been asked to skip nosync pages and this is a 1478 * nosync page, skip it. Note that the object flags were 1479 * not cleared in this case (because pass1 will have returned an 1480 * error), so we do not have to set them. 1481 */ 1482 if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { 1483 vm_page_flag_clear(p, PG_CLEANCHK); 1484 vm_page_wakeup(p); 1485 goto done; 1486 } 1487 1488 /* 1489 * Flush as many pages as we can. PG_CLEANCHK will be cleared on 1490 * the pages that get successfully flushed. Set info->error if 1491 * we raced an object modification. 1492 */ 1493 vm_object_page_collect_flush(info->object, p, info->pagerflags); 1494 /* vm_wait_nominal(); this can deadlock the system in syncer/pageout */ 1495 1496 /* 1497 * Must be at end to avoid SMP races, caller holds object token 1498 */ 1499 done: 1500 if ((++info->count & 63) == 0) 1501 lwkt_user_yield(); 1502 return(0); 1503 } 1504 1505 /* 1506 * Collect the specified page and nearby pages and flush them out. 1507 * The number of pages flushed is returned. The passed page is busied 1508 * by the caller and we are responsible for its disposition. 1509 * 1510 * The caller must hold the object. 1511 */ 1512 static void 1513 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags) 1514 { 1515 int error; 1516 int is; 1517 int ib; 1518 int i; 1519 int page_base; 1520 vm_pindex_t pi; 1521 vm_page_t ma[BLIST_MAX_ALLOC]; 1522 1523 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object)); 1524 1525 pi = p->pindex; 1526 page_base = pi % BLIST_MAX_ALLOC; 1527 ma[page_base] = p; 1528 ib = page_base - 1; 1529 is = page_base + 1; 1530 1531 while (ib >= 0) { 1532 vm_page_t tp; 1533 1534 tp = vm_page_lookup_busy_try(object, pi - page_base + ib, 1535 TRUE, &error); 1536 if (error) 1537 break; 1538 if (tp == NULL) 1539 break; 1540 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 1541 (tp->flags & PG_CLEANCHK) == 0) { 1542 vm_page_wakeup(tp); 1543 break; 1544 } 1545 if ((tp->queue - tp->pc) == PQ_CACHE) { 1546 vm_page_flag_clear(tp, PG_CLEANCHK); 1547 vm_page_wakeup(tp); 1548 break; 1549 } 1550 vm_page_test_dirty(tp); 1551 if ((tp->dirty & tp->valid) == 0 && 1552 (tp->flags & PG_NEED_COMMIT) == 0) { 1553 vm_page_flag_clear(tp, PG_CLEANCHK); 1554 vm_page_wakeup(tp); 1555 break; 1556 } 1557 ma[ib] = tp; 1558 --ib; 1559 } 1560 ++ib; /* fixup */ 1561 1562 while (is < BLIST_MAX_ALLOC && 1563 pi - page_base + is < object->size) { 1564 vm_page_t tp; 1565 1566 tp = vm_page_lookup_busy_try(object, pi - page_base + is, 1567 TRUE, &error); 1568 if (error) 1569 break; 1570 if (tp == NULL) 1571 break; 1572 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 1573 (tp->flags & PG_CLEANCHK) == 0) { 1574 vm_page_wakeup(tp); 1575 break; 1576 } 1577 if ((tp->queue - tp->pc) == PQ_CACHE) { 1578 vm_page_flag_clear(tp, PG_CLEANCHK); 1579 vm_page_wakeup(tp); 1580 break; 1581 } 1582 vm_page_test_dirty(tp); 1583 if ((tp->dirty & tp->valid) == 0 && 1584 (tp->flags & PG_NEED_COMMIT) == 0) { 1585 vm_page_flag_clear(tp, PG_CLEANCHK); 1586 vm_page_wakeup(tp); 1587 break; 1588 } 1589 ma[is] = tp; 1590 ++is; 1591 } 1592 1593 /* 1594 * All pages in the ma[] array are busied now 1595 */ 1596 for (i = ib; i < is; ++i) { 1597 vm_page_flag_clear(ma[i], PG_CLEANCHK); 1598 vm_page_hold(ma[i]); /* XXX need this any more? */ 1599 } 1600 vm_pageout_flush(&ma[ib], is - ib, pagerflags); 1601 for (i = ib; i < is; ++i) /* XXX need this any more? */ 1602 vm_page_unhold(ma[i]); 1603 } 1604 1605 /* 1606 * Same as vm_object_pmap_copy, except range checking really 1607 * works, and is meant for small sections of an object. 1608 * 1609 * This code protects resident pages by making them read-only 1610 * and is typically called on a fork or split when a page 1611 * is converted to copy-on-write. 1612 * 1613 * NOTE: If the page is already at VM_PROT_NONE, calling 1614 * vm_page_protect will have no effect. 1615 */ 1616 void 1617 vm_object_pmap_copy_1(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1618 { 1619 vm_pindex_t idx; 1620 vm_page_t p; 1621 1622 if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0) 1623 return; 1624 1625 vm_object_hold(object); 1626 for (idx = start; idx < end; idx++) { 1627 p = vm_page_lookup(object, idx); 1628 if (p == NULL) 1629 continue; 1630 vm_page_protect(p, VM_PROT_READ); 1631 } 1632 vm_object_drop(object); 1633 } 1634 1635 /* 1636 * Removes all physical pages in the specified object range from all 1637 * physical maps. 1638 * 1639 * The object must *not* be locked. 1640 */ 1641 1642 static int vm_object_pmap_remove_callback(vm_page_t p, void *data); 1643 1644 void 1645 vm_object_pmap_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1646 { 1647 struct rb_vm_page_scan_info info; 1648 1649 if (object == NULL) 1650 return; 1651 if (start == end) 1652 return; 1653 info.start_pindex = start; 1654 info.end_pindex = end - 1; 1655 info.count = 0; 1656 info.object = object; 1657 1658 vm_object_hold(object); 1659 do { 1660 info.error = 0; 1661 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp, 1662 vm_object_pmap_remove_callback, &info); 1663 } while (info.error); 1664 if (start == 0 && end == object->size) 1665 vm_object_clear_flag(object, OBJ_WRITEABLE); 1666 vm_object_drop(object); 1667 } 1668 1669 /* 1670 * The caller must hold the object 1671 */ 1672 static int 1673 vm_object_pmap_remove_callback(vm_page_t p, void *data) 1674 { 1675 struct rb_vm_page_scan_info *info = data; 1676 1677 if (info->object != p->object || 1678 p->pindex < info->start_pindex || 1679 p->pindex > info->end_pindex) { 1680 kprintf("vm_object_pmap_remove_callback: obj/pg race %p/%p\n", 1681 info->object, p); 1682 info->error = 1; 1683 return(0); 1684 } 1685 1686 vm_page_protect(p, VM_PROT_NONE); 1687 1688 /* 1689 * Must be at end to avoid SMP races, caller holds object token 1690 */ 1691 if ((++info->count & 63) == 0) 1692 lwkt_user_yield(); 1693 return(0); 1694 } 1695 1696 /* 1697 * Implements the madvise function at the object/page level. 1698 * 1699 * MADV_WILLNEED (any object) 1700 * 1701 * Activate the specified pages if they are resident. 1702 * 1703 * MADV_DONTNEED (any object) 1704 * 1705 * Deactivate the specified pages if they are resident. 1706 * 1707 * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects, OBJ_ONEMAPPING only) 1708 * 1709 * Deactivate and clean the specified pages if they are 1710 * resident. This permits the process to reuse the pages 1711 * without faulting or the kernel to reclaim the pages 1712 * without I/O. 1713 * 1714 * No requirements. 1715 */ 1716 void 1717 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, 1718 vm_pindex_t count, int advise) 1719 { 1720 vm_pindex_t end, tpindex; 1721 vm_object_t tobject; 1722 vm_object_t xobj; 1723 vm_page_t m; 1724 int error; 1725 1726 if (object == NULL) 1727 return; 1728 1729 end = pindex + count; 1730 1731 vm_object_hold(object); 1732 tobject = object; 1733 1734 /* 1735 * Locate and adjust resident pages 1736 */ 1737 for (; pindex < end; pindex += 1) { 1738 relookup: 1739 if (tobject != object) 1740 vm_object_drop(tobject); 1741 tobject = object; 1742 tpindex = pindex; 1743 shadowlookup: 1744 /* 1745 * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages 1746 * and those pages must be OBJ_ONEMAPPING. 1747 */ 1748 if (advise == MADV_FREE) { 1749 if ((tobject->type != OBJT_DEFAULT && 1750 tobject->type != OBJT_SWAP) || 1751 (tobject->flags & OBJ_ONEMAPPING) == 0) { 1752 continue; 1753 } 1754 } 1755 1756 m = vm_page_lookup_busy_try(tobject, tpindex, TRUE, &error); 1757 1758 if (error) { 1759 vm_page_sleep_busy(m, TRUE, "madvpo"); 1760 goto relookup; 1761 } 1762 if (m == NULL) { 1763 /* 1764 * There may be swap even if there is no backing page 1765 */ 1766 if (advise == MADV_FREE && tobject->type == OBJT_SWAP) 1767 swap_pager_freespace(tobject, tpindex, 1); 1768 1769 /* 1770 * next object 1771 */ 1772 while ((xobj = tobject->backing_object) != NULL) { 1773 KKASSERT(xobj != object); 1774 vm_object_hold(xobj); 1775 if (xobj == tobject->backing_object) 1776 break; 1777 vm_object_drop(xobj); 1778 } 1779 if (xobj == NULL) 1780 continue; 1781 tpindex += OFF_TO_IDX(tobject->backing_object_offset); 1782 if (tobject != object) { 1783 vm_object_lock_swap(); 1784 vm_object_drop(tobject); 1785 } 1786 tobject = xobj; 1787 goto shadowlookup; 1788 } 1789 1790 /* 1791 * If the page is not in a normal active state, we skip it. 1792 * If the page is not managed there are no page queues to 1793 * mess with. Things can break if we mess with pages in 1794 * any of the below states. 1795 */ 1796 if (m->wire_count || 1797 (m->flags & (PG_UNMANAGED | PG_NEED_COMMIT)) || 1798 m->valid != VM_PAGE_BITS_ALL 1799 ) { 1800 vm_page_wakeup(m); 1801 continue; 1802 } 1803 1804 /* 1805 * Theoretically once a page is known not to be busy, an 1806 * interrupt cannot come along and rip it out from under us. 1807 */ 1808 1809 if (advise == MADV_WILLNEED) { 1810 vm_page_activate(m); 1811 } else if (advise == MADV_DONTNEED) { 1812 vm_page_dontneed(m); 1813 } else if (advise == MADV_FREE) { 1814 /* 1815 * Mark the page clean. This will allow the page 1816 * to be freed up by the system. However, such pages 1817 * are often reused quickly by malloc()/free() 1818 * so we do not do anything that would cause 1819 * a page fault if we can help it. 1820 * 1821 * Specifically, we do not try to actually free 1822 * the page now nor do we try to put it in the 1823 * cache (which would cause a page fault on reuse). 1824 * 1825 * But we do make the page is freeable as we 1826 * can without actually taking the step of unmapping 1827 * it. 1828 */ 1829 pmap_clear_modify(m); 1830 m->dirty = 0; 1831 m->act_count = 0; 1832 vm_page_dontneed(m); 1833 if (tobject->type == OBJT_SWAP) 1834 swap_pager_freespace(tobject, tpindex, 1); 1835 } 1836 vm_page_wakeup(m); 1837 } 1838 if (tobject != object) 1839 vm_object_drop(tobject); 1840 vm_object_drop(object); 1841 } 1842 1843 /* 1844 * Create a new object which is backed by the specified existing object 1845 * range. Replace the pointer and offset that was pointing at the existing 1846 * object with the pointer/offset for the new object. 1847 * 1848 * If addref is non-zero the returned object is given an additional reference. 1849 * This mechanic exists to avoid the situation where refs might be 1 and 1850 * race against a collapse when the caller intends to bump it. So the 1851 * caller cannot add the ref after the fact. Used when the caller is 1852 * duplicating a vm_map_entry. 1853 * 1854 * No other requirements. 1855 */ 1856 void 1857 vm_object_shadow(vm_object_t *objectp, vm_ooffset_t *offset, vm_size_t length, 1858 int addref) 1859 { 1860 vm_object_t source; 1861 vm_object_t result; 1862 int useshadowlist; 1863 1864 source = *objectp; 1865 1866 /* 1867 * Don't create the new object if the old object isn't shared. 1868 * We have to chain wait before adding the reference to avoid 1869 * racing a collapse or deallocation. 1870 * 1871 * Clear OBJ_ONEMAPPING flag when shadowing. 1872 * 1873 * The caller owns a ref on source via *objectp which we are going 1874 * to replace. This ref is inherited by the backing_object assignment. 1875 * from nobject and does not need to be incremented here. 1876 * 1877 * However, we add a temporary extra reference to the original source 1878 * prior to holding nobject in case we block, to avoid races where 1879 * someone else might believe that the source can be collapsed. 1880 */ 1881 useshadowlist = 0; 1882 if (source) { 1883 if (source->type != OBJT_VNODE) { 1884 useshadowlist = 1; 1885 vm_object_hold(source); 1886 vm_object_chain_wait(source, 0); 1887 if (source->ref_count == 1 && 1888 source->handle == NULL && 1889 (source->type == OBJT_DEFAULT || 1890 source->type == OBJT_SWAP)) { 1891 if (addref) { 1892 vm_object_reference_locked(source); 1893 vm_object_clear_flag(source, 1894 OBJ_ONEMAPPING); 1895 } 1896 vm_object_drop(source); 1897 return; 1898 } 1899 vm_object_reference_locked(source); 1900 vm_object_clear_flag(source, OBJ_ONEMAPPING); 1901 } else { 1902 vm_object_reference_quick(source); 1903 vm_object_clear_flag(source, OBJ_ONEMAPPING); 1904 } 1905 } 1906 1907 /* 1908 * Allocate a new object with the given length. The new object 1909 * is returned referenced but we may have to add another one. 1910 * If we are adding a second reference we must clear OBJ_ONEMAPPING. 1911 * (typically because the caller is about to clone a vm_map_entry). 1912 * 1913 * The source object currently has an extra reference to prevent 1914 * collapses into it while we mess with its shadow list, which 1915 * we will remove later in this routine. 1916 * 1917 * The target object may require a second reference if asked for one 1918 * by the caller. 1919 */ 1920 result = vm_object_allocate(OBJT_DEFAULT, length); 1921 if (result == NULL) 1922 panic("vm_object_shadow: no object for shadowing"); 1923 vm_object_hold(result); 1924 if (addref) { 1925 vm_object_reference_locked(result); 1926 vm_object_clear_flag(result, OBJ_ONEMAPPING); 1927 } 1928 1929 /* 1930 * The new object shadows the source object. Chain wait before 1931 * adjusting shadow_count or the shadow list to avoid races. 1932 * 1933 * Try to optimize the result object's page color when shadowing 1934 * in order to maintain page coloring consistency in the combined 1935 * shadowed object. 1936 * 1937 * The backing_object reference to source requires adding a ref to 1938 * source. We simply inherit the ref from the original *objectp 1939 * (which we are replacing) so no additional refs need to be added. 1940 * (we must still clean up the extra ref we had to prevent collapse 1941 * races). 1942 * 1943 * SHADOWING IS NOT APPLICABLE TO OBJT_VNODE OBJECTS 1944 */ 1945 KKASSERT(result->backing_object == NULL); 1946 result->backing_object = source; 1947 if (source) { 1948 if (useshadowlist) { 1949 vm_object_chain_wait(source, 0); 1950 LIST_INSERT_HEAD(&source->shadow_head, 1951 result, shadow_list); 1952 source->shadow_count++; 1953 atomic_add_int(&source->generation, 1); 1954 vm_object_set_flag(result, OBJ_ONSHADOW); 1955 } 1956 /* cpu localization twist */ 1957 result->pg_color = vm_quickcolor(); 1958 } 1959 1960 /* 1961 * Adjust the return storage. Drop the ref on source before 1962 * returning. 1963 */ 1964 result->backing_object_offset = *offset; 1965 vm_object_drop(result); 1966 *offset = 0; 1967 if (source) { 1968 if (useshadowlist) { 1969 vm_object_deallocate_locked(source); 1970 vm_object_drop(source); 1971 } else { 1972 vm_object_deallocate(source); 1973 } 1974 } 1975 1976 /* 1977 * Return the new things 1978 */ 1979 *objectp = result; 1980 } 1981 1982 #define OBSC_TEST_ALL_SHADOWED 0x0001 1983 #define OBSC_COLLAPSE_NOWAIT 0x0002 1984 #define OBSC_COLLAPSE_WAIT 0x0004 1985 1986 static int vm_object_backing_scan_callback(vm_page_t p, void *data); 1987 1988 /* 1989 * The caller must hold the object. 1990 */ 1991 static __inline int 1992 vm_object_backing_scan(vm_object_t object, vm_object_t backing_object, int op) 1993 { 1994 struct rb_vm_page_scan_info info; 1995 struct vm_object_hash *hash; 1996 1997 vm_object_assert_held(object); 1998 vm_object_assert_held(backing_object); 1999 2000 KKASSERT(backing_object == object->backing_object); 2001 info.backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 2002 2003 /* 2004 * Initial conditions 2005 */ 2006 if (op & OBSC_TEST_ALL_SHADOWED) { 2007 /* 2008 * We do not want to have to test for the existence of 2009 * swap pages in the backing object. XXX but with the 2010 * new swapper this would be pretty easy to do. 2011 * 2012 * XXX what about anonymous MAP_SHARED memory that hasn't 2013 * been ZFOD faulted yet? If we do not test for this, the 2014 * shadow test may succeed! XXX 2015 */ 2016 if (backing_object->type != OBJT_DEFAULT) 2017 return(0); 2018 } 2019 if (op & OBSC_COLLAPSE_WAIT) { 2020 KKASSERT((backing_object->flags & OBJ_DEAD) == 0); 2021 vm_object_set_flag(backing_object, OBJ_DEAD); 2022 2023 hash = VMOBJ_HASH(backing_object); 2024 lwkt_gettoken(&hash->token); 2025 TAILQ_REMOVE(&hash->list, backing_object, object_list); 2026 lwkt_reltoken(&hash->token); 2027 } 2028 2029 /* 2030 * Our scan. We have to retry if a negative error code is returned, 2031 * otherwise 0 or 1 will be returned in info.error. 0 Indicates that 2032 * the scan had to be stopped because the parent does not completely 2033 * shadow the child. 2034 */ 2035 info.object = object; 2036 info.backing_object = backing_object; 2037 info.limit = op; 2038 info.count = 0; 2039 do { 2040 info.error = 1; 2041 vm_page_rb_tree_RB_SCAN(&backing_object->rb_memq, NULL, 2042 vm_object_backing_scan_callback, 2043 &info); 2044 } while (info.error < 0); 2045 2046 return(info.error); 2047 } 2048 2049 /* 2050 * The caller must hold the object. 2051 */ 2052 static int 2053 vm_object_backing_scan_callback(vm_page_t p, void *data) 2054 { 2055 struct rb_vm_page_scan_info *info = data; 2056 vm_object_t backing_object; 2057 vm_object_t object; 2058 vm_pindex_t pindex; 2059 vm_pindex_t new_pindex; 2060 vm_pindex_t backing_offset_index; 2061 int op; 2062 2063 pindex = p->pindex; 2064 new_pindex = pindex - info->backing_offset_index; 2065 op = info->limit; 2066 object = info->object; 2067 backing_object = info->backing_object; 2068 backing_offset_index = info->backing_offset_index; 2069 2070 if (op & OBSC_TEST_ALL_SHADOWED) { 2071 vm_page_t pp; 2072 2073 /* 2074 * Ignore pages outside the parent object's range 2075 * and outside the parent object's mapping of the 2076 * backing object. 2077 * 2078 * note that we do not busy the backing object's 2079 * page. 2080 */ 2081 if (pindex < backing_offset_index || 2082 new_pindex >= object->size 2083 ) { 2084 return(0); 2085 } 2086 2087 /* 2088 * See if the parent has the page or if the parent's 2089 * object pager has the page. If the parent has the 2090 * page but the page is not valid, the parent's 2091 * object pager must have the page. 2092 * 2093 * If this fails, the parent does not completely shadow 2094 * the object and we might as well give up now. 2095 */ 2096 pp = vm_page_lookup(object, new_pindex); 2097 if ((pp == NULL || pp->valid == 0) && 2098 !vm_pager_has_page(object, new_pindex) 2099 ) { 2100 info->error = 0; /* problemo */ 2101 return(-1); /* stop the scan */ 2102 } 2103 } 2104 2105 /* 2106 * Check for busy page. Note that we may have lost (p) when we 2107 * possibly blocked above. 2108 */ 2109 if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) { 2110 vm_page_t pp; 2111 2112 if (vm_page_busy_try(p, TRUE)) { 2113 if (op & OBSC_COLLAPSE_NOWAIT) { 2114 return(0); 2115 } else { 2116 /* 2117 * If we slept, anything could have 2118 * happened. Ask that the scan be restarted. 2119 * 2120 * Since the object is marked dead, the 2121 * backing offset should not have changed. 2122 */ 2123 vm_page_sleep_busy(p, TRUE, "vmocol"); 2124 info->error = -1; 2125 return(-1); 2126 } 2127 } 2128 2129 /* 2130 * If (p) is no longer valid restart the scan. 2131 */ 2132 if (p->object != backing_object || p->pindex != pindex) { 2133 kprintf("vm_object_backing_scan: Warning: page " 2134 "%p ripped out from under us\n", p); 2135 vm_page_wakeup(p); 2136 info->error = -1; 2137 return(-1); 2138 } 2139 2140 if (op & OBSC_COLLAPSE_NOWAIT) { 2141 if (p->valid == 0 || 2142 p->wire_count || 2143 (p->flags & PG_NEED_COMMIT)) { 2144 vm_page_wakeup(p); 2145 return(0); 2146 } 2147 } else { 2148 /* XXX what if p->valid == 0 , hold_count, etc? */ 2149 } 2150 2151 KASSERT( 2152 p->object == backing_object, 2153 ("vm_object_qcollapse(): object mismatch") 2154 ); 2155 2156 /* 2157 * Destroy any associated swap 2158 */ 2159 if (backing_object->type == OBJT_SWAP) 2160 swap_pager_freespace(backing_object, p->pindex, 1); 2161 2162 if ( 2163 p->pindex < backing_offset_index || 2164 new_pindex >= object->size 2165 ) { 2166 /* 2167 * Page is out of the parent object's range, we 2168 * can simply destroy it. 2169 */ 2170 vm_page_protect(p, VM_PROT_NONE); 2171 vm_page_free(p); 2172 return(0); 2173 } 2174 2175 pp = vm_page_lookup(object, new_pindex); 2176 if (pp != NULL || vm_pager_has_page(object, new_pindex)) { 2177 /* 2178 * page already exists in parent OR swap exists 2179 * for this location in the parent. Destroy 2180 * the original page from the backing object. 2181 * 2182 * Leave the parent's page alone 2183 */ 2184 vm_page_protect(p, VM_PROT_NONE); 2185 vm_page_free(p); 2186 return(0); 2187 } 2188 2189 /* 2190 * Page does not exist in parent, rename the 2191 * page from the backing object to the main object. 2192 * 2193 * If the page was mapped to a process, it can remain 2194 * mapped through the rename. 2195 */ 2196 if ((p->queue - p->pc) == PQ_CACHE) 2197 vm_page_deactivate(p); 2198 2199 vm_page_rename(p, object, new_pindex); 2200 vm_page_wakeup(p); 2201 /* page automatically made dirty by rename */ 2202 } 2203 return(0); 2204 } 2205 2206 /* 2207 * This version of collapse allows the operation to occur earlier and 2208 * when paging_in_progress is true for an object... This is not a complete 2209 * operation, but should plug 99.9% of the rest of the leaks. 2210 * 2211 * The caller must hold the object and backing_object and both must be 2212 * chainlocked. 2213 * 2214 * (only called from vm_object_collapse) 2215 */ 2216 static void 2217 vm_object_qcollapse(vm_object_t object, vm_object_t backing_object) 2218 { 2219 if (backing_object->ref_count == 1) { 2220 atomic_add_int(&backing_object->ref_count, 2); 2221 #if defined(DEBUG_LOCKS) 2222 debugvm_object_add(backing_object, "qcollapse", 1, 2); 2223 #endif 2224 vm_object_backing_scan(object, backing_object, 2225 OBSC_COLLAPSE_NOWAIT); 2226 atomic_add_int(&backing_object->ref_count, -2); 2227 #if defined(DEBUG_LOCKS) 2228 debugvm_object_add(backing_object, "qcollapse", 2, -2); 2229 #endif 2230 } 2231 } 2232 2233 /* 2234 * Collapse an object with the object backing it. Pages in the backing 2235 * object are moved into the parent, and the backing object is deallocated. 2236 * Any conflict is resolved in favor of the parent's existing pages. 2237 * 2238 * object must be held and chain-locked on call. 2239 * 2240 * The caller must have an extra ref on object to prevent a race from 2241 * destroying it during the collapse. 2242 */ 2243 void 2244 vm_object_collapse(vm_object_t object, struct vm_object_dealloc_list **dlistp) 2245 { 2246 struct vm_object_dealloc_list *dlist = NULL; 2247 vm_object_t backing_object; 2248 2249 /* 2250 * Only one thread is attempting a collapse at any given moment. 2251 * There are few restrictions for (object) that callers of this 2252 * function check so reentrancy is likely. 2253 */ 2254 KKASSERT(object != NULL); 2255 vm_object_assert_held(object); 2256 KKASSERT(object->chainlk & (CHAINLK_MASK | CHAINLK_EXCL)); 2257 2258 for (;;) { 2259 vm_object_t bbobj; 2260 int dodealloc; 2261 2262 /* 2263 * We can only collapse a DEFAULT/SWAP object with a 2264 * DEFAULT/SWAP object. 2265 */ 2266 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP) { 2267 backing_object = NULL; 2268 break; 2269 } 2270 2271 backing_object = object->backing_object; 2272 if (backing_object == NULL) 2273 break; 2274 if (backing_object->type != OBJT_DEFAULT && 2275 backing_object->type != OBJT_SWAP) { 2276 backing_object = NULL; 2277 break; 2278 } 2279 2280 /* 2281 * Hold the backing_object and check for races 2282 */ 2283 vm_object_hold(backing_object); 2284 if (backing_object != object->backing_object || 2285 (backing_object->type != OBJT_DEFAULT && 2286 backing_object->type != OBJT_SWAP)) { 2287 vm_object_drop(backing_object); 2288 continue; 2289 } 2290 2291 /* 2292 * Chain-lock the backing object too because if we 2293 * successfully merge its pages into the top object we 2294 * will collapse backing_object->backing_object as the 2295 * new backing_object. Re-check that it is still our 2296 * backing object. 2297 */ 2298 vm_object_chain_acquire(backing_object, 0); 2299 if (backing_object != object->backing_object) { 2300 vm_object_chain_release(backing_object); 2301 vm_object_drop(backing_object); 2302 continue; 2303 } 2304 2305 /* 2306 * we check the backing object first, because it is most likely 2307 * not collapsable. 2308 */ 2309 if (backing_object->handle != NULL || 2310 (backing_object->type != OBJT_DEFAULT && 2311 backing_object->type != OBJT_SWAP) || 2312 (backing_object->flags & OBJ_DEAD) || 2313 object->handle != NULL || 2314 (object->type != OBJT_DEFAULT && 2315 object->type != OBJT_SWAP) || 2316 (object->flags & OBJ_DEAD)) { 2317 break; 2318 } 2319 2320 /* 2321 * If paging is in progress we can't do a normal collapse. 2322 */ 2323 if ( 2324 object->paging_in_progress != 0 || 2325 backing_object->paging_in_progress != 0 2326 ) { 2327 vm_object_qcollapse(object, backing_object); 2328 break; 2329 } 2330 2331 /* 2332 * We know that we can either collapse the backing object (if 2333 * the parent is the only reference to it) or (perhaps) have 2334 * the parent bypass the object if the parent happens to shadow 2335 * all the resident pages in the entire backing object. 2336 * 2337 * This is ignoring pager-backed pages such as swap pages. 2338 * vm_object_backing_scan fails the shadowing test in this 2339 * case. 2340 */ 2341 if (backing_object->ref_count == 1) { 2342 /* 2343 * If there is exactly one reference to the backing 2344 * object, we can collapse it into the parent. 2345 */ 2346 KKASSERT(object->backing_object == backing_object); 2347 vm_object_backing_scan(object, backing_object, 2348 OBSC_COLLAPSE_WAIT); 2349 2350 /* 2351 * Move the pager from backing_object to object. 2352 */ 2353 if (backing_object->type == OBJT_SWAP) { 2354 vm_object_pip_add(backing_object, 1); 2355 2356 /* 2357 * scrap the paging_offset junk and do a 2358 * discrete copy. This also removes major 2359 * assumptions about how the swap-pager 2360 * works from where it doesn't belong. The 2361 * new swapper is able to optimize the 2362 * destroy-source case. 2363 */ 2364 vm_object_pip_add(object, 1); 2365 swap_pager_copy(backing_object, object, 2366 OFF_TO_IDX(object->backing_object_offset), 2367 TRUE); 2368 vm_object_pip_wakeup(object); 2369 vm_object_pip_wakeup(backing_object); 2370 } 2371 2372 /* 2373 * Object now shadows whatever backing_object did. 2374 * Remove object from backing_object's shadow_list. 2375 * 2376 * Removing object from backing_objects shadow list 2377 * requires releasing object, which we will do below. 2378 */ 2379 KKASSERT(object->backing_object == backing_object); 2380 if (object->flags & OBJ_ONSHADOW) { 2381 LIST_REMOVE(object, shadow_list); 2382 backing_object->shadow_count--; 2383 atomic_add_int(&backing_object->generation, 1); 2384 vm_object_clear_flag(object, OBJ_ONSHADOW); 2385 } 2386 2387 /* 2388 * backing_object->backing_object moves from within 2389 * backing_object to within object. 2390 * 2391 * OBJT_VNODE bbobj's should have empty shadow lists. 2392 */ 2393 while ((bbobj = backing_object->backing_object) != NULL) { 2394 if (bbobj->type == OBJT_VNODE) 2395 vm_object_hold_shared(bbobj); 2396 else 2397 vm_object_hold(bbobj); 2398 if (bbobj == backing_object->backing_object) 2399 break; 2400 vm_object_drop(bbobj); 2401 } 2402 2403 /* 2404 * We are removing backing_object from bbobj's 2405 * shadow list and adding object to bbobj's shadow 2406 * list, so the ref_count on bbobj is unchanged. 2407 */ 2408 if (bbobj) { 2409 if (backing_object->flags & OBJ_ONSHADOW) { 2410 /* not locked exclusively if vnode */ 2411 KKASSERT(bbobj->type != OBJT_VNODE); 2412 LIST_REMOVE(backing_object, 2413 shadow_list); 2414 bbobj->shadow_count--; 2415 atomic_add_int(&bbobj->generation, 1); 2416 vm_object_clear_flag(backing_object, 2417 OBJ_ONSHADOW); 2418 } 2419 backing_object->backing_object = NULL; 2420 } 2421 object->backing_object = bbobj; 2422 if (bbobj) { 2423 if (bbobj->type != OBJT_VNODE) { 2424 LIST_INSERT_HEAD(&bbobj->shadow_head, 2425 object, shadow_list); 2426 bbobj->shadow_count++; 2427 atomic_add_int(&bbobj->generation, 1); 2428 vm_object_set_flag(object, 2429 OBJ_ONSHADOW); 2430 } 2431 } 2432 2433 object->backing_object_offset += 2434 backing_object->backing_object_offset; 2435 2436 vm_object_drop(bbobj); 2437 2438 /* 2439 * Discard the old backing_object. Nothing should be 2440 * able to ref it, other than a vm_map_split(), 2441 * and vm_map_split() will stall on our chain lock. 2442 * And we control the parent so it shouldn't be 2443 * possible for it to go away either. 2444 * 2445 * Since the backing object has no pages, no pager 2446 * left, and no object references within it, all 2447 * that is necessary is to dispose of it. 2448 */ 2449 KASSERT(backing_object->ref_count == 1, 2450 ("backing_object %p was somehow " 2451 "re-referenced during collapse!", 2452 backing_object)); 2453 KASSERT(RB_EMPTY(&backing_object->rb_memq), 2454 ("backing_object %p somehow has left " 2455 "over pages during collapse!", 2456 backing_object)); 2457 2458 /* 2459 * The object can be destroyed. 2460 * 2461 * XXX just fall through and dodealloc instead 2462 * of forcing destruction? 2463 */ 2464 atomic_add_int(&backing_object->ref_count, -1); 2465 #if defined(DEBUG_LOCKS) 2466 debugvm_object_add(backing_object, "collapse", 1, -1); 2467 #endif 2468 if ((backing_object->flags & OBJ_DEAD) == 0) 2469 vm_object_terminate(backing_object); 2470 object_collapses++; 2471 dodealloc = 0; 2472 } else { 2473 /* 2474 * If we do not entirely shadow the backing object, 2475 * there is nothing we can do so we give up. 2476 */ 2477 if (vm_object_backing_scan(object, backing_object, 2478 OBSC_TEST_ALL_SHADOWED) == 0) { 2479 break; 2480 } 2481 2482 /* 2483 * bbobj is backing_object->backing_object. Since 2484 * object completely shadows backing_object we can 2485 * bypass it and become backed by bbobj instead. 2486 * 2487 * The shadow list for vnode backing objects is not 2488 * used and a shared hold is allowed. 2489 */ 2490 while ((bbobj = backing_object->backing_object) != NULL) { 2491 if (bbobj->type == OBJT_VNODE) 2492 vm_object_hold_shared(bbobj); 2493 else 2494 vm_object_hold(bbobj); 2495 if (bbobj == backing_object->backing_object) 2496 break; 2497 vm_object_drop(bbobj); 2498 } 2499 2500 /* 2501 * Make object shadow bbobj instead of backing_object. 2502 * Remove object from backing_object's shadow list. 2503 * 2504 * Deallocating backing_object will not remove 2505 * it, since its reference count is at least 2. 2506 * 2507 * Removing object from backing_object's shadow 2508 * list requires releasing a ref, which we do 2509 * below by setting dodealloc to 1. 2510 */ 2511 KKASSERT(object->backing_object == backing_object); 2512 if (object->flags & OBJ_ONSHADOW) { 2513 LIST_REMOVE(object, shadow_list); 2514 backing_object->shadow_count--; 2515 atomic_add_int(&backing_object->generation, 1); 2516 vm_object_clear_flag(object, OBJ_ONSHADOW); 2517 } 2518 2519 /* 2520 * Add a ref to bbobj, bbobj now shadows object. 2521 * 2522 * NOTE: backing_object->backing_object still points 2523 * to bbobj. That relationship remains intact 2524 * because backing_object has > 1 ref, so 2525 * someone else is pointing to it (hence why 2526 * we can't collapse it into object and can 2527 * only handle the all-shadowed bypass case). 2528 */ 2529 if (bbobj) { 2530 if (bbobj->type != OBJT_VNODE) { 2531 vm_object_chain_wait(bbobj, 0); 2532 vm_object_reference_locked(bbobj); 2533 LIST_INSERT_HEAD(&bbobj->shadow_head, 2534 object, shadow_list); 2535 bbobj->shadow_count++; 2536 atomic_add_int(&bbobj->generation, 1); 2537 vm_object_set_flag(object, 2538 OBJ_ONSHADOW); 2539 } else { 2540 vm_object_reference_quick(bbobj); 2541 } 2542 object->backing_object_offset += 2543 backing_object->backing_object_offset; 2544 object->backing_object = bbobj; 2545 vm_object_drop(bbobj); 2546 } else { 2547 object->backing_object = NULL; 2548 } 2549 2550 /* 2551 * Drop the reference count on backing_object. To 2552 * handle ref_count races properly we can't assume 2553 * that the ref_count is still at least 2 so we 2554 * have to actually call vm_object_deallocate() 2555 * (after clearing the chainlock). 2556 */ 2557 object_bypasses++; 2558 dodealloc = 1; 2559 } 2560 2561 /* 2562 * Ok, we want to loop on the new object->bbobj association, 2563 * possibly collapsing it further. However if dodealloc is 2564 * non-zero we have to deallocate the backing_object which 2565 * itself can potentially undergo a collapse, creating a 2566 * recursion depth issue with the LWKT token subsystem. 2567 * 2568 * In the case where we must deallocate the backing_object 2569 * it is possible now that the backing_object has a single 2570 * shadow count on some other object (not represented here 2571 * as yet), since it no longer shadows us. Thus when we 2572 * call vm_object_deallocate() it may attempt to collapse 2573 * itself into its remaining parent. 2574 */ 2575 if (dodealloc) { 2576 struct vm_object_dealloc_list *dtmp; 2577 2578 vm_object_chain_release(backing_object); 2579 vm_object_unlock(backing_object); 2580 /* backing_object remains held */ 2581 2582 /* 2583 * Auto-deallocation list for caller convenience. 2584 */ 2585 if (dlistp == NULL) 2586 dlistp = &dlist; 2587 2588 dtmp = kmalloc(sizeof(*dtmp), M_TEMP, M_WAITOK); 2589 dtmp->object = backing_object; 2590 dtmp->next = *dlistp; 2591 *dlistp = dtmp; 2592 } else { 2593 vm_object_chain_release(backing_object); 2594 vm_object_drop(backing_object); 2595 } 2596 /* backing_object = NULL; not needed */ 2597 /* loop */ 2598 } 2599 2600 /* 2601 * Clean up any left over backing_object 2602 */ 2603 if (backing_object) { 2604 vm_object_chain_release(backing_object); 2605 vm_object_drop(backing_object); 2606 } 2607 2608 /* 2609 * Clean up any auto-deallocation list. This is a convenience 2610 * for top-level callers so they don't have to pass &dlist. 2611 * Do not clean up any caller-passed dlistp, the caller will 2612 * do that. 2613 */ 2614 if (dlist) 2615 vm_object_deallocate_list(&dlist); 2616 2617 } 2618 2619 /* 2620 * vm_object_collapse() may collect additional objects in need of 2621 * deallocation. This routine deallocates these objects. The 2622 * deallocation itself can trigger additional collapses (which the 2623 * deallocate function takes care of). This procedure is used to 2624 * reduce procedural recursion since these vm_object shadow chains 2625 * can become quite long. 2626 */ 2627 void 2628 vm_object_deallocate_list(struct vm_object_dealloc_list **dlistp) 2629 { 2630 struct vm_object_dealloc_list *dlist; 2631 2632 while ((dlist = *dlistp) != NULL) { 2633 *dlistp = dlist->next; 2634 vm_object_lock(dlist->object); 2635 vm_object_deallocate_locked(dlist->object); 2636 vm_object_drop(dlist->object); 2637 kfree(dlist, M_TEMP); 2638 } 2639 } 2640 2641 /* 2642 * Removes all physical pages in the specified object range from the 2643 * object's list of pages. 2644 * 2645 * No requirements. 2646 */ 2647 static int vm_object_page_remove_callback(vm_page_t p, void *data); 2648 2649 void 2650 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 2651 boolean_t clean_only) 2652 { 2653 struct rb_vm_page_scan_info info; 2654 int all; 2655 2656 /* 2657 * Degenerate cases and assertions 2658 */ 2659 vm_object_hold(object); 2660 if (object == NULL || 2661 (object->resident_page_count == 0 && object->swblock_count == 0)) { 2662 vm_object_drop(object); 2663 return; 2664 } 2665 KASSERT(object->type != OBJT_PHYS, 2666 ("attempt to remove pages from a physical object")); 2667 2668 /* 2669 * Indicate that paging is occuring on the object 2670 */ 2671 vm_object_pip_add(object, 1); 2672 2673 /* 2674 * Figure out the actual removal range and whether we are removing 2675 * the entire contents of the object or not. If removing the entire 2676 * contents, be sure to get all pages, even those that might be 2677 * beyond the end of the object. 2678 */ 2679 info.object = object; 2680 info.start_pindex = start; 2681 if (end == 0) 2682 info.end_pindex = (vm_pindex_t)-1; 2683 else 2684 info.end_pindex = end - 1; 2685 info.limit = clean_only; 2686 info.count = 0; 2687 all = (start == 0 && info.end_pindex >= object->size - 1); 2688 2689 /* 2690 * Loop until we are sure we have gotten them all. 2691 */ 2692 do { 2693 info.error = 0; 2694 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp, 2695 vm_object_page_remove_callback, &info); 2696 } while (info.error); 2697 2698 /* 2699 * Remove any related swap if throwing away pages, or for 2700 * non-swap objects (the swap is a clean copy in that case). 2701 */ 2702 if (object->type != OBJT_SWAP || clean_only == FALSE) { 2703 if (all) 2704 swap_pager_freespace_all(object); 2705 else 2706 swap_pager_freespace(object, info.start_pindex, 2707 info.end_pindex - info.start_pindex + 1); 2708 } 2709 2710 /* 2711 * Cleanup 2712 */ 2713 vm_object_pip_wakeup(object); 2714 vm_object_drop(object); 2715 } 2716 2717 /* 2718 * The caller must hold the object. 2719 * 2720 * NOTE: User yields are allowed when removing more than one page, but not 2721 * allowed if only removing one page (the path for single page removals 2722 * might hold a spinlock). 2723 */ 2724 static int 2725 vm_object_page_remove_callback(vm_page_t p, void *data) 2726 { 2727 struct rb_vm_page_scan_info *info = data; 2728 2729 if (info->object != p->object || 2730 p->pindex < info->start_pindex || 2731 p->pindex > info->end_pindex) { 2732 kprintf("vm_object_page_remove_callbackA: obj/pg race %p/%p\n", 2733 info->object, p); 2734 return(0); 2735 } 2736 if (vm_page_busy_try(p, TRUE)) { 2737 vm_page_sleep_busy(p, TRUE, "vmopar"); 2738 info->error = 1; 2739 return(0); 2740 } 2741 if (info->object != p->object) { 2742 /* this should never happen */ 2743 kprintf("vm_object_page_remove_callbackB: obj/pg race %p/%p\n", 2744 info->object, p); 2745 vm_page_wakeup(p); 2746 return(0); 2747 } 2748 2749 /* 2750 * Wired pages cannot be destroyed, but they can be invalidated 2751 * and we do so if clean_only (limit) is not set. 2752 * 2753 * WARNING! The page may be wired due to being part of a buffer 2754 * cache buffer, and the buffer might be marked B_CACHE. 2755 * This is fine as part of a truncation but VFSs must be 2756 * sure to fix the buffer up when re-extending the file. 2757 * 2758 * NOTE! PG_NEED_COMMIT is ignored. 2759 */ 2760 if (p->wire_count != 0) { 2761 vm_page_protect(p, VM_PROT_NONE); 2762 if (info->limit == 0) 2763 p->valid = 0; 2764 vm_page_wakeup(p); 2765 goto done; 2766 } 2767 2768 /* 2769 * limit is our clean_only flag. If set and the page is dirty or 2770 * requires a commit, do not free it. If set and the page is being 2771 * held by someone, do not free it. 2772 */ 2773 if (info->limit && p->valid) { 2774 vm_page_test_dirty(p); 2775 if ((p->valid & p->dirty) || (p->flags & PG_NEED_COMMIT)) { 2776 vm_page_wakeup(p); 2777 goto done; 2778 } 2779 } 2780 2781 /* 2782 * Destroy the page 2783 */ 2784 vm_page_protect(p, VM_PROT_NONE); 2785 vm_page_free(p); 2786 2787 /* 2788 * Must be at end to avoid SMP races, caller holds object token 2789 */ 2790 done: 2791 if ((++info->count & 63) == 0) 2792 lwkt_user_yield(); 2793 2794 return(0); 2795 } 2796 2797 /* 2798 * Coalesces two objects backing up adjoining regions of memory into a 2799 * single object. 2800 * 2801 * returns TRUE if objects were combined. 2802 * 2803 * NOTE: Only works at the moment if the second object is NULL - 2804 * if it's not, which object do we lock first? 2805 * 2806 * Parameters: 2807 * prev_object First object to coalesce 2808 * prev_offset Offset into prev_object 2809 * next_object Second object into coalesce 2810 * next_offset Offset into next_object 2811 * 2812 * prev_size Size of reference to prev_object 2813 * next_size Size of reference to next_object 2814 * 2815 * The caller does not need to hold (prev_object) but must have a stable 2816 * pointer to it (typically by holding the vm_map locked). 2817 */ 2818 boolean_t 2819 vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex, 2820 vm_size_t prev_size, vm_size_t next_size) 2821 { 2822 vm_pindex_t next_pindex; 2823 2824 if (prev_object == NULL) 2825 return (TRUE); 2826 2827 vm_object_hold(prev_object); 2828 2829 if (prev_object->type != OBJT_DEFAULT && 2830 prev_object->type != OBJT_SWAP) { 2831 vm_object_drop(prev_object); 2832 return (FALSE); 2833 } 2834 2835 /* 2836 * Try to collapse the object first 2837 */ 2838 vm_object_chain_acquire(prev_object, 0); 2839 vm_object_collapse(prev_object, NULL); 2840 2841 /* 2842 * Can't coalesce if: . more than one reference . paged out . shadows 2843 * another object . has a copy elsewhere (any of which mean that the 2844 * pages not mapped to prev_entry may be in use anyway) 2845 */ 2846 2847 if (prev_object->backing_object != NULL) { 2848 vm_object_chain_release(prev_object); 2849 vm_object_drop(prev_object); 2850 return (FALSE); 2851 } 2852 2853 prev_size >>= PAGE_SHIFT; 2854 next_size >>= PAGE_SHIFT; 2855 next_pindex = prev_pindex + prev_size; 2856 2857 if ((prev_object->ref_count > 1) && 2858 (prev_object->size != next_pindex)) { 2859 vm_object_chain_release(prev_object); 2860 vm_object_drop(prev_object); 2861 return (FALSE); 2862 } 2863 2864 /* 2865 * Remove any pages that may still be in the object from a previous 2866 * deallocation. 2867 */ 2868 if (next_pindex < prev_object->size) { 2869 vm_object_page_remove(prev_object, 2870 next_pindex, 2871 next_pindex + next_size, FALSE); 2872 if (prev_object->type == OBJT_SWAP) 2873 swap_pager_freespace(prev_object, 2874 next_pindex, next_size); 2875 } 2876 2877 /* 2878 * Extend the object if necessary. 2879 */ 2880 if (next_pindex + next_size > prev_object->size) 2881 prev_object->size = next_pindex + next_size; 2882 2883 vm_object_chain_release(prev_object); 2884 vm_object_drop(prev_object); 2885 return (TRUE); 2886 } 2887 2888 /* 2889 * Make the object writable and flag is being possibly dirty. 2890 * 2891 * The object might not be held (or might be held but held shared), 2892 * the related vnode is probably not held either. Object and vnode are 2893 * stable by virtue of the vm_page busied by the caller preventing 2894 * destruction. 2895 * 2896 * If the related mount is flagged MNTK_THR_SYNC we need to call 2897 * vsetobjdirty(). Filesystems using this option usually shortcut 2898 * synchronization by only scanning the syncer list. 2899 */ 2900 void 2901 vm_object_set_writeable_dirty(vm_object_t object) 2902 { 2903 struct vnode *vp; 2904 2905 /*vm_object_assert_held(object);*/ 2906 /* 2907 * Avoid contention in vm fault path by checking the state before 2908 * issuing an atomic op on it. 2909 */ 2910 if ((object->flags & (OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY)) != 2911 (OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY)) { 2912 vm_object_set_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 2913 } 2914 if (object->type == OBJT_VNODE && 2915 (vp = (struct vnode *)object->handle) != NULL) { 2916 if ((vp->v_flag & VOBJDIRTY) == 0) { 2917 if (vp->v_mount && 2918 (vp->v_mount->mnt_kern_flag & MNTK_THR_SYNC)) { 2919 /* 2920 * New style THR_SYNC places vnodes on the 2921 * syncer list more deterministically. 2922 */ 2923 vsetobjdirty(vp); 2924 } else { 2925 /* 2926 * Old style scan would not necessarily place 2927 * a vnode on the syncer list when possibly 2928 * modified via mmap. 2929 */ 2930 vsetflags(vp, VOBJDIRTY); 2931 } 2932 } 2933 } 2934 } 2935 2936 #include "opt_ddb.h" 2937 #ifdef DDB 2938 #include <sys/kernel.h> 2939 2940 #include <sys/cons.h> 2941 2942 #include <ddb/ddb.h> 2943 2944 static int _vm_object_in_map (vm_map_t map, vm_object_t object, 2945 vm_map_entry_t entry); 2946 static int vm_object_in_map (vm_object_t object); 2947 2948 /* 2949 * The caller must hold the object. 2950 */ 2951 static int 2952 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry) 2953 { 2954 vm_map_t tmpm; 2955 vm_map_entry_t tmpe; 2956 vm_object_t obj, nobj; 2957 int entcount; 2958 2959 if (map == 0) 2960 return 0; 2961 if (entry == 0) { 2962 tmpe = map->header.next; 2963 entcount = map->nentries; 2964 while (entcount-- && (tmpe != &map->header)) { 2965 if( _vm_object_in_map(map, object, tmpe)) { 2966 return 1; 2967 } 2968 tmpe = tmpe->next; 2969 } 2970 return (0); 2971 } 2972 switch(entry->maptype) { 2973 case VM_MAPTYPE_SUBMAP: 2974 tmpm = entry->object.sub_map; 2975 tmpe = tmpm->header.next; 2976 entcount = tmpm->nentries; 2977 while (entcount-- && tmpe != &tmpm->header) { 2978 if( _vm_object_in_map(tmpm, object, tmpe)) { 2979 return 1; 2980 } 2981 tmpe = tmpe->next; 2982 } 2983 break; 2984 case VM_MAPTYPE_NORMAL: 2985 case VM_MAPTYPE_VPAGETABLE: 2986 obj = entry->object.vm_object; 2987 while (obj) { 2988 if (obj == object) { 2989 if (obj != entry->object.vm_object) 2990 vm_object_drop(obj); 2991 return 1; 2992 } 2993 while ((nobj = obj->backing_object) != NULL) { 2994 vm_object_hold(nobj); 2995 if (nobj == obj->backing_object) 2996 break; 2997 vm_object_drop(nobj); 2998 } 2999 if (obj != entry->object.vm_object) { 3000 if (nobj) 3001 vm_object_lock_swap(); 3002 vm_object_drop(obj); 3003 } 3004 obj = nobj; 3005 } 3006 break; 3007 default: 3008 break; 3009 } 3010 return 0; 3011 } 3012 3013 static int vm_object_in_map_callback(struct proc *p, void *data); 3014 3015 struct vm_object_in_map_info { 3016 vm_object_t object; 3017 int rv; 3018 }; 3019 3020 /* 3021 * Debugging only 3022 */ 3023 static int 3024 vm_object_in_map(vm_object_t object) 3025 { 3026 struct vm_object_in_map_info info; 3027 3028 info.rv = 0; 3029 info.object = object; 3030 3031 allproc_scan(vm_object_in_map_callback, &info); 3032 if (info.rv) 3033 return 1; 3034 if( _vm_object_in_map(&kernel_map, object, 0)) 3035 return 1; 3036 if( _vm_object_in_map(&pager_map, object, 0)) 3037 return 1; 3038 if( _vm_object_in_map(&buffer_map, object, 0)) 3039 return 1; 3040 return 0; 3041 } 3042 3043 /* 3044 * Debugging only 3045 */ 3046 static int 3047 vm_object_in_map_callback(struct proc *p, void *data) 3048 { 3049 struct vm_object_in_map_info *info = data; 3050 3051 if (p->p_vmspace) { 3052 if (_vm_object_in_map(&p->p_vmspace->vm_map, info->object, 0)) { 3053 info->rv = 1; 3054 return -1; 3055 } 3056 } 3057 return (0); 3058 } 3059 3060 DB_SHOW_COMMAND(vmochk, vm_object_check) 3061 { 3062 struct vm_object_hash *hash; 3063 vm_object_t object; 3064 int n; 3065 3066 /* 3067 * make sure that internal objs are in a map somewhere 3068 * and none have zero ref counts. 3069 */ 3070 for (n = 0; n < VMOBJ_HSIZE; ++n) { 3071 hash = &vm_object_hash[n]; 3072 for (object = TAILQ_FIRST(&hash->list); 3073 object != NULL; 3074 object = TAILQ_NEXT(object, object_list)) { 3075 if (object->type == OBJT_MARKER) 3076 continue; 3077 if (object->handle != NULL || 3078 (object->type != OBJT_DEFAULT && 3079 object->type != OBJT_SWAP)) { 3080 continue; 3081 } 3082 if (object->ref_count == 0) { 3083 db_printf("vmochk: internal obj has " 3084 "zero ref count: %ld\n", 3085 (long)object->size); 3086 } 3087 if (vm_object_in_map(object)) 3088 continue; 3089 db_printf("vmochk: internal obj is not in a map: " 3090 "ref: %d, size: %lu: 0x%lx, " 3091 "backing_object: %p\n", 3092 object->ref_count, (u_long)object->size, 3093 (u_long)object->size, 3094 (void *)object->backing_object); 3095 } 3096 } 3097 } 3098 3099 /* 3100 * Debugging only 3101 */ 3102 DB_SHOW_COMMAND(object, vm_object_print_static) 3103 { 3104 /* XXX convert args. */ 3105 vm_object_t object = (vm_object_t)addr; 3106 boolean_t full = have_addr; 3107 3108 vm_page_t p; 3109 3110 /* XXX count is an (unused) arg. Avoid shadowing it. */ 3111 #define count was_count 3112 3113 int count; 3114 3115 if (object == NULL) 3116 return; 3117 3118 db_iprintf( 3119 "Object %p: type=%d, size=0x%lx, res=%ld, ref=%d, flags=0x%x\n", 3120 object, (int)object->type, (u_long)object->size, 3121 object->resident_page_count, object->ref_count, object->flags); 3122 /* 3123 * XXX no %qd in kernel. Truncate object->backing_object_offset. 3124 */ 3125 db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%lx\n", 3126 object->shadow_count, 3127 object->backing_object ? object->backing_object->ref_count : 0, 3128 object->backing_object, (long)object->backing_object_offset); 3129 3130 if (!full) 3131 return; 3132 3133 db_indent += 2; 3134 count = 0; 3135 RB_FOREACH(p, vm_page_rb_tree, &object->rb_memq) { 3136 if (count == 0) 3137 db_iprintf("memory:="); 3138 else if (count == 6) { 3139 db_printf("\n"); 3140 db_iprintf(" ..."); 3141 count = 0; 3142 } else 3143 db_printf(","); 3144 count++; 3145 3146 db_printf("(off=0x%lx,page=0x%lx)", 3147 (u_long) p->pindex, (u_long) VM_PAGE_TO_PHYS(p)); 3148 } 3149 if (count != 0) 3150 db_printf("\n"); 3151 db_indent -= 2; 3152 } 3153 3154 /* XXX. */ 3155 #undef count 3156 3157 /* 3158 * XXX need this non-static entry for calling from vm_map_print. 3159 * 3160 * Debugging only 3161 */ 3162 void 3163 vm_object_print(/* db_expr_t */ long addr, 3164 boolean_t have_addr, 3165 /* db_expr_t */ long count, 3166 char *modif) 3167 { 3168 vm_object_print_static(addr, have_addr, count, modif); 3169 } 3170 3171 /* 3172 * Debugging only 3173 */ 3174 DB_SHOW_COMMAND(vmopag, vm_object_print_pages) 3175 { 3176 struct vm_object_hash *hash; 3177 vm_object_t object; 3178 int nl = 0; 3179 int c; 3180 int n; 3181 3182 for (n = 0; n < VMOBJ_HSIZE; ++n) { 3183 hash = &vm_object_hash[n]; 3184 for (object = TAILQ_FIRST(&hash->list); 3185 object != NULL; 3186 object = TAILQ_NEXT(object, object_list)) { 3187 vm_pindex_t idx, fidx; 3188 vm_pindex_t osize; 3189 vm_paddr_t pa = -1, padiff; 3190 int rcount; 3191 vm_page_t m; 3192 3193 if (object->type == OBJT_MARKER) 3194 continue; 3195 db_printf("new object: %p\n", (void *)object); 3196 if ( nl > 18) { 3197 c = cngetc(); 3198 if (c != ' ') 3199 return; 3200 nl = 0; 3201 } 3202 nl++; 3203 rcount = 0; 3204 fidx = 0; 3205 osize = object->size; 3206 if (osize > 128) 3207 osize = 128; 3208 for (idx = 0; idx < osize; idx++) { 3209 m = vm_page_lookup(object, idx); 3210 if (m == NULL) { 3211 if (rcount) { 3212 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 3213 (long)fidx, rcount, (long)pa); 3214 if ( nl > 18) { 3215 c = cngetc(); 3216 if (c != ' ') 3217 return; 3218 nl = 0; 3219 } 3220 nl++; 3221 rcount = 0; 3222 } 3223 continue; 3224 } 3225 3226 if (rcount && 3227 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) { 3228 ++rcount; 3229 continue; 3230 } 3231 if (rcount) { 3232 padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m); 3233 padiff >>= PAGE_SHIFT; 3234 padiff &= PQ_L2_MASK; 3235 if (padiff == 0) { 3236 pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE; 3237 ++rcount; 3238 continue; 3239 } 3240 db_printf(" index(%ld)run(%d)pa(0x%lx)", 3241 (long)fidx, rcount, (long)pa); 3242 db_printf("pd(%ld)\n", (long)padiff); 3243 if ( nl > 18) { 3244 c = cngetc(); 3245 if (c != ' ') 3246 return; 3247 nl = 0; 3248 } 3249 nl++; 3250 } 3251 fidx = idx; 3252 pa = VM_PAGE_TO_PHYS(m); 3253 rcount = 1; 3254 } 3255 if (rcount) { 3256 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 3257 (long)fidx, rcount, (long)pa); 3258 if ( nl > 18) { 3259 c = cngetc(); 3260 if (c != ' ') 3261 return; 3262 nl = 0; 3263 } 3264 nl++; 3265 } 3266 } 3267 } 3268 } 3269 #endif /* DDB */ 3270