1 /*- 2 * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU) 3 * 4 * Copyright (c) 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * The Mach Operating System project at Carnegie-Mellon University. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 35 * 36 * 37 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 38 * All rights reserved. 39 * 40 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 41 * 42 * Permission to use, copy, modify and distribute this software and 43 * its documentation is hereby granted, provided that both the copyright 44 * notice and this permission notice appear in all copies of the 45 * software, derivative works or modified versions, and any portions 46 * thereof, and that both notices appear in supporting documentation. 47 * 48 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 49 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 50 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 51 * 52 * Carnegie Mellon requests users of this software to return to 53 * 54 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 55 * School of Computer Science 56 * Carnegie Mellon University 57 * Pittsburgh PA 15213-3890 58 * 59 * any improvements or extensions that they make and grant Carnegie the 60 * rights to redistribute these changes. 61 */ 62 63 /* 64 * Virtual memory object module. 65 */ 66 67 #include <sys/cdefs.h> 68 __FBSDID("$FreeBSD$"); 69 70 #include "opt_vm.h" 71 72 #include <sys/param.h> 73 #include <sys/systm.h> 74 #include <sys/cpuset.h> 75 #include <sys/lock.h> 76 #include <sys/mman.h> 77 #include <sys/mount.h> 78 #include <sys/kernel.h> 79 #include <sys/pctrie.h> 80 #include <sys/sysctl.h> 81 #include <sys/mutex.h> 82 #include <sys/proc.h> /* for curproc, pageproc */ 83 #include <sys/refcount.h> 84 #include <sys/socket.h> 85 #include <sys/resourcevar.h> 86 #include <sys/rwlock.h> 87 #include <sys/user.h> 88 #include <sys/vnode.h> 89 #include <sys/vmmeter.h> 90 #include <sys/sx.h> 91 92 #include <vm/vm.h> 93 #include <vm/vm_param.h> 94 #include <vm/pmap.h> 95 #include <vm/vm_map.h> 96 #include <vm/vm_object.h> 97 #include <vm/vm_page.h> 98 #include <vm/vm_pageout.h> 99 #include <vm/vm_pager.h> 100 #include <vm/vm_phys.h> 101 #include <vm/vm_pagequeue.h> 102 #include <vm/swap_pager.h> 103 #include <vm/vm_kern.h> 104 #include <vm/vm_extern.h> 105 #include <vm/vm_radix.h> 106 #include <vm/vm_reserv.h> 107 #include <vm/uma.h> 108 109 static int old_msync; 110 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0, 111 "Use old (insecure) msync behavior"); 112 113 static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, 114 int pagerflags, int flags, boolean_t *clearobjflags, 115 boolean_t *eio); 116 static boolean_t vm_object_page_remove_write(vm_page_t p, int flags, 117 boolean_t *clearobjflags); 118 static void vm_object_qcollapse(vm_object_t object); 119 static void vm_object_vndeallocate(vm_object_t object); 120 121 /* 122 * Virtual memory objects maintain the actual data 123 * associated with allocated virtual memory. A given 124 * page of memory exists within exactly one object. 125 * 126 * An object is only deallocated when all "references" 127 * are given up. Only one "reference" to a given 128 * region of an object should be writeable. 129 * 130 * Associated with each object is a list of all resident 131 * memory pages belonging to that object; this list is 132 * maintained by the "vm_page" module, and locked by the object's 133 * lock. 134 * 135 * Each object also records a "pager" routine which is 136 * used to retrieve (and store) pages to the proper backing 137 * storage. In addition, objects may be backed by other 138 * objects from which they were virtual-copied. 139 * 140 * The only items within the object structure which are 141 * modified after time of creation are: 142 * reference count locked by object's lock 143 * pager routine locked by object's lock 144 * 145 */ 146 147 struct object_q vm_object_list; 148 struct mtx vm_object_list_mtx; /* lock for object list and count */ 149 150 struct vm_object kernel_object_store; 151 152 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0, 153 "VM object stats"); 154 155 static counter_u64_t object_collapses = EARLY_COUNTER; 156 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD, 157 &object_collapses, 158 "VM object collapses"); 159 160 static counter_u64_t object_bypasses = EARLY_COUNTER; 161 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD, 162 &object_bypasses, 163 "VM object bypasses"); 164 165 static void 166 counter_startup(void) 167 { 168 169 object_collapses = counter_u64_alloc(M_WAITOK); 170 object_bypasses = counter_u64_alloc(M_WAITOK); 171 } 172 SYSINIT(object_counters, SI_SUB_CPU, SI_ORDER_ANY, counter_startup, NULL); 173 174 static uma_zone_t obj_zone; 175 176 static int vm_object_zinit(void *mem, int size, int flags); 177 178 #ifdef INVARIANTS 179 static void vm_object_zdtor(void *mem, int size, void *arg); 180 181 static void 182 vm_object_zdtor(void *mem, int size, void *arg) 183 { 184 vm_object_t object; 185 186 object = (vm_object_t)mem; 187 KASSERT(object->ref_count == 0, 188 ("object %p ref_count = %d", object, object->ref_count)); 189 KASSERT(TAILQ_EMPTY(&object->memq), 190 ("object %p has resident pages in its memq", object)); 191 KASSERT(vm_radix_is_empty(&object->rtree), 192 ("object %p has resident pages in its trie", object)); 193 #if VM_NRESERVLEVEL > 0 194 KASSERT(LIST_EMPTY(&object->rvq), 195 ("object %p has reservations", 196 object)); 197 #endif 198 KASSERT(object->paging_in_progress == 0, 199 ("object %p paging_in_progress = %d", 200 object, object->paging_in_progress)); 201 KASSERT(object->resident_page_count == 0, 202 ("object %p resident_page_count = %d", 203 object, object->resident_page_count)); 204 KASSERT(object->shadow_count == 0, 205 ("object %p shadow_count = %d", 206 object, object->shadow_count)); 207 KASSERT(object->type == OBJT_DEAD, 208 ("object %p has non-dead type %d", 209 object, object->type)); 210 } 211 #endif 212 213 static int 214 vm_object_zinit(void *mem, int size, int flags) 215 { 216 vm_object_t object; 217 218 object = (vm_object_t)mem; 219 rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW); 220 221 /* These are true for any object that has been freed */ 222 object->type = OBJT_DEAD; 223 object->ref_count = 0; 224 vm_radix_init(&object->rtree); 225 refcount_init(&object->paging_in_progress, 0); 226 object->resident_page_count = 0; 227 object->shadow_count = 0; 228 object->flags = OBJ_DEAD; 229 230 mtx_lock(&vm_object_list_mtx); 231 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 232 mtx_unlock(&vm_object_list_mtx); 233 return (0); 234 } 235 236 static void 237 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object) 238 { 239 240 TAILQ_INIT(&object->memq); 241 LIST_INIT(&object->shadow_head); 242 243 object->type = type; 244 if (type == OBJT_SWAP) 245 pctrie_init(&object->un_pager.swp.swp_blks); 246 247 /* 248 * Ensure that swap_pager_swapoff() iteration over object_list 249 * sees up to date type and pctrie head if it observed 250 * non-dead object. 251 */ 252 atomic_thread_fence_rel(); 253 254 switch (type) { 255 case OBJT_DEAD: 256 panic("_vm_object_allocate: can't create OBJT_DEAD"); 257 case OBJT_DEFAULT: 258 case OBJT_SWAP: 259 object->flags = OBJ_ONEMAPPING; 260 break; 261 case OBJT_DEVICE: 262 case OBJT_SG: 263 object->flags = OBJ_FICTITIOUS | OBJ_UNMANAGED; 264 break; 265 case OBJT_MGTDEVICE: 266 object->flags = OBJ_FICTITIOUS; 267 break; 268 case OBJT_PHYS: 269 object->flags = OBJ_UNMANAGED; 270 break; 271 case OBJT_VNODE: 272 object->flags = 0; 273 break; 274 default: 275 panic("_vm_object_allocate: type %d is undefined", type); 276 } 277 object->size = size; 278 object->domain.dr_policy = NULL; 279 object->generation = 1; 280 object->ref_count = 1; 281 object->memattr = VM_MEMATTR_DEFAULT; 282 object->cred = NULL; 283 object->charge = 0; 284 object->handle = NULL; 285 object->backing_object = NULL; 286 object->backing_object_offset = (vm_ooffset_t) 0; 287 #if VM_NRESERVLEVEL > 0 288 LIST_INIT(&object->rvq); 289 #endif 290 umtx_shm_object_init(object); 291 } 292 293 /* 294 * vm_object_init: 295 * 296 * Initialize the VM objects module. 297 */ 298 void 299 vm_object_init(void) 300 { 301 TAILQ_INIT(&vm_object_list); 302 mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF); 303 304 rw_init(&kernel_object->lock, "kernel vm object"); 305 _vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS - 306 VM_MIN_KERNEL_ADDRESS), kernel_object); 307 #if VM_NRESERVLEVEL > 0 308 kernel_object->flags |= OBJ_COLORED; 309 kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS); 310 #endif 311 312 /* 313 * The lock portion of struct vm_object must be type stable due 314 * to vm_pageout_fallback_object_lock locking a vm object 315 * without holding any references to it. 316 */ 317 obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL, 318 #ifdef INVARIANTS 319 vm_object_zdtor, 320 #else 321 NULL, 322 #endif 323 vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); 324 325 vm_radix_zinit(); 326 } 327 328 void 329 vm_object_clear_flag(vm_object_t object, u_short bits) 330 { 331 332 VM_OBJECT_ASSERT_WLOCKED(object); 333 object->flags &= ~bits; 334 } 335 336 /* 337 * Sets the default memory attribute for the specified object. Pages 338 * that are allocated to this object are by default assigned this memory 339 * attribute. 340 * 341 * Presently, this function must be called before any pages are allocated 342 * to the object. In the future, this requirement may be relaxed for 343 * "default" and "swap" objects. 344 */ 345 int 346 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr) 347 { 348 349 VM_OBJECT_ASSERT_WLOCKED(object); 350 switch (object->type) { 351 case OBJT_DEFAULT: 352 case OBJT_DEVICE: 353 case OBJT_MGTDEVICE: 354 case OBJT_PHYS: 355 case OBJT_SG: 356 case OBJT_SWAP: 357 case OBJT_VNODE: 358 if (!TAILQ_EMPTY(&object->memq)) 359 return (KERN_FAILURE); 360 break; 361 case OBJT_DEAD: 362 return (KERN_INVALID_ARGUMENT); 363 default: 364 panic("vm_object_set_memattr: object %p is of undefined type", 365 object); 366 } 367 object->memattr = memattr; 368 return (KERN_SUCCESS); 369 } 370 371 void 372 vm_object_pip_add(vm_object_t object, short i) 373 { 374 375 refcount_acquiren(&object->paging_in_progress, i); 376 } 377 378 void 379 vm_object_pip_wakeup(vm_object_t object) 380 { 381 382 refcount_release(&object->paging_in_progress); 383 } 384 385 void 386 vm_object_pip_wakeupn(vm_object_t object, short i) 387 { 388 389 refcount_releasen(&object->paging_in_progress, i); 390 } 391 392 void 393 vm_object_pip_wait(vm_object_t object, char *waitid) 394 { 395 396 VM_OBJECT_ASSERT_WLOCKED(object); 397 398 while (object->paging_in_progress) { 399 VM_OBJECT_WUNLOCK(object); 400 refcount_wait(&object->paging_in_progress, waitid, PVM); 401 VM_OBJECT_WLOCK(object); 402 } 403 } 404 405 void 406 vm_object_pip_wait_unlocked(vm_object_t object, char *waitid) 407 { 408 409 VM_OBJECT_ASSERT_UNLOCKED(object); 410 411 while (object->paging_in_progress) 412 refcount_wait(&object->paging_in_progress, waitid, PVM); 413 } 414 415 /* 416 * vm_object_allocate: 417 * 418 * Returns a new object with the given size. 419 */ 420 vm_object_t 421 vm_object_allocate(objtype_t type, vm_pindex_t size) 422 { 423 vm_object_t object; 424 425 object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK); 426 _vm_object_allocate(type, size, object); 427 return (object); 428 } 429 430 431 /* 432 * vm_object_reference: 433 * 434 * Gets another reference to the given object. Note: OBJ_DEAD 435 * objects can be referenced during final cleaning. 436 */ 437 void 438 vm_object_reference(vm_object_t object) 439 { 440 if (object == NULL) 441 return; 442 VM_OBJECT_WLOCK(object); 443 vm_object_reference_locked(object); 444 VM_OBJECT_WUNLOCK(object); 445 } 446 447 /* 448 * vm_object_reference_locked: 449 * 450 * Gets another reference to the given object. 451 * 452 * The object must be locked. 453 */ 454 void 455 vm_object_reference_locked(vm_object_t object) 456 { 457 struct vnode *vp; 458 459 VM_OBJECT_ASSERT_WLOCKED(object); 460 object->ref_count++; 461 if (object->type == OBJT_VNODE) { 462 vp = object->handle; 463 vref(vp); 464 } 465 } 466 467 /* 468 * Handle deallocating an object of type OBJT_VNODE. 469 */ 470 static void 471 vm_object_vndeallocate(vm_object_t object) 472 { 473 struct vnode *vp = (struct vnode *) object->handle; 474 475 VM_OBJECT_ASSERT_WLOCKED(object); 476 KASSERT(object->type == OBJT_VNODE, 477 ("vm_object_vndeallocate: not a vnode object")); 478 KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); 479 #ifdef INVARIANTS 480 if (object->ref_count == 0) { 481 vn_printf(vp, "vm_object_vndeallocate "); 482 panic("vm_object_vndeallocate: bad object reference count"); 483 } 484 #endif 485 486 if (!umtx_shm_vnobj_persistent && object->ref_count == 1) 487 umtx_shm_object_terminated(object); 488 489 object->ref_count--; 490 491 /* vrele may need the vnode lock. */ 492 VM_OBJECT_WUNLOCK(object); 493 vrele(vp); 494 } 495 496 /* 497 * vm_object_deallocate: 498 * 499 * Release a reference to the specified object, 500 * gained either through a vm_object_allocate 501 * or a vm_object_reference call. When all references 502 * are gone, storage associated with this object 503 * may be relinquished. 504 * 505 * No object may be locked. 506 */ 507 void 508 vm_object_deallocate(vm_object_t object) 509 { 510 vm_object_t temp; 511 512 while (object != NULL) { 513 VM_OBJECT_WLOCK(object); 514 if (object->type == OBJT_VNODE) { 515 vm_object_vndeallocate(object); 516 return; 517 } 518 519 KASSERT(object->ref_count != 0, 520 ("vm_object_deallocate: object deallocated too many times: %d", object->type)); 521 522 /* 523 * If the reference count goes to 0 we start calling 524 * vm_object_terminate() on the object chain. 525 * A ref count of 1 may be a special case depending on the 526 * shadow count being 0 or 1. 527 */ 528 object->ref_count--; 529 if (object->ref_count > 1) { 530 VM_OBJECT_WUNLOCK(object); 531 return; 532 } else if (object->ref_count == 1) { 533 if (object->shadow_count == 0 && 534 object->handle == NULL && 535 (object->type == OBJT_DEFAULT || 536 (object->type == OBJT_SWAP && 537 (object->flags & OBJ_TMPFS_NODE) == 0))) { 538 vm_object_set_flag(object, OBJ_ONEMAPPING); 539 } else if ((object->shadow_count == 1) && 540 (object->handle == NULL) && 541 (object->type == OBJT_DEFAULT || 542 object->type == OBJT_SWAP)) { 543 vm_object_t robject; 544 545 robject = LIST_FIRST(&object->shadow_head); 546 KASSERT(robject != NULL, 547 ("vm_object_deallocate: ref_count: %d, shadow_count: %d", 548 object->ref_count, 549 object->shadow_count)); 550 KASSERT((robject->flags & OBJ_TMPFS_NODE) == 0, 551 ("shadowed tmpfs v_object %p", object)); 552 if (!VM_OBJECT_TRYWLOCK(robject)) { 553 /* 554 * Avoid a potential deadlock. 555 */ 556 object->ref_count++; 557 VM_OBJECT_WUNLOCK(object); 558 /* 559 * More likely than not the thread 560 * holding robject's lock has lower 561 * priority than the current thread. 562 * Let the lower priority thread run. 563 */ 564 pause("vmo_de", 1); 565 continue; 566 } 567 /* 568 * Collapse object into its shadow unless its 569 * shadow is dead. In that case, object will 570 * be deallocated by the thread that is 571 * deallocating its shadow. 572 */ 573 if ((robject->flags & OBJ_DEAD) == 0 && 574 (robject->handle == NULL) && 575 (robject->type == OBJT_DEFAULT || 576 robject->type == OBJT_SWAP)) { 577 578 robject->ref_count++; 579 retry: 580 if (robject->paging_in_progress) { 581 VM_OBJECT_WUNLOCK(object); 582 vm_object_pip_wait(robject, 583 "objde1"); 584 temp = robject->backing_object; 585 if (object == temp) { 586 VM_OBJECT_WLOCK(object); 587 goto retry; 588 } 589 } else if (object->paging_in_progress) { 590 VM_OBJECT_WUNLOCK(robject); 591 VM_OBJECT_WUNLOCK(object); 592 refcount_wait( 593 &object->paging_in_progress, 594 "objde2", PVM); 595 VM_OBJECT_WLOCK(robject); 596 temp = robject->backing_object; 597 if (object == temp) { 598 VM_OBJECT_WLOCK(object); 599 goto retry; 600 } 601 } else 602 VM_OBJECT_WUNLOCK(object); 603 604 if (robject->ref_count == 1) { 605 robject->ref_count--; 606 object = robject; 607 goto doterm; 608 } 609 object = robject; 610 vm_object_collapse(object); 611 VM_OBJECT_WUNLOCK(object); 612 continue; 613 } 614 VM_OBJECT_WUNLOCK(robject); 615 } 616 VM_OBJECT_WUNLOCK(object); 617 return; 618 } 619 doterm: 620 umtx_shm_object_terminated(object); 621 temp = object->backing_object; 622 if (temp != NULL) { 623 KASSERT((object->flags & OBJ_TMPFS_NODE) == 0, 624 ("shadowed tmpfs v_object 2 %p", object)); 625 VM_OBJECT_WLOCK(temp); 626 LIST_REMOVE(object, shadow_list); 627 temp->shadow_count--; 628 VM_OBJECT_WUNLOCK(temp); 629 object->backing_object = NULL; 630 } 631 /* 632 * Don't double-terminate, we could be in a termination 633 * recursion due to the terminate having to sync data 634 * to disk. 635 */ 636 if ((object->flags & OBJ_DEAD) == 0) { 637 vm_object_set_flag(object, OBJ_DEAD); 638 vm_object_terminate(object); 639 } else 640 VM_OBJECT_WUNLOCK(object); 641 object = temp; 642 } 643 } 644 645 /* 646 * vm_object_destroy removes the object from the global object list 647 * and frees the space for the object. 648 */ 649 void 650 vm_object_destroy(vm_object_t object) 651 { 652 653 /* 654 * Release the allocation charge. 655 */ 656 if (object->cred != NULL) { 657 swap_release_by_cred(object->charge, object->cred); 658 object->charge = 0; 659 crfree(object->cred); 660 object->cred = NULL; 661 } 662 663 /* 664 * Free the space for the object. 665 */ 666 uma_zfree(obj_zone, object); 667 } 668 669 /* 670 * vm_object_terminate_pages removes any remaining pageable pages 671 * from the object and resets the object to an empty state. 672 */ 673 static void 674 vm_object_terminate_pages(vm_object_t object) 675 { 676 vm_page_t p, p_next; 677 678 VM_OBJECT_ASSERT_WLOCKED(object); 679 680 /* 681 * Free any remaining pageable pages. This also removes them from the 682 * paging queues. However, don't free wired pages, just remove them 683 * from the object. Rather than incrementally removing each page from 684 * the object, the page and object are reset to any empty state. 685 */ 686 TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) { 687 vm_page_assert_unbusied(p); 688 KASSERT(p->object == object && 689 (p->ref_count & VPRC_OBJREF) != 0, 690 ("vm_object_terminate_pages: page %p is inconsistent", p)); 691 692 p->object = NULL; 693 if (vm_page_drop(p, VPRC_OBJREF) == VPRC_OBJREF) { 694 VM_CNT_INC(v_pfree); 695 vm_page_free(p); 696 } 697 } 698 699 /* 700 * If the object contained any pages, then reset it to an empty state. 701 * None of the object's fields, including "resident_page_count", were 702 * modified by the preceding loop. 703 */ 704 if (object->resident_page_count != 0) { 705 vm_radix_reclaim_allnodes(&object->rtree); 706 TAILQ_INIT(&object->memq); 707 object->resident_page_count = 0; 708 if (object->type == OBJT_VNODE) 709 vdrop(object->handle); 710 } 711 } 712 713 /* 714 * vm_object_terminate actually destroys the specified object, freeing 715 * up all previously used resources. 716 * 717 * The object must be locked. 718 * This routine may block. 719 */ 720 void 721 vm_object_terminate(vm_object_t object) 722 { 723 VM_OBJECT_ASSERT_WLOCKED(object); 724 KASSERT((object->flags & OBJ_DEAD) != 0, 725 ("terminating non-dead obj %p", object)); 726 727 /* 728 * wait for the pageout daemon to be done with the object 729 */ 730 vm_object_pip_wait(object, "objtrm"); 731 732 KASSERT(!object->paging_in_progress, 733 ("vm_object_terminate: pageout in progress")); 734 735 KASSERT(object->ref_count == 0, 736 ("vm_object_terminate: object with references, ref_count=%d", 737 object->ref_count)); 738 739 if ((object->flags & OBJ_PG_DTOR) == 0) 740 vm_object_terminate_pages(object); 741 742 #if VM_NRESERVLEVEL > 0 743 if (__predict_false(!LIST_EMPTY(&object->rvq))) 744 vm_reserv_break_all(object); 745 #endif 746 747 KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT || 748 object->type == OBJT_SWAP, 749 ("%s: non-swap obj %p has cred", __func__, object)); 750 751 /* 752 * Let the pager know object is dead. 753 */ 754 vm_pager_deallocate(object); 755 VM_OBJECT_WUNLOCK(object); 756 757 vm_object_destroy(object); 758 } 759 760 /* 761 * Make the page read-only so that we can clear the object flags. However, if 762 * this is a nosync mmap then the object is likely to stay dirty so do not 763 * mess with the page and do not clear the object flags. Returns TRUE if the 764 * page should be flushed, and FALSE otherwise. 765 */ 766 static boolean_t 767 vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *clearobjflags) 768 { 769 770 /* 771 * If we have been asked to skip nosync pages and this is a 772 * nosync page, skip it. Note that the object flags were not 773 * cleared in this case so we do not have to set them. 774 */ 775 if ((flags & OBJPC_NOSYNC) != 0 && (p->oflags & VPO_NOSYNC) != 0) { 776 *clearobjflags = FALSE; 777 return (FALSE); 778 } else { 779 pmap_remove_write(p); 780 return (p->dirty != 0); 781 } 782 } 783 784 /* 785 * vm_object_page_clean 786 * 787 * Clean all dirty pages in the specified range of object. Leaves page 788 * on whatever queue it is currently on. If NOSYNC is set then do not 789 * write out pages with VPO_NOSYNC set (originally comes from MAP_NOSYNC), 790 * leaving the object dirty. 791 * 792 * When stuffing pages asynchronously, allow clustering. XXX we need a 793 * synchronous clustering mode implementation. 794 * 795 * Odd semantics: if start == end, we clean everything. 796 * 797 * The object must be locked. 798 * 799 * Returns FALSE if some page from the range was not written, as 800 * reported by the pager, and TRUE otherwise. 801 */ 802 boolean_t 803 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end, 804 int flags) 805 { 806 vm_page_t np, p; 807 vm_pindex_t pi, tend, tstart; 808 int curgeneration, n, pagerflags; 809 boolean_t clearobjflags, eio, res; 810 811 VM_OBJECT_ASSERT_WLOCKED(object); 812 813 /* 814 * The OBJ_MIGHTBEDIRTY flag is only set for OBJT_VNODE 815 * objects. The check below prevents the function from 816 * operating on non-vnode objects. 817 */ 818 if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 || 819 object->resident_page_count == 0) 820 return (TRUE); 821 822 pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ? 823 VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; 824 pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0; 825 826 tstart = OFF_TO_IDX(start); 827 tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK); 828 clearobjflags = tstart == 0 && tend >= object->size; 829 res = TRUE; 830 831 rescan: 832 curgeneration = object->generation; 833 834 for (p = vm_page_find_least(object, tstart); p != NULL; p = np) { 835 pi = p->pindex; 836 if (pi >= tend) 837 break; 838 np = TAILQ_NEXT(p, listq); 839 if (p->valid == 0) 840 continue; 841 if (vm_page_sleep_if_busy(p, "vpcwai")) { 842 if (object->generation != curgeneration) { 843 if ((flags & OBJPC_SYNC) != 0) 844 goto rescan; 845 else 846 clearobjflags = FALSE; 847 } 848 np = vm_page_find_least(object, pi); 849 continue; 850 } 851 if (!vm_object_page_remove_write(p, flags, &clearobjflags)) 852 continue; 853 854 n = vm_object_page_collect_flush(object, p, pagerflags, 855 flags, &clearobjflags, &eio); 856 if (eio) { 857 res = FALSE; 858 clearobjflags = FALSE; 859 } 860 if (object->generation != curgeneration) { 861 if ((flags & OBJPC_SYNC) != 0) 862 goto rescan; 863 else 864 clearobjflags = FALSE; 865 } 866 867 /* 868 * If the VOP_PUTPAGES() did a truncated write, so 869 * that even the first page of the run is not fully 870 * written, vm_pageout_flush() returns 0 as the run 871 * length. Since the condition that caused truncated 872 * write may be permanent, e.g. exhausted free space, 873 * accepting n == 0 would cause an infinite loop. 874 * 875 * Forwarding the iterator leaves the unwritten page 876 * behind, but there is not much we can do there if 877 * filesystem refuses to write it. 878 */ 879 if (n == 0) { 880 n = 1; 881 clearobjflags = FALSE; 882 } 883 np = vm_page_find_least(object, pi + n); 884 } 885 #if 0 886 VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0); 887 #endif 888 889 if (clearobjflags) 890 vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY); 891 return (res); 892 } 893 894 static int 895 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags, 896 int flags, boolean_t *clearobjflags, boolean_t *eio) 897 { 898 vm_page_t ma[vm_pageout_page_count], p_first, tp; 899 int count, i, mreq, runlen; 900 901 vm_page_lock_assert(p, MA_NOTOWNED); 902 VM_OBJECT_ASSERT_WLOCKED(object); 903 904 count = 1; 905 mreq = 0; 906 907 for (tp = p; count < vm_pageout_page_count; count++) { 908 tp = vm_page_next(tp); 909 if (tp == NULL || vm_page_busied(tp)) 910 break; 911 if (!vm_object_page_remove_write(tp, flags, clearobjflags)) 912 break; 913 } 914 915 for (p_first = p; count < vm_pageout_page_count; count++) { 916 tp = vm_page_prev(p_first); 917 if (tp == NULL || vm_page_busied(tp)) 918 break; 919 if (!vm_object_page_remove_write(tp, flags, clearobjflags)) 920 break; 921 p_first = tp; 922 mreq++; 923 } 924 925 for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++) 926 ma[i] = tp; 927 928 vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio); 929 return (runlen); 930 } 931 932 /* 933 * Note that there is absolutely no sense in writing out 934 * anonymous objects, so we track down the vnode object 935 * to write out. 936 * We invalidate (remove) all pages from the address space 937 * for semantic correctness. 938 * 939 * If the backing object is a device object with unmanaged pages, then any 940 * mappings to the specified range of pages must be removed before this 941 * function is called. 942 * 943 * Note: certain anonymous maps, such as MAP_NOSYNC maps, 944 * may start out with a NULL object. 945 */ 946 boolean_t 947 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size, 948 boolean_t syncio, boolean_t invalidate) 949 { 950 vm_object_t backing_object; 951 struct vnode *vp; 952 struct mount *mp; 953 int error, flags, fsync_after; 954 boolean_t res; 955 956 if (object == NULL) 957 return (TRUE); 958 res = TRUE; 959 error = 0; 960 VM_OBJECT_WLOCK(object); 961 while ((backing_object = object->backing_object) != NULL) { 962 VM_OBJECT_WLOCK(backing_object); 963 offset += object->backing_object_offset; 964 VM_OBJECT_WUNLOCK(object); 965 object = backing_object; 966 if (object->size < OFF_TO_IDX(offset + size)) 967 size = IDX_TO_OFF(object->size) - offset; 968 } 969 /* 970 * Flush pages if writing is allowed, invalidate them 971 * if invalidation requested. Pages undergoing I/O 972 * will be ignored by vm_object_page_remove(). 973 * 974 * We cannot lock the vnode and then wait for paging 975 * to complete without deadlocking against vm_fault. 976 * Instead we simply call vm_object_page_remove() and 977 * allow it to block internally on a page-by-page 978 * basis when it encounters pages undergoing async 979 * I/O. 980 */ 981 if (object->type == OBJT_VNODE && 982 (object->flags & OBJ_MIGHTBEDIRTY) != 0 && 983 ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) { 984 VM_OBJECT_WUNLOCK(object); 985 (void) vn_start_write(vp, &mp, V_WAIT); 986 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 987 if (syncio && !invalidate && offset == 0 && 988 atop(size) == object->size) { 989 /* 990 * If syncing the whole mapping of the file, 991 * it is faster to schedule all the writes in 992 * async mode, also allowing the clustering, 993 * and then wait for i/o to complete. 994 */ 995 flags = 0; 996 fsync_after = TRUE; 997 } else { 998 flags = (syncio || invalidate) ? OBJPC_SYNC : 0; 999 flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0; 1000 fsync_after = FALSE; 1001 } 1002 VM_OBJECT_WLOCK(object); 1003 res = vm_object_page_clean(object, offset, offset + size, 1004 flags); 1005 VM_OBJECT_WUNLOCK(object); 1006 if (fsync_after) 1007 error = VOP_FSYNC(vp, MNT_WAIT, curthread); 1008 VOP_UNLOCK(vp, 0); 1009 vn_finished_write(mp); 1010 if (error != 0) 1011 res = FALSE; 1012 VM_OBJECT_WLOCK(object); 1013 } 1014 if ((object->type == OBJT_VNODE || 1015 object->type == OBJT_DEVICE) && invalidate) { 1016 if (object->type == OBJT_DEVICE) 1017 /* 1018 * The option OBJPR_NOTMAPPED must be passed here 1019 * because vm_object_page_remove() cannot remove 1020 * unmanaged mappings. 1021 */ 1022 flags = OBJPR_NOTMAPPED; 1023 else if (old_msync) 1024 flags = 0; 1025 else 1026 flags = OBJPR_CLEANONLY; 1027 vm_object_page_remove(object, OFF_TO_IDX(offset), 1028 OFF_TO_IDX(offset + size + PAGE_MASK), flags); 1029 } 1030 VM_OBJECT_WUNLOCK(object); 1031 return (res); 1032 } 1033 1034 /* 1035 * Determine whether the given advice can be applied to the object. Advice is 1036 * not applied to unmanaged pages since they never belong to page queues, and 1037 * since MADV_FREE is destructive, it can apply only to anonymous pages that 1038 * have been mapped at most once. 1039 */ 1040 static bool 1041 vm_object_advice_applies(vm_object_t object, int advice) 1042 { 1043 1044 if ((object->flags & OBJ_UNMANAGED) != 0) 1045 return (false); 1046 if (advice != MADV_FREE) 1047 return (true); 1048 return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) && 1049 (object->flags & OBJ_ONEMAPPING) != 0); 1050 } 1051 1052 static void 1053 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex, 1054 vm_size_t size) 1055 { 1056 1057 if (advice == MADV_FREE && object->type == OBJT_SWAP) 1058 swap_pager_freespace(object, pindex, size); 1059 } 1060 1061 /* 1062 * vm_object_madvise: 1063 * 1064 * Implements the madvise function at the object/page level. 1065 * 1066 * MADV_WILLNEED (any object) 1067 * 1068 * Activate the specified pages if they are resident. 1069 * 1070 * MADV_DONTNEED (any object) 1071 * 1072 * Deactivate the specified pages if they are resident. 1073 * 1074 * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects, 1075 * OBJ_ONEMAPPING only) 1076 * 1077 * Deactivate and clean the specified pages if they are 1078 * resident. This permits the process to reuse the pages 1079 * without faulting or the kernel to reclaim the pages 1080 * without I/O. 1081 */ 1082 void 1083 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end, 1084 int advice) 1085 { 1086 vm_pindex_t tpindex; 1087 vm_object_t backing_object, tobject; 1088 vm_page_t m, tm; 1089 1090 if (object == NULL) 1091 return; 1092 1093 relookup: 1094 VM_OBJECT_WLOCK(object); 1095 if (!vm_object_advice_applies(object, advice)) { 1096 VM_OBJECT_WUNLOCK(object); 1097 return; 1098 } 1099 for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) { 1100 tobject = object; 1101 1102 /* 1103 * If the next page isn't resident in the top-level object, we 1104 * need to search the shadow chain. When applying MADV_FREE, we 1105 * take care to release any swap space used to store 1106 * non-resident pages. 1107 */ 1108 if (m == NULL || pindex < m->pindex) { 1109 /* 1110 * Optimize a common case: if the top-level object has 1111 * no backing object, we can skip over the non-resident 1112 * range in constant time. 1113 */ 1114 if (object->backing_object == NULL) { 1115 tpindex = (m != NULL && m->pindex < end) ? 1116 m->pindex : end; 1117 vm_object_madvise_freespace(object, advice, 1118 pindex, tpindex - pindex); 1119 if ((pindex = tpindex) == end) 1120 break; 1121 goto next_page; 1122 } 1123 1124 tpindex = pindex; 1125 do { 1126 vm_object_madvise_freespace(tobject, advice, 1127 tpindex, 1); 1128 /* 1129 * Prepare to search the next object in the 1130 * chain. 1131 */ 1132 backing_object = tobject->backing_object; 1133 if (backing_object == NULL) 1134 goto next_pindex; 1135 VM_OBJECT_WLOCK(backing_object); 1136 tpindex += 1137 OFF_TO_IDX(tobject->backing_object_offset); 1138 if (tobject != object) 1139 VM_OBJECT_WUNLOCK(tobject); 1140 tobject = backing_object; 1141 if (!vm_object_advice_applies(tobject, advice)) 1142 goto next_pindex; 1143 } while ((tm = vm_page_lookup(tobject, tpindex)) == 1144 NULL); 1145 } else { 1146 next_page: 1147 tm = m; 1148 m = TAILQ_NEXT(m, listq); 1149 } 1150 1151 /* 1152 * If the page is not in a normal state, skip it. 1153 */ 1154 if (tm->valid != VM_PAGE_BITS_ALL || 1155 vm_page_wired(tm)) 1156 goto next_pindex; 1157 KASSERT((tm->flags & PG_FICTITIOUS) == 0, 1158 ("vm_object_madvise: page %p is fictitious", tm)); 1159 KASSERT((tm->oflags & VPO_UNMANAGED) == 0, 1160 ("vm_object_madvise: page %p is not managed", tm)); 1161 if (vm_page_busied(tm)) { 1162 if (object != tobject) 1163 VM_OBJECT_WUNLOCK(tobject); 1164 vm_page_lock(tm); 1165 VM_OBJECT_WUNLOCK(object); 1166 if (advice == MADV_WILLNEED) { 1167 /* 1168 * Reference the page before unlocking and 1169 * sleeping so that the page daemon is less 1170 * likely to reclaim it. 1171 */ 1172 vm_page_aflag_set(tm, PGA_REFERENCED); 1173 } 1174 vm_page_busy_sleep(tm, "madvpo", false); 1175 goto relookup; 1176 } 1177 vm_page_lock(tm); 1178 vm_page_advise(tm, advice); 1179 vm_page_unlock(tm); 1180 vm_object_madvise_freespace(tobject, advice, tm->pindex, 1); 1181 next_pindex: 1182 if (tobject != object) 1183 VM_OBJECT_WUNLOCK(tobject); 1184 } 1185 VM_OBJECT_WUNLOCK(object); 1186 } 1187 1188 /* 1189 * vm_object_shadow: 1190 * 1191 * Create a new object which is backed by the 1192 * specified existing object range. The source 1193 * object reference is deallocated. 1194 * 1195 * The new object and offset into that object 1196 * are returned in the source parameters. 1197 */ 1198 void 1199 vm_object_shadow( 1200 vm_object_t *object, /* IN/OUT */ 1201 vm_ooffset_t *offset, /* IN/OUT */ 1202 vm_size_t length) 1203 { 1204 vm_object_t source; 1205 vm_object_t result; 1206 1207 source = *object; 1208 1209 /* 1210 * Don't create the new object if the old object isn't shared. 1211 */ 1212 if (source != NULL) { 1213 VM_OBJECT_WLOCK(source); 1214 if (source->ref_count == 1 && 1215 source->handle == NULL && 1216 (source->type == OBJT_DEFAULT || 1217 source->type == OBJT_SWAP)) { 1218 VM_OBJECT_WUNLOCK(source); 1219 return; 1220 } 1221 VM_OBJECT_WUNLOCK(source); 1222 } 1223 1224 /* 1225 * Allocate a new object with the given length. 1226 */ 1227 result = vm_object_allocate(OBJT_DEFAULT, atop(length)); 1228 1229 /* 1230 * The new object shadows the source object, adding a reference to it. 1231 * Our caller changes his reference to point to the new object, 1232 * removing a reference to the source object. Net result: no change 1233 * of reference count. 1234 * 1235 * Try to optimize the result object's page color when shadowing 1236 * in order to maintain page coloring consistency in the combined 1237 * shadowed object. 1238 */ 1239 result->backing_object = source; 1240 /* 1241 * Store the offset into the source object, and fix up the offset into 1242 * the new object. 1243 */ 1244 result->backing_object_offset = *offset; 1245 if (source != NULL) { 1246 VM_OBJECT_WLOCK(source); 1247 result->domain = source->domain; 1248 LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list); 1249 source->shadow_count++; 1250 #if VM_NRESERVLEVEL > 0 1251 result->flags |= source->flags & OBJ_COLORED; 1252 result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) & 1253 ((1 << (VM_NFREEORDER - 1)) - 1); 1254 #endif 1255 VM_OBJECT_WUNLOCK(source); 1256 } 1257 1258 1259 /* 1260 * Return the new things 1261 */ 1262 *offset = 0; 1263 *object = result; 1264 } 1265 1266 /* 1267 * vm_object_split: 1268 * 1269 * Split the pages in a map entry into a new object. This affords 1270 * easier removal of unused pages, and keeps object inheritance from 1271 * being a negative impact on memory usage. 1272 */ 1273 void 1274 vm_object_split(vm_map_entry_t entry) 1275 { 1276 vm_page_t m, m_next; 1277 vm_object_t orig_object, new_object, source; 1278 vm_pindex_t idx, offidxstart; 1279 vm_size_t size; 1280 1281 orig_object = entry->object.vm_object; 1282 if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP) 1283 return; 1284 if (orig_object->ref_count <= 1) 1285 return; 1286 VM_OBJECT_WUNLOCK(orig_object); 1287 1288 offidxstart = OFF_TO_IDX(entry->offset); 1289 size = atop(entry->end - entry->start); 1290 1291 /* 1292 * If swap_pager_copy() is later called, it will convert new_object 1293 * into a swap object. 1294 */ 1295 new_object = vm_object_allocate(OBJT_DEFAULT, size); 1296 1297 /* 1298 * At this point, the new object is still private, so the order in 1299 * which the original and new objects are locked does not matter. 1300 */ 1301 VM_OBJECT_WLOCK(new_object); 1302 VM_OBJECT_WLOCK(orig_object); 1303 new_object->domain = orig_object->domain; 1304 source = orig_object->backing_object; 1305 if (source != NULL) { 1306 VM_OBJECT_WLOCK(source); 1307 if ((source->flags & OBJ_DEAD) != 0) { 1308 VM_OBJECT_WUNLOCK(source); 1309 VM_OBJECT_WUNLOCK(orig_object); 1310 VM_OBJECT_WUNLOCK(new_object); 1311 vm_object_deallocate(new_object); 1312 VM_OBJECT_WLOCK(orig_object); 1313 return; 1314 } 1315 LIST_INSERT_HEAD(&source->shadow_head, 1316 new_object, shadow_list); 1317 source->shadow_count++; 1318 vm_object_reference_locked(source); /* for new_object */ 1319 vm_object_clear_flag(source, OBJ_ONEMAPPING); 1320 VM_OBJECT_WUNLOCK(source); 1321 new_object->backing_object_offset = 1322 orig_object->backing_object_offset + entry->offset; 1323 new_object->backing_object = source; 1324 } 1325 if (orig_object->cred != NULL) { 1326 new_object->cred = orig_object->cred; 1327 crhold(orig_object->cred); 1328 new_object->charge = ptoa(size); 1329 KASSERT(orig_object->charge >= ptoa(size), 1330 ("orig_object->charge < 0")); 1331 orig_object->charge -= ptoa(size); 1332 } 1333 retry: 1334 m = vm_page_find_least(orig_object, offidxstart); 1335 for (; m != NULL && (idx = m->pindex - offidxstart) < size; 1336 m = m_next) { 1337 m_next = TAILQ_NEXT(m, listq); 1338 1339 /* 1340 * We must wait for pending I/O to complete before we can 1341 * rename the page. 1342 * 1343 * We do not have to VM_PROT_NONE the page as mappings should 1344 * not be changed by this operation. 1345 */ 1346 if (vm_page_busied(m)) { 1347 VM_OBJECT_WUNLOCK(new_object); 1348 vm_page_lock(m); 1349 VM_OBJECT_WUNLOCK(orig_object); 1350 vm_page_busy_sleep(m, "spltwt", false); 1351 VM_OBJECT_WLOCK(orig_object); 1352 VM_OBJECT_WLOCK(new_object); 1353 goto retry; 1354 } 1355 1356 /* vm_page_rename() will dirty the page. */ 1357 if (vm_page_rename(m, new_object, idx)) { 1358 VM_OBJECT_WUNLOCK(new_object); 1359 VM_OBJECT_WUNLOCK(orig_object); 1360 vm_radix_wait(); 1361 VM_OBJECT_WLOCK(orig_object); 1362 VM_OBJECT_WLOCK(new_object); 1363 goto retry; 1364 } 1365 #if VM_NRESERVLEVEL > 0 1366 /* 1367 * If some of the reservation's allocated pages remain with 1368 * the original object, then transferring the reservation to 1369 * the new object is neither particularly beneficial nor 1370 * particularly harmful as compared to leaving the reservation 1371 * with the original object. If, however, all of the 1372 * reservation's allocated pages are transferred to the new 1373 * object, then transferring the reservation is typically 1374 * beneficial. Determining which of these two cases applies 1375 * would be more costly than unconditionally renaming the 1376 * reservation. 1377 */ 1378 vm_reserv_rename(m, new_object, orig_object, offidxstart); 1379 #endif 1380 if (orig_object->type == OBJT_SWAP) 1381 vm_page_xbusy(m); 1382 } 1383 if (orig_object->type == OBJT_SWAP) { 1384 /* 1385 * swap_pager_copy() can sleep, in which case the orig_object's 1386 * and new_object's locks are released and reacquired. 1387 */ 1388 swap_pager_copy(orig_object, new_object, offidxstart, 0); 1389 TAILQ_FOREACH(m, &new_object->memq, listq) 1390 vm_page_xunbusy(m); 1391 } 1392 VM_OBJECT_WUNLOCK(orig_object); 1393 VM_OBJECT_WUNLOCK(new_object); 1394 entry->object.vm_object = new_object; 1395 entry->offset = 0LL; 1396 vm_object_deallocate(orig_object); 1397 VM_OBJECT_WLOCK(new_object); 1398 } 1399 1400 #define OBSC_COLLAPSE_NOWAIT 0x0002 1401 #define OBSC_COLLAPSE_WAIT 0x0004 1402 1403 static vm_page_t 1404 vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p, vm_page_t next, 1405 int op) 1406 { 1407 vm_object_t backing_object; 1408 1409 VM_OBJECT_ASSERT_WLOCKED(object); 1410 backing_object = object->backing_object; 1411 VM_OBJECT_ASSERT_WLOCKED(backing_object); 1412 1413 KASSERT(p == NULL || vm_page_busied(p), ("unbusy page %p", p)); 1414 KASSERT(p == NULL || p->object == object || p->object == backing_object, 1415 ("invalid ownership %p %p %p", p, object, backing_object)); 1416 if ((op & OBSC_COLLAPSE_NOWAIT) != 0) 1417 return (next); 1418 if (p != NULL) 1419 vm_page_lock(p); 1420 VM_OBJECT_WUNLOCK(object); 1421 VM_OBJECT_WUNLOCK(backing_object); 1422 /* The page is only NULL when rename fails. */ 1423 if (p == NULL) 1424 vm_radix_wait(); 1425 else 1426 vm_page_busy_sleep(p, "vmocol", false); 1427 VM_OBJECT_WLOCK(object); 1428 VM_OBJECT_WLOCK(backing_object); 1429 return (TAILQ_FIRST(&backing_object->memq)); 1430 } 1431 1432 static bool 1433 vm_object_scan_all_shadowed(vm_object_t object) 1434 { 1435 vm_object_t backing_object; 1436 vm_page_t p, pp; 1437 vm_pindex_t backing_offset_index, new_pindex, pi, ps; 1438 1439 VM_OBJECT_ASSERT_WLOCKED(object); 1440 VM_OBJECT_ASSERT_WLOCKED(object->backing_object); 1441 1442 backing_object = object->backing_object; 1443 1444 if (backing_object->type != OBJT_DEFAULT && 1445 backing_object->type != OBJT_SWAP) 1446 return (false); 1447 1448 pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 1449 p = vm_page_find_least(backing_object, pi); 1450 ps = swap_pager_find_least(backing_object, pi); 1451 1452 /* 1453 * Only check pages inside the parent object's range and 1454 * inside the parent object's mapping of the backing object. 1455 */ 1456 for (;; pi++) { 1457 if (p != NULL && p->pindex < pi) 1458 p = TAILQ_NEXT(p, listq); 1459 if (ps < pi) 1460 ps = swap_pager_find_least(backing_object, pi); 1461 if (p == NULL && ps >= backing_object->size) 1462 break; 1463 else if (p == NULL) 1464 pi = ps; 1465 else 1466 pi = MIN(p->pindex, ps); 1467 1468 new_pindex = pi - backing_offset_index; 1469 if (new_pindex >= object->size) 1470 break; 1471 1472 /* 1473 * See if the parent has the page or if the parent's object 1474 * pager has the page. If the parent has the page but the page 1475 * is not valid, the parent's object pager must have the page. 1476 * 1477 * If this fails, the parent does not completely shadow the 1478 * object and we might as well give up now. 1479 */ 1480 pp = vm_page_lookup(object, new_pindex); 1481 if ((pp == NULL || pp->valid == 0) && 1482 !vm_pager_has_page(object, new_pindex, NULL, NULL)) 1483 return (false); 1484 } 1485 return (true); 1486 } 1487 1488 static bool 1489 vm_object_collapse_scan(vm_object_t object, int op) 1490 { 1491 vm_object_t backing_object; 1492 vm_page_t next, p, pp; 1493 vm_pindex_t backing_offset_index, new_pindex; 1494 1495 VM_OBJECT_ASSERT_WLOCKED(object); 1496 VM_OBJECT_ASSERT_WLOCKED(object->backing_object); 1497 1498 backing_object = object->backing_object; 1499 backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 1500 1501 /* 1502 * Initial conditions 1503 */ 1504 if ((op & OBSC_COLLAPSE_WAIT) != 0) 1505 vm_object_set_flag(backing_object, OBJ_DEAD); 1506 1507 /* 1508 * Our scan 1509 */ 1510 for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) { 1511 next = TAILQ_NEXT(p, listq); 1512 new_pindex = p->pindex - backing_offset_index; 1513 1514 /* 1515 * Check for busy page 1516 */ 1517 if (vm_page_busied(p)) { 1518 next = vm_object_collapse_scan_wait(object, p, next, op); 1519 continue; 1520 } 1521 1522 KASSERT(p->object == backing_object, 1523 ("vm_object_collapse_scan: object mismatch")); 1524 1525 if (p->pindex < backing_offset_index || 1526 new_pindex >= object->size) { 1527 if (backing_object->type == OBJT_SWAP) 1528 swap_pager_freespace(backing_object, p->pindex, 1529 1); 1530 1531 KASSERT(!pmap_page_is_mapped(p), 1532 ("freeing mapped page %p", p)); 1533 if (vm_page_remove(p)) 1534 vm_page_free(p); 1535 continue; 1536 } 1537 1538 pp = vm_page_lookup(object, new_pindex); 1539 if (pp != NULL && vm_page_busied(pp)) { 1540 /* 1541 * The page in the parent is busy and possibly not 1542 * (yet) valid. Until its state is finalized by the 1543 * busy bit owner, we can't tell whether it shadows the 1544 * original page. Therefore, we must either skip it 1545 * and the original (backing_object) page or wait for 1546 * its state to be finalized. 1547 * 1548 * This is due to a race with vm_fault() where we must 1549 * unbusy the original (backing_obj) page before we can 1550 * (re)lock the parent. Hence we can get here. 1551 */ 1552 next = vm_object_collapse_scan_wait(object, pp, next, 1553 op); 1554 continue; 1555 } 1556 1557 KASSERT(pp == NULL || pp->valid != 0, 1558 ("unbusy invalid page %p", pp)); 1559 1560 if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL, 1561 NULL)) { 1562 /* 1563 * The page already exists in the parent OR swap exists 1564 * for this location in the parent. Leave the parent's 1565 * page alone. Destroy the original page from the 1566 * backing object. 1567 */ 1568 if (backing_object->type == OBJT_SWAP) 1569 swap_pager_freespace(backing_object, p->pindex, 1570 1); 1571 KASSERT(!pmap_page_is_mapped(p), 1572 ("freeing mapped page %p", p)); 1573 if (vm_page_remove(p)) 1574 vm_page_free(p); 1575 continue; 1576 } 1577 1578 /* 1579 * Page does not exist in parent, rename the page from the 1580 * backing object to the main object. 1581 * 1582 * If the page was mapped to a process, it can remain mapped 1583 * through the rename. vm_page_rename() will dirty the page. 1584 */ 1585 if (vm_page_rename(p, object, new_pindex)) { 1586 next = vm_object_collapse_scan_wait(object, NULL, next, 1587 op); 1588 continue; 1589 } 1590 1591 /* Use the old pindex to free the right page. */ 1592 if (backing_object->type == OBJT_SWAP) 1593 swap_pager_freespace(backing_object, 1594 new_pindex + backing_offset_index, 1); 1595 1596 #if VM_NRESERVLEVEL > 0 1597 /* 1598 * Rename the reservation. 1599 */ 1600 vm_reserv_rename(p, object, backing_object, 1601 backing_offset_index); 1602 #endif 1603 } 1604 return (true); 1605 } 1606 1607 1608 /* 1609 * this version of collapse allows the operation to occur earlier and 1610 * when paging_in_progress is true for an object... This is not a complete 1611 * operation, but should plug 99.9% of the rest of the leaks. 1612 */ 1613 static void 1614 vm_object_qcollapse(vm_object_t object) 1615 { 1616 vm_object_t backing_object = object->backing_object; 1617 1618 VM_OBJECT_ASSERT_WLOCKED(object); 1619 VM_OBJECT_ASSERT_WLOCKED(backing_object); 1620 1621 if (backing_object->ref_count != 1) 1622 return; 1623 1624 vm_object_collapse_scan(object, OBSC_COLLAPSE_NOWAIT); 1625 } 1626 1627 /* 1628 * vm_object_collapse: 1629 * 1630 * Collapse an object with the object backing it. 1631 * Pages in the backing object are moved into the 1632 * parent, and the backing object is deallocated. 1633 */ 1634 void 1635 vm_object_collapse(vm_object_t object) 1636 { 1637 vm_object_t backing_object, new_backing_object; 1638 1639 VM_OBJECT_ASSERT_WLOCKED(object); 1640 1641 while (TRUE) { 1642 /* 1643 * Verify that the conditions are right for collapse: 1644 * 1645 * The object exists and the backing object exists. 1646 */ 1647 if ((backing_object = object->backing_object) == NULL) 1648 break; 1649 1650 /* 1651 * we check the backing object first, because it is most likely 1652 * not collapsable. 1653 */ 1654 VM_OBJECT_WLOCK(backing_object); 1655 if (backing_object->handle != NULL || 1656 (backing_object->type != OBJT_DEFAULT && 1657 backing_object->type != OBJT_SWAP) || 1658 (backing_object->flags & (OBJ_DEAD | OBJ_NOSPLIT)) != 0 || 1659 object->handle != NULL || 1660 (object->type != OBJT_DEFAULT && 1661 object->type != OBJT_SWAP) || 1662 (object->flags & OBJ_DEAD)) { 1663 VM_OBJECT_WUNLOCK(backing_object); 1664 break; 1665 } 1666 1667 if (object->paging_in_progress != 0 || 1668 backing_object->paging_in_progress != 0) { 1669 vm_object_qcollapse(object); 1670 VM_OBJECT_WUNLOCK(backing_object); 1671 break; 1672 } 1673 1674 /* 1675 * We know that we can either collapse the backing object (if 1676 * the parent is the only reference to it) or (perhaps) have 1677 * the parent bypass the object if the parent happens to shadow 1678 * all the resident pages in the entire backing object. 1679 * 1680 * This is ignoring pager-backed pages such as swap pages. 1681 * vm_object_collapse_scan fails the shadowing test in this 1682 * case. 1683 */ 1684 if (backing_object->ref_count == 1) { 1685 vm_object_pip_add(object, 1); 1686 vm_object_pip_add(backing_object, 1); 1687 1688 /* 1689 * If there is exactly one reference to the backing 1690 * object, we can collapse it into the parent. 1691 */ 1692 vm_object_collapse_scan(object, OBSC_COLLAPSE_WAIT); 1693 1694 #if VM_NRESERVLEVEL > 0 1695 /* 1696 * Break any reservations from backing_object. 1697 */ 1698 if (__predict_false(!LIST_EMPTY(&backing_object->rvq))) 1699 vm_reserv_break_all(backing_object); 1700 #endif 1701 1702 /* 1703 * Move the pager from backing_object to object. 1704 */ 1705 if (backing_object->type == OBJT_SWAP) { 1706 /* 1707 * swap_pager_copy() can sleep, in which case 1708 * the backing_object's and object's locks are 1709 * released and reacquired. 1710 * Since swap_pager_copy() is being asked to 1711 * destroy the source, it will change the 1712 * backing_object's type to OBJT_DEFAULT. 1713 */ 1714 swap_pager_copy( 1715 backing_object, 1716 object, 1717 OFF_TO_IDX(object->backing_object_offset), TRUE); 1718 } 1719 /* 1720 * Object now shadows whatever backing_object did. 1721 * Note that the reference to 1722 * backing_object->backing_object moves from within 1723 * backing_object to within object. 1724 */ 1725 LIST_REMOVE(object, shadow_list); 1726 backing_object->shadow_count--; 1727 if (backing_object->backing_object) { 1728 VM_OBJECT_WLOCK(backing_object->backing_object); 1729 LIST_REMOVE(backing_object, shadow_list); 1730 LIST_INSERT_HEAD( 1731 &backing_object->backing_object->shadow_head, 1732 object, shadow_list); 1733 /* 1734 * The shadow_count has not changed. 1735 */ 1736 VM_OBJECT_WUNLOCK(backing_object->backing_object); 1737 } 1738 object->backing_object = backing_object->backing_object; 1739 object->backing_object_offset += 1740 backing_object->backing_object_offset; 1741 1742 /* 1743 * Discard backing_object. 1744 * 1745 * Since the backing object has no pages, no pager left, 1746 * and no object references within it, all that is 1747 * necessary is to dispose of it. 1748 */ 1749 KASSERT(backing_object->ref_count == 1, ( 1750 "backing_object %p was somehow re-referenced during collapse!", 1751 backing_object)); 1752 vm_object_pip_wakeup(backing_object); 1753 backing_object->type = OBJT_DEAD; 1754 backing_object->ref_count = 0; 1755 VM_OBJECT_WUNLOCK(backing_object); 1756 vm_object_destroy(backing_object); 1757 1758 vm_object_pip_wakeup(object); 1759 counter_u64_add(object_collapses, 1); 1760 } else { 1761 /* 1762 * If we do not entirely shadow the backing object, 1763 * there is nothing we can do so we give up. 1764 */ 1765 if (object->resident_page_count != object->size && 1766 !vm_object_scan_all_shadowed(object)) { 1767 VM_OBJECT_WUNLOCK(backing_object); 1768 break; 1769 } 1770 1771 /* 1772 * Make the parent shadow the next object in the 1773 * chain. Deallocating backing_object will not remove 1774 * it, since its reference count is at least 2. 1775 */ 1776 LIST_REMOVE(object, shadow_list); 1777 backing_object->shadow_count--; 1778 1779 new_backing_object = backing_object->backing_object; 1780 if ((object->backing_object = new_backing_object) != NULL) { 1781 VM_OBJECT_WLOCK(new_backing_object); 1782 LIST_INSERT_HEAD( 1783 &new_backing_object->shadow_head, 1784 object, 1785 shadow_list 1786 ); 1787 new_backing_object->shadow_count++; 1788 vm_object_reference_locked(new_backing_object); 1789 VM_OBJECT_WUNLOCK(new_backing_object); 1790 object->backing_object_offset += 1791 backing_object->backing_object_offset; 1792 } 1793 1794 /* 1795 * Drop the reference count on backing_object. Since 1796 * its ref_count was at least 2, it will not vanish. 1797 */ 1798 backing_object->ref_count--; 1799 VM_OBJECT_WUNLOCK(backing_object); 1800 counter_u64_add(object_bypasses, 1); 1801 } 1802 1803 /* 1804 * Try again with this object's new backing object. 1805 */ 1806 } 1807 } 1808 1809 /* 1810 * vm_object_page_remove: 1811 * 1812 * For the given object, either frees or invalidates each of the 1813 * specified pages. In general, a page is freed. However, if a page is 1814 * wired for any reason other than the existence of a managed, wired 1815 * mapping, then it may be invalidated but not removed from the object. 1816 * Pages are specified by the given range ["start", "end") and the option 1817 * OBJPR_CLEANONLY. As a special case, if "end" is zero, then the range 1818 * extends from "start" to the end of the object. If the option 1819 * OBJPR_CLEANONLY is specified, then only the non-dirty pages within the 1820 * specified range are affected. If the option OBJPR_NOTMAPPED is 1821 * specified, then the pages within the specified range must have no 1822 * mappings. Otherwise, if this option is not specified, any mappings to 1823 * the specified pages are removed before the pages are freed or 1824 * invalidated. 1825 * 1826 * In general, this operation should only be performed on objects that 1827 * contain managed pages. There are, however, two exceptions. First, it 1828 * is performed on the kernel and kmem objects by vm_map_entry_delete(). 1829 * Second, it is used by msync(..., MS_INVALIDATE) to invalidate device- 1830 * backed pages. In both of these cases, the option OBJPR_CLEANONLY must 1831 * not be specified and the option OBJPR_NOTMAPPED must be specified. 1832 * 1833 * The object must be locked. 1834 */ 1835 void 1836 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 1837 int options) 1838 { 1839 vm_page_t p, next; 1840 struct mtx *mtx; 1841 1842 VM_OBJECT_ASSERT_WLOCKED(object); 1843 KASSERT((object->flags & OBJ_UNMANAGED) == 0 || 1844 (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED, 1845 ("vm_object_page_remove: illegal options for object %p", object)); 1846 if (object->resident_page_count == 0) 1847 return; 1848 vm_object_pip_add(object, 1); 1849 again: 1850 p = vm_page_find_least(object, start); 1851 mtx = NULL; 1852 1853 /* 1854 * Here, the variable "p" is either (1) the page with the least pindex 1855 * greater than or equal to the parameter "start" or (2) NULL. 1856 */ 1857 for (; p != NULL && (p->pindex < end || end == 0); p = next) { 1858 next = TAILQ_NEXT(p, listq); 1859 1860 /* 1861 * If the page is wired for any reason besides the existence 1862 * of managed, wired mappings, then it cannot be freed. For 1863 * example, fictitious pages, which represent device memory, 1864 * are inherently wired and cannot be freed. They can, 1865 * however, be invalidated if the option OBJPR_CLEANONLY is 1866 * not specified. 1867 */ 1868 vm_page_change_lock(p, &mtx); 1869 if (vm_page_xbusied(p)) { 1870 VM_OBJECT_WUNLOCK(object); 1871 vm_page_busy_sleep(p, "vmopax", true); 1872 VM_OBJECT_WLOCK(object); 1873 goto again; 1874 } 1875 if (vm_page_busied(p)) { 1876 VM_OBJECT_WUNLOCK(object); 1877 vm_page_busy_sleep(p, "vmopar", false); 1878 VM_OBJECT_WLOCK(object); 1879 goto again; 1880 } 1881 if (vm_page_wired(p)) { 1882 wired: 1883 if ((options & OBJPR_NOTMAPPED) == 0 && 1884 object->ref_count != 0) 1885 pmap_remove_all(p); 1886 if ((options & OBJPR_CLEANONLY) == 0) { 1887 p->valid = 0; 1888 vm_page_undirty(p); 1889 } 1890 continue; 1891 } 1892 KASSERT((p->flags & PG_FICTITIOUS) == 0, 1893 ("vm_object_page_remove: page %p is fictitious", p)); 1894 if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) { 1895 if ((options & OBJPR_NOTMAPPED) == 0 && 1896 object->ref_count != 0 && 1897 !vm_page_try_remove_write(p)) 1898 goto wired; 1899 if (p->dirty != 0) 1900 continue; 1901 } 1902 if ((options & OBJPR_NOTMAPPED) == 0 && 1903 object->ref_count != 0 && !vm_page_try_remove_all(p)) 1904 goto wired; 1905 vm_page_free(p); 1906 } 1907 if (mtx != NULL) 1908 mtx_unlock(mtx); 1909 vm_object_pip_wakeup(object); 1910 } 1911 1912 /* 1913 * vm_object_page_noreuse: 1914 * 1915 * For the given object, attempt to move the specified pages to 1916 * the head of the inactive queue. This bypasses regular LRU 1917 * operation and allows the pages to be reused quickly under memory 1918 * pressure. If a page is wired for any reason, then it will not 1919 * be queued. Pages are specified by the range ["start", "end"). 1920 * As a special case, if "end" is zero, then the range extends from 1921 * "start" to the end of the object. 1922 * 1923 * This operation should only be performed on objects that 1924 * contain non-fictitious, managed pages. 1925 * 1926 * The object must be locked. 1927 */ 1928 void 1929 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1930 { 1931 struct mtx *mtx; 1932 vm_page_t p, next; 1933 1934 VM_OBJECT_ASSERT_LOCKED(object); 1935 KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0, 1936 ("vm_object_page_noreuse: illegal object %p", object)); 1937 if (object->resident_page_count == 0) 1938 return; 1939 p = vm_page_find_least(object, start); 1940 1941 /* 1942 * Here, the variable "p" is either (1) the page with the least pindex 1943 * greater than or equal to the parameter "start" or (2) NULL. 1944 */ 1945 mtx = NULL; 1946 for (; p != NULL && (p->pindex < end || end == 0); p = next) { 1947 next = TAILQ_NEXT(p, listq); 1948 vm_page_change_lock(p, &mtx); 1949 vm_page_deactivate_noreuse(p); 1950 } 1951 if (mtx != NULL) 1952 mtx_unlock(mtx); 1953 } 1954 1955 /* 1956 * Populate the specified range of the object with valid pages. Returns 1957 * TRUE if the range is successfully populated and FALSE otherwise. 1958 * 1959 * Note: This function should be optimized to pass a larger array of 1960 * pages to vm_pager_get_pages() before it is applied to a non- 1961 * OBJT_DEVICE object. 1962 * 1963 * The object must be locked. 1964 */ 1965 boolean_t 1966 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1967 { 1968 vm_page_t m; 1969 vm_pindex_t pindex; 1970 int rv; 1971 1972 VM_OBJECT_ASSERT_WLOCKED(object); 1973 for (pindex = start; pindex < end; pindex++) { 1974 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 1975 if (m->valid != VM_PAGE_BITS_ALL) { 1976 rv = vm_pager_get_pages(object, &m, 1, NULL, NULL); 1977 if (rv != VM_PAGER_OK) { 1978 vm_page_free(m); 1979 break; 1980 } 1981 } 1982 /* 1983 * Keep "m" busy because a subsequent iteration may unlock 1984 * the object. 1985 */ 1986 } 1987 if (pindex > start) { 1988 m = vm_page_lookup(object, start); 1989 while (m != NULL && m->pindex < pindex) { 1990 vm_page_xunbusy(m); 1991 m = TAILQ_NEXT(m, listq); 1992 } 1993 } 1994 return (pindex == end); 1995 } 1996 1997 /* 1998 * Routine: vm_object_coalesce 1999 * Function: Coalesces two objects backing up adjoining 2000 * regions of memory into a single object. 2001 * 2002 * returns TRUE if objects were combined. 2003 * 2004 * NOTE: Only works at the moment if the second object is NULL - 2005 * if it's not, which object do we lock first? 2006 * 2007 * Parameters: 2008 * prev_object First object to coalesce 2009 * prev_offset Offset into prev_object 2010 * prev_size Size of reference to prev_object 2011 * next_size Size of reference to the second object 2012 * reserved Indicator that extension region has 2013 * swap accounted for 2014 * 2015 * Conditions: 2016 * The object must *not* be locked. 2017 */ 2018 boolean_t 2019 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset, 2020 vm_size_t prev_size, vm_size_t next_size, boolean_t reserved) 2021 { 2022 vm_pindex_t next_pindex; 2023 2024 if (prev_object == NULL) 2025 return (TRUE); 2026 VM_OBJECT_WLOCK(prev_object); 2027 if ((prev_object->type != OBJT_DEFAULT && 2028 prev_object->type != OBJT_SWAP) || 2029 (prev_object->flags & OBJ_NOSPLIT) != 0) { 2030 VM_OBJECT_WUNLOCK(prev_object); 2031 return (FALSE); 2032 } 2033 2034 /* 2035 * Try to collapse the object first 2036 */ 2037 vm_object_collapse(prev_object); 2038 2039 /* 2040 * Can't coalesce if: . more than one reference . paged out . shadows 2041 * another object . has a copy elsewhere (any of which mean that the 2042 * pages not mapped to prev_entry may be in use anyway) 2043 */ 2044 if (prev_object->backing_object != NULL) { 2045 VM_OBJECT_WUNLOCK(prev_object); 2046 return (FALSE); 2047 } 2048 2049 prev_size >>= PAGE_SHIFT; 2050 next_size >>= PAGE_SHIFT; 2051 next_pindex = OFF_TO_IDX(prev_offset) + prev_size; 2052 2053 if (prev_object->ref_count > 1 && 2054 prev_object->size != next_pindex && 2055 (prev_object->flags & OBJ_ONEMAPPING) == 0) { 2056 VM_OBJECT_WUNLOCK(prev_object); 2057 return (FALSE); 2058 } 2059 2060 /* 2061 * Account for the charge. 2062 */ 2063 if (prev_object->cred != NULL) { 2064 2065 /* 2066 * If prev_object was charged, then this mapping, 2067 * although not charged now, may become writable 2068 * later. Non-NULL cred in the object would prevent 2069 * swap reservation during enabling of the write 2070 * access, so reserve swap now. Failed reservation 2071 * cause allocation of the separate object for the map 2072 * entry, and swap reservation for this entry is 2073 * managed in appropriate time. 2074 */ 2075 if (!reserved && !swap_reserve_by_cred(ptoa(next_size), 2076 prev_object->cred)) { 2077 VM_OBJECT_WUNLOCK(prev_object); 2078 return (FALSE); 2079 } 2080 prev_object->charge += ptoa(next_size); 2081 } 2082 2083 /* 2084 * Remove any pages that may still be in the object from a previous 2085 * deallocation. 2086 */ 2087 if (next_pindex < prev_object->size) { 2088 vm_object_page_remove(prev_object, next_pindex, next_pindex + 2089 next_size, 0); 2090 if (prev_object->type == OBJT_SWAP) 2091 swap_pager_freespace(prev_object, 2092 next_pindex, next_size); 2093 #if 0 2094 if (prev_object->cred != NULL) { 2095 KASSERT(prev_object->charge >= 2096 ptoa(prev_object->size - next_pindex), 2097 ("object %p overcharged 1 %jx %jx", prev_object, 2098 (uintmax_t)next_pindex, (uintmax_t)next_size)); 2099 prev_object->charge -= ptoa(prev_object->size - 2100 next_pindex); 2101 } 2102 #endif 2103 } 2104 2105 /* 2106 * Extend the object if necessary. 2107 */ 2108 if (next_pindex + next_size > prev_object->size) 2109 prev_object->size = next_pindex + next_size; 2110 2111 VM_OBJECT_WUNLOCK(prev_object); 2112 return (TRUE); 2113 } 2114 2115 void 2116 vm_object_set_writeable_dirty(vm_object_t object) 2117 { 2118 2119 VM_OBJECT_ASSERT_WLOCKED(object); 2120 if (object->type != OBJT_VNODE) { 2121 if ((object->flags & OBJ_TMPFS_NODE) != 0) { 2122 KASSERT(object->type == OBJT_SWAP, ("non-swap tmpfs")); 2123 vm_object_set_flag(object, OBJ_TMPFS_DIRTY); 2124 } 2125 return; 2126 } 2127 object->generation++; 2128 if ((object->flags & OBJ_MIGHTBEDIRTY) != 0) 2129 return; 2130 vm_object_set_flag(object, OBJ_MIGHTBEDIRTY); 2131 } 2132 2133 /* 2134 * vm_object_unwire: 2135 * 2136 * For each page offset within the specified range of the given object, 2137 * find the highest-level page in the shadow chain and unwire it. A page 2138 * must exist at every page offset, and the highest-level page must be 2139 * wired. 2140 */ 2141 void 2142 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length, 2143 uint8_t queue) 2144 { 2145 vm_object_t tobject, t1object; 2146 vm_page_t m, tm; 2147 vm_pindex_t end_pindex, pindex, tpindex; 2148 int depth, locked_depth; 2149 2150 KASSERT((offset & PAGE_MASK) == 0, 2151 ("vm_object_unwire: offset is not page aligned")); 2152 KASSERT((length & PAGE_MASK) == 0, 2153 ("vm_object_unwire: length is not a multiple of PAGE_SIZE")); 2154 /* The wired count of a fictitious page never changes. */ 2155 if ((object->flags & OBJ_FICTITIOUS) != 0) 2156 return; 2157 pindex = OFF_TO_IDX(offset); 2158 end_pindex = pindex + atop(length); 2159 again: 2160 locked_depth = 1; 2161 VM_OBJECT_RLOCK(object); 2162 m = vm_page_find_least(object, pindex); 2163 while (pindex < end_pindex) { 2164 if (m == NULL || pindex < m->pindex) { 2165 /* 2166 * The first object in the shadow chain doesn't 2167 * contain a page at the current index. Therefore, 2168 * the page must exist in a backing object. 2169 */ 2170 tobject = object; 2171 tpindex = pindex; 2172 depth = 0; 2173 do { 2174 tpindex += 2175 OFF_TO_IDX(tobject->backing_object_offset); 2176 tobject = tobject->backing_object; 2177 KASSERT(tobject != NULL, 2178 ("vm_object_unwire: missing page")); 2179 if ((tobject->flags & OBJ_FICTITIOUS) != 0) 2180 goto next_page; 2181 depth++; 2182 if (depth == locked_depth) { 2183 locked_depth++; 2184 VM_OBJECT_RLOCK(tobject); 2185 } 2186 } while ((tm = vm_page_lookup(tobject, tpindex)) == 2187 NULL); 2188 } else { 2189 tm = m; 2190 m = TAILQ_NEXT(m, listq); 2191 } 2192 if (vm_page_xbusied(tm)) { 2193 vm_page_lock(tm); 2194 for (tobject = object; locked_depth >= 1; 2195 locked_depth--) { 2196 t1object = tobject->backing_object; 2197 VM_OBJECT_RUNLOCK(tobject); 2198 tobject = t1object; 2199 } 2200 vm_page_busy_sleep(tm, "unwbo", true); 2201 goto again; 2202 } 2203 vm_page_unwire(tm, queue); 2204 next_page: 2205 pindex++; 2206 } 2207 /* Release the accumulated object locks. */ 2208 for (tobject = object; locked_depth >= 1; locked_depth--) { 2209 t1object = tobject->backing_object; 2210 VM_OBJECT_RUNLOCK(tobject); 2211 tobject = t1object; 2212 } 2213 } 2214 2215 /* 2216 * Return the vnode for the given object, or NULL if none exists. 2217 * For tmpfs objects, the function may return NULL if there is 2218 * no vnode allocated at the time of the call. 2219 */ 2220 struct vnode * 2221 vm_object_vnode(vm_object_t object) 2222 { 2223 struct vnode *vp; 2224 2225 VM_OBJECT_ASSERT_LOCKED(object); 2226 if (object->type == OBJT_VNODE) { 2227 vp = object->handle; 2228 KASSERT(vp != NULL, ("%s: OBJT_VNODE has no vnode", __func__)); 2229 } else if (object->type == OBJT_SWAP && 2230 (object->flags & OBJ_TMPFS) != 0) { 2231 vp = object->un_pager.swp.swp_tmpfs; 2232 KASSERT(vp != NULL, ("%s: OBJT_TMPFS has no vnode", __func__)); 2233 } else { 2234 vp = NULL; 2235 } 2236 return (vp); 2237 } 2238 2239 /* 2240 * Return the kvme type of the given object. 2241 * If vpp is not NULL, set it to the object's vm_object_vnode() or NULL. 2242 */ 2243 int 2244 vm_object_kvme_type(vm_object_t object, struct vnode **vpp) 2245 { 2246 2247 VM_OBJECT_ASSERT_LOCKED(object); 2248 if (vpp != NULL) 2249 *vpp = vm_object_vnode(object); 2250 switch (object->type) { 2251 case OBJT_DEFAULT: 2252 return (KVME_TYPE_DEFAULT); 2253 case OBJT_VNODE: 2254 return (KVME_TYPE_VNODE); 2255 case OBJT_SWAP: 2256 if ((object->flags & OBJ_TMPFS_NODE) != 0) 2257 return (KVME_TYPE_VNODE); 2258 return (KVME_TYPE_SWAP); 2259 case OBJT_DEVICE: 2260 return (KVME_TYPE_DEVICE); 2261 case OBJT_PHYS: 2262 return (KVME_TYPE_PHYS); 2263 case OBJT_DEAD: 2264 return (KVME_TYPE_DEAD); 2265 case OBJT_SG: 2266 return (KVME_TYPE_SG); 2267 case OBJT_MGTDEVICE: 2268 return (KVME_TYPE_MGTDEVICE); 2269 default: 2270 return (KVME_TYPE_UNKNOWN); 2271 } 2272 } 2273 2274 static int 2275 sysctl_vm_object_list(SYSCTL_HANDLER_ARGS) 2276 { 2277 struct kinfo_vmobject *kvo; 2278 char *fullpath, *freepath; 2279 struct vnode *vp; 2280 struct vattr va; 2281 vm_object_t obj; 2282 vm_page_t m; 2283 int count, error; 2284 2285 if (req->oldptr == NULL) { 2286 /* 2287 * If an old buffer has not been provided, generate an 2288 * estimate of the space needed for a subsequent call. 2289 */ 2290 mtx_lock(&vm_object_list_mtx); 2291 count = 0; 2292 TAILQ_FOREACH(obj, &vm_object_list, object_list) { 2293 if (obj->type == OBJT_DEAD) 2294 continue; 2295 count++; 2296 } 2297 mtx_unlock(&vm_object_list_mtx); 2298 return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) * 2299 count * 11 / 10)); 2300 } 2301 2302 kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK); 2303 error = 0; 2304 2305 /* 2306 * VM objects are type stable and are never removed from the 2307 * list once added. This allows us to safely read obj->object_list 2308 * after reacquiring the VM object lock. 2309 */ 2310 mtx_lock(&vm_object_list_mtx); 2311 TAILQ_FOREACH(obj, &vm_object_list, object_list) { 2312 if (obj->type == OBJT_DEAD) 2313 continue; 2314 VM_OBJECT_RLOCK(obj); 2315 if (obj->type == OBJT_DEAD) { 2316 VM_OBJECT_RUNLOCK(obj); 2317 continue; 2318 } 2319 mtx_unlock(&vm_object_list_mtx); 2320 kvo->kvo_size = ptoa(obj->size); 2321 kvo->kvo_resident = obj->resident_page_count; 2322 kvo->kvo_ref_count = obj->ref_count; 2323 kvo->kvo_shadow_count = obj->shadow_count; 2324 kvo->kvo_memattr = obj->memattr; 2325 kvo->kvo_active = 0; 2326 kvo->kvo_inactive = 0; 2327 TAILQ_FOREACH(m, &obj->memq, listq) { 2328 /* 2329 * A page may belong to the object but be 2330 * dequeued and set to PQ_NONE while the 2331 * object lock is not held. This makes the 2332 * reads of m->queue below racy, and we do not 2333 * count pages set to PQ_NONE. However, this 2334 * sysctl is only meant to give an 2335 * approximation of the system anyway. 2336 */ 2337 if (m->queue == PQ_ACTIVE) 2338 kvo->kvo_active++; 2339 else if (m->queue == PQ_INACTIVE) 2340 kvo->kvo_inactive++; 2341 } 2342 2343 kvo->kvo_vn_fileid = 0; 2344 kvo->kvo_vn_fsid = 0; 2345 kvo->kvo_vn_fsid_freebsd11 = 0; 2346 freepath = NULL; 2347 fullpath = ""; 2348 kvo->kvo_type = vm_object_kvme_type(obj, &vp); 2349 if (vp != NULL) 2350 vref(vp); 2351 VM_OBJECT_RUNLOCK(obj); 2352 if (vp != NULL) { 2353 vn_fullpath(curthread, vp, &fullpath, &freepath); 2354 vn_lock(vp, LK_SHARED | LK_RETRY); 2355 if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) { 2356 kvo->kvo_vn_fileid = va.va_fileid; 2357 kvo->kvo_vn_fsid = va.va_fsid; 2358 kvo->kvo_vn_fsid_freebsd11 = va.va_fsid; 2359 /* truncate */ 2360 } 2361 vput(vp); 2362 } 2363 2364 strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path)); 2365 if (freepath != NULL) 2366 free(freepath, M_TEMP); 2367 2368 /* Pack record size down */ 2369 kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path) 2370 + strlen(kvo->kvo_path) + 1; 2371 kvo->kvo_structsize = roundup(kvo->kvo_structsize, 2372 sizeof(uint64_t)); 2373 error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize); 2374 mtx_lock(&vm_object_list_mtx); 2375 if (error) 2376 break; 2377 } 2378 mtx_unlock(&vm_object_list_mtx); 2379 free(kvo, M_TEMP); 2380 return (error); 2381 } 2382 SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP | 2383 CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject", 2384 "List of VM objects"); 2385 2386 #include "opt_ddb.h" 2387 #ifdef DDB 2388 #include <sys/kernel.h> 2389 2390 #include <sys/cons.h> 2391 2392 #include <ddb/ddb.h> 2393 2394 static int 2395 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry) 2396 { 2397 vm_map_t tmpm; 2398 vm_map_entry_t tmpe; 2399 vm_object_t obj; 2400 int entcount; 2401 2402 if (map == 0) 2403 return 0; 2404 2405 if (entry == 0) { 2406 tmpe = map->header.next; 2407 entcount = map->nentries; 2408 while (entcount-- && (tmpe != &map->header)) { 2409 if (_vm_object_in_map(map, object, tmpe)) { 2410 return 1; 2411 } 2412 tmpe = tmpe->next; 2413 } 2414 } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { 2415 tmpm = entry->object.sub_map; 2416 tmpe = tmpm->header.next; 2417 entcount = tmpm->nentries; 2418 while (entcount-- && tmpe != &tmpm->header) { 2419 if (_vm_object_in_map(tmpm, object, tmpe)) { 2420 return 1; 2421 } 2422 tmpe = tmpe->next; 2423 } 2424 } else if ((obj = entry->object.vm_object) != NULL) { 2425 for (; obj; obj = obj->backing_object) 2426 if (obj == object) { 2427 return 1; 2428 } 2429 } 2430 return 0; 2431 } 2432 2433 static int 2434 vm_object_in_map(vm_object_t object) 2435 { 2436 struct proc *p; 2437 2438 /* sx_slock(&allproc_lock); */ 2439 FOREACH_PROC_IN_SYSTEM(p) { 2440 if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */) 2441 continue; 2442 if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) { 2443 /* sx_sunlock(&allproc_lock); */ 2444 return 1; 2445 } 2446 } 2447 /* sx_sunlock(&allproc_lock); */ 2448 if (_vm_object_in_map(kernel_map, object, 0)) 2449 return 1; 2450 return 0; 2451 } 2452 2453 DB_SHOW_COMMAND(vmochk, vm_object_check) 2454 { 2455 vm_object_t object; 2456 2457 /* 2458 * make sure that internal objs are in a map somewhere 2459 * and none have zero ref counts. 2460 */ 2461 TAILQ_FOREACH(object, &vm_object_list, object_list) { 2462 if (object->handle == NULL && 2463 (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) { 2464 if (object->ref_count == 0) { 2465 db_printf("vmochk: internal obj has zero ref count: %ld\n", 2466 (long)object->size); 2467 } 2468 if (!vm_object_in_map(object)) { 2469 db_printf( 2470 "vmochk: internal obj is not in a map: " 2471 "ref: %d, size: %lu: 0x%lx, backing_object: %p\n", 2472 object->ref_count, (u_long)object->size, 2473 (u_long)object->size, 2474 (void *)object->backing_object); 2475 } 2476 } 2477 } 2478 } 2479 2480 /* 2481 * vm_object_print: [ debug ] 2482 */ 2483 DB_SHOW_COMMAND(object, vm_object_print_static) 2484 { 2485 /* XXX convert args. */ 2486 vm_object_t object = (vm_object_t)addr; 2487 boolean_t full = have_addr; 2488 2489 vm_page_t p; 2490 2491 /* XXX count is an (unused) arg. Avoid shadowing it. */ 2492 #define count was_count 2493 2494 int count; 2495 2496 if (object == NULL) 2497 return; 2498 2499 db_iprintf( 2500 "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n", 2501 object, (int)object->type, (uintmax_t)object->size, 2502 object->resident_page_count, object->ref_count, object->flags, 2503 object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge); 2504 db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n", 2505 object->shadow_count, 2506 object->backing_object ? object->backing_object->ref_count : 0, 2507 object->backing_object, (uintmax_t)object->backing_object_offset); 2508 2509 if (!full) 2510 return; 2511 2512 db_indent += 2; 2513 count = 0; 2514 TAILQ_FOREACH(p, &object->memq, listq) { 2515 if (count == 0) 2516 db_iprintf("memory:="); 2517 else if (count == 6) { 2518 db_printf("\n"); 2519 db_iprintf(" ..."); 2520 count = 0; 2521 } else 2522 db_printf(","); 2523 count++; 2524 2525 db_printf("(off=0x%jx,page=0x%jx)", 2526 (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p)); 2527 } 2528 if (count != 0) 2529 db_printf("\n"); 2530 db_indent -= 2; 2531 } 2532 2533 /* XXX. */ 2534 #undef count 2535 2536 /* XXX need this non-static entry for calling from vm_map_print. */ 2537 void 2538 vm_object_print( 2539 /* db_expr_t */ long addr, 2540 boolean_t have_addr, 2541 /* db_expr_t */ long count, 2542 char *modif) 2543 { 2544 vm_object_print_static(addr, have_addr, count, modif); 2545 } 2546 2547 DB_SHOW_COMMAND(vmopag, vm_object_print_pages) 2548 { 2549 vm_object_t object; 2550 vm_pindex_t fidx; 2551 vm_paddr_t pa; 2552 vm_page_t m, prev_m; 2553 int rcount, nl, c; 2554 2555 nl = 0; 2556 TAILQ_FOREACH(object, &vm_object_list, object_list) { 2557 db_printf("new object: %p\n", (void *)object); 2558 if (nl > 18) { 2559 c = cngetc(); 2560 if (c != ' ') 2561 return; 2562 nl = 0; 2563 } 2564 nl++; 2565 rcount = 0; 2566 fidx = 0; 2567 pa = -1; 2568 TAILQ_FOREACH(m, &object->memq, listq) { 2569 if (m->pindex > 128) 2570 break; 2571 if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL && 2572 prev_m->pindex + 1 != m->pindex) { 2573 if (rcount) { 2574 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 2575 (long)fidx, rcount, (long)pa); 2576 if (nl > 18) { 2577 c = cngetc(); 2578 if (c != ' ') 2579 return; 2580 nl = 0; 2581 } 2582 nl++; 2583 rcount = 0; 2584 } 2585 } 2586 if (rcount && 2587 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) { 2588 ++rcount; 2589 continue; 2590 } 2591 if (rcount) { 2592 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 2593 (long)fidx, rcount, (long)pa); 2594 if (nl > 18) { 2595 c = cngetc(); 2596 if (c != ' ') 2597 return; 2598 nl = 0; 2599 } 2600 nl++; 2601 } 2602 fidx = m->pindex; 2603 pa = VM_PAGE_TO_PHYS(m); 2604 rcount = 1; 2605 } 2606 if (rcount) { 2607 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 2608 (long)fidx, rcount, (long)pa); 2609 if (nl > 18) { 2610 c = cngetc(); 2611 if (c != ' ') 2612 return; 2613 nl = 0; 2614 } 2615 nl++; 2616 } 2617 } 2618 } 2619 #endif /* DDB */ 2620