1 /*- 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * from: @(#)vm_map.c 8.3 (Berkeley) 1/12/94 33 * 34 * 35 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 36 * All rights reserved. 37 * 38 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 39 * 40 * Permission to use, copy, modify and distribute this software and 41 * its documentation is hereby granted, provided that both the copyright 42 * notice and this permission notice appear in all copies of the 43 * software, derivative works or modified versions, and any portions 44 * thereof, and that both notices appear in supporting documentation. 45 * 46 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 47 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 48 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 49 * 50 * Carnegie Mellon requests users of this software to return to 51 * 52 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 53 * School of Computer Science 54 * Carnegie Mellon University 55 * Pittsburgh PA 15213-3890 56 * 57 * any improvements or extensions that they make and grant Carnegie the 58 * rights to redistribute these changes. 59 */ 60 61 /* 62 * Virtual memory mapping module. 63 */ 64 65 #include <sys/cdefs.h> 66 __FBSDID("$FreeBSD$"); 67 68 #include <sys/param.h> 69 #include <sys/systm.h> 70 #include <sys/kernel.h> 71 #include <sys/ktr.h> 72 #include <sys/lock.h> 73 #include <sys/mutex.h> 74 #include <sys/proc.h> 75 #include <sys/vmmeter.h> 76 #include <sys/mman.h> 77 #include <sys/vnode.h> 78 #include <sys/racct.h> 79 #include <sys/resourcevar.h> 80 #include <sys/rwlock.h> 81 #include <sys/file.h> 82 #include <sys/sysctl.h> 83 #include <sys/sysent.h> 84 #include <sys/shm.h> 85 86 #include <vm/vm.h> 87 #include <vm/vm_param.h> 88 #include <vm/pmap.h> 89 #include <vm/vm_map.h> 90 #include <vm/vm_page.h> 91 #include <vm/vm_object.h> 92 #include <vm/vm_pager.h> 93 #include <vm/vm_kern.h> 94 #include <vm/vm_extern.h> 95 #include <vm/vnode_pager.h> 96 #include <vm/swap_pager.h> 97 #include <vm/uma.h> 98 99 /* 100 * Virtual memory maps provide for the mapping, protection, 101 * and sharing of virtual memory objects. In addition, 102 * this module provides for an efficient virtual copy of 103 * memory from one map to another. 104 * 105 * Synchronization is required prior to most operations. 106 * 107 * Maps consist of an ordered doubly-linked list of simple 108 * entries; a self-adjusting binary search tree of these 109 * entries is used to speed up lookups. 110 * 111 * Since portions of maps are specified by start/end addresses, 112 * which may not align with existing map entries, all 113 * routines merely "clip" entries to these start/end values. 114 * [That is, an entry is split into two, bordering at a 115 * start or end value.] Note that these clippings may not 116 * always be necessary (as the two resulting entries are then 117 * not changed); however, the clipping is done for convenience. 118 * 119 * As mentioned above, virtual copy operations are performed 120 * by copying VM object references from one map to 121 * another, and then marking both regions as copy-on-write. 122 */ 123 124 static struct mtx map_sleep_mtx; 125 static uma_zone_t mapentzone; 126 static uma_zone_t kmapentzone; 127 static uma_zone_t mapzone; 128 static uma_zone_t vmspace_zone; 129 static int vmspace_zinit(void *mem, int size, int flags); 130 static void vmspace_zfini(void *mem, int size); 131 static int vm_map_zinit(void *mem, int ize, int flags); 132 static void vm_map_zfini(void *mem, int size); 133 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, 134 vm_offset_t max); 135 static void vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map); 136 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry); 137 #ifdef INVARIANTS 138 static void vm_map_zdtor(void *mem, int size, void *arg); 139 static void vmspace_zdtor(void *mem, int size, void *arg); 140 #endif 141 142 #define ENTRY_CHARGED(e) ((e)->cred != NULL || \ 143 ((e)->object.vm_object != NULL && (e)->object.vm_object->cred != NULL && \ 144 !((e)->eflags & MAP_ENTRY_NEEDS_COPY))) 145 146 /* 147 * PROC_VMSPACE_{UN,}LOCK() can be a noop as long as vmspaces are type 148 * stable. 149 */ 150 #define PROC_VMSPACE_LOCK(p) do { } while (0) 151 #define PROC_VMSPACE_UNLOCK(p) do { } while (0) 152 153 /* 154 * VM_MAP_RANGE_CHECK: [ internal use only ] 155 * 156 * Asserts that the starting and ending region 157 * addresses fall within the valid range of the map. 158 */ 159 #define VM_MAP_RANGE_CHECK(map, start, end) \ 160 { \ 161 if (start < vm_map_min(map)) \ 162 start = vm_map_min(map); \ 163 if (end > vm_map_max(map)) \ 164 end = vm_map_max(map); \ 165 if (start > end) \ 166 start = end; \ 167 } 168 169 /* 170 * vm_map_startup: 171 * 172 * Initialize the vm_map module. Must be called before 173 * any other vm_map routines. 174 * 175 * Map and entry structures are allocated from the general 176 * purpose memory pool with some exceptions: 177 * 178 * - The kernel map and kmem submap are allocated statically. 179 * - Kernel map entries are allocated out of a static pool. 180 * 181 * These restrictions are necessary since malloc() uses the 182 * maps and requires map entries. 183 */ 184 185 void 186 vm_map_startup(void) 187 { 188 mtx_init(&map_sleep_mtx, "vm map sleep mutex", NULL, MTX_DEF); 189 mapzone = uma_zcreate("MAP", sizeof(struct vm_map), NULL, 190 #ifdef INVARIANTS 191 vm_map_zdtor, 192 #else 193 NULL, 194 #endif 195 vm_map_zinit, vm_map_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); 196 uma_prealloc(mapzone, MAX_KMAP); 197 kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry), 198 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 199 UMA_ZONE_MTXCLASS | UMA_ZONE_VM); 200 uma_prealloc(kmapentzone, MAX_KMAPENT); 201 mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry), 202 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 203 } 204 205 static void 206 vmspace_zfini(void *mem, int size) 207 { 208 struct vmspace *vm; 209 210 vm = (struct vmspace *)mem; 211 vm_map_zfini(&vm->vm_map, sizeof(vm->vm_map)); 212 } 213 214 static int 215 vmspace_zinit(void *mem, int size, int flags) 216 { 217 struct vmspace *vm; 218 219 vm = (struct vmspace *)mem; 220 221 vm->vm_map.pmap = NULL; 222 (void)vm_map_zinit(&vm->vm_map, sizeof(vm->vm_map), flags); 223 return (0); 224 } 225 226 static void 227 vm_map_zfini(void *mem, int size) 228 { 229 vm_map_t map; 230 231 map = (vm_map_t)mem; 232 mtx_destroy(&map->system_mtx); 233 sx_destroy(&map->lock); 234 } 235 236 static int 237 vm_map_zinit(void *mem, int size, int flags) 238 { 239 vm_map_t map; 240 241 map = (vm_map_t)mem; 242 memset(map, 0, sizeof(*map)); 243 mtx_init(&map->system_mtx, "vm map (system)", NULL, MTX_DEF | MTX_DUPOK); 244 sx_init(&map->lock, "vm map (user)"); 245 return (0); 246 } 247 248 #ifdef INVARIANTS 249 static void 250 vmspace_zdtor(void *mem, int size, void *arg) 251 { 252 struct vmspace *vm; 253 254 vm = (struct vmspace *)mem; 255 256 vm_map_zdtor(&vm->vm_map, sizeof(vm->vm_map), arg); 257 } 258 static void 259 vm_map_zdtor(void *mem, int size, void *arg) 260 { 261 vm_map_t map; 262 263 map = (vm_map_t)mem; 264 KASSERT(map->nentries == 0, 265 ("map %p nentries == %d on free.", 266 map, map->nentries)); 267 KASSERT(map->size == 0, 268 ("map %p size == %lu on free.", 269 map, (unsigned long)map->size)); 270 } 271 #endif /* INVARIANTS */ 272 273 /* 274 * Allocate a vmspace structure, including a vm_map and pmap, 275 * and initialize those structures. The refcnt is set to 1. 276 */ 277 struct vmspace * 278 vmspace_alloc(min, max) 279 vm_offset_t min, max; 280 { 281 struct vmspace *vm; 282 283 vm = uma_zalloc(vmspace_zone, M_WAITOK); 284 if (vm->vm_map.pmap == NULL && !pmap_pinit(vmspace_pmap(vm))) { 285 uma_zfree(vmspace_zone, vm); 286 return (NULL); 287 } 288 CTR1(KTR_VM, "vmspace_alloc: %p", vm); 289 _vm_map_init(&vm->vm_map, vmspace_pmap(vm), min, max); 290 vm->vm_refcnt = 1; 291 vm->vm_shm = NULL; 292 vm->vm_swrss = 0; 293 vm->vm_tsize = 0; 294 vm->vm_dsize = 0; 295 vm->vm_ssize = 0; 296 vm->vm_taddr = 0; 297 vm->vm_daddr = 0; 298 vm->vm_maxsaddr = 0; 299 return (vm); 300 } 301 302 void 303 vm_init2(void) 304 { 305 uma_zone_reserve_kva(kmapentzone, lmin(cnt.v_page_count, 306 (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE) / 8 + 307 maxproc * 2 + maxfiles); 308 vmspace_zone = uma_zcreate("VMSPACE", sizeof(struct vmspace), NULL, 309 #ifdef INVARIANTS 310 vmspace_zdtor, 311 #else 312 NULL, 313 #endif 314 vmspace_zinit, vmspace_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); 315 } 316 317 static void 318 vmspace_container_reset(struct proc *p) 319 { 320 321 #ifdef RACCT 322 PROC_LOCK(p); 323 racct_set(p, RACCT_DATA, 0); 324 racct_set(p, RACCT_STACK, 0); 325 racct_set(p, RACCT_RSS, 0); 326 racct_set(p, RACCT_MEMLOCK, 0); 327 racct_set(p, RACCT_VMEM, 0); 328 PROC_UNLOCK(p); 329 #endif 330 } 331 332 static inline void 333 vmspace_dofree(struct vmspace *vm) 334 { 335 336 CTR1(KTR_VM, "vmspace_free: %p", vm); 337 338 /* 339 * Make sure any SysV shm is freed, it might not have been in 340 * exit1(). 341 */ 342 shmexit(vm); 343 344 /* 345 * Lock the map, to wait out all other references to it. 346 * Delete all of the mappings and pages they hold, then call 347 * the pmap module to reclaim anything left. 348 */ 349 (void)vm_map_remove(&vm->vm_map, vm->vm_map.min_offset, 350 vm->vm_map.max_offset); 351 352 pmap_release(vmspace_pmap(vm)); 353 vm->vm_map.pmap = NULL; 354 uma_zfree(vmspace_zone, vm); 355 } 356 357 void 358 vmspace_free(struct vmspace *vm) 359 { 360 361 if (vm->vm_refcnt == 0) 362 panic("vmspace_free: attempt to free already freed vmspace"); 363 364 if (atomic_fetchadd_int(&vm->vm_refcnt, -1) == 1) 365 vmspace_dofree(vm); 366 } 367 368 void 369 vmspace_exitfree(struct proc *p) 370 { 371 struct vmspace *vm; 372 373 PROC_VMSPACE_LOCK(p); 374 vm = p->p_vmspace; 375 p->p_vmspace = NULL; 376 PROC_VMSPACE_UNLOCK(p); 377 KASSERT(vm == &vmspace0, ("vmspace_exitfree: wrong vmspace")); 378 vmspace_free(vm); 379 } 380 381 void 382 vmspace_exit(struct thread *td) 383 { 384 int refcnt; 385 struct vmspace *vm; 386 struct proc *p; 387 388 /* 389 * Release user portion of address space. 390 * This releases references to vnodes, 391 * which could cause I/O if the file has been unlinked. 392 * Need to do this early enough that we can still sleep. 393 * 394 * The last exiting process to reach this point releases as 395 * much of the environment as it can. vmspace_dofree() is the 396 * slower fallback in case another process had a temporary 397 * reference to the vmspace. 398 */ 399 400 p = td->td_proc; 401 vm = p->p_vmspace; 402 atomic_add_int(&vmspace0.vm_refcnt, 1); 403 do { 404 refcnt = vm->vm_refcnt; 405 if (refcnt > 1 && p->p_vmspace != &vmspace0) { 406 /* Switch now since other proc might free vmspace */ 407 PROC_VMSPACE_LOCK(p); 408 p->p_vmspace = &vmspace0; 409 PROC_VMSPACE_UNLOCK(p); 410 pmap_activate(td); 411 } 412 } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1)); 413 if (refcnt == 1) { 414 if (p->p_vmspace != vm) { 415 /* vmspace not yet freed, switch back */ 416 PROC_VMSPACE_LOCK(p); 417 p->p_vmspace = vm; 418 PROC_VMSPACE_UNLOCK(p); 419 pmap_activate(td); 420 } 421 pmap_remove_pages(vmspace_pmap(vm)); 422 /* Switch now since this proc will free vmspace */ 423 PROC_VMSPACE_LOCK(p); 424 p->p_vmspace = &vmspace0; 425 PROC_VMSPACE_UNLOCK(p); 426 pmap_activate(td); 427 vmspace_dofree(vm); 428 } 429 vmspace_container_reset(p); 430 } 431 432 /* Acquire reference to vmspace owned by another process. */ 433 434 struct vmspace * 435 vmspace_acquire_ref(struct proc *p) 436 { 437 struct vmspace *vm; 438 int refcnt; 439 440 PROC_VMSPACE_LOCK(p); 441 vm = p->p_vmspace; 442 if (vm == NULL) { 443 PROC_VMSPACE_UNLOCK(p); 444 return (NULL); 445 } 446 do { 447 refcnt = vm->vm_refcnt; 448 if (refcnt <= 0) { /* Avoid 0->1 transition */ 449 PROC_VMSPACE_UNLOCK(p); 450 return (NULL); 451 } 452 } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt + 1)); 453 if (vm != p->p_vmspace) { 454 PROC_VMSPACE_UNLOCK(p); 455 vmspace_free(vm); 456 return (NULL); 457 } 458 PROC_VMSPACE_UNLOCK(p); 459 return (vm); 460 } 461 462 void 463 _vm_map_lock(vm_map_t map, const char *file, int line) 464 { 465 466 if (map->system_map) 467 mtx_lock_flags_(&map->system_mtx, 0, file, line); 468 else 469 sx_xlock_(&map->lock, file, line); 470 map->timestamp++; 471 } 472 473 static void 474 vm_map_process_deferred(void) 475 { 476 struct thread *td; 477 vm_map_entry_t entry, next; 478 vm_object_t object; 479 480 td = curthread; 481 entry = td->td_map_def_user; 482 td->td_map_def_user = NULL; 483 while (entry != NULL) { 484 next = entry->next; 485 if ((entry->eflags & MAP_ENTRY_VN_WRITECNT) != 0) { 486 /* 487 * Decrement the object's writemappings and 488 * possibly the vnode's v_writecount. 489 */ 490 KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, 491 ("Submap with writecount")); 492 object = entry->object.vm_object; 493 KASSERT(object != NULL, ("No object for writecount")); 494 vnode_pager_release_writecount(object, entry->start, 495 entry->end); 496 } 497 vm_map_entry_deallocate(entry, FALSE); 498 entry = next; 499 } 500 } 501 502 void 503 _vm_map_unlock(vm_map_t map, const char *file, int line) 504 { 505 506 if (map->system_map) 507 mtx_unlock_flags_(&map->system_mtx, 0, file, line); 508 else { 509 sx_xunlock_(&map->lock, file, line); 510 vm_map_process_deferred(); 511 } 512 } 513 514 void 515 _vm_map_lock_read(vm_map_t map, const char *file, int line) 516 { 517 518 if (map->system_map) 519 mtx_lock_flags_(&map->system_mtx, 0, file, line); 520 else 521 sx_slock_(&map->lock, file, line); 522 } 523 524 void 525 _vm_map_unlock_read(vm_map_t map, const char *file, int line) 526 { 527 528 if (map->system_map) 529 mtx_unlock_flags_(&map->system_mtx, 0, file, line); 530 else { 531 sx_sunlock_(&map->lock, file, line); 532 vm_map_process_deferred(); 533 } 534 } 535 536 int 537 _vm_map_trylock(vm_map_t map, const char *file, int line) 538 { 539 int error; 540 541 error = map->system_map ? 542 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : 543 !sx_try_xlock_(&map->lock, file, line); 544 if (error == 0) 545 map->timestamp++; 546 return (error == 0); 547 } 548 549 int 550 _vm_map_trylock_read(vm_map_t map, const char *file, int line) 551 { 552 int error; 553 554 error = map->system_map ? 555 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : 556 !sx_try_slock_(&map->lock, file, line); 557 return (error == 0); 558 } 559 560 /* 561 * _vm_map_lock_upgrade: [ internal use only ] 562 * 563 * Tries to upgrade a read (shared) lock on the specified map to a write 564 * (exclusive) lock. Returns the value "0" if the upgrade succeeds and a 565 * non-zero value if the upgrade fails. If the upgrade fails, the map is 566 * returned without a read or write lock held. 567 * 568 * Requires that the map be read locked. 569 */ 570 int 571 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line) 572 { 573 unsigned int last_timestamp; 574 575 if (map->system_map) { 576 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); 577 } else { 578 if (!sx_try_upgrade_(&map->lock, file, line)) { 579 last_timestamp = map->timestamp; 580 sx_sunlock_(&map->lock, file, line); 581 vm_map_process_deferred(); 582 /* 583 * If the map's timestamp does not change while the 584 * map is unlocked, then the upgrade succeeds. 585 */ 586 sx_xlock_(&map->lock, file, line); 587 if (last_timestamp != map->timestamp) { 588 sx_xunlock_(&map->lock, file, line); 589 return (1); 590 } 591 } 592 } 593 map->timestamp++; 594 return (0); 595 } 596 597 void 598 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) 599 { 600 601 if (map->system_map) { 602 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); 603 } else 604 sx_downgrade_(&map->lock, file, line); 605 } 606 607 /* 608 * vm_map_locked: 609 * 610 * Returns a non-zero value if the caller holds a write (exclusive) lock 611 * on the specified map and the value "0" otherwise. 612 */ 613 int 614 vm_map_locked(vm_map_t map) 615 { 616 617 if (map->system_map) 618 return (mtx_owned(&map->system_mtx)); 619 else 620 return (sx_xlocked(&map->lock)); 621 } 622 623 #ifdef INVARIANTS 624 static void 625 _vm_map_assert_locked(vm_map_t map, const char *file, int line) 626 { 627 628 if (map->system_map) 629 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); 630 else 631 sx_assert_(&map->lock, SA_XLOCKED, file, line); 632 } 633 634 #define VM_MAP_ASSERT_LOCKED(map) \ 635 _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE) 636 #else 637 #define VM_MAP_ASSERT_LOCKED(map) 638 #endif 639 640 /* 641 * _vm_map_unlock_and_wait: 642 * 643 * Atomically releases the lock on the specified map and puts the calling 644 * thread to sleep. The calling thread will remain asleep until either 645 * vm_map_wakeup() is performed on the map or the specified timeout is 646 * exceeded. 647 * 648 * WARNING! This function does not perform deferred deallocations of 649 * objects and map entries. Therefore, the calling thread is expected to 650 * reacquire the map lock after reawakening and later perform an ordinary 651 * unlock operation, such as vm_map_unlock(), before completing its 652 * operation on the map. 653 */ 654 int 655 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) 656 { 657 658 mtx_lock(&map_sleep_mtx); 659 if (map->system_map) 660 mtx_unlock_flags_(&map->system_mtx, 0, file, line); 661 else 662 sx_xunlock_(&map->lock, file, line); 663 return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", 664 timo)); 665 } 666 667 /* 668 * vm_map_wakeup: 669 * 670 * Awaken any threads that have slept on the map using 671 * vm_map_unlock_and_wait(). 672 */ 673 void 674 vm_map_wakeup(vm_map_t map) 675 { 676 677 /* 678 * Acquire and release map_sleep_mtx to prevent a wakeup() 679 * from being performed (and lost) between the map unlock 680 * and the msleep() in _vm_map_unlock_and_wait(). 681 */ 682 mtx_lock(&map_sleep_mtx); 683 mtx_unlock(&map_sleep_mtx); 684 wakeup(&map->root); 685 } 686 687 void 688 vm_map_busy(vm_map_t map) 689 { 690 691 VM_MAP_ASSERT_LOCKED(map); 692 map->busy++; 693 } 694 695 void 696 vm_map_unbusy(vm_map_t map) 697 { 698 699 VM_MAP_ASSERT_LOCKED(map); 700 KASSERT(map->busy, ("vm_map_unbusy: not busy")); 701 if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) { 702 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP); 703 wakeup(&map->busy); 704 } 705 } 706 707 void 708 vm_map_wait_busy(vm_map_t map) 709 { 710 711 VM_MAP_ASSERT_LOCKED(map); 712 while (map->busy) { 713 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0); 714 if (map->system_map) 715 msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0); 716 else 717 sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0); 718 } 719 map->timestamp++; 720 } 721 722 long 723 vmspace_resident_count(struct vmspace *vmspace) 724 { 725 return pmap_resident_count(vmspace_pmap(vmspace)); 726 } 727 728 /* 729 * vm_map_create: 730 * 731 * Creates and returns a new empty VM map with 732 * the given physical map structure, and having 733 * the given lower and upper address bounds. 734 */ 735 vm_map_t 736 vm_map_create(pmap_t pmap, vm_offset_t min, vm_offset_t max) 737 { 738 vm_map_t result; 739 740 result = uma_zalloc(mapzone, M_WAITOK); 741 CTR1(KTR_VM, "vm_map_create: %p", result); 742 _vm_map_init(result, pmap, min, max); 743 return (result); 744 } 745 746 /* 747 * Initialize an existing vm_map structure 748 * such as that in the vmspace structure. 749 */ 750 static void 751 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) 752 { 753 754 map->header.next = map->header.prev = &map->header; 755 map->needs_wakeup = FALSE; 756 map->system_map = 0; 757 map->pmap = pmap; 758 map->min_offset = min; 759 map->max_offset = max; 760 map->flags = 0; 761 map->root = NULL; 762 map->timestamp = 0; 763 map->busy = 0; 764 } 765 766 void 767 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) 768 { 769 770 _vm_map_init(map, pmap, min, max); 771 mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK); 772 sx_init(&map->lock, "user map"); 773 } 774 775 /* 776 * vm_map_entry_dispose: [ internal use only ] 777 * 778 * Inverse of vm_map_entry_create. 779 */ 780 static void 781 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) 782 { 783 uma_zfree(map->system_map ? kmapentzone : mapentzone, entry); 784 } 785 786 /* 787 * vm_map_entry_create: [ internal use only ] 788 * 789 * Allocates a VM map entry for insertion. 790 * No entry fields are filled in. 791 */ 792 static vm_map_entry_t 793 vm_map_entry_create(vm_map_t map) 794 { 795 vm_map_entry_t new_entry; 796 797 if (map->system_map) 798 new_entry = uma_zalloc(kmapentzone, M_NOWAIT); 799 else 800 new_entry = uma_zalloc(mapentzone, M_WAITOK); 801 if (new_entry == NULL) 802 panic("vm_map_entry_create: kernel resources exhausted"); 803 return (new_entry); 804 } 805 806 /* 807 * vm_map_entry_set_behavior: 808 * 809 * Set the expected access behavior, either normal, random, or 810 * sequential. 811 */ 812 static inline void 813 vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior) 814 { 815 entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) | 816 (behavior & MAP_ENTRY_BEHAV_MASK); 817 } 818 819 /* 820 * vm_map_entry_set_max_free: 821 * 822 * Set the max_free field in a vm_map_entry. 823 */ 824 static inline void 825 vm_map_entry_set_max_free(vm_map_entry_t entry) 826 { 827 828 entry->max_free = entry->adj_free; 829 if (entry->left != NULL && entry->left->max_free > entry->max_free) 830 entry->max_free = entry->left->max_free; 831 if (entry->right != NULL && entry->right->max_free > entry->max_free) 832 entry->max_free = entry->right->max_free; 833 } 834 835 /* 836 * vm_map_entry_splay: 837 * 838 * The Sleator and Tarjan top-down splay algorithm with the 839 * following variation. Max_free must be computed bottom-up, so 840 * on the downward pass, maintain the left and right spines in 841 * reverse order. Then, make a second pass up each side to fix 842 * the pointers and compute max_free. The time bound is O(log n) 843 * amortized. 844 * 845 * The new root is the vm_map_entry containing "addr", or else an 846 * adjacent entry (lower or higher) if addr is not in the tree. 847 * 848 * The map must be locked, and leaves it so. 849 * 850 * Returns: the new root. 851 */ 852 static vm_map_entry_t 853 vm_map_entry_splay(vm_offset_t addr, vm_map_entry_t root) 854 { 855 vm_map_entry_t llist, rlist; 856 vm_map_entry_t ltree, rtree; 857 vm_map_entry_t y; 858 859 /* Special case of empty tree. */ 860 if (root == NULL) 861 return (root); 862 863 /* 864 * Pass One: Splay down the tree until we find addr or a NULL 865 * pointer where addr would go. llist and rlist are the two 866 * sides in reverse order (bottom-up), with llist linked by 867 * the right pointer and rlist linked by the left pointer in 868 * the vm_map_entry. Wait until Pass Two to set max_free on 869 * the two spines. 870 */ 871 llist = NULL; 872 rlist = NULL; 873 for (;;) { 874 /* root is never NULL in here. */ 875 if (addr < root->start) { 876 y = root->left; 877 if (y == NULL) 878 break; 879 if (addr < y->start && y->left != NULL) { 880 /* Rotate right and put y on rlist. */ 881 root->left = y->right; 882 y->right = root; 883 vm_map_entry_set_max_free(root); 884 root = y->left; 885 y->left = rlist; 886 rlist = y; 887 } else { 888 /* Put root on rlist. */ 889 root->left = rlist; 890 rlist = root; 891 root = y; 892 } 893 } else if (addr >= root->end) { 894 y = root->right; 895 if (y == NULL) 896 break; 897 if (addr >= y->end && y->right != NULL) { 898 /* Rotate left and put y on llist. */ 899 root->right = y->left; 900 y->left = root; 901 vm_map_entry_set_max_free(root); 902 root = y->right; 903 y->right = llist; 904 llist = y; 905 } else { 906 /* Put root on llist. */ 907 root->right = llist; 908 llist = root; 909 root = y; 910 } 911 } else 912 break; 913 } 914 915 /* 916 * Pass Two: Walk back up the two spines, flip the pointers 917 * and set max_free. The subtrees of the root go at the 918 * bottom of llist and rlist. 919 */ 920 ltree = root->left; 921 while (llist != NULL) { 922 y = llist->right; 923 llist->right = ltree; 924 vm_map_entry_set_max_free(llist); 925 ltree = llist; 926 llist = y; 927 } 928 rtree = root->right; 929 while (rlist != NULL) { 930 y = rlist->left; 931 rlist->left = rtree; 932 vm_map_entry_set_max_free(rlist); 933 rtree = rlist; 934 rlist = y; 935 } 936 937 /* 938 * Final assembly: add ltree and rtree as subtrees of root. 939 */ 940 root->left = ltree; 941 root->right = rtree; 942 vm_map_entry_set_max_free(root); 943 944 return (root); 945 } 946 947 /* 948 * vm_map_entry_{un,}link: 949 * 950 * Insert/remove entries from maps. 951 */ 952 static void 953 vm_map_entry_link(vm_map_t map, 954 vm_map_entry_t after_where, 955 vm_map_entry_t entry) 956 { 957 958 CTR4(KTR_VM, 959 "vm_map_entry_link: map %p, nentries %d, entry %p, after %p", map, 960 map->nentries, entry, after_where); 961 VM_MAP_ASSERT_LOCKED(map); 962 map->nentries++; 963 entry->prev = after_where; 964 entry->next = after_where->next; 965 entry->next->prev = entry; 966 after_where->next = entry; 967 968 if (after_where != &map->header) { 969 if (after_where != map->root) 970 vm_map_entry_splay(after_where->start, map->root); 971 entry->right = after_where->right; 972 entry->left = after_where; 973 after_where->right = NULL; 974 after_where->adj_free = entry->start - after_where->end; 975 vm_map_entry_set_max_free(after_where); 976 } else { 977 entry->right = map->root; 978 entry->left = NULL; 979 } 980 entry->adj_free = (entry->next == &map->header ? map->max_offset : 981 entry->next->start) - entry->end; 982 vm_map_entry_set_max_free(entry); 983 map->root = entry; 984 } 985 986 static void 987 vm_map_entry_unlink(vm_map_t map, 988 vm_map_entry_t entry) 989 { 990 vm_map_entry_t next, prev, root; 991 992 VM_MAP_ASSERT_LOCKED(map); 993 if (entry != map->root) 994 vm_map_entry_splay(entry->start, map->root); 995 if (entry->left == NULL) 996 root = entry->right; 997 else { 998 root = vm_map_entry_splay(entry->start, entry->left); 999 root->right = entry->right; 1000 root->adj_free = (entry->next == &map->header ? map->max_offset : 1001 entry->next->start) - root->end; 1002 vm_map_entry_set_max_free(root); 1003 } 1004 map->root = root; 1005 1006 prev = entry->prev; 1007 next = entry->next; 1008 next->prev = prev; 1009 prev->next = next; 1010 map->nentries--; 1011 CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map, 1012 map->nentries, entry); 1013 } 1014 1015 /* 1016 * vm_map_entry_resize_free: 1017 * 1018 * Recompute the amount of free space following a vm_map_entry 1019 * and propagate that value up the tree. Call this function after 1020 * resizing a map entry in-place, that is, without a call to 1021 * vm_map_entry_link() or _unlink(). 1022 * 1023 * The map must be locked, and leaves it so. 1024 */ 1025 static void 1026 vm_map_entry_resize_free(vm_map_t map, vm_map_entry_t entry) 1027 { 1028 1029 /* 1030 * Using splay trees without parent pointers, propagating 1031 * max_free up the tree is done by moving the entry to the 1032 * root and making the change there. 1033 */ 1034 if (entry != map->root) 1035 map->root = vm_map_entry_splay(entry->start, map->root); 1036 1037 entry->adj_free = (entry->next == &map->header ? map->max_offset : 1038 entry->next->start) - entry->end; 1039 vm_map_entry_set_max_free(entry); 1040 } 1041 1042 /* 1043 * vm_map_lookup_entry: [ internal use only ] 1044 * 1045 * Finds the map entry containing (or 1046 * immediately preceding) the specified address 1047 * in the given map; the entry is returned 1048 * in the "entry" parameter. The boolean 1049 * result indicates whether the address is 1050 * actually contained in the map. 1051 */ 1052 boolean_t 1053 vm_map_lookup_entry( 1054 vm_map_t map, 1055 vm_offset_t address, 1056 vm_map_entry_t *entry) /* OUT */ 1057 { 1058 vm_map_entry_t cur; 1059 boolean_t locked; 1060 1061 /* 1062 * If the map is empty, then the map entry immediately preceding 1063 * "address" is the map's header. 1064 */ 1065 cur = map->root; 1066 if (cur == NULL) 1067 *entry = &map->header; 1068 else if (address >= cur->start && cur->end > address) { 1069 *entry = cur; 1070 return (TRUE); 1071 } else if ((locked = vm_map_locked(map)) || 1072 sx_try_upgrade(&map->lock)) { 1073 /* 1074 * Splay requires a write lock on the map. However, it only 1075 * restructures the binary search tree; it does not otherwise 1076 * change the map. Thus, the map's timestamp need not change 1077 * on a temporary upgrade. 1078 */ 1079 map->root = cur = vm_map_entry_splay(address, cur); 1080 if (!locked) 1081 sx_downgrade(&map->lock); 1082 1083 /* 1084 * If "address" is contained within a map entry, the new root 1085 * is that map entry. Otherwise, the new root is a map entry 1086 * immediately before or after "address". 1087 */ 1088 if (address >= cur->start) { 1089 *entry = cur; 1090 if (cur->end > address) 1091 return (TRUE); 1092 } else 1093 *entry = cur->prev; 1094 } else 1095 /* 1096 * Since the map is only locked for read access, perform a 1097 * standard binary search tree lookup for "address". 1098 */ 1099 for (;;) { 1100 if (address < cur->start) { 1101 if (cur->left == NULL) { 1102 *entry = cur->prev; 1103 break; 1104 } 1105 cur = cur->left; 1106 } else if (cur->end > address) { 1107 *entry = cur; 1108 return (TRUE); 1109 } else { 1110 if (cur->right == NULL) { 1111 *entry = cur; 1112 break; 1113 } 1114 cur = cur->right; 1115 } 1116 } 1117 return (FALSE); 1118 } 1119 1120 /* 1121 * vm_map_insert: 1122 * 1123 * Inserts the given whole VM object into the target 1124 * map at the specified address range. The object's 1125 * size should match that of the address range. 1126 * 1127 * Requires that the map be locked, and leaves it so. 1128 * 1129 * If object is non-NULL, ref count must be bumped by caller 1130 * prior to making call to account for the new entry. 1131 */ 1132 int 1133 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, 1134 vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max, 1135 int cow) 1136 { 1137 vm_map_entry_t new_entry; 1138 vm_map_entry_t prev_entry; 1139 vm_map_entry_t temp_entry; 1140 vm_eflags_t protoeflags; 1141 struct ucred *cred; 1142 vm_inherit_t inheritance; 1143 boolean_t charge_prev_obj; 1144 1145 VM_MAP_ASSERT_LOCKED(map); 1146 1147 /* 1148 * Check that the start and end points are not bogus. 1149 */ 1150 if ((start < map->min_offset) || (end > map->max_offset) || 1151 (start >= end)) 1152 return (KERN_INVALID_ADDRESS); 1153 1154 /* 1155 * Find the entry prior to the proposed starting address; if it's part 1156 * of an existing entry, this range is bogus. 1157 */ 1158 if (vm_map_lookup_entry(map, start, &temp_entry)) 1159 return (KERN_NO_SPACE); 1160 1161 prev_entry = temp_entry; 1162 1163 /* 1164 * Assert that the next entry doesn't overlap the end point. 1165 */ 1166 if ((prev_entry->next != &map->header) && 1167 (prev_entry->next->start < end)) 1168 return (KERN_NO_SPACE); 1169 1170 protoeflags = 0; 1171 charge_prev_obj = FALSE; 1172 1173 if (cow & MAP_COPY_ON_WRITE) 1174 protoeflags |= MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY; 1175 1176 if (cow & MAP_NOFAULT) { 1177 protoeflags |= MAP_ENTRY_NOFAULT; 1178 1179 KASSERT(object == NULL, 1180 ("vm_map_insert: paradoxical MAP_NOFAULT request")); 1181 } 1182 if (cow & MAP_DISABLE_SYNCER) 1183 protoeflags |= MAP_ENTRY_NOSYNC; 1184 if (cow & MAP_DISABLE_COREDUMP) 1185 protoeflags |= MAP_ENTRY_NOCOREDUMP; 1186 if (cow & MAP_VN_WRITECOUNT) 1187 protoeflags |= MAP_ENTRY_VN_WRITECNT; 1188 if (cow & MAP_INHERIT_SHARE) 1189 inheritance = VM_INHERIT_SHARE; 1190 else 1191 inheritance = VM_INHERIT_DEFAULT; 1192 1193 cred = NULL; 1194 KASSERT((object != kmem_object && object != kernel_object) || 1195 ((object == kmem_object || object == kernel_object) && 1196 !(protoeflags & MAP_ENTRY_NEEDS_COPY)), 1197 ("kmem or kernel object and cow")); 1198 if (cow & (MAP_ACC_NO_CHARGE | MAP_NOFAULT)) 1199 goto charged; 1200 if ((cow & MAP_ACC_CHARGED) || ((prot & VM_PROT_WRITE) && 1201 ((protoeflags & MAP_ENTRY_NEEDS_COPY) || object == NULL))) { 1202 if (!(cow & MAP_ACC_CHARGED) && !swap_reserve(end - start)) 1203 return (KERN_RESOURCE_SHORTAGE); 1204 KASSERT(object == NULL || (protoeflags & MAP_ENTRY_NEEDS_COPY) || 1205 object->cred == NULL, 1206 ("OVERCOMMIT: vm_map_insert o %p", object)); 1207 cred = curthread->td_ucred; 1208 crhold(cred); 1209 if (object == NULL && !(protoeflags & MAP_ENTRY_NEEDS_COPY)) 1210 charge_prev_obj = TRUE; 1211 } 1212 1213 charged: 1214 /* Expand the kernel pmap, if necessary. */ 1215 if (map == kernel_map && end > kernel_vm_end) 1216 pmap_growkernel(end); 1217 if (object != NULL) { 1218 /* 1219 * OBJ_ONEMAPPING must be cleared unless this mapping 1220 * is trivially proven to be the only mapping for any 1221 * of the object's pages. (Object granularity 1222 * reference counting is insufficient to recognize 1223 * aliases with precision.) 1224 */ 1225 VM_OBJECT_WLOCK(object); 1226 if (object->ref_count > 1 || object->shadow_count != 0) 1227 vm_object_clear_flag(object, OBJ_ONEMAPPING); 1228 VM_OBJECT_WUNLOCK(object); 1229 } 1230 else if ((prev_entry != &map->header) && 1231 (prev_entry->eflags == protoeflags) && 1232 (prev_entry->end == start) && 1233 (prev_entry->wired_count == 0) && 1234 (prev_entry->cred == cred || 1235 (prev_entry->object.vm_object != NULL && 1236 (prev_entry->object.vm_object->cred == cred))) && 1237 vm_object_coalesce(prev_entry->object.vm_object, 1238 prev_entry->offset, 1239 (vm_size_t)(prev_entry->end - prev_entry->start), 1240 (vm_size_t)(end - prev_entry->end), charge_prev_obj)) { 1241 /* 1242 * We were able to extend the object. Determine if we 1243 * can extend the previous map entry to include the 1244 * new range as well. 1245 */ 1246 if ((prev_entry->inheritance == inheritance) && 1247 (prev_entry->protection == prot) && 1248 (prev_entry->max_protection == max)) { 1249 map->size += (end - prev_entry->end); 1250 prev_entry->end = end; 1251 vm_map_entry_resize_free(map, prev_entry); 1252 vm_map_simplify_entry(map, prev_entry); 1253 if (cred != NULL) 1254 crfree(cred); 1255 return (KERN_SUCCESS); 1256 } 1257 1258 /* 1259 * If we can extend the object but cannot extend the 1260 * map entry, we have to create a new map entry. We 1261 * must bump the ref count on the extended object to 1262 * account for it. object may be NULL. 1263 */ 1264 object = prev_entry->object.vm_object; 1265 offset = prev_entry->offset + 1266 (prev_entry->end - prev_entry->start); 1267 vm_object_reference(object); 1268 if (cred != NULL && object != NULL && object->cred != NULL && 1269 !(prev_entry->eflags & MAP_ENTRY_NEEDS_COPY)) { 1270 /* Object already accounts for this uid. */ 1271 crfree(cred); 1272 cred = NULL; 1273 } 1274 } 1275 1276 /* 1277 * NOTE: if conditionals fail, object can be NULL here. This occurs 1278 * in things like the buffer map where we manage kva but do not manage 1279 * backing objects. 1280 */ 1281 1282 /* 1283 * Create a new entry 1284 */ 1285 new_entry = vm_map_entry_create(map); 1286 new_entry->start = start; 1287 new_entry->end = end; 1288 new_entry->cred = NULL; 1289 1290 new_entry->eflags = protoeflags; 1291 new_entry->object.vm_object = object; 1292 new_entry->offset = offset; 1293 new_entry->avail_ssize = 0; 1294 1295 new_entry->inheritance = inheritance; 1296 new_entry->protection = prot; 1297 new_entry->max_protection = max; 1298 new_entry->wired_count = 0; 1299 new_entry->read_ahead = VM_FAULT_READ_AHEAD_INIT; 1300 new_entry->next_read = OFF_TO_IDX(offset); 1301 1302 KASSERT(cred == NULL || !ENTRY_CHARGED(new_entry), 1303 ("OVERCOMMIT: vm_map_insert leaks vm_map %p", new_entry)); 1304 new_entry->cred = cred; 1305 1306 /* 1307 * Insert the new entry into the list 1308 */ 1309 vm_map_entry_link(map, prev_entry, new_entry); 1310 map->size += new_entry->end - new_entry->start; 1311 1312 /* 1313 * It may be possible to merge the new entry with the next and/or 1314 * previous entries. However, due to MAP_STACK_* being a hack, a 1315 * panic can result from merging such entries. 1316 */ 1317 if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0) 1318 vm_map_simplify_entry(map, new_entry); 1319 1320 if (cow & (MAP_PREFAULT|MAP_PREFAULT_PARTIAL)) { 1321 vm_map_pmap_enter(map, start, prot, 1322 object, OFF_TO_IDX(offset), end - start, 1323 cow & MAP_PREFAULT_PARTIAL); 1324 } 1325 1326 return (KERN_SUCCESS); 1327 } 1328 1329 /* 1330 * vm_map_findspace: 1331 * 1332 * Find the first fit (lowest VM address) for "length" free bytes 1333 * beginning at address >= start in the given map. 1334 * 1335 * In a vm_map_entry, "adj_free" is the amount of free space 1336 * adjacent (higher address) to this entry, and "max_free" is the 1337 * maximum amount of contiguous free space in its subtree. This 1338 * allows finding a free region in one path down the tree, so 1339 * O(log n) amortized with splay trees. 1340 * 1341 * The map must be locked, and leaves it so. 1342 * 1343 * Returns: 0 on success, and starting address in *addr, 1344 * 1 if insufficient space. 1345 */ 1346 int 1347 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length, 1348 vm_offset_t *addr) /* OUT */ 1349 { 1350 vm_map_entry_t entry; 1351 vm_offset_t st; 1352 1353 /* 1354 * Request must fit within min/max VM address and must avoid 1355 * address wrap. 1356 */ 1357 if (start < map->min_offset) 1358 start = map->min_offset; 1359 if (start + length > map->max_offset || start + length < start) 1360 return (1); 1361 1362 /* Empty tree means wide open address space. */ 1363 if (map->root == NULL) { 1364 *addr = start; 1365 return (0); 1366 } 1367 1368 /* 1369 * After splay, if start comes before root node, then there 1370 * must be a gap from start to the root. 1371 */ 1372 map->root = vm_map_entry_splay(start, map->root); 1373 if (start + length <= map->root->start) { 1374 *addr = start; 1375 return (0); 1376 } 1377 1378 /* 1379 * Root is the last node that might begin its gap before 1380 * start, and this is the last comparison where address 1381 * wrap might be a problem. 1382 */ 1383 st = (start > map->root->end) ? start : map->root->end; 1384 if (length <= map->root->end + map->root->adj_free - st) { 1385 *addr = st; 1386 return (0); 1387 } 1388 1389 /* With max_free, can immediately tell if no solution. */ 1390 entry = map->root->right; 1391 if (entry == NULL || length > entry->max_free) 1392 return (1); 1393 1394 /* 1395 * Search the right subtree in the order: left subtree, root, 1396 * right subtree (first fit). The previous splay implies that 1397 * all regions in the right subtree have addresses > start. 1398 */ 1399 while (entry != NULL) { 1400 if (entry->left != NULL && entry->left->max_free >= length) 1401 entry = entry->left; 1402 else if (entry->adj_free >= length) { 1403 *addr = entry->end; 1404 return (0); 1405 } else 1406 entry = entry->right; 1407 } 1408 1409 /* Can't get here, so panic if we do. */ 1410 panic("vm_map_findspace: max_free corrupt"); 1411 } 1412 1413 int 1414 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, 1415 vm_offset_t start, vm_size_t length, vm_prot_t prot, 1416 vm_prot_t max, int cow) 1417 { 1418 vm_offset_t end; 1419 int result; 1420 1421 end = start + length; 1422 vm_map_lock(map); 1423 VM_MAP_RANGE_CHECK(map, start, end); 1424 (void) vm_map_delete(map, start, end); 1425 result = vm_map_insert(map, object, offset, start, end, prot, 1426 max, cow); 1427 vm_map_unlock(map); 1428 return (result); 1429 } 1430 1431 /* 1432 * vm_map_find finds an unallocated region in the target address 1433 * map with the given length. The search is defined to be 1434 * first-fit from the specified address; the region found is 1435 * returned in the same parameter. 1436 * 1437 * If object is non-NULL, ref count must be bumped by caller 1438 * prior to making call to account for the new entry. 1439 */ 1440 int 1441 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, 1442 vm_offset_t *addr, /* IN/OUT */ 1443 vm_size_t length, int find_space, vm_prot_t prot, 1444 vm_prot_t max, int cow) 1445 { 1446 vm_offset_t start, initial_addr; 1447 int result; 1448 1449 if (find_space == VMFS_OPTIMAL_SPACE && (object == NULL || 1450 (object->flags & OBJ_COLORED) == 0)) 1451 find_space = VMFS_ANY_SPACE; 1452 initial_addr = *addr; 1453 again: 1454 start = initial_addr; 1455 vm_map_lock(map); 1456 do { 1457 if (find_space != VMFS_NO_SPACE) { 1458 if (vm_map_findspace(map, start, length, addr)) { 1459 vm_map_unlock(map); 1460 if (find_space == VMFS_OPTIMAL_SPACE) { 1461 find_space = VMFS_ANY_SPACE; 1462 goto again; 1463 } 1464 return (KERN_NO_SPACE); 1465 } 1466 switch (find_space) { 1467 case VMFS_ALIGNED_SPACE: 1468 case VMFS_OPTIMAL_SPACE: 1469 pmap_align_superpage(object, offset, addr, 1470 length); 1471 break; 1472 #ifdef VMFS_TLB_ALIGNED_SPACE 1473 case VMFS_TLB_ALIGNED_SPACE: 1474 pmap_align_tlb(addr); 1475 break; 1476 #endif 1477 default: 1478 break; 1479 } 1480 1481 start = *addr; 1482 } 1483 result = vm_map_insert(map, object, offset, start, start + 1484 length, prot, max, cow); 1485 } while (result == KERN_NO_SPACE && (find_space == VMFS_ALIGNED_SPACE || 1486 #ifdef VMFS_TLB_ALIGNED_SPACE 1487 find_space == VMFS_TLB_ALIGNED_SPACE || 1488 #endif 1489 find_space == VMFS_OPTIMAL_SPACE)); 1490 vm_map_unlock(map); 1491 return (result); 1492 } 1493 1494 /* 1495 * vm_map_simplify_entry: 1496 * 1497 * Simplify the given map entry by merging with either neighbor. This 1498 * routine also has the ability to merge with both neighbors. 1499 * 1500 * The map must be locked. 1501 * 1502 * This routine guarentees that the passed entry remains valid (though 1503 * possibly extended). When merging, this routine may delete one or 1504 * both neighbors. 1505 */ 1506 void 1507 vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry) 1508 { 1509 vm_map_entry_t next, prev; 1510 vm_size_t prevsize, esize; 1511 1512 if (entry->eflags & (MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP)) 1513 return; 1514 1515 prev = entry->prev; 1516 if (prev != &map->header) { 1517 prevsize = prev->end - prev->start; 1518 if ( (prev->end == entry->start) && 1519 (prev->object.vm_object == entry->object.vm_object) && 1520 (!prev->object.vm_object || 1521 (prev->offset + prevsize == entry->offset)) && 1522 (prev->eflags == entry->eflags) && 1523 (prev->protection == entry->protection) && 1524 (prev->max_protection == entry->max_protection) && 1525 (prev->inheritance == entry->inheritance) && 1526 (prev->wired_count == entry->wired_count) && 1527 (prev->cred == entry->cred)) { 1528 vm_map_entry_unlink(map, prev); 1529 entry->start = prev->start; 1530 entry->offset = prev->offset; 1531 if (entry->prev != &map->header) 1532 vm_map_entry_resize_free(map, entry->prev); 1533 1534 /* 1535 * If the backing object is a vnode object, 1536 * vm_object_deallocate() calls vrele(). 1537 * However, vrele() does not lock the vnode 1538 * because the vnode has additional 1539 * references. Thus, the map lock can be kept 1540 * without causing a lock-order reversal with 1541 * the vnode lock. 1542 * 1543 * Since we count the number of virtual page 1544 * mappings in object->un_pager.vnp.writemappings, 1545 * the writemappings value should not be adjusted 1546 * when the entry is disposed of. 1547 */ 1548 if (prev->object.vm_object) 1549 vm_object_deallocate(prev->object.vm_object); 1550 if (prev->cred != NULL) 1551 crfree(prev->cred); 1552 vm_map_entry_dispose(map, prev); 1553 } 1554 } 1555 1556 next = entry->next; 1557 if (next != &map->header) { 1558 esize = entry->end - entry->start; 1559 if ((entry->end == next->start) && 1560 (next->object.vm_object == entry->object.vm_object) && 1561 (!entry->object.vm_object || 1562 (entry->offset + esize == next->offset)) && 1563 (next->eflags == entry->eflags) && 1564 (next->protection == entry->protection) && 1565 (next->max_protection == entry->max_protection) && 1566 (next->inheritance == entry->inheritance) && 1567 (next->wired_count == entry->wired_count) && 1568 (next->cred == entry->cred)) { 1569 vm_map_entry_unlink(map, next); 1570 entry->end = next->end; 1571 vm_map_entry_resize_free(map, entry); 1572 1573 /* 1574 * See comment above. 1575 */ 1576 if (next->object.vm_object) 1577 vm_object_deallocate(next->object.vm_object); 1578 if (next->cred != NULL) 1579 crfree(next->cred); 1580 vm_map_entry_dispose(map, next); 1581 } 1582 } 1583 } 1584 /* 1585 * vm_map_clip_start: [ internal use only ] 1586 * 1587 * Asserts that the given entry begins at or after 1588 * the specified address; if necessary, 1589 * it splits the entry into two. 1590 */ 1591 #define vm_map_clip_start(map, entry, startaddr) \ 1592 { \ 1593 if (startaddr > entry->start) \ 1594 _vm_map_clip_start(map, entry, startaddr); \ 1595 } 1596 1597 /* 1598 * This routine is called only when it is known that 1599 * the entry must be split. 1600 */ 1601 static void 1602 _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start) 1603 { 1604 vm_map_entry_t new_entry; 1605 1606 VM_MAP_ASSERT_LOCKED(map); 1607 1608 /* 1609 * Split off the front portion -- note that we must insert the new 1610 * entry BEFORE this one, so that this entry has the specified 1611 * starting address. 1612 */ 1613 vm_map_simplify_entry(map, entry); 1614 1615 /* 1616 * If there is no object backing this entry, we might as well create 1617 * one now. If we defer it, an object can get created after the map 1618 * is clipped, and individual objects will be created for the split-up 1619 * map. This is a bit of a hack, but is also about the best place to 1620 * put this improvement. 1621 */ 1622 if (entry->object.vm_object == NULL && !map->system_map) { 1623 vm_object_t object; 1624 object = vm_object_allocate(OBJT_DEFAULT, 1625 atop(entry->end - entry->start)); 1626 entry->object.vm_object = object; 1627 entry->offset = 0; 1628 if (entry->cred != NULL) { 1629 object->cred = entry->cred; 1630 object->charge = entry->end - entry->start; 1631 entry->cred = NULL; 1632 } 1633 } else if (entry->object.vm_object != NULL && 1634 ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) && 1635 entry->cred != NULL) { 1636 VM_OBJECT_WLOCK(entry->object.vm_object); 1637 KASSERT(entry->object.vm_object->cred == NULL, 1638 ("OVERCOMMIT: vm_entry_clip_start: both cred e %p", entry)); 1639 entry->object.vm_object->cred = entry->cred; 1640 entry->object.vm_object->charge = entry->end - entry->start; 1641 VM_OBJECT_WUNLOCK(entry->object.vm_object); 1642 entry->cred = NULL; 1643 } 1644 1645 new_entry = vm_map_entry_create(map); 1646 *new_entry = *entry; 1647 1648 new_entry->end = start; 1649 entry->offset += (start - entry->start); 1650 entry->start = start; 1651 if (new_entry->cred != NULL) 1652 crhold(entry->cred); 1653 1654 vm_map_entry_link(map, entry->prev, new_entry); 1655 1656 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { 1657 vm_object_reference(new_entry->object.vm_object); 1658 /* 1659 * The object->un_pager.vnp.writemappings for the 1660 * object of MAP_ENTRY_VN_WRITECNT type entry shall be 1661 * kept as is here. The virtual pages are 1662 * re-distributed among the clipped entries, so the sum is 1663 * left the same. 1664 */ 1665 } 1666 } 1667 1668 /* 1669 * vm_map_clip_end: [ internal use only ] 1670 * 1671 * Asserts that the given entry ends at or before 1672 * the specified address; if necessary, 1673 * it splits the entry into two. 1674 */ 1675 #define vm_map_clip_end(map, entry, endaddr) \ 1676 { \ 1677 if ((endaddr) < (entry->end)) \ 1678 _vm_map_clip_end((map), (entry), (endaddr)); \ 1679 } 1680 1681 /* 1682 * This routine is called only when it is known that 1683 * the entry must be split. 1684 */ 1685 static void 1686 _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end) 1687 { 1688 vm_map_entry_t new_entry; 1689 1690 VM_MAP_ASSERT_LOCKED(map); 1691 1692 /* 1693 * If there is no object backing this entry, we might as well create 1694 * one now. If we defer it, an object can get created after the map 1695 * is clipped, and individual objects will be created for the split-up 1696 * map. This is a bit of a hack, but is also about the best place to 1697 * put this improvement. 1698 */ 1699 if (entry->object.vm_object == NULL && !map->system_map) { 1700 vm_object_t object; 1701 object = vm_object_allocate(OBJT_DEFAULT, 1702 atop(entry->end - entry->start)); 1703 entry->object.vm_object = object; 1704 entry->offset = 0; 1705 if (entry->cred != NULL) { 1706 object->cred = entry->cred; 1707 object->charge = entry->end - entry->start; 1708 entry->cred = NULL; 1709 } 1710 } else if (entry->object.vm_object != NULL && 1711 ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) && 1712 entry->cred != NULL) { 1713 VM_OBJECT_WLOCK(entry->object.vm_object); 1714 KASSERT(entry->object.vm_object->cred == NULL, 1715 ("OVERCOMMIT: vm_entry_clip_end: both cred e %p", entry)); 1716 entry->object.vm_object->cred = entry->cred; 1717 entry->object.vm_object->charge = entry->end - entry->start; 1718 VM_OBJECT_WUNLOCK(entry->object.vm_object); 1719 entry->cred = NULL; 1720 } 1721 1722 /* 1723 * Create a new entry and insert it AFTER the specified entry 1724 */ 1725 new_entry = vm_map_entry_create(map); 1726 *new_entry = *entry; 1727 1728 new_entry->start = entry->end = end; 1729 new_entry->offset += (end - entry->start); 1730 if (new_entry->cred != NULL) 1731 crhold(entry->cred); 1732 1733 vm_map_entry_link(map, entry, new_entry); 1734 1735 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { 1736 vm_object_reference(new_entry->object.vm_object); 1737 } 1738 } 1739 1740 /* 1741 * vm_map_submap: [ kernel use only ] 1742 * 1743 * Mark the given range as handled by a subordinate map. 1744 * 1745 * This range must have been created with vm_map_find, 1746 * and no other operations may have been performed on this 1747 * range prior to calling vm_map_submap. 1748 * 1749 * Only a limited number of operations can be performed 1750 * within this rage after calling vm_map_submap: 1751 * vm_fault 1752 * [Don't try vm_map_copy!] 1753 * 1754 * To remove a submapping, one must first remove the 1755 * range from the superior map, and then destroy the 1756 * submap (if desired). [Better yet, don't try it.] 1757 */ 1758 int 1759 vm_map_submap( 1760 vm_map_t map, 1761 vm_offset_t start, 1762 vm_offset_t end, 1763 vm_map_t submap) 1764 { 1765 vm_map_entry_t entry; 1766 int result = KERN_INVALID_ARGUMENT; 1767 1768 vm_map_lock(map); 1769 1770 VM_MAP_RANGE_CHECK(map, start, end); 1771 1772 if (vm_map_lookup_entry(map, start, &entry)) { 1773 vm_map_clip_start(map, entry, start); 1774 } else 1775 entry = entry->next; 1776 1777 vm_map_clip_end(map, entry, end); 1778 1779 if ((entry->start == start) && (entry->end == end) && 1780 ((entry->eflags & MAP_ENTRY_COW) == 0) && 1781 (entry->object.vm_object == NULL)) { 1782 entry->object.sub_map = submap; 1783 entry->eflags |= MAP_ENTRY_IS_SUB_MAP; 1784 result = KERN_SUCCESS; 1785 } 1786 vm_map_unlock(map); 1787 1788 return (result); 1789 } 1790 1791 /* 1792 * The maximum number of pages to map 1793 */ 1794 #define MAX_INIT_PT 96 1795 1796 /* 1797 * vm_map_pmap_enter: 1798 * 1799 * Preload read-only mappings for the specified object's resident pages 1800 * into the target map. If "flags" is MAP_PREFAULT_PARTIAL, then only 1801 * the resident pages within the address range [addr, addr + ulmin(size, 1802 * ptoa(MAX_INIT_PT))) are mapped. Otherwise, all resident pages within 1803 * the specified address range are mapped. This eliminates many soft 1804 * faults on process startup and immediately after an mmap(2). Because 1805 * these are speculative mappings, cached pages are not reactivated and 1806 * mapped. 1807 */ 1808 void 1809 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, 1810 vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags) 1811 { 1812 vm_offset_t start; 1813 vm_page_t p, p_start; 1814 vm_pindex_t psize, tmpidx; 1815 1816 if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL) 1817 return; 1818 VM_OBJECT_RLOCK(object); 1819 if (object->type == OBJT_DEVICE || object->type == OBJT_SG) { 1820 VM_OBJECT_RUNLOCK(object); 1821 VM_OBJECT_WLOCK(object); 1822 if (object->type == OBJT_DEVICE || object->type == OBJT_SG) { 1823 pmap_object_init_pt(map->pmap, addr, object, pindex, 1824 size); 1825 VM_OBJECT_WUNLOCK(object); 1826 return; 1827 } 1828 VM_OBJECT_LOCK_DOWNGRADE(object); 1829 } 1830 1831 psize = atop(size); 1832 if (psize > MAX_INIT_PT && (flags & MAP_PREFAULT_PARTIAL) != 0) 1833 psize = MAX_INIT_PT; 1834 if (psize + pindex > object->size) { 1835 if (object->size < pindex) { 1836 VM_OBJECT_RUNLOCK(object); 1837 return; 1838 } 1839 psize = object->size - pindex; 1840 } 1841 1842 start = 0; 1843 p_start = NULL; 1844 1845 p = vm_page_find_least(object, pindex); 1846 /* 1847 * Assert: the variable p is either (1) the page with the 1848 * least pindex greater than or equal to the parameter pindex 1849 * or (2) NULL. 1850 */ 1851 for (; 1852 p != NULL && (tmpidx = p->pindex - pindex) < psize; 1853 p = TAILQ_NEXT(p, listq)) { 1854 /* 1855 * don't allow an madvise to blow away our really 1856 * free pages allocating pv entries. 1857 */ 1858 if ((flags & MAP_PREFAULT_MADVISE) && 1859 cnt.v_free_count < cnt.v_free_reserved) { 1860 psize = tmpidx; 1861 break; 1862 } 1863 if (p->valid == VM_PAGE_BITS_ALL) { 1864 if (p_start == NULL) { 1865 start = addr + ptoa(tmpidx); 1866 p_start = p; 1867 } 1868 } else if (p_start != NULL) { 1869 pmap_enter_object(map->pmap, start, addr + 1870 ptoa(tmpidx), p_start, prot); 1871 p_start = NULL; 1872 } 1873 } 1874 if (p_start != NULL) 1875 pmap_enter_object(map->pmap, start, addr + ptoa(psize), 1876 p_start, prot); 1877 VM_OBJECT_RUNLOCK(object); 1878 } 1879 1880 /* 1881 * vm_map_protect: 1882 * 1883 * Sets the protection of the specified address 1884 * region in the target map. If "set_max" is 1885 * specified, the maximum protection is to be set; 1886 * otherwise, only the current protection is affected. 1887 */ 1888 int 1889 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end, 1890 vm_prot_t new_prot, boolean_t set_max) 1891 { 1892 vm_map_entry_t current, entry; 1893 vm_object_t obj; 1894 struct ucred *cred; 1895 vm_prot_t old_prot; 1896 1897 vm_map_lock(map); 1898 1899 VM_MAP_RANGE_CHECK(map, start, end); 1900 1901 if (vm_map_lookup_entry(map, start, &entry)) { 1902 vm_map_clip_start(map, entry, start); 1903 } else { 1904 entry = entry->next; 1905 } 1906 1907 /* 1908 * Make a first pass to check for protection violations. 1909 */ 1910 current = entry; 1911 while ((current != &map->header) && (current->start < end)) { 1912 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) { 1913 vm_map_unlock(map); 1914 return (KERN_INVALID_ARGUMENT); 1915 } 1916 if ((new_prot & current->max_protection) != new_prot) { 1917 vm_map_unlock(map); 1918 return (KERN_PROTECTION_FAILURE); 1919 } 1920 current = current->next; 1921 } 1922 1923 1924 /* 1925 * Do an accounting pass for private read-only mappings that 1926 * now will do cow due to allowed write (e.g. debugger sets 1927 * breakpoint on text segment) 1928 */ 1929 for (current = entry; (current != &map->header) && 1930 (current->start < end); current = current->next) { 1931 1932 vm_map_clip_end(map, current, end); 1933 1934 if (set_max || 1935 ((new_prot & ~(current->protection)) & VM_PROT_WRITE) == 0 || 1936 ENTRY_CHARGED(current)) { 1937 continue; 1938 } 1939 1940 cred = curthread->td_ucred; 1941 obj = current->object.vm_object; 1942 1943 if (obj == NULL || (current->eflags & MAP_ENTRY_NEEDS_COPY)) { 1944 if (!swap_reserve(current->end - current->start)) { 1945 vm_map_unlock(map); 1946 return (KERN_RESOURCE_SHORTAGE); 1947 } 1948 crhold(cred); 1949 current->cred = cred; 1950 continue; 1951 } 1952 1953 VM_OBJECT_WLOCK(obj); 1954 if (obj->type != OBJT_DEFAULT && obj->type != OBJT_SWAP) { 1955 VM_OBJECT_WUNLOCK(obj); 1956 continue; 1957 } 1958 1959 /* 1960 * Charge for the whole object allocation now, since 1961 * we cannot distinguish between non-charged and 1962 * charged clipped mapping of the same object later. 1963 */ 1964 KASSERT(obj->charge == 0, 1965 ("vm_map_protect: object %p overcharged\n", obj)); 1966 if (!swap_reserve(ptoa(obj->size))) { 1967 VM_OBJECT_WUNLOCK(obj); 1968 vm_map_unlock(map); 1969 return (KERN_RESOURCE_SHORTAGE); 1970 } 1971 1972 crhold(cred); 1973 obj->cred = cred; 1974 obj->charge = ptoa(obj->size); 1975 VM_OBJECT_WUNLOCK(obj); 1976 } 1977 1978 /* 1979 * Go back and fix up protections. [Note that clipping is not 1980 * necessary the second time.] 1981 */ 1982 current = entry; 1983 while ((current != &map->header) && (current->start < end)) { 1984 old_prot = current->protection; 1985 1986 if (set_max) 1987 current->protection = 1988 (current->max_protection = new_prot) & 1989 old_prot; 1990 else 1991 current->protection = new_prot; 1992 1993 if ((current->eflags & (MAP_ENTRY_COW | MAP_ENTRY_USER_WIRED)) 1994 == (MAP_ENTRY_COW | MAP_ENTRY_USER_WIRED) && 1995 (current->protection & VM_PROT_WRITE) != 0 && 1996 (old_prot & VM_PROT_WRITE) == 0) { 1997 vm_fault_copy_entry(map, map, current, current, NULL); 1998 } 1999 2000 /* 2001 * When restricting access, update the physical map. Worry 2002 * about copy-on-write here. 2003 */ 2004 if ((old_prot & ~current->protection) != 0) { 2005 #define MASK(entry) (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \ 2006 VM_PROT_ALL) 2007 pmap_protect(map->pmap, current->start, 2008 current->end, 2009 current->protection & MASK(current)); 2010 #undef MASK 2011 } 2012 vm_map_simplify_entry(map, current); 2013 current = current->next; 2014 } 2015 vm_map_unlock(map); 2016 return (KERN_SUCCESS); 2017 } 2018 2019 /* 2020 * vm_map_madvise: 2021 * 2022 * This routine traverses a processes map handling the madvise 2023 * system call. Advisories are classified as either those effecting 2024 * the vm_map_entry structure, or those effecting the underlying 2025 * objects. 2026 */ 2027 int 2028 vm_map_madvise( 2029 vm_map_t map, 2030 vm_offset_t start, 2031 vm_offset_t end, 2032 int behav) 2033 { 2034 vm_map_entry_t current, entry; 2035 int modify_map = 0; 2036 2037 /* 2038 * Some madvise calls directly modify the vm_map_entry, in which case 2039 * we need to use an exclusive lock on the map and we need to perform 2040 * various clipping operations. Otherwise we only need a read-lock 2041 * on the map. 2042 */ 2043 switch(behav) { 2044 case MADV_NORMAL: 2045 case MADV_SEQUENTIAL: 2046 case MADV_RANDOM: 2047 case MADV_NOSYNC: 2048 case MADV_AUTOSYNC: 2049 case MADV_NOCORE: 2050 case MADV_CORE: 2051 modify_map = 1; 2052 vm_map_lock(map); 2053 break; 2054 case MADV_WILLNEED: 2055 case MADV_DONTNEED: 2056 case MADV_FREE: 2057 vm_map_lock_read(map); 2058 break; 2059 default: 2060 return (KERN_INVALID_ARGUMENT); 2061 } 2062 2063 /* 2064 * Locate starting entry and clip if necessary. 2065 */ 2066 VM_MAP_RANGE_CHECK(map, start, end); 2067 2068 if (vm_map_lookup_entry(map, start, &entry)) { 2069 if (modify_map) 2070 vm_map_clip_start(map, entry, start); 2071 } else { 2072 entry = entry->next; 2073 } 2074 2075 if (modify_map) { 2076 /* 2077 * madvise behaviors that are implemented in the vm_map_entry. 2078 * 2079 * We clip the vm_map_entry so that behavioral changes are 2080 * limited to the specified address range. 2081 */ 2082 for (current = entry; 2083 (current != &map->header) && (current->start < end); 2084 current = current->next 2085 ) { 2086 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) 2087 continue; 2088 2089 vm_map_clip_end(map, current, end); 2090 2091 switch (behav) { 2092 case MADV_NORMAL: 2093 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_NORMAL); 2094 break; 2095 case MADV_SEQUENTIAL: 2096 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_SEQUENTIAL); 2097 break; 2098 case MADV_RANDOM: 2099 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_RANDOM); 2100 break; 2101 case MADV_NOSYNC: 2102 current->eflags |= MAP_ENTRY_NOSYNC; 2103 break; 2104 case MADV_AUTOSYNC: 2105 current->eflags &= ~MAP_ENTRY_NOSYNC; 2106 break; 2107 case MADV_NOCORE: 2108 current->eflags |= MAP_ENTRY_NOCOREDUMP; 2109 break; 2110 case MADV_CORE: 2111 current->eflags &= ~MAP_ENTRY_NOCOREDUMP; 2112 break; 2113 default: 2114 break; 2115 } 2116 vm_map_simplify_entry(map, current); 2117 } 2118 vm_map_unlock(map); 2119 } else { 2120 vm_pindex_t pstart, pend; 2121 2122 /* 2123 * madvise behaviors that are implemented in the underlying 2124 * vm_object. 2125 * 2126 * Since we don't clip the vm_map_entry, we have to clip 2127 * the vm_object pindex and count. 2128 */ 2129 for (current = entry; 2130 (current != &map->header) && (current->start < end); 2131 current = current->next 2132 ) { 2133 vm_offset_t useStart; 2134 2135 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) 2136 continue; 2137 2138 pstart = OFF_TO_IDX(current->offset); 2139 pend = pstart + atop(current->end - current->start); 2140 useStart = current->start; 2141 2142 if (current->start < start) { 2143 pstart += atop(start - current->start); 2144 useStart = start; 2145 } 2146 if (current->end > end) 2147 pend -= atop(current->end - end); 2148 2149 if (pstart >= pend) 2150 continue; 2151 2152 vm_object_madvise(current->object.vm_object, pstart, 2153 pend, behav); 2154 if (behav == MADV_WILLNEED) { 2155 vm_map_pmap_enter(map, 2156 useStart, 2157 current->protection, 2158 current->object.vm_object, 2159 pstart, 2160 ptoa(pend - pstart), 2161 MAP_PREFAULT_MADVISE 2162 ); 2163 } 2164 } 2165 vm_map_unlock_read(map); 2166 } 2167 return (0); 2168 } 2169 2170 2171 /* 2172 * vm_map_inherit: 2173 * 2174 * Sets the inheritance of the specified address 2175 * range in the target map. Inheritance 2176 * affects how the map will be shared with 2177 * child maps at the time of vmspace_fork. 2178 */ 2179 int 2180 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, 2181 vm_inherit_t new_inheritance) 2182 { 2183 vm_map_entry_t entry; 2184 vm_map_entry_t temp_entry; 2185 2186 switch (new_inheritance) { 2187 case VM_INHERIT_NONE: 2188 case VM_INHERIT_COPY: 2189 case VM_INHERIT_SHARE: 2190 break; 2191 default: 2192 return (KERN_INVALID_ARGUMENT); 2193 } 2194 vm_map_lock(map); 2195 VM_MAP_RANGE_CHECK(map, start, end); 2196 if (vm_map_lookup_entry(map, start, &temp_entry)) { 2197 entry = temp_entry; 2198 vm_map_clip_start(map, entry, start); 2199 } else 2200 entry = temp_entry->next; 2201 while ((entry != &map->header) && (entry->start < end)) { 2202 vm_map_clip_end(map, entry, end); 2203 entry->inheritance = new_inheritance; 2204 vm_map_simplify_entry(map, entry); 2205 entry = entry->next; 2206 } 2207 vm_map_unlock(map); 2208 return (KERN_SUCCESS); 2209 } 2210 2211 /* 2212 * vm_map_unwire: 2213 * 2214 * Implements both kernel and user unwiring. 2215 */ 2216 int 2217 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, 2218 int flags) 2219 { 2220 vm_map_entry_t entry, first_entry, tmp_entry; 2221 vm_offset_t saved_start; 2222 unsigned int last_timestamp; 2223 int rv; 2224 boolean_t need_wakeup, result, user_unwire; 2225 2226 user_unwire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE; 2227 vm_map_lock(map); 2228 VM_MAP_RANGE_CHECK(map, start, end); 2229 if (!vm_map_lookup_entry(map, start, &first_entry)) { 2230 if (flags & VM_MAP_WIRE_HOLESOK) 2231 first_entry = first_entry->next; 2232 else { 2233 vm_map_unlock(map); 2234 return (KERN_INVALID_ADDRESS); 2235 } 2236 } 2237 last_timestamp = map->timestamp; 2238 entry = first_entry; 2239 while (entry != &map->header && entry->start < end) { 2240 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { 2241 /* 2242 * We have not yet clipped the entry. 2243 */ 2244 saved_start = (start >= entry->start) ? start : 2245 entry->start; 2246 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; 2247 if (vm_map_unlock_and_wait(map, 0)) { 2248 /* 2249 * Allow interruption of user unwiring? 2250 */ 2251 } 2252 vm_map_lock(map); 2253 if (last_timestamp+1 != map->timestamp) { 2254 /* 2255 * Look again for the entry because the map was 2256 * modified while it was unlocked. 2257 * Specifically, the entry may have been 2258 * clipped, merged, or deleted. 2259 */ 2260 if (!vm_map_lookup_entry(map, saved_start, 2261 &tmp_entry)) { 2262 if (flags & VM_MAP_WIRE_HOLESOK) 2263 tmp_entry = tmp_entry->next; 2264 else { 2265 if (saved_start == start) { 2266 /* 2267 * First_entry has been deleted. 2268 */ 2269 vm_map_unlock(map); 2270 return (KERN_INVALID_ADDRESS); 2271 } 2272 end = saved_start; 2273 rv = KERN_INVALID_ADDRESS; 2274 goto done; 2275 } 2276 } 2277 if (entry == first_entry) 2278 first_entry = tmp_entry; 2279 else 2280 first_entry = NULL; 2281 entry = tmp_entry; 2282 } 2283 last_timestamp = map->timestamp; 2284 continue; 2285 } 2286 vm_map_clip_start(map, entry, start); 2287 vm_map_clip_end(map, entry, end); 2288 /* 2289 * Mark the entry in case the map lock is released. (See 2290 * above.) 2291 */ 2292 entry->eflags |= MAP_ENTRY_IN_TRANSITION; 2293 entry->wiring_thread = curthread; 2294 /* 2295 * Check the map for holes in the specified region. 2296 * If VM_MAP_WIRE_HOLESOK was specified, skip this check. 2297 */ 2298 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) && 2299 (entry->end < end && (entry->next == &map->header || 2300 entry->next->start > entry->end))) { 2301 end = entry->end; 2302 rv = KERN_INVALID_ADDRESS; 2303 goto done; 2304 } 2305 /* 2306 * If system unwiring, require that the entry is system wired. 2307 */ 2308 if (!user_unwire && 2309 vm_map_entry_system_wired_count(entry) == 0) { 2310 end = entry->end; 2311 rv = KERN_INVALID_ARGUMENT; 2312 goto done; 2313 } 2314 entry = entry->next; 2315 } 2316 rv = KERN_SUCCESS; 2317 done: 2318 need_wakeup = FALSE; 2319 if (first_entry == NULL) { 2320 result = vm_map_lookup_entry(map, start, &first_entry); 2321 if (!result && (flags & VM_MAP_WIRE_HOLESOK)) 2322 first_entry = first_entry->next; 2323 else 2324 KASSERT(result, ("vm_map_unwire: lookup failed")); 2325 } 2326 for (entry = first_entry; entry != &map->header && entry->start < end; 2327 entry = entry->next) { 2328 /* 2329 * If VM_MAP_WIRE_HOLESOK was specified, an empty 2330 * space in the unwired region could have been mapped 2331 * while the map lock was dropped for draining 2332 * MAP_ENTRY_IN_TRANSITION. Moreover, another thread 2333 * could be simultaneously wiring this new mapping 2334 * entry. Detect these cases and skip any entries 2335 * marked as in transition by us. 2336 */ 2337 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || 2338 entry->wiring_thread != curthread) { 2339 KASSERT((flags & VM_MAP_WIRE_HOLESOK) != 0, 2340 ("vm_map_unwire: !HOLESOK and new/changed entry")); 2341 continue; 2342 } 2343 2344 if (rv == KERN_SUCCESS && (!user_unwire || 2345 (entry->eflags & MAP_ENTRY_USER_WIRED))) { 2346 if (user_unwire) 2347 entry->eflags &= ~MAP_ENTRY_USER_WIRED; 2348 entry->wired_count--; 2349 if (entry->wired_count == 0) { 2350 /* 2351 * Retain the map lock. 2352 */ 2353 vm_fault_unwire(map, entry->start, entry->end, 2354 entry->object.vm_object != NULL && 2355 (entry->object.vm_object->flags & 2356 OBJ_FICTITIOUS) != 0); 2357 } 2358 } 2359 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, 2360 ("vm_map_unwire: in-transition flag missing")); 2361 entry->eflags &= ~MAP_ENTRY_IN_TRANSITION; 2362 entry->wiring_thread = NULL; 2363 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { 2364 entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; 2365 need_wakeup = TRUE; 2366 } 2367 vm_map_simplify_entry(map, entry); 2368 } 2369 vm_map_unlock(map); 2370 if (need_wakeup) 2371 vm_map_wakeup(map); 2372 return (rv); 2373 } 2374 2375 /* 2376 * vm_map_wire: 2377 * 2378 * Implements both kernel and user wiring. 2379 */ 2380 int 2381 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, 2382 int flags) 2383 { 2384 vm_map_entry_t entry, first_entry, tmp_entry; 2385 vm_offset_t saved_end, saved_start; 2386 unsigned int last_timestamp; 2387 int rv; 2388 boolean_t fictitious, need_wakeup, result, user_wire; 2389 vm_prot_t prot; 2390 2391 prot = 0; 2392 if (flags & VM_MAP_WIRE_WRITE) 2393 prot |= VM_PROT_WRITE; 2394 user_wire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE; 2395 vm_map_lock(map); 2396 VM_MAP_RANGE_CHECK(map, start, end); 2397 if (!vm_map_lookup_entry(map, start, &first_entry)) { 2398 if (flags & VM_MAP_WIRE_HOLESOK) 2399 first_entry = first_entry->next; 2400 else { 2401 vm_map_unlock(map); 2402 return (KERN_INVALID_ADDRESS); 2403 } 2404 } 2405 last_timestamp = map->timestamp; 2406 entry = first_entry; 2407 while (entry != &map->header && entry->start < end) { 2408 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { 2409 /* 2410 * We have not yet clipped the entry. 2411 */ 2412 saved_start = (start >= entry->start) ? start : 2413 entry->start; 2414 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; 2415 if (vm_map_unlock_and_wait(map, 0)) { 2416 /* 2417 * Allow interruption of user wiring? 2418 */ 2419 } 2420 vm_map_lock(map); 2421 if (last_timestamp + 1 != map->timestamp) { 2422 /* 2423 * Look again for the entry because the map was 2424 * modified while it was unlocked. 2425 * Specifically, the entry may have been 2426 * clipped, merged, or deleted. 2427 */ 2428 if (!vm_map_lookup_entry(map, saved_start, 2429 &tmp_entry)) { 2430 if (flags & VM_MAP_WIRE_HOLESOK) 2431 tmp_entry = tmp_entry->next; 2432 else { 2433 if (saved_start == start) { 2434 /* 2435 * first_entry has been deleted. 2436 */ 2437 vm_map_unlock(map); 2438 return (KERN_INVALID_ADDRESS); 2439 } 2440 end = saved_start; 2441 rv = KERN_INVALID_ADDRESS; 2442 goto done; 2443 } 2444 } 2445 if (entry == first_entry) 2446 first_entry = tmp_entry; 2447 else 2448 first_entry = NULL; 2449 entry = tmp_entry; 2450 } 2451 last_timestamp = map->timestamp; 2452 continue; 2453 } 2454 vm_map_clip_start(map, entry, start); 2455 vm_map_clip_end(map, entry, end); 2456 /* 2457 * Mark the entry in case the map lock is released. (See 2458 * above.) 2459 */ 2460 entry->eflags |= MAP_ENTRY_IN_TRANSITION; 2461 entry->wiring_thread = curthread; 2462 if ((entry->protection & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 2463 || (entry->protection & prot) != prot) { 2464 entry->eflags |= MAP_ENTRY_WIRE_SKIPPED; 2465 if ((flags & VM_MAP_WIRE_HOLESOK) == 0) { 2466 end = entry->end; 2467 rv = KERN_INVALID_ADDRESS; 2468 goto done; 2469 } 2470 goto next_entry; 2471 } 2472 if (entry->wired_count == 0) { 2473 entry->wired_count++; 2474 saved_start = entry->start; 2475 saved_end = entry->end; 2476 fictitious = entry->object.vm_object != NULL && 2477 (entry->object.vm_object->flags & 2478 OBJ_FICTITIOUS) != 0; 2479 /* 2480 * Release the map lock, relying on the in-transition 2481 * mark. Mark the map busy for fork. 2482 */ 2483 vm_map_busy(map); 2484 vm_map_unlock(map); 2485 rv = vm_fault_wire(map, saved_start, saved_end, 2486 fictitious); 2487 vm_map_lock(map); 2488 vm_map_unbusy(map); 2489 if (last_timestamp + 1 != map->timestamp) { 2490 /* 2491 * Look again for the entry because the map was 2492 * modified while it was unlocked. The entry 2493 * may have been clipped, but NOT merged or 2494 * deleted. 2495 */ 2496 result = vm_map_lookup_entry(map, saved_start, 2497 &tmp_entry); 2498 KASSERT(result, ("vm_map_wire: lookup failed")); 2499 if (entry == first_entry) 2500 first_entry = tmp_entry; 2501 else 2502 first_entry = NULL; 2503 entry = tmp_entry; 2504 while (entry->end < saved_end) { 2505 if (rv != KERN_SUCCESS) { 2506 KASSERT(entry->wired_count == 1, 2507 ("vm_map_wire: bad count")); 2508 entry->wired_count = -1; 2509 } 2510 entry = entry->next; 2511 } 2512 } 2513 last_timestamp = map->timestamp; 2514 if (rv != KERN_SUCCESS) { 2515 KASSERT(entry->wired_count == 1, 2516 ("vm_map_wire: bad count")); 2517 /* 2518 * Assign an out-of-range value to represent 2519 * the failure to wire this entry. 2520 */ 2521 entry->wired_count = -1; 2522 end = entry->end; 2523 goto done; 2524 } 2525 } else if (!user_wire || 2526 (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { 2527 entry->wired_count++; 2528 } 2529 /* 2530 * Check the map for holes in the specified region. 2531 * If VM_MAP_WIRE_HOLESOK was specified, skip this check. 2532 */ 2533 next_entry: 2534 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) && 2535 (entry->end < end && (entry->next == &map->header || 2536 entry->next->start > entry->end))) { 2537 end = entry->end; 2538 rv = KERN_INVALID_ADDRESS; 2539 goto done; 2540 } 2541 entry = entry->next; 2542 } 2543 rv = KERN_SUCCESS; 2544 done: 2545 need_wakeup = FALSE; 2546 if (first_entry == NULL) { 2547 result = vm_map_lookup_entry(map, start, &first_entry); 2548 if (!result && (flags & VM_MAP_WIRE_HOLESOK)) 2549 first_entry = first_entry->next; 2550 else 2551 KASSERT(result, ("vm_map_wire: lookup failed")); 2552 } 2553 for (entry = first_entry; entry != &map->header && entry->start < end; 2554 entry = entry->next) { 2555 if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0) 2556 goto next_entry_done; 2557 2558 /* 2559 * If VM_MAP_WIRE_HOLESOK was specified, an empty 2560 * space in the unwired region could have been mapped 2561 * while the map lock was dropped for faulting in the 2562 * pages or draining MAP_ENTRY_IN_TRANSITION. 2563 * Moreover, another thread could be simultaneously 2564 * wiring this new mapping entry. Detect these cases 2565 * and skip any entries marked as in transition by us. 2566 */ 2567 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || 2568 entry->wiring_thread != curthread) { 2569 KASSERT((flags & VM_MAP_WIRE_HOLESOK) != 0, 2570 ("vm_map_wire: !HOLESOK and new/changed entry")); 2571 continue; 2572 } 2573 2574 if (rv == KERN_SUCCESS) { 2575 if (user_wire) 2576 entry->eflags |= MAP_ENTRY_USER_WIRED; 2577 } else if (entry->wired_count == -1) { 2578 /* 2579 * Wiring failed on this entry. Thus, unwiring is 2580 * unnecessary. 2581 */ 2582 entry->wired_count = 0; 2583 } else { 2584 if (!user_wire || 2585 (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) 2586 entry->wired_count--; 2587 if (entry->wired_count == 0) { 2588 /* 2589 * Retain the map lock. 2590 */ 2591 vm_fault_unwire(map, entry->start, entry->end, 2592 entry->object.vm_object != NULL && 2593 (entry->object.vm_object->flags & 2594 OBJ_FICTITIOUS) != 0); 2595 } 2596 } 2597 next_entry_done: 2598 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, 2599 ("vm_map_wire: in-transition flag missing %p", entry)); 2600 KASSERT(entry->wiring_thread == curthread, 2601 ("vm_map_wire: alien wire %p", entry)); 2602 entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION | 2603 MAP_ENTRY_WIRE_SKIPPED); 2604 entry->wiring_thread = NULL; 2605 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { 2606 entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; 2607 need_wakeup = TRUE; 2608 } 2609 vm_map_simplify_entry(map, entry); 2610 } 2611 vm_map_unlock(map); 2612 if (need_wakeup) 2613 vm_map_wakeup(map); 2614 return (rv); 2615 } 2616 2617 /* 2618 * vm_map_sync 2619 * 2620 * Push any dirty cached pages in the address range to their pager. 2621 * If syncio is TRUE, dirty pages are written synchronously. 2622 * If invalidate is TRUE, any cached pages are freed as well. 2623 * 2624 * If the size of the region from start to end is zero, we are 2625 * supposed to flush all modified pages within the region containing 2626 * start. Unfortunately, a region can be split or coalesced with 2627 * neighboring regions, making it difficult to determine what the 2628 * original region was. Therefore, we approximate this requirement by 2629 * flushing the current region containing start. 2630 * 2631 * Returns an error if any part of the specified range is not mapped. 2632 */ 2633 int 2634 vm_map_sync( 2635 vm_map_t map, 2636 vm_offset_t start, 2637 vm_offset_t end, 2638 boolean_t syncio, 2639 boolean_t invalidate) 2640 { 2641 vm_map_entry_t current; 2642 vm_map_entry_t entry; 2643 vm_size_t size; 2644 vm_object_t object; 2645 vm_ooffset_t offset; 2646 unsigned int last_timestamp; 2647 boolean_t failed; 2648 2649 vm_map_lock_read(map); 2650 VM_MAP_RANGE_CHECK(map, start, end); 2651 if (!vm_map_lookup_entry(map, start, &entry)) { 2652 vm_map_unlock_read(map); 2653 return (KERN_INVALID_ADDRESS); 2654 } else if (start == end) { 2655 start = entry->start; 2656 end = entry->end; 2657 } 2658 /* 2659 * Make a first pass to check for user-wired memory and holes. 2660 */ 2661 for (current = entry; current != &map->header && current->start < end; 2662 current = current->next) { 2663 if (invalidate && (current->eflags & MAP_ENTRY_USER_WIRED)) { 2664 vm_map_unlock_read(map); 2665 return (KERN_INVALID_ARGUMENT); 2666 } 2667 if (end > current->end && 2668 (current->next == &map->header || 2669 current->end != current->next->start)) { 2670 vm_map_unlock_read(map); 2671 return (KERN_INVALID_ADDRESS); 2672 } 2673 } 2674 2675 if (invalidate) 2676 pmap_remove(map->pmap, start, end); 2677 failed = FALSE; 2678 2679 /* 2680 * Make a second pass, cleaning/uncaching pages from the indicated 2681 * objects as we go. 2682 */ 2683 for (current = entry; current != &map->header && current->start < end;) { 2684 offset = current->offset + (start - current->start); 2685 size = (end <= current->end ? end : current->end) - start; 2686 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) { 2687 vm_map_t smap; 2688 vm_map_entry_t tentry; 2689 vm_size_t tsize; 2690 2691 smap = current->object.sub_map; 2692 vm_map_lock_read(smap); 2693 (void) vm_map_lookup_entry(smap, offset, &tentry); 2694 tsize = tentry->end - offset; 2695 if (tsize < size) 2696 size = tsize; 2697 object = tentry->object.vm_object; 2698 offset = tentry->offset + (offset - tentry->start); 2699 vm_map_unlock_read(smap); 2700 } else { 2701 object = current->object.vm_object; 2702 } 2703 vm_object_reference(object); 2704 last_timestamp = map->timestamp; 2705 vm_map_unlock_read(map); 2706 if (!vm_object_sync(object, offset, size, syncio, invalidate)) 2707 failed = TRUE; 2708 start += size; 2709 vm_object_deallocate(object); 2710 vm_map_lock_read(map); 2711 if (last_timestamp == map->timestamp || 2712 !vm_map_lookup_entry(map, start, ¤t)) 2713 current = current->next; 2714 } 2715 2716 vm_map_unlock_read(map); 2717 return (failed ? KERN_FAILURE : KERN_SUCCESS); 2718 } 2719 2720 /* 2721 * vm_map_entry_unwire: [ internal use only ] 2722 * 2723 * Make the region specified by this entry pageable. 2724 * 2725 * The map in question should be locked. 2726 * [This is the reason for this routine's existence.] 2727 */ 2728 static void 2729 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) 2730 { 2731 vm_fault_unwire(map, entry->start, entry->end, 2732 entry->object.vm_object != NULL && 2733 (entry->object.vm_object->flags & OBJ_FICTITIOUS) != 0); 2734 entry->wired_count = 0; 2735 } 2736 2737 static void 2738 vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map) 2739 { 2740 2741 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) 2742 vm_object_deallocate(entry->object.vm_object); 2743 uma_zfree(system_map ? kmapentzone : mapentzone, entry); 2744 } 2745 2746 /* 2747 * vm_map_entry_delete: [ internal use only ] 2748 * 2749 * Deallocate the given entry from the target map. 2750 */ 2751 static void 2752 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry) 2753 { 2754 vm_object_t object; 2755 vm_pindex_t offidxstart, offidxend, count, size1; 2756 vm_ooffset_t size; 2757 2758 vm_map_entry_unlink(map, entry); 2759 object = entry->object.vm_object; 2760 size = entry->end - entry->start; 2761 map->size -= size; 2762 2763 if (entry->cred != NULL) { 2764 swap_release_by_cred(size, entry->cred); 2765 crfree(entry->cred); 2766 } 2767 2768 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 && 2769 (object != NULL)) { 2770 KASSERT(entry->cred == NULL || object->cred == NULL || 2771 (entry->eflags & MAP_ENTRY_NEEDS_COPY), 2772 ("OVERCOMMIT vm_map_entry_delete: both cred %p", entry)); 2773 count = OFF_TO_IDX(size); 2774 offidxstart = OFF_TO_IDX(entry->offset); 2775 offidxend = offidxstart + count; 2776 VM_OBJECT_WLOCK(object); 2777 if (object->ref_count != 1 && 2778 ((object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING || 2779 object == kernel_object || object == kmem_object)) { 2780 vm_object_collapse(object); 2781 2782 /* 2783 * The option OBJPR_NOTMAPPED can be passed here 2784 * because vm_map_delete() already performed 2785 * pmap_remove() on the only mapping to this range 2786 * of pages. 2787 */ 2788 vm_object_page_remove(object, offidxstart, offidxend, 2789 OBJPR_NOTMAPPED); 2790 if (object->type == OBJT_SWAP) 2791 swap_pager_freespace(object, offidxstart, count); 2792 if (offidxend >= object->size && 2793 offidxstart < object->size) { 2794 size1 = object->size; 2795 object->size = offidxstart; 2796 if (object->cred != NULL) { 2797 size1 -= object->size; 2798 KASSERT(object->charge >= ptoa(size1), 2799 ("vm_map_entry_delete: object->charge < 0")); 2800 swap_release_by_cred(ptoa(size1), object->cred); 2801 object->charge -= ptoa(size1); 2802 } 2803 } 2804 } 2805 VM_OBJECT_WUNLOCK(object); 2806 } else 2807 entry->object.vm_object = NULL; 2808 if (map->system_map) 2809 vm_map_entry_deallocate(entry, TRUE); 2810 else { 2811 entry->next = curthread->td_map_def_user; 2812 curthread->td_map_def_user = entry; 2813 } 2814 } 2815 2816 /* 2817 * vm_map_delete: [ internal use only ] 2818 * 2819 * Deallocates the given address range from the target 2820 * map. 2821 */ 2822 int 2823 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) 2824 { 2825 vm_map_entry_t entry; 2826 vm_map_entry_t first_entry; 2827 2828 VM_MAP_ASSERT_LOCKED(map); 2829 2830 /* 2831 * Find the start of the region, and clip it 2832 */ 2833 if (!vm_map_lookup_entry(map, start, &first_entry)) 2834 entry = first_entry->next; 2835 else { 2836 entry = first_entry; 2837 vm_map_clip_start(map, entry, start); 2838 } 2839 2840 /* 2841 * Step through all entries in this region 2842 */ 2843 while ((entry != &map->header) && (entry->start < end)) { 2844 vm_map_entry_t next; 2845 2846 /* 2847 * Wait for wiring or unwiring of an entry to complete. 2848 * Also wait for any system wirings to disappear on 2849 * user maps. 2850 */ 2851 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 || 2852 (vm_map_pmap(map) != kernel_pmap && 2853 vm_map_entry_system_wired_count(entry) != 0)) { 2854 unsigned int last_timestamp; 2855 vm_offset_t saved_start; 2856 vm_map_entry_t tmp_entry; 2857 2858 saved_start = entry->start; 2859 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; 2860 last_timestamp = map->timestamp; 2861 (void) vm_map_unlock_and_wait(map, 0); 2862 vm_map_lock(map); 2863 if (last_timestamp + 1 != map->timestamp) { 2864 /* 2865 * Look again for the entry because the map was 2866 * modified while it was unlocked. 2867 * Specifically, the entry may have been 2868 * clipped, merged, or deleted. 2869 */ 2870 if (!vm_map_lookup_entry(map, saved_start, 2871 &tmp_entry)) 2872 entry = tmp_entry->next; 2873 else { 2874 entry = tmp_entry; 2875 vm_map_clip_start(map, entry, 2876 saved_start); 2877 } 2878 } 2879 continue; 2880 } 2881 vm_map_clip_end(map, entry, end); 2882 2883 next = entry->next; 2884 2885 /* 2886 * Unwire before removing addresses from the pmap; otherwise, 2887 * unwiring will put the entries back in the pmap. 2888 */ 2889 if (entry->wired_count != 0) { 2890 vm_map_entry_unwire(map, entry); 2891 } 2892 2893 pmap_remove(map->pmap, entry->start, entry->end); 2894 2895 /* 2896 * Delete the entry only after removing all pmap 2897 * entries pointing to its pages. (Otherwise, its 2898 * page frames may be reallocated, and any modify bits 2899 * will be set in the wrong object!) 2900 */ 2901 vm_map_entry_delete(map, entry); 2902 entry = next; 2903 } 2904 return (KERN_SUCCESS); 2905 } 2906 2907 /* 2908 * vm_map_remove: 2909 * 2910 * Remove the given address range from the target map. 2911 * This is the exported form of vm_map_delete. 2912 */ 2913 int 2914 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end) 2915 { 2916 int result; 2917 2918 vm_map_lock(map); 2919 VM_MAP_RANGE_CHECK(map, start, end); 2920 result = vm_map_delete(map, start, end); 2921 vm_map_unlock(map); 2922 return (result); 2923 } 2924 2925 /* 2926 * vm_map_check_protection: 2927 * 2928 * Assert that the target map allows the specified privilege on the 2929 * entire address region given. The entire region must be allocated. 2930 * 2931 * WARNING! This code does not and should not check whether the 2932 * contents of the region is accessible. For example a smaller file 2933 * might be mapped into a larger address space. 2934 * 2935 * NOTE! This code is also called by munmap(). 2936 * 2937 * The map must be locked. A read lock is sufficient. 2938 */ 2939 boolean_t 2940 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end, 2941 vm_prot_t protection) 2942 { 2943 vm_map_entry_t entry; 2944 vm_map_entry_t tmp_entry; 2945 2946 if (!vm_map_lookup_entry(map, start, &tmp_entry)) 2947 return (FALSE); 2948 entry = tmp_entry; 2949 2950 while (start < end) { 2951 if (entry == &map->header) 2952 return (FALSE); 2953 /* 2954 * No holes allowed! 2955 */ 2956 if (start < entry->start) 2957 return (FALSE); 2958 /* 2959 * Check protection associated with entry. 2960 */ 2961 if ((entry->protection & protection) != protection) 2962 return (FALSE); 2963 /* go to next entry */ 2964 start = entry->end; 2965 entry = entry->next; 2966 } 2967 return (TRUE); 2968 } 2969 2970 /* 2971 * vm_map_copy_entry: 2972 * 2973 * Copies the contents of the source entry to the destination 2974 * entry. The entries *must* be aligned properly. 2975 */ 2976 static void 2977 vm_map_copy_entry( 2978 vm_map_t src_map, 2979 vm_map_t dst_map, 2980 vm_map_entry_t src_entry, 2981 vm_map_entry_t dst_entry, 2982 vm_ooffset_t *fork_charge) 2983 { 2984 vm_object_t src_object; 2985 vm_map_entry_t fake_entry; 2986 vm_offset_t size; 2987 struct ucred *cred; 2988 int charged; 2989 2990 VM_MAP_ASSERT_LOCKED(dst_map); 2991 2992 if ((dst_entry->eflags|src_entry->eflags) & MAP_ENTRY_IS_SUB_MAP) 2993 return; 2994 2995 if (src_entry->wired_count == 0) { 2996 2997 /* 2998 * If the source entry is marked needs_copy, it is already 2999 * write-protected. 3000 */ 3001 if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) { 3002 pmap_protect(src_map->pmap, 3003 src_entry->start, 3004 src_entry->end, 3005 src_entry->protection & ~VM_PROT_WRITE); 3006 } 3007 3008 /* 3009 * Make a copy of the object. 3010 */ 3011 size = src_entry->end - src_entry->start; 3012 if ((src_object = src_entry->object.vm_object) != NULL) { 3013 VM_OBJECT_WLOCK(src_object); 3014 charged = ENTRY_CHARGED(src_entry); 3015 if ((src_object->handle == NULL) && 3016 (src_object->type == OBJT_DEFAULT || 3017 src_object->type == OBJT_SWAP)) { 3018 vm_object_collapse(src_object); 3019 if ((src_object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING) { 3020 vm_object_split(src_entry); 3021 src_object = src_entry->object.vm_object; 3022 } 3023 } 3024 vm_object_reference_locked(src_object); 3025 vm_object_clear_flag(src_object, OBJ_ONEMAPPING); 3026 if (src_entry->cred != NULL && 3027 !(src_entry->eflags & MAP_ENTRY_NEEDS_COPY)) { 3028 KASSERT(src_object->cred == NULL, 3029 ("OVERCOMMIT: vm_map_copy_entry: cred %p", 3030 src_object)); 3031 src_object->cred = src_entry->cred; 3032 src_object->charge = size; 3033 } 3034 VM_OBJECT_WUNLOCK(src_object); 3035 dst_entry->object.vm_object = src_object; 3036 if (charged) { 3037 cred = curthread->td_ucred; 3038 crhold(cred); 3039 dst_entry->cred = cred; 3040 *fork_charge += size; 3041 if (!(src_entry->eflags & 3042 MAP_ENTRY_NEEDS_COPY)) { 3043 crhold(cred); 3044 src_entry->cred = cred; 3045 *fork_charge += size; 3046 } 3047 } 3048 src_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY); 3049 dst_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY); 3050 dst_entry->offset = src_entry->offset; 3051 if (src_entry->eflags & MAP_ENTRY_VN_WRITECNT) { 3052 /* 3053 * MAP_ENTRY_VN_WRITECNT cannot 3054 * indicate write reference from 3055 * src_entry, since the entry is 3056 * marked as needs copy. Allocate a 3057 * fake entry that is used to 3058 * decrement object->un_pager.vnp.writecount 3059 * at the appropriate time. Attach 3060 * fake_entry to the deferred list. 3061 */ 3062 fake_entry = vm_map_entry_create(dst_map); 3063 fake_entry->eflags = MAP_ENTRY_VN_WRITECNT; 3064 src_entry->eflags &= ~MAP_ENTRY_VN_WRITECNT; 3065 vm_object_reference(src_object); 3066 fake_entry->object.vm_object = src_object; 3067 fake_entry->start = src_entry->start; 3068 fake_entry->end = src_entry->end; 3069 fake_entry->next = curthread->td_map_def_user; 3070 curthread->td_map_def_user = fake_entry; 3071 } 3072 } else { 3073 dst_entry->object.vm_object = NULL; 3074 dst_entry->offset = 0; 3075 if (src_entry->cred != NULL) { 3076 dst_entry->cred = curthread->td_ucred; 3077 crhold(dst_entry->cred); 3078 *fork_charge += size; 3079 } 3080 } 3081 3082 pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start, 3083 dst_entry->end - dst_entry->start, src_entry->start); 3084 } else { 3085 /* 3086 * Of course, wired down pages can't be set copy-on-write. 3087 * Cause wired pages to be copied into the new map by 3088 * simulating faults (the new pages are pageable) 3089 */ 3090 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry, 3091 fork_charge); 3092 } 3093 } 3094 3095 /* 3096 * vmspace_map_entry_forked: 3097 * Update the newly-forked vmspace each time a map entry is inherited 3098 * or copied. The values for vm_dsize and vm_tsize are approximate 3099 * (and mostly-obsolete ideas in the face of mmap(2) et al.) 3100 */ 3101 static void 3102 vmspace_map_entry_forked(const struct vmspace *vm1, struct vmspace *vm2, 3103 vm_map_entry_t entry) 3104 { 3105 vm_size_t entrysize; 3106 vm_offset_t newend; 3107 3108 entrysize = entry->end - entry->start; 3109 vm2->vm_map.size += entrysize; 3110 if (entry->eflags & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) { 3111 vm2->vm_ssize += btoc(entrysize); 3112 } else if (entry->start >= (vm_offset_t)vm1->vm_daddr && 3113 entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) { 3114 newend = MIN(entry->end, 3115 (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)); 3116 vm2->vm_dsize += btoc(newend - entry->start); 3117 } else if (entry->start >= (vm_offset_t)vm1->vm_taddr && 3118 entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) { 3119 newend = MIN(entry->end, 3120 (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)); 3121 vm2->vm_tsize += btoc(newend - entry->start); 3122 } 3123 } 3124 3125 /* 3126 * vmspace_fork: 3127 * Create a new process vmspace structure and vm_map 3128 * based on those of an existing process. The new map 3129 * is based on the old map, according to the inheritance 3130 * values on the regions in that map. 3131 * 3132 * XXX It might be worth coalescing the entries added to the new vmspace. 3133 * 3134 * The source map must not be locked. 3135 */ 3136 struct vmspace * 3137 vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge) 3138 { 3139 struct vmspace *vm2; 3140 vm_map_t new_map, old_map; 3141 vm_map_entry_t new_entry, old_entry; 3142 vm_object_t object; 3143 int locked; 3144 3145 old_map = &vm1->vm_map; 3146 /* Copy immutable fields of vm1 to vm2. */ 3147 vm2 = vmspace_alloc(old_map->min_offset, old_map->max_offset); 3148 if (vm2 == NULL) 3149 return (NULL); 3150 vm2->vm_taddr = vm1->vm_taddr; 3151 vm2->vm_daddr = vm1->vm_daddr; 3152 vm2->vm_maxsaddr = vm1->vm_maxsaddr; 3153 vm_map_lock(old_map); 3154 if (old_map->busy) 3155 vm_map_wait_busy(old_map); 3156 new_map = &vm2->vm_map; 3157 locked = vm_map_trylock(new_map); /* trylock to silence WITNESS */ 3158 KASSERT(locked, ("vmspace_fork: lock failed")); 3159 3160 old_entry = old_map->header.next; 3161 3162 while (old_entry != &old_map->header) { 3163 if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP) 3164 panic("vm_map_fork: encountered a submap"); 3165 3166 switch (old_entry->inheritance) { 3167 case VM_INHERIT_NONE: 3168 break; 3169 3170 case VM_INHERIT_SHARE: 3171 /* 3172 * Clone the entry, creating the shared object if necessary. 3173 */ 3174 object = old_entry->object.vm_object; 3175 if (object == NULL) { 3176 object = vm_object_allocate(OBJT_DEFAULT, 3177 atop(old_entry->end - old_entry->start)); 3178 old_entry->object.vm_object = object; 3179 old_entry->offset = 0; 3180 if (old_entry->cred != NULL) { 3181 object->cred = old_entry->cred; 3182 object->charge = old_entry->end - 3183 old_entry->start; 3184 old_entry->cred = NULL; 3185 } 3186 } 3187 3188 /* 3189 * Add the reference before calling vm_object_shadow 3190 * to insure that a shadow object is created. 3191 */ 3192 vm_object_reference(object); 3193 if (old_entry->eflags & MAP_ENTRY_NEEDS_COPY) { 3194 vm_object_shadow(&old_entry->object.vm_object, 3195 &old_entry->offset, 3196 old_entry->end - old_entry->start); 3197 old_entry->eflags &= ~MAP_ENTRY_NEEDS_COPY; 3198 /* Transfer the second reference too. */ 3199 vm_object_reference( 3200 old_entry->object.vm_object); 3201 3202 /* 3203 * As in vm_map_simplify_entry(), the 3204 * vnode lock will not be acquired in 3205 * this call to vm_object_deallocate(). 3206 */ 3207 vm_object_deallocate(object); 3208 object = old_entry->object.vm_object; 3209 } 3210 VM_OBJECT_WLOCK(object); 3211 vm_object_clear_flag(object, OBJ_ONEMAPPING); 3212 if (old_entry->cred != NULL) { 3213 KASSERT(object->cred == NULL, ("vmspace_fork both cred")); 3214 object->cred = old_entry->cred; 3215 object->charge = old_entry->end - old_entry->start; 3216 old_entry->cred = NULL; 3217 } 3218 3219 /* 3220 * Assert the correct state of the vnode 3221 * v_writecount while the object is locked, to 3222 * not relock it later for the assertion 3223 * correctness. 3224 */ 3225 if (old_entry->eflags & MAP_ENTRY_VN_WRITECNT && 3226 object->type == OBJT_VNODE) { 3227 KASSERT(((struct vnode *)object->handle)-> 3228 v_writecount > 0, 3229 ("vmspace_fork: v_writecount %p", object)); 3230 KASSERT(object->un_pager.vnp.writemappings > 0, 3231 ("vmspace_fork: vnp.writecount %p", 3232 object)); 3233 } 3234 VM_OBJECT_WUNLOCK(object); 3235 3236 /* 3237 * Clone the entry, referencing the shared object. 3238 */ 3239 new_entry = vm_map_entry_create(new_map); 3240 *new_entry = *old_entry; 3241 new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED | 3242 MAP_ENTRY_IN_TRANSITION); 3243 new_entry->wiring_thread = NULL; 3244 new_entry->wired_count = 0; 3245 if (new_entry->eflags & MAP_ENTRY_VN_WRITECNT) { 3246 vnode_pager_update_writecount(object, 3247 new_entry->start, new_entry->end); 3248 } 3249 3250 /* 3251 * Insert the entry into the new map -- we know we're 3252 * inserting at the end of the new map. 3253 */ 3254 vm_map_entry_link(new_map, new_map->header.prev, 3255 new_entry); 3256 vmspace_map_entry_forked(vm1, vm2, new_entry); 3257 3258 /* 3259 * Update the physical map 3260 */ 3261 pmap_copy(new_map->pmap, old_map->pmap, 3262 new_entry->start, 3263 (old_entry->end - old_entry->start), 3264 old_entry->start); 3265 break; 3266 3267 case VM_INHERIT_COPY: 3268 /* 3269 * Clone the entry and link into the map. 3270 */ 3271 new_entry = vm_map_entry_create(new_map); 3272 *new_entry = *old_entry; 3273 /* 3274 * Copied entry is COW over the old object. 3275 */ 3276 new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED | 3277 MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_VN_WRITECNT); 3278 new_entry->wiring_thread = NULL; 3279 new_entry->wired_count = 0; 3280 new_entry->object.vm_object = NULL; 3281 new_entry->cred = NULL; 3282 vm_map_entry_link(new_map, new_map->header.prev, 3283 new_entry); 3284 vmspace_map_entry_forked(vm1, vm2, new_entry); 3285 vm_map_copy_entry(old_map, new_map, old_entry, 3286 new_entry, fork_charge); 3287 break; 3288 } 3289 old_entry = old_entry->next; 3290 } 3291 /* 3292 * Use inlined vm_map_unlock() to postpone handling the deferred 3293 * map entries, which cannot be done until both old_map and 3294 * new_map locks are released. 3295 */ 3296 sx_xunlock(&old_map->lock); 3297 sx_xunlock(&new_map->lock); 3298 vm_map_process_deferred(); 3299 3300 return (vm2); 3301 } 3302 3303 int 3304 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, 3305 vm_prot_t prot, vm_prot_t max, int cow) 3306 { 3307 vm_map_entry_t new_entry, prev_entry; 3308 vm_offset_t bot, top; 3309 vm_size_t growsize, init_ssize; 3310 int orient, rv; 3311 rlim_t lmemlim, vmemlim; 3312 3313 /* 3314 * The stack orientation is piggybacked with the cow argument. 3315 * Extract it into orient and mask the cow argument so that we 3316 * don't pass it around further. 3317 * NOTE: We explicitly allow bi-directional stacks. 3318 */ 3319 orient = cow & (MAP_STACK_GROWS_DOWN|MAP_STACK_GROWS_UP); 3320 cow &= ~orient; 3321 KASSERT(orient != 0, ("No stack grow direction")); 3322 3323 if (addrbos < vm_map_min(map) || 3324 addrbos > vm_map_max(map) || 3325 addrbos + max_ssize < addrbos) 3326 return (KERN_NO_SPACE); 3327 3328 growsize = sgrowsiz; 3329 init_ssize = (max_ssize < growsize) ? max_ssize : growsize; 3330 3331 PROC_LOCK(curproc); 3332 lmemlim = lim_cur(curproc, RLIMIT_MEMLOCK); 3333 vmemlim = lim_cur(curproc, RLIMIT_VMEM); 3334 PROC_UNLOCK(curproc); 3335 3336 vm_map_lock(map); 3337 3338 /* If addr is already mapped, no go */ 3339 if (vm_map_lookup_entry(map, addrbos, &prev_entry)) { 3340 vm_map_unlock(map); 3341 return (KERN_NO_SPACE); 3342 } 3343 3344 if (!old_mlock && map->flags & MAP_WIREFUTURE) { 3345 if (ptoa(pmap_wired_count(map->pmap)) + init_ssize > lmemlim) { 3346 vm_map_unlock(map); 3347 return (KERN_NO_SPACE); 3348 } 3349 } 3350 3351 /* If we would blow our VMEM resource limit, no go */ 3352 if (map->size + init_ssize > vmemlim) { 3353 vm_map_unlock(map); 3354 return (KERN_NO_SPACE); 3355 } 3356 3357 /* 3358 * If we can't accomodate max_ssize in the current mapping, no go. 3359 * However, we need to be aware that subsequent user mappings might 3360 * map into the space we have reserved for stack, and currently this 3361 * space is not protected. 3362 * 3363 * Hopefully we will at least detect this condition when we try to 3364 * grow the stack. 3365 */ 3366 if ((prev_entry->next != &map->header) && 3367 (prev_entry->next->start < addrbos + max_ssize)) { 3368 vm_map_unlock(map); 3369 return (KERN_NO_SPACE); 3370 } 3371 3372 /* 3373 * We initially map a stack of only init_ssize. We will grow as 3374 * needed later. Depending on the orientation of the stack (i.e. 3375 * the grow direction) we either map at the top of the range, the 3376 * bottom of the range or in the middle. 3377 * 3378 * Note: we would normally expect prot and max to be VM_PROT_ALL, 3379 * and cow to be 0. Possibly we should eliminate these as input 3380 * parameters, and just pass these values here in the insert call. 3381 */ 3382 if (orient == MAP_STACK_GROWS_DOWN) 3383 bot = addrbos + max_ssize - init_ssize; 3384 else if (orient == MAP_STACK_GROWS_UP) 3385 bot = addrbos; 3386 else 3387 bot = round_page(addrbos + max_ssize/2 - init_ssize/2); 3388 top = bot + init_ssize; 3389 rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow); 3390 3391 /* Now set the avail_ssize amount. */ 3392 if (rv == KERN_SUCCESS) { 3393 if (prev_entry != &map->header) 3394 vm_map_clip_end(map, prev_entry, bot); 3395 new_entry = prev_entry->next; 3396 if (new_entry->end != top || new_entry->start != bot) 3397 panic("Bad entry start/end for new stack entry"); 3398 3399 new_entry->avail_ssize = max_ssize - init_ssize; 3400 if (orient & MAP_STACK_GROWS_DOWN) 3401 new_entry->eflags |= MAP_ENTRY_GROWS_DOWN; 3402 if (orient & MAP_STACK_GROWS_UP) 3403 new_entry->eflags |= MAP_ENTRY_GROWS_UP; 3404 } 3405 3406 vm_map_unlock(map); 3407 return (rv); 3408 } 3409 3410 static int stack_guard_page = 0; 3411 TUNABLE_INT("security.bsd.stack_guard_page", &stack_guard_page); 3412 SYSCTL_INT(_security_bsd, OID_AUTO, stack_guard_page, CTLFLAG_RW, 3413 &stack_guard_page, 0, 3414 "Insert stack guard page ahead of the growable segments."); 3415 3416 /* Attempts to grow a vm stack entry. Returns KERN_SUCCESS if the 3417 * desired address is already mapped, or if we successfully grow 3418 * the stack. Also returns KERN_SUCCESS if addr is outside the 3419 * stack range (this is strange, but preserves compatibility with 3420 * the grow function in vm_machdep.c). 3421 */ 3422 int 3423 vm_map_growstack(struct proc *p, vm_offset_t addr) 3424 { 3425 vm_map_entry_t next_entry, prev_entry; 3426 vm_map_entry_t new_entry, stack_entry; 3427 struct vmspace *vm = p->p_vmspace; 3428 vm_map_t map = &vm->vm_map; 3429 vm_offset_t end; 3430 vm_size_t growsize; 3431 size_t grow_amount, max_grow; 3432 rlim_t lmemlim, stacklim, vmemlim; 3433 int is_procstack, rv; 3434 struct ucred *cred; 3435 #ifdef notyet 3436 uint64_t limit; 3437 #endif 3438 #ifdef RACCT 3439 int error; 3440 #endif 3441 3442 Retry: 3443 PROC_LOCK(p); 3444 lmemlim = lim_cur(p, RLIMIT_MEMLOCK); 3445 stacklim = lim_cur(p, RLIMIT_STACK); 3446 vmemlim = lim_cur(p, RLIMIT_VMEM); 3447 PROC_UNLOCK(p); 3448 3449 vm_map_lock_read(map); 3450 3451 /* If addr is already in the entry range, no need to grow.*/ 3452 if (vm_map_lookup_entry(map, addr, &prev_entry)) { 3453 vm_map_unlock_read(map); 3454 return (KERN_SUCCESS); 3455 } 3456 3457 next_entry = prev_entry->next; 3458 if (!(prev_entry->eflags & MAP_ENTRY_GROWS_UP)) { 3459 /* 3460 * This entry does not grow upwards. Since the address lies 3461 * beyond this entry, the next entry (if one exists) has to 3462 * be a downward growable entry. The entry list header is 3463 * never a growable entry, so it suffices to check the flags. 3464 */ 3465 if (!(next_entry->eflags & MAP_ENTRY_GROWS_DOWN)) { 3466 vm_map_unlock_read(map); 3467 return (KERN_SUCCESS); 3468 } 3469 stack_entry = next_entry; 3470 } else { 3471 /* 3472 * This entry grows upward. If the next entry does not at 3473 * least grow downwards, this is the entry we need to grow. 3474 * otherwise we have two possible choices and we have to 3475 * select one. 3476 */ 3477 if (next_entry->eflags & MAP_ENTRY_GROWS_DOWN) { 3478 /* 3479 * We have two choices; grow the entry closest to 3480 * the address to minimize the amount of growth. 3481 */ 3482 if (addr - prev_entry->end <= next_entry->start - addr) 3483 stack_entry = prev_entry; 3484 else 3485 stack_entry = next_entry; 3486 } else 3487 stack_entry = prev_entry; 3488 } 3489 3490 if (stack_entry == next_entry) { 3491 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_DOWN, ("foo")); 3492 KASSERT(addr < stack_entry->start, ("foo")); 3493 end = (prev_entry != &map->header) ? prev_entry->end : 3494 stack_entry->start - stack_entry->avail_ssize; 3495 grow_amount = roundup(stack_entry->start - addr, PAGE_SIZE); 3496 max_grow = stack_entry->start - end; 3497 } else { 3498 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_UP, ("foo")); 3499 KASSERT(addr >= stack_entry->end, ("foo")); 3500 end = (next_entry != &map->header) ? next_entry->start : 3501 stack_entry->end + stack_entry->avail_ssize; 3502 grow_amount = roundup(addr + 1 - stack_entry->end, PAGE_SIZE); 3503 max_grow = end - stack_entry->end; 3504 } 3505 3506 if (grow_amount > stack_entry->avail_ssize) { 3507 vm_map_unlock_read(map); 3508 return (KERN_NO_SPACE); 3509 } 3510 3511 /* 3512 * If there is no longer enough space between the entries nogo, and 3513 * adjust the available space. Note: this should only happen if the 3514 * user has mapped into the stack area after the stack was created, 3515 * and is probably an error. 3516 * 3517 * This also effectively destroys any guard page the user might have 3518 * intended by limiting the stack size. 3519 */ 3520 if (grow_amount + (stack_guard_page ? PAGE_SIZE : 0) > max_grow) { 3521 if (vm_map_lock_upgrade(map)) 3522 goto Retry; 3523 3524 stack_entry->avail_ssize = max_grow; 3525 3526 vm_map_unlock(map); 3527 return (KERN_NO_SPACE); 3528 } 3529 3530 is_procstack = (addr >= (vm_offset_t)vm->vm_maxsaddr) ? 1 : 0; 3531 3532 /* 3533 * If this is the main process stack, see if we're over the stack 3534 * limit. 3535 */ 3536 if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) { 3537 vm_map_unlock_read(map); 3538 return (KERN_NO_SPACE); 3539 } 3540 #ifdef RACCT 3541 PROC_LOCK(p); 3542 if (is_procstack && 3543 racct_set(p, RACCT_STACK, ctob(vm->vm_ssize) + grow_amount)) { 3544 PROC_UNLOCK(p); 3545 vm_map_unlock_read(map); 3546 return (KERN_NO_SPACE); 3547 } 3548 PROC_UNLOCK(p); 3549 #endif 3550 3551 /* Round up the grow amount modulo sgrowsiz */ 3552 growsize = sgrowsiz; 3553 grow_amount = roundup(grow_amount, growsize); 3554 if (grow_amount > stack_entry->avail_ssize) 3555 grow_amount = stack_entry->avail_ssize; 3556 if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) { 3557 grow_amount = trunc_page((vm_size_t)stacklim) - 3558 ctob(vm->vm_ssize); 3559 } 3560 #ifdef notyet 3561 PROC_LOCK(p); 3562 limit = racct_get_available(p, RACCT_STACK); 3563 PROC_UNLOCK(p); 3564 if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > limit)) 3565 grow_amount = limit - ctob(vm->vm_ssize); 3566 #endif 3567 if (!old_mlock && map->flags & MAP_WIREFUTURE) { 3568 if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) { 3569 vm_map_unlock_read(map); 3570 rv = KERN_NO_SPACE; 3571 goto out; 3572 } 3573 #ifdef RACCT 3574 PROC_LOCK(p); 3575 if (racct_set(p, RACCT_MEMLOCK, 3576 ptoa(pmap_wired_count(map->pmap)) + grow_amount)) { 3577 PROC_UNLOCK(p); 3578 vm_map_unlock_read(map); 3579 rv = KERN_NO_SPACE; 3580 goto out; 3581 } 3582 PROC_UNLOCK(p); 3583 #endif 3584 } 3585 /* If we would blow our VMEM resource limit, no go */ 3586 if (map->size + grow_amount > vmemlim) { 3587 vm_map_unlock_read(map); 3588 rv = KERN_NO_SPACE; 3589 goto out; 3590 } 3591 #ifdef RACCT 3592 PROC_LOCK(p); 3593 if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) { 3594 PROC_UNLOCK(p); 3595 vm_map_unlock_read(map); 3596 rv = KERN_NO_SPACE; 3597 goto out; 3598 } 3599 PROC_UNLOCK(p); 3600 #endif 3601 3602 if (vm_map_lock_upgrade(map)) 3603 goto Retry; 3604 3605 if (stack_entry == next_entry) { 3606 /* 3607 * Growing downward. 3608 */ 3609 /* Get the preliminary new entry start value */ 3610 addr = stack_entry->start - grow_amount; 3611 3612 /* 3613 * If this puts us into the previous entry, cut back our 3614 * growth to the available space. Also, see the note above. 3615 */ 3616 if (addr < end) { 3617 stack_entry->avail_ssize = max_grow; 3618 addr = end; 3619 if (stack_guard_page) 3620 addr += PAGE_SIZE; 3621 } 3622 3623 rv = vm_map_insert(map, NULL, 0, addr, stack_entry->start, 3624 next_entry->protection, next_entry->max_protection, 0); 3625 3626 /* Adjust the available stack space by the amount we grew. */ 3627 if (rv == KERN_SUCCESS) { 3628 if (prev_entry != &map->header) 3629 vm_map_clip_end(map, prev_entry, addr); 3630 new_entry = prev_entry->next; 3631 KASSERT(new_entry == stack_entry->prev, ("foo")); 3632 KASSERT(new_entry->end == stack_entry->start, ("foo")); 3633 KASSERT(new_entry->start == addr, ("foo")); 3634 grow_amount = new_entry->end - new_entry->start; 3635 new_entry->avail_ssize = stack_entry->avail_ssize - 3636 grow_amount; 3637 stack_entry->eflags &= ~MAP_ENTRY_GROWS_DOWN; 3638 new_entry->eflags |= MAP_ENTRY_GROWS_DOWN; 3639 } 3640 } else { 3641 /* 3642 * Growing upward. 3643 */ 3644 addr = stack_entry->end + grow_amount; 3645 3646 /* 3647 * If this puts us into the next entry, cut back our growth 3648 * to the available space. Also, see the note above. 3649 */ 3650 if (addr > end) { 3651 stack_entry->avail_ssize = end - stack_entry->end; 3652 addr = end; 3653 if (stack_guard_page) 3654 addr -= PAGE_SIZE; 3655 } 3656 3657 grow_amount = addr - stack_entry->end; 3658 cred = stack_entry->cred; 3659 if (cred == NULL && stack_entry->object.vm_object != NULL) 3660 cred = stack_entry->object.vm_object->cred; 3661 if (cred != NULL && !swap_reserve_by_cred(grow_amount, cred)) 3662 rv = KERN_NO_SPACE; 3663 /* Grow the underlying object if applicable. */ 3664 else if (stack_entry->object.vm_object == NULL || 3665 vm_object_coalesce(stack_entry->object.vm_object, 3666 stack_entry->offset, 3667 (vm_size_t)(stack_entry->end - stack_entry->start), 3668 (vm_size_t)grow_amount, cred != NULL)) { 3669 map->size += (addr - stack_entry->end); 3670 /* Update the current entry. */ 3671 stack_entry->end = addr; 3672 stack_entry->avail_ssize -= grow_amount; 3673 vm_map_entry_resize_free(map, stack_entry); 3674 rv = KERN_SUCCESS; 3675 3676 if (next_entry != &map->header) 3677 vm_map_clip_start(map, next_entry, addr); 3678 } else 3679 rv = KERN_FAILURE; 3680 } 3681 3682 if (rv == KERN_SUCCESS && is_procstack) 3683 vm->vm_ssize += btoc(grow_amount); 3684 3685 vm_map_unlock(map); 3686 3687 /* 3688 * Heed the MAP_WIREFUTURE flag if it was set for this process. 3689 */ 3690 if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE)) { 3691 vm_map_wire(map, 3692 (stack_entry == next_entry) ? addr : addr - grow_amount, 3693 (stack_entry == next_entry) ? stack_entry->start : addr, 3694 (p->p_flag & P_SYSTEM) 3695 ? VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES 3696 : VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES); 3697 } 3698 3699 out: 3700 #ifdef RACCT 3701 if (rv != KERN_SUCCESS) { 3702 PROC_LOCK(p); 3703 error = racct_set(p, RACCT_VMEM, map->size); 3704 KASSERT(error == 0, ("decreasing RACCT_VMEM failed")); 3705 if (!old_mlock) { 3706 error = racct_set(p, RACCT_MEMLOCK, 3707 ptoa(pmap_wired_count(map->pmap))); 3708 KASSERT(error == 0, ("decreasing RACCT_MEMLOCK failed")); 3709 } 3710 error = racct_set(p, RACCT_STACK, ctob(vm->vm_ssize)); 3711 KASSERT(error == 0, ("decreasing RACCT_STACK failed")); 3712 PROC_UNLOCK(p); 3713 } 3714 #endif 3715 3716 return (rv); 3717 } 3718 3719 /* 3720 * Unshare the specified VM space for exec. If other processes are 3721 * mapped to it, then create a new one. The new vmspace is null. 3722 */ 3723 int 3724 vmspace_exec(struct proc *p, vm_offset_t minuser, vm_offset_t maxuser) 3725 { 3726 struct vmspace *oldvmspace = p->p_vmspace; 3727 struct vmspace *newvmspace; 3728 3729 newvmspace = vmspace_alloc(minuser, maxuser); 3730 if (newvmspace == NULL) 3731 return (ENOMEM); 3732 newvmspace->vm_swrss = oldvmspace->vm_swrss; 3733 /* 3734 * This code is written like this for prototype purposes. The 3735 * goal is to avoid running down the vmspace here, but let the 3736 * other process's that are still using the vmspace to finally 3737 * run it down. Even though there is little or no chance of blocking 3738 * here, it is a good idea to keep this form for future mods. 3739 */ 3740 PROC_VMSPACE_LOCK(p); 3741 p->p_vmspace = newvmspace; 3742 PROC_VMSPACE_UNLOCK(p); 3743 if (p == curthread->td_proc) 3744 pmap_activate(curthread); 3745 vmspace_free(oldvmspace); 3746 return (0); 3747 } 3748 3749 /* 3750 * Unshare the specified VM space for forcing COW. This 3751 * is called by rfork, for the (RFMEM|RFPROC) == 0 case. 3752 */ 3753 int 3754 vmspace_unshare(struct proc *p) 3755 { 3756 struct vmspace *oldvmspace = p->p_vmspace; 3757 struct vmspace *newvmspace; 3758 vm_ooffset_t fork_charge; 3759 3760 if (oldvmspace->vm_refcnt == 1) 3761 return (0); 3762 fork_charge = 0; 3763 newvmspace = vmspace_fork(oldvmspace, &fork_charge); 3764 if (newvmspace == NULL) 3765 return (ENOMEM); 3766 if (!swap_reserve_by_cred(fork_charge, p->p_ucred)) { 3767 vmspace_free(newvmspace); 3768 return (ENOMEM); 3769 } 3770 PROC_VMSPACE_LOCK(p); 3771 p->p_vmspace = newvmspace; 3772 PROC_VMSPACE_UNLOCK(p); 3773 if (p == curthread->td_proc) 3774 pmap_activate(curthread); 3775 vmspace_free(oldvmspace); 3776 return (0); 3777 } 3778 3779 /* 3780 * vm_map_lookup: 3781 * 3782 * Finds the VM object, offset, and 3783 * protection for a given virtual address in the 3784 * specified map, assuming a page fault of the 3785 * type specified. 3786 * 3787 * Leaves the map in question locked for read; return 3788 * values are guaranteed until a vm_map_lookup_done 3789 * call is performed. Note that the map argument 3790 * is in/out; the returned map must be used in 3791 * the call to vm_map_lookup_done. 3792 * 3793 * A handle (out_entry) is returned for use in 3794 * vm_map_lookup_done, to make that fast. 3795 * 3796 * If a lookup is requested with "write protection" 3797 * specified, the map may be changed to perform virtual 3798 * copying operations, although the data referenced will 3799 * remain the same. 3800 */ 3801 int 3802 vm_map_lookup(vm_map_t *var_map, /* IN/OUT */ 3803 vm_offset_t vaddr, 3804 vm_prot_t fault_typea, 3805 vm_map_entry_t *out_entry, /* OUT */ 3806 vm_object_t *object, /* OUT */ 3807 vm_pindex_t *pindex, /* OUT */ 3808 vm_prot_t *out_prot, /* OUT */ 3809 boolean_t *wired) /* OUT */ 3810 { 3811 vm_map_entry_t entry; 3812 vm_map_t map = *var_map; 3813 vm_prot_t prot; 3814 vm_prot_t fault_type = fault_typea; 3815 vm_object_t eobject; 3816 vm_size_t size; 3817 struct ucred *cred; 3818 3819 RetryLookup:; 3820 3821 vm_map_lock_read(map); 3822 3823 /* 3824 * Lookup the faulting address. 3825 */ 3826 if (!vm_map_lookup_entry(map, vaddr, out_entry)) { 3827 vm_map_unlock_read(map); 3828 return (KERN_INVALID_ADDRESS); 3829 } 3830 3831 entry = *out_entry; 3832 3833 /* 3834 * Handle submaps. 3835 */ 3836 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { 3837 vm_map_t old_map = map; 3838 3839 *var_map = map = entry->object.sub_map; 3840 vm_map_unlock_read(old_map); 3841 goto RetryLookup; 3842 } 3843 3844 /* 3845 * Check whether this task is allowed to have this page. 3846 */ 3847 prot = entry->protection; 3848 fault_type &= (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE); 3849 if ((fault_type & prot) != fault_type || prot == VM_PROT_NONE) { 3850 vm_map_unlock_read(map); 3851 return (KERN_PROTECTION_FAILURE); 3852 } 3853 if ((entry->eflags & MAP_ENTRY_USER_WIRED) && 3854 (entry->eflags & MAP_ENTRY_COW) && 3855 (fault_type & VM_PROT_WRITE)) { 3856 vm_map_unlock_read(map); 3857 return (KERN_PROTECTION_FAILURE); 3858 } 3859 if ((fault_typea & VM_PROT_COPY) != 0 && 3860 (entry->max_protection & VM_PROT_WRITE) == 0 && 3861 (entry->eflags & MAP_ENTRY_COW) == 0) { 3862 vm_map_unlock_read(map); 3863 return (KERN_PROTECTION_FAILURE); 3864 } 3865 3866 /* 3867 * If this page is not pageable, we have to get it for all possible 3868 * accesses. 3869 */ 3870 *wired = (entry->wired_count != 0); 3871 if (*wired) 3872 fault_type = entry->protection; 3873 size = entry->end - entry->start; 3874 /* 3875 * If the entry was copy-on-write, we either ... 3876 */ 3877 if (entry->eflags & MAP_ENTRY_NEEDS_COPY) { 3878 /* 3879 * If we want to write the page, we may as well handle that 3880 * now since we've got the map locked. 3881 * 3882 * If we don't need to write the page, we just demote the 3883 * permissions allowed. 3884 */ 3885 if ((fault_type & VM_PROT_WRITE) != 0 || 3886 (fault_typea & VM_PROT_COPY) != 0) { 3887 /* 3888 * Make a new object, and place it in the object 3889 * chain. Note that no new references have appeared 3890 * -- one just moved from the map to the new 3891 * object. 3892 */ 3893 if (vm_map_lock_upgrade(map)) 3894 goto RetryLookup; 3895 3896 if (entry->cred == NULL) { 3897 /* 3898 * The debugger owner is charged for 3899 * the memory. 3900 */ 3901 cred = curthread->td_ucred; 3902 crhold(cred); 3903 if (!swap_reserve_by_cred(size, cred)) { 3904 crfree(cred); 3905 vm_map_unlock(map); 3906 return (KERN_RESOURCE_SHORTAGE); 3907 } 3908 entry->cred = cred; 3909 } 3910 vm_object_shadow(&entry->object.vm_object, 3911 &entry->offset, size); 3912 entry->eflags &= ~MAP_ENTRY_NEEDS_COPY; 3913 eobject = entry->object.vm_object; 3914 if (eobject->cred != NULL) { 3915 /* 3916 * The object was not shadowed. 3917 */ 3918 swap_release_by_cred(size, entry->cred); 3919 crfree(entry->cred); 3920 entry->cred = NULL; 3921 } else if (entry->cred != NULL) { 3922 VM_OBJECT_WLOCK(eobject); 3923 eobject->cred = entry->cred; 3924 eobject->charge = size; 3925 VM_OBJECT_WUNLOCK(eobject); 3926 entry->cred = NULL; 3927 } 3928 3929 vm_map_lock_downgrade(map); 3930 } else { 3931 /* 3932 * We're attempting to read a copy-on-write page -- 3933 * don't allow writes. 3934 */ 3935 prot &= ~VM_PROT_WRITE; 3936 } 3937 } 3938 3939 /* 3940 * Create an object if necessary. 3941 */ 3942 if (entry->object.vm_object == NULL && 3943 !map->system_map) { 3944 if (vm_map_lock_upgrade(map)) 3945 goto RetryLookup; 3946 entry->object.vm_object = vm_object_allocate(OBJT_DEFAULT, 3947 atop(size)); 3948 entry->offset = 0; 3949 if (entry->cred != NULL) { 3950 VM_OBJECT_WLOCK(entry->object.vm_object); 3951 entry->object.vm_object->cred = entry->cred; 3952 entry->object.vm_object->charge = size; 3953 VM_OBJECT_WUNLOCK(entry->object.vm_object); 3954 entry->cred = NULL; 3955 } 3956 vm_map_lock_downgrade(map); 3957 } 3958 3959 /* 3960 * Return the object/offset from this entry. If the entry was 3961 * copy-on-write or empty, it has been fixed up. 3962 */ 3963 *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset); 3964 *object = entry->object.vm_object; 3965 3966 *out_prot = prot; 3967 return (KERN_SUCCESS); 3968 } 3969 3970 /* 3971 * vm_map_lookup_locked: 3972 * 3973 * Lookup the faulting address. A version of vm_map_lookup that returns 3974 * KERN_FAILURE instead of blocking on map lock or memory allocation. 3975 */ 3976 int 3977 vm_map_lookup_locked(vm_map_t *var_map, /* IN/OUT */ 3978 vm_offset_t vaddr, 3979 vm_prot_t fault_typea, 3980 vm_map_entry_t *out_entry, /* OUT */ 3981 vm_object_t *object, /* OUT */ 3982 vm_pindex_t *pindex, /* OUT */ 3983 vm_prot_t *out_prot, /* OUT */ 3984 boolean_t *wired) /* OUT */ 3985 { 3986 vm_map_entry_t entry; 3987 vm_map_t map = *var_map; 3988 vm_prot_t prot; 3989 vm_prot_t fault_type = fault_typea; 3990 3991 /* 3992 * Lookup the faulting address. 3993 */ 3994 if (!vm_map_lookup_entry(map, vaddr, out_entry)) 3995 return (KERN_INVALID_ADDRESS); 3996 3997 entry = *out_entry; 3998 3999 /* 4000 * Fail if the entry refers to a submap. 4001 */ 4002 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) 4003 return (KERN_FAILURE); 4004 4005 /* 4006 * Check whether this task is allowed to have this page. 4007 */ 4008 prot = entry->protection; 4009 fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE; 4010 if ((fault_type & prot) != fault_type) 4011 return (KERN_PROTECTION_FAILURE); 4012 if ((entry->eflags & MAP_ENTRY_USER_WIRED) && 4013 (entry->eflags & MAP_ENTRY_COW) && 4014 (fault_type & VM_PROT_WRITE)) 4015 return (KERN_PROTECTION_FAILURE); 4016 4017 /* 4018 * If this page is not pageable, we have to get it for all possible 4019 * accesses. 4020 */ 4021 *wired = (entry->wired_count != 0); 4022 if (*wired) 4023 fault_type = entry->protection; 4024 4025 if (entry->eflags & MAP_ENTRY_NEEDS_COPY) { 4026 /* 4027 * Fail if the entry was copy-on-write for a write fault. 4028 */ 4029 if (fault_type & VM_PROT_WRITE) 4030 return (KERN_FAILURE); 4031 /* 4032 * We're attempting to read a copy-on-write page -- 4033 * don't allow writes. 4034 */ 4035 prot &= ~VM_PROT_WRITE; 4036 } 4037 4038 /* 4039 * Fail if an object should be created. 4040 */ 4041 if (entry->object.vm_object == NULL && !map->system_map) 4042 return (KERN_FAILURE); 4043 4044 /* 4045 * Return the object/offset from this entry. If the entry was 4046 * copy-on-write or empty, it has been fixed up. 4047 */ 4048 *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset); 4049 *object = entry->object.vm_object; 4050 4051 *out_prot = prot; 4052 return (KERN_SUCCESS); 4053 } 4054 4055 /* 4056 * vm_map_lookup_done: 4057 * 4058 * Releases locks acquired by a vm_map_lookup 4059 * (according to the handle returned by that lookup). 4060 */ 4061 void 4062 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry) 4063 { 4064 /* 4065 * Unlock the main-level map 4066 */ 4067 vm_map_unlock_read(map); 4068 } 4069 4070 #include "opt_ddb.h" 4071 #ifdef DDB 4072 #include <sys/kernel.h> 4073 4074 #include <ddb/ddb.h> 4075 4076 static void 4077 vm_map_print(vm_map_t map) 4078 { 4079 vm_map_entry_t entry; 4080 4081 db_iprintf("Task map %p: pmap=%p, nentries=%d, version=%u\n", 4082 (void *)map, 4083 (void *)map->pmap, map->nentries, map->timestamp); 4084 4085 db_indent += 2; 4086 for (entry = map->header.next; entry != &map->header; 4087 entry = entry->next) { 4088 db_iprintf("map entry %p: start=%p, end=%p\n", 4089 (void *)entry, (void *)entry->start, (void *)entry->end); 4090 { 4091 static char *inheritance_name[4] = 4092 {"share", "copy", "none", "donate_copy"}; 4093 4094 db_iprintf(" prot=%x/%x/%s", 4095 entry->protection, 4096 entry->max_protection, 4097 inheritance_name[(int)(unsigned char)entry->inheritance]); 4098 if (entry->wired_count != 0) 4099 db_printf(", wired"); 4100 } 4101 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { 4102 db_printf(", share=%p, offset=0x%jx\n", 4103 (void *)entry->object.sub_map, 4104 (uintmax_t)entry->offset); 4105 if ((entry->prev == &map->header) || 4106 (entry->prev->object.sub_map != 4107 entry->object.sub_map)) { 4108 db_indent += 2; 4109 vm_map_print((vm_map_t)entry->object.sub_map); 4110 db_indent -= 2; 4111 } 4112 } else { 4113 if (entry->cred != NULL) 4114 db_printf(", ruid %d", entry->cred->cr_ruid); 4115 db_printf(", object=%p, offset=0x%jx", 4116 (void *)entry->object.vm_object, 4117 (uintmax_t)entry->offset); 4118 if (entry->object.vm_object && entry->object.vm_object->cred) 4119 db_printf(", obj ruid %d charge %jx", 4120 entry->object.vm_object->cred->cr_ruid, 4121 (uintmax_t)entry->object.vm_object->charge); 4122 if (entry->eflags & MAP_ENTRY_COW) 4123 db_printf(", copy (%s)", 4124 (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done"); 4125 db_printf("\n"); 4126 4127 if ((entry->prev == &map->header) || 4128 (entry->prev->object.vm_object != 4129 entry->object.vm_object)) { 4130 db_indent += 2; 4131 vm_object_print((db_expr_t)(intptr_t) 4132 entry->object.vm_object, 4133 1, 0, (char *)0); 4134 db_indent -= 2; 4135 } 4136 } 4137 } 4138 db_indent -= 2; 4139 } 4140 4141 DB_SHOW_COMMAND(map, map) 4142 { 4143 4144 if (!have_addr) { 4145 db_printf("usage: show map <addr>\n"); 4146 return; 4147 } 4148 vm_map_print((vm_map_t)addr); 4149 } 4150 4151 DB_SHOW_COMMAND(procvm, procvm) 4152 { 4153 struct proc *p; 4154 4155 if (have_addr) { 4156 p = (struct proc *) addr; 4157 } else { 4158 p = curproc; 4159 } 4160 4161 db_printf("p = %p, vmspace = %p, map = %p, pmap = %p\n", 4162 (void *)p, (void *)p->p_vmspace, (void *)&p->p_vmspace->vm_map, 4163 (void *)vmspace_pmap(p->p_vmspace)); 4164 4165 vm_map_print((vm_map_t)&p->p_vmspace->vm_map); 4166 } 4167 4168 #endif /* DDB */ 4169