1 /* 2 * Copyright (c) 1985, Avadis Tevanian, Jr., Michael Wayne Young 3 * Copyright (c) 1987 Carnegie-Mellon University 4 * Copyright (c) 1991 Regents of the University of California. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * The Mach Operating System project at Carnegie-Mellon University. 9 * 10 * The CMU software License Agreement specifies the terms and conditions 11 * for use and redistribution. 12 * 13 * @(#)vm_map.c 7.1 (Berkeley) 12/05/90 14 */ 15 16 /* 17 * Virtual memory mapping module. 18 */ 19 20 #include "types.h" 21 #include "malloc.h" 22 #include "../vm/vm_param.h" 23 #include "../vm/vm_map.h" 24 #include "../vm/vm_page.h" 25 #include "../vm/vm_object.h" 26 27 /* 28 * Virtual memory maps provide for the mapping, protection, 29 * and sharing of virtual memory objects. In addition, 30 * this module provides for an efficient virtual copy of 31 * memory from one map to another. 32 * 33 * Synchronization is required prior to most operations. 34 * 35 * Maps consist of an ordered doubly-linked list of simple 36 * entries; a single hint is used to speed up lookups. 37 * 38 * In order to properly represent the sharing of virtual 39 * memory regions among maps, the map structure is bi-level. 40 * Top-level ("address") maps refer to regions of sharable 41 * virtual memory. These regions are implemented as 42 * ("sharing") maps, which then refer to the actual virtual 43 * memory objects. When two address maps "share" memory, 44 * their top-level maps both have references to the same 45 * sharing map. When memory is virtual-copied from one 46 * address map to another, the references in the sharing 47 * maps are actually copied -- no copying occurs at the 48 * virtual memory object level. 49 * 50 * Since portions of maps are specified by start/end addreses, 51 * which may not align with existing map entries, all 52 * routines merely "clip" entries to these start/end values. 53 * [That is, an entry is split into two, bordering at a 54 * start or end value.] Note that these clippings may not 55 * always be necessary (as the two resulting entries are then 56 * not changed); however, the clipping is done for convenience. 57 * No attempt is currently made to "glue back together" two 58 * abutting entries. 59 * 60 * As mentioned above, virtual copy operations are performed 61 * by copying VM object references from one sharing map to 62 * another, and then marking both regions as copy-on-write. 63 * It is important to note that only one writeable reference 64 * to a VM object region exists in any map -- this means that 65 * shadow object creation can be delayed until a write operation 66 * occurs. 67 */ 68 69 /* 70 * vm_map_init: 71 * 72 * Initialize the vm_map module. Must be called before 73 * any other vm_map routines. 74 * 75 * Map and entry structures are allocated from the general 76 * purpose memory pool with some exceptions: 77 * 78 * - The kernel map and kmem submap are allocated statically. 79 * - Kernel map entries are allocated out of a static pool. 80 * 81 * These restrictions are necessary since malloc() uses the 82 * maps and requires map entries. 83 */ 84 85 vm_offset_t kentry_data; 86 vm_size_t kentry_data_size; 87 vm_map_entry_t kentry_free; 88 vm_map_t kmap_free; 89 90 void vm_map_init() 91 { 92 register int i; 93 register vm_map_entry_t mep; 94 vm_map_t mp; 95 96 /* 97 * Static map structures for allocation before initialization of 98 * kernel map or kmem map. vm_map_create knows how to deal with them. 99 */ 100 kmap_free = mp = (vm_map_t) kentry_data; 101 i = MAX_KMAP; 102 while (--i > 0) { 103 mp->header.next = (vm_map_entry_t) (mp + 1); 104 mp++; 105 } 106 mp++->header.next = VM_MAP_ENTRY_NULL; 107 108 /* 109 * Form a free list of statically allocated kernel map entries 110 * with the rest. 111 */ 112 kentry_free = mep = (vm_map_entry_t) mp; 113 i = (kentry_data_size - MAX_KMAP * sizeof *mp) / sizeof *mep; 114 while (--i > 0) { 115 mep->next = mep + 1; 116 mep++; 117 } 118 mep->next = VM_MAP_ENTRY_NULL; 119 } 120 121 /* 122 * vm_map_create: 123 * 124 * Creates and returns a new empty VM map with 125 * the given physical map structure, and having 126 * the given lower and upper address bounds. 127 */ 128 vm_map_t vm_map_create(pmap, min, max, pageable) 129 pmap_t pmap; 130 vm_offset_t min, max; 131 boolean_t pageable; 132 { 133 register vm_map_t result; 134 extern vm_map_t kernel_map, kmem_map; 135 136 if (kmem_map == VM_MAP_NULL) { 137 result = kmap_free; 138 kmap_free = (vm_map_t) result->header.next; 139 } else 140 MALLOC(result, vm_map_t, sizeof(struct vm_map), 141 M_VMMAP, M_WAITOK); 142 143 if (result == VM_MAP_NULL) 144 panic("vm_map_create: out of maps"); 145 146 result->header.next = result->header.prev = &result->header; 147 result->nentries = 0; 148 result->size = 0; 149 result->ref_count = 1; 150 result->pmap = pmap; 151 result->is_main_map = TRUE; 152 result->min_offset = min; 153 result->max_offset = max; 154 result->entries_pageable = pageable; 155 result->first_free = &result->header; 156 result->hint = &result->header; 157 result->timestamp = 0; 158 lock_init(&result->lock, TRUE); 159 simple_lock_init(&result->ref_lock); 160 simple_lock_init(&result->hint_lock); 161 return(result); 162 } 163 164 /* 165 * vm_map_entry_create: [ internal use only ] 166 * 167 * Allocates a VM map entry for insertion. 168 * No entry fields are filled in. This routine is 169 */ 170 vm_map_entry_t vm_map_entry_create(map) 171 vm_map_t map; 172 { 173 vm_map_entry_t entry; 174 extern vm_map_t kernel_map, kmem_map, mb_map; 175 176 if (map == kernel_map || map == kmem_map || map == mb_map) { 177 if (entry = kentry_free) 178 kentry_free = kentry_free->next; 179 } else 180 MALLOC(entry, vm_map_entry_t, sizeof(struct vm_map_entry), 181 M_VMMAPENT, M_WAITOK); 182 if (entry == VM_MAP_ENTRY_NULL) 183 panic("vm_map_entry_create: out of map entries"); 184 185 return(entry); 186 } 187 188 /* 189 * vm_map_entry_dispose: [ internal use only ] 190 * 191 * Inverse of vm_map_entry_create. 192 */ 193 void vm_map_entry_dispose(map, entry) 194 vm_map_t map; 195 vm_map_entry_t entry; 196 { 197 extern vm_map_t kernel_map, kmem_map, mb_map; 198 199 if (map == kernel_map || map == kmem_map || map == mb_map) { 200 entry->next = kentry_free; 201 kentry_free = entry; 202 } else 203 FREE(entry, M_VMMAPENT); 204 } 205 206 /* 207 * vm_map_entry_{un,}link: 208 * 209 * Insert/remove entries from maps. 210 */ 211 #define vm_map_entry_link(map, after_where, entry) \ 212 { \ 213 (map)->nentries++; \ 214 (entry)->prev = (after_where); \ 215 (entry)->next = (after_where)->next; \ 216 (entry)->prev->next = (entry); \ 217 (entry)->next->prev = (entry); \ 218 } 219 #define vm_map_entry_unlink(map, entry) \ 220 { \ 221 (map)->nentries--; \ 222 (entry)->next->prev = (entry)->prev; \ 223 (entry)->prev->next = (entry)->next; \ 224 } 225 226 /* 227 * vm_map_reference: 228 * 229 * Creates another valid reference to the given map. 230 * 231 */ 232 void vm_map_reference(map) 233 register vm_map_t map; 234 { 235 if (map == VM_MAP_NULL) 236 return; 237 238 simple_lock(&map->ref_lock); 239 map->ref_count++; 240 simple_unlock(&map->ref_lock); 241 } 242 243 /* 244 * vm_map_deallocate: 245 * 246 * Removes a reference from the specified map, 247 * destroying it if no references remain. 248 * The map should not be locked. 249 */ 250 void vm_map_deallocate(map) 251 register vm_map_t map; 252 { 253 register int c; 254 255 if (map == VM_MAP_NULL) 256 return; 257 258 simple_lock(&map->ref_lock); 259 c = --map->ref_count; 260 simple_unlock(&map->ref_lock); 261 262 if (c > 0) { 263 return; 264 } 265 266 /* 267 * Lock the map, to wait out all other references 268 * to it. 269 */ 270 271 vm_map_lock(map); 272 273 (void) vm_map_delete(map, map->min_offset, map->max_offset); 274 275 pmap_destroy(map->pmap); 276 277 FREE(map, M_VMMAP); 278 } 279 280 /* 281 * vm_map_insert: [ internal use only ] 282 * 283 * Inserts the given whole VM object into the target 284 * map at the specified address range. The object's 285 * size should match that of the address range. 286 * 287 * Requires that the map be locked, and leaves it so. 288 */ 289 vm_map_insert(map, object, offset, start, end) 290 vm_map_t map; 291 vm_object_t object; 292 vm_offset_t offset; 293 vm_offset_t start; 294 vm_offset_t end; 295 { 296 register vm_map_entry_t new_entry; 297 register vm_map_entry_t prev_entry; 298 vm_map_entry_t temp_entry; 299 300 /* 301 * Check that the start and end points are not bogus. 302 */ 303 304 if ((start < map->min_offset) || (end > map->max_offset) || 305 (start >= end)) 306 return(KERN_INVALID_ADDRESS); 307 308 /* 309 * Find the entry prior to the proposed 310 * starting address; if it's part of an 311 * existing entry, this range is bogus. 312 */ 313 314 if (vm_map_lookup_entry(map, start, &temp_entry)) 315 return(KERN_NO_SPACE); 316 317 prev_entry = temp_entry; 318 319 /* 320 * Assert that the next entry doesn't overlap the 321 * end point. 322 */ 323 324 if ((prev_entry->next != &map->header) && 325 (prev_entry->next->start < end)) 326 return(KERN_NO_SPACE); 327 328 /* 329 * See if we can avoid creating a new entry by 330 * extending one of our neighbors. 331 */ 332 333 if (object == VM_OBJECT_NULL) { 334 if ((prev_entry != &map->header) && 335 (prev_entry->end == start) && 336 (map->is_main_map) && 337 (prev_entry->is_a_map == FALSE) && 338 (prev_entry->is_sub_map == FALSE) && 339 (prev_entry->inheritance == VM_INHERIT_DEFAULT) && 340 (prev_entry->protection == VM_PROT_DEFAULT) && 341 (prev_entry->max_protection == VM_PROT_DEFAULT) && 342 (prev_entry->wired_count == 0)) { 343 344 if (vm_object_coalesce(prev_entry->object.vm_object, 345 VM_OBJECT_NULL, 346 prev_entry->offset, 347 (vm_offset_t) 0, 348 (vm_size_t)(prev_entry->end 349 - prev_entry->start), 350 (vm_size_t)(end - prev_entry->end))) { 351 /* 352 * Coalesced the two objects - can extend 353 * the previous map entry to include the 354 * new range. 355 */ 356 map->size += (end - prev_entry->end); 357 prev_entry->end = end; 358 return(KERN_SUCCESS); 359 } 360 } 361 } 362 363 /* 364 * Create a new entry 365 */ 366 367 new_entry = vm_map_entry_create(map); 368 new_entry->start = start; 369 new_entry->end = end; 370 371 new_entry->is_a_map = FALSE; 372 new_entry->is_sub_map = FALSE; 373 new_entry->object.vm_object = object; 374 new_entry->offset = offset; 375 376 new_entry->copy_on_write = FALSE; 377 new_entry->needs_copy = FALSE; 378 379 if (map->is_main_map) { 380 new_entry->inheritance = VM_INHERIT_DEFAULT; 381 new_entry->protection = VM_PROT_DEFAULT; 382 new_entry->max_protection = VM_PROT_DEFAULT; 383 new_entry->wired_count = 0; 384 } 385 386 /* 387 * Insert the new entry into the list 388 */ 389 390 vm_map_entry_link(map, prev_entry, new_entry); 391 map->size += new_entry->end - new_entry->start; 392 393 /* 394 * Update the free space hint 395 */ 396 397 if ((map->first_free == prev_entry) && (prev_entry->end >= new_entry->start)) 398 map->first_free = new_entry; 399 400 return(KERN_SUCCESS); 401 } 402 403 /* 404 * SAVE_HINT: 405 * 406 * Saves the specified entry as the hint for 407 * future lookups. Performs necessary interlocks. 408 */ 409 #define SAVE_HINT(map,value) \ 410 simple_lock(&(map)->hint_lock); \ 411 (map)->hint = (value); \ 412 simple_unlock(&(map)->hint_lock); 413 414 /* 415 * vm_map_lookup_entry: [ internal use only ] 416 * 417 * Finds the map entry containing (or 418 * immediately preceding) the specified address 419 * in the given map; the entry is returned 420 * in the "entry" parameter. The boolean 421 * result indicates whether the address is 422 * actually contained in the map. 423 */ 424 boolean_t vm_map_lookup_entry(map, address, entry) 425 register vm_map_t map; 426 register vm_offset_t address; 427 vm_map_entry_t *entry; /* OUT */ 428 { 429 register vm_map_entry_t cur; 430 register vm_map_entry_t last; 431 432 /* 433 * Start looking either from the head of the 434 * list, or from the hint. 435 */ 436 437 simple_lock(&map->hint_lock); 438 cur = map->hint; 439 simple_unlock(&map->hint_lock); 440 441 if (cur == &map->header) 442 cur = cur->next; 443 444 if (address >= cur->start) { 445 /* 446 * Go from hint to end of list. 447 * 448 * But first, make a quick check to see if 449 * we are already looking at the entry we 450 * want (which is usually the case). 451 * Note also that we don't need to save the hint 452 * here... it is the same hint (unless we are 453 * at the header, in which case the hint didn't 454 * buy us anything anyway). 455 */ 456 last = &map->header; 457 if ((cur != last) && (cur->end > address)) { 458 *entry = cur; 459 return(TRUE); 460 } 461 } 462 else { 463 /* 464 * Go from start to hint, *inclusively* 465 */ 466 last = cur->next; 467 cur = map->header.next; 468 } 469 470 /* 471 * Search linearly 472 */ 473 474 while (cur != last) { 475 if (cur->end > address) { 476 if (address >= cur->start) { 477 /* 478 * Save this lookup for future 479 * hints, and return 480 */ 481 482 *entry = cur; 483 SAVE_HINT(map, cur); 484 return(TRUE); 485 } 486 break; 487 } 488 cur = cur->next; 489 } 490 *entry = cur->prev; 491 SAVE_HINT(map, *entry); 492 return(FALSE); 493 } 494 495 /* 496 * vm_map_find finds an unallocated region in the target address 497 * map with the given length. The search is defined to be 498 * first-fit from the specified address; the region found is 499 * returned in the same parameter. 500 * 501 */ 502 vm_map_find(map, object, offset, addr, length, find_space) 503 vm_map_t map; 504 vm_object_t object; 505 vm_offset_t offset; 506 vm_offset_t *addr; /* IN/OUT */ 507 vm_size_t length; 508 boolean_t find_space; 509 { 510 register vm_map_entry_t entry; 511 register vm_offset_t start; 512 register vm_offset_t end; 513 int result; 514 515 start = *addr; 516 517 vm_map_lock(map); 518 519 if (find_space) { 520 /* 521 * Calculate the first possible address. 522 */ 523 524 if (start < map->min_offset) 525 start = map->min_offset; 526 if (start > map->max_offset) { 527 vm_map_unlock(map); 528 return (KERN_NO_SPACE); 529 } 530 531 /* 532 * Look for the first possible address; 533 * if there's already something at this 534 * address, we have to start after it. 535 */ 536 537 if (start == map->min_offset) { 538 if ((entry = map->first_free) != &map->header) 539 start = entry->end; 540 } else { 541 vm_map_entry_t tmp_entry; 542 if (vm_map_lookup_entry(map, start, &tmp_entry)) 543 start = tmp_entry->end; 544 entry = tmp_entry; 545 } 546 547 /* 548 * In any case, the "entry" always precedes 549 * the proposed new region throughout the 550 * loop: 551 */ 552 553 while (TRUE) { 554 register vm_map_entry_t next; 555 556 /* 557 * Find the end of the proposed new region. 558 * Be sure we didn't go beyond the end, or 559 * wrap around the address. 560 */ 561 562 end = start + length; 563 564 if ((end > map->max_offset) || (end < start)) { 565 vm_map_unlock(map); 566 return (KERN_NO_SPACE); 567 } 568 569 /* 570 * If there are no more entries, we must win. 571 */ 572 573 next = entry->next; 574 if (next == &map->header) 575 break; 576 577 /* 578 * If there is another entry, it must be 579 * after the end of the potential new region. 580 */ 581 582 if (next->start >= end) 583 break; 584 585 /* 586 * Didn't fit -- move to the next entry. 587 */ 588 589 entry = next; 590 start = entry->end; 591 } 592 *addr = start; 593 594 SAVE_HINT(map, entry); 595 } 596 597 result = vm_map_insert(map, object, offset, start, start + length); 598 599 vm_map_unlock(map); 600 return(result); 601 } 602 603 /* 604 * vm_map_simplify_entry: [ internal use only ] 605 * 606 * Simplify the given map entry by: 607 * removing extra sharing maps 608 * [XXX maybe later] merging with a neighbor 609 */ 610 void vm_map_simplify_entry(map, entry) 611 vm_map_t map; 612 vm_map_entry_t entry; 613 { 614 #ifdef lint 615 map++; 616 #endif lint 617 618 /* 619 * If this entry corresponds to a sharing map, then 620 * see if we can remove the level of indirection. 621 * If it's not a sharing map, then it points to 622 * a VM object, so see if we can merge with either 623 * of our neighbors. 624 */ 625 626 if (entry->is_sub_map) 627 return; 628 if (entry->is_a_map) { 629 #if 0 630 vm_map_t my_share_map; 631 int count; 632 633 my_share_map = entry->object.share_map; 634 simple_lock(&my_share_map->ref_lock); 635 count = my_share_map->ref_count; 636 simple_unlock(&my_share_map->ref_lock); 637 638 if (count == 1) { 639 /* Can move the region from 640 * entry->start to entry->end (+ entry->offset) 641 * in my_share_map into place of entry. 642 * Later. 643 */ 644 } 645 #endif 0 646 } 647 else { 648 /* 649 * Try to merge with our neighbors. 650 * 651 * Conditions for merge are: 652 * 653 * 1. entries are adjacent. 654 * 2. both entries point to objects 655 * with null pagers. 656 * 657 * If a merge is possible, we replace the two 658 * entries with a single entry, then merge 659 * the two objects into a single object. 660 * 661 * Now, all that is left to do is write the 662 * code! 663 */ 664 } 665 } 666 667 /* 668 * vm_map_clip_start: [ internal use only ] 669 * 670 * Asserts that the given entry begins at or after 671 * the specified address; if necessary, 672 * it splits the entry into two. 673 */ 674 #define vm_map_clip_start(map, entry, startaddr) \ 675 { \ 676 if (startaddr > entry->start) \ 677 _vm_map_clip_start(map, entry, startaddr); \ 678 } 679 680 /* 681 * This routine is called only when it is known that 682 * the entry must be split. 683 */ 684 void _vm_map_clip_start(map, entry, start) 685 register vm_map_t map; 686 register vm_map_entry_t entry; 687 register vm_offset_t start; 688 { 689 register vm_map_entry_t new_entry; 690 691 /* 692 * See if we can simplify this entry first 693 */ 694 695 vm_map_simplify_entry(map, entry); 696 697 /* 698 * Split off the front portion -- 699 * note that we must insert the new 700 * entry BEFORE this one, so that 701 * this entry has the specified starting 702 * address. 703 */ 704 705 new_entry = vm_map_entry_create(map); 706 *new_entry = *entry; 707 708 new_entry->end = start; 709 entry->offset += (start - entry->start); 710 entry->start = start; 711 712 vm_map_entry_link(map, entry->prev, new_entry); 713 714 if (entry->is_a_map || entry->is_sub_map) 715 vm_map_reference(new_entry->object.share_map); 716 else 717 vm_object_reference(new_entry->object.vm_object); 718 } 719 720 /* 721 * vm_map_clip_end: [ internal use only ] 722 * 723 * Asserts that the given entry ends at or before 724 * the specified address; if necessary, 725 * it splits the entry into two. 726 */ 727 728 void _vm_map_clip_end(); 729 #define vm_map_clip_end(map, entry, endaddr) \ 730 { \ 731 if (endaddr < entry->end) \ 732 _vm_map_clip_end(map, entry, endaddr); \ 733 } 734 735 /* 736 * This routine is called only when it is known that 737 * the entry must be split. 738 */ 739 void _vm_map_clip_end(map, entry, end) 740 register vm_map_t map; 741 register vm_map_entry_t entry; 742 register vm_offset_t end; 743 { 744 register vm_map_entry_t new_entry; 745 746 /* 747 * Create a new entry and insert it 748 * AFTER the specified entry 749 */ 750 751 new_entry = vm_map_entry_create(map); 752 *new_entry = *entry; 753 754 new_entry->start = entry->end = end; 755 new_entry->offset += (end - entry->start); 756 757 vm_map_entry_link(map, entry, new_entry); 758 759 if (entry->is_a_map || entry->is_sub_map) 760 vm_map_reference(new_entry->object.share_map); 761 else 762 vm_object_reference(new_entry->object.vm_object); 763 } 764 765 /* 766 * VM_MAP_RANGE_CHECK: [ internal use only ] 767 * 768 * Asserts that the starting and ending region 769 * addresses fall within the valid range of the map. 770 */ 771 #define VM_MAP_RANGE_CHECK(map, start, end) \ 772 { \ 773 if (start < vm_map_min(map)) \ 774 start = vm_map_min(map); \ 775 if (end > vm_map_max(map)) \ 776 end = vm_map_max(map); \ 777 if (start > end) \ 778 start = end; \ 779 } 780 781 /* 782 * vm_map_submap: [ kernel use only ] 783 * 784 * Mark the given range as handled by a subordinate map. 785 * 786 * This range must have been created with vm_map_find, 787 * and no other operations may have been performed on this 788 * range prior to calling vm_map_submap. 789 * 790 * Only a limited number of operations can be performed 791 * within this rage after calling vm_map_submap: 792 * vm_fault 793 * [Don't try vm_map_copy!] 794 * 795 * To remove a submapping, one must first remove the 796 * range from the superior map, and then destroy the 797 * submap (if desired). [Better yet, don't try it.] 798 */ 799 vm_map_submap(map, start, end, submap) 800 register vm_map_t map; 801 register vm_offset_t start; 802 register vm_offset_t end; 803 vm_map_t submap; 804 { 805 vm_map_entry_t entry; 806 register int result = KERN_INVALID_ARGUMENT; 807 808 vm_map_lock(map); 809 810 VM_MAP_RANGE_CHECK(map, start, end); 811 812 if (vm_map_lookup_entry(map, start, &entry)) { 813 vm_map_clip_start(map, entry, start); 814 } 815 else 816 entry = entry->next; 817 818 vm_map_clip_end(map, entry, end); 819 820 if ((entry->start == start) && (entry->end == end) && 821 (!entry->is_a_map) && 822 (entry->object.vm_object == VM_OBJECT_NULL) && 823 (!entry->copy_on_write)) { 824 entry->is_a_map = FALSE; 825 entry->is_sub_map = TRUE; 826 vm_map_reference(entry->object.sub_map = submap); 827 result = KERN_SUCCESS; 828 } 829 vm_map_unlock(map); 830 831 return(result); 832 } 833 834 /* 835 * vm_map_protect: 836 * 837 * Sets the protection of the specified address 838 * region in the target map. If "set_max" is 839 * specified, the maximum protection is to be set; 840 * otherwise, only the current protection is affected. 841 */ 842 vm_map_protect(map, start, end, new_prot, set_max) 843 register vm_map_t map; 844 register vm_offset_t start; 845 register vm_offset_t end; 846 register vm_prot_t new_prot; 847 register boolean_t set_max; 848 { 849 register vm_map_entry_t current; 850 vm_map_entry_t entry; 851 852 vm_map_lock(map); 853 854 VM_MAP_RANGE_CHECK(map, start, end); 855 856 if (vm_map_lookup_entry(map, start, &entry)) { 857 vm_map_clip_start(map, entry, start); 858 } 859 else 860 entry = entry->next; 861 862 /* 863 * Make a first pass to check for protection 864 * violations. 865 */ 866 867 current = entry; 868 while ((current != &map->header) && (current->start < end)) { 869 if (current->is_sub_map) 870 return(KERN_INVALID_ARGUMENT); 871 if ((new_prot & current->max_protection) != new_prot) { 872 vm_map_unlock(map); 873 return(KERN_PROTECTION_FAILURE); 874 } 875 876 current = current->next; 877 } 878 879 /* 880 * Go back and fix up protections. 881 * [Note that clipping is not necessary the second time.] 882 */ 883 884 current = entry; 885 886 while ((current != &map->header) && (current->start < end)) { 887 vm_prot_t old_prot; 888 889 vm_map_clip_end(map, current, end); 890 891 old_prot = current->protection; 892 if (set_max) 893 current->protection = 894 (current->max_protection = new_prot) & 895 old_prot; 896 else 897 current->protection = new_prot; 898 899 /* 900 * Update physical map if necessary. 901 * Worry about copy-on-write here -- CHECK THIS XXX 902 */ 903 904 if (current->protection != old_prot) { 905 906 #define MASK(entry) ((entry)->copy_on_write ? ~VM_PROT_WRITE : \ 907 VM_PROT_ALL) 908 #define max(a,b) ((a) > (b) ? (a) : (b)) 909 910 if (current->is_a_map) { 911 vm_map_entry_t share_entry; 912 vm_offset_t share_end; 913 914 vm_map_lock(current->object.share_map); 915 (void) vm_map_lookup_entry( 916 current->object.share_map, 917 current->offset, 918 &share_entry); 919 share_end = current->offset + 920 (current->end - current->start); 921 while ((share_entry != 922 ¤t->object.share_map->header) && 923 (share_entry->start < share_end)) { 924 925 pmap_protect(map->pmap, 926 (max(share_entry->start, 927 current->offset) - 928 current->offset + 929 current->start), 930 min(share_entry->end, 931 share_end) - 932 current->offset + 933 current->start, 934 current->protection & 935 MASK(share_entry)); 936 937 share_entry = share_entry->next; 938 } 939 vm_map_unlock(current->object.share_map); 940 } 941 else 942 pmap_protect(map->pmap, current->start, 943 current->end, 944 current->protection & MASK(entry)); 945 #undef max 946 #undef MASK 947 } 948 current = current->next; 949 } 950 951 vm_map_unlock(map); 952 return(KERN_SUCCESS); 953 } 954 955 /* 956 * vm_map_inherit: 957 * 958 * Sets the inheritance of the specified address 959 * range in the target map. Inheritance 960 * affects how the map will be shared with 961 * child maps at the time of vm_map_fork. 962 */ 963 vm_map_inherit(map, start, end, new_inheritance) 964 register vm_map_t map; 965 register vm_offset_t start; 966 register vm_offset_t end; 967 register vm_inherit_t new_inheritance; 968 { 969 register vm_map_entry_t entry; 970 vm_map_entry_t temp_entry; 971 972 switch (new_inheritance) { 973 case VM_INHERIT_NONE: 974 case VM_INHERIT_COPY: 975 case VM_INHERIT_SHARE: 976 break; 977 default: 978 return(KERN_INVALID_ARGUMENT); 979 } 980 981 vm_map_lock(map); 982 983 VM_MAP_RANGE_CHECK(map, start, end); 984 985 if (vm_map_lookup_entry(map, start, &temp_entry)) { 986 entry = temp_entry; 987 vm_map_clip_start(map, entry, start); 988 } 989 else 990 entry = temp_entry->next; 991 992 while ((entry != &map->header) && (entry->start < end)) { 993 vm_map_clip_end(map, entry, end); 994 995 entry->inheritance = new_inheritance; 996 997 entry = entry->next; 998 } 999 1000 vm_map_unlock(map); 1001 return(KERN_SUCCESS); 1002 } 1003 1004 /* 1005 * vm_map_pageable: 1006 * 1007 * Sets the pageability of the specified address 1008 * range in the target map. Regions specified 1009 * as not pageable require locked-down physical 1010 * memory and physical page maps. 1011 * 1012 * The map must not be locked, but a reference 1013 * must remain to the map throughout the call. 1014 */ 1015 vm_map_pageable(map, start, end, new_pageable) 1016 register vm_map_t map; 1017 register vm_offset_t start; 1018 register vm_offset_t end; 1019 register boolean_t new_pageable; 1020 { 1021 register vm_map_entry_t entry; 1022 vm_map_entry_t temp_entry; 1023 1024 vm_map_lock(map); 1025 1026 VM_MAP_RANGE_CHECK(map, start, end); 1027 1028 /* 1029 * Only one pageability change may take place at one 1030 * time, since vm_fault assumes it will be called 1031 * only once for each wiring/unwiring. Therefore, we 1032 * have to make sure we're actually changing the pageability 1033 * for the entire region. We do so before making any changes. 1034 */ 1035 1036 if (vm_map_lookup_entry(map, start, &temp_entry)) { 1037 entry = temp_entry; 1038 vm_map_clip_start(map, entry, start); 1039 } 1040 else 1041 entry = temp_entry->next; 1042 temp_entry = entry; 1043 1044 /* 1045 * Actions are rather different for wiring and unwiring, 1046 * so we have two separate cases. 1047 */ 1048 1049 if (new_pageable) { 1050 1051 /* 1052 * Unwiring. First ensure that the range to be 1053 * unwired is really wired down. 1054 */ 1055 while ((entry != &map->header) && (entry->start < end)) { 1056 1057 if (entry->wired_count == 0) { 1058 vm_map_unlock(map); 1059 return(KERN_INVALID_ARGUMENT); 1060 } 1061 entry = entry->next; 1062 } 1063 1064 /* 1065 * Now decrement the wiring count for each region. 1066 * If a region becomes completely unwired, 1067 * unwire its physical pages and mappings. 1068 */ 1069 lock_set_recursive(&map->lock); 1070 1071 entry = temp_entry; 1072 while ((entry != &map->header) && (entry->start < end)) { 1073 vm_map_clip_end(map, entry, end); 1074 1075 entry->wired_count--; 1076 if (entry->wired_count == 0) 1077 vm_fault_unwire(map, entry->start, entry->end); 1078 1079 entry = entry->next; 1080 } 1081 lock_clear_recursive(&map->lock); 1082 } 1083 1084 else { 1085 /* 1086 * Wiring. We must do this in two passes: 1087 * 1088 * 1. Holding the write lock, we increment the 1089 * wiring count. For any area that is not already 1090 * wired, we create any shadow objects that need 1091 * to be created. 1092 * 1093 * 2. We downgrade to a read lock, and call 1094 * vm_fault_wire to fault in the pages for any 1095 * newly wired area (wired_count is 1). 1096 * 1097 * Downgrading to a read lock for vm_fault_wire avoids 1098 * a possible deadlock with another thread that may have 1099 * faulted on one of the pages to be wired (it would mark 1100 * the page busy, blocking us, then in turn block on the 1101 * map lock that we hold). Because of problems in the 1102 * recursive lock package, we cannot upgrade to a write 1103 * lock in vm_map_lookup. Thus, any actions that require 1104 * the write lock must be done beforehand. Because we 1105 * keep the read lock on the map, the copy-on-write status 1106 * of the entries we modify here cannot change. 1107 */ 1108 1109 /* 1110 * Pass 1. 1111 */ 1112 entry = temp_entry; 1113 while ((entry != &map->header) && (entry->start < end)) { 1114 vm_map_clip_end(map, entry, end); 1115 1116 entry->wired_count++; 1117 if (entry->wired_count == 1) { 1118 1119 /* 1120 * Perform actions of vm_map_lookup that need 1121 * the write lock on the map: create a shadow 1122 * object for a copy-on-write region, or an 1123 * object for a zero-fill region. 1124 * 1125 * We don't have to do this for entries that 1126 * point to sharing maps, because we won't hold 1127 * the lock on the sharing map. 1128 */ 1129 if (!entry->is_a_map) { 1130 if (entry->needs_copy && 1131 ((entry->protection & VM_PROT_WRITE) != 0)) { 1132 1133 vm_object_shadow(&entry->object.vm_object, 1134 &entry->offset, 1135 (vm_size_t)(entry->end 1136 - entry->start)); 1137 entry->needs_copy = FALSE; 1138 } 1139 else if (entry->object.vm_object == VM_OBJECT_NULL) { 1140 entry->object.vm_object = 1141 vm_object_allocate((vm_size_t)(entry->end 1142 - entry->start)); 1143 entry->offset = (vm_offset_t)0; 1144 } 1145 } 1146 } 1147 1148 entry = entry->next; 1149 } 1150 1151 /* 1152 * Pass 2. 1153 */ 1154 1155 /* 1156 * HACK HACK HACK HACK 1157 * 1158 * If we are wiring in the kernel map or a submap of it, 1159 * unlock the map to avoid deadlocks. We trust that the 1160 * kernel threads are well-behaved, and therefore will 1161 * not do anything destructive to this region of the map 1162 * while we have it unlocked. We cannot trust user threads 1163 * to do the same. 1164 * 1165 * HACK HACK HACK HACK 1166 */ 1167 if (vm_map_pmap(map) == kernel_pmap) { 1168 vm_map_unlock(map); /* trust me ... */ 1169 } 1170 else { 1171 lock_set_recursive(&map->lock); 1172 lock_write_to_read(&map->lock); 1173 } 1174 1175 entry = temp_entry; 1176 while (entry != &map->header && entry->start < end) { 1177 if (entry->wired_count == 1) { 1178 vm_fault_wire(map, entry->start, entry->end); 1179 } 1180 entry = entry->next; 1181 } 1182 1183 if (vm_map_pmap(map) == kernel_pmap) { 1184 vm_map_lock(map); 1185 } 1186 else { 1187 lock_clear_recursive(&map->lock); 1188 } 1189 } 1190 1191 vm_map_unlock(map); 1192 1193 return(KERN_SUCCESS); 1194 } 1195 1196 /* 1197 * vm_map_entry_unwire: [ internal use only ] 1198 * 1199 * Make the region specified by this entry pageable. 1200 * 1201 * The map in question should be locked. 1202 * [This is the reason for this routine's existence.] 1203 */ 1204 void vm_map_entry_unwire(map, entry) 1205 vm_map_t map; 1206 register vm_map_entry_t entry; 1207 { 1208 vm_fault_unwire(map, entry->start, entry->end); 1209 entry->wired_count = 0; 1210 } 1211 1212 /* 1213 * vm_map_entry_delete: [ internal use only ] 1214 * 1215 * Deallocate the given entry from the target map. 1216 */ 1217 void vm_map_entry_delete(map, entry) 1218 register vm_map_t map; 1219 register vm_map_entry_t entry; 1220 { 1221 if (entry->wired_count != 0) 1222 vm_map_entry_unwire(map, entry); 1223 1224 vm_map_entry_unlink(map, entry); 1225 map->size -= entry->end - entry->start; 1226 1227 if (entry->is_a_map || entry->is_sub_map) 1228 vm_map_deallocate(entry->object.share_map); 1229 else 1230 vm_object_deallocate(entry->object.vm_object); 1231 1232 vm_map_entry_dispose(map, entry); 1233 } 1234 1235 /* 1236 * vm_map_delete: [ internal use only ] 1237 * 1238 * Deallocates the given address range from the target 1239 * map. 1240 * 1241 * When called with a sharing map, removes pages from 1242 * that region from all physical maps. 1243 */ 1244 vm_map_delete(map, start, end) 1245 register vm_map_t map; 1246 vm_offset_t start; 1247 register vm_offset_t end; 1248 { 1249 register vm_map_entry_t entry; 1250 vm_map_entry_t first_entry; 1251 1252 /* 1253 * Find the start of the region, and clip it 1254 */ 1255 1256 if (!vm_map_lookup_entry(map, start, &first_entry)) 1257 entry = first_entry->next; 1258 else { 1259 entry = first_entry; 1260 vm_map_clip_start(map, entry, start); 1261 1262 /* 1263 * Fix the lookup hint now, rather than each 1264 * time though the loop. 1265 */ 1266 1267 SAVE_HINT(map, entry->prev); 1268 } 1269 1270 /* 1271 * Save the free space hint 1272 */ 1273 1274 if (map->first_free->start >= start) 1275 map->first_free = entry->prev; 1276 1277 /* 1278 * Step through all entries in this region 1279 */ 1280 1281 while ((entry != &map->header) && (entry->start < end)) { 1282 vm_map_entry_t next; 1283 register vm_offset_t s, e; 1284 register vm_object_t object; 1285 1286 vm_map_clip_end(map, entry, end); 1287 1288 next = entry->next; 1289 s = entry->start; 1290 e = entry->end; 1291 1292 /* 1293 * Unwire before removing addresses from the pmap; 1294 * otherwise, unwiring will put the entries back in 1295 * the pmap. 1296 */ 1297 1298 object = entry->object.vm_object; 1299 if (entry->wired_count != 0) 1300 vm_map_entry_unwire(map, entry); 1301 1302 /* 1303 * If this is a sharing map, we must remove 1304 * *all* references to this data, since we can't 1305 * find all of the physical maps which are sharing 1306 * it. 1307 */ 1308 1309 if (object == kernel_object || object == kmem_object) 1310 vm_object_page_remove(object, entry->offset, 1311 entry->offset + (e - s)); 1312 else if (!map->is_main_map) 1313 vm_object_pmap_remove(object, 1314 entry->offset, 1315 entry->offset + (e - s)); 1316 else 1317 pmap_remove(map->pmap, s, e); 1318 1319 /* 1320 * Delete the entry (which may delete the object) 1321 * only after removing all pmap entries pointing 1322 * to its pages. (Otherwise, its page frames may 1323 * be reallocated, and any modify bits will be 1324 * set in the wrong object!) 1325 */ 1326 1327 vm_map_entry_delete(map, entry); 1328 entry = next; 1329 } 1330 return(KERN_SUCCESS); 1331 } 1332 1333 /* 1334 * vm_map_remove: 1335 * 1336 * Remove the given address range from the target map. 1337 * This is the exported form of vm_map_delete. 1338 */ 1339 vm_map_remove(map, start, end) 1340 register vm_map_t map; 1341 register vm_offset_t start; 1342 register vm_offset_t end; 1343 { 1344 register int result; 1345 1346 vm_map_lock(map); 1347 VM_MAP_RANGE_CHECK(map, start, end); 1348 result = vm_map_delete(map, start, end); 1349 vm_map_unlock(map); 1350 1351 return(result); 1352 } 1353 1354 /* 1355 * vm_map_check_protection: 1356 * 1357 * Assert that the target map allows the specified 1358 * privilege on the entire address region given. 1359 * The entire region must be allocated. 1360 */ 1361 boolean_t vm_map_check_protection(map, start, end, protection) 1362 register vm_map_t map; 1363 register vm_offset_t start; 1364 register vm_offset_t end; 1365 register vm_prot_t protection; 1366 { 1367 register vm_map_entry_t entry; 1368 vm_map_entry_t tmp_entry; 1369 1370 if (!vm_map_lookup_entry(map, start, &tmp_entry)) { 1371 return(FALSE); 1372 } 1373 1374 entry = tmp_entry; 1375 1376 while (start < end) { 1377 if (entry == &map->header) { 1378 return(FALSE); 1379 } 1380 1381 /* 1382 * No holes allowed! 1383 */ 1384 1385 if (start < entry->start) { 1386 return(FALSE); 1387 } 1388 1389 /* 1390 * Check protection associated with entry. 1391 */ 1392 1393 if ((entry->protection & protection) != protection) { 1394 return(FALSE); 1395 } 1396 1397 /* go to next entry */ 1398 1399 start = entry->end; 1400 entry = entry->next; 1401 } 1402 return(TRUE); 1403 } 1404 1405 /* 1406 * vm_map_copy_entry: 1407 * 1408 * Copies the contents of the source entry to the destination 1409 * entry. The entries *must* be aligned properly. 1410 */ 1411 void vm_map_copy_entry(src_map, dst_map, src_entry, dst_entry) 1412 vm_map_t src_map, dst_map; 1413 register vm_map_entry_t src_entry, dst_entry; 1414 { 1415 vm_object_t temp_object; 1416 1417 if (src_entry->is_sub_map || dst_entry->is_sub_map) 1418 return; 1419 1420 if (dst_entry->object.vm_object != VM_OBJECT_NULL && 1421 !dst_entry->object.vm_object->internal) 1422 printf("vm_map_copy_entry: copying over permanent data!\n"); 1423 1424 /* 1425 * If our destination map was wired down, 1426 * unwire it now. 1427 */ 1428 1429 if (dst_entry->wired_count != 0) 1430 vm_map_entry_unwire(dst_map, dst_entry); 1431 1432 /* 1433 * If we're dealing with a sharing map, we 1434 * must remove the destination pages from 1435 * all maps (since we cannot know which maps 1436 * this sharing map belongs in). 1437 */ 1438 1439 if (dst_map->is_main_map) 1440 pmap_remove(dst_map->pmap, dst_entry->start, dst_entry->end); 1441 else 1442 vm_object_pmap_remove(dst_entry->object.vm_object, 1443 dst_entry->offset, 1444 dst_entry->offset + 1445 (dst_entry->end - dst_entry->start)); 1446 1447 if (src_entry->wired_count == 0) { 1448 1449 boolean_t src_needs_copy; 1450 1451 /* 1452 * If the source entry is marked needs_copy, 1453 * it is already write-protected. 1454 */ 1455 if (!src_entry->needs_copy) { 1456 1457 boolean_t su; 1458 1459 /* 1460 * If the source entry has only one mapping, 1461 * we can just protect the virtual address 1462 * range. 1463 */ 1464 if (!(su = src_map->is_main_map)) { 1465 simple_lock(&src_map->ref_lock); 1466 su = (src_map->ref_count == 1); 1467 simple_unlock(&src_map->ref_lock); 1468 } 1469 1470 if (su) { 1471 pmap_protect(src_map->pmap, 1472 src_entry->start, 1473 src_entry->end, 1474 src_entry->protection & ~VM_PROT_WRITE); 1475 } 1476 else { 1477 vm_object_pmap_copy(src_entry->object.vm_object, 1478 src_entry->offset, 1479 src_entry->offset + (src_entry->end 1480 -src_entry->start)); 1481 } 1482 } 1483 1484 /* 1485 * Make a copy of the object. 1486 */ 1487 temp_object = dst_entry->object.vm_object; 1488 vm_object_copy(src_entry->object.vm_object, 1489 src_entry->offset, 1490 (vm_size_t)(src_entry->end - 1491 src_entry->start), 1492 &dst_entry->object.vm_object, 1493 &dst_entry->offset, 1494 &src_needs_copy); 1495 /* 1496 * If we didn't get a copy-object now, mark the 1497 * source map entry so that a shadow will be created 1498 * to hold its changed pages. 1499 */ 1500 if (src_needs_copy) 1501 src_entry->needs_copy = TRUE; 1502 1503 /* 1504 * The destination always needs to have a shadow 1505 * created. 1506 */ 1507 dst_entry->needs_copy = TRUE; 1508 1509 /* 1510 * Mark the entries copy-on-write, so that write-enabling 1511 * the entry won't make copy-on-write pages writable. 1512 */ 1513 src_entry->copy_on_write = TRUE; 1514 dst_entry->copy_on_write = TRUE; 1515 /* 1516 * Get rid of the old object. 1517 */ 1518 vm_object_deallocate(temp_object); 1519 1520 pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start, 1521 dst_entry->end - dst_entry->start, src_entry->start); 1522 } 1523 else { 1524 /* 1525 * Of course, wired down pages can't be set copy-on-write. 1526 * Cause wired pages to be copied into the new 1527 * map by simulating faults (the new pages are 1528 * pageable) 1529 */ 1530 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry); 1531 } 1532 } 1533 1534 /* 1535 * vm_map_copy: 1536 * 1537 * Perform a virtual memory copy from the source 1538 * address map/range to the destination map/range. 1539 * 1540 * If src_destroy or dst_alloc is requested, 1541 * the source and destination regions should be 1542 * disjoint, not only in the top-level map, but 1543 * in the sharing maps as well. [The best way 1544 * to guarantee this is to use a new intermediate 1545 * map to make copies. This also reduces map 1546 * fragmentation.] 1547 */ 1548 vm_map_copy(dst_map, src_map, 1549 dst_addr, len, src_addr, 1550 dst_alloc, src_destroy) 1551 vm_map_t dst_map; 1552 vm_map_t src_map; 1553 vm_offset_t dst_addr; 1554 vm_size_t len; 1555 vm_offset_t src_addr; 1556 boolean_t dst_alloc; 1557 boolean_t src_destroy; 1558 { 1559 register 1560 vm_map_entry_t src_entry; 1561 register 1562 vm_map_entry_t dst_entry; 1563 vm_map_entry_t tmp_entry; 1564 vm_offset_t src_start; 1565 vm_offset_t src_end; 1566 vm_offset_t dst_start; 1567 vm_offset_t dst_end; 1568 vm_offset_t src_clip; 1569 vm_offset_t dst_clip; 1570 int result; 1571 boolean_t old_src_destroy; 1572 1573 /* 1574 * XXX While we figure out why src_destroy screws up, 1575 * we'll do it by explicitly vm_map_delete'ing at the end. 1576 */ 1577 1578 old_src_destroy = src_destroy; 1579 src_destroy = FALSE; 1580 1581 /* 1582 * Compute start and end of region in both maps 1583 */ 1584 1585 src_start = src_addr; 1586 src_end = src_start + len; 1587 dst_start = dst_addr; 1588 dst_end = dst_start + len; 1589 1590 /* 1591 * Check that the region can exist in both source 1592 * and destination. 1593 */ 1594 1595 if ((dst_end < dst_start) || (src_end < src_start)) 1596 return(KERN_NO_SPACE); 1597 1598 /* 1599 * Lock the maps in question -- we avoid deadlock 1600 * by ordering lock acquisition by map value 1601 */ 1602 1603 if (src_map == dst_map) { 1604 vm_map_lock(src_map); 1605 } 1606 else if ((int) src_map < (int) dst_map) { 1607 vm_map_lock(src_map); 1608 vm_map_lock(dst_map); 1609 } else { 1610 vm_map_lock(dst_map); 1611 vm_map_lock(src_map); 1612 } 1613 1614 result = KERN_SUCCESS; 1615 1616 /* 1617 * Check protections... source must be completely readable and 1618 * destination must be completely writable. [Note that if we're 1619 * allocating the destination region, we don't have to worry 1620 * about protection, but instead about whether the region 1621 * exists.] 1622 */ 1623 1624 if (src_map->is_main_map && dst_map->is_main_map) { 1625 if (!vm_map_check_protection(src_map, src_start, src_end, 1626 VM_PROT_READ)) { 1627 result = KERN_PROTECTION_FAILURE; 1628 goto Return; 1629 } 1630 1631 if (dst_alloc) { 1632 /* XXX Consider making this a vm_map_find instead */ 1633 if ((result = vm_map_insert(dst_map, VM_OBJECT_NULL, 1634 (vm_offset_t) 0, dst_start, dst_end)) != KERN_SUCCESS) 1635 goto Return; 1636 } 1637 else if (!vm_map_check_protection(dst_map, dst_start, dst_end, 1638 VM_PROT_WRITE)) { 1639 result = KERN_PROTECTION_FAILURE; 1640 goto Return; 1641 } 1642 } 1643 1644 /* 1645 * Find the start entries and clip. 1646 * 1647 * Note that checking protection asserts that the 1648 * lookup cannot fail. 1649 * 1650 * Also note that we wait to do the second lookup 1651 * until we have done the first clip, as the clip 1652 * may affect which entry we get! 1653 */ 1654 1655 (void) vm_map_lookup_entry(src_map, src_addr, &tmp_entry); 1656 src_entry = tmp_entry; 1657 vm_map_clip_start(src_map, src_entry, src_start); 1658 1659 (void) vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry); 1660 dst_entry = tmp_entry; 1661 vm_map_clip_start(dst_map, dst_entry, dst_start); 1662 1663 /* 1664 * If both source and destination entries are the same, 1665 * retry the first lookup, as it may have changed. 1666 */ 1667 1668 if (src_entry == dst_entry) { 1669 (void) vm_map_lookup_entry(src_map, src_addr, &tmp_entry); 1670 src_entry = tmp_entry; 1671 } 1672 1673 /* 1674 * If source and destination entries are still the same, 1675 * a null copy is being performed. 1676 */ 1677 1678 if (src_entry == dst_entry) 1679 goto Return; 1680 1681 /* 1682 * Go through entries until we get to the end of the 1683 * region. 1684 */ 1685 1686 while (src_start < src_end) { 1687 /* 1688 * Clip the entries to the endpoint of the entire region. 1689 */ 1690 1691 vm_map_clip_end(src_map, src_entry, src_end); 1692 vm_map_clip_end(dst_map, dst_entry, dst_end); 1693 1694 /* 1695 * Clip each entry to the endpoint of the other entry. 1696 */ 1697 1698 src_clip = src_entry->start + (dst_entry->end - dst_entry->start); 1699 vm_map_clip_end(src_map, src_entry, src_clip); 1700 1701 dst_clip = dst_entry->start + (src_entry->end - src_entry->start); 1702 vm_map_clip_end(dst_map, dst_entry, dst_clip); 1703 1704 /* 1705 * Both entries now match in size and relative endpoints. 1706 * 1707 * If both entries refer to a VM object, we can 1708 * deal with them now. 1709 */ 1710 1711 if (!src_entry->is_a_map && !dst_entry->is_a_map) { 1712 vm_map_copy_entry(src_map, dst_map, src_entry, 1713 dst_entry); 1714 } 1715 else { 1716 register vm_map_t new_dst_map; 1717 vm_offset_t new_dst_start; 1718 vm_size_t new_size; 1719 vm_map_t new_src_map; 1720 vm_offset_t new_src_start; 1721 1722 /* 1723 * We have to follow at least one sharing map. 1724 */ 1725 1726 new_size = (dst_entry->end - dst_entry->start); 1727 1728 if (src_entry->is_a_map) { 1729 new_src_map = src_entry->object.share_map; 1730 new_src_start = src_entry->offset; 1731 } 1732 else { 1733 new_src_map = src_map; 1734 new_src_start = src_entry->start; 1735 lock_set_recursive(&src_map->lock); 1736 } 1737 1738 if (dst_entry->is_a_map) { 1739 vm_offset_t new_dst_end; 1740 1741 new_dst_map = dst_entry->object.share_map; 1742 new_dst_start = dst_entry->offset; 1743 1744 /* 1745 * Since the destination sharing entries 1746 * will be merely deallocated, we can 1747 * do that now, and replace the region 1748 * with a null object. [This prevents 1749 * splitting the source map to match 1750 * the form of the destination map.] 1751 * Note that we can only do so if the 1752 * source and destination do not overlap. 1753 */ 1754 1755 new_dst_end = new_dst_start + new_size; 1756 1757 if (new_dst_map != new_src_map) { 1758 vm_map_lock(new_dst_map); 1759 (void) vm_map_delete(new_dst_map, 1760 new_dst_start, 1761 new_dst_end); 1762 (void) vm_map_insert(new_dst_map, 1763 VM_OBJECT_NULL, 1764 (vm_offset_t) 0, 1765 new_dst_start, 1766 new_dst_end); 1767 vm_map_unlock(new_dst_map); 1768 } 1769 } 1770 else { 1771 new_dst_map = dst_map; 1772 new_dst_start = dst_entry->start; 1773 lock_set_recursive(&dst_map->lock); 1774 } 1775 1776 /* 1777 * Recursively copy the sharing map. 1778 */ 1779 1780 (void) vm_map_copy(new_dst_map, new_src_map, 1781 new_dst_start, new_size, new_src_start, 1782 FALSE, FALSE); 1783 1784 if (dst_map == new_dst_map) 1785 lock_clear_recursive(&dst_map->lock); 1786 if (src_map == new_src_map) 1787 lock_clear_recursive(&src_map->lock); 1788 } 1789 1790 /* 1791 * Update variables for next pass through the loop. 1792 */ 1793 1794 src_start = src_entry->end; 1795 src_entry = src_entry->next; 1796 dst_start = dst_entry->end; 1797 dst_entry = dst_entry->next; 1798 1799 /* 1800 * If the source is to be destroyed, here is the 1801 * place to do it. 1802 */ 1803 1804 if (src_destroy && src_map->is_main_map && 1805 dst_map->is_main_map) 1806 vm_map_entry_delete(src_map, src_entry->prev); 1807 } 1808 1809 /* 1810 * Update the physical maps as appropriate 1811 */ 1812 1813 if (src_map->is_main_map && dst_map->is_main_map) { 1814 if (src_destroy) 1815 pmap_remove(src_map->pmap, src_addr, src_addr + len); 1816 } 1817 1818 /* 1819 * Unlock the maps 1820 */ 1821 1822 Return: ; 1823 1824 if (old_src_destroy) 1825 vm_map_delete(src_map, src_addr, src_addr + len); 1826 1827 vm_map_unlock(src_map); 1828 if (src_map != dst_map) 1829 vm_map_unlock(dst_map); 1830 1831 return(result); 1832 } 1833 1834 /* 1835 * vm_map_fork: 1836 * 1837 * Create and return a new map based on the old 1838 * map, according to the inheritance values on the 1839 * regions in that map. 1840 * 1841 * The source map must not be locked. 1842 */ 1843 vm_map_t vm_map_fork(old_map) 1844 vm_map_t old_map; 1845 { 1846 vm_map_t new_map; 1847 vm_map_entry_t old_entry; 1848 vm_map_entry_t new_entry; 1849 pmap_t new_pmap; 1850 1851 vm_map_lock(old_map); 1852 1853 new_pmap = pmap_create((vm_size_t) 0); 1854 new_map = vm_map_create(new_pmap, 1855 old_map->min_offset, 1856 old_map->max_offset, 1857 old_map->entries_pageable); 1858 1859 old_entry = old_map->header.next; 1860 1861 while (old_entry != &old_map->header) { 1862 if (old_entry->is_sub_map) 1863 panic("vm_map_fork: encountered a submap"); 1864 1865 switch (old_entry->inheritance) { 1866 case VM_INHERIT_NONE: 1867 break; 1868 1869 case VM_INHERIT_SHARE: 1870 /* 1871 * If we don't already have a sharing map: 1872 */ 1873 1874 if (!old_entry->is_a_map) { 1875 vm_map_t new_share_map; 1876 vm_map_entry_t new_share_entry; 1877 1878 /* 1879 * Create a new sharing map 1880 */ 1881 1882 new_share_map = vm_map_create(PMAP_NULL, 1883 old_entry->start, 1884 old_entry->end, 1885 TRUE); 1886 new_share_map->is_main_map = FALSE; 1887 1888 /* 1889 * Create the only sharing entry from the 1890 * old task map entry. 1891 */ 1892 1893 new_share_entry = 1894 vm_map_entry_create(new_share_map); 1895 *new_share_entry = *old_entry; 1896 1897 /* 1898 * Insert the entry into the new sharing 1899 * map 1900 */ 1901 1902 vm_map_entry_link(new_share_map, 1903 new_share_map->header.prev, 1904 new_share_entry); 1905 1906 /* 1907 * Fix up the task map entry to refer 1908 * to the sharing map now. 1909 */ 1910 1911 old_entry->is_a_map = TRUE; 1912 old_entry->object.share_map = new_share_map; 1913 old_entry->offset = old_entry->start; 1914 } 1915 1916 /* 1917 * Clone the entry, referencing the sharing map. 1918 */ 1919 1920 new_entry = vm_map_entry_create(new_map); 1921 *new_entry = *old_entry; 1922 vm_map_reference(new_entry->object.share_map); 1923 1924 /* 1925 * Insert the entry into the new map -- we 1926 * know we're inserting at the end of the new 1927 * map. 1928 */ 1929 1930 vm_map_entry_link(new_map, new_map->header.prev, 1931 new_entry); 1932 1933 /* 1934 * Update the physical map 1935 */ 1936 1937 pmap_copy(new_map->pmap, old_map->pmap, 1938 new_entry->start, 1939 (old_entry->end - old_entry->start), 1940 old_entry->start); 1941 break; 1942 1943 case VM_INHERIT_COPY: 1944 /* 1945 * Clone the entry and link into the map. 1946 */ 1947 1948 new_entry = vm_map_entry_create(new_map); 1949 *new_entry = *old_entry; 1950 new_entry->wired_count = 0; 1951 new_entry->object.vm_object = VM_OBJECT_NULL; 1952 new_entry->is_a_map = FALSE; 1953 vm_map_entry_link(new_map, new_map->header.prev, 1954 new_entry); 1955 if (old_entry->is_a_map) { 1956 int check; 1957 1958 check = vm_map_copy(new_map, 1959 old_entry->object.share_map, 1960 new_entry->start, 1961 (vm_size_t)(new_entry->end - 1962 new_entry->start), 1963 old_entry->offset, 1964 FALSE, FALSE); 1965 if (check != KERN_SUCCESS) 1966 printf("vm_map_fork: copy in share_map region failed\n"); 1967 } 1968 else { 1969 vm_map_copy_entry(old_map, new_map, old_entry, 1970 new_entry); 1971 } 1972 break; 1973 } 1974 old_entry = old_entry->next; 1975 } 1976 1977 new_map->size = old_map->size; 1978 vm_map_unlock(old_map); 1979 1980 return(new_map); 1981 } 1982 1983 /* 1984 * vm_map_lookup: 1985 * 1986 * Finds the VM object, offset, and 1987 * protection for a given virtual address in the 1988 * specified map, assuming a page fault of the 1989 * type specified. 1990 * 1991 * Leaves the map in question locked for read; return 1992 * values are guaranteed until a vm_map_lookup_done 1993 * call is performed. Note that the map argument 1994 * is in/out; the returned map must be used in 1995 * the call to vm_map_lookup_done. 1996 * 1997 * A handle (out_entry) is returned for use in 1998 * vm_map_lookup_done, to make that fast. 1999 * 2000 * If a lookup is requested with "write protection" 2001 * specified, the map may be changed to perform virtual 2002 * copying operations, although the data referenced will 2003 * remain the same. 2004 */ 2005 vm_map_lookup(var_map, vaddr, fault_type, out_entry, 2006 object, offset, out_prot, wired, single_use) 2007 vm_map_t *var_map; /* IN/OUT */ 2008 register vm_offset_t vaddr; 2009 register vm_prot_t fault_type; 2010 2011 vm_map_entry_t *out_entry; /* OUT */ 2012 vm_object_t *object; /* OUT */ 2013 vm_offset_t *offset; /* OUT */ 2014 vm_prot_t *out_prot; /* OUT */ 2015 boolean_t *wired; /* OUT */ 2016 boolean_t *single_use; /* OUT */ 2017 { 2018 vm_map_t share_map; 2019 vm_offset_t share_offset; 2020 register vm_map_entry_t entry; 2021 register vm_map_t map = *var_map; 2022 register vm_prot_t prot; 2023 register boolean_t su; 2024 2025 RetryLookup: ; 2026 2027 /* 2028 * Lookup the faulting address. 2029 */ 2030 2031 vm_map_lock_read(map); 2032 2033 #define RETURN(why) \ 2034 { \ 2035 vm_map_unlock_read(map); \ 2036 return(why); \ 2037 } 2038 2039 /* 2040 * If the map has an interesting hint, try it before calling 2041 * full blown lookup routine. 2042 */ 2043 2044 simple_lock(&map->hint_lock); 2045 entry = map->hint; 2046 simple_unlock(&map->hint_lock); 2047 2048 *out_entry = entry; 2049 2050 if ((entry == &map->header) || 2051 (vaddr < entry->start) || (vaddr >= entry->end)) { 2052 vm_map_entry_t tmp_entry; 2053 2054 /* 2055 * Entry was either not a valid hint, or the vaddr 2056 * was not contained in the entry, so do a full lookup. 2057 */ 2058 if (!vm_map_lookup_entry(map, vaddr, &tmp_entry)) 2059 RETURN(KERN_INVALID_ADDRESS); 2060 2061 entry = tmp_entry; 2062 *out_entry = entry; 2063 } 2064 2065 /* 2066 * Handle submaps. 2067 */ 2068 2069 if (entry->is_sub_map) { 2070 vm_map_t old_map = map; 2071 2072 *var_map = map = entry->object.sub_map; 2073 vm_map_unlock_read(old_map); 2074 goto RetryLookup; 2075 } 2076 2077 /* 2078 * Check whether this task is allowed to have 2079 * this page. 2080 */ 2081 2082 prot = entry->protection; 2083 if ((fault_type & (prot)) != fault_type) 2084 RETURN(KERN_PROTECTION_FAILURE); 2085 2086 /* 2087 * If this page is not pageable, we have to get 2088 * it for all possible accesses. 2089 */ 2090 2091 if (*wired = (entry->wired_count != 0)) 2092 prot = fault_type = entry->protection; 2093 2094 /* 2095 * If we don't already have a VM object, track 2096 * it down. 2097 */ 2098 2099 if (su = !entry->is_a_map) { 2100 share_map = map; 2101 share_offset = vaddr; 2102 } 2103 else { 2104 vm_map_entry_t share_entry; 2105 2106 /* 2107 * Compute the sharing map, and offset into it. 2108 */ 2109 2110 share_map = entry->object.share_map; 2111 share_offset = (vaddr - entry->start) + entry->offset; 2112 2113 /* 2114 * Look for the backing store object and offset 2115 */ 2116 2117 vm_map_lock_read(share_map); 2118 2119 if (!vm_map_lookup_entry(share_map, share_offset, 2120 &share_entry)) { 2121 vm_map_unlock_read(share_map); 2122 RETURN(KERN_INVALID_ADDRESS); 2123 } 2124 entry = share_entry; 2125 } 2126 2127 /* 2128 * If the entry was copy-on-write, we either ... 2129 */ 2130 2131 if (entry->needs_copy) { 2132 /* 2133 * If we want to write the page, we may as well 2134 * handle that now since we've got the sharing 2135 * map locked. 2136 * 2137 * If we don't need to write the page, we just 2138 * demote the permissions allowed. 2139 */ 2140 2141 if (fault_type & VM_PROT_WRITE) { 2142 /* 2143 * Make a new object, and place it in the 2144 * object chain. Note that no new references 2145 * have appeared -- one just moved from the 2146 * share map to the new object. 2147 */ 2148 2149 if (lock_read_to_write(&share_map->lock)) { 2150 if (share_map != map) 2151 vm_map_unlock_read(map); 2152 goto RetryLookup; 2153 } 2154 2155 vm_object_shadow( 2156 &entry->object.vm_object, 2157 &entry->offset, 2158 (vm_size_t) (entry->end - entry->start)); 2159 2160 entry->needs_copy = FALSE; 2161 2162 lock_write_to_read(&share_map->lock); 2163 } 2164 else { 2165 /* 2166 * We're attempting to read a copy-on-write 2167 * page -- don't allow writes. 2168 */ 2169 2170 prot &= (~VM_PROT_WRITE); 2171 } 2172 } 2173 2174 /* 2175 * Create an object if necessary. 2176 */ 2177 if (entry->object.vm_object == VM_OBJECT_NULL) { 2178 2179 if (lock_read_to_write(&share_map->lock)) { 2180 if (share_map != map) 2181 vm_map_unlock_read(map); 2182 goto RetryLookup; 2183 } 2184 2185 entry->object.vm_object = vm_object_allocate( 2186 (vm_size_t)(entry->end - entry->start)); 2187 entry->offset = 0; 2188 lock_write_to_read(&share_map->lock); 2189 } 2190 2191 /* 2192 * Return the object/offset from this entry. If the entry 2193 * was copy-on-write or empty, it has been fixed up. 2194 */ 2195 2196 *offset = (share_offset - entry->start) + entry->offset; 2197 *object = entry->object.vm_object; 2198 2199 /* 2200 * Return whether this is the only map sharing this data. 2201 */ 2202 2203 if (!su) { 2204 simple_lock(&share_map->ref_lock); 2205 su = (share_map->ref_count == 1); 2206 simple_unlock(&share_map->ref_lock); 2207 } 2208 2209 *out_prot = prot; 2210 *single_use = su; 2211 2212 return(KERN_SUCCESS); 2213 2214 #undef RETURN 2215 } 2216 2217 /* 2218 * vm_map_lookup_done: 2219 * 2220 * Releases locks acquired by a vm_map_lookup 2221 * (according to the handle returned by that lookup). 2222 */ 2223 2224 void vm_map_lookup_done(map, entry) 2225 register vm_map_t map; 2226 vm_map_entry_t entry; 2227 { 2228 /* 2229 * If this entry references a map, unlock it first. 2230 */ 2231 2232 if (entry->is_a_map) 2233 vm_map_unlock_read(entry->object.share_map); 2234 2235 /* 2236 * Unlock the main-level map 2237 */ 2238 2239 vm_map_unlock_read(map); 2240 } 2241 2242 /* 2243 * Routine: vm_map_simplify 2244 * Purpose: 2245 * Attempt to simplify the map representation in 2246 * the vicinity of the given starting address. 2247 * Note: 2248 * This routine is intended primarily to keep the 2249 * kernel maps more compact -- they generally don't 2250 * benefit from the "expand a map entry" technology 2251 * at allocation time because the adjacent entry 2252 * is often wired down. 2253 */ 2254 void vm_map_simplify(map, start) 2255 vm_map_t map; 2256 vm_offset_t start; 2257 { 2258 vm_map_entry_t this_entry; 2259 vm_map_entry_t prev_entry; 2260 2261 vm_map_lock(map); 2262 if ( 2263 (vm_map_lookup_entry(map, start, &this_entry)) && 2264 ((prev_entry = this_entry->prev) != &map->header) && 2265 2266 (prev_entry->end == start) && 2267 (map->is_main_map) && 2268 2269 (prev_entry->is_a_map == FALSE) && 2270 (prev_entry->is_sub_map == FALSE) && 2271 2272 (this_entry->is_a_map == FALSE) && 2273 (this_entry->is_sub_map == FALSE) && 2274 2275 (prev_entry->inheritance == this_entry->inheritance) && 2276 (prev_entry->protection == this_entry->protection) && 2277 (prev_entry->max_protection == this_entry->max_protection) && 2278 (prev_entry->wired_count == this_entry->wired_count) && 2279 2280 (prev_entry->copy_on_write == this_entry->copy_on_write) && 2281 (prev_entry->needs_copy == this_entry->needs_copy) && 2282 2283 (prev_entry->object.vm_object == this_entry->object.vm_object) && 2284 ((prev_entry->offset + (prev_entry->end - prev_entry->start)) 2285 == this_entry->offset) 2286 ) { 2287 if (map->first_free == this_entry) 2288 map->first_free = prev_entry; 2289 2290 SAVE_HINT(map, prev_entry); 2291 vm_map_entry_unlink(map, this_entry); 2292 prev_entry->end = this_entry->end; 2293 vm_object_deallocate(this_entry->object.vm_object); 2294 vm_map_entry_dispose(map, this_entry); 2295 } 2296 vm_map_unlock(map); 2297 } 2298 2299 /* 2300 * vm_map_print: [ debug ] 2301 */ 2302 void vm_map_print(map, full) 2303 register vm_map_t map; 2304 boolean_t full; 2305 { 2306 register vm_map_entry_t entry; 2307 extern int indent; 2308 2309 iprintf("%s map 0x%x: pmap=0x%x,ref=%d,nentries=%d,version=%d\n", 2310 (map->is_main_map ? "Task" : "Share"), 2311 (int) map, (int) (map->pmap), map->ref_count, map->nentries, 2312 map->timestamp); 2313 2314 if (!full && indent) 2315 return; 2316 2317 indent += 2; 2318 for (entry = map->header.next; entry != &map->header; 2319 entry = entry->next) { 2320 iprintf("map entry 0x%x: start=0x%x, end=0x%x, ", 2321 (int) entry, (int) entry->start, (int) entry->end); 2322 if (map->is_main_map) { 2323 static char *inheritance_name[4] = 2324 { "share", "copy", "none", "donate_copy"}; 2325 printf("prot=%x/%x/%s, ", 2326 entry->protection, 2327 entry->max_protection, 2328 inheritance_name[entry->inheritance]); 2329 if (entry->wired_count != 0) 2330 printf("wired, "); 2331 } 2332 2333 if (entry->is_a_map || entry->is_sub_map) { 2334 printf("share=0x%x, offset=0x%x\n", 2335 (int) entry->object.share_map, 2336 (int) entry->offset); 2337 if ((entry->prev == &map->header) || 2338 (!entry->prev->is_a_map) || 2339 (entry->prev->object.share_map != 2340 entry->object.share_map)) { 2341 indent += 2; 2342 vm_map_print(entry->object.share_map, full); 2343 indent -= 2; 2344 } 2345 2346 } 2347 else { 2348 printf("object=0x%x, offset=0x%x", 2349 (int) entry->object.vm_object, 2350 (int) entry->offset); 2351 if (entry->copy_on_write) 2352 printf(", copy (%s)", 2353 entry->needs_copy ? "needed" : "done"); 2354 printf("\n"); 2355 2356 if ((entry->prev == &map->header) || 2357 (entry->prev->is_a_map) || 2358 (entry->prev->object.vm_object != 2359 entry->object.vm_object)) { 2360 indent += 2; 2361 vm_object_print(entry->object.vm_object, full); 2362 indent -= 2; 2363 } 2364 } 2365 } 2366 indent -= 2; 2367 } 2368