1 /* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_page.c 7.3 (Berkeley) 04/21/91 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Resident memory management module. 41 */ 42 43 #include "param.h" 44 45 #include "vm.h" 46 #include "vm_map.h" 47 #include "vm_page.h" 48 #include "vm_pageout.h" 49 50 /* 51 * Associated with page of user-allocatable memory is a 52 * page structure. 53 */ 54 55 queue_head_t *vm_page_buckets; /* Array of buckets */ 56 int vm_page_bucket_count = 0; /* How big is array? */ 57 int vm_page_hash_mask; /* Mask for hash function */ 58 simple_lock_data_t bucket_lock; /* lock for all buckets XXX */ 59 60 vm_size_t page_size = 4096; 61 vm_size_t page_mask = 4095; 62 int page_shift = 12; 63 64 queue_head_t vm_page_queue_free; 65 queue_head_t vm_page_queue_active; 66 queue_head_t vm_page_queue_inactive; 67 simple_lock_data_t vm_page_queue_lock; 68 simple_lock_data_t vm_page_queue_free_lock; 69 70 vm_page_t vm_page_array; 71 long first_page; 72 long last_page; 73 vm_offset_t first_phys_addr; 74 vm_offset_t last_phys_addr; 75 76 int vm_page_free_count; 77 int vm_page_active_count; 78 int vm_page_inactive_count; 79 int vm_page_wire_count; 80 int vm_page_laundry_count; 81 82 int vm_page_free_target = 0; 83 int vm_page_free_min = 0; 84 int vm_page_inactive_target = 0; 85 int vm_page_free_reserved = 0; 86 87 /* 88 * vm_set_page_size: 89 * 90 * Sets the page size, perhaps based upon the memory 91 * size. Must be called before any use of page-size 92 * dependent functions. 93 * 94 * Sets page_shift and page_mask from page_size. 95 */ 96 void vm_set_page_size() 97 { 98 page_mask = page_size - 1; 99 100 if ((page_mask & page_size) != 0) 101 panic("vm_set_page_size: page size not a power of two"); 102 103 for (page_shift = 0; ; page_shift++) 104 if ((1 << page_shift) == page_size) 105 break; 106 } 107 108 109 /* 110 * vm_page_startup: 111 * 112 * Initializes the resident memory module. 113 * 114 * Allocates memory for the page cells, and 115 * for the object/offset-to-page hash table headers. 116 * Each page cell is initialized and placed on the free list. 117 */ 118 vm_offset_t vm_page_startup(start, end, vaddr) 119 register vm_offset_t start; 120 vm_offset_t end; 121 register vm_offset_t vaddr; 122 { 123 register vm_offset_t mapped; 124 register vm_page_t m; 125 register queue_t bucket; 126 vm_size_t npages; 127 register vm_offset_t new_start; 128 int i; 129 vm_offset_t pa; 130 131 extern vm_offset_t kentry_data; 132 extern vm_size_t kentry_data_size; 133 134 135 /* 136 * Initialize the locks 137 */ 138 139 simple_lock_init(&vm_page_queue_free_lock); 140 simple_lock_init(&vm_page_queue_lock); 141 142 /* 143 * Initialize the queue headers for the free queue, 144 * the active queue and the inactive queue. 145 */ 146 147 queue_init(&vm_page_queue_free); 148 queue_init(&vm_page_queue_active); 149 queue_init(&vm_page_queue_inactive); 150 151 /* 152 * Allocate (and initialize) the hash table buckets. 153 * 154 * The number of buckets MUST BE a power of 2, and 155 * the actual value is the next power of 2 greater 156 * than the number of physical pages in the system. 157 * 158 * Note: 159 * This computation can be tweaked if desired. 160 */ 161 162 vm_page_buckets = (queue_t) vaddr; 163 bucket = vm_page_buckets; 164 if (vm_page_bucket_count == 0) { 165 vm_page_bucket_count = 1; 166 while (vm_page_bucket_count < atop(end - start)) 167 vm_page_bucket_count <<= 1; 168 } 169 170 vm_page_hash_mask = vm_page_bucket_count - 1; 171 172 /* 173 * Validate these addresses. 174 */ 175 176 new_start = round_page(((queue_t)start) + vm_page_bucket_count); 177 mapped = vaddr; 178 vaddr = pmap_map(mapped, start, new_start, 179 VM_PROT_READ|VM_PROT_WRITE); 180 start = new_start; 181 blkclr((caddr_t) mapped, vaddr - mapped); 182 mapped = vaddr; 183 184 for (i = vm_page_bucket_count; i--;) { 185 queue_init(bucket); 186 bucket++; 187 } 188 189 simple_lock_init(&bucket_lock); 190 191 /* 192 * round (or truncate) the addresses to our page size. 193 */ 194 195 end = trunc_page(end); 196 197 /* 198 * Pre-allocate maps and map entries that cannot be dynamically 199 * allocated via malloc(). The maps include the kernel_map and 200 * kmem_map which must be initialized before malloc() will 201 * work (obviously). Also could include pager maps which would 202 * be allocated before kmeminit. 203 * 204 * Allow some kernel map entries... this should be plenty 205 * since people shouldn't be cluttering up the kernel 206 * map (they should use their own maps). 207 */ 208 209 kentry_data_size = MAX_KMAP * sizeof(struct vm_map) + 210 MAX_KMAPENT * sizeof(struct vm_map_entry); 211 kentry_data_size = round_page(kentry_data_size); 212 kentry_data = (vm_offset_t) vaddr; 213 vaddr += kentry_data_size; 214 215 /* 216 * Validate these zone addresses. 217 */ 218 219 new_start = start + (vaddr - mapped); 220 pmap_map(mapped, start, new_start, VM_PROT_READ|VM_PROT_WRITE); 221 blkclr((caddr_t) mapped, (vaddr - mapped)); 222 mapped = vaddr; 223 start = new_start; 224 225 /* 226 * Compute the number of pages of memory that will be 227 * available for use (taking into account the overhead 228 * of a page structure per page). 229 */ 230 231 vm_page_free_count = npages = 232 (end - start)/(PAGE_SIZE + sizeof(struct vm_page)); 233 234 /* 235 * Initialize the mem entry structures now, and 236 * put them in the free queue. 237 */ 238 239 m = vm_page_array = (vm_page_t) vaddr; 240 first_page = start; 241 first_page += npages*sizeof(struct vm_page); 242 first_page = atop(round_page(first_page)); 243 last_page = first_page + npages - 1; 244 245 first_phys_addr = ptoa(first_page); 246 last_phys_addr = ptoa(last_page) + PAGE_MASK; 247 248 /* 249 * Validate these addresses. 250 */ 251 252 new_start = start + (round_page(m + npages) - mapped); 253 mapped = pmap_map(mapped, start, new_start, 254 VM_PROT_READ|VM_PROT_WRITE); 255 start = new_start; 256 257 /* 258 * Clear all of the page structures 259 */ 260 blkclr((caddr_t)m, npages * sizeof(*m)); 261 262 pa = first_phys_addr; 263 while (npages--) { 264 m->copy_on_write = FALSE; 265 m->wanted = FALSE; 266 m->inactive = FALSE; 267 m->active = FALSE; 268 m->busy = FALSE; 269 m->object = NULL; 270 m->phys_addr = pa; 271 queue_enter(&vm_page_queue_free, m, vm_page_t, pageq); 272 m++; 273 pa += PAGE_SIZE; 274 } 275 276 /* 277 * Initialize vm_pages_needed lock here - don't wait for pageout 278 * daemon XXX 279 */ 280 simple_lock_init(&vm_pages_needed_lock); 281 282 return(mapped); 283 } 284 285 /* 286 * vm_page_hash: 287 * 288 * Distributes the object/offset key pair among hash buckets. 289 * 290 * NOTE: This macro depends on vm_page_bucket_count being a power of 2. 291 */ 292 #define vm_page_hash(object, offset) \ 293 (((unsigned)object+(unsigned)atop(offset))&vm_page_hash_mask) 294 295 /* 296 * vm_page_insert: [ internal use only ] 297 * 298 * Inserts the given mem entry into the object/object-page 299 * table and object list. 300 * 301 * The object and page must be locked. 302 */ 303 304 void vm_page_insert(mem, object, offset) 305 register vm_page_t mem; 306 register vm_object_t object; 307 register vm_offset_t offset; 308 { 309 register queue_t bucket; 310 int spl; 311 312 VM_PAGE_CHECK(mem); 313 314 if (mem->tabled) 315 panic("vm_page_insert: already inserted"); 316 317 /* 318 * Record the object/offset pair in this page 319 */ 320 321 mem->object = object; 322 mem->offset = offset; 323 324 /* 325 * Insert it into the object_object/offset hash table 326 */ 327 328 bucket = &vm_page_buckets[vm_page_hash(object, offset)]; 329 spl = splimp(); 330 simple_lock(&bucket_lock); 331 queue_enter(bucket, mem, vm_page_t, hashq); 332 simple_unlock(&bucket_lock); 333 (void) splx(spl); 334 335 /* 336 * Now link into the object's list of backed pages. 337 */ 338 339 queue_enter(&object->memq, mem, vm_page_t, listq); 340 mem->tabled = TRUE; 341 342 /* 343 * And show that the object has one more resident 344 * page. 345 */ 346 347 object->resident_page_count++; 348 } 349 350 /* 351 * vm_page_remove: [ internal use only ] 352 * 353 * Removes the given mem entry from the object/offset-page 354 * table and the object page list. 355 * 356 * The object and page must be locked. 357 */ 358 359 void vm_page_remove(mem) 360 register vm_page_t mem; 361 { 362 register queue_t bucket; 363 int spl; 364 365 VM_PAGE_CHECK(mem); 366 367 if (!mem->tabled) 368 return; 369 370 /* 371 * Remove from the object_object/offset hash table 372 */ 373 374 bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)]; 375 spl = splimp(); 376 simple_lock(&bucket_lock); 377 queue_remove(bucket, mem, vm_page_t, hashq); 378 simple_unlock(&bucket_lock); 379 (void) splx(spl); 380 381 /* 382 * Now remove from the object's list of backed pages. 383 */ 384 385 queue_remove(&mem->object->memq, mem, vm_page_t, listq); 386 387 /* 388 * And show that the object has one fewer resident 389 * page. 390 */ 391 392 mem->object->resident_page_count--; 393 394 mem->tabled = FALSE; 395 } 396 397 /* 398 * vm_page_lookup: 399 * 400 * Returns the page associated with the object/offset 401 * pair specified; if none is found, NULL is returned. 402 * 403 * The object must be locked. No side effects. 404 */ 405 406 vm_page_t vm_page_lookup(object, offset) 407 register vm_object_t object; 408 register vm_offset_t offset; 409 { 410 register vm_page_t mem; 411 register queue_t bucket; 412 int spl; 413 414 /* 415 * Search the hash table for this object/offset pair 416 */ 417 418 bucket = &vm_page_buckets[vm_page_hash(object, offset)]; 419 420 spl = splimp(); 421 simple_lock(&bucket_lock); 422 mem = (vm_page_t) queue_first(bucket); 423 while (!queue_end(bucket, (queue_entry_t) mem)) { 424 VM_PAGE_CHECK(mem); 425 if ((mem->object == object) && (mem->offset == offset)) { 426 simple_unlock(&bucket_lock); 427 splx(spl); 428 return(mem); 429 } 430 mem = (vm_page_t) queue_next(&mem->hashq); 431 } 432 433 simple_unlock(&bucket_lock); 434 splx(spl); 435 return(NULL); 436 } 437 438 /* 439 * vm_page_rename: 440 * 441 * Move the given memory entry from its 442 * current object to the specified target object/offset. 443 * 444 * The object must be locked. 445 */ 446 void vm_page_rename(mem, new_object, new_offset) 447 register vm_page_t mem; 448 register vm_object_t new_object; 449 vm_offset_t new_offset; 450 { 451 if (mem->object == new_object) 452 return; 453 454 vm_page_lock_queues(); /* keep page from moving out from 455 under pageout daemon */ 456 vm_page_remove(mem); 457 vm_page_insert(mem, new_object, new_offset); 458 vm_page_unlock_queues(); 459 } 460 461 void vm_page_init(mem, object, offset) 462 vm_page_t mem; 463 vm_object_t object; 464 vm_offset_t offset; 465 { 466 #define vm_page_init(mem, object, offset) {\ 467 (mem)->busy = TRUE; \ 468 (mem)->tabled = FALSE; \ 469 vm_page_insert((mem), (object), (offset)); \ 470 (mem)->absent = FALSE; \ 471 (mem)->fictitious = FALSE; \ 472 (mem)->page_lock = VM_PROT_NONE; \ 473 (mem)->unlock_request = VM_PROT_NONE; \ 474 (mem)->laundry = FALSE; \ 475 (mem)->active = FALSE; \ 476 (mem)->inactive = FALSE; \ 477 (mem)->wire_count = 0; \ 478 (mem)->clean = TRUE; \ 479 (mem)->copy_on_write = FALSE; \ 480 (mem)->fake = TRUE; \ 481 } 482 483 vm_page_init(mem, object, offset); 484 } 485 486 /* 487 * vm_page_alloc: 488 * 489 * Allocate and return a memory cell associated 490 * with this VM object/offset pair. 491 * 492 * Object must be locked. 493 */ 494 vm_page_t vm_page_alloc(object, offset) 495 vm_object_t object; 496 vm_offset_t offset; 497 { 498 register vm_page_t mem; 499 int spl; 500 501 spl = splimp(); /* XXX */ 502 simple_lock(&vm_page_queue_free_lock); 503 if (queue_empty(&vm_page_queue_free)) { 504 simple_unlock(&vm_page_queue_free_lock); 505 splx(spl); 506 return(NULL); 507 } 508 509 queue_remove_first(&vm_page_queue_free, mem, vm_page_t, pageq); 510 511 vm_page_free_count--; 512 simple_unlock(&vm_page_queue_free_lock); 513 splx(spl); 514 515 vm_page_init(mem, object, offset); 516 517 /* 518 * Decide if we should poke the pageout daemon. 519 * We do this if the free count is less than the low 520 * water mark, or if the free count is less than the high 521 * water mark (but above the low water mark) and the inactive 522 * count is less than its target. 523 * 524 * We don't have the counts locked ... if they change a little, 525 * it doesn't really matter. 526 */ 527 528 if ((vm_page_free_count < vm_page_free_min) || 529 ((vm_page_free_count < vm_page_free_target) && 530 (vm_page_inactive_count < vm_page_inactive_target))) 531 thread_wakeup(&vm_pages_needed); 532 return(mem); 533 } 534 535 /* 536 * vm_page_free: 537 * 538 * Returns the given page to the free list, 539 * disassociating it with any VM object. 540 * 541 * Object and page must be locked prior to entry. 542 */ 543 void vm_page_free(mem) 544 register vm_page_t mem; 545 { 546 vm_page_remove(mem); 547 if (mem->active) { 548 queue_remove(&vm_page_queue_active, mem, vm_page_t, pageq); 549 mem->active = FALSE; 550 vm_page_active_count--; 551 } 552 553 if (mem->inactive) { 554 queue_remove(&vm_page_queue_inactive, mem, vm_page_t, pageq); 555 mem->inactive = FALSE; 556 vm_page_inactive_count--; 557 } 558 559 if (!mem->fictitious) { 560 int spl; 561 562 spl = splimp(); 563 simple_lock(&vm_page_queue_free_lock); 564 queue_enter(&vm_page_queue_free, mem, vm_page_t, pageq); 565 566 vm_page_free_count++; 567 simple_unlock(&vm_page_queue_free_lock); 568 splx(spl); 569 } 570 } 571 572 /* 573 * vm_page_wire: 574 * 575 * Mark this page as wired down by yet 576 * another map, removing it from paging queues 577 * as necessary. 578 * 579 * The page queues must be locked. 580 */ 581 void vm_page_wire(mem) 582 register vm_page_t mem; 583 { 584 VM_PAGE_CHECK(mem); 585 586 if (mem->wire_count == 0) { 587 if (mem->active) { 588 queue_remove(&vm_page_queue_active, mem, vm_page_t, 589 pageq); 590 vm_page_active_count--; 591 mem->active = FALSE; 592 } 593 if (mem->inactive) { 594 queue_remove(&vm_page_queue_inactive, mem, vm_page_t, 595 pageq); 596 vm_page_inactive_count--; 597 mem->inactive = FALSE; 598 } 599 vm_page_wire_count++; 600 } 601 mem->wire_count++; 602 } 603 604 /* 605 * vm_page_unwire: 606 * 607 * Release one wiring of this page, potentially 608 * enabling it to be paged again. 609 * 610 * The page queues must be locked. 611 */ 612 void vm_page_unwire(mem) 613 register vm_page_t mem; 614 { 615 VM_PAGE_CHECK(mem); 616 617 mem->wire_count--; 618 if (mem->wire_count == 0) { 619 queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq); 620 vm_page_active_count++; 621 mem->active = TRUE; 622 vm_page_wire_count--; 623 } 624 } 625 626 /* 627 * vm_page_deactivate: 628 * 629 * Returns the given page to the inactive list, 630 * indicating that no physical maps have access 631 * to this page. [Used by the physical mapping system.] 632 * 633 * The page queues must be locked. 634 */ 635 void vm_page_deactivate(m) 636 register vm_page_t m; 637 { 638 VM_PAGE_CHECK(m); 639 640 /* 641 * Only move active pages -- ignore locked or already 642 * inactive ones. 643 */ 644 645 if (m->active) { 646 pmap_clear_reference(VM_PAGE_TO_PHYS(m)); 647 queue_remove(&vm_page_queue_active, m, vm_page_t, pageq); 648 queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq); 649 m->active = FALSE; 650 m->inactive = TRUE; 651 vm_page_active_count--; 652 vm_page_inactive_count++; 653 if (pmap_is_modified(VM_PAGE_TO_PHYS(m))) 654 m->clean = FALSE; 655 m->laundry = !m->clean; 656 } 657 } 658 659 /* 660 * vm_page_activate: 661 * 662 * Put the specified page on the active list (if appropriate). 663 * 664 * The page queues must be locked. 665 */ 666 667 void vm_page_activate(m) 668 register vm_page_t m; 669 { 670 VM_PAGE_CHECK(m); 671 672 if (m->inactive) { 673 queue_remove(&vm_page_queue_inactive, m, vm_page_t, 674 pageq); 675 vm_page_inactive_count--; 676 m->inactive = FALSE; 677 } 678 if (m->wire_count == 0) { 679 if (m->active) 680 panic("vm_page_activate: already active"); 681 682 queue_enter(&vm_page_queue_active, m, vm_page_t, pageq); 683 m->active = TRUE; 684 vm_page_active_count++; 685 } 686 } 687 688 /* 689 * vm_page_zero_fill: 690 * 691 * Zero-fill the specified page. 692 * Written as a standard pagein routine, to 693 * be used by the zero-fill object. 694 */ 695 696 boolean_t vm_page_zero_fill(m) 697 vm_page_t m; 698 { 699 VM_PAGE_CHECK(m); 700 701 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 702 return(TRUE); 703 } 704 705 /* 706 * vm_page_copy: 707 * 708 * Copy one page to another 709 */ 710 711 void vm_page_copy(src_m, dest_m) 712 vm_page_t src_m; 713 vm_page_t dest_m; 714 { 715 VM_PAGE_CHECK(src_m); 716 VM_PAGE_CHECK(dest_m); 717 718 pmap_copy_page(VM_PAGE_TO_PHYS(src_m), VM_PAGE_TO_PHYS(dest_m)); 719 } 720