1 /* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_page.c 7.13 (Berkeley) 05/04/92 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Resident memory management module. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 46 #include <vm/vm.h> 47 #include <vm/vm_page.h> 48 #include <vm/vm_map.h> 49 #include <vm/vm_pageout.h> 50 51 /* 52 * Associated with page of user-allocatable memory is a 53 * page structure. 54 */ 55 56 queue_head_t *vm_page_buckets; /* Array of buckets */ 57 int vm_page_bucket_count = 0; /* How big is array? */ 58 int vm_page_hash_mask; /* Mask for hash function */ 59 simple_lock_data_t bucket_lock; /* lock for all buckets XXX */ 60 61 queue_head_t vm_page_queue_free; 62 queue_head_t vm_page_queue_active; 63 queue_head_t vm_page_queue_inactive; 64 simple_lock_data_t vm_page_queue_lock; 65 simple_lock_data_t vm_page_queue_free_lock; 66 67 /* has physical page allocation been initialized? */ 68 boolean_t vm_page_startup_initialized; 69 70 vm_page_t vm_page_array; 71 long first_page; 72 long last_page; 73 vm_offset_t first_phys_addr; 74 vm_offset_t last_phys_addr; 75 vm_size_t page_mask; 76 int page_shift; 77 78 /* 79 * vm_set_page_size: 80 * 81 * Sets the page size, perhaps based upon the memory 82 * size. Must be called before any use of page-size 83 * dependent functions. 84 * 85 * Sets page_shift and page_mask from cnt.v_page_size. 86 */ 87 void vm_set_page_size() 88 { 89 90 if (cnt.v_page_size == 0) 91 cnt.v_page_size = DEFAULT_PAGE_SIZE; 92 page_mask = cnt.v_page_size - 1; 93 if ((page_mask & cnt.v_page_size) != 0) 94 panic("vm_set_page_size: page size not a power of two"); 95 for (page_shift = 0; ; page_shift++) 96 if ((1 << page_shift) == cnt.v_page_size) 97 break; 98 } 99 100 101 /* 102 * vm_page_startup: 103 * 104 * Initializes the resident memory module. 105 * 106 * Allocates memory for the page cells, and 107 * for the object/offset-to-page hash table headers. 108 * Each page cell is initialized and placed on the free list. 109 */ 110 void vm_page_startup(start, end) 111 vm_offset_t *start; 112 vm_offset_t *end; 113 { 114 register vm_page_t m; 115 register queue_t bucket; 116 vm_size_t npages; 117 int i; 118 vm_offset_t pa; 119 extern vm_offset_t kentry_data; 120 extern vm_size_t kentry_data_size; 121 122 123 /* 124 * Initialize the locks 125 */ 126 127 simple_lock_init(&vm_page_queue_free_lock); 128 simple_lock_init(&vm_page_queue_lock); 129 130 /* 131 * Initialize the queue headers for the free queue, 132 * the active queue and the inactive queue. 133 */ 134 135 queue_init(&vm_page_queue_free); 136 queue_init(&vm_page_queue_active); 137 queue_init(&vm_page_queue_inactive); 138 139 /* 140 * Calculate the number of hash table buckets. 141 * 142 * The number of buckets MUST BE a power of 2, and 143 * the actual value is the next power of 2 greater 144 * than the number of physical pages in the system. 145 * 146 * Note: 147 * This computation can be tweaked if desired. 148 */ 149 150 if (vm_page_bucket_count == 0) { 151 vm_page_bucket_count = 1; 152 while (vm_page_bucket_count < atop(*end - *start)) 153 vm_page_bucket_count <<= 1; 154 } 155 156 vm_page_hash_mask = vm_page_bucket_count - 1; 157 158 /* 159 * Allocate (and initialize) the hash table buckets. 160 */ 161 vm_page_buckets = (queue_t) pmap_bootstrap_alloc(vm_page_bucket_count 162 * sizeof(struct queue_entry)); 163 bucket = vm_page_buckets; 164 165 for (i = vm_page_bucket_count; i--;) { 166 queue_init(bucket); 167 bucket++; 168 } 169 170 simple_lock_init(&bucket_lock); 171 172 /* 173 * Truncate the remainder of physical memory to our page size. 174 */ 175 176 *end = trunc_page(*end); 177 178 /* 179 * Pre-allocate maps and map entries that cannot be dynamically 180 * allocated via malloc(). The maps include the kernel_map and 181 * kmem_map which must be initialized before malloc() will 182 * work (obviously). Also could include pager maps which would 183 * be allocated before kmeminit. 184 * 185 * Allow some kernel map entries... this should be plenty 186 * since people shouldn't be cluttering up the kernel 187 * map (they should use their own maps). 188 */ 189 190 kentry_data_size = MAX_KMAP * sizeof(struct vm_map) + 191 MAX_KMAPENT * sizeof(struct vm_map_entry); 192 kentry_data = (vm_offset_t) pmap_bootstrap_alloc(kentry_data_size); 193 194 /* 195 * Compute the number of pages of memory that will be 196 * available for use (taking into account the overhead 197 * of a page structure per page). 198 */ 199 200 cnt.v_free_count = npages = 201 (*end - *start)/(PAGE_SIZE + sizeof(struct vm_page)); 202 203 /* 204 * Record the extent of physical memory that the 205 * virtual memory system manages. 206 */ 207 208 first_page = *start; 209 first_page += npages*sizeof(struct vm_page); 210 first_page = atop(round_page(first_page)); 211 last_page = first_page + npages - 1; 212 213 first_phys_addr = ptoa(first_page); 214 last_phys_addr = ptoa(last_page) + PAGE_MASK; 215 216 217 /* 218 * Allocate and clear the mem entry structures. 219 */ 220 221 m = vm_page_array = (vm_page_t) 222 pmap_bootstrap_alloc(npages * sizeof(struct vm_page)); 223 224 /* 225 * Initialize the mem entry structures now, and 226 * put them in the free queue. 227 */ 228 229 pa = first_phys_addr; 230 while (npages--) { 231 m->copy_on_write = FALSE; 232 m->wanted = FALSE; 233 m->inactive = FALSE; 234 m->active = FALSE; 235 m->busy = FALSE; 236 m->object = NULL; 237 m->phys_addr = pa; 238 #ifdef i386 239 if (pmap_isvalidphys(m->phys_addr)) { 240 queue_enter(&vm_page_queue_free, m, vm_page_t, pageq); 241 } else { 242 /* perhaps iomem needs it's own type, or dev pager? */ 243 m->fictitious = 1; 244 m->busy = TRUE; 245 cnt.v_free_count--; 246 } 247 #else /* i386 */ 248 queue_enter(&vm_page_queue_free, m, vm_page_t, pageq); 249 #endif /* i386 */ 250 m++; 251 pa += PAGE_SIZE; 252 } 253 254 /* 255 * Initialize vm_pages_needed lock here - don't wait for pageout 256 * daemon XXX 257 */ 258 simple_lock_init(&vm_pages_needed_lock); 259 260 /* from now on, pmap_bootstrap_alloc can't be used */ 261 vm_page_startup_initialized = TRUE; 262 } 263 264 /* 265 * vm_page_hash: 266 * 267 * Distributes the object/offset key pair among hash buckets. 268 * 269 * NOTE: This macro depends on vm_page_bucket_count being a power of 2. 270 */ 271 #define vm_page_hash(object, offset) \ 272 (((unsigned)object+(unsigned)atop(offset))&vm_page_hash_mask) 273 274 /* 275 * vm_page_insert: [ internal use only ] 276 * 277 * Inserts the given mem entry into the object/object-page 278 * table and object list. 279 * 280 * The object and page must be locked. 281 */ 282 283 void vm_page_insert(mem, object, offset) 284 register vm_page_t mem; 285 register vm_object_t object; 286 register vm_offset_t offset; 287 { 288 register queue_t bucket; 289 int spl; 290 291 VM_PAGE_CHECK(mem); 292 293 if (mem->tabled) 294 panic("vm_page_insert: already inserted"); 295 296 /* 297 * Record the object/offset pair in this page 298 */ 299 300 mem->object = object; 301 mem->offset = offset; 302 303 /* 304 * Insert it into the object_object/offset hash table 305 */ 306 307 bucket = &vm_page_buckets[vm_page_hash(object, offset)]; 308 spl = splimp(); 309 simple_lock(&bucket_lock); 310 queue_enter(bucket, mem, vm_page_t, hashq); 311 simple_unlock(&bucket_lock); 312 (void) splx(spl); 313 314 /* 315 * Now link into the object's list of backed pages. 316 */ 317 318 queue_enter(&object->memq, mem, vm_page_t, listq); 319 mem->tabled = TRUE; 320 321 /* 322 * And show that the object has one more resident 323 * page. 324 */ 325 326 object->resident_page_count++; 327 } 328 329 /* 330 * vm_page_remove: [ internal use only ] 331 * NOTE: used by device pager as well -wfj 332 * 333 * Removes the given mem entry from the object/offset-page 334 * table and the object page list. 335 * 336 * The object and page must be locked. 337 */ 338 339 void vm_page_remove(mem) 340 register vm_page_t mem; 341 { 342 register queue_t bucket; 343 int spl; 344 345 VM_PAGE_CHECK(mem); 346 347 if (!mem->tabled) 348 return; 349 350 /* 351 * Remove from the object_object/offset hash table 352 */ 353 354 bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)]; 355 spl = splimp(); 356 simple_lock(&bucket_lock); 357 queue_remove(bucket, mem, vm_page_t, hashq); 358 simple_unlock(&bucket_lock); 359 (void) splx(spl); 360 361 /* 362 * Now remove from the object's list of backed pages. 363 */ 364 365 queue_remove(&mem->object->memq, mem, vm_page_t, listq); 366 367 /* 368 * And show that the object has one fewer resident 369 * page. 370 */ 371 372 mem->object->resident_page_count--; 373 374 mem->tabled = FALSE; 375 } 376 377 /* 378 * vm_page_lookup: 379 * 380 * Returns the page associated with the object/offset 381 * pair specified; if none is found, NULL is returned. 382 * 383 * The object must be locked. No side effects. 384 */ 385 386 vm_page_t vm_page_lookup(object, offset) 387 register vm_object_t object; 388 register vm_offset_t offset; 389 { 390 register vm_page_t mem; 391 register queue_t bucket; 392 int spl; 393 394 /* 395 * Search the hash table for this object/offset pair 396 */ 397 398 bucket = &vm_page_buckets[vm_page_hash(object, offset)]; 399 400 spl = splimp(); 401 simple_lock(&bucket_lock); 402 mem = (vm_page_t) queue_first(bucket); 403 while (!queue_end(bucket, (queue_entry_t) mem)) { 404 VM_PAGE_CHECK(mem); 405 if ((mem->object == object) && (mem->offset == offset)) { 406 simple_unlock(&bucket_lock); 407 splx(spl); 408 return(mem); 409 } 410 mem = (vm_page_t) queue_next(&mem->hashq); 411 } 412 413 simple_unlock(&bucket_lock); 414 splx(spl); 415 return(NULL); 416 } 417 418 /* 419 * vm_page_rename: 420 * 421 * Move the given memory entry from its 422 * current object to the specified target object/offset. 423 * 424 * The object must be locked. 425 */ 426 void vm_page_rename(mem, new_object, new_offset) 427 register vm_page_t mem; 428 register vm_object_t new_object; 429 vm_offset_t new_offset; 430 { 431 if (mem->object == new_object) 432 return; 433 434 vm_page_lock_queues(); /* keep page from moving out from 435 under pageout daemon */ 436 vm_page_remove(mem); 437 vm_page_insert(mem, new_object, new_offset); 438 vm_page_unlock_queues(); 439 } 440 441 /* 442 * vm_page_alloc: 443 * 444 * Allocate and return a memory cell associated 445 * with this VM object/offset pair. 446 * 447 * Object must be locked. 448 */ 449 vm_page_t vm_page_alloc(object, offset) 450 vm_object_t object; 451 vm_offset_t offset; 452 { 453 register vm_page_t mem; 454 int spl; 455 456 spl = splimp(); /* XXX */ 457 simple_lock(&vm_page_queue_free_lock); 458 if (queue_empty(&vm_page_queue_free)) { 459 simple_unlock(&vm_page_queue_free_lock); 460 splx(spl); 461 return(NULL); 462 } 463 464 queue_remove_first(&vm_page_queue_free, mem, vm_page_t, pageq); 465 466 cnt.v_free_count--; 467 simple_unlock(&vm_page_queue_free_lock); 468 splx(spl); 469 470 VM_PAGE_INIT(mem, object, offset); 471 472 /* 473 * Decide if we should poke the pageout daemon. 474 * We do this if the free count is less than the low 475 * water mark, or if the free count is less than the high 476 * water mark (but above the low water mark) and the inactive 477 * count is less than its target. 478 * 479 * We don't have the counts locked ... if they change a little, 480 * it doesn't really matter. 481 */ 482 483 if (cnt.v_free_count < cnt.v_free_min || 484 (cnt.v_free_count < cnt.v_free_target && 485 cnt.v_inactive_count < cnt.v_inactive_target)) 486 thread_wakeup((int)&vm_pages_needed); 487 return (mem); 488 } 489 490 /* 491 * vm_page_free: 492 * 493 * Returns the given page to the free list, 494 * disassociating it with any VM object. 495 * 496 * Object and page must be locked prior to entry. 497 */ 498 void vm_page_free(mem) 499 register vm_page_t mem; 500 { 501 vm_page_remove(mem); 502 if (mem->active) { 503 queue_remove(&vm_page_queue_active, mem, vm_page_t, pageq); 504 mem->active = FALSE; 505 cnt.v_active_count--; 506 } 507 508 if (mem->inactive) { 509 queue_remove(&vm_page_queue_inactive, mem, vm_page_t, pageq); 510 mem->inactive = FALSE; 511 cnt.v_inactive_count--; 512 } 513 514 if (!mem->fictitious) { 515 int spl; 516 517 spl = splimp(); 518 simple_lock(&vm_page_queue_free_lock); 519 queue_enter(&vm_page_queue_free, mem, vm_page_t, pageq); 520 521 cnt.v_free_count++; 522 simple_unlock(&vm_page_queue_free_lock); 523 splx(spl); 524 } 525 } 526 527 /* 528 * vm_page_wire: 529 * 530 * Mark this page as wired down by yet 531 * another map, removing it from paging queues 532 * as necessary. 533 * 534 * The page queues must be locked. 535 */ 536 void vm_page_wire(mem) 537 register vm_page_t mem; 538 { 539 VM_PAGE_CHECK(mem); 540 541 if (mem->wire_count == 0) { 542 if (mem->active) { 543 queue_remove(&vm_page_queue_active, mem, vm_page_t, 544 pageq); 545 cnt.v_active_count--; 546 mem->active = FALSE; 547 } 548 if (mem->inactive) { 549 queue_remove(&vm_page_queue_inactive, mem, vm_page_t, 550 pageq); 551 cnt.v_inactive_count--; 552 mem->inactive = FALSE; 553 } 554 cnt.v_wire_count++; 555 } 556 mem->wire_count++; 557 } 558 559 /* 560 * vm_page_unwire: 561 * 562 * Release one wiring of this page, potentially 563 * enabling it to be paged again. 564 * 565 * The page queues must be locked. 566 */ 567 void vm_page_unwire(mem) 568 register vm_page_t mem; 569 { 570 VM_PAGE_CHECK(mem); 571 572 mem->wire_count--; 573 if (mem->wire_count == 0) { 574 queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq); 575 cnt.v_active_count++; 576 mem->active = TRUE; 577 cnt.v_wire_count--; 578 } 579 } 580 581 /* 582 * vm_page_deactivate: 583 * 584 * Returns the given page to the inactive list, 585 * indicating that no physical maps have access 586 * to this page. [Used by the physical mapping system.] 587 * 588 * The page queues must be locked. 589 */ 590 void vm_page_deactivate(m) 591 register vm_page_t m; 592 { 593 VM_PAGE_CHECK(m); 594 595 /* 596 * Only move active pages -- ignore locked or already 597 * inactive ones. 598 */ 599 600 if (m->active) { 601 pmap_clear_reference(VM_PAGE_TO_PHYS(m)); 602 queue_remove(&vm_page_queue_active, m, vm_page_t, pageq); 603 queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq); 604 m->active = FALSE; 605 m->inactive = TRUE; 606 cnt.v_active_count--; 607 cnt.v_inactive_count++; 608 if (pmap_is_modified(VM_PAGE_TO_PHYS(m))) 609 m->clean = FALSE; 610 m->laundry = !m->clean; 611 } 612 } 613 614 /* 615 * vm_page_activate: 616 * 617 * Put the specified page on the active list (if appropriate). 618 * 619 * The page queues must be locked. 620 */ 621 622 void vm_page_activate(m) 623 register vm_page_t m; 624 { 625 VM_PAGE_CHECK(m); 626 627 if (m->inactive) { 628 queue_remove(&vm_page_queue_inactive, m, vm_page_t, 629 pageq); 630 cnt.v_inactive_count--; 631 m->inactive = FALSE; 632 } 633 if (m->wire_count == 0) { 634 if (m->active) 635 panic("vm_page_activate: already active"); 636 637 queue_enter(&vm_page_queue_active, m, vm_page_t, pageq); 638 m->active = TRUE; 639 cnt.v_active_count++; 640 } 641 } 642 643 /* 644 * vm_page_zero_fill: 645 * 646 * Zero-fill the specified page. 647 * Written as a standard pagein routine, to 648 * be used by the zero-fill object. 649 */ 650 651 boolean_t vm_page_zero_fill(m) 652 vm_page_t m; 653 { 654 VM_PAGE_CHECK(m); 655 656 m->clean = 0; 657 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 658 return(TRUE); 659 } 660 661 /* 662 * vm_page_copy: 663 * 664 * Copy one page to another 665 */ 666 667 void vm_page_copy(src_m, dest_m) 668 vm_page_t src_m; 669 vm_page_t dest_m; 670 { 671 VM_PAGE_CHECK(src_m); 672 VM_PAGE_CHECK(dest_m); 673 674 dest_m->clean = 0; 675 pmap_copy_page(VM_PAGE_TO_PHYS(src_m), VM_PAGE_TO_PHYS(dest_m)); 676 } 677