1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * %sccs.include.redist.c% 9 * 10 * @(#)vm_page.c 8.1 (Berkeley) 06/11/93 11 * 12 * 13 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 14 * All rights reserved. 15 * 16 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 17 * 18 * Permission to use, copy, modify and distribute this software and 19 * its documentation is hereby granted, provided that both the copyright 20 * notice and this permission notice appear in all copies of the 21 * software, derivative works or modified versions, and any portions 22 * thereof, and that both notices appear in supporting documentation. 23 * 24 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 25 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 26 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 27 * 28 * Carnegie Mellon requests users of this software to return to 29 * 30 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 31 * School of Computer Science 32 * Carnegie Mellon University 33 * Pittsburgh PA 15213-3890 34 * 35 * any improvements or extensions that they make and grant Carnegie the 36 * rights to redistribute these changes. 37 */ 38 39 /* 40 * Resident memory management module. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 46 #include <vm/vm.h> 47 #include <vm/vm_page.h> 48 #include <vm/vm_map.h> 49 #include <vm/vm_pageout.h> 50 51 /* 52 * Associated with page of user-allocatable memory is a 53 * page structure. 54 */ 55 56 queue_head_t *vm_page_buckets; /* Array of buckets */ 57 int vm_page_bucket_count = 0; /* How big is array? */ 58 int vm_page_hash_mask; /* Mask for hash function */ 59 simple_lock_data_t bucket_lock; /* lock for all buckets XXX */ 60 61 queue_head_t vm_page_queue_free; 62 queue_head_t vm_page_queue_active; 63 queue_head_t vm_page_queue_inactive; 64 simple_lock_data_t vm_page_queue_lock; 65 simple_lock_data_t vm_page_queue_free_lock; 66 67 /* has physical page allocation been initialized? */ 68 boolean_t vm_page_startup_initialized; 69 70 vm_page_t vm_page_array; 71 long first_page; 72 long last_page; 73 vm_offset_t first_phys_addr; 74 vm_offset_t last_phys_addr; 75 vm_size_t page_mask; 76 int page_shift; 77 78 /* 79 * vm_set_page_size: 80 * 81 * Sets the page size, perhaps based upon the memory 82 * size. Must be called before any use of page-size 83 * dependent functions. 84 * 85 * Sets page_shift and page_mask from cnt.v_page_size. 86 */ 87 void vm_set_page_size() 88 { 89 90 if (cnt.v_page_size == 0) 91 cnt.v_page_size = DEFAULT_PAGE_SIZE; 92 page_mask = cnt.v_page_size - 1; 93 if ((page_mask & cnt.v_page_size) != 0) 94 panic("vm_set_page_size: page size not a power of two"); 95 for (page_shift = 0; ; page_shift++) 96 if ((1 << page_shift) == cnt.v_page_size) 97 break; 98 } 99 100 101 /* 102 * vm_page_startup: 103 * 104 * Initializes the resident memory module. 105 * 106 * Allocates memory for the page cells, and 107 * for the object/offset-to-page hash table headers. 108 * Each page cell is initialized and placed on the free list. 109 */ 110 void vm_page_startup(start, end) 111 vm_offset_t *start; 112 vm_offset_t *end; 113 { 114 register vm_page_t m; 115 register queue_t bucket; 116 vm_size_t npages; 117 int i; 118 vm_offset_t pa; 119 extern vm_offset_t kentry_data; 120 extern vm_size_t kentry_data_size; 121 122 123 /* 124 * Initialize the locks 125 */ 126 127 simple_lock_init(&vm_page_queue_free_lock); 128 simple_lock_init(&vm_page_queue_lock); 129 130 /* 131 * Initialize the queue headers for the free queue, 132 * the active queue and the inactive queue. 133 */ 134 135 queue_init(&vm_page_queue_free); 136 queue_init(&vm_page_queue_active); 137 queue_init(&vm_page_queue_inactive); 138 139 /* 140 * Calculate the number of hash table buckets. 141 * 142 * The number of buckets MUST BE a power of 2, and 143 * the actual value is the next power of 2 greater 144 * than the number of physical pages in the system. 145 * 146 * Note: 147 * This computation can be tweaked if desired. 148 */ 149 150 if (vm_page_bucket_count == 0) { 151 vm_page_bucket_count = 1; 152 while (vm_page_bucket_count < atop(*end - *start)) 153 vm_page_bucket_count <<= 1; 154 } 155 156 vm_page_hash_mask = vm_page_bucket_count - 1; 157 158 /* 159 * Allocate (and initialize) the hash table buckets. 160 */ 161 vm_page_buckets = (queue_t) pmap_bootstrap_alloc(vm_page_bucket_count 162 * sizeof(struct queue_entry)); 163 bucket = vm_page_buckets; 164 165 for (i = vm_page_bucket_count; i--;) { 166 queue_init(bucket); 167 bucket++; 168 } 169 170 simple_lock_init(&bucket_lock); 171 172 /* 173 * Truncate the remainder of physical memory to our page size. 174 */ 175 176 *end = trunc_page(*end); 177 178 /* 179 * Pre-allocate maps and map entries that cannot be dynamically 180 * allocated via malloc(). The maps include the kernel_map and 181 * kmem_map which must be initialized before malloc() will 182 * work (obviously). Also could include pager maps which would 183 * be allocated before kmeminit. 184 * 185 * Allow some kernel map entries... this should be plenty 186 * since people shouldn't be cluttering up the kernel 187 * map (they should use their own maps). 188 */ 189 190 kentry_data_size = MAX_KMAP * sizeof(struct vm_map) + 191 MAX_KMAPENT * sizeof(struct vm_map_entry); 192 kentry_data = (vm_offset_t) pmap_bootstrap_alloc(kentry_data_size); 193 194 /* 195 * Compute the number of pages of memory that will be 196 * available for use (taking into account the overhead 197 * of a page structure per page). 198 */ 199 200 cnt.v_free_count = npages = (*end - *start + sizeof(struct vm_page)) 201 / (PAGE_SIZE + sizeof(struct vm_page)); 202 203 /* 204 * Record the extent of physical memory that the 205 * virtual memory system manages. 206 */ 207 208 first_page = *start; 209 first_page += npages*sizeof(struct vm_page); 210 first_page = atop(round_page(first_page)); 211 last_page = first_page + npages - 1; 212 213 first_phys_addr = ptoa(first_page); 214 last_phys_addr = ptoa(last_page) + PAGE_MASK; 215 216 217 /* 218 * Allocate and clear the mem entry structures. 219 */ 220 221 m = vm_page_array = (vm_page_t) 222 pmap_bootstrap_alloc(npages * sizeof(struct vm_page)); 223 224 /* 225 * Initialize the mem entry structures now, and 226 * put them in the free queue. 227 */ 228 229 pa = first_phys_addr; 230 while (npages--) { 231 m->flags = 0; 232 m->object = NULL; 233 m->phys_addr = pa; 234 #ifdef i386 235 if (pmap_isvalidphys(m->phys_addr)) { 236 queue_enter(&vm_page_queue_free, m, vm_page_t, pageq); 237 } else { 238 /* perhaps iomem needs it's own type, or dev pager? */ 239 m->flags |= PG_FICTITIOUS | PG_BUSY; 240 cnt.v_free_count--; 241 } 242 #else /* i386 */ 243 queue_enter(&vm_page_queue_free, m, vm_page_t, pageq); 244 #endif /* i386 */ 245 m++; 246 pa += PAGE_SIZE; 247 } 248 249 /* 250 * Initialize vm_pages_needed lock here - don't wait for pageout 251 * daemon XXX 252 */ 253 simple_lock_init(&vm_pages_needed_lock); 254 255 /* from now on, pmap_bootstrap_alloc can't be used */ 256 vm_page_startup_initialized = TRUE; 257 } 258 259 /* 260 * vm_page_hash: 261 * 262 * Distributes the object/offset key pair among hash buckets. 263 * 264 * NOTE: This macro depends on vm_page_bucket_count being a power of 2. 265 */ 266 #define vm_page_hash(object, offset) \ 267 (((unsigned)object+(unsigned)atop(offset))&vm_page_hash_mask) 268 269 /* 270 * vm_page_insert: [ internal use only ] 271 * 272 * Inserts the given mem entry into the object/object-page 273 * table and object list. 274 * 275 * The object and page must be locked. 276 */ 277 278 void vm_page_insert(mem, object, offset) 279 register vm_page_t mem; 280 register vm_object_t object; 281 register vm_offset_t offset; 282 { 283 register queue_t bucket; 284 int spl; 285 286 VM_PAGE_CHECK(mem); 287 288 if (mem->flags & PG_TABLED) 289 panic("vm_page_insert: already inserted"); 290 291 /* 292 * Record the object/offset pair in this page 293 */ 294 295 mem->object = object; 296 mem->offset = offset; 297 298 /* 299 * Insert it into the object_object/offset hash table 300 */ 301 302 bucket = &vm_page_buckets[vm_page_hash(object, offset)]; 303 spl = splimp(); 304 simple_lock(&bucket_lock); 305 queue_enter(bucket, mem, vm_page_t, hashq); 306 simple_unlock(&bucket_lock); 307 (void) splx(spl); 308 309 /* 310 * Now link into the object's list of backed pages. 311 */ 312 313 queue_enter(&object->memq, mem, vm_page_t, listq); 314 mem->flags |= PG_TABLED; 315 316 /* 317 * And show that the object has one more resident 318 * page. 319 */ 320 321 object->resident_page_count++; 322 } 323 324 /* 325 * vm_page_remove: [ internal use only ] 326 * NOTE: used by device pager as well -wfj 327 * 328 * Removes the given mem entry from the object/offset-page 329 * table and the object page list. 330 * 331 * The object and page must be locked. 332 */ 333 334 void vm_page_remove(mem) 335 register vm_page_t mem; 336 { 337 register queue_t bucket; 338 int spl; 339 340 VM_PAGE_CHECK(mem); 341 342 if (!(mem->flags & PG_TABLED)) 343 return; 344 345 /* 346 * Remove from the object_object/offset hash table 347 */ 348 349 bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)]; 350 spl = splimp(); 351 simple_lock(&bucket_lock); 352 queue_remove(bucket, mem, vm_page_t, hashq); 353 simple_unlock(&bucket_lock); 354 (void) splx(spl); 355 356 /* 357 * Now remove from the object's list of backed pages. 358 */ 359 360 queue_remove(&mem->object->memq, mem, vm_page_t, listq); 361 362 /* 363 * And show that the object has one fewer resident 364 * page. 365 */ 366 367 mem->object->resident_page_count--; 368 369 mem->flags &= ~PG_TABLED; 370 } 371 372 /* 373 * vm_page_lookup: 374 * 375 * Returns the page associated with the object/offset 376 * pair specified; if none is found, NULL is returned. 377 * 378 * The object must be locked. No side effects. 379 */ 380 381 vm_page_t vm_page_lookup(object, offset) 382 register vm_object_t object; 383 register vm_offset_t offset; 384 { 385 register vm_page_t mem; 386 register queue_t bucket; 387 int spl; 388 389 /* 390 * Search the hash table for this object/offset pair 391 */ 392 393 bucket = &vm_page_buckets[vm_page_hash(object, offset)]; 394 395 spl = splimp(); 396 simple_lock(&bucket_lock); 397 mem = (vm_page_t) queue_first(bucket); 398 while (!queue_end(bucket, (queue_entry_t) mem)) { 399 VM_PAGE_CHECK(mem); 400 if ((mem->object == object) && (mem->offset == offset)) { 401 simple_unlock(&bucket_lock); 402 splx(spl); 403 return(mem); 404 } 405 mem = (vm_page_t) queue_next(&mem->hashq); 406 } 407 408 simple_unlock(&bucket_lock); 409 splx(spl); 410 return(NULL); 411 } 412 413 /* 414 * vm_page_rename: 415 * 416 * Move the given memory entry from its 417 * current object to the specified target object/offset. 418 * 419 * The object must be locked. 420 */ 421 void vm_page_rename(mem, new_object, new_offset) 422 register vm_page_t mem; 423 register vm_object_t new_object; 424 vm_offset_t new_offset; 425 { 426 if (mem->object == new_object) 427 return; 428 429 vm_page_lock_queues(); /* keep page from moving out from 430 under pageout daemon */ 431 vm_page_remove(mem); 432 vm_page_insert(mem, new_object, new_offset); 433 vm_page_unlock_queues(); 434 } 435 436 /* 437 * vm_page_alloc: 438 * 439 * Allocate and return a memory cell associated 440 * with this VM object/offset pair. 441 * 442 * Object must be locked. 443 */ 444 vm_page_t vm_page_alloc(object, offset) 445 vm_object_t object; 446 vm_offset_t offset; 447 { 448 register vm_page_t mem; 449 int spl; 450 451 spl = splimp(); /* XXX */ 452 simple_lock(&vm_page_queue_free_lock); 453 if (queue_empty(&vm_page_queue_free)) { 454 simple_unlock(&vm_page_queue_free_lock); 455 splx(spl); 456 return(NULL); 457 } 458 459 queue_remove_first(&vm_page_queue_free, mem, vm_page_t, pageq); 460 461 cnt.v_free_count--; 462 simple_unlock(&vm_page_queue_free_lock); 463 splx(spl); 464 465 VM_PAGE_INIT(mem, object, offset); 466 467 /* 468 * Decide if we should poke the pageout daemon. 469 * We do this if the free count is less than the low 470 * water mark, or if the free count is less than the high 471 * water mark (but above the low water mark) and the inactive 472 * count is less than its target. 473 * 474 * We don't have the counts locked ... if they change a little, 475 * it doesn't really matter. 476 */ 477 478 if (cnt.v_free_count < cnt.v_free_min || 479 (cnt.v_free_count < cnt.v_free_target && 480 cnt.v_inactive_count < cnt.v_inactive_target)) 481 thread_wakeup((int)&vm_pages_needed); 482 return (mem); 483 } 484 485 /* 486 * vm_page_free: 487 * 488 * Returns the given page to the free list, 489 * disassociating it with any VM object. 490 * 491 * Object and page must be locked prior to entry. 492 */ 493 void vm_page_free(mem) 494 register vm_page_t mem; 495 { 496 vm_page_remove(mem); 497 if (mem->flags & PG_ACTIVE) { 498 queue_remove(&vm_page_queue_active, mem, vm_page_t, pageq); 499 mem->flags &= ~PG_ACTIVE; 500 cnt.v_active_count--; 501 } 502 503 if (mem->flags & PG_INACTIVE) { 504 queue_remove(&vm_page_queue_inactive, mem, vm_page_t, pageq); 505 mem->flags &= ~PG_INACTIVE; 506 cnt.v_inactive_count--; 507 } 508 509 if (!(mem->flags & PG_FICTITIOUS)) { 510 int spl; 511 512 spl = splimp(); 513 simple_lock(&vm_page_queue_free_lock); 514 queue_enter(&vm_page_queue_free, mem, vm_page_t, pageq); 515 516 cnt.v_free_count++; 517 simple_unlock(&vm_page_queue_free_lock); 518 splx(spl); 519 } 520 } 521 522 /* 523 * vm_page_wire: 524 * 525 * Mark this page as wired down by yet 526 * another map, removing it from paging queues 527 * as necessary. 528 * 529 * The page queues must be locked. 530 */ 531 void vm_page_wire(mem) 532 register vm_page_t mem; 533 { 534 VM_PAGE_CHECK(mem); 535 536 if (mem->wire_count == 0) { 537 if (mem->flags & PG_ACTIVE) { 538 queue_remove(&vm_page_queue_active, mem, vm_page_t, 539 pageq); 540 cnt.v_active_count--; 541 mem->flags &= ~PG_ACTIVE; 542 } 543 if (mem->flags & PG_INACTIVE) { 544 queue_remove(&vm_page_queue_inactive, mem, vm_page_t, 545 pageq); 546 cnt.v_inactive_count--; 547 mem->flags &= ~PG_INACTIVE; 548 } 549 cnt.v_wire_count++; 550 } 551 mem->wire_count++; 552 } 553 554 /* 555 * vm_page_unwire: 556 * 557 * Release one wiring of this page, potentially 558 * enabling it to be paged again. 559 * 560 * The page queues must be locked. 561 */ 562 void vm_page_unwire(mem) 563 register vm_page_t mem; 564 { 565 VM_PAGE_CHECK(mem); 566 567 mem->wire_count--; 568 if (mem->wire_count == 0) { 569 queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq); 570 cnt.v_active_count++; 571 mem->flags |= PG_ACTIVE; 572 cnt.v_wire_count--; 573 } 574 } 575 576 /* 577 * vm_page_deactivate: 578 * 579 * Returns the given page to the inactive list, 580 * indicating that no physical maps have access 581 * to this page. [Used by the physical mapping system.] 582 * 583 * The page queues must be locked. 584 */ 585 void vm_page_deactivate(m) 586 register vm_page_t m; 587 { 588 VM_PAGE_CHECK(m); 589 590 /* 591 * Only move active pages -- ignore locked or already 592 * inactive ones. 593 */ 594 595 if (m->flags & PG_ACTIVE) { 596 pmap_clear_reference(VM_PAGE_TO_PHYS(m)); 597 queue_remove(&vm_page_queue_active, m, vm_page_t, pageq); 598 queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq); 599 m->flags &= ~PG_ACTIVE; 600 m->flags |= PG_INACTIVE; 601 cnt.v_active_count--; 602 cnt.v_inactive_count++; 603 if (pmap_is_modified(VM_PAGE_TO_PHYS(m))) 604 m->flags &= ~PG_CLEAN; 605 if (m->flags & PG_CLEAN) 606 m->flags &= ~PG_LAUNDRY; 607 else 608 m->flags |= PG_LAUNDRY; 609 } 610 } 611 612 /* 613 * vm_page_activate: 614 * 615 * Put the specified page on the active list (if appropriate). 616 * 617 * The page queues must be locked. 618 */ 619 620 void vm_page_activate(m) 621 register vm_page_t m; 622 { 623 VM_PAGE_CHECK(m); 624 625 if (m->flags & PG_INACTIVE) { 626 queue_remove(&vm_page_queue_inactive, m, vm_page_t, 627 pageq); 628 cnt.v_inactive_count--; 629 m->flags &= ~PG_INACTIVE; 630 } 631 if (m->wire_count == 0) { 632 if (m->flags & PG_ACTIVE) 633 panic("vm_page_activate: already active"); 634 635 queue_enter(&vm_page_queue_active, m, vm_page_t, pageq); 636 m->flags |= PG_ACTIVE; 637 cnt.v_active_count++; 638 } 639 } 640 641 /* 642 * vm_page_zero_fill: 643 * 644 * Zero-fill the specified page. 645 * Written as a standard pagein routine, to 646 * be used by the zero-fill object. 647 */ 648 649 boolean_t vm_page_zero_fill(m) 650 vm_page_t m; 651 { 652 VM_PAGE_CHECK(m); 653 654 m->flags &= ~PG_CLEAN; 655 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 656 return(TRUE); 657 } 658 659 /* 660 * vm_page_copy: 661 * 662 * Copy one page to another 663 */ 664 665 void vm_page_copy(src_m, dest_m) 666 vm_page_t src_m; 667 vm_page_t dest_m; 668 { 669 VM_PAGE_CHECK(src_m); 670 VM_PAGE_CHECK(dest_m); 671 672 dest_m->flags &= ~PG_CLEAN; 673 pmap_copy_page(VM_PAGE_TO_PHYS(src_m), VM_PAGE_TO_PHYS(dest_m)); 674 } 675