1 /*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 1991 Regents of the University of California. 5 * All rights reserved. 6 * Copyright (c) 1994 John S. Dyson 7 * All rights reserved. 8 * Copyright (c) 1994 David Greenman 9 * All rights reserved. 10 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu> 11 * All rights reserved. 12 * 13 * This code is derived from software contributed to Berkeley by 14 * the Systems Programming Group of the University of Utah Computer 15 * Science Department and William Jolitz of UUNET Technologies Inc. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. All advertising materials mentioning features or use of this software 26 * must display the following acknowledgement: 27 * This product includes software developed by the University of 28 * California, Berkeley and its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 46 */ 47 /*- 48 * Copyright (c) 2003 Networks Associates Technology, Inc. 49 * All rights reserved. 50 * Copyright (c) 2018 The FreeBSD Foundation 51 * All rights reserved. 52 * 53 * This software was developed for the FreeBSD Project by Jake Burkholder, 54 * Safeport Network Services, and Network Associates Laboratories, the 55 * Security Research Division of Network Associates, Inc. under 56 * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA 57 * CHATS research program. 58 * 59 * Portions of this software were developed by 60 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 61 * the FreeBSD Foundation. 62 * 63 * Redistribution and use in source and binary forms, with or without 64 * modification, are permitted provided that the following conditions 65 * are met: 66 * 1. Redistributions of source code must retain the above copyright 67 * notice, this list of conditions and the following disclaimer. 68 * 2. Redistributions in binary form must reproduce the above copyright 69 * notice, this list of conditions and the following disclaimer in the 70 * documentation and/or other materials provided with the distribution. 71 * 72 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 73 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 74 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 75 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 76 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 77 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 78 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 79 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 80 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 81 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 82 * SUCH DAMAGE. 83 */ 84 85 #include <sys/cdefs.h> 86 __FBSDID("$FreeBSD$"); 87 88 #include "opt_apic.h" 89 #include "opt_cpu.h" 90 #include "opt_pmap.h" 91 #include "opt_smp.h" 92 #include "opt_vm.h" 93 94 #include <sys/param.h> 95 #include <sys/systm.h> 96 #include <sys/kernel.h> 97 #include <sys/vmmeter.h> 98 #include <sys/sysctl.h> 99 #include <machine/bootinfo.h> 100 #include <machine/cpu.h> 101 #include <machine/cputypes.h> 102 #include <machine/md_var.h> 103 #ifdef DEV_APIC 104 #include <sys/bus.h> 105 #include <machine/intr_machdep.h> 106 #include <x86/apicvar.h> 107 #endif 108 #include <x86/ifunc.h> 109 110 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 111 "VM/pmap parameters"); 112 113 #include <machine/vmparam.h> 114 #include <vm/vm.h> 115 #include <vm/vm_page.h> 116 #include <vm/pmap.h> 117 #include <machine/pmap_base.h> 118 119 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 120 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 121 122 int unmapped_buf_allowed = 1; 123 124 int pti; 125 126 u_long physfree; /* phys addr of next free page */ 127 u_long vm86phystk; /* PA of vm86/bios stack */ 128 u_long vm86paddr; /* address of vm86 region */ 129 int vm86pa; /* phys addr of vm86 region */ 130 u_long KERNend; /* phys addr end of kernel (just after bss) */ 131 u_long KPTphys; /* phys addr of kernel page tables */ 132 caddr_t ptvmmap = 0; 133 vm_offset_t kernel_vm_end; 134 135 int i386_pmap_VM_NFREEORDER; 136 int i386_pmap_VM_LEVEL_0_ORDER; 137 int i386_pmap_PDRSHIFT; 138 139 int pat_works = 1; 140 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, 141 &pat_works, 0, 142 "Is page attribute table fully functional?"); 143 144 int pg_ps_enabled = 1; 145 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 146 &pg_ps_enabled, 0, 147 "Are large page mappings enabled?"); 148 149 int pv_entry_max = 0; 150 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, 151 &pv_entry_max, 0, 152 "Max number of PV entries"); 153 154 int pv_entry_count = 0; 155 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, 156 &pv_entry_count, 0, 157 "Current number of pv entries"); 158 159 #ifndef PMAP_SHPGPERPROC 160 #define PMAP_SHPGPERPROC 200 161 #endif 162 163 int shpgperproc = PMAP_SHPGPERPROC; 164 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, 165 &shpgperproc, 0, 166 "Page share factor per proc"); 167 168 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 169 "2/4MB page mapping counters"); 170 171 u_long pmap_pde_demotions; 172 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD, 173 &pmap_pde_demotions, 0, 174 "2/4MB page demotions"); 175 176 u_long pmap_pde_mappings; 177 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD, 178 &pmap_pde_mappings, 0, 179 "2/4MB page mappings"); 180 181 u_long pmap_pde_p_failures; 182 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD, 183 &pmap_pde_p_failures, 0, 184 "2/4MB page promotion failures"); 185 186 u_long pmap_pde_promotions; 187 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD, 188 &pmap_pde_promotions, 0, 189 "2/4MB page promotions"); 190 191 #ifdef SMP 192 int PMAP1changedcpu; 193 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 194 &PMAP1changedcpu, 0, 195 "Number of times pmap_pte_quick changed CPU with same PMAP1"); 196 #endif 197 198 int PMAP1changed; 199 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 200 &PMAP1changed, 0, 201 "Number of times pmap_pte_quick changed PMAP1"); 202 int PMAP1unchanged; 203 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 204 &PMAP1unchanged, 0, 205 "Number of times pmap_pte_quick didn't change PMAP1"); 206 207 static int 208 kvm_size(SYSCTL_HANDLER_ARGS) 209 { 210 unsigned long ksize; 211 212 ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE; 213 return (sysctl_handle_long(oidp, &ksize, 0, req)); 214 } 215 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 216 0, 0, kvm_size, "IU", 217 "Size of KVM"); 218 219 static int 220 kvm_free(SYSCTL_HANDLER_ARGS) 221 { 222 unsigned long kfree; 223 224 kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; 225 return (sysctl_handle_long(oidp, &kfree, 0, req)); 226 } 227 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 228 0, 0, kvm_free, "IU", 229 "Amount of KVM free"); 230 231 #ifdef PV_STATS 232 int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; 233 long pv_entry_frees, pv_entry_allocs; 234 int pv_entry_spare; 235 236 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, 237 &pc_chunk_count, 0, 238 "Current number of pv entry chunks"); 239 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, 240 &pc_chunk_allocs, 0, 241 "Current number of pv entry chunks allocated"); 242 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, 243 &pc_chunk_frees, 0, 244 "Current number of pv entry chunks frees"); 245 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, 246 &pc_chunk_tryfail, 0, 247 "Number of times tried to get a chunk page but failed."); 248 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, 249 &pv_entry_frees, 0, 250 "Current number of pv entry frees"); 251 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, 252 &pv_entry_allocs, 0, 253 "Current number of pv entry allocs"); 254 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, 255 &pv_entry_spare, 0, 256 "Current number of spare pv entries"); 257 #endif 258 259 struct pmap kernel_pmap_store; 260 static struct pmap_methods *pmap_methods_ptr; 261 262 static int 263 sysctl_kmaps(SYSCTL_HANDLER_ARGS) 264 { 265 return (pmap_methods_ptr->pm_sysctl_kmaps(oidp, arg1, arg2, req)); 266 } 267 SYSCTL_OID(_vm_pmap, OID_AUTO, kernel_maps, 268 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 269 NULL, 0, sysctl_kmaps, "A", 270 "Dump kernel address layout"); 271 272 273 /* 274 * Initialize a vm_page's machine-dependent fields. 275 */ 276 void 277 pmap_page_init(vm_page_t m) 278 { 279 280 TAILQ_INIT(&m->md.pv_list); 281 m->md.pat_mode = PAT_WRITE_BACK; 282 } 283 284 void 285 invltlb_glob(void) 286 { 287 288 invltlb(); 289 } 290 291 static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, 292 vm_offset_t eva); 293 static void pmap_invalidate_cache_range_all(vm_offset_t sva, 294 vm_offset_t eva); 295 296 void 297 pmap_flush_page(vm_page_t m) 298 { 299 300 pmap_methods_ptr->pm_flush_page(m); 301 } 302 303 DEFINE_IFUNC(, void, pmap_invalidate_cache_range, (vm_offset_t, vm_offset_t)) 304 { 305 306 if ((cpu_feature & CPUID_SS) != 0) 307 return (pmap_invalidate_cache_range_selfsnoop); 308 if ((cpu_feature & CPUID_CLFSH) != 0) 309 return (pmap_force_invalidate_cache_range); 310 return (pmap_invalidate_cache_range_all); 311 } 312 313 #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) 314 315 static void 316 pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva) 317 { 318 319 KASSERT((sva & PAGE_MASK) == 0, 320 ("pmap_invalidate_cache_range: sva not page-aligned")); 321 KASSERT((eva & PAGE_MASK) == 0, 322 ("pmap_invalidate_cache_range: eva not page-aligned")); 323 } 324 325 static void 326 pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva) 327 { 328 329 pmap_invalidate_cache_range_check_align(sva, eva); 330 } 331 332 void 333 pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 334 { 335 336 sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1); 337 if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) { 338 /* 339 * The supplied range is bigger than 2MB. 340 * Globally invalidate cache. 341 */ 342 pmap_invalidate_cache(); 343 return; 344 } 345 346 #ifdef DEV_APIC 347 /* 348 * XXX: Some CPUs fault, hang, or trash the local APIC 349 * registers if we use CLFLUSH on the local APIC 350 * range. The local APIC is always uncached, so we 351 * don't need to flush for that range anyway. 352 */ 353 if (pmap_kextract(sva) == lapic_paddr) 354 return; 355 #endif 356 357 if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) { 358 /* 359 * Do per-cache line flush. Use the sfence 360 * instruction to insure that previous stores are 361 * included in the write-back. The processor 362 * propagates flush to other processors in the cache 363 * coherence domain. 364 */ 365 sfence(); 366 for (; sva < eva; sva += cpu_clflush_line_size) 367 clflushopt(sva); 368 sfence(); 369 } else { 370 /* 371 * Writes are ordered by CLFLUSH on Intel CPUs. 372 */ 373 if (cpu_vendor_id != CPU_VENDOR_INTEL) 374 mfence(); 375 for (; sva < eva; sva += cpu_clflush_line_size) 376 clflush(sva); 377 if (cpu_vendor_id != CPU_VENDOR_INTEL) 378 mfence(); 379 } 380 } 381 382 static void 383 pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva) 384 { 385 386 pmap_invalidate_cache_range_check_align(sva, eva); 387 pmap_invalidate_cache(); 388 } 389 390 void 391 pmap_invalidate_cache_pages(vm_page_t *pages, int count) 392 { 393 int i; 394 395 if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE || 396 (cpu_feature & CPUID_CLFSH) == 0) { 397 pmap_invalidate_cache(); 398 } else { 399 for (i = 0; i < count; i++) 400 pmap_flush_page(pages[i]); 401 } 402 } 403 404 void 405 pmap_ksetrw(vm_offset_t va) 406 { 407 408 pmap_methods_ptr->pm_ksetrw(va); 409 } 410 411 void 412 pmap_remap_lower(bool enable) 413 { 414 415 pmap_methods_ptr->pm_remap_lower(enable); 416 } 417 418 void 419 pmap_remap_lowptdi(bool enable) 420 { 421 422 pmap_methods_ptr->pm_remap_lowptdi(enable); 423 } 424 425 void 426 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 427 vm_offset_t *addr, vm_size_t size) 428 { 429 430 return (pmap_methods_ptr->pm_align_superpage(object, offset, 431 addr, size)); 432 } 433 434 vm_offset_t 435 pmap_quick_enter_page(vm_page_t m) 436 { 437 438 return (pmap_methods_ptr->pm_quick_enter_page(m)); 439 } 440 441 void 442 pmap_quick_remove_page(vm_offset_t addr) 443 { 444 445 return (pmap_methods_ptr->pm_quick_remove_page(addr)); 446 } 447 448 void * 449 pmap_trm_alloc(size_t size, int flags) 450 { 451 452 return (pmap_methods_ptr->pm_trm_alloc(size, flags)); 453 } 454 455 void 456 pmap_trm_free(void *addr, size_t size) 457 { 458 459 pmap_methods_ptr->pm_trm_free(addr, size); 460 } 461 462 void 463 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 464 { 465 } 466 467 vm_offset_t 468 pmap_get_map_low(void) 469 { 470 471 return (pmap_methods_ptr->pm_get_map_low()); 472 } 473 474 vm_offset_t 475 pmap_get_vm_maxuser_address(void) 476 { 477 478 return (pmap_methods_ptr->pm_get_vm_maxuser_address()); 479 } 480 481 vm_paddr_t 482 pmap_kextract(vm_offset_t va) 483 { 484 485 return (pmap_methods_ptr->pm_kextract(va)); 486 } 487 488 vm_paddr_t 489 pmap_pg_frame(vm_paddr_t pa) 490 { 491 492 return (pmap_methods_ptr->pm_pg_frame(pa)); 493 } 494 495 void 496 pmap_sf_buf_map(struct sf_buf *sf) 497 { 498 499 pmap_methods_ptr->pm_sf_buf_map(sf); 500 } 501 502 void 503 pmap_cp_slow0_map(vm_offset_t kaddr, int plen, vm_page_t *ma) 504 { 505 506 pmap_methods_ptr->pm_cp_slow0_map(kaddr, plen, ma); 507 } 508 509 u_int 510 pmap_get_kcr3(void) 511 { 512 513 return (pmap_methods_ptr->pm_get_kcr3()); 514 } 515 516 u_int 517 pmap_get_cr3(pmap_t pmap) 518 { 519 520 return (pmap_methods_ptr->pm_get_cr3(pmap)); 521 } 522 523 caddr_t 524 pmap_cmap3(vm_paddr_t pa, u_int pte_flags) 525 { 526 527 return (pmap_methods_ptr->pm_cmap3(pa, pte_flags)); 528 } 529 530 void 531 pmap_basemem_setup(u_int basemem) 532 { 533 534 pmap_methods_ptr->pm_basemem_setup(basemem); 535 } 536 537 void 538 pmap_set_nx(void) 539 { 540 541 pmap_methods_ptr->pm_set_nx(); 542 } 543 544 void * 545 pmap_bios16_enter(void) 546 { 547 548 return (pmap_methods_ptr->pm_bios16_enter()); 549 } 550 551 void 552 pmap_bios16_leave(void *handle) 553 { 554 555 pmap_methods_ptr->pm_bios16_leave(handle); 556 } 557 558 void 559 pmap_bootstrap(vm_paddr_t firstaddr) 560 { 561 562 pmap_methods_ptr->pm_bootstrap(firstaddr); 563 } 564 565 boolean_t 566 pmap_is_valid_memattr(pmap_t pmap, vm_memattr_t mode) 567 { 568 569 return (pmap_methods_ptr->pm_is_valid_memattr(pmap, mode)); 570 } 571 572 int 573 pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde) 574 { 575 576 return (pmap_methods_ptr->pm_cache_bits(pmap, mode, is_pde)); 577 } 578 579 bool 580 pmap_ps_enabled(pmap_t pmap) 581 { 582 583 return (pmap_methods_ptr->pm_ps_enabled(pmap)); 584 } 585 586 void 587 pmap_pinit0(pmap_t pmap) 588 { 589 590 pmap_methods_ptr->pm_pinit0(pmap); 591 } 592 593 int 594 pmap_pinit(pmap_t pmap) 595 { 596 597 return (pmap_methods_ptr->pm_pinit(pmap)); 598 } 599 600 void 601 pmap_activate(struct thread *td) 602 { 603 604 pmap_methods_ptr->pm_activate(td); 605 } 606 607 void 608 pmap_activate_boot(pmap_t pmap) 609 { 610 611 pmap_methods_ptr->pm_activate_boot(pmap); 612 } 613 614 void 615 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) 616 { 617 618 pmap_methods_ptr->pm_advise(pmap, sva, eva, advice); 619 } 620 621 void 622 pmap_clear_modify(vm_page_t m) 623 { 624 625 pmap_methods_ptr->pm_clear_modify(m); 626 } 627 628 int 629 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) 630 { 631 632 return (pmap_methods_ptr->pm_change_attr(va, size, mode)); 633 } 634 635 int 636 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) 637 { 638 639 return (pmap_methods_ptr->pm_mincore(pmap, addr, pap)); 640 } 641 642 void 643 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 644 vm_offset_t src_addr) 645 { 646 647 pmap_methods_ptr->pm_copy(dst_pmap, src_pmap, dst_addr, len, src_addr); 648 } 649 650 void 651 pmap_copy_page(vm_page_t src, vm_page_t dst) 652 { 653 654 pmap_methods_ptr->pm_copy_page(src, dst); 655 } 656 657 void 658 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 659 vm_offset_t b_offset, int xfersize) 660 { 661 662 pmap_methods_ptr->pm_copy_pages(ma, a_offset, mb, b_offset, xfersize); 663 } 664 665 void 666 pmap_zero_page(vm_page_t m) 667 { 668 669 pmap_methods_ptr->pm_zero_page(m); 670 } 671 672 void 673 pmap_zero_page_area(vm_page_t m, int off, int size) 674 { 675 676 pmap_methods_ptr->pm_zero_page_area(m, off, size); 677 } 678 679 int 680 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 681 u_int flags, int8_t psind) 682 { 683 684 return (pmap_methods_ptr->pm_enter(pmap, va, m, prot, flags, psind)); 685 } 686 687 void 688 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 689 vm_page_t m_start, vm_prot_t prot) 690 { 691 692 pmap_methods_ptr->pm_enter_object(pmap, start, end, m_start, prot); 693 } 694 695 void 696 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 697 { 698 699 pmap_methods_ptr->pm_enter_quick(pmap, va, m, prot); 700 } 701 702 void * 703 pmap_kenter_temporary(vm_paddr_t pa, int i) 704 { 705 706 return (pmap_methods_ptr->pm_kenter_temporary(pa, i)); 707 } 708 709 void 710 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 711 vm_pindex_t pindex, vm_size_t size) 712 { 713 714 pmap_methods_ptr->pm_object_init_pt(pmap, addr, object, pindex, size); 715 } 716 717 void 718 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 719 { 720 721 pmap_methods_ptr->pm_unwire(pmap, sva, eva); 722 } 723 724 boolean_t 725 pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 726 { 727 728 return (pmap_methods_ptr->pm_page_exists_quick(pmap, m)); 729 } 730 731 int 732 pmap_page_wired_mappings(vm_page_t m) 733 { 734 735 return (pmap_methods_ptr->pm_page_wired_mappings(m)); 736 } 737 738 boolean_t 739 pmap_page_is_mapped(vm_page_t m) 740 { 741 742 return (pmap_methods_ptr->pm_page_is_mapped(m)); 743 } 744 745 void 746 pmap_remove_pages(pmap_t pmap) 747 { 748 749 pmap_methods_ptr->pm_remove_pages(pmap); 750 } 751 752 boolean_t 753 pmap_is_modified(vm_page_t m) 754 { 755 756 return (pmap_methods_ptr->pm_is_modified(m)); 757 } 758 759 boolean_t 760 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 761 { 762 763 return (pmap_methods_ptr->pm_is_prefaultable(pmap, addr)); 764 } 765 766 boolean_t 767 pmap_is_referenced(vm_page_t m) 768 { 769 770 return (pmap_methods_ptr->pm_is_referenced(m)); 771 } 772 773 void 774 pmap_remove_write(vm_page_t m) 775 { 776 777 pmap_methods_ptr->pm_remove_write(m); 778 } 779 780 int 781 pmap_ts_referenced(vm_page_t m) 782 { 783 784 return (pmap_methods_ptr->pm_ts_referenced(m)); 785 } 786 787 void * 788 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) 789 { 790 791 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, mode, 792 MAPDEV_SETATTR)); 793 } 794 795 void * 796 pmap_mapdev(vm_paddr_t pa, vm_size_t size) 797 { 798 799 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_UNCACHEABLE, 800 MAPDEV_SETATTR)); 801 } 802 803 void * 804 pmap_mapbios(vm_paddr_t pa, vm_size_t size) 805 { 806 807 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_WRITE_BACK, 0)); 808 } 809 810 void 811 pmap_unmapdev(vm_offset_t va, vm_size_t size) 812 { 813 814 pmap_methods_ptr->pm_unmapdev(va, size); 815 } 816 817 void 818 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 819 { 820 821 pmap_methods_ptr->pm_page_set_memattr(m, ma); 822 } 823 824 vm_paddr_t 825 pmap_extract(pmap_t pmap, vm_offset_t va) 826 { 827 828 return (pmap_methods_ptr->pm_extract(pmap, va)); 829 } 830 831 vm_page_t 832 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 833 { 834 835 return (pmap_methods_ptr->pm_extract_and_hold(pmap, va, prot)); 836 } 837 838 vm_offset_t 839 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 840 { 841 842 return (pmap_methods_ptr->pm_map(virt, start, end, prot)); 843 } 844 845 void 846 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) 847 { 848 849 pmap_methods_ptr->pm_qenter(sva, ma, count); 850 } 851 852 void 853 pmap_qremove(vm_offset_t sva, int count) 854 { 855 856 pmap_methods_ptr->pm_qremove(sva, count); 857 } 858 859 void 860 pmap_release(pmap_t pmap) 861 { 862 863 pmap_methods_ptr->pm_release(pmap); 864 } 865 866 void 867 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 868 { 869 870 pmap_methods_ptr->pm_remove(pmap, sva, eva); 871 } 872 873 void 874 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 875 { 876 877 pmap_methods_ptr->pm_protect(pmap, sva, eva, prot); 878 } 879 880 void 881 pmap_remove_all(vm_page_t m) 882 { 883 884 pmap_methods_ptr->pm_remove_all(m); 885 } 886 887 void 888 pmap_init(void) 889 { 890 891 pmap_methods_ptr->pm_init(); 892 } 893 894 void 895 pmap_init_pat(void) 896 { 897 898 pmap_methods_ptr->pm_init_pat(); 899 } 900 901 void 902 pmap_growkernel(vm_offset_t addr) 903 { 904 905 pmap_methods_ptr->pm_growkernel(addr); 906 } 907 908 void 909 pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 910 { 911 912 pmap_methods_ptr->pm_invalidate_page(pmap, va); 913 } 914 915 void 916 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 917 { 918 919 pmap_methods_ptr->pm_invalidate_range(pmap, sva, eva); 920 } 921 922 void 923 pmap_invalidate_all(pmap_t pmap) 924 { 925 926 pmap_methods_ptr->pm_invalidate_all(pmap); 927 } 928 929 void 930 pmap_invalidate_cache(void) 931 { 932 933 pmap_methods_ptr->pm_invalidate_cache(); 934 } 935 936 void 937 pmap_kenter(vm_offset_t va, vm_paddr_t pa) 938 { 939 940 pmap_methods_ptr->pm_kenter(va, pa); 941 } 942 943 void 944 pmap_kremove(vm_offset_t va) 945 { 946 947 pmap_methods_ptr->pm_kremove(va); 948 } 949 950 extern struct pmap_methods pmap_pae_methods, pmap_nopae_methods; 951 int pae_mode; 952 SYSCTL_INT(_vm_pmap, OID_AUTO, pae_mode, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 953 &pae_mode, 0, 954 "PAE"); 955 956 void 957 pmap_cold(void) 958 { 959 960 init_static_kenv((char *)bootinfo.bi_envp, 0); 961 pae_mode = (cpu_feature & CPUID_PAE) != 0; 962 if (pae_mode) 963 TUNABLE_INT_FETCH("vm.pmap.pae_mode", &pae_mode); 964 if (pae_mode) { 965 pmap_methods_ptr = &pmap_pae_methods; 966 pmap_pae_cold(); 967 } else { 968 pmap_methods_ptr = &pmap_nopae_methods; 969 pmap_nopae_cold(); 970 } 971 } 972