1 /*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 1991 Regents of the University of California. 5 * All rights reserved. 6 * Copyright (c) 1994 John S. Dyson 7 * All rights reserved. 8 * Copyright (c) 1994 David Greenman 9 * All rights reserved. 10 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu> 11 * All rights reserved. 12 * 13 * This code is derived from software contributed to Berkeley by 14 * the Systems Programming Group of the University of Utah Computer 15 * Science Department and William Jolitz of UUNET Technologies Inc. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. All advertising materials mentioning features or use of this software 26 * must display the following acknowledgement: 27 * This product includes software developed by the University of 28 * California, Berkeley and its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 46 */ 47 /*- 48 * Copyright (c) 2003 Networks Associates Technology, Inc. 49 * All rights reserved. 50 * Copyright (c) 2018 The FreeBSD Foundation 51 * All rights reserved. 52 * 53 * This software was developed for the FreeBSD Project by Jake Burkholder, 54 * Safeport Network Services, and Network Associates Laboratories, the 55 * Security Research Division of Network Associates, Inc. under 56 * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA 57 * CHATS research program. 58 * 59 * Portions of this software were developed by 60 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 61 * the FreeBSD Foundation. 62 * 63 * Redistribution and use in source and binary forms, with or without 64 * modification, are permitted provided that the following conditions 65 * are met: 66 * 1. Redistributions of source code must retain the above copyright 67 * notice, this list of conditions and the following disclaimer. 68 * 2. Redistributions in binary form must reproduce the above copyright 69 * notice, this list of conditions and the following disclaimer in the 70 * documentation and/or other materials provided with the distribution. 71 * 72 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 73 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 74 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 75 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 76 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 77 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 78 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 79 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 80 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 81 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 82 * SUCH DAMAGE. 83 */ 84 85 #include <sys/cdefs.h> 86 __FBSDID("$FreeBSD$"); 87 88 #include "opt_apic.h" 89 #include "opt_cpu.h" 90 #include "opt_pmap.h" 91 #include "opt_smp.h" 92 #include "opt_vm.h" 93 94 #include <sys/param.h> 95 #include <sys/systm.h> 96 #include <sys/kernel.h> 97 #include <sys/vmmeter.h> 98 #include <sys/sysctl.h> 99 #include <machine/bootinfo.h> 100 #include <machine/cpu.h> 101 #include <machine/cputypes.h> 102 #include <machine/md_var.h> 103 #ifdef DEV_APIC 104 #include <sys/bus.h> 105 #include <machine/intr_machdep.h> 106 #include <x86/apicvar.h> 107 #endif 108 #include <x86/ifunc.h> 109 110 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); 111 112 #include <machine/vmparam.h> 113 #include <vm/vm.h> 114 #include <vm/vm_page.h> 115 #include <vm/pmap.h> 116 #include <machine/pmap_base.h> 117 118 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 119 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 120 121 int unmapped_buf_allowed = 1; 122 123 int pti; 124 125 u_long physfree; /* phys addr of next free page */ 126 u_long vm86phystk; /* PA of vm86/bios stack */ 127 u_long vm86paddr; /* address of vm86 region */ 128 int vm86pa; /* phys addr of vm86 region */ 129 u_long KERNend; /* phys addr end of kernel (just after bss) */ 130 u_long KPTphys; /* phys addr of kernel page tables */ 131 caddr_t ptvmmap = 0; 132 vm_offset_t kernel_vm_end; 133 134 int i386_pmap_VM_NFREEORDER; 135 int i386_pmap_VM_LEVEL_0_ORDER; 136 int i386_pmap_PDRSHIFT; 137 138 int pat_works = 1; 139 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, 140 &pat_works, 0, 141 "Is page attribute table fully functional?"); 142 143 int pg_ps_enabled = 1; 144 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 145 &pg_ps_enabled, 0, 146 "Are large page mappings enabled?"); 147 148 int pv_entry_max = 0; 149 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, 150 &pv_entry_max, 0, 151 "Max number of PV entries"); 152 153 int pv_entry_count = 0; 154 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, 155 &pv_entry_count, 0, 156 "Current number of pv entries"); 157 158 #ifndef PMAP_SHPGPERPROC 159 #define PMAP_SHPGPERPROC 200 160 #endif 161 162 int shpgperproc = PMAP_SHPGPERPROC; 163 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, 164 &shpgperproc, 0, 165 "Page share factor per proc"); 166 167 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0, 168 "2/4MB page mapping counters"); 169 170 u_long pmap_pde_demotions; 171 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD, 172 &pmap_pde_demotions, 0, 173 "2/4MB page demotions"); 174 175 u_long pmap_pde_mappings; 176 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD, 177 &pmap_pde_mappings, 0, 178 "2/4MB page mappings"); 179 180 u_long pmap_pde_p_failures; 181 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD, 182 &pmap_pde_p_failures, 0, 183 "2/4MB page promotion failures"); 184 185 u_long pmap_pde_promotions; 186 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD, 187 &pmap_pde_promotions, 0, 188 "2/4MB page promotions"); 189 190 #ifdef SMP 191 int PMAP1changedcpu; 192 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 193 &PMAP1changedcpu, 0, 194 "Number of times pmap_pte_quick changed CPU with same PMAP1"); 195 #endif 196 197 int PMAP1changed; 198 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 199 &PMAP1changed, 0, 200 "Number of times pmap_pte_quick changed PMAP1"); 201 int PMAP1unchanged; 202 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 203 &PMAP1unchanged, 0, 204 "Number of times pmap_pte_quick didn't change PMAP1"); 205 206 static int 207 kvm_size(SYSCTL_HANDLER_ARGS) 208 { 209 unsigned long ksize; 210 211 ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE; 212 return (sysctl_handle_long(oidp, &ksize, 0, req)); 213 } 214 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 215 0, 0, kvm_size, "IU", 216 "Size of KVM"); 217 218 static int 219 kvm_free(SYSCTL_HANDLER_ARGS) 220 { 221 unsigned long kfree; 222 223 kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; 224 return (sysctl_handle_long(oidp, &kfree, 0, req)); 225 } 226 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 227 0, 0, kvm_free, "IU", 228 "Amount of KVM free"); 229 230 #ifdef PV_STATS 231 int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; 232 long pv_entry_frees, pv_entry_allocs; 233 int pv_entry_spare; 234 235 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, 236 &pc_chunk_count, 0, 237 "Current number of pv entry chunks"); 238 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, 239 &pc_chunk_allocs, 0, 240 "Current number of pv entry chunks allocated"); 241 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, 242 &pc_chunk_frees, 0, 243 "Current number of pv entry chunks frees"); 244 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, 245 &pc_chunk_tryfail, 0, 246 "Number of times tried to get a chunk page but failed."); 247 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, 248 &pv_entry_frees, 0, 249 "Current number of pv entry frees"); 250 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, 251 &pv_entry_allocs, 0, 252 "Current number of pv entry allocs"); 253 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, 254 &pv_entry_spare, 0, 255 "Current number of spare pv entries"); 256 #endif 257 258 struct pmap kernel_pmap_store; 259 static struct pmap_methods *pmap_methods_ptr; 260 261 /* 262 * Initialize a vm_page's machine-dependent fields. 263 */ 264 void 265 pmap_page_init(vm_page_t m) 266 { 267 268 TAILQ_INIT(&m->md.pv_list); 269 m->md.pat_mode = PAT_WRITE_BACK; 270 } 271 272 void 273 invltlb_glob(void) 274 { 275 276 invltlb(); 277 } 278 279 static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, 280 vm_offset_t eva); 281 static void pmap_invalidate_cache_range_all(vm_offset_t sva, 282 vm_offset_t eva); 283 284 void 285 pmap_flush_page(vm_page_t m) 286 { 287 288 pmap_methods_ptr->pm_flush_page(m); 289 } 290 291 DEFINE_IFUNC(, void, pmap_invalidate_cache_range, (vm_offset_t, vm_offset_t), 292 static) 293 { 294 295 if ((cpu_feature & CPUID_SS) != 0) 296 return (pmap_invalidate_cache_range_selfsnoop); 297 if ((cpu_feature & CPUID_CLFSH) != 0) 298 return (pmap_force_invalidate_cache_range); 299 return (pmap_invalidate_cache_range_all); 300 } 301 302 #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) 303 304 static void 305 pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva) 306 { 307 308 KASSERT((sva & PAGE_MASK) == 0, 309 ("pmap_invalidate_cache_range: sva not page-aligned")); 310 KASSERT((eva & PAGE_MASK) == 0, 311 ("pmap_invalidate_cache_range: eva not page-aligned")); 312 } 313 314 static void 315 pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva) 316 { 317 318 pmap_invalidate_cache_range_check_align(sva, eva); 319 } 320 321 void 322 pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 323 { 324 325 sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1); 326 if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) { 327 /* 328 * The supplied range is bigger than 2MB. 329 * Globally invalidate cache. 330 */ 331 pmap_invalidate_cache(); 332 return; 333 } 334 335 #ifdef DEV_APIC 336 /* 337 * XXX: Some CPUs fault, hang, or trash the local APIC 338 * registers if we use CLFLUSH on the local APIC 339 * range. The local APIC is always uncached, so we 340 * don't need to flush for that range anyway. 341 */ 342 if (pmap_kextract(sva) == lapic_paddr) 343 return; 344 #endif 345 346 if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) { 347 /* 348 * Do per-cache line flush. Use the sfence 349 * instruction to insure that previous stores are 350 * included in the write-back. The processor 351 * propagates flush to other processors in the cache 352 * coherence domain. 353 */ 354 sfence(); 355 for (; sva < eva; sva += cpu_clflush_line_size) 356 clflushopt(sva); 357 sfence(); 358 } else { 359 /* 360 * Writes are ordered by CLFLUSH on Intel CPUs. 361 */ 362 if (cpu_vendor_id != CPU_VENDOR_INTEL) 363 mfence(); 364 for (; sva < eva; sva += cpu_clflush_line_size) 365 clflush(sva); 366 if (cpu_vendor_id != CPU_VENDOR_INTEL) 367 mfence(); 368 } 369 } 370 371 static void 372 pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva) 373 { 374 375 pmap_invalidate_cache_range_check_align(sva, eva); 376 pmap_invalidate_cache(); 377 } 378 379 void 380 pmap_invalidate_cache_pages(vm_page_t *pages, int count) 381 { 382 int i; 383 384 if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE || 385 (cpu_feature & CPUID_CLFSH) == 0) { 386 pmap_invalidate_cache(); 387 } else { 388 for (i = 0; i < count; i++) 389 pmap_flush_page(pages[i]); 390 } 391 } 392 393 void 394 pmap_ksetrw(vm_offset_t va) 395 { 396 397 pmap_methods_ptr->pm_ksetrw(va); 398 } 399 400 void 401 pmap_remap_lower(bool enable) 402 { 403 404 pmap_methods_ptr->pm_remap_lower(enable); 405 } 406 407 void 408 pmap_remap_lowptdi(bool enable) 409 { 410 411 pmap_methods_ptr->pm_remap_lowptdi(enable); 412 } 413 414 void 415 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 416 vm_offset_t *addr, vm_size_t size) 417 { 418 419 return (pmap_methods_ptr->pm_align_superpage(object, offset, 420 addr, size)); 421 } 422 423 vm_offset_t 424 pmap_quick_enter_page(vm_page_t m) 425 { 426 427 return (pmap_methods_ptr->pm_quick_enter_page(m)); 428 } 429 430 void 431 pmap_quick_remove_page(vm_offset_t addr) 432 { 433 434 return (pmap_methods_ptr->pm_quick_remove_page(addr)); 435 } 436 437 void * 438 pmap_trm_alloc(size_t size, int flags) 439 { 440 441 return (pmap_methods_ptr->pm_trm_alloc(size, flags)); 442 } 443 444 void 445 pmap_trm_free(void *addr, size_t size) 446 { 447 448 pmap_methods_ptr->pm_trm_free(addr, size); 449 } 450 451 void 452 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 453 { 454 } 455 456 vm_offset_t 457 pmap_get_map_low(void) 458 { 459 460 return (pmap_methods_ptr->pm_get_map_low()); 461 } 462 463 vm_offset_t 464 pmap_get_vm_maxuser_address(void) 465 { 466 467 return (pmap_methods_ptr->pm_get_vm_maxuser_address()); 468 } 469 470 vm_paddr_t 471 pmap_kextract(vm_offset_t va) 472 { 473 474 return (pmap_methods_ptr->pm_kextract(va)); 475 } 476 477 vm_paddr_t 478 pmap_pg_frame(vm_paddr_t pa) 479 { 480 481 return (pmap_methods_ptr->pm_pg_frame(pa)); 482 } 483 484 void 485 pmap_sf_buf_map(struct sf_buf *sf) 486 { 487 488 pmap_methods_ptr->pm_sf_buf_map(sf); 489 } 490 491 void 492 pmap_cp_slow0_map(vm_offset_t kaddr, int plen, vm_page_t *ma) 493 { 494 495 pmap_methods_ptr->pm_cp_slow0_map(kaddr, plen, ma); 496 } 497 498 u_int 499 pmap_get_kcr3(void) 500 { 501 502 return (pmap_methods_ptr->pm_get_kcr3()); 503 } 504 505 u_int 506 pmap_get_cr3(pmap_t pmap) 507 { 508 509 return (pmap_methods_ptr->pm_get_cr3(pmap)); 510 } 511 512 caddr_t 513 pmap_cmap3(vm_paddr_t pa, u_int pte_flags) 514 { 515 516 return (pmap_methods_ptr->pm_cmap3(pa, pte_flags)); 517 } 518 519 void 520 pmap_basemem_setup(u_int basemem) 521 { 522 523 pmap_methods_ptr->pm_basemem_setup(basemem); 524 } 525 526 void 527 pmap_set_nx(void) 528 { 529 530 pmap_methods_ptr->pm_set_nx(); 531 } 532 533 void * 534 pmap_bios16_enter(void) 535 { 536 537 return (pmap_methods_ptr->pm_bios16_enter()); 538 } 539 540 void 541 pmap_bios16_leave(void *handle) 542 { 543 544 pmap_methods_ptr->pm_bios16_leave(handle); 545 } 546 547 void 548 pmap_bootstrap(vm_paddr_t firstaddr) 549 { 550 551 pmap_methods_ptr->pm_bootstrap(firstaddr); 552 } 553 554 boolean_t 555 pmap_is_valid_memattr(pmap_t pmap, vm_memattr_t mode) 556 { 557 558 return (pmap_methods_ptr->pm_is_valid_memattr(pmap, mode)); 559 } 560 561 int 562 pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde) 563 { 564 565 return (pmap_methods_ptr->pm_cache_bits(pmap, mode, is_pde)); 566 } 567 568 bool 569 pmap_ps_enabled(pmap_t pmap) 570 { 571 572 return (pmap_methods_ptr->pm_ps_enabled(pmap)); 573 } 574 575 void 576 pmap_pinit0(pmap_t pmap) 577 { 578 579 pmap_methods_ptr->pm_pinit0(pmap); 580 } 581 582 int 583 pmap_pinit(pmap_t pmap) 584 { 585 586 return (pmap_methods_ptr->pm_pinit(pmap)); 587 } 588 589 void 590 pmap_activate(struct thread *td) 591 { 592 593 pmap_methods_ptr->pm_activate(td); 594 } 595 596 void 597 pmap_activate_boot(pmap_t pmap) 598 { 599 600 pmap_methods_ptr->pm_activate_boot(pmap); 601 } 602 603 void 604 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) 605 { 606 607 pmap_methods_ptr->pm_advise(pmap, sva, eva, advice); 608 } 609 610 void 611 pmap_clear_modify(vm_page_t m) 612 { 613 614 pmap_methods_ptr->pm_clear_modify(m); 615 } 616 617 int 618 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) 619 { 620 621 return (pmap_methods_ptr->pm_change_attr(va, size, mode)); 622 } 623 624 int 625 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 626 { 627 628 return (pmap_methods_ptr->pm_mincore(pmap, addr, locked_pa)); 629 } 630 631 void 632 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 633 vm_offset_t src_addr) 634 { 635 636 pmap_methods_ptr->pm_copy(dst_pmap, src_pmap, dst_addr, len, src_addr); 637 } 638 639 void 640 pmap_copy_page(vm_page_t src, vm_page_t dst) 641 { 642 643 pmap_methods_ptr->pm_copy_page(src, dst); 644 } 645 646 void 647 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 648 vm_offset_t b_offset, int xfersize) 649 { 650 651 pmap_methods_ptr->pm_copy_pages(ma, a_offset, mb, b_offset, xfersize); 652 } 653 654 void 655 pmap_zero_page(vm_page_t m) 656 { 657 658 pmap_methods_ptr->pm_zero_page(m); 659 } 660 661 void 662 pmap_zero_page_area(vm_page_t m, int off, int size) 663 { 664 665 pmap_methods_ptr->pm_zero_page_area(m, off, size); 666 } 667 668 int 669 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 670 u_int flags, int8_t psind) 671 { 672 673 return (pmap_methods_ptr->pm_enter(pmap, va, m, prot, flags, psind)); 674 } 675 676 void 677 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 678 vm_page_t m_start, vm_prot_t prot) 679 { 680 681 pmap_methods_ptr->pm_enter_object(pmap, start, end, m_start, prot); 682 } 683 684 void 685 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 686 { 687 688 pmap_methods_ptr->pm_enter_quick(pmap, va, m, prot); 689 } 690 691 void * 692 pmap_kenter_temporary(vm_paddr_t pa, int i) 693 { 694 695 return (pmap_methods_ptr->pm_kenter_temporary(pa, i)); 696 } 697 698 void 699 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 700 vm_pindex_t pindex, vm_size_t size) 701 { 702 703 pmap_methods_ptr->pm_object_init_pt(pmap, addr, object, pindex, size); 704 } 705 706 void 707 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 708 { 709 710 pmap_methods_ptr->pm_unwire(pmap, sva, eva); 711 } 712 713 boolean_t 714 pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 715 { 716 717 return (pmap_methods_ptr->pm_page_exists_quick(pmap, m)); 718 } 719 720 int 721 pmap_page_wired_mappings(vm_page_t m) 722 { 723 724 return (pmap_methods_ptr->pm_page_wired_mappings(m)); 725 } 726 727 boolean_t 728 pmap_page_is_mapped(vm_page_t m) 729 { 730 731 return (pmap_methods_ptr->pm_page_is_mapped(m)); 732 } 733 734 void 735 pmap_remove_pages(pmap_t pmap) 736 { 737 738 pmap_methods_ptr->pm_remove_pages(pmap); 739 } 740 741 boolean_t 742 pmap_is_modified(vm_page_t m) 743 { 744 745 return (pmap_methods_ptr->pm_is_modified(m)); 746 } 747 748 boolean_t 749 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 750 { 751 752 return (pmap_methods_ptr->pm_is_prefaultable(pmap, addr)); 753 } 754 755 boolean_t 756 pmap_is_referenced(vm_page_t m) 757 { 758 759 return (pmap_methods_ptr->pm_is_referenced(m)); 760 } 761 762 void 763 pmap_remove_write(vm_page_t m) 764 { 765 766 pmap_methods_ptr->pm_remove_write(m); 767 } 768 769 int 770 pmap_ts_referenced(vm_page_t m) 771 { 772 773 return (pmap_methods_ptr->pm_ts_referenced(m)); 774 } 775 776 void * 777 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) 778 { 779 780 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, mode)); 781 } 782 783 void * 784 pmap_mapdev(vm_paddr_t pa, vm_size_t size) 785 { 786 787 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_UNCACHEABLE)); 788 } 789 790 void * 791 pmap_mapbios(vm_paddr_t pa, vm_size_t size) 792 { 793 794 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_WRITE_BACK)); 795 } 796 797 void 798 pmap_unmapdev(vm_offset_t va, vm_size_t size) 799 { 800 801 pmap_methods_ptr->pm_unmapdev(va, size); 802 } 803 804 void 805 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 806 { 807 808 pmap_methods_ptr->pm_page_set_memattr(m, ma); 809 } 810 811 vm_paddr_t 812 pmap_extract(pmap_t pmap, vm_offset_t va) 813 { 814 815 return (pmap_methods_ptr->pm_extract(pmap, va)); 816 } 817 818 vm_page_t 819 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 820 { 821 822 return (pmap_methods_ptr->pm_extract_and_hold(pmap, va, prot)); 823 } 824 825 vm_offset_t 826 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 827 { 828 829 return (pmap_methods_ptr->pm_map(virt, start, end, prot)); 830 } 831 832 void 833 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) 834 { 835 836 pmap_methods_ptr->pm_qenter(sva, ma, count); 837 } 838 839 void 840 pmap_qremove(vm_offset_t sva, int count) 841 { 842 843 pmap_methods_ptr->pm_qremove(sva, count); 844 } 845 846 void 847 pmap_release(pmap_t pmap) 848 { 849 850 pmap_methods_ptr->pm_release(pmap); 851 } 852 853 void 854 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 855 { 856 857 pmap_methods_ptr->pm_remove(pmap, sva, eva); 858 } 859 860 void 861 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 862 { 863 864 pmap_methods_ptr->pm_protect(pmap, sva, eva, prot); 865 } 866 867 void 868 pmap_remove_all(vm_page_t m) 869 { 870 871 pmap_methods_ptr->pm_remove_all(m); 872 } 873 874 void 875 pmap_init(void) 876 { 877 878 pmap_methods_ptr->pm_init(); 879 } 880 881 void 882 pmap_init_pat(void) 883 { 884 885 pmap_methods_ptr->pm_init_pat(); 886 } 887 888 void 889 pmap_growkernel(vm_offset_t addr) 890 { 891 892 pmap_methods_ptr->pm_growkernel(addr); 893 } 894 895 void 896 pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 897 { 898 899 pmap_methods_ptr->pm_invalidate_page(pmap, va); 900 } 901 902 void 903 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 904 { 905 906 pmap_methods_ptr->pm_invalidate_range(pmap, sva, eva); 907 } 908 909 void 910 pmap_invalidate_all(pmap_t pmap) 911 { 912 913 pmap_methods_ptr->pm_invalidate_all(pmap); 914 } 915 916 void 917 pmap_invalidate_cache(void) 918 { 919 920 pmap_methods_ptr->pm_invalidate_cache(); 921 } 922 923 void 924 pmap_kenter(vm_offset_t va, vm_paddr_t pa) 925 { 926 927 pmap_methods_ptr->pm_kenter(va, pa); 928 } 929 930 void 931 pmap_kremove(vm_offset_t va) 932 { 933 934 pmap_methods_ptr->pm_kremove(va); 935 } 936 937 extern struct pmap_methods pmap_pae_methods, pmap_nopae_methods; 938 int pae_mode; 939 SYSCTL_INT(_vm_pmap, OID_AUTO, pae_mode, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 940 &pae_mode, 0, 941 "PAE"); 942 943 void 944 pmap_cold(void) 945 { 946 947 init_static_kenv((char *)bootinfo.bi_envp, 0); 948 pae_mode = (cpu_feature & CPUID_PAE) != 0; 949 if (pae_mode) 950 TUNABLE_INT_FETCH("vm.pmap.pae_mode", &pae_mode); 951 if (pae_mode) { 952 pmap_methods_ptr = &pmap_pae_methods; 953 pmap_pae_cold(); 954 } else { 955 pmap_methods_ptr = &pmap_nopae_methods; 956 pmap_nopae_cold(); 957 } 958 } 959