1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018 Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 33 #include <sys/param.h> 34 #include <sys/kernel.h> 35 #include <sys/systm.h> 36 #include <sys/conf.h> 37 #include <sys/bitstring.h> 38 #include <sys/queue.h> 39 #include <sys/cpuset.h> 40 #include <sys/endian.h> 41 #include <sys/kerneldump.h> 42 #include <sys/ktr.h> 43 #include <sys/lock.h> 44 #include <sys/syslog.h> 45 #include <sys/msgbuf.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/mutex.h> 49 #include <sys/proc.h> 50 #include <sys/rwlock.h> 51 #include <sys/sched.h> 52 #include <sys/sysctl.h> 53 #include <sys/systm.h> 54 #include <sys/vmem.h> 55 #include <sys/vmmeter.h> 56 #include <sys/smp.h> 57 58 #include <sys/kdb.h> 59 60 #include <dev/ofw/openfirm.h> 61 62 #include <vm/vm.h> 63 #include <vm/pmap.h> 64 #include <vm/vm_param.h> 65 #include <vm/vm_kern.h> 66 #include <vm/vm_page.h> 67 #include <vm/vm_map.h> 68 #include <vm/vm_object.h> 69 #include <vm/vm_extern.h> 70 #include <vm/vm_pageout.h> 71 #include <vm/vm_phys.h> 72 #include <vm/vm_reserv.h> 73 #include <vm/uma.h> 74 75 #include <machine/_inttypes.h> 76 #include <machine/cpu.h> 77 #include <machine/platform.h> 78 #include <machine/frame.h> 79 #include <machine/md_var.h> 80 #include <machine/psl.h> 81 #include <machine/bat.h> 82 #include <machine/hid.h> 83 #include <machine/pte.h> 84 #include <machine/sr.h> 85 #include <machine/trap.h> 86 #include <machine/mmuvar.h> 87 88 #ifdef INVARIANTS 89 #include <vm/uma_dbg.h> 90 #endif 91 92 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit)) 93 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) 94 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit)) 95 96 #include "opt_ddb.h" 97 #ifdef DDB 98 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va); 99 #endif 100 101 #define PG_W RPTE_WIRED 102 #define PG_V RPTE_VALID 103 #define PG_MANAGED RPTE_MANAGED 104 #define PG_PROMOTED RPTE_PROMOTED 105 #define PG_M RPTE_C 106 #define PG_A RPTE_R 107 #define PG_X RPTE_EAA_X 108 #define PG_RW RPTE_EAA_W 109 #define PG_PTE_CACHE RPTE_ATTR_MASK 110 111 #define RPTE_SHIFT 9 112 #define NLS_MASK ((1UL<<5)-1) 113 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT) 114 #define RPTE_MASK (RPTE_ENTRIES-1) 115 116 #define NLB_SHIFT 0 117 #define NLB_MASK (((1UL<<52)-1) << 8) 118 119 extern int nkpt; 120 extern caddr_t crashdumpmap; 121 122 #define RIC_FLUSH_TLB 0 123 #define RIC_FLUSH_PWC 1 124 #define RIC_FLUSH_ALL 2 125 126 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ 127 128 #define PPC_INST_TLBIE 0x7c000264 129 #define PPC_INST_TLBIEL 0x7c000224 130 #define PPC_INST_SLBIA 0x7c0003e4 131 132 #define ___PPC_RA(a) (((a) & 0x1f) << 16) 133 #define ___PPC_RB(b) (((b) & 0x1f) << 11) 134 #define ___PPC_RS(s) (((s) & 0x1f) << 21) 135 #define ___PPC_RT(t) ___PPC_RS(t) 136 #define ___PPC_R(r) (((r) & 0x1) << 16) 137 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17) 138 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18) 139 140 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \ 141 ((IH & 0x7) << 21)) 142 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \ 143 __XSTRING(.long PPC_INST_TLBIE | \ 144 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 145 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 146 ___PPC_R(r)) 147 148 #define PPC_TLBIEL(rb,rs,ric,prs,r) \ 149 __XSTRING(.long PPC_INST_TLBIEL | \ 150 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 151 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 152 ___PPC_R(r)) 153 154 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7) 155 156 static __inline void 157 ttusync(void) 158 { 159 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 160 } 161 162 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ 163 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ 164 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */ 165 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ 166 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ 167 168 #define TLBIE_ACTUAL_PAGE_MASK 0xe0 169 #define TLBIE_ACTUAL_PAGE_4K 0x00 170 #define TLBIE_ACTUAL_PAGE_64K 0xa0 171 #define TLBIE_ACTUAL_PAGE_2M 0x20 172 #define TLBIE_ACTUAL_PAGE_1G 0x40 173 174 #define TLBIE_PRS_PARTITION_SCOPE 0x0 175 #define TLBIE_PRS_PROCESS_SCOPE 0x1 176 177 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */ 178 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */ 179 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC, 180 * cached {proc, part}tab entries 181 */ 182 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only: 183 * Invalidate a range of translations 184 */ 185 186 static __inline void 187 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid, 188 vm_offset_t va, uint16_t ap) 189 { 190 uint64_t rb, rs; 191 192 MPASS((va & PAGE_MASK) == 0); 193 194 rs = ((uint64_t)pid << 32) | lpid; 195 rb = va | is | ap; 196 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : : 197 "r" (rb), "r" (rs), "i" (ric), "i" (prs)); 198 } 199 200 static __inline void 201 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va) 202 { 203 204 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 205 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K); 206 } 207 208 static __inline void 209 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va) 210 { 211 212 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 213 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M); 214 } 215 216 static __inline void 217 radix_tlbie_invlpwc_user(uint32_t pid) 218 { 219 220 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 221 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 222 } 223 224 static __inline void 225 radix_tlbie_flush_user(uint32_t pid) 226 { 227 228 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 229 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 230 } 231 232 static __inline void 233 radix_tlbie_invlpg_kernel_4k(vm_offset_t va) 234 { 235 236 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 237 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K); 238 } 239 240 static __inline void 241 radix_tlbie_invlpg_kernel_2m(vm_offset_t va) 242 { 243 244 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 245 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M); 246 } 247 248 /* 1GB pages aren't currently supported. */ 249 static __inline __unused void 250 radix_tlbie_invlpg_kernel_1g(vm_offset_t va) 251 { 252 253 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 254 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G); 255 } 256 257 static __inline void 258 radix_tlbie_invlpwc_kernel(void) 259 { 260 261 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 262 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 263 } 264 265 static __inline void 266 radix_tlbie_flush_kernel(void) 267 { 268 269 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 270 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 271 } 272 273 static __inline vm_pindex_t 274 pmap_l3e_pindex(vm_offset_t va) 275 { 276 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT); 277 } 278 279 static __inline vm_pindex_t 280 pmap_pml3e_index(vm_offset_t va) 281 { 282 283 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK); 284 } 285 286 static __inline vm_pindex_t 287 pmap_pml2e_index(vm_offset_t va) 288 { 289 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK); 290 } 291 292 static __inline vm_pindex_t 293 pmap_pml1e_index(vm_offset_t va) 294 { 295 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT); 296 } 297 298 /* Return various clipped indexes for a given VA */ 299 static __inline vm_pindex_t 300 pmap_pte_index(vm_offset_t va) 301 { 302 303 return ((va >> PAGE_SHIFT) & RPTE_MASK); 304 } 305 306 /* Return a pointer to the PT slot that corresponds to a VA */ 307 static __inline pt_entry_t * 308 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va) 309 { 310 pt_entry_t *pte; 311 vm_paddr_t ptepa; 312 313 ptepa = (*l3e & NLB_MASK); 314 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa); 315 return (&pte[pmap_pte_index(va)]); 316 } 317 318 /* Return a pointer to the PD slot that corresponds to a VA */ 319 static __inline pt_entry_t * 320 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va) 321 { 322 pt_entry_t *l3e; 323 vm_paddr_t l3pa; 324 325 l3pa = (*l2e & NLB_MASK); 326 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa); 327 return (&l3e[pmap_pml3e_index(va)]); 328 } 329 330 /* Return a pointer to the PD slot that corresponds to a VA */ 331 static __inline pt_entry_t * 332 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va) 333 { 334 pt_entry_t *l2e; 335 vm_paddr_t l2pa; 336 337 l2pa = (*l1e & NLB_MASK); 338 339 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa); 340 return (&l2e[pmap_pml2e_index(va)]); 341 } 342 343 static __inline pml1_entry_t * 344 pmap_pml1e(pmap_t pmap, vm_offset_t va) 345 { 346 347 return (&pmap->pm_pml1[pmap_pml1e_index(va)]); 348 } 349 350 static pt_entry_t * 351 pmap_pml2e(pmap_t pmap, vm_offset_t va) 352 { 353 pt_entry_t *l1e; 354 355 l1e = pmap_pml1e(pmap, va); 356 if (l1e == NULL || (*l1e & RPTE_VALID) == 0) 357 return (NULL); 358 return (pmap_l1e_to_l2e(l1e, va)); 359 } 360 361 static __inline pt_entry_t * 362 pmap_pml3e(pmap_t pmap, vm_offset_t va) 363 { 364 pt_entry_t *l2e; 365 366 l2e = pmap_pml2e(pmap, va); 367 if (l2e == NULL || (*l2e & RPTE_VALID) == 0) 368 return (NULL); 369 return (pmap_l2e_to_l3e(l2e, va)); 370 } 371 372 static __inline pt_entry_t * 373 pmap_pte(pmap_t pmap, vm_offset_t va) 374 { 375 pt_entry_t *l3e; 376 377 l3e = pmap_pml3e(pmap, va); 378 if (l3e == NULL || (*l3e & RPTE_VALID) == 0) 379 return (NULL); 380 return (pmap_l3e_to_pte(l3e, va)); 381 } 382 383 int nkpt = 64; 384 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, 385 "Number of kernel page table pages allocated on bootup"); 386 387 vm_paddr_t dmaplimit; 388 389 SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); 390 391 static int pg_ps_enabled = 1; 392 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 393 &pg_ps_enabled, 0, "Are large page mappings enabled?"); 394 #ifdef INVARIANTS 395 #define VERBOSE_PMAP 0 396 #define VERBOSE_PROTECT 0 397 static int pmap_logging; 398 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN, 399 &pmap_logging, 0, "verbose debug logging"); 400 #endif 401 402 static u_int64_t KPTphys; /* phys addr of kernel level 1 */ 403 404 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ 405 406 static vm_offset_t qframe = 0; 407 static struct mtx qframe_mtx; 408 409 void mmu_radix_activate(struct thread *); 410 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int); 411 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *, 412 vm_size_t); 413 void mmu_radix_clear_modify(vm_page_t); 414 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); 415 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *); 416 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); 417 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 418 vm_prot_t); 419 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 420 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va); 421 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t); 422 void mmu_radix_kenter(vm_offset_t, vm_paddr_t); 423 vm_paddr_t mmu_radix_kextract(vm_offset_t); 424 void mmu_radix_kremove(vm_offset_t); 425 boolean_t mmu_radix_is_modified(vm_page_t); 426 boolean_t mmu_radix_is_prefaultable(pmap_t, vm_offset_t); 427 boolean_t mmu_radix_is_referenced(vm_page_t); 428 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t, 429 vm_pindex_t, vm_size_t); 430 boolean_t mmu_radix_page_exists_quick(pmap_t, vm_page_t); 431 void mmu_radix_page_init(vm_page_t); 432 boolean_t mmu_radix_page_is_mapped(vm_page_t m); 433 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t); 434 int mmu_radix_page_wired_mappings(vm_page_t); 435 int mmu_radix_pinit(pmap_t); 436 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 437 bool mmu_radix_ps_enabled(pmap_t); 438 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int); 439 void mmu_radix_qremove(vm_offset_t, int); 440 vm_offset_t mmu_radix_quick_enter_page(vm_page_t); 441 void mmu_radix_quick_remove_page(vm_offset_t); 442 boolean_t mmu_radix_ts_referenced(vm_page_t); 443 void mmu_radix_release(pmap_t); 444 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t); 445 void mmu_radix_remove_all(vm_page_t); 446 void mmu_radix_remove_pages(pmap_t); 447 void mmu_radix_remove_write(vm_page_t); 448 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t); 449 void mmu_radix_zero_page(vm_page_t); 450 void mmu_radix_zero_page_area(vm_page_t, int, int); 451 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); 452 void mmu_radix_page_array_startup(long pages); 453 454 #include "mmu_oea64.h" 455 456 /* 457 * Kernel MMU interface 458 */ 459 460 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t); 461 462 static void mmu_radix_copy_page(vm_page_t, vm_page_t); 463 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 464 vm_page_t *mb, vm_offset_t b_offset, int xfersize); 465 static void mmu_radix_growkernel(vm_offset_t); 466 static void mmu_radix_init(void); 467 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *); 468 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 469 static void mmu_radix_pinit0(pmap_t); 470 471 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t); 472 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); 473 static void mmu_radix_unmapdev(vm_offset_t, vm_size_t); 474 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma); 475 static boolean_t mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t); 476 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va); 477 static void mmu_radix_scan_init(void); 478 static void mmu_radix_cpu_bootstrap(int ap); 479 static void mmu_radix_tlbie_all(void); 480 481 static struct pmap_funcs mmu_radix_methods = { 482 .bootstrap = mmu_radix_bootstrap, 483 .copy_page = mmu_radix_copy_page, 484 .copy_pages = mmu_radix_copy_pages, 485 .cpu_bootstrap = mmu_radix_cpu_bootstrap, 486 .growkernel = mmu_radix_growkernel, 487 .init = mmu_radix_init, 488 .map = mmu_radix_map, 489 .mincore = mmu_radix_mincore, 490 .pinit = mmu_radix_pinit, 491 .pinit0 = mmu_radix_pinit0, 492 493 .mapdev = mmu_radix_mapdev, 494 .mapdev_attr = mmu_radix_mapdev_attr, 495 .unmapdev = mmu_radix_unmapdev, 496 .kenter_attr = mmu_radix_kenter_attr, 497 .dev_direct_mapped = mmu_radix_dev_direct_mapped, 498 .dumpsys_pa_init = mmu_radix_scan_init, 499 .dumpsys_map_chunk = mmu_radix_dumpsys_map, 500 .page_is_mapped = mmu_radix_page_is_mapped, 501 .ps_enabled = mmu_radix_ps_enabled, 502 .object_init_pt = mmu_radix_object_init_pt, 503 .protect = mmu_radix_protect, 504 /* pmap dispatcher interface */ 505 .clear_modify = mmu_radix_clear_modify, 506 .copy = mmu_radix_copy, 507 .enter = mmu_radix_enter, 508 .enter_object = mmu_radix_enter_object, 509 .enter_quick = mmu_radix_enter_quick, 510 .extract = mmu_radix_extract, 511 .extract_and_hold = mmu_radix_extract_and_hold, 512 .is_modified = mmu_radix_is_modified, 513 .is_prefaultable = mmu_radix_is_prefaultable, 514 .is_referenced = mmu_radix_is_referenced, 515 .ts_referenced = mmu_radix_ts_referenced, 516 .page_exists_quick = mmu_radix_page_exists_quick, 517 .page_init = mmu_radix_page_init, 518 .page_wired_mappings = mmu_radix_page_wired_mappings, 519 .qenter = mmu_radix_qenter, 520 .qremove = mmu_radix_qremove, 521 .release = mmu_radix_release, 522 .remove = mmu_radix_remove, 523 .remove_all = mmu_radix_remove_all, 524 .remove_write = mmu_radix_remove_write, 525 .unwire = mmu_radix_unwire, 526 .zero_page = mmu_radix_zero_page, 527 .zero_page_area = mmu_radix_zero_page_area, 528 .activate = mmu_radix_activate, 529 .quick_enter_page = mmu_radix_quick_enter_page, 530 .quick_remove_page = mmu_radix_quick_remove_page, 531 .page_set_memattr = mmu_radix_page_set_memattr, 532 .page_array_startup = mmu_radix_page_array_startup, 533 534 /* Internal interfaces */ 535 .kenter = mmu_radix_kenter, 536 .kextract = mmu_radix_kextract, 537 .kremove = mmu_radix_kremove, 538 .change_attr = mmu_radix_change_attr, 539 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr, 540 541 .tlbie_all = mmu_radix_tlbie_all, 542 }; 543 544 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods); 545 546 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 547 struct rwlock **lockp); 548 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va); 549 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *); 550 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 551 struct spglist *free, struct rwlock **lockp); 552 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 553 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 554 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 555 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde, 556 struct spglist *free); 557 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 558 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp); 559 560 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e, 561 u_int flags, struct rwlock **lockp); 562 #if VM_NRESERVLEVEL > 0 563 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 564 struct rwlock **lockp); 565 #endif 566 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 567 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); 568 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 569 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate); 570 571 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, 572 vm_prot_t prot, struct rwlock **lockp); 573 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, 574 u_int flags, vm_page_t m, struct rwlock **lockp); 575 576 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 577 static void free_pv_chunk(struct pv_chunk *pc); 578 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); 579 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va, 580 struct rwlock **lockp); 581 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, 582 struct rwlock **lockp); 583 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, 584 struct spglist *free); 585 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); 586 587 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start); 588 static void pmap_invalidate_all(pmap_t pmap); 589 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush); 590 591 /* 592 * Internal flags for pmap_enter()'s helper functions. 593 */ 594 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 595 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 596 597 #define UNIMPLEMENTED() panic("%s not implemented", __func__) 598 #define UNTESTED() panic("%s not yet tested", __func__) 599 600 601 602 /* Number of supported PID bits */ 603 static unsigned int isa3_pid_bits; 604 605 /* PID to start allocating from */ 606 static unsigned int isa3_base_pid; 607 608 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4) 609 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits) 610 611 612 /* 613 * Map of physical memory regions. 614 */ 615 static struct mem_region *regions, *pregions; 616 static struct numa_mem_region *numa_pregions; 617 static u_int phys_avail_count; 618 static int regions_sz, pregions_sz, numa_pregions_sz; 619 static struct pate *isa3_parttab; 620 static struct prte *isa3_proctab; 621 static vmem_t *asid_arena; 622 623 extern void bs_remap_earlyboot(void); 624 625 #define RADIX_PGD_SIZE_SHIFT 16 626 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT) 627 628 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3) 629 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t)) 630 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t)) 631 632 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */ 633 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */ 634 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */ 635 636 /* POWER9 only permits a 64k partition table size. */ 637 #define PARTTAB_SIZE_SHIFT 16 638 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT) 639 640 #define PARTTAB_HR (1UL << 63) /* host uses radix */ 641 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */ 642 643 /* TLB flush actions. Used as argument to tlbiel_all() */ 644 enum { 645 TLB_INVAL_SCOPE_LPID = 0, /* invalidate TLBs for current LPID */ 646 TLB_INVAL_SCOPE_GLOBAL = 1, /* invalidate all TLBs */ 647 }; 648 649 #define NPV_LIST_LOCKS MAXCPU 650 static int pmap_initialized; 651 static vm_paddr_t proctab0pa; 652 static vm_paddr_t parttab_phys; 653 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 654 655 /* 656 * Data for the pv entry allocation mechanism. 657 * Updates to pv_invl_gen are protected by the pv_list_locks[] 658 * elements, but reads are not. 659 */ 660 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 661 static struct mtx __exclusive_cache_line pv_chunks_mutex; 662 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; 663 static struct md_page *pv_table; 664 static struct md_page pv_dummy; 665 666 #ifdef PV_STATS 667 #define PV_STAT(x) do { x ; } while (0) 668 #else 669 #define PV_STAT(x) do { } while (0) 670 #endif 671 672 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT) 673 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)]) 674 675 #define PHYS_TO_PV_LIST_LOCK(pa) \ 676 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS]) 677 678 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 679 struct rwlock **_lockp = (lockp); \ 680 struct rwlock *_new_lock; \ 681 \ 682 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 683 if (_new_lock != *_lockp) { \ 684 if (*_lockp != NULL) \ 685 rw_wunlock(*_lockp); \ 686 *_lockp = _new_lock; \ 687 rw_wlock(*_lockp); \ 688 } \ 689 } while (0) 690 691 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 692 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 693 694 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 695 struct rwlock **_lockp = (lockp); \ 696 \ 697 if (*_lockp != NULL) { \ 698 rw_wunlock(*_lockp); \ 699 *_lockp = NULL; \ 700 } \ 701 } while (0) 702 703 #define VM_PAGE_TO_PV_LIST_LOCK(m) \ 704 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 705 706 /* 707 * We support 52 bits, hence: 708 * bits 52 - 31 = 21, 0b10101 709 * RTS encoding details 710 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long 711 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long 712 */ 713 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5)) 714 715 716 static int powernv_enabled = 1; 717 718 static inline void 719 tlbiel_radix_set_isa300(uint32_t set, uint32_t is, 720 uint32_t pid, uint32_t ric, uint32_t prs) 721 { 722 uint64_t rb; 723 uint64_t rs; 724 725 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53); 726 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31); 727 728 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 729 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 730 : "memory"); 731 } 732 733 static void 734 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is) 735 { 736 uint32_t set; 737 738 __asm __volatile("ptesync": : :"memory"); 739 740 /* 741 * Flush the first set of the TLB, and the entire Page Walk Cache 742 * and partition table entries. Then flush the remaining sets of the 743 * TLB. 744 */ 745 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 746 for (set = 1; set < num_sets; set++) 747 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 748 749 /* Do the same for process scoped entries. */ 750 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 751 for (set = 1; set < num_sets; set++) 752 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 753 754 __asm __volatile("ptesync": : :"memory"); 755 } 756 757 static void 758 mmu_radix_tlbiel_flush(int scope) 759 { 760 int is; 761 762 MPASS(scope == TLB_INVAL_SCOPE_LPID || 763 scope == TLB_INVAL_SCOPE_GLOBAL); 764 is = scope + 2; 765 766 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, is); 767 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); 768 } 769 770 static void 771 mmu_radix_tlbie_all() 772 { 773 /* TODO: LPID invalidate */ 774 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 775 } 776 777 static void 778 mmu_radix_init_amor(void) 779 { 780 /* 781 * In HV mode, we init AMOR (Authority Mask Override Register) so that 782 * the hypervisor and guest can setup IAMR (Instruction Authority Mask 783 * Register), enable key 0 and set it to 1. 784 * 785 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11) 786 */ 787 mtspr(SPR_AMOR, (3ul << 62)); 788 } 789 790 static void 791 mmu_radix_init_iamr(void) 792 { 793 /* 794 * Radix always uses key0 of the IAMR to determine if an access is 795 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 796 * fetch. 797 */ 798 mtspr(SPR_IAMR, (1ul << 62)); 799 } 800 801 static void 802 mmu_radix_pid_set(pmap_t pmap) 803 { 804 805 mtspr(SPR_PID, pmap->pm_pid); 806 isync(); 807 } 808 809 /* Quick sort callout for comparing physical addresses. */ 810 static int 811 pa_cmp(const void *a, const void *b) 812 { 813 const vm_paddr_t *pa = a, *pb = b; 814 815 if (*pa < *pb) 816 return (-1); 817 else if (*pa > *pb) 818 return (1); 819 else 820 return (0); 821 } 822 823 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) 824 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0) 825 #define pte_store(ptep, pte) do { \ 826 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \ 827 *(u_long *)(ptep) = (u_long)((pte) | PG_V | RPTE_LEAF); \ 828 } while (0) 829 /* 830 * NB: should only be used for adding directories - not for direct mappings 831 */ 832 #define pde_store(ptep, pa) do { \ 833 *(u_long *)(ptep) = (u_long)(pa|RPTE_VALID|RPTE_SHIFT); \ 834 } while (0) 835 836 #define pte_clear(ptep) do { \ 837 *(u_long *)(ptep) = (u_long)(0); \ 838 } while (0) 839 840 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */ 841 842 /* 843 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB 844 * (PTE) page mappings have identical settings for the following fields: 845 */ 846 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \ 847 PG_M | PG_A | RPTE_EAA_MASK | PG_V) 848 849 850 static __inline void 851 pmap_resident_count_inc(pmap_t pmap, int count) 852 { 853 854 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 855 pmap->pm_stats.resident_count += count; 856 } 857 858 static __inline void 859 pmap_resident_count_dec(pmap_t pmap, int count) 860 { 861 862 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 863 KASSERT(pmap->pm_stats.resident_count >= count, 864 ("pmap %p resident count underflow %ld %d", pmap, 865 pmap->pm_stats.resident_count, count)); 866 pmap->pm_stats.resident_count -= count; 867 } 868 869 static void 870 pagezero(vm_offset_t va) 871 { 872 va = trunc_page(va); 873 874 bzero((void *)va, PAGE_SIZE); 875 } 876 877 static uint64_t 878 allocpages(int n) 879 { 880 u_int64_t ret; 881 882 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE); 883 for (int i = 0; i < n; i++) 884 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE)); 885 return (ret); 886 } 887 888 static pt_entry_t * 889 kvtopte(vm_offset_t va) 890 { 891 pt_entry_t *l3e; 892 893 l3e = pmap_pml3e(kernel_pmap, va); 894 if ((*l3e & RPTE_VALID) == 0) 895 return (NULL); 896 return (pmap_l3e_to_pte(l3e, va)); 897 } 898 899 void 900 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa) 901 { 902 pt_entry_t *pte; 903 904 pte = kvtopte(va); 905 MPASS(pte != NULL); 906 *pte = pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | RPTE_EAA_W | \ 907 RPTE_EAA_P | PG_M | PG_A; 908 } 909 910 bool 911 mmu_radix_ps_enabled(pmap_t pmap) 912 { 913 return (pg_ps_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); 914 } 915 916 static pt_entry_t * 917 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e) 918 { 919 pml3_entry_t *l3e; 920 pt_entry_t *pte; 921 922 va &= PG_PS_FRAME; 923 l3e = pmap_pml3e(pmap, va); 924 if (l3e == NULL || (*l3e & PG_V) == 0) 925 return (NULL); 926 927 if (*l3e & RPTE_LEAF) { 928 *is_l3e = 1; 929 return (l3e); 930 } 931 *is_l3e = 0; 932 va &= PG_FRAME; 933 pte = pmap_l3e_to_pte(l3e, va); 934 if (pte == NULL || (*pte & PG_V) == 0) 935 return (NULL); 936 return (pte); 937 } 938 939 int 940 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags) 941 { 942 pt_entry_t *pte; 943 pt_entry_t startpte, origpte, newpte; 944 vm_page_t m; 945 int is_l3e; 946 947 startpte = 0; 948 retry: 949 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) 950 return (KERN_INVALID_ADDRESS); 951 origpte = newpte = *pte; 952 if (startpte == 0) { 953 startpte = origpte; 954 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) || 955 ((flags & VM_PROT_READ) && (startpte & PG_A))) { 956 pmap_invalidate_all(pmap); 957 #ifdef INVARIANTS 958 if (VERBOSE_PMAP || pmap_logging) 959 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n", 960 __func__, pmap, va, flags, origpte); 961 #endif 962 return (KERN_FAILURE); 963 } 964 } 965 #ifdef INVARIANTS 966 if (VERBOSE_PMAP || pmap_logging) 967 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va, 968 flags, origpte); 969 #endif 970 PMAP_LOCK(pmap); 971 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || 972 *pte != origpte) { 973 PMAP_UNLOCK(pmap); 974 return (KERN_FAILURE); 975 } 976 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME); 977 MPASS(m != NULL); 978 switch (flags) { 979 case VM_PROT_READ: 980 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0) 981 goto protfail; 982 newpte |= PG_A; 983 vm_page_aflag_set(m, PGA_REFERENCED); 984 break; 985 case VM_PROT_WRITE: 986 if ((newpte & RPTE_EAA_W) == 0) 987 goto protfail; 988 if (is_l3e) 989 goto protfail; 990 newpte |= PG_M; 991 vm_page_dirty(m); 992 break; 993 case VM_PROT_EXECUTE: 994 if ((newpte & RPTE_EAA_X) == 0) 995 goto protfail; 996 newpte |= PG_A; 997 vm_page_aflag_set(m, PGA_REFERENCED); 998 break; 999 } 1000 1001 if (!atomic_cmpset_long(pte, origpte, newpte)) 1002 goto retry; 1003 ptesync(); 1004 PMAP_UNLOCK(pmap); 1005 if (startpte == newpte) 1006 return (KERN_FAILURE); 1007 return (0); 1008 protfail: 1009 PMAP_UNLOCK(pmap); 1010 return (KERN_PROTECTION_FAILURE); 1011 } 1012 1013 /* 1014 * Returns TRUE if the given page is mapped individually or as part of 1015 * a 2mpage. Otherwise, returns FALSE. 1016 */ 1017 boolean_t 1018 mmu_radix_page_is_mapped(vm_page_t m) 1019 { 1020 struct rwlock *lock; 1021 boolean_t rv; 1022 1023 if ((m->oflags & VPO_UNMANAGED) != 0) 1024 return (FALSE); 1025 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 1026 rw_rlock(lock); 1027 rv = !TAILQ_EMPTY(&m->md.pv_list) || 1028 ((m->flags & PG_FICTITIOUS) == 0 && 1029 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); 1030 rw_runlock(lock); 1031 return (rv); 1032 } 1033 1034 /* 1035 * Determine the appropriate bits to set in a PTE or PDE for a specified 1036 * caching mode. 1037 */ 1038 static int 1039 pmap_cache_bits(vm_memattr_t ma) 1040 { 1041 if (ma != VM_MEMATTR_DEFAULT) { 1042 switch (ma) { 1043 case VM_MEMATTR_UNCACHEABLE: 1044 return (RPTE_ATTR_GUARDEDIO); 1045 case VM_MEMATTR_CACHEABLE: 1046 return (RPTE_ATTR_MEM); 1047 case VM_MEMATTR_WRITE_BACK: 1048 case VM_MEMATTR_PREFETCHABLE: 1049 case VM_MEMATTR_WRITE_COMBINING: 1050 return (RPTE_ATTR_UNGUARDEDIO); 1051 } 1052 } 1053 return (0); 1054 } 1055 1056 static void 1057 pmap_invalidate_page(pmap_t pmap, vm_offset_t start) 1058 { 1059 ptesync(); 1060 if (pmap == kernel_pmap) 1061 radix_tlbie_invlpg_kernel_4k(start); 1062 else 1063 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1064 ttusync(); 1065 } 1066 1067 static void 1068 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start) 1069 { 1070 ptesync(); 1071 if (pmap == kernel_pmap) 1072 radix_tlbie_invlpg_kernel_2m(start); 1073 else 1074 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start); 1075 ttusync(); 1076 } 1077 1078 static void 1079 pmap_invalidate_pwc(pmap_t pmap) 1080 { 1081 ptesync(); 1082 if (pmap == kernel_pmap) 1083 radix_tlbie_invlpwc_kernel(); 1084 else 1085 radix_tlbie_invlpwc_user(pmap->pm_pid); 1086 ttusync(); 1087 } 1088 1089 static void 1090 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end) 1091 { 1092 if (((start - end) >> PAGE_SHIFT) > 8) { 1093 pmap_invalidate_all(pmap); 1094 return; 1095 } 1096 ptesync(); 1097 if (pmap == kernel_pmap) { 1098 while (start < end) { 1099 radix_tlbie_invlpg_kernel_4k(start); 1100 start += PAGE_SIZE; 1101 } 1102 } else { 1103 while (start < end) { 1104 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1105 start += PAGE_SIZE; 1106 } 1107 } 1108 ttusync(); 1109 } 1110 1111 static void 1112 pmap_invalidate_all(pmap_t pmap) 1113 { 1114 ptesync(); 1115 if (pmap == kernel_pmap) 1116 radix_tlbie_flush_kernel(); 1117 else 1118 radix_tlbie_flush_user(pmap->pm_pid); 1119 ttusync(); 1120 } 1121 1122 static void 1123 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e) 1124 { 1125 1126 /* 1127 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created 1128 * by a promotion that did not invalidate the 512 4KB page mappings 1129 * that might exist in the TLB. Consequently, at this point, the TLB 1130 * may hold both 4KB and 2MB page mappings for the address range [va, 1131 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here. 1132 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any 1133 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a 1134 * single INVLPG suffices to invalidate the 2MB page mapping from the 1135 * TLB. 1136 */ 1137 ptesync(); 1138 if ((l3e & PG_PROMOTED) != 0) 1139 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1); 1140 else 1141 pmap_invalidate_page_2m(pmap, va); 1142 1143 pmap_invalidate_pwc(pmap); 1144 } 1145 1146 static __inline struct pv_chunk * 1147 pv_to_chunk(pv_entry_t pv) 1148 { 1149 1150 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1151 } 1152 1153 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1154 1155 #define PC_FREE0 0xfffffffffffffffful 1156 #define PC_FREE1 0x3ffffffffffffffful 1157 1158 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 }; 1159 1160 /* 1161 * Ensure that the number of spare PV entries in the specified pmap meets or 1162 * exceeds the given count, "needed". 1163 * 1164 * The given PV list lock may be released. 1165 */ 1166 static void 1167 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 1168 { 1169 struct pch new_tail; 1170 struct pv_chunk *pc; 1171 vm_page_t m; 1172 int avail, free; 1173 bool reclaimed; 1174 1175 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1176 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 1177 1178 /* 1179 * Newly allocated PV chunks must be stored in a private list until 1180 * the required number of PV chunks have been allocated. Otherwise, 1181 * reclaim_pv_chunk() could recycle one of these chunks. In 1182 * contrast, these chunks must be added to the pmap upon allocation. 1183 */ 1184 TAILQ_INIT(&new_tail); 1185 retry: 1186 avail = 0; 1187 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 1188 // if ((cpu_feature2 & CPUID2_POPCNT) == 0) 1189 bit_count((bitstr_t *)pc->pc_map, 0, 1190 sizeof(pc->pc_map) * NBBY, &free); 1191 #if 0 1192 free = popcnt_pc_map_pq(pc->pc_map); 1193 #endif 1194 if (free == 0) 1195 break; 1196 avail += free; 1197 if (avail >= needed) 1198 break; 1199 } 1200 for (reclaimed = false; avail < needed; avail += _NPCPV) { 1201 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1202 VM_ALLOC_WIRED); 1203 if (m == NULL) { 1204 m = reclaim_pv_chunk(pmap, lockp); 1205 if (m == NULL) 1206 goto retry; 1207 reclaimed = true; 1208 } 1209 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1210 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1211 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1212 pc->pc_pmap = pmap; 1213 pc->pc_map[0] = PC_FREE0; 1214 pc->pc_map[1] = PC_FREE1; 1215 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1216 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); 1217 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); 1218 1219 /* 1220 * The reclaim might have freed a chunk from the current pmap. 1221 * If that chunk contained available entries, we need to 1222 * re-count the number of available entries. 1223 */ 1224 if (reclaimed) 1225 goto retry; 1226 } 1227 if (!TAILQ_EMPTY(&new_tail)) { 1228 mtx_lock(&pv_chunks_mutex); 1229 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); 1230 mtx_unlock(&pv_chunks_mutex); 1231 } 1232 } 1233 1234 /* 1235 * First find and then remove the pv entry for the specified pmap and virtual 1236 * address from the specified pv list. Returns the pv entry if found and NULL 1237 * otherwise. This operation can be performed on pv lists for either 4KB or 1238 * 2MB page mappings. 1239 */ 1240 static __inline pv_entry_t 1241 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1242 { 1243 pv_entry_t pv; 1244 1245 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 1246 #ifdef INVARIANTS 1247 if (PV_PMAP(pv) == NULL) { 1248 printf("corrupted pv_chunk/pv %p\n", pv); 1249 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":"); 1250 } 1251 MPASS(PV_PMAP(pv) != NULL); 1252 MPASS(pv->pv_va != 0); 1253 #endif 1254 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1255 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 1256 pvh->pv_gen++; 1257 break; 1258 } 1259 } 1260 return (pv); 1261 } 1262 1263 /* 1264 * After demotion from a 2MB page mapping to 512 4KB page mappings, 1265 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 1266 * entries for each of the 4KB page mappings. 1267 */ 1268 static void 1269 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1270 struct rwlock **lockp) 1271 { 1272 struct md_page *pvh; 1273 struct pv_chunk *pc; 1274 pv_entry_t pv; 1275 vm_offset_t va_last; 1276 vm_page_t m; 1277 int bit, field; 1278 1279 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1280 KASSERT((pa & L3_PAGE_MASK) == 0, 1281 ("pmap_pv_demote_pde: pa is not 2mpage aligned")); 1282 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1283 1284 /* 1285 * Transfer the 2mpage's pv entry for this mapping to the first 1286 * page's pv list. Once this transfer begins, the pv list lock 1287 * must not be released until the last pv entry is reinstantiated. 1288 */ 1289 pvh = pa_to_pvh(pa); 1290 va = trunc_2mpage(va); 1291 pv = pmap_pvh_remove(pvh, pmap, va); 1292 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); 1293 m = PHYS_TO_VM_PAGE(pa); 1294 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1295 1296 m->md.pv_gen++; 1297 /* Instantiate the remaining NPTEPG - 1 pv entries. */ 1298 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); 1299 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1300 for (;;) { 1301 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1302 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 1303 , ("pmap_pv_demote_pde: missing spare")); 1304 for (field = 0; field < _NPCM; field++) { 1305 while (pc->pc_map[field]) { 1306 bit = cnttzd(pc->pc_map[field]); 1307 pc->pc_map[field] &= ~(1ul << bit); 1308 pv = &pc->pc_pventry[field * 64 + bit]; 1309 va += PAGE_SIZE; 1310 pv->pv_va = va; 1311 m++; 1312 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1313 ("pmap_pv_demote_pde: page %p is not managed", m)); 1314 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1315 1316 m->md.pv_gen++; 1317 if (va == va_last) 1318 goto out; 1319 } 1320 } 1321 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1322 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1323 } 1324 out: 1325 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1326 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1327 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1328 } 1329 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); 1330 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); 1331 } 1332 1333 static void 1334 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap) 1335 { 1336 1337 if (pmap == NULL) 1338 return; 1339 pmap_invalidate_all(pmap); 1340 if (pmap != locked_pmap) 1341 PMAP_UNLOCK(pmap); 1342 } 1343 1344 /* 1345 * We are in a serious low memory condition. Resort to 1346 * drastic measures to free some pages so we can allocate 1347 * another pv entry chunk. 1348 * 1349 * Returns NULL if PV entries were reclaimed from the specified pmap. 1350 * 1351 * We do not, however, unmap 2mpages because subsequent accesses will 1352 * allocate per-page pv entries until repromotion occurs, thereby 1353 * exacerbating the shortage of free pv entries. 1354 */ 1355 static int active_reclaims = 0; 1356 static vm_page_t 1357 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 1358 { 1359 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 1360 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 1361 struct md_page *pvh; 1362 pml3_entry_t *l3e; 1363 pmap_t next_pmap, pmap; 1364 pt_entry_t *pte, tpte; 1365 pv_entry_t pv; 1366 vm_offset_t va; 1367 vm_page_t m, m_pc; 1368 struct spglist free; 1369 uint64_t inuse; 1370 int bit, field, freed; 1371 1372 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1373 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 1374 pmap = NULL; 1375 m_pc = NULL; 1376 SLIST_INIT(&free); 1377 bzero(&pc_marker_b, sizeof(pc_marker_b)); 1378 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 1379 pc_marker = (struct pv_chunk *)&pc_marker_b; 1380 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 1381 1382 mtx_lock(&pv_chunks_mutex); 1383 active_reclaims++; 1384 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru); 1385 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru); 1386 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 1387 SLIST_EMPTY(&free)) { 1388 next_pmap = pc->pc_pmap; 1389 if (next_pmap == NULL) { 1390 /* 1391 * The next chunk is a marker. However, it is 1392 * not our marker, so active_reclaims must be 1393 * > 1. Consequently, the next_chunk code 1394 * will not rotate the pv_chunks list. 1395 */ 1396 goto next_chunk; 1397 } 1398 mtx_unlock(&pv_chunks_mutex); 1399 1400 /* 1401 * A pv_chunk can only be removed from the pc_lru list 1402 * when both pc_chunks_mutex is owned and the 1403 * corresponding pmap is locked. 1404 */ 1405 if (pmap != next_pmap) { 1406 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1407 pmap = next_pmap; 1408 /* Avoid deadlock and lock recursion. */ 1409 if (pmap > locked_pmap) { 1410 RELEASE_PV_LIST_LOCK(lockp); 1411 PMAP_LOCK(pmap); 1412 mtx_lock(&pv_chunks_mutex); 1413 continue; 1414 } else if (pmap != locked_pmap) { 1415 if (PMAP_TRYLOCK(pmap)) { 1416 mtx_lock(&pv_chunks_mutex); 1417 continue; 1418 } else { 1419 pmap = NULL; /* pmap is not locked */ 1420 mtx_lock(&pv_chunks_mutex); 1421 pc = TAILQ_NEXT(pc_marker, pc_lru); 1422 if (pc == NULL || 1423 pc->pc_pmap != next_pmap) 1424 continue; 1425 goto next_chunk; 1426 } 1427 } 1428 } 1429 1430 /* 1431 * Destroy every non-wired, 4 KB page mapping in the chunk. 1432 */ 1433 freed = 0; 1434 for (field = 0; field < _NPCM; field++) { 1435 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 1436 inuse != 0; inuse &= ~(1UL << bit)) { 1437 bit = cnttzd(inuse); 1438 pv = &pc->pc_pventry[field * 64 + bit]; 1439 va = pv->pv_va; 1440 l3e = pmap_pml3e(pmap, va); 1441 if ((*l3e & RPTE_LEAF) != 0) 1442 continue; 1443 pte = pmap_l3e_to_pte(l3e, va); 1444 if ((*pte & PG_W) != 0) 1445 continue; 1446 tpte = pte_load_clear(pte); 1447 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); 1448 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 1449 vm_page_dirty(m); 1450 if ((tpte & PG_A) != 0) 1451 vm_page_aflag_set(m, PGA_REFERENCED); 1452 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1453 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 1454 1455 m->md.pv_gen++; 1456 if (TAILQ_EMPTY(&m->md.pv_list) && 1457 (m->flags & PG_FICTITIOUS) == 0) { 1458 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 1459 if (TAILQ_EMPTY(&pvh->pv_list)) { 1460 vm_page_aflag_clear(m, 1461 PGA_WRITEABLE); 1462 } 1463 } 1464 pc->pc_map[field] |= 1UL << bit; 1465 pmap_unuse_pt(pmap, va, *l3e, &free); 1466 freed++; 1467 } 1468 } 1469 if (freed == 0) { 1470 mtx_lock(&pv_chunks_mutex); 1471 goto next_chunk; 1472 } 1473 /* Every freed mapping is for a 4 KB page. */ 1474 pmap_resident_count_dec(pmap, freed); 1475 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 1476 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 1477 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 1478 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1479 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) { 1480 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1481 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1482 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1483 /* Entire chunk is free; return it. */ 1484 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1485 mtx_lock(&pv_chunks_mutex); 1486 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1487 break; 1488 } 1489 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1490 mtx_lock(&pv_chunks_mutex); 1491 /* One freed pv entry in locked_pmap is sufficient. */ 1492 if (pmap == locked_pmap) 1493 break; 1494 next_chunk: 1495 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1496 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru); 1497 if (active_reclaims == 1 && pmap != NULL) { 1498 /* 1499 * Rotate the pv chunks list so that we do not 1500 * scan the same pv chunks that could not be 1501 * freed (because they contained a wired 1502 * and/or superpage mapping) on every 1503 * invocation of reclaim_pv_chunk(). 1504 */ 1505 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) { 1506 MPASS(pc->pc_pmap != NULL); 1507 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1508 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1509 } 1510 } 1511 } 1512 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1513 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); 1514 active_reclaims--; 1515 mtx_unlock(&pv_chunks_mutex); 1516 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1517 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 1518 m_pc = SLIST_FIRST(&free); 1519 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 1520 /* Recycle a freed page table page. */ 1521 m_pc->ref_count = 1; 1522 } 1523 vm_page_free_pages_toq(&free, true); 1524 return (m_pc); 1525 } 1526 1527 /* 1528 * free the pv_entry back to the free list 1529 */ 1530 static void 1531 free_pv_entry(pmap_t pmap, pv_entry_t pv) 1532 { 1533 struct pv_chunk *pc; 1534 int idx, field, bit; 1535 1536 #ifdef VERBOSE_PV 1537 if (pmap != kernel_pmap) 1538 printf("%s(%p, %p)\n", __func__, pmap, pv); 1539 #endif 1540 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1541 PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 1542 PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 1543 PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 1544 pc = pv_to_chunk(pv); 1545 idx = pv - &pc->pc_pventry[0]; 1546 field = idx / 64; 1547 bit = idx % 64; 1548 pc->pc_map[field] |= 1ul << bit; 1549 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) { 1550 /* 98% of the time, pc is already at the head of the list. */ 1551 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 1552 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1553 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1554 } 1555 return; 1556 } 1557 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1558 free_pv_chunk(pc); 1559 } 1560 1561 static void 1562 free_pv_chunk(struct pv_chunk *pc) 1563 { 1564 vm_page_t m; 1565 1566 mtx_lock(&pv_chunks_mutex); 1567 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1568 mtx_unlock(&pv_chunks_mutex); 1569 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1570 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1571 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1572 /* entire chunk is free, return it */ 1573 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1574 vm_page_unwire_noq(m); 1575 vm_page_free(m); 1576 } 1577 1578 /* 1579 * Returns a new PV entry, allocating a new PV chunk from the system when 1580 * needed. If this PV chunk allocation fails and a PV list lock pointer was 1581 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 1582 * returned. 1583 * 1584 * The given PV list lock may be released. 1585 */ 1586 static pv_entry_t 1587 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 1588 { 1589 int bit, field; 1590 pv_entry_t pv; 1591 struct pv_chunk *pc; 1592 vm_page_t m; 1593 1594 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1595 PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 1596 retry: 1597 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1598 if (pc != NULL) { 1599 for (field = 0; field < _NPCM; field++) { 1600 if (pc->pc_map[field]) { 1601 bit = cnttzd(pc->pc_map[field]); 1602 break; 1603 } 1604 } 1605 if (field < _NPCM) { 1606 pv = &pc->pc_pventry[field * 64 + bit]; 1607 pc->pc_map[field] &= ~(1ul << bit); 1608 /* If this was the last item, move it to tail */ 1609 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1610 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1611 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 1612 pc_list); 1613 } 1614 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1615 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 1616 MPASS(PV_PMAP(pv) != NULL); 1617 return (pv); 1618 } 1619 } 1620 /* No free items, allocate another chunk */ 1621 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1622 VM_ALLOC_WIRED); 1623 if (m == NULL) { 1624 if (lockp == NULL) { 1625 PV_STAT(pc_chunk_tryfail++); 1626 return (NULL); 1627 } 1628 m = reclaim_pv_chunk(pmap, lockp); 1629 if (m == NULL) 1630 goto retry; 1631 } 1632 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1633 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1634 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1635 pc->pc_pmap = pmap; 1636 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ 1637 pc->pc_map[1] = PC_FREE1; 1638 mtx_lock(&pv_chunks_mutex); 1639 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1640 mtx_unlock(&pv_chunks_mutex); 1641 pv = &pc->pc_pventry[0]; 1642 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1643 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1644 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 1645 MPASS(PV_PMAP(pv) != NULL); 1646 return (pv); 1647 } 1648 1649 #if VM_NRESERVLEVEL > 0 1650 /* 1651 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 1652 * replace the many pv entries for the 4KB page mappings by a single pv entry 1653 * for the 2MB page mapping. 1654 */ 1655 static void 1656 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1657 struct rwlock **lockp) 1658 { 1659 struct md_page *pvh; 1660 pv_entry_t pv; 1661 vm_offset_t va_last; 1662 vm_page_t m; 1663 1664 KASSERT((pa & L3_PAGE_MASK) == 0, 1665 ("pmap_pv_promote_pde: pa is not 2mpage aligned")); 1666 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1667 1668 /* 1669 * Transfer the first page's pv entry for this mapping to the 2mpage's 1670 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 1671 * a transfer avoids the possibility that get_pv_entry() calls 1672 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 1673 * mappings that is being promoted. 1674 */ 1675 m = PHYS_TO_VM_PAGE(pa); 1676 va = trunc_2mpage(va); 1677 pv = pmap_pvh_remove(&m->md, pmap, va); 1678 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); 1679 pvh = pa_to_pvh(pa); 1680 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 1681 pvh->pv_gen++; 1682 /* Free the remaining NPTEPG - 1 pv entries. */ 1683 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1684 do { 1685 m++; 1686 va += PAGE_SIZE; 1687 pmap_pvh_free(&m->md, pmap, va); 1688 } while (va < va_last); 1689 } 1690 #endif /* VM_NRESERVLEVEL > 0 */ 1691 1692 /* 1693 * First find and then destroy the pv entry for the specified pmap and virtual 1694 * address. This operation can be performed on pv lists for either 4KB or 2MB 1695 * page mappings. 1696 */ 1697 static void 1698 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1699 { 1700 pv_entry_t pv; 1701 1702 pv = pmap_pvh_remove(pvh, pmap, va); 1703 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 1704 free_pv_entry(pmap, pv); 1705 } 1706 1707 /* 1708 * Conditionally create the PV entry for a 4KB page mapping if the required 1709 * memory can be allocated without resorting to reclamation. 1710 */ 1711 static boolean_t 1712 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 1713 struct rwlock **lockp) 1714 { 1715 pv_entry_t pv; 1716 1717 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1718 /* Pass NULL instead of the lock pointer to disable reclamation. */ 1719 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 1720 pv->pv_va = va; 1721 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1722 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1723 m->md.pv_gen++; 1724 return (TRUE); 1725 } else 1726 return (FALSE); 1727 } 1728 1729 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX]; 1730 #ifdef INVARIANTS 1731 static void 1732 validate_addr(vm_paddr_t addr, vm_size_t size) 1733 { 1734 vm_paddr_t end = addr + size; 1735 bool found = false; 1736 1737 for (int i = 0; i < 2 * phys_avail_count; i += 2) { 1738 if (addr >= phys_avail_debug[i] && 1739 end <= phys_avail_debug[i + 1]) { 1740 found = true; 1741 break; 1742 } 1743 } 1744 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array", 1745 addr, end)); 1746 } 1747 #else 1748 static void validate_addr(vm_paddr_t addr, vm_size_t size) {} 1749 #endif 1750 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A) 1751 1752 static vm_paddr_t 1753 alloc_pt_page(void) 1754 { 1755 vm_paddr_t page; 1756 1757 page = allocpages(1); 1758 pagezero(PHYS_TO_DMAP(page)); 1759 return (page); 1760 } 1761 1762 static void 1763 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end) 1764 { 1765 pt_entry_t *pte, pteval; 1766 vm_paddr_t page; 1767 1768 if (bootverbose) 1769 printf("%s %lx -> %lx\n", __func__, start, end); 1770 while (start < end) { 1771 pteval = start | DMAP_PAGE_BITS; 1772 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start)); 1773 if ((*pte & RPTE_VALID) == 0) { 1774 page = alloc_pt_page(); 1775 pde_store(pte, page); 1776 } 1777 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start)); 1778 if ((start & L2_PAGE_MASK) == 0 && 1779 end - start >= L2_PAGE_SIZE) { 1780 start += L2_PAGE_SIZE; 1781 goto done; 1782 } else if ((*pte & RPTE_VALID) == 0) { 1783 page = alloc_pt_page(); 1784 pde_store(pte, page); 1785 } 1786 1787 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start)); 1788 if ((start & L3_PAGE_MASK) == 0 && 1789 end - start >= L3_PAGE_SIZE) { 1790 start += L3_PAGE_SIZE; 1791 goto done; 1792 } else if ((*pte & RPTE_VALID) == 0) { 1793 page = alloc_pt_page(); 1794 pde_store(pte, page); 1795 } 1796 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start)); 1797 start += PAGE_SIZE; 1798 done: 1799 pte_store(pte, pteval); 1800 } 1801 } 1802 1803 static void 1804 mmu_radix_dmap_populate(vm_size_t hwphyssz) 1805 { 1806 vm_paddr_t start, end; 1807 1808 for (int i = 0; i < pregions_sz; i++) { 1809 start = pregions[i].mr_start; 1810 end = start + pregions[i].mr_size; 1811 if (hwphyssz && start >= hwphyssz) 1812 break; 1813 if (hwphyssz && hwphyssz < end) 1814 end = hwphyssz; 1815 mmu_radix_dmap_range(start, end); 1816 } 1817 } 1818 1819 static void 1820 mmu_radix_setup_pagetables(vm_size_t hwphyssz) 1821 { 1822 vm_paddr_t ptpages, pages; 1823 pt_entry_t *pte; 1824 vm_paddr_t l1phys; 1825 1826 bzero(kernel_pmap, sizeof(struct pmap)); 1827 PMAP_LOCK_INIT(kernel_pmap); 1828 1829 ptpages = allocpages(2); 1830 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE); 1831 validate_addr(l1phys, RADIX_PGD_SIZE); 1832 if (bootverbose) 1833 printf("l1phys=%lx\n", l1phys); 1834 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0); 1835 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++) 1836 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE)); 1837 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys); 1838 1839 mmu_radix_dmap_populate(hwphyssz); 1840 1841 /* 1842 * Create page tables for first 128MB of KVA 1843 */ 1844 pages = ptpages; 1845 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS); 1846 *pte = (pages | RPTE_VALID | RPTE_SHIFT); 1847 pages += PAGE_SIZE; 1848 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS); 1849 *pte = (pages | RPTE_VALID | RPTE_SHIFT); 1850 pages += PAGE_SIZE; 1851 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS); 1852 /* 1853 * the kernel page table pages need to be preserved in 1854 * phys_avail and not overlap with previous allocations 1855 */ 1856 pages = allocpages(nkpt); 1857 if (bootverbose) { 1858 printf("phys_avail after dmap populate and nkpt allocation\n"); 1859 for (int j = 0; j < 2 * phys_avail_count; j+=2) 1860 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1861 j, phys_avail[j], j + 1, phys_avail[j + 1]); 1862 } 1863 KPTphys = pages; 1864 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE) 1865 *pte = (pages | RPTE_VALID | RPTE_SHIFT); 1866 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE; 1867 if (bootverbose) 1868 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1); 1869 /* 1870 * Add a physical memory segment (vm_phys_seg) corresponding to the 1871 * preallocated kernel page table pages so that vm_page structures 1872 * representing these pages will be created. The vm_page structures 1873 * are required for promotion of the corresponding kernel virtual 1874 * addresses to superpage mappings. 1875 */ 1876 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt)); 1877 } 1878 1879 static void 1880 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end) 1881 { 1882 vm_paddr_t kpstart, kpend; 1883 vm_size_t physsz, hwphyssz; 1884 //uint64_t l2virt; 1885 int rm_pavail, proctab_size; 1886 int i, j; 1887 1888 kpstart = start & ~DMAP_BASE_ADDRESS; 1889 kpend = end & ~DMAP_BASE_ADDRESS; 1890 1891 /* Get physical memory regions from firmware */ 1892 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 1893 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory"); 1894 1895 if (2 * VM_PHYSSEG_MAX < regions_sz) 1896 panic("mmu_radix_early_bootstrap: phys_avail too small"); 1897 1898 if (bootverbose) 1899 for (int i = 0; i < regions_sz; i++) 1900 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n", 1901 i, regions[i].mr_start, i, regions[i].mr_size); 1902 /* 1903 * XXX workaround a simulator bug 1904 */ 1905 for (int i = 0; i < regions_sz; i++) 1906 if (regions[i].mr_start & PAGE_MASK) { 1907 regions[i].mr_start += PAGE_MASK; 1908 regions[i].mr_start &= ~PAGE_MASK; 1909 regions[i].mr_size &= ~PAGE_MASK; 1910 } 1911 if (bootverbose) 1912 for (int i = 0; i < pregions_sz; i++) 1913 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n", 1914 i, pregions[i].mr_start, i, pregions[i].mr_size); 1915 1916 phys_avail_count = 0; 1917 physsz = 0; 1918 hwphyssz = 0; 1919 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 1920 for (i = 0, j = 0; i < regions_sz; i++) { 1921 if (bootverbose) 1922 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n", 1923 i, regions[i].mr_start, i, regions[i].mr_size); 1924 1925 if (regions[i].mr_size < PAGE_SIZE) 1926 continue; 1927 1928 if (hwphyssz != 0 && 1929 (physsz + regions[i].mr_size) >= hwphyssz) { 1930 if (physsz < hwphyssz) { 1931 phys_avail[j] = regions[i].mr_start; 1932 phys_avail[j + 1] = regions[i].mr_start + 1933 (hwphyssz - physsz); 1934 physsz = hwphyssz; 1935 phys_avail_count++; 1936 dump_avail[j] = phys_avail[j]; 1937 dump_avail[j + 1] = phys_avail[j + 1]; 1938 } 1939 break; 1940 } 1941 phys_avail[j] = regions[i].mr_start; 1942 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 1943 dump_avail[j] = phys_avail[j]; 1944 dump_avail[j + 1] = phys_avail[j + 1]; 1945 1946 phys_avail_count++; 1947 physsz += regions[i].mr_size; 1948 j += 2; 1949 } 1950 1951 /* Check for overlap with the kernel and exception vectors */ 1952 rm_pavail = 0; 1953 for (j = 0; j < 2 * phys_avail_count; j+=2) { 1954 if (phys_avail[j] < EXC_LAST) 1955 phys_avail[j] += EXC_LAST; 1956 1957 if (phys_avail[j] >= kpstart && 1958 phys_avail[j + 1] <= kpend) { 1959 phys_avail[j] = phys_avail[j + 1] = ~0; 1960 rm_pavail++; 1961 continue; 1962 } 1963 1964 if (kpstart >= phys_avail[j] && 1965 kpstart < phys_avail[j + 1]) { 1966 if (kpend < phys_avail[j + 1]) { 1967 phys_avail[2 * phys_avail_count] = 1968 (kpend & ~PAGE_MASK) + PAGE_SIZE; 1969 phys_avail[2 * phys_avail_count + 1] = 1970 phys_avail[j + 1]; 1971 phys_avail_count++; 1972 } 1973 1974 phys_avail[j + 1] = kpstart & ~PAGE_MASK; 1975 } 1976 1977 if (kpend >= phys_avail[j] && 1978 kpend < phys_avail[j + 1]) { 1979 if (kpstart > phys_avail[j]) { 1980 phys_avail[2 * phys_avail_count] = phys_avail[j]; 1981 phys_avail[2 * phys_avail_count + 1] = 1982 kpstart & ~PAGE_MASK; 1983 phys_avail_count++; 1984 } 1985 1986 phys_avail[j] = (kpend & ~PAGE_MASK) + 1987 PAGE_SIZE; 1988 } 1989 } 1990 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp); 1991 for (i = 0; i < 2 * phys_avail_count; i++) 1992 phys_avail_debug[i] = phys_avail[i]; 1993 1994 /* Remove physical available regions marked for removal (~0) */ 1995 if (rm_pavail) { 1996 phys_avail_count -= rm_pavail; 1997 for (i = 2 * phys_avail_count; 1998 i < 2*(phys_avail_count + rm_pavail); i+=2) 1999 phys_avail[i] = phys_avail[i + 1] = 0; 2000 } 2001 if (bootverbose) { 2002 printf("phys_avail ranges after filtering:\n"); 2003 for (j = 0; j < 2 * phys_avail_count; j+=2) 2004 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 2005 j, phys_avail[j], j + 1, phys_avail[j + 1]); 2006 } 2007 physmem = btoc(physsz); 2008 2009 /* XXX assume we're running non-virtualized and 2010 * we don't support BHYVE 2011 */ 2012 if (isa3_pid_bits == 0) 2013 isa3_pid_bits = 20; 2014 parttab_phys = moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE); 2015 validate_addr(parttab_phys, PARTTAB_SIZE); 2016 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++) 2017 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE)); 2018 2019 proctab_size = 1UL << PROCTAB_SIZE_SHIFT; 2020 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size); 2021 validate_addr(proctab0pa, proctab_size); 2022 for (int i = 0; i < proctab_size/PAGE_SIZE; i++) 2023 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE)); 2024 2025 mmu_radix_setup_pagetables(hwphyssz); 2026 } 2027 2028 static void 2029 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end) 2030 { 2031 int i; 2032 vm_paddr_t pa; 2033 void *dpcpu; 2034 vm_offset_t va; 2035 2036 /* 2037 * Set up the Open Firmware pmap and add its mappings if not in real 2038 * mode. 2039 */ 2040 if (bootverbose) 2041 printf("%s enter\n", __func__); 2042 2043 /* 2044 * Calculate the last available physical address, and reserve the 2045 * vm_page_array (upper bound). 2046 */ 2047 Maxmem = 0; 2048 for (i = 0; phys_avail[i + 2] != 0; i += 2) 2049 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1])); 2050 2051 /* 2052 * Set the start and end of kva. 2053 */ 2054 virtual_avail = VM_MIN_KERNEL_ADDRESS; 2055 virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; 2056 2057 /* 2058 * Remap any early IO mappings (console framebuffer, etc.) 2059 */ 2060 bs_remap_earlyboot(); 2061 2062 /* 2063 * Allocate a kernel stack with a guard page for thread0 and map it 2064 * into the kernel page map. 2065 */ 2066 pa = allocpages(kstack_pages); 2067 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 2068 virtual_avail = va + kstack_pages * PAGE_SIZE; 2069 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); 2070 thread0.td_kstack = va; 2071 for (i = 0; i < kstack_pages; i++) { 2072 mmu_radix_kenter(va, pa); 2073 pa += PAGE_SIZE; 2074 va += PAGE_SIZE; 2075 } 2076 thread0.td_kstack_pages = kstack_pages; 2077 2078 /* 2079 * Allocate virtual address space for the message buffer. 2080 */ 2081 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT); 2082 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa); 2083 2084 /* 2085 * Allocate virtual address space for the dynamic percpu area. 2086 */ 2087 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT); 2088 dpcpu = (void *)PHYS_TO_DMAP(pa); 2089 dpcpu_init(dpcpu, curcpu); 2090 /* 2091 * Reserve some special page table entries/VA space for temporary 2092 * mapping of pages. 2093 */ 2094 } 2095 2096 static void 2097 mmu_parttab_init(void) 2098 { 2099 uint64_t ptcr; 2100 2101 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys); 2102 2103 if (bootverbose) 2104 printf("%s parttab: %p\n", __func__, isa3_parttab); 2105 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2106 if (bootverbose) 2107 printf("setting ptcr %lx\n", ptcr); 2108 mtspr(SPR_PTCR, ptcr); 2109 } 2110 2111 static void 2112 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab) 2113 { 2114 uint64_t prev; 2115 2116 if (bootverbose) 2117 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab, 2118 lpid, pagetab, proctab); 2119 prev = be64toh(isa3_parttab[lpid].pagetab); 2120 isa3_parttab[lpid].pagetab = htobe64(pagetab); 2121 isa3_parttab[lpid].proctab = htobe64(proctab); 2122 2123 if (prev & PARTTAB_HR) { 2124 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : : 2125 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2126 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2127 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2128 } else { 2129 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : 2130 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2131 } 2132 ttusync(); 2133 } 2134 2135 static void 2136 mmu_radix_parttab_init(void) 2137 { 2138 uint64_t pagetab; 2139 2140 mmu_parttab_init(); 2141 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \ 2142 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR; 2143 mmu_parttab_update(0, pagetab, 0); 2144 } 2145 2146 static void 2147 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size) 2148 { 2149 uint64_t pagetab, proctab; 2150 2151 pagetab = be64toh(isa3_parttab[0].pagetab); 2152 proctab = proctabpa | table_size | PARTTAB_GR; 2153 mmu_parttab_update(0, pagetab, proctab); 2154 } 2155 2156 static void 2157 mmu_radix_proctab_init(void) 2158 { 2159 2160 isa3_base_pid = 1; 2161 2162 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa); 2163 isa3_proctab->proctab0 = 2164 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | 2165 RADIX_PGD_INDEX_SHIFT); 2166 2167 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12); 2168 2169 __asm __volatile("ptesync" : : : "memory"); 2170 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2171 "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); 2172 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory"); 2173 if (bootverbose) 2174 printf("process table %p and kernel radix PDE: %p\n", 2175 isa3_proctab, kernel_pmap->pm_pml1); 2176 mtmsr(mfmsr() | PSL_DR ); 2177 mtmsr(mfmsr() & ~PSL_DR); 2178 kernel_pmap->pm_pid = isa3_base_pid; 2179 isa3_base_pid++; 2180 } 2181 2182 void 2183 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2184 int advice) 2185 { 2186 struct rwlock *lock; 2187 pml1_entry_t *l1e; 2188 pml2_entry_t *l2e; 2189 pml3_entry_t oldl3e, *l3e; 2190 pt_entry_t *pte; 2191 vm_offset_t va, va_next; 2192 vm_page_t m; 2193 boolean_t anychanged; 2194 2195 if (advice != MADV_DONTNEED && advice != MADV_FREE) 2196 return; 2197 anychanged = FALSE; 2198 PMAP_LOCK(pmap); 2199 for (; sva < eva; sva = va_next) { 2200 l1e = pmap_pml1e(pmap, sva); 2201 if ((*l1e & PG_V) == 0) { 2202 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2203 if (va_next < sva) 2204 va_next = eva; 2205 continue; 2206 } 2207 l2e = pmap_l1e_to_l2e(l1e, sva); 2208 if ((*l2e & PG_V) == 0) { 2209 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2210 if (va_next < sva) 2211 va_next = eva; 2212 continue; 2213 } 2214 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2215 if (va_next < sva) 2216 va_next = eva; 2217 l3e = pmap_l2e_to_l3e(l2e, sva); 2218 oldl3e = *l3e; 2219 if ((oldl3e & PG_V) == 0) 2220 continue; 2221 else if ((oldl3e & RPTE_LEAF) != 0) { 2222 if ((oldl3e & PG_MANAGED) == 0) 2223 continue; 2224 lock = NULL; 2225 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) { 2226 if (lock != NULL) 2227 rw_wunlock(lock); 2228 2229 /* 2230 * The large page mapping was destroyed. 2231 */ 2232 continue; 2233 } 2234 2235 /* 2236 * Unless the page mappings are wired, remove the 2237 * mapping to a single page so that a subsequent 2238 * access may repromote. Since the underlying page 2239 * table page is fully populated, this removal never 2240 * frees a page table page. 2241 */ 2242 if ((oldl3e & PG_W) == 0) { 2243 pte = pmap_l3e_to_pte(l3e, sva); 2244 KASSERT((*pte & PG_V) != 0, 2245 ("pmap_advise: invalid PTE")); 2246 pmap_remove_pte(pmap, pte, sva, *l3e, NULL, 2247 &lock); 2248 anychanged = TRUE; 2249 } 2250 if (lock != NULL) 2251 rw_wunlock(lock); 2252 } 2253 if (va_next > eva) 2254 va_next = eva; 2255 va = va_next; 2256 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; 2257 pte++, sva += PAGE_SIZE) { 2258 MPASS(pte == pmap_pte(pmap, sva)); 2259 2260 if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) 2261 goto maybe_invlrng; 2262 else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2263 if (advice == MADV_DONTNEED) { 2264 /* 2265 * Future calls to pmap_is_modified() 2266 * can be avoided by making the page 2267 * dirty now. 2268 */ 2269 m = PHYS_TO_VM_PAGE(*pte & PG_FRAME); 2270 vm_page_dirty(m); 2271 } 2272 atomic_clear_long(pte, PG_M | PG_A); 2273 } else if ((*pte & PG_A) != 0) 2274 atomic_clear_long(pte, PG_A); 2275 else 2276 goto maybe_invlrng; 2277 anychanged = TRUE; 2278 continue; 2279 maybe_invlrng: 2280 if (va != va_next) { 2281 anychanged = true; 2282 va = va_next; 2283 } 2284 } 2285 if (va != va_next) 2286 anychanged = true; 2287 } 2288 if (anychanged) 2289 pmap_invalidate_all(pmap); 2290 PMAP_UNLOCK(pmap); 2291 } 2292 2293 /* 2294 * Routines used in machine-dependent code 2295 */ 2296 static void 2297 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end) 2298 { 2299 uint64_t lpcr; 2300 2301 if (bootverbose) 2302 printf("%s\n", __func__); 2303 hw_direct_map = 1; 2304 mmu_radix_early_bootstrap(start, end); 2305 if (bootverbose) 2306 printf("early bootstrap complete\n"); 2307 if (powernv_enabled) { 2308 lpcr = mfspr(SPR_LPCR); 2309 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2310 mmu_radix_parttab_init(); 2311 mmu_radix_init_amor(); 2312 if (bootverbose) 2313 printf("powernv init complete\n"); 2314 } 2315 mmu_radix_init_iamr(); 2316 mmu_radix_proctab_init(); 2317 mmu_radix_pid_set(kernel_pmap); 2318 /* XXX assume CPU_FTR_HVMODE */ 2319 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2320 2321 mmu_radix_late_bootstrap(start, end); 2322 numa_mem_regions(&numa_pregions, &numa_pregions_sz); 2323 if (bootverbose) 2324 printf("%s done\n", __func__); 2325 pmap_bootstrapped = 1; 2326 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE); 2327 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS); 2328 } 2329 2330 static void 2331 mmu_radix_cpu_bootstrap(int ap) 2332 { 2333 uint64_t lpcr; 2334 uint64_t ptcr; 2335 2336 if (powernv_enabled) { 2337 lpcr = mfspr(SPR_LPCR); 2338 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2339 2340 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2341 mtspr(SPR_PTCR, ptcr); 2342 mmu_radix_init_amor(); 2343 } 2344 mmu_radix_init_iamr(); 2345 mmu_radix_pid_set(kernel_pmap); 2346 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2347 } 2348 2349 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0, 2350 "2MB page mapping counters"); 2351 2352 static u_long pmap_l3e_demotions; 2353 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD, 2354 &pmap_l3e_demotions, 0, "2MB page demotions"); 2355 2356 static u_long pmap_l3e_mappings; 2357 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD, 2358 &pmap_l3e_mappings, 0, "2MB page mappings"); 2359 2360 static u_long pmap_l3e_p_failures; 2361 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD, 2362 &pmap_l3e_p_failures, 0, "2MB page promotion failures"); 2363 2364 static u_long pmap_l3e_promotions; 2365 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD, 2366 &pmap_l3e_promotions, 0, "2MB page promotions"); 2367 2368 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0, 2369 "1GB page mapping counters"); 2370 2371 static u_long pmap_l2e_demotions; 2372 SYSCTL_ULONG(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD, 2373 &pmap_l2e_demotions, 0, "1GB page demotions"); 2374 2375 void 2376 mmu_radix_clear_modify(vm_page_t m) 2377 { 2378 struct md_page *pvh; 2379 pmap_t pmap; 2380 pv_entry_t next_pv, pv; 2381 pml3_entry_t oldl3e, *l3e; 2382 pt_entry_t oldpte, *pte; 2383 struct rwlock *lock; 2384 vm_offset_t va; 2385 int md_gen, pvh_gen; 2386 2387 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2388 ("pmap_clear_modify: page %p is not managed", m)); 2389 vm_page_assert_busied(m); 2390 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 2391 2392 /* 2393 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. 2394 * If the object containing the page is locked and the page is not 2395 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 2396 */ 2397 if ((m->a.flags & PGA_WRITEABLE) == 0) 2398 return; 2399 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 2400 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 2401 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2402 rw_wlock(lock); 2403 restart: 2404 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 2405 pmap = PV_PMAP(pv); 2406 if (!PMAP_TRYLOCK(pmap)) { 2407 pvh_gen = pvh->pv_gen; 2408 rw_wunlock(lock); 2409 PMAP_LOCK(pmap); 2410 rw_wlock(lock); 2411 if (pvh_gen != pvh->pv_gen) { 2412 PMAP_UNLOCK(pmap); 2413 goto restart; 2414 } 2415 } 2416 va = pv->pv_va; 2417 l3e = pmap_pml3e(pmap, va); 2418 oldl3e = *l3e; 2419 if ((oldl3e & PG_RW) != 0) { 2420 if (pmap_demote_l3e_locked(pmap, l3e, va, &lock)) { 2421 if ((oldl3e & PG_W) == 0) { 2422 /* 2423 * Write protect the mapping to a 2424 * single page so that a subsequent 2425 * write access may repromote. 2426 */ 2427 va += VM_PAGE_TO_PHYS(m) - (oldl3e & 2428 PG_PS_FRAME); 2429 pte = pmap_l3e_to_pte(l3e, va); 2430 oldpte = *pte; 2431 if ((oldpte & PG_V) != 0) { 2432 while (!atomic_cmpset_long(pte, 2433 oldpte, 2434 (oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW))) 2435 oldpte = *pte; 2436 vm_page_dirty(m); 2437 pmap_invalidate_page(pmap, va); 2438 } 2439 } 2440 } 2441 } 2442 PMAP_UNLOCK(pmap); 2443 } 2444 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2445 pmap = PV_PMAP(pv); 2446 if (!PMAP_TRYLOCK(pmap)) { 2447 md_gen = m->md.pv_gen; 2448 pvh_gen = pvh->pv_gen; 2449 rw_wunlock(lock); 2450 PMAP_LOCK(pmap); 2451 rw_wlock(lock); 2452 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 2453 PMAP_UNLOCK(pmap); 2454 goto restart; 2455 } 2456 } 2457 l3e = pmap_pml3e(pmap, pv->pv_va); 2458 KASSERT((*l3e & RPTE_LEAF) == 0, ("pmap_clear_modify: found" 2459 " a 2mpage in page %p's pv list", m)); 2460 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 2461 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2462 atomic_clear_long(pte, PG_M); 2463 pmap_invalidate_page(pmap, pv->pv_va); 2464 } 2465 PMAP_UNLOCK(pmap); 2466 } 2467 rw_wunlock(lock); 2468 } 2469 2470 void 2471 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2472 vm_size_t len, vm_offset_t src_addr) 2473 { 2474 struct rwlock *lock; 2475 struct spglist free; 2476 vm_offset_t addr; 2477 vm_offset_t end_addr = src_addr + len; 2478 vm_offset_t va_next; 2479 vm_page_t dst_pdpg, dstmpte, srcmpte; 2480 bool invalidate_all; 2481 2482 CTR6(KTR_PMAP, 2483 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n", 2484 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr); 2485 2486 if (dst_addr != src_addr) 2487 return; 2488 lock = NULL; 2489 invalidate_all = false; 2490 if (dst_pmap < src_pmap) { 2491 PMAP_LOCK(dst_pmap); 2492 PMAP_LOCK(src_pmap); 2493 } else { 2494 PMAP_LOCK(src_pmap); 2495 PMAP_LOCK(dst_pmap); 2496 } 2497 2498 for (addr = src_addr; addr < end_addr; addr = va_next) { 2499 pml1_entry_t *l1e; 2500 pml2_entry_t *l2e; 2501 pml3_entry_t srcptepaddr, *l3e; 2502 pt_entry_t *src_pte, *dst_pte; 2503 2504 l1e = pmap_pml1e(src_pmap, addr); 2505 if ((*l1e & PG_V) == 0) { 2506 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2507 if (va_next < addr) 2508 va_next = end_addr; 2509 continue; 2510 } 2511 2512 l2e = pmap_l1e_to_l2e(l1e, addr); 2513 if ((*l2e & PG_V) == 0) { 2514 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2515 if (va_next < addr) 2516 va_next = end_addr; 2517 continue; 2518 } 2519 2520 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2521 if (va_next < addr) 2522 va_next = end_addr; 2523 2524 l3e = pmap_l2e_to_l3e(l2e, addr); 2525 srcptepaddr = *l3e; 2526 if (srcptepaddr == 0) 2527 continue; 2528 2529 if (srcptepaddr & RPTE_LEAF) { 2530 if ((addr & L3_PAGE_MASK) != 0 || 2531 addr + L3_PAGE_SIZE > end_addr) 2532 continue; 2533 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL); 2534 if (dst_pdpg == NULL) 2535 break; 2536 l3e = (pml3_entry_t *) 2537 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg)); 2538 l3e = &l3e[pmap_pml3e_index(addr)]; 2539 if (*l3e == 0 && ((srcptepaddr & PG_MANAGED) == 0 || 2540 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr, 2541 PMAP_ENTER_NORECLAIM, &lock))) { 2542 *l3e = srcptepaddr & ~PG_W; 2543 pmap_resident_count_inc(dst_pmap, 2544 L3_PAGE_SIZE / PAGE_SIZE); 2545 atomic_add_long(&pmap_l3e_mappings, 1); 2546 } else 2547 dst_pdpg->ref_count--; 2548 continue; 2549 } 2550 2551 srcptepaddr &= PG_FRAME; 2552 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 2553 KASSERT(srcmpte->ref_count > 0, 2554 ("pmap_copy: source page table page is unused")); 2555 2556 if (va_next > end_addr) 2557 va_next = end_addr; 2558 2559 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); 2560 src_pte = &src_pte[pmap_pte_index(addr)]; 2561 dstmpte = NULL; 2562 while (addr < va_next) { 2563 pt_entry_t ptetemp; 2564 ptetemp = *src_pte; 2565 /* 2566 * we only virtual copy managed pages 2567 */ 2568 if ((ptetemp & PG_MANAGED) != 0) { 2569 if (dstmpte != NULL && 2570 dstmpte->pindex == pmap_l3e_pindex(addr)) 2571 dstmpte->ref_count++; 2572 else if ((dstmpte = pmap_allocpte(dst_pmap, 2573 addr, NULL)) == NULL) 2574 goto out; 2575 dst_pte = (pt_entry_t *) 2576 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); 2577 dst_pte = &dst_pte[pmap_pte_index(addr)]; 2578 if (*dst_pte == 0 && 2579 pmap_try_insert_pv_entry(dst_pmap, addr, 2580 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), 2581 &lock)) { 2582 /* 2583 * Clear the wired, modified, and 2584 * accessed (referenced) bits 2585 * during the copy. 2586 */ 2587 *dst_pte = ptetemp & ~(PG_W | PG_M | 2588 PG_A); 2589 pmap_resident_count_inc(dst_pmap, 1); 2590 } else { 2591 SLIST_INIT(&free); 2592 if (pmap_unwire_ptp(dst_pmap, addr, 2593 dstmpte, &free)) { 2594 /* 2595 * Although "addr" is not 2596 * mapped, paging-structure 2597 * caches could nonetheless 2598 * have entries that refer to 2599 * the freed page table pages. 2600 * Invalidate those entries. 2601 */ 2602 invalidate_all = true; 2603 vm_page_free_pages_toq(&free, 2604 true); 2605 } 2606 goto out; 2607 } 2608 if (dstmpte->ref_count >= srcmpte->ref_count) 2609 break; 2610 } 2611 addr += PAGE_SIZE; 2612 if (__predict_false((addr & L3_PAGE_MASK) == 0)) 2613 src_pte = pmap_pte(src_pmap, addr); 2614 else 2615 src_pte++; 2616 } 2617 } 2618 out: 2619 if (invalidate_all) 2620 pmap_invalidate_all(dst_pmap); 2621 if (lock != NULL) 2622 rw_wunlock(lock); 2623 PMAP_UNLOCK(src_pmap); 2624 PMAP_UNLOCK(dst_pmap); 2625 } 2626 2627 static void 2628 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst) 2629 { 2630 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); 2631 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); 2632 2633 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst); 2634 /* 2635 * XXX slow 2636 */ 2637 bcopy((void *)src, (void *)dst, PAGE_SIZE); 2638 } 2639 2640 static void 2641 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2642 vm_offset_t b_offset, int xfersize) 2643 { 2644 2645 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma, 2646 a_offset, mb, b_offset, xfersize); 2647 UNIMPLEMENTED(); 2648 } 2649 2650 #if VM_NRESERVLEVEL > 0 2651 /* 2652 * Tries to promote the 512, contiguous 4KB page mappings that are within a 2653 * single page table page (PTP) to a single 2MB page mapping. For promotion 2654 * to occur, two conditions must be met: (1) the 4KB page mappings must map 2655 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 2656 * identical characteristics. 2657 */ 2658 static int 2659 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va, 2660 struct rwlock **lockp) 2661 { 2662 pml3_entry_t newpde; 2663 pt_entry_t *firstpte, oldpte, pa, *pte; 2664 vm_page_t mpte; 2665 2666 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2667 2668 /* 2669 * Examine the first PTE in the specified PTP. Abort if this PTE is 2670 * either invalid, unused, or does not map the first 4KB physical page 2671 * within a 2MB page. 2672 */ 2673 firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME); 2674 setpde: 2675 newpde = *firstpte; 2676 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) { 2677 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2678 " in pmap %p", va, pmap); 2679 goto fail; 2680 } 2681 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 2682 /* 2683 * When PG_M is already clear, PG_RW can be cleared without 2684 * a TLB invalidation. 2685 */ 2686 if (!atomic_cmpset_long(firstpte, newpde, (newpde | RPTE_EAA_R) & ~RPTE_EAA_W)) 2687 goto setpde; 2688 newpde &= ~RPTE_EAA_W; 2689 } 2690 2691 /* 2692 * Examine each of the other PTEs in the specified PTP. Abort if this 2693 * PTE maps an unexpected 4KB physical page or does not have identical 2694 * characteristics to the first PTE. 2695 */ 2696 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE; 2697 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { 2698 setpte: 2699 oldpte = *pte; 2700 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { 2701 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2702 " in pmap %p", va, pmap); 2703 goto fail; 2704 } 2705 if ((oldpte & (PG_M | PG_RW)) == PG_RW) { 2706 /* 2707 * When PG_M is already clear, PG_RW can be cleared 2708 * without a TLB invalidation. 2709 */ 2710 if (!atomic_cmpset_long(pte, oldpte, (oldpte | RPTE_EAA_R) & ~RPTE_EAA_W)) 2711 goto setpte; 2712 oldpte &= ~RPTE_EAA_W; 2713 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx" 2714 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) | 2715 (va & ~L3_PAGE_MASK), pmap); 2716 } 2717 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { 2718 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2719 " in pmap %p", va, pmap); 2720 goto fail; 2721 } 2722 pa -= PAGE_SIZE; 2723 } 2724 2725 /* 2726 * Save the page table page in its current state until the PDE 2727 * mapping the superpage is demoted by pmap_demote_pde() or 2728 * destroyed by pmap_remove_pde(). 2729 */ 2730 mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); 2731 KASSERT(mpte >= vm_page_array && 2732 mpte < &vm_page_array[vm_page_array_size], 2733 ("pmap_promote_l3e: page table page is out of range")); 2734 KASSERT(mpte->pindex == pmap_l3e_pindex(va), 2735 ("pmap_promote_l3e: page table page's pindex is wrong")); 2736 if (pmap_insert_pt_page(pmap, mpte)) { 2737 CTR2(KTR_PMAP, 2738 "pmap_promote_l3e: failure for va %#lx in pmap %p", va, 2739 pmap); 2740 goto fail; 2741 } 2742 2743 /* 2744 * Promote the pv entries. 2745 */ 2746 if ((newpde & PG_MANAGED) != 0) 2747 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp); 2748 2749 pte_store(pde, PG_PROMOTED | newpde); 2750 atomic_add_long(&pmap_l3e_promotions, 1); 2751 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx" 2752 " in pmap %p", va, pmap); 2753 return (0); 2754 fail: 2755 atomic_add_long(&pmap_l3e_p_failures, 1); 2756 return (KERN_FAILURE); 2757 } 2758 #endif /* VM_NRESERVLEVEL > 0 */ 2759 2760 int 2761 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, 2762 vm_prot_t prot, u_int flags, int8_t psind) 2763 { 2764 struct rwlock *lock; 2765 pml3_entry_t *l3e; 2766 pt_entry_t *pte; 2767 pt_entry_t newpte, origpte; 2768 pv_entry_t pv; 2769 vm_paddr_t opa, pa; 2770 vm_page_t mpte, om; 2771 int rv, retrycount; 2772 boolean_t nosleep, invalidate_all, invalidate_page; 2773 2774 va = trunc_page(va); 2775 retrycount = 0; 2776 invalidate_page = invalidate_all = false; 2777 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va, 2778 m, prot, flags, psind); 2779 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 2780 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || va < kmi.clean_sva || 2781 va >= kmi.clean_eva, 2782 ("pmap_enter: managed mapping within the clean submap")); 2783 if ((m->oflags & VPO_UNMANAGED) == 0) 2784 VM_PAGE_OBJECT_BUSY_ASSERT(m); 2785 2786 KASSERT((flags & PMAP_ENTER_RESERVED) == 0, 2787 ("pmap_enter: flags %u has reserved bits set", flags)); 2788 pa = VM_PAGE_TO_PHYS(m); 2789 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF); 2790 if ((flags & VM_PROT_WRITE) != 0) 2791 newpte |= PG_M; 2792 if ((flags & VM_PROT_READ) != 0) 2793 newpte |= PG_A; 2794 if (prot & VM_PROT_READ) 2795 newpte |= RPTE_EAA_R; 2796 if ((prot & VM_PROT_WRITE) != 0) 2797 newpte |= RPTE_EAA_W; 2798 KASSERT((newpte & (PG_M | PG_RW)) != PG_M, 2799 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); 2800 2801 if (prot & VM_PROT_EXECUTE) 2802 newpte |= PG_X; 2803 if ((flags & PMAP_ENTER_WIRED) != 0) 2804 newpte |= PG_W; 2805 if (va >= DMAP_MIN_ADDRESS) 2806 newpte |= RPTE_EAA_P; 2807 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs); 2808 /* 2809 * Set modified bit gratuitously for writeable mappings if 2810 * the page is unmanaged. We do not want to take a fault 2811 * to do the dirty bit accounting for these mappings. 2812 */ 2813 if ((m->oflags & VPO_UNMANAGED) != 0) { 2814 if ((newpte & PG_RW) != 0) 2815 newpte |= PG_M; 2816 } else 2817 newpte |= PG_MANAGED; 2818 2819 lock = NULL; 2820 PMAP_LOCK(pmap); 2821 if (psind == 1) { 2822 /* Assert the required virtual and physical alignment. */ 2823 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned")); 2824 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 2825 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock); 2826 goto out; 2827 } 2828 mpte = NULL; 2829 2830 /* 2831 * In the case that a page table page is not 2832 * resident, we are creating it here. 2833 */ 2834 retry: 2835 l3e = pmap_pml3e(pmap, va); 2836 if (l3e != NULL && (*l3e & PG_V) != 0 && ((*l3e & RPTE_LEAF) == 0 || 2837 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) { 2838 pte = pmap_l3e_to_pte(l3e, va); 2839 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { 2840 mpte = PHYS_TO_VM_PAGE(*l3e & PG_FRAME); 2841 mpte->ref_count++; 2842 } 2843 } else if (va < VM_MAXUSER_ADDRESS) { 2844 /* 2845 * Here if the pte page isn't mapped, or if it has been 2846 * deallocated. 2847 */ 2848 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 2849 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va), 2850 nosleep ? NULL : &lock); 2851 if (mpte == NULL && nosleep) { 2852 rv = KERN_RESOURCE_SHORTAGE; 2853 goto out; 2854 } 2855 if (__predict_false(retrycount++ == 6)) 2856 panic("too many retries"); 2857 invalidate_all = true; 2858 goto retry; 2859 } else 2860 panic("pmap_enter: invalid page directory va=%#lx", va); 2861 2862 origpte = *pte; 2863 pv = NULL; 2864 2865 /* 2866 * Is the specified virtual address already mapped? 2867 */ 2868 if ((origpte & PG_V) != 0) { 2869 #ifdef INVARIANTS 2870 if (VERBOSE_PMAP || pmap_logging) { 2871 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --" 2872 " asid=%lu curpid=%d name=%s origpte0x%lx\n", 2873 pmap, va, m, prot, flags, psind, pmap->pm_pid, 2874 curproc->p_pid, curproc->p_comm, origpte); 2875 pmap_pte_walk(pmap->pm_pml1, va); 2876 } 2877 #endif 2878 /* 2879 * Wiring change, just update stats. We don't worry about 2880 * wiring PT pages as they remain resident as long as there 2881 * are valid mappings in them. Hence, if a user page is wired, 2882 * the PT page will be also. 2883 */ 2884 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) 2885 pmap->pm_stats.wired_count++; 2886 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) 2887 pmap->pm_stats.wired_count--; 2888 2889 /* 2890 * Remove the extra PT page reference. 2891 */ 2892 if (mpte != NULL) { 2893 mpte->ref_count--; 2894 KASSERT(mpte->ref_count > 0, 2895 ("pmap_enter: missing reference to page table page," 2896 " va: 0x%lx", va)); 2897 } 2898 2899 /* 2900 * Has the physical page changed? 2901 */ 2902 opa = origpte & PG_FRAME; 2903 if (opa == pa) { 2904 /* 2905 * No, might be a protection or wiring change. 2906 */ 2907 if ((origpte & PG_MANAGED) != 0 && 2908 (newpte & PG_RW) != 0) 2909 vm_page_aflag_set(m, PGA_WRITEABLE); 2910 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) { 2911 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) { 2912 if (!atomic_cmpset_long(pte, origpte, newpte)) 2913 goto retry; 2914 if ((newpte & PG_M) != (origpte & PG_M)) 2915 vm_page_dirty(m); 2916 if ((newpte & PG_A) != (origpte & PG_A)) 2917 vm_page_aflag_set(m, PGA_REFERENCED); 2918 ptesync(); 2919 } else 2920 invalidate_all = true; 2921 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) 2922 goto unchanged; 2923 } 2924 goto validate; 2925 } 2926 2927 /* 2928 * The physical page has changed. Temporarily invalidate 2929 * the mapping. This ensures that all threads sharing the 2930 * pmap keep a consistent view of the mapping, which is 2931 * necessary for the correct handling of COW faults. It 2932 * also permits reuse of the old mapping's PV entry, 2933 * avoiding an allocation. 2934 * 2935 * For consistency, handle unmanaged mappings the same way. 2936 */ 2937 origpte = pte_load_clear(pte); 2938 KASSERT((origpte & PG_FRAME) == opa, 2939 ("pmap_enter: unexpected pa update for %#lx", va)); 2940 if ((origpte & PG_MANAGED) != 0) { 2941 om = PHYS_TO_VM_PAGE(opa); 2942 2943 /* 2944 * The pmap lock is sufficient to synchronize with 2945 * concurrent calls to pmap_page_test_mappings() and 2946 * pmap_ts_referenced(). 2947 */ 2948 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 2949 vm_page_dirty(om); 2950 if ((origpte & PG_A) != 0) 2951 vm_page_aflag_set(om, PGA_REFERENCED); 2952 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 2953 pv = pmap_pvh_remove(&om->md, pmap, va); 2954 if ((newpte & PG_MANAGED) == 0) 2955 free_pv_entry(pmap, pv); 2956 #ifdef INVARIANTS 2957 else if (origpte & PG_MANAGED) { 2958 if (pv == NULL) { 2959 pmap_page_print_mappings(om); 2960 MPASS(pv != NULL); 2961 } 2962 } 2963 #endif 2964 if ((om->a.flags & PGA_WRITEABLE) != 0 && 2965 TAILQ_EMPTY(&om->md.pv_list) && 2966 ((om->flags & PG_FICTITIOUS) != 0 || 2967 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) 2968 vm_page_aflag_clear(om, PGA_WRITEABLE); 2969 } 2970 if ((origpte & PG_A) != 0) 2971 invalidate_page = true; 2972 origpte = 0; 2973 } else { 2974 if (pmap != kernel_pmap) { 2975 #ifdef INVARIANTS 2976 if (VERBOSE_PMAP || pmap_logging) 2977 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n", 2978 pmap, va, m, prot, flags, psind, 2979 pmap->pm_pid, curproc->p_pid, 2980 curproc->p_comm); 2981 #endif 2982 } 2983 2984 /* 2985 * Increment the counters. 2986 */ 2987 if ((newpte & PG_W) != 0) 2988 pmap->pm_stats.wired_count++; 2989 pmap_resident_count_inc(pmap, 1); 2990 } 2991 2992 /* 2993 * Enter on the PV list if part of our managed memory. 2994 */ 2995 if ((newpte & PG_MANAGED) != 0) { 2996 if (pv == NULL) { 2997 pv = get_pv_entry(pmap, &lock); 2998 pv->pv_va = va; 2999 } 3000 #ifdef VERBOSE_PV 3001 else 3002 printf("reassigning pv: %p to pmap: %p\n", 3003 pv, pmap); 3004 #endif 3005 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 3006 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3007 m->md.pv_gen++; 3008 if ((newpte & PG_RW) != 0) 3009 vm_page_aflag_set(m, PGA_WRITEABLE); 3010 } 3011 3012 /* 3013 * Update the PTE. 3014 */ 3015 if ((origpte & PG_V) != 0) { 3016 validate: 3017 origpte = pte_load_store(pte, newpte); 3018 KASSERT((origpte & PG_FRAME) == pa, 3019 ("pmap_enter: unexpected pa update for %#lx", va)); 3020 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == 3021 (PG_M | PG_RW)) { 3022 if ((origpte & PG_MANAGED) != 0) 3023 vm_page_dirty(m); 3024 invalidate_page = true; 3025 3026 /* 3027 * Although the PTE may still have PG_RW set, TLB 3028 * invalidation may nonetheless be required because 3029 * the PTE no longer has PG_M set. 3030 */ 3031 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) { 3032 /* 3033 * Removing capabilities requires invalidation on POWER 3034 */ 3035 invalidate_page = true; 3036 goto unchanged; 3037 } 3038 if ((origpte & PG_A) != 0) 3039 invalidate_page = true; 3040 } else { 3041 pte_store(pte, newpte); 3042 ptesync(); 3043 } 3044 unchanged: 3045 3046 #if VM_NRESERVLEVEL > 0 3047 /* 3048 * If both the page table page and the reservation are fully 3049 * populated, then attempt promotion. 3050 */ 3051 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 3052 mmu_radix_ps_enabled(pmap) && 3053 (m->flags & PG_FICTITIOUS) == 0 && 3054 vm_reserv_level_iffullpop(m) == 0 && 3055 pmap_promote_l3e(pmap, l3e, va, &lock) == 0) 3056 invalidate_all = true; 3057 #endif 3058 if (invalidate_all) 3059 pmap_invalidate_all(pmap); 3060 else if (invalidate_page) 3061 pmap_invalidate_page(pmap, va); 3062 3063 rv = KERN_SUCCESS; 3064 out: 3065 if (lock != NULL) 3066 rw_wunlock(lock); 3067 PMAP_UNLOCK(pmap); 3068 3069 return (rv); 3070 } 3071 3072 3073 /* 3074 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true 3075 * if successful. Returns false if (1) a page table page cannot be allocated 3076 * without sleeping, (2) a mapping already exists at the specified virtual 3077 * address, or (3) a PV entry cannot be allocated without reclaiming another 3078 * PV entry. 3079 */ 3080 static bool 3081 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3082 struct rwlock **lockp) 3083 { 3084 pml3_entry_t newpde; 3085 3086 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3087 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) | 3088 RPTE_LEAF | PG_V; 3089 if ((m->oflags & VPO_UNMANAGED) == 0) 3090 newpde |= PG_MANAGED; 3091 if (prot & VM_PROT_EXECUTE) 3092 newpde |= PG_X; 3093 if (prot & VM_PROT_READ) 3094 newpde |= RPTE_EAA_R; 3095 if (va >= DMAP_MIN_ADDRESS) 3096 newpde |= RPTE_EAA_P; 3097 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP | 3098 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == 3099 KERN_SUCCESS); 3100 } 3101 3102 /* 3103 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if 3104 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE 3105 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and 3106 * a mapping already exists at the specified virtual address. Returns 3107 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table 3108 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if 3109 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. 3110 * 3111 * The parameter "m" is only used when creating a managed, writeable mapping. 3112 */ 3113 static int 3114 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags, 3115 vm_page_t m, struct rwlock **lockp) 3116 { 3117 struct spglist free; 3118 pml3_entry_t oldl3e, *l3e; 3119 vm_page_t mt, pdpg; 3120 3121 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW, 3122 ("pmap_enter_pde: newpde is missing PG_M")); 3123 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3124 3125 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 3126 NULL : lockp)) == NULL) { 3127 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3128 " in pmap %p", va, pmap); 3129 return (KERN_RESOURCE_SHORTAGE); 3130 } 3131 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 3132 l3e = &l3e[pmap_pml3e_index(va)]; 3133 oldl3e = *l3e; 3134 if ((oldl3e & PG_V) != 0) { 3135 KASSERT(pdpg->ref_count > 1, 3136 ("pmap_enter_pde: pdpg's wire count is too low")); 3137 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 3138 pdpg->ref_count--; 3139 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3140 " in pmap %p", va, pmap); 3141 return (KERN_FAILURE); 3142 } 3143 /* Break the existing mapping(s). */ 3144 SLIST_INIT(&free); 3145 if ((oldl3e & RPTE_LEAF) != 0) { 3146 /* 3147 * The reference to the PD page that was acquired by 3148 * pmap_allocl3e() ensures that it won't be freed. 3149 * However, if the PDE resulted from a promotion, then 3150 * a reserved PT page could be freed. 3151 */ 3152 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp); 3153 } else { 3154 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e, 3155 &free, lockp)) 3156 pmap_invalidate_all(pmap); 3157 } 3158 vm_page_free_pages_toq(&free, true); 3159 if (va >= VM_MAXUSER_ADDRESS) { 3160 mt = PHYS_TO_VM_PAGE(*l3e & PG_FRAME); 3161 if (pmap_insert_pt_page(pmap, mt)) { 3162 /* 3163 * XXX Currently, this can't happen because 3164 * we do not perform pmap_enter(psind == 1) 3165 * on the kernel pmap. 3166 */ 3167 panic("pmap_enter_pde: trie insert failed"); 3168 } 3169 } else 3170 KASSERT(*l3e == 0, ("pmap_enter_pde: non-zero pde %p", 3171 l3e)); 3172 } 3173 if ((newpde & PG_MANAGED) != 0) { 3174 /* 3175 * Abort this mapping if its PV entry could not be created. 3176 */ 3177 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) { 3178 SLIST_INIT(&free); 3179 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { 3180 /* 3181 * Although "va" is not mapped, paging- 3182 * structure caches could nonetheless have 3183 * entries that refer to the freed page table 3184 * pages. Invalidate those entries. 3185 */ 3186 pmap_invalidate_page(pmap, va); 3187 vm_page_free_pages_toq(&free, true); 3188 } 3189 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3190 " in pmap %p", va, pmap); 3191 return (KERN_RESOURCE_SHORTAGE); 3192 } 3193 if ((newpde & PG_RW) != 0) { 3194 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 3195 vm_page_aflag_set(mt, PGA_WRITEABLE); 3196 } 3197 } 3198 3199 /* 3200 * Increment counters. 3201 */ 3202 if ((newpde & PG_W) != 0) 3203 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE; 3204 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 3205 3206 /* 3207 * Map the superpage. (This is not a promoted mapping; there will not 3208 * be any lingering 4KB page mappings in the TLB.) 3209 */ 3210 pte_store(l3e, newpde); 3211 3212 atomic_add_long(&pmap_l3e_mappings, 1); 3213 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" 3214 " in pmap %p", va, pmap); 3215 return (KERN_SUCCESS); 3216 } 3217 3218 void 3219 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start, 3220 vm_offset_t end, vm_page_t m_start, vm_prot_t prot) 3221 { 3222 3223 struct rwlock *lock; 3224 vm_offset_t va; 3225 vm_page_t m, mpte; 3226 vm_pindex_t diff, psize; 3227 bool invalidate; 3228 VM_OBJECT_ASSERT_LOCKED(m_start->object); 3229 3230 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start, 3231 end, m_start, prot); 3232 3233 invalidate = false; 3234 psize = atop(end - start); 3235 mpte = NULL; 3236 m = m_start; 3237 lock = NULL; 3238 PMAP_LOCK(pmap); 3239 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 3240 va = start + ptoa(diff); 3241 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end && 3242 m->psind == 1 && mmu_radix_ps_enabled(pmap) && 3243 pmap_enter_2mpage(pmap, va, m, prot, &lock)) 3244 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1]; 3245 else 3246 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot, 3247 mpte, &lock, &invalidate); 3248 m = TAILQ_NEXT(m, listq); 3249 } 3250 ptesync(); 3251 if (lock != NULL) 3252 rw_wunlock(lock); 3253 if (invalidate) 3254 pmap_invalidate_all(pmap); 3255 PMAP_UNLOCK(pmap); 3256 } 3257 3258 static vm_page_t 3259 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 3260 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate) 3261 { 3262 struct spglist free; 3263 pt_entry_t *pte; 3264 vm_paddr_t pa; 3265 3266 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 3267 (m->oflags & VPO_UNMANAGED) != 0, 3268 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap")); 3269 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3270 3271 /* 3272 * In the case that a page table page is not 3273 * resident, we are creating it here. 3274 */ 3275 if (va < VM_MAXUSER_ADDRESS) { 3276 vm_pindex_t ptepindex; 3277 pml3_entry_t *ptepa; 3278 3279 /* 3280 * Calculate pagetable page index 3281 */ 3282 ptepindex = pmap_l3e_pindex(va); 3283 if (mpte && (mpte->pindex == ptepindex)) { 3284 mpte->ref_count++; 3285 } else { 3286 /* 3287 * Get the page directory entry 3288 */ 3289 ptepa = pmap_pml3e(pmap, va); 3290 3291 /* 3292 * If the page table page is mapped, we just increment 3293 * the hold count, and activate it. Otherwise, we 3294 * attempt to allocate a page table page. If this 3295 * attempt fails, we don't retry. Instead, we give up. 3296 */ 3297 if (ptepa && (*ptepa & PG_V) != 0) { 3298 if (*ptepa & RPTE_LEAF) 3299 return (NULL); 3300 mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME); 3301 mpte->ref_count++; 3302 } else { 3303 /* 3304 * Pass NULL instead of the PV list lock 3305 * pointer, because we don't intend to sleep. 3306 */ 3307 mpte = _pmap_allocpte(pmap, ptepindex, NULL); 3308 if (mpte == NULL) 3309 return (mpte); 3310 } 3311 } 3312 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); 3313 pte = &pte[pmap_pte_index(va)]; 3314 } else { 3315 mpte = NULL; 3316 pte = pmap_pte(pmap, va); 3317 } 3318 if (*pte) { 3319 if (mpte != NULL) { 3320 mpte->ref_count--; 3321 mpte = NULL; 3322 } 3323 return (mpte); 3324 } 3325 3326 /* 3327 * Enter on the PV list if part of our managed memory. 3328 */ 3329 if ((m->oflags & VPO_UNMANAGED) == 0 && 3330 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 3331 if (mpte != NULL) { 3332 SLIST_INIT(&free); 3333 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { 3334 /* 3335 * Although "va" is not mapped, paging- 3336 * structure caches could nonetheless have 3337 * entries that refer to the freed page table 3338 * pages. Invalidate those entries. 3339 */ 3340 *invalidate = true; 3341 vm_page_free_pages_toq(&free, true); 3342 } 3343 mpte = NULL; 3344 } 3345 return (mpte); 3346 } 3347 3348 /* 3349 * Increment counters 3350 */ 3351 pmap_resident_count_inc(pmap, 1); 3352 3353 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs); 3354 if (prot & VM_PROT_EXECUTE) 3355 pa |= PG_X; 3356 else 3357 pa |= RPTE_EAA_R; 3358 if ((m->oflags & VPO_UNMANAGED) == 0) 3359 pa |= PG_MANAGED; 3360 3361 pte_store(pte, pa); 3362 return (mpte); 3363 } 3364 3365 void 3366 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, 3367 vm_prot_t prot) 3368 { 3369 struct rwlock *lock; 3370 bool invalidate; 3371 3372 lock = NULL; 3373 invalidate = false; 3374 PMAP_LOCK(pmap); 3375 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock, 3376 &invalidate); 3377 ptesync(); 3378 if (lock != NULL) 3379 rw_wunlock(lock); 3380 if (invalidate) 3381 pmap_invalidate_all(pmap); 3382 PMAP_UNLOCK(pmap); 3383 } 3384 3385 vm_paddr_t 3386 mmu_radix_extract(pmap_t pmap, vm_offset_t va) 3387 { 3388 pml3_entry_t *l3e; 3389 pt_entry_t *pte; 3390 vm_paddr_t pa; 3391 3392 l3e = pmap_pml3e(pmap, va); 3393 if (__predict_false(l3e == NULL)) 3394 return (0); 3395 if (*l3e & RPTE_LEAF) { 3396 pa = (*l3e & PG_PS_FRAME) | (va & L3_PAGE_MASK); 3397 pa |= (va & L3_PAGE_MASK); 3398 } else { 3399 /* 3400 * Beware of a concurrent promotion that changes the 3401 * PDE at this point! For example, vtopte() must not 3402 * be used to access the PTE because it would use the 3403 * new PDE. It is, however, safe to use the old PDE 3404 * because the page table page is preserved by the 3405 * promotion. 3406 */ 3407 pte = pmap_l3e_to_pte(l3e, va); 3408 if (__predict_false(pte == NULL)) 3409 return (0); 3410 pa = *pte; 3411 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 3412 pa |= (va & PAGE_MASK); 3413 } 3414 return (pa); 3415 } 3416 3417 vm_page_t 3418 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3419 { 3420 pml3_entry_t l3e, *l3ep; 3421 pt_entry_t pte; 3422 vm_paddr_t pa; 3423 vm_page_t m; 3424 3425 pa = 0; 3426 m = NULL; 3427 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot); 3428 PMAP_LOCK(pmap); 3429 l3ep = pmap_pml3e(pmap, va); 3430 if (l3ep != NULL && (l3e = *l3ep)) { 3431 if (l3e & RPTE_LEAF) { 3432 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0) 3433 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) | 3434 (va & L3_PAGE_MASK)); 3435 } else { 3436 pte = *pmap_l3e_to_pte(l3ep, va); 3437 if ((pte & PG_V) && 3438 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) 3439 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 3440 } 3441 if (m != NULL && !vm_page_wire_mapped(m)) 3442 m = NULL; 3443 } 3444 PMAP_UNLOCK(pmap); 3445 return (m); 3446 } 3447 3448 static void 3449 mmu_radix_growkernel(vm_offset_t addr) 3450 { 3451 vm_paddr_t paddr; 3452 vm_page_t nkpg; 3453 pml3_entry_t *l3e; 3454 pml2_entry_t *l2e; 3455 3456 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 3457 if (VM_MIN_KERNEL_ADDRESS < addr && 3458 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE)) 3459 return; 3460 3461 addr = roundup2(addr, L3_PAGE_SIZE); 3462 if (addr - 1 >= vm_map_max(kernel_map)) 3463 addr = vm_map_max(kernel_map); 3464 while (kernel_vm_end < addr) { 3465 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); 3466 if ((*l2e & PG_V) == 0) { 3467 /* We need a new PDP entry */ 3468 nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_PAGE_SIZE_SHIFT, 3469 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | 3470 VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3471 if (nkpg == NULL) 3472 panic("pmap_growkernel: no memory to grow kernel"); 3473 if ((nkpg->flags & PG_ZERO) == 0) 3474 mmu_radix_zero_page(nkpg); 3475 paddr = VM_PAGE_TO_PHYS(nkpg); 3476 pde_store(l2e, paddr); 3477 continue; /* try again */ 3478 } 3479 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); 3480 if ((*l3e & PG_V) != 0) { 3481 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3482 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3483 kernel_vm_end = vm_map_max(kernel_map); 3484 break; 3485 } 3486 continue; 3487 } 3488 3489 nkpg = vm_page_alloc(NULL, pmap_l3e_pindex(kernel_vm_end), 3490 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 3491 VM_ALLOC_ZERO); 3492 if (nkpg == NULL) 3493 panic("pmap_growkernel: no memory to grow kernel"); 3494 if ((nkpg->flags & PG_ZERO) == 0) 3495 mmu_radix_zero_page(nkpg); 3496 paddr = VM_PAGE_TO_PHYS(nkpg); 3497 pde_store(l3e, paddr); 3498 3499 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3500 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3501 kernel_vm_end = vm_map_max(kernel_map); 3502 break; 3503 } 3504 } 3505 ptesync(); 3506 } 3507 3508 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory"); 3509 static uma_zone_t zone_radix_pgd; 3510 3511 static int 3512 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused, 3513 int flags) 3514 { 3515 3516 for (int i = 0; i < count; i++) { 3517 vm_page_t m = vm_page_alloc_contig(NULL, 0, 3518 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 3519 VM_ALLOC_ZERO | VM_ALLOC_WAITOK, RADIX_PGD_SIZE/PAGE_SIZE, 3520 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE, 3521 VM_MEMATTR_DEFAULT); 3522 /* XXX zero on alloc here so we don't have to later */ 3523 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 3524 } 3525 return (count); 3526 } 3527 3528 static void 3529 radix_pgd_release(void *arg __unused, void **store, int count) 3530 { 3531 vm_page_t m; 3532 struct spglist free; 3533 int page_count; 3534 3535 SLIST_INIT(&free); 3536 page_count = RADIX_PGD_SIZE/PAGE_SIZE; 3537 3538 for (int i = 0; i < count; i++) { 3539 /* 3540 * XXX selectively remove dmap and KVA entries so we don't 3541 * need to bzero 3542 */ 3543 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i])); 3544 for (int j = page_count-1; j >= 0; j--) { 3545 vm_page_unwire_noq(&m[j]); 3546 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss); 3547 } 3548 vm_page_free_pages_toq(&free, false); 3549 } 3550 } 3551 3552 static void 3553 mmu_radix_init() 3554 { 3555 vm_page_t mpte; 3556 vm_size_t s; 3557 int error, i, pv_npg; 3558 3559 /* L1TF, reserve page @0 unconditionally */ 3560 vm_page_blacklist_add(0, bootverbose); 3561 3562 zone_radix_pgd = uma_zcache_create("radix_pgd_cache", 3563 RADIX_PGD_SIZE, NULL, NULL, 3564 #ifdef INVARIANTS 3565 trash_init, trash_fini, 3566 #else 3567 NULL, NULL, 3568 #endif 3569 radix_pgd_import, radix_pgd_release, 3570 NULL, UMA_ZONE_NOBUCKET); 3571 3572 /* 3573 * Initialize the vm page array entries for the kernel pmap's 3574 * page table pages. 3575 */ 3576 PMAP_LOCK(kernel_pmap); 3577 for (i = 0; i < nkpt; i++) { 3578 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 3579 KASSERT(mpte >= vm_page_array && 3580 mpte < &vm_page_array[vm_page_array_size], 3581 ("pmap_init: page table page is out of range size: %lu", 3582 vm_page_array_size)); 3583 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i; 3584 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); 3585 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte); 3586 //pmap_insert_pt_page(kernel_pmap, mpte); 3587 mpte->ref_count = 1; 3588 } 3589 PMAP_UNLOCK(kernel_pmap); 3590 vm_wire_add(nkpt); 3591 3592 CTR1(KTR_PMAP, "%s()", __func__); 3593 TAILQ_INIT(&pv_dummy.pv_list); 3594 3595 /* 3596 * Are large page mappings enabled? 3597 */ 3598 TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled); 3599 if (pg_ps_enabled) { 3600 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 3601 ("pmap_init: can't assign to pagesizes[1]")); 3602 pagesizes[1] = L3_PAGE_SIZE; 3603 } 3604 3605 /* 3606 * Initialize the pv chunk list mutex. 3607 */ 3608 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); 3609 3610 /* 3611 * Initialize the pool of pv list locks. 3612 */ 3613 for (i = 0; i < NPV_LIST_LOCKS; i++) 3614 rw_init(&pv_list_locks[i], "pmap pv list"); 3615 3616 /* 3617 * Calculate the size of the pv head table for superpages. 3618 */ 3619 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE); 3620 3621 /* 3622 * Allocate memory for the pv head table for superpages. 3623 */ 3624 s = (vm_size_t)(pv_npg * sizeof(struct md_page)); 3625 s = round_page(s); 3626 pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO); 3627 for (i = 0; i < pv_npg; i++) 3628 TAILQ_INIT(&pv_table[i].pv_list); 3629 TAILQ_INIT(&pv_dummy.pv_list); 3630 3631 pmap_initialized = 1; 3632 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); 3633 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 3634 (vmem_addr_t *)&qframe); 3635 3636 if (error != 0) 3637 panic("qframe allocation failed"); 3638 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits), 3639 1, 1, M_WAITOK); 3640 } 3641 3642 static boolean_t 3643 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) 3644 { 3645 struct rwlock *lock; 3646 pv_entry_t pv; 3647 struct md_page *pvh; 3648 pt_entry_t *pte, mask; 3649 pmap_t pmap; 3650 int md_gen, pvh_gen; 3651 boolean_t rv; 3652 3653 rv = FALSE; 3654 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 3655 rw_rlock(lock); 3656 restart: 3657 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 3658 pmap = PV_PMAP(pv); 3659 if (!PMAP_TRYLOCK(pmap)) { 3660 md_gen = m->md.pv_gen; 3661 rw_runlock(lock); 3662 PMAP_LOCK(pmap); 3663 rw_rlock(lock); 3664 if (md_gen != m->md.pv_gen) { 3665 PMAP_UNLOCK(pmap); 3666 goto restart; 3667 } 3668 } 3669 pte = pmap_pte(pmap, pv->pv_va); 3670 mask = 0; 3671 if (modified) 3672 mask |= PG_RW | PG_M; 3673 if (accessed) 3674 mask |= PG_V | PG_A; 3675 rv = (*pte & mask) == mask; 3676 PMAP_UNLOCK(pmap); 3677 if (rv) 3678 goto out; 3679 } 3680 if ((m->flags & PG_FICTITIOUS) == 0) { 3681 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 3682 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 3683 pmap = PV_PMAP(pv); 3684 if (!PMAP_TRYLOCK(pmap)) { 3685 md_gen = m->md.pv_gen; 3686 pvh_gen = pvh->pv_gen; 3687 rw_runlock(lock); 3688 PMAP_LOCK(pmap); 3689 rw_rlock(lock); 3690 if (md_gen != m->md.pv_gen || 3691 pvh_gen != pvh->pv_gen) { 3692 PMAP_UNLOCK(pmap); 3693 goto restart; 3694 } 3695 } 3696 pte = pmap_pml3e(pmap, pv->pv_va); 3697 mask = 0; 3698 if (modified) 3699 mask |= PG_RW | PG_M; 3700 if (accessed) 3701 mask |= PG_V | PG_A; 3702 rv = (*pte & mask) == mask; 3703 PMAP_UNLOCK(pmap); 3704 if (rv) 3705 goto out; 3706 } 3707 } 3708 out: 3709 rw_runlock(lock); 3710 return (rv); 3711 } 3712 3713 /* 3714 * pmap_is_modified: 3715 * 3716 * Return whether or not the specified physical page was modified 3717 * in any physical maps. 3718 */ 3719 boolean_t 3720 mmu_radix_is_modified(vm_page_t m) 3721 { 3722 3723 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3724 ("pmap_is_modified: page %p is not managed", m)); 3725 3726 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3727 /* 3728 * If the page is not busied then this check is racy. 3729 */ 3730 if (!pmap_page_is_write_mapped(m)) 3731 return (FALSE); 3732 return (pmap_page_test_mappings(m, FALSE, TRUE)); 3733 } 3734 3735 boolean_t 3736 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3737 { 3738 pml3_entry_t *l3e; 3739 pt_entry_t *pte; 3740 boolean_t rv; 3741 3742 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 3743 rv = FALSE; 3744 PMAP_LOCK(pmap); 3745 l3e = pmap_pml3e(pmap, addr); 3746 if (l3e != NULL && (*l3e & (RPTE_LEAF | PG_V)) == PG_V) { 3747 pte = pmap_l3e_to_pte(l3e, addr); 3748 rv = (*pte & PG_V) == 0; 3749 } 3750 PMAP_UNLOCK(pmap); 3751 return (rv); 3752 } 3753 3754 boolean_t 3755 mmu_radix_is_referenced(vm_page_t m) 3756 { 3757 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3758 ("pmap_is_referenced: page %p is not managed", m)); 3759 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3760 return (pmap_page_test_mappings(m, TRUE, FALSE)); 3761 } 3762 3763 /* 3764 * pmap_ts_referenced: 3765 * 3766 * Return a count of reference bits for a page, clearing those bits. 3767 * It is not necessary for every reference bit to be cleared, but it 3768 * is necessary that 0 only be returned when there are truly no 3769 * reference bits set. 3770 * 3771 * As an optimization, update the page's dirty field if a modified bit is 3772 * found while counting reference bits. This opportunistic update can be 3773 * performed at low cost and can eliminate the need for some future calls 3774 * to pmap_is_modified(). However, since this function stops after 3775 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 3776 * dirty pages. Those dirty pages will only be detected by a future call 3777 * to pmap_is_modified(). 3778 * 3779 * A DI block is not needed within this function, because 3780 * invalidations are performed before the PV list lock is 3781 * released. 3782 */ 3783 boolean_t 3784 mmu_radix_ts_referenced(vm_page_t m) 3785 { 3786 struct md_page *pvh; 3787 pv_entry_t pv, pvf; 3788 pmap_t pmap; 3789 struct rwlock *lock; 3790 pml3_entry_t oldl3e, *l3e; 3791 pt_entry_t *pte; 3792 vm_paddr_t pa; 3793 int cleared, md_gen, not_cleared, pvh_gen; 3794 struct spglist free; 3795 3796 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3797 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3798 ("pmap_ts_referenced: page %p is not managed", m)); 3799 SLIST_INIT(&free); 3800 cleared = 0; 3801 pa = VM_PAGE_TO_PHYS(m); 3802 lock = PHYS_TO_PV_LIST_LOCK(pa); 3803 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); 3804 rw_wlock(lock); 3805 retry: 3806 not_cleared = 0; 3807 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 3808 goto small_mappings; 3809 pv = pvf; 3810 do { 3811 if (pvf == NULL) 3812 pvf = pv; 3813 pmap = PV_PMAP(pv); 3814 if (!PMAP_TRYLOCK(pmap)) { 3815 pvh_gen = pvh->pv_gen; 3816 rw_wunlock(lock); 3817 PMAP_LOCK(pmap); 3818 rw_wlock(lock); 3819 if (pvh_gen != pvh->pv_gen) { 3820 PMAP_UNLOCK(pmap); 3821 goto retry; 3822 } 3823 } 3824 l3e = pmap_pml3e(pmap, pv->pv_va); 3825 oldl3e = *l3e; 3826 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 3827 /* 3828 * Although "oldpde" is mapping a 2MB page, because 3829 * this function is called at a 4KB page granularity, 3830 * we only update the 4KB page under test. 3831 */ 3832 vm_page_dirty(m); 3833 } 3834 if ((oldl3e & PG_A) != 0) { 3835 /* 3836 * Since this reference bit is shared by 512 4KB 3837 * pages, it should not be cleared every time it is 3838 * tested. Apply a simple "hash" function on the 3839 * physical page number, the virtual superpage number, 3840 * and the pmap address to select one 4KB page out of 3841 * the 512 on which testing the reference bit will 3842 * result in clearing that reference bit. This 3843 * function is designed to avoid the selection of the 3844 * same 4KB page for every 2MB page mapping. 3845 * 3846 * On demotion, a mapping that hasn't been referenced 3847 * is simply destroyed. To avoid the possibility of a 3848 * subsequent page fault on a demoted wired mapping, 3849 * always leave its reference bit set. Moreover, 3850 * since the superpage is wired, the current state of 3851 * its reference bit won't affect page replacement. 3852 */ 3853 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^ 3854 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && 3855 (oldl3e & PG_W) == 0) { 3856 atomic_clear_long(l3e, PG_A); 3857 pmap_invalidate_page(pmap, pv->pv_va); 3858 cleared++; 3859 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 3860 ("inconsistent pv lock %p %p for page %p", 3861 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 3862 } else 3863 not_cleared++; 3864 } 3865 PMAP_UNLOCK(pmap); 3866 /* Rotate the PV list if it has more than one entry. */ 3867 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3868 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 3869 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 3870 pvh->pv_gen++; 3871 } 3872 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 3873 goto out; 3874 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 3875 small_mappings: 3876 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 3877 goto out; 3878 pv = pvf; 3879 do { 3880 if (pvf == NULL) 3881 pvf = pv; 3882 pmap = PV_PMAP(pv); 3883 if (!PMAP_TRYLOCK(pmap)) { 3884 pvh_gen = pvh->pv_gen; 3885 md_gen = m->md.pv_gen; 3886 rw_wunlock(lock); 3887 PMAP_LOCK(pmap); 3888 rw_wlock(lock); 3889 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 3890 PMAP_UNLOCK(pmap); 3891 goto retry; 3892 } 3893 } 3894 l3e = pmap_pml3e(pmap, pv->pv_va); 3895 KASSERT((*l3e & RPTE_LEAF) == 0, 3896 ("pmap_ts_referenced: found a 2mpage in page %p's pv list", 3897 m)); 3898 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 3899 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3900 vm_page_dirty(m); 3901 if ((*pte & PG_A) != 0) { 3902 atomic_clear_long(pte, PG_A); 3903 pmap_invalidate_page(pmap, pv->pv_va); 3904 cleared++; 3905 } 3906 PMAP_UNLOCK(pmap); 3907 /* Rotate the PV list if it has more than one entry. */ 3908 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3909 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 3910 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3911 m->md.pv_gen++; 3912 } 3913 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 3914 not_cleared < PMAP_TS_REFERENCED_MAX); 3915 out: 3916 rw_wunlock(lock); 3917 vm_page_free_pages_toq(&free, true); 3918 return (cleared + not_cleared); 3919 } 3920 3921 static vm_offset_t 3922 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start, 3923 vm_paddr_t end, int prot __unused) 3924 { 3925 3926 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end, 3927 prot); 3928 return (PHYS_TO_DMAP(start)); 3929 } 3930 3931 void 3932 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr, 3933 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 3934 { 3935 pml3_entry_t *l3e; 3936 vm_paddr_t pa, ptepa; 3937 vm_page_t p, pdpg; 3938 vm_memattr_t ma; 3939 3940 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr, 3941 object, pindex, size); 3942 VM_OBJECT_ASSERT_WLOCKED(object); 3943 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 3944 ("pmap_object_init_pt: non-device object")); 3945 /* NB: size can be logically ored with addr here */ 3946 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) { 3947 if (!mmu_radix_ps_enabled(pmap)) 3948 return; 3949 if (!vm_object_populate(object, pindex, pindex + atop(size))) 3950 return; 3951 p = vm_page_lookup(object, pindex); 3952 KASSERT(p->valid == VM_PAGE_BITS_ALL, 3953 ("pmap_object_init_pt: invalid page %p", p)); 3954 ma = p->md.mdpg_cache_attrs; 3955 3956 /* 3957 * Abort the mapping if the first page is not physically 3958 * aligned to a 2MB page boundary. 3959 */ 3960 ptepa = VM_PAGE_TO_PHYS(p); 3961 if (ptepa & L3_PAGE_MASK) 3962 return; 3963 3964 /* 3965 * Skip the first page. Abort the mapping if the rest of 3966 * the pages are not physically contiguous or have differing 3967 * memory attributes. 3968 */ 3969 p = TAILQ_NEXT(p, listq); 3970 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; 3971 pa += PAGE_SIZE) { 3972 KASSERT(p->valid == VM_PAGE_BITS_ALL, 3973 ("pmap_object_init_pt: invalid page %p", p)); 3974 if (pa != VM_PAGE_TO_PHYS(p) || 3975 ma != p->md.mdpg_cache_attrs) 3976 return; 3977 p = TAILQ_NEXT(p, listq); 3978 } 3979 3980 PMAP_LOCK(pmap); 3981 for (pa = ptepa | pmap_cache_bits(ma); 3982 pa < ptepa + size; pa += L3_PAGE_SIZE) { 3983 pdpg = pmap_allocl3e(pmap, addr, NULL); 3984 if (pdpg == NULL) { 3985 /* 3986 * The creation of mappings below is only an 3987 * optimization. If a page directory page 3988 * cannot be allocated without blocking, 3989 * continue on to the next mapping rather than 3990 * blocking. 3991 */ 3992 addr += L3_PAGE_SIZE; 3993 continue; 3994 } 3995 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 3996 l3e = &l3e[pmap_pml3e_index(addr)]; 3997 if ((*l3e & PG_V) == 0) { 3998 pa |= PG_M | PG_A | PG_RW; 3999 pte_store(l3e, pa); 4000 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4001 atomic_add_long(&pmap_l3e_mappings, 1); 4002 } else { 4003 /* Continue on if the PDE is already valid. */ 4004 pdpg->ref_count--; 4005 KASSERT(pdpg->ref_count > 0, 4006 ("pmap_object_init_pt: missing reference " 4007 "to page directory page, va: 0x%lx", addr)); 4008 } 4009 addr += L3_PAGE_SIZE; 4010 } 4011 ptesync(); 4012 PMAP_UNLOCK(pmap); 4013 } 4014 } 4015 4016 boolean_t 4017 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m) 4018 { 4019 struct md_page *pvh; 4020 struct rwlock *lock; 4021 pv_entry_t pv; 4022 int loops = 0; 4023 boolean_t rv; 4024 4025 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4026 ("pmap_page_exists_quick: page %p is not managed", m)); 4027 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m); 4028 rv = FALSE; 4029 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4030 rw_rlock(lock); 4031 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4032 if (PV_PMAP(pv) == pmap) { 4033 rv = TRUE; 4034 break; 4035 } 4036 loops++; 4037 if (loops >= 16) 4038 break; 4039 } 4040 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 4041 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4042 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4043 if (PV_PMAP(pv) == pmap) { 4044 rv = TRUE; 4045 break; 4046 } 4047 loops++; 4048 if (loops >= 16) 4049 break; 4050 } 4051 } 4052 rw_runlock(lock); 4053 return (rv); 4054 } 4055 4056 void 4057 mmu_radix_page_init(vm_page_t m) 4058 { 4059 4060 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4061 TAILQ_INIT(&m->md.pv_list); 4062 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT; 4063 } 4064 4065 int 4066 mmu_radix_page_wired_mappings(vm_page_t m) 4067 { 4068 struct rwlock *lock; 4069 struct md_page *pvh; 4070 pmap_t pmap; 4071 pt_entry_t *pte; 4072 pv_entry_t pv; 4073 int count, md_gen, pvh_gen; 4074 4075 if ((m->oflags & VPO_UNMANAGED) != 0) 4076 return (0); 4077 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4078 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4079 rw_rlock(lock); 4080 restart: 4081 count = 0; 4082 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4083 pmap = PV_PMAP(pv); 4084 if (!PMAP_TRYLOCK(pmap)) { 4085 md_gen = m->md.pv_gen; 4086 rw_runlock(lock); 4087 PMAP_LOCK(pmap); 4088 rw_rlock(lock); 4089 if (md_gen != m->md.pv_gen) { 4090 PMAP_UNLOCK(pmap); 4091 goto restart; 4092 } 4093 } 4094 pte = pmap_pte(pmap, pv->pv_va); 4095 if ((*pte & PG_W) != 0) 4096 count++; 4097 PMAP_UNLOCK(pmap); 4098 } 4099 if ((m->flags & PG_FICTITIOUS) == 0) { 4100 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4101 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4102 pmap = PV_PMAP(pv); 4103 if (!PMAP_TRYLOCK(pmap)) { 4104 md_gen = m->md.pv_gen; 4105 pvh_gen = pvh->pv_gen; 4106 rw_runlock(lock); 4107 PMAP_LOCK(pmap); 4108 rw_rlock(lock); 4109 if (md_gen != m->md.pv_gen || 4110 pvh_gen != pvh->pv_gen) { 4111 PMAP_UNLOCK(pmap); 4112 goto restart; 4113 } 4114 } 4115 pte = pmap_pml3e(pmap, pv->pv_va); 4116 if ((*pte & PG_W) != 0) 4117 count++; 4118 PMAP_UNLOCK(pmap); 4119 } 4120 } 4121 rw_runlock(lock); 4122 return (count); 4123 } 4124 4125 static void 4126 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa) 4127 { 4128 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT); 4129 } 4130 4131 int 4132 mmu_radix_pinit(pmap_t pmap) 4133 { 4134 vmem_addr_t pid; 4135 vm_paddr_t l1pa; 4136 4137 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4138 4139 /* 4140 * allocate the page directory page 4141 */ 4142 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK); 4143 4144 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++) 4145 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE); 4146 pmap->pm_radix.rt_root = 0; 4147 TAILQ_INIT(&pmap->pm_pvchunk); 4148 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4149 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4150 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid); 4151 4152 pmap->pm_pid = pid; 4153 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1); 4154 mmu_radix_update_proctab(pid, l1pa); 4155 __asm __volatile("ptesync;isync" : : : "memory"); 4156 4157 return (1); 4158 } 4159 4160 /* 4161 * This routine is called if the desired page table page does not exist. 4162 * 4163 * If page table page allocation fails, this routine may sleep before 4164 * returning NULL. It sleeps only if a lock pointer was given. 4165 * 4166 * Note: If a page allocation fails at page table level two or three, 4167 * one or two pages may be held during the wait, only to be released 4168 * afterwards. This conservative approach is easily argued to avoid 4169 * race conditions. 4170 */ 4171 static vm_page_t 4172 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 4173 { 4174 vm_page_t m, pdppg, pdpg; 4175 4176 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4177 4178 /* 4179 * Allocate a page table page. 4180 */ 4181 if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | 4182 VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 4183 if (lockp != NULL) { 4184 RELEASE_PV_LIST_LOCK(lockp); 4185 PMAP_UNLOCK(pmap); 4186 vm_wait(NULL); 4187 PMAP_LOCK(pmap); 4188 } 4189 /* 4190 * Indicate the need to retry. While waiting, the page table 4191 * page may have been allocated. 4192 */ 4193 return (NULL); 4194 } 4195 if ((m->flags & PG_ZERO) == 0) 4196 mmu_radix_zero_page(m); 4197 4198 /* 4199 * Map the pagetable page into the process address space, if 4200 * it isn't already there. 4201 */ 4202 4203 if (ptepindex >= (NUPDE + NUPDPE)) { 4204 pml1_entry_t *l1e; 4205 vm_pindex_t pml1index; 4206 4207 /* Wire up a new PDPE page */ 4208 pml1index = ptepindex - (NUPDE + NUPDPE); 4209 l1e = &pmap->pm_pml1[pml1index]; 4210 pde_store(l1e, VM_PAGE_TO_PHYS(m)); 4211 4212 } else if (ptepindex >= NUPDE) { 4213 vm_pindex_t pml1index; 4214 vm_pindex_t pdpindex; 4215 pml1_entry_t *l1e; 4216 pml2_entry_t *l2e; 4217 4218 /* Wire up a new l2e page */ 4219 pdpindex = ptepindex - NUPDE; 4220 pml1index = pdpindex >> RPTE_SHIFT; 4221 4222 l1e = &pmap->pm_pml1[pml1index]; 4223 if ((*l1e & PG_V) == 0) { 4224 /* Have to allocate a new pdp, recurse */ 4225 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index, 4226 lockp) == NULL) { 4227 vm_page_unwire_noq(m); 4228 vm_page_free_zero(m); 4229 return (NULL); 4230 } 4231 } else { 4232 /* Add reference to l2e page */ 4233 pdppg = PHYS_TO_VM_PAGE(*l1e & PG_FRAME); 4234 pdppg->ref_count++; 4235 } 4236 l2e = (pml2_entry_t *)PHYS_TO_DMAP(*l1e & PG_FRAME); 4237 4238 /* Now find the pdp page */ 4239 l2e = &l2e[pdpindex & RPTE_MASK]; 4240 pde_store(l2e, VM_PAGE_TO_PHYS(m)); 4241 4242 } else { 4243 vm_pindex_t pml1index; 4244 vm_pindex_t pdpindex; 4245 pml1_entry_t *l1e; 4246 pml2_entry_t *l2e; 4247 pml3_entry_t *l3e; 4248 4249 /* Wire up a new PTE page */ 4250 pdpindex = ptepindex >> RPTE_SHIFT; 4251 pml1index = pdpindex >> RPTE_SHIFT; 4252 4253 /* First, find the pdp and check that its valid. */ 4254 l1e = &pmap->pm_pml1[pml1index]; 4255 if ((*l1e & PG_V) == 0) { 4256 /* Have to allocate a new pd, recurse */ 4257 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4258 lockp) == NULL) { 4259 vm_page_unwire_noq(m); 4260 vm_page_free_zero(m); 4261 return (NULL); 4262 } 4263 l2e = (pml2_entry_t *)PHYS_TO_DMAP(*l1e & PG_FRAME); 4264 l2e = &l2e[pdpindex & RPTE_MASK]; 4265 } else { 4266 l2e = (pml2_entry_t *)PHYS_TO_DMAP(*l1e & PG_FRAME); 4267 l2e = &l2e[pdpindex & RPTE_MASK]; 4268 if ((*l2e & PG_V) == 0) { 4269 /* Have to allocate a new pd, recurse */ 4270 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4271 lockp) == NULL) { 4272 vm_page_unwire_noq(m); 4273 vm_page_free_zero(m); 4274 return (NULL); 4275 } 4276 } else { 4277 /* Add reference to the pd page */ 4278 pdpg = PHYS_TO_VM_PAGE(*l2e & PG_FRAME); 4279 pdpg->ref_count++; 4280 } 4281 } 4282 l3e = (pml3_entry_t *)PHYS_TO_DMAP(*l2e & PG_FRAME); 4283 4284 /* Now we know where the page directory page is */ 4285 l3e = &l3e[ptepindex & RPTE_MASK]; 4286 pde_store(l3e, VM_PAGE_TO_PHYS(m)); 4287 } 4288 4289 pmap_resident_count_inc(pmap, 1); 4290 return (m); 4291 } 4292 static vm_page_t 4293 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4294 { 4295 vm_pindex_t pdpindex, ptepindex; 4296 pml2_entry_t *pdpe; 4297 vm_page_t pdpg; 4298 4299 retry: 4300 pdpe = pmap_pml2e(pmap, va); 4301 if (pdpe != NULL && (*pdpe & PG_V) != 0) { 4302 /* Add a reference to the pd page. */ 4303 pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); 4304 pdpg->ref_count++; 4305 } else { 4306 /* Allocate a pd page. */ 4307 ptepindex = pmap_l3e_pindex(va); 4308 pdpindex = ptepindex >> RPTE_SHIFT; 4309 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); 4310 if (pdpg == NULL && lockp != NULL) 4311 goto retry; 4312 } 4313 return (pdpg); 4314 } 4315 4316 static vm_page_t 4317 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4318 { 4319 vm_pindex_t ptepindex; 4320 pml3_entry_t *pd; 4321 vm_page_t m; 4322 4323 /* 4324 * Calculate pagetable page index 4325 */ 4326 ptepindex = pmap_l3e_pindex(va); 4327 retry: 4328 /* 4329 * Get the page directory entry 4330 */ 4331 pd = pmap_pml3e(pmap, va); 4332 4333 /* 4334 * This supports switching from a 2MB page to a 4335 * normal 4K page. 4336 */ 4337 if (pd != NULL && (*pd & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) { 4338 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) { 4339 /* 4340 * Invalidation of the 2MB page mapping may have caused 4341 * the deallocation of the underlying PD page. 4342 */ 4343 pd = NULL; 4344 } 4345 } 4346 4347 /* 4348 * If the page table page is mapped, we just increment the 4349 * hold count, and activate it. 4350 */ 4351 if (pd != NULL && (*pd & PG_V) != 0) { 4352 m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); 4353 m->ref_count++; 4354 } else { 4355 /* 4356 * Here if the pte page isn't mapped, or if it has been 4357 * deallocated. 4358 */ 4359 m = _pmap_allocpte(pmap, ptepindex, lockp); 4360 if (m == NULL && lockp != NULL) 4361 goto retry; 4362 } 4363 return (m); 4364 } 4365 4366 static void 4367 mmu_radix_pinit0(pmap_t pmap) 4368 { 4369 4370 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4371 PMAP_LOCK_INIT(pmap); 4372 pmap->pm_pml1 = kernel_pmap->pm_pml1; 4373 pmap->pm_pid = kernel_pmap->pm_pid; 4374 4375 pmap->pm_radix.rt_root = 0; 4376 TAILQ_INIT(&pmap->pm_pvchunk); 4377 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4378 kernel_pmap->pm_flags = 4379 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4380 } 4381 /* 4382 * pmap_protect_l3e: do the things to protect a 2mpage in a process 4383 */ 4384 static boolean_t 4385 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot) 4386 { 4387 pt_entry_t newpde, oldpde; 4388 vm_offset_t eva, va; 4389 vm_page_t m; 4390 boolean_t anychanged; 4391 4392 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4393 KASSERT((sva & L3_PAGE_MASK) == 0, 4394 ("pmap_protect_l3e: sva is not 2mpage aligned")); 4395 anychanged = FALSE; 4396 retry: 4397 oldpde = newpde = *l3e; 4398 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) == 4399 (PG_MANAGED | PG_M | PG_RW)) { 4400 eva = sva + L3_PAGE_SIZE; 4401 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4402 va < eva; va += PAGE_SIZE, m++) 4403 vm_page_dirty(m); 4404 } 4405 if ((prot & VM_PROT_WRITE) == 0) { 4406 newpde &= ~(PG_RW | PG_M); 4407 newpde |= RPTE_EAA_R; 4408 } 4409 if (prot & VM_PROT_EXECUTE) 4410 newpde |= PG_X; 4411 if (newpde != oldpde) { 4412 /* 4413 * As an optimization to future operations on this PDE, clear 4414 * PG_PROMOTED. The impending invalidation will remove any 4415 * lingering 4KB page mappings from the TLB. 4416 */ 4417 if (!atomic_cmpset_long(l3e, oldpde, newpde & ~PG_PROMOTED)) 4418 goto retry; 4419 anychanged = TRUE; 4420 } 4421 return (anychanged); 4422 } 4423 4424 void 4425 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 4426 vm_prot_t prot) 4427 { 4428 vm_offset_t va_next; 4429 pml1_entry_t *l1e; 4430 pml2_entry_t *l2e; 4431 pml3_entry_t ptpaddr, *l3e; 4432 pt_entry_t *pte; 4433 boolean_t anychanged; 4434 4435 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva, 4436 prot); 4437 4438 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 4439 if (prot == VM_PROT_NONE) { 4440 mmu_radix_remove(pmap, sva, eva); 4441 return; 4442 } 4443 4444 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 4445 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 4446 return; 4447 4448 #ifdef INVARIANTS 4449 if (VERBOSE_PROTECT || pmap_logging) 4450 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n", 4451 pmap, sva, eva, prot, pmap->pm_pid); 4452 #endif 4453 anychanged = FALSE; 4454 4455 PMAP_LOCK(pmap); 4456 for (; sva < eva; sva = va_next) { 4457 l1e = pmap_pml1e(pmap, sva); 4458 if ((*l1e & PG_V) == 0) { 4459 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 4460 if (va_next < sva) 4461 va_next = eva; 4462 continue; 4463 } 4464 4465 l2e = pmap_l1e_to_l2e(l1e, sva); 4466 if ((*l2e & PG_V) == 0) { 4467 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 4468 if (va_next < sva) 4469 va_next = eva; 4470 continue; 4471 } 4472 4473 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 4474 if (va_next < sva) 4475 va_next = eva; 4476 4477 l3e = pmap_l2e_to_l3e(l2e, sva); 4478 ptpaddr = *l3e; 4479 4480 /* 4481 * Weed out invalid mappings. 4482 */ 4483 if (ptpaddr == 0) 4484 continue; 4485 4486 /* 4487 * Check for large page. 4488 */ 4489 if ((ptpaddr & RPTE_LEAF) != 0) { 4490 /* 4491 * Are we protecting the entire large page? If not, 4492 * demote the mapping and fall through. 4493 */ 4494 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 4495 if (pmap_protect_l3e(pmap, l3e, sva, prot)) 4496 anychanged = TRUE; 4497 continue; 4498 } else if (!pmap_demote_l3e(pmap, l3e, sva)) { 4499 /* 4500 * The large page mapping was destroyed. 4501 */ 4502 continue; 4503 } 4504 } 4505 4506 if (va_next > eva) 4507 va_next = eva; 4508 4509 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 4510 sva += PAGE_SIZE) { 4511 pt_entry_t obits, pbits; 4512 vm_page_t m; 4513 4514 retry: 4515 MPASS(pte == pmap_pte(pmap, sva)); 4516 obits = pbits = *pte; 4517 if ((pbits & PG_V) == 0) 4518 continue; 4519 4520 if ((prot & VM_PROT_WRITE) == 0) { 4521 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == 4522 (PG_MANAGED | PG_M | PG_RW)) { 4523 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); 4524 vm_page_dirty(m); 4525 } 4526 pbits &= ~(PG_RW | PG_M); 4527 pbits |= RPTE_EAA_R; 4528 } 4529 if (prot & VM_PROT_EXECUTE) 4530 pbits |= PG_X; 4531 4532 if (pbits != obits) { 4533 if (!atomic_cmpset_long(pte, obits, pbits)) 4534 goto retry; 4535 if (obits & (PG_A|PG_M)) { 4536 anychanged = TRUE; 4537 #ifdef INVARIANTS 4538 if (VERBOSE_PROTECT || pmap_logging) 4539 printf("%#lx %#lx -> %#lx\n", 4540 sva, obits, pbits); 4541 #endif 4542 } 4543 } 4544 } 4545 } 4546 if (anychanged) 4547 pmap_invalidate_all(pmap); 4548 PMAP_UNLOCK(pmap); 4549 } 4550 4551 void 4552 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count) 4553 { 4554 4555 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count); 4556 pt_entry_t oldpte, pa, *pte; 4557 vm_page_t m; 4558 uint64_t cache_bits, attr_bits; 4559 vm_offset_t va; 4560 4561 oldpte = 0; 4562 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 4563 va = sva; 4564 pte = kvtopte(va); 4565 while (va < sva + PAGE_SIZE * count) { 4566 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4567 pte = kvtopte(va); 4568 MPASS(pte == pmap_pte(kernel_pmap, va)); 4569 4570 /* 4571 * XXX there has to be a more efficient way than traversing 4572 * the page table every time - but go for correctness for 4573 * today 4574 */ 4575 4576 m = *ma++; 4577 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs); 4578 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits; 4579 if (*pte != pa) { 4580 oldpte |= *pte; 4581 pte_store(pte, pa); 4582 } 4583 va += PAGE_SIZE; 4584 pte++; 4585 } 4586 if (__predict_false((oldpte & RPTE_VALID) != 0)) 4587 pmap_invalidate_range(kernel_pmap, sva, sva + count * 4588 PAGE_SIZE); 4589 else 4590 ptesync(); 4591 } 4592 4593 void 4594 mmu_radix_qremove(vm_offset_t sva, int count) 4595 { 4596 vm_offset_t va; 4597 pt_entry_t *pte; 4598 4599 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count); 4600 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva)); 4601 4602 va = sva; 4603 pte = kvtopte(va); 4604 while (va < sva + PAGE_SIZE * count) { 4605 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4606 pte = kvtopte(va); 4607 pte_clear(pte); 4608 pte++; 4609 va += PAGE_SIZE; 4610 } 4611 pmap_invalidate_range(kernel_pmap, sva, va); 4612 } 4613 4614 /*************************************************** 4615 * Page table page management routines..... 4616 ***************************************************/ 4617 /* 4618 * Schedule the specified unused page table page to be freed. Specifically, 4619 * add the page to the specified list of pages that will be released to the 4620 * physical memory manager after the TLB has been updated. 4621 */ 4622 static __inline void 4623 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 4624 boolean_t set_PG_ZERO) 4625 { 4626 4627 if (set_PG_ZERO) 4628 m->flags |= PG_ZERO; 4629 else 4630 m->flags &= ~PG_ZERO; 4631 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 4632 } 4633 4634 /* 4635 * Inserts the specified page table page into the specified pmap's collection 4636 * of idle page table pages. Each of a pmap's page table pages is responsible 4637 * for mapping a distinct range of virtual addresses. The pmap's collection is 4638 * ordered by this virtual address range. 4639 */ 4640 static __inline int 4641 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) 4642 { 4643 4644 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4645 return (vm_radix_insert(&pmap->pm_radix, mpte)); 4646 } 4647 4648 /* 4649 * Removes the page table page mapping the specified virtual address from the 4650 * specified pmap's collection of idle page table pages, and returns it. 4651 * Otherwise, returns NULL if there is no page table page corresponding to the 4652 * specified virtual address. 4653 */ 4654 static __inline vm_page_t 4655 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 4656 { 4657 4658 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4659 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va))); 4660 } 4661 4662 /* 4663 * Decrements a page table page's wire count, which is used to record the 4664 * number of valid page table entries within the page. If the wire count 4665 * drops to zero, then the page table page is unmapped. Returns TRUE if the 4666 * page table page was unmapped and FALSE otherwise. 4667 */ 4668 static inline boolean_t 4669 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4670 { 4671 4672 --m->ref_count; 4673 if (m->ref_count == 0) { 4674 _pmap_unwire_ptp(pmap, va, m, free); 4675 return (TRUE); 4676 } else 4677 return (FALSE); 4678 } 4679 4680 static void 4681 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4682 { 4683 4684 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4685 /* 4686 * unmap the page table page 4687 */ 4688 if (m->pindex >= (NUPDE + NUPDPE)) { 4689 /* PDP page */ 4690 pml1_entry_t *pml1; 4691 pml1 = pmap_pml1e(pmap, va); 4692 *pml1 = 0; 4693 } else if (m->pindex >= NUPDE) { 4694 /* PD page */ 4695 pml2_entry_t *l2e; 4696 l2e = pmap_pml2e(pmap, va); 4697 *l2e = 0; 4698 } else { 4699 /* PTE page */ 4700 pml3_entry_t *l3e; 4701 l3e = pmap_pml3e(pmap, va); 4702 *l3e = 0; 4703 } 4704 pmap_resident_count_dec(pmap, 1); 4705 if (m->pindex < NUPDE) { 4706 /* We just released a PT, unhold the matching PD */ 4707 vm_page_t pdpg; 4708 4709 pdpg = PHYS_TO_VM_PAGE(*pmap_pml2e(pmap, va) & PG_FRAME); 4710 pmap_unwire_ptp(pmap, va, pdpg, free); 4711 } 4712 if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { 4713 /* We just released a PD, unhold the matching PDP */ 4714 vm_page_t pdppg; 4715 4716 pdppg = PHYS_TO_VM_PAGE(*pmap_pml1e(pmap, va) & PG_FRAME); 4717 pmap_unwire_ptp(pmap, va, pdppg, free); 4718 } 4719 4720 /* 4721 * Put page on a list so that it is released after 4722 * *ALL* TLB shootdown is done 4723 */ 4724 pmap_add_delayed_free_list(m, free, TRUE); 4725 } 4726 4727 /* 4728 * After removing a page table entry, this routine is used to 4729 * conditionally free the page, and manage the hold/wire counts. 4730 */ 4731 static int 4732 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde, 4733 struct spglist *free) 4734 { 4735 vm_page_t mpte; 4736 4737 if (va >= VM_MAXUSER_ADDRESS) 4738 return (0); 4739 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 4740 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 4741 return (pmap_unwire_ptp(pmap, va, mpte, free)); 4742 } 4743 4744 void 4745 mmu_radix_release(pmap_t pmap) 4746 { 4747 4748 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4749 KASSERT(pmap->pm_stats.resident_count == 0, 4750 ("pmap_release: pmap resident count %ld != 0", 4751 pmap->pm_stats.resident_count)); 4752 KASSERT(vm_radix_is_empty(&pmap->pm_radix), 4753 ("pmap_release: pmap has reserved page table page(s)")); 4754 4755 pmap_invalidate_all(pmap); 4756 isa3_proctab[pmap->pm_pid].proctab0 = 0; 4757 uma_zfree(zone_radix_pgd, pmap->pm_pml1); 4758 vmem_free(asid_arena, pmap->pm_pid, 1); 4759 } 4760 4761 /* 4762 * Create the PV entry for a 2MB page mapping. Always returns true unless the 4763 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 4764 * false if the PV entry cannot be allocated without resorting to reclamation. 4765 */ 4766 static bool 4767 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags, 4768 struct rwlock **lockp) 4769 { 4770 struct md_page *pvh; 4771 pv_entry_t pv; 4772 vm_paddr_t pa; 4773 4774 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4775 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4776 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 4777 NULL : lockp)) == NULL) 4778 return (false); 4779 pv->pv_va = va; 4780 pa = pde & PG_PS_FRAME; 4781 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4782 pvh = pa_to_pvh(pa); 4783 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 4784 pvh->pv_gen++; 4785 return (true); 4786 } 4787 4788 /* 4789 * Fills a page table page with mappings to consecutive physical pages. 4790 */ 4791 static void 4792 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) 4793 { 4794 pt_entry_t *pte; 4795 4796 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { 4797 *pte = newpte; 4798 newpte += PAGE_SIZE; 4799 } 4800 } 4801 4802 static boolean_t 4803 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va) 4804 { 4805 struct rwlock *lock; 4806 boolean_t rv; 4807 4808 lock = NULL; 4809 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock); 4810 if (lock != NULL) 4811 rw_wunlock(lock); 4812 return (rv); 4813 } 4814 4815 static boolean_t 4816 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 4817 struct rwlock **lockp) 4818 { 4819 pml3_entry_t oldpde; 4820 pt_entry_t *firstpte; 4821 vm_paddr_t mptepa; 4822 vm_page_t mpte; 4823 struct spglist free; 4824 vm_offset_t sva; 4825 4826 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4827 oldpde = *l3e; 4828 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 4829 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx", 4830 oldpde)); 4831 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == 4832 NULL) { 4833 KASSERT((oldpde & PG_W) == 0, 4834 ("pmap_demote_l3e: page table page for a wired mapping" 4835 " is missing")); 4836 4837 /* 4838 * Invalidate the 2MB page mapping and return "failure" if the 4839 * mapping was never accessed or the allocation of the new 4840 * page table page fails. If the 2MB page mapping belongs to 4841 * the direct map region of the kernel's address space, then 4842 * the page allocation request specifies the highest possible 4843 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is 4844 * normal. Page table pages are preallocated for every other 4845 * part of the kernel address space, so the direct map region 4846 * is the only part of the kernel address space that must be 4847 * handled here. 4848 */ 4849 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, 4850 pmap_l3e_pindex(va), (va >= DMAP_MIN_ADDRESS && va < 4851 DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | 4852 VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { 4853 SLIST_INIT(&free); 4854 sva = trunc_2mpage(va); 4855 pmap_remove_l3e(pmap, l3e, sva, &free, lockp); 4856 pmap_invalidate_l3e_page(pmap, sva, oldpde); 4857 vm_page_free_pages_toq(&free, true); 4858 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" 4859 " in pmap %p", va, pmap); 4860 return (FALSE); 4861 } 4862 if (va < VM_MAXUSER_ADDRESS) 4863 pmap_resident_count_inc(pmap, 1); 4864 } 4865 mptepa = VM_PAGE_TO_PHYS(mpte); 4866 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); 4867 KASSERT((oldpde & PG_A) != 0, 4868 ("pmap_demote_l3e: oldpde is missing PG_A")); 4869 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, 4870 ("pmap_demote_l3e: oldpde is missing PG_M")); 4871 4872 /* 4873 * If the page table page is new, initialize it. 4874 */ 4875 if (mpte->ref_count == 1) { 4876 mpte->ref_count = NPTEPG; 4877 pmap_fill_ptp(firstpte, oldpde); 4878 } 4879 4880 KASSERT((*firstpte & PG_FRAME) == (oldpde & PG_FRAME), 4881 ("pmap_demote_l3e: firstpte and newpte map different physical" 4882 " addresses")); 4883 4884 /* 4885 * If the mapping has changed attributes, update the page table 4886 * entries. 4887 */ 4888 if ((*firstpte & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE)) 4889 pmap_fill_ptp(firstpte, oldpde); 4890 4891 /* 4892 * The spare PV entries must be reserved prior to demoting the 4893 * mapping, that is, prior to changing the PDE. Otherwise, the state 4894 * of the PDE and the PV lists will be inconsistent, which can result 4895 * in reclaim_pv_chunk() attempting to remove a PV entry from the 4896 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected 4897 * PV entry for the 2MB page mapping that is being demoted. 4898 */ 4899 if ((oldpde & PG_MANAGED) != 0) 4900 reserve_pv_entries(pmap, NPTEPG - 1, lockp); 4901 4902 /* 4903 * Demote the mapping. This pmap is locked. The old PDE has 4904 * PG_A set. If the old PDE has PG_RW set, it also has PG_M 4905 * set. Thus, there is no danger of a race with another 4906 * processor changing the setting of PG_A and/or PG_M between 4907 * the read above and the store below. 4908 */ 4909 pde_store(l3e, mptepa); 4910 ptesync(); 4911 /* 4912 * Demote the PV entry. 4913 */ 4914 if ((oldpde & PG_MANAGED) != 0) 4915 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp); 4916 4917 4918 atomic_add_long(&pmap_l3e_demotions, 1); 4919 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx" 4920 " in pmap %p", va, pmap); 4921 return (TRUE); 4922 } 4923 4924 /* 4925 * pmap_remove_kernel_pde: Remove a kernel superpage mapping. 4926 */ 4927 static void 4928 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va) 4929 { 4930 vm_paddr_t mptepa; 4931 vm_page_t mpte; 4932 4933 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 4934 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4935 mpte = pmap_remove_pt_page(pmap, va); 4936 if (mpte == NULL) 4937 panic("pmap_remove_kernel_pde: Missing pt page."); 4938 4939 mptepa = VM_PAGE_TO_PHYS(mpte); 4940 4941 /* 4942 * Initialize the page table page. 4943 */ 4944 pagezero(PHYS_TO_DMAP(mptepa)); 4945 4946 /* 4947 * Demote the mapping. 4948 */ 4949 pde_store(l3e, mptepa); 4950 ptesync(); 4951 } 4952 4953 /* 4954 * pmap_remove_l3e: do the things to unmap a superpage in a process 4955 */ 4956 static int 4957 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 4958 struct spglist *free, struct rwlock **lockp) 4959 { 4960 struct md_page *pvh; 4961 pml3_entry_t oldpde; 4962 vm_offset_t eva, va; 4963 vm_page_t m, mpte; 4964 4965 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4966 KASSERT((sva & L3_PAGE_MASK) == 0, 4967 ("pmap_remove_l3e: sva is not 2mpage aligned")); 4968 oldpde = pte_load_clear(pdq); 4969 if (oldpde & PG_W) 4970 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE); 4971 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4972 if (oldpde & PG_MANAGED) { 4973 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); 4974 pvh = pa_to_pvh(oldpde & PG_PS_FRAME); 4975 pmap_pvh_free(pvh, pmap, sva); 4976 eva = sva + L3_PAGE_SIZE; 4977 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4978 va < eva; va += PAGE_SIZE, m++) { 4979 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) 4980 vm_page_dirty(m); 4981 if (oldpde & PG_A) 4982 vm_page_aflag_set(m, PGA_REFERENCED); 4983 if (TAILQ_EMPTY(&m->md.pv_list) && 4984 TAILQ_EMPTY(&pvh->pv_list)) 4985 vm_page_aflag_clear(m, PGA_WRITEABLE); 4986 } 4987 } 4988 if (pmap == kernel_pmap) { 4989 pmap_remove_kernel_l3e(pmap, pdq, sva); 4990 } else { 4991 mpte = pmap_remove_pt_page(pmap, sva); 4992 if (mpte != NULL) { 4993 pmap_resident_count_dec(pmap, 1); 4994 KASSERT(mpte->ref_count == NPTEPG, 4995 ("pmap_remove_l3e: pte page wire count error")); 4996 mpte->ref_count = 0; 4997 pmap_add_delayed_free_list(mpte, free, FALSE); 4998 } 4999 } 5000 return (pmap_unuse_pt(pmap, sva, *pmap_pml2e(pmap, sva), free)); 5001 } 5002 5003 5004 /* 5005 * pmap_remove_pte: do the things to unmap a page in a process 5006 */ 5007 static int 5008 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 5009 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp) 5010 { 5011 struct md_page *pvh; 5012 pt_entry_t oldpte; 5013 vm_page_t m; 5014 5015 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5016 oldpte = pte_load_clear(ptq); 5017 if (oldpte & RPTE_WIRED) 5018 pmap->pm_stats.wired_count -= 1; 5019 pmap_resident_count_dec(pmap, 1); 5020 if (oldpte & RPTE_MANAGED) { 5021 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); 5022 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5023 vm_page_dirty(m); 5024 if (oldpte & PG_A) 5025 vm_page_aflag_set(m, PGA_REFERENCED); 5026 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5027 pmap_pvh_free(&m->md, pmap, va); 5028 if (TAILQ_EMPTY(&m->md.pv_list) && 5029 (m->flags & PG_FICTITIOUS) == 0) { 5030 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5031 if (TAILQ_EMPTY(&pvh->pv_list)) 5032 vm_page_aflag_clear(m, PGA_WRITEABLE); 5033 } 5034 } 5035 return (pmap_unuse_pt(pmap, va, ptepde, free)); 5036 } 5037 5038 /* 5039 * Remove a single page from a process address space 5040 */ 5041 static bool 5042 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e, 5043 struct spglist *free) 5044 { 5045 struct rwlock *lock; 5046 pt_entry_t *pte; 5047 bool invalidate_all; 5048 5049 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5050 if ((*l3e & RPTE_VALID) == 0) { 5051 return (false); 5052 } 5053 pte = pmap_l3e_to_pte(l3e, va); 5054 if ((*pte & RPTE_VALID) == 0) { 5055 return (false); 5056 } 5057 lock = NULL; 5058 5059 invalidate_all = pmap_remove_pte(pmap, pte, va, *l3e, free, &lock); 5060 if (lock != NULL) 5061 rw_wunlock(lock); 5062 if (!invalidate_all) 5063 pmap_invalidate_page(pmap, va); 5064 return (invalidate_all); 5065 } 5066 5067 /* 5068 * Removes the specified range of addresses from the page table page. 5069 */ 5070 static bool 5071 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 5072 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp) 5073 { 5074 pt_entry_t *pte; 5075 vm_offset_t va; 5076 bool anyvalid; 5077 5078 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5079 anyvalid = false; 5080 va = eva; 5081 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++, 5082 sva += PAGE_SIZE) { 5083 MPASS(pte == pmap_pte(pmap, sva)); 5084 if (*pte == 0) { 5085 if (va != eva) { 5086 anyvalid = true; 5087 va = eva; 5088 } 5089 continue; 5090 } 5091 if (va == eva) 5092 va = sva; 5093 if (pmap_remove_pte(pmap, pte, sva, *l3e, free, lockp)) { 5094 anyvalid = true; 5095 sva += PAGE_SIZE; 5096 break; 5097 } 5098 } 5099 if (anyvalid) 5100 pmap_invalidate_all(pmap); 5101 else if (va != eva) 5102 pmap_invalidate_range(pmap, va, sva); 5103 return (anyvalid); 5104 } 5105 5106 5107 void 5108 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5109 { 5110 struct rwlock *lock; 5111 vm_offset_t va_next; 5112 pml1_entry_t *l1e; 5113 pml2_entry_t *l2e; 5114 pml3_entry_t ptpaddr, *l3e; 5115 struct spglist free; 5116 bool anyvalid; 5117 5118 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5119 5120 /* 5121 * Perform an unsynchronized read. This is, however, safe. 5122 */ 5123 if (pmap->pm_stats.resident_count == 0) 5124 return; 5125 5126 anyvalid = false; 5127 SLIST_INIT(&free); 5128 5129 /* XXX something fishy here */ 5130 sva = (sva + PAGE_MASK) & ~PAGE_MASK; 5131 eva = (eva + PAGE_MASK) & ~PAGE_MASK; 5132 5133 PMAP_LOCK(pmap); 5134 5135 /* 5136 * special handling of removing one page. a very 5137 * common operation and easy to short circuit some 5138 * code. 5139 */ 5140 if (sva + PAGE_SIZE == eva) { 5141 l3e = pmap_pml3e(pmap, sva); 5142 if (l3e && (*l3e & RPTE_LEAF) == 0) { 5143 anyvalid = pmap_remove_page(pmap, sva, l3e, &free); 5144 goto out; 5145 } 5146 } 5147 5148 lock = NULL; 5149 for (; sva < eva; sva = va_next) { 5150 5151 if (pmap->pm_stats.resident_count == 0) 5152 break; 5153 l1e = pmap_pml1e(pmap, sva); 5154 if (l1e == NULL || (*l1e & PG_V) == 0) { 5155 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5156 if (va_next < sva) 5157 va_next = eva; 5158 continue; 5159 } 5160 5161 l2e = pmap_l1e_to_l2e(l1e, sva); 5162 if (l2e == NULL || (*l2e & PG_V) == 0) { 5163 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5164 if (va_next < sva) 5165 va_next = eva; 5166 continue; 5167 } 5168 5169 /* 5170 * Calculate index for next page table. 5171 */ 5172 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5173 if (va_next < sva) 5174 va_next = eva; 5175 5176 l3e = pmap_l2e_to_l3e(l2e, sva); 5177 ptpaddr = *l3e; 5178 5179 /* 5180 * Weed out invalid mappings. 5181 */ 5182 if (ptpaddr == 0) 5183 continue; 5184 5185 /* 5186 * Check for large page. 5187 */ 5188 if ((ptpaddr & RPTE_LEAF) != 0) { 5189 /* 5190 * Are we removing the entire large page? If not, 5191 * demote the mapping and fall through. 5192 */ 5193 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5194 pmap_remove_l3e(pmap, l3e, sva, &free, &lock); 5195 continue; 5196 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva, 5197 &lock)) { 5198 /* The large page mapping was destroyed. */ 5199 continue; 5200 } else 5201 ptpaddr = *l3e; 5202 } 5203 5204 /* 5205 * Limit our scan to either the end of the va represented 5206 * by the current page table page, or to the end of the 5207 * range being removed. 5208 */ 5209 if (va_next > eva) 5210 va_next = eva; 5211 5212 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock)) 5213 anyvalid = true; 5214 } 5215 if (lock != NULL) 5216 rw_wunlock(lock); 5217 out: 5218 if (anyvalid) 5219 pmap_invalidate_all(pmap); 5220 PMAP_UNLOCK(pmap); 5221 vm_page_free_pages_toq(&free, true); 5222 } 5223 5224 void 5225 mmu_radix_remove_all(vm_page_t m) 5226 { 5227 struct md_page *pvh; 5228 pv_entry_t pv; 5229 pmap_t pmap; 5230 struct rwlock *lock; 5231 pt_entry_t *pte, tpte; 5232 pml3_entry_t *l3e; 5233 vm_offset_t va; 5234 struct spglist free; 5235 int pvh_gen, md_gen; 5236 5237 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5238 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5239 ("pmap_remove_all: page %p is not managed", m)); 5240 SLIST_INIT(&free); 5241 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5242 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5243 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5244 retry: 5245 rw_wlock(lock); 5246 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 5247 pmap = PV_PMAP(pv); 5248 if (!PMAP_TRYLOCK(pmap)) { 5249 pvh_gen = pvh->pv_gen; 5250 rw_wunlock(lock); 5251 PMAP_LOCK(pmap); 5252 rw_wlock(lock); 5253 if (pvh_gen != pvh->pv_gen) { 5254 rw_wunlock(lock); 5255 PMAP_UNLOCK(pmap); 5256 goto retry; 5257 } 5258 } 5259 va = pv->pv_va; 5260 l3e = pmap_pml3e(pmap, va); 5261 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock); 5262 PMAP_UNLOCK(pmap); 5263 } 5264 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 5265 pmap = PV_PMAP(pv); 5266 if (!PMAP_TRYLOCK(pmap)) { 5267 pvh_gen = pvh->pv_gen; 5268 md_gen = m->md.pv_gen; 5269 rw_wunlock(lock); 5270 PMAP_LOCK(pmap); 5271 rw_wlock(lock); 5272 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 5273 rw_wunlock(lock); 5274 PMAP_UNLOCK(pmap); 5275 goto retry; 5276 } 5277 } 5278 pmap_resident_count_dec(pmap, 1); 5279 l3e = pmap_pml3e(pmap, pv->pv_va); 5280 KASSERT((*l3e & RPTE_LEAF) == 0, ("pmap_remove_all: found" 5281 " a 2mpage in page %p's pv list", m)); 5282 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5283 tpte = pte_load_clear(pte); 5284 if (tpte & PG_W) 5285 pmap->pm_stats.wired_count--; 5286 if (tpte & PG_A) 5287 vm_page_aflag_set(m, PGA_REFERENCED); 5288 5289 /* 5290 * Update the vm_page_t clean and reference bits. 5291 */ 5292 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5293 vm_page_dirty(m); 5294 pmap_unuse_pt(pmap, pv->pv_va, *l3e, &free); 5295 pmap_invalidate_page(pmap, pv->pv_va); 5296 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5297 m->md.pv_gen++; 5298 free_pv_entry(pmap, pv); 5299 PMAP_UNLOCK(pmap); 5300 } 5301 vm_page_aflag_clear(m, PGA_WRITEABLE); 5302 rw_wunlock(lock); 5303 vm_page_free_pages_toq(&free, true); 5304 } 5305 5306 /* 5307 * Destroy all managed, non-wired mappings in the given user-space 5308 * pmap. This pmap cannot be active on any processor besides the 5309 * caller. 5310 * 5311 * This function cannot be applied to the kernel pmap. Moreover, it 5312 * is not intended for general use. It is only to be used during 5313 * process termination. Consequently, it can be implemented in ways 5314 * that make it faster than pmap_remove(). First, it can more quickly 5315 * destroy mappings by iterating over the pmap's collection of PV 5316 * entries, rather than searching the page table. Second, it doesn't 5317 * have to test and clear the page table entries atomically, because 5318 * no processor is currently accessing the user address space. In 5319 * particular, a page table entry's dirty bit won't change state once 5320 * this function starts. 5321 * 5322 * Although this function destroys all of the pmap's managed, 5323 * non-wired mappings, it can delay and batch the invalidation of TLB 5324 * entries without calling pmap_delayed_invl_started() and 5325 * pmap_delayed_invl_finished(). Because the pmap is not active on 5326 * any other processor, none of these TLB entries will ever be used 5327 * before their eventual invalidation. Consequently, there is no need 5328 * for either pmap_remove_all() or pmap_remove_write() to wait for 5329 * that eventual TLB invalidation. 5330 */ 5331 5332 void 5333 mmu_radix_remove_pages(pmap_t pmap) 5334 { 5335 5336 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 5337 pml3_entry_t ptel3e; 5338 pt_entry_t *pte, tpte; 5339 struct spglist free; 5340 vm_page_t m, mpte, mt; 5341 pv_entry_t pv; 5342 struct md_page *pvh; 5343 struct pv_chunk *pc, *npc; 5344 struct rwlock *lock; 5345 int64_t bit; 5346 uint64_t inuse, bitmask; 5347 int allfree, field, freed, idx; 5348 boolean_t superpage; 5349 vm_paddr_t pa; 5350 5351 /* 5352 * Assert that the given pmap is only active on the current 5353 * CPU. Unfortunately, we cannot block another CPU from 5354 * activating the pmap while this function is executing. 5355 */ 5356 KASSERT(pmap->pm_pid == mfspr(SPR_PID), 5357 ("non-current asid %lu - expected %lu", pmap->pm_pid, 5358 mfspr(SPR_PID))); 5359 5360 lock = NULL; 5361 5362 SLIST_INIT(&free); 5363 PMAP_LOCK(pmap); 5364 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 5365 allfree = 1; 5366 freed = 0; 5367 for (field = 0; field < _NPCM; field++) { 5368 inuse = ~pc->pc_map[field] & pc_freemask[field]; 5369 while (inuse != 0) { 5370 bit = cnttzd(inuse); 5371 bitmask = 1UL << bit; 5372 idx = field * 64 + bit; 5373 pv = &pc->pc_pventry[idx]; 5374 inuse &= ~bitmask; 5375 5376 pte = pmap_pml2e(pmap, pv->pv_va); 5377 ptel3e = *pte; 5378 pte = pmap_l2e_to_l3e(pte, pv->pv_va); 5379 tpte = *pte; 5380 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) { 5381 superpage = FALSE; 5382 ptel3e = tpte; 5383 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & 5384 PG_FRAME); 5385 pte = &pte[pmap_pte_index(pv->pv_va)]; 5386 tpte = *pte; 5387 } else { 5388 /* 5389 * Keep track whether 'tpte' is a 5390 * superpage explicitly instead of 5391 * relying on RPTE_LEAF being set. 5392 * 5393 * This is because RPTE_LEAF is numerically 5394 * identical to PG_PTE_PAT and thus a 5395 * regular page could be mistaken for 5396 * a superpage. 5397 */ 5398 superpage = TRUE; 5399 } 5400 5401 if ((tpte & PG_V) == 0) { 5402 panic("bad pte va %lx pte %lx", 5403 pv->pv_va, tpte); 5404 } 5405 5406 /* 5407 * We cannot remove wired pages from a process' mapping at this time 5408 */ 5409 if (tpte & PG_W) { 5410 allfree = 0; 5411 continue; 5412 } 5413 5414 if (superpage) 5415 pa = tpte & PG_PS_FRAME; 5416 else 5417 pa = tpte & PG_FRAME; 5418 5419 m = PHYS_TO_VM_PAGE(pa); 5420 KASSERT(m->phys_addr == pa, 5421 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 5422 m, (uintmax_t)m->phys_addr, 5423 (uintmax_t)tpte)); 5424 5425 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 5426 m < &vm_page_array[vm_page_array_size], 5427 ("pmap_remove_pages: bad tpte %#jx", 5428 (uintmax_t)tpte)); 5429 5430 pte_clear(pte); 5431 5432 /* 5433 * Update the vm_page_t clean/reference bits. 5434 */ 5435 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 5436 if (superpage) { 5437 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5438 vm_page_dirty(mt); 5439 } else 5440 vm_page_dirty(m); 5441 } 5442 5443 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 5444 5445 /* Mark free */ 5446 pc->pc_map[field] |= bitmask; 5447 if (superpage) { 5448 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5449 pvh = pa_to_pvh(tpte & PG_PS_FRAME); 5450 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 5451 pvh->pv_gen++; 5452 if (TAILQ_EMPTY(&pvh->pv_list)) { 5453 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5454 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 5455 TAILQ_EMPTY(&mt->md.pv_list)) 5456 vm_page_aflag_clear(mt, PGA_WRITEABLE); 5457 } 5458 mpte = pmap_remove_pt_page(pmap, pv->pv_va); 5459 if (mpte != NULL) { 5460 pmap_resident_count_dec(pmap, 1); 5461 KASSERT(mpte->ref_count == NPTEPG, 5462 ("pmap_remove_pages: pte page wire count error")); 5463 mpte->ref_count = 0; 5464 pmap_add_delayed_free_list(mpte, &free, FALSE); 5465 } 5466 } else { 5467 pmap_resident_count_dec(pmap, 1); 5468 #ifdef VERBOSE_PV 5469 printf("freeing pv (%p, %p)\n", 5470 pmap, pv); 5471 #endif 5472 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5473 m->md.pv_gen++; 5474 if ((m->a.flags & PGA_WRITEABLE) != 0 && 5475 TAILQ_EMPTY(&m->md.pv_list) && 5476 (m->flags & PG_FICTITIOUS) == 0) { 5477 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5478 if (TAILQ_EMPTY(&pvh->pv_list)) 5479 vm_page_aflag_clear(m, PGA_WRITEABLE); 5480 } 5481 } 5482 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free); 5483 freed++; 5484 } 5485 } 5486 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 5487 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 5488 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 5489 if (allfree) { 5490 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5491 free_pv_chunk(pc); 5492 } 5493 } 5494 if (lock != NULL) 5495 rw_wunlock(lock); 5496 pmap_invalidate_all(pmap); 5497 PMAP_UNLOCK(pmap); 5498 vm_page_free_pages_toq(&free, true); 5499 } 5500 5501 void 5502 mmu_radix_remove_write(vm_page_t m) 5503 { 5504 struct md_page *pvh; 5505 pmap_t pmap; 5506 struct rwlock *lock; 5507 pv_entry_t next_pv, pv; 5508 pml3_entry_t *l3e; 5509 pt_entry_t oldpte, *pte; 5510 int pvh_gen, md_gen; 5511 5512 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5513 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5514 ("pmap_remove_write: page %p is not managed", m)); 5515 vm_page_assert_busied(m); 5516 5517 if (!pmap_page_is_write_mapped(m)) 5518 return; 5519 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5520 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5521 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5522 retry_pv_loop: 5523 rw_wlock(lock); 5524 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 5525 pmap = PV_PMAP(pv); 5526 if (!PMAP_TRYLOCK(pmap)) { 5527 pvh_gen = pvh->pv_gen; 5528 rw_wunlock(lock); 5529 PMAP_LOCK(pmap); 5530 rw_wlock(lock); 5531 if (pvh_gen != pvh->pv_gen) { 5532 PMAP_UNLOCK(pmap); 5533 rw_wunlock(lock); 5534 goto retry_pv_loop; 5535 } 5536 } 5537 l3e = pmap_pml3e(pmap, pv->pv_va); 5538 if ((*l3e & PG_RW) != 0) 5539 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock); 5540 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 5541 ("inconsistent pv lock %p %p for page %p", 5542 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 5543 PMAP_UNLOCK(pmap); 5544 } 5545 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 5546 pmap = PV_PMAP(pv); 5547 if (!PMAP_TRYLOCK(pmap)) { 5548 pvh_gen = pvh->pv_gen; 5549 md_gen = m->md.pv_gen; 5550 rw_wunlock(lock); 5551 PMAP_LOCK(pmap); 5552 rw_wlock(lock); 5553 if (pvh_gen != pvh->pv_gen || 5554 md_gen != m->md.pv_gen) { 5555 PMAP_UNLOCK(pmap); 5556 rw_wunlock(lock); 5557 goto retry_pv_loop; 5558 } 5559 } 5560 l3e = pmap_pml3e(pmap, pv->pv_va); 5561 KASSERT((*l3e & RPTE_LEAF) == 0, 5562 ("pmap_remove_write: found a 2mpage in page %p's pv list", 5563 m)); 5564 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5565 retry: 5566 oldpte = *pte; 5567 if (oldpte & PG_RW) { 5568 if (!atomic_cmpset_long(pte, oldpte, 5569 (oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M))) 5570 goto retry; 5571 if ((oldpte & PG_M) != 0) 5572 vm_page_dirty(m); 5573 pmap_invalidate_page(pmap, pv->pv_va); 5574 } 5575 PMAP_UNLOCK(pmap); 5576 } 5577 rw_wunlock(lock); 5578 vm_page_aflag_clear(m, PGA_WRITEABLE); 5579 } 5580 5581 /* 5582 * Clear the wired attribute from the mappings for the specified range of 5583 * addresses in the given pmap. Every valid mapping within that range 5584 * must have the wired attribute set. In contrast, invalid mappings 5585 * cannot have the wired attribute set, so they are ignored. 5586 * 5587 * The wired attribute of the page table entry is not a hardware 5588 * feature, so there is no need to invalidate any TLB entries. 5589 * Since pmap_demote_l3e() for the wired entry must never fail, 5590 * pmap_delayed_invl_started()/finished() calls around the 5591 * function are not needed. 5592 */ 5593 void 5594 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5595 { 5596 vm_offset_t va_next; 5597 pml1_entry_t *l1e; 5598 pml2_entry_t *l2e; 5599 pml3_entry_t *l3e; 5600 pt_entry_t *pte; 5601 5602 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5603 PMAP_LOCK(pmap); 5604 for (; sva < eva; sva = va_next) { 5605 l1e = pmap_pml1e(pmap, sva); 5606 if ((*l1e & PG_V) == 0) { 5607 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5608 if (va_next < sva) 5609 va_next = eva; 5610 continue; 5611 } 5612 l2e = pmap_l1e_to_l2e(l1e, sva); 5613 if ((*l2e & PG_V) == 0) { 5614 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5615 if (va_next < sva) 5616 va_next = eva; 5617 continue; 5618 } 5619 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5620 if (va_next < sva) 5621 va_next = eva; 5622 l3e = pmap_l2e_to_l3e(l2e, sva); 5623 if ((*l3e & PG_V) == 0) 5624 continue; 5625 if ((*l3e & RPTE_LEAF) != 0) { 5626 if ((*l3e & PG_W) == 0) 5627 panic("pmap_unwire: pde %#jx is missing PG_W", 5628 (uintmax_t)*l3e); 5629 5630 /* 5631 * Are we unwiring the entire large page? If not, 5632 * demote the mapping and fall through. 5633 */ 5634 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5635 atomic_clear_long(l3e, PG_W); 5636 pmap->pm_stats.wired_count -= L3_PAGE_SIZE / 5637 PAGE_SIZE; 5638 continue; 5639 } else if (!pmap_demote_l3e(pmap, l3e, sva)) 5640 panic("pmap_unwire: demotion failed"); 5641 } 5642 if (va_next > eva) 5643 va_next = eva; 5644 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 5645 sva += PAGE_SIZE) { 5646 MPASS(pte == pmap_pte(pmap, sva)); 5647 if ((*pte & PG_V) == 0) 5648 continue; 5649 if ((*pte & PG_W) == 0) 5650 panic("pmap_unwire: pte %#jx is missing PG_W", 5651 (uintmax_t)*pte); 5652 5653 /* 5654 * PG_W must be cleared atomically. Although the pmap 5655 * lock synchronizes access to PG_W, another processor 5656 * could be setting PG_M and/or PG_A concurrently. 5657 */ 5658 atomic_clear_long(pte, PG_W); 5659 pmap->pm_stats.wired_count--; 5660 } 5661 } 5662 PMAP_UNLOCK(pmap); 5663 } 5664 5665 void 5666 mmu_radix_zero_page(vm_page_t m) 5667 { 5668 vm_offset_t addr; 5669 5670 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5671 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5672 pagezero(addr); 5673 } 5674 5675 void 5676 mmu_radix_zero_page_area(vm_page_t m, int off, int size) 5677 { 5678 caddr_t addr; 5679 5680 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size); 5681 MPASS(off + size <= PAGE_SIZE); 5682 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5683 memset(addr + off, 0, size); 5684 } 5685 5686 5687 5688 5689 static int 5690 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 5691 { 5692 pml3_entry_t *l3ep; 5693 pt_entry_t pte; 5694 vm_paddr_t pa; 5695 int val; 5696 5697 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 5698 PMAP_LOCK(pmap); 5699 5700 l3ep = pmap_pml3e(pmap, addr); 5701 if (l3ep != NULL && (*l3ep & PG_V)) { 5702 if (*l3ep & RPTE_LEAF) { 5703 pte = *l3ep; 5704 /* Compute the physical address of the 4KB page. */ 5705 pa = ((*l3ep & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) & 5706 PG_FRAME; 5707 val = MINCORE_SUPER; 5708 } else { 5709 pte = *pmap_l3e_to_pte(l3ep, addr); 5710 pa = pte & PG_FRAME; 5711 val = 0; 5712 } 5713 } else { 5714 pte = 0; 5715 pa = 0; 5716 val = 0; 5717 } 5718 if ((pte & PG_V) != 0) { 5719 val |= MINCORE_INCORE; 5720 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5721 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 5722 if ((pte & PG_A) != 0) 5723 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 5724 } 5725 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 5726 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 5727 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { 5728 *locked_pa = pa; 5729 } 5730 PMAP_UNLOCK(pmap); 5731 return (val); 5732 } 5733 5734 void 5735 mmu_radix_activate(struct thread *td) 5736 { 5737 pmap_t pmap; 5738 uint32_t curpid; 5739 5740 CTR2(KTR_PMAP, "%s(%p)", __func__, td); 5741 critical_enter(); 5742 pmap = vmspace_pmap(td->td_proc->p_vmspace); 5743 curpid = mfspr(SPR_PID); 5744 if (pmap->pm_pid > isa3_base_pid && 5745 curpid != pmap->pm_pid) { 5746 mmu_radix_pid_set(pmap); 5747 } 5748 critical_exit(); 5749 } 5750 5751 /* 5752 * Increase the starting virtual address of the given mapping if a 5753 * different alignment might result in more superpage mappings. 5754 */ 5755 void 5756 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset, 5757 vm_offset_t *addr, vm_size_t size) 5758 { 5759 5760 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr, 5761 size); 5762 vm_offset_t superpage_offset; 5763 5764 if (size < L3_PAGE_SIZE) 5765 return; 5766 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 5767 offset += ptoa(object->pg_color); 5768 superpage_offset = offset & L3_PAGE_MASK; 5769 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE || 5770 (*addr & L3_PAGE_MASK) == superpage_offset) 5771 return; 5772 if ((*addr & L3_PAGE_MASK) < superpage_offset) 5773 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset; 5774 else 5775 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset; 5776 } 5777 5778 static void * 5779 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr) 5780 { 5781 vm_offset_t va, tmpva, ppa, offset; 5782 5783 ppa = trunc_page(pa); 5784 offset = pa & PAGE_MASK; 5785 size = roundup2(offset + size, PAGE_SIZE); 5786 if (pa < powerpc_ptob(Maxmem)) 5787 panic("bad pa: %#lx less than Maxmem %#lx\n", 5788 pa, powerpc_ptob(Maxmem)); 5789 va = kva_alloc(size); 5790 if (bootverbose) 5791 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr); 5792 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr)); 5793 5794 if (!va) 5795 panic("%s: Couldn't alloc kernel virtual memory", __func__); 5796 5797 for (tmpva = va; size > 0;) { 5798 mmu_radix_kenter_attr(tmpva, ppa, attr); 5799 size -= PAGE_SIZE; 5800 tmpva += PAGE_SIZE; 5801 ppa += PAGE_SIZE; 5802 } 5803 ptesync(); 5804 5805 return ((void *)(va + offset)); 5806 } 5807 5808 static void * 5809 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size) 5810 { 5811 5812 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 5813 5814 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT)); 5815 } 5816 5817 void 5818 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma) 5819 { 5820 5821 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma); 5822 m->md.mdpg_cache_attrs = ma; 5823 5824 /* 5825 * If "m" is a normal page, update its direct mapping. This update 5826 * can be relied upon to perform any cache operations that are 5827 * required for data coherence. 5828 */ 5829 if ((m->flags & PG_FICTITIOUS) == 0 && 5830 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 5831 PAGE_SIZE, m->md.mdpg_cache_attrs)) 5832 panic("memory attribute change on the direct map failed"); 5833 } 5834 5835 static void 5836 mmu_radix_unmapdev(vm_offset_t va, vm_size_t size) 5837 { 5838 vm_offset_t offset; 5839 5840 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size); 5841 /* If we gave a direct map region in pmap_mapdev, do nothing */ 5842 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) 5843 return; 5844 5845 offset = va & PAGE_MASK; 5846 size = round_page(offset + size); 5847 va = trunc_page(va); 5848 5849 if (pmap_initialized) 5850 kva_free(va, size); 5851 } 5852 5853 static __inline void 5854 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask) 5855 { 5856 uint64_t opte, npte; 5857 5858 /* 5859 * The cache mode bits are all in the low 32-bits of the 5860 * PTE, so we can just spin on updating the low 32-bits. 5861 */ 5862 do { 5863 opte = *pte; 5864 npte = opte & ~mask; 5865 npte |= cache_bits; 5866 } while (npte != opte && !atomic_cmpset_long(pte, opte, npte)); 5867 } 5868 5869 /* 5870 * Tries to demote a 1GB page mapping. 5871 */ 5872 static boolean_t 5873 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) 5874 { 5875 pml2_entry_t oldpdpe; 5876 pml3_entry_t *firstpde, newpde, *pde; 5877 vm_paddr_t pdpgpa; 5878 vm_page_t pdpg; 5879 5880 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5881 oldpdpe = *l2e; 5882 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 5883 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); 5884 pdpg = vm_page_alloc(NULL, va >> L2_PAGE_SIZE_SHIFT, 5885 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); 5886 if (pdpg == NULL) { 5887 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" 5888 " in pmap %p", va, pmap); 5889 return (FALSE); 5890 } 5891 pdpgpa = VM_PAGE_TO_PHYS(pdpg); 5892 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); 5893 KASSERT((oldpdpe & PG_A) != 0, 5894 ("pmap_demote_pdpe: oldpdpe is missing PG_A")); 5895 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, 5896 ("pmap_demote_pdpe: oldpdpe is missing PG_M")); 5897 newpde = oldpdpe; 5898 5899 /* 5900 * Initialize the page directory page. 5901 */ 5902 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { 5903 *pde = newpde; 5904 newpde += L3_PAGE_SIZE; 5905 } 5906 5907 /* 5908 * Demote the mapping. 5909 */ 5910 pde_store(l2e, pdpgpa); 5911 5912 /* 5913 * Flush PWC --- XXX revisit 5914 */ 5915 pmap_invalidate_all(pmap); 5916 5917 pmap_l2e_demotions++; 5918 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" 5919 " in pmap %p", va, pmap); 5920 return (TRUE); 5921 } 5922 5923 vm_paddr_t 5924 mmu_radix_kextract(vm_offset_t va) 5925 { 5926 pml3_entry_t l3e; 5927 vm_paddr_t pa; 5928 5929 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 5930 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { 5931 pa = DMAP_TO_PHYS(va); 5932 } else { 5933 l3e = *pmap_pml3e(kernel_pmap, va); 5934 if (l3e & RPTE_LEAF) { 5935 pa = (l3e & PG_PS_FRAME) | (va & L3_PAGE_MASK); 5936 pa |= (va & L3_PAGE_MASK); 5937 } else { 5938 /* 5939 * Beware of a concurrent promotion that changes the 5940 * PDE at this point! For example, vtopte() must not 5941 * be used to access the PTE because it would use the 5942 * new PDE. It is, however, safe to use the old PDE 5943 * because the page table page is preserved by the 5944 * promotion. 5945 */ 5946 pa = *pmap_l3e_to_pte(&l3e, va); 5947 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 5948 pa |= (va & PAGE_MASK); 5949 } 5950 } 5951 return (pa); 5952 } 5953 5954 static pt_entry_t 5955 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 5956 { 5957 5958 if (ma != VM_MEMATTR_DEFAULT) { 5959 return pmap_cache_bits(ma); 5960 } 5961 5962 /* 5963 * Assume the page is cache inhibited and access is guarded unless 5964 * it's in our available memory array. 5965 */ 5966 for (int i = 0; i < pregions_sz; i++) { 5967 if ((pa >= pregions[i].mr_start) && 5968 (pa < (pregions[i].mr_start + pregions[i].mr_size))) 5969 return (RPTE_ATTR_MEM); 5970 } 5971 return (RPTE_ATTR_GUARDEDIO); 5972 } 5973 5974 static void 5975 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 5976 { 5977 pt_entry_t *pte, pteval; 5978 uint64_t cache_bits; 5979 5980 pte = kvtopte(va); 5981 MPASS(pte != NULL); 5982 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 5983 cache_bits = mmu_radix_calc_wimg(pa, ma); 5984 pte_store(pte, pteval | cache_bits); 5985 } 5986 5987 void 5988 mmu_radix_kremove(vm_offset_t va) 5989 { 5990 pt_entry_t *pte; 5991 5992 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 5993 5994 pte = kvtopte(va); 5995 pte_clear(pte); 5996 } 5997 5998 int 5999 mmu_radix_decode_kernel_ptr(vm_offset_t addr, 6000 int *is_user, vm_offset_t *decoded) 6001 { 6002 6003 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr); 6004 *decoded = addr; 6005 *is_user = (addr < VM_MAXUSER_ADDRESS); 6006 return (0); 6007 } 6008 6009 static boolean_t 6010 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size) 6011 { 6012 6013 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 6014 return (mem_valid(pa, size)); 6015 } 6016 6017 static void 6018 mmu_radix_scan_init() 6019 { 6020 6021 CTR1(KTR_PMAP, "%s()", __func__); 6022 UNIMPLEMENTED(); 6023 } 6024 6025 static void 6026 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, 6027 void **va) 6028 { 6029 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va); 6030 UNIMPLEMENTED(); 6031 } 6032 6033 vm_offset_t 6034 mmu_radix_quick_enter_page(vm_page_t m) 6035 { 6036 vm_paddr_t paddr; 6037 6038 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 6039 paddr = VM_PAGE_TO_PHYS(m); 6040 return (PHYS_TO_DMAP(paddr)); 6041 } 6042 6043 void 6044 mmu_radix_quick_remove_page(vm_offset_t addr __unused) 6045 { 6046 /* no work to do here */ 6047 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 6048 } 6049 6050 static void 6051 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 6052 { 6053 cpu_flush_dcache((void *)sva, eva - sva); 6054 } 6055 6056 int 6057 mmu_radix_change_attr(vm_offset_t va, vm_size_t size, 6058 vm_memattr_t mode) 6059 { 6060 int error; 6061 6062 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode); 6063 PMAP_LOCK(kernel_pmap); 6064 error = pmap_change_attr_locked(va, size, mode, true); 6065 PMAP_UNLOCK(kernel_pmap); 6066 return (error); 6067 } 6068 6069 static int 6070 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush) 6071 { 6072 vm_offset_t base, offset, tmpva; 6073 vm_paddr_t pa_start, pa_end, pa_end1; 6074 pml2_entry_t *l2e; 6075 pml3_entry_t *l3e; 6076 pt_entry_t *pte; 6077 int cache_bits, error; 6078 boolean_t changed; 6079 6080 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 6081 base = trunc_page(va); 6082 offset = va & PAGE_MASK; 6083 size = round_page(offset + size); 6084 6085 /* 6086 * Only supported on kernel virtual addresses, including the direct 6087 * map but excluding the recursive map. 6088 */ 6089 if (base < DMAP_MIN_ADDRESS) 6090 return (EINVAL); 6091 6092 cache_bits = pmap_cache_bits(mode); 6093 changed = FALSE; 6094 6095 /* 6096 * Pages that aren't mapped aren't supported. Also break down 2MB pages 6097 * into 4KB pages if required. 6098 */ 6099 for (tmpva = base; tmpva < base + size; ) { 6100 l2e = pmap_pml2e(kernel_pmap, tmpva); 6101 if (l2e == NULL || *l2e == 0) 6102 return (EINVAL); 6103 if (*l2e & RPTE_LEAF) { 6104 /* 6105 * If the current 1GB page already has the required 6106 * memory type, then we need not demote this page. Just 6107 * increment tmpva to the next 1GB page frame. 6108 */ 6109 if ((*l2e & RPTE_ATTR_MASK) == cache_bits) { 6110 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6111 continue; 6112 } 6113 6114 /* 6115 * If the current offset aligns with a 1GB page frame 6116 * and there is at least 1GB left within the range, then 6117 * we need not break down this page into 2MB pages. 6118 */ 6119 if ((tmpva & L2_PAGE_MASK) == 0 && 6120 tmpva + L2_PAGE_MASK < base + size) { 6121 tmpva += L2_PAGE_MASK; 6122 continue; 6123 } 6124 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva)) 6125 return (ENOMEM); 6126 } 6127 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6128 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n", 6129 tmpva, l2e)); 6130 if (*l3e == 0) 6131 return (EINVAL); 6132 if (*l3e & RPTE_LEAF) { 6133 /* 6134 * If the current 2MB page already has the required 6135 * memory type, then we need not demote this page. Just 6136 * increment tmpva to the next 2MB page frame. 6137 */ 6138 if ((*l3e & RPTE_ATTR_MASK) == cache_bits) { 6139 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6140 continue; 6141 } 6142 6143 /* 6144 * If the current offset aligns with a 2MB page frame 6145 * and there is at least 2MB left within the range, then 6146 * we need not break down this page into 4KB pages. 6147 */ 6148 if ((tmpva & L3_PAGE_MASK) == 0 && 6149 tmpva + L3_PAGE_MASK < base + size) { 6150 tmpva += L3_PAGE_SIZE; 6151 continue; 6152 } 6153 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva)) 6154 return (ENOMEM); 6155 } 6156 pte = pmap_l3e_to_pte(l3e, tmpva); 6157 if (*pte == 0) 6158 return (EINVAL); 6159 tmpva += PAGE_SIZE; 6160 } 6161 error = 0; 6162 6163 /* 6164 * Ok, all the pages exist, so run through them updating their 6165 * cache mode if required. 6166 */ 6167 pa_start = pa_end = 0; 6168 for (tmpva = base; tmpva < base + size; ) { 6169 l2e = pmap_pml2e(kernel_pmap, tmpva); 6170 if (*l2e & RPTE_LEAF) { 6171 if ((*l2e & RPTE_ATTR_MASK) != cache_bits) { 6172 pmap_pte_attr(l2e, cache_bits, 6173 RPTE_ATTR_MASK); 6174 changed = TRUE; 6175 } 6176 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6177 (*l2e & PG_PS_FRAME) < dmaplimit) { 6178 if (pa_start == pa_end) { 6179 /* Start physical address run. */ 6180 pa_start = *l2e & PG_PS_FRAME; 6181 pa_end = pa_start + L2_PAGE_SIZE; 6182 } else if (pa_end == (*l2e & PG_PS_FRAME)) 6183 pa_end += L2_PAGE_SIZE; 6184 else { 6185 /* Run ended, update direct map. */ 6186 error = pmap_change_attr_locked( 6187 PHYS_TO_DMAP(pa_start), 6188 pa_end - pa_start, mode, flush); 6189 if (error != 0) 6190 break; 6191 /* Start physical address run. */ 6192 pa_start = *l2e & PG_PS_FRAME; 6193 pa_end = pa_start + L2_PAGE_SIZE; 6194 } 6195 } 6196 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6197 continue; 6198 } 6199 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6200 if (*l3e & RPTE_LEAF) { 6201 if ((*l3e & RPTE_ATTR_MASK) != cache_bits) { 6202 pmap_pte_attr(l3e, cache_bits, 6203 RPTE_ATTR_MASK); 6204 changed = TRUE; 6205 } 6206 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6207 (*l3e & PG_PS_FRAME) < dmaplimit) { 6208 if (pa_start == pa_end) { 6209 /* Start physical address run. */ 6210 pa_start = *l3e & PG_PS_FRAME; 6211 pa_end = pa_start + L3_PAGE_SIZE; 6212 } else if (pa_end == (*l3e & PG_PS_FRAME)) 6213 pa_end += L3_PAGE_SIZE; 6214 else { 6215 /* Run ended, update direct map. */ 6216 error = pmap_change_attr_locked( 6217 PHYS_TO_DMAP(pa_start), 6218 pa_end - pa_start, mode, flush); 6219 if (error != 0) 6220 break; 6221 /* Start physical address run. */ 6222 pa_start = *l3e & PG_PS_FRAME; 6223 pa_end = pa_start + L3_PAGE_SIZE; 6224 } 6225 } 6226 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6227 } else { 6228 pte = pmap_l3e_to_pte(l3e, tmpva); 6229 if ((*pte & RPTE_ATTR_MASK) != cache_bits) { 6230 pmap_pte_attr(pte, cache_bits, 6231 RPTE_ATTR_MASK); 6232 changed = TRUE; 6233 } 6234 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6235 (*pte & PG_FRAME) < dmaplimit) { 6236 if (pa_start == pa_end) { 6237 /* Start physical address run. */ 6238 pa_start = *pte & PG_FRAME; 6239 pa_end = pa_start + PAGE_SIZE; 6240 } else if (pa_end == (*pte & PG_FRAME)) 6241 pa_end += PAGE_SIZE; 6242 else { 6243 /* Run ended, update direct map. */ 6244 error = pmap_change_attr_locked( 6245 PHYS_TO_DMAP(pa_start), 6246 pa_end - pa_start, mode, flush); 6247 if (error != 0) 6248 break; 6249 /* Start physical address run. */ 6250 pa_start = *pte & PG_FRAME; 6251 pa_end = pa_start + PAGE_SIZE; 6252 } 6253 } 6254 tmpva += PAGE_SIZE; 6255 } 6256 } 6257 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { 6258 pa_end1 = MIN(pa_end, dmaplimit); 6259 if (pa_start != pa_end1) 6260 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), 6261 pa_end1 - pa_start, mode, flush); 6262 } 6263 6264 /* 6265 * Flush CPU caches if required to make sure any data isn't cached that 6266 * shouldn't be, etc. 6267 */ 6268 if (changed) { 6269 pmap_invalidate_all(kernel_pmap); 6270 6271 if (flush) 6272 pmap_invalidate_cache_range(base, tmpva); 6273 6274 } 6275 return (error); 6276 } 6277 6278 /* 6279 * Allocate physical memory for the vm_page array and map it into KVA, 6280 * attempting to back the vm_pages with domain-local memory. 6281 */ 6282 void 6283 mmu_radix_page_array_startup(long pages) 6284 { 6285 #ifdef notyet 6286 pml2_entry_t *l2e; 6287 pml3_entry_t *pde; 6288 pml3_entry_t newl3; 6289 vm_offset_t va; 6290 long pfn; 6291 int domain, i; 6292 #endif 6293 vm_paddr_t pa; 6294 vm_offset_t start, end; 6295 6296 vm_page_array_size = pages; 6297 6298 start = VM_MIN_KERNEL_ADDRESS; 6299 end = start + pages * sizeof(struct vm_page); 6300 6301 pa = vm_phys_early_alloc(0, end - start); 6302 6303 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT); 6304 #ifdef notyet 6305 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */ 6306 for (va = start; va < end; va += L3_PAGE_SIZE) { 6307 pfn = first_page + (va - start) / sizeof(struct vm_page); 6308 domain = _vm_phys_domain(ptoa(pfn)); 6309 l2e = pmap_pml2e(kernel_pmap, va); 6310 if ((*l2e & PG_V) == 0) { 6311 pa = vm_phys_early_alloc(domain, PAGE_SIZE); 6312 dump_add_page(pa); 6313 pagezero(PHYS_TO_DMAP(pa)); 6314 pde_store(l2e, (pml2_entry_t)pa); 6315 } 6316 pde = pmap_l2e_to_l3e(l2e, va); 6317 if ((*pde & PG_V) != 0) 6318 panic("Unexpected pde %p", pde); 6319 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE); 6320 for (i = 0; i < NPDEPG; i++) 6321 dump_add_page(pa + i * PAGE_SIZE); 6322 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W); 6323 pte_store(pde, newl3); 6324 } 6325 #endif 6326 vm_page_array = (vm_page_t)start; 6327 } 6328 6329 #ifdef DDB 6330 #include <sys/kdb.h> 6331 #include <ddb/ddb.h> 6332 6333 static void 6334 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va) 6335 { 6336 pml1_entry_t *l1e; 6337 pml2_entry_t *l2e; 6338 pml3_entry_t *l3e; 6339 pt_entry_t *pte; 6340 6341 l1e = &l1[pmap_pml1e_index(va)]; 6342 db_printf("VA %#016lx l1e %#016lx", va, *l1e); 6343 if ((*l1e & PG_V) == 0) { 6344 db_printf("\n"); 6345 return; 6346 } 6347 l2e = pmap_l1e_to_l2e(l1e, va); 6348 db_printf(" l2e %#016lx", *l2e); 6349 if ((*l2e & PG_V) == 0 || (*l2e & RPTE_LEAF) != 0) { 6350 db_printf("\n"); 6351 return; 6352 } 6353 l3e = pmap_l2e_to_l3e(l2e, va); 6354 db_printf(" l3e %#016lx", *l3e); 6355 if ((*l3e & PG_V) == 0 || (*l3e & RPTE_LEAF) != 0) { 6356 db_printf("\n"); 6357 return; 6358 } 6359 pte = pmap_l3e_to_pte(l3e, va); 6360 db_printf(" pte %#016lx\n", *pte); 6361 } 6362 6363 void 6364 pmap_page_print_mappings(vm_page_t m) 6365 { 6366 pmap_t pmap; 6367 pv_entry_t pv; 6368 6369 db_printf("page %p(%lx)\n", m, m->phys_addr); 6370 /* need to elide locks if running in ddb */ 6371 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 6372 db_printf("pv: %p ", pv); 6373 db_printf("va: %#016lx ", pv->pv_va); 6374 pmap = PV_PMAP(pv); 6375 db_printf("pmap %p ", pmap); 6376 if (pmap != NULL) { 6377 db_printf("asid: %lu\n", pmap->pm_pid); 6378 pmap_pte_walk(pmap->pm_pml1, pv->pv_va); 6379 } 6380 } 6381 } 6382 6383 DB_SHOW_COMMAND(pte, pmap_print_pte) 6384 { 6385 vm_offset_t va; 6386 pmap_t pmap; 6387 6388 if (!have_addr) { 6389 db_printf("show pte addr\n"); 6390 return; 6391 } 6392 va = (vm_offset_t)addr; 6393 6394 if (va >= DMAP_MIN_ADDRESS) 6395 pmap = kernel_pmap; 6396 else if (kdb_thread != NULL) 6397 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); 6398 else 6399 pmap = vmspace_pmap(curthread->td_proc->p_vmspace); 6400 6401 pmap_pte_walk(pmap->pm_pml1, va); 6402 } 6403 6404 #endif 6405 6406