1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018 Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include "opt_platform.h" 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/kernel.h> 35 #include <sys/systm.h> 36 #include <sys/conf.h> 37 #include <sys/bitstring.h> 38 #include <sys/queue.h> 39 #include <sys/cpuset.h> 40 #include <sys/endian.h> 41 #include <sys/kerneldump.h> 42 #include <sys/ktr.h> 43 #include <sys/lock.h> 44 #include <sys/syslog.h> 45 #include <sys/msgbuf.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/mutex.h> 49 #include <sys/proc.h> 50 #include <sys/rwlock.h> 51 #include <sys/sched.h> 52 #include <sys/sysctl.h> 53 #include <sys/systm.h> 54 #include <sys/vmem.h> 55 #include <sys/vmmeter.h> 56 #include <sys/smp.h> 57 58 #include <sys/kdb.h> 59 60 #include <dev/ofw/openfirm.h> 61 62 #include <vm/vm.h> 63 #include <vm/pmap.h> 64 #include <vm/vm_param.h> 65 #include <vm/vm_kern.h> 66 #include <vm/vm_page.h> 67 #include <vm/vm_map.h> 68 #include <vm/vm_object.h> 69 #include <vm/vm_extern.h> 70 #include <vm/vm_pageout.h> 71 #include <vm/vm_phys.h> 72 #include <vm/vm_reserv.h> 73 #include <vm/vm_dumpset.h> 74 #include <vm/uma.h> 75 76 #include <machine/_inttypes.h> 77 #include <machine/cpu.h> 78 #include <machine/platform.h> 79 #include <machine/frame.h> 80 #include <machine/md_var.h> 81 #include <machine/psl.h> 82 #include <machine/bat.h> 83 #include <machine/hid.h> 84 #include <machine/pte.h> 85 #include <machine/sr.h> 86 #include <machine/trap.h> 87 #include <machine/mmuvar.h> 88 89 /* For pseries bit. */ 90 #include <powerpc/pseries/phyp-hvcall.h> 91 92 #ifdef INVARIANTS 93 #include <vm/uma_dbg.h> 94 #endif 95 96 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit)) 97 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) 98 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit)) 99 100 #include "opt_ddb.h" 101 102 #ifdef DDB 103 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va); 104 #endif 105 106 #define PG_W RPTE_WIRED 107 #define PG_V RPTE_VALID 108 #define PG_MANAGED RPTE_MANAGED 109 #define PG_PROMOTED RPTE_PROMOTED 110 #define PG_M RPTE_C 111 #define PG_A RPTE_R 112 #define PG_X RPTE_EAA_X 113 #define PG_RW RPTE_EAA_W 114 #define PG_PTE_CACHE RPTE_ATTR_MASK 115 116 #define RPTE_SHIFT 9 117 #define NLS_MASK ((1UL<<5)-1) 118 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT) 119 #define RPTE_MASK (RPTE_ENTRIES-1) 120 121 #define NLB_SHIFT 0 122 #define NLB_MASK (((1UL<<52)-1) << 8) 123 124 extern int nkpt; 125 extern caddr_t crashdumpmap; 126 127 #define RIC_FLUSH_TLB 0 128 #define RIC_FLUSH_PWC 1 129 #define RIC_FLUSH_ALL 2 130 131 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ 132 133 #define PPC_INST_TLBIE 0x7c000264 134 #define PPC_INST_TLBIEL 0x7c000224 135 #define PPC_INST_SLBIA 0x7c0003e4 136 137 #define ___PPC_RA(a) (((a) & 0x1f) << 16) 138 #define ___PPC_RB(b) (((b) & 0x1f) << 11) 139 #define ___PPC_RS(s) (((s) & 0x1f) << 21) 140 #define ___PPC_RT(t) ___PPC_RS(t) 141 #define ___PPC_R(r) (((r) & 0x1) << 16) 142 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17) 143 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18) 144 145 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \ 146 ((IH & 0x7) << 21)) 147 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \ 148 __XSTRING(.long PPC_INST_TLBIE | \ 149 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 150 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 151 ___PPC_R(r)) 152 153 #define PPC_TLBIEL(rb,rs,ric,prs,r) \ 154 __XSTRING(.long PPC_INST_TLBIEL | \ 155 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 156 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 157 ___PPC_R(r)) 158 159 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7) 160 161 static __inline void 162 ttusync(void) 163 { 164 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 165 } 166 167 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ 168 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ 169 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */ 170 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ 171 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ 172 173 #define TLBIE_ACTUAL_PAGE_MASK 0xe0 174 #define TLBIE_ACTUAL_PAGE_4K 0x00 175 #define TLBIE_ACTUAL_PAGE_64K 0xa0 176 #define TLBIE_ACTUAL_PAGE_2M 0x20 177 #define TLBIE_ACTUAL_PAGE_1G 0x40 178 179 #define TLBIE_PRS_PARTITION_SCOPE 0x0 180 #define TLBIE_PRS_PROCESS_SCOPE 0x1 181 182 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */ 183 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */ 184 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC, 185 * cached {proc, part}tab entries 186 */ 187 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only: 188 * Invalidate a range of translations 189 */ 190 191 static __always_inline void 192 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid, 193 vm_offset_t va, uint16_t ap) 194 { 195 uint64_t rb, rs; 196 197 MPASS((va & PAGE_MASK) == 0); 198 199 rs = ((uint64_t)pid << 32) | lpid; 200 rb = va | is | ap; 201 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : : 202 "r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory"); 203 } 204 205 static __inline void 206 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap) 207 { 208 209 __asm __volatile("ptesync" ::: "memory"); 210 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 211 TLBIEL_INVAL_PAGE, 0, 0, va, ap); 212 __asm __volatile("ptesync" ::: "memory"); 213 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 214 TLBIEL_INVAL_PAGE, pid, 0, va, ap); 215 } 216 217 static __inline void 218 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va) 219 { 220 221 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 222 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K); 223 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K); 224 } 225 226 static __inline void 227 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va) 228 { 229 230 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 231 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M); 232 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M); 233 } 234 235 static __inline void 236 radix_tlbie_invlpwc_user(uint32_t pid) 237 { 238 239 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 240 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 241 } 242 243 static __inline void 244 radix_tlbie_flush_user(uint32_t pid) 245 { 246 247 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 248 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 249 } 250 251 static __inline void 252 radix_tlbie_invlpg_kernel_4k(vm_offset_t va) 253 { 254 255 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 256 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K); 257 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K); 258 } 259 260 static __inline void 261 radix_tlbie_invlpg_kernel_2m(vm_offset_t va) 262 { 263 264 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 265 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M); 266 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M); 267 } 268 269 /* 1GB pages aren't currently supported. */ 270 static __inline __unused void 271 radix_tlbie_invlpg_kernel_1g(vm_offset_t va) 272 { 273 274 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 275 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G); 276 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G); 277 } 278 279 static __inline void 280 radix_tlbie_invlpwc_kernel(void) 281 { 282 283 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 284 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 285 } 286 287 static __inline void 288 radix_tlbie_flush_kernel(void) 289 { 290 291 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 292 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 293 } 294 295 static __inline vm_pindex_t 296 pmap_l3e_pindex(vm_offset_t va) 297 { 298 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT); 299 } 300 301 static __inline vm_pindex_t 302 pmap_pml3e_index(vm_offset_t va) 303 { 304 305 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK); 306 } 307 308 static __inline vm_pindex_t 309 pmap_pml2e_index(vm_offset_t va) 310 { 311 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK); 312 } 313 314 static __inline vm_pindex_t 315 pmap_pml1e_index(vm_offset_t va) 316 { 317 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT); 318 } 319 320 /* Return various clipped indexes for a given VA */ 321 static __inline vm_pindex_t 322 pmap_pte_index(vm_offset_t va) 323 { 324 325 return ((va >> PAGE_SHIFT) & RPTE_MASK); 326 } 327 328 /* Return a pointer to the PT slot that corresponds to a VA */ 329 static __inline pt_entry_t * 330 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va) 331 { 332 pt_entry_t *pte; 333 vm_paddr_t ptepa; 334 335 ptepa = (be64toh(*l3e) & NLB_MASK); 336 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa); 337 return (&pte[pmap_pte_index(va)]); 338 } 339 340 /* Return a pointer to the PD slot that corresponds to a VA */ 341 static __inline pt_entry_t * 342 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va) 343 { 344 pt_entry_t *l3e; 345 vm_paddr_t l3pa; 346 347 l3pa = (be64toh(*l2e) & NLB_MASK); 348 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa); 349 return (&l3e[pmap_pml3e_index(va)]); 350 } 351 352 /* Return a pointer to the PD slot that corresponds to a VA */ 353 static __inline pt_entry_t * 354 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va) 355 { 356 pt_entry_t *l2e; 357 vm_paddr_t l2pa; 358 359 l2pa = (be64toh(*l1e) & NLB_MASK); 360 361 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa); 362 return (&l2e[pmap_pml2e_index(va)]); 363 } 364 365 static __inline pml1_entry_t * 366 pmap_pml1e(pmap_t pmap, vm_offset_t va) 367 { 368 369 return (&pmap->pm_pml1[pmap_pml1e_index(va)]); 370 } 371 372 static pt_entry_t * 373 pmap_pml2e(pmap_t pmap, vm_offset_t va) 374 { 375 pt_entry_t *l1e; 376 377 l1e = pmap_pml1e(pmap, va); 378 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0) 379 return (NULL); 380 return (pmap_l1e_to_l2e(l1e, va)); 381 } 382 383 static __inline pt_entry_t * 384 pmap_pml3e(pmap_t pmap, vm_offset_t va) 385 { 386 pt_entry_t *l2e; 387 388 l2e = pmap_pml2e(pmap, va); 389 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0) 390 return (NULL); 391 return (pmap_l2e_to_l3e(l2e, va)); 392 } 393 394 static __inline pt_entry_t * 395 pmap_pte(pmap_t pmap, vm_offset_t va) 396 { 397 pt_entry_t *l3e; 398 399 l3e = pmap_pml3e(pmap, va); 400 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 401 return (NULL); 402 return (pmap_l3e_to_pte(l3e, va)); 403 } 404 405 int nkpt = 64; 406 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, 407 "Number of kernel page table pages allocated on bootup"); 408 409 vm_paddr_t dmaplimit; 410 411 SYSCTL_DECL(_vm_pmap); 412 413 #ifdef INVARIANTS 414 #define VERBOSE_PMAP 0 415 #define VERBOSE_PROTECT 0 416 static int pmap_logging; 417 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN, 418 &pmap_logging, 0, "verbose debug logging"); 419 #endif 420 421 static u_int64_t KPTphys; /* phys addr of kernel level 1 */ 422 423 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ 424 425 static vm_offset_t qframe = 0; 426 static struct mtx qframe_mtx; 427 428 void mmu_radix_activate(struct thread *); 429 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int); 430 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *, 431 vm_size_t); 432 void mmu_radix_clear_modify(vm_page_t); 433 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); 434 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *); 435 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); 436 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 437 vm_prot_t); 438 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 439 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va); 440 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t); 441 void mmu_radix_kenter(vm_offset_t, vm_paddr_t); 442 vm_paddr_t mmu_radix_kextract(vm_offset_t); 443 void mmu_radix_kremove(vm_offset_t); 444 boolean_t mmu_radix_is_modified(vm_page_t); 445 boolean_t mmu_radix_is_prefaultable(pmap_t, vm_offset_t); 446 boolean_t mmu_radix_is_referenced(vm_page_t); 447 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t, 448 vm_pindex_t, vm_size_t); 449 boolean_t mmu_radix_page_exists_quick(pmap_t, vm_page_t); 450 void mmu_radix_page_init(vm_page_t); 451 boolean_t mmu_radix_page_is_mapped(vm_page_t m); 452 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t); 453 int mmu_radix_page_wired_mappings(vm_page_t); 454 int mmu_radix_pinit(pmap_t); 455 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 456 bool mmu_radix_ps_enabled(pmap_t); 457 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int); 458 void mmu_radix_qremove(vm_offset_t, int); 459 vm_offset_t mmu_radix_quick_enter_page(vm_page_t); 460 void mmu_radix_quick_remove_page(vm_offset_t); 461 boolean_t mmu_radix_ts_referenced(vm_page_t); 462 void mmu_radix_release(pmap_t); 463 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t); 464 void mmu_radix_remove_all(vm_page_t); 465 void mmu_radix_remove_pages(pmap_t); 466 void mmu_radix_remove_write(vm_page_t); 467 void mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz); 468 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t); 469 void mmu_radix_zero_page(vm_page_t); 470 void mmu_radix_zero_page_area(vm_page_t, int, int); 471 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); 472 void mmu_radix_page_array_startup(long pages); 473 474 #include "mmu_oea64.h" 475 476 /* 477 * Kernel MMU interface 478 */ 479 480 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t); 481 482 static void mmu_radix_copy_page(vm_page_t, vm_page_t); 483 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 484 vm_page_t *mb, vm_offset_t b_offset, int xfersize); 485 static void mmu_radix_growkernel(vm_offset_t); 486 static void mmu_radix_init(void); 487 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *); 488 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 489 static void mmu_radix_pinit0(pmap_t); 490 491 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t); 492 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); 493 static void mmu_radix_unmapdev(void *, vm_size_t); 494 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma); 495 static boolean_t mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t); 496 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va); 497 static void mmu_radix_scan_init(void); 498 static void mmu_radix_cpu_bootstrap(int ap); 499 static void mmu_radix_tlbie_all(void); 500 501 static struct pmap_funcs mmu_radix_methods = { 502 .bootstrap = mmu_radix_bootstrap, 503 .copy_page = mmu_radix_copy_page, 504 .copy_pages = mmu_radix_copy_pages, 505 .cpu_bootstrap = mmu_radix_cpu_bootstrap, 506 .growkernel = mmu_radix_growkernel, 507 .init = mmu_radix_init, 508 .map = mmu_radix_map, 509 .mincore = mmu_radix_mincore, 510 .pinit = mmu_radix_pinit, 511 .pinit0 = mmu_radix_pinit0, 512 513 .mapdev = mmu_radix_mapdev, 514 .mapdev_attr = mmu_radix_mapdev_attr, 515 .unmapdev = mmu_radix_unmapdev, 516 .kenter_attr = mmu_radix_kenter_attr, 517 .dev_direct_mapped = mmu_radix_dev_direct_mapped, 518 .dumpsys_pa_init = mmu_radix_scan_init, 519 .dumpsys_map_chunk = mmu_radix_dumpsys_map, 520 .page_is_mapped = mmu_radix_page_is_mapped, 521 .ps_enabled = mmu_radix_ps_enabled, 522 .align_superpage = mmu_radix_align_superpage, 523 .object_init_pt = mmu_radix_object_init_pt, 524 .protect = mmu_radix_protect, 525 /* pmap dispatcher interface */ 526 .clear_modify = mmu_radix_clear_modify, 527 .copy = mmu_radix_copy, 528 .enter = mmu_radix_enter, 529 .enter_object = mmu_radix_enter_object, 530 .enter_quick = mmu_radix_enter_quick, 531 .extract = mmu_radix_extract, 532 .extract_and_hold = mmu_radix_extract_and_hold, 533 .is_modified = mmu_radix_is_modified, 534 .is_prefaultable = mmu_radix_is_prefaultable, 535 .is_referenced = mmu_radix_is_referenced, 536 .ts_referenced = mmu_radix_ts_referenced, 537 .page_exists_quick = mmu_radix_page_exists_quick, 538 .page_init = mmu_radix_page_init, 539 .page_wired_mappings = mmu_radix_page_wired_mappings, 540 .qenter = mmu_radix_qenter, 541 .qremove = mmu_radix_qremove, 542 .release = mmu_radix_release, 543 .remove = mmu_radix_remove, 544 .remove_all = mmu_radix_remove_all, 545 .remove_write = mmu_radix_remove_write, 546 .sync_icache = mmu_radix_sync_icache, 547 .unwire = mmu_radix_unwire, 548 .zero_page = mmu_radix_zero_page, 549 .zero_page_area = mmu_radix_zero_page_area, 550 .activate = mmu_radix_activate, 551 .quick_enter_page = mmu_radix_quick_enter_page, 552 .quick_remove_page = mmu_radix_quick_remove_page, 553 .page_set_memattr = mmu_radix_page_set_memattr, 554 .page_array_startup = mmu_radix_page_array_startup, 555 556 /* Internal interfaces */ 557 .kenter = mmu_radix_kenter, 558 .kextract = mmu_radix_kextract, 559 .kremove = mmu_radix_kremove, 560 .change_attr = mmu_radix_change_attr, 561 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr, 562 563 .tlbie_all = mmu_radix_tlbie_all, 564 }; 565 566 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods); 567 568 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 569 struct rwlock **lockp); 570 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va); 571 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *); 572 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 573 struct spglist *free, struct rwlock **lockp); 574 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 575 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 576 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 577 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde, 578 struct spglist *free); 579 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 580 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp); 581 582 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e, 583 u_int flags, struct rwlock **lockp); 584 #if VM_NRESERVLEVEL > 0 585 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 586 struct rwlock **lockp); 587 #endif 588 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 589 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); 590 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 591 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate); 592 593 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, 594 vm_prot_t prot, struct rwlock **lockp); 595 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, 596 u_int flags, vm_page_t m, struct rwlock **lockp); 597 598 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 599 static void free_pv_chunk(struct pv_chunk *pc); 600 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); 601 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va, 602 struct rwlock **lockp); 603 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, 604 struct rwlock **lockp); 605 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, 606 struct spglist *free); 607 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); 608 609 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start); 610 static void pmap_invalidate_all(pmap_t pmap); 611 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush); 612 613 /* 614 * Internal flags for pmap_enter()'s helper functions. 615 */ 616 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 617 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 618 619 #define UNIMPLEMENTED() panic("%s not implemented", __func__) 620 #define UNTESTED() panic("%s not yet tested", __func__) 621 622 /* Number of supported PID bits */ 623 static unsigned int isa3_pid_bits; 624 625 /* PID to start allocating from */ 626 static unsigned int isa3_base_pid; 627 628 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4) 629 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits) 630 631 /* 632 * Map of physical memory regions. 633 */ 634 static struct mem_region *regions, *pregions; 635 static struct numa_mem_region *numa_pregions; 636 static u_int phys_avail_count; 637 static int regions_sz, pregions_sz, numa_pregions_sz; 638 static struct pate *isa3_parttab; 639 static struct prte *isa3_proctab; 640 static vmem_t *asid_arena; 641 642 extern void bs_remap_earlyboot(void); 643 644 #define RADIX_PGD_SIZE_SHIFT 16 645 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT) 646 647 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3) 648 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t)) 649 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t)) 650 651 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */ 652 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */ 653 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */ 654 655 /* POWER9 only permits a 64k partition table size. */ 656 #define PARTTAB_SIZE_SHIFT 16 657 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT) 658 659 #define PARTTAB_HR (1UL << 63) /* host uses radix */ 660 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */ 661 662 /* TLB flush actions. Used as argument to tlbiel_flush() */ 663 enum { 664 TLB_INVAL_SCOPE_LPID = 2, /* invalidate TLBs for current LPID */ 665 TLB_INVAL_SCOPE_GLOBAL = 3, /* invalidate all TLBs */ 666 }; 667 668 #define NPV_LIST_LOCKS MAXCPU 669 static int pmap_initialized; 670 static vm_paddr_t proctab0pa; 671 static vm_paddr_t parttab_phys; 672 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 673 674 /* 675 * Data for the pv entry allocation mechanism. 676 * Updates to pv_invl_gen are protected by the pv_list_locks[] 677 * elements, but reads are not. 678 */ 679 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 680 static struct mtx __exclusive_cache_line pv_chunks_mutex; 681 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; 682 static struct md_page *pv_table; 683 static struct md_page pv_dummy; 684 685 #ifdef PV_STATS 686 #define PV_STAT(x) do { x ; } while (0) 687 #else 688 #define PV_STAT(x) do { } while (0) 689 #endif 690 691 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT) 692 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)]) 693 694 #define PHYS_TO_PV_LIST_LOCK(pa) \ 695 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS]) 696 697 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 698 struct rwlock **_lockp = (lockp); \ 699 struct rwlock *_new_lock; \ 700 \ 701 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 702 if (_new_lock != *_lockp) { \ 703 if (*_lockp != NULL) \ 704 rw_wunlock(*_lockp); \ 705 *_lockp = _new_lock; \ 706 rw_wlock(*_lockp); \ 707 } \ 708 } while (0) 709 710 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 711 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 712 713 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 714 struct rwlock **_lockp = (lockp); \ 715 \ 716 if (*_lockp != NULL) { \ 717 rw_wunlock(*_lockp); \ 718 *_lockp = NULL; \ 719 } \ 720 } while (0) 721 722 #define VM_PAGE_TO_PV_LIST_LOCK(m) \ 723 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 724 725 /* 726 * We support 52 bits, hence: 727 * bits 52 - 31 = 21, 0b10101 728 * RTS encoding details 729 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long 730 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long 731 */ 732 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5)) 733 734 static int powernv_enabled = 1; 735 736 static __always_inline void 737 tlbiel_radix_set_isa300(uint32_t set, uint32_t is, 738 uint32_t pid, uint32_t ric, uint32_t prs) 739 { 740 uint64_t rb; 741 uint64_t rs; 742 743 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53); 744 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31); 745 746 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 747 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 748 : "memory"); 749 } 750 751 static void 752 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is) 753 { 754 uint32_t set; 755 756 __asm __volatile("ptesync": : :"memory"); 757 758 /* 759 * Flush the first set of the TLB, and the entire Page Walk Cache 760 * and partition table entries. Then flush the remaining sets of the 761 * TLB. 762 */ 763 if (is == TLB_INVAL_SCOPE_GLOBAL) { 764 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 765 for (set = 1; set < num_sets; set++) 766 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 767 } 768 769 /* Do the same for process scoped entries. */ 770 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 771 for (set = 1; set < num_sets; set++) 772 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 773 774 __asm __volatile("ptesync": : :"memory"); 775 } 776 777 static void 778 mmu_radix_tlbiel_flush(int scope) 779 { 780 MPASS(scope == TLB_INVAL_SCOPE_LPID || 781 scope == TLB_INVAL_SCOPE_GLOBAL); 782 783 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, scope); 784 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); 785 } 786 787 static void 788 mmu_radix_tlbie_all(void) 789 { 790 if (powernv_enabled) 791 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 792 else 793 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 794 } 795 796 static void 797 mmu_radix_init_amor(void) 798 { 799 /* 800 * In HV mode, we init AMOR (Authority Mask Override Register) so that 801 * the hypervisor and guest can setup IAMR (Instruction Authority Mask 802 * Register), enable key 0 and set it to 1. 803 * 804 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11) 805 */ 806 mtspr(SPR_AMOR, (3ul << 62)); 807 } 808 809 static void 810 mmu_radix_init_iamr(void) 811 { 812 /* 813 * Radix always uses key0 of the IAMR to determine if an access is 814 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 815 * fetch. 816 */ 817 mtspr(SPR_IAMR, (1ul << 62)); 818 } 819 820 static void 821 mmu_radix_pid_set(pmap_t pmap) 822 { 823 824 mtspr(SPR_PID, pmap->pm_pid); 825 isync(); 826 } 827 828 /* Quick sort callout for comparing physical addresses. */ 829 static int 830 pa_cmp(const void *a, const void *b) 831 { 832 const vm_paddr_t *pa = a, *pb = b; 833 834 if (*pa < *pb) 835 return (-1); 836 else if (*pa > *pb) 837 return (1); 838 else 839 return (0); 840 } 841 842 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) 843 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0) 844 #define pte_store(ptep, pte) do { \ 845 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \ 846 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \ 847 } while (0) 848 /* 849 * NB: should only be used for adding directories - not for direct mappings 850 */ 851 #define pde_store(ptep, pa) do { \ 852 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \ 853 } while (0) 854 855 #define pte_clear(ptep) do { \ 856 *(u_long *)(ptep) = (u_long)(0); \ 857 } while (0) 858 859 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */ 860 861 /* 862 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB 863 * (PTE) page mappings have identical settings for the following fields: 864 */ 865 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \ 866 PG_M | PG_A | RPTE_EAA_MASK | PG_V) 867 868 static __inline void 869 pmap_resident_count_inc(pmap_t pmap, int count) 870 { 871 872 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 873 pmap->pm_stats.resident_count += count; 874 } 875 876 static __inline void 877 pmap_resident_count_dec(pmap_t pmap, int count) 878 { 879 880 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 881 KASSERT(pmap->pm_stats.resident_count >= count, 882 ("pmap %p resident count underflow %ld %d", pmap, 883 pmap->pm_stats.resident_count, count)); 884 pmap->pm_stats.resident_count -= count; 885 } 886 887 static void 888 pagezero(vm_offset_t va) 889 { 890 va = trunc_page(va); 891 892 bzero((void *)va, PAGE_SIZE); 893 } 894 895 static uint64_t 896 allocpages(int n) 897 { 898 u_int64_t ret; 899 900 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE); 901 for (int i = 0; i < n; i++) 902 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE)); 903 return (ret); 904 } 905 906 static pt_entry_t * 907 kvtopte(vm_offset_t va) 908 { 909 pt_entry_t *l3e; 910 911 l3e = pmap_pml3e(kernel_pmap, va); 912 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 913 return (NULL); 914 return (pmap_l3e_to_pte(l3e, va)); 915 } 916 917 void 918 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa) 919 { 920 pt_entry_t *pte; 921 922 pte = kvtopte(va); 923 MPASS(pte != NULL); 924 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \ 925 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A); 926 } 927 928 bool 929 mmu_radix_ps_enabled(pmap_t pmap) 930 { 931 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); 932 } 933 934 static pt_entry_t * 935 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e) 936 { 937 pml3_entry_t *l3e; 938 pt_entry_t *pte; 939 940 va &= PG_PS_FRAME; 941 l3e = pmap_pml3e(pmap, va); 942 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0) 943 return (NULL); 944 945 if (be64toh(*l3e) & RPTE_LEAF) { 946 *is_l3e = 1; 947 return (l3e); 948 } 949 *is_l3e = 0; 950 va &= PG_FRAME; 951 pte = pmap_l3e_to_pte(l3e, va); 952 if (pte == NULL || (be64toh(*pte) & PG_V) == 0) 953 return (NULL); 954 return (pte); 955 } 956 957 int 958 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags) 959 { 960 pt_entry_t *pte; 961 pt_entry_t startpte, origpte, newpte; 962 vm_page_t m; 963 int is_l3e; 964 965 startpte = 0; 966 retry: 967 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) 968 return (KERN_INVALID_ADDRESS); 969 origpte = newpte = be64toh(*pte); 970 if (startpte == 0) { 971 startpte = origpte; 972 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) || 973 ((flags & VM_PROT_READ) && (startpte & PG_A))) { 974 pmap_invalidate_all(pmap); 975 #ifdef INVARIANTS 976 if (VERBOSE_PMAP || pmap_logging) 977 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n", 978 __func__, pmap, va, flags, origpte); 979 #endif 980 return (KERN_FAILURE); 981 } 982 } 983 #ifdef INVARIANTS 984 if (VERBOSE_PMAP || pmap_logging) 985 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va, 986 flags, origpte); 987 #endif 988 PMAP_LOCK(pmap); 989 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || 990 be64toh(*pte) != origpte) { 991 PMAP_UNLOCK(pmap); 992 return (KERN_FAILURE); 993 } 994 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME); 995 MPASS(m != NULL); 996 switch (flags) { 997 case VM_PROT_READ: 998 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0) 999 goto protfail; 1000 newpte |= PG_A; 1001 vm_page_aflag_set(m, PGA_REFERENCED); 1002 break; 1003 case VM_PROT_WRITE: 1004 if ((newpte & RPTE_EAA_W) == 0) 1005 goto protfail; 1006 if (is_l3e) 1007 goto protfail; 1008 newpte |= PG_M; 1009 vm_page_dirty(m); 1010 break; 1011 case VM_PROT_EXECUTE: 1012 if ((newpte & RPTE_EAA_X) == 0) 1013 goto protfail; 1014 newpte |= PG_A; 1015 vm_page_aflag_set(m, PGA_REFERENCED); 1016 break; 1017 } 1018 1019 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 1020 goto retry; 1021 ptesync(); 1022 PMAP_UNLOCK(pmap); 1023 if (startpte == newpte) 1024 return (KERN_FAILURE); 1025 return (0); 1026 protfail: 1027 PMAP_UNLOCK(pmap); 1028 return (KERN_PROTECTION_FAILURE); 1029 } 1030 1031 /* 1032 * Returns TRUE if the given page is mapped individually or as part of 1033 * a 2mpage. Otherwise, returns FALSE. 1034 */ 1035 boolean_t 1036 mmu_radix_page_is_mapped(vm_page_t m) 1037 { 1038 struct rwlock *lock; 1039 boolean_t rv; 1040 1041 if ((m->oflags & VPO_UNMANAGED) != 0) 1042 return (FALSE); 1043 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 1044 rw_rlock(lock); 1045 rv = !TAILQ_EMPTY(&m->md.pv_list) || 1046 ((m->flags & PG_FICTITIOUS) == 0 && 1047 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); 1048 rw_runlock(lock); 1049 return (rv); 1050 } 1051 1052 /* 1053 * Determine the appropriate bits to set in a PTE or PDE for a specified 1054 * caching mode. 1055 */ 1056 static int 1057 pmap_cache_bits(vm_memattr_t ma) 1058 { 1059 if (ma != VM_MEMATTR_DEFAULT) { 1060 switch (ma) { 1061 case VM_MEMATTR_UNCACHEABLE: 1062 return (RPTE_ATTR_GUARDEDIO); 1063 case VM_MEMATTR_CACHEABLE: 1064 return (RPTE_ATTR_MEM); 1065 case VM_MEMATTR_WRITE_BACK: 1066 case VM_MEMATTR_PREFETCHABLE: 1067 case VM_MEMATTR_WRITE_COMBINING: 1068 return (RPTE_ATTR_UNGUARDEDIO); 1069 } 1070 } 1071 return (0); 1072 } 1073 1074 static void 1075 pmap_invalidate_page(pmap_t pmap, vm_offset_t start) 1076 { 1077 ptesync(); 1078 if (pmap == kernel_pmap) 1079 radix_tlbie_invlpg_kernel_4k(start); 1080 else 1081 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1082 ttusync(); 1083 } 1084 1085 static void 1086 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start) 1087 { 1088 ptesync(); 1089 if (pmap == kernel_pmap) 1090 radix_tlbie_invlpg_kernel_2m(start); 1091 else 1092 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start); 1093 ttusync(); 1094 } 1095 1096 static void 1097 pmap_invalidate_pwc(pmap_t pmap) 1098 { 1099 ptesync(); 1100 if (pmap == kernel_pmap) 1101 radix_tlbie_invlpwc_kernel(); 1102 else 1103 radix_tlbie_invlpwc_user(pmap->pm_pid); 1104 ttusync(); 1105 } 1106 1107 static void 1108 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end) 1109 { 1110 if (((start - end) >> PAGE_SHIFT) > 8) { 1111 pmap_invalidate_all(pmap); 1112 return; 1113 } 1114 ptesync(); 1115 if (pmap == kernel_pmap) { 1116 while (start < end) { 1117 radix_tlbie_invlpg_kernel_4k(start); 1118 start += PAGE_SIZE; 1119 } 1120 } else { 1121 while (start < end) { 1122 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1123 start += PAGE_SIZE; 1124 } 1125 } 1126 ttusync(); 1127 } 1128 1129 static void 1130 pmap_invalidate_all(pmap_t pmap) 1131 { 1132 ptesync(); 1133 if (pmap == kernel_pmap) 1134 radix_tlbie_flush_kernel(); 1135 else 1136 radix_tlbie_flush_user(pmap->pm_pid); 1137 ttusync(); 1138 } 1139 1140 static void 1141 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e) 1142 { 1143 1144 /* 1145 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created 1146 * by a promotion that did not invalidate the 512 4KB page mappings 1147 * that might exist in the TLB. Consequently, at this point, the TLB 1148 * may hold both 4KB and 2MB page mappings for the address range [va, 1149 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here. 1150 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any 1151 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a 1152 * single INVLPG suffices to invalidate the 2MB page mapping from the 1153 * TLB. 1154 */ 1155 ptesync(); 1156 if ((l3e & PG_PROMOTED) != 0) 1157 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1); 1158 else 1159 pmap_invalidate_page_2m(pmap, va); 1160 1161 pmap_invalidate_pwc(pmap); 1162 } 1163 1164 static __inline struct pv_chunk * 1165 pv_to_chunk(pv_entry_t pv) 1166 { 1167 1168 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1169 } 1170 1171 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1172 1173 #define PC_FREE0 0xfffffffffffffffful 1174 #define PC_FREE1 ((1ul << (_NPCPV % 64)) - 1) 1175 1176 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 }; 1177 1178 /* 1179 * Ensure that the number of spare PV entries in the specified pmap meets or 1180 * exceeds the given count, "needed". 1181 * 1182 * The given PV list lock may be released. 1183 */ 1184 static void 1185 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 1186 { 1187 struct pch new_tail; 1188 struct pv_chunk *pc; 1189 vm_page_t m; 1190 int avail, free; 1191 bool reclaimed; 1192 1193 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1194 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 1195 1196 /* 1197 * Newly allocated PV chunks must be stored in a private list until 1198 * the required number of PV chunks have been allocated. Otherwise, 1199 * reclaim_pv_chunk() could recycle one of these chunks. In 1200 * contrast, these chunks must be added to the pmap upon allocation. 1201 */ 1202 TAILQ_INIT(&new_tail); 1203 retry: 1204 avail = 0; 1205 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 1206 // if ((cpu_feature2 & CPUID2_POPCNT) == 0) 1207 bit_count((bitstr_t *)pc->pc_map, 0, 1208 sizeof(pc->pc_map) * NBBY, &free); 1209 #if 0 1210 free = popcnt_pc_map_pq(pc->pc_map); 1211 #endif 1212 if (free == 0) 1213 break; 1214 avail += free; 1215 if (avail >= needed) 1216 break; 1217 } 1218 for (reclaimed = false; avail < needed; avail += _NPCPV) { 1219 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 1220 if (m == NULL) { 1221 m = reclaim_pv_chunk(pmap, lockp); 1222 if (m == NULL) 1223 goto retry; 1224 reclaimed = true; 1225 } 1226 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1227 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1228 dump_add_page(m->phys_addr); 1229 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1230 pc->pc_pmap = pmap; 1231 pc->pc_map[0] = PC_FREE0; 1232 pc->pc_map[1] = PC_FREE1; 1233 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1234 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); 1235 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); 1236 1237 /* 1238 * The reclaim might have freed a chunk from the current pmap. 1239 * If that chunk contained available entries, we need to 1240 * re-count the number of available entries. 1241 */ 1242 if (reclaimed) 1243 goto retry; 1244 } 1245 if (!TAILQ_EMPTY(&new_tail)) { 1246 mtx_lock(&pv_chunks_mutex); 1247 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); 1248 mtx_unlock(&pv_chunks_mutex); 1249 } 1250 } 1251 1252 /* 1253 * First find and then remove the pv entry for the specified pmap and virtual 1254 * address from the specified pv list. Returns the pv entry if found and NULL 1255 * otherwise. This operation can be performed on pv lists for either 4KB or 1256 * 2MB page mappings. 1257 */ 1258 static __inline pv_entry_t 1259 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1260 { 1261 pv_entry_t pv; 1262 1263 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 1264 #ifdef INVARIANTS 1265 if (PV_PMAP(pv) == NULL) { 1266 printf("corrupted pv_chunk/pv %p\n", pv); 1267 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":"); 1268 } 1269 MPASS(PV_PMAP(pv) != NULL); 1270 MPASS(pv->pv_va != 0); 1271 #endif 1272 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1273 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 1274 pvh->pv_gen++; 1275 break; 1276 } 1277 } 1278 return (pv); 1279 } 1280 1281 /* 1282 * After demotion from a 2MB page mapping to 512 4KB page mappings, 1283 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 1284 * entries for each of the 4KB page mappings. 1285 */ 1286 static void 1287 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1288 struct rwlock **lockp) 1289 { 1290 struct md_page *pvh; 1291 struct pv_chunk *pc; 1292 pv_entry_t pv; 1293 vm_offset_t va_last; 1294 vm_page_t m; 1295 int bit, field; 1296 1297 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1298 KASSERT((pa & L3_PAGE_MASK) == 0, 1299 ("pmap_pv_demote_pde: pa is not 2mpage aligned")); 1300 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1301 1302 /* 1303 * Transfer the 2mpage's pv entry for this mapping to the first 1304 * page's pv list. Once this transfer begins, the pv list lock 1305 * must not be released until the last pv entry is reinstantiated. 1306 */ 1307 pvh = pa_to_pvh(pa); 1308 va = trunc_2mpage(va); 1309 pv = pmap_pvh_remove(pvh, pmap, va); 1310 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); 1311 m = PHYS_TO_VM_PAGE(pa); 1312 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1313 1314 m->md.pv_gen++; 1315 /* Instantiate the remaining NPTEPG - 1 pv entries. */ 1316 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); 1317 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1318 for (;;) { 1319 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1320 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 1321 , ("pmap_pv_demote_pde: missing spare")); 1322 for (field = 0; field < _NPCM; field++) { 1323 while (pc->pc_map[field]) { 1324 bit = cnttzd(pc->pc_map[field]); 1325 pc->pc_map[field] &= ~(1ul << bit); 1326 pv = &pc->pc_pventry[field * 64 + bit]; 1327 va += PAGE_SIZE; 1328 pv->pv_va = va; 1329 m++; 1330 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1331 ("pmap_pv_demote_pde: page %p is not managed", m)); 1332 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1333 1334 m->md.pv_gen++; 1335 if (va == va_last) 1336 goto out; 1337 } 1338 } 1339 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1340 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1341 } 1342 out: 1343 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1344 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1345 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1346 } 1347 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); 1348 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); 1349 } 1350 1351 static void 1352 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap) 1353 { 1354 1355 if (pmap == NULL) 1356 return; 1357 pmap_invalidate_all(pmap); 1358 if (pmap != locked_pmap) 1359 PMAP_UNLOCK(pmap); 1360 } 1361 1362 /* 1363 * We are in a serious low memory condition. Resort to 1364 * drastic measures to free some pages so we can allocate 1365 * another pv entry chunk. 1366 * 1367 * Returns NULL if PV entries were reclaimed from the specified pmap. 1368 * 1369 * We do not, however, unmap 2mpages because subsequent accesses will 1370 * allocate per-page pv entries until repromotion occurs, thereby 1371 * exacerbating the shortage of free pv entries. 1372 */ 1373 static int active_reclaims = 0; 1374 static vm_page_t 1375 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 1376 { 1377 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 1378 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 1379 struct md_page *pvh; 1380 pml3_entry_t *l3e; 1381 pmap_t next_pmap, pmap; 1382 pt_entry_t *pte, tpte; 1383 pv_entry_t pv; 1384 vm_offset_t va; 1385 vm_page_t m, m_pc; 1386 struct spglist free; 1387 uint64_t inuse; 1388 int bit, field, freed; 1389 1390 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1391 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 1392 pmap = NULL; 1393 m_pc = NULL; 1394 SLIST_INIT(&free); 1395 bzero(&pc_marker_b, sizeof(pc_marker_b)); 1396 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 1397 pc_marker = (struct pv_chunk *)&pc_marker_b; 1398 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 1399 1400 mtx_lock(&pv_chunks_mutex); 1401 active_reclaims++; 1402 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru); 1403 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru); 1404 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 1405 SLIST_EMPTY(&free)) { 1406 next_pmap = pc->pc_pmap; 1407 if (next_pmap == NULL) { 1408 /* 1409 * The next chunk is a marker. However, it is 1410 * not our marker, so active_reclaims must be 1411 * > 1. Consequently, the next_chunk code 1412 * will not rotate the pv_chunks list. 1413 */ 1414 goto next_chunk; 1415 } 1416 mtx_unlock(&pv_chunks_mutex); 1417 1418 /* 1419 * A pv_chunk can only be removed from the pc_lru list 1420 * when both pc_chunks_mutex is owned and the 1421 * corresponding pmap is locked. 1422 */ 1423 if (pmap != next_pmap) { 1424 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1425 pmap = next_pmap; 1426 /* Avoid deadlock and lock recursion. */ 1427 if (pmap > locked_pmap) { 1428 RELEASE_PV_LIST_LOCK(lockp); 1429 PMAP_LOCK(pmap); 1430 mtx_lock(&pv_chunks_mutex); 1431 continue; 1432 } else if (pmap != locked_pmap) { 1433 if (PMAP_TRYLOCK(pmap)) { 1434 mtx_lock(&pv_chunks_mutex); 1435 continue; 1436 } else { 1437 pmap = NULL; /* pmap is not locked */ 1438 mtx_lock(&pv_chunks_mutex); 1439 pc = TAILQ_NEXT(pc_marker, pc_lru); 1440 if (pc == NULL || 1441 pc->pc_pmap != next_pmap) 1442 continue; 1443 goto next_chunk; 1444 } 1445 } 1446 } 1447 1448 /* 1449 * Destroy every non-wired, 4 KB page mapping in the chunk. 1450 */ 1451 freed = 0; 1452 for (field = 0; field < _NPCM; field++) { 1453 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 1454 inuse != 0; inuse &= ~(1UL << bit)) { 1455 bit = cnttzd(inuse); 1456 pv = &pc->pc_pventry[field * 64 + bit]; 1457 va = pv->pv_va; 1458 l3e = pmap_pml3e(pmap, va); 1459 if ((be64toh(*l3e) & RPTE_LEAF) != 0) 1460 continue; 1461 pte = pmap_l3e_to_pte(l3e, va); 1462 if ((be64toh(*pte) & PG_W) != 0) 1463 continue; 1464 tpte = be64toh(pte_load_clear(pte)); 1465 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); 1466 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 1467 vm_page_dirty(m); 1468 if ((tpte & PG_A) != 0) 1469 vm_page_aflag_set(m, PGA_REFERENCED); 1470 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1471 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 1472 1473 m->md.pv_gen++; 1474 if (TAILQ_EMPTY(&m->md.pv_list) && 1475 (m->flags & PG_FICTITIOUS) == 0) { 1476 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 1477 if (TAILQ_EMPTY(&pvh->pv_list)) { 1478 vm_page_aflag_clear(m, 1479 PGA_WRITEABLE); 1480 } 1481 } 1482 pc->pc_map[field] |= 1UL << bit; 1483 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free); 1484 freed++; 1485 } 1486 } 1487 if (freed == 0) { 1488 mtx_lock(&pv_chunks_mutex); 1489 goto next_chunk; 1490 } 1491 /* Every freed mapping is for a 4 KB page. */ 1492 pmap_resident_count_dec(pmap, freed); 1493 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 1494 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 1495 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 1496 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1497 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) { 1498 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1499 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1500 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1501 /* Entire chunk is free; return it. */ 1502 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1503 dump_drop_page(m_pc->phys_addr); 1504 mtx_lock(&pv_chunks_mutex); 1505 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1506 break; 1507 } 1508 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1509 mtx_lock(&pv_chunks_mutex); 1510 /* One freed pv entry in locked_pmap is sufficient. */ 1511 if (pmap == locked_pmap) 1512 break; 1513 next_chunk: 1514 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1515 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru); 1516 if (active_reclaims == 1 && pmap != NULL) { 1517 /* 1518 * Rotate the pv chunks list so that we do not 1519 * scan the same pv chunks that could not be 1520 * freed (because they contained a wired 1521 * and/or superpage mapping) on every 1522 * invocation of reclaim_pv_chunk(). 1523 */ 1524 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) { 1525 MPASS(pc->pc_pmap != NULL); 1526 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1527 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1528 } 1529 } 1530 } 1531 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1532 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); 1533 active_reclaims--; 1534 mtx_unlock(&pv_chunks_mutex); 1535 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1536 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 1537 m_pc = SLIST_FIRST(&free); 1538 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 1539 /* Recycle a freed page table page. */ 1540 m_pc->ref_count = 1; 1541 } 1542 vm_page_free_pages_toq(&free, true); 1543 return (m_pc); 1544 } 1545 1546 /* 1547 * free the pv_entry back to the free list 1548 */ 1549 static void 1550 free_pv_entry(pmap_t pmap, pv_entry_t pv) 1551 { 1552 struct pv_chunk *pc; 1553 int idx, field, bit; 1554 1555 #ifdef VERBOSE_PV 1556 if (pmap != kernel_pmap) 1557 printf("%s(%p, %p)\n", __func__, pmap, pv); 1558 #endif 1559 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1560 PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 1561 PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 1562 PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 1563 pc = pv_to_chunk(pv); 1564 idx = pv - &pc->pc_pventry[0]; 1565 field = idx / 64; 1566 bit = idx % 64; 1567 pc->pc_map[field] |= 1ul << bit; 1568 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) { 1569 /* 98% of the time, pc is already at the head of the list. */ 1570 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 1571 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1572 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1573 } 1574 return; 1575 } 1576 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1577 free_pv_chunk(pc); 1578 } 1579 1580 static void 1581 free_pv_chunk(struct pv_chunk *pc) 1582 { 1583 vm_page_t m; 1584 1585 mtx_lock(&pv_chunks_mutex); 1586 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1587 mtx_unlock(&pv_chunks_mutex); 1588 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1589 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1590 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1591 /* entire chunk is free, return it */ 1592 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1593 dump_drop_page(m->phys_addr); 1594 vm_page_unwire_noq(m); 1595 vm_page_free(m); 1596 } 1597 1598 /* 1599 * Returns a new PV entry, allocating a new PV chunk from the system when 1600 * needed. If this PV chunk allocation fails and a PV list lock pointer was 1601 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 1602 * returned. 1603 * 1604 * The given PV list lock may be released. 1605 */ 1606 static pv_entry_t 1607 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 1608 { 1609 int bit, field; 1610 pv_entry_t pv; 1611 struct pv_chunk *pc; 1612 vm_page_t m; 1613 1614 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1615 PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 1616 retry: 1617 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1618 if (pc != NULL) { 1619 for (field = 0; field < _NPCM; field++) { 1620 if (pc->pc_map[field]) { 1621 bit = cnttzd(pc->pc_map[field]); 1622 break; 1623 } 1624 } 1625 if (field < _NPCM) { 1626 pv = &pc->pc_pventry[field * 64 + bit]; 1627 pc->pc_map[field] &= ~(1ul << bit); 1628 /* If this was the last item, move it to tail */ 1629 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1630 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1631 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 1632 pc_list); 1633 } 1634 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1635 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 1636 MPASS(PV_PMAP(pv) != NULL); 1637 return (pv); 1638 } 1639 } 1640 /* No free items, allocate another chunk */ 1641 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 1642 if (m == NULL) { 1643 if (lockp == NULL) { 1644 PV_STAT(pc_chunk_tryfail++); 1645 return (NULL); 1646 } 1647 m = reclaim_pv_chunk(pmap, lockp); 1648 if (m == NULL) 1649 goto retry; 1650 } 1651 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1652 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1653 dump_add_page(m->phys_addr); 1654 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1655 pc->pc_pmap = pmap; 1656 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ 1657 pc->pc_map[1] = PC_FREE1; 1658 mtx_lock(&pv_chunks_mutex); 1659 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1660 mtx_unlock(&pv_chunks_mutex); 1661 pv = &pc->pc_pventry[0]; 1662 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1663 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1664 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 1665 MPASS(PV_PMAP(pv) != NULL); 1666 return (pv); 1667 } 1668 1669 #if VM_NRESERVLEVEL > 0 1670 /* 1671 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 1672 * replace the many pv entries for the 4KB page mappings by a single pv entry 1673 * for the 2MB page mapping. 1674 */ 1675 static void 1676 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1677 struct rwlock **lockp) 1678 { 1679 struct md_page *pvh; 1680 pv_entry_t pv; 1681 vm_offset_t va_last; 1682 vm_page_t m; 1683 1684 KASSERT((pa & L3_PAGE_MASK) == 0, 1685 ("pmap_pv_promote_pde: pa is not 2mpage aligned")); 1686 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1687 1688 /* 1689 * Transfer the first page's pv entry for this mapping to the 2mpage's 1690 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 1691 * a transfer avoids the possibility that get_pv_entry() calls 1692 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 1693 * mappings that is being promoted. 1694 */ 1695 m = PHYS_TO_VM_PAGE(pa); 1696 va = trunc_2mpage(va); 1697 pv = pmap_pvh_remove(&m->md, pmap, va); 1698 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); 1699 pvh = pa_to_pvh(pa); 1700 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 1701 pvh->pv_gen++; 1702 /* Free the remaining NPTEPG - 1 pv entries. */ 1703 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1704 do { 1705 m++; 1706 va += PAGE_SIZE; 1707 pmap_pvh_free(&m->md, pmap, va); 1708 } while (va < va_last); 1709 } 1710 #endif /* VM_NRESERVLEVEL > 0 */ 1711 1712 /* 1713 * First find and then destroy the pv entry for the specified pmap and virtual 1714 * address. This operation can be performed on pv lists for either 4KB or 2MB 1715 * page mappings. 1716 */ 1717 static void 1718 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1719 { 1720 pv_entry_t pv; 1721 1722 pv = pmap_pvh_remove(pvh, pmap, va); 1723 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 1724 free_pv_entry(pmap, pv); 1725 } 1726 1727 /* 1728 * Conditionally create the PV entry for a 4KB page mapping if the required 1729 * memory can be allocated without resorting to reclamation. 1730 */ 1731 static boolean_t 1732 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 1733 struct rwlock **lockp) 1734 { 1735 pv_entry_t pv; 1736 1737 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1738 /* Pass NULL instead of the lock pointer to disable reclamation. */ 1739 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 1740 pv->pv_va = va; 1741 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1742 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1743 m->md.pv_gen++; 1744 return (TRUE); 1745 } else 1746 return (FALSE); 1747 } 1748 1749 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX]; 1750 #ifdef INVARIANTS 1751 static void 1752 validate_addr(vm_paddr_t addr, vm_size_t size) 1753 { 1754 vm_paddr_t end = addr + size; 1755 bool found = false; 1756 1757 for (int i = 0; i < 2 * phys_avail_count; i += 2) { 1758 if (addr >= phys_avail_debug[i] && 1759 end <= phys_avail_debug[i + 1]) { 1760 found = true; 1761 break; 1762 } 1763 } 1764 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array", 1765 addr, end)); 1766 } 1767 #else 1768 static void validate_addr(vm_paddr_t addr, vm_size_t size) {} 1769 #endif 1770 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A) 1771 1772 static vm_paddr_t 1773 alloc_pt_page(void) 1774 { 1775 vm_paddr_t page; 1776 1777 page = allocpages(1); 1778 pagezero(PHYS_TO_DMAP(page)); 1779 return (page); 1780 } 1781 1782 static void 1783 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end) 1784 { 1785 pt_entry_t *pte, pteval; 1786 vm_paddr_t page; 1787 1788 if (bootverbose) 1789 printf("%s %lx -> %lx\n", __func__, start, end); 1790 while (start < end) { 1791 pteval = start | DMAP_PAGE_BITS; 1792 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start)); 1793 if ((be64toh(*pte) & RPTE_VALID) == 0) { 1794 page = alloc_pt_page(); 1795 pde_store(pte, page); 1796 } 1797 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start)); 1798 if ((start & L2_PAGE_MASK) == 0 && 1799 end - start >= L2_PAGE_SIZE) { 1800 start += L2_PAGE_SIZE; 1801 goto done; 1802 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1803 page = alloc_pt_page(); 1804 pde_store(pte, page); 1805 } 1806 1807 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start)); 1808 if ((start & L3_PAGE_MASK) == 0 && 1809 end - start >= L3_PAGE_SIZE) { 1810 start += L3_PAGE_SIZE; 1811 goto done; 1812 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1813 page = alloc_pt_page(); 1814 pde_store(pte, page); 1815 } 1816 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start)); 1817 start += PAGE_SIZE; 1818 done: 1819 pte_store(pte, pteval); 1820 } 1821 } 1822 1823 static void 1824 mmu_radix_dmap_populate(vm_size_t hwphyssz) 1825 { 1826 vm_paddr_t start, end; 1827 1828 for (int i = 0; i < pregions_sz; i++) { 1829 start = pregions[i].mr_start; 1830 end = start + pregions[i].mr_size; 1831 if (hwphyssz && start >= hwphyssz) 1832 break; 1833 if (hwphyssz && hwphyssz < end) 1834 end = hwphyssz; 1835 mmu_radix_dmap_range(start, end); 1836 } 1837 } 1838 1839 static void 1840 mmu_radix_setup_pagetables(vm_size_t hwphyssz) 1841 { 1842 vm_paddr_t ptpages, pages; 1843 pt_entry_t *pte; 1844 vm_paddr_t l1phys; 1845 1846 bzero(kernel_pmap, sizeof(struct pmap)); 1847 PMAP_LOCK_INIT(kernel_pmap); 1848 1849 ptpages = allocpages(3); 1850 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE); 1851 validate_addr(l1phys, RADIX_PGD_SIZE); 1852 if (bootverbose) 1853 printf("l1phys=%lx\n", l1phys); 1854 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0); 1855 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++) 1856 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE)); 1857 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys); 1858 1859 mmu_radix_dmap_populate(hwphyssz); 1860 1861 /* 1862 * Create page tables for first 128MB of KVA 1863 */ 1864 pages = ptpages; 1865 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS); 1866 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1867 pages += PAGE_SIZE; 1868 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS); 1869 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1870 pages += PAGE_SIZE; 1871 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS); 1872 /* 1873 * the kernel page table pages need to be preserved in 1874 * phys_avail and not overlap with previous allocations 1875 */ 1876 pages = allocpages(nkpt); 1877 if (bootverbose) { 1878 printf("phys_avail after dmap populate and nkpt allocation\n"); 1879 for (int j = 0; j < 2 * phys_avail_count; j+=2) 1880 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1881 j, phys_avail[j], j + 1, phys_avail[j + 1]); 1882 } 1883 KPTphys = pages; 1884 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE) 1885 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1886 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE; 1887 if (bootverbose) 1888 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1); 1889 /* 1890 * Add a physical memory segment (vm_phys_seg) corresponding to the 1891 * preallocated kernel page table pages so that vm_page structures 1892 * representing these pages will be created. The vm_page structures 1893 * are required for promotion of the corresponding kernel virtual 1894 * addresses to superpage mappings. 1895 */ 1896 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt)); 1897 } 1898 1899 static void 1900 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end) 1901 { 1902 vm_paddr_t kpstart, kpend; 1903 vm_size_t physsz, hwphyssz; 1904 //uint64_t l2virt; 1905 int rm_pavail, proctab_size; 1906 int i, j; 1907 1908 kpstart = start & ~DMAP_BASE_ADDRESS; 1909 kpend = end & ~DMAP_BASE_ADDRESS; 1910 1911 /* Get physical memory regions from firmware */ 1912 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 1913 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory"); 1914 1915 if (2 * VM_PHYSSEG_MAX < regions_sz) 1916 panic("mmu_radix_early_bootstrap: phys_avail too small"); 1917 1918 if (bootverbose) 1919 for (int i = 0; i < regions_sz; i++) 1920 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n", 1921 i, regions[i].mr_start, i, regions[i].mr_size); 1922 /* 1923 * XXX workaround a simulator bug 1924 */ 1925 for (int i = 0; i < regions_sz; i++) 1926 if (regions[i].mr_start & PAGE_MASK) { 1927 regions[i].mr_start += PAGE_MASK; 1928 regions[i].mr_start &= ~PAGE_MASK; 1929 regions[i].mr_size &= ~PAGE_MASK; 1930 } 1931 if (bootverbose) 1932 for (int i = 0; i < pregions_sz; i++) 1933 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n", 1934 i, pregions[i].mr_start, i, pregions[i].mr_size); 1935 1936 phys_avail_count = 0; 1937 physsz = 0; 1938 hwphyssz = 0; 1939 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 1940 for (i = 0, j = 0; i < regions_sz; i++) { 1941 if (bootverbose) 1942 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n", 1943 i, regions[i].mr_start, i, regions[i].mr_size); 1944 1945 if (regions[i].mr_size < PAGE_SIZE) 1946 continue; 1947 1948 if (hwphyssz != 0 && 1949 (physsz + regions[i].mr_size) >= hwphyssz) { 1950 if (physsz < hwphyssz) { 1951 phys_avail[j] = regions[i].mr_start; 1952 phys_avail[j + 1] = regions[i].mr_start + 1953 (hwphyssz - physsz); 1954 physsz = hwphyssz; 1955 phys_avail_count++; 1956 dump_avail[j] = phys_avail[j]; 1957 dump_avail[j + 1] = phys_avail[j + 1]; 1958 } 1959 break; 1960 } 1961 phys_avail[j] = regions[i].mr_start; 1962 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 1963 dump_avail[j] = phys_avail[j]; 1964 dump_avail[j + 1] = phys_avail[j + 1]; 1965 1966 phys_avail_count++; 1967 physsz += regions[i].mr_size; 1968 j += 2; 1969 } 1970 1971 /* Check for overlap with the kernel and exception vectors */ 1972 rm_pavail = 0; 1973 for (j = 0; j < 2 * phys_avail_count; j+=2) { 1974 if (phys_avail[j] < EXC_LAST) 1975 phys_avail[j] += EXC_LAST; 1976 1977 if (phys_avail[j] >= kpstart && 1978 phys_avail[j + 1] <= kpend) { 1979 phys_avail[j] = phys_avail[j + 1] = ~0; 1980 rm_pavail++; 1981 continue; 1982 } 1983 1984 if (kpstart >= phys_avail[j] && 1985 kpstart < phys_avail[j + 1]) { 1986 if (kpend < phys_avail[j + 1]) { 1987 phys_avail[2 * phys_avail_count] = 1988 (kpend & ~PAGE_MASK) + PAGE_SIZE; 1989 phys_avail[2 * phys_avail_count + 1] = 1990 phys_avail[j + 1]; 1991 phys_avail_count++; 1992 } 1993 1994 phys_avail[j + 1] = kpstart & ~PAGE_MASK; 1995 } 1996 1997 if (kpend >= phys_avail[j] && 1998 kpend < phys_avail[j + 1]) { 1999 if (kpstart > phys_avail[j]) { 2000 phys_avail[2 * phys_avail_count] = phys_avail[j]; 2001 phys_avail[2 * phys_avail_count + 1] = 2002 kpstart & ~PAGE_MASK; 2003 phys_avail_count++; 2004 } 2005 2006 phys_avail[j] = (kpend & ~PAGE_MASK) + 2007 PAGE_SIZE; 2008 } 2009 } 2010 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp); 2011 for (i = 0; i < 2 * phys_avail_count; i++) 2012 phys_avail_debug[i] = phys_avail[i]; 2013 2014 /* Remove physical available regions marked for removal (~0) */ 2015 if (rm_pavail) { 2016 phys_avail_count -= rm_pavail; 2017 for (i = 2 * phys_avail_count; 2018 i < 2*(phys_avail_count + rm_pavail); i+=2) 2019 phys_avail[i] = phys_avail[i + 1] = 0; 2020 } 2021 if (bootverbose) { 2022 printf("phys_avail ranges after filtering:\n"); 2023 for (j = 0; j < 2 * phys_avail_count; j+=2) 2024 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 2025 j, phys_avail[j], j + 1, phys_avail[j + 1]); 2026 } 2027 physmem = btoc(physsz); 2028 2029 /* XXX assume we're running non-virtualized and 2030 * we don't support BHYVE 2031 */ 2032 if (isa3_pid_bits == 0) 2033 isa3_pid_bits = 20; 2034 if (powernv_enabled) { 2035 parttab_phys = 2036 moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE); 2037 validate_addr(parttab_phys, PARTTAB_SIZE); 2038 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++) 2039 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE)); 2040 2041 } 2042 proctab_size = 1UL << PROCTAB_SIZE_SHIFT; 2043 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size); 2044 validate_addr(proctab0pa, proctab_size); 2045 for (int i = 0; i < proctab_size/PAGE_SIZE; i++) 2046 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE)); 2047 2048 mmu_radix_setup_pagetables(hwphyssz); 2049 } 2050 2051 static void 2052 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end) 2053 { 2054 int i; 2055 vm_paddr_t pa; 2056 void *dpcpu; 2057 vm_offset_t va; 2058 2059 /* 2060 * Set up the Open Firmware pmap and add its mappings if not in real 2061 * mode. 2062 */ 2063 if (bootverbose) 2064 printf("%s enter\n", __func__); 2065 2066 /* 2067 * Calculate the last available physical address, and reserve the 2068 * vm_page_array (upper bound). 2069 */ 2070 Maxmem = 0; 2071 for (i = 0; phys_avail[i + 1] != 0; i += 2) 2072 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1])); 2073 2074 /* 2075 * Remap any early IO mappings (console framebuffer, etc.) 2076 */ 2077 bs_remap_earlyboot(); 2078 2079 /* 2080 * Allocate a kernel stack with a guard page for thread0 and map it 2081 * into the kernel page map. 2082 */ 2083 pa = allocpages(kstack_pages); 2084 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 2085 virtual_avail = va + kstack_pages * PAGE_SIZE; 2086 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); 2087 thread0.td_kstack = va; 2088 for (i = 0; i < kstack_pages; i++) { 2089 mmu_radix_kenter(va, pa); 2090 pa += PAGE_SIZE; 2091 va += PAGE_SIZE; 2092 } 2093 thread0.td_kstack_pages = kstack_pages; 2094 2095 /* 2096 * Allocate virtual address space for the message buffer. 2097 */ 2098 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT); 2099 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa); 2100 2101 /* 2102 * Allocate virtual address space for the dynamic percpu area. 2103 */ 2104 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT); 2105 dpcpu = (void *)PHYS_TO_DMAP(pa); 2106 dpcpu_init(dpcpu, curcpu); 2107 2108 crashdumpmap = (caddr_t)virtual_avail; 2109 virtual_avail += MAXDUMPPGS * PAGE_SIZE; 2110 2111 /* 2112 * Reserve some special page table entries/VA space for temporary 2113 * mapping of pages. 2114 */ 2115 } 2116 2117 static void 2118 mmu_parttab_init(void) 2119 { 2120 uint64_t ptcr; 2121 2122 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys); 2123 2124 if (bootverbose) 2125 printf("%s parttab: %p\n", __func__, isa3_parttab); 2126 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2127 if (bootverbose) 2128 printf("setting ptcr %lx\n", ptcr); 2129 mtspr(SPR_PTCR, ptcr); 2130 } 2131 2132 static void 2133 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab) 2134 { 2135 uint64_t prev; 2136 2137 if (bootverbose) 2138 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab, 2139 lpid, pagetab, proctab); 2140 prev = be64toh(isa3_parttab[lpid].pagetab); 2141 isa3_parttab[lpid].pagetab = htobe64(pagetab); 2142 isa3_parttab[lpid].proctab = htobe64(proctab); 2143 2144 if (prev & PARTTAB_HR) { 2145 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : : 2146 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2147 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2148 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2149 } else { 2150 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : 2151 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2152 } 2153 ttusync(); 2154 } 2155 2156 static void 2157 mmu_radix_parttab_init(void) 2158 { 2159 uint64_t pagetab; 2160 2161 mmu_parttab_init(); 2162 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \ 2163 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR; 2164 mmu_parttab_update(0, pagetab, 0); 2165 } 2166 2167 static void 2168 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size) 2169 { 2170 uint64_t pagetab, proctab; 2171 2172 pagetab = be64toh(isa3_parttab[0].pagetab); 2173 proctab = proctabpa | table_size | PARTTAB_GR; 2174 mmu_parttab_update(0, pagetab, proctab); 2175 } 2176 2177 static void 2178 mmu_radix_proctab_init(void) 2179 { 2180 2181 isa3_base_pid = 1; 2182 2183 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa); 2184 isa3_proctab->proctab0 = 2185 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | 2186 RADIX_PGD_INDEX_SHIFT); 2187 2188 if (powernv_enabled) { 2189 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12); 2190 __asm __volatile("ptesync" : : : "memory"); 2191 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2192 "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); 2193 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory"); 2194 #ifdef PSERIES 2195 } else { 2196 int64_t rc; 2197 2198 rc = phyp_hcall(H_REGISTER_PROC_TBL, 2199 PROC_TABLE_NEW | PROC_TABLE_RADIX | PROC_TABLE_GTSE, 2200 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12); 2201 if (rc != H_SUCCESS) 2202 panic("mmu_radix_proctab_init: " 2203 "failed to register process table: rc=%jd", 2204 (intmax_t)rc); 2205 #endif 2206 } 2207 2208 if (bootverbose) 2209 printf("process table %p and kernel radix PDE: %p\n", 2210 isa3_proctab, kernel_pmap->pm_pml1); 2211 mtmsr(mfmsr() | PSL_DR ); 2212 mtmsr(mfmsr() & ~PSL_DR); 2213 kernel_pmap->pm_pid = isa3_base_pid; 2214 isa3_base_pid++; 2215 } 2216 2217 void 2218 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2219 int advice) 2220 { 2221 struct rwlock *lock; 2222 pml1_entry_t *l1e; 2223 pml2_entry_t *l2e; 2224 pml3_entry_t oldl3e, *l3e; 2225 pt_entry_t *pte; 2226 vm_offset_t va, va_next; 2227 vm_page_t m; 2228 bool anychanged; 2229 2230 if (advice != MADV_DONTNEED && advice != MADV_FREE) 2231 return; 2232 anychanged = false; 2233 PMAP_LOCK(pmap); 2234 for (; sva < eva; sva = va_next) { 2235 l1e = pmap_pml1e(pmap, sva); 2236 if ((be64toh(*l1e) & PG_V) == 0) { 2237 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2238 if (va_next < sva) 2239 va_next = eva; 2240 continue; 2241 } 2242 l2e = pmap_l1e_to_l2e(l1e, sva); 2243 if ((be64toh(*l2e) & PG_V) == 0) { 2244 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2245 if (va_next < sva) 2246 va_next = eva; 2247 continue; 2248 } 2249 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2250 if (va_next < sva) 2251 va_next = eva; 2252 l3e = pmap_l2e_to_l3e(l2e, sva); 2253 oldl3e = be64toh(*l3e); 2254 if ((oldl3e & PG_V) == 0) 2255 continue; 2256 else if ((oldl3e & RPTE_LEAF) != 0) { 2257 if ((oldl3e & PG_MANAGED) == 0) 2258 continue; 2259 lock = NULL; 2260 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) { 2261 if (lock != NULL) 2262 rw_wunlock(lock); 2263 2264 /* 2265 * The large page mapping was destroyed. 2266 */ 2267 continue; 2268 } 2269 2270 /* 2271 * Unless the page mappings are wired, remove the 2272 * mapping to a single page so that a subsequent 2273 * access may repromote. Choosing the last page 2274 * within the address range [sva, min(va_next, eva)) 2275 * generally results in more repromotions. Since the 2276 * underlying page table page is fully populated, this 2277 * removal never frees a page table page. 2278 */ 2279 if ((oldl3e & PG_W) == 0) { 2280 va = eva; 2281 if (va > va_next) 2282 va = va_next; 2283 va -= PAGE_SIZE; 2284 KASSERT(va >= sva, 2285 ("mmu_radix_advise: no address gap")); 2286 pte = pmap_l3e_to_pte(l3e, va); 2287 KASSERT((be64toh(*pte) & PG_V) != 0, 2288 ("pmap_advise: invalid PTE")); 2289 pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL, 2290 &lock); 2291 anychanged = true; 2292 } 2293 if (lock != NULL) 2294 rw_wunlock(lock); 2295 } 2296 if (va_next > eva) 2297 va_next = eva; 2298 va = va_next; 2299 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; 2300 pte++, sva += PAGE_SIZE) { 2301 MPASS(pte == pmap_pte(pmap, sva)); 2302 2303 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) 2304 goto maybe_invlrng; 2305 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2306 if (advice == MADV_DONTNEED) { 2307 /* 2308 * Future calls to pmap_is_modified() 2309 * can be avoided by making the page 2310 * dirty now. 2311 */ 2312 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME); 2313 vm_page_dirty(m); 2314 } 2315 atomic_clear_long(pte, htobe64(PG_M | PG_A)); 2316 } else if ((be64toh(*pte) & PG_A) != 0) 2317 atomic_clear_long(pte, htobe64(PG_A)); 2318 else 2319 goto maybe_invlrng; 2320 anychanged = true; 2321 continue; 2322 maybe_invlrng: 2323 if (va != va_next) { 2324 anychanged = true; 2325 va = va_next; 2326 } 2327 } 2328 if (va != va_next) 2329 anychanged = true; 2330 } 2331 if (anychanged) 2332 pmap_invalidate_all(pmap); 2333 PMAP_UNLOCK(pmap); 2334 } 2335 2336 /* 2337 * Routines used in machine-dependent code 2338 */ 2339 static void 2340 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end) 2341 { 2342 uint64_t lpcr; 2343 2344 if (bootverbose) 2345 printf("%s\n", __func__); 2346 hw_direct_map = 1; 2347 powernv_enabled = (mfmsr() & PSL_HV) ? 1 : 0; 2348 mmu_radix_early_bootstrap(start, end); 2349 if (bootverbose) 2350 printf("early bootstrap complete\n"); 2351 if (powernv_enabled) { 2352 lpcr = mfspr(SPR_LPCR); 2353 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2354 mmu_radix_parttab_init(); 2355 mmu_radix_init_amor(); 2356 if (bootverbose) 2357 printf("powernv init complete\n"); 2358 } 2359 mmu_radix_init_iamr(); 2360 mmu_radix_proctab_init(); 2361 mmu_radix_pid_set(kernel_pmap); 2362 if (powernv_enabled) 2363 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2364 else 2365 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 2366 2367 mmu_radix_late_bootstrap(start, end); 2368 numa_mem_regions(&numa_pregions, &numa_pregions_sz); 2369 if (bootverbose) 2370 printf("%s done\n", __func__); 2371 pmap_bootstrapped = 1; 2372 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE); 2373 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS); 2374 } 2375 2376 static void 2377 mmu_radix_cpu_bootstrap(int ap) 2378 { 2379 uint64_t lpcr; 2380 uint64_t ptcr; 2381 2382 if (powernv_enabled) { 2383 lpcr = mfspr(SPR_LPCR); 2384 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2385 2386 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2387 mtspr(SPR_PTCR, ptcr); 2388 mmu_radix_init_amor(); 2389 } 2390 mmu_radix_init_iamr(); 2391 mmu_radix_pid_set(kernel_pmap); 2392 if (powernv_enabled) 2393 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2394 else 2395 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 2396 } 2397 2398 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0, 2399 "2MB page mapping counters"); 2400 2401 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_demotions); 2402 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD, 2403 &pmap_l3e_demotions, "2MB page demotions"); 2404 2405 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_mappings); 2406 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD, 2407 &pmap_l3e_mappings, "2MB page mappings"); 2408 2409 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_p_failures); 2410 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD, 2411 &pmap_l3e_p_failures, "2MB page promotion failures"); 2412 2413 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_promotions); 2414 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD, 2415 &pmap_l3e_promotions, "2MB page promotions"); 2416 2417 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0, 2418 "1GB page mapping counters"); 2419 2420 static COUNTER_U64_DEFINE_EARLY(pmap_l2e_demotions); 2421 SYSCTL_COUNTER_U64(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD, 2422 &pmap_l2e_demotions, "1GB page demotions"); 2423 2424 void 2425 mmu_radix_clear_modify(vm_page_t m) 2426 { 2427 struct md_page *pvh; 2428 pmap_t pmap; 2429 pv_entry_t next_pv, pv; 2430 pml3_entry_t oldl3e, *l3e; 2431 pt_entry_t oldpte, *pte; 2432 struct rwlock *lock; 2433 vm_offset_t va; 2434 int md_gen, pvh_gen; 2435 2436 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2437 ("pmap_clear_modify: page %p is not managed", m)); 2438 vm_page_assert_busied(m); 2439 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 2440 2441 /* 2442 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. 2443 * If the object containing the page is locked and the page is not 2444 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 2445 */ 2446 if ((m->a.flags & PGA_WRITEABLE) == 0) 2447 return; 2448 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 2449 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 2450 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2451 rw_wlock(lock); 2452 restart: 2453 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 2454 pmap = PV_PMAP(pv); 2455 if (!PMAP_TRYLOCK(pmap)) { 2456 pvh_gen = pvh->pv_gen; 2457 rw_wunlock(lock); 2458 PMAP_LOCK(pmap); 2459 rw_wlock(lock); 2460 if (pvh_gen != pvh->pv_gen) { 2461 PMAP_UNLOCK(pmap); 2462 goto restart; 2463 } 2464 } 2465 va = pv->pv_va; 2466 l3e = pmap_pml3e(pmap, va); 2467 oldl3e = be64toh(*l3e); 2468 if ((oldl3e & PG_RW) != 0 && 2469 pmap_demote_l3e_locked(pmap, l3e, va, &lock) && 2470 (oldl3e & PG_W) == 0) { 2471 /* 2472 * Write protect the mapping to a 2473 * single page so that a subsequent 2474 * write access may repromote. 2475 */ 2476 va += VM_PAGE_TO_PHYS(m) - (oldl3e & 2477 PG_PS_FRAME); 2478 pte = pmap_l3e_to_pte(l3e, va); 2479 oldpte = be64toh(*pte); 2480 while (!atomic_cmpset_long(pte, 2481 htobe64(oldpte), 2482 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW)))) 2483 oldpte = be64toh(*pte); 2484 vm_page_dirty(m); 2485 pmap_invalidate_page(pmap, va); 2486 } 2487 PMAP_UNLOCK(pmap); 2488 } 2489 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2490 pmap = PV_PMAP(pv); 2491 if (!PMAP_TRYLOCK(pmap)) { 2492 md_gen = m->md.pv_gen; 2493 pvh_gen = pvh->pv_gen; 2494 rw_wunlock(lock); 2495 PMAP_LOCK(pmap); 2496 rw_wlock(lock); 2497 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 2498 PMAP_UNLOCK(pmap); 2499 goto restart; 2500 } 2501 } 2502 l3e = pmap_pml3e(pmap, pv->pv_va); 2503 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found" 2504 " a 2mpage in page %p's pv list", m)); 2505 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 2506 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2507 atomic_clear_long(pte, htobe64(PG_M)); 2508 pmap_invalidate_page(pmap, pv->pv_va); 2509 } 2510 PMAP_UNLOCK(pmap); 2511 } 2512 rw_wunlock(lock); 2513 } 2514 2515 void 2516 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2517 vm_size_t len, vm_offset_t src_addr) 2518 { 2519 struct rwlock *lock; 2520 struct spglist free; 2521 vm_offset_t addr; 2522 vm_offset_t end_addr = src_addr + len; 2523 vm_offset_t va_next; 2524 vm_page_t dst_pdpg, dstmpte, srcmpte; 2525 bool invalidate_all; 2526 2527 CTR6(KTR_PMAP, 2528 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n", 2529 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr); 2530 2531 if (dst_addr != src_addr) 2532 return; 2533 lock = NULL; 2534 invalidate_all = false; 2535 if (dst_pmap < src_pmap) { 2536 PMAP_LOCK(dst_pmap); 2537 PMAP_LOCK(src_pmap); 2538 } else { 2539 PMAP_LOCK(src_pmap); 2540 PMAP_LOCK(dst_pmap); 2541 } 2542 2543 for (addr = src_addr; addr < end_addr; addr = va_next) { 2544 pml1_entry_t *l1e; 2545 pml2_entry_t *l2e; 2546 pml3_entry_t srcptepaddr, *l3e; 2547 pt_entry_t *src_pte, *dst_pte; 2548 2549 l1e = pmap_pml1e(src_pmap, addr); 2550 if ((be64toh(*l1e) & PG_V) == 0) { 2551 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2552 if (va_next < addr) 2553 va_next = end_addr; 2554 continue; 2555 } 2556 2557 l2e = pmap_l1e_to_l2e(l1e, addr); 2558 if ((be64toh(*l2e) & PG_V) == 0) { 2559 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2560 if (va_next < addr) 2561 va_next = end_addr; 2562 continue; 2563 } 2564 2565 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2566 if (va_next < addr) 2567 va_next = end_addr; 2568 2569 l3e = pmap_l2e_to_l3e(l2e, addr); 2570 srcptepaddr = be64toh(*l3e); 2571 if (srcptepaddr == 0) 2572 continue; 2573 2574 if (srcptepaddr & RPTE_LEAF) { 2575 if ((addr & L3_PAGE_MASK) != 0 || 2576 addr + L3_PAGE_SIZE > end_addr) 2577 continue; 2578 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL); 2579 if (dst_pdpg == NULL) 2580 break; 2581 l3e = (pml3_entry_t *) 2582 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg)); 2583 l3e = &l3e[pmap_pml3e_index(addr)]; 2584 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 || 2585 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr, 2586 PMAP_ENTER_NORECLAIM, &lock))) { 2587 *l3e = htobe64(srcptepaddr & ~PG_W); 2588 pmap_resident_count_inc(dst_pmap, 2589 L3_PAGE_SIZE / PAGE_SIZE); 2590 counter_u64_add(pmap_l3e_mappings, 1); 2591 } else 2592 dst_pdpg->ref_count--; 2593 continue; 2594 } 2595 2596 srcptepaddr &= PG_FRAME; 2597 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 2598 KASSERT(srcmpte->ref_count > 0, 2599 ("pmap_copy: source page table page is unused")); 2600 2601 if (va_next > end_addr) 2602 va_next = end_addr; 2603 2604 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); 2605 src_pte = &src_pte[pmap_pte_index(addr)]; 2606 dstmpte = NULL; 2607 while (addr < va_next) { 2608 pt_entry_t ptetemp; 2609 ptetemp = be64toh(*src_pte); 2610 /* 2611 * we only virtual copy managed pages 2612 */ 2613 if ((ptetemp & PG_MANAGED) != 0) { 2614 if (dstmpte != NULL && 2615 dstmpte->pindex == pmap_l3e_pindex(addr)) 2616 dstmpte->ref_count++; 2617 else if ((dstmpte = pmap_allocpte(dst_pmap, 2618 addr, NULL)) == NULL) 2619 goto out; 2620 dst_pte = (pt_entry_t *) 2621 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); 2622 dst_pte = &dst_pte[pmap_pte_index(addr)]; 2623 if (be64toh(*dst_pte) == 0 && 2624 pmap_try_insert_pv_entry(dst_pmap, addr, 2625 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), 2626 &lock)) { 2627 /* 2628 * Clear the wired, modified, and 2629 * accessed (referenced) bits 2630 * during the copy. 2631 */ 2632 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M | 2633 PG_A)); 2634 pmap_resident_count_inc(dst_pmap, 1); 2635 } else { 2636 SLIST_INIT(&free); 2637 if (pmap_unwire_ptp(dst_pmap, addr, 2638 dstmpte, &free)) { 2639 /* 2640 * Although "addr" is not 2641 * mapped, paging-structure 2642 * caches could nonetheless 2643 * have entries that refer to 2644 * the freed page table pages. 2645 * Invalidate those entries. 2646 */ 2647 invalidate_all = true; 2648 vm_page_free_pages_toq(&free, 2649 true); 2650 } 2651 goto out; 2652 } 2653 if (dstmpte->ref_count >= srcmpte->ref_count) 2654 break; 2655 } 2656 addr += PAGE_SIZE; 2657 if (__predict_false((addr & L3_PAGE_MASK) == 0)) 2658 src_pte = pmap_pte(src_pmap, addr); 2659 else 2660 src_pte++; 2661 } 2662 } 2663 out: 2664 if (invalidate_all) 2665 pmap_invalidate_all(dst_pmap); 2666 if (lock != NULL) 2667 rw_wunlock(lock); 2668 PMAP_UNLOCK(src_pmap); 2669 PMAP_UNLOCK(dst_pmap); 2670 } 2671 2672 static void 2673 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst) 2674 { 2675 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); 2676 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); 2677 2678 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst); 2679 /* 2680 * XXX slow 2681 */ 2682 bcopy((void *)src, (void *)dst, PAGE_SIZE); 2683 } 2684 2685 static void 2686 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2687 vm_offset_t b_offset, int xfersize) 2688 { 2689 void *a_cp, *b_cp; 2690 vm_offset_t a_pg_offset, b_pg_offset; 2691 int cnt; 2692 2693 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma, 2694 a_offset, mb, b_offset, xfersize); 2695 2696 while (xfersize > 0) { 2697 a_pg_offset = a_offset & PAGE_MASK; 2698 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 2699 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2700 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) + 2701 a_pg_offset; 2702 b_pg_offset = b_offset & PAGE_MASK; 2703 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 2704 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2705 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) + 2706 b_pg_offset; 2707 bcopy(a_cp, b_cp, cnt); 2708 a_offset += cnt; 2709 b_offset += cnt; 2710 xfersize -= cnt; 2711 } 2712 } 2713 2714 #if VM_NRESERVLEVEL > 0 2715 /* 2716 * Tries to promote the 512, contiguous 4KB page mappings that are within a 2717 * single page table page (PTP) to a single 2MB page mapping. For promotion 2718 * to occur, two conditions must be met: (1) the 4KB page mappings must map 2719 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 2720 * identical characteristics. 2721 */ 2722 static int 2723 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va, 2724 struct rwlock **lockp) 2725 { 2726 pml3_entry_t newpde; 2727 pt_entry_t *firstpte, oldpte, pa, *pte; 2728 vm_page_t mpte; 2729 2730 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2731 2732 /* 2733 * Examine the first PTE in the specified PTP. Abort if this PTE is 2734 * either invalid, unused, or does not map the first 4KB physical page 2735 * within a 2MB page. 2736 */ 2737 firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME); 2738 setpde: 2739 newpde = be64toh(*firstpte); 2740 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) { 2741 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2742 " in pmap %p", va, pmap); 2743 goto fail; 2744 } 2745 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 2746 /* 2747 * When PG_M is already clear, PG_RW can be cleared without 2748 * a TLB invalidation. 2749 */ 2750 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W))) 2751 goto setpde; 2752 newpde &= ~RPTE_EAA_W; 2753 } 2754 2755 /* 2756 * Examine each of the other PTEs in the specified PTP. Abort if this 2757 * PTE maps an unexpected 4KB physical page or does not have identical 2758 * characteristics to the first PTE. 2759 */ 2760 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE; 2761 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { 2762 setpte: 2763 oldpte = be64toh(*pte); 2764 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { 2765 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2766 " in pmap %p", va, pmap); 2767 goto fail; 2768 } 2769 if ((oldpte & (PG_M | PG_RW)) == PG_RW) { 2770 /* 2771 * When PG_M is already clear, PG_RW can be cleared 2772 * without a TLB invalidation. 2773 */ 2774 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W))) 2775 goto setpte; 2776 oldpte &= ~RPTE_EAA_W; 2777 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx" 2778 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) | 2779 (va & ~L3_PAGE_MASK), pmap); 2780 } 2781 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { 2782 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2783 " in pmap %p", va, pmap); 2784 goto fail; 2785 } 2786 pa -= PAGE_SIZE; 2787 } 2788 2789 /* 2790 * Save the page table page in its current state until the PDE 2791 * mapping the superpage is demoted by pmap_demote_pde() or 2792 * destroyed by pmap_remove_pde(). 2793 */ 2794 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME); 2795 KASSERT(mpte >= vm_page_array && 2796 mpte < &vm_page_array[vm_page_array_size], 2797 ("pmap_promote_l3e: page table page is out of range")); 2798 KASSERT(mpte->pindex == pmap_l3e_pindex(va), 2799 ("pmap_promote_l3e: page table page's pindex is wrong")); 2800 if (pmap_insert_pt_page(pmap, mpte)) { 2801 CTR2(KTR_PMAP, 2802 "pmap_promote_l3e: failure for va %#lx in pmap %p", va, 2803 pmap); 2804 goto fail; 2805 } 2806 2807 /* 2808 * Promote the pv entries. 2809 */ 2810 if ((newpde & PG_MANAGED) != 0) 2811 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp); 2812 2813 pte_store(pde, PG_PROMOTED | newpde); 2814 ptesync(); 2815 counter_u64_add(pmap_l3e_promotions, 1); 2816 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx" 2817 " in pmap %p", va, pmap); 2818 return (0); 2819 fail: 2820 counter_u64_add(pmap_l3e_p_failures, 1); 2821 return (KERN_FAILURE); 2822 } 2823 #endif /* VM_NRESERVLEVEL > 0 */ 2824 2825 int 2826 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, 2827 vm_prot_t prot, u_int flags, int8_t psind) 2828 { 2829 struct rwlock *lock; 2830 pml3_entry_t *l3e; 2831 pt_entry_t *pte; 2832 pt_entry_t newpte, origpte; 2833 pv_entry_t pv; 2834 vm_paddr_t opa, pa; 2835 vm_page_t mpte, om; 2836 int rv, retrycount; 2837 boolean_t nosleep, invalidate_all, invalidate_page; 2838 2839 va = trunc_page(va); 2840 retrycount = 0; 2841 invalidate_page = invalidate_all = false; 2842 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va, 2843 m, prot, flags, psind); 2844 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 2845 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va), 2846 ("pmap_enter: managed mapping within the clean submap")); 2847 if ((m->oflags & VPO_UNMANAGED) == 0) 2848 VM_PAGE_OBJECT_BUSY_ASSERT(m); 2849 2850 KASSERT((flags & PMAP_ENTER_RESERVED) == 0, 2851 ("pmap_enter: flags %u has reserved bits set", flags)); 2852 pa = VM_PAGE_TO_PHYS(m); 2853 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF); 2854 if ((flags & VM_PROT_WRITE) != 0) 2855 newpte |= PG_M; 2856 if ((flags & VM_PROT_READ) != 0) 2857 newpte |= PG_A; 2858 if (prot & VM_PROT_READ) 2859 newpte |= RPTE_EAA_R; 2860 if ((prot & VM_PROT_WRITE) != 0) 2861 newpte |= RPTE_EAA_W; 2862 KASSERT((newpte & (PG_M | PG_RW)) != PG_M, 2863 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); 2864 2865 if (prot & VM_PROT_EXECUTE) 2866 newpte |= PG_X; 2867 if ((flags & PMAP_ENTER_WIRED) != 0) 2868 newpte |= PG_W; 2869 if (va >= DMAP_MIN_ADDRESS) 2870 newpte |= RPTE_EAA_P; 2871 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs); 2872 /* 2873 * Set modified bit gratuitously for writeable mappings if 2874 * the page is unmanaged. We do not want to take a fault 2875 * to do the dirty bit accounting for these mappings. 2876 */ 2877 if ((m->oflags & VPO_UNMANAGED) != 0) { 2878 if ((newpte & PG_RW) != 0) 2879 newpte |= PG_M; 2880 } else 2881 newpte |= PG_MANAGED; 2882 2883 lock = NULL; 2884 PMAP_LOCK(pmap); 2885 if (psind == 1) { 2886 /* Assert the required virtual and physical alignment. */ 2887 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned")); 2888 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 2889 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock); 2890 goto out; 2891 } 2892 mpte = NULL; 2893 2894 /* 2895 * In the case that a page table page is not 2896 * resident, we are creating it here. 2897 */ 2898 retry: 2899 l3e = pmap_pml3e(pmap, va); 2900 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 || 2901 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) { 2902 pte = pmap_l3e_to_pte(l3e, va); 2903 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { 2904 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 2905 mpte->ref_count++; 2906 } 2907 } else if (va < VM_MAXUSER_ADDRESS) { 2908 /* 2909 * Here if the pte page isn't mapped, or if it has been 2910 * deallocated. 2911 */ 2912 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 2913 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va), 2914 nosleep ? NULL : &lock); 2915 if (mpte == NULL && nosleep) { 2916 rv = KERN_RESOURCE_SHORTAGE; 2917 goto out; 2918 } 2919 if (__predict_false(retrycount++ == 6)) 2920 panic("too many retries"); 2921 invalidate_all = true; 2922 goto retry; 2923 } else 2924 panic("pmap_enter: invalid page directory va=%#lx", va); 2925 2926 origpte = be64toh(*pte); 2927 pv = NULL; 2928 2929 /* 2930 * Is the specified virtual address already mapped? 2931 */ 2932 if ((origpte & PG_V) != 0) { 2933 #ifdef INVARIANTS 2934 if (VERBOSE_PMAP || pmap_logging) { 2935 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --" 2936 " asid=%lu curpid=%d name=%s origpte0x%lx\n", 2937 pmap, va, m, prot, flags, psind, pmap->pm_pid, 2938 curproc->p_pid, curproc->p_comm, origpte); 2939 #ifdef DDB 2940 pmap_pte_walk(pmap->pm_pml1, va); 2941 #endif 2942 } 2943 #endif 2944 /* 2945 * Wiring change, just update stats. We don't worry about 2946 * wiring PT pages as they remain resident as long as there 2947 * are valid mappings in them. Hence, if a user page is wired, 2948 * the PT page will be also. 2949 */ 2950 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) 2951 pmap->pm_stats.wired_count++; 2952 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) 2953 pmap->pm_stats.wired_count--; 2954 2955 /* 2956 * Remove the extra PT page reference. 2957 */ 2958 if (mpte != NULL) { 2959 mpte->ref_count--; 2960 KASSERT(mpte->ref_count > 0, 2961 ("pmap_enter: missing reference to page table page," 2962 " va: 0x%lx", va)); 2963 } 2964 2965 /* 2966 * Has the physical page changed? 2967 */ 2968 opa = origpte & PG_FRAME; 2969 if (opa == pa) { 2970 /* 2971 * No, might be a protection or wiring change. 2972 */ 2973 if ((origpte & PG_MANAGED) != 0 && 2974 (newpte & PG_RW) != 0) 2975 vm_page_aflag_set(m, PGA_WRITEABLE); 2976 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) { 2977 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) { 2978 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 2979 goto retry; 2980 if ((newpte & PG_M) != (origpte & PG_M)) 2981 vm_page_dirty(m); 2982 if ((newpte & PG_A) != (origpte & PG_A)) 2983 vm_page_aflag_set(m, PGA_REFERENCED); 2984 ptesync(); 2985 } else 2986 invalidate_all = true; 2987 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) 2988 goto unchanged; 2989 } 2990 goto validate; 2991 } 2992 2993 /* 2994 * The physical page has changed. Temporarily invalidate 2995 * the mapping. This ensures that all threads sharing the 2996 * pmap keep a consistent view of the mapping, which is 2997 * necessary for the correct handling of COW faults. It 2998 * also permits reuse of the old mapping's PV entry, 2999 * avoiding an allocation. 3000 * 3001 * For consistency, handle unmanaged mappings the same way. 3002 */ 3003 origpte = be64toh(pte_load_clear(pte)); 3004 KASSERT((origpte & PG_FRAME) == opa, 3005 ("pmap_enter: unexpected pa update for %#lx", va)); 3006 if ((origpte & PG_MANAGED) != 0) { 3007 om = PHYS_TO_VM_PAGE(opa); 3008 3009 /* 3010 * The pmap lock is sufficient to synchronize with 3011 * concurrent calls to pmap_page_test_mappings() and 3012 * pmap_ts_referenced(). 3013 */ 3014 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3015 vm_page_dirty(om); 3016 if ((origpte & PG_A) != 0) 3017 vm_page_aflag_set(om, PGA_REFERENCED); 3018 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 3019 pv = pmap_pvh_remove(&om->md, pmap, va); 3020 if ((newpte & PG_MANAGED) == 0) 3021 free_pv_entry(pmap, pv); 3022 #ifdef INVARIANTS 3023 else if (origpte & PG_MANAGED) { 3024 if (pv == NULL) { 3025 #ifdef DDB 3026 pmap_page_print_mappings(om); 3027 #endif 3028 MPASS(pv != NULL); 3029 } 3030 } 3031 #endif 3032 if ((om->a.flags & PGA_WRITEABLE) != 0 && 3033 TAILQ_EMPTY(&om->md.pv_list) && 3034 ((om->flags & PG_FICTITIOUS) != 0 || 3035 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) 3036 vm_page_aflag_clear(om, PGA_WRITEABLE); 3037 } 3038 if ((origpte & PG_A) != 0) 3039 invalidate_page = true; 3040 origpte = 0; 3041 } else { 3042 if (pmap != kernel_pmap) { 3043 #ifdef INVARIANTS 3044 if (VERBOSE_PMAP || pmap_logging) 3045 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n", 3046 pmap, va, m, prot, flags, psind, 3047 pmap->pm_pid, curproc->p_pid, 3048 curproc->p_comm); 3049 #endif 3050 } 3051 3052 /* 3053 * Increment the counters. 3054 */ 3055 if ((newpte & PG_W) != 0) 3056 pmap->pm_stats.wired_count++; 3057 pmap_resident_count_inc(pmap, 1); 3058 } 3059 3060 /* 3061 * Enter on the PV list if part of our managed memory. 3062 */ 3063 if ((newpte & PG_MANAGED) != 0) { 3064 if (pv == NULL) { 3065 pv = get_pv_entry(pmap, &lock); 3066 pv->pv_va = va; 3067 } 3068 #ifdef VERBOSE_PV 3069 else 3070 printf("reassigning pv: %p to pmap: %p\n", 3071 pv, pmap); 3072 #endif 3073 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 3074 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3075 m->md.pv_gen++; 3076 if ((newpte & PG_RW) != 0) 3077 vm_page_aflag_set(m, PGA_WRITEABLE); 3078 } 3079 3080 /* 3081 * Update the PTE. 3082 */ 3083 if ((origpte & PG_V) != 0) { 3084 validate: 3085 origpte = be64toh(pte_load_store(pte, htobe64(newpte))); 3086 KASSERT((origpte & PG_FRAME) == pa, 3087 ("pmap_enter: unexpected pa update for %#lx", va)); 3088 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == 3089 (PG_M | PG_RW)) { 3090 if ((origpte & PG_MANAGED) != 0) 3091 vm_page_dirty(m); 3092 invalidate_page = true; 3093 3094 /* 3095 * Although the PTE may still have PG_RW set, TLB 3096 * invalidation may nonetheless be required because 3097 * the PTE no longer has PG_M set. 3098 */ 3099 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) { 3100 /* 3101 * Removing capabilities requires invalidation on POWER 3102 */ 3103 invalidate_page = true; 3104 goto unchanged; 3105 } 3106 if ((origpte & PG_A) != 0) 3107 invalidate_page = true; 3108 } else { 3109 pte_store(pte, newpte); 3110 ptesync(); 3111 } 3112 unchanged: 3113 3114 #if VM_NRESERVLEVEL > 0 3115 /* 3116 * If both the page table page and the reservation are fully 3117 * populated, then attempt promotion. 3118 */ 3119 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 3120 mmu_radix_ps_enabled(pmap) && 3121 (m->flags & PG_FICTITIOUS) == 0 && 3122 vm_reserv_level_iffullpop(m) == 0 && 3123 pmap_promote_l3e(pmap, l3e, va, &lock) == 0) 3124 invalidate_all = true; 3125 #endif 3126 if (invalidate_all) 3127 pmap_invalidate_all(pmap); 3128 else if (invalidate_page) 3129 pmap_invalidate_page(pmap, va); 3130 3131 rv = KERN_SUCCESS; 3132 out: 3133 if (lock != NULL) 3134 rw_wunlock(lock); 3135 PMAP_UNLOCK(pmap); 3136 3137 return (rv); 3138 } 3139 3140 /* 3141 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true 3142 * if successful. Returns false if (1) a page table page cannot be allocated 3143 * without sleeping, (2) a mapping already exists at the specified virtual 3144 * address, or (3) a PV entry cannot be allocated without reclaiming another 3145 * PV entry. 3146 */ 3147 static bool 3148 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3149 struct rwlock **lockp) 3150 { 3151 pml3_entry_t newpde; 3152 3153 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3154 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) | 3155 RPTE_LEAF | PG_V; 3156 if ((m->oflags & VPO_UNMANAGED) == 0) 3157 newpde |= PG_MANAGED; 3158 if (prot & VM_PROT_EXECUTE) 3159 newpde |= PG_X; 3160 if (prot & VM_PROT_READ) 3161 newpde |= RPTE_EAA_R; 3162 if (va >= DMAP_MIN_ADDRESS) 3163 newpde |= RPTE_EAA_P; 3164 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP | 3165 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == 3166 KERN_SUCCESS); 3167 } 3168 3169 /* 3170 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if 3171 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE 3172 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and 3173 * a mapping already exists at the specified virtual address. Returns 3174 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table 3175 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if 3176 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. 3177 * 3178 * The parameter "m" is only used when creating a managed, writeable mapping. 3179 */ 3180 static int 3181 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags, 3182 vm_page_t m, struct rwlock **lockp) 3183 { 3184 struct spglist free; 3185 pml3_entry_t oldl3e, *l3e; 3186 vm_page_t mt, pdpg; 3187 3188 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW, 3189 ("pmap_enter_pde: newpde is missing PG_M")); 3190 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3191 3192 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 3193 NULL : lockp)) == NULL) { 3194 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3195 " in pmap %p", va, pmap); 3196 return (KERN_RESOURCE_SHORTAGE); 3197 } 3198 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 3199 l3e = &l3e[pmap_pml3e_index(va)]; 3200 oldl3e = be64toh(*l3e); 3201 if ((oldl3e & PG_V) != 0) { 3202 KASSERT(pdpg->ref_count > 1, 3203 ("pmap_enter_pde: pdpg's wire count is too low")); 3204 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 3205 pdpg->ref_count--; 3206 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3207 " in pmap %p", va, pmap); 3208 return (KERN_FAILURE); 3209 } 3210 /* Break the existing mapping(s). */ 3211 SLIST_INIT(&free); 3212 if ((oldl3e & RPTE_LEAF) != 0) { 3213 /* 3214 * The reference to the PD page that was acquired by 3215 * pmap_allocl3e() ensures that it won't be freed. 3216 * However, if the PDE resulted from a promotion, then 3217 * a reserved PT page could be freed. 3218 */ 3219 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp); 3220 pmap_invalidate_l3e_page(pmap, va, oldl3e); 3221 } else { 3222 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e, 3223 &free, lockp)) 3224 pmap_invalidate_all(pmap); 3225 } 3226 vm_page_free_pages_toq(&free, true); 3227 if (va >= VM_MAXUSER_ADDRESS) { 3228 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 3229 if (pmap_insert_pt_page(pmap, mt)) { 3230 /* 3231 * XXX Currently, this can't happen because 3232 * we do not perform pmap_enter(psind == 1) 3233 * on the kernel pmap. 3234 */ 3235 panic("pmap_enter_pde: trie insert failed"); 3236 } 3237 } else 3238 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p", 3239 l3e)); 3240 } 3241 if ((newpde & PG_MANAGED) != 0) { 3242 /* 3243 * Abort this mapping if its PV entry could not be created. 3244 */ 3245 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) { 3246 SLIST_INIT(&free); 3247 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { 3248 /* 3249 * Although "va" is not mapped, paging- 3250 * structure caches could nonetheless have 3251 * entries that refer to the freed page table 3252 * pages. Invalidate those entries. 3253 */ 3254 pmap_invalidate_page(pmap, va); 3255 vm_page_free_pages_toq(&free, true); 3256 } 3257 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3258 " in pmap %p", va, pmap); 3259 return (KERN_RESOURCE_SHORTAGE); 3260 } 3261 if ((newpde & PG_RW) != 0) { 3262 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 3263 vm_page_aflag_set(mt, PGA_WRITEABLE); 3264 } 3265 } 3266 3267 /* 3268 * Increment counters. 3269 */ 3270 if ((newpde & PG_W) != 0) 3271 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE; 3272 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 3273 3274 /* 3275 * Map the superpage. (This is not a promoted mapping; there will not 3276 * be any lingering 4KB page mappings in the TLB.) 3277 */ 3278 pte_store(l3e, newpde); 3279 ptesync(); 3280 3281 counter_u64_add(pmap_l3e_mappings, 1); 3282 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" 3283 " in pmap %p", va, pmap); 3284 return (KERN_SUCCESS); 3285 } 3286 3287 void 3288 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start, 3289 vm_offset_t end, vm_page_t m_start, vm_prot_t prot) 3290 { 3291 3292 struct rwlock *lock; 3293 vm_offset_t va; 3294 vm_page_t m, mpte; 3295 vm_pindex_t diff, psize; 3296 bool invalidate; 3297 VM_OBJECT_ASSERT_LOCKED(m_start->object); 3298 3299 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start, 3300 end, m_start, prot); 3301 3302 invalidate = false; 3303 psize = atop(end - start); 3304 mpte = NULL; 3305 m = m_start; 3306 lock = NULL; 3307 PMAP_LOCK(pmap); 3308 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 3309 va = start + ptoa(diff); 3310 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end && 3311 m->psind == 1 && mmu_radix_ps_enabled(pmap) && 3312 pmap_enter_2mpage(pmap, va, m, prot, &lock)) 3313 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1]; 3314 else 3315 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot, 3316 mpte, &lock, &invalidate); 3317 m = TAILQ_NEXT(m, listq); 3318 } 3319 ptesync(); 3320 if (lock != NULL) 3321 rw_wunlock(lock); 3322 if (invalidate) 3323 pmap_invalidate_all(pmap); 3324 PMAP_UNLOCK(pmap); 3325 } 3326 3327 static vm_page_t 3328 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 3329 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate) 3330 { 3331 struct spglist free; 3332 pt_entry_t *pte; 3333 vm_paddr_t pa; 3334 3335 KASSERT(!VA_IS_CLEANMAP(va) || 3336 (m->oflags & VPO_UNMANAGED) != 0, 3337 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap")); 3338 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3339 3340 /* 3341 * In the case that a page table page is not 3342 * resident, we are creating it here. 3343 */ 3344 if (va < VM_MAXUSER_ADDRESS) { 3345 vm_pindex_t ptepindex; 3346 pml3_entry_t *ptepa; 3347 3348 /* 3349 * Calculate pagetable page index 3350 */ 3351 ptepindex = pmap_l3e_pindex(va); 3352 if (mpte && (mpte->pindex == ptepindex)) { 3353 mpte->ref_count++; 3354 } else { 3355 /* 3356 * Get the page directory entry 3357 */ 3358 ptepa = pmap_pml3e(pmap, va); 3359 3360 /* 3361 * If the page table page is mapped, we just increment 3362 * the hold count, and activate it. Otherwise, we 3363 * attempt to allocate a page table page. If this 3364 * attempt fails, we don't retry. Instead, we give up. 3365 */ 3366 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) { 3367 if (be64toh(*ptepa) & RPTE_LEAF) 3368 return (NULL); 3369 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME); 3370 mpte->ref_count++; 3371 } else { 3372 /* 3373 * Pass NULL instead of the PV list lock 3374 * pointer, because we don't intend to sleep. 3375 */ 3376 mpte = _pmap_allocpte(pmap, ptepindex, NULL); 3377 if (mpte == NULL) 3378 return (mpte); 3379 } 3380 } 3381 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); 3382 pte = &pte[pmap_pte_index(va)]; 3383 } else { 3384 mpte = NULL; 3385 pte = pmap_pte(pmap, va); 3386 } 3387 if (be64toh(*pte)) { 3388 if (mpte != NULL) { 3389 mpte->ref_count--; 3390 mpte = NULL; 3391 } 3392 return (mpte); 3393 } 3394 3395 /* 3396 * Enter on the PV list if part of our managed memory. 3397 */ 3398 if ((m->oflags & VPO_UNMANAGED) == 0 && 3399 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 3400 if (mpte != NULL) { 3401 SLIST_INIT(&free); 3402 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { 3403 /* 3404 * Although "va" is not mapped, paging- 3405 * structure caches could nonetheless have 3406 * entries that refer to the freed page table 3407 * pages. Invalidate those entries. 3408 */ 3409 *invalidate = true; 3410 vm_page_free_pages_toq(&free, true); 3411 } 3412 mpte = NULL; 3413 } 3414 return (mpte); 3415 } 3416 3417 /* 3418 * Increment counters 3419 */ 3420 pmap_resident_count_inc(pmap, 1); 3421 3422 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs); 3423 if (prot & VM_PROT_EXECUTE) 3424 pa |= PG_X; 3425 else 3426 pa |= RPTE_EAA_R; 3427 if ((m->oflags & VPO_UNMANAGED) == 0) 3428 pa |= PG_MANAGED; 3429 3430 pte_store(pte, pa); 3431 return (mpte); 3432 } 3433 3434 void 3435 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, 3436 vm_prot_t prot) 3437 { 3438 struct rwlock *lock; 3439 bool invalidate; 3440 3441 lock = NULL; 3442 invalidate = false; 3443 PMAP_LOCK(pmap); 3444 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock, 3445 &invalidate); 3446 ptesync(); 3447 if (lock != NULL) 3448 rw_wunlock(lock); 3449 if (invalidate) 3450 pmap_invalidate_all(pmap); 3451 PMAP_UNLOCK(pmap); 3452 } 3453 3454 vm_paddr_t 3455 mmu_radix_extract(pmap_t pmap, vm_offset_t va) 3456 { 3457 pml3_entry_t *l3e; 3458 pt_entry_t *pte; 3459 vm_paddr_t pa; 3460 3461 l3e = pmap_pml3e(pmap, va); 3462 if (__predict_false(l3e == NULL)) 3463 return (0); 3464 if (be64toh(*l3e) & RPTE_LEAF) { 3465 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 3466 pa |= (va & L3_PAGE_MASK); 3467 } else { 3468 /* 3469 * Beware of a concurrent promotion that changes the 3470 * PDE at this point! For example, vtopte() must not 3471 * be used to access the PTE because it would use the 3472 * new PDE. It is, however, safe to use the old PDE 3473 * because the page table page is preserved by the 3474 * promotion. 3475 */ 3476 pte = pmap_l3e_to_pte(l3e, va); 3477 if (__predict_false(pte == NULL)) 3478 return (0); 3479 pa = be64toh(*pte); 3480 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 3481 pa |= (va & PAGE_MASK); 3482 } 3483 return (pa); 3484 } 3485 3486 vm_page_t 3487 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3488 { 3489 pml3_entry_t l3e, *l3ep; 3490 pt_entry_t pte; 3491 vm_page_t m; 3492 3493 m = NULL; 3494 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot); 3495 PMAP_LOCK(pmap); 3496 l3ep = pmap_pml3e(pmap, va); 3497 if (l3ep != NULL && (l3e = be64toh(*l3ep))) { 3498 if (l3e & RPTE_LEAF) { 3499 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0) 3500 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) | 3501 (va & L3_PAGE_MASK)); 3502 } else { 3503 /* Native endian PTE, do not pass to pmap functions */ 3504 pte = be64toh(*pmap_l3e_to_pte(l3ep, va)); 3505 if ((pte & PG_V) && 3506 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) 3507 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 3508 } 3509 if (m != NULL && !vm_page_wire_mapped(m)) 3510 m = NULL; 3511 } 3512 PMAP_UNLOCK(pmap); 3513 return (m); 3514 } 3515 3516 static void 3517 mmu_radix_growkernel(vm_offset_t addr) 3518 { 3519 vm_paddr_t paddr; 3520 vm_page_t nkpg; 3521 pml3_entry_t *l3e; 3522 pml2_entry_t *l2e; 3523 3524 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 3525 if (VM_MIN_KERNEL_ADDRESS < addr && 3526 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE)) 3527 return; 3528 3529 addr = roundup2(addr, L3_PAGE_SIZE); 3530 if (addr - 1 >= vm_map_max(kernel_map)) 3531 addr = vm_map_max(kernel_map); 3532 while (kernel_vm_end < addr) { 3533 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); 3534 if ((be64toh(*l2e) & PG_V) == 0) { 3535 /* We need a new PDP entry */ 3536 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | 3537 VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3538 if (nkpg == NULL) 3539 panic("pmap_growkernel: no memory to grow kernel"); 3540 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT; 3541 paddr = VM_PAGE_TO_PHYS(nkpg); 3542 pde_store(l2e, paddr); 3543 continue; /* try again */ 3544 } 3545 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); 3546 if ((be64toh(*l3e) & PG_V) != 0) { 3547 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3548 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3549 kernel_vm_end = vm_map_max(kernel_map); 3550 break; 3551 } 3552 continue; 3553 } 3554 3555 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | 3556 VM_ALLOC_ZERO); 3557 if (nkpg == NULL) 3558 panic("pmap_growkernel: no memory to grow kernel"); 3559 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end); 3560 paddr = VM_PAGE_TO_PHYS(nkpg); 3561 pde_store(l3e, paddr); 3562 3563 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3564 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3565 kernel_vm_end = vm_map_max(kernel_map); 3566 break; 3567 } 3568 } 3569 ptesync(); 3570 } 3571 3572 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory"); 3573 static uma_zone_t zone_radix_pgd; 3574 3575 static int 3576 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused, 3577 int flags) 3578 { 3579 int req; 3580 3581 req = VM_ALLOC_WIRED | malloc2vm_flags(flags); 3582 for (int i = 0; i < count; i++) { 3583 vm_page_t m = vm_page_alloc_noobj_contig(req, 3584 RADIX_PGD_SIZE / PAGE_SIZE, 3585 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE, 3586 VM_MEMATTR_DEFAULT); 3587 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 3588 } 3589 return (count); 3590 } 3591 3592 static void 3593 radix_pgd_release(void *arg __unused, void **store, int count) 3594 { 3595 vm_page_t m; 3596 struct spglist free; 3597 int page_count; 3598 3599 SLIST_INIT(&free); 3600 page_count = RADIX_PGD_SIZE/PAGE_SIZE; 3601 3602 for (int i = 0; i < count; i++) { 3603 /* 3604 * XXX selectively remove dmap and KVA entries so we don't 3605 * need to bzero 3606 */ 3607 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i])); 3608 for (int j = page_count-1; j >= 0; j--) { 3609 vm_page_unwire_noq(&m[j]); 3610 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss); 3611 } 3612 vm_page_free_pages_toq(&free, false); 3613 } 3614 } 3615 3616 static void 3617 mmu_radix_init(void) 3618 { 3619 vm_page_t mpte; 3620 vm_size_t s; 3621 int error, i, pv_npg; 3622 3623 /* XXX is this really needed for POWER? */ 3624 /* L1TF, reserve page @0 unconditionally */ 3625 vm_page_blacklist_add(0, bootverbose); 3626 3627 zone_radix_pgd = uma_zcache_create("radix_pgd_cache", 3628 RADIX_PGD_SIZE, NULL, NULL, 3629 #ifdef INVARIANTS 3630 trash_init, trash_fini, 3631 #else 3632 NULL, NULL, 3633 #endif 3634 radix_pgd_import, radix_pgd_release, 3635 NULL, UMA_ZONE_NOBUCKET); 3636 3637 /* 3638 * Initialize the vm page array entries for the kernel pmap's 3639 * page table pages. 3640 */ 3641 PMAP_LOCK(kernel_pmap); 3642 for (i = 0; i < nkpt; i++) { 3643 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 3644 KASSERT(mpte >= vm_page_array && 3645 mpte < &vm_page_array[vm_page_array_size], 3646 ("pmap_init: page table page is out of range size: %lu", 3647 vm_page_array_size)); 3648 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i; 3649 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); 3650 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte); 3651 //pmap_insert_pt_page(kernel_pmap, mpte); 3652 mpte->ref_count = 1; 3653 } 3654 PMAP_UNLOCK(kernel_pmap); 3655 vm_wire_add(nkpt); 3656 3657 CTR1(KTR_PMAP, "%s()", __func__); 3658 TAILQ_INIT(&pv_dummy.pv_list); 3659 3660 /* 3661 * Are large page mappings enabled? 3662 */ 3663 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled); 3664 if (superpages_enabled) { 3665 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 3666 ("pmap_init: can't assign to pagesizes[1]")); 3667 pagesizes[1] = L3_PAGE_SIZE; 3668 } 3669 3670 /* 3671 * Initialize the pv chunk list mutex. 3672 */ 3673 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); 3674 3675 /* 3676 * Initialize the pool of pv list locks. 3677 */ 3678 for (i = 0; i < NPV_LIST_LOCKS; i++) 3679 rw_init(&pv_list_locks[i], "pmap pv list"); 3680 3681 /* 3682 * Calculate the size of the pv head table for superpages. 3683 */ 3684 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE); 3685 3686 /* 3687 * Allocate memory for the pv head table for superpages. 3688 */ 3689 s = (vm_size_t)(pv_npg * sizeof(struct md_page)); 3690 s = round_page(s); 3691 pv_table = kmem_malloc(s, M_WAITOK | M_ZERO); 3692 for (i = 0; i < pv_npg; i++) 3693 TAILQ_INIT(&pv_table[i].pv_list); 3694 TAILQ_INIT(&pv_dummy.pv_list); 3695 3696 pmap_initialized = 1; 3697 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); 3698 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 3699 (vmem_addr_t *)&qframe); 3700 3701 if (error != 0) 3702 panic("qframe allocation failed"); 3703 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits), 3704 1, 1, M_WAITOK); 3705 } 3706 3707 static boolean_t 3708 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) 3709 { 3710 struct rwlock *lock; 3711 pv_entry_t pv; 3712 struct md_page *pvh; 3713 pt_entry_t *pte, mask; 3714 pmap_t pmap; 3715 int md_gen, pvh_gen; 3716 boolean_t rv; 3717 3718 rv = FALSE; 3719 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 3720 rw_rlock(lock); 3721 restart: 3722 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 3723 pmap = PV_PMAP(pv); 3724 if (!PMAP_TRYLOCK(pmap)) { 3725 md_gen = m->md.pv_gen; 3726 rw_runlock(lock); 3727 PMAP_LOCK(pmap); 3728 rw_rlock(lock); 3729 if (md_gen != m->md.pv_gen) { 3730 PMAP_UNLOCK(pmap); 3731 goto restart; 3732 } 3733 } 3734 pte = pmap_pte(pmap, pv->pv_va); 3735 mask = 0; 3736 if (modified) 3737 mask |= PG_RW | PG_M; 3738 if (accessed) 3739 mask |= PG_V | PG_A; 3740 rv = (be64toh(*pte) & mask) == mask; 3741 PMAP_UNLOCK(pmap); 3742 if (rv) 3743 goto out; 3744 } 3745 if ((m->flags & PG_FICTITIOUS) == 0) { 3746 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 3747 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 3748 pmap = PV_PMAP(pv); 3749 if (!PMAP_TRYLOCK(pmap)) { 3750 md_gen = m->md.pv_gen; 3751 pvh_gen = pvh->pv_gen; 3752 rw_runlock(lock); 3753 PMAP_LOCK(pmap); 3754 rw_rlock(lock); 3755 if (md_gen != m->md.pv_gen || 3756 pvh_gen != pvh->pv_gen) { 3757 PMAP_UNLOCK(pmap); 3758 goto restart; 3759 } 3760 } 3761 pte = pmap_pml3e(pmap, pv->pv_va); 3762 mask = 0; 3763 if (modified) 3764 mask |= PG_RW | PG_M; 3765 if (accessed) 3766 mask |= PG_V | PG_A; 3767 rv = (be64toh(*pte) & mask) == mask; 3768 PMAP_UNLOCK(pmap); 3769 if (rv) 3770 goto out; 3771 } 3772 } 3773 out: 3774 rw_runlock(lock); 3775 return (rv); 3776 } 3777 3778 /* 3779 * pmap_is_modified: 3780 * 3781 * Return whether or not the specified physical page was modified 3782 * in any physical maps. 3783 */ 3784 boolean_t 3785 mmu_radix_is_modified(vm_page_t m) 3786 { 3787 3788 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3789 ("pmap_is_modified: page %p is not managed", m)); 3790 3791 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3792 /* 3793 * If the page is not busied then this check is racy. 3794 */ 3795 if (!pmap_page_is_write_mapped(m)) 3796 return (FALSE); 3797 return (pmap_page_test_mappings(m, FALSE, TRUE)); 3798 } 3799 3800 boolean_t 3801 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3802 { 3803 pml3_entry_t *l3e; 3804 pt_entry_t *pte; 3805 boolean_t rv; 3806 3807 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 3808 rv = FALSE; 3809 PMAP_LOCK(pmap); 3810 l3e = pmap_pml3e(pmap, addr); 3811 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) { 3812 pte = pmap_l3e_to_pte(l3e, addr); 3813 rv = (be64toh(*pte) & PG_V) == 0; 3814 } 3815 PMAP_UNLOCK(pmap); 3816 return (rv); 3817 } 3818 3819 boolean_t 3820 mmu_radix_is_referenced(vm_page_t m) 3821 { 3822 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3823 ("pmap_is_referenced: page %p is not managed", m)); 3824 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3825 return (pmap_page_test_mappings(m, TRUE, FALSE)); 3826 } 3827 3828 /* 3829 * pmap_ts_referenced: 3830 * 3831 * Return a count of reference bits for a page, clearing those bits. 3832 * It is not necessary for every reference bit to be cleared, but it 3833 * is necessary that 0 only be returned when there are truly no 3834 * reference bits set. 3835 * 3836 * As an optimization, update the page's dirty field if a modified bit is 3837 * found while counting reference bits. This opportunistic update can be 3838 * performed at low cost and can eliminate the need for some future calls 3839 * to pmap_is_modified(). However, since this function stops after 3840 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 3841 * dirty pages. Those dirty pages will only be detected by a future call 3842 * to pmap_is_modified(). 3843 * 3844 * A DI block is not needed within this function, because 3845 * invalidations are performed before the PV list lock is 3846 * released. 3847 */ 3848 boolean_t 3849 mmu_radix_ts_referenced(vm_page_t m) 3850 { 3851 struct md_page *pvh; 3852 pv_entry_t pv, pvf; 3853 pmap_t pmap; 3854 struct rwlock *lock; 3855 pml3_entry_t oldl3e, *l3e; 3856 pt_entry_t *pte; 3857 vm_paddr_t pa; 3858 int cleared, md_gen, not_cleared, pvh_gen; 3859 struct spglist free; 3860 3861 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3862 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3863 ("pmap_ts_referenced: page %p is not managed", m)); 3864 SLIST_INIT(&free); 3865 cleared = 0; 3866 pa = VM_PAGE_TO_PHYS(m); 3867 lock = PHYS_TO_PV_LIST_LOCK(pa); 3868 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); 3869 rw_wlock(lock); 3870 retry: 3871 not_cleared = 0; 3872 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 3873 goto small_mappings; 3874 pv = pvf; 3875 do { 3876 if (pvf == NULL) 3877 pvf = pv; 3878 pmap = PV_PMAP(pv); 3879 if (!PMAP_TRYLOCK(pmap)) { 3880 pvh_gen = pvh->pv_gen; 3881 rw_wunlock(lock); 3882 PMAP_LOCK(pmap); 3883 rw_wlock(lock); 3884 if (pvh_gen != pvh->pv_gen) { 3885 PMAP_UNLOCK(pmap); 3886 goto retry; 3887 } 3888 } 3889 l3e = pmap_pml3e(pmap, pv->pv_va); 3890 oldl3e = be64toh(*l3e); 3891 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 3892 /* 3893 * Although "oldpde" is mapping a 2MB page, because 3894 * this function is called at a 4KB page granularity, 3895 * we only update the 4KB page under test. 3896 */ 3897 vm_page_dirty(m); 3898 } 3899 if ((oldl3e & PG_A) != 0) { 3900 /* 3901 * Since this reference bit is shared by 512 4KB 3902 * pages, it should not be cleared every time it is 3903 * tested. Apply a simple "hash" function on the 3904 * physical page number, the virtual superpage number, 3905 * and the pmap address to select one 4KB page out of 3906 * the 512 on which testing the reference bit will 3907 * result in clearing that reference bit. This 3908 * function is designed to avoid the selection of the 3909 * same 4KB page for every 2MB page mapping. 3910 * 3911 * On demotion, a mapping that hasn't been referenced 3912 * is simply destroyed. To avoid the possibility of a 3913 * subsequent page fault on a demoted wired mapping, 3914 * always leave its reference bit set. Moreover, 3915 * since the superpage is wired, the current state of 3916 * its reference bit won't affect page replacement. 3917 */ 3918 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^ 3919 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && 3920 (oldl3e & PG_W) == 0) { 3921 atomic_clear_long(l3e, htobe64(PG_A)); 3922 pmap_invalidate_page(pmap, pv->pv_va); 3923 cleared++; 3924 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 3925 ("inconsistent pv lock %p %p for page %p", 3926 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 3927 } else 3928 not_cleared++; 3929 } 3930 PMAP_UNLOCK(pmap); 3931 /* Rotate the PV list if it has more than one entry. */ 3932 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3933 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 3934 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 3935 pvh->pv_gen++; 3936 } 3937 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 3938 goto out; 3939 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 3940 small_mappings: 3941 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 3942 goto out; 3943 pv = pvf; 3944 do { 3945 if (pvf == NULL) 3946 pvf = pv; 3947 pmap = PV_PMAP(pv); 3948 if (!PMAP_TRYLOCK(pmap)) { 3949 pvh_gen = pvh->pv_gen; 3950 md_gen = m->md.pv_gen; 3951 rw_wunlock(lock); 3952 PMAP_LOCK(pmap); 3953 rw_wlock(lock); 3954 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 3955 PMAP_UNLOCK(pmap); 3956 goto retry; 3957 } 3958 } 3959 l3e = pmap_pml3e(pmap, pv->pv_va); 3960 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 3961 ("pmap_ts_referenced: found a 2mpage in page %p's pv list", 3962 m)); 3963 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 3964 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3965 vm_page_dirty(m); 3966 if ((be64toh(*pte) & PG_A) != 0) { 3967 atomic_clear_long(pte, htobe64(PG_A)); 3968 pmap_invalidate_page(pmap, pv->pv_va); 3969 cleared++; 3970 } 3971 PMAP_UNLOCK(pmap); 3972 /* Rotate the PV list if it has more than one entry. */ 3973 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3974 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 3975 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3976 m->md.pv_gen++; 3977 } 3978 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 3979 not_cleared < PMAP_TS_REFERENCED_MAX); 3980 out: 3981 rw_wunlock(lock); 3982 vm_page_free_pages_toq(&free, true); 3983 return (cleared + not_cleared); 3984 } 3985 3986 static vm_offset_t 3987 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start, 3988 vm_paddr_t end, int prot __unused) 3989 { 3990 3991 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end, 3992 prot); 3993 return (PHYS_TO_DMAP(start)); 3994 } 3995 3996 void 3997 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr, 3998 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 3999 { 4000 pml3_entry_t *l3e; 4001 vm_paddr_t pa, ptepa; 4002 vm_page_t p, pdpg; 4003 vm_memattr_t ma; 4004 4005 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr, 4006 object, pindex, size); 4007 VM_OBJECT_ASSERT_WLOCKED(object); 4008 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 4009 ("pmap_object_init_pt: non-device object")); 4010 /* NB: size can be logically ored with addr here */ 4011 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) { 4012 if (!mmu_radix_ps_enabled(pmap)) 4013 return; 4014 if (!vm_object_populate(object, pindex, pindex + atop(size))) 4015 return; 4016 p = vm_page_lookup(object, pindex); 4017 KASSERT(p->valid == VM_PAGE_BITS_ALL, 4018 ("pmap_object_init_pt: invalid page %p", p)); 4019 ma = p->md.mdpg_cache_attrs; 4020 4021 /* 4022 * Abort the mapping if the first page is not physically 4023 * aligned to a 2MB page boundary. 4024 */ 4025 ptepa = VM_PAGE_TO_PHYS(p); 4026 if (ptepa & L3_PAGE_MASK) 4027 return; 4028 4029 /* 4030 * Skip the first page. Abort the mapping if the rest of 4031 * the pages are not physically contiguous or have differing 4032 * memory attributes. 4033 */ 4034 p = TAILQ_NEXT(p, listq); 4035 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; 4036 pa += PAGE_SIZE) { 4037 KASSERT(p->valid == VM_PAGE_BITS_ALL, 4038 ("pmap_object_init_pt: invalid page %p", p)); 4039 if (pa != VM_PAGE_TO_PHYS(p) || 4040 ma != p->md.mdpg_cache_attrs) 4041 return; 4042 p = TAILQ_NEXT(p, listq); 4043 } 4044 4045 PMAP_LOCK(pmap); 4046 for (pa = ptepa | pmap_cache_bits(ma); 4047 pa < ptepa + size; pa += L3_PAGE_SIZE) { 4048 pdpg = pmap_allocl3e(pmap, addr, NULL); 4049 if (pdpg == NULL) { 4050 /* 4051 * The creation of mappings below is only an 4052 * optimization. If a page directory page 4053 * cannot be allocated without blocking, 4054 * continue on to the next mapping rather than 4055 * blocking. 4056 */ 4057 addr += L3_PAGE_SIZE; 4058 continue; 4059 } 4060 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 4061 l3e = &l3e[pmap_pml3e_index(addr)]; 4062 if ((be64toh(*l3e) & PG_V) == 0) { 4063 pa |= PG_M | PG_A | PG_RW; 4064 pte_store(l3e, pa); 4065 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4066 counter_u64_add(pmap_l3e_mappings, 1); 4067 } else { 4068 /* Continue on if the PDE is already valid. */ 4069 pdpg->ref_count--; 4070 KASSERT(pdpg->ref_count > 0, 4071 ("pmap_object_init_pt: missing reference " 4072 "to page directory page, va: 0x%lx", addr)); 4073 } 4074 addr += L3_PAGE_SIZE; 4075 } 4076 ptesync(); 4077 PMAP_UNLOCK(pmap); 4078 } 4079 } 4080 4081 boolean_t 4082 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m) 4083 { 4084 struct md_page *pvh; 4085 struct rwlock *lock; 4086 pv_entry_t pv; 4087 int loops = 0; 4088 boolean_t rv; 4089 4090 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4091 ("pmap_page_exists_quick: page %p is not managed", m)); 4092 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m); 4093 rv = FALSE; 4094 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4095 rw_rlock(lock); 4096 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4097 if (PV_PMAP(pv) == pmap) { 4098 rv = TRUE; 4099 break; 4100 } 4101 loops++; 4102 if (loops >= 16) 4103 break; 4104 } 4105 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 4106 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4107 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4108 if (PV_PMAP(pv) == pmap) { 4109 rv = TRUE; 4110 break; 4111 } 4112 loops++; 4113 if (loops >= 16) 4114 break; 4115 } 4116 } 4117 rw_runlock(lock); 4118 return (rv); 4119 } 4120 4121 void 4122 mmu_radix_page_init(vm_page_t m) 4123 { 4124 4125 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4126 TAILQ_INIT(&m->md.pv_list); 4127 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT; 4128 } 4129 4130 int 4131 mmu_radix_page_wired_mappings(vm_page_t m) 4132 { 4133 struct rwlock *lock; 4134 struct md_page *pvh; 4135 pmap_t pmap; 4136 pt_entry_t *pte; 4137 pv_entry_t pv; 4138 int count, md_gen, pvh_gen; 4139 4140 if ((m->oflags & VPO_UNMANAGED) != 0) 4141 return (0); 4142 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4143 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4144 rw_rlock(lock); 4145 restart: 4146 count = 0; 4147 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4148 pmap = PV_PMAP(pv); 4149 if (!PMAP_TRYLOCK(pmap)) { 4150 md_gen = m->md.pv_gen; 4151 rw_runlock(lock); 4152 PMAP_LOCK(pmap); 4153 rw_rlock(lock); 4154 if (md_gen != m->md.pv_gen) { 4155 PMAP_UNLOCK(pmap); 4156 goto restart; 4157 } 4158 } 4159 pte = pmap_pte(pmap, pv->pv_va); 4160 if ((be64toh(*pte) & PG_W) != 0) 4161 count++; 4162 PMAP_UNLOCK(pmap); 4163 } 4164 if ((m->flags & PG_FICTITIOUS) == 0) { 4165 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4166 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4167 pmap = PV_PMAP(pv); 4168 if (!PMAP_TRYLOCK(pmap)) { 4169 md_gen = m->md.pv_gen; 4170 pvh_gen = pvh->pv_gen; 4171 rw_runlock(lock); 4172 PMAP_LOCK(pmap); 4173 rw_rlock(lock); 4174 if (md_gen != m->md.pv_gen || 4175 pvh_gen != pvh->pv_gen) { 4176 PMAP_UNLOCK(pmap); 4177 goto restart; 4178 } 4179 } 4180 pte = pmap_pml3e(pmap, pv->pv_va); 4181 if ((be64toh(*pte) & PG_W) != 0) 4182 count++; 4183 PMAP_UNLOCK(pmap); 4184 } 4185 } 4186 rw_runlock(lock); 4187 return (count); 4188 } 4189 4190 static void 4191 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa) 4192 { 4193 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT); 4194 } 4195 4196 int 4197 mmu_radix_pinit(pmap_t pmap) 4198 { 4199 vmem_addr_t pid; 4200 vm_paddr_t l1pa; 4201 4202 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4203 4204 /* 4205 * allocate the page directory page 4206 */ 4207 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK); 4208 4209 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++) 4210 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE); 4211 vm_radix_init(&pmap->pm_radix); 4212 TAILQ_INIT(&pmap->pm_pvchunk); 4213 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4214 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4215 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid); 4216 4217 pmap->pm_pid = pid; 4218 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1); 4219 mmu_radix_update_proctab(pid, l1pa); 4220 __asm __volatile("ptesync;isync" : : : "memory"); 4221 4222 return (1); 4223 } 4224 4225 /* 4226 * This routine is called if the desired page table page does not exist. 4227 * 4228 * If page table page allocation fails, this routine may sleep before 4229 * returning NULL. It sleeps only if a lock pointer was given. 4230 * 4231 * Note: If a page allocation fails at page table level two or three, 4232 * one or two pages may be held during the wait, only to be released 4233 * afterwards. This conservative approach is easily argued to avoid 4234 * race conditions. 4235 */ 4236 static vm_page_t 4237 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 4238 { 4239 vm_page_t m, pdppg, pdpg; 4240 4241 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4242 4243 /* 4244 * Allocate a page table page. 4245 */ 4246 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 4247 if (lockp != NULL) { 4248 RELEASE_PV_LIST_LOCK(lockp); 4249 PMAP_UNLOCK(pmap); 4250 vm_wait(NULL); 4251 PMAP_LOCK(pmap); 4252 } 4253 /* 4254 * Indicate the need to retry. While waiting, the page table 4255 * page may have been allocated. 4256 */ 4257 return (NULL); 4258 } 4259 m->pindex = ptepindex; 4260 4261 /* 4262 * Map the pagetable page into the process address space, if 4263 * it isn't already there. 4264 */ 4265 4266 if (ptepindex >= (NUPDE + NUPDPE)) { 4267 pml1_entry_t *l1e; 4268 vm_pindex_t pml1index; 4269 4270 /* Wire up a new PDPE page */ 4271 pml1index = ptepindex - (NUPDE + NUPDPE); 4272 l1e = &pmap->pm_pml1[pml1index]; 4273 KASSERT((be64toh(*l1e) & PG_V) == 0, 4274 ("%s: L1 entry %#lx is valid", __func__, *l1e)); 4275 pde_store(l1e, VM_PAGE_TO_PHYS(m)); 4276 } else if (ptepindex >= NUPDE) { 4277 vm_pindex_t pml1index; 4278 vm_pindex_t pdpindex; 4279 pml1_entry_t *l1e; 4280 pml2_entry_t *l2e; 4281 4282 /* Wire up a new l2e page */ 4283 pdpindex = ptepindex - NUPDE; 4284 pml1index = pdpindex >> RPTE_SHIFT; 4285 4286 l1e = &pmap->pm_pml1[pml1index]; 4287 if ((be64toh(*l1e) & PG_V) == 0) { 4288 /* Have to allocate a new pdp, recurse */ 4289 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index, 4290 lockp) == NULL) { 4291 vm_page_unwire_noq(m); 4292 vm_page_free_zero(m); 4293 return (NULL); 4294 } 4295 } else { 4296 /* Add reference to l2e page */ 4297 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME); 4298 pdppg->ref_count++; 4299 } 4300 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4301 4302 /* Now find the pdp page */ 4303 l2e = &l2e[pdpindex & RPTE_MASK]; 4304 KASSERT((be64toh(*l2e) & PG_V) == 0, 4305 ("%s: L2 entry %#lx is valid", __func__, *l2e)); 4306 pde_store(l2e, VM_PAGE_TO_PHYS(m)); 4307 } else { 4308 vm_pindex_t pml1index; 4309 vm_pindex_t pdpindex; 4310 pml1_entry_t *l1e; 4311 pml2_entry_t *l2e; 4312 pml3_entry_t *l3e; 4313 4314 /* Wire up a new PTE page */ 4315 pdpindex = ptepindex >> RPTE_SHIFT; 4316 pml1index = pdpindex >> RPTE_SHIFT; 4317 4318 /* First, find the pdp and check that its valid. */ 4319 l1e = &pmap->pm_pml1[pml1index]; 4320 if ((be64toh(*l1e) & PG_V) == 0) { 4321 /* Have to allocate a new pd, recurse */ 4322 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4323 lockp) == NULL) { 4324 vm_page_unwire_noq(m); 4325 vm_page_free_zero(m); 4326 return (NULL); 4327 } 4328 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4329 l2e = &l2e[pdpindex & RPTE_MASK]; 4330 } else { 4331 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4332 l2e = &l2e[pdpindex & RPTE_MASK]; 4333 if ((be64toh(*l2e) & PG_V) == 0) { 4334 /* Have to allocate a new pd, recurse */ 4335 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4336 lockp) == NULL) { 4337 vm_page_unwire_noq(m); 4338 vm_page_free_zero(m); 4339 return (NULL); 4340 } 4341 } else { 4342 /* Add reference to the pd page */ 4343 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME); 4344 pdpg->ref_count++; 4345 } 4346 } 4347 l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME); 4348 4349 /* Now we know where the page directory page is */ 4350 l3e = &l3e[ptepindex & RPTE_MASK]; 4351 KASSERT((be64toh(*l3e) & PG_V) == 0, 4352 ("%s: L3 entry %#lx is valid", __func__, *l3e)); 4353 pde_store(l3e, VM_PAGE_TO_PHYS(m)); 4354 } 4355 4356 pmap_resident_count_inc(pmap, 1); 4357 return (m); 4358 } 4359 static vm_page_t 4360 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4361 { 4362 vm_pindex_t pdpindex, ptepindex; 4363 pml2_entry_t *pdpe; 4364 vm_page_t pdpg; 4365 4366 retry: 4367 pdpe = pmap_pml2e(pmap, va); 4368 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) { 4369 /* Add a reference to the pd page. */ 4370 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME); 4371 pdpg->ref_count++; 4372 } else { 4373 /* Allocate a pd page. */ 4374 ptepindex = pmap_l3e_pindex(va); 4375 pdpindex = ptepindex >> RPTE_SHIFT; 4376 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); 4377 if (pdpg == NULL && lockp != NULL) 4378 goto retry; 4379 } 4380 return (pdpg); 4381 } 4382 4383 static vm_page_t 4384 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4385 { 4386 vm_pindex_t ptepindex; 4387 pml3_entry_t *pd; 4388 vm_page_t m; 4389 4390 /* 4391 * Calculate pagetable page index 4392 */ 4393 ptepindex = pmap_l3e_pindex(va); 4394 retry: 4395 /* 4396 * Get the page directory entry 4397 */ 4398 pd = pmap_pml3e(pmap, va); 4399 4400 /* 4401 * This supports switching from a 2MB page to a 4402 * normal 4K page. 4403 */ 4404 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) { 4405 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) { 4406 /* 4407 * Invalidation of the 2MB page mapping may have caused 4408 * the deallocation of the underlying PD page. 4409 */ 4410 pd = NULL; 4411 } 4412 } 4413 4414 /* 4415 * If the page table page is mapped, we just increment the 4416 * hold count, and activate it. 4417 */ 4418 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) { 4419 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME); 4420 m->ref_count++; 4421 } else { 4422 /* 4423 * Here if the pte page isn't mapped, or if it has been 4424 * deallocated. 4425 */ 4426 m = _pmap_allocpte(pmap, ptepindex, lockp); 4427 if (m == NULL && lockp != NULL) 4428 goto retry; 4429 } 4430 return (m); 4431 } 4432 4433 static void 4434 mmu_radix_pinit0(pmap_t pmap) 4435 { 4436 4437 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4438 PMAP_LOCK_INIT(pmap); 4439 pmap->pm_pml1 = kernel_pmap->pm_pml1; 4440 pmap->pm_pid = kernel_pmap->pm_pid; 4441 4442 vm_radix_init(&pmap->pm_radix); 4443 TAILQ_INIT(&pmap->pm_pvchunk); 4444 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4445 kernel_pmap->pm_flags = 4446 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4447 } 4448 /* 4449 * pmap_protect_l3e: do the things to protect a 2mpage in a process 4450 */ 4451 static boolean_t 4452 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot) 4453 { 4454 pt_entry_t newpde, oldpde; 4455 vm_offset_t eva, va; 4456 vm_page_t m; 4457 boolean_t anychanged; 4458 4459 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4460 KASSERT((sva & L3_PAGE_MASK) == 0, 4461 ("pmap_protect_l3e: sva is not 2mpage aligned")); 4462 anychanged = FALSE; 4463 retry: 4464 oldpde = newpde = be64toh(*l3e); 4465 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) == 4466 (PG_MANAGED | PG_M | PG_RW)) { 4467 eva = sva + L3_PAGE_SIZE; 4468 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4469 va < eva; va += PAGE_SIZE, m++) 4470 vm_page_dirty(m); 4471 } 4472 if ((prot & VM_PROT_WRITE) == 0) { 4473 newpde &= ~(PG_RW | PG_M); 4474 newpde |= RPTE_EAA_R; 4475 } 4476 if (prot & VM_PROT_EXECUTE) 4477 newpde |= PG_X; 4478 if (newpde != oldpde) { 4479 /* 4480 * As an optimization to future operations on this PDE, clear 4481 * PG_PROMOTED. The impending invalidation will remove any 4482 * lingering 4KB page mappings from the TLB. 4483 */ 4484 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED))) 4485 goto retry; 4486 anychanged = TRUE; 4487 } 4488 return (anychanged); 4489 } 4490 4491 void 4492 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 4493 vm_prot_t prot) 4494 { 4495 vm_offset_t va_next; 4496 pml1_entry_t *l1e; 4497 pml2_entry_t *l2e; 4498 pml3_entry_t ptpaddr, *l3e; 4499 pt_entry_t *pte; 4500 boolean_t anychanged; 4501 4502 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva, 4503 prot); 4504 4505 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 4506 if (prot == VM_PROT_NONE) { 4507 mmu_radix_remove(pmap, sva, eva); 4508 return; 4509 } 4510 4511 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 4512 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 4513 return; 4514 4515 #ifdef INVARIANTS 4516 if (VERBOSE_PROTECT || pmap_logging) 4517 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n", 4518 pmap, sva, eva, prot, pmap->pm_pid); 4519 #endif 4520 anychanged = FALSE; 4521 4522 PMAP_LOCK(pmap); 4523 for (; sva < eva; sva = va_next) { 4524 l1e = pmap_pml1e(pmap, sva); 4525 if ((be64toh(*l1e) & PG_V) == 0) { 4526 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 4527 if (va_next < sva) 4528 va_next = eva; 4529 continue; 4530 } 4531 4532 l2e = pmap_l1e_to_l2e(l1e, sva); 4533 if ((be64toh(*l2e) & PG_V) == 0) { 4534 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 4535 if (va_next < sva) 4536 va_next = eva; 4537 continue; 4538 } 4539 4540 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 4541 if (va_next < sva) 4542 va_next = eva; 4543 4544 l3e = pmap_l2e_to_l3e(l2e, sva); 4545 ptpaddr = be64toh(*l3e); 4546 4547 /* 4548 * Weed out invalid mappings. 4549 */ 4550 if (ptpaddr == 0) 4551 continue; 4552 4553 /* 4554 * Check for large page. 4555 */ 4556 if ((ptpaddr & RPTE_LEAF) != 0) { 4557 /* 4558 * Are we protecting the entire large page? If not, 4559 * demote the mapping and fall through. 4560 */ 4561 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 4562 if (pmap_protect_l3e(pmap, l3e, sva, prot)) 4563 anychanged = TRUE; 4564 continue; 4565 } else if (!pmap_demote_l3e(pmap, l3e, sva)) { 4566 /* 4567 * The large page mapping was destroyed. 4568 */ 4569 continue; 4570 } 4571 } 4572 4573 if (va_next > eva) 4574 va_next = eva; 4575 4576 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 4577 sva += PAGE_SIZE) { 4578 pt_entry_t obits, pbits; 4579 vm_page_t m; 4580 4581 retry: 4582 MPASS(pte == pmap_pte(pmap, sva)); 4583 obits = pbits = be64toh(*pte); 4584 if ((pbits & PG_V) == 0) 4585 continue; 4586 4587 if ((prot & VM_PROT_WRITE) == 0) { 4588 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == 4589 (PG_MANAGED | PG_M | PG_RW)) { 4590 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); 4591 vm_page_dirty(m); 4592 } 4593 pbits &= ~(PG_RW | PG_M); 4594 pbits |= RPTE_EAA_R; 4595 } 4596 if (prot & VM_PROT_EXECUTE) 4597 pbits |= PG_X; 4598 4599 if (pbits != obits) { 4600 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits))) 4601 goto retry; 4602 if (obits & (PG_A|PG_M)) { 4603 anychanged = TRUE; 4604 #ifdef INVARIANTS 4605 if (VERBOSE_PROTECT || pmap_logging) 4606 printf("%#lx %#lx -> %#lx\n", 4607 sva, obits, pbits); 4608 #endif 4609 } 4610 } 4611 } 4612 } 4613 if (anychanged) 4614 pmap_invalidate_all(pmap); 4615 PMAP_UNLOCK(pmap); 4616 } 4617 4618 void 4619 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count) 4620 { 4621 4622 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count); 4623 pt_entry_t oldpte, pa, *pte; 4624 vm_page_t m; 4625 uint64_t cache_bits, attr_bits; 4626 vm_offset_t va; 4627 4628 oldpte = 0; 4629 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 4630 va = sva; 4631 pte = kvtopte(va); 4632 while (va < sva + PAGE_SIZE * count) { 4633 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4634 pte = kvtopte(va); 4635 MPASS(pte == pmap_pte(kernel_pmap, va)); 4636 4637 /* 4638 * XXX there has to be a more efficient way than traversing 4639 * the page table every time - but go for correctness for 4640 * today 4641 */ 4642 4643 m = *ma++; 4644 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs); 4645 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits; 4646 if (be64toh(*pte) != pa) { 4647 oldpte |= be64toh(*pte); 4648 pte_store(pte, pa); 4649 } 4650 va += PAGE_SIZE; 4651 pte++; 4652 } 4653 if (__predict_false((oldpte & RPTE_VALID) != 0)) 4654 pmap_invalidate_range(kernel_pmap, sva, sva + count * 4655 PAGE_SIZE); 4656 else 4657 ptesync(); 4658 } 4659 4660 void 4661 mmu_radix_qremove(vm_offset_t sva, int count) 4662 { 4663 vm_offset_t va; 4664 pt_entry_t *pte; 4665 4666 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count); 4667 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva)); 4668 4669 va = sva; 4670 pte = kvtopte(va); 4671 while (va < sva + PAGE_SIZE * count) { 4672 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4673 pte = kvtopte(va); 4674 pte_clear(pte); 4675 pte++; 4676 va += PAGE_SIZE; 4677 } 4678 pmap_invalidate_range(kernel_pmap, sva, va); 4679 } 4680 4681 /*************************************************** 4682 * Page table page management routines..... 4683 ***************************************************/ 4684 /* 4685 * Schedule the specified unused page table page to be freed. Specifically, 4686 * add the page to the specified list of pages that will be released to the 4687 * physical memory manager after the TLB has been updated. 4688 */ 4689 static __inline void 4690 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 4691 boolean_t set_PG_ZERO) 4692 { 4693 4694 if (set_PG_ZERO) 4695 m->flags |= PG_ZERO; 4696 else 4697 m->flags &= ~PG_ZERO; 4698 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 4699 } 4700 4701 /* 4702 * Inserts the specified page table page into the specified pmap's collection 4703 * of idle page table pages. Each of a pmap's page table pages is responsible 4704 * for mapping a distinct range of virtual addresses. The pmap's collection is 4705 * ordered by this virtual address range. 4706 */ 4707 static __inline int 4708 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) 4709 { 4710 4711 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4712 return (vm_radix_insert(&pmap->pm_radix, mpte)); 4713 } 4714 4715 /* 4716 * Removes the page table page mapping the specified virtual address from the 4717 * specified pmap's collection of idle page table pages, and returns it. 4718 * Otherwise, returns NULL if there is no page table page corresponding to the 4719 * specified virtual address. 4720 */ 4721 static __inline vm_page_t 4722 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 4723 { 4724 4725 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4726 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va))); 4727 } 4728 4729 /* 4730 * Decrements a page table page's wire count, which is used to record the 4731 * number of valid page table entries within the page. If the wire count 4732 * drops to zero, then the page table page is unmapped. Returns TRUE if the 4733 * page table page was unmapped and FALSE otherwise. 4734 */ 4735 static inline boolean_t 4736 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4737 { 4738 4739 --m->ref_count; 4740 if (m->ref_count == 0) { 4741 _pmap_unwire_ptp(pmap, va, m, free); 4742 return (TRUE); 4743 } else 4744 return (FALSE); 4745 } 4746 4747 static void 4748 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4749 { 4750 4751 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4752 /* 4753 * unmap the page table page 4754 */ 4755 if (m->pindex >= NUPDE + NUPDPE) { 4756 /* PDP page */ 4757 pml1_entry_t *pml1; 4758 pml1 = pmap_pml1e(pmap, va); 4759 *pml1 = 0; 4760 } else if (m->pindex >= NUPDE) { 4761 /* PD page */ 4762 pml2_entry_t *l2e; 4763 l2e = pmap_pml2e(pmap, va); 4764 *l2e = 0; 4765 } else { 4766 /* PTE page */ 4767 pml3_entry_t *l3e; 4768 l3e = pmap_pml3e(pmap, va); 4769 *l3e = 0; 4770 } 4771 pmap_resident_count_dec(pmap, 1); 4772 if (m->pindex < NUPDE) { 4773 /* We just released a PT, unhold the matching PD */ 4774 vm_page_t pdpg; 4775 4776 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME); 4777 pmap_unwire_ptp(pmap, va, pdpg, free); 4778 } 4779 else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { 4780 /* We just released a PD, unhold the matching PDP */ 4781 vm_page_t pdppg; 4782 4783 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME); 4784 pmap_unwire_ptp(pmap, va, pdppg, free); 4785 } 4786 4787 /* 4788 * Put page on a list so that it is released after 4789 * *ALL* TLB shootdown is done 4790 */ 4791 pmap_add_delayed_free_list(m, free, TRUE); 4792 } 4793 4794 /* 4795 * After removing a page table entry, this routine is used to 4796 * conditionally free the page, and manage the hold/wire counts. 4797 */ 4798 static int 4799 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde, 4800 struct spglist *free) 4801 { 4802 vm_page_t mpte; 4803 4804 if (va >= VM_MAXUSER_ADDRESS) 4805 return (0); 4806 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 4807 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 4808 return (pmap_unwire_ptp(pmap, va, mpte, free)); 4809 } 4810 4811 void 4812 mmu_radix_release(pmap_t pmap) 4813 { 4814 4815 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4816 KASSERT(pmap->pm_stats.resident_count == 0, 4817 ("pmap_release: pmap resident count %ld != 0", 4818 pmap->pm_stats.resident_count)); 4819 KASSERT(vm_radix_is_empty(&pmap->pm_radix), 4820 ("pmap_release: pmap has reserved page table page(s)")); 4821 4822 pmap_invalidate_all(pmap); 4823 isa3_proctab[pmap->pm_pid].proctab0 = 0; 4824 uma_zfree(zone_radix_pgd, pmap->pm_pml1); 4825 vmem_free(asid_arena, pmap->pm_pid, 1); 4826 } 4827 4828 /* 4829 * Create the PV entry for a 2MB page mapping. Always returns true unless the 4830 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 4831 * false if the PV entry cannot be allocated without resorting to reclamation. 4832 */ 4833 static bool 4834 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags, 4835 struct rwlock **lockp) 4836 { 4837 struct md_page *pvh; 4838 pv_entry_t pv; 4839 vm_paddr_t pa; 4840 4841 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4842 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4843 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 4844 NULL : lockp)) == NULL) 4845 return (false); 4846 pv->pv_va = va; 4847 pa = pde & PG_PS_FRAME; 4848 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4849 pvh = pa_to_pvh(pa); 4850 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 4851 pvh->pv_gen++; 4852 return (true); 4853 } 4854 4855 /* 4856 * Fills a page table page with mappings to consecutive physical pages. 4857 */ 4858 static void 4859 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) 4860 { 4861 pt_entry_t *pte; 4862 4863 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { 4864 *pte = htobe64(newpte); 4865 newpte += PAGE_SIZE; 4866 } 4867 } 4868 4869 static boolean_t 4870 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va) 4871 { 4872 struct rwlock *lock; 4873 boolean_t rv; 4874 4875 lock = NULL; 4876 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock); 4877 if (lock != NULL) 4878 rw_wunlock(lock); 4879 return (rv); 4880 } 4881 4882 static boolean_t 4883 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 4884 struct rwlock **lockp) 4885 { 4886 pml3_entry_t oldpde; 4887 pt_entry_t *firstpte; 4888 vm_paddr_t mptepa; 4889 vm_page_t mpte; 4890 struct spglist free; 4891 vm_offset_t sva; 4892 4893 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4894 oldpde = be64toh(*l3e); 4895 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 4896 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx", 4897 oldpde)); 4898 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == 4899 NULL) { 4900 KASSERT((oldpde & PG_W) == 0, 4901 ("pmap_demote_l3e: page table page for a wired mapping" 4902 " is missing")); 4903 4904 /* 4905 * Invalidate the 2MB page mapping and return "failure" if the 4906 * mapping was never accessed or the allocation of the new 4907 * page table page fails. If the 2MB page mapping belongs to 4908 * the direct map region of the kernel's address space, then 4909 * the page allocation request specifies the highest possible 4910 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is 4911 * normal. Page table pages are preallocated for every other 4912 * part of the kernel address space, so the direct map region 4913 * is the only part of the kernel address space that must be 4914 * handled here. 4915 */ 4916 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj( 4917 (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ? 4918 VM_ALLOC_INTERRUPT : 0) | VM_ALLOC_WIRED)) == NULL) { 4919 SLIST_INIT(&free); 4920 sva = trunc_2mpage(va); 4921 pmap_remove_l3e(pmap, l3e, sva, &free, lockp); 4922 pmap_invalidate_l3e_page(pmap, sva, oldpde); 4923 vm_page_free_pages_toq(&free, true); 4924 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" 4925 " in pmap %p", va, pmap); 4926 return (FALSE); 4927 } 4928 mpte->pindex = pmap_l3e_pindex(va); 4929 if (va < VM_MAXUSER_ADDRESS) 4930 pmap_resident_count_inc(pmap, 1); 4931 } 4932 mptepa = VM_PAGE_TO_PHYS(mpte); 4933 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); 4934 KASSERT((oldpde & PG_A) != 0, 4935 ("pmap_demote_l3e: oldpde is missing PG_A")); 4936 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, 4937 ("pmap_demote_l3e: oldpde is missing PG_M")); 4938 4939 /* 4940 * If the page table page is new, initialize it. 4941 */ 4942 if (mpte->ref_count == 1) { 4943 mpte->ref_count = NPTEPG; 4944 pmap_fill_ptp(firstpte, oldpde); 4945 } 4946 4947 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME), 4948 ("pmap_demote_l3e: firstpte and newpte map different physical" 4949 " addresses")); 4950 4951 /* 4952 * If the mapping has changed attributes, update the page table 4953 * entries. 4954 */ 4955 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE)) 4956 pmap_fill_ptp(firstpte, oldpde); 4957 4958 /* 4959 * The spare PV entries must be reserved prior to demoting the 4960 * mapping, that is, prior to changing the PDE. Otherwise, the state 4961 * of the PDE and the PV lists will be inconsistent, which can result 4962 * in reclaim_pv_chunk() attempting to remove a PV entry from the 4963 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected 4964 * PV entry for the 2MB page mapping that is being demoted. 4965 */ 4966 if ((oldpde & PG_MANAGED) != 0) 4967 reserve_pv_entries(pmap, NPTEPG - 1, lockp); 4968 4969 /* 4970 * Demote the mapping. This pmap is locked. The old PDE has 4971 * PG_A set. If the old PDE has PG_RW set, it also has PG_M 4972 * set. Thus, there is no danger of a race with another 4973 * processor changing the setting of PG_A and/or PG_M between 4974 * the read above and the store below. 4975 */ 4976 pde_store(l3e, mptepa); 4977 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde); 4978 /* 4979 * Demote the PV entry. 4980 */ 4981 if ((oldpde & PG_MANAGED) != 0) 4982 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp); 4983 4984 counter_u64_add(pmap_l3e_demotions, 1); 4985 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx" 4986 " in pmap %p", va, pmap); 4987 return (TRUE); 4988 } 4989 4990 /* 4991 * pmap_remove_kernel_pde: Remove a kernel superpage mapping. 4992 */ 4993 static void 4994 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va) 4995 { 4996 vm_paddr_t mptepa; 4997 vm_page_t mpte; 4998 4999 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 5000 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5001 mpte = pmap_remove_pt_page(pmap, va); 5002 if (mpte == NULL) 5003 panic("pmap_remove_kernel_pde: Missing pt page."); 5004 5005 mptepa = VM_PAGE_TO_PHYS(mpte); 5006 5007 /* 5008 * Initialize the page table page. 5009 */ 5010 pagezero(PHYS_TO_DMAP(mptepa)); 5011 5012 /* 5013 * Demote the mapping. 5014 */ 5015 pde_store(l3e, mptepa); 5016 ptesync(); 5017 } 5018 5019 /* 5020 * pmap_remove_l3e: do the things to unmap a superpage in a process 5021 */ 5022 static int 5023 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 5024 struct spglist *free, struct rwlock **lockp) 5025 { 5026 struct md_page *pvh; 5027 pml3_entry_t oldpde; 5028 vm_offset_t eva, va; 5029 vm_page_t m, mpte; 5030 5031 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5032 KASSERT((sva & L3_PAGE_MASK) == 0, 5033 ("pmap_remove_l3e: sva is not 2mpage aligned")); 5034 oldpde = be64toh(pte_load_clear(pdq)); 5035 if (oldpde & PG_W) 5036 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE); 5037 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5038 if (oldpde & PG_MANAGED) { 5039 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); 5040 pvh = pa_to_pvh(oldpde & PG_PS_FRAME); 5041 pmap_pvh_free(pvh, pmap, sva); 5042 eva = sva + L3_PAGE_SIZE; 5043 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 5044 va < eva; va += PAGE_SIZE, m++) { 5045 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5046 vm_page_dirty(m); 5047 if (oldpde & PG_A) 5048 vm_page_aflag_set(m, PGA_REFERENCED); 5049 if (TAILQ_EMPTY(&m->md.pv_list) && 5050 TAILQ_EMPTY(&pvh->pv_list)) 5051 vm_page_aflag_clear(m, PGA_WRITEABLE); 5052 } 5053 } 5054 if (pmap == kernel_pmap) { 5055 pmap_remove_kernel_l3e(pmap, pdq, sva); 5056 } else { 5057 mpte = pmap_remove_pt_page(pmap, sva); 5058 if (mpte != NULL) { 5059 pmap_resident_count_dec(pmap, 1); 5060 KASSERT(mpte->ref_count == NPTEPG, 5061 ("pmap_remove_l3e: pte page wire count error")); 5062 mpte->ref_count = 0; 5063 pmap_add_delayed_free_list(mpte, free, FALSE); 5064 } 5065 } 5066 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free)); 5067 } 5068 5069 /* 5070 * pmap_remove_pte: do the things to unmap a page in a process 5071 */ 5072 static int 5073 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 5074 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp) 5075 { 5076 struct md_page *pvh; 5077 pt_entry_t oldpte; 5078 vm_page_t m; 5079 5080 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5081 oldpte = be64toh(pte_load_clear(ptq)); 5082 if (oldpte & RPTE_WIRED) 5083 pmap->pm_stats.wired_count -= 1; 5084 pmap_resident_count_dec(pmap, 1); 5085 if (oldpte & RPTE_MANAGED) { 5086 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); 5087 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5088 vm_page_dirty(m); 5089 if (oldpte & PG_A) 5090 vm_page_aflag_set(m, PGA_REFERENCED); 5091 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5092 pmap_pvh_free(&m->md, pmap, va); 5093 if (TAILQ_EMPTY(&m->md.pv_list) && 5094 (m->flags & PG_FICTITIOUS) == 0) { 5095 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5096 if (TAILQ_EMPTY(&pvh->pv_list)) 5097 vm_page_aflag_clear(m, PGA_WRITEABLE); 5098 } 5099 } 5100 return (pmap_unuse_pt(pmap, va, ptepde, free)); 5101 } 5102 5103 /* 5104 * Remove a single page from a process address space 5105 */ 5106 static bool 5107 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e, 5108 struct spglist *free) 5109 { 5110 struct rwlock *lock; 5111 pt_entry_t *pte; 5112 bool invalidate_all; 5113 5114 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5115 if ((be64toh(*l3e) & RPTE_VALID) == 0) { 5116 return (false); 5117 } 5118 pte = pmap_l3e_to_pte(l3e, va); 5119 if ((be64toh(*pte) & RPTE_VALID) == 0) { 5120 return (false); 5121 } 5122 lock = NULL; 5123 5124 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock); 5125 if (lock != NULL) 5126 rw_wunlock(lock); 5127 if (!invalidate_all) 5128 pmap_invalidate_page(pmap, va); 5129 return (invalidate_all); 5130 } 5131 5132 /* 5133 * Removes the specified range of addresses from the page table page. 5134 */ 5135 static bool 5136 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 5137 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp) 5138 { 5139 pt_entry_t *pte; 5140 vm_offset_t va; 5141 bool anyvalid; 5142 5143 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5144 anyvalid = false; 5145 va = eva; 5146 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++, 5147 sva += PAGE_SIZE) { 5148 MPASS(pte == pmap_pte(pmap, sva)); 5149 if (*pte == 0) { 5150 if (va != eva) { 5151 anyvalid = true; 5152 va = eva; 5153 } 5154 continue; 5155 } 5156 if (va == eva) 5157 va = sva; 5158 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) { 5159 anyvalid = true; 5160 sva += PAGE_SIZE; 5161 break; 5162 } 5163 } 5164 if (anyvalid) 5165 pmap_invalidate_all(pmap); 5166 else if (va != eva) 5167 pmap_invalidate_range(pmap, va, sva); 5168 return (anyvalid); 5169 } 5170 5171 void 5172 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5173 { 5174 struct rwlock *lock; 5175 vm_offset_t va_next; 5176 pml1_entry_t *l1e; 5177 pml2_entry_t *l2e; 5178 pml3_entry_t ptpaddr, *l3e; 5179 struct spglist free; 5180 bool anyvalid; 5181 5182 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5183 5184 /* 5185 * Perform an unsynchronized read. This is, however, safe. 5186 */ 5187 if (pmap->pm_stats.resident_count == 0) 5188 return; 5189 5190 anyvalid = false; 5191 SLIST_INIT(&free); 5192 5193 /* XXX something fishy here */ 5194 sva = (sva + PAGE_MASK) & ~PAGE_MASK; 5195 eva = (eva + PAGE_MASK) & ~PAGE_MASK; 5196 5197 PMAP_LOCK(pmap); 5198 5199 /* 5200 * special handling of removing one page. a very 5201 * common operation and easy to short circuit some 5202 * code. 5203 */ 5204 if (sva + PAGE_SIZE == eva) { 5205 l3e = pmap_pml3e(pmap, sva); 5206 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) { 5207 anyvalid = pmap_remove_page(pmap, sva, l3e, &free); 5208 goto out; 5209 } 5210 } 5211 5212 lock = NULL; 5213 for (; sva < eva; sva = va_next) { 5214 if (pmap->pm_stats.resident_count == 0) 5215 break; 5216 l1e = pmap_pml1e(pmap, sva); 5217 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) { 5218 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5219 if (va_next < sva) 5220 va_next = eva; 5221 continue; 5222 } 5223 5224 l2e = pmap_l1e_to_l2e(l1e, sva); 5225 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) { 5226 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5227 if (va_next < sva) 5228 va_next = eva; 5229 continue; 5230 } 5231 5232 /* 5233 * Calculate index for next page table. 5234 */ 5235 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5236 if (va_next < sva) 5237 va_next = eva; 5238 5239 l3e = pmap_l2e_to_l3e(l2e, sva); 5240 ptpaddr = be64toh(*l3e); 5241 5242 /* 5243 * Weed out invalid mappings. 5244 */ 5245 if (ptpaddr == 0) 5246 continue; 5247 5248 /* 5249 * Check for large page. 5250 */ 5251 if ((ptpaddr & RPTE_LEAF) != 0) { 5252 /* 5253 * Are we removing the entire large page? If not, 5254 * demote the mapping and fall through. 5255 */ 5256 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5257 pmap_remove_l3e(pmap, l3e, sva, &free, &lock); 5258 anyvalid = true; 5259 continue; 5260 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva, 5261 &lock)) { 5262 /* The large page mapping was destroyed. */ 5263 continue; 5264 } else 5265 ptpaddr = be64toh(*l3e); 5266 } 5267 5268 /* 5269 * Limit our scan to either the end of the va represented 5270 * by the current page table page, or to the end of the 5271 * range being removed. 5272 */ 5273 if (va_next > eva) 5274 va_next = eva; 5275 5276 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock)) 5277 anyvalid = true; 5278 } 5279 if (lock != NULL) 5280 rw_wunlock(lock); 5281 out: 5282 if (anyvalid) 5283 pmap_invalidate_all(pmap); 5284 PMAP_UNLOCK(pmap); 5285 vm_page_free_pages_toq(&free, true); 5286 } 5287 5288 void 5289 mmu_radix_remove_all(vm_page_t m) 5290 { 5291 struct md_page *pvh; 5292 pv_entry_t pv; 5293 pmap_t pmap; 5294 struct rwlock *lock; 5295 pt_entry_t *pte, tpte; 5296 pml3_entry_t *l3e; 5297 vm_offset_t va; 5298 struct spglist free; 5299 int pvh_gen, md_gen; 5300 5301 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5302 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5303 ("pmap_remove_all: page %p is not managed", m)); 5304 SLIST_INIT(&free); 5305 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5306 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5307 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5308 retry: 5309 rw_wlock(lock); 5310 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 5311 pmap = PV_PMAP(pv); 5312 if (!PMAP_TRYLOCK(pmap)) { 5313 pvh_gen = pvh->pv_gen; 5314 rw_wunlock(lock); 5315 PMAP_LOCK(pmap); 5316 rw_wlock(lock); 5317 if (pvh_gen != pvh->pv_gen) { 5318 rw_wunlock(lock); 5319 PMAP_UNLOCK(pmap); 5320 goto retry; 5321 } 5322 } 5323 va = pv->pv_va; 5324 l3e = pmap_pml3e(pmap, va); 5325 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock); 5326 PMAP_UNLOCK(pmap); 5327 } 5328 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 5329 pmap = PV_PMAP(pv); 5330 if (!PMAP_TRYLOCK(pmap)) { 5331 pvh_gen = pvh->pv_gen; 5332 md_gen = m->md.pv_gen; 5333 rw_wunlock(lock); 5334 PMAP_LOCK(pmap); 5335 rw_wlock(lock); 5336 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 5337 rw_wunlock(lock); 5338 PMAP_UNLOCK(pmap); 5339 goto retry; 5340 } 5341 } 5342 pmap_resident_count_dec(pmap, 1); 5343 l3e = pmap_pml3e(pmap, pv->pv_va); 5344 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found" 5345 " a 2mpage in page %p's pv list", m)); 5346 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5347 tpte = be64toh(pte_load_clear(pte)); 5348 if (tpte & PG_W) 5349 pmap->pm_stats.wired_count--; 5350 if (tpte & PG_A) 5351 vm_page_aflag_set(m, PGA_REFERENCED); 5352 5353 /* 5354 * Update the vm_page_t clean and reference bits. 5355 */ 5356 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5357 vm_page_dirty(m); 5358 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free); 5359 pmap_invalidate_page(pmap, pv->pv_va); 5360 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5361 m->md.pv_gen++; 5362 free_pv_entry(pmap, pv); 5363 PMAP_UNLOCK(pmap); 5364 } 5365 vm_page_aflag_clear(m, PGA_WRITEABLE); 5366 rw_wunlock(lock); 5367 vm_page_free_pages_toq(&free, true); 5368 } 5369 5370 /* 5371 * Destroy all managed, non-wired mappings in the given user-space 5372 * pmap. This pmap cannot be active on any processor besides the 5373 * caller. 5374 * 5375 * This function cannot be applied to the kernel pmap. Moreover, it 5376 * is not intended for general use. It is only to be used during 5377 * process termination. Consequently, it can be implemented in ways 5378 * that make it faster than pmap_remove(). First, it can more quickly 5379 * destroy mappings by iterating over the pmap's collection of PV 5380 * entries, rather than searching the page table. Second, it doesn't 5381 * have to test and clear the page table entries atomically, because 5382 * no processor is currently accessing the user address space. In 5383 * particular, a page table entry's dirty bit won't change state once 5384 * this function starts. 5385 * 5386 * Although this function destroys all of the pmap's managed, 5387 * non-wired mappings, it can delay and batch the invalidation of TLB 5388 * entries without calling pmap_delayed_invl_started() and 5389 * pmap_delayed_invl_finished(). Because the pmap is not active on 5390 * any other processor, none of these TLB entries will ever be used 5391 * before their eventual invalidation. Consequently, there is no need 5392 * for either pmap_remove_all() or pmap_remove_write() to wait for 5393 * that eventual TLB invalidation. 5394 */ 5395 5396 void 5397 mmu_radix_remove_pages(pmap_t pmap) 5398 { 5399 5400 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 5401 pml3_entry_t ptel3e; 5402 pt_entry_t *pte, tpte; 5403 struct spglist free; 5404 vm_page_t m, mpte, mt; 5405 pv_entry_t pv; 5406 struct md_page *pvh; 5407 struct pv_chunk *pc, *npc; 5408 struct rwlock *lock; 5409 int64_t bit; 5410 uint64_t inuse, bitmask; 5411 int allfree, field, idx; 5412 #ifdef PV_STATS 5413 int freed; 5414 #endif 5415 boolean_t superpage; 5416 vm_paddr_t pa; 5417 5418 /* 5419 * Assert that the given pmap is only active on the current 5420 * CPU. Unfortunately, we cannot block another CPU from 5421 * activating the pmap while this function is executing. 5422 */ 5423 KASSERT(pmap->pm_pid == mfspr(SPR_PID), 5424 ("non-current asid %lu - expected %lu", pmap->pm_pid, 5425 mfspr(SPR_PID))); 5426 5427 lock = NULL; 5428 5429 SLIST_INIT(&free); 5430 PMAP_LOCK(pmap); 5431 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 5432 allfree = 1; 5433 #ifdef PV_STATS 5434 freed = 0; 5435 #endif 5436 for (field = 0; field < _NPCM; field++) { 5437 inuse = ~pc->pc_map[field] & pc_freemask[field]; 5438 while (inuse != 0) { 5439 bit = cnttzd(inuse); 5440 bitmask = 1UL << bit; 5441 idx = field * 64 + bit; 5442 pv = &pc->pc_pventry[idx]; 5443 inuse &= ~bitmask; 5444 5445 pte = pmap_pml2e(pmap, pv->pv_va); 5446 ptel3e = be64toh(*pte); 5447 pte = pmap_l2e_to_l3e(pte, pv->pv_va); 5448 tpte = be64toh(*pte); 5449 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) { 5450 superpage = FALSE; 5451 ptel3e = tpte; 5452 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & 5453 PG_FRAME); 5454 pte = &pte[pmap_pte_index(pv->pv_va)]; 5455 tpte = be64toh(*pte); 5456 } else { 5457 /* 5458 * Keep track whether 'tpte' is a 5459 * superpage explicitly instead of 5460 * relying on RPTE_LEAF being set. 5461 * 5462 * This is because RPTE_LEAF is numerically 5463 * identical to PG_PTE_PAT and thus a 5464 * regular page could be mistaken for 5465 * a superpage. 5466 */ 5467 superpage = TRUE; 5468 } 5469 5470 if ((tpte & PG_V) == 0) { 5471 panic("bad pte va %lx pte %lx", 5472 pv->pv_va, tpte); 5473 } 5474 5475 /* 5476 * We cannot remove wired pages from a process' mapping at this time 5477 */ 5478 if (tpte & PG_W) { 5479 allfree = 0; 5480 continue; 5481 } 5482 5483 if (superpage) 5484 pa = tpte & PG_PS_FRAME; 5485 else 5486 pa = tpte & PG_FRAME; 5487 5488 m = PHYS_TO_VM_PAGE(pa); 5489 KASSERT(m->phys_addr == pa, 5490 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 5491 m, (uintmax_t)m->phys_addr, 5492 (uintmax_t)tpte)); 5493 5494 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 5495 m < &vm_page_array[vm_page_array_size], 5496 ("pmap_remove_pages: bad tpte %#jx", 5497 (uintmax_t)tpte)); 5498 5499 pte_clear(pte); 5500 5501 /* 5502 * Update the vm_page_t clean/reference bits. 5503 */ 5504 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 5505 if (superpage) { 5506 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5507 vm_page_dirty(mt); 5508 } else 5509 vm_page_dirty(m); 5510 } 5511 5512 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 5513 5514 /* Mark free */ 5515 pc->pc_map[field] |= bitmask; 5516 if (superpage) { 5517 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5518 pvh = pa_to_pvh(tpte & PG_PS_FRAME); 5519 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 5520 pvh->pv_gen++; 5521 if (TAILQ_EMPTY(&pvh->pv_list)) { 5522 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5523 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 5524 TAILQ_EMPTY(&mt->md.pv_list)) 5525 vm_page_aflag_clear(mt, PGA_WRITEABLE); 5526 } 5527 mpte = pmap_remove_pt_page(pmap, pv->pv_va); 5528 if (mpte != NULL) { 5529 pmap_resident_count_dec(pmap, 1); 5530 KASSERT(mpte->ref_count == NPTEPG, 5531 ("pmap_remove_pages: pte page wire count error")); 5532 mpte->ref_count = 0; 5533 pmap_add_delayed_free_list(mpte, &free, FALSE); 5534 } 5535 } else { 5536 pmap_resident_count_dec(pmap, 1); 5537 #ifdef VERBOSE_PV 5538 printf("freeing pv (%p, %p)\n", 5539 pmap, pv); 5540 #endif 5541 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5542 m->md.pv_gen++; 5543 if ((m->a.flags & PGA_WRITEABLE) != 0 && 5544 TAILQ_EMPTY(&m->md.pv_list) && 5545 (m->flags & PG_FICTITIOUS) == 0) { 5546 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5547 if (TAILQ_EMPTY(&pvh->pv_list)) 5548 vm_page_aflag_clear(m, PGA_WRITEABLE); 5549 } 5550 } 5551 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free); 5552 #ifdef PV_STATS 5553 freed++; 5554 #endif 5555 } 5556 } 5557 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 5558 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 5559 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 5560 if (allfree) { 5561 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5562 free_pv_chunk(pc); 5563 } 5564 } 5565 if (lock != NULL) 5566 rw_wunlock(lock); 5567 pmap_invalidate_all(pmap); 5568 PMAP_UNLOCK(pmap); 5569 vm_page_free_pages_toq(&free, true); 5570 } 5571 5572 void 5573 mmu_radix_remove_write(vm_page_t m) 5574 { 5575 struct md_page *pvh; 5576 pmap_t pmap; 5577 struct rwlock *lock; 5578 pv_entry_t next_pv, pv; 5579 pml3_entry_t *l3e; 5580 pt_entry_t oldpte, *pte; 5581 int pvh_gen, md_gen; 5582 5583 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5584 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5585 ("pmap_remove_write: page %p is not managed", m)); 5586 vm_page_assert_busied(m); 5587 5588 if (!pmap_page_is_write_mapped(m)) 5589 return; 5590 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5591 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5592 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5593 retry_pv_loop: 5594 rw_wlock(lock); 5595 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 5596 pmap = PV_PMAP(pv); 5597 if (!PMAP_TRYLOCK(pmap)) { 5598 pvh_gen = pvh->pv_gen; 5599 rw_wunlock(lock); 5600 PMAP_LOCK(pmap); 5601 rw_wlock(lock); 5602 if (pvh_gen != pvh->pv_gen) { 5603 PMAP_UNLOCK(pmap); 5604 rw_wunlock(lock); 5605 goto retry_pv_loop; 5606 } 5607 } 5608 l3e = pmap_pml3e(pmap, pv->pv_va); 5609 if ((be64toh(*l3e) & PG_RW) != 0) 5610 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock); 5611 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 5612 ("inconsistent pv lock %p %p for page %p", 5613 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 5614 PMAP_UNLOCK(pmap); 5615 } 5616 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 5617 pmap = PV_PMAP(pv); 5618 if (!PMAP_TRYLOCK(pmap)) { 5619 pvh_gen = pvh->pv_gen; 5620 md_gen = m->md.pv_gen; 5621 rw_wunlock(lock); 5622 PMAP_LOCK(pmap); 5623 rw_wlock(lock); 5624 if (pvh_gen != pvh->pv_gen || 5625 md_gen != m->md.pv_gen) { 5626 PMAP_UNLOCK(pmap); 5627 rw_wunlock(lock); 5628 goto retry_pv_loop; 5629 } 5630 } 5631 l3e = pmap_pml3e(pmap, pv->pv_va); 5632 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 5633 ("pmap_remove_write: found a 2mpage in page %p's pv list", 5634 m)); 5635 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5636 retry: 5637 oldpte = be64toh(*pte); 5638 if (oldpte & PG_RW) { 5639 if (!atomic_cmpset_long(pte, htobe64(oldpte), 5640 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M)))) 5641 goto retry; 5642 if ((oldpte & PG_M) != 0) 5643 vm_page_dirty(m); 5644 pmap_invalidate_page(pmap, pv->pv_va); 5645 } 5646 PMAP_UNLOCK(pmap); 5647 } 5648 rw_wunlock(lock); 5649 vm_page_aflag_clear(m, PGA_WRITEABLE); 5650 } 5651 5652 /* 5653 * Clear the wired attribute from the mappings for the specified range of 5654 * addresses in the given pmap. Every valid mapping within that range 5655 * must have the wired attribute set. In contrast, invalid mappings 5656 * cannot have the wired attribute set, so they are ignored. 5657 * 5658 * The wired attribute of the page table entry is not a hardware 5659 * feature, so there is no need to invalidate any TLB entries. 5660 * Since pmap_demote_l3e() for the wired entry must never fail, 5661 * pmap_delayed_invl_started()/finished() calls around the 5662 * function are not needed. 5663 */ 5664 void 5665 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5666 { 5667 vm_offset_t va_next; 5668 pml1_entry_t *l1e; 5669 pml2_entry_t *l2e; 5670 pml3_entry_t *l3e; 5671 pt_entry_t *pte; 5672 5673 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5674 PMAP_LOCK(pmap); 5675 for (; sva < eva; sva = va_next) { 5676 l1e = pmap_pml1e(pmap, sva); 5677 if ((be64toh(*l1e) & PG_V) == 0) { 5678 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5679 if (va_next < sva) 5680 va_next = eva; 5681 continue; 5682 } 5683 l2e = pmap_l1e_to_l2e(l1e, sva); 5684 if ((be64toh(*l2e) & PG_V) == 0) { 5685 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5686 if (va_next < sva) 5687 va_next = eva; 5688 continue; 5689 } 5690 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5691 if (va_next < sva) 5692 va_next = eva; 5693 l3e = pmap_l2e_to_l3e(l2e, sva); 5694 if ((be64toh(*l3e) & PG_V) == 0) 5695 continue; 5696 if ((be64toh(*l3e) & RPTE_LEAF) != 0) { 5697 if ((be64toh(*l3e) & PG_W) == 0) 5698 panic("pmap_unwire: pde %#jx is missing PG_W", 5699 (uintmax_t)(be64toh(*l3e))); 5700 5701 /* 5702 * Are we unwiring the entire large page? If not, 5703 * demote the mapping and fall through. 5704 */ 5705 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5706 atomic_clear_long(l3e, htobe64(PG_W)); 5707 pmap->pm_stats.wired_count -= L3_PAGE_SIZE / 5708 PAGE_SIZE; 5709 continue; 5710 } else if (!pmap_demote_l3e(pmap, l3e, sva)) 5711 panic("pmap_unwire: demotion failed"); 5712 } 5713 if (va_next > eva) 5714 va_next = eva; 5715 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 5716 sva += PAGE_SIZE) { 5717 MPASS(pte == pmap_pte(pmap, sva)); 5718 if ((be64toh(*pte) & PG_V) == 0) 5719 continue; 5720 if ((be64toh(*pte) & PG_W) == 0) 5721 panic("pmap_unwire: pte %#jx is missing PG_W", 5722 (uintmax_t)(be64toh(*pte))); 5723 5724 /* 5725 * PG_W must be cleared atomically. Although the pmap 5726 * lock synchronizes access to PG_W, another processor 5727 * could be setting PG_M and/or PG_A concurrently. 5728 */ 5729 atomic_clear_long(pte, htobe64(PG_W)); 5730 pmap->pm_stats.wired_count--; 5731 } 5732 } 5733 PMAP_UNLOCK(pmap); 5734 } 5735 5736 void 5737 mmu_radix_zero_page(vm_page_t m) 5738 { 5739 vm_offset_t addr; 5740 5741 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5742 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5743 pagezero(addr); 5744 } 5745 5746 void 5747 mmu_radix_zero_page_area(vm_page_t m, int off, int size) 5748 { 5749 caddr_t addr; 5750 5751 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size); 5752 MPASS(off + size <= PAGE_SIZE); 5753 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5754 memset(addr + off, 0, size); 5755 } 5756 5757 static int 5758 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 5759 { 5760 pml3_entry_t *l3ep; 5761 pt_entry_t pte; 5762 vm_paddr_t pa; 5763 int val; 5764 5765 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 5766 PMAP_LOCK(pmap); 5767 5768 l3ep = pmap_pml3e(pmap, addr); 5769 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) { 5770 if (be64toh(*l3ep) & RPTE_LEAF) { 5771 pte = be64toh(*l3ep); 5772 /* Compute the physical address of the 4KB page. */ 5773 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) & 5774 PG_FRAME; 5775 val = MINCORE_PSIND(1); 5776 } else { 5777 /* Native endian PTE, do not pass to functions */ 5778 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr)); 5779 pa = pte & PG_FRAME; 5780 val = 0; 5781 } 5782 } else { 5783 pte = 0; 5784 pa = 0; 5785 val = 0; 5786 } 5787 if ((pte & PG_V) != 0) { 5788 val |= MINCORE_INCORE; 5789 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5790 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 5791 if ((pte & PG_A) != 0) 5792 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 5793 } 5794 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 5795 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 5796 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { 5797 *locked_pa = pa; 5798 } 5799 PMAP_UNLOCK(pmap); 5800 return (val); 5801 } 5802 5803 void 5804 mmu_radix_activate(struct thread *td) 5805 { 5806 pmap_t pmap; 5807 uint32_t curpid; 5808 5809 CTR2(KTR_PMAP, "%s(%p)", __func__, td); 5810 critical_enter(); 5811 pmap = vmspace_pmap(td->td_proc->p_vmspace); 5812 curpid = mfspr(SPR_PID); 5813 if (pmap->pm_pid > isa3_base_pid && 5814 curpid != pmap->pm_pid) { 5815 mmu_radix_pid_set(pmap); 5816 } 5817 critical_exit(); 5818 } 5819 5820 /* 5821 * Increase the starting virtual address of the given mapping if a 5822 * different alignment might result in more superpage mappings. 5823 */ 5824 void 5825 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset, 5826 vm_offset_t *addr, vm_size_t size) 5827 { 5828 5829 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr, 5830 size); 5831 vm_offset_t superpage_offset; 5832 5833 if (size < L3_PAGE_SIZE) 5834 return; 5835 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 5836 offset += ptoa(object->pg_color); 5837 superpage_offset = offset & L3_PAGE_MASK; 5838 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE || 5839 (*addr & L3_PAGE_MASK) == superpage_offset) 5840 return; 5841 if ((*addr & L3_PAGE_MASK) < superpage_offset) 5842 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset; 5843 else 5844 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset; 5845 } 5846 5847 static void * 5848 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr) 5849 { 5850 vm_offset_t va, tmpva, ppa, offset; 5851 5852 ppa = trunc_page(pa); 5853 offset = pa & PAGE_MASK; 5854 size = roundup2(offset + size, PAGE_SIZE); 5855 if (pa < powerpc_ptob(Maxmem)) 5856 panic("bad pa: %#lx less than Maxmem %#lx\n", 5857 pa, powerpc_ptob(Maxmem)); 5858 va = kva_alloc(size); 5859 if (bootverbose) 5860 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr); 5861 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr)); 5862 5863 if (!va) 5864 panic("%s: Couldn't alloc kernel virtual memory", __func__); 5865 5866 for (tmpva = va; size > 0;) { 5867 mmu_radix_kenter_attr(tmpva, ppa, attr); 5868 size -= PAGE_SIZE; 5869 tmpva += PAGE_SIZE; 5870 ppa += PAGE_SIZE; 5871 } 5872 ptesync(); 5873 5874 return ((void *)(va + offset)); 5875 } 5876 5877 static void * 5878 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size) 5879 { 5880 5881 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 5882 5883 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT)); 5884 } 5885 5886 void 5887 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma) 5888 { 5889 5890 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma); 5891 m->md.mdpg_cache_attrs = ma; 5892 5893 /* 5894 * If "m" is a normal page, update its direct mapping. This update 5895 * can be relied upon to perform any cache operations that are 5896 * required for data coherence. 5897 */ 5898 if ((m->flags & PG_FICTITIOUS) == 0 && 5899 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 5900 PAGE_SIZE, m->md.mdpg_cache_attrs)) 5901 panic("memory attribute change on the direct map failed"); 5902 } 5903 5904 static void 5905 mmu_radix_unmapdev(void *p, vm_size_t size) 5906 { 5907 vm_offset_t offset, va; 5908 5909 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, p, size); 5910 5911 /* If we gave a direct map region in pmap_mapdev, do nothing */ 5912 va = (vm_offset_t)p; 5913 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) 5914 return; 5915 5916 offset = va & PAGE_MASK; 5917 size = round_page(offset + size); 5918 va = trunc_page(va); 5919 5920 if (pmap_initialized) { 5921 mmu_radix_qremove(va, atop(size)); 5922 kva_free(va, size); 5923 } 5924 } 5925 5926 void 5927 mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 5928 { 5929 vm_paddr_t pa = 0; 5930 int sync_sz; 5931 5932 while (sz > 0) { 5933 pa = pmap_extract(pm, va); 5934 sync_sz = PAGE_SIZE - (va & PAGE_MASK); 5935 sync_sz = min(sync_sz, sz); 5936 if (pa != 0) { 5937 pa += (va & PAGE_MASK); 5938 __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz); 5939 } 5940 va += sync_sz; 5941 sz -= sync_sz; 5942 } 5943 } 5944 5945 static __inline void 5946 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask) 5947 { 5948 uint64_t opte, npte; 5949 5950 /* 5951 * The cache mode bits are all in the low 32-bits of the 5952 * PTE, so we can just spin on updating the low 32-bits. 5953 */ 5954 do { 5955 opte = be64toh(*pte); 5956 npte = opte & ~mask; 5957 npte |= cache_bits; 5958 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte))); 5959 } 5960 5961 /* 5962 * Tries to demote a 1GB page mapping. 5963 */ 5964 static boolean_t 5965 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) 5966 { 5967 pml2_entry_t oldpdpe; 5968 pml3_entry_t *firstpde, newpde, *pde; 5969 vm_paddr_t pdpgpa; 5970 vm_page_t pdpg; 5971 5972 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5973 oldpdpe = be64toh(*l2e); 5974 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 5975 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); 5976 pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 5977 if (pdpg == NULL) { 5978 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" 5979 " in pmap %p", va, pmap); 5980 return (FALSE); 5981 } 5982 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT; 5983 pdpgpa = VM_PAGE_TO_PHYS(pdpg); 5984 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); 5985 KASSERT((oldpdpe & PG_A) != 0, 5986 ("pmap_demote_pdpe: oldpdpe is missing PG_A")); 5987 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, 5988 ("pmap_demote_pdpe: oldpdpe is missing PG_M")); 5989 newpde = oldpdpe; 5990 5991 /* 5992 * Initialize the page directory page. 5993 */ 5994 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { 5995 *pde = htobe64(newpde); 5996 newpde += L3_PAGE_SIZE; 5997 } 5998 5999 /* 6000 * Demote the mapping. 6001 */ 6002 pde_store(l2e, pdpgpa); 6003 6004 /* 6005 * Flush PWC --- XXX revisit 6006 */ 6007 pmap_invalidate_all(pmap); 6008 6009 counter_u64_add(pmap_l2e_demotions, 1); 6010 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" 6011 " in pmap %p", va, pmap); 6012 return (TRUE); 6013 } 6014 6015 vm_paddr_t 6016 mmu_radix_kextract(vm_offset_t va) 6017 { 6018 pml3_entry_t l3e; 6019 vm_paddr_t pa; 6020 6021 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6022 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { 6023 pa = DMAP_TO_PHYS(va); 6024 } else { 6025 /* Big-endian PTE on stack */ 6026 l3e = *pmap_pml3e(kernel_pmap, va); 6027 if (be64toh(l3e) & RPTE_LEAF) { 6028 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 6029 pa |= (va & L3_PAGE_MASK); 6030 } else { 6031 /* 6032 * Beware of a concurrent promotion that changes the 6033 * PDE at this point! For example, vtopte() must not 6034 * be used to access the PTE because it would use the 6035 * new PDE. It is, however, safe to use the old PDE 6036 * because the page table page is preserved by the 6037 * promotion. 6038 */ 6039 pa = be64toh(*pmap_l3e_to_pte(&l3e, va)); 6040 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 6041 pa |= (va & PAGE_MASK); 6042 } 6043 } 6044 return (pa); 6045 } 6046 6047 static pt_entry_t 6048 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 6049 { 6050 6051 if (ma != VM_MEMATTR_DEFAULT) { 6052 return pmap_cache_bits(ma); 6053 } 6054 6055 /* 6056 * Assume the page is cache inhibited and access is guarded unless 6057 * it's in our available memory array. 6058 */ 6059 for (int i = 0; i < pregions_sz; i++) { 6060 if ((pa >= pregions[i].mr_start) && 6061 (pa < (pregions[i].mr_start + pregions[i].mr_size))) 6062 return (RPTE_ATTR_MEM); 6063 } 6064 return (RPTE_ATTR_GUARDEDIO); 6065 } 6066 6067 static void 6068 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 6069 { 6070 pt_entry_t *pte, pteval; 6071 uint64_t cache_bits; 6072 6073 pte = kvtopte(va); 6074 MPASS(pte != NULL); 6075 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 6076 cache_bits = mmu_radix_calc_wimg(pa, ma); 6077 pte_store(pte, pteval | cache_bits); 6078 } 6079 6080 void 6081 mmu_radix_kremove(vm_offset_t va) 6082 { 6083 pt_entry_t *pte; 6084 6085 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6086 6087 pte = kvtopte(va); 6088 pte_clear(pte); 6089 } 6090 6091 int 6092 mmu_radix_decode_kernel_ptr(vm_offset_t addr, 6093 int *is_user, vm_offset_t *decoded) 6094 { 6095 6096 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr); 6097 *decoded = addr; 6098 *is_user = (addr < VM_MAXUSER_ADDRESS); 6099 return (0); 6100 } 6101 6102 static boolean_t 6103 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size) 6104 { 6105 6106 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 6107 return (mem_valid(pa, size)); 6108 } 6109 6110 static void 6111 mmu_radix_scan_init(void) 6112 { 6113 6114 CTR1(KTR_PMAP, "%s()", __func__); 6115 UNIMPLEMENTED(); 6116 } 6117 6118 static void 6119 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, 6120 void **va) 6121 { 6122 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va); 6123 UNIMPLEMENTED(); 6124 } 6125 6126 vm_offset_t 6127 mmu_radix_quick_enter_page(vm_page_t m) 6128 { 6129 vm_paddr_t paddr; 6130 6131 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 6132 paddr = VM_PAGE_TO_PHYS(m); 6133 return (PHYS_TO_DMAP(paddr)); 6134 } 6135 6136 void 6137 mmu_radix_quick_remove_page(vm_offset_t addr __unused) 6138 { 6139 /* no work to do here */ 6140 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 6141 } 6142 6143 static void 6144 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 6145 { 6146 cpu_flush_dcache((void *)sva, eva - sva); 6147 } 6148 6149 int 6150 mmu_radix_change_attr(vm_offset_t va, vm_size_t size, 6151 vm_memattr_t mode) 6152 { 6153 int error; 6154 6155 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode); 6156 PMAP_LOCK(kernel_pmap); 6157 error = pmap_change_attr_locked(va, size, mode, true); 6158 PMAP_UNLOCK(kernel_pmap); 6159 return (error); 6160 } 6161 6162 static int 6163 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush) 6164 { 6165 vm_offset_t base, offset, tmpva; 6166 vm_paddr_t pa_start, pa_end, pa_end1; 6167 pml2_entry_t *l2e; 6168 pml3_entry_t *l3e; 6169 pt_entry_t *pte; 6170 int cache_bits, error; 6171 boolean_t changed; 6172 6173 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 6174 base = trunc_page(va); 6175 offset = va & PAGE_MASK; 6176 size = round_page(offset + size); 6177 6178 /* 6179 * Only supported on kernel virtual addresses, including the direct 6180 * map but excluding the recursive map. 6181 */ 6182 if (base < DMAP_MIN_ADDRESS) 6183 return (EINVAL); 6184 6185 cache_bits = pmap_cache_bits(mode); 6186 changed = FALSE; 6187 6188 /* 6189 * Pages that aren't mapped aren't supported. Also break down 2MB pages 6190 * into 4KB pages if required. 6191 */ 6192 for (tmpva = base; tmpva < base + size; ) { 6193 l2e = pmap_pml2e(kernel_pmap, tmpva); 6194 if (l2e == NULL || *l2e == 0) 6195 return (EINVAL); 6196 if (be64toh(*l2e) & RPTE_LEAF) { 6197 /* 6198 * If the current 1GB page already has the required 6199 * memory type, then we need not demote this page. Just 6200 * increment tmpva to the next 1GB page frame. 6201 */ 6202 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) { 6203 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6204 continue; 6205 } 6206 6207 /* 6208 * If the current offset aligns with a 1GB page frame 6209 * and there is at least 1GB left within the range, then 6210 * we need not break down this page into 2MB pages. 6211 */ 6212 if ((tmpva & L2_PAGE_MASK) == 0 && 6213 tmpva + L2_PAGE_MASK < base + size) { 6214 tmpva += L2_PAGE_MASK; 6215 continue; 6216 } 6217 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva)) 6218 return (ENOMEM); 6219 } 6220 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6221 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n", 6222 tmpva, l2e)); 6223 if (*l3e == 0) 6224 return (EINVAL); 6225 if (be64toh(*l3e) & RPTE_LEAF) { 6226 /* 6227 * If the current 2MB page already has the required 6228 * memory type, then we need not demote this page. Just 6229 * increment tmpva to the next 2MB page frame. 6230 */ 6231 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) { 6232 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6233 continue; 6234 } 6235 6236 /* 6237 * If the current offset aligns with a 2MB page frame 6238 * and there is at least 2MB left within the range, then 6239 * we need not break down this page into 4KB pages. 6240 */ 6241 if ((tmpva & L3_PAGE_MASK) == 0 && 6242 tmpva + L3_PAGE_MASK < base + size) { 6243 tmpva += L3_PAGE_SIZE; 6244 continue; 6245 } 6246 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva)) 6247 return (ENOMEM); 6248 } 6249 pte = pmap_l3e_to_pte(l3e, tmpva); 6250 if (*pte == 0) 6251 return (EINVAL); 6252 tmpva += PAGE_SIZE; 6253 } 6254 error = 0; 6255 6256 /* 6257 * Ok, all the pages exist, so run through them updating their 6258 * cache mode if required. 6259 */ 6260 pa_start = pa_end = 0; 6261 for (tmpva = base; tmpva < base + size; ) { 6262 l2e = pmap_pml2e(kernel_pmap, tmpva); 6263 if (be64toh(*l2e) & RPTE_LEAF) { 6264 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) { 6265 pmap_pte_attr(l2e, cache_bits, 6266 RPTE_ATTR_MASK); 6267 changed = TRUE; 6268 } 6269 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6270 (*l2e & PG_PS_FRAME) < dmaplimit) { 6271 if (pa_start == pa_end) { 6272 /* Start physical address run. */ 6273 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6274 pa_end = pa_start + L2_PAGE_SIZE; 6275 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME)) 6276 pa_end += L2_PAGE_SIZE; 6277 else { 6278 /* Run ended, update direct map. */ 6279 error = pmap_change_attr_locked( 6280 PHYS_TO_DMAP(pa_start), 6281 pa_end - pa_start, mode, flush); 6282 if (error != 0) 6283 break; 6284 /* Start physical address run. */ 6285 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6286 pa_end = pa_start + L2_PAGE_SIZE; 6287 } 6288 } 6289 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6290 continue; 6291 } 6292 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6293 if (be64toh(*l3e) & RPTE_LEAF) { 6294 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) { 6295 pmap_pte_attr(l3e, cache_bits, 6296 RPTE_ATTR_MASK); 6297 changed = TRUE; 6298 } 6299 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6300 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) { 6301 if (pa_start == pa_end) { 6302 /* Start physical address run. */ 6303 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6304 pa_end = pa_start + L3_PAGE_SIZE; 6305 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME)) 6306 pa_end += L3_PAGE_SIZE; 6307 else { 6308 /* Run ended, update direct map. */ 6309 error = pmap_change_attr_locked( 6310 PHYS_TO_DMAP(pa_start), 6311 pa_end - pa_start, mode, flush); 6312 if (error != 0) 6313 break; 6314 /* Start physical address run. */ 6315 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6316 pa_end = pa_start + L3_PAGE_SIZE; 6317 } 6318 } 6319 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6320 } else { 6321 pte = pmap_l3e_to_pte(l3e, tmpva); 6322 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) { 6323 pmap_pte_attr(pte, cache_bits, 6324 RPTE_ATTR_MASK); 6325 changed = TRUE; 6326 } 6327 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6328 (be64toh(*pte) & PG_FRAME) < dmaplimit) { 6329 if (pa_start == pa_end) { 6330 /* Start physical address run. */ 6331 pa_start = be64toh(*pte) & PG_FRAME; 6332 pa_end = pa_start + PAGE_SIZE; 6333 } else if (pa_end == (be64toh(*pte) & PG_FRAME)) 6334 pa_end += PAGE_SIZE; 6335 else { 6336 /* Run ended, update direct map. */ 6337 error = pmap_change_attr_locked( 6338 PHYS_TO_DMAP(pa_start), 6339 pa_end - pa_start, mode, flush); 6340 if (error != 0) 6341 break; 6342 /* Start physical address run. */ 6343 pa_start = be64toh(*pte) & PG_FRAME; 6344 pa_end = pa_start + PAGE_SIZE; 6345 } 6346 } 6347 tmpva += PAGE_SIZE; 6348 } 6349 } 6350 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { 6351 pa_end1 = MIN(pa_end, dmaplimit); 6352 if (pa_start != pa_end1) 6353 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), 6354 pa_end1 - pa_start, mode, flush); 6355 } 6356 6357 /* 6358 * Flush CPU caches if required to make sure any data isn't cached that 6359 * shouldn't be, etc. 6360 */ 6361 if (changed) { 6362 pmap_invalidate_all(kernel_pmap); 6363 6364 if (flush) 6365 pmap_invalidate_cache_range(base, tmpva); 6366 } 6367 return (error); 6368 } 6369 6370 /* 6371 * Allocate physical memory for the vm_page array and map it into KVA, 6372 * attempting to back the vm_pages with domain-local memory. 6373 */ 6374 void 6375 mmu_radix_page_array_startup(long pages) 6376 { 6377 #ifdef notyet 6378 pml2_entry_t *l2e; 6379 pml3_entry_t *pde; 6380 pml3_entry_t newl3; 6381 vm_offset_t va; 6382 long pfn; 6383 int domain, i; 6384 #endif 6385 vm_paddr_t pa; 6386 vm_offset_t start, end; 6387 6388 vm_page_array_size = pages; 6389 6390 start = VM_MIN_KERNEL_ADDRESS; 6391 end = start + pages * sizeof(struct vm_page); 6392 6393 pa = vm_phys_early_alloc(0, end - start); 6394 6395 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT); 6396 #ifdef notyet 6397 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */ 6398 for (va = start; va < end; va += L3_PAGE_SIZE) { 6399 pfn = first_page + (va - start) / sizeof(struct vm_page); 6400 domain = vm_phys_domain(ptoa(pfn)); 6401 l2e = pmap_pml2e(kernel_pmap, va); 6402 if ((be64toh(*l2e) & PG_V) == 0) { 6403 pa = vm_phys_early_alloc(domain, PAGE_SIZE); 6404 dump_add_page(pa); 6405 pagezero(PHYS_TO_DMAP(pa)); 6406 pde_store(l2e, (pml2_entry_t)pa); 6407 } 6408 pde = pmap_l2e_to_l3e(l2e, va); 6409 if ((be64toh(*pde) & PG_V) != 0) 6410 panic("Unexpected pde %p", pde); 6411 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE); 6412 for (i = 0; i < NPDEPG; i++) 6413 dump_add_page(pa + i * PAGE_SIZE); 6414 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W); 6415 pte_store(pde, newl3); 6416 } 6417 #endif 6418 vm_page_array = (vm_page_t)start; 6419 } 6420 6421 #ifdef DDB 6422 #include <sys/kdb.h> 6423 #include <ddb/ddb.h> 6424 6425 static void 6426 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va) 6427 { 6428 pml1_entry_t *l1e; 6429 pml2_entry_t *l2e; 6430 pml3_entry_t *l3e; 6431 pt_entry_t *pte; 6432 6433 l1e = &l1[pmap_pml1e_index(va)]; 6434 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e)); 6435 if ((be64toh(*l1e) & PG_V) == 0) { 6436 db_printf("\n"); 6437 return; 6438 } 6439 l2e = pmap_l1e_to_l2e(l1e, va); 6440 db_printf(" l2e %#016lx", be64toh(*l2e)); 6441 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) { 6442 db_printf("\n"); 6443 return; 6444 } 6445 l3e = pmap_l2e_to_l3e(l2e, va); 6446 db_printf(" l3e %#016lx", be64toh(*l3e)); 6447 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) { 6448 db_printf("\n"); 6449 return; 6450 } 6451 pte = pmap_l3e_to_pte(l3e, va); 6452 db_printf(" pte %#016lx\n", be64toh(*pte)); 6453 } 6454 6455 void 6456 pmap_page_print_mappings(vm_page_t m) 6457 { 6458 pmap_t pmap; 6459 pv_entry_t pv; 6460 6461 db_printf("page %p(%lx)\n", m, m->phys_addr); 6462 /* need to elide locks if running in ddb */ 6463 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 6464 db_printf("pv: %p ", pv); 6465 db_printf("va: %#016lx ", pv->pv_va); 6466 pmap = PV_PMAP(pv); 6467 db_printf("pmap %p ", pmap); 6468 if (pmap != NULL) { 6469 db_printf("asid: %lu\n", pmap->pm_pid); 6470 pmap_pte_walk(pmap->pm_pml1, pv->pv_va); 6471 } 6472 } 6473 } 6474 6475 DB_SHOW_COMMAND(pte, pmap_print_pte) 6476 { 6477 vm_offset_t va; 6478 pmap_t pmap; 6479 6480 if (!have_addr) { 6481 db_printf("show pte addr\n"); 6482 return; 6483 } 6484 va = (vm_offset_t)addr; 6485 6486 if (va >= DMAP_MIN_ADDRESS) 6487 pmap = kernel_pmap; 6488 else if (kdb_thread != NULL) 6489 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); 6490 else 6491 pmap = vmspace_pmap(curthread->td_proc->p_vmspace); 6492 6493 pmap_pte_walk(pmap->pm_pml1, va); 6494 } 6495 6496 #endif 6497