1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2018 Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include "opt_platform.h" 29 30 #include <sys/param.h> 31 #include <sys/kernel.h> 32 #include <sys/systm.h> 33 #include <sys/conf.h> 34 #include <sys/bitstring.h> 35 #include <sys/queue.h> 36 #include <sys/cpuset.h> 37 #include <sys/endian.h> 38 #include <sys/kerneldump.h> 39 #include <sys/ktr.h> 40 #include <sys/lock.h> 41 #include <sys/syslog.h> 42 #include <sys/msgbuf.h> 43 #include <sys/malloc.h> 44 #include <sys/mman.h> 45 #include <sys/mutex.h> 46 #include <sys/proc.h> 47 #include <sys/rwlock.h> 48 #include <sys/sched.h> 49 #include <sys/sysctl.h> 50 #include <sys/systm.h> 51 #include <sys/vmem.h> 52 #include <sys/vmmeter.h> 53 #include <sys/smp.h> 54 55 #include <sys/kdb.h> 56 57 #include <dev/ofw/openfirm.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #include <vm/vm_param.h> 62 #include <vm/vm_kern.h> 63 #include <vm/vm_page.h> 64 #include <vm/vm_map.h> 65 #include <vm/vm_object.h> 66 #include <vm/vm_extern.h> 67 #include <vm/vm_pageout.h> 68 #include <vm/vm_phys.h> 69 #include <vm/vm_radix.h> 70 #include <vm/vm_reserv.h> 71 #include <vm/vm_dumpset.h> 72 #include <vm/uma.h> 73 74 #include <machine/_inttypes.h> 75 #include <machine/cpu.h> 76 #include <machine/platform.h> 77 #include <machine/frame.h> 78 #include <machine/md_var.h> 79 #include <machine/psl.h> 80 #include <machine/bat.h> 81 #include <machine/hid.h> 82 #include <machine/pte.h> 83 #include <machine/sr.h> 84 #include <machine/trap.h> 85 #include <machine/mmuvar.h> 86 87 /* For pseries bit. */ 88 #include <powerpc/pseries/phyp-hvcall.h> 89 90 #ifdef INVARIANTS 91 #include <vm/uma_dbg.h> 92 #endif 93 94 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit)) 95 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) 96 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit)) 97 98 #include "opt_ddb.h" 99 100 #ifdef DDB 101 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va); 102 #endif 103 104 #define PG_W RPTE_WIRED 105 #define PG_V RPTE_VALID 106 #define PG_MANAGED RPTE_MANAGED 107 #define PG_PROMOTED RPTE_PROMOTED 108 #define PG_M RPTE_C 109 #define PG_A RPTE_R 110 #define PG_X RPTE_EAA_X 111 #define PG_RW RPTE_EAA_W 112 #define PG_PTE_CACHE RPTE_ATTR_MASK 113 114 #define RPTE_SHIFT 9 115 #define NLS_MASK ((1UL<<5)-1) 116 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT) 117 #define RPTE_MASK (RPTE_ENTRIES-1) 118 119 #define NLB_SHIFT 0 120 #define NLB_MASK (((1UL<<52)-1) << 8) 121 122 extern int nkpt; 123 extern caddr_t crashdumpmap; 124 125 #define RIC_FLUSH_TLB 0 126 #define RIC_FLUSH_PWC 1 127 #define RIC_FLUSH_ALL 2 128 129 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ 130 131 #define PPC_INST_TLBIE 0x7c000264 132 #define PPC_INST_TLBIEL 0x7c000224 133 #define PPC_INST_SLBIA 0x7c0003e4 134 135 #define ___PPC_RA(a) (((a) & 0x1f) << 16) 136 #define ___PPC_RB(b) (((b) & 0x1f) << 11) 137 #define ___PPC_RS(s) (((s) & 0x1f) << 21) 138 #define ___PPC_RT(t) ___PPC_RS(t) 139 #define ___PPC_R(r) (((r) & 0x1) << 16) 140 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17) 141 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18) 142 143 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \ 144 ((IH & 0x7) << 21)) 145 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \ 146 __XSTRING(.long PPC_INST_TLBIE | \ 147 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 148 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 149 ___PPC_R(r)) 150 151 #define PPC_TLBIEL(rb,rs,ric,prs,r) \ 152 __XSTRING(.long PPC_INST_TLBIEL | \ 153 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 154 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 155 ___PPC_R(r)) 156 157 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7) 158 159 static __inline void 160 ttusync(void) 161 { 162 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 163 } 164 165 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ 166 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ 167 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */ 168 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ 169 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ 170 171 #define TLBIE_ACTUAL_PAGE_MASK 0xe0 172 #define TLBIE_ACTUAL_PAGE_4K 0x00 173 #define TLBIE_ACTUAL_PAGE_64K 0xa0 174 #define TLBIE_ACTUAL_PAGE_2M 0x20 175 #define TLBIE_ACTUAL_PAGE_1G 0x40 176 177 #define TLBIE_PRS_PARTITION_SCOPE 0x0 178 #define TLBIE_PRS_PROCESS_SCOPE 0x1 179 180 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */ 181 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */ 182 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC, 183 * cached {proc, part}tab entries 184 */ 185 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only: 186 * Invalidate a range of translations 187 */ 188 189 static __always_inline void 190 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid, 191 vm_offset_t va, uint16_t ap) 192 { 193 uint64_t rb, rs; 194 195 MPASS((va & PAGE_MASK) == 0); 196 197 rs = ((uint64_t)pid << 32) | lpid; 198 rb = va | is | ap; 199 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : : 200 "r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory"); 201 } 202 203 static __inline void 204 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap) 205 { 206 207 __asm __volatile("ptesync" ::: "memory"); 208 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 209 TLBIEL_INVAL_PAGE, 0, 0, va, ap); 210 __asm __volatile("ptesync" ::: "memory"); 211 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 212 TLBIEL_INVAL_PAGE, pid, 0, va, ap); 213 } 214 215 static __inline void 216 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va) 217 { 218 219 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 220 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K); 221 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K); 222 } 223 224 static __inline void 225 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va) 226 { 227 228 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 229 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M); 230 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M); 231 } 232 233 static __inline void 234 radix_tlbie_invlpwc_user(uint32_t pid) 235 { 236 237 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 238 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 239 } 240 241 static __inline void 242 radix_tlbie_flush_user(uint32_t pid) 243 { 244 245 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 246 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 247 } 248 249 static __inline void 250 radix_tlbie_invlpg_kernel_4k(vm_offset_t va) 251 { 252 253 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 254 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K); 255 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K); 256 } 257 258 static __inline void 259 radix_tlbie_invlpg_kernel_2m(vm_offset_t va) 260 { 261 262 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 263 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M); 264 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M); 265 } 266 267 /* 1GB pages aren't currently supported. */ 268 static __inline __unused void 269 radix_tlbie_invlpg_kernel_1g(vm_offset_t va) 270 { 271 272 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 273 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G); 274 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G); 275 } 276 277 static __inline void 278 radix_tlbie_invlpwc_kernel(void) 279 { 280 281 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 282 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 283 } 284 285 static __inline void 286 radix_tlbie_flush_kernel(void) 287 { 288 289 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 290 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 291 } 292 293 static __inline vm_pindex_t 294 pmap_l3e_pindex(vm_offset_t va) 295 { 296 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT); 297 } 298 299 static __inline vm_pindex_t 300 pmap_pml3e_index(vm_offset_t va) 301 { 302 303 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK); 304 } 305 306 static __inline vm_pindex_t 307 pmap_pml2e_index(vm_offset_t va) 308 { 309 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK); 310 } 311 312 static __inline vm_pindex_t 313 pmap_pml1e_index(vm_offset_t va) 314 { 315 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT); 316 } 317 318 /* Return various clipped indexes for a given VA */ 319 static __inline vm_pindex_t 320 pmap_pte_index(vm_offset_t va) 321 { 322 323 return ((va >> PAGE_SHIFT) & RPTE_MASK); 324 } 325 326 /* Return a pointer to the PT slot that corresponds to a VA */ 327 static __inline pt_entry_t * 328 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va) 329 { 330 pt_entry_t *pte; 331 vm_paddr_t ptepa; 332 333 ptepa = (be64toh(*l3e) & NLB_MASK); 334 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa); 335 return (&pte[pmap_pte_index(va)]); 336 } 337 338 /* Return a pointer to the PD slot that corresponds to a VA */ 339 static __inline pt_entry_t * 340 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va) 341 { 342 pt_entry_t *l3e; 343 vm_paddr_t l3pa; 344 345 l3pa = (be64toh(*l2e) & NLB_MASK); 346 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa); 347 return (&l3e[pmap_pml3e_index(va)]); 348 } 349 350 /* Return a pointer to the PD slot that corresponds to a VA */ 351 static __inline pt_entry_t * 352 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va) 353 { 354 pt_entry_t *l2e; 355 vm_paddr_t l2pa; 356 357 l2pa = (be64toh(*l1e) & NLB_MASK); 358 359 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa); 360 return (&l2e[pmap_pml2e_index(va)]); 361 } 362 363 static __inline pml1_entry_t * 364 pmap_pml1e(pmap_t pmap, vm_offset_t va) 365 { 366 367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]); 368 } 369 370 static pt_entry_t * 371 pmap_pml2e(pmap_t pmap, vm_offset_t va) 372 { 373 pt_entry_t *l1e; 374 375 l1e = pmap_pml1e(pmap, va); 376 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0) 377 return (NULL); 378 return (pmap_l1e_to_l2e(l1e, va)); 379 } 380 381 static __inline pt_entry_t * 382 pmap_pml3e(pmap_t pmap, vm_offset_t va) 383 { 384 pt_entry_t *l2e; 385 386 l2e = pmap_pml2e(pmap, va); 387 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0) 388 return (NULL); 389 return (pmap_l2e_to_l3e(l2e, va)); 390 } 391 392 static __inline pt_entry_t * 393 pmap_pte(pmap_t pmap, vm_offset_t va) 394 { 395 pt_entry_t *l3e; 396 397 l3e = pmap_pml3e(pmap, va); 398 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 399 return (NULL); 400 return (pmap_l3e_to_pte(l3e, va)); 401 } 402 403 int nkpt = 64; 404 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, 405 "Number of kernel page table pages allocated on bootup"); 406 407 vm_paddr_t dmaplimit; 408 409 SYSCTL_DECL(_vm_pmap); 410 411 #ifdef INVARIANTS 412 #define VERBOSE_PMAP 0 413 #define VERBOSE_PROTECT 0 414 static int pmap_logging; 415 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN, 416 &pmap_logging, 0, "verbose debug logging"); 417 #endif 418 419 static u_int64_t KPTphys; /* phys addr of kernel level 1 */ 420 421 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ 422 423 static vm_offset_t qframe = 0; 424 static struct mtx qframe_mtx; 425 426 void mmu_radix_activate(struct thread *); 427 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int); 428 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *, 429 vm_size_t); 430 void mmu_radix_clear_modify(vm_page_t); 431 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); 432 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *); 433 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); 434 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 435 vm_prot_t); 436 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 437 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va); 438 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t); 439 void mmu_radix_kenter(vm_offset_t, vm_paddr_t); 440 vm_paddr_t mmu_radix_kextract(vm_offset_t); 441 void mmu_radix_kremove(vm_offset_t); 442 boolean_t mmu_radix_is_modified(vm_page_t); 443 boolean_t mmu_radix_is_prefaultable(pmap_t, vm_offset_t); 444 boolean_t mmu_radix_is_referenced(vm_page_t); 445 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t, 446 vm_pindex_t, vm_size_t); 447 boolean_t mmu_radix_page_exists_quick(pmap_t, vm_page_t); 448 void mmu_radix_page_init(vm_page_t); 449 boolean_t mmu_radix_page_is_mapped(vm_page_t m); 450 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t); 451 int mmu_radix_page_wired_mappings(vm_page_t); 452 int mmu_radix_pinit(pmap_t); 453 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 454 bool mmu_radix_ps_enabled(pmap_t); 455 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int); 456 void mmu_radix_qremove(vm_offset_t, int); 457 vm_offset_t mmu_radix_quick_enter_page(vm_page_t); 458 void mmu_radix_quick_remove_page(vm_offset_t); 459 int mmu_radix_ts_referenced(vm_page_t); 460 void mmu_radix_release(pmap_t); 461 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t); 462 void mmu_radix_remove_all(vm_page_t); 463 void mmu_radix_remove_pages(pmap_t); 464 void mmu_radix_remove_write(vm_page_t); 465 void mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz); 466 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t); 467 void mmu_radix_zero_page(vm_page_t); 468 void mmu_radix_zero_page_area(vm_page_t, int, int); 469 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); 470 void mmu_radix_page_array_startup(long pages); 471 472 #include "mmu_oea64.h" 473 474 /* 475 * Kernel MMU interface 476 */ 477 478 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t); 479 480 static void mmu_radix_copy_page(vm_page_t, vm_page_t); 481 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 482 vm_page_t *mb, vm_offset_t b_offset, int xfersize); 483 static void mmu_radix_growkernel(vm_offset_t); 484 static void mmu_radix_init(void); 485 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *); 486 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 487 static void mmu_radix_pinit0(pmap_t); 488 489 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t); 490 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); 491 static void mmu_radix_unmapdev(void *, vm_size_t); 492 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma); 493 static int mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t); 494 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va); 495 static void mmu_radix_scan_init(void); 496 static void mmu_radix_cpu_bootstrap(int ap); 497 static void mmu_radix_tlbie_all(void); 498 499 static struct pmap_funcs mmu_radix_methods = { 500 .bootstrap = mmu_radix_bootstrap, 501 .copy_page = mmu_radix_copy_page, 502 .copy_pages = mmu_radix_copy_pages, 503 .cpu_bootstrap = mmu_radix_cpu_bootstrap, 504 .growkernel = mmu_radix_growkernel, 505 .init = mmu_radix_init, 506 .map = mmu_radix_map, 507 .mincore = mmu_radix_mincore, 508 .pinit = mmu_radix_pinit, 509 .pinit0 = mmu_radix_pinit0, 510 511 .mapdev = mmu_radix_mapdev, 512 .mapdev_attr = mmu_radix_mapdev_attr, 513 .unmapdev = mmu_radix_unmapdev, 514 .kenter_attr = mmu_radix_kenter_attr, 515 .dev_direct_mapped = mmu_radix_dev_direct_mapped, 516 .dumpsys_pa_init = mmu_radix_scan_init, 517 .dumpsys_map_chunk = mmu_radix_dumpsys_map, 518 .page_is_mapped = mmu_radix_page_is_mapped, 519 .ps_enabled = mmu_radix_ps_enabled, 520 .align_superpage = mmu_radix_align_superpage, 521 .object_init_pt = mmu_radix_object_init_pt, 522 .protect = mmu_radix_protect, 523 /* pmap dispatcher interface */ 524 .clear_modify = mmu_radix_clear_modify, 525 .copy = mmu_radix_copy, 526 .enter = mmu_radix_enter, 527 .enter_object = mmu_radix_enter_object, 528 .enter_quick = mmu_radix_enter_quick, 529 .extract = mmu_radix_extract, 530 .extract_and_hold = mmu_radix_extract_and_hold, 531 .is_modified = mmu_radix_is_modified, 532 .is_prefaultable = mmu_radix_is_prefaultable, 533 .is_referenced = mmu_radix_is_referenced, 534 .ts_referenced = mmu_radix_ts_referenced, 535 .page_exists_quick = mmu_radix_page_exists_quick, 536 .page_init = mmu_radix_page_init, 537 .page_wired_mappings = mmu_radix_page_wired_mappings, 538 .qenter = mmu_radix_qenter, 539 .qremove = mmu_radix_qremove, 540 .release = mmu_radix_release, 541 .remove = mmu_radix_remove, 542 .remove_all = mmu_radix_remove_all, 543 .remove_write = mmu_radix_remove_write, 544 .sync_icache = mmu_radix_sync_icache, 545 .unwire = mmu_radix_unwire, 546 .zero_page = mmu_radix_zero_page, 547 .zero_page_area = mmu_radix_zero_page_area, 548 .activate = mmu_radix_activate, 549 .quick_enter_page = mmu_radix_quick_enter_page, 550 .quick_remove_page = mmu_radix_quick_remove_page, 551 .page_set_memattr = mmu_radix_page_set_memattr, 552 .page_array_startup = mmu_radix_page_array_startup, 553 554 /* Internal interfaces */ 555 .kenter = mmu_radix_kenter, 556 .kextract = mmu_radix_kextract, 557 .kremove = mmu_radix_kremove, 558 .change_attr = mmu_radix_change_attr, 559 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr, 560 561 .tlbie_all = mmu_radix_tlbie_all, 562 }; 563 564 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods); 565 566 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 567 struct rwlock **lockp); 568 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va); 569 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *); 570 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 571 struct spglist *free, struct rwlock **lockp); 572 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 573 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 574 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 575 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde, 576 struct spglist *free); 577 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 578 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp); 579 580 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e, 581 u_int flags, struct rwlock **lockp); 582 #if VM_NRESERVLEVEL > 0 583 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 584 struct rwlock **lockp); 585 #endif 586 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 587 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); 588 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 589 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate); 590 591 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, 592 vm_prot_t prot, struct rwlock **lockp); 593 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, 594 u_int flags, vm_page_t m, struct rwlock **lockp); 595 596 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 597 static void free_pv_chunk(struct pv_chunk *pc); 598 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); 599 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va, 600 struct rwlock **lockp); 601 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, 602 struct rwlock **lockp); 603 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, 604 struct spglist *free); 605 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); 606 607 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start); 608 static void pmap_invalidate_all(pmap_t pmap); 609 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush); 610 611 /* 612 * Internal flags for pmap_enter()'s helper functions. 613 */ 614 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 615 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 616 617 #define UNIMPLEMENTED() panic("%s not implemented", __func__) 618 #define UNTESTED() panic("%s not yet tested", __func__) 619 620 /* Number of supported PID bits */ 621 static unsigned int isa3_pid_bits; 622 623 /* PID to start allocating from */ 624 static unsigned int isa3_base_pid; 625 626 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4) 627 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits) 628 629 /* 630 * Map of physical memory regions. 631 */ 632 static struct mem_region *regions, *pregions; 633 static struct numa_mem_region *numa_pregions; 634 static u_int phys_avail_count; 635 static int regions_sz, pregions_sz, numa_pregions_sz; 636 static struct pate *isa3_parttab; 637 static struct prte *isa3_proctab; 638 static vmem_t *asid_arena; 639 640 extern void bs_remap_earlyboot(void); 641 642 #define RADIX_PGD_SIZE_SHIFT 16 643 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT) 644 645 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3) 646 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t)) 647 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t)) 648 649 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */ 650 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */ 651 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */ 652 653 /* POWER9 only permits a 64k partition table size. */ 654 #define PARTTAB_SIZE_SHIFT 16 655 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT) 656 657 #define PARTTAB_HR (1UL << 63) /* host uses radix */ 658 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */ 659 660 /* TLB flush actions. Used as argument to tlbiel_flush() */ 661 enum { 662 TLB_INVAL_SCOPE_LPID = 2, /* invalidate TLBs for current LPID */ 663 TLB_INVAL_SCOPE_GLOBAL = 3, /* invalidate all TLBs */ 664 }; 665 666 #define NPV_LIST_LOCKS MAXCPU 667 static int pmap_initialized; 668 static vm_paddr_t proctab0pa; 669 static vm_paddr_t parttab_phys; 670 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 671 672 /* 673 * Data for the pv entry allocation mechanism. 674 * Updates to pv_invl_gen are protected by the pv_list_locks[] 675 * elements, but reads are not. 676 */ 677 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 678 static struct mtx __exclusive_cache_line pv_chunks_mutex; 679 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; 680 static struct md_page *pv_table; 681 static struct md_page pv_dummy; 682 683 #ifdef PV_STATS 684 #define PV_STAT(x) do { x ; } while (0) 685 #else 686 #define PV_STAT(x) do { } while (0) 687 #endif 688 689 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT) 690 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)]) 691 692 #define PHYS_TO_PV_LIST_LOCK(pa) \ 693 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS]) 694 695 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 696 struct rwlock **_lockp = (lockp); \ 697 struct rwlock *_new_lock; \ 698 \ 699 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 700 if (_new_lock != *_lockp) { \ 701 if (*_lockp != NULL) \ 702 rw_wunlock(*_lockp); \ 703 *_lockp = _new_lock; \ 704 rw_wlock(*_lockp); \ 705 } \ 706 } while (0) 707 708 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 709 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 710 711 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 712 struct rwlock **_lockp = (lockp); \ 713 \ 714 if (*_lockp != NULL) { \ 715 rw_wunlock(*_lockp); \ 716 *_lockp = NULL; \ 717 } \ 718 } while (0) 719 720 #define VM_PAGE_TO_PV_LIST_LOCK(m) \ 721 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 722 723 /* 724 * We support 52 bits, hence: 725 * bits 52 - 31 = 21, 0b10101 726 * RTS encoding details 727 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long 728 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long 729 */ 730 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5)) 731 732 static int powernv_enabled = 1; 733 734 static __always_inline void 735 tlbiel_radix_set_isa300(uint32_t set, uint32_t is, 736 uint32_t pid, uint32_t ric, uint32_t prs) 737 { 738 uint64_t rb; 739 uint64_t rs; 740 741 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53); 742 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31); 743 744 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 745 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 746 : "memory"); 747 } 748 749 static void 750 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is) 751 { 752 uint32_t set; 753 754 __asm __volatile("ptesync": : :"memory"); 755 756 /* 757 * Flush the first set of the TLB, and the entire Page Walk Cache 758 * and partition table entries. Then flush the remaining sets of the 759 * TLB. 760 */ 761 if (is == TLB_INVAL_SCOPE_GLOBAL) { 762 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 763 for (set = 1; set < num_sets; set++) 764 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 765 } 766 767 /* Do the same for process scoped entries. */ 768 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 769 for (set = 1; set < num_sets; set++) 770 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 771 772 __asm __volatile("ptesync": : :"memory"); 773 } 774 775 static void 776 mmu_radix_tlbiel_flush(int scope) 777 { 778 MPASS(scope == TLB_INVAL_SCOPE_LPID || 779 scope == TLB_INVAL_SCOPE_GLOBAL); 780 781 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, scope); 782 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); 783 } 784 785 static void 786 mmu_radix_tlbie_all(void) 787 { 788 if (powernv_enabled) 789 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 790 else 791 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 792 } 793 794 static void 795 mmu_radix_init_amor(void) 796 { 797 /* 798 * In HV mode, we init AMOR (Authority Mask Override Register) so that 799 * the hypervisor and guest can setup IAMR (Instruction Authority Mask 800 * Register), enable key 0 and set it to 1. 801 * 802 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11) 803 */ 804 mtspr(SPR_AMOR, (3ul << 62)); 805 } 806 807 static void 808 mmu_radix_init_iamr(void) 809 { 810 /* 811 * Radix always uses key0 of the IAMR to determine if an access is 812 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 813 * fetch. 814 */ 815 mtspr(SPR_IAMR, (1ul << 62)); 816 } 817 818 static void 819 mmu_radix_pid_set(pmap_t pmap) 820 { 821 822 mtspr(SPR_PID, pmap->pm_pid); 823 isync(); 824 } 825 826 /* Quick sort callout for comparing physical addresses. */ 827 static int 828 pa_cmp(const void *a, const void *b) 829 { 830 const vm_paddr_t *pa = a, *pb = b; 831 832 if (*pa < *pb) 833 return (-1); 834 else if (*pa > *pb) 835 return (1); 836 else 837 return (0); 838 } 839 840 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) 841 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0) 842 #define pte_store(ptep, pte) do { \ 843 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \ 844 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \ 845 } while (0) 846 /* 847 * NB: should only be used for adding directories - not for direct mappings 848 */ 849 #define pde_store(ptep, pa) do { \ 850 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \ 851 } while (0) 852 853 #define pte_clear(ptep) do { \ 854 *(u_long *)(ptep) = (u_long)(0); \ 855 } while (0) 856 857 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */ 858 859 /* 860 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB 861 * (PTE) page mappings have identical settings for the following fields: 862 */ 863 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \ 864 PG_M | PG_A | RPTE_EAA_MASK | PG_V) 865 866 static __inline void 867 pmap_resident_count_inc(pmap_t pmap, int count) 868 { 869 870 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 871 pmap->pm_stats.resident_count += count; 872 } 873 874 static __inline void 875 pmap_resident_count_dec(pmap_t pmap, int count) 876 { 877 878 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 879 KASSERT(pmap->pm_stats.resident_count >= count, 880 ("pmap %p resident count underflow %ld %d", pmap, 881 pmap->pm_stats.resident_count, count)); 882 pmap->pm_stats.resident_count -= count; 883 } 884 885 static void 886 pagezero(vm_offset_t va) 887 { 888 va = trunc_page(va); 889 890 bzero((void *)va, PAGE_SIZE); 891 } 892 893 static uint64_t 894 allocpages(int n) 895 { 896 u_int64_t ret; 897 898 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE); 899 for (int i = 0; i < n; i++) 900 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE)); 901 return (ret); 902 } 903 904 static pt_entry_t * 905 kvtopte(vm_offset_t va) 906 { 907 pt_entry_t *l3e; 908 909 l3e = pmap_pml3e(kernel_pmap, va); 910 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 911 return (NULL); 912 return (pmap_l3e_to_pte(l3e, va)); 913 } 914 915 void 916 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa) 917 { 918 pt_entry_t *pte; 919 920 pte = kvtopte(va); 921 MPASS(pte != NULL); 922 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \ 923 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A); 924 } 925 926 bool 927 mmu_radix_ps_enabled(pmap_t pmap) 928 { 929 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); 930 } 931 932 static pt_entry_t * 933 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e) 934 { 935 pml3_entry_t *l3e; 936 pt_entry_t *pte; 937 938 va &= PG_PS_FRAME; 939 l3e = pmap_pml3e(pmap, va); 940 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0) 941 return (NULL); 942 943 if (be64toh(*l3e) & RPTE_LEAF) { 944 *is_l3e = 1; 945 return (l3e); 946 } 947 *is_l3e = 0; 948 va &= PG_FRAME; 949 pte = pmap_l3e_to_pte(l3e, va); 950 if (pte == NULL || (be64toh(*pte) & PG_V) == 0) 951 return (NULL); 952 return (pte); 953 } 954 955 int 956 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags) 957 { 958 pt_entry_t *pte; 959 pt_entry_t startpte, origpte, newpte; 960 vm_page_t m; 961 int is_l3e; 962 963 startpte = 0; 964 retry: 965 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) 966 return (KERN_INVALID_ADDRESS); 967 origpte = newpte = be64toh(*pte); 968 if (startpte == 0) { 969 startpte = origpte; 970 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) || 971 ((flags & VM_PROT_READ) && (startpte & PG_A))) { 972 pmap_invalidate_all(pmap); 973 #ifdef INVARIANTS 974 if (VERBOSE_PMAP || pmap_logging) 975 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n", 976 __func__, pmap, va, flags, origpte); 977 #endif 978 return (KERN_FAILURE); 979 } 980 } 981 #ifdef INVARIANTS 982 if (VERBOSE_PMAP || pmap_logging) 983 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va, 984 flags, origpte); 985 #endif 986 PMAP_LOCK(pmap); 987 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || 988 be64toh(*pte) != origpte) { 989 PMAP_UNLOCK(pmap); 990 return (KERN_FAILURE); 991 } 992 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME); 993 MPASS(m != NULL); 994 switch (flags) { 995 case VM_PROT_READ: 996 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0) 997 goto protfail; 998 newpte |= PG_A; 999 vm_page_aflag_set(m, PGA_REFERENCED); 1000 break; 1001 case VM_PROT_WRITE: 1002 if ((newpte & RPTE_EAA_W) == 0) 1003 goto protfail; 1004 if (is_l3e) 1005 goto protfail; 1006 newpte |= PG_M; 1007 vm_page_dirty(m); 1008 break; 1009 case VM_PROT_EXECUTE: 1010 if ((newpte & RPTE_EAA_X) == 0) 1011 goto protfail; 1012 newpte |= PG_A; 1013 vm_page_aflag_set(m, PGA_REFERENCED); 1014 break; 1015 } 1016 1017 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 1018 goto retry; 1019 ptesync(); 1020 PMAP_UNLOCK(pmap); 1021 if (startpte == newpte) 1022 return (KERN_FAILURE); 1023 return (0); 1024 protfail: 1025 PMAP_UNLOCK(pmap); 1026 return (KERN_PROTECTION_FAILURE); 1027 } 1028 1029 /* 1030 * Returns TRUE if the given page is mapped individually or as part of 1031 * a 2mpage. Otherwise, returns FALSE. 1032 */ 1033 boolean_t 1034 mmu_radix_page_is_mapped(vm_page_t m) 1035 { 1036 struct rwlock *lock; 1037 boolean_t rv; 1038 1039 if ((m->oflags & VPO_UNMANAGED) != 0) 1040 return (FALSE); 1041 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 1042 rw_rlock(lock); 1043 rv = !TAILQ_EMPTY(&m->md.pv_list) || 1044 ((m->flags & PG_FICTITIOUS) == 0 && 1045 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); 1046 rw_runlock(lock); 1047 return (rv); 1048 } 1049 1050 /* 1051 * Determine the appropriate bits to set in a PTE or PDE for a specified 1052 * caching mode. 1053 */ 1054 static int 1055 pmap_cache_bits(vm_memattr_t ma) 1056 { 1057 if (ma != VM_MEMATTR_DEFAULT) { 1058 switch (ma) { 1059 case VM_MEMATTR_UNCACHEABLE: 1060 return (RPTE_ATTR_GUARDEDIO); 1061 case VM_MEMATTR_CACHEABLE: 1062 return (RPTE_ATTR_MEM); 1063 case VM_MEMATTR_WRITE_BACK: 1064 case VM_MEMATTR_PREFETCHABLE: 1065 case VM_MEMATTR_WRITE_COMBINING: 1066 return (RPTE_ATTR_UNGUARDEDIO); 1067 } 1068 } 1069 return (0); 1070 } 1071 1072 static void 1073 pmap_invalidate_page(pmap_t pmap, vm_offset_t start) 1074 { 1075 ptesync(); 1076 if (pmap == kernel_pmap) 1077 radix_tlbie_invlpg_kernel_4k(start); 1078 else 1079 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1080 ttusync(); 1081 } 1082 1083 static void 1084 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start) 1085 { 1086 ptesync(); 1087 if (pmap == kernel_pmap) 1088 radix_tlbie_invlpg_kernel_2m(start); 1089 else 1090 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start); 1091 ttusync(); 1092 } 1093 1094 static void 1095 pmap_invalidate_pwc(pmap_t pmap) 1096 { 1097 ptesync(); 1098 if (pmap == kernel_pmap) 1099 radix_tlbie_invlpwc_kernel(); 1100 else 1101 radix_tlbie_invlpwc_user(pmap->pm_pid); 1102 ttusync(); 1103 } 1104 1105 static void 1106 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end) 1107 { 1108 if (((start - end) >> PAGE_SHIFT) > 8) { 1109 pmap_invalidate_all(pmap); 1110 return; 1111 } 1112 ptesync(); 1113 if (pmap == kernel_pmap) { 1114 while (start < end) { 1115 radix_tlbie_invlpg_kernel_4k(start); 1116 start += PAGE_SIZE; 1117 } 1118 } else { 1119 while (start < end) { 1120 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1121 start += PAGE_SIZE; 1122 } 1123 } 1124 ttusync(); 1125 } 1126 1127 static void 1128 pmap_invalidate_all(pmap_t pmap) 1129 { 1130 ptesync(); 1131 if (pmap == kernel_pmap) 1132 radix_tlbie_flush_kernel(); 1133 else 1134 radix_tlbie_flush_user(pmap->pm_pid); 1135 ttusync(); 1136 } 1137 1138 static void 1139 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e) 1140 { 1141 1142 /* 1143 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created 1144 * by a promotion that did not invalidate the 512 4KB page mappings 1145 * that might exist in the TLB. Consequently, at this point, the TLB 1146 * may hold both 4KB and 2MB page mappings for the address range [va, 1147 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here. 1148 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any 1149 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a 1150 * single INVLPG suffices to invalidate the 2MB page mapping from the 1151 * TLB. 1152 */ 1153 ptesync(); 1154 if ((l3e & PG_PROMOTED) != 0) 1155 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1); 1156 else 1157 pmap_invalidate_page_2m(pmap, va); 1158 1159 pmap_invalidate_pwc(pmap); 1160 } 1161 1162 static __inline struct pv_chunk * 1163 pv_to_chunk(pv_entry_t pv) 1164 { 1165 1166 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1167 } 1168 1169 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1170 1171 #define PC_FREE0 0xfffffffffffffffful 1172 #define PC_FREE1 ((1ul << (_NPCPV % 64)) - 1) 1173 1174 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 }; 1175 1176 /* 1177 * Ensure that the number of spare PV entries in the specified pmap meets or 1178 * exceeds the given count, "needed". 1179 * 1180 * The given PV list lock may be released. 1181 */ 1182 static void 1183 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 1184 { 1185 struct pch new_tail; 1186 struct pv_chunk *pc; 1187 vm_page_t m; 1188 int avail, free; 1189 bool reclaimed; 1190 1191 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1192 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 1193 1194 /* 1195 * Newly allocated PV chunks must be stored in a private list until 1196 * the required number of PV chunks have been allocated. Otherwise, 1197 * reclaim_pv_chunk() could recycle one of these chunks. In 1198 * contrast, these chunks must be added to the pmap upon allocation. 1199 */ 1200 TAILQ_INIT(&new_tail); 1201 retry: 1202 avail = 0; 1203 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 1204 // if ((cpu_feature2 & CPUID2_POPCNT) == 0) 1205 bit_count((bitstr_t *)pc->pc_map, 0, 1206 sizeof(pc->pc_map) * NBBY, &free); 1207 #if 0 1208 free = popcnt_pc_map_pq(pc->pc_map); 1209 #endif 1210 if (free == 0) 1211 break; 1212 avail += free; 1213 if (avail >= needed) 1214 break; 1215 } 1216 for (reclaimed = false; avail < needed; avail += _NPCPV) { 1217 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 1218 if (m == NULL) { 1219 m = reclaim_pv_chunk(pmap, lockp); 1220 if (m == NULL) 1221 goto retry; 1222 reclaimed = true; 1223 } 1224 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1225 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1226 dump_add_page(m->phys_addr); 1227 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1228 pc->pc_pmap = pmap; 1229 pc->pc_map[0] = PC_FREE0; 1230 pc->pc_map[1] = PC_FREE1; 1231 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1232 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); 1233 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); 1234 1235 /* 1236 * The reclaim might have freed a chunk from the current pmap. 1237 * If that chunk contained available entries, we need to 1238 * re-count the number of available entries. 1239 */ 1240 if (reclaimed) 1241 goto retry; 1242 } 1243 if (!TAILQ_EMPTY(&new_tail)) { 1244 mtx_lock(&pv_chunks_mutex); 1245 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); 1246 mtx_unlock(&pv_chunks_mutex); 1247 } 1248 } 1249 1250 /* 1251 * First find and then remove the pv entry for the specified pmap and virtual 1252 * address from the specified pv list. Returns the pv entry if found and NULL 1253 * otherwise. This operation can be performed on pv lists for either 4KB or 1254 * 2MB page mappings. 1255 */ 1256 static __inline pv_entry_t 1257 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1258 { 1259 pv_entry_t pv; 1260 1261 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 1262 #ifdef INVARIANTS 1263 if (PV_PMAP(pv) == NULL) { 1264 printf("corrupted pv_chunk/pv %p\n", pv); 1265 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":"); 1266 } 1267 MPASS(PV_PMAP(pv) != NULL); 1268 MPASS(pv->pv_va != 0); 1269 #endif 1270 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1271 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 1272 pvh->pv_gen++; 1273 break; 1274 } 1275 } 1276 return (pv); 1277 } 1278 1279 /* 1280 * After demotion from a 2MB page mapping to 512 4KB page mappings, 1281 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 1282 * entries for each of the 4KB page mappings. 1283 */ 1284 static void 1285 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1286 struct rwlock **lockp) 1287 { 1288 struct md_page *pvh; 1289 struct pv_chunk *pc; 1290 pv_entry_t pv; 1291 vm_offset_t va_last; 1292 vm_page_t m; 1293 int bit, field; 1294 1295 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1296 KASSERT((pa & L3_PAGE_MASK) == 0, 1297 ("pmap_pv_demote_pde: pa is not 2mpage aligned")); 1298 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1299 1300 /* 1301 * Transfer the 2mpage's pv entry for this mapping to the first 1302 * page's pv list. Once this transfer begins, the pv list lock 1303 * must not be released until the last pv entry is reinstantiated. 1304 */ 1305 pvh = pa_to_pvh(pa); 1306 va = trunc_2mpage(va); 1307 pv = pmap_pvh_remove(pvh, pmap, va); 1308 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); 1309 m = PHYS_TO_VM_PAGE(pa); 1310 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1311 1312 m->md.pv_gen++; 1313 /* Instantiate the remaining NPTEPG - 1 pv entries. */ 1314 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); 1315 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1316 for (;;) { 1317 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1318 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 1319 , ("pmap_pv_demote_pde: missing spare")); 1320 for (field = 0; field < _NPCM; field++) { 1321 while (pc->pc_map[field]) { 1322 bit = cnttzd(pc->pc_map[field]); 1323 pc->pc_map[field] &= ~(1ul << bit); 1324 pv = &pc->pc_pventry[field * 64 + bit]; 1325 va += PAGE_SIZE; 1326 pv->pv_va = va; 1327 m++; 1328 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1329 ("pmap_pv_demote_pde: page %p is not managed", m)); 1330 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1331 1332 m->md.pv_gen++; 1333 if (va == va_last) 1334 goto out; 1335 } 1336 } 1337 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1338 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1339 } 1340 out: 1341 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1342 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1343 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1344 } 1345 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); 1346 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); 1347 } 1348 1349 static void 1350 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap) 1351 { 1352 1353 if (pmap == NULL) 1354 return; 1355 pmap_invalidate_all(pmap); 1356 if (pmap != locked_pmap) 1357 PMAP_UNLOCK(pmap); 1358 } 1359 1360 /* 1361 * We are in a serious low memory condition. Resort to 1362 * drastic measures to free some pages so we can allocate 1363 * another pv entry chunk. 1364 * 1365 * Returns NULL if PV entries were reclaimed from the specified pmap. 1366 * 1367 * We do not, however, unmap 2mpages because subsequent accesses will 1368 * allocate per-page pv entries until repromotion occurs, thereby 1369 * exacerbating the shortage of free pv entries. 1370 */ 1371 static int active_reclaims = 0; 1372 static vm_page_t 1373 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 1374 { 1375 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 1376 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 1377 struct md_page *pvh; 1378 pml3_entry_t *l3e; 1379 pmap_t next_pmap, pmap; 1380 pt_entry_t *pte, tpte; 1381 pv_entry_t pv; 1382 vm_offset_t va; 1383 vm_page_t m, m_pc; 1384 struct spglist free; 1385 uint64_t inuse; 1386 int bit, field, freed; 1387 1388 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1389 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 1390 pmap = NULL; 1391 m_pc = NULL; 1392 SLIST_INIT(&free); 1393 bzero(&pc_marker_b, sizeof(pc_marker_b)); 1394 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 1395 pc_marker = (struct pv_chunk *)&pc_marker_b; 1396 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 1397 1398 mtx_lock(&pv_chunks_mutex); 1399 active_reclaims++; 1400 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru); 1401 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru); 1402 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 1403 SLIST_EMPTY(&free)) { 1404 next_pmap = pc->pc_pmap; 1405 if (next_pmap == NULL) { 1406 /* 1407 * The next chunk is a marker. However, it is 1408 * not our marker, so active_reclaims must be 1409 * > 1. Consequently, the next_chunk code 1410 * will not rotate the pv_chunks list. 1411 */ 1412 goto next_chunk; 1413 } 1414 mtx_unlock(&pv_chunks_mutex); 1415 1416 /* 1417 * A pv_chunk can only be removed from the pc_lru list 1418 * when both pc_chunks_mutex is owned and the 1419 * corresponding pmap is locked. 1420 */ 1421 if (pmap != next_pmap) { 1422 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1423 pmap = next_pmap; 1424 /* Avoid deadlock and lock recursion. */ 1425 if (pmap > locked_pmap) { 1426 RELEASE_PV_LIST_LOCK(lockp); 1427 PMAP_LOCK(pmap); 1428 mtx_lock(&pv_chunks_mutex); 1429 continue; 1430 } else if (pmap != locked_pmap) { 1431 if (PMAP_TRYLOCK(pmap)) { 1432 mtx_lock(&pv_chunks_mutex); 1433 continue; 1434 } else { 1435 pmap = NULL; /* pmap is not locked */ 1436 mtx_lock(&pv_chunks_mutex); 1437 pc = TAILQ_NEXT(pc_marker, pc_lru); 1438 if (pc == NULL || 1439 pc->pc_pmap != next_pmap) 1440 continue; 1441 goto next_chunk; 1442 } 1443 } 1444 } 1445 1446 /* 1447 * Destroy every non-wired, 4 KB page mapping in the chunk. 1448 */ 1449 freed = 0; 1450 for (field = 0; field < _NPCM; field++) { 1451 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 1452 inuse != 0; inuse &= ~(1UL << bit)) { 1453 bit = cnttzd(inuse); 1454 pv = &pc->pc_pventry[field * 64 + bit]; 1455 va = pv->pv_va; 1456 l3e = pmap_pml3e(pmap, va); 1457 if ((be64toh(*l3e) & RPTE_LEAF) != 0) 1458 continue; 1459 pte = pmap_l3e_to_pte(l3e, va); 1460 if ((be64toh(*pte) & PG_W) != 0) 1461 continue; 1462 tpte = be64toh(pte_load_clear(pte)); 1463 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); 1464 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 1465 vm_page_dirty(m); 1466 if ((tpte & PG_A) != 0) 1467 vm_page_aflag_set(m, PGA_REFERENCED); 1468 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1469 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 1470 1471 m->md.pv_gen++; 1472 if (TAILQ_EMPTY(&m->md.pv_list) && 1473 (m->flags & PG_FICTITIOUS) == 0) { 1474 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 1475 if (TAILQ_EMPTY(&pvh->pv_list)) { 1476 vm_page_aflag_clear(m, 1477 PGA_WRITEABLE); 1478 } 1479 } 1480 pc->pc_map[field] |= 1UL << bit; 1481 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free); 1482 freed++; 1483 } 1484 } 1485 if (freed == 0) { 1486 mtx_lock(&pv_chunks_mutex); 1487 goto next_chunk; 1488 } 1489 /* Every freed mapping is for a 4 KB page. */ 1490 pmap_resident_count_dec(pmap, freed); 1491 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 1492 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 1493 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 1494 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1495 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) { 1496 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1497 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1498 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1499 /* Entire chunk is free; return it. */ 1500 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1501 dump_drop_page(m_pc->phys_addr); 1502 mtx_lock(&pv_chunks_mutex); 1503 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1504 break; 1505 } 1506 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1507 mtx_lock(&pv_chunks_mutex); 1508 /* One freed pv entry in locked_pmap is sufficient. */ 1509 if (pmap == locked_pmap) 1510 break; 1511 next_chunk: 1512 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1513 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru); 1514 if (active_reclaims == 1 && pmap != NULL) { 1515 /* 1516 * Rotate the pv chunks list so that we do not 1517 * scan the same pv chunks that could not be 1518 * freed (because they contained a wired 1519 * and/or superpage mapping) on every 1520 * invocation of reclaim_pv_chunk(). 1521 */ 1522 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) { 1523 MPASS(pc->pc_pmap != NULL); 1524 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1525 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1526 } 1527 } 1528 } 1529 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1530 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); 1531 active_reclaims--; 1532 mtx_unlock(&pv_chunks_mutex); 1533 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1534 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 1535 m_pc = SLIST_FIRST(&free); 1536 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 1537 /* Recycle a freed page table page. */ 1538 m_pc->ref_count = 1; 1539 } 1540 vm_page_free_pages_toq(&free, true); 1541 return (m_pc); 1542 } 1543 1544 /* 1545 * free the pv_entry back to the free list 1546 */ 1547 static void 1548 free_pv_entry(pmap_t pmap, pv_entry_t pv) 1549 { 1550 struct pv_chunk *pc; 1551 int idx, field, bit; 1552 1553 #ifdef VERBOSE_PV 1554 if (pmap != kernel_pmap) 1555 printf("%s(%p, %p)\n", __func__, pmap, pv); 1556 #endif 1557 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1558 PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 1559 PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 1560 PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 1561 pc = pv_to_chunk(pv); 1562 idx = pv - &pc->pc_pventry[0]; 1563 field = idx / 64; 1564 bit = idx % 64; 1565 pc->pc_map[field] |= 1ul << bit; 1566 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) { 1567 /* 98% of the time, pc is already at the head of the list. */ 1568 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 1569 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1570 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1571 } 1572 return; 1573 } 1574 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1575 free_pv_chunk(pc); 1576 } 1577 1578 static void 1579 free_pv_chunk(struct pv_chunk *pc) 1580 { 1581 vm_page_t m; 1582 1583 mtx_lock(&pv_chunks_mutex); 1584 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1585 mtx_unlock(&pv_chunks_mutex); 1586 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1587 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1588 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1589 /* entire chunk is free, return it */ 1590 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1591 dump_drop_page(m->phys_addr); 1592 vm_page_unwire_noq(m); 1593 vm_page_free(m); 1594 } 1595 1596 /* 1597 * Returns a new PV entry, allocating a new PV chunk from the system when 1598 * needed. If this PV chunk allocation fails and a PV list lock pointer was 1599 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 1600 * returned. 1601 * 1602 * The given PV list lock may be released. 1603 */ 1604 static pv_entry_t 1605 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 1606 { 1607 int bit, field; 1608 pv_entry_t pv; 1609 struct pv_chunk *pc; 1610 vm_page_t m; 1611 1612 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1613 PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 1614 retry: 1615 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1616 if (pc != NULL) { 1617 for (field = 0; field < _NPCM; field++) { 1618 if (pc->pc_map[field]) { 1619 bit = cnttzd(pc->pc_map[field]); 1620 break; 1621 } 1622 } 1623 if (field < _NPCM) { 1624 pv = &pc->pc_pventry[field * 64 + bit]; 1625 pc->pc_map[field] &= ~(1ul << bit); 1626 /* If this was the last item, move it to tail */ 1627 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1628 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1629 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 1630 pc_list); 1631 } 1632 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1633 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 1634 MPASS(PV_PMAP(pv) != NULL); 1635 return (pv); 1636 } 1637 } 1638 /* No free items, allocate another chunk */ 1639 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 1640 if (m == NULL) { 1641 if (lockp == NULL) { 1642 PV_STAT(pc_chunk_tryfail++); 1643 return (NULL); 1644 } 1645 m = reclaim_pv_chunk(pmap, lockp); 1646 if (m == NULL) 1647 goto retry; 1648 } 1649 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1650 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1651 dump_add_page(m->phys_addr); 1652 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1653 pc->pc_pmap = pmap; 1654 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ 1655 pc->pc_map[1] = PC_FREE1; 1656 mtx_lock(&pv_chunks_mutex); 1657 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1658 mtx_unlock(&pv_chunks_mutex); 1659 pv = &pc->pc_pventry[0]; 1660 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1661 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1662 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 1663 MPASS(PV_PMAP(pv) != NULL); 1664 return (pv); 1665 } 1666 1667 #if VM_NRESERVLEVEL > 0 1668 /* 1669 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 1670 * replace the many pv entries for the 4KB page mappings by a single pv entry 1671 * for the 2MB page mapping. 1672 */ 1673 static void 1674 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1675 struct rwlock **lockp) 1676 { 1677 struct md_page *pvh; 1678 pv_entry_t pv; 1679 vm_offset_t va_last; 1680 vm_page_t m; 1681 1682 KASSERT((pa & L3_PAGE_MASK) == 0, 1683 ("pmap_pv_promote_pde: pa is not 2mpage aligned")); 1684 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1685 1686 /* 1687 * Transfer the first page's pv entry for this mapping to the 2mpage's 1688 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 1689 * a transfer avoids the possibility that get_pv_entry() calls 1690 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 1691 * mappings that is being promoted. 1692 */ 1693 m = PHYS_TO_VM_PAGE(pa); 1694 va = trunc_2mpage(va); 1695 pv = pmap_pvh_remove(&m->md, pmap, va); 1696 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); 1697 pvh = pa_to_pvh(pa); 1698 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 1699 pvh->pv_gen++; 1700 /* Free the remaining NPTEPG - 1 pv entries. */ 1701 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1702 do { 1703 m++; 1704 va += PAGE_SIZE; 1705 pmap_pvh_free(&m->md, pmap, va); 1706 } while (va < va_last); 1707 } 1708 #endif /* VM_NRESERVLEVEL > 0 */ 1709 1710 /* 1711 * First find and then destroy the pv entry for the specified pmap and virtual 1712 * address. This operation can be performed on pv lists for either 4KB or 2MB 1713 * page mappings. 1714 */ 1715 static void 1716 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1717 { 1718 pv_entry_t pv; 1719 1720 pv = pmap_pvh_remove(pvh, pmap, va); 1721 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 1722 free_pv_entry(pmap, pv); 1723 } 1724 1725 /* 1726 * Conditionally create the PV entry for a 4KB page mapping if the required 1727 * memory can be allocated without resorting to reclamation. 1728 */ 1729 static boolean_t 1730 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 1731 struct rwlock **lockp) 1732 { 1733 pv_entry_t pv; 1734 1735 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1736 /* Pass NULL instead of the lock pointer to disable reclamation. */ 1737 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 1738 pv->pv_va = va; 1739 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1740 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1741 m->md.pv_gen++; 1742 return (TRUE); 1743 } else 1744 return (FALSE); 1745 } 1746 1747 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX]; 1748 #ifdef INVARIANTS 1749 static void 1750 validate_addr(vm_paddr_t addr, vm_size_t size) 1751 { 1752 vm_paddr_t end = addr + size; 1753 bool found = false; 1754 1755 for (int i = 0; i < 2 * phys_avail_count; i += 2) { 1756 if (addr >= phys_avail_debug[i] && 1757 end <= phys_avail_debug[i + 1]) { 1758 found = true; 1759 break; 1760 } 1761 } 1762 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array", 1763 addr, end)); 1764 } 1765 #else 1766 static void validate_addr(vm_paddr_t addr, vm_size_t size) {} 1767 #endif 1768 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A) 1769 1770 static vm_paddr_t 1771 alloc_pt_page(void) 1772 { 1773 vm_paddr_t page; 1774 1775 page = allocpages(1); 1776 pagezero(PHYS_TO_DMAP(page)); 1777 return (page); 1778 } 1779 1780 static void 1781 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end) 1782 { 1783 pt_entry_t *pte, pteval; 1784 vm_paddr_t page; 1785 1786 if (bootverbose) 1787 printf("%s %lx -> %lx\n", __func__, start, end); 1788 while (start < end) { 1789 pteval = start | DMAP_PAGE_BITS; 1790 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start)); 1791 if ((be64toh(*pte) & RPTE_VALID) == 0) { 1792 page = alloc_pt_page(); 1793 pde_store(pte, page); 1794 } 1795 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start)); 1796 if ((start & L2_PAGE_MASK) == 0 && 1797 end - start >= L2_PAGE_SIZE) { 1798 start += L2_PAGE_SIZE; 1799 goto done; 1800 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1801 page = alloc_pt_page(); 1802 pde_store(pte, page); 1803 } 1804 1805 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start)); 1806 if ((start & L3_PAGE_MASK) == 0 && 1807 end - start >= L3_PAGE_SIZE) { 1808 start += L3_PAGE_SIZE; 1809 goto done; 1810 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1811 page = alloc_pt_page(); 1812 pde_store(pte, page); 1813 } 1814 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start)); 1815 start += PAGE_SIZE; 1816 done: 1817 pte_store(pte, pteval); 1818 } 1819 } 1820 1821 static void 1822 mmu_radix_dmap_populate(vm_size_t hwphyssz) 1823 { 1824 vm_paddr_t start, end; 1825 1826 for (int i = 0; i < pregions_sz; i++) { 1827 start = pregions[i].mr_start; 1828 end = start + pregions[i].mr_size; 1829 if (hwphyssz && start >= hwphyssz) 1830 break; 1831 if (hwphyssz && hwphyssz < end) 1832 end = hwphyssz; 1833 mmu_radix_dmap_range(start, end); 1834 } 1835 } 1836 1837 static void 1838 mmu_radix_setup_pagetables(vm_size_t hwphyssz) 1839 { 1840 vm_paddr_t ptpages, pages; 1841 pt_entry_t *pte; 1842 vm_paddr_t l1phys; 1843 1844 bzero(kernel_pmap, sizeof(struct pmap)); 1845 PMAP_LOCK_INIT(kernel_pmap); 1846 vm_radix_init(&kernel_pmap->pm_radix); 1847 1848 ptpages = allocpages(3); 1849 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE); 1850 validate_addr(l1phys, RADIX_PGD_SIZE); 1851 if (bootverbose) 1852 printf("l1phys=%lx\n", l1phys); 1853 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0); 1854 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++) 1855 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE)); 1856 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys); 1857 1858 mmu_radix_dmap_populate(hwphyssz); 1859 1860 /* 1861 * Create page tables for first 128MB of KVA 1862 */ 1863 pages = ptpages; 1864 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS); 1865 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1866 pages += PAGE_SIZE; 1867 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS); 1868 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1869 pages += PAGE_SIZE; 1870 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS); 1871 /* 1872 * the kernel page table pages need to be preserved in 1873 * phys_avail and not overlap with previous allocations 1874 */ 1875 pages = allocpages(nkpt); 1876 if (bootverbose) { 1877 printf("phys_avail after dmap populate and nkpt allocation\n"); 1878 for (int j = 0; j < 2 * phys_avail_count; j+=2) 1879 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1880 j, phys_avail[j], j + 1, phys_avail[j + 1]); 1881 } 1882 KPTphys = pages; 1883 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE) 1884 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1885 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE; 1886 if (bootverbose) 1887 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1); 1888 /* 1889 * Add a physical memory segment (vm_phys_seg) corresponding to the 1890 * preallocated kernel page table pages so that vm_page structures 1891 * representing these pages will be created. The vm_page structures 1892 * are required for promotion of the corresponding kernel virtual 1893 * addresses to superpage mappings. 1894 */ 1895 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt)); 1896 } 1897 1898 static void 1899 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end) 1900 { 1901 vm_paddr_t kpstart, kpend; 1902 vm_size_t physsz, hwphyssz; 1903 //uint64_t l2virt; 1904 int rm_pavail, proctab_size; 1905 int i, j; 1906 1907 kpstart = start & ~DMAP_BASE_ADDRESS; 1908 kpend = end & ~DMAP_BASE_ADDRESS; 1909 1910 /* Get physical memory regions from firmware */ 1911 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 1912 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory"); 1913 1914 if (2 * VM_PHYSSEG_MAX < regions_sz) 1915 panic("mmu_radix_early_bootstrap: phys_avail too small"); 1916 1917 if (bootverbose) 1918 for (int i = 0; i < regions_sz; i++) 1919 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n", 1920 i, regions[i].mr_start, i, regions[i].mr_size); 1921 /* 1922 * XXX workaround a simulator bug 1923 */ 1924 for (int i = 0; i < regions_sz; i++) 1925 if (regions[i].mr_start & PAGE_MASK) { 1926 regions[i].mr_start += PAGE_MASK; 1927 regions[i].mr_start &= ~PAGE_MASK; 1928 regions[i].mr_size &= ~PAGE_MASK; 1929 } 1930 if (bootverbose) 1931 for (int i = 0; i < pregions_sz; i++) 1932 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n", 1933 i, pregions[i].mr_start, i, pregions[i].mr_size); 1934 1935 phys_avail_count = 0; 1936 physsz = 0; 1937 hwphyssz = 0; 1938 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 1939 for (i = 0, j = 0; i < regions_sz; i++) { 1940 if (bootverbose) 1941 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n", 1942 i, regions[i].mr_start, i, regions[i].mr_size); 1943 1944 if (regions[i].mr_size < PAGE_SIZE) 1945 continue; 1946 1947 if (hwphyssz != 0 && 1948 (physsz + regions[i].mr_size) >= hwphyssz) { 1949 if (physsz < hwphyssz) { 1950 phys_avail[j] = regions[i].mr_start; 1951 phys_avail[j + 1] = regions[i].mr_start + 1952 (hwphyssz - physsz); 1953 physsz = hwphyssz; 1954 phys_avail_count++; 1955 dump_avail[j] = phys_avail[j]; 1956 dump_avail[j + 1] = phys_avail[j + 1]; 1957 } 1958 break; 1959 } 1960 phys_avail[j] = regions[i].mr_start; 1961 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 1962 dump_avail[j] = phys_avail[j]; 1963 dump_avail[j + 1] = phys_avail[j + 1]; 1964 1965 phys_avail_count++; 1966 physsz += regions[i].mr_size; 1967 j += 2; 1968 } 1969 1970 /* Check for overlap with the kernel and exception vectors */ 1971 rm_pavail = 0; 1972 for (j = 0; j < 2 * phys_avail_count; j+=2) { 1973 if (phys_avail[j] < EXC_LAST) 1974 phys_avail[j] += EXC_LAST; 1975 1976 if (phys_avail[j] >= kpstart && 1977 phys_avail[j + 1] <= kpend) { 1978 phys_avail[j] = phys_avail[j + 1] = ~0; 1979 rm_pavail++; 1980 continue; 1981 } 1982 1983 if (kpstart >= phys_avail[j] && 1984 kpstart < phys_avail[j + 1]) { 1985 if (kpend < phys_avail[j + 1]) { 1986 phys_avail[2 * phys_avail_count] = 1987 (kpend & ~PAGE_MASK) + PAGE_SIZE; 1988 phys_avail[2 * phys_avail_count + 1] = 1989 phys_avail[j + 1]; 1990 phys_avail_count++; 1991 } 1992 1993 phys_avail[j + 1] = kpstart & ~PAGE_MASK; 1994 } 1995 1996 if (kpend >= phys_avail[j] && 1997 kpend < phys_avail[j + 1]) { 1998 if (kpstart > phys_avail[j]) { 1999 phys_avail[2 * phys_avail_count] = phys_avail[j]; 2000 phys_avail[2 * phys_avail_count + 1] = 2001 kpstart & ~PAGE_MASK; 2002 phys_avail_count++; 2003 } 2004 2005 phys_avail[j] = (kpend & ~PAGE_MASK) + 2006 PAGE_SIZE; 2007 } 2008 } 2009 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp); 2010 for (i = 0; i < 2 * phys_avail_count; i++) 2011 phys_avail_debug[i] = phys_avail[i]; 2012 2013 /* Remove physical available regions marked for removal (~0) */ 2014 if (rm_pavail) { 2015 phys_avail_count -= rm_pavail; 2016 for (i = 2 * phys_avail_count; 2017 i < 2*(phys_avail_count + rm_pavail); i+=2) 2018 phys_avail[i] = phys_avail[i + 1] = 0; 2019 } 2020 if (bootverbose) { 2021 printf("phys_avail ranges after filtering:\n"); 2022 for (j = 0; j < 2 * phys_avail_count; j+=2) 2023 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 2024 j, phys_avail[j], j + 1, phys_avail[j + 1]); 2025 } 2026 physmem = btoc(physsz); 2027 2028 /* XXX assume we're running non-virtualized and 2029 * we don't support BHYVE 2030 */ 2031 if (isa3_pid_bits == 0) 2032 isa3_pid_bits = 20; 2033 if (powernv_enabled) { 2034 parttab_phys = 2035 moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE); 2036 validate_addr(parttab_phys, PARTTAB_SIZE); 2037 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++) 2038 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE)); 2039 2040 } 2041 proctab_size = 1UL << PROCTAB_SIZE_SHIFT; 2042 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size); 2043 validate_addr(proctab0pa, proctab_size); 2044 for (int i = 0; i < proctab_size/PAGE_SIZE; i++) 2045 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE)); 2046 2047 mmu_radix_setup_pagetables(hwphyssz); 2048 } 2049 2050 static void 2051 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end) 2052 { 2053 int i; 2054 vm_paddr_t pa; 2055 void *dpcpu; 2056 vm_offset_t va; 2057 2058 /* 2059 * Set up the Open Firmware pmap and add its mappings if not in real 2060 * mode. 2061 */ 2062 if (bootverbose) 2063 printf("%s enter\n", __func__); 2064 2065 /* 2066 * Calculate the last available physical address, and reserve the 2067 * vm_page_array (upper bound). 2068 */ 2069 Maxmem = 0; 2070 for (i = 0; phys_avail[i + 1] != 0; i += 2) 2071 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1])); 2072 2073 /* 2074 * Remap any early IO mappings (console framebuffer, etc.) 2075 */ 2076 bs_remap_earlyboot(); 2077 2078 /* 2079 * Allocate a kernel stack with a guard page for thread0 and map it 2080 * into the kernel page map. 2081 */ 2082 pa = allocpages(kstack_pages); 2083 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 2084 virtual_avail = va + kstack_pages * PAGE_SIZE; 2085 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); 2086 thread0.td_kstack = va; 2087 for (i = 0; i < kstack_pages; i++) { 2088 mmu_radix_kenter(va, pa); 2089 pa += PAGE_SIZE; 2090 va += PAGE_SIZE; 2091 } 2092 thread0.td_kstack_pages = kstack_pages; 2093 2094 /* 2095 * Allocate virtual address space for the message buffer. 2096 */ 2097 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT); 2098 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa); 2099 2100 /* 2101 * Allocate virtual address space for the dynamic percpu area. 2102 */ 2103 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT); 2104 dpcpu = (void *)PHYS_TO_DMAP(pa); 2105 dpcpu_init(dpcpu, curcpu); 2106 2107 crashdumpmap = (caddr_t)virtual_avail; 2108 virtual_avail += MAXDUMPPGS * PAGE_SIZE; 2109 2110 /* 2111 * Reserve some special page table entries/VA space for temporary 2112 * mapping of pages. 2113 */ 2114 } 2115 2116 static void 2117 mmu_parttab_init(void) 2118 { 2119 uint64_t ptcr; 2120 2121 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys); 2122 2123 if (bootverbose) 2124 printf("%s parttab: %p\n", __func__, isa3_parttab); 2125 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2126 if (bootverbose) 2127 printf("setting ptcr %lx\n", ptcr); 2128 mtspr(SPR_PTCR, ptcr); 2129 } 2130 2131 static void 2132 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab) 2133 { 2134 uint64_t prev; 2135 2136 if (bootverbose) 2137 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab, 2138 lpid, pagetab, proctab); 2139 prev = be64toh(isa3_parttab[lpid].pagetab); 2140 isa3_parttab[lpid].pagetab = htobe64(pagetab); 2141 isa3_parttab[lpid].proctab = htobe64(proctab); 2142 2143 if (prev & PARTTAB_HR) { 2144 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : : 2145 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2146 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2147 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2148 } else { 2149 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : 2150 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2151 } 2152 ttusync(); 2153 } 2154 2155 static void 2156 mmu_radix_parttab_init(void) 2157 { 2158 uint64_t pagetab; 2159 2160 mmu_parttab_init(); 2161 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \ 2162 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR; 2163 mmu_parttab_update(0, pagetab, 0); 2164 } 2165 2166 static void 2167 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size) 2168 { 2169 uint64_t pagetab, proctab; 2170 2171 pagetab = be64toh(isa3_parttab[0].pagetab); 2172 proctab = proctabpa | table_size | PARTTAB_GR; 2173 mmu_parttab_update(0, pagetab, proctab); 2174 } 2175 2176 static void 2177 mmu_radix_proctab_init(void) 2178 { 2179 2180 isa3_base_pid = 1; 2181 2182 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa); 2183 isa3_proctab->proctab0 = 2184 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | 2185 RADIX_PGD_INDEX_SHIFT); 2186 2187 if (powernv_enabled) { 2188 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12); 2189 __asm __volatile("ptesync" : : : "memory"); 2190 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2191 "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); 2192 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory"); 2193 #ifdef PSERIES 2194 } else { 2195 int64_t rc; 2196 2197 rc = phyp_hcall(H_REGISTER_PROC_TBL, 2198 PROC_TABLE_NEW | PROC_TABLE_RADIX | PROC_TABLE_GTSE, 2199 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12); 2200 if (rc != H_SUCCESS) 2201 panic("mmu_radix_proctab_init: " 2202 "failed to register process table: rc=%jd", 2203 (intmax_t)rc); 2204 #endif 2205 } 2206 2207 if (bootverbose) 2208 printf("process table %p and kernel radix PDE: %p\n", 2209 isa3_proctab, kernel_pmap->pm_pml1); 2210 mtmsr(mfmsr() | PSL_DR ); 2211 mtmsr(mfmsr() & ~PSL_DR); 2212 kernel_pmap->pm_pid = isa3_base_pid; 2213 isa3_base_pid++; 2214 } 2215 2216 void 2217 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2218 int advice) 2219 { 2220 struct rwlock *lock; 2221 pml1_entry_t *l1e; 2222 pml2_entry_t *l2e; 2223 pml3_entry_t oldl3e, *l3e; 2224 pt_entry_t *pte; 2225 vm_offset_t va, va_next; 2226 vm_page_t m; 2227 bool anychanged; 2228 2229 if (advice != MADV_DONTNEED && advice != MADV_FREE) 2230 return; 2231 anychanged = false; 2232 PMAP_LOCK(pmap); 2233 for (; sva < eva; sva = va_next) { 2234 l1e = pmap_pml1e(pmap, sva); 2235 if ((be64toh(*l1e) & PG_V) == 0) { 2236 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2237 if (va_next < sva) 2238 va_next = eva; 2239 continue; 2240 } 2241 l2e = pmap_l1e_to_l2e(l1e, sva); 2242 if ((be64toh(*l2e) & PG_V) == 0) { 2243 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2244 if (va_next < sva) 2245 va_next = eva; 2246 continue; 2247 } 2248 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2249 if (va_next < sva) 2250 va_next = eva; 2251 l3e = pmap_l2e_to_l3e(l2e, sva); 2252 oldl3e = be64toh(*l3e); 2253 if ((oldl3e & PG_V) == 0) 2254 continue; 2255 else if ((oldl3e & RPTE_LEAF) != 0) { 2256 if ((oldl3e & PG_MANAGED) == 0) 2257 continue; 2258 lock = NULL; 2259 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) { 2260 if (lock != NULL) 2261 rw_wunlock(lock); 2262 2263 /* 2264 * The large page mapping was destroyed. 2265 */ 2266 continue; 2267 } 2268 2269 /* 2270 * Unless the page mappings are wired, remove the 2271 * mapping to a single page so that a subsequent 2272 * access may repromote. Choosing the last page 2273 * within the address range [sva, min(va_next, eva)) 2274 * generally results in more repromotions. Since the 2275 * underlying page table page is fully populated, this 2276 * removal never frees a page table page. 2277 */ 2278 if ((oldl3e & PG_W) == 0) { 2279 va = eva; 2280 if (va > va_next) 2281 va = va_next; 2282 va -= PAGE_SIZE; 2283 KASSERT(va >= sva, 2284 ("mmu_radix_advise: no address gap")); 2285 pte = pmap_l3e_to_pte(l3e, va); 2286 KASSERT((be64toh(*pte) & PG_V) != 0, 2287 ("pmap_advise: invalid PTE")); 2288 pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL, 2289 &lock); 2290 anychanged = true; 2291 } 2292 if (lock != NULL) 2293 rw_wunlock(lock); 2294 } 2295 if (va_next > eva) 2296 va_next = eva; 2297 va = va_next; 2298 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; 2299 pte++, sva += PAGE_SIZE) { 2300 MPASS(pte == pmap_pte(pmap, sva)); 2301 2302 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) 2303 goto maybe_invlrng; 2304 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2305 if (advice == MADV_DONTNEED) { 2306 /* 2307 * Future calls to pmap_is_modified() 2308 * can be avoided by making the page 2309 * dirty now. 2310 */ 2311 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME); 2312 vm_page_dirty(m); 2313 } 2314 atomic_clear_long(pte, htobe64(PG_M | PG_A)); 2315 } else if ((be64toh(*pte) & PG_A) != 0) 2316 atomic_clear_long(pte, htobe64(PG_A)); 2317 else 2318 goto maybe_invlrng; 2319 anychanged = true; 2320 continue; 2321 maybe_invlrng: 2322 if (va != va_next) { 2323 anychanged = true; 2324 va = va_next; 2325 } 2326 } 2327 if (va != va_next) 2328 anychanged = true; 2329 } 2330 if (anychanged) 2331 pmap_invalidate_all(pmap); 2332 PMAP_UNLOCK(pmap); 2333 } 2334 2335 /* 2336 * Routines used in machine-dependent code 2337 */ 2338 static void 2339 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end) 2340 { 2341 uint64_t lpcr; 2342 2343 if (bootverbose) 2344 printf("%s\n", __func__); 2345 hw_direct_map = 1; 2346 powernv_enabled = (mfmsr() & PSL_HV) ? 1 : 0; 2347 mmu_radix_early_bootstrap(start, end); 2348 if (bootverbose) 2349 printf("early bootstrap complete\n"); 2350 if (powernv_enabled) { 2351 lpcr = mfspr(SPR_LPCR); 2352 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2353 mmu_radix_parttab_init(); 2354 mmu_radix_init_amor(); 2355 if (bootverbose) 2356 printf("powernv init complete\n"); 2357 } 2358 mmu_radix_init_iamr(); 2359 mmu_radix_proctab_init(); 2360 mmu_radix_pid_set(kernel_pmap); 2361 if (powernv_enabled) 2362 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2363 else 2364 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 2365 2366 mmu_radix_late_bootstrap(start, end); 2367 numa_mem_regions(&numa_pregions, &numa_pregions_sz); 2368 if (bootverbose) 2369 printf("%s done\n", __func__); 2370 pmap_bootstrapped = 1; 2371 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE); 2372 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS); 2373 } 2374 2375 static void 2376 mmu_radix_cpu_bootstrap(int ap) 2377 { 2378 uint64_t lpcr; 2379 uint64_t ptcr; 2380 2381 if (powernv_enabled) { 2382 lpcr = mfspr(SPR_LPCR); 2383 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2384 2385 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2386 mtspr(SPR_PTCR, ptcr); 2387 mmu_radix_init_amor(); 2388 } 2389 mmu_radix_init_iamr(); 2390 mmu_radix_pid_set(kernel_pmap); 2391 if (powernv_enabled) 2392 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2393 else 2394 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 2395 } 2396 2397 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0, 2398 "2MB page mapping counters"); 2399 2400 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_demotions); 2401 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD, 2402 &pmap_l3e_demotions, "2MB page demotions"); 2403 2404 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_mappings); 2405 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD, 2406 &pmap_l3e_mappings, "2MB page mappings"); 2407 2408 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_p_failures); 2409 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD, 2410 &pmap_l3e_p_failures, "2MB page promotion failures"); 2411 2412 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_promotions); 2413 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD, 2414 &pmap_l3e_promotions, "2MB page promotions"); 2415 2416 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0, 2417 "1GB page mapping counters"); 2418 2419 static COUNTER_U64_DEFINE_EARLY(pmap_l2e_demotions); 2420 SYSCTL_COUNTER_U64(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD, 2421 &pmap_l2e_demotions, "1GB page demotions"); 2422 2423 void 2424 mmu_radix_clear_modify(vm_page_t m) 2425 { 2426 struct md_page *pvh; 2427 pmap_t pmap; 2428 pv_entry_t next_pv, pv; 2429 pml3_entry_t oldl3e, *l3e; 2430 pt_entry_t oldpte, *pte; 2431 struct rwlock *lock; 2432 vm_offset_t va; 2433 int md_gen, pvh_gen; 2434 2435 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2436 ("pmap_clear_modify: page %p is not managed", m)); 2437 vm_page_assert_busied(m); 2438 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 2439 2440 /* 2441 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. 2442 * If the object containing the page is locked and the page is not 2443 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 2444 */ 2445 if ((m->a.flags & PGA_WRITEABLE) == 0) 2446 return; 2447 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 2448 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 2449 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2450 rw_wlock(lock); 2451 restart: 2452 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 2453 pmap = PV_PMAP(pv); 2454 if (!PMAP_TRYLOCK(pmap)) { 2455 pvh_gen = pvh->pv_gen; 2456 rw_wunlock(lock); 2457 PMAP_LOCK(pmap); 2458 rw_wlock(lock); 2459 if (pvh_gen != pvh->pv_gen) { 2460 PMAP_UNLOCK(pmap); 2461 goto restart; 2462 } 2463 } 2464 va = pv->pv_va; 2465 l3e = pmap_pml3e(pmap, va); 2466 oldl3e = be64toh(*l3e); 2467 if ((oldl3e & PG_RW) != 0 && 2468 pmap_demote_l3e_locked(pmap, l3e, va, &lock) && 2469 (oldl3e & PG_W) == 0) { 2470 /* 2471 * Write protect the mapping to a 2472 * single page so that a subsequent 2473 * write access may repromote. 2474 */ 2475 va += VM_PAGE_TO_PHYS(m) - (oldl3e & 2476 PG_PS_FRAME); 2477 pte = pmap_l3e_to_pte(l3e, va); 2478 oldpte = be64toh(*pte); 2479 while (!atomic_cmpset_long(pte, 2480 htobe64(oldpte), 2481 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW)))) 2482 oldpte = be64toh(*pte); 2483 vm_page_dirty(m); 2484 pmap_invalidate_page(pmap, va); 2485 } 2486 PMAP_UNLOCK(pmap); 2487 } 2488 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2489 pmap = PV_PMAP(pv); 2490 if (!PMAP_TRYLOCK(pmap)) { 2491 md_gen = m->md.pv_gen; 2492 pvh_gen = pvh->pv_gen; 2493 rw_wunlock(lock); 2494 PMAP_LOCK(pmap); 2495 rw_wlock(lock); 2496 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 2497 PMAP_UNLOCK(pmap); 2498 goto restart; 2499 } 2500 } 2501 l3e = pmap_pml3e(pmap, pv->pv_va); 2502 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found" 2503 " a 2mpage in page %p's pv list", m)); 2504 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 2505 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2506 atomic_clear_long(pte, htobe64(PG_M)); 2507 pmap_invalidate_page(pmap, pv->pv_va); 2508 } 2509 PMAP_UNLOCK(pmap); 2510 } 2511 rw_wunlock(lock); 2512 } 2513 2514 void 2515 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2516 vm_size_t len, vm_offset_t src_addr) 2517 { 2518 struct rwlock *lock; 2519 struct spglist free; 2520 vm_offset_t addr; 2521 vm_offset_t end_addr = src_addr + len; 2522 vm_offset_t va_next; 2523 vm_page_t dst_pdpg, dstmpte, srcmpte; 2524 bool invalidate_all; 2525 2526 CTR6(KTR_PMAP, 2527 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n", 2528 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr); 2529 2530 if (dst_addr != src_addr) 2531 return; 2532 lock = NULL; 2533 invalidate_all = false; 2534 if (dst_pmap < src_pmap) { 2535 PMAP_LOCK(dst_pmap); 2536 PMAP_LOCK(src_pmap); 2537 } else { 2538 PMAP_LOCK(src_pmap); 2539 PMAP_LOCK(dst_pmap); 2540 } 2541 2542 for (addr = src_addr; addr < end_addr; addr = va_next) { 2543 pml1_entry_t *l1e; 2544 pml2_entry_t *l2e; 2545 pml3_entry_t srcptepaddr, *l3e; 2546 pt_entry_t *src_pte, *dst_pte; 2547 2548 l1e = pmap_pml1e(src_pmap, addr); 2549 if ((be64toh(*l1e) & PG_V) == 0) { 2550 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2551 if (va_next < addr) 2552 va_next = end_addr; 2553 continue; 2554 } 2555 2556 l2e = pmap_l1e_to_l2e(l1e, addr); 2557 if ((be64toh(*l2e) & PG_V) == 0) { 2558 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2559 if (va_next < addr) 2560 va_next = end_addr; 2561 continue; 2562 } 2563 2564 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2565 if (va_next < addr) 2566 va_next = end_addr; 2567 2568 l3e = pmap_l2e_to_l3e(l2e, addr); 2569 srcptepaddr = be64toh(*l3e); 2570 if (srcptepaddr == 0) 2571 continue; 2572 2573 if (srcptepaddr & RPTE_LEAF) { 2574 if ((addr & L3_PAGE_MASK) != 0 || 2575 addr + L3_PAGE_SIZE > end_addr) 2576 continue; 2577 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL); 2578 if (dst_pdpg == NULL) 2579 break; 2580 l3e = (pml3_entry_t *) 2581 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg)); 2582 l3e = &l3e[pmap_pml3e_index(addr)]; 2583 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 || 2584 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr, 2585 PMAP_ENTER_NORECLAIM, &lock))) { 2586 *l3e = htobe64(srcptepaddr & ~PG_W); 2587 pmap_resident_count_inc(dst_pmap, 2588 L3_PAGE_SIZE / PAGE_SIZE); 2589 counter_u64_add(pmap_l3e_mappings, 1); 2590 } else 2591 dst_pdpg->ref_count--; 2592 continue; 2593 } 2594 2595 srcptepaddr &= PG_FRAME; 2596 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 2597 KASSERT(srcmpte->ref_count > 0, 2598 ("pmap_copy: source page table page is unused")); 2599 2600 if (va_next > end_addr) 2601 va_next = end_addr; 2602 2603 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); 2604 src_pte = &src_pte[pmap_pte_index(addr)]; 2605 dstmpte = NULL; 2606 while (addr < va_next) { 2607 pt_entry_t ptetemp; 2608 ptetemp = be64toh(*src_pte); 2609 /* 2610 * we only virtual copy managed pages 2611 */ 2612 if ((ptetemp & PG_MANAGED) != 0) { 2613 if (dstmpte != NULL && 2614 dstmpte->pindex == pmap_l3e_pindex(addr)) 2615 dstmpte->ref_count++; 2616 else if ((dstmpte = pmap_allocpte(dst_pmap, 2617 addr, NULL)) == NULL) 2618 goto out; 2619 dst_pte = (pt_entry_t *) 2620 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); 2621 dst_pte = &dst_pte[pmap_pte_index(addr)]; 2622 if (be64toh(*dst_pte) == 0 && 2623 pmap_try_insert_pv_entry(dst_pmap, addr, 2624 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), 2625 &lock)) { 2626 /* 2627 * Clear the wired, modified, and 2628 * accessed (referenced) bits 2629 * during the copy. 2630 */ 2631 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M | 2632 PG_A)); 2633 pmap_resident_count_inc(dst_pmap, 1); 2634 } else { 2635 SLIST_INIT(&free); 2636 if (pmap_unwire_ptp(dst_pmap, addr, 2637 dstmpte, &free)) { 2638 /* 2639 * Although "addr" is not 2640 * mapped, paging-structure 2641 * caches could nonetheless 2642 * have entries that refer to 2643 * the freed page table pages. 2644 * Invalidate those entries. 2645 */ 2646 invalidate_all = true; 2647 vm_page_free_pages_toq(&free, 2648 true); 2649 } 2650 goto out; 2651 } 2652 if (dstmpte->ref_count >= srcmpte->ref_count) 2653 break; 2654 } 2655 addr += PAGE_SIZE; 2656 if (__predict_false((addr & L3_PAGE_MASK) == 0)) 2657 src_pte = pmap_pte(src_pmap, addr); 2658 else 2659 src_pte++; 2660 } 2661 } 2662 out: 2663 if (invalidate_all) 2664 pmap_invalidate_all(dst_pmap); 2665 if (lock != NULL) 2666 rw_wunlock(lock); 2667 PMAP_UNLOCK(src_pmap); 2668 PMAP_UNLOCK(dst_pmap); 2669 } 2670 2671 static void 2672 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst) 2673 { 2674 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); 2675 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); 2676 2677 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst); 2678 /* 2679 * XXX slow 2680 */ 2681 bcopy((void *)src, (void *)dst, PAGE_SIZE); 2682 } 2683 2684 static void 2685 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2686 vm_offset_t b_offset, int xfersize) 2687 { 2688 void *a_cp, *b_cp; 2689 vm_offset_t a_pg_offset, b_pg_offset; 2690 int cnt; 2691 2692 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma, 2693 a_offset, mb, b_offset, xfersize); 2694 2695 while (xfersize > 0) { 2696 a_pg_offset = a_offset & PAGE_MASK; 2697 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 2698 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2699 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) + 2700 a_pg_offset; 2701 b_pg_offset = b_offset & PAGE_MASK; 2702 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 2703 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2704 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) + 2705 b_pg_offset; 2706 bcopy(a_cp, b_cp, cnt); 2707 a_offset += cnt; 2708 b_offset += cnt; 2709 xfersize -= cnt; 2710 } 2711 } 2712 2713 #if VM_NRESERVLEVEL > 0 2714 /* 2715 * Tries to promote the 512, contiguous 4KB page mappings that are within a 2716 * single page table page (PTP) to a single 2MB page mapping. For promotion 2717 * to occur, two conditions must be met: (1) the 4KB page mappings must map 2718 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 2719 * identical characteristics. 2720 */ 2721 static int 2722 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va, 2723 struct rwlock **lockp) 2724 { 2725 pml3_entry_t newpde; 2726 pt_entry_t *firstpte, oldpte, pa, *pte; 2727 vm_page_t mpte; 2728 2729 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2730 2731 /* 2732 * Examine the first PTE in the specified PTP. Abort if this PTE is 2733 * either invalid, unused, or does not map the first 4KB physical page 2734 * within a 2MB page. 2735 */ 2736 firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME); 2737 setpde: 2738 newpde = be64toh(*firstpte); 2739 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) { 2740 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2741 " in pmap %p", va, pmap); 2742 goto fail; 2743 } 2744 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 2745 /* 2746 * When PG_M is already clear, PG_RW can be cleared without 2747 * a TLB invalidation. 2748 */ 2749 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W))) 2750 goto setpde; 2751 newpde &= ~RPTE_EAA_W; 2752 } 2753 2754 /* 2755 * Examine each of the other PTEs in the specified PTP. Abort if this 2756 * PTE maps an unexpected 4KB physical page or does not have identical 2757 * characteristics to the first PTE. 2758 */ 2759 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE; 2760 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { 2761 setpte: 2762 oldpte = be64toh(*pte); 2763 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { 2764 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2765 " in pmap %p", va, pmap); 2766 goto fail; 2767 } 2768 if ((oldpte & (PG_M | PG_RW)) == PG_RW) { 2769 /* 2770 * When PG_M is already clear, PG_RW can be cleared 2771 * without a TLB invalidation. 2772 */ 2773 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W))) 2774 goto setpte; 2775 oldpte &= ~RPTE_EAA_W; 2776 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx" 2777 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) | 2778 (va & ~L3_PAGE_MASK), pmap); 2779 } 2780 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { 2781 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2782 " in pmap %p", va, pmap); 2783 goto fail; 2784 } 2785 pa -= PAGE_SIZE; 2786 } 2787 2788 /* 2789 * Save the page table page in its current state until the PDE 2790 * mapping the superpage is demoted by pmap_demote_pde() or 2791 * destroyed by pmap_remove_pde(). 2792 */ 2793 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME); 2794 KASSERT(mpte >= vm_page_array && 2795 mpte < &vm_page_array[vm_page_array_size], 2796 ("pmap_promote_l3e: page table page is out of range")); 2797 KASSERT(mpte->pindex == pmap_l3e_pindex(va), 2798 ("pmap_promote_l3e: page table page's pindex is wrong")); 2799 if (pmap_insert_pt_page(pmap, mpte)) { 2800 CTR2(KTR_PMAP, 2801 "pmap_promote_l3e: failure for va %#lx in pmap %p", va, 2802 pmap); 2803 goto fail; 2804 } 2805 2806 /* 2807 * Promote the pv entries. 2808 */ 2809 if ((newpde & PG_MANAGED) != 0) 2810 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp); 2811 2812 pte_store(pde, PG_PROMOTED | newpde); 2813 ptesync(); 2814 counter_u64_add(pmap_l3e_promotions, 1); 2815 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx" 2816 " in pmap %p", va, pmap); 2817 return (0); 2818 fail: 2819 counter_u64_add(pmap_l3e_p_failures, 1); 2820 return (KERN_FAILURE); 2821 } 2822 #endif /* VM_NRESERVLEVEL > 0 */ 2823 2824 int 2825 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, 2826 vm_prot_t prot, u_int flags, int8_t psind) 2827 { 2828 struct rwlock *lock; 2829 pml3_entry_t *l3e; 2830 pt_entry_t *pte; 2831 pt_entry_t newpte, origpte; 2832 pv_entry_t pv; 2833 vm_paddr_t opa, pa; 2834 vm_page_t mpte, om; 2835 int rv, retrycount; 2836 boolean_t nosleep, invalidate_all, invalidate_page; 2837 2838 va = trunc_page(va); 2839 retrycount = 0; 2840 invalidate_page = invalidate_all = false; 2841 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va, 2842 m, prot, flags, psind); 2843 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 2844 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va), 2845 ("pmap_enter: managed mapping within the clean submap")); 2846 if ((m->oflags & VPO_UNMANAGED) == 0) 2847 VM_PAGE_OBJECT_BUSY_ASSERT(m); 2848 2849 KASSERT((flags & PMAP_ENTER_RESERVED) == 0, 2850 ("pmap_enter: flags %u has reserved bits set", flags)); 2851 pa = VM_PAGE_TO_PHYS(m); 2852 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF); 2853 if ((flags & VM_PROT_WRITE) != 0) 2854 newpte |= PG_M; 2855 if ((flags & VM_PROT_READ) != 0) 2856 newpte |= PG_A; 2857 if (prot & VM_PROT_READ) 2858 newpte |= RPTE_EAA_R; 2859 if ((prot & VM_PROT_WRITE) != 0) 2860 newpte |= RPTE_EAA_W; 2861 KASSERT((newpte & (PG_M | PG_RW)) != PG_M, 2862 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); 2863 2864 if (prot & VM_PROT_EXECUTE) 2865 newpte |= PG_X; 2866 if ((flags & PMAP_ENTER_WIRED) != 0) 2867 newpte |= PG_W; 2868 if (va >= DMAP_MIN_ADDRESS) 2869 newpte |= RPTE_EAA_P; 2870 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs); 2871 /* 2872 * Set modified bit gratuitously for writeable mappings if 2873 * the page is unmanaged. We do not want to take a fault 2874 * to do the dirty bit accounting for these mappings. 2875 */ 2876 if ((m->oflags & VPO_UNMANAGED) != 0) { 2877 if ((newpte & PG_RW) != 0) 2878 newpte |= PG_M; 2879 } else 2880 newpte |= PG_MANAGED; 2881 2882 lock = NULL; 2883 PMAP_LOCK(pmap); 2884 if (psind == 1) { 2885 /* Assert the required virtual and physical alignment. */ 2886 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned")); 2887 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 2888 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock); 2889 goto out; 2890 } 2891 mpte = NULL; 2892 2893 /* 2894 * In the case that a page table page is not 2895 * resident, we are creating it here. 2896 */ 2897 retry: 2898 l3e = pmap_pml3e(pmap, va); 2899 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 || 2900 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) { 2901 pte = pmap_l3e_to_pte(l3e, va); 2902 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { 2903 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 2904 mpte->ref_count++; 2905 } 2906 } else if (va < VM_MAXUSER_ADDRESS) { 2907 /* 2908 * Here if the pte page isn't mapped, or if it has been 2909 * deallocated. 2910 */ 2911 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 2912 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va), 2913 nosleep ? NULL : &lock); 2914 if (mpte == NULL && nosleep) { 2915 rv = KERN_RESOURCE_SHORTAGE; 2916 goto out; 2917 } 2918 if (__predict_false(retrycount++ == 6)) 2919 panic("too many retries"); 2920 invalidate_all = true; 2921 goto retry; 2922 } else 2923 panic("pmap_enter: invalid page directory va=%#lx", va); 2924 2925 origpte = be64toh(*pte); 2926 pv = NULL; 2927 2928 /* 2929 * Is the specified virtual address already mapped? 2930 */ 2931 if ((origpte & PG_V) != 0) { 2932 #ifdef INVARIANTS 2933 if (VERBOSE_PMAP || pmap_logging) { 2934 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --" 2935 " asid=%lu curpid=%d name=%s origpte0x%lx\n", 2936 pmap, va, m, prot, flags, psind, pmap->pm_pid, 2937 curproc->p_pid, curproc->p_comm, origpte); 2938 #ifdef DDB 2939 pmap_pte_walk(pmap->pm_pml1, va); 2940 #endif 2941 } 2942 #endif 2943 /* 2944 * Wiring change, just update stats. We don't worry about 2945 * wiring PT pages as they remain resident as long as there 2946 * are valid mappings in them. Hence, if a user page is wired, 2947 * the PT page will be also. 2948 */ 2949 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) 2950 pmap->pm_stats.wired_count++; 2951 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) 2952 pmap->pm_stats.wired_count--; 2953 2954 /* 2955 * Remove the extra PT page reference. 2956 */ 2957 if (mpte != NULL) { 2958 mpte->ref_count--; 2959 KASSERT(mpte->ref_count > 0, 2960 ("pmap_enter: missing reference to page table page," 2961 " va: 0x%lx", va)); 2962 } 2963 2964 /* 2965 * Has the physical page changed? 2966 */ 2967 opa = origpte & PG_FRAME; 2968 if (opa == pa) { 2969 /* 2970 * No, might be a protection or wiring change. 2971 */ 2972 if ((origpte & PG_MANAGED) != 0 && 2973 (newpte & PG_RW) != 0) 2974 vm_page_aflag_set(m, PGA_WRITEABLE); 2975 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) { 2976 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) { 2977 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 2978 goto retry; 2979 if ((newpte & PG_M) != (origpte & PG_M)) 2980 vm_page_dirty(m); 2981 if ((newpte & PG_A) != (origpte & PG_A)) 2982 vm_page_aflag_set(m, PGA_REFERENCED); 2983 ptesync(); 2984 } else 2985 invalidate_all = true; 2986 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) 2987 goto unchanged; 2988 } 2989 goto validate; 2990 } 2991 2992 /* 2993 * The physical page has changed. Temporarily invalidate 2994 * the mapping. This ensures that all threads sharing the 2995 * pmap keep a consistent view of the mapping, which is 2996 * necessary for the correct handling of COW faults. It 2997 * also permits reuse of the old mapping's PV entry, 2998 * avoiding an allocation. 2999 * 3000 * For consistency, handle unmanaged mappings the same way. 3001 */ 3002 origpte = be64toh(pte_load_clear(pte)); 3003 KASSERT((origpte & PG_FRAME) == opa, 3004 ("pmap_enter: unexpected pa update for %#lx", va)); 3005 if ((origpte & PG_MANAGED) != 0) { 3006 om = PHYS_TO_VM_PAGE(opa); 3007 3008 /* 3009 * The pmap lock is sufficient to synchronize with 3010 * concurrent calls to pmap_page_test_mappings() and 3011 * pmap_ts_referenced(). 3012 */ 3013 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3014 vm_page_dirty(om); 3015 if ((origpte & PG_A) != 0) 3016 vm_page_aflag_set(om, PGA_REFERENCED); 3017 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 3018 pv = pmap_pvh_remove(&om->md, pmap, va); 3019 if ((newpte & PG_MANAGED) == 0) 3020 free_pv_entry(pmap, pv); 3021 #ifdef INVARIANTS 3022 else if (origpte & PG_MANAGED) { 3023 if (pv == NULL) { 3024 #ifdef DDB 3025 pmap_page_print_mappings(om); 3026 #endif 3027 MPASS(pv != NULL); 3028 } 3029 } 3030 #endif 3031 if ((om->a.flags & PGA_WRITEABLE) != 0 && 3032 TAILQ_EMPTY(&om->md.pv_list) && 3033 ((om->flags & PG_FICTITIOUS) != 0 || 3034 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) 3035 vm_page_aflag_clear(om, PGA_WRITEABLE); 3036 } 3037 if ((origpte & PG_A) != 0) 3038 invalidate_page = true; 3039 origpte = 0; 3040 } else { 3041 if (pmap != kernel_pmap) { 3042 #ifdef INVARIANTS 3043 if (VERBOSE_PMAP || pmap_logging) 3044 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n", 3045 pmap, va, m, prot, flags, psind, 3046 pmap->pm_pid, curproc->p_pid, 3047 curproc->p_comm); 3048 #endif 3049 } 3050 3051 /* 3052 * Increment the counters. 3053 */ 3054 if ((newpte & PG_W) != 0) 3055 pmap->pm_stats.wired_count++; 3056 pmap_resident_count_inc(pmap, 1); 3057 } 3058 3059 /* 3060 * Enter on the PV list if part of our managed memory. 3061 */ 3062 if ((newpte & PG_MANAGED) != 0) { 3063 if (pv == NULL) { 3064 pv = get_pv_entry(pmap, &lock); 3065 pv->pv_va = va; 3066 } 3067 #ifdef VERBOSE_PV 3068 else 3069 printf("reassigning pv: %p to pmap: %p\n", 3070 pv, pmap); 3071 #endif 3072 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 3073 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3074 m->md.pv_gen++; 3075 if ((newpte & PG_RW) != 0) 3076 vm_page_aflag_set(m, PGA_WRITEABLE); 3077 } 3078 3079 /* 3080 * Update the PTE. 3081 */ 3082 if ((origpte & PG_V) != 0) { 3083 validate: 3084 origpte = be64toh(pte_load_store(pte, htobe64(newpte))); 3085 KASSERT((origpte & PG_FRAME) == pa, 3086 ("pmap_enter: unexpected pa update for %#lx", va)); 3087 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == 3088 (PG_M | PG_RW)) { 3089 if ((origpte & PG_MANAGED) != 0) 3090 vm_page_dirty(m); 3091 invalidate_page = true; 3092 3093 /* 3094 * Although the PTE may still have PG_RW set, TLB 3095 * invalidation may nonetheless be required because 3096 * the PTE no longer has PG_M set. 3097 */ 3098 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) { 3099 /* 3100 * Removing capabilities requires invalidation on POWER 3101 */ 3102 invalidate_page = true; 3103 goto unchanged; 3104 } 3105 if ((origpte & PG_A) != 0) 3106 invalidate_page = true; 3107 } else { 3108 pte_store(pte, newpte); 3109 ptesync(); 3110 } 3111 unchanged: 3112 3113 #if VM_NRESERVLEVEL > 0 3114 /* 3115 * If both the page table page and the reservation are fully 3116 * populated, then attempt promotion. 3117 */ 3118 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 3119 mmu_radix_ps_enabled(pmap) && 3120 (m->flags & PG_FICTITIOUS) == 0 && 3121 vm_reserv_level_iffullpop(m) == 0 && 3122 pmap_promote_l3e(pmap, l3e, va, &lock) == 0) 3123 invalidate_all = true; 3124 #endif 3125 if (invalidate_all) 3126 pmap_invalidate_all(pmap); 3127 else if (invalidate_page) 3128 pmap_invalidate_page(pmap, va); 3129 3130 rv = KERN_SUCCESS; 3131 out: 3132 if (lock != NULL) 3133 rw_wunlock(lock); 3134 PMAP_UNLOCK(pmap); 3135 3136 return (rv); 3137 } 3138 3139 /* 3140 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true 3141 * if successful. Returns false if (1) a page table page cannot be allocated 3142 * without sleeping, (2) a mapping already exists at the specified virtual 3143 * address, or (3) a PV entry cannot be allocated without reclaiming another 3144 * PV entry. 3145 */ 3146 static bool 3147 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3148 struct rwlock **lockp) 3149 { 3150 pml3_entry_t newpde; 3151 3152 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3153 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) | 3154 RPTE_LEAF | PG_V; 3155 if ((m->oflags & VPO_UNMANAGED) == 0) 3156 newpde |= PG_MANAGED; 3157 if (prot & VM_PROT_EXECUTE) 3158 newpde |= PG_X; 3159 if (prot & VM_PROT_READ) 3160 newpde |= RPTE_EAA_R; 3161 if (va >= DMAP_MIN_ADDRESS) 3162 newpde |= RPTE_EAA_P; 3163 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP | 3164 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == 3165 KERN_SUCCESS); 3166 } 3167 3168 /* 3169 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if 3170 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE 3171 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and 3172 * a mapping already exists at the specified virtual address. Returns 3173 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table 3174 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if 3175 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. 3176 * 3177 * The parameter "m" is only used when creating a managed, writeable mapping. 3178 */ 3179 static int 3180 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags, 3181 vm_page_t m, struct rwlock **lockp) 3182 { 3183 struct spglist free; 3184 pml3_entry_t oldl3e, *l3e; 3185 vm_page_t mt, pdpg; 3186 3187 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW, 3188 ("pmap_enter_pde: newpde is missing PG_M")); 3189 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3190 3191 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 3192 NULL : lockp)) == NULL) { 3193 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3194 " in pmap %p", va, pmap); 3195 return (KERN_RESOURCE_SHORTAGE); 3196 } 3197 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 3198 l3e = &l3e[pmap_pml3e_index(va)]; 3199 oldl3e = be64toh(*l3e); 3200 if ((oldl3e & PG_V) != 0) { 3201 KASSERT(pdpg->ref_count > 1, 3202 ("pmap_enter_pde: pdpg's wire count is too low")); 3203 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 3204 pdpg->ref_count--; 3205 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3206 " in pmap %p", va, pmap); 3207 return (KERN_FAILURE); 3208 } 3209 /* Break the existing mapping(s). */ 3210 SLIST_INIT(&free); 3211 if ((oldl3e & RPTE_LEAF) != 0) { 3212 /* 3213 * The reference to the PD page that was acquired by 3214 * pmap_allocl3e() ensures that it won't be freed. 3215 * However, if the PDE resulted from a promotion, then 3216 * a reserved PT page could be freed. 3217 */ 3218 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp); 3219 pmap_invalidate_l3e_page(pmap, va, oldl3e); 3220 } else { 3221 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e, 3222 &free, lockp)) 3223 pmap_invalidate_all(pmap); 3224 } 3225 vm_page_free_pages_toq(&free, true); 3226 if (va >= VM_MAXUSER_ADDRESS) { 3227 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 3228 if (pmap_insert_pt_page(pmap, mt)) { 3229 /* 3230 * XXX Currently, this can't happen because 3231 * we do not perform pmap_enter(psind == 1) 3232 * on the kernel pmap. 3233 */ 3234 panic("pmap_enter_pde: trie insert failed"); 3235 } 3236 } else 3237 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p", 3238 l3e)); 3239 } 3240 if ((newpde & PG_MANAGED) != 0) { 3241 /* 3242 * Abort this mapping if its PV entry could not be created. 3243 */ 3244 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) { 3245 SLIST_INIT(&free); 3246 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { 3247 /* 3248 * Although "va" is not mapped, paging- 3249 * structure caches could nonetheless have 3250 * entries that refer to the freed page table 3251 * pages. Invalidate those entries. 3252 */ 3253 pmap_invalidate_page(pmap, va); 3254 vm_page_free_pages_toq(&free, true); 3255 } 3256 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3257 " in pmap %p", va, pmap); 3258 return (KERN_RESOURCE_SHORTAGE); 3259 } 3260 if ((newpde & PG_RW) != 0) { 3261 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 3262 vm_page_aflag_set(mt, PGA_WRITEABLE); 3263 } 3264 } 3265 3266 /* 3267 * Increment counters. 3268 */ 3269 if ((newpde & PG_W) != 0) 3270 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE; 3271 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 3272 3273 /* 3274 * Map the superpage. (This is not a promoted mapping; there will not 3275 * be any lingering 4KB page mappings in the TLB.) 3276 */ 3277 pte_store(l3e, newpde); 3278 ptesync(); 3279 3280 counter_u64_add(pmap_l3e_mappings, 1); 3281 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" 3282 " in pmap %p", va, pmap); 3283 return (KERN_SUCCESS); 3284 } 3285 3286 void 3287 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start, 3288 vm_offset_t end, vm_page_t m_start, vm_prot_t prot) 3289 { 3290 3291 struct rwlock *lock; 3292 vm_offset_t va; 3293 vm_page_t m, mpte; 3294 vm_pindex_t diff, psize; 3295 bool invalidate; 3296 VM_OBJECT_ASSERT_LOCKED(m_start->object); 3297 3298 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start, 3299 end, m_start, prot); 3300 3301 invalidate = false; 3302 psize = atop(end - start); 3303 mpte = NULL; 3304 m = m_start; 3305 lock = NULL; 3306 PMAP_LOCK(pmap); 3307 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 3308 va = start + ptoa(diff); 3309 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end && 3310 m->psind == 1 && mmu_radix_ps_enabled(pmap) && 3311 pmap_enter_2mpage(pmap, va, m, prot, &lock)) 3312 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1]; 3313 else 3314 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot, 3315 mpte, &lock, &invalidate); 3316 m = TAILQ_NEXT(m, listq); 3317 } 3318 ptesync(); 3319 if (lock != NULL) 3320 rw_wunlock(lock); 3321 if (invalidate) 3322 pmap_invalidate_all(pmap); 3323 PMAP_UNLOCK(pmap); 3324 } 3325 3326 static vm_page_t 3327 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 3328 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate) 3329 { 3330 struct spglist free; 3331 pt_entry_t *pte; 3332 vm_paddr_t pa; 3333 3334 KASSERT(!VA_IS_CLEANMAP(va) || 3335 (m->oflags & VPO_UNMANAGED) != 0, 3336 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap")); 3337 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3338 3339 /* 3340 * In the case that a page table page is not 3341 * resident, we are creating it here. 3342 */ 3343 if (va < VM_MAXUSER_ADDRESS) { 3344 vm_pindex_t ptepindex; 3345 pml3_entry_t *ptepa; 3346 3347 /* 3348 * Calculate pagetable page index 3349 */ 3350 ptepindex = pmap_l3e_pindex(va); 3351 if (mpte && (mpte->pindex == ptepindex)) { 3352 mpte->ref_count++; 3353 } else { 3354 /* 3355 * Get the page directory entry 3356 */ 3357 ptepa = pmap_pml3e(pmap, va); 3358 3359 /* 3360 * If the page table page is mapped, we just increment 3361 * the hold count, and activate it. Otherwise, we 3362 * attempt to allocate a page table page. If this 3363 * attempt fails, we don't retry. Instead, we give up. 3364 */ 3365 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) { 3366 if (be64toh(*ptepa) & RPTE_LEAF) 3367 return (NULL); 3368 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME); 3369 mpte->ref_count++; 3370 } else { 3371 /* 3372 * Pass NULL instead of the PV list lock 3373 * pointer, because we don't intend to sleep. 3374 */ 3375 mpte = _pmap_allocpte(pmap, ptepindex, NULL); 3376 if (mpte == NULL) 3377 return (mpte); 3378 } 3379 } 3380 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); 3381 pte = &pte[pmap_pte_index(va)]; 3382 } else { 3383 mpte = NULL; 3384 pte = pmap_pte(pmap, va); 3385 } 3386 if (be64toh(*pte)) { 3387 if (mpte != NULL) { 3388 mpte->ref_count--; 3389 mpte = NULL; 3390 } 3391 return (mpte); 3392 } 3393 3394 /* 3395 * Enter on the PV list if part of our managed memory. 3396 */ 3397 if ((m->oflags & VPO_UNMANAGED) == 0 && 3398 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 3399 if (mpte != NULL) { 3400 SLIST_INIT(&free); 3401 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { 3402 /* 3403 * Although "va" is not mapped, paging- 3404 * structure caches could nonetheless have 3405 * entries that refer to the freed page table 3406 * pages. Invalidate those entries. 3407 */ 3408 *invalidate = true; 3409 vm_page_free_pages_toq(&free, true); 3410 } 3411 mpte = NULL; 3412 } 3413 return (mpte); 3414 } 3415 3416 /* 3417 * Increment counters 3418 */ 3419 pmap_resident_count_inc(pmap, 1); 3420 3421 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs); 3422 if (prot & VM_PROT_EXECUTE) 3423 pa |= PG_X; 3424 else 3425 pa |= RPTE_EAA_R; 3426 if ((m->oflags & VPO_UNMANAGED) == 0) 3427 pa |= PG_MANAGED; 3428 3429 pte_store(pte, pa); 3430 return (mpte); 3431 } 3432 3433 void 3434 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, 3435 vm_prot_t prot) 3436 { 3437 struct rwlock *lock; 3438 bool invalidate; 3439 3440 lock = NULL; 3441 invalidate = false; 3442 PMAP_LOCK(pmap); 3443 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock, 3444 &invalidate); 3445 ptesync(); 3446 if (lock != NULL) 3447 rw_wunlock(lock); 3448 if (invalidate) 3449 pmap_invalidate_all(pmap); 3450 PMAP_UNLOCK(pmap); 3451 } 3452 3453 vm_paddr_t 3454 mmu_radix_extract(pmap_t pmap, vm_offset_t va) 3455 { 3456 pml3_entry_t *l3e; 3457 pt_entry_t *pte; 3458 vm_paddr_t pa; 3459 3460 l3e = pmap_pml3e(pmap, va); 3461 if (__predict_false(l3e == NULL)) 3462 return (0); 3463 if (be64toh(*l3e) & RPTE_LEAF) { 3464 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 3465 pa |= (va & L3_PAGE_MASK); 3466 } else { 3467 /* 3468 * Beware of a concurrent promotion that changes the 3469 * PDE at this point! For example, vtopte() must not 3470 * be used to access the PTE because it would use the 3471 * new PDE. It is, however, safe to use the old PDE 3472 * because the page table page is preserved by the 3473 * promotion. 3474 */ 3475 pte = pmap_l3e_to_pte(l3e, va); 3476 if (__predict_false(pte == NULL)) 3477 return (0); 3478 pa = be64toh(*pte); 3479 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 3480 pa |= (va & PAGE_MASK); 3481 } 3482 return (pa); 3483 } 3484 3485 vm_page_t 3486 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3487 { 3488 pml3_entry_t l3e, *l3ep; 3489 pt_entry_t pte; 3490 vm_page_t m; 3491 3492 m = NULL; 3493 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot); 3494 PMAP_LOCK(pmap); 3495 l3ep = pmap_pml3e(pmap, va); 3496 if (l3ep != NULL && (l3e = be64toh(*l3ep))) { 3497 if (l3e & RPTE_LEAF) { 3498 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0) 3499 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) | 3500 (va & L3_PAGE_MASK)); 3501 } else { 3502 /* Native endian PTE, do not pass to pmap functions */ 3503 pte = be64toh(*pmap_l3e_to_pte(l3ep, va)); 3504 if ((pte & PG_V) && 3505 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) 3506 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 3507 } 3508 if (m != NULL && !vm_page_wire_mapped(m)) 3509 m = NULL; 3510 } 3511 PMAP_UNLOCK(pmap); 3512 return (m); 3513 } 3514 3515 static void 3516 mmu_radix_growkernel(vm_offset_t addr) 3517 { 3518 vm_paddr_t paddr; 3519 vm_page_t nkpg; 3520 pml3_entry_t *l3e; 3521 pml2_entry_t *l2e; 3522 3523 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 3524 if (VM_MIN_KERNEL_ADDRESS < addr && 3525 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE)) 3526 return; 3527 3528 addr = roundup2(addr, L3_PAGE_SIZE); 3529 if (addr - 1 >= vm_map_max(kernel_map)) 3530 addr = vm_map_max(kernel_map); 3531 while (kernel_vm_end < addr) { 3532 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); 3533 if ((be64toh(*l2e) & PG_V) == 0) { 3534 /* We need a new PDP entry */ 3535 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | 3536 VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3537 if (nkpg == NULL) 3538 panic("pmap_growkernel: no memory to grow kernel"); 3539 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT; 3540 paddr = VM_PAGE_TO_PHYS(nkpg); 3541 pde_store(l2e, paddr); 3542 continue; /* try again */ 3543 } 3544 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); 3545 if ((be64toh(*l3e) & PG_V) != 0) { 3546 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3547 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3548 kernel_vm_end = vm_map_max(kernel_map); 3549 break; 3550 } 3551 continue; 3552 } 3553 3554 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | 3555 VM_ALLOC_ZERO); 3556 if (nkpg == NULL) 3557 panic("pmap_growkernel: no memory to grow kernel"); 3558 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end); 3559 paddr = VM_PAGE_TO_PHYS(nkpg); 3560 pde_store(l3e, paddr); 3561 3562 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3563 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3564 kernel_vm_end = vm_map_max(kernel_map); 3565 break; 3566 } 3567 } 3568 ptesync(); 3569 } 3570 3571 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory"); 3572 static uma_zone_t zone_radix_pgd; 3573 3574 static int 3575 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused, 3576 int flags) 3577 { 3578 int req; 3579 3580 req = VM_ALLOC_WIRED | malloc2vm_flags(flags); 3581 for (int i = 0; i < count; i++) { 3582 vm_page_t m = vm_page_alloc_noobj_contig(req, 3583 RADIX_PGD_SIZE / PAGE_SIZE, 3584 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE, 3585 VM_MEMATTR_DEFAULT); 3586 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 3587 } 3588 return (count); 3589 } 3590 3591 static void 3592 radix_pgd_release(void *arg __unused, void **store, int count) 3593 { 3594 vm_page_t m; 3595 struct spglist free; 3596 int page_count; 3597 3598 SLIST_INIT(&free); 3599 page_count = RADIX_PGD_SIZE/PAGE_SIZE; 3600 3601 for (int i = 0; i < count; i++) { 3602 /* 3603 * XXX selectively remove dmap and KVA entries so we don't 3604 * need to bzero 3605 */ 3606 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i])); 3607 for (int j = page_count-1; j >= 0; j--) { 3608 vm_page_unwire_noq(&m[j]); 3609 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss); 3610 } 3611 vm_page_free_pages_toq(&free, false); 3612 } 3613 } 3614 3615 static void 3616 mmu_radix_init(void) 3617 { 3618 vm_page_t mpte; 3619 vm_size_t s; 3620 int error, i, pv_npg; 3621 3622 /* XXX is this really needed for POWER? */ 3623 /* L1TF, reserve page @0 unconditionally */ 3624 vm_page_blacklist_add(0, bootverbose); 3625 3626 zone_radix_pgd = uma_zcache_create("radix_pgd_cache", 3627 RADIX_PGD_SIZE, NULL, NULL, 3628 #ifdef INVARIANTS 3629 trash_init, trash_fini, 3630 #else 3631 NULL, NULL, 3632 #endif 3633 radix_pgd_import, radix_pgd_release, 3634 NULL, UMA_ZONE_NOBUCKET); 3635 3636 /* 3637 * Initialize the vm page array entries for the kernel pmap's 3638 * page table pages. 3639 */ 3640 PMAP_LOCK(kernel_pmap); 3641 for (i = 0; i < nkpt; i++) { 3642 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 3643 KASSERT(mpte >= vm_page_array && 3644 mpte < &vm_page_array[vm_page_array_size], 3645 ("pmap_init: page table page is out of range size: %lu", 3646 vm_page_array_size)); 3647 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i; 3648 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); 3649 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte); 3650 //pmap_insert_pt_page(kernel_pmap, mpte); 3651 mpte->ref_count = 1; 3652 } 3653 PMAP_UNLOCK(kernel_pmap); 3654 vm_wire_add(nkpt); 3655 3656 CTR1(KTR_PMAP, "%s()", __func__); 3657 TAILQ_INIT(&pv_dummy.pv_list); 3658 3659 /* 3660 * Are large page mappings enabled? 3661 */ 3662 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled); 3663 if (superpages_enabled) { 3664 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 3665 ("pmap_init: can't assign to pagesizes[1]")); 3666 pagesizes[1] = L3_PAGE_SIZE; 3667 } 3668 3669 /* 3670 * Initialize the pv chunk list mutex. 3671 */ 3672 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); 3673 3674 /* 3675 * Initialize the pool of pv list locks. 3676 */ 3677 for (i = 0; i < NPV_LIST_LOCKS; i++) 3678 rw_init(&pv_list_locks[i], "pmap pv list"); 3679 3680 /* 3681 * Calculate the size of the pv head table for superpages. 3682 */ 3683 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE); 3684 3685 /* 3686 * Allocate memory for the pv head table for superpages. 3687 */ 3688 s = (vm_size_t)(pv_npg * sizeof(struct md_page)); 3689 s = round_page(s); 3690 pv_table = kmem_malloc(s, M_WAITOK | M_ZERO); 3691 for (i = 0; i < pv_npg; i++) 3692 TAILQ_INIT(&pv_table[i].pv_list); 3693 TAILQ_INIT(&pv_dummy.pv_list); 3694 3695 pmap_initialized = 1; 3696 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); 3697 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 3698 (vmem_addr_t *)&qframe); 3699 3700 if (error != 0) 3701 panic("qframe allocation failed"); 3702 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits), 3703 1, 1, M_WAITOK); 3704 } 3705 3706 static boolean_t 3707 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) 3708 { 3709 struct rwlock *lock; 3710 pv_entry_t pv; 3711 struct md_page *pvh; 3712 pt_entry_t *pte, mask; 3713 pmap_t pmap; 3714 int md_gen, pvh_gen; 3715 boolean_t rv; 3716 3717 rv = FALSE; 3718 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 3719 rw_rlock(lock); 3720 restart: 3721 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 3722 pmap = PV_PMAP(pv); 3723 if (!PMAP_TRYLOCK(pmap)) { 3724 md_gen = m->md.pv_gen; 3725 rw_runlock(lock); 3726 PMAP_LOCK(pmap); 3727 rw_rlock(lock); 3728 if (md_gen != m->md.pv_gen) { 3729 PMAP_UNLOCK(pmap); 3730 goto restart; 3731 } 3732 } 3733 pte = pmap_pte(pmap, pv->pv_va); 3734 mask = 0; 3735 if (modified) 3736 mask |= PG_RW | PG_M; 3737 if (accessed) 3738 mask |= PG_V | PG_A; 3739 rv = (be64toh(*pte) & mask) == mask; 3740 PMAP_UNLOCK(pmap); 3741 if (rv) 3742 goto out; 3743 } 3744 if ((m->flags & PG_FICTITIOUS) == 0) { 3745 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 3746 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 3747 pmap = PV_PMAP(pv); 3748 if (!PMAP_TRYLOCK(pmap)) { 3749 md_gen = m->md.pv_gen; 3750 pvh_gen = pvh->pv_gen; 3751 rw_runlock(lock); 3752 PMAP_LOCK(pmap); 3753 rw_rlock(lock); 3754 if (md_gen != m->md.pv_gen || 3755 pvh_gen != pvh->pv_gen) { 3756 PMAP_UNLOCK(pmap); 3757 goto restart; 3758 } 3759 } 3760 pte = pmap_pml3e(pmap, pv->pv_va); 3761 mask = 0; 3762 if (modified) 3763 mask |= PG_RW | PG_M; 3764 if (accessed) 3765 mask |= PG_V | PG_A; 3766 rv = (be64toh(*pte) & mask) == mask; 3767 PMAP_UNLOCK(pmap); 3768 if (rv) 3769 goto out; 3770 } 3771 } 3772 out: 3773 rw_runlock(lock); 3774 return (rv); 3775 } 3776 3777 /* 3778 * pmap_is_modified: 3779 * 3780 * Return whether or not the specified physical page was modified 3781 * in any physical maps. 3782 */ 3783 boolean_t 3784 mmu_radix_is_modified(vm_page_t m) 3785 { 3786 3787 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3788 ("pmap_is_modified: page %p is not managed", m)); 3789 3790 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3791 /* 3792 * If the page is not busied then this check is racy. 3793 */ 3794 if (!pmap_page_is_write_mapped(m)) 3795 return (FALSE); 3796 return (pmap_page_test_mappings(m, FALSE, TRUE)); 3797 } 3798 3799 boolean_t 3800 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3801 { 3802 pml3_entry_t *l3e; 3803 pt_entry_t *pte; 3804 boolean_t rv; 3805 3806 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 3807 rv = FALSE; 3808 PMAP_LOCK(pmap); 3809 l3e = pmap_pml3e(pmap, addr); 3810 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) { 3811 pte = pmap_l3e_to_pte(l3e, addr); 3812 rv = (be64toh(*pte) & PG_V) == 0; 3813 } 3814 PMAP_UNLOCK(pmap); 3815 return (rv); 3816 } 3817 3818 boolean_t 3819 mmu_radix_is_referenced(vm_page_t m) 3820 { 3821 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3822 ("pmap_is_referenced: page %p is not managed", m)); 3823 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3824 return (pmap_page_test_mappings(m, TRUE, FALSE)); 3825 } 3826 3827 /* 3828 * pmap_ts_referenced: 3829 * 3830 * Return a count of reference bits for a page, clearing those bits. 3831 * It is not necessary for every reference bit to be cleared, but it 3832 * is necessary that 0 only be returned when there are truly no 3833 * reference bits set. 3834 * 3835 * As an optimization, update the page's dirty field if a modified bit is 3836 * found while counting reference bits. This opportunistic update can be 3837 * performed at low cost and can eliminate the need for some future calls 3838 * to pmap_is_modified(). However, since this function stops after 3839 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 3840 * dirty pages. Those dirty pages will only be detected by a future call 3841 * to pmap_is_modified(). 3842 * 3843 * A DI block is not needed within this function, because 3844 * invalidations are performed before the PV list lock is 3845 * released. 3846 */ 3847 int 3848 mmu_radix_ts_referenced(vm_page_t m) 3849 { 3850 struct md_page *pvh; 3851 pv_entry_t pv, pvf; 3852 pmap_t pmap; 3853 struct rwlock *lock; 3854 pml3_entry_t oldl3e, *l3e; 3855 pt_entry_t *pte; 3856 vm_paddr_t pa; 3857 int cleared, md_gen, not_cleared, pvh_gen; 3858 struct spglist free; 3859 3860 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3861 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3862 ("pmap_ts_referenced: page %p is not managed", m)); 3863 SLIST_INIT(&free); 3864 cleared = 0; 3865 pa = VM_PAGE_TO_PHYS(m); 3866 lock = PHYS_TO_PV_LIST_LOCK(pa); 3867 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); 3868 rw_wlock(lock); 3869 retry: 3870 not_cleared = 0; 3871 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 3872 goto small_mappings; 3873 pv = pvf; 3874 do { 3875 if (pvf == NULL) 3876 pvf = pv; 3877 pmap = PV_PMAP(pv); 3878 if (!PMAP_TRYLOCK(pmap)) { 3879 pvh_gen = pvh->pv_gen; 3880 rw_wunlock(lock); 3881 PMAP_LOCK(pmap); 3882 rw_wlock(lock); 3883 if (pvh_gen != pvh->pv_gen) { 3884 PMAP_UNLOCK(pmap); 3885 goto retry; 3886 } 3887 } 3888 l3e = pmap_pml3e(pmap, pv->pv_va); 3889 oldl3e = be64toh(*l3e); 3890 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 3891 /* 3892 * Although "oldpde" is mapping a 2MB page, because 3893 * this function is called at a 4KB page granularity, 3894 * we only update the 4KB page under test. 3895 */ 3896 vm_page_dirty(m); 3897 } 3898 if ((oldl3e & PG_A) != 0) { 3899 /* 3900 * Since this reference bit is shared by 512 4KB 3901 * pages, it should not be cleared every time it is 3902 * tested. Apply a simple "hash" function on the 3903 * physical page number, the virtual superpage number, 3904 * and the pmap address to select one 4KB page out of 3905 * the 512 on which testing the reference bit will 3906 * result in clearing that reference bit. This 3907 * function is designed to avoid the selection of the 3908 * same 4KB page for every 2MB page mapping. 3909 * 3910 * On demotion, a mapping that hasn't been referenced 3911 * is simply destroyed. To avoid the possibility of a 3912 * subsequent page fault on a demoted wired mapping, 3913 * always leave its reference bit set. Moreover, 3914 * since the superpage is wired, the current state of 3915 * its reference bit won't affect page replacement. 3916 */ 3917 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^ 3918 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && 3919 (oldl3e & PG_W) == 0) { 3920 atomic_clear_long(l3e, htobe64(PG_A)); 3921 pmap_invalidate_page(pmap, pv->pv_va); 3922 cleared++; 3923 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 3924 ("inconsistent pv lock %p %p for page %p", 3925 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 3926 } else 3927 not_cleared++; 3928 } 3929 PMAP_UNLOCK(pmap); 3930 /* Rotate the PV list if it has more than one entry. */ 3931 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3932 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 3933 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 3934 pvh->pv_gen++; 3935 } 3936 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 3937 goto out; 3938 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 3939 small_mappings: 3940 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 3941 goto out; 3942 pv = pvf; 3943 do { 3944 if (pvf == NULL) 3945 pvf = pv; 3946 pmap = PV_PMAP(pv); 3947 if (!PMAP_TRYLOCK(pmap)) { 3948 pvh_gen = pvh->pv_gen; 3949 md_gen = m->md.pv_gen; 3950 rw_wunlock(lock); 3951 PMAP_LOCK(pmap); 3952 rw_wlock(lock); 3953 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 3954 PMAP_UNLOCK(pmap); 3955 goto retry; 3956 } 3957 } 3958 l3e = pmap_pml3e(pmap, pv->pv_va); 3959 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 3960 ("pmap_ts_referenced: found a 2mpage in page %p's pv list", 3961 m)); 3962 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 3963 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3964 vm_page_dirty(m); 3965 if ((be64toh(*pte) & PG_A) != 0) { 3966 atomic_clear_long(pte, htobe64(PG_A)); 3967 pmap_invalidate_page(pmap, pv->pv_va); 3968 cleared++; 3969 } 3970 PMAP_UNLOCK(pmap); 3971 /* Rotate the PV list if it has more than one entry. */ 3972 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3973 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 3974 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3975 m->md.pv_gen++; 3976 } 3977 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 3978 not_cleared < PMAP_TS_REFERENCED_MAX); 3979 out: 3980 rw_wunlock(lock); 3981 vm_page_free_pages_toq(&free, true); 3982 return (cleared + not_cleared); 3983 } 3984 3985 static vm_offset_t 3986 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start, 3987 vm_paddr_t end, int prot __unused) 3988 { 3989 3990 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end, 3991 prot); 3992 return (PHYS_TO_DMAP(start)); 3993 } 3994 3995 void 3996 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr, 3997 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 3998 { 3999 pml3_entry_t *l3e; 4000 vm_paddr_t pa, ptepa; 4001 vm_page_t p, pdpg; 4002 vm_memattr_t ma; 4003 4004 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr, 4005 object, pindex, size); 4006 VM_OBJECT_ASSERT_WLOCKED(object); 4007 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 4008 ("pmap_object_init_pt: non-device object")); 4009 /* NB: size can be logically ored with addr here */ 4010 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) { 4011 if (!mmu_radix_ps_enabled(pmap)) 4012 return; 4013 if (!vm_object_populate(object, pindex, pindex + atop(size))) 4014 return; 4015 p = vm_page_lookup(object, pindex); 4016 KASSERT(p->valid == VM_PAGE_BITS_ALL, 4017 ("pmap_object_init_pt: invalid page %p", p)); 4018 ma = p->md.mdpg_cache_attrs; 4019 4020 /* 4021 * Abort the mapping if the first page is not physically 4022 * aligned to a 2MB page boundary. 4023 */ 4024 ptepa = VM_PAGE_TO_PHYS(p); 4025 if (ptepa & L3_PAGE_MASK) 4026 return; 4027 4028 /* 4029 * Skip the first page. Abort the mapping if the rest of 4030 * the pages are not physically contiguous or have differing 4031 * memory attributes. 4032 */ 4033 p = TAILQ_NEXT(p, listq); 4034 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; 4035 pa += PAGE_SIZE) { 4036 KASSERT(p->valid == VM_PAGE_BITS_ALL, 4037 ("pmap_object_init_pt: invalid page %p", p)); 4038 if (pa != VM_PAGE_TO_PHYS(p) || 4039 ma != p->md.mdpg_cache_attrs) 4040 return; 4041 p = TAILQ_NEXT(p, listq); 4042 } 4043 4044 PMAP_LOCK(pmap); 4045 for (pa = ptepa | pmap_cache_bits(ma); 4046 pa < ptepa + size; pa += L3_PAGE_SIZE) { 4047 pdpg = pmap_allocl3e(pmap, addr, NULL); 4048 if (pdpg == NULL) { 4049 /* 4050 * The creation of mappings below is only an 4051 * optimization. If a page directory page 4052 * cannot be allocated without blocking, 4053 * continue on to the next mapping rather than 4054 * blocking. 4055 */ 4056 addr += L3_PAGE_SIZE; 4057 continue; 4058 } 4059 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 4060 l3e = &l3e[pmap_pml3e_index(addr)]; 4061 if ((be64toh(*l3e) & PG_V) == 0) { 4062 pa |= PG_M | PG_A | PG_RW; 4063 pte_store(l3e, pa); 4064 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4065 counter_u64_add(pmap_l3e_mappings, 1); 4066 } else { 4067 /* Continue on if the PDE is already valid. */ 4068 pdpg->ref_count--; 4069 KASSERT(pdpg->ref_count > 0, 4070 ("pmap_object_init_pt: missing reference " 4071 "to page directory page, va: 0x%lx", addr)); 4072 } 4073 addr += L3_PAGE_SIZE; 4074 } 4075 ptesync(); 4076 PMAP_UNLOCK(pmap); 4077 } 4078 } 4079 4080 boolean_t 4081 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m) 4082 { 4083 struct md_page *pvh; 4084 struct rwlock *lock; 4085 pv_entry_t pv; 4086 int loops = 0; 4087 boolean_t rv; 4088 4089 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4090 ("pmap_page_exists_quick: page %p is not managed", m)); 4091 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m); 4092 rv = FALSE; 4093 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4094 rw_rlock(lock); 4095 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4096 if (PV_PMAP(pv) == pmap) { 4097 rv = TRUE; 4098 break; 4099 } 4100 loops++; 4101 if (loops >= 16) 4102 break; 4103 } 4104 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 4105 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4106 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4107 if (PV_PMAP(pv) == pmap) { 4108 rv = TRUE; 4109 break; 4110 } 4111 loops++; 4112 if (loops >= 16) 4113 break; 4114 } 4115 } 4116 rw_runlock(lock); 4117 return (rv); 4118 } 4119 4120 void 4121 mmu_radix_page_init(vm_page_t m) 4122 { 4123 4124 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4125 TAILQ_INIT(&m->md.pv_list); 4126 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT; 4127 } 4128 4129 int 4130 mmu_radix_page_wired_mappings(vm_page_t m) 4131 { 4132 struct rwlock *lock; 4133 struct md_page *pvh; 4134 pmap_t pmap; 4135 pt_entry_t *pte; 4136 pv_entry_t pv; 4137 int count, md_gen, pvh_gen; 4138 4139 if ((m->oflags & VPO_UNMANAGED) != 0) 4140 return (0); 4141 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4142 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4143 rw_rlock(lock); 4144 restart: 4145 count = 0; 4146 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4147 pmap = PV_PMAP(pv); 4148 if (!PMAP_TRYLOCK(pmap)) { 4149 md_gen = m->md.pv_gen; 4150 rw_runlock(lock); 4151 PMAP_LOCK(pmap); 4152 rw_rlock(lock); 4153 if (md_gen != m->md.pv_gen) { 4154 PMAP_UNLOCK(pmap); 4155 goto restart; 4156 } 4157 } 4158 pte = pmap_pte(pmap, pv->pv_va); 4159 if ((be64toh(*pte) & PG_W) != 0) 4160 count++; 4161 PMAP_UNLOCK(pmap); 4162 } 4163 if ((m->flags & PG_FICTITIOUS) == 0) { 4164 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4165 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4166 pmap = PV_PMAP(pv); 4167 if (!PMAP_TRYLOCK(pmap)) { 4168 md_gen = m->md.pv_gen; 4169 pvh_gen = pvh->pv_gen; 4170 rw_runlock(lock); 4171 PMAP_LOCK(pmap); 4172 rw_rlock(lock); 4173 if (md_gen != m->md.pv_gen || 4174 pvh_gen != pvh->pv_gen) { 4175 PMAP_UNLOCK(pmap); 4176 goto restart; 4177 } 4178 } 4179 pte = pmap_pml3e(pmap, pv->pv_va); 4180 if ((be64toh(*pte) & PG_W) != 0) 4181 count++; 4182 PMAP_UNLOCK(pmap); 4183 } 4184 } 4185 rw_runlock(lock); 4186 return (count); 4187 } 4188 4189 static void 4190 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa) 4191 { 4192 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT); 4193 } 4194 4195 int 4196 mmu_radix_pinit(pmap_t pmap) 4197 { 4198 vmem_addr_t pid; 4199 vm_paddr_t l1pa; 4200 4201 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4202 4203 /* 4204 * allocate the page directory page 4205 */ 4206 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK); 4207 4208 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++) 4209 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE); 4210 vm_radix_init(&pmap->pm_radix); 4211 TAILQ_INIT(&pmap->pm_pvchunk); 4212 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4213 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4214 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid); 4215 4216 pmap->pm_pid = pid; 4217 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1); 4218 mmu_radix_update_proctab(pid, l1pa); 4219 __asm __volatile("ptesync;isync" : : : "memory"); 4220 4221 return (1); 4222 } 4223 4224 /* 4225 * This routine is called if the desired page table page does not exist. 4226 * 4227 * If page table page allocation fails, this routine may sleep before 4228 * returning NULL. It sleeps only if a lock pointer was given. 4229 * 4230 * Note: If a page allocation fails at page table level two or three, 4231 * one or two pages may be held during the wait, only to be released 4232 * afterwards. This conservative approach is easily argued to avoid 4233 * race conditions. 4234 */ 4235 static vm_page_t 4236 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 4237 { 4238 vm_page_t m, pdppg, pdpg; 4239 4240 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4241 4242 /* 4243 * Allocate a page table page. 4244 */ 4245 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 4246 if (lockp != NULL) { 4247 RELEASE_PV_LIST_LOCK(lockp); 4248 PMAP_UNLOCK(pmap); 4249 vm_wait(NULL); 4250 PMAP_LOCK(pmap); 4251 } 4252 /* 4253 * Indicate the need to retry. While waiting, the page table 4254 * page may have been allocated. 4255 */ 4256 return (NULL); 4257 } 4258 m->pindex = ptepindex; 4259 4260 /* 4261 * Map the pagetable page into the process address space, if 4262 * it isn't already there. 4263 */ 4264 4265 if (ptepindex >= (NUPDE + NUPDPE)) { 4266 pml1_entry_t *l1e; 4267 vm_pindex_t pml1index; 4268 4269 /* Wire up a new PDPE page */ 4270 pml1index = ptepindex - (NUPDE + NUPDPE); 4271 l1e = &pmap->pm_pml1[pml1index]; 4272 KASSERT((be64toh(*l1e) & PG_V) == 0, 4273 ("%s: L1 entry %#lx is valid", __func__, *l1e)); 4274 pde_store(l1e, VM_PAGE_TO_PHYS(m)); 4275 } else if (ptepindex >= NUPDE) { 4276 vm_pindex_t pml1index; 4277 vm_pindex_t pdpindex; 4278 pml1_entry_t *l1e; 4279 pml2_entry_t *l2e; 4280 4281 /* Wire up a new l2e page */ 4282 pdpindex = ptepindex - NUPDE; 4283 pml1index = pdpindex >> RPTE_SHIFT; 4284 4285 l1e = &pmap->pm_pml1[pml1index]; 4286 if ((be64toh(*l1e) & PG_V) == 0) { 4287 /* Have to allocate a new pdp, recurse */ 4288 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index, 4289 lockp) == NULL) { 4290 vm_page_unwire_noq(m); 4291 vm_page_free_zero(m); 4292 return (NULL); 4293 } 4294 } else { 4295 /* Add reference to l2e page */ 4296 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME); 4297 pdppg->ref_count++; 4298 } 4299 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4300 4301 /* Now find the pdp page */ 4302 l2e = &l2e[pdpindex & RPTE_MASK]; 4303 KASSERT((be64toh(*l2e) & PG_V) == 0, 4304 ("%s: L2 entry %#lx is valid", __func__, *l2e)); 4305 pde_store(l2e, VM_PAGE_TO_PHYS(m)); 4306 } else { 4307 vm_pindex_t pml1index; 4308 vm_pindex_t pdpindex; 4309 pml1_entry_t *l1e; 4310 pml2_entry_t *l2e; 4311 pml3_entry_t *l3e; 4312 4313 /* Wire up a new PTE page */ 4314 pdpindex = ptepindex >> RPTE_SHIFT; 4315 pml1index = pdpindex >> RPTE_SHIFT; 4316 4317 /* First, find the pdp and check that its valid. */ 4318 l1e = &pmap->pm_pml1[pml1index]; 4319 if ((be64toh(*l1e) & PG_V) == 0) { 4320 /* Have to allocate a new pd, recurse */ 4321 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4322 lockp) == NULL) { 4323 vm_page_unwire_noq(m); 4324 vm_page_free_zero(m); 4325 return (NULL); 4326 } 4327 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4328 l2e = &l2e[pdpindex & RPTE_MASK]; 4329 } else { 4330 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4331 l2e = &l2e[pdpindex & RPTE_MASK]; 4332 if ((be64toh(*l2e) & PG_V) == 0) { 4333 /* Have to allocate a new pd, recurse */ 4334 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4335 lockp) == NULL) { 4336 vm_page_unwire_noq(m); 4337 vm_page_free_zero(m); 4338 return (NULL); 4339 } 4340 } else { 4341 /* Add reference to the pd page */ 4342 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME); 4343 pdpg->ref_count++; 4344 } 4345 } 4346 l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME); 4347 4348 /* Now we know where the page directory page is */ 4349 l3e = &l3e[ptepindex & RPTE_MASK]; 4350 KASSERT((be64toh(*l3e) & PG_V) == 0, 4351 ("%s: L3 entry %#lx is valid", __func__, *l3e)); 4352 pde_store(l3e, VM_PAGE_TO_PHYS(m)); 4353 } 4354 4355 pmap_resident_count_inc(pmap, 1); 4356 return (m); 4357 } 4358 static vm_page_t 4359 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4360 { 4361 vm_pindex_t pdpindex, ptepindex; 4362 pml2_entry_t *pdpe; 4363 vm_page_t pdpg; 4364 4365 retry: 4366 pdpe = pmap_pml2e(pmap, va); 4367 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) { 4368 /* Add a reference to the pd page. */ 4369 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME); 4370 pdpg->ref_count++; 4371 } else { 4372 /* Allocate a pd page. */ 4373 ptepindex = pmap_l3e_pindex(va); 4374 pdpindex = ptepindex >> RPTE_SHIFT; 4375 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); 4376 if (pdpg == NULL && lockp != NULL) 4377 goto retry; 4378 } 4379 return (pdpg); 4380 } 4381 4382 static vm_page_t 4383 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4384 { 4385 vm_pindex_t ptepindex; 4386 pml3_entry_t *pd; 4387 vm_page_t m; 4388 4389 /* 4390 * Calculate pagetable page index 4391 */ 4392 ptepindex = pmap_l3e_pindex(va); 4393 retry: 4394 /* 4395 * Get the page directory entry 4396 */ 4397 pd = pmap_pml3e(pmap, va); 4398 4399 /* 4400 * This supports switching from a 2MB page to a 4401 * normal 4K page. 4402 */ 4403 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) { 4404 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) { 4405 /* 4406 * Invalidation of the 2MB page mapping may have caused 4407 * the deallocation of the underlying PD page. 4408 */ 4409 pd = NULL; 4410 } 4411 } 4412 4413 /* 4414 * If the page table page is mapped, we just increment the 4415 * hold count, and activate it. 4416 */ 4417 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) { 4418 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME); 4419 m->ref_count++; 4420 } else { 4421 /* 4422 * Here if the pte page isn't mapped, or if it has been 4423 * deallocated. 4424 */ 4425 m = _pmap_allocpte(pmap, ptepindex, lockp); 4426 if (m == NULL && lockp != NULL) 4427 goto retry; 4428 } 4429 return (m); 4430 } 4431 4432 static void 4433 mmu_radix_pinit0(pmap_t pmap) 4434 { 4435 4436 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4437 PMAP_LOCK_INIT(pmap); 4438 pmap->pm_pml1 = kernel_pmap->pm_pml1; 4439 pmap->pm_pid = kernel_pmap->pm_pid; 4440 4441 vm_radix_init(&pmap->pm_radix); 4442 TAILQ_INIT(&pmap->pm_pvchunk); 4443 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4444 kernel_pmap->pm_flags = 4445 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4446 } 4447 /* 4448 * pmap_protect_l3e: do the things to protect a 2mpage in a process 4449 */ 4450 static boolean_t 4451 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot) 4452 { 4453 pt_entry_t newpde, oldpde; 4454 vm_offset_t eva, va; 4455 vm_page_t m; 4456 boolean_t anychanged; 4457 4458 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4459 KASSERT((sva & L3_PAGE_MASK) == 0, 4460 ("pmap_protect_l3e: sva is not 2mpage aligned")); 4461 anychanged = FALSE; 4462 retry: 4463 oldpde = newpde = be64toh(*l3e); 4464 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) == 4465 (PG_MANAGED | PG_M | PG_RW)) { 4466 eva = sva + L3_PAGE_SIZE; 4467 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4468 va < eva; va += PAGE_SIZE, m++) 4469 vm_page_dirty(m); 4470 } 4471 if ((prot & VM_PROT_WRITE) == 0) { 4472 newpde &= ~(PG_RW | PG_M); 4473 newpde |= RPTE_EAA_R; 4474 } 4475 if (prot & VM_PROT_EXECUTE) 4476 newpde |= PG_X; 4477 if (newpde != oldpde) { 4478 /* 4479 * As an optimization to future operations on this PDE, clear 4480 * PG_PROMOTED. The impending invalidation will remove any 4481 * lingering 4KB page mappings from the TLB. 4482 */ 4483 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED))) 4484 goto retry; 4485 anychanged = TRUE; 4486 } 4487 return (anychanged); 4488 } 4489 4490 void 4491 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 4492 vm_prot_t prot) 4493 { 4494 vm_offset_t va_next; 4495 pml1_entry_t *l1e; 4496 pml2_entry_t *l2e; 4497 pml3_entry_t ptpaddr, *l3e; 4498 pt_entry_t *pte; 4499 boolean_t anychanged; 4500 4501 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva, 4502 prot); 4503 4504 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 4505 if (prot == VM_PROT_NONE) { 4506 mmu_radix_remove(pmap, sva, eva); 4507 return; 4508 } 4509 4510 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 4511 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 4512 return; 4513 4514 #ifdef INVARIANTS 4515 if (VERBOSE_PROTECT || pmap_logging) 4516 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n", 4517 pmap, sva, eva, prot, pmap->pm_pid); 4518 #endif 4519 anychanged = FALSE; 4520 4521 PMAP_LOCK(pmap); 4522 for (; sva < eva; sva = va_next) { 4523 l1e = pmap_pml1e(pmap, sva); 4524 if ((be64toh(*l1e) & PG_V) == 0) { 4525 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 4526 if (va_next < sva) 4527 va_next = eva; 4528 continue; 4529 } 4530 4531 l2e = pmap_l1e_to_l2e(l1e, sva); 4532 if ((be64toh(*l2e) & PG_V) == 0) { 4533 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 4534 if (va_next < sva) 4535 va_next = eva; 4536 continue; 4537 } 4538 4539 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 4540 if (va_next < sva) 4541 va_next = eva; 4542 4543 l3e = pmap_l2e_to_l3e(l2e, sva); 4544 ptpaddr = be64toh(*l3e); 4545 4546 /* 4547 * Weed out invalid mappings. 4548 */ 4549 if (ptpaddr == 0) 4550 continue; 4551 4552 /* 4553 * Check for large page. 4554 */ 4555 if ((ptpaddr & RPTE_LEAF) != 0) { 4556 /* 4557 * Are we protecting the entire large page? If not, 4558 * demote the mapping and fall through. 4559 */ 4560 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 4561 if (pmap_protect_l3e(pmap, l3e, sva, prot)) 4562 anychanged = TRUE; 4563 continue; 4564 } else if (!pmap_demote_l3e(pmap, l3e, sva)) { 4565 /* 4566 * The large page mapping was destroyed. 4567 */ 4568 continue; 4569 } 4570 } 4571 4572 if (va_next > eva) 4573 va_next = eva; 4574 4575 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 4576 sva += PAGE_SIZE) { 4577 pt_entry_t obits, pbits; 4578 vm_page_t m; 4579 4580 retry: 4581 MPASS(pte == pmap_pte(pmap, sva)); 4582 obits = pbits = be64toh(*pte); 4583 if ((pbits & PG_V) == 0) 4584 continue; 4585 4586 if ((prot & VM_PROT_WRITE) == 0) { 4587 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == 4588 (PG_MANAGED | PG_M | PG_RW)) { 4589 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); 4590 vm_page_dirty(m); 4591 } 4592 pbits &= ~(PG_RW | PG_M); 4593 pbits |= RPTE_EAA_R; 4594 } 4595 if (prot & VM_PROT_EXECUTE) 4596 pbits |= PG_X; 4597 4598 if (pbits != obits) { 4599 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits))) 4600 goto retry; 4601 if (obits & (PG_A|PG_M)) { 4602 anychanged = TRUE; 4603 #ifdef INVARIANTS 4604 if (VERBOSE_PROTECT || pmap_logging) 4605 printf("%#lx %#lx -> %#lx\n", 4606 sva, obits, pbits); 4607 #endif 4608 } 4609 } 4610 } 4611 } 4612 if (anychanged) 4613 pmap_invalidate_all(pmap); 4614 PMAP_UNLOCK(pmap); 4615 } 4616 4617 void 4618 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count) 4619 { 4620 4621 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count); 4622 pt_entry_t oldpte, pa, *pte; 4623 vm_page_t m; 4624 uint64_t cache_bits, attr_bits; 4625 vm_offset_t va; 4626 4627 oldpte = 0; 4628 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 4629 va = sva; 4630 pte = kvtopte(va); 4631 while (va < sva + PAGE_SIZE * count) { 4632 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4633 pte = kvtopte(va); 4634 MPASS(pte == pmap_pte(kernel_pmap, va)); 4635 4636 /* 4637 * XXX there has to be a more efficient way than traversing 4638 * the page table every time - but go for correctness for 4639 * today 4640 */ 4641 4642 m = *ma++; 4643 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs); 4644 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits; 4645 if (be64toh(*pte) != pa) { 4646 oldpte |= be64toh(*pte); 4647 pte_store(pte, pa); 4648 } 4649 va += PAGE_SIZE; 4650 pte++; 4651 } 4652 if (__predict_false((oldpte & RPTE_VALID) != 0)) 4653 pmap_invalidate_range(kernel_pmap, sva, sva + count * 4654 PAGE_SIZE); 4655 else 4656 ptesync(); 4657 } 4658 4659 void 4660 mmu_radix_qremove(vm_offset_t sva, int count) 4661 { 4662 vm_offset_t va; 4663 pt_entry_t *pte; 4664 4665 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count); 4666 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva)); 4667 4668 va = sva; 4669 pte = kvtopte(va); 4670 while (va < sva + PAGE_SIZE * count) { 4671 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4672 pte = kvtopte(va); 4673 pte_clear(pte); 4674 pte++; 4675 va += PAGE_SIZE; 4676 } 4677 pmap_invalidate_range(kernel_pmap, sva, va); 4678 } 4679 4680 /*************************************************** 4681 * Page table page management routines..... 4682 ***************************************************/ 4683 /* 4684 * Schedule the specified unused page table page to be freed. Specifically, 4685 * add the page to the specified list of pages that will be released to the 4686 * physical memory manager after the TLB has been updated. 4687 */ 4688 static __inline void 4689 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 4690 boolean_t set_PG_ZERO) 4691 { 4692 4693 if (set_PG_ZERO) 4694 m->flags |= PG_ZERO; 4695 else 4696 m->flags &= ~PG_ZERO; 4697 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 4698 } 4699 4700 /* 4701 * Inserts the specified page table page into the specified pmap's collection 4702 * of idle page table pages. Each of a pmap's page table pages is responsible 4703 * for mapping a distinct range of virtual addresses. The pmap's collection is 4704 * ordered by this virtual address range. 4705 */ 4706 static __inline int 4707 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) 4708 { 4709 4710 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4711 return (vm_radix_insert(&pmap->pm_radix, mpte)); 4712 } 4713 4714 /* 4715 * Removes the page table page mapping the specified virtual address from the 4716 * specified pmap's collection of idle page table pages, and returns it. 4717 * Otherwise, returns NULL if there is no page table page corresponding to the 4718 * specified virtual address. 4719 */ 4720 static __inline vm_page_t 4721 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 4722 { 4723 4724 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4725 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va))); 4726 } 4727 4728 /* 4729 * Decrements a page table page's wire count, which is used to record the 4730 * number of valid page table entries within the page. If the wire count 4731 * drops to zero, then the page table page is unmapped. Returns TRUE if the 4732 * page table page was unmapped and FALSE otherwise. 4733 */ 4734 static inline boolean_t 4735 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4736 { 4737 4738 --m->ref_count; 4739 if (m->ref_count == 0) { 4740 _pmap_unwire_ptp(pmap, va, m, free); 4741 return (TRUE); 4742 } else 4743 return (FALSE); 4744 } 4745 4746 static void 4747 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4748 { 4749 4750 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4751 /* 4752 * unmap the page table page 4753 */ 4754 if (m->pindex >= NUPDE + NUPDPE) { 4755 /* PDP page */ 4756 pml1_entry_t *pml1; 4757 pml1 = pmap_pml1e(pmap, va); 4758 *pml1 = 0; 4759 } else if (m->pindex >= NUPDE) { 4760 /* PD page */ 4761 pml2_entry_t *l2e; 4762 l2e = pmap_pml2e(pmap, va); 4763 *l2e = 0; 4764 } else { 4765 /* PTE page */ 4766 pml3_entry_t *l3e; 4767 l3e = pmap_pml3e(pmap, va); 4768 *l3e = 0; 4769 } 4770 pmap_resident_count_dec(pmap, 1); 4771 if (m->pindex < NUPDE) { 4772 /* We just released a PT, unhold the matching PD */ 4773 vm_page_t pdpg; 4774 4775 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME); 4776 pmap_unwire_ptp(pmap, va, pdpg, free); 4777 } 4778 else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { 4779 /* We just released a PD, unhold the matching PDP */ 4780 vm_page_t pdppg; 4781 4782 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME); 4783 pmap_unwire_ptp(pmap, va, pdppg, free); 4784 } 4785 4786 /* 4787 * Put page on a list so that it is released after 4788 * *ALL* TLB shootdown is done 4789 */ 4790 pmap_add_delayed_free_list(m, free, TRUE); 4791 } 4792 4793 /* 4794 * After removing a page table entry, this routine is used to 4795 * conditionally free the page, and manage the hold/wire counts. 4796 */ 4797 static int 4798 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde, 4799 struct spglist *free) 4800 { 4801 vm_page_t mpte; 4802 4803 if (va >= VM_MAXUSER_ADDRESS) 4804 return (0); 4805 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 4806 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 4807 return (pmap_unwire_ptp(pmap, va, mpte, free)); 4808 } 4809 4810 void 4811 mmu_radix_release(pmap_t pmap) 4812 { 4813 4814 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4815 KASSERT(pmap->pm_stats.resident_count == 0, 4816 ("pmap_release: pmap resident count %ld != 0", 4817 pmap->pm_stats.resident_count)); 4818 KASSERT(vm_radix_is_empty(&pmap->pm_radix), 4819 ("pmap_release: pmap has reserved page table page(s)")); 4820 4821 pmap_invalidate_all(pmap); 4822 isa3_proctab[pmap->pm_pid].proctab0 = 0; 4823 uma_zfree(zone_radix_pgd, pmap->pm_pml1); 4824 vmem_free(asid_arena, pmap->pm_pid, 1); 4825 } 4826 4827 /* 4828 * Create the PV entry for a 2MB page mapping. Always returns true unless the 4829 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 4830 * false if the PV entry cannot be allocated without resorting to reclamation. 4831 */ 4832 static bool 4833 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags, 4834 struct rwlock **lockp) 4835 { 4836 struct md_page *pvh; 4837 pv_entry_t pv; 4838 vm_paddr_t pa; 4839 4840 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4841 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4842 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 4843 NULL : lockp)) == NULL) 4844 return (false); 4845 pv->pv_va = va; 4846 pa = pde & PG_PS_FRAME; 4847 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4848 pvh = pa_to_pvh(pa); 4849 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 4850 pvh->pv_gen++; 4851 return (true); 4852 } 4853 4854 /* 4855 * Fills a page table page with mappings to consecutive physical pages. 4856 */ 4857 static void 4858 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) 4859 { 4860 pt_entry_t *pte; 4861 4862 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { 4863 *pte = htobe64(newpte); 4864 newpte += PAGE_SIZE; 4865 } 4866 } 4867 4868 static boolean_t 4869 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va) 4870 { 4871 struct rwlock *lock; 4872 boolean_t rv; 4873 4874 lock = NULL; 4875 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock); 4876 if (lock != NULL) 4877 rw_wunlock(lock); 4878 return (rv); 4879 } 4880 4881 static boolean_t 4882 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 4883 struct rwlock **lockp) 4884 { 4885 pml3_entry_t oldpde; 4886 pt_entry_t *firstpte; 4887 vm_paddr_t mptepa; 4888 vm_page_t mpte; 4889 struct spglist free; 4890 vm_offset_t sva; 4891 4892 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4893 oldpde = be64toh(*l3e); 4894 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 4895 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx", 4896 oldpde)); 4897 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == 4898 NULL) { 4899 KASSERT((oldpde & PG_W) == 0, 4900 ("pmap_demote_l3e: page table page for a wired mapping" 4901 " is missing")); 4902 4903 /* 4904 * Invalidate the 2MB page mapping and return "failure" if the 4905 * mapping was never accessed or the allocation of the new 4906 * page table page fails. If the 2MB page mapping belongs to 4907 * the direct map region of the kernel's address space, then 4908 * the page allocation request specifies the highest possible 4909 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is 4910 * normal. Page table pages are preallocated for every other 4911 * part of the kernel address space, so the direct map region 4912 * is the only part of the kernel address space that must be 4913 * handled here. 4914 */ 4915 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj( 4916 (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ? 4917 VM_ALLOC_INTERRUPT : 0) | VM_ALLOC_WIRED)) == NULL) { 4918 SLIST_INIT(&free); 4919 sva = trunc_2mpage(va); 4920 pmap_remove_l3e(pmap, l3e, sva, &free, lockp); 4921 pmap_invalidate_l3e_page(pmap, sva, oldpde); 4922 vm_page_free_pages_toq(&free, true); 4923 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" 4924 " in pmap %p", va, pmap); 4925 return (FALSE); 4926 } 4927 mpte->pindex = pmap_l3e_pindex(va); 4928 if (va < VM_MAXUSER_ADDRESS) 4929 pmap_resident_count_inc(pmap, 1); 4930 } 4931 mptepa = VM_PAGE_TO_PHYS(mpte); 4932 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); 4933 KASSERT((oldpde & PG_A) != 0, 4934 ("pmap_demote_l3e: oldpde is missing PG_A")); 4935 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, 4936 ("pmap_demote_l3e: oldpde is missing PG_M")); 4937 4938 /* 4939 * If the page table page is new, initialize it. 4940 */ 4941 if (mpte->ref_count == 1) { 4942 mpte->ref_count = NPTEPG; 4943 pmap_fill_ptp(firstpte, oldpde); 4944 } 4945 4946 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME), 4947 ("pmap_demote_l3e: firstpte and newpte map different physical" 4948 " addresses")); 4949 4950 /* 4951 * If the mapping has changed attributes, update the page table 4952 * entries. 4953 */ 4954 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE)) 4955 pmap_fill_ptp(firstpte, oldpde); 4956 4957 /* 4958 * The spare PV entries must be reserved prior to demoting the 4959 * mapping, that is, prior to changing the PDE. Otherwise, the state 4960 * of the PDE and the PV lists will be inconsistent, which can result 4961 * in reclaim_pv_chunk() attempting to remove a PV entry from the 4962 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected 4963 * PV entry for the 2MB page mapping that is being demoted. 4964 */ 4965 if ((oldpde & PG_MANAGED) != 0) 4966 reserve_pv_entries(pmap, NPTEPG - 1, lockp); 4967 4968 /* 4969 * Demote the mapping. This pmap is locked. The old PDE has 4970 * PG_A set. If the old PDE has PG_RW set, it also has PG_M 4971 * set. Thus, there is no danger of a race with another 4972 * processor changing the setting of PG_A and/or PG_M between 4973 * the read above and the store below. 4974 */ 4975 pde_store(l3e, mptepa); 4976 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde); 4977 /* 4978 * Demote the PV entry. 4979 */ 4980 if ((oldpde & PG_MANAGED) != 0) 4981 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp); 4982 4983 counter_u64_add(pmap_l3e_demotions, 1); 4984 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx" 4985 " in pmap %p", va, pmap); 4986 return (TRUE); 4987 } 4988 4989 /* 4990 * pmap_remove_kernel_pde: Remove a kernel superpage mapping. 4991 */ 4992 static void 4993 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va) 4994 { 4995 vm_paddr_t mptepa; 4996 vm_page_t mpte; 4997 4998 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 4999 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5000 mpte = pmap_remove_pt_page(pmap, va); 5001 if (mpte == NULL) 5002 panic("pmap_remove_kernel_pde: Missing pt page."); 5003 5004 mptepa = VM_PAGE_TO_PHYS(mpte); 5005 5006 /* 5007 * Initialize the page table page. 5008 */ 5009 pagezero(PHYS_TO_DMAP(mptepa)); 5010 5011 /* 5012 * Demote the mapping. 5013 */ 5014 pde_store(l3e, mptepa); 5015 ptesync(); 5016 } 5017 5018 /* 5019 * pmap_remove_l3e: do the things to unmap a superpage in a process 5020 */ 5021 static int 5022 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 5023 struct spglist *free, struct rwlock **lockp) 5024 { 5025 struct md_page *pvh; 5026 pml3_entry_t oldpde; 5027 vm_offset_t eva, va; 5028 vm_page_t m, mpte; 5029 5030 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5031 KASSERT((sva & L3_PAGE_MASK) == 0, 5032 ("pmap_remove_l3e: sva is not 2mpage aligned")); 5033 oldpde = be64toh(pte_load_clear(pdq)); 5034 if (oldpde & PG_W) 5035 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE); 5036 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5037 if (oldpde & PG_MANAGED) { 5038 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); 5039 pvh = pa_to_pvh(oldpde & PG_PS_FRAME); 5040 pmap_pvh_free(pvh, pmap, sva); 5041 eva = sva + L3_PAGE_SIZE; 5042 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 5043 va < eva; va += PAGE_SIZE, m++) { 5044 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5045 vm_page_dirty(m); 5046 if (oldpde & PG_A) 5047 vm_page_aflag_set(m, PGA_REFERENCED); 5048 if (TAILQ_EMPTY(&m->md.pv_list) && 5049 TAILQ_EMPTY(&pvh->pv_list)) 5050 vm_page_aflag_clear(m, PGA_WRITEABLE); 5051 } 5052 } 5053 if (pmap == kernel_pmap) { 5054 pmap_remove_kernel_l3e(pmap, pdq, sva); 5055 } else { 5056 mpte = pmap_remove_pt_page(pmap, sva); 5057 if (mpte != NULL) { 5058 pmap_resident_count_dec(pmap, 1); 5059 KASSERT(mpte->ref_count == NPTEPG, 5060 ("pmap_remove_l3e: pte page wire count error")); 5061 mpte->ref_count = 0; 5062 pmap_add_delayed_free_list(mpte, free, FALSE); 5063 } 5064 } 5065 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free)); 5066 } 5067 5068 /* 5069 * pmap_remove_pte: do the things to unmap a page in a process 5070 */ 5071 static int 5072 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 5073 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp) 5074 { 5075 struct md_page *pvh; 5076 pt_entry_t oldpte; 5077 vm_page_t m; 5078 5079 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5080 oldpte = be64toh(pte_load_clear(ptq)); 5081 if (oldpte & RPTE_WIRED) 5082 pmap->pm_stats.wired_count -= 1; 5083 pmap_resident_count_dec(pmap, 1); 5084 if (oldpte & RPTE_MANAGED) { 5085 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); 5086 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5087 vm_page_dirty(m); 5088 if (oldpte & PG_A) 5089 vm_page_aflag_set(m, PGA_REFERENCED); 5090 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5091 pmap_pvh_free(&m->md, pmap, va); 5092 if (TAILQ_EMPTY(&m->md.pv_list) && 5093 (m->flags & PG_FICTITIOUS) == 0) { 5094 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5095 if (TAILQ_EMPTY(&pvh->pv_list)) 5096 vm_page_aflag_clear(m, PGA_WRITEABLE); 5097 } 5098 } 5099 return (pmap_unuse_pt(pmap, va, ptepde, free)); 5100 } 5101 5102 /* 5103 * Remove a single page from a process address space 5104 */ 5105 static bool 5106 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e, 5107 struct spglist *free) 5108 { 5109 struct rwlock *lock; 5110 pt_entry_t *pte; 5111 bool invalidate_all; 5112 5113 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5114 if ((be64toh(*l3e) & RPTE_VALID) == 0) { 5115 return (false); 5116 } 5117 pte = pmap_l3e_to_pte(l3e, va); 5118 if ((be64toh(*pte) & RPTE_VALID) == 0) { 5119 return (false); 5120 } 5121 lock = NULL; 5122 5123 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock); 5124 if (lock != NULL) 5125 rw_wunlock(lock); 5126 if (!invalidate_all) 5127 pmap_invalidate_page(pmap, va); 5128 return (invalidate_all); 5129 } 5130 5131 /* 5132 * Removes the specified range of addresses from the page table page. 5133 */ 5134 static bool 5135 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 5136 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp) 5137 { 5138 pt_entry_t *pte; 5139 vm_offset_t va; 5140 bool anyvalid; 5141 5142 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5143 anyvalid = false; 5144 va = eva; 5145 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++, 5146 sva += PAGE_SIZE) { 5147 MPASS(pte == pmap_pte(pmap, sva)); 5148 if (*pte == 0) { 5149 if (va != eva) { 5150 anyvalid = true; 5151 va = eva; 5152 } 5153 continue; 5154 } 5155 if (va == eva) 5156 va = sva; 5157 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) { 5158 anyvalid = true; 5159 sva += PAGE_SIZE; 5160 break; 5161 } 5162 } 5163 if (anyvalid) 5164 pmap_invalidate_all(pmap); 5165 else if (va != eva) 5166 pmap_invalidate_range(pmap, va, sva); 5167 return (anyvalid); 5168 } 5169 5170 void 5171 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5172 { 5173 struct rwlock *lock; 5174 vm_offset_t va_next; 5175 pml1_entry_t *l1e; 5176 pml2_entry_t *l2e; 5177 pml3_entry_t ptpaddr, *l3e; 5178 struct spglist free; 5179 bool anyvalid; 5180 5181 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5182 5183 /* 5184 * Perform an unsynchronized read. This is, however, safe. 5185 */ 5186 if (pmap->pm_stats.resident_count == 0) 5187 return; 5188 5189 anyvalid = false; 5190 SLIST_INIT(&free); 5191 5192 /* XXX something fishy here */ 5193 sva = (sva + PAGE_MASK) & ~PAGE_MASK; 5194 eva = (eva + PAGE_MASK) & ~PAGE_MASK; 5195 5196 PMAP_LOCK(pmap); 5197 5198 /* 5199 * special handling of removing one page. a very 5200 * common operation and easy to short circuit some 5201 * code. 5202 */ 5203 if (sva + PAGE_SIZE == eva) { 5204 l3e = pmap_pml3e(pmap, sva); 5205 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) { 5206 anyvalid = pmap_remove_page(pmap, sva, l3e, &free); 5207 goto out; 5208 } 5209 } 5210 5211 lock = NULL; 5212 for (; sva < eva; sva = va_next) { 5213 if (pmap->pm_stats.resident_count == 0) 5214 break; 5215 l1e = pmap_pml1e(pmap, sva); 5216 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) { 5217 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5218 if (va_next < sva) 5219 va_next = eva; 5220 continue; 5221 } 5222 5223 l2e = pmap_l1e_to_l2e(l1e, sva); 5224 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) { 5225 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5226 if (va_next < sva) 5227 va_next = eva; 5228 continue; 5229 } 5230 5231 /* 5232 * Calculate index for next page table. 5233 */ 5234 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5235 if (va_next < sva) 5236 va_next = eva; 5237 5238 l3e = pmap_l2e_to_l3e(l2e, sva); 5239 ptpaddr = be64toh(*l3e); 5240 5241 /* 5242 * Weed out invalid mappings. 5243 */ 5244 if (ptpaddr == 0) 5245 continue; 5246 5247 /* 5248 * Check for large page. 5249 */ 5250 if ((ptpaddr & RPTE_LEAF) != 0) { 5251 /* 5252 * Are we removing the entire large page? If not, 5253 * demote the mapping and fall through. 5254 */ 5255 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5256 pmap_remove_l3e(pmap, l3e, sva, &free, &lock); 5257 anyvalid = true; 5258 continue; 5259 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva, 5260 &lock)) { 5261 /* The large page mapping was destroyed. */ 5262 continue; 5263 } else 5264 ptpaddr = be64toh(*l3e); 5265 } 5266 5267 /* 5268 * Limit our scan to either the end of the va represented 5269 * by the current page table page, or to the end of the 5270 * range being removed. 5271 */ 5272 if (va_next > eva) 5273 va_next = eva; 5274 5275 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock)) 5276 anyvalid = true; 5277 } 5278 if (lock != NULL) 5279 rw_wunlock(lock); 5280 out: 5281 if (anyvalid) 5282 pmap_invalidate_all(pmap); 5283 PMAP_UNLOCK(pmap); 5284 vm_page_free_pages_toq(&free, true); 5285 } 5286 5287 void 5288 mmu_radix_remove_all(vm_page_t m) 5289 { 5290 struct md_page *pvh; 5291 pv_entry_t pv; 5292 pmap_t pmap; 5293 struct rwlock *lock; 5294 pt_entry_t *pte, tpte; 5295 pml3_entry_t *l3e; 5296 vm_offset_t va; 5297 struct spglist free; 5298 int pvh_gen, md_gen; 5299 5300 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5301 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5302 ("pmap_remove_all: page %p is not managed", m)); 5303 SLIST_INIT(&free); 5304 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5305 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5306 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5307 retry: 5308 rw_wlock(lock); 5309 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 5310 pmap = PV_PMAP(pv); 5311 if (!PMAP_TRYLOCK(pmap)) { 5312 pvh_gen = pvh->pv_gen; 5313 rw_wunlock(lock); 5314 PMAP_LOCK(pmap); 5315 rw_wlock(lock); 5316 if (pvh_gen != pvh->pv_gen) { 5317 rw_wunlock(lock); 5318 PMAP_UNLOCK(pmap); 5319 goto retry; 5320 } 5321 } 5322 va = pv->pv_va; 5323 l3e = pmap_pml3e(pmap, va); 5324 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock); 5325 PMAP_UNLOCK(pmap); 5326 } 5327 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 5328 pmap = PV_PMAP(pv); 5329 if (!PMAP_TRYLOCK(pmap)) { 5330 pvh_gen = pvh->pv_gen; 5331 md_gen = m->md.pv_gen; 5332 rw_wunlock(lock); 5333 PMAP_LOCK(pmap); 5334 rw_wlock(lock); 5335 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 5336 rw_wunlock(lock); 5337 PMAP_UNLOCK(pmap); 5338 goto retry; 5339 } 5340 } 5341 pmap_resident_count_dec(pmap, 1); 5342 l3e = pmap_pml3e(pmap, pv->pv_va); 5343 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found" 5344 " a 2mpage in page %p's pv list", m)); 5345 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5346 tpte = be64toh(pte_load_clear(pte)); 5347 if (tpte & PG_W) 5348 pmap->pm_stats.wired_count--; 5349 if (tpte & PG_A) 5350 vm_page_aflag_set(m, PGA_REFERENCED); 5351 5352 /* 5353 * Update the vm_page_t clean and reference bits. 5354 */ 5355 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5356 vm_page_dirty(m); 5357 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free); 5358 pmap_invalidate_page(pmap, pv->pv_va); 5359 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5360 m->md.pv_gen++; 5361 free_pv_entry(pmap, pv); 5362 PMAP_UNLOCK(pmap); 5363 } 5364 vm_page_aflag_clear(m, PGA_WRITEABLE); 5365 rw_wunlock(lock); 5366 vm_page_free_pages_toq(&free, true); 5367 } 5368 5369 /* 5370 * Destroy all managed, non-wired mappings in the given user-space 5371 * pmap. This pmap cannot be active on any processor besides the 5372 * caller. 5373 * 5374 * This function cannot be applied to the kernel pmap. Moreover, it 5375 * is not intended for general use. It is only to be used during 5376 * process termination. Consequently, it can be implemented in ways 5377 * that make it faster than pmap_remove(). First, it can more quickly 5378 * destroy mappings by iterating over the pmap's collection of PV 5379 * entries, rather than searching the page table. Second, it doesn't 5380 * have to test and clear the page table entries atomically, because 5381 * no processor is currently accessing the user address space. In 5382 * particular, a page table entry's dirty bit won't change state once 5383 * this function starts. 5384 * 5385 * Although this function destroys all of the pmap's managed, 5386 * non-wired mappings, it can delay and batch the invalidation of TLB 5387 * entries without calling pmap_delayed_invl_started() and 5388 * pmap_delayed_invl_finished(). Because the pmap is not active on 5389 * any other processor, none of these TLB entries will ever be used 5390 * before their eventual invalidation. Consequently, there is no need 5391 * for either pmap_remove_all() or pmap_remove_write() to wait for 5392 * that eventual TLB invalidation. 5393 */ 5394 5395 void 5396 mmu_radix_remove_pages(pmap_t pmap) 5397 { 5398 5399 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 5400 pml3_entry_t ptel3e; 5401 pt_entry_t *pte, tpte; 5402 struct spglist free; 5403 vm_page_t m, mpte, mt; 5404 pv_entry_t pv; 5405 struct md_page *pvh; 5406 struct pv_chunk *pc, *npc; 5407 struct rwlock *lock; 5408 int64_t bit; 5409 uint64_t inuse, bitmask; 5410 int allfree, field, idx; 5411 #ifdef PV_STATS 5412 int freed; 5413 #endif 5414 boolean_t superpage; 5415 vm_paddr_t pa; 5416 5417 /* 5418 * Assert that the given pmap is only active on the current 5419 * CPU. Unfortunately, we cannot block another CPU from 5420 * activating the pmap while this function is executing. 5421 */ 5422 KASSERT(pmap->pm_pid == mfspr(SPR_PID), 5423 ("non-current asid %lu - expected %lu", pmap->pm_pid, 5424 mfspr(SPR_PID))); 5425 5426 lock = NULL; 5427 5428 SLIST_INIT(&free); 5429 PMAP_LOCK(pmap); 5430 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 5431 allfree = 1; 5432 #ifdef PV_STATS 5433 freed = 0; 5434 #endif 5435 for (field = 0; field < _NPCM; field++) { 5436 inuse = ~pc->pc_map[field] & pc_freemask[field]; 5437 while (inuse != 0) { 5438 bit = cnttzd(inuse); 5439 bitmask = 1UL << bit; 5440 idx = field * 64 + bit; 5441 pv = &pc->pc_pventry[idx]; 5442 inuse &= ~bitmask; 5443 5444 pte = pmap_pml2e(pmap, pv->pv_va); 5445 ptel3e = be64toh(*pte); 5446 pte = pmap_l2e_to_l3e(pte, pv->pv_va); 5447 tpte = be64toh(*pte); 5448 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) { 5449 superpage = FALSE; 5450 ptel3e = tpte; 5451 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & 5452 PG_FRAME); 5453 pte = &pte[pmap_pte_index(pv->pv_va)]; 5454 tpte = be64toh(*pte); 5455 } else { 5456 /* 5457 * Keep track whether 'tpte' is a 5458 * superpage explicitly instead of 5459 * relying on RPTE_LEAF being set. 5460 * 5461 * This is because RPTE_LEAF is numerically 5462 * identical to PG_PTE_PAT and thus a 5463 * regular page could be mistaken for 5464 * a superpage. 5465 */ 5466 superpage = TRUE; 5467 } 5468 5469 if ((tpte & PG_V) == 0) { 5470 panic("bad pte va %lx pte %lx", 5471 pv->pv_va, tpte); 5472 } 5473 5474 /* 5475 * We cannot remove wired pages from a process' mapping at this time 5476 */ 5477 if (tpte & PG_W) { 5478 allfree = 0; 5479 continue; 5480 } 5481 5482 if (superpage) 5483 pa = tpte & PG_PS_FRAME; 5484 else 5485 pa = tpte & PG_FRAME; 5486 5487 m = PHYS_TO_VM_PAGE(pa); 5488 KASSERT(m->phys_addr == pa, 5489 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 5490 m, (uintmax_t)m->phys_addr, 5491 (uintmax_t)tpte)); 5492 5493 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 5494 m < &vm_page_array[vm_page_array_size], 5495 ("pmap_remove_pages: bad tpte %#jx", 5496 (uintmax_t)tpte)); 5497 5498 pte_clear(pte); 5499 5500 /* 5501 * Update the vm_page_t clean/reference bits. 5502 */ 5503 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 5504 if (superpage) { 5505 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5506 vm_page_dirty(mt); 5507 } else 5508 vm_page_dirty(m); 5509 } 5510 5511 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 5512 5513 /* Mark free */ 5514 pc->pc_map[field] |= bitmask; 5515 if (superpage) { 5516 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5517 pvh = pa_to_pvh(tpte & PG_PS_FRAME); 5518 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 5519 pvh->pv_gen++; 5520 if (TAILQ_EMPTY(&pvh->pv_list)) { 5521 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5522 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 5523 TAILQ_EMPTY(&mt->md.pv_list)) 5524 vm_page_aflag_clear(mt, PGA_WRITEABLE); 5525 } 5526 mpte = pmap_remove_pt_page(pmap, pv->pv_va); 5527 if (mpte != NULL) { 5528 pmap_resident_count_dec(pmap, 1); 5529 KASSERT(mpte->ref_count == NPTEPG, 5530 ("pmap_remove_pages: pte page wire count error")); 5531 mpte->ref_count = 0; 5532 pmap_add_delayed_free_list(mpte, &free, FALSE); 5533 } 5534 } else { 5535 pmap_resident_count_dec(pmap, 1); 5536 #ifdef VERBOSE_PV 5537 printf("freeing pv (%p, %p)\n", 5538 pmap, pv); 5539 #endif 5540 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5541 m->md.pv_gen++; 5542 if ((m->a.flags & PGA_WRITEABLE) != 0 && 5543 TAILQ_EMPTY(&m->md.pv_list) && 5544 (m->flags & PG_FICTITIOUS) == 0) { 5545 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5546 if (TAILQ_EMPTY(&pvh->pv_list)) 5547 vm_page_aflag_clear(m, PGA_WRITEABLE); 5548 } 5549 } 5550 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free); 5551 #ifdef PV_STATS 5552 freed++; 5553 #endif 5554 } 5555 } 5556 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 5557 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 5558 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 5559 if (allfree) { 5560 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5561 free_pv_chunk(pc); 5562 } 5563 } 5564 if (lock != NULL) 5565 rw_wunlock(lock); 5566 pmap_invalidate_all(pmap); 5567 PMAP_UNLOCK(pmap); 5568 vm_page_free_pages_toq(&free, true); 5569 } 5570 5571 void 5572 mmu_radix_remove_write(vm_page_t m) 5573 { 5574 struct md_page *pvh; 5575 pmap_t pmap; 5576 struct rwlock *lock; 5577 pv_entry_t next_pv, pv; 5578 pml3_entry_t *l3e; 5579 pt_entry_t oldpte, *pte; 5580 int pvh_gen, md_gen; 5581 5582 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5583 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5584 ("pmap_remove_write: page %p is not managed", m)); 5585 vm_page_assert_busied(m); 5586 5587 if (!pmap_page_is_write_mapped(m)) 5588 return; 5589 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5590 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5591 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5592 retry_pv_loop: 5593 rw_wlock(lock); 5594 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 5595 pmap = PV_PMAP(pv); 5596 if (!PMAP_TRYLOCK(pmap)) { 5597 pvh_gen = pvh->pv_gen; 5598 rw_wunlock(lock); 5599 PMAP_LOCK(pmap); 5600 rw_wlock(lock); 5601 if (pvh_gen != pvh->pv_gen) { 5602 PMAP_UNLOCK(pmap); 5603 rw_wunlock(lock); 5604 goto retry_pv_loop; 5605 } 5606 } 5607 l3e = pmap_pml3e(pmap, pv->pv_va); 5608 if ((be64toh(*l3e) & PG_RW) != 0) 5609 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock); 5610 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 5611 ("inconsistent pv lock %p %p for page %p", 5612 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 5613 PMAP_UNLOCK(pmap); 5614 } 5615 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 5616 pmap = PV_PMAP(pv); 5617 if (!PMAP_TRYLOCK(pmap)) { 5618 pvh_gen = pvh->pv_gen; 5619 md_gen = m->md.pv_gen; 5620 rw_wunlock(lock); 5621 PMAP_LOCK(pmap); 5622 rw_wlock(lock); 5623 if (pvh_gen != pvh->pv_gen || 5624 md_gen != m->md.pv_gen) { 5625 PMAP_UNLOCK(pmap); 5626 rw_wunlock(lock); 5627 goto retry_pv_loop; 5628 } 5629 } 5630 l3e = pmap_pml3e(pmap, pv->pv_va); 5631 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 5632 ("pmap_remove_write: found a 2mpage in page %p's pv list", 5633 m)); 5634 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5635 retry: 5636 oldpte = be64toh(*pte); 5637 if (oldpte & PG_RW) { 5638 if (!atomic_cmpset_long(pte, htobe64(oldpte), 5639 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M)))) 5640 goto retry; 5641 if ((oldpte & PG_M) != 0) 5642 vm_page_dirty(m); 5643 pmap_invalidate_page(pmap, pv->pv_va); 5644 } 5645 PMAP_UNLOCK(pmap); 5646 } 5647 rw_wunlock(lock); 5648 vm_page_aflag_clear(m, PGA_WRITEABLE); 5649 } 5650 5651 /* 5652 * Clear the wired attribute from the mappings for the specified range of 5653 * addresses in the given pmap. Every valid mapping within that range 5654 * must have the wired attribute set. In contrast, invalid mappings 5655 * cannot have the wired attribute set, so they are ignored. 5656 * 5657 * The wired attribute of the page table entry is not a hardware 5658 * feature, so there is no need to invalidate any TLB entries. 5659 * Since pmap_demote_l3e() for the wired entry must never fail, 5660 * pmap_delayed_invl_started()/finished() calls around the 5661 * function are not needed. 5662 */ 5663 void 5664 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5665 { 5666 vm_offset_t va_next; 5667 pml1_entry_t *l1e; 5668 pml2_entry_t *l2e; 5669 pml3_entry_t *l3e; 5670 pt_entry_t *pte; 5671 5672 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5673 PMAP_LOCK(pmap); 5674 for (; sva < eva; sva = va_next) { 5675 l1e = pmap_pml1e(pmap, sva); 5676 if ((be64toh(*l1e) & PG_V) == 0) { 5677 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5678 if (va_next < sva) 5679 va_next = eva; 5680 continue; 5681 } 5682 l2e = pmap_l1e_to_l2e(l1e, sva); 5683 if ((be64toh(*l2e) & PG_V) == 0) { 5684 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5685 if (va_next < sva) 5686 va_next = eva; 5687 continue; 5688 } 5689 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5690 if (va_next < sva) 5691 va_next = eva; 5692 l3e = pmap_l2e_to_l3e(l2e, sva); 5693 if ((be64toh(*l3e) & PG_V) == 0) 5694 continue; 5695 if ((be64toh(*l3e) & RPTE_LEAF) != 0) { 5696 if ((be64toh(*l3e) & PG_W) == 0) 5697 panic("pmap_unwire: pde %#jx is missing PG_W", 5698 (uintmax_t)(be64toh(*l3e))); 5699 5700 /* 5701 * Are we unwiring the entire large page? If not, 5702 * demote the mapping and fall through. 5703 */ 5704 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5705 atomic_clear_long(l3e, htobe64(PG_W)); 5706 pmap->pm_stats.wired_count -= L3_PAGE_SIZE / 5707 PAGE_SIZE; 5708 continue; 5709 } else if (!pmap_demote_l3e(pmap, l3e, sva)) 5710 panic("pmap_unwire: demotion failed"); 5711 } 5712 if (va_next > eva) 5713 va_next = eva; 5714 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 5715 sva += PAGE_SIZE) { 5716 MPASS(pte == pmap_pte(pmap, sva)); 5717 if ((be64toh(*pte) & PG_V) == 0) 5718 continue; 5719 if ((be64toh(*pte) & PG_W) == 0) 5720 panic("pmap_unwire: pte %#jx is missing PG_W", 5721 (uintmax_t)(be64toh(*pte))); 5722 5723 /* 5724 * PG_W must be cleared atomically. Although the pmap 5725 * lock synchronizes access to PG_W, another processor 5726 * could be setting PG_M and/or PG_A concurrently. 5727 */ 5728 atomic_clear_long(pte, htobe64(PG_W)); 5729 pmap->pm_stats.wired_count--; 5730 } 5731 } 5732 PMAP_UNLOCK(pmap); 5733 } 5734 5735 void 5736 mmu_radix_zero_page(vm_page_t m) 5737 { 5738 vm_offset_t addr; 5739 5740 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5741 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5742 pagezero(addr); 5743 } 5744 5745 void 5746 mmu_radix_zero_page_area(vm_page_t m, int off, int size) 5747 { 5748 caddr_t addr; 5749 5750 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size); 5751 MPASS(off + size <= PAGE_SIZE); 5752 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5753 memset(addr + off, 0, size); 5754 } 5755 5756 static int 5757 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 5758 { 5759 pml3_entry_t *l3ep; 5760 pt_entry_t pte; 5761 vm_paddr_t pa; 5762 int val; 5763 5764 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 5765 PMAP_LOCK(pmap); 5766 5767 l3ep = pmap_pml3e(pmap, addr); 5768 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) { 5769 if (be64toh(*l3ep) & RPTE_LEAF) { 5770 pte = be64toh(*l3ep); 5771 /* Compute the physical address of the 4KB page. */ 5772 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) & 5773 PG_FRAME; 5774 val = MINCORE_PSIND(1); 5775 } else { 5776 /* Native endian PTE, do not pass to functions */ 5777 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr)); 5778 pa = pte & PG_FRAME; 5779 val = 0; 5780 } 5781 } else { 5782 pte = 0; 5783 pa = 0; 5784 val = 0; 5785 } 5786 if ((pte & PG_V) != 0) { 5787 val |= MINCORE_INCORE; 5788 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5789 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 5790 if ((pte & PG_A) != 0) 5791 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 5792 } 5793 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 5794 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 5795 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { 5796 *locked_pa = pa; 5797 } 5798 PMAP_UNLOCK(pmap); 5799 return (val); 5800 } 5801 5802 void 5803 mmu_radix_activate(struct thread *td) 5804 { 5805 pmap_t pmap; 5806 uint32_t curpid; 5807 5808 CTR2(KTR_PMAP, "%s(%p)", __func__, td); 5809 critical_enter(); 5810 pmap = vmspace_pmap(td->td_proc->p_vmspace); 5811 curpid = mfspr(SPR_PID); 5812 if (pmap->pm_pid > isa3_base_pid && 5813 curpid != pmap->pm_pid) { 5814 mmu_radix_pid_set(pmap); 5815 } 5816 critical_exit(); 5817 } 5818 5819 /* 5820 * Increase the starting virtual address of the given mapping if a 5821 * different alignment might result in more superpage mappings. 5822 */ 5823 void 5824 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset, 5825 vm_offset_t *addr, vm_size_t size) 5826 { 5827 5828 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr, 5829 size); 5830 vm_offset_t superpage_offset; 5831 5832 if (size < L3_PAGE_SIZE) 5833 return; 5834 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 5835 offset += ptoa(object->pg_color); 5836 superpage_offset = offset & L3_PAGE_MASK; 5837 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE || 5838 (*addr & L3_PAGE_MASK) == superpage_offset) 5839 return; 5840 if ((*addr & L3_PAGE_MASK) < superpage_offset) 5841 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset; 5842 else 5843 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset; 5844 } 5845 5846 static void * 5847 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr) 5848 { 5849 vm_offset_t va, tmpva, ppa, offset; 5850 5851 ppa = trunc_page(pa); 5852 offset = pa & PAGE_MASK; 5853 size = roundup2(offset + size, PAGE_SIZE); 5854 if (pa < powerpc_ptob(Maxmem)) 5855 panic("bad pa: %#lx less than Maxmem %#lx\n", 5856 pa, powerpc_ptob(Maxmem)); 5857 va = kva_alloc(size); 5858 if (bootverbose) 5859 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr); 5860 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr)); 5861 5862 if (!va) 5863 panic("%s: Couldn't alloc kernel virtual memory", __func__); 5864 5865 for (tmpva = va; size > 0;) { 5866 mmu_radix_kenter_attr(tmpva, ppa, attr); 5867 size -= PAGE_SIZE; 5868 tmpva += PAGE_SIZE; 5869 ppa += PAGE_SIZE; 5870 } 5871 ptesync(); 5872 5873 return ((void *)(va + offset)); 5874 } 5875 5876 static void * 5877 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size) 5878 { 5879 5880 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 5881 5882 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT)); 5883 } 5884 5885 void 5886 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma) 5887 { 5888 5889 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma); 5890 m->md.mdpg_cache_attrs = ma; 5891 5892 /* 5893 * If "m" is a normal page, update its direct mapping. This update 5894 * can be relied upon to perform any cache operations that are 5895 * required for data coherence. 5896 */ 5897 if ((m->flags & PG_FICTITIOUS) == 0 && 5898 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 5899 PAGE_SIZE, m->md.mdpg_cache_attrs)) 5900 panic("memory attribute change on the direct map failed"); 5901 } 5902 5903 static void 5904 mmu_radix_unmapdev(void *p, vm_size_t size) 5905 { 5906 vm_offset_t offset, va; 5907 5908 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, p, size); 5909 5910 /* If we gave a direct map region in pmap_mapdev, do nothing */ 5911 va = (vm_offset_t)p; 5912 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) 5913 return; 5914 5915 offset = va & PAGE_MASK; 5916 size = round_page(offset + size); 5917 va = trunc_page(va); 5918 5919 if (pmap_initialized) { 5920 mmu_radix_qremove(va, atop(size)); 5921 kva_free(va, size); 5922 } 5923 } 5924 5925 void 5926 mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 5927 { 5928 vm_paddr_t pa = 0; 5929 int sync_sz; 5930 5931 if (__predict_false(pm == NULL)) 5932 pm = &curthread->td_proc->p_vmspace->vm_pmap; 5933 5934 while (sz > 0) { 5935 pa = pmap_extract(pm, va); 5936 sync_sz = PAGE_SIZE - (va & PAGE_MASK); 5937 sync_sz = min(sync_sz, sz); 5938 if (pa != 0) { 5939 pa += (va & PAGE_MASK); 5940 __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz); 5941 } 5942 va += sync_sz; 5943 sz -= sync_sz; 5944 } 5945 } 5946 5947 static __inline void 5948 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask) 5949 { 5950 uint64_t opte, npte; 5951 5952 /* 5953 * The cache mode bits are all in the low 32-bits of the 5954 * PTE, so we can just spin on updating the low 32-bits. 5955 */ 5956 do { 5957 opte = be64toh(*pte); 5958 npte = opte & ~mask; 5959 npte |= cache_bits; 5960 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte))); 5961 } 5962 5963 /* 5964 * Tries to demote a 1GB page mapping. 5965 */ 5966 static boolean_t 5967 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) 5968 { 5969 pml2_entry_t oldpdpe; 5970 pml3_entry_t *firstpde, newpde, *pde; 5971 vm_paddr_t pdpgpa; 5972 vm_page_t pdpg; 5973 5974 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5975 oldpdpe = be64toh(*l2e); 5976 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 5977 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); 5978 pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 5979 if (pdpg == NULL) { 5980 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" 5981 " in pmap %p", va, pmap); 5982 return (FALSE); 5983 } 5984 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT; 5985 pdpgpa = VM_PAGE_TO_PHYS(pdpg); 5986 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); 5987 KASSERT((oldpdpe & PG_A) != 0, 5988 ("pmap_demote_pdpe: oldpdpe is missing PG_A")); 5989 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, 5990 ("pmap_demote_pdpe: oldpdpe is missing PG_M")); 5991 newpde = oldpdpe; 5992 5993 /* 5994 * Initialize the page directory page. 5995 */ 5996 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { 5997 *pde = htobe64(newpde); 5998 newpde += L3_PAGE_SIZE; 5999 } 6000 6001 /* 6002 * Demote the mapping. 6003 */ 6004 pde_store(l2e, pdpgpa); 6005 6006 /* 6007 * Flush PWC --- XXX revisit 6008 */ 6009 pmap_invalidate_all(pmap); 6010 6011 counter_u64_add(pmap_l2e_demotions, 1); 6012 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" 6013 " in pmap %p", va, pmap); 6014 return (TRUE); 6015 } 6016 6017 vm_paddr_t 6018 mmu_radix_kextract(vm_offset_t va) 6019 { 6020 pml3_entry_t l3e; 6021 vm_paddr_t pa; 6022 6023 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6024 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { 6025 pa = DMAP_TO_PHYS(va); 6026 } else { 6027 /* Big-endian PTE on stack */ 6028 l3e = *pmap_pml3e(kernel_pmap, va); 6029 if (be64toh(l3e) & RPTE_LEAF) { 6030 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 6031 pa |= (va & L3_PAGE_MASK); 6032 } else { 6033 /* 6034 * Beware of a concurrent promotion that changes the 6035 * PDE at this point! For example, vtopte() must not 6036 * be used to access the PTE because it would use the 6037 * new PDE. It is, however, safe to use the old PDE 6038 * because the page table page is preserved by the 6039 * promotion. 6040 */ 6041 pa = be64toh(*pmap_l3e_to_pte(&l3e, va)); 6042 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 6043 pa |= (va & PAGE_MASK); 6044 } 6045 } 6046 return (pa); 6047 } 6048 6049 static pt_entry_t 6050 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 6051 { 6052 6053 if (ma != VM_MEMATTR_DEFAULT) { 6054 return pmap_cache_bits(ma); 6055 } 6056 6057 /* 6058 * Assume the page is cache inhibited and access is guarded unless 6059 * it's in our available memory array. 6060 */ 6061 for (int i = 0; i < pregions_sz; i++) { 6062 if ((pa >= pregions[i].mr_start) && 6063 (pa < (pregions[i].mr_start + pregions[i].mr_size))) 6064 return (RPTE_ATTR_MEM); 6065 } 6066 return (RPTE_ATTR_GUARDEDIO); 6067 } 6068 6069 static void 6070 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 6071 { 6072 pt_entry_t *pte, pteval; 6073 uint64_t cache_bits; 6074 6075 pte = kvtopte(va); 6076 MPASS(pte != NULL); 6077 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 6078 cache_bits = mmu_radix_calc_wimg(pa, ma); 6079 pte_store(pte, pteval | cache_bits); 6080 } 6081 6082 void 6083 mmu_radix_kremove(vm_offset_t va) 6084 { 6085 pt_entry_t *pte; 6086 6087 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6088 6089 pte = kvtopte(va); 6090 pte_clear(pte); 6091 } 6092 6093 int 6094 mmu_radix_decode_kernel_ptr(vm_offset_t addr, 6095 int *is_user, vm_offset_t *decoded) 6096 { 6097 6098 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr); 6099 *decoded = addr; 6100 *is_user = (addr < VM_MAXUSER_ADDRESS); 6101 return (0); 6102 } 6103 6104 static int 6105 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size) 6106 { 6107 6108 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 6109 return (mem_valid(pa, size)); 6110 } 6111 6112 static void 6113 mmu_radix_scan_init(void) 6114 { 6115 6116 CTR1(KTR_PMAP, "%s()", __func__); 6117 UNIMPLEMENTED(); 6118 } 6119 6120 static void 6121 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, 6122 void **va) 6123 { 6124 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va); 6125 UNIMPLEMENTED(); 6126 } 6127 6128 vm_offset_t 6129 mmu_radix_quick_enter_page(vm_page_t m) 6130 { 6131 vm_paddr_t paddr; 6132 6133 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 6134 paddr = VM_PAGE_TO_PHYS(m); 6135 return (PHYS_TO_DMAP(paddr)); 6136 } 6137 6138 void 6139 mmu_radix_quick_remove_page(vm_offset_t addr __unused) 6140 { 6141 /* no work to do here */ 6142 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 6143 } 6144 6145 static void 6146 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 6147 { 6148 cpu_flush_dcache((void *)sva, eva - sva); 6149 } 6150 6151 int 6152 mmu_radix_change_attr(vm_offset_t va, vm_size_t size, 6153 vm_memattr_t mode) 6154 { 6155 int error; 6156 6157 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode); 6158 PMAP_LOCK(kernel_pmap); 6159 error = pmap_change_attr_locked(va, size, mode, true); 6160 PMAP_UNLOCK(kernel_pmap); 6161 return (error); 6162 } 6163 6164 static int 6165 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush) 6166 { 6167 vm_offset_t base, offset, tmpva; 6168 vm_paddr_t pa_start, pa_end, pa_end1; 6169 pml2_entry_t *l2e; 6170 pml3_entry_t *l3e; 6171 pt_entry_t *pte; 6172 int cache_bits, error; 6173 boolean_t changed; 6174 6175 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 6176 base = trunc_page(va); 6177 offset = va & PAGE_MASK; 6178 size = round_page(offset + size); 6179 6180 /* 6181 * Only supported on kernel virtual addresses, including the direct 6182 * map but excluding the recursive map. 6183 */ 6184 if (base < DMAP_MIN_ADDRESS) 6185 return (EINVAL); 6186 6187 cache_bits = pmap_cache_bits(mode); 6188 changed = FALSE; 6189 6190 /* 6191 * Pages that aren't mapped aren't supported. Also break down 2MB pages 6192 * into 4KB pages if required. 6193 */ 6194 for (tmpva = base; tmpva < base + size; ) { 6195 l2e = pmap_pml2e(kernel_pmap, tmpva); 6196 if (l2e == NULL || *l2e == 0) 6197 return (EINVAL); 6198 if (be64toh(*l2e) & RPTE_LEAF) { 6199 /* 6200 * If the current 1GB page already has the required 6201 * memory type, then we need not demote this page. Just 6202 * increment tmpva to the next 1GB page frame. 6203 */ 6204 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) { 6205 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6206 continue; 6207 } 6208 6209 /* 6210 * If the current offset aligns with a 1GB page frame 6211 * and there is at least 1GB left within the range, then 6212 * we need not break down this page into 2MB pages. 6213 */ 6214 if ((tmpva & L2_PAGE_MASK) == 0 && 6215 tmpva + L2_PAGE_MASK < base + size) { 6216 tmpva += L2_PAGE_MASK; 6217 continue; 6218 } 6219 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva)) 6220 return (ENOMEM); 6221 } 6222 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6223 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n", 6224 tmpva, l2e)); 6225 if (*l3e == 0) 6226 return (EINVAL); 6227 if (be64toh(*l3e) & RPTE_LEAF) { 6228 /* 6229 * If the current 2MB page already has the required 6230 * memory type, then we need not demote this page. Just 6231 * increment tmpva to the next 2MB page frame. 6232 */ 6233 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) { 6234 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6235 continue; 6236 } 6237 6238 /* 6239 * If the current offset aligns with a 2MB page frame 6240 * and there is at least 2MB left within the range, then 6241 * we need not break down this page into 4KB pages. 6242 */ 6243 if ((tmpva & L3_PAGE_MASK) == 0 && 6244 tmpva + L3_PAGE_MASK < base + size) { 6245 tmpva += L3_PAGE_SIZE; 6246 continue; 6247 } 6248 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva)) 6249 return (ENOMEM); 6250 } 6251 pte = pmap_l3e_to_pte(l3e, tmpva); 6252 if (*pte == 0) 6253 return (EINVAL); 6254 tmpva += PAGE_SIZE; 6255 } 6256 error = 0; 6257 6258 /* 6259 * Ok, all the pages exist, so run through them updating their 6260 * cache mode if required. 6261 */ 6262 pa_start = pa_end = 0; 6263 for (tmpva = base; tmpva < base + size; ) { 6264 l2e = pmap_pml2e(kernel_pmap, tmpva); 6265 if (be64toh(*l2e) & RPTE_LEAF) { 6266 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) { 6267 pmap_pte_attr(l2e, cache_bits, 6268 RPTE_ATTR_MASK); 6269 changed = TRUE; 6270 } 6271 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6272 (*l2e & PG_PS_FRAME) < dmaplimit) { 6273 if (pa_start == pa_end) { 6274 /* Start physical address run. */ 6275 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6276 pa_end = pa_start + L2_PAGE_SIZE; 6277 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME)) 6278 pa_end += L2_PAGE_SIZE; 6279 else { 6280 /* Run ended, update direct map. */ 6281 error = pmap_change_attr_locked( 6282 PHYS_TO_DMAP(pa_start), 6283 pa_end - pa_start, mode, flush); 6284 if (error != 0) 6285 break; 6286 /* Start physical address run. */ 6287 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6288 pa_end = pa_start + L2_PAGE_SIZE; 6289 } 6290 } 6291 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6292 continue; 6293 } 6294 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6295 if (be64toh(*l3e) & RPTE_LEAF) { 6296 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) { 6297 pmap_pte_attr(l3e, cache_bits, 6298 RPTE_ATTR_MASK); 6299 changed = TRUE; 6300 } 6301 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6302 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) { 6303 if (pa_start == pa_end) { 6304 /* Start physical address run. */ 6305 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6306 pa_end = pa_start + L3_PAGE_SIZE; 6307 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME)) 6308 pa_end += L3_PAGE_SIZE; 6309 else { 6310 /* Run ended, update direct map. */ 6311 error = pmap_change_attr_locked( 6312 PHYS_TO_DMAP(pa_start), 6313 pa_end - pa_start, mode, flush); 6314 if (error != 0) 6315 break; 6316 /* Start physical address run. */ 6317 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6318 pa_end = pa_start + L3_PAGE_SIZE; 6319 } 6320 } 6321 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6322 } else { 6323 pte = pmap_l3e_to_pte(l3e, tmpva); 6324 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) { 6325 pmap_pte_attr(pte, cache_bits, 6326 RPTE_ATTR_MASK); 6327 changed = TRUE; 6328 } 6329 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6330 (be64toh(*pte) & PG_FRAME) < dmaplimit) { 6331 if (pa_start == pa_end) { 6332 /* Start physical address run. */ 6333 pa_start = be64toh(*pte) & PG_FRAME; 6334 pa_end = pa_start + PAGE_SIZE; 6335 } else if (pa_end == (be64toh(*pte) & PG_FRAME)) 6336 pa_end += PAGE_SIZE; 6337 else { 6338 /* Run ended, update direct map. */ 6339 error = pmap_change_attr_locked( 6340 PHYS_TO_DMAP(pa_start), 6341 pa_end - pa_start, mode, flush); 6342 if (error != 0) 6343 break; 6344 /* Start physical address run. */ 6345 pa_start = be64toh(*pte) & PG_FRAME; 6346 pa_end = pa_start + PAGE_SIZE; 6347 } 6348 } 6349 tmpva += PAGE_SIZE; 6350 } 6351 } 6352 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { 6353 pa_end1 = MIN(pa_end, dmaplimit); 6354 if (pa_start != pa_end1) 6355 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), 6356 pa_end1 - pa_start, mode, flush); 6357 } 6358 6359 /* 6360 * Flush CPU caches if required to make sure any data isn't cached that 6361 * shouldn't be, etc. 6362 */ 6363 if (changed) { 6364 pmap_invalidate_all(kernel_pmap); 6365 6366 if (flush) 6367 pmap_invalidate_cache_range(base, tmpva); 6368 } 6369 return (error); 6370 } 6371 6372 /* 6373 * Allocate physical memory for the vm_page array and map it into KVA, 6374 * attempting to back the vm_pages with domain-local memory. 6375 */ 6376 void 6377 mmu_radix_page_array_startup(long pages) 6378 { 6379 #ifdef notyet 6380 pml2_entry_t *l2e; 6381 pml3_entry_t *pde; 6382 pml3_entry_t newl3; 6383 vm_offset_t va; 6384 long pfn; 6385 int domain, i; 6386 #endif 6387 vm_paddr_t pa; 6388 vm_offset_t start, end; 6389 6390 vm_page_array_size = pages; 6391 6392 start = VM_MIN_KERNEL_ADDRESS; 6393 end = start + pages * sizeof(struct vm_page); 6394 6395 pa = vm_phys_early_alloc(0, end - start); 6396 6397 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT); 6398 #ifdef notyet 6399 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */ 6400 for (va = start; va < end; va += L3_PAGE_SIZE) { 6401 pfn = first_page + (va - start) / sizeof(struct vm_page); 6402 domain = vm_phys_domain(ptoa(pfn)); 6403 l2e = pmap_pml2e(kernel_pmap, va); 6404 if ((be64toh(*l2e) & PG_V) == 0) { 6405 pa = vm_phys_early_alloc(domain, PAGE_SIZE); 6406 dump_add_page(pa); 6407 pagezero(PHYS_TO_DMAP(pa)); 6408 pde_store(l2e, (pml2_entry_t)pa); 6409 } 6410 pde = pmap_l2e_to_l3e(l2e, va); 6411 if ((be64toh(*pde) & PG_V) != 0) 6412 panic("Unexpected pde %p", pde); 6413 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE); 6414 for (i = 0; i < NPDEPG; i++) 6415 dump_add_page(pa + i * PAGE_SIZE); 6416 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W); 6417 pte_store(pde, newl3); 6418 } 6419 #endif 6420 vm_page_array = (vm_page_t)start; 6421 } 6422 6423 #ifdef DDB 6424 #include <sys/kdb.h> 6425 #include <ddb/ddb.h> 6426 6427 static void 6428 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va) 6429 { 6430 pml1_entry_t *l1e; 6431 pml2_entry_t *l2e; 6432 pml3_entry_t *l3e; 6433 pt_entry_t *pte; 6434 6435 l1e = &l1[pmap_pml1e_index(va)]; 6436 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e)); 6437 if ((be64toh(*l1e) & PG_V) == 0) { 6438 db_printf("\n"); 6439 return; 6440 } 6441 l2e = pmap_l1e_to_l2e(l1e, va); 6442 db_printf(" l2e %#016lx", be64toh(*l2e)); 6443 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) { 6444 db_printf("\n"); 6445 return; 6446 } 6447 l3e = pmap_l2e_to_l3e(l2e, va); 6448 db_printf(" l3e %#016lx", be64toh(*l3e)); 6449 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) { 6450 db_printf("\n"); 6451 return; 6452 } 6453 pte = pmap_l3e_to_pte(l3e, va); 6454 db_printf(" pte %#016lx\n", be64toh(*pte)); 6455 } 6456 6457 void 6458 pmap_page_print_mappings(vm_page_t m) 6459 { 6460 pmap_t pmap; 6461 pv_entry_t pv; 6462 6463 db_printf("page %p(%lx)\n", m, m->phys_addr); 6464 /* need to elide locks if running in ddb */ 6465 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 6466 db_printf("pv: %p ", pv); 6467 db_printf("va: %#016lx ", pv->pv_va); 6468 pmap = PV_PMAP(pv); 6469 db_printf("pmap %p ", pmap); 6470 if (pmap != NULL) { 6471 db_printf("asid: %lu\n", pmap->pm_pid); 6472 pmap_pte_walk(pmap->pm_pml1, pv->pv_va); 6473 } 6474 } 6475 } 6476 6477 DB_SHOW_COMMAND(pte, pmap_print_pte) 6478 { 6479 vm_offset_t va; 6480 pmap_t pmap; 6481 6482 if (!have_addr) { 6483 db_printf("show pte addr\n"); 6484 return; 6485 } 6486 va = (vm_offset_t)addr; 6487 6488 if (va >= DMAP_MIN_ADDRESS) 6489 pmap = kernel_pmap; 6490 else if (kdb_thread != NULL) 6491 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); 6492 else 6493 pmap = vmspace_pmap(curthread->td_proc->p_vmspace); 6494 6495 pmap_pte_walk(pmap->pm_pml1, va); 6496 } 6497 6498 #endif 6499