1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2010 Andreas Tobler 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/kernel.h> 33 #include <sys/ktr.h> 34 #include <sys/lock.h> 35 #include <sys/rmlock.h> 36 #include <sys/mutex.h> 37 #include <sys/proc.h> 38 #include <sys/sysctl.h> 39 #include <sys/systm.h> 40 #include <sys/vmmeter.h> 41 42 #include <dev/ofw/openfirm.h> 43 #include <machine/ofw_machdep.h> 44 45 #include <vm/vm.h> 46 #include <vm/vm_param.h> 47 #include <vm/vm_kern.h> 48 #include <vm/vm_page.h> 49 #include <vm/vm_map.h> 50 #include <vm/vm_object.h> 51 #include <vm/vm_extern.h> 52 #include <vm/vm_pageout.h> 53 #include <vm/uma.h> 54 55 #include <powerpc/aim/mmu_oea64.h> 56 57 #include "phyp-hvcall.h" 58 59 #define MMU_PHYP_DEBUG 0 60 #define MMU_PHYP_ID "mmu_phyp: " 61 #if MMU_PHYP_DEBUG 62 #define dprintf(fmt, ...) printf(fmt, ## __VA_ARGS__) 63 #define dprintf0(fmt, ...) dprintf(MMU_PHYP_ID fmt, ## __VA_ARGS__) 64 #else 65 #define dprintf(fmt, args...) do { ; } while(0) 66 #define dprintf0(fmt, args...) do { ; } while(0) 67 #endif 68 69 static struct rmlock mphyp_eviction_lock; 70 71 /* 72 * Kernel MMU interface 73 */ 74 75 static void mphyp_install(void); 76 static void mphyp_bootstrap(vm_offset_t kernelstart, 77 vm_offset_t kernelend); 78 static void mphyp_cpu_bootstrap(int ap); 79 static void *mphyp_dump_pmap(void *ctx, void *buf, 80 u_long *nbytes); 81 static int64_t mphyp_pte_synch(struct pvo_entry *pvo); 82 static int64_t mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit); 83 static int64_t mphyp_pte_unset(struct pvo_entry *pvo); 84 static int64_t mphyp_pte_insert(struct pvo_entry *pvo); 85 static int64_t mphyp_pte_unset_sp(struct pvo_entry *pvo); 86 static int64_t mphyp_pte_insert_sp(struct pvo_entry *pvo); 87 static int64_t mphyp_pte_replace_sp(struct pvo_entry *pvo); 88 89 static struct pmap_funcs mphyp_methods = { 90 .install = mphyp_install, 91 .bootstrap = mphyp_bootstrap, 92 .cpu_bootstrap = mphyp_cpu_bootstrap, 93 .dumpsys_dump_pmap = mphyp_dump_pmap, 94 }; 95 96 static struct moea64_funcs mmu_phyp_funcs = { 97 .pte_synch = mphyp_pte_synch, 98 .pte_clear = mphyp_pte_clear, 99 .pte_unset = mphyp_pte_unset, 100 .pte_insert = mphyp_pte_insert, 101 .pte_unset_sp = mphyp_pte_unset_sp, 102 .pte_insert_sp = mphyp_pte_insert_sp, 103 .pte_replace_sp = mphyp_pte_replace_sp, 104 }; 105 106 MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, oea64_mmu); 107 108 static int brokenkvm = 0; 109 110 static void 111 print_kvm_bug_warning(void *data) 112 { 113 114 if (brokenkvm) 115 printf("WARNING: Running on a broken hypervisor that does " 116 "not support mandatory H_CLEAR_MOD and H_CLEAR_REF " 117 "hypercalls. Performance will be suboptimal.\n"); 118 } 119 120 SYSINIT(kvmbugwarn1, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1, 121 print_kvm_bug_warning, NULL); 122 SYSINIT(kvmbugwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 1, print_kvm_bug_warning, 123 NULL); 124 125 static void 126 mphyp_install() 127 { 128 129 moea64_ops = &mmu_phyp_funcs; 130 131 moea64_install(); 132 } 133 134 static void 135 mphyp_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend) 136 { 137 uint64_t final_pteg_count = 0; 138 char buf[8]; 139 uint32_t prop[2]; 140 uint32_t nptlp, shift = 0, slb_encoding = 0; 141 uint32_t lp_size, lp_encoding; 142 struct lpte old; 143 uint64_t vsid; 144 phandle_t dev, node, root; 145 int idx, len, res; 146 bool has_lp; 147 148 rm_init(&mphyp_eviction_lock, "pte eviction"); 149 150 moea64_early_bootstrap(kernelstart, kernelend); 151 152 root = OF_peer(0); 153 154 dev = OF_child(root); 155 while (dev != 0) { 156 res = OF_getprop(dev, "name", buf, sizeof(buf)); 157 if (res > 0 && strcmp(buf, "cpus") == 0) 158 break; 159 dev = OF_peer(dev); 160 } 161 162 node = OF_child(dev); 163 164 while (node != 0) { 165 res = OF_getprop(node, "device_type", buf, sizeof(buf)); 166 if (res > 0 && strcmp(buf, "cpu") == 0) 167 break; 168 node = OF_peer(node); 169 } 170 171 res = OF_getencprop(node, "ibm,pft-size", prop, sizeof(prop)); 172 if (res <= 0) 173 panic("mmu_phyp: unknown PFT size"); 174 final_pteg_count = 1 << prop[1]; 175 res = OF_getencprop(node, "ibm,slb-size", prop, sizeof(prop[0])); 176 if (res > 0) 177 n_slbs = prop[0]; 178 dprintf0("slb-size=%i\n", n_slbs); 179 180 moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 181 182 /* Clear any old page table entries */ 183 for (idx = 0; idx < moea64_pteg_count*8; idx++) { 184 phyp_pft_hcall(H_READ, 0, idx, 0, 0, &old.pte_hi, 185 &old.pte_lo, &old.pte_lo); 186 vsid = (old.pte_hi << (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) >> 28; 187 if (vsid == VSID_VRMA || vsid == 0 /* Older VRMA */) 188 continue; 189 190 if (old.pte_hi & LPTE_VALID) 191 phyp_hcall(H_REMOVE, 0, idx, 0); 192 } 193 194 /* 195 * Scan the large page size property for PAPR compatible machines. 196 * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties' 197 * for the encoding of the property. 198 */ 199 200 len = OF_getproplen(node, "ibm,segment-page-sizes"); 201 if (len > 0) { 202 /* 203 * We have to use a variable length array on the stack 204 * since we have very limited stack space. 205 */ 206 pcell_t arr[len/sizeof(cell_t)]; 207 res = OF_getencprop(node, "ibm,segment-page-sizes", arr, 208 sizeof(arr)); 209 len /= 4; 210 idx = 0; 211 has_lp = false; 212 while (len > 0) { 213 shift = arr[idx]; 214 slb_encoding = arr[idx + 1]; 215 nptlp = arr[idx + 2]; 216 217 dprintf0("Segment Page Size: " 218 "%uKB, slb_enc=0x%X: {size, encoding}[%u] =", 219 shift > 10? 1 << (shift-10) : 0, 220 slb_encoding, nptlp); 221 222 idx += 3; 223 len -= 3; 224 while (len > 0 && nptlp) { 225 lp_size = arr[idx]; 226 lp_encoding = arr[idx+1]; 227 228 dprintf(" {%uKB, 0x%X}", 229 lp_size > 10? 1 << (lp_size-10) : 0, 230 lp_encoding); 231 232 if (slb_encoding == SLBV_L && lp_encoding == 0) 233 has_lp = true; 234 235 if (slb_encoding == SLB_PGSZ_4K_4K && 236 lp_encoding == LP_4K_16M) 237 moea64_has_lp_4k_16m = true; 238 239 idx += 2; 240 len -= 2; 241 nptlp--; 242 } 243 dprintf("\n"); 244 if (has_lp && moea64_has_lp_4k_16m) 245 break; 246 } 247 248 if (has_lp) { 249 moea64_large_page_shift = shift; 250 moea64_large_page_size = 1ULL << lp_size; 251 moea64_large_page_mask = moea64_large_page_size - 1; 252 hw_direct_map = 1; 253 printf(MMU_PHYP_ID 254 "Support for hugepages of %uKB detected\n", 255 moea64_large_page_shift > 10? 256 1 << (moea64_large_page_shift-10) : 0); 257 } else { 258 moea64_large_page_size = 0; 259 moea64_large_page_shift = 0; 260 moea64_large_page_mask = 0; 261 hw_direct_map = 0; 262 printf(MMU_PHYP_ID 263 "Support for hugepages not found\n"); 264 } 265 } 266 267 moea64_mid_bootstrap(kernelstart, kernelend); 268 moea64_late_bootstrap(kernelstart, kernelend); 269 270 /* Test for broken versions of KVM that don't conform to the spec */ 271 if (phyp_hcall(H_CLEAR_MOD, 0, 0) == H_FUNCTION) 272 brokenkvm = 1; 273 } 274 275 static void 276 mphyp_cpu_bootstrap(int ap) 277 { 278 struct slb *slb = PCPU_GET(aim.slb); 279 register_t seg0; 280 int i; 281 282 /* 283 * Install kernel SLB entries 284 */ 285 286 __asm __volatile ("slbia"); 287 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 288 for (i = 0; i < 64; i++) { 289 if (!(slb[i].slbe & SLBE_VALID)) 290 continue; 291 292 __asm __volatile ("slbmte %0, %1" :: 293 "r"(slb[i].slbv), "r"(slb[i].slbe)); 294 } 295 } 296 297 static int64_t 298 mphyp_pte_synch(struct pvo_entry *pvo) 299 { 300 struct lpte pte; 301 uint64_t junk; 302 303 __asm __volatile("ptesync"); 304 phyp_pft_hcall(H_READ, 0, pvo->pvo_pte.slot, 0, 0, &pte.pte_hi, 305 &pte.pte_lo, &junk); 306 if ((pte.pte_hi & LPTE_AVPN_MASK) != 307 ((pvo->pvo_vpn >> (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) & 308 LPTE_AVPN_MASK)) 309 return (-1); 310 if (!(pte.pte_hi & LPTE_VALID)) 311 return (-1); 312 313 return (pte.pte_lo & (LPTE_CHG | LPTE_REF)); 314 } 315 316 static int64_t 317 mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit) 318 { 319 struct rm_priotracker track; 320 int64_t refchg; 321 uint64_t ptelo, junk; 322 int err; 323 324 /* 325 * This involves two steps (synch and clear) so we need the entry 326 * not to change in the middle. We are protected against deliberate 327 * unset by virtue of holding the pmap lock. Protection against 328 * incidental unset (page table eviction) comes from holding the 329 * shared eviction lock. 330 */ 331 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 332 rm_rlock(&mphyp_eviction_lock, &track); 333 334 refchg = mphyp_pte_synch(pvo); 335 if (refchg < 0) { 336 rm_runlock(&mphyp_eviction_lock, &track); 337 return (refchg); 338 } 339 340 if (brokenkvm) { 341 /* 342 * No way to clear either bit, which is total madness. 343 * Pessimistically claim that, once modified, it stays so 344 * forever and that it is never referenced. 345 */ 346 rm_runlock(&mphyp_eviction_lock, &track); 347 return (refchg & ~LPTE_REF); 348 } 349 350 if (ptebit & LPTE_CHG) { 351 err = phyp_pft_hcall(H_CLEAR_MOD, 0, pvo->pvo_pte.slot, 0, 0, 352 &ptelo, &junk, &junk); 353 KASSERT(err == H_SUCCESS, 354 ("Error clearing page change bit: %d", err)); 355 refchg |= (ptelo & LPTE_CHG); 356 } 357 if (ptebit & LPTE_REF) { 358 err = phyp_pft_hcall(H_CLEAR_REF, 0, pvo->pvo_pte.slot, 0, 0, 359 &ptelo, &junk, &junk); 360 KASSERT(err == H_SUCCESS, 361 ("Error clearing page reference bit: %d", err)); 362 refchg |= (ptelo & LPTE_REF); 363 } 364 365 rm_runlock(&mphyp_eviction_lock, &track); 366 367 return (refchg); 368 } 369 370 static int64_t 371 mphyp_pte_unset(struct pvo_entry *pvo) 372 { 373 struct lpte pte; 374 uint64_t junk; 375 int err; 376 377 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 378 379 moea64_pte_from_pvo(pvo, &pte); 380 381 err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot, 382 pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, 383 &junk); 384 KASSERT(err == H_SUCCESS || err == H_NOT_FOUND, 385 ("Error removing page: %d", err)); 386 387 if (err == H_NOT_FOUND) { 388 STAT_MOEA64(moea64_pte_overflow--); 389 return (-1); 390 } 391 392 return (pte.pte_lo & (LPTE_REF | LPTE_CHG)); 393 } 394 395 static uintptr_t 396 mphyp_pte_spillable_ident(uintptr_t ptegbase, struct lpte *to_evict) 397 { 398 uint64_t slot, junk, k; 399 struct lpte pt; 400 int i, j; 401 402 /* Start at a random slot */ 403 i = mftb() % 8; 404 k = -1; 405 for (j = 0; j < 8; j++) { 406 slot = ptegbase + (i + j) % 8; 407 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, 408 &pt.pte_lo, &junk); 409 410 if ((pt.pte_hi & (LPTE_WIRED | LPTE_BIG)) != 0) 411 continue; 412 413 /* This is a candidate, so remember it */ 414 k = slot; 415 416 /* Try to get a page that has not been used lately */ 417 if (!(pt.pte_hi & LPTE_VALID) || !(pt.pte_lo & LPTE_REF)) { 418 memcpy(to_evict, &pt, sizeof(struct lpte)); 419 return (k); 420 } 421 } 422 423 if (k == -1) 424 return (k); 425 426 phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi, 427 &to_evict->pte_lo, &junk); 428 return (k); 429 } 430 431 static __inline int64_t 432 mphyp_pte_insert_locked(struct pvo_entry *pvo, struct lpte *pte) 433 { 434 struct lpte evicted; 435 uint64_t index, junk; 436 int64_t result; 437 438 /* 439 * First try primary hash. 440 */ 441 pvo->pvo_pte.slot &= ~7UL; /* Base slot address */ 442 result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte->pte_hi, 443 pte->pte_lo, &index, &evicted.pte_lo, &junk); 444 if (result == H_SUCCESS) { 445 pvo->pvo_pte.slot = index; 446 return (0); 447 } 448 KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " 449 "(ptegidx: %#zx/%#lx, PTE %#lx/%#lx", result, pvo->pvo_pte.slot, 450 moea64_pteg_count, pte->pte_hi, pte->pte_lo)); 451 452 /* 453 * Next try secondary hash. 454 */ 455 pvo->pvo_vaddr ^= PVO_HID; 456 pte->pte_hi ^= LPTE_HID; 457 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 458 459 result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, 460 pte->pte_hi, pte->pte_lo, &index, &evicted.pte_lo, &junk); 461 if (result == H_SUCCESS) { 462 pvo->pvo_pte.slot = index; 463 return (0); 464 } 465 KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", 466 result)); 467 468 return (-1); 469 } 470 471 472 static __inline int64_t 473 mphyp_pte_evict_and_insert_locked(struct pvo_entry *pvo, struct lpte *pte) 474 { 475 struct lpte evicted; 476 uint64_t index, junk, lastptelo; 477 int64_t result; 478 479 evicted.pte_hi = 0; 480 481 index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); 482 if (index == -1L) { 483 /* Try other hash table? */ 484 pvo->pvo_vaddr ^= PVO_HID; 485 pte->pte_hi ^= LPTE_HID; 486 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 487 index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); 488 } 489 490 if (index == -1L) { 491 /* No freeable slots in either PTEG? We're hosed. */ 492 rm_wunlock(&mphyp_eviction_lock); 493 panic("mphyp_pte_insert: overflow"); 494 return (-1); 495 } 496 497 /* Victim acquired: update page before waving goodbye */ 498 if (evicted.pte_hi & LPTE_VALID) { 499 result = phyp_pft_hcall(H_REMOVE, H_AVPN, index, 500 evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo, 501 &junk); 502 STAT_MOEA64(moea64_pte_overflow++); 503 KASSERT(result == H_SUCCESS || result == H_NOT_FOUND, 504 ("Error evicting page: %d", (int)result)); 505 } 506 507 /* 508 * Set the new PTE. 509 */ 510 result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte->pte_hi, 511 pte->pte_lo, &index, &evicted.pte_lo, &junk); 512 513 pvo->pvo_pte.slot = index; 514 if (result == H_SUCCESS) 515 return (0); 516 517 rm_wunlock(&mphyp_eviction_lock); 518 panic("Page replacement error: %ld", result); 519 return (result); 520 } 521 522 static int64_t 523 mphyp_pte_insert(struct pvo_entry *pvo) 524 { 525 struct rm_priotracker track; 526 int64_t ret; 527 struct lpte pte; 528 529 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 530 531 /* Initialize PTE */ 532 moea64_pte_from_pvo(pvo, &pte); 533 534 /* Make sure further insertion is locked out during evictions */ 535 rm_rlock(&mphyp_eviction_lock, &track); 536 537 ret = mphyp_pte_insert_locked(pvo, &pte); 538 rm_runlock(&mphyp_eviction_lock, &track); 539 540 if (ret == -1) { 541 /* 542 * Out of luck. Find a PTE to sacrifice. 543 */ 544 545 /* Lock out all insertions for a bit */ 546 rm_wlock(&mphyp_eviction_lock); 547 ret = mphyp_pte_evict_and_insert_locked(pvo, &pte); 548 rm_wunlock(&mphyp_eviction_lock); /* All clear */ 549 } 550 551 return (ret); 552 } 553 554 static void * 555 mphyp_dump_pmap(void *ctx, void *buf, u_long *nbytes) 556 { 557 struct dump_context *dctx; 558 struct lpte p, *pbuf; 559 int bufidx; 560 uint64_t junk; 561 u_long ptex, ptex_end; 562 563 dctx = (struct dump_context *)ctx; 564 pbuf = (struct lpte *)buf; 565 bufidx = 0; 566 ptex = dctx->ptex; 567 ptex_end = ptex + dctx->blksz / sizeof(struct lpte); 568 ptex_end = MIN(ptex_end, dctx->ptex_end); 569 *nbytes = (ptex_end - ptex) * sizeof(struct lpte); 570 571 if (*nbytes == 0) 572 return (NULL); 573 574 for (; ptex < ptex_end; ptex++) { 575 phyp_pft_hcall(H_READ, 0, ptex, 0, 0, 576 &p.pte_hi, &p.pte_lo, &junk); 577 pbuf[bufidx++] = p; 578 } 579 580 dctx->ptex = ptex; 581 return (buf); 582 } 583 584 static int64_t 585 mphyp_pte_unset_sp(struct pvo_entry *pvo) 586 { 587 struct lpte pte; 588 uint64_t junk, refchg; 589 int err; 590 vm_offset_t eva; 591 pmap_t pm; 592 593 pm = pvo->pvo_pmap; 594 PMAP_LOCK_ASSERT(pm, MA_OWNED); 595 KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0, 596 ("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo))); 597 598 refchg = 0; 599 eva = PVO_VADDR(pvo) + HPT_SP_SIZE; 600 601 for (; pvo != NULL && PVO_VADDR(pvo) < eva; 602 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { 603 moea64_pte_from_pvo(pvo, &pte); 604 605 err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot, 606 pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, 607 &junk); 608 KASSERT(err == H_SUCCESS || err == H_NOT_FOUND, 609 ("Error removing page: %d", err)); 610 611 if (err == H_NOT_FOUND) 612 STAT_MOEA64(moea64_pte_overflow--); 613 refchg |= pte.pte_lo & (LPTE_REF | LPTE_CHG); 614 } 615 616 return (refchg); 617 } 618 619 static int64_t 620 mphyp_pte_insert_sp(struct pvo_entry *pvo) 621 { 622 struct rm_priotracker track; 623 int64_t ret; 624 struct lpte pte; 625 vm_offset_t eva; 626 pmap_t pm; 627 628 pm = pvo->pvo_pmap; 629 PMAP_LOCK_ASSERT(pm, MA_OWNED); 630 KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0, 631 ("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo))); 632 633 eva = PVO_VADDR(pvo) + HPT_SP_SIZE; 634 635 /* Make sure further insertion is locked out during evictions */ 636 rm_rlock(&mphyp_eviction_lock, &track); 637 638 for (; pvo != NULL && PVO_VADDR(pvo) < eva; 639 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { 640 /* Initialize PTE */ 641 moea64_pte_from_pvo(pvo, &pte); 642 643 ret = mphyp_pte_insert_locked(pvo, &pte); 644 if (ret == -1) { 645 /* 646 * Out of luck. Find a PTE to sacrifice. 647 */ 648 649 /* Lock out all insertions for a bit */ 650 rm_runlock(&mphyp_eviction_lock, &track); 651 rm_wlock(&mphyp_eviction_lock); 652 mphyp_pte_evict_and_insert_locked(pvo, &pte); 653 rm_wunlock(&mphyp_eviction_lock); /* All clear */ 654 rm_rlock(&mphyp_eviction_lock, &track); 655 } 656 } 657 658 rm_runlock(&mphyp_eviction_lock, &track); 659 return (0); 660 } 661 662 static int64_t 663 mphyp_pte_replace_sp(struct pvo_entry *pvo) 664 { 665 int64_t refchg; 666 667 refchg = mphyp_pte_unset_sp(pvo); 668 mphyp_pte_insert_sp(pvo); 669 return (refchg); 670 } 671