1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2020 Justin Hibbits 5 * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com> 6 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * Some hw specific parts of this pmap were derived or influenced 30 * by NetBSD's ibm4xx pmap module. More generic code is shared with 31 * a few other pmap modules from the FreeBSD tree. 32 */ 33 34 /* 35 * VM layout notes: 36 * 37 * Kernel and user threads run within one common virtual address space 38 * defined by AS=0. 39 * 40 * 32-bit pmap: 41 * Virtual address space layout: 42 * ----------------------------- 43 * 0x0000_0000 - 0x7fff_ffff : user process 44 * 0x8000_0000 - 0xbfff_ffff : pmap_mapdev()-ed area (PCI/PCIE etc.) 45 * 0xc000_0000 - 0xffff_efff : KVA 46 */ 47 48 #include <sys/cdefs.h> 49 __FBSDID("$FreeBSD$"); 50 51 #include "opt_ddb.h" 52 #include "opt_kstack_pages.h" 53 54 #include <sys/param.h> 55 #include <sys/conf.h> 56 #include <sys/malloc.h> 57 #include <sys/ktr.h> 58 #include <sys/proc.h> 59 #include <sys/user.h> 60 #include <sys/queue.h> 61 #include <sys/systm.h> 62 #include <sys/kernel.h> 63 #include <sys/kerneldump.h> 64 #include <sys/linker.h> 65 #include <sys/msgbuf.h> 66 #include <sys/lock.h> 67 #include <sys/mutex.h> 68 #include <sys/rwlock.h> 69 #include <sys/sched.h> 70 #include <sys/smp.h> 71 #include <sys/vmmeter.h> 72 73 #include <vm/vm.h> 74 #include <vm/vm_page.h> 75 #include <vm/vm_kern.h> 76 #include <vm/vm_pageout.h> 77 #include <vm/vm_extern.h> 78 #include <vm/vm_object.h> 79 #include <vm/vm_param.h> 80 #include <vm/vm_map.h> 81 #include <vm/vm_pager.h> 82 #include <vm/vm_phys.h> 83 #include <vm/vm_pagequeue.h> 84 #include <vm/uma.h> 85 86 #include <machine/_inttypes.h> 87 #include <machine/cpu.h> 88 #include <machine/pcb.h> 89 #include <machine/platform.h> 90 91 #include <machine/tlb.h> 92 #include <machine/spr.h> 93 #include <machine/md_var.h> 94 #include <machine/mmuvar.h> 95 #include <machine/pmap.h> 96 #include <machine/pte.h> 97 98 #include <ddb/ddb.h> 99 100 #define PRI0ptrX "08x" 101 102 /* Reserved KVA space and mutex for mmu_booke_zero_page. */ 103 static vm_offset_t zero_page_va; 104 static struct mtx zero_page_mutex; 105 106 /* Reserved KVA space and mutex for mmu_booke_copy_page. */ 107 static vm_offset_t copy_page_src_va; 108 static vm_offset_t copy_page_dst_va; 109 static struct mtx copy_page_mutex; 110 111 static vm_offset_t kernel_ptbl_root; 112 static unsigned int kernel_ptbls; /* Number of KVA ptbls. */ 113 114 /**************************************************************************/ 115 /* PMAP */ 116 /**************************************************************************/ 117 118 #define VM_MAPDEV_BASE ((vm_offset_t)VM_MAXUSER_ADDRESS + PAGE_SIZE) 119 120 static void tid_flush(tlbtid_t tid); 121 static unsigned long ilog2(unsigned long); 122 123 /**************************************************************************/ 124 /* Page table management */ 125 /**************************************************************************/ 126 127 #define PMAP_ROOT_SIZE (sizeof(pte_t**) * PDIR_NENTRIES) 128 static void ptbl_init(void); 129 static struct ptbl_buf *ptbl_buf_alloc(void); 130 static void ptbl_buf_free(struct ptbl_buf *); 131 static void ptbl_free_pmap_ptbl(pmap_t, pte_t *); 132 133 static pte_t *ptbl_alloc(pmap_t, unsigned int, boolean_t); 134 static void ptbl_free(pmap_t, unsigned int); 135 static void ptbl_hold(pmap_t, unsigned int); 136 static int ptbl_unhold(pmap_t, unsigned int); 137 138 static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t); 139 static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t); 140 static int pte_remove(pmap_t, vm_offset_t, uint8_t); 141 static pte_t *pte_find(pmap_t, vm_offset_t); 142 143 struct ptbl_buf { 144 TAILQ_ENTRY(ptbl_buf) link; /* list link */ 145 vm_offset_t kva; /* va of mapping */ 146 }; 147 148 /* Number of kva ptbl buffers, each covering one ptbl (PTBL_PAGES). */ 149 #define PTBL_BUFS (128 * 16) 150 151 /* ptbl free list and a lock used for access synchronization. */ 152 static TAILQ_HEAD(, ptbl_buf) ptbl_buf_freelist; 153 static struct mtx ptbl_buf_freelist_lock; 154 155 /* Base address of kva space allocated fot ptbl bufs. */ 156 static vm_offset_t ptbl_buf_pool_vabase; 157 158 /* Pointer to ptbl_buf structures. */ 159 static struct ptbl_buf *ptbl_bufs; 160 161 /**************************************************************************/ 162 /* Page table related */ 163 /**************************************************************************/ 164 165 166 /* Initialize pool of kva ptbl buffers. */ 167 static void 168 ptbl_init(void) 169 { 170 int i; 171 172 CTR3(KTR_PMAP, "%s: s (ptbl_bufs = 0x%08x size 0x%08x)", __func__, 173 (uint32_t)ptbl_bufs, sizeof(struct ptbl_buf) * PTBL_BUFS); 174 CTR3(KTR_PMAP, "%s: s (ptbl_buf_pool_vabase = 0x%08x size = 0x%08x)", 175 __func__, ptbl_buf_pool_vabase, PTBL_BUFS * PTBL_PAGES * PAGE_SIZE); 176 177 mtx_init(&ptbl_buf_freelist_lock, "ptbl bufs lock", NULL, MTX_DEF); 178 TAILQ_INIT(&ptbl_buf_freelist); 179 180 for (i = 0; i < PTBL_BUFS; i++) { 181 ptbl_bufs[i].kva = 182 ptbl_buf_pool_vabase + i * PTBL_PAGES * PAGE_SIZE; 183 TAILQ_INSERT_TAIL(&ptbl_buf_freelist, &ptbl_bufs[i], link); 184 } 185 } 186 187 /* Get a ptbl_buf from the freelist. */ 188 static struct ptbl_buf * 189 ptbl_buf_alloc(void) 190 { 191 struct ptbl_buf *buf; 192 193 mtx_lock(&ptbl_buf_freelist_lock); 194 buf = TAILQ_FIRST(&ptbl_buf_freelist); 195 if (buf != NULL) 196 TAILQ_REMOVE(&ptbl_buf_freelist, buf, link); 197 mtx_unlock(&ptbl_buf_freelist_lock); 198 199 CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf); 200 201 return (buf); 202 } 203 204 /* Return ptbl buff to free pool. */ 205 static void 206 ptbl_buf_free(struct ptbl_buf *buf) 207 { 208 209 CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf); 210 211 mtx_lock(&ptbl_buf_freelist_lock); 212 TAILQ_INSERT_TAIL(&ptbl_buf_freelist, buf, link); 213 mtx_unlock(&ptbl_buf_freelist_lock); 214 } 215 216 /* 217 * Search the list of allocated ptbl bufs and find on list of allocated ptbls 218 */ 219 static void 220 ptbl_free_pmap_ptbl(pmap_t pmap, pte_t *ptbl) 221 { 222 struct ptbl_buf *pbuf; 223 224 CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl); 225 226 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 227 228 TAILQ_FOREACH(pbuf, &pmap->pm_ptbl_list, link) 229 if (pbuf->kva == (vm_offset_t)ptbl) { 230 /* Remove from pmap ptbl buf list. */ 231 TAILQ_REMOVE(&pmap->pm_ptbl_list, pbuf, link); 232 233 /* Free corresponding ptbl buf. */ 234 ptbl_buf_free(pbuf); 235 break; 236 } 237 } 238 239 /* Allocate page table. */ 240 static pte_t * 241 ptbl_alloc(pmap_t pmap, unsigned int pdir_idx, boolean_t nosleep) 242 { 243 vm_page_t mtbl[PTBL_PAGES]; 244 vm_page_t m; 245 struct ptbl_buf *pbuf; 246 unsigned int pidx; 247 pte_t *ptbl; 248 int i, j; 249 250 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap, 251 (pmap == kernel_pmap), pdir_idx); 252 253 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)), 254 ("ptbl_alloc: invalid pdir_idx")); 255 KASSERT((pmap->pm_pdir[pdir_idx] == NULL), 256 ("pte_alloc: valid ptbl entry exists!")); 257 258 pbuf = ptbl_buf_alloc(); 259 if (pbuf == NULL) 260 panic("pte_alloc: couldn't alloc kernel virtual memory"); 261 262 ptbl = (pte_t *)pbuf->kva; 263 264 CTR2(KTR_PMAP, "%s: ptbl kva = %p", __func__, ptbl); 265 266 for (i = 0; i < PTBL_PAGES; i++) { 267 pidx = (PTBL_PAGES * pdir_idx) + i; 268 while ((m = vm_page_alloc(NULL, pidx, 269 VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { 270 if (nosleep) { 271 ptbl_free_pmap_ptbl(pmap, ptbl); 272 for (j = 0; j < i; j++) 273 vm_page_free(mtbl[j]); 274 vm_wire_sub(i); 275 return (NULL); 276 } 277 PMAP_UNLOCK(pmap); 278 rw_wunlock(&pvh_global_lock); 279 vm_wait(NULL); 280 rw_wlock(&pvh_global_lock); 281 PMAP_LOCK(pmap); 282 } 283 mtbl[i] = m; 284 } 285 286 /* Map allocated pages into kernel_pmap. */ 287 mmu_booke_qenter((vm_offset_t)ptbl, mtbl, PTBL_PAGES); 288 289 /* Zero whole ptbl. */ 290 bzero((caddr_t)ptbl, PTBL_PAGES * PAGE_SIZE); 291 292 /* Add pbuf to the pmap ptbl bufs list. */ 293 TAILQ_INSERT_TAIL(&pmap->pm_ptbl_list, pbuf, link); 294 295 return (ptbl); 296 } 297 298 /* Free ptbl pages and invalidate pdir entry. */ 299 static void 300 ptbl_free(pmap_t pmap, unsigned int pdir_idx) 301 { 302 pte_t *ptbl; 303 vm_paddr_t pa; 304 vm_offset_t va; 305 vm_page_t m; 306 int i; 307 308 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap, 309 (pmap == kernel_pmap), pdir_idx); 310 311 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)), 312 ("ptbl_free: invalid pdir_idx")); 313 314 ptbl = pmap->pm_pdir[pdir_idx]; 315 316 CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl); 317 318 KASSERT((ptbl != NULL), ("ptbl_free: null ptbl")); 319 320 /* 321 * Invalidate the pdir entry as soon as possible, so that other CPUs 322 * don't attempt to look up the page tables we are releasing. 323 */ 324 mtx_lock_spin(&tlbivax_mutex); 325 tlb_miss_lock(); 326 327 pmap->pm_pdir[pdir_idx] = NULL; 328 329 tlb_miss_unlock(); 330 mtx_unlock_spin(&tlbivax_mutex); 331 332 for (i = 0; i < PTBL_PAGES; i++) { 333 va = ((vm_offset_t)ptbl + (i * PAGE_SIZE)); 334 pa = pte_vatopa(kernel_pmap, va); 335 m = PHYS_TO_VM_PAGE(pa); 336 vm_page_free_zero(m); 337 vm_wire_sub(1); 338 mmu_booke_kremove(va); 339 } 340 341 ptbl_free_pmap_ptbl(pmap, ptbl); 342 } 343 344 /* 345 * Decrement ptbl pages hold count and attempt to free ptbl pages. 346 * Called when removing pte entry from ptbl. 347 * 348 * Return 1 if ptbl pages were freed. 349 */ 350 static int 351 ptbl_unhold(pmap_t pmap, unsigned int pdir_idx) 352 { 353 pte_t *ptbl; 354 vm_paddr_t pa; 355 vm_page_t m; 356 int i; 357 358 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap, 359 (pmap == kernel_pmap), pdir_idx); 360 361 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)), 362 ("ptbl_unhold: invalid pdir_idx")); 363 KASSERT((pmap != kernel_pmap), 364 ("ptbl_unhold: unholding kernel ptbl!")); 365 366 ptbl = pmap->pm_pdir[pdir_idx]; 367 368 //debugf("ptbl_unhold: ptbl = 0x%08x\n", (u_int32_t)ptbl); 369 KASSERT(((vm_offset_t)ptbl >= VM_MIN_KERNEL_ADDRESS), 370 ("ptbl_unhold: non kva ptbl")); 371 372 /* decrement hold count */ 373 for (i = 0; i < PTBL_PAGES; i++) { 374 pa = pte_vatopa(kernel_pmap, 375 (vm_offset_t)ptbl + (i * PAGE_SIZE)); 376 m = PHYS_TO_VM_PAGE(pa); 377 m->ref_count--; 378 } 379 380 /* 381 * Free ptbl pages if there are no pte etries in this ptbl. 382 * ref_count has the same value for all ptbl pages, so check the last 383 * page. 384 */ 385 if (m->ref_count == 0) { 386 ptbl_free(pmap, pdir_idx); 387 388 //debugf("ptbl_unhold: e (freed ptbl)\n"); 389 return (1); 390 } 391 392 return (0); 393 } 394 395 /* 396 * Increment hold count for ptbl pages. This routine is used when a new pte 397 * entry is being inserted into the ptbl. 398 */ 399 static void 400 ptbl_hold(pmap_t pmap, unsigned int pdir_idx) 401 { 402 vm_paddr_t pa; 403 pte_t *ptbl; 404 vm_page_t m; 405 int i; 406 407 CTR3(KTR_PMAP, "%s: pmap = %p pdir_idx = %d", __func__, pmap, 408 pdir_idx); 409 410 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)), 411 ("ptbl_hold: invalid pdir_idx")); 412 KASSERT((pmap != kernel_pmap), 413 ("ptbl_hold: holding kernel ptbl!")); 414 415 ptbl = pmap->pm_pdir[pdir_idx]; 416 417 KASSERT((ptbl != NULL), ("ptbl_hold: null ptbl")); 418 419 for (i = 0; i < PTBL_PAGES; i++) { 420 pa = pte_vatopa(kernel_pmap, 421 (vm_offset_t)ptbl + (i * PAGE_SIZE)); 422 m = PHYS_TO_VM_PAGE(pa); 423 m->ref_count++; 424 } 425 } 426 427 /* 428 * Clean pte entry, try to free page table page if requested. 429 * 430 * Return 1 if ptbl pages were freed, otherwise return 0. 431 */ 432 static int 433 pte_remove(pmap_t pmap, vm_offset_t va, uint8_t flags) 434 { 435 unsigned int pdir_idx = PDIR_IDX(va); 436 unsigned int ptbl_idx = PTBL_IDX(va); 437 vm_page_t m; 438 pte_t *ptbl; 439 pte_t *pte; 440 441 //int su = (pmap == kernel_pmap); 442 //debugf("pte_remove: s (su = %d pmap = 0x%08x va = 0x%08x flags = %d)\n", 443 // su, (u_int32_t)pmap, va, flags); 444 445 ptbl = pmap->pm_pdir[pdir_idx]; 446 KASSERT(ptbl, ("pte_remove: null ptbl")); 447 448 pte = &ptbl[ptbl_idx]; 449 450 if (pte == NULL || !PTE_ISVALID(pte)) 451 return (0); 452 453 if (PTE_ISWIRED(pte)) 454 pmap->pm_stats.wired_count--; 455 456 /* Get vm_page_t for mapped pte. */ 457 m = PHYS_TO_VM_PAGE(PTE_PA(pte)); 458 459 /* Handle managed entry. */ 460 if (PTE_ISMANAGED(pte)) { 461 462 if (PTE_ISMODIFIED(pte)) 463 vm_page_dirty(m); 464 465 if (PTE_ISREFERENCED(pte)) 466 vm_page_aflag_set(m, PGA_REFERENCED); 467 468 pv_remove(pmap, va, m); 469 } else if (pmap == kernel_pmap && m && m->md.pv_tracked) { 470 /* 471 * Always pv_insert()/pv_remove() on MPC85XX, in case DPAA is 472 * used. This is needed by the NCSW support code for fast 473 * VA<->PA translation. 474 */ 475 pv_remove(pmap, va, m); 476 if (TAILQ_EMPTY(&m->md.pv_list)) 477 m->md.pv_tracked = false; 478 } 479 480 mtx_lock_spin(&tlbivax_mutex); 481 tlb_miss_lock(); 482 483 tlb0_flush_entry(va); 484 *pte = 0; 485 486 tlb_miss_unlock(); 487 mtx_unlock_spin(&tlbivax_mutex); 488 489 pmap->pm_stats.resident_count--; 490 491 if (flags & PTBL_UNHOLD) { 492 //debugf("pte_remove: e (unhold)\n"); 493 return (ptbl_unhold(pmap, pdir_idx)); 494 } 495 496 //debugf("pte_remove: e\n"); 497 return (0); 498 } 499 500 /* 501 * Insert PTE for a given page and virtual address. 502 */ 503 static int 504 pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags, 505 boolean_t nosleep) 506 { 507 unsigned int pdir_idx = PDIR_IDX(va); 508 unsigned int ptbl_idx = PTBL_IDX(va); 509 pte_t *ptbl, *pte, pte_tmp; 510 511 CTR4(KTR_PMAP, "%s: su = %d pmap = %p va = %p", __func__, 512 pmap == kernel_pmap, pmap, va); 513 514 /* Get the page table pointer. */ 515 ptbl = pmap->pm_pdir[pdir_idx]; 516 517 if (ptbl == NULL) { 518 /* Allocate page table pages. */ 519 ptbl = ptbl_alloc(pmap, pdir_idx, nosleep); 520 if (ptbl == NULL) { 521 KASSERT(nosleep, ("nosleep and NULL ptbl")); 522 return (ENOMEM); 523 } 524 pmap->pm_pdir[pdir_idx] = ptbl; 525 pte = &ptbl[ptbl_idx]; 526 } else { 527 /* 528 * Check if there is valid mapping for requested 529 * va, if there is, remove it. 530 */ 531 pte = &pmap->pm_pdir[pdir_idx][ptbl_idx]; 532 if (PTE_ISVALID(pte)) { 533 pte_remove(pmap, va, PTBL_HOLD); 534 } else { 535 /* 536 * pte is not used, increment hold count 537 * for ptbl pages. 538 */ 539 if (pmap != kernel_pmap) 540 ptbl_hold(pmap, pdir_idx); 541 } 542 } 543 544 /* 545 * Insert pv_entry into pv_list for mapped page if part of managed 546 * memory. 547 */ 548 if ((m->oflags & VPO_UNMANAGED) == 0) { 549 flags |= PTE_MANAGED; 550 551 /* Create and insert pv entry. */ 552 pv_insert(pmap, va, m); 553 } 554 555 pmap->pm_stats.resident_count++; 556 557 pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m)); 558 pte_tmp |= (PTE_VALID | flags | PTE_PS_4KB); /* 4KB pages only */ 559 560 mtx_lock_spin(&tlbivax_mutex); 561 tlb_miss_lock(); 562 563 tlb0_flush_entry(va); 564 *pte = pte_tmp; 565 566 tlb_miss_unlock(); 567 mtx_unlock_spin(&tlbivax_mutex); 568 return (0); 569 } 570 571 /* Return the pa for the given pmap/va. */ 572 static vm_paddr_t 573 pte_vatopa(pmap_t pmap, vm_offset_t va) 574 { 575 vm_paddr_t pa = 0; 576 pte_t *pte; 577 578 pte = pte_find(pmap, va); 579 if ((pte != NULL) && PTE_ISVALID(pte)) 580 pa = (PTE_PA(pte) | (va & PTE_PA_MASK)); 581 return (pa); 582 } 583 584 /* Get a pointer to a PTE in a page table. */ 585 static pte_t * 586 pte_find(pmap_t pmap, vm_offset_t va) 587 { 588 unsigned int pdir_idx = PDIR_IDX(va); 589 unsigned int ptbl_idx = PTBL_IDX(va); 590 591 KASSERT((pmap != NULL), ("pte_find: invalid pmap")); 592 593 if (pmap->pm_pdir[pdir_idx]) 594 return (&(pmap->pm_pdir[pdir_idx][ptbl_idx])); 595 596 return (NULL); 597 } 598 599 /* Get a pointer to a PTE in a page table, or the next closest (greater) one. */ 600 static __inline pte_t * 601 pte_find_next(pmap_t pmap, vm_offset_t *pva) 602 { 603 vm_offset_t va; 604 pte_t **pdir; 605 pte_t *pte; 606 unsigned long i, j; 607 608 KASSERT((pmap != NULL), ("pte_find: invalid pmap")); 609 610 va = *pva; 611 i = PDIR_IDX(va); 612 j = PTBL_IDX(va); 613 pdir = pmap->pm_pdir; 614 for (; i < PDIR_NENTRIES; i++, j = 0) { 615 if (pdir[i] == NULL) 616 continue; 617 for (; j < PTBL_NENTRIES; j++) { 618 pte = &pdir[i][j]; 619 if (!PTE_ISVALID(pte)) 620 continue; 621 *pva = PDIR_SIZE * i + PAGE_SIZE * j; 622 return (pte); 623 } 624 } 625 return (NULL); 626 } 627 628 /* Set up kernel page tables. */ 629 static void 630 kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr) 631 { 632 pte_t *pte; 633 vm_offset_t va; 634 vm_offset_t pdir_start; 635 int i; 636 637 kptbl_min = VM_MIN_KERNEL_ADDRESS / PDIR_SIZE; 638 kernel_pmap->pm_pdir = (pte_t **)kernel_ptbl_root; 639 640 pdir_start = kernel_ptbl_root + PDIR_NENTRIES * sizeof(pte_t); 641 642 /* Initialize kernel pdir */ 643 for (i = 0; i < kernel_ptbls; i++) { 644 kernel_pmap->pm_pdir[kptbl_min + i] = 645 (pte_t *)(pdir_start + (i * PAGE_SIZE * PTBL_PAGES)); 646 } 647 648 /* 649 * Fill in PTEs covering kernel code and data. They are not required 650 * for address translation, as this area is covered by static TLB1 651 * entries, but for pte_vatopa() to work correctly with kernel area 652 * addresses. 653 */ 654 for (va = addr; va < data_end; va += PAGE_SIZE) { 655 pte = &(kernel_pmap->pm_pdir[PDIR_IDX(va)][PTBL_IDX(va)]); 656 powerpc_sync(); 657 *pte = PTE_RPN_FROM_PA(kernload + (va - kernstart)); 658 *pte |= PTE_M | PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | 659 PTE_VALID | PTE_PS_4KB; 660 } 661 } 662 663 static vm_offset_t 664 mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end) 665 { 666 /* Allocate space for ptbl_bufs. */ 667 ptbl_bufs = (struct ptbl_buf *)data_end; 668 data_end += sizeof(struct ptbl_buf) * PTBL_BUFS; 669 debugf(" ptbl_bufs at 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n", 670 (uintptr_t)ptbl_bufs, data_end); 671 672 data_end = round_page(data_end); 673 674 kernel_ptbl_root = data_end; 675 data_end += PDIR_NENTRIES * sizeof(pte_t*); 676 677 /* Allocate PTE tables for kernel KVA. */ 678 kernel_ptbls = howmany(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, 679 PDIR_SIZE); 680 data_end += kernel_ptbls * PTBL_PAGES * PAGE_SIZE; 681 debugf(" kernel ptbls: %d\n", kernel_ptbls); 682 debugf(" kernel pdir at %#jx end = %#jx\n", 683 (uintmax_t)kernel_ptbl_root, (uintmax_t)data_end); 684 685 return (data_end); 686 } 687 688 /* 689 * Initialize a preallocated and zeroed pmap structure, 690 * such as one in a vmspace structure. 691 */ 692 static int 693 mmu_booke_pinit(pmap_t pmap) 694 { 695 int i; 696 697 CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap, 698 curthread->td_proc->p_pid, curthread->td_proc->p_comm); 699 700 KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap")); 701 702 for (i = 0; i < MAXCPU; i++) 703 pmap->pm_tid[i] = TID_NONE; 704 CPU_ZERO(&kernel_pmap->pm_active); 705 bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); 706 pmap->pm_pdir = uma_zalloc(ptbl_root_zone, M_WAITOK); 707 bzero(pmap->pm_pdir, sizeof(pte_t *) * PDIR_NENTRIES); 708 TAILQ_INIT(&pmap->pm_ptbl_list); 709 710 return (1); 711 } 712 713 /* 714 * Release any resources held by the given physical map. 715 * Called when a pmap initialized by mmu_booke_pinit is being released. 716 * Should only be called if the map contains no valid mappings. 717 */ 718 static void 719 mmu_booke_release(pmap_t pmap) 720 { 721 722 KASSERT(pmap->pm_stats.resident_count == 0, 723 ("pmap_release: pmap resident count %ld != 0", 724 pmap->pm_stats.resident_count)); 725 uma_zfree(ptbl_root_zone, pmap->pm_pdir); 726 } 727 728 static void 729 mmu_booke_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 730 { 731 pte_t *pte; 732 vm_paddr_t pa = 0; 733 int sync_sz, valid; 734 pmap_t pmap; 735 vm_page_t m; 736 vm_offset_t addr; 737 int active; 738 739 rw_wlock(&pvh_global_lock); 740 pmap = PCPU_GET(curpmap); 741 active = (pm == kernel_pmap || pm == pmap) ? 1 : 0; 742 while (sz > 0) { 743 PMAP_LOCK(pm); 744 pte = pte_find(pm, va); 745 valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0; 746 if (valid) 747 pa = PTE_PA(pte); 748 PMAP_UNLOCK(pm); 749 sync_sz = PAGE_SIZE - (va & PAGE_MASK); 750 sync_sz = min(sync_sz, sz); 751 if (valid) { 752 if (!active) { 753 /* Create a mapping in the active pmap. */ 754 addr = 0; 755 m = PHYS_TO_VM_PAGE(pa); 756 PMAP_LOCK(pmap); 757 pte_enter(pmap, m, addr, 758 PTE_SR | PTE_VALID, FALSE); 759 addr += (va & PAGE_MASK); 760 __syncicache((void *)addr, sync_sz); 761 pte_remove(pmap, addr, PTBL_UNHOLD); 762 PMAP_UNLOCK(pmap); 763 } else 764 __syncicache((void *)va, sync_sz); 765 } 766 va += sync_sz; 767 sz -= sync_sz; 768 } 769 rw_wunlock(&pvh_global_lock); 770 } 771 772 /* 773 * mmu_booke_zero_page_area zeros the specified hardware page by 774 * mapping it into virtual memory and using bzero to clear 775 * its contents. 776 * 777 * off and size must reside within a single page. 778 */ 779 static void 780 mmu_booke_zero_page_area(vm_page_t m, int off, int size) 781 { 782 vm_offset_t va; 783 784 /* XXX KASSERT off and size are within a single page? */ 785 786 mtx_lock(&zero_page_mutex); 787 va = zero_page_va; 788 789 mmu_booke_kenter(va, VM_PAGE_TO_PHYS(m)); 790 bzero((caddr_t)va + off, size); 791 mmu_booke_kremove(va); 792 793 mtx_unlock(&zero_page_mutex); 794 } 795 796 /* 797 * mmu_booke_zero_page zeros the specified hardware page. 798 */ 799 static void 800 mmu_booke_zero_page(vm_page_t m) 801 { 802 vm_offset_t off, va; 803 804 va = zero_page_va; 805 mtx_lock(&zero_page_mutex); 806 807 mmu_booke_kenter(va, VM_PAGE_TO_PHYS(m)); 808 809 for (off = 0; off < PAGE_SIZE; off += cacheline_size) 810 __asm __volatile("dcbz 0,%0" :: "r"(va + off)); 811 812 mmu_booke_kremove(va); 813 814 mtx_unlock(&zero_page_mutex); 815 } 816 817 /* 818 * mmu_booke_copy_page copies the specified (machine independent) page by 819 * mapping the page into virtual memory and using memcopy to copy the page, 820 * one machine dependent page at a time. 821 */ 822 static void 823 mmu_booke_copy_page(vm_page_t sm, vm_page_t dm) 824 { 825 vm_offset_t sva, dva; 826 827 sva = copy_page_src_va; 828 dva = copy_page_dst_va; 829 830 mtx_lock(©_page_mutex); 831 mmu_booke_kenter(sva, VM_PAGE_TO_PHYS(sm)); 832 mmu_booke_kenter(dva, VM_PAGE_TO_PHYS(dm)); 833 834 memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE); 835 836 mmu_booke_kremove(dva); 837 mmu_booke_kremove(sva); 838 mtx_unlock(©_page_mutex); 839 } 840 841 static inline void 842 mmu_booke_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 843 vm_page_t *mb, vm_offset_t b_offset, int xfersize) 844 { 845 void *a_cp, *b_cp; 846 vm_offset_t a_pg_offset, b_pg_offset; 847 int cnt; 848 849 mtx_lock(©_page_mutex); 850 while (xfersize > 0) { 851 a_pg_offset = a_offset & PAGE_MASK; 852 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 853 mmu_booke_kenter(copy_page_src_va, 854 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])); 855 a_cp = (char *)copy_page_src_va + a_pg_offset; 856 b_pg_offset = b_offset & PAGE_MASK; 857 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 858 mmu_booke_kenter(copy_page_dst_va, 859 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])); 860 b_cp = (char *)copy_page_dst_va + b_pg_offset; 861 bcopy(a_cp, b_cp, cnt); 862 mmu_booke_kremove(copy_page_dst_va); 863 mmu_booke_kremove(copy_page_src_va); 864 a_offset += cnt; 865 b_offset += cnt; 866 xfersize -= cnt; 867 } 868 mtx_unlock(©_page_mutex); 869 } 870 871 static vm_offset_t 872 mmu_booke_quick_enter_page(vm_page_t m) 873 { 874 vm_paddr_t paddr; 875 vm_offset_t qaddr; 876 uint32_t flags; 877 pte_t *pte; 878 879 paddr = VM_PAGE_TO_PHYS(m); 880 881 flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID; 882 flags |= tlb_calc_wimg(paddr, pmap_page_get_memattr(m)) << PTE_MAS2_SHIFT; 883 flags |= PTE_PS_4KB; 884 885 critical_enter(); 886 qaddr = PCPU_GET(qmap_addr); 887 888 pte = pte_find(kernel_pmap, qaddr); 889 890 KASSERT(*pte == 0, ("mmu_booke_quick_enter_page: PTE busy")); 891 892 /* 893 * XXX: tlbivax is broadcast to other cores, but qaddr should 894 * not be present in other TLBs. Is there a better instruction 895 * sequence to use? Or just forget it & use mmu_booke_kenter()... 896 */ 897 __asm __volatile("tlbivax 0, %0" :: "r"(qaddr & MAS2_EPN_MASK)); 898 __asm __volatile("isync; msync"); 899 900 *pte = PTE_RPN_FROM_PA(paddr) | flags; 901 902 /* Flush the real memory from the instruction cache. */ 903 if ((flags & (PTE_I | PTE_G)) == 0) 904 __syncicache((void *)qaddr, PAGE_SIZE); 905 906 return (qaddr); 907 } 908 909 static void 910 mmu_booke_quick_remove_page(vm_offset_t addr) 911 { 912 pte_t *pte; 913 914 pte = pte_find(kernel_pmap, addr); 915 916 KASSERT(PCPU_GET(qmap_addr) == addr, 917 ("mmu_booke_quick_remove_page: invalid address")); 918 KASSERT(*pte != 0, 919 ("mmu_booke_quick_remove_page: PTE not in use")); 920 921 *pte = 0; 922 critical_exit(); 923 } 924 925 /**************************************************************************/ 926 /* TID handling */ 927 /**************************************************************************/ 928 929 /* 930 * Return the largest uint value log such that 2^log <= num. 931 */ 932 static unsigned long 933 ilog2(unsigned long num) 934 { 935 long lz; 936 937 __asm ("cntlzw %0, %1" : "=r" (lz) : "r" (num)); 938 return (31 - lz); 939 } 940 941 /* 942 * Invalidate all TLB0 entries which match the given TID. Note this is 943 * dedicated for cases when invalidations should NOT be propagated to other 944 * CPUs. 945 */ 946 static void 947 tid_flush(tlbtid_t tid) 948 { 949 register_t msr; 950 uint32_t mas0, mas1, mas2; 951 int entry, way; 952 953 954 /* Don't evict kernel translations */ 955 if (tid == TID_KERNEL) 956 return; 957 958 msr = mfmsr(); 959 __asm __volatile("wrteei 0"); 960 961 /* 962 * Newer (e500mc and later) have tlbilx, which doesn't broadcast, so use 963 * it for PID invalidation. 964 */ 965 switch ((mfpvr() >> 16) & 0xffff) { 966 case FSL_E500mc: 967 case FSL_E5500: 968 case FSL_E6500: 969 mtspr(SPR_MAS6, tid << MAS6_SPID0_SHIFT); 970 /* tlbilxpid */ 971 __asm __volatile("isync; .long 0x7c200024; isync; msync"); 972 __asm __volatile("wrtee %0" :: "r"(msr)); 973 return; 974 } 975 976 for (way = 0; way < TLB0_WAYS; way++) 977 for (entry = 0; entry < TLB0_ENTRIES_PER_WAY; entry++) { 978 979 mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way); 980 mtspr(SPR_MAS0, mas0); 981 982 mas2 = entry << MAS2_TLB0_ENTRY_IDX_SHIFT; 983 mtspr(SPR_MAS2, mas2); 984 985 __asm __volatile("isync; tlbre"); 986 987 mas1 = mfspr(SPR_MAS1); 988 989 if (!(mas1 & MAS1_VALID)) 990 continue; 991 if (((mas1 & MAS1_TID_MASK) >> MAS1_TID_SHIFT) != tid) 992 continue; 993 mas1 &= ~MAS1_VALID; 994 mtspr(SPR_MAS1, mas1); 995 __asm __volatile("isync; tlbwe; isync; msync"); 996 } 997 __asm __volatile("wrtee %0" :: "r"(msr)); 998 } 999