1 /* $NetBSD: pmap.c,v 1.130 2002/11/11 10:15:46 chs Exp $ */ 2 #undef NO_VCACHE /* Don't forget the locked TLB in dostart */ 3 #define HWREF 4 /* 5 * 6 * Copyright (C) 1996-1999 Eduardo Horvath. 7 * All rights reserved. 8 * 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30 #include "opt_ddb.h" 31 32 #include <sys/param.h> 33 #include <sys/malloc.h> 34 #include <sys/queue.h> 35 #include <sys/systm.h> 36 #include <sys/msgbuf.h> 37 #include <sys/lock.h> 38 #include <sys/pool.h> 39 #include <sys/exec.h> 40 #include <sys/core.h> 41 #include <sys/kcore.h> 42 #include <sys/proc.h> 43 44 #include <uvm/uvm.h> 45 46 #include <machine/pcb.h> 47 #include <machine/sparc64.h> 48 #include <machine/ctlreg.h> 49 #include <machine/openfirm.h> 50 #include <machine/kcore.h> 51 52 #include "cache.h" 53 54 #ifdef DDB 55 #include <machine/db_machdep.h> 56 #include <ddb/db_command.h> 57 #include <ddb/db_sym.h> 58 #include <ddb/db_variables.h> 59 #include <ddb/db_extern.h> 60 #include <ddb/db_access.h> 61 #include <ddb/db_output.h> 62 #else 63 #define Debugger() 64 #define db_printf printf 65 #endif 66 67 #define MEG (1<<20) /* 1MB */ 68 #define KB (1<<10) /* 1KB */ 69 70 paddr_t cpu0paddr;/* XXXXXXXXXXXXXXXX */ 71 72 /* These routines are in assembly to allow access thru physical mappings */ 73 extern int64_t pseg_get __P((struct pmap *, vaddr_t)); 74 extern int pseg_set __P((struct pmap *, vaddr_t, int64_t, paddr_t)); 75 76 /* 77 * Diatribe on ref/mod counting: 78 * 79 * First of all, ref/mod info must be non-volatile. Hence we need to keep it 80 * in the pv_entry structure for each page. (We could bypass this for the 81 * vm_page, but that's a long story....) 82 * 83 * This architecture has nice, fast traps with lots of space for software bits 84 * in the TTE. To accelerate ref/mod counts we make use of these features. 85 * 86 * When we map a page initially, we place a TTE in the page table. It's 87 * inserted with the TLB_W and TLB_ACCESS bits cleared. If a page is really 88 * writeable we set the TLB_REAL_W bit for the trap handler. 89 * 90 * Whenever we take a TLB miss trap, the trap handler will set the TLB_ACCESS 91 * bit in the approprate TTE in the page table. Whenever we take a protection 92 * fault, if the TLB_REAL_W bit is set then we flip both the TLB_W and TLB_MOD 93 * bits to enable writing and mark the page as modified. 94 * 95 * This means that we may have ref/mod information all over the place. The 96 * pmap routines must traverse the page tables of all pmaps with a given page 97 * and collect/clear all the ref/mod information and copy it into the pv_entry. 98 */ 99 100 #ifdef NO_VCACHE 101 #define FORCE_ALIAS 1 102 #else 103 #define FORCE_ALIAS 0 104 #endif 105 106 #define PV_ALIAS 0x1LL 107 #define PV_REF 0x2LL 108 #define PV_MOD 0x4LL 109 #define PV_NVC 0x8LL 110 #define PV_NC 0x10LL 111 #define PV_WE 0x20LL /* Debug -- this page was writable somtime */ 112 #define PV_MASK (0x03fLL) 113 #define PV_VAMASK (~(NBPG - 1)) 114 #define PV_MATCH(pv,va) (!(((pv)->pv_va ^ (va)) & PV_VAMASK)) 115 #define PV_SETVA(pv,va) ((pv)->pv_va = (((va) & PV_VAMASK) | \ 116 (((pv)->pv_va) & PV_MASK))) 117 118 struct pool pmap_pmap_pool; 119 struct pool pmap_pv_pool; 120 121 pv_entry_t pmap_remove_pv __P((struct pmap *, vaddr_t, struct vm_page *)); 122 void pmap_enter_pv __P((struct pmap *, vaddr_t, paddr_t, struct vm_page *, 123 pv_entry_t)); 124 void pmap_page_cache __P((struct pmap *, paddr_t, int)); 125 126 /* 127 * First and last managed physical addresses. 128 * XXX only used for dumping the system. 129 */ 130 paddr_t vm_first_phys, vm_num_phys; 131 132 u_int64_t first_phys_addr; 133 #define pa_index(pa) atop((pa) - first_phys_addr) 134 135 /* 136 * Here's the CPU TSB stuff. It's allocated in pmap_bootstrap. 137 */ 138 pte_t *tsb; 139 int tsbsize; /* tsbents = 512 * 2^^tsbsize */ 140 #define TSBENTS (512<<tsbsize) 141 #define TSBSIZE (TSBENTS * 16) 142 143 struct pmap kernel_pmap_; 144 145 int physmem; 146 /* 147 * Virtual and physical addresses of the start and end of kernel text 148 * and data segments. 149 */ 150 vaddr_t ktext; 151 paddr_t ktextp; 152 vaddr_t ektext; 153 paddr_t ektextp; 154 vaddr_t kdata; 155 paddr_t kdatap; 156 vaddr_t ekdata; 157 paddr_t ekdatap; 158 159 static int npgs; 160 static u_int nextavail; 161 static struct mem_region memlist[8]; /* Pick a random size here */ 162 163 vaddr_t vmmap; /* one reserved MI vpage for /dev/mem */ 164 165 struct mem_region *mem, *avail, *orig; 166 int memsize; 167 168 static int memh = 0, vmemh = 0; /* Handles to OBP devices */ 169 170 paddr_t avail_start, avail_end; /* These are used by ps & family */ 171 172 static int ptelookup_va __P((vaddr_t va)); /* sun4u */ 173 #if notyet 174 static void tsb_enter __P((int ctx, int64_t va, int64_t data)); 175 #endif 176 177 struct pmap_stats { 178 int ps_unlink_pvfirst; /* # of pv_unlinks on head */ 179 int ps_unlink_pvsearch; /* # of pv_unlink searches */ 180 int ps_changeprots; /* # of calls to changeprot */ 181 int ps_useless_changeprots; /* # of changeprots for wiring */ 182 int ps_enter_firstpv; /* pv heads entered */ 183 int ps_enter_secondpv; /* pv nonheads entered */ 184 int ps_useless_changewire; /* useless wiring changes */ 185 int ps_npg_prot_all; /* # of active pages protected */ 186 int ps_npg_prot_actual; /* # pages actually affected */ 187 } pmap_stats; 188 189 struct prom_map *prom_map; 190 int prom_map_size; 191 192 #ifdef DEBUG 193 struct { 194 int kernel; /* entering kernel mapping */ 195 int user; /* entering user mapping */ 196 int ptpneeded; /* needed to allocate a PT page */ 197 int pwchange; /* no mapping change, just wiring or protection */ 198 int wchange; /* no mapping change, just wiring */ 199 int mchange; /* was mapped but mapping to different page */ 200 int managed; /* a managed page */ 201 int firstpv; /* first mapping for this PA */ 202 int secondpv; /* second mapping for this PA */ 203 int ci; /* cache inhibited */ 204 int unmanaged; /* not a managed page */ 205 int flushes; /* cache flushes */ 206 int cachehit; /* new entry forced valid entry out */ 207 } enter_stats; 208 struct { 209 int calls; 210 int removes; 211 int flushes; 212 int tflushes; /* TLB flushes */ 213 int pidflushes; /* HW pid stolen */ 214 int pvfirst; 215 int pvsearch; 216 } remove_stats; 217 #define ENTER_STAT(x) enter_stats.x ++ 218 #define REMOVE_STAT(x) remove_stats.x ++ 219 220 #define PDB_CREATE 0x0001 221 #define PDB_DESTROY 0x0002 222 #define PDB_REMOVE 0x0004 223 #define PDB_CHANGEPROT 0x0008 224 #define PDB_ENTER 0x0010 225 #define PDB_DEMAP 0x0020 226 #define PDB_REF 0x0040 227 #define PDB_COPY 0x0080 228 229 #define PDB_MMU_ALLOC 0x0100 230 #define PDB_MMU_STEAL 0x0200 231 #define PDB_CTX_ALLOC 0x0400 232 #define PDB_CTX_STEAL 0x0800 233 #define PDB_MMUREG_ALLOC 0x1000 234 #define PDB_MMUREG_STEAL 0x2000 235 #define PDB_CACHESTUFF 0x4000 236 #define PDB_ALIAS 0x8000 237 #define PDB_EXTRACT 0x10000 238 #define PDB_BOOT 0x20000 239 #define PDB_BOOT1 0x40000 240 #define PDB_GROW 0x80000 241 int pmapdebug = 0; 242 /* Number of H/W pages stolen for page tables */ 243 int pmap_pages_stolen = 0; 244 245 #define BDPRINTF(n, f) if (pmapdebug & (n)) prom_printf f 246 #define DPRINTF(n, f) if (pmapdebug & (n)) printf f 247 #else 248 #define ENTER_STAT(x) 249 #define REMOVE_STAT(x) 250 #define BDPRINTF(n, f) 251 #define DPRINTF(n, f) 252 #endif 253 254 #ifdef NOTDEF_DEBUG 255 void pv_check __P((void)); 256 void 257 pv_check() 258 { 259 int i, j, s; 260 261 s = splhigh(); 262 for (i = 0; i < physmem; i++) { 263 struct pv_entry *pv; 264 for (pv = &pv_table[i]; pv; pv = pv->pv_next) { 265 if (pv->pv_pmap && 266 !(pseg_get(pv->pv_pmap, pv->pv_va) & TLB_V)) { 267 printf("pv_check(): unreferenced pv=%p pa=%p va=%p pm=%p\n", 268 i, ptoa(first_phys_addr+i), pv->pv_va, pv->pv_pmap); 269 Debugger(); 270 } 271 } 272 } 273 splx(s); 274 } 275 #else 276 #define pv_check() 277 #endif 278 279 /* 280 * 281 * A context is simply a small number that differentiates multiple mappings 282 * of the same address. Contexts on the spitfire are 13 bits, but could 283 * be as large as 17 bits. 284 * 285 * Each context is either free or attached to a pmap. 286 * 287 * The context table is an array of pointers to psegs. Just dereference 288 * the right pointer and you get to the pmap segment tables. These are 289 * physical addresses, of course. 290 * 291 */ 292 293 int pmap_next_ctx = 1; 294 paddr_t *ctxbusy; 295 LIST_HEAD(, pmap) pmap_ctxlist; 296 int numctx; 297 #define CTXENTRY (sizeof(paddr_t)) 298 #define CTXSIZE (numctx*CTXENTRY) 299 300 static int pmap_get_page(paddr_t *p); 301 static void pmap_free_page(paddr_t pa); 302 303 304 /* 305 * Support for big page sizes. This maps the page size to the 306 * page bits. That is: these are the bits between 8K pages and 307 * larger page sizes that cause aliasing. 308 */ 309 struct page_size_map page_size_map[] = { 310 #ifdef DEBUG 311 { 0, PGSZ_8K & 0 }, /* Disable large pages */ 312 #endif 313 { (4 * 1024 * 1024 - 1) & ~(8 * 1024 - 1), PGSZ_4M }, 314 { (512 * 1024 - 1) & ~(8 * 1024 - 1), PGSZ_512K }, 315 { (64 * 1024 - 1) & ~(8 * 1024 - 1), PGSZ_64K }, 316 { (8 * 1024 - 1) & ~(8 * 1024 - 1), PGSZ_8K }, 317 { 0, 0 } 318 }; 319 320 /* 321 * Calculate the largest page size that will map this. 322 * 323 * You really need to do this both on VA and PA. 324 */ 325 #define PMAP_PAGE_SIZE(va, pa, len, pgsz, pglen) \ 326 do { \ 327 for ((pgsz) = PGSZ_4M; (pgsz); (pgsz)--) { \ 328 (pglen) = PG_SZ(pgsz); \ 329 \ 330 if (((len) >= (pgsz)) && \ 331 ((pa) & ((pglen) - 1) & ~PG_SZ(PGSZ_8K)) == 0 &&\ 332 ((va) & ((pglen) - 1) & ~PG_SZ(PGSZ_8K)) == 0) \ 333 break; \ 334 } \ 335 (pgsz) = 0; \ 336 (pglen) = PG_SZ(pgsz); \ 337 } while (0) 338 339 340 /* 341 * Enter a TTE into the kernel pmap only. Don't do anything else. 342 * 343 * Use only during bootstrapping since it does no locking and 344 * can lose ref/mod info!!!! 345 * 346 */ 347 static void pmap_enter_kpage __P((vaddr_t, int64_t)); 348 static void 349 pmap_enter_kpage(va, data) 350 vaddr_t va; 351 int64_t data; 352 { 353 paddr_t newp; 354 355 newp = NULL; 356 while (pseg_set(pmap_kernel(), va, data, newp) & 1) { 357 newp = NULL; 358 pmap_get_page(&newp); 359 if (!newp) { 360 prom_printf("pmap_enter_kpage: out of pages\n"); 361 panic("pmap_enter_kpage"); 362 } 363 364 ENTER_STAT(ptpneeded); 365 BDPRINTF(PDB_BOOT1, 366 ("pseg_set: pm=%p va=%p data=%lx newp %lx\r\n", 367 pmap_kernel(), va, (long)data, (long)newp)); 368 #ifdef DEBUG 369 if (pmapdebug & PDB_BOOT1) 370 {int i; for (i=0; i<140000000; i++) ;} 371 #endif 372 } 373 } 374 375 /* 376 * Check the bootargs to see if we need to enable bootdebug. 377 */ 378 #ifdef DEBUG 379 void pmap_bootdebug __P((void)); 380 void 381 pmap_bootdebug() 382 { 383 int chosen; 384 char *cp; 385 char buf[128]; 386 387 /* 388 * Grab boot args from PROM 389 */ 390 chosen = OF_finddevice("/chosen"); 391 /* Setup pointer to boot flags */ 392 OF_getprop(chosen, "bootargs", buf, sizeof(buf)); 393 cp = buf; 394 while (*cp != '-') 395 if (*cp++ == '\0') 396 return; 397 for (;;) 398 switch (*++cp) { 399 case '\0': 400 return; 401 case 'V': 402 pmapdebug |= PDB_BOOT|PDB_BOOT1; 403 break; 404 case 'D': 405 pmapdebug |= PDB_BOOT1; 406 break; 407 } 408 } 409 #endif 410 411 412 /* 413 * Calculate the correct number of page colors to use. This should be the 414 * size of the E$/NBPG. However, different CPUs can have different sized 415 * E$, so we need to take the GCM of the E$ size. 416 */ 417 static int pmap_calculate_colors __P((void)); 418 static int 419 pmap_calculate_colors() { 420 int node = 0; 421 int size, assoc, color, maxcolor = 1; 422 char buf[80]; 423 424 while ((node = OF_peer(node))) { 425 if ((OF_getprop(node, "device_type", buf, sizeof(buf)) > 0) && 426 strcmp("cpu", buf) == 0) { 427 /* Found a CPU, get the E$ info. */ 428 if (OF_getprop(node,"ecache-size", &size, 429 sizeof(size)) != sizeof(size)) { 430 printf("pmap_calculate_colors: node %x has " 431 "no ecache-size\n", node); 432 /* If we can't get the E$ size, skip the node */ 433 continue; 434 } 435 if (OF_getprop(node, "ecache-associativity", &assoc, 436 sizeof(assoc)) != sizeof(assoc)) 437 /* Fake asociativity of 1 */ 438 assoc = 1; 439 color = size/assoc/NBPG; 440 if (color > maxcolor) 441 maxcolor = color; 442 } 443 } 444 return (maxcolor); 445 } 446 447 /* 448 * This is called during bootstrap, before the system is really initialized. 449 * 450 * It's called with the start and end virtual addresses of the kernel. We 451 * bootstrap the pmap allocator now. We will allocate the basic structures we 452 * need to bootstrap the VM system here: the page frame tables, the TSB, and 453 * the free memory lists. 454 * 455 * Now all this is becoming a bit obsolete. maxctx is still important, but by 456 * separating the kernel text and data segments we really would need to 457 * provide the start and end of each segment. But we can't. The rodata 458 * segment is attached to the end of the kernel segment and has nothing to 459 * delimit its end. We could still pass in the beginning of the kernel and 460 * the beginning and end of the data segment but we could also just as easily 461 * calculate that all in here. 462 * 463 * To handle the kernel text, we need to do a reverse mapping of the start of 464 * the kernel, then traverse the free memory lists to find out how big it is. 465 */ 466 467 void 468 pmap_bootstrap(kernelstart, kernelend, maxctx) 469 u_long kernelstart, kernelend; 470 u_int maxctx; 471 { 472 extern int data_start[], end[]; /* start of data segment */ 473 extern int msgbufmapped; 474 struct mem_region *mp, *mp1; 475 int msgbufsiz; 476 int pcnt; 477 size_t s, sz; 478 int i, j; 479 int64_t data; 480 vaddr_t va; 481 u_int64_t phys_msgbuf; 482 paddr_t newp, newkp; 483 vaddr_t newkv, firstaddr, intstk; 484 vsize_t kdsize, ktsize; 485 486 #ifdef DEBUG 487 pmap_bootdebug(); 488 #endif 489 490 BDPRINTF(PDB_BOOT, ("Entered pmap_bootstrap.\r\n")); 491 /* 492 * set machine page size 493 */ 494 uvmexp.pagesize = NBPG; 495 uvmexp.ncolors = pmap_calculate_colors(); 496 uvm_setpagesize(); 497 498 /* 499 * Find out how big the kernel's virtual address 500 * space is. The *$#@$ prom loses this info 501 */ 502 if ((vmemh = OF_finddevice("/virtual-memory")) == -1) { 503 prom_printf("no virtual-memory?"); 504 OF_exit(); 505 } 506 bzero((caddr_t)memlist, sizeof(memlist)); 507 if (OF_getprop(vmemh, "available", memlist, sizeof(memlist)) <= 0) { 508 prom_printf("no vmemory avail?"); 509 OF_exit(); 510 } 511 512 #ifdef DEBUG 513 if (pmapdebug & PDB_BOOT) { 514 /* print out mem list */ 515 prom_printf("Available virtual memory:\r\n"); 516 for (mp = memlist; mp->size; mp++) { 517 prom_printf("memlist start %p size %lx\r\n", 518 (void *)(u_long)mp->start, 519 (u_long)mp->size); 520 } 521 prom_printf("End of available virtual memory\r\n"); 522 } 523 #endif 524 /* 525 * Get hold or the message buffer. 526 */ 527 msgbufp = (struct kern_msgbuf *)(vaddr_t)MSGBUF_VA; 528 /* XXXXX -- increase msgbufsiz for uvmhist printing */ 529 msgbufsiz = 4*NBPG /* round_page(sizeof(struct msgbuf)) */; 530 BDPRINTF(PDB_BOOT, ("Trying to allocate msgbuf at %lx, size %lx\r\n", 531 (long)msgbufp, (long)msgbufsiz)); 532 if ((long)msgbufp != 533 (long)(phys_msgbuf = prom_claim_virt((vaddr_t)msgbufp, msgbufsiz))) 534 prom_printf( 535 "cannot get msgbuf VA, msgbufp=%p, phys_msgbuf=%lx\r\n", 536 (void *)msgbufp, (long)phys_msgbuf); 537 phys_msgbuf = prom_get_msgbuf(msgbufsiz, MMU_PAGE_ALIGN); 538 BDPRINTF(PDB_BOOT, 539 ("We should have the memory at %lx, let's map it in\r\n", 540 phys_msgbuf)); 541 if (prom_map_phys(phys_msgbuf, msgbufsiz, (vaddr_t)msgbufp, 542 -1/* sunos does this */) == -1) 543 prom_printf("Failed to map msgbuf\r\n"); 544 else 545 BDPRINTF(PDB_BOOT, ("msgbuf mapped at %p\r\n", 546 (void *)msgbufp)); 547 msgbufmapped = 1; /* enable message buffer */ 548 initmsgbuf((caddr_t)msgbufp, msgbufsiz); 549 550 /* 551 * Record kernel mapping -- we will map these with a permanent 4MB 552 * TLB entry when we initialize the CPU later. 553 */ 554 BDPRINTF(PDB_BOOT, ("translating kernelstart %p\r\n", 555 (void *)kernelstart)); 556 ktext = kernelstart; 557 ktextp = prom_vtop(kernelstart); 558 559 kdata = (vaddr_t)data_start; 560 kdatap = prom_vtop(kdata); 561 ekdata = (vaddr_t)end; 562 563 /* 564 * Find the real size of the kernel. Locate the smallest starting 565 * address > kernelstart. 566 */ 567 for (mp1 = mp = memlist; mp->size; mp++) { 568 /* 569 * Check whether this region is at the end of the kernel. 570 */ 571 if (mp->start >= ekdata && (mp1->start < ekdata || 572 mp1->start > mp->start)) 573 mp1 = mp; 574 } 575 if (mp1->start < kdata) 576 prom_printf("Kernel at end of vmem???\r\n"); 577 578 BDPRINTF(PDB_BOOT1, 579 ("Kernel data is mapped at %lx, next free seg: %lx, %lx\r\n", 580 (long)kdata, (u_long)mp1->start, (u_long)mp1->size)); 581 582 /* 583 * We save where we can start allocating memory. 584 */ 585 firstaddr = (ekdata + 07) & ~ 07; /* Longword align */ 586 587 /* 588 * We reserve 100K to grow. 589 */ 590 ekdata += 100*KB; 591 592 /* 593 * And set the end of the data segment to the end of what our 594 * bootloader allocated for us, if we still fit in there. 595 */ 596 if (ekdata < mp1->start) 597 ekdata = mp1->start; 598 599 #if 1 600 #define valloc(name, type, num) (name) = (type *)firstaddr; firstaddr += (num) 601 #else 602 #define valloc(name, type, num) (name) = (type *)firstaddr; firstaddr = \ 603 (vaddr_t)((name)+(num)) 604 #endif 605 606 /* 607 * Since we can't always give the loader the hint to align us on a 4MB 608 * boundary, we will need to do the alignment ourselves. First 609 * allocate a new 4MB aligned segment for the kernel, then map it 610 * in, copy the kernel over, swap mappings, then finally, free the 611 * old kernel. Then we can continue with this. 612 * 613 * We'll do the data segment up here since we know how big it is. 614 * We'll do the text segment after we've read in the PROM translations 615 * so we can figure out its size. 616 * 617 * The ctxbusy table takes about 64KB, the TSB up to 32KB, and the 618 * rest should be less than 1K, so 100KB extra should be plenty. 619 */ 620 kdsize = round_page(ekdata - kdata); 621 BDPRINTF(PDB_BOOT1, ("Kernel data size is %lx\r\n", (long)kdsize)); 622 623 if ((kdatap & (4*MEG-1)) == 0) { 624 /* We were at a 4MB boundary -- claim the rest */ 625 psize_t szdiff = (4*MEG - kdsize) & (4*MEG - 1); 626 627 BDPRINTF(PDB_BOOT1, ("Need to extend dseg by %lx\r\n", 628 (long)szdiff)); 629 if (szdiff) { 630 /* Claim the rest of the physical page. */ 631 newkp = kdatap + kdsize; 632 newkv = kdata + kdsize; 633 if (newkp != prom_claim_phys(newkp, szdiff)) { 634 prom_printf("pmap_bootstrap: could not claim " 635 "physical dseg extension " 636 "at %lx size %lx\r\n", 637 newkp, szdiff); 638 goto remap_data; 639 } 640 641 /* And the rest of the virtual page. */ 642 if (prom_claim_virt(newkv, szdiff) != newkv) 643 prom_printf("pmap_bootstrap: could not claim " 644 "virtual dseg extension " 645 "at size %lx\r\n", newkv, szdiff); 646 647 /* Make sure all 4MB are mapped */ 648 prom_map_phys(newkp, szdiff, newkv, -1); 649 } 650 } else { 651 psize_t sz; 652 remap_data: 653 /* 654 * Either we're not at a 4MB boundary or we can't get the rest 655 * of the 4MB extension. We need to move the data segment. 656 * Leave 1MB of extra fiddle space in the calculations. 657 */ 658 659 sz = (kdsize + 4*MEG - 1) & ~(4*MEG-1); 660 BDPRINTF(PDB_BOOT1, 661 ("Allocating new %lx kernel data at 4MB boundary\r\n", 662 (u_long)sz)); 663 if ((newkp = prom_alloc_phys(sz, 4*MEG)) == (paddr_t)-1 ) { 664 prom_printf("Cannot allocate new kernel\r\n"); 665 OF_exit(); 666 } 667 BDPRINTF(PDB_BOOT1, ("Allocating new va for buffer at %llx\r\n", 668 (u_int64_t)newkp)); 669 if ((newkv = (vaddr_t)prom_alloc_virt(sz, 8)) == 670 (vaddr_t)-1) { 671 prom_printf("Cannot allocate new kernel va\r\n"); 672 OF_exit(); 673 } 674 BDPRINTF(PDB_BOOT1, ("Mapping in buffer %llx at %llx\r\n", 675 (u_int64_t)newkp, (u_int64_t)newkv)); 676 prom_map_phys(newkp, sz, (vaddr_t)newkv, -1); 677 BDPRINTF(PDB_BOOT1, ("Copying %ld bytes kernel data...", 678 kdsize)); 679 bzero((void *)newkv, sz); 680 bcopy((void *)kdata, (void *)newkv, kdsize); 681 BDPRINTF(PDB_BOOT1, ("done. Swapping maps..unmap new\r\n")); 682 prom_unmap_virt((vaddr_t)newkv, sz); 683 BDPRINTF(PDB_BOOT, ("remap old ")); 684 #if 0 685 /* 686 * calling the prom will probably require reading part of the 687 * data segment so we can't do this. */ 688 prom_unmap_virt((vaddr_t)kdatap, kdsize); 689 #endif 690 prom_map_phys(newkp, sz, kdata, -1); 691 /* 692 * we will map in 4MB, more than we allocated, to allow 693 * further allocation 694 */ 695 BDPRINTF(PDB_BOOT1, ("free old\r\n")); 696 prom_free_phys(kdatap, kdsize); 697 kdatap = newkp; 698 BDPRINTF(PDB_BOOT1, 699 ("pmap_bootstrap: firstaddr is %lx virt (%lx phys)" 700 "avail for kernel\r\n", (u_long)firstaddr, 701 (u_long)prom_vtop(firstaddr))); 702 } 703 704 /* 705 * Find out how much RAM we have installed. 706 */ 707 BDPRINTF(PDB_BOOT, ("pmap_bootstrap: getting phys installed\r\n")); 708 if ((memh = OF_finddevice("/memory")) == -1) { 709 prom_printf("no memory?"); 710 OF_exit(); 711 } 712 memsize = OF_getproplen(memh, "reg") + 2 * sizeof(struct mem_region); 713 valloc(mem, struct mem_region, memsize); 714 bzero((caddr_t)mem, memsize); 715 if (OF_getprop(memh, "reg", mem, memsize) <= 0) { 716 prom_printf("no memory installed?"); 717 OF_exit(); 718 } 719 720 #ifdef DEBUG 721 if (pmapdebug & PDB_BOOT1) { 722 /* print out mem list */ 723 prom_printf("Installed physical memory:\r\n"); 724 for (mp = mem; mp->size; mp++) { 725 prom_printf("memlist start %lx size %lx\r\n", 726 (u_long)mp->start, (u_long)mp->size); 727 } 728 } 729 #endif 730 BDPRINTF(PDB_BOOT1, ("Calculating physmem:")); 731 732 for (mp = mem; mp->size; mp++) 733 physmem += btoc(mp->size); 734 BDPRINTF(PDB_BOOT1, (" result %x or %d pages\r\n", 735 (int)physmem, (int)physmem)); 736 /* 737 * Calculate approx TSB size. This probably needs tweaking. 738 */ 739 if (physmem < btoc(64 * 1024 * 1024)) 740 tsbsize = 0; 741 else if (physmem < btoc(512 * 1024 * 1024)) 742 tsbsize = 1; 743 else 744 tsbsize = 2; 745 746 /* 747 * Save the prom translations 748 */ 749 sz = OF_getproplen(vmemh, "translations"); 750 valloc(prom_map, struct prom_map, sz); 751 if (OF_getprop(vmemh, "translations", (void*)prom_map, sz) <= 0) { 752 prom_printf("no translations installed?"); 753 OF_exit(); 754 } 755 prom_map_size = sz / sizeof(struct prom_map); 756 #ifdef DEBUG 757 if (pmapdebug & PDB_BOOT) { 758 /* print out mem list */ 759 prom_printf("Prom xlations:\r\n"); 760 for (i = 0; i < prom_map_size; i++) { 761 prom_printf("start %016lx size %016lx tte %016lx\r\n", 762 (u_long)prom_map[i].vstart, 763 (u_long)prom_map[i].vsize, 764 (u_long)prom_map[i].tte); 765 } 766 prom_printf("End of prom xlations\r\n"); 767 } 768 #endif 769 /* 770 * Hunt for the kernel text segment and figure out it size and 771 * alignment. 772 */ 773 for (i = 0; i < prom_map_size; i++) 774 if (prom_map[i].vstart == ktext) 775 break; 776 if (i == prom_map_size) 777 panic("No kernel text segment!"); 778 ktsize = prom_map[i].vsize; 779 ektext = ktext + ktsize; 780 781 if (ktextp & (4*MEG-1)) { 782 /* Kernel text is not 4MB aligned -- need to fix that */ 783 BDPRINTF(PDB_BOOT1, 784 ("Allocating new %lx kernel text at 4MB boundary\r\n", 785 (u_long)ktsize)); 786 if ((newkp = prom_alloc_phys(ktsize, 4*MEG)) == 0 ) { 787 prom_printf("Cannot allocate new kernel text\r\n"); 788 OF_exit(); 789 } 790 BDPRINTF(PDB_BOOT1, ("Allocating new va for buffer at %llx\r\n", 791 (u_int64_t)newkp)); 792 if ((newkv = (vaddr_t)prom_alloc_virt(ktsize, 8)) == 793 (vaddr_t)-1) { 794 prom_printf("Cannot allocate new kernel text va\r\n"); 795 OF_exit(); 796 } 797 BDPRINTF(PDB_BOOT1, ("Mapping in buffer %lx at %lx\r\n", 798 (u_long)newkp, (u_long)newkv)); 799 prom_map_phys(newkp, ktsize, (vaddr_t)newkv, -1); 800 BDPRINTF(PDB_BOOT1, ("Copying %ld bytes kernel text...", 801 ktsize)); 802 bcopy((void *)ktext, (void *)newkv, 803 ktsize); 804 BDPRINTF(PDB_BOOT1, ("done. Swapping maps..unmap new\r\n")); 805 prom_unmap_virt((vaddr_t)newkv, 4*MEG); 806 BDPRINTF(PDB_BOOT, ("remap old ")); 807 #if 0 808 /* 809 * calling the prom will probably require reading part of the 810 * text segment so we can't do this. 811 */ 812 prom_unmap_virt((vaddr_t)ktextp, ktsize); 813 #endif 814 prom_map_phys(newkp, ktsize, ktext, -1); 815 /* 816 * we will map in 4MB, more than we allocated, to allow 817 * further allocation 818 */ 819 BDPRINTF(PDB_BOOT1, ("free old\r\n")); 820 prom_free_phys(ktextp, ktsize); 821 ktextp = newkp; 822 823 BDPRINTF(PDB_BOOT1, 824 ("pmap_bootstrap: firstaddr is %lx virt (%lx phys)" 825 "avail for kernel\r\n", (u_long)firstaddr, 826 (u_long)prom_vtop(firstaddr))); 827 828 /* 829 * Re-fetch translations -- they've certainly changed. 830 */ 831 if (OF_getprop(vmemh, "translations", (void*)prom_map, sz) <= 832 0) { 833 prom_printf("no translations installed?"); 834 OF_exit(); 835 } 836 #ifdef DEBUG 837 if (pmapdebug & PDB_BOOT) { 838 /* print out mem list */ 839 prom_printf("New prom xlations:\r\n"); 840 for (i = 0; i < prom_map_size; i++) { 841 prom_printf("start %016lx size %016lx tte %016lx\r\n", 842 (u_long)prom_map[i].vstart, 843 (u_long)prom_map[i].vsize, 844 (u_long)prom_map[i].tte); 845 } 846 prom_printf("End of prom xlations\r\n"); 847 } 848 #endif 849 } 850 ektextp = ktextp + ktsize; 851 852 /* 853 * Here's a quick in-lined reverse bubble sort. It gets rid of 854 * any translations inside the kernel data VA range. 855 */ 856 for(i = 0; i < prom_map_size; i++) { 857 if (prom_map[i].vstart >= kdata && 858 prom_map[i].vstart <= firstaddr) { 859 prom_map[i].vstart = 0; 860 prom_map[i].vsize = 0; 861 } 862 if (prom_map[i].vstart >= ktext && 863 prom_map[i].vstart <= ektext) { 864 prom_map[i].vstart = 0; 865 prom_map[i].vsize = 0; 866 } 867 for(j = i; j < prom_map_size; j++) { 868 if (prom_map[j].vstart >= kdata && 869 prom_map[j].vstart <= firstaddr) 870 continue; /* this is inside the kernel */ 871 if (prom_map[j].vstart >= ktext && 872 prom_map[j].vstart <= ektext) 873 continue; /* this is inside the kernel */ 874 if (prom_map[j].vstart > prom_map[i].vstart) { 875 struct prom_map tmp; 876 tmp = prom_map[i]; 877 prom_map[i] = prom_map[j]; 878 prom_map[j] = tmp; 879 } 880 } 881 } 882 #ifdef DEBUG 883 if (pmapdebug & PDB_BOOT) { 884 /* print out mem list */ 885 prom_printf("Prom xlations:\r\n"); 886 for (i = 0; i < prom_map_size; i++) { 887 prom_printf("start %016lx size %016lx tte %016lx\r\n", 888 (u_long)prom_map[i].vstart, 889 (u_long)prom_map[i].vsize, 890 (u_long)prom_map[i].tte); 891 } 892 prom_printf("End of prom xlations\r\n"); 893 } 894 #endif 895 896 /* 897 * Allocate a 64MB page for the cpu_info structure now. 898 */ 899 if ((cpu0paddr = prom_alloc_phys(8*NBPG, 8*NBPG)) == 0 ) { 900 prom_printf("Cannot allocate new cpu_info\r\n"); 901 OF_exit(); 902 } 903 904 /* 905 * Now the kernel text segment is in its final location we can try to 906 * find out how much memory really is free. 907 */ 908 sz = OF_getproplen(memh, "available") + sizeof(struct mem_region); 909 valloc(orig, struct mem_region, sz); 910 bzero((caddr_t)orig, sz); 911 if (OF_getprop(memh, "available", orig, sz) <= 0) { 912 prom_printf("no available RAM?"); 913 OF_exit(); 914 } 915 #ifdef DEBUG 916 if (pmapdebug & PDB_BOOT1) { 917 /* print out mem list */ 918 prom_printf("Available physical memory:\r\n"); 919 for (mp = orig; mp->size; mp++) { 920 prom_printf("memlist start %lx size %lx\r\n", 921 (u_long)mp->start, (u_long)mp->size); 922 } 923 prom_printf("End of available physical memory\r\n"); 924 } 925 #endif 926 valloc(avail, struct mem_region, sz); 927 bzero((caddr_t)avail, sz); 928 for (pcnt = 0, mp = orig, mp1 = avail; (mp1->size = mp->size); 929 mp++, mp1++) { 930 mp1->start = mp->start; 931 pcnt++; 932 } 933 934 /* 935 * Allocate and initialize a context table 936 */ 937 numctx = maxctx; 938 valloc(ctxbusy, paddr_t, CTXSIZE); 939 bzero((caddr_t)ctxbusy, CTXSIZE); 940 LIST_INIT(&pmap_ctxlist); 941 942 /* 943 * Allocate our TSB. 944 * 945 * We will use the left over space to flesh out the kernel pmap. 946 */ 947 BDPRINTF(PDB_BOOT1, ("firstaddr before TSB=%lx\r\n", 948 (u_long)firstaddr)); 949 firstaddr = ((firstaddr + TSBSIZE - 1) & ~(TSBSIZE-1)); 950 #ifdef DEBUG 951 i = (firstaddr + (NBPG-1)) & ~(NBPG-1); /* First, page align */ 952 if ((int)firstaddr < i) { 953 prom_printf("TSB alloc fixup failed\r\n"); 954 prom_printf("frobbed i, firstaddr before TSB=%x, %lx\r\n", 955 (int)i, (u_long)firstaddr); 956 panic("TSB alloc"); 957 OF_exit(); 958 } 959 #endif 960 BDPRINTF(PDB_BOOT, ("frobbed i, firstaddr before TSB=%x, %lx\r\n", 961 (int)i, (u_long)firstaddr)); 962 valloc(tsb, pte_t, TSBSIZE); 963 bzero(tsb, TSBSIZE); 964 965 BDPRINTF(PDB_BOOT1, ("firstaddr after TSB=%lx\r\n", (u_long)firstaddr)); 966 BDPRINTF(PDB_BOOT1, ("TSB allocated at %p size %08x\r\n", (void*)tsb, 967 (int)TSBSIZE)); 968 969 first_phys_addr = mem->start; 970 BDPRINTF(PDB_BOOT1, ("firstaddr after pmap=%08lx\r\n", 971 (u_long)firstaddr)); 972 973 /* 974 * Page align all regions. 975 * Non-page memory isn't very interesting to us. 976 * Also, sort the entries for ascending addresses. 977 * 978 * And convert from virtual to physical addresses. 979 */ 980 981 BDPRINTF(PDB_BOOT, ("kernel virtual size %08lx - %08lx\r\n", 982 (u_long)kernelstart, (u_long)firstaddr)); 983 kdata = kdata & ~PGOFSET; 984 ekdata = firstaddr; 985 ekdata = (ekdata + PGOFSET) & ~PGOFSET; 986 BDPRINTF(PDB_BOOT1, ("kernel virtual size %08lx - %08lx\r\n", 987 (u_long)kernelstart, (u_long)kernelend)); 988 ekdatap = ekdata - kdata + kdatap; 989 /* Switch from vaddrs to paddrs */ 990 if(ekdatap > (kdatap + 4*MEG)) { 991 prom_printf("Kernel size exceeds 4MB\r\n"); 992 } 993 994 #ifdef DEBUG 995 if (pmapdebug & PDB_BOOT1) { 996 /* print out mem list */ 997 prom_printf("Available %lx physical memory before cleanup:\r\n", 998 (u_long)avail); 999 for (mp = avail; mp->size; mp++) { 1000 prom_printf("memlist start %lx size %lx\r\n", 1001 (u_long)mp->start, 1002 (u_long)mp->size); 1003 } 1004 prom_printf("End of available physical memory before cleanup\r\n"); 1005 prom_printf("kernel physical text size %08lx - %08lx\r\n", 1006 (u_long)ktextp, (u_long)ektextp); 1007 prom_printf("kernel physical data size %08lx - %08lx\r\n", 1008 (u_long)kdatap, (u_long)ekdatap); 1009 } 1010 #endif 1011 /* 1012 * Here's a another quick in-lined bubble sort. 1013 */ 1014 for (i = 0; i < pcnt; i++) { 1015 for (j = i; j < pcnt; j++) { 1016 if (avail[j].start < avail[i].start) { 1017 struct mem_region tmp; 1018 tmp = avail[i]; 1019 avail[i] = avail[j]; 1020 avail[j] = tmp; 1021 } 1022 } 1023 } 1024 1025 /* Throw away page zero if we have it. */ 1026 if (avail->start == 0) { 1027 avail->start += NBPG; 1028 avail->size -= NBPG; 1029 } 1030 /* 1031 * Now we need to remove the area we valloc'ed from the available 1032 * memory lists. (NB: we may have already alloc'ed the entire space). 1033 */ 1034 npgs = 0; 1035 for (mp = avail; mp->size; mp++) { 1036 /* 1037 * Check whether this region holds all of the kernel. 1038 */ 1039 s = mp->start + mp->size; 1040 if (mp->start < kdatap && s > roundup(ekdatap, 4*MEG)) { 1041 avail[pcnt].start = roundup(ekdatap, 4*MEG); 1042 avail[pcnt++].size = s - kdatap; 1043 mp->size = kdatap - mp->start; 1044 } 1045 /* 1046 * Look whether this regions starts within the kernel. 1047 */ 1048 if (mp->start >= kdatap && 1049 mp->start < roundup(ekdatap, 4*MEG)) { 1050 s = ekdatap - mp->start; 1051 if (mp->size > s) 1052 mp->size -= s; 1053 else 1054 mp->size = 0; 1055 mp->start = roundup(ekdatap, 4*MEG); 1056 } 1057 /* 1058 * Now look whether this region ends within the kernel. 1059 */ 1060 s = mp->start + mp->size; 1061 if (s > kdatap && s < roundup(ekdatap, 4*MEG)) 1062 mp->size -= s - kdatap; 1063 /* 1064 * Now page align the start of the region. 1065 */ 1066 s = mp->start % NBPG; 1067 if (mp->size >= s) { 1068 mp->size -= s; 1069 mp->start += s; 1070 } 1071 /* 1072 * And now align the size of the region. 1073 */ 1074 mp->size -= mp->size % NBPG; 1075 /* 1076 * Check whether some memory is left here. 1077 */ 1078 if (mp->size == 0) { 1079 bcopy(mp + 1, mp, 1080 (pcnt - (mp - avail)) * sizeof *mp); 1081 pcnt--; 1082 mp--; 1083 continue; 1084 } 1085 s = mp->start; 1086 sz = mp->size; 1087 npgs += btoc(sz); 1088 for (mp1 = avail; mp1 < mp; mp1++) 1089 if (s < mp1->start) 1090 break; 1091 if (mp1 < mp) { 1092 bcopy(mp1, mp1 + 1, (char *)mp - (char *)mp1); 1093 mp1->start = s; 1094 mp1->size = sz; 1095 } 1096 #ifdef DEBUG 1097 /* Clear all memory we give to the VM system. I want to make sure 1098 * the PROM isn't using it for something, so this should break the PROM. 1099 */ 1100 1101 /* Calling pmap_zero_page() at this point also hangs some machines 1102 * so don't do it at all. -- pk 26/02/2002 1103 */ 1104 #if 0 1105 { 1106 paddr_t p; 1107 for (p = mp->start; p < mp->start+mp->size; p += NBPG) 1108 pmap_zero_page(p); 1109 } 1110 #endif 1111 #endif /* DEBUG */ 1112 /* 1113 * In future we should be able to specify both allocated 1114 * and free. 1115 */ 1116 uvm_page_physload( 1117 atop(mp->start), 1118 atop(mp->start+mp->size), 1119 atop(mp->start), 1120 atop(mp->start+mp->size), 1121 VM_FREELIST_DEFAULT); 1122 } 1123 1124 #if 0 1125 /* finally, free up any space that valloc did not use */ 1126 prom_unmap_virt((vaddr_t)ekdata, roundup(ekdata, 4*MEG) - ekdata); 1127 if (ekdatap < roundup(kdatap, 4*MEG)) { 1128 uvm_page_physload(atop(ekdatap), 1129 atop(roundup(ekdatap, (4*MEG))), 1130 atop(ekdatap), 1131 atop(roundup(ekdatap, (4*MEG))), 1132 VM_FREELIST_DEFAULT); 1133 } 1134 #endif 1135 1136 #ifdef DEBUG 1137 if (pmapdebug & PDB_BOOT) { 1138 /* print out mem list */ 1139 prom_printf("Available physical memory after cleanup:\r\n"); 1140 for (mp = avail; mp->size; mp++) { 1141 prom_printf("avail start %lx size %lx\r\n", 1142 (long)mp->start, (long)mp->size); 1143 } 1144 prom_printf("End of available physical memory after cleanup\r\n"); 1145 } 1146 #endif 1147 /* 1148 * Allocate and clear out pmap_kernel()->pm_segs[] 1149 */ 1150 simple_lock_init(&pmap_kernel()->pm_lock); 1151 pmap_kernel()->pm_refs = 1; 1152 pmap_kernel()->pm_ctx = 0; 1153 1154 /* Throw away page zero */ 1155 do { 1156 pmap_get_page(&newp); 1157 } while (!newp); 1158 pmap_kernel()->pm_segs=(paddr_t *)(u_long)newp; 1159 pmap_kernel()->pm_physaddr = newp; 1160 /* mark kernel context as busy */ 1161 ctxbusy[0] = pmap_kernel()->pm_physaddr; 1162 1163 /* 1164 * finish filling out kernel pmap. 1165 */ 1166 1167 BDPRINTF(PDB_BOOT, ("pmap_kernel()->pm_physaddr = %lx\r\n", 1168 (long)pmap_kernel()->pm_physaddr)); 1169 /* 1170 * Tell pmap about our mesgbuf -- Hope this works already 1171 */ 1172 #ifdef DEBUG 1173 BDPRINTF(PDB_BOOT1, ("Calling consinit()\r\n")); 1174 if (pmapdebug & PDB_BOOT1) 1175 consinit(); 1176 BDPRINTF(PDB_BOOT1, ("Inserting mesgbuf into pmap_kernel()\r\n")); 1177 #endif 1178 /* it's not safe to call pmap_enter so we need to do this ourselves */ 1179 va = (vaddr_t)msgbufp; 1180 prom_map_phys(phys_msgbuf, msgbufsiz, (vaddr_t)msgbufp, -1); 1181 while (msgbufsiz) { 1182 int pgsz; 1183 psize_t psize; 1184 1185 PMAP_PAGE_SIZE(va, phys_msgbuf, msgbufsiz, pgsz, psize); 1186 data = TSB_DATA(0 /* global */, 1187 pgsz, 1188 phys_msgbuf, 1189 1 /* priv */, 1190 1 /* Write */, 1191 1 /* Cacheable */, 1192 FORCE_ALIAS /* ALIAS -- Disable D$ */, 1193 1 /* valid */, 1194 0 /* IE */); 1195 do { 1196 pmap_enter_kpage(va, data); 1197 va += NBPG; 1198 msgbufsiz -= NBPG; 1199 phys_msgbuf += NBPG; 1200 } while (psize-=NBPG); 1201 } 1202 BDPRINTF(PDB_BOOT1, ("Done inserting mesgbuf into pmap_kernel()\r\n")); 1203 1204 BDPRINTF(PDB_BOOT1, ("Inserting PROM mappings into pmap_kernel()\r\n")); 1205 for (i = 0; i < prom_map_size; i++) 1206 if (prom_map[i].vstart && ((prom_map[i].vstart>>32) == 0)) 1207 for (j = 0; j < prom_map[i].vsize; j += NBPG) { 1208 int k; 1209 1210 for (k = 0; page_size_map[k].mask; k++) { 1211 if (((prom_map[i].vstart | 1212 prom_map[i].tte) & 1213 page_size_map[k].mask) == 0 && 1214 page_size_map[k].mask < 1215 prom_map[i].vsize) 1216 break; 1217 } 1218 #ifdef DEBUG 1219 page_size_map[k].use++; 1220 #endif 1221 /* Enter PROM map into pmap_kernel() */ 1222 pmap_enter_kpage(prom_map[i].vstart + j, 1223 (prom_map[i].tte + j)| 1224 page_size_map[k].code); 1225 } 1226 BDPRINTF(PDB_BOOT1, ("Done inserting PROM mappings into pmap_kernel()\r\n")); 1227 1228 /* 1229 * Fix up start of kernel heap. 1230 */ 1231 vmmap = (vaddr_t)roundup(ekdata, 4*MEG); 1232 /* Let's keep 1 page of redzone after the kernel */ 1233 vmmap += NBPG; 1234 { 1235 extern vaddr_t u0[2]; 1236 extern struct pcb* proc0paddr; 1237 extern void main __P((void)); 1238 paddr_t pa; 1239 1240 /* Initialize all the pointers to u0 */ 1241 cpcb = (struct pcb *)vmmap; 1242 proc0paddr = cpcb; 1243 u0[0] = vmmap; 1244 /* Allocate some VAs for u0 */ 1245 u0[1] = vmmap + 2*USPACE; 1246 1247 BDPRINTF(PDB_BOOT1, 1248 ("Inserting stack 0 into pmap_kernel() at %p\r\n", 1249 vmmap)); 1250 1251 while (vmmap < u0[1]) { 1252 int64_t data; 1253 1254 pmap_get_page(&pa); 1255 prom_map_phys(pa, NBPG, vmmap, -1); 1256 data = TSB_DATA(0 /* global */, 1257 PGSZ_8K, 1258 pa, 1259 1 /* priv */, 1260 1 /* Write */, 1261 1 /* Cacheable */, 1262 FORCE_ALIAS /* ALIAS -- Disable D$ */, 1263 1 /* valid */, 1264 0 /* IE */); 1265 pmap_enter_kpage(vmmap, data); 1266 vmmap += NBPG; 1267 } 1268 BDPRINTF(PDB_BOOT1, 1269 ("Done inserting stack 0 into pmap_kernel()\r\n")); 1270 1271 /* Now map in and initialize our cpu_info structure */ 1272 #ifdef DIAGNOSTIC 1273 vmmap += NBPG; /* redzone -- XXXX do we need one? */ 1274 #endif 1275 if ((vmmap ^ INTSTACK) & VA_ALIAS_MASK) 1276 vmmap += NBPG; /* Matchup virtual color for D$ */ 1277 intstk = vmmap; 1278 cpus = (struct cpu_info *)(intstk+CPUINFO_VA-INTSTACK); 1279 1280 BDPRINTF(PDB_BOOT1, 1281 ("Inserting cpu_info into pmap_kernel() at %p\r\n", 1282 cpus)); 1283 /* Now map in all 8 pages of cpu_info */ 1284 pa = cpu0paddr; 1285 prom_map_phys(pa, 64*KB, vmmap, -1); 1286 /* 1287 * Also map it in as the interrupt stack. 1288 * This lets the PROM see this if needed. 1289 * 1290 * XXXX locore.s does not flush these mappings 1291 * before installing the locked TTE. 1292 */ 1293 prom_map_phys(pa, 64*KB, CPUINFO_VA, -1); 1294 for (i=0; i<8; i++) { 1295 int64_t data; 1296 1297 data = TSB_DATA(0 /* global */, 1298 PGSZ_8K, 1299 pa, 1300 1 /* priv */, 1301 1 /* Write */, 1302 1 /* Cacheable */, 1303 FORCE_ALIAS /* ALIAS -- Disable D$ */, 1304 1 /* valid */, 1305 0 /* IE */); 1306 pmap_enter_kpage(vmmap, data); 1307 vmmap += NBPG; 1308 pa += NBPG; 1309 } 1310 BDPRINTF(PDB_BOOT1, ("Initializing cpu_info\r\n")); 1311 1312 /* Initialize our cpu_info structure */ 1313 bzero((void *)intstk, 8*NBPG); 1314 cpus->ci_next = NULL; /* Redundant, I know. */ 1315 cpus->ci_curproc = &proc0; 1316 cpus->ci_cpcb = (struct pcb *)u0[0]; /* Need better source */ 1317 cpus->ci_upaid = CPU_UPAID; 1318 cpus->ci_number = cpus->ci_upaid; /* How do we figure this out? */ 1319 cpus->ci_fpproc = NULL; 1320 cpus->ci_spinup = main; /* Call main when we're running. */ 1321 cpus->ci_initstack = (void *)u0[1]; 1322 cpus->ci_paddr = cpu0paddr; 1323 /* The rest will be done at CPU attach time. */ 1324 BDPRINTF(PDB_BOOT1, 1325 ("Done inserting cpu_info into pmap_kernel()\r\n")); 1326 } 1327 1328 vmmap = (vaddr_t)reserve_dumppages((caddr_t)(u_long)vmmap); 1329 /* 1330 * Set up bounds of allocatable memory for vmstat et al. 1331 */ 1332 nextavail = avail->start; 1333 avail_start = nextavail; 1334 for (mp = avail; mp->size; mp++) 1335 avail_end = mp->start+mp->size; 1336 BDPRINTF(PDB_BOOT1, ("Finished pmap_bootstrap()\r\n")); 1337 1338 } 1339 1340 /* 1341 * Initialize anything else for pmap handling. 1342 * Called during vm_init(). 1343 */ 1344 void 1345 pmap_init() 1346 { 1347 struct vm_page *pg; 1348 struct pglist pglist; 1349 u_int64_t data; 1350 paddr_t pa; 1351 psize_t size; 1352 vaddr_t va; 1353 1354 BDPRINTF(PDB_BOOT1, ("pmap_init()\r\n")); 1355 if (PAGE_SIZE != NBPG) 1356 panic("pmap_init: PAGE_SIZE!=NBPG"); 1357 1358 size = sizeof(struct pv_entry) * physmem; 1359 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 1360 (paddr_t)NBPG, (paddr_t)0, &pglist, 1, 0) != 0) 1361 panic("cpu_start: no memory"); 1362 1363 va = uvm_km_valloc(kernel_map, size); 1364 if (va == 0) 1365 panic("cpu_start: no memory"); 1366 1367 /* Map the pages */ 1368 TAILQ_FOREACH(pg, &pglist, pageq) { 1369 pa = VM_PAGE_TO_PHYS(pg); 1370 pmap_zero_page(pa); 1371 data = TSB_DATA(0 /* global */, 1372 PGSZ_8K, 1373 pa, 1374 1 /* priv */, 1375 1 /* Write */, 1376 1 /* Cacheable */, 1377 FORCE_ALIAS /* ALIAS -- Disable D$ */, 1378 1 /* valid */, 1379 0 /* IE */); 1380 pmap_enter_kpage(va, data); 1381 va += NBPG; 1382 } 1383 1384 /* 1385 * initialize the pmap pools. 1386 */ 1387 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1388 &pool_allocator_nointr); 1389 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", 1390 &pool_allocator_nointr); 1391 1392 vm_first_phys = avail_start; 1393 vm_num_phys = avail_end - avail_start; 1394 } 1395 1396 /* 1397 * How much virtual space is available to the kernel? 1398 */ 1399 static vaddr_t kbreak; /* End of kernel VA */ 1400 void 1401 pmap_virtual_space(start, end) 1402 vaddr_t *start, *end; 1403 { 1404 1405 /* 1406 * Reserve one segment for kernel virtual memory 1407 */ 1408 /* Reserve two pages for pmap_copy_page && /dev/mem */ 1409 *start = kbreak = (vaddr_t)(vmmap + 2*NBPG); 1410 *end = VM_MAX_KERNEL_ADDRESS; 1411 BDPRINTF(PDB_BOOT1, ("pmap_virtual_space: %x-%x\r\n", *start, *end)); 1412 } 1413 1414 /* 1415 * Preallocate kernel page tables to a specified VA. 1416 * This simply loops through the first TTE for each 1417 * page table from the beginning of the kernel pmap, 1418 * reads the entry, and if the result is 1419 * zero (either invalid entry or no page table) it stores 1420 * a zero there, populating page tables in the process. 1421 * This is not the most efficient technique but i don't 1422 * expect it to be called that often. 1423 */ 1424 vaddr_t 1425 pmap_growkernel(maxkvaddr) 1426 vaddr_t maxkvaddr; 1427 { 1428 struct pmap *pm = pmap_kernel(); 1429 paddr_t pa; 1430 1431 if (maxkvaddr >= KERNEND) { 1432 printf("WARNING: cannot extend kernel pmap beyond %p to %p\n", 1433 (void *)KERNEND, (void *)maxkvaddr); 1434 return (kbreak); 1435 } 1436 simple_lock(&pm->pm_lock); 1437 DPRINTF(PDB_GROW, ("pmap_growkernel(%lx...%lx)\n", kbreak, maxkvaddr)); 1438 /* Align with the start of a page table */ 1439 for (kbreak &= (-1 << PDSHIFT); kbreak < maxkvaddr; 1440 kbreak += (1 << PDSHIFT)) { 1441 if (pseg_get(pm, kbreak)) 1442 continue; 1443 1444 pa = 0; 1445 while (pseg_set(pm, kbreak, 0, pa) & 1) { 1446 DPRINTF(PDB_GROW, 1447 ("pmap_growkernel: extending %lx\n", kbreak)); 1448 pa = 0; 1449 if (!pmap_get_page(&pa)) 1450 panic("pmap_grow_kernel: no pages"); 1451 ENTER_STAT(ptpneeded); 1452 } 1453 } 1454 simple_unlock(&pm->pm_lock); 1455 return (kbreak); 1456 } 1457 1458 /* 1459 * Create and return a physical map. 1460 */ 1461 struct pmap * 1462 pmap_create() 1463 { 1464 struct pmap *pm; 1465 1466 DPRINTF(PDB_CREATE, ("pmap_create()\n")); 1467 1468 pm = pool_get(&pmap_pmap_pool, PR_WAITOK); 1469 memset(pm, 0, sizeof *pm); 1470 DPRINTF(PDB_CREATE, ("pmap_create(): created %p\n", pm)); 1471 1472 simple_lock_init(&pm->pm_lock); 1473 pm->pm_refs = 1; 1474 TAILQ_INIT(&pm->pm_obj.memq); 1475 if (pm != pmap_kernel()) { 1476 pmap_get_page(&pm->pm_physaddr); 1477 while (pm->pm_physaddr == NULL) { 1478 uvm_wait("pmap_create"); 1479 pmap_get_page(&pm->pm_physaddr); 1480 } 1481 pm->pm_segs = (paddr_t *)(u_long)pm->pm_physaddr; 1482 if (!pm->pm_physaddr) 1483 panic("pmap_create"); 1484 ctx_alloc(pm); 1485 } 1486 DPRINTF(PDB_CREATE, ("pmap_create(%p): ctx %d\n", pm, pm->pm_ctx)); 1487 return pm; 1488 } 1489 1490 /* 1491 * Add a reference to the given pmap. 1492 */ 1493 void 1494 pmap_reference(pm) 1495 struct pmap *pm; 1496 { 1497 1498 simple_lock(&pm->pm_lock); 1499 pm->pm_refs++; 1500 simple_unlock(&pm->pm_lock); 1501 } 1502 1503 /* 1504 * Retire the given pmap from service. 1505 * Should only be called if the map contains no valid mappings. 1506 */ 1507 void 1508 pmap_destroy(pm) 1509 struct pmap *pm; 1510 { 1511 struct vm_page *pg, *nextpg; 1512 int refs; 1513 1514 simple_lock(&pm->pm_lock); 1515 refs = --pm->pm_refs; 1516 simple_unlock(&pm->pm_lock); 1517 if (refs > 0) { 1518 return; 1519 } 1520 DPRINTF(PDB_DESTROY, ("pmap_destroy: freeing pmap %p\n", pm)); 1521 1522 /* we could be a little smarter and leave pages zeroed */ 1523 for (pg = TAILQ_FIRST(&pm->pm_obj.memq); pg != NULL; pg = nextpg) { 1524 nextpg = TAILQ_NEXT(pg, listq); 1525 TAILQ_REMOVE(&pm->pm_obj.memq, pg, listq); 1526 KASSERT(pg->mdpage.mdpg_pvh.pv_pmap == NULL); 1527 uvm_pagefree(pg); 1528 } 1529 pmap_free_page((paddr_t)(u_long)pm->pm_segs); 1530 pool_put(&pmap_pmap_pool, pm); 1531 } 1532 1533 /* 1534 * Copy the range specified by src_addr/len 1535 * from the source map to the range dst_addr/len 1536 * in the destination map. 1537 * 1538 * This routine is only advisory and need not do anything. 1539 */ 1540 void 1541 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 1542 struct pmap *dst_pmap, *src_pmap; 1543 vaddr_t dst_addr, src_addr; 1544 vsize_t len; 1545 { 1546 1547 DPRINTF(PDB_CREATE, ("pmap_copy(%p, %p, %p, %lx, %p)\n", 1548 dst_pmap, src_pmap, (void *)(u_long)dst_addr, 1549 (u_long)len, (void *)(u_long)src_addr)); 1550 } 1551 1552 /* 1553 * Garbage collects the physical map system for 1554 * pages which are no longer used. 1555 * Success need not be guaranteed -- that is, there 1556 * may well be pages which are not referenced, but 1557 * others may be collected. 1558 * Called by the pageout daemon when pages are scarce. 1559 */ 1560 void 1561 pmap_collect(pm) 1562 struct pmap *pm; 1563 { 1564 int64_t data; 1565 paddr_t pa, *pdir, *ptbl; 1566 struct vm_page *pg; 1567 int i, j, k, n, m; 1568 1569 /* 1570 * This is a good place to scan the pmaps for page tables with 1571 * no valid mappings in them and free them. 1572 */ 1573 1574 /* NEVER GARBAGE COLLECT THE KERNEL PMAP */ 1575 if (pm == pmap_kernel()) 1576 return; 1577 1578 simple_lock(&pm->pm_lock); 1579 for (i = 0; i < STSZ; i++) { 1580 pdir = (paddr_t *)(u_long)ldxa((vaddr_t)&pm->pm_segs[i], 1581 ASI_PHYS_CACHED); 1582 if (pdir == NULL) { 1583 continue; 1584 } 1585 m = 0; 1586 for (k = 0; k < PDSZ; k++) { 1587 ptbl = (paddr_t *)(u_long)ldxa((vaddr_t)&pdir[k], 1588 ASI_PHYS_CACHED); 1589 if (ptbl == NULL) { 1590 continue; 1591 } 1592 m++; 1593 n = 0; 1594 for (j = 0; j < PTSZ; j++) { 1595 data = ldxa((vaddr_t)&ptbl[j], ASI_PHYS_CACHED); 1596 if (data & TLB_V) 1597 n++; 1598 } 1599 if (!n) { 1600 stxa((paddr_t)(u_long)&pdir[k], 1601 ASI_PHYS_CACHED, 0); 1602 pa = (paddr_t)(u_long)ptbl; 1603 pg = PHYS_TO_VM_PAGE(pa); 1604 TAILQ_REMOVE(&pm->pm_obj.memq, pg, listq); 1605 pmap_free_page(pa); 1606 } 1607 } 1608 if (!m) { 1609 stxa((paddr_t)(u_long)&pm->pm_segs[i], 1610 ASI_PHYS_CACHED, 0); 1611 pa = (paddr_t)(u_long)pdir; 1612 pg = PHYS_TO_VM_PAGE(pa); 1613 TAILQ_REMOVE(&pm->pm_obj.memq, pg, listq); 1614 pmap_free_page(pa); 1615 } 1616 } 1617 simple_unlock(&pm->pm_lock); 1618 } 1619 1620 #if 0 1621 /* 1622 * The two following routines are now in locore.s so I can code them in assembly 1623 * They can bypass the MMU or use VIS bcopy extensions for speed. 1624 */ 1625 /* 1626 * Fill the given physical page with zeroes. 1627 */ 1628 void 1629 pmap_zero_page(pa) 1630 paddr_t pa; 1631 { 1632 /* 1633 * We don't need to worry about flushing caches 1634 * since all our virtual caches are write-through. 1635 * All we need to do is map the page in somewhere, bzero it, 1636 * and unmap it. However, we need to be sure we don't 1637 * map it in anywhere near the kernel or we may lose, badly. 1638 */ 1639 bzero((caddr_t)pa, NBPG); 1640 } 1641 1642 /* 1643 * Copy the given physical source page to its destination. 1644 * 1645 * I will code this in assembly RSN. 1646 */ 1647 void 1648 pmap_copy_page(src, dst) 1649 paddr_t src, dst; 1650 { 1651 bcopy((caddr_t)src, (caddr_t)dst, NBPG); 1652 } 1653 #endif 1654 1655 /* 1656 * Activate the address space for the specified process. If the 1657 * process is the current process, load the new MMU context. 1658 */ 1659 void 1660 pmap_activate(p) 1661 struct proc *p; 1662 { 1663 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1664 1665 /* 1666 * This is essentially the same thing that happens in cpu_switch() 1667 * when the newly selected process is about to run, except that we 1668 * have to make sure to clean the register windows before we set 1669 * the new context. 1670 */ 1671 1672 if (p != curproc) { 1673 return; 1674 } 1675 write_user_windows(); 1676 pmap_activate_pmap(pmap); 1677 } 1678 1679 void 1680 pmap_activate_pmap(struct pmap *pmap) 1681 { 1682 1683 if (pmap->pm_ctx == 0) { 1684 (void) ctx_alloc(pmap); 1685 } 1686 stxa(CTX_SECONDARY, ASI_DMMU, pmap->pm_ctx); 1687 } 1688 1689 /* 1690 * Deactivate the address space of the specified process. 1691 */ 1692 void 1693 pmap_deactivate(p) 1694 struct proc *p; 1695 { 1696 } 1697 1698 /* 1699 * pmap_kenter_pa: [ INTERFACE ] 1700 * 1701 * Enter a va -> pa mapping into the kernel pmap without any 1702 * physical->virtual tracking. 1703 * 1704 * Note: no locking is necessary in this function. 1705 */ 1706 void 1707 pmap_kenter_pa(va, pa, prot) 1708 vaddr_t va; 1709 paddr_t pa; 1710 vm_prot_t prot; 1711 { 1712 pte_t tte; 1713 paddr_t ptp; 1714 struct pmap *pm = pmap_kernel(); 1715 int i; 1716 1717 KASSERT(va < INTSTACK || va > EINTSTACK); 1718 KASSERT(va < kdata || va > ekdata); 1719 1720 /* 1721 * Construct the TTE. 1722 */ 1723 1724 ENTER_STAT(unmanaged); 1725 if (pa & (PMAP_NVC|PMAP_NC)) 1726 ENTER_STAT(ci); 1727 1728 tte.data = TSB_DATA(0, PGSZ_8K, pa, 1 /* Privileged */, 1729 (VM_PROT_WRITE & prot), 1730 !(pa & PMAP_NC), pa & (PMAP_NVC), 1, 0); 1731 /* We don't track mod/ref here. */ 1732 if (prot & VM_PROT_WRITE) 1733 tte.data |= TLB_REAL_W|TLB_W; 1734 tte.data |= TLB_TSB_LOCK; /* wired */ 1735 KASSERT((tte.data & TLB_NFO) == 0); 1736 ptp = 0; 1737 1738 retry: 1739 i = pseg_set(pm, va, tte.data, ptp); 1740 KASSERT((i & 6) == 0); 1741 if (i & 1) { 1742 KASSERT((i & 4) == 0); 1743 ptp = 0; 1744 if (!pmap_get_page(&ptp)) 1745 panic("pmap_kenter_pa: no pages"); 1746 ENTER_STAT(ptpneeded); 1747 goto retry; 1748 } 1749 if (ptp && i == 0) { 1750 /* We allocated a spare page but didn't use it. Free it. */ 1751 printf("pmap_kenter_pa: freeing unused page %llx\n", 1752 (long long)ptp); 1753 pmap_free_page(ptp); 1754 } 1755 #ifdef DEBUG 1756 i = ptelookup_va(va); 1757 if (pmapdebug & PDB_ENTER) 1758 prom_printf("pmap_kenter_pa: va=%08x data=%08x:%08x " 1759 "tsb[%d]=%08x\r\n", va, (int)(tte.data>>32), 1760 (int)tte.data, i, &tsb[i]); 1761 if (pmapdebug & PDB_MMU_STEAL && tsb[i].data) { 1762 prom_printf("pmap_kenter_pa: evicting entry tag=%x:%08x " 1763 "data=%08x:%08x tsb[%d]=%08x\r\n", 1764 (int)(tsb[i].tag>>32), (int)tsb[i].tag, 1765 (int)(tsb[i].data>>32), (int)tsb[i].data, 1766 i, &tsb[i]); 1767 prom_printf("with va=%08x data=%08x:%08x tsb[%d]=%08x\r\n", 1768 va, (int)(tte.data>>32), (int)tte.data, i, &tsb[i]); 1769 } 1770 #endif 1771 KASSERT((tsb[i].data & TLB_NFO) == 0); 1772 } 1773 1774 /* 1775 * pmap_kremove: [ INTERFACE ] 1776 * 1777 * Remove a mapping entered with pmap_kenter_pa() starting at va, 1778 * for size bytes (assumed to be page rounded). 1779 */ 1780 void 1781 pmap_kremove(va, size) 1782 vaddr_t va; 1783 vsize_t size; 1784 { 1785 struct pmap *pm = pmap_kernel(); 1786 int64_t data; 1787 vaddr_t flushva = va; 1788 vsize_t flushsize = size; 1789 paddr_t pa; 1790 int i; 1791 boolean_t flush = FALSE; 1792 1793 KASSERT(va < INTSTACK || va > EINTSTACK); 1794 KASSERT(va < kdata || va > ekdata); 1795 1796 DPRINTF(PDB_DEMAP, ("pmap_kremove: start 0x%lx size %lx\n", va, size)); 1797 for (; size >= NBPG; va += NBPG, size -= NBPG) { 1798 1799 #ifdef DIAGNOSTIC 1800 /* 1801 * Is this part of the permanent 4MB mapping? 1802 */ 1803 if (va >= ktext && va < roundup(ekdata, 4*MEG)) 1804 panic("pmap_kremove: va=%08x in locked TLB\r", 1805 (u_int)va); 1806 #endif 1807 1808 data = pseg_get(pm, va); 1809 if (data == 0) { 1810 continue; 1811 } 1812 1813 flush = TRUE; 1814 pa = data & TLB_PA_MASK; 1815 1816 /* 1817 * We need to flip the valid bit and 1818 * clear the access statistics. 1819 */ 1820 1821 if (pseg_set(pm, va, 0, 0)) { 1822 printf("pmap_kremove: pseg empty!\n"); 1823 Debugger(); 1824 /* panic? */ 1825 } 1826 DPRINTF(PDB_DEMAP, ("pmap_kremove: seg %x pdir %x pte %x\n", 1827 (int)va_to_seg(va), (int)va_to_dir(va), 1828 (int)va_to_pte(va))); 1829 REMOVE_STAT(removes); 1830 1831 i = ptelookup_va(va); 1832 if (tsb[i].tag > 0 && 1833 tsb[i].tag == TSB_TAG(0, pm->pm_ctx, va)) { 1834 tsb[i].data = 0; 1835 KASSERT((tsb[i].data & TLB_NFO) == 0); 1836 } 1837 REMOVE_STAT(tflushes); 1838 1839 /* 1840 * Here we assume nothing can get into the TLB 1841 * unless it has a PTE. 1842 */ 1843 1844 tlb_flush_pte(va, pm->pm_ctx); 1845 } 1846 if (flush) { 1847 REMOVE_STAT(flushes); 1848 cache_flush_virt(flushva, flushsize); 1849 } 1850 } 1851 1852 /* 1853 * Insert physical page at pa into the given pmap at virtual address va. 1854 * Supports 64-bit pa so we can map I/O space. 1855 */ 1856 int 1857 pmap_enter(pm, va, pa, prot, flags) 1858 struct pmap *pm; 1859 vaddr_t va; 1860 paddr_t pa; 1861 vm_prot_t prot; 1862 int flags; 1863 { 1864 pte_t tte; 1865 int64_t data; 1866 paddr_t opa, ptp; 1867 pv_entry_t pvh, npv = NULL; 1868 struct vm_page *pg, *opg, *ptpg; 1869 int s, i, uncached = 0; 1870 int size = 0; /* PMAP_SZ_TO_TTE(pa); */ 1871 boolean_t wired = (flags & PMAP_WIRED) != 0; 1872 boolean_t wasmapped = FALSE; 1873 boolean_t dopv = TRUE; 1874 1875 /* 1876 * Is this part of the permanent mappings? 1877 */ 1878 KASSERT(pm != pmap_kernel() || va < INTSTACK || va > EINTSTACK); 1879 KASSERT(pm != pmap_kernel() || va < kdata || va > ekdata); 1880 1881 /* 1882 * If a mapping at this address already exists, check if we're 1883 * entering the same PA again. if it's different remove it. 1884 */ 1885 1886 simple_lock(&pm->pm_lock); 1887 data = pseg_get(pm, va); 1888 if (data & TLB_V) { 1889 wasmapped = TRUE; 1890 opa = data & TLB_PA_MASK; 1891 if (opa != pa) { 1892 opg = PHYS_TO_VM_PAGE(opa); 1893 if (opg != NULL) { 1894 npv = pmap_remove_pv(pm, va, opg); 1895 } 1896 } 1897 } 1898 1899 /* 1900 * Construct the TTE. 1901 */ 1902 pg = PHYS_TO_VM_PAGE(pa); 1903 if (pg) { 1904 pvh = &pg->mdpage.mdpg_pvh; 1905 uncached = (pvh->pv_va & (PV_ALIAS|PV_NVC)); 1906 #ifdef DIAGNOSTIC 1907 if ((flags & VM_PROT_ALL) & ~prot) 1908 panic("pmap_enter: access_type exceeds prot"); 1909 #endif 1910 /* 1911 * If we don't have the traphandler do it, 1912 * set the ref/mod bits now. 1913 */ 1914 if (flags & VM_PROT_ALL) 1915 pvh->pv_va |= PV_REF; 1916 if (flags & VM_PROT_WRITE) 1917 pvh->pv_va |= PV_MOD; 1918 1919 /* 1920 * make sure we have a pv entry ready if we need one. 1921 */ 1922 if (pvh->pv_pmap == NULL || (wasmapped && opa == pa)) { 1923 if (npv != NULL) { 1924 pool_put(&pmap_pv_pool, npv); 1925 npv = NULL; 1926 } 1927 if (wasmapped && opa == pa) { 1928 dopv = FALSE; 1929 } 1930 } else if (npv == NULL) { 1931 npv = pool_get(&pmap_pv_pool, PR_NOWAIT); 1932 if (npv == NULL) { 1933 if (flags & PMAP_CANFAIL) { 1934 simple_unlock(&pm->pm_lock); 1935 return (ENOMEM); 1936 } 1937 panic("pmap_enter: no pv entries available"); 1938 } 1939 } 1940 ENTER_STAT(managed); 1941 } else { 1942 ENTER_STAT(unmanaged); 1943 dopv = FALSE; 1944 if (npv != NULL) { 1945 pool_put(&pmap_pv_pool, npv); 1946 npv = NULL; 1947 } 1948 } 1949 1950 #ifndef NO_VCACHE 1951 if (pa & PMAP_NVC) 1952 #endif 1953 uncached = 1; 1954 if (uncached) 1955 ENTER_STAT(ci); 1956 tte.data = TSB_DATA(0, size, pa, pm == pmap_kernel(), 1957 flags & VM_PROT_WRITE, !(pa & PMAP_NC), 1958 uncached, 1, pa & PMAP_LITTLE); 1959 #ifdef HWREF 1960 if (prot & VM_PROT_WRITE) 1961 tte.data |= TLB_REAL_W; 1962 #else 1963 /* If it needs ref accounting do nothing. */ 1964 if (!(flags & VM_PROT_READ)) { 1965 simple_unlock(&pm->pm_lock); 1966 return 0; 1967 } 1968 #endif 1969 if (flags & VM_PROT_EXECUTE) { 1970 if ((flags & (VM_PROT_READ|VM_PROT_WRITE)) == 0) 1971 tte.data |= TLB_EXEC_ONLY|TLB_EXEC; 1972 else 1973 tte.data |= TLB_EXEC; 1974 } 1975 if (wired) 1976 tte.data |= TLB_TSB_LOCK; 1977 KASSERT((tte.data & TLB_NFO) == 0); 1978 ptp = 0; 1979 1980 retry: 1981 i = pseg_set(pm, va, tte.data, ptp); 1982 if (i & 4) { 1983 /* ptp used as L3 */ 1984 KASSERT(ptp != 0); 1985 KASSERT((i & 3) == 0); 1986 ptpg = PHYS_TO_VM_PAGE(ptp); 1987 if (ptpg) { 1988 ptpg->offset = (uint64_t)va & (0xfffffLL << 23); 1989 TAILQ_INSERT_TAIL(&pm->pm_obj.memq, ptpg, listq); 1990 } else { 1991 KASSERT(pm == pmap_kernel()); 1992 } 1993 } 1994 if (i & 2) { 1995 /* ptp used as L2 */ 1996 KASSERT(ptp != 0); 1997 KASSERT((i & 4) == 0); 1998 ptpg = PHYS_TO_VM_PAGE(ptp); 1999 if (ptpg) { 2000 ptpg->offset = (((uint64_t)va >> 43) & 0x3ffLL) << 13; 2001 TAILQ_INSERT_TAIL(&pm->pm_obj.memq, ptpg, listq); 2002 } else { 2003 KASSERT(pm == pmap_kernel()); 2004 } 2005 } 2006 if (i & 1) { 2007 KASSERT((i & 4) == 0); 2008 ptp = 0; 2009 if (!pmap_get_page(&ptp)) { 2010 if (flags & PMAP_CANFAIL) { 2011 simple_unlock(&pm->pm_lock); 2012 if (npv != NULL) { 2013 pool_put(&pmap_pv_pool, npv); 2014 } 2015 return (ENOMEM); 2016 } else { 2017 panic("pmap_enter: no pages"); 2018 } 2019 } 2020 ENTER_STAT(ptpneeded); 2021 goto retry; 2022 } 2023 if (ptp && i == 0) { 2024 /* We allocated a spare page but didn't use it. Free it. */ 2025 printf("pmap_enter: freeing unused page %llx\n", 2026 (long long)ptp); 2027 pmap_free_page(ptp); 2028 } 2029 if (dopv) { 2030 pmap_enter_pv(pm, va, pa, pg, npv); 2031 } 2032 2033 simple_unlock(&pm->pm_lock); 2034 #ifdef DEBUG 2035 i = ptelookup_va(va); 2036 if (pmapdebug & PDB_ENTER) 2037 prom_printf("pmap_enter: va=%08x data=%08x:%08x " 2038 "tsb[%d]=%08x\r\n", va, (int)(tte.data>>32), 2039 (int)tte.data, i, &tsb[i]); 2040 if (pmapdebug & PDB_MMU_STEAL && tsb[i].data) { 2041 prom_printf("pmap_enter: evicting entry tag=%x:%08x " 2042 "data=%08x:%08x tsb[%d]=%08x\r\n", 2043 (int)(tsb[i].tag>>32), (int)tsb[i].tag, 2044 (int)(tsb[i].data>>32), (int)tsb[i].data, i, &tsb[i]); 2045 prom_printf("with va=%08x data=%08x:%08x tsb[%d]=%08x\r\n", 2046 va, (int)(tte.data>>32), (int)tte.data, i, &tsb[i]); 2047 } 2048 #endif 2049 if (flags & (VM_PROT_READ | VM_PROT_WRITE)) { 2050 2051 /* 2052 * preload the TSB with the new entry, 2053 * since we're going to need it immediately anyway. 2054 */ 2055 2056 i = ptelookup_va(va); 2057 tte.tag = TSB_TAG(0, pm->pm_ctx, va); 2058 s = splhigh(); 2059 tsb[i].tag = tte.tag; 2060 __asm __volatile("" : : : "memory"); 2061 tsb[i].data = tte.data; 2062 2063 /* 2064 * it's only necessary to flush the TLB if this page was 2065 * previously mapped, but for some reason it's a lot faster 2066 * for the fork+exit microbenchmark if we always do it. 2067 */ 2068 2069 tlb_flush_pte(va, pm->pm_ctx); 2070 splx(s); 2071 } else if (wasmapped && (pm->pm_ctx || pm == pmap_kernel())) { 2072 i = ptelookup_va(va); 2073 if (tsb[i].tag > 0 && 2074 tsb[i].tag == TSB_TAG(0, pm->pm_ctx, va)) { 2075 tsb[i].data = 0; 2076 } 2077 /* Force reload -- protections may be changed */ 2078 tlb_flush_pte(va, pm->pm_ctx); 2079 KASSERT((tsb[i].data & TLB_NFO) == 0); 2080 } 2081 2082 /* We will let the fast mmu miss interrupt load the new translation */ 2083 pv_check(); 2084 return 0; 2085 } 2086 2087 void 2088 pmap_remove_all(pm) 2089 struct pmap *pm; 2090 { 2091 2092 if (pm == pmap_kernel()) { 2093 return; 2094 } 2095 stxa(CTX_SECONDARY, ASI_DMMU, 0); 2096 pm->pm_refs = 0; 2097 ctx_free(pm); 2098 } 2099 2100 /* 2101 * Remove the given range of mapping entries. 2102 */ 2103 void 2104 pmap_remove(pm, va, endva) 2105 struct pmap *pm; 2106 vaddr_t va, endva; 2107 { 2108 int64_t data; 2109 vaddr_t flushva = va; 2110 paddr_t pa; 2111 struct vm_page *pg; 2112 pv_entry_t pv; 2113 int i; 2114 boolean_t flush = FALSE; 2115 2116 /* 2117 * In here we should check each pseg and if there are no more entries, 2118 * free it. It's just that linear scans of 8K pages gets expensive. 2119 */ 2120 2121 KASSERT(pm != pmap_kernel() || endva < INTSTACK || va > EINTSTACK); 2122 KASSERT(pm != pmap_kernel() || endva < kdata || va > ekdata); 2123 2124 simple_lock(&pm->pm_lock); 2125 DPRINTF(PDB_REMOVE, ("pmap_remove(pm=%p, va=%p, endva=%p):", pm, 2126 (void *)(u_long)va, (void *)(u_long)endva)); 2127 REMOVE_STAT(calls); 2128 2129 /* Now do the real work */ 2130 for (; va < endva; va += NBPG) { 2131 #ifdef DIAGNOSTIC 2132 /* 2133 * Is this part of the permanent 4MB mapping? 2134 */ 2135 if (pm == pmap_kernel() && va >= ktext && 2136 va < roundup(ekdata, 4*MEG)) 2137 panic("pmap_remove: va=%08llx in locked TLB\r", 2138 (long long)va); 2139 #endif 2140 2141 data = pseg_get(pm, va); 2142 if (data == 0) { 2143 continue; 2144 } 2145 2146 flush = TRUE; 2147 /* First remove the pv entry, if there is one */ 2148 pa = data & TLB_PA_MASK; 2149 pg = PHYS_TO_VM_PAGE(pa); 2150 if (pg) { 2151 pv = pmap_remove_pv(pm, va, pg); 2152 if (pv != NULL) { 2153 pool_put(&pmap_pv_pool, pv); 2154 } 2155 } 2156 2157 /* 2158 * We need to flip the valid bit and 2159 * clear the access statistics. 2160 */ 2161 2162 if (pseg_set(pm, va, 0, 0)) { 2163 printf("pmap_remove: pseg empty!\n"); 2164 Debugger(); 2165 /* panic? */ 2166 } 2167 DPRINTF(PDB_REMOVE, (" clearing seg %x pte %x\n", 2168 (int)va_to_seg(va), (int)va_to_pte(va))); 2169 REMOVE_STAT(removes); 2170 2171 if (!pm->pm_ctx && pm != pmap_kernel()) 2172 continue; 2173 i = ptelookup_va(va); 2174 if (tsb[i].tag > 0 && 2175 tsb[i].tag == TSB_TAG(0, pm->pm_ctx, va)) { 2176 DPRINTF(PDB_REMOVE, (" clearing TSB [%d]\n", i)); 2177 tsb[i].data = 0; 2178 } 2179 REMOVE_STAT(tflushes); 2180 2181 /* 2182 * Here we assume nothing can get into the TLB 2183 * unless it has a PTE. 2184 */ 2185 2186 if (pm->pm_refs) { 2187 tlb_flush_pte(va, pm->pm_ctx); 2188 } 2189 } 2190 simple_unlock(&pm->pm_lock); 2191 if (flush && pm->pm_refs) { 2192 REMOVE_STAT(flushes); 2193 cache_flush_virt(flushva, endva - flushva); 2194 } 2195 DPRINTF(PDB_REMOVE, ("\n")); 2196 pv_check(); 2197 } 2198 2199 /* 2200 * Change the protection on the specified range of this pmap. 2201 */ 2202 void 2203 pmap_protect(pm, sva, eva, prot) 2204 struct pmap *pm; 2205 vaddr_t sva, eva; 2206 vm_prot_t prot; 2207 { 2208 paddr_t pa; 2209 int64_t data; 2210 struct vm_page *pg; 2211 pv_entry_t pv; 2212 int i; 2213 2214 KASSERT(pm != pmap_kernel() || eva < INTSTACK || sva > EINTSTACK); 2215 KASSERT(pm != pmap_kernel() || eva < kdata || sva > ekdata); 2216 2217 if ((prot & (VM_PROT_WRITE|PMAP_WIRED)) == VM_PROT_WRITE) 2218 return; 2219 2220 if (prot == VM_PROT_NONE) { 2221 pmap_remove(pm, sva, eva); 2222 return; 2223 } 2224 2225 simple_lock(&pm->pm_lock); 2226 sva = sva & ~PGOFSET; 2227 for (; sva < eva; sva += NBPG) { 2228 #ifdef DEBUG 2229 /* 2230 * Is this part of the permanent 4MB mapping? 2231 */ 2232 if (pm == pmap_kernel() && sva >= ktext && 2233 sva < roundup(ekdata, 4 * MEG)) { 2234 prom_printf("pmap_protect: va=%08x in locked TLB\r\n", 2235 sva); 2236 OF_enter(); 2237 return; 2238 } 2239 #endif 2240 DPRINTF(PDB_CHANGEPROT, ("pmap_protect: va %p\n", 2241 (void *)(u_long)sva)); 2242 data = pseg_get(pm, sva); 2243 if ((data & TLB_V) == 0) { 2244 continue; 2245 } 2246 2247 pa = data & TLB_PA_MASK; 2248 DPRINTF(PDB_CHANGEPROT|PDB_REF, 2249 ("pmap_protect: va=%08x data=%08llx " 2250 "seg=%08x pte=%08x\r\n", 2251 (u_int)sva, (long long)pa, (int)va_to_seg(sva), 2252 (int)va_to_pte(sva))); 2253 2254 #ifdef DEBUG 2255 /* Catch this before the assertion */ 2256 if (data & TLB_NFO) { 2257 printf("pmap_protect: pm=%p NFO mapping " 2258 "va=%x data=%08llx\n", 2259 pm, (u_int)sva, (long long)data); 2260 Debugger(); 2261 } 2262 #endif 2263 pg = PHYS_TO_VM_PAGE(pa); 2264 if (pg) { 2265 /* Save REF/MOD info */ 2266 pv = &pg->mdpage.mdpg_pvh; 2267 if (data & TLB_ACCESS) 2268 pv->pv_va |= PV_REF; 2269 if (data & TLB_MODIFY) 2270 pv->pv_va |= PV_MOD; 2271 } 2272 2273 /* Just do the pmap and TSB, not the pv_list */ 2274 data &= ~(TLB_W|TLB_REAL_W); 2275 /* Turn *ON* write to wired mappings. */ 2276 if ((prot & (VM_PROT_WRITE|PMAP_WIRED)) == 2277 (VM_PROT_WRITE|PMAP_WIRED)) 2278 data |= (TLB_W|TLB_REAL_W); 2279 KASSERT((data & TLB_NFO) == 0); 2280 if (pseg_set(pm, sva, data, 0)) { 2281 printf("pmap_protect: gotten pseg empty!\n"); 2282 Debugger(); 2283 /* panic? */ 2284 } 2285 2286 if (!pm->pm_ctx && pm != pmap_kernel()) 2287 continue; 2288 i = ptelookup_va(sva); 2289 if (tsb[i].tag > 0 && 2290 tsb[i].tag == TSB_TAG(0, pm->pm_ctx, sva)) { 2291 tsb[i].data = data; 2292 KASSERT((tsb[i].data & TLB_NFO) == 0); 2293 } 2294 tlb_flush_pte(sva, pm->pm_ctx); 2295 } 2296 simple_unlock(&pm->pm_lock); 2297 pv_check(); 2298 } 2299 2300 /* 2301 * Extract the physical page address associated 2302 * with the given map/virtual_address pair. 2303 */ 2304 boolean_t 2305 pmap_extract(pm, va, pap) 2306 struct pmap *pm; 2307 vaddr_t va; 2308 paddr_t *pap; 2309 { 2310 paddr_t pa; 2311 2312 if (pm == pmap_kernel() && va >= kdata && va < roundup(ekdata, 4*MEG)) { 2313 /* Need to deal w/locked TLB entry specially. */ 2314 pa = (paddr_t)(kdatap - kdata + va); 2315 DPRINTF(PDB_EXTRACT, ("pmap_extract: va=%lx pa=%llx\n", 2316 (u_long)va, (unsigned long long)pa)); 2317 } else if (pm == pmap_kernel() && va >= ktext && va < ektext) { 2318 /* Need to deal w/locked TLB entry specially. */ 2319 pa = (paddr_t)(ktextp - ktext + va); 2320 DPRINTF(PDB_EXTRACT, ("pmap_extract: va=%lx pa=%llx\n", 2321 (u_long)va, (unsigned long long)pa)); 2322 } else { 2323 if (pm != pmap_kernel()) { 2324 simple_lock(&pm->pm_lock); 2325 } 2326 pa = pseg_get(pm, va) & TLB_PA_MASK; 2327 #ifdef DEBUG 2328 if (pmapdebug & PDB_EXTRACT) { 2329 paddr_t npa = ldxa((vaddr_t)&pm->pm_segs[va_to_seg(va)], 2330 ASI_PHYS_CACHED); 2331 printf("pmap_extract: va=%p segs[%ld]=%llx", 2332 (void *)(u_long)va, (long)va_to_seg(va), 2333 (unsigned long long)npa); 2334 if (npa) { 2335 npa = (paddr_t) 2336 ldxa((vaddr_t)&((paddr_t *)(u_long)npa) 2337 [va_to_dir(va)], 2338 ASI_PHYS_CACHED); 2339 printf(" segs[%ld][%ld]=%lx", 2340 (long)va_to_seg(va), 2341 (long)va_to_dir(va), (long)npa); 2342 } 2343 if (npa) { 2344 npa = (paddr_t) 2345 ldxa((vaddr_t)&((paddr_t *)(u_long)npa) 2346 [va_to_pte(va)], 2347 ASI_PHYS_CACHED); 2348 printf(" segs[%ld][%ld][%ld]=%lx", 2349 (long)va_to_seg(va), 2350 (long)va_to_dir(va), 2351 (long)va_to_pte(va), (long)npa); 2352 } 2353 printf(" pseg_get: %lx\n", (long)pa); 2354 } 2355 #endif 2356 if (pm != pmap_kernel()) { 2357 simple_unlock(&pm->pm_lock); 2358 } 2359 } 2360 if (pa == 0) 2361 return (FALSE); 2362 if (pap != NULL) 2363 *pap = pa + (va & PGOFSET); 2364 return (TRUE); 2365 } 2366 2367 /* 2368 * Return the number bytes that pmap_dumpmmu() will dump. 2369 */ 2370 int 2371 pmap_dumpsize() 2372 { 2373 int sz; 2374 2375 sz = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)); 2376 sz += memsize * sizeof(phys_ram_seg_t); 2377 2378 return btodb(sz + DEV_BSIZE - 1); 2379 } 2380 2381 /* 2382 * Write the mmu contents to the dump device. 2383 * This gets appended to the end of a crash dump since 2384 * there is no in-core copy of kernel memory mappings on a 4/4c machine. 2385 * 2386 * Write the core dump headers and MD data to the dump device. 2387 * We dump the following items: 2388 * 2389 * kcore_seg_t MI header defined in <sys/kcore.h>) 2390 * cpu_kcore_hdr_t MD header defined in <machine/kcore.h>) 2391 * phys_ram_seg_t[memsize] physical memory segments 2392 */ 2393 int 2394 pmap_dumpmmu(dump, blkno) 2395 int (*dump) __P((dev_t, daddr_t, caddr_t, size_t)); 2396 daddr_t blkno; 2397 { 2398 kcore_seg_t *kseg; 2399 cpu_kcore_hdr_t *kcpu; 2400 phys_ram_seg_t memseg; 2401 int error = 0; 2402 int i, memsegoffset; 2403 int buffer[dbtob(1) / sizeof(int)]; 2404 int *bp, *ep; 2405 2406 #define EXPEDITE(p,n) do { \ 2407 int *sp = (int *)(p); \ 2408 int sz = (n); \ 2409 while (sz > 0) { \ 2410 *bp++ = *sp++; \ 2411 if (bp >= ep) { \ 2412 error = (*dump)(dumpdev, blkno, \ 2413 (caddr_t)buffer, dbtob(1)); \ 2414 if (error != 0) \ 2415 return (error); \ 2416 ++blkno; \ 2417 bp = buffer; \ 2418 } \ 2419 sz -= 4; \ 2420 } \ 2421 } while (0) 2422 2423 /* Setup bookkeeping pointers */ 2424 bp = buffer; 2425 ep = &buffer[sizeof(buffer) / sizeof(buffer[0])]; 2426 2427 /* Fill in MI segment header */ 2428 kseg = (kcore_seg_t *)bp; 2429 CORE_SETMAGIC(*kseg, KCORE_MAGIC, MID_MACHINE, CORE_CPU); 2430 kseg->c_size = dbtob(pmap_dumpsize()) - ALIGN(sizeof(kcore_seg_t)); 2431 2432 /* Fill in MD segment header (interpreted by MD part of libkvm) */ 2433 kcpu = (cpu_kcore_hdr_t *)((long)bp + ALIGN(sizeof(kcore_seg_t))); 2434 kcpu->cputype = CPU_SUN4U; 2435 kcpu->kernbase = (u_int64_t)KERNBASE; 2436 kcpu->cpubase = (u_int64_t)CPUINFO_VA; 2437 2438 /* Describe the locked text segment */ 2439 kcpu->ktextbase = (u_int64_t)ktext; 2440 kcpu->ktextp = (u_int64_t)ktextp; 2441 kcpu->ktextsz = (u_int64_t)ektextp - ktextp; 2442 2443 /* Describe locked data segment */ 2444 kcpu->kdatabase = (u_int64_t)kdata; 2445 kcpu->kdatap = (u_int64_t)kdatap; 2446 kcpu->kdatasz = (u_int64_t)ekdatap - kdatap; 2447 2448 /* Now the memsegs */ 2449 kcpu->nmemseg = memsize; 2450 kcpu->memsegoffset = memsegoffset = ALIGN(sizeof(cpu_kcore_hdr_t)); 2451 2452 /* Now we need to point this at our kernel pmap. */ 2453 kcpu->nsegmap = STSZ; 2454 kcpu->segmapoffset = (u_int64_t)pmap_kernel()->pm_physaddr; 2455 2456 /* Note: we have assumed everything fits in buffer[] so far... */ 2457 bp = (int *)((long)kcpu + ALIGN(sizeof(cpu_kcore_hdr_t))); 2458 2459 for (i = 0; i < memsize; i++) { 2460 memseg.start = mem[i].start; 2461 memseg.size = mem[i].size; 2462 EXPEDITE(&memseg, sizeof(phys_ram_seg_t)); 2463 } 2464 2465 if (bp != buffer) 2466 error = (*dump)(dumpdev, blkno++, (caddr_t)buffer, dbtob(1)); 2467 2468 return (error); 2469 } 2470 2471 /* 2472 * Determine (non)existance of physical page 2473 */ 2474 int 2475 pmap_pa_exists(pa) 2476 paddr_t pa; 2477 { 2478 struct mem_region *mp; 2479 2480 /* Just go through physical memory list & see if we're there */ 2481 for (mp = mem; mp->size && mp->start <= pa; mp++) 2482 if (mp->start <= pa && mp->start + mp->size >= pa) 2483 return 1; 2484 return 0; 2485 } 2486 2487 /* 2488 * Lookup the appropriate TSB entry. 2489 * 2490 * Here is the full official pseudo code: 2491 * 2492 */ 2493 2494 #ifdef NOTYET 2495 int64 GenerateTSBPointer( 2496 int64 va, /* Missing VA */ 2497 PointerType type, /* 8K_POINTER or 16K_POINTER */ 2498 int64 TSBBase, /* TSB Register[63:13] << 13 */ 2499 Boolean split, /* TSB Register[12] */ 2500 int TSBSize) /* TSB Register[2:0] */ 2501 { 2502 int64 vaPortion; 2503 int64 TSBBaseMask; 2504 int64 splitMask; 2505 2506 /* TSBBaseMask marks the bits from TSB Base Reg */ 2507 TSBBaseMask = 0xffffffffffffe000 << 2508 (split? (TSBsize + 1) : TSBsize); 2509 2510 /* Shift va towards lsb appropriately and */ 2511 /* zero out the original va page offset */ 2512 vaPortion = (va >> ((type == 8K_POINTER)? 9: 12)) & 2513 0xfffffffffffffff0; 2514 2515 if (split) { 2516 /* There's only one bit in question for split */ 2517 splitMask = 1 << (13 + TSBsize); 2518 if (type == 8K_POINTER) 2519 /* Make sure we're in the lower half */ 2520 vaPortion &= ~splitMask; 2521 else 2522 /* Make sure we're in the upper half */ 2523 vaPortion |= splitMask; 2524 } 2525 return (TSBBase & TSBBaseMask) | (vaPortion & ~TSBBaseMask); 2526 } 2527 #endif 2528 /* 2529 * Of course, since we are not using a split TSB or variable page sizes, 2530 * we can optimize this a bit. 2531 * 2532 * The following only works for a unified 8K TSB. It will find the slot 2533 * for that particular va and return it. IT MAY BE FOR ANOTHER MAPPING! 2534 */ 2535 int 2536 ptelookup_va(va) 2537 vaddr_t va; 2538 { 2539 long tsbptr; 2540 #define TSBBASEMASK (0xffffffffffffe000LL << tsbsize) 2541 2542 tsbptr = (((va >> 9) & 0xfffffffffffffff0LL) & ~TSBBASEMASK); 2543 return (tsbptr / sizeof(pte_t)); 2544 } 2545 2546 #if notyet 2547 void 2548 tsb_enter(ctx, va, data) 2549 int ctx; 2550 int64_t va; 2551 int64_t data; 2552 { 2553 int i, s; 2554 int64_t pa; 2555 2556 i = ptelookup_va(va); 2557 s = splvm(); 2558 pa = tsb[i].data&TLB_PA_MASK; 2559 2560 /* 2561 * If we use fast DMMU access fault handlers to track 2562 * referenced and modified bits, we should save the 2563 * TSB entry's state here. Since we don't, we don't. 2564 */ 2565 /* Do not use global entries */ 2566 tsb[i].tag = TSB_TAG(0,ctx,va); 2567 tsb[i].data = data; 2568 tlb_flush_pte(va, ctx); /* Force reload -- protections may be changed */ 2569 splx(s); 2570 } 2571 #endif 2572 2573 /* 2574 * Do whatever is needed to sync the MOD/REF flags 2575 */ 2576 2577 boolean_t 2578 pmap_clear_modify(pg) 2579 struct vm_page *pg; 2580 { 2581 pv_entry_t pv; 2582 int i, changed = 0; 2583 #ifdef DEBUG 2584 int modified = 0; 2585 2586 DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_clear_modify(%p)\n", pg)); 2587 #endif 2588 2589 #if defined(DEBUG) 2590 modified = pmap_is_modified(pg); 2591 #endif 2592 /* Clear all mappings */ 2593 pv = &pg->mdpage.mdpg_pvh; 2594 #ifdef DEBUG 2595 if (pv->pv_va & PV_MOD) 2596 pv->pv_va |= PV_WE; /* Remember this was modified */ 2597 #endif 2598 if (pv->pv_va & PV_MOD) 2599 changed |= 1; 2600 pv->pv_va &= ~(PV_MOD); 2601 #ifdef DEBUG 2602 if (pv->pv_next && !pv->pv_pmap) { 2603 printf("pmap_clear_modify: npv but no pmap for pv %p\n", pv); 2604 Debugger(); 2605 } 2606 #endif 2607 if (pv->pv_pmap != NULL) 2608 for (; pv; pv = pv->pv_next) { 2609 int64_t data; 2610 struct pmap *pmap = pv->pv_pmap; 2611 vaddr_t va = pv->pv_va & PV_VAMASK; 2612 2613 simple_lock(&pmap->pm_lock); 2614 /* First clear the mod bit in the PTE and make it R/O */ 2615 data = pseg_get(pmap, va); 2616 /* Need to both clear the modify and write bits */ 2617 if (data & TLB_MODIFY) 2618 changed |= 1; 2619 #ifdef HWREF 2620 data &= ~(TLB_MODIFY|TLB_W); 2621 #else 2622 data &= ~(TLB_MODIFY|TLB_W|TLB_REAL_W); 2623 #endif 2624 KASSERT((data & TLB_NFO) == 0); 2625 if (pseg_set(pmap, va, data, 0)) { 2626 printf("pmap_clear_modify: pseg empty!\n"); 2627 Debugger(); 2628 /* panic? */ 2629 } 2630 if (pmap->pm_ctx || pmap == pmap_kernel()) { 2631 i = ptelookup_va(va); 2632 if (tsb[i].tag == TSB_TAG(0, pmap->pm_ctx, va)) 2633 tsb[i].data = 0; 2634 tlb_flush_pte(va, pmap->pm_ctx); 2635 } 2636 /* Then clear the mod bit in the pv */ 2637 if (pv->pv_va & PV_MOD) 2638 changed |= 1; 2639 pv->pv_va &= ~(PV_MOD); 2640 simple_unlock(&pmap->pm_lock); 2641 } 2642 pv_check(); 2643 #ifdef DEBUG 2644 if (pmap_is_modified(pg)) { 2645 printf("pmap_clear_modify(): %p still modified!\n", pg); 2646 Debugger(); 2647 } 2648 DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_clear_modify: pg %p %s\n", pg, 2649 (changed ? "was modified" : "was not modified"))); 2650 if (modified != changed) { 2651 printf("pmap_clear_modify: modified %d changed %d\n", 2652 modified, changed); 2653 Debugger(); 2654 } else return (modified); 2655 #endif 2656 return (changed); 2657 } 2658 2659 boolean_t 2660 pmap_clear_reference(pg) 2661 struct vm_page *pg; 2662 { 2663 paddr_t pa = VM_PAGE_TO_PHYS(pg); 2664 pv_entry_t pv; 2665 int i, changed = 0; 2666 #ifdef DEBUG 2667 int referenced = 0; 2668 #endif 2669 2670 #ifdef DEBUG 2671 DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_clear_reference(%p)\n", pg)); 2672 referenced = pmap_is_referenced(pg); 2673 #endif 2674 /* Clear all references */ 2675 pv = &pg->mdpage.mdpg_pvh; 2676 #ifdef NOT_DEBUG 2677 if (pv->pv_va & PV_MOD) 2678 printf("pmap_clear_reference(): pg %p still modified\n", pg); 2679 #endif 2680 if (pv->pv_va & PV_REF) 2681 changed |= 1; 2682 pv->pv_va &= ~(PV_REF); 2683 #ifdef DEBUG 2684 if (pv->pv_next && !pv->pv_pmap) { 2685 printf("pmap_clear_reference: npv but no pmap for pv %p\n", pv); 2686 Debugger(); 2687 } 2688 #endif 2689 if (pv->pv_pmap != NULL) { 2690 for (; pv; pv = pv->pv_next) { 2691 int64_t data; 2692 struct pmap *pmap = pv->pv_pmap; 2693 vaddr_t va = pv->pv_va & PV_VAMASK; 2694 2695 simple_lock(&pmap->pm_lock); 2696 data = pseg_get(pmap, va); 2697 DPRINTF(PDB_CHANGEPROT, 2698 ("clearing ref pm:%p va:%p ctx:%lx data:%llx\n", 2699 pmap, (void *)(u_long)va, (u_long)pmap->pm_ctx, 2700 (long long)data)); 2701 #ifdef HWREF 2702 if (data & TLB_ACCESS) 2703 changed |= 1; 2704 data &= ~TLB_ACCESS; 2705 #else 2706 if (data < 0) 2707 changed |= 1; 2708 data = 0; 2709 #endif 2710 KASSERT((data & TLB_NFO) == 0); 2711 if (pseg_set(pmap, va, data, 0)) { 2712 printf("pmap_clear_reference: pseg empty!\n"); 2713 Debugger(); 2714 /* panic? */ 2715 } 2716 if (pmap->pm_ctx || pmap == pmap_kernel()) { 2717 i = ptelookup_va(va); 2718 2719 /* Invalidate our TSB entry since ref info 2720 is in the PTE */ 2721 if (tsb[i].tag == TSB_TAG(0, pmap->pm_ctx, va)) 2722 tsb[i].data = 0; 2723 tlb_flush_pte(va, pmap->pm_ctx); 2724 } 2725 if (pv->pv_va & PV_REF) 2726 changed |= 1; 2727 pv->pv_va &= ~(PV_REF); 2728 simple_unlock(&pmap->pm_lock); 2729 } 2730 } 2731 dcache_flush_page(pa); 2732 pv_check(); 2733 #ifdef DEBUG 2734 if (pmap_is_referenced(pg)) { 2735 printf("pmap_clear_reference(): %p still referenced!\n", pg); 2736 Debugger(); 2737 } 2738 DPRINTF(PDB_CHANGEPROT|PDB_REF, 2739 ("pmap_clear_reference: pg %p %s\n", pg, 2740 (changed ? "was referenced" : "was not referenced"))); 2741 if (referenced != changed) { 2742 printf("pmap_clear_reference: referenced %d changed %d\n", 2743 referenced, changed); 2744 Debugger(); 2745 } else return (referenced); 2746 #endif 2747 return (changed); 2748 } 2749 2750 boolean_t 2751 pmap_is_modified(pg) 2752 struct vm_page *pg; 2753 { 2754 pv_entry_t pv, npv; 2755 int i = 0; 2756 2757 /* Check if any mapping has been modified */ 2758 pv = &pg->mdpage.mdpg_pvh; 2759 if (pv->pv_va & PV_MOD) 2760 i = 1; 2761 #ifdef HWREF 2762 #ifdef DEBUG 2763 if (pv->pv_next && !pv->pv_pmap) { 2764 printf("pmap_is_modified: npv but no pmap for pv %p\n", pv); 2765 Debugger(); 2766 } 2767 #endif 2768 if (!i && pv->pv_pmap != NULL) 2769 for (npv = pv; i == 0 && npv && npv->pv_pmap; 2770 npv = npv->pv_next) { 2771 int64_t data; 2772 2773 data = pseg_get(npv->pv_pmap, npv->pv_va & PV_VAMASK); 2774 if (data & TLB_MODIFY) 2775 i = 1; 2776 2777 /* Migrate modify info to head pv */ 2778 if (npv->pv_va & PV_MOD) 2779 i = 1; 2780 npv->pv_va &= ~PV_MOD; 2781 } 2782 /* Save modify info */ 2783 if (i) 2784 pv->pv_va |= PV_MOD; 2785 #ifdef DEBUG 2786 if (i) 2787 pv->pv_va |= PV_WE; 2788 #endif 2789 #endif 2790 2791 DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_is_modified(%p) = %d\n", pg, i)); 2792 pv_check(); 2793 return (i); 2794 } 2795 2796 boolean_t 2797 pmap_is_referenced(pg) 2798 struct vm_page *pg; 2799 { 2800 pv_entry_t pv, npv; 2801 int i = 0; 2802 2803 /* Check if any mapping has been referenced */ 2804 pv = &pg->mdpage.mdpg_pvh; 2805 if (pv->pv_va & PV_REF) 2806 i = 1; 2807 #ifdef HWREF 2808 #ifdef DEBUG 2809 if (pv->pv_next && !pv->pv_pmap) { 2810 printf("pmap_is_referenced: npv but no pmap for pv %p\n", pv); 2811 Debugger(); 2812 } 2813 #endif 2814 if (!i && (pv->pv_pmap != NULL)) 2815 for (npv = pv; npv; npv = npv->pv_next) { 2816 int64_t data; 2817 2818 data = pseg_get(npv->pv_pmap, npv->pv_va & PV_VAMASK); 2819 if (data & TLB_ACCESS) 2820 i = 1; 2821 2822 /* Migrate ref info to head pv */ 2823 if (npv->pv_va & PV_REF) 2824 i = 1; 2825 npv->pv_va &= ~PV_REF; 2826 } 2827 /* Save ref info */ 2828 if (i) 2829 pv->pv_va |= PV_REF; 2830 #endif 2831 2832 DPRINTF(PDB_CHANGEPROT|PDB_REF, 2833 ("pmap_is_referenced(%p) = %d\n", pg, i)); 2834 pv_check(); 2835 return i; 2836 } 2837 2838 2839 2840 /* 2841 * Routine: pmap_unwire 2842 * Function: Clear the wired attribute for a map/virtual-address 2843 * pair. 2844 * In/out conditions: 2845 * The mapping must already exist in the pmap. 2846 */ 2847 void 2848 pmap_unwire(pmap, va) 2849 pmap_t pmap; 2850 vaddr_t va; 2851 { 2852 int64_t data; 2853 2854 DPRINTF(PDB_MMU_STEAL, ("pmap_unwire(%p, %lx)\n", pmap, va)); 2855 2856 #ifdef DEBUG 2857 /* 2858 * Is this part of the permanent 4MB mapping? 2859 */ 2860 if (pmap == pmap_kernel() && va >= ktext && 2861 va < roundup(ekdata, 4*MEG)) { 2862 prom_printf("pmap_unwire: va=%08x in locked TLB\r\n", va); 2863 OF_enter(); 2864 return; 2865 } 2866 #endif 2867 simple_lock(&pmap->pm_lock); 2868 data = pseg_get(pmap, va & PV_VAMASK); 2869 data &= ~TLB_TSB_LOCK; 2870 if (pseg_set(pmap, va & PV_VAMASK, data, 0)) { 2871 printf("pmap_unwire: gotten pseg empty!\n"); 2872 Debugger(); 2873 /* panic? */ 2874 } 2875 simple_unlock(&pmap->pm_lock); 2876 pv_check(); 2877 } 2878 2879 /* 2880 * Lower the protection on the specified physical page. 2881 * 2882 * Never enable writing as it will break COW 2883 */ 2884 2885 void 2886 pmap_page_protect(pg, prot) 2887 struct vm_page *pg; 2888 vm_prot_t prot; 2889 { 2890 int64_t clear, set; 2891 int64_t data = 0; 2892 paddr_t pa = VM_PAGE_TO_PHYS(pg); 2893 pv_entry_t pv, npv, firstpv; 2894 struct pmap *pmap; 2895 vaddr_t va; 2896 int i; 2897 boolean_t needflush = FALSE; 2898 2899 DPRINTF(PDB_CHANGEPROT, 2900 ("pmap_page_protect: pg %p prot %x\n", pg, prot)); 2901 2902 pv = &pg->mdpage.mdpg_pvh; 2903 if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { 2904 /* copy_on_write */ 2905 2906 set = TLB_V; 2907 clear = TLB_REAL_W|TLB_W; 2908 if (VM_PROT_EXECUTE & prot) 2909 set |= TLB_EXEC; 2910 else 2911 clear |= TLB_EXEC; 2912 if (VM_PROT_EXECUTE == prot) 2913 set |= TLB_EXEC_ONLY; 2914 2915 #ifdef DEBUG 2916 if (pv->pv_next && !pv->pv_pmap) { 2917 printf("pmap_page_protect: no pmap for pv %p\n", pv); 2918 Debugger(); 2919 } 2920 #endif 2921 if (pv->pv_pmap != NULL) { 2922 for (; pv; pv = pv->pv_next) { 2923 pmap = pv->pv_pmap; 2924 va = pv->pv_va & PV_VAMASK; 2925 2926 simple_lock(&pmap->pm_lock); 2927 DPRINTF(PDB_CHANGEPROT | PDB_REF, 2928 ("pmap_page_protect: " 2929 "RO va %p of pg %p...\n", 2930 (void *)(u_long)pv->pv_va, pg)); 2931 data = pseg_get(pmap, va); 2932 2933 /* Save REF/MOD info */ 2934 if (data & TLB_ACCESS) 2935 pv->pv_va |= PV_REF; 2936 if (data & TLB_MODIFY) 2937 pv->pv_va |= PV_MOD; 2938 2939 data &= ~clear; 2940 data |= set; 2941 KASSERT((data & TLB_NFO) == 0); 2942 if (pseg_set(pmap, va, data, 0)) { 2943 printf("pmap_page_protect: " 2944 "pseg empty!\n"); 2945 Debugger(); 2946 /* panic? */ 2947 } 2948 if (pmap->pm_ctx || pmap == pmap_kernel()) { 2949 i = ptelookup_va(va); 2950 2951 /* 2952 * since we already know the va 2953 * for each mapping, we don't need to 2954 * scan the entire TSB. 2955 */ 2956 2957 if (tsb[i].tag == 2958 TSB_TAG(0, pmap->pm_ctx, va)) 2959 tsb[i].data = 0; 2960 tlb_flush_pte(va, pmap->pm_ctx); 2961 } 2962 simple_unlock(&pmap->pm_lock); 2963 } 2964 } 2965 } else { 2966 /* remove mappings */ 2967 DPRINTF(PDB_REMOVE, 2968 ("pmap_page_protect: demapping pg %p\n", pg)); 2969 2970 firstpv = pv; 2971 2972 /* First remove the entire list of continuation pv's*/ 2973 for (npv = pv->pv_next; npv; npv = pv->pv_next) { 2974 pmap = npv->pv_pmap; 2975 va = npv->pv_va & PV_VAMASK; 2976 2977 /* We're removing npv from pv->pv_next */ 2978 simple_lock(&pmap->pm_lock); 2979 DPRINTF(PDB_CHANGEPROT|PDB_REF|PDB_REMOVE, 2980 ("pmap_page_protect: " 2981 "demap va %p of pg %p in pmap %p...\n", 2982 (void *)(u_long)va, pg, pmap)); 2983 2984 /* clear the entry in the page table */ 2985 data = pseg_get(pmap, va); 2986 2987 /* Save ref/mod info */ 2988 if (data & TLB_ACCESS) 2989 firstpv->pv_va |= PV_REF; 2990 if (data & TLB_MODIFY) 2991 firstpv->pv_va |= PV_MOD; 2992 /* Clear mapping */ 2993 if (pseg_set(pmap, va, 0, 0)) { 2994 printf("pmap_page_protect: pseg empty!\n"); 2995 Debugger(); 2996 /* panic? */ 2997 } 2998 if (pmap->pm_ctx || pmap == pmap_kernel()) { 2999 /* clear the entry in the TSB */ 3000 i = ptelookup_va(va); 3001 3002 /* 3003 * since we already know the va for 3004 * each mapping we don't need to scan 3005 * the entire TSB. 3006 */ 3007 3008 if (tsb[i].tag == TSB_TAG(0, pmap->pm_ctx, va)) 3009 tsb[i].data = 0; 3010 tlb_flush_pte(va, pmap->pm_ctx); 3011 } 3012 if (pmap->pm_refs > 0) { 3013 needflush = TRUE; 3014 } 3015 simple_unlock(&pmap->pm_lock); 3016 3017 /* free the pv */ 3018 pv->pv_next = npv->pv_next; 3019 pool_put(&pmap_pv_pool, npv); 3020 } 3021 3022 pv = firstpv; 3023 3024 /* Then remove the primary pv */ 3025 #ifdef DEBUG 3026 if (pv->pv_next && !pv->pv_pmap) { 3027 printf("pmap_page_protect: no pmap for pv %p\n", pv); 3028 Debugger(); 3029 } 3030 #endif 3031 if (pv->pv_pmap != NULL) { 3032 pmap = pv->pv_pmap; 3033 va = pv->pv_va & PV_VAMASK; 3034 3035 simple_lock(&pmap->pm_lock); 3036 DPRINTF(PDB_CHANGEPROT|PDB_REF|PDB_REMOVE, 3037 ("pmap_page_protect: " 3038 "demap va %p of pg %p from pm %p...\n", 3039 (void *)(u_long)va, pg, pmap)); 3040 3041 data = pseg_get(pmap, va); 3042 /* Save ref/mod info */ 3043 if (data & TLB_ACCESS) 3044 pv->pv_va |= PV_REF; 3045 if (data & TLB_MODIFY) 3046 pv->pv_va |= PV_MOD; 3047 if (pseg_set(pmap, va, 0, 0)) { 3048 printf("pmap_page_protect: pseg empty!\n"); 3049 Debugger(); 3050 /* panic? */ 3051 } 3052 if (pv->pv_pmap->pm_ctx || 3053 pv->pv_pmap == pmap_kernel()) { 3054 i = ptelookup_va(va); 3055 3056 /* 3057 * since we already know the va for 3058 * each mapping we don't need to scan 3059 * the entire TSB. 3060 */ 3061 3062 if (tsb[i].tag == TSB_TAG(0, pmap->pm_ctx, va)) 3063 tsb[i].data = 0; 3064 tlb_flush_pte(va, pmap->pm_ctx); 3065 } 3066 if (pmap->pm_refs > 0) { 3067 needflush = TRUE; 3068 } 3069 simple_unlock(&pmap->pm_lock); 3070 npv = pv->pv_next; 3071 /* dump the first pv */ 3072 if (npv) { 3073 /* First save mod/ref bits */ 3074 pv->pv_va |= npv->pv_va & PV_MASK; 3075 pv->pv_next = npv->pv_next; 3076 pv->pv_pmap = npv->pv_pmap; 3077 pool_put(&pmap_pv_pool, npv); 3078 } else { 3079 pv->pv_pmap = NULL; 3080 pv->pv_next = NULL; 3081 } 3082 } 3083 if (needflush) { 3084 dcache_flush_page(pa); 3085 } 3086 } 3087 /* We should really only flush the pages we demapped. */ 3088 pv_check(); 3089 } 3090 3091 /* 3092 * count pages in pmap -- this can be slow. 3093 */ 3094 int 3095 pmap_count_res(pm) 3096 struct pmap *pm; 3097 { 3098 int64_t data; 3099 paddr_t *pdir, *ptbl; 3100 int i, j, k, n; 3101 3102 /* Almost the same as pmap_collect() */ 3103 /* Don't want one of these pages reused while we're reading it. */ 3104 simple_lock(&pm->pm_lock); 3105 n = 0; 3106 for (i = 0; i < STSZ; i++) { 3107 pdir = (paddr_t *)(u_long)ldxa((vaddr_t)&pm->pm_segs[i], 3108 ASI_PHYS_CACHED); 3109 if (pdir == NULL) { 3110 continue; 3111 } 3112 for (k = 0; k < PDSZ; k++) { 3113 ptbl = (paddr_t *)(u_long)ldxa((vaddr_t)&pdir[k], 3114 ASI_PHYS_CACHED); 3115 if (ptbl == NULL) { 3116 continue; 3117 } 3118 for (j = 0; j < PTSZ; j++) { 3119 data = (int64_t)ldxa((vaddr_t)&ptbl[j], 3120 ASI_PHYS_CACHED); 3121 if (data & TLB_V) 3122 n++; 3123 } 3124 } 3125 } 3126 simple_unlock(&pm->pm_lock); 3127 return n; 3128 } 3129 3130 /* 3131 * count wired pages in pmap -- this can be slow. 3132 */ 3133 int 3134 pmap_count_wired(pm) 3135 struct pmap *pm; 3136 { 3137 int64_t data; 3138 paddr_t *pdir, *ptbl; 3139 int i, j, k, n; 3140 3141 /* Almost the same as pmap_collect() */ 3142 /* Don't want one of these pages reused while we're reading it. */ 3143 simple_lock(&pm->pm_lock); 3144 n = 0; 3145 for (i = 0; i < STSZ; i++) { 3146 pdir = (paddr_t *)(u_long)ldxa((vaddr_t)&pm->pm_segs[i], 3147 ASI_PHYS_CACHED); 3148 if (pdir == NULL) { 3149 continue; 3150 } 3151 for (k = 0; k < PDSZ; k++) { 3152 ptbl = (paddr_t *)(u_long)ldxa((vaddr_t)&pdir[k], 3153 ASI_PHYS_CACHED); 3154 if (ptbl == NULL) { 3155 continue; 3156 } 3157 for (j = 0; j < PTSZ; j++) { 3158 data = (int64_t)ldxa((vaddr_t)&ptbl[j], 3159 ASI_PHYS_CACHED); 3160 if (data & TLB_TSB_LOCK) 3161 n++; 3162 } 3163 } 3164 } 3165 simple_unlock(&pm->pm_lock); 3166 return n; 3167 } 3168 3169 void 3170 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 3171 { 3172 3173 blast_icache(); 3174 } 3175 3176 /* 3177 * Allocate a hardware context to the given pmap. 3178 */ 3179 int 3180 ctx_alloc(pm) 3181 struct pmap *pm; 3182 { 3183 int64_t *tsbaddr; 3184 int i, ctx; 3185 3186 simple_lock(&pm->pm_lock); 3187 ctx = pmap_next_ctx++; 3188 3189 /* 3190 * if we have run out of contexts, remove all user entries from 3191 * the TSB, TLB and dcache and start over with context 1 again. 3192 * note that we skip the highest-numbered context because 3193 * the PROM keeps some locked TLB entries there. 3194 */ 3195 3196 if (ctx == numctx - 1) { 3197 write_user_windows(); 3198 while (!LIST_EMPTY(&pmap_ctxlist)) { 3199 ctx_free(LIST_FIRST(&pmap_ctxlist)); 3200 } 3201 for (i = TSBENTS - 1; i >= 0; i--) { 3202 if (TSB_TAG_CTX(tsb[i].tag) != 0) { 3203 tsbaddr = &tsb[i].data; 3204 __asm __volatile("clrx [%0]" 3205 : : "r" (tsbaddr) : "memory"); 3206 } 3207 } 3208 tlb_flush_all(); 3209 blast_dcache(); 3210 ctx = 1; 3211 pmap_next_ctx = 2; 3212 } 3213 ctxbusy[ctx] = pm->pm_physaddr; 3214 LIST_INSERT_HEAD(&pmap_ctxlist, pm, pm_list); 3215 pm->pm_ctx = ctx; 3216 simple_unlock(&pm->pm_lock); 3217 DPRINTF(PDB_CTX_ALLOC, ("ctx_alloc: allocated ctx %d\n", ctx)); 3218 return ctx; 3219 } 3220 3221 /* 3222 * Give away a context. 3223 */ 3224 void 3225 ctx_free(pm) 3226 struct pmap *pm; 3227 { 3228 int oldctx; 3229 3230 oldctx = pm->pm_ctx; 3231 if (oldctx == 0) { 3232 return; 3233 } 3234 3235 #ifdef DIAGNOSTIC 3236 if (pm == pmap_kernel()) 3237 panic("ctx_free: freeing kernel context"); 3238 if (ctxbusy[oldctx] == 0) 3239 printf("ctx_free: freeing free context %d\n", oldctx); 3240 if (ctxbusy[oldctx] != pm->pm_physaddr) { 3241 printf("ctx_free: freeing someone else's context\n " 3242 "ctxbusy[%d] = %p, pm(%p)->pm_ctx = %p\n", 3243 oldctx, (void *)(u_long)ctxbusy[oldctx], pm, 3244 (void *)(u_long)pm->pm_physaddr); 3245 Debugger(); 3246 } 3247 #endif 3248 /* We should verify it has not been stolen and reallocated... */ 3249 DPRINTF(PDB_CTX_ALLOC, ("ctx_free: freeing ctx %d\n", oldctx)); 3250 ctxbusy[oldctx] = NULL; 3251 pm->pm_ctx = 0; 3252 LIST_REMOVE(pm, pm_list); 3253 } 3254 3255 /* 3256 * Enter the pmap and virtual address into the 3257 * physical to virtual map table. 3258 * 3259 * We enter here with the pmap locked. 3260 */ 3261 3262 void 3263 pmap_enter_pv(pmap, va, pa, pg, npv) 3264 struct pmap *pmap; 3265 vaddr_t va; 3266 paddr_t pa; 3267 struct vm_page *pg; 3268 pv_entry_t npv; 3269 { 3270 pv_entry_t pvh; 3271 3272 pvh = &pg->mdpage.mdpg_pvh; 3273 DPRINTF(PDB_ENTER, ("pmap_enter: pvh %p: was %lx/%p/%p\n", 3274 pvh, pvh->pv_va, pvh->pv_pmap, pvh->pv_next)); 3275 if (pvh->pv_pmap == NULL) { 3276 3277 /* 3278 * No entries yet, use header as the first entry 3279 */ 3280 DPRINTF(PDB_ENTER, ("pmap_enter: first pv: pmap %p va %lx\n", 3281 pmap, va)); 3282 ENTER_STAT(firstpv); 3283 PV_SETVA(pvh, va); 3284 pvh->pv_pmap = pmap; 3285 pvh->pv_next = NULL; 3286 KASSERT(npv == NULL); 3287 } else { 3288 if (pg->loan_count == 0 && !(pvh->pv_va & PV_ALIAS)) { 3289 3290 /* 3291 * There is at least one other VA mapping this page. 3292 * Check if they are cache index compatible. If not 3293 * remove all mappings, flush the cache and set page 3294 * to be mapped uncached. Caching will be restored 3295 * when pages are mapped compatible again. 3296 */ 3297 if ((pvh->pv_va ^ va) & VA_ALIAS_MASK) { 3298 pvh->pv_va |= PV_ALIAS; 3299 pmap_page_cache(pmap, pa, 0); 3300 ENTER_STAT(ci); 3301 } 3302 } 3303 3304 /* 3305 * There is at least one other VA mapping this page. 3306 * Place this entry after the header. 3307 */ 3308 3309 DPRINTF(PDB_ENTER, ("pmap_enter: new pv: pmap %p va %lx\n", 3310 pmap, va)); 3311 npv->pv_pmap = pmap; 3312 npv->pv_va = va & PV_VAMASK; 3313 npv->pv_next = pvh->pv_next; 3314 pvh->pv_next = npv; 3315 3316 if (!npv->pv_next) 3317 ENTER_STAT(secondpv); 3318 } 3319 } 3320 3321 /* 3322 * Remove a physical to virtual address translation. 3323 */ 3324 3325 pv_entry_t 3326 pmap_remove_pv(pmap, va, pg) 3327 struct pmap *pmap; 3328 vaddr_t va; 3329 struct vm_page *pg; 3330 { 3331 pv_entry_t pvh, npv, pv; 3332 int64_t data = 0; 3333 3334 pvh = &pg->mdpage.mdpg_pvh; 3335 3336 DPRINTF(PDB_REMOVE, ("pmap_remove_pv(pm=%p, va=%p, pg=%p)\n", pmap, 3337 (void *)(u_long)va, pg)); 3338 pv_check(); 3339 3340 /* 3341 * Remove page from the PV table. 3342 * If it is the first entry on the list, it is actually 3343 * in the header and we must copy the following entry up 3344 * to the header. Otherwise we must search the list for 3345 * the entry. In either case we free the now unused entry. 3346 */ 3347 if (pmap == pvh->pv_pmap && PV_MATCH(pvh, va)) { 3348 data = pseg_get(pvh->pv_pmap, pvh->pv_va & PV_VAMASK); 3349 npv = pvh->pv_next; 3350 if (npv) { 3351 /* First save mod/ref bits */ 3352 pvh->pv_va = (pvh->pv_va & PV_MASK) | npv->pv_va; 3353 pvh->pv_next = npv->pv_next; 3354 pvh->pv_pmap = npv->pv_pmap; 3355 } else { 3356 pvh->pv_pmap = NULL; 3357 pvh->pv_next = NULL; 3358 pvh->pv_va &= (PV_REF|PV_MOD); 3359 } 3360 #ifdef DEBUG 3361 remove_stats.pvfirst++; 3362 #endif 3363 } else { 3364 for (pv = pvh, npv = pvh->pv_next; npv; 3365 pv = npv, npv = npv->pv_next) { 3366 #ifdef DEBUG 3367 remove_stats.pvsearch++; 3368 #endif 3369 if (pmap == npv->pv_pmap && PV_MATCH(npv, va)) 3370 break; 3371 } 3372 pv->pv_next = npv->pv_next; 3373 data = pseg_get(npv->pv_pmap, npv->pv_va & PV_VAMASK); 3374 } 3375 3376 /* Save ref/mod info */ 3377 if (data & TLB_ACCESS) 3378 pvh->pv_va |= PV_REF; 3379 if (data & TLB_MODIFY) 3380 pvh->pv_va |= PV_MOD; 3381 3382 /* Check to see if the alias went away */ 3383 if (pvh->pv_va & PV_ALIAS) { 3384 pvh->pv_va &= ~PV_ALIAS; 3385 for (pv = pvh; pv; pv = pv->pv_next) { 3386 if ((pv->pv_va ^ pvh->pv_va) & VA_ALIAS_MASK) { 3387 pvh->pv_va |= PV_ALIAS; 3388 break; 3389 } 3390 } 3391 if (!(pvh->pv_va & PV_ALIAS)) 3392 pmap_page_cache(pmap, VM_PAGE_TO_PHYS(pg), 1); 3393 } 3394 pv_check(); 3395 return npv; 3396 } 3397 3398 /* 3399 * pmap_page_cache: 3400 * 3401 * Change all mappings of a page to cached/uncached. 3402 */ 3403 void 3404 pmap_page_cache(pm, pa, mode) 3405 struct pmap *pm; 3406 paddr_t pa; 3407 int mode; 3408 { 3409 struct vm_page *pg; 3410 pv_entry_t pv; 3411 vaddr_t va; 3412 int i; 3413 3414 DPRINTF(PDB_ENTER, ("pmap_page_uncache(%llx)\n", 3415 (unsigned long long)pa)); 3416 pg = PHYS_TO_VM_PAGE(pa); 3417 pv = &pg->mdpage.mdpg_pvh; 3418 while (pv) { 3419 va = pv->pv_va & PV_VAMASK; 3420 if (pv->pv_pmap != pm) 3421 simple_lock(&pv->pv_pmap->pm_lock); 3422 if (pv->pv_va & PV_NC) { 3423 /* Non-cached -- I/O mapping */ 3424 if (pseg_set(pv->pv_pmap, va, 3425 pseg_get(pv->pv_pmap, va) & 3426 ~(TLB_CV|TLB_CP), 0)) { 3427 printf("pmap_page_cache: pseg empty!\n"); 3428 Debugger(); 3429 /* panic? */ 3430 } 3431 } else if (mode && (!(pv->pv_va & PV_NVC))) { 3432 /* Enable caching */ 3433 if (pseg_set(pv->pv_pmap, va, 3434 pseg_get(pv->pv_pmap, va) | TLB_CV, 0)) { 3435 printf("pmap_page_cache: pseg empty!\n"); 3436 Debugger(); 3437 /* panic? */ 3438 } 3439 } else { 3440 /* Disable caching */ 3441 if (pseg_set(pv->pv_pmap, va, 3442 pseg_get(pv->pv_pmap, va) & ~TLB_CV, 0)) { 3443 printf("pmap_page_cache: pseg empty!\n"); 3444 Debugger(); 3445 /* panic? */ 3446 } 3447 } 3448 if (pv->pv_pmap != pm) 3449 simple_unlock(&pv->pv_pmap->pm_lock); 3450 if (pv->pv_pmap->pm_ctx || pv->pv_pmap == pmap_kernel()) { 3451 i = ptelookup_va(va); 3452 if (tsb[i].tag > 0 && tsb[i].tag == 3453 TSB_TAG(0, pv->pv_pmap->pm_ctx, va)) { 3454 tsb[i].data = 0; 3455 } 3456 /* Force reload -- cache bits have changed */ 3457 tlb_flush_pte(va, pv->pv_pmap->pm_ctx); 3458 } 3459 pv = pv->pv_next; 3460 } 3461 } 3462 3463 3464 static int 3465 pmap_get_page(paddr_t *p) 3466 { 3467 struct vm_page *pg; 3468 paddr_t pa; 3469 3470 if (uvm.page_init_done) { 3471 pg = uvm_pagealloc(NULL, 0, NULL, 3472 UVM_PGA_ZERO | UVM_PGA_USERESERVE); 3473 if (pg == NULL) 3474 return (0); 3475 pa = VM_PAGE_TO_PHYS(pg); 3476 } else { 3477 if (!uvm_page_physget(&pa)) 3478 return (0); 3479 pmap_zero_page(pa); 3480 } 3481 *p = pa; 3482 return (1); 3483 } 3484 3485 static void 3486 pmap_free_page(paddr_t pa) 3487 { 3488 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3489 3490 uvm_pagefree(pg); 3491 } 3492 3493 3494 #ifdef DDB 3495 3496 void db_dump_pv __P((db_expr_t, int, db_expr_t, char *)); 3497 void 3498 db_dump_pv(addr, have_addr, count, modif) 3499 db_expr_t addr; 3500 int have_addr; 3501 db_expr_t count; 3502 char *modif; 3503 { 3504 struct vm_page *pg; 3505 struct pv_entry *pv; 3506 3507 if (!have_addr) { 3508 db_printf("Need addr for pv\n"); 3509 return; 3510 } 3511 3512 pg = PHYS_TO_VM_PAGE((paddr_t)addr); 3513 if (pg == NULL) { 3514 db_printf("page is not managed\n"); 3515 return; 3516 } 3517 for (pv = &pg->mdpage.mdpg_pvh; pv; pv = pv->pv_next) 3518 db_printf("pv@%p: next=%p pmap=%p va=0x%llx\n", 3519 pv, pv->pv_next, pv->pv_pmap, 3520 (unsigned long long)pv->pv_va); 3521 } 3522 3523 #endif 3524 3525 #ifdef DEBUG 3526 /* 3527 * Test ref/modify handling. */ 3528 void pmap_testout __P((void)); 3529 void 3530 pmap_testout() 3531 { 3532 vaddr_t va; 3533 volatile int *loc; 3534 int val = 0; 3535 paddr_t pa; 3536 struct vm_page *pg; 3537 int ref, mod; 3538 3539 /* Allocate a page */ 3540 va = (vaddr_t)(vmmap - NBPG); 3541 KASSERT(va != NULL); 3542 loc = (int*)va; 3543 3544 pmap_get_page(&pa); 3545 pg = PHYS_TO_VM_PAGE(pa); 3546 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, VM_PROT_ALL); 3547 pmap_update(pmap_kernel()); 3548 3549 /* Now clear reference and modify */ 3550 ref = pmap_clear_reference(pg); 3551 mod = pmap_clear_modify(pg); 3552 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3553 (void *)(u_long)va, (long)pa, 3554 ref, mod); 3555 3556 /* Check it's properly cleared */ 3557 ref = pmap_is_referenced(pg); 3558 mod = pmap_is_modified(pg); 3559 printf("Checking cleared page: ref %d, mod %d\n", 3560 ref, mod); 3561 3562 /* Reference page */ 3563 val = *loc; 3564 3565 ref = pmap_is_referenced(pg); 3566 mod = pmap_is_modified(pg); 3567 printf("Referenced page: ref %d, mod %d val %x\n", 3568 ref, mod, val); 3569 3570 /* Now clear reference and modify */ 3571 ref = pmap_clear_reference(pg); 3572 mod = pmap_clear_modify(pg); 3573 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3574 (void *)(u_long)va, (long)pa, 3575 ref, mod); 3576 3577 /* Modify page */ 3578 *loc = 1; 3579 3580 ref = pmap_is_referenced(pg); 3581 mod = pmap_is_modified(pg); 3582 printf("Modified page: ref %d, mod %d\n", 3583 ref, mod); 3584 3585 /* Now clear reference and modify */ 3586 ref = pmap_clear_reference(pg); 3587 mod = pmap_clear_modify(pg); 3588 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3589 (void *)(u_long)va, (long)pa, 3590 ref, mod); 3591 3592 /* Check it's properly cleared */ 3593 ref = pmap_is_referenced(pg); 3594 mod = pmap_is_modified(pg); 3595 printf("Checking cleared page: ref %d, mod %d\n", 3596 ref, mod); 3597 3598 /* Modify page */ 3599 *loc = 1; 3600 3601 ref = pmap_is_referenced(pg); 3602 mod = pmap_is_modified(pg); 3603 printf("Modified page: ref %d, mod %d\n", 3604 ref, mod); 3605 3606 /* Check pmap_protect() */ 3607 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 3608 pmap_update(pmap_kernel()); 3609 ref = pmap_is_referenced(pg); 3610 mod = pmap_is_modified(pg); 3611 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 3612 ref, mod); 3613 3614 /* Now clear reference and modify */ 3615 ref = pmap_clear_reference(pg); 3616 mod = pmap_clear_modify(pg); 3617 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3618 (void *)(u_long)va, (long)pa, 3619 ref, mod); 3620 3621 /* Modify page */ 3622 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, VM_PROT_ALL); 3623 pmap_update(pmap_kernel()); 3624 *loc = 1; 3625 3626 ref = pmap_is_referenced(pg); 3627 mod = pmap_is_modified(pg); 3628 printf("Modified page: ref %d, mod %d\n", 3629 ref, mod); 3630 3631 /* Check pmap_protect() */ 3632 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 3633 pmap_update(pmap_kernel()); 3634 ref = pmap_is_referenced(pg); 3635 mod = pmap_is_modified(pg); 3636 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 3637 ref, mod); 3638 3639 /* Now clear reference and modify */ 3640 ref = pmap_clear_reference(pg); 3641 mod = pmap_clear_modify(pg); 3642 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3643 (void *)(u_long)va, (long)pa, 3644 ref, mod); 3645 3646 /* Modify page */ 3647 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, VM_PROT_ALL); 3648 pmap_update(pmap_kernel()); 3649 *loc = 1; 3650 3651 ref = pmap_is_referenced(pg); 3652 mod = pmap_is_modified(pg); 3653 printf("Modified page: ref %d, mod %d\n", 3654 ref, mod); 3655 3656 /* Check pmap_pag_protect() */ 3657 pmap_page_protect(pg, VM_PROT_READ); 3658 ref = pmap_is_referenced(pg); 3659 mod = pmap_is_modified(pg); 3660 printf("pmap_protect(): ref %d, mod %d\n", 3661 ref, mod); 3662 3663 /* Now clear reference and modify */ 3664 ref = pmap_clear_reference(pg); 3665 mod = pmap_clear_modify(pg); 3666 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3667 (void *)(u_long)va, (long)pa, 3668 ref, mod); 3669 3670 3671 /* Modify page */ 3672 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, VM_PROT_ALL); 3673 pmap_update(pmap_kernel()); 3674 *loc = 1; 3675 3676 ref = pmap_is_referenced(pg); 3677 mod = pmap_is_modified(pg); 3678 printf("Modified page: ref %d, mod %d\n", 3679 ref, mod); 3680 3681 /* Check pmap_pag_protect() */ 3682 pmap_page_protect(pg, VM_PROT_NONE); 3683 ref = pmap_is_referenced(pg); 3684 mod = pmap_is_modified(pg); 3685 printf("pmap_protect(): ref %d, mod %d\n", 3686 ref, mod); 3687 3688 /* Now clear reference and modify */ 3689 ref = pmap_clear_reference(pg); 3690 mod = pmap_clear_modify(pg); 3691 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3692 (void *)(u_long)va, (long)pa, 3693 ref, mod); 3694 3695 /* Unmap page */ 3696 pmap_remove(pmap_kernel(), va, va+1); 3697 pmap_update(pmap_kernel()); 3698 ref = pmap_is_referenced(pg); 3699 mod = pmap_is_modified(pg); 3700 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 3701 3702 /* Now clear reference and modify */ 3703 ref = pmap_clear_reference(pg); 3704 mod = pmap_clear_modify(pg); 3705 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3706 (void *)(u_long)va, (long)pa, ref, mod); 3707 3708 /* Check it's properly cleared */ 3709 ref = pmap_is_referenced(pg); 3710 mod = pmap_is_modified(pg); 3711 printf("Checking cleared page: ref %d, mod %d\n", 3712 ref, mod); 3713 3714 pmap_remove(pmap_kernel(), va, va+1); 3715 pmap_update(pmap_kernel()); 3716 pmap_free_page(pa); 3717 } 3718 #endif 3719