1 /* $NetBSD: pmap.c,v 1.340 2010/04/26 09:26:25 martin Exp $ */ 2 3 /* 4 * Copyright (c) 1996 5 * The President and Fellows of Harvard College. All rights reserved. 6 * Copyright (c) 1992, 1993 7 * The Regents of the University of California. All rights reserved. 8 * 9 * This software was developed by the Computer Systems Engineering group 10 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 11 * contributed to Berkeley. 12 * 13 * All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by Harvard University. 16 * This product includes software developed by the University of 17 * California, Lawrence Berkeley Laboratory. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * 1. Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * 2. Redistributions in binary form must reproduce the above copyright 26 * notice, this list of conditions and the following disclaimer in the 27 * documentation and/or other materials provided with the distribution. 28 * 3. All advertising materials mentioning features or use of this software 29 * must display the following acknowledgement: 30 * This product includes software developed by Aaron Brown and 31 * Harvard University. 32 * This product includes software developed by the University of 33 * California, Berkeley and its contributors. 34 * 4. Neither the name of the University nor the names of its contributors 35 * may be used to endorse or promote products derived from this software 36 * without specific prior written permission. 37 * 38 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 39 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 40 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 41 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 42 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 43 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 44 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 45 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 46 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 47 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 48 * SUCH DAMAGE. 49 * 50 * @(#)pmap.c 8.4 (Berkeley) 2/5/94 51 * 52 */ 53 54 /* 55 * SPARC physical map management code. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.340 2010/04/26 09:26:25 martin Exp $"); 60 61 #include "opt_ddb.h" 62 #include "opt_kgdb.h" 63 #include "opt_sparc_arch.h" 64 65 #include <sys/param.h> 66 #include <sys/systm.h> 67 #include <sys/device.h> 68 #include <sys/proc.h> 69 #include <sys/queue.h> 70 #include <sys/pool.h> 71 #include <sys/exec.h> 72 #include <sys/core.h> 73 #include <sys/kcore.h> 74 #include <sys/kernel.h> 75 #include <sys/atomic.h> 76 77 #include <sys/exec_aout.h> /* for MID_* */ 78 79 #include <uvm/uvm.h> 80 81 #include <machine/autoconf.h> 82 #include <machine/bsd_openprom.h> 83 #include <machine/oldmon.h> 84 #include <machine/cpu.h> 85 #include <machine/ctlreg.h> 86 #include <machine/kcore.h> 87 88 #include <sparc/sparc/asm.h> 89 #include <sparc/sparc/cache.h> 90 #include <sparc/sparc/vaddrs.h> 91 #include <sparc/sparc/cpuvar.h> 92 93 /* 94 * The SPARCstation offers us the following challenges: 95 * 96 * 1. A virtual address cache. This is, strictly speaking, not 97 * part of the architecture, but the code below assumes one. 98 * This is a write-through cache on the 4c and a write-back cache 99 * on others. 100 * 101 * 2. (4/4c only) An MMU that acts like a cache. There is not enough 102 * space in the MMU to map everything all the time. Instead, we need 103 * to load MMU with the `working set' of translations for each 104 * process. The sun4m does not act like a cache; tables are maintained 105 * in physical memory. 106 * 107 * 3. Segmented virtual and physical spaces. The upper 12 bits of 108 * a virtual address (the virtual segment) index a segment table, 109 * giving a physical segment. The physical segment selects a 110 * `Page Map Entry Group' (PMEG) and the virtual page number---the 111 * next 5 or 6 bits of the virtual address---select the particular 112 * `Page Map Entry' for the page. We call the latter a PTE and 113 * call each Page Map Entry Group a pmeg (for want of a better name). 114 * Note that the sun4m has an unsegmented 36-bit physical space. 115 * 116 * Since there are no valid bits in the segment table, the only way 117 * to have an invalid segment is to make one full pmeg of invalid PTEs. 118 * We use the last one (since the ROM does as well) (sun4/4c only) 119 * 120 * 4. Discontiguous physical pages. The Mach VM expects physical pages 121 * to be in one sequential lump. 122 * 123 * 5. The MMU is always on: it is not possible to disable it. This is 124 * mainly a startup hassle. 125 */ 126 127 struct pmap_stats { 128 int ps_unlink_pvfirst; /* # of pv_unlinks on head */ 129 int ps_unlink_pvsearch; /* # of pv_unlink searches */ 130 int ps_changeprots; /* # of calls to changeprot */ 131 int ps_enter_firstpv; /* pv heads entered */ 132 int ps_enter_secondpv; /* pv nonheads entered */ 133 int ps_useless_changewire; /* useless wiring changes */ 134 int ps_npg_prot_all; /* # of active pages protected */ 135 int ps_npg_prot_actual; /* # pages actually affected */ 136 int ps_npmeg_free; /* # of free pmegs */ 137 int ps_npmeg_locked; /* # of pmegs on locked list */ 138 int ps_npmeg_lru; /* # of pmegs on lru list */ 139 } pmap_stats; 140 141 #if defined(SUN4) || defined(SUN4C) 142 struct evcnt mmu_stolenpmegs_evcnt = 143 EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","stln pmgs"); 144 EVCNT_ATTACH_STATIC(mmu_stolenpmegs_evcnt); 145 146 struct evcnt mmu_pagein_evcnt = 147 EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","pagein"); 148 EVCNT_ATTACH_STATIC(mmu_pagein_evcnt); 149 #endif /* SUN4 || SUN4C */ 150 151 #ifdef DEBUG 152 #define PDB_CREATE 0x0001 153 #define PDB_DESTROY 0x0002 154 #define PDB_REMOVE 0x0004 155 #define PDB_CHANGEPROT 0x0008 156 #define PDB_ENTER 0x0010 157 #define PDB_FOLLOW 0x0020 158 159 #define PDB_MMU_ALLOC 0x0100 160 #define PDB_MMU_STEAL 0x0200 161 #define PDB_CTX_ALLOC 0x0400 162 #define PDB_CTX_STEAL 0x0800 163 #define PDB_MMUREG_ALLOC 0x1000 164 #define PDB_MMUREG_STEAL 0x2000 165 #define PDB_CACHESTUFF 0x4000 166 #define PDB_SWITCHMAP 0x8000 167 #define PDB_SANITYCHK 0x10000 168 int pmapdebug = 0; 169 #endif 170 171 /* 172 * Bounds on managed physical addresses. Used by (MD) users 173 * of uvm_pglistalloc() to provide search hints. 174 */ 175 paddr_t vm_first_phys = (paddr_t)-1; 176 paddr_t vm_last_phys = 0; 177 psize_t vm_num_phys; 178 179 #define PMAP_LOCK() KERNEL_LOCK(1, NULL) 180 #define PMAP_UNLOCK() KERNEL_UNLOCK_ONE(NULL) 181 182 /* 183 * Flags in pvlist.pv_flags. Note that PV_MOD must be 1 and PV_REF must be 2 184 * since they must line up with the bits in the hardware PTEs (see pte.h). 185 * SUN4M bits are at a slightly different location in the PTE. 186 * 187 * Note: the REF, MOD and ANC flag bits occur only in the head of a pvlist. 188 * The NC bit is meaningful in each individual pv entry and reflects the 189 * requested non-cacheability at the time the entry was made through 190 * pv_link() or when subsequently altered by kvm_uncache() (but the latter 191 * does not happen in kernels as of the time of this writing (March 2001)). 192 */ 193 #define PV_MOD 1 /* page modified */ 194 #define PV_REF 2 /* page referenced */ 195 #define PV_NC 4 /* page cannot be cached */ 196 #define PV_REF4M 1 /* page referenced (SRMMU) */ 197 #define PV_MOD4M 2 /* page modified (SRMMU) */ 198 #define PV_ANC 0x10 /* page has incongruent aliases */ 199 200 static struct pool pv_pool; 201 202 /* 203 * pvhead(pte): find a VM page given a PTE entry. 204 */ 205 #if defined(SUN4) || defined(SUN4C) 206 static struct vm_page * 207 pvhead4_4c(u_int pte) 208 { 209 paddr_t pa = (pte & PG_PFNUM) << PGSHIFT; 210 211 return (PHYS_TO_VM_PAGE(pa)); 212 } 213 #endif 214 215 #if defined(SUN4M) || defined(SUN4D) 216 static struct vm_page * 217 pvhead4m(u_int pte) 218 { 219 paddr_t pa = (pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT; 220 221 return (PHYS_TO_VM_PAGE(pa)); 222 } 223 #endif 224 225 /* 226 * Each virtual segment within each pmap is either valid or invalid. 227 * It is valid if pm_npte[VA_VSEG(va)] is not 0. This does not mean 228 * it is in the MMU, however; that is true iff pm_segmap[VA_VSEG(va)] 229 * does not point to the invalid PMEG. 230 * 231 * In the older SPARC architectures (sun4/sun4c), page tables are cached in 232 * the MMU. The following discussion applies to these architectures: 233 * 234 * If a virtual segment is valid and loaded, the correct PTEs appear 235 * in the MMU only. If it is valid and unloaded, the correct PTEs appear 236 * in the pm_pte[VA_VSEG(va)] only. However, some effort is made to keep 237 * the software copies consistent enough with the MMU so that libkvm can 238 * do user address translations. In particular, pv_changepte() and 239 * pmap_enu() maintain consistency, while less critical changes are 240 * not maintained. pm_pte[VA_VSEG(va)] always points to space for those 241 * PTEs. 242 * 243 * Each PMEG in the MMU is either free or contains PTEs corresponding to 244 * some pmap and virtual segment. If it contains some PTEs, it also contains 245 * reference and modify bits that belong in the pv_table. If we need 246 * to steal a PMEG from some process (if we need one and none are free) 247 * we must copy the ref and mod bits, and update pm_segmap in the other 248 * pmap to show that its virtual segment is no longer in the MMU. 249 * 250 * There are 128 PMEGs in a small Sun-4, of which only a few dozen are 251 * tied down permanently, leaving `about' 100 to be spread among 252 * running processes. These are managed as an LRU cache. Before 253 * calling the VM paging code for a user page fault, the fault handler 254 * calls mmu_load(pmap, va) to try to get a set of PTEs put into the 255 * MMU. mmu_load will check the validity of the segment and tell whether 256 * it did something. 257 * 258 * Since I hate the name PMEG I call this data structure an `mmu entry'. 259 * Each mmuentry is on exactly one of three `usage' lists: free, LRU, 260 * or locked. The locked list is only used for kernel mappings that need 261 * to be wired down. 262 * 263 * 264 * In the sun4m architecture using the SPARC Reference MMU (SRMMU), three 265 * levels of page tables are maintained in physical memory. We use the same 266 * structures as with the 3-level old-style MMU (pm_regmap, pm_segmap, 267 * rg_segmap, sg_pte, etc) to maintain kernel-edible page tables; we also 268 * build a parallel set of physical tables that can be used by the MMU. 269 * (XXX: This seems redundant, but is it necessary for the unified kernel?) 270 * 271 * If a virtual segment is valid, its entries will be in both parallel lists. 272 * If it is not valid, then its entry in the kernel tables will be zero, and 273 * its entry in the MMU tables will either be nonexistent or zero as well. 274 * 275 * The Reference MMU generally uses a Translation Look-aside Buffer (TLB) 276 * to cache the result of recently executed page table walks. When 277 * manipulating page tables, we need to ensure consistency of the 278 * in-memory and TLB copies of the page table entries. This is handled 279 * by flushing (and invalidating) a TLB entry when appropriate before 280 * altering an in-memory page table entry. 281 */ 282 struct mmuentry { 283 CIRCLEQ_ENTRY(mmuentry) me_list; /* usage list link */ 284 TAILQ_ENTRY(mmuentry) me_pmchain; /* pmap owner link */ 285 struct pmap *me_pmap; /* pmap, if in use */ 286 u_short me_vreg; /* associated virtual region/segment */ 287 u_short me_vseg; /* associated virtual region/segment */ 288 u_short me_cookie; /* hardware SMEG/PMEG number */ 289 #ifdef DIAGNOSTIC 290 int *me_statp;/*XXX*/ 291 #endif 292 }; 293 struct mmuentry *mmusegments; /* allocated in pmap_bootstrap */ 294 struct mmuentry *mmuregions; /* allocated in pmap_bootstrap */ 295 296 CIRCLEQ_HEAD(mmuq, mmuentry); 297 struct mmuq segm_freelist, segm_lru, segm_locked; 298 struct mmuq region_freelist, region_lru, region_locked; 299 /* 300 * We use a circular queue, since that allows us to remove an element 301 * from a list without knowing the list header. 302 */ 303 #define CIRCLEQ_REMOVE_NOH(elm, field) do { \ 304 (elm)->field.cqe_next->field.cqe_prev = (elm)->field.cqe_prev; \ 305 (elm)->field.cqe_prev->field.cqe_next = (elm)->field.cqe_next; \ 306 } while (/*CONSTCOND*/0) 307 308 #define MMUQ_INIT(head) CIRCLEQ_INIT(head) 309 #define MMUQ_REMOVE(elm,field) CIRCLEQ_REMOVE_NOH(elm,field) 310 #define MMUQ_INSERT_TAIL(head,elm,field)CIRCLEQ_INSERT_TAIL(head,elm,field) 311 #define MMUQ_EMPTY(head) CIRCLEQ_EMPTY(head) 312 #define MMUQ_FIRST(head) CIRCLEQ_FIRST(head) 313 314 315 int seginval; /* [4/4c] the invalid segment number */ 316 int reginval; /* [4/3mmu] the invalid region number */ 317 318 static kmutex_t demap_lock; 319 static bool lock_available = false; /* demap_lock has been initialized */ 320 321 /* 322 * (sun4/4c) 323 * A context is simply a small number that dictates which set of 4096 324 * segment map entries the MMU uses. The Sun 4c has eight (SS1,IPC) or 325 * sixteen (SS2,IPX) such sets. These are alloted in an `almost MRU' fashion. 326 * (sun4m) 327 * A context is simply a small number that indexes the context table, the 328 * root-level page table mapping 4G areas. Each entry in this table points 329 * to a 1st-level region table. A SPARC reference MMU will usually use 16 330 * such contexts, but some offer as many as 64k contexts; the theoretical 331 * maximum is 2^32 - 1, but this would create overlarge context tables. 332 * 333 * Each context is either free or attached to a pmap. 334 * 335 * Since the virtual address cache is tagged by context, when we steal 336 * a context we have to flush (that part of) the cache. 337 */ 338 union ctxinfo { 339 union ctxinfo *c_nextfree; /* free list (if free) */ 340 struct pmap *c_pmap; /* pmap (if busy) */ 341 }; 342 343 static kmutex_t ctx_lock; /* lock for below */ 344 union ctxinfo *ctxinfo; /* allocated at in pmap_bootstrap */ 345 union ctxinfo *ctx_freelist; /* context free list */ 346 int ctx_kick; /* allocation rover when none free */ 347 int ctx_kickdir; /* ctx_kick roves both directions */ 348 int ncontext; /* sizeof ctx_freelist */ 349 350 void ctx_alloc(struct pmap *); 351 void ctx_free(struct pmap *); 352 353 void * vmmap; /* one reserved MI vpage for /dev/mem */ 354 /*void * vdumppages; -* 32KB worth of reserved dump pages */ 355 356 smeg_t tregion; /* [4/3mmu] Region for temporary mappings */ 357 358 static struct pmap kernel_pmap_store; /* the kernel's pmap */ 359 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; /* pmap_kernel() */ 360 struct regmap kernel_regmap_store[NKREG]; /* the kernel's regmap */ 361 struct segmap kernel_segmap_store[NKREG*NSEGRG];/* the kernel's segmaps */ 362 363 #if defined(SUN4M) || defined(SUN4D) 364 u_int *kernel_regtable_store; /* 1k of storage to map the kernel */ 365 u_int *kernel_segtable_store; /* 2k of storage to map the kernel */ 366 u_int *kernel_pagtable_store; /* 128k of storage to map the kernel */ 367 368 /* 369 * Memory pools and back-end supplier for SRMMU page tables. 370 * Share a pool between the level 2 and level 3 page tables, 371 * since these are equal in size. 372 */ 373 static struct pool L1_pool; 374 static struct pool L23_pool; 375 376 static void *pgt_page_alloc(struct pool *, int); 377 static void pgt_page_free(struct pool *, void *); 378 379 static struct pool_allocator pgt_page_allocator = { 380 pgt_page_alloc, pgt_page_free, 0, 381 }; 382 383 #endif /* SUN4M || SUN4D */ 384 385 #if defined(SUN4) || defined(SUN4C) 386 /* 387 * Memory pool for user and kernel PTE tables. 388 */ 389 static struct pool pte_pool; 390 #endif 391 392 struct memarr *pmemarr; /* physical memory regions */ 393 int npmemarr; /* number of entries in pmemarr */ 394 395 static paddr_t avail_start; /* first available physical page, other 396 than the `etext gap' defined below */ 397 static vaddr_t etext_gap_start;/* start of gap between text & data */ 398 static vaddr_t etext_gap_end; /* end of gap between text & data */ 399 static vaddr_t virtual_avail; /* first free kernel virtual address */ 400 static vaddr_t virtual_end; /* last free kernel virtual address */ 401 402 static void pmap_page_upload(void); 403 404 int mmu_has_hole; 405 406 vaddr_t prom_vstart; /* For /dev/kmem */ 407 vaddr_t prom_vend; 408 409 /* 410 * Memory pool for pmap structures. 411 */ 412 static struct pool_cache pmap_cache; 413 static int pmap_pmap_pool_ctor(void *, void *, int); 414 static void pmap_pmap_pool_dtor(void *, void *); 415 static struct pool segmap_pool; 416 417 #if defined(SUN4) 418 /* 419 * [sun4]: segfixmask: on some systems (4/110) "getsegmap()" returns a 420 * partly invalid value. getsegmap returns a 16 bit value on the sun4, 421 * but only the first 8 or so bits are valid (the rest are *supposed* to 422 * be zero. On the 4/110 the bits that are supposed to be zero are 423 * all one instead. e.g. KERNBASE is usually mapped by pmeg number zero. 424 * On a 4/300 getsegmap(KERNBASE) == 0x0000, but 425 * on a 4/100 getsegmap(KERNBASE) == 0xff00 426 * 427 * This confuses mmu_reservemon() and causes it to not reserve the PROM's 428 * pmegs. Then the PROM's pmegs get used during autoconfig and everything 429 * falls apart! (not very fun to debug, BTW.) 430 * 431 * solution: mask the invalid bits in the getsetmap macro. 432 */ 433 434 static u_int segfixmask = 0xffffffff; /* all bits valid to start */ 435 #else 436 #define segfixmask 0xffffffff /* It's in getsegmap's scope */ 437 #endif 438 439 /* 440 * pseudo-functions for mnemonic value 441 */ 442 #define getsegmap(va) (CPU_ISSUN4C \ 443 ? lduba(va, ASI_SEGMAP) \ 444 : (lduha(va, ASI_SEGMAP) & segfixmask)) 445 #define setsegmap(va, pmeg) (CPU_ISSUN4C \ 446 ? stba(va, ASI_SEGMAP, pmeg) \ 447 : stha(va, ASI_SEGMAP, pmeg)) 448 449 /* 3-level sun4 MMU only: */ 450 #define getregmap(va) ((unsigned)lduha((va)+2, ASI_REGMAP) >> 8) 451 #define setregmap(va, smeg) stha((va)+2, ASI_REGMAP, (smeg << 8)) 452 453 454 #if defined(SUN4M) || defined(SUN4D) 455 #if 0 456 #if VM_PROT_READ != 1 || VM_PROT_WRITE != 2 || VM_PROT_EXECUTE != 4 457 #error fix protection code translation table 458 #endif 459 #endif 460 /* 461 * Translation table for kernel vs. PTE protection bits. 462 */ 463 const u_int protection_codes[2][8] = { 464 /* kernel */ 465 { 466 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE */ 467 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_READ */ 468 PPROT_N_RWX, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE */ 469 PPROT_N_RWX, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_READ */ 470 PPROT_N_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_NONE */ 471 PPROT_N_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_READ */ 472 PPROT_N_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */ 473 PPROT_N_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */ 474 }, 475 476 /* user */ 477 { 478 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE */ 479 PPROT_R_R, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_READ */ 480 PPROT_RW_RW, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE */ 481 PPROT_RW_RW, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_READ */ 482 PPROT_X_X, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_NONE */ 483 PPROT_RX_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_READ */ 484 PPROT_RWX_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */ 485 PPROT_RWX_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */ 486 } 487 }; 488 #define pte_kprot4m(prot) (protection_codes[0][(prot)]) 489 #define pte_uprot4m(prot) (protection_codes[1][(prot)]) 490 #define pte_prot4m(pm, prot) \ 491 (protection_codes[(pm) == pmap_kernel() ? 0 : 1][(prot)]) 492 493 void setpte4m(vaddr_t va, int pte); 494 void setpgt4m(int *ptep, int pte); 495 void setpgt4m_va(vaddr_t, int *, int, int, int, u_int); 496 int updatepte4m(vaddr_t, int *, int, int, int, u_int); 497 #endif /* SUN4M || SUN4D */ 498 499 #if defined(MULTIPROCESSOR) 500 #define PMAP_SET_CPUSET(pmap, cpi) \ 501 (pmap->pm_cpuset |= (1 << (cpi)->ci_cpuid)) 502 #define PMAP_CLR_CPUSET(pmap, cpi) \ 503 (pmap->pm_cpuset &= ~(1 << (cpi)->ci_cpuid)) 504 #define PMAP_CPUSET(pmap) (pmap->pm_cpuset) 505 #else 506 #define PMAP_SET_CPUSET(pmap, cpi) /* nothing */ 507 #define PMAP_CLR_CPUSET(pmap, cpi) /* nothing */ 508 #define PMAP_CPUSET(pmap) 1 /* XXX: 1 or 0? */ 509 #endif /* MULTIPROCESSOR */ 510 511 512 /* Function pointer messiness for supporting multiple sparc architectures 513 * within a single kernel: notice that there are two versions of many of the 514 * functions within this file/module, one for the sun4/sun4c and the other 515 * for the sun4m. For performance reasons (since things like pte bits don't 516 * map nicely between the two architectures), there are separate functions 517 * rather than unified functions which test the cputyp variable. If only 518 * one architecture is being used, then the non-suffixed function calls 519 * are macro-translated into the appropriate xxx4_4c or xxx4m call. If 520 * multiple architectures are defined, the calls translate to (*xxx_p), 521 * i.e. they indirect through function pointers initialized as appropriate 522 * to the run-time architecture in pmap_bootstrap. See also pmap.h. 523 */ 524 525 #if defined(SUN4M) || defined(SUN4D) 526 static void mmu_setup4m_L1(int, struct pmap *); 527 static void mmu_setup4m_L2(int, struct regmap *); 528 static void mmu_setup4m_L3(int, struct segmap *); 529 /*static*/ void mmu_reservemon4m(struct pmap *); 530 531 /*static*/ void pmap_changeprot4m(pmap_t, vaddr_t, vm_prot_t, int); 532 /*static*/ void pmap_rmk4m(struct pmap *, vaddr_t, vaddr_t, int, int); 533 /*static*/ void pmap_rmu4m(struct pmap *, vaddr_t, vaddr_t, int, int); 534 /*static*/ int pmap_enk4m(struct pmap *, vaddr_t, vm_prot_t, 535 int, struct vm_page *, int); 536 /*static*/ int pmap_enu4m(struct pmap *, vaddr_t, vm_prot_t, 537 int, struct vm_page *, int); 538 /*static*/ void pv_changepte4m(struct vm_page *, int, int); 539 /*static*/ int pv_syncflags4m(struct vm_page *); 540 /*static*/ int pv_link4m(struct vm_page *, struct pmap *, vaddr_t, u_int *); 541 /*static*/ void pv_unlink4m(struct vm_page *, struct pmap *, vaddr_t); 542 #endif 543 544 #if defined(SUN4) || defined(SUN4C) 545 /*static*/ void mmu_reservemon4_4c(int *, int *); 546 /*static*/ void pmap_changeprot4_4c(pmap_t, vaddr_t, vm_prot_t, int); 547 /*static*/ void pmap_rmk4_4c(struct pmap *, vaddr_t, vaddr_t, int, int); 548 /*static*/ void pmap_rmu4_4c(struct pmap *, vaddr_t, vaddr_t, int, int); 549 /*static*/ int pmap_enk4_4c(struct pmap *, vaddr_t, vm_prot_t, 550 int, struct vm_page *, int); 551 /*static*/ int pmap_enu4_4c(struct pmap *, vaddr_t, vm_prot_t, 552 int, struct vm_page *, int); 553 /*static*/ void pv_changepte4_4c(struct vm_page *, int, int); 554 /*static*/ int pv_syncflags4_4c(struct vm_page *); 555 /*static*/ int pv_link4_4c(struct vm_page *, struct pmap *, vaddr_t, u_int *); 556 /*static*/ void pv_unlink4_4c(struct vm_page *, struct pmap *, vaddr_t); 557 #endif 558 559 #if !(defined(SUN4M) || defined(SUN4D)) && (defined(SUN4) || defined(SUN4C)) 560 #define pmap_rmk pmap_rmk4_4c 561 #define pmap_rmu pmap_rmu4_4c 562 563 #elif (defined(SUN4M) || defined(SUN4D)) && !(defined(SUN4) || defined(SUN4C)) 564 #define pmap_rmk pmap_rmk4m 565 #define pmap_rmu pmap_rmu4m 566 567 #else /* must use function pointers */ 568 569 /* function pointer declarations */ 570 /* from pmap.h: */ 571 bool (*pmap_clear_modify_p)(struct vm_page *); 572 bool (*pmap_clear_reference_p)(struct vm_page *); 573 int (*pmap_enter_p)(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int); 574 bool (*pmap_extract_p)(pmap_t, vaddr_t, paddr_t *); 575 bool (*pmap_is_modified_p)(struct vm_page *); 576 bool (*pmap_is_referenced_p)(struct vm_page *); 577 void (*pmap_kenter_pa_p)(vaddr_t, paddr_t, vm_prot_t, u_int); 578 void (*pmap_kremove_p)(vaddr_t, vsize_t); 579 void (*pmap_kprotect_p)(vaddr_t, vsize_t, vm_prot_t); 580 void (*pmap_page_protect_p)(struct vm_page *, vm_prot_t); 581 void (*pmap_protect_p)(pmap_t, vaddr_t, vaddr_t, vm_prot_t); 582 /* local: */ 583 void (*pmap_rmk_p)(struct pmap *, vaddr_t, vaddr_t, int, int); 584 void (*pmap_rmu_p)(struct pmap *, vaddr_t, vaddr_t, int, int); 585 586 #define pmap_rmk (*pmap_rmk_p) 587 #define pmap_rmu (*pmap_rmu_p) 588 589 #endif 590 591 /* --------------------------------------------------------------*/ 592 593 /* 594 * Next we have some sun4m/4d-specific routines which have no 4/4c 595 * counterparts, or which are 4/4c macros. 596 */ 597 598 #if defined(SUN4M) || defined(SUN4D) 599 /* 600 * SP versions of the tlb flush operations. 601 * 602 * Turn off traps to prevent register window overflows 603 * from writing user windows to the wrong stack. 604 */ 605 static void 606 sp_tlb_flush(int va, int ctx, int lvl) 607 { 608 609 /* Traps off */ 610 __asm("rd %psr, %o3"); 611 __asm("wr %%o3, %0, %%psr" :: "n" (PSR_ET)); 612 613 /* Save context */ 614 __asm("mov %0, %%o4" :: "n"(SRMMU_CXR)); 615 __asm("lda [%%o4]%0, %%o5" :: "n"(ASI_SRMMU)); 616 617 /* Set new context and flush type bits */ 618 __asm("andn %o0, 0xfff, %o0"); 619 __asm("sta %%o1, [%%o4]%0" :: "n"(ASI_SRMMU)); 620 __asm("or %o0, %o2, %o0"); 621 622 /* Do the TLB flush */ 623 __asm("sta %%g0, [%%o0]%0" :: "n"(ASI_SRMMUFP)); 624 625 /* Restore context */ 626 __asm("sta %%o5, [%%o4]%0" :: "n"(ASI_SRMMU)); 627 628 /* and turn traps on again */ 629 __asm("wr %o3, 0, %psr"); 630 __asm("nop"); 631 __asm("nop"); 632 __asm("nop"); 633 } 634 635 static inline void 636 sp_tlb_flush_all(void) 637 { 638 639 sta(ASI_SRMMUFP_LN, ASI_SRMMUFP, 0); 640 } 641 642 #if defined(MULTIPROCESSOR) 643 /* 644 * The SMP versions of the tlb flush routines. We only need to 645 * do a cross call for these on sun4m (Mbus) systems. sun4d systems 646 * have an Xbus which broadcasts TLB demaps in hardware. 647 */ 648 649 static inline void smp_tlb_flush_page (int va, int ctx, u_int cpuset); 650 static inline void smp_tlb_flush_segment (int va, int ctx, u_int cpuset); 651 static inline void smp_tlb_flush_region (int va, int ctx, u_int cpuset); 652 static inline void smp_tlb_flush_context (int ctx, u_int cpuset); 653 static inline void smp_tlb_flush_all (void); 654 655 /* From locore: */ 656 extern void ft_tlb_flush(int va, int ctx, int lvl); 657 658 static inline void 659 smp_tlb_flush_page(int va, int ctx, u_int cpuset) 660 { 661 662 if (CPU_ISSUN4D) { 663 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L3); 664 } else 665 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L3, cpuset); 666 } 667 668 static inline void 669 smp_tlb_flush_segment(int va, int ctx, u_int cpuset) 670 { 671 672 if (CPU_ISSUN4D) { 673 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L2); 674 } else 675 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L2, cpuset); 676 } 677 678 static inline void 679 smp_tlb_flush_region(int va, int ctx, u_int cpuset) 680 { 681 682 if (CPU_ISSUN4D) { 683 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L1); 684 } else 685 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L1, cpuset); 686 } 687 688 static inline void 689 smp_tlb_flush_context(int ctx, u_int cpuset) 690 { 691 692 if (CPU_ISSUN4D) { 693 sp_tlb_flush(ctx, 0, ASI_SRMMUFP_L0); 694 } else 695 FXCALL3(sp_tlb_flush, ft_tlb_flush, 0, ctx, ASI_SRMMUFP_L0, cpuset); 696 } 697 698 static inline void 699 smp_tlb_flush_all(void) 700 { 701 702 if (CPU_ISSUN4D) { 703 sp_tlb_flush_all(); 704 } else 705 XCALL0(sp_tlb_flush_all, CPUSET_ALL); 706 } 707 #endif /* MULTIPROCESSOR */ 708 709 #if defined(MULTIPROCESSOR) 710 #define tlb_flush_page(va,ctx,s) smp_tlb_flush_page(va,ctx,s) 711 #define tlb_flush_segment(va,ctx,s) smp_tlb_flush_segment(va,ctx,s) 712 #define tlb_flush_region(va,ctx,s) smp_tlb_flush_region(va,ctx,s) 713 #define tlb_flush_context(ctx,s) smp_tlb_flush_context(ctx,s) 714 #define tlb_flush_all() smp_tlb_flush_all() 715 #else 716 #define tlb_flush_page(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L3) 717 #define tlb_flush_segment(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L2) 718 #define tlb_flush_region(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L1) 719 #define tlb_flush_context(ctx,s) sp_tlb_flush(ctx,0,ASI_SRMMUFP_L0) 720 #define tlb_flush_all() sp_tlb_flush_all() 721 #endif /* MULTIPROCESSOR */ 722 723 static u_int VA2PA(void *); 724 static u_long srmmu_bypass_read(u_long); 725 726 /* 727 * VA2PA(addr) -- converts a virtual address to a physical address using 728 * the MMU's currently-installed page tables. As a side effect, the address 729 * translation used may cause the associated pte to be encached. The correct 730 * context for VA must be set before this is called. 731 * 732 * This routine should work with any level of mapping, as it is used 733 * during bootup to interact with the ROM's initial L1 mapping of the kernel. 734 */ 735 static u_int 736 VA2PA(void *addr) 737 { 738 u_int pte; 739 740 /* 741 * We'll use that handy SRMMU flush/probe. 742 * Try each level in turn until we find a valid pte. Otherwise panic. 743 */ 744 745 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L3, ASI_SRMMUFP); 746 /* Unlock fault status; required on Hypersparc modules */ 747 (void)lda(SRMMU_SFSR, ASI_SRMMU); 748 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE) 749 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 750 ((u_int)addr & 0xfff)); 751 752 /* A `TLB Flush Entire' is required before any L0, L1 or L2 probe */ 753 tlb_flush_all_real(); 754 755 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L2, ASI_SRMMUFP); 756 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE) 757 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 758 ((u_int)addr & 0x3ffff)); 759 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L1, ASI_SRMMUFP); 760 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE) 761 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 762 ((u_int)addr & 0xffffff)); 763 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L0, ASI_SRMMUFP); 764 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE) 765 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 766 ((u_int)addr & 0xffffffff)); 767 768 #ifdef DIAGNOSTIC 769 panic("VA2PA: Asked to translate unmapped VA %p", addr); 770 #else 771 return (0); 772 #endif 773 } 774 775 /* 776 * Atomically update a PTE entry, coping with hardware updating the 777 * PTE at the same time we are. This is the procedure that is 778 * recommended in the SuperSPARC user's manual. 779 */ 780 int 781 updatepte4m(vaddr_t va, int *pte, int bic, int bis, int ctx, u_int cpuset) 782 { 783 int oldval, swapval; 784 volatile int *vpte = (volatile int *)pte; 785 bool can_lock = lock_available; 786 787 /* 788 * Can only be one of these happening in the system 789 * at any one time. 790 */ 791 if (__predict_true(can_lock)) 792 mutex_spin_enter(&demap_lock); 793 794 /* 795 * The idea is to loop swapping zero into the pte, flushing 796 * it, and repeating until it stays zero. At this point, 797 * there should be no more hardware accesses to this PTE 798 * so we can modify it without losing any mod/ref info. 799 */ 800 oldval = 0; 801 do { 802 swapval = 0; 803 swap(vpte, swapval); 804 tlb_flush_page(va, ctx, cpuset); 805 oldval |= swapval; 806 } while (__predict_false(*vpte != 0)); 807 808 swapval = (oldval & ~bic) | bis; 809 swap(vpte, swapval); 810 811 if (__predict_true(can_lock)) 812 mutex_spin_exit(&demap_lock); 813 814 return (oldval); 815 } 816 817 inline void 818 setpgt4m(int *ptep, int pte) 819 { 820 821 swap(ptep, pte); 822 } 823 824 inline void 825 setpgt4m_va(vaddr_t va, int *ptep, int pte, int pageflush, int ctx, 826 u_int cpuset) 827 { 828 829 #if defined(MULTIPROCESSOR) 830 updatepte4m(va, ptep, 0xffffffff, pte, pageflush ? ctx : 0, cpuset); 831 #else 832 if (__predict_true(pageflush)) 833 tlb_flush_page(va, ctx, 0); 834 setpgt4m(ptep, pte); 835 #endif /* MULTIPROCESSOR */ 836 } 837 838 /* Set the page table entry for va to pte. */ 839 void 840 setpte4m(vaddr_t va, int pte) 841 { 842 struct pmap *pm; 843 struct regmap *rp; 844 struct segmap *sp; 845 846 #ifdef DEBUG 847 if (getcontext4m() != 0) 848 panic("setpte4m: user context"); 849 #endif 850 851 pm = pmap_kernel(); 852 rp = &pm->pm_regmap[VA_VREG(va)]; 853 sp = &rp->rg_segmap[VA_VSEG(va)]; 854 855 tlb_flush_page(va, 0, CPUSET_ALL); 856 setpgt4m(sp->sg_pte + VA_SUN4M_VPG(va), pte); 857 } 858 859 /* 860 * Page table pool back-end. 861 */ 862 void * 863 pgt_page_alloc(struct pool *pp, int flags) 864 { 865 int cacheit = (cpuinfo.flags & CPUFLG_CACHEPAGETABLES) != 0; 866 struct vm_page *pg; 867 vaddr_t va; 868 paddr_t pa; 869 870 /* Allocate a page of physical memory */ 871 if ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL) 872 return (NULL); 873 874 /* Allocate virtual memory */ 875 va = uvm_km_alloc(kmem_map, PAGE_SIZE, 0, UVM_KMF_VAONLY | 876 ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK)); 877 if (va == 0) { 878 uvm_pagefree(pg); 879 return (NULL); 880 } 881 882 /* 883 * On systems with a physical data cache we need to flush this page 884 * from the cache if the pagetables cannot be cached. 885 * On systems with a virtually indexed data cache, we only need 886 * to map it non-cacheable, since the page is not currently mapped. 887 */ 888 pa = VM_PAGE_TO_PHYS(pg); 889 if (cacheit == 0) 890 pcache_flush_page(pa, 1); 891 892 /* Map the page */ 893 pmap_kenter_pa(va, pa | (cacheit ? 0 : PMAP_NC), 894 VM_PROT_READ | VM_PROT_WRITE, 0); 895 pmap_update(pmap_kernel()); 896 897 return ((void *)va); 898 } 899 900 void 901 pgt_page_free(struct pool *pp, void *v) 902 { 903 vaddr_t va; 904 paddr_t pa; 905 bool rv; 906 907 va = (vaddr_t)v; 908 rv = pmap_extract(pmap_kernel(), va, &pa); 909 KASSERT(rv); 910 uvm_pagefree(PHYS_TO_VM_PAGE(pa)); 911 pmap_kremove(va, PAGE_SIZE); 912 uvm_km_free(kmem_map, va, PAGE_SIZE, UVM_KMF_VAONLY); 913 } 914 #endif /* SUN4M || SUN4D */ 915 916 /*----------------------------------------------------------------*/ 917 918 /* 919 * The following three macros are to be used in sun4/sun4c code only. 920 */ 921 #if defined(SUN4_MMU3L) 922 #define CTX_USABLE(pm,rp) ( \ 923 ((pm)->pm_ctx != NULL && \ 924 (!HASSUN4_MMU3L || (rp)->rg_smeg != reginval)) \ 925 ) 926 #else 927 #define CTX_USABLE(pm,rp) ((pm)->pm_ctx != NULL ) 928 #endif 929 930 #define GAP_WIDEN(pm,vr) do if (CPU_HAS_SUNMMU) { \ 931 if (vr + 1 == pm->pm_gap_start) \ 932 pm->pm_gap_start = vr; \ 933 if (vr == pm->pm_gap_end) \ 934 pm->pm_gap_end = vr + 1; \ 935 } while (0) 936 937 #define GAP_SHRINK(pm,vr) do if (CPU_HAS_SUNMMU) { \ 938 int x; \ 939 x = pm->pm_gap_start + (pm->pm_gap_end - pm->pm_gap_start) / 2; \ 940 if (vr > x) { \ 941 if (vr < pm->pm_gap_end) \ 942 pm->pm_gap_end = vr; \ 943 } else { \ 944 if (vr >= pm->pm_gap_start && x != pm->pm_gap_start) \ 945 pm->pm_gap_start = vr + 1; \ 946 } \ 947 } while (0) 948 949 950 static void get_phys_mem(void **); 951 #if 0 /* not used */ 952 void kvm_iocache(char *, int); 953 #endif 954 955 #ifdef DEBUG 956 void pm_check(char *, struct pmap *); 957 void pm_check_k(char *, struct pmap *); 958 void pm_check_u(char *, struct pmap *); 959 #endif 960 961 /* 962 * During the PMAP bootstrap, we can use a simple translation to map a 963 * kernel virtual address to a psysical memory address (this is arranged 964 * in locore). Usually, KERNBASE maps to physical address 0. This is always 965 * the case on sun4 and sun4c machines. On sun4m machines -- if no memory is 966 * installed in the bank corresponding to physical address 0 -- the PROM may 967 * elect to load us at some other address, presumably at the start of 968 * the first memory bank that is available. We set the up the variable 969 * `va2pa_offset' to hold the physical address corresponding to KERNBASE. 970 */ 971 972 static u_long va2pa_offset; 973 #define PMAP_BOOTSTRAP_VA2PA(v) ((paddr_t)((u_long)(v) - va2pa_offset)) 974 #define PMAP_BOOTSTRAP_PA2VA(p) ((vaddr_t)((u_long)(p) + va2pa_offset)) 975 976 /* 977 * Grab physical memory list. 978 * While here, compute `physmem'. 979 */ 980 void 981 get_phys_mem(void **top) 982 { 983 struct memarr *mp; 984 char *p; 985 int i; 986 987 /* Load the memory descriptor array at the current kernel top */ 988 p = (void *)ALIGN(*top); 989 pmemarr = (struct memarr *)p; 990 npmemarr = prom_makememarr(pmemarr, 1000, MEMARR_AVAILPHYS); 991 992 /* Update kernel top */ 993 p += npmemarr * sizeof(struct memarr); 994 *top = p; 995 996 for (physmem = 0, mp = pmemarr, i = npmemarr; --i >= 0; mp++) 997 physmem += btoc(mp->len); 998 } 999 1000 1001 /* 1002 * Support functions for vm_page_bootstrap(). 1003 */ 1004 1005 /* 1006 * How much virtual space does this kernel have? 1007 * (After mapping kernel text, data, etc.) 1008 */ 1009 void 1010 pmap_virtual_space(vaddr_t *v_start, vaddr_t *v_end) 1011 { 1012 1013 *v_start = virtual_avail; 1014 *v_end = virtual_end; 1015 } 1016 1017 #ifdef PMAP_GROWKERNEL 1018 vaddr_t 1019 pmap_growkernel(vaddr_t eva) 1020 { 1021 struct regmap *rp; 1022 struct segmap *sp; 1023 int vr, evr, M, N, i; 1024 struct vm_page *pg; 1025 vaddr_t va; 1026 1027 if (eva <= virtual_end) 1028 return (virtual_end); 1029 1030 /* For now, only implemented for sun4/sun4c */ 1031 KASSERT(CPU_HAS_SUNMMU); 1032 1033 /* 1034 * Map in the next region(s) 1035 */ 1036 1037 /* Get current end-of-kernel */ 1038 vr = virtual_end >> RGSHIFT; 1039 evr = (eva + NBPRG - 1) >> RGSHIFT; 1040 eva = evr << RGSHIFT; 1041 1042 if (eva > VM_MAX_KERNEL_ADDRESS) 1043 panic("growkernel: grown too large: %lx", eva); 1044 1045 /* 1046 * Divide a region in N blocks of M segments, where each segment 1047 * block can have its PTEs mapped by one page. 1048 * N should come out to 1 for 8K pages and to 4 for 4K pages. 1049 */ 1050 M = NBPG / (NPTESG * sizeof(int)); 1051 N = (NBPRG/NBPSG) / M; 1052 1053 while (vr < evr) { 1054 rp = &pmap_kernel()->pm_regmap[vr]; 1055 for (i = 0; i < N; i++) { 1056 sp = &rp->rg_segmap[i * M]; 1057 va = (vaddr_t)sp->sg_pte; 1058 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 1059 if (pg == NULL) 1060 panic("growkernel: out of memory"); 1061 pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), 1062 VM_PROT_READ | VM_PROT_WRITE, 0); 1063 } 1064 } 1065 1066 virtual_end = eva; 1067 return (eva); 1068 } 1069 #endif 1070 1071 /* 1072 * Helper routine that hands off available physical pages to the VM system. 1073 */ 1074 static void 1075 pmap_page_upload(void) 1076 { 1077 int n; 1078 paddr_t start, end; 1079 1080 /* First, the `etext gap' */ 1081 start = PMAP_BOOTSTRAP_VA2PA(etext_gap_start); 1082 end = PMAP_BOOTSTRAP_VA2PA(etext_gap_end); 1083 1084 #ifdef DIAGNOSTIC 1085 if (avail_start <= start) 1086 panic("pmap_page_upload: etext gap overlap: %lx < %lx", 1087 (u_long)avail_start, (u_long)start); 1088 #endif 1089 if (etext_gap_start < etext_gap_end) { 1090 vm_first_phys = start; 1091 uvm_page_physload( 1092 atop(start), 1093 atop(end), 1094 atop(start), 1095 atop(end), VM_FREELIST_DEFAULT); 1096 } 1097 1098 for (n = 0; n < npmemarr; n++) { 1099 1100 start = pmemarr[n].addr; 1101 end = start + pmemarr[n].len; 1102 1103 /* Update vm_{first_last}_phys */ 1104 if (vm_first_phys > start) 1105 vm_first_phys = start; 1106 if (vm_last_phys < end) 1107 vm_last_phys = end; 1108 1109 /* 1110 * Exclude any memory allocated for the kernel as computed 1111 * by pmap_bootstrap(), i.e. the range 1112 * [KERNBASE_PA, avail_start>. 1113 * Note that this will also exclude the `etext gap' range 1114 * already uploaded above. 1115 */ 1116 if (start < PMAP_BOOTSTRAP_VA2PA(KERNBASE)) { 1117 /* 1118 * This segment starts below the kernel load address. 1119 * Chop it off at the start of the kernel. 1120 */ 1121 paddr_t chop = PMAP_BOOTSTRAP_VA2PA(KERNBASE); 1122 1123 if (end < chop) 1124 chop = end; 1125 #ifdef DEBUG 1126 prom_printf("bootstrap gap: start %lx, chop %lx, end %lx\n", 1127 start, chop, end); 1128 #endif 1129 uvm_page_physload( 1130 atop(start), 1131 atop(chop), 1132 atop(start), 1133 atop(chop), 1134 VM_FREELIST_DEFAULT); 1135 1136 /* 1137 * Adjust the start address to reflect the 1138 * uploaded portion of this segment. 1139 */ 1140 start = chop; 1141 } 1142 1143 /* Skip the current kernel address range */ 1144 if (start <= avail_start && avail_start < end) 1145 start = avail_start; 1146 1147 if (start == end) 1148 continue; 1149 1150 /* Upload (the rest of) this segment */ 1151 uvm_page_physload( 1152 atop(start), 1153 atop(end), 1154 atop(start), 1155 atop(end), VM_FREELIST_DEFAULT); 1156 } 1157 1158 #if defined(MULTIPROCESSOR) 1159 { 1160 CPU_INFO_ITERATOR cpunum; 1161 struct cpu_info *cpi; 1162 1163 for (CPU_INFO_FOREACH(cpunum, cpi)) { 1164 if (cpi->ci_free_sva1) 1165 uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)), 1166 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)), 1167 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)), 1168 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)), 1169 VM_FREELIST_DEFAULT); 1170 if (cpi->ci_free_sva2) 1171 uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)), 1172 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)), 1173 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)), 1174 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)), 1175 VM_FREELIST_DEFAULT); 1176 } 1177 } 1178 #endif 1179 } 1180 1181 /* 1182 * This routine is used by mmrw() to validate access to `/dev/mem'. 1183 */ 1184 int 1185 pmap_pa_exists(paddr_t pa) 1186 { 1187 int nmem; 1188 struct memarr *mp; 1189 1190 for (mp = pmemarr, nmem = npmemarr; --nmem >= 0; mp++) { 1191 if (pa >= mp->addr && pa < mp->addr + mp->len) 1192 return 1; 1193 } 1194 1195 return 0; 1196 } 1197 1198 /* update pv_flags given a valid pte */ 1199 #define MR4_4C(pte) (((pte) >> PG_M_SHIFT) & (PV_MOD | PV_REF)) 1200 #define MR4M(pte) (((pte) >> PG_M_SHIFT4M) & (PV_MOD4M | PV_REF4M)) 1201 1202 /*----------------------------------------------------------------*/ 1203 1204 /* 1205 * Agree with the monitor ROM as to how many MMU entries are 1206 * to be reserved, and map all of its segments into all contexts. 1207 * 1208 * Unfortunately, while the Version 0 PROM had a nice linked list of 1209 * taken virtual memory, the Version 2 PROM provides instead a convoluted 1210 * description of *free* virtual memory. Rather than invert this, we 1211 * resort to two magic constants from the PROM vector description file. 1212 */ 1213 #if defined(SUN4) || defined(SUN4C) 1214 void 1215 mmu_reservemon4_4c(int *nrp, int *nsp) 1216 { 1217 u_int va = 0, eva = 0; 1218 int mmuseg, i, nr, ns, vr, lastvr; 1219 int *pte; 1220 #if defined(SUN4_MMU3L) 1221 int mmureg; 1222 #endif 1223 struct regmap *rp; 1224 1225 #if defined(SUN4) 1226 if (CPU_ISSUN4) { 1227 prom_vstart = va = OLDMON_STARTVADDR; 1228 prom_vend = eva = OLDMON_ENDVADDR; 1229 } 1230 #endif 1231 #if defined(SUN4C) 1232 if (CPU_ISSUN4C) { 1233 prom_vstart = va = OPENPROM_STARTVADDR; 1234 prom_vend = eva = OPENPROM_ENDVADDR; 1235 } 1236 #endif 1237 ns = *nsp; 1238 nr = *nrp; 1239 lastvr = 0; 1240 while (va < eva) { 1241 vr = VA_VREG(va); 1242 rp = &pmap_kernel()->pm_regmap[vr]; 1243 1244 #if defined(SUN4_MMU3L) 1245 if (HASSUN4_MMU3L && vr != lastvr) { 1246 lastvr = vr; 1247 mmureg = getregmap(va); 1248 if (mmureg < nr) 1249 rp->rg_smeg = nr = mmureg; 1250 /* 1251 * On 3-level MMU machines, we distribute regions, 1252 * rather than segments, amongst the contexts. 1253 */ 1254 for (i = ncontext; --i > 0;) 1255 prom_setcontext(i, (void *)va, mmureg); 1256 } 1257 #endif 1258 mmuseg = getsegmap(va); 1259 if (mmuseg < ns) 1260 ns = mmuseg; 1261 1262 if (!HASSUN4_MMU3L) 1263 for (i = ncontext; --i > 0;) 1264 prom_setcontext(i, (void *)va, mmuseg); 1265 1266 if (mmuseg == seginval) { 1267 va += NBPSG; 1268 continue; 1269 } 1270 /* 1271 * Another PROM segment. Enter into region map. 1272 * Assume the entire segment is valid. 1273 */ 1274 rp->rg_nsegmap += 1; 1275 rp->rg_segmap[VA_VSEG(va)].sg_pmeg = mmuseg; 1276 rp->rg_segmap[VA_VSEG(va)].sg_npte = NPTESG; 1277 pte = rp->rg_segmap[VA_VSEG(va)].sg_pte; 1278 1279 /* PROM maps its memory user-accessible: fix it. */ 1280 for (i = NPTESG; --i >= 0; va += NBPG, pte++) { 1281 *pte = getpte4(va) | PG_S; 1282 setpte4(va, *pte); 1283 } 1284 } 1285 *nsp = ns; 1286 *nrp = nr; 1287 return; 1288 } 1289 #endif 1290 1291 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */ 1292 1293 u_long 1294 srmmu_bypass_read(u_long paddr) 1295 { 1296 unsigned long v; 1297 1298 if (cpuinfo.mxcc) { 1299 /* 1300 * We're going to have to use MMU passthrough. If we're on 1301 * a Viking SuperSPARC with a MultiCache Controller, we 1302 * need to set the AC (Alternate Cacheable) bit in the MMU's 1303 * control register in order to not by-pass the cache. 1304 */ 1305 1306 unsigned long s = lda(SRMMU_PCR, ASI_SRMMU); 1307 1308 /* set MMU AC bit */ 1309 sta(SRMMU_PCR, ASI_SRMMU, s | VIKING_PCR_AC); 1310 v = lda(paddr, ASI_BYPASS); 1311 sta(SRMMU_PCR, ASI_SRMMU, s); 1312 } else 1313 v = lda(paddr, ASI_BYPASS); 1314 1315 return (v); 1316 } 1317 1318 1319 /* 1320 * Take the monitor's initial page table layout, convert it to 3rd-level pte's 1321 * (it starts out as a L1 mapping), and install it along with a set of kernel 1322 * mapping tables as the kernel's initial page table setup. Also create and 1323 * enable a context table. I suppose we also want to block user-mode access 1324 * to the new kernel/ROM mappings. 1325 */ 1326 1327 /* 1328 * mmu_reservemon4m(): Copies the existing (ROM) page tables to kernel space, 1329 * converting any L1/L2 PTEs to L3 PTEs. Does *not* copy the L1 entry mapping 1330 * the kernel at KERNBASE since we don't want to map 16M of physical 1331 * memory for the kernel. Thus the kernel must be installed later! 1332 * Also installs ROM mappings into the kernel pmap. 1333 * NOTE: This also revokes all user-mode access to the mapped regions. 1334 */ 1335 void 1336 mmu_reservemon4m(struct pmap *kpmap) 1337 { 1338 unsigned int rom_ctxtbl; 1339 int te; 1340 1341 #if !(defined(PROM_AT_F0) || defined(MSIIEP)) 1342 prom_vstart = OPENPROM_STARTVADDR; 1343 prom_vend = OPENPROM_ENDVADDR; 1344 #else /* OBP3/OFW in JavaStations */ 1345 prom_vstart = 0xf0000000; 1346 #if defined(MSIIEP) 1347 prom_vend = 0xf0800000; 1348 #else 1349 prom_vend = 0xf0080000; 1350 #endif 1351 #endif 1352 1353 /* 1354 * XXX: although the sun4m can handle 36 bits of physical 1355 * address space, we assume that all these page tables, etc 1356 * are in the lower 4G (32-bits) of address space, i.e. out of I/O 1357 * space. Eventually this should be changed to support the 36 bit 1358 * physical addressing, in case some crazed ROM designer decides to 1359 * stick the pagetables up there. In that case, we should use MMU 1360 * transparent mode, (i.e. ASI 0x20 to 0x2f) to access 1361 * physical memory. 1362 */ 1363 1364 rom_ctxtbl = (lda(SRMMU_CXTPTR,ASI_SRMMU) << SRMMU_PPNPASHIFT); 1365 1366 te = srmmu_bypass_read(rom_ctxtbl); /* i.e. context 0 */ 1367 1368 switch (te & SRMMU_TETYPE) { 1369 case SRMMU_TEINVALID: 1370 cpuinfo.ctx_tbl[0] = SRMMU_TEINVALID; 1371 panic("mmu_reservemon4m: no existing L0 mapping! " 1372 "(How are we running?"); 1373 break; 1374 case SRMMU_TEPTE: 1375 panic("mmu_reservemon4m: can't handle ROM 4G page size"); 1376 /* XXX: Should make this work, however stupid it is */ 1377 break; 1378 case SRMMU_TEPTD: 1379 mmu_setup4m_L1(te, kpmap); 1380 break; 1381 default: 1382 panic("mmu_reservemon4m: unknown pagetable entry type"); 1383 } 1384 } 1385 1386 /* regtblptd - PTD for region table to be remapped */ 1387 void 1388 mmu_setup4m_L1(int regtblptd, struct pmap *kpmap) 1389 { 1390 unsigned int regtblrover; 1391 int i; 1392 unsigned int te; 1393 struct regmap *rp; 1394 int j, k; 1395 1396 /* 1397 * Here we scan the region table to copy any entries which appear. 1398 * We are only concerned with regions in kernel space and above 1399 * (i.e. regions VA_VREG(KERNBASE)+1 to 0xff). We ignore the first 1400 * region (at VA_VREG(KERNBASE)), since that is the 16MB L1 mapping 1401 * that the ROM used to map the kernel in initially. Later, we will 1402 * rebuild a new L3 mapping for the kernel and install it before 1403 * switching to the new pagetables. 1404 */ 1405 regtblrover = 1406 ((regtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT) + 1407 (VA_VREG(KERNBASE)+1) * sizeof(long); /* kernel only */ 1408 1409 for (i = VA_VREG(KERNBASE) + 1; i < SRMMU_L1SIZE; 1410 i++, regtblrover += sizeof(long)) { 1411 1412 /* The region we're dealing with */ 1413 rp = &kpmap->pm_regmap[i]; 1414 1415 te = srmmu_bypass_read(regtblrover); 1416 switch(te & SRMMU_TETYPE) { 1417 case SRMMU_TEINVALID: 1418 break; 1419 1420 case SRMMU_TEPTE: 1421 #ifdef DEBUG 1422 prom_printf("mmu_setup4m_L1: " 1423 "converting region 0x%x from L1->L3\n", i); 1424 #endif 1425 /* 1426 * This region entry covers 64MB of memory -- or 1427 * (NSEGRG * NPTESG) pages -- which we must convert 1428 * into a 3-level description. 1429 */ 1430 1431 for (j = 0; j < SRMMU_L2SIZE; j++) { 1432 struct segmap *sp = &rp->rg_segmap[j]; 1433 1434 for (k = 0; k < SRMMU_L3SIZE; k++) { 1435 setpgt4m(&sp->sg_pte[k], 1436 (te & SRMMU_L1PPNMASK) | 1437 (j << SRMMU_L2PPNSHFT) | 1438 (k << SRMMU_L3PPNSHFT) | 1439 (te & SRMMU_PGBITSMSK) | 1440 ((te & SRMMU_PROT_MASK) | 1441 PPROT_U2S_OMASK) | 1442 SRMMU_TEPTE); 1443 } 1444 } 1445 break; 1446 1447 case SRMMU_TEPTD: 1448 mmu_setup4m_L2(te, rp); 1449 break; 1450 1451 default: 1452 panic("mmu_setup4m_L1: unknown pagetable entry type"); 1453 } 1454 } 1455 } 1456 1457 void 1458 mmu_setup4m_L2(int segtblptd, struct regmap *rp) 1459 { 1460 unsigned int segtblrover; 1461 int i, k; 1462 unsigned int te; 1463 struct segmap *sp; 1464 1465 segtblrover = (segtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT; 1466 for (i = 0; i < SRMMU_L2SIZE; i++, segtblrover += sizeof(long)) { 1467 1468 sp = &rp->rg_segmap[i]; 1469 1470 te = srmmu_bypass_read(segtblrover); 1471 switch(te & SRMMU_TETYPE) { 1472 case SRMMU_TEINVALID: 1473 break; 1474 1475 case SRMMU_TEPTE: 1476 #ifdef DEBUG 1477 prom_printf("mmu_setup4m_L2: converting L2 entry at segment 0x%x to L3\n",i); 1478 #endif 1479 /* 1480 * This segment entry covers 256KB of memory -- or 1481 * (NPTESG) pages -- which we must convert 1482 * into a 3-level description. 1483 */ 1484 for (k = 0; k < SRMMU_L3SIZE; k++) { 1485 setpgt4m(&sp->sg_pte[k], 1486 (te & SRMMU_L1PPNMASK) | 1487 (te & SRMMU_L2PPNMASK) | 1488 (k << SRMMU_L3PPNSHFT) | 1489 (te & SRMMU_PGBITSMSK) | 1490 ((te & SRMMU_PROT_MASK) | 1491 PPROT_U2S_OMASK) | 1492 SRMMU_TEPTE); 1493 } 1494 break; 1495 1496 case SRMMU_TEPTD: 1497 mmu_setup4m_L3(te, sp); 1498 break; 1499 1500 default: 1501 panic("mmu_setup4m_L2: unknown pagetable entry type"); 1502 } 1503 } 1504 } 1505 1506 void 1507 mmu_setup4m_L3(int pagtblptd, struct segmap *sp) 1508 { 1509 unsigned int pagtblrover; 1510 int i; 1511 unsigned int te; 1512 1513 pagtblrover = (pagtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT; 1514 for (i = 0; i < SRMMU_L3SIZE; i++, pagtblrover += sizeof(long)) { 1515 te = srmmu_bypass_read(pagtblrover); 1516 switch(te & SRMMU_TETYPE) { 1517 case SRMMU_TEINVALID: 1518 break; 1519 case SRMMU_TEPTE: 1520 setpgt4m(&sp->sg_pte[i], te | PPROT_U2S_OMASK); 1521 pmap_kernel()->pm_stats.resident_count++; 1522 break; 1523 case SRMMU_TEPTD: 1524 panic("mmu_setup4m_L3: PTD found in L3 page table"); 1525 default: 1526 panic("mmu_setup4m_L3: unknown pagetable entry type"); 1527 } 1528 } 1529 } 1530 #endif /* defined SUN4M || defined SUN4D */ 1531 1532 /*----------------------------------------------------------------*/ 1533 1534 #if defined(SUN4) || defined(SUN4C) 1535 /* 1536 * MMU management. 1537 */ 1538 static int me_alloc(struct mmuq *, struct pmap *, int, int); 1539 static void me_free(struct pmap *, u_int); 1540 #if defined(SUN4_MMU3L) 1541 static int region_alloc(struct mmuq *, struct pmap *, int); 1542 static void region_free(struct pmap *, u_int); 1543 #endif 1544 1545 1546 /* 1547 * Allocate an MMU entry (i.e., a PMEG). 1548 * If necessary, steal one from someone else. 1549 * Put it on the tail of the given queue 1550 * (which is either the LRU list or the locked list). 1551 * The locked list is not actually ordered, but this is easiest. 1552 * Also put it on the given (new) pmap's chain, 1553 * enter its pmeg number into that pmap's segmap, 1554 * and store the pmeg's new virtual segment number (me->me_vseg). 1555 * 1556 * This routine is large and complicated, but it must be fast 1557 * since it implements the dynamic allocation of MMU entries. 1558 */ 1559 1560 static inline int 1561 me_alloc(struct mmuq *mh, struct pmap *newpm, int newvreg, int newvseg) 1562 { 1563 struct mmuentry *me; 1564 struct pmap *pm; 1565 int i, va, *ptep, pte; 1566 int ctx; 1567 struct regmap *rp; 1568 struct segmap *sp; 1569 1570 /* try free list first */ 1571 if (!MMUQ_EMPTY(&segm_freelist)) { 1572 me = MMUQ_FIRST(&segm_freelist); 1573 MMUQ_REMOVE(me, me_list); 1574 #ifdef DEBUG 1575 if (me->me_pmap != NULL) 1576 panic("me_alloc: freelist entry has pmap"); 1577 if (pmapdebug & PDB_MMU_ALLOC) 1578 printf("me_alloc: got pmeg %d\n", me->me_cookie); 1579 #endif 1580 MMUQ_INSERT_TAIL(mh, me, me_list); 1581 1582 /* onto on pmap chain; pmap is already locked, if needed */ 1583 TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain); 1584 #ifdef DIAGNOSTIC 1585 pmap_stats.ps_npmeg_free--; 1586 if (mh == &segm_locked) { 1587 pmap_stats.ps_npmeg_locked++; 1588 me->me_statp = &pmap_stats.ps_npmeg_locked; 1589 } else { 1590 pmap_stats.ps_npmeg_lru++; 1591 me->me_statp = &pmap_stats.ps_npmeg_lru; 1592 } 1593 #endif 1594 1595 /* into pmap segment table, with backpointers */ 1596 me->me_pmap = newpm; 1597 me->me_vseg = newvseg; 1598 me->me_vreg = newvreg; 1599 1600 return (me->me_cookie); 1601 } 1602 1603 /* no luck, take head of LRU list */ 1604 if (MMUQ_EMPTY(&segm_lru)) 1605 panic("me_alloc: all pmegs gone"); 1606 1607 me = MMUQ_FIRST(&segm_lru); 1608 pm = me->me_pmap; 1609 #ifdef DEBUG 1610 if (pmapdebug & (PDB_MMU_ALLOC | PDB_MMU_STEAL)) 1611 printf("me_alloc: stealing pmeg 0x%x from pmap %p\n", 1612 me->me_cookie, pm); 1613 #endif 1614 1615 mmu_stolenpmegs_evcnt.ev_count++; 1616 1617 /* 1618 * Remove from LRU list, and insert at end of new list 1619 * (probably the LRU list again, but so what?). 1620 */ 1621 MMUQ_REMOVE(me, me_list); 1622 MMUQ_INSERT_TAIL(mh, me, me_list); 1623 1624 #ifdef DIAGNOSTIC 1625 if (mh == &segm_locked) { 1626 pmap_stats.ps_npmeg_lru--; 1627 pmap_stats.ps_npmeg_locked++; 1628 me->me_statp = &pmap_stats.ps_npmeg_locked; 1629 } else { 1630 me->me_statp = &pmap_stats.ps_npmeg_lru; 1631 } 1632 #endif 1633 1634 rp = &pm->pm_regmap[me->me_vreg]; 1635 sp = &rp->rg_segmap[me->me_vseg]; 1636 ptep = sp->sg_pte; 1637 1638 #ifdef DEBUG 1639 if (sp->sg_pmeg != me->me_cookie) 1640 panic("me_alloc: wrong sg_pmeg (%d != %d)", 1641 sp->sg_pmeg, me->me_cookie); 1642 #endif 1643 1644 /* 1645 * The PMEG must be mapped into some context so that we can 1646 * read its PTEs. Use its current context if it has one; 1647 * if not, and since context 0 is reserved for the kernel, 1648 * the simplest method is to switch to 0 and map the PMEG 1649 * to virtual address 0---which, being a user space address, 1650 * is by definition not in use. 1651 * 1652 * XXX do not have to flush cache immediately 1653 */ 1654 ctx = getcontext4(); 1655 1656 /* 1657 * Even if we're stealing a PMEG from ourselves (i.e. if pm==newpm), 1658 * we must make sure there are no user register windows in the CPU 1659 * for the following reasons: 1660 * (1) if we have a write-allocate cache and the segment we are 1661 * stealing contains stack pages, an interrupt during the 1662 * interval that starts at cache_flush_segment() below and ends 1663 * when the segment is finally removed from the MMU, may cause 1664 * dirty cache lines to reappear. 1665 * (2) when re-wiring this PMEG for use by another segment (e.g. 1666 * in mmu_pagein()) a window exists where the PTEs in this PMEG 1667 * point at arbitrary pages allocated to this address space. 1668 * Again, a register window flush at this point is likely to 1669 * cause data corruption in case the segment being rewired 1670 * contains stack virtual addresses. 1671 */ 1672 write_user_windows(); 1673 if (CTX_USABLE(pm,rp)) { 1674 setcontext4(pm->pm_ctxnum); 1675 va = VSTOVA(me->me_vreg, me->me_vseg); 1676 #ifdef DEBUG 1677 if (getsegmap(va) != me->me_cookie) 1678 panic("me_alloc: wrong pmeg in MMU (%d != %d)", 1679 getsegmap(va), me->me_cookie); 1680 #endif 1681 cache_flush_segment(me->me_vreg, me->me_vseg, pm->pm_ctxnum); 1682 } else { 1683 va = 0; 1684 setcontext4(0); 1685 if (HASSUN4_MMU3L) 1686 setregmap(va, tregion); 1687 setsegmap(va, me->me_cookie); 1688 /* 1689 * No cache flush needed: it happened earlier when 1690 * the old context was taken. 1691 */ 1692 } 1693 1694 /* 1695 * Record reference and modify bits for each page, 1696 * and copy PTEs into kernel memory so that they can 1697 * be reloaded later. 1698 */ 1699 i = NPTESG; 1700 do { 1701 int swbits = *ptep & PG_MBZ; 1702 pte = getpte4(va); 1703 if ((pte & (PG_V | PG_TYPE)) == (PG_V | PG_OBMEM)) { 1704 struct vm_page *pg; 1705 if ((pg = pvhead4_4c(pte)) != NULL) 1706 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte); 1707 } 1708 *ptep++ = swbits | (pte & ~(PG_U|PG_M)); 1709 va += NBPG; 1710 } while (--i > 0); 1711 1712 /* update segment tables */ 1713 if (CTX_USABLE(pm,rp)) { 1714 va = VSTOVA(me->me_vreg,me->me_vseg); 1715 if (pm != pmap_kernel() || HASSUN4_MMU3L) 1716 setsegmap(va, seginval); 1717 else { 1718 /* Unmap segment from all contexts */ 1719 for (i = ncontext; --i >= 0;) { 1720 setcontext4(i); 1721 setsegmap(va, seginval); 1722 } 1723 } 1724 } 1725 sp->sg_pmeg = seginval; 1726 1727 /* off old pmap chain */ 1728 TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain); 1729 setcontext4(ctx); 1730 1731 /* onto new pmap chain; new pmap is already locked, if needed */ 1732 TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain); 1733 1734 /* into new segment table, with backpointers */ 1735 me->me_pmap = newpm; 1736 me->me_vseg = newvseg; 1737 me->me_vreg = newvreg; 1738 1739 return (me->me_cookie); 1740 } 1741 1742 /* 1743 * Free an MMU entry. 1744 * 1745 * Assumes the corresponding pmap is already locked. 1746 * Caller must update hardware. 1747 */ 1748 static inline void 1749 me_free(struct pmap *pm, u_int pmeg) 1750 { 1751 struct mmuentry *me = &mmusegments[pmeg]; 1752 #ifdef DEBUG 1753 struct regmap *rp; 1754 int i, va, tpte, ctx; 1755 #endif 1756 1757 #ifdef DEBUG 1758 rp = &pm->pm_regmap[me->me_vreg]; 1759 if (pmapdebug & PDB_MMU_ALLOC) 1760 printf("me_free: freeing pmeg %d from pmap %p\n", 1761 me->me_cookie, pm); 1762 if (me->me_cookie != pmeg) 1763 panic("me_free: wrong mmuentry"); 1764 if (pm != me->me_pmap) 1765 panic("me_free: pm != me_pmap"); 1766 if (rp->rg_segmap[me->me_vseg].sg_pmeg != pmeg && 1767 rp->rg_segmap[me->me_vseg].sg_pmeg != seginval) 1768 panic("me_free: wrong sg_pmeg (%d != %d)", 1769 rp->rg_segmap[me->me_vseg].sg_pmeg, pmeg); 1770 1771 /* check for spurious mappings (using temp. mapping in context 0) */ 1772 ctx = getcontext4(); 1773 setcontext4(0); 1774 if (HASSUN4_MMU3L) 1775 setregmap(0, tregion); 1776 setsegmap(0, me->me_cookie); 1777 va = 0; 1778 i = NPTESG; 1779 do { 1780 tpte = getpte4(va); 1781 if ((tpte & PG_V) == PG_V) 1782 panic("me_free: segment not clean (pte=%x)", tpte); 1783 va += NBPG; 1784 } while (--i > 0); 1785 setcontext4(ctx); 1786 #endif /* DEBUG */ 1787 1788 /* take mmu entry off pmap chain */ 1789 TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain); 1790 1791 /* off LRU or lock chain */ 1792 MMUQ_REMOVE(me, me_list); 1793 #ifdef DIAGNOSTIC 1794 if (me->me_statp == NULL) 1795 panic("me_statp"); 1796 (*me->me_statp)--; 1797 me->me_statp = NULL; 1798 #endif 1799 1800 /* no associated pmap; on free list */ 1801 me->me_pmap = NULL; 1802 MMUQ_INSERT_TAIL(&segm_freelist, me, me_list); 1803 #ifdef DIAGNOSTIC 1804 pmap_stats.ps_npmeg_free++; 1805 #endif 1806 } 1807 1808 #if defined(SUN4_MMU3L) 1809 1810 /* XXX - Merge with segm_alloc/segm_free ? */ 1811 1812 int 1813 region_alloc(struct mmuq *mh, struct pmap *newpm, int newvr) 1814 { 1815 struct mmuentry *me; 1816 struct pmap *pm; 1817 int ctx; 1818 struct regmap *rp; 1819 1820 /* try free list first */ 1821 if (!MMUQ_EMPTY(®ion_freelist)) { 1822 me = MMUQ_FIRST(®ion_freelist); 1823 MMUQ_REMOVE(me, me_list); 1824 #ifdef DEBUG 1825 if (me->me_pmap != NULL) 1826 panic("region_alloc: freelist entry has pmap"); 1827 if (pmapdebug & PDB_MMUREG_ALLOC) 1828 printf("region_alloc: got smeg 0x%x\n", me->me_cookie); 1829 #endif 1830 MMUQ_INSERT_TAIL(mh, me, me_list); 1831 1832 /* onto on pmap chain; pmap is already locked, if needed */ 1833 TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain); 1834 1835 /* into pmap segment table, with backpointers */ 1836 me->me_pmap = newpm; 1837 me->me_vreg = newvr; 1838 1839 return (me->me_cookie); 1840 } 1841 1842 /* no luck, take head of LRU list */ 1843 if (MMUQ_EMPTY(®ion_lru)) 1844 panic("region_alloc: all smegs gone"); 1845 1846 me = MMUQ_FIRST(®ion_lru); 1847 1848 pm = me->me_pmap; 1849 if (pm == NULL) 1850 panic("region_alloc: LRU entry has no pmap"); 1851 if (pm == pmap_kernel()) 1852 panic("region_alloc: stealing from kernel"); 1853 #ifdef DEBUG 1854 if (pmapdebug & (PDB_MMUREG_ALLOC | PDB_MMUREG_STEAL)) 1855 printf("region_alloc: stealing smeg 0x%x from pmap %p\n", 1856 me->me_cookie, pm); 1857 #endif 1858 /* 1859 * Remove from LRU list, and insert at end of new list 1860 * (probably the LRU list again, but so what?). 1861 */ 1862 MMUQ_REMOVE(me, me_list); 1863 MMUQ_INSERT_TAIL(mh, me, me_list); 1864 1865 rp = &pm->pm_regmap[me->me_vreg]; 1866 ctx = getcontext4(); 1867 1868 /* Flush register windows; see comment in me_alloc() */ 1869 write_user_windows(); 1870 if (pm->pm_ctx) { 1871 setcontext4(pm->pm_ctxnum); 1872 cache_flush_region(me->me_vreg, pm->pm_ctxnum); 1873 } 1874 1875 /* update region tables */ 1876 if (pm->pm_ctx) 1877 setregmap(VRTOVA(me->me_vreg), reginval); 1878 rp->rg_smeg = reginval; 1879 1880 /* off old pmap chain */ 1881 TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain); 1882 setcontext4(ctx); /* done with old context */ 1883 1884 /* onto new pmap chain; new pmap is already locked, if needed */ 1885 TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain); 1886 1887 /* into new segment table, with backpointers */ 1888 me->me_pmap = newpm; 1889 me->me_vreg = newvr; 1890 1891 return (me->me_cookie); 1892 } 1893 1894 /* 1895 * Free an MMU entry. 1896 * Assumes the corresponding pmap is already locked. 1897 * Caller must update hardware. 1898 */ 1899 void 1900 region_free(struct pmap *pm, u_int smeg) 1901 { 1902 struct mmuentry *me = &mmuregions[smeg]; 1903 1904 #ifdef DEBUG 1905 if (pmapdebug & PDB_MMUREG_ALLOC) 1906 printf("region_free: freeing smeg 0x%x from pmap %p\n", 1907 me->me_cookie, pm); 1908 if (me->me_cookie != smeg) 1909 panic("region_free: wrong mmuentry"); 1910 if (pm != me->me_pmap) 1911 panic("region_free: pm != me_pmap"); 1912 #endif 1913 1914 /* take mmu entry off pmap chain */ 1915 TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain); 1916 1917 /* off LRU or lock chain */ 1918 MMUQ_REMOVE(me, me_list); 1919 1920 /* no associated pmap; on free list */ 1921 me->me_pmap = NULL; 1922 MMUQ_INSERT_TAIL(®ion_freelist, me, me_list); 1923 } 1924 1925 static void 1926 mmu_pagein_reg(struct pmap *pm, struct regmap *rp, vaddr_t va, 1927 int vr, struct mmuq *mh) 1928 { 1929 int i, s, smeg; 1930 1931 va = VA_ROUNDDOWNTOREG(va); 1932 rp->rg_smeg = smeg = region_alloc(mh, pm, vr); 1933 1934 s = splvm(); 1935 if (pm == pmap_kernel()) { 1936 /* Map region into all contexts */ 1937 int ctx = getcontext4(); 1938 i = ncontext - 1; 1939 do { 1940 setcontext4(i); 1941 setregmap(va, smeg); 1942 } while (--i >= 0); 1943 setcontext4(ctx); 1944 } else 1945 setregmap(va, smeg); 1946 1947 /* Load PMEGs into this region */ 1948 for (i = 0; i < NSEGRG; i++) { 1949 setsegmap(va, rp->rg_segmap[i].sg_pmeg); 1950 va += NBPSG; 1951 } 1952 splx(s); 1953 } 1954 #endif /* SUN4_MMU3L */ 1955 1956 static void 1957 mmu_pmeg_lock(int pmeg) 1958 { 1959 struct mmuentry *me = &mmusegments[pmeg]; 1960 1961 MMUQ_REMOVE(me, me_list); 1962 MMUQ_INSERT_TAIL(&segm_locked, me, me_list); 1963 #ifdef DIAGNOSTIC 1964 (*me->me_statp)--; 1965 pmap_stats.ps_npmeg_locked++; 1966 me->me_statp = &pmap_stats.ps_npmeg_locked; 1967 #endif 1968 } 1969 1970 static void 1971 mmu_pmeg_unlock(int pmeg) 1972 { 1973 struct mmuentry *me = &mmusegments[pmeg]; 1974 1975 MMUQ_REMOVE(me, me_list); 1976 MMUQ_INSERT_TAIL(&segm_lru, me, me_list); 1977 #ifdef DIAGNOSTIC 1978 (*me->me_statp)--; 1979 pmap_stats.ps_npmeg_lru++; 1980 me->me_statp = &pmap_stats.ps_npmeg_lru; 1981 #endif 1982 } 1983 1984 static void 1985 mmu_pagein_seg(struct pmap *pm, struct segmap *sp, vaddr_t va, 1986 int vr, int vs, struct mmuq *mh) 1987 { 1988 int s, i, pmeg, *pte; 1989 1990 mmu_pagein_evcnt.ev_count++; 1991 1992 va = VA_ROUNDDOWNTOSEG(va); 1993 s = splvm(); /* paranoid */ 1994 sp->sg_pmeg = pmeg = me_alloc(mh, pm, vr, vs); 1995 if (pm != pmap_kernel() || HASSUN4_MMU3L) 1996 setsegmap(va, pmeg); 1997 else { 1998 /* Map kernel address into all contexts */ 1999 int ctx = getcontext4(); 2000 i = ncontext - 1; 2001 do { 2002 setcontext4(i); 2003 setsegmap(va, pmeg); 2004 } while (--i >= 0); 2005 setcontext4(ctx); 2006 } 2007 2008 /* reload segment: write PTEs into a the MMU */ 2009 pte = sp->sg_pte; 2010 i = NPTESG; 2011 do { 2012 setpte4(va, *pte++ & ~PG_MBZ); 2013 va += NBPG; 2014 } while (--i > 0); 2015 splx(s); 2016 } 2017 2018 /* 2019 * `Page in' (load or inspect) an MMU entry; called on page faults. 2020 * Returns 1 if we reloaded the segment, -1 if the segment was 2021 * already loaded and the page was marked valid (in which case the 2022 * fault must be a bus error or something), or 0 (segment loaded but 2023 * PTE not valid, or segment not loaded at all). 2024 */ 2025 int 2026 mmu_pagein(struct pmap *pm, vaddr_t va, int prot) 2027 { 2028 int vr, vs, bits; 2029 struct regmap *rp; 2030 struct segmap *sp; 2031 2032 PMAP_LOCK(); 2033 2034 if (prot != VM_PROT_NONE) 2035 bits = PG_V | ((prot & VM_PROT_WRITE) ? PG_W : 0); 2036 else 2037 bits = 0; 2038 2039 vr = VA_VREG(va); 2040 vs = VA_VSEG(va); 2041 rp = &pm->pm_regmap[vr]; 2042 2043 /* return 0 if we have no PMEGs to load */ 2044 if (rp->rg_nsegmap == 0) { 2045 PMAP_UNLOCK(); 2046 return (0); 2047 } 2048 2049 #ifdef DIAGNOSTIC 2050 if (rp->rg_segmap == NULL) 2051 panic("pagein: no segmap"); 2052 #endif 2053 2054 #if defined(SUN4_MMU3L) 2055 if (HASSUN4_MMU3L && rp->rg_smeg == reginval) 2056 mmu_pagein_reg(pm, rp, va, vr, ®ion_lru); 2057 #endif 2058 sp = &rp->rg_segmap[vs]; 2059 2060 /* return 0 if we have no PTEs to load */ 2061 if (sp->sg_npte == 0) { 2062 PMAP_UNLOCK(); 2063 return (0); 2064 } 2065 2066 /* return -1 if the fault is `hard', 0 if not */ 2067 if (sp->sg_pmeg != seginval) { 2068 PMAP_UNLOCK(); 2069 return (bits && (getpte4(va) & bits) == bits ? -1 : 0); 2070 } 2071 2072 mmu_pagein_seg(pm, sp, va, vr, vs, &segm_lru); 2073 PMAP_UNLOCK(); 2074 return (1); 2075 } 2076 #endif /* SUN4 or SUN4C */ 2077 2078 /* 2079 * Allocate a context. If necessary, steal one from someone else. 2080 * Changes hardware context number and loads segment map. 2081 * 2082 * This routine is only ever called from locore.s just after it has 2083 * saved away the previous process, so there are no active user windows. 2084 */ 2085 void 2086 ctx_alloc(struct pmap *pm) 2087 { 2088 union ctxinfo *c; 2089 int cnum, i = 0, doflush; 2090 struct regmap *rp; 2091 int gap_start, gap_end; 2092 vaddr_t va; 2093 #if defined(SUN4M) || defined(SUN4D) 2094 struct cpu_info *cpi; 2095 #endif 2096 2097 /*XXX-GCC!*/gap_start=gap_end=0; 2098 #ifdef DEBUG 2099 if (pm->pm_ctx) 2100 panic("ctx_alloc pm_ctx"); 2101 if (pmapdebug & PDB_CTX_ALLOC) 2102 printf("ctx_alloc[%d](%p)\n", cpu_number(), pm); 2103 #endif 2104 if (CPU_HAS_SUNMMU) { 2105 gap_start = pm->pm_gap_start; 2106 gap_end = pm->pm_gap_end; 2107 } 2108 2109 mutex_spin_enter(&ctx_lock); 2110 if ((c = ctx_freelist) != NULL) { 2111 ctx_freelist = c->c_nextfree; 2112 cnum = c - ctxinfo; 2113 doflush = 0; 2114 } else { 2115 if ((ctx_kick += ctx_kickdir) >= ncontext) { 2116 ctx_kick = ncontext - 1; 2117 ctx_kickdir = -1; 2118 } else if (ctx_kick < 1) { 2119 ctx_kick = 1; 2120 ctx_kickdir = 1; 2121 } 2122 c = &ctxinfo[cnum = ctx_kick]; 2123 #ifdef DEBUG 2124 if (c->c_pmap == NULL) 2125 panic("ctx_alloc cu_pmap"); 2126 if (pmapdebug & (PDB_CTX_ALLOC | PDB_CTX_STEAL)) 2127 printf("ctx_alloc[%d]: steal context %d from %p\n", 2128 cpu_number(), cnum, c->c_pmap); 2129 #endif 2130 c->c_pmap->pm_ctx = NULL; 2131 c->c_pmap->pm_ctxnum = 0; 2132 doflush = (CACHEINFO.c_vactype != VAC_NONE); 2133 if (CPU_HAS_SUNMMU) { 2134 if (gap_start < c->c_pmap->pm_gap_start) 2135 gap_start = c->c_pmap->pm_gap_start; 2136 if (gap_end > c->c_pmap->pm_gap_end) 2137 gap_end = c->c_pmap->pm_gap_end; 2138 } 2139 } 2140 2141 c->c_pmap = pm; 2142 pm->pm_ctx = c; 2143 pm->pm_ctxnum = cnum; 2144 2145 if (CPU_HAS_SUNMMU) { 2146 2147 /* 2148 * Write pmap's region (3-level MMU) or segment table into 2149 * the MMU. 2150 * 2151 * Only write those entries that actually map something in 2152 * this context by maintaining a pair of region numbers in 2153 * between which the pmap has no valid mappings. 2154 * 2155 * If a context was just allocated from the free list, trust 2156 * that all its pmeg numbers are `seginval'. We make sure this 2157 * is the case initially in pmap_bootstrap(). Otherwise, the 2158 * context was freed by calling ctx_free() in pmap_release(), 2159 * which in turn is supposedly called only when all mappings 2160 * have been removed. 2161 * 2162 * On the other hand, if the context had to be stolen from 2163 * another pmap, we possibly shrink the gap to be the 2164 * disjuction of the new and the previous map. 2165 */ 2166 2167 setcontext4(cnum); 2168 if (doflush) 2169 cache_flush_context(cnum); 2170 2171 rp = pm->pm_regmap; 2172 for (va = 0, i = NUREG; --i >= 0; ) { 2173 if (VA_VREG(va) >= gap_start) { 2174 va = VRTOVA(gap_end); 2175 i -= gap_end - gap_start; 2176 rp += gap_end - gap_start; 2177 if (i < 0) 2178 break; 2179 /* mustn't re-enter this branch */ 2180 gap_start = NUREG; 2181 } 2182 if (HASSUN4_MMU3L) { 2183 setregmap(va, rp++->rg_smeg); 2184 va += NBPRG; 2185 } else { 2186 int j; 2187 struct segmap *sp = rp->rg_segmap; 2188 for (j = NSEGRG; --j >= 0; va += NBPSG) 2189 setsegmap(va, 2190 sp?sp++->sg_pmeg:seginval); 2191 rp++; 2192 } 2193 } 2194 2195 } else if (CPU_HAS_SRMMU) { 2196 2197 #if defined(SUN4M) || defined(SUN4D) 2198 /* 2199 * Reload page and context tables to activate the page tables 2200 * for this context. 2201 * 2202 * The gap stuff isn't really needed in the sun4m architecture, 2203 * since we don't have to worry about excessive mappings (all 2204 * mappings exist since the page tables must be complete for 2205 * the mmu to be happy). 2206 * 2207 * If a context was just allocated from the free list, trust 2208 * that all of its mmu-edible page tables are zeroed out 2209 * (except for those associated with the kernel). We make 2210 * sure this is the case initially in pmap_bootstrap() and 2211 * pmap_init() (?). 2212 * Otherwise, the context was freed by calling ctx_free() in 2213 * pmap_release(), which in turn is supposedly called only 2214 * when all mappings have been removed. 2215 * 2216 * XXX: Do we have to flush cache after reloading ctx tbl? 2217 */ 2218 2219 /* 2220 * We need to flush the cache only when stealing a context 2221 * from another pmap. In that case it's Ok to switch the 2222 * context and leave it set, since the context table 2223 * will have a valid region table entry for this context 2224 * number. 2225 * 2226 * Otherwise, we switch to the new context after loading 2227 * the context table entry with the new pmap's region. 2228 */ 2229 if (doflush) { 2230 cache_flush_context(cnum); 2231 } 2232 2233 /* 2234 * The context allocated to a process is the same on all CPUs. 2235 * Here we install the per-CPU region table in each CPU's 2236 * context table slot. 2237 * 2238 * Note on multi-threaded processes: a context must remain 2239 * valid as long as any thread is still running on a CPU. 2240 */ 2241 for (CPU_INFO_FOREACH(i, cpi)) { 2242 setpgt4m(&cpi->ctx_tbl[cnum], 2243 (pm->pm_reg_ptps_pa[i] >> SRMMU_PPNPASHIFT) | 2244 SRMMU_TEPTD); 2245 } 2246 2247 /* And finally switch to the new context */ 2248 (*cpuinfo.pure_vcache_flush)(); 2249 setcontext4m(cnum); 2250 #endif /* SUN4M || SUN4D */ 2251 } 2252 mutex_spin_exit(&ctx_lock); 2253 } 2254 2255 /* 2256 * Give away a context. 2257 */ 2258 void 2259 ctx_free(struct pmap *pm) 2260 { 2261 union ctxinfo *c; 2262 int ctx; 2263 #if defined(SUN4M) || defined(SUN4D) 2264 struct cpu_info *cpi; 2265 #endif 2266 2267 c = pm->pm_ctx; 2268 ctx = pm->pm_ctxnum; 2269 pm->pm_ctx = NULL; 2270 pm->pm_ctxnum = 0; 2271 #if defined(SUN4) || defined(SUN4C) 2272 if (CPU_HAS_SUNMMU) { 2273 int octx = getcontext4(); 2274 setcontext4(ctx); 2275 cache_flush_context(ctx); 2276 setcontext4(octx); 2277 } 2278 #endif /* SUN4 || SUN4C */ 2279 2280 mutex_spin_enter(&ctx_lock); 2281 2282 #if defined(SUN4M) || defined(SUN4D) 2283 if (CPU_HAS_SRMMU) { 2284 int i; 2285 2286 cache_flush_context(ctx); 2287 tlb_flush_context(ctx, PMAP_CPUSET(pm)); 2288 for (CPU_INFO_FOREACH(i, cpi)) { 2289 setpgt4m(&cpi->ctx_tbl[ctx], SRMMU_TEINVALID); 2290 } 2291 } 2292 #endif 2293 2294 c->c_nextfree = ctx_freelist; 2295 ctx_freelist = c; 2296 mutex_spin_exit(&ctx_lock); 2297 } 2298 2299 2300 /*----------------------------------------------------------------*/ 2301 2302 /* 2303 * pvlist functions. 2304 */ 2305 2306 /* 2307 * Walk the given pv list, and for each PTE, set or clear some bits 2308 * (e.g., PG_W or PG_NC). 2309 * 2310 * This routine flushes the cache for any page whose PTE changes, 2311 * as long as the process has a context; this is overly conservative. 2312 * It also copies ref and mod bits to the pvlist, on the theory that 2313 * this might save work later. (XXX should test this theory) 2314 */ 2315 2316 #if defined(SUN4) || defined(SUN4C) 2317 2318 void 2319 pv_changepte4_4c(struct vm_page *pg, int bis, int bic) 2320 { 2321 int pte, *ptep; 2322 struct pvlist *pv; 2323 struct pmap *pm; 2324 int va, vr, vs; 2325 int ctx, s; 2326 struct regmap *rp; 2327 struct segmap *sp; 2328 2329 pv = VM_MDPAGE_PVHEAD(pg); 2330 2331 write_user_windows(); /* paranoid? */ 2332 s = splvm(); /* paranoid? */ 2333 if (pv->pv_pmap == NULL) { 2334 splx(s); 2335 return; 2336 } 2337 ctx = getcontext4(); 2338 for (; pv != NULL; pv = pv->pv_next) { 2339 pm = pv->pv_pmap; 2340 va = pv->pv_va; 2341 vr = VA_VREG(va); 2342 vs = VA_VSEG(va); 2343 rp = &pm->pm_regmap[vr]; 2344 sp = &rp->rg_segmap[vs]; 2345 ptep = &sp->sg_pte[VA_VPG(va)]; 2346 2347 if (sp->sg_pmeg == seginval) { 2348 /* not in hardware: just fix software copy */ 2349 *ptep = (*ptep | bis) & ~bic; 2350 } else { 2351 /* in hardware: fix hardware copy */ 2352 if (CTX_USABLE(pm,rp)) { 2353 setcontext4(pm->pm_ctxnum); 2354 /* XXX should flush only when necessary */ 2355 pte = getpte4(va); 2356 /* 2357 * XXX: always flush cache; conservative, but 2358 * needed to invalidate cache tag protection 2359 * bits and when disabling caching. 2360 */ 2361 cache_flush_page(va, pm->pm_ctxnum); 2362 } else { 2363 /* Make temp map in ctx 0 to access the PTE */ 2364 setcontext4(0); 2365 if (HASSUN4_MMU3L) 2366 setregmap(0, tregion); 2367 setsegmap(0, sp->sg_pmeg); 2368 va = VA_VPG(va) << PGSHIFT; 2369 pte = getpte4(va); 2370 } 2371 if (pte & PG_V) 2372 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte); 2373 pte = (pte | bis) & ~bic; 2374 setpte4(va, pte); 2375 *ptep = (*ptep & PG_MBZ) | pte; 2376 } 2377 } 2378 setcontext4(ctx); 2379 splx(s); 2380 } 2381 2382 /* 2383 * Sync ref and mod bits in pvlist (turns off same in hardware PTEs). 2384 * Returns the new flags. 2385 * 2386 * This is just like pv_changepte, but we never add or remove bits, 2387 * hence never need to adjust software copies. 2388 */ 2389 int 2390 pv_syncflags4_4c(struct vm_page *pg) 2391 { 2392 struct pvlist *pv; 2393 struct pmap *pm; 2394 int pte, va, vr, vs, pmeg, flags; 2395 int ctx, s; 2396 struct regmap *rp; 2397 struct segmap *sp; 2398 2399 pv = VM_MDPAGE_PVHEAD(pg); 2400 2401 s = splvm(); /* paranoid? */ 2402 if (pv->pv_pmap == NULL) { 2403 /* Page not mapped; pv_flags is already up to date */ 2404 splx(s); 2405 return (0); 2406 } 2407 ctx = getcontext4(); 2408 flags = pv->pv_flags; 2409 for (; pv != NULL; pv = pv->pv_next) { 2410 pm = pv->pv_pmap; 2411 va = pv->pv_va; 2412 vr = VA_VREG(va); 2413 vs = VA_VSEG(va); 2414 rp = &pm->pm_regmap[vr]; 2415 sp = &rp->rg_segmap[vs]; 2416 if ((pmeg = sp->sg_pmeg) == seginval) 2417 continue; 2418 if (CTX_USABLE(pm,rp)) { 2419 setcontext4(pm->pm_ctxnum); 2420 /* XXX should flush only when necessary */ 2421 pte = getpte4(va); 2422 if (pte & PG_M) 2423 cache_flush_page(va, pm->pm_ctxnum); 2424 } else { 2425 /* Make temp map in ctx 0 to access the PTE */ 2426 setcontext4(0); 2427 if (HASSUN4_MMU3L) 2428 setregmap(0, tregion); 2429 setsegmap(0, pmeg); 2430 va = VA_VPG(va) << PGSHIFT; 2431 pte = getpte4(va); 2432 } 2433 if (pte & (PG_M|PG_U) && pte & PG_V) { 2434 flags |= MR4_4C(pte); 2435 pte &= ~(PG_M|PG_U); 2436 setpte4(va, pte); 2437 } 2438 } 2439 2440 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags; 2441 setcontext4(ctx); 2442 splx(s); 2443 return (flags); 2444 } 2445 2446 /* 2447 * pv_unlink is a helper function for pmap_remove. 2448 * It takes a pointer to the pv_table head for some physical address 2449 * and removes the appropriate (pmap, va) entry. 2450 * 2451 * Once the entry is removed, if the pv_table head has the cache 2452 * inhibit bit set, see if we can turn that off; if so, walk the 2453 * pvlist and turn off PG_NC in each PTE. (The pvlist is by 2454 * definition nonempty, since it must have at least two elements 2455 * in it to have PV_NC set, and we only remove one here.) 2456 */ 2457 /*static*/ void 2458 pv_unlink4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va) 2459 { 2460 struct pvlist *pv0, *npv; 2461 2462 pv0 = VM_MDPAGE_PVHEAD(pg); 2463 npv = pv0->pv_next; 2464 2465 /* 2466 * First entry is special (sigh). 2467 */ 2468 if (pv0->pv_pmap == pm && pv0->pv_va == va) { 2469 pmap_stats.ps_unlink_pvfirst++; 2470 if (npv != NULL) { 2471 /* 2472 * Shift next entry into the head. 2473 * Make sure to retain the REF, MOD and ANC flags. 2474 */ 2475 pv0->pv_next = npv->pv_next; 2476 pv0->pv_pmap = npv->pv_pmap; 2477 pv0->pv_va = npv->pv_va; 2478 pv0->pv_flags &= ~PV_NC; 2479 pv0->pv_flags |= (npv->pv_flags & PV_NC); 2480 pool_put(&pv_pool, npv); 2481 } else { 2482 /* 2483 * No mappings left; we still need to maintain 2484 * the REF and MOD flags. since pmap_is_modified() 2485 * can still be called for this page. 2486 */ 2487 pv0->pv_pmap = NULL; 2488 pv0->pv_flags &= ~(PV_NC|PV_ANC); 2489 return; 2490 } 2491 } else { 2492 struct pvlist *prev; 2493 2494 pmap_stats.ps_unlink_pvsearch++; 2495 for (prev = pv0;; prev = npv, npv = npv->pv_next) { 2496 if (npv == NULL) { 2497 panic("pv_unlink: pm %p is missing on pg %p", 2498 pm, pg); 2499 } 2500 if (npv->pv_pmap == pm && npv->pv_va == va) 2501 break; 2502 } 2503 prev->pv_next = npv->pv_next; 2504 pool_put(&pv_pool, npv); 2505 } 2506 if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) { 2507 /* 2508 * Not cached: check whether we can fix that now. 2509 */ 2510 va = pv0->pv_va; 2511 for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next) 2512 if (BADALIAS(va, npv->pv_va) || 2513 (npv->pv_flags & PV_NC) != 0) 2514 return; 2515 pv0->pv_flags &= ~PV_ANC; 2516 pv_changepte4_4c(pg, 0, PG_NC); 2517 } 2518 } 2519 2520 /* 2521 * pv_link is the inverse of pv_unlink, and is used in pmap_enter. 2522 * It returns PG_NC if the (new) pvlist says that the address cannot 2523 * be cached. 2524 */ 2525 /*static*/ int 2526 pv_link4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va, 2527 unsigned int *pteprotop) 2528 { 2529 struct pvlist *pv0, *pv, *npv; 2530 int nc = (*pteprotop & PG_NC) != 0 ? PV_NC : 0; 2531 2532 pv0 = VM_MDPAGE_PVHEAD(pg); 2533 2534 if (pv0->pv_pmap == NULL) { 2535 /* no pvlist entries yet */ 2536 pmap_stats.ps_enter_firstpv++; 2537 pv0->pv_next = NULL; 2538 pv0->pv_pmap = pm; 2539 pv0->pv_va = va; 2540 pv0->pv_flags |= nc; 2541 return (0); 2542 } 2543 2544 /* 2545 * Allocate the new PV entry now, and, if that fails, bail out 2546 * before changing the cacheable state of the existing mappings. 2547 */ 2548 npv = pool_get(&pv_pool, PR_NOWAIT); 2549 if (npv == NULL) 2550 return (ENOMEM); 2551 2552 pmap_stats.ps_enter_secondpv++; 2553 2554 /* 2555 * Before entering the new mapping, see if 2556 * it will cause old mappings to become aliased 2557 * and thus need to be `discached'. 2558 */ 2559 if (pv0->pv_flags & PV_ANC) { 2560 /* already uncached, just stay that way */ 2561 *pteprotop |= PG_NC; 2562 goto link_npv; 2563 } 2564 2565 for (pv = pv0; pv != NULL; pv = pv->pv_next) { 2566 if ((pv->pv_flags & PV_NC) != 0) { 2567 *pteprotop |= PG_NC; 2568 #ifdef DEBUG 2569 /* Check currently illegal condition */ 2570 if (nc == 0) 2571 printf("pv_link: proc %s, va=0x%lx: " 2572 "unexpected uncached mapping at 0x%lx\n", 2573 curproc ? curproc->p_comm : "--", 2574 va, pv->pv_va); 2575 #endif 2576 } 2577 if (BADALIAS(va, pv->pv_va)) { 2578 #ifdef DEBUG 2579 if (pmapdebug & PDB_CACHESTUFF) 2580 printf( 2581 "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p\n", 2582 curproc ? curproc->p_comm : "--", 2583 va, pv->pv_va, pg); 2584 #endif 2585 /* Mark list head `uncached due to aliases' */ 2586 pv0->pv_flags |= PV_ANC; 2587 pv_changepte4_4c(pg, PG_NC, 0); 2588 *pteprotop |= PG_NC; 2589 break; 2590 } 2591 } 2592 2593 link_npv: 2594 npv->pv_next = pv0->pv_next; 2595 npv->pv_pmap = pm; 2596 npv->pv_va = va; 2597 npv->pv_flags = nc; 2598 pv0->pv_next = npv; 2599 return (0); 2600 } 2601 2602 #endif /* SUN4 || SUN4C */ 2603 2604 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */ 2605 /* 2606 * Walk the given pv list, and for each PTE, set or clear some bits 2607 * (e.g., PG_W or PG_NC). 2608 * 2609 * This routine flushes the cache for any page whose PTE changes, 2610 * as long as the process has a context; this is overly conservative. 2611 * It also copies ref and mod bits to the pvlist, on the theory that 2612 * this might save work later. (XXX should test this theory) 2613 * 2614 * Called with PV lock and pmap main lock held. 2615 */ 2616 void 2617 pv_changepte4m(struct vm_page *pg, int bis, int bic) 2618 { 2619 struct pvlist *pv; 2620 struct pmap *pm; 2621 vaddr_t va; 2622 struct regmap *rp; 2623 struct segmap *sp; 2624 2625 pv = VM_MDPAGE_PVHEAD(pg); 2626 if (pv->pv_pmap == NULL) 2627 return; 2628 2629 for (; pv != NULL; pv = pv->pv_next) { 2630 int tpte; 2631 pm = pv->pv_pmap; 2632 /* XXXSMP: should lock pm */ 2633 va = pv->pv_va; 2634 rp = &pm->pm_regmap[VA_VREG(va)]; 2635 sp = &rp->rg_segmap[VA_VSEG(va)]; 2636 2637 if (pm->pm_ctx) { 2638 /* 2639 * XXX: always flush cache; conservative, but 2640 * needed to invalidate cache tag protection 2641 * bits and when disabling caching. 2642 */ 2643 cache_flush_page(va, pm->pm_ctxnum); 2644 } 2645 2646 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 2647 KASSERT((tpte & SRMMU_TETYPE) == SRMMU_TEPTE); 2648 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(updatepte4m(va, 2649 &sp->sg_pte[VA_SUN4M_VPG(va)], bic, bis, pm->pm_ctxnum, 2650 PMAP_CPUSET(pm))); 2651 } 2652 } 2653 2654 /* 2655 * Sync ref and mod bits in pvlist. If page has been ref'd or modified, 2656 * update ref/mod bits in pvlist, and clear the hardware bits. 2657 * 2658 * Return the new flags. 2659 */ 2660 int 2661 pv_syncflags4m(struct vm_page *pg) 2662 { 2663 struct pvlist *pv; 2664 struct pmap *pm; 2665 int va, flags; 2666 int s; 2667 struct regmap *rp; 2668 struct segmap *sp; 2669 int tpte; 2670 2671 s = splvm(); 2672 PMAP_LOCK(); 2673 pv = VM_MDPAGE_PVHEAD(pg); 2674 if (pv->pv_pmap == NULL) { 2675 /* Page not mapped; pv_flags is already up to date */ 2676 flags = 0; 2677 goto out; 2678 } 2679 2680 flags = pv->pv_flags; 2681 for (; pv != NULL; pv = pv->pv_next) { 2682 pm = pv->pv_pmap; 2683 va = pv->pv_va; 2684 rp = &pm->pm_regmap[VA_VREG(va)]; 2685 sp = &rp->rg_segmap[VA_VSEG(va)]; 2686 2687 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 2688 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE && 2689 (tpte & (SRMMU_PG_R|SRMMU_PG_M)) != 0) { 2690 /* 2691 * Flush cache if modified to make sure the PTE 2692 * M bit will be set again on the next write access. 2693 */ 2694 if (pm->pm_ctx && (tpte & SRMMU_PG_M) == SRMMU_PG_M) 2695 cache_flush_page(va, pm->pm_ctxnum); 2696 2697 flags |= MR4M(updatepte4m(va, 2698 &sp->sg_pte[VA_SUN4M_VPG(va)], 2699 SRMMU_PG_M | SRMMU_PG_R, 2700 0, pm->pm_ctxnum, PMAP_CPUSET(pm))); 2701 } 2702 } 2703 2704 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags; 2705 out: 2706 PMAP_UNLOCK(); 2707 splx(s); 2708 return (flags); 2709 } 2710 2711 /* 2712 * Should be called with pmap already locked. 2713 */ 2714 void 2715 pv_unlink4m(struct vm_page *pg, struct pmap *pm, vaddr_t va) 2716 { 2717 struct pvlist *pv0, *npv; 2718 2719 pv0 = VM_MDPAGE_PVHEAD(pg); 2720 2721 npv = pv0->pv_next; 2722 /* 2723 * First entry is special (sigh). 2724 */ 2725 if (pv0->pv_pmap == pm && pv0->pv_va == va) { 2726 pmap_stats.ps_unlink_pvfirst++; 2727 if (npv != NULL) { 2728 /* 2729 * Shift next entry into the head. 2730 * Make sure to retain the REF, MOD and ANC flags 2731 * on the list head. 2732 */ 2733 pv0->pv_next = npv->pv_next; 2734 pv0->pv_pmap = npv->pv_pmap; 2735 pv0->pv_va = npv->pv_va; 2736 pv0->pv_flags &= ~PV_NC; 2737 pv0->pv_flags |= (npv->pv_flags & PV_NC); 2738 pool_put(&pv_pool, npv); 2739 } else { 2740 /* 2741 * No mappings left; we need to maintain 2742 * the REF and MOD flags, since pmap_is_modified() 2743 * can still be called for this page. 2744 */ 2745 pv0->pv_pmap = NULL; 2746 pv0->pv_flags &= ~(PV_NC|PV_ANC); 2747 return; 2748 } 2749 } else { 2750 struct pvlist *prev; 2751 2752 pmap_stats.ps_unlink_pvsearch++; 2753 for (prev = pv0;; prev = npv, npv = npv->pv_next) { 2754 if (npv == NULL) { 2755 panic("pv_unlink: pm %p is missing on pg %p", 2756 pm, pg); 2757 return; 2758 } 2759 if (npv->pv_pmap == pm && npv->pv_va == va) 2760 break; 2761 } 2762 prev->pv_next = npv->pv_next; 2763 pool_put(&pv_pool, npv); 2764 } 2765 2766 if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) { 2767 2768 /* 2769 * Not cached: check whether we can fix that now. 2770 */ 2771 va = pv0->pv_va; 2772 for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next) 2773 if (BADALIAS(va, npv->pv_va) || 2774 (npv->pv_flags & PV_NC) != 0) 2775 return; 2776 #ifdef DEBUG 2777 if (pmapdebug & PDB_CACHESTUFF) 2778 printf( 2779 "pv_unlink: alias ok: proc %s, va 0x%lx, pg %p\n", 2780 curproc ? curproc->p_comm : "--", 2781 va, pg); 2782 #endif 2783 pv0->pv_flags &= ~PV_ANC; 2784 pv_changepte4m(pg, SRMMU_PG_C, 0); 2785 } 2786 } 2787 2788 /* 2789 * pv_link is the inverse of pv_unlink, and is used in pmap_enter. 2790 * May turn off the cacheable bit in the pte prototype for the new mapping. 2791 * Called with pm locked. 2792 */ 2793 /*static*/ int 2794 pv_link4m(struct vm_page *pg, struct pmap *pm, vaddr_t va, 2795 unsigned int *pteprotop) 2796 { 2797 struct pvlist *pv0, *pv, *npv; 2798 int nc = (*pteprotop & SRMMU_PG_C) == 0 ? PV_NC : 0; 2799 int error = 0; 2800 2801 pv0 = VM_MDPAGE_PVHEAD(pg); 2802 2803 if (pv0->pv_pmap == NULL) { 2804 /* no pvlist entries yet */ 2805 pmap_stats.ps_enter_firstpv++; 2806 pv0->pv_next = NULL; 2807 pv0->pv_pmap = pm; 2808 pv0->pv_va = va; 2809 pv0->pv_flags |= nc; 2810 goto out; 2811 } 2812 2813 /* 2814 * Allocate the new PV entry now, and, if that fails, bail out 2815 * before changing the cacheable state of the existing mappings. 2816 */ 2817 npv = pool_get(&pv_pool, PR_NOWAIT); 2818 if (npv == NULL) { 2819 error = ENOMEM; 2820 goto out; 2821 } 2822 2823 pmap_stats.ps_enter_secondpv++; 2824 2825 /* 2826 * See if the new mapping will cause old mappings to 2827 * become aliased and thus need to be `discached'. 2828 */ 2829 if ((pv0->pv_flags & PV_ANC) != 0) { 2830 /* already uncached, just stay that way */ 2831 *pteprotop &= ~SRMMU_PG_C; 2832 goto link_npv; 2833 } 2834 2835 for (pv = pv0; pv != NULL; pv = pv->pv_next) { 2836 if ((pv->pv_flags & PV_NC) != 0) { 2837 *pteprotop &= ~SRMMU_PG_C; 2838 #ifdef DEBUG 2839 /* Check currently illegal condition */ 2840 if (nc == 0) 2841 printf("pv_link: proc %s, va=0x%lx: " 2842 "unexpected uncached mapping at 0x%lx\n", 2843 curproc ? curproc->p_comm : "--", 2844 va, pv->pv_va); 2845 #endif 2846 } 2847 if (BADALIAS(va, pv->pv_va)) { 2848 #ifdef DEBUG 2849 if (pmapdebug & PDB_CACHESTUFF) 2850 printf( 2851 "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p\n", 2852 curproc ? curproc->p_comm : "--", 2853 va, pv->pv_va, pg); 2854 #endif 2855 /* Mark list head `uncached due to aliases' */ 2856 pv0->pv_flags |= PV_ANC; 2857 pv_changepte4m(pg, 0, SRMMU_PG_C); 2858 *pteprotop &= ~SRMMU_PG_C; 2859 break; 2860 } 2861 } 2862 2863 link_npv: 2864 /* Now link in the new PV entry */ 2865 npv->pv_next = pv0->pv_next; 2866 npv->pv_pmap = pm; 2867 npv->pv_va = va; 2868 npv->pv_flags = nc; 2869 pv0->pv_next = npv; 2870 2871 out: 2872 return (error); 2873 } 2874 #endif 2875 2876 /* 2877 * Uncache all entries on behalf of kvm_uncache(). In addition to 2878 * removing the cache bit from the PTE, we are also setting PV_NC 2879 * in each entry to stop pv_unlink() from re-caching (i.e. when a 2880 * a bad alias is going away). 2881 */ 2882 static void 2883 pv_uncache(struct vm_page *pg) 2884 { 2885 struct pvlist *pv; 2886 int s; 2887 2888 s = splvm(); 2889 PMAP_LOCK(); 2890 2891 for (pv = VM_MDPAGE_PVHEAD(pg); pv != NULL; pv = pv->pv_next) 2892 pv->pv_flags |= PV_NC; 2893 2894 #if defined(SUN4M) || defined(SUN4D) 2895 if (CPU_HAS_SRMMU) 2896 pv_changepte4m(pg, 0, SRMMU_PG_C); 2897 #endif 2898 #if defined(SUN4) || defined(SUN4C) 2899 if (CPU_HAS_SUNMMU) 2900 pv_changepte4_4c(pg, PG_NC, 0); 2901 #endif 2902 PMAP_UNLOCK(); 2903 splx(s); 2904 } 2905 2906 /* 2907 * Walk the given list and flush the cache for each (MI) page that is 2908 * potentially in the cache. Called only if vactype != VAC_NONE. 2909 */ 2910 #if defined(SUN4) || defined(SUN4C) 2911 static void 2912 pv_flushcache4_4c(struct vm_page *pg) 2913 { 2914 struct pvlist *pv; 2915 struct pmap *pm; 2916 int s, ctx; 2917 2918 pv = VM_MDPAGE_PVHEAD(pg); 2919 2920 write_user_windows(); /* paranoia? */ 2921 s = splvm(); /* XXX extreme paranoia */ 2922 if ((pm = pv->pv_pmap) != NULL) { 2923 ctx = getcontext4(); 2924 for (;;) { 2925 if (pm->pm_ctx) { 2926 setcontext4(pm->pm_ctxnum); 2927 cache_flush_page(pv->pv_va, pm->pm_ctxnum); 2928 } 2929 pv = pv->pv_next; 2930 if (pv == NULL) 2931 break; 2932 pm = pv->pv_pmap; 2933 } 2934 setcontext4(ctx); 2935 } 2936 splx(s); 2937 } 2938 #endif /* SUN4 || SUN4C */ 2939 2940 #if defined(SUN4M) || defined(SUN4D) 2941 static void 2942 pv_flushcache4m(struct vm_page *pg) 2943 { 2944 struct pvlist *pv; 2945 struct pmap *pm; 2946 int s; 2947 2948 pv = VM_MDPAGE_PVHEAD(pg); 2949 2950 s = splvm(); /* XXX extreme paranoia */ 2951 if ((pm = pv->pv_pmap) != NULL) { 2952 for (;;) { 2953 if (pm->pm_ctx) { 2954 cache_flush_page(pv->pv_va, pm->pm_ctxnum); 2955 } 2956 pv = pv->pv_next; 2957 if (pv == NULL) 2958 break; 2959 pm = pv->pv_pmap; 2960 } 2961 } 2962 splx(s); 2963 } 2964 #endif /* SUN4M || SUN4D */ 2965 2966 /*----------------------------------------------------------------*/ 2967 2968 /* 2969 * At last, pmap code. 2970 */ 2971 2972 #if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D)) 2973 int nptesg; 2974 #endif 2975 2976 #if defined(SUN4M) || defined(SUN4D) 2977 static void pmap_bootstrap4m(void *); 2978 #endif 2979 #if defined(SUN4) || defined(SUN4C) 2980 static void pmap_bootstrap4_4c(void *, int, int, int); 2981 #endif 2982 2983 /* 2984 * Bootstrap the system enough to run with VM enabled. 2985 * 2986 * nsegment is the number of mmu segment entries (``PMEGs''); 2987 * nregion is the number of mmu region entries (``SMEGs''); 2988 * nctx is the number of contexts. 2989 */ 2990 void 2991 pmap_bootstrap(int nctx, int nregion, int nsegment) 2992 { 2993 void *p; 2994 extern char etext[], kernel_data_start[]; 2995 extern char *kernel_top; 2996 2997 uvmexp.pagesize = NBPG; 2998 uvm_setpagesize(); 2999 3000 #if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D)) 3001 /* In this case NPTESG is a variable */ 3002 nptesg = (NBPSG >> pgshift); 3003 #endif 3004 3005 /* 3006 * Grab physical memory list. 3007 */ 3008 p = kernel_top; 3009 get_phys_mem(&p); 3010 3011 /* 3012 * The data segment in sparc ELF images is aligned to a 64KB 3013 * (the maximum page size defined by the ELF/sparc ABI) boundary. 3014 * This results in a unused portion of physical memory in between 3015 * the text/rodata and the data segment. We pick up that gap 3016 * here to remove it from the kernel map and give it to the 3017 * VM manager later. 3018 */ 3019 etext_gap_start = (vaddr_t)(etext + NBPG - 1) & ~PGOFSET; 3020 etext_gap_end = (vaddr_t)kernel_data_start & ~PGOFSET; 3021 3022 if (CPU_HAS_SRMMU) { 3023 #if defined(SUN4M) || defined(SUN4D) 3024 pmap_bootstrap4m(p); 3025 #endif 3026 } else if (CPU_HAS_SUNMMU) { 3027 #if defined(SUN4) || defined(SUN4C) 3028 pmap_bootstrap4_4c(p, nctx, nregion, nsegment); 3029 #endif 3030 } 3031 3032 pmap_page_upload(); 3033 mutex_init(&demap_lock, MUTEX_DEFAULT, IPL_VM); 3034 mutex_init(&ctx_lock, MUTEX_DEFAULT, IPL_SCHED); 3035 lock_available = true; 3036 } 3037 3038 #if defined(SUN4) || defined(SUN4C) 3039 void 3040 pmap_bootstrap4_4c(void *top, int nctx, int nregion, int nsegment) 3041 { 3042 union ctxinfo *ci; 3043 struct mmuentry *mmuseg; 3044 #if defined(SUN4_MMU3L) 3045 struct mmuentry *mmureg; 3046 #endif 3047 struct regmap *rp; 3048 struct segmap *sp; 3049 int i, j; 3050 int npte, zseg, vr, vs; 3051 int startscookie, scookie; 3052 #if defined(SUN4_MMU3L) 3053 int startrcookie = 0, rcookie = 0; 3054 #endif 3055 int *kptes; 3056 int lastpage; 3057 vaddr_t va; 3058 vaddr_t p; 3059 extern char kernel_text[]; 3060 3061 /* 3062 * Compute `va2pa_offset'. 3063 * Use `kernel_text' to probe the MMU translation since 3064 * the pages at KERNBASE might not be mapped. 3065 */ 3066 va2pa_offset = (vaddr_t)kernel_text - 3067 ((getpte4(kernel_text) & PG_PFNUM) << PGSHIFT); 3068 3069 ncontext = nctx; 3070 3071 switch (cputyp) { 3072 case CPU_SUN4C: 3073 mmu_has_hole = 1; 3074 break; 3075 case CPU_SUN4: 3076 if (cpuinfo.cpu_type != CPUTYP_4_400) { 3077 mmu_has_hole = 1; 3078 break; 3079 } 3080 } 3081 3082 #if defined(SUN4) 3083 /* 3084 * set up the segfixmask to mask off invalid bits 3085 */ 3086 segfixmask = nsegment - 1; /* assume nsegment is a power of 2 */ 3087 #ifdef DIAGNOSTIC 3088 if (((nsegment & segfixmask) | (nsegment & ~segfixmask)) != nsegment) { 3089 printf("pmap_bootstrap: unsuitable number of segments (%d)\n", 3090 nsegment); 3091 callrom(); 3092 } 3093 #endif 3094 #endif 3095 3096 #if defined(SUN4M) || defined(SUN4D) /* We're in a dual-arch kernel. 3097 Setup 4/4c fn. ptrs */ 3098 pmap_clear_modify_p = pmap_clear_modify4_4c; 3099 pmap_clear_reference_p = pmap_clear_reference4_4c; 3100 pmap_enter_p = pmap_enter4_4c; 3101 pmap_extract_p = pmap_extract4_4c; 3102 pmap_is_modified_p = pmap_is_modified4_4c; 3103 pmap_is_referenced_p = pmap_is_referenced4_4c; 3104 pmap_kenter_pa_p = pmap_kenter_pa4_4c; 3105 pmap_kremove_p = pmap_kremove4_4c; 3106 pmap_kprotect_p = pmap_kprotect4_4c; 3107 pmap_page_protect_p = pmap_page_protect4_4c; 3108 pmap_protect_p = pmap_protect4_4c; 3109 pmap_rmk_p = pmap_rmk4_4c; 3110 pmap_rmu_p = pmap_rmu4_4c; 3111 #endif /* defined SUN4M || defined SUN4D */ 3112 3113 p = (vaddr_t)top; 3114 3115 /* 3116 * Last segment is the `invalid' one (one PMEG of pte's with !pg_v). 3117 * It will never be used for anything else. 3118 */ 3119 seginval = --nsegment; 3120 3121 #if defined(SUN4_MMU3L) 3122 if (HASSUN4_MMU3L) 3123 reginval = --nregion; 3124 #endif 3125 3126 /* 3127 * Allocate and initialise mmu entries and context structures. 3128 */ 3129 #if defined(SUN4_MMU3L) 3130 mmuregions = mmureg = (struct mmuentry *)p; 3131 p += nregion * sizeof(struct mmuentry); 3132 memset(mmuregions, 0, nregion * sizeof(struct mmuentry)); 3133 #endif 3134 mmusegments = mmuseg = (struct mmuentry *)p; 3135 p += nsegment * sizeof(struct mmuentry); 3136 memset(mmusegments, 0, nsegment * sizeof(struct mmuentry)); 3137 3138 pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p; 3139 p += nctx * sizeof *ci; 3140 3141 /* Initialize MMU resource queues */ 3142 #if defined(SUN4_MMU3L) 3143 MMUQ_INIT(®ion_freelist); 3144 MMUQ_INIT(®ion_lru); 3145 MMUQ_INIT(®ion_locked); 3146 #endif 3147 MMUQ_INIT(&segm_freelist); 3148 MMUQ_INIT(&segm_lru); 3149 MMUQ_INIT(&segm_locked); 3150 3151 3152 /* 3153 * Intialize the kernel pmap. 3154 */ 3155 /* kernel_pmap_store.pm_ctxnum = 0; */ 3156 kernel_pmap_store.pm_refcount = 1; 3157 #if defined(SUN4_MMU3L) 3158 TAILQ_INIT(&kernel_pmap_store.pm_reglist); 3159 #endif 3160 TAILQ_INIT(&kernel_pmap_store.pm_seglist); 3161 3162 /* 3163 * Allocate memory for kernel PTEs 3164 * XXX Consider allocating memory for only a few regions 3165 * and use growkernel() to allocate more as needed. 3166 */ 3167 kptes = (int *)p; 3168 p += NKREG * NSEGRG * NPTESG * sizeof(int); 3169 memset(kptes, 0, NKREG * NSEGRG * NPTESG * sizeof(int)); 3170 3171 /* 3172 * Set up pm_regmap for kernel to point NUREG *below* the beginning 3173 * of kernel regmap storage. Since the kernel only uses regions 3174 * above NUREG, we save storage space and can index kernel and 3175 * user regions in the same way. 3176 */ 3177 kernel_pmap_store.pm_regmap = &kernel_regmap_store[-NUREG]; 3178 for (i = NKREG; --i >= 0;) { 3179 #if defined(SUN4_MMU3L) 3180 kernel_regmap_store[i].rg_smeg = reginval; 3181 #endif 3182 kernel_regmap_store[i].rg_segmap = 3183 &kernel_segmap_store[i * NSEGRG]; 3184 for (j = NSEGRG; --j >= 0;) { 3185 sp = &kernel_segmap_store[i * NSEGRG + j]; 3186 sp->sg_pmeg = seginval; 3187 sp->sg_pte = &kptes[(i * NSEGRG + j) * NPTESG]; 3188 } 3189 } 3190 3191 /* 3192 * Preserve the monitor ROM's reserved VM region, so that 3193 * we can use L1-A or the monitor's debugger. As a side 3194 * effect we map the ROM's reserved VM into all contexts 3195 * (otherwise L1-A crashes the machine!). 3196 */ 3197 3198 mmu_reservemon4_4c(&nregion, &nsegment); 3199 3200 #if defined(SUN4_MMU3L) 3201 /* Reserve one region for temporary mappings */ 3202 if (HASSUN4_MMU3L) 3203 tregion = --nregion; 3204 #endif 3205 3206 /* 3207 * Set up the `constants' for the call to vm_init() 3208 * in main(). All pages beginning at p (rounded up to 3209 * the next whole page) and continuing through the number 3210 * of available pages are free, but they start at a higher 3211 * virtual address. This gives us two mappable MD pages 3212 * for pmap_zero_page and pmap_copy_page, and one MI page 3213 * for /dev/mem, all with no associated physical memory. 3214 */ 3215 p = (p + NBPG - 1) & ~PGOFSET; 3216 3217 avail_start = PMAP_BOOTSTRAP_VA2PA(p); 3218 3219 i = p; 3220 cpuinfo.vpage[0] = (void *)p, p += NBPG; 3221 cpuinfo.vpage[1] = (void *)p, p += NBPG; 3222 vmmap = (void *)p, p += NBPG; 3223 p = (vaddr_t)reserve_dumppages((void *)p); 3224 3225 virtual_avail = p; 3226 virtual_end = VM_MAX_KERNEL_ADDRESS; 3227 3228 p = i; /* retract to first free phys */ 3229 3230 3231 /* 3232 * All contexts are free except the kernel's. 3233 * 3234 * XXX sun4c could use context 0 for users? 3235 */ 3236 ci->c_pmap = pmap_kernel(); 3237 ctx_freelist = ci + 1; 3238 for (i = 1; i < ncontext; i++) { 3239 ci++; 3240 ci->c_nextfree = ci + 1; 3241 } 3242 ci->c_nextfree = NULL; 3243 ctx_kick = 0; 3244 ctx_kickdir = -1; 3245 3246 /* 3247 * Init mmu entries that map the kernel physical addresses. 3248 * 3249 * All the other MMU entries are free. 3250 * 3251 * THIS ASSUMES THE KERNEL IS MAPPED BY A CONTIGUOUS RANGE OF 3252 * MMU SEGMENTS/REGIONS DURING THE BOOT PROCESS 3253 */ 3254 3255 /* Compute the number of segments used by the kernel */ 3256 zseg = (((p + NBPSG - 1) & ~SGOFSET) - KERNBASE) >> SGSHIFT; 3257 lastpage = VA_VPG(p); 3258 if (lastpage == 0) 3259 /* 3260 * If the page bits in p are 0, we filled the last segment 3261 * exactly; if not, it is the last page filled in the 3262 * last segment. 3263 */ 3264 lastpage = NPTESG; 3265 3266 p = KERNBASE; /* first va */ 3267 vs = VA_VSEG(KERNBASE); /* first virtual segment */ 3268 vr = VA_VREG(KERNBASE); /* first virtual region */ 3269 rp = &pmap_kernel()->pm_regmap[vr]; 3270 3271 /* Get region/segment where kernel addresses start */ 3272 #if defined(SUN4_MMU3L) 3273 if (HASSUN4_MMU3L) 3274 startrcookie = rcookie = getregmap(p); 3275 mmureg = &mmuregions[rcookie]; 3276 #endif 3277 3278 startscookie = scookie = getsegmap(p); 3279 mmuseg = &mmusegments[scookie]; 3280 zseg += scookie; /* First free segment */ 3281 3282 for (;;) { 3283 3284 /* 3285 * Distribute each kernel region/segment into all contexts. 3286 * This is done through the monitor ROM, rather than 3287 * directly here: if we do a setcontext we will fault, 3288 * as we are not (yet) mapped in any other context. 3289 */ 3290 3291 if ((vs % NSEGRG) == 0) { 3292 /* Entering a new region */ 3293 if (VA_VREG(p) > vr) { 3294 #ifdef DEBUG 3295 printf("note: giant kernel!\n"); 3296 #endif 3297 vr++, rp++; 3298 } 3299 #if defined(SUN4_MMU3L) 3300 if (HASSUN4_MMU3L) { 3301 for (i = 1; i < nctx; i++) 3302 prom_setcontext(i, (void *)p, rcookie); 3303 3304 MMUQ_INSERT_TAIL(®ion_locked, 3305 mmureg, me_list); 3306 TAILQ_INSERT_TAIL(&pmap_kernel()->pm_reglist, 3307 mmureg, me_pmchain); 3308 #ifdef DIAGNOSTIC 3309 mmuseg->me_statp = NULL; 3310 #endif 3311 mmureg->me_cookie = rcookie; 3312 mmureg->me_pmap = pmap_kernel(); 3313 mmureg->me_vreg = vr; 3314 rp->rg_smeg = rcookie; 3315 mmureg++; 3316 rcookie++; 3317 } 3318 #endif /* SUN4_MMU3L */ 3319 } 3320 3321 #if defined(SUN4_MMU3L) 3322 if (!HASSUN4_MMU3L) 3323 #endif 3324 for (i = 1; i < nctx; i++) 3325 prom_setcontext(i, (void *)p, scookie); 3326 3327 /* set up the mmu entry */ 3328 MMUQ_INSERT_TAIL(&segm_locked, mmuseg, me_list); 3329 #ifdef DIAGNOSTIC 3330 mmuseg->me_statp = &pmap_stats.ps_npmeg_locked; 3331 #endif 3332 TAILQ_INSERT_TAIL(&pmap_kernel()->pm_seglist, mmuseg, me_pmchain); 3333 pmap_stats.ps_npmeg_locked++; 3334 mmuseg->me_cookie = scookie; 3335 mmuseg->me_pmap = pmap_kernel(); 3336 mmuseg->me_vreg = vr; 3337 mmuseg->me_vseg = vs % NSEGRG; 3338 sp = &rp->rg_segmap[vs % NSEGRG]; 3339 sp->sg_pmeg = scookie; 3340 npte = ++scookie < zseg ? NPTESG : lastpage; 3341 sp->sg_npte = npte; 3342 sp->sg_nwired = npte; 3343 pmap_kernel()->pm_stats.resident_count += npte; 3344 rp->rg_nsegmap += 1; 3345 for (i = 0; i < npte; i++) 3346 sp->sg_pte[i] = getpte4(p + i * NBPG) | PG_WIRED; 3347 mmuseg++; 3348 vs++; 3349 if (scookie < zseg) { 3350 p += NBPSG; 3351 continue; 3352 } 3353 3354 /* 3355 * Unmap the pages, if any, that are not part of 3356 * the final segment. 3357 */ 3358 for (p += npte << PGSHIFT; npte < NPTESG; npte++, p += NBPG) 3359 setpte4(p, 0); 3360 3361 #if defined(SUN4_MMU3L) 3362 if (HASSUN4_MMU3L) { 3363 /* 3364 * Unmap the segments, if any, that are not part of 3365 * the final region. 3366 */ 3367 for (i = rp->rg_nsegmap; i < NSEGRG; i++, p += NBPSG) 3368 setsegmap(p, seginval); 3369 3370 /* 3371 * Unmap any kernel regions that we aren't using. 3372 */ 3373 for (i = 0; i < nctx; i++) { 3374 setcontext4(i); 3375 for (va = p; 3376 va < (OPENPROM_STARTVADDR & ~(NBPRG - 1)); 3377 va += NBPRG) 3378 setregmap(va, reginval); 3379 } 3380 3381 } else 3382 #endif 3383 { 3384 /* 3385 * Unmap any kernel segments that we aren't using. 3386 */ 3387 for (i = 0; i < nctx; i++) { 3388 setcontext4(i); 3389 for (va = p; 3390 va < (OPENPROM_STARTVADDR & ~(NBPSG - 1)); 3391 va += NBPSG) 3392 setsegmap(va, seginval); 3393 } 3394 } 3395 break; 3396 } 3397 3398 #if defined(SUN4_MMU3L) 3399 if (HASSUN4_MMU3L) 3400 for (rcookie = 0; rcookie < nregion; rcookie++) { 3401 if (rcookie == startrcookie) 3402 /* Kernel must fit in one region! */ 3403 rcookie++; 3404 mmureg = &mmuregions[rcookie]; 3405 mmureg->me_cookie = rcookie; 3406 MMUQ_INSERT_TAIL(®ion_freelist, mmureg, me_list); 3407 #ifdef DIAGNOSTIC 3408 mmuseg->me_statp = NULL; 3409 #endif 3410 } 3411 #endif /* SUN4_MMU3L */ 3412 3413 for (scookie = 0; scookie < nsegment; scookie++) { 3414 if (scookie == startscookie) 3415 /* Skip static kernel image */ 3416 scookie = zseg; 3417 mmuseg = &mmusegments[scookie]; 3418 mmuseg->me_cookie = scookie; 3419 MMUQ_INSERT_TAIL(&segm_freelist, mmuseg, me_list); 3420 pmap_stats.ps_npmeg_free++; 3421 #ifdef DIAGNOSTIC 3422 mmuseg->me_statp = NULL; 3423 #endif 3424 } 3425 3426 /* Erase all spurious user-space segmaps */ 3427 for (i = 1; i < ncontext; i++) { 3428 setcontext4(i); 3429 if (HASSUN4_MMU3L) 3430 for (p = 0, j = NUREG; --j >= 0; p += NBPRG) 3431 setregmap(p, reginval); 3432 else 3433 for (p = 0, vr = 0; vr < NUREG; vr++) { 3434 if (VA_INHOLE(p)) { 3435 p = MMU_HOLE_END; 3436 vr = VA_VREG(p); 3437 } 3438 for (j = NSEGRG; --j >= 0; p += NBPSG) 3439 setsegmap(p, seginval); 3440 } 3441 } 3442 setcontext4(0); 3443 3444 /* 3445 * write protect & encache kernel text; 3446 * set red zone at kernel base; 3447 * enable cache on message buffer and cpuinfo. 3448 */ 3449 { 3450 extern char etext[]; 3451 3452 /* Enable cache on message buffer and cpuinfo */ 3453 for (p = KERNBASE; p < (vaddr_t)trapbase; p += NBPG) 3454 setpte4(p, getpte4(p) & ~PG_NC); 3455 3456 /* Enable cache and write protext kernel text */ 3457 for (p = (vaddr_t)trapbase; p < (vaddr_t)etext; p += NBPG) 3458 setpte4(p, getpte4(p) & ~(PG_NC|PG_W)); 3459 3460 /* 3461 * Unmap the `etext gap'; it'll be made available 3462 * to the VM manager. 3463 */ 3464 for (p = etext_gap_start; p < etext_gap_end; p += NBPG) { 3465 rp = &pmap_kernel()->pm_regmap[VA_VREG(p)]; 3466 sp = &rp->rg_segmap[VA_VSEG(p)]; 3467 sp->sg_nwired--; 3468 sp->sg_npte--; 3469 pmap_kernel()->pm_stats.resident_count--; 3470 sp->sg_pte[VA_VPG(p)] = 0; 3471 setpte4(p, 0); 3472 } 3473 3474 /* Enable cache on data & bss */ 3475 for (p = etext_gap_end; p < virtual_avail; p += NBPG) 3476 setpte4(p, getpte4(p) & ~PG_NC); 3477 3478 } 3479 cpus[0] = (struct cpu_info *)CPUINFO_VA; 3480 } 3481 #endif 3482 3483 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_bootstrap */ 3484 /* 3485 * Bootstrap the system enough to run with VM enabled on a sun4m machine. 3486 * 3487 * Switches from ROM to kernel page tables, and sets up initial mappings. 3488 */ 3489 static void 3490 pmap_bootstrap4m(void *top) 3491 { 3492 int i, j; 3493 vaddr_t p, q; 3494 union ctxinfo *ci; 3495 int reg, seg; 3496 unsigned int ctxtblsize; 3497 vaddr_t pagetables_start, pagetables_end; 3498 paddr_t pagetables_start_pa; 3499 extern char etext[]; 3500 extern char kernel_text[]; 3501 vaddr_t va; 3502 #ifdef MULTIPROCESSOR 3503 vsize_t off; 3504 size_t cpuinfo_len = sizeof(struct cpu_info); 3505 uint8_t *cpuinfo_data; 3506 int align = PAGE_SIZE; 3507 vaddr_t sva, cpuinfo_va; 3508 vsize_t sz; 3509 #endif 3510 3511 /* 3512 * Compute `va2pa_offset'. 3513 * Use `kernel_text' to probe the MMU translation since 3514 * the pages at KERNBASE might not be mapped. 3515 */ 3516 va2pa_offset = (vaddr_t)kernel_text - VA2PA(kernel_text); 3517 3518 ncontext = cpuinfo.mmu_ncontext; 3519 3520 #if defined(SUN4) || defined(SUN4C) /* setup SRMMU fn. ptrs for dual-arch 3521 kernel */ 3522 pmap_clear_modify_p = pmap_clear_modify4m; 3523 pmap_clear_reference_p = pmap_clear_reference4m; 3524 pmap_enter_p = pmap_enter4m; 3525 pmap_extract_p = pmap_extract4m; 3526 pmap_is_modified_p = pmap_is_modified4m; 3527 pmap_is_referenced_p = pmap_is_referenced4m; 3528 pmap_kenter_pa_p = pmap_kenter_pa4m; 3529 pmap_kremove_p = pmap_kremove4m; 3530 pmap_kprotect_p = pmap_kprotect4m; 3531 pmap_page_protect_p = pmap_page_protect4m; 3532 pmap_protect_p = pmap_protect4m; 3533 pmap_rmk_p = pmap_rmk4m; 3534 pmap_rmu_p = pmap_rmu4m; 3535 #endif /* defined SUN4/SUN4C */ 3536 3537 /* 3538 * p points to top of kernel mem 3539 */ 3540 p = (vaddr_t)top; 3541 3542 p = (p + NBPG - 1) & ~PGOFSET; 3543 3544 /* 3545 * Intialize the kernel pmap. 3546 */ 3547 /* kernel_pmap_store.pm_ctxnum = 0; */ 3548 kernel_pmap_store.pm_refcount = 1; 3549 3550 /* 3551 * Set up pm_regmap for kernel to point NUREG *below* the beginning 3552 * of kernel regmap storage. Since the kernel only uses regions 3553 * above NUREG, we save storage space and can index kernel and 3554 * user regions in the same way. 3555 */ 3556 kernel_pmap_store.pm_regmap = &kernel_regmap_store[-NUREG]; 3557 memset(kernel_regmap_store, 0, NKREG * sizeof(struct regmap)); 3558 memset(kernel_segmap_store, 0, NKREG * NSEGRG * sizeof(struct segmap)); 3559 for (i = NKREG; --i >= 0;) { 3560 kernel_regmap_store[i].rg_segmap = 3561 &kernel_segmap_store[i * NSEGRG]; 3562 kernel_regmap_store[i].rg_seg_ptps = NULL; 3563 for (j = NSEGRG; --j >= 0;) 3564 kernel_segmap_store[i * NSEGRG + j].sg_pte = NULL; 3565 } 3566 3567 /* Allocate kernel region pointer tables */ 3568 pmap_kernel()->pm_reg_ptps = (int **)(q = p); 3569 p += sparc_ncpus * sizeof(int **); 3570 memset((void *)q, 0, (u_int)p - (u_int)q); 3571 3572 pmap_kernel()->pm_reg_ptps_pa = (int *)(q = p); 3573 p += sparc_ncpus * sizeof(int *); 3574 memset((void *)q, 0, (u_int)p - (u_int)q); 3575 3576 /* Allocate context administration */ 3577 pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p; 3578 p += ncontext * sizeof *ci; 3579 memset((void *)ci, 0, (u_int)p - (u_int)ci); 3580 3581 /* 3582 * Set up the `constants' for the call to vm_init() 3583 * in main(). All pages beginning at p (rounded up to 3584 * the next whole page) and continuing through the number 3585 * of available pages are free. 3586 */ 3587 p = (p + NBPG - 1) & ~PGOFSET; 3588 3589 /* 3590 * Reserve memory for MMU pagetables. Some of these have severe 3591 * alignment restrictions. We allocate in a sequence that 3592 * minimizes alignment gaps. 3593 */ 3594 3595 pagetables_start = p; 3596 pagetables_start_pa = PMAP_BOOTSTRAP_VA2PA(p); 3597 3598 /* 3599 * Allocate context table. 3600 * To keep supersparc happy, minimum aligment is on a 4K boundary. 3601 */ 3602 ctxtblsize = max(ncontext,1024) * sizeof(int); 3603 cpuinfo.ctx_tbl = (int *)roundup((u_int)p, ctxtblsize); 3604 cpuinfo.ctx_tbl_pa = PMAP_BOOTSTRAP_VA2PA(cpuinfo.ctx_tbl); 3605 p = (u_int)cpuinfo.ctx_tbl + ctxtblsize; 3606 3607 #if defined(MULTIPROCESSOR) 3608 /* 3609 * Make sure all smp_tlb_flush*() routines for kernel pmap are 3610 * broadcast to all CPU's. 3611 */ 3612 pmap_kernel()->pm_cpuset = CPUSET_ALL; 3613 #endif 3614 3615 /* 3616 * Reserve memory for segment and page tables needed to map the entire 3617 * kernel. This takes (2K + NKREG * 16K) of space, but unfortunately 3618 * is necessary since pmap_enter() *must* be able to enter a kernel 3619 * mapping without delay. 3620 */ 3621 p = (vaddr_t) roundup(p, SRMMU_L1SIZE * sizeof(u_int)); 3622 qzero((void *)p, SRMMU_L1SIZE * sizeof(u_int)); 3623 kernel_regtable_store = (u_int *)p; 3624 p += SRMMU_L1SIZE * sizeof(u_int); 3625 3626 p = (vaddr_t) roundup(p, SRMMU_L2SIZE * sizeof(u_int)); 3627 qzero((void *)p, (SRMMU_L2SIZE * sizeof(u_int)) * NKREG); 3628 kernel_segtable_store = (u_int *)p; 3629 p += (SRMMU_L2SIZE * sizeof(u_int)) * NKREG; 3630 3631 p = (vaddr_t) roundup(p, SRMMU_L3SIZE * sizeof(u_int)); 3632 /* zero it: all will be SRMMU_TEINVALID */ 3633 qzero((void *)p, ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG); 3634 kernel_pagtable_store = (u_int *)p; 3635 p += ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG; 3636 3637 /* Round to next page and mark end of pre-wired kernel space */ 3638 p = (p + NBPG - 1) & ~PGOFSET; 3639 pagetables_end = p; 3640 3641 #if defined(MULTIPROCESSOR) 3642 /* 3643 * Allocate aligned KVA. `cpuinfo' resides at a fixed virtual 3644 * address. Since we need to access an other CPU's cpuinfo 3645 * structure occasionally, this must be done at a virtual address 3646 * that's cache congruent to the fixed address CPUINFO_VA. 3647 * 3648 * NOTE: we're using the cache properties of the boot CPU to 3649 * determine the alignment (XXX). 3650 */ 3651 sz = sizeof(struct cpu_info); 3652 if (sparc_ncpus > 1) { 3653 if (CACHEINFO.c_totalsize > align) { 3654 /* Need a power of two */ 3655 while (align <= CACHEINFO.c_totalsize) 3656 align <<= 1; 3657 align >>= 1; 3658 } 3659 3660 sz = (sz + PAGE_SIZE - 1) & -PAGE_SIZE; 3661 cpuinfo_len = sz + align - PAGE_SIZE; 3662 3663 /* Grab as much space as we need */ 3664 cpuinfo_data = (uint8_t *)p; 3665 p += (cpuinfo_len * sparc_ncpus); 3666 } else 3667 cpuinfo_data = (uint8_t *)CPUINFO_VA; 3668 #endif 3669 3670 avail_start = PMAP_BOOTSTRAP_VA2PA(p); 3671 3672 /* 3673 * Now wire the region and segment tables of the kernel map. 3674 */ 3675 pmap_kernel()->pm_reg_ptps[0] = (int *) kernel_regtable_store; 3676 pmap_kernel()->pm_reg_ptps_pa[0] = 3677 PMAP_BOOTSTRAP_VA2PA(kernel_regtable_store); 3678 3679 /* Install L1 table in context 0 */ 3680 setpgt4m(&cpuinfo.ctx_tbl[0], 3681 (pmap_kernel()->pm_reg_ptps_pa[0] >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 3682 3683 for (reg = 0; reg < NKREG; reg++) { 3684 struct regmap *rp; 3685 void *kphyssegtbl; 3686 3687 /* 3688 * Entering new region; install & build segtbl 3689 */ 3690 3691 rp = &pmap_kernel()->pm_regmap[reg + VA_VREG(KERNBASE)]; 3692 3693 kphyssegtbl = (void *) 3694 &kernel_segtable_store[reg * SRMMU_L2SIZE]; 3695 3696 setpgt4m(&pmap_kernel()->pm_reg_ptps[0][reg + VA_VREG(KERNBASE)], 3697 (PMAP_BOOTSTRAP_VA2PA(kphyssegtbl) >> SRMMU_PPNPASHIFT) | 3698 SRMMU_TEPTD); 3699 3700 rp->rg_seg_ptps = (int *)kphyssegtbl; 3701 3702 for (seg = 0; seg < NSEGRG; seg++) { 3703 struct segmap *sp; 3704 void *kphyspagtbl; 3705 3706 rp->rg_nsegmap++; 3707 3708 sp = &rp->rg_segmap[seg]; 3709 kphyspagtbl = (void *) 3710 &kernel_pagtable_store 3711 [((reg * NSEGRG) + seg) * SRMMU_L3SIZE]; 3712 3713 setpgt4m(&rp->rg_seg_ptps[seg], 3714 (PMAP_BOOTSTRAP_VA2PA(kphyspagtbl) >> SRMMU_PPNPASHIFT) | 3715 SRMMU_TEPTD); 3716 sp->sg_pte = (int *) kphyspagtbl; 3717 } 3718 } 3719 3720 /* 3721 * Preserve the monitor ROM's reserved VM region, so that 3722 * we can use L1-A or the monitor's debugger. 3723 */ 3724 mmu_reservemon4m(&kernel_pmap_store); 3725 3726 /* 3727 * Reserve virtual address space for two mappable MD pages 3728 * for pmap_zero_page and pmap_copy_page, one MI page 3729 * for /dev/mem, and some more for dumpsys(). 3730 */ 3731 q = p; 3732 cpuinfo.vpage[0] = (void *)p, p += NBPG; 3733 cpuinfo.vpage[1] = (void *)p, p += NBPG; 3734 vmmap = (void *)p, p += NBPG; 3735 p = (vaddr_t)reserve_dumppages((void *)p); 3736 3737 /* Find PTE locations of vpage[] to optimize zero_fill() et.al. */ 3738 for (i = 0; i < 2; i++) { 3739 struct regmap *rp; 3740 struct segmap *sp; 3741 rp = &pmap_kernel()->pm_regmap[VA_VREG(cpuinfo.vpage[i])]; 3742 sp = &rp->rg_segmap[VA_VSEG(cpuinfo.vpage[i])]; 3743 cpuinfo.vpage_pte[i] = 3744 &sp->sg_pte[VA_SUN4M_VPG(cpuinfo.vpage[i])]; 3745 } 3746 3747 #if !(defined(PROM_AT_F0) || defined(MSIIEP)) 3748 virtual_avail = p; 3749 #elif defined(MSIIEP) 3750 virtual_avail = (vaddr_t)0xf0800000; /* Krups */ 3751 #else 3752 virtual_avail = (vaddr_t)0xf0080000; /* Mr.Coffee/OFW */ 3753 #endif 3754 virtual_end = VM_MAX_KERNEL_ADDRESS; 3755 3756 p = q; /* retract to first free phys */ 3757 3758 /* 3759 * Set up the ctxinfo structures (freelist of contexts) 3760 */ 3761 ci->c_pmap = pmap_kernel(); 3762 ctx_freelist = ci + 1; 3763 for (i = 1; i < ncontext; i++) { 3764 ci++; 3765 ci->c_nextfree = ci + 1; 3766 } 3767 ci->c_nextfree = NULL; 3768 ctx_kick = 0; 3769 ctx_kickdir = -1; 3770 3771 /* 3772 * Now map the kernel into our new set of page tables, then 3773 * (finally) switch over to our running page tables. 3774 * We map from KERNBASE to p into context 0's page tables (and 3775 * the kernel pmap). 3776 */ 3777 #ifdef DEBUG /* Sanity checks */ 3778 if (p % NBPG != 0) 3779 panic("pmap_bootstrap4m: p misaligned?!?"); 3780 if (KERNBASE % NBPRG != 0) 3781 panic("pmap_bootstrap4m: KERNBASE not region-aligned"); 3782 #endif 3783 3784 for (q = KERNBASE; q < p; q += NBPG) { 3785 struct regmap *rp; 3786 struct segmap *sp; 3787 int pte, *ptep; 3788 3789 /* 3790 * Now install entry for current page. 3791 */ 3792 rp = &pmap_kernel()->pm_regmap[VA_VREG(q)]; 3793 sp = &rp->rg_segmap[VA_VSEG(q)]; 3794 ptep = &sp->sg_pte[VA_VPG(q)]; 3795 3796 /* 3797 * Unmap the `etext gap'; it'll be made available 3798 * to the VM manager. 3799 */ 3800 if (q >= etext_gap_start && q < etext_gap_end) { 3801 setpgt4m(ptep, 0); 3802 continue; 3803 } 3804 3805 pte = PMAP_BOOTSTRAP_VA2PA(q) >> SRMMU_PPNPASHIFT; 3806 pte |= PPROT_N_RX | SRMMU_TEPTE; 3807 3808 /* Deal with the cacheable bit for pagetable memory */ 3809 if ((cpuinfo.flags & CPUFLG_CACHEPAGETABLES) != 0 || 3810 q < pagetables_start || q >= pagetables_end) 3811 pte |= SRMMU_PG_C; 3812 3813 /* write-protect kernel text */ 3814 if (q < (vaddr_t)trapbase || q >= (vaddr_t)etext) 3815 pte |= PPROT_WRITE; 3816 3817 setpgt4m(ptep, pte); 3818 pmap_kernel()->pm_stats.resident_count++; 3819 } 3820 3821 if ((cpuinfo.flags & CPUFLG_CACHEPAGETABLES) == 0) { 3822 /* 3823 * The page tables have been setup. Since we're still 3824 * running on the PROM's memory map, the memory we 3825 * allocated for our page tables might still be cached. 3826 * Flush it now, and don't touch it again until we 3827 * switch to our own tables (will be done immediately below). 3828 */ 3829 int size = pagetables_end - pagetables_start; 3830 if (CACHEINFO.c_vactype != VAC_NONE) { 3831 va = (vaddr_t)pagetables_start; 3832 while (size > 0) { 3833 cache_flush_page(va, 0); 3834 va += NBPG; 3835 size -= NBPG; 3836 } 3837 } else if (cpuinfo.pcache_flush_page != NULL) { 3838 paddr_t pa = pagetables_start_pa; 3839 while (size > 0) { 3840 pcache_flush_page(pa, 0); 3841 pa += NBPG; 3842 size -= NBPG; 3843 } 3844 } 3845 } 3846 3847 /* 3848 * Now switch to kernel pagetables (finally!) 3849 */ 3850 mmu_install_tables(&cpuinfo); 3851 3852 #if defined(MULTIPROCESSOR) 3853 /* 3854 * Initialise any cpu-specific data now. 3855 */ 3856 cpu_init_system(); 3857 3858 /* 3859 * Setup the cpus[] array and the ci_self links. 3860 */ 3861 prom_printf("setting cpus self reference\n"); 3862 for (i = 0; i < sparc_ncpus; i++) { 3863 sva = (vaddr_t) (cpuinfo_data + (cpuinfo_len * i)); 3864 cpuinfo_va = sva + 3865 (((CPUINFO_VA & (align - 1)) + align - sva) & (align - 1)); 3866 3867 /* 3868 * Either remap from CPUINFO_VA to the new correct value 3869 * or clear out this cpuinfo. 3870 */ 3871 if (i == 0) { 3872 for (off = 0, va = cpuinfo_va; 3873 sparc_ncpus > 1 && off < sizeof(struct cpu_info); 3874 va += NBPG, off += NBPG) { 3875 paddr_t pa = 3876 PMAP_BOOTSTRAP_VA2PA(CPUINFO_VA + off); 3877 3878 prom_printf("going to pmap_kenter_pa" 3879 "(va=%p, pa=%p)\n", va, pa); 3880 pmap_kremove(va, NBPG); 3881 pmap_kenter_pa(va, pa, 3882 VM_PROT_READ | VM_PROT_WRITE, 0); 3883 } 3884 3885 } else 3886 memset((void *)cpuinfo_va, 0, sizeof(struct cpu_info)); 3887 3888 cpus[i] = (struct cpu_info *)cpuinfo_va; 3889 cpus[i]->ci_self = cpus[i]; 3890 prom_printf("set cpu%d ci_self address: %p\n", i, cpus[i]); 3891 3892 /* Unmap and prepare to return unused pages */ 3893 if (cpuinfo_va != sva) { 3894 cpus[i]->ci_free_sva1 = sva; 3895 cpus[i]->ci_free_eva1 = cpuinfo_va; 3896 for (va = cpus[i]->ci_free_sva1; 3897 va < cpus[i]->ci_free_eva1; 3898 va += NBPG) 3899 setpte4m(va, 0); 3900 } 3901 if (cpuinfo_va + sz != sva + cpuinfo_len) { 3902 cpus[i]->ci_free_sva2 = cpuinfo_va + sz; 3903 cpus[i]->ci_free_eva2 = sva + cpuinfo_len; 3904 for (va = cpus[i]->ci_free_sva2; 3905 va < cpus[i]->ci_free_eva2; 3906 va += NBPG) 3907 setpte4m(va, 0); 3908 } 3909 } 3910 #else 3911 cpus[0] = (struct cpu_info *)CPUINFO_VA; 3912 #endif 3913 3914 pmap_update(pmap_kernel()); 3915 3916 #ifdef DIAGNOSTIC 3917 if (curcpu()->ci_self != cpus[0]) { 3918 prom_printf("curcpu()->ci_self %p != cpus[0] %p\n", curcpu()->ci_self, cpus[0]); 3919 panic("cpuinfo inconsistent"); 3920 } 3921 #endif 3922 3923 prom_printf("pmap_bootstrap4m done\n"); 3924 } 3925 3926 static u_long prom_ctxreg; 3927 3928 void 3929 mmu_install_tables(struct cpu_info *sc) 3930 { 3931 3932 #ifdef DEBUG 3933 prom_printf("pmap_bootstrap: installing kernel page tables..."); 3934 #endif 3935 setcontext4m(0); /* paranoia? %%%: Make 0x3 a define! below */ 3936 3937 /* Enable MMU tablewalk caching, flush TLB */ 3938 if (sc->mmu_enable != 0) 3939 sc->mmu_enable(); 3940 3941 tlb_flush_all_real(); 3942 prom_ctxreg = lda(SRMMU_CXTPTR, ASI_SRMMU); 3943 3944 sta(SRMMU_CXTPTR, ASI_SRMMU, 3945 (sc->ctx_tbl_pa >> SRMMU_PPNPASHIFT) & ~0x3); 3946 3947 tlb_flush_all_real(); 3948 3949 #ifdef DEBUG 3950 prom_printf("done.\n"); 3951 #endif 3952 } 3953 3954 void srmmu_restore_prom_ctx(void); 3955 3956 void 3957 srmmu_restore_prom_ctx(void) 3958 { 3959 3960 tlb_flush_all(); 3961 sta(SRMMU_CXTPTR, ASI_SRMMU, prom_ctxreg); 3962 tlb_flush_all(); 3963 } 3964 #endif /* SUN4M || SUN4D */ 3965 3966 #if defined(MULTIPROCESSOR) 3967 /* 3968 * Allocate per-CPU page tables. One region, segment and page table 3969 * is needed to map CPUINFO_VA to different physical addresses on 3970 * each CPU. Since the kernel region and segment tables are all 3971 * pre-wired (in bootstrap() above) and we also assume that the 3972 * first segment (256K) of kernel space is fully populated with 3973 * pages from the start, these per-CPU tables will never need 3974 * to be updated when mapping kernel virtual memory. 3975 * 3976 * Note: this routine is called in the context of the boot CPU 3977 * during autoconfig. 3978 */ 3979 void 3980 pmap_alloc_cpu(struct cpu_info *sc) 3981 { 3982 #if defined(SUN4M) || defined(SUN4D) /* Only implemented for SUN4M/D */ 3983 vaddr_t va; 3984 paddr_t pa; 3985 paddr_t alignment; 3986 u_int *ctxtable, *regtable, *segtable, *pagtable; 3987 u_int *ctxtable_pa, *regtable_pa, *segtable_pa, *pagtable_pa; 3988 psize_t ctxsize, size; 3989 int vr, vs, vpg; 3990 struct regmap *rp; 3991 struct segmap *sp; 3992 struct pglist mlist; 3993 int cachebit; 3994 int pagesz = NBPG; 3995 int i; 3996 3997 cachebit = (cpuinfo.flags & CPUFLG_CACHEPAGETABLES) != 0; 3998 3999 /* 4000 * Allocate properly aligned and contiguous physically memory 4001 * for the PTE tables. 4002 */ 4003 ctxsize = (sc->mmu_ncontext * sizeof(int) + pagesz - 1) & -pagesz; 4004 alignment = ctxsize; 4005 4006 /* The region, segment and page table we need fit in one page */ 4007 size = ctxsize + pagesz; 4008 4009 if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys, 4010 alignment, 0, &mlist, 1, 0) != 0) 4011 panic("pmap_alloc_cpu: no memory"); 4012 4013 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&mlist)); 4014 4015 /* Allocate virtual memory */ 4016 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 4017 if (va == 0) 4018 panic("pmap_alloc_cpu: no memory"); 4019 4020 /* 4021 * Layout the page tables in our chunk of memory 4022 */ 4023 ctxtable = (u_int *)va; 4024 regtable = (u_int *)(va + ctxsize); 4025 segtable = regtable + SRMMU_L1SIZE; 4026 pagtable = segtable + SRMMU_L2SIZE; 4027 4028 ctxtable_pa = (u_int *)pa; 4029 regtable_pa = (u_int *)(pa + ctxsize); 4030 segtable_pa = regtable_pa + SRMMU_L1SIZE; 4031 pagtable_pa = segtable_pa + SRMMU_L2SIZE; 4032 4033 /* Map the pages */ 4034 while (size != 0) { 4035 pmap_kenter_pa(va, pa | (cachebit ? 0 : PMAP_NC), 4036 VM_PROT_READ | VM_PROT_WRITE, 0); 4037 va += pagesz; 4038 pa += pagesz; 4039 size -= pagesz; 4040 } 4041 pmap_update(pmap_kernel()); 4042 4043 /* 4044 * Store the region table pointer (and its corresponding physical 4045 * address) in the CPU's slot in the kernel pmap region table 4046 * pointer table. 4047 */ 4048 pmap_kernel()->pm_reg_ptps[sc->ci_cpuid] = regtable; 4049 pmap_kernel()->pm_reg_ptps_pa[sc->ci_cpuid] = (paddr_t)regtable_pa; 4050 4051 vr = VA_VREG(CPUINFO_VA); 4052 vs = VA_VSEG(CPUINFO_VA); 4053 vpg = VA_VPG(CPUINFO_VA); 4054 rp = &pmap_kernel()->pm_regmap[vr]; 4055 sp = &rp->rg_segmap[vs]; 4056 4057 /* 4058 * Copy page tables from CPU #0, then modify entry for CPUINFO_VA 4059 * so that it points at the per-CPU pages. 4060 */ 4061 qcopy(pmap_kernel()->pm_reg_ptps[0], regtable, 4062 SRMMU_L1SIZE * sizeof(int)); 4063 qcopy(rp->rg_seg_ptps, segtable, SRMMU_L2SIZE * sizeof(int)); 4064 qcopy(sp->sg_pte, pagtable, SRMMU_L3SIZE * sizeof(int)); 4065 4066 setpgt4m(&ctxtable[0], 4067 ((u_long)regtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 4068 setpgt4m(®table[vr], 4069 ((u_long)segtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 4070 setpgt4m(&segtable[vs], 4071 ((u_long)pagtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 4072 setpgt4m(&pagtable[vpg], 4073 (VA2PA((void *)sc) >> SRMMU_PPNPASHIFT) | 4074 (SRMMU_TEPTE | PPROT_N_RWX | SRMMU_PG_C)); 4075 4076 /* Install this CPU's context table */ 4077 sc->ctx_tbl = ctxtable; 4078 sc->ctx_tbl_pa = (paddr_t)ctxtable_pa; 4079 4080 /* Pre-compute this CPU's vpage[] PTEs */ 4081 for (i = 0; i < 2; i++) { 4082 rp = &pmap_kernel()->pm_regmap[VA_VREG(sc->vpage[i])]; 4083 sp = &rp->rg_segmap[VA_VSEG(sc->vpage[i])]; 4084 sc->vpage_pte[i] = &sp->sg_pte[VA_SUN4M_VPG(sc->vpage[i])]; 4085 } 4086 #endif /* SUN4M || SUN4D */ 4087 } 4088 #endif /* MULTIPROCESSOR */ 4089 4090 4091 void 4092 pmap_init(void) 4093 { 4094 u_int sz; 4095 4096 if (PAGE_SIZE != NBPG) 4097 panic("pmap_init: PAGE_SIZE!=NBPG"); 4098 4099 vm_num_phys = vm_last_phys - vm_first_phys; 4100 4101 /* Setup a pool for additional pvlist structures */ 4102 pool_init(&pv_pool, sizeof(struct pvlist), 0, 0, 0, "pvtable", NULL, 4103 IPL_NONE); 4104 4105 /* 4106 * Setup a pool for pmap structures. 4107 * The pool size includes space for an array of per-CPU 4108 * region table pointers & physical addresses 4109 */ 4110 sz = ALIGN(sizeof(struct pmap)) + 4111 ALIGN(NUREG * sizeof(struct regmap)) + 4112 sparc_ncpus * sizeof(int *) + /* pm_reg_ptps */ 4113 sparc_ncpus * sizeof(int); /* pm_reg_ptps_pa */ 4114 pool_cache_bootstrap(&pmap_cache, sz, 0, 0, 0, "pmappl", NULL, 4115 IPL_NONE, pmap_pmap_pool_ctor, pmap_pmap_pool_dtor, NULL); 4116 4117 sz = NSEGRG * sizeof (struct segmap); 4118 pool_init(&segmap_pool, sz, 0, 0, 0, "segmap", NULL, IPL_NONE); 4119 4120 #if defined(SUN4M) || defined(SUN4D) 4121 if (CPU_HAS_SRMMU) { 4122 /* 4123 * The SRMMU only ever needs chunks in one of two sizes: 4124 * 1024 (for region level tables) and 256 (for segment 4125 * and page level tables). 4126 */ 4127 sz = SRMMU_L1SIZE * sizeof(int); 4128 pool_init(&L1_pool, sz, sz, 0, 0, "L1 pagetable", 4129 &pgt_page_allocator, IPL_NONE); 4130 4131 sz = SRMMU_L2SIZE * sizeof(int); 4132 pool_init(&L23_pool, sz, sz, 0, 0, "L2/L3 pagetable", 4133 &pgt_page_allocator, IPL_NONE); 4134 } 4135 #endif /* SUN4M || SUN4D */ 4136 #if defined(SUN4) || defined(SUN4C) 4137 if (CPU_HAS_SUNMMU) { 4138 sz = NPTESG * sizeof(int); 4139 pool_init(&pte_pool, sz, 0, 0, 0, "ptemap", NULL, 4140 IPL_NONE); 4141 } 4142 #endif /* SUN4 || SUN4C */ 4143 } 4144 4145 4146 /* 4147 * Map physical addresses into kernel VM. 4148 */ 4149 vaddr_t 4150 pmap_map(vaddr_t va, paddr_t pa, paddr_t endpa, int prot) 4151 { 4152 int pgsize = PAGE_SIZE; 4153 4154 while (pa < endpa) { 4155 pmap_kenter_pa(va, pa, prot, 0); 4156 va += pgsize; 4157 pa += pgsize; 4158 } 4159 pmap_update(pmap_kernel()); 4160 return (va); 4161 } 4162 4163 #ifdef DEBUG 4164 /* 4165 * Check a pmap for spuriously lingering mappings 4166 */ 4167 static inline void 4168 pmap_quiet_check(struct pmap *pm) 4169 { 4170 int vs, vr; 4171 4172 if (CPU_HAS_SUNMMU) { 4173 #if defined(SUN4_MMU3L) 4174 if (TAILQ_FIRST(&pm->pm_reglist)) 4175 panic("pmap_destroy: region list not empty"); 4176 #endif 4177 if (TAILQ_FIRST(&pm->pm_seglist)) 4178 panic("pmap_destroy: segment list not empty"); 4179 } 4180 4181 for (vr = 0; vr < NUREG; vr++) { 4182 struct regmap *rp = &pm->pm_regmap[vr]; 4183 4184 if (HASSUN4_MMU3L) { 4185 if (rp->rg_smeg != reginval) 4186 printf("pmap_chk: spurious smeg in " 4187 "user region %d\n", vr); 4188 } 4189 if (CPU_HAS_SRMMU) { 4190 int n; 4191 #if defined(MULTIPROCESSOR) 4192 for (n = 0; n < sparc_ncpus; n++) 4193 #else 4194 n = 0; 4195 #endif 4196 { 4197 /* Did this cpu attach? */ 4198 if (pmap_kernel()->pm_reg_ptps[n] == 0) 4199 continue; 4200 4201 if (pm->pm_reg_ptps[n][vr] != SRMMU_TEINVALID) 4202 printf("pmap_chk: spurious PTP in user " 4203 "region %d on CPU %d\n", vr, n); 4204 } 4205 } 4206 if (rp->rg_nsegmap != 0) 4207 printf("pmap_chk: %d segments remain in " 4208 "region %d\n", rp->rg_nsegmap, vr); 4209 if (rp->rg_segmap != NULL) { 4210 printf("pmap_chk: segments still " 4211 "allocated in region %d\n", vr); 4212 for (vs = 0; vs < NSEGRG; vs++) { 4213 struct segmap *sp = &rp->rg_segmap[vs]; 4214 if (sp->sg_npte != 0) 4215 printf("pmap_chk: %d ptes " 4216 "remain in segment %d\n", 4217 sp->sg_npte, vs); 4218 if (sp->sg_pte != NULL) { 4219 printf("pmap_chk: ptes still " 4220 "allocated in segment %d\n", vs); 4221 } 4222 if (CPU_HAS_SUNMMU) { 4223 if (sp->sg_pmeg != seginval) 4224 printf("pmap_chk: pm %p(%d,%d) " 4225 "spurious soft pmeg %d\n", 4226 pm, vr, vs, sp->sg_pmeg); 4227 } 4228 } 4229 } 4230 4231 /* Check for spurious pmeg entries in the MMU */ 4232 if (pm->pm_ctx == NULL) 4233 continue; 4234 if (CPU_HAS_SUNMMU) { 4235 int ctx; 4236 if (mmu_has_hole && (vr >= 32 || vr < (256 - 32))) 4237 continue; 4238 ctx = getcontext4(); 4239 setcontext4(pm->pm_ctxnum); 4240 for (vs = 0; vs < NSEGRG; vs++) { 4241 vaddr_t va = VSTOVA(vr,vs); 4242 int pmeg = getsegmap(va); 4243 if (pmeg != seginval) 4244 printf("pmap_chk: pm %p(%d,%d:%x): " 4245 "spurious pmeg %d\n", 4246 pm, vr, vs, (u_int)va, pmeg); 4247 } 4248 setcontext4(ctx); 4249 } 4250 } 4251 if (pm->pm_stats.resident_count) { 4252 printf("pmap_chk: res count %ld\n", 4253 pm->pm_stats.resident_count); 4254 } 4255 if (pm->pm_stats.wired_count) { 4256 printf("pmap_chk: wired count %ld\n", 4257 pm->pm_stats.wired_count); 4258 } 4259 } 4260 #endif /* DEBUG */ 4261 4262 int 4263 pmap_pmap_pool_ctor(void *arg, void *object, int flags) 4264 { 4265 struct pmap *pm = object; 4266 u_long addr; 4267 4268 memset(pm, 0, sizeof *pm); 4269 4270 /* 4271 * `pmap_pool' entries include space for the per-CPU 4272 * region table pointer arrays. 4273 */ 4274 addr = (u_long)pm + ALIGN(sizeof(struct pmap)); 4275 pm->pm_regmap = (void *)addr; 4276 addr += ALIGN(NUREG * sizeof(struct regmap)); 4277 pm->pm_reg_ptps = (int **)addr; 4278 addr += sparc_ncpus * sizeof(int *); 4279 pm->pm_reg_ptps_pa = (int *)addr; 4280 4281 qzero((void *)pm->pm_regmap, NUREG * sizeof(struct regmap)); 4282 4283 /* pm->pm_ctx = NULL; // already done */ 4284 4285 if (CPU_HAS_SUNMMU) { 4286 TAILQ_INIT(&pm->pm_seglist); 4287 #if defined(SUN4_MMU3L) 4288 TAILQ_INIT(&pm->pm_reglist); 4289 if (HASSUN4_MMU3L) { 4290 int i; 4291 for (i = NUREG; --i >= 0;) 4292 pm->pm_regmap[i].rg_smeg = reginval; 4293 } 4294 #endif 4295 } 4296 #if defined(SUN4M) || defined(SUN4D) 4297 else { 4298 int i, n; 4299 4300 /* 4301 * We must allocate and initialize hardware-readable (MMU) 4302 * pagetables. We must also map the kernel regions into this 4303 * pmap's pagetables, so that we can access the kernel from 4304 * this user context. 4305 */ 4306 #if defined(MULTIPROCESSOR) 4307 for (n = 0; n < sparc_ncpus; n++) 4308 #else 4309 n = 0; 4310 #endif 4311 { 4312 int *upt, *kpt; 4313 4314 #if defined(MULTIPROCESSOR) 4315 /* Did this cpu attach? */ 4316 if (pmap_kernel()->pm_reg_ptps[n] == 0) 4317 continue; 4318 #endif 4319 4320 upt = pool_get(&L1_pool, flags); 4321 pm->pm_reg_ptps[n] = upt; 4322 pm->pm_reg_ptps_pa[n] = VA2PA((char *)upt); 4323 4324 /* Invalidate user space regions */ 4325 for (i = 0; i < NUREG; i++) 4326 setpgt4m(upt++, SRMMU_TEINVALID); 4327 4328 /* Copy kernel regions */ 4329 kpt = &pmap_kernel()->pm_reg_ptps[n][VA_VREG(KERNBASE)]; 4330 for (i = 0; i < NKREG; i++) 4331 setpgt4m(upt++, kpt[i]); 4332 } 4333 } 4334 #endif /* SUN4M || SUN4D */ 4335 4336 /* XXX - a peculiar place to do this, but we can't do it in pmap_init 4337 * and here at least it's off the beaten code track. 4338 */ 4339 {static int x; if (x == 0) pool_setlowat(&pv_pool, 512), x = 1; } 4340 4341 return (0); 4342 } 4343 4344 void 4345 pmap_pmap_pool_dtor(void *arg, void *object) 4346 { 4347 struct pmap *pm = object; 4348 union ctxinfo *c; 4349 int s = splvm(); /* paranoia */ 4350 4351 #ifdef DEBUG 4352 if (pmapdebug & PDB_DESTROY) 4353 printf("pmap_pmap_pool_dtor(%p)\n", pm); 4354 #endif 4355 4356 if ((c = pm->pm_ctx) != NULL) { 4357 ctx_free(pm); 4358 } 4359 4360 #if defined(SUN4M) || defined(SUN4D) 4361 if (CPU_HAS_SRMMU) { 4362 int n; 4363 4364 #if defined(MULTIPROCESSOR) 4365 for (n = 0; n < sparc_ncpus; n++) 4366 #else 4367 n = 0; 4368 #endif 4369 { 4370 int *pt; 4371 4372 #if defined(MULTIPROCESSOR) 4373 /* Did this cpu attach? */ 4374 if (pmap_kernel()->pm_reg_ptps[n] == 0) 4375 continue; 4376 #endif 4377 4378 pt = pm->pm_reg_ptps[n]; 4379 pm->pm_reg_ptps[n] = NULL; 4380 pm->pm_reg_ptps_pa[n] = 0; 4381 pool_put(&L1_pool, pt); 4382 } 4383 } 4384 #endif /* SUN4M || SUN4D */ 4385 splx(s); 4386 } 4387 4388 /* 4389 * Create and return a physical map. 4390 */ 4391 struct pmap * 4392 pmap_create(void) 4393 { 4394 struct pmap *pm; 4395 4396 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 4397 4398 /* 4399 * Reset fields that are not preserved in the pmap cache pool. 4400 */ 4401 pm->pm_refcount = 1; 4402 #if defined(MULTIPROCESSOR) 4403 /* reset active CPU set */ 4404 pm->pm_cpuset = 0; 4405 #endif 4406 if (CPU_HAS_SUNMMU) { 4407 /* reset the region gap */ 4408 pm->pm_gap_start = 0; 4409 pm->pm_gap_end = VA_VREG(VM_MAXUSER_ADDRESS); 4410 } 4411 4412 #ifdef DEBUG 4413 if (pmapdebug & PDB_CREATE) 4414 printf("pmap_create[%d]: created %p\n", cpu_number(), pm); 4415 pmap_quiet_check(pm); 4416 #endif 4417 return (pm); 4418 } 4419 4420 /* 4421 * Retire the given pmap from service. 4422 * Should only be called if the map contains no valid mappings. 4423 */ 4424 void 4425 pmap_destroy(struct pmap *pm) 4426 { 4427 4428 #ifdef DEBUG 4429 if (pmapdebug & PDB_DESTROY) 4430 printf("pmap_destroy[%d](%p)\n", cpu_number(), pm); 4431 #endif 4432 if (atomic_dec_uint_nv(&pm->pm_refcount) == 0) { 4433 #ifdef DEBUG 4434 pmap_quiet_check(pm); 4435 #endif 4436 pool_cache_put(&pmap_cache, pm); 4437 } 4438 } 4439 4440 /* 4441 * Add a reference to the given pmap. 4442 */ 4443 void 4444 pmap_reference(struct pmap *pm) 4445 { 4446 4447 atomic_inc_uint(&pm->pm_refcount); 4448 } 4449 4450 #if defined(SUN4) || defined(SUN4C) 4451 /* 4452 * helper to deallocate level 2 & 3 page tables. 4453 */ 4454 static void 4455 pgt_lvl23_remove4_4c(struct pmap *pm, struct regmap *rp, struct segmap *sp, 4456 int vr, int vs) 4457 { 4458 vaddr_t va, tva; 4459 int i, pmeg; 4460 4461 va = VSTOVA(vr,vs); 4462 if ((pmeg = sp->sg_pmeg) != seginval) { 4463 if (CTX_USABLE(pm,rp)) { 4464 setcontext4(pm->pm_ctxnum); 4465 setsegmap(va, seginval); 4466 } else { 4467 /* no context, use context 0 */ 4468 setcontext4(0); 4469 if (HASSUN4_MMU3L && rp->rg_smeg != reginval) { 4470 setregmap(0, rp->rg_smeg); 4471 tva = vs << SGSHIFT; 4472 setsegmap(tva, seginval); 4473 } 4474 } 4475 if (!HASSUN4_MMU3L) { 4476 if (pm == pmap_kernel()) { 4477 /* Unmap segment from all contexts */ 4478 for (i = ncontext; --i >= 0;) { 4479 setcontext4(i); 4480 setsegmap(va, seginval); 4481 } 4482 } 4483 } 4484 me_free(pm, pmeg); 4485 sp->sg_pmeg = seginval; 4486 } 4487 /* Free software tables for non-kernel maps */ 4488 if (pm != pmap_kernel()) { 4489 pool_put(&pte_pool, sp->sg_pte); 4490 sp->sg_pte = NULL; 4491 } 4492 4493 if (rp->rg_nsegmap <= 0) 4494 panic("pgt_rm: pm %p: nsegmap = %d\n", pm, rp->rg_nsegmap); 4495 4496 if (--rp->rg_nsegmap == 0) { 4497 #if defined(SUN4_MMU3L) 4498 if (HASSUN4_MMU3L) { 4499 if (rp->rg_smeg != reginval) { 4500 if (pm == pmap_kernel()) { 4501 /* Unmap from all contexts */ 4502 for (i = ncontext; --i >= 0;) { 4503 setcontext4(i); 4504 setregmap(va, reginval); 4505 } 4506 } else if (pm->pm_ctx) { 4507 setcontext4(pm->pm_ctxnum); 4508 setregmap(va, reginval); 4509 } 4510 4511 /* Release MMU resource */ 4512 region_free(pm, rp->rg_smeg); 4513 rp->rg_smeg = reginval; 4514 } 4515 } 4516 #endif /* SUN4_MMU3L */ 4517 /* Free software tables for non-kernel maps */ 4518 if (pm != pmap_kernel()) { 4519 GAP_WIDEN(pm,vr); 4520 pool_put(&segmap_pool, rp->rg_segmap); 4521 rp->rg_segmap = NULL; 4522 } 4523 } 4524 } 4525 #endif /* SUN4 || SUN4C */ 4526 4527 #if defined(SUN4M) || defined(SUN4D) 4528 /* 4529 * SRMMU helper to deallocate level 2 & 3 page tables. 4530 */ 4531 static void 4532 pgt_lvl23_remove4m(struct pmap *pm, struct regmap *rp, struct segmap *sp, 4533 int vr, int vs) 4534 { 4535 4536 /* Invalidate level 2 PTP entry */ 4537 if (pm->pm_ctx) 4538 tlb_flush_segment(VSTOVA(vr,vs), pm->pm_ctxnum, 4539 PMAP_CPUSET(pm)); 4540 setpgt4m(&rp->rg_seg_ptps[vs], SRMMU_TEINVALID); 4541 pool_put(&L23_pool, sp->sg_pte); 4542 sp->sg_pte = NULL; 4543 4544 /* If region is now empty, remove level 2 pagetable as well */ 4545 if (--rp->rg_nsegmap == 0) { 4546 int n = 0; 4547 if (pm->pm_ctx) 4548 tlb_flush_region(VRTOVA(vr), pm->pm_ctxnum, 4549 PMAP_CPUSET(pm)); 4550 #ifdef MULTIPROCESSOR 4551 /* Invalidate level 1 PTP entries on all CPUs */ 4552 for (; n < sparc_ncpus; n++) { 4553 if ((cpus[n]->flags & CPUFLG_HATCHED) == 0) 4554 continue; 4555 #endif 4556 setpgt4m(&pm->pm_reg_ptps[n][vr], SRMMU_TEINVALID); 4557 #ifdef MULTIPROCESSOR 4558 } 4559 #endif 4560 4561 pool_put(&segmap_pool, rp->rg_segmap); 4562 rp->rg_segmap = NULL; 4563 pool_put(&L23_pool, rp->rg_seg_ptps); 4564 } 4565 } 4566 #endif /* SUN4M || SUN4D */ 4567 4568 void 4569 pmap_remove_all(struct pmap *pm) 4570 { 4571 if (pm->pm_ctx == NULL) 4572 return; 4573 4574 #if defined(SUN4) || defined(SUN4C) 4575 if (CPU_HAS_SUNMMU) { 4576 int ctx = getcontext4(); 4577 setcontext4(pm->pm_ctxnum); 4578 cache_flush_context(pm->pm_ctxnum); 4579 setcontext4(ctx); 4580 } 4581 #endif 4582 4583 #if defined(SUN4M) || defined(SUN4D) 4584 if (CPU_HAS_SRMMU) { 4585 cache_flush_context(pm->pm_ctxnum); 4586 } 4587 #endif 4588 4589 pm->pm_flags |= PMAP_USERCACHECLEAN; 4590 } 4591 4592 /* 4593 * Remove the given range of mapping entries. 4594 * The starting and ending addresses are already rounded to pages. 4595 * Sheer lunacy: pmap_remove is often asked to remove nonexistent 4596 * mappings. 4597 */ 4598 void 4599 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 4600 { 4601 vaddr_t nva; 4602 int vr, vs, s, ctx; 4603 void (*rm)(struct pmap *, vaddr_t, vaddr_t, int, int); 4604 4605 #ifdef DEBUG 4606 if (pmapdebug & PDB_REMOVE) 4607 printf("pmap_remove[%d](%p, 0x%lx, 0x%lx)\n", 4608 cpu_number(), pm, va, endva); 4609 #endif 4610 4611 if (!CPU_HAS_SRMMU) 4612 write_user_windows(); 4613 4614 if (pm == pmap_kernel()) { 4615 /* 4616 * Removing from kernel address space. 4617 */ 4618 rm = pmap_rmk; 4619 } else { 4620 /* 4621 * Removing from user address space. 4622 */ 4623 rm = pmap_rmu; 4624 } 4625 4626 ctx = getcontext(); 4627 s = splvm(); /* XXX conservative */ 4628 PMAP_LOCK(); 4629 for (; va < endva; va = nva) { 4630 /* do one virtual segment at a time */ 4631 vr = VA_VREG(va); 4632 vs = VA_VSEG(va); 4633 nva = VSTOVA(vr, vs + 1); 4634 if (nva == 0 || nva > endva) 4635 nva = endva; 4636 if (pm->pm_regmap[vr].rg_nsegmap != 0) 4637 (*rm)(pm, va, nva, vr, vs); 4638 } 4639 PMAP_UNLOCK(); 4640 splx(s); 4641 setcontext(ctx); 4642 } 4643 4644 /* 4645 * It is the same amount of work to cache_flush_page 16 pages 4646 * as to cache_flush_segment 1 segment, assuming a 64K cache size 4647 * and a 4K page size or a 128K cache size and 8K page size. 4648 */ 4649 #define PMAP_SFL_THRESHOLD 16 /* if > magic, use cache_flush_segment */ 4650 4651 /* 4652 * Remove a range contained within a single segment. 4653 * These are egregiously complicated routines. 4654 */ 4655 4656 #if defined(SUN4) || defined(SUN4C) 4657 4658 /* remove from kernel */ 4659 /*static*/ void 4660 pmap_rmk4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs) 4661 { 4662 int pte, mmupte, *ptep, perpage, npg; 4663 struct vm_page *pg; 4664 int nleft, pmeg, inmmu; 4665 struct regmap *rp; 4666 struct segmap *sp; 4667 4668 rp = &pm->pm_regmap[vr]; 4669 sp = &rp->rg_segmap[vs]; 4670 4671 if (rp->rg_nsegmap == 0) 4672 return; 4673 if ((nleft = sp->sg_npte) == 0) 4674 return; 4675 pmeg = sp->sg_pmeg; 4676 inmmu = pmeg != seginval; 4677 ptep = &sp->sg_pte[VA_VPG(va)]; 4678 4679 /* decide how to flush cache */ 4680 npg = (endva - va) >> PGSHIFT; 4681 if (!inmmu) { 4682 perpage = 0; 4683 } else if (npg > PMAP_SFL_THRESHOLD) { 4684 /* flush the whole segment */ 4685 perpage = 0; 4686 cache_flush_segment(vr, vs, 0); 4687 } else { 4688 /* flush each page individually; some never need flushing */ 4689 perpage = (CACHEINFO.c_vactype != VAC_NONE); 4690 } 4691 4692 for (; va < endva; va += NBPG, ptep++) { 4693 pte = *ptep; 4694 mmupte = inmmu ? getpte4(va) : 0; 4695 if ((pte & PG_V) == 0) { 4696 #ifdef DIAGNOSTIC 4697 if (inmmu && (mmupte & PG_V) != 0) 4698 printf("rmk: inconsistent ptes va=%lx\n", va); 4699 #endif 4700 continue; 4701 } 4702 if ((pte & PG_TYPE) == PG_OBMEM) { 4703 /* if cacheable, flush page as needed */ 4704 if (perpage && (mmupte & PG_NC) == 0) 4705 cache_flush_page(va, 0); 4706 if ((pg = pvhead4_4c(pte)) != NULL) { 4707 if (inmmu) 4708 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte); 4709 pv_unlink4_4c(pg, pm, va); 4710 } 4711 } 4712 nleft--; 4713 #ifdef DIAGNOSTIC 4714 if (nleft < 0) 4715 panic("pmap_rmk: too many PTEs in segment; " 4716 "va 0x%lx; endva 0x%lx", va, endva); 4717 #endif 4718 if (pte & PG_WIRED) { 4719 sp->sg_nwired--; 4720 pm->pm_stats.wired_count--; 4721 } 4722 4723 if (inmmu) 4724 setpte4(va, 0); 4725 *ptep = 0; 4726 pm->pm_stats.resident_count--; 4727 } 4728 4729 #ifdef DIAGNOSTIC 4730 if (sp->sg_nwired > nleft || sp->sg_nwired < 0) 4731 panic("pmap_rmk: pm %p, va %lx: nleft=%d, nwired=%d", 4732 pm, va, nleft, sp->sg_nwired); 4733 #endif 4734 if ((sp->sg_npte = nleft) == 0) 4735 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 4736 else if (sp->sg_nwired == 0) { 4737 if (sp->sg_pmeg != seginval) 4738 mmu_pmeg_unlock(sp->sg_pmeg); 4739 } 4740 } 4741 4742 #endif /* SUN4 || SUN4C */ 4743 4744 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_rmk */ 4745 4746 /* remove from kernel (4m)*/ 4747 /* pm is already locked */ 4748 /*static*/ void 4749 pmap_rmk4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs) 4750 { 4751 int tpte, perpage, npg; 4752 struct vm_page *pg; 4753 struct regmap *rp; 4754 struct segmap *sp; 4755 4756 rp = &pm->pm_regmap[vr]; 4757 sp = &rp->rg_segmap[vs]; 4758 if (rp->rg_nsegmap == 0) 4759 return; 4760 4761 /* decide how to flush cache */ 4762 npg = (endva - va) >> PGSHIFT; 4763 if (npg > PMAP_SFL_THRESHOLD) { 4764 /* flush the whole segment */ 4765 perpage = 0; 4766 if (CACHEINFO.c_vactype != VAC_NONE) 4767 cache_flush_segment(vr, vs, 0); 4768 } else { 4769 /* flush each page individually; some never need flushing */ 4770 perpage = (CACHEINFO.c_vactype != VAC_NONE); 4771 } 4772 while (va < endva) { 4773 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 4774 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) { 4775 #ifdef DEBUG 4776 if ((pmapdebug & PDB_SANITYCHK) && 4777 (getpte4m(va) & SRMMU_TETYPE) == SRMMU_TEPTE) 4778 panic("pmap_rmk: Spurious kTLB entry for 0x%lx", 4779 va); 4780 #endif 4781 va += NBPG; 4782 continue; 4783 } 4784 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 4785 /* if cacheable, flush page as needed */ 4786 if (perpage && (tpte & SRMMU_PG_C)) 4787 cache_flush_page(va, 0); 4788 if ((pg = pvhead4m(tpte)) != NULL) { 4789 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte); 4790 pv_unlink4m(pg, pm, va); 4791 } 4792 } 4793 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 4794 SRMMU_TEINVALID, 1, 0, CPUSET_ALL); 4795 pm->pm_stats.resident_count--; 4796 va += NBPG; 4797 } 4798 } 4799 #endif /* SUN4M || SUN4D */ 4800 4801 #if defined(SUN4) || defined(SUN4C) 4802 4803 /* remove from user */ 4804 /*static*/ void 4805 pmap_rmu4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs) 4806 { 4807 int *ptep, pteva, pte, perpage, npg; 4808 struct vm_page *pg; 4809 int nleft, pmeg, inmmu; 4810 struct regmap *rp; 4811 struct segmap *sp; 4812 4813 rp = &pm->pm_regmap[vr]; 4814 if (rp->rg_nsegmap == 0) 4815 return; 4816 sp = &rp->rg_segmap[vs]; 4817 if ((nleft = sp->sg_npte) == 0) 4818 return; 4819 pmeg = sp->sg_pmeg; 4820 inmmu = pmeg != seginval; 4821 4822 /* 4823 * PTEs are in MMU. Invalidate in hardware, update ref & 4824 * mod bits, and flush cache if required. 4825 */ 4826 if (!inmmu) { 4827 perpage = 0; 4828 pteva = 0; 4829 } else if (CTX_USABLE(pm,rp)) { 4830 /* process has a context, must flush cache */ 4831 npg = (endva - va) >> PGSHIFT; 4832 setcontext4(pm->pm_ctxnum); 4833 if ((pm->pm_flags & PMAP_USERCACHECLEAN) != 0) 4834 perpage = 0; 4835 else if (npg > PMAP_SFL_THRESHOLD) { 4836 perpage = 0; /* flush the whole segment */ 4837 cache_flush_segment(vr, vs, pm->pm_ctxnum); 4838 } else 4839 perpage = (CACHEINFO.c_vactype != VAC_NONE); 4840 pteva = va; 4841 } else { 4842 /* no context, use context 0; cache flush unnecessary */ 4843 setcontext4(0); 4844 if (HASSUN4_MMU3L) 4845 setregmap(0, tregion); 4846 /* XXX use per-CPU pteva? */ 4847 setsegmap(0, pmeg); 4848 pteva = VA_VPG(va) << PGSHIFT; 4849 perpage = 0; 4850 } 4851 4852 ptep = sp->sg_pte + VA_VPG(va); 4853 for (; va < endva; ptep++, pteva += NBPG, va += NBPG) { 4854 int mmupte; 4855 pte = *ptep; 4856 mmupte = inmmu ? getpte4(pteva) : 0; 4857 4858 if ((pte & PG_V) == 0) { 4859 #ifdef DIAGNOSTIC 4860 if (inmmu && (mmupte & PG_V) != 0) 4861 printf("pmap_rmu: pte=%x, mmupte=%x\n", 4862 pte, getpte4(pteva)); 4863 #endif 4864 continue; 4865 } 4866 if ((pte & PG_TYPE) == PG_OBMEM) { 4867 /* if cacheable, flush page as needed */ 4868 if (perpage && (mmupte & PG_NC) == 0) 4869 cache_flush_page(va, pm->pm_ctxnum); 4870 if ((pg = pvhead4_4c(pte)) != NULL) { 4871 if (inmmu) 4872 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte); 4873 pv_unlink4_4c(pg, pm, va); 4874 } 4875 } 4876 nleft--; 4877 #ifdef DIAGNOSTIC 4878 if (nleft < 0) 4879 panic("pmap_rmu: too many PTEs in segment; " 4880 "va 0x%lx; endva 0x%lx", va, endva); 4881 #endif 4882 if (inmmu) 4883 setpte4(pteva, 0); 4884 4885 if (pte & PG_WIRED) { 4886 sp->sg_nwired--; 4887 pm->pm_stats.wired_count--; 4888 } 4889 *ptep = 0; 4890 pm->pm_stats.resident_count--; 4891 } 4892 4893 #ifdef DIAGNOSTIC 4894 if (sp->sg_nwired > nleft || sp->sg_nwired < 0) 4895 panic("pmap_rmu: pm %p, va %lx: nleft=%d, nwired=%d", 4896 pm, va, nleft, sp->sg_nwired); 4897 #endif 4898 if ((sp->sg_npte = nleft) == 0) 4899 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 4900 else if (sp->sg_nwired == 0) { 4901 if (sp->sg_pmeg != seginval) 4902 mmu_pmeg_unlock(sp->sg_pmeg); 4903 } 4904 } 4905 4906 #endif /* SUN4 || SUN4C */ 4907 4908 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_rmu */ 4909 /* remove from user */ 4910 /* Note: pm is already locked */ 4911 /*static*/ void 4912 pmap_rmu4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs) 4913 { 4914 int *pte0, perpage, npg; 4915 struct vm_page *pg; 4916 int nleft; 4917 struct regmap *rp; 4918 struct segmap *sp; 4919 4920 rp = &pm->pm_regmap[vr]; 4921 if (rp->rg_nsegmap == 0) 4922 return; 4923 sp = &rp->rg_segmap[vs]; 4924 if ((nleft = sp->sg_npte) == 0) 4925 return; 4926 pte0 = sp->sg_pte; 4927 4928 /* 4929 * Invalidate PTE in MMU pagetables. Flush cache if necessary. 4930 */ 4931 if (pm->pm_ctx && (pm->pm_flags & PMAP_USERCACHECLEAN) == 0) { 4932 /* process has a context, must flush cache */ 4933 if (CACHEINFO.c_vactype != VAC_NONE) { 4934 npg = (endva - va) >> PGSHIFT; 4935 if (npg > PMAP_SFL_THRESHOLD) { 4936 perpage = 0; /* flush the whole segment */ 4937 cache_flush_segment(vr, vs, pm->pm_ctxnum); 4938 } else 4939 perpage = 1; 4940 } else 4941 perpage = 0; 4942 } else { 4943 /* no context; cache flush unnecessary */ 4944 perpage = 0; 4945 } 4946 for (; va < endva; va += NBPG) { 4947 int tpte; 4948 4949 tpte = pte0[VA_SUN4M_VPG(va)]; 4950 4951 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) { 4952 #ifdef DEBUG 4953 if ((pmapdebug & PDB_SANITYCHK) && 4954 pm->pm_ctx && 4955 (getpte4m(va) & SRMMU_TEPTE) == SRMMU_TEPTE) 4956 panic("pmap_rmu: Spurious uTLB entry for 0x%lx", 4957 va); 4958 #endif 4959 continue; 4960 } 4961 4962 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 4963 /* if cacheable, flush page as needed */ 4964 if (perpage && (tpte & SRMMU_PG_C)) 4965 cache_flush_page(va, pm->pm_ctxnum); 4966 if ((pg = pvhead4m(tpte)) != NULL) { 4967 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte); 4968 pv_unlink4m(pg, pm, va); 4969 } 4970 } 4971 nleft--; 4972 #ifdef DIAGNOSTIC 4973 if (nleft < 0) 4974 panic("pmap_rmu: too many PTEs in segment; " 4975 "va 0x%lx; endva 0x%lx", va, endva); 4976 #endif 4977 setpgt4m_va(va, &pte0[VA_SUN4M_VPG(va)], SRMMU_TEINVALID, 4978 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm)); 4979 pm->pm_stats.resident_count--; 4980 if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) { 4981 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 4982 pm->pm_stats.wired_count--; 4983 } 4984 } 4985 4986 /* 4987 * If the segment is all gone, and the context is loaded, give 4988 * the segment back. 4989 */ 4990 if ((sp->sg_npte = nleft) == 0) 4991 pgt_lvl23_remove4m(pm, rp, sp, vr, vs); 4992 } 4993 #endif /* SUN4M || SUN4D */ 4994 4995 /* 4996 * Lower (make more strict) the protection on the specified 4997 * physical page. 4998 * 4999 * There are only two cases: either the protection is going to 0 5000 * (in which case we do the dirty work here), or it is going from 5001 * to read-only (in which case pv_changepte does the trick). 5002 */ 5003 5004 #if defined(SUN4) || defined(SUN4C) 5005 void 5006 pmap_page_protect4_4c(struct vm_page *pg, vm_prot_t prot) 5007 { 5008 struct pvlist *pv, *npv; 5009 struct pmap *pm; 5010 vaddr_t va; 5011 int vr, vs, pteva, pte, *ptep; 5012 int flags, nleft, s, ctx; 5013 struct regmap *rp; 5014 struct segmap *sp; 5015 5016 #ifdef DEBUG 5017 if ((pmapdebug & PDB_CHANGEPROT) || 5018 (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE)) 5019 printf("pmap_page_protect(0x%lx, 0x%x)\n", 5020 VM_PAGE_TO_PHYS(pg), prot); 5021 #endif 5022 /* 5023 * Skip unmanaged pages, or operations that do not take 5024 * away write permission. 5025 */ 5026 if (prot & VM_PROT_WRITE) 5027 return; 5028 5029 write_user_windows(); /* paranoia */ 5030 if (prot & VM_PROT_READ) { 5031 pv_changepte4_4c(pg, 0, PG_W); 5032 return; 5033 } 5034 5035 /* 5036 * Remove all access to all people talking to this page. 5037 * Walk down PV list, removing all mappings. 5038 * The logic is much like that for pmap_remove, 5039 * but we know we are removing exactly one page. 5040 */ 5041 s = splvm(); 5042 pv = VM_MDPAGE_PVHEAD(pg); 5043 if (pv->pv_pmap == NULL) { 5044 splx(s); 5045 return; 5046 } 5047 ctx = getcontext4(); 5048 5049 /* This pv head will become empty, so clear caching state flags */ 5050 flags = pv->pv_flags & ~(PV_NC|PV_ANC); 5051 5052 while (pv != NULL) { 5053 pm = pv->pv_pmap; 5054 va = pv->pv_va; 5055 vr = VA_VREG(va); 5056 vs = VA_VSEG(va); 5057 rp = &pm->pm_regmap[vr]; 5058 sp = &rp->rg_segmap[vs]; 5059 if ((nleft = sp->sg_npte) <= 0) 5060 panic("pmap_page_protect: empty vseg"); 5061 sp->sg_npte = --nleft; 5062 ptep = &sp->sg_pte[VA_VPG(va)]; 5063 5064 if (*ptep & PG_WIRED) { 5065 sp->sg_nwired--; 5066 pm->pm_stats.wired_count--; 5067 } 5068 5069 if (sp->sg_pmeg != seginval) { 5070 /* Update PV flags */ 5071 if (CTX_USABLE(pm,rp)) { 5072 setcontext4(pm->pm_ctxnum); 5073 pteva = va; 5074 cache_flush_page(va, pm->pm_ctxnum); 5075 } else { 5076 setcontext4(0); 5077 /* XXX use per-CPU pteva? */ 5078 if (HASSUN4_MMU3L) 5079 setregmap(0, tregion); 5080 setsegmap(0, sp->sg_pmeg); 5081 pteva = VA_VPG(va) << PGSHIFT; 5082 } 5083 5084 pte = getpte4(pteva); 5085 #ifdef DIAGNOSTIC 5086 if ((pte & PG_V) == 0) 5087 panic("pmap_page_protect !PG_V: pg %p " 5088 "ctx %d, va 0x%lx, pte 0x%x", 5089 pg, pm->pm_ctxnum, va, pte); 5090 #endif 5091 flags |= MR4_4C(pte); 5092 5093 setpte4(pteva, 0); 5094 #ifdef DIAGNOSTIC 5095 if (sp->sg_nwired > nleft || sp->sg_nwired < 0) 5096 panic("pmap_page_protect: pm %p, va %lx: nleft=%d, nwired=%d", 5097 pm, va, nleft, sp->sg_nwired); 5098 #endif 5099 if (sp->sg_nwired == 0) 5100 mmu_pmeg_unlock(sp->sg_pmeg); 5101 } 5102 5103 *ptep = 0; 5104 pm->pm_stats.resident_count--; 5105 if (nleft == 0) 5106 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 5107 npv = pv->pv_next; 5108 if (pv != VM_MDPAGE_PVHEAD(pg)) 5109 pool_put(&pv_pool, pv); 5110 pv = npv; 5111 } 5112 5113 /* Finally, update pv head */ 5114 VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL; 5115 VM_MDPAGE_PVHEAD(pg)->pv_next = NULL; 5116 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags; 5117 setcontext4(ctx); 5118 splx(s); 5119 } 5120 5121 /* 5122 * Lower (make more strict) the protection on the specified 5123 * range of this pmap. 5124 * 5125 * There are only two cases: either the protection is going to 0 5126 * (in which case we call pmap_remove to do the dirty work), or 5127 * it is going from read/write to read-only. The latter is 5128 * fairly easy. 5129 */ 5130 void 5131 pmap_protect4_4c(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 5132 { 5133 int va, nva, vr, vs; 5134 int s, ctx; 5135 struct regmap *rp; 5136 struct segmap *sp; 5137 5138 if ((prot & VM_PROT_READ) == 0) { 5139 pmap_remove(pm, sva, eva); 5140 return; 5141 } 5142 5143 write_user_windows(); 5144 ctx = getcontext4(); 5145 s = splvm(); 5146 PMAP_LOCK(); 5147 for (va = sva; va < eva;) { 5148 vr = VA_VREG(va); 5149 vs = VA_VSEG(va); 5150 rp = &pm->pm_regmap[vr]; 5151 nva = VSTOVA(vr,vs + 1); 5152 if (nva > eva) 5153 nva = eva; 5154 if (rp->rg_nsegmap == 0) { 5155 va = nva; 5156 continue; 5157 } 5158 #ifdef DEBUG 5159 if (rp->rg_segmap == NULL) 5160 panic("pmap_protect: no segments"); 5161 #endif 5162 sp = &rp->rg_segmap[vs]; 5163 if (sp->sg_npte == 0) { 5164 va = nva; 5165 continue; 5166 } 5167 #ifdef DEBUG 5168 if (sp->sg_pte == NULL) 5169 panic("pmap_protect: no pages"); 5170 #endif 5171 if (sp->sg_pmeg == seginval) { 5172 int *ptep = &sp->sg_pte[VA_VPG(va)]; 5173 5174 /* not in MMU; just clear PG_W from core copies */ 5175 for (; va < nva; va += NBPG) 5176 *ptep++ &= ~PG_W; 5177 } else { 5178 /* in MMU: take away write bits from MMU PTEs */ 5179 if (CTX_USABLE(pm,rp)) { 5180 int pte; 5181 5182 /* 5183 * Flush cache so that any existing cache 5184 * tags are updated. This is really only 5185 * needed for PTEs that lose PG_W. 5186 */ 5187 pmap_stats.ps_npg_prot_all += 5188 (nva - va) >> PGSHIFT; 5189 setcontext4(pm->pm_ctxnum); 5190 for (; va < nva; va += NBPG) { 5191 pte = getpte4(va); 5192 if ((pte & (PG_W|PG_TYPE)) == 5193 (PG_W|PG_OBMEM)) { 5194 pmap_stats.ps_npg_prot_actual++; 5195 cache_flush_page(va, pm->pm_ctxnum); 5196 setpte4(va, pte & ~PG_W); 5197 } 5198 } 5199 } else { 5200 int pteva; 5201 5202 /* 5203 * No context, hence not cached; 5204 * just update PTEs. 5205 */ 5206 setcontext4(0); 5207 /* XXX use per-CPU pteva? */ 5208 if (HASSUN4_MMU3L) 5209 setregmap(0, tregion); 5210 setsegmap(0, sp->sg_pmeg); 5211 pteva = VA_VPG(va) << PGSHIFT; 5212 for (; va < nva; pteva += NBPG, va += NBPG) 5213 setpte4(pteva, getpte4(pteva) & ~PG_W); 5214 } 5215 } 5216 } 5217 PMAP_UNLOCK(); 5218 splx(s); 5219 setcontext4(ctx); 5220 } 5221 5222 /* 5223 * Change the protection and/or wired status of the given (MI) virtual page. 5224 * XXX: should have separate function (or flag) telling whether only wiring 5225 * is changing. 5226 */ 5227 void 5228 pmap_changeprot4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags) 5229 { 5230 int vr, vs, newprot, ctx, pte, *ptep; 5231 int pmeg; 5232 struct regmap *rp; 5233 struct segmap *sp; 5234 5235 #ifdef DEBUG 5236 if (pmapdebug & PDB_CHANGEPROT) 5237 printf("pmap_changeprot(%p, 0x%lx, 0x%x, 0x%x)\n", 5238 pm, va, prot, flags); 5239 #endif 5240 5241 if (pm == pmap_kernel()) 5242 newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S; 5243 else 5244 newprot = prot & VM_PROT_WRITE ? PG_W : 0; 5245 vr = VA_VREG(va); 5246 vs = VA_VSEG(va); 5247 rp = &pm->pm_regmap[vr]; 5248 sp = &rp->rg_segmap[vs]; 5249 ptep = &sp->sg_pte[VA_VPG(va)]; 5250 5251 pmap_stats.ps_changeprots++; 5252 5253 pte = *ptep; 5254 if (pte & PG_WIRED && (flags & PMAP_WIRED) == 0) { 5255 pte &= ~PG_WIRED; 5256 sp->sg_nwired--; 5257 pm->pm_stats.wired_count--; 5258 } else if ((pte & PG_WIRED) == 0 && flags & PMAP_WIRED) { 5259 pte |= PG_WIRED; 5260 sp->sg_nwired++; 5261 pm->pm_stats.wired_count++; 5262 } 5263 pte = (pte & ~PG_PROT) | newprot; 5264 /* Update S/W pte entry */ 5265 *ptep = pte; 5266 5267 /* update PTEs in software or hardware */ 5268 if ((pmeg = sp->sg_pmeg) != seginval) { 5269 /* update in hardware */ 5270 ctx = getcontext4(); 5271 if (CTX_USABLE(pm,rp)) { 5272 /* 5273 * Use current context. 5274 * Flush cache if page has been referenced to 5275 * avoid stale protection bits in the cache tags. 5276 */ 5277 setcontext4(pm->pm_ctxnum); 5278 pte = getpte4(va); 5279 if ((pte & (PG_U|PG_NC|PG_TYPE)) == (PG_U|PG_OBMEM)) 5280 cache_flush_page(va, pm->pm_ctxnum); 5281 } else { 5282 setcontext4(0); 5283 /* XXX use per-CPU va? */ 5284 if (HASSUN4_MMU3L) 5285 setregmap(0, tregion); 5286 setsegmap(0, pmeg); 5287 va = VA_VPG(va) << PGSHIFT; 5288 pte = getpte4(va); 5289 } 5290 pte = (pte & ~PG_PROT) | newprot; 5291 setpte4(va, pte); 5292 setcontext4(ctx); 5293 #ifdef DIAGNOSTIC 5294 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 5295 panic("pmap_protect: pm %p, va %lx: nleft=%d, nwired=%d", 5296 pm, va, sp->sg_npte, sp->sg_nwired); 5297 #endif 5298 if (sp->sg_nwired == 0) 5299 mmu_pmeg_unlock(pmeg); 5300 else 5301 mmu_pmeg_lock(pmeg); 5302 } 5303 } 5304 5305 #endif /* SUN4 || SUN4C */ 5306 5307 #if defined(SUN4M) || defined(SUN4D) 5308 /* 5309 * Lower (make more strict) the protection on the specified 5310 * physical page. 5311 * 5312 * There are only two cases: either the protection is going to 0 5313 * (in which case we do the dirty work here), or it is going 5314 * to read-only (in which case pv_changepte does the trick). 5315 */ 5316 void 5317 pmap_page_protect4m(struct vm_page *pg, vm_prot_t prot) 5318 { 5319 struct pvlist *pv, *npv; 5320 struct pmap *pm; 5321 vaddr_t va; 5322 int vr, vs, tpte; 5323 int flags, nleft, s; 5324 struct regmap *rp; 5325 struct segmap *sp; 5326 5327 #ifdef DEBUG 5328 if ((pmapdebug & PDB_CHANGEPROT) || 5329 (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE)) 5330 printf("pmap_page_protect[%d](0x%lx, 0x%x)\n", 5331 cpu_number(), VM_PAGE_TO_PHYS(pg), prot); 5332 #endif 5333 s = splvm(); 5334 PMAP_LOCK(); 5335 5336 if (prot & VM_PROT_READ) { 5337 pv_changepte4m(pg, 0, PPROT_WRITE); 5338 goto out; 5339 } 5340 5341 /* 5342 * Remove all access to all people talking to this page. 5343 * Walk down PV list, removing all mappings. The logic is much 5344 * like that for pmap_remove, but we know we are removing exactly 5345 * one page. 5346 */ 5347 pv = VM_MDPAGE_PVHEAD(pg); 5348 if (pv->pv_pmap == NULL) 5349 goto out; 5350 5351 /* This pv head will become empty, so clear caching state flags */ 5352 flags = pv->pv_flags & ~(PV_NC|PV_ANC); 5353 while (pv != NULL) { 5354 pm = pv->pv_pmap; 5355 va = pv->pv_va; 5356 vr = VA_VREG(va); 5357 vs = VA_VSEG(va); 5358 rp = &pm->pm_regmap[vr]; 5359 if (rp->rg_nsegmap == 0) 5360 panic("pmap_remove_all: empty vreg"); 5361 sp = &rp->rg_segmap[vs]; 5362 nleft = sp->sg_npte; 5363 if (pm != pmap_kernel()) { 5364 if (nleft <= 0) 5365 panic("pmap_page_protect: empty vseg"); 5366 sp->sg_npte = --nleft; 5367 } 5368 5369 /* 5370 * Invalidate PTE in MMU pagetables. 5371 * Flush cache if necessary. 5372 */ 5373 if (pm->pm_ctx) { 5374 cache_flush_page(va, pm->pm_ctxnum); 5375 } 5376 5377 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 5378 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], SRMMU_TEINVALID, 5379 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm)); 5380 5381 pm->pm_stats.resident_count--; 5382 if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) { 5383 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 5384 pm->pm_stats.wired_count--; 5385 } 5386 5387 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) 5388 panic("pmap_page_protect !PG_V: pg %p va %lx", pg, va); 5389 5390 flags |= MR4M(tpte); 5391 5392 if (pm != pmap_kernel() && nleft == 0) 5393 /* 5394 * Entire user mode segment is gone 5395 */ 5396 pgt_lvl23_remove4m(pm, rp, sp, vr, vs); 5397 5398 npv = pv->pv_next; 5399 if (pv != VM_MDPAGE_PVHEAD(pg)) 5400 pool_put(&pv_pool, pv); 5401 pv = npv; 5402 } 5403 5404 /* Finally, update pv head */ 5405 VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL; 5406 VM_MDPAGE_PVHEAD(pg)->pv_next = NULL; 5407 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags; 5408 5409 out: 5410 PMAP_UNLOCK(); 5411 splx(s); 5412 } 5413 5414 /* 5415 * Lower (make more strict) the protection on the specified 5416 * range of this pmap. 5417 */ 5418 void 5419 pmap_protect4m(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 5420 { 5421 vaddr_t va, nva; 5422 int s, vr, vs; 5423 struct regmap *rp; 5424 struct segmap *sp; 5425 int newprot; 5426 5427 if ((prot & VM_PROT_READ) == 0) { 5428 pmap_remove(pm, sva, eva); 5429 return; 5430 } 5431 5432 #ifdef DEBUG 5433 if (pmapdebug & PDB_CHANGEPROT) 5434 printf("pmap_protect[%d][curpid %d, ctx %d,%d](%lx, %lx, %x)\n", 5435 cpu_number(), curproc->p_pid, 5436 getcontext4m(), pm->pm_ctx ? pm->pm_ctxnum : -1, 5437 sva, eva, prot); 5438 #endif 5439 5440 newprot = pte_prot4m(pm, prot); 5441 5442 write_user_windows(); 5443 s = splvm(); 5444 PMAP_LOCK(); 5445 5446 for (va = sva; va < eva;) { 5447 vr = VA_VREG(va); 5448 vs = VA_VSEG(va); 5449 rp = &pm->pm_regmap[vr]; 5450 nva = VSTOVA(vr,vs + 1); 5451 if (nva > eva) 5452 nva = eva; 5453 if (rp->rg_nsegmap == 0) { 5454 va = nva; 5455 continue; 5456 } 5457 sp = &rp->rg_segmap[vs]; 5458 if (pm != pmap_kernel() && sp->sg_npte == 0) { 5459 va = nva; 5460 continue; 5461 } 5462 5463 /* 5464 * pages loaded: take away write bits from MMU PTEs 5465 */ 5466 pmap_stats.ps_npg_prot_all += (nva - va) >> PGSHIFT; 5467 for (; va < nva; va += NBPG) { 5468 int tpte, npte; 5469 5470 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 5471 if ((tpte & SRMMU_PGTYPE) != PG_SUN4M_OBMEM) 5472 continue; 5473 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) 5474 continue; 5475 npte = (tpte & ~SRMMU_PROT_MASK) | newprot; 5476 if (npte == tpte) 5477 continue; 5478 5479 /* 5480 * Flush cache so that any existing cache 5481 * tags are updated. 5482 */ 5483 5484 pmap_stats.ps_npg_prot_actual++; 5485 if (pm->pm_ctx) { 5486 cache_flush_page(va, pm->pm_ctxnum); 5487 } 5488 updatepte4m(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 5489 SRMMU_PROT_MASK, newprot, pm->pm_ctxnum, 5490 PMAP_CPUSET(pm)); 5491 } 5492 } 5493 PMAP_UNLOCK(); 5494 splx(s); 5495 } 5496 5497 /* 5498 * Change the protection and/or wired status of the given (MI) virtual page. 5499 * XXX: should have separate function (or flag) telling whether only wiring 5500 * is changing. 5501 */ 5502 void 5503 pmap_changeprot4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags) 5504 { 5505 int pte, newprot; 5506 struct regmap *rp; 5507 struct segmap *sp; 5508 bool owired; 5509 5510 #ifdef DEBUG 5511 if (pmapdebug & PDB_CHANGEPROT) 5512 printf("pmap_changeprot[%d](%p, 0x%lx, 0x%x, 0x%x)\n", 5513 cpu_number(), pm, va, prot, flags); 5514 #endif 5515 5516 newprot = pte_prot4m(pm, prot); 5517 5518 pmap_stats.ps_changeprots++; 5519 5520 rp = &pm->pm_regmap[VA_VREG(va)]; 5521 sp = &rp->rg_segmap[VA_VSEG(va)]; 5522 5523 pte = sp->sg_pte[VA_SUN4M_VPG(va)]; 5524 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va)); 5525 5526 if (owired) { 5527 pm->pm_stats.wired_count--; 5528 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 5529 } 5530 if (flags & PMAP_WIRED) { 5531 pm->pm_stats.wired_count++; 5532 sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va)); 5533 } 5534 5535 if (pm->pm_ctx) { 5536 /* 5537 * Use current context. 5538 * Flush cache if page has been referenced to 5539 * avoid stale protection bits in the cache tags. 5540 */ 5541 5542 if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) == 5543 (SRMMU_PG_C|PG_SUN4M_OBMEM)) 5544 cache_flush_page(va, pm->pm_ctxnum); 5545 } 5546 5547 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 5548 (pte & ~SRMMU_PROT_MASK) | newprot, 5549 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm)); 5550 5551 } 5552 #endif /* SUN4M || SUN4D */ 5553 5554 /* 5555 * Insert (MI) physical page pa at virtual address va in the given pmap. 5556 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary. 5557 * 5558 * If pa is not in the `managed' range it will not be `bank mapped'. 5559 * This works during bootstrap only because the first 4MB happens to 5560 * map one-to-one. 5561 * 5562 * There may already be something else there, or we might just be 5563 * changing protections and/or wiring on an existing mapping. 5564 * XXX should have different entry points for changing! 5565 */ 5566 5567 #if defined(SUN4) || defined(SUN4C) 5568 5569 int 5570 pmap_enter4_4c(struct pmap *pm, vaddr_t va, paddr_t pa, 5571 vm_prot_t prot, u_int flags) 5572 { 5573 struct vm_page *pg; 5574 int pteproto, ctx; 5575 int error; 5576 5577 if (VA_INHOLE(va)) { 5578 #ifdef DEBUG 5579 printf("pmap_enter: pm %p, va 0x%lx, pa 0x%lx: in MMU hole\n", 5580 pm, va, pa); 5581 #endif 5582 return 0; 5583 } 5584 5585 #ifdef DEBUG 5586 if (pmapdebug & PDB_ENTER) 5587 printf("pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)\n", 5588 pm, va, pa, prot, flags); 5589 #endif 5590 5591 pg = PHYS_TO_VM_PAGE(pa); 5592 pteproto = PG_V | PMAP_T2PTE_4(pa); 5593 pa &= ~PMAP_TNC_4; 5594 5595 /* 5596 * Set up prototype for new PTE. Cannot set PG_NC from PV_NC yet 5597 * since the pvlist no-cache bit might change as a result of the 5598 * new mapping. 5599 */ 5600 pteproto |= atop(pa) & PG_PFNUM; 5601 if (prot & VM_PROT_WRITE) 5602 pteproto |= PG_W; 5603 if ((flags & PMAP_WIRED) != 0) 5604 pteproto |= PG_WIRED; 5605 if (flags & VM_PROT_ALL) { 5606 pteproto |= PG_U; 5607 if (flags & VM_PROT_WRITE) { 5608 pteproto |= PG_M; 5609 } 5610 } 5611 5612 write_user_windows(); 5613 ctx = getcontext4(); 5614 if (pm == pmap_kernel()) 5615 error = pmap_enk4_4c(pm, va, prot, flags, pg, pteproto | PG_S); 5616 else 5617 error = pmap_enu4_4c(pm, va, prot, flags, pg, pteproto); 5618 setcontext4(ctx); 5619 return (error); 5620 } 5621 5622 /* enter new (or change existing) kernel mapping */ 5623 int 5624 pmap_enk4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags, 5625 struct vm_page *pg, int pteproto) 5626 { 5627 int vr, vs, pte, s, inmmu; 5628 int *ptep; 5629 struct regmap *rp; 5630 struct segmap *sp; 5631 int error = 0; 5632 5633 vr = VA_VREG(va); 5634 vs = VA_VSEG(va); 5635 rp = &pm->pm_regmap[vr]; 5636 sp = &rp->rg_segmap[vs]; 5637 ptep = &sp->sg_pte[VA_VPG(va)]; 5638 s = splvm(); /* XXX way too conservative */ 5639 5640 #if defined(SUN4_MMU3L) 5641 if (HASSUN4_MMU3L && rp->rg_smeg == reginval) 5642 mmu_pagein_reg(pm, rp, va, vr, ®ion_locked); 5643 #endif 5644 5645 inmmu = sp->sg_pmeg != seginval; 5646 if ((pte = *ptep) & PG_V) { 5647 5648 /* old mapping exists, and is of the same pa type */ 5649 if ((pte & (PG_PFNUM|PG_TYPE)) == 5650 (pteproto & (PG_PFNUM|PG_TYPE))) { 5651 /* just changing protection and/or wiring */ 5652 pmap_changeprot4_4c(pm, va, prot, flags); 5653 splx(s); 5654 return (0); 5655 } 5656 5657 if ((pte & PG_TYPE) == PG_OBMEM) { 5658 struct vm_page *opg; 5659 5660 /* 5661 * Switcheroo: changing pa for this va. 5662 * If old pa was managed, remove from pvlist. 5663 * If old page was cached, flush cache. 5664 */ 5665 if ((opg = pvhead4_4c(pte)) != NULL) 5666 pv_unlink4_4c(opg, pm, va); 5667 if (inmmu && (pte & PG_NC) == 0) { 5668 setcontext4(0); /* ??? */ 5669 cache_flush_page(va, 0); 5670 } 5671 } 5672 *ptep = 0; 5673 if (inmmu) 5674 setpte4(va, 0); 5675 if (pte & PG_WIRED) { 5676 sp->sg_nwired--; 5677 pm->pm_stats.wired_count--; 5678 } 5679 pm->pm_stats.resident_count--; 5680 } else { 5681 /* adding new entry */ 5682 if (sp->sg_npte++ == 0) { 5683 #ifdef DIAGNOSTIC 5684 int i; for (i = 0; i < NPTESG; i++) { 5685 if (sp->sg_pte[i] == 0) 5686 continue; 5687 panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n", 5688 pm, va, i); 5689 } 5690 #endif 5691 rp->rg_nsegmap++; 5692 } 5693 } 5694 5695 /* 5696 * If the new mapping is for a managed PA, enter into pvlist. 5697 */ 5698 if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) { 5699 if (--sp->sg_npte == 0) 5700 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 5701 if ((flags & PMAP_CANFAIL) != 0) 5702 goto out; 5703 panic("pmap_enter: cannot allocate PV entry"); 5704 } 5705 5706 /* Update S/W page table */ 5707 *ptep = pteproto; 5708 if (pteproto & PG_WIRED) { 5709 sp->sg_nwired++; 5710 pm->pm_stats.wired_count++; 5711 } 5712 pm->pm_stats.resident_count++; 5713 5714 #ifdef DIAGNOSTIC 5715 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 5716 panic("pmap_enk: pm %p, va %lx: nleft=%d, nwired=%d", 5717 pm, va, sp->sg_npte, sp->sg_nwired); 5718 #endif 5719 if (sp->sg_pmeg == seginval) 5720 mmu_pagein_seg(pm, sp, va, vr, vs, 5721 (pteproto & PG_WIRED) != 0 ? &segm_locked : &segm_lru); 5722 else if ((pteproto & PG_WIRED) != 0) 5723 mmu_pmeg_lock(sp->sg_pmeg); 5724 5725 /* Update H/W page table */ 5726 setpte4(va, pteproto & ~PG_MBZ); 5727 out: 5728 splx(s); 5729 return (error); 5730 } 5731 5732 /* enter new (or change existing) user mapping */ 5733 int 5734 pmap_enu4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags, 5735 struct vm_page *pg, int pteproto) 5736 { 5737 int vr, vs, *ptep, pte, pmeg, s; 5738 int error = 0; 5739 struct regmap *rp; 5740 struct segmap *sp; 5741 5742 pm->pm_flags &= ~PMAP_USERCACHECLEAN; 5743 5744 vr = VA_VREG(va); 5745 vs = VA_VSEG(va); 5746 rp = &pm->pm_regmap[vr]; 5747 s = splvm(); /* XXX conservative */ 5748 5749 /* 5750 * If there is no space in which the PTEs can be written 5751 * while they are not in the hardware, this must be a new 5752 * virtual segment. Get PTE space and count the segment. 5753 * 5754 * TO SPEED UP CTX ALLOC, PUT SEGMENT BOUNDS STUFF HERE 5755 * AND IN pmap_rmu() 5756 */ 5757 5758 GAP_SHRINK(pm,vr); 5759 5760 #ifdef DEBUG 5761 if (pm->pm_gap_end < pm->pm_gap_start) { 5762 printf("pmap_enu: gap_start 0x%x, gap_end 0x%x", 5763 pm->pm_gap_start, pm->pm_gap_end); 5764 panic("pmap_enu: gap botch"); 5765 } 5766 #endif 5767 5768 if (rp->rg_segmap == NULL) { 5769 /* definitely a new mapping */ 5770 int i; 5771 int mflag = PR_NOWAIT; 5772 5773 rretry: 5774 sp = (struct segmap *)pool_get(&segmap_pool, mflag); 5775 if (sp == NULL) { 5776 if ((flags & PMAP_CANFAIL) != 0) { 5777 error = ENOMEM; 5778 goto out; 5779 } 5780 mflag = PR_WAITOK; 5781 goto rretry; 5782 } 5783 #ifdef DEBUG 5784 if (rp->rg_segmap != NULL) 5785 panic("pmap_enter: segment filled during sleep"); 5786 #endif 5787 qzero((void *)sp, NSEGRG * sizeof (struct segmap)); 5788 rp->rg_segmap = sp; 5789 rp->rg_nsegmap = 0; 5790 for (i = NSEGRG; --i >= 0;) 5791 sp++->sg_pmeg = seginval; 5792 } 5793 5794 sp = &rp->rg_segmap[vs]; 5795 5796 if ((ptep = sp->sg_pte) == NULL) { 5797 /* definitely a new mapping */ 5798 int size = NPTESG * sizeof *ptep; 5799 int mflag = PR_NOWAIT; 5800 5801 sretry: 5802 ptep = (int *)pool_get(&pte_pool, mflag); 5803 if (ptep == NULL) { 5804 if ((flags & PMAP_CANFAIL) != 0) { 5805 error = ENOMEM; 5806 goto out; 5807 } 5808 mflag = PR_WAITOK; 5809 goto sretry; 5810 } 5811 #ifdef DEBUG 5812 if (sp->sg_pte != NULL) 5813 panic("pmap_enter: pte filled during sleep"); 5814 if (sp->sg_pmeg != seginval) 5815 panic("pmap_enter: new ptes, but not seginval"); 5816 #endif 5817 qzero((void *)ptep, size); 5818 sp->sg_pte = ptep; 5819 sp->sg_npte = 1; 5820 rp->rg_nsegmap++; 5821 } else { 5822 /* might be a change: fetch old pte */ 5823 pte = ptep[VA_VPG(va)]; 5824 if (pte & PG_V) { 5825 /* old mapping exists, and is of the same pa type */ 5826 if ((pte & (PG_PFNUM|PG_TYPE)) == 5827 (pteproto & (PG_PFNUM|PG_TYPE))) { 5828 /* just changing prot and/or wiring */ 5829 pmap_changeprot4_4c(pm, va, prot, flags); 5830 splx(s); 5831 return (0); 5832 } 5833 /* 5834 * Switcheroo: changing pa for this va. 5835 * If old pa was managed, remove from pvlist. 5836 * If old page was cached, flush cache. 5837 */ 5838 #if 0 5839 printf("%s[%d]: pmap_enu: changing existing " 5840 "va(0x%lx)=>pa entry\n", 5841 curproc->p_comm, curproc->p_pid, va); 5842 #endif 5843 if ((pte & PG_TYPE) == PG_OBMEM) { 5844 struct vm_page *opg; 5845 if ((opg = pvhead4_4c(pte)) != NULL) 5846 pv_unlink4_4c(opg, pm, va); 5847 if (CACHEINFO.c_vactype != VAC_NONE && 5848 (pmeg = sp->sg_pmeg) != seginval) { 5849 /* hardware pte */ 5850 if (CTX_USABLE(pm,rp)) { 5851 setcontext4(pm->pm_ctxnum); 5852 } else { 5853 setcontext4(0); 5854 /* XXX use per-CPU pteva? */ 5855 if (HASSUN4_MMU3L) 5856 setregmap(0, tregion); 5857 setsegmap(0, pmeg); 5858 } 5859 cache_flush_page(va, pm->pm_ctxnum); 5860 } 5861 } 5862 if (pte & PG_WIRED) { 5863 sp->sg_nwired--; 5864 pm->pm_stats.wired_count--; 5865 } 5866 pm->pm_stats.resident_count--; 5867 ptep[VA_VPG(va)] = 0; 5868 if (sp->sg_pmeg != seginval) 5869 setpte4(va, 0); 5870 } else { 5871 /* adding new entry */ 5872 sp->sg_npte++; 5873 } 5874 } 5875 5876 if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) { 5877 if (--sp->sg_npte == 0) 5878 /* Sigh, undo pgt allocations */ 5879 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 5880 5881 if ((flags & PMAP_CANFAIL) != 0) 5882 goto out; 5883 panic("pmap_enter: cannot allocate PV entry"); 5884 } 5885 5886 /* Update S/W page table */ 5887 ptep += VA_VPG(va); 5888 *ptep = pteproto; 5889 if (pteproto & PG_WIRED) { 5890 sp->sg_nwired++; 5891 pm->pm_stats.wired_count++; 5892 } 5893 pm->pm_stats.resident_count++; 5894 5895 #ifdef DIAGNOSTIC 5896 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 5897 panic("pmap_enu: pm %p, va %lx: nleft=%d, nwired=%d", 5898 pm, va, sp->sg_npte, sp->sg_nwired); 5899 #endif 5900 5901 if ((pmeg = sp->sg_pmeg) != seginval) { 5902 /* Update H/W page table */ 5903 if (CTX_USABLE(pm,rp)) 5904 setcontext4(pm->pm_ctxnum); 5905 else { 5906 setcontext4(0); 5907 if (HASSUN4_MMU3L) 5908 setregmap(0, tregion); 5909 setsegmap(0, pmeg); 5910 va = VA_VPG(va) << PGSHIFT; 5911 } 5912 setpte4(va, pteproto & ~PG_MBZ); 5913 } 5914 5915 out: 5916 splx(s); 5917 return (error); 5918 } 5919 5920 void 5921 pmap_kenter_pa4_4c(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 5922 { 5923 struct pmap *pm = pmap_kernel(); 5924 struct regmap *rp; 5925 struct segmap *sp; 5926 int vr, vs, s; 5927 int *ptep, pteproto; 5928 int lockit = 1; 5929 5930 pteproto = PG_S | PG_V | PMAP_T2PTE_4(pa); 5931 pa &= ~PMAP_TNC_4; 5932 pteproto |= atop(pa) & PG_PFNUM; 5933 if (prot & VM_PROT_WRITE) 5934 pteproto |= PG_W; 5935 5936 vr = VA_VREG(va); 5937 vs = VA_VSEG(va); 5938 rp = &pm->pm_regmap[vr]; 5939 sp = &rp->rg_segmap[vs]; 5940 ptep = &sp->sg_pte[VA_VPG(va)]; 5941 5942 if (lockit) { 5943 pteproto |= PG_WIRED; 5944 sp->sg_nwired++; 5945 } 5946 5947 KASSERT((*ptep & PG_V) == 0); 5948 5949 s = splvm(); 5950 #if defined(SUN4_MMU3L) 5951 if (HASSUN4_MMU3L && rp->rg_smeg == reginval) 5952 mmu_pagein_reg(pm, rp, va, vr, ®ion_locked); 5953 #endif 5954 5955 if (sp->sg_npte++ == 0) { 5956 #ifdef DIAGNOSTIC 5957 int i; for (i = 0; i < NPTESG; i++) { 5958 if (sp->sg_pte[i] == 0) 5959 continue; 5960 panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n", 5961 pm, va, i); 5962 } 5963 #endif 5964 rp->rg_nsegmap++; 5965 } 5966 5967 /* Update S/W page table */ 5968 *ptep = pteproto; 5969 5970 #ifdef DIAGNOSTIC 5971 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 5972 panic("pmap_kenter: pm %p, va %lx: nleft=%d, nwired=%d", 5973 pm, va, sp->sg_npte, sp->sg_nwired); 5974 #endif 5975 5976 if (sp->sg_pmeg == seginval) { 5977 mmu_pagein_seg(pm, sp, va, vr, vs, 5978 lockit ? &segm_locked : &segm_lru); 5979 } else if (lockit) 5980 mmu_pmeg_lock(sp->sg_pmeg); 5981 5982 /* Update H/W page table */ 5983 setpte4(va, pteproto & ~PG_MBZ); 5984 splx(s); 5985 } 5986 5987 #if notyet 5988 void pmap_lockmmu(vaddr_t sva, size_t sz); 5989 5990 void 5991 pmap_lockmmu(vaddr_t sva, size_t sz) 5992 { 5993 struct pmap *pm = pmap_kernel(); 5994 vaddr_t va, eva; 5995 struct regmap *rp; 5996 struct segmap *sp; 5997 int vr, vs; 5998 5999 if (CPU_HAS_SRMMU) 6000 return; 6001 6002 eva = sva + sz; 6003 va = VA_ROUNDDOWNTOSEG(sva); 6004 6005 for (; va < eva; va += NBPSG) { 6006 vr = VA_VREG(va); 6007 vs = VA_VSEG(va); 6008 rp = &pm->pm_regmap[vr]; 6009 sp = &rp->rg_segmap[vs]; 6010 6011 KASSERT(sp->sg_npte != 0); 6012 6013 if (sp->sg_pmeg == seginval) 6014 mmu_pagein_seg(pm, sp, va, vr, vs, &segm_locked); 6015 else 6016 mmu_pmeg_lock(sp->sg_pmeg); 6017 } 6018 } 6019 #endif 6020 6021 void 6022 pmap_kremove4_4c(vaddr_t va, vsize_t len) 6023 { 6024 struct pmap *pm = pmap_kernel(); 6025 struct regmap *rp; 6026 struct segmap *sp; 6027 vaddr_t nva, endva; 6028 int pte, mmupte, *ptep, perpage, npg, inmmu; 6029 int nleft, pmeg; 6030 int vr, vs, s, ctx; 6031 6032 endva = va + len; 6033 #ifdef DEBUG 6034 if (pmapdebug & PDB_REMOVE) 6035 printf("pmap_kremove(0x%lx, 0x%lx)\n", va, endva); 6036 #endif 6037 6038 write_user_windows(); 6039 6040 s = splvm(); 6041 ctx = getcontext(); 6042 PMAP_LOCK(); 6043 setcontext4(0); 6044 for (; va < endva; va = nva) { 6045 /* do one virtual segment at a time */ 6046 vr = VA_VREG(va); 6047 vs = VA_VSEG(va); 6048 nva = VSTOVA(vr, vs + 1); 6049 if (nva == 0 || nva > endva) 6050 nva = endva; 6051 6052 rp = &pm->pm_regmap[vr]; 6053 sp = &rp->rg_segmap[vs]; 6054 6055 if (rp->rg_nsegmap == 0) 6056 continue; 6057 nleft = sp->sg_npte; 6058 if (nleft == 0) 6059 continue; 6060 pmeg = sp->sg_pmeg; 6061 inmmu = (pmeg != seginval); 6062 ptep = &sp->sg_pte[VA_VPG(va)]; 6063 6064 /* decide how to flush cache */ 6065 npg = (nva - va) >> PGSHIFT; 6066 if (!inmmu) { 6067 perpage = 0; 6068 } else if (npg > PMAP_SFL_THRESHOLD) { 6069 /* flush the whole segment */ 6070 perpage = 0; 6071 cache_flush_segment(vr, vs, 0); 6072 } else { 6073 /* 6074 * flush each page individually; 6075 * some never need flushing 6076 */ 6077 perpage = (CACHEINFO.c_vactype != VAC_NONE); 6078 } 6079 6080 for (; va < nva; va += NBPG, ptep++) { 6081 pte = *ptep; 6082 mmupte = inmmu ? getpte4(va) : 0; 6083 if ((pte & PG_V) == 0) { 6084 #ifdef DIAGNOSTIC 6085 if (inmmu && (mmupte & PG_V) != 0) 6086 printf("rmk: inconsistent ptes va=%lx\n", va); 6087 #endif 6088 continue; 6089 } 6090 if ((pte & PG_TYPE) == PG_OBMEM) { 6091 /* if cacheable, flush page as needed */ 6092 if (perpage && (mmupte & PG_NC) == 0) 6093 cache_flush_page(va, 0); 6094 } 6095 nleft--; 6096 #ifdef DIAGNOSTIC 6097 if (nleft < 0) 6098 panic("pmap_kremove: too many PTEs in segment; " 6099 "va 0x%lx; endva 0x%lx", va, endva); 6100 #endif 6101 if (pte & PG_WIRED) 6102 sp->sg_nwired--; 6103 6104 if (inmmu) 6105 setpte4(va, 0); 6106 *ptep = 0; 6107 } 6108 6109 #ifdef DIAGNOSTIC 6110 if (sp->sg_nwired > nleft || sp->sg_nwired < 0) 6111 panic("pmap_kremove: pm %p, va %lx: nleft=%d, nwired=%d", 6112 pm, va, nleft, sp->sg_nwired); 6113 #endif 6114 6115 if ((sp->sg_npte = nleft) == 0) 6116 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 6117 else if (sp->sg_nwired == 0) { 6118 if (sp->sg_pmeg != seginval) 6119 mmu_pmeg_unlock(sp->sg_pmeg); 6120 } 6121 } 6122 PMAP_UNLOCK(); 6123 setcontext4(ctx); 6124 splx(s); 6125 } 6126 6127 /* 6128 * Change protection on a range of kernel addresses. 6129 */ 6130 void 6131 pmap_kprotect4_4c(vaddr_t va, vsize_t size, vm_prot_t prot) 6132 { 6133 int pte, newprot, ctx; 6134 6135 size = roundup(size,NBPG); 6136 newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S; 6137 6138 ctx = getcontext4(); 6139 setcontext4(0); 6140 while (size > 0) { 6141 pte = getpte4(va); 6142 6143 /* 6144 * Flush cache if page has been referenced to 6145 * avoid stale protection bits in the cache tags. 6146 */ 6147 if ((pte & (PG_NC|PG_TYPE)) == PG_OBMEM) 6148 cache_flush_page(va, 0); 6149 6150 pte = (pte & ~PG_PROT) | newprot; 6151 setpte4(va, pte); 6152 6153 va += NBPG; 6154 size -= NBPG; 6155 } 6156 setcontext4(ctx); 6157 } 6158 #endif /* SUN4 || SUN4C */ 6159 6160 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of enter routines */ 6161 /* 6162 * Insert (MI) physical page pa at virtual address va in the given pmap. 6163 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary. 6164 * 6165 * If pa is not in the `managed' range it will not be `bank mapped'. 6166 * This works during bootstrap only because the first 4MB happens to 6167 * map one-to-one. 6168 * 6169 * There may already be something else there, or we might just be 6170 * changing protections and/or wiring on an existing mapping. 6171 * XXX should have different entry points for changing! 6172 */ 6173 6174 int 6175 pmap_enter4m(struct pmap *pm, vaddr_t va, paddr_t pa, 6176 vm_prot_t prot, u_int flags) 6177 { 6178 struct vm_page *pg; 6179 int pteproto; 6180 int error; 6181 6182 #ifdef DEBUG 6183 if (pmapdebug & PDB_ENTER) 6184 printf("pmap_enter[curcpu %d, curpid %d, ctx %d,%d]" 6185 "(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)\n", 6186 cpu_number(), curproc==NULL ? -1 : curproc->p_pid, 6187 getcontext4m(), pm->pm_ctx==NULL ? -1 : pm->pm_ctxnum, 6188 pm, va, pa, prot, flags); 6189 #endif 6190 6191 pg = PHYS_TO_VM_PAGE(pa); 6192 6193 /* Initialise pteproto with cache bit */ 6194 pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0; 6195 6196 #ifdef DEBUG 6197 if (pa & PMAP_TYPE_SRMMU) { /* this page goes in an iospace */ 6198 if (cpuinfo.cpu_type == CPUTYP_MS1) 6199 panic("pmap_enter4m: attempt to use 36-bit iospace on" 6200 " MicroSPARC"); 6201 } 6202 #endif 6203 pteproto |= SRMMU_TEPTE; 6204 pteproto |= PMAP_T2PTE_SRMMU(pa); 6205 pa &= ~PMAP_TNC_SRMMU; 6206 6207 /* 6208 * Set up prototype for new PTE. Cannot set PG_NC from PV_NC yet 6209 * since the pvlist no-cache bit might change as a result of the 6210 * new mapping. 6211 */ 6212 pteproto |= (atop(pa) << SRMMU_PPNSHIFT); 6213 6214 /* Make sure we get a pte with appropriate perms! */ 6215 pteproto |= pte_prot4m(pm, prot); 6216 if (flags & VM_PROT_ALL) { 6217 pteproto |= SRMMU_PG_R; 6218 if (flags & VM_PROT_WRITE) { 6219 pteproto |= SRMMU_PG_M; 6220 } 6221 } 6222 6223 if (pm == pmap_kernel()) 6224 error = pmap_enk4m(pm, va, prot, flags, pg, pteproto | PPROT_S); 6225 else 6226 error = pmap_enu4m(pm, va, prot, flags, pg, pteproto); 6227 6228 return (error); 6229 } 6230 6231 /* enter new (or change existing) kernel mapping */ 6232 int 6233 pmap_enk4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags, 6234 struct vm_page *pg, int pteproto) 6235 { 6236 int vr, vs, tpte, s; 6237 struct regmap *rp; 6238 struct segmap *sp; 6239 int error = 0; 6240 6241 #ifdef DEBUG 6242 if (va < KERNBASE) 6243 panic("pmap_enk4m: can't enter va 0x%lx below KERNBASE", va); 6244 #endif 6245 vr = VA_VREG(va); 6246 vs = VA_VSEG(va); 6247 rp = &pm->pm_regmap[vr]; 6248 sp = &rp->rg_segmap[vs]; 6249 6250 s = splvm(); /* XXX way too conservative */ 6251 PMAP_LOCK(); 6252 6253 if (rp->rg_seg_ptps == NULL) /* enter new region */ 6254 panic("pmap_enk4m: missing kernel region table for va 0x%lx",va); 6255 6256 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6257 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) { 6258 6259 /* old mapping exists, and is of the same pa type */ 6260 6261 if ((tpte & SRMMU_PPNMASK) == (pteproto & SRMMU_PPNMASK)) { 6262 /* just changing protection and/or wiring */ 6263 pmap_changeprot4m(pm, va, prot, flags); 6264 PMAP_UNLOCK(); 6265 splx(s); 6266 return (0); 6267 } 6268 6269 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 6270 struct vm_page *opg; 6271 #ifdef DEBUG 6272 printf("pmap_enk4m: changing existing va=>pa entry: va 0x%lx, pteproto 0x%x, " 6273 "oldpte 0x%x\n", va, pteproto, tpte); 6274 #endif 6275 /* 6276 * Switcheroo: changing pa for this va. 6277 * If old pa was managed, remove from pvlist. 6278 * If old page was cached, flush cache. 6279 */ 6280 if ((opg = pvhead4m(tpte)) != NULL) 6281 pv_unlink4m(opg, pm, va); 6282 if (tpte & SRMMU_PG_C) { 6283 cache_flush_page(va, 0); 6284 } 6285 } 6286 6287 /* 6288 * Invalidate the mapping now, so we can avoid the 6289 * de-map and update protocol when setting the new 6290 * PTE below. 6291 */ 6292 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 6293 SRMMU_TEINVALID, pm->pm_ctx != NULL, 6294 pm->pm_ctxnum, PMAP_CPUSET(pm)); 6295 pm->pm_stats.resident_count--; 6296 } 6297 6298 /* 6299 * If the new mapping is for a managed PA, enter into pvlist. 6300 */ 6301 if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) { 6302 if ((flags & PMAP_CANFAIL) != 0) 6303 goto out; 6304 panic("pmap_enter: cannot allocate PV entry"); 6305 } 6306 6307 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto); 6308 pm->pm_stats.resident_count++; 6309 out: 6310 PMAP_UNLOCK(); 6311 splx(s); 6312 return (error); 6313 } 6314 6315 /* enter new (or change existing) user mapping */ 6316 int 6317 pmap_enu4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags, 6318 struct vm_page *pg, int pteproto) 6319 { 6320 int vr, vs, *pte, tpte, s; 6321 int error = 0; 6322 struct regmap *rp; 6323 struct segmap *sp; 6324 bool owired; 6325 6326 #ifdef DEBUG 6327 if (KERNBASE < va) 6328 panic("pmap_enu4m: can't enter va 0x%lx above KERNBASE", va); 6329 #endif 6330 6331 pm->pm_flags &= ~PMAP_USERCACHECLEAN; 6332 6333 vr = VA_VREG(va); 6334 vs = VA_VSEG(va); 6335 rp = &pm->pm_regmap[vr]; 6336 s = splvm(); /* XXX conservative */ 6337 PMAP_LOCK(); 6338 6339 if (rp->rg_segmap == NULL) { 6340 /* definitely a new mapping */ 6341 int mflag = PR_NOWAIT; 6342 6343 rretry: 6344 sp = (struct segmap *)pool_get(&segmap_pool, mflag); 6345 if (sp == NULL) { 6346 if ((flags & PMAP_CANFAIL) != 0) { 6347 error = ENOMEM; 6348 goto out; 6349 } 6350 mflag = PR_WAITOK; 6351 goto rretry; 6352 } 6353 #ifdef DEBUG 6354 if (rp->rg_segmap != NULL) 6355 panic("pmap_enu4m: segment filled during sleep"); 6356 #endif 6357 qzero((void *)sp, NSEGRG * sizeof (struct segmap)); 6358 rp->rg_segmap = sp; 6359 rp->rg_nsegmap = 0; 6360 rp->rg_seg_ptps = NULL; 6361 } 6362 if (rp->rg_seg_ptps == NULL) { 6363 /* Need a segment table */ 6364 int i, *ptd; 6365 int mflag = PR_NOWAIT; 6366 6367 sretry: 6368 ptd = pool_get(&L23_pool, mflag); 6369 if (ptd == NULL) { 6370 if ((flags & PMAP_CANFAIL) != 0) { 6371 error = ENOMEM; 6372 goto out; 6373 } 6374 mflag = PR_WAITOK; 6375 goto sretry; 6376 } 6377 6378 rp->rg_seg_ptps = ptd; 6379 for (i = 0; i < SRMMU_L2SIZE; i++) 6380 setpgt4m(&ptd[i], SRMMU_TEINVALID); 6381 6382 /* Replicate segment allocation in each CPU's region table */ 6383 #ifdef MULTIPROCESSOR 6384 for (i = 0; i < sparc_ncpus; i++) 6385 #else 6386 i = 0; 6387 #endif 6388 { 6389 #if defined(MULTIPROCESSOR) 6390 if ((cpus[i]->flags & CPUFLG_HATCHED) == 0) 6391 continue; 6392 #endif 6393 setpgt4m(&pm->pm_reg_ptps[i][vr], 6394 (VA2PA((void *)ptd) >> SRMMU_PPNPASHIFT) | 6395 SRMMU_TEPTD); 6396 } 6397 } 6398 6399 sp = &rp->rg_segmap[vs]; 6400 6401 owired = false; 6402 if ((pte = sp->sg_pte) == NULL) { 6403 /* definitely a new mapping */ 6404 int i; 6405 int mflag = PR_NOWAIT; 6406 6407 pte = pool_get(&L23_pool, mflag); 6408 if (pte == NULL) { 6409 if ((flags & PMAP_CANFAIL) != 0) { 6410 error = ENOMEM; 6411 goto out; 6412 } 6413 panic("pmap_enter: cannot allocate PTE table"); 6414 } 6415 6416 sp->sg_pte = pte; 6417 sp->sg_npte = 1; 6418 rp->rg_nsegmap++; 6419 for (i = 0; i < SRMMU_L3SIZE; i++) 6420 setpgt4m(&pte[i], SRMMU_TEINVALID); 6421 setpgt4m(&rp->rg_seg_ptps[vs], 6422 (VA2PA((void *)pte) >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 6423 } else { 6424 #ifdef DIAGNOSTIC 6425 if (sp->sg_npte <= 0) 6426 panic("pm %p: npte %d", pm, sp->sg_npte); 6427 #endif 6428 /* 6429 * Might be a change: fetch old pte 6430 */ 6431 tpte = pte[VA_SUN4M_VPG(va)]; 6432 6433 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) { 6434 6435 /* old mapping exists, and is of the same pa type */ 6436 if ((tpte & SRMMU_PPNMASK) == 6437 (pteproto & SRMMU_PPNMASK)) { 6438 /* just changing prot and/or wiring */ 6439 /* caller should call this directly: */ 6440 pmap_changeprot4m(pm, va, prot, flags); 6441 PMAP_UNLOCK(); 6442 splx(s); 6443 return (0); 6444 } 6445 /* 6446 * Switcheroo: changing pa for this va. 6447 * If old pa was managed, remove from pvlist. 6448 * If old page was cached, flush cache. 6449 */ 6450 #ifdef DEBUG 6451 if (pmapdebug & PDB_SWITCHMAP) 6452 printf("%s[%d]: pmap_enu: changing existing " 6453 "va 0x%x: pte 0x%x=>0x%x\n", 6454 curproc->p_comm, curproc->p_pid, 6455 (int)va, tpte, pteproto); 6456 #endif 6457 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 6458 struct vm_page *opg; 6459 if ((opg = pvhead4m(tpte)) != NULL) { 6460 VM_MDPAGE_PVHEAD(opg)->pv_flags |= 6461 MR4M(tpte); 6462 pv_unlink4m(opg, pm, va); 6463 } 6464 if (pm->pm_ctx && (tpte & SRMMU_PG_C)) 6465 cache_flush_page(va, pm->pm_ctxnum); 6466 } 6467 /* 6468 * We end up in this `change map' branch relatively 6469 * infrequently. 6470 * Invalidate the mapping now, so we can avoid the 6471 * de-map and update protocol when setting the new 6472 * PTE below. 6473 */ 6474 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 6475 SRMMU_TEINVALID, pm->pm_ctx != NULL, 6476 pm->pm_ctxnum, PMAP_CPUSET(pm)); 6477 pm->pm_stats.resident_count--; 6478 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va)); 6479 } else { 6480 /* adding new entry */ 6481 sp->sg_npte++; 6482 } 6483 } 6484 6485 if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) { 6486 if (--sp->sg_npte == 0) 6487 /* Sigh, undo pgt allocations */ 6488 pgt_lvl23_remove4m(pm, rp, sp, vr, vs); 6489 6490 if ((flags & PMAP_CANFAIL) != 0) 6491 goto out; 6492 panic("pmap_enter: cannot allocate PV entry"); 6493 } 6494 6495 /* 6496 * Update PTEs, flush TLB as necessary. 6497 */ 6498 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto); 6499 pm->pm_stats.resident_count++; 6500 if (owired) { 6501 pm->pm_stats.wired_count--; 6502 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 6503 } 6504 if (flags & PMAP_WIRED) { 6505 pm->pm_stats.wired_count++; 6506 sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va)); 6507 } 6508 6509 out: 6510 PMAP_UNLOCK(); 6511 splx(s); 6512 return (error); 6513 } 6514 6515 void 6516 pmap_kenter_pa4m(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 6517 { 6518 struct pmap *pm = pmap_kernel(); 6519 struct regmap *rp; 6520 struct segmap *sp; 6521 int pteproto, vr, vs; 6522 6523 /* Initialise pteproto with cache bit */ 6524 pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0; 6525 pteproto |= SRMMU_TEPTE | PPROT_S; 6526 pteproto |= PMAP_T2PTE_SRMMU(pa); 6527 pteproto |= (atop(pa & ~PMAP_TNC_SRMMU) << SRMMU_PPNSHIFT); 6528 pteproto |= pte_kprot4m(prot); 6529 6530 vr = VA_VREG(va); 6531 vs = VA_VSEG(va); 6532 rp = &pm->pm_regmap[vr]; 6533 sp = &rp->rg_segmap[vs]; 6534 6535 KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) != SRMMU_TEPTE); 6536 6537 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto); 6538 } 6539 6540 void 6541 pmap_kremove4m(vaddr_t va, vsize_t len) 6542 { 6543 struct pmap *pm = pmap_kernel(); 6544 struct regmap *rp; 6545 struct segmap *sp; 6546 vaddr_t endva, nva; 6547 int vr, vs; 6548 int tpte, perpage, npg, s; 6549 6550 /* 6551 * The kernel pmap doesn't need to be locked, but the demap lock 6552 * in updatepte() requires interrupt protection. 6553 */ 6554 s = splvm(); 6555 6556 endva = va + len; 6557 for (; va < endva; va = nva) { 6558 /* do one virtual segment at a time */ 6559 vr = VA_VREG(va); 6560 vs = VA_VSEG(va); 6561 nva = VSTOVA(vr, vs + 1); 6562 if (nva == 0 || nva > endva) { 6563 nva = endva; 6564 } 6565 6566 rp = &pm->pm_regmap[vr]; 6567 sp = &rp->rg_segmap[vs]; 6568 6569 /* decide how to flush the cache */ 6570 npg = (nva - va) >> PGSHIFT; 6571 if (npg > PMAP_SFL_THRESHOLD) { 6572 /* flush the whole segment */ 6573 perpage = 0; 6574 if (CACHEINFO.c_vactype != VAC_NONE) { 6575 cache_flush_segment(vr, vs, 0); 6576 } 6577 } else { 6578 /* 6579 * flush each page individually; 6580 * some never need flushing 6581 */ 6582 perpage = (CACHEINFO.c_vactype != VAC_NONE); 6583 } 6584 for (; va < nva; va += NBPG) { 6585 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6586 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) 6587 continue; 6588 6589 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 6590 /* if cacheable, flush page as needed */ 6591 if (perpage && (tpte & SRMMU_PG_C)) 6592 cache_flush_page(va, 0); 6593 } 6594 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 6595 SRMMU_TEINVALID, 1, 0, CPUSET_ALL); 6596 } 6597 } 6598 splx(s); 6599 } 6600 6601 /* 6602 * Change protection on a range of kernel addresses. 6603 */ 6604 void 6605 pmap_kprotect4m(vaddr_t va, vsize_t size, vm_prot_t prot) 6606 { 6607 struct pmap *pm = pmap_kernel(); 6608 int pte, newprot, s; 6609 struct regmap *rp; 6610 struct segmap *sp; 6611 6612 size = roundup(size,NBPG); 6613 newprot = pte_kprot4m(prot); 6614 6615 /* 6616 * The kernel pmap doesn't need to be locked, but the demap lock 6617 * in updatepte() requires interrupt protection. 6618 */ 6619 s = splvm(); 6620 6621 while (size > 0) { 6622 rp = &pm->pm_regmap[VA_VREG(va)]; 6623 sp = &rp->rg_segmap[VA_VSEG(va)]; 6624 pte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6625 6626 /* 6627 * Flush cache if page has been referenced to 6628 * avoid stale protection bits in the cache tags. 6629 */ 6630 if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) == 6631 (SRMMU_PG_C|PG_SUN4M_OBMEM)) 6632 cache_flush_page(va, 0); 6633 6634 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 6635 (pte & ~SRMMU_PROT_MASK) | newprot, 6636 1, pm->pm_ctxnum, PMAP_CPUSET(pm)); 6637 6638 va += NBPG; 6639 size -= NBPG; 6640 } 6641 splx(s); 6642 } 6643 #endif /* SUN4M || SUN4D */ 6644 6645 /* 6646 * Clear the wiring attribute for a map/virtual-address pair. 6647 */ 6648 /* ARGSUSED */ 6649 void 6650 pmap_unwire(struct pmap *pm, vaddr_t va) 6651 { 6652 int vr, vs, *ptep; 6653 struct regmap *rp; 6654 struct segmap *sp; 6655 bool owired; 6656 6657 vr = VA_VREG(va); 6658 vs = VA_VSEG(va); 6659 rp = &pm->pm_regmap[vr]; 6660 sp = &rp->rg_segmap[vs]; 6661 6662 owired = false; 6663 if (CPU_HAS_SUNMMU) { 6664 ptep = &sp->sg_pte[VA_VPG(va)]; 6665 owired = *ptep & PG_WIRED; 6666 *ptep &= ~PG_WIRED; 6667 } 6668 if (CPU_HAS_SRMMU) { 6669 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va)); 6670 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 6671 } 6672 if (!owired) { 6673 pmap_stats.ps_useless_changewire++; 6674 return; 6675 } 6676 6677 pm->pm_stats.wired_count--; 6678 #if defined(SUN4) || defined(SUN4C) 6679 if (CPU_HAS_SUNMMU && --sp->sg_nwired <= 0) { 6680 #ifdef DIAGNOSTIC 6681 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 6682 panic("pmap_unwire: pm %p, va %lx: nleft=%d, nwired=%d", 6683 pm, va, sp->sg_npte, sp->sg_nwired); 6684 #endif 6685 if (sp->sg_pmeg != seginval) 6686 mmu_pmeg_unlock(sp->sg_pmeg); 6687 } 6688 #endif /* SUN4 || SUN4C */ 6689 } 6690 6691 /* 6692 * Extract the physical page address associated 6693 * with the given map/virtual_address pair. 6694 * GRR, the vm code knows; we should not have to do this! 6695 */ 6696 6697 #if defined(SUN4) || defined(SUN4C) 6698 bool 6699 pmap_extract4_4c(struct pmap *pm, vaddr_t va, paddr_t *pap) 6700 { 6701 int vr, vs; 6702 struct regmap *rp; 6703 struct segmap *sp; 6704 int pte, *ptep; 6705 6706 vr = VA_VREG(va); 6707 vs = VA_VSEG(va); 6708 rp = &pm->pm_regmap[vr]; 6709 if (rp->rg_segmap == NULL) { 6710 #ifdef DEBUG 6711 if (pmapdebug & PDB_FOLLOW) 6712 printf("pmap_extract: invalid segment (%d)\n", vr); 6713 #endif 6714 return (false); 6715 } 6716 sp = &rp->rg_segmap[vs]; 6717 ptep = sp->sg_pte; 6718 if (ptep == NULL) { 6719 #ifdef DEBUG 6720 if (pmapdebug & PDB_FOLLOW) 6721 printf("pmap_extract: invalid segment\n"); 6722 #endif 6723 return (false); 6724 } 6725 pte = ptep[VA_VPG(va)]; 6726 6727 if ((pte & PG_V) == 0) { 6728 #ifdef DEBUG 6729 if (pmapdebug & PDB_FOLLOW) 6730 printf("pmap_extract: invalid pte\n"); 6731 #endif 6732 return (false); 6733 } 6734 pte &= PG_PFNUM; 6735 if (pap != NULL) 6736 *pap = (pte << PGSHIFT) | (va & PGOFSET); 6737 return (true); 6738 } 6739 #endif /* SUN4 || SUN4C */ 6740 6741 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_extract */ 6742 /* 6743 * Extract the physical page address associated 6744 * with the given map/virtual_address pair. 6745 * GRR, the vm code knows; we should not have to do this! 6746 */ 6747 bool 6748 pmap_extract4m(struct pmap *pm, vaddr_t va, paddr_t *pap) 6749 { 6750 struct regmap *rp; 6751 struct segmap *sp; 6752 int pte; 6753 int vr, vs, s, v = false; 6754 bool can_lock = lock_available; 6755 6756 vr = VA_VREG(va); 6757 vs = VA_VSEG(va); 6758 6759 /* 6760 * The kernel pmap doesn't need to be locked, but the demap lock 6761 * requires interrupt protection. 6762 */ 6763 s = splvm(); 6764 if (pm != pmap_kernel()) { 6765 PMAP_LOCK(); 6766 } 6767 6768 rp = &pm->pm_regmap[vr]; 6769 if (rp->rg_segmap == NULL) { 6770 #ifdef DEBUG 6771 if (pmapdebug & PDB_FOLLOW) 6772 printf("pmap_extract: no segmap\n"); 6773 #endif 6774 goto out; 6775 } 6776 6777 sp = &rp->rg_segmap[vs]; 6778 if (sp->sg_pte == NULL) { 6779 #ifdef DEBUG 6780 if (pmapdebug & PDB_FOLLOW) 6781 printf("pmap_extract: no ptes\n"); 6782 #endif 6783 goto out; 6784 } 6785 6786 pte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6787 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) { 6788 #ifdef DEBUG 6789 if (pmapdebug & PDB_FOLLOW) 6790 printf("pmap_extract: invalid pte of type %d\n", 6791 pte & SRMMU_TETYPE); 6792 #endif 6793 /* 6794 * We can read a spurious invalid pte if the system is in 6795 * the middle of the PTE update protocol. So, acquire the 6796 * demap lock and retry. 6797 */ 6798 if (__predict_true(can_lock)) 6799 mutex_spin_enter(&demap_lock); 6800 pte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6801 if (__predict_true(can_lock)) 6802 mutex_spin_exit(&demap_lock); 6803 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) 6804 goto out; 6805 } 6806 #ifdef DIAGNOSTIC 6807 if (pm != pmap_kernel() && sp->sg_npte <= 0) 6808 panic("pmap_extract: pm %p: npte = %d\n", pm, sp->sg_npte); 6809 #endif 6810 6811 if (pap != NULL) 6812 *pap = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT) | 6813 VA_OFF(va); 6814 6815 v = true; 6816 out: 6817 if (pm != pmap_kernel()) { 6818 PMAP_UNLOCK(); 6819 } 6820 splx(s); 6821 return (v); 6822 } 6823 #endif /* sun4m */ 6824 6825 int pmap_copy_disabled=0; 6826 6827 /* 6828 * Copy the range specified by src_addr/len 6829 * from the source map to the range dst_addr/len 6830 * in the destination map. 6831 * 6832 * This routine is only advisory and need not do anything. 6833 */ 6834 /* ARGSUSED */ 6835 void 6836 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, 6837 vaddr_t dst_addr, vsize_t len, vaddr_t src_addr) 6838 { 6839 #if notyet 6840 struct regmap *rp; 6841 struct segmap *sp; 6842 6843 if (pmap_copy_disabled) 6844 return; 6845 #ifdef DIAGNOSTIC 6846 if (VA_OFF(src_addr) != 0) 6847 printf("pmap_copy: addr not page aligned: 0x%lx\n", src_addr); 6848 if ((len & (NBPG-1)) != 0) 6849 printf("pmap_copy: length not page aligned: 0x%lx\n", len); 6850 #endif 6851 6852 if (src_pmap == NULL) 6853 return; 6854 6855 if (CPU_HAS_SRMMU) { 6856 int i, npg, pte; 6857 paddr_t pa; 6858 6859 npg = len >> PGSHIFT; 6860 for (i = 0; i < npg; i++) { 6861 if ((rp = src_pmap->pm_regmap) == NULL) 6862 continue; 6863 rp += VA_VREG(src_addr); 6864 6865 if ((sp = rp->rg_segmap) == NULL) 6866 continue; 6867 sp += VA_VSEG(src_addr); 6868 if (sp->sg_npte == 0) 6869 continue; 6870 6871 pte = sp->sg_pte[VA_SUN4M_VPG(src_addr)]; 6872 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) 6873 continue; 6874 6875 pa = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT); 6876 pmap_enter(dst_pmap, dst_addr, 6877 pa, 6878 /* XXX - need to copy VM_PROT_EXEC too */ 6879 (pte & PPROT_WRITE) 6880 ? (VM_PROT_WRITE | VM_PROT_READ) 6881 : VM_PROT_READ, 6882 0); 6883 src_addr += NBPG; 6884 dst_addr += NBPG; 6885 } 6886 pmap_update(dst_pmap); 6887 } 6888 #endif 6889 } 6890 6891 #if defined(SUN4) || defined(SUN4C) 6892 /* 6893 * Clear the modify bit for the given physical page. 6894 */ 6895 bool 6896 pmap_clear_modify4_4c(struct vm_page *pg) 6897 { 6898 bool rv; 6899 6900 (void) pv_syncflags4_4c(pg); 6901 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD; 6902 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD; 6903 return (rv); 6904 } 6905 6906 /* 6907 * Tell whether the given physical page has been modified. 6908 */ 6909 bool 6910 pmap_is_modified4_4c(struct vm_page *pg) 6911 { 6912 6913 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD || 6914 pv_syncflags4_4c(pg) & PV_MOD); 6915 } 6916 6917 /* 6918 * Clear the reference bit for the given physical page. 6919 */ 6920 bool 6921 pmap_clear_reference4_4c(struct vm_page *pg) 6922 { 6923 bool rv; 6924 6925 (void) pv_syncflags4_4c(pg); 6926 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF; 6927 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF; 6928 return (rv); 6929 } 6930 6931 /* 6932 * Tell whether the given physical page has been referenced. 6933 */ 6934 bool 6935 pmap_is_referenced4_4c(struct vm_page *pg) 6936 { 6937 6938 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF || 6939 pv_syncflags4_4c(pg) & PV_REF); 6940 } 6941 #endif /* SUN4 || SUN4C */ 6942 6943 #if defined(SUN4M) || defined(SUN4D) 6944 6945 /* 6946 * SRMMU versions of bit test/set routines 6947 * 6948 * Note that the 4m-specific routines should eventually service these 6949 * requests from their page tables, and the whole pvlist bit mess should 6950 * be dropped for the 4m (unless this causes a performance hit from 6951 * tracing down pagetables/regmap/segmaps). 6952 */ 6953 6954 /* 6955 * Clear the modify bit for the given physical page. 6956 */ 6957 bool 6958 pmap_clear_modify4m(struct vm_page *pg) 6959 { 6960 bool rv; 6961 6962 (void) pv_syncflags4m(pg); 6963 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M; 6964 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD4M; 6965 return (rv); 6966 } 6967 6968 /* 6969 * Tell whether the given physical page has been modified. 6970 */ 6971 bool 6972 pmap_is_modified4m(struct vm_page *pg) 6973 { 6974 6975 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M || 6976 pv_syncflags4m(pg) & PV_MOD4M); 6977 } 6978 6979 /* 6980 * Clear the reference bit for the given physical page. 6981 */ 6982 bool 6983 pmap_clear_reference4m(struct vm_page *pg) 6984 { 6985 bool rv; 6986 6987 (void) pv_syncflags4m(pg); 6988 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M; 6989 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF4M; 6990 return (rv); 6991 } 6992 6993 /* 6994 * Tell whether the given physical page has been referenced. 6995 */ 6996 bool 6997 pmap_is_referenced4m(struct vm_page *pg) 6998 { 6999 7000 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M || 7001 pv_syncflags4m(pg) & PV_REF4M); 7002 } 7003 #endif /* SUN4M || SUN4D */ 7004 7005 /* 7006 * Fill the given MI physical page with zero bytes. 7007 * 7008 * We avoid stomping on the cache. 7009 * XXX might be faster to use destination's context and allow cache to fill? 7010 */ 7011 7012 #if defined(SUN4) || defined(SUN4C) 7013 7014 void 7015 pmap_zero_page4_4c(paddr_t pa) 7016 { 7017 struct vm_page *pg; 7018 void *va; 7019 int pte; 7020 7021 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 7022 /* 7023 * The following might not be necessary since the page 7024 * is being cleared because it is about to be allocated, 7025 * i.e., is in use by no one. 7026 */ 7027 pv_flushcache4_4c(pg); 7028 } 7029 pte = PG_V | PG_S | PG_W | PG_NC | (atop(pa) & PG_PFNUM); 7030 7031 va = cpuinfo.vpage[0]; 7032 setpte4(va, pte); 7033 qzero(va, NBPG); 7034 setpte4(va, 0); 7035 } 7036 7037 /* 7038 * Copy the given MI physical source page to its destination. 7039 * 7040 * We avoid stomping on the cache as above (with same `XXX' note). 7041 * We must first flush any write-back cache for the source page. 7042 * We go ahead and stomp on the kernel's virtual cache for the 7043 * source page, since the cache can read memory MUCH faster than 7044 * the processor. 7045 */ 7046 void 7047 pmap_copy_page4_4c(paddr_t src, paddr_t dst) 7048 { 7049 struct vm_page *pg; 7050 char *sva, *dva; 7051 int spte, dpte; 7052 7053 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) { 7054 if (CACHEINFO.c_vactype == VAC_WRITEBACK) 7055 pv_flushcache4_4c(pg); 7056 } 7057 spte = PG_V | PG_S | (atop(src) & PG_PFNUM); 7058 7059 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) { 7060 /* similar `might not be necessary' comment applies */ 7061 if (CACHEINFO.c_vactype != VAC_NONE) 7062 pv_flushcache4_4c(pg); 7063 } 7064 dpte = PG_V | PG_S | PG_W | PG_NC | (atop(dst) & PG_PFNUM); 7065 7066 sva = cpuinfo.vpage[0]; 7067 dva = cpuinfo.vpage[1]; 7068 setpte4(sva, spte); 7069 setpte4(dva, dpte); 7070 qcopy(sva, dva, NBPG); /* loads cache, so we must ... */ 7071 cache_flush_page((vaddr_t)sva, getcontext4()); 7072 setpte4(sva, 0); 7073 setpte4(dva, 0); 7074 } 7075 #endif /* SUN4 || SUN4C */ 7076 7077 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of copy/zero routines */ 7078 /* 7079 * Fill the given MI physical page with zero bytes. 7080 * 7081 * We avoid stomping on the cache. 7082 * XXX might be faster to use destination's context and allow cache to fill? 7083 */ 7084 void 7085 pmap_zero_page4m(paddr_t pa) 7086 { 7087 struct vm_page *pg; 7088 void *va; 7089 int pte; 7090 7091 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 7092 /* 7093 * The following VAC flush might not be necessary since the 7094 * page is being cleared because it is about to be allocated, 7095 * i.e., is in use by no one. 7096 * In the case of a physical cache, a flush (or just an 7097 * invalidate, if possible) is usually necessary when using 7098 * uncached access to clear it. 7099 */ 7100 if (CACHEINFO.c_vactype != VAC_NONE) 7101 pv_flushcache4m(pg); 7102 else 7103 pcache_flush_page(pa, 1); 7104 } 7105 pte = SRMMU_TEPTE | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT); 7106 if (cpuinfo.flags & CPUFLG_CACHE_MANDATORY) 7107 pte |= SRMMU_PG_C; 7108 7109 va = cpuinfo.vpage[0]; 7110 setpgt4m(cpuinfo.vpage_pte[0], pte); 7111 qzero(va, NBPG); 7112 /* 7113 * Remove temporary mapping (which is kernel-only, so the 7114 * context used for TLB flushing does not matter) 7115 */ 7116 sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3); 7117 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID); 7118 } 7119 7120 /* 7121 * Viking/MXCC specific version of pmap_zero_page 7122 */ 7123 void 7124 pmap_zero_page_viking_mxcc(paddr_t pa) 7125 { 7126 u_int offset; 7127 u_int stream_data_addr = MXCC_STREAM_DATA; 7128 uint64_t v = (uint64_t)pa; 7129 7130 /* Load MXCC stream data register with 0 (bottom 32 bytes only) */ 7131 stda(stream_data_addr+0, ASI_CONTROL, 0); 7132 stda(stream_data_addr+8, ASI_CONTROL, 0); 7133 stda(stream_data_addr+16, ASI_CONTROL, 0); 7134 stda(stream_data_addr+24, ASI_CONTROL, 0); 7135 7136 /* Then write the stream data register to each block in the page */ 7137 v |= MXCC_STREAM_C; 7138 for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) { 7139 stda(MXCC_STREAM_DST, ASI_CONTROL, v | offset); 7140 } 7141 } 7142 7143 /* 7144 * HyperSPARC/RT625 specific version of pmap_zero_page 7145 */ 7146 void 7147 pmap_zero_page_hypersparc(paddr_t pa) 7148 { 7149 struct vm_page *pg; 7150 void *va; 7151 int pte; 7152 int offset; 7153 7154 /* 7155 * We still have to map the page, since ASI_BLOCKFILL 7156 * takes virtual addresses. This also means we have to 7157 * consider cache aliasing; therefore we still need 7158 * to flush the cache here. All we gain is the speed-up 7159 * in zero-fill loop itself.. 7160 */ 7161 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 7162 /* 7163 * The following might not be necessary since the page 7164 * is being cleared because it is about to be allocated, 7165 * i.e., is in use by no one. 7166 */ 7167 if (CACHEINFO.c_vactype != VAC_NONE) 7168 pv_flushcache4m(pg); 7169 } 7170 pte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT); 7171 7172 va = cpuinfo.vpage[0]; 7173 setpgt4m(cpuinfo.vpage_pte[0], pte); 7174 for (offset = 0; offset < NBPG; offset += 32) { 7175 sta((char *)va + offset, ASI_BLOCKFILL, 0); 7176 } 7177 /* Remove temporary mapping */ 7178 sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3); 7179 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID); 7180 } 7181 7182 /* 7183 * Copy the given MI physical source page to its destination. 7184 * 7185 * We avoid stomping on the cache as above (with same `XXX' note). 7186 * We must first flush any write-back cache for the source page. 7187 * We go ahead and stomp on the kernel's virtual cache for the 7188 * source page, since the cache can read memory MUCH faster than 7189 * the processor. 7190 */ 7191 void 7192 pmap_copy_page4m(paddr_t src, paddr_t dst) 7193 { 7194 struct vm_page *pg; 7195 void *sva, *dva; 7196 int spte, dpte; 7197 7198 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) { 7199 if (CACHEINFO.c_vactype == VAC_WRITEBACK) 7200 pv_flushcache4m(pg); 7201 } 7202 7203 spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX | 7204 (src >> SRMMU_PPNPASHIFT); 7205 7206 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) { 7207 /* similar `might not be necessary' comment applies */ 7208 if (CACHEINFO.c_vactype != VAC_NONE) 7209 pv_flushcache4m(pg); 7210 else 7211 pcache_flush_page(dst, 1); 7212 } 7213 7214 dpte = SRMMU_TEPTE | PPROT_N_RWX | (dst >> SRMMU_PPNPASHIFT); 7215 if (cpuinfo.flags & CPUFLG_CACHE_MANDATORY) 7216 dpte |= SRMMU_PG_C; 7217 7218 sva = cpuinfo.vpage[0]; 7219 dva = cpuinfo.vpage[1]; 7220 setpgt4m(cpuinfo.vpage_pte[0], spte); 7221 setpgt4m(cpuinfo.vpage_pte[1], dpte); 7222 qcopy(sva, dva, NBPG); /* loads cache, so we must ... */ 7223 cpuinfo.sp_vcache_flush_page((vaddr_t)sva, getcontext4m()); 7224 sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3); 7225 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID); 7226 sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3); 7227 setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID); 7228 } 7229 7230 /* 7231 * Viking/MXCC specific version of pmap_copy_page 7232 */ 7233 void 7234 pmap_copy_page_viking_mxcc(paddr_t src, paddr_t dst) 7235 { 7236 u_int offset; 7237 uint64_t v1 = (uint64_t)src; 7238 uint64_t v2 = (uint64_t)dst; 7239 7240 /* Enable cache-coherency */ 7241 v1 |= MXCC_STREAM_C; 7242 v2 |= MXCC_STREAM_C; 7243 7244 /* Copy through stream data register */ 7245 for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) { 7246 stda(MXCC_STREAM_SRC, ASI_CONTROL, v1 | offset); 7247 stda(MXCC_STREAM_DST, ASI_CONTROL, v2 | offset); 7248 } 7249 } 7250 7251 /* 7252 * HyperSPARC/RT625 specific version of pmap_copy_page 7253 */ 7254 void 7255 pmap_copy_page_hypersparc(paddr_t src, paddr_t dst) 7256 { 7257 struct vm_page *pg; 7258 void *sva, *dva; 7259 int spte, dpte; 7260 int offset; 7261 7262 /* 7263 * We still have to map the pages, since ASI_BLOCKCOPY 7264 * takes virtual addresses. This also means we have to 7265 * consider cache aliasing; therefore we still need 7266 * to flush the cache here. All we gain is the speed-up 7267 * in copy loop itself.. 7268 */ 7269 7270 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) { 7271 if (CACHEINFO.c_vactype == VAC_WRITEBACK) 7272 pv_flushcache4m(pg); 7273 } 7274 7275 spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX | 7276 (src >> SRMMU_PPNPASHIFT); 7277 7278 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) { 7279 /* similar `might not be necessary' comment applies */ 7280 if (CACHEINFO.c_vactype != VAC_NONE) 7281 pv_flushcache4m(pg); 7282 } 7283 7284 dpte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX | 7285 (dst >> SRMMU_PPNPASHIFT); 7286 7287 sva = cpuinfo.vpage[0]; 7288 dva = cpuinfo.vpage[1]; 7289 setpgt4m(cpuinfo.vpage_pte[0], spte); 7290 setpgt4m(cpuinfo.vpage_pte[1], dpte); 7291 7292 for (offset = 0; offset < NBPG; offset += 32) { 7293 sta((char *)dva + offset, ASI_BLOCKCOPY, (char *)sva + offset); 7294 } 7295 7296 sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3); 7297 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID); 7298 sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3); 7299 setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID); 7300 } 7301 #endif /* SUN4M || SUN4D */ 7302 7303 /* 7304 * Turn off cache for a given (va, number of pages). 7305 * 7306 * We just assert PG_NC for each PTE; the addresses must reside 7307 * in locked kernel space. A cache flush is also done. 7308 */ 7309 void 7310 kvm_uncache(char *va, int npages) 7311 { 7312 struct vm_page *pg; 7313 int pte; 7314 7315 if (CPU_HAS_SRMMU) { 7316 #if defined(SUN4M) || defined(SUN4D) 7317 for (; --npages >= 0; va = (char *)va + NBPG) { 7318 pte = getpte4m((vaddr_t) va); 7319 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) 7320 panic("kvm_uncache: table entry not pte"); 7321 7322 if ((pte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 7323 if ((pg = pvhead4m(pte)) != NULL) { 7324 pv_uncache(pg); 7325 return; 7326 } 7327 cache_flush_page((vaddr_t)va, 0); 7328 } 7329 7330 pte &= ~SRMMU_PG_C; 7331 setpte4m((vaddr_t)va, pte); 7332 } 7333 #endif 7334 } else { 7335 #if defined(SUN4) || defined(SUN4C) 7336 for (; --npages >= 0; va += NBPG) { 7337 pte = getpte4(va); 7338 if ((pte & PG_V) == 0) 7339 panic("kvm_uncache !pg_v"); 7340 7341 if ((pte & PG_TYPE) == PG_OBMEM) { 7342 if ((pg = pvhead4_4c(pte)) != NULL) { 7343 pv_uncache(pg); 7344 return; 7345 } 7346 cache_flush_page((vaddr_t)va, 0); 7347 } 7348 pte |= PG_NC; 7349 setpte4(va, pte); 7350 } 7351 #endif 7352 } 7353 } 7354 7355 #if 0 /* not used */ 7356 /* 7357 * Turn on IO cache for a given (va, number of pages). 7358 * 7359 * We just assert PG_NC for each PTE; the addresses must reside 7360 * in locked kernel space. A cache flush is also done. 7361 */ 7362 void 7363 kvm_iocache(char *va, int npages) 7364 { 7365 7366 #if defined(SUN4M) 7367 if (CPU_ISSUN4M) /* %%%: Implement! */ 7368 panic("kvm_iocache: 4m iocache not implemented"); 7369 #endif 7370 #if defined(SUN4D) 7371 if (CPU_ISSUN4D) /* %%%: Implement! */ 7372 panic("kvm_iocache: 4d iocache not implemented"); 7373 #endif 7374 #if defined(SUN4) || defined(SUN4C) 7375 for (; --npages >= 0; va += NBPG) { 7376 int pte = getpte4(va); 7377 if ((pte & PG_V) == 0) 7378 panic("kvm_iocache !pg_v"); 7379 pte |= PG_IOC; 7380 setpte4(va, pte); 7381 } 7382 #endif 7383 } 7384 #endif 7385 7386 /* 7387 * Find first virtual address >= *va that is 7388 * least likely to cause cache aliases. 7389 * (This will just seg-align mappings.) 7390 */ 7391 void 7392 pmap_prefer(vaddr_t foff, vaddr_t *vap) 7393 { 7394 vaddr_t va = *vap; 7395 long d, m; 7396 7397 if (VA_INHOLE(va)) 7398 va = MMU_HOLE_END; 7399 7400 m = CACHE_ALIAS_DIST; 7401 if (m == 0) /* m=0 => no cache aliasing */ 7402 return; 7403 7404 d = foff - va; 7405 d &= (m - 1); 7406 *vap = va + d; 7407 } 7408 7409 void 7410 pmap_redzone(void) 7411 { 7412 7413 pmap_remove(pmap_kernel(), KERNBASE, KERNBASE+NBPG); 7414 } 7415 7416 /* 7417 * Activate the address space for the specified process. If the 7418 * process is the current process, load the new MMU context. 7419 */ 7420 void 7421 pmap_activate(struct lwp *l) 7422 { 7423 pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap; 7424 int s; 7425 7426 /* 7427 * This is essentially the same thing that happens in cpu_switch() 7428 * when the newly selected process is about to run, except that we 7429 * have to make sure to clean the register windows before we set 7430 * the new context. 7431 */ 7432 7433 s = splvm(); 7434 if (l == curlwp) { 7435 write_user_windows(); 7436 if (pm->pm_ctx == NULL) { 7437 ctx_alloc(pm); /* performs setcontext() */ 7438 } else { 7439 /* Do any cache flush needed on context switch */ 7440 (*cpuinfo.pure_vcache_flush)(); 7441 setcontext(pm->pm_ctxnum); 7442 } 7443 #if defined(MULTIPROCESSOR) 7444 if (pm != pmap_kernel()) 7445 PMAP_SET_CPUSET(pm, &cpuinfo); 7446 #endif 7447 } 7448 splx(s); 7449 } 7450 7451 /* 7452 * Deactivate the address space of the specified process. 7453 */ 7454 void 7455 pmap_deactivate(struct lwp *l) 7456 { 7457 #if defined(MULTIPROCESSOR) 7458 pmap_t pm; 7459 struct proc *p; 7460 7461 p = l->l_proc; 7462 if (p->p_vmspace && 7463 (pm = p->p_vmspace->vm_map.pmap) != pmap_kernel()) { 7464 #if defined(SUN4M) || defined(SUN4D) 7465 if (pm->pm_ctx && CPU_HAS_SRMMU) 7466 sp_tlb_flush(0, pm->pm_ctxnum, ASI_SRMMUFP_L0); 7467 #endif 7468 7469 /* we no longer need broadcast tlb flushes for this pmap. */ 7470 PMAP_CLR_CPUSET(pm, &cpuinfo); 7471 } 7472 #endif 7473 } 7474 7475 #ifdef DEBUG 7476 /* 7477 * Check consistency of a pmap (time consuming!). 7478 */ 7479 void 7480 pm_check(char *s, struct pmap *pm) 7481 { 7482 7483 if (pm == pmap_kernel()) 7484 pm_check_k(s, pm); 7485 else 7486 pm_check_u(s, pm); 7487 } 7488 7489 void 7490 pm_check_u(char *s, struct pmap *pm) 7491 { 7492 struct regmap *rp; 7493 struct segmap *sp; 7494 int cpu, n, vs, vr, j, m, *pte; 7495 7496 cpu = cpuinfo.ci_cpuid; 7497 7498 if (pm->pm_regmap == NULL) 7499 panic("%s: CPU %d: CHK(pmap %p): no region mapping", 7500 s, cpu, pm); 7501 7502 #if defined(SUN4M) || defined(SUN4D) 7503 if (CPU_HAS_SRMMU && 7504 (pm->pm_reg_ptps[cpu] == NULL || 7505 pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu]))) 7506 panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: " 7507 "tblva=%p, tblpa=0x%x", 7508 s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]); 7509 7510 if (CPU_HAS_SRMMU && pm->pm_ctx != NULL && 7511 (cpuinfo.ctx_tbl[pm->pm_ctxnum] != ((VA2PA((void *)pm->pm_reg_ptps[cpu]) 7512 >> SRMMU_PPNPASHIFT) | 7513 SRMMU_TEPTD))) 7514 panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed " 7515 "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], pm->pm_ctxnum); 7516 #endif 7517 7518 for (vr = 0; vr < NUREG; vr++) { 7519 rp = &pm->pm_regmap[vr]; 7520 if (rp->rg_nsegmap == 0) 7521 continue; 7522 if (rp->rg_segmap == NULL) 7523 panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL", 7524 s, cpu, vr, rp->rg_nsegmap); 7525 #if defined(SUN4M) || defined(SUN4D) 7526 if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL) 7527 panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table", 7528 s, cpu, vr, rp->rg_nsegmap); 7529 if (CPU_HAS_SRMMU && 7530 pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >> 7531 SRMMU_PPNPASHIFT) | SRMMU_TEPTD)) 7532 panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed", 7533 s, cpu, vr); 7534 #endif 7535 if ((unsigned int)rp < KERNBASE) 7536 panic("%s: CPU %d: rp=%p", s, cpu, rp); 7537 n = 0; 7538 for (vs = 0; vs < NSEGRG; vs++) { 7539 sp = &rp->rg_segmap[vs]; 7540 if ((unsigned int)sp < KERNBASE) 7541 panic("%s: CPU %d: sp=%p", s, cpu, sp); 7542 if (sp->sg_npte != 0) { 7543 n++; 7544 if (sp->sg_pte == NULL) 7545 panic("%s: CPU %d: CHK(vr %d, vs %d): npte=%d, " 7546 "pte=NULL", s, cpu, vr, vs, sp->sg_npte); 7547 #if defined(SUN4M) || defined(SUN4D) 7548 if (CPU_HAS_SRMMU && 7549 rp->rg_seg_ptps[vs] != 7550 ((VA2PA((void *)sp->sg_pte) 7551 >> SRMMU_PPNPASHIFT) | 7552 SRMMU_TEPTD)) 7553 panic("%s: CPU %d: CHK(vr %d, vs %d): SRMMU page " 7554 "table not installed correctly", 7555 s, cpu, vr, vs); 7556 #endif 7557 pte=sp->sg_pte; 7558 m = 0; 7559 for (j=0; j<NPTESG; j++,pte++) 7560 if ((CPU_HAS_SRMMU 7561 ?((*pte & SRMMU_TETYPE) == SRMMU_TEPTE) 7562 :(*pte & PG_V))) 7563 m++; 7564 if (m != sp->sg_npte) 7565 printf("%s: CPU %d: user CHK(vr %d, vs %d): " 7566 "npte(%d) != # valid(%d)\n", 7567 s, cpu, vr, vs, sp->sg_npte, m); 7568 } 7569 } 7570 if (n != rp->rg_nsegmap) 7571 panic("%s: CPU %d: CHK(vr %d): inconsistent " 7572 "# of pte's: %d, should be %d", 7573 s, cpu, vr, rp->rg_nsegmap, n); 7574 } 7575 return; 7576 } 7577 7578 /* Note: not as extensive as pm_check_u. */ 7579 void 7580 pm_check_k(char *s, struct pmap *pm) 7581 { 7582 struct regmap *rp; 7583 int cpu, vr, vs, n; 7584 7585 cpu = cpu_number(); 7586 7587 if (pm->pm_regmap == NULL) 7588 panic("%s: CHK(pmap %p): no region mapping", s, pm); 7589 7590 #if defined(SUN4M) || defined(SUN4D) 7591 if (CPU_HAS_SRMMU && 7592 (pm->pm_reg_ptps[cpu] == NULL || 7593 pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu]))) 7594 panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: tblva=%p, tblpa=0x%x", 7595 s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]); 7596 7597 if (CPU_HAS_SRMMU && 7598 (cpuinfo.ctx_tbl[0] != ((VA2PA((void *)pm->pm_reg_ptps[cpu]) >> 7599 SRMMU_PPNPASHIFT) | SRMMU_TEPTD))) 7600 panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed " 7601 "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], 0); 7602 #endif 7603 for (vr = NUREG; vr < NUREG+NKREG; vr++) { 7604 rp = &pm->pm_regmap[vr]; 7605 if (rp->rg_segmap == NULL) 7606 panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL", 7607 s, cpu, vr, rp->rg_nsegmap); 7608 if (rp->rg_nsegmap == 0) 7609 continue; 7610 #if defined(SUN4M) || defined(SUN4D) 7611 if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL) 7612 panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table", 7613 s, cpu, vr, rp->rg_nsegmap); 7614 7615 if (CPU_HAS_SRMMU && vr != NUREG /* 1st kseg is per CPU */ && 7616 pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >> 7617 SRMMU_PPNPASHIFT) | SRMMU_TEPTD)) 7618 panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed", 7619 s, cpu, vr); 7620 #endif 7621 if (CPU_HAS_SRMMU) { 7622 n = NSEGRG; 7623 } else { 7624 for (n = 0, vs = 0; vs < NSEGRG; vs++) { 7625 if (rp->rg_segmap[vs].sg_npte) 7626 n++; 7627 } 7628 } 7629 if (n != rp->rg_nsegmap) 7630 printf("%s: CPU %d: kernel CHK(vr %d): inconsistent " 7631 "# of pte's: %d, should be %d\n", 7632 s, cpu, vr, rp->rg_nsegmap, n); 7633 } 7634 return; 7635 } 7636 #endif 7637 7638 /* 7639 * Return the number of disk blocks that pmap_dumpmmu() will dump. 7640 */ 7641 int 7642 pmap_dumpsize(void) 7643 { 7644 int sz; 7645 7646 sz = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)); 7647 sz += npmemarr * sizeof(phys_ram_seg_t); 7648 sz += sizeof(kernel_segmap_store); 7649 7650 if (CPU_HAS_SUNMMU) 7651 /* For each pmeg in the MMU, we'll write NPTESG PTEs. */ 7652 sz += (seginval + 1) * NPTESG * sizeof(int); 7653 7654 return btodb(sz + DEV_BSIZE - 1); 7655 } 7656 7657 /* 7658 * Write the core dump headers and MD data to the dump device. 7659 * We dump the following items: 7660 * 7661 * kcore_seg_t MI header defined in <sys/kcore.h>) 7662 * cpu_kcore_hdr_t MD header defined in <machine/kcore.h>) 7663 * phys_ram_seg_t[npmemarr] physical memory segments 7664 * segmap_t[NKREG*NSEGRG] the kernel's segment map 7665 * the MMU pmegs on sun4/sun4c 7666 */ 7667 int 7668 pmap_dumpmmu(int (*dump)(dev_t, daddr_t, void *, size_t), 7669 daddr_t blkno) 7670 { 7671 kcore_seg_t *ksegp; 7672 cpu_kcore_hdr_t *kcpup; 7673 phys_ram_seg_t memseg; 7674 int error = 0; 7675 int i, memsegoffset, segmapoffset, pmegoffset; 7676 int buffer[dbtob(1) / sizeof(int)]; 7677 int *bp, *ep; 7678 #if defined(SUN4C) || defined(SUN4) 7679 int pmeg; 7680 #endif 7681 7682 #define EXPEDITE(p,n) do { \ 7683 int *sp = (int *)(p); \ 7684 int sz = (n); \ 7685 while (sz > 0) { \ 7686 *bp++ = *sp++; \ 7687 if (bp >= ep) { \ 7688 error = (*dump)(dumpdev, blkno, \ 7689 (void *)buffer, dbtob(1)); \ 7690 if (error != 0) \ 7691 return (error); \ 7692 ++blkno; \ 7693 bp = buffer; \ 7694 } \ 7695 sz -= 4; \ 7696 } \ 7697 } while (0) 7698 7699 setcontext(0); 7700 7701 /* Setup bookkeeping pointers */ 7702 bp = buffer; 7703 ep = &buffer[sizeof(buffer) / sizeof(buffer[0])]; 7704 7705 /* Fill in MI segment header */ 7706 ksegp = (kcore_seg_t *)bp; 7707 CORE_SETMAGIC(*ksegp, KCORE_MAGIC, MID_MACHINE, CORE_CPU); 7708 ksegp->c_size = dbtob(pmap_dumpsize()) - ALIGN(sizeof(kcore_seg_t)); 7709 7710 /* Fill in MD segment header (interpreted by MD part of libkvm) */ 7711 kcpup = (cpu_kcore_hdr_t *)((int)bp + ALIGN(sizeof(kcore_seg_t))); 7712 kcpup->cputype = cputyp; 7713 kcpup->kernbase = KERNBASE; 7714 kcpup->nmemseg = npmemarr; 7715 kcpup->memsegoffset = memsegoffset = ALIGN(sizeof(cpu_kcore_hdr_t)); 7716 kcpup->nsegmap = NKREG*NSEGRG; 7717 kcpup->segmapoffset = segmapoffset = 7718 memsegoffset + npmemarr * sizeof(phys_ram_seg_t); 7719 7720 kcpup->npmeg = (CPU_HAS_SUNMMU) ? seginval + 1 : 0; 7721 kcpup->pmegoffset = pmegoffset = 7722 segmapoffset + kcpup->nsegmap * sizeof(struct segmap); 7723 7724 /* Note: we have assumed everything fits in buffer[] so far... */ 7725 bp = (int *)((int)kcpup + ALIGN(sizeof(cpu_kcore_hdr_t))); 7726 7727 #if 0 7728 /* Align storage for upcoming quad-aligned segment array */ 7729 while (bp != (int *)ALIGN(bp)) { 7730 int dummy = 0; 7731 EXPEDITE(&dummy, 4); 7732 } 7733 #endif 7734 7735 for (i = 0; i < npmemarr; i++) { 7736 memseg.start = pmemarr[i].addr; 7737 memseg.size = pmemarr[i].len; 7738 EXPEDITE((void *)&memseg, sizeof(phys_ram_seg_t)); 7739 } 7740 7741 EXPEDITE(&kernel_segmap_store, sizeof(kernel_segmap_store)); 7742 7743 if (CPU_HAS_SRMMU) 7744 goto out; 7745 7746 #if defined(SUN4C) || defined(SUN4) 7747 /* 7748 * dump page table entries 7749 * 7750 * We dump each pmeg in order (by segment number). Since the MMU 7751 * automatically maps the given virtual segment to a pmeg we must 7752 * iterate over the segments by incrementing an unused segment slot 7753 * in the MMU. This fixed segment number is used in the virtual 7754 * address argument to getpte(). 7755 */ 7756 7757 /* 7758 * Go through the pmegs and dump each one. 7759 */ 7760 for (pmeg = 0; pmeg <= seginval; ++pmeg) { 7761 int va = 0; 7762 7763 setsegmap(va, pmeg); 7764 i = NPTESG; 7765 do { 7766 int pte = getpte4(va); 7767 EXPEDITE(&pte, sizeof(pte)); 7768 va += NBPG; 7769 } while (--i > 0); 7770 } 7771 setsegmap(0, seginval); 7772 #endif 7773 7774 out: 7775 if (bp != buffer) 7776 error = (*dump)(dumpdev, blkno++, (void *)buffer, dbtob(1)); 7777 7778 return (error); 7779 } 7780 7781 /* 7782 * Helper function for debuggers. 7783 */ 7784 void 7785 pmap_writetext(unsigned char *dst, int ch) 7786 { 7787 int s, pte0, pte, ctx; 7788 vaddr_t va; 7789 7790 s = splvm(); 7791 va = (unsigned long)dst & (~PGOFSET); 7792 cache_flush(dst, 1); 7793 7794 ctx = getcontext(); 7795 setcontext(0); 7796 7797 #if defined(SUN4M) || defined(SUN4D) 7798 if (CPU_HAS_SRMMU) { 7799 pte0 = getpte4m(va); 7800 if ((pte0 & SRMMU_TETYPE) != SRMMU_TEPTE) { 7801 splx(s); 7802 return; 7803 } 7804 pte = pte0 | PPROT_WRITE; 7805 setpte4m(va, pte); 7806 *dst = (unsigned char)ch; 7807 setpte4m(va, pte0); 7808 7809 } 7810 #endif 7811 #if defined(SUN4) || defined(SUN4C) 7812 if (CPU_ISSUN4C || CPU_ISSUN4) { 7813 pte0 = getpte4(va); 7814 if ((pte0 & PG_V) == 0) { 7815 splx(s); 7816 return; 7817 } 7818 pte = pte0 | PG_W; 7819 setpte4(va, pte); 7820 *dst = (unsigned char)ch; 7821 setpte4(va, pte0); 7822 } 7823 #endif 7824 cache_flush(dst, 1); 7825 setcontext(ctx); 7826 splx(s); 7827 } 7828 7829 #ifdef EXTREME_DEBUG 7830 7831 void debug_pagetables(void); 7832 void print_fe_map(void); 7833 7834 static void test_region(int, int, int); 7835 7836 7837 void 7838 debug_pagetables(void) 7839 { 7840 struct promvec *promvec = romp; 7841 int *regtbl; 7842 int te; 7843 int i; 7844 7845 printf("\nncontext=%d. ", ncontext); 7846 printf("Context table is at va %p. Level 0 PTP: 0x%x\n", 7847 cpuinfo.ctx_tbl, cpuinfo.ctx_tbl[0]); 7848 printf("Context 0 region table is at va %p, pa 0x%x. Contents:\n", 7849 pmap_kernel()->pm_reg_ptps[0], pmap_kernel()->pm_reg_ptps_pa[0]); 7850 7851 regtbl = pmap_kernel()->pm_reg_ptps[0]; 7852 7853 printf("PROM vector is at %p\n", promvec); 7854 printf("PROM reboot routine is at %p\n", promvec->pv_reboot); 7855 printf("PROM abort routine is at %p\n", promvec->pv_abort); 7856 printf("PROM halt routine is at %p\n", promvec->pv_halt); 7857 7858 printf("Testing region 0xfe: "); 7859 test_region(0xfe,0,16*1024*1024); 7860 printf("Testing region 0xff: "); 7861 test_region(0xff,0,16*1024*1024); 7862 printf("Testing kernel region 0x%x: ", VA_VREG(KERNBASE)); 7863 test_region(VA_VREG(KERNBASE), 4096, avail_start); 7864 cngetc(); 7865 7866 for (i = 0; i < SRMMU_L1SIZE; i++) { 7867 te = regtbl[i]; 7868 if ((te & SRMMU_TETYPE) == SRMMU_TEINVALID) 7869 continue; 7870 printf("Region 0x%x: PTE=0x%x <%s> L2PA=0x%x kernL2VA=%p\n", 7871 i, te, ((te & SRMMU_TETYPE) == SRMMU_TEPTE ? "pte" : 7872 ((te & SRMMU_TETYPE) == SRMMU_TEPTD ? "ptd" : 7873 ((te & SRMMU_TETYPE) == SRMMU_TEINVALID ? 7874 "invalid" : "reserved"))), 7875 (te & ~0x3) << SRMMU_PPNPASHIFT, 7876 pmap_kernel()->pm_regmap[i].rg_seg_ptps); 7877 } 7878 printf("Press q to halt...\n"); 7879 if (cngetc()=='q') 7880 callrom(); 7881 } 7882 7883 static u_int 7884 VA2PAsw(int ctx, void *addr, int *pte) 7885 { 7886 int *curtbl; 7887 int curpte; 7888 7889 #ifdef EXTREME_EXTREME_DEBUG 7890 printf("Looking up addr 0x%x in context 0x%x\n",addr,ctx); 7891 #endif 7892 /* L0 */ 7893 *pte = curpte = cpuinfo.ctx_tbl[ctx]; 7894 #ifdef EXTREME_EXTREME_DEBUG 7895 printf("Got L0 pte 0x%x\n",pte); 7896 #endif 7897 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) { 7898 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 7899 ((u_int)addr & 0xffffffff)); 7900 } 7901 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) { 7902 printf("Bad context table entry 0x%x for context 0x%x\n", 7903 curpte, ctx); 7904 return 0; 7905 } 7906 /* L1 */ 7907 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */ 7908 *pte = curpte = curtbl[VA_VREG(addr)]; 7909 #ifdef EXTREME_EXTREME_DEBUG 7910 printf("L1 table at 0x%x.\nGot L1 pte 0x%x\n",curtbl,curpte); 7911 #endif 7912 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) 7913 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 7914 ((u_int)addr & 0xffffff)); 7915 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) { 7916 printf("Bad region table entry 0x%x for region 0x%x\n", 7917 curpte, VA_VREG(addr)); 7918 return 0; 7919 } 7920 /* L2 */ 7921 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */ 7922 *pte = curpte = curtbl[VA_VSEG(addr)]; 7923 #ifdef EXTREME_EXTREME_DEBUG 7924 printf("L2 table at 0x%x.\nGot L2 pte 0x%x\n",curtbl,curpte); 7925 #endif 7926 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) 7927 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 7928 ((u_int)addr & 0x3ffff)); 7929 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) { 7930 printf("Bad segment table entry 0x%x for reg 0x%x, seg 0x%x\n", 7931 curpte, VA_VREG(addr), VA_VSEG(addr)); 7932 return 0; 7933 } 7934 /* L3 */ 7935 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */ 7936 *pte = curpte = curtbl[VA_VPG(addr)]; 7937 #ifdef EXTREME_EXTREME_DEBUG 7938 printf("L3 table at %p.\nGot L3 pte 0x%x\n", curtbl, curpte); 7939 #endif 7940 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) 7941 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 7942 ((u_int)addr & 0xfff)); 7943 else { 7944 printf("Bad L3 pte 0x%x for reg 0x%x, seg 0x%x, pg 0x%x\n", 7945 curpte, VA_VREG(addr), VA_VSEG(addr), VA_VPG(addr)); 7946 return 0; 7947 } 7948 printf("Bizarreness with address %p!\n", addr); 7949 } 7950 7951 static void 7952 test_region(int reg, int start, int stop) 7953 { 7954 int i; 7955 int addr; 7956 int pte; 7957 int ptesw; 7958 /* int cnt=0; 7959 */ 7960 7961 for (i = start; i < stop; i += NBPG) { 7962 addr = (reg << RGSHIFT) | i; 7963 pte = lda(((u_int)(addr)) | ASI_SRMMUFP_LN, ASI_SRMMUFP); 7964 if (pte) { 7965 /* printf("Valid address 0x%x\n",addr); 7966 if (++cnt == 20) { 7967 cngetc(); 7968 cnt = 0; 7969 } 7970 */ 7971 if (VA2PA((void *)addr) != VA2PAsw(0, (void *)addr, &ptesw)) { 7972 printf("Mismatch at address 0x%x.\n", addr); 7973 if (cngetc() == 'q') 7974 break; 7975 } 7976 if (reg == VA_VREG(KERNBASE)) 7977 /* kernel permissions are different */ 7978 continue; 7979 if ((pte & SRMMU_PROT_MASK) != (ptesw & SRMMU_PROT_MASK)) { 7980 printf("Mismatched protections at address " 7981 "0x%x; pte=0x%x, ptesw=0x%x\n", 7982 addr, pte, ptesw); 7983 if (cngetc() == 'q') 7984 break; 7985 } 7986 } 7987 } 7988 printf("done.\n"); 7989 } 7990 7991 7992 void 7993 print_fe_map(void) 7994 { 7995 u_int i, pte; 7996 7997 printf("map of region 0xfe:\n"); 7998 for (i = 0xfe000000; i < 0xff000000; i += 4096) { 7999 if (((pte = getpte4m(i)) & SRMMU_TETYPE) != SRMMU_TEPTE) 8000 continue; 8001 printf("0x%x -> 0x%x%x (pte 0x%x)\n", i, pte >> 28, 8002 (pte & ~0xff) << 4, pte); 8003 } 8004 printf("done\n"); 8005 } 8006 #endif /* EXTREME_DEBUG */ 8007 8008 #ifdef DDB 8009 int pmap_dump(struct pmap *pm); 8010 8011 int 8012 pmap_dump(struct pmap *pm) 8013 { 8014 int startvr, endvr, vr, vs, i, n; 8015 struct regmap *rp; 8016 struct segmap *sp; 8017 8018 if (pm == NULL) 8019 pm = pmap_kernel(); 8020 8021 if (pm == pmap_kernel()) { 8022 startvr = NUREG; 8023 endvr = 256; 8024 } else { 8025 startvr = 0; 8026 endvr = NUREG; 8027 } 8028 8029 for (vr = startvr; vr < endvr; vr++) { 8030 rp = &pm->pm_regmap[vr]; 8031 if (rp->rg_nsegmap == 0) 8032 continue; 8033 printf("vr %d: %d segments", vr, rp->rg_nsegmap); 8034 if (rp->rg_segmap == NULL) { 8035 printf("[no segments]\n"); 8036 continue; 8037 } 8038 for (vs = 0; vs < NSEGRG; vs++) { 8039 sp = &rp->rg_segmap[vs]; 8040 if (sp->sg_npte == 0) 8041 continue; 8042 if ((vs & 3) == 0) 8043 printf("\n "); 8044 printf(" %d: n %d w %d p %d,", vs, 8045 sp->sg_npte, sp->sg_nwired, sp->sg_pmeg); 8046 if (sp->sg_pte == NULL) { 8047 printf("[no ptes]"); 8048 continue; 8049 } 8050 for (n = 0, i = 0; i < NPTESG; i++) { 8051 if (CPU_HAS_SUNMMU && sp->sg_pte[i] & PG_WIRED) 8052 n++; 8053 if (CPU_HAS_SRMMU && sp->sg_wiremap & (1 << i)) 8054 n++; 8055 } 8056 if (n != sp->sg_nwired) 8057 printf("[wired count %d]", n); 8058 } 8059 printf("\n"); 8060 } 8061 8062 return (0); 8063 } 8064 #endif /* DDB */ 8065