xref: /netbsd/sys/arch/sparc/sparc/pmap.c (revision 6550d01e)
1 /*	$NetBSD: pmap.c,v 1.340 2010/04/26 09:26:25 martin Exp $ */
2 
3 /*
4  * Copyright (c) 1996
5  * 	The President and Fellows of Harvard College. All rights reserved.
6  * Copyright (c) 1992, 1993
7  *	The Regents of the University of California.  All rights reserved.
8  *
9  * This software was developed by the Computer Systems Engineering group
10  * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
11  * contributed to Berkeley.
12  *
13  * All advertising materials mentioning features or use of this software
14  * must display the following acknowledgement:
15  *	This product includes software developed by Harvard University.
16  *	This product includes software developed by the University of
17  *	California, Lawrence Berkeley Laboratory.
18  *
19  * Redistribution and use in source and binary forms, with or without
20  * modification, are permitted provided that the following conditions
21  * are met:
22  *
23  * 1. Redistributions of source code must retain the above copyright
24  *    notice, this list of conditions and the following disclaimer.
25  * 2. Redistributions in binary form must reproduce the above copyright
26  *    notice, this list of conditions and the following disclaimer in the
27  *    documentation and/or other materials provided with the distribution.
28  * 3. All advertising materials mentioning features or use of this software
29  *    must display the following acknowledgement:
30  *	This product includes software developed by Aaron Brown and
31  *	Harvard University.
32  *      This product includes software developed by the University of
33  *      California, Berkeley and its contributors.
34  * 4. Neither the name of the University nor the names of its contributors
35  *    may be used to endorse or promote products derived from this software
36  *    without specific prior written permission.
37  *
38  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
39  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
40  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
41  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
42  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
43  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
44  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
45  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
46  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
47  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
48  * SUCH DAMAGE.
49  *
50  *	@(#)pmap.c	8.4 (Berkeley) 2/5/94
51  *
52  */
53 
54 /*
55  * SPARC physical map management code.
56  */
57 
58 #include <sys/cdefs.h>
59 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.340 2010/04/26 09:26:25 martin Exp $");
60 
61 #include "opt_ddb.h"
62 #include "opt_kgdb.h"
63 #include "opt_sparc_arch.h"
64 
65 #include <sys/param.h>
66 #include <sys/systm.h>
67 #include <sys/device.h>
68 #include <sys/proc.h>
69 #include <sys/queue.h>
70 #include <sys/pool.h>
71 #include <sys/exec.h>
72 #include <sys/core.h>
73 #include <sys/kcore.h>
74 #include <sys/kernel.h>
75 #include <sys/atomic.h>
76 
77 #include <sys/exec_aout.h>		/* for MID_* */
78 
79 #include <uvm/uvm.h>
80 
81 #include <machine/autoconf.h>
82 #include <machine/bsd_openprom.h>
83 #include <machine/oldmon.h>
84 #include <machine/cpu.h>
85 #include <machine/ctlreg.h>
86 #include <machine/kcore.h>
87 
88 #include <sparc/sparc/asm.h>
89 #include <sparc/sparc/cache.h>
90 #include <sparc/sparc/vaddrs.h>
91 #include <sparc/sparc/cpuvar.h>
92 
93 /*
94  * The SPARCstation offers us the following challenges:
95  *
96  *   1. A virtual address cache.  This is, strictly speaking, not
97  *	part of the architecture, but the code below assumes one.
98  *	This is a write-through cache on the 4c and a write-back cache
99  *	on others.
100  *
101  *   2. (4/4c only) An MMU that acts like a cache.  There is not enough
102  *	space in the MMU to map everything all the time.  Instead, we need
103  *	to load MMU with the `working set' of translations for each
104  *	process. The sun4m does not act like a cache; tables are maintained
105  *	in physical memory.
106  *
107  *   3.	Segmented virtual and physical spaces.  The upper 12 bits of
108  *	a virtual address (the virtual segment) index a segment table,
109  *	giving a physical segment.  The physical segment selects a
110  *	`Page Map Entry Group' (PMEG) and the virtual page number---the
111  *	next 5 or 6 bits of the virtual address---select the particular
112  *	`Page Map Entry' for the page.  We call the latter a PTE and
113  *	call each Page Map Entry Group a pmeg (for want of a better name).
114  *	Note that the sun4m has an unsegmented 36-bit physical space.
115  *
116  *	Since there are no valid bits in the segment table, the only way
117  *	to have an invalid segment is to make one full pmeg of invalid PTEs.
118  *	We use the last one (since the ROM does as well) (sun4/4c only)
119  *
120  *   4. Discontiguous physical pages.  The Mach VM expects physical pages
121  *	to be in one sequential lump.
122  *
123  *   5. The MMU is always on: it is not possible to disable it.  This is
124  *	mainly a startup hassle.
125  */
126 
127 struct pmap_stats {
128 	int	ps_unlink_pvfirst;	/* # of pv_unlinks on head */
129 	int	ps_unlink_pvsearch;	/* # of pv_unlink searches */
130 	int	ps_changeprots;		/* # of calls to changeprot */
131 	int	ps_enter_firstpv;	/* pv heads entered */
132 	int	ps_enter_secondpv;	/* pv nonheads entered */
133 	int	ps_useless_changewire;	/* useless wiring changes */
134 	int	ps_npg_prot_all;	/* # of active pages protected */
135 	int	ps_npg_prot_actual;	/* # pages actually affected */
136 	int	ps_npmeg_free;		/* # of free pmegs */
137 	int	ps_npmeg_locked;	/* # of pmegs on locked list */
138 	int	ps_npmeg_lru;		/* # of pmegs on lru list */
139 } pmap_stats;
140 
141 #if defined(SUN4) || defined(SUN4C)
142 struct evcnt mmu_stolenpmegs_evcnt =
143 	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","stln pmgs");
144 EVCNT_ATTACH_STATIC(mmu_stolenpmegs_evcnt);
145 
146 struct evcnt mmu_pagein_evcnt =
147 	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","pagein");
148 EVCNT_ATTACH_STATIC(mmu_pagein_evcnt);
149 #endif /* SUN4 || SUN4C */
150 
151 #ifdef DEBUG
152 #define	PDB_CREATE	0x0001
153 #define	PDB_DESTROY	0x0002
154 #define	PDB_REMOVE	0x0004
155 #define	PDB_CHANGEPROT	0x0008
156 #define	PDB_ENTER	0x0010
157 #define	PDB_FOLLOW	0x0020
158 
159 #define	PDB_MMU_ALLOC	0x0100
160 #define	PDB_MMU_STEAL	0x0200
161 #define	PDB_CTX_ALLOC	0x0400
162 #define	PDB_CTX_STEAL	0x0800
163 #define	PDB_MMUREG_ALLOC	0x1000
164 #define	PDB_MMUREG_STEAL	0x2000
165 #define	PDB_CACHESTUFF	0x4000
166 #define	PDB_SWITCHMAP	0x8000
167 #define	PDB_SANITYCHK	0x10000
168 int	pmapdebug = 0;
169 #endif
170 
171 /*
172  * Bounds on managed physical addresses. Used by (MD) users
173  * of uvm_pglistalloc() to provide search hints.
174  */
175 paddr_t	vm_first_phys = (paddr_t)-1;
176 paddr_t	vm_last_phys = 0;
177 psize_t vm_num_phys;
178 
179 #define	PMAP_LOCK()	KERNEL_LOCK(1, NULL)
180 #define	PMAP_UNLOCK()	KERNEL_UNLOCK_ONE(NULL)
181 
182 /*
183  * Flags in pvlist.pv_flags.  Note that PV_MOD must be 1 and PV_REF must be 2
184  * since they must line up with the bits in the hardware PTEs (see pte.h).
185  * SUN4M bits are at a slightly different location in the PTE.
186  *
187  * Note: the REF, MOD and ANC flag bits occur only in the head of a pvlist.
188  * The NC bit is meaningful in each individual pv entry and reflects the
189  * requested non-cacheability at the time the entry was made through
190  * pv_link() or when subsequently altered by kvm_uncache() (but the latter
191  * does not happen in kernels as of the time of this writing (March 2001)).
192  */
193 #define PV_MOD		1	/* page modified */
194 #define PV_REF		2	/* page referenced */
195 #define PV_NC		4	/* page cannot be cached */
196 #define PV_REF4M	1	/* page referenced (SRMMU) */
197 #define PV_MOD4M	2	/* page modified (SRMMU) */
198 #define PV_ANC		0x10	/* page has incongruent aliases */
199 
200 static struct pool pv_pool;
201 
202 /*
203  * pvhead(pte): find a VM page given a PTE entry.
204  */
205 #if defined(SUN4) || defined(SUN4C)
206 static struct vm_page *
207 pvhead4_4c(u_int pte)
208 {
209 	paddr_t pa = (pte & PG_PFNUM) << PGSHIFT;
210 
211 	return (PHYS_TO_VM_PAGE(pa));
212 }
213 #endif
214 
215 #if defined(SUN4M) || defined(SUN4D)
216 static struct vm_page *
217 pvhead4m(u_int pte)
218 {
219 	paddr_t pa = (pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT;
220 
221 	return (PHYS_TO_VM_PAGE(pa));
222 }
223 #endif
224 
225 /*
226  * Each virtual segment within each pmap is either valid or invalid.
227  * It is valid if pm_npte[VA_VSEG(va)] is not 0.  This does not mean
228  * it is in the MMU, however; that is true iff pm_segmap[VA_VSEG(va)]
229  * does not point to the invalid PMEG.
230  *
231  * In the older SPARC architectures (sun4/sun4c), page tables are cached in
232  * the MMU. The following discussion applies to these architectures:
233  *
234  * If a virtual segment is valid and loaded, the correct PTEs appear
235  * in the MMU only.  If it is valid and unloaded, the correct PTEs appear
236  * in the pm_pte[VA_VSEG(va)] only.  However, some effort is made to keep
237  * the software copies consistent enough with the MMU so that libkvm can
238  * do user address translations.  In particular, pv_changepte() and
239  * pmap_enu() maintain consistency, while less critical changes are
240  * not maintained.  pm_pte[VA_VSEG(va)] always points to space for those
241  * PTEs.
242  *
243  * Each PMEG in the MMU is either free or contains PTEs corresponding to
244  * some pmap and virtual segment.  If it contains some PTEs, it also contains
245  * reference and modify bits that belong in the pv_table.  If we need
246  * to steal a PMEG from some process (if we need one and none are free)
247  * we must copy the ref and mod bits, and update pm_segmap in the other
248  * pmap to show that its virtual segment is no longer in the MMU.
249  *
250  * There are 128 PMEGs in a small Sun-4, of which only a few dozen are
251  * tied down permanently, leaving `about' 100 to be spread among
252  * running processes.  These are managed as an LRU cache.  Before
253  * calling the VM paging code for a user page fault, the fault handler
254  * calls mmu_load(pmap, va) to try to get a set of PTEs put into the
255  * MMU.  mmu_load will check the validity of the segment and tell whether
256  * it did something.
257  *
258  * Since I hate the name PMEG I call this data structure an `mmu entry'.
259  * Each mmuentry is on exactly one of three `usage' lists: free, LRU,
260  * or locked.  The locked list is only used for kernel mappings that need
261  * to be wired down.
262  *
263  *
264  * In the sun4m architecture using the SPARC Reference MMU (SRMMU), three
265  * levels of page tables are maintained in physical memory. We use the same
266  * structures as with the 3-level old-style MMU (pm_regmap, pm_segmap,
267  * rg_segmap, sg_pte, etc) to maintain kernel-edible page tables; we also
268  * build a parallel set of physical tables that can be used by the MMU.
269  * (XXX: This seems redundant, but is it necessary for the unified kernel?)
270  *
271  * If a virtual segment is valid, its entries will be in both parallel lists.
272  * If it is not valid, then its entry in the kernel tables will be zero, and
273  * its entry in the MMU tables will either be nonexistent or zero as well.
274  *
275  * The Reference MMU generally uses a Translation Look-aside Buffer (TLB)
276  * to cache the result of recently executed page table walks. When
277  * manipulating page tables, we need to ensure consistency of the
278  * in-memory and TLB copies of the page table entries. This is handled
279  * by flushing (and invalidating) a TLB entry when appropriate before
280  * altering an in-memory page table entry.
281  */
282 struct mmuentry {
283 	CIRCLEQ_ENTRY(mmuentry)	me_list;	/* usage list link */
284 	TAILQ_ENTRY(mmuentry)	me_pmchain;	/* pmap owner link */
285 	struct	pmap *me_pmap;		/* pmap, if in use */
286 	u_short	me_vreg;		/* associated virtual region/segment */
287 	u_short	me_vseg;		/* associated virtual region/segment */
288 	u_short	me_cookie;		/* hardware SMEG/PMEG number */
289 #ifdef DIAGNOSTIC
290 	int *me_statp;/*XXX*/
291 #endif
292 };
293 struct mmuentry *mmusegments;	/* allocated in pmap_bootstrap */
294 struct mmuentry *mmuregions;	/* allocated in pmap_bootstrap */
295 
296 CIRCLEQ_HEAD(mmuq, mmuentry);
297 struct mmuq segm_freelist, segm_lru, segm_locked;
298 struct mmuq region_freelist, region_lru, region_locked;
299 /*
300  * We use a circular queue, since that allows us to remove an element
301  * from a list without knowing the list header.
302  */
303 #define CIRCLEQ_REMOVE_NOH(elm, field) do {				\
304 	(elm)->field.cqe_next->field.cqe_prev = (elm)->field.cqe_prev;	\
305 	(elm)->field.cqe_prev->field.cqe_next = (elm)->field.cqe_next;	\
306 } while (/*CONSTCOND*/0)
307 
308 #define MMUQ_INIT(head)			CIRCLEQ_INIT(head)
309 #define MMUQ_REMOVE(elm,field)		CIRCLEQ_REMOVE_NOH(elm,field)
310 #define MMUQ_INSERT_TAIL(head,elm,field)CIRCLEQ_INSERT_TAIL(head,elm,field)
311 #define MMUQ_EMPTY(head)		CIRCLEQ_EMPTY(head)
312 #define MMUQ_FIRST(head)		CIRCLEQ_FIRST(head)
313 
314 
315 int	seginval;		/* [4/4c] the invalid segment number */
316 int	reginval;		/* [4/3mmu] the invalid region number */
317 
318 static kmutex_t demap_lock;
319 static bool	lock_available = false;	/* demap_lock has been initialized */
320 
321 /*
322  * (sun4/4c)
323  * A context is simply a small number that dictates which set of 4096
324  * segment map entries the MMU uses.  The Sun 4c has eight (SS1,IPC) or
325  * sixteen (SS2,IPX) such sets. These are alloted in an `almost MRU' fashion.
326  * (sun4m)
327  * A context is simply a small number that indexes the context table, the
328  * root-level page table mapping 4G areas. Each entry in this table points
329  * to a 1st-level region table. A SPARC reference MMU will usually use 16
330  * such contexts, but some offer as many as 64k contexts; the theoretical
331  * maximum is 2^32 - 1, but this would create overlarge context tables.
332  *
333  * Each context is either free or attached to a pmap.
334  *
335  * Since the virtual address cache is tagged by context, when we steal
336  * a context we have to flush (that part of) the cache.
337  */
338 union ctxinfo {
339 	union	ctxinfo *c_nextfree;	/* free list (if free) */
340 	struct	pmap *c_pmap;		/* pmap (if busy) */
341 };
342 
343 static kmutex_t	ctx_lock;		/* lock for below */
344 union	ctxinfo *ctxinfo;		/* allocated at in pmap_bootstrap */
345 union	ctxinfo *ctx_freelist;		/* context free list */
346 int	ctx_kick;			/* allocation rover when none free */
347 int	ctx_kickdir;			/* ctx_kick roves both directions */
348 int	ncontext;			/* sizeof ctx_freelist */
349 
350 void	ctx_alloc(struct pmap *);
351 void	ctx_free(struct pmap *);
352 
353 void *	vmmap;			/* one reserved MI vpage for /dev/mem */
354 /*void *	vdumppages;	-* 32KB worth of reserved dump pages */
355 
356 smeg_t		tregion;	/* [4/3mmu] Region for temporary mappings */
357 
358 static struct pmap	kernel_pmap_store;	/* the kernel's pmap */
359 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; /* pmap_kernel() */
360 struct regmap	kernel_regmap_store[NKREG];	/* the kernel's regmap */
361 struct segmap	kernel_segmap_store[NKREG*NSEGRG];/* the kernel's segmaps */
362 
363 #if defined(SUN4M) || defined(SUN4D)
364 u_int 	*kernel_regtable_store;		/* 1k of storage to map the kernel */
365 u_int	*kernel_segtable_store;		/* 2k of storage to map the kernel */
366 u_int	*kernel_pagtable_store;		/* 128k of storage to map the kernel */
367 
368 /*
369  * Memory pools and back-end supplier for SRMMU page tables.
370  * Share a pool between the level 2 and level 3 page tables,
371  * since these are equal in size.
372  */
373 static struct pool L1_pool;
374 static struct pool L23_pool;
375 
376 static void *pgt_page_alloc(struct pool *, int);
377 static void  pgt_page_free(struct pool *, void *);
378 
379 static struct pool_allocator pgt_page_allocator = {
380 	pgt_page_alloc, pgt_page_free, 0,
381 };
382 
383 #endif /* SUN4M || SUN4D */
384 
385 #if defined(SUN4) || defined(SUN4C)
386 /*
387  * Memory pool for user and kernel PTE tables.
388  */
389 static struct pool pte_pool;
390 #endif
391 
392 struct	memarr *pmemarr;	/* physical memory regions */
393 int	npmemarr;		/* number of entries in pmemarr */
394 
395 static paddr_t	avail_start;	/* first available physical page, other
396 				   than the `etext gap' defined below */
397 static vaddr_t	etext_gap_start;/* start of gap between text & data */
398 static vaddr_t	etext_gap_end;	/* end of gap between text & data */
399 static vaddr_t	virtual_avail;	/* first free kernel virtual address */
400 static vaddr_t	virtual_end;	/* last free kernel virtual address */
401 
402 static void pmap_page_upload(void);
403 
404 int mmu_has_hole;
405 
406 vaddr_t prom_vstart;	/* For /dev/kmem */
407 vaddr_t prom_vend;
408 
409 /*
410  * Memory pool for pmap structures.
411  */
412 static struct pool_cache pmap_cache;
413 static int	pmap_pmap_pool_ctor(void *, void *, int);
414 static void	pmap_pmap_pool_dtor(void *, void *);
415 static struct pool segmap_pool;
416 
417 #if defined(SUN4)
418 /*
419  * [sun4]: segfixmask: on some systems (4/110) "getsegmap()" returns a
420  * partly invalid value. getsegmap returns a 16 bit value on the sun4,
421  * but only the first 8 or so bits are valid (the rest are *supposed* to
422  * be zero. On the 4/110 the bits that are supposed to be zero are
423  * all one instead. e.g. KERNBASE is usually mapped by pmeg number zero.
424  * On a 4/300 getsegmap(KERNBASE) == 0x0000, but
425  * on a 4/100 getsegmap(KERNBASE) == 0xff00
426  *
427  * This confuses mmu_reservemon() and causes it to not reserve the PROM's
428  * pmegs. Then the PROM's pmegs get used during autoconfig and everything
429  * falls apart!  (not very fun to debug, BTW.)
430  *
431  * solution: mask the invalid bits in the getsetmap macro.
432  */
433 
434 static u_int segfixmask = 0xffffffff; /* all bits valid to start */
435 #else
436 #define segfixmask 0xffffffff	/* It's in getsegmap's scope */
437 #endif
438 
439 /*
440  * pseudo-functions for mnemonic value
441  */
442 #define	getsegmap(va)		(CPU_ISSUN4C \
443 					? lduba(va, ASI_SEGMAP) \
444 					: (lduha(va, ASI_SEGMAP) & segfixmask))
445 #define	setsegmap(va, pmeg)	(CPU_ISSUN4C \
446 					? stba(va, ASI_SEGMAP, pmeg) \
447 					: stha(va, ASI_SEGMAP, pmeg))
448 
449 /* 3-level sun4 MMU only: */
450 #define	getregmap(va)		((unsigned)lduha((va)+2, ASI_REGMAP) >> 8)
451 #define	setregmap(va, smeg)	stha((va)+2, ASI_REGMAP, (smeg << 8))
452 
453 
454 #if defined(SUN4M) || defined(SUN4D)
455 #if 0
456 #if VM_PROT_READ != 1 || VM_PROT_WRITE != 2 || VM_PROT_EXECUTE != 4
457 #error fix protection code translation table
458 #endif
459 #endif
460 /*
461  * Translation table for kernel vs. PTE protection bits.
462  */
463 const u_int protection_codes[2][8] = {
464 	/* kernel */
465 	{
466 	PPROT_N_RX,	/* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_NONE */
467 	PPROT_N_RX,	/* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_READ */
468 	PPROT_N_RWX,	/* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_NONE */
469 	PPROT_N_RWX,	/* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_READ */
470 	PPROT_N_RX,	/* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_NONE */
471 	PPROT_N_RX,	/* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_READ */
472 	PPROT_N_RWX,	/* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */
473 	PPROT_N_RWX,	/* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */
474 	},
475 
476 	/* user */
477 	{
478 	PPROT_N_RX,	/* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_NONE */
479 	PPROT_R_R,	/* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_READ */
480 	PPROT_RW_RW,	/* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_NONE */
481 	PPROT_RW_RW,	/* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_READ */
482 	PPROT_X_X,	/* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_NONE */
483 	PPROT_RX_RX,	/* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_READ */
484 	PPROT_RWX_RWX,	/* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */
485 	PPROT_RWX_RWX,	/* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */
486 	}
487 };
488 #define pte_kprot4m(prot) (protection_codes[0][(prot)])
489 #define pte_uprot4m(prot) (protection_codes[1][(prot)])
490 #define pte_prot4m(pm, prot) \
491 	(protection_codes[(pm) == pmap_kernel() ? 0 : 1][(prot)])
492 
493 void		setpte4m(vaddr_t va, int pte);
494 void		setpgt4m(int *ptep, int pte);
495 void		setpgt4m_va(vaddr_t, int *, int, int, int, u_int);
496 int		updatepte4m(vaddr_t, int *, int, int, int, u_int);
497 #endif /* SUN4M || SUN4D */
498 
499 #if defined(MULTIPROCESSOR)
500 #define PMAP_SET_CPUSET(pmap, cpi)	\
501 	(pmap->pm_cpuset |= (1 << (cpi)->ci_cpuid))
502 #define PMAP_CLR_CPUSET(pmap, cpi)	\
503 	(pmap->pm_cpuset &= ~(1 << (cpi)->ci_cpuid))
504 #define PMAP_CPUSET(pmap)		(pmap->pm_cpuset)
505 #else
506 #define PMAP_SET_CPUSET(pmap, cpi)	/* nothing */
507 #define PMAP_CLR_CPUSET(pmap, cpi)	/* nothing */
508 #define PMAP_CPUSET(pmap)		1	/* XXX: 1 or 0? */
509 #endif /* MULTIPROCESSOR */
510 
511 
512 /* Function pointer messiness for supporting multiple sparc architectures
513  * within a single kernel: notice that there are two versions of many of the
514  * functions within this file/module, one for the sun4/sun4c and the other
515  * for the sun4m. For performance reasons (since things like pte bits don't
516  * map nicely between the two architectures), there are separate functions
517  * rather than unified functions which test the cputyp variable. If only
518  * one architecture is being used, then the non-suffixed function calls
519  * are macro-translated into the appropriate xxx4_4c or xxx4m call. If
520  * multiple architectures are defined, the calls translate to (*xxx_p),
521  * i.e. they indirect through function pointers initialized as appropriate
522  * to the run-time architecture in pmap_bootstrap. See also pmap.h.
523  */
524 
525 #if defined(SUN4M) || defined(SUN4D)
526 static void mmu_setup4m_L1(int, struct pmap *);
527 static void mmu_setup4m_L2(int, struct regmap *);
528 static void  mmu_setup4m_L3(int, struct segmap *);
529 /*static*/ void	mmu_reservemon4m(struct pmap *);
530 
531 /*static*/ void pmap_changeprot4m(pmap_t, vaddr_t, vm_prot_t, int);
532 /*static*/ void pmap_rmk4m(struct pmap *, vaddr_t, vaddr_t, int, int);
533 /*static*/ void pmap_rmu4m(struct pmap *, vaddr_t, vaddr_t, int, int);
534 /*static*/ int  pmap_enk4m(struct pmap *, vaddr_t, vm_prot_t,
535 				int, struct vm_page *, int);
536 /*static*/ int  pmap_enu4m(struct pmap *, vaddr_t, vm_prot_t,
537 				int, struct vm_page *, int);
538 /*static*/ void pv_changepte4m(struct vm_page *, int, int);
539 /*static*/ int  pv_syncflags4m(struct vm_page *);
540 /*static*/ int  pv_link4m(struct vm_page *, struct pmap *, vaddr_t, u_int *);
541 /*static*/ void pv_unlink4m(struct vm_page *, struct pmap *, vaddr_t);
542 #endif
543 
544 #if defined(SUN4) || defined(SUN4C)
545 /*static*/ void	mmu_reservemon4_4c(int *, int *);
546 /*static*/ void pmap_changeprot4_4c(pmap_t, vaddr_t, vm_prot_t, int);
547 /*static*/ void pmap_rmk4_4c(struct pmap *, vaddr_t, vaddr_t, int, int);
548 /*static*/ void pmap_rmu4_4c(struct pmap *, vaddr_t, vaddr_t, int, int);
549 /*static*/ int  pmap_enk4_4c(struct pmap *, vaddr_t, vm_prot_t,
550 				  int, struct vm_page *, int);
551 /*static*/ int  pmap_enu4_4c(struct pmap *, vaddr_t, vm_prot_t,
552 				  int, struct vm_page *, int);
553 /*static*/ void pv_changepte4_4c(struct vm_page *, int, int);
554 /*static*/ int  pv_syncflags4_4c(struct vm_page *);
555 /*static*/ int  pv_link4_4c(struct vm_page *, struct pmap *, vaddr_t, u_int *);
556 /*static*/ void pv_unlink4_4c(struct vm_page *, struct pmap *, vaddr_t);
557 #endif
558 
559 #if !(defined(SUN4M) || defined(SUN4D)) && (defined(SUN4) || defined(SUN4C))
560 #define		pmap_rmk	pmap_rmk4_4c
561 #define		pmap_rmu	pmap_rmu4_4c
562 
563 #elif (defined(SUN4M) || defined(SUN4D)) && !(defined(SUN4) || defined(SUN4C))
564 #define		pmap_rmk	pmap_rmk4m
565 #define		pmap_rmu	pmap_rmu4m
566 
567 #else  /* must use function pointers */
568 
569 /* function pointer declarations */
570 /* from pmap.h: */
571 bool		(*pmap_clear_modify_p)(struct vm_page *);
572 bool		(*pmap_clear_reference_p)(struct vm_page *);
573 int		(*pmap_enter_p)(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int);
574 bool		(*pmap_extract_p)(pmap_t, vaddr_t, paddr_t *);
575 bool		(*pmap_is_modified_p)(struct vm_page *);
576 bool		(*pmap_is_referenced_p)(struct vm_page *);
577 void		(*pmap_kenter_pa_p)(vaddr_t, paddr_t, vm_prot_t, u_int);
578 void		(*pmap_kremove_p)(vaddr_t, vsize_t);
579 void		(*pmap_kprotect_p)(vaddr_t, vsize_t, vm_prot_t);
580 void		(*pmap_page_protect_p)(struct vm_page *, vm_prot_t);
581 void		(*pmap_protect_p)(pmap_t, vaddr_t, vaddr_t, vm_prot_t);
582 /* local: */
583 void 		(*pmap_rmk_p)(struct pmap *, vaddr_t, vaddr_t, int, int);
584 void 		(*pmap_rmu_p)(struct pmap *, vaddr_t, vaddr_t, int, int);
585 
586 #define		pmap_rmk	(*pmap_rmk_p)
587 #define		pmap_rmu	(*pmap_rmu_p)
588 
589 #endif
590 
591 /* --------------------------------------------------------------*/
592 
593 /*
594  * Next we have some sun4m/4d-specific routines which have no 4/4c
595  * counterparts, or which are 4/4c macros.
596  */
597 
598 #if defined(SUN4M) || defined(SUN4D)
599 /*
600  * SP versions of the tlb flush operations.
601  *
602  * Turn off traps to prevent register window overflows
603  * from writing user windows to the wrong stack.
604  */
605 static void
606 sp_tlb_flush(int va, int ctx, int lvl)
607 {
608 
609 	/* Traps off */
610 	__asm("rd	%psr, %o3");
611 	__asm("wr	%%o3, %0, %%psr" :: "n" (PSR_ET));
612 
613 	/* Save context */
614 	__asm("mov	%0, %%o4" :: "n"(SRMMU_CXR));
615 	__asm("lda	[%%o4]%0, %%o5" :: "n"(ASI_SRMMU));
616 
617 	/* Set new context and flush type bits */
618 	__asm("andn	%o0, 0xfff, %o0");
619 	__asm("sta	%%o1, [%%o4]%0" :: "n"(ASI_SRMMU));
620 	__asm("or	%o0, %o2, %o0");
621 
622 	/* Do the TLB flush */
623 	__asm("sta	%%g0, [%%o0]%0" :: "n"(ASI_SRMMUFP));
624 
625 	/* Restore context */
626 	__asm("sta	%%o5, [%%o4]%0" :: "n"(ASI_SRMMU));
627 
628 	/* and turn traps on again */
629 	__asm("wr	%o3, 0, %psr");
630 	__asm("nop");
631 	__asm("nop");
632 	__asm("nop");
633 }
634 
635 static inline void
636 sp_tlb_flush_all(void)
637 {
638 
639 	sta(ASI_SRMMUFP_LN, ASI_SRMMUFP, 0);
640 }
641 
642 #if defined(MULTIPROCESSOR)
643 /*
644  * The SMP versions of the tlb flush routines.  We only need to
645  * do a cross call for these on sun4m (Mbus) systems. sun4d systems
646  * have an Xbus which broadcasts TLB demaps in hardware.
647  */
648 
649 static inline void	smp_tlb_flush_page (int va, int ctx, u_int cpuset);
650 static inline void	smp_tlb_flush_segment (int va, int ctx, u_int cpuset);
651 static inline void	smp_tlb_flush_region (int va, int ctx, u_int cpuset);
652 static inline void	smp_tlb_flush_context (int ctx, u_int cpuset);
653 static inline void	smp_tlb_flush_all (void);
654 
655 /* From locore: */
656 extern void ft_tlb_flush(int va, int ctx, int lvl);
657 
658 static inline void
659 smp_tlb_flush_page(int va, int ctx, u_int cpuset)
660 {
661 
662 	if (CPU_ISSUN4D) {
663 		sp_tlb_flush(va, ctx, ASI_SRMMUFP_L3);
664 	} else
665 		FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L3, cpuset);
666 }
667 
668 static inline void
669 smp_tlb_flush_segment(int va, int ctx, u_int cpuset)
670 {
671 
672 	if (CPU_ISSUN4D) {
673 		sp_tlb_flush(va, ctx, ASI_SRMMUFP_L2);
674 	} else
675 		FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L2, cpuset);
676 }
677 
678 static inline void
679 smp_tlb_flush_region(int va, int ctx, u_int cpuset)
680 {
681 
682 	if (CPU_ISSUN4D) {
683 		sp_tlb_flush(va, ctx, ASI_SRMMUFP_L1);
684 	} else
685 		FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L1, cpuset);
686 }
687 
688 static inline void
689 smp_tlb_flush_context(int ctx, u_int cpuset)
690 {
691 
692 	if (CPU_ISSUN4D) {
693 		sp_tlb_flush(ctx, 0, ASI_SRMMUFP_L0);
694 	} else
695 		FXCALL3(sp_tlb_flush, ft_tlb_flush, 0, ctx, ASI_SRMMUFP_L0, cpuset);
696 }
697 
698 static inline void
699 smp_tlb_flush_all(void)
700 {
701 
702 	if (CPU_ISSUN4D) {
703 		sp_tlb_flush_all();
704 	} else
705 		XCALL0(sp_tlb_flush_all, CPUSET_ALL);
706 }
707 #endif /* MULTIPROCESSOR */
708 
709 #if defined(MULTIPROCESSOR)
710 #define tlb_flush_page(va,ctx,s)	smp_tlb_flush_page(va,ctx,s)
711 #define tlb_flush_segment(va,ctx,s)	smp_tlb_flush_segment(va,ctx,s)
712 #define tlb_flush_region(va,ctx,s)	smp_tlb_flush_region(va,ctx,s)
713 #define tlb_flush_context(ctx,s)	smp_tlb_flush_context(ctx,s)
714 #define tlb_flush_all()			smp_tlb_flush_all()
715 #else
716 #define tlb_flush_page(va,ctx,s)	sp_tlb_flush(va,ctx,ASI_SRMMUFP_L3)
717 #define tlb_flush_segment(va,ctx,s)	sp_tlb_flush(va,ctx,ASI_SRMMUFP_L2)
718 #define tlb_flush_region(va,ctx,s)	sp_tlb_flush(va,ctx,ASI_SRMMUFP_L1)
719 #define tlb_flush_context(ctx,s)	sp_tlb_flush(ctx,0,ASI_SRMMUFP_L0)
720 #define tlb_flush_all()			sp_tlb_flush_all()
721 #endif /* MULTIPROCESSOR */
722 
723 static u_int	VA2PA(void *);
724 static u_long	srmmu_bypass_read(u_long);
725 
726 /*
727  * VA2PA(addr) -- converts a virtual address to a physical address using
728  * the MMU's currently-installed page tables. As a side effect, the address
729  * translation used may cause the associated pte to be encached. The correct
730  * context for VA must be set before this is called.
731  *
732  * This routine should work with any level of mapping, as it is used
733  * during bootup to interact with the ROM's initial L1 mapping of the kernel.
734  */
735 static u_int
736 VA2PA(void *addr)
737 {
738 	u_int pte;
739 
740 	/*
741 	 * We'll use that handy SRMMU flush/probe.
742 	 * Try each level in turn until we find a valid pte. Otherwise panic.
743 	 */
744 
745 	pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L3, ASI_SRMMUFP);
746 	/* Unlock fault status; required on Hypersparc modules */
747 	(void)lda(SRMMU_SFSR, ASI_SRMMU);
748 	if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
749 	    return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
750 		    ((u_int)addr & 0xfff));
751 
752 	/* A `TLB Flush Entire' is required before any L0, L1 or L2 probe */
753 	tlb_flush_all_real();
754 
755 	pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L2, ASI_SRMMUFP);
756 	if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
757 	    return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
758 		    ((u_int)addr & 0x3ffff));
759 	pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L1, ASI_SRMMUFP);
760 	if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
761 	    return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
762 		    ((u_int)addr & 0xffffff));
763 	pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L0, ASI_SRMMUFP);
764 	if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
765 	    return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
766 		    ((u_int)addr & 0xffffffff));
767 
768 #ifdef DIAGNOSTIC
769 	panic("VA2PA: Asked to translate unmapped VA %p", addr);
770 #else
771 	return (0);
772 #endif
773 }
774 
775 /*
776  * Atomically update a PTE entry, coping with hardware updating the
777  * PTE at the same time we are.  This is the procedure that is
778  * recommended in the SuperSPARC user's manual.
779  */
780 int
781 updatepte4m(vaddr_t va, int *pte, int bic, int bis, int ctx, u_int cpuset)
782 {
783 	int oldval, swapval;
784 	volatile int *vpte = (volatile int *)pte;
785 	bool can_lock = lock_available;
786 
787 	/*
788 	 * Can only be one of these happening in the system
789 	 * at any one time.
790 	 */
791 	if (__predict_true(can_lock))
792 		mutex_spin_enter(&demap_lock);
793 
794 	/*
795 	 * The idea is to loop swapping zero into the pte, flushing
796 	 * it, and repeating until it stays zero.  At this point,
797 	 * there should be no more hardware accesses to this PTE
798 	 * so we can modify it without losing any mod/ref info.
799 	 */
800 	oldval = 0;
801 	do {
802 		swapval = 0;
803 		swap(vpte, swapval);
804 		tlb_flush_page(va, ctx, cpuset);
805 		oldval |= swapval;
806 	} while (__predict_false(*vpte != 0));
807 
808 	swapval = (oldval & ~bic) | bis;
809 	swap(vpte, swapval);
810 
811 	if (__predict_true(can_lock))
812 		mutex_spin_exit(&demap_lock);
813 
814 	return (oldval);
815 }
816 
817 inline void
818 setpgt4m(int *ptep, int pte)
819 {
820 
821 	swap(ptep, pte);
822 }
823 
824 inline void
825 setpgt4m_va(vaddr_t va, int *ptep, int pte, int pageflush, int ctx,
826 	    u_int cpuset)
827 {
828 
829 #if defined(MULTIPROCESSOR)
830 	updatepte4m(va, ptep, 0xffffffff, pte, pageflush ? ctx : 0, cpuset);
831 #else
832 	if (__predict_true(pageflush))
833 		tlb_flush_page(va, ctx, 0);
834 	setpgt4m(ptep, pte);
835 #endif /* MULTIPROCESSOR */
836 }
837 
838 /* Set the page table entry for va to pte. */
839 void
840 setpte4m(vaddr_t va, int pte)
841 {
842 	struct pmap *pm;
843 	struct regmap *rp;
844 	struct segmap *sp;
845 
846 #ifdef DEBUG
847 	if (getcontext4m() != 0)
848 		panic("setpte4m: user context");
849 #endif
850 
851 	pm = pmap_kernel();
852 	rp = &pm->pm_regmap[VA_VREG(va)];
853 	sp = &rp->rg_segmap[VA_VSEG(va)];
854 
855 	tlb_flush_page(va, 0, CPUSET_ALL);
856 	setpgt4m(sp->sg_pte + VA_SUN4M_VPG(va), pte);
857 }
858 
859 /*
860  * Page table pool back-end.
861  */
862 void *
863 pgt_page_alloc(struct pool *pp, int flags)
864 {
865 	int cacheit = (cpuinfo.flags & CPUFLG_CACHEPAGETABLES) != 0;
866 	struct vm_page *pg;
867 	vaddr_t va;
868 	paddr_t pa;
869 
870 	/* Allocate a page of physical memory */
871 	if ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL)
872 		return (NULL);
873 
874 	/* Allocate virtual memory */
875 	va = uvm_km_alloc(kmem_map, PAGE_SIZE, 0, UVM_KMF_VAONLY |
876 		((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK));
877 	if (va == 0) {
878 		uvm_pagefree(pg);
879 		return (NULL);
880 	}
881 
882 	/*
883 	 * On systems with a physical data cache we need to flush this page
884 	 * from the cache if the pagetables cannot be cached.
885 	 * On systems with a virtually indexed data cache, we only need
886 	 * to map it non-cacheable, since the page is not currently mapped.
887 	 */
888 	pa = VM_PAGE_TO_PHYS(pg);
889 	if (cacheit == 0)
890 		pcache_flush_page(pa, 1);
891 
892 	/* Map the page */
893 	pmap_kenter_pa(va, pa | (cacheit ? 0 : PMAP_NC),
894 	    VM_PROT_READ | VM_PROT_WRITE, 0);
895 	pmap_update(pmap_kernel());
896 
897 	return ((void *)va);
898 }
899 
900 void
901 pgt_page_free(struct pool *pp, void *v)
902 {
903 	vaddr_t va;
904 	paddr_t pa;
905 	bool rv;
906 
907 	va = (vaddr_t)v;
908 	rv = pmap_extract(pmap_kernel(), va, &pa);
909 	KASSERT(rv);
910 	uvm_pagefree(PHYS_TO_VM_PAGE(pa));
911 	pmap_kremove(va, PAGE_SIZE);
912 	uvm_km_free(kmem_map, va, PAGE_SIZE, UVM_KMF_VAONLY);
913 }
914 #endif /* SUN4M || SUN4D */
915 
916 /*----------------------------------------------------------------*/
917 
918 /*
919  * The following three macros are to be used in sun4/sun4c code only.
920  */
921 #if defined(SUN4_MMU3L)
922 #define CTX_USABLE(pm,rp) (					\
923 		((pm)->pm_ctx != NULL &&			\
924 		 (!HASSUN4_MMU3L || (rp)->rg_smeg != reginval))	\
925 )
926 #else
927 #define CTX_USABLE(pm,rp)	((pm)->pm_ctx != NULL )
928 #endif
929 
930 #define GAP_WIDEN(pm,vr) do if (CPU_HAS_SUNMMU) {		\
931 	if (vr + 1 == pm->pm_gap_start)				\
932 		pm->pm_gap_start = vr;				\
933 	if (vr == pm->pm_gap_end)				\
934 		pm->pm_gap_end = vr + 1;			\
935 } while (0)
936 
937 #define GAP_SHRINK(pm,vr) do if (CPU_HAS_SUNMMU) {			\
938 	int x;								\
939 	x = pm->pm_gap_start + (pm->pm_gap_end - pm->pm_gap_start) / 2;	\
940 	if (vr > x) {							\
941 		if (vr < pm->pm_gap_end)				\
942 			pm->pm_gap_end = vr;				\
943 	} else {							\
944 		if (vr >= pm->pm_gap_start && x != pm->pm_gap_start)	\
945 			pm->pm_gap_start = vr + 1;			\
946 	}								\
947 } while (0)
948 
949 
950 static void get_phys_mem(void **);
951 #if 0 /* not used */
952 void	kvm_iocache(char *, int);
953 #endif
954 
955 #ifdef DEBUG
956 void	pm_check(char *, struct pmap *);
957 void	pm_check_k(char *, struct pmap *);
958 void	pm_check_u(char *, struct pmap *);
959 #endif
960 
961 /*
962  * During the PMAP bootstrap, we can use a simple translation to map a
963  * kernel virtual address to a psysical memory address (this is arranged
964  * in locore).  Usually, KERNBASE maps to physical address 0. This is always
965  * the case on sun4 and sun4c machines. On sun4m machines -- if no memory is
966  * installed in the bank corresponding to physical address 0 -- the PROM may
967  * elect to load us at some other address, presumably at the start of
968  * the first memory bank that is available. We set the up the variable
969  * `va2pa_offset' to hold the physical address corresponding to KERNBASE.
970  */
971 
972 static u_long va2pa_offset;
973 #define PMAP_BOOTSTRAP_VA2PA(v) ((paddr_t)((u_long)(v) - va2pa_offset))
974 #define PMAP_BOOTSTRAP_PA2VA(p) ((vaddr_t)((u_long)(p) + va2pa_offset))
975 
976 /*
977  * Grab physical memory list.
978  * While here, compute `physmem'.
979  */
980 void
981 get_phys_mem(void **top)
982 {
983 	struct memarr *mp;
984 	char *p;
985 	int i;
986 
987 	/* Load the memory descriptor array at the current kernel top */
988 	p = (void *)ALIGN(*top);
989 	pmemarr = (struct memarr *)p;
990 	npmemarr = prom_makememarr(pmemarr, 1000, MEMARR_AVAILPHYS);
991 
992 	/* Update kernel top */
993 	p += npmemarr * sizeof(struct memarr);
994 	*top = p;
995 
996 	for (physmem = 0, mp = pmemarr, i = npmemarr; --i >= 0; mp++)
997 		physmem += btoc(mp->len);
998 }
999 
1000 
1001 /*
1002  * Support functions for vm_page_bootstrap().
1003  */
1004 
1005 /*
1006  * How much virtual space does this kernel have?
1007  * (After mapping kernel text, data, etc.)
1008  */
1009 void
1010 pmap_virtual_space(vaddr_t *v_start, vaddr_t *v_end)
1011 {
1012 
1013         *v_start = virtual_avail;
1014         *v_end   = virtual_end;
1015 }
1016 
1017 #ifdef PMAP_GROWKERNEL
1018 vaddr_t
1019 pmap_growkernel(vaddr_t eva)
1020 {
1021 	struct regmap *rp;
1022 	struct segmap *sp;
1023 	int vr, evr, M, N, i;
1024 	struct vm_page *pg;
1025 	vaddr_t va;
1026 
1027 	if (eva <= virtual_end)
1028 		return (virtual_end);
1029 
1030 	/* For now, only implemented for sun4/sun4c */
1031 	KASSERT(CPU_HAS_SUNMMU);
1032 
1033 	/*
1034 	 * Map in the next region(s)
1035 	 */
1036 
1037 	/* Get current end-of-kernel */
1038 	vr = virtual_end >> RGSHIFT;
1039 	evr = (eva + NBPRG - 1) >> RGSHIFT;
1040 	eva = evr << RGSHIFT;
1041 
1042 	if (eva > VM_MAX_KERNEL_ADDRESS)
1043 		panic("growkernel: grown too large: %lx", eva);
1044 
1045 	/*
1046 	 * Divide a region in N blocks of M segments, where each segment
1047 	 * block can have its PTEs mapped by one page.
1048 	 * N should come out to 1 for 8K pages and to 4 for 4K pages.
1049 	 */
1050 	M = NBPG / (NPTESG * sizeof(int));
1051 	N = (NBPRG/NBPSG) / M;
1052 
1053 	while (vr < evr) {
1054 		rp = &pmap_kernel()->pm_regmap[vr];
1055 		for (i = 0; i < N; i++) {
1056 			sp = &rp->rg_segmap[i * M];
1057 			va = (vaddr_t)sp->sg_pte;
1058 			pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
1059 			if (pg == NULL)
1060 				panic("growkernel: out of memory");
1061 			pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
1062 					VM_PROT_READ | VM_PROT_WRITE, 0);
1063 		}
1064 	}
1065 
1066 	virtual_end = eva;
1067 	return (eva);
1068 }
1069 #endif
1070 
1071 /*
1072  * Helper routine that hands off available physical pages to the VM system.
1073  */
1074 static void
1075 pmap_page_upload(void)
1076 {
1077 	int	n;
1078 	paddr_t	start, end;
1079 
1080 	/* First, the `etext gap' */
1081 	start = PMAP_BOOTSTRAP_VA2PA(etext_gap_start);
1082 	end = PMAP_BOOTSTRAP_VA2PA(etext_gap_end);
1083 
1084 #ifdef DIAGNOSTIC
1085 	if (avail_start <= start)
1086 		panic("pmap_page_upload: etext gap overlap: %lx < %lx",
1087 			(u_long)avail_start, (u_long)start);
1088 #endif
1089 	if (etext_gap_start < etext_gap_end) {
1090 		vm_first_phys = start;
1091 		uvm_page_physload(
1092 			atop(start),
1093 			atop(end),
1094 			atop(start),
1095 			atop(end), VM_FREELIST_DEFAULT);
1096 	}
1097 
1098 	for (n = 0; n < npmemarr; n++) {
1099 
1100 		start = pmemarr[n].addr;
1101 		end = start + pmemarr[n].len;
1102 
1103 		/* Update vm_{first_last}_phys */
1104 		if (vm_first_phys > start)
1105 			vm_first_phys = start;
1106 		if (vm_last_phys < end)
1107 			vm_last_phys = end;
1108 
1109 		/*
1110 		 * Exclude any memory allocated for the kernel as computed
1111 		 * by pmap_bootstrap(), i.e. the range
1112 		 *	[KERNBASE_PA, avail_start>.
1113 		 * Note that this will also exclude the `etext gap' range
1114 		 * already uploaded above.
1115 		 */
1116 		if (start < PMAP_BOOTSTRAP_VA2PA(KERNBASE)) {
1117 			/*
1118 			 * This segment starts below the kernel load address.
1119 			 * Chop it off at the start of the kernel.
1120 			 */
1121 			paddr_t	chop = PMAP_BOOTSTRAP_VA2PA(KERNBASE);
1122 
1123 			if (end < chop)
1124 				chop = end;
1125 #ifdef DEBUG
1126 			prom_printf("bootstrap gap: start %lx, chop %lx, end %lx\n",
1127 				start, chop, end);
1128 #endif
1129 			uvm_page_physload(
1130 				atop(start),
1131 				atop(chop),
1132 				atop(start),
1133 				atop(chop),
1134 				VM_FREELIST_DEFAULT);
1135 
1136 			/*
1137 			 * Adjust the start address to reflect the
1138 			 * uploaded portion of this segment.
1139 			 */
1140 			start = chop;
1141 		}
1142 
1143 		/* Skip the current kernel address range */
1144 		if (start <= avail_start && avail_start < end)
1145 			start = avail_start;
1146 
1147 		if (start == end)
1148 			continue;
1149 
1150 		/* Upload (the rest of) this segment */
1151 		uvm_page_physload(
1152 			atop(start),
1153 			atop(end),
1154 			atop(start),
1155 			atop(end), VM_FREELIST_DEFAULT);
1156 	}
1157 
1158 #if defined(MULTIPROCESSOR)
1159 	{
1160 		CPU_INFO_ITERATOR cpunum;
1161 		struct cpu_info *cpi;
1162 
1163 		for (CPU_INFO_FOREACH(cpunum, cpi)) {
1164 			if (cpi->ci_free_sva1)
1165 				uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)),
1166 						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)),
1167 						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)),
1168 						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)),
1169 						  VM_FREELIST_DEFAULT);
1170 			if (cpi->ci_free_sva2)
1171 				uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)),
1172 						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)),
1173 						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)),
1174 						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)),
1175 						  VM_FREELIST_DEFAULT);
1176 		}
1177 	}
1178 #endif
1179 }
1180 
1181 /*
1182  * This routine is used by mmrw() to validate access to `/dev/mem'.
1183  */
1184 int
1185 pmap_pa_exists(paddr_t pa)
1186 {
1187 	int nmem;
1188 	struct memarr *mp;
1189 
1190 	for (mp = pmemarr, nmem = npmemarr; --nmem >= 0; mp++) {
1191 		if (pa >= mp->addr && pa < mp->addr + mp->len)
1192 			return 1;
1193 	}
1194 
1195 	return 0;
1196 }
1197 
1198 /* update pv_flags given a valid pte */
1199 #define	MR4_4C(pte) (((pte) >> PG_M_SHIFT) & (PV_MOD | PV_REF))
1200 #define MR4M(pte) (((pte) >> PG_M_SHIFT4M) & (PV_MOD4M | PV_REF4M))
1201 
1202 /*----------------------------------------------------------------*/
1203 
1204 /*
1205  * Agree with the monitor ROM as to how many MMU entries are
1206  * to be reserved, and map all of its segments into all contexts.
1207  *
1208  * Unfortunately, while the Version 0 PROM had a nice linked list of
1209  * taken virtual memory, the Version 2 PROM provides instead a convoluted
1210  * description of *free* virtual memory.  Rather than invert this, we
1211  * resort to two magic constants from the PROM vector description file.
1212  */
1213 #if defined(SUN4) || defined(SUN4C)
1214 void
1215 mmu_reservemon4_4c(int *nrp, int *nsp)
1216 {
1217 	u_int va = 0, eva = 0;
1218 	int mmuseg, i, nr, ns, vr, lastvr;
1219 	int *pte;
1220 #if defined(SUN4_MMU3L)
1221 	int mmureg;
1222 #endif
1223 	struct regmap *rp;
1224 
1225 #if defined(SUN4)
1226 	if (CPU_ISSUN4) {
1227 		prom_vstart = va = OLDMON_STARTVADDR;
1228 		prom_vend = eva = OLDMON_ENDVADDR;
1229 	}
1230 #endif
1231 #if defined(SUN4C)
1232 	if (CPU_ISSUN4C) {
1233 		prom_vstart = va = OPENPROM_STARTVADDR;
1234 		prom_vend = eva = OPENPROM_ENDVADDR;
1235 	}
1236 #endif
1237 	ns = *nsp;
1238 	nr = *nrp;
1239 	lastvr = 0;
1240 	while (va < eva) {
1241 		vr = VA_VREG(va);
1242 		rp = &pmap_kernel()->pm_regmap[vr];
1243 
1244 #if defined(SUN4_MMU3L)
1245 		if (HASSUN4_MMU3L && vr != lastvr) {
1246 			lastvr = vr;
1247 			mmureg = getregmap(va);
1248 			if (mmureg < nr)
1249 				rp->rg_smeg = nr = mmureg;
1250 			/*
1251 			 * On 3-level MMU machines, we distribute regions,
1252 			 * rather than segments, amongst the contexts.
1253 			 */
1254 			for (i = ncontext; --i > 0;)
1255 				prom_setcontext(i, (void *)va, mmureg);
1256 		}
1257 #endif
1258 		mmuseg = getsegmap(va);
1259 		if (mmuseg < ns)
1260 			ns = mmuseg;
1261 
1262 		if (!HASSUN4_MMU3L)
1263 			for (i = ncontext; --i > 0;)
1264 				prom_setcontext(i, (void *)va, mmuseg);
1265 
1266 		if (mmuseg == seginval) {
1267 			va += NBPSG;
1268 			continue;
1269 		}
1270 		/*
1271 		 * Another PROM segment. Enter into region map.
1272 		 * Assume the entire segment is valid.
1273 		 */
1274 		rp->rg_nsegmap += 1;
1275 		rp->rg_segmap[VA_VSEG(va)].sg_pmeg = mmuseg;
1276 		rp->rg_segmap[VA_VSEG(va)].sg_npte = NPTESG;
1277 		pte = rp->rg_segmap[VA_VSEG(va)].sg_pte;
1278 
1279 		/* PROM maps its memory user-accessible: fix it. */
1280 		for (i = NPTESG; --i >= 0; va += NBPG, pte++) {
1281 			*pte = getpte4(va) | PG_S;
1282 			setpte4(va, *pte);
1283 		}
1284 	}
1285 	*nsp = ns;
1286 	*nrp = nr;
1287 	return;
1288 }
1289 #endif
1290 
1291 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */
1292 
1293 u_long
1294 srmmu_bypass_read(u_long paddr)
1295 {
1296 	unsigned long v;
1297 
1298 	if (cpuinfo.mxcc) {
1299 		/*
1300 		 * We're going to have to use MMU passthrough. If we're on
1301 		 * a Viking SuperSPARC with a MultiCache Controller, we
1302 		 * need to set the AC (Alternate Cacheable) bit in the MMU's
1303 		 * control register in order to not by-pass the cache.
1304 		 */
1305 
1306 		unsigned long s = lda(SRMMU_PCR, ASI_SRMMU);
1307 
1308 		/* set MMU AC bit */
1309 		sta(SRMMU_PCR, ASI_SRMMU, s | VIKING_PCR_AC);
1310 		v = lda(paddr, ASI_BYPASS);
1311 		sta(SRMMU_PCR, ASI_SRMMU, s);
1312 	} else
1313 		v = lda(paddr, ASI_BYPASS);
1314 
1315 	return (v);
1316 }
1317 
1318 
1319 /*
1320  * Take the monitor's initial page table layout, convert it to 3rd-level pte's
1321  * (it starts out as a L1 mapping), and install it along with a set of kernel
1322  * mapping tables as the kernel's initial page table setup. Also create and
1323  * enable a context table. I suppose we also want to block user-mode access
1324  * to the new kernel/ROM mappings.
1325  */
1326 
1327 /*
1328  * mmu_reservemon4m(): Copies the existing (ROM) page tables to kernel space,
1329  * converting any L1/L2 PTEs to L3 PTEs. Does *not* copy the L1 entry mapping
1330  * the kernel at KERNBASE since we don't want to map 16M of physical
1331  * memory for the kernel. Thus the kernel must be installed later!
1332  * Also installs ROM mappings into the kernel pmap.
1333  * NOTE: This also revokes all user-mode access to the mapped regions.
1334  */
1335 void
1336 mmu_reservemon4m(struct pmap *kpmap)
1337 {
1338 	unsigned int rom_ctxtbl;
1339 	int te;
1340 
1341 #if !(defined(PROM_AT_F0) || defined(MSIIEP))
1342 	prom_vstart = OPENPROM_STARTVADDR;
1343 	prom_vend = OPENPROM_ENDVADDR;
1344 #else /* OBP3/OFW in JavaStations */
1345 	prom_vstart = 0xf0000000;
1346 #if defined(MSIIEP)
1347 	prom_vend = 0xf0800000;
1348 #else
1349 	prom_vend = 0xf0080000;
1350 #endif
1351 #endif
1352 
1353 	/*
1354 	 * XXX: although the sun4m can handle 36 bits of physical
1355 	 * address space, we assume that all these page tables, etc
1356 	 * are in the lower 4G (32-bits) of address space, i.e. out of I/O
1357 	 * space. Eventually this should be changed to support the 36 bit
1358 	 * physical addressing, in case some crazed ROM designer decides to
1359 	 * stick the pagetables up there. In that case, we should use MMU
1360 	 * transparent mode, (i.e. ASI 0x20 to 0x2f) to access
1361 	 * physical memory.
1362 	 */
1363 
1364 	rom_ctxtbl = (lda(SRMMU_CXTPTR,ASI_SRMMU) << SRMMU_PPNPASHIFT);
1365 
1366 	te = srmmu_bypass_read(rom_ctxtbl);	/* i.e. context 0 */
1367 
1368 	switch (te & SRMMU_TETYPE) {
1369 	case SRMMU_TEINVALID:
1370 		cpuinfo.ctx_tbl[0] = SRMMU_TEINVALID;
1371 		panic("mmu_reservemon4m: no existing L0 mapping! "
1372 		      "(How are we running?");
1373 		break;
1374 	case SRMMU_TEPTE:
1375 		panic("mmu_reservemon4m: can't handle ROM 4G page size");
1376 		/* XXX: Should make this work, however stupid it is */
1377 		break;
1378 	case SRMMU_TEPTD:
1379 		mmu_setup4m_L1(te, kpmap);
1380 		break;
1381 	default:
1382 		panic("mmu_reservemon4m: unknown pagetable entry type");
1383 	}
1384 }
1385 
1386 /* regtblptd - PTD for region table to be remapped */
1387 void
1388 mmu_setup4m_L1(int regtblptd, struct pmap *kpmap)
1389 {
1390 	unsigned int regtblrover;
1391 	int i;
1392 	unsigned int te;
1393 	struct regmap *rp;
1394 	int j, k;
1395 
1396 	/*
1397 	 * Here we scan the region table to copy any entries which appear.
1398 	 * We are only concerned with regions in kernel space and above
1399 	 * (i.e. regions VA_VREG(KERNBASE)+1 to 0xff). We ignore the first
1400 	 * region (at VA_VREG(KERNBASE)), since that is the 16MB L1 mapping
1401 	 * that the ROM used to map the kernel in initially. Later, we will
1402 	 * rebuild a new L3 mapping for the kernel and install it before
1403 	 * switching to the new pagetables.
1404 	 */
1405 	regtblrover =
1406 		((regtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT) +
1407 		(VA_VREG(KERNBASE)+1) * sizeof(long);	/* kernel only */
1408 
1409 	for (i = VA_VREG(KERNBASE) + 1; i < SRMMU_L1SIZE;
1410 	     i++, regtblrover += sizeof(long)) {
1411 
1412 		/* The region we're dealing with */
1413 		rp = &kpmap->pm_regmap[i];
1414 
1415 		te = srmmu_bypass_read(regtblrover);
1416 		switch(te & SRMMU_TETYPE) {
1417 		case SRMMU_TEINVALID:
1418 			break;
1419 
1420 		case SRMMU_TEPTE:
1421 #ifdef DEBUG
1422 			prom_printf("mmu_setup4m_L1: "
1423 			       "converting region 0x%x from L1->L3\n", i);
1424 #endif
1425 			/*
1426 			 * This region entry covers 64MB of memory -- or
1427 			 * (NSEGRG * NPTESG) pages -- which we must convert
1428 			 * into a 3-level description.
1429 			 */
1430 
1431 			for (j = 0; j < SRMMU_L2SIZE; j++) {
1432 				struct segmap *sp = &rp->rg_segmap[j];
1433 
1434 				for (k = 0; k < SRMMU_L3SIZE; k++) {
1435 					setpgt4m(&sp->sg_pte[k],
1436 						(te & SRMMU_L1PPNMASK) |
1437 						(j << SRMMU_L2PPNSHFT) |
1438 						(k << SRMMU_L3PPNSHFT) |
1439 						(te & SRMMU_PGBITSMSK) |
1440 						((te & SRMMU_PROT_MASK) |
1441 						 PPROT_U2S_OMASK) |
1442 						SRMMU_TEPTE);
1443 				}
1444 			}
1445 			break;
1446 
1447 		case SRMMU_TEPTD:
1448 			mmu_setup4m_L2(te, rp);
1449 			break;
1450 
1451 		default:
1452 			panic("mmu_setup4m_L1: unknown pagetable entry type");
1453 		}
1454 	}
1455 }
1456 
1457 void
1458 mmu_setup4m_L2(int segtblptd, struct regmap *rp)
1459 {
1460 	unsigned int segtblrover;
1461 	int i, k;
1462 	unsigned int te;
1463 	struct segmap *sp;
1464 
1465 	segtblrover = (segtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT;
1466 	for (i = 0; i < SRMMU_L2SIZE; i++, segtblrover += sizeof(long)) {
1467 
1468 		sp = &rp->rg_segmap[i];
1469 
1470 		te = srmmu_bypass_read(segtblrover);
1471 		switch(te & SRMMU_TETYPE) {
1472 		case SRMMU_TEINVALID:
1473 			break;
1474 
1475 		case SRMMU_TEPTE:
1476 #ifdef DEBUG
1477 			prom_printf("mmu_setup4m_L2: converting L2 entry at segment 0x%x to L3\n",i);
1478 #endif
1479 			/*
1480 			 * This segment entry covers 256KB of memory -- or
1481 			 * (NPTESG) pages -- which we must convert
1482 			 * into a 3-level description.
1483 			 */
1484 			for (k = 0; k < SRMMU_L3SIZE; k++) {
1485 				setpgt4m(&sp->sg_pte[k],
1486 					(te & SRMMU_L1PPNMASK) |
1487 					(te & SRMMU_L2PPNMASK) |
1488 					(k << SRMMU_L3PPNSHFT) |
1489 					(te & SRMMU_PGBITSMSK) |
1490 					((te & SRMMU_PROT_MASK) |
1491 					 PPROT_U2S_OMASK) |
1492 					SRMMU_TEPTE);
1493 			}
1494 			break;
1495 
1496 		case SRMMU_TEPTD:
1497 			mmu_setup4m_L3(te, sp);
1498 			break;
1499 
1500 		default:
1501 			panic("mmu_setup4m_L2: unknown pagetable entry type");
1502 		}
1503 	}
1504 }
1505 
1506 void
1507 mmu_setup4m_L3(int pagtblptd, struct segmap *sp)
1508 {
1509 	unsigned int pagtblrover;
1510 	int i;
1511 	unsigned int te;
1512 
1513 	pagtblrover = (pagtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT;
1514 	for (i = 0; i < SRMMU_L3SIZE; i++, pagtblrover += sizeof(long)) {
1515 		te = srmmu_bypass_read(pagtblrover);
1516 		switch(te & SRMMU_TETYPE) {
1517 		case SRMMU_TEINVALID:
1518 			break;
1519 		case SRMMU_TEPTE:
1520 			setpgt4m(&sp->sg_pte[i], te | PPROT_U2S_OMASK);
1521 			pmap_kernel()->pm_stats.resident_count++;
1522 			break;
1523 		case SRMMU_TEPTD:
1524 			panic("mmu_setup4m_L3: PTD found in L3 page table");
1525 		default:
1526 			panic("mmu_setup4m_L3: unknown pagetable entry type");
1527 		}
1528 	}
1529 }
1530 #endif /* defined SUN4M || defined SUN4D */
1531 
1532 /*----------------------------------------------------------------*/
1533 
1534 #if defined(SUN4) || defined(SUN4C)
1535 /*
1536  * MMU management.
1537  */
1538 static int	me_alloc(struct mmuq *, struct pmap *, int, int);
1539 static void	me_free(struct pmap *, u_int);
1540 #if defined(SUN4_MMU3L)
1541 static int	region_alloc(struct mmuq *, struct pmap *, int);
1542 static void	region_free(struct pmap *, u_int);
1543 #endif
1544 
1545 
1546 /*
1547  * Allocate an MMU entry (i.e., a PMEG).
1548  * If necessary, steal one from someone else.
1549  * Put it on the tail of the given queue
1550  * (which is either the LRU list or the locked list).
1551  * The locked list is not actually ordered, but this is easiest.
1552  * Also put it on the given (new) pmap's chain,
1553  * enter its pmeg number into that pmap's segmap,
1554  * and store the pmeg's new virtual segment number (me->me_vseg).
1555  *
1556  * This routine is large and complicated, but it must be fast
1557  * since it implements the dynamic allocation of MMU entries.
1558  */
1559 
1560 static inline int
1561 me_alloc(struct mmuq *mh, struct pmap *newpm, int newvreg, int newvseg)
1562 {
1563 	struct mmuentry *me;
1564 	struct pmap *pm;
1565 	int i, va, *ptep, pte;
1566 	int ctx;
1567 	struct regmap *rp;
1568 	struct segmap *sp;
1569 
1570 	/* try free list first */
1571 	if (!MMUQ_EMPTY(&segm_freelist)) {
1572 		me = MMUQ_FIRST(&segm_freelist);
1573 		MMUQ_REMOVE(me, me_list);
1574 #ifdef DEBUG
1575 		if (me->me_pmap != NULL)
1576 			panic("me_alloc: freelist entry has pmap");
1577 		if (pmapdebug & PDB_MMU_ALLOC)
1578 			printf("me_alloc: got pmeg %d\n", me->me_cookie);
1579 #endif
1580 		MMUQ_INSERT_TAIL(mh, me, me_list);
1581 
1582 		/* onto on pmap chain; pmap is already locked, if needed */
1583 		TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain);
1584 #ifdef DIAGNOSTIC
1585 		pmap_stats.ps_npmeg_free--;
1586 		if (mh == &segm_locked) {
1587 			pmap_stats.ps_npmeg_locked++;
1588 			me->me_statp = &pmap_stats.ps_npmeg_locked;
1589 		} else {
1590 			pmap_stats.ps_npmeg_lru++;
1591 			me->me_statp = &pmap_stats.ps_npmeg_lru;
1592 		}
1593 #endif
1594 
1595 		/* into pmap segment table, with backpointers */
1596 		me->me_pmap = newpm;
1597 		me->me_vseg = newvseg;
1598 		me->me_vreg = newvreg;
1599 
1600 		return (me->me_cookie);
1601 	}
1602 
1603 	/* no luck, take head of LRU list */
1604 	if (MMUQ_EMPTY(&segm_lru))
1605 		panic("me_alloc: all pmegs gone");
1606 
1607 	me = MMUQ_FIRST(&segm_lru);
1608 	pm = me->me_pmap;
1609 #ifdef DEBUG
1610 	if (pmapdebug & (PDB_MMU_ALLOC | PDB_MMU_STEAL))
1611 		printf("me_alloc: stealing pmeg 0x%x from pmap %p\n",
1612 		    me->me_cookie, pm);
1613 #endif
1614 
1615 	mmu_stolenpmegs_evcnt.ev_count++;
1616 
1617 	/*
1618 	 * Remove from LRU list, and insert at end of new list
1619 	 * (probably the LRU list again, but so what?).
1620 	 */
1621 	MMUQ_REMOVE(me, me_list);
1622 	MMUQ_INSERT_TAIL(mh, me, me_list);
1623 
1624 #ifdef DIAGNOSTIC
1625 	if (mh == &segm_locked) {
1626 		pmap_stats.ps_npmeg_lru--;
1627 		pmap_stats.ps_npmeg_locked++;
1628 		me->me_statp = &pmap_stats.ps_npmeg_locked;
1629 	} else {
1630 		me->me_statp = &pmap_stats.ps_npmeg_lru;
1631 	}
1632 #endif
1633 
1634 	rp = &pm->pm_regmap[me->me_vreg];
1635 	sp = &rp->rg_segmap[me->me_vseg];
1636 	ptep = sp->sg_pte;
1637 
1638 #ifdef DEBUG
1639 	if (sp->sg_pmeg != me->me_cookie)
1640 		panic("me_alloc: wrong sg_pmeg (%d != %d)",
1641 				sp->sg_pmeg, me->me_cookie);
1642 #endif
1643 
1644 	/*
1645 	 * The PMEG must be mapped into some context so that we can
1646 	 * read its PTEs.  Use its current context if it has one;
1647 	 * if not, and since context 0 is reserved for the kernel,
1648 	 * the simplest method is to switch to 0 and map the PMEG
1649 	 * to virtual address 0---which, being a user space address,
1650 	 * is by definition not in use.
1651 	 *
1652 	 * XXX do not have to flush cache immediately
1653 	 */
1654 	ctx = getcontext4();
1655 
1656 	/*
1657 	 * Even if we're stealing a PMEG from ourselves (i.e. if pm==newpm),
1658 	 * we must make sure there are no user register windows in the CPU
1659 	 * for the following reasons:
1660 	 * (1) if we have a write-allocate cache and the segment we are
1661 	 *     stealing contains stack pages, an interrupt during the
1662 	 *     interval that starts at cache_flush_segment() below and ends
1663 	 *     when the segment is finally removed from the MMU, may cause
1664 	 *     dirty cache lines to reappear.
1665 	 * (2) when re-wiring this PMEG for use by another segment (e.g.
1666 	 *     in mmu_pagein()) a window exists where the PTEs in this PMEG
1667 	 *     point at arbitrary pages allocated to this address space.
1668 	 *     Again, a register window flush at this point is likely to
1669 	 *     cause data corruption in case the segment being rewired
1670 	 *     contains stack virtual addresses.
1671 	 */
1672 	write_user_windows();
1673 	if (CTX_USABLE(pm,rp)) {
1674 		setcontext4(pm->pm_ctxnum);
1675 		va = VSTOVA(me->me_vreg, me->me_vseg);
1676 #ifdef DEBUG
1677 		if (getsegmap(va) != me->me_cookie)
1678 			panic("me_alloc: wrong pmeg in MMU (%d != %d)",
1679 				getsegmap(va), me->me_cookie);
1680 #endif
1681 		cache_flush_segment(me->me_vreg, me->me_vseg, pm->pm_ctxnum);
1682 	} else {
1683 		va = 0;
1684 		setcontext4(0);
1685 		if (HASSUN4_MMU3L)
1686 			setregmap(va, tregion);
1687 		setsegmap(va, me->me_cookie);
1688 		/*
1689 		 * No cache flush needed: it happened earlier when
1690 		 * the old context was taken.
1691 		 */
1692 	}
1693 
1694 	/*
1695 	 * Record reference and modify bits for each page,
1696 	 * and copy PTEs into kernel memory so that they can
1697 	 * be reloaded later.
1698 	 */
1699 	i = NPTESG;
1700 	do {
1701 		int swbits = *ptep & PG_MBZ;
1702 		pte = getpte4(va);
1703 		if ((pte & (PG_V | PG_TYPE)) == (PG_V | PG_OBMEM)) {
1704 			struct vm_page *pg;
1705 			if ((pg = pvhead4_4c(pte)) != NULL)
1706 				VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte);
1707 		}
1708 		*ptep++ = swbits | (pte & ~(PG_U|PG_M));
1709 		va += NBPG;
1710 	} while (--i > 0);
1711 
1712 	/* update segment tables */
1713 	if (CTX_USABLE(pm,rp)) {
1714 		va = VSTOVA(me->me_vreg,me->me_vseg);
1715 		if (pm != pmap_kernel() || HASSUN4_MMU3L)
1716 			setsegmap(va, seginval);
1717 		else {
1718 			/* Unmap segment from all contexts */
1719 			for (i = ncontext; --i >= 0;) {
1720 				setcontext4(i);
1721 				setsegmap(va, seginval);
1722 			}
1723 		}
1724 	}
1725 	sp->sg_pmeg = seginval;
1726 
1727 	/* off old pmap chain */
1728 	TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain);
1729 	setcontext4(ctx);
1730 
1731 	/* onto new pmap chain; new pmap is already locked, if needed */
1732 	TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain);
1733 
1734 	/* into new segment table, with backpointers */
1735 	me->me_pmap = newpm;
1736 	me->me_vseg = newvseg;
1737 	me->me_vreg = newvreg;
1738 
1739 	return (me->me_cookie);
1740 }
1741 
1742 /*
1743  * Free an MMU entry.
1744  *
1745  * Assumes the corresponding pmap is already locked.
1746  * Caller must update hardware.
1747  */
1748 static inline void
1749 me_free(struct pmap *pm, u_int pmeg)
1750 {
1751 	struct mmuentry *me = &mmusegments[pmeg];
1752 #ifdef DEBUG
1753 	struct regmap *rp;
1754 	int i, va, tpte, ctx;
1755 #endif
1756 
1757 #ifdef DEBUG
1758 	rp = &pm->pm_regmap[me->me_vreg];
1759 	if (pmapdebug & PDB_MMU_ALLOC)
1760 		printf("me_free: freeing pmeg %d from pmap %p\n",
1761 		    me->me_cookie, pm);
1762 	if (me->me_cookie != pmeg)
1763 		panic("me_free: wrong mmuentry");
1764 	if (pm != me->me_pmap)
1765 		panic("me_free: pm != me_pmap");
1766 	if (rp->rg_segmap[me->me_vseg].sg_pmeg != pmeg &&
1767 	    rp->rg_segmap[me->me_vseg].sg_pmeg != seginval)
1768 		panic("me_free: wrong sg_pmeg (%d != %d)",
1769 			rp->rg_segmap[me->me_vseg].sg_pmeg, pmeg);
1770 
1771 	/* check for spurious mappings (using temp. mapping in context 0) */
1772 	ctx = getcontext4();
1773 	setcontext4(0);
1774 	if (HASSUN4_MMU3L)
1775 		setregmap(0, tregion);
1776 	setsegmap(0, me->me_cookie);
1777 	va = 0;
1778 	i = NPTESG;
1779 	do {
1780 		tpte = getpte4(va);
1781 		if ((tpte & PG_V) == PG_V)
1782 			panic("me_free: segment not clean (pte=%x)", tpte);
1783 		va += NBPG;
1784 	} while (--i > 0);
1785 	setcontext4(ctx);
1786 #endif /* DEBUG */
1787 
1788 	/* take mmu entry off pmap chain */
1789 	TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain);
1790 
1791 	/* off LRU or lock chain */
1792 	MMUQ_REMOVE(me, me_list);
1793 #ifdef DIAGNOSTIC
1794 	if (me->me_statp == NULL)
1795 		panic("me_statp");
1796 	(*me->me_statp)--;
1797 	me->me_statp = NULL;
1798 #endif
1799 
1800 	/* no associated pmap; on free list */
1801 	me->me_pmap = NULL;
1802 	MMUQ_INSERT_TAIL(&segm_freelist, me, me_list);
1803 #ifdef DIAGNOSTIC
1804 	pmap_stats.ps_npmeg_free++;
1805 #endif
1806 }
1807 
1808 #if defined(SUN4_MMU3L)
1809 
1810 /* XXX - Merge with segm_alloc/segm_free ? */
1811 
1812 int
1813 region_alloc(struct mmuq *mh, struct pmap *newpm, int newvr)
1814 {
1815 	struct mmuentry *me;
1816 	struct pmap *pm;
1817 	int ctx;
1818 	struct regmap *rp;
1819 
1820 	/* try free list first */
1821 	if (!MMUQ_EMPTY(&region_freelist)) {
1822 		me = MMUQ_FIRST(&region_freelist);
1823 		MMUQ_REMOVE(me, me_list);
1824 #ifdef DEBUG
1825 		if (me->me_pmap != NULL)
1826 			panic("region_alloc: freelist entry has pmap");
1827 		if (pmapdebug & PDB_MMUREG_ALLOC)
1828 			printf("region_alloc: got smeg 0x%x\n", me->me_cookie);
1829 #endif
1830 		MMUQ_INSERT_TAIL(mh, me, me_list);
1831 
1832 		/* onto on pmap chain; pmap is already locked, if needed */
1833 		TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain);
1834 
1835 		/* into pmap segment table, with backpointers */
1836 		me->me_pmap = newpm;
1837 		me->me_vreg = newvr;
1838 
1839 		return (me->me_cookie);
1840 	}
1841 
1842 	/* no luck, take head of LRU list */
1843 	if (MMUQ_EMPTY(&region_lru))
1844 		panic("region_alloc: all smegs gone");
1845 
1846 	me = MMUQ_FIRST(&region_lru);
1847 
1848 	pm = me->me_pmap;
1849 	if (pm == NULL)
1850 		panic("region_alloc: LRU entry has no pmap");
1851 	if (pm == pmap_kernel())
1852 		panic("region_alloc: stealing from kernel");
1853 #ifdef DEBUG
1854 	if (pmapdebug & (PDB_MMUREG_ALLOC | PDB_MMUREG_STEAL))
1855 		printf("region_alloc: stealing smeg 0x%x from pmap %p\n",
1856 		    me->me_cookie, pm);
1857 #endif
1858 	/*
1859 	 * Remove from LRU list, and insert at end of new list
1860 	 * (probably the LRU list again, but so what?).
1861 	 */
1862 	MMUQ_REMOVE(me, me_list);
1863 	MMUQ_INSERT_TAIL(mh, me, me_list);
1864 
1865 	rp = &pm->pm_regmap[me->me_vreg];
1866 	ctx = getcontext4();
1867 
1868 	/* Flush register windows; see comment in me_alloc() */
1869 	write_user_windows();
1870 	if (pm->pm_ctx) {
1871 		setcontext4(pm->pm_ctxnum);
1872 		cache_flush_region(me->me_vreg, pm->pm_ctxnum);
1873 	}
1874 
1875 	/* update region tables */
1876 	if (pm->pm_ctx)
1877 		setregmap(VRTOVA(me->me_vreg), reginval);
1878 	rp->rg_smeg = reginval;
1879 
1880 	/* off old pmap chain */
1881 	TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain);
1882 	setcontext4(ctx);	/* done with old context */
1883 
1884 	/* onto new pmap chain; new pmap is already locked, if needed */
1885 	TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain);
1886 
1887 	/* into new segment table, with backpointers */
1888 	me->me_pmap = newpm;
1889 	me->me_vreg = newvr;
1890 
1891 	return (me->me_cookie);
1892 }
1893 
1894 /*
1895  * Free an MMU entry.
1896  * Assumes the corresponding pmap is already locked.
1897  * Caller must update hardware.
1898  */
1899 void
1900 region_free(struct pmap *pm, u_int smeg)
1901 {
1902 	struct mmuentry *me = &mmuregions[smeg];
1903 
1904 #ifdef DEBUG
1905 	if (pmapdebug & PDB_MMUREG_ALLOC)
1906 		printf("region_free: freeing smeg 0x%x from pmap %p\n",
1907 		    me->me_cookie, pm);
1908 	if (me->me_cookie != smeg)
1909 		panic("region_free: wrong mmuentry");
1910 	if (pm != me->me_pmap)
1911 		panic("region_free: pm != me_pmap");
1912 #endif
1913 
1914 	/* take mmu entry off pmap chain */
1915 	TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain);
1916 
1917 	/* off LRU or lock chain */
1918 	MMUQ_REMOVE(me, me_list);
1919 
1920 	/* no associated pmap; on free list */
1921 	me->me_pmap = NULL;
1922 	MMUQ_INSERT_TAIL(&region_freelist, me, me_list);
1923 }
1924 
1925 static void
1926 mmu_pagein_reg(struct pmap *pm, struct regmap *rp, vaddr_t va,
1927 		int vr, struct mmuq *mh)
1928 {
1929 	int i, s, smeg;
1930 
1931 	va = VA_ROUNDDOWNTOREG(va);
1932 	rp->rg_smeg = smeg = region_alloc(mh, pm, vr);
1933 
1934 	s = splvm();
1935 	if (pm == pmap_kernel()) {
1936 		/* Map region into all contexts */
1937 		int ctx = getcontext4();
1938 		i = ncontext - 1;
1939 		do {
1940 			setcontext4(i);
1941 			setregmap(va, smeg);
1942 		} while (--i >= 0);
1943 		setcontext4(ctx);
1944 	} else
1945 		setregmap(va, smeg);
1946 
1947 	/* Load PMEGs into this region */
1948 	for (i = 0; i < NSEGRG; i++) {
1949 		setsegmap(va, rp->rg_segmap[i].sg_pmeg);
1950 		va += NBPSG;
1951 	}
1952 	splx(s);
1953 }
1954 #endif /* SUN4_MMU3L */
1955 
1956 static void
1957 mmu_pmeg_lock(int pmeg)
1958 {
1959 	struct mmuentry *me = &mmusegments[pmeg];
1960 
1961 	MMUQ_REMOVE(me, me_list);
1962 	MMUQ_INSERT_TAIL(&segm_locked, me, me_list);
1963 #ifdef DIAGNOSTIC
1964 	(*me->me_statp)--;
1965 	pmap_stats.ps_npmeg_locked++;
1966 	me->me_statp = &pmap_stats.ps_npmeg_locked;
1967 #endif
1968 }
1969 
1970 static void
1971 mmu_pmeg_unlock(int pmeg)
1972 {
1973 	struct mmuentry *me = &mmusegments[pmeg];
1974 
1975 	MMUQ_REMOVE(me, me_list);
1976 	MMUQ_INSERT_TAIL(&segm_lru, me, me_list);
1977 #ifdef DIAGNOSTIC
1978 	(*me->me_statp)--;
1979 	pmap_stats.ps_npmeg_lru++;
1980 	me->me_statp = &pmap_stats.ps_npmeg_lru;
1981 #endif
1982 }
1983 
1984 static void
1985 mmu_pagein_seg(struct pmap *pm, struct segmap *sp, vaddr_t va,
1986 		int vr, int vs, struct mmuq *mh)
1987 {
1988 	int s, i, pmeg, *pte;
1989 
1990 	mmu_pagein_evcnt.ev_count++;
1991 
1992 	va = VA_ROUNDDOWNTOSEG(va);
1993 	s = splvm();		/* paranoid */
1994 	sp->sg_pmeg = pmeg = me_alloc(mh, pm, vr, vs);
1995 	if (pm != pmap_kernel() || HASSUN4_MMU3L)
1996 		setsegmap(va, pmeg);
1997 	else {
1998 		/* Map kernel address into all contexts */
1999 		int ctx = getcontext4();
2000 		i = ncontext - 1;
2001 		do {
2002 			setcontext4(i);
2003 			setsegmap(va, pmeg);
2004 		} while (--i >= 0);
2005 		setcontext4(ctx);
2006 	}
2007 
2008 	/* reload segment: write PTEs into a the MMU */
2009 	pte = sp->sg_pte;
2010 	i = NPTESG;
2011 	do {
2012 		setpte4(va, *pte++ & ~PG_MBZ);
2013 		va += NBPG;
2014 	} while (--i > 0);
2015 	splx(s);
2016 }
2017 
2018 /*
2019  * `Page in' (load or inspect) an MMU entry; called on page faults.
2020  * Returns 1 if we reloaded the segment, -1 if the segment was
2021  * already loaded and the page was marked valid (in which case the
2022  * fault must be a bus error or something), or 0 (segment loaded but
2023  * PTE not valid, or segment not loaded at all).
2024  */
2025 int
2026 mmu_pagein(struct pmap *pm, vaddr_t va, int prot)
2027 {
2028 	int vr, vs, bits;
2029 	struct regmap *rp;
2030 	struct segmap *sp;
2031 
2032 	PMAP_LOCK();
2033 
2034 	if (prot != VM_PROT_NONE)
2035 		bits = PG_V | ((prot & VM_PROT_WRITE) ? PG_W : 0);
2036 	else
2037 		bits = 0;
2038 
2039 	vr = VA_VREG(va);
2040 	vs = VA_VSEG(va);
2041 	rp = &pm->pm_regmap[vr];
2042 
2043 	/* return 0 if we have no PMEGs to load */
2044 	if (rp->rg_nsegmap == 0) {
2045 		PMAP_UNLOCK();
2046 		return (0);
2047 	}
2048 
2049 #ifdef DIAGNOSTIC
2050 	if (rp->rg_segmap == NULL)
2051 		panic("pagein: no segmap");
2052 #endif
2053 
2054 #if defined(SUN4_MMU3L)
2055 	if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
2056 		mmu_pagein_reg(pm, rp, va, vr, &region_lru);
2057 #endif
2058 	sp = &rp->rg_segmap[vs];
2059 
2060 	/* return 0 if we have no PTEs to load */
2061 	if (sp->sg_npte == 0) {
2062 		PMAP_UNLOCK();
2063 		return (0);
2064 	}
2065 
2066 	/* return -1 if the fault is `hard', 0 if not */
2067 	if (sp->sg_pmeg != seginval) {
2068 		PMAP_UNLOCK();
2069 		return (bits && (getpte4(va) & bits) == bits ? -1 : 0);
2070 	}
2071 
2072 	mmu_pagein_seg(pm, sp, va, vr, vs, &segm_lru);
2073 	PMAP_UNLOCK();
2074 	return (1);
2075 }
2076 #endif /* SUN4 or SUN4C */
2077 
2078 /*
2079  * Allocate a context.  If necessary, steal one from someone else.
2080  * Changes hardware context number and loads segment map.
2081  *
2082  * This routine is only ever called from locore.s just after it has
2083  * saved away the previous process, so there are no active user windows.
2084  */
2085 void
2086 ctx_alloc(struct pmap *pm)
2087 {
2088 	union ctxinfo *c;
2089 	int cnum, i = 0, doflush;
2090 	struct regmap *rp;
2091 	int gap_start, gap_end;
2092 	vaddr_t va;
2093 #if defined(SUN4M) || defined(SUN4D)
2094 	struct cpu_info *cpi;
2095 #endif
2096 
2097 /*XXX-GCC!*/gap_start=gap_end=0;
2098 #ifdef DEBUG
2099 	if (pm->pm_ctx)
2100 		panic("ctx_alloc pm_ctx");
2101 	if (pmapdebug & PDB_CTX_ALLOC)
2102 		printf("ctx_alloc[%d](%p)\n", cpu_number(), pm);
2103 #endif
2104 	if (CPU_HAS_SUNMMU) {
2105 		gap_start = pm->pm_gap_start;
2106 		gap_end = pm->pm_gap_end;
2107 	}
2108 
2109 	mutex_spin_enter(&ctx_lock);
2110 	if ((c = ctx_freelist) != NULL) {
2111 		ctx_freelist = c->c_nextfree;
2112 		cnum = c - ctxinfo;
2113 		doflush = 0;
2114 	} else {
2115 		if ((ctx_kick += ctx_kickdir) >= ncontext) {
2116 			ctx_kick = ncontext - 1;
2117 			ctx_kickdir = -1;
2118 		} else if (ctx_kick < 1) {
2119 			ctx_kick = 1;
2120 			ctx_kickdir = 1;
2121 		}
2122 		c = &ctxinfo[cnum = ctx_kick];
2123 #ifdef DEBUG
2124 		if (c->c_pmap == NULL)
2125 			panic("ctx_alloc cu_pmap");
2126 		if (pmapdebug & (PDB_CTX_ALLOC | PDB_CTX_STEAL))
2127 			printf("ctx_alloc[%d]: steal context %d from %p\n",
2128 			    cpu_number(), cnum, c->c_pmap);
2129 #endif
2130 		c->c_pmap->pm_ctx = NULL;
2131 		c->c_pmap->pm_ctxnum = 0;
2132 		doflush = (CACHEINFO.c_vactype != VAC_NONE);
2133 		if (CPU_HAS_SUNMMU) {
2134 			if (gap_start < c->c_pmap->pm_gap_start)
2135 				gap_start = c->c_pmap->pm_gap_start;
2136 			if (gap_end > c->c_pmap->pm_gap_end)
2137 				gap_end = c->c_pmap->pm_gap_end;
2138 		}
2139 	}
2140 
2141 	c->c_pmap = pm;
2142 	pm->pm_ctx = c;
2143 	pm->pm_ctxnum = cnum;
2144 
2145 	if (CPU_HAS_SUNMMU) {
2146 
2147 		/*
2148 		 * Write pmap's region (3-level MMU) or segment table into
2149 		 * the MMU.
2150 		 *
2151 		 * Only write those entries that actually map something in
2152 		 * this context by maintaining a pair of region numbers in
2153 		 * between which the pmap has no valid mappings.
2154 		 *
2155 		 * If a context was just allocated from the free list, trust
2156 		 * that all its pmeg numbers are `seginval'. We make sure this
2157 		 * is the case initially in pmap_bootstrap(). Otherwise, the
2158 		 * context was freed by calling ctx_free() in pmap_release(),
2159 		 * which in turn is supposedly called only when all mappings
2160 		 * have been removed.
2161 		 *
2162 		 * On the other hand, if the context had to be stolen from
2163 		 * another pmap, we possibly shrink the gap to be the
2164 		 * disjuction of the new and the previous map.
2165 		 */
2166 
2167 		setcontext4(cnum);
2168 		if (doflush)
2169 			cache_flush_context(cnum);
2170 
2171 		rp = pm->pm_regmap;
2172 		for (va = 0, i = NUREG; --i >= 0; ) {
2173 			if (VA_VREG(va) >= gap_start) {
2174 				va = VRTOVA(gap_end);
2175 				i -= gap_end - gap_start;
2176 				rp += gap_end - gap_start;
2177 				if (i < 0)
2178 					break;
2179 				/* mustn't re-enter this branch */
2180 				gap_start = NUREG;
2181 			}
2182 			if (HASSUN4_MMU3L) {
2183 				setregmap(va, rp++->rg_smeg);
2184 				va += NBPRG;
2185 			} else {
2186 				int j;
2187 				struct segmap *sp = rp->rg_segmap;
2188 				for (j = NSEGRG; --j >= 0; va += NBPSG)
2189 					setsegmap(va,
2190 						  sp?sp++->sg_pmeg:seginval);
2191 				rp++;
2192 			}
2193 		}
2194 
2195 	} else if (CPU_HAS_SRMMU) {
2196 
2197 #if defined(SUN4M) || defined(SUN4D)
2198 		/*
2199 		 * Reload page and context tables to activate the page tables
2200 		 * for this context.
2201 		 *
2202 		 * The gap stuff isn't really needed in the sun4m architecture,
2203 		 * since we don't have to worry about excessive mappings (all
2204 		 * mappings exist since the page tables must be complete for
2205 		 * the mmu to be happy).
2206 		 *
2207 		 * If a context was just allocated from the free list, trust
2208 		 * that all of its mmu-edible page tables are zeroed out
2209 		 * (except for those associated with the kernel). We make
2210 		 * sure this is the case initially in pmap_bootstrap() and
2211 		 * pmap_init() (?).
2212 		 * Otherwise, the context was freed by calling ctx_free() in
2213 		 * pmap_release(), which in turn is supposedly called only
2214 		 * when all mappings have been removed.
2215 		 *
2216 		 * XXX: Do we have to flush cache after reloading ctx tbl?
2217 		 */
2218 
2219 		/*
2220 		 * We need to flush the cache only when stealing a context
2221 		 * from another pmap. In that case it's Ok to switch the
2222 		 * context and leave it set, since the context table
2223 		 * will have a valid region table entry for this context
2224 		 * number.
2225 		 *
2226 		 * Otherwise, we switch to the new context after loading
2227 		 * the context table entry with the new pmap's region.
2228 		 */
2229 		if (doflush) {
2230 			cache_flush_context(cnum);
2231 		}
2232 
2233 		/*
2234 		 * The context allocated to a process is the same on all CPUs.
2235 		 * Here we install the per-CPU region table in each CPU's
2236 		 * context table slot.
2237 		 *
2238 		 * Note on multi-threaded processes: a context must remain
2239 		 * valid as long as any thread is still running on a CPU.
2240 		 */
2241 		for (CPU_INFO_FOREACH(i, cpi)) {
2242 			setpgt4m(&cpi->ctx_tbl[cnum],
2243 				 (pm->pm_reg_ptps_pa[i] >> SRMMU_PPNPASHIFT) |
2244 					SRMMU_TEPTD);
2245 		}
2246 
2247 		/* And finally switch to the new context */
2248 		(*cpuinfo.pure_vcache_flush)();
2249 		setcontext4m(cnum);
2250 #endif /* SUN4M || SUN4D */
2251 	}
2252 	mutex_spin_exit(&ctx_lock);
2253 }
2254 
2255 /*
2256  * Give away a context.
2257  */
2258 void
2259 ctx_free(struct pmap *pm)
2260 {
2261 	union ctxinfo *c;
2262 	int ctx;
2263 #if defined(SUN4M) || defined(SUN4D)
2264 	struct cpu_info *cpi;
2265 #endif
2266 
2267 	c = pm->pm_ctx;
2268 	ctx = pm->pm_ctxnum;
2269 	pm->pm_ctx = NULL;
2270 	pm->pm_ctxnum = 0;
2271 #if defined(SUN4) || defined(SUN4C)
2272 	if (CPU_HAS_SUNMMU) {
2273 		int octx = getcontext4();
2274 		setcontext4(ctx);
2275 		cache_flush_context(ctx);
2276 		setcontext4(octx);
2277 	}
2278 #endif /* SUN4 || SUN4C */
2279 
2280 	mutex_spin_enter(&ctx_lock);
2281 
2282 #if defined(SUN4M) || defined(SUN4D)
2283 	if (CPU_HAS_SRMMU) {
2284 		int i;
2285 
2286 		cache_flush_context(ctx);
2287 		tlb_flush_context(ctx, PMAP_CPUSET(pm));
2288 		for (CPU_INFO_FOREACH(i, cpi)) {
2289 			setpgt4m(&cpi->ctx_tbl[ctx], SRMMU_TEINVALID);
2290 		}
2291 	}
2292 #endif
2293 
2294 	c->c_nextfree = ctx_freelist;
2295 	ctx_freelist = c;
2296 	mutex_spin_exit(&ctx_lock);
2297 }
2298 
2299 
2300 /*----------------------------------------------------------------*/
2301 
2302 /*
2303  * pvlist functions.
2304  */
2305 
2306 /*
2307  * Walk the given pv list, and for each PTE, set or clear some bits
2308  * (e.g., PG_W or PG_NC).
2309  *
2310  * This routine flushes the cache for any page whose PTE changes,
2311  * as long as the process has a context; this is overly conservative.
2312  * It also copies ref and mod bits to the pvlist, on the theory that
2313  * this might save work later.  (XXX should test this theory)
2314  */
2315 
2316 #if defined(SUN4) || defined(SUN4C)
2317 
2318 void
2319 pv_changepte4_4c(struct vm_page *pg, int bis, int bic)
2320 {
2321 	int pte, *ptep;
2322 	struct pvlist *pv;
2323 	struct pmap *pm;
2324 	int va, vr, vs;
2325 	int ctx, s;
2326 	struct regmap *rp;
2327 	struct segmap *sp;
2328 
2329 	pv = VM_MDPAGE_PVHEAD(pg);
2330 
2331 	write_user_windows();		/* paranoid? */
2332 	s = splvm();			/* paranoid? */
2333 	if (pv->pv_pmap == NULL) {
2334 		splx(s);
2335 		return;
2336 	}
2337 	ctx = getcontext4();
2338 	for (; pv != NULL; pv = pv->pv_next) {
2339 		pm = pv->pv_pmap;
2340 		va = pv->pv_va;
2341 		vr = VA_VREG(va);
2342 		vs = VA_VSEG(va);
2343 		rp = &pm->pm_regmap[vr];
2344 		sp = &rp->rg_segmap[vs];
2345 		ptep = &sp->sg_pte[VA_VPG(va)];
2346 
2347 		if (sp->sg_pmeg == seginval) {
2348 			/* not in hardware: just fix software copy */
2349 			*ptep = (*ptep | bis) & ~bic;
2350 		} else {
2351 			/* in hardware: fix hardware copy */
2352 			if (CTX_USABLE(pm,rp)) {
2353 				setcontext4(pm->pm_ctxnum);
2354 				/* XXX should flush only when necessary */
2355 				pte = getpte4(va);
2356 				/*
2357 				 * XXX: always flush cache; conservative, but
2358 				 * needed to invalidate cache tag protection
2359 				 * bits and when disabling caching.
2360 				 */
2361 				cache_flush_page(va, pm->pm_ctxnum);
2362 			} else {
2363 				/* Make temp map in ctx 0 to access the PTE */
2364 				setcontext4(0);
2365 				if (HASSUN4_MMU3L)
2366 					setregmap(0, tregion);
2367 				setsegmap(0, sp->sg_pmeg);
2368 				va = VA_VPG(va) << PGSHIFT;
2369 				pte = getpte4(va);
2370 			}
2371 			if (pte & PG_V)
2372 				VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte);
2373 			pte = (pte | bis) & ~bic;
2374 			setpte4(va, pte);
2375 			*ptep = (*ptep & PG_MBZ) | pte;
2376 		}
2377 	}
2378 	setcontext4(ctx);
2379 	splx(s);
2380 }
2381 
2382 /*
2383  * Sync ref and mod bits in pvlist (turns off same in hardware PTEs).
2384  * Returns the new flags.
2385  *
2386  * This is just like pv_changepte, but we never add or remove bits,
2387  * hence never need to adjust software copies.
2388  */
2389 int
2390 pv_syncflags4_4c(struct vm_page *pg)
2391 {
2392 	struct pvlist *pv;
2393 	struct pmap *pm;
2394 	int pte, va, vr, vs, pmeg, flags;
2395 	int ctx, s;
2396 	struct regmap *rp;
2397 	struct segmap *sp;
2398 
2399 	pv = VM_MDPAGE_PVHEAD(pg);
2400 
2401 	s = splvm();			/* paranoid? */
2402 	if (pv->pv_pmap == NULL) {
2403 		/* Page not mapped; pv_flags is already up to date */
2404 		splx(s);
2405 		return (0);
2406 	}
2407 	ctx = getcontext4();
2408 	flags = pv->pv_flags;
2409 	for (; pv != NULL; pv = pv->pv_next) {
2410 		pm = pv->pv_pmap;
2411 		va = pv->pv_va;
2412 		vr = VA_VREG(va);
2413 		vs = VA_VSEG(va);
2414 		rp = &pm->pm_regmap[vr];
2415 		sp = &rp->rg_segmap[vs];
2416 		if ((pmeg = sp->sg_pmeg) == seginval)
2417 			continue;
2418 		if (CTX_USABLE(pm,rp)) {
2419 			setcontext4(pm->pm_ctxnum);
2420 			/* XXX should flush only when necessary */
2421 			pte = getpte4(va);
2422 			if (pte & PG_M)
2423 				cache_flush_page(va, pm->pm_ctxnum);
2424 		} else {
2425 			/* Make temp map in ctx 0 to access the PTE */
2426 			setcontext4(0);
2427 			if (HASSUN4_MMU3L)
2428 				setregmap(0, tregion);
2429 			setsegmap(0, pmeg);
2430 			va = VA_VPG(va) << PGSHIFT;
2431 			pte = getpte4(va);
2432 		}
2433 		if (pte & (PG_M|PG_U) && pte & PG_V) {
2434 			flags |= MR4_4C(pte);
2435 			pte &= ~(PG_M|PG_U);
2436 			setpte4(va, pte);
2437 		}
2438 	}
2439 
2440 	VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
2441 	setcontext4(ctx);
2442 	splx(s);
2443 	return (flags);
2444 }
2445 
2446 /*
2447  * pv_unlink is a helper function for pmap_remove.
2448  * It takes a pointer to the pv_table head for some physical address
2449  * and removes the appropriate (pmap, va) entry.
2450  *
2451  * Once the entry is removed, if the pv_table head has the cache
2452  * inhibit bit set, see if we can turn that off; if so, walk the
2453  * pvlist and turn off PG_NC in each PTE.  (The pvlist is by
2454  * definition nonempty, since it must have at least two elements
2455  * in it to have PV_NC set, and we only remove one here.)
2456  */
2457 /*static*/ void
2458 pv_unlink4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va)
2459 {
2460 	struct pvlist *pv0, *npv;
2461 
2462 	pv0 = VM_MDPAGE_PVHEAD(pg);
2463 	npv = pv0->pv_next;
2464 
2465 	/*
2466 	 * First entry is special (sigh).
2467 	 */
2468 	if (pv0->pv_pmap == pm && pv0->pv_va == va) {
2469 		pmap_stats.ps_unlink_pvfirst++;
2470 		if (npv != NULL) {
2471 			/*
2472 			 * Shift next entry into the head.
2473 			 * Make sure to retain the REF, MOD and ANC flags.
2474 			 */
2475 			pv0->pv_next = npv->pv_next;
2476 			pv0->pv_pmap = npv->pv_pmap;
2477 			pv0->pv_va = npv->pv_va;
2478 			pv0->pv_flags &= ~PV_NC;
2479 			pv0->pv_flags |= (npv->pv_flags & PV_NC);
2480 			pool_put(&pv_pool, npv);
2481 		} else {
2482 			/*
2483 			 * No mappings left; we still need to maintain
2484 			 * the REF and MOD flags. since pmap_is_modified()
2485 			 * can still be called for this page.
2486 			 */
2487 			pv0->pv_pmap = NULL;
2488 			pv0->pv_flags &= ~(PV_NC|PV_ANC);
2489 			return;
2490 		}
2491 	} else {
2492 		struct pvlist *prev;
2493 
2494 		pmap_stats.ps_unlink_pvsearch++;
2495 		for (prev = pv0;; prev = npv, npv = npv->pv_next) {
2496 			if (npv == NULL) {
2497 				panic("pv_unlink: pm %p is missing on pg %p",
2498 					pm, pg);
2499 			}
2500 			if (npv->pv_pmap == pm && npv->pv_va == va)
2501 				break;
2502 		}
2503 		prev->pv_next = npv->pv_next;
2504 		pool_put(&pv_pool, npv);
2505 	}
2506 	if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) {
2507 		/*
2508 		 * Not cached: check whether we can fix that now.
2509 		 */
2510 		va = pv0->pv_va;
2511 		for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next)
2512 			if (BADALIAS(va, npv->pv_va) ||
2513 			    (npv->pv_flags & PV_NC) != 0)
2514 				return;
2515 		pv0->pv_flags &= ~PV_ANC;
2516 		pv_changepte4_4c(pg, 0, PG_NC);
2517 	}
2518 }
2519 
2520 /*
2521  * pv_link is the inverse of pv_unlink, and is used in pmap_enter.
2522  * It returns PG_NC if the (new) pvlist says that the address cannot
2523  * be cached.
2524  */
2525 /*static*/ int
2526 pv_link4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va,
2527 	    unsigned int *pteprotop)
2528 {
2529 	struct pvlist *pv0, *pv, *npv;
2530 	int nc = (*pteprotop & PG_NC) != 0 ? PV_NC : 0;
2531 
2532 	pv0 = VM_MDPAGE_PVHEAD(pg);
2533 
2534 	if (pv0->pv_pmap == NULL) {
2535 		/* no pvlist entries yet */
2536 		pmap_stats.ps_enter_firstpv++;
2537 		pv0->pv_next = NULL;
2538 		pv0->pv_pmap = pm;
2539 		pv0->pv_va = va;
2540 		pv0->pv_flags |= nc;
2541 		return (0);
2542 	}
2543 
2544 	/*
2545 	 * Allocate the new PV entry now, and, if that fails, bail out
2546 	 * before changing the cacheable state of the existing mappings.
2547 	 */
2548 	npv = pool_get(&pv_pool, PR_NOWAIT);
2549 	if (npv == NULL)
2550 		return (ENOMEM);
2551 
2552 	pmap_stats.ps_enter_secondpv++;
2553 
2554 	/*
2555 	 * Before entering the new mapping, see if
2556 	 * it will cause old mappings to become aliased
2557 	 * and thus need to be `discached'.
2558 	 */
2559 	if (pv0->pv_flags & PV_ANC) {
2560 		/* already uncached, just stay that way */
2561 		*pteprotop |= PG_NC;
2562 		goto link_npv;
2563 	}
2564 
2565 	for (pv = pv0; pv != NULL; pv = pv->pv_next) {
2566 		if ((pv->pv_flags & PV_NC) != 0) {
2567 			*pteprotop |= PG_NC;
2568 #ifdef DEBUG
2569 			/* Check currently illegal condition */
2570 			if (nc == 0)
2571 				printf("pv_link: proc %s, va=0x%lx: "
2572 				"unexpected uncached mapping at 0x%lx\n",
2573 				    curproc ? curproc->p_comm : "--",
2574 				    va, pv->pv_va);
2575 #endif
2576 		}
2577 		if (BADALIAS(va, pv->pv_va)) {
2578 #ifdef DEBUG
2579 			if (pmapdebug & PDB_CACHESTUFF)
2580 				printf(
2581 			"pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p\n",
2582 				curproc ? curproc->p_comm : "--",
2583 				va, pv->pv_va, pg);
2584 #endif
2585 			/* Mark list head `uncached due to aliases' */
2586 			pv0->pv_flags |= PV_ANC;
2587 			pv_changepte4_4c(pg, PG_NC, 0);
2588 			*pteprotop |= PG_NC;
2589 			break;
2590 		}
2591 	}
2592 
2593 link_npv:
2594 	npv->pv_next = pv0->pv_next;
2595 	npv->pv_pmap = pm;
2596 	npv->pv_va = va;
2597 	npv->pv_flags = nc;
2598 	pv0->pv_next = npv;
2599 	return (0);
2600 }
2601 
2602 #endif /* SUN4 || SUN4C */
2603 
2604 #if defined(SUN4M) || defined(SUN4D)	/* SRMMU versions of above */
2605 /*
2606  * Walk the given pv list, and for each PTE, set or clear some bits
2607  * (e.g., PG_W or PG_NC).
2608  *
2609  * This routine flushes the cache for any page whose PTE changes,
2610  * as long as the process has a context; this is overly conservative.
2611  * It also copies ref and mod bits to the pvlist, on the theory that
2612  * this might save work later.  (XXX should test this theory)
2613  *
2614  * Called with PV lock and pmap main lock held.
2615  */
2616 void
2617 pv_changepte4m(struct vm_page *pg, int bis, int bic)
2618 {
2619 	struct pvlist *pv;
2620 	struct pmap *pm;
2621 	vaddr_t va;
2622 	struct regmap *rp;
2623 	struct segmap *sp;
2624 
2625 	pv = VM_MDPAGE_PVHEAD(pg);
2626 	if (pv->pv_pmap == NULL)
2627 		return;
2628 
2629 	for (; pv != NULL; pv = pv->pv_next) {
2630 		int tpte;
2631 		pm = pv->pv_pmap;
2632 		/* XXXSMP: should lock pm */
2633 		va = pv->pv_va;
2634 		rp = &pm->pm_regmap[VA_VREG(va)];
2635 		sp = &rp->rg_segmap[VA_VSEG(va)];
2636 
2637 		if (pm->pm_ctx) {
2638 			/*
2639 			 * XXX: always flush cache; conservative, but
2640 			 * needed to invalidate cache tag protection
2641 			 * bits and when disabling caching.
2642 			 */
2643 			cache_flush_page(va, pm->pm_ctxnum);
2644 		}
2645 
2646 		tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
2647 		KASSERT((tpte & SRMMU_TETYPE) == SRMMU_TEPTE);
2648 		VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(updatepte4m(va,
2649 		    &sp->sg_pte[VA_SUN4M_VPG(va)], bic, bis, pm->pm_ctxnum,
2650 		    PMAP_CPUSET(pm)));
2651 	}
2652 }
2653 
2654 /*
2655  * Sync ref and mod bits in pvlist. If page has been ref'd or modified,
2656  * update ref/mod bits in pvlist, and clear the hardware bits.
2657  *
2658  * Return the new flags.
2659  */
2660 int
2661 pv_syncflags4m(struct vm_page *pg)
2662 {
2663 	struct pvlist *pv;
2664 	struct pmap *pm;
2665 	int va, flags;
2666 	int s;
2667 	struct regmap *rp;
2668 	struct segmap *sp;
2669 	int tpte;
2670 
2671 	s = splvm();
2672 	PMAP_LOCK();
2673 	pv = VM_MDPAGE_PVHEAD(pg);
2674 	if (pv->pv_pmap == NULL) {
2675 		/* Page not mapped; pv_flags is already up to date */
2676 		flags = 0;
2677 		goto out;
2678 	}
2679 
2680 	flags = pv->pv_flags;
2681 	for (; pv != NULL; pv = pv->pv_next) {
2682 		pm = pv->pv_pmap;
2683 		va = pv->pv_va;
2684 		rp = &pm->pm_regmap[VA_VREG(va)];
2685 		sp = &rp->rg_segmap[VA_VSEG(va)];
2686 
2687 		tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
2688 		if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE &&
2689 		    (tpte & (SRMMU_PG_R|SRMMU_PG_M)) != 0) {
2690 			/*
2691 			 * Flush cache if modified to make sure the PTE
2692 			 * M bit will be set again on the next write access.
2693 			 */
2694 			if (pm->pm_ctx && (tpte & SRMMU_PG_M) == SRMMU_PG_M)
2695 				cache_flush_page(va, pm->pm_ctxnum);
2696 
2697 			flags |= MR4M(updatepte4m(va,
2698 					&sp->sg_pte[VA_SUN4M_VPG(va)],
2699 					SRMMU_PG_M | SRMMU_PG_R,
2700 					0, pm->pm_ctxnum, PMAP_CPUSET(pm)));
2701 		}
2702 	}
2703 
2704 	VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
2705 out:
2706 	PMAP_UNLOCK();
2707 	splx(s);
2708 	return (flags);
2709 }
2710 
2711 /*
2712  * Should be called with pmap already locked.
2713  */
2714 void
2715 pv_unlink4m(struct vm_page *pg, struct pmap *pm, vaddr_t va)
2716 {
2717 	struct pvlist *pv0, *npv;
2718 
2719 	pv0 = VM_MDPAGE_PVHEAD(pg);
2720 
2721 	npv = pv0->pv_next;
2722 	/*
2723 	 * First entry is special (sigh).
2724 	 */
2725 	if (pv0->pv_pmap == pm && pv0->pv_va == va) {
2726 		pmap_stats.ps_unlink_pvfirst++;
2727 		if (npv != NULL) {
2728 			/*
2729 			 * Shift next entry into the head.
2730 			 * Make sure to retain the REF, MOD and ANC flags
2731 			 * on the list head.
2732 			 */
2733 			pv0->pv_next = npv->pv_next;
2734 			pv0->pv_pmap = npv->pv_pmap;
2735 			pv0->pv_va = npv->pv_va;
2736 			pv0->pv_flags &= ~PV_NC;
2737 			pv0->pv_flags |= (npv->pv_flags & PV_NC);
2738 			pool_put(&pv_pool, npv);
2739 		} else {
2740 			/*
2741 			 * No mappings left; we need to maintain
2742 			 * the REF and MOD flags, since pmap_is_modified()
2743 			 * can still be called for this page.
2744 			 */
2745 			pv0->pv_pmap = NULL;
2746 			pv0->pv_flags &= ~(PV_NC|PV_ANC);
2747 			return;
2748 		}
2749 	} else {
2750 		struct pvlist *prev;
2751 
2752 		pmap_stats.ps_unlink_pvsearch++;
2753 		for (prev = pv0;; prev = npv, npv = npv->pv_next) {
2754 			if (npv == NULL) {
2755 				panic("pv_unlink: pm %p is missing on pg %p",
2756 					pm, pg);
2757 				return;
2758 			}
2759 			if (npv->pv_pmap == pm && npv->pv_va == va)
2760 				break;
2761 		}
2762 		prev->pv_next = npv->pv_next;
2763 		pool_put(&pv_pool, npv);
2764 	}
2765 
2766 	if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) {
2767 
2768 		/*
2769 		 * Not cached: check whether we can fix that now.
2770 		 */
2771 		va = pv0->pv_va;
2772 		for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next)
2773 			if (BADALIAS(va, npv->pv_va) ||
2774 			    (npv->pv_flags & PV_NC) != 0)
2775 				return;
2776 #ifdef DEBUG
2777 		if (pmapdebug & PDB_CACHESTUFF)
2778 			printf(
2779 			"pv_unlink: alias ok: proc %s, va 0x%lx, pg %p\n",
2780 				curproc ? curproc->p_comm : "--",
2781 				va, pg);
2782 #endif
2783 		pv0->pv_flags &= ~PV_ANC;
2784 		pv_changepte4m(pg, SRMMU_PG_C, 0);
2785 	}
2786 }
2787 
2788 /*
2789  * pv_link is the inverse of pv_unlink, and is used in pmap_enter.
2790  * May turn off the cacheable bit in the pte prototype for the new mapping.
2791  * Called with pm locked.
2792  */
2793 /*static*/ int
2794 pv_link4m(struct vm_page *pg, struct pmap *pm, vaddr_t va,
2795 	  unsigned int *pteprotop)
2796 {
2797 	struct pvlist *pv0, *pv, *npv;
2798 	int nc = (*pteprotop & SRMMU_PG_C) == 0 ? PV_NC : 0;
2799 	int error = 0;
2800 
2801 	pv0 = VM_MDPAGE_PVHEAD(pg);
2802 
2803 	if (pv0->pv_pmap == NULL) {
2804 		/* no pvlist entries yet */
2805 		pmap_stats.ps_enter_firstpv++;
2806 		pv0->pv_next = NULL;
2807 		pv0->pv_pmap = pm;
2808 		pv0->pv_va = va;
2809 		pv0->pv_flags |= nc;
2810 		goto out;
2811 	}
2812 
2813 	/*
2814 	 * Allocate the new PV entry now, and, if that fails, bail out
2815 	 * before changing the cacheable state of the existing mappings.
2816 	 */
2817 	npv = pool_get(&pv_pool, PR_NOWAIT);
2818 	if (npv == NULL) {
2819 		error = ENOMEM;
2820 		goto out;
2821 	}
2822 
2823 	pmap_stats.ps_enter_secondpv++;
2824 
2825 	/*
2826 	 * See if the new mapping will cause old mappings to
2827 	 * become aliased and thus need to be `discached'.
2828 	 */
2829 	if ((pv0->pv_flags & PV_ANC) != 0) {
2830 		/* already uncached, just stay that way */
2831 		*pteprotop &= ~SRMMU_PG_C;
2832 		goto link_npv;
2833 	}
2834 
2835 	for (pv = pv0; pv != NULL; pv = pv->pv_next) {
2836 		if ((pv->pv_flags & PV_NC) != 0) {
2837 			*pteprotop &= ~SRMMU_PG_C;
2838 #ifdef DEBUG
2839 			/* Check currently illegal condition */
2840 			if (nc == 0)
2841 				printf("pv_link: proc %s, va=0x%lx: "
2842 				"unexpected uncached mapping at 0x%lx\n",
2843 				    curproc ? curproc->p_comm : "--",
2844 				    va, pv->pv_va);
2845 #endif
2846 		}
2847 		if (BADALIAS(va, pv->pv_va)) {
2848 #ifdef DEBUG
2849 			if (pmapdebug & PDB_CACHESTUFF)
2850 				printf(
2851 			"pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p\n",
2852 				curproc ? curproc->p_comm : "--",
2853 				va, pv->pv_va, pg);
2854 #endif
2855 			/* Mark list head `uncached due to aliases' */
2856 			pv0->pv_flags |= PV_ANC;
2857 			pv_changepte4m(pg, 0, SRMMU_PG_C);
2858 			*pteprotop &= ~SRMMU_PG_C;
2859 			break;
2860 		}
2861 	}
2862 
2863 link_npv:
2864 	/* Now link in the new PV entry */
2865 	npv->pv_next = pv0->pv_next;
2866 	npv->pv_pmap = pm;
2867 	npv->pv_va = va;
2868 	npv->pv_flags = nc;
2869 	pv0->pv_next = npv;
2870 
2871 out:
2872 	return (error);
2873 }
2874 #endif
2875 
2876 /*
2877  * Uncache all entries on behalf of kvm_uncache(). In addition to
2878  * removing the cache bit from the PTE, we are also setting PV_NC
2879  * in each entry to stop pv_unlink() from re-caching (i.e. when a
2880  * a bad alias is going away).
2881  */
2882 static void
2883 pv_uncache(struct vm_page *pg)
2884 {
2885 	struct pvlist *pv;
2886 	int s;
2887 
2888 	s = splvm();
2889 	PMAP_LOCK();
2890 
2891 	for (pv = VM_MDPAGE_PVHEAD(pg); pv != NULL; pv = pv->pv_next)
2892 		pv->pv_flags |= PV_NC;
2893 
2894 #if defined(SUN4M) || defined(SUN4D)
2895 	if (CPU_HAS_SRMMU)
2896 		pv_changepte4m(pg, 0, SRMMU_PG_C);
2897 #endif
2898 #if defined(SUN4) || defined(SUN4C)
2899 	if (CPU_HAS_SUNMMU)
2900 		pv_changepte4_4c(pg, PG_NC, 0);
2901 #endif
2902 	PMAP_UNLOCK();
2903 	splx(s);
2904 }
2905 
2906 /*
2907  * Walk the given list and flush the cache for each (MI) page that is
2908  * potentially in the cache. Called only if vactype != VAC_NONE.
2909  */
2910 #if defined(SUN4) || defined(SUN4C)
2911 static void
2912 pv_flushcache4_4c(struct vm_page *pg)
2913 {
2914 	struct pvlist *pv;
2915 	struct pmap *pm;
2916 	int s, ctx;
2917 
2918 	pv = VM_MDPAGE_PVHEAD(pg);
2919 
2920 	write_user_windows();	/* paranoia? */
2921 	s = splvm();		/* XXX extreme paranoia */
2922 	if ((pm = pv->pv_pmap) != NULL) {
2923 		ctx = getcontext4();
2924 		for (;;) {
2925 			if (pm->pm_ctx) {
2926 				setcontext4(pm->pm_ctxnum);
2927 				cache_flush_page(pv->pv_va, pm->pm_ctxnum);
2928 			}
2929 			pv = pv->pv_next;
2930 			if (pv == NULL)
2931 				break;
2932 			pm = pv->pv_pmap;
2933 		}
2934 		setcontext4(ctx);
2935 	}
2936 	splx(s);
2937 }
2938 #endif /* SUN4 || SUN4C */
2939 
2940 #if defined(SUN4M) || defined(SUN4D)
2941 static void
2942 pv_flushcache4m(struct vm_page *pg)
2943 {
2944 	struct pvlist *pv;
2945 	struct pmap *pm;
2946 	int s;
2947 
2948 	pv = VM_MDPAGE_PVHEAD(pg);
2949 
2950 	s = splvm();		/* XXX extreme paranoia */
2951 	if ((pm = pv->pv_pmap) != NULL) {
2952 		for (;;) {
2953 			if (pm->pm_ctx) {
2954 				cache_flush_page(pv->pv_va, pm->pm_ctxnum);
2955 			}
2956 			pv = pv->pv_next;
2957 			if (pv == NULL)
2958 				break;
2959 			pm = pv->pv_pmap;
2960 		}
2961 	}
2962 	splx(s);
2963 }
2964 #endif /* SUN4M || SUN4D */
2965 
2966 /*----------------------------------------------------------------*/
2967 
2968 /*
2969  * At last, pmap code.
2970  */
2971 
2972 #if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D))
2973 int nptesg;
2974 #endif
2975 
2976 #if defined(SUN4M) || defined(SUN4D)
2977 static void pmap_bootstrap4m(void *);
2978 #endif
2979 #if defined(SUN4) || defined(SUN4C)
2980 static void pmap_bootstrap4_4c(void *, int, int, int);
2981 #endif
2982 
2983 /*
2984  * Bootstrap the system enough to run with VM enabled.
2985  *
2986  * nsegment is the number of mmu segment entries (``PMEGs'');
2987  * nregion is the number of mmu region entries (``SMEGs'');
2988  * nctx is the number of contexts.
2989  */
2990 void
2991 pmap_bootstrap(int nctx, int nregion, int nsegment)
2992 {
2993 	void *p;
2994 	extern char etext[], kernel_data_start[];
2995 	extern char *kernel_top;
2996 
2997 	uvmexp.pagesize = NBPG;
2998 	uvm_setpagesize();
2999 
3000 #if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D))
3001 	/* In this case NPTESG is a variable */
3002 	nptesg = (NBPSG >> pgshift);
3003 #endif
3004 
3005 	/*
3006 	 * Grab physical memory list.
3007 	 */
3008 	p = kernel_top;
3009 	get_phys_mem(&p);
3010 
3011 	/*
3012 	 * The data segment in sparc ELF images is aligned to a 64KB
3013 	 * (the maximum page size defined by the ELF/sparc ABI) boundary.
3014 	 * This results in a unused portion of physical memory in between
3015 	 * the text/rodata and the data segment. We pick up that gap
3016 	 * here to remove it from the kernel map and give it to the
3017 	 * VM manager later.
3018 	 */
3019 	etext_gap_start = (vaddr_t)(etext + NBPG - 1) & ~PGOFSET;
3020 	etext_gap_end = (vaddr_t)kernel_data_start & ~PGOFSET;
3021 
3022 	if (CPU_HAS_SRMMU) {
3023 #if defined(SUN4M) || defined(SUN4D)
3024 		pmap_bootstrap4m(p);
3025 #endif
3026 	} else if (CPU_HAS_SUNMMU) {
3027 #if defined(SUN4) || defined(SUN4C)
3028 		pmap_bootstrap4_4c(p, nctx, nregion, nsegment);
3029 #endif
3030 	}
3031 
3032 	pmap_page_upload();
3033 	mutex_init(&demap_lock, MUTEX_DEFAULT, IPL_VM);
3034 	mutex_init(&ctx_lock, MUTEX_DEFAULT, IPL_SCHED);
3035 	lock_available = true;
3036 }
3037 
3038 #if defined(SUN4) || defined(SUN4C)
3039 void
3040 pmap_bootstrap4_4c(void *top, int nctx, int nregion, int nsegment)
3041 {
3042 	union ctxinfo *ci;
3043 	struct mmuentry *mmuseg;
3044 #if defined(SUN4_MMU3L)
3045 	struct mmuentry *mmureg;
3046 #endif
3047 	struct regmap *rp;
3048 	struct segmap *sp;
3049 	int i, j;
3050 	int npte, zseg, vr, vs;
3051 	int startscookie, scookie;
3052 #if defined(SUN4_MMU3L)
3053 	int startrcookie = 0, rcookie = 0;
3054 #endif
3055 	int *kptes;
3056 	int lastpage;
3057 	vaddr_t va;
3058 	vaddr_t p;
3059 	extern char kernel_text[];
3060 
3061 	/*
3062 	 * Compute `va2pa_offset'.
3063 	 * Use `kernel_text' to probe the MMU translation since
3064 	 * the pages at KERNBASE might not be mapped.
3065 	 */
3066 	va2pa_offset = (vaddr_t)kernel_text -
3067 			((getpte4(kernel_text) & PG_PFNUM) << PGSHIFT);
3068 
3069 	ncontext = nctx;
3070 
3071 	switch (cputyp) {
3072 	case CPU_SUN4C:
3073 		mmu_has_hole = 1;
3074 		break;
3075 	case CPU_SUN4:
3076 		if (cpuinfo.cpu_type != CPUTYP_4_400) {
3077 			mmu_has_hole = 1;
3078 			break;
3079 		}
3080 	}
3081 
3082 #if defined(SUN4)
3083 	/*
3084 	 * set up the segfixmask to mask off invalid bits
3085 	 */
3086 	segfixmask =  nsegment - 1; /* assume nsegment is a power of 2 */
3087 #ifdef DIAGNOSTIC
3088 	if (((nsegment & segfixmask) | (nsegment & ~segfixmask)) != nsegment) {
3089 		printf("pmap_bootstrap: unsuitable number of segments (%d)\n",
3090 			nsegment);
3091 		callrom();
3092 	}
3093 #endif
3094 #endif
3095 
3096 #if defined(SUN4M) || defined(SUN4D) /* We're in a dual-arch kernel.
3097 					Setup 4/4c fn. ptrs */
3098 	pmap_clear_modify_p 	=	pmap_clear_modify4_4c;
3099 	pmap_clear_reference_p 	= 	pmap_clear_reference4_4c;
3100 	pmap_enter_p 		=	pmap_enter4_4c;
3101 	pmap_extract_p 		=	pmap_extract4_4c;
3102 	pmap_is_modified_p 	=	pmap_is_modified4_4c;
3103 	pmap_is_referenced_p	=	pmap_is_referenced4_4c;
3104 	pmap_kenter_pa_p 	=	pmap_kenter_pa4_4c;
3105 	pmap_kremove_p	 	=	pmap_kremove4_4c;
3106 	pmap_kprotect_p	 	=	pmap_kprotect4_4c;
3107 	pmap_page_protect_p	=	pmap_page_protect4_4c;
3108 	pmap_protect_p		=	pmap_protect4_4c;
3109 	pmap_rmk_p		=	pmap_rmk4_4c;
3110 	pmap_rmu_p		=	pmap_rmu4_4c;
3111 #endif /* defined SUN4M || defined SUN4D */
3112 
3113 	p = (vaddr_t)top;
3114 
3115 	/*
3116 	 * Last segment is the `invalid' one (one PMEG of pte's with !pg_v).
3117 	 * It will never be used for anything else.
3118 	 */
3119 	seginval = --nsegment;
3120 
3121 #if defined(SUN4_MMU3L)
3122 	if (HASSUN4_MMU3L)
3123 		reginval = --nregion;
3124 #endif
3125 
3126 	/*
3127 	 * Allocate and initialise mmu entries and context structures.
3128 	 */
3129 #if defined(SUN4_MMU3L)
3130 	mmuregions = mmureg = (struct mmuentry *)p;
3131 	p += nregion * sizeof(struct mmuentry);
3132 	memset(mmuregions, 0, nregion * sizeof(struct mmuentry));
3133 #endif
3134 	mmusegments = mmuseg = (struct mmuentry *)p;
3135 	p += nsegment * sizeof(struct mmuentry);
3136 	memset(mmusegments, 0, nsegment * sizeof(struct mmuentry));
3137 
3138 	pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
3139 	p += nctx * sizeof *ci;
3140 
3141 	/* Initialize MMU resource queues */
3142 #if defined(SUN4_MMU3L)
3143 	MMUQ_INIT(&region_freelist);
3144 	MMUQ_INIT(&region_lru);
3145 	MMUQ_INIT(&region_locked);
3146 #endif
3147 	MMUQ_INIT(&segm_freelist);
3148 	MMUQ_INIT(&segm_lru);
3149 	MMUQ_INIT(&segm_locked);
3150 
3151 
3152 	/*
3153 	 * Intialize the kernel pmap.
3154 	 */
3155 	/* kernel_pmap_store.pm_ctxnum = 0; */
3156 	kernel_pmap_store.pm_refcount = 1;
3157 #if defined(SUN4_MMU3L)
3158 	TAILQ_INIT(&kernel_pmap_store.pm_reglist);
3159 #endif
3160 	TAILQ_INIT(&kernel_pmap_store.pm_seglist);
3161 
3162 	/*
3163 	 * Allocate memory for kernel PTEs
3164 	 * XXX Consider allocating memory for only a few regions
3165 	 * and use growkernel() to allocate more as needed.
3166 	 */
3167 	kptes = (int *)p;
3168 	p += NKREG * NSEGRG * NPTESG * sizeof(int);
3169 	memset(kptes, 0, NKREG * NSEGRG * NPTESG * sizeof(int));
3170 
3171 	/*
3172 	 * Set up pm_regmap for kernel to point NUREG *below* the beginning
3173 	 * of kernel regmap storage. Since the kernel only uses regions
3174 	 * above NUREG, we save storage space and can index kernel and
3175 	 * user regions in the same way.
3176 	 */
3177 	kernel_pmap_store.pm_regmap = &kernel_regmap_store[-NUREG];
3178 	for (i = NKREG; --i >= 0;) {
3179 #if defined(SUN4_MMU3L)
3180 		kernel_regmap_store[i].rg_smeg = reginval;
3181 #endif
3182 		kernel_regmap_store[i].rg_segmap =
3183 			&kernel_segmap_store[i * NSEGRG];
3184 		for (j = NSEGRG; --j >= 0;) {
3185 			sp = &kernel_segmap_store[i * NSEGRG + j];
3186 			sp->sg_pmeg = seginval;
3187 			sp->sg_pte = &kptes[(i * NSEGRG + j) * NPTESG];
3188 		}
3189 	}
3190 
3191 	/*
3192 	 * Preserve the monitor ROM's reserved VM region, so that
3193 	 * we can use L1-A or the monitor's debugger.  As a side
3194 	 * effect we map the ROM's reserved VM into all contexts
3195 	 * (otherwise L1-A crashes the machine!).
3196 	 */
3197 
3198 	mmu_reservemon4_4c(&nregion, &nsegment);
3199 
3200 #if defined(SUN4_MMU3L)
3201 	/* Reserve one region for temporary mappings */
3202 	if (HASSUN4_MMU3L)
3203 		tregion = --nregion;
3204 #endif
3205 
3206 	/*
3207 	 * Set up the `constants' for the call to vm_init()
3208 	 * in main().  All pages beginning at p (rounded up to
3209 	 * the next whole page) and continuing through the number
3210 	 * of available pages are free, but they start at a higher
3211 	 * virtual address.  This gives us two mappable MD pages
3212 	 * for pmap_zero_page and pmap_copy_page, and one MI page
3213 	 * for /dev/mem, all with no associated physical memory.
3214 	 */
3215 	p = (p + NBPG - 1) & ~PGOFSET;
3216 
3217 	avail_start = PMAP_BOOTSTRAP_VA2PA(p);
3218 
3219 	i = p;
3220 	cpuinfo.vpage[0] = (void *)p, p += NBPG;
3221 	cpuinfo.vpage[1] = (void *)p, p += NBPG;
3222 	vmmap = (void *)p, p += NBPG;
3223 	p = (vaddr_t)reserve_dumppages((void *)p);
3224 
3225 	virtual_avail = p;
3226 	virtual_end = VM_MAX_KERNEL_ADDRESS;
3227 
3228 	p = i;			/* retract to first free phys */
3229 
3230 
3231 	/*
3232 	 * All contexts are free except the kernel's.
3233 	 *
3234 	 * XXX sun4c could use context 0 for users?
3235 	 */
3236 	ci->c_pmap = pmap_kernel();
3237 	ctx_freelist = ci + 1;
3238 	for (i = 1; i < ncontext; i++) {
3239 		ci++;
3240 		ci->c_nextfree = ci + 1;
3241 	}
3242 	ci->c_nextfree = NULL;
3243 	ctx_kick = 0;
3244 	ctx_kickdir = -1;
3245 
3246 	/*
3247 	 * Init mmu entries that map the kernel physical addresses.
3248 	 *
3249 	 * All the other MMU entries are free.
3250 	 *
3251 	 * THIS ASSUMES THE KERNEL IS MAPPED BY A CONTIGUOUS RANGE OF
3252 	 * MMU SEGMENTS/REGIONS DURING THE BOOT PROCESS
3253 	 */
3254 
3255 	/* Compute the number of segments used by the kernel */
3256 	zseg = (((p + NBPSG - 1) & ~SGOFSET) - KERNBASE) >> SGSHIFT;
3257 	lastpage = VA_VPG(p);
3258 	if (lastpage == 0)
3259 		/*
3260 		 * If the page bits in p are 0, we filled the last segment
3261 		 * exactly; if not, it is the last page filled in the
3262 		 * last segment.
3263 		 */
3264 		lastpage = NPTESG;
3265 
3266 	p = KERNBASE;			/* first va */
3267 	vs = VA_VSEG(KERNBASE);		/* first virtual segment */
3268 	vr = VA_VREG(KERNBASE);		/* first virtual region */
3269 	rp = &pmap_kernel()->pm_regmap[vr];
3270 
3271 	/* Get region/segment where kernel addresses start */
3272 #if defined(SUN4_MMU3L)
3273 	if (HASSUN4_MMU3L)
3274 		startrcookie = rcookie = getregmap(p);
3275 	mmureg = &mmuregions[rcookie];
3276 #endif
3277 
3278 	startscookie = scookie = getsegmap(p);
3279 	mmuseg = &mmusegments[scookie];
3280 	zseg += scookie;	/* First free segment */
3281 
3282 	for (;;) {
3283 
3284 		/*
3285 		 * Distribute each kernel region/segment into all contexts.
3286 		 * This is done through the monitor ROM, rather than
3287 		 * directly here: if we do a setcontext we will fault,
3288 		 * as we are not (yet) mapped in any other context.
3289 		 */
3290 
3291 		if ((vs % NSEGRG) == 0) {
3292 			/* Entering a new region */
3293 			if (VA_VREG(p) > vr) {
3294 #ifdef DEBUG
3295 				printf("note: giant kernel!\n");
3296 #endif
3297 				vr++, rp++;
3298 			}
3299 #if defined(SUN4_MMU3L)
3300 			if (HASSUN4_MMU3L) {
3301 				for (i = 1; i < nctx; i++)
3302 					prom_setcontext(i, (void *)p, rcookie);
3303 
3304 				MMUQ_INSERT_TAIL(&region_locked,
3305 						  mmureg, me_list);
3306 				TAILQ_INSERT_TAIL(&pmap_kernel()->pm_reglist,
3307 						  mmureg, me_pmchain);
3308 #ifdef DIAGNOSTIC
3309 				mmuseg->me_statp = NULL;
3310 #endif
3311 				mmureg->me_cookie = rcookie;
3312 				mmureg->me_pmap = pmap_kernel();
3313 				mmureg->me_vreg = vr;
3314 				rp->rg_smeg = rcookie;
3315 				mmureg++;
3316 				rcookie++;
3317 			}
3318 #endif /* SUN4_MMU3L */
3319 		}
3320 
3321 #if defined(SUN4_MMU3L)
3322 		if (!HASSUN4_MMU3L)
3323 #endif
3324 			for (i = 1; i < nctx; i++)
3325 				prom_setcontext(i, (void *)p, scookie);
3326 
3327 		/* set up the mmu entry */
3328 		MMUQ_INSERT_TAIL(&segm_locked, mmuseg, me_list);
3329 #ifdef DIAGNOSTIC
3330 		mmuseg->me_statp = &pmap_stats.ps_npmeg_locked;
3331 #endif
3332 		TAILQ_INSERT_TAIL(&pmap_kernel()->pm_seglist, mmuseg, me_pmchain);
3333 		pmap_stats.ps_npmeg_locked++;
3334 		mmuseg->me_cookie = scookie;
3335 		mmuseg->me_pmap = pmap_kernel();
3336 		mmuseg->me_vreg = vr;
3337 		mmuseg->me_vseg = vs % NSEGRG;
3338 		sp = &rp->rg_segmap[vs % NSEGRG];
3339 		sp->sg_pmeg = scookie;
3340 		npte = ++scookie < zseg ? NPTESG : lastpage;
3341 		sp->sg_npte = npte;
3342 		sp->sg_nwired = npte;
3343 		pmap_kernel()->pm_stats.resident_count += npte;
3344 		rp->rg_nsegmap += 1;
3345 		for (i = 0; i < npte; i++)
3346 			sp->sg_pte[i] = getpte4(p + i * NBPG) | PG_WIRED;
3347 		mmuseg++;
3348 		vs++;
3349 		if (scookie < zseg) {
3350 			p += NBPSG;
3351 			continue;
3352 		}
3353 
3354 		/*
3355 		 * Unmap the pages, if any, that are not part of
3356 		 * the final segment.
3357 		 */
3358 		for (p += npte << PGSHIFT; npte < NPTESG; npte++, p += NBPG)
3359 			setpte4(p, 0);
3360 
3361 #if defined(SUN4_MMU3L)
3362 		if (HASSUN4_MMU3L) {
3363 			/*
3364 			 * Unmap the segments, if any, that are not part of
3365 			 * the final region.
3366 			 */
3367 			for (i = rp->rg_nsegmap; i < NSEGRG; i++, p += NBPSG)
3368 				setsegmap(p, seginval);
3369 
3370 			/*
3371 			 * Unmap any kernel regions that we aren't using.
3372 			 */
3373 			for (i = 0; i < nctx; i++) {
3374 				setcontext4(i);
3375 				for (va = p;
3376 				     va < (OPENPROM_STARTVADDR & ~(NBPRG - 1));
3377 				     va += NBPRG)
3378 					setregmap(va, reginval);
3379 			}
3380 
3381 		} else
3382 #endif
3383 		{
3384 			/*
3385 			 * Unmap any kernel segments that we aren't using.
3386 			 */
3387 			for (i = 0; i < nctx; i++) {
3388 				setcontext4(i);
3389 				for (va = p;
3390 				     va < (OPENPROM_STARTVADDR & ~(NBPSG - 1));
3391 				     va += NBPSG)
3392 					setsegmap(va, seginval);
3393 			}
3394 		}
3395 		break;
3396 	}
3397 
3398 #if defined(SUN4_MMU3L)
3399 	if (HASSUN4_MMU3L)
3400 		for (rcookie = 0; rcookie < nregion; rcookie++) {
3401 			if (rcookie == startrcookie)
3402 				/* Kernel must fit in one region! */
3403 				rcookie++;
3404 			mmureg = &mmuregions[rcookie];
3405 			mmureg->me_cookie = rcookie;
3406 			MMUQ_INSERT_TAIL(&region_freelist, mmureg, me_list);
3407 #ifdef DIAGNOSTIC
3408 			mmuseg->me_statp = NULL;
3409 #endif
3410 		}
3411 #endif /* SUN4_MMU3L */
3412 
3413 	for (scookie = 0; scookie < nsegment; scookie++) {
3414 		if (scookie == startscookie)
3415 			/* Skip static kernel image */
3416 			scookie = zseg;
3417 		mmuseg = &mmusegments[scookie];
3418 		mmuseg->me_cookie = scookie;
3419 		MMUQ_INSERT_TAIL(&segm_freelist, mmuseg, me_list);
3420 		pmap_stats.ps_npmeg_free++;
3421 #ifdef DIAGNOSTIC
3422 		mmuseg->me_statp = NULL;
3423 #endif
3424 	}
3425 
3426 	/* Erase all spurious user-space segmaps */
3427 	for (i = 1; i < ncontext; i++) {
3428 		setcontext4(i);
3429 		if (HASSUN4_MMU3L)
3430 			for (p = 0, j = NUREG; --j >= 0; p += NBPRG)
3431 				setregmap(p, reginval);
3432 		else
3433 			for (p = 0, vr = 0; vr < NUREG; vr++) {
3434 				if (VA_INHOLE(p)) {
3435 					p = MMU_HOLE_END;
3436 					vr = VA_VREG(p);
3437 				}
3438 				for (j = NSEGRG; --j >= 0; p += NBPSG)
3439 					setsegmap(p, seginval);
3440 			}
3441 	}
3442 	setcontext4(0);
3443 
3444 	/*
3445 	 * write protect & encache kernel text;
3446 	 * set red zone at kernel base;
3447 	 * enable cache on message buffer and cpuinfo.
3448 	 */
3449 	{
3450 		extern char etext[];
3451 
3452 		/* Enable cache on message buffer and cpuinfo */
3453 		for (p = KERNBASE; p < (vaddr_t)trapbase; p += NBPG)
3454 			setpte4(p, getpte4(p) & ~PG_NC);
3455 
3456 		/* Enable cache and write protext kernel text */
3457 		for (p = (vaddr_t)trapbase; p < (vaddr_t)etext; p += NBPG)
3458 			setpte4(p, getpte4(p) & ~(PG_NC|PG_W));
3459 
3460 		/*
3461 		 * Unmap the `etext gap'; it'll be made available
3462 		 * to the VM manager.
3463 		 */
3464 		for (p = etext_gap_start; p < etext_gap_end; p += NBPG) {
3465 			rp = &pmap_kernel()->pm_regmap[VA_VREG(p)];
3466 			sp = &rp->rg_segmap[VA_VSEG(p)];
3467 			sp->sg_nwired--;
3468 			sp->sg_npte--;
3469 			pmap_kernel()->pm_stats.resident_count--;
3470 			sp->sg_pte[VA_VPG(p)] = 0;
3471 			setpte4(p, 0);
3472 		}
3473 
3474 		/* Enable cache on data & bss */
3475 		for (p = etext_gap_end; p < virtual_avail; p += NBPG)
3476 			setpte4(p, getpte4(p) & ~PG_NC);
3477 
3478 	}
3479 	cpus[0] = (struct cpu_info *)CPUINFO_VA;
3480 }
3481 #endif
3482 
3483 #if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of pmap_bootstrap */
3484 /*
3485  * Bootstrap the system enough to run with VM enabled on a sun4m machine.
3486  *
3487  * Switches from ROM to kernel page tables, and sets up initial mappings.
3488  */
3489 static void
3490 pmap_bootstrap4m(void *top)
3491 {
3492 	int i, j;
3493 	vaddr_t p, q;
3494 	union ctxinfo *ci;
3495 	int reg, seg;
3496 	unsigned int ctxtblsize;
3497 	vaddr_t pagetables_start, pagetables_end;
3498 	paddr_t pagetables_start_pa;
3499 	extern char etext[];
3500 	extern char kernel_text[];
3501 	vaddr_t va;
3502 #ifdef MULTIPROCESSOR
3503 	vsize_t off;
3504 	size_t cpuinfo_len = sizeof(struct cpu_info);
3505 	uint8_t *cpuinfo_data;
3506 	int align = PAGE_SIZE;
3507 	vaddr_t sva, cpuinfo_va;
3508 	vsize_t sz;
3509 #endif
3510 
3511 	/*
3512 	 * Compute `va2pa_offset'.
3513 	 * Use `kernel_text' to probe the MMU translation since
3514 	 * the pages at KERNBASE might not be mapped.
3515 	 */
3516 	va2pa_offset = (vaddr_t)kernel_text - VA2PA(kernel_text);
3517 
3518 	ncontext = cpuinfo.mmu_ncontext;
3519 
3520 #if defined(SUN4) || defined(SUN4C) /* setup SRMMU fn. ptrs for dual-arch
3521 				       kernel */
3522 	pmap_clear_modify_p 	=	pmap_clear_modify4m;
3523 	pmap_clear_reference_p 	= 	pmap_clear_reference4m;
3524 	pmap_enter_p 		=	pmap_enter4m;
3525 	pmap_extract_p 		=	pmap_extract4m;
3526 	pmap_is_modified_p 	=	pmap_is_modified4m;
3527 	pmap_is_referenced_p	=	pmap_is_referenced4m;
3528 	pmap_kenter_pa_p 	=	pmap_kenter_pa4m;
3529 	pmap_kremove_p	 	=	pmap_kremove4m;
3530 	pmap_kprotect_p	 	=	pmap_kprotect4m;
3531 	pmap_page_protect_p	=	pmap_page_protect4m;
3532 	pmap_protect_p		=	pmap_protect4m;
3533 	pmap_rmk_p		=	pmap_rmk4m;
3534 	pmap_rmu_p		=	pmap_rmu4m;
3535 #endif /* defined SUN4/SUN4C */
3536 
3537 	/*
3538 	 * p points to top of kernel mem
3539 	 */
3540 	p = (vaddr_t)top;
3541 
3542 	p = (p + NBPG - 1) & ~PGOFSET;
3543 
3544 	/*
3545 	 * Intialize the kernel pmap.
3546 	 */
3547 	/* kernel_pmap_store.pm_ctxnum = 0; */
3548 	kernel_pmap_store.pm_refcount = 1;
3549 
3550 	/*
3551 	 * Set up pm_regmap for kernel to point NUREG *below* the beginning
3552 	 * of kernel regmap storage. Since the kernel only uses regions
3553 	 * above NUREG, we save storage space and can index kernel and
3554 	 * user regions in the same way.
3555 	 */
3556 	kernel_pmap_store.pm_regmap = &kernel_regmap_store[-NUREG];
3557 	memset(kernel_regmap_store, 0, NKREG * sizeof(struct regmap));
3558 	memset(kernel_segmap_store, 0, NKREG * NSEGRG * sizeof(struct segmap));
3559 	for (i = NKREG; --i >= 0;) {
3560 		kernel_regmap_store[i].rg_segmap =
3561 			&kernel_segmap_store[i * NSEGRG];
3562 		kernel_regmap_store[i].rg_seg_ptps = NULL;
3563 		for (j = NSEGRG; --j >= 0;)
3564 			kernel_segmap_store[i * NSEGRG + j].sg_pte = NULL;
3565 	}
3566 
3567 	/* Allocate kernel region pointer tables */
3568 	pmap_kernel()->pm_reg_ptps = (int **)(q = p);
3569 	p += sparc_ncpus * sizeof(int **);
3570 	memset((void *)q, 0, (u_int)p - (u_int)q);
3571 
3572 	pmap_kernel()->pm_reg_ptps_pa = (int *)(q = p);
3573 	p += sparc_ncpus * sizeof(int *);
3574 	memset((void *)q, 0, (u_int)p - (u_int)q);
3575 
3576 	/* Allocate context administration */
3577 	pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
3578 	p += ncontext * sizeof *ci;
3579 	memset((void *)ci, 0, (u_int)p - (u_int)ci);
3580 
3581 	/*
3582 	 * Set up the `constants' for the call to vm_init()
3583 	 * in main().  All pages beginning at p (rounded up to
3584 	 * the next whole page) and continuing through the number
3585 	 * of available pages are free.
3586 	 */
3587 	p = (p + NBPG - 1) & ~PGOFSET;
3588 
3589 	/*
3590 	 * Reserve memory for MMU pagetables. Some of these have severe
3591 	 * alignment restrictions. We allocate in a sequence that
3592 	 * minimizes alignment gaps.
3593 	 */
3594 
3595 	pagetables_start = p;
3596 	pagetables_start_pa = PMAP_BOOTSTRAP_VA2PA(p);
3597 
3598 	/*
3599 	 * Allocate context table.
3600 	 * To keep supersparc happy, minimum aligment is on a 4K boundary.
3601 	 */
3602 	ctxtblsize = max(ncontext,1024) * sizeof(int);
3603 	cpuinfo.ctx_tbl = (int *)roundup((u_int)p, ctxtblsize);
3604 	cpuinfo.ctx_tbl_pa = PMAP_BOOTSTRAP_VA2PA(cpuinfo.ctx_tbl);
3605 	p = (u_int)cpuinfo.ctx_tbl + ctxtblsize;
3606 
3607 #if defined(MULTIPROCESSOR)
3608 	/*
3609 	 * Make sure all smp_tlb_flush*() routines for kernel pmap are
3610 	 * broadcast to all CPU's.
3611 	 */
3612 	pmap_kernel()->pm_cpuset = CPUSET_ALL;
3613 #endif
3614 
3615 	/*
3616 	 * Reserve memory for segment and page tables needed to map the entire
3617 	 * kernel. This takes (2K + NKREG * 16K) of space, but unfortunately
3618 	 * is necessary since pmap_enter() *must* be able to enter a kernel
3619 	 * mapping without delay.
3620 	 */
3621 	p = (vaddr_t) roundup(p, SRMMU_L1SIZE * sizeof(u_int));
3622 	qzero((void *)p, SRMMU_L1SIZE * sizeof(u_int));
3623 	kernel_regtable_store = (u_int *)p;
3624 	p += SRMMU_L1SIZE * sizeof(u_int);
3625 
3626 	p = (vaddr_t) roundup(p, SRMMU_L2SIZE * sizeof(u_int));
3627 	qzero((void *)p, (SRMMU_L2SIZE * sizeof(u_int)) * NKREG);
3628 	kernel_segtable_store = (u_int *)p;
3629 	p += (SRMMU_L2SIZE * sizeof(u_int)) * NKREG;
3630 
3631 	p = (vaddr_t) roundup(p, SRMMU_L3SIZE * sizeof(u_int));
3632 	/* zero it: all will be SRMMU_TEINVALID */
3633 	qzero((void *)p, ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG);
3634 	kernel_pagtable_store = (u_int *)p;
3635 	p += ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG;
3636 
3637 	/* Round to next page and mark end of pre-wired kernel space */
3638 	p = (p + NBPG - 1) & ~PGOFSET;
3639 	pagetables_end = p;
3640 
3641 #if defined(MULTIPROCESSOR)
3642 	/*
3643 	 * Allocate aligned KVA.  `cpuinfo' resides at a fixed virtual
3644 	 * address. Since we need to access an other CPU's cpuinfo
3645 	 * structure occasionally, this must be done at a virtual address
3646 	 * that's cache congruent to the fixed address CPUINFO_VA.
3647 	 *
3648 	 * NOTE: we're using the cache properties of the boot CPU to
3649 	 * determine the alignment (XXX).
3650 	 */
3651 	sz = sizeof(struct cpu_info);
3652 	if (sparc_ncpus > 1) {
3653 		if (CACHEINFO.c_totalsize > align) {
3654 			/* Need a power of two */
3655 			while (align <= CACHEINFO.c_totalsize)
3656 				align <<= 1;
3657 			align >>= 1;
3658 		}
3659 
3660 		sz = (sz + PAGE_SIZE - 1) & -PAGE_SIZE;
3661 		cpuinfo_len = sz + align - PAGE_SIZE;
3662 
3663 		/* Grab as much space as we need */
3664 		cpuinfo_data = (uint8_t *)p;
3665 		p += (cpuinfo_len * sparc_ncpus);
3666 	} else
3667 		cpuinfo_data = (uint8_t *)CPUINFO_VA;
3668 #endif
3669 
3670 	avail_start = PMAP_BOOTSTRAP_VA2PA(p);
3671 
3672 	/*
3673 	 * Now wire the region and segment tables of the kernel map.
3674 	 */
3675 	pmap_kernel()->pm_reg_ptps[0] = (int *) kernel_regtable_store;
3676 	pmap_kernel()->pm_reg_ptps_pa[0] =
3677 		 PMAP_BOOTSTRAP_VA2PA(kernel_regtable_store);
3678 
3679 	/* Install L1 table in context 0 */
3680 	setpgt4m(&cpuinfo.ctx_tbl[0],
3681 	    (pmap_kernel()->pm_reg_ptps_pa[0] >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
3682 
3683 	for (reg = 0; reg < NKREG; reg++) {
3684 		struct regmap *rp;
3685 		void *kphyssegtbl;
3686 
3687 		/*
3688 		 * Entering new region; install & build segtbl
3689 		 */
3690 
3691 		rp = &pmap_kernel()->pm_regmap[reg + VA_VREG(KERNBASE)];
3692 
3693 		kphyssegtbl = (void *)
3694 		    &kernel_segtable_store[reg * SRMMU_L2SIZE];
3695 
3696 		setpgt4m(&pmap_kernel()->pm_reg_ptps[0][reg + VA_VREG(KERNBASE)],
3697 		    (PMAP_BOOTSTRAP_VA2PA(kphyssegtbl) >> SRMMU_PPNPASHIFT) |
3698 		    SRMMU_TEPTD);
3699 
3700 		rp->rg_seg_ptps = (int *)kphyssegtbl;
3701 
3702 		for (seg = 0; seg < NSEGRG; seg++) {
3703 			struct segmap *sp;
3704 			void *kphyspagtbl;
3705 
3706 			rp->rg_nsegmap++;
3707 
3708 			sp = &rp->rg_segmap[seg];
3709 			kphyspagtbl = (void *)
3710 			    &kernel_pagtable_store
3711 				[((reg * NSEGRG) + seg) * SRMMU_L3SIZE];
3712 
3713 			setpgt4m(&rp->rg_seg_ptps[seg],
3714 				 (PMAP_BOOTSTRAP_VA2PA(kphyspagtbl) >> SRMMU_PPNPASHIFT) |
3715 				 SRMMU_TEPTD);
3716 			sp->sg_pte = (int *) kphyspagtbl;
3717 		}
3718 	}
3719 
3720 	/*
3721 	 * Preserve the monitor ROM's reserved VM region, so that
3722 	 * we can use L1-A or the monitor's debugger.
3723 	 */
3724 	mmu_reservemon4m(&kernel_pmap_store);
3725 
3726 	/*
3727 	 * Reserve virtual address space for two mappable MD pages
3728 	 * for pmap_zero_page and pmap_copy_page, one MI page
3729 	 * for /dev/mem, and some more for dumpsys().
3730 	 */
3731 	q = p;
3732 	cpuinfo.vpage[0] = (void *)p, p += NBPG;
3733 	cpuinfo.vpage[1] = (void *)p, p += NBPG;
3734 	vmmap = (void *)p, p += NBPG;
3735 	p = (vaddr_t)reserve_dumppages((void *)p);
3736 
3737 	/* Find PTE locations of vpage[] to optimize zero_fill() et.al. */
3738 	for (i = 0; i < 2; i++) {
3739 		struct regmap *rp;
3740 		struct segmap *sp;
3741 		rp = &pmap_kernel()->pm_regmap[VA_VREG(cpuinfo.vpage[i])];
3742 		sp = &rp->rg_segmap[VA_VSEG(cpuinfo.vpage[i])];
3743 		cpuinfo.vpage_pte[i] =
3744 			&sp->sg_pte[VA_SUN4M_VPG(cpuinfo.vpage[i])];
3745 	}
3746 
3747 #if !(defined(PROM_AT_F0) || defined(MSIIEP))
3748 	virtual_avail = p;
3749 #elif defined(MSIIEP)
3750 	virtual_avail = (vaddr_t)0xf0800000; /* Krups */
3751 #else
3752 	virtual_avail = (vaddr_t)0xf0080000; /* Mr.Coffee/OFW */
3753 #endif
3754 	virtual_end = VM_MAX_KERNEL_ADDRESS;
3755 
3756 	p = q;			/* retract to first free phys */
3757 
3758 	/*
3759 	 * Set up the ctxinfo structures (freelist of contexts)
3760 	 */
3761 	ci->c_pmap = pmap_kernel();
3762 	ctx_freelist = ci + 1;
3763 	for (i = 1; i < ncontext; i++) {
3764 		ci++;
3765 		ci->c_nextfree = ci + 1;
3766 	}
3767 	ci->c_nextfree = NULL;
3768 	ctx_kick = 0;
3769 	ctx_kickdir = -1;
3770 
3771 	/*
3772 	 * Now map the kernel into our new set of page tables, then
3773 	 * (finally) switch over to our running page tables.
3774 	 * We map from KERNBASE to p into context 0's page tables (and
3775 	 * the kernel pmap).
3776 	 */
3777 #ifdef DEBUG			/* Sanity checks */
3778 	if (p % NBPG != 0)
3779 		panic("pmap_bootstrap4m: p misaligned?!?");
3780 	if (KERNBASE % NBPRG != 0)
3781 		panic("pmap_bootstrap4m: KERNBASE not region-aligned");
3782 #endif
3783 
3784 	for (q = KERNBASE; q < p; q += NBPG) {
3785 		struct regmap *rp;
3786 		struct segmap *sp;
3787 		int pte, *ptep;
3788 
3789 		/*
3790 		 * Now install entry for current page.
3791 		 */
3792 		rp = &pmap_kernel()->pm_regmap[VA_VREG(q)];
3793 		sp = &rp->rg_segmap[VA_VSEG(q)];
3794 		ptep = &sp->sg_pte[VA_VPG(q)];
3795 
3796 		/*
3797 		 * Unmap the `etext gap'; it'll be made available
3798 		 * to the VM manager.
3799 		 */
3800 		if (q >= etext_gap_start && q < etext_gap_end) {
3801 			setpgt4m(ptep, 0);
3802 			continue;
3803 		}
3804 
3805 		pte = PMAP_BOOTSTRAP_VA2PA(q) >> SRMMU_PPNPASHIFT;
3806 		pte |= PPROT_N_RX | SRMMU_TEPTE;
3807 
3808 		/* Deal with the cacheable bit for pagetable memory */
3809 		if ((cpuinfo.flags & CPUFLG_CACHEPAGETABLES) != 0 ||
3810 		    q < pagetables_start || q >= pagetables_end)
3811 			pte |= SRMMU_PG_C;
3812 
3813 		/* write-protect kernel text */
3814 		if (q < (vaddr_t)trapbase || q >= (vaddr_t)etext)
3815 			pte |= PPROT_WRITE;
3816 
3817 		setpgt4m(ptep, pte);
3818 		pmap_kernel()->pm_stats.resident_count++;
3819 	}
3820 
3821 	if ((cpuinfo.flags & CPUFLG_CACHEPAGETABLES) == 0) {
3822 		/*
3823 		 * The page tables have been setup. Since we're still
3824 		 * running on the PROM's memory map, the memory we
3825 		 * allocated for our page tables might still be cached.
3826 		 * Flush it now, and don't touch it again until we
3827 		 * switch to our own tables (will be done immediately below).
3828 		 */
3829 		int size = pagetables_end - pagetables_start;
3830 		if (CACHEINFO.c_vactype != VAC_NONE) {
3831 			va = (vaddr_t)pagetables_start;
3832 			while (size > 0) {
3833 				cache_flush_page(va, 0);
3834 				va += NBPG;
3835 				size -= NBPG;
3836 			}
3837 		} else if (cpuinfo.pcache_flush_page != NULL) {
3838 			paddr_t pa = pagetables_start_pa;
3839 			while (size > 0) {
3840 				pcache_flush_page(pa, 0);
3841 				pa += NBPG;
3842 				size -= NBPG;
3843 			}
3844 		}
3845 	}
3846 
3847 	/*
3848 	 * Now switch to kernel pagetables (finally!)
3849 	 */
3850 	mmu_install_tables(&cpuinfo);
3851 
3852 #if defined(MULTIPROCESSOR)
3853 	/*
3854 	 * Initialise any cpu-specific data now.
3855 	 */
3856 	cpu_init_system();
3857 
3858 	/*
3859 	 * Setup the cpus[] array and the ci_self links.
3860 	 */
3861 	prom_printf("setting cpus self reference\n");
3862 	for (i = 0; i < sparc_ncpus; i++) {
3863 		sva = (vaddr_t) (cpuinfo_data + (cpuinfo_len * i));
3864 		cpuinfo_va = sva +
3865 		   (((CPUINFO_VA & (align - 1)) + align - sva) & (align - 1));
3866 
3867 		/*
3868 		 * Either remap from CPUINFO_VA to the new correct value
3869 		 * or clear out this cpuinfo.
3870 		 */
3871 		if (i == 0) {
3872 			for (off = 0, va = cpuinfo_va;
3873 			     sparc_ncpus > 1 && off < sizeof(struct cpu_info);
3874 			     va += NBPG, off += NBPG) {
3875 				paddr_t pa =
3876 				    PMAP_BOOTSTRAP_VA2PA(CPUINFO_VA + off);
3877 
3878 				prom_printf("going to pmap_kenter_pa"
3879 					    "(va=%p, pa=%p)\n", va, pa);
3880 				pmap_kremove(va, NBPG);
3881 				pmap_kenter_pa(va, pa,
3882 					       VM_PROT_READ | VM_PROT_WRITE, 0);
3883 			}
3884 
3885 		} else
3886 			memset((void *)cpuinfo_va, 0, sizeof(struct cpu_info));
3887 
3888 		cpus[i] = (struct cpu_info *)cpuinfo_va;
3889 		cpus[i]->ci_self = cpus[i];
3890 		prom_printf("set cpu%d ci_self address: %p\n", i, cpus[i]);
3891 
3892 		/* Unmap and prepare to return unused pages */
3893 		if (cpuinfo_va != sva) {
3894 			cpus[i]->ci_free_sva1 = sva;
3895 			cpus[i]->ci_free_eva1 = cpuinfo_va;
3896 			for (va = cpus[i]->ci_free_sva1;
3897 			     va < cpus[i]->ci_free_eva1;
3898 			     va += NBPG)
3899 				setpte4m(va, 0);
3900 		}
3901 		if (cpuinfo_va + sz != sva + cpuinfo_len) {
3902 			cpus[i]->ci_free_sva2 = cpuinfo_va + sz;
3903 			cpus[i]->ci_free_eva2 = sva + cpuinfo_len;
3904 			for (va = cpus[i]->ci_free_sva2;
3905 			     va < cpus[i]->ci_free_eva2;
3906 			     va += NBPG)
3907 				setpte4m(va, 0);
3908 		}
3909 	}
3910 #else
3911 	cpus[0] = (struct cpu_info *)CPUINFO_VA;
3912 #endif
3913 
3914 	pmap_update(pmap_kernel());
3915 
3916 #ifdef DIAGNOSTIC
3917 	if (curcpu()->ci_self != cpus[0]) {
3918 		prom_printf("curcpu()->ci_self %p != cpus[0] %p\n", curcpu()->ci_self, cpus[0]);
3919 		panic("cpuinfo inconsistent");
3920 	}
3921 #endif
3922 
3923 	prom_printf("pmap_bootstrap4m done\n");
3924 }
3925 
3926 static u_long prom_ctxreg;
3927 
3928 void
3929 mmu_install_tables(struct cpu_info *sc)
3930 {
3931 
3932 #ifdef DEBUG
3933 	prom_printf("pmap_bootstrap: installing kernel page tables...");
3934 #endif
3935 	setcontext4m(0);	/* paranoia? %%%: Make 0x3 a define! below */
3936 
3937 	/* Enable MMU tablewalk caching, flush TLB */
3938 	if (sc->mmu_enable != 0)
3939 		sc->mmu_enable();
3940 
3941 	tlb_flush_all_real();
3942 	prom_ctxreg = lda(SRMMU_CXTPTR, ASI_SRMMU);
3943 
3944 	sta(SRMMU_CXTPTR, ASI_SRMMU,
3945 		(sc->ctx_tbl_pa >> SRMMU_PPNPASHIFT) & ~0x3);
3946 
3947 	tlb_flush_all_real();
3948 
3949 #ifdef DEBUG
3950 	prom_printf("done.\n");
3951 #endif
3952 }
3953 
3954 void srmmu_restore_prom_ctx(void);
3955 
3956 void
3957 srmmu_restore_prom_ctx(void)
3958 {
3959 
3960 	tlb_flush_all();
3961 	sta(SRMMU_CXTPTR, ASI_SRMMU, prom_ctxreg);
3962 	tlb_flush_all();
3963 }
3964 #endif /* SUN4M || SUN4D */
3965 
3966 #if defined(MULTIPROCESSOR)
3967 /*
3968  * Allocate per-CPU page tables. One region, segment and page table
3969  * is needed to map CPUINFO_VA to different physical addresses on
3970  * each CPU. Since the kernel region and segment tables are all
3971  * pre-wired (in bootstrap() above) and we also assume that the
3972  * first segment (256K) of kernel space is fully populated with
3973  * pages from the start, these per-CPU tables will never need
3974  * to be updated when mapping kernel virtual memory.
3975  *
3976  * Note: this routine is called in the context of the boot CPU
3977  * during autoconfig.
3978  */
3979 void
3980 pmap_alloc_cpu(struct cpu_info *sc)
3981 {
3982 #if defined(SUN4M) || defined(SUN4D)	/* Only implemented for SUN4M/D */
3983 	vaddr_t va;
3984 	paddr_t pa;
3985 	paddr_t alignment;
3986 	u_int *ctxtable, *regtable, *segtable, *pagtable;
3987 	u_int *ctxtable_pa, *regtable_pa, *segtable_pa, *pagtable_pa;
3988 	psize_t ctxsize, size;
3989 	int vr, vs, vpg;
3990 	struct regmap *rp;
3991 	struct segmap *sp;
3992 	struct pglist mlist;
3993 	int cachebit;
3994 	int pagesz = NBPG;
3995 	int i;
3996 
3997 	cachebit = (cpuinfo.flags & CPUFLG_CACHEPAGETABLES) != 0;
3998 
3999 	/*
4000 	 * Allocate properly aligned and contiguous physically memory
4001 	 * for the PTE tables.
4002 	 */
4003 	ctxsize = (sc->mmu_ncontext * sizeof(int) + pagesz - 1) & -pagesz;
4004 	alignment = ctxsize;
4005 
4006 	/* The region, segment and page table we need fit in one page */
4007 	size = ctxsize + pagesz;
4008 
4009 	if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys,
4010 			    alignment, 0, &mlist, 1, 0) != 0)
4011 		panic("pmap_alloc_cpu: no memory");
4012 
4013 	pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&mlist));
4014 
4015 	/* Allocate virtual memory */
4016 	va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
4017 	if (va == 0)
4018 		panic("pmap_alloc_cpu: no memory");
4019 
4020 	/*
4021 	 * Layout the page tables in our chunk of memory
4022 	 */
4023 	ctxtable = (u_int *)va;
4024 	regtable = (u_int *)(va + ctxsize);
4025 	segtable = regtable + SRMMU_L1SIZE;
4026 	pagtable = segtable + SRMMU_L2SIZE;
4027 
4028 	ctxtable_pa = (u_int *)pa;
4029 	regtable_pa = (u_int *)(pa + ctxsize);
4030 	segtable_pa = regtable_pa + SRMMU_L1SIZE;
4031 	pagtable_pa = segtable_pa + SRMMU_L2SIZE;
4032 
4033 	/* Map the pages */
4034 	while (size != 0) {
4035 		pmap_kenter_pa(va, pa | (cachebit ? 0 : PMAP_NC),
4036 		    VM_PROT_READ | VM_PROT_WRITE, 0);
4037 		va += pagesz;
4038 		pa += pagesz;
4039 		size -= pagesz;
4040 	}
4041 	pmap_update(pmap_kernel());
4042 
4043 	/*
4044 	 * Store the region table pointer (and its corresponding physical
4045 	 * address) in the CPU's slot in the kernel pmap region table
4046 	 * pointer table.
4047 	 */
4048 	pmap_kernel()->pm_reg_ptps[sc->ci_cpuid] = regtable;
4049 	pmap_kernel()->pm_reg_ptps_pa[sc->ci_cpuid] = (paddr_t)regtable_pa;
4050 
4051 	vr = VA_VREG(CPUINFO_VA);
4052 	vs = VA_VSEG(CPUINFO_VA);
4053 	vpg = VA_VPG(CPUINFO_VA);
4054 	rp = &pmap_kernel()->pm_regmap[vr];
4055 	sp = &rp->rg_segmap[vs];
4056 
4057 	/*
4058 	 * Copy page tables from CPU #0, then modify entry for CPUINFO_VA
4059 	 * so that it points at the per-CPU pages.
4060 	 */
4061 	qcopy(pmap_kernel()->pm_reg_ptps[0], regtable,
4062 		SRMMU_L1SIZE * sizeof(int));
4063 	qcopy(rp->rg_seg_ptps, segtable, SRMMU_L2SIZE * sizeof(int));
4064 	qcopy(sp->sg_pte, pagtable, SRMMU_L3SIZE * sizeof(int));
4065 
4066 	setpgt4m(&ctxtable[0],
4067 		 ((u_long)regtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
4068 	setpgt4m(&regtable[vr],
4069 		 ((u_long)segtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
4070 	setpgt4m(&segtable[vs],
4071 		 ((u_long)pagtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
4072 	setpgt4m(&pagtable[vpg],
4073 		(VA2PA((void *)sc) >> SRMMU_PPNPASHIFT) |
4074 		(SRMMU_TEPTE | PPROT_N_RWX | SRMMU_PG_C));
4075 
4076 	/* Install this CPU's context table */
4077 	sc->ctx_tbl = ctxtable;
4078 	sc->ctx_tbl_pa = (paddr_t)ctxtable_pa;
4079 
4080 	/* Pre-compute this CPU's vpage[] PTEs */
4081 	for (i = 0; i < 2; i++) {
4082 		rp = &pmap_kernel()->pm_regmap[VA_VREG(sc->vpage[i])];
4083 		sp = &rp->rg_segmap[VA_VSEG(sc->vpage[i])];
4084 		sc->vpage_pte[i] = &sp->sg_pte[VA_SUN4M_VPG(sc->vpage[i])];
4085 	}
4086 #endif /* SUN4M || SUN4D */
4087 }
4088 #endif /* MULTIPROCESSOR */
4089 
4090 
4091 void
4092 pmap_init(void)
4093 {
4094 	u_int sz;
4095 
4096 	if (PAGE_SIZE != NBPG)
4097 		panic("pmap_init: PAGE_SIZE!=NBPG");
4098 
4099 	vm_num_phys = vm_last_phys - vm_first_phys;
4100 
4101 	/* Setup a pool for additional pvlist structures */
4102 	pool_init(&pv_pool, sizeof(struct pvlist), 0, 0, 0, "pvtable", NULL,
4103 	    IPL_NONE);
4104 
4105 	/*
4106 	 * Setup a pool for pmap structures.
4107 	 * The pool size includes space for an array of per-CPU
4108 	 * region table pointers & physical addresses
4109 	 */
4110 	sz = ALIGN(sizeof(struct pmap)) +
4111 	     ALIGN(NUREG * sizeof(struct regmap)) +
4112 	     sparc_ncpus * sizeof(int *) +	/* pm_reg_ptps */
4113 	     sparc_ncpus * sizeof(int);		/* pm_reg_ptps_pa */
4114 	pool_cache_bootstrap(&pmap_cache, sz, 0, 0, 0, "pmappl", NULL,
4115 	    IPL_NONE, pmap_pmap_pool_ctor, pmap_pmap_pool_dtor, NULL);
4116 
4117 	sz = NSEGRG * sizeof (struct segmap);
4118 	pool_init(&segmap_pool, sz, 0, 0, 0, "segmap", NULL, IPL_NONE);
4119 
4120 #if defined(SUN4M) || defined(SUN4D)
4121 	if (CPU_HAS_SRMMU) {
4122 		/*
4123 		 * The SRMMU only ever needs chunks in one of two sizes:
4124 		 * 1024 (for region level tables) and 256 (for segment
4125 		 * and page level tables).
4126 		 */
4127 		sz = SRMMU_L1SIZE * sizeof(int);
4128 		pool_init(&L1_pool, sz, sz, 0, 0, "L1 pagetable",
4129 			  &pgt_page_allocator, IPL_NONE);
4130 
4131 		sz = SRMMU_L2SIZE * sizeof(int);
4132 		pool_init(&L23_pool, sz, sz, 0, 0, "L2/L3 pagetable",
4133 			  &pgt_page_allocator, IPL_NONE);
4134 	}
4135 #endif /* SUN4M || SUN4D */
4136 #if defined(SUN4) || defined(SUN4C)
4137 	if (CPU_HAS_SUNMMU) {
4138 		sz = NPTESG * sizeof(int);
4139 		pool_init(&pte_pool, sz, 0, 0, 0, "ptemap", NULL,
4140 		    IPL_NONE);
4141 	}
4142 #endif /* SUN4 || SUN4C */
4143 }
4144 
4145 
4146 /*
4147  * Map physical addresses into kernel VM.
4148  */
4149 vaddr_t
4150 pmap_map(vaddr_t va, paddr_t pa, paddr_t endpa, int prot)
4151 {
4152 	int pgsize = PAGE_SIZE;
4153 
4154 	while (pa < endpa) {
4155 		pmap_kenter_pa(va, pa, prot, 0);
4156 		va += pgsize;
4157 		pa += pgsize;
4158 	}
4159 	pmap_update(pmap_kernel());
4160 	return (va);
4161 }
4162 
4163 #ifdef DEBUG
4164 /*
4165  * Check a pmap for spuriously lingering mappings
4166  */
4167 static inline void
4168 pmap_quiet_check(struct pmap *pm)
4169 {
4170 	int vs, vr;
4171 
4172 	if (CPU_HAS_SUNMMU) {
4173 #if defined(SUN4_MMU3L)
4174 		if (TAILQ_FIRST(&pm->pm_reglist))
4175 			panic("pmap_destroy: region list not empty");
4176 #endif
4177 		if (TAILQ_FIRST(&pm->pm_seglist))
4178 			panic("pmap_destroy: segment list not empty");
4179 	}
4180 
4181 	for (vr = 0; vr < NUREG; vr++) {
4182 		struct regmap *rp = &pm->pm_regmap[vr];
4183 
4184 		if (HASSUN4_MMU3L) {
4185 			if (rp->rg_smeg != reginval)
4186 				printf("pmap_chk: spurious smeg in "
4187 				       "user region %d\n", vr);
4188 		}
4189 		if (CPU_HAS_SRMMU) {
4190 			int n;
4191 #if defined(MULTIPROCESSOR)
4192 			for (n = 0; n < sparc_ncpus; n++)
4193 #else
4194 			n = 0;
4195 #endif
4196 			{
4197 				/* Did this cpu attach? */
4198 				if (pmap_kernel()->pm_reg_ptps[n] == 0)
4199 					continue;
4200 
4201 				if (pm->pm_reg_ptps[n][vr] != SRMMU_TEINVALID)
4202 					printf("pmap_chk: spurious PTP in user "
4203 						"region %d on CPU %d\n", vr, n);
4204 			}
4205 		}
4206 		if (rp->rg_nsegmap != 0)
4207 			printf("pmap_chk: %d segments remain in "
4208 				"region %d\n", rp->rg_nsegmap, vr);
4209 		if (rp->rg_segmap != NULL) {
4210 			printf("pmap_chk: segments still "
4211 				"allocated in region %d\n", vr);
4212 			for (vs = 0; vs < NSEGRG; vs++) {
4213 				struct segmap *sp = &rp->rg_segmap[vs];
4214 				if (sp->sg_npte != 0)
4215 					printf("pmap_chk: %d ptes "
4216 					     "remain in segment %d\n",
4217 						sp->sg_npte, vs);
4218 				if (sp->sg_pte != NULL) {
4219 					printf("pmap_chk: ptes still "
4220 					     "allocated in segment %d\n", vs);
4221 				}
4222 				if (CPU_HAS_SUNMMU) {
4223 					if (sp->sg_pmeg != seginval)
4224 						printf("pmap_chk: pm %p(%d,%d) "
4225 						  "spurious soft pmeg %d\n",
4226 						  pm, vr, vs, sp->sg_pmeg);
4227 				}
4228 			}
4229 		}
4230 
4231 		/* Check for spurious pmeg entries in the MMU */
4232 		if (pm->pm_ctx == NULL)
4233 			continue;
4234 		if (CPU_HAS_SUNMMU) {
4235 			int ctx;
4236 			if (mmu_has_hole && (vr >= 32 || vr < (256 - 32)))
4237 				continue;
4238 			ctx = getcontext4();
4239 			setcontext4(pm->pm_ctxnum);
4240 			for (vs = 0; vs < NSEGRG; vs++) {
4241 				vaddr_t va = VSTOVA(vr,vs);
4242 				int pmeg = getsegmap(va);
4243 				if (pmeg != seginval)
4244 					printf("pmap_chk: pm %p(%d,%d:%x): "
4245 						"spurious pmeg %d\n",
4246 						pm, vr, vs, (u_int)va, pmeg);
4247 			}
4248 			setcontext4(ctx);
4249 		}
4250 	}
4251 	if (pm->pm_stats.resident_count) {
4252 		printf("pmap_chk: res count %ld\n",
4253 		       pm->pm_stats.resident_count);
4254 	}
4255 	if (pm->pm_stats.wired_count) {
4256 		printf("pmap_chk: wired count %ld\n",
4257 		       pm->pm_stats.wired_count);
4258 	}
4259 }
4260 #endif /* DEBUG */
4261 
4262 int
4263 pmap_pmap_pool_ctor(void *arg, void *object, int flags)
4264 {
4265 	struct pmap *pm = object;
4266 	u_long addr;
4267 
4268 	memset(pm, 0, sizeof *pm);
4269 
4270 	/*
4271 	 * `pmap_pool' entries include space for the per-CPU
4272 	 * region table pointer arrays.
4273 	 */
4274 	addr = (u_long)pm + ALIGN(sizeof(struct pmap));
4275 	pm->pm_regmap = (void *)addr;
4276 	addr += ALIGN(NUREG * sizeof(struct regmap));
4277 	pm->pm_reg_ptps = (int **)addr;
4278 	addr += sparc_ncpus * sizeof(int *);
4279 	pm->pm_reg_ptps_pa = (int *)addr;
4280 
4281 	qzero((void *)pm->pm_regmap, NUREG * sizeof(struct regmap));
4282 
4283 	/* pm->pm_ctx = NULL; // already done */
4284 
4285 	if (CPU_HAS_SUNMMU) {
4286 		TAILQ_INIT(&pm->pm_seglist);
4287 #if defined(SUN4_MMU3L)
4288 		TAILQ_INIT(&pm->pm_reglist);
4289 		if (HASSUN4_MMU3L) {
4290 			int i;
4291 			for (i = NUREG; --i >= 0;)
4292 				pm->pm_regmap[i].rg_smeg = reginval;
4293 		}
4294 #endif
4295 	}
4296 #if defined(SUN4M) || defined(SUN4D)
4297 	else {
4298 		int i, n;
4299 
4300 		/*
4301 		 * We must allocate and initialize hardware-readable (MMU)
4302 		 * pagetables. We must also map the kernel regions into this
4303 		 * pmap's pagetables, so that we can access the kernel from
4304 		 * this user context.
4305 		 */
4306 #if defined(MULTIPROCESSOR)
4307 		for (n = 0; n < sparc_ncpus; n++)
4308 #else
4309 		n = 0;
4310 #endif
4311 		{
4312 			int *upt, *kpt;
4313 
4314 #if defined(MULTIPROCESSOR)
4315 			/* Did this cpu attach? */
4316 			if (pmap_kernel()->pm_reg_ptps[n] == 0)
4317 				continue;
4318 #endif
4319 
4320 			upt = pool_get(&L1_pool, flags);
4321 			pm->pm_reg_ptps[n] = upt;
4322 			pm->pm_reg_ptps_pa[n] = VA2PA((char *)upt);
4323 
4324 			/* Invalidate user space regions */
4325 			for (i = 0; i < NUREG; i++)
4326 				setpgt4m(upt++, SRMMU_TEINVALID);
4327 
4328 			/* Copy kernel regions */
4329 			kpt = &pmap_kernel()->pm_reg_ptps[n][VA_VREG(KERNBASE)];
4330 			for (i = 0; i < NKREG; i++)
4331 				setpgt4m(upt++, kpt[i]);
4332 		}
4333 	}
4334 #endif /* SUN4M || SUN4D */
4335 
4336 	/* XXX - a peculiar place to do this, but we can't do it in pmap_init
4337 	 * and here at least it's off the beaten code track.
4338 	 */
4339 {static int x; if (x == 0) pool_setlowat(&pv_pool, 512), x = 1; }
4340 
4341 	return (0);
4342 }
4343 
4344 void
4345 pmap_pmap_pool_dtor(void *arg, void *object)
4346 {
4347 	struct pmap *pm = object;
4348 	union ctxinfo *c;
4349 	int s = splvm();	/* paranoia */
4350 
4351 #ifdef DEBUG
4352 	if (pmapdebug & PDB_DESTROY)
4353 		printf("pmap_pmap_pool_dtor(%p)\n", pm);
4354 #endif
4355 
4356 	if ((c = pm->pm_ctx) != NULL) {
4357 		ctx_free(pm);
4358 	}
4359 
4360 #if defined(SUN4M) || defined(SUN4D)
4361 	if (CPU_HAS_SRMMU) {
4362 		int n;
4363 
4364 #if defined(MULTIPROCESSOR)
4365 		for (n = 0; n < sparc_ncpus; n++)
4366 #else
4367 		n = 0;
4368 #endif
4369 		{
4370 			int *pt;
4371 
4372 #if defined(MULTIPROCESSOR)
4373 			/* Did this cpu attach? */
4374 			if (pmap_kernel()->pm_reg_ptps[n] == 0)
4375 				continue;
4376 #endif
4377 
4378 			pt = pm->pm_reg_ptps[n];
4379 			pm->pm_reg_ptps[n] = NULL;
4380 			pm->pm_reg_ptps_pa[n] = 0;
4381 			pool_put(&L1_pool, pt);
4382 		}
4383 	}
4384 #endif /* SUN4M || SUN4D */
4385 	splx(s);
4386 }
4387 
4388 /*
4389  * Create and return a physical map.
4390  */
4391 struct pmap *
4392 pmap_create(void)
4393 {
4394 	struct pmap *pm;
4395 
4396 	pm = pool_cache_get(&pmap_cache, PR_WAITOK);
4397 
4398 	/*
4399 	 * Reset fields that are not preserved in the pmap cache pool.
4400 	 */
4401 	pm->pm_refcount = 1;
4402 #if defined(MULTIPROCESSOR)
4403 	/* reset active CPU set */
4404 	pm->pm_cpuset = 0;
4405 #endif
4406 	if (CPU_HAS_SUNMMU) {
4407 		/* reset the region gap */
4408 		pm->pm_gap_start = 0;
4409 		pm->pm_gap_end = VA_VREG(VM_MAXUSER_ADDRESS);
4410 	}
4411 
4412 #ifdef DEBUG
4413 	if (pmapdebug & PDB_CREATE)
4414 		printf("pmap_create[%d]: created %p\n", cpu_number(), pm);
4415 	pmap_quiet_check(pm);
4416 #endif
4417 	return (pm);
4418 }
4419 
4420 /*
4421  * Retire the given pmap from service.
4422  * Should only be called if the map contains no valid mappings.
4423  */
4424 void
4425 pmap_destroy(struct pmap *pm)
4426 {
4427 
4428 #ifdef DEBUG
4429 	if (pmapdebug & PDB_DESTROY)
4430 		printf("pmap_destroy[%d](%p)\n", cpu_number(), pm);
4431 #endif
4432 	if (atomic_dec_uint_nv(&pm->pm_refcount) == 0) {
4433 #ifdef DEBUG
4434 		pmap_quiet_check(pm);
4435 #endif
4436 		pool_cache_put(&pmap_cache, pm);
4437 	}
4438 }
4439 
4440 /*
4441  * Add a reference to the given pmap.
4442  */
4443 void
4444 pmap_reference(struct pmap *pm)
4445 {
4446 
4447 	atomic_inc_uint(&pm->pm_refcount);
4448 }
4449 
4450 #if defined(SUN4) || defined(SUN4C)
4451 /*
4452  * helper to deallocate level 2 & 3 page tables.
4453  */
4454 static void
4455 pgt_lvl23_remove4_4c(struct pmap *pm, struct regmap *rp, struct segmap *sp,
4456 		     int vr, int vs)
4457 {
4458 	vaddr_t va, tva;
4459 	int i, pmeg;
4460 
4461 	va = VSTOVA(vr,vs);
4462 	if ((pmeg = sp->sg_pmeg) != seginval) {
4463 		if (CTX_USABLE(pm,rp)) {
4464 			setcontext4(pm->pm_ctxnum);
4465 			setsegmap(va, seginval);
4466 		} else {
4467 			/* no context, use context 0 */
4468 			setcontext4(0);
4469 			if (HASSUN4_MMU3L && rp->rg_smeg != reginval) {
4470 				setregmap(0, rp->rg_smeg);
4471 				tva = vs << SGSHIFT;
4472 				setsegmap(tva, seginval);
4473 			}
4474 		}
4475 		if (!HASSUN4_MMU3L) {
4476 			if (pm == pmap_kernel()) {
4477 				/* Unmap segment from all contexts */
4478 				for (i = ncontext; --i >= 0;) {
4479 					setcontext4(i);
4480 					setsegmap(va, seginval);
4481 				}
4482 			}
4483 		}
4484 		me_free(pm, pmeg);
4485 		sp->sg_pmeg = seginval;
4486 	}
4487 	/* Free software tables for non-kernel maps */
4488 	if (pm != pmap_kernel()) {
4489 		pool_put(&pte_pool, sp->sg_pte);
4490 		sp->sg_pte = NULL;
4491 	}
4492 
4493 	if (rp->rg_nsegmap <= 0)
4494 		panic("pgt_rm: pm %p: nsegmap = %d\n", pm, rp->rg_nsegmap);
4495 
4496 	if (--rp->rg_nsegmap == 0) {
4497 #if defined(SUN4_MMU3L)
4498 		if (HASSUN4_MMU3L) {
4499 			if (rp->rg_smeg != reginval) {
4500 				if (pm == pmap_kernel()) {
4501 					/* Unmap from all contexts */
4502 					for (i = ncontext; --i >= 0;) {
4503 						setcontext4(i);
4504 						setregmap(va, reginval);
4505 					}
4506 				} else if (pm->pm_ctx) {
4507 					setcontext4(pm->pm_ctxnum);
4508 					setregmap(va, reginval);
4509 				}
4510 
4511 				/* Release MMU resource */
4512 				region_free(pm, rp->rg_smeg);
4513 				rp->rg_smeg = reginval;
4514 			}
4515 		}
4516 #endif /* SUN4_MMU3L */
4517 		/* Free software tables for non-kernel maps */
4518 		if (pm != pmap_kernel()) {
4519 			GAP_WIDEN(pm,vr);
4520 			pool_put(&segmap_pool, rp->rg_segmap);
4521 			rp->rg_segmap = NULL;
4522 		}
4523 	}
4524 }
4525 #endif /* SUN4 || SUN4C */
4526 
4527 #if defined(SUN4M) || defined(SUN4D)
4528 /*
4529  * SRMMU helper to deallocate level 2 & 3 page tables.
4530  */
4531 static void
4532 pgt_lvl23_remove4m(struct pmap *pm, struct regmap *rp, struct segmap *sp,
4533     int vr, int vs)
4534 {
4535 
4536 	/* Invalidate level 2 PTP entry */
4537 	if (pm->pm_ctx)
4538 		tlb_flush_segment(VSTOVA(vr,vs), pm->pm_ctxnum,
4539 				  PMAP_CPUSET(pm));
4540 	setpgt4m(&rp->rg_seg_ptps[vs], SRMMU_TEINVALID);
4541 	pool_put(&L23_pool, sp->sg_pte);
4542 	sp->sg_pte = NULL;
4543 
4544 	/* If region is now empty, remove level 2 pagetable as well */
4545 	if (--rp->rg_nsegmap == 0) {
4546 		int n = 0;
4547 		if (pm->pm_ctx)
4548 			tlb_flush_region(VRTOVA(vr), pm->pm_ctxnum,
4549 					 PMAP_CPUSET(pm));
4550 #ifdef MULTIPROCESSOR
4551 		/* Invalidate level 1 PTP entries on all CPUs */
4552 		for (; n < sparc_ncpus; n++) {
4553 			if ((cpus[n]->flags & CPUFLG_HATCHED) == 0)
4554 				continue;
4555 #endif
4556 			setpgt4m(&pm->pm_reg_ptps[n][vr], SRMMU_TEINVALID);
4557 #ifdef MULTIPROCESSOR
4558 		}
4559 #endif
4560 
4561 		pool_put(&segmap_pool, rp->rg_segmap);
4562 		rp->rg_segmap = NULL;
4563 		pool_put(&L23_pool, rp->rg_seg_ptps);
4564 	}
4565 }
4566 #endif /* SUN4M || SUN4D */
4567 
4568 void
4569 pmap_remove_all(struct pmap *pm)
4570 {
4571 	if (pm->pm_ctx == NULL)
4572 		return;
4573 
4574 #if defined(SUN4) || defined(SUN4C)
4575 	if (CPU_HAS_SUNMMU) {
4576 		int ctx = getcontext4();
4577 		setcontext4(pm->pm_ctxnum);
4578 		cache_flush_context(pm->pm_ctxnum);
4579 		setcontext4(ctx);
4580 	}
4581 #endif
4582 
4583 #if defined(SUN4M) || defined(SUN4D)
4584 	if (CPU_HAS_SRMMU) {
4585 		cache_flush_context(pm->pm_ctxnum);
4586 	}
4587 #endif
4588 
4589 	pm->pm_flags |= PMAP_USERCACHECLEAN;
4590 }
4591 
4592 /*
4593  * Remove the given range of mapping entries.
4594  * The starting and ending addresses are already rounded to pages.
4595  * Sheer lunacy: pmap_remove is often asked to remove nonexistent
4596  * mappings.
4597  */
4598 void
4599 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva)
4600 {
4601 	vaddr_t nva;
4602 	int vr, vs, s, ctx;
4603 	void (*rm)(struct pmap *, vaddr_t, vaddr_t, int, int);
4604 
4605 #ifdef DEBUG
4606 	if (pmapdebug & PDB_REMOVE)
4607 		printf("pmap_remove[%d](%p, 0x%lx, 0x%lx)\n",
4608 			cpu_number(), pm, va, endva);
4609 #endif
4610 
4611 	if (!CPU_HAS_SRMMU)
4612 		write_user_windows();
4613 
4614 	if (pm == pmap_kernel()) {
4615 		/*
4616 		 * Removing from kernel address space.
4617 		 */
4618 		rm = pmap_rmk;
4619 	} else {
4620 		/*
4621 		 * Removing from user address space.
4622 		 */
4623 		rm = pmap_rmu;
4624 	}
4625 
4626 	ctx = getcontext();
4627 	s = splvm();		/* XXX conservative */
4628 	PMAP_LOCK();
4629 	for (; va < endva; va = nva) {
4630 		/* do one virtual segment at a time */
4631 		vr = VA_VREG(va);
4632 		vs = VA_VSEG(va);
4633 		nva = VSTOVA(vr, vs + 1);
4634 		if (nva == 0 || nva > endva)
4635 			nva = endva;
4636 		if (pm->pm_regmap[vr].rg_nsegmap != 0)
4637 			(*rm)(pm, va, nva, vr, vs);
4638 	}
4639 	PMAP_UNLOCK();
4640 	splx(s);
4641 	setcontext(ctx);
4642 }
4643 
4644 /*
4645  * It is the same amount of work to cache_flush_page 16 pages
4646  * as to cache_flush_segment 1 segment, assuming a 64K cache size
4647  * and a 4K page size or a 128K cache size and 8K page size.
4648  */
4649 #define	PMAP_SFL_THRESHOLD	16	/* if > magic, use cache_flush_segment */
4650 
4651 /*
4652  * Remove a range contained within a single segment.
4653  * These are egregiously complicated routines.
4654  */
4655 
4656 #if defined(SUN4) || defined(SUN4C)
4657 
4658 /* remove from kernel */
4659 /*static*/ void
4660 pmap_rmk4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
4661 {
4662 	int pte, mmupte, *ptep, perpage, npg;
4663 	struct vm_page *pg;
4664 	int nleft, pmeg, inmmu;
4665 	struct regmap *rp;
4666 	struct segmap *sp;
4667 
4668 	rp = &pm->pm_regmap[vr];
4669 	sp = &rp->rg_segmap[vs];
4670 
4671 	if (rp->rg_nsegmap == 0)
4672 		return;
4673 	if ((nleft = sp->sg_npte) == 0)
4674 		return;
4675 	pmeg = sp->sg_pmeg;
4676 	inmmu = pmeg != seginval;
4677 	ptep = &sp->sg_pte[VA_VPG(va)];
4678 
4679 	/* decide how to flush cache */
4680 	npg = (endva - va) >> PGSHIFT;
4681 	if (!inmmu) {
4682 		perpage = 0;
4683 	} else if (npg > PMAP_SFL_THRESHOLD) {
4684 		/* flush the whole segment */
4685 		perpage = 0;
4686 		cache_flush_segment(vr, vs, 0);
4687 	} else {
4688 		/* flush each page individually; some never need flushing */
4689 		perpage = (CACHEINFO.c_vactype != VAC_NONE);
4690 	}
4691 
4692 	for (; va < endva; va += NBPG, ptep++) {
4693 		pte = *ptep;
4694 		mmupte = inmmu ? getpte4(va) : 0;
4695 		if ((pte & PG_V) == 0) {
4696 #ifdef DIAGNOSTIC
4697 			if (inmmu && (mmupte & PG_V) != 0)
4698 				printf("rmk: inconsistent ptes va=%lx\n", va);
4699 #endif
4700 			continue;
4701 		}
4702 		if ((pte & PG_TYPE) == PG_OBMEM) {
4703 			/* if cacheable, flush page as needed */
4704 			if (perpage && (mmupte & PG_NC) == 0)
4705 				cache_flush_page(va, 0);
4706 			if ((pg = pvhead4_4c(pte)) != NULL) {
4707 				if (inmmu)
4708 					VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte);
4709 				pv_unlink4_4c(pg, pm, va);
4710 			}
4711 		}
4712 		nleft--;
4713 #ifdef DIAGNOSTIC
4714 		if (nleft < 0)
4715 			panic("pmap_rmk: too many PTEs in segment; "
4716 			      "va 0x%lx; endva 0x%lx", va, endva);
4717 #endif
4718 		if (pte & PG_WIRED) {
4719 			sp->sg_nwired--;
4720 			pm->pm_stats.wired_count--;
4721 		}
4722 
4723 		if (inmmu)
4724 			setpte4(va, 0);
4725 		*ptep = 0;
4726 		pm->pm_stats.resident_count--;
4727 	}
4728 
4729 #ifdef DIAGNOSTIC
4730 	if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
4731 		panic("pmap_rmk: pm %p, va %lx: nleft=%d, nwired=%d",
4732 			pm, va, nleft, sp->sg_nwired);
4733 #endif
4734 	if ((sp->sg_npte = nleft) == 0)
4735 		pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
4736 	else if (sp->sg_nwired == 0) {
4737 		if (sp->sg_pmeg != seginval)
4738 			mmu_pmeg_unlock(sp->sg_pmeg);
4739 	}
4740 }
4741 
4742 #endif /* SUN4 || SUN4C */
4743 
4744 #if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of pmap_rmk */
4745 
4746 /* remove from kernel (4m)*/
4747 /* pm is already locked */
4748 /*static*/ void
4749 pmap_rmk4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
4750 {
4751 	int tpte, perpage, npg;
4752 	struct vm_page *pg;
4753 	struct regmap *rp;
4754 	struct segmap *sp;
4755 
4756 	rp = &pm->pm_regmap[vr];
4757 	sp = &rp->rg_segmap[vs];
4758 	if (rp->rg_nsegmap == 0)
4759 		return;
4760 
4761 	/* decide how to flush cache */
4762 	npg = (endva - va) >> PGSHIFT;
4763 	if (npg > PMAP_SFL_THRESHOLD) {
4764 		/* flush the whole segment */
4765 		perpage = 0;
4766 		if (CACHEINFO.c_vactype != VAC_NONE)
4767 			cache_flush_segment(vr, vs, 0);
4768 	} else {
4769 		/* flush each page individually; some never need flushing */
4770 		perpage = (CACHEINFO.c_vactype != VAC_NONE);
4771 	}
4772 	while (va < endva) {
4773 		tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
4774 		if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) {
4775 #ifdef DEBUG
4776 			if ((pmapdebug & PDB_SANITYCHK) &&
4777 			    (getpte4m(va) & SRMMU_TETYPE) == SRMMU_TEPTE)
4778 				panic("pmap_rmk: Spurious kTLB entry for 0x%lx",
4779 				      va);
4780 #endif
4781 			va += NBPG;
4782 			continue;
4783 		}
4784 		if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
4785 			/* if cacheable, flush page as needed */
4786 			if (perpage && (tpte & SRMMU_PG_C))
4787 				cache_flush_page(va, 0);
4788 			if ((pg = pvhead4m(tpte)) != NULL) {
4789 				VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte);
4790 				pv_unlink4m(pg, pm, va);
4791 			}
4792 		}
4793 		setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
4794 		    SRMMU_TEINVALID, 1, 0, CPUSET_ALL);
4795 		pm->pm_stats.resident_count--;
4796 		va += NBPG;
4797 	}
4798 }
4799 #endif /* SUN4M || SUN4D */
4800 
4801 #if defined(SUN4) || defined(SUN4C)
4802 
4803 /* remove from user */
4804 /*static*/ void
4805 pmap_rmu4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
4806 {
4807 	int *ptep, pteva, pte, perpage, npg;
4808 	struct vm_page *pg;
4809 	int nleft, pmeg, inmmu;
4810 	struct regmap *rp;
4811 	struct segmap *sp;
4812 
4813 	rp = &pm->pm_regmap[vr];
4814 	if (rp->rg_nsegmap == 0)
4815 		return;
4816 	sp = &rp->rg_segmap[vs];
4817 	if ((nleft = sp->sg_npte) == 0)
4818 		return;
4819 	pmeg = sp->sg_pmeg;
4820 	inmmu = pmeg != seginval;
4821 
4822 	/*
4823 	 * PTEs are in MMU.  Invalidate in hardware, update ref &
4824 	 * mod bits, and flush cache if required.
4825 	 */
4826 	if (!inmmu) {
4827 		perpage = 0;
4828 		pteva = 0;
4829 	} else if (CTX_USABLE(pm,rp)) {
4830 		/* process has a context, must flush cache */
4831 		npg = (endva - va) >> PGSHIFT;
4832 		setcontext4(pm->pm_ctxnum);
4833 		if ((pm->pm_flags & PMAP_USERCACHECLEAN) != 0)
4834 			perpage = 0;
4835 		else if (npg > PMAP_SFL_THRESHOLD) {
4836 			perpage = 0; /* flush the whole segment */
4837 			cache_flush_segment(vr, vs, pm->pm_ctxnum);
4838 		} else
4839 			perpage = (CACHEINFO.c_vactype != VAC_NONE);
4840 		pteva = va;
4841 	} else {
4842 		/* no context, use context 0; cache flush unnecessary */
4843 		setcontext4(0);
4844 		if (HASSUN4_MMU3L)
4845 			setregmap(0, tregion);
4846 		/* XXX use per-CPU pteva? */
4847 		setsegmap(0, pmeg);
4848 		pteva = VA_VPG(va) << PGSHIFT;
4849 		perpage = 0;
4850 	}
4851 
4852 	ptep = sp->sg_pte + VA_VPG(va);
4853 	for (; va < endva; ptep++, pteva += NBPG, va += NBPG) {
4854 		int mmupte;
4855 		pte = *ptep;
4856 		mmupte = inmmu ? getpte4(pteva) : 0;
4857 
4858 		if ((pte & PG_V) == 0) {
4859 #ifdef DIAGNOSTIC
4860 			if (inmmu && (mmupte & PG_V) != 0)
4861 				printf("pmap_rmu: pte=%x, mmupte=%x\n",
4862 					pte, getpte4(pteva));
4863 #endif
4864 			continue;
4865 		}
4866 		if ((pte & PG_TYPE) == PG_OBMEM) {
4867 			/* if cacheable, flush page as needed */
4868 			if (perpage && (mmupte & PG_NC) == 0)
4869 				cache_flush_page(va, pm->pm_ctxnum);
4870 			if ((pg = pvhead4_4c(pte)) != NULL) {
4871 				if (inmmu)
4872 					VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte);
4873 				pv_unlink4_4c(pg, pm, va);
4874 			}
4875 		}
4876 		nleft--;
4877 #ifdef DIAGNOSTIC
4878 		if (nleft < 0)
4879 			panic("pmap_rmu: too many PTEs in segment; "
4880 			     "va 0x%lx; endva 0x%lx", va, endva);
4881 #endif
4882 		if (inmmu)
4883 			setpte4(pteva, 0);
4884 
4885 		if (pte & PG_WIRED) {
4886 			sp->sg_nwired--;
4887 			pm->pm_stats.wired_count--;
4888 		}
4889 		*ptep = 0;
4890 		pm->pm_stats.resident_count--;
4891 	}
4892 
4893 #ifdef DIAGNOSTIC
4894 	if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
4895 		panic("pmap_rmu: pm %p, va %lx: nleft=%d, nwired=%d",
4896 			pm, va, nleft, sp->sg_nwired);
4897 #endif
4898 	if ((sp->sg_npte = nleft) == 0)
4899 		pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
4900 	else if (sp->sg_nwired == 0) {
4901 		if (sp->sg_pmeg != seginval)
4902 			mmu_pmeg_unlock(sp->sg_pmeg);
4903 	}
4904 }
4905 
4906 #endif /* SUN4 || SUN4C */
4907 
4908 #if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of pmap_rmu */
4909 /* remove from user */
4910 /* Note: pm is already locked */
4911 /*static*/ void
4912 pmap_rmu4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
4913 {
4914 	int *pte0, perpage, npg;
4915 	struct vm_page *pg;
4916 	int nleft;
4917 	struct regmap *rp;
4918 	struct segmap *sp;
4919 
4920 	rp = &pm->pm_regmap[vr];
4921 	if (rp->rg_nsegmap == 0)
4922 		return;
4923 	sp = &rp->rg_segmap[vs];
4924 	if ((nleft = sp->sg_npte) == 0)
4925 		return;
4926 	pte0 = sp->sg_pte;
4927 
4928 	/*
4929 	 * Invalidate PTE in MMU pagetables. Flush cache if necessary.
4930 	 */
4931 	if (pm->pm_ctx && (pm->pm_flags & PMAP_USERCACHECLEAN) == 0) {
4932 		/* process has a context, must flush cache */
4933 		if (CACHEINFO.c_vactype != VAC_NONE) {
4934 			npg = (endva - va) >> PGSHIFT;
4935 			if (npg > PMAP_SFL_THRESHOLD) {
4936 				perpage = 0; /* flush the whole segment */
4937 				cache_flush_segment(vr, vs, pm->pm_ctxnum);
4938 			} else
4939 				perpage = 1;
4940 		} else
4941 			perpage = 0;
4942 	} else {
4943 		/* no context; cache flush unnecessary */
4944 		perpage = 0;
4945 	}
4946 	for (; va < endva; va += NBPG) {
4947 		int tpte;
4948 
4949 		tpte = pte0[VA_SUN4M_VPG(va)];
4950 
4951 		if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) {
4952 #ifdef DEBUG
4953 			if ((pmapdebug & PDB_SANITYCHK) &&
4954 			    pm->pm_ctx &&
4955 			    (getpte4m(va) & SRMMU_TEPTE) == SRMMU_TEPTE)
4956 				panic("pmap_rmu: Spurious uTLB entry for 0x%lx",
4957 				      va);
4958 #endif
4959 			continue;
4960 		}
4961 
4962 		if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
4963 			/* if cacheable, flush page as needed */
4964 			if (perpage && (tpte & SRMMU_PG_C))
4965 				cache_flush_page(va, pm->pm_ctxnum);
4966 			if ((pg = pvhead4m(tpte)) != NULL) {
4967 				VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte);
4968 				pv_unlink4m(pg, pm, va);
4969 			}
4970 		}
4971 		nleft--;
4972 #ifdef DIAGNOSTIC
4973 		if (nleft < 0)
4974 			panic("pmap_rmu: too many PTEs in segment; "
4975 			      "va 0x%lx; endva 0x%lx", va, endva);
4976 #endif
4977 		setpgt4m_va(va, &pte0[VA_SUN4M_VPG(va)], SRMMU_TEINVALID,
4978 		    pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));
4979 		pm->pm_stats.resident_count--;
4980 		if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) {
4981 			sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
4982 			pm->pm_stats.wired_count--;
4983 		}
4984 	}
4985 
4986 	/*
4987 	 * If the segment is all gone, and the context is loaded, give
4988 	 * the segment back.
4989 	 */
4990 	if ((sp->sg_npte = nleft) == 0)
4991 		pgt_lvl23_remove4m(pm, rp, sp, vr, vs);
4992 }
4993 #endif /* SUN4M || SUN4D */
4994 
4995 /*
4996  * Lower (make more strict) the protection on the specified
4997  * physical page.
4998  *
4999  * There are only two cases: either the protection is going to 0
5000  * (in which case we do the dirty work here), or it is going from
5001  * to read-only (in which case pv_changepte does the trick).
5002  */
5003 
5004 #if defined(SUN4) || defined(SUN4C)
5005 void
5006 pmap_page_protect4_4c(struct vm_page *pg, vm_prot_t prot)
5007 {
5008 	struct pvlist *pv, *npv;
5009 	struct pmap *pm;
5010 	vaddr_t va;
5011 	int vr, vs, pteva, pte, *ptep;
5012 	int flags, nleft, s, ctx;
5013 	struct regmap *rp;
5014 	struct segmap *sp;
5015 
5016 #ifdef DEBUG
5017 	if ((pmapdebug & PDB_CHANGEPROT) ||
5018 	    (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE))
5019 		printf("pmap_page_protect(0x%lx, 0x%x)\n",
5020 			VM_PAGE_TO_PHYS(pg), prot);
5021 #endif
5022 	/*
5023 	 * Skip unmanaged pages, or operations that do not take
5024 	 * away write permission.
5025 	 */
5026 	if (prot & VM_PROT_WRITE)
5027 		return;
5028 
5029 	write_user_windows();	/* paranoia */
5030 	if (prot & VM_PROT_READ) {
5031 		pv_changepte4_4c(pg, 0, PG_W);
5032 		return;
5033 	}
5034 
5035 	/*
5036 	 * Remove all access to all people talking to this page.
5037 	 * Walk down PV list, removing all mappings.
5038 	 * The logic is much like that for pmap_remove,
5039 	 * but we know we are removing exactly one page.
5040 	 */
5041 	s = splvm();
5042 	pv = VM_MDPAGE_PVHEAD(pg);
5043 	if (pv->pv_pmap == NULL) {
5044 		splx(s);
5045 		return;
5046 	}
5047 	ctx = getcontext4();
5048 
5049 	/* This pv head will become empty, so clear caching state flags */
5050 	flags = pv->pv_flags & ~(PV_NC|PV_ANC);
5051 
5052 	while (pv != NULL) {
5053 		pm = pv->pv_pmap;
5054 		va = pv->pv_va;
5055 		vr = VA_VREG(va);
5056 		vs = VA_VSEG(va);
5057 		rp = &pm->pm_regmap[vr];
5058 		sp = &rp->rg_segmap[vs];
5059 		if ((nleft = sp->sg_npte) <= 0)
5060 			panic("pmap_page_protect: empty vseg");
5061 		sp->sg_npte = --nleft;
5062 		ptep = &sp->sg_pte[VA_VPG(va)];
5063 
5064 		if (*ptep & PG_WIRED) {
5065 			sp->sg_nwired--;
5066 			pm->pm_stats.wired_count--;
5067 		}
5068 
5069 		if (sp->sg_pmeg != seginval) {
5070 			/* Update PV flags */
5071 			if (CTX_USABLE(pm,rp)) {
5072 				setcontext4(pm->pm_ctxnum);
5073 				pteva = va;
5074 				cache_flush_page(va, pm->pm_ctxnum);
5075 			} else {
5076 				setcontext4(0);
5077 				/* XXX use per-CPU pteva? */
5078 				if (HASSUN4_MMU3L)
5079 					setregmap(0, tregion);
5080 				setsegmap(0, sp->sg_pmeg);
5081 				pteva = VA_VPG(va) << PGSHIFT;
5082 			}
5083 
5084 			pte = getpte4(pteva);
5085 #ifdef DIAGNOSTIC
5086 			if ((pte & PG_V) == 0)
5087 				panic("pmap_page_protect !PG_V: pg %p "
5088 				      "ctx %d, va 0x%lx, pte 0x%x",
5089 				      pg, pm->pm_ctxnum, va, pte);
5090 #endif
5091 			flags |= MR4_4C(pte);
5092 
5093 			setpte4(pteva, 0);
5094 #ifdef DIAGNOSTIC
5095 			if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
5096 				panic("pmap_page_protect: pm %p, va %lx: nleft=%d, nwired=%d",
5097 					pm, va, nleft, sp->sg_nwired);
5098 #endif
5099 			if (sp->sg_nwired == 0)
5100 				mmu_pmeg_unlock(sp->sg_pmeg);
5101 		}
5102 
5103 		*ptep = 0;
5104 		pm->pm_stats.resident_count--;
5105 		if (nleft == 0)
5106 			pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
5107 		npv = pv->pv_next;
5108 		if (pv != VM_MDPAGE_PVHEAD(pg))
5109 			pool_put(&pv_pool, pv);
5110 		pv = npv;
5111 	}
5112 
5113 	/* Finally, update pv head */
5114 	VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL;
5115 	VM_MDPAGE_PVHEAD(pg)->pv_next = NULL;
5116 	VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
5117 	setcontext4(ctx);
5118 	splx(s);
5119 }
5120 
5121 /*
5122  * Lower (make more strict) the protection on the specified
5123  * range of this pmap.
5124  *
5125  * There are only two cases: either the protection is going to 0
5126  * (in which case we call pmap_remove to do the dirty work), or
5127  * it is going from read/write to read-only.  The latter is
5128  * fairly easy.
5129  */
5130 void
5131 pmap_protect4_4c(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
5132 {
5133 	int va, nva, vr, vs;
5134 	int s, ctx;
5135 	struct regmap *rp;
5136 	struct segmap *sp;
5137 
5138 	if ((prot & VM_PROT_READ) == 0) {
5139 		pmap_remove(pm, sva, eva);
5140 		return;
5141 	}
5142 
5143 	write_user_windows();
5144 	ctx = getcontext4();
5145 	s = splvm();
5146 	PMAP_LOCK();
5147 	for (va = sva; va < eva;) {
5148 		vr = VA_VREG(va);
5149 		vs = VA_VSEG(va);
5150 		rp = &pm->pm_regmap[vr];
5151 		nva = VSTOVA(vr,vs + 1);
5152 		if (nva > eva)
5153 			nva = eva;
5154 		if (rp->rg_nsegmap == 0) {
5155 			va = nva;
5156 			continue;
5157 		}
5158 #ifdef DEBUG
5159 		if (rp->rg_segmap == NULL)
5160 			panic("pmap_protect: no segments");
5161 #endif
5162 		sp = &rp->rg_segmap[vs];
5163 		if (sp->sg_npte == 0) {
5164 			va = nva;
5165 			continue;
5166 		}
5167 #ifdef DEBUG
5168 		if (sp->sg_pte == NULL)
5169 			panic("pmap_protect: no pages");
5170 #endif
5171 		if (sp->sg_pmeg == seginval) {
5172 			int *ptep = &sp->sg_pte[VA_VPG(va)];
5173 
5174 			/* not in MMU; just clear PG_W from core copies */
5175 			for (; va < nva; va += NBPG)
5176 				*ptep++ &= ~PG_W;
5177 		} else {
5178 			/* in MMU: take away write bits from MMU PTEs */
5179 			if (CTX_USABLE(pm,rp)) {
5180 				int pte;
5181 
5182 				/*
5183 				 * Flush cache so that any existing cache
5184 				 * tags are updated.  This is really only
5185 				 * needed for PTEs that lose PG_W.
5186 				 */
5187 				pmap_stats.ps_npg_prot_all +=
5188 					(nva - va) >> PGSHIFT;
5189 				setcontext4(pm->pm_ctxnum);
5190 				for (; va < nva; va += NBPG) {
5191 					pte = getpte4(va);
5192 					if ((pte & (PG_W|PG_TYPE)) ==
5193 					    (PG_W|PG_OBMEM)) {
5194 						pmap_stats.ps_npg_prot_actual++;
5195 						cache_flush_page(va, pm->pm_ctxnum);
5196 						setpte4(va, pte & ~PG_W);
5197 					}
5198 				}
5199 			} else {
5200 				int pteva;
5201 
5202 				/*
5203 				 * No context, hence not cached;
5204 				 * just update PTEs.
5205 				 */
5206 				setcontext4(0);
5207 				/* XXX use per-CPU pteva? */
5208 				if (HASSUN4_MMU3L)
5209 					setregmap(0, tregion);
5210 				setsegmap(0, sp->sg_pmeg);
5211 				pteva = VA_VPG(va) << PGSHIFT;
5212 				for (; va < nva; pteva += NBPG, va += NBPG)
5213 					setpte4(pteva, getpte4(pteva) & ~PG_W);
5214 			}
5215 		}
5216 	}
5217 	PMAP_UNLOCK();
5218 	splx(s);
5219 	setcontext4(ctx);
5220 }
5221 
5222 /*
5223  * Change the protection and/or wired status of the given (MI) virtual page.
5224  * XXX: should have separate function (or flag) telling whether only wiring
5225  * is changing.
5226  */
5227 void
5228 pmap_changeprot4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags)
5229 {
5230 	int vr, vs, newprot, ctx, pte, *ptep;
5231 	int pmeg;
5232 	struct regmap *rp;
5233 	struct segmap *sp;
5234 
5235 #ifdef DEBUG
5236 	if (pmapdebug & PDB_CHANGEPROT)
5237 		printf("pmap_changeprot(%p, 0x%lx, 0x%x, 0x%x)\n",
5238 		    pm, va, prot, flags);
5239 #endif
5240 
5241 	if (pm == pmap_kernel())
5242 		newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S;
5243 	else
5244 		newprot = prot & VM_PROT_WRITE ? PG_W : 0;
5245 	vr = VA_VREG(va);
5246 	vs = VA_VSEG(va);
5247 	rp = &pm->pm_regmap[vr];
5248 	sp = &rp->rg_segmap[vs];
5249 	ptep = &sp->sg_pte[VA_VPG(va)];
5250 
5251 	pmap_stats.ps_changeprots++;
5252 
5253 	pte = *ptep;
5254 	if (pte & PG_WIRED && (flags & PMAP_WIRED) == 0) {
5255 		pte &= ~PG_WIRED;
5256 		sp->sg_nwired--;
5257 		pm->pm_stats.wired_count--;
5258 	} else if ((pte & PG_WIRED) == 0 && flags & PMAP_WIRED) {
5259 		pte |= PG_WIRED;
5260 		sp->sg_nwired++;
5261 		pm->pm_stats.wired_count++;
5262 	}
5263 	pte = (pte & ~PG_PROT) | newprot;
5264 	/* Update S/W pte entry */
5265 	*ptep = pte;
5266 
5267 	/* update PTEs in software or hardware */
5268 	if ((pmeg = sp->sg_pmeg) != seginval) {
5269 		/* update in hardware */
5270 		ctx = getcontext4();
5271 		if (CTX_USABLE(pm,rp)) {
5272 			/*
5273 			 * Use current context.
5274 			 * Flush cache if page has been referenced to
5275 			 * avoid stale protection bits in the cache tags.
5276 			 */
5277 			setcontext4(pm->pm_ctxnum);
5278 			pte = getpte4(va);
5279 			if ((pte & (PG_U|PG_NC|PG_TYPE)) == (PG_U|PG_OBMEM))
5280 				cache_flush_page(va, pm->pm_ctxnum);
5281 		} else {
5282 			setcontext4(0);
5283 			/* XXX use per-CPU va? */
5284 			if (HASSUN4_MMU3L)
5285 				setregmap(0, tregion);
5286 			setsegmap(0, pmeg);
5287 			va = VA_VPG(va) << PGSHIFT;
5288 			pte = getpte4(va);
5289 		}
5290 		pte = (pte & ~PG_PROT) | newprot;
5291 		setpte4(va, pte);
5292 		setcontext4(ctx);
5293 #ifdef DIAGNOSTIC
5294 		if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
5295 			panic("pmap_protect: pm %p, va %lx: nleft=%d, nwired=%d",
5296 				pm, va, sp->sg_npte, sp->sg_nwired);
5297 #endif
5298 		if (sp->sg_nwired == 0)
5299 			mmu_pmeg_unlock(pmeg);
5300 		else
5301 			mmu_pmeg_lock(pmeg);
5302 	}
5303 }
5304 
5305 #endif /* SUN4 || SUN4C */
5306 
5307 #if defined(SUN4M) || defined(SUN4D)
5308 /*
5309  * Lower (make more strict) the protection on the specified
5310  * physical page.
5311  *
5312  * There are only two cases: either the protection is going to 0
5313  * (in which case we do the dirty work here), or it is going
5314  * to read-only (in which case pv_changepte does the trick).
5315  */
5316 void
5317 pmap_page_protect4m(struct vm_page *pg, vm_prot_t prot)
5318 {
5319 	struct pvlist *pv, *npv;
5320 	struct pmap *pm;
5321 	vaddr_t va;
5322 	int vr, vs, tpte;
5323 	int flags, nleft, s;
5324 	struct regmap *rp;
5325 	struct segmap *sp;
5326 
5327 #ifdef DEBUG
5328 	if ((pmapdebug & PDB_CHANGEPROT) ||
5329 	    (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE))
5330 		printf("pmap_page_protect[%d](0x%lx, 0x%x)\n",
5331 			cpu_number(), VM_PAGE_TO_PHYS(pg), prot);
5332 #endif
5333 	s = splvm();
5334 	PMAP_LOCK();
5335 
5336 	if (prot & VM_PROT_READ) {
5337 		pv_changepte4m(pg, 0, PPROT_WRITE);
5338 		goto out;
5339 	}
5340 
5341 	/*
5342 	 * Remove all access to all people talking to this page.
5343 	 * Walk down PV list, removing all mappings. The logic is much
5344 	 * like that for pmap_remove, but we know we are removing exactly
5345 	 * one page.
5346 	 */
5347 	pv = VM_MDPAGE_PVHEAD(pg);
5348 	if (pv->pv_pmap == NULL)
5349 		goto out;
5350 
5351 	/* This pv head will become empty, so clear caching state flags */
5352 	flags = pv->pv_flags & ~(PV_NC|PV_ANC);
5353 	while (pv != NULL) {
5354 		pm = pv->pv_pmap;
5355 		va = pv->pv_va;
5356 		vr = VA_VREG(va);
5357 		vs = VA_VSEG(va);
5358 		rp = &pm->pm_regmap[vr];
5359 		if (rp->rg_nsegmap == 0)
5360 			panic("pmap_remove_all: empty vreg");
5361 		sp = &rp->rg_segmap[vs];
5362 		nleft = sp->sg_npte;
5363 		if (pm != pmap_kernel()) {
5364 			if (nleft <= 0)
5365 				panic("pmap_page_protect: empty vseg");
5366 			sp->sg_npte = --nleft;
5367 		}
5368 
5369 		/*
5370 		 * Invalidate PTE in MMU pagetables.
5371 		 * Flush cache if necessary.
5372 		 */
5373 		if (pm->pm_ctx) {
5374 			cache_flush_page(va, pm->pm_ctxnum);
5375 		}
5376 
5377 		tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
5378 		setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], SRMMU_TEINVALID,
5379 		    pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));
5380 
5381 		pm->pm_stats.resident_count--;
5382 		if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) {
5383 			sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
5384 			pm->pm_stats.wired_count--;
5385 		}
5386 
5387 		if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
5388 			panic("pmap_page_protect !PG_V: pg %p va %lx", pg, va);
5389 
5390 		flags |= MR4M(tpte);
5391 
5392 		if (pm != pmap_kernel() && nleft == 0)
5393 			/*
5394 			 * Entire user mode segment is gone
5395 			 */
5396 			pgt_lvl23_remove4m(pm, rp, sp, vr, vs);
5397 
5398 		npv = pv->pv_next;
5399 		if (pv != VM_MDPAGE_PVHEAD(pg))
5400 			pool_put(&pv_pool, pv);
5401 		pv = npv;
5402 	}
5403 
5404 	/* Finally, update pv head */
5405 	VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL;
5406 	VM_MDPAGE_PVHEAD(pg)->pv_next = NULL;
5407 	VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
5408 
5409 out:
5410 	PMAP_UNLOCK();
5411 	splx(s);
5412 }
5413 
5414 /*
5415  * Lower (make more strict) the protection on the specified
5416  * range of this pmap.
5417  */
5418 void
5419 pmap_protect4m(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
5420 {
5421 	vaddr_t va, nva;
5422 	int s, vr, vs;
5423 	struct regmap *rp;
5424 	struct segmap *sp;
5425 	int newprot;
5426 
5427 	if ((prot & VM_PROT_READ) == 0) {
5428 		pmap_remove(pm, sva, eva);
5429 		return;
5430 	}
5431 
5432 #ifdef DEBUG
5433 	if (pmapdebug & PDB_CHANGEPROT)
5434 		printf("pmap_protect[%d][curpid %d, ctx %d,%d](%lx, %lx, %x)\n",
5435 			cpu_number(), curproc->p_pid,
5436 			getcontext4m(), pm->pm_ctx ? pm->pm_ctxnum : -1,
5437 			sva, eva, prot);
5438 #endif
5439 
5440 	newprot = pte_prot4m(pm, prot);
5441 
5442 	write_user_windows();
5443 	s = splvm();
5444 	PMAP_LOCK();
5445 
5446 	for (va = sva; va < eva;) {
5447 		vr = VA_VREG(va);
5448 		vs = VA_VSEG(va);
5449 		rp = &pm->pm_regmap[vr];
5450 		nva = VSTOVA(vr,vs + 1);
5451 		if (nva > eva)
5452 			nva = eva;
5453 		if (rp->rg_nsegmap == 0) {
5454 			va = nva;
5455 			continue;
5456 		}
5457 		sp = &rp->rg_segmap[vs];
5458 		if (pm != pmap_kernel() && sp->sg_npte == 0) {
5459 			va = nva;
5460 			continue;
5461 		}
5462 
5463 		/*
5464 		 * pages loaded: take away write bits from MMU PTEs
5465 		 */
5466 		pmap_stats.ps_npg_prot_all += (nva - va) >> PGSHIFT;
5467 		for (; va < nva; va += NBPG) {
5468 			int tpte, npte;
5469 
5470 			tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
5471 			if ((tpte & SRMMU_PGTYPE) != PG_SUN4M_OBMEM)
5472 				continue;
5473 			if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
5474 				continue;
5475 			npte = (tpte & ~SRMMU_PROT_MASK) | newprot;
5476 			if (npte == tpte)
5477 				continue;
5478 
5479 			/*
5480 			 * Flush cache so that any existing cache
5481 			 * tags are updated.
5482 			 */
5483 
5484 			pmap_stats.ps_npg_prot_actual++;
5485 			if (pm->pm_ctx) {
5486 				cache_flush_page(va, pm->pm_ctxnum);
5487 			}
5488 			updatepte4m(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
5489 			    SRMMU_PROT_MASK, newprot, pm->pm_ctxnum,
5490 			    PMAP_CPUSET(pm));
5491 		}
5492 	}
5493 	PMAP_UNLOCK();
5494 	splx(s);
5495 }
5496 
5497 /*
5498  * Change the protection and/or wired status of the given (MI) virtual page.
5499  * XXX: should have separate function (or flag) telling whether only wiring
5500  * is changing.
5501  */
5502 void
5503 pmap_changeprot4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags)
5504 {
5505 	int pte, newprot;
5506 	struct regmap *rp;
5507 	struct segmap *sp;
5508 	bool owired;
5509 
5510 #ifdef DEBUG
5511 	if (pmapdebug & PDB_CHANGEPROT)
5512 		printf("pmap_changeprot[%d](%p, 0x%lx, 0x%x, 0x%x)\n",
5513 		    cpu_number(), pm, va, prot, flags);
5514 #endif
5515 
5516 	newprot = pte_prot4m(pm, prot);
5517 
5518 	pmap_stats.ps_changeprots++;
5519 
5520 	rp = &pm->pm_regmap[VA_VREG(va)];
5521 	sp = &rp->rg_segmap[VA_VSEG(va)];
5522 
5523 	pte = sp->sg_pte[VA_SUN4M_VPG(va)];
5524 	owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
5525 
5526 	if (owired) {
5527 		pm->pm_stats.wired_count--;
5528 		sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
5529 	}
5530 	if (flags & PMAP_WIRED) {
5531 		pm->pm_stats.wired_count++;
5532 		sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va));
5533 	}
5534 
5535 	if (pm->pm_ctx) {
5536 		/*
5537 		 * Use current context.
5538 		 * Flush cache if page has been referenced to
5539 		 * avoid stale protection bits in the cache tags.
5540 		 */
5541 
5542 		if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) ==
5543 		    (SRMMU_PG_C|PG_SUN4M_OBMEM))
5544 			cache_flush_page(va, pm->pm_ctxnum);
5545 	}
5546 
5547 	setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
5548 		 (pte & ~SRMMU_PROT_MASK) | newprot,
5549 		 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));
5550 
5551 }
5552 #endif /* SUN4M || SUN4D */
5553 
5554 /*
5555  * Insert (MI) physical page pa at virtual address va in the given pmap.
5556  * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary.
5557  *
5558  * If pa is not in the `managed' range it will not be `bank mapped'.
5559  * This works during bootstrap only because the first 4MB happens to
5560  * map one-to-one.
5561  *
5562  * There may already be something else there, or we might just be
5563  * changing protections and/or wiring on an existing mapping.
5564  *	XXX	should have different entry points for changing!
5565  */
5566 
5567 #if defined(SUN4) || defined(SUN4C)
5568 
5569 int
5570 pmap_enter4_4c(struct pmap *pm, vaddr_t va, paddr_t pa,
5571 	       vm_prot_t prot, u_int flags)
5572 {
5573 	struct vm_page *pg;
5574 	int pteproto, ctx;
5575 	int error;
5576 
5577 	if (VA_INHOLE(va)) {
5578 #ifdef DEBUG
5579 		printf("pmap_enter: pm %p, va 0x%lx, pa 0x%lx: in MMU hole\n",
5580 			pm, va, pa);
5581 #endif
5582 		return 0;
5583 	}
5584 
5585 #ifdef DEBUG
5586 	if (pmapdebug & PDB_ENTER)
5587 		printf("pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)\n",
5588 		    pm, va, pa, prot, flags);
5589 #endif
5590 
5591 	pg = PHYS_TO_VM_PAGE(pa);
5592 	pteproto = PG_V | PMAP_T2PTE_4(pa);
5593 	pa &= ~PMAP_TNC_4;
5594 
5595 	/*
5596 	 * Set up prototype for new PTE.  Cannot set PG_NC from PV_NC yet
5597 	 * since the pvlist no-cache bit might change as a result of the
5598 	 * new mapping.
5599 	 */
5600 	pteproto |= atop(pa) & PG_PFNUM;
5601 	if (prot & VM_PROT_WRITE)
5602 		pteproto |= PG_W;
5603 	if ((flags & PMAP_WIRED) != 0)
5604 		pteproto |= PG_WIRED;
5605 	if (flags & VM_PROT_ALL) {
5606 		pteproto |= PG_U;
5607 		if (flags & VM_PROT_WRITE) {
5608 			pteproto |= PG_M;
5609 		}
5610 	}
5611 
5612 	write_user_windows();
5613 	ctx = getcontext4();
5614 	if (pm == pmap_kernel())
5615 		error = pmap_enk4_4c(pm, va, prot, flags, pg, pteproto | PG_S);
5616 	else
5617 		error = pmap_enu4_4c(pm, va, prot, flags, pg, pteproto);
5618 	setcontext4(ctx);
5619 	return (error);
5620 }
5621 
5622 /* enter new (or change existing) kernel mapping */
5623 int
5624 pmap_enk4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
5625 	     struct vm_page *pg, int pteproto)
5626 {
5627 	int vr, vs, pte, s, inmmu;
5628 	int *ptep;
5629 	struct regmap *rp;
5630 	struct segmap *sp;
5631 	int error = 0;
5632 
5633 	vr = VA_VREG(va);
5634 	vs = VA_VSEG(va);
5635 	rp = &pm->pm_regmap[vr];
5636 	sp = &rp->rg_segmap[vs];
5637 	ptep = &sp->sg_pte[VA_VPG(va)];
5638 	s = splvm();		/* XXX way too conservative */
5639 
5640 #if defined(SUN4_MMU3L)
5641 	if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
5642 		mmu_pagein_reg(pm, rp, va, vr, &region_locked);
5643 #endif
5644 
5645 	inmmu = sp->sg_pmeg != seginval;
5646 	if ((pte = *ptep) & PG_V) {
5647 
5648 		/* old mapping exists, and is of the same pa type */
5649 		if ((pte & (PG_PFNUM|PG_TYPE)) ==
5650 		    (pteproto & (PG_PFNUM|PG_TYPE))) {
5651 			/* just changing protection and/or wiring */
5652 			pmap_changeprot4_4c(pm, va, prot, flags);
5653 			splx(s);
5654 			return (0);
5655 		}
5656 
5657 		if ((pte & PG_TYPE) == PG_OBMEM) {
5658 			struct vm_page *opg;
5659 
5660 			/*
5661 			 * Switcheroo: changing pa for this va.
5662 			 * If old pa was managed, remove from pvlist.
5663 			 * If old page was cached, flush cache.
5664 			 */
5665 			if ((opg = pvhead4_4c(pte)) != NULL)
5666 				pv_unlink4_4c(opg, pm, va);
5667 			if (inmmu && (pte & PG_NC) == 0) {
5668 				setcontext4(0);	/* ??? */
5669 				cache_flush_page(va, 0);
5670 			}
5671 		}
5672 		*ptep = 0;
5673 		if (inmmu)
5674 			setpte4(va, 0);
5675 		if (pte & PG_WIRED) {
5676 			sp->sg_nwired--;
5677 			pm->pm_stats.wired_count--;
5678 		}
5679 		pm->pm_stats.resident_count--;
5680 	} else {
5681 		/* adding new entry */
5682 		if (sp->sg_npte++ == 0) {
5683 #ifdef DIAGNOSTIC
5684 			int i; for (i = 0; i < NPTESG; i++) {
5685 				if (sp->sg_pte[i] == 0)
5686 					continue;
5687 				panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n",
5688 					pm, va, i);
5689 			}
5690 #endif
5691 			rp->rg_nsegmap++;
5692 		}
5693 	}
5694 
5695 	/*
5696 	 * If the new mapping is for a managed PA, enter into pvlist.
5697 	 */
5698 	if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) {
5699 		if (--sp->sg_npte == 0)
5700 			pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
5701 		if ((flags & PMAP_CANFAIL) != 0)
5702 			goto out;
5703 		panic("pmap_enter: cannot allocate PV entry");
5704 	}
5705 
5706 	/* Update S/W page table */
5707 	*ptep = pteproto;
5708 	if (pteproto & PG_WIRED) {
5709 		sp->sg_nwired++;
5710 		pm->pm_stats.wired_count++;
5711 	}
5712 	pm->pm_stats.resident_count++;
5713 
5714 #ifdef DIAGNOSTIC
5715 	if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
5716 		panic("pmap_enk: pm %p, va %lx: nleft=%d, nwired=%d",
5717 			pm, va, sp->sg_npte, sp->sg_nwired);
5718 #endif
5719 	if (sp->sg_pmeg == seginval)
5720 		mmu_pagein_seg(pm, sp, va, vr, vs,
5721 			(pteproto & PG_WIRED) != 0 ? &segm_locked : &segm_lru);
5722 	else if ((pteproto & PG_WIRED) != 0)
5723 		mmu_pmeg_lock(sp->sg_pmeg);
5724 
5725 	/* Update H/W page table */
5726 	setpte4(va, pteproto & ~PG_MBZ);
5727 out:
5728 	splx(s);
5729 	return (error);
5730 }
5731 
5732 /* enter new (or change existing) user mapping */
5733 int
5734 pmap_enu4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
5735 	     struct vm_page *pg, int pteproto)
5736 {
5737 	int vr, vs, *ptep, pte, pmeg, s;
5738 	int error = 0;
5739 	struct regmap *rp;
5740 	struct segmap *sp;
5741 
5742 	pm->pm_flags &= ~PMAP_USERCACHECLEAN;
5743 
5744 	vr = VA_VREG(va);
5745 	vs = VA_VSEG(va);
5746 	rp = &pm->pm_regmap[vr];
5747 	s = splvm();			/* XXX conservative */
5748 
5749 	/*
5750 	 * If there is no space in which the PTEs can be written
5751 	 * while they are not in the hardware, this must be a new
5752 	 * virtual segment.  Get PTE space and count the segment.
5753 	 *
5754 	 * TO SPEED UP CTX ALLOC, PUT SEGMENT BOUNDS STUFF HERE
5755 	 * AND IN pmap_rmu()
5756 	 */
5757 
5758 	GAP_SHRINK(pm,vr);
5759 
5760 #ifdef DEBUG
5761 	if (pm->pm_gap_end < pm->pm_gap_start) {
5762 		printf("pmap_enu: gap_start 0x%x, gap_end 0x%x",
5763 			pm->pm_gap_start, pm->pm_gap_end);
5764 		panic("pmap_enu: gap botch");
5765 	}
5766 #endif
5767 
5768 	if (rp->rg_segmap == NULL) {
5769 		/* definitely a new mapping */
5770 		int i;
5771 		int mflag = PR_NOWAIT;
5772 
5773 	rretry:
5774 		sp = (struct segmap *)pool_get(&segmap_pool, mflag);
5775 		if (sp == NULL) {
5776 			if ((flags & PMAP_CANFAIL) != 0) {
5777 				error = ENOMEM;
5778 				goto out;
5779 			}
5780 			mflag = PR_WAITOK;
5781 			goto rretry;
5782 		}
5783 #ifdef DEBUG
5784 		if (rp->rg_segmap != NULL)
5785 			panic("pmap_enter: segment filled during sleep");
5786 #endif
5787 		qzero((void *)sp, NSEGRG * sizeof (struct segmap));
5788 		rp->rg_segmap = sp;
5789 		rp->rg_nsegmap = 0;
5790 		for (i = NSEGRG; --i >= 0;)
5791 			sp++->sg_pmeg = seginval;
5792 	}
5793 
5794 	sp = &rp->rg_segmap[vs];
5795 
5796 	if ((ptep = sp->sg_pte) == NULL) {
5797 		/* definitely a new mapping */
5798 		int size = NPTESG * sizeof *ptep;
5799 		int mflag = PR_NOWAIT;
5800 
5801 	sretry:
5802 		ptep = (int *)pool_get(&pte_pool, mflag);
5803 		if (ptep == NULL) {
5804 			if ((flags & PMAP_CANFAIL) != 0) {
5805 				error = ENOMEM;
5806 				goto out;
5807 			}
5808 			mflag = PR_WAITOK;
5809 			goto sretry;
5810 		}
5811 #ifdef DEBUG
5812 		if (sp->sg_pte != NULL)
5813 			panic("pmap_enter: pte filled during sleep");
5814 		if (sp->sg_pmeg != seginval)
5815 			panic("pmap_enter: new ptes, but not seginval");
5816 #endif
5817 		qzero((void *)ptep, size);
5818 		sp->sg_pte = ptep;
5819 		sp->sg_npte = 1;
5820 		rp->rg_nsegmap++;
5821 	} else {
5822 		/* might be a change: fetch old pte */
5823 		pte = ptep[VA_VPG(va)];
5824 		if (pte & PG_V) {
5825 			/* old mapping exists, and is of the same pa type */
5826 			if ((pte & (PG_PFNUM|PG_TYPE)) ==
5827 			    (pteproto & (PG_PFNUM|PG_TYPE))) {
5828 				/* just changing prot and/or wiring */
5829 				pmap_changeprot4_4c(pm, va, prot, flags);
5830 				splx(s);
5831 				return (0);
5832 			}
5833 			/*
5834 			 * Switcheroo: changing pa for this va.
5835 			 * If old pa was managed, remove from pvlist.
5836 			 * If old page was cached, flush cache.
5837 			 */
5838 #if 0
5839 			printf("%s[%d]: pmap_enu: changing existing "
5840 				"va(0x%lx)=>pa entry\n",
5841 				curproc->p_comm, curproc->p_pid, va);
5842 #endif
5843 			if ((pte & PG_TYPE) == PG_OBMEM) {
5844 				struct vm_page *opg;
5845 				if ((opg = pvhead4_4c(pte)) != NULL)
5846 					pv_unlink4_4c(opg, pm, va);
5847 				if (CACHEINFO.c_vactype != VAC_NONE &&
5848 				    (pmeg = sp->sg_pmeg) != seginval) {
5849 					/* hardware pte */
5850 					if (CTX_USABLE(pm,rp)) {
5851 						setcontext4(pm->pm_ctxnum);
5852 					} else {
5853 						setcontext4(0);
5854 						/* XXX use per-CPU pteva? */
5855 						if (HASSUN4_MMU3L)
5856 							setregmap(0, tregion);
5857 						setsegmap(0, pmeg);
5858 					}
5859 					cache_flush_page(va, pm->pm_ctxnum);
5860 				}
5861 			}
5862 			if (pte & PG_WIRED) {
5863 				sp->sg_nwired--;
5864 				pm->pm_stats.wired_count--;
5865 			}
5866 			pm->pm_stats.resident_count--;
5867 			ptep[VA_VPG(va)] = 0;
5868 			if (sp->sg_pmeg != seginval)
5869 				setpte4(va, 0);
5870 		} else {
5871 			/* adding new entry */
5872 			sp->sg_npte++;
5873 		}
5874 	}
5875 
5876 	if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) {
5877 		if (--sp->sg_npte == 0)
5878 			/* Sigh, undo pgt allocations */
5879 			pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
5880 
5881 		if ((flags & PMAP_CANFAIL) != 0)
5882 			goto out;
5883 		panic("pmap_enter: cannot allocate PV entry");
5884 	}
5885 
5886 	/* Update S/W page table */
5887 	ptep += VA_VPG(va);
5888 	*ptep = pteproto;
5889 	if (pteproto & PG_WIRED) {
5890 		sp->sg_nwired++;
5891 		pm->pm_stats.wired_count++;
5892 	}
5893 	pm->pm_stats.resident_count++;
5894 
5895 #ifdef DIAGNOSTIC
5896 	if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
5897 		panic("pmap_enu: pm %p, va %lx: nleft=%d, nwired=%d",
5898 			pm, va, sp->sg_npte, sp->sg_nwired);
5899 #endif
5900 
5901 	if ((pmeg = sp->sg_pmeg) != seginval) {
5902 		/* Update H/W page table */
5903 		if (CTX_USABLE(pm,rp))
5904 			setcontext4(pm->pm_ctxnum);
5905 		else {
5906 			setcontext4(0);
5907 			if (HASSUN4_MMU3L)
5908 				setregmap(0, tregion);
5909 			setsegmap(0, pmeg);
5910 			va = VA_VPG(va) << PGSHIFT;
5911 		}
5912 		setpte4(va, pteproto & ~PG_MBZ);
5913 	}
5914 
5915 out:
5916 	splx(s);
5917 	return (error);
5918 }
5919 
5920 void
5921 pmap_kenter_pa4_4c(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
5922 {
5923 	struct pmap *pm = pmap_kernel();
5924 	struct regmap *rp;
5925 	struct segmap *sp;
5926 	int vr, vs, s;
5927 	int *ptep, pteproto;
5928 	int lockit = 1;
5929 
5930 	pteproto = PG_S | PG_V | PMAP_T2PTE_4(pa);
5931 	pa &= ~PMAP_TNC_4;
5932 	pteproto |= atop(pa) & PG_PFNUM;
5933 	if (prot & VM_PROT_WRITE)
5934 		pteproto |= PG_W;
5935 
5936 	vr = VA_VREG(va);
5937 	vs = VA_VSEG(va);
5938 	rp = &pm->pm_regmap[vr];
5939 	sp = &rp->rg_segmap[vs];
5940 	ptep = &sp->sg_pte[VA_VPG(va)];
5941 
5942 	if (lockit) {
5943 		pteproto |= PG_WIRED;
5944 		sp->sg_nwired++;
5945 	}
5946 
5947 	KASSERT((*ptep & PG_V) == 0);
5948 
5949 	s = splvm();
5950 #if defined(SUN4_MMU3L)
5951 	if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
5952 		mmu_pagein_reg(pm, rp, va, vr, &region_locked);
5953 #endif
5954 
5955 	if (sp->sg_npte++ == 0) {
5956 #ifdef DIAGNOSTIC
5957 		int i; for (i = 0; i < NPTESG; i++) {
5958 			if (sp->sg_pte[i] == 0)
5959 				continue;
5960 			panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n",
5961 				pm, va, i);
5962 		}
5963 #endif
5964 		rp->rg_nsegmap++;
5965 	}
5966 
5967 	/* Update S/W page table */
5968 	*ptep = pteproto;
5969 
5970 #ifdef DIAGNOSTIC
5971 	if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
5972 		panic("pmap_kenter: pm %p, va %lx: nleft=%d, nwired=%d",
5973 			pm, va, sp->sg_npte, sp->sg_nwired);
5974 #endif
5975 
5976 	if (sp->sg_pmeg == seginval) {
5977 		mmu_pagein_seg(pm, sp, va, vr, vs,
5978 				lockit ? &segm_locked : &segm_lru);
5979 	} else if (lockit)
5980 		mmu_pmeg_lock(sp->sg_pmeg);
5981 
5982 	/* Update H/W page table */
5983 	setpte4(va, pteproto & ~PG_MBZ);
5984 	splx(s);
5985 }
5986 
5987 #if notyet
5988 void pmap_lockmmu(vaddr_t sva, size_t sz);
5989 
5990 void
5991 pmap_lockmmu(vaddr_t sva, size_t sz)
5992 {
5993 	struct pmap *pm = pmap_kernel();
5994 	vaddr_t va, eva;
5995 	struct regmap *rp;
5996 	struct segmap *sp;
5997 	int vr, vs;
5998 
5999 	if (CPU_HAS_SRMMU)
6000 		return;
6001 
6002 	eva = sva + sz;
6003 	va = VA_ROUNDDOWNTOSEG(sva);
6004 
6005 	for (; va < eva; va += NBPSG) {
6006 		vr = VA_VREG(va);
6007 		vs = VA_VSEG(va);
6008 		rp = &pm->pm_regmap[vr];
6009 		sp = &rp->rg_segmap[vs];
6010 
6011 		KASSERT(sp->sg_npte != 0);
6012 
6013 		if (sp->sg_pmeg == seginval)
6014 			mmu_pagein_seg(pm, sp, va, vr, vs, &segm_locked);
6015 		else
6016 			mmu_pmeg_lock(sp->sg_pmeg);
6017 	}
6018 }
6019 #endif
6020 
6021 void
6022 pmap_kremove4_4c(vaddr_t va, vsize_t len)
6023 {
6024 	struct pmap *pm = pmap_kernel();
6025 	struct regmap *rp;
6026 	struct segmap *sp;
6027 	vaddr_t nva, endva;
6028 	int pte, mmupte, *ptep, perpage, npg, inmmu;
6029 	int nleft, pmeg;
6030 	int vr, vs, s, ctx;
6031 
6032 	endva = va + len;
6033 #ifdef DEBUG
6034 	if (pmapdebug & PDB_REMOVE)
6035 		printf("pmap_kremove(0x%lx, 0x%lx)\n", va, endva);
6036 #endif
6037 
6038 	write_user_windows();
6039 
6040 	s = splvm();
6041 	ctx = getcontext();
6042 	PMAP_LOCK();
6043 	setcontext4(0);
6044 	for (; va < endva; va = nva) {
6045 		/* do one virtual segment at a time */
6046 		vr = VA_VREG(va);
6047 		vs = VA_VSEG(va);
6048 		nva = VSTOVA(vr, vs + 1);
6049 		if (nva == 0 || nva > endva)
6050 			nva = endva;
6051 
6052 		rp = &pm->pm_regmap[vr];
6053 		sp = &rp->rg_segmap[vs];
6054 
6055 		if (rp->rg_nsegmap == 0)
6056 			continue;
6057 		nleft = sp->sg_npte;
6058 		if (nleft == 0)
6059 			continue;
6060 		pmeg = sp->sg_pmeg;
6061 		inmmu = (pmeg != seginval);
6062 		ptep = &sp->sg_pte[VA_VPG(va)];
6063 
6064 		/* decide how to flush cache */
6065 		npg = (nva - va) >> PGSHIFT;
6066 		if (!inmmu) {
6067 			perpage = 0;
6068 		} else if (npg > PMAP_SFL_THRESHOLD) {
6069 			/* flush the whole segment */
6070 			perpage = 0;
6071 			cache_flush_segment(vr, vs, 0);
6072 		} else {
6073 			/*
6074 			 * flush each page individually;
6075 			 * some never need flushing
6076 			 */
6077 			perpage = (CACHEINFO.c_vactype != VAC_NONE);
6078 		}
6079 
6080 		for (; va < nva; va += NBPG, ptep++) {
6081 			pte = *ptep;
6082 			mmupte = inmmu ? getpte4(va) : 0;
6083 			if ((pte & PG_V) == 0) {
6084 #ifdef DIAGNOSTIC
6085 				if (inmmu && (mmupte & PG_V) != 0)
6086 					printf("rmk: inconsistent ptes va=%lx\n", va);
6087 #endif
6088 				continue;
6089 			}
6090 			if ((pte & PG_TYPE) == PG_OBMEM) {
6091 				/* if cacheable, flush page as needed */
6092 				if (perpage && (mmupte & PG_NC) == 0)
6093 					cache_flush_page(va, 0);
6094 			}
6095 			nleft--;
6096 #ifdef DIAGNOSTIC
6097 			if (nleft < 0)
6098 				panic("pmap_kremove: too many PTEs in segment; "
6099 				      "va 0x%lx; endva 0x%lx", va, endva);
6100 #endif
6101 			if (pte & PG_WIRED)
6102 				sp->sg_nwired--;
6103 
6104 			if (inmmu)
6105 				setpte4(va, 0);
6106 			*ptep = 0;
6107 		}
6108 
6109 #ifdef DIAGNOSTIC
6110 		if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
6111 			panic("pmap_kremove: pm %p, va %lx: nleft=%d, nwired=%d",
6112 				pm, va, nleft, sp->sg_nwired);
6113 #endif
6114 
6115 		if ((sp->sg_npte = nleft) == 0)
6116 			pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
6117 		else if (sp->sg_nwired == 0) {
6118 			if (sp->sg_pmeg != seginval)
6119 				mmu_pmeg_unlock(sp->sg_pmeg);
6120 		}
6121 	}
6122 	PMAP_UNLOCK();
6123 	setcontext4(ctx);
6124 	splx(s);
6125 }
6126 
6127 /*
6128  * Change protection on a range of kernel addresses.
6129  */
6130 void
6131 pmap_kprotect4_4c(vaddr_t va, vsize_t size, vm_prot_t prot)
6132 {
6133 	int pte, newprot, ctx;
6134 
6135 	size = roundup(size,NBPG);
6136 	newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S;
6137 
6138 	ctx = getcontext4();
6139 	setcontext4(0);
6140 	while (size > 0) {
6141 		pte = getpte4(va);
6142 
6143 		/*
6144 		 * Flush cache if page has been referenced to
6145 		 * avoid stale protection bits in the cache tags.
6146 		 */
6147 		if ((pte & (PG_NC|PG_TYPE)) == PG_OBMEM)
6148 			cache_flush_page(va, 0);
6149 
6150 		pte = (pte & ~PG_PROT) | newprot;
6151 		setpte4(va, pte);
6152 
6153 		va += NBPG;
6154 		size -= NBPG;
6155 	}
6156 	setcontext4(ctx);
6157 }
6158 #endif /* SUN4 || SUN4C */
6159 
6160 #if defined(SUN4M) || defined(SUN4D)	/* SRMMU versions of enter routines */
6161 /*
6162  * Insert (MI) physical page pa at virtual address va in the given pmap.
6163  * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary.
6164  *
6165  * If pa is not in the `managed' range it will not be `bank mapped'.
6166  * This works during bootstrap only because the first 4MB happens to
6167  * map one-to-one.
6168  *
6169  * There may already be something else there, or we might just be
6170  * changing protections and/or wiring on an existing mapping.
6171  *	XXX	should have different entry points for changing!
6172  */
6173 
6174 int
6175 pmap_enter4m(struct pmap *pm, vaddr_t va, paddr_t pa,
6176 	     vm_prot_t prot, u_int flags)
6177 {
6178 	struct vm_page *pg;
6179 	int pteproto;
6180 	int error;
6181 
6182 #ifdef DEBUG
6183 	if (pmapdebug & PDB_ENTER)
6184 		printf("pmap_enter[curcpu %d, curpid %d, ctx %d,%d]"
6185 			"(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)\n",
6186 			cpu_number(), curproc==NULL ? -1 : curproc->p_pid,
6187 			getcontext4m(), pm->pm_ctx==NULL ? -1 : pm->pm_ctxnum,
6188 			pm, va, pa, prot, flags);
6189 #endif
6190 
6191 	pg = PHYS_TO_VM_PAGE(pa);
6192 
6193 	/* Initialise pteproto with cache bit */
6194 	pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0;
6195 
6196 #ifdef DEBUG
6197 	if (pa & PMAP_TYPE_SRMMU) {	/* this page goes in an iospace */
6198 		if (cpuinfo.cpu_type == CPUTYP_MS1)
6199 			panic("pmap_enter4m: attempt to use 36-bit iospace on"
6200 			      " MicroSPARC");
6201 	}
6202 #endif
6203 	pteproto |= SRMMU_TEPTE;
6204 	pteproto |= PMAP_T2PTE_SRMMU(pa);
6205 	pa &= ~PMAP_TNC_SRMMU;
6206 
6207 	/*
6208 	 * Set up prototype for new PTE.  Cannot set PG_NC from PV_NC yet
6209 	 * since the pvlist no-cache bit might change as a result of the
6210 	 * new mapping.
6211 	 */
6212 	pteproto |= (atop(pa) << SRMMU_PPNSHIFT);
6213 
6214 	/* Make sure we get a pte with appropriate perms! */
6215 	pteproto |= pte_prot4m(pm, prot);
6216 	if (flags & VM_PROT_ALL) {
6217 		pteproto |= SRMMU_PG_R;
6218 		if (flags & VM_PROT_WRITE) {
6219 			pteproto |= SRMMU_PG_M;
6220 		}
6221 	}
6222 
6223 	if (pm == pmap_kernel())
6224 		error = pmap_enk4m(pm, va, prot, flags, pg, pteproto | PPROT_S);
6225 	else
6226 		error = pmap_enu4m(pm, va, prot, flags, pg, pteproto);
6227 
6228 	return (error);
6229 }
6230 
6231 /* enter new (or change existing) kernel mapping */
6232 int
6233 pmap_enk4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
6234 	   struct vm_page *pg, int pteproto)
6235 {
6236 	int vr, vs, tpte, s;
6237 	struct regmap *rp;
6238 	struct segmap *sp;
6239 	int error = 0;
6240 
6241 #ifdef DEBUG
6242 	if (va < KERNBASE)
6243 		panic("pmap_enk4m: can't enter va 0x%lx below KERNBASE", va);
6244 #endif
6245 	vr = VA_VREG(va);
6246 	vs = VA_VSEG(va);
6247 	rp = &pm->pm_regmap[vr];
6248 	sp = &rp->rg_segmap[vs];
6249 
6250 	s = splvm();		/* XXX way too conservative */
6251 	PMAP_LOCK();
6252 
6253 	if (rp->rg_seg_ptps == NULL) /* enter new region */
6254 		panic("pmap_enk4m: missing kernel region table for va 0x%lx",va);
6255 
6256 	tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
6257 	if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) {
6258 
6259 		/* old mapping exists, and is of the same pa type */
6260 
6261 		if ((tpte & SRMMU_PPNMASK) == (pteproto & SRMMU_PPNMASK)) {
6262 			/* just changing protection and/or wiring */
6263 			pmap_changeprot4m(pm, va, prot, flags);
6264 			PMAP_UNLOCK();
6265 			splx(s);
6266 			return (0);
6267 		}
6268 
6269 		if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
6270 			struct vm_page *opg;
6271 #ifdef DEBUG
6272 printf("pmap_enk4m: changing existing va=>pa entry: va 0x%lx, pteproto 0x%x, "
6273        "oldpte 0x%x\n", va, pteproto, tpte);
6274 #endif
6275 			/*
6276 			 * Switcheroo: changing pa for this va.
6277 			 * If old pa was managed, remove from pvlist.
6278 			 * If old page was cached, flush cache.
6279 			 */
6280 			if ((opg = pvhead4m(tpte)) != NULL)
6281 				pv_unlink4m(opg, pm, va);
6282 			if (tpte & SRMMU_PG_C) {
6283 				cache_flush_page(va, 0);
6284 			}
6285 		}
6286 
6287 		/*
6288 		 * Invalidate the mapping now, so we can avoid the
6289 		 * de-map and update protocol when setting the new
6290 		 * PTE below.
6291 		 */
6292 		setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
6293 			SRMMU_TEINVALID, pm->pm_ctx != NULL,
6294 			pm->pm_ctxnum, PMAP_CPUSET(pm));
6295 		pm->pm_stats.resident_count--;
6296 	}
6297 
6298 	/*
6299 	 * If the new mapping is for a managed PA, enter into pvlist.
6300 	 */
6301 	if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) {
6302 		if ((flags & PMAP_CANFAIL) != 0)
6303 			goto out;
6304 		panic("pmap_enter: cannot allocate PV entry");
6305 	}
6306 
6307 	setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
6308 	pm->pm_stats.resident_count++;
6309 out:
6310 	PMAP_UNLOCK();
6311 	splx(s);
6312 	return (error);
6313 }
6314 
6315 /* enter new (or change existing) user mapping */
6316 int
6317 pmap_enu4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
6318 	   struct vm_page *pg, int pteproto)
6319 {
6320 	int vr, vs, *pte, tpte, s;
6321 	int error = 0;
6322 	struct regmap *rp;
6323 	struct segmap *sp;
6324 	bool owired;
6325 
6326 #ifdef DEBUG
6327 	if (KERNBASE < va)
6328 		panic("pmap_enu4m: can't enter va 0x%lx above KERNBASE", va);
6329 #endif
6330 
6331 	pm->pm_flags &= ~PMAP_USERCACHECLEAN;
6332 
6333 	vr = VA_VREG(va);
6334 	vs = VA_VSEG(va);
6335 	rp = &pm->pm_regmap[vr];
6336 	s = splvm();			/* XXX conservative */
6337 	PMAP_LOCK();
6338 
6339 	if (rp->rg_segmap == NULL) {
6340 		/* definitely a new mapping */
6341 		int mflag = PR_NOWAIT;
6342 
6343 	rretry:
6344 		sp = (struct segmap *)pool_get(&segmap_pool, mflag);
6345 		if (sp == NULL) {
6346 			if ((flags & PMAP_CANFAIL) != 0) {
6347 				error = ENOMEM;
6348 				goto out;
6349 			}
6350 			mflag = PR_WAITOK;
6351 			goto rretry;
6352 		}
6353 #ifdef DEBUG
6354 		if (rp->rg_segmap != NULL)
6355 			panic("pmap_enu4m: segment filled during sleep");
6356 #endif
6357 		qzero((void *)sp, NSEGRG * sizeof (struct segmap));
6358 		rp->rg_segmap = sp;
6359 		rp->rg_nsegmap = 0;
6360 		rp->rg_seg_ptps = NULL;
6361 	}
6362 	if (rp->rg_seg_ptps == NULL) {
6363 		/* Need a segment table */
6364 		int i, *ptd;
6365 		int mflag = PR_NOWAIT;
6366 
6367 	sretry:
6368 		ptd = pool_get(&L23_pool, mflag);
6369 		if (ptd == NULL) {
6370 			if ((flags & PMAP_CANFAIL) != 0) {
6371 				error = ENOMEM;
6372 				goto out;
6373 			}
6374 			mflag = PR_WAITOK;
6375 			goto sretry;
6376 		}
6377 
6378 		rp->rg_seg_ptps = ptd;
6379 		for (i = 0; i < SRMMU_L2SIZE; i++)
6380 			setpgt4m(&ptd[i], SRMMU_TEINVALID);
6381 
6382 		/* Replicate segment allocation in each CPU's region table */
6383 #ifdef MULTIPROCESSOR
6384 		for (i = 0; i < sparc_ncpus; i++)
6385 #else
6386 		i = 0;
6387 #endif
6388 		{
6389 #if defined(MULTIPROCESSOR)
6390 			if ((cpus[i]->flags & CPUFLG_HATCHED) == 0)
6391 				continue;
6392 #endif
6393 			setpgt4m(&pm->pm_reg_ptps[i][vr],
6394 				 (VA2PA((void *)ptd) >> SRMMU_PPNPASHIFT) |
6395 					SRMMU_TEPTD);
6396 		}
6397 	}
6398 
6399 	sp = &rp->rg_segmap[vs];
6400 
6401 	owired = false;
6402 	if ((pte = sp->sg_pte) == NULL) {
6403 		/* definitely a new mapping */
6404 		int i;
6405 		int mflag = PR_NOWAIT;
6406 
6407 		pte = pool_get(&L23_pool, mflag);
6408 		if (pte == NULL) {
6409 			if ((flags & PMAP_CANFAIL) != 0) {
6410 				error = ENOMEM;
6411 				goto out;
6412 			}
6413 			panic("pmap_enter: cannot allocate PTE table");
6414 		}
6415 
6416 		sp->sg_pte = pte;
6417 		sp->sg_npte = 1;
6418 		rp->rg_nsegmap++;
6419 		for (i = 0; i < SRMMU_L3SIZE; i++)
6420 			setpgt4m(&pte[i], SRMMU_TEINVALID);
6421 		setpgt4m(&rp->rg_seg_ptps[vs],
6422 			(VA2PA((void *)pte) >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
6423 	} else {
6424 #ifdef DIAGNOSTIC
6425 		if (sp->sg_npte <= 0)
6426 			panic("pm %p: npte %d", pm, sp->sg_npte);
6427 #endif
6428 		/*
6429 		 * Might be a change: fetch old pte
6430 		 */
6431 		tpte = pte[VA_SUN4M_VPG(va)];
6432 
6433 		if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) {
6434 
6435 			/* old mapping exists, and is of the same pa type */
6436 			if ((tpte & SRMMU_PPNMASK) ==
6437 			    (pteproto & SRMMU_PPNMASK)) {
6438 				/* just changing prot and/or wiring */
6439 				/* caller should call this directly: */
6440 				pmap_changeprot4m(pm, va, prot, flags);
6441 				PMAP_UNLOCK();
6442 				splx(s);
6443 				return (0);
6444 			}
6445 			/*
6446 			 * Switcheroo: changing pa for this va.
6447 			 * If old pa was managed, remove from pvlist.
6448 			 * If old page was cached, flush cache.
6449 			 */
6450 #ifdef DEBUG
6451 			if (pmapdebug & PDB_SWITCHMAP)
6452 				printf("%s[%d]: pmap_enu: changing existing "
6453 					"va 0x%x: pte 0x%x=>0x%x\n",
6454 					curproc->p_comm, curproc->p_pid,
6455 					(int)va, tpte, pteproto);
6456 #endif
6457 			if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
6458 				struct vm_page *opg;
6459 				if ((opg = pvhead4m(tpte)) != NULL) {
6460 					VM_MDPAGE_PVHEAD(opg)->pv_flags |=
6461 							MR4M(tpte);
6462 					pv_unlink4m(opg, pm, va);
6463 				}
6464 				if (pm->pm_ctx && (tpte & SRMMU_PG_C))
6465 					cache_flush_page(va, pm->pm_ctxnum);
6466 			}
6467 			/*
6468 			 * We end up in this `change map' branch relatively
6469 			 * infrequently.
6470 			 * Invalidate the mapping now, so we can avoid the
6471 			 * de-map and update protocol when setting the new
6472 			 * PTE below.
6473 			 */
6474 			setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
6475 				SRMMU_TEINVALID, pm->pm_ctx != NULL,
6476 				pm->pm_ctxnum, PMAP_CPUSET(pm));
6477 			pm->pm_stats.resident_count--;
6478 			owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
6479 		} else {
6480 			/* adding new entry */
6481 			sp->sg_npte++;
6482 		}
6483 	}
6484 
6485 	if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) {
6486 		if (--sp->sg_npte == 0)
6487 			/* Sigh, undo pgt allocations */
6488 			pgt_lvl23_remove4m(pm, rp, sp, vr, vs);
6489 
6490 		if ((flags & PMAP_CANFAIL) != 0)
6491 			goto out;
6492 		panic("pmap_enter: cannot allocate PV entry");
6493 	}
6494 
6495 	/*
6496 	 * Update PTEs, flush TLB as necessary.
6497 	 */
6498 	setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
6499 	pm->pm_stats.resident_count++;
6500 	if (owired) {
6501 		pm->pm_stats.wired_count--;
6502 		sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
6503 	}
6504 	if (flags & PMAP_WIRED) {
6505 		pm->pm_stats.wired_count++;
6506 		sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va));
6507 	}
6508 
6509 out:
6510 	PMAP_UNLOCK();
6511 	splx(s);
6512 	return (error);
6513 }
6514 
6515 void
6516 pmap_kenter_pa4m(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
6517 {
6518 	struct pmap *pm = pmap_kernel();
6519 	struct regmap *rp;
6520 	struct segmap *sp;
6521 	int pteproto, vr, vs;
6522 
6523 	/* Initialise pteproto with cache bit */
6524 	pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0;
6525 	pteproto |= SRMMU_TEPTE | PPROT_S;
6526 	pteproto |= PMAP_T2PTE_SRMMU(pa);
6527 	pteproto |= (atop(pa & ~PMAP_TNC_SRMMU) << SRMMU_PPNSHIFT);
6528 	pteproto |= pte_kprot4m(prot);
6529 
6530 	vr = VA_VREG(va);
6531 	vs = VA_VSEG(va);
6532 	rp = &pm->pm_regmap[vr];
6533 	sp = &rp->rg_segmap[vs];
6534 
6535 	KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) != SRMMU_TEPTE);
6536 
6537 	setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
6538 }
6539 
6540 void
6541 pmap_kremove4m(vaddr_t va, vsize_t len)
6542 {
6543 	struct pmap *pm = pmap_kernel();
6544 	struct regmap *rp;
6545 	struct segmap *sp;
6546 	vaddr_t endva, nva;
6547 	int vr, vs;
6548 	int tpte, perpage, npg, s;
6549 
6550 	/*
6551 	 * The kernel pmap doesn't need to be locked, but the demap lock
6552 	 * in updatepte() requires interrupt protection.
6553 	 */
6554 	s = splvm();
6555 
6556 	endva = va + len;
6557 	for (; va < endva; va = nva) {
6558 		/* do one virtual segment at a time */
6559 		vr = VA_VREG(va);
6560 		vs = VA_VSEG(va);
6561 		nva = VSTOVA(vr, vs + 1);
6562 		if (nva == 0 || nva > endva) {
6563 			nva = endva;
6564 		}
6565 
6566 		rp = &pm->pm_regmap[vr];
6567 		sp = &rp->rg_segmap[vs];
6568 
6569 		/* decide how to flush the cache */
6570 		npg = (nva - va) >> PGSHIFT;
6571 		if (npg > PMAP_SFL_THRESHOLD) {
6572 			/* flush the whole segment */
6573 			perpage = 0;
6574 			if (CACHEINFO.c_vactype != VAC_NONE) {
6575 				cache_flush_segment(vr, vs, 0);
6576 			}
6577 		} else {
6578 			/*
6579 			 * flush each page individually;
6580 			 * some never need flushing
6581 			 */
6582 			perpage = (CACHEINFO.c_vactype != VAC_NONE);
6583 		}
6584 		for (; va < nva; va += NBPG) {
6585 			tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
6586 			if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
6587 				continue;
6588 
6589 			if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
6590 				/* if cacheable, flush page as needed */
6591 				if (perpage && (tpte & SRMMU_PG_C))
6592 					cache_flush_page(va, 0);
6593 			}
6594 			setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
6595 				 SRMMU_TEINVALID, 1, 0, CPUSET_ALL);
6596 		}
6597 	}
6598 	splx(s);
6599 }
6600 
6601 /*
6602  * Change protection on a range of kernel addresses.
6603  */
6604 void
6605 pmap_kprotect4m(vaddr_t va, vsize_t size, vm_prot_t prot)
6606 {
6607 	struct pmap *pm = pmap_kernel();
6608 	int pte, newprot, s;
6609 	struct regmap *rp;
6610 	struct segmap *sp;
6611 
6612 	size = roundup(size,NBPG);
6613 	newprot = pte_kprot4m(prot);
6614 
6615 	/*
6616 	 * The kernel pmap doesn't need to be locked, but the demap lock
6617 	 * in updatepte() requires interrupt protection.
6618 	 */
6619 	s = splvm();
6620 
6621 	while (size > 0) {
6622 		rp = &pm->pm_regmap[VA_VREG(va)];
6623 		sp = &rp->rg_segmap[VA_VSEG(va)];
6624 		pte = sp->sg_pte[VA_SUN4M_VPG(va)];
6625 
6626 		/*
6627 		 * Flush cache if page has been referenced to
6628 		 * avoid stale protection bits in the cache tags.
6629 		 */
6630 		if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) ==
6631 		    (SRMMU_PG_C|PG_SUN4M_OBMEM))
6632 			cache_flush_page(va, 0);
6633 
6634 		setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
6635 			 (pte & ~SRMMU_PROT_MASK) | newprot,
6636 			 1, pm->pm_ctxnum, PMAP_CPUSET(pm));
6637 
6638 		va += NBPG;
6639 		size -= NBPG;
6640 	}
6641 	splx(s);
6642 }
6643 #endif /* SUN4M || SUN4D */
6644 
6645 /*
6646  * Clear the wiring attribute for a map/virtual-address pair.
6647  */
6648 /* ARGSUSED */
6649 void
6650 pmap_unwire(struct pmap *pm, vaddr_t va)
6651 {
6652 	int vr, vs, *ptep;
6653 	struct regmap *rp;
6654 	struct segmap *sp;
6655 	bool owired;
6656 
6657 	vr = VA_VREG(va);
6658 	vs = VA_VSEG(va);
6659 	rp = &pm->pm_regmap[vr];
6660 	sp = &rp->rg_segmap[vs];
6661 
6662 	owired = false;
6663 	if (CPU_HAS_SUNMMU) {
6664 		ptep = &sp->sg_pte[VA_VPG(va)];
6665 		owired = *ptep & PG_WIRED;
6666 		*ptep &= ~PG_WIRED;
6667 	}
6668 	if (CPU_HAS_SRMMU) {
6669 		owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
6670 		sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
6671 	}
6672 	if (!owired) {
6673 		pmap_stats.ps_useless_changewire++;
6674 		return;
6675 	}
6676 
6677 	pm->pm_stats.wired_count--;
6678 #if defined(SUN4) || defined(SUN4C)
6679 	if (CPU_HAS_SUNMMU && --sp->sg_nwired <= 0) {
6680 #ifdef DIAGNOSTIC
6681 		if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
6682 			panic("pmap_unwire: pm %p, va %lx: nleft=%d, nwired=%d",
6683 				pm, va, sp->sg_npte, sp->sg_nwired);
6684 #endif
6685 		if (sp->sg_pmeg != seginval)
6686 			mmu_pmeg_unlock(sp->sg_pmeg);
6687 	}
6688 #endif /* SUN4 || SUN4C */
6689 }
6690 
6691 /*
6692  * Extract the physical page address associated
6693  * with the given map/virtual_address pair.
6694  * GRR, the vm code knows; we should not have to do this!
6695  */
6696 
6697 #if defined(SUN4) || defined(SUN4C)
6698 bool
6699 pmap_extract4_4c(struct pmap *pm, vaddr_t va, paddr_t *pap)
6700 {
6701 	int vr, vs;
6702 	struct regmap *rp;
6703 	struct segmap *sp;
6704 	int pte, *ptep;
6705 
6706 	vr = VA_VREG(va);
6707 	vs = VA_VSEG(va);
6708 	rp = &pm->pm_regmap[vr];
6709 	if (rp->rg_segmap == NULL) {
6710 #ifdef DEBUG
6711 		if (pmapdebug & PDB_FOLLOW)
6712 			printf("pmap_extract: invalid segment (%d)\n", vr);
6713 #endif
6714 		return (false);
6715 	}
6716 	sp = &rp->rg_segmap[vs];
6717 	ptep = sp->sg_pte;
6718 	if (ptep == NULL) {
6719 #ifdef DEBUG
6720 		if (pmapdebug & PDB_FOLLOW)
6721 			printf("pmap_extract: invalid segment\n");
6722 #endif
6723 		return (false);
6724 	}
6725 	pte = ptep[VA_VPG(va)];
6726 
6727 	if ((pte & PG_V) == 0) {
6728 #ifdef DEBUG
6729 		if (pmapdebug & PDB_FOLLOW)
6730 			printf("pmap_extract: invalid pte\n");
6731 #endif
6732 		return (false);
6733 	}
6734 	pte &= PG_PFNUM;
6735 	if (pap != NULL)
6736 		*pap = (pte << PGSHIFT) | (va & PGOFSET);
6737 	return (true);
6738 }
6739 #endif /* SUN4 || SUN4C */
6740 
6741 #if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of pmap_extract */
6742 /*
6743  * Extract the physical page address associated
6744  * with the given map/virtual_address pair.
6745  * GRR, the vm code knows; we should not have to do this!
6746  */
6747 bool
6748 pmap_extract4m(struct pmap *pm, vaddr_t va, paddr_t *pap)
6749 {
6750 	struct regmap *rp;
6751 	struct segmap *sp;
6752 	int pte;
6753 	int vr, vs, s, v = false;
6754 	bool can_lock = lock_available;
6755 
6756 	vr = VA_VREG(va);
6757 	vs = VA_VSEG(va);
6758 
6759 	/*
6760 	 * The kernel pmap doesn't need to be locked, but the demap lock
6761 	 * requires interrupt protection.
6762 	 */
6763 	s = splvm();
6764 	if (pm != pmap_kernel()) {
6765 		PMAP_LOCK();
6766 	}
6767 
6768 	rp = &pm->pm_regmap[vr];
6769 	if (rp->rg_segmap == NULL) {
6770 #ifdef DEBUG
6771 		if (pmapdebug & PDB_FOLLOW)
6772 			printf("pmap_extract: no segmap\n");
6773 #endif
6774 		goto out;
6775 	}
6776 
6777 	sp = &rp->rg_segmap[vs];
6778 	if (sp->sg_pte == NULL) {
6779 #ifdef DEBUG
6780 		if (pmapdebug & PDB_FOLLOW)
6781 			printf("pmap_extract: no ptes\n");
6782 #endif
6783 		goto out;
6784 	}
6785 
6786 	pte = sp->sg_pte[VA_SUN4M_VPG(va)];
6787 	if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) {
6788 #ifdef DEBUG
6789 		if (pmapdebug & PDB_FOLLOW)
6790 			printf("pmap_extract: invalid pte of type %d\n",
6791 			       pte & SRMMU_TETYPE);
6792 #endif
6793 		/*
6794 		 * We can read a spurious invalid pte if the system is in
6795 		 * the middle of the PTE update protocol. So, acquire the
6796 		 * demap lock and retry.
6797 		 */
6798 		if (__predict_true(can_lock))
6799 			mutex_spin_enter(&demap_lock);
6800 		pte = sp->sg_pte[VA_SUN4M_VPG(va)];
6801 		if (__predict_true(can_lock))
6802 			mutex_spin_exit(&demap_lock);
6803 		if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
6804 			goto out;
6805 	}
6806 #ifdef DIAGNOSTIC
6807 	if (pm != pmap_kernel() && sp->sg_npte <= 0)
6808 		panic("pmap_extract: pm %p: npte = %d\n", pm, sp->sg_npte);
6809 #endif
6810 
6811 	if (pap != NULL)
6812 		*pap = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT) |
6813 		    VA_OFF(va);
6814 
6815 	v = true;
6816 out:
6817 	if (pm != pmap_kernel()) {
6818 		PMAP_UNLOCK();
6819 	}
6820 	splx(s);
6821 	return (v);
6822 }
6823 #endif /* sun4m */
6824 
6825 int pmap_copy_disabled=0;
6826 
6827 /*
6828  * Copy the range specified by src_addr/len
6829  * from the source map to the range dst_addr/len
6830  * in the destination map.
6831  *
6832  * This routine is only advisory and need not do anything.
6833  */
6834 /* ARGSUSED */
6835 void
6836 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap,
6837 	  vaddr_t dst_addr, vsize_t len, vaddr_t src_addr)
6838 {
6839 #if notyet
6840 	struct regmap *rp;
6841 	struct segmap *sp;
6842 
6843 	if (pmap_copy_disabled)
6844 		return;
6845 #ifdef DIAGNOSTIC
6846 	if (VA_OFF(src_addr) != 0)
6847 		printf("pmap_copy: addr not page aligned: 0x%lx\n", src_addr);
6848 	if ((len & (NBPG-1)) != 0)
6849 		printf("pmap_copy: length not page aligned: 0x%lx\n", len);
6850 #endif
6851 
6852 	if (src_pmap == NULL)
6853 		return;
6854 
6855 	if (CPU_HAS_SRMMU) {
6856 		int i, npg, pte;
6857 		paddr_t pa;
6858 
6859 		npg = len >> PGSHIFT;
6860 		for (i = 0; i < npg; i++) {
6861 			if ((rp = src_pmap->pm_regmap) == NULL)
6862 				continue;
6863 			rp += VA_VREG(src_addr);
6864 
6865 			if ((sp = rp->rg_segmap) == NULL)
6866 				continue;
6867 			sp += VA_VSEG(src_addr);
6868 			if (sp->sg_npte == 0)
6869 				continue;
6870 
6871 			pte = sp->sg_pte[VA_SUN4M_VPG(src_addr)];
6872 			if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
6873 				continue;
6874 
6875 			pa = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT);
6876 			pmap_enter(dst_pmap, dst_addr,
6877 				   pa,
6878 				   /* XXX - need to copy VM_PROT_EXEC too */
6879 				   (pte & PPROT_WRITE)
6880 					? (VM_PROT_WRITE | VM_PROT_READ)
6881 					: VM_PROT_READ,
6882 				   0);
6883 			src_addr += NBPG;
6884 			dst_addr += NBPG;
6885 		}
6886 		pmap_update(dst_pmap);
6887 	}
6888 #endif
6889 }
6890 
6891 #if defined(SUN4) || defined(SUN4C)
6892 /*
6893  * Clear the modify bit for the given physical page.
6894  */
6895 bool
6896 pmap_clear_modify4_4c(struct vm_page *pg)
6897 {
6898 	bool rv;
6899 
6900 	(void) pv_syncflags4_4c(pg);
6901 	rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD;
6902 	VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD;
6903 	return (rv);
6904 }
6905 
6906 /*
6907  * Tell whether the given physical page has been modified.
6908  */
6909 bool
6910 pmap_is_modified4_4c(struct vm_page *pg)
6911 {
6912 
6913 	return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD ||
6914 		pv_syncflags4_4c(pg) & PV_MOD);
6915 }
6916 
6917 /*
6918  * Clear the reference bit for the given physical page.
6919  */
6920 bool
6921 pmap_clear_reference4_4c(struct vm_page *pg)
6922 {
6923 	bool rv;
6924 
6925 	(void) pv_syncflags4_4c(pg);
6926 	rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF;
6927 	VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF;
6928 	return (rv);
6929 }
6930 
6931 /*
6932  * Tell whether the given physical page has been referenced.
6933  */
6934 bool
6935 pmap_is_referenced4_4c(struct vm_page *pg)
6936 {
6937 
6938 	return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF ||
6939 		pv_syncflags4_4c(pg) & PV_REF);
6940 }
6941 #endif /* SUN4 || SUN4C */
6942 
6943 #if defined(SUN4M) || defined(SUN4D)
6944 
6945 /*
6946  * SRMMU versions of bit test/set routines
6947  *
6948  * Note that the 4m-specific routines should eventually service these
6949  * requests from their page tables, and the whole pvlist bit mess should
6950  * be dropped for the 4m (unless this causes a performance hit from
6951  * tracing down pagetables/regmap/segmaps).
6952  */
6953 
6954 /*
6955  * Clear the modify bit for the given physical page.
6956  */
6957 bool
6958 pmap_clear_modify4m(struct vm_page *pg)
6959 {
6960 	bool rv;
6961 
6962 	(void) pv_syncflags4m(pg);
6963 	rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M;
6964 	VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD4M;
6965 	return (rv);
6966 }
6967 
6968 /*
6969  * Tell whether the given physical page has been modified.
6970  */
6971 bool
6972 pmap_is_modified4m(struct vm_page *pg)
6973 {
6974 
6975 	return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M ||
6976 		pv_syncflags4m(pg) & PV_MOD4M);
6977 }
6978 
6979 /*
6980  * Clear the reference bit for the given physical page.
6981  */
6982 bool
6983 pmap_clear_reference4m(struct vm_page *pg)
6984 {
6985 	bool rv;
6986 
6987 	(void) pv_syncflags4m(pg);
6988 	rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M;
6989 	VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF4M;
6990 	return (rv);
6991 }
6992 
6993 /*
6994  * Tell whether the given physical page has been referenced.
6995  */
6996 bool
6997 pmap_is_referenced4m(struct vm_page *pg)
6998 {
6999 
7000 	return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M ||
7001 		pv_syncflags4m(pg) & PV_REF4M);
7002 }
7003 #endif /* SUN4M || SUN4D */
7004 
7005 /*
7006  * Fill the given MI physical page with zero bytes.
7007  *
7008  * We avoid stomping on the cache.
7009  * XXX	might be faster to use destination's context and allow cache to fill?
7010  */
7011 
7012 #if defined(SUN4) || defined(SUN4C)
7013 
7014 void
7015 pmap_zero_page4_4c(paddr_t pa)
7016 {
7017 	struct vm_page *pg;
7018 	void *va;
7019 	int pte;
7020 
7021 	if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
7022 		/*
7023 		 * The following might not be necessary since the page
7024 		 * is being cleared because it is about to be allocated,
7025 		 * i.e., is in use by no one.
7026 		 */
7027 		pv_flushcache4_4c(pg);
7028 	}
7029 	pte = PG_V | PG_S | PG_W | PG_NC | (atop(pa) & PG_PFNUM);
7030 
7031 	va = cpuinfo.vpage[0];
7032 	setpte4(va, pte);
7033 	qzero(va, NBPG);
7034 	setpte4(va, 0);
7035 }
7036 
7037 /*
7038  * Copy the given MI physical source page to its destination.
7039  *
7040  * We avoid stomping on the cache as above (with same `XXX' note).
7041  * We must first flush any write-back cache for the source page.
7042  * We go ahead and stomp on the kernel's virtual cache for the
7043  * source page, since the cache can read memory MUCH faster than
7044  * the processor.
7045  */
7046 void
7047 pmap_copy_page4_4c(paddr_t src, paddr_t dst)
7048 {
7049 	struct vm_page *pg;
7050 	char *sva, *dva;
7051 	int spte, dpte;
7052 
7053 	if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
7054 		if (CACHEINFO.c_vactype == VAC_WRITEBACK)
7055 			pv_flushcache4_4c(pg);
7056 	}
7057 	spte = PG_V | PG_S | (atop(src) & PG_PFNUM);
7058 
7059 	if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
7060 		/* similar `might not be necessary' comment applies */
7061 		if (CACHEINFO.c_vactype != VAC_NONE)
7062 			pv_flushcache4_4c(pg);
7063 	}
7064 	dpte = PG_V | PG_S | PG_W | PG_NC | (atop(dst) & PG_PFNUM);
7065 
7066 	sva = cpuinfo.vpage[0];
7067 	dva = cpuinfo.vpage[1];
7068 	setpte4(sva, spte);
7069 	setpte4(dva, dpte);
7070 	qcopy(sva, dva, NBPG);	/* loads cache, so we must ... */
7071 	cache_flush_page((vaddr_t)sva, getcontext4());
7072 	setpte4(sva, 0);
7073 	setpte4(dva, 0);
7074 }
7075 #endif /* SUN4 || SUN4C */
7076 
7077 #if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of copy/zero routines */
7078 /*
7079  * Fill the given MI physical page with zero bytes.
7080  *
7081  * We avoid stomping on the cache.
7082  * XXX	might be faster to use destination's context and allow cache to fill?
7083  */
7084 void
7085 pmap_zero_page4m(paddr_t pa)
7086 {
7087 	struct vm_page *pg;
7088 	void *va;
7089 	int pte;
7090 
7091 	if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
7092 		/*
7093 		 * The following VAC flush might not be necessary since the
7094 		 * page is being cleared because it is about to be allocated,
7095 		 * i.e., is in use by no one.
7096 		 * In the case of a physical cache, a flush (or just an
7097 		 * invalidate, if possible) is usually necessary when using
7098 		 * uncached access to clear it.
7099 		 */
7100 		if (CACHEINFO.c_vactype != VAC_NONE)
7101 			pv_flushcache4m(pg);
7102 		else
7103 			pcache_flush_page(pa, 1);
7104 	}
7105 	pte = SRMMU_TEPTE | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT);
7106 	if (cpuinfo.flags & CPUFLG_CACHE_MANDATORY)
7107 		pte |= SRMMU_PG_C;
7108 
7109 	va = cpuinfo.vpage[0];
7110 	setpgt4m(cpuinfo.vpage_pte[0], pte);
7111 	qzero(va, NBPG);
7112 	/*
7113 	 * Remove temporary mapping (which is kernel-only, so the
7114 	 * context used for TLB flushing does not matter)
7115 	 */
7116 	sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3);
7117 	setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
7118 }
7119 
7120 /*
7121  * Viking/MXCC specific version of pmap_zero_page
7122  */
7123 void
7124 pmap_zero_page_viking_mxcc(paddr_t pa)
7125 {
7126 	u_int offset;
7127 	u_int stream_data_addr = MXCC_STREAM_DATA;
7128 	uint64_t v = (uint64_t)pa;
7129 
7130 	/* Load MXCC stream data register with 0 (bottom 32 bytes only) */
7131 	stda(stream_data_addr+0, ASI_CONTROL, 0);
7132 	stda(stream_data_addr+8, ASI_CONTROL, 0);
7133 	stda(stream_data_addr+16, ASI_CONTROL, 0);
7134 	stda(stream_data_addr+24, ASI_CONTROL, 0);
7135 
7136 	/* Then write the stream data register to each block in the page */
7137 	v |= MXCC_STREAM_C;
7138 	for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) {
7139 		stda(MXCC_STREAM_DST, ASI_CONTROL, v | offset);
7140 	}
7141 }
7142 
7143 /*
7144  * HyperSPARC/RT625 specific version of pmap_zero_page
7145  */
7146 void
7147 pmap_zero_page_hypersparc(paddr_t pa)
7148 {
7149 	struct vm_page *pg;
7150 	void *va;
7151 	int pte;
7152 	int offset;
7153 
7154 	/*
7155 	 * We still have to map the page, since ASI_BLOCKFILL
7156 	 * takes virtual addresses. This also means we have to
7157 	 * consider cache aliasing; therefore we still need
7158 	 * to flush the cache here. All we gain is the speed-up
7159 	 * in zero-fill loop itself..
7160 	 */
7161 	if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
7162 		/*
7163 		 * The following might not be necessary since the page
7164 		 * is being cleared because it is about to be allocated,
7165 		 * i.e., is in use by no one.
7166 		 */
7167 		if (CACHEINFO.c_vactype != VAC_NONE)
7168 			pv_flushcache4m(pg);
7169 	}
7170 	pte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT);
7171 
7172 	va = cpuinfo.vpage[0];
7173 	setpgt4m(cpuinfo.vpage_pte[0], pte);
7174 	for (offset = 0; offset < NBPG; offset += 32) {
7175 		sta((char *)va + offset, ASI_BLOCKFILL, 0);
7176 	}
7177 	/* Remove temporary mapping */
7178 	sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3);
7179 	setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
7180 }
7181 
7182 /*
7183  * Copy the given MI physical source page to its destination.
7184  *
7185  * We avoid stomping on the cache as above (with same `XXX' note).
7186  * We must first flush any write-back cache for the source page.
7187  * We go ahead and stomp on the kernel's virtual cache for the
7188  * source page, since the cache can read memory MUCH faster than
7189  * the processor.
7190  */
7191 void
7192 pmap_copy_page4m(paddr_t src, paddr_t dst)
7193 {
7194 	struct vm_page *pg;
7195 	void *sva, *dva;
7196 	int spte, dpte;
7197 
7198 	if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
7199 		if (CACHEINFO.c_vactype == VAC_WRITEBACK)
7200 			pv_flushcache4m(pg);
7201 	}
7202 
7203 	spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX |
7204 		(src >> SRMMU_PPNPASHIFT);
7205 
7206 	if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
7207 		/* similar `might not be necessary' comment applies */
7208 		if (CACHEINFO.c_vactype != VAC_NONE)
7209 			pv_flushcache4m(pg);
7210 		else
7211 			pcache_flush_page(dst, 1);
7212 	}
7213 
7214 	dpte = SRMMU_TEPTE | PPROT_N_RWX | (dst >> SRMMU_PPNPASHIFT);
7215 	if (cpuinfo.flags & CPUFLG_CACHE_MANDATORY)
7216 		dpte |= SRMMU_PG_C;
7217 
7218 	sva = cpuinfo.vpage[0];
7219 	dva = cpuinfo.vpage[1];
7220 	setpgt4m(cpuinfo.vpage_pte[0], spte);
7221 	setpgt4m(cpuinfo.vpage_pte[1], dpte);
7222 	qcopy(sva, dva, NBPG);	/* loads cache, so we must ... */
7223 	cpuinfo.sp_vcache_flush_page((vaddr_t)sva, getcontext4m());
7224 	sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3);
7225 	setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
7226 	sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3);
7227 	setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID);
7228 }
7229 
7230 /*
7231  * Viking/MXCC specific version of pmap_copy_page
7232  */
7233 void
7234 pmap_copy_page_viking_mxcc(paddr_t src, paddr_t dst)
7235 {
7236 	u_int offset;
7237 	uint64_t v1 = (uint64_t)src;
7238 	uint64_t v2 = (uint64_t)dst;
7239 
7240 	/* Enable cache-coherency */
7241 	v1 |= MXCC_STREAM_C;
7242 	v2 |= MXCC_STREAM_C;
7243 
7244 	/* Copy through stream data register */
7245 	for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) {
7246 		stda(MXCC_STREAM_SRC, ASI_CONTROL, v1 | offset);
7247 		stda(MXCC_STREAM_DST, ASI_CONTROL, v2 | offset);
7248 	}
7249 }
7250 
7251 /*
7252  * HyperSPARC/RT625 specific version of pmap_copy_page
7253  */
7254 void
7255 pmap_copy_page_hypersparc(paddr_t src, paddr_t dst)
7256 {
7257 	struct vm_page *pg;
7258 	void *sva, *dva;
7259 	int spte, dpte;
7260 	int offset;
7261 
7262 	/*
7263 	 * We still have to map the pages, since ASI_BLOCKCOPY
7264 	 * takes virtual addresses. This also means we have to
7265 	 * consider cache aliasing; therefore we still need
7266 	 * to flush the cache here. All we gain is the speed-up
7267 	 * in copy loop itself..
7268 	 */
7269 
7270 	if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
7271 		if (CACHEINFO.c_vactype == VAC_WRITEBACK)
7272 			pv_flushcache4m(pg);
7273 	}
7274 
7275 	spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX |
7276 		(src >> SRMMU_PPNPASHIFT);
7277 
7278 	if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
7279 		/* similar `might not be necessary' comment applies */
7280 		if (CACHEINFO.c_vactype != VAC_NONE)
7281 			pv_flushcache4m(pg);
7282 	}
7283 
7284 	dpte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX |
7285 		(dst >> SRMMU_PPNPASHIFT);
7286 
7287 	sva = cpuinfo.vpage[0];
7288 	dva = cpuinfo.vpage[1];
7289 	setpgt4m(cpuinfo.vpage_pte[0], spte);
7290 	setpgt4m(cpuinfo.vpage_pte[1], dpte);
7291 
7292 	for (offset = 0; offset < NBPG; offset += 32) {
7293 		sta((char *)dva + offset, ASI_BLOCKCOPY, (char *)sva + offset);
7294 	}
7295 
7296 	sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3);
7297 	setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
7298 	sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3);
7299 	setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID);
7300 }
7301 #endif /* SUN4M || SUN4D */
7302 
7303 /*
7304  * Turn off cache for a given (va, number of pages).
7305  *
7306  * We just assert PG_NC for each PTE; the addresses must reside
7307  * in locked kernel space.  A cache flush is also done.
7308  */
7309 void
7310 kvm_uncache(char *va, int npages)
7311 {
7312 	struct vm_page *pg;
7313 	int pte;
7314 
7315 	if (CPU_HAS_SRMMU) {
7316 #if defined(SUN4M) || defined(SUN4D)
7317 		for (; --npages >= 0; va = (char *)va + NBPG) {
7318 			pte = getpte4m((vaddr_t) va);
7319 			if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
7320 				panic("kvm_uncache: table entry not pte");
7321 
7322 			if ((pte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
7323 				if ((pg = pvhead4m(pte)) != NULL) {
7324 					pv_uncache(pg);
7325 					return;
7326 				}
7327 				cache_flush_page((vaddr_t)va, 0);
7328 			}
7329 
7330 			pte &= ~SRMMU_PG_C;
7331 			setpte4m((vaddr_t)va, pte);
7332 		}
7333 #endif
7334 	} else {
7335 #if defined(SUN4) || defined(SUN4C)
7336 		for (; --npages >= 0; va += NBPG) {
7337 			pte = getpte4(va);
7338 			if ((pte & PG_V) == 0)
7339 				panic("kvm_uncache !pg_v");
7340 
7341 			if ((pte & PG_TYPE) == PG_OBMEM) {
7342 				if ((pg = pvhead4_4c(pte)) != NULL) {
7343 					pv_uncache(pg);
7344 					return;
7345 				}
7346 				cache_flush_page((vaddr_t)va, 0);
7347 			}
7348 			pte |= PG_NC;
7349 			setpte4(va, pte);
7350 		}
7351 #endif
7352 	}
7353 }
7354 
7355 #if 0 /* not used */
7356 /*
7357  * Turn on IO cache for a given (va, number of pages).
7358  *
7359  * We just assert PG_NC for each PTE; the addresses must reside
7360  * in locked kernel space.  A cache flush is also done.
7361  */
7362 void
7363 kvm_iocache(char *va, int npages)
7364 {
7365 
7366 #if defined(SUN4M)
7367 	if (CPU_ISSUN4M) /* %%%: Implement! */
7368 		panic("kvm_iocache: 4m iocache not implemented");
7369 #endif
7370 #if defined(SUN4D)
7371 	if (CPU_ISSUN4D) /* %%%: Implement! */
7372 		panic("kvm_iocache: 4d iocache not implemented");
7373 #endif
7374 #if defined(SUN4) || defined(SUN4C)
7375 	for (; --npages >= 0; va += NBPG) {
7376 		int pte = getpte4(va);
7377 		if ((pte & PG_V) == 0)
7378 			panic("kvm_iocache !pg_v");
7379 		pte |= PG_IOC;
7380 		setpte4(va, pte);
7381 	}
7382 #endif
7383 }
7384 #endif
7385 
7386 /*
7387  * Find first virtual address >= *va that is
7388  * least likely to cause cache aliases.
7389  * (This will just seg-align mappings.)
7390  */
7391 void
7392 pmap_prefer(vaddr_t foff, vaddr_t *vap)
7393 {
7394 	vaddr_t va = *vap;
7395 	long d, m;
7396 
7397 	if (VA_INHOLE(va))
7398 		va = MMU_HOLE_END;
7399 
7400 	m = CACHE_ALIAS_DIST;
7401 	if (m == 0)		/* m=0 => no cache aliasing */
7402 		return;
7403 
7404 	d = foff - va;
7405 	d &= (m - 1);
7406 	*vap = va + d;
7407 }
7408 
7409 void
7410 pmap_redzone(void)
7411 {
7412 
7413 	pmap_remove(pmap_kernel(), KERNBASE, KERNBASE+NBPG);
7414 }
7415 
7416 /*
7417  * Activate the address space for the specified process.  If the
7418  * process is the current process, load the new MMU context.
7419  */
7420 void
7421 pmap_activate(struct lwp *l)
7422 {
7423 	pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap;
7424 	int s;
7425 
7426 	/*
7427 	 * This is essentially the same thing that happens in cpu_switch()
7428 	 * when the newly selected process is about to run, except that we
7429 	 * have to make sure to clean the register windows before we set
7430 	 * the new context.
7431 	 */
7432 
7433 	s = splvm();
7434 	if (l == curlwp) {
7435 		write_user_windows();
7436 		if (pm->pm_ctx == NULL) {
7437 			ctx_alloc(pm);	/* performs setcontext() */
7438 		} else {
7439 			/* Do any cache flush needed on context switch */
7440 			(*cpuinfo.pure_vcache_flush)();
7441 			setcontext(pm->pm_ctxnum);
7442 		}
7443 #if defined(MULTIPROCESSOR)
7444 		if (pm != pmap_kernel())
7445 			PMAP_SET_CPUSET(pm, &cpuinfo);
7446 #endif
7447 	}
7448 	splx(s);
7449 }
7450 
7451 /*
7452  * Deactivate the address space of the specified process.
7453  */
7454 void
7455 pmap_deactivate(struct lwp *l)
7456 {
7457 #if defined(MULTIPROCESSOR)
7458 	pmap_t pm;
7459 	struct proc *p;
7460 
7461 	p = l->l_proc;
7462 	if (p->p_vmspace &&
7463 	    (pm = p->p_vmspace->vm_map.pmap) != pmap_kernel()) {
7464 #if defined(SUN4M) || defined(SUN4D)
7465 		if (pm->pm_ctx && CPU_HAS_SRMMU)
7466 			sp_tlb_flush(0, pm->pm_ctxnum, ASI_SRMMUFP_L0);
7467 #endif
7468 
7469 		/* we no longer need broadcast tlb flushes for this pmap. */
7470 		PMAP_CLR_CPUSET(pm, &cpuinfo);
7471 	}
7472 #endif
7473 }
7474 
7475 #ifdef DEBUG
7476 /*
7477  * Check consistency of a pmap (time consuming!).
7478  */
7479 void
7480 pm_check(char *s, struct pmap *pm)
7481 {
7482 
7483 	if (pm == pmap_kernel())
7484 		pm_check_k(s, pm);
7485 	else
7486 		pm_check_u(s, pm);
7487 }
7488 
7489 void
7490 pm_check_u(char *s, struct pmap *pm)
7491 {
7492 	struct regmap *rp;
7493 	struct segmap *sp;
7494 	int cpu, n, vs, vr, j, m, *pte;
7495 
7496 	cpu = cpuinfo.ci_cpuid;
7497 
7498 	if (pm->pm_regmap == NULL)
7499 		panic("%s: CPU %d: CHK(pmap %p): no region mapping",
7500 			s, cpu, pm);
7501 
7502 #if defined(SUN4M) || defined(SUN4D)
7503 	if (CPU_HAS_SRMMU &&
7504 	    (pm->pm_reg_ptps[cpu] == NULL ||
7505 	     pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu])))
7506 		panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: "
7507 		      "tblva=%p, tblpa=0x%x",
7508 			s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]);
7509 
7510 	if (CPU_HAS_SRMMU && pm->pm_ctx != NULL &&
7511 	    (cpuinfo.ctx_tbl[pm->pm_ctxnum] != ((VA2PA((void *)pm->pm_reg_ptps[cpu])
7512 					      >> SRMMU_PPNPASHIFT) |
7513 					     SRMMU_TEPTD)))
7514 	    panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed "
7515 		  "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], pm->pm_ctxnum);
7516 #endif
7517 
7518 	for (vr = 0; vr < NUREG; vr++) {
7519 		rp = &pm->pm_regmap[vr];
7520 		if (rp->rg_nsegmap == 0)
7521 			continue;
7522 		if (rp->rg_segmap == NULL)
7523 			panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL",
7524 				s, cpu, vr, rp->rg_nsegmap);
7525 #if defined(SUN4M) || defined(SUN4D)
7526 		if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL)
7527 		    panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table",
7528 			  s, cpu, vr, rp->rg_nsegmap);
7529 		if (CPU_HAS_SRMMU &&
7530 		    pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >>
7531 					    SRMMU_PPNPASHIFT) | SRMMU_TEPTD))
7532 		    panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed",
7533 				s, cpu, vr);
7534 #endif
7535 		if ((unsigned int)rp < KERNBASE)
7536 			panic("%s: CPU %d: rp=%p", s, cpu, rp);
7537 		n = 0;
7538 		for (vs = 0; vs < NSEGRG; vs++) {
7539 			sp = &rp->rg_segmap[vs];
7540 			if ((unsigned int)sp < KERNBASE)
7541 				panic("%s: CPU %d: sp=%p", s, cpu, sp);
7542 			if (sp->sg_npte != 0) {
7543 				n++;
7544 				if (sp->sg_pte == NULL)
7545 					panic("%s: CPU %d: CHK(vr %d, vs %d): npte=%d, "
7546 					   "pte=NULL", s, cpu, vr, vs, sp->sg_npte);
7547 #if defined(SUN4M) || defined(SUN4D)
7548 				if (CPU_HAS_SRMMU &&
7549 				    rp->rg_seg_ptps[vs] !=
7550 				     ((VA2PA((void *)sp->sg_pte)
7551 					>> SRMMU_PPNPASHIFT) |
7552 				       SRMMU_TEPTD))
7553 				    panic("%s: CPU %d: CHK(vr %d, vs %d): SRMMU page "
7554 					  "table not installed correctly",
7555 						s, cpu, vr, vs);
7556 #endif
7557 				pte=sp->sg_pte;
7558 				m = 0;
7559 				for (j=0; j<NPTESG; j++,pte++)
7560 				    if ((CPU_HAS_SRMMU
7561 					 ?((*pte & SRMMU_TETYPE) == SRMMU_TEPTE)
7562 					 :(*pte & PG_V)))
7563 					m++;
7564 				if (m != sp->sg_npte)
7565 					printf("%s: CPU %d: user CHK(vr %d, vs %d): "
7566 					    "npte(%d) != # valid(%d)\n",
7567 						s, cpu, vr, vs, sp->sg_npte, m);
7568 			}
7569 		}
7570 		if (n != rp->rg_nsegmap)
7571 			panic("%s: CPU %d: CHK(vr %d): inconsistent "
7572 				"# of pte's: %d, should be %d",
7573 				s, cpu, vr, rp->rg_nsegmap, n);
7574 	}
7575 	return;
7576 }
7577 
7578 /* Note: not as extensive as pm_check_u. */
7579 void
7580 pm_check_k(char *s, struct pmap *pm)
7581 {
7582 	struct regmap *rp;
7583 	int cpu, vr, vs, n;
7584 
7585 	cpu = cpu_number();
7586 
7587 	if (pm->pm_regmap == NULL)
7588 		panic("%s: CHK(pmap %p): no region mapping", s, pm);
7589 
7590 #if defined(SUN4M) || defined(SUN4D)
7591 	if (CPU_HAS_SRMMU &&
7592 	    (pm->pm_reg_ptps[cpu] == NULL ||
7593 	     pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu])))
7594 	    panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: tblva=%p, tblpa=0x%x",
7595 		  s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]);
7596 
7597 	if (CPU_HAS_SRMMU &&
7598 	    (cpuinfo.ctx_tbl[0] != ((VA2PA((void *)pm->pm_reg_ptps[cpu]) >>
7599 					     SRMMU_PPNPASHIFT) | SRMMU_TEPTD)))
7600 	    panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed "
7601 		  "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], 0);
7602 #endif
7603 	for (vr = NUREG; vr < NUREG+NKREG; vr++) {
7604 		rp = &pm->pm_regmap[vr];
7605 		if (rp->rg_segmap == NULL)
7606 			panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL",
7607 				s, cpu, vr, rp->rg_nsegmap);
7608 		if (rp->rg_nsegmap == 0)
7609 			continue;
7610 #if defined(SUN4M) || defined(SUN4D)
7611 		if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL)
7612 		    panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table",
7613 			  s, cpu, vr, rp->rg_nsegmap);
7614 
7615 		if (CPU_HAS_SRMMU && vr != NUREG /* 1st kseg is per CPU */ &&
7616 		    pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >>
7617 					    SRMMU_PPNPASHIFT) | SRMMU_TEPTD))
7618 		    panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed",
7619 				s, cpu, vr);
7620 #endif
7621 		if (CPU_HAS_SRMMU) {
7622 			n = NSEGRG;
7623 		} else {
7624 			for (n = 0, vs = 0; vs < NSEGRG; vs++) {
7625 				if (rp->rg_segmap[vs].sg_npte)
7626 					n++;
7627 			}
7628 		}
7629 		if (n != rp->rg_nsegmap)
7630 			printf("%s: CPU %d: kernel CHK(vr %d): inconsistent "
7631 				"# of pte's: %d, should be %d\n",
7632 				s, cpu, vr, rp->rg_nsegmap, n);
7633 	}
7634 	return;
7635 }
7636 #endif
7637 
7638 /*
7639  * Return the number of disk blocks that pmap_dumpmmu() will dump.
7640  */
7641 int
7642 pmap_dumpsize(void)
7643 {
7644 	int	sz;
7645 
7646 	sz = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t));
7647 	sz += npmemarr * sizeof(phys_ram_seg_t);
7648 	sz += sizeof(kernel_segmap_store);
7649 
7650 	if (CPU_HAS_SUNMMU)
7651 		/* For each pmeg in the MMU, we'll write NPTESG PTEs. */
7652 		sz += (seginval + 1) * NPTESG * sizeof(int);
7653 
7654 	return btodb(sz + DEV_BSIZE - 1);
7655 }
7656 
7657 /*
7658  * Write the core dump headers and MD data to the dump device.
7659  * We dump the following items:
7660  *
7661  *	kcore_seg_t		 MI header defined in <sys/kcore.h>)
7662  *	cpu_kcore_hdr_t		 MD header defined in <machine/kcore.h>)
7663  *	phys_ram_seg_t[npmemarr] physical memory segments
7664  *	segmap_t[NKREG*NSEGRG]	 the kernel's segment map
7665  *	the MMU pmegs on sun4/sun4c
7666  */
7667 int
7668 pmap_dumpmmu(int (*dump)(dev_t, daddr_t, void *, size_t),
7669 	     daddr_t blkno)
7670 {
7671 	kcore_seg_t	*ksegp;
7672 	cpu_kcore_hdr_t	*kcpup;
7673 	phys_ram_seg_t	memseg;
7674 	int		error = 0;
7675 	int		i, memsegoffset, segmapoffset, pmegoffset;
7676 	int		buffer[dbtob(1) / sizeof(int)];
7677 	int		*bp, *ep;
7678 #if defined(SUN4C) || defined(SUN4)
7679 	int	pmeg;
7680 #endif
7681 
7682 #define EXPEDITE(p,n) do {						\
7683 	int *sp = (int *)(p);						\
7684 	int sz = (n);							\
7685 	while (sz > 0) {						\
7686 		*bp++ = *sp++;						\
7687 		if (bp >= ep) {						\
7688 			error = (*dump)(dumpdev, blkno,			\
7689 					(void *)buffer, dbtob(1));	\
7690 			if (error != 0)					\
7691 				return (error);				\
7692 			++blkno;					\
7693 			bp = buffer;					\
7694 		}							\
7695 		sz -= 4;						\
7696 	}								\
7697 } while (0)
7698 
7699 	setcontext(0);
7700 
7701 	/* Setup bookkeeping pointers */
7702 	bp = buffer;
7703 	ep = &buffer[sizeof(buffer) / sizeof(buffer[0])];
7704 
7705 	/* Fill in MI segment header */
7706 	ksegp = (kcore_seg_t *)bp;
7707 	CORE_SETMAGIC(*ksegp, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
7708 	ksegp->c_size = dbtob(pmap_dumpsize()) - ALIGN(sizeof(kcore_seg_t));
7709 
7710 	/* Fill in MD segment header (interpreted by MD part of libkvm) */
7711 	kcpup = (cpu_kcore_hdr_t *)((int)bp + ALIGN(sizeof(kcore_seg_t)));
7712 	kcpup->cputype = cputyp;
7713 	kcpup->kernbase = KERNBASE;
7714 	kcpup->nmemseg = npmemarr;
7715 	kcpup->memsegoffset = memsegoffset = ALIGN(sizeof(cpu_kcore_hdr_t));
7716 	kcpup->nsegmap = NKREG*NSEGRG;
7717 	kcpup->segmapoffset = segmapoffset =
7718 		memsegoffset + npmemarr * sizeof(phys_ram_seg_t);
7719 
7720 	kcpup->npmeg = (CPU_HAS_SUNMMU) ? seginval + 1 : 0;
7721 	kcpup->pmegoffset = pmegoffset =
7722 		segmapoffset + kcpup->nsegmap * sizeof(struct segmap);
7723 
7724 	/* Note: we have assumed everything fits in buffer[] so far... */
7725 	bp = (int *)((int)kcpup + ALIGN(sizeof(cpu_kcore_hdr_t)));
7726 
7727 #if 0
7728 	/* Align storage for upcoming quad-aligned segment array */
7729 	while (bp != (int *)ALIGN(bp)) {
7730 		int dummy = 0;
7731 		EXPEDITE(&dummy, 4);
7732 	}
7733 #endif
7734 
7735 	for (i = 0; i < npmemarr; i++) {
7736 		memseg.start = pmemarr[i].addr;
7737 		memseg.size = pmemarr[i].len;
7738 		EXPEDITE((void *)&memseg, sizeof(phys_ram_seg_t));
7739 	}
7740 
7741 	EXPEDITE(&kernel_segmap_store, sizeof(kernel_segmap_store));
7742 
7743 	if (CPU_HAS_SRMMU)
7744 		goto out;
7745 
7746 #if defined(SUN4C) || defined(SUN4)
7747 	/*
7748 	 * dump page table entries
7749 	 *
7750 	 * We dump each pmeg in order (by segment number).  Since the MMU
7751 	 * automatically maps the given virtual segment to a pmeg we must
7752 	 * iterate over the segments by incrementing an unused segment slot
7753 	 * in the MMU.  This fixed segment number is used in the virtual
7754 	 * address argument to getpte().
7755 	 */
7756 
7757 	/*
7758 	 * Go through the pmegs and dump each one.
7759 	 */
7760 	for (pmeg = 0; pmeg <= seginval; ++pmeg) {
7761 		int va = 0;
7762 
7763 		setsegmap(va, pmeg);
7764 		i = NPTESG;
7765 		do {
7766 			int pte = getpte4(va);
7767 			EXPEDITE(&pte, sizeof(pte));
7768 			va += NBPG;
7769 		} while (--i > 0);
7770 	}
7771 	setsegmap(0, seginval);
7772 #endif
7773 
7774 out:
7775 	if (bp != buffer)
7776 		error = (*dump)(dumpdev, blkno++, (void *)buffer, dbtob(1));
7777 
7778 	return (error);
7779 }
7780 
7781 /*
7782  * Helper function for debuggers.
7783  */
7784 void
7785 pmap_writetext(unsigned char *dst, int ch)
7786 {
7787 	int s, pte0, pte, ctx;
7788 	vaddr_t va;
7789 
7790 	s = splvm();
7791 	va = (unsigned long)dst & (~PGOFSET);
7792 	cache_flush(dst, 1);
7793 
7794 	ctx = getcontext();
7795 	setcontext(0);
7796 
7797 #if defined(SUN4M) || defined(SUN4D)
7798 	if (CPU_HAS_SRMMU) {
7799 		pte0 = getpte4m(va);
7800 		if ((pte0 & SRMMU_TETYPE) != SRMMU_TEPTE) {
7801 			splx(s);
7802 			return;
7803 		}
7804 		pte = pte0 | PPROT_WRITE;
7805 		setpte4m(va, pte);
7806 		*dst = (unsigned char)ch;
7807 		setpte4m(va, pte0);
7808 
7809 	}
7810 #endif
7811 #if defined(SUN4) || defined(SUN4C)
7812 	if (CPU_ISSUN4C || CPU_ISSUN4) {
7813 		pte0 = getpte4(va);
7814 		if ((pte0 & PG_V) == 0) {
7815 			splx(s);
7816 			return;
7817 		}
7818 		pte = pte0 | PG_W;
7819 		setpte4(va, pte);
7820 		*dst = (unsigned char)ch;
7821 		setpte4(va, pte0);
7822 	}
7823 #endif
7824 	cache_flush(dst, 1);
7825 	setcontext(ctx);
7826 	splx(s);
7827 }
7828 
7829 #ifdef EXTREME_DEBUG
7830 
7831 void debug_pagetables(void);
7832 void print_fe_map(void);
7833 
7834 static void test_region(int, int, int);
7835 
7836 
7837 void
7838 debug_pagetables(void)
7839 {
7840 	struct promvec *promvec = romp;
7841 	int *regtbl;
7842 	int te;
7843 	int i;
7844 
7845 	printf("\nncontext=%d. ", ncontext);
7846 	printf("Context table is at va %p. Level 0 PTP: 0x%x\n",
7847 	       cpuinfo.ctx_tbl, cpuinfo.ctx_tbl[0]);
7848 	printf("Context 0 region table is at va %p, pa 0x%x. Contents:\n",
7849 	       pmap_kernel()->pm_reg_ptps[0], pmap_kernel()->pm_reg_ptps_pa[0]);
7850 
7851 	regtbl = pmap_kernel()->pm_reg_ptps[0];
7852 
7853 	printf("PROM vector is at %p\n", promvec);
7854 	printf("PROM reboot routine is at %p\n", promvec->pv_reboot);
7855 	printf("PROM abort routine is at %p\n", promvec->pv_abort);
7856 	printf("PROM halt routine is at %p\n", promvec->pv_halt);
7857 
7858 	printf("Testing region 0xfe: ");
7859 	test_region(0xfe,0,16*1024*1024);
7860 	printf("Testing region 0xff: ");
7861 	test_region(0xff,0,16*1024*1024);
7862 	printf("Testing kernel region 0x%x: ", VA_VREG(KERNBASE));
7863 	test_region(VA_VREG(KERNBASE), 4096, avail_start);
7864 	cngetc();
7865 
7866 	for (i = 0; i < SRMMU_L1SIZE; i++) {
7867 		te = regtbl[i];
7868 		if ((te & SRMMU_TETYPE) == SRMMU_TEINVALID)
7869 		    continue;
7870 		printf("Region 0x%x: PTE=0x%x <%s> L2PA=0x%x kernL2VA=%p\n",
7871 		       i, te, ((te & SRMMU_TETYPE) == SRMMU_TEPTE ? "pte" :
7872 			       ((te & SRMMU_TETYPE) == SRMMU_TEPTD ? "ptd" :
7873 				((te & SRMMU_TETYPE) == SRMMU_TEINVALID ?
7874 				 "invalid" : "reserved"))),
7875 		       (te & ~0x3) << SRMMU_PPNPASHIFT,
7876 		       pmap_kernel()->pm_regmap[i].rg_seg_ptps);
7877 	}
7878 	printf("Press q to halt...\n");
7879 	if (cngetc()=='q')
7880 	    callrom();
7881 }
7882 
7883 static u_int
7884 VA2PAsw(int ctx, void *addr, int *pte)
7885 {
7886 	int *curtbl;
7887 	int curpte;
7888 
7889 #ifdef EXTREME_EXTREME_DEBUG
7890 	printf("Looking up addr 0x%x in context 0x%x\n",addr,ctx);
7891 #endif
7892 	/* L0 */
7893 	*pte = curpte = cpuinfo.ctx_tbl[ctx];
7894 #ifdef EXTREME_EXTREME_DEBUG
7895 	printf("Got L0 pte 0x%x\n",pte);
7896 #endif
7897 	if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) {
7898 		return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
7899 			((u_int)addr & 0xffffffff));
7900 	}
7901 	if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
7902 		printf("Bad context table entry 0x%x for context 0x%x\n",
7903 		       curpte, ctx);
7904 		return 0;
7905 	}
7906 	/* L1 */
7907 	curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
7908 	*pte = curpte = curtbl[VA_VREG(addr)];
7909 #ifdef EXTREME_EXTREME_DEBUG
7910 	printf("L1 table at 0x%x.\nGot L1 pte 0x%x\n",curtbl,curpte);
7911 #endif
7912 	if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
7913 	    return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
7914 		    ((u_int)addr & 0xffffff));
7915 	if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
7916 		printf("Bad region table entry 0x%x for region 0x%x\n",
7917 		       curpte, VA_VREG(addr));
7918 		return 0;
7919 	}
7920 	/* L2 */
7921 	curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
7922 	*pte = curpte = curtbl[VA_VSEG(addr)];
7923 #ifdef EXTREME_EXTREME_DEBUG
7924 	printf("L2 table at 0x%x.\nGot L2 pte 0x%x\n",curtbl,curpte);
7925 #endif
7926 	if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
7927 	    return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
7928 		    ((u_int)addr & 0x3ffff));
7929 	if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
7930 		printf("Bad segment table entry 0x%x for reg 0x%x, seg 0x%x\n",
7931 		       curpte, VA_VREG(addr), VA_VSEG(addr));
7932 		return 0;
7933 	}
7934 	/* L3 */
7935 	curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
7936 	*pte = curpte = curtbl[VA_VPG(addr)];
7937 #ifdef EXTREME_EXTREME_DEBUG
7938 	printf("L3 table at %p.\nGot L3 pte 0x%x\n", curtbl, curpte);
7939 #endif
7940 	if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
7941 	    return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
7942 		    ((u_int)addr & 0xfff));
7943 	else {
7944 		printf("Bad L3 pte 0x%x for reg 0x%x, seg 0x%x, pg 0x%x\n",
7945 		       curpte, VA_VREG(addr), VA_VSEG(addr), VA_VPG(addr));
7946 		return 0;
7947 	}
7948 	printf("Bizarreness with address %p!\n", addr);
7949 }
7950 
7951 static void
7952 test_region(int reg, int start, int stop)
7953 {
7954 	int i;
7955 	int addr;
7956 	int pte;
7957 	int ptesw;
7958 /*	int cnt=0;
7959 */
7960 
7961 	for (i = start; i < stop; i += NBPG) {
7962 		addr = (reg << RGSHIFT) | i;
7963 		pte = lda(((u_int)(addr)) | ASI_SRMMUFP_LN, ASI_SRMMUFP);
7964 		if (pte) {
7965 /*			printf("Valid address 0x%x\n",addr);
7966 			if (++cnt == 20) {
7967 				cngetc();
7968 				cnt = 0;
7969 			}
7970 */
7971 			if (VA2PA((void *)addr) != VA2PAsw(0, (void *)addr, &ptesw)) {
7972 				printf("Mismatch at address 0x%x.\n", addr);
7973 				if (cngetc() == 'q')
7974 					break;
7975 			}
7976 			if (reg == VA_VREG(KERNBASE))
7977 				/* kernel permissions are different */
7978 				continue;
7979 			if ((pte & SRMMU_PROT_MASK) != (ptesw & SRMMU_PROT_MASK)) {
7980 				printf("Mismatched protections at address "
7981 				       "0x%x; pte=0x%x, ptesw=0x%x\n",
7982 				       addr, pte, ptesw);
7983 				if (cngetc() == 'q')
7984 					break;
7985 			}
7986 		}
7987 	}
7988 	printf("done.\n");
7989 }
7990 
7991 
7992 void
7993 print_fe_map(void)
7994 {
7995 	u_int i, pte;
7996 
7997 	printf("map of region 0xfe:\n");
7998 	for (i = 0xfe000000; i < 0xff000000; i += 4096) {
7999 		if (((pte = getpte4m(i)) & SRMMU_TETYPE) != SRMMU_TEPTE)
8000 		    continue;
8001 		printf("0x%x -> 0x%x%x (pte 0x%x)\n", i, pte >> 28,
8002 		       (pte & ~0xff) << 4, pte);
8003 	}
8004 	printf("done\n");
8005 }
8006 #endif /* EXTREME_DEBUG */
8007 
8008 #ifdef DDB
8009 int pmap_dump(struct pmap *pm);
8010 
8011 int
8012 pmap_dump(struct pmap *pm)
8013 {
8014 	int startvr, endvr, vr, vs, i, n;
8015 	struct regmap *rp;
8016 	struct segmap *sp;
8017 
8018 	if (pm == NULL)
8019 		pm = pmap_kernel();
8020 
8021 	if (pm == pmap_kernel()) {
8022 		startvr = NUREG;
8023 		endvr = 256;
8024 	} else {
8025 		startvr = 0;
8026 		endvr = NUREG;
8027 	}
8028 
8029 	for (vr = startvr; vr < endvr; vr++) {
8030 		rp = &pm->pm_regmap[vr];
8031 		if (rp->rg_nsegmap == 0)
8032 			continue;
8033 		printf("vr %d: %d segments", vr, rp->rg_nsegmap);
8034 		if (rp->rg_segmap == NULL) {
8035 			printf("[no segments]\n");
8036 			continue;
8037 		}
8038 		for (vs = 0; vs < NSEGRG; vs++) {
8039 			sp = &rp->rg_segmap[vs];
8040 			if (sp->sg_npte == 0)
8041 				continue;
8042 			if ((vs & 3) == 0)
8043 				printf("\n   ");
8044 			printf(" %d: n %d w %d p %d,", vs,
8045 				sp->sg_npte, sp->sg_nwired, sp->sg_pmeg);
8046 			if (sp->sg_pte == NULL) {
8047 				printf("[no ptes]");
8048 				continue;
8049 			}
8050 			for (n = 0, i = 0; i < NPTESG; i++) {
8051 				if (CPU_HAS_SUNMMU && sp->sg_pte[i] & PG_WIRED)
8052 					n++;
8053 				if (CPU_HAS_SRMMU && sp->sg_wiremap & (1 << i))
8054 					n++;
8055 			}
8056 			if (n != sp->sg_nwired)
8057 				printf("[wired count %d]", n);
8058 		}
8059 		printf("\n");
8060 	}
8061 
8062 	return (0);
8063 }
8064 #endif /* DDB */
8065