xref: /netbsd/sys/arch/sparc/sparc/pmap.c (revision 1e11a48f)

/*	$NetBSD: pmap.c,v 1.377 2022/04/09 23:38:32 riastradh Exp $ */

/*
 * Copyright (c) 1996
 * 	The President and Fellows of Harvard College. All rights reserved.
 * Copyright (c) 1992, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * This software was developed by the Computer Systems Engineering group
 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
 * contributed to Berkeley.
 *
 * All advertising materials mentioning features or use of this software
 * must display the following acknowledgement:
 *	This product includes software developed by Harvard University.
 *	This product includes software developed by the University of
 *	California, Lawrence Berkeley Laboratory.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by Aaron Brown and
 *	Harvard University.
 *      This product includes software developed by the University of
 *      California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)pmap.c	8.4 (Berkeley) 2/5/94
 *
 */

/*
 * SPARC physical map management code.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.377 2022/04/09 23:38:32 riastradh Exp $");

#include "opt_ddb.h"
#include "opt_kgdb.h"
#include "opt_sparc_arch.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/pool.h>
#include <sys/exec.h>
#include <sys/core.h>
#include <sys/kcore.h>
#include <sys/kernel.h>
#include <sys/atomic.h>

#include <sys/exec_aout.h>		/* for MID_* */

#include <uvm/uvm.h>

#include <machine/autoconf.h>
#include <machine/bsd_openprom.h>
#include <machine/oldmon.h>
#include <machine/cpu.h>
#include <machine/ctlreg.h>
#include <machine/kcore.h>
#include <machine/locore.h>

#include <sparc/sparc/asm.h>
#include <sparc/sparc/cache.h>
#include <sparc/sparc/vaddrs.h>
#include <sparc/sparc/cpuvar.h>

/*
 * The SPARCstation offers us the following challenges:
 *
 *   1. A virtual address cache.  This is, strictly speaking, not
 *	part of the architecture, but the code below assumes one.
 *	This is a write-through cache on the 4c and a write-back cache
 *	on others.
 *
 *   2. (4/4c only) An MMU that acts like a cache.  There is not enough
 *	space in the MMU to map everything all the time.  Instead, we need
 *	to load MMU with the `working set' of translations for each
 *	process. The sun4m does not act like a cache; tables are maintained
 *	in physical memory.
 *
 *   3.	Segmented virtual and physical spaces.  The upper 12 bits of
 *	a virtual address (the virtual segment) index a segment table,
 *	giving a physical segment.  The physical segment selects a
 *	`Page Map Entry Group' (PMEG) and the virtual page number---the
 *	next 5 or 6 bits of the virtual address---select the particular
 *	`Page Map Entry' for the page.  We call the latter a PTE and
 *	call each Page Map Entry Group a pmeg (for want of a better name).
 *	Note that the sun4m has an unsegmented 36-bit physical space.
 *
 *	Since there are no valid bits in the segment table, the only way
 *	to have an invalid segment is to make one full pmeg of invalid PTEs.
 *	We use the last one (since the ROM does as well) (sun4/4c only)
 *
 *   4. Discontiguous physical pages.  The Mach VM expects physical pages
 *	to be in one sequential lump.
 *
 *   5. The MMU is always on: it is not possible to disable it.  This is
 *	mainly a startup hassle.
 */

struct pmap_stats {
	int	ps_unlink_pvfirst;	/* # of pv_unlinks on head */
	int	ps_unlink_pvsearch;	/* # of pv_unlink searches */
	int	ps_changeprots;		/* # of calls to changeprot */
	int	ps_enter_firstpv;	/* pv heads entered */
	int	ps_enter_secondpv;	/* pv nonheads entered */
	int	ps_useless_changewire;	/* useless wiring changes */
	int	ps_npg_prot_all;	/* # of active pages protected */
	int	ps_npg_prot_actual;	/* # pages actually affected */
	int	ps_npmeg_free;		/* # of free pmegs */
	int	ps_npmeg_locked;	/* # of pmegs on locked list */
	int	ps_npmeg_lru;		/* # of pmegs on lru list */
} pmap_stats;

#if defined(SUN4) || defined(SUN4C)
struct evcnt mmu_stolenpmegs_evcnt =
	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","stln pmgs");
EVCNT_ATTACH_STATIC(mmu_stolenpmegs_evcnt);

struct evcnt mmu_pagein_evcnt =
	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","pagein");
EVCNT_ATTACH_STATIC(mmu_pagein_evcnt);
#endif /* SUN4 || SUN4C */

#ifdef DEBUG
#define	PDB_CREATE	0x0001
#define	PDB_DESTROY	0x0002
#define	PDB_REMOVE	0x0004
#define	PDB_CHANGEPROT	0x0008
#define	PDB_ENTER	0x0010
#define	PDB_FOLLOW	0x0020
#define	PDB_INITLOUD	0x0040

#define	PDB_MMU_ALLOC	0x0100
#define	PDB_MMU_STEAL	0x0200
#define	PDB_CTX_ALLOC	0x0400
#define	PDB_CTX_STEAL	0x0800
#define	PDB_MMUREG_ALLOC	0x1000
#define	PDB_MMUREG_STEAL	0x2000
#define	PDB_CACHESTUFF	0x4000
#define	PDB_SWITCHMAP	0x8000
#define	PDB_SANITYCHK	0x10000
int	pmapdebug = 0;
#define DPRINTF(level, fmt, ...) do { \
	if (pmapdebug & (level)) \
		printf("%s:%d: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__); \
} while (0)
#else
#define DPRINTF(level, fmt, ...) /* nothing */
#endif

/*
 * Bounds on managed physical addresses. Used by (MD) users
 * of uvm_pglistalloc() to provide search hints.
 */
paddr_t	vm_first_phys = (paddr_t)-1;
paddr_t	vm_last_phys = 0;
psize_t vm_num_phys;

#define	PMAP_LOCK()	mutex_enter(&pmap_lock)
#define	PMAP_UNLOCK()	mutex_exit(&pmap_lock)

/*
 * Flags in pvlist.pv_flags.  Note that PV_MOD must be 1 and PV_REF must be 2
 * since they must line up with the bits in the hardware PTEs (see pte.h).
 * SUN4M bits are at a slightly different location in the PTE.
 *
 * Note: the REF, MOD and ANC flag bits occur only in the head of a pvlist.
 * The NC bit is meaningful in each individual pv entry and reflects the
 * requested non-cacheability at the time the entry was made through
 * pv_link() or when subsequently altered by kvm_uncache() (but the latter
 * does not happen in kernels as of the time of this writing (March 2001)).
 */
#define PV_MOD		1	/* page modified */
#define PV_REF		2	/* page referenced */
#define PV_NC		4	/* page cannot be cached */
#define PV_REF4M	1	/* page referenced (SRMMU) */
#define PV_MOD4M	2	/* page modified (SRMMU) */
#define PV_ANC		0x10	/* page has incongruent aliases */

static struct pool pv_pool;

/*
 * pvhead(pte): find a VM page given a PTE entry.
 */
#if defined(SUN4) || defined(SUN4C)
static struct vm_page *
pvhead4_4c(u_int pte)
{
	paddr_t pa = (pte & PG_PFNUM) << PGSHIFT;

	return (PHYS_TO_VM_PAGE(pa));
}
#endif

#if defined(SUN4M) || defined(SUN4D)
static struct vm_page *
pvhead4m(u_int pte)
{
	paddr_t pa = (pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT;

	return (PHYS_TO_VM_PAGE(pa));
}
#endif

/*
 * Each virtual segment within each pmap is either valid or invalid.
 * It is valid if pm_npte[VA_VSEG(va)] is not 0.  This does not mean
 * it is in the MMU, however; that is true iff pm_segmap[VA_VSEG(va)]
 * does not point to the invalid PMEG.
 *
 * In the older SPARC architectures (sun4/sun4c), page tables are cached in
 * the MMU. The following discussion applies to these architectures:
 *
 * If a virtual segment is valid and loaded, the correct PTEs appear
 * in the MMU only.  If it is valid and unloaded, the correct PTEs appear
 * in the pm_pte[VA_VSEG(va)] only.  However, some effort is made to keep
 * the software copies consistent enough with the MMU so that libkvm can
 * do user address translations.  In particular, pv_changepte() and
 * pmap_enu() maintain consistency, while less critical changes are
 * not maintained.  pm_pte[VA_VSEG(va)] always points to space for those
 * PTEs.
 *
 * Each PMEG in the MMU is either free or contains PTEs corresponding to
 * some pmap and virtual segment.  If it contains some PTEs, it also contains
 * reference and modify bits that belong in the pv_table.  If we need
 * to steal a PMEG from some process (if we need one and none are free)
 * we must copy the ref and mod bits, and update pm_segmap in the other
 * pmap to show that its virtual segment is no longer in the MMU.
 *
 * There are 128 PMEGs in a small Sun-4, of which only a few dozen are
 * tied down permanently, leaving `about' 100 to be spread among
 * running processes.  These are managed as an LRU cache.  Before
 * calling the VM paging code for a user page fault, the fault handler
 * calls mmu_load(pmap, va) to try to get a set of PTEs put into the
 * MMU.  mmu_load will check the validity of the segment and tell whether
 * it did something.
 *
 * Since I hate the name PMEG I call this data structure an `mmu entry'.
 * Each mmuentry is on exactly one of three `usage' lists: free, LRU,
 * or locked.  The locked list is only used for kernel mappings that need
 * to be wired down.
 *
 *
 * In the sun4m architecture using the SPARC Reference MMU (SRMMU), three
 * levels of page tables are maintained in physical memory. We use the same
 * structures as with the 3-level old-style MMU (pm_regmap, pm_segmap,
 * rg_segmap, sg_pte, etc) to maintain kernel-edible page tables; we also
 * build a parallel set of physical tables that can be used by the MMU.
 * (XXX: This seems redundant, but is it necessary for the unified kernel?)
 *
 * If a virtual segment is valid, its entries will be in both parallel lists.
 * If it is not valid, then its entry in the kernel tables will be zero, and
 * its entry in the MMU tables will either be nonexistent or zero as well.
 *
 * The Reference MMU generally uses a Translation Look-aside Buffer (TLB)
 * to cache the result of recently executed page table walks. When
 * manipulating page tables, we need to ensure consistency of the
 * in-memory and TLB copies of the page table entries. This is handled
 * by flushing (and invalidating) a TLB entry when appropriate before
 * altering an in-memory page table entry.
 */
struct mmuentry {
	struct {
	    struct mmuentry *prev, *next;
	}			me_list;	/* usage list link */
	TAILQ_ENTRY(mmuentry)	me_pmchain;	/* pmap owner link */
	struct	pmap *me_pmap;		/* pmap, if in use */
	u_short	me_vreg;		/* associated virtual region/segment */
	u_short	me_vseg;		/* associated virtual region/segment */
	u_short	me_cookie;		/* hardware SMEG/PMEG number */
#ifdef DIAGNOSTIC
	int *me_statp;/*XXX*/
#endif
};
struct mmuentry *mmusegments;	/* allocated in pmap_bootstrap */
struct mmuentry *mmuregions;	/* allocated in pmap_bootstrap */

#if defined(SUN4) || defined(SUN4C)
struct mmuentry segm_freelist, segm_lru, segm_locked;
#if defined(SUN4_MMU3L)
struct mmuentry region_freelist, region_lru, region_locked;
#endif
/*
 * We use a double linked list looping through its static head (which
 * always remains on the list), so we can remove any other member from
 * a list without knowing which list it is on.
 */
static void inline
mmuq_remove(struct mmuentry *e)
{
	e->me_list.next->me_list.prev = e->me_list.prev;
	e->me_list.prev->me_list.next = e->me_list.next;
}

static void inline
mmuq_init(struct mmuentry *e)
{
	memset(e, 0, sizeof(*e));
	e->me_list.next = e;
	e->me_list.prev = e;
}

static inline struct mmuentry *
mmuq_first(struct mmuentry *head)
{
	KASSERT(head->me_list.next != head);
	return head->me_list.next;
}

static inline bool
mmuq_empty(struct mmuentry *head)
{
	return head->me_list.next == head;
}

static inline void
mmuq_insert_tail(struct mmuentry *head, struct mmuentry *e)
{
	e->me_list.prev = head->me_list.prev;
	e->me_list.next = head;
	head->me_list.prev->me_list.next = e;
	head->me_list.prev = e;
}
#endif


int	seginval;		/* [4/4c] the invalid segment number */
int	reginval;		/* [4/3mmu] the invalid region number */

static kmutex_t pmap_lock;
static kmutex_t demap_lock;
static bool	lock_available = false;	/* demap_lock has been initialized */

/*
 * (sun4/4c)
 * A context is simply a small number that dictates which set of 4096
 * segment map entries the MMU uses.  The Sun 4c has eight (SS1,IPC) or
 * sixteen (SS2,IPX) such sets. These are alloted in an `almost MRU' fashion.
 * (sun4m)
 * A context is simply a small number that indexes the context table, the
 * root-level page table mapping 4G areas. Each entry in this table points
 * to a 1st-level region table. A SPARC reference MMU will usually use 16
 * such contexts, but some offer as many as 64k contexts; the theoretical
 * maximum is 2^32 - 1, but this would create overlarge context tables.
 *
 * Each context is either free or attached to a pmap.
 *
 * Since the virtual address cache is tagged by context, when we steal
 * a context we have to flush (that part of) the cache.
 */
union ctxinfo {
	union	ctxinfo *c_nextfree;	/* free list (if free) */
	struct	pmap *c_pmap;		/* pmap (if busy) */
};

static kmutex_t	ctx_lock;		/* lock for below, and {,de}activate */
union	ctxinfo *ctxinfo;		/* allocated at in pmap_bootstrap */
union	ctxinfo *ctx_freelist;		/* context free list */
int	ctx_kick;			/* allocation rover when none free */
int	ctx_kickdir;			/* ctx_kick roves both directions */
int	ncontext;			/* sizeof ctx_freelist */

static void	ctx_alloc(struct pmap *);
static void	ctx_free(struct pmap *);

/*void *	vdumppages;	-* 32KB worth of reserved dump pages */

smeg_t		tregion;	/* [4/3mmu] Region for temporary mappings */

static struct pmap	kernel_pmap_store;	/* the kernel's pmap */
struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; /* pmap_kernel() */
struct regmap	kernel_regmap_store[NKREG];	/* the kernel's regmap */
struct segmap	kernel_segmap_store[NKREG*NSEGRG];/* the kernel's segmaps */

#if defined(SUN4M) || defined(SUN4D)
u_int 	*kernel_regtable_store;		/* 1k of storage to map the kernel */
u_int	*kernel_segtable_store;		/* 2k of storage to map the kernel */
u_int	*kernel_pagtable_store;		/* 128k of storage to map the kernel */

/*
 * Memory pools and back-end supplier for SRMMU page tables.
 * Share a pool between the level 2 and level 3 page tables,
 * since these are equal in size.
 */
static struct pool L1_pool;
static struct pool L23_pool;

static void *pgt_page_alloc(struct pool *, int);
static void  pgt_page_free(struct pool *, void *);

static struct pool_allocator pgt_page_allocator = {
	pgt_page_alloc, pgt_page_free, 0,
};

#endif /* SUN4M || SUN4D */

#if defined(SUN4) || defined(SUN4C)
/*
 * Memory pool for user and kernel PTE tables.
 */
static struct pool pte_pool;
#endif

struct	memarr *pmemarr;	/* physical memory regions */
int	npmemarr;		/* number of entries in pmemarr */

static paddr_t	avail_start;	/* first available physical page, other
				   than the `etext gap' defined below */
static vaddr_t	etext_gap_start;/* start of gap between text & data */
static vaddr_t	etext_gap_end;	/* end of gap between text & data */
static vaddr_t	virtual_avail;	/* first free kernel virtual address */
static vaddr_t	virtual_end;	/* last free kernel virtual address */

static void pmap_page_upload(void);

int mmu_has_hole;

vaddr_t prom_vstart;	/* For /dev/kmem */
vaddr_t prom_vend;

/*
 * Memory pool for pmap structures.
 */
static struct pool_cache pmap_cache;
static int	pmap_pmap_pool_ctor(void *, void *, int);
static void	pmap_pmap_pool_dtor(void *, void *);
static struct pool segmap_pool;

#if defined(SUN4)
/*
 * [sun4]: segfixmask: on some systems (4/110) "getsegmap()" returns a
 * partly invalid value. getsegmap returns a 16 bit value on the sun4,
 * but only the first 8 or so bits are valid (the rest are *supposed* to
 * be zero. On the 4/110 the bits that are supposed to be zero are
 * all one instead. e.g. KERNBASE is usually mapped by pmeg number zero.
 * On a 4/300 getsegmap(KERNBASE) == 0x0000, but
 * on a 4/100 getsegmap(KERNBASE) == 0xff00
 *
 * This confuses mmu_reservemon() and causes it to not reserve the PROM's
 * pmegs. Then the PROM's pmegs get used during autoconfig and everything
 * falls apart!  (not very fun to debug, BTW.)
 *
 * solution: mask the invalid bits in the getsetmap macro.
 */

static u_int segfixmask = 0xffffffff; /* all bits valid to start */
#else
#define segfixmask 0xffffffff	/* It's in getsegmap's scope */
#endif

/*
 * pseudo-functions for mnemonic value
 */
#define	getsegmap(va)		(CPU_ISSUN4C \
					? lduba(va, ASI_SEGMAP) \
					: (lduha(va, ASI_SEGMAP) & segfixmask))
#define	setsegmap(va, pmeg)	(CPU_ISSUN4C \
					? stba(va, ASI_SEGMAP, pmeg) \
					: stha(va, ASI_SEGMAP, pmeg))

/* 3-level sun4 MMU only: */
#define	getregmap(va)		((unsigned)lduha((va)+2, ASI_REGMAP) >> 8)
#define	setregmap(va, smeg)	stha((va)+2, ASI_REGMAP, (smeg << 8))


#if defined(SUN4M) || defined(SUN4D)
#if 0
#if VM_PROT_READ != 1 || VM_PROT_WRITE != 2 || VM_PROT_EXECUTE != 4
#error fix protection code translation table
#endif
#endif
/*
 * Translation table for kernel vs. PTE protection bits.
 */
const u_int protection_codes[2][8] = {
	/* kernel */
	{
	PPROT_N_RX,	/* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_NONE */
	PPROT_N_RX,	/* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_READ */
	PPROT_N_RWX,	/* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_NONE */
	PPROT_N_RWX,	/* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_READ */
	PPROT_N_RX,	/* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_NONE */
	PPROT_N_RX,	/* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_READ */
	PPROT_N_RWX,	/* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */
	PPROT_N_RWX,	/* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */
	},

	/* user */
	{
	PPROT_N_RX,	/* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_NONE */
	PPROT_R_R,	/* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_READ */
	PPROT_RW_RW,	/* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_NONE */
	PPROT_RW_RW,	/* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_READ */
	PPROT_X_X,	/* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_NONE */
	PPROT_RX_RX,	/* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_READ */
	PPROT_RWX_RWX,	/* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */
	PPROT_RWX_RWX,	/* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */
	}
};
#define pte_kprot4m(prot) (protection_codes[0][(prot)])
#define pte_uprot4m(prot) (protection_codes[1][(prot)])
#define pte_prot4m(pm, prot) \
	(protection_codes[(pm) == pmap_kernel() ? 0 : 1][(prot)])

void		setpte4m(vaddr_t va, int pte);
void		setpgt4m(int *ptep, int pte);
void		setpgt4m_va(vaddr_t, int *, int, int, int, u_int);
int		updatepte4m(vaddr_t, int *, int, int, int, u_int);
#endif /* SUN4M || SUN4D */

#if defined(MULTIPROCESSOR)
#define PMAP_SET_CPUSET(pmap, cpi)	\
	(pmap->pm_cpuset |= (1 << (cpi)->ci_cpuid))
#define PMAP_CLR_CPUSET(pmap, cpi)	\
	(pmap->pm_cpuset &= ~(1 << (cpi)->ci_cpuid))
#define PMAP_CPUSET(pmap)		(pmap->pm_cpuset)
#else
#define PMAP_SET_CPUSET(pmap, cpi)	/* nothing */
#define PMAP_CLR_CPUSET(pmap, cpi)	/* nothing */
#define PMAP_CPUSET(pmap)		1	/* XXX: 1 or 0? */
#endif /* MULTIPROCESSOR */


/* Function pointer messiness for supporting multiple sparc architectures
 * within a single kernel: notice that there are two versions of many of the
 * functions within this file/module, one for the sun4/sun4c and the other
 * for the sun4m. For performance reasons (since things like pte bits don't
 * map nicely between the two architectures), there are separate functions
 * rather than unified functions which test the cputyp variable. If only
 * one architecture is being used, then the non-suffixed function calls
 * are macro-translated into the appropriate xxx4_4c or xxx4m call. If
 * multiple architectures are defined, the calls translate to (*xxx_p),
 * i.e. they indirect through function pointers initialized as appropriate
 * to the run-time architecture in pmap_bootstrap. See also pmap.h.
 */

#if defined(SUN4M) || defined(SUN4D)
static void mmu_setup4m_L1(int, struct pmap *);
static void mmu_setup4m_L2(int, struct regmap *);
static void  mmu_setup4m_L3(int, struct segmap *);
/*static*/ void	mmu_reservemon4m(struct pmap *);

/*static*/ void pmap_changeprot4m(pmap_t, vaddr_t, vm_prot_t, int);
/*static*/ void pmap_rmk4m(struct pmap *, vaddr_t, vaddr_t, int, int);
/*static*/ void pmap_rmu4m(struct pmap *, vaddr_t, vaddr_t, int, int);
/*static*/ int  pmap_enk4m(struct pmap *, vaddr_t, vm_prot_t,
				int, struct vm_page *, int);
/*static*/ int  pmap_enu4m(struct pmap *, vaddr_t, vm_prot_t,
				int, struct vm_page *, int);
/*static*/ void pv_changepte4m(struct vm_page *, int, int);
/*static*/ int  pv_syncflags4m(struct vm_page *);
/*static*/ int  pv_link4m(struct vm_page *, struct pmap *, vaddr_t, u_int *);
/*static*/ void pv_unlink4m(struct vm_page *, struct pmap *, vaddr_t);
#endif

#if defined(SUN4) || defined(SUN4C)
/*static*/ void	mmu_reservemon4_4c(int *, int *);
/*static*/ void pmap_changeprot4_4c(pmap_t, vaddr_t, vm_prot_t, int);
/*static*/ void pmap_rmk4_4c(struct pmap *, vaddr_t, vaddr_t, int, int);
/*static*/ void pmap_rmu4_4c(struct pmap *, vaddr_t, vaddr_t, int, int);
/*static*/ int  pmap_enk4_4c(struct pmap *, vaddr_t, vm_prot_t,
				  int, struct vm_page *, int);
/*static*/ int  pmap_enu4_4c(struct pmap *, vaddr_t, vm_prot_t,
				  int, struct vm_page *, int);
/*static*/ void pv_changepte4_4c(struct vm_page *, int, int);
/*static*/ int  pv_syncflags4_4c(struct vm_page *);
/*static*/ int  pv_link4_4c(struct vm_page *, struct pmap *, vaddr_t, u_int *);
/*static*/ void pv_unlink4_4c(struct vm_page *, struct pmap *, vaddr_t);
#endif

#if !(defined(SUN4M) || defined(SUN4D)) && (defined(SUN4) || defined(SUN4C))
#define		pmap_rmk	pmap_rmk4_4c
#define		pmap_rmu	pmap_rmu4_4c

#elif (defined(SUN4M) || defined(SUN4D)) && !(defined(SUN4) || defined(SUN4C))
#define		pmap_rmk	pmap_rmk4m
#define		pmap_rmu	pmap_rmu4m

#else  /* must use function pointers */

/* function pointer declarations */
/* from pmap.h: */
bool		(*pmap_clear_modify_p)(struct vm_page *);
bool		(*pmap_clear_reference_p)(struct vm_page *);
int		(*pmap_enter_p)(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int);
bool		(*pmap_extract_p)(pmap_t, vaddr_t, paddr_t *);
bool		(*pmap_is_modified_p)(struct vm_page *);
bool		(*pmap_is_referenced_p)(struct vm_page *);
void		(*pmap_kenter_pa_p)(vaddr_t, paddr_t, vm_prot_t, u_int);
void		(*pmap_kremove_p)(vaddr_t, vsize_t);
void		(*pmap_kprotect_p)(vaddr_t, vsize_t, vm_prot_t);
void		(*pmap_page_protect_p)(struct vm_page *, vm_prot_t);
void		(*pmap_protect_p)(pmap_t, vaddr_t, vaddr_t, vm_prot_t);
/* local: */
void 		(*pmap_rmk_p)(struct pmap *, vaddr_t, vaddr_t, int, int);
void 		(*pmap_rmu_p)(struct pmap *, vaddr_t, vaddr_t, int, int);

#define		pmap_rmk	(*pmap_rmk_p)
#define		pmap_rmu	(*pmap_rmu_p)

#endif

/* --------------------------------------------------------------*/

/*
 * Next we have some sun4m/4d-specific routines which have no 4/4c
 * counterparts, or which are 4/4c macros.
 */

#if defined(SUN4M) || defined(SUN4D)
/*
 * SP versions of the tlb flush operations.
 *
 * Turn off traps to prevent register window overflows from writing
 * user windows to the wrong stack.  Cf. tlb_flush_page_real() &c.
 */
static void
sp_tlb_flush(int va, int ctx, int lvl)
{
	int opsr, octx;

	va &= ~0xfff;
	va |= lvl;

	/*
	 * Turn off traps.
	 *
	 * Like setpsr((opsr = getpsr()) & ~PSR_ET); but we can shave
	 * off one instruction b/c we never disable traps recursively,
	 * so we can use the xor done by wrpsr itself to clear the
	 * bit.
	 *
	 * XXX: Add to asm.h?  We can use this in cache.c too.
	 */
	opsr = getpsr();	/* KDASSERT(opsr & PSR_ET); */
	__asm volatile ("wr %0, %1, %%psr"
			:: "r"(opsr), "n"(PSR_ET) : "memory");
	__asm volatile ("nop; nop; nop");

	octx = getcontext4m();	/* save context */

	/* Do the TLB flush in "ctx" */
	setcontext4m(ctx);
	__asm volatile ("sta %%g0, [%0]%1" :: "r"(va), "n"(ASI_SRMMUFP));

	setcontext4m(octx);	/* restore context */
	setpsr(opsr);		/* turn traps on again */
}

static inline void
sp_tlb_flush_all(void)
{

	sta(ASI_SRMMUFP_LN, ASI_SRMMUFP, 0);
}

#if defined(MULTIPROCESSOR)
/*
 * The SMP versions of the tlb flush routines.  We only need to
 * do a cross call for these on sun4m (Mbus) systems. sun4d systems
 * have an Xbus which broadcasts TLB demaps in hardware.
 */

static inline void	smp_tlb_flush_page (int va, int ctx, u_int cpuset);
static inline void	smp_tlb_flush_segment (int va, int ctx, u_int cpuset);
static inline void	smp_tlb_flush_region (int va, int ctx, u_int cpuset);
static inline void	smp_tlb_flush_context (int ctx, u_int cpuset);
static inline void	smp_tlb_flush_all (void);

static inline void
smp_tlb_flush_page(int va, int ctx, u_int cpuset)
{

	if (CPU_ISSUN4D) {
		sp_tlb_flush(va, ctx, ASI_SRMMUFP_L3);
	} else
		FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L3, cpuset);
}

static inline void
smp_tlb_flush_segment(int va, int ctx, u_int cpuset)
{

	if (CPU_ISSUN4D) {
		sp_tlb_flush(va, ctx, ASI_SRMMUFP_L2);
	} else
		FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L2, cpuset);
}

static inline void
smp_tlb_flush_region(int va, int ctx, u_int cpuset)
{

	if (CPU_ISSUN4D) {
		sp_tlb_flush(va, ctx, ASI_SRMMUFP_L1);
	} else
		FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L1, cpuset);
}

static inline void
smp_tlb_flush_context(int ctx, u_int cpuset)
{

	if (CPU_ISSUN4D) {
		sp_tlb_flush(0, ctx, ASI_SRMMUFP_L0);
	} else
		FXCALL3(sp_tlb_flush, ft_tlb_flush, 0, ctx, ASI_SRMMUFP_L0, cpuset);
}

static inline void
smp_tlb_flush_all(void)
{

	if (CPU_ISSUN4D) {
		sp_tlb_flush_all();
	} else
		XCALL0(sp_tlb_flush_all, CPUSET_ALL);
}
#endif /* MULTIPROCESSOR */

#if defined(MULTIPROCESSOR)
#define tlb_flush_page(va,ctx,s)	smp_tlb_flush_page(va,ctx,s)
#define tlb_flush_segment(va,ctx,s)	smp_tlb_flush_segment(va,ctx,s)
#define tlb_flush_region(va,ctx,s)	smp_tlb_flush_region(va,ctx,s)
#define tlb_flush_context(ctx,s)	smp_tlb_flush_context(ctx,s)
#define tlb_flush_all()			smp_tlb_flush_all()
#else
#define tlb_flush_page(va,ctx,s)	sp_tlb_flush(va,ctx,ASI_SRMMUFP_L3)
#define tlb_flush_segment(va,ctx,s)	sp_tlb_flush(va,ctx,ASI_SRMMUFP_L2)
#define tlb_flush_region(va,ctx,s)	sp_tlb_flush(va,ctx,ASI_SRMMUFP_L1)
#define tlb_flush_context(ctx,s)	sp_tlb_flush(0,ctx,ASI_SRMMUFP_L0)
#define tlb_flush_all()			sp_tlb_flush_all()
#endif /* MULTIPROCESSOR */

static u_int	VA2PA(void *);
static u_long	srmmu_bypass_read(u_long);

/*
 * VA2PA(addr) -- converts a virtual address to a physical address using
 * the MMU's currently-installed page tables. As a side effect, the address
 * translation used may cause the associated pte to be encached. The correct
 * context for VA must be set before this is called.
 *
 * This routine should work with any level of mapping, as it is used
 * during bootup to interact with the ROM's initial L1 mapping of the kernel.
 */
static u_int
VA2PA(void *addr)
{
	u_int pte;

	/*
	 * We'll use that handy SRMMU flush/probe.
	 * Try each level in turn until we find a valid pte. Otherwise panic.
	 */

	pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L3, ASI_SRMMUFP);
	/* Unlock fault status; required on Hypersparc modules */
	(void)lda(SRMMU_SFSR, ASI_SRMMU);
	if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
	    return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
		    ((u_int)addr & 0xfff));

	/* A `TLB Flush Entire' is required before any L0, L1 or L2 probe */
	tlb_flush_all_real();

	pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L2, ASI_SRMMUFP);
	if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
	    return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
		    ((u_int)addr & 0x3ffff));
	pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L1, ASI_SRMMUFP);
	if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
	    return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
		    ((u_int)addr & 0xffffff));
	pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L0, ASI_SRMMUFP);
	if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
	    return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
		    ((u_int)addr & 0xffffffff));

#ifdef DIAGNOSTIC
	panic("VA2PA: Asked to translate unmapped VA %p", addr);
#else
	return (0);
#endif
}

/*
 * Atomically update a PTE entry, coping with hardware updating the
 * PTE at the same time we are.  This is the procedure that is
 * recommended in the SuperSPARC user's manual.
 */
int
updatepte4m(vaddr_t va, int *pte, int bic, int bis, int ctx, u_int cpuset)
{
	int oldval, swapval;
	volatile int *vpte = (volatile int *)pte;
	bool can_lock = lock_available;

	/*
	 * Can only be one of these happening in the system
	 * at any one time.
	 */
	if (__predict_true(can_lock))
		mutex_spin_enter(&demap_lock);

	/*
	 * The idea is to loop swapping zero into the pte, flushing
	 * it, and repeating until it stays zero.  At this point,
	 * there should be no more hardware accesses to this PTE
	 * so we can modify it without losing any mod/ref info.
	 */
	oldval = 0;
	do {
		swapval = 0;
		swap(vpte, swapval);
		tlb_flush_page(va, ctx, cpuset);
		oldval |= swapval;
	} while (__predict_false(*vpte != 0));

	swapval = (oldval & ~bic) | bis;
	swap(vpte, swapval);

	if (__predict_true(can_lock))
		mutex_spin_exit(&demap_lock);

	return (oldval);
}

inline void
setpgt4m(int *ptep, int pte)
{

	kpreempt_disable();
	swap(ptep, pte);
	kpreempt_enable();
}

inline void
setpgt4m_va(vaddr_t va, int *ptep, int pte, int pageflush, int ctx,
	    u_int cpuset)
{

#if defined(MULTIPROCESSOR)
	updatepte4m(va, ptep, 0xffffffff, pte, pageflush ? ctx : 0, cpuset);
#else
	kpreempt_disable();
	if (__predict_true(pageflush))
		tlb_flush_page(va, ctx, 0);
	setpgt4m(ptep, pte);
	kpreempt_enable();
#endif /* MULTIPROCESSOR */
}

/* Set the page table entry for va to pte. */
void
setpte4m(vaddr_t va, int pte)
{
	struct pmap *pm;
	struct regmap *rp;
	struct segmap *sp;

#ifdef DEBUG
	if (getcontext4m() != 0)
		panic("setpte4m: user context");
#endif

	pm = pmap_kernel();
	rp = &pm->pm_regmap[VA_VREG(va)];
	sp = &rp->rg_segmap[VA_VSEG(va)];

	tlb_flush_page(va, 0, CPUSET_ALL);
	setpgt4m(sp->sg_pte + VA_SUN4M_VPG(va), pte);
}

/*
 * Page table pool back-end.
 */
void *
pgt_page_alloc(struct pool *pp, int flags)
{
	int cacheit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0;
	struct vm_page *pg;
	vaddr_t va;
	paddr_t pa;

	/* Allocate a page of physical memory */
	while ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL &&
	       (flags & PR_WAITOK) != 0) {
		uvm_wait("pgtpg");
	}
	if (pg == NULL) {
		KASSERT((flags & PR_WAITOK) == 0);
		return NULL;
	}

	/* Allocate virtual memory */
	va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY |
		((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK));
	if (va == 0) {
		KASSERT((flags & PR_WAITOK) == 0);
		uvm_pagefree(pg);
		return (NULL);
	}

	/*
	 * On systems with a physical data cache we need to flush this page
	 * from the cache if the pagetables cannot be cached.
	 * On systems with a virtually indexed data cache, we only need
	 * to map it non-cacheable, since the page is not currently mapped.
	 */
	pa = VM_PAGE_TO_PHYS(pg);
	if (cacheit == 0)
		pcache_flush_page(pa, 1);

	/* Map the page */
	pmap_kenter_pa(va, pa | (cacheit ? 0 : PMAP_NC),
	    VM_PROT_READ | VM_PROT_WRITE, 0);
	pmap_update(pmap_kernel());

	return ((void *)va);
}

void
pgt_page_free(struct pool *pp, void *v)
{
	vaddr_t va;
	paddr_t pa;
	bool rv __diagused;

	va = (vaddr_t)v;
	rv = pmap_extract(pmap_kernel(), va, &pa);
	KASSERT(rv);
	uvm_pagefree(PHYS_TO_VM_PAGE(pa));
	pmap_kremove(va, PAGE_SIZE);
	uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY);
}
#endif /* SUN4M || SUN4D */

/*----------------------------------------------------------------*/

/*
 * The following three macros are to be used in sun4/sun4c code only.
 */
#if defined(SUN4_MMU3L)
#define CTX_USABLE(pm,rp) (					\
		((pm)->pm_ctx != NULL &&			\
		 (!HASSUN4_MMU3L || (rp)->rg_smeg != reginval))	\
)
#else
#define CTX_USABLE(pm,rp)	((pm)->pm_ctx != NULL )
#endif

#define GAP_WIDEN(pm,vr) do if (CPU_HAS_SUNMMU) {		\
	if (vr + 1 == pm->pm_gap_start)				\
		pm->pm_gap_start = vr;				\
	if (vr == pm->pm_gap_end)				\
		pm->pm_gap_end = vr + 1;			\
} while (0)

#define GAP_SHRINK(pm,vr) do if (CPU_HAS_SUNMMU) {			\
	int x;								\
	x = pm->pm_gap_start + (pm->pm_gap_end - pm->pm_gap_start) / 2;	\
	if (vr > x) {							\
		if (vr < pm->pm_gap_end)				\
			pm->pm_gap_end = vr;				\
	} else {							\
		if (vr >= pm->pm_gap_start && x != pm->pm_gap_start)	\
			pm->pm_gap_start = vr + 1;			\
	}								\
} while (0)


static void get_phys_mem(void **);
#if 0 /* not used */
void	kvm_iocache(char *, int);
#endif

#ifdef DEBUG
void	pm_check(char *, struct pmap *);
void	pm_check_k(char *, struct pmap *);
void	pm_check_u(char *, struct pmap *);
#endif

/*
 * During the PMAP bootstrap, we can use a simple translation to map a
 * kernel virtual address to a psysical memory address (this is arranged
 * in locore).  Usually, KERNBASE maps to physical address 0. This is always
 * the case on sun4 and sun4c machines. On sun4m machines -- if no memory is
 * installed in the bank corresponding to physical address 0 -- the PROM may
 * elect to load us at some other address, presumably at the start of
 * the first memory bank that is available. We set the up the variable
 * `va2pa_offset' to hold the physical address corresponding to KERNBASE.
 */

static u_long va2pa_offset;
#define PMAP_BOOTSTRAP_VA2PA(v) ((paddr_t)((u_long)(v) - va2pa_offset))
#define PMAP_BOOTSTRAP_PA2VA(p) ((vaddr_t)((u_long)(p) + va2pa_offset))

/*
 * Grab physical memory list.
 * While here, compute `physmem'.
 */
void
get_phys_mem(void **top)
{
	struct memarr *mp;
	char *p;
	int i;

	/* Load the memory descriptor array at the current kernel top */
	p = (void *)ALIGN(*top);
	pmemarr = (struct memarr *)p;
	npmemarr = prom_makememarr(pmemarr, 1000, MEMARR_AVAILPHYS);

	/* Update kernel top */
	p += npmemarr * sizeof(struct memarr);
	*top = p;

	for (physmem = 0, mp = pmemarr, i = npmemarr; --i >= 0; mp++)
		physmem += btoc(mp->len);
}


/*
 * Support functions for vm_page_bootstrap().
 */

/*
 * How much virtual space does this kernel have?
 * (After mapping kernel text, data, etc.)
 */
void
pmap_virtual_space(vaddr_t *v_start, vaddr_t *v_end)
{

        *v_start = virtual_avail;
        *v_end   = virtual_end;
}

#ifdef PMAP_GROWKERNEL
vaddr_t
pmap_growkernel(vaddr_t eva)
{
	struct regmap *rp;
	struct segmap *sp;
	int vr, evr, M, N, i;
	struct vm_page *pg;
	vaddr_t va;

	if (eva <= virtual_end)
		return (virtual_end);

	/* For now, only implemented for sun4/sun4c */
	KASSERT(CPU_HAS_SUNMMU);

	/*
	 * Map in the next region(s)
	 */

	/* Get current end-of-kernel */
	vr = virtual_end >> RGSHIFT;
	evr = (eva + NBPRG - 1) >> RGSHIFT;
	eva = evr << RGSHIFT;

	if (eva > VM_MAX_KERNEL_ADDRESS)
		panic("growkernel: grown too large: %lx", eva);

	/*
	 * Divide a region in N blocks of M segments, where each segment
	 * block can have its PTEs mapped by one page.
	 * N should come out to 1 for 8K pages and to 4 for 4K pages.
	 */
	M = NBPG / (NPTESG * sizeof(int));
	N = (NBPRG/NBPSG) / M;

	while (vr < evr) {
		rp = &pmap_kernel()->pm_regmap[vr];
		for (i = 0; i < N; i++) {
			sp = &rp->rg_segmap[i * M];
			va = (vaddr_t)sp->sg_pte;
			pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
			if (pg == NULL)
				panic("growkernel: out of memory");
			pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
					VM_PROT_READ | VM_PROT_WRITE, 0);
		}
	}

	virtual_end = eva;
	return (eva);
}
#endif

/*
 * Helper routine that hands off available physical pages to the VM system.
 */
static void
pmap_page_upload(void)
{
	int	n;
	paddr_t	pstart, pend;

	/* First, the `etext gap' */
	pstart = PMAP_BOOTSTRAP_VA2PA(etext_gap_start);
	pend = PMAP_BOOTSTRAP_VA2PA(etext_gap_end);

#ifdef DIAGNOSTIC
	if (avail_start <= pstart)
		panic("pmap_page_upload: etext gap overlap: %lx < %lx",
			(u_long)avail_start, (u_long)pstart);
#endif
	if (etext_gap_start < etext_gap_end) {
		vm_first_phys = pstart;
		uvm_page_physload(
			atop(pstart),
			atop(pend),
			atop(pstart),
			atop(pend), VM_FREELIST_DEFAULT);
	}

	for (n = 0; n < npmemarr; n++) {

		pstart = pmemarr[n].addr;
		pend = pstart + pmemarr[n].len;

		/* Update vm_{first_last}_phys */
		if (vm_first_phys > pstart)
			vm_first_phys = pstart;
		if (vm_last_phys < pend)
			vm_last_phys = pend;

		/*
		 * Exclude any memory allocated for the kernel as computed
		 * by pmap_bootstrap(), i.e. the range
		 *	[KERNBASE_PA, avail_start>.
		 * Note that this will also exclude the `etext gap' range
		 * already uploaded above.
		 */
		if (pstart < PMAP_BOOTSTRAP_VA2PA(KERNBASE)) {
			/*
			 * This segment starts below the kernel load address.
			 * Chop it off at the pstart of the kernel.
			 */
			paddr_t	chop = PMAP_BOOTSTRAP_VA2PA(KERNBASE);

			if (pend < chop)
				chop = pend;
#ifdef DEBUG
			prom_printf("bootstrap gap: pstart %lx, chop %lx, pend %lx\n",
				pstart, chop, pend);
#endif
			uvm_page_physload(
				atop(pstart),
				atop(chop),
				atop(pstart),
				atop(chop),
				VM_FREELIST_DEFAULT);

			/*
			 * Adjust the start address to reflect the
			 * uploaded portion of this segment.
			 */
			pstart = chop;
		}

		/* Skip the current kernel address range */
		if (pstart <= avail_start && avail_start < pend)
			pstart = avail_start;

		if (pstart == pend)
			continue;

		/* Upload (the rest of) this segment */
		uvm_page_physload(
			atop(pstart),
			atop(pend),
			atop(pstart),
			atop(pend), VM_FREELIST_DEFAULT);
	}

#if defined(MULTIPROCESSOR)
	{
		CPU_INFO_ITERATOR cpunum;
		struct cpu_info *cpi;

		for (CPU_INFO_FOREACH(cpunum, cpi)) {
			if (cpi->ci_free_sva1)
				uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)),
						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)),
						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)),
						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)),
						  VM_FREELIST_DEFAULT);
			if (cpi->ci_free_sva2)
				uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)),
						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)),
						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)),
						  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)),
						  VM_FREELIST_DEFAULT);
		}
	}
#endif
}

/*
 * This routine is used by mmrw() to validate access to `/dev/mem'.
 */
int
pmap_pa_exists(paddr_t pa)
{
	int nmem;
	struct memarr *mp;

	for (mp = pmemarr, nmem = npmemarr; --nmem >= 0; mp++) {
		if (pa >= mp->addr && pa < mp->addr + mp->len)
			return 1;
	}

	return 0;
}

/* update pv_flags given a valid pte */
#define	MR4_4C(pte) (((pte) >> PG_M_SHIFT) & (PV_MOD | PV_REF))
#define MR4M(pte) (((pte) >> PG_M_SHIFT4M) & (PV_MOD4M | PV_REF4M))

/*----------------------------------------------------------------*/

/*
 * Agree with the monitor ROM as to how many MMU entries are
 * to be reserved, and map all of its segments into all contexts.
 *
 * Unfortunately, while the Version 0 PROM had a nice linked list of
 * taken virtual memory, the Version 2 PROM provides instead a convoluted
 * description of *free* virtual memory.  Rather than invert this, we
 * resort to two magic constants from the PROM vector description file.
 */
#if defined(SUN4) || defined(SUN4C)
void
mmu_reservemon4_4c(int *nrp, int *nsp)
{
	u_int va = 0, eva = 0;
	int mmuseg, i, nr, ns, vr;
	int *pte;
#if defined(SUN4_MMU3L)
	int mmureg, lastvr = 0;
#endif
	struct regmap *rp;

#if defined(SUN4)
	if (CPU_ISSUN4) {
		prom_vstart = va = OLDMON_STARTVADDR;
		prom_vend = eva = OLDMON_ENDVADDR;
	}
#endif
#if defined(SUN4C)
	if (CPU_ISSUN4C) {
		prom_vstart = va = OPENPROM_STARTVADDR;
		prom_vend = eva = OPENPROM_ENDVADDR;
	}
#endif
	ns = *nsp;
	nr = *nrp;
	while (va < eva) {
		vr = VA_VREG(va);
		rp = &pmap_kernel()->pm_regmap[vr];

#if defined(SUN4_MMU3L)
		if (HASSUN4_MMU3L && vr != lastvr) {
			lastvr = vr;
			mmureg = getregmap(va);
			if (mmureg < nr)
				rp->rg_smeg = nr = mmureg;
			/*
			 * On 3-level MMU machines, we distribute regions,
			 * rather than segments, amongst the contexts.
			 */
			for (i = ncontext; --i > 0;)
				prom_setcontext(i, (void *)va, mmureg);
		}
#endif
		mmuseg = getsegmap(va);
		if (mmuseg < ns)
			ns = mmuseg;

		if (!HASSUN4_MMU3L)
			for (i = ncontext; --i > 0;)
				prom_setcontext(i, (void *)va, mmuseg);

		if (mmuseg == seginval) {
			va += NBPSG;
			continue;
		}
		/*
		 * Another PROM segment. Enter into region map.
		 * Assume the entire segment is valid.
		 */
		rp->rg_nsegmap += 1;
		rp->rg_segmap[VA_VSEG(va)].sg_pmeg = mmuseg;
		rp->rg_segmap[VA_VSEG(va)].sg_npte = NPTESG;
		pte = rp->rg_segmap[VA_VSEG(va)].sg_pte;

		/* PROM maps its memory user-accessible: fix it. */
		for (i = NPTESG; --i >= 0; va += NBPG, pte++) {
			*pte = getpte4(va) | PG_S;
			setpte4(va, *pte);
		}
	}
	*nsp = ns;
	*nrp = nr;
	return;
}
#endif

#if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */

u_long
srmmu_bypass_read(u_long paddr)
{
	unsigned long v;

	if (cpuinfo.mxcc) {
		/*
		 * We're going to have to use MMU passthrough. If we're on
		 * a Viking SuperSPARC with a MultiCache Controller, we
		 * need to set the AC (Alternate Cacheable) bit in the MMU's
		 * control register in order to not by-pass the cache.
		 */

		unsigned long s = lda(SRMMU_PCR, ASI_SRMMU);

		/* set MMU AC bit */
		sta(SRMMU_PCR, ASI_SRMMU, s | VIKING_PCR_AC);
		v = lda(paddr, ASI_BYPASS);
		sta(SRMMU_PCR, ASI_SRMMU, s);
	} else
		v = lda(paddr, ASI_BYPASS);

	return (v);
}


/*
 * Take the monitor's initial page table layout, convert it to 3rd-level pte's
 * (it starts out as a L1 mapping), and install it along with a set of kernel
 * mapping tables as the kernel's initial page table setup. Also create and
 * enable a context table. I suppose we also want to block user-mode access
 * to the new kernel/ROM mappings.
 */

/*
 * mmu_reservemon4m(): Copies the existing (ROM) page tables to kernel space,
 * converting any L1/L2 PTEs to L3 PTEs. Does *not* copy the L1 entry mapping
 * the kernel at KERNBASE since we don't want to map 16M of physical
 * memory for the kernel. Thus the kernel must be installed later!
 * Also installs ROM mappings into the kernel pmap.
 * NOTE: This also revokes all user-mode access to the mapped regions.
 */
void
mmu_reservemon4m(struct pmap *kpmap)
{
	unsigned int rom_ctxtbl;
	int te;

#if !(defined(PROM_AT_F0) || defined(MSIIEP))
	prom_vstart = OPENPROM_STARTVADDR;
	prom_vend = OPENPROM_ENDVADDR;
#else /* OBP3/OFW in JavaStations */
	prom_vstart = 0xf0000000;
#if defined(MSIIEP)
	prom_vend = 0xf0800000;
#else
	prom_vend = 0xf0080000;
#endif
#endif

	/*
	 * XXX: although the sun4m can handle 36 bits of physical
	 * address space, we assume that all these page tables, etc
	 * are in the lower 4G (32-bits) of address space, i.e. out of I/O
	 * space. Eventually this should be changed to support the 36 bit
	 * physical addressing, in case some crazed ROM designer decides to
	 * stick the pagetables up there. In that case, we should use MMU
	 * transparent mode, (i.e. ASI 0x20 to 0x2f) to access
	 * physical memory.
	 */

	rom_ctxtbl = (lda(SRMMU_CXTPTR,ASI_SRMMU) << SRMMU_PPNPASHIFT);

	te = srmmu_bypass_read(rom_ctxtbl);	/* i.e. context 0 */

	switch (te & SRMMU_TETYPE) {
	case SRMMU_TEINVALID:
		cpuinfo.ctx_tbl[0] = SRMMU_TEINVALID;
		panic("mmu_reservemon4m: no existing L0 mapping! "
		      "(How are we running?");
		break;
	case SRMMU_TEPTE:
		panic("mmu_reservemon4m: can't handle ROM 4G page size");
		/* XXX: Should make this work, however stupid it is */
		break;
	case SRMMU_TEPTD:
		mmu_setup4m_L1(te, kpmap);
		break;
	default:
		panic("mmu_reservemon4m: unknown pagetable entry type");
	}
}

/* regtblptd - PTD for region table to be remapped */
void
mmu_setup4m_L1(int regtblptd, struct pmap *kpmap)
{
	unsigned int regtblrover;
	int i;
	unsigned int te;
	struct regmap *rp;
	int j, k;

	/*
	 * Here we scan the region table to copy any entries which appear.
	 * We are only concerned with regions in kernel space and above
	 * (i.e. regions VA_VREG(KERNBASE)+1 to 0xff). We ignore the first
	 * region (at VA_VREG(KERNBASE)), since that is the 16MB L1 mapping
	 * that the ROM used to map the kernel in initially. Later, we will
	 * rebuild a new L3 mapping for the kernel and install it before
	 * switching to the new pagetables.
	 */
	regtblrover =
		((regtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT) +
		(VA_VREG(KERNBASE)+1) * sizeof(long);	/* kernel only */

	for (i = VA_VREG(KERNBASE) + 1; i < SRMMU_L1SIZE;
	     i++, regtblrover += sizeof(long)) {

		/* The region we're dealing with */
		rp = &kpmap->pm_regmap[i];

		te = srmmu_bypass_read(regtblrover);
		switch(te & SRMMU_TETYPE) {
		case SRMMU_TEINVALID:
			break;

		case SRMMU_TEPTE:
#ifdef DEBUG
			prom_printf("mmu_setup4m_L1: "
			       "converting region 0x%x from L1->L3\n", i);
#endif
			/*
			 * This region entry covers 64MB of memory -- or
			 * (NSEGRG * NPTESG) pages -- which we must convert
			 * into a 3-level description.
			 */

			for (j = 0; j < SRMMU_L2SIZE; j++) {
				struct segmap *sp = &rp->rg_segmap[j];

				for (k = 0; k < SRMMU_L3SIZE; k++) {
					setpgt4m(&sp->sg_pte[k],
						(te & SRMMU_L1PPNMASK) |
						(j << SRMMU_L2PPNSHFT) |
						(k << SRMMU_L3PPNSHFT) |
						(te & SRMMU_PGBITSMSK) |
						((te & SRMMU_PROT_MASK) |
						 PPROT_U2S_OMASK) |
						SRMMU_TEPTE);
				}
			}
			break;

		case SRMMU_TEPTD:
			mmu_setup4m_L2(te, rp);
			break;

		default:
			panic("mmu_setup4m_L1: unknown pagetable entry type");
		}
	}
}

void
mmu_setup4m_L2(int segtblptd, struct regmap *rp)
{
	unsigned int segtblrover;
	int i, k;
	unsigned int te;
	struct segmap *sp;

	segtblrover = (segtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT;
	for (i = 0; i < SRMMU_L2SIZE; i++, segtblrover += sizeof(long)) {

		sp = &rp->rg_segmap[i];

		te = srmmu_bypass_read(segtblrover);
		switch(te & SRMMU_TETYPE) {
		case SRMMU_TEINVALID:
			break;

		case SRMMU_TEPTE:
#ifdef DEBUG
			prom_printf("mmu_setup4m_L2: converting L2 entry at segment 0x%x to L3\n",i);
#endif
			/*
			 * This segment entry covers 256KB of memory -- or
			 * (NPTESG) pages -- which we must convert
			 * into a 3-level description.
			 */
			for (k = 0; k < SRMMU_L3SIZE; k++) {
				setpgt4m(&sp->sg_pte[k],
					(te & SRMMU_L1PPNMASK) |
					(te & SRMMU_L2PPNMASK) |
					(k << SRMMU_L3PPNSHFT) |
					(te & SRMMU_PGBITSMSK) |
					((te & SRMMU_PROT_MASK) |
					 PPROT_U2S_OMASK) |
					SRMMU_TEPTE);
			}
			break;

		case SRMMU_TEPTD:
			mmu_setup4m_L3(te, sp);
			break;

		default:
			panic("mmu_setup4m_L2: unknown pagetable entry type");
		}
	}
}

void
mmu_setup4m_L3(int pagtblptd, struct segmap *sp)
{
	unsigned int pagtblrover;
	int i;
	unsigned int te;

	pagtblrover = (pagtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT;
	for (i = 0; i < SRMMU_L3SIZE; i++, pagtblrover += sizeof(long)) {
		te = srmmu_bypass_read(pagtblrover);
		switch(te & SRMMU_TETYPE) {
		case SRMMU_TEINVALID:
			break;
		case SRMMU_TEPTE:
			setpgt4m(&sp->sg_pte[i], te | PPROT_U2S_OMASK);
			pmap_kernel()->pm_stats.resident_count++;
			break;
		case SRMMU_TEPTD:
			panic("mmu_setup4m_L3: PTD found in L3 page table");
		default:
			panic("mmu_setup4m_L3: unknown pagetable entry type");
		}
	}
}
#endif /* defined SUN4M || defined SUN4D */

/*----------------------------------------------------------------*/

#if defined(SUN4) || defined(SUN4C)
/*
 * MMU management.
 */
static int	me_alloc(struct mmuentry *, struct pmap *, int, int);
static void	me_free(struct pmap *, u_int);
#if defined(SUN4_MMU3L)
static int	region_alloc(struct mmuentry *, struct pmap *, int);
static void	region_free(struct pmap *, u_int);
#endif


/*
 * Allocate an MMU entry (i.e., a PMEG).
 * If necessary, steal one from someone else.
 * Put it on the tail of the given queue
 * (which is either the LRU list or the locked list).
 * The locked list is not actually ordered, but this is easiest.
 * Also put it on the given (new) pmap's chain,
 * enter its pmeg number into that pmap's segmap,
 * and store the pmeg's new virtual segment number (me->me_vseg).
 *
 * This routine is large and complicated, but it must be fast
 * since it implements the dynamic allocation of MMU entries.
 */

static inline int
me_alloc(struct mmuentry *mh, struct pmap *newpm, int newvreg, int newvseg)
{
	struct mmuentry *me;
	struct pmap *pm;
	int i, va, *ptep, pte;
	int ctx;
	struct regmap *rp;
	struct segmap *sp;

	/* try free list first */
	if (!mmuq_empty(&segm_freelist)) {
		me = mmuq_first(&segm_freelist);
		mmuq_remove(me);
#ifdef DEBUG
		if (me->me_pmap != NULL)
			panic("me_alloc: freelist entry has pmap");
		DPRINTF(PDB_MMU_ALLOC,
		    "me_alloc: got pmeg %d", me->me_cookie);
#endif
		mmuq_insert_tail(mh, me);

		/* onto on pmap chain; pmap is already locked, if needed */
		TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain);
#ifdef DIAGNOSTIC
		pmap_stats.ps_npmeg_free--;
		if (mh == &segm_locked) {
			pmap_stats.ps_npmeg_locked++;
			me->me_statp = &pmap_stats.ps_npmeg_locked;
		} else {
			pmap_stats.ps_npmeg_lru++;
			me->me_statp = &pmap_stats.ps_npmeg_lru;
		}
#endif

		/* into pmap segment table, with backpointers */
		me->me_pmap = newpm;
		me->me_vseg = newvseg;
		me->me_vreg = newvreg;

		return (me->me_cookie);
	}

	/* no luck, take head of LRU list */
	if (mmuq_empty(&segm_lru))
		panic("me_alloc: all pmegs gone");

	me = mmuq_first(&segm_lru);
	pm = me->me_pmap;
	DPRINTF(PDB_MMU_ALLOC | PDB_MMU_STEAL,
	    "me_alloc: stealing pmeg 0x%x from pmap %p", me->me_cookie, pm);

	mmu_stolenpmegs_evcnt.ev_count++;

	/*
	 * Remove from LRU list, and insert at end of new list
	 * (probably the LRU list again, but so what?).
	 */
	mmuq_remove(me);
	mmuq_insert_tail(mh, me);

#ifdef DIAGNOSTIC
	if (mh == &segm_locked) {
		pmap_stats.ps_npmeg_lru--;
		pmap_stats.ps_npmeg_locked++;
		me->me_statp = &pmap_stats.ps_npmeg_locked;
	} else {
		me->me_statp = &pmap_stats.ps_npmeg_lru;
	}
#endif

	rp = &pm->pm_regmap[me->me_vreg];
	sp = &rp->rg_segmap[me->me_vseg];
	ptep = sp->sg_pte;

#ifdef DEBUG
	if (sp->sg_pmeg != me->me_cookie)
		panic("me_alloc: wrong sg_pmeg (%d != %d)",
				sp->sg_pmeg, me->me_cookie);
#endif

	/*
	 * The PMEG must be mapped into some context so that we can
	 * read its PTEs.  Use its current context if it has one;
	 * if not, and since context 0 is reserved for the kernel,
	 * the simplest method is to switch to 0 and map the PMEG
	 * to virtual address 0---which, being a user space address,
	 * is by definition not in use.
	 *
	 * XXX do not have to flush cache immediately
	 */
	ctx = getcontext4();

	/*
	 * Even if we're stealing a PMEG from ourselves (i.e. if pm==newpm),
	 * we must make sure there are no user register windows in the CPU
	 * for the following reasons:
	 * (1) if we have a write-allocate cache and the segment we are
	 *     stealing contains stack pages, an interrupt during the
	 *     interval that starts at cache_flush_segment() below and ends
	 *     when the segment is finally removed from the MMU, may cause
	 *     dirty cache lines to reappear.
	 * (2) when re-wiring this PMEG for use by another segment (e.g.
	 *     in mmu_pagein()) a window exists where the PTEs in this PMEG
	 *     point at arbitrary pages allocated to this address space.
	 *     Again, a register window flush at this point is likely to
	 *     cause data corruption in case the segment being rewired
	 *     contains stack virtual addresses.
	 */
	write_user_windows();
	if (CTX_USABLE(pm,rp)) {
		setcontext4(pm->pm_ctxnum);
		va = VSTOVA(me->me_vreg, me->me_vseg);
#ifdef DEBUG
		if (getsegmap(va) != me->me_cookie)
			panic("me_alloc: wrong pmeg in MMU (%d != %d)",
				getsegmap(va), me->me_cookie);
#endif
		cache_flush_segment(me->me_vreg, me->me_vseg, pm->pm_ctxnum);
	} else {
		va = 0;
		setcontext4(0);
		if (HASSUN4_MMU3L)
			setregmap(va, tregion);
		setsegmap(va, me->me_cookie);
		/*
		 * No cache flush needed: it happened earlier when
		 * the old context was taken.
		 */
	}

	/*
	 * Record reference and modify bits for each page,
	 * and copy PTEs into kernel memory so that they can
	 * be reloaded later.
	 */
	i = NPTESG;
	do {
		int swbits = *ptep & PG_MBZ;
		pte = getpte4(va);
		if ((pte & (PG_V | PG_TYPE)) == (PG_V | PG_OBMEM)) {
			struct vm_page *pg;
			if ((pg = pvhead4_4c(pte)) != NULL)
				VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte);
		}
		*ptep++ = swbits | (pte & ~(PG_U|PG_M));
		va += NBPG;
	} while (--i > 0);

	/* update segment tables */
	if (CTX_USABLE(pm,rp)) {
		va = VSTOVA(me->me_vreg,me->me_vseg);
		if (pm != pmap_kernel() || HASSUN4_MMU3L)
			setsegmap(va, seginval);
		else {
			/* Unmap segment from all contexts */
			for (i = ncontext; --i >= 0;) {
				setcontext4(i);
				setsegmap(va, seginval);
			}
		}
	}
	sp->sg_pmeg = seginval;

	/* off old pmap chain */
	TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain);
	setcontext4(ctx);

	/* onto new pmap chain; new pmap is already locked, if needed */
	TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain);

	/* into new segment table, with backpointers */
	me->me_pmap = newpm;
	me->me_vseg = newvseg;
	me->me_vreg = newvreg;

	return (me->me_cookie);
}

/*
 * Free an MMU entry.
 *
 * Assumes the corresponding pmap is already locked.
 * Caller must update hardware.
 */
static inline void
me_free(struct pmap *pm, u_int pmeg)
{
	struct mmuentry *me = &mmusegments[pmeg];
#ifdef DEBUG
	struct regmap *rp;
	int i, va, tpte, ctx;
#endif

#ifdef DEBUG
	rp = &pm->pm_regmap[me->me_vreg];
	DPRINTF(PDB_MMU_ALLOC,
	   "me_free: freeing pmeg %d from pmap %p", me->me_cookie, pm);
	if (me->me_cookie != pmeg)
		panic("me_free: wrong mmuentry");
	if (pm != me->me_pmap)
		panic("me_free: pm != me_pmap");
	if (rp->rg_segmap[me->me_vseg].sg_pmeg != pmeg &&
	    rp->rg_segmap[me->me_vseg].sg_pmeg != seginval)
		panic("me_free: wrong sg_pmeg (%d != %d)",
			rp->rg_segmap[me->me_vseg].sg_pmeg, pmeg);

	/* check for spurious mappings (using temp. mapping in context 0) */
	ctx = getcontext4();
	setcontext4(0);
	if (HASSUN4_MMU3L)
		setregmap(0, tregion);
	setsegmap(0, me->me_cookie);
	va = 0;
	i = NPTESG;
	do {
		tpte = getpte4(va);
		if ((tpte & PG_V) == PG_V)
			panic("me_free: segment not clean (pte=%x)", tpte);
		va += NBPG;
	} while (--i > 0);
	setcontext4(ctx);
#endif /* DEBUG */

	/* take mmu entry off pmap chain */
	TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain);

	/* off LRU or lock chain */
	mmuq_remove(me);
#ifdef DIAGNOSTIC
	if (me->me_statp == NULL)
		panic("me_statp");
	(*me->me_statp)--;
	me->me_statp = NULL;
#endif

	/* no associated pmap; on free list */
	me->me_pmap = NULL;
	mmuq_insert_tail(&segm_freelist, me);
#ifdef DIAGNOSTIC
	pmap_stats.ps_npmeg_free++;
#endif
}

#if defined(SUN4_MMU3L)

/* XXX - Merge with segm_alloc/segm_free ? */

int
region_alloc(struct mmuentry *mh, struct pmap *newpm, int newvr)
{
	struct mmuentry *me;
	struct pmap *pm;
	int ctx;
	struct regmap *rp;

	/* try free list first */
	if (!mmuq_empty(&region_freelist)) {
		me = mmuq_first(&region_freelist);
		mmuq_remove(me);
#ifdef DEBUG
		if (me->me_pmap != NULL)
			panic("region_alloc: freelist entry has pmap");
		DPRINTF(PDB_MMUREG_ALLOC,
		    "region_alloc: got smeg 0x%x", me->me_cookie);
#endif
		mmuq_insert_tail(mh, me);

		/* onto on pmap chain; pmap is already locked, if needed */
		TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain);

		/* into pmap segment table, with backpointers */
		me->me_pmap = newpm;
		me->me_vreg = newvr;

		return (me->me_cookie);
	}

	/* no luck, take head of LRU list */
	if (mmuq_empty(&region_lru))
		panic("region_alloc: all smegs gone");

	me = mmuq_first(&region_lru);

	pm = me->me_pmap;
	if (pm == NULL)
		panic("region_alloc: LRU entry has no pmap");
	if (pm == pmap_kernel())
		panic("region_alloc: stealing from kernel");
	DPRINTF(PDB_MMUREG_ALLOC | PDB_MMUREG_STEAL,
	    "region_alloc: stealing smeg 0x%x from pmap %p",
	    me->me_cookie, pm);

	/*
	 * Remove from LRU list, and insert at end of new list
	 * (probably the LRU list again, but so what?).
	 */
	mmuq_remove(me);
	mmuq_insert_tail(mh, me);

	rp = &pm->pm_regmap[me->me_vreg];
	ctx = getcontext4();

	/* Flush register windows; see comment in me_alloc() */
	write_user_windows();
	if (pm->pm_ctx) {
		setcontext4(pm->pm_ctxnum);
		cache_flush_region(me->me_vreg, pm->pm_ctxnum);
	}

	/* update region tables */
	if (pm->pm_ctx)
		setregmap(VRTOVA(me->me_vreg), reginval);
	rp->rg_smeg = reginval;

	/* off old pmap chain */
	TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain);
	setcontext4(ctx);	/* done with old context */

	/* onto new pmap chain; new pmap is already locked, if needed */
	TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain);

	/* into new segment table, with backpointers */
	me->me_pmap = newpm;
	me->me_vreg = newvr;

	return (me->me_cookie);
}

/*
 * Free an MMU entry.
 * Assumes the corresponding pmap is already locked.
 * Caller must update hardware.
 */
void
region_free(struct pmap *pm, u_int smeg)
{
	struct mmuentry *me = &mmuregions[smeg];

	DPRINTF(PDB_MMUREG_ALLOC,
	    "region_free: freeing smeg 0x%x from pmap %p", me->me_cookie, pm);
#ifdef DEBUG
	if (me->me_cookie != smeg)
		panic("region_free: wrong mmuentry");
	if (pm != me->me_pmap)
		panic("region_free: pm != me_pmap");
#endif

	/* take mmu entry off pmap chain */
	TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain);

	/* off LRU or lock chain */
	mmuq_remove(me);

	/* no associated pmap; on free list */
	me->me_pmap = NULL;
	mmuq_insert_tail(&region_freelist, me);
}

static void
mmu_pagein_reg(struct pmap *pm, struct regmap *rp, vaddr_t va,
		int vr, struct mmuentry *mh)
{
	int i, s, smeg;

	va = VA_ROUNDDOWNTOREG(va);
	rp->rg_smeg = smeg = region_alloc(mh, pm, vr);

	s = splvm();
	if (pm == pmap_kernel()) {
		/* Map region into all contexts */
		int ctx = getcontext4();
		i = ncontext - 1;
		do {
			setcontext4(i);
			setregmap(va, smeg);
		} while (--i >= 0);
		setcontext4(ctx);
	} else
		setregmap(va, smeg);

	/* Load PMEGs into this region */
	for (i = 0; i < NSEGRG; i++) {
		setsegmap(va, rp->rg_segmap[i].sg_pmeg);
		va += NBPSG;
	}
	splx(s);
}
#endif /* SUN4_MMU3L */

static void
mmu_pmeg_lock(int pmeg)
{
	struct mmuentry *me = &mmusegments[pmeg];

	mmuq_remove(me);
	mmuq_insert_tail(&segm_locked, me);
#ifdef DIAGNOSTIC
	(*me->me_statp)--;
	pmap_stats.ps_npmeg_locked++;
	me->me_statp = &pmap_stats.ps_npmeg_locked;
#endif
}

static void
mmu_pmeg_unlock(int pmeg)
{
	struct mmuentry *me = &mmusegments[pmeg];

	mmuq_remove(me);
	mmuq_insert_tail(&segm_lru, me);
#ifdef DIAGNOSTIC
	(*me->me_statp)--;
	pmap_stats.ps_npmeg_lru++;
	me->me_statp = &pmap_stats.ps_npmeg_lru;
#endif
}

static void
mmu_pagein_seg(struct pmap *pm, struct segmap *sp, vaddr_t va,
		int vr, int vs, struct mmuentry *mh)
{
	int s, i, pmeg, *pte;

	mmu_pagein_evcnt.ev_count++;

	va = VA_ROUNDDOWNTOSEG(va);
	s = splvm();		/* paranoid */
	sp->sg_pmeg = pmeg = me_alloc(mh, pm, vr, vs);
	if (pm != pmap_kernel() || HASSUN4_MMU3L)
		setsegmap(va, pmeg);
	else {
		/* Map kernel address into all contexts */
		int ctx = getcontext4();
		i = ncontext - 1;
		do {
			setcontext4(i);
			setsegmap(va, pmeg);
		} while (--i >= 0);
		setcontext4(ctx);
	}

	/* reload segment: write PTEs into a the MMU */
	pte = sp->sg_pte;
	i = NPTESG;
	do {
		setpte4(va, *pte++ & ~PG_MBZ);
		va += NBPG;
	} while (--i > 0);
	splx(s);
}

/*
 * `Page in' (load or inspect) an MMU entry; called on page faults.
 * Returns 1 if we reloaded the segment, -1 if the segment was
 * already loaded and the page was marked valid (in which case the
 * fault must be a bus error or something), or 0 (segment loaded but
 * PTE not valid, or segment not loaded at all).
 */
int
mmu_pagein(struct pmap *pm, vaddr_t va, int prot)
{
	int vr, vs, bits;
	struct regmap *rp;
	struct segmap *sp;

	PMAP_LOCK();

	if (prot != VM_PROT_NONE)
		bits = PG_V | ((prot & VM_PROT_WRITE) ? PG_W : 0);
	else
		bits = 0;

	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];

	/* return 0 if we have no PMEGs to load */
	if (rp->rg_nsegmap == 0) {
		PMAP_UNLOCK();
		return (0);
	}

#ifdef DIAGNOSTIC
	if (rp->rg_segmap == NULL)
		panic("pagein: no segmap");
#endif

#if defined(SUN4_MMU3L)
	if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
		mmu_pagein_reg(pm, rp, va, vr, &region_lru);
#endif
	sp = &rp->rg_segmap[vs];

	/* return 0 if we have no PTEs to load */
	if (sp->sg_npte == 0) {
		PMAP_UNLOCK();
		return (0);
	}

	/* return -1 if the fault is `hard', 0 if not */
	if (sp->sg_pmeg != seginval) {
		PMAP_UNLOCK();
		return (bits && (getpte4(va) & bits) == bits ? -1 : 0);
	}

	mmu_pagein_seg(pm, sp, va, vr, vs, &segm_lru);
	PMAP_UNLOCK();
	return (1);
}
#endif /* SUN4 or SUN4C */

/*
 * Allocate a context.  If necessary, steal one from someone else.
 * Changes hardware context number and loads segment map.
 *
 * This routine is only ever called from locore.s just after it has
 * saved away the previous process, so there are no active user windows.
 */
static void
ctx_alloc(struct pmap *pm)
{
	union ctxinfo *c;
	int cnum, i = 0, doflush;
	struct regmap *rp;
	int gap_start, gap_end;
	vaddr_t va;
#if defined(SUN4M) || defined(SUN4D)
	struct cpu_info *cpi;
#endif

	KASSERT(mutex_owned(&ctx_lock));

/*XXX-GCC!*/gap_start=gap_end=0;
#ifdef DEBUG
	if (pm->pm_ctx)
		panic("ctx_alloc pm_ctx");
#endif
	DPRINTF(PDB_CTX_ALLOC,
	    "ctx_alloc[%d](%p)", cpu_number(), pm);

	if (CPU_HAS_SUNMMU) {
		gap_start = pm->pm_gap_start;
		gap_end = pm->pm_gap_end;
	}

	if ((c = ctx_freelist) != NULL) {
		ctx_freelist = c->c_nextfree;
		cnum = c - ctxinfo;
		doflush = 0;
	} else {
		if ((ctx_kick += ctx_kickdir) >= ncontext) {
			ctx_kick = ncontext - 1;
			ctx_kickdir = -1;
		} else if (ctx_kick < 1) {
			ctx_kick = 1;
			ctx_kickdir = 1;
		}
		c = &ctxinfo[cnum = ctx_kick];
#ifdef DEBUG
		if (c->c_pmap == NULL)
			panic("ctx_alloc cu_pmap");
#endif
		DPRINTF(PDB_CTX_ALLOC | PDB_CTX_STEAL,
		    "ctx_alloc[%d]: steal context %d from %p",
		    cpu_number(), cnum, c->c_pmap);

		c->c_pmap->pm_ctx = NULL;
		c->c_pmap->pm_ctxnum = 0;
		doflush = (CACHEINFO.c_vactype != VAC_NONE);
		if (CPU_HAS_SUNMMU) {
			if (gap_start < c->c_pmap->pm_gap_start)
				gap_start = c->c_pmap->pm_gap_start;
			if (gap_end > c->c_pmap->pm_gap_end)
				gap_end = c->c_pmap->pm_gap_end;
		}
	}

	c->c_pmap = pm;
	pm->pm_ctx = c;
	pm->pm_ctxnum = cnum;

	if (CPU_HAS_SUNMMU) {

		/*
		 * Write pmap's region (3-level MMU) or segment table into
		 * the MMU.
		 *
		 * Only write those entries that actually map something in
		 * this context by maintaining a pair of region numbers in
		 * between which the pmap has no valid mappings.
		 *
		 * If a context was just allocated from the free list, trust
		 * that all its pmeg numbers are `seginval'. We make sure this
		 * is the case initially in pmap_bootstrap(). Otherwise, the
		 * context was freed by calling ctx_free() in pmap_release(),
		 * which in turn is supposedly called only when all mappings
		 * have been removed.
		 *
		 * On the other hand, if the context had to be stolen from
		 * another pmap, we possibly shrink the gap to be the
		 * disjuction of the new and the previous map.
		 */

		setcontext4(cnum);
		if (doflush)
			cache_flush_context(cnum);

		rp = pm->pm_regmap;
		for (va = 0, i = NUREG; --i >= 0; ) {
			if (VA_VREG(va) >= gap_start) {
				va = VRTOVA(gap_end);
				i -= gap_end - gap_start;
				rp += gap_end - gap_start;
				if (i < 0)
					break;
				/* mustn't re-enter this branch */
				gap_start = NUREG;
			}
			if (HASSUN4_MMU3L) {
				setregmap(va, rp++->rg_smeg);
				va += NBPRG;
			} else {
				int j;
				struct segmap *sp = rp->rg_segmap;
				for (j = NSEGRG; --j >= 0; va += NBPSG)
					setsegmap(va,
						  sp?sp++->sg_pmeg:seginval);
				rp++;
			}
		}

	} else if (CPU_HAS_SRMMU) {

#if defined(SUN4M) || defined(SUN4D)
		/*
		 * Reload page and context tables to activate the page tables
		 * for this context.
		 *
		 * The gap stuff isn't really needed in the sun4m architecture,
		 * since we don't have to worry about excessive mappings (all
		 * mappings exist since the page tables must be complete for
		 * the mmu to be happy).
		 *
		 * If a context was just allocated from the free list, trust
		 * that all of its mmu-edible page tables are zeroed out
		 * (except for those associated with the kernel). We make
		 * sure this is the case initially in pmap_bootstrap() and
		 * pmap_init() (?).
		 * Otherwise, the context was freed by calling ctx_free() in
		 * pmap_release(), which in turn is supposedly called only
		 * when all mappings have been removed.
		 *
		 * XXX: Do we have to flush cache after reloading ctx tbl?
		 */

		/*
		 * We need to flush the cache only when stealing a context
		 * from another pmap. In that case it's Ok to switch the
		 * context and leave it set, since the context table
		 * will have a valid region table entry for this context
		 * number.
		 *
		 * Otherwise, we switch to the new context after loading
		 * the context table entry with the new pmap's region.
		 */
		if (doflush) {
			cache_flush_context(cnum);
		}

		/*
		 * The context allocated to a process is the same on all CPUs.
		 * Here we install the per-CPU region table in each CPU's
		 * context table slot.
		 *
		 * Note on multi-threaded processes: a context must remain
		 * valid as long as any thread is still running on a CPU.
		 */
		for (CPU_INFO_FOREACH(i, cpi)) {
			setpgt4m(&cpi->ctx_tbl[cnum],
				 (pm->pm_reg_ptps_pa[i] >> SRMMU_PPNPASHIFT) |
					SRMMU_TEPTD);
		}

		/* And finally switch to the new context */
		(*cpuinfo.pure_vcache_flush)();
		setcontext4m(cnum);
#endif /* SUN4M || SUN4D */
	}
}

/*
 * Give away a context.
 */
static void
ctx_free(struct pmap *pm)
{
	union ctxinfo *c;
	int ctx;
#if defined(SUN4M) || defined(SUN4D)
	struct cpu_info *cpi;
#endif

	KASSERT(mutex_owned(&ctx_lock));

	c = pm->pm_ctx;
	ctx = pm->pm_ctxnum;
	pm->pm_ctx = NULL;
	pm->pm_ctxnum = 0;
#if defined(SUN4) || defined(SUN4C)
	if (CPU_HAS_SUNMMU) {
		int octx = getcontext4();
		setcontext4(ctx);
		cache_flush_context(ctx);
		setcontext4(octx);
	}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)
	if (CPU_HAS_SRMMU) {
		CPU_INFO_ITERATOR i;

		__USE(i);

		cache_flush_context(ctx);
		tlb_flush_context(ctx, PMAP_CPUSET(pm));
		for (CPU_INFO_FOREACH(i, cpi)) {
			setpgt4m(&cpi->ctx_tbl[ctx], SRMMU_TEINVALID);
		}
	}
#endif

	c->c_nextfree = ctx_freelist;
	ctx_freelist = c;
}


/*----------------------------------------------------------------*/

/*
 * pvlist functions.
 */

/*
 * Walk the given pv list, and for each PTE, set or clear some bits
 * (e.g., PG_W or PG_NC).
 *
 * This routine flushes the cache for any page whose PTE changes,
 * as long as the process has a context; this is overly conservative.
 * It also copies ref and mod bits to the pvlist, on the theory that
 * this might save work later.  (XXX should test this theory)
 */

#if defined(SUN4) || defined(SUN4C)

void
pv_changepte4_4c(struct vm_page *pg, int bis, int bic)
{
	int pte, *ptep;
	struct pvlist *pv;
	struct pmap *pm;
	int va, vr, vs;
	int ctx, s;
	struct regmap *rp;
	struct segmap *sp;

	pv = VM_MDPAGE_PVHEAD(pg);

	write_user_windows();		/* paranoid? */
	s = splvm();			/* paranoid? */
	if (pv->pv_pmap == NULL) {
		splx(s);
		return;
	}
	ctx = getcontext4();
	for (; pv != NULL; pv = pv->pv_next) {
		pm = pv->pv_pmap;
		va = pv->pv_va;
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		rp = &pm->pm_regmap[vr];
		sp = &rp->rg_segmap[vs];
		ptep = &sp->sg_pte[VA_VPG(va)];

		if (sp->sg_pmeg == seginval) {
			/* not in hardware: just fix software copy */
			*ptep = (*ptep | bis) & ~bic;
		} else {
			/* in hardware: fix hardware copy */
			if (CTX_USABLE(pm,rp)) {
				setcontext4(pm->pm_ctxnum);
				/* XXX should flush only when necessary */
				pte = getpte4(va);
				/*
				 * XXX: always flush cache; conservative, but
				 * needed to invalidate cache tag protection
				 * bits and when disabling caching.
				 */
				cache_flush_page(va, pm->pm_ctxnum);
			} else {
				/* Make temp map in ctx 0 to access the PTE */
				setcontext4(0);
				if (HASSUN4_MMU3L)
					setregmap(0, tregion);
				setsegmap(0, sp->sg_pmeg);
				va = VA_VPG(va) << PGSHIFT;
				pte = getpte4(va);
			}
			if (pte & PG_V)
				VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte);
			pte = (pte | bis) & ~bic;
			setpte4(va, pte);
			*ptep = (*ptep & PG_MBZ) | pte;
		}
	}
	setcontext4(ctx);
	splx(s);
}

/*
 * Sync ref and mod bits in pvlist (turns off same in hardware PTEs).
 * Returns the new flags.
 *
 * This is just like pv_changepte, but we never add or remove bits,
 * hence never need to adjust software copies.
 */
int
pv_syncflags4_4c(struct vm_page *pg)
{
	struct pvlist *pv;
	struct pmap *pm;
	int pte, va, vr, vs, pmeg, flags;
	int ctx, s;
	struct regmap *rp;
	struct segmap *sp;

	pv = VM_MDPAGE_PVHEAD(pg);

	s = splvm();			/* paranoid? */
	if (pv->pv_pmap == NULL) {
		/* Page not mapped; pv_flags is already up to date */
		splx(s);
		return (0);
	}
	ctx = getcontext4();
	flags = pv->pv_flags;
	for (; pv != NULL; pv = pv->pv_next) {
		pm = pv->pv_pmap;
		va = pv->pv_va;
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		rp = &pm->pm_regmap[vr];
		sp = &rp->rg_segmap[vs];
		if ((pmeg = sp->sg_pmeg) == seginval)
			continue;
		if (CTX_USABLE(pm,rp)) {
			setcontext4(pm->pm_ctxnum);
			/* XXX should flush only when necessary */
			pte = getpte4(va);
			if (pte & PG_M)
				cache_flush_page(va, pm->pm_ctxnum);
		} else {
			/* Make temp map in ctx 0 to access the PTE */
			setcontext4(0);
			if (HASSUN4_MMU3L)
				setregmap(0, tregion);
			setsegmap(0, pmeg);
			va = VA_VPG(va) << PGSHIFT;
			pte = getpte4(va);
		}
		if (pte & (PG_M|PG_U) && pte & PG_V) {
			flags |= MR4_4C(pte);
			pte &= ~(PG_M|PG_U);
			setpte4(va, pte);
		}
	}

	VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
	setcontext4(ctx);
	splx(s);
	return (flags);
}

/*
 * pv_unlink is a helper function for pmap_remove.
 * It takes a pointer to the pv_table head for some physical address
 * and removes the appropriate (pmap, va) entry.
 *
 * Once the entry is removed, if the pv_table head has the cache
 * inhibit bit set, see if we can turn that off; if so, walk the
 * pvlist and turn off PG_NC in each PTE.  (The pvlist is by
 * definition nonempty, since it must have at least two elements
 * in it to have PV_NC set, and we only remove one here.)
 */
/*static*/ void
pv_unlink4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va)
{
	struct pvlist *pv0, *npv;

	pv0 = VM_MDPAGE_PVHEAD(pg);
	npv = pv0->pv_next;

	/*
	 * First entry is special (sigh).
	 */
	if (pv0->pv_pmap == pm && pv0->pv_va == va) {
		pmap_stats.ps_unlink_pvfirst++;
		if (npv != NULL) {
			/*
			 * Shift next entry into the head.
			 * Make sure to retain the REF, MOD and ANC flags.
			 */
			pv0->pv_next = npv->pv_next;
			pv0->pv_pmap = npv->pv_pmap;
			pv0->pv_va = npv->pv_va;
			pv0->pv_flags &= ~PV_NC;
			pv0->pv_flags |= (npv->pv_flags & PV_NC);
			pool_put(&pv_pool, npv);
		} else {
			/*
			 * No mappings left; we still need to maintain
			 * the REF and MOD flags. since pmap_is_modified()
			 * can still be called for this page.
			 */
			pv0->pv_pmap = NULL;
			pv0->pv_flags &= ~(PV_NC|PV_ANC);
			return;
		}
	} else {
		struct pvlist *prev;

		pmap_stats.ps_unlink_pvsearch++;
		for (prev = pv0;; prev = npv, npv = npv->pv_next) {
			if (npv == NULL) {
				panic("pv_unlink: pm %p is missing on pg %p",
					pm, pg);
			}
			if (npv->pv_pmap == pm && npv->pv_va == va)
				break;
		}
		prev->pv_next = npv->pv_next;
		pool_put(&pv_pool, npv);
	}
	if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) {
		/*
		 * Not cached: check whether we can fix that now.
		 */
		va = pv0->pv_va;
		for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next)
			if (BADALIAS(va, npv->pv_va) ||
			    (npv->pv_flags & PV_NC) != 0)
				return;
		pv0->pv_flags &= ~PV_ANC;
		pv_changepte4_4c(pg, 0, PG_NC);
	}
}

/*
 * pv_link is the inverse of pv_unlink, and is used in pmap_enter.
 * It returns PG_NC if the (new) pvlist says that the address cannot
 * be cached.
 */
/*static*/ int
pv_link4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va,
	    unsigned int *pteprotop)
{
	struct pvlist *pv0, *pv, *npv;
	int nc = (*pteprotop & PG_NC) != 0 ? PV_NC : 0;

	pv0 = VM_MDPAGE_PVHEAD(pg);

	if (pv0->pv_pmap == NULL) {
		/* no pvlist entries yet */
		pmap_stats.ps_enter_firstpv++;
		pv0->pv_next = NULL;
		pv0->pv_pmap = pm;
		pv0->pv_va = va;
		pv0->pv_flags |= nc;
		return (0);
	}

	/*
	 * Allocate the new PV entry now, and, if that fails, bail out
	 * before changing the cacheable state of the existing mappings.
	 */
	npv = pool_get(&pv_pool, PR_NOWAIT);
	if (npv == NULL)
		return (ENOMEM);

	pmap_stats.ps_enter_secondpv++;

	/*
	 * Before entering the new mapping, see if
	 * it will cause old mappings to become aliased
	 * and thus need to be `discached'.
	 */
	if (pv0->pv_flags & PV_ANC) {
		/* already uncached, just stay that way */
		*pteprotop |= PG_NC;
		goto link_npv;
	}

	for (pv = pv0; pv != NULL; pv = pv->pv_next) {
		if ((pv->pv_flags & PV_NC) != 0) {
			*pteprotop |= PG_NC;
#ifdef DEBUG
			/* Check currently illegal condition */
			if (nc == 0)
				printf("pv_link: proc %s, va=0x%lx: "
				"unexpected uncached mapping at 0x%lx\n",
				    curproc ? curproc->p_comm : "--",
				    va, pv->pv_va);
#endif
		}
		if (BADALIAS(va, pv->pv_va)) {
			DPRINTF(PDB_CACHESTUFF,
			    "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p",
			    curproc ? curproc->p_comm : "--",
			    va, pv->pv_va, pg);
			/* Mark list head `uncached due to aliases' */
			pv0->pv_flags |= PV_ANC;
			pv_changepte4_4c(pg, PG_NC, 0);
			*pteprotop |= PG_NC;
			break;
		}
	}

link_npv:
	npv->pv_next = pv0->pv_next;
	npv->pv_pmap = pm;
	npv->pv_va = va;
	npv->pv_flags = nc;
	pv0->pv_next = npv;
	return (0);
}

#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)	/* SRMMU versions of above */
/*
 * Walk the given pv list, and for each PTE, set or clear some bits
 * (e.g., PG_W or PG_NC).
 *
 * This routine flushes the cache for any page whose PTE changes,
 * as long as the process has a context; this is overly conservative.
 * It also copies ref and mod bits to the pvlist, on the theory that
 * this might save work later.  (XXX should test this theory)
 *
 * Called with PV lock and pmap main lock held.
 */
void
pv_changepte4m(struct vm_page *pg, int bis, int bic)
{
	struct pvlist *pv;
	struct pmap *pm;
	vaddr_t va;
	struct regmap *rp;
	struct segmap *sp;

	pv = VM_MDPAGE_PVHEAD(pg);
	if (pv->pv_pmap == NULL)
		return;

	for (; pv != NULL; pv = pv->pv_next) {
		pm = pv->pv_pmap;
		/* XXXSMP: should lock pm */
		va = pv->pv_va;
		rp = &pm->pm_regmap[VA_VREG(va)];
		sp = &rp->rg_segmap[VA_VSEG(va)];

		if (pm->pm_ctx) {
			/*
			 * XXX: always flush cache; conservative, but
			 * needed to invalidate cache tag protection
			 * bits and when disabling caching.
			 */
			cache_flush_page(va, pm->pm_ctxnum);
		}

		KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) ==
			SRMMU_TEPTE);
		VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(updatepte4m(va,
		    &sp->sg_pte[VA_SUN4M_VPG(va)], bic, bis, pm->pm_ctxnum,
		    PMAP_CPUSET(pm)));
	}
}

/*
 * Sync ref and mod bits in pvlist. If page has been ref'd or modified,
 * update ref/mod bits in pvlist, and clear the hardware bits.
 *
 * Return the new flags.
 */
int
pv_syncflags4m(struct vm_page *pg)
{
	struct pvlist *pv;
	struct pmap *pm;
	int va, flags;
	int s;
	struct regmap *rp;
	struct segmap *sp;
	int tpte;

	s = splvm();
	PMAP_LOCK();
	pv = VM_MDPAGE_PVHEAD(pg);
	if (pv->pv_pmap == NULL) {
		/* Page not mapped; pv_flags is already up to date */
		flags = 0;
		goto out;
	}

	flags = pv->pv_flags;
	for (; pv != NULL; pv = pv->pv_next) {
		pm = pv->pv_pmap;
		va = pv->pv_va;
		rp = &pm->pm_regmap[VA_VREG(va)];
		sp = &rp->rg_segmap[VA_VSEG(va)];

		tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
		if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE &&
		    (tpte & (SRMMU_PG_R|SRMMU_PG_M)) != 0) {
			/*
			 * Flush cache if modified to make sure the PTE
			 * M bit will be set again on the next write access.
			 */
			if (pm->pm_ctx && (tpte & SRMMU_PG_M) == SRMMU_PG_M)
				cache_flush_page(va, pm->pm_ctxnum);

			flags |= MR4M(updatepte4m(va,
					&sp->sg_pte[VA_SUN4M_VPG(va)],
					SRMMU_PG_M | SRMMU_PG_R,
					0, pm->pm_ctxnum, PMAP_CPUSET(pm)));
		}
	}

	VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
out:
	PMAP_UNLOCK();
	splx(s);
	return (flags);
}

/*
 * Should be called with pmap already locked.
 */
void
pv_unlink4m(struct vm_page *pg, struct pmap *pm, vaddr_t va)
{
	struct pvlist *pv0, *npv;

	pv0 = VM_MDPAGE_PVHEAD(pg);

	npv = pv0->pv_next;
	/*
	 * First entry is special (sigh).
	 */
	if (pv0->pv_pmap == pm && pv0->pv_va == va) {
		pmap_stats.ps_unlink_pvfirst++;
		if (npv != NULL) {
			/*
			 * Shift next entry into the head.
			 * Make sure to retain the REF, MOD and ANC flags
			 * on the list head.
			 */
			pv0->pv_next = npv->pv_next;
			pv0->pv_pmap = npv->pv_pmap;
			pv0->pv_va = npv->pv_va;
			pv0->pv_flags &= ~PV_NC;
			pv0->pv_flags |= (npv->pv_flags & PV_NC);
			pool_put(&pv_pool, npv);
		} else {
			/*
			 * No mappings left; we need to maintain
			 * the REF and MOD flags, since pmap_is_modified()
			 * can still be called for this page.
			 */
			pv0->pv_pmap = NULL;
			pv0->pv_flags &= ~(PV_NC|PV_ANC);
			return;
		}
	} else {
		struct pvlist *prev;

		pmap_stats.ps_unlink_pvsearch++;
		for (prev = pv0;; prev = npv, npv = npv->pv_next) {
			if (npv == NULL) {
				panic("pv_unlink: pm %p is missing on pg %p",
					pm, pg);
				return;
			}
			if (npv->pv_pmap == pm && npv->pv_va == va)
				break;
		}
		prev->pv_next = npv->pv_next;
		pool_put(&pv_pool, npv);
	}

	if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) {

		/*
		 * Not cached: check whether we can fix that now.
		 */
		va = pv0->pv_va;
		for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next)
			if (BADALIAS(va, npv->pv_va) ||
			    (npv->pv_flags & PV_NC) != 0)
				return;
		DPRINTF(PDB_CACHESTUFF,
		    "pv_unlink: alias ok: proc %s, va 0x%lx, pg %p",
		    curproc ? curproc->p_comm : "--", va, pg);
		pv0->pv_flags &= ~PV_ANC;
		pv_changepte4m(pg, SRMMU_PG_C, 0);
	}
}

/*
 * pv_link is the inverse of pv_unlink, and is used in pmap_enter.
 * May turn off the cacheable bit in the pte prototype for the new mapping.
 * Called with pm locked.
 */
/*static*/ int
pv_link4m(struct vm_page *pg, struct pmap *pm, vaddr_t va,
	  unsigned int *pteprotop)
{
	struct pvlist *pv0, *pv, *npv;
	int nc = (*pteprotop & SRMMU_PG_C) == 0 ? PV_NC : 0;
	int error = 0;

	pv0 = VM_MDPAGE_PVHEAD(pg);

	if (pv0->pv_pmap == NULL) {
		/* no pvlist entries yet */
		pmap_stats.ps_enter_firstpv++;
		pv0->pv_next = NULL;
		pv0->pv_pmap = pm;
		pv0->pv_va = va;
		pv0->pv_flags |= nc;
		goto out;
	}

	/*
	 * Allocate the new PV entry now, and, if that fails, bail out
	 * before changing the cacheable state of the existing mappings.
	 */
	npv = pool_get(&pv_pool, PR_NOWAIT);
	if (npv == NULL) {
		error = ENOMEM;
		goto out;
	}

	pmap_stats.ps_enter_secondpv++;

	/*
	 * See if the new mapping will cause old mappings to
	 * become aliased and thus need to be `discached'.
	 */
	if ((pv0->pv_flags & PV_ANC) != 0) {
		/* already uncached, just stay that way */
		*pteprotop &= ~SRMMU_PG_C;
		goto link_npv;
	}

	for (pv = pv0; pv != NULL; pv = pv->pv_next) {
		if ((pv->pv_flags & PV_NC) != 0) {
			*pteprotop &= ~SRMMU_PG_C;
#ifdef DEBUG
			/* Check currently illegal condition */
			if (nc == 0)
				printf("pv_link: proc %s, va=0x%lx: "
				"unexpected uncached mapping at 0x%lx\n",
				    curproc ? curproc->p_comm : "--",
				    va, pv->pv_va);
#endif
		}
		if (BADALIAS(va, pv->pv_va)) {
			DPRINTF(PDB_CACHESTUFF,
			    "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p",
				curproc ? curproc->p_comm : "--",
				va, pv->pv_va, pg);
			/* Mark list head `uncached due to aliases' */
			pv0->pv_flags |= PV_ANC;
			pv_changepte4m(pg, 0, SRMMU_PG_C);
			*pteprotop &= ~SRMMU_PG_C;
			break;
		}
	}

link_npv:
	/* Now link in the new PV entry */
	npv->pv_next = pv0->pv_next;
	npv->pv_pmap = pm;
	npv->pv_va = va;
	npv->pv_flags = nc;
	pv0->pv_next = npv;

out:
	return (error);
}
#endif

/*
 * Uncache all entries on behalf of kvm_uncache(). In addition to
 * removing the cache bit from the PTE, we are also setting PV_NC
 * in each entry to stop pv_unlink() from re-caching (i.e. when a
 * a bad alias is going away).
 */
static void
pv_uncache(struct vm_page *pg)
{
	struct pvlist *pv;
	int s;

	s = splvm();
	PMAP_LOCK();

	for (pv = VM_MDPAGE_PVHEAD(pg); pv != NULL; pv = pv->pv_next)
		pv->pv_flags |= PV_NC;

#if defined(SUN4M) || defined(SUN4D)
	if (CPU_HAS_SRMMU)
		pv_changepte4m(pg, 0, SRMMU_PG_C);
#endif
#if defined(SUN4) || defined(SUN4C)
	if (CPU_HAS_SUNMMU)
		pv_changepte4_4c(pg, PG_NC, 0);
#endif
	PMAP_UNLOCK();
	splx(s);
}

/*
 * Walk the given list and flush the cache for each (MI) page that is
 * potentially in the cache. Called only if vactype != VAC_NONE.
 */
#if defined(SUN4) || defined(SUN4C)
static void
pv_flushcache4_4c(struct vm_page *pg)
{
	struct pvlist *pv;
	struct pmap *pm;
	int s, ctx;

	pv = VM_MDPAGE_PVHEAD(pg);

	write_user_windows();	/* paranoia? */
	s = splvm();		/* XXX extreme paranoia */
	if ((pm = pv->pv_pmap) != NULL) {
		ctx = getcontext4();
		for (;;) {
			if (pm->pm_ctx) {
				setcontext4(pm->pm_ctxnum);
				cache_flush_page(pv->pv_va, pm->pm_ctxnum);
			}
			pv = pv->pv_next;
			if (pv == NULL)
				break;
			pm = pv->pv_pmap;
		}
		setcontext4(ctx);
	}
	splx(s);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)
static void
pv_flushcache4m(struct vm_page *pg)
{
	struct pvlist *pv;
	struct pmap *pm;
	int s;

	pv = VM_MDPAGE_PVHEAD(pg);

	s = splvm();		/* XXX extreme paranoia */
	if ((pm = pv->pv_pmap) != NULL) {
		for (;;) {
			if (pm->pm_ctx) {
				cache_flush_page(pv->pv_va, pm->pm_ctxnum);
			}
			pv = pv->pv_next;
			if (pv == NULL)
				break;
			pm = pv->pv_pmap;
		}
	}
	splx(s);
}
#endif /* SUN4M || SUN4D */

/*----------------------------------------------------------------*/

/*
 * At last, pmap code.
 */

#if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D))
int nptesg;
#endif

#if defined(SUN4M) || defined(SUN4D)
static void pmap_bootstrap4m(void *);
#endif
#if defined(SUN4) || defined(SUN4C)
static void pmap_bootstrap4_4c(void *, int, int, int);
#endif

/*
 * Bootstrap the system enough to run with VM enabled.
 *
 * nsegment is the number of mmu segment entries (``PMEGs'');
 * nregion is the number of mmu region entries (``SMEGs'');
 * nctx is the number of contexts.
 */
void
pmap_bootstrap(int nctx, int nregion, int nsegment)
{
	void *p;

	uvmexp.pagesize = NBPG;
	uvm_md_init();

#if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D))
	/* In this case NPTESG is a variable */
	nptesg = (NBPSG >> pgshift);
#endif

	/*
	 * Grab physical memory list.
	 */
	p = kernel_top;
	get_phys_mem(&p);

	/*
	 * The data segment in sparc ELF images is aligned to a 64KB
	 * (the maximum page size defined by the ELF/sparc ABI) boundary.
	 * This results in a unused portion of physical memory in between
	 * the text/rodata and the data segment. We pick up that gap
	 * here to remove it from the kernel map and give it to the
	 * VM manager later.
	 */
	etext_gap_start = (vaddr_t)(etext + NBPG - 1) & ~PGOFSET;
	etext_gap_end = (vaddr_t)kernel_data_start & ~PGOFSET;

	if (CPU_HAS_SRMMU) {
#if defined(SUN4M) || defined(SUN4D)
		pmap_bootstrap4m(p);
#endif
	} else if (CPU_HAS_SUNMMU) {
#if defined(SUN4) || defined(SUN4C)
		pmap_bootstrap4_4c(p, nctx, nregion, nsegment);
#endif
	}

	pmap_page_upload();
	mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_NONE);
	mutex_init(&demap_lock, MUTEX_DEFAULT, IPL_VM);
	mutex_init(&ctx_lock, MUTEX_DEFAULT, IPL_SCHED);
	lock_available = true;
}

#if defined(SUN4) || defined(SUN4C)
void
pmap_bootstrap4_4c(void *top, int nctx, int nregion, int nsegment)
{
	union ctxinfo *ci;
	struct mmuentry *mmuseg;
#if defined(SUN4_MMU3L)
	struct mmuentry *mmureg;
#endif
	struct regmap *rp;
	struct segmap *sp;
	int i, j;
	int npte, zseg, vr, vs;
	int startscookie, scookie;
#if defined(SUN4_MMU3L)
	int startrcookie = 0, rcookie = 0;
#endif
	int *kptes;
	int lastpage;
	vaddr_t va;
	vaddr_t p;

	/*
	 * Compute `va2pa_offset'.
	 * Use `kernel_text' to probe the MMU translation since
	 * the pages at KERNBASE might not be mapped.
	 */
	va2pa_offset = (vaddr_t)kernel_text -
			((getpte4(kernel_text) & PG_PFNUM) << PGSHIFT);

	ncontext = nctx;

	switch (cputyp) {
	case CPU_SUN4C:
		mmu_has_hole = 1;
		break;
	case CPU_SUN4:
		if (cpuinfo.cpu_type != CPUTYP_4_400) {
			mmu_has_hole = 1;
			break;
		}
	}

#if defined(SUN4)
	/*
	 * set up the segfixmask to mask off invalid bits
	 */
	segfixmask =  nsegment - 1; /* assume nsegment is a power of 2 */
#ifdef DIAGNOSTIC
	if (((nsegment & segfixmask) | (nsegment & ~segfixmask)) != nsegment) {
		printf("pmap_bootstrap: unsuitable number of segments (%d)\n",
			nsegment);
		callrom();
	}
#endif
#endif

#if defined(SUN4M) || defined(SUN4D) /* We're in a dual-arch kernel.
					Setup 4/4c fn. ptrs */
	pmap_clear_modify_p 	=	pmap_clear_modify4_4c;
	pmap_clear_reference_p 	= 	pmap_clear_reference4_4c;
	pmap_enter_p 		=	pmap_enter4_4c;
	pmap_extract_p 		=	pmap_extract4_4c;
	pmap_is_modified_p 	=	pmap_is_modified4_4c;
	pmap_is_referenced_p	=	pmap_is_referenced4_4c;
	pmap_kenter_pa_p 	=	pmap_kenter_pa4_4c;
	pmap_kremove_p	 	=	pmap_kremove4_4c;
	pmap_kprotect_p	 	=	pmap_kprotect4_4c;
	pmap_page_protect_p	=	pmap_page_protect4_4c;
	pmap_protect_p		=	pmap_protect4_4c;
	pmap_rmk_p		=	pmap_rmk4_4c;
	pmap_rmu_p		=	pmap_rmu4_4c;
#endif /* defined SUN4M || defined SUN4D */

	p = (vaddr_t)top;

	/*
	 * Last segment is the `invalid' one (one PMEG of pte's with !pg_v).
	 * It will never be used for anything else.
	 */
	seginval = --nsegment;

#if defined(SUN4_MMU3L)
	if (HASSUN4_MMU3L)
		reginval = --nregion;
#endif

	/*
	 * Allocate and initialise mmu entries and context structures.
	 */
#if defined(SUN4_MMU3L)
	mmuregions = mmureg = (struct mmuentry *)p;
	p += nregion * sizeof(struct mmuentry);
	memset(mmuregions, 0, nregion * sizeof(struct mmuentry));
#endif
	mmusegments = mmuseg = (struct mmuentry *)p;
	p += nsegment * sizeof(struct mmuentry);
	memset(mmusegments, 0, nsegment * sizeof(struct mmuentry));

	pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
	p += nctx * sizeof *ci;

	/* Initialize MMU resource queues */
#if defined(SUN4_MMU3L)
	mmuq_init(&region_freelist);
	mmuq_init(&region_lru);
	mmuq_init(&region_locked);
#endif
	mmuq_init(&segm_freelist);
	mmuq_init(&segm_lru);
	mmuq_init(&segm_locked);


	/*
	 * Initialize the kernel pmap.
	 */
	/* kernel_pmap_store.pm_ctxnum = 0; */
	kernel_pmap_store.pm_refcount = 1;
#if defined(SUN4_MMU3L)
	TAILQ_INIT(&kernel_pmap_store.pm_reglist);
#endif
	TAILQ_INIT(&kernel_pmap_store.pm_seglist);

	/*
	 * Allocate memory for kernel PTEs
	 * XXX Consider allocating memory for only a few regions
	 * and use growkernel() to allocate more as needed.
	 */
	kptes = (int *)p;
	p += NKREG * NSEGRG * NPTESG * sizeof(int);
	memset(kptes, 0, NKREG * NSEGRG * NPTESG * sizeof(int));

	/*
	 * Set up pm_regmap for kernel to point NUREG *below* the beginning
	 * of kernel regmap storage. Since the kernel only uses regions
	 * above NUREG, we save storage space and can index kernel and
	 * user regions in the same way.
	 */
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Warray-bounds"
	kernel_pmap_store.pm_regmap = kernel_regmap_store - NUREG;
#pragma GCC diagnostic pop
	for (i = NKREG; --i >= 0;) {
#if defined(SUN4_MMU3L)
		kernel_regmap_store[i].rg_smeg = reginval;
#endif
		kernel_regmap_store[i].rg_segmap =
			&kernel_segmap_store[i * NSEGRG];
		for (j = NSEGRG; --j >= 0;) {
			sp = &kernel_segmap_store[i * NSEGRG + j];
			sp->sg_pmeg = seginval;
			sp->sg_pte = &kptes[(i * NSEGRG + j) * NPTESG];
		}
	}

	/*
	 * Preserve the monitor ROM's reserved VM region, so that
	 * we can use L1-A or the monitor's debugger.  As a side
	 * effect we map the ROM's reserved VM into all contexts
	 * (otherwise L1-A crashes the machine!).
	 */

	mmu_reservemon4_4c(&nregion, &nsegment);

#if defined(SUN4_MMU3L)
	/* Reserve one region for temporary mappings */
	if (HASSUN4_MMU3L)
		tregion = --nregion;
#endif

	/*
	 * Set up the `constants' for the call to vm_init()
	 * in main().  All pages beginning at p (rounded up to
	 * the next whole page) and continuing through the number
	 * of available pages are free, but they start at a higher
	 * virtual address.  This gives us two mappable MD pages
	 * for pmap_zero_page and pmap_copy_page, and one MI page
	 * for /dev/mem, all with no associated physical memory.
	 */
	p = (p + NBPG - 1) & ~PGOFSET;

	avail_start = PMAP_BOOTSTRAP_VA2PA(p);

	i = p;
	cpuinfo.vpage[0] = (void *)p, p += NBPG;
	cpuinfo.vpage[1] = (void *)p, p += NBPG;
	p = (vaddr_t)reserve_dumppages((void *)p);

	virtual_avail = p;
	virtual_end = VM_MAX_KERNEL_ADDRESS;

	p = i;			/* retract to first free phys */


	/*
	 * All contexts are free except the kernel's.
	 *
	 * XXX sun4c could use context 0 for users?
	 */
	ci->c_pmap = pmap_kernel();
	ctx_freelist = ci + 1;
	for (i = 1; i < ncontext; i++) {
		ci++;
		ci->c_nextfree = ci + 1;
	}
	ci->c_nextfree = NULL;
	ctx_kick = 0;
	ctx_kickdir = -1;

	/*
	 * Init mmu entries that map the kernel physical addresses.
	 *
	 * All the other MMU entries are free.
	 *
	 * THIS ASSUMES THE KERNEL IS MAPPED BY A CONTIGUOUS RANGE OF
	 * MMU SEGMENTS/REGIONS DURING THE BOOT PROCESS
	 */

	/* Compute the number of segments used by the kernel */
	zseg = (((p + NBPSG - 1) & ~SGOFSET) - KERNBASE) >> SGSHIFT;
	lastpage = VA_VPG(p);
	if (lastpage == 0)
		/*
		 * If the page bits in p are 0, we filled the last segment
		 * exactly; if not, it is the last page filled in the
		 * last segment.
		 */
		lastpage = NPTESG;

	p = KERNBASE;			/* first va */
	vs = VA_VSEG(KERNBASE);		/* first virtual segment */
	vr = VA_VREG(KERNBASE);		/* first virtual region */
	rp = &pmap_kernel()->pm_regmap[vr];

	/* Get region/segment where kernel addresses start */
#if defined(SUN4_MMU3L)
	if (HASSUN4_MMU3L)
		startrcookie = rcookie = getregmap(p);
	mmureg = &mmuregions[rcookie];
#endif

	startscookie = scookie = getsegmap(p);
	mmuseg = &mmusegments[scookie];
	zseg += scookie;	/* First free segment */

	for (;;) {

		/*
		 * Distribute each kernel region/segment into all contexts.
		 * This is done through the monitor ROM, rather than
		 * directly here: if we do a setcontext we will fault,
		 * as we are not (yet) mapped in any other context.
		 */

		if ((vs % NSEGRG) == 0) {
			/* Entering a new region */
			if (VA_VREG(p) > vr) {
#ifdef DEBUG
				printf("note: giant kernel!\n");
#endif
				vr++, rp++;
			}
#if defined(SUN4_MMU3L)
			if (HASSUN4_MMU3L) {
				for (i = 1; i < nctx; i++)
					prom_setcontext(i, (void *)p, rcookie);

				mmuq_insert_tail(&region_locked,
						  mmureg);
				TAILQ_INSERT_TAIL(&pmap_kernel()->pm_reglist,
						  mmureg, me_pmchain);
#ifdef DIAGNOSTIC
				mmuseg->me_statp = NULL;
#endif
				mmureg->me_cookie = rcookie;
				mmureg->me_pmap = pmap_kernel();
				mmureg->me_vreg = vr;
				rp->rg_smeg = rcookie;
				mmureg++;
				rcookie++;
			}
#endif /* SUN4_MMU3L */
		}

#if defined(SUN4_MMU3L)
		if (!HASSUN4_MMU3L)
#endif
			for (i = 1; i < nctx; i++)
				prom_setcontext(i, (void *)p, scookie);

		/* set up the mmu entry */
		mmuq_insert_tail(&segm_locked, mmuseg);
#ifdef DIAGNOSTIC
		mmuseg->me_statp = &pmap_stats.ps_npmeg_locked;
#endif
		TAILQ_INSERT_TAIL(&pmap_kernel()->pm_seglist, mmuseg, me_pmchain);
		pmap_stats.ps_npmeg_locked++;
		mmuseg->me_cookie = scookie;
		mmuseg->me_pmap = pmap_kernel();
		mmuseg->me_vreg = vr;
		mmuseg->me_vseg = vs % NSEGRG;
		sp = &rp->rg_segmap[vs % NSEGRG];
		sp->sg_pmeg = scookie;
		npte = ++scookie < zseg ? NPTESG : lastpage;
		sp->sg_npte = npte;
		sp->sg_nwired = npte;
		pmap_kernel()->pm_stats.resident_count += npte;
		rp->rg_nsegmap += 1;
		for (i = 0; i < npte; i++)
			sp->sg_pte[i] = getpte4(p + i * NBPG) | PG_WIRED;
		mmuseg++;
		vs++;
		if (scookie < zseg) {
			p += NBPSG;
			continue;
		}

		/*
		 * Unmap the pages, if any, that are not part of
		 * the final segment.
		 */
		for (p += npte << PGSHIFT; npte < NPTESG; npte++, p += NBPG)
			setpte4(p, 0);

#if defined(SUN4_MMU3L)
		if (HASSUN4_MMU3L) {
			/*
			 * Unmap the segments, if any, that are not part of
			 * the final region.
			 */
			for (i = rp->rg_nsegmap; i < NSEGRG; i++, p += NBPSG)
				setsegmap(p, seginval);

			/*
			 * Unmap any kernel regions that we aren't using.
			 */
			for (i = 0; i < nctx; i++) {
				setcontext4(i);
				for (va = p;
				     va < (OPENPROM_STARTVADDR & ~(NBPRG - 1));
				     va += NBPRG)
					setregmap(va, reginval);
			}

		} else
#endif
		{
			/*
			 * Unmap any kernel segments that we aren't using.
			 */
			for (i = 0; i < nctx; i++) {
				setcontext4(i);
				for (va = p;
				     va < (OPENPROM_STARTVADDR & ~(NBPSG - 1));
				     va += NBPSG)
					setsegmap(va, seginval);
			}
		}
		break;
	}

#if defined(SUN4_MMU3L)
	if (HASSUN4_MMU3L)
		for (rcookie = 0; rcookie < nregion; rcookie++) {
			if (rcookie == startrcookie)
				/* Kernel must fit in one region! */
				rcookie++;
			mmureg = &mmuregions[rcookie];
			mmureg->me_cookie = rcookie;
			mmuq_insert_tail(&region_freelist, mmureg);
#ifdef DIAGNOSTIC
			mmuseg->me_statp = NULL;
#endif
		}
#endif /* SUN4_MMU3L */

	for (scookie = 0; scookie < nsegment; scookie++) {
		if (scookie == startscookie)
			/* Skip static kernel image */
			scookie = zseg;
		mmuseg = &mmusegments[scookie];
		mmuseg->me_cookie = scookie;
		mmuq_insert_tail(&segm_freelist, mmuseg);
		pmap_stats.ps_npmeg_free++;
#ifdef DIAGNOSTIC
		mmuseg->me_statp = NULL;
#endif
	}

	/* Erase all spurious user-space segmaps */
	for (i = 1; i < ncontext; i++) {
		setcontext4(i);
		if (HASSUN4_MMU3L)
			for (p = 0, j = NUREG; --j >= 0; p += NBPRG)
				setregmap(p, reginval);
		else
			for (p = 0, vr = 0; vr < NUREG; vr++) {
				if (VA_INHOLE(p)) {
					p = MMU_HOLE_END;
					vr = VA_VREG(p);
				}
				for (j = NSEGRG; --j >= 0; p += NBPSG)
					setsegmap(p, seginval);
			}
	}
	setcontext4(0);

	/*
	 * write protect & encache kernel text;
	 * set red zone at kernel base;
	 * enable cache on message buffer and cpuinfo.
	 */

	/* Enable cache on message buffer and cpuinfo */
	for (p = KERNBASE; p < (vaddr_t)trapbase; p += NBPG)
		setpte4(p, getpte4(p) & ~PG_NC);

	/* Enable cache and write protext kernel text */
	for (p = (vaddr_t)trapbase; p < (vaddr_t)etext; p += NBPG)
		setpte4(p, getpte4(p) & ~(PG_NC|PG_W));

	/*
	 * Unmap the `etext gap'; it'll be made available
	 * to the VM manager.
	 */
	for (p = etext_gap_start; p < etext_gap_end; p += NBPG) {
		rp = &pmap_kernel()->pm_regmap[VA_VREG(p)];
		sp = &rp->rg_segmap[VA_VSEG(p)];
		sp->sg_nwired--;
		sp->sg_npte--;
		pmap_kernel()->pm_stats.resident_count--;
		sp->sg_pte[VA_VPG(p)] = 0;
		setpte4(p, 0);
	}

	/* Enable cache on data & bss */
	for (p = etext_gap_end; p < virtual_avail; p += NBPG)
		setpte4(p, getpte4(p) & ~PG_NC);

	cpus[0] = (struct cpu_info *)CPUINFO_VA;
}
#endif

#if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of pmap_bootstrap */
/*
 * Bootstrap the system enough to run with VM enabled on a sun4m machine.
 *
 * Switches from ROM to kernel page tables, and sets up initial mappings.
 */
static void
pmap_bootstrap4m(void *top)
{
	int i, j;
	vaddr_t p, q;
	union ctxinfo *ci;
	int reg, seg;
	unsigned int ctxtblsize;
	vaddr_t pagetables_start, pagetables_end;
	paddr_t pagetables_start_pa;
	vaddr_t va;
#if defined(MULTIPROCESSOR)
	vsize_t off;
	size_t cpuinfo_len = sizeof(struct cpu_info);
	uint8_t *cpuinfo_data;
	int align = PAGE_SIZE;
	vaddr_t sva, cpuinfo_va;
	vsize_t sz;
#endif

	/*
	 * Compute `va2pa_offset'.
	 * Use `kernel_text' to probe the MMU translation since
	 * the pages at KERNBASE might not be mapped.
	 */
	va2pa_offset = (vaddr_t)kernel_text - VA2PA(kernel_text);

	ncontext = cpuinfo.mmu_ncontext;

#if defined(SUN4) || defined(SUN4C) /* setup SRMMU fn. ptrs for dual-arch
				       kernel */
	pmap_clear_modify_p 	=	pmap_clear_modify4m;
	pmap_clear_reference_p 	= 	pmap_clear_reference4m;
	pmap_enter_p 		=	pmap_enter4m;
	pmap_extract_p 		=	pmap_extract4m;
	pmap_is_modified_p 	=	pmap_is_modified4m;
	pmap_is_referenced_p	=	pmap_is_referenced4m;
	pmap_kenter_pa_p 	=	pmap_kenter_pa4m;
	pmap_kremove_p	 	=	pmap_kremove4m;
	pmap_kprotect_p	 	=	pmap_kprotect4m;
	pmap_page_protect_p	=	pmap_page_protect4m;
	pmap_protect_p		=	pmap_protect4m;
	pmap_rmk_p		=	pmap_rmk4m;
	pmap_rmu_p		=	pmap_rmu4m;
#endif /* defined SUN4/SUN4C */

	/*
	 * p points to top of kernel mem
	 */
	p = (vaddr_t)top;

	p = (p + NBPG - 1) & ~PGOFSET;
	DPRINTF(PDB_INITLOUD, "initial p=%lx", p);

	/*
	 * Initialize the kernel pmap.
	 */
	/* kernel_pmap_store.pm_ctxnum = 0; */
	kernel_pmap_store.pm_refcount = 1;

	/*
	 * Set up pm_regmap for kernel to point NUREG *below* the beginning
	 * of kernel regmap storage. Since the kernel only uses regions
	 * above NUREG, we save storage space and can index kernel and
	 * user regions in the same way.
	 */
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Warray-bounds"
	kernel_pmap_store.pm_regmap = kernel_regmap_store - NUREG;
#pragma GCC diagnostic pop
	memset(kernel_regmap_store, 0, sizeof kernel_regmap_store);
	memset(kernel_segmap_store, 0, sizeof kernel_segmap_store);
	for (i = NKREG; --i >= 0;) {
		kernel_regmap_store[i].rg_segmap =
			&kernel_segmap_store[i * NSEGRG];
		kernel_regmap_store[i].rg_seg_ptps = NULL;
		for (j = NSEGRG; --j >= 0;)
			kernel_segmap_store[i * NSEGRG + j].sg_pte = NULL;
	}

	/* Allocate kernel region pointer tables */
	pmap_kernel()->pm_reg_ptps = (int **)(q = p);
	DPRINTF(PDB_INITLOUD, "kernel region pointer tables p=%lx", p);
	p += sparc_ncpus * sizeof(int **);
	memset((void *)q, 0, (u_int)p - (u_int)q);

	pmap_kernel()->pm_reg_ptps_pa = (int *)(q = p);
	DPRINTF(PDB_INITLOUD, "kernel region pointer tables pa p=%lx", p);
	p += sparc_ncpus * sizeof(int *);
	memset((void *)q, 0, (u_int)p - (u_int)q);

	/* Allocate context administration */
	pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
	DPRINTF(PDB_INITLOUD, "context administration p=%lx", p);
	p += ncontext * sizeof *ci;
	memset((void *)ci, 0, (u_int)p - (u_int)ci);

	/*
	 * Set up the `constants' for the call to vm_init()
	 * in main().  All pages beginning at p (rounded up to
	 * the next whole page) and continuing through the number
	 * of available pages are free.
	 */
	p = (p + NBPG - 1) & ~PGOFSET;
	DPRINTF(PDB_INITLOUD, "align p=%lx", p);

	/*
	 * Reserve memory for MMU pagetables. Some of these have severe
	 * alignment restrictions. We allocate in a sequence that
	 * minimizes alignment gaps.
	 */

	pagetables_start = p;
	pagetables_start_pa = PMAP_BOOTSTRAP_VA2PA(p);

	/*
	 * Allocate context table.
	 * To keep supersparc happy, minimum alignment is on a 4K boundary.
	 */
	ctxtblsize = uimax(ncontext,1024) * sizeof(int);
	cpuinfo.ctx_tbl = (int *)roundup((u_int)p, ctxtblsize);
	cpuinfo.ctx_tbl_pa = PMAP_BOOTSTRAP_VA2PA(cpuinfo.ctx_tbl);
	p = (u_int)cpuinfo.ctx_tbl + ctxtblsize;
	DPRINTF(PDB_INITLOUD, "post ctx table p=%lx", p);

#if defined(MULTIPROCESSOR)
	/*
	 * Make sure all smp_tlb_flush*() routines for kernel pmap are
	 * broadcast to all CPU's.
	 */
	pmap_kernel()->pm_cpuset = CPUSET_ALL;
#endif

	/*
	 * Reserve memory for segment and page tables needed to map the entire
	 * kernel. This takes (2K + NKREG * 16K) of space, but unfortunately
	 * is necessary since pmap_enter() *must* be able to enter a kernel
	 * mapping without delay.
	 */
	p = (vaddr_t) roundup(p, SRMMU_L1SIZE * sizeof(u_int));
	DPRINTF(PDB_INITLOUD, "roundup kernel_regtable_store p=%lx", p);
	qzero((void *)p, SRMMU_L1SIZE * sizeof(u_int));
	kernel_regtable_store = (u_int *)p;
	p += SRMMU_L1SIZE * sizeof(u_int);
	DPRINTF(PDB_INITLOUD, "L1 pages p=%lx", p);

	p = (vaddr_t) roundup(p, SRMMU_L2SIZE * sizeof(u_int));
	DPRINTF(PDB_INITLOUD, "roundup kernel_segtable_store p=%lx", p);
	qzero((void *)p, (SRMMU_L2SIZE * sizeof(u_int)) * NKREG);
	kernel_segtable_store = (u_int *)p;
	p += (SRMMU_L2SIZE * sizeof(u_int)) * NKREG;
	DPRINTF(PDB_INITLOUD, "L2 pages p=%lx", p);

	p = (vaddr_t) roundup(p, SRMMU_L3SIZE * sizeof(u_int));
	DPRINTF(PDB_INITLOUD, "roundup kernel_pagtable_store p=%lx", p);
	/* zero it: all will be SRMMU_TEINVALID */
	qzero((void *)p, ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG);
	kernel_pagtable_store = (u_int *)p;
	p += ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG;
	DPRINTF(PDB_INITLOUD, "L3 pages p=%lx", p);

	/* Round to next page and mark end of pre-wired kernel space */
	p = (p + NBPG - 1) & ~PGOFSET;
	DPRINTF(PDB_INITLOUD, "align p=%lx", p);
	pagetables_end = p;

#if defined(MULTIPROCESSOR)
	/*
	 * Allocate aligned KVA.  `cpuinfo' resides at a fixed virtual
	 * address. Since we need to access an other CPU's cpuinfo
	 * structure occasionally, this must be done at a virtual address
	 * that's cache congruent to the fixed address CPUINFO_VA.
	 *
	 * NOTE: we're using the cache properties of the boot CPU to
	 * determine the alignment (XXX).
	 */
	sz = sizeof(struct cpu_info);
	if (sparc_ncpus > 1) {
		if (CACHEINFO.c_totalsize > align) {
			/* Need a power of two */
			while (align <= CACHEINFO.c_totalsize)
				align <<= 1;
			align >>= 1;
		}

		sz = (sz + PAGE_SIZE - 1) & -PAGE_SIZE;
		cpuinfo_len = sz + align - PAGE_SIZE;

		/* Grab as much space as we need */
		DPRINTF(PDB_INITLOUD, "cpuinfo=%lx", p);
		cpuinfo_data = (uint8_t *)p;
		p += (cpuinfo_len * sparc_ncpus);
	} else
		cpuinfo_data = (uint8_t *)CPUINFO_VA;
#endif

	DPRINTF(PDB_INITLOUD, "avail_start=%lx", p);
	avail_start = PMAP_BOOTSTRAP_VA2PA(p);

	/*
	 * Now wire the region and segment tables of the kernel map.
	 */
	pmap_kernel()->pm_reg_ptps[0] = (int *) kernel_regtable_store;
	pmap_kernel()->pm_reg_ptps_pa[0] =
		 PMAP_BOOTSTRAP_VA2PA(kernel_regtable_store);

	/* Install L1 table in context 0 */
	setpgt4m(&cpuinfo.ctx_tbl[0],
	    (pmap_kernel()->pm_reg_ptps_pa[0] >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);

	for (reg = 0; reg < NKREG; reg++) {
		struct regmap *rp;
		void *kphyssegtbl;

		/*
		 * Entering new region; install & build segtbl
		 */

		rp = &pmap_kernel()->pm_regmap[reg + VA_VREG(KERNBASE)];

		kphyssegtbl = (void *)
		    &kernel_segtable_store[reg * SRMMU_L2SIZE];

		setpgt4m(&pmap_kernel()->pm_reg_ptps[0][reg + VA_VREG(KERNBASE)],
		    (PMAP_BOOTSTRAP_VA2PA(kphyssegtbl) >> SRMMU_PPNPASHIFT) |
		    SRMMU_TEPTD);

		rp->rg_seg_ptps = (int *)kphyssegtbl;

		for (seg = 0; seg < NSEGRG; seg++) {
			struct segmap *sp;
			void *kphyspagtbl;

			rp->rg_nsegmap++;

			sp = &rp->rg_segmap[seg];
			kphyspagtbl = (void *)
			    &kernel_pagtable_store
				[((reg * NSEGRG) + seg) * SRMMU_L3SIZE];

			setpgt4m(&rp->rg_seg_ptps[seg],
				 (PMAP_BOOTSTRAP_VA2PA(kphyspagtbl) >> SRMMU_PPNPASHIFT) |
				 SRMMU_TEPTD);
			sp->sg_pte = (int *) kphyspagtbl;
		}
	}

	/*
	 * Preserve the monitor ROM's reserved VM region, so that
	 * we can use L1-A or the monitor's debugger.
	 */
	mmu_reservemon4m(&kernel_pmap_store);

	/*
	 * Reserve virtual address space for two mappable MD pages
	 * for pmap_zero_page and pmap_copy_page, one MI page
	 * for /dev/mem, and some more for dumpsys().
	 */
	q = p;
	cpuinfo.vpage[0] = (void *)p, p += NBPG;
	cpuinfo.vpage[1] = (void *)p, p += NBPG;
	p = (vaddr_t)reserve_dumppages((void *)p);

	/* Find PTE locations of vpage[] to optimize zero_fill() et.al. */
	for (i = 0; i < 2; i++) {
		struct regmap *rp;
		struct segmap *sp;
		rp = &pmap_kernel()->pm_regmap[VA_VREG(cpuinfo.vpage[i])];
		sp = &rp->rg_segmap[VA_VSEG(cpuinfo.vpage[i])];
		cpuinfo.vpage_pte[i] =
			&sp->sg_pte[VA_SUN4M_VPG(cpuinfo.vpage[i])];
	}

#if !(defined(PROM_AT_F0) || defined(MSIIEP))
	virtual_avail = p;
#elif defined(MSIIEP)
	virtual_avail = (vaddr_t)0xf0800000; /* Krups */
#else
	virtual_avail = (vaddr_t)0xf0080000; /* Mr.Coffee/OFW */
#endif
	virtual_end = VM_MAX_KERNEL_ADDRESS;

	p = q;			/* retract to first free phys */

	/*
	 * Set up the ctxinfo structures (freelist of contexts)
	 */
	ci->c_pmap = pmap_kernel();
	ctx_freelist = ci + 1;
	for (i = 1; i < ncontext; i++) {
		ci++;
		ci->c_nextfree = ci + 1;
	}
	ci->c_nextfree = NULL;
	ctx_kick = 0;
	ctx_kickdir = -1;

	/*
	 * Now map the kernel into our new set of page tables, then
	 * (finally) switch over to our running page tables.
	 * We map from KERNBASE to p into context 0's page tables (and
	 * the kernel pmap).
	 */
#ifdef DEBUG			/* Sanity checks */
	if (p % NBPG != 0)
		panic("pmap_bootstrap4m: p misaligned?!?");
	if (KERNBASE % NBPRG != 0)
		panic("pmap_bootstrap4m: KERNBASE not region-aligned");
#endif

	for (q = KERNBASE; q < p; q += NBPG) {
		struct regmap *rp;
		struct segmap *sp;
		int pte, *ptep;

		/*
		 * Now install entry for current page.
		 */
		rp = &pmap_kernel()->pm_regmap[VA_VREG(q)];
		sp = &rp->rg_segmap[VA_VSEG(q)];
		ptep = &sp->sg_pte[VA_VPG(q)];

		/*
		 * Unmap the `etext gap'; it'll be made available
		 * to the VM manager.
		 */
		if (q >= etext_gap_start && q < etext_gap_end) {
			setpgt4m(ptep, 0);
			continue;
		}

		pte = PMAP_BOOTSTRAP_VA2PA(q) >> SRMMU_PPNPASHIFT;
		pte |= PPROT_N_RX | SRMMU_TEPTE;

		/* Deal with the cacheable bit for pagetable memory */
		if ((CACHEINFO.c_flags & CACHE_PAGETABLES) != 0 ||
		    q < pagetables_start || q >= pagetables_end)
			pte |= SRMMU_PG_C;

		/* write-protect kernel text */
		if (q < (vaddr_t)trapbase || q >= (vaddr_t)etext)
			pte |= PPROT_WRITE;

		setpgt4m(ptep, pte);
		pmap_kernel()->pm_stats.resident_count++;
	}

	if ((CACHEINFO.c_flags & CACHE_PAGETABLES) == 0) {
		/*
		 * The page tables have been setup. Since we're still
		 * running on the PROM's memory map, the memory we
		 * allocated for our page tables might still be cached.
		 * Flush it now, and don't touch it again until we
		 * switch to our own tables (will be done immediately below).
		 */
		int size = pagetables_end - pagetables_start;
		if (CACHEINFO.c_vactype != VAC_NONE) {
			va = (vaddr_t)pagetables_start;
			while (size > 0) {
				cache_flush_page(va, 0);
				va += NBPG;
				size -= NBPG;
			}
		} else if (cpuinfo.pcache_flush_page != NULL) {
			paddr_t pa = pagetables_start_pa;
			while (size > 0) {
				pcache_flush_page(pa, 0);
				pa += NBPG;
				size -= NBPG;
			}
		}
	}

	/*
	 * Now switch to kernel pagetables (finally!)
	 */
	mmu_install_tables(&cpuinfo);

#if defined(MULTIPROCESSOR)
	/*
	 * Initialise any cpu-specific data now.
	 */
	cpu_init_system();

	/*
	 * Setup the cpus[] array and the ci_self links.
	 */
	for (i = 0; i < sparc_ncpus; i++) {
		sva = (vaddr_t) (cpuinfo_data + (cpuinfo_len * i));
		cpuinfo_va = sva +
		   (((CPUINFO_VA & (align - 1)) + align - sva) & (align - 1));

		/*
		 * Either remap from CPUINFO_VA to the new correct value
		 * or clear out this cpuinfo.
		 */
		if (i == 0) {
			for (off = 0, va = cpuinfo_va;
			     sparc_ncpus > 1 && off < sizeof(struct cpu_info);
			     va += NBPG, off += NBPG) {
				paddr_t pa =
				    PMAP_BOOTSTRAP_VA2PA(CPUINFO_VA + off);

				pmap_kremove(va, NBPG);
				pmap_kenter_pa(va, pa,
					       VM_PROT_READ | VM_PROT_WRITE, 0);
			}

		} else
			memset((void *)cpuinfo_va, 0, sizeof(struct cpu_info));

		cpus[i] = (struct cpu_info *)cpuinfo_va;
		cpus[i]->ci_self = cpus[i];

		/* Unmap and prepare to return unused pages */
		if (cpuinfo_va != sva) {
			cpus[i]->ci_free_sva1 = sva;
			cpus[i]->ci_free_eva1 = cpuinfo_va;
			for (va = cpus[i]->ci_free_sva1;
			     va < cpus[i]->ci_free_eva1;
			     va += NBPG)
				setpte4m(va, 0);
		}
		if (cpuinfo_va + sz != sva + cpuinfo_len) {
			cpus[i]->ci_free_sva2 = cpuinfo_va + sz;
			cpus[i]->ci_free_eva2 = sva + cpuinfo_len;
			for (va = cpus[i]->ci_free_sva2;
			     va < cpus[i]->ci_free_eva2;
			     va += NBPG)
				setpte4m(va, 0);
		}
	}
#else
	cpus[0] = (struct cpu_info *)CPUINFO_VA;
#endif

	pmap_update(pmap_kernel());

#ifdef DIAGNOSTIC
	if (curcpu()->ci_self != cpus[0]) {
		prom_printf("curcpu()->ci_self %p != cpus[0] %p\n", curcpu()->ci_self, cpus[0]);
		panic("cpuinfo inconsistent");
	}
#endif
}

static u_long prom_ctxreg;

void
mmu_install_tables(struct cpu_info *sc)
{

#ifdef DEBUG
	prom_printf("pmap_bootstrap: installing kernel page tables...");
#endif
	setcontext4m(0);	/* paranoia? %%%: Make 0x3 a define! below */

	/* Enable MMU tablewalk caching, flush TLB */
	if (sc->mmu_enable != 0)
		sc->mmu_enable();

	tlb_flush_all_real();
	prom_ctxreg = lda(SRMMU_CXTPTR, ASI_SRMMU);

	sta(SRMMU_CXTPTR, ASI_SRMMU,
		(sc->ctx_tbl_pa >> SRMMU_PPNPASHIFT) & ~0x3);

	tlb_flush_all_real();

#ifdef DEBUG
	prom_printf("done.\n");
#endif
}

void srmmu_restore_prom_ctx(void);

void
srmmu_restore_prom_ctx(void)
{

	tlb_flush_all();
	sta(SRMMU_CXTPTR, ASI_SRMMU, prom_ctxreg);
	tlb_flush_all();
}
#endif /* SUN4M || SUN4D */

#if defined(MULTIPROCESSOR)
/*
 * Allocate per-CPU page tables. One region, segment and page table
 * is needed to map CPUINFO_VA to different physical addresses on
 * each CPU. Since the kernel region and segment tables are all
 * pre-wired (in bootstrap() above) and we also assume that the
 * first segment (256K) of kernel space is fully populated with
 * pages from the start, these per-CPU tables will never need
 * to be updated when mapping kernel virtual memory.
 *
 * Note: this routine is called in the context of the boot CPU
 * during autoconfig.
 */
void
pmap_alloc_cpu(struct cpu_info *sc)
{
#if defined(SUN4M) || defined(SUN4D)	/* Only implemented for SUN4M/D */
	vaddr_t va;
	paddr_t pa;
	paddr_t alignment;
	u_int *ctxtable, *regtable, *segtable, *pagtable;
	u_int *ctxtable_pa, *regtable_pa, *segtable_pa, *pagtable_pa;
	psize_t ctxsize, size;
	int vr, vs, vpg;
	struct regmap *rp;
	struct segmap *sp;
	struct pglist mlist;
	int cachebit;
	int pagesz = NBPG;
	int i;

	cachebit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0;

	/*
	 * Allocate properly aligned and contiguous physically memory
	 * for the PTE tables.
	 */
	ctxsize = (sc->mmu_ncontext * sizeof(int) + pagesz - 1) & -pagesz;
	alignment = ctxsize;

	/* The region, segment and page table we need fit in one page */
	size = ctxsize + pagesz;

	if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys,
			    alignment, 0, &mlist, 1, 0) != 0)
		panic("pmap_alloc_cpu: no memory");

	pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&mlist));

	/* Allocate virtual memory */
	va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
	if (va == 0)
		panic("pmap_alloc_cpu: no memory");

	/*
	 * Layout the page tables in our chunk of memory
	 */
	ctxtable = (u_int *)va;
	regtable = (u_int *)(va + ctxsize);
	segtable = regtable + SRMMU_L1SIZE;
	pagtable = segtable + SRMMU_L2SIZE;

	ctxtable_pa = (u_int *)pa;
	regtable_pa = (u_int *)(pa + ctxsize);
	segtable_pa = regtable_pa + SRMMU_L1SIZE;
	pagtable_pa = segtable_pa + SRMMU_L2SIZE;

	/* Map the pages */
	while (size != 0) {
		pmap_kenter_pa(va, pa | (cachebit ? 0 : PMAP_NC),
		    VM_PROT_READ | VM_PROT_WRITE, 0);
		va += pagesz;
		pa += pagesz;
		size -= pagesz;
	}
	pmap_update(pmap_kernel());

	/*
	 * Store the region table pointer (and its corresponding physical
	 * address) in the CPU's slot in the kernel pmap region table
	 * pointer table.
	 */
	pmap_kernel()->pm_reg_ptps[sc->ci_cpuid] = regtable;
	pmap_kernel()->pm_reg_ptps_pa[sc->ci_cpuid] = (paddr_t)regtable_pa;

	vr = VA_VREG(CPUINFO_VA);
	vs = VA_VSEG(CPUINFO_VA);
	vpg = VA_VPG(CPUINFO_VA);
	rp = &pmap_kernel()->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];

	/*
	 * Copy page tables from CPU #0, then modify entry for CPUINFO_VA
	 * so that it points at the per-CPU pages.
	 */
	qcopy(pmap_kernel()->pm_reg_ptps[0], regtable,
		SRMMU_L1SIZE * sizeof(int));
	qcopy(rp->rg_seg_ptps, segtable, SRMMU_L2SIZE * sizeof(int));
	qcopy(sp->sg_pte, pagtable, SRMMU_L3SIZE * sizeof(int));

	setpgt4m(&ctxtable[0],
		 ((u_long)regtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
	setpgt4m(&regtable[vr],
		 ((u_long)segtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
	setpgt4m(&segtable[vs],
		 ((u_long)pagtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
	setpgt4m(&pagtable[vpg],
		(VA2PA((void *)sc) >> SRMMU_PPNPASHIFT) |
		(SRMMU_TEPTE | PPROT_N_RWX | SRMMU_PG_C));

	/* Install this CPU's context table */
	sc->ctx_tbl = ctxtable;
	sc->ctx_tbl_pa = (paddr_t)ctxtable_pa;

	/* Pre-compute this CPU's vpage[] PTEs */
	for (i = 0; i < 2; i++) {
		rp = &pmap_kernel()->pm_regmap[VA_VREG(sc->vpage[i])];
		sp = &rp->rg_segmap[VA_VSEG(sc->vpage[i])];
		sc->vpage_pte[i] = &sp->sg_pte[VA_SUN4M_VPG(sc->vpage[i])];
	}
#endif /* SUN4M || SUN4D */
}
#endif /* MULTIPROCESSOR */


void
pmap_init(void)
{
	u_int sz;

	if (PAGE_SIZE != NBPG)
		panic("pmap_init: PAGE_SIZE!=NBPG");

	vm_num_phys = vm_last_phys - vm_first_phys;

	/* Setup a pool for additional pvlist structures */
	pool_init(&pv_pool, sizeof(struct pvlist), 0, 0, 0, "pvtable", NULL,
	    IPL_NONE);

	/*
	 * Setup a pool for pmap structures.
	 * The pool size includes space for an array of per-CPU
	 * region table pointers & physical addresses
	 */
	sz = ALIGN(sizeof(struct pmap)) +
	     ALIGN(NUREG * sizeof(struct regmap)) +
	     sparc_ncpus * sizeof(int *) +	/* pm_reg_ptps */
	     sparc_ncpus * sizeof(int);		/* pm_reg_ptps_pa */
	pool_cache_bootstrap(&pmap_cache, sz, 0, 0, 0, "pmappl", NULL,
	    IPL_NONE, pmap_pmap_pool_ctor, pmap_pmap_pool_dtor, NULL);

	sz = NSEGRG * sizeof (struct segmap);
	pool_init(&segmap_pool, sz, 0, 0, 0, "segmap", NULL, IPL_NONE);

#if defined(SUN4M) || defined(SUN4D)
	if (CPU_HAS_SRMMU) {
		/*
		 * The SRMMU only ever needs chunks in one of two sizes:
		 * 1024 (for region level tables) and 256 (for segment
		 * and page level tables).
		 */
		sz = SRMMU_L1SIZE * sizeof(int);
		pool_init(&L1_pool, sz, sz, 0, 0, "L1 pagetable",
			  &pgt_page_allocator, IPL_NONE);

		sz = SRMMU_L2SIZE * sizeof(int);
		pool_init(&L23_pool, sz, sz, 0, 0, "L2/L3 pagetable",
			  &pgt_page_allocator, IPL_NONE);
	}
#endif /* SUN4M || SUN4D */
#if defined(SUN4) || defined(SUN4C)
	if (CPU_HAS_SUNMMU) {
		sz = NPTESG * sizeof(int);
		pool_init(&pte_pool, sz, 0, 0, 0, "ptemap", NULL,
		    IPL_NONE);
	}
#endif /* SUN4 || SUN4C */
}


/*
 * Map physical addresses into kernel VM.
 */
vaddr_t
pmap_map(vaddr_t va, paddr_t pa, paddr_t endpa, int prot)
{
	int pgsize = PAGE_SIZE;

	while (pa < endpa) {
		pmap_kenter_pa(va, pa, prot, 0);
		va += pgsize;
		pa += pgsize;
	}
	pmap_update(pmap_kernel());
	return (va);
}

/*
 * Check a pmap for spuriously lingering mappings
 */
static inline void
pmap_quiet_check(struct pmap *pm)
{
#ifdef DEBUG
	int vs, vr;

	if (CPU_HAS_SUNMMU) {
#if defined(SUN4_MMU3L)
		if (TAILQ_FIRST(&pm->pm_reglist))
			panic("pmap_destroy: region list not empty");
#endif
		if (TAILQ_FIRST(&pm->pm_seglist))
			panic("pmap_destroy: segment list not empty");
	}

	for (vr = 0; vr < NUREG; vr++) {
		struct regmap *rp = &pm->pm_regmap[vr];

		if (HASSUN4_MMU3L) {
			if (rp->rg_smeg != reginval)
				printf("pmap_chk: spurious smeg in "
				       "user region %d\n", vr);
		}
		if (CPU_HAS_SRMMU) {
			int n;
#if defined(MULTIPROCESSOR)
			for (n = 0; n < sparc_ncpus; n++)
#else
			n = 0;
#endif
			{
				/* Did this cpu attach? */
				if (pmap_kernel()->pm_reg_ptps[n] == 0)
					continue;

				if (pm->pm_reg_ptps[n][vr] != SRMMU_TEINVALID)
					printf("pmap_chk: spurious PTP in user "
						"region %d on CPU %d\n", vr, n);
			}
		}
		if (rp->rg_nsegmap != 0)
			printf("pmap_chk: %d segments remain in "
				"region %d\n", rp->rg_nsegmap, vr);
		if (rp->rg_segmap != NULL) {
			printf("pmap_chk: segments still "
				"allocated in region %d\n", vr);
			for (vs = 0; vs < NSEGRG; vs++) {
				struct segmap *sp = &rp->rg_segmap[vs];
				if (sp->sg_npte != 0)
					printf("pmap_chk: %d ptes "
					     "remain in segment %d\n",
						sp->sg_npte, vs);
				if (sp->sg_pte != NULL) {
					printf("pmap_chk: ptes still "
					     "allocated in segment %d\n", vs);
				}
				if (CPU_HAS_SUNMMU) {
					if (sp->sg_pmeg != seginval)
						printf("pmap_chk: pm %p(%d,%d) "
						  "spurious soft pmeg %d\n",
						  pm, vr, vs, sp->sg_pmeg);
				}
			}
		}

		/* Check for spurious pmeg entries in the MMU */
		if (pm->pm_ctx == NULL)
			continue;
		if (CPU_HAS_SUNMMU) {
			int ctx;
			if (mmu_has_hole && (vr >= 32 && vr < (256 - 32)))
				continue;
			ctx = getcontext4();
			setcontext4(pm->pm_ctxnum);
			for (vs = 0; vs < NSEGRG; vs++) {
				vaddr_t va = VSTOVA(vr,vs);
				int pmeg = getsegmap(va);
				if (pmeg != seginval)
					printf("pmap_chk: pm %p(%d,%d:%x): "
						"spurious pmeg %d\n",
						pm, vr, vs, (u_int)va, pmeg);
			}
			setcontext4(ctx);
		}
	}
	if (pm->pm_stats.resident_count) {
		printf("pmap_chk: res count %ld\n",
		       pm->pm_stats.resident_count);
	}
	if (pm->pm_stats.wired_count) {
		printf("pmap_chk: wired count %ld\n",
		       pm->pm_stats.wired_count);
	}
#endif /* DEBUG */
}

int
pmap_pmap_pool_ctor(void *arg, void *object, int flags)
{
	struct pmap *pm = object;
	u_long addr;

	memset(pm, 0, sizeof *pm);

	/*
	 * `pmap_pool' entries include space for the per-CPU
	 * region table pointer arrays.
	 */
	addr = (u_long)pm + ALIGN(sizeof(struct pmap));
	pm->pm_regmap = (void *)addr;
	addr += ALIGN(NUREG * sizeof(struct regmap));
	pm->pm_reg_ptps = (int **)addr;
	addr += sparc_ncpus * sizeof(int *);
	pm->pm_reg_ptps_pa = (int *)addr;

	qzero((void *)pm->pm_regmap, NUREG * sizeof(struct regmap));

	/* pm->pm_ctx = NULL; // already done */

	if (CPU_HAS_SUNMMU) {
		TAILQ_INIT(&pm->pm_seglist);
#if defined(SUN4_MMU3L)
		TAILQ_INIT(&pm->pm_reglist);
		if (HASSUN4_MMU3L) {
			int i;
			for (i = NUREG; --i >= 0;)
				pm->pm_regmap[i].rg_smeg = reginval;
		}
#endif
	}
#if defined(SUN4M) || defined(SUN4D)
	else {
		int i, n;

		/*
		 * We must allocate and initialize hardware-readable (MMU)
		 * pagetables. We must also map the kernel regions into this
		 * pmap's pagetables, so that we can access the kernel from
		 * this user context.
		 */
#if defined(MULTIPROCESSOR)
		for (n = 0; n < sparc_ncpus; n++)
#else
		n = 0;
#endif
		{
			int *upt, *kpt;

#if defined(MULTIPROCESSOR)
			/* Did this cpu attach? */
			if (pmap_kernel()->pm_reg_ptps[n] == 0)
				continue;
#endif

			upt = pool_get(&L1_pool, flags);
			pm->pm_reg_ptps[n] = upt;
			pm->pm_reg_ptps_pa[n] = VA2PA((char *)upt);

			/* Invalidate user space regions */
			for (i = 0; i < NUREG; i++)
				setpgt4m(upt++, SRMMU_TEINVALID);

			/* Copy kernel regions */
			kpt = &pmap_kernel()->pm_reg_ptps[n][VA_VREG(KERNBASE)];
			for (i = 0; i < NKREG; i++)
				setpgt4m(upt++, kpt[i]);
		}
	}
#endif /* SUN4M || SUN4D */

	/* XXX - a peculiar place to do this, but we can't do it in pmap_init
	 * and here at least it's off the beaten code track.
	 */
{static int x; if (x == 0) pool_setlowat(&pv_pool, 512), x = 1; }

	return (0);
}

void
pmap_pmap_pool_dtor(void *arg, void *object)
{
	struct pmap *pm = object;
	union ctxinfo *c;
	int s = splvm();	/* paranoia */

	DPRINTF(PDB_DESTROY, "pmap_pmap_pool_dtor(%p)", pm);

	if ((c = pm->pm_ctx) != NULL) {
		mutex_spin_enter(&ctx_lock);
		ctx_free(pm);
		mutex_spin_exit(&ctx_lock);
	}

#if defined(SUN4M) || defined(SUN4D)
	if (CPU_HAS_SRMMU) {
		int n;

#if defined(MULTIPROCESSOR)
		for (n = 0; n < sparc_ncpus; n++)
#else
		n = 0;
#endif
		{
			int *pt;

#if defined(MULTIPROCESSOR)
			/* Did this cpu attach? */
			if (pmap_kernel()->pm_reg_ptps[n] == 0)
				continue;
#endif

			pt = pm->pm_reg_ptps[n];
			pm->pm_reg_ptps[n] = NULL;
			pm->pm_reg_ptps_pa[n] = 0;
			pool_put(&L1_pool, pt);
		}
	}
#endif /* SUN4M || SUN4D */
	splx(s);
}

/*
 * Create and return a physical map.
 */
struct pmap *
pmap_create(void)
{
	struct pmap *pm;

	pm = pool_cache_get(&pmap_cache, PR_WAITOK);

	/*
	 * Reset fields that are not preserved in the pmap cache pool.
	 */
	pm->pm_refcount = 1;
#if defined(MULTIPROCESSOR)
	/* reset active CPU set */
	pm->pm_cpuset = 0;
#endif
	if (CPU_HAS_SUNMMU) {
		/* reset the region gap */
		pm->pm_gap_start = 0;
		pm->pm_gap_end = VA_VREG(VM_MAXUSER_ADDRESS);
	}

	DPRINTF(PDB_CREATE, "pmap_create[%d]: created %p", cpu_number(), pm);
	pmap_quiet_check(pm);

	return (pm);
}

/*
 * Retire the given pmap from service.
 * Should only be called if the map contains no valid mappings.
 */
void
pmap_destroy(struct pmap *pm)
{

	DPRINTF(PDB_DESTROY, "pmap_destroy[%d](%p)", cpu_number(), pm);
	membar_release();
	if (atomic_dec_uint_nv(&pm->pm_refcount) == 0) {
		membar_acquire();
		pmap_quiet_check(pm);
		pool_cache_put(&pmap_cache, pm);
	}
}

/*
 * Add a reference to the given pmap.
 */
void
pmap_reference(struct pmap *pm)
{

	atomic_inc_uint(&pm->pm_refcount);
}

#if defined(SUN4) || defined(SUN4C)
/*
 * helper to deallocate level 2 & 3 page tables.
 */
static void
pgt_lvl23_remove4_4c(struct pmap *pm, struct regmap *rp, struct segmap *sp,
		     int vr, int vs)
{
	vaddr_t va, tva;
	int i, pmeg;

	va = VSTOVA(vr,vs);
	if ((pmeg = sp->sg_pmeg) != seginval) {
		if (CTX_USABLE(pm,rp)) {
			setcontext4(pm->pm_ctxnum);
			setsegmap(va, seginval);
		} else {
			/* no context, use context 0 */
			setcontext4(0);
			if (HASSUN4_MMU3L && rp->rg_smeg != reginval) {
				setregmap(0, rp->rg_smeg);
				tva = vs << SGSHIFT;
				setsegmap(tva, seginval);
			}
		}
		if (!HASSUN4_MMU3L) {
			if (pm == pmap_kernel()) {
				/* Unmap segment from all contexts */
				for (i = ncontext; --i >= 0;) {
					setcontext4(i);
					setsegmap(va, seginval);
				}
			}
		}
		me_free(pm, pmeg);
		sp->sg_pmeg = seginval;
	}
	/* Free software tables for non-kernel maps */
	if (pm != pmap_kernel()) {
		pool_put(&pte_pool, sp->sg_pte);
		sp->sg_pte = NULL;
	}

	if (rp->rg_nsegmap <= 0)
		panic("pgt_rm: pm %p: nsegmap = %d\n", pm, rp->rg_nsegmap);

	if (--rp->rg_nsegmap == 0) {
#if defined(SUN4_MMU3L)
		if (HASSUN4_MMU3L) {
			if (rp->rg_smeg != reginval) {
				if (pm == pmap_kernel()) {
					/* Unmap from all contexts */
					for (i = ncontext; --i >= 0;) {
						setcontext4(i);
						setregmap(va, reginval);
					}
				} else if (pm->pm_ctx) {
					setcontext4(pm->pm_ctxnum);
					setregmap(va, reginval);
				}

				/* Release MMU resource */
				region_free(pm, rp->rg_smeg);
				rp->rg_smeg = reginval;
			}
		}
#endif /* SUN4_MMU3L */
		/* Free software tables for non-kernel maps */
		if (pm != pmap_kernel()) {
			GAP_WIDEN(pm,vr);
			pool_put(&segmap_pool, rp->rg_segmap);
			rp->rg_segmap = NULL;
		}
	}
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)
/*
 * SRMMU helper to deallocate level 2 & 3 page tables.
 */
static void
pgt_lvl23_remove4m(struct pmap *pm, struct regmap *rp, struct segmap *sp,
    int vr, int vs)
{

	/* Invalidate level 2 PTP entry */
	if (pm->pm_ctx)
		tlb_flush_segment(VSTOVA(vr,vs), pm->pm_ctxnum,
				  PMAP_CPUSET(pm));
	setpgt4m(&rp->rg_seg_ptps[vs], SRMMU_TEINVALID);
	pool_put(&L23_pool, sp->sg_pte);
	sp->sg_pte = NULL;

	/* If region is now empty, remove level 2 pagetable as well */
	if (--rp->rg_nsegmap == 0) {
		int n = 0;
		if (pm->pm_ctx)
			tlb_flush_region(VRTOVA(vr), pm->pm_ctxnum,
					 PMAP_CPUSET(pm));
#if defined(MULTIPROCESSOR)
		/* Invalidate level 1 PTP entries on all CPUs */
		for (; n < sparc_ncpus; n++) {
			if ((cpus[n]->flags & CPUFLG_HATCHED) == 0)
				continue;
#endif
			setpgt4m(&pm->pm_reg_ptps[n][vr], SRMMU_TEINVALID);
#if defined(MULTIPROCESSOR)
		}
#endif

		pool_put(&segmap_pool, rp->rg_segmap);
		rp->rg_segmap = NULL;
		pool_put(&L23_pool, rp->rg_seg_ptps);
	}
}
#endif /* SUN4M || SUN4D */

bool
pmap_remove_all(struct pmap *pm)
{
	if (pm->pm_ctx == NULL)
		return false;

#if defined(SUN4) || defined(SUN4C)
	if (CPU_HAS_SUNMMU) {
		int ctx = getcontext4();
		setcontext4(pm->pm_ctxnum);
		cache_flush_context(pm->pm_ctxnum);
		setcontext4(ctx);
	}
#endif

#if defined(SUN4M) || defined(SUN4D)
	if (CPU_HAS_SRMMU) {
		cache_flush_context(pm->pm_ctxnum);
	}
#endif

	pm->pm_flags |= PMAP_USERCACHECLEAN;
	return false;
}

/*
 * Remove the given range of mapping entries.
 * The starting and ending addresses are already rounded to pages.
 * Sheer lunacy: pmap_remove is often asked to remove nonexistent
 * mappings.
 */
void
pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva)
{
	vaddr_t nva;
	int vr, vs, s, ctx;
	void (*rm)(struct pmap *, vaddr_t, vaddr_t, int, int);

	DPRINTF(PDB_REMOVE, "pmap_remove[%d](%p, 0x%lx, 0x%lx)",
	    cpu_number(), pm, va, endva);

	if (!CPU_HAS_SRMMU)
		write_user_windows();

	if (pm == pmap_kernel()) {
		/*
		 * Removing from kernel address space.
		 */
		rm = pmap_rmk;
	} else {
		/*
		 * Removing from user address space.
		 */
		rm = pmap_rmu;
	}

	ctx = getcontext();
	s = splvm();
	PMAP_LOCK();
	for (; va < endva; va = nva) {
		/* do one virtual segment at a time */
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		nva = VSTOVA(vr, vs + 1);
		if (nva == 0 || nva > endva)
			nva = endva;
		if (pm->pm_regmap[vr].rg_nsegmap != 0)
			(*rm)(pm, va, nva, vr, vs);
	}
	PMAP_UNLOCK();
	splx(s);
	setcontext(ctx);
}

/*
 * It is the same amount of work to cache_flush_page 16 pages
 * as to cache_flush_segment 1 segment, assuming a 64K cache size
 * and a 4K page size or a 128K cache size and 8K page size.
 */
#define	PMAP_SFL_THRESHOLD	16	/* if > magic, use cache_flush_segment */

/*
 * Remove a range contained within a single segment.
 * These are egregiously complicated routines.
 */

#if defined(SUN4) || defined(SUN4C)

/* remove from kernel */
/*static*/ void
pmap_rmk4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
{
	int pte, mmupte, *ptep, perpage, npg;
	struct vm_page *pg;
	int nleft, pmeg, inmmu;
	struct regmap *rp;
	struct segmap *sp;

	rp = &pm->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];

	if (rp->rg_nsegmap == 0)
		return;
	if ((nleft = sp->sg_npte) == 0)
		return;
	pmeg = sp->sg_pmeg;
	inmmu = pmeg != seginval;
	ptep = &sp->sg_pte[VA_VPG(va)];

	/* decide how to flush cache */
	npg = (endva - va) >> PGSHIFT;
	if (!inmmu) {
		perpage = 0;
	} else if (npg > PMAP_SFL_THRESHOLD) {
		/* flush the whole segment */
		perpage = 0;
		cache_flush_segment(vr, vs, 0);
	} else {
		/* flush each page individually; some never need flushing */
		perpage = (CACHEINFO.c_vactype != VAC_NONE);
	}

	for (; va < endva; va += NBPG, ptep++) {
		pte = *ptep;
		mmupte = inmmu ? getpte4(va) : 0;
		if ((pte & PG_V) == 0) {
#ifdef DIAGNOSTIC
			if (inmmu && (mmupte & PG_V) != 0)
				printf("rmk: inconsistent ptes va=%lx\n", va);
#endif
			continue;
		}
		if ((pte & PG_TYPE) == PG_OBMEM) {
			/* if cacheable, flush page as needed */
			if (perpage && (mmupte & PG_NC) == 0)
				cache_flush_page(va, 0);
			if ((pg = pvhead4_4c(pte)) != NULL) {
				if (inmmu)
					VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte);
				pv_unlink4_4c(pg, pm, va);
			}
		}
		nleft--;
#ifdef DIAGNOSTIC
		if (nleft < 0)
			panic("pmap_rmk: too many PTEs in segment; "
			      "va 0x%lx; endva 0x%lx", va, endva);
#endif
		if (pte & PG_WIRED) {
			sp->sg_nwired--;
			pm->pm_stats.wired_count--;
		}

		if (inmmu)
			setpte4(va, 0);
		*ptep = 0;
		pm->pm_stats.resident_count--;
	}

#ifdef DIAGNOSTIC
	if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
		panic("pmap_rmk: pm %p, va %lx: nleft=%d, nwired=%d",
			pm, va, nleft, sp->sg_nwired);
#endif
	if ((sp->sg_npte = nleft) == 0)
		pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
	else if (sp->sg_nwired == 0) {
		if (sp->sg_pmeg != seginval)
			mmu_pmeg_unlock(sp->sg_pmeg);
	}
}

#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of pmap_rmk */

/* remove from kernel (4m)*/
/* pm is already locked */
/*static*/ void
pmap_rmk4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
{
	int tpte, perpage, npg;
	struct vm_page *pg;
	struct regmap *rp;
	struct segmap *sp;

	rp = &pm->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];
	if (rp->rg_nsegmap == 0)
		return;

	/* decide how to flush cache */
	npg = (endva - va) >> PGSHIFT;
	if (npg > PMAP_SFL_THRESHOLD) {
		/* flush the whole segment */
		perpage = 0;
		if (CACHEINFO.c_vactype != VAC_NONE)
			cache_flush_segment(vr, vs, 0);
	} else {
		/* flush each page individually; some never need flushing */
		perpage = (CACHEINFO.c_vactype != VAC_NONE);
	}
	while (va < endva) {
		tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
		if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) {
#ifdef DEBUG
			if ((pmapdebug & PDB_SANITYCHK) &&
			    (getpte4m(va) & SRMMU_TETYPE) == SRMMU_TEPTE)
				panic("pmap_rmk: Spurious kTLB entry for 0x%lx",
				      va);
#endif
			va += NBPG;
			continue;
		}
		if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
			/* if cacheable, flush page as needed */
			if (perpage && (tpte & SRMMU_PG_C))
				cache_flush_page(va, 0);
			if ((pg = pvhead4m(tpte)) != NULL) {
				VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte);
				pv_unlink4m(pg, pm, va);
			}
		}
		setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
		    SRMMU_TEINVALID, 1, 0, CPUSET_ALL);
		pm->pm_stats.resident_count--;
		va += NBPG;
	}
}
#endif /* SUN4M || SUN4D */

#if defined(SUN4) || defined(SUN4C)

/* remove from user */
/*static*/ void
pmap_rmu4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
{
	int *ptep, pteva, pte, perpage, npg;
	struct vm_page *pg;
	int nleft, pmeg, inmmu;
	struct regmap *rp;
	struct segmap *sp;

	rp = &pm->pm_regmap[vr];
	if (rp->rg_nsegmap == 0)
		return;
	sp = &rp->rg_segmap[vs];
	if ((nleft = sp->sg_npte) == 0)
		return;
	pmeg = sp->sg_pmeg;
	inmmu = pmeg != seginval;

	/*
	 * PTEs are in MMU.  Invalidate in hardware, update ref &
	 * mod bits, and flush cache if required.
	 */
	if (!inmmu) {
		perpage = 0;
		pteva = 0;
	} else if (CTX_USABLE(pm,rp)) {
		/* process has a context, must flush cache */
		npg = (endva - va) >> PGSHIFT;
		setcontext4(pm->pm_ctxnum);
		if ((pm->pm_flags & PMAP_USERCACHECLEAN) != 0)
			perpage = 0;
		else if (npg > PMAP_SFL_THRESHOLD) {
			perpage = 0; /* flush the whole segment */
			cache_flush_segment(vr, vs, pm->pm_ctxnum);
		} else
			perpage = (CACHEINFO.c_vactype != VAC_NONE);
		pteva = va;
	} else {
		/* no context, use context 0; cache flush unnecessary */
		setcontext4(0);
		if (HASSUN4_MMU3L)
			setregmap(0, tregion);
		/* XXX use per-CPU pteva? */
		setsegmap(0, pmeg);
		pteva = VA_VPG(va) << PGSHIFT;
		perpage = 0;
	}

	ptep = sp->sg_pte + VA_VPG(va);
	for (; va < endva; ptep++, pteva += NBPG, va += NBPG) {
		int mmupte;
		pte = *ptep;
		mmupte = inmmu ? getpte4(pteva) : 0;

		if ((pte & PG_V) == 0) {
#ifdef DIAGNOSTIC
			if (inmmu && (mmupte & PG_V) != 0)
				printf("pmap_rmu: pte=%x, mmupte=%x\n",
					pte, getpte4(pteva));
#endif
			continue;
		}
		if ((pte & PG_TYPE) == PG_OBMEM) {
			/* if cacheable, flush page as needed */
			if (perpage && (mmupte & PG_NC) == 0)
				cache_flush_page(va, pm->pm_ctxnum);
			if ((pg = pvhead4_4c(pte)) != NULL) {
				if (inmmu)
					VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte);
				pv_unlink4_4c(pg, pm, va);
			}
		}
		nleft--;
#ifdef DIAGNOSTIC
		if (nleft < 0)
			panic("pmap_rmu: too many PTEs in segment; "
			     "va 0x%lx; endva 0x%lx", va, endva);
#endif
		if (inmmu)
			setpte4(pteva, 0);

		if (pte & PG_WIRED) {
			sp->sg_nwired--;
			pm->pm_stats.wired_count--;
		}
		*ptep = 0;
		pm->pm_stats.resident_count--;
	}

#ifdef DIAGNOSTIC
	if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
		panic("pmap_rmu: pm %p, va %lx: nleft=%d, nwired=%d",
			pm, va, nleft, sp->sg_nwired);
#endif
	if ((sp->sg_npte = nleft) == 0)
		pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
	else if (sp->sg_nwired == 0) {
		if (sp->sg_pmeg != seginval)
			mmu_pmeg_unlock(sp->sg_pmeg);
	}
}

#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of pmap_rmu */
/* remove from user */
/* Note: pm is already locked */
/*static*/ void
pmap_rmu4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
{
	int *pte0, perpage, npg;
	struct vm_page *pg;
	int nleft;
	struct regmap *rp;
	struct segmap *sp;

	rp = &pm->pm_regmap[vr];
	if (rp->rg_nsegmap == 0)
		return;
	sp = &rp->rg_segmap[vs];
	if ((nleft = sp->sg_npte) == 0)
		return;
	pte0 = sp->sg_pte;

	/*
	 * Invalidate PTE in MMU pagetables. Flush cache if necessary.
	 */
	if (pm->pm_ctx && (pm->pm_flags & PMAP_USERCACHECLEAN) == 0) {
		/* process has a context, must flush cache */
		if (CACHEINFO.c_vactype != VAC_NONE) {
			npg = (endva - va) >> PGSHIFT;
			if (npg > PMAP_SFL_THRESHOLD) {
				perpage = 0; /* flush the whole segment */
				cache_flush_segment(vr, vs, pm->pm_ctxnum);
			} else
				perpage = 1;
		} else
			perpage = 0;
	} else {
		/* no context; cache flush unnecessary */
		perpage = 0;
	}
	for (; va < endva; va += NBPG) {
		int tpte;

		tpte = pte0[VA_SUN4M_VPG(va)];

		if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) {
#ifdef DEBUG
			if ((pmapdebug & PDB_SANITYCHK) &&
			    pm->pm_ctx &&
			    (getpte4m(va) & SRMMU_TEPTE) == SRMMU_TEPTE)
				panic("pmap_rmu: Spurious uTLB entry for 0x%lx",
				      va);
#endif
			continue;
		}

		if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
			/* if cacheable, flush page as needed */
			if (perpage && (tpte & SRMMU_PG_C))
				cache_flush_page(va, pm->pm_ctxnum);
			if ((pg = pvhead4m(tpte)) != NULL) {
				VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte);
				pv_unlink4m(pg, pm, va);
			}
		}
		nleft--;
#ifdef DIAGNOSTIC
		if (nleft < 0)
			panic("pmap_rmu: too many PTEs in segment; "
			      "va 0x%lx; endva 0x%lx", va, endva);
#endif
		setpgt4m_va(va, &pte0[VA_SUN4M_VPG(va)], SRMMU_TEINVALID,
		    pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));
		pm->pm_stats.resident_count--;
		if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) {
			sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
			pm->pm_stats.wired_count--;
		}
	}

	/*
	 * If the segment is all gone, and the context is loaded, give
	 * the segment back.
	 */
	if ((sp->sg_npte = nleft) == 0)
		pgt_lvl23_remove4m(pm, rp, sp, vr, vs);
}
#endif /* SUN4M || SUN4D */

/*
 * Lower (make more strict) the protection on the specified
 * physical page.
 *
 * There are only two cases: either the protection is going to 0
 * (in which case we do the dirty work here), or it is going from
 * to read-only (in which case pv_changepte does the trick).
 */

#if defined(SUN4) || defined(SUN4C)
void
pmap_page_protect4_4c(struct vm_page *pg, vm_prot_t prot)
{
	struct pvlist *pv, *npv;
	struct pmap *pm;
	vaddr_t va;
	int vr, vs, pteva, pte, *ptep;
	int flags, nleft, s, ctx;
	struct regmap *rp;
	struct segmap *sp;

#ifdef DEBUG
	if ((pmapdebug & PDB_CHANGEPROT) ||
	    (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE))
		printf("pmap_page_protect(0x%lx, 0x%x)\n",
			VM_PAGE_TO_PHYS(pg), prot);
#endif

	/*
	 * Skip unmanaged pages, or operations that do not take
	 * away write permission.
	 */
	if (prot & VM_PROT_WRITE)
		return;

	write_user_windows();	/* paranoia */
	if (prot & VM_PROT_READ) {
		pv_changepte4_4c(pg, 0, PG_W);
		return;
	}

	/*
	 * Remove all access to all people talking to this page.
	 * Walk down PV list, removing all mappings.
	 * The logic is much like that for pmap_remove,
	 * but we know we are removing exactly one page.
	 */
	s = splvm();
	pv = VM_MDPAGE_PVHEAD(pg);
	if (pv->pv_pmap == NULL) {
		splx(s);
		return;
	}
	ctx = getcontext4();

	/* This pv head will become empty, so clear caching state flags */
	flags = pv->pv_flags & ~(PV_NC|PV_ANC);

	while (pv != NULL) {
		pm = pv->pv_pmap;
		va = pv->pv_va;
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		rp = &pm->pm_regmap[vr];
		sp = &rp->rg_segmap[vs];
		if ((nleft = sp->sg_npte) <= 0)
			panic("pmap_page_protect: empty vseg");
		sp->sg_npte = --nleft;
		ptep = &sp->sg_pte[VA_VPG(va)];

		if (*ptep & PG_WIRED) {
			sp->sg_nwired--;
			pm->pm_stats.wired_count--;
		}

		if (sp->sg_pmeg != seginval) {
			/* Update PV flags */
			if (CTX_USABLE(pm,rp)) {
				setcontext4(pm->pm_ctxnum);
				pteva = va;
				cache_flush_page(va, pm->pm_ctxnum);
			} else {
				setcontext4(0);
				/* XXX use per-CPU pteva? */
				if (HASSUN4_MMU3L)
					setregmap(0, tregion);
				setsegmap(0, sp->sg_pmeg);
				pteva = VA_VPG(va) << PGSHIFT;
			}

			pte = getpte4(pteva);
#ifdef DIAGNOSTIC
			if ((pte & PG_V) == 0)
				panic("pmap_page_protect !PG_V: pg %p "
				      "ctx %d, va 0x%lx, pte 0x%x",
				      pg, pm->pm_ctxnum, va, pte);
#endif
			flags |= MR4_4C(pte);

			setpte4(pteva, 0);
#ifdef DIAGNOSTIC
			if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
				panic("pmap_page_protect: pm %p, va %lx: nleft=%d, nwired=%d",
					pm, va, nleft, sp->sg_nwired);
#endif
			if (sp->sg_nwired == 0)
				mmu_pmeg_unlock(sp->sg_pmeg);
		}

		*ptep = 0;
		pm->pm_stats.resident_count--;
		if (nleft == 0)
			pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
		npv = pv->pv_next;
		if (pv != VM_MDPAGE_PVHEAD(pg))
			pool_put(&pv_pool, pv);
		pv = npv;
	}

	/* Finally, update pv head */
	VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL;
	VM_MDPAGE_PVHEAD(pg)->pv_next = NULL;
	VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
	setcontext4(ctx);
	splx(s);
}

/*
 * Lower (make more strict) the protection on the specified
 * range of this pmap.
 *
 * There are only two cases: either the protection is going to 0
 * (in which case we call pmap_remove to do the dirty work), or
 * it is going from read/write to read-only.  The latter is
 * fairly easy.
 */
void
pmap_protect4_4c(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
	int va, nva, vr, vs;
	int s, ctx;
	struct regmap *rp;
	struct segmap *sp;

	if ((prot & VM_PROT_READ) == 0) {
		pmap_remove(pm, sva, eva);
		return;
	}

	write_user_windows();
	ctx = getcontext4();
	s = splvm();
	PMAP_LOCK();
	for (va = sva; va < eva;) {
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		rp = &pm->pm_regmap[vr];
		nva = VSTOVA(vr,vs + 1);
		if (nva > eva)
			nva = eva;
		if (rp->rg_nsegmap == 0) {
			va = nva;
			continue;
		}
#ifdef DEBUG
		if (rp->rg_segmap == NULL)
			panic("pmap_protect: no segments");
#endif
		sp = &rp->rg_segmap[vs];
		if (sp->sg_npte == 0) {
			va = nva;
			continue;
		}
#ifdef DEBUG
		if (sp->sg_pte == NULL)
			panic("pmap_protect: no pages");
#endif
		if (sp->sg_pmeg == seginval) {
			int *ptep = &sp->sg_pte[VA_VPG(va)];

			/* not in MMU; just clear PG_W from core copies */
			for (; va < nva; va += NBPG)
				*ptep++ &= ~PG_W;
		} else {
			/* in MMU: take away write bits from MMU PTEs */
			if (CTX_USABLE(pm,rp)) {
				int pte;

				/*
				 * Flush cache so that any existing cache
				 * tags are updated.  This is really only
				 * needed for PTEs that lose PG_W.
				 */
				pmap_stats.ps_npg_prot_all +=
					(nva - va) >> PGSHIFT;
				setcontext4(pm->pm_ctxnum);
				for (; va < nva; va += NBPG) {
					pte = getpte4(va);
					if ((pte & (PG_W|PG_TYPE)) ==
					    (PG_W|PG_OBMEM)) {
						pmap_stats.ps_npg_prot_actual++;
						cache_flush_page(va, pm->pm_ctxnum);
						setpte4(va, pte & ~PG_W);
					}
				}
			} else {
				int pteva;

				/*
				 * No context, hence not cached;
				 * just update PTEs.
				 */
				setcontext4(0);
				/* XXX use per-CPU pteva? */
				if (HASSUN4_MMU3L)
					setregmap(0, tregion);
				setsegmap(0, sp->sg_pmeg);
				pteva = VA_VPG(va) << PGSHIFT;
				for (; va < nva; pteva += NBPG, va += NBPG)
					setpte4(pteva, getpte4(pteva) & ~PG_W);
			}
		}
	}
	PMAP_UNLOCK();
	splx(s);
	setcontext4(ctx);
}

/*
 * Change the protection and/or wired status of the given (MI) virtual page.
 * XXX: should have separate function (or flag) telling whether only wiring
 * is changing.
 */
void
pmap_changeprot4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags)
{
	int vr, vs, newprot, ctx, pte, *ptep;
	int pmeg;
	struct regmap *rp;
	struct segmap *sp;

	DPRINTF(PDB_CHANGEPROT, "pmap_changeprot(%p, 0x%lx, 0x%x, 0x%x)",
	    pm, va, prot, flags);

	if (pm == pmap_kernel())
		newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S;
	else
		newprot = prot & VM_PROT_WRITE ? PG_W : 0;
	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];
	ptep = &sp->sg_pte[VA_VPG(va)];

	pmap_stats.ps_changeprots++;

	pte = *ptep;
	if (pte & PG_WIRED && (flags & PMAP_WIRED) == 0) {
		pte &= ~PG_WIRED;
		sp->sg_nwired--;
		pm->pm_stats.wired_count--;
	} else if ((pte & PG_WIRED) == 0 && flags & PMAP_WIRED) {
		pte |= PG_WIRED;
		sp->sg_nwired++;
		pm->pm_stats.wired_count++;
	}
	pte = (pte & ~PG_PROT) | newprot;
	/* Update S/W pte entry */
	*ptep = pte;

	/* update PTEs in software or hardware */
	if ((pmeg = sp->sg_pmeg) != seginval) {
		/* update in hardware */
		ctx = getcontext4();
		if (CTX_USABLE(pm,rp)) {
			/*
			 * Use current context.
			 * Flush cache if page has been referenced to
			 * avoid stale protection bits in the cache tags.
			 */
			setcontext4(pm->pm_ctxnum);
			pte = getpte4(va);
			if ((pte & (PG_U|PG_NC|PG_TYPE)) == (PG_U|PG_OBMEM))
				cache_flush_page(va, pm->pm_ctxnum);
		} else {
			setcontext4(0);
			/* XXX use per-CPU va? */
			if (HASSUN4_MMU3L)
				setregmap(0, tregion);
			setsegmap(0, pmeg);
			va = VA_VPG(va) << PGSHIFT;
			pte = getpte4(va);
		}
		pte = (pte & ~PG_PROT) | newprot;
		setpte4(va, pte);
		setcontext4(ctx);
#ifdef DIAGNOSTIC
		if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
			panic("pmap_protect: pm %p, va %lx: nleft=%d, nwired=%d",
				pm, va, sp->sg_npte, sp->sg_nwired);
#endif
		if (sp->sg_nwired == 0)
			mmu_pmeg_unlock(pmeg);
		else
			mmu_pmeg_lock(pmeg);
	}
}

#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)
/*
 * Lower (make more strict) the protection on the specified
 * physical page.
 *
 * There are only two cases: either the protection is going to 0
 * (in which case we do the dirty work here), or it is going
 * to read-only (in which case pv_changepte does the trick).
 */
void
pmap_page_protect4m(struct vm_page *pg, vm_prot_t prot)
{
	struct pvlist *pv, *npv;
	struct pmap *pm;
	vaddr_t va;
	int vr, vs, tpte;
	int flags, nleft, s;
	struct regmap *rp;
	struct segmap *sp;

#ifdef DEBUG
	if ((pmapdebug & PDB_CHANGEPROT) ||
	    (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE))
		printf("pmap_page_protect[%d](0x%lx, 0x%x)\n",
			cpu_number(), VM_PAGE_TO_PHYS(pg), prot);
#endif
	s = splvm();
	PMAP_LOCK();

	if (prot & VM_PROT_READ) {
		pv_changepte4m(pg, 0, PPROT_WRITE);
		goto out;
	}

	/*
	 * Remove all access to all people talking to this page.
	 * Walk down PV list, removing all mappings. The logic is much
	 * like that for pmap_remove, but we know we are removing exactly
	 * one page.
	 */
	pv = VM_MDPAGE_PVHEAD(pg);
	if (pv->pv_pmap == NULL)
		goto out;

	/* This pv head will become empty, so clear caching state flags */
	flags = pv->pv_flags & ~(PV_NC|PV_ANC);
	while (pv != NULL) {
		pm = pv->pv_pmap;
		va = pv->pv_va;
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		rp = &pm->pm_regmap[vr];
		if (rp->rg_nsegmap == 0)
			panic("pmap_remove_all: empty vreg");
		sp = &rp->rg_segmap[vs];
		nleft = sp->sg_npte;
		if (pm != pmap_kernel()) {
			if (nleft <= 0)
				panic("pmap_page_protect: empty vseg");
			sp->sg_npte = --nleft;
		}

		/*
		 * Invalidate PTE in MMU pagetables.
		 * Flush cache if necessary.
		 */
		if (pm->pm_ctx) {
			cache_flush_page(va, pm->pm_ctxnum);
		}

		tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
		setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], SRMMU_TEINVALID,
		    pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));

		pm->pm_stats.resident_count--;
		if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) {
			sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
			pm->pm_stats.wired_count--;
		}

		if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
			panic("pmap_page_protect !PG_V: pg %p va %lx", pg, va);

		flags |= MR4M(tpte);

		if (pm != pmap_kernel() && nleft == 0)
			/*
			 * Entire user mode segment is gone
			 */
			pgt_lvl23_remove4m(pm, rp, sp, vr, vs);

		npv = pv->pv_next;
		if (pv != VM_MDPAGE_PVHEAD(pg))
			pool_put(&pv_pool, pv);
		pv = npv;
	}

	/* Finally, update pv head */
	VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL;
	VM_MDPAGE_PVHEAD(pg)->pv_next = NULL;
	VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;

out:
	PMAP_UNLOCK();
	splx(s);
}

/*
 * Lower (make more strict) the protection on the specified
 * range of this pmap.
 */
void
pmap_protect4m(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
	vaddr_t va, nva;
	int s, vr, vs;
	struct regmap *rp;
	struct segmap *sp;
	int newprot;

	if ((prot & VM_PROT_READ) == 0) {
		pmap_remove(pm, sva, eva);
		return;
	}

	DPRINTF(PDB_CHANGEPROT,
	    "pmap_protect[%d][curpid %d, ctx %d,%d](%lx, %lx, %x)",
	    cpu_number(), curproc->p_pid, getcontext4m(),
	    pm->pm_ctx ? pm->pm_ctxnum : -1, sva, eva, prot);

	newprot = pte_prot4m(pm, prot);

	write_user_windows();
	s = splvm();
	PMAP_LOCK();

	for (va = sva; va < eva;) {
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		rp = &pm->pm_regmap[vr];
		nva = VSTOVA(vr,vs + 1);
		if (nva > eva)
			nva = eva;
		if (rp->rg_nsegmap == 0) {
			va = nva;
			continue;
		}
		sp = &rp->rg_segmap[vs];
		if (pm != pmap_kernel() && sp->sg_npte == 0) {
			va = nva;
			continue;
		}

		/*
		 * pages loaded: take away write bits from MMU PTEs
		 */
		pmap_stats.ps_npg_prot_all += (nva - va) >> PGSHIFT;
		for (; va < nva; va += NBPG) {
			int tpte, npte;

			tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
			if ((tpte & SRMMU_PGTYPE) != PG_SUN4M_OBMEM)
				continue;
			if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
				continue;
			npte = (tpte & ~SRMMU_PROT_MASK) | newprot;
			if (npte == tpte)
				continue;

			/*
			 * Flush cache so that any existing cache
			 * tags are updated.
			 */

			pmap_stats.ps_npg_prot_actual++;
			if (pm->pm_ctx) {
				cache_flush_page(va, pm->pm_ctxnum);
			}
			updatepte4m(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
			    SRMMU_PROT_MASK, newprot, pm->pm_ctxnum,
			    PMAP_CPUSET(pm));
		}
	}
	PMAP_UNLOCK();
	splx(s);
}

/*
 * Change the protection and/or wired status of the given (MI) virtual page.
 * XXX: should have separate function (or flag) telling whether only wiring
 * is changing.
 */
void
pmap_changeprot4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags)
{
	int pte, newprot;
	struct regmap *rp;
	struct segmap *sp;
	bool owired;

	DPRINTF(PDB_CHANGEPROT, "pmap_changeprot[%d](%p, 0x%lx, 0x%x, 0x%x)",
	    cpu_number(), pm, va, prot, flags);

	newprot = pte_prot4m(pm, prot);

	pmap_stats.ps_changeprots++;

	rp = &pm->pm_regmap[VA_VREG(va)];
	sp = &rp->rg_segmap[VA_VSEG(va)];

	pte = sp->sg_pte[VA_SUN4M_VPG(va)];
	owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));

	if (owired) {
		pm->pm_stats.wired_count--;
		sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
	}
	if (flags & PMAP_WIRED) {
		pm->pm_stats.wired_count++;
		sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va));
	}

	if (pm->pm_ctx) {
		/*
		 * Use current context.
		 * Flush cache if page has been referenced to
		 * avoid stale protection bits in the cache tags.
		 */

		if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) ==
		    (SRMMU_PG_C|PG_SUN4M_OBMEM))
			cache_flush_page(va, pm->pm_ctxnum);
	}

	setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
		 (pte & ~SRMMU_PROT_MASK) | newprot,
		 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));

}
#endif /* SUN4M || SUN4D */

/*
 * Insert (MI) physical page pa at virtual address va in the given pmap.
 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary.
 *
 * If pa is not in the `managed' range it will not be `bank mapped'.
 * This works during bootstrap only because the first 4MB happens to
 * map one-to-one.
 *
 * There may already be something else there, or we might just be
 * changing protections and/or wiring on an existing mapping.
 *	XXX	should have different entry points for changing!
 */

#if defined(SUN4) || defined(SUN4C)

int
pmap_enter4_4c(struct pmap *pm, vaddr_t va, paddr_t pa,
	       vm_prot_t prot, u_int flags)
{
	struct vm_page *pg;
	int pteproto, ctx;
	int error;

	if (VA_INHOLE(va)) {
#ifdef DEBUG
		printf("pmap_enter: pm %p, va 0x%lx, pa 0x%lx: in MMU hole\n",
			pm, va, pa);
#endif
		return 0;
	}

	DPRINTF(PDB_ENTER, "pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)",
	    pm, va, pa, prot, flags);

	pg = PHYS_TO_VM_PAGE(pa);
	pteproto = PG_V | PMAP_T2PTE_4(pa);
	pa &= ~PMAP_TNC_4;

	/*
	 * Set up prototype for new PTE.  Cannot set PG_NC from PV_NC yet
	 * since the pvlist no-cache bit might change as a result of the
	 * new mapping.
	 */
	pteproto |= atop(pa) & PG_PFNUM;
	if (prot & VM_PROT_WRITE)
		pteproto |= PG_W;
	if ((flags & PMAP_WIRED) != 0)
		pteproto |= PG_WIRED;
	if (flags & VM_PROT_ALL) {
		pteproto |= PG_U;
		if (flags & VM_PROT_WRITE) {
			pteproto |= PG_M;
		}
	}

	write_user_windows();
	ctx = getcontext4();
	if (pm == pmap_kernel())
		error = pmap_enk4_4c(pm, va, prot, flags, pg, pteproto | PG_S);
	else
		error = pmap_enu4_4c(pm, va, prot, flags, pg, pteproto);
	setcontext4(ctx);
	return (error);
}

/* enter new (or change existing) kernel mapping */
int
pmap_enk4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
	     struct vm_page *pg, int pteproto)
{
	int vr, vs, pte, s, inmmu;
	int *ptep;
	struct regmap *rp;
	struct segmap *sp;
	int error = 0;

	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];
	ptep = &sp->sg_pte[VA_VPG(va)];
	s = splvm();		/* XXX way too conservative */

#if defined(SUN4_MMU3L)
	if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
		mmu_pagein_reg(pm, rp, va, vr, &region_locked);
#endif

	inmmu = sp->sg_pmeg != seginval;
	if ((pte = *ptep) & PG_V) {

		/* old mapping exists, and is of the same pa type */
		if ((pte & (PG_PFNUM|PG_TYPE)) ==
		    (pteproto & (PG_PFNUM|PG_TYPE))) {
			/* just changing protection and/or wiring */
			pmap_changeprot4_4c(pm, va, prot, flags);
			splx(s);
			return (0);
		}

		if ((pte & PG_TYPE) == PG_OBMEM) {
			struct vm_page *opg;

			/*
			 * Switcheroo: changing pa for this va.
			 * If old pa was managed, remove from pvlist.
			 * If old page was cached, flush cache.
			 */
			if ((opg = pvhead4_4c(pte)) != NULL)
				pv_unlink4_4c(opg, pm, va);
			if (inmmu && (pte & PG_NC) == 0) {
				setcontext4(0);	/* ??? */
				cache_flush_page(va, 0);
			}
		}
		*ptep = 0;
		if (inmmu)
			setpte4(va, 0);
		if (pte & PG_WIRED) {
			sp->sg_nwired--;
			pm->pm_stats.wired_count--;
		}
		pm->pm_stats.resident_count--;
	} else {
		/* adding new entry */
		if (sp->sg_npte++ == 0) {
#ifdef DIAGNOSTIC
			int i; for (i = 0; i < NPTESG; i++) {
				if (sp->sg_pte[i] == 0)
					continue;
				panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n",
					pm, va, i);
			}
#endif
			rp->rg_nsegmap++;
		}
	}

	/*
	 * If the new mapping is for a managed PA, enter into pvlist.
	 */
	if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) {
		if (--sp->sg_npte == 0)
			pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
		if ((flags & PMAP_CANFAIL) != 0)
			goto out;
		panic("pmap_enter: cannot allocate PV entry");
	}

	/* Update S/W page table */
	*ptep = pteproto;
	if (pteproto & PG_WIRED) {
		sp->sg_nwired++;
		pm->pm_stats.wired_count++;
	}
	pm->pm_stats.resident_count++;

#ifdef DIAGNOSTIC
	if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
		panic("pmap_enk: pm %p, va %lx: nleft=%d, nwired=%d",
			pm, va, sp->sg_npte, sp->sg_nwired);
#endif
	if (sp->sg_pmeg == seginval)
		mmu_pagein_seg(pm, sp, va, vr, vs,
			(pteproto & PG_WIRED) != 0 ? &segm_locked : &segm_lru);
	else if ((pteproto & PG_WIRED) != 0)
		mmu_pmeg_lock(sp->sg_pmeg);

	/* Update H/W page table */
	setpte4(va, pteproto & ~PG_MBZ);
out:
	splx(s);
	return (error);
}

/* enter new (or change existing) user mapping */
int
pmap_enu4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
	     struct vm_page *pg, int pteproto)
{
	int vr, vs, *ptep, pte, pmeg, s;
	int error = 0;
	struct regmap *rp;
	struct segmap *sp;

	pm->pm_flags &= ~PMAP_USERCACHECLEAN;

	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	s = splvm();			/* XXX conservative */

	/*
	 * If there is no space in which the PTEs can be written
	 * while they are not in the hardware, this must be a new
	 * virtual segment.  Get PTE space and count the segment.
	 *
	 * TO SPEED UP CTX ALLOC, PUT SEGMENT BOUNDS STUFF HERE
	 * AND IN pmap_rmu()
	 */

	GAP_SHRINK(pm,vr);

#ifdef DEBUG
	if (pm->pm_gap_end < pm->pm_gap_start) {
		printf("pmap_enu: gap_start 0x%x, gap_end 0x%x",
			pm->pm_gap_start, pm->pm_gap_end);
		panic("pmap_enu: gap botch");
	}
#endif

	if (rp->rg_segmap == NULL) {
		/* definitely a new mapping */
		int i;
		int mflag = PR_NOWAIT;

	rretry:
		sp = (struct segmap *)pool_get(&segmap_pool, mflag);
		if (sp == NULL) {
			if ((flags & PMAP_CANFAIL) != 0) {
				error = ENOMEM;
				goto out;
			}
			mflag = PR_WAITOK;
			goto rretry;
		}
#ifdef DEBUG
		if (rp->rg_segmap != NULL)
			panic("pmap_enter: segment filled during sleep");
#endif
		qzero((void *)sp, NSEGRG * sizeof (struct segmap));
		rp->rg_segmap = sp;
		rp->rg_nsegmap = 0;
		for (i = NSEGRG; --i >= 0;)
			sp++->sg_pmeg = seginval;
	}

	sp = &rp->rg_segmap[vs];

	if ((ptep = sp->sg_pte) == NULL) {
		/* definitely a new mapping */
		int size = NPTESG * sizeof *ptep;
		int mflag = PR_NOWAIT;

	sretry:
		ptep = (int *)pool_get(&pte_pool, mflag);
		if (ptep == NULL) {
			if ((flags & PMAP_CANFAIL) != 0) {
				error = ENOMEM;
				goto out;
			}
			mflag = PR_WAITOK;
			goto sretry;
		}
#ifdef DEBUG
		if (sp->sg_pte != NULL)
			panic("pmap_enter: pte filled during sleep");
		if (sp->sg_pmeg != seginval)
			panic("pmap_enter: new ptes, but not seginval");
#endif
		qzero((void *)ptep, size);
		sp->sg_pte = ptep;
		sp->sg_npte = 1;
		rp->rg_nsegmap++;
	} else {
		/* might be a change: fetch old pte */
		pte = ptep[VA_VPG(va)];
		if (pte & PG_V) {
			/* old mapping exists, and is of the same pa type */
			if ((pte & (PG_PFNUM|PG_TYPE)) ==
			    (pteproto & (PG_PFNUM|PG_TYPE))) {
				/* just changing prot and/or wiring */
				pmap_changeprot4_4c(pm, va, prot, flags);
				splx(s);
				return (0);
			}
			/*
			 * Switcheroo: changing pa for this va.
			 * If old pa was managed, remove from pvlist.
			 * If old page was cached, flush cache.
			 */
#if 0
			printf("%s[%d]: pmap_enu: changing existing "
				"va(0x%lx)=>pa entry\n",
				curproc->p_comm, curproc->p_pid, va);
#endif
			if ((pte & PG_TYPE) == PG_OBMEM) {
				struct vm_page *opg;
				if ((opg = pvhead4_4c(pte)) != NULL)
					pv_unlink4_4c(opg, pm, va);
				if (CACHEINFO.c_vactype != VAC_NONE &&
				    (pmeg = sp->sg_pmeg) != seginval) {
					/* hardware pte */
					if (CTX_USABLE(pm,rp)) {
						setcontext4(pm->pm_ctxnum);
					} else {
						setcontext4(0);
						/* XXX use per-CPU pteva? */
						if (HASSUN4_MMU3L)
							setregmap(0, tregion);
						setsegmap(0, pmeg);
					}
					cache_flush_page(va, pm->pm_ctxnum);
				}
			}
			if (pte & PG_WIRED) {
				sp->sg_nwired--;
				pm->pm_stats.wired_count--;
			}
			pm->pm_stats.resident_count--;
			ptep[VA_VPG(va)] = 0;
			if (sp->sg_pmeg != seginval)
				setpte4(va, 0);
		} else {
			/* adding new entry */
			sp->sg_npte++;
		}
	}

	if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) {
		if (--sp->sg_npte == 0)
			/* Sigh, undo pgt allocations */
			pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);

		if ((flags & PMAP_CANFAIL) != 0)
			goto out;
		panic("pmap_enter: cannot allocate PV entry");
	}

	/* Update S/W page table */
	ptep += VA_VPG(va);
	*ptep = pteproto;
	if (pteproto & PG_WIRED) {
		sp->sg_nwired++;
		pm->pm_stats.wired_count++;
	}
	pm->pm_stats.resident_count++;

#ifdef DIAGNOSTIC
	if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
		panic("pmap_enu: pm %p, va %lx: nleft=%d, nwired=%d",
			pm, va, sp->sg_npte, sp->sg_nwired);
#endif

	if ((pmeg = sp->sg_pmeg) != seginval) {
		/* Update H/W page table */
		if (CTX_USABLE(pm,rp))
			setcontext4(pm->pm_ctxnum);
		else {
			setcontext4(0);
			if (HASSUN4_MMU3L)
				setregmap(0, tregion);
			setsegmap(0, pmeg);
			va = VA_VPG(va) << PGSHIFT;
		}
		setpte4(va, pteproto & ~PG_MBZ);
	}

out:
	splx(s);
	return (error);
}

void
pmap_kenter_pa4_4c(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
	struct pmap *pm = pmap_kernel();
	struct regmap *rp;
	struct segmap *sp;
	int vr, vs, s;
	int *ptep, pteproto;
	int lockit = 1;

	pteproto = PG_S | PG_V | PMAP_T2PTE_4(pa);
	pa &= ~PMAP_TNC_4;
	pteproto |= atop(pa) & PG_PFNUM;
	if (prot & VM_PROT_WRITE)
		pteproto |= PG_W;

	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];
	ptep = &sp->sg_pte[VA_VPG(va)];

	if (lockit) {
		pteproto |= PG_WIRED;
		sp->sg_nwired++;
	}

	KASSERT((*ptep & PG_V) == 0);

	s = splvm();
#if defined(SUN4_MMU3L)
	if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
		mmu_pagein_reg(pm, rp, va, vr, &region_locked);
#endif

	if (sp->sg_npte++ == 0) {
#ifdef DIAGNOSTIC
		int i; for (i = 0; i < NPTESG; i++) {
			if (sp->sg_pte[i] == 0)
				continue;
			panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n",
				pm, va, i);
		}
#endif
		rp->rg_nsegmap++;
	}

	/* Update S/W page table */
	*ptep = pteproto;

#ifdef DIAGNOSTIC
	if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
		panic("pmap_kenter: pm %p, va %lx: nleft=%d, nwired=%d",
			pm, va, sp->sg_npte, sp->sg_nwired);
#endif

	if (sp->sg_pmeg == seginval) {
		mmu_pagein_seg(pm, sp, va, vr, vs,
				lockit ? &segm_locked : &segm_lru);
	} else if (lockit)
		mmu_pmeg_lock(sp->sg_pmeg);

	/* Update H/W page table */
	setpte4(va, pteproto & ~PG_MBZ);
	splx(s);
}

#if notyet /* XXXMRG delete */
void pmap_lockmmu(vaddr_t sva, size_t sz);

void
pmap_lockmmu(vaddr_t sva, size_t sz)
{
	struct pmap *pm = pmap_kernel();
	vaddr_t va, eva;
	struct regmap *rp;
	struct segmap *sp;
	int vr, vs;

	if (CPU_HAS_SRMMU)
		return;

	eva = sva + sz;
	va = VA_ROUNDDOWNTOSEG(sva);

	for (; va < eva; va += NBPSG) {
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		rp = &pm->pm_regmap[vr];
		sp = &rp->rg_segmap[vs];

		KASSERT(sp->sg_npte != 0);

		if (sp->sg_pmeg == seginval)
			mmu_pagein_seg(pm, sp, va, vr, vs, &segm_locked);
		else
			mmu_pmeg_lock(sp->sg_pmeg);
	}
}
#endif

void
pmap_kremove4_4c(vaddr_t va, vsize_t len)
{
	struct pmap *pm = pmap_kernel();
	struct regmap *rp;
	struct segmap *sp;
	vaddr_t nva, endva;
	int pte, mmupte, *ptep, perpage, npg, inmmu;
	int nleft, pmeg;
	int vr, vs, s, ctx;

	endva = va + len;
	DPRINTF(PDB_REMOVE, "pmap_kremove(0x%lx, 0x%lx)", va, endva);

	write_user_windows();

	s = splvm();
	ctx = getcontext();
	PMAP_LOCK();
	setcontext4(0);
	for (; va < endva; va = nva) {
		/* do one virtual segment at a time */
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		nva = VSTOVA(vr, vs + 1);
		if (nva == 0 || nva > endva)
			nva = endva;

		rp = &pm->pm_regmap[vr];
		sp = &rp->rg_segmap[vs];

		if (rp->rg_nsegmap == 0)
			continue;
		nleft = sp->sg_npte;
		if (nleft == 0)
			continue;
		pmeg = sp->sg_pmeg;
		inmmu = (pmeg != seginval);
		ptep = &sp->sg_pte[VA_VPG(va)];

		/* decide how to flush cache */
		npg = (nva - va) >> PGSHIFT;
		if (!inmmu) {
			perpage = 0;
		} else if (npg > PMAP_SFL_THRESHOLD) {
			/* flush the whole segment */
			perpage = 0;
			cache_flush_segment(vr, vs, 0);
		} else {
			/*
			 * flush each page individually;
			 * some never need flushing
			 */
			perpage = (CACHEINFO.c_vactype != VAC_NONE);
		}

		for (; va < nva; va += NBPG, ptep++) {
			pte = *ptep;
			mmupte = inmmu ? getpte4(va) : 0;
			if ((pte & PG_V) == 0) {
#ifdef DIAGNOSTIC
				if (inmmu && (mmupte & PG_V) != 0)
					printf("rmk: inconsistent ptes va=%lx\n", va);
#endif
				continue;
			}
			if ((pte & PG_TYPE) == PG_OBMEM) {
				/* if cacheable, flush page as needed */
				if (perpage && (mmupte & PG_NC) == 0)
					cache_flush_page(va, 0);
			}
			nleft--;
#ifdef DIAGNOSTIC
			if (nleft < 0)
				panic("pmap_kremove: too many PTEs in segment; "
				      "va 0x%lx; endva 0x%lx", va, endva);
#endif
			if (pte & PG_WIRED)
				sp->sg_nwired--;

			if (inmmu)
				setpte4(va, 0);
			*ptep = 0;
		}

#ifdef DIAGNOSTIC
		if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
			panic("pmap_kremove: pm %p, va %lx: nleft=%d, nwired=%d",
				pm, va, nleft, sp->sg_nwired);
#endif

		if ((sp->sg_npte = nleft) == 0)
			pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
		else if (sp->sg_nwired == 0) {
			if (sp->sg_pmeg != seginval)
				mmu_pmeg_unlock(sp->sg_pmeg);
		}
	}
	PMAP_UNLOCK();
	setcontext4(ctx);
	splx(s);
}

/*
 * Change protection on a range of kernel addresses.
 */
void
pmap_kprotect4_4c(vaddr_t va, vsize_t size, vm_prot_t prot)
{
	int pte, newprot, ctx;

	size = roundup(size,NBPG);
	newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S;

	ctx = getcontext4();
	setcontext4(0);
	while (size > 0) {
		pte = getpte4(va);

		/*
		 * Flush cache if page has been referenced to
		 * avoid stale protection bits in the cache tags.
		 */
		if ((pte & (PG_NC|PG_TYPE)) == PG_OBMEM)
			cache_flush_page(va, 0);

		pte = (pte & ~PG_PROT) | newprot;
		setpte4(va, pte);

		va += NBPG;
		size -= NBPG;
	}
	setcontext4(ctx);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)	/* SRMMU versions of enter routines */
/*
 * Insert (MI) physical page pa at virtual address va in the given pmap.
 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary.
 *
 * If pa is not in the `managed' range it will not be `bank mapped'.
 * This works during bootstrap only because the first 4MB happens to
 * map one-to-one.
 *
 * There may already be something else there, or we might just be
 * changing protections and/or wiring on an existing mapping.
 *	XXX	should have different entry points for changing!
 */

int
pmap_enter4m(struct pmap *pm, vaddr_t va, paddr_t pa,
	     vm_prot_t prot, u_int flags)
{
	struct vm_page *pg;
	int pteproto;
	int error;

	DPRINTF(PDB_ENTER, "pmap_enter[curcpu %d, curpid %d, ctx %d,%d]"
	    "(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)",
	    cpu_number(), curproc == NULL ? -1 : curproc->p_pid,
	    getcontext4m(), pm->pm_ctx == NULL ? -1 : pm->pm_ctxnum,
	    pm, va, pa, prot, flags);

	pg = PHYS_TO_VM_PAGE(pa);

	/* Initialise pteproto with cache bit */
	pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0;

#ifdef DEBUG
	if (pa & PMAP_TYPE_SRMMU) {	/* this page goes in an iospace */
		if (cpuinfo.cpu_type == CPUTYP_MS1)
			panic("pmap_enter4m: attempt to use 36-bit iospace on"
			      " MicroSPARC");
	}
#endif
	pteproto |= SRMMU_TEPTE;
	pteproto |= PMAP_T2PTE_SRMMU(pa);
	pa &= ~PMAP_TNC_SRMMU;

	/*
	 * Set up prototype for new PTE.  Cannot set PG_NC from PV_NC yet
	 * since the pvlist no-cache bit might change as a result of the
	 * new mapping.
	 */
	pteproto |= (atop(pa) << SRMMU_PPNSHIFT);

	/* Make sure we get a pte with appropriate perms! */
	pteproto |= pte_prot4m(pm, prot);
	if (flags & VM_PROT_ALL) {
		pteproto |= SRMMU_PG_R;
		if (flags & VM_PROT_WRITE) {
			pteproto |= SRMMU_PG_M;
		}
	}

	if (pm == pmap_kernel())
		error = pmap_enk4m(pm, va, prot, flags, pg, pteproto | PPROT_S);
	else
		error = pmap_enu4m(pm, va, prot, flags, pg, pteproto);

	return (error);
}

/* enter new (or change existing) kernel mapping */
int
pmap_enk4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
	   struct vm_page *pg, int pteproto)
{
	int vr, vs, tpte, s;
	struct regmap *rp;
	struct segmap *sp;
	int error = 0;

#ifdef DEBUG
	if (va < KERNBASE)
		panic("pmap_enk4m: can't enter va 0x%lx below KERNBASE", va);
#endif
	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];

	kpreempt_disable();
	s = splvm();
	PMAP_LOCK();

	if (rp->rg_seg_ptps == NULL) /* enter new region */
		panic("pmap_enk4m: missing kernel region table for va 0x%lx",va);

	tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
	if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) {

		/* old mapping exists, and is of the same pa type */

		if ((tpte & SRMMU_PPNMASK) == (pteproto & SRMMU_PPNMASK)) {
			/* just changing protection and/or wiring */
			pmap_changeprot4m(pm, va, prot, flags);
			error = 0;
			goto out;
		}

		if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
			struct vm_page *opg;
#ifdef DEBUG
printf("pmap_enk4m: changing existing va=>pa entry: va 0x%lx, pteproto 0x%x, "
       "oldpte 0x%x\n", va, pteproto, tpte);
#endif
			/*
			 * Switcheroo: changing pa for this va.
			 * If old pa was managed, remove from pvlist.
			 * If old page was cached, flush cache.
			 */
			if ((opg = pvhead4m(tpte)) != NULL)
				pv_unlink4m(opg, pm, va);
			if (tpte & SRMMU_PG_C) {
				cache_flush_page(va, 0);
			}
		}

		/*
		 * Invalidate the mapping now, so we can avoid the
		 * de-map and update protocol when setting the new
		 * PTE below.
		 */
		setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
			SRMMU_TEINVALID, pm->pm_ctx != NULL,
			pm->pm_ctxnum, PMAP_CPUSET(pm));
		pm->pm_stats.resident_count--;
	}

	/*
	 * If the new mapping is for a managed PA, enter into pvlist.
	 */
	if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) {
		if ((flags & PMAP_CANFAIL) != 0)
			goto out;
		panic("pmap_enter: cannot allocate PV entry");
	}

	setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
	pm->pm_stats.resident_count++;
out:
	PMAP_UNLOCK();
	splx(s);
	kpreempt_enable();
	return (error);
}

/* enter new (or change existing) user mapping */
int
pmap_enu4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
	   struct vm_page *pg, int pteproto)
{
	int vr, vs, *pte, tpte, s;
	int error = 0;
	struct regmap *rp;
	struct segmap *sp;
	bool owired;

#ifdef DEBUG
	if (KERNBASE < va)
		panic("pmap_enu4m: can't enter va 0x%lx above KERNBASE", va);
#endif

	pm->pm_flags &= ~PMAP_USERCACHECLEAN;

	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	s = splvm();
	PMAP_LOCK();

	if (rp->rg_segmap == NULL) {
		/* definitely a new mapping */
		int mflag = PR_NOWAIT;

	rretry:
		sp = (struct segmap *)pool_get(&segmap_pool, mflag);
		if (sp == NULL) {
			if ((flags & PMAP_CANFAIL) != 0) {
				error = ENOMEM;
				goto out;
			}
			mflag = PR_WAITOK;
			goto rretry;
		}
#ifdef DEBUG
		if (rp->rg_segmap != NULL)
			panic("pmap_enu4m: segment filled during sleep");
#endif
		qzero((void *)sp, NSEGRG * sizeof (struct segmap));
		rp->rg_segmap = sp;
		rp->rg_nsegmap = 0;
		rp->rg_seg_ptps = NULL;
	}
	if (rp->rg_seg_ptps == NULL) {
		/* Need a segment table */
		int i, *ptd;
		int mflag = PR_NOWAIT;

	sretry:
		ptd = pool_get(&L23_pool, mflag);
		if (ptd == NULL) {
			if ((flags & PMAP_CANFAIL) != 0) {
				error = ENOMEM;
				goto out;
			}
			mflag = PR_WAITOK;
			goto sretry;
		}

		rp->rg_seg_ptps = ptd;
		for (i = 0; i < SRMMU_L2SIZE; i++)
			setpgt4m(&ptd[i], SRMMU_TEINVALID);

		/* Replicate segment allocation in each CPU's region table */
#if defined(MULTIPROCESSOR)
		for (i = 0; i < sparc_ncpus; i++)
#else
		i = 0;
#endif
		{
#if defined(MULTIPROCESSOR)
			if ((cpus[i]->flags & CPUFLG_HATCHED) == 0)
				continue;
#endif
			setpgt4m(&pm->pm_reg_ptps[i][vr],
				 (VA2PA((void *)ptd) >> SRMMU_PPNPASHIFT) |
					SRMMU_TEPTD);
		}
	}

	sp = &rp->rg_segmap[vs];

	owired = false;
	if ((pte = sp->sg_pte) == NULL) {
		/* definitely a new mapping */
		int i;
		int mflag = PR_NOWAIT;

		pte = pool_get(&L23_pool, mflag);
		if (pte == NULL) {
			if ((flags & PMAP_CANFAIL) != 0) {
				error = ENOMEM;
				goto out;
			}
			panic("pmap_enter: cannot allocate PTE table");
		}

		sp->sg_pte = pte;
		sp->sg_npte = 1;
		rp->rg_nsegmap++;
		for (i = 0; i < SRMMU_L3SIZE; i++)
			setpgt4m(&pte[i], SRMMU_TEINVALID);
		setpgt4m(&rp->rg_seg_ptps[vs],
			(VA2PA((void *)pte) >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
	} else {
#ifdef DIAGNOSTIC
		if (sp->sg_npte <= 0)
			panic("pm %p: npte %d", pm, sp->sg_npte);
#endif
		/*
		 * Might be a change: fetch old pte
		 */
		tpte = pte[VA_SUN4M_VPG(va)];

		if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) {

			/* old mapping exists, and is of the same pa type */
			if ((tpte & SRMMU_PPNMASK) ==
			    (pteproto & SRMMU_PPNMASK)) {
				/* just changing prot and/or wiring */
				/* caller should call this directly: */
				pmap_changeprot4m(pm, va, prot, flags);
				error = 0;
				goto out;
			}
			/*
			 * Switcheroo: changing pa for this va.
			 * If old pa was managed, remove from pvlist.
			 * If old page was cached, flush cache.
			 */
			DPRINTF(PDB_SWITCHMAP,
			    "%s[%d]: pmap_enu: changing existing "
			    "va 0x%x: pte 0x%x=>0x%x",
			    curproc->p_comm, curproc->p_pid,
			    (int)va, tpte, pteproto);

			if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
				struct vm_page *opg;
				if ((opg = pvhead4m(tpte)) != NULL) {
					VM_MDPAGE_PVHEAD(opg)->pv_flags |=
							MR4M(tpte);
					pv_unlink4m(opg, pm, va);
				}
				if (pm->pm_ctx && (tpte & SRMMU_PG_C))
					cache_flush_page(va, pm->pm_ctxnum);
			}
			/*
			 * We end up in this `change map' branch relatively
			 * infrequently.
			 * Invalidate the mapping now, so we can avoid the
			 * de-map and update protocol when setting the new
			 * PTE below.
			 */
			setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
				SRMMU_TEINVALID, pm->pm_ctx != NULL,
				pm->pm_ctxnum, PMAP_CPUSET(pm));
			pm->pm_stats.resident_count--;
			owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
		} else {
			/* adding new entry */
			sp->sg_npte++;
		}
	}

	if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) {
		if (--sp->sg_npte == 0)
			/* Sigh, undo pgt allocations */
			pgt_lvl23_remove4m(pm, rp, sp, vr, vs);

		if ((flags & PMAP_CANFAIL) != 0)
			goto out;
		panic("pmap_enter: cannot allocate PV entry");
	}

	/*
	 * Update PTEs, flush TLB as necessary.
	 */
	setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
	pm->pm_stats.resident_count++;
	if (owired) {
		pm->pm_stats.wired_count--;
		sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
	}
	if (flags & PMAP_WIRED) {
		pm->pm_stats.wired_count++;
		sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va));
	}

out:
	PMAP_UNLOCK();
	splx(s);
	return (error);
}

void
pmap_kenter_pa4m(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
	struct pmap *pm = pmap_kernel();
	struct regmap *rp;
	struct segmap *sp;
	int pteproto, vr, vs;

	/* Initialise pteproto with cache bit */
	pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0;
	pteproto |= SRMMU_TEPTE | PPROT_S;
	pteproto |= PMAP_T2PTE_SRMMU(pa);
	pteproto |= (atop(pa & ~PMAP_TNC_SRMMU) << SRMMU_PPNSHIFT);
	pteproto |= pte_kprot4m(prot);

	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];

	KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) != SRMMU_TEPTE);

	setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
}

void
pmap_kremove4m(vaddr_t va, vsize_t len)
{
	struct pmap *pm = pmap_kernel();
	struct regmap *rp;
	struct segmap *sp;
	vaddr_t endva, nva;
	int vr, vs;
	int tpte, perpage, npg, s;

	/*
	 * The kernel pmap doesn't need to be locked, but the demap lock
	 * in updatepte() requires interrupt protection.
	 */
	kpreempt_disable();
	s = splvm();

	endva = va + len;
	for (; va < endva; va = nva) {
		/* do one virtual segment at a time */
		vr = VA_VREG(va);
		vs = VA_VSEG(va);
		nva = VSTOVA(vr, vs + 1);
		if (nva == 0 || nva > endva) {
			nva = endva;
		}

		rp = &pm->pm_regmap[vr];
		sp = &rp->rg_segmap[vs];

		/* decide how to flush the cache */
		npg = (nva - va) >> PGSHIFT;
		if (npg > PMAP_SFL_THRESHOLD) {
			/* flush the whole segment */
			perpage = 0;
			if (CACHEINFO.c_vactype != VAC_NONE) {
				cache_flush_segment(vr, vs, 0);
			}
		} else {
			/*
			 * flush each page individually;
			 * some never need flushing
			 */
			perpage = (CACHEINFO.c_vactype != VAC_NONE);
		}
		for (; va < nva; va += NBPG) {
			tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
			if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
				continue;

			if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
				/* if cacheable, flush page as needed */
				if (perpage && (tpte & SRMMU_PG_C))
					cache_flush_page(va, 0);
			}
			setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
				 SRMMU_TEINVALID, 1, 0, CPUSET_ALL);
		}
	}
	splx(s);
	kpreempt_enable();
}

/*
 * Change protection on a range of kernel addresses.
 */
void
pmap_kprotect4m(vaddr_t va, vsize_t size, vm_prot_t prot)
{
	struct pmap *pm = pmap_kernel();
	int pte, newprot, s;
	struct regmap *rp;
	struct segmap *sp;

	size = roundup(size,NBPG);
	newprot = pte_kprot4m(prot);

	/*
	 * The kernel pmap doesn't need to be locked, but the demap lock
	 * in updatepte() requires interrupt protection.
	 */
	kpreempt_disable();
	s = splvm();

	while (size > 0) {
		rp = &pm->pm_regmap[VA_VREG(va)];
		sp = &rp->rg_segmap[VA_VSEG(va)];
		pte = sp->sg_pte[VA_SUN4M_VPG(va)];

		/*
		 * Flush cache if page has been referenced to
		 * avoid stale protection bits in the cache tags.
		 */
		if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) ==
		    (SRMMU_PG_C|PG_SUN4M_OBMEM))
			cache_flush_page(va, 0);

		setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
			 (pte & ~SRMMU_PROT_MASK) | newprot,
			 1, pm->pm_ctxnum, PMAP_CPUSET(pm));

		va += NBPG;
		size -= NBPG;
	}
	splx(s);
	kpreempt_enable();
}
#endif /* SUN4M || SUN4D */

/*
 * Clear the wiring attribute for a map/virtual-address pair.
 */
/* ARGSUSED */
void
pmap_unwire(struct pmap *pm, vaddr_t va)
{
	int vr, vs, *ptep;
	struct regmap *rp;
	struct segmap *sp;
	bool owired;

	kpreempt_disable();
	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	sp = &rp->rg_segmap[vs];

	owired = false;
	if (CPU_HAS_SUNMMU) {
		ptep = &sp->sg_pte[VA_VPG(va)];
		owired = *ptep & PG_WIRED;
		*ptep &= ~PG_WIRED;
	}
	if (CPU_HAS_SRMMU) {
		owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
		sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
	}
	if (!owired) {
		pmap_stats.ps_useless_changewire++;
		kpreempt_enable();
		return;
	}

	pm->pm_stats.wired_count--;
#if defined(SUN4) || defined(SUN4C)
	if (CPU_HAS_SUNMMU && --sp->sg_nwired <= 0) {
#ifdef DIAGNOSTIC
		if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
			panic("pmap_unwire: pm %p, va %lx: nleft=%d, nwired=%d",
				pm, va, sp->sg_npte, sp->sg_nwired);
#endif
		if (sp->sg_pmeg != seginval)
			mmu_pmeg_unlock(sp->sg_pmeg);
	}
#endif /* SUN4 || SUN4C */
	kpreempt_enable();
}

/*
 * Extract the physical page address associated
 * with the given map/virtual_address pair.
 * GRR, the vm code knows; we should not have to do this!
 */

#if defined(SUN4) || defined(SUN4C)
bool
pmap_extract4_4c(struct pmap *pm, vaddr_t va, paddr_t *pap)
{
	int vr, vs;
	struct regmap *rp;
	struct segmap *sp;
	int pte, *ptep;

	vr = VA_VREG(va);
	vs = VA_VSEG(va);
	rp = &pm->pm_regmap[vr];
	if (rp->rg_segmap == NULL) {
		DPRINTF(PDB_FOLLOW, "pmap_extract: invalid segment (%d)", vr);
		return (false);
	}
	sp = &rp->rg_segmap[vs];
	ptep = sp->sg_pte;
	if (ptep == NULL) {
		DPRINTF(PDB_FOLLOW, "pmap_extract: invalid segment");
		return (false);
	}
	pte = ptep[VA_VPG(va)];

	if ((pte & PG_V) == 0) {
		DPRINTF(PDB_FOLLOW, "pmap_extract: invalid pte");
		return (false);
	}
	pte &= PG_PFNUM;
	if (pap != NULL)
		*pap = (pte << PGSHIFT) | (va & PGOFSET);
	return (true);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of pmap_extract */
/*
 * Extract the physical page address associated
 * with the given map/virtual_address pair.
 * GRR, the vm code knows; we should not have to do this!
 */
bool
pmap_extract4m(struct pmap *pm, vaddr_t va, paddr_t *pap)
{
	struct regmap *rp;
	struct segmap *sp;
	int pte;
	int vr, vs, s, v = false;
	bool can_lock = lock_available;

	vr = VA_VREG(va);
	vs = VA_VSEG(va);

	/*
	 * The kernel pmap doesn't need to be locked, but the demap lock
	 * requires interrupt protection.
	 */
	s = splvm();
	if (pm != pmap_kernel()) {
		PMAP_LOCK();
	}

	rp = &pm->pm_regmap[vr];
	if (rp->rg_segmap == NULL) {
		DPRINTF(PDB_FOLLOW, "pmap_extract: no segmap");
		goto out;
	}

	sp = &rp->rg_segmap[vs];
	if (sp->sg_pte == NULL) {
		DPRINTF(PDB_FOLLOW, "pmap_extract: no ptes");
		goto out;
	}

	pte = sp->sg_pte[VA_SUN4M_VPG(va)];
	if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) {
		DPRINTF(PDB_FOLLOW, "pmap_extract: invalid pte of type %d",
		    pte & SRMMU_TETYPE);
		/*
		 * We can read a spurious invalid pte if the system is in
		 * the middle of the PTE update protocol. So, acquire the
		 * demap lock and retry.
		 */
		if (__predict_true(can_lock))
			mutex_spin_enter(&demap_lock);
		pte = sp->sg_pte[VA_SUN4M_VPG(va)];
		if (__predict_true(can_lock))
			mutex_spin_exit(&demap_lock);
		if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
			goto out;
	}
#ifdef DIAGNOSTIC
	if (pm != pmap_kernel() && sp->sg_npte <= 0)
		panic("pmap_extract: pm %p: npte = %d\n", pm, sp->sg_npte);
#endif

	if (pap != NULL)
		*pap = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT) |
		    VA_OFF(va);

	v = true;
out:
	if (pm != pmap_kernel()) {
		PMAP_UNLOCK();
	}
	splx(s);
	return (v);
}
#endif /* sun4m */

int pmap_copy_disabled=0;

/*
 * Copy the range specified by src_addr/len
 * from the source map to the range dst_addr/len
 * in the destination map.
 *
 * This routine is only advisory and need not do anything.
 */
/* ARGSUSED */
void
pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap,
	  vaddr_t dst_addr, vsize_t len, vaddr_t src_addr)
{
#if notyet
	struct regmap *rp;
	struct segmap *sp;

	if (pmap_copy_disabled)
		return;
#ifdef DIAGNOSTIC
	if (VA_OFF(src_addr) != 0)
		printf("pmap_copy: addr not page aligned: 0x%lx\n", src_addr);
	if ((len & (NBPG-1)) != 0)
		printf("pmap_copy: length not page aligned: 0x%lx\n", len);
#endif

	if (src_pmap == NULL)
		return;

	if (CPU_HAS_SRMMU) {
		int i, npg, pte;
		paddr_t pa;

		npg = len >> PGSHIFT;
		for (i = 0; i < npg; i++) {
			if ((rp = src_pmap->pm_regmap) == NULL)
				continue;
			rp += VA_VREG(src_addr);

			if ((sp = rp->rg_segmap) == NULL)
				continue;
			sp += VA_VSEG(src_addr);
			if (sp->sg_npte == 0)
				continue;

			pte = sp->sg_pte[VA_SUN4M_VPG(src_addr)];
			if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
				continue;

			pa = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT);
			pmap_enter(dst_pmap, dst_addr,
				   pa,
				   /* XXX - need to copy VM_PROT_EXEC too */
				   (pte & PPROT_WRITE)
					? (VM_PROT_WRITE | VM_PROT_READ)
					: VM_PROT_READ,
				   0);
			src_addr += NBPG;
			dst_addr += NBPG;
		}
		pmap_update(dst_pmap);
	}
#endif
}

#if defined(SUN4) || defined(SUN4C)
/*
 * Clear the modify bit for the given physical page.
 */
bool
pmap_clear_modify4_4c(struct vm_page *pg)
{
	bool rv;

	(void) pv_syncflags4_4c(pg);
	rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD;
	VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD;
	return (rv);
}

/*
 * Tell whether the given physical page has been modified.
 */
bool
pmap_is_modified4_4c(struct vm_page *pg)
{

	return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD ||
		pv_syncflags4_4c(pg) & PV_MOD);
}

/*
 * Clear the reference bit for the given physical page.
 */
bool
pmap_clear_reference4_4c(struct vm_page *pg)
{
	bool rv;

	(void) pv_syncflags4_4c(pg);
	rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF;
	VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF;
	return (rv);
}

/*
 * Tell whether the given physical page has been referenced.
 */
bool
pmap_is_referenced4_4c(struct vm_page *pg)
{

	return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF ||
		pv_syncflags4_4c(pg) & PV_REF);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)

/*
 * SRMMU versions of bit test/set routines
 *
 * Note that the 4m-specific routines should eventually service these
 * requests from their page tables, and the whole pvlist bit mess should
 * be dropped for the 4m (unless this causes a performance hit from
 * tracing down pagetables/regmap/segmaps).
 */

/*
 * Clear the modify bit for the given physical page.
 */
bool
pmap_clear_modify4m(struct vm_page *pg)
{
	bool rv;

	(void) pv_syncflags4m(pg);
	rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M;
	VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD4M;
	return (rv);
}

/*
 * Tell whether the given physical page has been modified.
 */
bool
pmap_is_modified4m(struct vm_page *pg)
{

	return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M ||
		pv_syncflags4m(pg) & PV_MOD4M);
}

/*
 * Clear the reference bit for the given physical page.
 */
bool
pmap_clear_reference4m(struct vm_page *pg)
{
	bool rv;

	(void) pv_syncflags4m(pg);
	rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M;
	VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF4M;
	return (rv);
}

/*
 * Tell whether the given physical page has been referenced.
 */
bool
pmap_is_referenced4m(struct vm_page *pg)
{

	return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M ||
		pv_syncflags4m(pg) & PV_REF4M);
}
#endif /* SUN4M || SUN4D */

/*
 * Fill the given MI physical page with zero bytes.
 *
 * We avoid stomping on the cache.
 * XXX	might be faster to use destination's context and allow cache to fill?
 */

#if defined(SUN4) || defined(SUN4C)

void
pmap_zero_page4_4c(paddr_t pa)
{
	struct vm_page *pg;
	void *va;
	int pte;

	if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
		/*
		 * The following might not be necessary since the page
		 * is being cleared because it is about to be allocated,
		 * i.e., is in use by no one.
		 */
		pv_flushcache4_4c(pg);
	}
	pte = PG_V | PG_S | PG_W | PG_NC | (atop(pa) & PG_PFNUM);

	va = cpuinfo.vpage[0];
	setpte4(va, pte);
	qzero(va, NBPG);
	setpte4(va, 0);
}

/*
 * Copy the given MI physical source page to its destination.
 *
 * We avoid stomping on the cache as above (with same `XXX' note).
 * We must first flush any write-back cache for the source page.
 * We go ahead and stomp on the kernel's virtual cache for the
 * source page, since the cache can read memory MUCH faster than
 * the processor.
 */
void
pmap_copy_page4_4c(paddr_t src, paddr_t dst)
{
	struct vm_page *pg;
	char *sva, *dva;
	int spte, dpte;

	if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
		if (CACHEINFO.c_vactype == VAC_WRITEBACK)
			pv_flushcache4_4c(pg);
	}
	spte = PG_V | PG_S | (atop(src) & PG_PFNUM);

	if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
		/* similar `might not be necessary' comment applies */
		if (CACHEINFO.c_vactype != VAC_NONE)
			pv_flushcache4_4c(pg);
	}
	dpte = PG_V | PG_S | PG_W | PG_NC | (atop(dst) & PG_PFNUM);

	sva = cpuinfo.vpage[0];
	dva = cpuinfo.vpage[1];
	setpte4(sva, spte);
	setpte4(dva, dpte);
	qcopy(sva, dva, NBPG);	/* loads cache, so we must ... */
	cache_flush_page((vaddr_t)sva, getcontext4());
	setpte4(sva, 0);
	setpte4(dva, 0);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)	/* SRMMU version of copy/zero routines */
/*
 * Fill the given MI physical page with zero bytes.
 *
 * We avoid stomping on the cache.
 * XXX	might be faster to use destination's context and allow cache to fill?
 */
void
pmap_zero_page4m(paddr_t pa)
{
	struct vm_page *pg;
	void *va;
	int pte;

	kpreempt_disable();
	if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
		/*
		 * The following VAC flush might not be necessary since the
		 * page is being cleared because it is about to be allocated,
		 * i.e., is in use by no one.
		 * In the case of a physical cache, a flush (or just an
		 * invalidate, if possible) is usually necessary when using
		 * uncached access to clear it.
		 */
		if (CACHEINFO.c_vactype != VAC_NONE)
			pv_flushcache4m(pg);
		else
			pcache_flush_page(pa, 1);
	}
	pte = SRMMU_TEPTE | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT);
	if (CACHEINFO.c_flags & CACHE_MANDATORY)
		pte |= SRMMU_PG_C;

	va = cpuinfo.vpage[0];
	setpgt4m(cpuinfo.vpage_pte[0], pte);
	qzero(va, NBPG);
	/*
	 * Remove temporary mapping (which is kernel-only, so the
	 * context used for TLB flushing does not matter)
	 */
	sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3);
	setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
	kpreempt_enable();
}

/*
 * Viking/MXCC specific version of pmap_zero_page
 */
void
pmap_zero_page_viking_mxcc(paddr_t pa)
{
	u_int offset;
	u_int stream_data_addr = MXCC_STREAM_DATA;
	uint64_t v = (uint64_t)pa;

	kpreempt_disable();
	/* Load MXCC stream data register with 0 (bottom 32 bytes only) */
	stda(stream_data_addr+0, ASI_CONTROL, 0);
	stda(stream_data_addr+8, ASI_CONTROL, 0);
	stda(stream_data_addr+16, ASI_CONTROL, 0);
	stda(stream_data_addr+24, ASI_CONTROL, 0);

	/* Then write the stream data register to each block in the page */
	v |= MXCC_STREAM_C;
	for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) {
		stda(MXCC_STREAM_DST, ASI_CONTROL, v | offset);
	}
	kpreempt_enable();
}

/*
 * HyperSPARC/RT625 specific version of pmap_zero_page
 */
void
pmap_zero_page_hypersparc(paddr_t pa)
{
	struct vm_page *pg;
	void *va;
	int pte;
	int offset;

	kpreempt_disable();
	/*
	 * We still have to map the page, since ASI_BLOCKFILL
	 * takes virtual addresses. This also means we have to
	 * consider cache aliasing; therefore we still need
	 * to flush the cache here. All we gain is the speed-up
	 * in zero-fill loop itself..
	 */
	if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
		/*
		 * The following might not be necessary since the page
		 * is being cleared because it is about to be allocated,
		 * i.e., is in use by no one.
		 */
		if (CACHEINFO.c_vactype != VAC_NONE)
			pv_flushcache4m(pg);
	}
	pte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT);

	va = cpuinfo.vpage[0];
	setpgt4m(cpuinfo.vpage_pte[0], pte);
	for (offset = 0; offset < NBPG; offset += 32) {
		sta((char *)va + offset, ASI_BLOCKFILL, 0);
	}
	/* Remove temporary mapping */
	sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3);
	setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
	kpreempt_enable();
}

/*
 * Copy the given MI physical source page to its destination.
 *
 * We avoid stomping on the cache as above (with same `XXX' note).
 * We must first flush any write-back cache for the source page.
 * We go ahead and stomp on the kernel's virtual cache for the
 * source page, since the cache can read memory MUCH faster than
 * the processor.
 */
void
pmap_copy_page4m(paddr_t src, paddr_t dst)
{
	struct vm_page *pg;
	void *sva, *dva;
	int spte, dpte;

	kpreempt_disable();
	if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
		if (CACHEINFO.c_vactype == VAC_WRITEBACK)
			pv_flushcache4m(pg);
	}

	spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX |
		(src >> SRMMU_PPNPASHIFT);

	if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
		/* similar `might not be necessary' comment applies */
		if (CACHEINFO.c_vactype != VAC_NONE)
			pv_flushcache4m(pg);
		else
			pcache_flush_page(dst, 1);
	}

	dpte = SRMMU_TEPTE | PPROT_N_RWX | (dst >> SRMMU_PPNPASHIFT);
	if (CACHEINFO.c_flags & CACHE_MANDATORY)
		dpte |= SRMMU_PG_C;

	sva = cpuinfo.vpage[0];
	dva = cpuinfo.vpage[1];
	setpgt4m(cpuinfo.vpage_pte[0], spte);
	setpgt4m(cpuinfo.vpage_pte[1], dpte);
	qcopy(sva, dva, NBPG);	/* loads cache, so we must ... */
	cpuinfo.sp_vcache_flush_page((vaddr_t)sva, getcontext4m());
	sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3);
	setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
	sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3);
	setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID);
	kpreempt_enable();
}

/*
 * Viking/MXCC specific version of pmap_copy_page
 */
void
pmap_copy_page_viking_mxcc(paddr_t src, paddr_t dst)
{
	u_int offset;
	uint64_t v1 = (uint64_t)src;
	uint64_t v2 = (uint64_t)dst;

	kpreempt_disable();
	/* Enable cache-coherency */
	v1 |= MXCC_STREAM_C;
	v2 |= MXCC_STREAM_C;

	/* Copy through stream data register */
	for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) {
		stda(MXCC_STREAM_SRC, ASI_CONTROL, v1 | offset);
		stda(MXCC_STREAM_DST, ASI_CONTROL, v2 | offset);
	}
	kpreempt_enable();
}

/*
 * HyperSPARC/RT625 specific version of pmap_copy_page
 */
void
pmap_copy_page_hypersparc(paddr_t src, paddr_t dst)
{
	struct vm_page *pg;
	void *sva, *dva;
	int spte, dpte;
	int offset;

	kpreempt_disable();
	/*
	 * We still have to map the pages, since ASI_BLOCKCOPY
	 * takes virtual addresses. This also means we have to
	 * consider cache aliasing; therefore we still need
	 * to flush the cache here. All we gain is the speed-up
	 * in copy loop itself..
	 */

	if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
		if (CACHEINFO.c_vactype == VAC_WRITEBACK)
			pv_flushcache4m(pg);
	}

	spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX |
		(src >> SRMMU_PPNPASHIFT);

	if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
		/* similar `might not be necessary' comment applies */
		if (CACHEINFO.c_vactype != VAC_NONE)
			pv_flushcache4m(pg);
	}

	dpte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX |
		(dst >> SRMMU_PPNPASHIFT);

	sva = cpuinfo.vpage[0];
	dva = cpuinfo.vpage[1];
	setpgt4m(cpuinfo.vpage_pte[0], spte);
	setpgt4m(cpuinfo.vpage_pte[1], dpte);

	for (offset = 0; offset < NBPG; offset += 32) {
		sta((char *)dva + offset, ASI_BLOCKCOPY, (char *)sva + offset);
	}

	sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3);
	setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
	sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3);
	setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID);
	kpreempt_enable();
}
#endif /* SUN4M || SUN4D */

/*
 * Turn off cache for a given (va, number of pages).
 *
 * We just assert PG_NC for each PTE; the addresses must reside
 * in locked kernel space.  A cache flush is also done.
 */
void
kvm_uncache(char *va, int npages)
{
	struct vm_page *pg;
	int pte;

	if (CPU_HAS_SRMMU) {
#if defined(SUN4M) || defined(SUN4D)
		for (; --npages >= 0; va = (char *)va + NBPG) {
			pte = getpte4m((vaddr_t) va);
			if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
				panic("kvm_uncache: table entry not pte");

			if ((pte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
				if ((pg = pvhead4m(pte)) != NULL) {
					pv_uncache(pg);
					return;
				}
				cache_flush_page((vaddr_t)va, 0);
			}

			pte &= ~SRMMU_PG_C;
			setpte4m((vaddr_t)va, pte);
		}
#endif
	} else {
#if defined(SUN4) || defined(SUN4C)
		for (; --npages >= 0; va += NBPG) {
			pte = getpte4(va);
			if ((pte & PG_V) == 0)
				panic("kvm_uncache !pg_v");

			if ((pte & PG_TYPE) == PG_OBMEM) {
				if ((pg = pvhead4_4c(pte)) != NULL) {
					pv_uncache(pg);
					return;
				}
				cache_flush_page((vaddr_t)va, 0);
			}
			pte |= PG_NC;
			setpte4(va, pte);
		}
#endif
	}
}

#if 0 /* not used */
/*
 * Turn on IO cache for a given (va, number of pages).
 *
 * We just assert PG_NC for each PTE; the addresses must reside
 * in locked kernel space.  A cache flush is also done.
 */
void
kvm_iocache(char *va, int npages)
{

#if defined(SUN4M)
	if (CPU_ISSUN4M) /* %%%: Implement! */
		panic("kvm_iocache: 4m iocache not implemented");
#endif
#if defined(SUN4D)
	if (CPU_ISSUN4D) /* %%%: Implement! */
		panic("kvm_iocache: 4d iocache not implemented");
#endif
#if defined(SUN4) || defined(SUN4C)
	for (; --npages >= 0; va += NBPG) {
		int pte = getpte4(va);
		if ((pte & PG_V) == 0)
			panic("kvm_iocache !pg_v");
		pte |= PG_IOC;
		setpte4(va, pte);
	}
#endif
}
#endif

/*
 * Find first virtual address >= *va that is
 * least likely to cause cache aliases.
 * (This will just seg-align mappings.)
 */
void
pmap_prefer(vaddr_t foff, vaddr_t *vap, size_t size, int td)
{
	vaddr_t va = *vap;
	long m;

	m = CACHE_ALIAS_DIST;
	if (m == 0)		/* m=0 => no cache aliasing */
		return;

	if (VA_INHOLE(va)) {
		if (td)
			va = MMU_HOLE_START - size;
		else
			va = MMU_HOLE_END;
	}

	va = (va & ~(m - 1)) | (foff & (m - 1));

	if (td) {
		if (va > *vap)
			va -= m;
	} else {
		if (va < *vap)
			va += m;
	}
	*vap = va;
}

void
pmap_redzone(void)
{

	pmap_remove(pmap_kernel(), KERNBASE, KERNBASE+NBPG);
}

/*
 * Activate the address space for the specified process.  If the
 * process is the current process, load the new MMU context.
 */
void
pmap_activate(struct lwp *l)
{
	pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap;

	if (pm == pmap_kernel() || l != curlwp) {
		return;
	}

	mutex_spin_enter(&ctx_lock);
	if (pm->pm_ctx == NULL) {
		ctx_alloc(pm);	/* performs setcontext() */
	} else {
		setcontext(pm->pm_ctxnum);
	}
	PMAP_SET_CPUSET(pm, &cpuinfo);
	mutex_spin_exit(&ctx_lock);
}

/*
 * Deactivate the address space of the specified process.
 */
void
pmap_deactivate(struct lwp *l)
{
	struct proc *p = l->l_proc;
	pmap_t pm = p->p_vmspace->vm_map.pmap;

	if (pm == pmap_kernel() || l != curlwp) {
		return;
	}

	write_user_windows();
	mutex_spin_enter(&ctx_lock);
	if (pm->pm_ctx) {
		(*cpuinfo.pure_vcache_flush)();

#if defined(SUN4M) || defined(SUN4D)
		if (CPU_HAS_SRMMU)
			sp_tlb_flush(0, pm->pm_ctxnum, ASI_SRMMUFP_L0);
#endif
	}

	/* we no longer need broadcast tlb flushes for this pmap. */
	PMAP_CLR_CPUSET(pm, &cpuinfo);
	mutex_spin_exit(&ctx_lock);
}

#ifdef DEBUG
/*
 * Check consistency of a pmap (time consuming!).
 */
void
pm_check(char *s, struct pmap *pm)
{

	if (pm == pmap_kernel())
		pm_check_k(s, pm);
	else
		pm_check_u(s, pm);
}

void
pm_check_u(char *s, struct pmap *pm)
{
	struct regmap *rp;
	struct segmap *sp;
	int cpu, n, vs, vr, j, m, *pte;

	cpu = cpuinfo.ci_cpuid;

	if (pm->pm_regmap == NULL)
		panic("%s: CPU %d: CHK(pmap %p): no region mapping",
			s, cpu, pm);

#if defined(SUN4M) || defined(SUN4D)
	if (CPU_HAS_SRMMU &&
	    (pm->pm_reg_ptps[cpu] == NULL ||
	     pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu])))
		panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: "
		      "tblva=%p, tblpa=0x%x",
			s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]);

	if (CPU_HAS_SRMMU && pm->pm_ctx != NULL &&
	    (cpuinfo.ctx_tbl[pm->pm_ctxnum] != ((VA2PA((void *)pm->pm_reg_ptps[cpu])
					      >> SRMMU_PPNPASHIFT) |
					     SRMMU_TEPTD)))
	    panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed "
		  "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], pm->pm_ctxnum);
#endif

	for (vr = 0; vr < NUREG; vr++) {
		rp = &pm->pm_regmap[vr];
		if (rp->rg_nsegmap == 0)
			continue;
		if (rp->rg_segmap == NULL)
			panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL",
				s, cpu, vr, rp->rg_nsegmap);
#if defined(SUN4M) || defined(SUN4D)
		if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL)
		    panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table",
			  s, cpu, vr, rp->rg_nsegmap);
		if (CPU_HAS_SRMMU &&
		    pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >>
					    SRMMU_PPNPASHIFT) | SRMMU_TEPTD))
		    panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed",
				s, cpu, vr);
#endif
		if ((unsigned int)rp < KERNBASE)
			panic("%s: CPU %d: rp=%p", s, cpu, rp);
		n = 0;
		for (vs = 0; vs < NSEGRG; vs++) {
			sp = &rp->rg_segmap[vs];
			if ((unsigned int)sp < KERNBASE)
				panic("%s: CPU %d: sp=%p", s, cpu, sp);
			if (sp->sg_npte != 0) {
				n++;
				if (sp->sg_pte == NULL)
					panic("%s: CPU %d: CHK(vr %d, vs %d): npte=%d, "
					   "pte=NULL", s, cpu, vr, vs, sp->sg_npte);
#if defined(SUN4M) || defined(SUN4D)
				if (CPU_HAS_SRMMU &&
				    rp->rg_seg_ptps[vs] !=
				     ((VA2PA((void *)sp->sg_pte)
					>> SRMMU_PPNPASHIFT) |
				       SRMMU_TEPTD))
				    panic("%s: CPU %d: CHK(vr %d, vs %d): SRMMU page "
					  "table not installed correctly",
						s, cpu, vr, vs);
#endif
				pte=sp->sg_pte;
				m = 0;
				for (j=0; j<NPTESG; j++,pte++)
				    if ((CPU_HAS_SRMMU
					 ?((*pte & SRMMU_TETYPE) == SRMMU_TEPTE)
					 :(*pte & PG_V)))
					m++;
				if (m != sp->sg_npte)
					printf("%s: CPU %d: user CHK(vr %d, vs %d): "
					    "npte(%d) != # valid(%d)\n",
						s, cpu, vr, vs, sp->sg_npte, m);
			}
		}
		if (n != rp->rg_nsegmap)
			panic("%s: CPU %d: CHK(vr %d): inconsistent "
				"# of pte's: %d, should be %d",
				s, cpu, vr, rp->rg_nsegmap, n);
	}
	return;
}

/* Note: not as extensive as pm_check_u. */
void
pm_check_k(char *s, struct pmap *pm)
{
	struct regmap *rp;
	int cpu, vr, vs, n;

	cpu = cpu_number();

	if (pm->pm_regmap == NULL)
		panic("%s: CHK(pmap %p): no region mapping", s, pm);

#if defined(SUN4M) || defined(SUN4D)
	if (CPU_HAS_SRMMU &&
	    (pm->pm_reg_ptps[cpu] == NULL ||
	     pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu])))
	    panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: tblva=%p, tblpa=0x%x",
		  s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]);

	if (CPU_HAS_SRMMU &&
	    (cpuinfo.ctx_tbl[0] != ((VA2PA((void *)pm->pm_reg_ptps[cpu]) >>
					     SRMMU_PPNPASHIFT) | SRMMU_TEPTD)))
	    panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed "
		  "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], 0);
#endif
	for (vr = NUREG; vr < NUREG+NKREG; vr++) {
		rp = &pm->pm_regmap[vr];
		if (rp->rg_segmap == NULL)
			panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL",
				s, cpu, vr, rp->rg_nsegmap);
		if (rp->rg_nsegmap == 0)
			continue;
#if defined(SUN4M) || defined(SUN4D)
		if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL)
		    panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table",
			  s, cpu, vr, rp->rg_nsegmap);

		if (CPU_HAS_SRMMU && vr != NUREG /* 1st kseg is per CPU */ &&
		    pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >>
					    SRMMU_PPNPASHIFT) | SRMMU_TEPTD))
		    panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed",
				s, cpu, vr);
#endif
		if (CPU_HAS_SRMMU) {
			n = NSEGRG;
		} else {
			for (n = 0, vs = 0; vs < NSEGRG; vs++) {
				if (rp->rg_segmap[vs].sg_npte)
					n++;
			}
		}
		if (n != rp->rg_nsegmap)
			printf("%s: CPU %d: kernel CHK(vr %d): inconsistent "
				"# of pte's: %d, should be %d\n",
				s, cpu, vr, rp->rg_nsegmap, n);
	}
	return;
}
#endif

/*
 * Return the number of disk blocks that pmap_dumpmmu() will dump.
 */
int
pmap_dumpsize(void)
{
	int	sz;

	sz = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t));
	sz += npmemarr * sizeof(phys_ram_seg_t);
	sz += sizeof(kernel_segmap_store);

	if (CPU_HAS_SUNMMU)
		/* For each pmeg in the MMU, we'll write NPTESG PTEs. */
		sz += (seginval + 1) * NPTESG * sizeof(int);

	return btodb(sz + DEV_BSIZE - 1);
}

/*
 * Write the core dump headers and MD data to the dump device.
 * We dump the following items:
 *
 *	kcore_seg_t		 MI header defined in <sys/kcore.h>)
 *	cpu_kcore_hdr_t		 MD header defined in <machine/kcore.h>)
 *	phys_ram_seg_t[npmemarr] physical memory segments
 *	segmap_t[NKREG*NSEGRG]	 the kernel's segment map
 *	the MMU pmegs on sun4/sun4c
 */
int
pmap_dumpmmu(int (*dump)(dev_t, daddr_t, void *, size_t),
	     daddr_t blkno)
{
	kcore_seg_t	*ksegp;
	cpu_kcore_hdr_t	*kcpup;
	phys_ram_seg_t	memseg;
	int		error = 0;
	int		i, memsegoffset, segmapoffset, pmegoffset;
	int		buffer[dbtob(1) / sizeof(int)];
	int		*bp, *ep;
#if defined(SUN4C) || defined(SUN4)
	int	pmeg;
#endif

#define EXPEDITE(p,n) do {						\
	int *sp = (int *)(p);						\
	int sz = (n);							\
	while (sz > 0) {						\
		*bp++ = *sp++;						\
		if (bp >= ep) {						\
			error = (*dump)(dumpdev, blkno,			\
					(void *)buffer, dbtob(1));	\
			if (error != 0)					\
				return (error);				\
			++blkno;					\
			bp = buffer;					\
		}							\
		sz -= 4;						\
	}								\
} while (0)

	setcontext(0);

	/* Setup bookkeeping pointers */
	bp = buffer;
	ep = &buffer[sizeof(buffer) / sizeof(buffer[0])];

	/* Fill in MI segment header */
	ksegp = (kcore_seg_t *)bp;
	CORE_SETMAGIC(*ksegp, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
	ksegp->c_size = dbtob(pmap_dumpsize()) - ALIGN(sizeof(kcore_seg_t));

	/* Fill in MD segment header (interpreted by MD part of libkvm) */
	kcpup = (cpu_kcore_hdr_t *)((int)bp + ALIGN(sizeof(kcore_seg_t)));
	kcpup->cputype = cputyp;
	kcpup->kernbase = KERNBASE;
	kcpup->nmemseg = npmemarr;
	kcpup->memsegoffset = memsegoffset = ALIGN(sizeof(cpu_kcore_hdr_t));
	kcpup->nsegmap = NKREG*NSEGRG;
	kcpup->segmapoffset = segmapoffset =
		memsegoffset + npmemarr * sizeof(phys_ram_seg_t);

	kcpup->npmeg = (CPU_HAS_SUNMMU) ? seginval + 1 : 0;
	kcpup->pmegoffset = pmegoffset =
		segmapoffset + kcpup->nsegmap * sizeof(struct segmap);

	/* Note: we have assumed everything fits in buffer[] so far... */
	bp = (int *)((int)kcpup + ALIGN(sizeof(cpu_kcore_hdr_t)));

#if 0
	/* Align storage for upcoming quad-aligned segment array */
	while (bp != (int *)ALIGN(bp)) {
		int dummy = 0;
		EXPEDITE(&dummy, 4);
	}
#endif

	for (i = 0; i < npmemarr; i++) {
		memseg.start = pmemarr[i].addr;
		memseg.size = pmemarr[i].len;
		EXPEDITE((void *)&memseg, sizeof(phys_ram_seg_t));
	}

	EXPEDITE(&kernel_segmap_store, sizeof(kernel_segmap_store));

	if (CPU_HAS_SRMMU)
		goto out;

#if defined(SUN4C) || defined(SUN4)
	/*
	 * dump page table entries
	 *
	 * We dump each pmeg in order (by segment number).  Since the MMU
	 * automatically maps the given virtual segment to a pmeg we must
	 * iterate over the segments by incrementing an unused segment slot
	 * in the MMU.  This fixed segment number is used in the virtual
	 * address argument to getpte().
	 */

	/*
	 * Go through the pmegs and dump each one.
	 */
	for (pmeg = 0; pmeg <= seginval; ++pmeg) {
		int va = 0;

		setsegmap(va, pmeg);
		i = NPTESG;
		do {
			int pte = getpte4(va);
			EXPEDITE(&pte, sizeof(pte));
			va += NBPG;
		} while (--i > 0);
	}
	setsegmap(0, seginval);
#endif

out:
	if (bp != buffer)
		error = (*dump)(dumpdev, blkno++, (void *)buffer, dbtob(1));

	return (error);
}

/*
 * Helper function for debuggers.
 */
void
pmap_writetext(unsigned char *dst, int ch)
{
	int s, pte0, pte, ctx;
	vaddr_t va;

	s = splvm();
	va = (unsigned long)dst & (~PGOFSET);
	cache_flush(dst, 1);

	ctx = getcontext();
	setcontext(0);

#if defined(SUN4M) || defined(SUN4D)
	if (CPU_HAS_SRMMU) {
		pte0 = getpte4m(va);
		if ((pte0 & SRMMU_TETYPE) != SRMMU_TEPTE) {
			goto out;
		}
		pte = pte0 | PPROT_WRITE;
		setpte4m(va, pte);
		*dst = (unsigned char)ch;
		setpte4m(va, pte0);

	}
#endif
#if defined(SUN4) || defined(SUN4C)
	if (CPU_ISSUN4C || CPU_ISSUN4) {
		pte0 = getpte4(va);
		if ((pte0 & PG_V) == 0) {
			goto out;
		}
		pte = pte0 | PG_W;
		setpte4(va, pte);
		*dst = (unsigned char)ch;
		setpte4(va, pte0);
	}
#endif
	cache_flush(dst, 1);

out:
	setcontext(ctx);
	splx(s);
}

#ifdef EXTREME_DEBUG

void debug_pagetables(void);
void print_fe_map(void);

static void test_region(int, int, int);


void
debug_pagetables(void)
{
	struct promvec *promvec = romp;
	int *regtbl;
	int te;
	int i;

	printf("\nncontext=%d. ", ncontext);
	printf("Context table is at va %p. Level 0 PTP: 0x%x\n",
	       cpuinfo.ctx_tbl, cpuinfo.ctx_tbl[0]);
	printf("Context 0 region table is at va %p, pa 0x%x. Contents:\n",
	       pmap_kernel()->pm_reg_ptps[0], pmap_kernel()->pm_reg_ptps_pa[0]);

	regtbl = pmap_kernel()->pm_reg_ptps[0];

	printf("PROM vector is at %p\n", promvec);
	printf("PROM reboot routine is at %p\n", promvec->pv_reboot);
	printf("PROM abort routine is at %p\n", promvec->pv_abort);
	printf("PROM halt routine is at %p\n", promvec->pv_halt);

	printf("Testing region 0xfe: ");
	test_region(0xfe,0,16*1024*1024);
	printf("Testing region 0xff: ");
	test_region(0xff,0,16*1024*1024);
	printf("Testing kernel region 0x%x: ", VA_VREG(KERNBASE));
	test_region(VA_VREG(KERNBASE), 4096, avail_start);
	cngetc();

	for (i = 0; i < SRMMU_L1SIZE; i++) {
		te = regtbl[i];
		if ((te & SRMMU_TETYPE) == SRMMU_TEINVALID)
		    continue;
		printf("Region 0x%x: PTE=0x%x <%s> L2PA=0x%x kernL2VA=%p\n",
		       i, te, ((te & SRMMU_TETYPE) == SRMMU_TEPTE ? "pte" :
			       ((te & SRMMU_TETYPE) == SRMMU_TEPTD ? "ptd" :
				((te & SRMMU_TETYPE) == SRMMU_TEINVALID ?
				 "invalid" : "reserved"))),
		       (te & ~0x3) << SRMMU_PPNPASHIFT,
		       pmap_kernel()->pm_regmap[i].rg_seg_ptps);
	}
	printf("Press q to halt...\n");
	if (cngetc()=='q')
	    callrom();
}

static u_int
VA2PAsw(int ctx, void *addr, int *pte)
{
	int *curtbl;
	int curpte;

#ifdef EXTREME_EXTREME_DEBUG
	printf("Looking up addr 0x%x in context 0x%x\n",addr,ctx);
#endif
	/* L0 */
	*pte = curpte = cpuinfo.ctx_tbl[ctx];
#ifdef EXTREME_EXTREME_DEBUG
	printf("Got L0 pte 0x%x\n",pte);
#endif
	if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) {
		return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
			((u_int)addr & 0xffffffff));
	}
	if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
		printf("Bad context table entry 0x%x for context 0x%x\n",
		       curpte, ctx);
		return 0;
	}
	/* L1 */
	curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
	*pte = curpte = curtbl[VA_VREG(addr)];
#ifdef EXTREME_EXTREME_DEBUG
	printf("L1 table at 0x%x.\nGot L1 pte 0x%x\n",curtbl,curpte);
#endif
	if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
	    return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
		    ((u_int)addr & 0xffffff));
	if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
		printf("Bad region table entry 0x%x for region 0x%x\n",
		       curpte, VA_VREG(addr));
		return 0;
	}
	/* L2 */
	curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
	*pte = curpte = curtbl[VA_VSEG(addr)];
#ifdef EXTREME_EXTREME_DEBUG
	printf("L2 table at 0x%x.\nGot L2 pte 0x%x\n",curtbl,curpte);
#endif
	if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
	    return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
		    ((u_int)addr & 0x3ffff));
	if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
		printf("Bad segment table entry 0x%x for reg 0x%x, seg 0x%x\n",
		       curpte, VA_VREG(addr), VA_VSEG(addr));
		return 0;
	}
	/* L3 */
	curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
	*pte = curpte = curtbl[VA_VPG(addr)];
#ifdef EXTREME_EXTREME_DEBUG
	printf("L3 table at %p.\nGot L3 pte 0x%x\n", curtbl, curpte);
#endif
	if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
	    return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
		    ((u_int)addr & 0xfff));
	else {
		printf("Bad L3 pte 0x%x for reg 0x%x, seg 0x%x, pg 0x%x\n",
		       curpte, VA_VREG(addr), VA_VSEG(addr), VA_VPG(addr));
		return 0;
	}
	printf("Bizarreness with address %p!\n", addr);
}

static void
test_region(int reg, int start, int stop)
{
	int i;
	int addr;
	int pte;
	int ptesw;
/*	int cnt=0;
*/

	for (i = start; i < stop; i += NBPG) {
		addr = (reg << RGSHIFT) | i;
		pte = lda(((u_int)(addr)) | ASI_SRMMUFP_LN, ASI_SRMMUFP);
		if (pte) {
/*			printf("Valid address 0x%x\n",addr);
			if (++cnt == 20) {
				cngetc();
				cnt = 0;
			}
*/
			if (VA2PA((void *)addr) != VA2PAsw(0, (void *)addr, &ptesw)) {
				printf("Mismatch at address 0x%x.\n", addr);
				if (cngetc() == 'q')
					break;
			}
			if (reg == VA_VREG(KERNBASE))
				/* kernel permissions are different */
				continue;
			if ((pte & SRMMU_PROT_MASK) != (ptesw & SRMMU_PROT_MASK)) {
				printf("Mismatched protections at address "
				       "0x%x; pte=0x%x, ptesw=0x%x\n",
				       addr, pte, ptesw);
				if (cngetc() == 'q')
					break;
			}
		}
	}
	printf("done.\n");
}


void
print_fe_map(void)
{
	u_int i, pte;

	printf("map of region 0xfe:\n");
	for (i = 0xfe000000; i < 0xff000000; i += 4096) {
		if (((pte = getpte4m(i)) & SRMMU_TETYPE) != SRMMU_TEPTE)
		    continue;
		printf("0x%x -> 0x%x%x (pte 0x%x)\n", i, pte >> 28,
		       (pte & ~0xff) << 4, pte);
	}
	printf("done\n");
}
#endif /* EXTREME_DEBUG */

#ifdef DDB
int pmap_dump(struct pmap *pm);

int
pmap_dump(struct pmap *pm)
{
	int startvr, endvr, vr, vs, i, n;
	struct regmap *rp;
	struct segmap *sp;

	if (pm == NULL)
		pm = pmap_kernel();

	if (pm == pmap_kernel()) {
		startvr = NUREG;
		endvr = 256;
	} else {
		startvr = 0;
		endvr = NUREG;
	}

	for (vr = startvr; vr < endvr; vr++) {
		rp = &pm->pm_regmap[vr];
		if (rp->rg_nsegmap == 0)
			continue;
		printf("vr %d: %d segments", vr, rp->rg_nsegmap);
		if (rp->rg_segmap == NULL) {
			printf("[no segments]\n");
			continue;
		}
		for (vs = 0; vs < NSEGRG; vs++) {
			sp = &rp->rg_segmap[vs];
			if (sp->sg_npte == 0)
				continue;
			if ((vs & 3) == 0)
				printf("\n   ");
			printf(" %d: n %d w %d p %d,", vs,
				sp->sg_npte, sp->sg_nwired, sp->sg_pmeg);
			if (sp->sg_pte == NULL) {
				printf("[no ptes]");
				continue;
			}
			for (n = 0, i = 0; i < NPTESG; i++) {
				if (CPU_HAS_SUNMMU && sp->sg_pte[i] & PG_WIRED)
					n++;
				if (CPU_HAS_SRMMU && sp->sg_wiremap & (1 << i))
					n++;
			}
			if (n != sp->sg_nwired)
				printf("[wired count %d]", n);
		}
		printf("\n");
	}

	return (0);
}
#endif /* DDB */