1da673940SJordan Gordeev /*
2da673940SJordan Gordeev * Copyright (c) 1991 Regents of the University of California.
3da673940SJordan Gordeev * Copyright (c) 1994 John S. Dyson
4da673940SJordan Gordeev * Copyright (c) 1994 David Greenman
5da673940SJordan Gordeev * Copyright (c) 2003 Peter Wemm
6da673940SJordan Gordeev * Copyright (c) 2005-2008 Alan L. Cox <alc@cs.rice.edu>
7a70dbf04SMatthew Dillon * Copyright (c) 2008-2019 The DragonFly Project.
8da673940SJordan Gordeev * Copyright (c) 2008, 2009 Jordan Gordeev.
9da673940SJordan Gordeev * All rights reserved.
10da673940SJordan Gordeev *
11da673940SJordan Gordeev * This code is derived from software contributed to Berkeley by
12da673940SJordan Gordeev * the Systems Programming Group of the University of Utah Computer
13da673940SJordan Gordeev * Science Department and William Jolitz of UUNET Technologies Inc.
14da673940SJordan Gordeev *
15da673940SJordan Gordeev * Redistribution and use in source and binary forms, with or without
16da673940SJordan Gordeev * modification, are permitted provided that the following conditions
17da673940SJordan Gordeev * are met:
18da673940SJordan Gordeev * 1. Redistributions of source code must retain the above copyright
19da673940SJordan Gordeev * notice, this list of conditions and the following disclaimer.
20da673940SJordan Gordeev * 2. Redistributions in binary form must reproduce the above copyright
21da673940SJordan Gordeev * notice, this list of conditions and the following disclaimer in the
22da673940SJordan Gordeev * documentation and/or other materials provided with the distribution.
23da673940SJordan Gordeev * 3. All advertising materials mentioning features or use of this software
24da673940SJordan Gordeev * must display the following acknowledgement:
25da673940SJordan Gordeev * This product includes software developed by the University of
26da673940SJordan Gordeev * California, Berkeley and its contributors.
27da673940SJordan Gordeev * 4. Neither the name of the University nor the names of its contributors
28da673940SJordan Gordeev * may be used to endorse or promote products derived from this software
29da673940SJordan Gordeev * without specific prior written permission.
30da673940SJordan Gordeev *
31da673940SJordan Gordeev * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32da673940SJordan Gordeev * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33da673940SJordan Gordeev * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34da673940SJordan Gordeev * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35da673940SJordan Gordeev * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36da673940SJordan Gordeev * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37da673940SJordan Gordeev * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38da673940SJordan Gordeev * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39da673940SJordan Gordeev * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40da673940SJordan Gordeev * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41da673940SJordan Gordeev * SUCH DAMAGE.
42da673940SJordan Gordeev *
43da673940SJordan Gordeev * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
44da673940SJordan Gordeev * $FreeBSD: src/sys/i386/i386/pmap.c,v 1.250.2.18 2002/03/06 22:48:53 silby Exp $
45da673940SJordan Gordeev */
46da673940SJordan Gordeev
47da673940SJordan Gordeev /*
48da673940SJordan Gordeev * Manages physical address maps.
49da673940SJordan Gordeev */
50da673940SJordan Gordeev
51da673940SJordan Gordeev #include "opt_msgbuf.h"
52da673940SJordan Gordeev
53da673940SJordan Gordeev #include <sys/param.h>
54da673940SJordan Gordeev #include <sys/systm.h>
55da673940SJordan Gordeev #include <sys/kernel.h>
56da673940SJordan Gordeev #include <sys/proc.h>
57da673940SJordan Gordeev #include <sys/msgbuf.h>
58da673940SJordan Gordeev #include <sys/vmmeter.h>
59da673940SJordan Gordeev #include <sys/mman.h>
60da673940SJordan Gordeev #include <sys/vmspace.h>
61da673940SJordan Gordeev
62da673940SJordan Gordeev #include <vm/vm.h>
63da673940SJordan Gordeev #include <vm/vm_param.h>
64da673940SJordan Gordeev #include <sys/sysctl.h>
65da673940SJordan Gordeev #include <sys/lock.h>
66da673940SJordan Gordeev #include <vm/vm_kern.h>
67da673940SJordan Gordeev #include <vm/vm_page.h>
68da673940SJordan Gordeev #include <vm/vm_map.h>
69da673940SJordan Gordeev #include <vm/vm_object.h>
70da673940SJordan Gordeev #include <vm/vm_extern.h>
71da673940SJordan Gordeev #include <vm/vm_pageout.h>
72da673940SJordan Gordeev #include <vm/vm_pager.h>
73da673940SJordan Gordeev #include <vm/vm_zone.h>
74da673940SJordan Gordeev
75da673940SJordan Gordeev #include <sys/thread2.h>
76b12defdcSMatthew Dillon #include <sys/spinlock2.h>
77a86ce0cdSMatthew Dillon #include <vm/vm_page2.h>
78da673940SJordan Gordeev
79da673940SJordan Gordeev #include <machine/cputypes.h>
80da673940SJordan Gordeev #include <machine/md_var.h>
81da673940SJordan Gordeev #include <machine/specialreg.h>
82da673940SJordan Gordeev #include <machine/smp.h>
83da673940SJordan Gordeev #include <machine/globaldata.h>
842a7bd4d8SSascha Wildner #include <machine/pcb.h>
85da673940SJordan Gordeev #include <machine/pmap.h>
86da673940SJordan Gordeev #include <machine/pmap_inval.h>
87da673940SJordan Gordeev
88da673940SJordan Gordeev #include <ddb/ddb.h>
89da673940SJordan Gordeev
90da673940SJordan Gordeev #include <stdio.h>
91da673940SJordan Gordeev #include <assert.h>
92da673940SJordan Gordeev #include <stdlib.h>
93da673940SJordan Gordeev
94da673940SJordan Gordeev #define PMAP_KEEP_PDIRS
95da673940SJordan Gordeev #ifndef PMAP_SHPGPERPROC
96f1d3f422SMatthew Dillon #define PMAP_SHPGPERPROC 1000
97da673940SJordan Gordeev #endif
98da673940SJordan Gordeev
99da673940SJordan Gordeev #if defined(DIAGNOSTIC)
100da673940SJordan Gordeev #define PMAP_DIAGNOSTIC
101da673940SJordan Gordeev #endif
102da673940SJordan Gordeev
103da673940SJordan Gordeev #define MINPV 2048
104da673940SJordan Gordeev
105da673940SJordan Gordeev #if !defined(PMAP_DIAGNOSTIC)
106da673940SJordan Gordeev #define PMAP_INLINE __inline
107da673940SJordan Gordeev #else
108da673940SJordan Gordeev #define PMAP_INLINE
109da673940SJordan Gordeev #endif
110da673940SJordan Gordeev
111da673940SJordan Gordeev /*
112da673940SJordan Gordeev * Get PDEs and PTEs for user/kernel address space
113da673940SJordan Gordeev */
114da673940SJordan Gordeev static pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va);
115da673940SJordan Gordeev #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
116da673940SJordan Gordeev
117da673940SJordan Gordeev #define pmap_pde_v(pte) ((*(pd_entry_t *)pte & VPTE_V) != 0)
118da673940SJordan Gordeev #define pmap_pte_w(pte) ((*(pt_entry_t *)pte & VPTE_WIRED) != 0)
119da673940SJordan Gordeev #define pmap_pte_m(pte) ((*(pt_entry_t *)pte & VPTE_M) != 0)
120da673940SJordan Gordeev #define pmap_pte_u(pte) ((*(pt_entry_t *)pte & VPTE_A) != 0)
121da673940SJordan Gordeev #define pmap_pte_v(pte) ((*(pt_entry_t *)pte & VPTE_V) != 0)
122da673940SJordan Gordeev
123da673940SJordan Gordeev /*
124da673940SJordan Gordeev * Given a map and a machine independent protection code,
125da673940SJordan Gordeev * convert to a vax protection code.
126da673940SJordan Gordeev */
127da673940SJordan Gordeev #define pte_prot(m, p) \
128da673940SJordan Gordeev (protection_codes[p & (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE)])
12973d64b98SMatthew Dillon static uint64_t protection_codes[8];
130da673940SJordan Gordeev
131*c713db65SAaron LI static struct pmap kernel_pmap_store;
132*c713db65SAaron LI struct pmap *kernel_pmap = &kernel_pmap_store;
133da673940SJordan Gordeev
134da673940SJordan Gordeev static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */
135da673940SJordan Gordeev
13615d6dd34SAntonio Huete Jimenez static struct vm_object kptobj;
137da673940SJordan Gordeev static int nkpt;
138da673940SJordan Gordeev
139da673940SJordan Gordeev static uint64_t KPDphys; /* phys addr of kernel level 2 */
140da673940SJordan Gordeev uint64_t KPDPphys; /* phys addr of kernel level 3 */
141da673940SJordan Gordeev uint64_t KPML4phys; /* phys addr of kernel level 4 */
142da673940SJordan Gordeev
143a86ce0cdSMatthew Dillon extern void *vkernel_stack;
144da673940SJordan Gordeev
145da673940SJordan Gordeev /*
146da673940SJordan Gordeev * Data for the pv entry allocation mechanism
147da673940SJordan Gordeev */
148da673940SJordan Gordeev static vm_zone_t pvzone;
149da673940SJordan Gordeev static struct vm_zone pvzone_store;
150b7ea2f3fSMatthew Dillon static vm_pindex_t pv_entry_count = 0;
151b7ea2f3fSMatthew Dillon static vm_pindex_t pv_entry_max = 0;
152b7ea2f3fSMatthew Dillon static vm_pindex_t pv_entry_high_water = 0;
153da673940SJordan Gordeev static int pmap_pagedaemon_waken = 0;
154da673940SJordan Gordeev static struct pv_entry *pvinit;
155da673940SJordan Gordeev
156da673940SJordan Gordeev /*
157da673940SJordan Gordeev * All those kernel PT submaps that BSD is so fond of
158da673940SJordan Gordeev */
1594090d6ffSSascha Wildner pt_entry_t *CMAP1 = NULL, *ptmmap;
1604c0cc8bbSSascha Wildner caddr_t CADDR1 = NULL;
161da673940SJordan Gordeev static pt_entry_t *msgbufmap;
162da673940SJordan Gordeev
163da673940SJordan Gordeev uint64_t KPTphys;
164da673940SJordan Gordeev
165da673940SJordan Gordeev static PMAP_INLINE void free_pv_entry (pv_entry_t pv);
166da673940SJordan Gordeev static pv_entry_t get_pv_entry (void);
167d87f4462Szrj static void x86_64_protection_init (void);
168da673940SJordan Gordeev static __inline void pmap_clearbit (vm_page_t m, int bit);
169da673940SJordan Gordeev
170da673940SJordan Gordeev static void pmap_remove_all (vm_page_t m);
171da673940SJordan Gordeev static int pmap_remove_pte (struct pmap *pmap, pt_entry_t *ptq,
17200eb801eSMatthew Dillon pt_entry_t oldpte, vm_offset_t sva);
173da673940SJordan Gordeev static void pmap_remove_page (struct pmap *pmap, vm_offset_t va);
174da673940SJordan Gordeev static int pmap_remove_entry (struct pmap *pmap, vm_page_t m,
175da673940SJordan Gordeev vm_offset_t va);
176da673940SJordan Gordeev static boolean_t pmap_testbit (vm_page_t m, int bit);
177da673940SJordan Gordeev static void pmap_insert_entry (pmap_t pmap, vm_offset_t va,
178c78d5661SMatthew Dillon vm_page_t mpte, vm_page_t m, pv_entry_t);
179da673940SJordan Gordeev
180da673940SJordan Gordeev static vm_page_t pmap_allocpte (pmap_t pmap, vm_offset_t va);
181da673940SJordan Gordeev
182da673940SJordan Gordeev static int pmap_release_free_page (pmap_t pmap, vm_page_t p);
183da673940SJordan Gordeev static vm_page_t _pmap_allocpte (pmap_t pmap, vm_pindex_t ptepindex);
184da673940SJordan Gordeev static vm_page_t pmap_page_lookup (vm_object_t object, vm_pindex_t pindex);
185da673940SJordan Gordeev static int pmap_unuse_pt (pmap_t, vm_offset_t, vm_page_t);
186da673940SJordan Gordeev
187c50e690bSMatthew Dillon static int
pv_entry_compare(pv_entry_t pv1,pv_entry_t pv2)188c50e690bSMatthew Dillon pv_entry_compare(pv_entry_t pv1, pv_entry_t pv2)
189c50e690bSMatthew Dillon {
190c50e690bSMatthew Dillon if (pv1->pv_va < pv2->pv_va)
191c50e690bSMatthew Dillon return(-1);
192c50e690bSMatthew Dillon if (pv1->pv_va > pv2->pv_va)
193c50e690bSMatthew Dillon return(1);
194c50e690bSMatthew Dillon return(0);
195c50e690bSMatthew Dillon }
196c50e690bSMatthew Dillon
197c50e690bSMatthew Dillon RB_GENERATE2(pv_entry_rb_tree, pv_entry, pv_entry,
198c50e690bSMatthew Dillon pv_entry_compare, vm_offset_t, pv_va);
199c50e690bSMatthew Dillon
200da673940SJordan Gordeev static __inline vm_pindex_t
pmap_pt_pindex(vm_offset_t va)201c78d5661SMatthew Dillon pmap_pt_pindex(vm_offset_t va)
202da673940SJordan Gordeev {
203da673940SJordan Gordeev return va >> PDRSHIFT;
204da673940SJordan Gordeev }
205da673940SJordan Gordeev
206da673940SJordan Gordeev static __inline vm_pindex_t
pmap_pte_index(vm_offset_t va)207da673940SJordan Gordeev pmap_pte_index(vm_offset_t va)
208da673940SJordan Gordeev {
209da673940SJordan Gordeev return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
210da673940SJordan Gordeev }
211da673940SJordan Gordeev
212da673940SJordan Gordeev static __inline vm_pindex_t
pmap_pde_index(vm_offset_t va)213da673940SJordan Gordeev pmap_pde_index(vm_offset_t va)
214da673940SJordan Gordeev {
215da673940SJordan Gordeev return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
216da673940SJordan Gordeev }
217da673940SJordan Gordeev
218da673940SJordan Gordeev static __inline vm_pindex_t
pmap_pdpe_index(vm_offset_t va)219da673940SJordan Gordeev pmap_pdpe_index(vm_offset_t va)
220da673940SJordan Gordeev {
221da673940SJordan Gordeev return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
222da673940SJordan Gordeev }
223da673940SJordan Gordeev
224da673940SJordan Gordeev static __inline vm_pindex_t
pmap_pml4e_index(vm_offset_t va)225da673940SJordan Gordeev pmap_pml4e_index(vm_offset_t va)
226da673940SJordan Gordeev {
227da673940SJordan Gordeev return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
228da673940SJordan Gordeev }
229da673940SJordan Gordeev
230da673940SJordan Gordeev /* Return a pointer to the PML4 slot that corresponds to a VA */
231da673940SJordan Gordeev static __inline pml4_entry_t *
pmap_pml4e(pmap_t pmap,vm_offset_t va)232da673940SJordan Gordeev pmap_pml4e(pmap_t pmap, vm_offset_t va)
233da673940SJordan Gordeev {
234da673940SJordan Gordeev return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
235da673940SJordan Gordeev }
236da673940SJordan Gordeev
237da673940SJordan Gordeev /* Return a pointer to the PDP slot that corresponds to a VA */
238da673940SJordan Gordeev static __inline pdp_entry_t *
pmap_pml4e_to_pdpe(pml4_entry_t * pml4e,vm_offset_t va)239da673940SJordan Gordeev pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
240da673940SJordan Gordeev {
241da673940SJordan Gordeev pdp_entry_t *pdpe;
242da673940SJordan Gordeev
243da673940SJordan Gordeev pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & VPTE_FRAME);
244da673940SJordan Gordeev return (&pdpe[pmap_pdpe_index(va)]);
245da673940SJordan Gordeev }
246da673940SJordan Gordeev
247da673940SJordan Gordeev /* Return a pointer to the PDP slot that corresponds to a VA */
248da673940SJordan Gordeev static __inline pdp_entry_t *
pmap_pdpe(pmap_t pmap,vm_offset_t va)249da673940SJordan Gordeev pmap_pdpe(pmap_t pmap, vm_offset_t va)
250da673940SJordan Gordeev {
251da673940SJordan Gordeev pml4_entry_t *pml4e;
252da673940SJordan Gordeev
253da673940SJordan Gordeev pml4e = pmap_pml4e(pmap, va);
254da673940SJordan Gordeev if ((*pml4e & VPTE_V) == 0)
255da673940SJordan Gordeev return NULL;
256da673940SJordan Gordeev return (pmap_pml4e_to_pdpe(pml4e, va));
257da673940SJordan Gordeev }
258da673940SJordan Gordeev
259da673940SJordan Gordeev /* Return a pointer to the PD slot that corresponds to a VA */
260da673940SJordan Gordeev static __inline pd_entry_t *
pmap_pdpe_to_pde(pdp_entry_t * pdpe,vm_offset_t va)261da673940SJordan Gordeev pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
262da673940SJordan Gordeev {
263da673940SJordan Gordeev pd_entry_t *pde;
264da673940SJordan Gordeev
265da673940SJordan Gordeev pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & VPTE_FRAME);
266da673940SJordan Gordeev return (&pde[pmap_pde_index(va)]);
267da673940SJordan Gordeev }
268da673940SJordan Gordeev
269da673940SJordan Gordeev /* Return a pointer to the PD slot that corresponds to a VA */
270da673940SJordan Gordeev static __inline pd_entry_t *
pmap_pde(pmap_t pmap,vm_offset_t va)271da673940SJordan Gordeev pmap_pde(pmap_t pmap, vm_offset_t va)
272da673940SJordan Gordeev {
273da673940SJordan Gordeev pdp_entry_t *pdpe;
274da673940SJordan Gordeev
275da673940SJordan Gordeev pdpe = pmap_pdpe(pmap, va);
276da673940SJordan Gordeev if (pdpe == NULL || (*pdpe & VPTE_V) == 0)
277da673940SJordan Gordeev return NULL;
278da673940SJordan Gordeev return (pmap_pdpe_to_pde(pdpe, va));
279da673940SJordan Gordeev }
280da673940SJordan Gordeev
281da673940SJordan Gordeev /* Return a pointer to the PT slot that corresponds to a VA */
282da673940SJordan Gordeev static __inline pt_entry_t *
pmap_pde_to_pte(pd_entry_t * pde,vm_offset_t va)283da673940SJordan Gordeev pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
284da673940SJordan Gordeev {
285da673940SJordan Gordeev pt_entry_t *pte;
286da673940SJordan Gordeev
287da673940SJordan Gordeev pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & VPTE_FRAME);
288da673940SJordan Gordeev return (&pte[pmap_pte_index(va)]);
289da673940SJordan Gordeev }
290da673940SJordan Gordeev
291eb36cb6bSMatthew Dillon /*
292eb36cb6bSMatthew Dillon * Hold pt_m for page table scans to prevent it from getting reused out
293eb36cb6bSMatthew Dillon * from under us across blocking conditions in the body of the loop.
294eb36cb6bSMatthew Dillon */
295eb36cb6bSMatthew Dillon static __inline
296eb36cb6bSMatthew Dillon vm_page_t
pmap_hold_pt_page(pd_entry_t * pde,vm_offset_t va)297eb36cb6bSMatthew Dillon pmap_hold_pt_page(pd_entry_t *pde, vm_offset_t va)
298eb36cb6bSMatthew Dillon {
299eb36cb6bSMatthew Dillon pt_entry_t pte;
300eb36cb6bSMatthew Dillon vm_page_t pt_m;
301eb36cb6bSMatthew Dillon
302eb36cb6bSMatthew Dillon pte = (pt_entry_t)*pde;
303eb36cb6bSMatthew Dillon KKASSERT(pte != 0);
304eb36cb6bSMatthew Dillon pt_m = PHYS_TO_VM_PAGE(pte & VPTE_FRAME);
305eb36cb6bSMatthew Dillon vm_page_hold(pt_m);
306eb36cb6bSMatthew Dillon
307eb36cb6bSMatthew Dillon return pt_m;
308eb36cb6bSMatthew Dillon }
309eb36cb6bSMatthew Dillon
310da673940SJordan Gordeev /* Return a pointer to the PT slot that corresponds to a VA */
311da673940SJordan Gordeev static __inline pt_entry_t *
pmap_pte(pmap_t pmap,vm_offset_t va)312da673940SJordan Gordeev pmap_pte(pmap_t pmap, vm_offset_t va)
313da673940SJordan Gordeev {
314da673940SJordan Gordeev pd_entry_t *pde;
315da673940SJordan Gordeev
316da673940SJordan Gordeev pde = pmap_pde(pmap, va);
317da673940SJordan Gordeev if (pde == NULL || (*pde & VPTE_V) == 0)
318da673940SJordan Gordeev return NULL;
319d87f4462Szrj if ((*pde & VPTE_PS) != 0) /* compat with x86 pmap_pte() */
320da673940SJordan Gordeev return ((pt_entry_t *)pde);
321da673940SJordan Gordeev return (pmap_pde_to_pte(pde, va));
322da673940SJordan Gordeev }
323da673940SJordan Gordeev
3242c2e847cSSascha Wildner static PMAP_INLINE pt_entry_t *
vtopte(vm_offset_t va)325da673940SJordan Gordeev vtopte(vm_offset_t va)
326da673940SJordan Gordeev {
327da673940SJordan Gordeev pt_entry_t *x;
328*c713db65SAaron LI x = pmap_pte(kernel_pmap, va);
329da673940SJordan Gordeev assert(x != NULL);
330da673940SJordan Gordeev return x;
331da673940SJordan Gordeev }
332da673940SJordan Gordeev
333da673940SJordan Gordeev static __inline pd_entry_t *
vtopde(vm_offset_t va)334da673940SJordan Gordeev vtopde(vm_offset_t va)
335da673940SJordan Gordeev {
336da673940SJordan Gordeev pd_entry_t *x;
337*c713db65SAaron LI x = pmap_pde(kernel_pmap, va);
338da673940SJordan Gordeev assert(x != NULL);
339da673940SJordan Gordeev return x;
340da673940SJordan Gordeev }
341da673940SJordan Gordeev
342a94cabebSMatthew Dillon /*
343a94cabebSMatthew Dillon * Returns the physical address translation from va for a user address.
344a94cabebSMatthew Dillon * (vm_paddr_t)-1 is returned on failure.
345a94cabebSMatthew Dillon */
346a94cabebSMatthew Dillon vm_paddr_t
uservtophys(vm_offset_t va)347a94cabebSMatthew Dillon uservtophys(vm_offset_t va)
348a94cabebSMatthew Dillon {
349fd8b44bdSMatthew Dillon struct vmspace *vm = curproc->p_vmspace;
350fd8b44bdSMatthew Dillon vm_page_t m;
351a94cabebSMatthew Dillon vm_paddr_t pa;
352fd8b44bdSMatthew Dillon int error;
353fd8b44bdSMatthew Dillon int busy;
354a94cabebSMatthew Dillon
355fd8b44bdSMatthew Dillon /* XXX No idea how to handle this case in a simple way, just abort */
356fd8b44bdSMatthew Dillon if (PAGE_SIZE - (va & PAGE_MASK) < sizeof(u_int))
357fd8b44bdSMatthew Dillon return ((vm_paddr_t)-1);
358fd8b44bdSMatthew Dillon
359fd8b44bdSMatthew Dillon m = vm_fault_page(&vm->vm_map, trunc_page(va),
360fd8b44bdSMatthew Dillon VM_PROT_READ|VM_PROT_WRITE,
361fd8b44bdSMatthew Dillon VM_FAULT_NORMAL,
362fd8b44bdSMatthew Dillon &error, &busy);
363fd8b44bdSMatthew Dillon if (error)
364fd8b44bdSMatthew Dillon return ((vm_paddr_t)-1);
365fd8b44bdSMatthew Dillon
366beaef97aSMatthew Dillon pa = VM_PAGE_TO_PHYS(m) | (va & PAGE_MASK);
367fd8b44bdSMatthew Dillon if (busy)
368fd8b44bdSMatthew Dillon vm_page_wakeup(m);
369fd8b44bdSMatthew Dillon else
370fd8b44bdSMatthew Dillon vm_page_unhold(m);
371fd8b44bdSMatthew Dillon
372a94cabebSMatthew Dillon return pa;
373a94cabebSMatthew Dillon }
374a94cabebSMatthew Dillon
375da673940SJordan Gordeev static uint64_t
allocpages(vm_paddr_t * firstaddr,int n)376da673940SJordan Gordeev allocpages(vm_paddr_t *firstaddr, int n)
377da673940SJordan Gordeev {
378da673940SJordan Gordeev uint64_t ret;
379da673940SJordan Gordeev
380da673940SJordan Gordeev ret = *firstaddr;
381c78d5661SMatthew Dillon /*bzero((void *)ret, n * PAGE_SIZE); not mapped yet */
382da673940SJordan Gordeev *firstaddr += n * PAGE_SIZE;
383da673940SJordan Gordeev return (ret);
384da673940SJordan Gordeev }
385da673940SJordan Gordeev
3862c2e847cSSascha Wildner static void
create_pagetables(vm_paddr_t * firstaddr,int64_t ptov_offset)387da673940SJordan Gordeev create_pagetables(vm_paddr_t *firstaddr, int64_t ptov_offset)
388da673940SJordan Gordeev {
389da673940SJordan Gordeev int i;
390da673940SJordan Gordeev pml4_entry_t *KPML4virt;
391da673940SJordan Gordeev pdp_entry_t *KPDPvirt;
392da673940SJordan Gordeev pd_entry_t *KPDvirt;
393da673940SJordan Gordeev pt_entry_t *KPTvirt;
394da673940SJordan Gordeev int kpml4i = pmap_pml4e_index(ptov_offset);
395da673940SJordan Gordeev int kpdpi = pmap_pdpe_index(ptov_offset);
396a86ce0cdSMatthew Dillon int kpdi = pmap_pde_index(ptov_offset);
397da673940SJordan Gordeev
398ad54aa11SMatthew Dillon /*
399ad54aa11SMatthew Dillon * Calculate NKPT - number of kernel page tables. We have to
400ad54aa11SMatthew Dillon * accomodoate prealloction of the vm_page_array, dump bitmap,
401ad54aa11SMatthew Dillon * MSGBUF_SIZE, and other stuff. Be generous.
402ad54aa11SMatthew Dillon *
403ad54aa11SMatthew Dillon * Maxmem is in pages.
404ad54aa11SMatthew Dillon */
405ad54aa11SMatthew Dillon nkpt = (Maxmem * (sizeof(struct vm_page) * 2) + MSGBUF_SIZE) / NBPDR;
406ad54aa11SMatthew Dillon /*
407ad54aa11SMatthew Dillon * Allocate pages
408ad54aa11SMatthew Dillon */
409da673940SJordan Gordeev KPML4phys = allocpages(firstaddr, 1);
410da673940SJordan Gordeev KPDPphys = allocpages(firstaddr, NKPML4E);
411da673940SJordan Gordeev KPDphys = allocpages(firstaddr, NKPDPE);
412ad54aa11SMatthew Dillon KPTphys = allocpages(firstaddr, nkpt);
413da673940SJordan Gordeev
414da673940SJordan Gordeev KPML4virt = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
415da673940SJordan Gordeev KPDPvirt = (pdp_entry_t *)PHYS_TO_DMAP(KPDPphys);
416da673940SJordan Gordeev KPDvirt = (pd_entry_t *)PHYS_TO_DMAP(KPDphys);
417da673940SJordan Gordeev KPTvirt = (pt_entry_t *)PHYS_TO_DMAP(KPTphys);
418da673940SJordan Gordeev
419da673940SJordan Gordeev bzero(KPML4virt, 1 * PAGE_SIZE);
420da673940SJordan Gordeev bzero(KPDPvirt, NKPML4E * PAGE_SIZE);
421da673940SJordan Gordeev bzero(KPDvirt, NKPDPE * PAGE_SIZE);
422ad54aa11SMatthew Dillon bzero(KPTvirt, nkpt * PAGE_SIZE);
423da673940SJordan Gordeev
424da673940SJordan Gordeev /* Now map the page tables at their location within PTmap */
425ad54aa11SMatthew Dillon for (i = 0; i < nkpt; i++) {
426a86ce0cdSMatthew Dillon KPDvirt[i + kpdi] = KPTphys + (i << PAGE_SHIFT);
427a86ce0cdSMatthew Dillon KPDvirt[i + kpdi] |= VPTE_RW | VPTE_V | VPTE_U;
428da673940SJordan Gordeev }
429da673940SJordan Gordeev
430da673940SJordan Gordeev /* And connect up the PD to the PDP */
431da673940SJordan Gordeev for (i = 0; i < NKPDPE; i++) {
432da673940SJordan Gordeev KPDPvirt[i + kpdpi] = KPDphys + (i << PAGE_SHIFT);
433a86ce0cdSMatthew Dillon KPDPvirt[i + kpdpi] |= VPTE_RW | VPTE_V | VPTE_U;
434da673940SJordan Gordeev }
435da673940SJordan Gordeev
436da673940SJordan Gordeev /* And recursively map PML4 to itself in order to get PTmap */
437da673940SJordan Gordeev KPML4virt[PML4PML4I] = KPML4phys;
438a86ce0cdSMatthew Dillon KPML4virt[PML4PML4I] |= VPTE_RW | VPTE_V | VPTE_U;
439da673940SJordan Gordeev
440da673940SJordan Gordeev /* Connect the KVA slot up to the PML4 */
441da673940SJordan Gordeev KPML4virt[kpml4i] = KPDPphys;
442a86ce0cdSMatthew Dillon KPML4virt[kpml4i] |= VPTE_RW | VPTE_V | VPTE_U;
443da673940SJordan Gordeev }
444da673940SJordan Gordeev
445da673940SJordan Gordeev /*
4469e5e1578SMatthew Dillon * Typically used to initialize a fictitious page by vm/device_pager.c
4479e5e1578SMatthew Dillon */
4489e5e1578SMatthew Dillon void
pmap_page_init(struct vm_page * m)4499e5e1578SMatthew Dillon pmap_page_init(struct vm_page *m)
4509e5e1578SMatthew Dillon {
4519e5e1578SMatthew Dillon vm_page_init(m);
4529e5e1578SMatthew Dillon TAILQ_INIT(&m->md.pv_list);
4539e5e1578SMatthew Dillon }
4549e5e1578SMatthew Dillon
4559e5e1578SMatthew Dillon /*
456da673940SJordan Gordeev * Bootstrap the system enough to run with virtual memory.
457da673940SJordan Gordeev *
458d87f4462Szrj * On x86_64 this is called after mapping has already been enabled
459da673940SJordan Gordeev * and just syncs the pmap module with what has already been done.
460da673940SJordan Gordeev * [We can't call it easily with mapping off since the kernel is not
461da673940SJordan Gordeev * mapped with PA == VA, hence we would have to relocate every address
462da673940SJordan Gordeev * from the linked base (virtual) address "KERNBASE" to the actual
463da673940SJordan Gordeev * (physical) address starting relative to 0]
464da673940SJordan Gordeev */
465da673940SJordan Gordeev void
pmap_bootstrap(vm_paddr_t * firstaddr,int64_t ptov_offset)466da673940SJordan Gordeev pmap_bootstrap(vm_paddr_t *firstaddr, int64_t ptov_offset)
467da673940SJordan Gordeev {
468da673940SJordan Gordeev vm_offset_t va;
469da673940SJordan Gordeev pt_entry_t *pte;
470da673940SJordan Gordeev
471da673940SJordan Gordeev /*
472da673940SJordan Gordeev * Create an initial set of page tables to run the kernel in.
473da673940SJordan Gordeev */
474da673940SJordan Gordeev create_pagetables(firstaddr, ptov_offset);
475da673940SJordan Gordeev
476a86ce0cdSMatthew Dillon virtual_start = KvaStart;
477da673940SJordan Gordeev virtual_end = KvaEnd;
478da673940SJordan Gordeev
479da673940SJordan Gordeev /*
480da673940SJordan Gordeev * Initialize protection array.
481da673940SJordan Gordeev */
482d87f4462Szrj x86_64_protection_init();
483da673940SJordan Gordeev
484da673940SJordan Gordeev /*
485da673940SJordan Gordeev * The kernel's pmap is statically allocated so we don't have to use
486da673940SJordan Gordeev * pmap_create, which is unlikely to work correctly at this part of
487da673940SJordan Gordeev * the boot sequence (XXX and which no longer exists).
488b12defdcSMatthew Dillon *
489b12defdcSMatthew Dillon * The kernel_pmap's pm_pteobj is used only for locking and not
490b12defdcSMatthew Dillon * for mmu pages.
491da673940SJordan Gordeev */
492*c713db65SAaron LI kernel_pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
493*c713db65SAaron LI kernel_pmap->pm_count = 1;
4948421b1adSMatthew Dillon /* don't allow deactivation */
495*c713db65SAaron LI CPUMASK_ASSALLONES(kernel_pmap->pm_active);
496*c713db65SAaron LI kernel_pmap->pm_pteobj = NULL; /* see pmap_init */
497*c713db65SAaron LI RB_INIT(&kernel_pmap->pm_pvroot);
498*c713db65SAaron LI spin_init(&kernel_pmap->pm_spin, "pmapbootstrap");
499da673940SJordan Gordeev
500da673940SJordan Gordeev /*
501da673940SJordan Gordeev * Reserve some special page table entries/VA space for temporary
502da673940SJordan Gordeev * mapping of pages.
503da673940SJordan Gordeev */
504da673940SJordan Gordeev #define SYSMAP(c, p, v, n) \
505da673940SJordan Gordeev v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
506da673940SJordan Gordeev
507da673940SJordan Gordeev va = virtual_start;
508*c713db65SAaron LI pte = pmap_pte(kernel_pmap, va);
509da673940SJordan Gordeev /*
510da673940SJordan Gordeev * CMAP1/CMAP2 are used for zeroing and copying pages.
511da673940SJordan Gordeev */
512da673940SJordan Gordeev SYSMAP(caddr_t, CMAP1, CADDR1, 1)
513da673940SJordan Gordeev
514c521f80cSSascha Wildner #if 0 /* JGV */
515da673940SJordan Gordeev /*
516da673940SJordan Gordeev * Crashdump maps.
517da673940SJordan Gordeev */
518da673940SJordan Gordeev SYSMAP(caddr_t, pt_crashdumpmap, crashdumpmap, MAXDUMPPGS);
519da673940SJordan Gordeev #endif
520da673940SJordan Gordeev
521da673940SJordan Gordeev /*
522da673940SJordan Gordeev * ptvmmap is used for reading arbitrary physical pages via
523da673940SJordan Gordeev * /dev/mem.
524da673940SJordan Gordeev */
525da673940SJordan Gordeev SYSMAP(caddr_t, ptmmap, ptvmmap, 1)
526da673940SJordan Gordeev
527da673940SJordan Gordeev /*
528da673940SJordan Gordeev * msgbufp is used to map the system message buffer.
529da673940SJordan Gordeev * XXX msgbufmap is not used.
530da673940SJordan Gordeev */
531da673940SJordan Gordeev SYSMAP(struct msgbuf *, msgbufmap, msgbufp,
532da673940SJordan Gordeev atop(round_page(MSGBUF_SIZE)))
533da673940SJordan Gordeev
534da673940SJordan Gordeev virtual_start = va;
535da673940SJordan Gordeev
536da673940SJordan Gordeev *CMAP1 = 0;
537da673940SJordan Gordeev cpu_invltlb();
538da673940SJordan Gordeev }
539da673940SJordan Gordeev
540da673940SJordan Gordeev /*
541da673940SJordan Gordeev * Initialize the pmap module.
542da673940SJordan Gordeev * Called by vm_init, to initialize any structures that the pmap
543da673940SJordan Gordeev * system needs to map virtual memory.
544da673940SJordan Gordeev * pmap_init has been enhanced to support in a fairly consistant
545da673940SJordan Gordeev * way, discontiguous physical memory.
546da673940SJordan Gordeev */
547da673940SJordan Gordeev void
pmap_init(void)548da673940SJordan Gordeev pmap_init(void)
549da673940SJordan Gordeev {
550b7ea2f3fSMatthew Dillon vm_pindex_t i;
551b7ea2f3fSMatthew Dillon vm_pindex_t initial_pvs;
552da673940SJordan Gordeev
553da673940SJordan Gordeev /*
554da673940SJordan Gordeev * object for kernel page table pages
555da673940SJordan Gordeev */
556da673940SJordan Gordeev /* JG I think the number can be arbitrary */
55715d6dd34SAntonio Huete Jimenez vm_object_init(&kptobj, 5);
558*c713db65SAaron LI kernel_pmap->pm_pteobj = &kptobj;
559da673940SJordan Gordeev
560da673940SJordan Gordeev /*
561da673940SJordan Gordeev * Allocate memory for random pmap data structures. Includes the
562da673940SJordan Gordeev * pv_head_table.
563da673940SJordan Gordeev */
564da673940SJordan Gordeev for (i = 0; i < vm_page_array_size; i++) {
565da673940SJordan Gordeev vm_page_t m;
566da673940SJordan Gordeev
567da673940SJordan Gordeev m = &vm_page_array[i];
568da673940SJordan Gordeev TAILQ_INIT(&m->md.pv_list);
569da673940SJordan Gordeev m->md.pv_list_count = 0;
570da673940SJordan Gordeev }
571da673940SJordan Gordeev
572da673940SJordan Gordeev /*
573da673940SJordan Gordeev * init the pv free list
574da673940SJordan Gordeev */
575da673940SJordan Gordeev initial_pvs = vm_page_array_size;
576da673940SJordan Gordeev if (initial_pvs < MINPV)
577da673940SJordan Gordeev initial_pvs = MINPV;
578da673940SJordan Gordeev pvzone = &pvzone_store;
5793091de50SMatthew Dillon pvinit = (struct pv_entry *)
5801eeaf6b2SAaron LI kmem_alloc(kernel_map,
5813091de50SMatthew Dillon initial_pvs * sizeof (struct pv_entry),
5823091de50SMatthew Dillon VM_SUBSYS_PVENTRY);
583da673940SJordan Gordeev zbootinit(pvzone, "PV ENTRY", sizeof (struct pv_entry), pvinit,
584da673940SJordan Gordeev initial_pvs);
585da673940SJordan Gordeev
586da673940SJordan Gordeev /*
587da673940SJordan Gordeev * Now it is safe to enable pv_table recording.
588da673940SJordan Gordeev */
589da673940SJordan Gordeev pmap_initialized = TRUE;
590da673940SJordan Gordeev }
591da673940SJordan Gordeev
592da673940SJordan Gordeev /*
593da673940SJordan Gordeev * Initialize the address space (zone) for the pv_entries. Set a
594da673940SJordan Gordeev * high water mark so that the system can recover from excessive
595da673940SJordan Gordeev * numbers of pv entries.
596da673940SJordan Gordeev */
597da673940SJordan Gordeev void
pmap_init2(void)598da673940SJordan Gordeev pmap_init2(void)
599da673940SJordan Gordeev {
600b7ea2f3fSMatthew Dillon vm_pindex_t shpgperproc = PMAP_SHPGPERPROC;
601da673940SJordan Gordeev
602b7ea2f3fSMatthew Dillon TUNABLE_LONG_FETCH("vm.pmap.shpgperproc", &shpgperproc);
603da673940SJordan Gordeev pv_entry_max = shpgperproc * maxproc + vm_page_array_size;
604b7ea2f3fSMatthew Dillon TUNABLE_LONG_FETCH("vm.pmap.pv_entries", &pv_entry_max);
605da673940SJordan Gordeev pv_entry_high_water = 9 * (pv_entry_max / 10);
606e16c650dSMatthew Dillon zinitna(pvzone, NULL, 0, pv_entry_max, ZONE_INTERRUPT);
607da673940SJordan Gordeev }
608da673940SJordan Gordeev
609da673940SJordan Gordeev
610da673940SJordan Gordeev /***************************************************
611da673940SJordan Gordeev * Low level helper routines.....
612da673940SJordan Gordeev ***************************************************/
613da673940SJordan Gordeev
614da673940SJordan Gordeev /*
615da673940SJordan Gordeev * The modification bit is not tracked for any pages in this range. XXX
616da673940SJordan Gordeev * such pages in this maps should always use pmap_k*() functions and not
617da673940SJordan Gordeev * be managed anyhow.
618da673940SJordan Gordeev *
619da673940SJordan Gordeev * XXX User and kernel address spaces are independant for virtual kernels,
620da673940SJordan Gordeev * this function only applies to the kernel pmap.
621da673940SJordan Gordeev */
622ae442b2eSMatthew Dillon static void
pmap_track_modified(pmap_t pmap,vm_offset_t va)623da673940SJordan Gordeev pmap_track_modified(pmap_t pmap, vm_offset_t va)
624da673940SJordan Gordeev {
625*c713db65SAaron LI KKASSERT(pmap != kernel_pmap ||
626ffd470cbSMatthew Dillon va < clean_sva || va >= clean_eva);
627da673940SJordan Gordeev }
628da673940SJordan Gordeev
629da673940SJordan Gordeev /*
630da673940SJordan Gordeev * Extract the physical page address associated with the map/VA pair.
631f7230403SMatthew Dillon *
632f7230403SMatthew Dillon * No requirements.
633da673940SJordan Gordeev */
634da673940SJordan Gordeev vm_paddr_t
pmap_extract(pmap_t pmap,vm_offset_t va,void ** handlep)63576f1911eSMatthew Dillon pmap_extract(pmap_t pmap, vm_offset_t va, void **handlep)
636da673940SJordan Gordeev {
637da673940SJordan Gordeev vm_paddr_t rtval;
638da673940SJordan Gordeev pt_entry_t *pte;
639da673940SJordan Gordeev pd_entry_t pde, *pdep;
640da673940SJordan Gordeev
641c91894e0SMatthew Dillon vm_object_hold(pmap->pm_pteobj);
642da673940SJordan Gordeev rtval = 0;
643da673940SJordan Gordeev pdep = pmap_pde(pmap, va);
644da673940SJordan Gordeev if (pdep != NULL) {
645da673940SJordan Gordeev pde = *pdep;
646da673940SJordan Gordeev if (pde) {
647da673940SJordan Gordeev if ((pde & VPTE_PS) != 0) {
648da673940SJordan Gordeev /* JGV */
649da673940SJordan Gordeev rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
650da673940SJordan Gordeev } else {
651da673940SJordan Gordeev pte = pmap_pde_to_pte(pdep, va);
652da673940SJordan Gordeev rtval = (*pte & VPTE_FRAME) | (va & PAGE_MASK);
653da673940SJordan Gordeev }
654da673940SJordan Gordeev }
655da673940SJordan Gordeev }
65676f1911eSMatthew Dillon if (handlep)
65776f1911eSMatthew Dillon *handlep = NULL; /* XXX */
658c91894e0SMatthew Dillon vm_object_drop(pmap->pm_pteobj);
659c91894e0SMatthew Dillon
660da673940SJordan Gordeev return rtval;
661da673940SJordan Gordeev }
662da673940SJordan Gordeev
66376f1911eSMatthew Dillon void
pmap_extract_done(void * handle)66476f1911eSMatthew Dillon pmap_extract_done(void *handle)
66576f1911eSMatthew Dillon {
66676f1911eSMatthew Dillon pmap_t pmap;
66776f1911eSMatthew Dillon
66876f1911eSMatthew Dillon if (handle) {
66976f1911eSMatthew Dillon pmap = handle;
67076f1911eSMatthew Dillon vm_object_drop(pmap->pm_pteobj);
67176f1911eSMatthew Dillon }
67276f1911eSMatthew Dillon }
67376f1911eSMatthew Dillon
674da673940SJordan Gordeev /*
675a86ce0cdSMatthew Dillon * Similar to extract but checks protections, SMP-friendly short-cut for
676a86ce0cdSMatthew Dillon * vm_fault_page[_quick]().
677a36803d2SMatthew Dillon *
678a36803d2SMatthew Dillon * WARNING! THE RETURNED PAGE IS ONLY HELD AND NEITHER IT NOR ITS TARGET
679a36803d2SMatthew Dillon * DATA IS SUITABLE FOR WRITING. Writing can interfere with
680a36803d2SMatthew Dillon * pageouts flushes, msync, etc. The hold_count is not enough
681a36803d2SMatthew Dillon * to avoid races against pageouts and other flush code doesn't
682a36803d2SMatthew Dillon * care about hold_count.
683a86ce0cdSMatthew Dillon */
684a86ce0cdSMatthew Dillon vm_page_t
pmap_fault_page_quick(pmap_t pmap __unused,vm_offset_t vaddr __unused,vm_prot_t prot __unused,int * busyp __unused)685a86ce0cdSMatthew Dillon pmap_fault_page_quick(pmap_t pmap __unused, vm_offset_t vaddr __unused,
686dc039ae0SMatthew Dillon vm_prot_t prot __unused, int *busyp __unused)
687a86ce0cdSMatthew Dillon {
688a86ce0cdSMatthew Dillon return(NULL);
689a86ce0cdSMatthew Dillon }
690a86ce0cdSMatthew Dillon
691a86ce0cdSMatthew Dillon /*
692da673940SJordan Gordeev * Routine: pmap_kextract
693da673940SJordan Gordeev * Function:
694da673940SJordan Gordeev * Extract the physical page address associated
695da673940SJordan Gordeev * kernel virtual address.
696da673940SJordan Gordeev */
697da673940SJordan Gordeev vm_paddr_t
pmap_kextract(vm_offset_t va)698da673940SJordan Gordeev pmap_kextract(vm_offset_t va)
699da673940SJordan Gordeev {
700da673940SJordan Gordeev pd_entry_t pde;
701da673940SJordan Gordeev vm_paddr_t pa;
702da673940SJordan Gordeev
703da673940SJordan Gordeev KKASSERT(va >= KvaStart && va < KvaEnd);
704da673940SJordan Gordeev
705da673940SJordan Gordeev /*
706da673940SJordan Gordeev * The DMAP region is not included in [KvaStart, KvaEnd)
707da673940SJordan Gordeev */
708da673940SJordan Gordeev #if 0
709da673940SJordan Gordeev if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
710da673940SJordan Gordeev pa = DMAP_TO_PHYS(va);
711da673940SJordan Gordeev } else {
712da673940SJordan Gordeev #endif
713da673940SJordan Gordeev pde = *vtopde(va);
714da673940SJordan Gordeev if (pde & VPTE_PS) {
715da673940SJordan Gordeev /* JGV */
716da673940SJordan Gordeev pa = (pde & PG_PS_FRAME) | (va & PDRMASK);
717da673940SJordan Gordeev } else {
718da673940SJordan Gordeev /*
719da673940SJordan Gordeev * Beware of a concurrent promotion that changes the
720da673940SJordan Gordeev * PDE at this point! For example, vtopte() must not
721da673940SJordan Gordeev * be used to access the PTE because it would use the
722da673940SJordan Gordeev * new PDE. It is, however, safe to use the old PDE
723da673940SJordan Gordeev * because the page table page is preserved by the
724da673940SJordan Gordeev * promotion.
725da673940SJordan Gordeev */
726da673940SJordan Gordeev pa = *pmap_pde_to_pte(&pde, va);
727da673940SJordan Gordeev pa = (pa & VPTE_FRAME) | (va & PAGE_MASK);
728da673940SJordan Gordeev }
729da673940SJordan Gordeev #if 0
730da673940SJordan Gordeev }
731da673940SJordan Gordeev #endif
732da673940SJordan Gordeev return pa;
733da673940SJordan Gordeev }
734da673940SJordan Gordeev
735da673940SJordan Gordeev /***************************************************
736da673940SJordan Gordeev * Low level mapping routines.....
737da673940SJordan Gordeev ***************************************************/
738da673940SJordan Gordeev
739da673940SJordan Gordeev /*
740da673940SJordan Gordeev * Enter a mapping into kernel_pmap. Mappings created in this fashion
741da673940SJordan Gordeev * are not managed. Mappings must be immediately accessible on all cpus.
742da673940SJordan Gordeev *
743da673940SJordan Gordeev * Call pmap_inval_pte() to invalidate the virtual pte and clean out the
744e1bcf416SMatthew Dillon * real pmap and handle related races before storing the new vpte. The
745e1bcf416SMatthew Dillon * new semantics for kenter require use to do an UNCONDITIONAL invalidation,
746e1bcf416SMatthew Dillon * because the entry may have previously been cleared without an invalidation.
747da673940SJordan Gordeev */
748da673940SJordan Gordeev void
pmap_kenter(vm_offset_t va,vm_paddr_t pa)749da673940SJordan Gordeev pmap_kenter(vm_offset_t va, vm_paddr_t pa)
750da673940SJordan Gordeev {
751dc039ae0SMatthew Dillon pt_entry_t *ptep;
752da673940SJordan Gordeev pt_entry_t npte;
753da673940SJordan Gordeev
754da673940SJordan Gordeev KKASSERT(va >= KvaStart && va < KvaEnd);
755a86ce0cdSMatthew Dillon npte = pa | VPTE_RW | VPTE_V | VPTE_U;
756dc039ae0SMatthew Dillon ptep = vtopte(va);
757e1bcf416SMatthew Dillon
758e1bcf416SMatthew Dillon #if 1
759*c713db65SAaron LI pmap_inval_pte(ptep, kernel_pmap, va);
760e1bcf416SMatthew Dillon #else
761da673940SJordan Gordeev if (*pte & VPTE_V)
762*c713db65SAaron LI pmap_inval_pte(ptep, kernel_pmap, va);
763e1bcf416SMatthew Dillon #endif
764dc039ae0SMatthew Dillon atomic_swap_long(ptep, npte);
765da673940SJordan Gordeev }
766da673940SJordan Gordeev
767da673940SJordan Gordeev /*
768da673940SJordan Gordeev * Enter an unmanaged KVA mapping for the private use of the current
76979f2da03SMatthew Dillon * cpu only.
770da673940SJordan Gordeev *
77179f2da03SMatthew Dillon * It is illegal for the mapping to be accessed by other cpus without
77279f2da03SMatthew Dillon * proper invalidation.
773da673940SJordan Gordeev */
77467d1661cSSascha Wildner int
pmap_kenter_quick(vm_offset_t va,vm_paddr_t pa)775da673940SJordan Gordeev pmap_kenter_quick(vm_offset_t va, vm_paddr_t pa)
776da673940SJordan Gordeev {
77767d1661cSSascha Wildner pt_entry_t *ptep;
778da673940SJordan Gordeev pt_entry_t npte;
77967d1661cSSascha Wildner int res;
780da673940SJordan Gordeev
781da673940SJordan Gordeev KKASSERT(va >= KvaStart && va < KvaEnd);
782da673940SJordan Gordeev
783a86ce0cdSMatthew Dillon npte = (vpte_t)pa | VPTE_RW | VPTE_V | VPTE_U;
78467d1661cSSascha Wildner ptep = vtopte(va);
785dc039ae0SMatthew Dillon
7862f0acc22SMatthew Dillon #if 1
787*c713db65SAaron LI pmap_inval_pte_quick(ptep, kernel_pmap, va);
7882f0acc22SMatthew Dillon res = 1;
7892f0acc22SMatthew Dillon #else
7902f0acc22SMatthew Dillon /* FUTURE */
79167d1661cSSascha Wildner res = (*ptep != 0);
792dc039ae0SMatthew Dillon if (*pte & VPTE_V)
793*c713db65SAaron LI pmap_inval_pte(pte, kernel_pmap, va);
7942f0acc22SMatthew Dillon #endif
795dc039ae0SMatthew Dillon atomic_swap_long(ptep, npte);
7962f0acc22SMatthew Dillon
79767d1661cSSascha Wildner return res;
798da673940SJordan Gordeev }
799da673940SJordan Gordeev
800dc039ae0SMatthew Dillon /*
801dc039ae0SMatthew Dillon * Invalidation will occur later, ok to be lazy here.
802dc039ae0SMatthew Dillon */
8032f0acc22SMatthew Dillon int
pmap_kenter_noinval(vm_offset_t va,vm_paddr_t pa)804ccd67bf6SMatthew Dillon pmap_kenter_noinval(vm_offset_t va, vm_paddr_t pa)
805ccd67bf6SMatthew Dillon {
8062f0acc22SMatthew Dillon pt_entry_t *ptep;
807ccd67bf6SMatthew Dillon pt_entry_t npte;
8082f0acc22SMatthew Dillon int res;
809ccd67bf6SMatthew Dillon
810ccd67bf6SMatthew Dillon KKASSERT(va >= KvaStart && va < KvaEnd);
811ccd67bf6SMatthew Dillon
812ccd67bf6SMatthew Dillon npte = (vpte_t)pa | VPTE_RW | VPTE_V | VPTE_U;
8132f0acc22SMatthew Dillon ptep = vtopte(va);
8142f0acc22SMatthew Dillon #if 1
8152f0acc22SMatthew Dillon res = 1;
8162f0acc22SMatthew Dillon #else
8172f0acc22SMatthew Dillon /* FUTURE */
8182f0acc22SMatthew Dillon res = (*ptep != 0);
8192f0acc22SMatthew Dillon #endif
820dc039ae0SMatthew Dillon atomic_swap_long(ptep, npte);
8212f0acc22SMatthew Dillon
8222f0acc22SMatthew Dillon return res;
823ccd67bf6SMatthew Dillon }
824ccd67bf6SMatthew Dillon
825da673940SJordan Gordeev /*
826da673940SJordan Gordeev * Remove an unmanaged mapping created with pmap_kenter*().
827da673940SJordan Gordeev */
828da673940SJordan Gordeev void
pmap_kremove(vm_offset_t va)829da673940SJordan Gordeev pmap_kremove(vm_offset_t va)
830da673940SJordan Gordeev {
831dc039ae0SMatthew Dillon pt_entry_t *ptep;
832da673940SJordan Gordeev
833da673940SJordan Gordeev KKASSERT(va >= KvaStart && va < KvaEnd);
834da673940SJordan Gordeev
835dc039ae0SMatthew Dillon ptep = vtopte(va);
836dc039ae0SMatthew Dillon atomic_swap_long(ptep, 0);
837*c713db65SAaron LI pmap_inval_pte(ptep, kernel_pmap, va);
838da673940SJordan Gordeev }
839da673940SJordan Gordeev
840da673940SJordan Gordeev /*
841da673940SJordan Gordeev * Remove an unmanaged mapping created with pmap_kenter*() but synchronize
842da673940SJordan Gordeev * only with this cpu.
843da673940SJordan Gordeev *
844da673940SJordan Gordeev * Unfortunately because we optimize new entries by testing VPTE_V later
845da673940SJordan Gordeev * on, we actually still have to synchronize with all the cpus. XXX maybe
846da673940SJordan Gordeev * store a junk value and test against 0 in the other places instead?
847da673940SJordan Gordeev */
848da673940SJordan Gordeev void
pmap_kremove_quick(vm_offset_t va)849da673940SJordan Gordeev pmap_kremove_quick(vm_offset_t va)
850da673940SJordan Gordeev {
851dc039ae0SMatthew Dillon pt_entry_t *ptep;
852da673940SJordan Gordeev
853da673940SJordan Gordeev KKASSERT(va >= KvaStart && va < KvaEnd);
854da673940SJordan Gordeev
855dc039ae0SMatthew Dillon ptep = vtopte(va);
856dc039ae0SMatthew Dillon atomic_swap_long(ptep, 0);
857*c713db65SAaron LI pmap_inval_pte(ptep, kernel_pmap, va); /* NOT _quick */
858da673940SJordan Gordeev }
859da673940SJordan Gordeev
860dc039ae0SMatthew Dillon /*
861dc039ae0SMatthew Dillon * Invalidation will occur later, ok to be lazy here.
862dc039ae0SMatthew Dillon */
863ccd67bf6SMatthew Dillon void
pmap_kremove_noinval(vm_offset_t va)864ccd67bf6SMatthew Dillon pmap_kremove_noinval(vm_offset_t va)
865ccd67bf6SMatthew Dillon {
866dc039ae0SMatthew Dillon pt_entry_t *ptep;
867ccd67bf6SMatthew Dillon
868ccd67bf6SMatthew Dillon KKASSERT(va >= KvaStart && va < KvaEnd);
869ccd67bf6SMatthew Dillon
870dc039ae0SMatthew Dillon ptep = vtopte(va);
871dc039ae0SMatthew Dillon atomic_swap_long(ptep, 0);
872ccd67bf6SMatthew Dillon }
873ccd67bf6SMatthew Dillon
874da673940SJordan Gordeev /*
875da673940SJordan Gordeev * Used to map a range of physical addresses into kernel
876da673940SJordan Gordeev * virtual address space.
877da673940SJordan Gordeev *
878da673940SJordan Gordeev * For now, VM is already on, we only need to map the
879da673940SJordan Gordeev * specified memory.
880da673940SJordan Gordeev */
881da673940SJordan Gordeev vm_offset_t
pmap_map(vm_offset_t * virtp,vm_paddr_t start,vm_paddr_t end,int prot)8820e6594a8SSascha Wildner pmap_map(vm_offset_t *virtp, vm_paddr_t start, vm_paddr_t end, int prot)
883da673940SJordan Gordeev {
884da673940SJordan Gordeev return PHYS_TO_DMAP(start);
885da673940SJordan Gordeev }
886da673940SJordan Gordeev
887da673940SJordan Gordeev /*
888da673940SJordan Gordeev * Map a set of unmanaged VM pages into KVM.
889da673940SJordan Gordeev */
890f3bd2fceSMatthew Dillon static __inline void
_pmap_qenter(vm_offset_t beg_va,vm_page_t * m,int count,int doinval)891f3bd2fceSMatthew Dillon _pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count, int doinval)
892da673940SJordan Gordeev {
893da673940SJordan Gordeev vm_offset_t end_va;
894dc039ae0SMatthew Dillon vm_offset_t va;
895da673940SJordan Gordeev
896dc039ae0SMatthew Dillon end_va = beg_va + count * PAGE_SIZE;
89795270b7eSMatthew Dillon KKASSERT(beg_va >= KvaStart && end_va <= KvaEnd);
898da673940SJordan Gordeev
899dc039ae0SMatthew Dillon for (va = beg_va; va < end_va; va += PAGE_SIZE) {
900dc039ae0SMatthew Dillon pt_entry_t *ptep;
901da673940SJordan Gordeev
902dc039ae0SMatthew Dillon ptep = vtopte(va);
903dc039ae0SMatthew Dillon atomic_swap_long(ptep, VM_PAGE_TO_PHYS(*m) |
904c78d5661SMatthew Dillon VPTE_RW | VPTE_V | VPTE_U);
905dc039ae0SMatthew Dillon ++m;
906da673940SJordan Gordeev }
907f3bd2fceSMatthew Dillon if (doinval)
908*c713db65SAaron LI pmap_invalidate_range(kernel_pmap, beg_va, end_va);
909*c713db65SAaron LI /* pmap_inval_pte(pte, kernel_pmap, va); */
910da673940SJordan Gordeev }
911da673940SJordan Gordeev
912f3bd2fceSMatthew Dillon void
pmap_qenter(vm_offset_t beg_va,vm_page_t * m,int count)913f3bd2fceSMatthew Dillon pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count)
914f3bd2fceSMatthew Dillon {
915f3bd2fceSMatthew Dillon _pmap_qenter(beg_va, m, count, 1);
916f3bd2fceSMatthew Dillon }
917f3bd2fceSMatthew Dillon
918f3bd2fceSMatthew Dillon void
pmap_qenter_noinval(vm_offset_t beg_va,vm_page_t * m,int count)919f3bd2fceSMatthew Dillon pmap_qenter_noinval(vm_offset_t beg_va, vm_page_t *m, int count)
920f3bd2fceSMatthew Dillon {
921f3bd2fceSMatthew Dillon _pmap_qenter(beg_va, m, count, 0);
922f3bd2fceSMatthew Dillon }
923f3bd2fceSMatthew Dillon
924da673940SJordan Gordeev /*
925da673940SJordan Gordeev * Undo the effects of pmap_qenter*().
926da673940SJordan Gordeev */
927da673940SJordan Gordeev void
pmap_qremove(vm_offset_t beg_va,int count)928dc039ae0SMatthew Dillon pmap_qremove(vm_offset_t beg_va, int count)
929da673940SJordan Gordeev {
930da673940SJordan Gordeev vm_offset_t end_va;
931dc039ae0SMatthew Dillon vm_offset_t va;
932da673940SJordan Gordeev
933dc039ae0SMatthew Dillon end_va = beg_va + count * PAGE_SIZE;
934dc039ae0SMatthew Dillon KKASSERT(beg_va >= KvaStart && end_va < KvaEnd);
935da673940SJordan Gordeev
936dc039ae0SMatthew Dillon for (va = beg_va; va < end_va; va += PAGE_SIZE) {
937dc039ae0SMatthew Dillon pt_entry_t *ptep;
938da673940SJordan Gordeev
939dc039ae0SMatthew Dillon ptep = vtopte(va);
940dc039ae0SMatthew Dillon atomic_swap_long(ptep, 0);
941d0f59917SMatthew Dillon }
942*c713db65SAaron LI pmap_invalidate_range(kernel_pmap, beg_va, end_va);
943d0f59917SMatthew Dillon }
944d0f59917SMatthew Dillon
945dc039ae0SMatthew Dillon /*
946dc039ae0SMatthew Dillon * Unlike the real pmap code, we can't avoid calling the real-kernel.
947dc039ae0SMatthew Dillon */
948d0f59917SMatthew Dillon void
pmap_qremove_quick(vm_offset_t va,int count)949d0f59917SMatthew Dillon pmap_qremove_quick(vm_offset_t va, int count)
950d0f59917SMatthew Dillon {
951dc039ae0SMatthew Dillon pmap_qremove(va, count);
952d0f59917SMatthew Dillon }
953d0f59917SMatthew Dillon
954d0f59917SMatthew Dillon void
pmap_qremove_noinval(vm_offset_t va,int count)955d0f59917SMatthew Dillon pmap_qremove_noinval(vm_offset_t va, int count)
956d0f59917SMatthew Dillon {
957dc039ae0SMatthew Dillon pmap_qremove(va, count);
958da673940SJordan Gordeev }
959da673940SJordan Gordeev
960da673940SJordan Gordeev /*
961da673940SJordan Gordeev * This routine works like vm_page_lookup() but also blocks as long as the
962da673940SJordan Gordeev * page is busy. This routine does not busy the page it returns.
963da673940SJordan Gordeev *
964da673940SJordan Gordeev * Unless the caller is managing objects whos pages are in a known state,
965da673940SJordan Gordeev * the call should be made with a critical section held so the page's object
966da673940SJordan Gordeev * association remains valid on return.
967da673940SJordan Gordeev */
968da673940SJordan Gordeev static vm_page_t
pmap_page_lookup(vm_object_t object,vm_pindex_t pindex)969da673940SJordan Gordeev pmap_page_lookup(vm_object_t object, vm_pindex_t pindex)
970da673940SJordan Gordeev {
971da673940SJordan Gordeev vm_page_t m;
972da673940SJordan Gordeev
973b12defdcSMatthew Dillon ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
97495270b7eSMatthew Dillon m = vm_page_lookup_busy_wait(object, pindex, TRUE, "pplookp");
975da673940SJordan Gordeev
976da673940SJordan Gordeev return(m);
977da673940SJordan Gordeev }
978da673940SJordan Gordeev
979da673940SJordan Gordeev /*
980da673940SJordan Gordeev * Create a new thread and optionally associate it with a (new) process.
981da673940SJordan Gordeev * NOTE! the new thread's cpu may not equal the current cpu.
982da673940SJordan Gordeev */
983da673940SJordan Gordeev void
pmap_init_thread(thread_t td)984da673940SJordan Gordeev pmap_init_thread(thread_t td)
985da673940SJordan Gordeev {
986da673940SJordan Gordeev /* enforce pcb placement */
987da673940SJordan Gordeev td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_size) - 1;
988da673940SJordan Gordeev td->td_savefpu = &td->td_pcb->pcb_save;
989a76ca9b9SSascha Wildner td->td_sp = (char *)td->td_pcb - 16; /* JG is -16 needed on x86_64? */
990da673940SJordan Gordeev }
991da673940SJordan Gordeev
992da673940SJordan Gordeev /*
993da673940SJordan Gordeev * This routine directly affects the fork perf for a process.
994da673940SJordan Gordeev */
995da673940SJordan Gordeev void
pmap_init_proc(struct proc * p)996da673940SJordan Gordeev pmap_init_proc(struct proc *p)
997da673940SJordan Gordeev {
998da673940SJordan Gordeev }
999da673940SJordan Gordeev
1000da673940SJordan Gordeev /*
1001c78d5661SMatthew Dillon * Unwire a page table which has been removed from the pmap. We own the
1002c78d5661SMatthew Dillon * wire_count, so the page cannot go away. The page representing the page
1003c78d5661SMatthew Dillon * table is passed in unbusied and must be busied if we cannot trivially
1004c78d5661SMatthew Dillon * unwire it.
100595270b7eSMatthew Dillon *
100695270b7eSMatthew Dillon * XXX NOTE! This code is not usually run because we do not currently
100795270b7eSMatthew Dillon * implement dynamic page table page removal. The page in
100895270b7eSMatthew Dillon * its parent assumes at least 1 wire count, so no call to this
100995270b7eSMatthew Dillon * function ever sees a wire count less than 2.
1010da673940SJordan Gordeev */
1011da673940SJordan Gordeev static int
pmap_unwire_pgtable(pmap_t pmap,vm_offset_t va,vm_page_t m)1012c78d5661SMatthew Dillon pmap_unwire_pgtable(pmap_t pmap, vm_offset_t va, vm_page_t m)
1013da673940SJordan Gordeev {
1014c78d5661SMatthew Dillon /*
1015c78d5661SMatthew Dillon * Try to unwire optimally. If non-zero is returned the wire_count
1016c78d5661SMatthew Dillon * is 1 and we must busy the page to unwire it.
1017c78d5661SMatthew Dillon */
1018c78d5661SMatthew Dillon if (vm_page_unwire_quick(m) == 0)
1019c78d5661SMatthew Dillon return 0;
1020c78d5661SMatthew Dillon
102195270b7eSMatthew Dillon vm_page_busy_wait(m, TRUE, "pmuwpt");
1022da673940SJordan Gordeev KASSERT(m->queue == PQ_NONE,
1023c78d5661SMatthew Dillon ("_pmap_unwire_pgtable: %p->queue != PQ_NONE", m));
1024da673940SJordan Gordeev
1025c78d5661SMatthew Dillon if (m->wire_count == 1) {
1026da673940SJordan Gordeev /*
1027da673940SJordan Gordeev * Unmap the page table page.
1028da673940SJordan Gordeev */
1029da673940SJordan Gordeev /* pmap_inval_add(info, pmap, -1); */
1030da673940SJordan Gordeev
1031c78d5661SMatthew Dillon if (m->pindex >= (NUPT_TOTAL + NUPD_TOTAL)) {
1032da673940SJordan Gordeev /* PDP page */
1033da673940SJordan Gordeev pml4_entry_t *pml4;
1034da673940SJordan Gordeev pml4 = pmap_pml4e(pmap, va);
1035da673940SJordan Gordeev *pml4 = 0;
1036c78d5661SMatthew Dillon } else if (m->pindex >= NUPT_TOTAL) {
1037da673940SJordan Gordeev /* PD page */
1038da673940SJordan Gordeev pdp_entry_t *pdp;
1039da673940SJordan Gordeev pdp = pmap_pdpe(pmap, va);
1040da673940SJordan Gordeev *pdp = 0;
1041da673940SJordan Gordeev } else {
1042da673940SJordan Gordeev /* PT page */
1043da673940SJordan Gordeev pd_entry_t *pd;
1044da673940SJordan Gordeev pd = pmap_pde(pmap, va);
1045da673940SJordan Gordeev *pd = 0;
1046da673940SJordan Gordeev }
1047da673940SJordan Gordeev
1048da673940SJordan Gordeev KKASSERT(pmap->pm_stats.resident_count > 0);
1049c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count, -1);
1050da673940SJordan Gordeev
1051da673940SJordan Gordeev if (pmap->pm_ptphint == m)
1052da673940SJordan Gordeev pmap->pm_ptphint = NULL;
1053da673940SJordan Gordeev
1054c78d5661SMatthew Dillon if (m->pindex < NUPT_TOTAL) {
1055da673940SJordan Gordeev /* We just released a PT, unhold the matching PD */
1056da673940SJordan Gordeev vm_page_t pdpg;
1057da673940SJordan Gordeev
1058c78d5661SMatthew Dillon pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) &
1059c78d5661SMatthew Dillon VPTE_FRAME);
1060c78d5661SMatthew Dillon pmap_unwire_pgtable(pmap, va, pdpg);
1061da673940SJordan Gordeev }
1062c78d5661SMatthew Dillon if (m->pindex >= NUPT_TOTAL &&
1063c78d5661SMatthew Dillon m->pindex < (NUPT_TOTAL + NUPD_TOTAL)) {
1064da673940SJordan Gordeev /* We just released a PD, unhold the matching PDP */
1065da673940SJordan Gordeev vm_page_t pdppg;
1066da673940SJordan Gordeev
1067c78d5661SMatthew Dillon pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) &
1068c78d5661SMatthew Dillon VPTE_FRAME);
1069c78d5661SMatthew Dillon pmap_unwire_pgtable(pmap, va, pdppg);
1070da673940SJordan Gordeev }
1071da673940SJordan Gordeev
1072da673940SJordan Gordeev /*
1073c78d5661SMatthew Dillon * This was our last wire, the page had better be unwired
1074da673940SJordan Gordeev * after we decrement wire_count.
1075da673940SJordan Gordeev *
1076da673940SJordan Gordeev * FUTURE NOTE: shared page directory page could result in
1077da673940SJordan Gordeev * multiple wire counts.
1078da673940SJordan Gordeev */
1079c78d5661SMatthew Dillon vm_page_unwire(m, 0);
1080da673940SJordan Gordeev KKASSERT(m->wire_count == 0);
1081da673940SJordan Gordeev vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
1082da673940SJordan Gordeev vm_page_flash(m);
1083b443039bSMatthew Dillon vm_page_free(m);
1084da673940SJordan Gordeev return 1;
1085da673940SJordan Gordeev } else {
1086c78d5661SMatthew Dillon /* XXX SMP race to 1 if not holding vmobj */
1087c78d5661SMatthew Dillon vm_page_unwire(m, 0);
1088b12defdcSMatthew Dillon vm_page_wakeup(m);
1089da673940SJordan Gordeev return 0;
1090da673940SJordan Gordeev }
1091da673940SJordan Gordeev }
1092da673940SJordan Gordeev
1093da673940SJordan Gordeev /*
1094da673940SJordan Gordeev * After removing a page table entry, this routine is used to
1095da673940SJordan Gordeev * conditionally free the page, and manage the hold/wire counts.
1096c78d5661SMatthew Dillon *
1097c78d5661SMatthew Dillon * If not NULL the caller owns a wire_count on mpte, so it can't disappear.
1098c78d5661SMatthew Dillon * If NULL the caller owns a wire_count on what would be the mpte, we must
1099c78d5661SMatthew Dillon * look it up.
1100da673940SJordan Gordeev */
1101da673940SJordan Gordeev static int
pmap_unuse_pt(pmap_t pmap,vm_offset_t va,vm_page_t mpte)1102da673940SJordan Gordeev pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte)
1103da673940SJordan Gordeev {
1104da673940SJordan Gordeev vm_pindex_t ptepindex;
1105da673940SJordan Gordeev
1106b12defdcSMatthew Dillon ASSERT_LWKT_TOKEN_HELD(vm_object_token(pmap->pm_pteobj));
1107b12defdcSMatthew Dillon
1108da673940SJordan Gordeev if (mpte == NULL) {
1109da673940SJordan Gordeev /*
1110da673940SJordan Gordeev * page table pages in the kernel_pmap are not managed.
1111da673940SJordan Gordeev */
1112*c713db65SAaron LI if (pmap == kernel_pmap)
1113da673940SJordan Gordeev return(0);
1114c78d5661SMatthew Dillon ptepindex = pmap_pt_pindex(va);
1115da673940SJordan Gordeev if (pmap->pm_ptphint &&
1116da673940SJordan Gordeev (pmap->pm_ptphint->pindex == ptepindex)) {
1117da673940SJordan Gordeev mpte = pmap->pm_ptphint;
1118da673940SJordan Gordeev } else {
1119da673940SJordan Gordeev mpte = pmap_page_lookup(pmap->pm_pteobj, ptepindex);
1120da673940SJordan Gordeev pmap->pm_ptphint = mpte;
1121b12defdcSMatthew Dillon vm_page_wakeup(mpte);
1122da673940SJordan Gordeev }
1123da673940SJordan Gordeev }
1124c78d5661SMatthew Dillon return pmap_unwire_pgtable(pmap, va, mpte);
1125da673940SJordan Gordeev }
1126da673940SJordan Gordeev
1127da673940SJordan Gordeev /*
1128da673940SJordan Gordeev * Initialize pmap0/vmspace0 . Since process 0 never enters user mode we
1129da673940SJordan Gordeev * just dummy it up so it works well enough for fork().
1130da673940SJordan Gordeev *
1131da673940SJordan Gordeev * In DragonFly, process pmaps may only be used to manipulate user address
1132da673940SJordan Gordeev * space, never kernel address space.
1133da673940SJordan Gordeev */
1134da673940SJordan Gordeev void
pmap_pinit0(struct pmap * pmap)1135da673940SJordan Gordeev pmap_pinit0(struct pmap *pmap)
1136da673940SJordan Gordeev {
1137da673940SJordan Gordeev pmap_pinit(pmap);
1138da673940SJordan Gordeev }
1139da673940SJordan Gordeev
1140da673940SJordan Gordeev /*
1141da673940SJordan Gordeev * Initialize a preallocated and zeroed pmap structure,
1142da673940SJordan Gordeev * such as one in a vmspace structure.
1143da673940SJordan Gordeev */
1144da673940SJordan Gordeev void
pmap_pinit(struct pmap * pmap)1145da673940SJordan Gordeev pmap_pinit(struct pmap *pmap)
1146da673940SJordan Gordeev {
1147da673940SJordan Gordeev vm_page_t ptdpg;
1148da673940SJordan Gordeev
1149da673940SJordan Gordeev /*
1150da673940SJordan Gordeev * No need to allocate page table space yet but we do need a valid
1151da673940SJordan Gordeev * page directory table.
1152da673940SJordan Gordeev */
1153da673940SJordan Gordeev if (pmap->pm_pml4 == NULL) {
11543091de50SMatthew Dillon pmap->pm_pml4 = (pml4_entry_t *)
11551eeaf6b2SAaron LI kmem_alloc_pageable(kernel_map, PAGE_SIZE,
11563091de50SMatthew Dillon VM_SUBSYS_PML4);
1157da673940SJordan Gordeev }
1158da673940SJordan Gordeev
1159da673940SJordan Gordeev /*
1160da673940SJordan Gordeev * Allocate an object for the ptes
1161da673940SJordan Gordeev */
1162da673940SJordan Gordeev if (pmap->pm_pteobj == NULL)
1163c78d5661SMatthew Dillon pmap->pm_pteobj = vm_object_allocate(OBJT_DEFAULT, NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL + 1);
1164da673940SJordan Gordeev
1165da673940SJordan Gordeev /*
1166da673940SJordan Gordeev * Allocate the page directory page, unless we already have
1167da673940SJordan Gordeev * one cached. If we used the cached page the wire_count will
1168da673940SJordan Gordeev * already be set appropriately.
1169da673940SJordan Gordeev */
1170da673940SJordan Gordeev if ((ptdpg = pmap->pm_pdirm) == NULL) {
1171d2d8515bSMatthew Dillon ptdpg = vm_page_grab(pmap->pm_pteobj,
1172c78d5661SMatthew Dillon NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL,
1173d2d8515bSMatthew Dillon VM_ALLOC_NORMAL | VM_ALLOC_RETRY |
1174d2d8515bSMatthew Dillon VM_ALLOC_ZERO);
1175da673940SJordan Gordeev pmap->pm_pdirm = ptdpg;
1176b443039bSMatthew Dillon vm_page_flag_clear(ptdpg, PG_MAPPED | PG_WRITEABLE);
117754341a3bSMatthew Dillon vm_page_wire(ptdpg);
1178b12defdcSMatthew Dillon vm_page_wakeup(ptdpg);
1179da673940SJordan Gordeev pmap_kenter((vm_offset_t)pmap->pm_pml4, VM_PAGE_TO_PHYS(ptdpg));
1180da673940SJordan Gordeev }
1181da673940SJordan Gordeev pmap->pm_count = 1;
11821ad93419SNuno Antunes CPUMASK_ASSZERO(pmap->pm_active);
1183da673940SJordan Gordeev pmap->pm_ptphint = NULL;
1184c50e690bSMatthew Dillon RB_INIT(&pmap->pm_pvroot);
1185ba87a4abSSascha Wildner spin_init(&pmap->pm_spin, "pmapinit");
1186da673940SJordan Gordeev bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
1187da673940SJordan Gordeev pmap->pm_stats.resident_count = 1;
1188c50e690bSMatthew Dillon pmap->pm_stats.wired_count = 1;
1189da673940SJordan Gordeev }
1190da673940SJordan Gordeev
1191da673940SJordan Gordeev /*
1192da673940SJordan Gordeev * Clean up a pmap structure so it can be physically freed. This routine
1193da673940SJordan Gordeev * is called by the vmspace dtor function. A great deal of pmap data is
1194da673940SJordan Gordeev * left passively mapped to improve vmspace management so we have a bit
1195da673940SJordan Gordeev * of cleanup work to do here.
1196f7230403SMatthew Dillon *
1197f7230403SMatthew Dillon * No requirements.
1198da673940SJordan Gordeev */
1199da673940SJordan Gordeev void
pmap_puninit(pmap_t pmap)1200da673940SJordan Gordeev pmap_puninit(pmap_t pmap)
1201da673940SJordan Gordeev {
1202da673940SJordan Gordeev vm_page_t p;
1203da673940SJordan Gordeev
12041ad93419SNuno Antunes KKASSERT(CPUMASK_TESTZERO(pmap->pm_active));
1205da673940SJordan Gordeev if ((p = pmap->pm_pdirm) != NULL) {
1206da673940SJordan Gordeev KKASSERT(pmap->pm_pml4 != NULL);
1207da673940SJordan Gordeev pmap_kremove((vm_offset_t)pmap->pm_pml4);
120895270b7eSMatthew Dillon vm_page_busy_wait(p, TRUE, "pgpun");
1209b443039bSMatthew Dillon vm_page_unwire(p, 0);
1210b443039bSMatthew Dillon vm_page_flag_clear(p, PG_MAPPED | PG_WRITEABLE);
1211b443039bSMatthew Dillon vm_page_free(p);
1212da673940SJordan Gordeev pmap->pm_pdirm = NULL;
1213c50e690bSMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, -1);
1214c50e690bSMatthew Dillon KKASSERT(pmap->pm_stats.wired_count == 0);
1215da673940SJordan Gordeev }
1216da673940SJordan Gordeev if (pmap->pm_pml4) {
12171eeaf6b2SAaron LI kmem_free(kernel_map, (vm_offset_t)pmap->pm_pml4, PAGE_SIZE);
1218da673940SJordan Gordeev pmap->pm_pml4 = NULL;
1219da673940SJordan Gordeev }
1220da673940SJordan Gordeev if (pmap->pm_pteobj) {
1221da673940SJordan Gordeev vm_object_deallocate(pmap->pm_pteobj);
1222da673940SJordan Gordeev pmap->pm_pteobj = NULL;
1223da673940SJordan Gordeev }
1224da673940SJordan Gordeev }
1225da673940SJordan Gordeev
1226da673940SJordan Gordeev /*
122773b1bfb1SMatthew Dillon * This function is now unused (used to add the pmap to the pmap_list)
1228da673940SJordan Gordeev */
1229da673940SJordan Gordeev void
pmap_pinit2(struct pmap * pmap)1230da673940SJordan Gordeev pmap_pinit2(struct pmap *pmap)
1231da673940SJordan Gordeev {
1232da673940SJordan Gordeev }
1233da673940SJordan Gordeev
1234da673940SJordan Gordeev /*
1235da673940SJordan Gordeev * Attempt to release and free a vm_page in a pmap. Returns 1 on success,
1236da673940SJordan Gordeev * 0 on failure (if the procedure had to sleep).
1237da673940SJordan Gordeev *
1238da673940SJordan Gordeev * When asked to remove the page directory page itself, we actually just
1239da673940SJordan Gordeev * leave it cached so we do not have to incur the SMP inval overhead of
1240da673940SJordan Gordeev * removing the kernel mapping. pmap_puninit() will take care of it.
1241da673940SJordan Gordeev */
1242da673940SJordan Gordeev static int
pmap_release_free_page(struct pmap * pmap,vm_page_t p)1243da673940SJordan Gordeev pmap_release_free_page(struct pmap *pmap, vm_page_t p)
1244da673940SJordan Gordeev {
1245da673940SJordan Gordeev /*
1246da673940SJordan Gordeev * This code optimizes the case of freeing non-busy
1247da673940SJordan Gordeev * page-table pages. Those pages are zero now, and
1248da673940SJordan Gordeev * might as well be placed directly into the zero queue.
1249da673940SJordan Gordeev */
125095270b7eSMatthew Dillon if (vm_page_busy_try(p, TRUE)) {
125195270b7eSMatthew Dillon vm_page_sleep_busy(p, TRUE, "pmaprl");
1252c78d5661SMatthew Dillon return 1;
1253b12defdcSMatthew Dillon }
1254da673940SJordan Gordeev
1255da673940SJordan Gordeev /*
1256da673940SJordan Gordeev * Remove the page table page from the processes address space.
1257da673940SJordan Gordeev */
1258c78d5661SMatthew Dillon if (p->pindex == NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL) {
1259da673940SJordan Gordeev /*
1260da673940SJordan Gordeev * We are the pml4 table itself.
1261da673940SJordan Gordeev */
1262da673940SJordan Gordeev /* XXX anything to do here? */
1263c78d5661SMatthew Dillon } else if (p->pindex >= (NUPT_TOTAL + NUPD_TOTAL)) {
1264da673940SJordan Gordeev /*
1265da673940SJordan Gordeev * We are a PDP page.
1266da673940SJordan Gordeev * We look for the PML4 entry that points to us.
1267da673940SJordan Gordeev */
1268c78d5661SMatthew Dillon vm_page_t m4;
1269c78d5661SMatthew Dillon pml4_entry_t *pml4;
1270c78d5661SMatthew Dillon int idx;
1271c78d5661SMatthew Dillon
1272c78d5661SMatthew Dillon m4 = vm_page_lookup(pmap->pm_pteobj,
1273c78d5661SMatthew Dillon NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL);
1274da673940SJordan Gordeev KKASSERT(m4 != NULL);
1275c78d5661SMatthew Dillon pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m4));
1276c78d5661SMatthew Dillon idx = (p->pindex - (NUPT_TOTAL + NUPD_TOTAL)) % NPML4EPG;
1277da673940SJordan Gordeev KKASSERT(pml4[idx] != 0);
1278c78d5661SMatthew Dillon if (pml4[idx] == 0)
1279c78d5661SMatthew Dillon kprintf("pmap_release: Unmapped PML4\n");
1280da673940SJordan Gordeev pml4[idx] = 0;
1281c78d5661SMatthew Dillon vm_page_unwire_quick(m4);
1282c78d5661SMatthew Dillon } else if (p->pindex >= NUPT_TOTAL) {
1283da673940SJordan Gordeev /*
1284da673940SJordan Gordeev * We are a PD page.
1285da673940SJordan Gordeev * We look for the PDP entry that points to us.
1286da673940SJordan Gordeev */
1287c78d5661SMatthew Dillon vm_page_t m3;
1288c78d5661SMatthew Dillon pdp_entry_t *pdp;
1289c78d5661SMatthew Dillon int idx;
1290c78d5661SMatthew Dillon
1291c78d5661SMatthew Dillon m3 = vm_page_lookup(pmap->pm_pteobj,
1292c78d5661SMatthew Dillon NUPT_TOTAL + NUPD_TOTAL +
1293c78d5661SMatthew Dillon (p->pindex - NUPT_TOTAL) / NPDPEPG);
1294da673940SJordan Gordeev KKASSERT(m3 != NULL);
1295c78d5661SMatthew Dillon pdp = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m3));
1296c78d5661SMatthew Dillon idx = (p->pindex - NUPT_TOTAL) % NPDPEPG;
1297da673940SJordan Gordeev KKASSERT(pdp[idx] != 0);
1298c78d5661SMatthew Dillon if (pdp[idx] == 0)
1299c78d5661SMatthew Dillon kprintf("pmap_release: Unmapped PDP %d\n", idx);
1300da673940SJordan Gordeev pdp[idx] = 0;
1301c78d5661SMatthew Dillon vm_page_unwire_quick(m3);
1302da673940SJordan Gordeev } else {
1303da673940SJordan Gordeev /* We are a PT page.
1304da673940SJordan Gordeev * We look for the PD entry that points to us.
1305da673940SJordan Gordeev */
1306c78d5661SMatthew Dillon vm_page_t m2;
1307c78d5661SMatthew Dillon pd_entry_t *pd;
1308c78d5661SMatthew Dillon int idx;
1309c78d5661SMatthew Dillon
1310c78d5661SMatthew Dillon m2 = vm_page_lookup(pmap->pm_pteobj,
1311c78d5661SMatthew Dillon NUPT_TOTAL + p->pindex / NPDEPG);
1312da673940SJordan Gordeev KKASSERT(m2 != NULL);
1313c78d5661SMatthew Dillon pd = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m2));
1314c78d5661SMatthew Dillon idx = p->pindex % NPDEPG;
1315c78d5661SMatthew Dillon if (pd[idx] == 0)
1316c78d5661SMatthew Dillon kprintf("pmap_release: Unmapped PD %d\n", idx);
1317da673940SJordan Gordeev pd[idx] = 0;
1318c78d5661SMatthew Dillon vm_page_unwire_quick(m2);
1319da673940SJordan Gordeev }
1320da673940SJordan Gordeev KKASSERT(pmap->pm_stats.resident_count > 0);
1321c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count, -1);
1322da673940SJordan Gordeev
1323c78d5661SMatthew Dillon if (p->wire_count > 1) {
1324274b5c29SMatthew Dillon panic("pmap_release: freeing held pt page "
1325274b5c29SMatthew Dillon "pmap=%p pg=%p dmap=%p pi=%ld {%ld,%ld,%ld}",
1326274b5c29SMatthew Dillon pmap, p, (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(p)),
1327c78d5661SMatthew Dillon p->pindex, NUPT_TOTAL, NUPD_TOTAL, NUPDP_TOTAL);
1328da673940SJordan Gordeev }
132995270b7eSMatthew Dillon
133095270b7eSMatthew Dillon if (pmap->pm_ptphint == p)
1331da673940SJordan Gordeev pmap->pm_ptphint = NULL;
1332da673940SJordan Gordeev
1333da673940SJordan Gordeev /*
1334da673940SJordan Gordeev * We leave the top-level page table page cached, wired, and mapped in
1335da673940SJordan Gordeev * the pmap until the dtor function (pmap_puninit()) gets called.
1336afd2da4dSMatthew Dillon * However, still clean it up.
1337da673940SJordan Gordeev */
1338c78d5661SMatthew Dillon if (p->pindex == NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL) {
1339da673940SJordan Gordeev bzero(pmap->pm_pml4, PAGE_SIZE);
1340da673940SJordan Gordeev vm_page_wakeup(p);
1341da673940SJordan Gordeev } else {
1342c78d5661SMatthew Dillon vm_page_unwire(p, 0);
1343c78d5661SMatthew Dillon vm_page_flag_clear(p, PG_MAPPED | PG_WRITEABLE);
1344da673940SJordan Gordeev vm_page_free(p);
1345c50e690bSMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, -1);
1346da673940SJordan Gordeev }
1347c78d5661SMatthew Dillon return 0;
1348da673940SJordan Gordeev }
1349da673940SJordan Gordeev
1350da673940SJordan Gordeev /*
1351c78d5661SMatthew Dillon * Locate the requested PT, PD, or PDP page table page.
1352c78d5661SMatthew Dillon *
1353c78d5661SMatthew Dillon * Returns a busied page, caller must vm_page_wakeup() when done.
1354da673940SJordan Gordeev */
1355da673940SJordan Gordeev static vm_page_t
_pmap_allocpte(pmap_t pmap,vm_pindex_t ptepindex)1356da673940SJordan Gordeev _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex)
1357da673940SJordan Gordeev {
1358c78d5661SMatthew Dillon vm_page_t m;
1359c78d5661SMatthew Dillon vm_page_t pm;
1360c78d5661SMatthew Dillon vm_pindex_t pindex;
1361c78d5661SMatthew Dillon pt_entry_t *ptep;
1362c78d5661SMatthew Dillon pt_entry_t data;
1363da673940SJordan Gordeev
1364da673940SJordan Gordeev /*
1365c78d5661SMatthew Dillon * Find or fabricate a new pagetable page. A non-zero wire_count
1366c78d5661SMatthew Dillon * indicates that the page has already been mapped into its parent.
1367da673940SJordan Gordeev */
1368da673940SJordan Gordeev m = vm_page_grab(pmap->pm_pteobj, ptepindex,
1369da673940SJordan Gordeev VM_ALLOC_NORMAL | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
1370c78d5661SMatthew Dillon if (m->wire_count != 0)
1371c78d5661SMatthew Dillon return m;
1372da673940SJordan Gordeev
1373da673940SJordan Gordeev /*
1374c78d5661SMatthew Dillon * Map the page table page into its parent, giving it 1 wire count.
1375da673940SJordan Gordeev */
137654341a3bSMatthew Dillon vm_page_wire(m);
1377ffd470cbSMatthew Dillon vm_page_unqueue(m);
1378c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count, 1);
1379c50e690bSMatthew Dillon vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE);
1380da673940SJordan Gordeev
1381c78d5661SMatthew Dillon data = VM_PAGE_TO_PHYS(m) |
1382c50e690bSMatthew Dillon VPTE_RW | VPTE_V | VPTE_U | VPTE_A | VPTE_M | VPTE_WIRED;
1383c50e690bSMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, 1);
1384da673940SJordan Gordeev
1385c78d5661SMatthew Dillon if (ptepindex >= (NUPT_TOTAL + NUPD_TOTAL)) {
1386c78d5661SMatthew Dillon /*
1387c78d5661SMatthew Dillon * Map PDP into the PML4
1388da673940SJordan Gordeev */
1389c78d5661SMatthew Dillon pindex = ptepindex - (NUPT_TOTAL + NUPD_TOTAL);
1390c78d5661SMatthew Dillon pindex &= (NUPDP_TOTAL - 1);
1391c78d5661SMatthew Dillon ptep = (pt_entry_t *)pmap->pm_pml4;
1392c78d5661SMatthew Dillon pm = NULL;
1393c78d5661SMatthew Dillon } else if (ptepindex >= NUPT_TOTAL) {
1394c78d5661SMatthew Dillon /*
1395c78d5661SMatthew Dillon * Map PD into its PDP
1396c78d5661SMatthew Dillon */
1397c78d5661SMatthew Dillon pindex = (ptepindex - NUPT_TOTAL) >> NPDPEPGSHIFT;
1398c78d5661SMatthew Dillon pindex += NUPT_TOTAL + NUPD_TOTAL;
1399c78d5661SMatthew Dillon pm = _pmap_allocpte(pmap, pindex);
1400c78d5661SMatthew Dillon pindex = (ptepindex - NUPT_TOTAL) & (NPDPEPG - 1);
1401c78d5661SMatthew Dillon ptep = (void *)PHYS_TO_DMAP(pm->phys_addr);
1402da673940SJordan Gordeev } else {
1403c78d5661SMatthew Dillon /*
1404c78d5661SMatthew Dillon * Map PT into its PD
1405c78d5661SMatthew Dillon */
1406c78d5661SMatthew Dillon pindex = ptepindex >> NPDPEPGSHIFT;
1407c78d5661SMatthew Dillon pindex += NUPT_TOTAL;
1408c78d5661SMatthew Dillon pm = _pmap_allocpte(pmap, pindex);
1409c78d5661SMatthew Dillon pindex = ptepindex & (NPTEPG - 1);
1410c78d5661SMatthew Dillon ptep = (void *)PHYS_TO_DMAP(pm->phys_addr);
1411da673940SJordan Gordeev }
1412da673940SJordan Gordeev
1413da673940SJordan Gordeev /*
1414c78d5661SMatthew Dillon * Install the pte in (pm). (m) prevents races.
1415da673940SJordan Gordeev */
1416c78d5661SMatthew Dillon ptep += pindex;
1417c78d5661SMatthew Dillon data = atomic_swap_long(ptep, data);
1418c78d5661SMatthew Dillon if (pm) {
1419c78d5661SMatthew Dillon vm_page_wire_quick(pm);
1420c78d5661SMatthew Dillon vm_page_wakeup(pm);
1421c78d5661SMatthew Dillon }
1422c78d5661SMatthew Dillon pmap->pm_ptphint = pm;
1423da673940SJordan Gordeev
1424da673940SJordan Gordeev return m;
1425da673940SJordan Gordeev }
1426da673940SJordan Gordeev
1427da673940SJordan Gordeev /*
1428da673940SJordan Gordeev * Determine the page table page required to access the VA in the pmap
1429da673940SJordan Gordeev * and allocate it if necessary. Return a held vm_page_t for the page.
1430da673940SJordan Gordeev *
1431da673940SJordan Gordeev * Only used with user pmaps.
1432da673940SJordan Gordeev */
1433da673940SJordan Gordeev static vm_page_t
pmap_allocpte(pmap_t pmap,vm_offset_t va)1434da673940SJordan Gordeev pmap_allocpte(pmap_t pmap, vm_offset_t va)
1435da673940SJordan Gordeev {
1436da673940SJordan Gordeev vm_pindex_t ptepindex;
1437da673940SJordan Gordeev vm_page_t m;
1438da673940SJordan Gordeev
1439b12defdcSMatthew Dillon ASSERT_LWKT_TOKEN_HELD(vm_object_token(pmap->pm_pteobj));
1440b12defdcSMatthew Dillon
1441da673940SJordan Gordeev /*
1442c78d5661SMatthew Dillon * Calculate pagetable page index, and return the PT page to
1443c78d5661SMatthew Dillon * the caller.
1444da673940SJordan Gordeev */
1445c78d5661SMatthew Dillon ptepindex = pmap_pt_pindex(va);
1446c78d5661SMatthew Dillon m = _pmap_allocpte(pmap, ptepindex);
1447da673940SJordan Gordeev
1448da673940SJordan Gordeev return m;
1449da673940SJordan Gordeev }
1450da673940SJordan Gordeev
1451da673940SJordan Gordeev /***************************************************
1452da673940SJordan Gordeev * Pmap allocation/deallocation routines.
1453da673940SJordan Gordeev ***************************************************/
1454da673940SJordan Gordeev
1455da673940SJordan Gordeev /*
1456da673940SJordan Gordeev * Release any resources held by the given physical map.
1457da673940SJordan Gordeev * Called when a pmap initialized by pmap_pinit is being released.
1458da673940SJordan Gordeev * Should only be called if the map contains no valid mappings.
1459da673940SJordan Gordeev */
1460da673940SJordan Gordeev static int pmap_release_callback(struct vm_page *p, void *data);
1461da673940SJordan Gordeev
1462da673940SJordan Gordeev void
pmap_release(struct pmap * pmap)1463da673940SJordan Gordeev pmap_release(struct pmap *pmap)
1464da673940SJordan Gordeev {
1465da673940SJordan Gordeev vm_object_t object = pmap->pm_pteobj;
1466da673940SJordan Gordeev struct rb_vm_page_scan_info info;
1467da673940SJordan Gordeev
1468*c713db65SAaron LI KKASSERT(pmap != kernel_pmap);
1469da673940SJordan Gordeev
1470da673940SJordan Gordeev #if defined(DIAGNOSTIC)
1471da673940SJordan Gordeev if (object->ref_count != 1)
1472da673940SJordan Gordeev panic("pmap_release: pteobj reference count != 1");
1473da673940SJordan Gordeev #endif
1474da673940SJordan Gordeev
1475da673940SJordan Gordeev info.pmap = pmap;
1476da673940SJordan Gordeev info.object = object;
1477da673940SJordan Gordeev
1478685f4bf5SMatthew Dillon KASSERT(CPUMASK_TESTZERO(pmap->pm_active),
1479685f4bf5SMatthew Dillon ("pmap %p still active! %016jx",
1480685f4bf5SMatthew Dillon pmap,
1481685f4bf5SMatthew Dillon (uintmax_t)CPUMASK_LOWMASK(pmap->pm_active)));
1482685f4bf5SMatthew Dillon
1483b12defdcSMatthew Dillon vm_object_hold(object);
1484da673940SJordan Gordeev do {
1485da673940SJordan Gordeev info.error = 0;
1486da673940SJordan Gordeev info.mpte = NULL;
1487da673940SJordan Gordeev info.limit = object->generation;
1488da673940SJordan Gordeev
1489da673940SJordan Gordeev vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
1490da673940SJordan Gordeev pmap_release_callback, &info);
1491da673940SJordan Gordeev if (info.error == 0 && info.mpte) {
1492c78d5661SMatthew Dillon if (pmap_release_free_page(pmap, info.mpte))
1493da673940SJordan Gordeev info.error = 1;
1494da673940SJordan Gordeev }
1495da673940SJordan Gordeev } while (info.error);
1496c50e690bSMatthew Dillon
149795270b7eSMatthew Dillon pmap->pm_ptphint = NULL;
149895270b7eSMatthew Dillon
1499c50e690bSMatthew Dillon KASSERT((pmap->pm_stats.wired_count == (pmap->pm_pdirm != NULL)),
1500c50e690bSMatthew Dillon ("pmap_release: dangling count %p %ld",
1501c50e690bSMatthew Dillon pmap, pmap->pm_stats.wired_count));
1502c50e690bSMatthew Dillon
1503b12defdcSMatthew Dillon vm_object_drop(object);
1504da673940SJordan Gordeev }
1505da673940SJordan Gordeev
1506da673940SJordan Gordeev static int
pmap_release_callback(struct vm_page * p,void * data)1507da673940SJordan Gordeev pmap_release_callback(struct vm_page *p, void *data)
1508da673940SJordan Gordeev {
1509da673940SJordan Gordeev struct rb_vm_page_scan_info *info = data;
1510da673940SJordan Gordeev
1511c78d5661SMatthew Dillon if (p->pindex == NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL) {
1512da673940SJordan Gordeev info->mpte = p;
1513da673940SJordan Gordeev return(0);
1514da673940SJordan Gordeev }
1515c78d5661SMatthew Dillon if (pmap_release_free_page(info->pmap, p)) {
1516da673940SJordan Gordeev info->error = 1;
1517da673940SJordan Gordeev return(-1);
1518da673940SJordan Gordeev }
1519da673940SJordan Gordeev if (info->object->generation != info->limit) {
1520da673940SJordan Gordeev info->error = 1;
1521da673940SJordan Gordeev return(-1);
1522da673940SJordan Gordeev }
1523da673940SJordan Gordeev return(0);
1524da673940SJordan Gordeev }
1525da673940SJordan Gordeev
1526da673940SJordan Gordeev /*
1527da673940SJordan Gordeev * Grow the number of kernel page table entries, if needed.
1528f7230403SMatthew Dillon *
1529d95d5e03SMatthew Dillon * kernel_map must be locked exclusively by the caller.
1530da673940SJordan Gordeev */
1531da673940SJordan Gordeev void
pmap_growkernel(vm_offset_t kstart,vm_offset_t kend)1532a8cf2878SMatthew Dillon pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
1533da673940SJordan Gordeev {
1534a8cf2878SMatthew Dillon vm_offset_t addr;
1535da673940SJordan Gordeev vm_paddr_t paddr;
1536da673940SJordan Gordeev vm_offset_t ptppaddr;
1537da673940SJordan Gordeev vm_page_t nkpg;
1538da673940SJordan Gordeev pd_entry_t *pde, newpdir;
1539da673940SJordan Gordeev pdp_entry_t newpdp;
1540da673940SJordan Gordeev
1541a8cf2878SMatthew Dillon addr = kend;
1542a8cf2878SMatthew Dillon
154315d6dd34SAntonio Huete Jimenez vm_object_hold(&kptobj);
1544da673940SJordan Gordeev if (kernel_vm_end == 0) {
1545da673940SJordan Gordeev kernel_vm_end = KvaStart;
1546da673940SJordan Gordeev nkpt = 0;
1547*c713db65SAaron LI while ((*pmap_pde(kernel_pmap, kernel_vm_end) & VPTE_V) != 0) {
15483f7b7260SSascha Wildner kernel_vm_end =
15493f7b7260SSascha Wildner rounddown2(kernel_vm_end + PAGE_SIZE * NPTEPG,
15503f7b7260SSascha Wildner PAGE_SIZE * NPTEPG);
1551da673940SJordan Gordeev nkpt++;
15521eeaf6b2SAaron LI if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
15531eeaf6b2SAaron LI kernel_vm_end = vm_map_max(kernel_map);
1554da673940SJordan Gordeev break;
1555da673940SJordan Gordeev }
1556da673940SJordan Gordeev }
1557da673940SJordan Gordeev }
1558da673940SJordan Gordeev addr = roundup2(addr, PAGE_SIZE * NPTEPG);
15591eeaf6b2SAaron LI if (addr - 1 >= vm_map_max(kernel_map))
15601eeaf6b2SAaron LI addr = vm_map_max(kernel_map);
1561da673940SJordan Gordeev while (kernel_vm_end < addr) {
1562*c713db65SAaron LI pde = pmap_pde(kernel_pmap, kernel_vm_end);
1563da673940SJordan Gordeev if (pde == NULL) {
1564da673940SJordan Gordeev /* We need a new PDP entry */
156515d6dd34SAntonio Huete Jimenez nkpg = vm_page_alloc(&kptobj, nkpt,
156600eb801eSMatthew Dillon VM_ALLOC_NORMAL |
156700eb801eSMatthew Dillon VM_ALLOC_SYSTEM |
156800eb801eSMatthew Dillon VM_ALLOC_INTERRUPT);
1569a8cf2878SMatthew Dillon if (nkpg == NULL) {
1570a8cf2878SMatthew Dillon panic("pmap_growkernel: no memory to "
1571a8cf2878SMatthew Dillon "grow kernel");
1572a8cf2878SMatthew Dillon }
1573da673940SJordan Gordeev paddr = VM_PAGE_TO_PHYS(nkpg);
1574da673940SJordan Gordeev pmap_zero_page(paddr);
1575a86ce0cdSMatthew Dillon newpdp = (pdp_entry_t)(paddr |
1576a86ce0cdSMatthew Dillon VPTE_V | VPTE_RW | VPTE_U |
1577c50e690bSMatthew Dillon VPTE_A | VPTE_M | VPTE_WIRED);
1578*c713db65SAaron LI *pmap_pdpe(kernel_pmap, kernel_vm_end) = newpdp;
1579*c713db65SAaron LI atomic_add_long(&kernel_pmap->pm_stats.wired_count, 1);
1580da673940SJordan Gordeev nkpt++;
1581da673940SJordan Gordeev continue; /* try again */
1582da673940SJordan Gordeev }
1583da673940SJordan Gordeev if ((*pde & VPTE_V) != 0) {
15843f7b7260SSascha Wildner kernel_vm_end =
15853f7b7260SSascha Wildner rounddown2(kernel_vm_end + PAGE_SIZE * NPTEPG,
15863f7b7260SSascha Wildner PAGE_SIZE * NPTEPG);
15871eeaf6b2SAaron LI if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
15881eeaf6b2SAaron LI kernel_vm_end = vm_map_max(kernel_map);
1589da673940SJordan Gordeev break;
1590da673940SJordan Gordeev }
1591da673940SJordan Gordeev continue;
1592da673940SJordan Gordeev }
1593da673940SJordan Gordeev
1594da673940SJordan Gordeev /*
1595da673940SJordan Gordeev * This index is bogus, but out of the way
1596da673940SJordan Gordeev */
159715d6dd34SAntonio Huete Jimenez nkpg = vm_page_alloc(&kptobj, nkpt,
1598a8cf2878SMatthew Dillon VM_ALLOC_NORMAL |
1599a8cf2878SMatthew Dillon VM_ALLOC_SYSTEM |
1600a8cf2878SMatthew Dillon VM_ALLOC_INTERRUPT);
1601da673940SJordan Gordeev if (nkpg == NULL)
1602da673940SJordan Gordeev panic("pmap_growkernel: no memory to grow kernel");
1603da673940SJordan Gordeev
1604da673940SJordan Gordeev vm_page_wire(nkpg);
1605da673940SJordan Gordeev ptppaddr = VM_PAGE_TO_PHYS(nkpg);
1606da673940SJordan Gordeev pmap_zero_page(ptppaddr);
1607a86ce0cdSMatthew Dillon newpdir = (pd_entry_t)(ptppaddr |
1608a86ce0cdSMatthew Dillon VPTE_V | VPTE_RW | VPTE_U |
1609c50e690bSMatthew Dillon VPTE_A | VPTE_M | VPTE_WIRED);
1610*c713db65SAaron LI *pmap_pde(kernel_pmap, kernel_vm_end) = newpdir;
1611*c713db65SAaron LI atomic_add_long(&kernel_pmap->pm_stats.wired_count, 1);
1612da673940SJordan Gordeev nkpt++;
1613da673940SJordan Gordeev
16143f7b7260SSascha Wildner kernel_vm_end =
16153f7b7260SSascha Wildner rounddown2(kernel_vm_end + PAGE_SIZE * NPTEPG,
16163f7b7260SSascha Wildner PAGE_SIZE * NPTEPG);
16171eeaf6b2SAaron LI if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
16181eeaf6b2SAaron LI kernel_vm_end = vm_map_max(kernel_map);
1619da673940SJordan Gordeev break;
1620da673940SJordan Gordeev }
1621da673940SJordan Gordeev }
162215d6dd34SAntonio Huete Jimenez vm_object_drop(&kptobj);
1623da673940SJordan Gordeev }
1624da673940SJordan Gordeev
1625da673940SJordan Gordeev /*
1626da673940SJordan Gordeev * Add a reference to the specified pmap.
1627f7230403SMatthew Dillon *
1628f7230403SMatthew Dillon * No requirements.
1629da673940SJordan Gordeev */
1630da673940SJordan Gordeev void
pmap_reference(pmap_t pmap)1631da673940SJordan Gordeev pmap_reference(pmap_t pmap)
1632da673940SJordan Gordeev {
1633c91894e0SMatthew Dillon if (pmap)
1634c91894e0SMatthew Dillon atomic_add_int(&pmap->pm_count, 1);
1635da673940SJordan Gordeev }
1636da673940SJordan Gordeev
1637da673940SJordan Gordeev /************************************************************************
1638da673940SJordan Gordeev * VMSPACE MANAGEMENT *
1639da673940SJordan Gordeev ************************************************************************
1640da673940SJordan Gordeev *
1641da673940SJordan Gordeev * The VMSPACE management we do in our virtual kernel must be reflected
1642da673940SJordan Gordeev * in the real kernel. This is accomplished by making vmspace system
1643da673940SJordan Gordeev * calls to the real kernel.
1644da673940SJordan Gordeev */
1645da673940SJordan Gordeev void
cpu_vmspace_alloc(struct vmspace * vm)1646da673940SJordan Gordeev cpu_vmspace_alloc(struct vmspace *vm)
1647da673940SJordan Gordeev {
1648da673940SJordan Gordeev int r;
1649da673940SJordan Gordeev void *rp;
1650da673940SJordan Gordeev vpte_t vpte;
1651da673940SJordan Gordeev
1652da673940SJordan Gordeev #define USER_SIZE (VM_MAX_USER_ADDRESS - VM_MIN_USER_ADDRESS)
1653da673940SJordan Gordeev
1654da673940SJordan Gordeev if (vmspace_create(&vm->vm_pmap, 0, NULL) < 0)
1655da673940SJordan Gordeev panic("vmspace_create() failed");
1656da673940SJordan Gordeev
1657da673940SJordan Gordeev rp = vmspace_mmap(&vm->vm_pmap, VM_MIN_USER_ADDRESS, USER_SIZE,
165873d64b98SMatthew Dillon PROT_READ|PROT_WRITE|PROT_EXEC,
1659da673940SJordan Gordeev MAP_FILE|MAP_SHARED|MAP_VPAGETABLE|MAP_FIXED,
1660da673940SJordan Gordeev MemImageFd, 0);
1661da673940SJordan Gordeev if (rp == MAP_FAILED)
1662da673940SJordan Gordeev panic("vmspace_mmap: failed");
1663da673940SJordan Gordeev vmspace_mcontrol(&vm->vm_pmap, VM_MIN_USER_ADDRESS, USER_SIZE,
1664da673940SJordan Gordeev MADV_NOSYNC, 0);
1665c50e690bSMatthew Dillon vpte = VM_PAGE_TO_PHYS(vmspace_pmap(vm)->pm_pdirm) |
1666c50e690bSMatthew Dillon VPTE_RW | VPTE_V | VPTE_U;
1667da673940SJordan Gordeev r = vmspace_mcontrol(&vm->vm_pmap, VM_MIN_USER_ADDRESS, USER_SIZE,
1668da673940SJordan Gordeev MADV_SETMAP, vpte);
1669da673940SJordan Gordeev if (r < 0)
1670da673940SJordan Gordeev panic("vmspace_mcontrol: failed");
1671da673940SJordan Gordeev }
1672da673940SJordan Gordeev
1673da673940SJordan Gordeev void
cpu_vmspace_free(struct vmspace * vm)1674da673940SJordan Gordeev cpu_vmspace_free(struct vmspace *vm)
1675da673940SJordan Gordeev {
1676da673940SJordan Gordeev if (vmspace_destroy(&vm->vm_pmap) < 0)
1677da673940SJordan Gordeev panic("vmspace_destroy() failed");
1678da673940SJordan Gordeev }
1679da673940SJordan Gordeev
1680da673940SJordan Gordeev /***************************************************
1681da673940SJordan Gordeev * page management routines.
1682da673940SJordan Gordeev ***************************************************/
1683da673940SJordan Gordeev
1684da673940SJordan Gordeev /*
1685da673940SJordan Gordeev * free the pv_entry back to the free list. This function may be
1686da673940SJordan Gordeev * called from an interrupt.
1687da673940SJordan Gordeev */
1688da673940SJordan Gordeev static __inline void
free_pv_entry(pv_entry_t pv)1689da673940SJordan Gordeev free_pv_entry(pv_entry_t pv)
1690da673940SJordan Gordeev {
1691b7ea2f3fSMatthew Dillon atomic_add_long(&pv_entry_count, -1);
1692da673940SJordan Gordeev zfree(pvzone, pv);
1693da673940SJordan Gordeev }
1694da673940SJordan Gordeev
1695da673940SJordan Gordeev /*
1696da673940SJordan Gordeev * get a new pv_entry, allocating a block from the system
1697da673940SJordan Gordeev * when needed. This function may be called from an interrupt.
1698da673940SJordan Gordeev */
1699da673940SJordan Gordeev static pv_entry_t
get_pv_entry(void)1700da673940SJordan Gordeev get_pv_entry(void)
1701da673940SJordan Gordeev {
1702b7ea2f3fSMatthew Dillon atomic_add_long(&pv_entry_count, 1);
1703da673940SJordan Gordeev if (pv_entry_high_water &&
1704da673940SJordan Gordeev (pv_entry_count > pv_entry_high_water) &&
1705005328d6SMatthew Dillon atomic_swap_int(&pmap_pagedaemon_waken, 1) == 0) {
1706da673940SJordan Gordeev wakeup(&vm_pages_needed);
1707da673940SJordan Gordeev }
1708da673940SJordan Gordeev return zalloc(pvzone);
1709da673940SJordan Gordeev }
1710da673940SJordan Gordeev
1711da673940SJordan Gordeev /*
1712da673940SJordan Gordeev * This routine is very drastic, but can save the system
1713da673940SJordan Gordeev * in a pinch.
1714f7230403SMatthew Dillon *
1715f7230403SMatthew Dillon * No requirements.
1716da673940SJordan Gordeev */
1717da673940SJordan Gordeev void
pmap_collect(void)1718da673940SJordan Gordeev pmap_collect(void)
1719da673940SJordan Gordeev {
1720da673940SJordan Gordeev int i;
1721da673940SJordan Gordeev vm_page_t m;
1722da673940SJordan Gordeev static int warningdone=0;
1723da673940SJordan Gordeev
1724da673940SJordan Gordeev if (pmap_pagedaemon_waken == 0)
1725da673940SJordan Gordeev return;
1726da673940SJordan Gordeev pmap_pagedaemon_waken = 0;
1727da673940SJordan Gordeev
1728da673940SJordan Gordeev if (warningdone < 5) {
1729685f4bf5SMatthew Dillon kprintf("pmap_collect: collecting pv entries -- "
1730685f4bf5SMatthew Dillon "suggest increasing PMAP_SHPGPERPROC\n");
1731da673940SJordan Gordeev warningdone++;
1732da673940SJordan Gordeev }
1733da673940SJordan Gordeev
1734da673940SJordan Gordeev for (i = 0; i < vm_page_array_size; i++) {
1735da673940SJordan Gordeev m = &vm_page_array[i];
1736b12defdcSMatthew Dillon if (m->wire_count || m->hold_count)
1737da673940SJordan Gordeev continue;
1738b12defdcSMatthew Dillon if (vm_page_busy_try(m, TRUE) == 0) {
1739b12defdcSMatthew Dillon if (m->wire_count == 0 && m->hold_count == 0) {
1740da673940SJordan Gordeev pmap_remove_all(m);
1741da673940SJordan Gordeev }
1742b12defdcSMatthew Dillon vm_page_wakeup(m);
1743b12defdcSMatthew Dillon }
1744b12defdcSMatthew Dillon }
1745da673940SJordan Gordeev }
1746da673940SJordan Gordeev
1747da673940SJordan Gordeev
1748da673940SJordan Gordeev /*
1749da673940SJordan Gordeev * If it is the first entry on the list, it is actually
1750da673940SJordan Gordeev * in the header and we must copy the following entry up
1751da673940SJordan Gordeev * to the header. Otherwise we must search the list for
1752da673940SJordan Gordeev * the entry. In either case we free the now unused entry.
1753b12defdcSMatthew Dillon *
1754c91894e0SMatthew Dillon * pmap->pm_pteobj must be held and (m) must be spin-locked by the caller.
1755da673940SJordan Gordeev */
1756da673940SJordan Gordeev static int
pmap_remove_entry(struct pmap * pmap,vm_page_t m,vm_offset_t va)1757da673940SJordan Gordeev pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va)
1758da673940SJordan Gordeev {
1759da673940SJordan Gordeev pv_entry_t pv;
1760da673940SJordan Gordeev int rtval;
1761da673940SJordan Gordeev
1762c78d5661SMatthew Dillon vm_page_spin_lock(m);
1763c50e690bSMatthew Dillon pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot, va);
1764da673940SJordan Gordeev
1765da673940SJordan Gordeev /*
1766da673940SJordan Gordeev * Note that pv_ptem is NULL if the page table page itself is not
1767da673940SJordan Gordeev * managed, even if the page being removed IS managed.
1768da673940SJordan Gordeev */
1769da673940SJordan Gordeev rtval = 0;
1770da673940SJordan Gordeev if (pv) {
1771da673940SJordan Gordeev TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
1772da673940SJordan Gordeev if (TAILQ_EMPTY(&m->md.pv_list))
1773da673940SJordan Gordeev vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
1774b443039bSMatthew Dillon m->md.pv_list_count--;
1775b443039bSMatthew Dillon KKASSERT(m->md.pv_list_count >= 0);
1776c50e690bSMatthew Dillon pv_entry_rb_tree_RB_REMOVE(&pmap->pm_pvroot, pv);
1777c91894e0SMatthew Dillon atomic_add_int(&pmap->pm_generation, 1);
1778c78d5661SMatthew Dillon vm_page_spin_unlock(m);
1779da673940SJordan Gordeev rtval = pmap_unuse_pt(pmap, va, pv->pv_ptem);
1780da673940SJordan Gordeev free_pv_entry(pv);
1781c78d5661SMatthew Dillon } else {
1782c78d5661SMatthew Dillon vm_page_spin_unlock(m);
1783c78d5661SMatthew Dillon kprintf("pmap_remove_entry: could not find "
1784c78d5661SMatthew Dillon "pmap=%p m=%p va=%016jx\n",
1785c78d5661SMatthew Dillon pmap, m, va);
1786da673940SJordan Gordeev }
1787da673940SJordan Gordeev return rtval;
1788da673940SJordan Gordeev }
1789da673940SJordan Gordeev
1790da673940SJordan Gordeev /*
1791da673940SJordan Gordeev * Create a pv entry for page at pa for (pmap, va). If the page table page
1792da673940SJordan Gordeev * holding the VA is managed, mpte will be non-NULL.
1793c91894e0SMatthew Dillon *
1794c91894e0SMatthew Dillon * pmap->pm_pteobj must be held and (m) must be spin-locked by the caller.
1795da673940SJordan Gordeev */
1796da673940SJordan Gordeev static void
pmap_insert_entry(pmap_t pmap,vm_offset_t va,vm_page_t mpte,vm_page_t m,pv_entry_t pv)1797c78d5661SMatthew Dillon pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_page_t m,
1798c78d5661SMatthew Dillon pv_entry_t pv)
1799da673940SJordan Gordeev {
1800da673940SJordan Gordeev pv->pv_va = va;
1801da673940SJordan Gordeev pv->pv_pmap = pmap;
1802da673940SJordan Gordeev pv->pv_ptem = mpte;
1803da673940SJordan Gordeev
1804da673940SJordan Gordeev m->md.pv_list_count++;
1805c50e690bSMatthew Dillon TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1806c50e690bSMatthew Dillon pv = pv_entry_rb_tree_RB_INSERT(&pmap->pm_pvroot, pv);
180795270b7eSMatthew Dillon vm_page_flag_set(m, PG_MAPPED);
1808c50e690bSMatthew Dillon KKASSERT(pv == NULL);
1809da673940SJordan Gordeev }
1810da673940SJordan Gordeev
1811da673940SJordan Gordeev /*
1812da673940SJordan Gordeev * pmap_remove_pte: do the things to unmap a page in a process
1813c91894e0SMatthew Dillon *
1814c78d5661SMatthew Dillon * Caller holds pmap->pm_pteobj and holds the associated page table
1815c78d5661SMatthew Dillon * page busy to prevent races.
1816da673940SJordan Gordeev */
1817da673940SJordan Gordeev static int
pmap_remove_pte(struct pmap * pmap,pt_entry_t * ptq,pt_entry_t oldpte,vm_offset_t va)181800eb801eSMatthew Dillon pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, pt_entry_t oldpte,
181900eb801eSMatthew Dillon vm_offset_t va)
1820da673940SJordan Gordeev {
1821da673940SJordan Gordeev vm_page_t m;
1822c91894e0SMatthew Dillon int error;
1823da673940SJordan Gordeev
182400eb801eSMatthew Dillon if (ptq)
1825da673940SJordan Gordeev oldpte = pmap_inval_loadandclear(ptq, pmap, va);
182600eb801eSMatthew Dillon
1827da673940SJordan Gordeev if (oldpte & VPTE_WIRED)
1828c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, -1);
1829da673940SJordan Gordeev KKASSERT(pmap->pm_stats.wired_count >= 0);
1830da673940SJordan Gordeev
1831da673940SJordan Gordeev #if 0
1832da673940SJordan Gordeev /*
1833da673940SJordan Gordeev * Machines that don't support invlpg, also don't support
1834da673940SJordan Gordeev * PG_G. XXX PG_G is disabled for SMP so don't worry about
1835da673940SJordan Gordeev * the SMP case.
1836da673940SJordan Gordeev */
1837da673940SJordan Gordeev if (oldpte & PG_G)
1838da673940SJordan Gordeev cpu_invlpg((void *)va);
1839da673940SJordan Gordeev #endif
1840da673940SJordan Gordeev KKASSERT(pmap->pm_stats.resident_count > 0);
1841c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count, -1);
1842da673940SJordan Gordeev if (oldpte & VPTE_MANAGED) {
1843da673940SJordan Gordeev m = PHYS_TO_VM_PAGE(oldpte);
1844c78d5661SMatthew Dillon
1845c78d5661SMatthew Dillon /*
1846c78d5661SMatthew Dillon * NOTE: pmap_remove_entry() will spin-lock the page
1847c78d5661SMatthew Dillon */
1848da673940SJordan Gordeev if (oldpte & VPTE_M) {
1849da673940SJordan Gordeev #if defined(PMAP_DIAGNOSTIC)
185006bb314fSSascha Wildner if (pmap_nw_modified(oldpte)) {
1851b12defdcSMatthew Dillon kprintf("pmap_remove: modified page not "
1852b12defdcSMatthew Dillon "writable: va: 0x%lx, pte: 0x%lx\n",
1853da673940SJordan Gordeev va, oldpte);
1854da673940SJordan Gordeev }
1855da673940SJordan Gordeev #endif
1856ae442b2eSMatthew Dillon pmap_track_modified(pmap, va);
1857da673940SJordan Gordeev vm_page_dirty(m);
1858da673940SJordan Gordeev }
1859da673940SJordan Gordeev if (oldpte & VPTE_A)
1860da673940SJordan Gordeev vm_page_flag_set(m, PG_REFERENCED);
1861c91894e0SMatthew Dillon error = pmap_remove_entry(pmap, m, va);
1862da673940SJordan Gordeev } else {
1863c91894e0SMatthew Dillon error = pmap_unuse_pt(pmap, va, NULL);
1864da673940SJordan Gordeev }
1865c91894e0SMatthew Dillon return error;
1866da673940SJordan Gordeev }
1867da673940SJordan Gordeev
1868da673940SJordan Gordeev /*
1869da673940SJordan Gordeev * pmap_remove_page:
1870da673940SJordan Gordeev *
1871da673940SJordan Gordeev * Remove a single page from a process address space.
1872da673940SJordan Gordeev *
1873da673940SJordan Gordeev * This function may not be called from an interrupt if the pmap is
1874da673940SJordan Gordeev * not kernel_pmap.
1875c91894e0SMatthew Dillon *
1876c91894e0SMatthew Dillon * Caller holds pmap->pm_pteobj
1877da673940SJordan Gordeev */
1878da673940SJordan Gordeev static void
pmap_remove_page(struct pmap * pmap,vm_offset_t va)1879da673940SJordan Gordeev pmap_remove_page(struct pmap *pmap, vm_offset_t va)
1880da673940SJordan Gordeev {
1881da673940SJordan Gordeev pt_entry_t *pte;
1882da673940SJordan Gordeev
1883da673940SJordan Gordeev pte = pmap_pte(pmap, va);
1884da673940SJordan Gordeev if (pte == NULL)
1885da673940SJordan Gordeev return;
1886da673940SJordan Gordeev if ((*pte & VPTE_V) == 0)
1887da673940SJordan Gordeev return;
188800eb801eSMatthew Dillon pmap_remove_pte(pmap, pte, 0, va);
1889da673940SJordan Gordeev }
1890da673940SJordan Gordeev
1891da673940SJordan Gordeev /*
1892da673940SJordan Gordeev * Remove the given range of addresses from the specified map.
1893da673940SJordan Gordeev *
1894f7230403SMatthew Dillon * It is assumed that the start and end are properly rounded to
1895f7230403SMatthew Dillon * the page size.
1896da673940SJordan Gordeev *
1897da673940SJordan Gordeev * This function may not be called from an interrupt if the pmap is
1898da673940SJordan Gordeev * not kernel_pmap.
1899f7230403SMatthew Dillon *
1900f7230403SMatthew Dillon * No requirements.
1901da673940SJordan Gordeev */
1902da673940SJordan Gordeev void
pmap_remove(struct pmap * pmap,vm_offset_t sva,vm_offset_t eva)1903da673940SJordan Gordeev pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
1904da673940SJordan Gordeev {
1905da673940SJordan Gordeev vm_offset_t va_next;
1906da673940SJordan Gordeev pml4_entry_t *pml4e;
1907da673940SJordan Gordeev pdp_entry_t *pdpe;
1908da673940SJordan Gordeev pd_entry_t ptpaddr, *pde;
1909da673940SJordan Gordeev pt_entry_t *pte;
1910eb36cb6bSMatthew Dillon vm_page_t pt_m;
1911da673940SJordan Gordeev
1912da673940SJordan Gordeev if (pmap == NULL)
1913da673940SJordan Gordeev return;
1914da673940SJordan Gordeev
1915b12defdcSMatthew Dillon vm_object_hold(pmap->pm_pteobj);
1916da673940SJordan Gordeev KKASSERT(pmap->pm_stats.resident_count >= 0);
1917f7230403SMatthew Dillon if (pmap->pm_stats.resident_count == 0) {
1918b12defdcSMatthew Dillon vm_object_drop(pmap->pm_pteobj);
1919da673940SJordan Gordeev return;
1920f7230403SMatthew Dillon }
1921da673940SJordan Gordeev
1922da673940SJordan Gordeev /*
1923da673940SJordan Gordeev * special handling of removing one page. a very
1924da673940SJordan Gordeev * common operation and easy to short circuit some
1925da673940SJordan Gordeev * code.
1926da673940SJordan Gordeev */
1927da673940SJordan Gordeev if (sva + PAGE_SIZE == eva) {
1928da673940SJordan Gordeev pde = pmap_pde(pmap, sva);
1929da673940SJordan Gordeev if (pde && (*pde & VPTE_PS) == 0) {
1930da673940SJordan Gordeev pmap_remove_page(pmap, sva);
1931b12defdcSMatthew Dillon vm_object_drop(pmap->pm_pteobj);
1932da673940SJordan Gordeev return;
1933da673940SJordan Gordeev }
1934da673940SJordan Gordeev }
1935da673940SJordan Gordeev
1936da673940SJordan Gordeev for (; sva < eva; sva = va_next) {
1937da673940SJordan Gordeev pml4e = pmap_pml4e(pmap, sva);
1938da673940SJordan Gordeev if ((*pml4e & VPTE_V) == 0) {
1939da673940SJordan Gordeev va_next = (sva + NBPML4) & ~PML4MASK;
1940da673940SJordan Gordeev if (va_next < sva)
1941da673940SJordan Gordeev va_next = eva;
1942da673940SJordan Gordeev continue;
1943da673940SJordan Gordeev }
1944da673940SJordan Gordeev
1945da673940SJordan Gordeev pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
1946da673940SJordan Gordeev if ((*pdpe & VPTE_V) == 0) {
1947da673940SJordan Gordeev va_next = (sva + NBPDP) & ~PDPMASK;
1948da673940SJordan Gordeev if (va_next < sva)
1949da673940SJordan Gordeev va_next = eva;
1950da673940SJordan Gordeev continue;
1951da673940SJordan Gordeev }
1952da673940SJordan Gordeev
1953da673940SJordan Gordeev /*
1954da673940SJordan Gordeev * Calculate index for next page table.
1955da673940SJordan Gordeev */
1956da673940SJordan Gordeev va_next = (sva + NBPDR) & ~PDRMASK;
1957da673940SJordan Gordeev if (va_next < sva)
1958da673940SJordan Gordeev va_next = eva;
1959da673940SJordan Gordeev
1960da673940SJordan Gordeev pde = pmap_pdpe_to_pde(pdpe, sva);
1961da673940SJordan Gordeev ptpaddr = *pde;
1962da673940SJordan Gordeev
1963da673940SJordan Gordeev /*
1964da673940SJordan Gordeev * Weed out invalid mappings.
1965da673940SJordan Gordeev */
1966da673940SJordan Gordeev if (ptpaddr == 0)
1967da673940SJordan Gordeev continue;
1968da673940SJordan Gordeev
1969da673940SJordan Gordeev /*
1970da673940SJordan Gordeev * Check for large page.
1971da673940SJordan Gordeev */
1972da673940SJordan Gordeev if ((ptpaddr & VPTE_PS) != 0) {
1973da673940SJordan Gordeev /* JG FreeBSD has more complex treatment here */
1974da673940SJordan Gordeev KKASSERT(*pde != 0);
1975da673940SJordan Gordeev pmap_inval_pde(pde, pmap, sva);
1976c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count,
1977c91894e0SMatthew Dillon -NBPDR / PAGE_SIZE);
1978da673940SJordan Gordeev continue;
1979da673940SJordan Gordeev }
1980da673940SJordan Gordeev
1981da673940SJordan Gordeev /*
1982da673940SJordan Gordeev * Limit our scan to either the end of the va represented
1983da673940SJordan Gordeev * by the current page table page, or to the end of the
1984da673940SJordan Gordeev * range being removed.
1985da673940SJordan Gordeev */
1986da673940SJordan Gordeev if (va_next > eva)
1987da673940SJordan Gordeev va_next = eva;
1988da673940SJordan Gordeev
1989da673940SJordan Gordeev /*
1990da673940SJordan Gordeev * NOTE: pmap_remove_pte() can block.
1991da673940SJordan Gordeev */
1992eb36cb6bSMatthew Dillon pt_m = pmap_hold_pt_page(pde, sva);
1993da673940SJordan Gordeev for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
1994da673940SJordan Gordeev sva += PAGE_SIZE) {
1995eb36cb6bSMatthew Dillon if (*pte) {
199600eb801eSMatthew Dillon if (pmap_remove_pte(pmap, pte, 0, sva))
1997da673940SJordan Gordeev break;
1998da673940SJordan Gordeev }
1999da673940SJordan Gordeev }
2000eb36cb6bSMatthew Dillon vm_page_unhold(pt_m);
2001eb36cb6bSMatthew Dillon }
2002b12defdcSMatthew Dillon vm_object_drop(pmap->pm_pteobj);
2003da673940SJordan Gordeev }
2004da673940SJordan Gordeev
2005da673940SJordan Gordeev /*
2006da673940SJordan Gordeev * Removes this physical page from all physical maps in which it resides.
2007da673940SJordan Gordeev * Reflects back modify bits to the pager.
2008da673940SJordan Gordeev *
2009da673940SJordan Gordeev * This routine may not be called from an interrupt.
2010f7230403SMatthew Dillon *
2011f7230403SMatthew Dillon * No requirements.
2012da673940SJordan Gordeev */
2013da673940SJordan Gordeev static void
pmap_remove_all(vm_page_t m)2014da673940SJordan Gordeev pmap_remove_all(vm_page_t m)
2015da673940SJordan Gordeev {
2016da673940SJordan Gordeev pt_entry_t *pte, tpte;
2017da673940SJordan Gordeev pv_entry_t pv;
2018c91894e0SMatthew Dillon vm_object_t pmobj;
2019c91894e0SMatthew Dillon pmap_t pmap;
2020da673940SJordan Gordeev
2021da673940SJordan Gordeev #if defined(PMAP_DIAGNOSTIC)
2022da673940SJordan Gordeev /*
2023da673940SJordan Gordeev * XXX this makes pmap_page_protect(NONE) illegal for non-managed
2024da673940SJordan Gordeev * pages!
2025da673940SJordan Gordeev */
2026da673940SJordan Gordeev if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) {
2027da673940SJordan Gordeev panic("pmap_page_protect: illegal for unmanaged page, va: 0x%08llx", (long long)VM_PAGE_TO_PHYS(m));
2028da673940SJordan Gordeev }
2029da673940SJordan Gordeev #endif
2030da673940SJordan Gordeev
2031c91894e0SMatthew Dillon restart:
2032c91894e0SMatthew Dillon vm_page_spin_lock(m);
2033da673940SJordan Gordeev while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
2034c91894e0SMatthew Dillon pmap = pv->pv_pmap;
2035c91894e0SMatthew Dillon pmobj = pmap->pm_pteobj;
2036da673940SJordan Gordeev
2037c91894e0SMatthew Dillon /*
2038c91894e0SMatthew Dillon * Handle reversed lock ordering
2039c91894e0SMatthew Dillon */
2040c91894e0SMatthew Dillon if (vm_object_hold_try(pmobj) == 0) {
2041c91894e0SMatthew Dillon refcount_acquire(&pmobj->hold_count);
2042c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2043c91894e0SMatthew Dillon vm_object_lock(pmobj);
2044c91894e0SMatthew Dillon vm_page_spin_lock(m);
2045c91894e0SMatthew Dillon if (pv != TAILQ_FIRST(&m->md.pv_list) ||
2046c91894e0SMatthew Dillon pmap != pv->pv_pmap ||
2047c91894e0SMatthew Dillon pmobj != pmap->pm_pteobj) {
2048c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2049c91894e0SMatthew Dillon vm_object_drop(pmobj);
2050c91894e0SMatthew Dillon goto restart;
2051c91894e0SMatthew Dillon }
2052c91894e0SMatthew Dillon }
2053c91894e0SMatthew Dillon
2054c91894e0SMatthew Dillon KKASSERT(pmap->pm_stats.resident_count > 0);
2055c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count, -1);
2056c91894e0SMatthew Dillon
2057c91894e0SMatthew Dillon pte = pmap_pte(pmap, pv->pv_va);
2058da673940SJordan Gordeev KKASSERT(pte != NULL);
2059da673940SJordan Gordeev
2060c91894e0SMatthew Dillon tpte = pmap_inval_loadandclear(pte, pmap, pv->pv_va);
2061da673940SJordan Gordeev if (tpte & VPTE_WIRED)
2062c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, -1);
2063c91894e0SMatthew Dillon KKASSERT(pmap->pm_stats.wired_count >= 0);
2064da673940SJordan Gordeev
2065da673940SJordan Gordeev if (tpte & VPTE_A)
2066da673940SJordan Gordeev vm_page_flag_set(m, PG_REFERENCED);
2067da673940SJordan Gordeev
2068da673940SJordan Gordeev /*
2069da673940SJordan Gordeev * Update the vm_page_t clean and reference bits.
2070da673940SJordan Gordeev */
2071da673940SJordan Gordeev if (tpte & VPTE_M) {
2072da673940SJordan Gordeev #if defined(PMAP_DIAGNOSTIC)
2073da673940SJordan Gordeev if (pmap_nw_modified(tpte)) {
2074da673940SJordan Gordeev kprintf(
2075da673940SJordan Gordeev "pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
2076da673940SJordan Gordeev pv->pv_va, tpte);
2077da673940SJordan Gordeev }
2078da673940SJordan Gordeev #endif
2079ae442b2eSMatthew Dillon pmap_track_modified(pmap, pv->pv_va);
2080da673940SJordan Gordeev vm_page_dirty(m);
2081da673940SJordan Gordeev }
2082da673940SJordan Gordeev TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
2083da673940SJordan Gordeev if (TAILQ_EMPTY(&m->md.pv_list))
2084da673940SJordan Gordeev vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
2085b443039bSMatthew Dillon m->md.pv_list_count--;
2086b443039bSMatthew Dillon KKASSERT(m->md.pv_list_count >= 0);
2087b443039bSMatthew Dillon pv_entry_rb_tree_RB_REMOVE(&pmap->pm_pvroot, pv);
2088b443039bSMatthew Dillon atomic_add_int(&pmap->pm_generation, 1);
2089c78d5661SMatthew Dillon vm_page_spin_unlock(m);
2090c91894e0SMatthew Dillon pmap_unuse_pt(pmap, pv->pv_va, pv->pv_ptem);
2091da673940SJordan Gordeev free_pv_entry(pv);
2092b443039bSMatthew Dillon
2093b443039bSMatthew Dillon vm_object_drop(pmobj);
2094c78d5661SMatthew Dillon vm_page_spin_lock(m);
2095da673940SJordan Gordeev }
2096da673940SJordan Gordeev KKASSERT((m->flags & (PG_MAPPED|PG_WRITEABLE)) == 0);
2097c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2098da673940SJordan Gordeev }
2099da673940SJordan Gordeev
2100da673940SJordan Gordeev /*
2101a7a03a5fSMatthew Dillon * Removes the page from a particular pmap
2102a7a03a5fSMatthew Dillon */
2103a7a03a5fSMatthew Dillon void
pmap_remove_specific(pmap_t pmap,vm_page_t m)2104a7a03a5fSMatthew Dillon pmap_remove_specific(pmap_t pmap, vm_page_t m)
2105a7a03a5fSMatthew Dillon {
2106a7a03a5fSMatthew Dillon pt_entry_t *pte, tpte;
2107a7a03a5fSMatthew Dillon pv_entry_t pv;
2108a7a03a5fSMatthew Dillon
2109c91894e0SMatthew Dillon vm_object_hold(pmap->pm_pteobj);
2110a7a03a5fSMatthew Dillon again:
2111c91894e0SMatthew Dillon vm_page_spin_lock(m);
2112a7a03a5fSMatthew Dillon TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2113a7a03a5fSMatthew Dillon if (pv->pv_pmap != pmap)
2114a7a03a5fSMatthew Dillon continue;
2115a7a03a5fSMatthew Dillon
2116c91894e0SMatthew Dillon KKASSERT(pmap->pm_stats.resident_count > 0);
2117c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count, -1);
2118a7a03a5fSMatthew Dillon
2119c91894e0SMatthew Dillon pte = pmap_pte(pmap, pv->pv_va);
2120a7a03a5fSMatthew Dillon KKASSERT(pte != NULL);
2121a7a03a5fSMatthew Dillon
2122c91894e0SMatthew Dillon tpte = pmap_inval_loadandclear(pte, pmap, pv->pv_va);
2123a7a03a5fSMatthew Dillon if (tpte & VPTE_WIRED)
2124c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, -1);
2125c91894e0SMatthew Dillon KKASSERT(pmap->pm_stats.wired_count >= 0);
2126a7a03a5fSMatthew Dillon
2127a7a03a5fSMatthew Dillon if (tpte & VPTE_A)
2128a7a03a5fSMatthew Dillon vm_page_flag_set(m, PG_REFERENCED);
2129a7a03a5fSMatthew Dillon
2130a7a03a5fSMatthew Dillon /*
2131a7a03a5fSMatthew Dillon * Update the vm_page_t clean and reference bits.
2132a7a03a5fSMatthew Dillon */
2133a7a03a5fSMatthew Dillon if (tpte & VPTE_M) {
2134ae442b2eSMatthew Dillon pmap_track_modified(pmap, pv->pv_va);
2135a7a03a5fSMatthew Dillon vm_page_dirty(m);
2136a7a03a5fSMatthew Dillon }
2137a7a03a5fSMatthew Dillon TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
2138c50e690bSMatthew Dillon pv_entry_rb_tree_RB_REMOVE(&pmap->pm_pvroot, pv);
2139c91894e0SMatthew Dillon atomic_add_int(&pmap->pm_generation, 1);
2140a7a03a5fSMatthew Dillon m->md.pv_list_count--;
2141a7a03a5fSMatthew Dillon KKASSERT(m->md.pv_list_count >= 0);
2142a7a03a5fSMatthew Dillon if (TAILQ_EMPTY(&m->md.pv_list))
2143a7a03a5fSMatthew Dillon vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
2144c91894e0SMatthew Dillon pmap_unuse_pt(pmap, pv->pv_va, pv->pv_ptem);
2145c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2146a7a03a5fSMatthew Dillon free_pv_entry(pv);
2147a7a03a5fSMatthew Dillon goto again;
2148a7a03a5fSMatthew Dillon }
2149c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2150c91894e0SMatthew Dillon vm_object_drop(pmap->pm_pteobj);
2151a7a03a5fSMatthew Dillon }
2152a7a03a5fSMatthew Dillon
2153a7a03a5fSMatthew Dillon /*
2154da673940SJordan Gordeev * Set the physical protection on the specified range of this map
2155da673940SJordan Gordeev * as requested.
2156da673940SJordan Gordeev *
2157da673940SJordan Gordeev * This function may not be called from an interrupt if the map is
2158da673940SJordan Gordeev * not the kernel_pmap.
2159f7230403SMatthew Dillon *
2160f7230403SMatthew Dillon * No requirements.
2161da673940SJordan Gordeev */
2162da673940SJordan Gordeev void
pmap_protect(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,vm_prot_t prot)2163da673940SJordan Gordeev pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
2164da673940SJordan Gordeev {
2165da673940SJordan Gordeev vm_offset_t va_next;
2166da673940SJordan Gordeev pml4_entry_t *pml4e;
2167da673940SJordan Gordeev pdp_entry_t *pdpe;
2168da673940SJordan Gordeev pd_entry_t ptpaddr, *pde;
2169da673940SJordan Gordeev pt_entry_t *pte;
2170eb36cb6bSMatthew Dillon vm_page_t pt_m;
2171da673940SJordan Gordeev
2172da673940SJordan Gordeev if (pmap == NULL)
2173da673940SJordan Gordeev return;
2174da673940SJordan Gordeev
217573d64b98SMatthew Dillon if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == VM_PROT_NONE) {
2176da673940SJordan Gordeev pmap_remove(pmap, sva, eva);
2177da673940SJordan Gordeev return;
2178da673940SJordan Gordeev }
2179da673940SJordan Gordeev
2180da673940SJordan Gordeev if (prot & VM_PROT_WRITE)
2181da673940SJordan Gordeev return;
2182da673940SJordan Gordeev
2183c91894e0SMatthew Dillon vm_object_hold(pmap->pm_pteobj);
2184f7230403SMatthew Dillon
2185da673940SJordan Gordeev for (; sva < eva; sva = va_next) {
2186da673940SJordan Gordeev pml4e = pmap_pml4e(pmap, sva);
2187da673940SJordan Gordeev if ((*pml4e & VPTE_V) == 0) {
2188da673940SJordan Gordeev va_next = (sva + NBPML4) & ~PML4MASK;
2189da673940SJordan Gordeev if (va_next < sva)
2190da673940SJordan Gordeev va_next = eva;
2191da673940SJordan Gordeev continue;
2192da673940SJordan Gordeev }
2193da673940SJordan Gordeev
2194da673940SJordan Gordeev pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
2195da673940SJordan Gordeev if ((*pdpe & VPTE_V) == 0) {
2196da673940SJordan Gordeev va_next = (sva + NBPDP) & ~PDPMASK;
2197da673940SJordan Gordeev if (va_next < sva)
2198da673940SJordan Gordeev va_next = eva;
2199da673940SJordan Gordeev continue;
2200da673940SJordan Gordeev }
2201da673940SJordan Gordeev
2202da673940SJordan Gordeev va_next = (sva + NBPDR) & ~PDRMASK;
2203da673940SJordan Gordeev if (va_next < sva)
2204da673940SJordan Gordeev va_next = eva;
2205da673940SJordan Gordeev
2206da673940SJordan Gordeev pde = pmap_pdpe_to_pde(pdpe, sva);
2207da673940SJordan Gordeev ptpaddr = *pde;
2208da673940SJordan Gordeev
2209c78d5661SMatthew Dillon #if 0
2210da673940SJordan Gordeev /*
2211da673940SJordan Gordeev * Check for large page.
2212da673940SJordan Gordeev */
2213da673940SJordan Gordeev if ((ptpaddr & VPTE_PS) != 0) {
2214da673940SJordan Gordeev /* JG correct? */
2215da673940SJordan Gordeev pmap_clean_pde(pde, pmap, sva);
2216c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count,
2217c91894e0SMatthew Dillon -NBPDR / PAGE_SIZE);
2218da673940SJordan Gordeev continue;
2219da673940SJordan Gordeev }
2220c78d5661SMatthew Dillon #endif
2221da673940SJordan Gordeev
2222da673940SJordan Gordeev /*
2223da673940SJordan Gordeev * Weed out invalid mappings. Note: we assume that the page
2224da673940SJordan Gordeev * directory table is always allocated, and in kernel virtual.
2225da673940SJordan Gordeev */
2226da673940SJordan Gordeev if (ptpaddr == 0)
2227da673940SJordan Gordeev continue;
2228da673940SJordan Gordeev
2229da673940SJordan Gordeev if (va_next > eva)
2230da673940SJordan Gordeev va_next = eva;
2231da673940SJordan Gordeev
2232eb36cb6bSMatthew Dillon pt_m = pmap_hold_pt_page(pde, sva);
2233da673940SJordan Gordeev for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
2234da673940SJordan Gordeev sva += PAGE_SIZE) {
2235da673940SJordan Gordeev /*
2236da673940SJordan Gordeev * Clean managed pages and also check the accessed
2237da673940SJordan Gordeev * bit. Just remove write perms for unmanaged
2238da673940SJordan Gordeev * pages. Be careful of races, turning off write
2239da673940SJordan Gordeev * access will force a fault rather then setting
2240da673940SJordan Gordeev * the modified bit at an unexpected time.
2241da673940SJordan Gordeev */
2242ae442b2eSMatthew Dillon pmap_track_modified(pmap, sva);
224395270b7eSMatthew Dillon pmap_clean_pte(pte, pmap, sva, NULL);
2244da673940SJordan Gordeev }
2245eb36cb6bSMatthew Dillon vm_page_unhold(pt_m);
2246da673940SJordan Gordeev }
2247c91894e0SMatthew Dillon vm_object_drop(pmap->pm_pteobj);
2248da673940SJordan Gordeev }
2249da673940SJordan Gordeev
2250da673940SJordan Gordeev /*
2251da673940SJordan Gordeev * Enter a managed page into a pmap. If the page is not wired related pmap
2252da673940SJordan Gordeev * data can be destroyed at any time for later demand-operation.
2253da673940SJordan Gordeev *
2254da673940SJordan Gordeev * Insert the vm_page (m) at virtual address (v) in (pmap), with the
2255da673940SJordan Gordeev * specified protection, and wire the mapping if requested.
2256da673940SJordan Gordeev *
2257da673940SJordan Gordeev * NOTE: This routine may not lazy-evaluate or lose information. The
2258da673940SJordan Gordeev * page must actually be inserted into the given map NOW.
2259da673940SJordan Gordeev *
2260da673940SJordan Gordeev * NOTE: When entering a page at a KVA address, the pmap must be the
2261da673940SJordan Gordeev * kernel_pmap.
2262f7230403SMatthew Dillon *
2263f7230403SMatthew Dillon * No requirements.
2264da673940SJordan Gordeev */
2265da673940SJordan Gordeev void
pmap_enter(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,boolean_t wired,vm_map_entry_t entry __unused)2266da673940SJordan Gordeev pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
2267921c891eSMatthew Dillon boolean_t wired, vm_map_entry_t entry __unused)
2268da673940SJordan Gordeev {
2269da673940SJordan Gordeev vm_paddr_t pa;
2270c78d5661SMatthew Dillon pv_entry_t pv;
2271da673940SJordan Gordeev pt_entry_t *pte;
2272da673940SJordan Gordeev pt_entry_t origpte, newpte;
2273c78d5661SMatthew Dillon vm_paddr_t opa;
2274da673940SJordan Gordeev vm_page_t mpte;
2275da673940SJordan Gordeev
2276da673940SJordan Gordeev if (pmap == NULL)
2277da673940SJordan Gordeev return;
2278da673940SJordan Gordeev
2279da673940SJordan Gordeev va = trunc_page(va);
2280da673940SJordan Gordeev
2281b12defdcSMatthew Dillon vm_object_hold(pmap->pm_pteobj);
2282f7230403SMatthew Dillon
2283da673940SJordan Gordeev /*
2284da673940SJordan Gordeev * Get the page table page. The kernel_pmap's page table pages
2285da673940SJordan Gordeev * are preallocated and have no associated vm_page_t.
2286c78d5661SMatthew Dillon *
2287c78d5661SMatthew Dillon * If not NULL, mpte will be busied and we must vm_page_wakeup()
2288c78d5661SMatthew Dillon * to cleanup. There will already be at least one wire count from
2289c78d5661SMatthew Dillon * it being mapped into its parent.
2290da673940SJordan Gordeev */
2291*c713db65SAaron LI if (pmap == kernel_pmap) {
2292da673940SJordan Gordeev mpte = NULL;
2293c78d5661SMatthew Dillon pte = vtopte(va);
2294f7230403SMatthew Dillon } else {
2295c78d5661SMatthew Dillon mpte = pmap_allocpte(pmap, va);
2296c78d5661SMatthew Dillon pte = (void *)PHYS_TO_DMAP(mpte->phys_addr);
2297c78d5661SMatthew Dillon pte += pmap_pte_index(va);
2298f7230403SMatthew Dillon }
2299da673940SJordan Gordeev
2300da673940SJordan Gordeev /*
230100eb801eSMatthew Dillon * Deal with races against the kernel's real MMU by cleaning the
230200eb801eSMatthew Dillon * page, even if we are re-entering the same page.
2303da673940SJordan Gordeev */
2304da673940SJordan Gordeev pa = VM_PAGE_TO_PHYS(m);
230500eb801eSMatthew Dillon origpte = pmap_inval_loadandclear(pte, pmap, va);
230695270b7eSMatthew Dillon /*origpte = pmap_clean_pte(pte, pmap, va, NULL);*/
2307da673940SJordan Gordeev opa = origpte & VPTE_FRAME;
2308da673940SJordan Gordeev
2309da673940SJordan Gordeev if (origpte & VPTE_PS)
2310da673940SJordan Gordeev panic("pmap_enter: attempted pmap_enter on 2MB page");
2311da673940SJordan Gordeev
2312eb36cb6bSMatthew Dillon if ((origpte & (VPTE_MANAGED|VPTE_M)) == (VPTE_MANAGED|VPTE_M)) {
2313ae442b2eSMatthew Dillon vm_page_t om;
2314ae442b2eSMatthew Dillon
2315ae442b2eSMatthew Dillon pmap_track_modified(pmap, va);
2316ae442b2eSMatthew Dillon om = PHYS_TO_VM_PAGE(opa);
2317eb36cb6bSMatthew Dillon vm_page_dirty(om);
2318eb36cb6bSMatthew Dillon }
2319eb36cb6bSMatthew Dillon
2320da673940SJordan Gordeev /*
2321da673940SJordan Gordeev * Mapping has not changed, must be protection or wiring change.
2322da673940SJordan Gordeev */
2323da673940SJordan Gordeev if (origpte && (opa == pa)) {
2324da673940SJordan Gordeev /*
2325da673940SJordan Gordeev * Wiring change, just update stats. We don't worry about
2326da673940SJordan Gordeev * wiring PT pages as they remain resident as long as there
2327da673940SJordan Gordeev * are valid mappings in them. Hence, if a user page is wired,
2328da673940SJordan Gordeev * the PT page will be also.
2329da673940SJordan Gordeev */
2330da673940SJordan Gordeev if (wired && ((origpte & VPTE_WIRED) == 0))
2331c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, 1);
2332da673940SJordan Gordeev else if (!wired && (origpte & VPTE_WIRED))
2333c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, -1);
2334da673940SJordan Gordeev
2335da673940SJordan Gordeev if (origpte & VPTE_MANAGED) {
2336da673940SJordan Gordeev pa |= VPTE_MANAGED;
2337da673940SJordan Gordeev KKASSERT(m->flags & PG_MAPPED);
2338a70dbf04SMatthew Dillon KKASSERT((m->flags & PG_FICTITIOUS) == 0);
23395ee06c6cSMatthew Dillon } else {
2340a70dbf04SMatthew Dillon KKASSERT((m->flags & PG_FICTITIOUS));
2341da673940SJordan Gordeev }
234295270b7eSMatthew Dillon vm_page_spin_lock(m);
2343da673940SJordan Gordeev goto validate;
2344da673940SJordan Gordeev }
2345c78d5661SMatthew Dillon
2346c78d5661SMatthew Dillon /*
2347c78d5661SMatthew Dillon * Bump the wire_count for the page table page.
2348c78d5661SMatthew Dillon */
2349c78d5661SMatthew Dillon if (mpte)
2350c78d5661SMatthew Dillon vm_page_wire_quick(mpte);
2351c78d5661SMatthew Dillon
2352da673940SJordan Gordeev /*
2353da673940SJordan Gordeev * Mapping has changed, invalidate old range and fall through to
235495270b7eSMatthew Dillon * handle validating new mapping. Don't inherit anything from
235595270b7eSMatthew Dillon * oldpte.
2356da673940SJordan Gordeev */
2357da673940SJordan Gordeev if (opa) {
2358da673940SJordan Gordeev int err;
235900eb801eSMatthew Dillon err = pmap_remove_pte(pmap, NULL, origpte, va);
236095270b7eSMatthew Dillon origpte = 0;
2361da673940SJordan Gordeev if (err)
2362da673940SJordan Gordeev panic("pmap_enter: pte vanished, va: 0x%lx", va);
2363da673940SJordan Gordeev }
2364da673940SJordan Gordeev
2365da673940SJordan Gordeev /*
2366da673940SJordan Gordeev * Enter on the PV list if part of our managed memory. Note that we
2367da673940SJordan Gordeev * raise IPL while manipulating pv_table since pmap_enter can be
2368da673940SJordan Gordeev * called at interrupt time.
2369da673940SJordan Gordeev */
2370c78d5661SMatthew Dillon if (pmap_initialized) {
2371a70dbf04SMatthew Dillon if ((m->flags & PG_FICTITIOUS) == 0) {
2372b443039bSMatthew Dillon /*
2373b443039bSMatthew Dillon * WARNING! We are using m's spin-lock as a
2374b443039bSMatthew Dillon * man's pte lock to interlock against
2375b443039bSMatthew Dillon * pmap_page_protect() operations.
2376b443039bSMatthew Dillon *
2377b443039bSMatthew Dillon * This is a bad hack (obviously).
2378b443039bSMatthew Dillon */
2379c78d5661SMatthew Dillon pv = get_pv_entry();
2380c91894e0SMatthew Dillon vm_page_spin_lock(m);
2381c78d5661SMatthew Dillon pmap_insert_entry(pmap, va, mpte, m, pv);
2382da673940SJordan Gordeev pa |= VPTE_MANAGED;
2383b443039bSMatthew Dillon /* vm_page_spin_unlock(m); */
238495270b7eSMatthew Dillon } else {
238595270b7eSMatthew Dillon vm_page_spin_lock(m);
2386c78d5661SMatthew Dillon }
238795270b7eSMatthew Dillon } else {
238895270b7eSMatthew Dillon vm_page_spin_lock(m);
2389c78d5661SMatthew Dillon }
2390da673940SJordan Gordeev
2391da673940SJordan Gordeev /*
2392da673940SJordan Gordeev * Increment counters
2393da673940SJordan Gordeev */
2394c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count, 1);
2395da673940SJordan Gordeev if (wired)
2396c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, 1);
2397da673940SJordan Gordeev
2398da673940SJordan Gordeev validate:
2399da673940SJordan Gordeev /*
2400da673940SJordan Gordeev * Now validate mapping with desired protection/wiring.
2401da673940SJordan Gordeev */
2402a86ce0cdSMatthew Dillon newpte = (pt_entry_t)(pa | pte_prot(pmap, prot) | VPTE_V | VPTE_U);
2403c50e690bSMatthew Dillon newpte |= VPTE_A;
2404da673940SJordan Gordeev
2405da673940SJordan Gordeev if (wired)
2406da673940SJordan Gordeev newpte |= VPTE_WIRED;
2407*c713db65SAaron LI // if (pmap != kernel_pmap)
2408da673940SJordan Gordeev newpte |= VPTE_U;
2409a86ce0cdSMatthew Dillon if (newpte & VPTE_RW)
2410da673940SJordan Gordeev vm_page_flag_set(m, PG_WRITEABLE);
2411da673940SJordan Gordeev KKASSERT((newpte & VPTE_MANAGED) == 0 || (m->flags & PG_MAPPED));
2412c78d5661SMatthew Dillon
241300eb801eSMatthew Dillon origpte = atomic_swap_long(pte, newpte);
241400eb801eSMatthew Dillon if (origpte & VPTE_M) {
241500eb801eSMatthew Dillon kprintf("pmap [M] race @ %016jx\n", va);
241600eb801eSMatthew Dillon atomic_set_long(pte, VPTE_M);
241700eb801eSMatthew Dillon }
2418b443039bSMatthew Dillon vm_page_spin_unlock(m);
2419c50e690bSMatthew Dillon
2420c78d5661SMatthew Dillon if (mpte)
2421c78d5661SMatthew Dillon vm_page_wakeup(mpte);
2422b12defdcSMatthew Dillon vm_object_drop(pmap->pm_pteobj);
2423da673940SJordan Gordeev }
2424da673940SJordan Gordeev
2425da673940SJordan Gordeev /*
2426da673940SJordan Gordeev * Make a temporary mapping for a physical address. This is only intended
2427da673940SJordan Gordeev * to be used for panic dumps.
2428fb8345e6SMatthew Dillon *
2429fb8345e6SMatthew Dillon * The caller is responsible for calling smp_invltlb().
2430da673940SJordan Gordeev */
2431da673940SJordan Gordeev void *
pmap_kenter_temporary(vm_paddr_t pa,long i)24328e5ea5f7SMatthew Dillon pmap_kenter_temporary(vm_paddr_t pa, long i)
2433da673940SJordan Gordeev {
2434fb8345e6SMatthew Dillon pmap_kenter_quick(crashdumpmap + (i * PAGE_SIZE), pa);
2435da673940SJordan Gordeev return ((void *)crashdumpmap);
2436da673940SJordan Gordeev }
2437da673940SJordan Gordeev
2438da673940SJordan Gordeev #define MAX_INIT_PT (96)
2439da673940SJordan Gordeev
2440da673940SJordan Gordeev /*
2441da673940SJordan Gordeev * This routine preloads the ptes for a given object into the specified pmap.
2442da673940SJordan Gordeev * This eliminates the blast of soft faults on process startup and
2443da673940SJordan Gordeev * immediately after an mmap.
2444f7230403SMatthew Dillon *
2445f7230403SMatthew Dillon * No requirements.
2446da673940SJordan Gordeev */
2447da673940SJordan Gordeev static int pmap_object_init_pt_callback(vm_page_t p, void *data);
2448da673940SJordan Gordeev
2449da673940SJordan Gordeev void
pmap_object_init_pt(pmap_t pmap,vm_map_entry_t entry,vm_offset_t addr,vm_size_t size,int limit)2450530e94fcSMatthew Dillon pmap_object_init_pt(pmap_t pmap, vm_map_entry_t entry,
2451530e94fcSMatthew Dillon vm_offset_t addr, vm_size_t size, int limit)
2452da673940SJordan Gordeev {
2453530e94fcSMatthew Dillon vm_prot_t prot = entry->protection;
2454530e94fcSMatthew Dillon vm_object_t object = entry->ba.object;
2455530e94fcSMatthew Dillon vm_pindex_t pindex = atop(entry->ba.offset + (addr - entry->ba.start));
2456da673940SJordan Gordeev struct rb_vm_page_scan_info info;
2457da673940SJordan Gordeev struct lwp *lp;
2458da673940SJordan Gordeev vm_size_t psize;
2459da673940SJordan Gordeev
2460da673940SJordan Gordeev /*
2461da673940SJordan Gordeev * We can't preinit if read access isn't set or there is no pmap
2462da673940SJordan Gordeev * or object.
2463da673940SJordan Gordeev */
2464da673940SJordan Gordeev if ((prot & VM_PROT_READ) == 0 || pmap == NULL || object == NULL)
2465da673940SJordan Gordeev return;
2466da673940SJordan Gordeev
2467da673940SJordan Gordeev /*
2468da673940SJordan Gordeev * We can't preinit if the pmap is not the current pmap
2469da673940SJordan Gordeev */
2470da673940SJordan Gordeev lp = curthread->td_lwp;
2471da673940SJordan Gordeev if (lp == NULL || pmap != vmspace_pmap(lp->lwp_vmspace))
2472da673940SJordan Gordeev return;
2473da673940SJordan Gordeev
247400eb801eSMatthew Dillon /*
247500eb801eSMatthew Dillon * Misc additional checks
247600eb801eSMatthew Dillon */
24770e6594a8SSascha Wildner psize = x86_64_btop(size);
2478da673940SJordan Gordeev
2479da673940SJordan Gordeev if ((object->type != OBJT_VNODE) ||
2480da673940SJordan Gordeev ((limit & MAP_PREFAULT_PARTIAL) && (psize > MAX_INIT_PT) &&
2481da673940SJordan Gordeev (object->resident_page_count > MAX_INIT_PT))) {
2482da673940SJordan Gordeev return;
2483da673940SJordan Gordeev }
2484da673940SJordan Gordeev
2485da673940SJordan Gordeev if (psize + pindex > object->size) {
2486da673940SJordan Gordeev if (object->size < pindex)
2487da673940SJordan Gordeev return;
2488da673940SJordan Gordeev psize = object->size - pindex;
2489da673940SJordan Gordeev }
2490da673940SJordan Gordeev
2491da673940SJordan Gordeev if (psize == 0)
2492da673940SJordan Gordeev return;
2493da673940SJordan Gordeev
2494da673940SJordan Gordeev /*
2495da673940SJordan Gordeev * Use a red-black scan to traverse the requested range and load
2496da673940SJordan Gordeev * any valid pages found into the pmap.
2497da673940SJordan Gordeev *
2498da673940SJordan Gordeev * We cannot safely scan the object's memq unless we are in a
2499da673940SJordan Gordeev * critical section since interrupts can remove pages from objects.
2500da673940SJordan Gordeev */
2501da673940SJordan Gordeev info.start_pindex = pindex;
2502da673940SJordan Gordeev info.end_pindex = pindex + psize - 1;
2503da673940SJordan Gordeev info.limit = limit;
2504da673940SJordan Gordeev info.mpte = NULL;
2505da673940SJordan Gordeev info.addr = addr;
2506da673940SJordan Gordeev info.pmap = pmap;
2507530e94fcSMatthew Dillon info.entry = entry;
2508da673940SJordan Gordeev
2509b3371fc1SMatthew Dillon vm_object_hold_shared(object);
2510da673940SJordan Gordeev vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
2511da673940SJordan Gordeev pmap_object_init_pt_callback, &info);
2512b12defdcSMatthew Dillon vm_object_drop(object);
2513da673940SJordan Gordeev }
2514da673940SJordan Gordeev
2515da673940SJordan Gordeev static
2516da673940SJordan Gordeev int
pmap_object_init_pt_callback(vm_page_t p,void * data)2517da673940SJordan Gordeev pmap_object_init_pt_callback(vm_page_t p, void *data)
2518da673940SJordan Gordeev {
2519da673940SJordan Gordeev struct rb_vm_page_scan_info *info = data;
2520da673940SJordan Gordeev vm_pindex_t rel_index;
2521da673940SJordan Gordeev /*
2522da673940SJordan Gordeev * don't allow an madvise to blow away our really
2523da673940SJordan Gordeev * free pages allocating pv entries.
2524da673940SJordan Gordeev */
2525da673940SJordan Gordeev if ((info->limit & MAP_PREFAULT_MADVISE) &&
2526da673940SJordan Gordeev vmstats.v_free_count < vmstats.v_free_reserved) {
2527da673940SJordan Gordeev return(-1);
2528da673940SJordan Gordeev }
25290d987a03SMatthew Dillon
25300d987a03SMatthew Dillon /*
25310d987a03SMatthew Dillon * Ignore list markers and ignore pages we cannot instantly
25320d987a03SMatthew Dillon * busy (while holding the object token).
25330d987a03SMatthew Dillon */
25340d987a03SMatthew Dillon if (p->flags & PG_MARKER)
25350d987a03SMatthew Dillon return 0;
2536b12defdcSMatthew Dillon if (vm_page_busy_try(p, TRUE))
2537b12defdcSMatthew Dillon return 0;
2538da673940SJordan Gordeev if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2539b12defdcSMatthew Dillon (p->flags & PG_FICTITIOUS) == 0) {
2540da673940SJordan Gordeev if ((p->queue - p->pc) == PQ_CACHE)
2541da673940SJordan Gordeev vm_page_deactivate(p);
2542da673940SJordan Gordeev rel_index = p->pindex - info->start_pindex;
2543530e94fcSMatthew Dillon pmap_enter(info->pmap, info->addr + x86_64_ptob(rel_index), p,
2544530e94fcSMatthew Dillon VM_PROT_READ, FALSE, info->entry);
2545da673940SJordan Gordeev }
2546b12defdcSMatthew Dillon vm_page_wakeup(p);
2547da673940SJordan Gordeev return(0);
2548da673940SJordan Gordeev }
2549da673940SJordan Gordeev
2550da673940SJordan Gordeev /*
25510e6594a8SSascha Wildner * Return TRUE if the pmap is in shape to trivially
25520e6594a8SSascha Wildner * pre-fault the specified address.
25530e6594a8SSascha Wildner *
25540e6594a8SSascha Wildner * Returns FALSE if it would be non-trivial or if a
25550e6594a8SSascha Wildner * pte is already loaded into the slot.
2556f7230403SMatthew Dillon *
2557f7230403SMatthew Dillon * No requirements.
2558da673940SJordan Gordeev */
25590e6594a8SSascha Wildner int
pmap_prefault_ok(pmap_t pmap,vm_offset_t addr)25600e6594a8SSascha Wildner pmap_prefault_ok(pmap_t pmap, vm_offset_t addr)
2561da673940SJordan Gordeev {
2562da673940SJordan Gordeev pt_entry_t *pte;
2563da673940SJordan Gordeev pd_entry_t *pde;
2564f7230403SMatthew Dillon int ret;
2565da673940SJordan Gordeev
2566c91894e0SMatthew Dillon vm_object_hold(pmap->pm_pteobj);
2567da673940SJordan Gordeev pde = pmap_pde(pmap, addr);
2568f7230403SMatthew Dillon if (pde == NULL || *pde == 0) {
2569f7230403SMatthew Dillon ret = 0;
2570f7230403SMatthew Dillon } else {
2571da673940SJordan Gordeev pte = pmap_pde_to_pte(pde, addr);
2572f7230403SMatthew Dillon ret = (*pte) ? 0 : 1;
2573f7230403SMatthew Dillon }
2574c91894e0SMatthew Dillon vm_object_drop(pmap->pm_pteobj);
2575c91894e0SMatthew Dillon
2576f7230403SMatthew Dillon return (ret);
2577da673940SJordan Gordeev }
2578da673940SJordan Gordeev
2579da673940SJordan Gordeev /*
2580f7230403SMatthew Dillon * Change the wiring attribute for a map/virtual-address pair.
2581f7230403SMatthew Dillon *
2582da673940SJordan Gordeev * The mapping must already exist in the pmap.
2583f7230403SMatthew Dillon * No other requirements.
2584da673940SJordan Gordeev */
258576f1911eSMatthew Dillon vm_page_t
pmap_unwire(pmap_t pmap,vm_offset_t va)258676f1911eSMatthew Dillon pmap_unwire(pmap_t pmap, vm_offset_t va)
2587da673940SJordan Gordeev {
2588da673940SJordan Gordeev pt_entry_t *pte;
258976f1911eSMatthew Dillon vm_paddr_t pa;
259076f1911eSMatthew Dillon vm_page_t m;
2591da673940SJordan Gordeev
2592da673940SJordan Gordeev if (pmap == NULL)
259376f1911eSMatthew Dillon return NULL;
2594da673940SJordan Gordeev
2595c91894e0SMatthew Dillon vm_object_hold(pmap->pm_pteobj);
2596da673940SJordan Gordeev pte = pmap_pte(pmap, va);
2597da673940SJordan Gordeev
259800eb801eSMatthew Dillon if (pte == NULL || (*pte & VPTE_V) == 0) {
259900eb801eSMatthew Dillon vm_object_drop(pmap->pm_pteobj);
260076f1911eSMatthew Dillon return NULL;
260100eb801eSMatthew Dillon }
2602da673940SJordan Gordeev
2603da673940SJordan Gordeev /*
2604da673940SJordan Gordeev * Wiring is not a hardware characteristic so there is no need to
2605da673940SJordan Gordeev * invalidate TLB. However, in an SMP environment we must use
2606da673940SJordan Gordeev * a locked bus cycle to update the pte (if we are not using
2607da673940SJordan Gordeev * the pmap_inval_*() API that is)... it's ok to do this for simple
2608da673940SJordan Gordeev * wiring changes.
2609da673940SJordan Gordeev */
261076f1911eSMatthew Dillon if (pmap_pte_w(pte))
261176f1911eSMatthew Dillon atomic_add_long(&pmap->pm_stats.wired_count, -1);
261276f1911eSMatthew Dillon /* XXX else return NULL so caller doesn't unwire m ? */
2613da673940SJordan Gordeev atomic_clear_long(pte, VPTE_WIRED);
261476f1911eSMatthew Dillon
261576f1911eSMatthew Dillon pa = *pte & VPTE_FRAME;
261676f1911eSMatthew Dillon m = PHYS_TO_VM_PAGE(pa); /* held by wired count */
261776f1911eSMatthew Dillon
2618c91894e0SMatthew Dillon vm_object_drop(pmap->pm_pteobj);
261976f1911eSMatthew Dillon
262076f1911eSMatthew Dillon return m;
2621da673940SJordan Gordeev }
2622da673940SJordan Gordeev
2623da673940SJordan Gordeev /*
2624da673940SJordan Gordeev * Copy the range specified by src_addr/len
2625da673940SJordan Gordeev * from the source map to the range dst_addr/len
2626da673940SJordan Gordeev * in the destination map.
2627da673940SJordan Gordeev *
2628da673940SJordan Gordeev * This routine is only advisory and need not do anything.
2629da673940SJordan Gordeev */
2630da673940SJordan Gordeev void
pmap_copy(pmap_t dst_pmap,pmap_t src_pmap,vm_offset_t dst_addr,vm_size_t len,vm_offset_t src_addr)2631da673940SJordan Gordeev pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
2632da673940SJordan Gordeev vm_size_t len, vm_offset_t src_addr)
2633da673940SJordan Gordeev {
2634da673940SJordan Gordeev /*
2635da673940SJordan Gordeev * XXX BUGGY. Amoung other things srcmpte is assumed to remain
2636da673940SJordan Gordeev * valid through blocking calls, and that's just not going to
2637da673940SJordan Gordeev * be the case.
2638da673940SJordan Gordeev *
2639da673940SJordan Gordeev * FIXME!
2640da673940SJordan Gordeev */
2641da673940SJordan Gordeev return;
2642da673940SJordan Gordeev }
2643da673940SJordan Gordeev
2644da673940SJordan Gordeev /*
2645da673940SJordan Gordeev * pmap_zero_page:
2646da673940SJordan Gordeev *
2647da673940SJordan Gordeev * Zero the specified physical page.
2648da673940SJordan Gordeev *
2649da673940SJordan Gordeev * This function may be called from an interrupt and no locking is
2650da673940SJordan Gordeev * required.
2651da673940SJordan Gordeev */
2652da673940SJordan Gordeev void
pmap_zero_page(vm_paddr_t phys)2653da673940SJordan Gordeev pmap_zero_page(vm_paddr_t phys)
2654da673940SJordan Gordeev {
2655da673940SJordan Gordeev vm_offset_t va = PHYS_TO_DMAP(phys);
2656da673940SJordan Gordeev
2657da673940SJordan Gordeev bzero((void *)va, PAGE_SIZE);
2658da673940SJordan Gordeev }
2659da673940SJordan Gordeev
2660da673940SJordan Gordeev /*
2661da673940SJordan Gordeev * pmap_zero_page:
2662da673940SJordan Gordeev *
2663da673940SJordan Gordeev * Zero part of a physical page by mapping it into memory and clearing
2664da673940SJordan Gordeev * its contents with bzero.
2665da673940SJordan Gordeev *
2666da673940SJordan Gordeev * off and size may not cover an area beyond a single hardware page.
2667da673940SJordan Gordeev */
2668da673940SJordan Gordeev void
pmap_zero_page_area(vm_paddr_t phys,int off,int size)2669da673940SJordan Gordeev pmap_zero_page_area(vm_paddr_t phys, int off, int size)
2670da673940SJordan Gordeev {
2671da673940SJordan Gordeev vm_offset_t virt = PHYS_TO_DMAP(phys);
2672c78d5661SMatthew Dillon
2673da673940SJordan Gordeev bzero((char *)virt + off, size);
2674da673940SJordan Gordeev }
2675da673940SJordan Gordeev
2676da673940SJordan Gordeev /*
2677da673940SJordan Gordeev * pmap_copy_page:
2678da673940SJordan Gordeev *
2679da673940SJordan Gordeev * Copy the physical page from the source PA to the target PA.
2680da673940SJordan Gordeev * This function may be called from an interrupt. No locking
2681da673940SJordan Gordeev * is required.
2682da673940SJordan Gordeev */
2683da673940SJordan Gordeev void
pmap_copy_page(vm_paddr_t src,vm_paddr_t dst)2684da673940SJordan Gordeev pmap_copy_page(vm_paddr_t src, vm_paddr_t dst)
2685da673940SJordan Gordeev {
2686da673940SJordan Gordeev vm_offset_t src_virt, dst_virt;
2687da673940SJordan Gordeev
2688da673940SJordan Gordeev src_virt = PHYS_TO_DMAP(src);
2689da673940SJordan Gordeev dst_virt = PHYS_TO_DMAP(dst);
26902c2e847cSSascha Wildner bcopy((void *)src_virt, (void *)dst_virt, PAGE_SIZE);
2691da673940SJordan Gordeev }
2692da673940SJordan Gordeev
2693da673940SJordan Gordeev /*
2694da673940SJordan Gordeev * pmap_copy_page_frag:
2695da673940SJordan Gordeev *
2696da673940SJordan Gordeev * Copy the physical page from the source PA to the target PA.
2697da673940SJordan Gordeev * This function may be called from an interrupt. No locking
2698da673940SJordan Gordeev * is required.
2699da673940SJordan Gordeev */
2700da673940SJordan Gordeev void
pmap_copy_page_frag(vm_paddr_t src,vm_paddr_t dst,size_t bytes)2701da673940SJordan Gordeev pmap_copy_page_frag(vm_paddr_t src, vm_paddr_t dst, size_t bytes)
2702da673940SJordan Gordeev {
2703da673940SJordan Gordeev vm_offset_t src_virt, dst_virt;
2704da673940SJordan Gordeev
2705da673940SJordan Gordeev src_virt = PHYS_TO_DMAP(src);
2706da673940SJordan Gordeev dst_virt = PHYS_TO_DMAP(dst);
2707da673940SJordan Gordeev bcopy((char *)src_virt + (src & PAGE_MASK),
2708da673940SJordan Gordeev (char *)dst_virt + (dst & PAGE_MASK),
2709da673940SJordan Gordeev bytes);
2710da673940SJordan Gordeev }
2711da673940SJordan Gordeev
2712da673940SJordan Gordeev /*
2713f7230403SMatthew Dillon * Remove all pages from specified address space this aids process
2714f7230403SMatthew Dillon * exit speeds. Also, this code is special cased for current
2715f7230403SMatthew Dillon * process only, but can have the more generic (and slightly slower)
2716f7230403SMatthew Dillon * mode enabled. This is much faster than pmap_remove in the case
2717f7230403SMatthew Dillon * of running down an entire address space.
2718f7230403SMatthew Dillon *
2719f7230403SMatthew Dillon * No other requirements.
2720da673940SJordan Gordeev */
2721da673940SJordan Gordeev void
pmap_remove_pages(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)2722da673940SJordan Gordeev pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
2723da673940SJordan Gordeev {
2724c50e690bSMatthew Dillon pmap_remove(pmap, sva, eva);
2725c50e690bSMatthew Dillon #if 0
2726da673940SJordan Gordeev pt_entry_t *pte, tpte;
2727da673940SJordan Gordeev pv_entry_t pv, npv;
2728da673940SJordan Gordeev vm_page_t m;
2729da673940SJordan Gordeev int save_generation;
2730da673940SJordan Gordeev
2731b12defdcSMatthew Dillon if (pmap->pm_pteobj)
2732b12defdcSMatthew Dillon vm_object_hold(pmap->pm_pteobj);
2733b12defdcSMatthew Dillon
2734c50e690bSMatthew Dillon pmap_invalidate_range(pmap, sva, eva);
2735c50e690bSMatthew Dillon
2736da673940SJordan Gordeev for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
2737da673940SJordan Gordeev if (pv->pv_va >= eva || pv->pv_va < sva) {
2738da673940SJordan Gordeev npv = TAILQ_NEXT(pv, pv_plist);
2739da673940SJordan Gordeev continue;
2740da673940SJordan Gordeev }
2741da673940SJordan Gordeev
2742da673940SJordan Gordeev KKASSERT(pmap == pv->pv_pmap);
2743da673940SJordan Gordeev
2744da673940SJordan Gordeev pte = pmap_pte(pmap, pv->pv_va);
2745da673940SJordan Gordeev
2746da673940SJordan Gordeev /*
2747da673940SJordan Gordeev * We cannot remove wired pages from a process' mapping
2748da673940SJordan Gordeev * at this time
2749da673940SJordan Gordeev */
2750da673940SJordan Gordeev if (*pte & VPTE_WIRED) {
2751da673940SJordan Gordeev npv = TAILQ_NEXT(pv, pv_plist);
2752da673940SJordan Gordeev continue;
2753da673940SJordan Gordeev }
2754da673940SJordan Gordeev tpte = pmap_inval_loadandclear(pte, pmap, pv->pv_va);
2755da673940SJordan Gordeev
2756da673940SJordan Gordeev m = PHYS_TO_VM_PAGE(tpte & VPTE_FRAME);
2757c91894e0SMatthew Dillon vm_page_spin_lock(m);
2758da673940SJordan Gordeev
2759da673940SJordan Gordeev KASSERT(m < &vm_page_array[vm_page_array_size],
2760da673940SJordan Gordeev ("pmap_remove_pages: bad tpte %lx", tpte));
2761da673940SJordan Gordeev
2762da673940SJordan Gordeev KKASSERT(pmap->pm_stats.resident_count > 0);
2763c91894e0SMatthew Dillon atomic_add_long(&pmap->pm_stats.resident_count, -1);
2764da673940SJordan Gordeev
2765da673940SJordan Gordeev /*
2766da673940SJordan Gordeev * Update the vm_page_t clean and reference bits.
2767da673940SJordan Gordeev */
2768da673940SJordan Gordeev if (tpte & VPTE_M) {
2769da673940SJordan Gordeev vm_page_dirty(m);
2770da673940SJordan Gordeev }
2771da673940SJordan Gordeev
2772da673940SJordan Gordeev npv = TAILQ_NEXT(pv, pv_plist);
2773da673940SJordan Gordeev TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
2774c91894e0SMatthew Dillon atomic_add_int(&pmap->pm_generation, 1);
2775c91894e0SMatthew Dillon save_generation = pmap->pm_generation;
2776da673940SJordan Gordeev m->md.pv_list_count--;
2777da673940SJordan Gordeev TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
2778da673940SJordan Gordeev if (TAILQ_EMPTY(&m->md.pv_list))
2779da673940SJordan Gordeev vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE);
2780c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2781da673940SJordan Gordeev
2782da673940SJordan Gordeev pmap_unuse_pt(pmap, pv->pv_va, pv->pv_ptem);
2783da673940SJordan Gordeev free_pv_entry(pv);
2784da673940SJordan Gordeev
2785da673940SJordan Gordeev /*
2786da673940SJordan Gordeev * Restart the scan if we blocked during the unuse or free
2787da673940SJordan Gordeev * calls and other removals were made.
2788da673940SJordan Gordeev */
2789da673940SJordan Gordeev if (save_generation != pmap->pm_generation) {
2790da673940SJordan Gordeev kprintf("Warning: pmap_remove_pages race-A avoided\n");
2791cd2a0876SSascha Wildner npv = TAILQ_FIRST(&pmap->pm_pvlist);
2792da673940SJordan Gordeev }
2793da673940SJordan Gordeev }
2794b12defdcSMatthew Dillon if (pmap->pm_pteobj)
2795b12defdcSMatthew Dillon vm_object_drop(pmap->pm_pteobj);
2796c50e690bSMatthew Dillon pmap_remove(pmap, sva, eva);
2797c50e690bSMatthew Dillon #endif
2798da673940SJordan Gordeev }
2799da673940SJordan Gordeev
2800da673940SJordan Gordeev /*
2801da673940SJordan Gordeev * pmap_testbit tests bits in active mappings of a VM page.
2802da673940SJordan Gordeev */
2803da673940SJordan Gordeev static boolean_t
pmap_testbit(vm_page_t m,int bit)2804da673940SJordan Gordeev pmap_testbit(vm_page_t m, int bit)
2805da673940SJordan Gordeev {
2806da673940SJordan Gordeev pv_entry_t pv;
2807da673940SJordan Gordeev pt_entry_t *pte;
2808da673940SJordan Gordeev
2809da673940SJordan Gordeev if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
2810da673940SJordan Gordeev return FALSE;
2811da673940SJordan Gordeev
2812da673940SJordan Gordeev if (TAILQ_FIRST(&m->md.pv_list) == NULL)
2813da673940SJordan Gordeev return FALSE;
2814da673940SJordan Gordeev
2815c91894e0SMatthew Dillon vm_page_spin_lock(m);
2816da673940SJordan Gordeev TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2817da673940SJordan Gordeev /*
2818da673940SJordan Gordeev * if the bit being tested is the modified bit, then
2819da673940SJordan Gordeev * mark clean_map and ptes as never
2820da673940SJordan Gordeev * modified.
2821da673940SJordan Gordeev */
2822ae442b2eSMatthew Dillon if (bit & (VPTE_A|VPTE_M))
2823ae442b2eSMatthew Dillon pmap_track_modified(pv->pv_pmap, pv->pv_va);
2824da673940SJordan Gordeev
2825da673940SJordan Gordeev #if defined(PMAP_DIAGNOSTIC)
2826da673940SJordan Gordeev if (pv->pv_pmap == NULL) {
2827da673940SJordan Gordeev kprintf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va);
2828da673940SJordan Gordeev continue;
2829da673940SJordan Gordeev }
2830da673940SJordan Gordeev #endif
2831da673940SJordan Gordeev pte = pmap_pte(pv->pv_pmap, pv->pv_va);
2832da673940SJordan Gordeev if (*pte & bit) {
2833c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2834da673940SJordan Gordeev return TRUE;
2835da673940SJordan Gordeev }
2836da673940SJordan Gordeev }
2837c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2838da673940SJordan Gordeev return (FALSE);
2839da673940SJordan Gordeev }
2840da673940SJordan Gordeev
2841da673940SJordan Gordeev /*
2842da673940SJordan Gordeev * This routine is used to clear bits in ptes. Certain bits require special
2843da673940SJordan Gordeev * handling, in particular (on virtual kernels) the VPTE_M (modify) bit.
2844da673940SJordan Gordeev *
2845da673940SJordan Gordeev * This routine is only called with certain VPTE_* bit combinations.
2846da673940SJordan Gordeev */
2847da673940SJordan Gordeev static __inline void
pmap_clearbit(vm_page_t m,int bit)2848da673940SJordan Gordeev pmap_clearbit(vm_page_t m, int bit)
2849da673940SJordan Gordeev {
2850da673940SJordan Gordeev pv_entry_t pv;
2851da673940SJordan Gordeev pt_entry_t *pte;
2852da673940SJordan Gordeev pt_entry_t pbits;
285395270b7eSMatthew Dillon vm_object_t pmobj;
285495270b7eSMatthew Dillon pmap_t pmap;
2855da673940SJordan Gordeev
2856b443039bSMatthew Dillon if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) {
2857c50e690bSMatthew Dillon if (bit == VPTE_RW)
2858c50e690bSMatthew Dillon vm_page_flag_clear(m, PG_WRITEABLE);
2859da673940SJordan Gordeev return;
2860b443039bSMatthew Dillon }
2861da673940SJordan Gordeev
2862da673940SJordan Gordeev /*
2863da673940SJordan Gordeev * Loop over all current mappings setting/clearing as appropos If
2864da673940SJordan Gordeev * setting RO do we need to clear the VAC?
2865da673940SJordan Gordeev */
2866c50e690bSMatthew Dillon restart:
286795270b7eSMatthew Dillon vm_page_spin_lock(m);
2868da673940SJordan Gordeev TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2869da673940SJordan Gordeev /*
287095270b7eSMatthew Dillon * Need the pmap object lock(?)
287195270b7eSMatthew Dillon */
287295270b7eSMatthew Dillon pmap = pv->pv_pmap;
287395270b7eSMatthew Dillon pmobj = pmap->pm_pteobj;
287495270b7eSMatthew Dillon
287595270b7eSMatthew Dillon if (vm_object_hold_try(pmobj) == 0) {
287695270b7eSMatthew Dillon refcount_acquire(&pmobj->hold_count);
287795270b7eSMatthew Dillon vm_page_spin_unlock(m);
287895270b7eSMatthew Dillon vm_object_lock(pmobj);
287995270b7eSMatthew Dillon vm_object_drop(pmobj);
288095270b7eSMatthew Dillon goto restart;
288195270b7eSMatthew Dillon }
288295270b7eSMatthew Dillon
288395270b7eSMatthew Dillon /*
2884da673940SJordan Gordeev * don't write protect pager mappings
2885da673940SJordan Gordeev */
2886a86ce0cdSMatthew Dillon if (bit == VPTE_RW) {
2887ae442b2eSMatthew Dillon pmap_track_modified(pv->pv_pmap, pv->pv_va);
288895270b7eSMatthew Dillon }
2889da673940SJordan Gordeev
2890da673940SJordan Gordeev #if defined(PMAP_DIAGNOSTIC)
2891da673940SJordan Gordeev if (pv->pv_pmap == NULL) {
2892da673940SJordan Gordeev kprintf("Null pmap (cb) at va: 0x%lx\n", pv->pv_va);
289395270b7eSMatthew Dillon vm_object_drop(pmobj);
2894da673940SJordan Gordeev continue;
2895da673940SJordan Gordeev }
2896da673940SJordan Gordeev #endif
2897da673940SJordan Gordeev
2898da673940SJordan Gordeev /*
2899da673940SJordan Gordeev * Careful here. We can use a locked bus instruction to
2900da673940SJordan Gordeev * clear VPTE_A or VPTE_M safely but we need to synchronize
2901a86ce0cdSMatthew Dillon * with the target cpus when we mess with VPTE_RW.
2902da673940SJordan Gordeev *
2903da673940SJordan Gordeev * On virtual kernels we must force a new fault-on-write
2904da673940SJordan Gordeev * in the real kernel if we clear the Modify bit ourselves,
2905da673940SJordan Gordeev * otherwise the real kernel will not get a new fault and
2906da673940SJordan Gordeev * will never set our Modify bit again.
2907da673940SJordan Gordeev */
2908da673940SJordan Gordeev pte = pmap_pte(pv->pv_pmap, pv->pv_va);
2909da673940SJordan Gordeev if (*pte & bit) {
2910a86ce0cdSMatthew Dillon if (bit == VPTE_RW) {
2911da673940SJordan Gordeev /*
2912da673940SJordan Gordeev * We must also clear VPTE_M when clearing
2913c50e690bSMatthew Dillon * VPTE_RW and synchronize its state to
2914c50e690bSMatthew Dillon * the page.
2915da673940SJordan Gordeev */
2916ae442b2eSMatthew Dillon pmap_track_modified(pv->pv_pmap, pv->pv_va);
2917da673940SJordan Gordeev pbits = pmap_clean_pte(pte, pv->pv_pmap,
291895270b7eSMatthew Dillon pv->pv_va, m);
2919da673940SJordan Gordeev } else if (bit == VPTE_M) {
2920da673940SJordan Gordeev /*
2921c50e690bSMatthew Dillon * We must invalidate the real-kernel pte
2922c50e690bSMatthew Dillon * when clearing VPTE_M bit to force the
2923c50e690bSMatthew Dillon * real-kernel to take a new fault to re-set
2924c50e690bSMatthew Dillon * VPTE_M.
2925da673940SJordan Gordeev */
2926da673940SJordan Gordeev atomic_clear_long(pte, VPTE_M);
292700eb801eSMatthew Dillon if (*pte & VPTE_RW) {
2928c50e690bSMatthew Dillon pmap_invalidate_range(pv->pv_pmap,
2929c50e690bSMatthew Dillon pv->pv_va,
2930c50e690bSMatthew Dillon pv->pv_va + PAGE_SIZE);
293100eb801eSMatthew Dillon }
2932c50e690bSMatthew Dillon } else if ((bit & (VPTE_RW|VPTE_M)) ==
2933c50e690bSMatthew Dillon (VPTE_RW|VPTE_M)) {
2934da673940SJordan Gordeev /*
2935da673940SJordan Gordeev * We've been asked to clear W & M, I guess
2936da673940SJordan Gordeev * the caller doesn't want us to update
2937da673940SJordan Gordeev * the dirty status of the VM page.
2938da673940SJordan Gordeev */
2939ae442b2eSMatthew Dillon pmap_track_modified(pv->pv_pmap, pv->pv_va);
294095270b7eSMatthew Dillon pmap_clean_pte(pte, pv->pv_pmap, pv->pv_va, m);
2941eb36cb6bSMatthew Dillon panic("shouldn't be called");
2942da673940SJordan Gordeev } else {
2943da673940SJordan Gordeev /*
2944da673940SJordan Gordeev * We've been asked to clear bits that do
2945da673940SJordan Gordeev * not interact with hardware.
2946da673940SJordan Gordeev */
2947da673940SJordan Gordeev atomic_clear_long(pte, bit);
2948da673940SJordan Gordeev }
2949da673940SJordan Gordeev }
295095270b7eSMatthew Dillon vm_object_drop(pmobj);
2951da673940SJordan Gordeev }
2952b443039bSMatthew Dillon if (bit == VPTE_RW)
2953b443039bSMatthew Dillon vm_page_flag_clear(m, PG_WRITEABLE);
2954c91894e0SMatthew Dillon vm_page_spin_unlock(m);
2955da673940SJordan Gordeev }
2956da673940SJordan Gordeev
2957da673940SJordan Gordeev /*
2958da673940SJordan Gordeev * Lower the permission for all mappings to a given page.
2959f7230403SMatthew Dillon *
2960f7230403SMatthew Dillon * No other requirements.
2961da673940SJordan Gordeev */
2962da673940SJordan Gordeev void
pmap_page_protect(vm_page_t m,vm_prot_t prot)2963da673940SJordan Gordeev pmap_page_protect(vm_page_t m, vm_prot_t prot)
2964da673940SJordan Gordeev {
2965da673940SJordan Gordeev if ((prot & VM_PROT_WRITE) == 0) {
2966da673940SJordan Gordeev if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
2967a86ce0cdSMatthew Dillon pmap_clearbit(m, VPTE_RW);
2968da673940SJordan Gordeev } else {
2969da673940SJordan Gordeev pmap_remove_all(m);
2970da673940SJordan Gordeev }
2971da673940SJordan Gordeev }
2972da673940SJordan Gordeev }
2973da673940SJordan Gordeev
2974da673940SJordan Gordeev vm_paddr_t
pmap_phys_address(vm_pindex_t ppn)2975da673940SJordan Gordeev pmap_phys_address(vm_pindex_t ppn)
2976da673940SJordan Gordeev {
29770e6594a8SSascha Wildner return (x86_64_ptob(ppn));
2978da673940SJordan Gordeev }
2979da673940SJordan Gordeev
2980da673940SJordan Gordeev /*
2981da673940SJordan Gordeev * Return a count of reference bits for a page, clearing those bits.
2982da673940SJordan Gordeev * It is not necessary for every reference bit to be cleared, but it
2983da673940SJordan Gordeev * is necessary that 0 only be returned when there are truly no
2984da673940SJordan Gordeev * reference bits set.
2985da673940SJordan Gordeev *
2986da673940SJordan Gordeev * XXX: The exact number of bits to check and clear is a matter that
2987da673940SJordan Gordeev * should be tested and standardized at some point in the future for
2988da673940SJordan Gordeev * optimal aging of shared pages.
2989f7230403SMatthew Dillon *
2990f7230403SMatthew Dillon * No other requirements.
2991da673940SJordan Gordeev */
2992da673940SJordan Gordeev int
pmap_ts_referenced(vm_page_t m)2993da673940SJordan Gordeev pmap_ts_referenced(vm_page_t m)
2994da673940SJordan Gordeev {
2995da673940SJordan Gordeev pv_entry_t pv, pvf, pvn;
2996da673940SJordan Gordeev pt_entry_t *pte;
2997da673940SJordan Gordeev int rtval = 0;
2998da673940SJordan Gordeev
2999da673940SJordan Gordeev if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
3000da673940SJordan Gordeev return (rtval);
3001da673940SJordan Gordeev
3002c91894e0SMatthew Dillon vm_page_spin_lock(m);
3003da673940SJordan Gordeev if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
3004da673940SJordan Gordeev pvf = pv;
3005da673940SJordan Gordeev do {
3006da673940SJordan Gordeev pvn = TAILQ_NEXT(pv, pv_list);
3007da673940SJordan Gordeev TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
3008da673940SJordan Gordeev TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
3009da673940SJordan Gordeev
3010ae442b2eSMatthew Dillon pmap_track_modified(pv->pv_pmap, pv->pv_va);
3011da673940SJordan Gordeev pte = pmap_pte(pv->pv_pmap, pv->pv_va);
3012da673940SJordan Gordeev
3013da673940SJordan Gordeev if (pte && (*pte & VPTE_A)) {
3014da673940SJordan Gordeev atomic_clear_long(pte, VPTE_A);
3015da673940SJordan Gordeev rtval++;
3016da673940SJordan Gordeev if (rtval > 4) {
3017da673940SJordan Gordeev break;
3018da673940SJordan Gordeev }
3019da673940SJordan Gordeev }
3020da673940SJordan Gordeev } while ((pv = pvn) != NULL && pv != pvf);
3021da673940SJordan Gordeev }
3022c91894e0SMatthew Dillon vm_page_spin_unlock(m);
3023da673940SJordan Gordeev
3024da673940SJordan Gordeev return (rtval);
3025da673940SJordan Gordeev }
3026da673940SJordan Gordeev
3027da673940SJordan Gordeev /*
3028da673940SJordan Gordeev * Return whether or not the specified physical page was modified
3029da673940SJordan Gordeev * in any physical maps.
3030f7230403SMatthew Dillon *
3031f7230403SMatthew Dillon * No other requirements.
3032da673940SJordan Gordeev */
3033da673940SJordan Gordeev boolean_t
pmap_is_modified(vm_page_t m)3034da673940SJordan Gordeev pmap_is_modified(vm_page_t m)
3035da673940SJordan Gordeev {
3036f7230403SMatthew Dillon boolean_t res;
3037f7230403SMatthew Dillon
3038f7230403SMatthew Dillon res = pmap_testbit(m, VPTE_M);
3039c91894e0SMatthew Dillon
3040f7230403SMatthew Dillon return (res);
3041da673940SJordan Gordeev }
3042da673940SJordan Gordeev
3043da673940SJordan Gordeev /*
304495270b7eSMatthew Dillon * Clear the modify bits on the specified physical page. For the vkernel
304595270b7eSMatthew Dillon * we really need to clean the page, which clears VPTE_RW and VPTE_M, in
304695270b7eSMatthew Dillon * order to ensure that we take a fault on the next write to the page.
304795270b7eSMatthew Dillon * Otherwise the page may become dirty without us knowing it.
3048f7230403SMatthew Dillon *
3049f7230403SMatthew Dillon * No other requirements.
3050da673940SJordan Gordeev */
3051da673940SJordan Gordeev void
pmap_clear_modify(vm_page_t m)3052da673940SJordan Gordeev pmap_clear_modify(vm_page_t m)
3053da673940SJordan Gordeev {
305495270b7eSMatthew Dillon pmap_clearbit(m, VPTE_RW);
3055da673940SJordan Gordeev }
3056da673940SJordan Gordeev
3057da673940SJordan Gordeev /*
3058da673940SJordan Gordeev * Clear the reference bit on the specified physical page.
3059f7230403SMatthew Dillon *
3060f7230403SMatthew Dillon * No other requirements.
3061da673940SJordan Gordeev */
3062da673940SJordan Gordeev void
pmap_clear_reference(vm_page_t m)3063da673940SJordan Gordeev pmap_clear_reference(vm_page_t m)
3064da673940SJordan Gordeev {
3065da673940SJordan Gordeev pmap_clearbit(m, VPTE_A);
3066da673940SJordan Gordeev }
3067da673940SJordan Gordeev
3068da673940SJordan Gordeev /*
3069da673940SJordan Gordeev * Miscellaneous support routines follow
3070da673940SJordan Gordeev */
3071da673940SJordan Gordeev static void
x86_64_protection_init(void)3072d87f4462Szrj x86_64_protection_init(void)
3073da673940SJordan Gordeev {
307473d64b98SMatthew Dillon uint64_t *kp;
307573d64b98SMatthew Dillon int prot;
3076da673940SJordan Gordeev
3077da673940SJordan Gordeev kp = protection_codes;
3078da673940SJordan Gordeev for (prot = 0; prot < 8; prot++) {
3079da673940SJordan Gordeev if (prot & VM_PROT_READ)
308073d64b98SMatthew Dillon *kp |= 0; /* R */
3081da673940SJordan Gordeev if (prot & VM_PROT_WRITE)
308273d64b98SMatthew Dillon *kp |= VPTE_RW; /* R+W */
308373d64b98SMatthew Dillon if (prot && (prot & VM_PROT_EXECUTE) == 0)
308473d64b98SMatthew Dillon *kp |= VPTE_NX; /* NX - !executable */
3085da673940SJordan Gordeev ++kp;
3086da673940SJordan Gordeev }
3087da673940SJordan Gordeev }
3088da673940SJordan Gordeev
3089da673940SJordan Gordeev /*
3090c4d9bf46SFrançois Tigeot * Sets the memory attribute for the specified page.
3091c4d9bf46SFrançois Tigeot */
3092c4d9bf46SFrançois Tigeot void
pmap_page_set_memattr(vm_page_t m,vm_memattr_t ma)3093c4d9bf46SFrançois Tigeot pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
3094c4d9bf46SFrançois Tigeot {
3095c4d9bf46SFrançois Tigeot /* This is a vkernel, do nothing */
3096c4d9bf46SFrançois Tigeot }
3097c4d9bf46SFrançois Tigeot
3098c4d9bf46SFrançois Tigeot /*
3099c9fa2b57SFrançois Tigeot * Change the PAT attribute on an existing kernel memory map. Caller
3100c9fa2b57SFrançois Tigeot * must ensure that the virtual memory in question is not accessed
3101c9fa2b57SFrançois Tigeot * during the adjustment.
3102c9fa2b57SFrançois Tigeot */
3103c9fa2b57SFrançois Tigeot void
pmap_change_attr(vm_offset_t va,vm_size_t count,int mode)3104c9fa2b57SFrançois Tigeot pmap_change_attr(vm_offset_t va, vm_size_t count, int mode)
3105c9fa2b57SFrançois Tigeot {
3106c9fa2b57SFrançois Tigeot /* This is a vkernel, do nothing */
3107c9fa2b57SFrançois Tigeot }
3108c9fa2b57SFrançois Tigeot
3109c9fa2b57SFrançois Tigeot /*
3110f7230403SMatthew Dillon * Perform the pmap work for mincore
3111f7230403SMatthew Dillon *
3112f7230403SMatthew Dillon * No other requirements.
3113da673940SJordan Gordeev */
3114da673940SJordan Gordeev int
pmap_mincore(pmap_t pmap,vm_offset_t addr)3115da673940SJordan Gordeev pmap_mincore(pmap_t pmap, vm_offset_t addr)
3116da673940SJordan Gordeev {
3117da673940SJordan Gordeev pt_entry_t *ptep, pte;
3118da673940SJordan Gordeev vm_page_t m;
3119da673940SJordan Gordeev int val = 0;
3120da673940SJordan Gordeev
3121c91894e0SMatthew Dillon vm_object_hold(pmap->pm_pteobj);
3122da673940SJordan Gordeev ptep = pmap_pte(pmap, addr);
3123da673940SJordan Gordeev
3124f7230403SMatthew Dillon if (ptep && (pte = *ptep) != 0) {
31258608b858SMatthew Dillon vm_paddr_t pa;
3126da673940SJordan Gordeev
3127da673940SJordan Gordeev val = MINCORE_INCORE;
3128da673940SJordan Gordeev if ((pte & VPTE_MANAGED) == 0)
3129f7230403SMatthew Dillon goto done;
3130da673940SJordan Gordeev
3131da673940SJordan Gordeev pa = pte & VPTE_FRAME;
3132da673940SJordan Gordeev
3133da673940SJordan Gordeev m = PHYS_TO_VM_PAGE(pa);
3134da673940SJordan Gordeev
3135da673940SJordan Gordeev /*
3136da673940SJordan Gordeev * Modified by us
3137da673940SJordan Gordeev */
3138da673940SJordan Gordeev if (pte & VPTE_M)
3139da673940SJordan Gordeev val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
3140da673940SJordan Gordeev /*
3141da673940SJordan Gordeev * Modified by someone
3142da673940SJordan Gordeev */
3143da673940SJordan Gordeev else if (m->dirty || pmap_is_modified(m))
3144da673940SJordan Gordeev val |= MINCORE_MODIFIED_OTHER;
3145da673940SJordan Gordeev /*
3146da673940SJordan Gordeev * Referenced by us
3147da673940SJordan Gordeev */
3148da673940SJordan Gordeev if (pte & VPTE_A)
3149da673940SJordan Gordeev val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
3150da673940SJordan Gordeev
3151da673940SJordan Gordeev /*
3152da673940SJordan Gordeev * Referenced by someone
3153da673940SJordan Gordeev */
3154da673940SJordan Gordeev else if ((m->flags & PG_REFERENCED) || pmap_ts_referenced(m)) {
3155da673940SJordan Gordeev val |= MINCORE_REFERENCED_OTHER;
3156da673940SJordan Gordeev vm_page_flag_set(m, PG_REFERENCED);
3157da673940SJordan Gordeev }
3158da673940SJordan Gordeev }
3159f7230403SMatthew Dillon done:
3160c91894e0SMatthew Dillon vm_object_drop(pmap->pm_pteobj);
3161c91894e0SMatthew Dillon
3162da673940SJordan Gordeev return val;
3163da673940SJordan Gordeev }
3164da673940SJordan Gordeev
3165da673940SJordan Gordeev /*
3166da673940SJordan Gordeev * Replace p->p_vmspace with a new one. If adjrefs is non-zero the new
3167da673940SJordan Gordeev * vmspace will be ref'd and the old one will be deref'd.
3168b12defdcSMatthew Dillon *
3169b12defdcSMatthew Dillon * Caller must hold vmspace->vm_map.token for oldvm and newvm
3170da673940SJordan Gordeev */
3171da673940SJordan Gordeev void
pmap_replacevm(struct proc * p,struct vmspace * newvm,int adjrefs)3172da673940SJordan Gordeev pmap_replacevm(struct proc *p, struct vmspace *newvm, int adjrefs)
3173da673940SJordan Gordeev {
3174da673940SJordan Gordeev struct vmspace *oldvm;
3175da673940SJordan Gordeev struct lwp *lp;
3176da673940SJordan Gordeev
3177da673940SJordan Gordeev oldvm = p->p_vmspace;
3178da673940SJordan Gordeev if (oldvm != newvm) {
3179685f4bf5SMatthew Dillon if (adjrefs)
3180685f4bf5SMatthew Dillon vmspace_ref(newvm);
3181a3a33e50SMatthew Dillon KKASSERT((newvm->vm_refcnt & VM_REF_DELETED) == 0);
3182da673940SJordan Gordeev p->p_vmspace = newvm;
3183da673940SJordan Gordeev KKASSERT(p->p_nthreads == 1);
3184da673940SJordan Gordeev lp = RB_ROOT(&p->p_lwp_tree);
3185da673940SJordan Gordeev pmap_setlwpvm(lp, newvm);
3186685f4bf5SMatthew Dillon if (adjrefs)
318793f86408SMatthew Dillon vmspace_rel(oldvm);
3188da673940SJordan Gordeev }
3189da673940SJordan Gordeev }
3190da673940SJordan Gordeev
3191da673940SJordan Gordeev /*
3192da673940SJordan Gordeev * Set the vmspace for a LWP. The vmspace is almost universally set the
3193da673940SJordan Gordeev * same as the process vmspace, but virtual kernels need to swap out contexts
3194da673940SJordan Gordeev * on a per-lwp basis.
3195da673940SJordan Gordeev */
3196da673940SJordan Gordeev void
pmap_setlwpvm(struct lwp * lp,struct vmspace * newvm)3197da673940SJordan Gordeev pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
3198da673940SJordan Gordeev {
3199da673940SJordan Gordeev struct vmspace *oldvm;
3200da673940SJordan Gordeev struct pmap *pmap;
3201da673940SJordan Gordeev
3202da673940SJordan Gordeev oldvm = lp->lwp_vmspace;
3203685f4bf5SMatthew Dillon if (oldvm != newvm) {
3204a86ce0cdSMatthew Dillon crit_enter();
3205a3a33e50SMatthew Dillon KKASSERT((newvm->vm_refcnt & VM_REF_DELETED) == 0);
3206685f4bf5SMatthew Dillon lp->lwp_vmspace = newvm;
3207685f4bf5SMatthew Dillon if (curthread->td_lwp == lp) {
3208da673940SJordan Gordeev pmap = vmspace_pmap(newvm);
3209c07315c4SMatthew Dillon ATOMIC_CPUMASK_ORBIT(pmap->pm_active, mycpu->gd_cpuid);
3210685f4bf5SMatthew Dillon if (pmap->pm_active_lock & CPULOCK_EXCL)
3211685f4bf5SMatthew Dillon pmap_interlock_wait(newvm);
3212da673940SJordan Gordeev #if defined(SWTCH_OPTIM_STATS)
3213da673940SJordan Gordeev tlb_flush_count++;
3214da673940SJordan Gordeev #endif
3215da673940SJordan Gordeev pmap = vmspace_pmap(oldvm);
3216685f4bf5SMatthew Dillon ATOMIC_CPUMASK_NANDBIT(pmap->pm_active,
3217685f4bf5SMatthew Dillon mycpu->gd_cpuid);
3218685f4bf5SMatthew Dillon }
3219da673940SJordan Gordeev crit_exit();
3220da673940SJordan Gordeev }
3221685f4bf5SMatthew Dillon }
3222da673940SJordan Gordeev
3223a86ce0cdSMatthew Dillon /*
3224a86ce0cdSMatthew Dillon * The swtch code tried to switch in a heavy weight process whos pmap
3225a86ce0cdSMatthew Dillon * is locked by another cpu. We have to wait for the lock to clear before
3226a86ce0cdSMatthew Dillon * the pmap can be used.
3227a86ce0cdSMatthew Dillon */
3228a86ce0cdSMatthew Dillon void
pmap_interlock_wait(struct vmspace * vm)3229a86ce0cdSMatthew Dillon pmap_interlock_wait (struct vmspace *vm)
3230a86ce0cdSMatthew Dillon {
3231a86ce0cdSMatthew Dillon pmap_t pmap = vmspace_pmap(vm);
3232a86ce0cdSMatthew Dillon
3233685f4bf5SMatthew Dillon if (pmap->pm_active_lock & CPULOCK_EXCL) {
3234685f4bf5SMatthew Dillon crit_enter();
3235685f4bf5SMatthew Dillon while (pmap->pm_active_lock & CPULOCK_EXCL) {
3236685f4bf5SMatthew Dillon cpu_ccfence();
3237eb8c4738Szrj vkernel_yield();
3238a86ce0cdSMatthew Dillon }
3239685f4bf5SMatthew Dillon crit_exit();
3240685f4bf5SMatthew Dillon }
3241685f4bf5SMatthew Dillon }
3242a86ce0cdSMatthew Dillon
3243da673940SJordan Gordeev vm_offset_t
pmap_addr_hint(vm_object_t obj,vm_offset_t addr,vm_size_t size)3244da673940SJordan Gordeev pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
3245da673940SJordan Gordeev {
3246da673940SJordan Gordeev
3247da673940SJordan Gordeev if ((obj == NULL) || (size < NBPDR) || (obj->type != OBJT_DEVICE)) {
3248da673940SJordan Gordeev return addr;
3249da673940SJordan Gordeev }
3250da673940SJordan Gordeev
3251965b839fSSascha Wildner addr = roundup2(addr, NBPDR);
3252da673940SJordan Gordeev return addr;
3253da673940SJordan Gordeev }
3254722871d3SMatthew Dillon
3255722871d3SMatthew Dillon /*
3256722871d3SMatthew Dillon * Used by kmalloc/kfree, page already exists at va
3257722871d3SMatthew Dillon */
3258722871d3SMatthew Dillon vm_page_t
pmap_kvtom(vm_offset_t va)3259722871d3SMatthew Dillon pmap_kvtom(vm_offset_t va)
3260722871d3SMatthew Dillon {
3261722871d3SMatthew Dillon vpte_t *ptep;
3262722871d3SMatthew Dillon
3263722871d3SMatthew Dillon KKASSERT(va >= KvaStart && va < KvaEnd);
32645e6356abSSascha Wildner ptep = vtopte(va);
32655e6356abSSascha Wildner return(PHYS_TO_VM_PAGE(*ptep & PG_FRAME));
3266722871d3SMatthew Dillon }
3267921c891eSMatthew Dillon
3268921c891eSMatthew Dillon void
pmap_object_init(vm_object_t object)3269921c891eSMatthew Dillon pmap_object_init(vm_object_t object)
3270921c891eSMatthew Dillon {
3271921c891eSMatthew Dillon /* empty */
3272921c891eSMatthew Dillon }
3273921c891eSMatthew Dillon
3274921c891eSMatthew Dillon void
pmap_object_free(vm_object_t object)3275921c891eSMatthew Dillon pmap_object_free(vm_object_t object)
3276921c891eSMatthew Dillon {
3277921c891eSMatthew Dillon /* empty */
3278921c891eSMatthew Dillon }
3279a7a03a5fSMatthew Dillon
3280a7a03a5fSMatthew Dillon void
pmap_pgscan(struct pmap_pgscan_info * pginfo)3281a7a03a5fSMatthew Dillon pmap_pgscan(struct pmap_pgscan_info *pginfo)
3282a7a03a5fSMatthew Dillon {
3283a7a03a5fSMatthew Dillon pmap_t pmap = pginfo->pmap;
3284a7a03a5fSMatthew Dillon vm_offset_t sva = pginfo->beg_addr;
3285a7a03a5fSMatthew Dillon vm_offset_t eva = pginfo->end_addr;
3286a7a03a5fSMatthew Dillon vm_offset_t va_next;
3287a7a03a5fSMatthew Dillon pml4_entry_t *pml4e;
3288a7a03a5fSMatthew Dillon pdp_entry_t *pdpe;
3289a7a03a5fSMatthew Dillon pd_entry_t ptpaddr, *pde;
3290a7a03a5fSMatthew Dillon pt_entry_t *pte;
3291eb36cb6bSMatthew Dillon vm_page_t pt_m;
3292a7a03a5fSMatthew Dillon int stop = 0;
3293a7a03a5fSMatthew Dillon
3294c91894e0SMatthew Dillon vm_object_hold(pmap->pm_pteobj);
3295a7a03a5fSMatthew Dillon
3296a7a03a5fSMatthew Dillon for (; sva < eva; sva = va_next) {
3297a7a03a5fSMatthew Dillon if (stop)
3298a7a03a5fSMatthew Dillon break;
3299a7a03a5fSMatthew Dillon
3300a7a03a5fSMatthew Dillon pml4e = pmap_pml4e(pmap, sva);
3301a7a03a5fSMatthew Dillon if ((*pml4e & VPTE_V) == 0) {
3302a7a03a5fSMatthew Dillon va_next = (sva + NBPML4) & ~PML4MASK;
3303a7a03a5fSMatthew Dillon if (va_next < sva)
3304a7a03a5fSMatthew Dillon va_next = eva;
3305a7a03a5fSMatthew Dillon continue;
3306a7a03a5fSMatthew Dillon }
3307a7a03a5fSMatthew Dillon
3308a7a03a5fSMatthew Dillon pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
3309a7a03a5fSMatthew Dillon if ((*pdpe & VPTE_V) == 0) {
3310a7a03a5fSMatthew Dillon va_next = (sva + NBPDP) & ~PDPMASK;
3311a7a03a5fSMatthew Dillon if (va_next < sva)
3312a7a03a5fSMatthew Dillon va_next = eva;
3313a7a03a5fSMatthew Dillon continue;
3314a7a03a5fSMatthew Dillon }
3315a7a03a5fSMatthew Dillon
3316a7a03a5fSMatthew Dillon va_next = (sva + NBPDR) & ~PDRMASK;
3317a7a03a5fSMatthew Dillon if (va_next < sva)
3318a7a03a5fSMatthew Dillon va_next = eva;
3319a7a03a5fSMatthew Dillon
3320a7a03a5fSMatthew Dillon pde = pmap_pdpe_to_pde(pdpe, sva);
3321a7a03a5fSMatthew Dillon ptpaddr = *pde;
3322a7a03a5fSMatthew Dillon
3323c78d5661SMatthew Dillon #if 0
3324a7a03a5fSMatthew Dillon /*
3325a7a03a5fSMatthew Dillon * Check for large page (ignore).
3326a7a03a5fSMatthew Dillon */
3327a7a03a5fSMatthew Dillon if ((ptpaddr & VPTE_PS) != 0) {
3328a7a03a5fSMatthew Dillon #if 0
3329a7a03a5fSMatthew Dillon pmap_clean_pde(pde, pmap, sva);
3330a7a03a5fSMatthew Dillon pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
3331a7a03a5fSMatthew Dillon #endif
3332a7a03a5fSMatthew Dillon continue;
3333a7a03a5fSMatthew Dillon }
3334c78d5661SMatthew Dillon #endif
3335a7a03a5fSMatthew Dillon
3336a7a03a5fSMatthew Dillon /*
3337a7a03a5fSMatthew Dillon * Weed out invalid mappings. Note: we assume that the page
3338a7a03a5fSMatthew Dillon * directory table is always allocated, and in kernel virtual.
3339a7a03a5fSMatthew Dillon */
3340a7a03a5fSMatthew Dillon if (ptpaddr == 0)
3341a7a03a5fSMatthew Dillon continue;
3342a7a03a5fSMatthew Dillon
3343a7a03a5fSMatthew Dillon if (va_next > eva)
3344a7a03a5fSMatthew Dillon va_next = eva;
3345a7a03a5fSMatthew Dillon
3346eb36cb6bSMatthew Dillon pt_m = pmap_hold_pt_page(pde, sva);
3347a7a03a5fSMatthew Dillon for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
3348a7a03a5fSMatthew Dillon sva += PAGE_SIZE) {
3349a7a03a5fSMatthew Dillon vm_page_t m;
3350a7a03a5fSMatthew Dillon
3351a7a03a5fSMatthew Dillon if (stop)
3352a7a03a5fSMatthew Dillon break;
3353a7a03a5fSMatthew Dillon if ((*pte & VPTE_MANAGED) == 0)
3354a7a03a5fSMatthew Dillon continue;
3355a7a03a5fSMatthew Dillon
3356a7a03a5fSMatthew Dillon m = PHYS_TO_VM_PAGE(*pte & VPTE_FRAME);
3357a7a03a5fSMatthew Dillon if (vm_page_busy_try(m, TRUE) == 0) {
3358a7a03a5fSMatthew Dillon if (pginfo->callback(pginfo, sva, m) < 0)
3359a7a03a5fSMatthew Dillon stop = 1;
3360a7a03a5fSMatthew Dillon }
3361a7a03a5fSMatthew Dillon }
3362eb36cb6bSMatthew Dillon vm_page_unhold(pt_m);
3363a7a03a5fSMatthew Dillon }
3364c91894e0SMatthew Dillon vm_object_drop(pmap->pm_pteobj);
3365a7a03a5fSMatthew Dillon }
3366e3c330f0SMatthew Dillon
3367e3c330f0SMatthew Dillon void
pmap_maybethreaded(pmap_t pmap)3368e3c330f0SMatthew Dillon pmap_maybethreaded(pmap_t pmap)
3369e3c330f0SMatthew Dillon {
3370e3c330f0SMatthew Dillon /* nop */
3371e3c330f0SMatthew Dillon }
3372e3c330f0SMatthew Dillon
3373e3c330f0SMatthew Dillon /*
3374e3c330f0SMatthew Dillon * Called while page is hard-busied to clear the PG_MAPPED and PG_WRITEABLE
3375e3c330f0SMatthew Dillon * flags if able.
3376e3c330f0SMatthew Dillon *
3377e3c330f0SMatthew Dillon * vkernel code is using the old pmap style so the flags should already
3378e3c330f0SMatthew Dillon * be properly set.
3379e3c330f0SMatthew Dillon */
3380e3c330f0SMatthew Dillon int
pmap_mapped_sync(vm_page_t m)3381e3c330f0SMatthew Dillon pmap_mapped_sync(vm_page_t m)
3382e3c330f0SMatthew Dillon {
3383e3c330f0SMatthew Dillon return (m->flags);
3384e3c330f0SMatthew Dillon }
3385