1 /* $NetBSD: pmap.h,v 1.117 2014/04/21 19:12:11 christos Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 /*
29 * Copyright (c) 2001 Wasabi Systems, Inc.
30 * All rights reserved.
31 *
32 * Written by Frank van der Linden for Wasabi Systems, Inc.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in the
41 * documentation and/or other materials provided with the distribution.
42 * 3. All advertising materials mentioning features or use of this software
43 * must display the following acknowledgement:
44 * This product includes software developed for the NetBSD Project by
45 * Wasabi Systems, Inc.
46 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
47 * or promote products derived from this software without specific prior
48 * written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
52 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
53 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
54 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
60 * POSSIBILITY OF SUCH DAMAGE.
61 */
62
63 #ifndef _I386_PMAP_H_
64 #define _I386_PMAP_H_
65
66 #if defined(_KERNEL_OPT)
67 #include "opt_user_ldt.h"
68 #include "opt_xen.h"
69 #endif
70
71 #include <sys/atomic.h>
72
73 #include <i386/pte.h>
74 #include <machine/segments.h>
75 #if defined(_KERNEL)
76 #include <machine/cpufunc.h>
77 #endif
78
79 #include <uvm/uvm_object.h>
80 #ifdef XEN
81 #include <xen/xenfunc.h>
82 #include <xen/xenpmap.h>
83 #endif /* XEN */
84
85 /*
86 * see pte.h for a description of i386 MMU terminology and hardware
87 * interface.
88 *
89 * a pmap describes a processes' 4GB virtual address space. when PAE
90 * is not in use, this virtual address space can be broken up into 1024 4MB
91 * regions which are described by PDEs in the PDP. the PDEs are defined as
92 * follows:
93 *
94 * (ranges are inclusive -> exclusive, just like vm_map_entry start/end)
95 * (the following assumes that KERNBASE is 0xc0000000)
96 *
97 * PDE#s VA range usage
98 * 0->766 0x0 -> 0xbfc00000 user address space
99 * 767 0xbfc00000-> recursive mapping of PDP (used for
100 * 0xc0000000 linear mapping of PTPs)
101 * 768->1023 0xc0000000-> kernel address space (constant
102 * 0xffc00000 across all pmap's/processes)
103 * <end>
104 *
105 *
106 * note: a recursive PDP mapping provides a way to map all the PTEs for
107 * a 4GB address space into a linear chunk of virtual memory. in other
108 * words, the PTE for page 0 is the first int mapped into the 4MB recursive
109 * area. the PTE for page 1 is the second int. the very last int in the
110 * 4MB range is the PTE that maps VA 0xfffff000 (the last page in a 4GB
111 * address).
112 *
113 * all pmap's PD's must have the same values in slots 768->1023 so that
114 * the kernel is always mapped in every process. these values are loaded
115 * into the PD at pmap creation time.
116 *
117 * at any one time only one pmap can be active on a processor. this is
118 * the pmap whose PDP is pointed to by processor register %cr3. this pmap
119 * will have all its PTEs mapped into memory at the recursive mapping
120 * point (slot #767 as show above). when the pmap code wants to find the
121 * PTE for a virtual address, all it has to do is the following:
122 *
123 * address of PTE = (767 * 4MB) + (VA / PAGE_SIZE) * sizeof(pt_entry_t)
124 * = 0xbfc00000 + (VA / 4096) * 4
125 *
126 * what happens if the pmap layer is asked to perform an operation
127 * on a pmap that is not the one which is currently active? in that
128 * case we temporarily load this pmap, perform the operation, and mark
129 * the currently active one as pending lazy reload.
130 *
131 * the following figure shows the effects of the recursive PDP mapping:
132 *
133 * PDP (%cr3)
134 * +----+
135 * | 0| -> PTP#0 that maps VA 0x0 -> 0x400000
136 * | |
137 * | |
138 * | 767| -> points back to PDP (%cr3) mapping VA 0xbfc00000 -> 0xc0000000
139 * | 768| -> first kernel PTP (maps 0xc0000000 -> 0xc0400000)
140 * | |
141 * +----+
142 *
143 * note that the PDE#767 VA (0xbfc00000) is defined as "PTE_BASE"
144 *
145 * starting at VA 0xbfc00000 the current active PDP (%cr3) acts as a
146 * PTP:
147 *
148 * PTP#767 == PDP(%cr3) => maps VA 0xbfc00000 -> 0xc0000000
149 * +----+
150 * | 0| -> maps the contents of PTP#0 at VA 0xbfc00000->0xbfc01000
151 * | |
152 * | |
153 * | 767| -> maps contents of PTP#767 (the PDP) at VA 0xbfeff000
154 * | 768| -> maps contents of first kernel PTP
155 * | |
156 * |1023|
157 * +----+
158 *
159 * note that mapping of the PDP at PTP#767's VA (0xbfeff000) is
160 * defined as "PDP_BASE".... within that mapping there are two
161 * defines:
162 * "PDP_PDE" (0xbfeffbfc) is the VA of the PDE in the PDP
163 * which points back to itself.
164 *
165 * - PAE support -
166 * ---------------
167 *
168 * PAE adds another layer of indirection during address translation, breaking
169 * up the translation process in 3 different levels:
170 * - L3 page directory, containing 4 * 64-bits addresses (index determined by
171 * bits [31:30] from the virtual address). This breaks up the address space
172 * in 4 1GB regions.
173 * - the PD (L2), containing 512 64-bits addresses, breaking each L3 region
174 * in 512 * 2MB regions.
175 * - the PT (L1), also containing 512 64-bits addresses (at L1, the size of
176 * the pages is still 4K).
177 *
178 * The kernel virtual space is mapped by the last entry in the L3 page,
179 * the first 3 entries mapping the user VA space.
180 *
181 * Because the L3 has only 4 entries of 1GB each, we can't use recursive
182 * mappings at this level for PDP_PDE (this would eat up 2 of the 4GB
183 * virtual space). There are also restrictions imposed by Xen on the
184 * last entry of the L3 PD (reference count to this page cannot be
185 * bigger than 1), which makes it hard to use one L3 page per pmap to
186 * switch between pmaps using %cr3.
187 *
188 * As such, each CPU gets its own L3 page that is always loaded into its %cr3
189 * (ci_pae_l3_pd in the associated cpu_info struct). We claim that the VM has
190 * only a 2-level PTP (similar to the non-PAE case). L2 PD is now 4 contiguous
191 * pages long (corresponding to the 4 entries of the L3), and the different
192 * index/slots (like PDP_PDE) are adapted accordingly.
193 *
194 * Kernel space remains in L3[3], L3[0-2] maps the user VA space. Switching
195 * between pmaps consists in modifying the first 3 entries of the CPU's L3 page.
196 *
197 * PTE_BASE will need 4 entries in the L2 PD pages to map the L2 pages
198 * recursively.
199 *
200 * In addition, for Xen, we can't recursively map L3[3] (Xen wants the ref
201 * count on this page to be exactly one), so we use a shadow PD page for
202 * the last L2 PD. The shadow page could be static too, but to make pm_pdir[]
203 * contiguous we'll allocate/copy one page per pmap.
204 */
205
206 /*
207 * Mask to get rid of the sign-extended part of addresses.
208 */
209 #define VA_SIGN_MASK 0
210 #define VA_SIGN_NEG(va) ((va) | VA_SIGN_MASK)
211 /*
212 * XXXfvdl this one's not right.
213 */
214 #define VA_SIGN_POS(va) ((va) & ~VA_SIGN_MASK)
215
216 /*
217 * the following defines identify the slots used as described above.
218 */
219 #ifdef PAE
220 #define L2_SLOT_PTE (KERNBASE/NBPD_L2-4) /* 1532: for recursive PDP map */
221 #define L2_SLOT_KERN (KERNBASE/NBPD_L2) /* 1536: start of kernel space */
222 #else /* PAE */
223 #define L2_SLOT_PTE (KERNBASE/NBPD_L2-1) /* 767: for recursive PDP map */
224 #define L2_SLOT_KERN (KERNBASE/NBPD_L2) /* 768: start of kernel space */
225 #endif /* PAE */
226
227 #define L2_SLOT_KERNBASE L2_SLOT_KERN
228
229 #define PDIR_SLOT_KERN L2_SLOT_KERN
230 #define PDIR_SLOT_PTE L2_SLOT_PTE
231
232 /*
233 * the following defines give the virtual addresses of various MMU
234 * data structures:
235 * PTE_BASE: the base VA of the linear PTE mappings
236 * PDP_BASE: the base VA of the recursive mapping of the PDP
237 * PDP_PDE: the VA of the PDE that points back to the PDP
238 */
239
240 #define PTE_BASE ((pt_entry_t *) (PDIR_SLOT_PTE * NBPD_L2))
241
242 #define L1_BASE PTE_BASE
243
244 #define L2_BASE ((pd_entry_t *)((char *)L1_BASE + L2_SLOT_PTE * NBPD_L1))
245
246 #define PDP_PDE (L2_BASE + PDIR_SLOT_PTE)
247
248 #define PDP_BASE L2_BASE
249
250 /* largest value (-1 for APTP space) */
251 #define NKL2_MAX_ENTRIES (NTOPLEVEL_PDES - (KERNBASE/NBPD_L2) - 1)
252 #define NKL1_MAX_ENTRIES (unsigned long)(NKL2_MAX_ENTRIES * NPDPG)
253
254 #define NKL2_KIMG_ENTRIES 0 /* XXX unused */
255
256 #define NKL2_START_ENTRIES 0 /* XXX computed on runtime */
257 #define NKL1_START_ENTRIES 0 /* XXX unused */
258
259 #ifndef XEN
260 #define NTOPLEVEL_PDES (PAGE_SIZE * PDP_SIZE / (sizeof (pd_entry_t)))
261 #else /* !XEN */
262 #ifdef PAE
263 #define NTOPLEVEL_PDES 1964 /* 1964-2047 reserved by Xen */
264 #else /* PAE */
265 #define NTOPLEVEL_PDES 1008 /* 1008-1023 reserved by Xen */
266 #endif /* PAE */
267 #endif /* !XEN */
268 #define NPDPG (PAGE_SIZE / sizeof (pd_entry_t))
269
270 #define PTP_MASK_INITIALIZER { L1_FRAME, L2_FRAME }
271 #define PTP_SHIFT_INITIALIZER { L1_SHIFT, L2_SHIFT }
272 #define NKPTP_INITIALIZER { NKL1_START_ENTRIES, NKL2_START_ENTRIES }
273 #define NKPTPMAX_INITIALIZER { NKL1_MAX_ENTRIES, NKL2_MAX_ENTRIES }
274 #define NBPD_INITIALIZER { NBPD_L1, NBPD_L2 }
275 #define PDES_INITIALIZER { L2_BASE }
276
277 #define PTP_LEVELS 2
278
279 /*
280 * PG_AVAIL usage: we make use of the ignored bits of the PTE
281 */
282
283 #define PG_W PG_AVAIL1 /* "wired" mapping */
284 #define PG_PVLIST PG_AVAIL2 /* mapping has entry on pvlist */
285 #define PG_X PG_AVAIL3 /* executable mapping */
286
287 /*
288 * Number of PTE's per cache line. 4 byte pte, 32-byte cache line
289 * Used to avoid false sharing of cache lines.
290 */
291 #ifdef PAE
292 #define NPTECL 4
293 #else
294 #define NPTECL 8
295 #endif
296
297 #include <x86/pmap.h>
298
299 #ifndef XEN
300 #define pmap_pa2pte(a) (a)
301 #define pmap_pte2pa(a) ((a) & PG_FRAME)
302 #define pmap_pte_set(p, n) do { *(p) = (n); } while (0)
303 #define pmap_pte_flush() /* nothing */
304
305 #ifdef PAE
306 #define pmap_pte_cas(p, o, n) atomic_cas_64((p), (o), (n))
307 #define pmap_pte_testset(p, n) \
308 atomic_swap_64((volatile uint64_t *)p, n)
309 #define pmap_pte_setbits(p, b) \
310 atomic_or_64((volatile uint64_t *)p, b)
311 #define pmap_pte_clearbits(p, b) \
312 atomic_and_64((volatile uint64_t *)p, ~(b))
313 #else /* PAE */
314 #define pmap_pte_cas(p, o, n) atomic_cas_32((p), (o), (n))
315 #define pmap_pte_testset(p, n) \
316 atomic_swap_ulong((volatile unsigned long *)p, n)
317 #define pmap_pte_setbits(p, b) \
318 atomic_or_ulong((volatile unsigned long *)p, b)
319 #define pmap_pte_clearbits(p, b) \
320 atomic_and_ulong((volatile unsigned long *)p, ~(b))
321 #endif /* PAE */
322
323 #else /* XEN */
324 extern kmutex_t pte_lock;
325
326 static __inline pt_entry_t
pmap_pa2pte(paddr_t pa)327 pmap_pa2pte(paddr_t pa)
328 {
329 return (pt_entry_t)xpmap_ptom_masked(pa);
330 }
331
332 static __inline paddr_t
pmap_pte2pa(pt_entry_t pte)333 pmap_pte2pa(pt_entry_t pte)
334 {
335 return xpmap_mtop_masked(pte & PG_FRAME);
336 }
337 static __inline void
pmap_pte_set(pt_entry_t * pte,pt_entry_t npte)338 pmap_pte_set(pt_entry_t *pte, pt_entry_t npte)
339 {
340 int s = splvm();
341 xpq_queue_pte_update(xpmap_ptetomach(pte), npte);
342 splx(s);
343 }
344
345 static __inline pt_entry_t
pmap_pte_cas(volatile pt_entry_t * ptep,pt_entry_t o,pt_entry_t n)346 pmap_pte_cas(volatile pt_entry_t *ptep, pt_entry_t o, pt_entry_t n)
347 {
348 pt_entry_t opte;
349
350 mutex_enter(&pte_lock);
351 opte = *ptep;
352 if (opte == o) {
353 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(ptep)), n);
354 xpq_flush_queue();
355 }
356 mutex_exit(&pte_lock);
357 return opte;
358 }
359
360 static __inline pt_entry_t
pmap_pte_testset(volatile pt_entry_t * pte,pt_entry_t npte)361 pmap_pte_testset(volatile pt_entry_t *pte, pt_entry_t npte)
362 {
363 pt_entry_t opte;
364
365 mutex_enter(&pte_lock);
366 opte = *pte;
367 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)),
368 npte);
369 xpq_flush_queue();
370 mutex_exit(&pte_lock);
371 return opte;
372 }
373
374 static __inline void
pmap_pte_setbits(volatile pt_entry_t * pte,pt_entry_t bits)375 pmap_pte_setbits(volatile pt_entry_t *pte, pt_entry_t bits)
376 {
377 mutex_enter(&pte_lock);
378 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), (*pte) | bits);
379 xpq_flush_queue();
380 mutex_exit(&pte_lock);
381 }
382
383 static __inline void
pmap_pte_clearbits(volatile pt_entry_t * pte,pt_entry_t bits)384 pmap_pte_clearbits(volatile pt_entry_t *pte, pt_entry_t bits)
385 {
386 mutex_enter(&pte_lock);
387 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)),
388 (*pte) & ~bits);
389 xpq_flush_queue();
390 mutex_exit(&pte_lock);
391 }
392
393 static __inline void
pmap_pte_flush(void)394 pmap_pte_flush(void)
395 {
396 int s = splvm();
397 xpq_flush_queue();
398 splx(s);
399 }
400
401 #endif
402
403 struct vm_map;
404 struct trapframe;
405 struct pcb;
406
407 int pmap_exec_fixup(struct vm_map *, struct trapframe *, struct pcb *);
408 void pmap_ldt_cleanup(struct lwp *);
409
410 #include <x86/pmap_pv.h>
411
412 #define __HAVE_VM_PAGE_MD
413 #define VM_MDPAGE_INIT(pg) \
414 memset(&(pg)->mdpage, 0, sizeof((pg)->mdpage)); \
415 PMAP_PAGE_INIT(&(pg)->mdpage.mp_pp)
416
417 struct vm_page_md {
418 struct pmap_page mp_pp;
419 };
420
421 #endif /* _I386_PMAP_H_ */
422