1 /* $NetBSD: pmap_private.h,v 1.4 2022/08/21 09:12:43 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 /*
29 * Copyright (c) 2001 Wasabi Systems, Inc.
30 * All rights reserved.
31 *
32 * Written by Frank van der Linden for Wasabi Systems, Inc.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in the
41 * documentation and/or other materials provided with the distribution.
42 * 3. All advertising materials mentioning features or use of this software
43 * must display the following acknowledgement:
44 * This product includes software developed for the NetBSD Project by
45 * Wasabi Systems, Inc.
46 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
47 * or promote products derived from this software without specific prior
48 * written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
52 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
53 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
54 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
60 * POSSIBILITY OF SUCH DAMAGE.
61 */
62
63 #ifndef _AMD64_PMAP_PRIVATE_H_
64 #define _AMD64_PMAP_PRIVATE_H_
65
66 #ifdef __x86_64__
67
68 #if defined(_KERNEL_OPT)
69 #include "opt_xen.h"
70 #include "opt_kasan.h"
71 #include "opt_kmsan.h"
72 #include "opt_kubsan.h"
73 #endif
74
75 #include <sys/atomic.h>
76
77 #include <machine/pte.h>
78 #include <machine/segments.h>
79 #ifdef _KERNEL
80 #include <machine/cpufunc.h>
81 #endif
82
83 #include <uvm/uvm_object.h>
84 #ifdef XENPV
85 #include <xen/xenfunc.h>
86 #include <xen/xenpmap.h>
87 #endif
88
89 #ifdef KASAN
90 #define L4_SLOT_KASAN 256
91 #define NL4_SLOT_KASAN 32
92 #endif
93
94 #ifdef KMSAN
95 #define L4_SLOT_KMSAN 256
96 #define NL4_SLOT_KMSAN 4
97 #endif
98
99 #define NL4_SLOT_DIRECT 32
100
101 #ifndef XENPV
102 #define L4_SLOT_PTE slotspace.area[SLAREA_PTE].sslot
103 #else
104 #define L4_SLOT_PTE 509
105 #endif
106 #define L4_SLOT_KERN slotspace.area[SLAREA_MAIN].sslot
107 #define L4_SLOT_KERNBASE 511 /* pl4_i(KERNBASE) */
108
109 #define PDIR_SLOT_USERLIM 255
110 #define PDIR_SLOT_KERN L4_SLOT_KERN
111 #define PDIR_SLOT_PTE L4_SLOT_PTE
112
113 /*
114 * The following defines give the virtual addresses of various MMU
115 * data structures:
116 * PTE_BASE: the base VA of the linear PTE mappings
117 * PDP_BASE: the base VA of the recursive mapping of the PTD
118 */
119
120 #ifndef XENPV
121 extern pt_entry_t *pte_base;
122 #define PTE_BASE pte_base
123 #else
124 #define PTE_BASE ((pt_entry_t *)VA_SIGN_NEG((L4_SLOT_PTE * NBPD_L4)))
125 #endif
126
127 #define L1_BASE PTE_BASE
128 #define L2_BASE ((pd_entry_t *)((char *)L1_BASE + L4_SLOT_PTE * NBPD_L3))
129 #define L3_BASE ((pd_entry_t *)((char *)L2_BASE + L4_SLOT_PTE * NBPD_L2))
130 #define L4_BASE ((pd_entry_t *)((char *)L3_BASE + L4_SLOT_PTE * NBPD_L1))
131
132 #define PDP_BASE L4_BASE
133
134 #if defined(KMSAN)
135 #define NKL4_MAX_ENTRIES (unsigned long)1 /* 512GB only */
136 #else
137 #define NKL4_MAX_ENTRIES (unsigned long)64
138 #endif
139 #define NKL3_MAX_ENTRIES (unsigned long)(NKL4_MAX_ENTRIES * 512)
140 #define NKL2_MAX_ENTRIES (unsigned long)(NKL3_MAX_ENTRIES * 512)
141 #define NKL1_MAX_ENTRIES (unsigned long)(NKL2_MAX_ENTRIES * 512)
142
143 #define NKL4_KIMG_ENTRIES 1
144 #define NKL3_KIMG_ENTRIES 1
145 #if defined(KUBSAN) || defined(KMSAN)
146 #define NKL2_KIMG_ENTRIES 64 /* really big kernel */
147 #else
148 #define NKL2_KIMG_ENTRIES 48
149 #endif
150
151 /*
152 * Since kva space is below the kernel in its entirety, we start off
153 * with zero entries on each level.
154 */
155 #define NKL4_START_ENTRIES 0
156 #define NKL3_START_ENTRIES 0
157 #define NKL2_START_ENTRIES 0
158 #define NKL1_START_ENTRIES 0
159
160 #define PTP_MASK_INITIALIZER { L1_MASK, L2_MASK, L3_MASK, L4_MASK }
161 #define PTP_FRAME_INITIALIZER { L1_FRAME, L2_FRAME, L3_FRAME, L4_FRAME }
162 #define PTP_SHIFT_INITIALIZER { L1_SHIFT, L2_SHIFT, L3_SHIFT, L4_SHIFT }
163 #define NKPTP_INITIALIZER { NKL1_START_ENTRIES, NKL2_START_ENTRIES, \
164 NKL3_START_ENTRIES, NKL4_START_ENTRIES }
165 #define NKPTPMAX_INITIALIZER { NKL1_MAX_ENTRIES, NKL2_MAX_ENTRIES, \
166 NKL3_MAX_ENTRIES, NKL4_MAX_ENTRIES }
167 #define NBPD_INITIALIZER { NBPD_L1, NBPD_L2, NBPD_L3, NBPD_L4 }
168 #define PDES_INITIALIZER { L2_BASE, L3_BASE, L4_BASE }
169
170 #define PTP_LEVELS 4
171
172 /*
173 * PTE_AVL usage: we make use of the ignored bits of the PTE
174 */
175 #define PTE_WIRED PTE_AVL1 /* Wired Mapping */
176 #define PTE_PVLIST PTE_AVL2 /* Mapping has entry on pvlist */
177 #define PTE_X 0 /* Dummy */
178
179 /* XXX To be deleted. */
180 #define PG_W PTE_WIRED
181 #define PG_PVLIST PTE_PVLIST
182 #define PG_X PTE_X
183
184 void svs_pmap_sync(struct pmap *, int);
185 void svs_ldt_sync(struct pmap *);
186 void svs_lwp_switch(struct lwp *, struct lwp *);
187 void svs_pdir_switch(struct pmap *);
188 void svs_init(void);
189 extern bool svs_enabled;
190 extern bool svs_pcid;
191
192 #define _MACHINE_PMAP_PRIVATE_H_X86
193 #include <x86/pmap_private.h>
194 #undef _MACHINE_PMAP_PRIVATE_H_X86
195
196 #ifndef XENPV
197
198 #define pmap_pa2pte(a) (a)
199 #define pmap_pte2pa(a) ((a) & PTE_FRAME)
200 #define pmap_pte_set(p, n) do { *(p) = (n); } while (0)
201 #define pmap_pte_cas(p, o, n) atomic_cas_64((p), (o), (n))
202 #define pmap_pte_testset(p, n) \
203 atomic_swap_ulong((volatile unsigned long *)p, n)
204 #define pmap_pte_setbits(p, b) \
205 atomic_or_ulong((volatile unsigned long *)p, b)
206 #define pmap_pte_clearbits(p, b) \
207 atomic_and_ulong((volatile unsigned long *)p, ~(b))
208 #define pmap_pte_flush() /* nothing */
209
210 #else
211
212 extern kmutex_t pte_lock;
213
214 static __inline pt_entry_t
pmap_pa2pte(paddr_t pa)215 pmap_pa2pte(paddr_t pa)
216 {
217 return (pt_entry_t)xpmap_ptom_masked(pa);
218 }
219
220 static __inline paddr_t
pmap_pte2pa(pt_entry_t pte)221 pmap_pte2pa(pt_entry_t pte)
222 {
223 return xpmap_mtop_masked(pte & PTE_FRAME);
224 }
225
226 static __inline void
pmap_pte_set(pt_entry_t * pte,pt_entry_t npte)227 pmap_pte_set(pt_entry_t *pte, pt_entry_t npte)
228 {
229 int s = splvm();
230 xpq_queue_pte_update(xpmap_ptetomach(pte), npte);
231 splx(s);
232 }
233
234 static __inline pt_entry_t
pmap_pte_cas(volatile pt_entry_t * ptep,pt_entry_t o,pt_entry_t n)235 pmap_pte_cas(volatile pt_entry_t *ptep, pt_entry_t o, pt_entry_t n)
236 {
237 pt_entry_t opte;
238
239 mutex_enter(&pte_lock);
240 opte = *ptep;
241 if (opte == o) {
242 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(ptep)), n);
243 xpq_flush_queue();
244 }
245 mutex_exit(&pte_lock);
246 return opte;
247 }
248
249 static __inline pt_entry_t
pmap_pte_testset(volatile pt_entry_t * pte,pt_entry_t npte)250 pmap_pte_testset(volatile pt_entry_t *pte, pt_entry_t npte)
251 {
252 pt_entry_t opte;
253
254 mutex_enter(&pte_lock);
255 opte = *pte;
256 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), npte);
257 xpq_flush_queue();
258 mutex_exit(&pte_lock);
259 return opte;
260 }
261
262 static __inline void
pmap_pte_setbits(volatile pt_entry_t * pte,pt_entry_t bits)263 pmap_pte_setbits(volatile pt_entry_t *pte, pt_entry_t bits)
264 {
265 mutex_enter(&pte_lock);
266 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), (*pte) | bits);
267 xpq_flush_queue();
268 mutex_exit(&pte_lock);
269 }
270
271 static __inline void
pmap_pte_clearbits(volatile pt_entry_t * pte,pt_entry_t bits)272 pmap_pte_clearbits(volatile pt_entry_t *pte, pt_entry_t bits)
273 {
274 mutex_enter(&pte_lock);
275 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)),
276 (*pte) & ~bits);
277 xpq_flush_queue();
278 mutex_exit(&pte_lock);
279 }
280
281 static __inline void
pmap_pte_flush(void)282 pmap_pte_flush(void)
283 {
284 int s = splvm();
285 xpq_flush_queue();
286 splx(s);
287 }
288
289 #endif
290
291 #ifdef __HAVE_DIRECT_MAP
292 #define PMAP_DIRECT
293
294 static __inline int
pmap_direct_process(paddr_t pa,voff_t pgoff,size_t len,int (* process)(void *,size_t,void *),void * arg)295 pmap_direct_process(paddr_t pa, voff_t pgoff, size_t len,
296 int (*process)(void *, size_t, void *), void *arg)
297 {
298 vaddr_t va = PMAP_DIRECT_MAP(pa);
299
300 return process((void *)(va + pgoff), len, arg);
301 }
302
303 #endif /* __HAVE_DIRECT_MAP */
304
305 void pmap_changeprot_local(vaddr_t, vm_prot_t);
306
307 #else /* !__x86_64__ */
308
309 #include <i386/pmap_private.h>
310
311 #endif /* __x86_64__ */
312
313 #endif /* _AMD64_PMAP_PRIVATE_H_ */
314