1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
5 *
6 * This software was developed by Alexandru Elisei under sponsorship
7 * from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32
33 #include <sys/types.h>
34 #include <sys/malloc.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <vm/vm.h>
38 #include <vm/pmap.h>
39 #include <vm/vm_page.h>
40 #include <vm/vm_param.h>
41 #include <vm/vm_phys.h>
42
43 #include <machine/atomic.h>
44 #include <machine/machdep.h>
45 #include <machine/vm.h>
46 #include <machine/vmm.h>
47 #include <machine/vmparam.h>
48
49 #include "mmu.h"
50 #include "arm64.h"
51
52 static struct mtx vmmpmap_mtx;
53 static pt_entry_t *l0;
54 static vm_paddr_t l0_paddr;
55
56 bool
vmmpmap_init(void)57 vmmpmap_init(void)
58 {
59 vm_page_t m;
60
61 m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
62 if (m == NULL)
63 return (false);
64
65 l0_paddr = VM_PAGE_TO_PHYS(m);
66 l0 = (pd_entry_t *)PHYS_TO_DMAP(l0_paddr);
67
68 mtx_init(&vmmpmap_mtx, "vmm pmap", NULL, MTX_DEF);
69
70 return (true);
71 }
72
73 static void
vmmpmap_release_l3(pd_entry_t l2e)74 vmmpmap_release_l3(pd_entry_t l2e)
75 {
76 pt_entry_t *l3 __diagused;
77 vm_page_t m;
78 int i;
79
80 l3 = (pd_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
81 for (i = 0; i < Ln_ENTRIES; i++) {
82 KASSERT(l3[i] == 0, ("%s: l3 still mapped: %p %lx", __func__,
83 &l3[i], l3[i]));
84 }
85
86 m = PHYS_TO_VM_PAGE(l2e & ~ATTR_MASK);
87 vm_page_unwire_noq(m);
88 vm_page_free(m);
89 }
90
91 static void
vmmpmap_release_l2(pd_entry_t l1e)92 vmmpmap_release_l2(pd_entry_t l1e)
93 {
94 pt_entry_t *l2;
95 vm_page_t m;
96 int i;
97
98 l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
99 for (i = 0; i < Ln_ENTRIES; i++) {
100 if (l2[i] != 0) {
101 vmmpmap_release_l3(l2[i]);
102 }
103 }
104
105 m = PHYS_TO_VM_PAGE(l1e & ~ATTR_MASK);
106 vm_page_unwire_noq(m);
107 vm_page_free(m);
108 }
109
110 static void
vmmpmap_release_l1(pd_entry_t l0e)111 vmmpmap_release_l1(pd_entry_t l0e)
112 {
113 pt_entry_t *l1;
114 vm_page_t m;
115 int i;
116
117 l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
118 for (i = 0; i < Ln_ENTRIES; i++) {
119 if (l1[i] != 0) {
120 vmmpmap_release_l2(l1[i]);
121 }
122 }
123
124 m = PHYS_TO_VM_PAGE(l0e & ~ATTR_MASK);
125 vm_page_unwire_noq(m);
126 vm_page_free(m);
127 }
128
129 void
vmmpmap_fini(void)130 vmmpmap_fini(void)
131 {
132 vm_page_t m;
133 int i;
134
135 /* Remove the remaining entries */
136 for (i = 0; i < L0_ENTRIES; i++) {
137 if (l0[i] != 0) {
138 vmmpmap_release_l1(l0[i]);
139 }
140 }
141
142 m = PHYS_TO_VM_PAGE(l0_paddr);
143 vm_page_unwire_noq(m);
144 vm_page_free(m);
145
146 mtx_destroy(&vmmpmap_mtx);
147 }
148
149 uint64_t
vmmpmap_to_ttbr0(void)150 vmmpmap_to_ttbr0(void)
151 {
152
153 return (l0_paddr);
154 }
155
156 /* Returns a pointer to the level 1 table, allocating if needed. */
157 static pt_entry_t *
vmmpmap_l1_table(vm_offset_t va)158 vmmpmap_l1_table(vm_offset_t va)
159 {
160 pt_entry_t new_l0e, l0e, *l1;
161 vm_page_t m;
162 int rv;
163
164 m = NULL;
165 again:
166 l0e = atomic_load_64(&l0[pmap_l0_index(va)]);
167 if ((l0e & ATTR_DESCR_VALID) == 0) {
168 /* Allocate a page for the level 1 table */
169 if (m == NULL) {
170 m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
171 if (m == NULL)
172 return (NULL);
173 }
174
175 new_l0e = VM_PAGE_TO_PHYS(m) | L0_TABLE;
176
177 mtx_lock(&vmmpmap_mtx);
178 rv = atomic_cmpset_64(&l0[pmap_l0_index(va)], l0e, new_l0e);
179 mtx_unlock(&vmmpmap_mtx);
180 /* We may have raced another thread, try again */
181 if (rv == 0)
182 goto again;
183
184 /* The cmpset succeeded */
185 l0e = new_l0e;
186 } else if (m != NULL) {
187 /* We allocated a page that wasn't used */
188 vm_page_unwire_noq(m);
189 vm_page_free_zero(m);
190 }
191
192 l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
193 return (l1);
194 }
195
196 static pt_entry_t *
vmmpmap_l2_table(vm_offset_t va)197 vmmpmap_l2_table(vm_offset_t va)
198 {
199 pt_entry_t new_l1e, l1e, *l1, *l2;
200 vm_page_t m;
201 int rv;
202
203 l1 = vmmpmap_l1_table(va);
204 if (l1 == NULL)
205 return (NULL);
206
207 m = NULL;
208 again:
209 l1e = atomic_load_64(&l1[pmap_l1_index(va)]);
210 if ((l1e & ATTR_DESCR_VALID) == 0) {
211 /* Allocate a page for the level 2 table */
212 if (m == NULL) {
213 m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
214 if (m == NULL)
215 return (NULL);
216 }
217
218 new_l1e = VM_PAGE_TO_PHYS(m) | L1_TABLE;
219
220 mtx_lock(&vmmpmap_mtx);
221 rv = atomic_cmpset_64(&l1[pmap_l1_index(va)], l1e, new_l1e);
222 mtx_unlock(&vmmpmap_mtx);
223 /* We may have raced another thread, try again */
224 if (rv == 0)
225 goto again;
226
227 /* The cmpset succeeded */
228 l1e = new_l1e;
229 } else if (m != NULL) {
230 /* We allocated a page that wasn't used */
231 vm_page_unwire_noq(m);
232 vm_page_free_zero(m);
233 }
234
235 l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
236 return (l2);
237 }
238
239 static pd_entry_t *
vmmpmap_l3_table(vm_offset_t va)240 vmmpmap_l3_table(vm_offset_t va)
241 {
242 pt_entry_t new_l2e, l2e, *l2, *l3;
243 vm_page_t m;
244 int rv;
245
246 l2 = vmmpmap_l2_table(va);
247 if (l2 == NULL)
248 return (NULL);
249
250 m = NULL;
251 again:
252 l2e = atomic_load_64(&l2[pmap_l2_index(va)]);
253 if ((l2e & ATTR_DESCR_VALID) == 0) {
254 /* Allocate a page for the level 3 table */
255 if (m == NULL) {
256 m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
257 if (m == NULL)
258 return (NULL);
259 }
260
261 new_l2e = VM_PAGE_TO_PHYS(m) | L2_TABLE;
262
263 mtx_lock(&vmmpmap_mtx);
264 rv = atomic_cmpset_64(&l2[pmap_l2_index(va)], l2e, new_l2e);
265 mtx_unlock(&vmmpmap_mtx);
266 /* We may have raced another thread, try again */
267 if (rv == 0)
268 goto again;
269
270 /* The cmpset succeeded */
271 l2e = new_l2e;
272 } else if (m != NULL) {
273 /* We allocated a page that wasn't used */
274 vm_page_unwire_noq(m);
275 vm_page_free_zero(m);
276 }
277
278 l3 = (pt_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
279 return (l3);
280 }
281
282 /*
283 * Creates an EL2 entry in the hyp_pmap. Similar to pmap_kenter.
284 */
285 bool
vmmpmap_enter(vm_offset_t va,vm_size_t size,vm_paddr_t pa,vm_prot_t prot)286 vmmpmap_enter(vm_offset_t va, vm_size_t size, vm_paddr_t pa, vm_prot_t prot)
287 {
288 pd_entry_t l3e, *l3;
289
290 KASSERT((pa & L3_OFFSET) == 0,
291 ("%s: Invalid physical address", __func__));
292 KASSERT((va & L3_OFFSET) == 0,
293 ("%s: Invalid virtual address", __func__));
294 KASSERT((size & PAGE_MASK) == 0,
295 ("%s: Mapping is not page-sized", __func__));
296
297 l3e = ATTR_DEFAULT | L3_PAGE;
298 /* This bit is res1 at EL2 */
299 l3e |= ATTR_S1_AP(ATTR_S1_AP_USER);
300 /* Only normal memory is used at EL2 */
301 l3e |= ATTR_S1_IDX(VM_MEMATTR_DEFAULT);
302
303 if ((prot & VM_PROT_EXECUTE) == 0) {
304 /* PXN is res0 at EL2. UXN is XN */
305 l3e |= ATTR_S1_UXN;
306 }
307 if ((prot & VM_PROT_WRITE) == 0) {
308 l3e |= ATTR_S1_AP(ATTR_S1_AP_RO);
309 }
310
311 while (size > 0) {
312 l3 = vmmpmap_l3_table(va);
313 if (l3 == NULL)
314 return (false);
315
316 #ifdef INVARIANTS
317 /*
318 * Ensure no other threads can write to l3 between the KASSERT
319 * and store.
320 */
321 mtx_lock(&vmmpmap_mtx);
322 #endif
323 KASSERT(atomic_load_64(&l3[pmap_l3_index(va)]) == 0,
324 ("%s: VA already mapped", __func__));
325
326 atomic_store_64(&l3[pmap_l3_index(va)], l3e | pa);
327 #ifdef INVARIANTS
328 mtx_unlock(&vmmpmap_mtx);
329 #endif
330
331 size -= PAGE_SIZE;
332 pa += PAGE_SIZE;
333 va += PAGE_SIZE;
334 }
335
336 return (true);
337 }
338
339 void
vmmpmap_remove(vm_offset_t va,vm_size_t size,bool invalidate)340 vmmpmap_remove(vm_offset_t va, vm_size_t size, bool invalidate)
341 {
342 pt_entry_t l0e, *l1, l1e, *l2, l2e;
343 pd_entry_t *l3, l3e, **l3_list;
344 vm_offset_t eva, va_next, sva;
345 size_t i;
346
347 KASSERT((va & L3_OFFSET) == 0,
348 ("%s: Invalid virtual address", __func__));
349 KASSERT((size & PAGE_MASK) == 0,
350 ("%s: Mapping is not page-sized", __func__));
351
352 if (invalidate) {
353 l3_list = malloc((size / PAGE_SIZE) * sizeof(l3_list[0]),
354 M_TEMP, M_WAITOK | M_ZERO);
355 }
356
357 sva = va;
358 eva = va + size;
359 mtx_lock(&vmmpmap_mtx);
360 for (i = 0; va < eva; va = va_next) {
361 l0e = atomic_load_64(&l0[pmap_l0_index(va)]);
362 if (l0e == 0) {
363 va_next = (va + L0_SIZE) & ~L0_OFFSET;
364 if (va_next < va)
365 va_next = eva;
366 continue;
367 }
368 MPASS((l0e & ATTR_DESCR_MASK) == L0_TABLE);
369
370 l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
371 l1e = atomic_load_64(&l1[pmap_l1_index(va)]);
372 if (l1e == 0) {
373 va_next = (va + L1_SIZE) & ~L1_OFFSET;
374 if (va_next < va)
375 va_next = eva;
376 continue;
377 }
378 MPASS((l1e & ATTR_DESCR_MASK) == L1_TABLE);
379
380 l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
381 l2e = atomic_load_64(&l2[pmap_l2_index(va)]);
382 if (l2e == 0) {
383 va_next = (va + L2_SIZE) & ~L2_OFFSET;
384 if (va_next < va)
385 va_next = eva;
386 continue;
387 }
388 MPASS((l2e & ATTR_DESCR_MASK) == L2_TABLE);
389
390 l3 = (pd_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
391 if (invalidate) {
392 l3e = atomic_load_64(&l3[pmap_l3_index(va)]);
393 MPASS(l3e != 0);
394 /*
395 * Mark memory as read-only so we can invalidate
396 * the cache.
397 */
398 l3e &= ~ATTR_S1_AP_MASK;
399 l3e |= ATTR_S1_AP(ATTR_S1_AP_RO);
400 atomic_store_64(&l3[pmap_l3_index(va)], l3e);
401
402 l3_list[i] = &l3[pmap_l3_index(va)];
403 i++;
404 } else {
405 /*
406 * The caller is responsible for clearing the cache &
407 * handling the TLB
408 */
409 atomic_store_64(&l3[pmap_l3_index(va)], 0);
410 }
411
412 va_next = (va + L3_SIZE) & ~L3_OFFSET;
413 if (va_next < va)
414 va_next = eva;
415 }
416 mtx_unlock(&vmmpmap_mtx);
417
418 if (invalidate) {
419 /* Invalidate the memory from the D-cache */
420 vmm_call_hyp(HYP_DC_CIVAC, sva, size);
421
422 for (i = 0; i < (size / PAGE_SIZE); i++) {
423 atomic_store_64(l3_list[i], 0);
424 }
425
426 vmm_call_hyp(HYP_EL2_TLBI, HYP_EL2_TLBI_VA, sva, size);
427
428 free(l3_list, M_TEMP);
429 }
430 }
431