1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2020 Justin Hibbits
5 * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
6 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * Some hw specific parts of this pmap were derived or influenced
30 * by NetBSD's ibm4xx pmap module. More generic code is shared with
31 * a few other pmap modules from the FreeBSD tree.
32 */
33
34 /*
35 * VM layout notes:
36 *
37 * Kernel and user threads run within one common virtual address space
38 * defined by AS=0.
39 *
40 * 64-bit pmap:
41 * Virtual address space layout:
42 * -----------------------------
43 * 0x0000_0000_0000_0000 - 0x3fff_ffff_ffff_ffff : user process
44 * 0x4000_0000_0000_0000 - 0x7fff_ffff_ffff_ffff : unused
45 * 0x8000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : mmio region
46 * 0xc000_0000_0000_0000 - 0xdfff_ffff_ffff_ffff : direct map
47 * 0xe000_0000_0000_0000 - 0xffff_ffff_ffff_ffff : KVA
48 */
49
50 #include <sys/cdefs.h>
51 #include "opt_ddb.h"
52 #include "opt_kstack_pages.h"
53
54 #include <sys/param.h>
55 #include <sys/conf.h>
56 #include <sys/malloc.h>
57 #include <sys/ktr.h>
58 #include <sys/proc.h>
59 #include <sys/user.h>
60 #include <sys/queue.h>
61 #include <sys/systm.h>
62 #include <sys/kernel.h>
63 #include <sys/kerneldump.h>
64 #include <sys/linker.h>
65 #include <sys/msgbuf.h>
66 #include <sys/lock.h>
67 #include <sys/mutex.h>
68 #include <sys/rwlock.h>
69 #include <sys/sched.h>
70 #include <sys/smp.h>
71 #include <sys/vmmeter.h>
72
73 #include <vm/vm.h>
74 #include <vm/vm_page.h>
75 #include <vm/vm_kern.h>
76 #include <vm/vm_pageout.h>
77 #include <vm/vm_extern.h>
78 #include <vm/vm_object.h>
79 #include <vm/vm_param.h>
80 #include <vm/vm_map.h>
81 #include <vm/vm_pager.h>
82 #include <vm/vm_phys.h>
83 #include <vm/vm_pagequeue.h>
84 #include <vm/uma.h>
85
86 #include <machine/_inttypes.h>
87 #include <machine/cpu.h>
88 #include <machine/pcb.h>
89 #include <machine/platform.h>
90
91 #include <machine/tlb.h>
92 #include <machine/spr.h>
93 #include <machine/md_var.h>
94 #include <machine/mmuvar.h>
95 #include <machine/pmap.h>
96 #include <machine/pte.h>
97
98 #include <ddb/ddb.h>
99
100 #ifdef DEBUG
101 #define debugf(fmt, args...) printf(fmt, ##args)
102 #else
103 #define debugf(fmt, args...)
104 #endif
105
106 #define PRI0ptrX "016lx"
107
108 /**************************************************************************/
109 /* PMAP */
110 /**************************************************************************/
111
112 unsigned int kernel_pdirs;
113 static uma_zone_t ptbl_root_zone;
114 static pte_t ****kernel_ptbl_root;
115
116 /*
117 * Base of the pmap_mapdev() region. On 32-bit it immediately follows the
118 * userspace address range. On On 64-bit it's far above, at (1 << 63), and
119 * ranges up to the DMAP, giving 62 bits of PA allowed. This is far larger than
120 * the widest Book-E address bus, the e6500 has a 40-bit PA space. This allows
121 * us to map akin to the DMAP, with addresses identical to the PA, offset by the
122 * base.
123 */
124 #define VM_MAPDEV_BASE 0x8000000000000000
125 #define VM_MAPDEV_PA_MAX 0x4000000000000000 /* Don't encroach on DMAP */
126
127 static void tid_flush(tlbtid_t tid);
128
129 /**************************************************************************/
130 /* Page table management */
131 /**************************************************************************/
132
133 #define PMAP_ROOT_SIZE (sizeof(pte_t****) * PG_ROOT_NENTRIES)
134 static pte_t *ptbl_alloc(pmap_t pmap, vm_offset_t va,
135 bool nosleep, bool *is_new);
136 static void ptbl_hold(pmap_t, pte_t *);
137 static int ptbl_unhold(pmap_t, vm_offset_t);
138
139 static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t);
140 static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, bool);
141 static int pte_remove(pmap_t, vm_offset_t, uint8_t);
142 static pte_t *pte_find(pmap_t, vm_offset_t);
143 static pte_t *pte_find_next(pmap_t, vm_offset_t *);
144 static void kernel_pte_alloc(vm_offset_t, vm_offset_t);
145
146 /**************************************************************************/
147 /* Page table related */
148 /**************************************************************************/
149
150 /* Allocate a page, to be used in a page table. */
151 static vm_offset_t
mmu_booke_alloc_page(pmap_t pmap,unsigned int idx,bool nosleep)152 mmu_booke_alloc_page(pmap_t pmap, unsigned int idx, bool nosleep)
153 {
154 vm_page_t m;
155 int req;
156
157 req = VM_ALLOC_WIRED | VM_ALLOC_ZERO;
158 while ((m = vm_page_alloc_noobj(req)) == NULL) {
159 if (nosleep)
160 return (0);
161
162 PMAP_UNLOCK(pmap);
163 rw_wunlock(&pvh_global_lock);
164 vm_wait(NULL);
165 rw_wlock(&pvh_global_lock);
166 PMAP_LOCK(pmap);
167 }
168 m->pindex = idx;
169
170 return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
171 }
172
173 /* Initialize pool of kva ptbl buffers. */
174 static void
ptbl_init(void)175 ptbl_init(void)
176 {
177 }
178
179 /* Get a pointer to a PTE in a page table. */
180 static __inline pte_t *
pte_find(pmap_t pmap,vm_offset_t va)181 pte_find(pmap_t pmap, vm_offset_t va)
182 {
183 pte_t ***pdir_l1;
184 pte_t **pdir;
185 pte_t *ptbl;
186
187 KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
188
189 pdir_l1 = pmap->pm_root[PG_ROOT_IDX(va)];
190 if (pdir_l1 == NULL)
191 return (NULL);
192 pdir = pdir_l1[PDIR_L1_IDX(va)];
193 if (pdir == NULL)
194 return (NULL);
195 ptbl = pdir[PDIR_IDX(va)];
196
197 return ((ptbl != NULL) ? &ptbl[PTBL_IDX(va)] : NULL);
198 }
199
200 /* Get a pointer to a PTE in a page table, or the next closest (greater) one. */
201 static __inline pte_t *
pte_find_next(pmap_t pmap,vm_offset_t * pva)202 pte_find_next(pmap_t pmap, vm_offset_t *pva)
203 {
204 vm_offset_t va;
205 pte_t ****pm_root;
206 pte_t *pte;
207 unsigned long i, j, k, l;
208
209 KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
210
211 va = *pva;
212 i = PG_ROOT_IDX(va);
213 j = PDIR_L1_IDX(va);
214 k = PDIR_IDX(va);
215 l = PTBL_IDX(va);
216 pm_root = pmap->pm_root;
217
218 /* truncate the VA for later. */
219 va &= ~((1UL << (PG_ROOT_H + 1)) - 1);
220 for (; i < PG_ROOT_NENTRIES; i++, j = 0, k = 0, l = 0) {
221 if (pm_root[i] == 0)
222 continue;
223 for (; j < PDIR_L1_NENTRIES; j++, k = 0, l = 0) {
224 if (pm_root[i][j] == 0)
225 continue;
226 for (; k < PDIR_NENTRIES; k++, l = 0) {
227 if (pm_root[i][j][k] == NULL)
228 continue;
229 for (; l < PTBL_NENTRIES; l++) {
230 pte = &pm_root[i][j][k][l];
231 if (!PTE_ISVALID(pte))
232 continue;
233 *pva = va + PG_ROOT_SIZE * i +
234 PDIR_L1_SIZE * j +
235 PDIR_SIZE * k +
236 PAGE_SIZE * l;
237 return (pte);
238 }
239 }
240 }
241 }
242 return (NULL);
243 }
244
245 static bool
unhold_free_page(pmap_t pmap,vm_page_t m)246 unhold_free_page(pmap_t pmap, vm_page_t m)
247 {
248
249 if (vm_page_unwire_noq(m)) {
250 vm_page_free_zero(m);
251 return (true);
252 }
253
254 return (false);
255 }
256
257 static vm_offset_t
get_pgtbl_page(pmap_t pmap,vm_offset_t * ptr_tbl,uint32_t index,bool nosleep,bool hold_parent,bool * isnew)258 get_pgtbl_page(pmap_t pmap, vm_offset_t *ptr_tbl, uint32_t index,
259 bool nosleep, bool hold_parent, bool *isnew)
260 {
261 vm_offset_t page;
262 vm_page_t m;
263
264 page = ptr_tbl[index];
265 KASSERT(page != 0 || pmap != kernel_pmap,
266 ("NULL page table page found in kernel pmap!"));
267 if (page == 0) {
268 page = mmu_booke_alloc_page(pmap, index, nosleep);
269 if (ptr_tbl[index] == 0) {
270 *isnew = true;
271 ptr_tbl[index] = page;
272 if (hold_parent) {
273 m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)ptr_tbl));
274 m->ref_count++;
275 }
276 return (page);
277 }
278 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS(page));
279 page = ptr_tbl[index];
280 vm_page_unwire_noq(m);
281 vm_page_free_zero(m);
282 }
283
284 *isnew = false;
285
286 return (page);
287 }
288
289 /* Allocate page table. */
290 static pte_t*
ptbl_alloc(pmap_t pmap,vm_offset_t va,bool nosleep,bool * is_new)291 ptbl_alloc(pmap_t pmap, vm_offset_t va, bool nosleep, bool *is_new)
292 {
293 unsigned int pg_root_idx = PG_ROOT_IDX(va);
294 unsigned int pdir_l1_idx = PDIR_L1_IDX(va);
295 unsigned int pdir_idx = PDIR_IDX(va);
296 vm_offset_t pdir_l1, pdir, ptbl;
297
298 /* When holding a parent, no need to hold the root index pages. */
299 pdir_l1 = get_pgtbl_page(pmap, (vm_offset_t *)pmap->pm_root,
300 pg_root_idx, nosleep, false, is_new);
301 if (pdir_l1 == 0)
302 return (NULL);
303 pdir = get_pgtbl_page(pmap, (vm_offset_t *)pdir_l1, pdir_l1_idx,
304 nosleep, !*is_new, is_new);
305 if (pdir == 0)
306 return (NULL);
307 ptbl = get_pgtbl_page(pmap, (vm_offset_t *)pdir, pdir_idx,
308 nosleep, !*is_new, is_new);
309
310 return ((pte_t *)ptbl);
311 }
312
313 /*
314 * Decrement ptbl pages hold count and attempt to free ptbl pages. Called
315 * when removing pte entry from ptbl.
316 *
317 * Return 1 if ptbl pages were freed.
318 */
319 static int
ptbl_unhold(pmap_t pmap,vm_offset_t va)320 ptbl_unhold(pmap_t pmap, vm_offset_t va)
321 {
322 pte_t *ptbl;
323 vm_page_t m;
324 u_int pg_root_idx;
325 pte_t ***pdir_l1;
326 u_int pdir_l1_idx;
327 pte_t **pdir;
328 u_int pdir_idx;
329
330 pg_root_idx = PG_ROOT_IDX(va);
331 pdir_l1_idx = PDIR_L1_IDX(va);
332 pdir_idx = PDIR_IDX(va);
333
334 KASSERT((pmap != kernel_pmap),
335 ("ptbl_unhold: unholding kernel ptbl!"));
336
337 pdir_l1 = pmap->pm_root[pg_root_idx];
338 pdir = pdir_l1[pdir_l1_idx];
339 ptbl = pdir[pdir_idx];
340
341 /* decrement hold count */
342 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl));
343
344 if (!unhold_free_page(pmap, m))
345 return (0);
346
347 pdir[pdir_idx] = NULL;
348 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir));
349
350 if (!unhold_free_page(pmap, m))
351 return (1);
352
353 pdir_l1[pdir_l1_idx] = NULL;
354 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir_l1));
355
356 if (!unhold_free_page(pmap, m))
357 return (1);
358 pmap->pm_root[pg_root_idx] = NULL;
359
360 return (1);
361 }
362
363 /*
364 * Increment hold count for ptbl pages. This routine is used when new pte
365 * entry is being inserted into ptbl.
366 */
367 static void
ptbl_hold(pmap_t pmap,pte_t * ptbl)368 ptbl_hold(pmap_t pmap, pte_t *ptbl)
369 {
370 vm_page_t m;
371
372 KASSERT((pmap != kernel_pmap),
373 ("ptbl_hold: holding kernel ptbl!"));
374
375 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl));
376 m->ref_count++;
377 }
378
379 /*
380 * Clean pte entry, try to free page table page if requested.
381 *
382 * Return 1 if ptbl pages were freed, otherwise return 0.
383 */
384 static int
pte_remove(pmap_t pmap,vm_offset_t va,u_int8_t flags)385 pte_remove(pmap_t pmap, vm_offset_t va, u_int8_t flags)
386 {
387 vm_page_t m;
388 pte_t *pte;
389
390 pte = pte_find(pmap, va);
391 KASSERT(pte != NULL, ("%s: NULL pte for va %#jx, pmap %p",
392 __func__, (uintmax_t)va, pmap));
393
394 if (!PTE_ISVALID(pte))
395 return (0);
396
397 /* Get vm_page_t for mapped pte. */
398 m = PHYS_TO_VM_PAGE(PTE_PA(pte));
399
400 if (PTE_ISWIRED(pte))
401 pmap->pm_stats.wired_count--;
402
403 /* Handle managed entry. */
404 if (PTE_ISMANAGED(pte)) {
405 /* Handle modified pages. */
406 if (PTE_ISMODIFIED(pte))
407 vm_page_dirty(m);
408
409 /* Referenced pages. */
410 if (PTE_ISREFERENCED(pte))
411 vm_page_aflag_set(m, PGA_REFERENCED);
412
413 /* Remove pv_entry from pv_list. */
414 pv_remove(pmap, va, m);
415 } else if (pmap == kernel_pmap && m && m->md.pv_tracked) {
416 pv_remove(pmap, va, m);
417 if (TAILQ_EMPTY(&m->md.pv_list))
418 m->md.pv_tracked = false;
419 }
420 mtx_lock_spin(&tlbivax_mutex);
421 tlb_miss_lock();
422
423 tlb0_flush_entry(va);
424 *pte = 0;
425
426 tlb_miss_unlock();
427 mtx_unlock_spin(&tlbivax_mutex);
428
429 pmap->pm_stats.resident_count--;
430
431 if (flags & PTBL_UNHOLD) {
432 return (ptbl_unhold(pmap, va));
433 }
434 return (0);
435 }
436
437 /*
438 * Insert PTE for a given page and virtual address.
439 */
440 static int
pte_enter(pmap_t pmap,vm_page_t m,vm_offset_t va,uint32_t flags,bool nosleep)441 pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,
442 bool nosleep)
443 {
444 unsigned int ptbl_idx = PTBL_IDX(va);
445 pte_t *ptbl, *pte, pte_tmp;
446 bool is_new;
447
448 /* Get the page directory pointer. */
449 ptbl = ptbl_alloc(pmap, va, nosleep, &is_new);
450 if (ptbl == NULL) {
451 KASSERT(nosleep, ("nosleep and NULL ptbl"));
452 return (ENOMEM);
453 }
454 if (is_new) {
455 pte = &ptbl[ptbl_idx];
456 } else {
457 /*
458 * Check if there is valid mapping for requested va, if there
459 * is, remove it.
460 */
461 pte = &ptbl[ptbl_idx];
462 if (PTE_ISVALID(pte)) {
463 pte_remove(pmap, va, PTBL_HOLD);
464 } else {
465 /*
466 * pte is not used, increment hold count for ptbl
467 * pages.
468 */
469 if (pmap != kernel_pmap)
470 ptbl_hold(pmap, ptbl);
471 }
472 }
473
474 /*
475 * Insert pv_entry into pv_list for mapped page if part of managed
476 * memory.
477 */
478 if ((m->oflags & VPO_UNMANAGED) == 0) {
479 flags |= PTE_MANAGED;
480
481 /* Create and insert pv entry. */
482 pv_insert(pmap, va, m);
483 }
484
485 pmap->pm_stats.resident_count++;
486
487 pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
488 pte_tmp |= (PTE_VALID | flags);
489
490 mtx_lock_spin(&tlbivax_mutex);
491 tlb_miss_lock();
492
493 tlb0_flush_entry(va);
494 *pte = pte_tmp;
495
496 tlb_miss_unlock();
497 mtx_unlock_spin(&tlbivax_mutex);
498
499 return (0);
500 }
501
502 /* Return the pa for the given pmap/va. */
503 static vm_paddr_t
pte_vatopa(pmap_t pmap,vm_offset_t va)504 pte_vatopa(pmap_t pmap, vm_offset_t va)
505 {
506 vm_paddr_t pa = 0;
507 pte_t *pte;
508
509 pte = pte_find(pmap, va);
510 if ((pte != NULL) && PTE_ISVALID(pte))
511 pa = (PTE_PA(pte) | (va & PTE_PA_MASK));
512 return (pa);
513 }
514
515 /* allocate pte entries to manage (addr & mask) to (addr & mask) + size */
516 static void
kernel_pte_alloc(vm_offset_t data_end,vm_offset_t addr)517 kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
518 {
519 pte_t *pte;
520 vm_size_t kva_size;
521 int kernel_pdirs, kernel_pgtbls, pdir_l1s;
522 vm_offset_t va, l1_va, pdir_va, ptbl_va;
523 int i, j, k;
524
525 kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
526 kernel_pmap->pm_root = kernel_ptbl_root;
527 pdir_l1s = howmany(kva_size, PG_ROOT_SIZE);
528 kernel_pdirs = howmany(kva_size, PDIR_L1_SIZE);
529 kernel_pgtbls = howmany(kva_size, PDIR_SIZE);
530
531 /* Initialize kernel pdir */
532 l1_va = (vm_offset_t)kernel_ptbl_root +
533 round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***));
534 pdir_va = l1_va + pdir_l1s * PAGE_SIZE;
535 ptbl_va = pdir_va + kernel_pdirs * PAGE_SIZE;
536 if (bootverbose) {
537 printf("ptbl_root_va: %#lx\n", (vm_offset_t)kernel_ptbl_root);
538 printf("l1_va: %#lx (%d entries)\n", l1_va, pdir_l1s);
539 printf("pdir_va: %#lx(%d entries)\n", pdir_va, kernel_pdirs);
540 printf("ptbl_va: %#lx(%d entries)\n", ptbl_va, kernel_pgtbls);
541 }
542
543 va = VM_MIN_KERNEL_ADDRESS;
544 for (i = PG_ROOT_IDX(va); i < PG_ROOT_IDX(va) + pdir_l1s;
545 i++, l1_va += PAGE_SIZE) {
546 kernel_pmap->pm_root[i] = (pte_t ***)l1_va;
547 for (j = 0;
548 j < PDIR_L1_NENTRIES && va < VM_MAX_KERNEL_ADDRESS;
549 j++, pdir_va += PAGE_SIZE) {
550 kernel_pmap->pm_root[i][j] = (pte_t **)pdir_va;
551 for (k = 0;
552 k < PDIR_NENTRIES && va < VM_MAX_KERNEL_ADDRESS;
553 k++, va += PDIR_SIZE, ptbl_va += PAGE_SIZE)
554 kernel_pmap->pm_root[i][j][k] = (pte_t *)ptbl_va;
555 }
556 }
557 /*
558 * Fill in PTEs covering kernel code and data. They are not required
559 * for address translation, as this area is covered by static TLB1
560 * entries, but for pte_vatopa() to work correctly with kernel area
561 * addresses.
562 */
563 for (va = addr; va < data_end; va += PAGE_SIZE) {
564 pte = &(kernel_pmap->pm_root[PG_ROOT_IDX(va)][PDIR_L1_IDX(va)][PDIR_IDX(va)][PTBL_IDX(va)]);
565 *pte = PTE_RPN_FROM_PA(kernload + (va - kernstart));
566 *pte |= PTE_M | PTE_SR | PTE_SW | PTE_SX | PTE_WIRED |
567 PTE_VALID | PTE_PS_4KB;
568 }
569 }
570
571 static vm_offset_t
mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)572 mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)
573 {
574 vm_size_t kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
575 kernel_ptbl_root = (pte_t ****)data_end;
576
577 data_end += round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***));
578 data_end += howmany(kva_size, PG_ROOT_SIZE) * PAGE_SIZE;
579 data_end += howmany(kva_size, PDIR_L1_SIZE) * PAGE_SIZE;
580 data_end += howmany(kva_size, PDIR_SIZE) * PAGE_SIZE;
581
582 return (data_end);
583 }
584
585 /*
586 * Initialize a preallocated and zeroed pmap structure,
587 * such as one in a vmspace structure.
588 */
589 static int
mmu_booke_pinit(pmap_t pmap)590 mmu_booke_pinit(pmap_t pmap)
591 {
592 int i;
593
594 CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,
595 curthread->td_proc->p_pid, curthread->td_proc->p_comm);
596
597 KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));
598
599 for (i = 0; i < MAXCPU; i++)
600 pmap->pm_tid[i] = TID_NONE;
601 CPU_ZERO(&kernel_pmap->pm_active);
602 bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
603 pmap->pm_root = uma_zalloc(ptbl_root_zone, M_WAITOK);
604 bzero(pmap->pm_root, sizeof(pte_t **) * PG_ROOT_NENTRIES);
605
606 return (1);
607 }
608
609 /*
610 * Release any resources held by the given physical map.
611 * Called when a pmap initialized by mmu_booke_pinit is being released.
612 * Should only be called if the map contains no valid mappings.
613 */
614 static void
mmu_booke_release(pmap_t pmap)615 mmu_booke_release(pmap_t pmap)
616 {
617
618 KASSERT(pmap->pm_stats.resident_count == 0,
619 ("pmap_release: pmap resident count %ld != 0",
620 pmap->pm_stats.resident_count));
621 #ifdef INVARIANTS
622 /*
623 * Verify that all page directories are gone.
624 * Protects against reference count leakage.
625 */
626 for (int i = 0; i < PG_ROOT_NENTRIES; i++)
627 KASSERT(pmap->pm_root[i] == 0,
628 ("Index %d on root page %p is non-zero!\n", i, pmap->pm_root));
629 #endif
630 uma_zfree(ptbl_root_zone, pmap->pm_root);
631 }
632
633 static void
mmu_booke_sync_icache(pmap_t pm,vm_offset_t va,vm_size_t sz)634 mmu_booke_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
635 {
636 pte_t *pte;
637 vm_paddr_t pa = 0;
638 int sync_sz, valid;
639
640 while (sz > 0) {
641 PMAP_LOCK(pm);
642 pte = pte_find(pm, va);
643 valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0;
644 if (valid)
645 pa = PTE_PA(pte);
646 PMAP_UNLOCK(pm);
647 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
648 sync_sz = min(sync_sz, sz);
649 if (valid) {
650 pa += (va & PAGE_MASK);
651 __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz);
652 }
653 va += sync_sz;
654 sz -= sync_sz;
655 }
656 }
657
658 /*
659 * mmu_booke_zero_page_area zeros the specified hardware page by
660 * mapping it into virtual memory and using bzero to clear
661 * its contents.
662 *
663 * off and size must reside within a single page.
664 */
665 static void
mmu_booke_zero_page_area(vm_page_t m,int off,int size)666 mmu_booke_zero_page_area(vm_page_t m, int off, int size)
667 {
668 vm_offset_t va;
669
670 /* XXX KASSERT off and size are within a single page? */
671
672 va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
673 bzero((caddr_t)va + off, size);
674 }
675
676 /*
677 * mmu_booke_zero_page zeros the specified hardware page.
678 */
679 static void
mmu_booke_zero_page(vm_page_t m)680 mmu_booke_zero_page(vm_page_t m)
681 {
682 vm_offset_t off, va;
683
684 va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
685
686 for (off = 0; off < PAGE_SIZE; off += cacheline_size)
687 __asm __volatile("dcbz 0,%0" :: "r"(va + off));
688 }
689
690 /*
691 * mmu_booke_copy_page copies the specified (machine independent) page by
692 * mapping the page into virtual memory and using memcopy to copy the page,
693 * one machine dependent page at a time.
694 */
695 static void
mmu_booke_copy_page(vm_page_t sm,vm_page_t dm)696 mmu_booke_copy_page(vm_page_t sm, vm_page_t dm)
697 {
698 vm_offset_t sva, dva;
699
700 sva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(sm));
701 dva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dm));
702 memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE);
703 }
704
705 static inline void
mmu_booke_copy_pages(vm_page_t * ma,vm_offset_t a_offset,vm_page_t * mb,vm_offset_t b_offset,int xfersize)706 mmu_booke_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
707 vm_page_t *mb, vm_offset_t b_offset, int xfersize)
708 {
709 void *a_cp, *b_cp;
710 vm_offset_t a_pg_offset, b_pg_offset;
711 int cnt;
712
713 vm_page_t pa, pb;
714
715 while (xfersize > 0) {
716 a_pg_offset = a_offset & PAGE_MASK;
717 pa = ma[a_offset >> PAGE_SHIFT];
718 b_pg_offset = b_offset & PAGE_MASK;
719 pb = mb[b_offset >> PAGE_SHIFT];
720 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
721 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
722 a_cp = (caddr_t)((uintptr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pa)) +
723 a_pg_offset);
724 b_cp = (caddr_t)((uintptr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pb)) +
725 b_pg_offset);
726 bcopy(a_cp, b_cp, cnt);
727 a_offset += cnt;
728 b_offset += cnt;
729 xfersize -= cnt;
730 }
731 }
732
733 static vm_offset_t
mmu_booke_quick_enter_page(vm_page_t m)734 mmu_booke_quick_enter_page(vm_page_t m)
735 {
736 return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
737 }
738
739 static void
mmu_booke_quick_remove_page(vm_offset_t addr)740 mmu_booke_quick_remove_page(vm_offset_t addr)
741 {
742 }
743
744 /**************************************************************************/
745 /* TID handling */
746 /**************************************************************************/
747
748 /*
749 * Invalidate all TLB0 entries which match the given TID. Note this is
750 * dedicated for cases when invalidations should NOT be propagated to other
751 * CPUs.
752 */
753 static void
tid_flush(tlbtid_t tid)754 tid_flush(tlbtid_t tid)
755 {
756 register_t msr;
757
758 /* Don't evict kernel translations */
759 if (tid == TID_KERNEL)
760 return;
761
762 msr = mfmsr();
763 __asm __volatile("wrteei 0");
764
765 /*
766 * Newer (e500mc and later) have tlbilx, which doesn't broadcast, so use
767 * it for PID invalidation.
768 */
769 mtspr(SPR_MAS6, tid << MAS6_SPID0_SHIFT);
770 __asm __volatile("isync; .long 0x7c200024; isync; msync");
771
772 __asm __volatile("wrtee %0" :: "r"(msr));
773 }
774