1 /* $OpenBSD: pmap.c,v 1.125 2023/04/13 15:23:22 miod Exp $ */
2
3 /*
4 * Copyright (c) 2001-2004 Opsycon AB (www.opsycon.se / www.opsycon.com)
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
19 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/proc.h>
31 #include <sys/malloc.h>
32 #include <sys/user.h>
33 #include <sys/buf.h>
34 #include <sys/pool.h>
35 #ifdef SYSVSHM
36 #include <sys/shm.h>
37 #endif
38 #include <sys/atomic.h>
39
40 #include <mips64/cache.h>
41 #include <mips64/mips_cpu.h>
42 #include <machine/autoconf.h>
43 #include <machine/cpu.h>
44 #include <machine/vmparam.h>
45
46 #include <uvm/uvm.h>
47
48 extern void mem_zero_page(vaddr_t);
49
50 struct pool pmap_pmap_pool;
51 struct pool pmap_pv_pool;
52 struct pool pmap_pg_pool;
53
54 #define pmap_pv_alloc() (pv_entry_t)pool_get(&pmap_pv_pool, PR_NOWAIT)
55 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv))
56
57 #ifndef PMAP_PV_LOWAT
58 #define PMAP_PV_LOWAT 16
59 #endif
60 int pmap_pv_lowat = PMAP_PV_LOWAT;
61
62 /*
63 * Maximum number of pages that can be invalidated from remote TLBs at once.
64 * This limit prevents the IPI handler from blocking other high-priority
65 * interrupts for too long.
66 */
67 #define SHOOTDOWN_MAX 128
68
69 uint pmap_alloc_tlbpid(struct proc *);
70 void pmap_do_page_cache(vm_page_t, u_int);
71 void pmap_do_remove(pmap_t, vaddr_t, vaddr_t);
72 int pmap_enter_pv(pmap_t, vaddr_t, vm_page_t, pt_entry_t *);
73 void pmap_remove_pv(pmap_t, vaddr_t, paddr_t);
74 void pmap_page_remove(struct vm_page *);
75 void pmap_page_wrprotect(struct vm_page *, vm_prot_t);
76 void *pmap_pg_alloc(struct pool *, int, int *);
77 void pmap_pg_free(struct pool *, void *);
78
79 struct pool_allocator pmap_pg_allocator = {
80 pmap_pg_alloc, pmap_pg_free
81 };
82
83 #define pmap_invalidate_kernel_page(va) tlb_flush_addr(va)
84 #define pmap_update_kernel_page(va, entry) tlb_update((va), (entry))
85
86 void pmap_invalidate_user_page(pmap_t, vaddr_t);
87 void pmap_invalidate_icache(pmap_t, vaddr_t, pt_entry_t);
88 void pmap_update_user_page(pmap_t, vaddr_t, pt_entry_t);
89 #ifdef MULTIPROCESSOR
90 void pmap_invalidate_icache_action(void *);
91 void pmap_shootdown_range(pmap_t, vaddr_t, vaddr_t, int);
92 void pmap_shootdown_range_action(void *);
93 #else
94 #define pmap_shootdown_range(pmap, sva, eva, needisync) \
95 do { /* nothing */ } while (0)
96 #endif
97
98 #ifdef PMAPDEBUG
99 struct {
100 int kernel; /* entering kernel mapping */
101 int user; /* entering user mapping */
102 int ptpneeded; /* needed to allocate a PT page */
103 int pwchange; /* no mapping change, just wiring or protection */
104 int wchange; /* no mapping change, just wiring */
105 int mchange; /* was mapped but mapping to different page */
106 int managed; /* a managed page */
107 int firstpv; /* first mapping for this PA */
108 int secondpv; /* second mapping for this PA */
109 int ci; /* cache inhibited */
110 int unmanaged; /* not a managed page */
111 int flushes; /* cache flushes */
112 int cachehit; /* new entry forced valid entry out */
113 } enter_stats;
114 struct {
115 int calls;
116 int removes;
117 int flushes;
118 int pidflushes; /* HW pid stolen */
119 int pvfirst;
120 int pvsearch;
121 } remove_stats;
122
123 #define PDB_FOLLOW 0x0001
124 #define PDB_INIT 0x0002
125 #define PDB_ENTER 0x0004
126 #define PDB_REMOVE 0x0008
127 #define PDB_CREATE 0x0010
128 #define PDB_PTPAGE 0x0020
129 #define PDB_PVENTRY 0x0040
130 #define PDB_BITS 0x0080
131 #define PDB_COLLECT 0x0100
132 #define PDB_PROTECT 0x0200
133 #define PDB_TLBPID 0x0400
134 #define PDB_PARANOIA 0x2000
135 #define PDB_WIRING 0x4000
136 #define PDB_PVDUMP 0x8000
137
138 #define DPRINTF(flag, printdata) \
139 if (pmapdebug & (flag)) \
140 printf printdata;
141
142 #define stat_count(what) atomic_inc_int(&(what))
143 int pmapdebug = PDB_ENTER|PDB_FOLLOW;
144
145 #else
146
147 #define DPRINTF(flag, printdata)
148 #define stat_count(what)
149
150 #endif /* PMAPDEBUG */
151
152 static struct pmap kernel_pmap_store
153 [(PMAP_SIZEOF(MAXCPUS) + sizeof(struct pmap) - 1)
154 / sizeof(struct pmap)];
155 struct pmap *const kernel_pmap_ptr = kernel_pmap_store;
156
157 vaddr_t virtual_start; /* VA of first avail page (after kernel bss)*/
158 vaddr_t virtual_end; /* VA of last avail page (end of kernel AS) */
159
160 vaddr_t pmap_prefer_mask;
161
162 static struct pmap_asid_info pmap_asid_info[MAXCPUS];
163
164 pt_entry_t *Sysmap; /* kernel pte table */
165 u_int Sysmapsize; /* number of pte's in Sysmap */
166 const vaddr_t Sysmapbase = VM_MIN_KERNEL_ADDRESS; /* for libkvm */
167
168 pt_entry_t protection_codes[8];
169 pt_entry_t pg_ri;
170 pt_entry_t pg_xi;
171
172 void
pmap_invalidate_user_page(pmap_t pmap,vaddr_t va)173 pmap_invalidate_user_page(pmap_t pmap, vaddr_t va)
174 {
175 u_long cpuid = cpu_number();
176 u_long asid = pmap->pm_asid[cpuid].pma_asid << PG_ASID_SHIFT;
177
178 if (pmap->pm_asid[cpuid].pma_asidgen ==
179 pmap_asid_info[cpuid].pma_asidgen)
180 tlb_flush_addr(va | asid);
181 }
182
183 void
pmap_update_user_page(pmap_t pmap,vaddr_t va,pt_entry_t entry)184 pmap_update_user_page(pmap_t pmap, vaddr_t va, pt_entry_t entry)
185 {
186 u_long cpuid = cpu_number();
187 u_long asid = pmap->pm_asid[cpuid].pma_asid << PG_ASID_SHIFT;
188
189 if (pmap->pm_asid[cpuid].pma_asidgen ==
190 pmap_asid_info[cpuid].pma_asidgen)
191 tlb_update(va | asid, entry);
192 }
193
194 #ifdef MULTIPROCESSOR
195
196 #define PMAP_ASSERT_LOCKED(pm) \
197 do { \
198 if ((pm) != pmap_kernel()) \
199 MUTEX_ASSERT_LOCKED(&(pm)->pm_mtx); \
200 } while (0)
201
202 static inline void
pmap_lock(pmap_t pmap)203 pmap_lock(pmap_t pmap)
204 {
205 if (pmap != pmap_kernel())
206 mtx_enter(&pmap->pm_mtx);
207 }
208
209 static inline void
pmap_unlock(pmap_t pmap)210 pmap_unlock(pmap_t pmap)
211 {
212 if (pmap != pmap_kernel())
213 mtx_leave(&pmap->pm_mtx);
214 }
215
216 static inline void
pmap_swlock(pmap_t pmap)217 pmap_swlock(pmap_t pmap)
218 {
219 if (pmap != pmap_kernel())
220 mtx_enter(&pmap->pm_swmtx);
221 }
222
223 static inline void
pmap_swunlock(pmap_t pmap)224 pmap_swunlock(pmap_t pmap)
225 {
226 if (pmap != pmap_kernel())
227 mtx_leave(&pmap->pm_swmtx);
228 }
229
230 static inline pt_entry_t
pmap_pte_cas(pt_entry_t * pte,pt_entry_t o,pt_entry_t n)231 pmap_pte_cas(pt_entry_t *pte, pt_entry_t o, pt_entry_t n)
232 {
233 #ifdef MIPS_PTE64
234 return atomic_cas_ulong((unsigned long *)pte, o, n);
235 #else
236 return atomic_cas_uint((unsigned int *)pte, o, n);
237 #endif
238 }
239
240 struct pmap_invalidate_icache_arg {
241 vaddr_t va;
242 pt_entry_t entry;
243 };
244
245 void
pmap_invalidate_icache(pmap_t pmap,vaddr_t va,pt_entry_t entry)246 pmap_invalidate_icache(pmap_t pmap, vaddr_t va, pt_entry_t entry)
247 {
248 struct pmap_invalidate_icache_arg ii_args;
249 unsigned long cpumask = 0;
250 struct cpu_info *ci;
251 CPU_INFO_ITERATOR cii;
252
253 pmap_swlock(pmap);
254 CPU_INFO_FOREACH(cii, ci) {
255 if (CPU_IS_RUNNING(ci) &&
256 pmap->pm_asid[ci->ci_cpuid].pma_asidgen != 0)
257 cpumask |= 1UL << ci->ci_cpuid;
258 }
259 pmap_swunlock(pmap);
260 if (cpumask != 0) {
261 ii_args.va = va;
262 ii_args.entry = entry;
263 smp_rendezvous_cpus(cpumask, pmap_invalidate_icache_action,
264 &ii_args);
265 }
266 }
267
268 void
pmap_invalidate_icache_action(void * arg)269 pmap_invalidate_icache_action(void *arg)
270 {
271 struct cpu_info *ci = curcpu();
272 struct pmap_invalidate_icache_arg *ii_args = arg;
273
274 Mips_SyncDCachePage(ci, ii_args->va, pfn_to_pad(ii_args->entry));
275 Mips_InvalidateICache(ci, ii_args->va, PAGE_SIZE);
276 }
277
278 struct pmap_shootdown_range_arg {
279 pmap_t pmap;
280 vaddr_t sva;
281 vaddr_t eva;
282 int needisync;
283 };
284
285 void
pmap_shootdown_range(pmap_t pmap,vaddr_t sva,vaddr_t eva,int needisync)286 pmap_shootdown_range(pmap_t pmap, vaddr_t sva, vaddr_t eva, int needisync)
287 {
288 struct pmap_shootdown_range_arg sr_arg;
289 struct cpu_info *ci, *self = curcpu();
290 CPU_INFO_ITERATOR cii;
291 vaddr_t va;
292 unsigned long cpumask = 0;
293
294 pmap_swlock(pmap);
295 CPU_INFO_FOREACH(cii, ci) {
296 if (ci == self)
297 continue;
298 if (!CPU_IS_RUNNING(ci))
299 continue;
300 if (pmap != pmap_kernel()) {
301 if (pmap->pm_asid[ci->ci_cpuid].pma_asidgen !=
302 pmap_asid_info[ci->ci_cpuid].pma_asidgen) {
303 continue;
304 } else if (ci->ci_curpmap != pmap) {
305 pmap->pm_asid[ci->ci_cpuid].pma_asidgen = 0;
306 continue;
307 }
308 }
309 cpumask |= 1UL << ci->ci_cpuid;
310 }
311 pmap_swunlock(pmap);
312 if (cpumask != 0) {
313 sr_arg.pmap = pmap;
314 for (va = sva; va < eva; va += SHOOTDOWN_MAX * PAGE_SIZE) {
315 sr_arg.sva = va;
316 sr_arg.eva = va + SHOOTDOWN_MAX * PAGE_SIZE;
317 if (sr_arg.eva > eva)
318 sr_arg.eva = eva;
319 sr_arg.needisync = needisync;
320 smp_rendezvous_cpus(cpumask,
321 pmap_shootdown_range_action, &sr_arg);
322 }
323 }
324 }
325
326 void
pmap_shootdown_range_action(void * arg)327 pmap_shootdown_range_action(void *arg)
328 {
329 struct pmap_shootdown_range_arg *sr_arg = arg;
330 vaddr_t va;
331
332 if (sr_arg->pmap == pmap_kernel()) {
333 for (va = sr_arg->sva; va < sr_arg->eva; va += PAGE_SIZE)
334 pmap_invalidate_kernel_page(va);
335 } else {
336 for (va = sr_arg->sva; va < sr_arg->eva; va += PAGE_SIZE)
337 pmap_invalidate_user_page(sr_arg->pmap, va);
338 if (sr_arg->needisync)
339 Mips_InvalidateICache(curcpu(), sr_arg->sva,
340 sr_arg->eva - sr_arg->sva);
341 }
342 }
343
344 #else /* MULTIPROCESSOR */
345
346 #define PMAP_ASSERT_LOCKED(pm) do { /* nothing */ } while (0)
347 #define pmap_lock(pm) do { /* nothing */ } while (0)
348 #define pmap_unlock(pm) do { /* nothing */ } while (0)
349 #define pmap_swlock(pm) do { /* nothing */ } while (0)
350 #define pmap_swunlock(pm) do { /* nothing */ } while (0)
351
352 void
pmap_invalidate_icache(pmap_t pmap,vaddr_t va,pt_entry_t entry)353 pmap_invalidate_icache(pmap_t pmap, vaddr_t va, pt_entry_t entry)
354 {
355 struct cpu_info *ci = curcpu();
356
357 Mips_SyncDCachePage(ci, va, pfn_to_pad(entry));
358 Mips_InvalidateICache(ci, va, PAGE_SIZE);
359 }
360
361 #endif /* MULTIPROCESSOR */
362
363 /*
364 * Bootstrap the system enough to run with virtual memory.
365 */
366 void
pmap_bootstrap(void)367 pmap_bootstrap(void)
368 {
369 u_int i;
370 pt_entry_t *spte;
371
372 /*
373 * Create a mapping table for kernel virtual memory. This
374 * table is a linear table in contrast to the user process
375 * mapping tables which are built with segment/page tables.
376 * Create 1GB of map (this will only use 1MB of memory).
377 */
378 virtual_start = VM_MIN_KERNEL_ADDRESS;
379 virtual_end = VM_MAX_KERNEL_ADDRESS;
380
381 Sysmapsize = (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) /
382 PAGE_SIZE;
383 if (Sysmapsize & 1)
384 Sysmapsize++; /* force even number of pages */
385
386 Sysmap = (pt_entry_t *)
387 uvm_pageboot_alloc(sizeof(pt_entry_t) * Sysmapsize);
388
389 pool_init(&pmap_pmap_pool, PMAP_SIZEOF(ncpusfound), 0, IPL_VM, 0,
390 "pmappl", NULL);
391 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, IPL_VM, 0,
392 "pvpl", NULL);
393 pool_init(&pmap_pg_pool, PMAP_PGSIZE, PMAP_PGSIZE, IPL_VM, 0,
394 "pmappgpl", &pmap_pg_allocator);
395
396 pmap_kernel()->pm_count = 1;
397
398 /*
399 * The 64 bit Mips architecture stores the AND result
400 * of the Global bits in the pte pair in the on chip
401 * translation lookaside buffer. Thus invalid entries
402 * must have the Global bit set so when Entry LO and
403 * Entry HI G bits are ANDed together they will produce
404 * a global bit to store in the tlb.
405 */
406 for (i = Sysmapsize, spte = Sysmap; i != 0; i--, spte++)
407 *spte = PG_G;
408
409 for (i = 0; i < MAXCPUS; i++) {
410 pmap_asid_info[i].pma_asidgen = 1;
411 pmap_asid_info[i].pma_asid = MIN_USER_ASID + 1;
412 }
413
414 #if defined(CPU_MIPS64R2) && !defined(CPU_LOONGSON2)
415 if (cp0_get_pagegrain() & PGRAIN_RIE)
416 pg_ri = PG_RI;
417 if (cp0_get_pagegrain() & PGRAIN_XIE)
418 pg_xi = PG_XI;
419 #endif
420
421 for (i = 0; i < 8; i++) {
422 if ((i & PROT_READ) == 0)
423 protection_codes[i] |= pg_ri;
424 if ((i & PROT_WRITE) == 0)
425 protection_codes[i] |= PG_RO;
426 if ((i & PROT_EXEC) == 0)
427 protection_codes[i] |= pg_xi;
428 }
429 }
430
431 /*
432 * Page steal allocator used during bootup.
433 */
434 vaddr_t
pmap_steal_memory(vsize_t size,vaddr_t * vstartp,vaddr_t * vendp)435 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
436 {
437 int i, j;
438 uint npg;
439 vaddr_t va;
440 paddr_t pa;
441
442 #ifdef DIAGNOSTIC
443 if (uvm.page_init_done) {
444 panic("pmap_steal_memory: too late, vm is running!");
445 }
446 #endif
447
448 size = round_page(size);
449 npg = atop(size);
450 va = 0;
451
452 for(i = 0; i < vm_nphysseg && va == 0; i++) {
453 if (vm_physmem[i].avail_start != vm_physmem[i].start ||
454 vm_physmem[i].avail_start >= vm_physmem[i].avail_end) {
455 continue;
456 }
457
458 if ((vm_physmem[i].avail_end - vm_physmem[i].avail_start) < npg)
459 continue;
460
461 pa = ptoa(vm_physmem[i].avail_start);
462 vm_physmem[i].avail_start += npg;
463 vm_physmem[i].start += npg;
464
465 if (vm_physmem[i].avail_start == vm_physmem[i].end) {
466 if (vm_nphysseg == 1)
467 panic("pmap_steal_memory: out of memory!");
468
469 vm_nphysseg--;
470 for (j = i; j < vm_nphysseg; j++)
471 vm_physmem[j] = vm_physmem[j + 1];
472 }
473 if (vstartp)
474 *vstartp = round_page(virtual_start);
475 if (vendp)
476 *vendp = virtual_end;
477
478 va = PHYS_TO_XKPHYS(pa, CCA_CACHED);
479
480 bzero((void *)va, size);
481 return (va);
482 }
483
484 panic("pmap_steal_memory: no memory to steal");
485 }
486
487 /*
488 * Initialize the pmap module.
489 * Called by uvm_init, to initialize any structures that the pmap
490 * system needs to map virtual memory.
491 */
492 void
pmap_init()493 pmap_init()
494 {
495
496 DPRINTF(PDB_FOLLOW|PDB_INIT, ("pmap_init()\n"));
497
498 #if 0 /* too early */
499 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat);
500 #endif
501 }
502
503 static pv_entry_t pg_to_pvh(struct vm_page *);
504 static __inline pv_entry_t
pg_to_pvh(struct vm_page * pg)505 pg_to_pvh(struct vm_page *pg)
506 {
507 return &pg->mdpage.pv_ent;
508 }
509
510 /*
511 * Create and return a physical map.
512 */
513 pmap_t
pmap_create()514 pmap_create()
515 {
516 pmap_t pmap;
517
518 DPRINTF(PDB_FOLLOW|PDB_CREATE, ("pmap_create()\n"));
519
520 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK | PR_ZERO);
521 pmap->pm_segtab = pool_get(&pmap_pg_pool, PR_WAITOK | PR_ZERO);
522 pmap->pm_count = 1;
523 mtx_init(&pmap->pm_mtx, IPL_VM);
524 mtx_init(&pmap->pm_swmtx, IPL_SCHED);
525
526 return (pmap);
527 }
528
529 /*
530 * Retire the given physical map from service.
531 * Should only be called if the map contains
532 * no valid mappings.
533 */
534 void
pmap_destroy(pmap_t pmap)535 pmap_destroy(pmap_t pmap)
536 {
537 pt_entry_t **pde, *pte;
538 int count;
539 unsigned int i, j;
540 #ifdef PARANOIA
541 unsigned int k;
542 #endif
543
544 DPRINTF(PDB_FOLLOW|PDB_CREATE, ("pmap_destroy(%p)\n", pmap));
545
546 count = atomic_dec_int_nv(&pmap->pm_count);
547 if (count > 0)
548 return;
549
550 if (pmap->pm_segtab) {
551 for (i = 0; i < PMAP_SEGTABSIZE; i++) {
552 /* get pointer to segment map */
553 if ((pde = pmap->pm_segtab->seg_tab[i]) == NULL)
554 continue;
555 for (j = 0; j < NPDEPG; j++) {
556 if ((pte = pde[j]) == NULL)
557 continue;
558 #ifdef PARANOIA
559 for (k = 0; k < NPTEPG; k++) {
560 if (pte[k] != PG_NV)
561 panic("pmap_destroy(%p): "
562 "pgtab %p not empty at "
563 "index %u", pmap, pte, k);
564 }
565 #endif
566 pool_put(&pmap_pg_pool, pte);
567 #ifdef PARANOIA
568 pde[j] = NULL;
569 #endif
570 }
571 pool_put(&pmap_pg_pool, pde);
572 #ifdef PARANOIA
573 pmap->pm_segtab->seg_tab[i] = NULL;
574 #endif
575 }
576 pool_put(&pmap_pg_pool, pmap->pm_segtab);
577 #ifdef PARANOIA
578 pmap->pm_segtab = NULL;
579 #endif
580 }
581
582 pool_put(&pmap_pmap_pool, pmap);
583 }
584
585 void
pmap_collect(pmap_t pmap)586 pmap_collect(pmap_t pmap)
587 {
588 void *pmpg;
589 pt_entry_t **pde, *pte;
590 unsigned int i, j, k;
591 unsigned int m, n;
592
593 DPRINTF(PDB_FOLLOW, ("pmap_collect(%p)\n", pmap));
594
595 /* There is nothing to garbage collect in the kernel pmap. */
596 if (pmap == pmap_kernel())
597 return;
598
599 pmap_lock(pmap);
600
601 /*
602 * When unlinking a directory page, the subsequent call to
603 * pmap_shootdown_range() lets any parallel lockless directory
604 * traversals end before the page gets freed.
605 */
606
607 for (i = 0; i < PMAP_SEGTABSIZE; i++) {
608 if ((pde = pmap->pm_segtab->seg_tab[i]) == NULL)
609 continue;
610 m = 0;
611 for (j = 0; j < NPDEPG; j++) {
612 if ((pte = pde[j]) == NULL)
613 continue;
614 n = 0;
615 for (k = 0; k < NPTEPG; k++) {
616 if (pte[k] & PG_V) {
617 n++;
618 break;
619 }
620 }
621 if (n == 0) {
622 pmpg = pde[j];
623 pde[j] = NULL;
624 pmap_shootdown_range(pmap, 0, 0, 0);
625 pool_put(&pmap_pg_pool, pmpg);
626 } else
627 m++;
628 }
629 if (m == 0) {
630 pmpg = pmap->pm_segtab->seg_tab[i];
631 pmap->pm_segtab->seg_tab[i] = NULL;
632 pmap_shootdown_range(pmap, 0, 0, 0);
633 pool_put(&pmap_pg_pool, pmpg);
634 }
635 }
636
637 pmap_unlock(pmap);
638 }
639
640 /*
641 * Add a reference to the specified pmap.
642 */
643 void
pmap_reference(pmap_t pmap)644 pmap_reference(pmap_t pmap)
645 {
646
647 DPRINTF(PDB_FOLLOW, ("pmap_reference(%p)\n", pmap));
648
649 atomic_inc_int(&pmap->pm_count);
650 }
651
652 /*
653 * Make a new pmap (vmspace) active for the given process.
654 */
655 void
pmap_activate(struct proc * p)656 pmap_activate(struct proc *p)
657 {
658 pmap_t pmap = p->p_vmspace->vm_map.pmap;
659 struct cpu_info *ci = curcpu();
660 uint id;
661
662 pmap_swlock(pmap);
663 ci->ci_curpmap = pmap;
664 p->p_addr->u_pcb.pcb_segtab = pmap->pm_segtab;
665 id = pmap_alloc_tlbpid(p);
666 if (p == ci->ci_curproc)
667 tlb_set_pid(id);
668 pmap_swunlock(pmap);
669 }
670
671 /*
672 * Make a previously active pmap (vmspace) inactive.
673 */
674 void
pmap_deactivate(struct proc * p)675 pmap_deactivate(struct proc *p)
676 {
677 struct cpu_info *ci = curcpu();
678
679 ci->ci_curpmap = NULL;
680 }
681
682 /*
683 * Remove the given range of addresses from the specified map.
684 *
685 * It is assumed that the start and end are properly
686 * rounded to the page size.
687 */
688 void
pmap_do_remove(pmap_t pmap,vaddr_t sva,vaddr_t eva)689 pmap_do_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
690 {
691 vaddr_t ndsva, nssva, va;
692 pt_entry_t ***seg, **pde, *pte, entry;
693 paddr_t pa;
694 struct cpu_info *ci = curcpu();
695 int needisync = 0;
696
697 PMAP_ASSERT_LOCKED(pmap);
698
699 DPRINTF(PDB_FOLLOW|PDB_REMOVE|PDB_PROTECT,
700 ("pmap_remove(%p, %p, %p)\n", pmap, (void *)sva, (void *)eva));
701
702 stat_count(remove_stats.calls);
703
704 if (pmap == pmap_kernel()) {
705 /* remove entries from kernel pmap */
706 #ifdef DIAGNOSTIC
707 if (sva < VM_MIN_KERNEL_ADDRESS ||
708 eva >= VM_MAX_KERNEL_ADDRESS || eva < sva)
709 panic("pmap_remove(%p, %p): not in range",
710 (void *)sva, (void *)eva);
711 #endif
712 pte = kvtopte(sva);
713 for (va = sva; va < eva; va += PAGE_SIZE, pte++) {
714 entry = *pte;
715 if (!(entry & PG_V))
716 continue;
717 if (entry & PG_WIRED)
718 atomic_dec_long(&pmap->pm_stats.wired_count);
719 atomic_dec_long(&pmap->pm_stats.resident_count);
720 pa = pfn_to_pad(entry);
721 if ((entry & PG_CACHEMODE) == PG_CACHED)
722 Mips_HitSyncDCachePage(ci, va, pa);
723 pmap_remove_pv(pmap, va, pa);
724 *pte = PG_NV | PG_G;
725 /*
726 * Flush the TLB for the given address.
727 */
728 pmap_invalidate_kernel_page(va);
729 stat_count(remove_stats.flushes);
730 }
731 pmap_shootdown_range(pmap_kernel(), sva, eva, 0);
732 return;
733 }
734
735 #ifdef DIAGNOSTIC
736 if (eva > VM_MAXUSER_ADDRESS)
737 panic("pmap_remove: uva not in range");
738 #endif
739 /*
740 * Invalidate every valid mapping within the range.
741 */
742 seg = &pmap_segmap(pmap, sva);
743 for (va = sva ; va < eva; va = nssva, seg++) {
744 nssva = mips_trunc_seg(va) + NBSEG;
745 if (*seg == NULL)
746 continue;
747 pde = *seg + uvtopde(va);
748 for ( ; va < eva && va < nssva; va = ndsva, pde++) {
749 ndsva = mips_trunc_dir(va) + NBDIR;
750 if (*pde == NULL)
751 continue;
752 pte = *pde + uvtopte(va);
753 for ( ; va < eva && va < ndsva;
754 va += PAGE_SIZE, pte++) {
755 entry = *pte;
756 if (!(entry & PG_V))
757 continue;
758 if (entry & PG_WIRED)
759 atomic_dec_long(
760 &pmap->pm_stats.wired_count);
761 atomic_dec_long(&pmap->pm_stats.resident_count);
762 pa = pfn_to_pad(entry);
763 if ((entry & PG_CACHEMODE) == PG_CACHED)
764 Mips_SyncDCachePage(ci, va, pa);
765 if (pg_xi != 0 && (entry & pg_xi) == 0)
766 needisync = 1;
767 pmap_remove_pv(pmap, va, pa);
768 *pte = PG_NV;
769 /*
770 * Flush the TLB for the given address.
771 */
772 pmap_invalidate_user_page(pmap, va);
773 if (needisync)
774 Mips_InvalidateICache(ci, va,
775 PAGE_SIZE);
776 stat_count(remove_stats.flushes);
777 }
778 }
779 }
780 pmap_shootdown_range(pmap, sva, eva, needisync);
781 }
782
783 void
pmap_remove(pmap_t pmap,vaddr_t sva,vaddr_t eva)784 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
785 {
786 pmap_lock(pmap);
787 pmap_do_remove(pmap, sva, eva);
788 pmap_unlock(pmap);
789 }
790
791 /*
792 * Makes all mappings to a given page read-only.
793 */
794 void
pmap_page_wrprotect(struct vm_page * pg,vm_prot_t prot)795 pmap_page_wrprotect(struct vm_page *pg, vm_prot_t prot)
796 {
797 struct cpu_info *ci = curcpu();
798 pt_entry_t *pte, entry, p;
799 pv_entry_t pv;
800 int needisync = 0;
801
802 p = protection_codes[prot];
803
804 mtx_enter(&pg->mdpage.pv_mtx);
805 for (pv = pg_to_pvh(pg); pv != NULL; pv = pv->pv_next) {
806 if (pv->pv_pmap == pmap_kernel()) {
807 #ifdef DIAGNOSTIC
808 if (pv->pv_va < VM_MIN_KERNEL_ADDRESS ||
809 pv->pv_va >= VM_MAX_KERNEL_ADDRESS)
810 panic("%s(%p)", __func__, (void *)pv->pv_va);
811 #endif
812 pte = kvtopte(pv->pv_va);
813 entry = *pte;
814 if (!(entry & PG_V))
815 continue;
816 if ((entry & PG_M) != 0 &&
817 (entry & PG_CACHEMODE) == PG_CACHED)
818 Mips_HitSyncDCachePage(ci, pv->pv_va,
819 pfn_to_pad(entry));
820 entry = (entry & ~PG_PROTMASK) | p;
821 *pte = entry;
822 pmap_update_kernel_page(pv->pv_va, entry);
823 pmap_shootdown_range(pmap_kernel(), pv->pv_va,
824 pv->pv_va + PAGE_SIZE, 0);
825 } else if (pv->pv_pmap != NULL) {
826 pte = pmap_pte_lookup(pv->pv_pmap, pv->pv_va);
827 if (pte == NULL)
828 continue;
829 entry = *pte;
830 if (!(entry & PG_V))
831 continue;
832 if ((entry & PG_M) != 0 &&
833 (entry & PG_CACHEMODE) == PG_CACHED)
834 Mips_SyncDCachePage(ci, pv->pv_va,
835 pfn_to_pad(entry));
836 if (pg_xi != 0 && (entry & pg_xi) == 0)
837 needisync = 1;
838 entry = (entry & ~PG_PROTMASK) | p;
839 *pte = entry;
840 pmap_update_user_page(pv->pv_pmap, pv->pv_va, entry);
841 if (needisync)
842 Mips_InvalidateICache(ci, pv->pv_va, PAGE_SIZE);
843 pmap_shootdown_range(pv->pv_pmap, pv->pv_va,
844 pv->pv_va + PAGE_SIZE, needisync);
845 }
846 }
847 mtx_leave(&pg->mdpage.pv_mtx);
848 }
849
850 /*
851 * Removes all mappings to a given page.
852 */
853 void
pmap_page_remove(struct vm_page * pg)854 pmap_page_remove(struct vm_page *pg)
855 {
856 pmap_t pmap;
857 pv_entry_t pv;
858 vaddr_t va;
859
860 mtx_enter(&pg->mdpage.pv_mtx);
861 while ((pv = pg_to_pvh(pg))->pv_pmap != NULL) {
862 pmap = pv->pv_pmap;
863 va = pv->pv_va;
864
865 /*
866 * The PV list lock has to be released for pmap_do_remove().
867 * The lock ordering prevents locking the pmap before the
868 * release, so another CPU might remove or replace the page at
869 * the virtual address in the pmap. Continue with this PV entry
870 * only if the list head is unchanged after reacquiring
871 * the locks.
872 */
873 pmap_reference(pmap);
874 mtx_leave(&pg->mdpage.pv_mtx);
875 pmap_lock(pmap);
876 mtx_enter(&pg->mdpage.pv_mtx);
877 if (pg_to_pvh(pg)->pv_pmap != pmap ||
878 pg_to_pvh(pg)->pv_va != va) {
879 mtx_leave(&pg->mdpage.pv_mtx);
880 pmap_unlock(pmap);
881 pmap_destroy(pmap);
882 mtx_enter(&pg->mdpage.pv_mtx);
883 continue;
884 }
885 mtx_leave(&pg->mdpage.pv_mtx);
886
887 pmap_do_remove(pmap, va, va + PAGE_SIZE);
888
889 pmap_unlock(pmap);
890 pmap_destroy(pmap);
891 mtx_enter(&pg->mdpage.pv_mtx);
892 }
893 mtx_leave(&pg->mdpage.pv_mtx);
894 }
895
896 /*
897 * pmap_page_protect:
898 *
899 * Lower the permission for all mappings to a given page.
900 */
901 void
pmap_page_protect(struct vm_page * pg,vm_prot_t prot)902 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
903 {
904 if (prot == PROT_NONE) {
905 DPRINTF(PDB_REMOVE, ("pmap_page_protect(%p, 0x%x)\n", pg, prot));
906 } else {
907 DPRINTF(PDB_FOLLOW|PDB_PROTECT,
908 ("pmap_page_protect(%p, 0x%x)\n", pg, prot));
909 }
910
911 switch (prot) {
912 case PROT_READ | PROT_WRITE:
913 case PROT_MASK:
914 break;
915
916 /* copy_on_write */
917 case PROT_EXEC:
918 case PROT_READ:
919 case PROT_READ | PROT_EXEC:
920 pmap_page_wrprotect(pg, prot);
921 break;
922
923 /* remove_all */
924 default:
925 pmap_page_remove(pg);
926 break;
927 }
928 }
929
930 /*
931 * Set the physical protection on the
932 * specified range of this map as requested.
933 */
934 void
pmap_protect(pmap_t pmap,vaddr_t sva,vaddr_t eva,vm_prot_t prot)935 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
936 {
937 vaddr_t ndsva, nssva, va;
938 pt_entry_t ***seg, **pde, *pte, entry, p;
939 struct cpu_info *ci = curcpu();
940 int needisync = 0;
941
942 DPRINTF(PDB_FOLLOW|PDB_PROTECT,
943 ("pmap_protect(%p, %p, %p, 0x%x)\n",
944 pmap, (void *)sva, (void *)eva, prot));
945
946 if ((prot & PROT_READ) == PROT_NONE) {
947 pmap_remove(pmap, sva, eva);
948 return;
949 }
950
951 p = protection_codes[prot];
952 if (prot & PROT_WRITE)
953 p |= PG_M;
954
955 pmap_lock(pmap);
956
957 if (pmap == pmap_kernel()) {
958 /*
959 * Change entries in kernel pmap.
960 * This will trap if the page is writeable (in order to set
961 * the dirty bit) even if the dirty bit is already set. The
962 * optimization isn't worth the effort since this code isn't
963 * executed much. The common case is to make a user page
964 * read-only.
965 */
966 #ifdef DIAGNOSTIC
967 if (sva < VM_MIN_KERNEL_ADDRESS ||
968 eva >= VM_MAX_KERNEL_ADDRESS || eva < sva)
969 panic("pmap_protect(%p, %p): not in range",
970 (void *)sva, (void *)eva);
971 #endif
972 pte = kvtopte(sva);
973 for (va = sva; va < eva; va += PAGE_SIZE, pte++) {
974 entry = *pte;
975 if (!(entry & PG_V))
976 continue;
977 if ((entry & PG_M) != 0 /* && p != PG_M */)
978 if ((entry & PG_CACHEMODE) == PG_CACHED)
979 Mips_HitSyncDCachePage(ci, va,
980 pfn_to_pad(entry));
981 entry = (entry & ~PG_PROTMASK) | p;
982 *pte = entry;
983 /*
984 * Update the TLB if the given address is in the cache.
985 */
986 pmap_update_kernel_page(va, entry);
987 }
988 pmap_shootdown_range(pmap_kernel(), sva, eva, 0);
989 pmap_unlock(pmap);
990 return;
991 }
992
993 #ifdef DIAGNOSTIC
994 if (eva > VM_MAXUSER_ADDRESS)
995 panic("pmap_protect: uva not in range");
996 #endif
997 /*
998 * Change protection on every valid mapping within the range.
999 */
1000 seg = &pmap_segmap(pmap, sva);
1001 for (va = sva ; va < eva; va = nssva, seg++) {
1002 nssva = mips_trunc_seg(va) + NBSEG;
1003 if (*seg == NULL)
1004 continue;
1005 pde = *seg + uvtopde(va);
1006 for ( ; va < eva && va < nssva; va = ndsva, pde++) {
1007 ndsva = mips_trunc_dir(va) + NBDIR;
1008 if (*pde == NULL)
1009 continue;
1010 pte = *pde + uvtopte(va);
1011 for ( ; va < eva && va < ndsva;
1012 va += PAGE_SIZE, pte++) {
1013 entry = *pte;
1014 if (!(entry & PG_V))
1015 continue;
1016 if ((entry & PG_M) != 0 /* && p != PG_M */ &&
1017 (entry & PG_CACHEMODE) == PG_CACHED) {
1018 if (prot & PROT_EXEC) {
1019 /* This will also sync D$. */
1020 pmap_invalidate_icache(pmap,
1021 va, entry);
1022 } else
1023 Mips_SyncDCachePage(ci, va,
1024 pfn_to_pad(entry));
1025 }
1026 if (pg_xi != 0 && (entry & pg_xi) == 0)
1027 needisync = 1;
1028 entry = (entry & ~PG_PROTMASK) | p;
1029 *pte = entry;
1030 pmap_update_user_page(pmap, va, entry);
1031 if (needisync)
1032 Mips_InvalidateICache(ci, va,
1033 PAGE_SIZE);
1034 }
1035 }
1036 }
1037 pmap_shootdown_range(pmap, sva, eva, needisync);
1038
1039 pmap_unlock(pmap);
1040 }
1041
1042 /*
1043 * Insert the given physical page (p) at
1044 * the specified virtual address (v) in the
1045 * target physical map with the protection requested.
1046 *
1047 * NB: This is the only routine which MAY NOT lazy-evaluate
1048 * or lose information. That is, this routine must actually
1049 * insert this page into the given map NOW.
1050 */
1051 int
pmap_enter(pmap_t pmap,vaddr_t va,paddr_t pa,vm_prot_t prot,int flags)1052 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
1053 {
1054 pt_entry_t **pde, *pte, npte, opte;
1055 vm_page_t pg;
1056 struct cpu_info *ci = curcpu();
1057 u_long cpuid = ci->ci_cpuid;
1058 int needisync = 0;
1059 int wired = (flags & PMAP_WIRED) != 0;
1060
1061 DPRINTF(PDB_FOLLOW|PDB_ENTER,
1062 ("pmap_enter(%p, %p, %p, 0x%x, 0x%x)\n",
1063 pmap, (void *)va, (void *)pa, prot, flags));
1064
1065 pg = PHYS_TO_VM_PAGE(pa);
1066
1067 pmap_lock(pmap);
1068
1069 /*
1070 * Get the PTE. Allocate page directory and page table pages
1071 * if necessary.
1072 */
1073 if (pmap == pmap_kernel()) {
1074 #ifdef DIAGNOSTIC
1075 if (va < VM_MIN_KERNEL_ADDRESS ||
1076 va >= VM_MAX_KERNEL_ADDRESS)
1077 panic("%s: kva %p", __func__, (void *)va);
1078 #endif
1079 stat_count(enter_stats.kernel);
1080 pte = kvtopte(va);
1081 } else {
1082 #ifdef DIAGNOSTIC
1083 if (va >= VM_MAXUSER_ADDRESS)
1084 panic("%s: uva %p", __func__, (void *)va);
1085 #endif
1086 stat_count(enter_stats.user);
1087 if ((pde = pmap_segmap(pmap, va)) == NULL) {
1088 pde = pool_get(&pmap_pg_pool, PR_NOWAIT | PR_ZERO);
1089 if (pde == NULL) {
1090 if (flags & PMAP_CANFAIL) {
1091 pmap_unlock(pmap);
1092 return ENOMEM;
1093 }
1094 panic("%s: out of memory", __func__);
1095 }
1096 pmap_segmap(pmap, va) = pde;
1097 }
1098 if ((pte = pde[uvtopde(va)]) == NULL) {
1099 pte = pool_get(&pmap_pg_pool, PR_NOWAIT | PR_ZERO);
1100 if (pte == NULL) {
1101 if (flags & PMAP_CANFAIL) {
1102 pmap_unlock(pmap);
1103 return ENOMEM;
1104 }
1105 panic("%s: out of memory", __func__);
1106 }
1107 pde[uvtopde(va)] = pte;
1108 }
1109 pte += uvtopte(va);
1110 }
1111
1112 if ((*pte & PG_V) && pa != pfn_to_pad(*pte)) {
1113 pmap_do_remove(pmap, va, va + PAGE_SIZE);
1114 stat_count(enter_stats.mchange);
1115 }
1116
1117 if ((*pte & PG_V) == 0) {
1118 atomic_inc_long(&pmap->pm_stats.resident_count);
1119 if (wired)
1120 atomic_inc_long(&pmap->pm_stats.wired_count);
1121 } else {
1122 if ((*pte & PG_WIRED) != 0 && wired == 0)
1123 atomic_dec_long(&pmap->pm_stats.wired_count);
1124 else if ((*pte & PG_WIRED) == 0 && wired != 0)
1125 atomic_inc_long(&pmap->pm_stats.wired_count);
1126 }
1127
1128 npte = protection_codes[prot] | PG_V;
1129
1130 if (pg != NULL) {
1131 mtx_enter(&pg->mdpage.pv_mtx);
1132
1133 /* Set page referenced/modified status based on flags */
1134 if (flags & PROT_WRITE)
1135 atomic_setbits_int(&pg->pg_flags,
1136 PGF_ATTR_MOD | PGF_ATTR_REF);
1137 else if (flags & PROT_MASK)
1138 atomic_setbits_int(&pg->pg_flags, PGF_ATTR_REF);
1139
1140 if (prot & PROT_WRITE) {
1141 if (pmap == pmap_kernel()) {
1142 /*
1143 * Don't bother to trap on kernel writes,
1144 * just record page as dirty.
1145 */
1146 npte |= PG_M;
1147 } else if (pg->pg_flags & PGF_ATTR_MOD) {
1148 npte |= PG_M;
1149 }
1150 }
1151
1152 if (flags & PMAP_NOCACHE)
1153 npte |= PG_UNCACHED;
1154 else
1155 npte |= PG_CACHED;
1156
1157 stat_count(enter_stats.managed);
1158 } else {
1159 /*
1160 * Assumption: if it is not part of our managed memory
1161 * then it must be device memory which may be volatile.
1162 */
1163 stat_count(enter_stats.unmanaged);
1164 npte |= PG_UNCACHED;
1165 if (prot & PROT_WRITE)
1166 npte |= PG_M;
1167 }
1168
1169 if (pmap == pmap_kernel()) {
1170 /*
1171 * Enter kernel space mapping.
1172 */
1173
1174 if (pg != NULL) {
1175 if (pmap_enter_pv(pmap, va, pg, &npte) != 0) {
1176 if (flags & PMAP_CANFAIL) {
1177 mtx_leave(&pg->mdpage.pv_mtx);
1178 pmap_unlock(pmap);
1179 return ENOMEM;
1180 }
1181 panic("pmap_enter: pmap_enter_pv() failed");
1182 }
1183 }
1184
1185 npte |= vad_to_pfn(pa) | PG_G;
1186 if (wired)
1187 npte |= PG_WIRED;
1188
1189 /*
1190 * Update the same virtual address entry.
1191 */
1192 opte = *pte;
1193 *pte = npte;
1194 pmap_update_kernel_page(va, npte);
1195 if ((opte & PG_V) != 0)
1196 pmap_shootdown_range(pmap_kernel(), va, va + PAGE_SIZE,
1197 0);
1198 if (pg != NULL)
1199 mtx_leave(&pg->mdpage.pv_mtx);
1200 pmap_unlock(pmap);
1201 return 0;
1202 }
1203
1204 /*
1205 * Enter user space mapping.
1206 */
1207
1208 if (pg != NULL) {
1209 if (pmap_enter_pv(pmap, va, pg, &npte) != 0) {
1210 if (flags & PMAP_CANFAIL) {
1211 mtx_leave(&pg->mdpage.pv_mtx);
1212 pmap_unlock(pmap);
1213 return ENOMEM;
1214 }
1215 panic("pmap_enter: pmap_enter_pv() failed");
1216 }
1217 }
1218
1219 npte |= vad_to_pfn(pa);
1220 if (wired)
1221 npte |= PG_WIRED;
1222
1223 if (pmap->pm_asid[cpuid].pma_asidgen ==
1224 pmap_asid_info[cpuid].pma_asidgen) {
1225 DPRINTF(PDB_ENTER, ("pmap_enter: new pte 0x%08x tlbpid %u\n",
1226 npte, pmap->pm_asid[cpuid].pma_asid));
1227 } else {
1228 DPRINTF(PDB_ENTER, ("pmap_enter: new pte 0x%08x\n", npte));
1229 }
1230
1231 opte = *pte;
1232 *pte = npte;
1233 pmap_update_user_page(pmap, va, npte);
1234 if ((opte & PG_V) != 0) {
1235 if (pg_xi != 0 && (opte & pg_xi) == 0) {
1236 needisync = 1;
1237 Mips_InvalidateICache(ci, va, PAGE_SIZE);
1238 }
1239 pmap_shootdown_range(pmap, va, va + PAGE_SIZE, needisync);
1240 }
1241
1242 /*
1243 * If mapping an executable page, invalidate ICache
1244 * and make sure there are no pending writes.
1245 */
1246 if (pg != NULL && (prot & PROT_EXEC)) {
1247 if ((npte & PG_CACHEMODE) == PG_CACHED) {
1248 /* This will also sync D$. */
1249 pmap_invalidate_icache(pmap, va, npte);
1250 } else
1251 Mips_InvalidateICache(ci, va, PAGE_SIZE);
1252 }
1253
1254 if (pg != NULL)
1255 mtx_leave(&pg->mdpage.pv_mtx);
1256 pmap_unlock(pmap);
1257
1258 return 0;
1259 }
1260
1261 void
pmap_kenter_pa(vaddr_t va,paddr_t pa,vm_prot_t prot)1262 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
1263 {
1264 pt_entry_t *pte, npte, opte;
1265
1266 DPRINTF(PDB_FOLLOW|PDB_ENTER,
1267 ("pmap_kenter_pa(%p, %p, 0x%x)\n", (void *)va, (void *)pa, prot));
1268
1269 #ifdef DIAGNOSTIC
1270 if (va < VM_MIN_KERNEL_ADDRESS ||
1271 va >= VM_MAX_KERNEL_ADDRESS)
1272 panic("pmap_kenter_pa: kva %p", (void *)va);
1273 #endif
1274
1275 npte = vad_to_pfn(pa) | protection_codes[prot] |
1276 PG_V | PG_G | PG_CACHED | PG_WIRED;
1277 if (prot & PROT_WRITE)
1278 npte |= PG_M;
1279 pte = kvtopte(va);
1280 if ((*pte & PG_V) == 0) {
1281 atomic_inc_long(&pmap_kernel()->pm_stats.resident_count);
1282 atomic_inc_long(&pmap_kernel()->pm_stats.wired_count);
1283 } else {
1284 if ((*pte & PG_WIRED) == 0)
1285 atomic_inc_long(&pmap_kernel()->pm_stats.wired_count);
1286 }
1287 opte = *pte;
1288 *pte = npte;
1289 pmap_update_kernel_page(va, npte);
1290 if ((opte & PG_V) != 0)
1291 pmap_shootdown_range(pmap_kernel(), va, va + PAGE_SIZE, 0);
1292 }
1293
1294 /*
1295 * Remove a mapping from the kernel map table. When doing this
1296 * the cache must be synced for the VA mapped since we mapped
1297 * pages behind the back of the VP tracking system.
1298 */
1299 void
pmap_kremove(vaddr_t va,vsize_t len)1300 pmap_kremove(vaddr_t va, vsize_t len)
1301 {
1302 pt_entry_t *pte, entry;
1303 vaddr_t eva;
1304 struct cpu_info *ci = curcpu();
1305
1306 DPRINTF(PDB_FOLLOW|PDB_REMOVE,
1307 ("pmap_kremove(%p, 0x%lx)\n", (void *)va, len));
1308
1309 eva = va + len;
1310 #ifdef DIAGNOSTIC
1311 if (va < VM_MIN_KERNEL_ADDRESS ||
1312 eva >= VM_MAX_KERNEL_ADDRESS || eva < va)
1313 panic("pmap_kremove: va %p len %lx", (void *)va, len);
1314 #endif
1315 pte = kvtopte(va);
1316 for (; va < eva; va += PAGE_SIZE, pte++) {
1317 entry = *pte;
1318 if (!(entry & PG_V))
1319 continue;
1320 if ((entry & PG_CACHEMODE) == PG_CACHED)
1321 Mips_HitSyncDCachePage(ci, va, pfn_to_pad(entry));
1322 *pte = PG_NV | PG_G;
1323 pmap_invalidate_kernel_page(va);
1324 pmap_shootdown_range(pmap_kernel(), va, va + PAGE_SIZE, 0);
1325 atomic_dec_long(&pmap_kernel()->pm_stats.wired_count);
1326 atomic_dec_long(&pmap_kernel()->pm_stats.resident_count);
1327 }
1328 }
1329
1330 void
pmap_unwire(pmap_t pmap,vaddr_t va)1331 pmap_unwire(pmap_t pmap, vaddr_t va)
1332 {
1333 pt_entry_t *pte;
1334
1335 pmap_lock(pmap);
1336
1337 if (pmap == pmap_kernel())
1338 pte = kvtopte(va);
1339 else {
1340 pte = pmap_pte_lookup(pmap, va);
1341 if (pte == NULL)
1342 goto out;
1343 }
1344
1345 if (*pte & PG_V) {
1346 if (*pte & PG_WIRED) {
1347 *pte &= ~PG_WIRED;
1348 atomic_dec_long(&pmap->pm_stats.wired_count);
1349 }
1350 }
1351
1352 out:
1353 pmap_unlock(pmap);
1354 }
1355
1356 /*
1357 * Routine: pmap_extract
1358 * Function:
1359 * Extract the physical page address associated
1360 * with the given map/virtual_address pair.
1361 */
1362 int
pmap_extract(pmap_t pmap,vaddr_t va,paddr_t * pap)1363 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap)
1364 {
1365 paddr_t pa;
1366 pt_entry_t *pte;
1367 int found = 1;
1368
1369 if (pmap == pmap_kernel()) {
1370 if (IS_XKPHYS(va))
1371 pa = XKPHYS_TO_PHYS(va);
1372 else if (va >= (vaddr_t)CKSEG0_BASE &&
1373 va < (vaddr_t)CKSEG0_BASE + CKSEG_SIZE)
1374 pa = CKSEG0_TO_PHYS(va);
1375 else if (va >= (vaddr_t)CKSEG1_BASE &&
1376 va < (vaddr_t)CKSEG1_BASE + CKSEG_SIZE)
1377 pa = CKSEG1_TO_PHYS(va);
1378 else {
1379 #ifdef DIAGNOSTIC
1380 if (va < VM_MIN_KERNEL_ADDRESS ||
1381 va >= VM_MAX_KERNEL_ADDRESS)
1382 panic("pmap_extract(%p, %p)", pmap, (void *)va);
1383 #endif
1384 pte = kvtopte(va);
1385 if (*pte & PG_V)
1386 pa = pfn_to_pad(*pte) | (va & PAGE_MASK);
1387 else
1388 found = 0;
1389 }
1390 } else {
1391 pmap_lock(pmap);
1392 pte = pmap_pte_lookup(pmap, va);
1393 if (pte != NULL && (*pte & PG_V) != 0)
1394 pa = pfn_to_pad(*pte) | (va & PAGE_MASK);
1395 else
1396 found = 0;
1397 pmap_unlock(pmap);
1398 }
1399
1400 if (found != 0)
1401 *pap = pa;
1402
1403 DPRINTF(PDB_FOLLOW, ("pmap_extract(%p, %p)=%p(%d)",
1404 pmap, (void *)va, (void *)pa, rv));
1405
1406 return found;
1407 }
1408
1409 /*
1410 * pmap_zero_page zeros the specified (machine independent) page.
1411 */
1412 void
pmap_zero_page(struct vm_page * pg)1413 pmap_zero_page(struct vm_page *pg)
1414 {
1415 paddr_t phys = VM_PAGE_TO_PHYS(pg);
1416 vaddr_t va;
1417 pv_entry_t pv;
1418 struct cpu_info *ci = curcpu();
1419 int df = 0;
1420
1421 DPRINTF(PDB_FOLLOW, ("pmap_zero_page(%p)\n", (void *)phys));
1422
1423 va = (vaddr_t)PHYS_TO_XKPHYS(phys, CCA_CACHED);
1424 if (pg->pg_flags & PGF_UNCACHED)
1425 df = 1;
1426 else if (pg->pg_flags & PGF_CACHED) {
1427 mtx_enter(&pg->mdpage.pv_mtx);
1428 pv = pg_to_pvh(pg);
1429 df = ((pv->pv_va ^ va) & cache_valias_mask) != 0;
1430 if (df)
1431 Mips_SyncDCachePage(ci, pv->pv_va, phys);
1432 mtx_leave(&pg->mdpage.pv_mtx);
1433 }
1434 mem_zero_page(va);
1435 if (df || cache_valias_mask != 0)
1436 Mips_HitSyncDCachePage(ci, va, phys);
1437 }
1438
1439 /*
1440 * pmap_copy_page copies the specified (machine independent) page.
1441 *
1442 * We do the copy phys to phys and need to check if there may be
1443 * a virtual coherence problem. If so flush the cache for the
1444 * areas before copying, and flush afterwards.
1445 */
1446 void
pmap_copy_page(struct vm_page * srcpg,struct vm_page * dstpg)1447 pmap_copy_page(struct vm_page *srcpg, struct vm_page *dstpg)
1448 {
1449 paddr_t src, dst;
1450 vaddr_t s, d;
1451 int sf, df;
1452 pv_entry_t pv;
1453 struct cpu_info *ci = curcpu();
1454
1455 sf = df = 0;
1456 src = VM_PAGE_TO_PHYS(srcpg);
1457 dst = VM_PAGE_TO_PHYS(dstpg);
1458 s = (vaddr_t)PHYS_TO_XKPHYS(src, CCA_CACHED);
1459 d = (vaddr_t)PHYS_TO_XKPHYS(dst, CCA_CACHED);
1460
1461 DPRINTF(PDB_FOLLOW,
1462 ("pmap_copy_page(%p, %p)\n", (void *)src, (void *)dst));
1463
1464 mtx_enter(&srcpg->mdpage.pv_mtx);
1465 pv = pg_to_pvh(srcpg);
1466 if (srcpg->pg_flags & PGF_UNCACHED)
1467 sf = 1;
1468 else if (srcpg->pg_flags & PGF_CACHED) {
1469 sf = ((pv->pv_va ^ s) & cache_valias_mask) != 0;
1470 if (sf)
1471 Mips_SyncDCachePage(ci, pv->pv_va, src);
1472 }
1473 mtx_leave(&srcpg->mdpage.pv_mtx);
1474
1475 mtx_enter(&dstpg->mdpage.pv_mtx);
1476 pv = pg_to_pvh(dstpg);
1477 if (dstpg->pg_flags & PGF_UNCACHED)
1478 df = 1;
1479 else if (dstpg->pg_flags & PGF_CACHED) {
1480 df = ((pv->pv_va ^ s) & cache_valias_mask) != 0;
1481 if (df)
1482 Mips_SyncDCachePage(ci, pv->pv_va, dst);
1483 }
1484 mtx_leave(&dstpg->mdpage.pv_mtx);
1485
1486 memcpy((void *)d, (void *)s, PAGE_SIZE);
1487
1488 if (sf)
1489 Mips_HitInvalidateDCache(ci, s, PAGE_SIZE);
1490 if (df || cache_valias_mask != 0)
1491 Mips_HitSyncDCachePage(ci, d, dst);
1492 }
1493
1494 /*
1495 * Clear the modify bits on the specified physical page.
1496 * Also sync the cache so it reflects the new clean state of the page.
1497 */
1498 int
pmap_clear_modify(struct vm_page * pg)1499 pmap_clear_modify(struct vm_page *pg)
1500 {
1501 pv_entry_t pv;
1502 pt_entry_t *pte, entry;
1503 paddr_t pa;
1504 struct cpu_info *ci = curcpu();
1505 int modified = 0;
1506
1507 DPRINTF(PDB_FOLLOW,
1508 ("pmap_clear_modify(%p)\n", (void *)VM_PAGE_TO_PHYS(pg)));
1509
1510 mtx_enter(&pg->mdpage.pv_mtx);
1511
1512 if (pg->pg_flags & PGF_ATTR_MOD) {
1513 atomic_clearbits_int(&pg->pg_flags, PGF_ATTR_MOD);
1514 modified = 1;
1515 }
1516
1517 pa = VM_PAGE_TO_PHYS(pg);
1518 for (pv = pg_to_pvh(pg); pv != NULL; pv = pv->pv_next) {
1519 if (pv->pv_pmap == pmap_kernel()) {
1520 #ifdef DIAGNOSTIC
1521 if (pv->pv_va < VM_MIN_KERNEL_ADDRESS ||
1522 pv->pv_va >= VM_MAX_KERNEL_ADDRESS)
1523 panic("pmap_clear_modify(%p)",
1524 (void *)pv->pv_va);
1525 #endif
1526 pte = kvtopte(pv->pv_va);
1527 entry = *pte;
1528 if ((entry & PG_V) != 0 && (entry & PG_M) != 0) {
1529 if (pg->pg_flags & PGF_CACHED)
1530 Mips_HitSyncDCachePage(ci, pv->pv_va,
1531 pfn_to_pad(entry));
1532 modified = 1;
1533 entry &= ~PG_M;
1534 *pte = entry;
1535 pmap_update_kernel_page(pv->pv_va, entry);
1536 pmap_shootdown_range(pmap_kernel(), pv->pv_va,
1537 pv->pv_va + PAGE_SIZE, 0);
1538 }
1539 } else if (pv->pv_pmap != NULL) {
1540 pte = pmap_pte_lookup(pv->pv_pmap, pv->pv_va);
1541 if (pte == NULL)
1542 continue;
1543 entry = *pte;
1544 if ((entry & PG_V) != 0 && (entry & PG_M) != 0) {
1545 if (pg->pg_flags & PGF_CACHED)
1546 Mips_SyncDCachePage(ci, pv->pv_va, pa);
1547 modified = 1;
1548 entry &= ~PG_M;
1549 *pte = entry;
1550 pmap_update_user_page(pv->pv_pmap, pv->pv_va,
1551 entry);
1552 pmap_shootdown_range(pv->pv_pmap, pv->pv_va,
1553 pv->pv_va + PAGE_SIZE, 0);
1554 }
1555 }
1556 }
1557
1558 mtx_leave(&pg->mdpage.pv_mtx);
1559
1560 return modified;
1561 }
1562
1563 /*
1564 * pmap_clear_reference:
1565 *
1566 * Clear the reference bit on the specified physical page.
1567 */
1568 int
pmap_clear_reference(struct vm_page * pg)1569 pmap_clear_reference(struct vm_page *pg)
1570 {
1571 int referenced;
1572
1573 DPRINTF(PDB_FOLLOW, ("pmap_clear_reference(%p)\n",
1574 (void *)VM_PAGE_TO_PHYS(pg)));
1575
1576 referenced = (pg->pg_flags & PGF_ATTR_REF) != 0;
1577 atomic_clearbits_int(&pg->pg_flags, PGF_ATTR_REF);
1578 return referenced;
1579 }
1580
1581 /*
1582 * pmap_is_referenced:
1583 *
1584 * Return whether or not the specified physical page is referenced
1585 * by any physical maps.
1586 */
1587 int
pmap_is_referenced(struct vm_page * pg)1588 pmap_is_referenced(struct vm_page *pg)
1589 {
1590 return (pg->pg_flags & PGF_ATTR_REF) != 0;
1591 }
1592
1593 /*
1594 * pmap_is_modified:
1595 *
1596 * Return whether or not the specified physical page is modified
1597 * by any physical maps.
1598 */
1599 int
pmap_is_modified(struct vm_page * pg)1600 pmap_is_modified(struct vm_page *pg)
1601 {
1602 return (pg->pg_flags & PGF_ATTR_MOD) != 0;
1603 }
1604
1605 /*
1606 * Miscellaneous support routines not part of the pmap API
1607 */
1608
1609 /*
1610 * Sets the modify bit on the page that contains virtual address va.
1611 * Returns 0 on success. If the page is mapped read-only, the bit is left
1612 * unchanged and the function returns non-zero.
1613 */
1614 int
pmap_emulate_modify(pmap_t pmap,vaddr_t va)1615 pmap_emulate_modify(pmap_t pmap, vaddr_t va)
1616 {
1617 pt_entry_t *pte, entry;
1618 paddr_t pa;
1619 vm_page_t pg;
1620 #ifdef MULTIPROCESSOR
1621 /* Keep back TLB shootdowns. */
1622 register_t sr = disableintr();
1623 pt_entry_t old_entry;
1624 #endif
1625 int readonly = 0;
1626
1627 if (pmap == pmap_kernel()) {
1628 pte = kvtopte(va);
1629 } else {
1630 pte = pmap_pte_lookup(pmap, va);
1631 if (pte == NULL)
1632 panic("%s: invalid page dir in pmap %p va %p", __func__,
1633 pmap, (void *)va);
1634 }
1635 entry = *pte;
1636 if (!(entry & PG_V) || (entry & PG_M)) {
1637 #ifdef MULTIPROCESSOR
1638 /* Another CPU might have changed the mapping. */
1639 if (pmap == pmap_kernel())
1640 pmap_update_kernel_page(trunc_page(va), entry);
1641 else
1642 pmap_update_user_page(pmap, trunc_page(va), entry);
1643 goto out;
1644 #else
1645 panic("%s: invalid pte 0x%lx in pmap %p va %p", __func__,
1646 (unsigned long)entry, pmap, (void *)va);
1647 #endif
1648 }
1649 if (entry & PG_RO) {
1650 readonly = 1;
1651 goto out;
1652 }
1653 #ifdef MULTIPROCESSOR
1654 old_entry = entry;
1655 entry |= PG_M;
1656 if (pmap_pte_cas(pte, old_entry, entry) != old_entry) {
1657 /* Refault to resolve the conflict. */
1658 goto out;
1659 }
1660 #else
1661 entry |= PG_M;
1662 *pte = entry;
1663 #endif
1664 if (pmap == pmap_kernel())
1665 pmap_update_kernel_page(trunc_page(va), entry);
1666 else
1667 pmap_update_user_page(pmap, trunc_page(va), entry);
1668 pa = pfn_to_pad(entry);
1669 pg = PHYS_TO_VM_PAGE(pa);
1670 if (pg == NULL)
1671 panic("%s: unmanaged page %p in pmap %p va %p", __func__,
1672 (void *)pa, pmap, (void *)va);
1673 atomic_setbits_int(&pg->pg_flags, PGF_ATTR_MOD | PGF_ATTR_REF);
1674 out:
1675 #ifdef MULTIPROCESSOR
1676 setsr(sr);
1677 #endif
1678 return readonly;
1679 }
1680
1681 /*
1682 * Walk the PV tree for a physical page and change all its
1683 * mappings to cached or uncached.
1684 */
1685 void
pmap_do_page_cache(vm_page_t pg,u_int mode)1686 pmap_do_page_cache(vm_page_t pg, u_int mode)
1687 {
1688 pv_entry_t pv;
1689 pt_entry_t *pte, entry;
1690 pt_entry_t newmode;
1691
1692 MUTEX_ASSERT_LOCKED(&pg->mdpage.pv_mtx);
1693
1694 DPRINTF(PDB_FOLLOW|PDB_ENTER, ("pmap_page_cache(%p)\n", pg));
1695
1696 newmode = mode & PGF_CACHED ? PG_CACHED : PG_UNCACHED;
1697
1698 for (pv = pg_to_pvh(pg); pv != NULL; pv = pv->pv_next) {
1699 if (pv->pv_pmap == pmap_kernel()) {
1700 #ifdef DIAGNOSTIC
1701 if (pv->pv_va < VM_MIN_KERNEL_ADDRESS ||
1702 pv->pv_va >= VM_MAX_KERNEL_ADDRESS)
1703 panic("pmap_page_cache(%p)", (void *)pv->pv_va);
1704 #endif
1705 pte = kvtopte(pv->pv_va);
1706 entry = *pte;
1707 if (entry & PG_V) {
1708 entry = (entry & ~PG_CACHEMODE) | newmode;
1709 *pte = entry;
1710 pmap_update_kernel_page(pv->pv_va, entry);
1711 pmap_shootdown_range(pmap_kernel(), pv->pv_va,
1712 pv->pv_va + PAGE_SIZE, 0);
1713 }
1714 } else if (pv->pv_pmap != NULL) {
1715 pte = pmap_pte_lookup(pv->pv_pmap, pv->pv_va);
1716 if (pte == NULL)
1717 continue;
1718 entry = *pte;
1719 if (entry & PG_V) {
1720 entry = (entry & ~PG_CACHEMODE) | newmode;
1721 *pte = entry;
1722 pmap_update_user_page(pv->pv_pmap, pv->pv_va,
1723 entry);
1724 pmap_shootdown_range(pv->pv_pmap, pv->pv_va,
1725 pv->pv_va + PAGE_SIZE, 0);
1726 }
1727 }
1728 }
1729 atomic_clearbits_int(&pg->pg_flags, PGF_CACHED | PGF_UNCACHED);
1730 atomic_setbits_int(&pg->pg_flags, mode);
1731 }
1732
1733 void
pmap_page_cache(vm_page_t pg,u_int mode)1734 pmap_page_cache(vm_page_t pg, u_int mode)
1735 {
1736 mtx_enter(&pg->mdpage.pv_mtx);
1737 pmap_do_page_cache(pg, mode);
1738 mtx_leave(&pg->mdpage.pv_mtx);
1739 }
1740
1741 /*
1742 * Allocate a hardware PID and return it.
1743 * It takes almost as much or more time to search the TLB for a
1744 * specific PID and flush those entries as it does to flush the entire TLB.
1745 * Therefore, when we allocate a new PID, we just take the next number. When
1746 * we run out of numbers, we flush the TLB, increment the generation count
1747 * and start over. PID zero is reserved for kernel use.
1748 * This is called only by switch().
1749 */
1750 uint
pmap_alloc_tlbpid(struct proc * p)1751 pmap_alloc_tlbpid(struct proc *p)
1752 {
1753 pmap_t pmap;
1754 uint id;
1755 struct cpu_info *ci = curcpu();
1756 u_long cpuid = ci->ci_cpuid;
1757
1758 pmap = p->p_vmspace->vm_map.pmap;
1759 if (pmap->pm_asid[cpuid].pma_asidgen !=
1760 pmap_asid_info[cpuid].pma_asidgen) {
1761 id = pmap_asid_info[cpuid].pma_asid;
1762 if (id >= PG_ASID_COUNT) {
1763 tlb_asid_wrap(ci);
1764 /* reserve tlbpid_gen == 0 to always mean invalid */
1765 if (++pmap_asid_info[cpuid].pma_asidgen == 0)
1766 pmap_asid_info[cpuid].pma_asidgen = 1;
1767 id = MIN_USER_ASID;
1768 }
1769 pmap_asid_info[cpuid].pma_asid = id + 1;
1770 pmap->pm_asid[cpuid].pma_asid = id;
1771 pmap->pm_asid[cpuid].pma_asidgen =
1772 pmap_asid_info[cpuid].pma_asidgen;
1773 } else {
1774 id = pmap->pm_asid[cpuid].pma_asid;
1775 }
1776
1777 if (curproc) {
1778 DPRINTF(PDB_FOLLOW|PDB_TLBPID,
1779 ("pmap_alloc_tlbpid: curproc %d '%s' ",
1780 curproc->p_p->ps_pid, curproc->p_p->ps_comm));
1781 } else {
1782 DPRINTF(PDB_FOLLOW|PDB_TLBPID,
1783 ("pmap_alloc_tlbpid: curproc <none> "));
1784 }
1785 DPRINTF(PDB_FOLLOW|PDB_TLBPID, ("segtab %p tlbpid %u pid %d '%s'\n",
1786 pmap->pm_segtab, id, p->p_p->ps_pid, p->p_p->ps_comm));
1787
1788 return (id);
1789 }
1790
1791 /*
1792 * Enter the pmap and virtual address into the physical to virtual map table.
1793 */
1794 int
pmap_enter_pv(pmap_t pmap,vaddr_t va,vm_page_t pg,pt_entry_t * npte)1795 pmap_enter_pv(pmap_t pmap, vaddr_t va, vm_page_t pg, pt_entry_t *npte)
1796 {
1797 pv_entry_t pv, npv;
1798
1799 MUTEX_ASSERT_LOCKED(&pg->mdpage.pv_mtx);
1800
1801 pv = pg_to_pvh(pg);
1802 if (pv->pv_pmap == NULL) {
1803 /*
1804 * No entries yet, use header as the first entry
1805 */
1806
1807 DPRINTF(PDB_PVENTRY,
1808 ("pmap_enter: first pv: pmap %p va %p pa %p\n",
1809 pmap, (void *)va, (void *)VM_PAGE_TO_PHYS(pg)));
1810
1811 stat_count(enter_stats.firstpv);
1812
1813 pv->pv_va = va;
1814 if (*npte & PG_CACHED)
1815 atomic_setbits_int(&pg->pg_flags, PGF_CACHED);
1816 if (*npte & PG_UNCACHED)
1817 atomic_setbits_int(&pg->pg_flags, PGF_UNCACHED);
1818 pv->pv_pmap = pmap;
1819 pv->pv_next = NULL;
1820 } else {
1821 /*
1822 * There is at least one other VA mapping this page.
1823 * We'll place this entry after the header.
1824 */
1825
1826 if ((pg->pg_flags & PGF_CACHED) == 0) {
1827 /*
1828 * If page is not mapped cached it's either because
1829 * an uncached mapping was explicitly requested or
1830 * we have a VAC situation.
1831 * Map this page uncached as well.
1832 */
1833 *npte = (*npte & ~PG_CACHEMODE) | PG_UNCACHED;
1834 }
1835
1836 /*
1837 * The entry may already be in the list if
1838 * we are only changing the protection bits.
1839 */
1840 for (npv = pv; npv; npv = npv->pv_next) {
1841 if (pmap == npv->pv_pmap && va == npv->pv_va)
1842 return 0;
1843 }
1844
1845 DPRINTF(PDB_PVENTRY,
1846 ("pmap_enter: new pv: pmap %p va %p pg %p\n",
1847 pmap, (void *)va, (void *)VM_PAGE_TO_PHYS(pg)));
1848
1849 npv = pmap_pv_alloc();
1850 if (npv == NULL)
1851 return ENOMEM;
1852
1853 if (cache_valias_mask != 0 && (*npte & PG_CACHED) != 0 &&
1854 (pg->pg_flags & PGF_CACHED) != 0) {
1855 /*
1856 * We have a VAC possibility. Check if virtual
1857 * address of current mappings are compatible
1858 * with this new mapping. Only need to check first
1859 * since all others have been checked compatible
1860 * when added. If they are incompatible, update
1861 * all mappings to be mapped uncached, flush the
1862 * cache and set page to not be mapped cached.
1863 */
1864 if (((pv->pv_va ^ va) & cache_valias_mask) != 0) {
1865 #ifdef PMAPDEBUG
1866 printf("%s: uncaching page pa %p, va %p/%p\n",
1867 __func__, (void *)VM_PAGE_TO_PHYS(pg),
1868 (void *)pv->pv_va, (void *)va);
1869 #endif
1870 pmap_do_page_cache(pg, 0);
1871 Mips_SyncDCachePage(curcpu(), pv->pv_va,
1872 VM_PAGE_TO_PHYS(pg));
1873 *npte = (*npte & ~PG_CACHEMODE) | PG_UNCACHED;
1874 }
1875 }
1876
1877 npv->pv_va = va;
1878 npv->pv_pmap = pmap;
1879 npv->pv_next = pv->pv_next;
1880 pv->pv_next = npv;
1881
1882 if (!npv->pv_next)
1883 stat_count(enter_stats.secondpv);
1884 }
1885
1886 return 0;
1887 }
1888
1889 /*
1890 * Remove a physical to virtual address translation from the PV table.
1891 */
1892 void
pmap_remove_pv(pmap_t pmap,vaddr_t va,paddr_t pa)1893 pmap_remove_pv(pmap_t pmap, vaddr_t va, paddr_t pa)
1894 {
1895 pv_entry_t pv, npv;
1896 vm_page_t pg;
1897
1898 DPRINTF(PDB_FOLLOW|PDB_PVENTRY,
1899 ("pmap_remove_pv(%p, %p, %p)\n", pmap, (void *)va, (void *)pa));
1900
1901 /*
1902 * Remove page from the PV table
1903 */
1904 pg = PHYS_TO_VM_PAGE(pa);
1905 if (pg == NULL)
1906 return;
1907
1908 mtx_enter(&pg->mdpage.pv_mtx);
1909
1910 /*
1911 * If we are removing the first entry on the list, copy up
1912 * the next entry, if any, and free that pv item since the
1913 * first root item can't be freed. Else walk the list.
1914 */
1915 pv = pg_to_pvh(pg);
1916 if (pmap == pv->pv_pmap && va == pv->pv_va) {
1917 npv = pv->pv_next;
1918 if (npv) {
1919 *pv = *npv;
1920 pmap_pv_free(npv);
1921 } else {
1922 pv->pv_pmap = NULL;
1923 atomic_clearbits_int(&pg->pg_flags,
1924 PG_PMAPMASK & ~PGF_PRESERVE);
1925 }
1926 stat_count(remove_stats.pvfirst);
1927 } else {
1928 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) {
1929 stat_count(remove_stats.pvsearch);
1930 if (pmap == npv->pv_pmap && va == npv->pv_va)
1931 break;
1932 }
1933 if (npv != NULL) {
1934 pv->pv_next = npv->pv_next;
1935 pmap_pv_free(npv);
1936 } else {
1937 #ifdef DIAGNOSTIC
1938 panic("pmap_remove_pv(%p, %p, %p) not found",
1939 pmap, (void *)va, (void *)pa);
1940 #endif
1941 }
1942 }
1943
1944 if (cache_valias_mask != 0 && pv->pv_pmap != NULL &&
1945 (pg->pg_flags & (PGF_CACHED | PGF_UNCACHED)) == 0) {
1946 /*
1947 * If this page had been mapped uncached due to aliasing,
1948 * check if it can be mapped cached again after the current
1949 * entry's removal.
1950 */
1951 pv = pg_to_pvh(pg);
1952 va = pv->pv_va;
1953 for (pv = pv->pv_next; pv != NULL; pv = pv->pv_next) {
1954 if (((pv->pv_va ^ va) & cache_valias_mask) != 0)
1955 break;
1956 }
1957
1958 if (pv == NULL) {
1959 #ifdef PMAPDEBUG
1960 printf("%s: caching page pa %p, va %p again\n",
1961 __func__, (void *)VM_PAGE_TO_PHYS(pg), (void *)va);
1962 #endif
1963 pmap_do_page_cache(pg, PGF_CACHED);
1964 }
1965 }
1966
1967 mtx_leave(&pg->mdpage.pv_mtx);
1968 }
1969
1970 /*
1971 * Allocator for smaller-than-a-page structures pool (pm_segtab, and
1972 * second level page tables). Pages backing this poll are mapped in
1973 * XKPHYS to avoid additional page faults when servicing a TLB miss.
1974 */
1975
1976 void *
pmap_pg_alloc(struct pool * pp,int flags,int * slowdown)1977 pmap_pg_alloc(struct pool *pp, int flags, int *slowdown)
1978 {
1979 vm_page_t pg;
1980
1981 *slowdown = 0;
1982 for (;;) {
1983 pg = uvm_pagealloc(NULL, 0, NULL,
1984 UVM_PGA_USERESERVE | UVM_PGA_ZERO);
1985 if (pg != NULL)
1986 break;
1987
1988 *slowdown = 1;
1989 if (flags & PR_WAITOK)
1990 uvm_wait(__func__);
1991 else
1992 break;
1993 }
1994
1995 if (pg != NULL)
1996 return (void *)PHYS_TO_XKPHYS(VM_PAGE_TO_PHYS(pg), CCA_CACHED);
1997 else
1998 return NULL;
1999 }
2000
2001 void
pmap_pg_free(struct pool * pp,void * item)2002 pmap_pg_free(struct pool *pp, void *item)
2003 {
2004 vaddr_t va = (vaddr_t)item;
2005 paddr_t pa = XKPHYS_TO_PHYS(va);
2006 vm_page_t pg = PHYS_TO_VM_PAGE(pa);
2007
2008 if (cache_valias_mask)
2009 Mips_HitSyncDCachePage(curcpu(), va, pa);
2010 uvm_pagefree(pg);
2011 }
2012
2013 void
pmap_proc_iflush(struct process * pr,vaddr_t va,vsize_t len)2014 pmap_proc_iflush(struct process *pr, vaddr_t va, vsize_t len)
2015 {
2016 #ifdef MULTIPROCESSOR
2017 struct pmap *pmap = vm_map_pmap(&pr->ps_vmspace->vm_map);
2018 CPU_INFO_ITERATOR cii;
2019 struct cpu_info *ci;
2020
2021 CPU_INFO_FOREACH(cii, ci) {
2022 if (ci->ci_curpmap == pmap) {
2023 Mips_InvalidateICache(ci, va, len);
2024 break;
2025 }
2026 }
2027 #else
2028 Mips_InvalidateICache(curcpu(), va, len);
2029 #endif
2030 }
2031
2032 #ifdef __HAVE_PMAP_DIRECT
2033 vaddr_t
pmap_map_direct(vm_page_t pg)2034 pmap_map_direct(vm_page_t pg)
2035 {
2036 paddr_t pa = VM_PAGE_TO_PHYS(pg);
2037 vaddr_t va;
2038
2039 va = PHYS_TO_XKPHYS(pa, CCA_CACHED);
2040
2041 return va;
2042 }
2043
2044 vm_page_t
pmap_unmap_direct(vaddr_t va)2045 pmap_unmap_direct(vaddr_t va)
2046 {
2047 paddr_t pa;
2048 vm_page_t pg;
2049
2050 pa = XKPHYS_TO_PHYS(va);
2051 pg = PHYS_TO_VM_PAGE(pa);
2052 if (cache_valias_mask)
2053 Mips_HitSyncDCachePage(curcpu(), va, pa);
2054
2055 return pg;
2056 }
2057 #endif
2058
2059 void
pmap_update(struct pmap * pmap)2060 pmap_update(struct pmap *pmap)
2061 {
2062 Mips_SyncICache(curcpu());
2063 }
2064
2065 /*
2066 * Read an instruction from a given virtual memory address.
2067 * TLB read-inhibition is bypassed.
2068 */
2069 int
pmap_copyinsn(pmap_t pmap,vaddr_t uva,uint32_t * insn)2070 pmap_copyinsn(pmap_t pmap, vaddr_t uva, uint32_t *insn)
2071 {
2072 pt_entry_t *pte;
2073 paddr_t pa;
2074 int found = 0;
2075
2076 if (uva >= VM_MAXUSER_ADDRESS || pmap == pmap_kernel()) {
2077 panic("%s(%p, %p): invalid params", __func__,
2078 pmap, (void *)uva);
2079 }
2080
2081 /*
2082 * Read the instruction through the direct map region.
2083 *
2084 * The pmap lock prevents other threads from changing the mapping
2085 * and repurposing the page frame while this thread accesses the
2086 * direct map.
2087 */
2088 pmap_lock(pmap);
2089 pte = pmap_pte_lookup(pmap, uva);
2090 if (pte != NULL && (*pte & PG_V) != 0 && (*pte & pg_xi) == 0) {
2091 pa = pfn_to_pad(*pte) | (uva & PAGE_MASK);
2092 *insn = *(uint32_t *)PHYS_TO_XKPHYS(pa, CCA_CACHED);
2093 found = 1;
2094 }
2095 pmap_unlock(pmap);
2096
2097 return found;
2098 }
2099