1 /* $NetBSD: pmap.c,v 1.42 2022/04/09 23:38:32 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center and by Chris G. Demetriou.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*-
34 * Copyright (c) 1991 Regents of the University of California.
35 * All rights reserved.
36 * Copyright (c) 1994 John S. Dyson
37 * All rights reserved.
38 * Copyright (c) 1994 David Greenman
39 * All rights reserved.
40 * Copyright (c) 1998,2000 Doug Rabson
41 * All rights reserved.
42 *
43 * This code is derived from software contributed to Berkeley by
44 * the Systems Programming Group of the University of Utah Computer
45 * Science Department and William Jolitz of UUNET Technologies Inc.
46 *
47 * Redistribution and use in source and binary forms, with or without
48 * modification, are permitted provided that the following conditions
49 * are met:
50 * 1. Redistributions of source code must retain the above copyright
51 * notice, this list of conditions and the following disclaimer.
52 * 2. Redistributions in binary form must reproduce the above copyright
53 * notice, this list of conditions and the following disclaimer in the
54 * documentation and/or other materials provided with the distribution.
55 * 3. All advertising materials mentioning features or use of this software
56 * must display the following acknowledgement:
57 * This product includes software developed by the University of
58 * California, Berkeley and its contributors.
59 * 4. Neither the name of the University nor the names of its contributors
60 * may be used to endorse or promote products derived from this software
61 * without specific prior written permission.
62 *
63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
73 * SUCH DAMAGE.
74 *
75 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
76 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp
77 * with some ideas from NetBSD's alpha pmap
78 */
79
80 /* __FBSDID("$FreeBSD: src/sys/ia64/ia64/pmap.c,v 1.172 2005/11/20 06:09:48 alc Exp $"); */
81
82 #include <sys/cdefs.h>
83
84 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.42 2022/04/09 23:38:32 riastradh Exp $");
85
86 #include <sys/param.h>
87 #include <sys/atomic.h>
88 #include <sys/systm.h>
89 #include <sys/buf.h>
90 #include <sys/reboot.h>
91 #include <sys/lock.h>
92 #include <sys/pool.h>
93 #include <sys/sched.h>
94 #include <sys/bitops.h>
95
96 #include <uvm/uvm.h>
97 #include <uvm/uvm_physseg.h>
98
99 #include <machine/pal.h>
100 #include <machine/atomic.h>
101 #include <machine/pte.h>
102 #include <machine/cpufunc.h>
103 #include <machine/md_var.h>
104 #include <machine/vmparam.h>
105
106 /*
107 * Manages physical address maps.
108 *
109 * Since the information managed by this module is
110 * also stored by the logical address mapping module,
111 * this module may throw away valid virtual-to-physical
112 * mappings at almost any time. However, invalidations
113 * of virtual-to-physical mappings must be done as
114 * requested.
115 *
116 * In order to cope with hardware architectures which
117 * make virtual-to-physical map invalidates expensive,
118 * this module may delay invalidate or reduced protection
119 * operations until such time as they are actually
120 * necessary. This module is given full information as
121 * to which processors are currently using which maps,
122 * and to when physical maps must be made correct.
123 */
124
125 /*
126 * Following the Linux model, region IDs are allocated in groups of
127 * eight so that a single region ID can be used for as many RRs as we
128 * want by encoding the RR number into the low bits of the ID.
129 *
130 * We reserve region ID 0 for the kernel and allocate the remaining
131 * IDs for user pmaps.
132 *
133 * Region 0-3: User virtually mapped
134 * Region 4: PBVM and special mappings
135 * Region 5: Kernel virtual memory
136 * Region 6: Direct-mapped uncacheable
137 * Region 7: Direct-mapped cacheable
138 */
139
140 #if !defined(DIAGNOSTIC)
141 #define PMAP_INLINE __inline
142 #else
143 #define PMAP_INLINE
144 #endif
145
146 #ifdef PV_STATS
147 #define PV_STAT(x) do { x ; } while (0)
148 #else
149 #define PV_STAT(x) do { } while (0)
150 #endif
151
152 #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED)
153 #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY)
154 #define pmap_exec(lpte) ((lpte)->pte & PTE_AR_RX)
155 #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED)
156 #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK)
157 #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT)
158 #define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56)
159 #define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED)
160
161 #define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED
162 #define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY
163 #define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT
164 #define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED
165
166 #define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED
167
168 /*
169 * Individual PV entries are stored in per-pmap chunks. This saves
170 * space by eliminating the need to record the pmap within every PV
171 * entry.
172 */
173 #if PAGE_SIZE == 8192
174 #define _NPCM 6
175 #define _NPCPV 337
176 #define _NPCS 2
177 #elif PAGE_SIZE == 16384
178 #define _NPCM 11
179 #define _NPCPV 677
180 #define _NPCS 1
181 #else
182 #error "invalid page size"
183 #endif
184
185 struct pv_chunk {
186 pmap_t pc_pmap;
187 TAILQ_ENTRY(pv_chunk) pc_list;
188 u_long pc_map[_NPCM]; /* bitmap; 1 = free */
189 TAILQ_ENTRY(pv_chunk) pc_lru;
190 u_long pc_spare[_NPCS];
191 struct pv_entry pc_pventry[_NPCPV];
192 };
193
194 /*
195 * The VHPT bucket head structure.
196 */
197 struct ia64_bucket {
198 uint64_t chain;
199 kmutex_t mutex;
200 u_int length;
201 };
202
203 /*
204 * Statically allocated kernel pmap
205 */
206 static struct pmap kernel_pmap_store;/* the kernel's pmap (proc0) */
207 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store;
208
209 vaddr_t virtual_avail; /* VA of first avail page (after kernel bss) */
210 vaddr_t virtual_end; /* VA of last avail page (end of kernel AS) */
211
212 /* XXX freebsd, needs to be sorted out */
213 #define kernel_pmap pmap_kernel()
214 #define critical_enter() kpreempt_disable()
215 #define critical_exit() kpreempt_enable()
216 /* flags the entire page as dirty */
217 #define vm_page_dirty(page) (page->flags &= ~PG_CLEAN)
218 #define vm_page_is_managed(page) (pmap_initialized && uvm_pageismanaged(VM_PAGE_TO_PHYS(page)))
219
220 /*
221 * Kernel virtual memory management.
222 */
223 static int nkpt;
224
225 extern struct ia64_lpte ***ia64_kptdir;
226
227 #define KPTE_DIR0_INDEX(va) \
228 (((va) >> (3*PAGE_SHIFT-8)) & ((1<<(PAGE_SHIFT-3))-1))
229 #define KPTE_DIR1_INDEX(va) \
230 (((va) >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
231 #define KPTE_PTE_INDEX(va) \
232 (((va) >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1))
233
234 #define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte))
235
236 vaddr_t kernel_vm_end;
237
238 /* Defaults for ptc.e. */
239 /*
240 static uint64_t pmap_ptc_e_base = 0;
241 static uint32_t pmap_ptc_e_count1 = 1;
242 static uint32_t pmap_ptc_e_count2 = 1;
243 static uint32_t pmap_ptc_e_stride1 = 0;
244 static uint32_t pmap_ptc_e_stride2 = 0;
245 */
246 /* Values for ptc.e. XXX values for SKI, add SKI kernel option methinks */
247 static uint64_t pmap_ptc_e_base = 0x100000000;
248 static uint64_t pmap_ptc_e_count1 = 3;
249 static uint64_t pmap_ptc_e_count2 = 2;
250 static uint64_t pmap_ptc_e_stride1 = 0x2000;
251 static uint64_t pmap_ptc_e_stride2 = 0x100000000;
252
253 kmutex_t pmap_ptc_mutex;
254
255 /*
256 * Data for the RID allocator
257 */
258 static int pmap_ridcount;
259 static int pmap_rididx;
260 static int pmap_ridmapsz;
261 static int pmap_ridmax;
262 static uint64_t *pmap_ridmap;
263 kmutex_t pmap_ridmutex;
264
265 static krwlock_t pvh_global_lock __attribute__ ((aligned (128)));
266
267 static pool_cache_t pmap_pool_cache;
268
269 /*
270 * Data for the pv entry allocation mechanism
271 */
272 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
273 static int pv_entry_count;
274
275 /*
276 * Data for allocating PTEs for user processes.
277 */
278 static pool_cache_t pte_pool_cache;
279
280 struct ia64_bucket *pmap_vhpt_bucket;
281
282 /* XXX For freebsd, these are sysctl variables */
283 static uint64_t pmap_vhpt_nbuckets = 0;
284 uint64_t pmap_vhpt_log2size = 0;
285 static uint64_t pmap_vhpt_inserts = 0;
286
287 static bool pmap_initialized = false; /* Has pmap_init completed? */
288 static uint64_t pmap_pages_stolen = 0; /* instrumentation */
289
290 static struct ia64_lpte *pmap_find_vhpt(vaddr_t va);
291
292 static void free_pv_chunk(struct pv_chunk *pc);
293 static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
294 static pv_entry_t get_pv_entry(pmap_t pmap, bool try);
295 static struct vm_page *pmap_pv_reclaim(pmap_t locked_pmap);
296
297 static void pmap_free_pte(struct ia64_lpte *pte, vaddr_t va);
298 static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte,
299 vaddr_t va, pv_entry_t pv, int freepte);
300 static int pmap_remove_vhpt(vaddr_t va);
301
302 static vaddr_t pmap_steal_vhpt_memory(vsize_t);
303
304 static struct vm_page *vm_page_alloc1(void);
305 static void vm_page_free1(struct vm_page *pg);
306
307 static vm_memattr_t pmap_flags_to_memattr(u_int flags);
308
309 #if DEBUG
310 static void pmap_testout(void);
311 #endif
312
313 static void
pmap_initialize_vhpt(vaddr_t vhpt)314 pmap_initialize_vhpt(vaddr_t vhpt)
315 {
316 struct ia64_lpte *pte;
317 u_int i;
318
319 pte = (struct ia64_lpte *)vhpt;
320 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
321 pte[i].pte = 0;
322 pte[i].itir = 0;
323 pte[i].tag = 1UL << 63; /* Invalid tag */
324 pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
325 }
326 }
327
328 #ifdef MULTIPROCESSOR
329 vaddr_t
pmap_alloc_vhpt(void)330 pmap_alloc_vhpt(void)
331 {
332 vaddr_t vhpt;
333 struct vm_page *m;
334 vsize_t size;
335
336 size = 1UL << pmap_vhpt_log2size;
337 m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
338 VM_ALLOC_WIRED, atop(size), 0UL, ~0UL, size, 0UL,
339 VM_MEMATTR_DEFAULT);
340 if (m != NULL) {
341 vhpt = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
342 pmap_initialize_vhpt(vhpt);
343 return (vhpt);
344 }
345 return (0);
346 }
347 #endif
348
349 /*
350 * Bootstrap the system enough to run with virtual memory.
351 */
352 void
pmap_bootstrap(void)353 pmap_bootstrap(void)
354 {
355 struct ia64_pal_result res;
356 vaddr_t base;
357 size_t size;
358 vsize_t bufsz;
359 int i, ridbits;
360
361 /*
362 * Query the PAL Code to find the loop parameters for the
363 * ptc.e instruction.
364 */
365 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0);
366 if (res.pal_status != 0)
367 panic("Can't configure ptc.e parameters");
368 pmap_ptc_e_base = res.pal_result[0];
369 pmap_ptc_e_count1 = res.pal_result[1] >> 32;
370 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1);
371 pmap_ptc_e_stride1 = res.pal_result[2] >> 32;
372 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1);
373 if (bootverbose)
374 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, "
375 "stride1=0x%lx, stride2=0x%lx\n",
376 pmap_ptc_e_base,
377 pmap_ptc_e_count1,
378 pmap_ptc_e_count2,
379 pmap_ptc_e_stride1,
380 pmap_ptc_e_stride2);
381
382 mutex_init(&pmap_ptc_mutex, MUTEX_DEFAULT, IPL_VM);
383
384 /*
385 * Setup RIDs. RIDs 0..7 are reserved for the kernel.
386 *
387 * We currently need at least 19 bits in the RID because PID_MAX
388 * can only be encoded in 17 bits and we need RIDs for 4 regions
389 * per process. With PID_MAX equalling 99999 this means that we
390 * need to be able to encode 399996 (=4*PID_MAX).
391 * The Itanium processor only has 18 bits and the architected
392 * minimum is exactly that. So, we cannot use a PID based scheme
393 * in those cases. Enter pmap_ridmap...
394 * We should avoid the map when running on a processor that has
395 * implemented enough bits. This means that we should pass the
396 * process/thread ID to pmap. This we currently don't do, so we
397 * use the map anyway. However, we don't want to allocate a map
398 * that is large enough to cover the range dictated by the number
399 * of bits in the RID, because that may result in a RID map of
400 * 2MB in size for a 24-bit RID. A 64KB map is enough.
401 * The bottomline: we create a 32KB map when the processor only
402 * implements 18 bits (or when we can't figure it out). Otherwise
403 * we create a 64KB map.
404 */
405 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
406 if (res.pal_status != 0) {
407 if (bootverbose)
408 printf("Can't read VM Summary - assuming 18 Region ID bits\n");
409 ridbits = 18; /* guaranteed minimum */
410 } else {
411 ridbits = (res.pal_result[1] >> 8) & 0xff;
412 if (bootverbose)
413 printf("Processor supports %d Region ID bits\n",
414 ridbits);
415 }
416 if (ridbits > 19)
417 ridbits = 19;
418
419 pmap_ridmax = (1 << ridbits);
420 pmap_ridmapsz = pmap_ridmax / 64;
421 pmap_ridmap = (uint64_t *)uvm_pageboot_alloc(pmap_ridmax / 8);
422 pmap_ridmap[0] |= 0xff;
423 pmap_rididx = 0;
424 pmap_ridcount = 8;
425 mutex_init(&pmap_ridmutex, MUTEX_DEFAULT, IPL_VM);
426
427 /*
428 * Compute the number of pages kmem_map will have.
429 */
430 kmeminit_nkmempages();
431
432 /*
433 * Figure out how many initial PTE's are necessary to map the
434 * kernel. We also reserve space for kmem_alloc_pageable()
435 * for vm_fork().
436 */
437
438 /* Get size of buffer cache and set an upper limit */
439 bufsz = buf_memcalc();
440 buf_setvalimit(bufsz);
441
442 nkpt = (((ubc_nwins << ubc_winshift) +
443 bufsz + 16 * NCARGS + pager_map_size) / PAGE_SIZE +
444 USRIOSIZE + (maxproc * UPAGES) + nkmempages) / NKPTEPG;
445
446 /*
447 * Allocate some memory for initial kernel 'page tables'.
448 */
449 ia64_kptdir = (void *)uvm_pageboot_alloc(PAGE_SIZE);
450 memset((void *)ia64_kptdir, 0, PAGE_SIZE);
451 nkpt = 0;
452 kernel_vm_end = VM_INIT_KERNEL_ADDRESS;
453
454 /*
455 * Determine a valid (mappable) VHPT size.
456 */
457 if (pmap_vhpt_log2size == 0)
458 pmap_vhpt_log2size = 20;
459 else if (pmap_vhpt_log2size < 16)
460 pmap_vhpt_log2size = 16;
461 else if (pmap_vhpt_log2size > 28)
462 pmap_vhpt_log2size = 28;
463 if (pmap_vhpt_log2size & 1)
464 pmap_vhpt_log2size--;
465
466 size = 1UL << pmap_vhpt_log2size;
467 /* XXX add some retries here */
468 base = pmap_steal_vhpt_memory(size);
469
470 curcpu()->ci_vhpt = base;
471
472 if (base == 0)
473 panic("Unable to allocate VHPT");
474
475 pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
476 pmap_vhpt_bucket = (void *)uvm_pageboot_alloc(pmap_vhpt_nbuckets *
477 sizeof(struct ia64_bucket));
478 if (bootverbose)
479 printf("VHPT: address=%#lx, size=%#lx, buckets=%ld, address=%lx\n",
480 base, size, pmap_vhpt_nbuckets, (long unsigned int)&pmap_vhpt_bucket[0]);
481
482 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
483 /* Stolen memory is zeroed. */
484 mutex_init(&pmap_vhpt_bucket[i].mutex, MUTEX_DEFAULT, IPL_VM);
485 }
486
487 pmap_initialize_vhpt(base);
488 map_vhpt(base);
489 ia64_set_pta(base + (1 << 8) + (pmap_vhpt_log2size << 2) + 1);
490 ia64_srlz_i();
491
492 /* XXX
493 virtual_avail = VM_INIT_KERNEL_ADDRESS;
494 virtual_end = VM_MAX_KERNEL_ADDRESS;
495 */
496
497 /*
498 * Initialize the kernel pmap (which is statically allocated).
499 */
500 PMAP_LOCK_INIT(kernel_pmap);
501 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
502 kernel_pmap->pm_rid[i] = 0;
503 TAILQ_INIT(&kernel_pmap->pm_pvchunk);
504
505 bzero(&kernel_pmap->pm_stats, sizeof kernel_pmap->pm_stats);
506 kernel_pmap->pm_refcount = 1;
507
508 curcpu()->ci_pmap = kernel_pmap;
509
510 /*
511 * Initialize the global pv list lock.
512 */
513 rw_init(&pvh_global_lock);
514
515 /* Region 5 is mapped via the VHPT. */
516 ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1);
517
518 /*
519 * Clear out any random TLB entries left over from booting.
520 */
521 pmap_invalidate_all();
522
523 map_gateway_page();
524 }
525
526 vaddr_t
pmap_page_to_va(struct vm_page * m)527 pmap_page_to_va(struct vm_page *m)
528 {
529 paddr_t pa;
530 vaddr_t va;
531
532 pa = VM_PAGE_TO_PHYS(m);
533 va = (m->mdpage.memattr == VM_MEMATTR_UNCACHEABLE) ? IA64_PHYS_TO_RR6(pa) :
534 IA64_PHYS_TO_RR7(pa);
535 return (va);
536 }
537
538 /***************************************************
539 * Manipulate TLBs for a pmap
540 ***************************************************/
541
542 static void
pmap_invalidate_page(vaddr_t va)543 pmap_invalidate_page(vaddr_t va)
544 {
545 struct ia64_lpte *pte;
546 //struct pcpu *pc;
547 uint64_t tag;
548 u_int vhpt_ofs;
549 struct cpu_info *ci;
550 CPU_INFO_ITERATOR cii;
551
552 critical_enter();
553
554 vhpt_ofs = ia64_thash(va) - curcpu()->ci_vhpt;
555
556 tag = ia64_ttag(va);
557
558 for (CPU_INFO_FOREACH(cii,ci)) {
559 pte = (struct ia64_lpte *)(ci->ci_vhpt + vhpt_ofs);
560 atomic_cmpset_64(&pte->tag, tag, 1UL << 63);
561 }
562
563 mutex_spin_enter(&pmap_ptc_mutex);
564
565 ia64_ptc_ga(va, PAGE_SHIFT << 2);
566 ia64_mf();
567 ia64_srlz_i();
568
569 mutex_spin_exit(&pmap_ptc_mutex);
570
571 ia64_invala();
572
573 critical_exit();
574 }
575
576 void
pmap_invalidate_all(void)577 pmap_invalidate_all(void)
578 {
579 uint64_t addr;
580 int i, j;
581
582 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
583
584 addr = pmap_ptc_e_base;
585 for (i = 0; i < pmap_ptc_e_count1; i++) {
586 for (j = 0; j < pmap_ptc_e_count2; j++) {
587 ia64_ptc_e(addr);
588 addr += pmap_ptc_e_stride2;
589 }
590 addr += pmap_ptc_e_stride1;
591 }
592 ia64_srlz_i();
593 }
594
595 static uint32_t
pmap_allocate_rid(void)596 pmap_allocate_rid(void)
597 {
598 uint64_t bit, bits;
599 int rid;
600
601 mutex_enter(&pmap_ridmutex);
602
603 if (pmap_ridcount == pmap_ridmax)
604 panic("pmap_allocate_rid: All Region IDs used");
605
606 /* Find an index with a free bit. */
607 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) {
608 pmap_rididx++;
609 if (pmap_rididx == pmap_ridmapsz)
610 pmap_rididx = 0;
611 }
612 rid = pmap_rididx * 64;
613
614 /* Find a free bit. */
615 bit = 1UL;
616 while (bits & bit) {
617 rid++;
618 bit <<= 1;
619 }
620
621 pmap_ridmap[pmap_rididx] |= bit;
622 pmap_ridcount++;
623
624 mutex_exit(&pmap_ridmutex);
625
626 return rid;
627 }
628
629 static void
pmap_free_rid(uint32_t rid)630 pmap_free_rid(uint32_t rid)
631 {
632 uint64_t bit;
633 int idx;
634
635 idx = rid / 64;
636 bit = ~(1UL << (rid & 63));
637
638 mutex_enter(&pmap_ridmutex);
639 pmap_ridmap[idx] &= bit;
640 pmap_ridcount--;
641
642 mutex_exit(&pmap_ridmutex);
643 }
644
645 /***************************************************
646 * Page table page management routines.....
647 ***************************************************/
648 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
649
650 static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)651 pv_to_chunk(pv_entry_t pv)
652 {
653 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
654 }
655
656 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
657
658 #define PC_FREE_FULL 0xfffffffffffffffful
659 #define PC_FREE_PARTIAL \
660 ((1UL << (_NPCPV - sizeof(u_long) * 8 * (_NPCM - 1))) - 1)
661
662 #if PAGE_SIZE == 8192
663 static const u_long pc_freemask[_NPCM] = {
664 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
665 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_PARTIAL
666 };
667 #elif PAGE_SIZE == 16384
668 static const u_long pc_freemask[_NPCM] = {
669 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
670 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
671 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
672 PC_FREE_FULL, PC_FREE_PARTIAL
673 };
674 #endif
675
676 #ifdef PV_STATS
677 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
678 static long pv_entry_frees, pv_entry_allocs;
679 static int pv_entry_spare;
680 #endif
681
682 /*
683 * We are in a serious low memory condition. Resort to
684 * drastic measures to free some pages so we can allocate
685 * another pv entry chunk.
686 */
687 static struct vm_page
pmap_pv_reclaim(pmap_t locked_pmap)688 *pmap_pv_reclaim(pmap_t locked_pmap)
689 {
690 struct pch newtail;
691 struct pv_chunk *pc;
692 struct ia64_lpte *pte;
693 pmap_t pmap;
694 pv_entry_t pv;
695 vaddr_t va;
696 struct vm_page *m;
697 struct vm_page *m_pc;
698 u_long inuse;
699 int bit, field, freed, idx;
700
701 PMAP_LOCK_ASSERT(locked_pmap);
702 pmap = NULL;
703 m_pc = NULL;
704 TAILQ_INIT(&newtail);
705 while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL) {
706 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
707 if (pmap != pc->pc_pmap) {
708 if (pmap != NULL) {
709 if (pmap != locked_pmap) {
710 pmap_switch(locked_pmap);
711 PMAP_UNLOCK(pmap);
712 }
713 }
714 pmap = pc->pc_pmap;
715 /* Avoid deadlock and lock recursion. */
716 if (pmap > locked_pmap)
717 PMAP_LOCK(pmap);
718 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
719 pmap = NULL;
720 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
721 continue;
722 }
723 pmap_switch(pmap);
724 }
725
726 /*
727 * Destroy every non-wired, 8 KB page mapping in the chunk.
728 */
729 freed = 0;
730 for (field = 0; field < _NPCM; field++) {
731 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
732 inuse != 0; inuse &= ~(1UL << bit)) {
733 bit = ffs64(inuse) - 1;
734 idx = field * sizeof(inuse) * NBBY + bit;
735 pv = &pc->pc_pventry[idx];
736 va = pv->pv_va;
737 pte = pmap_find_vhpt(va);
738 KASSERTMSG(pte != NULL, "pte");
739 if (pmap_wired(pte))
740 continue;
741 pmap_remove_vhpt(va);
742 pmap_invalidate_page(va);
743 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
744 if (pmap_dirty(pte))
745 vm_page_dirty(m);
746 pmap_free_pte(pte, va);
747 TAILQ_REMOVE(&m->mdpage.pv_list, pv, pv_list);
748 /* XXX
749 if (TAILQ_EMPTY(&m->mdpage.pv_list))
750 //vm_page_aflag_clear(m, PGA_WRITEABLE);
751 m->flags |= PG_RDONLY;
752 */
753 pc->pc_map[field] |= 1UL << bit;
754 freed++;
755 }
756 }
757 if (freed == 0) {
758 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
759 continue;
760 }
761 /* Every freed mapping is for a 8 KB page. */
762 pmap->pm_stats.resident_count -= freed;
763 PV_STAT(pv_entry_frees += freed);
764 PV_STAT(pv_entry_spare += freed);
765 pv_entry_count -= freed;
766 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
767 for (field = 0; field < _NPCM; field++)
768 if (pc->pc_map[field] != pc_freemask[field]) {
769 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
770 pc_list);
771 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
772
773 /*
774 * One freed pv entry in locked_pmap is
775 * sufficient.
776 */
777 if (pmap == locked_pmap)
778 goto out;
779 break;
780 }
781 if (field == _NPCM) {
782 PV_STAT(pv_entry_spare -= _NPCPV);
783 PV_STAT(pc_chunk_count--);
784 PV_STAT(pc_chunk_frees++);
785 /* Entire chunk is free; return it. */
786 m_pc = PHYS_TO_VM_PAGE(IA64_RR_MASK((vaddr_t)pc));
787 break;
788 }
789 }
790 out:
791 TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
792 if (pmap != NULL) {
793 if (pmap != locked_pmap) {
794 pmap_switch(locked_pmap);
795 PMAP_UNLOCK(pmap);
796 }
797 }
798 return (m_pc);
799 }
800
801 /*
802 * free the pv_entry back to the free list
803 */
804 static void
free_pv_entry(pmap_t pmap,pv_entry_t pv)805 free_pv_entry(pmap_t pmap, pv_entry_t pv)
806 {
807 struct pv_chunk *pc;
808 int bit, field, idx;
809
810 KASSERT(rw_write_held(&pvh_global_lock));
811 PMAP_LOCK_ASSERT(pmap);
812 PV_STAT(pv_entry_frees++);
813 PV_STAT(pv_entry_spare++);
814 pv_entry_count--;
815 pc = pv_to_chunk(pv);
816 idx = pv - &pc->pc_pventry[0];
817 field = idx / (sizeof(u_long) * NBBY);
818 bit = idx % (sizeof(u_long) * NBBY);
819 pc->pc_map[field] |= 1ul << bit;
820 for (idx = 0; idx < _NPCM; idx++)
821 if (pc->pc_map[idx] != pc_freemask[idx]) {
822 /*
823 * 98% of the time, pc is already at the head of the
824 * list. If it isn't already, move it to the head.
825 */
826 if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
827 pc)) {
828 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
829 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
830 pc_list);
831 }
832 return;
833 }
834 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
835 free_pv_chunk(pc);
836 }
837
838 static void
free_pv_chunk(struct pv_chunk * pc)839 free_pv_chunk(struct pv_chunk *pc)
840 {
841 struct vm_page *m;
842
843 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
844 PV_STAT(pv_entry_spare -= _NPCPV);
845 PV_STAT(pc_chunk_count--);
846 PV_STAT(pc_chunk_frees++);
847 /* entire chunk is free, return it */
848 m = PHYS_TO_VM_PAGE(IA64_RR_MASK((vaddr_t)pc));
849 #if 0
850 /* XXX freebsd these move pages around in queue */
851 vm_page_unwire(m, 0); // releases one wiring and moves page back active/inactive queue
852 vm_page_free(m); // moves page to "free" queue & disassociate with object
853
854 /* XXX might to need locks/other checks here, uvm_unwire, pmap_kremove... */
855 uvm_pagefree(m);
856 #endif
857 vm_page_free1(m);
858 }
859
860 /*
861 * get a new pv_entry, allocating a block from the system
862 * when needed.
863 */
864 static pv_entry_t
get_pv_entry(pmap_t pmap,bool try)865 get_pv_entry(pmap_t pmap, bool try)
866 {
867 struct pv_chunk *pc;
868 pv_entry_t pv;
869 struct vm_page *m;
870 int bit, field, idx;
871
872 KASSERT(rw_write_held(&pvh_global_lock));
873 PMAP_LOCK_ASSERT(pmap);
874 PV_STAT(pv_entry_allocs++);
875 pv_entry_count++;
876 retry:
877 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
878 if (pc != NULL) {
879 for (field = 0; field < _NPCM; field++) {
880 if (pc->pc_map[field]) {
881 bit = ffs64(pc->pc_map[field]) - 1;
882 break;
883 }
884 }
885 if (field < _NPCM) {
886 idx = field * sizeof(pc->pc_map[field]) * NBBY + bit;
887 pv = &pc->pc_pventry[idx];
888 pc->pc_map[field] &= ~(1ul << bit);
889 /* If this was the last item, move it to tail */
890 for (field = 0; field < _NPCM; field++)
891 if (pc->pc_map[field] != 0) {
892 PV_STAT(pv_entry_spare--);
893 return (pv); /* not full, return */
894 }
895 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
896 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
897 PV_STAT(pv_entry_spare--);
898 return (pv);
899 }
900 }
901
902 /* No free items, allocate another chunk */
903 m = vm_page_alloc1();
904
905 if (m == NULL) {
906 if (try) {
907 pv_entry_count--;
908 PV_STAT(pc_chunk_tryfail++);
909 return (NULL);
910 }
911 m = pmap_pv_reclaim(pmap);
912 if (m == NULL)
913 goto retry;
914 }
915
916 PV_STAT(pc_chunk_count++);
917 PV_STAT(pc_chunk_allocs++);
918 pc = (struct pv_chunk *)IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
919 pc->pc_pmap = pmap;
920 pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */
921 for (field = 1; field < _NPCM; field++)
922 pc->pc_map[field] = pc_freemask[field];
923 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
924 pv = &pc->pc_pventry[0];
925 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
926 PV_STAT(pv_entry_spare += _NPCPV - 1);
927 return (pv);
928 }
929
930 /*
931 * Conditionally create a pv entry.
932 */
933 #if 0
934 static bool
935 pmap_try_insert_pv_entry(pmap_t pmap, vaddr_t va, struct vm_page *m)
936 {
937 pv_entry_t pv;
938
939 PMAP_LOCK_ASSERT(pmap);
940 KASSERT(rw_write_held(&pvh_global_lock));
941 if ((pv = get_pv_entry(pmap, true)) != NULL) {
942 pv->pv_va = va;
943 TAILQ_INSERT_TAIL(&m->mdpage.pv_list, pv, pv_list);
944 return (true);
945 } else
946 return (false);
947 }
948 #endif
949
950 /*
951 * Add an ia64_lpte to the VHPT.
952 */
953 static void
pmap_enter_vhpt(struct ia64_lpte * pte,vaddr_t va)954 pmap_enter_vhpt(struct ia64_lpte *pte, vaddr_t va)
955 {
956 struct ia64_bucket *bckt;
957 struct ia64_lpte *vhpte;
958 uint64_t pte_pa;
959
960 /* Can fault, so get it out of the way. */
961 pte_pa = ia64_tpa((vaddr_t)pte);
962
963 vhpte = (struct ia64_lpte *)ia64_thash(va);
964 bckt = (struct ia64_bucket *)vhpte->chain;
965
966 mutex_spin_enter(&bckt->mutex);
967 pte->chain = bckt->chain;
968 ia64_mf();
969 bckt->chain = pte_pa;
970
971 pmap_vhpt_inserts++;
972 bckt->length++;
973 mutex_spin_exit(&bckt->mutex);
974 }
975
976 /*
977 * Remove the ia64_lpte matching va from the VHPT. Return zero if it
978 * worked or an appropriate error code otherwise.
979 */
980 static int
pmap_remove_vhpt(vaddr_t va)981 pmap_remove_vhpt(vaddr_t va)
982 {
983 struct ia64_bucket *bckt;
984 struct ia64_lpte *pte;
985 struct ia64_lpte *lpte;
986 struct ia64_lpte *vhpte;
987 uint64_t chain, tag;
988
989 tag = ia64_ttag(va);
990 vhpte = (struct ia64_lpte *)ia64_thash(va);
991 bckt = (struct ia64_bucket *)vhpte->chain;
992
993 lpte = NULL;
994 mutex_spin_enter(&bckt->mutex);
995 chain = bckt->chain;
996 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
997 while (chain != 0 && pte->tag != tag) {
998 lpte = pte;
999 chain = pte->chain;
1000 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1001 }
1002 if (chain == 0) {
1003 mutex_spin_exit(&bckt->mutex);
1004 return (ENOENT);
1005 }
1006
1007 /* Snip this pv_entry out of the collision chain. */
1008 if (lpte == NULL)
1009 bckt->chain = pte->chain;
1010 else
1011 lpte->chain = pte->chain;
1012 ia64_mf();
1013
1014 bckt->length--;
1015
1016 mutex_spin_exit(&bckt->mutex);
1017 return (0);
1018 }
1019
1020 /*
1021 * Find the ia64_lpte for the given va, if any.
1022 */
1023 static struct ia64_lpte *
pmap_find_vhpt(vaddr_t va)1024 pmap_find_vhpt(vaddr_t va)
1025 {
1026 struct ia64_bucket *bckt;
1027 struct ia64_lpte *pte;
1028 uint64_t chain, tag;
1029
1030 tag = ia64_ttag(va);
1031 pte = (struct ia64_lpte *)ia64_thash(va);
1032 bckt = (struct ia64_bucket *)pte->chain;
1033
1034 mutex_spin_enter(&bckt->mutex);
1035 chain = bckt->chain;
1036 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1037 while (chain != 0 && pte->tag != tag) {
1038 chain = pte->chain;
1039 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1040 }
1041
1042 mutex_spin_exit(&bckt->mutex);
1043 return ((chain != 0) ? pte : NULL);
1044 }
1045
1046 /*
1047 * Remove an entry from the list of managed mappings.
1048 */
1049 static int
pmap_remove_entry(pmap_t pmap,struct vm_page * m,vaddr_t va,pv_entry_t pv)1050 pmap_remove_entry(pmap_t pmap, struct vm_page *m, vaddr_t va, pv_entry_t pv)
1051 {
1052
1053 KASSERT(rw_write_held(&pvh_global_lock));
1054 if (!pv) {
1055 TAILQ_FOREACH(pv, &m->mdpage.pv_list, pv_list) {
1056 if (pmap == PV_PMAP(pv) && va == pv->pv_va)
1057 break;
1058 }
1059 }
1060
1061 if (pv) {
1062 TAILQ_REMOVE(&m->mdpage.pv_list, pv, pv_list);
1063 /* XXX
1064 if (TAILQ_FIRST(&m->mdpage.pv_list) == NULL)
1065 //vm_page_aflag_clear(m, PGA_WRITEABLE);
1066 m->flags |= PG_RDONLY;
1067 */
1068 free_pv_entry(pmap, pv);
1069 return 0;
1070 } else {
1071 return ENOENT;
1072 }
1073 }
1074
1075 /*
1076 * Create a pv entry for page at pa for
1077 * (pmap, va).
1078 */
1079 static void
pmap_insert_entry(pmap_t pmap,vaddr_t va,struct vm_page * m)1080 pmap_insert_entry(pmap_t pmap, vaddr_t va, struct vm_page *m)
1081 {
1082 pv_entry_t pv;
1083
1084 KASSERT(rw_write_held(&pvh_global_lock));
1085 pv = get_pv_entry(pmap, false);
1086 pv->pv_va = va;
1087 TAILQ_INSERT_TAIL(&m->mdpage.pv_list, pv, pv_list);
1088 }
1089
1090 /***************************************************
1091 * Low level mapping routines.....
1092 ***************************************************/
1093
1094 /*
1095 * Find the kernel lpte for mapping the given virtual address, which
1096 * must be in the part of region 5 which we can cover with our kernel
1097 * 'page tables'.
1098 */
1099 static struct ia64_lpte *
pmap_find_kpte(vaddr_t va)1100 pmap_find_kpte(vaddr_t va)
1101 {
1102 struct ia64_lpte **dir1;
1103 struct ia64_lpte *leaf;
1104
1105 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1106 UVMHIST_LOG(maphist, "(va=%p)", va, 0, 0, 0);
1107
1108 KASSERTMSG((va >> 61) == 5, "kernel mapping 0x%lx not in region 5", va);
1109
1110 KASSERTMSG(va < kernel_vm_end, "kernel mapping 0x%lx out of range", va);
1111
1112 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(va)];
1113 leaf = dir1[KPTE_DIR1_INDEX(va)];
1114
1115 UVMHIST_LOG(maphist, "(kpte_dir0=%#lx, kpte_dir1=%#lx, kpte_pte=%#lx)",
1116 KPTE_DIR0_INDEX(va), KPTE_DIR1_INDEX(va), KPTE_PTE_INDEX(va), 0);
1117 UVMHIST_LOG(maphist, "(dir1=%p, leaf=%p ret=%p)",
1118 dir1, leaf, &leaf[KPTE_PTE_INDEX(va)], 0);
1119
1120 return (&leaf[KPTE_PTE_INDEX(va)]);
1121 }
1122
1123 /*
1124 * Find a pte suitable for mapping a user-space address. If one exists
1125 * in the VHPT, that one will be returned, otherwise a new pte is
1126 * allocated.
1127 */
1128 static struct ia64_lpte *
pmap_find_pte(vaddr_t va)1129 pmap_find_pte(vaddr_t va)
1130 {
1131 struct ia64_lpte *pte;
1132
1133 if (va >= VM_MAXUSER_ADDRESS)
1134 return pmap_find_kpte(va);
1135
1136 pte = pmap_find_vhpt(va);
1137 if (pte == NULL) {
1138 pte = pool_cache_get(pte_pool_cache, PR_NOWAIT);
1139 if (pte != NULL) {
1140 memset((void *)pte, 0, sizeof(struct ia64_lpte));
1141 pte->tag = 1UL << 63;
1142 }
1143 }
1144
1145 return (pte);
1146 }
1147
1148 /*
1149 * Free a pte which is now unused. This simply returns it to the zone
1150 * allocator if it is a user mapping. For kernel mappings, clear the
1151 * valid bit to make it clear that the mapping is not currently used.
1152 */
1153 static void
pmap_free_pte(struct ia64_lpte * pte,vaddr_t va)1154 pmap_free_pte(struct ia64_lpte *pte, vaddr_t va)
1155 {
1156 if (va < VM_MAXUSER_ADDRESS)
1157 pool_cache_put(pte_pool_cache, pte);
1158 else
1159 pmap_clear_present(pte);
1160 }
1161
1162 static PMAP_INLINE void
pmap_pte_prot(pmap_t pm,struct ia64_lpte * pte,vm_prot_t prot)1163 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot)
1164 {
1165 static long prot2ar[4] = {
1166 PTE_AR_R, /* VM_PROT_NONE */
1167 PTE_AR_RW, /* VM_PROT_WRITE */
1168 PTE_AR_RX|PTE_ED, /* VM_PROT_EXECUTE */
1169 PTE_AR_RWX|PTE_ED /* VM_PROT_WRITE|VM_PROT_EXECUTE */
1170 };
1171
1172 pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED);
1173 pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56;
1174 pte->pte |= (prot == VM_PROT_NONE || pm == kernel_pmap)
1175 ? PTE_PL_KERN : PTE_PL_USER;
1176 pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1];
1177 }
1178
1179 static PMAP_INLINE void
pmap_pte_attr(struct ia64_lpte * pte,vm_memattr_t ma)1180 pmap_pte_attr(struct ia64_lpte *pte, vm_memattr_t ma)
1181 {
1182 pte->pte &= ~PTE_MA_MASK;
1183 pte->pte |= (ma & PTE_MA_MASK);
1184 }
1185
1186 /*
1187 * Set a pte to contain a valid mapping and enter it in the VHPT. If
1188 * the pte was orginally valid, then its assumed to already be in the
1189 * VHPT.
1190 * This functions does not set the protection bits. It's expected
1191 * that those have been set correctly prior to calling this function.
1192 */
1193 static void
pmap_set_pte(struct ia64_lpte * pte,vaddr_t va,vaddr_t pa,bool wired,bool managed)1194 pmap_set_pte(struct ia64_lpte *pte, vaddr_t va, vaddr_t pa,
1195 bool wired, bool managed)
1196 {
1197 pte->pte &= PTE_PROT_MASK | PTE_MA_MASK | PTE_PL_MASK |
1198 PTE_AR_MASK | PTE_ED;
1199 pte->pte |= PTE_PRESENT;
1200 pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED);
1201 pte->pte |= (wired) ? PTE_WIRED : 0;
1202 pte->pte |= pa & PTE_PPN_MASK;
1203
1204 pte->itir = PAGE_SHIFT << 2;
1205
1206 ia64_mf();
1207
1208 pte->tag = ia64_ttag(va);
1209 }
1210
1211 /*
1212 * Remove the (possibly managed) mapping represented by pte from the
1213 * given pmap.
1214 */
1215 static int
pmap_remove_pte(pmap_t pmap,struct ia64_lpte * pte,vaddr_t va,pv_entry_t pv,int freepte)1216 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vaddr_t va,
1217 pv_entry_t pv, int freepte)
1218 {
1219 int error;
1220 struct vm_page *m;
1221
1222 /*
1223 * First remove from the VHPT.
1224 */
1225 error = pmap_remove_vhpt(va);
1226 KASSERTMSG(error == 0, "%s: pmap_remove_vhpt returned %d",__func__, error);
1227
1228 pmap_invalidate_page(va);
1229
1230 if (pmap_wired(pte))
1231 pmap->pm_stats.wired_count -= 1;
1232
1233 pmap->pm_stats.resident_count -= 1;
1234 if (pmap_managed(pte)) {
1235 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1236 if (pmap_dirty(pte))
1237 vm_page_dirty(m);
1238
1239 error = pmap_remove_entry(pmap, m, va, pv);
1240 }
1241 if (freepte)
1242 pmap_free_pte(pte, va);
1243
1244 return (error);
1245 }
1246
1247 /*
1248 * pmap_init: [ INTERFACE ]
1249 *
1250 * Initialize the pmap module. Called by vm_init(), to initialize any
1251 * structures that the pmap system needs to map virtual memory.
1252 *
1253 * Note: no locking is necessary in this function.
1254 */
1255 void
pmap_init(void)1256 pmap_init(void)
1257 {
1258 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1259
1260 pmap_pool_cache = pool_cache_init(sizeof(struct pmap), 0, 0, 0,
1261 "pmap_pool_cache", NULL, IPL_VM,
1262 NULL, NULL, NULL);
1263 if (pmap_pool_cache == NULL)
1264 panic("%s cannot allocate pmap pool", __func__);
1265
1266 pte_pool_cache = pool_cache_init(sizeof(struct ia64_lpte), 0, 0, 0,
1267 "pte_pool_cache", NULL, IPL_VM,
1268 NULL, NULL, NULL);
1269 if (pte_pool_cache == NULL)
1270 panic("%s cannot allocate pte pool", __func__);
1271
1272
1273 pmap_initialized = true;
1274
1275 #if DEBUG
1276 if (0) pmap_testout();
1277 #endif
1278 }
1279
1280 /*
1281 * pmap_virtual_space: [ INTERFACE ]
1282 *
1283 * Define the initial bounds of the kernel virtual address space.
1284 */
1285 void
pmap_virtual_space(vaddr_t * vstartp,vaddr_t * vendp)1286 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
1287 {
1288 *vstartp = VM_MIN_KERNEL_ADDRESS;
1289 *vendp = VM_MAX_KERNEL_ADDRESS;
1290 }
1291
1292 /*
1293 * pmap_steal_memory: [ INTERFACE ]
1294 *
1295 * Bootstrap memory allocator (alternative to uvm_pageboot_alloc()).
1296 * This function allows for early dynamic memory allocation until the
1297 * virtual memory system has been bootstrapped. After that point, either
1298 * kmem_alloc or malloc should be used. This function works by stealing
1299 * pages from the (to be) managed page pool, then implicitly mapping the
1300 * pages (by using their RR7 addresses) and zeroing them.
1301 *
1302 * It may be used once the physical memory segments have been pre-loaded
1303 * into the vm_physmem[] array. Early memory allocation MUST use this
1304 * interface! This cannot be used after uvm_page_init(), and will
1305 * generate a panic if tried.
1306 *
1307 * Note that this memory will never be freed, and in essence it is wired
1308 * down.
1309 *
1310 * We must adjust *vstartp and/or *vendp iff we use address space
1311 * from the kernel virtual address range defined by pmap_virtual_space().
1312 *
1313 * Note: no locking is necessary in this function.
1314 */
1315 vaddr_t
pmap_steal_memory(vsize_t size,vaddr_t * vstartp,vaddr_t * vendp)1316 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
1317 {
1318 int npgs;
1319 uvm_physseg_t upm;
1320 vaddr_t va;
1321 paddr_t pa;
1322
1323 size = round_page(size);
1324 npgs = atop(size);
1325
1326 for (upm = uvm_physseg_get_first();
1327 uvm_physseg_valid_p(upm);
1328 upm = uvm_physseg_get_next(upm)) {
1329 if (uvm.page_init_done == true)
1330 panic("pmap_steal_memory: called _after_ bootstrap");
1331
1332 if (uvm_physseg_get_avail_start(upm) != uvm_physseg_get_start(upm) ||
1333 uvm_physseg_get_avail_start(upm) >= uvm_physseg_get_avail_end(upm))
1334 continue;
1335
1336 if ((uvm_physseg_get_avail_end(upm) - uvm_physseg_get_avail_start(upm))
1337 < npgs)
1338 continue;
1339
1340 /*
1341 * There are enough pages here; steal them!
1342 */
1343 pa = ptoa(uvm_physseg_get_start(upm));
1344 uvm_physseg_unplug(atop(pa), npgs);
1345
1346 va = IA64_PHYS_TO_RR7(pa);
1347 memset((void *)va, 0, size);
1348 pmap_pages_stolen += npgs;
1349 return va;
1350 }
1351
1352 /*
1353 * If we got here, this was no memory left.
1354 */
1355 panic("pmap_steal_memory: no memory to steal");
1356 }
1357
1358 /*
1359 * pmap_steal_vhpt_memory: Derived from alpha/pmap.c:pmap_steal_memory()
1360 * Note: This function is not visible outside the pmap module.
1361 * Based on pmap_steal_memory();
1362 * Assumptions: size is always a power of 2.
1363 * Returns: Allocated memory at a naturally aligned address
1364 */
1365 static vaddr_t
pmap_steal_vhpt_memory(vsize_t size)1366 pmap_steal_vhpt_memory(vsize_t size)
1367 {
1368 int npgs;
1369 uvm_physseg_t upm;
1370 vaddr_t va;
1371 paddr_t pa = 0;
1372 paddr_t vhpt_start = 0, start1, start2, end1, end2;
1373
1374 size = round_page(size);
1375 npgs = atop(size);
1376
1377 for (upm = uvm_physseg_get_first();
1378 uvm_physseg_valid_p(upm);
1379 upm = uvm_physseg_get_next(upm)) {
1380 if (uvm.page_init_done == true)
1381 panic("pmap_vhpt_steal_memory: called _after_ bootstrap");
1382
1383 if (uvm_physseg_get_avail_start(upm) != uvm_physseg_get_start(upm) || /* XXX: ??? */
1384 uvm_physseg_get_avail_start(upm) >= uvm_physseg_get_avail_end(upm))
1385 continue;
1386
1387 /* Break off a VHPT sized, aligned chunk off this segment. */
1388
1389 start1 = uvm_physseg_get_avail_start(upm);
1390
1391 /* Align requested start address on requested size boundary */
1392 end1 = vhpt_start = roundup(start1, npgs);
1393
1394 start2 = vhpt_start + npgs;
1395 end2 = uvm_physseg_get_avail_end(upm);
1396
1397 /* Case 1: Doesn't fit. skip this segment */
1398
1399 if (start2 > end2) {
1400 vhpt_start = 0;
1401 continue;
1402 }
1403
1404 /* For all cases of fit:
1405 * - Remove segment.
1406 * - Re-insert fragments via uvm_page_physload();
1407 */
1408
1409 /*
1410 * We _fail_ on a vhpt request which exhausts memory.
1411 */
1412 if (start1 == end1 &&
1413 start2 == end2 &&
1414 uvm_physseg_get_first() == uvm_physseg_get_last() /* single segment */) {
1415 #ifdef DEBUG
1416 printf("pmap_vhpt_steal_memory: out of memory!");
1417 #endif
1418 return -1;
1419 }
1420
1421 /* Remove this segment from the list. */
1422 if (uvm_physseg_unplug(uvm_physseg_get_start(upm),
1423 uvm_physseg_get_end(upm) - uvm_physseg_get_start(upm)) == false) {
1424 panic("%s: uvm_physseg_unplug(%"PRIxPADDR", %"PRIxPADDR") failed\n",
1425 __func__, uvm_physseg_get_start(upm),
1426 uvm_physseg_get_end(upm) - uvm_physseg_get_start(upm));
1427 }
1428
1429 /* Case 2: Perfect fit - skip segment reload. */
1430
1431 if (start1 == end1 && start2 == end2) break;
1432
1433 /* Case 3: Left unfit - reload it.
1434 */
1435
1436 if (start1 != end1)
1437 uvm_page_physload(start1, end1, start1, end1,
1438 VM_FREELIST_DEFAULT);
1439
1440 /* Case 4: Right unfit - reload it. */
1441
1442 if (start2 != end2)
1443 uvm_page_physload(start2, end2, start2, end2,
1444 VM_FREELIST_DEFAULT);
1445
1446 /* Case 5: Both unfit - Redundant, isn't it ? */
1447 break;
1448 }
1449
1450 /*
1451 * If we got here, we couldn't find a fit.
1452 */
1453 if (vhpt_start == 0) {
1454 #ifdef DEBUG
1455 printf("pmap_steal_vhpt_memory: no VHPT aligned fit found.");
1456 #endif
1457 return -1;
1458 }
1459
1460 /*
1461 * There are enough pages here; steal them!
1462 */
1463 pa = ptoa(vhpt_start);
1464 va = IA64_PHYS_TO_RR7(pa);
1465 memset((void *)va, 0, size);
1466 pmap_pages_stolen += npgs;
1467 return va;
1468 }
1469
1470 /*
1471 * pmap_create: [ INTERFACE ]
1472 *
1473 * Create and return a physical map.
1474 *
1475 * Note: no locking is necessary in this function.
1476 */
1477 pmap_t
pmap_create(void)1478 pmap_create(void)
1479 {
1480 pmap_t pmap;
1481 int i;
1482
1483 pmap = pool_cache_get(pmap_pool_cache, PR_WAITOK);
1484
1485 if (pmap == NULL)
1486 panic("%s no pool", __func__);
1487
1488 PMAP_LOCK_INIT(pmap);
1489
1490 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
1491 pmap->pm_rid[i] = pmap_allocate_rid();
1492
1493 TAILQ_INIT(&pmap->pm_pvchunk);
1494
1495 pmap->pm_stats.resident_count = 0;
1496 pmap->pm_stats.wired_count = 0;
1497
1498 pmap->pm_refcount = 1;
1499
1500 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1501 UVMHIST_LOG(maphist, "(pm=%p)", pmap, 0, 0, 0);
1502
1503 return pmap;
1504 }
1505
1506 /*
1507 * pmap_destroy: [ INTERFACE ]
1508 *
1509 * Drop the reference count on the specified pmap, releasing
1510 * all resources if the reference count drops to zero.
1511 */
1512 void
pmap_destroy(pmap_t pmap)1513 pmap_destroy(pmap_t pmap)
1514 {
1515 int i;
1516
1517 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1518 UVMHIST_LOG(maphist, "(pm=%p)", pmap, 0, 0, 0);
1519
1520 membar_release();
1521 if (atomic_dec_64_nv(&pmap->pm_refcount) > 0)
1522 return;
1523 membar_acquire();
1524
1525 KASSERT(pmap->pm_stats.resident_count == 0);
1526 KASSERT(pmap->pm_stats.wired_count == 0);
1527
1528 KASSERT(TAILQ_EMPTY(&pmap->pm_pvchunk)); /* XXX hmmm */
1529 KASSERT(!PMAP_LOCKED(pmap)); /* XXX hmmm */
1530 /*PMAP_LOCK(pmap); */ /* XXX overkill */
1531
1532 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
1533 if (pmap->pm_rid[i])
1534 pmap_free_rid(pmap->pm_rid[i]);
1535
1536 /*PMAP_UNLOCK(pmap);*/ /* XXX hmm */
1537 PMAP_LOCK_DESTROY(pmap);
1538
1539 pool_cache_put(pmap_pool_cache, pmap);
1540 }
1541
1542 /*
1543 * pmap_reference: [ INTERFACE ]
1544 *
1545 * Add a reference to the specified pmap.
1546 */
1547 void
pmap_reference(pmap_t pmap)1548 pmap_reference(pmap_t pmap)
1549 {
1550 atomic_inc_64(&pmap->pm_refcount);
1551 }
1552
1553 /*
1554 * pmap_resident_count: [ INTERFACE ]
1555 *
1556 * Query the ``resident pages'' statistic for pmap.
1557 */
1558 long
pmap_resident_count(pmap_t pmap)1559 pmap_resident_count(pmap_t pmap)
1560 {
1561 return (pmap->pm_stats.resident_count);
1562 }
1563
1564 /*
1565 * pmap_wired_count: [ INTERFACE ]
1566 *
1567 * Query the ``wired pages'' statistic for pmap.
1568 *
1569 */
1570 long
pmap_wired_count(pmap_t pmap)1571 pmap_wired_count(pmap_t pmap)
1572 {
1573 return (pmap->pm_stats.wired_count);
1574 }
1575
1576 /*
1577 * pmap_growkernel: [ INTERFACE ]
1578 *
1579 */
1580 vaddr_t
pmap_growkernel(vaddr_t maxkvaddr)1581 pmap_growkernel(vaddr_t maxkvaddr)
1582 {
1583 struct ia64_lpte **dir1;
1584 struct ia64_lpte *leaf;
1585 struct vm_page *pg;
1586 #if 0
1587 struct vm_page *nkpg;
1588 paddr_t pa;
1589 vaddr_t va;
1590 #endif
1591
1592 /* XXX this function may still need serious fixin' */
1593 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1594 UVMHIST_LOG(maphist, "(va=%#lx, nkpt=%ld, kvm_end=%#lx before)", maxkvaddr, nkpt, kernel_vm_end, 0);
1595
1596 vaddr_t addr = maxkvaddr;
1597
1598 /* XXX use uvm_pageboot_alloc if not done? */
1599 if (!uvm.page_init_done)
1600 panic("uvm_page init not done");
1601
1602 while (kernel_vm_end <= addr) {
1603 if (nkpt == PAGE_SIZE/8 + PAGE_SIZE*PAGE_SIZE/64)
1604 panic("%s: out of kernel address space", __func__);
1605 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)];
1606
1607 if (dir1 == NULL) {
1608 #if 0
1609 /* FreeBSD does it this way... */
1610 nkpg = vm_page_alloc(NULL, nkpt++, VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
1611 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
1612 #endif
1613 pg = vm_page_alloc1();
1614 if (!pg)
1615 panic("%s: cannot add dir1 page", __func__);
1616 nkpt++;
1617
1618 #if 0
1619 dir1 = (struct ia64_lpte **)pmap_page_to_va(nkpg);
1620 bzero(dir1, PAGE_SIZE);
1621 #endif
1622 dir1 = (struct ia64_lpte **)pmap_page_to_va(pg);
1623
1624 ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)] = dir1;
1625 }
1626
1627 #if 0
1628 nkpg = vm_page_alloc(NULL, nkpt++, VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
1629 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
1630 #endif
1631 pg = vm_page_alloc1();
1632 if (!pg)
1633 panic("%s: cannot add PTE page", __func__);
1634 nkpt++;
1635 #if 0
1636 leaf = (struct ia64_lpte *)pmap_page_to_va(nkpg);
1637 bzero(leaf, PAGE_SIZE);
1638 #endif
1639 leaf = (struct ia64_lpte *)pmap_page_to_va(pg);
1640
1641 dir1[KPTE_DIR1_INDEX(kernel_vm_end)] = leaf;
1642
1643 kernel_vm_end += PAGE_SIZE * NKPTEPG;
1644 }
1645
1646 UVMHIST_LOG(maphist, "(va=%#lx, nkpt=%ld, kvm_end=%#lx after)", maxkvaddr, nkpt, kernel_vm_end, 0);
1647
1648 /* XXX fix */
1649 return kernel_vm_end;
1650 }
1651
1652 /*
1653 * pmap_enter: [ INTERFACE ]
1654 *
1655 * Create a mapping in physical map pmap for the physical
1656 * address pa at the virtual address va with protection speci-
1657 * fied by bits in prot.
1658 */
1659 int
pmap_enter(pmap_t pmap,vaddr_t va,paddr_t pa,vm_prot_t prot,u_int flags)1660 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1661 {
1662 pmap_t oldpmap;
1663 struct vm_page *m;
1664 vaddr_t opa;
1665 struct ia64_lpte origpte;
1666 struct ia64_lpte *pte;
1667 bool icache_inval, managed, wired, canfail;
1668 /* vm_memattr_t ma; */
1669
1670 /* XXX this needs work */
1671
1672 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1673 UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, pa=%p, prot=%#x)", pmap, va, pa, prot);
1674
1675 /* wired = (flags & PMAP_ENTER_WIRED) != 0; */
1676 wired = (flags & PMAP_WIRED) != 0;
1677 canfail = (flags & PMAP_CANFAIL) != 0;
1678
1679 /* ma = pmap_flags_to_memattr(flags); */
1680
1681 rw_enter(&pvh_global_lock, RW_WRITER);
1682 PMAP_LOCK(pmap);
1683 oldpmap = pmap_switch(pmap);
1684
1685 va &= ~PAGE_MASK;
1686 KASSERTMSG(va <= VM_MAX_KERNEL_ADDRESS, "%s: toobig", __func__);
1687
1688 /* XXX
1689 if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m))
1690 VM_OBJECT_ASSERT_LOCKED(m->object);
1691 */
1692
1693 /*
1694 * Find (or create) a pte for the given mapping.
1695 */
1696 pte = pmap_find_pte(va);
1697
1698 if (pte == NULL) {
1699 pmap_switch(oldpmap);
1700 PMAP_UNLOCK(pmap);
1701 rw_exit(&pvh_global_lock);
1702
1703 if (canfail)
1704 return (ENOMEM);
1705 else
1706 panic("%s: no pte available", __func__);
1707 }
1708
1709 origpte = *pte;
1710 if (!pmap_present(pte)) {
1711 opa = ~0UL;
1712 pmap_enter_vhpt(pte, va);
1713 } else
1714 opa = pmap_ppn(pte);
1715
1716 managed = false;
1717 /* XXX hmm
1718 pa = VM_PAGE_TO_PHYS(m);
1719 */
1720
1721 m = PHYS_TO_VM_PAGE(pa);
1722 if (m == NULL) {
1723 /* implies page not managed? */
1724 panic("%s: new page needed", __func__);
1725 }
1726 icache_inval = (prot & VM_PROT_EXECUTE) ? true : false;
1727
1728 /*
1729 * Mapping has not changed, must be protection or wiring change.
1730 */
1731 if (opa == pa) {
1732 /*
1733 * Wiring change, just update stats. We don't worry about
1734 * wiring PT pages as they remain resident as long as there
1735 * are valid mappings in them. Hence, if a user page is wired,
1736 * the PT page will be also.
1737 */
1738 if (wired && !pmap_wired(&origpte))
1739 pmap->pm_stats.wired_count++;
1740 else if (!wired && pmap_wired(&origpte))
1741 pmap->pm_stats.wired_count--;
1742
1743 managed = (pmap_managed(&origpte)) ? true : false;
1744
1745 /*
1746 * We might be turning off write access to the page,
1747 * so we go ahead and sense modify status. Otherwise,
1748 * we can avoid I-cache invalidation if the page
1749 * already allowed execution.
1750 */
1751 if (managed && pmap_dirty(&origpte))
1752 vm_page_dirty(m);
1753 else if (pmap_exec(&origpte))
1754 icache_inval = false;
1755
1756 pmap_invalidate_page(va);
1757 goto validate;
1758 }
1759
1760 /*
1761 * Mapping has changed, invalidate old range and fall
1762 * through to handle validating new mapping.
1763 */
1764 if (opa != ~0UL) {
1765 pmap_remove_pte(pmap, pte, va, 0, 0);
1766 pmap_enter_vhpt(pte, va);
1767 }
1768
1769 /*
1770 * Enter on the PV list if part of our managed memory.
1771 */
1772 if (vm_page_is_managed(m)) {
1773 #if 0
1774 KASSERTMSG(va < kmi.clean_sva || va >= kmi.clean_eva,
1775 ("pmap_enter: managed mapping within the clean submap"));
1776 #endif
1777 pmap_insert_entry(pmap, va, m);
1778 managed = true;
1779 }
1780
1781 /*
1782 * Increment counters
1783 */
1784 pmap->pm_stats.resident_count++;
1785 if (wired)
1786 pmap->pm_stats.wired_count++;
1787
1788 validate:
1789
1790 /*
1791 * Now validate mapping with desired protection/wiring. This
1792 * adds the pte to the VHPT if necessary.
1793 */
1794 pmap_pte_prot(pmap, pte, prot);
1795 pmap_pte_attr(pte, m->mdpage.memattr);
1796 pmap_set_pte(pte, va, pa, wired, managed);
1797
1798 /* Invalidate the I-cache when needed. */
1799 if (icache_inval)
1800 ia64_sync_icache(va, PAGE_SIZE);
1801
1802 /* XXX
1803 if ((prot & VM_PROT_WRITE) != 0 && managed)
1804 //vm_page_aflag_set(m, PGA_WRITEABLE);
1805 m->flags &= ~PG_RDONLY;
1806 */
1807 rw_exit(&pvh_global_lock);
1808 pmap_switch(oldpmap);
1809 PMAP_UNLOCK(pmap);
1810 return (0);
1811 }
1812
1813 /*
1814 * pmap_remove: [ INTERFACE ]
1815 *
1816 * Remove the given range of addresses from the specified map.
1817 *
1818 * It is assumed that the start and end are properly
1819 * rounded to the page size.
1820 *
1821 * Sparsely used ranges are inefficiently removed. The VHPT is
1822 * probed for every page within the range. XXX
1823 */
1824 void
pmap_remove(pmap_t pmap,vaddr_t sva,vaddr_t eva)1825 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
1826 {
1827 pmap_t oldpmap;
1828 vaddr_t va;
1829 struct ia64_lpte *pte;
1830
1831 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1832 UVMHIST_LOG(maphist, "(pm=%p, sva=%#lx, eva=%#lx)", pmap, sva, eva, 0);
1833
1834 /*
1835 * Perform an unsynchronized read. This is, however, safe.
1836 */
1837 if (pmap->pm_stats.resident_count == 0)
1838 return;
1839
1840 rw_enter(&pvh_global_lock, RW_WRITER);
1841 PMAP_LOCK(pmap);
1842 oldpmap = pmap_switch(pmap);
1843 for (va = sva; va < eva; va += PAGE_SIZE) {
1844 pte = pmap_find_vhpt(va);
1845 if (pte != NULL)
1846 pmap_remove_pte(pmap, pte, va, 0, 1);
1847 }
1848
1849 rw_exit(&pvh_global_lock);
1850 pmap_switch(oldpmap);
1851 PMAP_UNLOCK(pmap);
1852 }
1853
1854 /*
1855 * pmap_remove_all: [ INTERFACE ]
1856 *
1857 * This function is a hint to the pmap implementation that all
1858 * entries in pmap will be removed before any more entries are
1859 * entered.
1860 */
1861 bool
pmap_remove_all(pmap_t pmap)1862 pmap_remove_all(pmap_t pmap)
1863 {
1864 /* XXX do nothing */
1865 return false;
1866 }
1867
1868 /*
1869 * pmap_protect: [ INTERFACE ]
1870 *
1871 * Set the physical protection on the specified range of this map
1872 * as requested.
1873 */
1874 void
pmap_protect(pmap_t pmap,vaddr_t sva,vaddr_t eva,vm_prot_t prot)1875 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
1876 {
1877 pmap_t oldpmap;
1878 struct ia64_lpte *pte;
1879
1880 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1881 UVMHIST_LOG(maphist, "(pm=%p, sva=%#lx, eva=%#lx, prot=%#x)",
1882 pmap, sva, eva, prot);
1883 UVMHIST_LOG(maphist, "(VM_PROT_READ=%u, VM_PROT_WRITE=%u, VM_PROT_EXECUTE=%u)",
1884 (prot & VM_PROT_READ) != 0, (prot & VM_PROT_WRITE) != 0,
1885 (prot & VM_PROT_EXECUTE) != 0, 0);
1886
1887 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1888 pmap_remove(pmap, sva, eva);
1889 return;
1890 }
1891
1892 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
1893 (VM_PROT_WRITE|VM_PROT_EXECUTE))
1894 return;
1895
1896 if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
1897 panic("pmap_protect: unaligned addresses");
1898 sva = trunc_page(sva);
1899 eva = round_page(eva) - 1;
1900
1901 PMAP_LOCK(pmap);
1902 oldpmap = pmap_switch(pmap);
1903 for ( ; sva < eva; sva += PAGE_SIZE) {
1904 /* If page is invalid, skip this page */
1905 pte = pmap_find_vhpt(sva);
1906 if (pte == NULL)
1907 continue;
1908
1909 /* If there's no change, skip it too */
1910 if (pmap_prot(pte) == prot)
1911 continue;
1912
1913 /* wired pages unaffected by prot changes */
1914 if (pmap_wired(pte))
1915 continue;
1916
1917 if ((prot & VM_PROT_WRITE) == 0 &&
1918 pmap_managed(pte) && pmap_dirty(pte)) {
1919 paddr_t pa = pmap_ppn(pte);
1920 struct vm_page *m = PHYS_TO_VM_PAGE(pa);
1921
1922 vm_page_dirty(m);
1923 pmap_clear_dirty(pte);
1924 }
1925
1926 if (prot & VM_PROT_EXECUTE)
1927 ia64_sync_icache(sva, PAGE_SIZE);
1928
1929 pmap_pte_prot(pmap, pte, prot);
1930 pmap_invalidate_page(sva);
1931 }
1932 pmap_switch(oldpmap);
1933 PMAP_UNLOCK(pmap);
1934 }
1935
1936 /*
1937 * pmap_unwire: [ INTERFACE ]
1938 *
1939 * Clear the wired attribute for a map/virtual-address pair.
1940 *
1941 * The wired attribute of the page table entry is not a hardware feature,
1942 * so there is no need to invalidate any TLB entries.
1943 *
1944 * The mapping must already exist in the pmap.
1945 */
1946 void
pmap_unwire(pmap_t pmap,vaddr_t va)1947 pmap_unwire(pmap_t pmap, vaddr_t va)
1948 {
1949 pmap_t oldpmap;
1950 struct ia64_lpte *pte;
1951
1952 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1953 UVMHIST_LOG(maphist, "(pm=%p, va=%#x)", pmap, va, 0, 0);
1954
1955 PMAP_LOCK(pmap);
1956 oldpmap = pmap_switch(pmap);
1957
1958 pte = pmap_find_vhpt(va);
1959
1960 /* XXX panic if no pte or not wired? */
1961 if (pte == NULL)
1962 panic("pmap_unwire: %lx not found in vhpt", va);
1963
1964 if (!pmap_wired(pte))
1965 panic("pmap_unwire: pte %p isn't wired", pte);
1966
1967 pmap->pm_stats.wired_count--;
1968 pmap_clear_wired(pte);
1969
1970 pmap_switch(oldpmap);
1971 PMAP_UNLOCK(pmap);
1972 }
1973
1974 /*
1975 * pmap_extract: [ INTERFACE ]
1976 *
1977 * Extract the physical address associated with the given
1978 * pmap/virtual address pair.
1979 */
1980 bool
pmap_extract(pmap_t pmap,vaddr_t va,paddr_t * pap)1981 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap)
1982 {
1983 struct ia64_lpte *pte;
1984 pmap_t oldpmap;
1985 paddr_t pa;
1986
1987 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1988 UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, pap=%#lx)", pmap, va, pap, 0);
1989
1990 pa = 0;
1991
1992 PMAP_LOCK(pmap);
1993 oldpmap = pmap_switch(pmap);
1994 pte = pmap_find_vhpt(va);
1995 if (pte != NULL && pmap_present(pte))
1996 pa = pmap_ppn(pte);
1997 pmap_switch(oldpmap);
1998 PMAP_UNLOCK(pmap);
1999
2000 if (pa && (pap != NULL))
2001 *pap = pa;
2002
2003 UVMHIST_LOG(maphist, "(pa=%#lx)", pa, 0, 0, 0);
2004
2005 return (pa != 0);
2006 }
2007 /*
2008 * Extract the physical page address associated with a kernel
2009 * virtual address.
2010 */
2011 paddr_t
pmap_kextract(vaddr_t va)2012 pmap_kextract(vaddr_t va)
2013 {
2014 struct ia64_lpte *pte;
2015 /*uint64_t *pbvm_pgtbl;*/
2016 paddr_t pa;
2017 /*u_int idx;*/
2018
2019 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2020 UVMHIST_LOG(maphist, "(va=%#lx)", va, 0, 0, 0);
2021
2022 KASSERTMSG(va >= VM_MAXUSER_ADDRESS, "Must be kernel VA");
2023
2024 /* Regions 6 and 7 are direct mapped. */
2025 if (va >= IA64_RR_BASE(6)) {
2026 pa = IA64_RR_MASK(va);
2027 goto out;
2028 }
2029
2030 /* Region 5 is our KVA. Bail out if the VA is beyond our limits. */
2031 if (va >= kernel_vm_end)
2032 goto err_out;
2033 if (va >= VM_INIT_KERNEL_ADDRESS) {
2034 pte = pmap_find_kpte(va);
2035 pa = pmap_present(pte) ? pmap_ppn(pte) | (va & PAGE_MASK) : 0;
2036 goto out;
2037 }
2038 #if 0 /* XXX fix */
2039 /* The PBVM page table. */
2040 if (va >= IA64_PBVM_PGTBL + bootinfo->bi_pbvm_pgtblsz)
2041 goto err_out;
2042 if (va >= IA64_PBVM_PGTBL) {
2043 pa = (va - IA64_PBVM_PGTBL) + bootinfo->bi_pbvm_pgtbl;
2044 goto out;
2045 }
2046
2047 /* The PBVM itself. */
2048 if (va >= IA64_PBVM_BASE) {
2049 pbvm_pgtbl = (void *)IA64_PBVM_PGTBL;
2050 idx = (va - IA64_PBVM_BASE) >> IA64_PBVM_PAGE_SHIFT;
2051 if (idx >= (bootinfo->bi_pbvm_pgtblsz >> 3))
2052 goto err_out;
2053 if ((pbvm_pgtbl[idx] & PTE_PRESENT) == 0)
2054 goto err_out;
2055 pa = (pbvm_pgtbl[idx] & PTE_PPN_MASK) +
2056 (va & IA64_PBVM_PAGE_MASK);
2057 goto out;
2058 }
2059 #endif
2060
2061 err_out:
2062 KASSERT(1);
2063 printf("XXX: %s: va=%#lx is invalid\n", __func__, va);
2064 pa = 0;
2065 /* FALLTHROUGH */
2066
2067 out:
2068 UVMHIST_LOG(maphist, "(pa=%#lx)", pa, 0, 0, 0);
2069 return (pa);
2070 }
2071
2072 /*
2073 * pmap_kenter_pa: [ INTERFACE ]
2074 *
2075 * Enter a va -> pa mapping into the kernel pmap without any
2076 * physical->virtual tracking.
2077 *
2078 * Note: no locking is necessary in this function.
2079 */
2080 void
pmap_kenter_pa(vaddr_t va,paddr_t pa,vm_prot_t prot,u_int flags)2081 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
2082 {
2083 struct ia64_lpte *pte;
2084 vm_memattr_t attr;
2085 const bool managed = false; /* don't gather ref/mod info */
2086 const bool wired = true; /* pmap_kenter_pa always wired */
2087
2088 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2089 UVMHIST_LOG(maphist, "(va=%#lx, pa=%#lx, prot=%p, flags=%p)", va, pa, prot, flags);
2090
2091 KASSERT(va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS);
2092
2093 attr = pmap_flags_to_memattr(flags);
2094
2095 pte = pmap_find_kpte(va);
2096 if (pmap_present(pte))
2097 pmap_invalidate_page(va);
2098 else
2099 pmap_enter_vhpt(pte, va);
2100
2101 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
2102 pmap_pte_attr(pte, attr);
2103 pmap_set_pte(pte, va, pa, wired, managed);
2104 }
2105
2106 /*
2107 * pmap_kremove: [ INTERFACE ]
2108 *
2109 * Remove all mappings starting at virtual address va for size
2110 * bytes from the kernel physical map. All mappings that are
2111 * removed must be the ``unmanaged'' type created with
2112 * pmap_kenter_pa(). The implementation may assert this.
2113 */
2114 void
pmap_kremove(vaddr_t va,vsize_t size)2115 pmap_kremove(vaddr_t va, vsize_t size)
2116 {
2117 struct ia64_lpte *pte;
2118 vaddr_t eva = va + size;
2119
2120 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2121 UVMHIST_LOG(maphist, "(va=%#lx)", va, 0, 0, 0);
2122
2123 while (va < eva) {
2124 pte = pmap_find_kpte(va);
2125 if (pmap_present(pte)) {
2126 KASSERT(pmap_managed(pte) != 0);
2127 pmap_remove_vhpt(va);
2128 pmap_invalidate_page(va);
2129 pmap_clear_present(pte);
2130 }
2131 va += PAGE_SIZE;
2132 }
2133 }
2134
2135 /*
2136 * pmap_copy: [ INTERFACE ]
2137 *
2138 * This function copies the mappings starting at src_addr in
2139 * src_map for len bytes into dst_map starting at dst_addr.
2140 *
2141 * Note that while this function is required to be provided by
2142 * a pmap implementation, it is not actually required to do
2143 * anything. pmap_copy() is merely advisory (it is used in
2144 * the fork(2) path to ``pre-fault'' the child's address
2145 * space).
2146 */
2147 void
pmap_copy(pmap_t dst_map,pmap_t src_map,vaddr_t dst_addr,vsize_t len,vaddr_t src_addr)2148 pmap_copy(pmap_t dst_map, pmap_t src_map, vaddr_t dst_addr, vsize_t len,
2149 vaddr_t src_addr)
2150 {
2151 /* nothing required */
2152 }
2153
2154 /*
2155 * pmap_update: [ INTERFACE ]
2156 *
2157 * If a pmap implementation does not delay virtual-to-physical
2158 * mapping updates, pmap_update() has no operation.
2159 */
2160 void
pmap_update(pmap_t pmap)2161 pmap_update(pmap_t pmap)
2162 {
2163 /* nothing required */
2164 }
2165
2166 /*
2167 * pmap_activate: [ INTERFACE ]
2168 *
2169 * Activate the pmap used by the specified process. This includes
2170 * reloading the MMU context if the current process, and marking
2171 * the pmap in use by the processor.
2172 *
2173 * Note: We may use only spin locks here, since we are called
2174 * by a critical section in cpu_switch()!
2175 */
2176 void
pmap_activate(struct lwp * l)2177 pmap_activate(struct lwp *l)
2178 {
2179 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2180 UVMHIST_LOG(maphist, "(lwp=%p)", l, 0, 0, 0);
2181 KASSERT(l == curlwp);
2182 #if 0
2183 /* pmap_switch(vmspace_pmap(td->td_proc->p_vmspace)); */
2184 pmap_switch(vm_map_pmap(&l->l_proc->p_vmspace->vm_map));
2185 #else
2186 struct pmap *pmap = l->l_proc->p_vmspace->vm_map.pmap;
2187
2188 if (pmap == pmap_kernel()) {
2189 return;
2190 }
2191
2192 if (l != curlwp) {
2193 return;
2194 }
2195
2196 pmap_switch(pmap);
2197 #endif
2198 }
2199
2200 /*
2201 * pmap_deactivate: [ INTERFACE ]
2202 *
2203 * Deactivate the physical map used by the process behind lwp
2204 * l. It is generally used in conjunction with
2205 * pmap_activate(). Like pmap_activate(), pmap_deactivate()
2206 * may not always be called when l is the current lwp.
2207 *
2208 */
2209 void
pmap_deactivate(struct lwp * l)2210 pmap_deactivate(struct lwp *l)
2211 {
2212 /* XXX ? */
2213 }
2214
2215 /*
2216 * pmap_zero_page: [ INTERFACE ]
2217 *
2218 * Zero the specified (machine independent) page by mapping the page
2219 * into virtual memory and clear its contents, one machine dependent
2220 * page at a time.
2221 *
2222 * Note: no locking is necessary in this function.
2223 */
2224 void
pmap_zero_page(paddr_t phys)2225 pmap_zero_page(paddr_t phys)
2226 {
2227 struct vm_page *m;
2228 vaddr_t va;
2229
2230 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2231 UVMHIST_LOG(maphist, "(pa=%p)", phys, 0, 0, 0);
2232
2233 m = PHYS_TO_VM_PAGE(phys);
2234 KASSERT(m != NULL);
2235
2236 va = pmap_page_to_va(m);
2237 KASSERT(trunc_page(va) == va);
2238
2239 UVMHIST_LOG(maphist, "(pa=%p, va=%p)", phys, va, 0, 0);
2240 memset((void *)va, 0, PAGE_SIZE);
2241 }
2242
2243 /*
2244 * pmap_copy_page: [ INTERFACE ]
2245 *
2246 * Copy the specified (machine independent) page by mapping the page
2247 * into virtual memory and using memcpy to copy the page, one machine
2248 * dependent page at a time.
2249 *
2250 * Note: no locking is necessary in this function.
2251 */
2252 void
pmap_copy_page(paddr_t psrc,paddr_t pdst)2253 pmap_copy_page(paddr_t psrc, paddr_t pdst)
2254 {
2255 struct vm_page *md, *ms;
2256 vaddr_t dst_va, src_va;
2257
2258 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2259 UVMHIST_LOG(maphist, "(sp=%p, dp=%p)", psrc, pdst, 0, 0);
2260
2261 md = PHYS_TO_VM_PAGE(pdst);
2262 ms = PHYS_TO_VM_PAGE(psrc);
2263 KASSERT(md != NULL && ms != NULL);
2264
2265 dst_va = pmap_page_to_va(md);
2266 src_va = pmap_page_to_va(ms);
2267 KASSERT(trunc_page(dst_va) == dst_va && trunc_page(src_va) == src_va);
2268
2269 memcpy((void *)dst_va, (void *)src_va, PAGE_SIZE);
2270 }
2271
2272 /*
2273 * pmap_page_protect: [ INTERFACE ]
2274 *
2275 * Lower the permissions for all mappings of the page pg to
2276 * prot. This function is used by the virtual memory system
2277 * to implement copy-on-write (called with VM_PROT_READ set in
2278 * prot) and to revoke all mappings when cleaning a page
2279 * (called with no bits set in prot). Access permissions must
2280 * never be added to a page as a result of this call.
2281 */
2282 void
pmap_page_protect(struct vm_page * pg,vm_prot_t prot)2283 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
2284 {
2285 //struct ia64_lpte *pte;
2286 pmap_t pmap;
2287 pv_entry_t pv;
2288 vaddr_t va;
2289
2290 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2291 UVMHIST_LOG(maphist, "(m=%p, prot=%p)", pg, prot, 0, 0);
2292
2293 if ((prot & VM_PROT_WRITE) != 0)
2294 return;
2295 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
2296 /* XXX FreeBSD
2297 if ((pg->mdpage.aflags & PGA_WRITEABLE) == 0)
2298 return;
2299 */
2300 if (pg->flags & PG_RDONLY)
2301 return;
2302 rw_enter(&pvh_global_lock, RW_WRITER);
2303 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
2304 pmap = PV_PMAP(pv);
2305 va = pv->pv_va;
2306 // locking of pmap done in pmap_protect
2307 pmap_protect(pmap, va, va + PAGE_SIZE, prot);
2308 }
2309 /* XXX
2310 vm_page_aflag_clear(pg, PGA_WRITEABLE);
2311 */
2312 pg->flags |= PG_RDONLY;
2313
2314 rw_exit(&pvh_global_lock);
2315 } else {
2316 pmap_remove_all_phys(pg);
2317 }
2318 }
2319
2320 /*
2321 * pmap_clear_modify: [ INTERFACE ]
2322 *
2323 * Clear the modify bits on the specified physical page.
2324 */
2325 bool
pmap_clear_modify(struct vm_page * pg)2326 pmap_clear_modify(struct vm_page *pg)
2327 {
2328 struct ia64_lpte *pte;
2329 pmap_t oldpmap, pmap;
2330 pv_entry_t pv;
2331 bool rv;
2332
2333 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2334 UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0);
2335
2336 KASSERTMSG(vm_page_is_managed(pg), "%s : page %p not managed",
2337 __func__, pg);
2338
2339 rv = false;
2340
2341 //VM_OBJECT_ASSERT_WLOCKED(m->object);
2342 //KASSERT(!vm_page_xbusied(m),
2343 // ("pmap_clear_modify: page %p is exclusive busied", m));
2344
2345 /*
2346 * If the page is not PGA_WRITEABLE, then no PTEs can be modified.
2347 * If the object containing the page is locked and the page is not
2348 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
2349 */
2350
2351 #if 0 /* XXX freebsd does this, looks faster but not required */
2352 if ((m->aflags & PGA_WRITEABLE) == 0)
2353 return;
2354 if (pg->flags & PG_RDONLY)
2355 return (rv);
2356 #endif
2357 rw_enter(&pvh_global_lock, RW_WRITER);
2358 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
2359 pmap = PV_PMAP(pv);
2360 PMAP_LOCK(pmap);
2361 oldpmap = pmap_switch(pmap);
2362 pte = pmap_find_vhpt(pv->pv_va);
2363 KASSERTMSG(pte != NULL, "pte");
2364 if (pmap_dirty(pte)) {
2365 pmap_clear_dirty(pte);
2366 pmap_invalidate_page(pv->pv_va);
2367 rv = true;
2368 }
2369 pmap_switch(oldpmap);
2370 PMAP_UNLOCK(pmap);
2371 }
2372
2373 rw_exit(&pvh_global_lock);
2374
2375 return (rv);
2376 }
2377
2378 /*
2379 * pmap_clear_reference: [ INTERFACE ]
2380 *
2381 * Clear the reference bit on the specified physical page.
2382 *
2383 * returns true or false
2384 * indicating whether or not the ``referenced'' attribute was
2385 * set on the page before it was cleared
2386 */
2387 bool
pmap_clear_reference(struct vm_page * pg)2388 pmap_clear_reference(struct vm_page *pg)
2389 {
2390 struct ia64_lpte *pte;
2391 pmap_t oldpmap, pmap;
2392 pv_entry_t pv;
2393 bool rv;
2394
2395 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2396 UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0);
2397
2398 KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed",
2399 __func__, pg);
2400
2401 rv = false;
2402
2403 rw_enter(&pvh_global_lock, RW_WRITER);
2404 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
2405 pmap = PV_PMAP(pv);
2406 PMAP_LOCK(pmap);
2407 oldpmap = pmap_switch(pmap);
2408 pte = pmap_find_vhpt(pv->pv_va);
2409 KASSERTMSG(pte != NULL, "pte");
2410 if (pmap_accessed(pte)) {
2411 rv = true;
2412 pmap_clear_accessed(pte);
2413 pmap_invalidate_page(pv->pv_va);
2414 }
2415 pmap_switch(oldpmap);
2416 PMAP_UNLOCK(pmap);
2417 }
2418
2419 rw_exit(&pvh_global_lock);
2420 return (rv);
2421 }
2422
2423 /*
2424 * pmap_is_modified: [ INTERFACE ]
2425 *
2426 */
2427 bool
pmap_is_modified(struct vm_page * pg)2428 pmap_is_modified(struct vm_page *pg)
2429 {
2430 struct ia64_lpte *pte;
2431 pmap_t oldpmap, pmap;
2432 pv_entry_t pv;
2433 bool rv;
2434
2435 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2436 UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0);
2437
2438 KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed",
2439 __func__, pg);
2440 rv = false;
2441
2442 /*
2443 * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
2444 * concurrently set while the object is locked. Thus, if PGA_WRITEABLE
2445 * is clear, no PTEs can be dirty.
2446 */
2447 /* XXX freebsd
2448 VM_OBJECT_ASSERT_WLOCKED(m->object);
2449 if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
2450 return (rv);
2451 */
2452 rw_enter(&pvh_global_lock, RW_WRITER);
2453 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
2454 pmap = PV_PMAP(pv);
2455 PMAP_LOCK(pmap);
2456 oldpmap = pmap_switch(pmap);
2457 pte = pmap_find_vhpt(pv->pv_va);
2458 pmap_switch(oldpmap);
2459 KASSERTMSG(pte != NULL, "pte");
2460
2461 rv = pmap_dirty(pte) ? true : false;
2462 PMAP_UNLOCK(pmap);
2463 if (rv)
2464 break;
2465 }
2466
2467 rw_exit(&pvh_global_lock);
2468 return (rv);
2469 }
2470
2471 /*
2472 * pmap_is_referenced: [ INTERFACE ]
2473 *
2474 * Test whether or not the ``referenced'' attribute is set on
2475 * page pg.
2476 *
2477 */
2478 bool
pmap_is_referenced(struct vm_page * pg)2479 pmap_is_referenced(struct vm_page *pg)
2480 {
2481 struct ia64_lpte *pte;
2482 pmap_t oldpmap, pmap;
2483 pv_entry_t pv;
2484 bool rv;
2485
2486 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2487 UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0);
2488
2489 KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed",
2490 __func__, pg);
2491
2492 rv = false;
2493 rw_enter(&pvh_global_lock, RW_WRITER);
2494 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
2495 pmap = PV_PMAP(pv);
2496 PMAP_LOCK(pmap);
2497 oldpmap = pmap_switch(pmap);
2498 pte = pmap_find_vhpt(pv->pv_va);
2499 KASSERTMSG(pte != NULL, "pte");
2500 rv = pmap_accessed(pte) ? true : false;
2501 pmap_switch(oldpmap);
2502 PMAP_UNLOCK(pmap);
2503 if (rv)
2504 break;
2505 }
2506
2507 rw_exit(&pvh_global_lock);
2508 return (rv);
2509 }
2510
2511 /*
2512 * pmap_phys_address: [ INTERFACE ]
2513 *
2514 * Return the physical address corresponding to the specified
2515 * cookie. Used by the device pager to decode a device driver's
2516 * mmap entry point return value.
2517 *
2518 * Note: no locking is necessary in this function.
2519 */
2520 paddr_t
pmap_phys_address(paddr_t ppn)2521 pmap_phys_address(paddr_t ppn)
2522 {
2523 return ia64_ptob(ppn);
2524 }
2525
2526 pmap_t
pmap_switch(pmap_t pm)2527 pmap_switch(pmap_t pm)
2528 {
2529 pmap_t prevpm;
2530 int i;
2531
2532 critical_enter();
2533
2534 prevpm = curcpu()->ci_pmap;
2535 if (prevpm == pm)
2536 goto out;
2537 if (pm == NULL) {
2538 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) {
2539 ia64_set_rr(IA64_RR_BASE(i),
2540 (i << 8)|(PAGE_SHIFT << 2)|1);
2541 }
2542 } else {
2543 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) {
2544 ia64_set_rr(IA64_RR_BASE(i),
2545 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1);
2546 }
2547 }
2548
2549 /* XXX */
2550 ia64_srlz_d();
2551 curcpu()->ci_pmap = pm;
2552 ia64_srlz_d();
2553
2554 out:
2555 critical_exit();
2556 return (prevpm);
2557 }
2558
2559
2560 /*
2561 * Synchronize CPU instruction caches of the specified range.
2562 * Same as freebsd
2563 * void pmap_sync_icache(pmap_t pm, vaddr_t va, vsize_t sz)
2564 */
2565 void
pmap_procwr(struct proc * p,vaddr_t va,vsize_t sz)2566 pmap_procwr(struct proc *p, vaddr_t va, vsize_t sz)
2567 {
2568 pmap_t oldpm;
2569 struct ia64_lpte *pte;
2570 vaddr_t lim;
2571 vsize_t len;
2572
2573 struct pmap * const pm = p->p_vmspace->vm_map.pmap;
2574
2575 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2576 UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, sz=%#lx)", pm, va, sz, 0);
2577
2578 sz += va & 31;
2579 va &= ~31;
2580 sz = (sz + 31) & ~31;
2581
2582 PMAP_LOCK(pm);
2583 oldpm = pmap_switch(pm);
2584 while (sz > 0) {
2585 lim = round_page(va);
2586 len = MIN(lim - va, sz);
2587 pte = pmap_find_vhpt(va);
2588 if (pte != NULL && pmap_present(pte))
2589 ia64_sync_icache(va, len);
2590 va += len;
2591 sz -= len;
2592 }
2593 pmap_switch(oldpm);
2594 PMAP_UNLOCK(pm);
2595 }
2596
2597 /*
2598 * Routine: pmap_remove_all_phys
2599 * Function:
2600 * Removes this physical page from
2601 * all physical maps in which it resides.
2602 * Reflects back modify bits to the pager.
2603 *
2604 * Notes:
2605 * Original versions of this routine were very
2606 * inefficient because they iteratively called
2607 * pmap_remove (slow...)
2608 */
2609 void
pmap_remove_all_phys(struct vm_page * m)2610 pmap_remove_all_phys(struct vm_page *m)
2611 {
2612 pmap_t oldpmap;
2613 pv_entry_t pv;
2614
2615 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2616 UVMHIST_LOG(maphist, "(m=%p)", m, 0, 0, 0);
2617
2618 KASSERTMSG(vm_page_is_managed(m), "%s: page %p is not managed",
2619 __func__, m);
2620
2621 rw_enter(&pvh_global_lock, RW_WRITER);
2622 while ((pv = TAILQ_FIRST(&m->mdpage.pv_list)) != NULL) {
2623 struct ia64_lpte *pte;
2624 pmap_t pmap = PV_PMAP(pv);
2625 vaddr_t va = pv->pv_va;
2626
2627 PMAP_LOCK(pmap);
2628 oldpmap = pmap_switch(pmap);
2629 pte = pmap_find_vhpt(va);
2630 KASSERTMSG(pte != NULL, "pte");
2631 if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(m))
2632 panic("%s: pv_table for %lx is inconsistent",
2633 __func__, VM_PAGE_TO_PHYS(m));
2634 pmap_remove_pte(pmap, pte, va, pv, 1);
2635 pmap_switch(oldpmap);
2636 PMAP_UNLOCK(pmap);
2637 }
2638 /* XXX freebsd
2639 vm_page_aflag_clear(m, PGA_WRITEABLE);
2640 */
2641 m->flags |= PG_RDONLY;
2642
2643 rw_exit(&pvh_global_lock);
2644 }
2645
2646 /*
2647 * vm_page_alloc1:
2648 *
2649 * Allocate and return a memory cell with no associated object.
2650 */
2651 static struct vm_page
vm_page_alloc1(void)2652 *vm_page_alloc1(void)
2653 {
2654 struct vm_page *pg;
2655
2656 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
2657 if (pg) {
2658 pg->wire_count = 1; /* no mappings yet */
2659 pg->flags &= ~PG_BUSY; /* never busy */
2660 }
2661 return pg;
2662 }
2663
2664 /*
2665 * vm_page_free1:
2666 *
2667 * Returns the given page to the free list,
2668 * disassociating it with any VM object.
2669 *
2670 * Object and page must be locked prior to entry.
2671 */
2672 static void
vm_page_free1(struct vm_page * pg)2673 vm_page_free1(struct vm_page *pg)
2674 {
2675 KASSERT(pg->flags != (PG_CLEAN|PG_FAKE)); /* Freeing invalid page */
2676
2677 pg->flags |= PG_BUSY;
2678 pg->wire_count = 0;
2679 uvm_pagefree(pg);
2680 }
2681
2682 /*
2683 * pmap_flag_to_attr
2684 *
2685 * Convert pmap_enter/pmap_kenter_pa flags to memory attributes
2686 */
2687 static vm_memattr_t
pmap_flags_to_memattr(u_int flags)2688 pmap_flags_to_memattr(u_int flags)
2689 {
2690 u_int cacheflags = flags & PMAP_CACHE_MASK;
2691
2692 #if 0
2693 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2694 UVMHIST_LOG(maphist, "(PMAP_NOCACHE=%u, PMAP_WRITE_COMBINE=%u, "
2695 "PMAP_WRITE_BACK=%u, PMAP_NOCACHE_OVR=%u)",
2696 (flags & PMAP_NOCACHE) != 0, (flags & PMAP_WRITE_COMBINE) != 0,
2697 (flags & PMAP_WRITE_BACK) != 0, (flags & PMAP_NOCACHE_OVR) != 0);
2698 #endif
2699 switch (cacheflags) {
2700 case PMAP_NOCACHE:
2701 return VM_MEMATTR_UNCACHEABLE;
2702 case PMAP_WRITE_COMBINE:
2703 /* XXX implement if possible */
2704 KASSERT(1);
2705 return VM_MEMATTR_WRITE_COMBINING;
2706 case PMAP_WRITE_BACK:
2707 return VM_MEMATTR_WRITE_BACK;
2708 case PMAP_NOCACHE_OVR:
2709 default:
2710 return VM_MEMATTR_DEFAULT;
2711 }
2712 }
2713
2714 #ifdef DEBUG
2715 /*
2716 * Test ref/modify handling.
2717 */
2718
2719 static void
pmap_testout(void)2720 pmap_testout(void)
2721 {
2722 vaddr_t va;
2723 volatile int *loc;
2724 int val = 0;
2725 paddr_t pa;
2726 struct vm_page *pg;
2727 int ref, mod;
2728 bool extracted;
2729
2730 /* Allocate a page */
2731 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED | UVM_KMF_ZERO);
2732 KASSERT(va != 0);
2733 loc = (int*)va;
2734
2735 extracted = pmap_extract(pmap_kernel(), va, &pa);
2736 printf("va %p pa %lx extracted %u\n", (void *)(u_long)va, pa, extracted);
2737 printf("kextract %lx\n", pmap_kextract(va));
2738
2739 pg = PHYS_TO_VM_PAGE(pa);
2740 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
2741 pmap_update(pmap_kernel());
2742
2743 ref = pmap_is_referenced(pg);
2744 mod = pmap_is_modified(pg);
2745
2746 printf("Entered page va %p pa %lx: ref %d, mod %d\n",
2747 (void *)(u_long)va, (long)pa, ref, mod);
2748
2749 /* Now clear reference and modify */
2750 ref = pmap_clear_reference(pg);
2751 mod = pmap_clear_modify(pg);
2752 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
2753 (void *)(u_long)va, (long)pa,
2754 ref, mod);
2755
2756 /* Check it's properly cleared */
2757 ref = pmap_is_referenced(pg);
2758 mod = pmap_is_modified(pg);
2759 printf("Checking cleared page: ref %d, mod %d\n",
2760 ref, mod);
2761
2762 /* Reference page */
2763 val = *loc;
2764
2765 ref = pmap_is_referenced(pg);
2766 mod = pmap_is_modified(pg);
2767 printf("Referenced page: ref %d, mod %d val %x\n",
2768 ref, mod, val);
2769
2770 /* Now clear reference and modify */
2771 ref = pmap_clear_reference(pg);
2772 mod = pmap_clear_modify(pg);
2773 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
2774 (void *)(u_long)va, (long)pa,
2775 ref, mod);
2776
2777 /* Modify page */
2778 *loc = 1;
2779
2780 ref = pmap_is_referenced(pg);
2781 mod = pmap_is_modified(pg);
2782 printf("Modified page: ref %d, mod %d\n",
2783 ref, mod);
2784
2785 /* Now clear reference and modify */
2786 ref = pmap_clear_reference(pg);
2787 mod = pmap_clear_modify(pg);
2788 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
2789 (void *)(u_long)va, (long)pa,
2790 ref, mod);
2791
2792 /* Check it's properly cleared */
2793 ref = pmap_is_referenced(pg);
2794 mod = pmap_is_modified(pg);
2795 printf("Checking cleared page: ref %d, mod %d\n",
2796 ref, mod);
2797
2798 /* Modify page */
2799 *loc = 1;
2800
2801 ref = pmap_is_referenced(pg);
2802 mod = pmap_is_modified(pg);
2803 printf("Modified page: ref %d, mod %d\n",
2804 ref, mod);
2805
2806 /* Check pmap_protect() */
2807 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ);
2808 pmap_update(pmap_kernel());
2809 ref = pmap_is_referenced(pg);
2810 mod = pmap_is_modified(pg);
2811 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n",
2812 ref, mod);
2813
2814 /* Now clear reference and modify */
2815 ref = pmap_clear_reference(pg);
2816 mod = pmap_clear_modify(pg);
2817 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
2818 (void *)(u_long)va, (long)pa,
2819 ref, mod);
2820
2821 /* Modify page */
2822 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
2823 pmap_update(pmap_kernel());
2824 *loc = 1;
2825
2826 ref = pmap_is_referenced(pg);
2827 mod = pmap_is_modified(pg);
2828 printf("Modified page: ref %d, mod %d\n",
2829 ref, mod);
2830
2831 /* Check pmap_protect() */
2832 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE);
2833 pmap_update(pmap_kernel());
2834 ref = pmap_is_referenced(pg);
2835 mod = pmap_is_modified(pg);
2836 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n",
2837 ref, mod);
2838
2839 /* Now clear reference and modify */
2840 ref = pmap_clear_reference(pg);
2841 mod = pmap_clear_modify(pg);
2842 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
2843 (void *)(u_long)va, (long)pa,
2844 ref, mod);
2845
2846 /* Modify page */
2847 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
2848 pmap_update(pmap_kernel());
2849 *loc = 1;
2850
2851 ref = pmap_is_referenced(pg);
2852 mod = pmap_is_modified(pg);
2853 printf("Modified page: ref %d, mod %d\n",
2854 ref, mod);
2855
2856 /* Check pmap_pag_protect() */
2857 pmap_page_protect(pg, VM_PROT_READ);
2858 ref = pmap_is_referenced(pg);
2859 mod = pmap_is_modified(pg);
2860 printf("pmap_protect(): ref %d, mod %d\n",
2861 ref, mod);
2862
2863 /* Now clear reference and modify */
2864 ref = pmap_clear_reference(pg);
2865 mod = pmap_clear_modify(pg);
2866 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
2867 (void *)(u_long)va, (long)pa,
2868 ref, mod);
2869
2870
2871 /* Modify page */
2872 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
2873 pmap_update(pmap_kernel());
2874 *loc = 1;
2875
2876 ref = pmap_is_referenced(pg);
2877 mod = pmap_is_modified(pg);
2878 printf("Modified page: ref %d, mod %d\n",
2879 ref, mod);
2880
2881 /* Check pmap_pag_protect() */
2882 pmap_page_protect(pg, VM_PROT_NONE);
2883 ref = pmap_is_referenced(pg);
2884 mod = pmap_is_modified(pg);
2885 printf("pmap_protect(): ref %d, mod %d\n",
2886 ref, mod);
2887
2888 /* Now clear reference and modify */
2889 ref = pmap_clear_reference(pg);
2890 mod = pmap_clear_modify(pg);
2891 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
2892 (void *)(u_long)va, (long)pa,
2893 ref, mod);
2894
2895 /* Unmap page */
2896 pmap_remove(pmap_kernel(), va, va+1);
2897 pmap_update(pmap_kernel());
2898 ref = pmap_is_referenced(pg);
2899 mod = pmap_is_modified(pg);
2900 printf("Unmapped page: ref %d, mod %d\n", ref, mod);
2901
2902 /* Now clear reference and modify */
2903 ref = pmap_clear_reference(pg);
2904 mod = pmap_clear_modify(pg);
2905 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
2906 (void *)(u_long)va, (long)pa, ref, mod);
2907
2908 /* Check it's properly cleared */
2909 ref = pmap_is_referenced(pg);
2910 mod = pmap_is_modified(pg);
2911 printf("Checking cleared page: ref %d, mod %d\n",
2912 ref, mod);
2913
2914 pmap_remove(pmap_kernel(), va, va+1);
2915 pmap_update(pmap_kernel());
2916 //pmap_free_page(pa, cpus_active);
2917 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE,
2918 UVM_KMF_WIRED | UVM_KMF_ZERO);
2919
2920 panic("end of testout");
2921 }
2922 #endif
2923