1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2005 Peter Grehan 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30 #include <sys/cdefs.h> 31 /* 32 * Dispatch MI pmap calls to the appropriate MMU implementation 33 * through a previously registered kernel object. 34 * 35 * Before pmap_bootstrap() can be called, a CPU module must have 36 * called pmap_mmu_install(). This may be called multiple times: 37 * the highest priority call will be installed as the default 38 * MMU handler when pmap_bootstrap() is called. 39 * 40 * It is required that mutex_init() be called before pmap_bootstrap(), 41 * as the PMAP layer makes extensive use of mutexes. 42 */ 43 44 #include <sys/param.h> 45 #include <sys/kernel.h> 46 #include <sys/conf.h> 47 #include <sys/lock.h> 48 #include <sys/kerneldump.h> 49 #include <sys/ktr.h> 50 #include <sys/mutex.h> 51 #include <sys/sysctl.h> 52 #include <sys/systm.h> 53 54 #include <vm/vm.h> 55 #include <vm/vm_extern.h> 56 #include <vm/vm_page.h> 57 58 #include <machine/dump.h> 59 #include <machine/ifunc.h> 60 #include <machine/md_var.h> 61 #include <machine/mmuvar.h> 62 #include <machine/smp.h> 63 64 mmu_t mmu_obj; 65 66 /* 67 * pmap globals 68 */ 69 struct pmap kernel_pmap_store; 70 71 vm_offset_t msgbuf_phys; 72 73 vm_offset_t kernel_vm_end; 74 vm_offset_t virtual_avail; 75 vm_offset_t virtual_end; 76 caddr_t crashdumpmap; 77 78 int pmap_bootstrapped; 79 /* Default level 0 reservations consist of 512 pages (2MB superpage). */ 80 int vm_level_0_order = 9; 81 82 SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); 83 84 int superpages_enabled = 1; 85 SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, CTLFLAG_RDTUN, 86 &superpages_enabled, 0, "Enable support for transparent superpages"); 87 88 #ifdef AIM 89 int 90 pvo_vaddr_compare(struct pvo_entry *a, struct pvo_entry *b) 91 { 92 if (PVO_VADDR(a) < PVO_VADDR(b)) 93 return (-1); 94 else if (PVO_VADDR(a) > PVO_VADDR(b)) 95 return (1); 96 return (0); 97 } 98 RB_GENERATE(pvo_tree, pvo_entry, pvo_plink, pvo_vaddr_compare); 99 #endif 100 101 static int 102 pmap_nomethod(void) 103 { 104 return (0); 105 } 106 107 #define DEFINE_PMAP_IFUNC(ret, func, args) \ 108 DEFINE_IFUNC(, ret, pmap_##func, args) { \ 109 pmap_##func##_t f; \ 110 f = PMAP_RESOLVE_FUNC(func); \ 111 return (f != NULL ? f : (pmap_##func##_t)pmap_nomethod);\ 112 } 113 #define DEFINE_DUMPSYS_IFUNC(ret, func, args) \ 114 DEFINE_IFUNC(, ret, dumpsys_##func, args) { \ 115 pmap_dumpsys_##func##_t f; \ 116 f = PMAP_RESOLVE_FUNC(dumpsys_##func); \ 117 return (f != NULL ? f : (pmap_dumpsys_##func##_t)pmap_nomethod);\ 118 } 119 120 DEFINE_PMAP_IFUNC(void, activate, (struct thread *)); 121 DEFINE_PMAP_IFUNC(void, advise, (pmap_t, vm_offset_t, vm_offset_t, int)); 122 DEFINE_PMAP_IFUNC(void, align_superpage, (vm_object_t, vm_ooffset_t, 123 vm_offset_t *, vm_size_t)); 124 DEFINE_PMAP_IFUNC(void, clear_modify, (vm_page_t)); 125 DEFINE_PMAP_IFUNC(void, copy, (pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t)); 126 DEFINE_PMAP_IFUNC(int, enter, (pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t)); 127 DEFINE_PMAP_IFUNC(void, enter_quick, (pmap_t, vm_offset_t, vm_page_t, vm_prot_t)); 128 DEFINE_PMAP_IFUNC(void, enter_object, (pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 129 vm_prot_t)); 130 DEFINE_PMAP_IFUNC(vm_paddr_t, extract, (pmap_t, vm_offset_t)); 131 DEFINE_PMAP_IFUNC(vm_page_t, extract_and_hold, (pmap_t, vm_offset_t, vm_prot_t)); 132 DEFINE_PMAP_IFUNC(void, kenter, (vm_offset_t, vm_paddr_t)); 133 DEFINE_PMAP_IFUNC(void, kenter_attr, (vm_offset_t, vm_paddr_t, vm_memattr_t)); 134 DEFINE_PMAP_IFUNC(vm_paddr_t, kextract, (vm_offset_t)); 135 DEFINE_PMAP_IFUNC(void, kremove, (vm_offset_t)); 136 DEFINE_PMAP_IFUNC(void, object_init_pt, (pmap_t, vm_offset_t, vm_object_t, vm_pindex_t, 137 vm_size_t)); 138 DEFINE_PMAP_IFUNC(bool, is_modified, (vm_page_t)); 139 DEFINE_PMAP_IFUNC(bool, is_prefaultable, (pmap_t, vm_offset_t)); 140 DEFINE_PMAP_IFUNC(bool, is_referenced, (vm_page_t)); 141 DEFINE_PMAP_IFUNC(bool, page_exists_quick, (pmap_t, vm_page_t)); 142 DEFINE_PMAP_IFUNC(void, page_init, (vm_page_t)); 143 DEFINE_PMAP_IFUNC(bool, page_is_mapped, (vm_page_t)); 144 DEFINE_PMAP_IFUNC(int, page_wired_mappings, (vm_page_t)); 145 DEFINE_PMAP_IFUNC(void, protect, (pmap_t, vm_offset_t, vm_offset_t, vm_prot_t)); 146 DEFINE_PMAP_IFUNC(bool, ps_enabled, (pmap_t)); 147 DEFINE_PMAP_IFUNC(void, qenter, (vm_offset_t, vm_page_t *, int)); 148 DEFINE_PMAP_IFUNC(void, qremove, (vm_offset_t, int)); 149 DEFINE_PMAP_IFUNC(vm_offset_t, quick_enter_page, (vm_page_t)); 150 DEFINE_PMAP_IFUNC(void, quick_remove_page, (vm_offset_t)); 151 DEFINE_PMAP_IFUNC(int, ts_referenced, (vm_page_t)); 152 DEFINE_PMAP_IFUNC(void, release, (pmap_t)); 153 DEFINE_PMAP_IFUNC(void, remove, (pmap_t, vm_offset_t, vm_offset_t)); 154 DEFINE_PMAP_IFUNC(void, remove_all, (vm_page_t)); 155 DEFINE_PMAP_IFUNC(void, remove_pages, (pmap_t)); 156 DEFINE_PMAP_IFUNC(void, remove_write, (vm_page_t)); 157 DEFINE_PMAP_IFUNC(void, unwire, (pmap_t, vm_offset_t, vm_offset_t)); 158 DEFINE_PMAP_IFUNC(void, zero_page, (vm_page_t)); 159 DEFINE_PMAP_IFUNC(void, zero_page_area, (vm_page_t, int, int)); 160 DEFINE_PMAP_IFUNC(void, copy_page, (vm_page_t, vm_page_t)); 161 DEFINE_PMAP_IFUNC(void, copy_pages, 162 (vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 163 vm_offset_t b_offset, int xfersize)); 164 DEFINE_PMAP_IFUNC(void, growkernel, (vm_offset_t)); 165 DEFINE_PMAP_IFUNC(void, init, (void)); 166 DEFINE_PMAP_IFUNC(vm_offset_t, map, (vm_offset_t *, vm_paddr_t, vm_paddr_t, int)); 167 DEFINE_PMAP_IFUNC(int, pinit, (pmap_t)); 168 DEFINE_PMAP_IFUNC(void, pinit0, (pmap_t)); 169 DEFINE_PMAP_IFUNC(int, mincore, (pmap_t, vm_offset_t, vm_paddr_t *)); 170 DEFINE_PMAP_IFUNC(void, deactivate, (struct thread *)); 171 DEFINE_PMAP_IFUNC(void, bootstrap, (vm_offset_t, vm_offset_t)); 172 DEFINE_PMAP_IFUNC(void, cpu_bootstrap, (int)); 173 DEFINE_PMAP_IFUNC(void *, mapdev, (vm_paddr_t, vm_size_t)); 174 DEFINE_PMAP_IFUNC(void *, mapdev_attr, (vm_paddr_t, vm_size_t, vm_memattr_t)); 175 DEFINE_PMAP_IFUNC(void, page_set_memattr, (vm_page_t, vm_memattr_t)); 176 DEFINE_PMAP_IFUNC(void, unmapdev, (void *, vm_size_t)); 177 DEFINE_PMAP_IFUNC(int, map_user_ptr, 178 (pmap_t, volatile const void *, void **, size_t, size_t *)); 179 DEFINE_PMAP_IFUNC(int, decode_kernel_ptr, (vm_offset_t, int *, vm_offset_t *)); 180 DEFINE_PMAP_IFUNC(int, dev_direct_mapped, (vm_paddr_t, vm_size_t)); 181 DEFINE_PMAP_IFUNC(void, sync_icache, (pmap_t, vm_offset_t, vm_size_t)); 182 DEFINE_PMAP_IFUNC(int, change_attr, (vm_offset_t, vm_size_t, vm_memattr_t)); 183 DEFINE_PMAP_IFUNC(void, page_array_startup, (long)); 184 DEFINE_PMAP_IFUNC(void, tlbie_all, (void)); 185 186 DEFINE_DUMPSYS_IFUNC(void, map_chunk, (vm_paddr_t, size_t, void **)); 187 DEFINE_DUMPSYS_IFUNC(void, unmap_chunk, (vm_paddr_t, size_t, void *)); 188 DEFINE_DUMPSYS_IFUNC(void, pa_init, (void)); 189 DEFINE_DUMPSYS_IFUNC(size_t, scan_pmap, (struct bitset *)); 190 DEFINE_DUMPSYS_IFUNC(void *, dump_pmap_init, (unsigned)); 191 DEFINE_DUMPSYS_IFUNC(void *, dump_pmap, (void *, void *, u_long *)); 192 193 /* 194 * MMU install routines. Highest priority wins, equal priority also 195 * overrides allowing last-set to win. 196 */ 197 SET_DECLARE(mmu_set, struct mmu_kobj); 198 199 bool 200 pmap_mmu_install(char *name, int prio) 201 { 202 mmu_t *mmupp, mmup; 203 static int curr_prio = 0; 204 205 /* 206 * Try and locate the MMU kobj corresponding to the name 207 */ 208 SET_FOREACH(mmupp, mmu_set) { 209 mmup = *mmupp; 210 211 if (mmup->name && 212 !strcmp(mmup->name, name) && 213 (prio >= curr_prio || mmu_obj == NULL)) { 214 curr_prio = prio; 215 mmu_obj = mmup; 216 return (true); 217 } 218 } 219 220 return (false); 221 } 222 223 /* MMU "pre-bootstrap" init, used to install extra resolvers, etc. */ 224 void 225 pmap_mmu_init(void) 226 { 227 if (mmu_obj->funcs->install != NULL) 228 (mmu_obj->funcs->install)(); 229 } 230 231 const char * 232 pmap_mmu_name(void) 233 { 234 return (mmu_obj->name); 235 } 236 237 int unmapped_buf_allowed; 238 239 bool 240 pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode) 241 { 242 243 switch (mode) { 244 case VM_MEMATTR_DEFAULT: 245 case VM_MEMATTR_UNCACHEABLE: 246 case VM_MEMATTR_CACHEABLE: 247 case VM_MEMATTR_WRITE_COMBINING: 248 case VM_MEMATTR_WRITE_BACK: 249 case VM_MEMATTR_WRITE_THROUGH: 250 case VM_MEMATTR_PREFETCHABLE: 251 return (true); 252 default: 253 return (false); 254 } 255 } 256 257 void 258 pmap_active_cpus(pmap_t pmap, cpuset_t *res) 259 { 260 *res = pmap->pm_active; 261 } 262