1 /* 2 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $DragonFly: src/sys/platform/vkernel/platform/pmap_inval.c,v 1.4 2007/07/02 02:22:58 dillon Exp $ 35 */ 36 37 /* 38 * pmap invalidation support code. Certain hardware requirements must 39 * be dealt with when manipulating page table entries and page directory 40 * entries within a pmap. In particular, we cannot safely manipulate 41 * page tables which are in active use by another cpu (even if it is 42 * running in userland) for two reasons: First, TLB writebacks will 43 * race against our own modifications and tests. Second, even if we 44 * were to use bus-locked instruction we can still screw up the 45 * target cpu's instruction pipeline due to Intel cpu errata. 46 * 47 * For our virtual page tables, the real kernel will handle SMP interactions 48 * with pmaps that may be active on other cpus. Even so, we have to be 49 * careful about bit setting races particularly when we are trying to clean 50 * a page and test the modified bit to avoid races where the modified bit 51 * might get set after our poll but before we clear the field. 52 */ 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/kernel.h> 56 #include <sys/proc.h> 57 #include <sys/vmmeter.h> 58 #include <sys/thread2.h> 59 60 #include <sys/mman.h> 61 #include <sys/vmspace.h> 62 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 #include <vm/vm_object.h> 66 67 #include <machine/cputypes.h> 68 #include <machine/md_var.h> 69 #include <machine/specialreg.h> 70 #include <machine/smp.h> 71 #include <machine/globaldata.h> 72 #include <machine/pmap.h> 73 #include <machine/pmap_inval.h> 74 75 static __inline 76 void 77 pmap_inval_cpu(struct pmap *pmap, vm_offset_t va, size_t bytes) 78 { 79 if (pmap == &kernel_pmap) { 80 madvise((void *)va, bytes, MADV_INVAL); 81 } else { 82 vmspace_mcontrol(pmap, (void *)va, bytes, MADV_INVAL, 0); 83 } 84 } 85 86 /* 87 * Invalidate a pte in a pmap and synchronize with target cpus 88 * as required. Throw away the modified and access bits. Use 89 * pmap_clean_pte() to do the same thing but also get an interlocked 90 * modified/access status. 91 * 92 * Clearing the field first (basically clearing VPTE_V) prevents any 93 * new races from occuring while we invalidate the TLB (i.e. the pmap 94 * on the real cpu), then clear it again to clean out any race that 95 * might have occured before the invalidation completed. 96 */ 97 void 98 pmap_inval_pte(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va) 99 { 100 *ptep = 0; 101 pmap_inval_cpu(pmap, va, PAGE_SIZE); 102 *ptep = 0; 103 } 104 105 /* 106 * Same as pmap_inval_pte() but only synchronize with the current 107 * cpu. For the moment its the same as the non-quick version. 108 */ 109 void 110 pmap_inval_pte_quick(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va) 111 { 112 *ptep = 0; 113 pmap_inval_cpu(pmap, va, PAGE_SIZE); 114 *ptep = 0; 115 } 116 117 /* 118 * Invalidating page directory entries requires some additional 119 * sophistication. The cachemask must be cleared so the kernel 120 * resynchronizes its temporary page table mappings cache. 121 */ 122 void 123 pmap_inval_pde(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va) 124 { 125 *ptep = 0; 126 pmap_inval_cpu(pmap, va, SEG_SIZE); 127 *ptep = 0; 128 } 129 130 void 131 pmap_inval_pde_quick(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va) 132 { 133 pmap_inval_pde(ptep, pmap, va); 134 } 135 136 /* 137 * These carefully handle interactions with other cpus and return 138 * the original vpte. Clearing VPTE_W prevents us from racing the 139 * setting of VPTE_M, allowing us to invalidate the tlb (the real cpu's 140 * pmap) and get good status for VPTE_M. 141 * 142 * When messing with page directory entries we have to clear the cpu 143 * mask to force a reload of the kernel's page table mapping cache. 144 * 145 * clean: clear VPTE_M and VPTE_W 146 * setro: clear VPTE_W 147 * load&clear: clear entire field 148 */ 149 vpte_t 150 pmap_clean_pte(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va) 151 { 152 vpte_t pte; 153 154 pte = *ptep; 155 if (pte & VPTE_V) { 156 atomic_clear_long(ptep, VPTE_W); 157 pmap_inval_cpu(pmap, va, PAGE_SIZE); 158 pte = *ptep; 159 atomic_clear_long(ptep, VPTE_W|VPTE_M); 160 } 161 return(pte); 162 } 163 164 vpte_t 165 pmap_clean_pde(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va) 166 { 167 vpte_t pte; 168 169 pte = *ptep; 170 if (pte & VPTE_V) { 171 atomic_clear_long(ptep, VPTE_W); 172 pmap_inval_cpu(pmap, va, SEG_SIZE); 173 pte = *ptep; 174 atomic_clear_long(ptep, VPTE_W|VPTE_M); 175 } 176 return(pte); 177 } 178 179 /* 180 * This is an odd case and I'm not sure whether it even occurs in normal 181 * operation. Turn off write access to the page, clean out the tlb 182 * (the real cpu's pmap), and deal with any VPTE_M race that may have 183 * occured. VPTE_M is not cleared. 184 */ 185 vpte_t 186 pmap_setro_pte(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va) 187 { 188 vpte_t pte; 189 190 pte = *ptep; 191 if (pte & VPTE_V) { 192 pte = *ptep; 193 atomic_clear_long(ptep, VPTE_W); 194 pmap_inval_cpu(pmap, va, PAGE_SIZE); 195 pte |= *ptep & VPTE_M; 196 } 197 return(pte); 198 } 199 200 /* 201 * This is a combination of pmap_inval_pte() and pmap_clean_pte(). 202 * Firts prevent races with the 'A' and 'M' bits, then clean out 203 * the tlb (the real cpu's pmap), then incorporate any races that 204 * may have occured in the mean time, and finally zero out the pte. 205 */ 206 vpte_t 207 pmap_inval_loadandclear(volatile vpte_t *ptep, struct pmap *pmap, 208 vm_offset_t va) 209 { 210 vpte_t pte; 211 212 pte = *ptep; 213 if (pte & VPTE_V) { 214 pte = *ptep; 215 atomic_clear_long(ptep, VPTE_R|VPTE_W); 216 pmap_inval_cpu(pmap, va, PAGE_SIZE); 217 pte |= *ptep & (VPTE_A | VPTE_M); 218 } 219 *ptep = 0; 220 return(pte); 221 } 222