1 /* 2 * Copyright (c) 2003-2011 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * pmap invalidation support code. Certain hardware requirements must 37 * be dealt with when manipulating page table entries and page directory 38 * entries within a pmap. In particular, we cannot safely manipulate 39 * page tables which are in active use by another cpu (even if it is 40 * running in userland) for two reasons: First, TLB writebacks will 41 * race against our own modifications and tests. Second, even if we 42 * were to use bus-locked instruction we can still screw up the 43 * target cpu's instruction pipeline due to Intel cpu errata. 44 */ 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/proc.h> 50 #include <sys/vmmeter.h> 51 #include <sys/thread2.h> 52 53 #include <vm/vm.h> 54 #include <vm/pmap.h> 55 #include <vm/vm_object.h> 56 57 #include <machine/cputypes.h> 58 #include <machine/md_var.h> 59 #include <machine/specialreg.h> 60 #include <machine/smp.h> 61 #include <machine/globaldata.h> 62 #include <machine/pmap.h> 63 #include <machine/pmap_inval.h> 64 65 static void pmap_inval_callback(void *arg); 66 67 /* 68 * Initialize for add or flush 69 * 70 * The critical section is required to prevent preemption, allowing us to 71 * set CPULOCK_EXCL on the pmap. The critical section is also assumed 72 * when lwkt_process_ipiq() is called. 73 */ 74 void 75 pmap_inval_init(pmap_inval_info_t info) 76 { 77 info->pir_flags = 0; 78 crit_enter_id("inval"); 79 } 80 81 /* 82 * Add a (pmap, va) pair to the invalidation list and protect access 83 * as appropriate. 84 * 85 * CPULOCK_EXCL is used to interlock thread switchins, otherwise another 86 * cpu can switch in a pmap that we are unaware of and interfere with our 87 * pte operation. 88 * 89 * NOTE! If pmap_fast_kernel_cpusync is enabled, interlocks on kernel_pmap 90 * are effectively NOPs and will not quiesce target cpus. The 91 * deinterlock will then issue the IPI and wait for completion, 92 * which avoids spinning all AP cpus at once. 93 * 94 * This needs testing before it can be enabled by default. 95 */ 96 void 97 pmap_inval_interlock(pmap_inval_info_t info, pmap_t pmap, vm_offset_t va) 98 { 99 cpulock_t olock; 100 cpulock_t nlock; 101 102 DEBUG_PUSH_INFO("pmap_inval_interlock"); 103 for (;;) { 104 olock = pmap->pm_active_lock; 105 cpu_ccfence(); 106 nlock = olock | CPULOCK_EXCL; 107 if (olock != nlock && 108 atomic_cmpset_int(&pmap->pm_active_lock, olock, nlock)) { 109 break; 110 } 111 lwkt_process_ipiq(); 112 cpu_pause(); 113 } 114 DEBUG_POP_INFO(); 115 KKASSERT((info->pir_flags & PIRF_CPUSYNC) == 0); 116 117 info->pir_va = va; 118 info->pir_flags = PIRF_CPUSYNC; 119 lwkt_cpusync_init(&info->pir_cpusync, pmap->pm_active, 120 pmap_inval_callback, info); 121 if (pmap == &kernel_pmap && pmap_fast_kernel_cpusync) { 122 info->pir_flags |= PIRF_QUICK; 123 } else { 124 lwkt_cpusync_interlock(&info->pir_cpusync); 125 atomic_add_acq_long(&pmap->pm_invgen, 1); 126 } 127 } 128 129 void 130 pmap_inval_invltlb(pmap_inval_info_t info) 131 { 132 info->pir_va = (vm_offset_t)-1; 133 } 134 135 /* 136 * Deinterlock a pmap after making a change to a PTE. 137 * 138 * WARNING! We currently do not use a fully synchronous cpusync for 139 * kernel_map adjustments. We assume that all use cases for 140 * accesses via the kernel map are locally interlocked, so instead 141 * we use a semi-synchronous smp_invltlb(). 142 */ 143 void 144 pmap_inval_deinterlock(pmap_inval_info_t info, pmap_t pmap) 145 { 146 KKASSERT(info->pir_flags & PIRF_CPUSYNC); 147 atomic_clear_int(&pmap->pm_active_lock, CPULOCK_EXCL); 148 if (info->pir_flags & PIRF_QUICK) { 149 atomic_add_acq_long(&pmap->pm_invgen, 1); 150 lwkt_cpusync_quick(&info->pir_cpusync); 151 } else { 152 lwkt_cpusync_deinterlock(&info->pir_cpusync); 153 } 154 info->pir_flags = 0; 155 } 156 157 static void 158 pmap_inval_callback(void *arg) 159 { 160 pmap_inval_info_t info = arg; 161 162 if (info->pir_va == (vm_offset_t)-1) 163 cpu_invltlb(); 164 else 165 cpu_invlpg((void *)info->pir_va); 166 } 167 168 void 169 pmap_inval_done(pmap_inval_info_t info) 170 { 171 KKASSERT((info->pir_flags & PIRF_CPUSYNC) == 0); 172 crit_exit_id("inval"); 173 } 174