1 /* 2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Jeffrey M. Hsu. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of The DragonFly Project nor the names of its 16 * contributors may be used to endorse or promote products derived 17 * from this software without specific, prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * $DragonFly: src/sys/sys/spinlock2.h,v 1.11 2006/06/01 19:02:39 dillon Exp $ 33 */ 34 35 #ifndef _SYS_SPINLOCK2_H_ 36 #define _SYS_SPINLOCK2_H_ 37 38 #ifndef _KERNEL 39 40 #error "This file should not be included by userland programs." 41 42 #else 43 44 #ifndef _SYS_SYSTM_H_ 45 #include <sys/systm.h> 46 #endif 47 #ifndef _SYS_THREAD2_H_ 48 #include <sys/thread2.h> 49 #endif 50 #ifndef _SYS_GLOBALDATA_H_ 51 #include <sys/globaldata.h> 52 #endif 53 #ifndef _MACHINE_ATOMIC_H_ 54 #include <machine/atomic.h> 55 #endif 56 #ifndef _MACHINE_CPUFUNC_H_ 57 #include <machine/cpufunc.h> 58 #endif 59 60 /* 61 * SPECIAL NOTE! Obtaining a spinlock does not enter a critical section 62 * or protect against FAST interrupts but it will prevent thread preemption. 63 * Because the spinlock code path is ultra critical, we do not check for 64 * LWKT reschedule requests (due to an interrupt thread not being able to 65 * preempt). 66 */ 67 68 #ifdef SMP 69 70 extern int spin_trylock_wr_contested(struct spinlock *mtx, int value); 71 extern void spin_lock_wr_contested(struct spinlock *mtx, int value); 72 extern void spin_lock_rd_contested(struct spinlock *mtx); 73 74 #endif 75 76 #ifdef SMP 77 78 /* 79 * Attempt to obtain an exclusive spinlock. Returns FALSE on failure, 80 * TRUE on success. Since the caller assumes that spinlocks must actually 81 * work when using this function, it is only made available to SMP builds. 82 */ 83 static __inline boolean_t 84 spin_trylock_wr(struct spinlock *mtx) 85 { 86 globaldata_t gd = mycpu; 87 int value; 88 89 ++gd->gd_spinlocks_wr; 90 if ((value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE)) != 0) 91 return (spin_trylock_wr_contested(mtx, value)); 92 return (TRUE); 93 } 94 95 #endif 96 97 /* 98 * Obtain an exclusive spinlock and return. Shortcut the case where the only 99 * cached read lock was from our own cpu (it can just be cleared). 100 */ 101 static __inline void 102 spin_lock_wr_quick(globaldata_t gd, struct spinlock *mtx) 103 { 104 #ifdef SMP 105 int value; 106 #endif 107 108 ++gd->gd_spinlocks_wr; 109 #ifdef SMP 110 if ((value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE)) != 0) { 111 value &= ~gd->gd_cpumask; 112 if (value) 113 spin_lock_wr_contested(mtx, value); 114 } 115 #endif 116 } 117 118 static __inline void 119 spin_lock_wr(struct spinlock *mtx) 120 { 121 spin_lock_wr_quick(mycpu, mtx); 122 } 123 124 #if 0 125 126 /* 127 * Upgrade a shared spinlock to exclusive. Return TRUE if we were 128 * able to upgrade without another exclusive holder getting in before 129 * us, FALSE otherwise. 130 */ 131 static __inline int 132 spin_lock_upgrade(struct spinlock *mtx) 133 { 134 globaldata_t gd = mycpu; 135 #ifdef SMP 136 int value; 137 #endif 138 139 ++gd->gd_spinlocks_wr; 140 #ifdef SMP 141 value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE); 142 cpu_sfence(); 143 #endif 144 gd->gd_spinlock_rd = NULL; 145 #ifdef SMP 146 value &= ~gd->gd_cpumask; 147 if (value) { 148 spin_lock_wr_contested(mtx, value); 149 if (value & SPINLOCK_EXCLUSIVE) 150 return (FALSE); 151 XXX regain original shared lock? 152 } 153 return (TRUE); 154 #endif 155 } 156 157 #endif 158 159 /* 160 * Obtain a shared spinlock and return. This is a critical code path. 161 * 162 * The vast majority of the overhead is in the cpu_mfence() (5ns vs 1ns for 163 * the entire rest of the procedure). Unfortunately we have to ensure that 164 * spinlock pointer is written out before we check the cpumask to interlock 165 * against an exclusive spinlock that clears the cpumask and then checks 166 * the spinlock pointer. 167 * 168 * But what is EXTREMELY important here is that we do not have to perform 169 * a locked bus cycle on the spinlock itself if the shared bit for our cpu 170 * is already found to be set. We only need the mfence, and the mfence is 171 * local to the cpu and never conflicts with other cpu's. 172 * 173 * This means that multiple parallel shared acessors (e.g. filedescriptor 174 * table lookups, namecache lookups) run at full speed and incur NO cache 175 * contention at all. It is the difference between 10ns and 40-100ns. 176 */ 177 static __inline void 178 spin_lock_rd_quick(globaldata_t gd, struct spinlock *mtx) 179 { 180 gd->gd_spinlock_rd = mtx; 181 #ifdef SMP 182 cpu_mfence(); 183 if ((mtx->lock & gd->gd_cpumask) == 0) 184 spin_lock_rd_contested(mtx); 185 #endif 186 } 187 188 static __inline void 189 spin_lock_rd(struct spinlock *mtx) 190 { 191 spin_lock_rd_quick(mycpu,mtx); 192 } 193 194 /* 195 * Release an exclusive spinlock. We can just do this passively, only 196 * ensuring that our spinlock count is left intact until the mutex is 197 * cleared. 198 */ 199 static __inline void 200 spin_unlock_wr_quick(globaldata_t gd, struct spinlock *mtx) 201 { 202 #ifdef SMP 203 mtx->lock = 0; 204 #endif 205 --gd->gd_spinlocks_wr; 206 } 207 208 static __inline void 209 spin_unlock_wr(struct spinlock *mtx) 210 { 211 spin_unlock_wr_quick(mycpu, mtx); 212 } 213 214 /* 215 * Release a shared spinlock. We leave the shared bit set in the spinlock 216 * as a cache and simply clear the spinlock pointer for the cpu. This 217 * fast-paths another shared lock later at the cost of an exclusive lock 218 * having to check per-cpu spinlock pointers to determine when there are no 219 * shared holders remaining. 220 */ 221 static __inline void 222 spin_unlock_rd_quick(globaldata_t gd, struct spinlock *mtx) 223 { 224 KKASSERT(gd->gd_spinlock_rd == mtx); 225 gd->gd_spinlock_rd = NULL; 226 } 227 228 static __inline void 229 spin_unlock_rd(struct spinlock *mtx) 230 { 231 spin_unlock_rd_quick(mycpu, mtx); 232 } 233 234 static __inline void 235 spin_init(struct spinlock *mtx) 236 { 237 mtx->lock = 0; 238 } 239 240 static __inline void 241 spin_uninit(struct spinlock *mtx) 242 { 243 /* unused */ 244 } 245 246 #endif /* _KERNEL */ 247 #endif /* _SYS_SPINLOCK2_H_ */ 248 249