1 /* 2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Jeffrey M. Hsu. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of The DragonFly Project nor the names of its 16 * contributors may be used to endorse or promote products derived 17 * from this software without specific, prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * $DragonFly: src/sys/sys/spinlock2.h,v 1.12 2008/06/04 04:34:54 nth Exp $ 33 */ 34 35 #ifndef _SYS_SPINLOCK2_H_ 36 #define _SYS_SPINLOCK2_H_ 37 38 #ifndef _KERNEL 39 40 #error "This file should not be included by userland programs." 41 42 #else 43 44 #ifndef _SYS_SYSTM_H_ 45 #include <sys/systm.h> 46 #endif 47 #ifndef _SYS_THREAD2_H_ 48 #include <sys/thread2.h> 49 #endif 50 #ifndef _SYS_GLOBALDATA_H_ 51 #include <sys/globaldata.h> 52 #endif 53 #ifndef _MACHINE_ATOMIC_H_ 54 #include <machine/atomic.h> 55 #endif 56 #ifndef _MACHINE_CPUFUNC_H_ 57 #include <machine/cpufunc.h> 58 #endif 59 60 /* 61 * SPECIAL NOTE! Obtaining a spinlock does not enter a critical section 62 * or protect against FAST interrupts but it will prevent thread preemption. 63 * Because the spinlock code path is ultra critical, we do not check for 64 * LWKT reschedule requests (due to an interrupt thread not being able to 65 * preempt). 66 */ 67 68 #ifdef SMP 69 70 extern int spin_trylock_wr_contested(globaldata_t gd, struct spinlock *mtx, 71 int value); 72 extern void spin_lock_wr_contested(struct spinlock *mtx, int value); 73 extern void spin_lock_rd_contested(struct spinlock *mtx); 74 75 #endif 76 77 #ifdef SMP 78 79 /* 80 * Attempt to obtain an exclusive spinlock. Returns FALSE on failure, 81 * TRUE on success. Since the caller assumes that spinlocks must actually 82 * work when using this function, it is only made available to SMP builds. 83 */ 84 static __inline boolean_t 85 spin_trylock_wr(struct spinlock *mtx) 86 { 87 globaldata_t gd = mycpu; 88 int value; 89 90 ++gd->gd_spinlocks_wr; 91 if ((value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE)) != 0) 92 return (spin_trylock_wr_contested(gd, mtx, value)); 93 return (TRUE); 94 } 95 96 #endif 97 98 /* 99 * Obtain an exclusive spinlock and return. Shortcut the case where the only 100 * cached read lock was from our own cpu (it can just be cleared). 101 */ 102 static __inline void 103 spin_lock_wr_quick(globaldata_t gd, struct spinlock *mtx) 104 { 105 #ifdef SMP 106 int value; 107 #endif 108 109 ++gd->gd_spinlocks_wr; 110 #ifdef SMP 111 if ((value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE)) != 0) { 112 value &= ~gd->gd_cpumask; 113 if (value) 114 spin_lock_wr_contested(mtx, value); 115 } 116 #endif 117 } 118 119 static __inline void 120 spin_lock_wr(struct spinlock *mtx) 121 { 122 spin_lock_wr_quick(mycpu, mtx); 123 } 124 125 #if 0 126 127 /* 128 * Upgrade a shared spinlock to exclusive. Return TRUE if we were 129 * able to upgrade without another exclusive holder getting in before 130 * us, FALSE otherwise. 131 */ 132 static __inline int 133 spin_lock_upgrade(struct spinlock *mtx) 134 { 135 globaldata_t gd = mycpu; 136 #ifdef SMP 137 int value; 138 #endif 139 140 ++gd->gd_spinlocks_wr; 141 #ifdef SMP 142 value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE); 143 cpu_sfence(); 144 #endif 145 gd->gd_spinlock_rd = NULL; 146 #ifdef SMP 147 value &= ~gd->gd_cpumask; 148 if (value) { 149 spin_lock_wr_contested(mtx, value); 150 if (value & SPINLOCK_EXCLUSIVE) 151 return (FALSE); 152 XXX regain original shared lock? 153 } 154 return (TRUE); 155 #endif 156 } 157 158 #endif 159 160 /* 161 * Obtain a shared spinlock and return. This is a critical code path. 162 * 163 * The vast majority of the overhead is in the cpu_mfence() (5ns vs 1ns for 164 * the entire rest of the procedure). Unfortunately we have to ensure that 165 * spinlock pointer is written out before we check the cpumask to interlock 166 * against an exclusive spinlock that clears the cpumask and then checks 167 * the spinlock pointer. 168 * 169 * But what is EXTREMELY important here is that we do not have to perform 170 * a locked bus cycle on the spinlock itself if the shared bit for our cpu 171 * is already found to be set. We only need the mfence, and the mfence is 172 * local to the cpu and never conflicts with other cpu's. 173 * 174 * This means that multiple parallel shared acessors (e.g. filedescriptor 175 * table lookups, namecache lookups) run at full speed and incur NO cache 176 * contention at all. It is the difference between 10ns and 40-100ns. 177 */ 178 static __inline void 179 spin_lock_rd_quick(globaldata_t gd, struct spinlock *mtx) 180 { 181 gd->gd_spinlock_rd = mtx; 182 #ifdef SMP 183 cpu_mfence(); 184 if ((mtx->lock & gd->gd_cpumask) == 0) 185 spin_lock_rd_contested(mtx); 186 #endif 187 } 188 189 static __inline void 190 spin_lock_rd(struct spinlock *mtx) 191 { 192 spin_lock_rd_quick(mycpu,mtx); 193 } 194 195 /* 196 * Release an exclusive spinlock. We can just do this passively, only 197 * ensuring that our spinlock count is left intact until the mutex is 198 * cleared. 199 */ 200 static __inline void 201 spin_unlock_wr_quick(globaldata_t gd, struct spinlock *mtx) 202 { 203 #ifdef SMP 204 mtx->lock = 0; 205 #endif 206 --gd->gd_spinlocks_wr; 207 } 208 209 static __inline void 210 spin_unlock_wr(struct spinlock *mtx) 211 { 212 spin_unlock_wr_quick(mycpu, mtx); 213 } 214 215 /* 216 * Release a shared spinlock. We leave the shared bit set in the spinlock 217 * as a cache and simply clear the spinlock pointer for the cpu. This 218 * fast-paths another shared lock later at the cost of an exclusive lock 219 * having to check per-cpu spinlock pointers to determine when there are no 220 * shared holders remaining. 221 */ 222 static __inline void 223 spin_unlock_rd_quick(globaldata_t gd, struct spinlock *mtx) 224 { 225 KKASSERT(gd->gd_spinlock_rd == mtx); 226 gd->gd_spinlock_rd = NULL; 227 } 228 229 static __inline void 230 spin_unlock_rd(struct spinlock *mtx) 231 { 232 spin_unlock_rd_quick(mycpu, mtx); 233 } 234 235 static __inline void 236 spin_init(struct spinlock *mtx) 237 { 238 mtx->lock = 0; 239 } 240 241 static __inline void 242 spin_uninit(struct spinlock *mtx) 243 { 244 /* unused */ 245 } 246 247 #endif /* _KERNEL */ 248 #endif /* _SYS_SPINLOCK2_H_ */ 249 250