1 /* 2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Jeffrey M. Hsu. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of The DragonFly Project nor the names of its 16 * contributors may be used to endorse or promote products derived 17 * from this software without specific, prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #ifndef _SYS_SPINLOCK2_H_ 34 #define _SYS_SPINLOCK2_H_ 35 36 #ifndef _KERNEL 37 38 #error "This file should not be included by userland programs." 39 40 #else 41 42 #ifndef _SYS_SYSTM_H_ 43 #include <sys/systm.h> 44 #endif 45 #ifndef _SYS_THREAD2_H_ 46 #include <sys/thread2.h> 47 #endif 48 #ifndef _SYS_GLOBALDATA_H_ 49 #include <sys/globaldata.h> 50 #endif 51 #include <machine/atomic.h> 52 #include <machine/cpufunc.h> 53 54 extern struct spinlock pmap_spin; 55 56 #ifdef SMP 57 58 int spin_trylock_contested(struct spinlock *spin); 59 void spin_lock_contested(struct spinlock *spin); 60 void _spin_pool_lock(void *chan); 61 void _spin_pool_unlock(void *chan); 62 63 #endif 64 65 #ifdef SMP 66 67 /* 68 * Attempt to obtain an exclusive spinlock. Returns FALSE on failure, 69 * TRUE on success. 70 */ 71 static __inline boolean_t 72 spin_trylock(struct spinlock *spin) 73 { 74 globaldata_t gd = mycpu; 75 76 ++gd->gd_curthread->td_critcount; 77 cpu_ccfence(); 78 ++gd->gd_spinlocks_wr; 79 if (atomic_swap_int(&spin->counta, 1)) 80 return (spin_trylock_contested(spin)); 81 #ifdef DEBUG_LOCKS 82 int i; 83 for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) { 84 if (gd->gd_curthread->td_spinlock_stack_id[i] == 0) { 85 gd->gd_curthread->td_spinlock_stack_id[i] = 1; 86 gd->gd_curthread->td_spinlock_stack[i] = spin; 87 gd->gd_curthread->td_spinlock_caller_pc[i] = 88 __builtin_return_address(0); 89 break; 90 } 91 } 92 #endif 93 return (TRUE); 94 } 95 96 #else 97 98 static __inline boolean_t 99 spin_trylock(struct spinlock *spin) 100 { 101 globaldata_t gd = mycpu; 102 103 ++gd->gd_curthread->td_critcount; 104 cpu_ccfence(); 105 ++gd->gd_spinlocks_wr; 106 return (TRUE); 107 } 108 109 #endif 110 111 /* 112 * Return TRUE if the spinlock is held (we can't tell by whom, though) 113 */ 114 static __inline int 115 spin_held(struct spinlock *spin) 116 { 117 return(spin->counta != 0); 118 } 119 120 /* 121 * Obtain an exclusive spinlock and return. 122 */ 123 static __inline void 124 spin_lock_quick(globaldata_t gd, struct spinlock *spin) 125 { 126 ++gd->gd_curthread->td_critcount; 127 cpu_ccfence(); 128 ++gd->gd_spinlocks_wr; 129 #ifdef SMP 130 if (atomic_swap_int(&spin->counta, 1)) 131 spin_lock_contested(spin); 132 #ifdef DEBUG_LOCKS 133 int i; 134 for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) { 135 if (gd->gd_curthread->td_spinlock_stack_id[i] == 0) { 136 gd->gd_curthread->td_spinlock_stack_id[i] = 1; 137 gd->gd_curthread->td_spinlock_stack[i] = spin; 138 gd->gd_curthread->td_spinlock_caller_pc[i] = 139 __builtin_return_address(0); 140 break; 141 } 142 } 143 #endif 144 #endif 145 } 146 147 static __inline void 148 spin_lock(struct spinlock *spin) 149 { 150 spin_lock_quick(mycpu, spin); 151 } 152 153 /* 154 * Release an exclusive spinlock. We can just do this passively, only 155 * ensuring that our spinlock count is left intact until the mutex is 156 * cleared. 157 */ 158 static __inline void 159 spin_unlock_quick(globaldata_t gd, struct spinlock *spin) 160 { 161 #ifdef SMP 162 #ifdef DEBUG_LOCKS 163 int i; 164 for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) { 165 if ((gd->gd_curthread->td_spinlock_stack_id[i] == 1) && 166 (gd->gd_curthread->td_spinlock_stack[i] == spin)) { 167 gd->gd_curthread->td_spinlock_stack_id[i] = 0; 168 gd->gd_curthread->td_spinlock_stack[i] = NULL; 169 gd->gd_curthread->td_spinlock_caller_pc[i] = NULL; 170 break; 171 } 172 } 173 #endif 174 /* 175 * Don't use a locked instruction here. To reduce latency we avoid 176 * reading spin->counta prior to writing to it. 177 */ 178 #ifdef DEBUG_LOCKS 179 KKASSERT(spin->counta != 0); 180 #endif 181 cpu_sfence(); 182 spin->counta = 0; 183 cpu_sfence(); 184 #endif 185 #ifdef DEBUG_LOCKS 186 KKASSERT(gd->gd_spinlocks_wr > 0); 187 #endif 188 --gd->gd_spinlocks_wr; 189 cpu_ccfence(); 190 --gd->gd_curthread->td_critcount; 191 #if 0 192 /* FUTURE */ 193 if (__predict_false(gd->gd_reqflags & RQF_IDLECHECK_MASK)) 194 lwkt_maybe_splz(gd->gd_curthread); 195 #endif 196 } 197 198 static __inline void 199 spin_unlock(struct spinlock *spin) 200 { 201 spin_unlock_quick(mycpu, spin); 202 } 203 204 static __inline void 205 spin_pool_lock(void *chan) 206 { 207 #ifdef SMP 208 _spin_pool_lock(chan); 209 #else 210 spin_lock(NULL); 211 #endif 212 } 213 214 static __inline void 215 spin_pool_unlock(void *chan) 216 { 217 #ifdef SMP 218 _spin_pool_unlock(chan); 219 #else 220 spin_unlock(NULL); 221 #endif 222 } 223 224 static __inline void 225 spin_init(struct spinlock *spin) 226 { 227 spin->counta = 0; 228 spin->countb = 0; 229 } 230 231 static __inline void 232 spin_uninit(struct spinlock *spin) 233 { 234 /* unused */ 235 } 236 237 #endif /* _KERNEL */ 238 #endif /* _SYS_SPINLOCK2_H_ */ 239 240