1 /* $OpenBSD: atomic.h,v 1.17 2015/01/06 00:38:32 dlg Exp $ */ 2 /* $NetBSD: atomic.h,v 1.1 2003/04/26 18:39:37 fvdl Exp $ */ 3 4 /* 5 * Copyright 2002 (c) Wasabi Systems, Inc. 6 * All rights reserved. 7 * 8 * Written by Frank van der Linden for Wasabi Systems, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed for the NetBSD Project by 21 * Wasabi Systems, Inc. 22 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 23 * or promote products derived from this software without specific prior 24 * written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 #ifndef _MACHINE_ATOMIC_H_ 40 #define _MACHINE_ATOMIC_H_ 41 42 /* 43 * Perform atomic operations on memory. Should be atomic with respect 44 * to interrupts and multiple processors. 45 * 46 * void atomic_setbits_int(volatile u_int *a, u_int mask) { *a |= mask; } 47 * void atomic_clearbits_int(volatile u_int *a, u_int mas) { *a &= ~mask; } 48 */ 49 50 #if defined(_KERNEL) && !defined(_LOCORE) 51 52 #ifdef MULTIPROCESSOR 53 #define LOCK "lock" 54 #else 55 #define LOCK 56 #endif 57 58 static inline unsigned int 59 _atomic_cas_uint(volatile unsigned int *p, unsigned int e, unsigned int n) 60 { 61 __asm volatile(LOCK " cmpxchgl %2, %1" 62 : "=a" (n), "=m" (*p) 63 : "r" (n), "a" (e), "m" (*p)); 64 65 return (n); 66 } 67 #define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n)) 68 69 static inline unsigned long 70 _atomic_cas_ulong(volatile unsigned long *p, unsigned long e, unsigned long n) 71 { 72 __asm volatile(LOCK " cmpxchgq %2, %1" 73 : "=a" (n), "=m" (*p) 74 : "r" (n), "a" (e), "m" (*p)); 75 76 return (n); 77 } 78 #define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n)) 79 80 static inline void * 81 _atomic_cas_ptr(volatile void *p, void *e, void *n) 82 { 83 __asm volatile(LOCK " cmpxchgq %2, %1" 84 : "=a" (n), "=m" (*(unsigned long *)p) 85 : "r" (n), "a" (e), "m" (*(unsigned long *)p)); 86 87 return (n); 88 } 89 #define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n)) 90 91 static inline unsigned int 92 _atomic_swap_uint(volatile unsigned int *p, unsigned int n) 93 { 94 __asm volatile("xchgl %0, %1" 95 : "=a" (n), "=m" (*p) 96 : "0" (n), "m" (*p)); 97 98 return (n); 99 } 100 #define atomic_swap_uint(_p, _n) _atomic_swap_uint((_p), (_n)) 101 #define atomic_swap_32(_p, _n) _atomic_swap_uint((_p), (_n)) 102 103 static inline unsigned long 104 _atomic_swap_ulong(volatile unsigned long *p, unsigned long n) 105 { 106 __asm volatile("xchgq %0, %1" 107 : "=a" (n), "=m" (*p) 108 : "0" (n), "m" (*p)); 109 110 return (n); 111 } 112 #define atomic_swap_ulong(_p, _n) _atomic_swap_ulong((_p), (_n)) 113 114 static inline uint64_t 115 _atomic_swap_64(volatile uint64_t *p, uint64_t n) 116 { 117 __asm volatile("xchgq %0, %1" 118 : "=a" (n), "=m" (*p) 119 : "0" (n), "m" (*p)); 120 121 return (n); 122 } 123 #define atomic_swap_64(_p, _n) _atomic_swap_64((_p), (_n)) 124 125 static inline void * 126 _atomic_swap_ptr(volatile void *p, void *n) 127 { 128 __asm volatile("xchgq %0, %1" 129 : "=a" (n), "=m" (*(unsigned long *)p) 130 : "0" (n), "m" (*(unsigned long *)p)); 131 132 return (n); 133 } 134 #define atomic_swap_ptr(_p, _n) _atomic_swap_ptr((_p), (_n)) 135 136 static inline void 137 _atomic_inc_int(volatile unsigned int *p) 138 { 139 __asm volatile(LOCK " incl %0" 140 : "+m" (*p)); 141 } 142 #define atomic_inc_int(_p) _atomic_inc_int(_p) 143 144 static inline void 145 _atomic_inc_long(volatile unsigned long *p) 146 { 147 __asm volatile(LOCK " incq %0" 148 : "+m" (*p)); 149 } 150 #define atomic_inc_long(_p) _atomic_inc_long(_p) 151 152 static inline void 153 _atomic_dec_int(volatile unsigned int *p) 154 { 155 __asm volatile(LOCK " decl %0" 156 : "+m" (*p)); 157 } 158 #define atomic_dec_int(_p) _atomic_dec_int(_p) 159 160 static inline void 161 _atomic_dec_long(volatile unsigned long *p) 162 { 163 __asm volatile(LOCK " decq %0" 164 : "+m" (*p)); 165 } 166 #define atomic_dec_long(_p) _atomic_dec_long(_p) 167 168 static inline void 169 _atomic_add_int(volatile unsigned int *p, unsigned int v) 170 { 171 __asm volatile(LOCK " addl %1,%0" 172 : "+m" (*p) 173 : "a" (v)); 174 } 175 #define atomic_add_int(_p, _v) _atomic_add_int(_p, _v) 176 177 static inline void 178 _atomic_add_long(volatile unsigned long *p, unsigned long v) 179 { 180 __asm volatile(LOCK " addq %1,%0" 181 : "+m" (*p) 182 : "a" (v)); 183 } 184 #define atomic_add_long(_p, _v) _atomic_add_long(_p, _v) 185 186 static inline void 187 _atomic_sub_int(volatile unsigned int *p, unsigned int v) 188 { 189 __asm volatile(LOCK " subl %1,%0" 190 : "+m" (*p) 191 : "a" (v)); 192 } 193 #define atomic_sub_int(_p, _v) _atomic_sub_int(_p, _v) 194 195 static inline void 196 _atomic_sub_long(volatile unsigned long *p, unsigned long v) 197 { 198 __asm volatile(LOCK " subq %1,%0" 199 : "+m" (*p) 200 : "a" (v)); 201 } 202 #define atomic_sub_long(_p, _v) _atomic_sub_long(_p, _v) 203 204 205 static inline unsigned long 206 _atomic_add_int_nv(volatile unsigned int *p, unsigned int v) 207 { 208 unsigned int rv = v; 209 210 __asm volatile(LOCK " xaddl %0,%1" 211 : "+a" (rv), "+m" (*p)); 212 213 return (rv + v); 214 } 215 #define atomic_add_int_nv(_p, _v) _atomic_add_int_nv(_p, _v) 216 217 static inline unsigned long 218 _atomic_add_long_nv(volatile unsigned long *p, unsigned long v) 219 { 220 unsigned long rv = v; 221 222 __asm volatile(LOCK " xaddq %0,%1" 223 : "+a" (rv), "+m" (*p)); 224 225 return (rv + v); 226 } 227 #define atomic_add_long_nv(_p, _v) _atomic_add_long_nv(_p, _v) 228 229 static inline unsigned long 230 _atomic_sub_int_nv(volatile unsigned int *p, unsigned int v) 231 { 232 unsigned int rv = 0 - v; 233 234 __asm volatile(LOCK " xaddl %0,%1" 235 : "+a" (rv), "+m" (*p)); 236 237 return (rv - v); 238 } 239 #define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv(_p, _v) 240 241 static inline unsigned long 242 _atomic_sub_long_nv(volatile unsigned long *p, unsigned long v) 243 { 244 unsigned long rv = 0 - v; 245 246 __asm volatile(LOCK " xaddq %0,%1" 247 : "+a" (rv), "+m" (*p)); 248 249 return (rv - v); 250 } 251 #define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv(_p, _v) 252 253 /* 254 * The AMD64 architecture is rather strongly ordered. When accessing 255 * normal write-back cachable memory, only reads may be reordered with 256 * older writes to different locations. There are a few instructions 257 * (clfush, non-temporal move instructions) that obey weaker ordering 258 * rules, but those instructions will only be used in (inline) 259 * assembly code where we can add the necessary fence instructions 260 * ourselves. 261 */ 262 263 #define __membar(_f) do { __asm __volatile(_f ::: "memory"); } while (0) 264 265 #ifdef MULTIPROCESSOR 266 #define membar_enter() __membar("mfence") 267 #define membar_exit() __membar("") 268 #define membar_producer() __membar("") 269 #define membar_consumer() __membar("") 270 #define membar_sync() __membar("mfence") 271 #else 272 #define membar_enter() __membar("") 273 #define membar_exit() __membar("") 274 #define membar_producer() __membar("") 275 #define membar_consumer() __membar("") 276 #define membar_sync() __membar("") 277 #endif 278 279 /* virtio needs MP membars even on SP kernels */ 280 #define virtio_membar_producer() __membar("") 281 #define virtio_membar_consumer() __membar("") 282 #define virtio_membar_sync() __membar("mfence") 283 284 static __inline void 285 x86_atomic_setbits_u32(volatile u_int32_t *ptr, u_int32_t bits) 286 { 287 __asm volatile(LOCK " orl %1,%0" : "=m" (*ptr) : "ir" (bits)); 288 } 289 290 static __inline void 291 x86_atomic_clearbits_u32(volatile u_int32_t *ptr, u_int32_t bits) 292 { 293 __asm volatile(LOCK " andl %1,%0" : "=m" (*ptr) : "ir" (~bits)); 294 } 295 296 /* 297 * XXX XXX XXX 298 * theoretically 64bit cannot be used as 299 * an "i" and thus if we ever try to give 300 * these anything from the high dword there 301 * is an asm error pending 302 */ 303 static __inline void 304 x86_atomic_setbits_u64(volatile u_int64_t *ptr, u_int64_t bits) 305 { 306 __asm volatile(LOCK " orq %1,%0" : "=m" (*ptr) : "ir" (bits)); 307 } 308 309 static __inline void 310 x86_atomic_clearbits_u64(volatile u_int64_t *ptr, u_int64_t bits) 311 { 312 __asm volatile(LOCK " andq %1,%0" : "=m" (*ptr) : "ir" (~bits)); 313 } 314 315 #define x86_atomic_testset_ul x86_atomic_testset_u64 316 #define x86_atomic_setbits_ul x86_atomic_setbits_u64 317 #define x86_atomic_clearbits_ul x86_atomic_clearbits_u64 318 319 #define atomic_setbits_int x86_atomic_setbits_u32 320 #define atomic_clearbits_int x86_atomic_clearbits_u32 321 322 #undef LOCK 323 324 #endif /* defined(_KERNEL) && !defined(_LOCORE) */ 325 #endif /* _MACHINE_ATOMIC_H_ */ 326