1/* $NetBSD: atomic.S,v 1.22 2014/05/23 03:17:31 uebayasi Exp $ */ 2 3/*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/param.h> 33#include <machine/asm.h> 34/* 35 * __HAVE_ constants should not be in <machine/types.h> 36 * because we can't use them from assembly. OTOH we 37 * only need __HAVE_ATOMIC64_OPS here, and we don't. 38 */ 39#ifdef _KERNEL 40#define ALIAS(f, t) STRONG_ALIAS(f,t) 41#else 42#define ALIAS(f, t) WEAK_ALIAS(f,t) 43#endif 44 45#ifdef _HARDKERNEL 46#define LOCK(n) .Lpatch ## n: lock 47#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 48#else 49#define LOCK(n) lock 50#define ENDLABEL(a) /* nothing */ 51#endif 52 53 .text 54 55ENTRY(_atomic_add_32) 56 movl 4(%esp), %edx 57 movl 8(%esp), %eax 58 LOCK(1) 59 addl %eax, (%edx) 60 ret 61END(_atomic_add_32) 62 63ENTRY(_atomic_add_32_nv) 64 movl 4(%esp), %edx 65 movl 8(%esp), %eax 66 movl %eax, %ecx 67 LOCK(2) 68 xaddl %eax, (%edx) 69 addl %ecx, %eax 70 ret 71END(_atomic_add_32_nv) 72 73ENTRY(_atomic_and_32) 74 movl 4(%esp), %edx 75 movl 8(%esp), %eax 76 LOCK(3) 77 andl %eax, (%edx) 78 ret 79END(_atomic_and_32) 80 81ENTRY(_atomic_and_32_nv) 82 movl 4(%esp), %edx 83 movl (%edx), %eax 840: 85 movl %eax, %ecx 86 andl 8(%esp), %ecx 87 LOCK(4) 88 cmpxchgl %ecx, (%edx) 89 jnz 1f 90 movl %ecx, %eax 91 ret 921: 93 jmp 0b 94END(_atomic_and_32_nv) 95 96ENTRY(_atomic_dec_32) 97 movl 4(%esp), %edx 98 LOCK(5) 99 decl (%edx) 100 ret 101END(_atomic_dec_32) 102 103ENTRY(_atomic_dec_32_nv) 104 movl 4(%esp), %edx 105 movl $-1, %eax 106 LOCK(6) 107 xaddl %eax, (%edx) 108 decl %eax 109 ret 110END(_atomic_dec_32_nv) 111 112ENTRY(_atomic_inc_32) 113 movl 4(%esp), %edx 114 LOCK(7) 115 incl (%edx) 116 ret 117END(_atomic_inc_32) 118 119ENTRY(_atomic_inc_32_nv) 120 movl 4(%esp), %edx 121 movl $1, %eax 122 LOCK(8) 123 xaddl %eax, (%edx) 124 incl %eax 125 ret 126END(_atomic_inc_32_nv) 127 128ENTRY(_atomic_or_32) 129 movl 4(%esp), %edx 130 movl 8(%esp), %eax 131 LOCK(9) 132 orl %eax, (%edx) 133 ret 134END(_atomic_or_32) 135 136ENTRY(_atomic_or_32_nv) 137 movl 4(%esp), %edx 138 movl (%edx), %eax 1390: 140 movl %eax, %ecx 141 orl 8(%esp), %ecx 142 LOCK(10) 143 cmpxchgl %ecx, (%edx) 144 jnz 1f 145 movl %ecx, %eax 146 ret 1471: 148 jmp 0b 149END(_atomic_or_32_nv) 150 151ENTRY(_atomic_swap_32) 152 movl 4(%esp), %edx 153 movl 8(%esp), %eax 154 xchgl %eax, (%edx) 155 ret 156END(_atomic_swap_32) 157 158ENTRY(_atomic_cas_32) 159 movl 4(%esp), %edx 160 movl 8(%esp), %eax 161 movl 12(%esp), %ecx 162 LOCK(12) 163 cmpxchgl %ecx, (%edx) 164 /* %eax now contains the old value */ 165 ret 166END(_atomic_cas_32) 167 168ENTRY(_atomic_cas_32_ni) 169 movl 4(%esp), %edx 170 movl 8(%esp), %eax 171 movl 12(%esp), %ecx 172 cmpxchgl %ecx, (%edx) 173 /* %eax now contains the old value */ 174 ret 175END(_atomic_cas_32_ni) 176 177ENTRY(_membar_consumer) 178 LOCK(13) 179 addl $0, -4(%esp) 180 ret 181END(_membar_consumer) 182ENDLABEL(membar_consumer_end) 183 184ENTRY(_membar_producer) 185 /* A store is enough */ 186 movl $0, -4(%esp) 187 ret 188END(_membar_producer) 189ENDLABEL(membar_producer_end) 190 191ENTRY(_membar_sync) 192 LOCK(14) 193 addl $0, -4(%esp) 194 ret 195END(_membar_sync) 196ENDLABEL(membar_sync_end) 197 198#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 199ENTRY(_atomic_cas_64) 200#ifdef _HARDKERNEL 201 pushf 202 cli 203#endif /* _HARDKERNEL */ 204 pushl %edi 205 pushl %ebx 206 movl 12(%esp), %edi 207 movl 16(%esp), %eax 208 movl 20(%esp), %edx 209 movl 24(%esp), %ebx 210 movl 28(%esp), %ecx 211 cmpl 0(%edi), %eax 212 jne 2f 213 cmpl 4(%edi), %edx 214 jne 2f 215 movl %ebx, 0(%edi) 216 movl %ecx, 4(%edi) 2171: 218 popl %ebx 219 popl %edi 220#ifdef _HARDKERNEL 221 popf 222#endif /* _HARDKERNEL */ 223 ret 2242: 225 movl 0(%edi), %eax 226 movl 4(%edi), %edx 227 jmp 1b 228END(_atomic_cas_64) 229ENDLABEL(_atomic_cas_64_end) 230 231ENTRY(_atomic_cas_cx8) 232 pushl %edi 233 pushl %ebx 234 movl 12(%esp), %edi 235 movl 16(%esp), %eax 236 movl 20(%esp), %edx 237 movl 24(%esp), %ebx 238 movl 28(%esp), %ecx 239 LOCK(15) 240 cmpxchg8b (%edi) 241 popl %ebx 242 popl %edi 243 ret 244#ifdef _HARDKERNEL 245#ifdef GPROF 246 .space 16, 0x90 247#else 248 .space 32, 0x90 249#endif 250#endif /* _HARDKERNEL */ 251END(_atomic_cas_cx8) 252ENDLABEL(_atomic_cas_cx8_end) 253#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 254 255#ifdef _HARDKERNEL 256ENTRY(sse2_lfence) 257 lfence 258 ret 259END(sse2_lfence) 260ENDLABEL(sse2_lfence_end) 261 262ENTRY(sse2_mfence) 263 mfence 264 ret 265END(sse2_mfence) 266ENDLABEL(sse2_mfence_end) 267 268atomic_lockpatch: 269 .globl atomic_lockpatch 270 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 271 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 272 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 273#endif /* _HARDKERNEL */ 274 275ALIAS(atomic_add_32,_atomic_add_32) 276ALIAS(atomic_add_int,_atomic_add_32) 277ALIAS(atomic_add_long,_atomic_add_32) 278ALIAS(atomic_add_ptr,_atomic_add_32) 279 280ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 281ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 282ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 283ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 284 285ALIAS(atomic_and_32,_atomic_and_32) 286ALIAS(atomic_and_uint,_atomic_and_32) 287ALIAS(atomic_and_ulong,_atomic_and_32) 288ALIAS(atomic_and_ptr,_atomic_and_32) 289 290ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 291ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 292ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 293ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 294 295ALIAS(atomic_dec_32,_atomic_dec_32) 296ALIAS(atomic_dec_uint,_atomic_dec_32) 297ALIAS(atomic_dec_ulong,_atomic_dec_32) 298ALIAS(atomic_dec_ptr,_atomic_dec_32) 299 300ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 301ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 302ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 303ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 304 305ALIAS(atomic_inc_32,_atomic_inc_32) 306ALIAS(atomic_inc_uint,_atomic_inc_32) 307ALIAS(atomic_inc_ulong,_atomic_inc_32) 308ALIAS(atomic_inc_ptr,_atomic_inc_32) 309 310ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 311ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 312ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 313ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 314 315ALIAS(atomic_or_32,_atomic_or_32) 316ALIAS(atomic_or_uint,_atomic_or_32) 317ALIAS(atomic_or_ulong,_atomic_or_32) 318ALIAS(atomic_or_ptr,_atomic_or_32) 319 320ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 321ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 322ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 323ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 324 325ALIAS(atomic_swap_32,_atomic_swap_32) 326ALIAS(atomic_swap_uint,_atomic_swap_32) 327ALIAS(atomic_swap_ulong,_atomic_swap_32) 328ALIAS(atomic_swap_ptr,_atomic_swap_32) 329 330ALIAS(atomic_cas_32,_atomic_cas_32) 331ALIAS(atomic_cas_uint,_atomic_cas_32) 332ALIAS(atomic_cas_ulong,_atomic_cas_32) 333ALIAS(atomic_cas_ptr,_atomic_cas_32) 334 335ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 336ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 337ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 338ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 339 340#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 341ALIAS(atomic_cas_64,_atomic_cas_64) 342ALIAS(atomic_cas_64_ni,_atomic_cas_64) 343ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64) 344#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 345 346ALIAS(membar_consumer,_membar_consumer) 347ALIAS(membar_producer,_membar_producer) 348ALIAS(membar_enter,_membar_consumer) 349ALIAS(membar_exit,_membar_producer) 350ALIAS(membar_sync,_membar_sync) 351 352STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 353STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 354STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 355 356STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 357STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 358STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 359 360STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 361STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 362STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 363 364STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 365STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 366STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 367 368STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 369STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 370STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 371 372STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 373STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 374STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 375 376STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 377STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 378STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 379 380STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 381STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 382STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 383 384STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 385STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 386STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 387 388STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 389STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 390STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 391 392STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 393STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 394STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 395 396STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 397STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 398STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 399 400STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 401STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 402STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 403 404STRONG_ALIAS(_membar_enter,_membar_consumer) 405STRONG_ALIAS(_membar_exit,_membar_producer) 406