1 /*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _MACHINE_ATOMIC_H_ 30 #define _MACHINE_ATOMIC_H_ 31 32 #include <sys/atomic_common.h> 33 34 #define isb() __asm __volatile("isb" : : : "memory") 35 36 /* 37 * Options for DMB and DSB: 38 * oshld Outer Shareable, load 39 * oshst Outer Shareable, store 40 * osh Outer Shareable, all 41 * nshld Non-shareable, load 42 * nshst Non-shareable, store 43 * nsh Non-shareable, all 44 * ishld Inner Shareable, load 45 * ishst Inner Shareable, store 46 * ish Inner Shareable, all 47 * ld Full system, load 48 * st Full system, store 49 * sy Full system, all 50 */ 51 #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") 52 #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") 53 54 #define mb() dmb(sy) /* Full system memory barrier all */ 55 #define wmb() dmb(st) /* Full system memory barrier store */ 56 #define rmb() dmb(ld) /* Full system memory barrier load */ 57 58 #define ATOMIC_OP(op, asm_op, bar, a, l) \ 59 static __inline void \ 60 atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val) \ 61 { \ 62 uint32_t tmp; \ 63 int res; \ 64 \ 65 __asm __volatile( \ 66 "1: ld"#a"xr %w0, [%2] \n" \ 67 " "#asm_op" %w0, %w0, %w3 \n" \ 68 " st"#l"xr %w1, %w0, [%2] \n" \ 69 " cbnz %w1, 1b \n" \ 70 : "=&r"(tmp), "=&r"(res) \ 71 : "r" (p), "r" (val) \ 72 : "memory" \ 73 ); \ 74 } \ 75 \ 76 static __inline void \ 77 atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val) \ 78 { \ 79 uint64_t tmp; \ 80 int res; \ 81 \ 82 __asm __volatile( \ 83 "1: ld"#a"xr %0, [%2] \n" \ 84 " "#asm_op" %0, %0, %3 \n" \ 85 " st"#l"xr %w1, %0, [%2] \n" \ 86 " cbnz %w1, 1b \n" \ 87 : "=&r"(tmp), "=&r"(res) \ 88 : "r" (p), "r" (val) \ 89 : "memory" \ 90 ); \ 91 } 92 93 #define ATOMIC(op, asm_op) \ 94 ATOMIC_OP(op, asm_op, , , ) \ 95 ATOMIC_OP(op, asm_op, acq_, a, ) \ 96 ATOMIC_OP(op, asm_op, rel_, , l) \ 97 98 ATOMIC(add, add) 99 ATOMIC(clear, bic) 100 ATOMIC(set, orr) 101 ATOMIC(subtract, sub) 102 103 #define ATOMIC_FCMPSET(bar, a, l) \ 104 static __inline int \ 105 atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval, \ 106 uint32_t newval) \ 107 { \ 108 uint32_t tmp; \ 109 uint32_t _cmpval = *cmpval; \ 110 int res; \ 111 \ 112 __asm __volatile( \ 113 "1: mov %w1, #1 \n" \ 114 " ld"#a"xr %w0, [%2] \n" \ 115 " cmp %w0, %w3 \n" \ 116 " b.ne 2f \n" \ 117 " st"#l"xr %w1, %w4, [%2] \n" \ 118 "2:" \ 119 : "=&r"(tmp), "=&r"(res) \ 120 : "r" (p), "r" (_cmpval), "r" (newval) \ 121 : "cc", "memory" \ 122 ); \ 123 *cmpval = tmp; \ 124 \ 125 return (!res); \ 126 } \ 127 \ 128 static __inline int \ 129 atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval, \ 130 uint64_t newval) \ 131 { \ 132 uint64_t tmp; \ 133 uint64_t _cmpval = *cmpval; \ 134 int res; \ 135 \ 136 __asm __volatile( \ 137 "1: mov %w1, #1 \n" \ 138 " ld"#a"xr %0, [%2] \n" \ 139 " cmp %0, %3 \n" \ 140 " b.ne 2f \n" \ 141 " st"#l"xr %w1, %4, [%2] \n" \ 142 "2:" \ 143 : "=&r"(tmp), "=&r"(res) \ 144 : "r" (p), "r" (_cmpval), "r" (newval) \ 145 : "cc", "memory" \ 146 ); \ 147 *cmpval = tmp; \ 148 \ 149 return (!res); \ 150 } 151 152 ATOMIC_FCMPSET( , , ) 153 ATOMIC_FCMPSET(acq_, a, ) 154 ATOMIC_FCMPSET(rel_, ,l) 155 156 #undef ATOMIC_FCMPSET 157 158 #define ATOMIC_CMPSET(bar, a, l) \ 159 static __inline int \ 160 atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \ 161 uint32_t newval) \ 162 { \ 163 uint32_t tmp; \ 164 int res; \ 165 \ 166 __asm __volatile( \ 167 "1: mov %w1, #1 \n" \ 168 " ld"#a"xr %w0, [%2] \n" \ 169 " cmp %w0, %w3 \n" \ 170 " b.ne 2f \n" \ 171 " st"#l"xr %w1, %w4, [%2] \n" \ 172 " cbnz %w1, 1b \n" \ 173 "2:" \ 174 : "=&r"(tmp), "=&r"(res) \ 175 : "r" (p), "r" (cmpval), "r" (newval) \ 176 : "cc", "memory" \ 177 ); \ 178 \ 179 return (!res); \ 180 } \ 181 \ 182 static __inline int \ 183 atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval, \ 184 uint64_t newval) \ 185 { \ 186 uint64_t tmp; \ 187 int res; \ 188 \ 189 __asm __volatile( \ 190 "1: mov %w1, #1 \n" \ 191 " ld"#a"xr %0, [%2] \n" \ 192 " cmp %0, %3 \n" \ 193 " b.ne 2f \n" \ 194 " st"#l"xr %w1, %4, [%2] \n" \ 195 " cbnz %w1, 1b \n" \ 196 "2:" \ 197 : "=&r"(tmp), "=&r"(res) \ 198 : "r" (p), "r" (cmpval), "r" (newval) \ 199 : "cc", "memory" \ 200 ); \ 201 \ 202 return (!res); \ 203 } 204 205 ATOMIC_CMPSET( , , ) 206 ATOMIC_CMPSET(acq_, a, ) 207 ATOMIC_CMPSET(rel_, ,l) 208 209 static __inline uint32_t 210 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 211 { 212 uint32_t tmp, ret; 213 int res; 214 215 __asm __volatile( 216 "1: ldxr %w2, [%3] \n" 217 " add %w0, %w2, %w4 \n" 218 " stxr %w1, %w0, [%3] \n" 219 " cbnz %w1, 1b \n" 220 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 221 : "r" (p), "r" (val) 222 : "memory" 223 ); 224 225 return (ret); 226 } 227 228 static __inline uint64_t 229 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 230 { 231 uint64_t tmp, ret; 232 int res; 233 234 __asm __volatile( 235 "1: ldxr %2, [%3] \n" 236 " add %0, %2, %4 \n" 237 " stxr %w1, %0, [%3] \n" 238 " cbnz %w1, 1b \n" 239 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 240 : "r" (p), "r" (val) 241 : "memory" 242 ); 243 244 return (ret); 245 } 246 247 static __inline uint32_t 248 atomic_readandclear_32(volatile uint32_t *p) 249 { 250 uint32_t ret; 251 int res; 252 253 __asm __volatile( 254 "1: ldxr %w1, [%2] \n" 255 " stxr %w0, wzr, [%2] \n" 256 " cbnz %w0, 1b \n" 257 : "=&r"(res), "=&r"(ret) 258 : "r" (p) 259 : "memory" 260 ); 261 262 return (ret); 263 } 264 265 static __inline uint64_t 266 atomic_readandclear_64(volatile uint64_t *p) 267 { 268 uint64_t ret; 269 int res; 270 271 __asm __volatile( 272 "1: ldxr %1, [%2] \n" 273 " stxr %w0, xzr, [%2] \n" 274 " cbnz %w0, 1b \n" 275 : "=&r"(res), "=&r"(ret) 276 : "r" (p) 277 : "memory" 278 ); 279 280 return (ret); 281 } 282 283 static __inline uint32_t 284 atomic_swap_32(volatile uint32_t *p, uint32_t val) 285 { 286 uint32_t ret; 287 int res; 288 289 __asm __volatile( 290 "1: ldxr %w0, [%2] \n" 291 " stxr %w1, %w3, [%2] \n" 292 " cbnz %w1, 1b \n" 293 : "=&r"(ret), "=&r"(res) 294 : "r" (p), "r" (val) 295 : "memory" 296 ); 297 298 return (ret); 299 } 300 301 static __inline uint64_t 302 atomic_swap_64(volatile uint64_t *p, uint64_t val) 303 { 304 uint64_t ret; 305 int res; 306 307 __asm __volatile( 308 "1: ldxr %0, [%2] \n" 309 " stxr %w1, %3, [%2] \n" 310 " cbnz %w1, 1b \n" 311 : "=&r"(ret), "=&r"(res) 312 : "r" (p), "r" (val) 313 : "memory" 314 ); 315 316 return (ret); 317 } 318 319 static __inline uint32_t 320 atomic_load_acq_32(volatile uint32_t *p) 321 { 322 uint32_t ret; 323 324 __asm __volatile( 325 "ldar %w0, [%1] \n" 326 : "=&r" (ret) 327 : "r" (p) 328 : "memory"); 329 330 return (ret); 331 } 332 333 static __inline uint64_t 334 atomic_load_acq_64(volatile uint64_t *p) 335 { 336 uint64_t ret; 337 338 __asm __volatile( 339 "ldar %0, [%1] \n" 340 : "=&r" (ret) 341 : "r" (p) 342 : "memory"); 343 344 return (ret); 345 } 346 347 static __inline void 348 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 349 { 350 351 __asm __volatile( 352 "stlr %w0, [%1] \n" 353 : 354 : "r" (val), "r" (p) 355 : "memory"); 356 } 357 358 static __inline void 359 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 360 { 361 362 __asm __volatile( 363 "stlr %0, [%1] \n" 364 : 365 : "r" (val), "r" (p) 366 : "memory"); 367 } 368 369 370 #define atomic_add_int atomic_add_32 371 #define atomic_fcmpset_int atomic_fcmpset_32 372 #define atomic_clear_int atomic_clear_32 373 #define atomic_cmpset_int atomic_cmpset_32 374 #define atomic_fetchadd_int atomic_fetchadd_32 375 #define atomic_readandclear_int atomic_readandclear_32 376 #define atomic_set_int atomic_set_32 377 #define atomic_swap_int atomic_swap_32 378 #define atomic_subtract_int atomic_subtract_32 379 380 #define atomic_add_acq_int atomic_add_acq_32 381 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 382 #define atomic_clear_acq_int atomic_clear_acq_32 383 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 384 #define atomic_load_acq_int atomic_load_acq_32 385 #define atomic_set_acq_int atomic_set_acq_32 386 #define atomic_subtract_acq_int atomic_subtract_acq_32 387 388 #define atomic_add_rel_int atomic_add_rel_32 389 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 390 #define atomic_clear_rel_int atomic_clear_rel_32 391 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 392 #define atomic_set_rel_int atomic_set_rel_32 393 #define atomic_subtract_rel_int atomic_subtract_rel_32 394 #define atomic_store_rel_int atomic_store_rel_32 395 396 #define atomic_add_long atomic_add_64 397 #define atomic_fcmpset_long atomic_fcmpset_64 398 #define atomic_clear_long atomic_clear_64 399 #define atomic_cmpset_long atomic_cmpset_64 400 #define atomic_fetchadd_long atomic_fetchadd_64 401 #define atomic_readandclear_long atomic_readandclear_64 402 #define atomic_set_long atomic_set_64 403 #define atomic_swap_long atomic_swap_64 404 #define atomic_subtract_long atomic_subtract_64 405 406 #define atomic_add_ptr atomic_add_64 407 #define atomic_fcmpset_ptr atomic_fcmpset_64 408 #define atomic_clear_ptr atomic_clear_64 409 #define atomic_cmpset_ptr atomic_cmpset_64 410 #define atomic_fetchadd_ptr atomic_fetchadd_64 411 #define atomic_readandclear_ptr atomic_readandclear_64 412 #define atomic_set_ptr atomic_set_64 413 #define atomic_swap_ptr atomic_swap_64 414 #define atomic_subtract_ptr atomic_subtract_64 415 416 #define atomic_add_acq_long atomic_add_acq_64 417 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 418 #define atomic_clear_acq_long atomic_clear_acq_64 419 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 420 #define atomic_load_acq_long atomic_load_acq_64 421 #define atomic_set_acq_long atomic_set_acq_64 422 #define atomic_subtract_acq_long atomic_subtract_acq_64 423 424 #define atomic_add_acq_ptr atomic_add_acq_64 425 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 426 #define atomic_clear_acq_ptr atomic_clear_acq_64 427 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 428 #define atomic_load_acq_ptr atomic_load_acq_64 429 #define atomic_set_acq_ptr atomic_set_acq_64 430 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 431 432 #define atomic_add_rel_long atomic_add_rel_64 433 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 434 #define atomic_clear_rel_long atomic_clear_rel_64 435 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 436 #define atomic_set_rel_long atomic_set_rel_64 437 #define atomic_subtract_rel_long atomic_subtract_rel_64 438 #define atomic_store_rel_long atomic_store_rel_64 439 440 #define atomic_add_rel_ptr atomic_add_rel_64 441 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 442 #define atomic_clear_rel_ptr atomic_clear_rel_64 443 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 444 #define atomic_set_rel_ptr atomic_set_rel_64 445 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 446 #define atomic_store_rel_ptr atomic_store_rel_64 447 448 static __inline void 449 atomic_thread_fence_acq(void) 450 { 451 452 dmb(ld); 453 } 454 455 static __inline void 456 atomic_thread_fence_rel(void) 457 { 458 459 dmb(sy); 460 } 461 462 static __inline void 463 atomic_thread_fence_acq_rel(void) 464 { 465 466 dmb(sy); 467 } 468 469 static __inline void 470 atomic_thread_fence_seq_cst(void) 471 { 472 473 dmb(sy); 474 } 475 476 #endif /* _MACHINE_ATOMIC_H_ */ 477 478