1 /*- 2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD$ 35 */ 36 37 #ifndef _MACHINE_ATOMIC_H_ 38 #define _MACHINE_ATOMIC_H_ 39 40 #include <sys/atomic_common.h> 41 42 #define fence() __asm __volatile("fence" ::: "memory"); 43 #define mb() fence() 44 #define rmb() fence() 45 #define wmb() fence() 46 47 static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); 48 static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); 49 static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); 50 static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *, 51 uint16_t); 52 53 #define ATOMIC_ACQ_REL(NAME, WIDTH) \ 54 static __inline void \ 55 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 56 { \ 57 atomic_##NAME##_##WIDTH(p, v); \ 58 fence(); \ 59 } \ 60 \ 61 static __inline void \ 62 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 63 { \ 64 fence(); \ 65 atomic_##NAME##_##WIDTH(p, v); \ 66 } 67 68 #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \ 69 static __inline int \ 70 atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 71 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 72 { \ 73 int retval; \ 74 \ 75 retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \ 76 fence(); \ 77 return (retval); \ 78 } \ 79 \ 80 static __inline int \ 81 atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 82 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 83 { \ 84 fence(); \ 85 return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \ 86 } 87 88 #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \ 89 static __inline int \ 90 atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 91 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 92 { \ 93 int retval; \ 94 \ 95 retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \ 96 fence(); \ 97 return (retval); \ 98 } \ 99 \ 100 static __inline int \ 101 atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 102 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 103 { \ 104 fence(); \ 105 return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \ 106 } 107 108 ATOMIC_CMPSET_ACQ_REL(8); 109 ATOMIC_FCMPSET_ACQ_REL(8); 110 ATOMIC_CMPSET_ACQ_REL(16); 111 ATOMIC_FCMPSET_ACQ_REL(16); 112 113 #define atomic_cmpset_char atomic_cmpset_8 114 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 115 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 116 #define atomic_fcmpset_char atomic_fcmpset_8 117 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 118 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 119 120 121 #define atomic_cmpset_short atomic_cmpset_16 122 #define atomic_cmpset_acq_short atomic_cmpset_acq_16 123 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 124 #define atomic_fcmpset_short atomic_fcmpset_16 125 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 126 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 127 128 static __inline void 129 atomic_add_32(volatile uint32_t *p, uint32_t val) 130 { 131 132 __asm __volatile("amoadd.w zero, %1, %0" 133 : "+A" (*p) 134 : "r" (val) 135 : "memory"); 136 } 137 138 static __inline void 139 atomic_subtract_32(volatile uint32_t *p, uint32_t val) 140 { 141 142 __asm __volatile("amoadd.w zero, %1, %0" 143 : "+A" (*p) 144 : "r" (-val) 145 : "memory"); 146 } 147 148 static __inline void 149 atomic_set_32(volatile uint32_t *p, uint32_t val) 150 { 151 152 __asm __volatile("amoor.w zero, %1, %0" 153 : "+A" (*p) 154 : "r" (val) 155 : "memory"); 156 } 157 158 static __inline void 159 atomic_clear_32(volatile uint32_t *p, uint32_t val) 160 { 161 162 __asm __volatile("amoand.w zero, %1, %0" 163 : "+A" (*p) 164 : "r" (~val) 165 : "memory"); 166 } 167 168 static __inline int 169 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 170 { 171 uint32_t tmp; 172 int res; 173 174 res = 0; 175 176 __asm __volatile( 177 "0:" 178 "li %1, 1\n" /* Preset to fail */ 179 "lr.w %0, %2\n" 180 "bne %0, %z3, 1f\n" 181 "sc.w %1, %z4, %2\n" 182 "bnez %1, 0b\n" 183 "1:" 184 : "=&r" (tmp), "=&r" (res), "+A" (*p) 185 : "rJ" ((long)(int32_t)cmpval), "rJ" (newval) 186 : "memory"); 187 188 return (!res); 189 } 190 191 static __inline int 192 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 193 { 194 uint32_t tmp; 195 int res; 196 197 res = 0; 198 199 __asm __volatile( 200 "0:" 201 "li %1, 1\n" /* Preset to fail */ 202 "lr.w %0, %2\n" /* Load old value */ 203 "bne %0, %z4, 1f\n" /* Compare */ 204 "sc.w %1, %z5, %2\n" /* Try to store new value */ 205 "j 2f\n" 206 "1:" 207 "sw %0, %3\n" /* Save old value */ 208 "2:" 209 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 210 : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval) 211 : "memory"); 212 213 return (!res); 214 } 215 216 static __inline uint32_t 217 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 218 { 219 uint32_t ret; 220 221 __asm __volatile("amoadd.w %0, %2, %1" 222 : "=&r" (ret), "+A" (*p) 223 : "r" (val) 224 : "memory"); 225 226 return (ret); 227 } 228 229 static __inline uint32_t 230 atomic_readandclear_32(volatile uint32_t *p) 231 { 232 uint32_t ret; 233 uint32_t val; 234 235 val = 0; 236 237 __asm __volatile("amoswap.w %0, %2, %1" 238 : "=&r"(ret), "+A" (*p) 239 : "r" (val) 240 : "memory"); 241 242 return (ret); 243 } 244 245 #define atomic_add_int atomic_add_32 246 #define atomic_clear_int atomic_clear_32 247 #define atomic_cmpset_int atomic_cmpset_32 248 #define atomic_fcmpset_int atomic_fcmpset_32 249 #define atomic_fetchadd_int atomic_fetchadd_32 250 #define atomic_readandclear_int atomic_readandclear_32 251 #define atomic_set_int atomic_set_32 252 #define atomic_subtract_int atomic_subtract_32 253 254 ATOMIC_ACQ_REL(set, 32) 255 ATOMIC_ACQ_REL(clear, 32) 256 ATOMIC_ACQ_REL(add, 32) 257 ATOMIC_ACQ_REL(subtract, 32) 258 259 ATOMIC_CMPSET_ACQ_REL(32); 260 ATOMIC_FCMPSET_ACQ_REL(32); 261 262 static __inline uint32_t 263 atomic_load_acq_32(volatile uint32_t *p) 264 { 265 uint32_t ret; 266 267 ret = *p; 268 269 fence(); 270 271 return (ret); 272 } 273 274 static __inline void 275 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 276 { 277 278 fence(); 279 280 *p = val; 281 } 282 283 #define atomic_add_acq_int atomic_add_acq_32 284 #define atomic_clear_acq_int atomic_clear_acq_32 285 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 286 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 287 #define atomic_load_acq_int atomic_load_acq_32 288 #define atomic_set_acq_int atomic_set_acq_32 289 #define atomic_subtract_acq_int atomic_subtract_acq_32 290 291 #define atomic_add_rel_int atomic_add_rel_32 292 #define atomic_clear_rel_int atomic_add_rel_32 293 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 294 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 295 #define atomic_set_rel_int atomic_set_rel_32 296 #define atomic_subtract_rel_int atomic_subtract_rel_32 297 #define atomic_store_rel_int atomic_store_rel_32 298 299 static __inline void 300 atomic_add_64(volatile uint64_t *p, uint64_t val) 301 { 302 303 __asm __volatile("amoadd.d zero, %1, %0" 304 : "+A" (*p) 305 : "r" (val) 306 : "memory"); 307 } 308 309 static __inline void 310 atomic_subtract_64(volatile uint64_t *p, uint64_t val) 311 { 312 313 __asm __volatile("amoadd.d zero, %1, %0" 314 : "+A" (*p) 315 : "r" (-val) 316 : "memory"); 317 } 318 319 static __inline void 320 atomic_set_64(volatile uint64_t *p, uint64_t val) 321 { 322 323 __asm __volatile("amoor.d zero, %1, %0" 324 : "+A" (*p) 325 : "r" (val) 326 : "memory"); 327 } 328 329 static __inline void 330 atomic_clear_64(volatile uint64_t *p, uint64_t val) 331 { 332 333 __asm __volatile("amoand.d zero, %1, %0" 334 : "+A" (*p) 335 : "r" (~val) 336 : "memory"); 337 } 338 339 static __inline int 340 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 341 { 342 uint64_t tmp; 343 int res; 344 345 res = 0; 346 347 __asm __volatile( 348 "0:" 349 "li %1, 1\n" /* Preset to fail */ 350 "lr.d %0, %2\n" 351 "bne %0, %z3, 1f\n" 352 "sc.d %1, %z4, %2\n" 353 "bnez %1, 0b\n" 354 "1:" 355 : "=&r" (tmp), "=&r" (res), "+A" (*p) 356 : "rJ" (cmpval), "rJ" (newval) 357 : "memory"); 358 359 return (!res); 360 } 361 362 static __inline int 363 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 364 { 365 uint64_t tmp; 366 int res; 367 368 res = 0; 369 370 __asm __volatile( 371 "0:" 372 "li %1, 1\n" /* Preset to fail */ 373 "lr.d %0, %2\n" /* Load old value */ 374 "bne %0, %z4, 1f\n" /* Compare */ 375 "sc.d %1, %z5, %2\n" /* Try to store new value */ 376 "j 2f\n" 377 "1:" 378 "sd %0, %3\n" /* Save old value */ 379 "2:" 380 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 381 : "rJ" (*cmpval), "rJ" (newval) 382 : "memory"); 383 384 return (!res); 385 } 386 387 static __inline uint64_t 388 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 389 { 390 uint64_t ret; 391 392 __asm __volatile("amoadd.d %0, %2, %1" 393 : "=&r" (ret), "+A" (*p) 394 : "r" (val) 395 : "memory"); 396 397 return (ret); 398 } 399 400 static __inline uint64_t 401 atomic_readandclear_64(volatile uint64_t *p) 402 { 403 uint64_t ret; 404 uint64_t val; 405 406 val = 0; 407 408 __asm __volatile("amoswap.d %0, %2, %1" 409 : "=&r"(ret), "+A" (*p) 410 : "r" (val) 411 : "memory"); 412 413 return (ret); 414 } 415 416 static __inline uint32_t 417 atomic_swap_32(volatile uint32_t *p, uint32_t val) 418 { 419 uint32_t old; 420 421 __asm __volatile("amoswap.w %0, %2, %1" 422 : "=&r"(old), "+A" (*p) 423 : "r" (val) 424 : "memory"); 425 426 return (old); 427 } 428 429 static __inline uint64_t 430 atomic_swap_64(volatile uint64_t *p, uint64_t val) 431 { 432 uint64_t old; 433 434 __asm __volatile("amoswap.d %0, %2, %1" 435 : "=&r"(old), "+A" (*p) 436 : "r" (val) 437 : "memory"); 438 439 return (old); 440 } 441 442 #define atomic_swap_int atomic_swap_32 443 444 #define atomic_add_long atomic_add_64 445 #define atomic_clear_long atomic_clear_64 446 #define atomic_cmpset_long atomic_cmpset_64 447 #define atomic_fcmpset_long atomic_fcmpset_64 448 #define atomic_fetchadd_long atomic_fetchadd_64 449 #define atomic_readandclear_long atomic_readandclear_64 450 #define atomic_set_long atomic_set_64 451 #define atomic_subtract_long atomic_subtract_64 452 #define atomic_swap_long atomic_swap_64 453 454 #define atomic_add_ptr atomic_add_64 455 #define atomic_clear_ptr atomic_clear_64 456 #define atomic_cmpset_ptr atomic_cmpset_64 457 #define atomic_fcmpset_ptr atomic_fcmpset_64 458 #define atomic_fetchadd_ptr atomic_fetchadd_64 459 #define atomic_readandclear_ptr atomic_readandclear_64 460 #define atomic_set_ptr atomic_set_64 461 #define atomic_subtract_ptr atomic_subtract_64 462 #define atomic_swap_ptr atomic_swap_64 463 464 ATOMIC_ACQ_REL(set, 64) 465 ATOMIC_ACQ_REL(clear, 64) 466 ATOMIC_ACQ_REL(add, 64) 467 ATOMIC_ACQ_REL(subtract, 64) 468 469 ATOMIC_CMPSET_ACQ_REL(64); 470 ATOMIC_FCMPSET_ACQ_REL(64); 471 472 static __inline uint64_t 473 atomic_load_acq_64(volatile uint64_t *p) 474 { 475 uint64_t ret; 476 477 ret = *p; 478 479 fence(); 480 481 return (ret); 482 } 483 484 static __inline void 485 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 486 { 487 488 fence(); 489 490 *p = val; 491 } 492 493 #define atomic_add_acq_long atomic_add_acq_64 494 #define atomic_clear_acq_long atomic_add_acq_64 495 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 496 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 497 #define atomic_load_acq_long atomic_load_acq_64 498 #define atomic_set_acq_long atomic_set_acq_64 499 #define atomic_subtract_acq_long atomic_subtract_acq_64 500 501 #define atomic_add_acq_ptr atomic_add_acq_64 502 #define atomic_clear_acq_ptr atomic_add_acq_64 503 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 504 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 505 #define atomic_load_acq_ptr atomic_load_acq_64 506 #define atomic_set_acq_ptr atomic_set_acq_64 507 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 508 509 #undef ATOMIC_ACQ_REL 510 511 static __inline void 512 atomic_thread_fence_acq(void) 513 { 514 515 fence(); 516 } 517 518 static __inline void 519 atomic_thread_fence_rel(void) 520 { 521 522 fence(); 523 } 524 525 static __inline void 526 atomic_thread_fence_acq_rel(void) 527 { 528 529 fence(); 530 } 531 532 static __inline void 533 atomic_thread_fence_seq_cst(void) 534 { 535 536 fence(); 537 } 538 539 #define atomic_add_rel_long atomic_add_rel_64 540 #define atomic_clear_rel_long atomic_clear_rel_64 541 542 #define atomic_add_rel_long atomic_add_rel_64 543 #define atomic_clear_rel_long atomic_clear_rel_64 544 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 545 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 546 #define atomic_set_rel_long atomic_set_rel_64 547 #define atomic_subtract_rel_long atomic_subtract_rel_64 548 #define atomic_store_rel_long atomic_store_rel_64 549 550 #define atomic_add_rel_ptr atomic_add_rel_64 551 #define atomic_clear_rel_ptr atomic_clear_rel_64 552 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 553 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 554 #define atomic_set_rel_ptr atomic_set_rel_64 555 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 556 #define atomic_store_rel_ptr atomic_store_rel_64 557 558 #include <sys/_atomic_subword.h> 559 560 #endif /* _MACHINE_ATOMIC_H_ */ 561