1 /* 2 * Distributed under the Boost Software License, Version 1.0. 3 * (See accompanying file LICENSE_1_0.txt or copy at 4 * http://www.boost.org/LICENSE_1_0.txt) 5 * 6 * Copyright (c) 2009 Helge Bahmann 7 * Copyright (c) 2013 Tim Blechmann 8 * Copyright (c) 2014 Andrey Semashev 9 */ 10 /*! 11 * \file atomic/detail/core_arch_ops_gcc_ppc.hpp 12 * 13 * This header contains implementation of the \c core_arch_operations template. 14 */ 15 16 #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_ 17 #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_ 18 19 #include <cstddef> 20 #include <boost/memory_order.hpp> 21 #include <boost/atomic/detail/config.hpp> 22 #include <boost/atomic/detail/storage_traits.hpp> 23 #include <boost/atomic/detail/core_arch_operations_fwd.hpp> 24 #include <boost/atomic/detail/ops_gcc_ppc_common.hpp> 25 #include <boost/atomic/detail/capabilities.hpp> 26 #include <boost/atomic/detail/header.hpp> 27 28 #ifdef BOOST_HAS_PRAGMA_ONCE 29 #pragma once 30 #endif 31 32 namespace boost { 33 namespace atomics { 34 namespace detail { 35 36 // The implementation below uses information from this document: 37 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2010.02.19a.html 38 39 /* 40 Refer to: Motorola: "Programming Environments Manual for 32-Bit 41 Implementations of the PowerPC Architecture", Appendix E: 42 "Synchronization Programming Examples" for an explanation of what is 43 going on here (can be found on the web at various places by the 44 name "MPCFPE32B.pdf", Google is your friend...) 45 46 Most of the atomic operations map to instructions in a relatively 47 straight-forward fashion, but "load"s may at first glance appear 48 a bit strange as they map to: 49 50 lwz %rX, addr 51 cmpw %rX, %rX 52 bne- 1f 53 1: 54 55 That is, the CPU is forced to perform a branch that "formally" depends 56 on the value retrieved from memory. This scheme has an overhead of 57 about 1-2 clock cycles per load, but it allows to map "acquire" to 58 the "isync" instruction instead of "sync" uniformly and for all type 59 of atomic operations. Since "isync" has a cost of about 15 clock 60 cycles, while "sync" hast a cost of about 50 clock cycles, the small 61 penalty to atomic loads more than compensates for this. 62 63 Byte- and halfword-sized atomic values are implemented in two ways. 64 When 8 and 16-bit instructions are available (in Power8 and later), 65 they are used. Otherwise operations are realized by encoding the 66 value to be represented into a word, performing sign/zero extension 67 as appropriate. This means that after add/sub operations the value 68 needs fixing up to accurately preserve the wrap-around semantic of 69 the smaller type. (Nothing special needs to be done for the bit-wise 70 and the "exchange type" operators as the compiler already sees to 71 it that values carried in registers are extended appropriately and 72 everything falls into place naturally). 73 74 The register constraint "b" instructs gcc to use any register 75 except r0; this is sometimes required because the encoding for 76 r0 is used to signify "constant zero" in a number of instructions, 77 making r0 unusable in this place. For simplicity this constraint 78 is used everywhere since I am to lazy to look this up on a 79 per-instruction basis, and ppc has enough registers for this not 80 to pose a problem. 81 */ 82 83 template< bool Signed, bool Interprocess > 84 struct core_arch_operations< 4u, Signed, Interprocess > : 85 public core_arch_operations_gcc_ppc_base 86 { 87 typedef typename storage_traits< 4u >::type storage_type; 88 89 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u; 90 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 4u; 91 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; 92 static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; 93 storeboost::atomics::detail::core_arch_operations94 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 95 { 96 fence_before(order); 97 __asm__ __volatile__ 98 ( 99 "stw %1, %0\n\t" 100 : "+m" (storage) 101 : "r" (v) 102 ); 103 } 104 loadboost::atomics::detail::core_arch_operations105 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT 106 { 107 storage_type v; 108 if (order == memory_order_seq_cst) 109 __asm__ __volatile__ ("sync" ::: "memory"); 110 if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u) 111 { 112 __asm__ __volatile__ 113 ( 114 "lwz %0, %1\n\t" 115 "cmpw %0, %0\n\t" 116 "bne- 1f\n\t" 117 "1:\n\t" 118 "isync\n\t" 119 : "=&r" (v) 120 : "m" (storage) 121 : "cr0", "memory" 122 ); 123 } 124 else 125 { 126 __asm__ __volatile__ 127 ( 128 "lwz %0, %1\n\t" 129 : "=&r" (v) 130 : "m" (storage) 131 ); 132 } 133 return v; 134 } 135 exchangeboost::atomics::detail::core_arch_operations136 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 137 { 138 storage_type original; 139 fence_before(order); 140 __asm__ __volatile__ 141 ( 142 "1:\n\t" 143 "lwarx %0,%y1\n\t" 144 "stwcx. %2,%y1\n\t" 145 "bne- 1b\n\t" 146 : "=&b" (original), "+Z" (storage) 147 : "b" (v) 148 : "cr0" 149 ); 150 fence_after(order); 151 return original; 152 } 153 compare_exchange_weakboost::atomics::detail::core_arch_operations154 static BOOST_FORCEINLINE bool compare_exchange_weak( 155 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT 156 { 157 int success; 158 fence_before(success_order); 159 __asm__ __volatile__ 160 ( 161 "li %1, 0\n\t" 162 "lwarx %0,%y2\n\t" 163 "cmpw %0, %3\n\t" 164 "bne- 1f\n\t" 165 "stwcx. %4,%y2\n\t" 166 "bne- 1f\n\t" 167 "li %1, 1\n\t" 168 "1:\n\t" 169 : "=&b" (expected), "=&b" (success), "+Z" (storage) 170 : "b" (expected), "b" (desired) 171 : "cr0" 172 ); 173 if (success) 174 fence_after(success_order); 175 else 176 fence_after(failure_order); 177 return !!success; 178 } 179 compare_exchange_strongboost::atomics::detail::core_arch_operations180 static BOOST_FORCEINLINE bool compare_exchange_strong( 181 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT 182 { 183 int success; 184 fence_before(success_order); 185 __asm__ __volatile__ 186 ( 187 "li %1, 0\n\t" 188 "0: lwarx %0,%y2\n\t" 189 "cmpw %0, %3\n\t" 190 "bne- 1f\n\t" 191 "stwcx. %4,%y2\n\t" 192 "bne- 0b\n\t" 193 "li %1, 1\n\t" 194 "1:\n\t" 195 : "=&b" (expected), "=&b" (success), "+Z" (storage) 196 : "b" (expected), "b" (desired) 197 : "cr0" 198 ); 199 if (success) 200 fence_after(success_order); 201 else 202 fence_after(failure_order); 203 return !!success; 204 } 205 fetch_addboost::atomics::detail::core_arch_operations206 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 207 { 208 storage_type original, result; 209 fence_before(order); 210 __asm__ __volatile__ 211 ( 212 "1:\n\t" 213 "lwarx %0,%y2\n\t" 214 "add %1,%0,%3\n\t" 215 "stwcx. %1,%y2\n\t" 216 "bne- 1b\n\t" 217 : "=&b" (original), "=&b" (result), "+Z" (storage) 218 : "b" (v) 219 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 220 ); 221 fence_after(order); 222 return original; 223 } 224 fetch_subboost::atomics::detail::core_arch_operations225 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 226 { 227 storage_type original, result; 228 fence_before(order); 229 __asm__ __volatile__ 230 ( 231 "1:\n\t" 232 "lwarx %0,%y2\n\t" 233 "sub %1,%0,%3\n\t" 234 "stwcx. %1,%y2\n\t" 235 "bne- 1b\n\t" 236 : "=&b" (original), "=&b" (result), "+Z" (storage) 237 : "b" (v) 238 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 239 ); 240 fence_after(order); 241 return original; 242 } 243 fetch_andboost::atomics::detail::core_arch_operations244 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 245 { 246 storage_type original, result; 247 fence_before(order); 248 __asm__ __volatile__ 249 ( 250 "1:\n\t" 251 "lwarx %0,%y2\n\t" 252 "and %1,%0,%3\n\t" 253 "stwcx. %1,%y2\n\t" 254 "bne- 1b\n\t" 255 : "=&b" (original), "=&b" (result), "+Z" (storage) 256 : "b" (v) 257 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 258 ); 259 fence_after(order); 260 return original; 261 } 262 fetch_orboost::atomics::detail::core_arch_operations263 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 264 { 265 storage_type original, result; 266 fence_before(order); 267 __asm__ __volatile__ 268 ( 269 "1:\n\t" 270 "lwarx %0,%y2\n\t" 271 "or %1,%0,%3\n\t" 272 "stwcx. %1,%y2\n\t" 273 "bne- 1b\n\t" 274 : "=&b" (original), "=&b" (result), "+Z" (storage) 275 : "b" (v) 276 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 277 ); 278 fence_after(order); 279 return original; 280 } 281 fetch_xorboost::atomics::detail::core_arch_operations282 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 283 { 284 storage_type original, result; 285 fence_before(order); 286 __asm__ __volatile__ 287 ( 288 "1:\n\t" 289 "lwarx %0,%y2\n\t" 290 "xor %1,%0,%3\n\t" 291 "stwcx. %1,%y2\n\t" 292 "bne- 1b\n\t" 293 : "=&b" (original), "=&b" (result), "+Z" (storage) 294 : "b" (v) 295 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 296 ); 297 fence_after(order); 298 return original; 299 } 300 test_and_setboost::atomics::detail::core_arch_operations301 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 302 { 303 return !!exchange(storage, (storage_type)1, order); 304 } 305 clearboost::atomics::detail::core_arch_operations306 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 307 { 308 store(storage, (storage_type)0, order); 309 } 310 }; 311 312 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX) 313 314 template< bool Signed, bool Interprocess > 315 struct core_arch_operations< 1u, Signed, Interprocess > : 316 public core_arch_operations_gcc_ppc_base 317 { 318 typedef typename storage_traits< 1u >::type storage_type; 319 320 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u; 321 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 1u; 322 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; 323 static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; 324 storeboost::atomics::detail::core_arch_operations325 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 326 { 327 fence_before(order); 328 __asm__ __volatile__ 329 ( 330 "stb %1, %0\n\t" 331 : "+m" (storage) 332 : "r" (v) 333 ); 334 } 335 loadboost::atomics::detail::core_arch_operations336 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT 337 { 338 storage_type v; 339 if (order == memory_order_seq_cst) 340 __asm__ __volatile__ ("sync" ::: "memory"); 341 if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u) 342 { 343 __asm__ __volatile__ 344 ( 345 "lbz %0, %1\n\t" 346 "cmpw %0, %0\n\t" 347 "bne- 1f\n\t" 348 "1:\n\t" 349 "isync\n\t" 350 : "=&r" (v) 351 : "m" (storage) 352 : "cr0", "memory" 353 ); 354 } 355 else 356 { 357 __asm__ __volatile__ 358 ( 359 "lbz %0, %1\n\t" 360 : "=&r" (v) 361 : "m" (storage) 362 ); 363 } 364 return v; 365 } 366 exchangeboost::atomics::detail::core_arch_operations367 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 368 { 369 storage_type original; 370 fence_before(order); 371 __asm__ __volatile__ 372 ( 373 "1:\n\t" 374 "lbarx %0,%y1\n\t" 375 "stbcx. %2,%y1\n\t" 376 "bne- 1b\n\t" 377 : "=&b" (original), "+Z" (storage) 378 : "b" (v) 379 : "cr0" 380 ); 381 fence_after(order); 382 return original; 383 } 384 compare_exchange_weakboost::atomics::detail::core_arch_operations385 static BOOST_FORCEINLINE bool compare_exchange_weak( 386 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT 387 { 388 int success; 389 fence_before(success_order); 390 __asm__ __volatile__ 391 ( 392 "li %1, 0\n\t" 393 "lbarx %0,%y2\n\t" 394 "cmpw %0, %3\n\t" 395 "bne- 1f\n\t" 396 "stbcx. %4,%y2\n\t" 397 "bne- 1f\n\t" 398 "li %1, 1\n\t" 399 "1:\n\t" 400 : "=&b" (expected), "=&b" (success), "+Z" (storage) 401 : "b" (expected), "b" (desired) 402 : "cr0" 403 ); 404 if (success) 405 fence_after(success_order); 406 else 407 fence_after(failure_order); 408 return !!success; 409 } 410 compare_exchange_strongboost::atomics::detail::core_arch_operations411 static BOOST_FORCEINLINE bool compare_exchange_strong( 412 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT 413 { 414 int success; 415 fence_before(success_order); 416 __asm__ __volatile__ 417 ( 418 "li %1, 0\n\t" 419 "0: lbarx %0,%y2\n\t" 420 "cmpw %0, %3\n\t" 421 "bne- 1f\n\t" 422 "stbcx. %4,%y2\n\t" 423 "bne- 0b\n\t" 424 "li %1, 1\n\t" 425 "1:\n\t" 426 : "=&b" (expected), "=&b" (success), "+Z" (storage) 427 : "b" (expected), "b" (desired) 428 : "cr0" 429 ); 430 if (success) 431 fence_after(success_order); 432 else 433 fence_after(failure_order); 434 return !!success; 435 } 436 fetch_addboost::atomics::detail::core_arch_operations437 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 438 { 439 storage_type original, result; 440 fence_before(order); 441 __asm__ __volatile__ 442 ( 443 "1:\n\t" 444 "lbarx %0,%y2\n\t" 445 "add %1,%0,%3\n\t" 446 "stbcx. %1,%y2\n\t" 447 "bne- 1b\n\t" 448 : "=&b" (original), "=&b" (result), "+Z" (storage) 449 : "b" (v) 450 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 451 ); 452 fence_after(order); 453 return original; 454 } 455 fetch_subboost::atomics::detail::core_arch_operations456 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 457 { 458 storage_type original, result; 459 fence_before(order); 460 __asm__ __volatile__ 461 ( 462 "1:\n\t" 463 "lbarx %0,%y2\n\t" 464 "sub %1,%0,%3\n\t" 465 "stbcx. %1,%y2\n\t" 466 "bne- 1b\n\t" 467 : "=&b" (original), "=&b" (result), "+Z" (storage) 468 : "b" (v) 469 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 470 ); 471 fence_after(order); 472 return original; 473 } 474 fetch_andboost::atomics::detail::core_arch_operations475 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 476 { 477 storage_type original, result; 478 fence_before(order); 479 __asm__ __volatile__ 480 ( 481 "1:\n\t" 482 "lbarx %0,%y2\n\t" 483 "and %1,%0,%3\n\t" 484 "stbcx. %1,%y2\n\t" 485 "bne- 1b\n\t" 486 : "=&b" (original), "=&b" (result), "+Z" (storage) 487 : "b" (v) 488 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 489 ); 490 fence_after(order); 491 return original; 492 } 493 fetch_orboost::atomics::detail::core_arch_operations494 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 495 { 496 storage_type original, result; 497 fence_before(order); 498 __asm__ __volatile__ 499 ( 500 "1:\n\t" 501 "lbarx %0,%y2\n\t" 502 "or %1,%0,%3\n\t" 503 "stbcx. %1,%y2\n\t" 504 "bne- 1b\n\t" 505 : "=&b" (original), "=&b" (result), "+Z" (storage) 506 : "b" (v) 507 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 508 ); 509 fence_after(order); 510 return original; 511 } 512 fetch_xorboost::atomics::detail::core_arch_operations513 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 514 { 515 storage_type original, result; 516 fence_before(order); 517 __asm__ __volatile__ 518 ( 519 "1:\n\t" 520 "lbarx %0,%y2\n\t" 521 "xor %1,%0,%3\n\t" 522 "stbcx. %1,%y2\n\t" 523 "bne- 1b\n\t" 524 : "=&b" (original), "=&b" (result), "+Z" (storage) 525 : "b" (v) 526 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 527 ); 528 fence_after(order); 529 return original; 530 } 531 test_and_setboost::atomics::detail::core_arch_operations532 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 533 { 534 return !!exchange(storage, (storage_type)1, order); 535 } 536 clearboost::atomics::detail::core_arch_operations537 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 538 { 539 store(storage, (storage_type)0, order); 540 } 541 }; 542 543 #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX) 544 545 template< bool Interprocess > 546 struct core_arch_operations< 1u, false, Interprocess > : 547 public core_arch_operations< 4u, false, Interprocess > 548 { 549 typedef core_arch_operations< 4u, false, Interprocess > base_type; 550 typedef typename base_type::storage_type storage_type; 551 fetch_addboost::atomics::detail::core_arch_operations552 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 553 { 554 storage_type original, result; 555 base_type::fence_before(order); 556 __asm__ __volatile__ 557 ( 558 "1:\n\t" 559 "lwarx %0,%y2\n\t" 560 "add %1,%0,%3\n\t" 561 "rlwinm %1, %1, 0, 0xff\n\t" 562 "stwcx. %1,%y2\n\t" 563 "bne- 1b\n\t" 564 : "=&b" (original), "=&b" (result), "+Z" (storage) 565 : "b" (v) 566 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 567 ); 568 base_type::fence_after(order); 569 return original; 570 } 571 fetch_subboost::atomics::detail::core_arch_operations572 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 573 { 574 storage_type original, result; 575 base_type::fence_before(order); 576 __asm__ __volatile__ 577 ( 578 "1:\n\t" 579 "lwarx %0,%y2\n\t" 580 "sub %1,%0,%3\n\t" 581 "rlwinm %1, %1, 0, 0xff\n\t" 582 "stwcx. %1,%y2\n\t" 583 "bne- 1b\n\t" 584 : "=&b" (original), "=&b" (result), "+Z" (storage) 585 : "b" (v) 586 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 587 ); 588 base_type::fence_after(order); 589 return original; 590 } 591 }; 592 593 template< bool Interprocess > 594 struct core_arch_operations< 1u, true, Interprocess > : 595 public core_arch_operations< 4u, true, Interprocess > 596 { 597 typedef core_arch_operations< 4u, true, Interprocess > base_type; 598 typedef typename base_type::storage_type storage_type; 599 fetch_addboost::atomics::detail::core_arch_operations600 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 601 { 602 storage_type original, result; 603 base_type::fence_before(order); 604 __asm__ __volatile__ 605 ( 606 "1:\n\t" 607 "lwarx %0,%y2\n\t" 608 "add %1,%0,%3\n\t" 609 "extsb %1, %1\n\t" 610 "stwcx. %1,%y2\n\t" 611 "bne- 1b\n\t" 612 : "=&b" (original), "=&b" (result), "+Z" (storage) 613 : "b" (v) 614 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 615 ); 616 base_type::fence_after(order); 617 return original; 618 } 619 fetch_subboost::atomics::detail::core_arch_operations620 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 621 { 622 storage_type original, result; 623 base_type::fence_before(order); 624 __asm__ __volatile__ 625 ( 626 "1:\n\t" 627 "lwarx %0,%y2\n\t" 628 "sub %1,%0,%3\n\t" 629 "extsb %1, %1\n\t" 630 "stwcx. %1,%y2\n\t" 631 "bne- 1b\n\t" 632 : "=&b" (original), "=&b" (result), "+Z" (storage) 633 : "b" (v) 634 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 635 ); 636 base_type::fence_after(order); 637 return original; 638 } 639 }; 640 641 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX) 642 643 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX) 644 645 template< bool Signed, bool Interprocess > 646 struct core_arch_operations< 2u, Signed, Interprocess > : 647 public core_arch_operations_gcc_ppc_base 648 { 649 typedef typename storage_traits< 2u >::type storage_type; 650 651 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u; 652 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 2u; 653 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; 654 static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; 655 storeboost::atomics::detail::core_arch_operations656 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 657 { 658 fence_before(order); 659 __asm__ __volatile__ 660 ( 661 "sth %1, %0\n\t" 662 : "+m" (storage) 663 : "r" (v) 664 ); 665 } 666 loadboost::atomics::detail::core_arch_operations667 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT 668 { 669 storage_type v; 670 if (order == memory_order_seq_cst) 671 __asm__ __volatile__ ("sync" ::: "memory"); 672 if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u) 673 { 674 __asm__ __volatile__ 675 ( 676 "lhz %0, %1\n\t" 677 "cmpw %0, %0\n\t" 678 "bne- 1f\n\t" 679 "1:\n\t" 680 "isync\n\t" 681 : "=&r" (v) 682 : "m" (storage) 683 : "cr0", "memory" 684 ); 685 } 686 else 687 { 688 __asm__ __volatile__ 689 ( 690 "lhz %0, %1\n\t" 691 : "=&r" (v) 692 : "m" (storage) 693 ); 694 } 695 return v; 696 } 697 exchangeboost::atomics::detail::core_arch_operations698 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 699 { 700 storage_type original; 701 fence_before(order); 702 __asm__ __volatile__ 703 ( 704 "1:\n\t" 705 "lharx %0,%y1\n\t" 706 "sthcx. %2,%y1\n\t" 707 "bne- 1b\n\t" 708 : "=&b" (original), "+Z" (storage) 709 : "b" (v) 710 : "cr0" 711 ); 712 fence_after(order); 713 return original; 714 } 715 compare_exchange_weakboost::atomics::detail::core_arch_operations716 static BOOST_FORCEINLINE bool compare_exchange_weak( 717 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT 718 { 719 int success; 720 fence_before(success_order); 721 __asm__ __volatile__ 722 ( 723 "li %1, 0\n\t" 724 "lharx %0,%y2\n\t" 725 "cmpw %0, %3\n\t" 726 "bne- 1f\n\t" 727 "sthcx. %4,%y2\n\t" 728 "bne- 1f\n\t" 729 "li %1, 1\n\t" 730 "1:\n\t" 731 : "=&b" (expected), "=&b" (success), "+Z" (storage) 732 : "b" (expected), "b" (desired) 733 : "cr0" 734 ); 735 if (success) 736 fence_after(success_order); 737 else 738 fence_after(failure_order); 739 return !!success; 740 } 741 compare_exchange_strongboost::atomics::detail::core_arch_operations742 static BOOST_FORCEINLINE bool compare_exchange_strong( 743 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT 744 { 745 int success; 746 fence_before(success_order); 747 __asm__ __volatile__ 748 ( 749 "li %1, 0\n\t" 750 "0: lharx %0,%y2\n\t" 751 "cmpw %0, %3\n\t" 752 "bne- 1f\n\t" 753 "sthcx. %4,%y2\n\t" 754 "bne- 0b\n\t" 755 "li %1, 1\n\t" 756 "1:\n\t" 757 : "=&b" (expected), "=&b" (success), "+Z" (storage) 758 : "b" (expected), "b" (desired) 759 : "cr0" 760 ); 761 if (success) 762 fence_after(success_order); 763 else 764 fence_after(failure_order); 765 return !!success; 766 } 767 fetch_addboost::atomics::detail::core_arch_operations768 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 769 { 770 storage_type original, result; 771 fence_before(order); 772 __asm__ __volatile__ 773 ( 774 "1:\n\t" 775 "lharx %0,%y2\n\t" 776 "add %1,%0,%3\n\t" 777 "sthcx. %1,%y2\n\t" 778 "bne- 1b\n\t" 779 : "=&b" (original), "=&b" (result), "+Z" (storage) 780 : "b" (v) 781 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 782 ); 783 fence_after(order); 784 return original; 785 } 786 fetch_subboost::atomics::detail::core_arch_operations787 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 788 { 789 storage_type original, result; 790 fence_before(order); 791 __asm__ __volatile__ 792 ( 793 "1:\n\t" 794 "lharx %0,%y2\n\t" 795 "sub %1,%0,%3\n\t" 796 "sthcx. %1,%y2\n\t" 797 "bne- 1b\n\t" 798 : "=&b" (original), "=&b" (result), "+Z" (storage) 799 : "b" (v) 800 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 801 ); 802 fence_after(order); 803 return original; 804 } 805 fetch_andboost::atomics::detail::core_arch_operations806 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 807 { 808 storage_type original, result; 809 fence_before(order); 810 __asm__ __volatile__ 811 ( 812 "1:\n\t" 813 "lharx %0,%y2\n\t" 814 "and %1,%0,%3\n\t" 815 "sthcx. %1,%y2\n\t" 816 "bne- 1b\n\t" 817 : "=&b" (original), "=&b" (result), "+Z" (storage) 818 : "b" (v) 819 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 820 ); 821 fence_after(order); 822 return original; 823 } 824 fetch_orboost::atomics::detail::core_arch_operations825 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 826 { 827 storage_type original, result; 828 fence_before(order); 829 __asm__ __volatile__ 830 ( 831 "1:\n\t" 832 "lharx %0,%y2\n\t" 833 "or %1,%0,%3\n\t" 834 "sthcx. %1,%y2\n\t" 835 "bne- 1b\n\t" 836 : "=&b" (original), "=&b" (result), "+Z" (storage) 837 : "b" (v) 838 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 839 ); 840 fence_after(order); 841 return original; 842 } 843 fetch_xorboost::atomics::detail::core_arch_operations844 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 845 { 846 storage_type original, result; 847 fence_before(order); 848 __asm__ __volatile__ 849 ( 850 "1:\n\t" 851 "lharx %0,%y2\n\t" 852 "xor %1,%0,%3\n\t" 853 "sthcx. %1,%y2\n\t" 854 "bne- 1b\n\t" 855 : "=&b" (original), "=&b" (result), "+Z" (storage) 856 : "b" (v) 857 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 858 ); 859 fence_after(order); 860 return original; 861 } 862 test_and_setboost::atomics::detail::core_arch_operations863 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 864 { 865 return !!exchange(storage, (storage_type)1, order); 866 } 867 clearboost::atomics::detail::core_arch_operations868 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 869 { 870 store(storage, (storage_type)0, order); 871 } 872 }; 873 874 #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX) 875 876 template< bool Interprocess > 877 struct core_arch_operations< 2u, false, Interprocess > : 878 public core_arch_operations< 4u, false, Interprocess > 879 { 880 typedef core_arch_operations< 4u, false, Interprocess > base_type; 881 typedef typename base_type::storage_type storage_type; 882 fetch_addboost::atomics::detail::core_arch_operations883 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 884 { 885 storage_type original, result; 886 base_type::fence_before(order); 887 __asm__ __volatile__ 888 ( 889 "1:\n\t" 890 "lwarx %0,%y2\n\t" 891 "add %1,%0,%3\n\t" 892 "rlwinm %1, %1, 0, 0xffff\n\t" 893 "stwcx. %1,%y2\n\t" 894 "bne- 1b\n\t" 895 : "=&b" (original), "=&b" (result), "+Z" (storage) 896 : "b" (v) 897 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 898 ); 899 base_type::fence_after(order); 900 return original; 901 } 902 fetch_subboost::atomics::detail::core_arch_operations903 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 904 { 905 storage_type original, result; 906 base_type::fence_before(order); 907 __asm__ __volatile__ 908 ( 909 "1:\n\t" 910 "lwarx %0,%y2\n\t" 911 "sub %1,%0,%3\n\t" 912 "rlwinm %1, %1, 0, 0xffff\n\t" 913 "stwcx. %1,%y2\n\t" 914 "bne- 1b\n\t" 915 : "=&b" (original), "=&b" (result), "+Z" (storage) 916 : "b" (v) 917 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 918 ); 919 base_type::fence_after(order); 920 return original; 921 } 922 }; 923 924 template< bool Interprocess > 925 struct core_arch_operations< 2u, true, Interprocess > : 926 public core_arch_operations< 4u, true, Interprocess > 927 { 928 typedef core_arch_operations< 4u, true, Interprocess > base_type; 929 typedef typename base_type::storage_type storage_type; 930 fetch_addboost::atomics::detail::core_arch_operations931 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 932 { 933 storage_type original, result; 934 base_type::fence_before(order); 935 __asm__ __volatile__ 936 ( 937 "1:\n\t" 938 "lwarx %0,%y2\n\t" 939 "add %1,%0,%3\n\t" 940 "extsh %1, %1\n\t" 941 "stwcx. %1,%y2\n\t" 942 "bne- 1b\n\t" 943 : "=&b" (original), "=&b" (result), "+Z" (storage) 944 : "b" (v) 945 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 946 ); 947 base_type::fence_after(order); 948 return original; 949 } 950 fetch_subboost::atomics::detail::core_arch_operations951 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 952 { 953 storage_type original, result; 954 base_type::fence_before(order); 955 __asm__ __volatile__ 956 ( 957 "1:\n\t" 958 "lwarx %0,%y2\n\t" 959 "sub %1,%0,%3\n\t" 960 "extsh %1, %1\n\t" 961 "stwcx. %1,%y2\n\t" 962 "bne- 1b\n\t" 963 : "=&b" (original), "=&b" (result), "+Z" (storage) 964 : "b" (v) 965 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 966 ); 967 base_type::fence_after(order); 968 return original; 969 } 970 }; 971 972 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX) 973 974 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX) 975 976 template< bool Signed, bool Interprocess > 977 struct core_arch_operations< 8u, Signed, Interprocess > : 978 public core_arch_operations_gcc_ppc_base 979 { 980 typedef typename storage_traits< 8u >::type storage_type; 981 982 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u; 983 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u; 984 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; 985 static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; 986 storeboost::atomics::detail::core_arch_operations987 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 988 { 989 fence_before(order); 990 __asm__ __volatile__ 991 ( 992 "std %1, %0\n\t" 993 : "+m" (storage) 994 : "r" (v) 995 ); 996 } 997 loadboost::atomics::detail::core_arch_operations998 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT 999 { 1000 storage_type v; 1001 if (order == memory_order_seq_cst) 1002 __asm__ __volatile__ ("sync" ::: "memory"); 1003 if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u) 1004 { 1005 __asm__ __volatile__ 1006 ( 1007 "ld %0, %1\n\t" 1008 "cmpd %0, %0\n\t" 1009 "bne- 1f\n\t" 1010 "1:\n\t" 1011 "isync\n\t" 1012 : "=&b" (v) 1013 : "m" (storage) 1014 : "cr0", "memory" 1015 ); 1016 } 1017 else 1018 { 1019 __asm__ __volatile__ 1020 ( 1021 "ld %0, %1\n\t" 1022 : "=&b" (v) 1023 : "m" (storage) 1024 ); 1025 } 1026 return v; 1027 } 1028 exchangeboost::atomics::detail::core_arch_operations1029 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1030 { 1031 storage_type original; 1032 fence_before(order); 1033 __asm__ __volatile__ 1034 ( 1035 "1:\n\t" 1036 "ldarx %0,%y1\n\t" 1037 "stdcx. %2,%y1\n\t" 1038 "bne- 1b\n\t" 1039 : "=&b" (original), "+Z" (storage) 1040 : "b" (v) 1041 : "cr0" 1042 ); 1043 fence_after(order); 1044 return original; 1045 } 1046 compare_exchange_weakboost::atomics::detail::core_arch_operations1047 static BOOST_FORCEINLINE bool compare_exchange_weak( 1048 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT 1049 { 1050 int success; 1051 fence_before(success_order); 1052 __asm__ __volatile__ 1053 ( 1054 "li %1, 0\n\t" 1055 "ldarx %0,%y2\n\t" 1056 "cmpd %0, %3\n\t" 1057 "bne- 1f\n\t" 1058 "stdcx. %4,%y2\n\t" 1059 "bne- 1f\n\t" 1060 "li %1, 1\n\t" 1061 "1:" 1062 : "=&b" (expected), "=&b" (success), "+Z" (storage) 1063 : "b" (expected), "b" (desired) 1064 : "cr0" 1065 ); 1066 if (success) 1067 fence_after(success_order); 1068 else 1069 fence_after(failure_order); 1070 return !!success; 1071 } 1072 compare_exchange_strongboost::atomics::detail::core_arch_operations1073 static BOOST_FORCEINLINE bool compare_exchange_strong( 1074 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT 1075 { 1076 int success; 1077 fence_before(success_order); 1078 __asm__ __volatile__ 1079 ( 1080 "li %1, 0\n\t" 1081 "0: ldarx %0,%y2\n\t" 1082 "cmpd %0, %3\n\t" 1083 "bne- 1f\n\t" 1084 "stdcx. %4,%y2\n\t" 1085 "bne- 0b\n\t" 1086 "li %1, 1\n\t" 1087 "1:\n\t" 1088 : "=&b" (expected), "=&b" (success), "+Z" (storage) 1089 : "b" (expected), "b" (desired) 1090 : "cr0" 1091 ); 1092 if (success) 1093 fence_after(success_order); 1094 else 1095 fence_after(failure_order); 1096 return !!success; 1097 } 1098 fetch_addboost::atomics::detail::core_arch_operations1099 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1100 { 1101 storage_type original, result; 1102 fence_before(order); 1103 __asm__ __volatile__ 1104 ( 1105 "1:\n\t" 1106 "ldarx %0,%y2\n\t" 1107 "add %1,%0,%3\n\t" 1108 "stdcx. %1,%y2\n\t" 1109 "bne- 1b\n\t" 1110 : "=&b" (original), "=&b" (result), "+Z" (storage) 1111 : "b" (v) 1112 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 1113 ); 1114 fence_after(order); 1115 return original; 1116 } 1117 fetch_subboost::atomics::detail::core_arch_operations1118 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1119 { 1120 storage_type original, result; 1121 fence_before(order); 1122 __asm__ __volatile__ 1123 ( 1124 "1:\n\t" 1125 "ldarx %0,%y2\n\t" 1126 "sub %1,%0,%3\n\t" 1127 "stdcx. %1,%y2\n\t" 1128 "bne- 1b\n\t" 1129 : "=&b" (original), "=&b" (result), "+Z" (storage) 1130 : "b" (v) 1131 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 1132 ); 1133 fence_after(order); 1134 return original; 1135 } 1136 fetch_andboost::atomics::detail::core_arch_operations1137 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1138 { 1139 storage_type original, result; 1140 fence_before(order); 1141 __asm__ __volatile__ 1142 ( 1143 "1:\n\t" 1144 "ldarx %0,%y2\n\t" 1145 "and %1,%0,%3\n\t" 1146 "stdcx. %1,%y2\n\t" 1147 "bne- 1b\n\t" 1148 : "=&b" (original), "=&b" (result), "+Z" (storage) 1149 : "b" (v) 1150 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 1151 ); 1152 fence_after(order); 1153 return original; 1154 } 1155 fetch_orboost::atomics::detail::core_arch_operations1156 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1157 { 1158 storage_type original, result; 1159 fence_before(order); 1160 __asm__ __volatile__ 1161 ( 1162 "1:\n\t" 1163 "ldarx %0,%y2\n\t" 1164 "or %1,%0,%3\n\t" 1165 "stdcx. %1,%y2\n\t" 1166 "bne- 1b\n\t" 1167 : "=&b" (original), "=&b" (result), "+Z" (storage) 1168 : "b" (v) 1169 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 1170 ); 1171 fence_after(order); 1172 return original; 1173 } 1174 fetch_xorboost::atomics::detail::core_arch_operations1175 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1176 { 1177 storage_type original, result; 1178 fence_before(order); 1179 __asm__ __volatile__ 1180 ( 1181 "1:\n\t" 1182 "ldarx %0,%y2\n\t" 1183 "xor %1,%0,%3\n\t" 1184 "stdcx. %1,%y2\n\t" 1185 "bne- 1b\n\t" 1186 : "=&b" (original), "=&b" (result), "+Z" (storage) 1187 : "b" (v) 1188 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC 1189 ); 1190 fence_after(order); 1191 return original; 1192 } 1193 test_and_setboost::atomics::detail::core_arch_operations1194 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1195 { 1196 return !!exchange(storage, (storage_type)1, order); 1197 } 1198 clearboost::atomics::detail::core_arch_operations1199 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1200 { 1201 store(storage, (storage_type)0, order); 1202 } 1203 }; 1204 1205 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX) 1206 1207 } // namespace detail 1208 } // namespace atomics 1209 } // namespace boost 1210 1211 #include <boost/atomic/detail/footer.hpp> 1212 1213 #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_ 1214