1 /*
2 * Distributed under the Boost Software License, Version 1.0.
3 * (See accompanying file LICENSE_1_0.txt or copy at
4 * http://www.boost.org/LICENSE_1_0.txt)
5 *
6 * Copyright (c) 2009, 2011 Helge Bahmann
7 * Copyright (c) 2009 Phil Endecott
8 * Copyright (c) 2013 Tim Blechmann
9 * Linux-specific code by Phil Endecott
10 * Copyright (c) 2014 Andrey Semashev
11 */
12 /*!
13 * \file atomic/detail/ops_linux_arm.hpp
14 *
15 * This header contains implementation of the \c operations template.
16 */
17
18 #ifndef BOOST_ATOMIC_DETAIL_OPS_LINUX_ARM_HPP_INCLUDED_
19 #define BOOST_ATOMIC_DETAIL_OPS_LINUX_ARM_HPP_INCLUDED_
20
21 #include <cstddef>
22 #include <boost/memory_order.hpp>
23 #include <boost/atomic/detail/config.hpp>
24 #include <boost/atomic/detail/storage_type.hpp>
25 #include <boost/atomic/detail/operations_fwd.hpp>
26 #include <boost/atomic/capabilities.hpp>
27 #include <boost/atomic/detail/ops_cas_based.hpp>
28 #include <boost/atomic/detail/ops_extending_cas_based.hpp>
29
30 #ifdef BOOST_HAS_PRAGMA_ONCE
31 #pragma once
32 #endif
33
34 namespace boost {
35 namespace atomics {
36 namespace detail {
37
38 // Different ARM processors have different atomic instructions. In particular,
39 // architecture versions before v6 (which are still in widespread use, e.g. the
40 // Intel/Marvell XScale chips like the one in the NSLU2) have only atomic swap.
41 // On Linux the kernel provides some support that lets us abstract away from
42 // these differences: it provides emulated CAS and barrier functions at special
43 // addresses that are guaranteed not to be interrupted by the kernel. Using
44 // this facility is slightly slower than inline assembler would be, but much
45 // faster than a system call.
46 //
47 // While this emulated CAS is "strong" in the sense that it does not fail
48 // "spuriously" (i.e.: it never fails to perform the exchange when the value
49 // found equals the value expected), it does not return the found value on
50 // failure. To satisfy the atomic API, compare_exchange_{weak|strong} must
51 // return the found value on failure, and we have to manually load this value
52 // after the emulated CAS reports failure. This in turn introduces a race
53 // between the CAS failing (due to the "wrong" value being found) and subsequently
54 // loading (which might turn up the "right" value). From an application's
55 // point of view this looks like "spurious failure", and therefore the
56 // emulated CAS is only good enough to provide compare_exchange_weak
57 // semantics.
58
59 struct linux_arm_cas_base
60 {
61 static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = true;
62 static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
63
fence_before_storeboost::atomics::detail::linux_arm_cas_base64 static BOOST_FORCEINLINE void fence_before_store(memory_order order) BOOST_NOEXCEPT
65 {
66 if ((static_cast< unsigned int >(order) & static_cast< unsigned int >(memory_order_release)) != 0u)
67 hardware_full_fence();
68 }
69
fence_after_storeboost::atomics::detail::linux_arm_cas_base70 static BOOST_FORCEINLINE void fence_after_store(memory_order order) BOOST_NOEXCEPT
71 {
72 if (order == memory_order_seq_cst)
73 hardware_full_fence();
74 }
75
fence_after_loadboost::atomics::detail::linux_arm_cas_base76 static BOOST_FORCEINLINE void fence_after_load(memory_order order) BOOST_NOEXCEPT
77 {
78 if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
79 hardware_full_fence();
80 }
81
hardware_full_fenceboost::atomics::detail::linux_arm_cas_base82 static BOOST_FORCEINLINE void hardware_full_fence() BOOST_NOEXCEPT
83 {
84 typedef void (*kernel_dmb_t)(void);
85 ((kernel_dmb_t)0xffff0fa0)();
86 }
87 };
88
89 template< bool Signed >
90 struct linux_arm_cas :
91 public linux_arm_cas_base
92 {
93 typedef typename make_storage_type< 4u >::type storage_type;
94 typedef typename make_storage_type< 4u >::aligned aligned_storage_type;
95
96 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
97 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
98
storeboost::atomics::detail::linux_arm_cas99 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
100 {
101 fence_before_store(order);
102 storage = v;
103 fence_after_store(order);
104 }
105
loadboost::atomics::detail::linux_arm_cas106 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
107 {
108 storage_type v = storage;
109 fence_after_load(order);
110 return v;
111 }
112
compare_exchange_strongboost::atomics::detail::linux_arm_cas113 static BOOST_FORCEINLINE bool compare_exchange_strong(
114 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
115 {
116 while (true)
117 {
118 storage_type tmp = expected;
119 if (compare_exchange_weak(storage, tmp, desired, success_order, failure_order))
120 return true;
121 if (tmp != expected)
122 {
123 expected = tmp;
124 return false;
125 }
126 }
127 }
128
compare_exchange_weakboost::atomics::detail::linux_arm_cas129 static BOOST_FORCEINLINE bool compare_exchange_weak(
130 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
131 {
132 typedef storage_type (*kernel_cmpxchg32_t)(storage_type oldval, storage_type newval, volatile storage_type* ptr);
133
134 if (((kernel_cmpxchg32_t)0xffff0fc0)(expected, desired, &storage) == 0)
135 {
136 return true;
137 }
138 else
139 {
140 expected = storage;
141 return false;
142 }
143 }
144 };
145
146 template< bool Signed >
147 struct operations< 1u, Signed > :
148 public extending_cas_based_operations< cas_based_operations< cas_based_exchange< linux_arm_cas< Signed > > >, 1u, Signed >
149 {
150 };
151
152 template< bool Signed >
153 struct operations< 2u, Signed > :
154 public extending_cas_based_operations< cas_based_operations< cas_based_exchange< linux_arm_cas< Signed > > >, 2u, Signed >
155 {
156 };
157
158 template< bool Signed >
159 struct operations< 4u, Signed > :
160 public cas_based_operations< cas_based_exchange< linux_arm_cas< Signed > > >
161 {
162 };
163
thread_fence(memory_order order)164 BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
165 {
166 if (order != memory_order_relaxed)
167 linux_arm_cas_base::hardware_full_fence();
168 }
169
signal_fence(memory_order order)170 BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
171 {
172 if (order != memory_order_relaxed)
173 __asm__ __volatile__ ("" ::: "memory");
174 }
175
176 } // namespace detail
177 } // namespace atomics
178 } // namespace boost
179
180 #endif // BOOST_ATOMIC_DETAIL_OPS_LINUX_ARM_HPP_INCLUDED_
181