1 /*
2  * Distributed under the Boost Software License, Version 1.0.
3  * (See accompanying file LICENSE_1_0.txt or copy at
4  * http://www.boost.org/LICENSE_1_0.txt)
5  *
6  * Copyright (c) 2009 Helge Bahmann
7  * Copyright (c) 2012 Tim Blechmann
8  * Copyright (c) 2014 Andrey Semashev
9  */
10 /*!
11  * \file   atomic/detail/ops_msvc_arm.hpp
12  *
13  * This header contains implementation of the \c operations template.
14  */
15 
16 #ifndef BOOST_ATOMIC_DETAIL_OPS_MSVC_ARM_HPP_INCLUDED_
17 #define BOOST_ATOMIC_DETAIL_OPS_MSVC_ARM_HPP_INCLUDED_
18 
19 #include <intrin.h>
20 #include <cstddef>
21 #include <boost/memory_order.hpp>
22 #include <boost/atomic/detail/config.hpp>
23 #include <boost/atomic/detail/interlocked.hpp>
24 #include <boost/atomic/detail/storage_type.hpp>
25 #include <boost/atomic/detail/operations_fwd.hpp>
26 #include <boost/atomic/detail/type_traits/make_signed.hpp>
27 #include <boost/atomic/capabilities.hpp>
28 #include <boost/atomic/detail/ops_msvc_common.hpp>
29 
30 #ifdef BOOST_HAS_PRAGMA_ONCE
31 #pragma once
32 #endif
33 
34 #define BOOST_ATOMIC_DETAIL_ARM_LOAD8(p) __iso_volatile_load8((const volatile __int8*)(p))
35 #define BOOST_ATOMIC_DETAIL_ARM_LOAD16(p) __iso_volatile_load16((const volatile __int16*)(p))
36 #define BOOST_ATOMIC_DETAIL_ARM_LOAD32(p) __iso_volatile_load32((const volatile __int32*)(p))
37 #define BOOST_ATOMIC_DETAIL_ARM_LOAD64(p) __iso_volatile_load64((const volatile __int64*)(p))
38 #define BOOST_ATOMIC_DETAIL_ARM_STORE8(p, v) __iso_volatile_store8((volatile __int8*)(p), (__int8)(v))
39 #define BOOST_ATOMIC_DETAIL_ARM_STORE16(p, v) __iso_volatile_store16((volatile __int16*)(p), (__int16)(v))
40 #define BOOST_ATOMIC_DETAIL_ARM_STORE32(p, v) __iso_volatile_store32((volatile __int32*)(p), (__int32)(v))
41 #define BOOST_ATOMIC_DETAIL_ARM_STORE64(p, v) __iso_volatile_store64((volatile __int64*)(p), (__int64)(v))
42 
43 namespace boost {
44 namespace atomics {
45 namespace detail {
46 
47 // A note about memory_order_consume. Technically, this architecture allows to avoid
48 // unnecessary memory barrier after consume load since it supports data dependency ordering.
49 // However, some compiler optimizations may break a seemingly valid code relying on data
50 // dependency tracking by injecting bogus branches to aid out of order execution.
51 // This may happen not only in Boost.Atomic code but also in user's code, which we have no
52 // control of. See this thread: http://lists.boost.org/Archives/boost/2014/06/213890.php.
53 // For this reason we promote memory_order_consume to memory_order_acquire.
54 
55 struct msvc_arm_operations_base
56 {
57     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
58 
hardware_full_fenceboost::atomics::detail::msvc_arm_operations_base59     static BOOST_FORCEINLINE void hardware_full_fence() BOOST_NOEXCEPT
60     {
61         __dmb(0xB); // _ARM_BARRIER_ISH, see armintr.h from MSVC 11 and later
62     }
63 
fence_before_storeboost::atomics::detail::msvc_arm_operations_base64     static BOOST_FORCEINLINE void fence_before_store(memory_order order) BOOST_NOEXCEPT
65     {
66         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
67 
68         if ((order & memory_order_release) != 0)
69             hardware_full_fence();
70 
71         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
72     }
73 
fence_after_storeboost::atomics::detail::msvc_arm_operations_base74     static BOOST_FORCEINLINE void fence_after_store(memory_order order) BOOST_NOEXCEPT
75     {
76         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
77 
78         if (order == memory_order_seq_cst)
79             hardware_full_fence();
80 
81         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
82     }
83 
fence_after_loadboost::atomics::detail::msvc_arm_operations_base84     static BOOST_FORCEINLINE void fence_after_load(memory_order order) BOOST_NOEXCEPT
85     {
86         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
87 
88         if ((order & (memory_order_consume | memory_order_acquire)) != 0)
89             hardware_full_fence();
90 
91         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
92     }
93 
cas_common_orderboost::atomics::detail::msvc_arm_operations_base94     static BOOST_FORCEINLINE BOOST_CONSTEXPR memory_order cas_common_order(memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
95     {
96         // Combine order flags together and promote memory_order_consume to memory_order_acquire
97         return static_cast< memory_order >(((failure_order | success_order) & ~memory_order_consume) | (((failure_order | success_order) & memory_order_consume) << 1u));
98     }
99 };
100 
101 template< typename T, typename Derived >
102 struct msvc_arm_operations :
103     public msvc_arm_operations_base
104 {
105     typedef T storage_type;
106 
fetch_subboost::atomics::detail::msvc_arm_operations107     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
108     {
109         typedef typename boost::atomics::detail::make_signed< storage_type >::type signed_storage_type;
110         return Derived::fetch_add(storage, static_cast< storage_type >(-static_cast< signed_storage_type >(v)), order);
111     }
112 
compare_exchange_weakboost::atomics::detail::msvc_arm_operations113     static BOOST_FORCEINLINE bool compare_exchange_weak(
114         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
115     {
116         return Derived::compare_exchange_strong(storage, expected, desired, success_order, failure_order);
117     }
118 
test_and_setboost::atomics::detail::msvc_arm_operations119     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
120     {
121         return !!Derived::exchange(storage, (storage_type)1, order);
122     }
123 
clearboost::atomics::detail::msvc_arm_operations124     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
125     {
126         Derived::store(storage, (storage_type)0, order);
127     }
128 };
129 
130 template< bool Signed >
131 struct operations< 1u, Signed > :
132     public msvc_arm_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > >
133 {
134     typedef msvc_arm_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > > base_type;
135     typedef typename base_type::storage_type storage_type;
136     typedef typename make_storage_type< 1u, Signed >::aligned aligned_storage_type;
137 
138     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
139     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
140 
storeboost::atomics::detail::operations141     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
142     {
143         base_type::fence_before_store(order);
144         BOOST_ATOMIC_DETAIL_ARM_STORE8(&storage, v);
145         base_type::fence_after_store(order);
146     }
147 
loadboost::atomics::detail::operations148     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
149     {
150         storage_type v = BOOST_ATOMIC_DETAIL_ARM_LOAD8(&storage);
151         base_type::fence_after_load(order);
152         return v;
153     }
154 
fetch_addboost::atomics::detail::operations155     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
156     {
157         switch (order)
158         {
159         case memory_order_relaxed:
160             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD8_RELAXED(&storage, v));
161             break;
162         case memory_order_consume:
163         case memory_order_acquire:
164             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD8_ACQUIRE(&storage, v));
165             break;
166         case memory_order_release:
167             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD8_RELEASE(&storage, v));
168             break;
169         case memory_order_acq_rel:
170         case memory_order_seq_cst:
171         default:
172             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD8(&storage, v));
173             break;
174         }
175         return v;
176     }
177 
exchangeboost::atomics::detail::operations178     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
179     {
180         switch (order)
181         {
182         case memory_order_relaxed:
183             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE8_RELAXED(&storage, v));
184             break;
185         case memory_order_consume:
186         case memory_order_acquire:
187             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE8_ACQUIRE(&storage, v));
188             break;
189         case memory_order_release:
190             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE8_RELEASE(&storage, v));
191             break;
192         case memory_order_acq_rel:
193         case memory_order_seq_cst:
194         default:
195             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE8(&storage, v));
196             break;
197         }
198         return v;
199     }
200 
compare_exchange_strongboost::atomics::detail::operations201     static BOOST_FORCEINLINE bool compare_exchange_strong(
202         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
203     {
204         storage_type previous = expected, old_val;
205 
206         switch (cas_common_order(success_order, failure_order))
207         {
208         case memory_order_relaxed:
209             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8_RELAXED(&storage, desired, previous));
210             break;
211         case memory_order_consume:
212         case memory_order_acquire:
213             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8_ACQUIRE(&storage, desired, previous));
214             break;
215         case memory_order_release:
216             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8_RELEASE(&storage, desired, previous));
217             break;
218         case memory_order_acq_rel:
219         case memory_order_seq_cst:
220         default:
221             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8(&storage, desired, previous));
222             break;
223         }
224         expected = old_val;
225 
226         return (previous == old_val);
227     }
228 
fetch_andboost::atomics::detail::operations229     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
230     {
231         switch (order)
232         {
233         case memory_order_relaxed:
234             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND8_RELAXED(&storage, v));
235             break;
236         case memory_order_consume:
237         case memory_order_acquire:
238             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND8_ACQUIRE(&storage, v));
239             break;
240         case memory_order_release:
241             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND8_RELEASE(&storage, v));
242             break;
243         case memory_order_acq_rel:
244         case memory_order_seq_cst:
245         default:
246             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND8(&storage, v));
247             break;
248         }
249         return v;
250     }
251 
fetch_orboost::atomics::detail::operations252     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
253     {
254         switch (order)
255         {
256         case memory_order_relaxed:
257             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR8_RELAXED(&storage, v));
258             break;
259         case memory_order_consume:
260         case memory_order_acquire:
261             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR8_ACQUIRE(&storage, v));
262             break;
263         case memory_order_release:
264             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR8_RELEASE(&storage, v));
265             break;
266         case memory_order_acq_rel:
267         case memory_order_seq_cst:
268         default:
269             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR8(&storage, v));
270             break;
271         }
272         return v;
273     }
274 
fetch_xorboost::atomics::detail::operations275     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
276     {
277         switch (order)
278         {
279         case memory_order_relaxed:
280             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR8_RELAXED(&storage, v));
281             break;
282         case memory_order_consume:
283         case memory_order_acquire:
284             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR8_ACQUIRE(&storage, v));
285             break;
286         case memory_order_release:
287             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR8_RELEASE(&storage, v));
288             break;
289         case memory_order_acq_rel:
290         case memory_order_seq_cst:
291         default:
292             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR8(&storage, v));
293             break;
294         }
295         return v;
296     }
297 };
298 
299 template< bool Signed >
300 struct operations< 2u, Signed > :
301     public msvc_arm_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > >
302 {
303     typedef msvc_arm_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > > base_type;
304     typedef typename base_type::storage_type storage_type;
305     typedef typename make_storage_type< 2u, Signed >::aligned aligned_storage_type;
306 
307     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
308     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
309 
storeboost::atomics::detail::operations310     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
311     {
312         base_type::fence_before_store(order);
313         BOOST_ATOMIC_DETAIL_ARM_STORE16(&storage, v);
314         base_type::fence_after_store(order);
315     }
316 
loadboost::atomics::detail::operations317     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
318     {
319         storage_type v = BOOST_ATOMIC_DETAIL_ARM_LOAD16(&storage);
320         base_type::fence_after_load(order);
321         return v;
322     }
323 
fetch_addboost::atomics::detail::operations324     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
325     {
326         switch (order)
327         {
328         case memory_order_relaxed:
329             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD16_RELAXED(&storage, v));
330             break;
331         case memory_order_consume:
332         case memory_order_acquire:
333             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD16_ACQUIRE(&storage, v));
334             break;
335         case memory_order_release:
336             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD16_RELEASE(&storage, v));
337             break;
338         case memory_order_acq_rel:
339         case memory_order_seq_cst:
340         default:
341             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD16(&storage, v));
342             break;
343         }
344         return v;
345     }
346 
exchangeboost::atomics::detail::operations347     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
348     {
349         switch (order)
350         {
351         case memory_order_relaxed:
352             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE16_RELAXED(&storage, v));
353             break;
354         case memory_order_consume:
355         case memory_order_acquire:
356             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE16_ACQUIRE(&storage, v));
357             break;
358         case memory_order_release:
359             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE16_RELEASE(&storage, v));
360             break;
361         case memory_order_acq_rel:
362         case memory_order_seq_cst:
363         default:
364             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE16(&storage, v));
365             break;
366         }
367         return v;
368     }
369 
compare_exchange_strongboost::atomics::detail::operations370     static BOOST_FORCEINLINE bool compare_exchange_strong(
371         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
372     {
373         storage_type previous = expected, old_val;
374 
375         switch (cas_common_order(success_order, failure_order))
376         {
377         case memory_order_relaxed:
378             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16_RELAXED(&storage, desired, previous));
379             break;
380         case memory_order_consume:
381         case memory_order_acquire:
382             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16_ACQUIRE(&storage, desired, previous));
383             break;
384         case memory_order_release:
385             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16_RELEASE(&storage, desired, previous));
386             break;
387         case memory_order_acq_rel:
388         case memory_order_seq_cst:
389         default:
390             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16(&storage, desired, previous));
391             break;
392         }
393         expected = old_val;
394 
395         return (previous == old_val);
396     }
397 
fetch_andboost::atomics::detail::operations398     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
399     {
400         switch (order)
401         {
402         case memory_order_relaxed:
403             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND16_RELAXED(&storage, v));
404             break;
405         case memory_order_consume:
406         case memory_order_acquire:
407             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND16_ACQUIRE(&storage, v));
408             break;
409         case memory_order_release:
410             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND16_RELEASE(&storage, v));
411             break;
412         case memory_order_acq_rel:
413         case memory_order_seq_cst:
414         default:
415             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND16(&storage, v));
416             break;
417         }
418         return v;
419     }
420 
fetch_orboost::atomics::detail::operations421     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
422     {
423         switch (order)
424         {
425         case memory_order_relaxed:
426             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR16_RELAXED(&storage, v));
427             break;
428         case memory_order_consume:
429         case memory_order_acquire:
430             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR16_ACQUIRE(&storage, v));
431             break;
432         case memory_order_release:
433             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR16_RELEASE(&storage, v));
434             break;
435         case memory_order_acq_rel:
436         case memory_order_seq_cst:
437         default:
438             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR16(&storage, v));
439             break;
440         }
441         return v;
442     }
443 
fetch_xorboost::atomics::detail::operations444     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
445     {
446         switch (order)
447         {
448         case memory_order_relaxed:
449             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR16_RELAXED(&storage, v));
450             break;
451         case memory_order_consume:
452         case memory_order_acquire:
453             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR16_ACQUIRE(&storage, v));
454             break;
455         case memory_order_release:
456             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR16_RELEASE(&storage, v));
457             break;
458         case memory_order_acq_rel:
459         case memory_order_seq_cst:
460         default:
461             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR16(&storage, v));
462             break;
463         }
464         return v;
465     }
466 };
467 
468 template< bool Signed >
469 struct operations< 4u, Signed > :
470     public msvc_arm_operations< typename make_storage_type< 4u, Signed >::type, operations< 4u, Signed > >
471 {
472     typedef msvc_arm_operations< typename make_storage_type< 4u, Signed >::type, operations< 4u, Signed > > base_type;
473     typedef typename base_type::storage_type storage_type;
474     typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type;
475 
476     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
477     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
478 
storeboost::atomics::detail::operations479     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
480     {
481         base_type::fence_before_store(order);
482         BOOST_ATOMIC_DETAIL_ARM_STORE32(&storage, v);
483         base_type::fence_after_store(order);
484     }
485 
loadboost::atomics::detail::operations486     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
487     {
488         storage_type v = BOOST_ATOMIC_DETAIL_ARM_LOAD32(&storage);
489         base_type::fence_after_load(order);
490         return v;
491     }
492 
fetch_addboost::atomics::detail::operations493     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
494     {
495         switch (order)
496         {
497         case memory_order_relaxed:
498             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD_RELAXED(&storage, v));
499             break;
500         case memory_order_consume:
501         case memory_order_acquire:
502             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD_ACQUIRE(&storage, v));
503             break;
504         case memory_order_release:
505             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD_RELEASE(&storage, v));
506             break;
507         case memory_order_acq_rel:
508         case memory_order_seq_cst:
509         default:
510             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD(&storage, v));
511             break;
512         }
513         return v;
514     }
515 
exchangeboost::atomics::detail::operations516     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
517     {
518         switch (order)
519         {
520         case memory_order_relaxed:
521             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_RELAXED(&storage, v));
522             break;
523         case memory_order_consume:
524         case memory_order_acquire:
525             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ACQUIRE(&storage, v));
526             break;
527         case memory_order_release:
528             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_RELEASE(&storage, v));
529             break;
530         case memory_order_acq_rel:
531         case memory_order_seq_cst:
532         default:
533             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&storage, v));
534             break;
535         }
536         return v;
537     }
538 
compare_exchange_strongboost::atomics::detail::operations539     static BOOST_FORCEINLINE bool compare_exchange_strong(
540         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
541     {
542         storage_type previous = expected, old_val;
543 
544         switch (cas_common_order(success_order, failure_order))
545         {
546         case memory_order_relaxed:
547             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE_RELAXED(&storage, desired, previous));
548             break;
549         case memory_order_consume:
550         case memory_order_acquire:
551             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE_ACQUIRE(&storage, desired, previous));
552             break;
553         case memory_order_release:
554             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE_RELEASE(&storage, desired, previous));
555             break;
556         case memory_order_acq_rel:
557         case memory_order_seq_cst:
558         default:
559             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE(&storage, desired, previous));
560             break;
561         }
562         expected = old_val;
563 
564         return (previous == old_val);
565     }
566 
fetch_andboost::atomics::detail::operations567     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
568     {
569         switch (order)
570         {
571         case memory_order_relaxed:
572             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND_RELAXED(&storage, v));
573             break;
574         case memory_order_consume:
575         case memory_order_acquire:
576             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND_ACQUIRE(&storage, v));
577             break;
578         case memory_order_release:
579             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND_RELEASE(&storage, v));
580             break;
581         case memory_order_acq_rel:
582         case memory_order_seq_cst:
583         default:
584             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND(&storage, v));
585             break;
586         }
587         return v;
588     }
589 
fetch_orboost::atomics::detail::operations590     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
591     {
592         switch (order)
593         {
594         case memory_order_relaxed:
595             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR_RELAXED(&storage, v));
596             break;
597         case memory_order_consume:
598         case memory_order_acquire:
599             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR_ACQUIRE(&storage, v));
600             break;
601         case memory_order_release:
602             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR_RELEASE(&storage, v));
603             break;
604         case memory_order_acq_rel:
605         case memory_order_seq_cst:
606         default:
607             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR(&storage, v));
608             break;
609         }
610         return v;
611     }
612 
fetch_xorboost::atomics::detail::operations613     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
614     {
615         switch (order)
616         {
617         case memory_order_relaxed:
618             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR_RELAXED(&storage, v));
619             break;
620         case memory_order_consume:
621         case memory_order_acquire:
622             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR_ACQUIRE(&storage, v));
623             break;
624         case memory_order_release:
625             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR_RELEASE(&storage, v));
626             break;
627         case memory_order_acq_rel:
628         case memory_order_seq_cst:
629         default:
630             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR(&storage, v));
631             break;
632         }
633         return v;
634     }
635 };
636 
637 template< bool Signed >
638 struct operations< 8u, Signed > :
639     public msvc_arm_operations< typename make_storage_type< 8u, Signed >::type, operations< 8u, Signed > >
640 {
641     typedef msvc_arm_operations< typename make_storage_type< 8u, Signed >::type, operations< 8u, Signed > > base_type;
642     typedef typename base_type::storage_type storage_type;
643     typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type;
644 
645     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
646     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
647 
storeboost::atomics::detail::operations648     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
649     {
650         base_type::fence_before_store(order);
651         BOOST_ATOMIC_DETAIL_ARM_STORE64(&storage, v);
652         base_type::fence_after_store(order);
653     }
654 
loadboost::atomics::detail::operations655     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
656     {
657         storage_type v = BOOST_ATOMIC_DETAIL_ARM_LOAD64(&storage);
658         base_type::fence_after_load(order);
659         return v;
660     }
661 
fetch_addboost::atomics::detail::operations662     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
663     {
664         switch (order)
665         {
666         case memory_order_relaxed:
667             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64_RELAXED(&storage, v));
668             break;
669         case memory_order_consume:
670         case memory_order_acquire:
671             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64_ACQUIRE(&storage, v));
672             break;
673         case memory_order_release:
674             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64_RELEASE(&storage, v));
675             break;
676         case memory_order_acq_rel:
677         case memory_order_seq_cst:
678         default:
679             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(&storage, v));
680             break;
681         }
682         return v;
683     }
684 
exchangeboost::atomics::detail::operations685     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
686     {
687         switch (order)
688         {
689         case memory_order_relaxed:
690             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64_RELAXED(&storage, v));
691             break;
692         case memory_order_consume:
693         case memory_order_acquire:
694             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64_ACQUIRE(&storage, v));
695             break;
696         case memory_order_release:
697             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64_RELEASE(&storage, v));
698             break;
699         case memory_order_acq_rel:
700         case memory_order_seq_cst:
701         default:
702             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(&storage, v));
703             break;
704         }
705         return v;
706     }
707 
compare_exchange_strongboost::atomics::detail::operations708     static BOOST_FORCEINLINE bool compare_exchange_strong(
709         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
710     {
711         storage_type previous = expected, old_val;
712 
713         switch (cas_common_order(success_order, failure_order))
714         {
715         case memory_order_relaxed:
716             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64_RELAXED(&storage, desired, previous));
717             break;
718         case memory_order_consume:
719         case memory_order_acquire:
720             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64_ACQUIRE(&storage, desired, previous));
721             break;
722         case memory_order_release:
723             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64_RELEASE(&storage, desired, previous));
724             break;
725         case memory_order_acq_rel:
726         case memory_order_seq_cst:
727         default:
728             old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(&storage, desired, previous));
729             break;
730         }
731         expected = old_val;
732 
733         return (previous == old_val);
734     }
735 
fetch_andboost::atomics::detail::operations736     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
737     {
738         switch (order)
739         {
740         case memory_order_relaxed:
741             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND64_RELAXED(&storage, v));
742             break;
743         case memory_order_consume:
744         case memory_order_acquire:
745             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND64_ACQUIRE(&storage, v));
746             break;
747         case memory_order_release:
748             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND64_RELEASE(&storage, v));
749             break;
750         case memory_order_acq_rel:
751         case memory_order_seq_cst:
752         default:
753             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND64(&storage, v));
754             break;
755         }
756         return v;
757     }
758 
fetch_orboost::atomics::detail::operations759     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
760     {
761         switch (order)
762         {
763         case memory_order_relaxed:
764             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR64_RELAXED(&storage, v));
765             break;
766         case memory_order_consume:
767         case memory_order_acquire:
768             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR64_ACQUIRE(&storage, v));
769             break;
770         case memory_order_release:
771             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR64_RELEASE(&storage, v));
772             break;
773         case memory_order_acq_rel:
774         case memory_order_seq_cst:
775         default:
776             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR64(&storage, v));
777             break;
778         }
779         return v;
780     }
781 
fetch_xorboost::atomics::detail::operations782     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
783     {
784         switch (order)
785         {
786         case memory_order_relaxed:
787             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR64_RELAXED(&storage, v));
788             break;
789         case memory_order_consume:
790         case memory_order_acquire:
791             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR64_ACQUIRE(&storage, v));
792             break;
793         case memory_order_release:
794             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR64_RELEASE(&storage, v));
795             break;
796         case memory_order_acq_rel:
797         case memory_order_seq_cst:
798         default:
799             v = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR64(&storage, v));
800             break;
801         }
802         return v;
803     }
804 };
805 
806 
thread_fence(memory_order order)807 BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
808 {
809     BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
810     if (order != memory_order_relaxed)
811         msvc_arm_operations_base::hardware_full_fence();
812     BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
813 }
814 
signal_fence(memory_order order)815 BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
816 {
817     if (order != memory_order_relaxed)
818         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
819 }
820 
821 } // namespace detail
822 } // namespace atomics
823 } // namespace boost
824 
825 #undef BOOST_ATOMIC_DETAIL_ARM_LOAD8
826 #undef BOOST_ATOMIC_DETAIL_ARM_LOAD16
827 #undef BOOST_ATOMIC_DETAIL_ARM_LOAD32
828 #undef BOOST_ATOMIC_DETAIL_ARM_LOAD64
829 #undef BOOST_ATOMIC_DETAIL_ARM_STORE8
830 #undef BOOST_ATOMIC_DETAIL_ARM_STORE16
831 #undef BOOST_ATOMIC_DETAIL_ARM_STORE32
832 #undef BOOST_ATOMIC_DETAIL_ARM_STORE64
833 
834 #endif // BOOST_ATOMIC_DETAIL_OPS_MSVC_ARM_HPP_INCLUDED_
835