1 /*
2 * Distributed under the Boost Software License, Version 1.0.
3 * (See accompanying file LICENSE_1_0.txt or copy at
4 * http://www.boost.org/LICENSE_1_0.txt)
5 *
6 * Copyright (c) 2009 Helge Bahmann
7 * Copyright (c) 2012 Tim Blechmann
8 * Copyright (c) 2014 Andrey Semashev
9 */
10 /*!
11 * \file atomic/detail/ops_gcc_x86.hpp
12 *
13 * This header contains implementation of the \c operations template.
14 */
15
16 #ifndef BOOST_ATOMIC_DETAIL_OPS_GCC_X86_HPP_INCLUDED_
17 #define BOOST_ATOMIC_DETAIL_OPS_GCC_X86_HPP_INCLUDED_
18
19 #include <boost/memory_order.hpp>
20 #include <boost/atomic/detail/config.hpp>
21 #include <boost/atomic/detail/storage_type.hpp>
22 #include <boost/atomic/detail/operations_fwd.hpp>
23 #include <boost/atomic/capabilities.hpp>
24 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B) || defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
25 #include <boost/atomic/detail/ops_gcc_x86_dcas.hpp>
26 #include <boost/atomic/detail/ops_cas_based.hpp>
27 #endif
28
29 #ifdef BOOST_HAS_PRAGMA_ONCE
30 #pragma once
31 #endif
32
33 #if defined(__x86_64__)
34 #define BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER "rdx"
35 #else
36 #define BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER "edx"
37 #endif
38
39 namespace boost {
40 namespace atomics {
41 namespace detail {
42
43 struct gcc_x86_operations_base
44 {
fence_beforeboost::atomics::detail::gcc_x86_operations_base45 static BOOST_FORCEINLINE void fence_before(memory_order order) BOOST_NOEXCEPT
46 {
47 if ((order & memory_order_release) != 0)
48 __asm__ __volatile__ ("" ::: "memory");
49 }
50
fence_afterboost::atomics::detail::gcc_x86_operations_base51 static BOOST_FORCEINLINE void fence_after(memory_order order) BOOST_NOEXCEPT
52 {
53 if ((order & memory_order_acquire) != 0)
54 __asm__ __volatile__ ("" ::: "memory");
55 }
56 };
57
58 template< typename T, typename Derived >
59 struct gcc_x86_operations :
60 public gcc_x86_operations_base
61 {
62 typedef T storage_type;
63
storeboost::atomics::detail::gcc_x86_operations64 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
65 {
66 if (order != memory_order_seq_cst)
67 {
68 fence_before(order);
69 storage = v;
70 fence_after(order);
71 }
72 else
73 {
74 Derived::exchange(storage, v, order);
75 }
76 }
77
loadboost::atomics::detail::gcc_x86_operations78 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
79 {
80 storage_type v = storage;
81 fence_after(order);
82 return v;
83 }
84
fetch_subboost::atomics::detail::gcc_x86_operations85 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
86 {
87 return Derived::fetch_add(storage, -v, order);
88 }
89
compare_exchange_weakboost::atomics::detail::gcc_x86_operations90 static BOOST_FORCEINLINE bool compare_exchange_weak(
91 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
92 {
93 return Derived::compare_exchange_strong(storage, expected, desired, success_order, failure_order);
94 }
95
test_and_setboost::atomics::detail::gcc_x86_operations96 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
97 {
98 return !!Derived::exchange(storage, (storage_type)1, order);
99 }
100
clearboost::atomics::detail::gcc_x86_operations101 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
102 {
103 store(storage, (storage_type)0, order);
104 }
105
is_lock_freeboost::atomics::detail::gcc_x86_operations106 static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
107 {
108 return true;
109 }
110 };
111
112 template< bool Signed >
113 struct operations< 1u, Signed > :
114 public gcc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > >
115 {
116 typedef gcc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > > base_type;
117 typedef typename base_type::storage_type storage_type;
118
fetch_addboost::atomics::detail::operations119 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
120 {
121 __asm__ __volatile__
122 (
123 "lock; xaddb %0, %1"
124 : "+q" (v), "+m" (storage)
125 :
126 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
127 );
128 return v;
129 }
130
exchangeboost::atomics::detail::operations131 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
132 {
133 __asm__ __volatile__
134 (
135 "xchgb %0, %1"
136 : "+q" (v), "+m" (storage)
137 :
138 : "memory"
139 );
140 return v;
141 }
142
compare_exchange_strongboost::atomics::detail::operations143 static BOOST_FORCEINLINE bool compare_exchange_strong(
144 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
145 {
146 storage_type previous = expected;
147 bool success;
148 __asm__ __volatile__
149 (
150 "lock; cmpxchgb %3, %1\n\t"
151 "sete %2"
152 : "+a" (previous), "+m" (storage), "=q" (success)
153 : "q" (desired)
154 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
155 );
156 expected = previous;
157 return success;
158 }
159
160 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, result)\
161 __asm__ __volatile__\
162 (\
163 "xor %%" BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER ", %%" BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER "\n\t"\
164 ".align 16\n\t"\
165 "1: movb %[arg], %%dl\n\t"\
166 op " %%al, %%dl\n\t"\
167 "lock; cmpxchgb %%dl, %[storage]\n\t"\
168 "jne 1b"\
169 : [res] "+a" (result), [storage] "+m" (storage)\
170 : [arg] "q" (argument)\
171 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER, "memory"\
172 )
173
fetch_andboost::atomics::detail::operations174 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
175 {
176 storage_type res = storage;
177 BOOST_ATOMIC_DETAIL_CAS_LOOP("andb", v, res);
178 return res;
179 }
180
fetch_orboost::atomics::detail::operations181 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
182 {
183 storage_type res = storage;
184 BOOST_ATOMIC_DETAIL_CAS_LOOP("orb", v, res);
185 return res;
186 }
187
fetch_xorboost::atomics::detail::operations188 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
189 {
190 storage_type res = storage;
191 BOOST_ATOMIC_DETAIL_CAS_LOOP("xorb", v, res);
192 return res;
193 }
194
195 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
196 };
197
198 template< bool Signed >
199 struct operations< 2u, Signed > :
200 public gcc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > >
201 {
202 typedef gcc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > > base_type;
203 typedef typename base_type::storage_type storage_type;
204
fetch_addboost::atomics::detail::operations205 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
206 {
207 __asm__ __volatile__
208 (
209 "lock; xaddw %0, %1"
210 : "+q" (v), "+m" (storage)
211 :
212 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
213 );
214 return v;
215 }
216
exchangeboost::atomics::detail::operations217 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
218 {
219 __asm__ __volatile__
220 (
221 "xchgw %0, %1"
222 : "+q" (v), "+m" (storage)
223 :
224 : "memory"
225 );
226 return v;
227 }
228
compare_exchange_strongboost::atomics::detail::operations229 static BOOST_FORCEINLINE bool compare_exchange_strong(
230 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
231 {
232 storage_type previous = expected;
233 bool success;
234 __asm__ __volatile__
235 (
236 "lock; cmpxchgw %3, %1\n\t"
237 "sete %2"
238 : "+a" (previous), "+m" (storage), "=q" (success)
239 : "q" (desired)
240 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
241 );
242 expected = previous;
243 return success;
244 }
245
246 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, result)\
247 __asm__ __volatile__\
248 (\
249 "xor %%" BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER ", %%" BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER "\n\t"\
250 ".align 16\n\t"\
251 "1: movw %[arg], %%dx\n\t"\
252 op " %%ax, %%dx\n\t"\
253 "lock; cmpxchgw %%dx, %[storage]\n\t"\
254 "jne 1b"\
255 : [res] "+a" (result), [storage] "+m" (storage)\
256 : [arg] "q" (argument)\
257 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER, "memory"\
258 )
259
fetch_andboost::atomics::detail::operations260 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
261 {
262 storage_type res = storage;
263 BOOST_ATOMIC_DETAIL_CAS_LOOP("andw", v, res);
264 return res;
265 }
266
fetch_orboost::atomics::detail::operations267 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
268 {
269 storage_type res = storage;
270 BOOST_ATOMIC_DETAIL_CAS_LOOP("orw", v, res);
271 return res;
272 }
273
fetch_xorboost::atomics::detail::operations274 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
275 {
276 storage_type res = storage;
277 BOOST_ATOMIC_DETAIL_CAS_LOOP("xorw", v, res);
278 return res;
279 }
280
281 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
282 };
283
284 template< bool Signed >
285 struct operations< 4u, Signed > :
286 public gcc_x86_operations< typename make_storage_type< 4u, Signed >::type, operations< 4u, Signed > >
287 {
288 typedef gcc_x86_operations< typename make_storage_type< 4u, Signed >::type, operations< 4u, Signed > > base_type;
289 typedef typename base_type::storage_type storage_type;
290
fetch_addboost::atomics::detail::operations291 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
292 {
293 __asm__ __volatile__
294 (
295 "lock; xaddl %0, %1"
296 : "+r" (v), "+m" (storage)
297 :
298 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
299 );
300 return v;
301 }
302
exchangeboost::atomics::detail::operations303 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
304 {
305 __asm__ __volatile__
306 (
307 "xchgl %0, %1"
308 : "+r" (v), "+m" (storage)
309 :
310 : "memory"
311 );
312 return v;
313 }
314
compare_exchange_strongboost::atomics::detail::operations315 static BOOST_FORCEINLINE bool compare_exchange_strong(
316 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
317 {
318 storage_type previous = expected;
319 bool success;
320 __asm__ __volatile__
321 (
322 "lock; cmpxchgl %3, %1\n\t"
323 "sete %2"
324 : "+a" (previous), "+m" (storage), "=q" (success)
325 : "r" (desired)
326 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
327 );
328 expected = previous;
329 return success;
330 }
331
332 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, result)\
333 __asm__ __volatile__\
334 (\
335 "xor %%" BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER ", %%" BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER "\n\t"\
336 ".align 16\n\t"\
337 "1: movl %[arg], %%edx\n\t"\
338 op " %%eax, %%edx\n\t"\
339 "lock; cmpxchgl %%edx, %[storage]\n\t"\
340 "jne 1b"\
341 : [res] "+a" (result), [storage] "+m" (storage)\
342 : [arg] "r" (argument)\
343 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER, "memory"\
344 )
345
fetch_andboost::atomics::detail::operations346 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
347 {
348 storage_type res = storage;
349 BOOST_ATOMIC_DETAIL_CAS_LOOP("andl", v, res);
350 return res;
351 }
352
fetch_orboost::atomics::detail::operations353 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
354 {
355 storage_type res = storage;
356 BOOST_ATOMIC_DETAIL_CAS_LOOP("orl", v, res);
357 return res;
358 }
359
fetch_xorboost::atomics::detail::operations360 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
361 {
362 storage_type res = storage;
363 BOOST_ATOMIC_DETAIL_CAS_LOOP("xorl", v, res);
364 return res;
365 }
366
367 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
368 };
369
370 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B)
371
372 template< bool Signed >
373 struct operations< 8u, Signed > :
374 public cas_based_operations< gcc_dcas_x86< Signed > >
375 {
376 };
377
378 #elif defined(__x86_64__)
379
380 template< bool Signed >
381 struct operations< 8u, Signed > :
382 public gcc_x86_operations< typename make_storage_type< 8u, Signed >::type, operations< 8u, Signed > >
383 {
384 typedef gcc_x86_operations< typename make_storage_type< 8u, Signed >::type, operations< 8u, Signed > > base_type;
385 typedef typename base_type::storage_type storage_type;
386
fetch_addboost::atomics::detail::operations387 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
388 {
389 __asm__ __volatile__
390 (
391 "lock; xaddq %0, %1"
392 : "+r" (v), "+m" (storage)
393 :
394 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
395 );
396 return v;
397 }
398
exchangeboost::atomics::detail::operations399 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
400 {
401 __asm__ __volatile__
402 (
403 "xchgq %0, %1"
404 : "+r" (v), "+m" (storage)
405 :
406 : "memory"
407 );
408 return v;
409 }
410
compare_exchange_strongboost::atomics::detail::operations411 static BOOST_FORCEINLINE bool compare_exchange_strong(
412 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
413 {
414 storage_type previous = expected;
415 bool success;
416 __asm__ __volatile__
417 (
418 "lock; cmpxchgq %3, %1\n\t"
419 "sete %2"
420 : "+a" (previous), "+m" (storage), "=q" (success)
421 : "r" (desired)
422 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
423 );
424 expected = previous;
425 return success;
426 }
427
428 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, result)\
429 __asm__ __volatile__\
430 (\
431 "xor %%" BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER ", %%" BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER "\n\t"\
432 ".align 16\n\t"\
433 "1: movq %[arg], %%rdx\n\t"\
434 op " %%rax, %%rdx\n\t"\
435 "lock; cmpxchgq %%rdx, %[storage]\n\t"\
436 "jne 1b"\
437 : [res] "+a" (result), [storage] "+m" (storage)\
438 : [arg] "r" (argument)\
439 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER, "memory"\
440 )
441
fetch_andboost::atomics::detail::operations442 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
443 {
444 storage_type res = storage;
445 BOOST_ATOMIC_DETAIL_CAS_LOOP("andq", v, res);
446 return res;
447 }
448
fetch_orboost::atomics::detail::operations449 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
450 {
451 storage_type res = storage;
452 BOOST_ATOMIC_DETAIL_CAS_LOOP("orq", v, res);
453 return res;
454 }
455
fetch_xorboost::atomics::detail::operations456 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
457 {
458 storage_type res = storage;
459 BOOST_ATOMIC_DETAIL_CAS_LOOP("xorq", v, res);
460 return res;
461 }
462
463 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
464 };
465
466 #endif
467
468 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
469
470 template< bool Signed >
471 struct operations< 16u, Signed > :
472 public cas_based_operations< gcc_dcas_x86_64< Signed > >
473 {
474 };
475
476 #endif // defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
477
thread_fence(memory_order order)478 BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
479 {
480 if (order == memory_order_seq_cst)
481 {
482 __asm__ __volatile__
483 (
484 #if defined(__x86_64__) || defined(__SSE2__)
485 "mfence\n"
486 #else
487 "lock; addl $0, (%%esp)\n"
488 #endif
489 ::: "memory"
490 );
491 }
492 else if ((order & (memory_order_acquire | memory_order_release)) != 0)
493 {
494 __asm__ __volatile__ ("" ::: "memory");
495 }
496 }
497
signal_fence(memory_order order)498 BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
499 {
500 if (order != memory_order_relaxed)
501 __asm__ __volatile__ ("" ::: "memory");
502 }
503
504 } // namespace detail
505 } // namespace atomics
506 } // namespace boost
507
508 #undef BOOST_ATOMIC_DETAIL_TEMP_CAS_REGISTER
509
510 #endif // BOOST_ATOMIC_DETAIL_OPS_GCC_X86_HPP_INCLUDED_
511