1 #ifndef Py_ATOMIC_H
2 #define Py_ATOMIC_H
3 #ifdef Py_BUILD_CORE
4 
5 #include "dynamic_annotations.h"
6 
7 #include "pyconfig.h"
8 
9 #if defined(HAVE_STD_ATOMIC)
10 #include <stdatomic.h>
11 #endif
12 
13 /* This is modeled after the atomics interface from C1x, according to
14  * the draft at
15  * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf.
16  * Operations and types are named the same except with a _Py_ prefix
17  * and have the same semantics.
18  *
19  * Beware, the implementations here are deep magic.
20  */
21 
22 #if defined(HAVE_STD_ATOMIC)
23 
24 typedef enum _Py_memory_order {
25     _Py_memory_order_relaxed = memory_order_relaxed,
26     _Py_memory_order_acquire = memory_order_acquire,
27     _Py_memory_order_release = memory_order_release,
28     _Py_memory_order_acq_rel = memory_order_acq_rel,
29     _Py_memory_order_seq_cst = memory_order_seq_cst
30 } _Py_memory_order;
31 
32 typedef struct _Py_atomic_address {
33     _Atomic void *_value;
34 } _Py_atomic_address;
35 
36 typedef struct _Py_atomic_int {
37     atomic_int _value;
38 } _Py_atomic_int;
39 
40 #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
41     atomic_signal_fence(ORDER)
42 
43 #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
44     atomic_thread_fence(ORDER)
45 
46 #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
47     atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)
48 
49 #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
50     atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER)
51 
52 /* Use builtin atomic operations in GCC >= 4.7 */
53 #elif defined(HAVE_BUILTIN_ATOMIC)
54 
55 typedef enum _Py_memory_order {
56     _Py_memory_order_relaxed = __ATOMIC_RELAXED,
57     _Py_memory_order_acquire = __ATOMIC_ACQUIRE,
58     _Py_memory_order_release = __ATOMIC_RELEASE,
59     _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL,
60     _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST
61 } _Py_memory_order;
62 
63 typedef struct _Py_atomic_address {
64     void *_value;
65 } _Py_atomic_address;
66 
67 typedef struct _Py_atomic_int {
68     int _value;
69 } _Py_atomic_int;
70 
71 #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
72     __atomic_signal_fence(ORDER)
73 
74 #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
75     __atomic_thread_fence(ORDER)
76 
77 #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
78     (assert((ORDER) == __ATOMIC_RELAXED                       \
79             || (ORDER) == __ATOMIC_SEQ_CST                    \
80             || (ORDER) == __ATOMIC_RELEASE),                  \
81      __atomic_store_n(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER))
82 
83 #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER)           \
84     (assert((ORDER) == __ATOMIC_RELAXED                       \
85             || (ORDER) == __ATOMIC_SEQ_CST                    \
86             || (ORDER) == __ATOMIC_ACQUIRE                    \
87             || (ORDER) == __ATOMIC_CONSUME),                  \
88      __atomic_load_n(&(ATOMIC_VAL)->_value, ORDER))
89 
90 #else
91 
92 typedef enum _Py_memory_order {
93     _Py_memory_order_relaxed,
94     _Py_memory_order_acquire,
95     _Py_memory_order_release,
96     _Py_memory_order_acq_rel,
97     _Py_memory_order_seq_cst
98 } _Py_memory_order;
99 
100 typedef struct _Py_atomic_address {
101     void *_value;
102 } _Py_atomic_address;
103 
104 typedef struct _Py_atomic_int {
105     int _value;
106 } _Py_atomic_int;
107 
108 /* Only support GCC (for expression statements) and x86 (for simple
109  * atomic semantics) for now */
110 #if defined(__GNUC__) && (defined(__i386__) || defined(__amd64))
111 
112 static __inline__ void
_Py_atomic_signal_fence(_Py_memory_order order)113 _Py_atomic_signal_fence(_Py_memory_order order)
114 {
115     if (order != _Py_memory_order_relaxed)
116         __asm__ volatile("":::"memory");
117 }
118 
119 static __inline__ void
_Py_atomic_thread_fence(_Py_memory_order order)120 _Py_atomic_thread_fence(_Py_memory_order order)
121 {
122     if (order != _Py_memory_order_relaxed)
123         __asm__ volatile("mfence":::"memory");
124 }
125 
126 /* Tell the race checker about this operation's effects. */
127 static __inline__ void
_Py_ANNOTATE_MEMORY_ORDER(const volatile void * address,_Py_memory_order order)128 _Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order)
129 {
130     (void)address;		/* shut up -Wunused-parameter */
131     switch(order) {
132     case _Py_memory_order_release:
133     case _Py_memory_order_acq_rel:
134     case _Py_memory_order_seq_cst:
135         _Py_ANNOTATE_HAPPENS_BEFORE(address);
136         break;
137     case _Py_memory_order_relaxed:
138     case _Py_memory_order_acquire:
139         break;
140     }
141     switch(order) {
142     case _Py_memory_order_acquire:
143     case _Py_memory_order_acq_rel:
144     case _Py_memory_order_seq_cst:
145         _Py_ANNOTATE_HAPPENS_AFTER(address);
146         break;
147     case _Py_memory_order_relaxed:
148     case _Py_memory_order_release:
149         break;
150     }
151 }
152 
153 #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
154     __extension__ ({ \
155         __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
156         __typeof__(atomic_val->_value) new_val = NEW_VAL;\
157         volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \
158         _Py_memory_order order = ORDER; \
159         _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
160         \
161         /* Perform the operation. */ \
162         _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \
163         switch(order) { \
164         case _Py_memory_order_release: \
165             _Py_atomic_signal_fence(_Py_memory_order_release); \
166             /* fallthrough */ \
167         case _Py_memory_order_relaxed: \
168             *volatile_data = new_val; \
169             break; \
170         \
171         case _Py_memory_order_acquire: \
172         case _Py_memory_order_acq_rel: \
173         case _Py_memory_order_seq_cst: \
174             __asm__ volatile("xchg %0, %1" \
175                          : "+r"(new_val) \
176                          : "m"(atomic_val->_value) \
177                          : "memory"); \
178             break; \
179         } \
180         _Py_ANNOTATE_IGNORE_WRITES_END(); \
181     })
182 
183 #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
184     __extension__ ({  \
185         __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
186         __typeof__(atomic_val->_value) result; \
187         volatile __typeof__(result) *volatile_data = &atomic_val->_value; \
188         _Py_memory_order order = ORDER; \
189         _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
190         \
191         /* Perform the operation. */ \
192         _Py_ANNOTATE_IGNORE_READS_BEGIN(); \
193         switch(order) { \
194         case _Py_memory_order_release: \
195         case _Py_memory_order_acq_rel: \
196         case _Py_memory_order_seq_cst: \
197             /* Loads on x86 are not releases by default, so need a */ \
198             /* thread fence. */ \
199             _Py_atomic_thread_fence(_Py_memory_order_release); \
200             break; \
201         default: \
202             /* No fence */ \
203             break; \
204         } \
205         result = *volatile_data; \
206         switch(order) { \
207         case _Py_memory_order_acquire: \
208         case _Py_memory_order_acq_rel: \
209         case _Py_memory_order_seq_cst: \
210             /* Loads on x86 are automatically acquire operations so */ \
211             /* can get by with just a compiler fence. */ \
212             _Py_atomic_signal_fence(_Py_memory_order_acquire); \
213             break; \
214         default: \
215             /* No fence */ \
216             break; \
217         } \
218         _Py_ANNOTATE_IGNORE_READS_END(); \
219         result; \
220     })
221 
222 #else  /* !gcc x86 */
223 /* Fall back to other compilers and processors by assuming that simple
224    volatile accesses are atomic.  This is false, so people should port
225    this. */
226 #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0)
227 #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0)
228 #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
229     ((ATOMIC_VAL)->_value = NEW_VAL)
230 #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
231     ((ATOMIC_VAL)->_value)
232 
233 #endif  /* !gcc x86 */
234 #endif
235 
236 /* Standardized shortcuts. */
237 #define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \
238     _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_seq_cst)
239 #define _Py_atomic_load(ATOMIC_VAL) \
240     _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_seq_cst)
241 
242 /* Python-local extensions */
243 
244 #define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \
245     _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_relaxed)
246 #define _Py_atomic_load_relaxed(ATOMIC_VAL) \
247     _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed)
248 
249 #endif  /* Py_BUILD_CORE */
250 #endif  /* Py_ATOMIC_H */
251