1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
2 //
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
5 //
6 //===----------------------------------------------------------------------===//
7 //
8 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
9 // Not intended for direct inclusion. Include sanitizer_atomic.h.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #ifndef SANITIZER_ATOMIC_MSVC_H
14 #define SANITIZER_ATOMIC_MSVC_H
15
16 extern "C" void _ReadWriteBarrier();
17 #pragma intrinsic(_ReadWriteBarrier)
18 extern "C" void _mm_mfence();
19 #pragma intrinsic(_mm_mfence)
20 extern "C" void _mm_pause();
21 #pragma intrinsic(_mm_pause)
22 extern "C" char _InterlockedExchange8( // NOLINT
23 char volatile *Addend, char Value); // NOLINT
24 #pragma intrinsic(_InterlockedExchange8)
25 extern "C" short _InterlockedExchange16( // NOLINT
26 short volatile *Addend, short Value); // NOLINT
27 #pragma intrinsic(_InterlockedExchange16)
28 extern "C" long _InterlockedExchange( // NOLINT
29 long volatile *Addend, long Value); // NOLINT
30 #pragma intrinsic(_InterlockedExchange)
31 extern "C" long _InterlockedExchangeAdd( // NOLINT
32 long volatile * Addend, long Value); // NOLINT
33 #pragma intrinsic(_InterlockedExchangeAdd)
34 extern "C" char _InterlockedCompareExchange8( // NOLINT
35 char volatile *Destination, // NOLINT
36 char Exchange, char Comparand); // NOLINT
37 #pragma intrinsic(_InterlockedCompareExchange8)
38 extern "C" short _InterlockedCompareExchange16( // NOLINT
39 short volatile *Destination, // NOLINT
40 short Exchange, short Comparand); // NOLINT
41 #pragma intrinsic(_InterlockedCompareExchange16)
42 extern "C"
43 long long _InterlockedCompareExchange64( // NOLINT
44 long long volatile *Destination, // NOLINT
45 long long Exchange, long long Comparand); // NOLINT
46 #pragma intrinsic(_InterlockedCompareExchange64)
47 extern "C" void *_InterlockedCompareExchangePointer(
48 void *volatile *Destination,
49 void *Exchange, void *Comparand);
50 #pragma intrinsic(_InterlockedCompareExchangePointer)
51 extern "C"
52 long __cdecl _InterlockedCompareExchange( // NOLINT
53 long volatile *Destination, // NOLINT
54 long Exchange, long Comparand); // NOLINT
55 #pragma intrinsic(_InterlockedCompareExchange)
56
57 #ifdef _WIN64
58 extern "C" long long _InterlockedExchangeAdd64( // NOLINT
59 long long volatile * Addend, long long Value); // NOLINT
60 #pragma intrinsic(_InterlockedExchangeAdd64)
61 #endif
62
63 namespace __sanitizer {
64
atomic_signal_fence(memory_order)65 INLINE void atomic_signal_fence(memory_order) {
66 _ReadWriteBarrier();
67 }
68
atomic_thread_fence(memory_order)69 INLINE void atomic_thread_fence(memory_order) {
70 _mm_mfence();
71 }
72
proc_yield(int cnt)73 INLINE void proc_yield(int cnt) {
74 for (int i = 0; i < cnt; i++)
75 _mm_pause();
76 }
77
78 template<typename T>
atomic_load(const volatile T * a,memory_order mo)79 INLINE typename T::Type atomic_load(
80 const volatile T *a, memory_order mo) {
81 DCHECK(mo & (memory_order_relaxed | memory_order_consume
82 | memory_order_acquire | memory_order_seq_cst));
83 DCHECK(!((uptr)a % sizeof(*a)));
84 typename T::Type v;
85 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
86 if (mo == memory_order_relaxed) {
87 v = a->val_dont_use;
88 } else {
89 atomic_signal_fence(memory_order_seq_cst);
90 v = a->val_dont_use;
91 atomic_signal_fence(memory_order_seq_cst);
92 }
93 return v;
94 }
95
96 template<typename T>
atomic_store(volatile T * a,typename T::Type v,memory_order mo)97 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
98 DCHECK(mo & (memory_order_relaxed | memory_order_release
99 | memory_order_seq_cst));
100 DCHECK(!((uptr)a % sizeof(*a)));
101 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
102 if (mo == memory_order_relaxed) {
103 a->val_dont_use = v;
104 } else {
105 atomic_signal_fence(memory_order_seq_cst);
106 a->val_dont_use = v;
107 atomic_signal_fence(memory_order_seq_cst);
108 }
109 if (mo == memory_order_seq_cst)
110 atomic_thread_fence(memory_order_seq_cst);
111 }
112
atomic_fetch_add(volatile atomic_uint32_t * a,u32 v,memory_order mo)113 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
114 u32 v, memory_order mo) {
115 (void)mo;
116 DCHECK(!((uptr)a % sizeof(*a)));
117 return (u32)_InterlockedExchangeAdd(
118 (volatile long*)&a->val_dont_use, (long)v); // NOLINT
119 }
120
atomic_fetch_add(volatile atomic_uintptr_t * a,uptr v,memory_order mo)121 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
122 uptr v, memory_order mo) {
123 (void)mo;
124 DCHECK(!((uptr)a % sizeof(*a)));
125 #ifdef _WIN64
126 return (uptr)_InterlockedExchangeAdd64(
127 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT
128 #else
129 return (uptr)_InterlockedExchangeAdd(
130 (volatile long*)&a->val_dont_use, (long)v); // NOLINT
131 #endif
132 }
133
atomic_fetch_sub(volatile atomic_uint32_t * a,u32 v,memory_order mo)134 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
135 u32 v, memory_order mo) {
136 (void)mo;
137 DCHECK(!((uptr)a % sizeof(*a)));
138 return (u32)_InterlockedExchangeAdd(
139 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT
140 }
141
atomic_fetch_sub(volatile atomic_uintptr_t * a,uptr v,memory_order mo)142 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
143 uptr v, memory_order mo) {
144 (void)mo;
145 DCHECK(!((uptr)a % sizeof(*a)));
146 #ifdef _WIN64
147 return (uptr)_InterlockedExchangeAdd64(
148 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT
149 #else
150 return (uptr)_InterlockedExchangeAdd(
151 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT
152 #endif
153 }
154
atomic_exchange(volatile atomic_uint8_t * a,u8 v,memory_order mo)155 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
156 u8 v, memory_order mo) {
157 (void)mo;
158 DCHECK(!((uptr)a % sizeof(*a)));
159 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
160 }
161
atomic_exchange(volatile atomic_uint16_t * a,u16 v,memory_order mo)162 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
163 u16 v, memory_order mo) {
164 (void)mo;
165 DCHECK(!((uptr)a % sizeof(*a)));
166 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
167 }
168
atomic_exchange(volatile atomic_uint32_t * a,u32 v,memory_order mo)169 INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
170 u32 v, memory_order mo) {
171 (void)mo;
172 DCHECK(!((uptr)a % sizeof(*a)));
173 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
174 }
175
atomic_compare_exchange_strong(volatile atomic_uint8_t * a,u8 * cmp,u8 xchgv,memory_order mo)176 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
177 u8 *cmp,
178 u8 xchgv,
179 memory_order mo) {
180 (void)mo;
181 DCHECK(!((uptr)a % sizeof(*a)));
182 u8 cmpv = *cmp;
183 #ifdef _WIN64
184 u8 prev = (u8)_InterlockedCompareExchange8(
185 (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
186 #else
187 u8 prev;
188 __asm {
189 mov al, cmpv
190 mov ecx, a
191 mov dl, xchgv
192 lock cmpxchg [ecx], dl
193 mov prev, al
194 }
195 #endif
196 if (prev == cmpv)
197 return true;
198 *cmp = prev;
199 return false;
200 }
201
atomic_compare_exchange_strong(volatile atomic_uintptr_t * a,uptr * cmp,uptr xchg,memory_order mo)202 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
203 uptr *cmp,
204 uptr xchg,
205 memory_order mo) {
206 uptr cmpv = *cmp;
207 uptr prev = (uptr)_InterlockedCompareExchangePointer(
208 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
209 if (prev == cmpv)
210 return true;
211 *cmp = prev;
212 return false;
213 }
214
atomic_compare_exchange_strong(volatile atomic_uint16_t * a,u16 * cmp,u16 xchg,memory_order mo)215 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
216 u16 *cmp,
217 u16 xchg,
218 memory_order mo) {
219 u16 cmpv = *cmp;
220 u16 prev = (u16)_InterlockedCompareExchange16(
221 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
222 if (prev == cmpv)
223 return true;
224 *cmp = prev;
225 return false;
226 }
227
atomic_compare_exchange_strong(volatile atomic_uint32_t * a,u32 * cmp,u32 xchg,memory_order mo)228 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
229 u32 *cmp,
230 u32 xchg,
231 memory_order mo) {
232 u32 cmpv = *cmp;
233 u32 prev = (u32)_InterlockedCompareExchange(
234 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
235 if (prev == cmpv)
236 return true;
237 *cmp = prev;
238 return false;
239 }
240
atomic_compare_exchange_strong(volatile atomic_uint64_t * a,u64 * cmp,u64 xchg,memory_order mo)241 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
242 u64 *cmp,
243 u64 xchg,
244 memory_order mo) {
245 u64 cmpv = *cmp;
246 u64 prev = (u64)_InterlockedCompareExchange64(
247 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
248 if (prev == cmpv)
249 return true;
250 *cmp = prev;
251 return false;
252 }
253
254 template<typename T>
atomic_compare_exchange_weak(volatile T * a,typename T::Type * cmp,typename T::Type xchg,memory_order mo)255 INLINE bool atomic_compare_exchange_weak(volatile T *a,
256 typename T::Type *cmp,
257 typename T::Type xchg,
258 memory_order mo) {
259 return atomic_compare_exchange_strong(a, cmp, xchg, mo);
260 }
261
262 } // namespace __sanitizer
263
264 #endif // SANITIZER_ATOMIC_CLANG_H
265