1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=8 sts=4 et sw=4 tw=99:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 /* For documentation, see jit/AtomicOperations.h, both the comment block at the
8 * beginning and the #ifdef nest near the end.
9 *
10 * This is a common file for tier-3 platforms that are not providing
11 * hardware-specific implementations of the atomic operations. Please keep it
12 * reasonably platform-independent by adding #ifdefs at the beginning as much as
13 * possible, not throughout the file.
14 *
15 *
16 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
17 * !!!! NOTE !!!!
18 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
19 *
20 * The implementations in this file are NOT SAFE and cannot be safe even in
21 * principle because they rely on C++ undefined behavior. However, they are
22 * frequently good enough for tier-3 platforms.
23 */
24
25 #ifndef jit_none_AtomicOperations_feeling_lucky_h
26 #define jit_none_AtomicOperations_feeling_lucky_h
27
28 #include "mozilla/Assertions.h"
29 #include "mozilla/Types.h"
30
31 // 64-bit atomics are not required by the JS spec, and you can compile
32 // SpiderMonkey without them.
33 //
34 // 64-bit lock-free atomics are however required for WebAssembly, and
35 // WebAssembly will be disabled if you do not define both HAS_64BIT_ATOMICS and
36 // HAS_64BIT_LOCKFREE.
37 //
38 // If you are only able to provide 64-bit non-lock-free atomics and you really
39 // want WebAssembly support you can always just lie about the lock-freedom.
40 // After all, you're already feeling lucky.
41
42 #if defined(__ppc__) || defined(__PPC__)
43 #define GNUC_COMPATIBLE
44 #endif
45
46 #if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \
47 defined(__PPC64LE__)
48 #define HAS_64BIT_ATOMICS
49 #define HAS_64BIT_LOCKFREE
50 #define GNUC_COMPATIBLE
51 #endif
52
53 #ifdef __sparc__
54 #define GNUC_COMPATIBLE
55 #ifdef __LP64__
56 #define HAS_64BIT_ATOMICS
57 #define HAS_64BIT_LOCKFREE
58 #endif
59 #endif
60
61 #ifdef __alpha__
62 #define GNUC_COMPATIBLE
63 #endif
64
65 #ifdef __hppa__
66 #define GNUC_COMPATIBLE
67 #endif
68
69 #ifdef __sh__
70 #define GNUC_COMPATIBLE
71 #endif
72
73 #ifdef __s390__
74 #define GNUC_COMPATIBLE
75 #endif
76
77 #ifdef __s390x__
78 #define HAS_64BIT_ATOMICS
79 #define HAS_64BIT_LOCKFREE
80 #define GNUC_COMPATIBLE
81 #endif
82
83 // The default implementation tactic for gcc/clang is to use the newer
84 // __atomic intrinsics added for use in C++11 <atomic>. Where that
85 // isn't available, we use GCC's older __sync functions instead.
86 //
87 // ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS is kept as a backward
88 // compatible option for older compilers: enable this to use GCC's old
89 // __sync functions instead of the newer __atomic functions. This
90 // will be required for GCC 4.6.x and earlier, and probably for Clang
91 // 3.1, should we need to use those versions.
92
93 //#define ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
94
95 // Sanity check.
96
97 #if defined(HAS_64BIT_LOCKFREE) && !defined(HAS_64BIT_ATOMICS)
98 #error "This combination of features is senseless, please fix"
99 #endif
100
101 // Try to avoid platform #ifdefs below this point.
102
103 #ifdef GNUC_COMPATIBLE
104
hasAtomic8()105 inline bool js::jit::AtomicOperations::hasAtomic8() {
106 #if defined(HAS_64BIT_ATOMICS)
107 return true;
108 #else
109 return false;
110 #endif
111 }
112
isLockfree8()113 inline bool js::jit::AtomicOperations::isLockfree8() {
114 #if defined(HAS_64BIT_LOCKFREE)
115 return true;
116 #else
117 return false;
118 #endif
119 }
120
fenceSeqCst()121 inline void js::jit::AtomicOperations::fenceSeqCst() {
122 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
123 __sync_synchronize();
124 #else
125 __atomic_thread_fence(__ATOMIC_SEQ_CST);
126 #endif
127 }
128
129 template <typename T>
loadSeqCst(T * addr)130 inline T js::jit::AtomicOperations::loadSeqCst(T* addr) {
131 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
132 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
133 __sync_synchronize();
134 T v = *addr;
135 __sync_synchronize();
136 #else
137 T v;
138 __atomic_load(addr, &v, __ATOMIC_SEQ_CST);
139 #endif
140 return v;
141 }
142
143 #ifndef HAS_64BIT_ATOMICS
144 namespace js {
145 namespace jit {
146
147 template <>
loadSeqCst(int64_t * addr)148 inline int64_t AtomicOperations::loadSeqCst(int64_t* addr) {
149 MOZ_CRASH("No 64-bit atomics");
150 }
151
152 template <>
loadSeqCst(uint64_t * addr)153 inline uint64_t AtomicOperations::loadSeqCst(uint64_t* addr) {
154 MOZ_CRASH("No 64-bit atomics");
155 }
156
157 } // namespace jit
158 } // namespace js
159 #endif
160
161 template <typename T>
storeSeqCst(T * addr,T val)162 inline void js::jit::AtomicOperations::storeSeqCst(T* addr, T val) {
163 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
164 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
165 __sync_synchronize();
166 *addr = val;
167 __sync_synchronize();
168 #else
169 __atomic_store(addr, &val, __ATOMIC_SEQ_CST);
170 #endif
171 }
172
173 #ifndef HAS_64BIT_ATOMICS
174 namespace js {
175 namespace jit {
176
177 template <>
storeSeqCst(int64_t * addr,int64_t val)178 inline void AtomicOperations::storeSeqCst(int64_t* addr, int64_t val) {
179 MOZ_CRASH("No 64-bit atomics");
180 }
181
182 template <>
storeSeqCst(uint64_t * addr,uint64_t val)183 inline void AtomicOperations::storeSeqCst(uint64_t* addr, uint64_t val) {
184 MOZ_CRASH("No 64-bit atomics");
185 }
186
187 } // namespace jit
188 } // namespace js
189 #endif
190
191 template <typename T>
compareExchangeSeqCst(T * addr,T oldval,T newval)192 inline T js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval,
193 T newval) {
194 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
195 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
196 return __sync_val_compare_and_swap(addr, oldval, newval);
197 #else
198 __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST,
199 __ATOMIC_SEQ_CST);
200 return oldval;
201 #endif
202 }
203
204 #ifndef HAS_64BIT_ATOMICS
205 namespace js {
206 namespace jit {
207
208 template <>
compareExchangeSeqCst(int64_t * addr,int64_t oldval,int64_t newval)209 inline int64_t AtomicOperations::compareExchangeSeqCst(int64_t* addr,
210 int64_t oldval,
211 int64_t newval) {
212 MOZ_CRASH("No 64-bit atomics");
213 }
214
215 template <>
compareExchangeSeqCst(uint64_t * addr,uint64_t oldval,uint64_t newval)216 inline uint64_t AtomicOperations::compareExchangeSeqCst(uint64_t* addr,
217 uint64_t oldval,
218 uint64_t newval) {
219 MOZ_CRASH("No 64-bit atomics");
220 }
221
222 } // namespace jit
223 } // namespace js
224 #endif
225
226 template <typename T>
fetchAddSeqCst(T * addr,T val)227 inline T js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val) {
228 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
229 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
230 return __sync_fetch_and_add(addr, val);
231 #else
232 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
233 #endif
234 }
235
236 #ifndef HAS_64BIT_ATOMICS
237 namespace js {
238 namespace jit {
239
240 template <>
fetchAddSeqCst(int64_t * addr,int64_t val)241 inline int64_t AtomicOperations::fetchAddSeqCst(int64_t* addr, int64_t val) {
242 MOZ_CRASH("No 64-bit atomics");
243 }
244
245 template <>
fetchAddSeqCst(uint64_t * addr,uint64_t val)246 inline uint64_t AtomicOperations::fetchAddSeqCst(uint64_t* addr, uint64_t val) {
247 MOZ_CRASH("No 64-bit atomics");
248 }
249
250 } // namespace jit
251 } // namespace js
252 #endif
253
254 template <typename T>
fetchSubSeqCst(T * addr,T val)255 inline T js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val) {
256 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
257 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
258 return __sync_fetch_and_sub(addr, val);
259 #else
260 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
261 #endif
262 }
263
264 #ifndef HAS_64BIT_ATOMICS
265 namespace js {
266 namespace jit {
267
268 template <>
fetchSubSeqCst(int64_t * addr,int64_t val)269 inline int64_t AtomicOperations::fetchSubSeqCst(int64_t* addr, int64_t val) {
270 MOZ_CRASH("No 64-bit atomics");
271 }
272
273 template <>
fetchSubSeqCst(uint64_t * addr,uint64_t val)274 inline uint64_t AtomicOperations::fetchSubSeqCst(uint64_t* addr, uint64_t val) {
275 MOZ_CRASH("No 64-bit atomics");
276 }
277
278 } // namespace jit
279 } // namespace js
280 #endif
281
282 template <typename T>
fetchAndSeqCst(T * addr,T val)283 inline T js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val) {
284 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
285 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
286 return __sync_fetch_and_and(addr, val);
287 #else
288 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
289 #endif
290 }
291
292 #ifndef HAS_64BIT_ATOMICS
293 namespace js {
294 namespace jit {
295
296 template <>
fetchAndSeqCst(int64_t * addr,int64_t val)297 inline int64_t AtomicOperations::fetchAndSeqCst(int64_t* addr, int64_t val) {
298 MOZ_CRASH("No 64-bit atomics");
299 }
300
301 template <>
fetchAndSeqCst(uint64_t * addr,uint64_t val)302 inline uint64_t AtomicOperations::fetchAndSeqCst(uint64_t* addr, uint64_t val) {
303 MOZ_CRASH("No 64-bit atomics");
304 }
305
306 } // namespace jit
307 } // namespace js
308 #endif
309
310 template <typename T>
fetchOrSeqCst(T * addr,T val)311 inline T js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val) {
312 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
313 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
314 return __sync_fetch_and_or(addr, val);
315 #else
316 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
317 #endif
318 }
319
320 #ifndef HAS_64BIT_ATOMICS
321 namespace js {
322 namespace jit {
323
324 template <>
fetchOrSeqCst(int64_t * addr,int64_t val)325 inline int64_t AtomicOperations::fetchOrSeqCst(int64_t* addr, int64_t val) {
326 MOZ_CRASH("No 64-bit atomics");
327 }
328
329 template <>
fetchOrSeqCst(uint64_t * addr,uint64_t val)330 inline uint64_t AtomicOperations::fetchOrSeqCst(uint64_t* addr, uint64_t val) {
331 MOZ_CRASH("No 64-bit atomics");
332 }
333
334 } // namespace jit
335 } // namespace js
336 #endif
337
338 template <typename T>
fetchXorSeqCst(T * addr,T val)339 inline T js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val) {
340 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
341 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
342 return __sync_fetch_and_xor(addr, val);
343 #else
344 return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST);
345 #endif
346 }
347
348 #ifndef HAS_64BIT_ATOMICS
349 namespace js {
350 namespace jit {
351
352 template <>
fetchXorSeqCst(int64_t * addr,int64_t val)353 inline int64_t AtomicOperations::fetchXorSeqCst(int64_t* addr, int64_t val) {
354 MOZ_CRASH("No 64-bit atomics");
355 }
356
357 template <>
fetchXorSeqCst(uint64_t * addr,uint64_t val)358 inline uint64_t AtomicOperations::fetchXorSeqCst(uint64_t* addr, uint64_t val) {
359 MOZ_CRASH("No 64-bit atomics");
360 }
361
362 } // namespace jit
363 } // namespace js
364 #endif
365
366 template <typename T>
loadSafeWhenRacy(T * addr)367 inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) {
368 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
369 // This is actually roughly right even on 32-bit platforms since in that
370 // case, double, int64, and uint64 loads need not be access-atomic.
371 return *addr;
372 }
373
374 template <typename T>
storeSafeWhenRacy(T * addr,T val)375 inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) {
376 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
377 // This is actually roughly right even on 32-bit platforms since in that
378 // case, double, int64, and uint64 loads need not be access-atomic.
379 *addr = val;
380 }
381
memcpySafeWhenRacy(void * dest,const void * src,size_t nbytes)382 inline void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest,
383 const void* src,
384 size_t nbytes) {
385 MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes));
386 MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes));
387 ::memcpy(dest, src, nbytes);
388 }
389
memmoveSafeWhenRacy(void * dest,const void * src,size_t nbytes)390 inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest,
391 const void* src,
392 size_t nbytes) {
393 ::memmove(dest, src, nbytes);
394 }
395
396 template <typename T>
exchangeSeqCst(T * addr,T val)397 inline T js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val) {
398 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
399 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
400 T v;
401 __sync_synchronize();
402 do {
403 v = *addr;
404 } while (__sync_val_compare_and_swap(addr, v, val) != v);
405 return v;
406 #else
407 T v;
408 __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST);
409 return v;
410 #endif
411 }
412
413 #ifndef HAS_64BIT_ATOMICS
414 namespace js {
415 namespace jit {
416
417 template <>
exchangeSeqCst(int64_t * addr,int64_t val)418 inline int64_t AtomicOperations::exchangeSeqCst(int64_t* addr, int64_t val) {
419 MOZ_CRASH("No 64-bit atomics");
420 }
421
422 template <>
exchangeSeqCst(uint64_t * addr,uint64_t val)423 inline uint64_t AtomicOperations::exchangeSeqCst(uint64_t* addr, uint64_t val) {
424 MOZ_CRASH("No 64-bit atomics");
425 }
426
427 } // namespace jit
428 } // namespace js
429 #endif
430
431 #elif defined(ENABLE_SHARED_ARRAY_BUFFER)
432
433 #error "Either disable JS shared memory, use GCC or Clang, or add code here"
434
435 #endif
436
437 #undef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
438 #undef GNUC_COMPATIBLE
439 #undef HAS_64BIT_ATOMICS
440 #undef HAS_64BIT_LOCKFREE
441
442 #endif // jit_none_AtomicOperations_feeling_lucky_h
443