1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 /* For documentation, see jit/AtomicOperations.h, both the comment block at the
8 * beginning and the #ifdef nest near the end.
9 *
10 * This is a common file for tier-3 platforms (including simulators for our
11 * tier-1 platforms) that are not providing hardware-specific implementations of
12 * the atomic operations. Please keep it reasonably platform-independent by
13 * adding #ifdefs at the beginning as much as possible, not throughout the file.
14 *
15 *
16 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
17 * !!!! NOTE !!!!
18 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
19 *
20 * The implementations in this file are NOT SAFE and cannot be safe even in
21 * principle because they rely on C++ undefined behavior. However, they are
22 * frequently good enough for tier-3 platforms.
23 */
24
25 #ifndef jit_shared_AtomicOperations_feeling_lucky_gcc_h
26 #define jit_shared_AtomicOperations_feeling_lucky_gcc_h
27
28 #include "mozilla/Assertions.h"
29 #include "mozilla/Types.h"
30
31 // Explicitly exclude tier-1 platforms.
32
33 #if ((defined(__x86_64__) || defined(_M_X64)) && defined(JS_CODEGEN_X64)) || \
34 ((defined(__i386__) || defined(_M_IX86)) && defined(JS_CODEGEN_X86)) || \
35 (defined(__arm__) && defined(JS_CODEGEN_ARM)) || \
36 ((defined(__aarch64__) || defined(_M_ARM64)) && defined(JS_CODEGEN_ARM64))
37 # error "Do not use this code on a tier-1 platform when a JIT is available"
38 #endif
39
40 #if !(defined(__clang__) || defined(__GNUC__))
41 # error "This file only for gcc/Clang"
42 #endif
43
44 // 64-bit atomics are not required by the JS spec, and you can compile
45 // SpiderMonkey without them. 64-bit atomics are required for BigInt
46 // support.
47 //
48 // 64-bit lock-free atomics are required for WebAssembly, but gating in the
49 // WebAssembly subsystem ensures that no WebAssembly-supporting platforms need
50 // code in this file.
51
52 #if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \
53 defined(JS_SIMULATOR_MIPS64)
54 // On some x86 (32-bit) systems this will not work because the compiler does not
55 // open-code 64-bit atomics. If so, try linking with -latomic. If that doesn't
56 // work, you're mostly on your own.
57 # define HAS_64BIT_ATOMICS
58 # define HAS_64BIT_LOCKFREE
59 #endif
60
61 #if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \
62 defined(__PPC64LE__)
63 # define HAS_64BIT_ATOMICS
64 # define HAS_64BIT_LOCKFREE
65 #endif
66
67 #if defined(__riscv) && __riscv_xlen == 64
68 # define HAS_64BIT_ATOMICS
69 # define HAS_64BIT_LOCKFREE
70 #endif
71
72 #ifdef __sparc__
73 # ifdef __LP64__
74 # define HAS_64BIT_ATOMICS
75 # define HAS_64BIT_LOCKFREE
76 # endif
77 #endif
78
79 #ifdef JS_CODEGEN_NONE
80 # ifdef JS_64BIT
81 # define HAS_64BIT_ATOMICS
82 # define HAS_64BIT_LOCKFREE
83 # endif
84 #endif
85
86 // The default implementation tactic for gcc/clang is to use the newer __atomic
87 // intrinsics added for use in C++11 <atomic>. Where that isn't available, we
88 // use GCC's older __sync functions instead.
89 //
90 // ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS is kept as a backward compatible
91 // option for older compilers: enable this to use GCC's old __sync functions
92 // instead of the newer __atomic functions. This will be required for GCC 4.6.x
93 // and earlier, and probably for Clang 3.1, should we need to use those
94 // versions. Firefox no longer supports compilers that old.
95
96 //#define ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
97
98 // Sanity check.
99
100 #if defined(HAS_64BIT_LOCKFREE) && !defined(HAS_64BIT_ATOMICS)
101 # error "This combination of features is senseless, please fix"
102 #endif
103
104 // Try to avoid platform #ifdefs below this point.
105
Initialize()106 inline bool js::jit::AtomicOperations::Initialize() {
107 // Nothing
108 return true;
109 }
110
ShutDown()111 inline void js::jit::AtomicOperations::ShutDown() {
112 // Nothing
113 }
114
115 // When compiling with Clang on 32-bit linux it will be necessary to link with
116 // -latomic to get the proper 64-bit intrinsics.
117
hasAtomic8()118 inline bool js::jit::AtomicOperations::hasAtomic8() {
119 #if defined(HAS_64BIT_ATOMICS)
120 return true;
121 #else
122 return false;
123 #endif
124 }
125
isLockfree8()126 inline bool js::jit::AtomicOperations::isLockfree8() {
127 #if defined(HAS_64BIT_LOCKFREE)
128 return true;
129 #else
130 return false;
131 #endif
132 }
133
fenceSeqCst()134 inline void js::jit::AtomicOperations::fenceSeqCst() {
135 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
136 __sync_synchronize();
137 #else
138 __atomic_thread_fence(__ATOMIC_SEQ_CST);
139 #endif
140 }
141
142 template <typename T>
loadSeqCst(T * addr)143 inline T js::jit::AtomicOperations::loadSeqCst(T* addr) {
144 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
145 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
146 __sync_synchronize();
147 T v = *addr;
148 __sync_synchronize();
149 #else
150 T v;
151 __atomic_load(addr, &v, __ATOMIC_SEQ_CST);
152 #endif
153 return v;
154 }
155
156 #ifndef HAS_64BIT_ATOMICS
157 namespace js {
158 namespace jit {
159
160 template <>
loadSeqCst(int64_t * addr)161 inline int64_t AtomicOperations::loadSeqCst(int64_t* addr) {
162 MOZ_CRASH("No 64-bit atomics");
163 }
164
165 template <>
loadSeqCst(uint64_t * addr)166 inline uint64_t AtomicOperations::loadSeqCst(uint64_t* addr) {
167 MOZ_CRASH("No 64-bit atomics");
168 }
169
170 } // namespace jit
171 } // namespace js
172 #endif
173
174 template <typename T>
storeSeqCst(T * addr,T val)175 inline void js::jit::AtomicOperations::storeSeqCst(T* addr, T val) {
176 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
177 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
178 __sync_synchronize();
179 *addr = val;
180 __sync_synchronize();
181 #else
182 __atomic_store(addr, &val, __ATOMIC_SEQ_CST);
183 #endif
184 }
185
186 #ifndef HAS_64BIT_ATOMICS
187 namespace js {
188 namespace jit {
189
190 template <>
storeSeqCst(int64_t * addr,int64_t val)191 inline void AtomicOperations::storeSeqCst(int64_t* addr, int64_t val) {
192 MOZ_CRASH("No 64-bit atomics");
193 }
194
195 template <>
storeSeqCst(uint64_t * addr,uint64_t val)196 inline void AtomicOperations::storeSeqCst(uint64_t* addr, uint64_t val) {
197 MOZ_CRASH("No 64-bit atomics");
198 }
199
200 } // namespace jit
201 } // namespace js
202 #endif
203
204 template <typename T>
exchangeSeqCst(T * addr,T val)205 inline T js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val) {
206 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
207 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
208 T v;
209 __sync_synchronize();
210 do {
211 v = *addr;
212 } while (__sync_val_compare_and_swap(addr, v, val) != v);
213 return v;
214 #else
215 T v;
216 __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST);
217 return v;
218 #endif
219 }
220
221 #ifndef HAS_64BIT_ATOMICS
222 namespace js {
223 namespace jit {
224
225 template <>
exchangeSeqCst(int64_t * addr,int64_t val)226 inline int64_t AtomicOperations::exchangeSeqCst(int64_t* addr, int64_t val) {
227 MOZ_CRASH("No 64-bit atomics");
228 }
229
230 template <>
exchangeSeqCst(uint64_t * addr,uint64_t val)231 inline uint64_t AtomicOperations::exchangeSeqCst(uint64_t* addr, uint64_t val) {
232 MOZ_CRASH("No 64-bit atomics");
233 }
234
235 } // namespace jit
236 } // namespace js
237 #endif
238
239 template <typename T>
compareExchangeSeqCst(T * addr,T oldval,T newval)240 inline T js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval,
241 T newval) {
242 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
243 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
244 return __sync_val_compare_and_swap(addr, oldval, newval);
245 #else
246 __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST,
247 __ATOMIC_SEQ_CST);
248 return oldval;
249 #endif
250 }
251
252 #ifndef HAS_64BIT_ATOMICS
253 namespace js {
254 namespace jit {
255
256 template <>
compareExchangeSeqCst(int64_t * addr,int64_t oldval,int64_t newval)257 inline int64_t AtomicOperations::compareExchangeSeqCst(int64_t* addr,
258 int64_t oldval,
259 int64_t newval) {
260 MOZ_CRASH("No 64-bit atomics");
261 }
262
263 template <>
compareExchangeSeqCst(uint64_t * addr,uint64_t oldval,uint64_t newval)264 inline uint64_t AtomicOperations::compareExchangeSeqCst(uint64_t* addr,
265 uint64_t oldval,
266 uint64_t newval) {
267 MOZ_CRASH("No 64-bit atomics");
268 }
269
270 } // namespace jit
271 } // namespace js
272 #endif
273
274 template <typename T>
fetchAddSeqCst(T * addr,T val)275 inline T js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val) {
276 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
277 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
278 return __sync_fetch_and_add(addr, val);
279 #else
280 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
281 #endif
282 }
283
284 #ifndef HAS_64BIT_ATOMICS
285 namespace js {
286 namespace jit {
287
288 template <>
fetchAddSeqCst(int64_t * addr,int64_t val)289 inline int64_t AtomicOperations::fetchAddSeqCst(int64_t* addr, int64_t val) {
290 MOZ_CRASH("No 64-bit atomics");
291 }
292
293 template <>
fetchAddSeqCst(uint64_t * addr,uint64_t val)294 inline uint64_t AtomicOperations::fetchAddSeqCst(uint64_t* addr, uint64_t val) {
295 MOZ_CRASH("No 64-bit atomics");
296 }
297
298 } // namespace jit
299 } // namespace js
300 #endif
301
302 template <typename T>
fetchSubSeqCst(T * addr,T val)303 inline T js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val) {
304 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
305 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
306 return __sync_fetch_and_sub(addr, val);
307 #else
308 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
309 #endif
310 }
311
312 #ifndef HAS_64BIT_ATOMICS
313 namespace js {
314 namespace jit {
315
316 template <>
fetchSubSeqCst(int64_t * addr,int64_t val)317 inline int64_t AtomicOperations::fetchSubSeqCst(int64_t* addr, int64_t val) {
318 MOZ_CRASH("No 64-bit atomics");
319 }
320
321 template <>
fetchSubSeqCst(uint64_t * addr,uint64_t val)322 inline uint64_t AtomicOperations::fetchSubSeqCst(uint64_t* addr, uint64_t val) {
323 MOZ_CRASH("No 64-bit atomics");
324 }
325
326 } // namespace jit
327 } // namespace js
328 #endif
329
330 template <typename T>
fetchAndSeqCst(T * addr,T val)331 inline T js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val) {
332 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
333 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
334 return __sync_fetch_and_and(addr, val);
335 #else
336 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
337 #endif
338 }
339
340 #ifndef HAS_64BIT_ATOMICS
341 namespace js {
342 namespace jit {
343
344 template <>
fetchAndSeqCst(int64_t * addr,int64_t val)345 inline int64_t AtomicOperations::fetchAndSeqCst(int64_t* addr, int64_t val) {
346 MOZ_CRASH("No 64-bit atomics");
347 }
348
349 template <>
fetchAndSeqCst(uint64_t * addr,uint64_t val)350 inline uint64_t AtomicOperations::fetchAndSeqCst(uint64_t* addr, uint64_t val) {
351 MOZ_CRASH("No 64-bit atomics");
352 }
353
354 } // namespace jit
355 } // namespace js
356 #endif
357
358 template <typename T>
fetchOrSeqCst(T * addr,T val)359 inline T js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val) {
360 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
361 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
362 return __sync_fetch_and_or(addr, val);
363 #else
364 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
365 #endif
366 }
367
368 #ifndef HAS_64BIT_ATOMICS
369 namespace js {
370 namespace jit {
371
372 template <>
fetchOrSeqCst(int64_t * addr,int64_t val)373 inline int64_t AtomicOperations::fetchOrSeqCst(int64_t* addr, int64_t val) {
374 MOZ_CRASH("No 64-bit atomics");
375 }
376
377 template <>
fetchOrSeqCst(uint64_t * addr,uint64_t val)378 inline uint64_t AtomicOperations::fetchOrSeqCst(uint64_t* addr, uint64_t val) {
379 MOZ_CRASH("No 64-bit atomics");
380 }
381
382 } // namespace jit
383 } // namespace js
384 #endif
385
386 template <typename T>
fetchXorSeqCst(T * addr,T val)387 inline T js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val) {
388 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
389 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
390 return __sync_fetch_and_xor(addr, val);
391 #else
392 return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST);
393 #endif
394 }
395
396 #ifndef HAS_64BIT_ATOMICS
397 namespace js {
398 namespace jit {
399
400 template <>
fetchXorSeqCst(int64_t * addr,int64_t val)401 inline int64_t AtomicOperations::fetchXorSeqCst(int64_t* addr, int64_t val) {
402 MOZ_CRASH("No 64-bit atomics");
403 }
404
405 template <>
fetchXorSeqCst(uint64_t * addr,uint64_t val)406 inline uint64_t AtomicOperations::fetchXorSeqCst(uint64_t* addr, uint64_t val) {
407 MOZ_CRASH("No 64-bit atomics");
408 }
409
410 } // namespace jit
411 } // namespace js
412 #endif
413
414 template <typename T>
loadSafeWhenRacy(T * addr)415 inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) {
416 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
417 // This is actually roughly right even on 32-bit platforms since in that
418 // case, double, int64, and uint64 loads need not be access-atomic.
419 //
420 // We could use __atomic_load, but it would be needlessly expensive on
421 // 32-bit platforms that could support it and just plain wrong on others.
422 return *addr;
423 }
424
425 template <typename T>
storeSafeWhenRacy(T * addr,T val)426 inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) {
427 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
428 // This is actually roughly right even on 32-bit platforms since in that
429 // case, double, int64, and uint64 loads need not be access-atomic.
430 //
431 // We could use __atomic_store, but it would be needlessly expensive on
432 // 32-bit platforms that could support it and just plain wrong on others.
433 *addr = val;
434 }
435
memcpySafeWhenRacy(void * dest,const void * src,size_t nbytes)436 inline void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest,
437 const void* src,
438 size_t nbytes) {
439 MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes));
440 MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes));
441 ::memcpy(dest, src, nbytes);
442 }
443
memmoveSafeWhenRacy(void * dest,const void * src,size_t nbytes)444 inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest,
445 const void* src,
446 size_t nbytes) {
447 ::memmove(dest, src, nbytes);
448 }
449
450 #undef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
451 #undef HAS_64BIT_ATOMICS
452 #undef HAS_64BIT_LOCKFREE
453
454 #endif // jit_shared_AtomicOperations_feeling_lucky_gcc_h
455