1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 /* For documentation, see jit/AtomicOperations.h, both the comment block at the
8 * beginning and the #ifdef nest near the end.
9 *
10 * This is a common file for tier-3 platforms (including simulators for our
11 * tier-1 platforms) that are not providing hardware-specific implementations of
12 * the atomic operations. Please keep it reasonably platform-independent by
13 * adding #ifdefs at the beginning as much as possible, not throughout the file.
14 *
15 *
16 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
17 * !!!! NOTE !!!!
18 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
19 *
20 * The implementations in this file are NOT SAFE and cannot be safe even in
21 * principle because they rely on C++ undefined behavior. However, they are
22 * frequently good enough for tier-3 platforms.
23 */
24
25 #ifndef jit_shared_AtomicOperations_feeling_lucky_gcc_h
26 #define jit_shared_AtomicOperations_feeling_lucky_gcc_h
27
28 #include "mozilla/Assertions.h"
29 #include "mozilla/Types.h"
30
31 // Explicitly exclude tier-1 platforms.
32
33 #if ((defined(__x86_64__) || defined(_M_X64)) && defined(JS_CODEGEN_X64)) || \
34 ((defined(__i386__) || defined(_M_IX86)) && defined(JS_CODEGEN_X86)) || \
35 (defined(__arm__) && defined(JS_CODEGEN_ARM)) || \
36 ((defined(__aarch64__) || defined(_M_ARM64)) && defined(JS_CODEGEN_ARM64))
37 # error "Do not use this code on a tier-1 platform when a JIT is available"
38 #endif
39
40 #if !(defined(__clang__) || defined(__GNUC__))
41 # error "This file only for gcc/Clang"
42 #endif
43
44 // 64-bit atomics are not required by the JS spec, and you can compile
45 // SpiderMonkey without them. 64-bit atomics are required for BigInt
46 // support.
47 //
48 // 64-bit lock-free atomics are required for WebAssembly, but gating in the
49 // WebAssembly subsystem ensures that no WebAssembly-supporting platforms need
50 // code in this file.
51
52 #if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM)
53 // On some x86 (32-bit) systems this will not work because the compiler does not
54 // open-code 64-bit atomics. If so, try linking with -latomic. If that doesn't
55 // work, you're mostly on your own.
56 # define HAS_64BIT_ATOMICS
57 # define HAS_64BIT_LOCKFREE
58 #endif
59
60 #if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \
61 defined(__PPC64LE__)
62 # define HAS_64BIT_ATOMICS
63 # define HAS_64BIT_LOCKFREE
64 #endif
65
66 #ifdef __sparc__
67 # ifdef __LP64__
68 # define HAS_64BIT_ATOMICS
69 # define HAS_64BIT_LOCKFREE
70 # endif
71 #endif
72
73 #ifdef JS_CODEGEN_NONE
74 # ifdef JS_64BIT
75 # define HAS_64BIT_ATOMICS
76 # define HAS_64BIT_LOCKFREE
77 # endif
78 #endif
79
80 // The default implementation tactic for gcc/clang is to use the newer __atomic
81 // intrinsics added for use in C++11 <atomic>. Where that isn't available, we
82 // use GCC's older __sync functions instead.
83 //
84 // ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS is kept as a backward compatible
85 // option for older compilers: enable this to use GCC's old __sync functions
86 // instead of the newer __atomic functions. This will be required for GCC 4.6.x
87 // and earlier, and probably for Clang 3.1, should we need to use those
88 // versions. Firefox no longer supports compilers that old.
89
90 //#define ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
91
92 // Sanity check.
93
94 #if defined(HAS_64BIT_LOCKFREE) && !defined(HAS_64BIT_ATOMICS)
95 # error "This combination of features is senseless, please fix"
96 #endif
97
98 // Try to avoid platform #ifdefs below this point.
99
Initialize()100 inline bool js::jit::AtomicOperations::Initialize() {
101 // Nothing
102 return true;
103 }
104
ShutDown()105 inline void js::jit::AtomicOperations::ShutDown() {
106 // Nothing
107 }
108
109 // When compiling with Clang on 32-bit linux it will be necessary to link with
110 // -latomic to get the proper 64-bit intrinsics.
111
hasAtomic8()112 inline bool js::jit::AtomicOperations::hasAtomic8() {
113 #if defined(HAS_64BIT_ATOMICS)
114 return true;
115 #else
116 return false;
117 #endif
118 }
119
isLockfree8()120 inline bool js::jit::AtomicOperations::isLockfree8() {
121 #if defined(HAS_64BIT_LOCKFREE)
122 return true;
123 #else
124 return false;
125 #endif
126 }
127
fenceSeqCst()128 inline void js::jit::AtomicOperations::fenceSeqCst() {
129 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
130 __sync_synchronize();
131 #else
132 __atomic_thread_fence(__ATOMIC_SEQ_CST);
133 #endif
134 }
135
136 template <typename T>
loadSeqCst(T * addr)137 inline T js::jit::AtomicOperations::loadSeqCst(T* addr) {
138 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
139 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
140 __sync_synchronize();
141 T v = *addr;
142 __sync_synchronize();
143 #else
144 T v;
145 __atomic_load(addr, &v, __ATOMIC_SEQ_CST);
146 #endif
147 return v;
148 }
149
150 #ifndef HAS_64BIT_ATOMICS
151 namespace js {
152 namespace jit {
153
154 template <>
loadSeqCst(int64_t * addr)155 inline int64_t AtomicOperations::loadSeqCst(int64_t* addr) {
156 MOZ_CRASH("No 64-bit atomics");
157 }
158
159 template <>
loadSeqCst(uint64_t * addr)160 inline uint64_t AtomicOperations::loadSeqCst(uint64_t* addr) {
161 MOZ_CRASH("No 64-bit atomics");
162 }
163
164 } // namespace jit
165 } // namespace js
166 #endif
167
168 template <typename T>
storeSeqCst(T * addr,T val)169 inline void js::jit::AtomicOperations::storeSeqCst(T* addr, T val) {
170 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
171 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
172 __sync_synchronize();
173 *addr = val;
174 __sync_synchronize();
175 #else
176 __atomic_store(addr, &val, __ATOMIC_SEQ_CST);
177 #endif
178 }
179
180 #ifndef HAS_64BIT_ATOMICS
181 namespace js {
182 namespace jit {
183
184 template <>
storeSeqCst(int64_t * addr,int64_t val)185 inline void AtomicOperations::storeSeqCst(int64_t* addr, int64_t val) {
186 MOZ_CRASH("No 64-bit atomics");
187 }
188
189 template <>
storeSeqCst(uint64_t * addr,uint64_t val)190 inline void AtomicOperations::storeSeqCst(uint64_t* addr, uint64_t val) {
191 MOZ_CRASH("No 64-bit atomics");
192 }
193
194 } // namespace jit
195 } // namespace js
196 #endif
197
198 template <typename T>
exchangeSeqCst(T * addr,T val)199 inline T js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val) {
200 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
201 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
202 T v;
203 __sync_synchronize();
204 do {
205 v = *addr;
206 } while (__sync_val_compare_and_swap(addr, v, val) != v);
207 return v;
208 #else
209 T v;
210 __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST);
211 return v;
212 #endif
213 }
214
215 #ifndef HAS_64BIT_ATOMICS
216 namespace js {
217 namespace jit {
218
219 template <>
exchangeSeqCst(int64_t * addr,int64_t val)220 inline int64_t AtomicOperations::exchangeSeqCst(int64_t* addr, int64_t val) {
221 MOZ_CRASH("No 64-bit atomics");
222 }
223
224 template <>
exchangeSeqCst(uint64_t * addr,uint64_t val)225 inline uint64_t AtomicOperations::exchangeSeqCst(uint64_t* addr, uint64_t val) {
226 MOZ_CRASH("No 64-bit atomics");
227 }
228
229 } // namespace jit
230 } // namespace js
231 #endif
232
233 template <typename T>
compareExchangeSeqCst(T * addr,T oldval,T newval)234 inline T js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval,
235 T newval) {
236 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
237 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
238 return __sync_val_compare_and_swap(addr, oldval, newval);
239 #else
240 __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST,
241 __ATOMIC_SEQ_CST);
242 return oldval;
243 #endif
244 }
245
246 #ifndef HAS_64BIT_ATOMICS
247 namespace js {
248 namespace jit {
249
250 template <>
compareExchangeSeqCst(int64_t * addr,int64_t oldval,int64_t newval)251 inline int64_t AtomicOperations::compareExchangeSeqCst(int64_t* addr,
252 int64_t oldval,
253 int64_t newval) {
254 MOZ_CRASH("No 64-bit atomics");
255 }
256
257 template <>
compareExchangeSeqCst(uint64_t * addr,uint64_t oldval,uint64_t newval)258 inline uint64_t AtomicOperations::compareExchangeSeqCst(uint64_t* addr,
259 uint64_t oldval,
260 uint64_t newval) {
261 MOZ_CRASH("No 64-bit atomics");
262 }
263
264 } // namespace jit
265 } // namespace js
266 #endif
267
268 template <typename T>
fetchAddSeqCst(T * addr,T val)269 inline T js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val) {
270 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
271 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
272 return __sync_fetch_and_add(addr, val);
273 #else
274 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
275 #endif
276 }
277
278 #ifndef HAS_64BIT_ATOMICS
279 namespace js {
280 namespace jit {
281
282 template <>
fetchAddSeqCst(int64_t * addr,int64_t val)283 inline int64_t AtomicOperations::fetchAddSeqCst(int64_t* addr, int64_t val) {
284 MOZ_CRASH("No 64-bit atomics");
285 }
286
287 template <>
fetchAddSeqCst(uint64_t * addr,uint64_t val)288 inline uint64_t AtomicOperations::fetchAddSeqCst(uint64_t* addr, uint64_t val) {
289 MOZ_CRASH("No 64-bit atomics");
290 }
291
292 } // namespace jit
293 } // namespace js
294 #endif
295
296 template <typename T>
fetchSubSeqCst(T * addr,T val)297 inline T js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val) {
298 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
299 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
300 return __sync_fetch_and_sub(addr, val);
301 #else
302 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
303 #endif
304 }
305
306 #ifndef HAS_64BIT_ATOMICS
307 namespace js {
308 namespace jit {
309
310 template <>
fetchSubSeqCst(int64_t * addr,int64_t val)311 inline int64_t AtomicOperations::fetchSubSeqCst(int64_t* addr, int64_t val) {
312 MOZ_CRASH("No 64-bit atomics");
313 }
314
315 template <>
fetchSubSeqCst(uint64_t * addr,uint64_t val)316 inline uint64_t AtomicOperations::fetchSubSeqCst(uint64_t* addr, uint64_t val) {
317 MOZ_CRASH("No 64-bit atomics");
318 }
319
320 } // namespace jit
321 } // namespace js
322 #endif
323
324 template <typename T>
fetchAndSeqCst(T * addr,T val)325 inline T js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val) {
326 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
327 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
328 return __sync_fetch_and_and(addr, val);
329 #else
330 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
331 #endif
332 }
333
334 #ifndef HAS_64BIT_ATOMICS
335 namespace js {
336 namespace jit {
337
338 template <>
fetchAndSeqCst(int64_t * addr,int64_t val)339 inline int64_t AtomicOperations::fetchAndSeqCst(int64_t* addr, int64_t val) {
340 MOZ_CRASH("No 64-bit atomics");
341 }
342
343 template <>
fetchAndSeqCst(uint64_t * addr,uint64_t val)344 inline uint64_t AtomicOperations::fetchAndSeqCst(uint64_t* addr, uint64_t val) {
345 MOZ_CRASH("No 64-bit atomics");
346 }
347
348 } // namespace jit
349 } // namespace js
350 #endif
351
352 template <typename T>
fetchOrSeqCst(T * addr,T val)353 inline T js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val) {
354 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
355 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
356 return __sync_fetch_and_or(addr, val);
357 #else
358 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
359 #endif
360 }
361
362 #ifndef HAS_64BIT_ATOMICS
363 namespace js {
364 namespace jit {
365
366 template <>
fetchOrSeqCst(int64_t * addr,int64_t val)367 inline int64_t AtomicOperations::fetchOrSeqCst(int64_t* addr, int64_t val) {
368 MOZ_CRASH("No 64-bit atomics");
369 }
370
371 template <>
fetchOrSeqCst(uint64_t * addr,uint64_t val)372 inline uint64_t AtomicOperations::fetchOrSeqCst(uint64_t* addr, uint64_t val) {
373 MOZ_CRASH("No 64-bit atomics");
374 }
375
376 } // namespace jit
377 } // namespace js
378 #endif
379
380 template <typename T>
fetchXorSeqCst(T * addr,T val)381 inline T js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val) {
382 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
383 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
384 return __sync_fetch_and_xor(addr, val);
385 #else
386 return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST);
387 #endif
388 }
389
390 #ifndef HAS_64BIT_ATOMICS
391 namespace js {
392 namespace jit {
393
394 template <>
fetchXorSeqCst(int64_t * addr,int64_t val)395 inline int64_t AtomicOperations::fetchXorSeqCst(int64_t* addr, int64_t val) {
396 MOZ_CRASH("No 64-bit atomics");
397 }
398
399 template <>
fetchXorSeqCst(uint64_t * addr,uint64_t val)400 inline uint64_t AtomicOperations::fetchXorSeqCst(uint64_t* addr, uint64_t val) {
401 MOZ_CRASH("No 64-bit atomics");
402 }
403
404 } // namespace jit
405 } // namespace js
406 #endif
407
408 template <typename T>
loadSafeWhenRacy(T * addr)409 inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) {
410 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
411 // This is actually roughly right even on 32-bit platforms since in that
412 // case, double, int64, and uint64 loads need not be access-atomic.
413 //
414 // We could use __atomic_load, but it would be needlessly expensive on
415 // 32-bit platforms that could support it and just plain wrong on others.
416 return *addr;
417 }
418
419 template <typename T>
storeSafeWhenRacy(T * addr,T val)420 inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) {
421 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
422 // This is actually roughly right even on 32-bit platforms since in that
423 // case, double, int64, and uint64 loads need not be access-atomic.
424 //
425 // We could use __atomic_store, but it would be needlessly expensive on
426 // 32-bit platforms that could support it and just plain wrong on others.
427 *addr = val;
428 }
429
memcpySafeWhenRacy(void * dest,const void * src,size_t nbytes)430 inline void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest,
431 const void* src,
432 size_t nbytes) {
433 MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes));
434 MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes));
435 ::memcpy(dest, src, nbytes);
436 }
437
memmoveSafeWhenRacy(void * dest,const void * src,size_t nbytes)438 inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest,
439 const void* src,
440 size_t nbytes) {
441 ::memmove(dest, src, nbytes);
442 }
443
444 #undef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
445 #undef HAS_64BIT_ATOMICS
446 #undef HAS_64BIT_LOCKFREE
447
448 #endif // jit_shared_AtomicOperations_feeling_lucky_gcc_h
449