1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef jit_AtomicOperations_h
8 #define jit_AtomicOperations_h
9
10 #include "mozilla/Types.h"
11
12 #include <string.h>
13
14 #include "vm/SharedMem.h"
15
16 namespace js {
17 namespace jit {
18
19 /*
20 * [SMDOC] Atomic Operations
21 *
22 * The atomic operations layer defines types and functions for
23 * JIT-compatible atomic operation.
24 *
25 * The fundamental constraints on the functions are:
26 *
27 * - That their realization here MUST be compatible with code the JIT
28 * generates for its Atomics operations, so that an atomic access
29 * from the interpreter or runtime - from any C++ code - really is
30 * atomic relative to a concurrent, compatible atomic access from
31 * jitted code. That is, these primitives expose JIT-compatible
32 * atomicity functionality to C++.
33 *
34 * - That accesses may race without creating C++ undefined behavior:
35 * atomic accesses (marked "SeqCst") may race with non-atomic
36 * accesses (marked "SafeWhenRacy"); overlapping but non-matching,
37 * and hence incompatible, atomic accesses may race; and non-atomic
38 * accesses may race. The effects of races need not be predictable,
39 * so garbage can be produced by a read or written by a write, but
40 * the effects must be benign: the program must continue to run, and
41 * only the memory in the union of addresses named in the racing
42 * accesses may be affected.
43 *
44 * The compatibility constraint means that if the JIT makes dynamic
45 * decisions about how to implement atomic operations then
46 * corresponding dynamic decisions MUST be made in the implementations
47 * of the functions below.
48 *
49 * The safe-for-races constraint means that by and large, it is hard
50 * to implement these primitives in C++. See "Implementation notes"
51 * below.
52 *
53 * The "SeqCst" suffix on operations means "sequentially consistent"
54 * and means such a function's operation must have "sequentially
55 * consistent" memory ordering. See mfbt/Atomics.h for an explanation
56 * of this memory ordering.
57 *
58 * Note that a "SafeWhenRacy" access does not provide the atomicity of
59 * a "relaxed atomic" access: it can read or write garbage if there's
60 * a race.
61 *
62 *
63 * Implementation notes.
64 *
65 * It's not a requirement that these functions be inlined; performance
66 * is not a great concern. On some platforms these functions may call
67 * out to code that's generated at run time.
68 *
69 * In principle these functions will not be written in C++, thus
70 * making races defined behavior if all racy accesses from C++ go via
71 * these functions. (Jitted code will always be safe for races and
72 * provides the same guarantees as these functions.)
73 *
74 * The appropriate implementations will be platform-specific and
75 * there are some obvious implementation strategies to choose
76 * from, sometimes a combination is appropriate:
77 *
78 * - generating the code at run-time with the JIT;
79 * - hand-written assembler (maybe inline); or
80 * - using special compiler intrinsics or directives.
81 *
82 * Trusting the compiler not to generate code that blows up on a
83 * race definitely won't work in the presence of TSan, or even of
84 * optimizing compilers in seemingly-"innocuous" conditions. (See
85 * https://www.usenix.org/legacy/event/hotpar11/tech/final_files/Boehm.pdf
86 * for details.)
87 */
88 class AtomicOperations {
89 // The following functions are defined for T = int8_t, uint8_t,
90 // int16_t, uint16_t, int32_t, uint32_t, int64_t, and uint64_t.
91
92 // Atomically read *addr.
93 template <typename T>
94 static inline T loadSeqCst(T* addr);
95
96 // Atomically store val in *addr.
97 template <typename T>
98 static inline void storeSeqCst(T* addr, T val);
99
100 // Atomically store val in *addr and return the old value of *addr.
101 template <typename T>
102 static inline T exchangeSeqCst(T* addr, T val);
103
104 // Atomically check that *addr contains oldval and if so replace it
105 // with newval, in any case returning the old contents of *addr.
106 template <typename T>
107 static inline T compareExchangeSeqCst(T* addr, T oldval, T newval);
108
109 // Atomically add, subtract, bitwise-AND, bitwise-OR, or bitwise-XOR
110 // val into *addr and return the old value of *addr.
111 template <typename T>
112 static inline T fetchAddSeqCst(T* addr, T val);
113
114 template <typename T>
115 static inline T fetchSubSeqCst(T* addr, T val);
116
117 template <typename T>
118 static inline T fetchAndSeqCst(T* addr, T val);
119
120 template <typename T>
121 static inline T fetchOrSeqCst(T* addr, T val);
122
123 template <typename T>
124 static inline T fetchXorSeqCst(T* addr, T val);
125
126 // The SafeWhenRacy functions are to be used when C++ code has to access
127 // memory without synchronization and can't guarantee that there won't be a
128 // race on the access. But they are access-atomic for integer data so long
129 // as any racing writes are of the same size and to the same address.
130
131 // Defined for all the integral types as well as for float32 and float64,
132 // but not access-atomic for floats, nor for int64 and uint64 on 32-bit
133 // platforms.
134 template <typename T>
135 static inline T loadSafeWhenRacy(T* addr);
136
137 // Defined for all the integral types as well as for float32 and float64,
138 // but not access-atomic for floats, nor for int64 and uint64 on 32-bit
139 // platforms.
140 template <typename T>
141 static inline void storeSafeWhenRacy(T* addr, T val);
142
143 // Replacement for memcpy(). No access-atomicity guarantees.
144 static inline void memcpySafeWhenRacy(void* dest, const void* src,
145 size_t nbytes);
146
147 // Replacement for memmove(). No access-atomicity guarantees.
148 static inline void memmoveSafeWhenRacy(void* dest, const void* src,
149 size_t nbytes);
150
151 public:
152 // On some platforms we generate code for the atomics at run-time; that
153 // happens here.
154 static bool Initialize();
155
156 // Deallocate the code segment for generated atomics functions.
157 static void ShutDown();
158
159 // Test lock-freedom for any int32 value. This implements the
160 // Atomics::isLockFree() operation in the ECMAScript Shared Memory and
161 // Atomics specification, as follows:
162 //
163 // 4-byte accesses are always lock free (in the spec).
164 // 1-, 2-, and 8-byte accesses are always lock free (in SpiderMonkey).
165 //
166 // There is no lock-freedom for JS for any other values on any platform.
167 static constexpr inline bool isLockfreeJS(int32_t n);
168
169 // If the return value is true then the templated functions below are
170 // supported for int64_t and uint64_t. If the return value is false then
171 // those functions will MOZ_CRASH. The value of this call does not change
172 // during execution.
173 static inline bool hasAtomic8();
174
175 // If the return value is true then hasAtomic8() is true and the atomic
176 // operations are indeed lock-free. The value of this call does not change
177 // during execution.
178 static inline bool isLockfree8();
179
180 // Execute a full memory barrier (LoadLoad+LoadStore+StoreLoad+StoreStore).
181 static inline void fenceSeqCst();
182
183 // All clients should use the APIs that take SharedMem pointers.
184 // See above for semantics and acceptable types.
185
186 template <typename T>
loadSeqCst(SharedMem<T * > addr)187 static T loadSeqCst(SharedMem<T*> addr) {
188 return loadSeqCst(addr.unwrap());
189 }
190
191 template <typename T>
storeSeqCst(SharedMem<T * > addr,T val)192 static void storeSeqCst(SharedMem<T*> addr, T val) {
193 return storeSeqCst(addr.unwrap(), val);
194 }
195
196 template <typename T>
exchangeSeqCst(SharedMem<T * > addr,T val)197 static T exchangeSeqCst(SharedMem<T*> addr, T val) {
198 return exchangeSeqCst(addr.unwrap(), val);
199 }
200
201 template <typename T>
compareExchangeSeqCst(SharedMem<T * > addr,T oldval,T newval)202 static T compareExchangeSeqCst(SharedMem<T*> addr, T oldval, T newval) {
203 return compareExchangeSeqCst(addr.unwrap(), oldval, newval);
204 }
205
206 template <typename T>
fetchAddSeqCst(SharedMem<T * > addr,T val)207 static T fetchAddSeqCst(SharedMem<T*> addr, T val) {
208 return fetchAddSeqCst(addr.unwrap(), val);
209 }
210
211 template <typename T>
fetchSubSeqCst(SharedMem<T * > addr,T val)212 static T fetchSubSeqCst(SharedMem<T*> addr, T val) {
213 return fetchSubSeqCst(addr.unwrap(), val);
214 }
215
216 template <typename T>
fetchAndSeqCst(SharedMem<T * > addr,T val)217 static T fetchAndSeqCst(SharedMem<T*> addr, T val) {
218 return fetchAndSeqCst(addr.unwrap(), val);
219 }
220
221 template <typename T>
fetchOrSeqCst(SharedMem<T * > addr,T val)222 static T fetchOrSeqCst(SharedMem<T*> addr, T val) {
223 return fetchOrSeqCst(addr.unwrap(), val);
224 }
225
226 template <typename T>
fetchXorSeqCst(SharedMem<T * > addr,T val)227 static T fetchXorSeqCst(SharedMem<T*> addr, T val) {
228 return fetchXorSeqCst(addr.unwrap(), val);
229 }
230
231 template <typename T>
loadSafeWhenRacy(SharedMem<T * > addr)232 static T loadSafeWhenRacy(SharedMem<T*> addr) {
233 return loadSafeWhenRacy(addr.unwrap());
234 }
235
236 template <typename T>
storeSafeWhenRacy(SharedMem<T * > addr,T val)237 static void storeSafeWhenRacy(SharedMem<T*> addr, T val) {
238 return storeSafeWhenRacy(addr.unwrap(), val);
239 }
240
241 template <typename T>
memcpySafeWhenRacy(SharedMem<T * > dest,SharedMem<T * > src,size_t nbytes)242 static void memcpySafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src,
243 size_t nbytes) {
244 memcpySafeWhenRacy(dest.template cast<void*>().unwrap(),
245 src.template cast<void*>().unwrap(), nbytes);
246 }
247
248 template <typename T>
memcpySafeWhenRacy(SharedMem<T * > dest,T * src,size_t nbytes)249 static void memcpySafeWhenRacy(SharedMem<T*> dest, T* src, size_t nbytes) {
250 memcpySafeWhenRacy(dest.template cast<void*>().unwrap(),
251 static_cast<void*>(src), nbytes);
252 }
253
254 template <typename T>
memcpySafeWhenRacy(T * dest,SharedMem<T * > src,size_t nbytes)255 static void memcpySafeWhenRacy(T* dest, SharedMem<T*> src, size_t nbytes) {
256 memcpySafeWhenRacy(static_cast<void*>(dest),
257 src.template cast<void*>().unwrap(), nbytes);
258 }
259
260 template <typename T>
memmoveSafeWhenRacy(SharedMem<T * > dest,SharedMem<T * > src,size_t nbytes)261 static void memmoveSafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src,
262 size_t nbytes) {
263 memmoveSafeWhenRacy(dest.template cast<void*>().unwrap(),
264 src.template cast<void*>().unwrap(), nbytes);
265 }
266
memsetSafeWhenRacy(SharedMem<uint8_t * > dest,int value,size_t nbytes)267 static void memsetSafeWhenRacy(SharedMem<uint8_t*> dest, int value,
268 size_t nbytes) {
269 uint8_t buf[1024];
270 size_t iterations = nbytes / sizeof(buf);
271 size_t tail = nbytes % sizeof(buf);
272 size_t offs = 0;
273 if (iterations > 0) {
274 memset(buf, value, sizeof(buf));
275 while (iterations--) {
276 memcpySafeWhenRacy(dest + offs, SharedMem<uint8_t*>::unshared(buf),
277 sizeof(buf));
278 offs += sizeof(buf);
279 }
280 } else {
281 memset(buf, value, tail);
282 }
283 memcpySafeWhenRacy(dest + offs, SharedMem<uint8_t*>::unshared(buf), tail);
284 }
285
286 template <typename T>
podCopySafeWhenRacy(SharedMem<T * > dest,SharedMem<T * > src,size_t nelem)287 static void podCopySafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src,
288 size_t nelem) {
289 memcpySafeWhenRacy(dest, src, nelem * sizeof(T));
290 }
291
292 template <typename T>
podMoveSafeWhenRacy(SharedMem<T * > dest,SharedMem<T * > src,size_t nelem)293 static void podMoveSafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src,
294 size_t nelem) {
295 memmoveSafeWhenRacy(dest, src, nelem * sizeof(T));
296 }
297 };
298
isLockfreeJS(int32_t size)299 constexpr inline bool AtomicOperations::isLockfreeJS(int32_t size) {
300 // Keep this in sync with atomicIsLockFreeJS() in jit/MacroAssembler.cpp.
301
302 switch (size) {
303 case 1:
304 return true;
305 case 2:
306 return true;
307 case 4:
308 // The spec requires Atomics.isLockFree(4) to return true.
309 return true;
310 case 8:
311 return true;
312 default:
313 return false;
314 }
315 }
316
317 } // namespace jit
318 } // namespace js
319
320 // As explained above, our atomic operations are not portable even in principle,
321 // so we must include platform+compiler specific definitions here.
322 //
323 // x86, x64, arm, and arm64 are maintained by Mozilla. All other platform
324 // setups are by platform maintainers' request and are not maintained by
325 // Mozilla.
326 //
327 // If you are using a platform+compiler combination that causes an error below
328 // (and if the problem isn't just that the compiler uses a different name for a
329 // known architecture), you have basically three options:
330 //
331 // - find an already-supported compiler for the platform and use that instead
332 //
333 // - write your own support code for the platform+compiler and create a new
334 // case below
335 //
336 // - include jit/shared/AtomicOperations-feeling-lucky.h in a case for the
337 // platform below, if you have a gcc-compatible compiler and truly feel
338 // lucky. You may have to add a little code to that file, too.
339 //
340 // Simulators are confusing. These atomic primitives must be compatible with
341 // the code that the JIT emits, but of course for an ARM simulator running on
342 // x86 the primitives here will be for x86, not for ARM, while the JIT emits ARM
343 // code. Our ARM simulator solves that the easy way: by using these primitives
344 // to implement its atomic operations. For other simulators there may need to
345 // be special cases below to provide simulator-compatible primitives, for
346 // example, for our ARM64 simulator the primitives could in principle
347 // participate in the memory exclusivity monitors implemented by the simulator.
348 // Such a solution is likely to be difficult.
349
350 #if defined(JS_SIMULATOR_MIPS32)
351 # if defined(__clang__) || defined(__GNUC__)
352 # include "jit/mips-shared/AtomicOperations-mips-shared.h"
353 # else
354 # error "AtomicOperations on MIPS-32 for unknown compiler"
355 # endif
356 #elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
357 defined(_M_IX86)
358 # if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
359 # include "jit/shared/AtomicOperations-shared-jit.h"
360 # else
361 # include "jit/shared/AtomicOperations-feeling-lucky.h"
362 # endif
363 #elif defined(__arm__)
364 # if defined(JS_CODEGEN_ARM)
365 # include "jit/shared/AtomicOperations-shared-jit.h"
366 # else
367 # include "jit/shared/AtomicOperations-feeling-lucky.h"
368 # endif
369 #elif defined(__aarch64__) || defined(_M_ARM64)
370 # if defined(JS_CODEGEN_ARM64)
371 # include "jit/shared/AtomicOperations-shared-jit.h"
372 # else
373 # include "jit/shared/AtomicOperations-feeling-lucky.h"
374 # endif
375 #elif defined(__mips__)
376 # if defined(__clang__) || defined(__GNUC__)
377 # include "jit/mips-shared/AtomicOperations-mips-shared.h"
378 # else
379 # error "AtomicOperations on MIPS for an unknown compiler"
380 # endif
381 #elif defined(__ppc__) || defined(__PPC__) || defined(__sparc__) || \
382 defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \
383 defined(__PPC64LE__) || defined(__alpha__) || defined(__hppa__) || \
384 defined(__sh__) || defined(__s390__) || defined(__s390x__) || \
385 defined(__m68k__) || defined(__riscv) || defined(__wasi__)
386 # include "jit/shared/AtomicOperations-feeling-lucky.h"
387 #else
388 # error "No AtomicOperations support provided for this platform"
389 #endif
390
391 #endif // jit_AtomicOperations_h
392