1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=8 sts=4 et sw=4 tw=99:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 /* For documentation, see jit/AtomicOperations.h */
8
9 #ifndef jit_arm64_AtomicOperations_arm64_h
10 #define jit_arm64_AtomicOperations_arm64_h
11
12 #include "mozilla/Assertions.h"
13 #include "mozilla/Types.h"
14
15 #include "jit/arm64/Architecture-arm64.h"
16
17 inline bool
isLockfree8()18 js::jit::AtomicOperations::isLockfree8()
19 {
20 MOZ_ASSERT(__atomic_always_lock_free(sizeof(int8_t), 0));
21 MOZ_ASSERT(__atomic_always_lock_free(sizeof(int16_t), 0));
22 MOZ_ASSERT(__atomic_always_lock_free(sizeof(int32_t), 0));
23 return true;
24 }
25
26 inline void
fenceSeqCst()27 js::jit::AtomicOperations::fenceSeqCst()
28 {
29 __atomic_thread_fence(__ATOMIC_SEQ_CST);
30 }
31
32 template<typename T>
33 inline T
loadSeqCst(T * addr)34 js::jit::AtomicOperations::loadSeqCst(T* addr)
35 {
36 MOZ_ASSERT(sizeof(T) < 8 || isLockfree8());
37 T v;
38 __atomic_load(addr, &v, __ATOMIC_SEQ_CST);
39 return v;
40 }
41
42 template<typename T>
43 inline void
storeSeqCst(T * addr,T val)44 js::jit::AtomicOperations::storeSeqCst(T* addr, T val)
45 {
46 MOZ_ASSERT(sizeof(T) < 8 || isLockfree8());
47 __atomic_store(addr, &val, __ATOMIC_SEQ_CST);
48 }
49
50 template<typename T>
51 inline T
exchangeSeqCst(T * addr,T val)52 js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val)
53 {
54 MOZ_ASSERT(sizeof(T) < 8 || isLockfree8());
55 T v;
56 __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST);
57 return v;
58 }
59
60 template<typename T>
61 inline T
compareExchangeSeqCst(T * addr,T oldval,T newval)62 js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval, T newval)
63 {
64 MOZ_ASSERT(sizeof(T) < 8 || isLockfree8());
65 __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
66 return oldval;
67 }
68
69 template<typename T>
70 inline T
fetchAddSeqCst(T * addr,T val)71 js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val)
72 {
73 static_assert(sizeof(T) <= 4, "not available for 8-byte values yet");
74 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
75 }
76
77 template<typename T>
78 inline T
fetchSubSeqCst(T * addr,T val)79 js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val)
80 {
81 static_assert(sizeof(T) <= 4, "not available for 8-byte values yet");
82 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
83 }
84
85 template<typename T>
86 inline T
fetchAndSeqCst(T * addr,T val)87 js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val)
88 {
89 static_assert(sizeof(T) <= 4, "not available for 8-byte values yet");
90 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
91 }
92
93 template<typename T>
94 inline T
fetchOrSeqCst(T * addr,T val)95 js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val)
96 {
97 static_assert(sizeof(T) <= 4, "not available for 8-byte values yet");
98 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
99 }
100
101 template<typename T>
102 inline T
fetchXorSeqCst(T * addr,T val)103 js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val)
104 {
105 static_assert(sizeof(T) <= 4, "not available for 8-byte values yet");
106 return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST);
107 }
108
109 template <typename T>
110 inline T
loadSafeWhenRacy(T * addr)111 js::jit::AtomicOperations::loadSafeWhenRacy(T* addr)
112 {
113 return *addr; // FIXME (1208663): not yet safe
114 }
115
116 template <typename T>
117 inline void
storeSafeWhenRacy(T * addr,T val)118 js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val)
119 {
120 *addr = val; // FIXME (1208663): not yet safe
121 }
122
123 inline void
memcpySafeWhenRacy(void * dest,const void * src,size_t nbytes)124 js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest, const void* src,
125 size_t nbytes)
126 {
127 memcpy(dest, src, nbytes); // FIXME (1208663): not yet safe
128 }
129
130 inline void
memmoveSafeWhenRacy(void * dest,const void * src,size_t nbytes)131 js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest, const void* src,
132 size_t nbytes)
133 {
134 memmove(dest, src, nbytes); // FIXME (1208663): not yet safe
135 }
136
137 template<size_t nbytes>
138 inline void
acquire(void * addr)139 js::jit::RegionLock::acquire(void* addr)
140 {
141 uint32_t zero = 0;
142 uint32_t one = 1;
143 while (!__atomic_compare_exchange(&spinlock, &zero, &one, false, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE)) {
144 zero = 0;
145 continue;
146 }
147 }
148
149 template<size_t nbytes>
150 inline void
release(void * addr)151 js::jit::RegionLock::release(void* addr)
152 {
153 MOZ_ASSERT(AtomicOperations::loadSeqCst(&spinlock) == 1, "releasing unlocked region lock");
154 uint32_t zero = 0;
155 __atomic_store(&spinlock, &zero, __ATOMIC_SEQ_CST);
156 }
157
158 #endif // jit_arm64_AtomicOperations_arm64_h
159