1 /*
2 * Copyright (c) 2012-2014, Bruno Levy
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * * Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * * Neither the name of the ALICE Project-Team nor the names of its
14 * contributors may be used to endorse or promote products derived from this
15 * software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 * POSSIBILITY OF SUCH DAMAGE.
28 *
29 * If you modify this software, you should include a notice giving the
30 * name of the person performing the modification, the date of modification,
31 * and the reason for such modification.
32 *
33 * Contact: Bruno Levy
34 *
35 * Bruno.Levy@inria.fr
36 * http://www.loria.fr/~levy
37 *
38 * ALICE Project
39 * LORIA, INRIA Lorraine,
40 * Campus Scientifique, BP 239
41 * 54506 VANDOEUVRE LES NANCY CEDEX
42 * FRANCE
43 *
44 */
45
46 #ifndef GEOGRAM_BASIC_ATOMICS
47 #define GEOGRAM_BASIC_ATOMICS
48
49 #include <geogram/basic/common.h>
50 #include <geogram/basic/numeric.h>
51
52 /**
53 * \file geogram/basic/atomics.h
54 * \brief Functions for atomic operations
55 */
56
57 #if defined(GEO_OS_LINUX) || defined(GEO_OS_DRAGONFLY)
58 # if defined(GEO_OS_EMSCRIPTEN)
59 # define GEO_USE_DUMMY_ATOMICS
60 # elif defined(GEO_OS_RASPBERRY)
61 # define GEO_USE_ARM32_ATOMICS
62 # elif defined(GEO_OS_ANDROID)
63 # define GEO_USE_ANDROID_ATOMICS
64 # else
65 # define GEO_USE_X86_ATOMICS
66 # endif
67 #endif
68
69 #if defined(GEO_USE_DUMMY_ATOMICS)
70
geo_pause()71 inline void geo_pause() {
72 }
73
atomic_bittestandset_x86(volatile unsigned int *,unsigned int)74 inline char atomic_bittestandset_x86(volatile unsigned int*, unsigned int) {
75 return 0;
76 }
77
atomic_bittestandreset_x86(volatile unsigned int *,unsigned int)78 inline char atomic_bittestandreset_x86(volatile unsigned int*, unsigned int) {
79 return 0;
80 }
81
82 #elif defined(GEO_USE_ANDROID_ATOMICS)
83
84 /** A mutex for Android */
85 typedef GEO::Numeric::uint32 android_mutex_t;
86
lock_mutex_android(volatile android_mutex_t * lock)87 inline void lock_mutex_android(volatile android_mutex_t* lock) {
88 while(__sync_lock_test_and_set(lock, 1) != 0);
89 }
90
unlock_mutex_android(volatile android_mutex_t * lock)91 inline void unlock_mutex_android(volatile android_mutex_t* lock) {
92 __sync_lock_release(lock);
93 }
94
atomic_bitset_android(volatile unsigned int * ptr,unsigned int bit)95 inline unsigned int atomic_bitset_android(volatile unsigned int* ptr, unsigned int bit) {
96 return __sync_fetch_and_or(ptr, 1u << bit) & (1u << bit);
97 }
98
atomic_bitreset_android(volatile unsigned int * ptr,unsigned int bit)99 inline unsigned int atomic_bitreset_android(volatile unsigned int* ptr, unsigned int bit) {
100 return __sync_fetch_and_and(ptr, ~(1u << bit)) & (1u << bit);
101 }
102
memory_barrier_android()103 inline void memory_barrier_android() {
104 // Full memory barrier.
105 __sync_synchronize();
106 }
107
wait_for_event_android()108 inline void wait_for_event_android() {
109 /* TODO */
110 }
111
send_event_android()112 inline void send_event_android() {
113 /* TODO */
114 }
115
116 #elif defined(GEO_USE_ARM32_ATOMICS)
117
118 /** A mutex for ARM processors */
119 typedef GEO::Numeric::uint32 arm32_mutex_t;
120
121 /**
122 * \brief Acquires a lock (ARM only)
123 * \param[in,out] lock the mutex to lock
124 */
lock_mutex_arm32(volatile arm32_mutex_t * lock)125 inline void lock_mutex_arm32(volatile arm32_mutex_t* lock) {
126 arm_mutex_t tmp;
127 __asm__ __volatile__ (
128 "1: ldrex %0, [%1] \n" // read lock
129 " cmp %0, #0 \n" // check if zero
130 " wfene \n" // wait for event if non-zero
131 " strexeq %0, %2, [%1] \n" // attempt to store new value
132 " cmpeq %0, #0 \n" // test if store succeeded
133 " bne 1b \n" // retry if not
134 " dmb \n" // memory barrier
135 : "=&r" (tmp)
136 : "r" (lock), "r" (1)
137 : "cc", "memory");
138 }
139
140 /**
141 * \brief Releases a lock (ARM only)
142 * \param[in,out] lock the mutex to unlock
143 */
unlock_mutex_arm32(volatile arm32_mutex_t * lock)144 inline void unlock_mutex_arm32(volatile arm32_mutex_t* lock) {
145 __asm__ __volatile__ (
146 " dmb \n" // ensure all previous access are observed
147 " str %1, [%0] \n" // clear the lock
148 " dsb \n" // ensure completion of clear lock ...
149 " sev \n" // ... before sending the event
150 :
151 : "r" (lock), "r" (0)
152 : "cc", "memory");
153 }
154
155 /**
156 * \brief Atomically tests and sets a bit (ARM only)
157 * \details Sets the bit \p bit of the *\p ptr.
158 * The function is atomic and acts as a read-write memory barrier.
159 * \param[in] ptr a pointer to an unsigned integer
160 * \param[in] bit index of the bit to set in *\p ptr
161 * \retval a non-zero integer if the bit was previously set
162 * \retval 0 if the bit was previously not set
163 */
atomic_bitset_arm32(volatile unsigned int * ptr,unsigned int bit)164 inline unsigned int atomic_bitset_arm32(volatile unsigned int* ptr, unsigned int bit) {
165 unsigned int tmp;
166 unsigned int result;
167 unsigned int OK;
168 __asm__ __volatile__ (
169 "1: ldrex %1, [%5] \n" // result = *ptr
170 " orr %0, %1, %6, LSL %4 \n" // tmp = result OR (1 << bit)
171 " strex %2, %0, [%5] \n" // *ptr = tmp, status in OK
172 " teq %2, #0 \n" // if !OK then
173 " bne 1b \n" // goto 1:
174 " and %1, %1, %6, LSL %4 \n" // result = result AND (1 << bit)
175 : "=&r" (tmp), "=&r" (result), "=&r" (OK), "+m" (*ptr)
176 : "r" (bit), "r" (ptr), "r" (1)
177 : "cc"
178 );
179 return result;
180 }
181
182 /**
183 * \brief Atomically tests and resets a bit (ARM only)
184 * \details Resets the bit \p bit of *\p ptr.
185 * The function is atomic and acts as a read-write memory barrier.
186 * \param[in] ptr a pointer to an unsigned integer
187 * \param[in] bit index of the bit to reset in *\p ptr
188 * \retval a non-zero integer if the bit was previously reset
189 * \retval 0 if the bit was previously not reset
190 */
atomic_bitreset_arm32(volatile unsigned int * ptr,unsigned int bit)191 inline unsigned int atomic_bitreset_arm32(volatile unsigned int* ptr, unsigned int bit) {
192 unsigned int tmp;
193 unsigned int result;
194 unsigned int OK;
195 __asm__ __volatile__ (
196 "1: ldrex %1, [%5] \n" // result = *ptr
197 " bic %0, %1, %6, LSL %4 \n" // tmp = result AND NOT(1 << bit)
198 " strex %2, %0, [%5] \n" // *ptr = tmp, status in OK
199 " teq %2, #0 \n" // if !OK then
200 " bne 1b \n" // goto 1:
201 " and %1, %1, %6, LSL %4 \n" // result = result AND (1 << bit)
202 : "=&r" (tmp), "=&r" (result), "=&r" (OK), "+m" (*ptr)
203 : "r" (bit), "r" (ptr), "r" (1)
204 : "cc"
205 );
206 return result;
207 }
208
209 /**
210 * \brief Issues a memory and compiler barrier (ARM only)
211 */
memory_barrier_arm32()212 inline void memory_barrier_arm32() {
213 __asm__ __volatile__ (
214 "dmb \n"
215 : : : "memory"
216 );
217 }
218
219 /**
220 * \brief Waits for an event (ARM only)
221 */
wait_for_event_arm32()222 inline void wait_for_event_arm32() {
223 __asm__ __volatile__ (
224 "wfe \n"
225 : : :
226 );
227 }
228
229 /**
230 * \brief Sends an event (ARM only)
231 */
send_event_arm32()232 inline void send_event_arm32() {
233 __asm__ __volatile__ (
234 "dsb \n" // ensure completion of store operations
235 "sev \n"
236 : : :
237 );
238 }
239
240 #elif defined(GEO_USE_X86_ATOMICS)
241
242 # define GEO_USE_X86_PAUSE
243
244 # ifdef GEO_USE_X86_PAUSE
245
246 /**
247 * \brief Issues a processor pause (INTEL only)
248 */
geo_pause()249 inline void geo_pause() {
250 __asm__ __volatile__ (
251 "pause;\n"
252 );
253 }
254
255 # else
256 # ifdef __ICC
257 # define geo_pause _mm_pause
258 # else
259 # define geo_pause __builtin_ia32_pause
260 # endif
261
262 # endif
263
264 /**
265 * \brief Atomically tests and sets a bit (INTEL only)
266 * \details Sets bit \p bit of *\p ptr and returns its previous value.
267 * The function is atomic and acts as a read-write memory barrier.
268 * \param[in] ptr a pointer to an unsigned integer
269 * \param[in] bit index of the bit to set in *\p ptr
270 * \return the previous value of bit \p bit
271 */
atomic_bittestandset_x86(volatile unsigned int * ptr,unsigned int bit)272 inline char atomic_bittestandset_x86(volatile unsigned int* ptr, unsigned int bit) {
273 char out;
274 #if defined(__x86_64)
275 __asm__ __volatile__ (
276 "lock; bts %2,%1\n" // set carry flag if bit %2 (bit) of %1 (ptr) is set
277 // then set bit %2 of %1
278 "sbb %0,%0\n" // set %0 (out) if carry flag is set
279 : "=r" (out), "=m" (*ptr)
280 : "Ir" (bit)
281 : "memory"
282 );
283 #else
284 __asm__ __volatile__ (
285 "lock; bts %2,%1\n" // set carry flag if bit %2 (bit) of %1 (ptr) is set
286 // then set bit %2 of %1
287 "sbb %0,%0\n" // set %0 (out) if carry flag is set
288 : "=q" (out), "=m" (*ptr)
289 : "Ir" (bit)
290 : "memory"
291 );
292 #endif
293 return out;
294 }
295
296 /**
297 * \brief Atomically tests and resets a bit (INTEL only)
298 * \details Resets bit \p bit of *\p ptr and returns its previous value.
299 * The function is atomic and acts as a read-write memory barrier
300 * \param[in] ptr a pointer to an unsigned integer
301 * \param[in] bit index of the bit to reset in \p ptr
302 * \return the previous value of bit \p bit
303 */
atomic_bittestandreset_x86(volatile unsigned int * ptr,unsigned int bit)304 inline char atomic_bittestandreset_x86(volatile unsigned int* ptr, unsigned int bit) {
305 char out;
306 #if defined(__x86_64)
307 __asm__ __volatile__ (
308 "lock; btr %2,%1\n" // set carry flag if bit %2 (bit) of %1 (ptr) is set
309 // then reset bit %2 of %1
310 "sbb %0,%0\n" // set %0 (out) if carry flag is set
311 : "=r" (out), "=m" (*ptr)
312 : "Ir" (bit)
313 : "memory"
314 );
315 #else
316 __asm__ __volatile__ (
317 "lock; btr %2,%1\n" // set carry flag if bit %2 (bit) of %1 (ptr) is set
318 // then reset bit %2 of %1
319 "sbb %0,%0\n" // set %0 (out) if carry flag is set
320 : "=q" (out), "=m" (*ptr)
321 : "Ir" (bit)
322 : "memory"
323 );
324 #endif
325 return out;
326 }
327
328 #elif defined(GEO_OS_APPLE)
329
330 #include <libkern/OSAtomic.h>
331
332 #elif defined(GEO_OS_WINDOWS)
333
334 #include <windows.h>
335 #include <intrin.h>
336 #pragma intrinsic(_InterlockedCompareExchange8)
337 #pragma intrinsic(_InterlockedCompareExchange16)
338 #pragma intrinsic(_InterlockedCompareExchange)
339 #pragma intrinsic(_interlockedbittestandset)
340 #pragma intrinsic(_interlockedbittestandreset)
341 #pragma intrinsic(_ReadBarrier)
342 #pragma intrinsic(_WriteBarrier)
343 #pragma intrinsic(_ReadWriteBarrier)
344
345 # ifdef GEO_COMPILER_MINGW
geo_pause()346 inline void geo_pause() {
347 }
348 # endif
349
350 #endif // GEO_OS_WINDOWS
351
352 #endif
353
354