1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2001-2020. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 /* Description: Error checking thread interface to the ethread library.
22 * All functions terminates the emulator on failure.
23 * Author: Rickard Green
24 */
25
26 #ifndef ERL_THREAD_H__
27 #define ERL_THREAD_H__
28
29 /*
30 * --- Documentation of atomics and memory barriers --------------------------
31 *
32 * The following explicit memory barriers exist:
33 *
34 * - ERTS_THR_MEMORY_BARRIER
35 * Full memory barrier. Orders both loads, and stores. No
36 * load or store is allowed to be reordered over the
37 * barrier.
38 * - ERTS_THR_WRITE_MEMORY_BARRIER
39 * Write barrier. Orders *only* stores. These are not
40 * allowed to be reordered over the barrier.
41 * - ERTS_THR_READ_MEMORY_BARRIER
42 * Read barrier. Orders *only* loads. These are not
43 * allowed to be reordered over the barrier.
44 * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
45 * Data dependency read barrier. Orders *only* loads
46 * according to data dependency across the barrier.
47 *
48 * --- Atomic operations ---
49 *
50 * Atomics operations exist for 32-bit, word size, and double word size
51 * integers. Function prototypes are listed below.
52 *
53 * Each function implementing an atomic operation exist with the following
54 * implied memory barrier semantics. Not all combinations are useful, but
55 * all of them exist for simplicity. <B> is suffix in function name:
56 *
57 * - <B> - Description
58 *
59 * - mb - Full memory barrier. Orders both loads, and
60 * stores before, and after the atomic operation.
61 * No load or store is allowed to be reordered
62 * over the atomic operation.
63 * - relb - Release barrier. Orders both loads, and
64 * stores appearing *before* the atomic
65 * operation. These are not allowed to be
66 * reordered over the atomic operation.
67 * - acqb - Acquire barrier. Orders both loads, and stores
68 * appearing *after* the atomic operation. These
69 * are not allowed to be reordered over the
70 * atomic operation.
71 * - wb - Write barrier. Orders *only* stores. These are
72 * not allowed to be reordered over the barrier.
73 * Store in atomic operation is ordered *after*
74 * the barrier.
75 * - rb - Read barrier. Orders *only* loads. These are
76 * not allowed to be reordered over the barrier.
77 * Load in atomic operation is ordered *before*
78 * the barrier.
79 * - ddrb - Data dependency read barrier. Orders *only*
80 * loads according to data dependency across the
81 * barrier. Load in atomic operation is ordered
82 * before the barrier.
83 *
84 * --- 32-bit atomic operations ---
85 *
86 * The following 32-bit atomic operations exist. <B> should be
87 * replaced with a supported memory barrier (see above). Note
88 * that sizeof(erts_atomic32_t) might be larger than 4!
89 *
90 *
91 * Initialize (not necessarily the same as the set operation):
92 * void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
93 * erts_aint32_t val);
94 *
95 * Set value:
96 * void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
97 * erts_aint32_t val);
98 *
99 * Read; returns current value:
100 * erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
101 *
102 * Increment; returns resulting value:
103 * erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
104 *
105 * Decrement; returns resulting value:
106 * erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
107 *
108 * Increment:
109 * void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
110 *
111 * Decrement:
112 * void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
113 *
114 * Add value; returns resulting value:
115 * erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
116 * erts_aint32_t val);
117 *
118 * Add value:
119 * void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
120 * erts_aint32_t val);
121 *
122 * Bitwise-or; returns previous value:
123 * erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
124 * erts_aint32_t val);
125 *
126 * Bitwise-and; returns previous value:
127 * erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
128 * erts_aint32_t val);
129 *
130 * Exchange; returns previous value:
131 * erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
132 * erts_aint32_t val);
133 *
134 * Compare and exchange; returns previous or current value. If
135 * returned value equals 'exp' the value was changed to 'new';
136 * otherwise not:
137 * erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
138 * erts_aint32_t new,
139 * erts_aint32_t exp);
140 *
141 * --- Word size atomic operations ---
142 *
143 * The following word size (same size as sizeof(void *)) atomic
144 * operations exist. <B> should be replaced with a supported
145 * memory barrier (see above). Note that sizeof(erts_atomic_t)
146 * might be larger than sizeof(void *)!
147 *
148 * Initialize (not necessarily the same as the set operation):
149 * void erts_atomic_init_<B>(erts_atomic_t *atmc,
150 * erts_aint_t val);
151 *
152 * Set value;
153 * void erts_atomic_set_<B>(erts_atomic_t *atmc,
154 * erts_aint_t val);
155 *
156 * Read; returns current value:
157 * erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
158 *
159 * Increment; returns resulting value:
160 * erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
161 *
162 * Decrement; returns resulting value:
163 * erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
164 *
165 * Increment:
166 * void erts_atomic_inc_<B>(erts_atomic_t *atmc);
167 *
168 * Decrement:
169 * void erts_atomic_dec_<B>(erts_atomic_t *atmc);
170 *
171 * Add value; returns resulting value:
172 * erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
173 * erts_aint_t val);
174 *
175 * Add value:
176 * void erts_atomic_add_<B>(erts_atomic_t *atmc,
177 * erts_aint_t val);
178 *
179 * Bitwise-or; returns previous value:
180 * erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
181 * erts_aint_t val);
182 *
183 * Bitwise-and; returns previous value:
184 * erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
185 * erts_aint_t val);
186 *
187 * Exchange; returns previous value:
188 * erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
189 * erts_aint_t val);
190 *
191 * Compare and exchange; returns previous or current value. If
192 * returned value equals 'exp' the value was changed to 'new';
193 * otherwise not:
194 * erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
195 * erts_aint_t new,
196 * erts_aint_t exp);
197 *
198 * --- Double word size atomic operations ---
199 *
200 * The following double word atomic operations exist. <B> should be
201 * replaced with a supported memory barrier (see above).
202 *
203 * Note that sizeof(erts_dw_atomic_t) usually is larger than
204 * 2*sizeof(void *)!
205 *
206 * The erts_dw_aint_t data type should be accessed as if it was defined
207 * like this:
208 *
209 * typedef struct {
210 * erts_aint_t sint[2];
211 * } erts_dw_aint_t;
212 *
213 * Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
214 * significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
215 *
216 *
217 * Initialize (not necessarily the same as the set operation):
218 * void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
219 * erts_dw_aint_t *val);
220 *
221 * Set; value is written into 'val':
222 * void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
223 * erts_dw_aint_t *val);
224 *
225 * Read; value is written into 'val':
226 * void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
227 * erts_dw_aint_t *val);
228 *
229 * Compare and exchange; returns a value != 0 if exchange was
230 * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
231 * contains the same value as in memory when the function is called,
232 * 'new' is written to memory; otherwise, not. If exchange was not
233 * made, 'exp_act' contains the actual value in memory:
234 * int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
235 * erts_dw_aint_t *new,
236 * erts_dw_aint_t *exp_act);
237 */
238
239 #define ERTS_SPIN_BODY ETHR_SPIN_BODY
240
241 #include "sys.h"
242
243 #include "erl_lock_flags.h"
244 #include "erl_term.h"
245
246
247 #define ETHR_TRY_INLINE_FUNCS
248 #include "ethread.h"
249
250 #include "erl_lock_check.h"
251 #include "erl_lock_count.h"
252 #include "erl_dyn_lock_check.h"
253
254 #if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 5
255 /*
256 * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have
257 * only seen this bug in glibc versions before 2.5. Note that condition
258 * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by
259 * this bug since these implementations may use mutexes internally.
260 */
261 # define ERTS_THR_HAVE_BUSY_DESTROY_BUG
262 #endif
263
264 #define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
265 #define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
266 #define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
267 #define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER
268
269 #ifdef ERTS_ENABLE_LOCK_POSITION
270 #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
271 #define erts_mtx_trylock(L) erts_mtx_trylock_x(L, __FILE__, __LINE__)
272 #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__)
273 #define erts_rwmtx_tryrlock(L) erts_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
274 #define erts_rwmtx_rlock(L) erts_rwmtx_rlock_x(L, __FILE__, __LINE__)
275 #define erts_rwmtx_tryrwlock(L) erts_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
276 #define erts_rwmtx_rwlock(L) erts_rwmtx_rwlock_x(L, __FILE__, __LINE__)
277 #define erts_read_lock(L) erts_read_lock_x(L, __FILE__, __LINE__)
278 #define erts_write_lock(L) erts_write_lock_x(L, __FILE__, __LINE__)
279 #endif
280
281 #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER
282 typedef ethr_thr_opts erts_thr_opts_t;
283 typedef ethr_init_data erts_thr_init_data_t;
284 typedef ethr_late_init_data erts_thr_late_init_data_t;
285 typedef ethr_tid erts_tid_t;
286
287 /* mutex */
288 typedef struct {
289 ethr_mutex mtx;
290 #ifdef ERTS_ENABLE_LOCK_CHECK
291 erts_lc_lock_t lc;
292 #endif
293 #ifdef ERTS_ENABLE_LOCK_COUNT
294 erts_lcnt_ref_t lcnt;
295 #endif
296 #ifdef DEBUG
297 erts_lock_flags_t flags;
298 #endif
299 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
300 erts_dlc_t dlc;
301 #endif
302 } erts_mtx_t;
303 typedef ethr_cond erts_cnd_t;
304
305 /* rwmutex */
306 typedef struct {
307 ethr_rwmutex rwmtx;
308 #ifdef ERTS_ENABLE_LOCK_CHECK
309 erts_lc_lock_t lc;
310 #endif
311 #ifdef ERTS_ENABLE_LOCK_COUNT
312 erts_lcnt_ref_t lcnt;
313 #endif
314 #ifdef DEBUG
315 erts_lock_flags_t flags;
316 #endif
317 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
318 erts_dlc_t dlc;
319 #endif
320 } erts_rwmtx_t;
321
322 #define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER
323 #define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER
324 #define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER
325 #define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL
326 #define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ
327 #define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
328 ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
329 #define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED
330 #define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED
331 #define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED
332 typedef ethr_rwmutex_opt erts_rwmtx_opt_t;
333
334 typedef ethr_tsd_key erts_tsd_key_t;
335 typedef ethr_ts_event erts_tse_t;
336 #define erts_dw_aint_t ethr_dw_sint_t
337 #define erts_dw_atomic_t ethr_dw_atomic_t
338 #define erts_aint_t ethr_sint_t
339 #define erts_atomic_t ethr_atomic_t
340 #define erts_aint32_t ethr_sint32_t
341 #define erts_atomic32_t ethr_atomic32_t
342
343 #if defined(ARCH_32)
344 # define erts_atomic64_t ethr_dw_atomic_t
345 # define erts_aint64_t ethr_sint64_t
346 #elif defined(ARCH_64)
347 # define erts_atomic64_t ethr_atomic_t
348 # define erts_aint64_t ethr_sint_t
349 #else
350 # error "Not supported architecture"
351 #endif
352
353 #define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
354 #define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD
355
356 /* spinlock */
357 typedef struct {
358 ethr_spinlock_t slck;
359 #ifdef ERTS_ENABLE_LOCK_CHECK
360 erts_lc_lock_t lc;
361 #endif
362 #ifdef ERTS_ENABLE_LOCK_COUNT
363 erts_lcnt_ref_t lcnt;
364 #endif
365 #ifdef DEBUG
366 erts_lock_flags_t flags;
367 #endif
368 } erts_spinlock_t;
369
370 /* rwlock */
371 typedef struct {
372 ethr_rwlock_t rwlck;
373 #ifdef ERTS_ENABLE_LOCK_CHECK
374 erts_lc_lock_t lc;
375 #endif
376 #ifdef ERTS_ENABLE_LOCK_COUNT
377 erts_lcnt_ref_t lcnt;
378 #endif
379 #ifdef DEBUG
380 erts_lock_flags_t flags;
381 #endif
382 } erts_rwlock_t;
383
384 __decl_noreturn void __noreturn erts_thr_fatal_error(int, const char *);
385 /* implemented in erl_init.c */
386
387 #define ERTS_THR_INIT_DATA_DEF_INITER ETHR_INIT_DATA_DEFAULT_INITER
388 #define ERTS_THR_LATE_INIT_DATA_DEF_INITER \
389 ETHR_LATE_INIT_DATA_DEFAULT_INITER
390
391 #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
392 # define ERTS_HAVE_REC_MTX_INIT ETHR_HAVE_ETHR_REC_MUTEX_INIT
393 #endif
394
395 #define ERTS_AINT_NULL ((erts_aint_t) NULL)
396
397 #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
398 #define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
399 #define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
400 #define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
401
402 ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id);
403 ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id);
404 ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *),
405 void *arg, erts_thr_opts_t *opts);
406 ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res);
407 ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid);
408 ERTS_GLB_INLINE void erts_thr_exit(void *res);
409 ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
410 ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
411 ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
412 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
413 ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx,
414 const char *name,
415 Eterm extra,
416 erts_lock_flags_t flags);
417 ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx,
418 char *name,
419 Eterm extra,
420 erts_lock_flags_t flags);
421 ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
422 #ifdef ERTS_ENABLE_LOCK_POSITION
423 ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, const char *file,
424 unsigned int line);
425 ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, const char *file,
426 unsigned int line);
427 #else
428 ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
429 ERTS_GLB_INLINE void erts_mtx_lock(erts_mtx_t *mtx);
430 #endif
431 ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx);
432 ERTS_GLB_INLINE int erts_lc_mtx_is_locked(erts_mtx_t *mtx);
433 ERTS_GLB_INLINE void erts_cnd_init(erts_cnd_t *cnd);
434 ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd);
435 ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx);
436 ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd);
437 ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd);
438 ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no);
439 ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
440 erts_rwmtx_opt_t *opt,
441 char *name,
442 Eterm extra,
443 erts_lock_flags_t flags);
444 ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
445 char *name,
446 Eterm extra,
447 erts_lock_flags_t flags);
448 ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
449 #ifdef ERTS_ENABLE_LOCK_POSITION
450 ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line);
451 ERTS_GLB_INLINE void erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line);
452 ERTS_GLB_INLINE void erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line);
453 ERTS_GLB_INLINE int erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line);
454 #else
455 ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx);
456 ERTS_GLB_INLINE void erts_rwmtx_rlock(erts_rwmtx_t *rwmtx);
457 ERTS_GLB_INLINE void erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx);
458 ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
459 #endif
460 ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx);
461 ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
462 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
463 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);
464 ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
465 char *name,
466 Eterm extra,
467 erts_lock_flags_t flags);
468 ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock);
469 ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock);
470 #ifdef ERTS_ENABLE_LOCK_POSITION
471 ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, const char *file, unsigned int line);
472 #else
473 ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock);
474 #endif
475 ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock);
476 ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock,
477 char *name,
478 Eterm extra,
479 erts_lock_flags_t flags);
480 ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock);
481 ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock);
482 #ifdef ERTS_ENABLE_LOCK_POSITION
483 ERTS_GLB_INLINE void erts_read_lock_x(erts_rwlock_t *lock, const char *file, unsigned int line);
484 ERTS_GLB_INLINE void erts_write_lock_x(erts_rwlock_t *lock, const char *file, unsigned int line);
485 #else
486 ERTS_GLB_INLINE void erts_read_lock(erts_rwlock_t *lock);
487 ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock);
488 #endif
489 ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock);
490 ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock);
491 ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock);
492 ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname);
493 ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key);
494 ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
495 ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
496 ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void);
497 ERTS_GLB_INLINE void erts_tse_use(erts_tse_t *ep);
498 ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep);
499 ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep);
500 ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
501 ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
502 ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
503 ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount);
504 ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo);
505 ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo);
506 ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep);
507 ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
508 ERTS_GLB_INLINE int erts_thr_get_main_status(void);
509 ERTS_GLB_INLINE void erts_thr_yield(void);
510
511
512 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
513 #define ERTS_THR_HAVE_SIG_FUNCS 1
514 ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
515 sigset_t *oset);
516 ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
517
518 ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig);
519
520 #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
521
522
523 ERTS_GLB_INLINE erts_aint_t
524 erts_atomic_read_bset_nob(erts_atomic_t *var,
525 erts_aint_t mask,
526 erts_aint_t set);
527 ERTS_GLB_INLINE erts_aint_t
528 erts_atomic_read_bset_ddrb(erts_atomic_t *var,
529 erts_aint_t mask,
530 erts_aint_t set);
531 ERTS_GLB_INLINE erts_aint_t
532 erts_atomic_read_bset_rb(erts_atomic_t *var,
533 erts_aint_t mask,
534 erts_aint_t set);
535 ERTS_GLB_INLINE erts_aint_t
536 erts_atomic_read_bset_wb(erts_atomic_t *var,
537 erts_aint_t mask,
538 erts_aint_t set);
539 ERTS_GLB_INLINE erts_aint_t
540 erts_atomic_read_bset_acqb(erts_atomic_t *var,
541 erts_aint_t mask,
542 erts_aint_t set);
543 ERTS_GLB_INLINE erts_aint_t
544 erts_atomic_read_bset_relb(erts_atomic_t *var,
545 erts_aint_t mask,
546 erts_aint_t set);
547 ERTS_GLB_INLINE erts_aint_t
548 erts_atomic_read_bset_mb(erts_atomic_t *var,
549 erts_aint_t mask,
550 erts_aint_t set);
551 ERTS_GLB_INLINE erts_aint32_t
552 erts_atomic32_read_bset_nob(erts_atomic32_t *var,
553 erts_aint32_t mask,
554 erts_aint32_t set);
555 ERTS_GLB_INLINE erts_aint32_t
556 erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
557 erts_aint32_t mask,
558 erts_aint32_t set);
559 ERTS_GLB_INLINE erts_aint32_t
560 erts_atomic32_read_bset_rb(erts_atomic32_t *var,
561 erts_aint32_t mask,
562 erts_aint32_t set);
563 ERTS_GLB_INLINE erts_aint32_t
564 erts_atomic32_read_bset_wb(erts_atomic32_t *var,
565 erts_aint32_t mask,
566 erts_aint32_t set);
567 ERTS_GLB_INLINE erts_aint32_t
568 erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
569 erts_aint32_t mask,
570 erts_aint32_t set);
571 ERTS_GLB_INLINE erts_aint32_t
572 erts_atomic32_read_bset_relb(erts_atomic32_t *var,
573 erts_aint32_t mask,
574 erts_aint32_t set);
575 ERTS_GLB_INLINE erts_aint32_t
576 erts_atomic32_read_bset_mb(erts_atomic32_t *var,
577 erts_aint32_t mask,
578 erts_aint32_t set);
579
580 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
581 #define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \
582 do { \
583 Type act = ReadOp((VarP)); \
584 while (1) { \
585 Type exp = act; \
586 Type new_value = exp & ~(Mask); \
587 new_value |= ((Mask) & (Set)); \
588 act = CmpxchgOp((VarP), new_value, exp); \
589 if (act == exp) \
590 return act; \
591 } \
592 } while (0)
593 #endif
594
595 ERTS_GLB_INLINE void
596 erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
597 ERTS_GLB_INLINE void
598 erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
599 ERTS_GLB_INLINE void
600 erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val);
601 ERTS_GLB_INLINE erts_aint_t
602 erts_atomic_read_dirty(erts_atomic_t *var);
603 ERTS_GLB_INLINE void
604 erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val);
605 ERTS_GLB_INLINE erts_aint32_t
606 erts_atomic32_read_dirty(erts_atomic32_t *var);
607
608 /*
609 * See "Documentation of atomics and memory barriers" at the top
610 * of this file for info on atomics.
611 */
612
613 /* Double word size atomics */
614
615 #define erts_dw_atomic_init_nob ethr_dw_atomic_init
616 #define erts_dw_atomic_set_nob ethr_dw_atomic_set
617 #define erts_dw_atomic_read_nob ethr_dw_atomic_read
618 #define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg
619
620 #define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb
621 #define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb
622 #define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb
623 #define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb
624
625 #define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb
626 #define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb
627 #define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb
628 #define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb
629
630 #define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb
631 #define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb
632 #define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
633 #define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb
634
635 #define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
636 #define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
637 #define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
638 #define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb
639
640 #define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
641 #define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
642 #define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
643 #define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb
644
645 #define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb
646 #define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb
647 #define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
648 #define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb
649
650 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
651
652 ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t * var,erts_dw_aint_t * val)653 erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
654 {
655 ethr_sint_t *sint = ethr_dw_atomic_addr(var);
656 sint[0] = val->sint[0];
657 sint[1] = val->sint[1];
658 }
659
660 ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t * var,erts_dw_aint_t * val)661 erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
662 {
663 ethr_sint_t *sint = ethr_dw_atomic_addr(var);
664 val->sint[0] = sint[0];
665 val->sint[1] = sint[1];
666 }
667
668 #endif
669
670 /* Word size atomics */
671
672 #define erts_atomic_init_nob ethr_atomic_init
673 #define erts_atomic_set_nob ethr_atomic_set
674 #define erts_atomic_read_nob ethr_atomic_read
675 #define erts_atomic_inc_read_nob ethr_atomic_inc_read
676 #define erts_atomic_dec_read_nob ethr_atomic_dec_read
677 #define erts_atomic_inc_nob ethr_atomic_inc
678 #define erts_atomic_dec_nob ethr_atomic_dec
679 #define erts_atomic_add_read_nob ethr_atomic_add_read
680 #define erts_atomic_add_nob ethr_atomic_add
681 #define erts_atomic_read_bor_nob ethr_atomic_read_bor
682 #define erts_atomic_read_band_nob ethr_atomic_read_band
683 #define erts_atomic_xchg_nob ethr_atomic_xchg
684 #define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg
685
686 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
687 ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t * var,erts_aint_t mask,erts_aint_t set)688 erts_atomic_read_bset_nob(erts_atomic_t *var,
689 erts_aint_t mask,
690 erts_aint_t set)
691 {
692 ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
693 ethr_atomic_read,
694 ethr_atomic_cmpxchg,
695 var, mask, set);
696 }
697 #endif
698
699 #define erts_atomic_init_mb ethr_atomic_init_mb
700 #define erts_atomic_set_mb ethr_atomic_set_mb
701 #define erts_atomic_read_mb ethr_atomic_read_mb
702 #define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb
703 #define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb
704 #define erts_atomic_inc_mb ethr_atomic_inc_mb
705 #define erts_atomic_dec_mb ethr_atomic_dec_mb
706 #define erts_atomic_add_read_mb ethr_atomic_add_read_mb
707 #define erts_atomic_add_mb ethr_atomic_add_mb
708 #define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb
709 #define erts_atomic_read_band_mb ethr_atomic_read_band_mb
710 #define erts_atomic_xchg_mb ethr_atomic_xchg_mb
711 #define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb
712
713 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
714 ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t * var,erts_aint_t mask,erts_aint_t set)715 erts_atomic_read_bset_mb(erts_atomic_t *var,
716 erts_aint_t mask,
717 erts_aint_t set)
718 {
719 ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
720 ethr_atomic_read,
721 ethr_atomic_cmpxchg_mb,
722 var, mask, set);
723 }
724 #endif
725
726 #define erts_atomic_init_acqb ethr_atomic_init_acqb
727 #define erts_atomic_set_acqb ethr_atomic_set_acqb
728 #define erts_atomic_read_acqb ethr_atomic_read_acqb
729 #define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb
730 #define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb
731 #define erts_atomic_inc_acqb ethr_atomic_inc_acqb
732 #define erts_atomic_dec_acqb ethr_atomic_dec_acqb
733 #define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb
734 #define erts_atomic_add_acqb ethr_atomic_add_acqb
735 #define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb
736 #define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb
737 #define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
738 #define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
739
740 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
741 ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t * var,erts_aint_t mask,erts_aint_t set)742 erts_atomic_read_bset_acqb(erts_atomic_t *var,
743 erts_aint_t mask,
744 erts_aint_t set)
745 {
746 ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
747 ethr_atomic_read,
748 ethr_atomic_cmpxchg_acqb,
749 var, mask, set);
750 }
751 #endif
752
753 #define erts_atomic_init_relb ethr_atomic_init_relb
754 #define erts_atomic_set_relb ethr_atomic_set_relb
755 #define erts_atomic_read_relb ethr_atomic_read_relb
756 #define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb
757 #define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb
758 #define erts_atomic_inc_relb ethr_atomic_inc_relb
759 #define erts_atomic_dec_relb ethr_atomic_dec_relb
760 #define erts_atomic_add_read_relb ethr_atomic_add_read_relb
761 #define erts_atomic_add_relb ethr_atomic_add_relb
762 #define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb
763 #define erts_atomic_read_band_relb ethr_atomic_read_band_relb
764 #define erts_atomic_xchg_relb ethr_atomic_xchg_relb
765 #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
766
767 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
768 ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t * var,erts_aint_t mask,erts_aint_t set)769 erts_atomic_read_bset_relb(erts_atomic_t *var,
770 erts_aint_t mask,
771 erts_aint_t set)
772 {
773 ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
774 ethr_atomic_read,
775 ethr_atomic_cmpxchg_relb,
776 var, mask, set);
777 }
778 #endif
779
780 #define erts_atomic_init_ddrb ethr_atomic_init_ddrb
781 #define erts_atomic_set_ddrb ethr_atomic_set_ddrb
782 #define erts_atomic_read_ddrb ethr_atomic_read_ddrb
783 #define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
784 #define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
785 #define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
786 #define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
787 #define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
788 #define erts_atomic_add_ddrb ethr_atomic_add_ddrb
789 #define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
790 #define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
791 #define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
792 #define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
793
794 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
795 ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t * var,erts_aint_t mask,erts_aint_t set)796 erts_atomic_read_bset_ddrb(erts_atomic_t *var,
797 erts_aint_t mask,
798 erts_aint_t set)
799 {
800 ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
801 ethr_atomic_read,
802 ethr_atomic_cmpxchg_ddrb,
803 var, mask, set);
804 }
805 #endif
806
807 #define erts_atomic_init_rb ethr_atomic_init_rb
808 #define erts_atomic_set_rb ethr_atomic_set_rb
809 #define erts_atomic_read_rb ethr_atomic_read_rb
810 #define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb
811 #define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb
812 #define erts_atomic_inc_rb ethr_atomic_inc_rb
813 #define erts_atomic_dec_rb ethr_atomic_dec_rb
814 #define erts_atomic_add_read_rb ethr_atomic_add_read_rb
815 #define erts_atomic_add_rb ethr_atomic_add_rb
816 #define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb
817 #define erts_atomic_read_band_rb ethr_atomic_read_band_rb
818 #define erts_atomic_xchg_rb ethr_atomic_xchg_rb
819 #define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb
820
821 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
822 ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t * var,erts_aint_t mask,erts_aint_t set)823 erts_atomic_read_bset_rb(erts_atomic_t *var,
824 erts_aint_t mask,
825 erts_aint_t set)
826 {
827 ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
828 ethr_atomic_read,
829 ethr_atomic_cmpxchg_rb,
830 var, mask, set);
831 }
832 #endif
833
834 #define erts_atomic_init_wb ethr_atomic_init_wb
835 #define erts_atomic_set_wb ethr_atomic_set_wb
836 #define erts_atomic_read_wb ethr_atomic_read_wb
837 #define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb
838 #define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb
839 #define erts_atomic_inc_wb ethr_atomic_inc_wb
840 #define erts_atomic_dec_wb ethr_atomic_dec_wb
841 #define erts_atomic_add_read_wb ethr_atomic_add_read_wb
842 #define erts_atomic_add_wb ethr_atomic_add_wb
843 #define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb
844 #define erts_atomic_read_band_wb ethr_atomic_read_band_wb
845 #define erts_atomic_xchg_wb ethr_atomic_xchg_wb
846 #define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb
847
848 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
849
850 ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t * var,erts_aint_t mask,erts_aint_t set)851 erts_atomic_read_bset_wb(erts_atomic_t *var,
852 erts_aint_t mask,
853 erts_aint_t set)
854 {
855 ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
856 ethr_atomic_read,
857 ethr_atomic_cmpxchg_wb,
858 var, mask, set);
859 }
860
861 #endif
862
863 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
864
865 ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t * var,erts_aint_t val)866 erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val)
867 {
868 ethr_sint_t *sint = ethr_atomic_addr(var);
869 *sint = val;
870 }
871
872 ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t * var)873 erts_atomic_read_dirty(erts_atomic_t *var)
874 {
875 ethr_sint_t *sint = ethr_atomic_addr(var);
876 return *sint;
877 }
878
879 #endif
880
881 /* 32-bit atomics */
882
883 #define erts_atomic32_init_nob ethr_atomic32_init
884 #define erts_atomic32_set_nob ethr_atomic32_set
885 #define erts_atomic32_read_nob ethr_atomic32_read
886 #define erts_atomic32_inc_read_nob ethr_atomic32_inc_read
887 #define erts_atomic32_dec_read_nob ethr_atomic32_dec_read
888 #define erts_atomic32_inc_nob ethr_atomic32_inc
889 #define erts_atomic32_dec_nob ethr_atomic32_dec
890 #define erts_atomic32_add_read_nob ethr_atomic32_add_read
891 #define erts_atomic32_add_nob ethr_atomic32_add
892 #define erts_atomic32_read_bor_nob ethr_atomic32_read_bor
893 #define erts_atomic32_read_band_nob ethr_atomic32_read_band
894 #define erts_atomic32_xchg_nob ethr_atomic32_xchg
895 #define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg
896
897 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
898 ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t * var,erts_aint32_t mask,erts_aint32_t set)899 erts_atomic32_read_bset_nob(erts_atomic32_t *var,
900 erts_aint32_t mask,
901 erts_aint32_t set)
902 {
903 ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
904 ethr_atomic32_read,
905 ethr_atomic32_cmpxchg,
906 var, mask, set);
907 }
908 #endif
909
910 #define erts_atomic32_init_mb ethr_atomic32_init_mb
911 #define erts_atomic32_set_mb ethr_atomic32_set_mb
912 #define erts_atomic32_read_mb ethr_atomic32_read_mb
913 #define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb
914 #define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb
915 #define erts_atomic32_inc_mb ethr_atomic32_inc_mb
916 #define erts_atomic32_dec_mb ethr_atomic32_dec_mb
917 #define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb
918 #define erts_atomic32_add_mb ethr_atomic32_add_mb
919 #define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb
920 #define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb
921 #define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
922 #define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb
923
924 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
925 ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t * var,erts_aint32_t mask,erts_aint32_t set)926 erts_atomic32_read_bset_mb(erts_atomic32_t *var,
927 erts_aint32_t mask,
928 erts_aint32_t set)
929 {
930 ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
931 ethr_atomic32_read,
932 ethr_atomic32_cmpxchg_mb,
933 var, mask, set);
934 }
935 #endif
936
937 #define erts_atomic32_init_acqb ethr_atomic32_init_acqb
938 #define erts_atomic32_set_acqb ethr_atomic32_set_acqb
939 #define erts_atomic32_read_acqb ethr_atomic32_read_acqb
940 #define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb
941 #define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb
942 #define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb
943 #define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb
944 #define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb
945 #define erts_atomic32_add_acqb ethr_atomic32_add_acqb
946 #define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb
947 #define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb
948 #define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
949 #define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb
950
951 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
952 ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t * var,erts_aint32_t mask,erts_aint32_t set)953 erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
954 erts_aint32_t mask,
955 erts_aint32_t set)
956 {
957 ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
958 ethr_atomic32_read,
959 ethr_atomic32_cmpxchg_acqb,
960 var, mask, set);
961 }
962 #endif
963
964 #define erts_atomic32_init_relb ethr_atomic32_init_relb
965 #define erts_atomic32_set_relb ethr_atomic32_set_relb
966 #define erts_atomic32_read_relb ethr_atomic32_read_relb
967 #define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb
968 #define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb
969 #define erts_atomic32_inc_relb ethr_atomic32_inc_relb
970 #define erts_atomic32_dec_relb ethr_atomic32_dec_relb
971 #define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb
972 #define erts_atomic32_add_relb ethr_atomic32_add_relb
973 #define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb
974 #define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb
975 #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
976 #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
977
978 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
979 ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t * var,erts_aint32_t mask,erts_aint32_t set)980 erts_atomic32_read_bset_relb(erts_atomic32_t *var,
981 erts_aint32_t mask,
982 erts_aint32_t set)
983 {
984 ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
985 ethr_atomic32_read,
986 ethr_atomic32_cmpxchg_relb,
987 var, mask, set);
988 }
989 #endif
990
991 #define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
992 #define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
993 #define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
994 #define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
995 #define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
996 #define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
997 #define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
998 #define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
999 #define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
1000 #define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
1001 #define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
1002 #define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
1003 #define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb
1004
1005 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
1006 ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t * var,erts_aint32_t mask,erts_aint32_t set)1007 erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
1008 erts_aint32_t mask,
1009 erts_aint32_t set)
1010 {
1011 ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
1012 ethr_atomic32_read,
1013 ethr_atomic32_cmpxchg_ddrb,
1014 var, mask, set);
1015 }
1016 #endif
1017
1018 #define erts_atomic32_init_rb ethr_atomic32_init_rb
1019 #define erts_atomic32_set_rb ethr_atomic32_set_rb
1020 #define erts_atomic32_read_rb ethr_atomic32_read_rb
1021 #define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb
1022 #define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb
1023 #define erts_atomic32_inc_rb ethr_atomic32_inc_rb
1024 #define erts_atomic32_dec_rb ethr_atomic32_dec_rb
1025 #define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb
1026 #define erts_atomic32_add_rb ethr_atomic32_add_rb
1027 #define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb
1028 #define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb
1029 #define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
1030 #define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb
1031
1032 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
1033 ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t * var,erts_aint32_t mask,erts_aint32_t set)1034 erts_atomic32_read_bset_rb(erts_atomic32_t *var,
1035 erts_aint32_t mask,
1036 erts_aint32_t set)
1037 {
1038 ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
1039 ethr_atomic32_read,
1040 ethr_atomic32_cmpxchg_rb,
1041 var, mask, set);
1042 }
1043 #endif
1044
1045 #define erts_atomic32_init_wb ethr_atomic32_init_wb
1046 #define erts_atomic32_set_wb ethr_atomic32_set_wb
1047 #define erts_atomic32_read_wb ethr_atomic32_read_wb
1048 #define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb
1049 #define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb
1050 #define erts_atomic32_inc_wb ethr_atomic32_inc_wb
1051 #define erts_atomic32_dec_wb ethr_atomic32_dec_wb
1052 #define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb
1053 #define erts_atomic32_add_wb ethr_atomic32_add_wb
1054 #define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb
1055 #define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb
1056 #define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
1057 #define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb
1058
1059 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
1060
1061 ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t * var,erts_aint32_t mask,erts_aint32_t set)1062 erts_atomic32_read_bset_wb(erts_atomic32_t *var,
1063 erts_aint32_t mask,
1064 erts_aint32_t set)
1065 {
1066 ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
1067 ethr_atomic32_read,
1068 ethr_atomic32_cmpxchg_wb,
1069 var, mask, set);
1070 }
1071
1072 #endif
1073
1074 #undef ERTS_ATOMIC_BSET_IMPL__
1075
1076 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
1077
1078 ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t * var,erts_aint32_t val)1079 erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val)
1080 {
1081 ethr_sint32_t *sint = ethr_atomic32_addr(var);
1082 *sint = val;
1083 }
1084
1085 ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t * var)1086 erts_atomic32_read_dirty(erts_atomic32_t *var)
1087 {
1088 ethr_sint32_t *sint = ethr_atomic32_addr(var);
1089 return *sint;
1090 }
1091
1092 #endif
1093
1094 /* 64-bit atomics */
1095
1096 #if defined(ARCH_64)
1097
1098 #define erts_atomic64_init_nob ethr_atomic_init
1099 #define erts_atomic64_set_nob ethr_atomic_set
1100 #define erts_atomic64_read_nob ethr_atomic_read
1101 #define erts_atomic64_inc_read_nob ethr_atomic_inc_read
1102 #define erts_atomic64_dec_read_nob ethr_atomic_dec_read
1103 #define erts_atomic64_inc_nob ethr_atomic_inc
1104 #define erts_atomic64_dec_nob ethr_atomic_dec
1105 #define erts_atomic64_add_read_nob ethr_atomic_add_read
1106 #define erts_atomic64_add_nob ethr_atomic_add
1107 #define erts_atomic64_read_bor_nob ethr_atomic_read_bor
1108 #define erts_atomic64_read_band_nob ethr_atomic_read_band
1109 #define erts_atomic64_xchg_nob ethr_atomic_xchg
1110 #define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg
1111 #define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob
1112
1113 #define erts_atomic64_init_mb ethr_atomic_init_mb
1114 #define erts_atomic64_set_mb ethr_atomic_set_mb
1115 #define erts_atomic64_read_mb ethr_atomic_read_mb
1116 #define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb
1117 #define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb
1118 #define erts_atomic64_inc_mb ethr_atomic_inc_mb
1119 #define erts_atomic64_dec_mb ethr_atomic_dec_mb
1120 #define erts_atomic64_add_read_mb ethr_atomic_add_read_mb
1121 #define erts_atomic64_add_mb ethr_atomic_add_mb
1122 #define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb
1123 #define erts_atomic64_read_band_mb ethr_atomic_read_band_mb
1124 #define erts_atomic64_xchg_mb ethr_atomic_xchg_mb
1125 #define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb
1126 #define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb
1127
1128 #define erts_atomic64_init_acqb ethr_atomic_init_acqb
1129 #define erts_atomic64_set_acqb ethr_atomic_set_acqb
1130 #define erts_atomic64_read_acqb ethr_atomic_read_acqb
1131 #define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb
1132 #define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb
1133 #define erts_atomic64_inc_acqb ethr_atomic_inc_acqb
1134 #define erts_atomic64_dec_acqb ethr_atomic_dec_acqb
1135 #define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb
1136 #define erts_atomic64_add_acqb ethr_atomic_add_acqb
1137 #define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb
1138 #define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb
1139 #define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb
1140 #define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
1141 #define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb
1142
1143 #define erts_atomic64_init_relb ethr_atomic_init_relb
1144 #define erts_atomic64_set_relb ethr_atomic_set_relb
1145 #define erts_atomic64_read_relb ethr_atomic_read_relb
1146 #define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb
1147 #define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb
1148 #define erts_atomic64_inc_relb ethr_atomic_inc_relb
1149 #define erts_atomic64_dec_relb ethr_atomic_dec_relb
1150 #define erts_atomic64_add_read_relb ethr_atomic_add_read_relb
1151 #define erts_atomic64_add_relb ethr_atomic_add_relb
1152 #define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb
1153 #define erts_atomic64_read_band_relb ethr_atomic_read_band_relb
1154 #define erts_atomic64_xchg_relb ethr_atomic_xchg_relb
1155 #define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb
1156 #define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb
1157
1158 #define erts_atomic64_init_ddrb ethr_atomic_init_ddrb
1159 #define erts_atomic64_set_ddrb ethr_atomic_set_ddrb
1160 #define erts_atomic64_read_ddrb ethr_atomic_read_ddrb
1161 #define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb
1162 #define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb
1163 #define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb
1164 #define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb
1165 #define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb
1166 #define erts_atomic64_add_ddrb ethr_atomic_add_ddrb
1167 #define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb
1168 #define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb
1169 #define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb
1170 #define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
1171 #define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb
1172
1173 #define erts_atomic64_init_rb ethr_atomic_init_rb
1174 #define erts_atomic64_set_rb ethr_atomic_set_rb
1175 #define erts_atomic64_read_rb ethr_atomic_read_rb
1176 #define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb
1177 #define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb
1178 #define erts_atomic64_inc_rb ethr_atomic_inc_rb
1179 #define erts_atomic64_dec_rb ethr_atomic_dec_rb
1180 #define erts_atomic64_add_read_rb ethr_atomic_add_read_rb
1181 #define erts_atomic64_add_rb ethr_atomic_add_rb
1182 #define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb
1183 #define erts_atomic64_read_band_rb ethr_atomic_read_band_rb
1184 #define erts_atomic64_xchg_rb ethr_atomic_xchg_rb
1185 #define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb
1186 #define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb
1187
1188 #define erts_atomic64_init_wb ethr_atomic_init_wb
1189 #define erts_atomic64_set_wb ethr_atomic_set_wb
1190 #define erts_atomic64_read_wb ethr_atomic_read_wb
1191 #define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb
1192 #define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb
1193 #define erts_atomic64_inc_wb ethr_atomic_inc_wb
1194 #define erts_atomic64_dec_wb ethr_atomic_dec_wb
1195 #define erts_atomic64_add_read_wb ethr_atomic_add_read_wb
1196 #define erts_atomic64_add_wb ethr_atomic_add_wb
1197 #define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb
1198 #define erts_atomic64_read_band_wb ethr_atomic_read_band_wb
1199 #define erts_atomic64_xchg_wb ethr_atomic_xchg_wb
1200 #define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb
1201 #define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb
1202
1203 #define erts_atomic64_set_dirty erts_atomic_set_dirty
1204 #define erts_atomic64_read_dirty erts_atomic_read_dirty
1205
1206 #elif defined(ARCH_32)
1207
1208 #undef ERTS_ATOMIC64_OPS_DECL__
1209
1210 #define ERTS_ATOMIC64_OPS_DECL__(BARRIER) \
1211 ERTS_GLB_INLINE void \
1212 erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \
1213 erts_aint64_t val); \
1214 ERTS_GLB_INLINE void \
1215 erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \
1216 erts_aint64_t val); \
1217 ERTS_GLB_INLINE erts_aint64_t \
1218 erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var); \
1219 ERTS_GLB_INLINE erts_aint64_t \
1220 erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var); \
1221 ERTS_GLB_INLINE erts_aint64_t \
1222 erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var); \
1223 ERTS_GLB_INLINE void \
1224 erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var); \
1225 ERTS_GLB_INLINE void \
1226 erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var); \
1227 ERTS_GLB_INLINE erts_aint64_t \
1228 erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \
1229 erts_aint64_t val); \
1230 ERTS_GLB_INLINE void \
1231 erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \
1232 erts_aint64_t val); \
1233 ERTS_GLB_INLINE erts_aint64_t \
1234 erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \
1235 erts_aint64_t val); \
1236 ERTS_GLB_INLINE erts_aint64_t \
1237 erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \
1238 erts_aint64_t val); \
1239 ERTS_GLB_INLINE erts_aint64_t \
1240 erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \
1241 erts_aint64_t val); \
1242 ERTS_GLB_INLINE erts_aint64_t \
1243 erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \
1244 erts_aint64_t new_value, \
1245 erts_aint64_t exp); \
1246 ERTS_GLB_INLINE erts_aint64_t \
1247 erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \
1248 erts_aint64_t mask, \
1249 erts_aint64_t set)
1250
1251 ERTS_ATOMIC64_OPS_DECL__(nob);
1252 ERTS_ATOMIC64_OPS_DECL__(mb);
1253 ERTS_ATOMIC64_OPS_DECL__(acqb);
1254 ERTS_ATOMIC64_OPS_DECL__(relb);
1255 ERTS_ATOMIC64_OPS_DECL__(ddrb);
1256 ERTS_ATOMIC64_OPS_DECL__(rb);
1257 ERTS_ATOMIC64_OPS_DECL__(wb);
1258
1259 #undef ERTS_ATOMIC64_OPS_DECL__
1260
1261 ERTS_GLB_INLINE void
1262 erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val);
1263 ERTS_GLB_INLINE erts_aint64_t
1264 erts_atomic64_read_dirty(erts_atomic64_t *var);
1265
1266 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
1267
1268 /*
1269 * The ethr_dw_atomic_*_nob() functions below
1270 * are here to make it possible for the
1271 * ERTS_ATOMIC64_OPS_IMPL__() to map erts
1272 * barriers to ethread barriers...
1273 */
1274 static ERTS_INLINE void
ethr_dw_atomic_init_nob(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1275 ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var,
1276 ethr_dw_sint_t *val)
1277 {
1278 ethr_dw_atomic_init(var, val);
1279 }
1280
1281 static ERTS_INLINE void
ethr_dw_atomic_set_nob(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1282 ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var,
1283 ethr_dw_sint_t *val)
1284 {
1285 ethr_dw_atomic_set(var, val);
1286 }
1287
1288 static ERTS_INLINE void
ethr_dw_atomic_read_nob(ethr_dw_atomic_t * var,ethr_dw_sint_t * val)1289 ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var,
1290 ethr_dw_sint_t *val)
1291 {
1292 ethr_dw_atomic_read(var, val);
1293 }
1294
1295 static ERTS_INLINE int
ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t * var,ethr_dw_sint_t * new_value,ethr_dw_sint_t * xchg)1296 ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var,
1297 ethr_dw_sint_t *new_value,
1298 ethr_dw_sint_t *xchg)
1299 {
1300 return ethr_dw_atomic_cmpxchg(var, new_value, xchg);
1301 }
1302
1303 #undef ERTS_ATOMIC64_OPS_IMPL__
1304 #undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
1305 #undef ERTS_DW_SINT_TO_AINT64__
1306 #undef ERTS_AINT64_TO_DW_SINT__
1307
1308 #ifdef ETHR_SU_DW_NAINT_T__
1309 #define ERTS_DW_SINT_TO_AINT64__(DW) \
1310 ((erts_aint64_t) DW.dw_sint)
1311 #define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \
1312 (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64)
1313 #else /* !ETHR_SU_DW_NAINT_T__ */
1314 #define ERTS_DW_SINT_TO_AINT64__(DW) \
1315 ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32) \
1316 | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD]) \
1317 & ((erts_aint64_t) 0xffffffff)))
1318 #define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \
1319 do { \
1320 DW.sint[ETHR_DW_SINT_LOW_WORD] = \
1321 (ethr_sint_t) (AINT64 & 0xffffffff); \
1322 DW.sint[ETHR_DW_SINT_HIGH_WORD] = \
1323 (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff); \
1324 } while (0)
1325 #endif /* !ETHR_SU_DW_NAINT_T__ */
1326
1327 #define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, \
1328 AVarP, XchgVar, NewVar, \
1329 ModificationCode) \
1330 do { \
1331 ethr_dw_sint_t dw_xchg__, dw_new__; \
1332 ethr_dw_atomic_read(AVarP, &dw_xchg__); \
1333 do { \
1334 XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__); \
1335 { \
1336 ModificationCode; \
1337 } \
1338 ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar); \
1339 } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__)); \
1340 } while (0)
1341
1342 #define ERTS_ATOMIC64_OPS_IMPL__(BARRIER) \
1343 \
1344 ERTS_GLB_INLINE void \
1345 erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \
1346 erts_aint64_t val) \
1347 { \
1348 ethr_dw_sint_t dw; \
1349 ERTS_AINT64_TO_DW_SINT__(dw, val); \
1350 ethr_dw_atomic_init_ ## BARRIER(var, &dw); \
1351 } \
1352 \
1353 ERTS_GLB_INLINE void \
1354 erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \
1355 erts_aint64_t val) \
1356 { \
1357 ethr_dw_sint_t dw; \
1358 ERTS_AINT64_TO_DW_SINT__(dw, val); \
1359 ethr_dw_atomic_set_ ## BARRIER(var, &dw); \
1360 } \
1361 \
1362 ERTS_GLB_INLINE erts_aint64_t \
1363 erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var) \
1364 { \
1365 ethr_dw_sint_t dw; \
1366 ethr_dw_atomic_read_ ## BARRIER(var, &dw); \
1367 return ERTS_DW_SINT_TO_AINT64__(dw); \
1368 } \
1369 \
1370 ERTS_GLB_INLINE erts_aint64_t \
1371 erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var) \
1372 { \
1373 erts_aint64_t xchg, new_value; \
1374 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1375 var, xchg, new_value, \
1376 new_value = xchg + 1); \
1377 return new_value; \
1378 } \
1379 \
1380 ERTS_GLB_INLINE erts_aint64_t \
1381 erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var) \
1382 { \
1383 erts_aint64_t xchg, new_value; \
1384 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1385 var, xchg, new_value, \
1386 new_value = xchg - 1); \
1387 return new_value; \
1388 } \
1389 \
1390 ERTS_GLB_INLINE void \
1391 erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var) \
1392 { \
1393 erts_aint64_t xchg, new_value; \
1394 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1395 var, xchg, new_value, \
1396 new_value = xchg + 1); \
1397 } \
1398 \
1399 ERTS_GLB_INLINE void \
1400 erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var) \
1401 { \
1402 erts_aint64_t xchg, new_value; \
1403 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1404 var, xchg, new_value, \
1405 new_value = xchg - 1); \
1406 } \
1407 \
1408 ERTS_GLB_INLINE erts_aint64_t \
1409 erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \
1410 erts_aint64_t val) \
1411 { \
1412 erts_aint64_t xchg, new_value; \
1413 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1414 var, xchg, new_value, \
1415 new_value = xchg + val); \
1416 return new_value; \
1417 } \
1418 \
1419 ERTS_GLB_INLINE void \
1420 erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \
1421 erts_aint64_t val) \
1422 { \
1423 erts_aint64_t xchg, new_value; \
1424 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1425 var, xchg, new_value, \
1426 new_value = xchg + val); \
1427 } \
1428 \
1429 ERTS_GLB_INLINE erts_aint64_t \
1430 erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \
1431 erts_aint64_t val) \
1432 { \
1433 erts_aint64_t xchg, new_value; \
1434 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1435 var, xchg, new_value, \
1436 new_value = xchg | val); \
1437 return xchg; \
1438 } \
1439 \
1440 ERTS_GLB_INLINE erts_aint64_t \
1441 erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \
1442 erts_aint64_t val) \
1443 { \
1444 erts_aint64_t xchg, new_value; \
1445 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1446 var, xchg, new_value, \
1447 new_value = xchg & val); \
1448 return xchg; \
1449 } \
1450 \
1451 ERTS_GLB_INLINE erts_aint64_t \
1452 erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \
1453 erts_aint64_t val) \
1454 { \
1455 erts_aint64_t xchg, new_value; \
1456 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1457 var, xchg, new_value, \
1458 new_value = val); \
1459 return xchg; \
1460 } \
1461 \
1462 ERTS_GLB_INLINE erts_aint64_t \
1463 erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \
1464 erts_aint64_t new_value, \
1465 erts_aint64_t exp) \
1466 { \
1467 ethr_dw_sint_t dw_xchg, dw_new; \
1468 ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp); \
1469 ERTS_AINT64_TO_DW_SINT__(dw_new, new_value); \
1470 if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg)) \
1471 return exp; \
1472 return ERTS_DW_SINT_TO_AINT64__(dw_xchg); \
1473 } \
1474 \
1475 ERTS_GLB_INLINE erts_aint64_t \
1476 erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \
1477 erts_aint64_t mask, \
1478 erts_aint64_t set) \
1479 { \
1480 erts_aint64_t xchg, new_value; \
1481 ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \
1482 var, xchg, new_value, \
1483 { \
1484 new_value = xchg & ~mask; \
1485 new_value |= mask & set; \
1486 }); \
1487 return xchg; \
1488 }
1489
1490 ERTS_ATOMIC64_OPS_IMPL__(nob)
ERTS_ATOMIC64_OPS_IMPL__(mb)1491 ERTS_ATOMIC64_OPS_IMPL__(mb)
1492 ERTS_ATOMIC64_OPS_IMPL__(acqb)
1493 ERTS_ATOMIC64_OPS_IMPL__(relb)
1494 ERTS_ATOMIC64_OPS_IMPL__(ddrb)
1495 ERTS_ATOMIC64_OPS_IMPL__(rb)
1496 ERTS_ATOMIC64_OPS_IMPL__(wb)
1497
1498 #undef ERTS_ATOMIC64_OPS_IMPL__
1499 #undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
1500
1501 ERTS_GLB_INLINE void
1502 erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val)
1503 {
1504 ethr_sint_t *sint = ethr_dw_atomic_addr(var);
1505 ethr_dw_sint_t dw;
1506 ERTS_AINT64_TO_DW_SINT__(dw, val);
1507 sint[0] = dw.sint[0];
1508 sint[1] = dw.sint[1];
1509 }
1510
1511 ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t * var)1512 erts_atomic64_read_dirty(erts_atomic64_t *var)
1513 {
1514 ethr_sint_t *sint;
1515 ethr_dw_sint_t dw;
1516 sint = ethr_dw_atomic_addr(var);
1517 dw.sint[0] = sint[0];
1518 dw.sint[1] = sint[1];
1519 return ERTS_DW_SINT_TO_AINT64__(dw);
1520 }
1521
1522 #undef ERTS_DW_SINT_TO_AINT64__
1523 #undef ERTS_AINT64_TO_DW_SINT__
1524
1525 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
1526
1527 #endif /* ARCH_32 */
1528
1529
1530 #include "erl_msacc.h"
1531
1532 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
1533
1534 ERTS_GLB_INLINE void
erts_thr_init(erts_thr_init_data_t * id)1535 erts_thr_init(erts_thr_init_data_t *id)
1536 {
1537 int res = ethr_init(id);
1538 if (res)
1539 erts_thr_fatal_error(res, "initialize thread library");
1540 }
1541
1542 ERTS_GLB_INLINE void
erts_thr_late_init(erts_thr_late_init_data_t * id)1543 erts_thr_late_init(erts_thr_late_init_data_t *id)
1544 {
1545 int res = ethr_late_init(id);
1546 if (res)
1547 erts_thr_fatal_error(res, "complete initialization of thread library");
1548 }
1549
1550 ERTS_GLB_INLINE void
erts_thr_create(erts_tid_t * tid,void * (* func)(void *),void * arg,erts_thr_opts_t * opts)1551 erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg,
1552 erts_thr_opts_t *opts)
1553 {
1554 int res = ethr_thr_create(tid, func, arg, opts);
1555 if (res)
1556 erts_thr_fatal_error(res, "create thread");
1557 }
1558
1559 ERTS_GLB_INLINE void
erts_thr_join(erts_tid_t tid,void ** thr_res)1560 erts_thr_join(erts_tid_t tid, void **thr_res)
1561 {
1562 int res = ethr_thr_join(tid, thr_res);
1563 if (res)
1564 erts_thr_fatal_error(res, "join thread");
1565 }
1566
1567
1568 ERTS_GLB_INLINE void
erts_thr_detach(erts_tid_t tid)1569 erts_thr_detach(erts_tid_t tid)
1570 {
1571 int res = ethr_thr_detach(tid);
1572 if (res)
1573 erts_thr_fatal_error(res, "detach thread");
1574 }
1575
1576
1577 ERTS_GLB_INLINE void
erts_thr_exit(void * res)1578 erts_thr_exit(void *res)
1579 {
1580 ethr_thr_exit(res);
1581 erts_thr_fatal_error(0, "terminate thread");
1582 }
1583
1584 ERTS_GLB_INLINE void
erts_thr_install_exit_handler(void (* exit_handler)(void))1585 erts_thr_install_exit_handler(void (*exit_handler)(void))
1586 {
1587 int res = ethr_install_exit_handler(exit_handler);
1588 if (res != 0)
1589 erts_thr_fatal_error(res, "install thread exit handler");
1590 }
1591
1592 ERTS_GLB_INLINE erts_tid_t
erts_thr_self(void)1593 erts_thr_self(void)
1594 {
1595 return ethr_self();
1596 }
1597
1598 ERTS_GLB_INLINE int
erts_thr_getname(erts_tid_t tid,char * buf,size_t len)1599 erts_thr_getname(erts_tid_t tid, char *buf, size_t len)
1600 {
1601 return ethr_getname(tid, buf, len);
1602 }
1603
1604
1605 ERTS_GLB_INLINE int
erts_equal_tids(erts_tid_t x,erts_tid_t y)1606 erts_equal_tids(erts_tid_t x, erts_tid_t y)
1607 {
1608 return ethr_equal_tids(x, y);
1609 }
1610
1611 ERTS_GLB_INLINE void
erts_mtx_init(erts_mtx_t * mtx,const char * name,Eterm extra,erts_lock_flags_t flags)1612 erts_mtx_init(erts_mtx_t *mtx, const char *name, Eterm extra, erts_lock_flags_t flags)
1613 {
1614 int res = ethr_mutex_init(&mtx->mtx);
1615 if (res) {
1616 erts_thr_fatal_error(res, "initialize mutex");
1617 }
1618
1619 flags |= ERTS_LOCK_TYPE_MUTEX;
1620 #ifdef DEBUG
1621 mtx->flags = flags;
1622 #endif
1623
1624 #ifdef ERTS_ENABLE_LOCK_CHECK
1625 erts_lc_init_lock_x(&mtx->lc, name, flags, extra);
1626 #endif
1627 #ifdef ERTS_ENABLE_LOCK_COUNT
1628 erts_lcnt_init_ref_x(&mtx->lcnt, name, extra, flags);
1629 #endif
1630 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1631 erts_dlc_create_lock(&mtx->dlc, name);
1632 #endif
1633 }
1634
1635 ERTS_GLB_INLINE void
erts_mtx_init_locked(erts_mtx_t * mtx,char * name,Eterm extra,erts_lock_flags_t flags)1636 erts_mtx_init_locked(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
1637 {
1638 erts_mtx_init(mtx, name, extra, flags);
1639
1640 ethr_mutex_lock(&mtx->mtx);
1641 #ifdef ERTS_ENABLE_LOCK_CHECK
1642 erts_lc_trylock(1, &mtx->lc);
1643 #endif
1644 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1645 erts_dlc_trylock(&mtx->dlc, 1);
1646 #endif
1647 #ifdef ERTS_ENABLE_LOCK_COUNT
1648 erts_lcnt_trylock(&mtx->lcnt, 1);
1649 #endif
1650 }
1651
1652 ERTS_GLB_INLINE void
erts_mtx_destroy(erts_mtx_t * mtx)1653 erts_mtx_destroy(erts_mtx_t *mtx)
1654 {
1655 int res;
1656
1657 ASSERT(!(mtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
1658
1659 #ifdef ERTS_ENABLE_LOCK_CHECK
1660 erts_lc_destroy_lock(&mtx->lc);
1661 #endif
1662 #ifdef ERTS_ENABLE_LOCK_COUNT
1663 erts_lcnt_uninstall(&mtx->lcnt);
1664 #endif
1665 res = ethr_mutex_destroy(&mtx->mtx);
1666 if (res != 0) {
1667 #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
1668 if (res == EBUSY) {
1669 char *warn = "Ignoring busy mutex destroy. "
1670 "Most likely a bug in pthread implementation.";
1671 erts_send_warning_to_logger_str_nogl(warn);
1672 }
1673 else
1674 #endif
1675 erts_thr_fatal_error(res, "destroy mutex");
1676 }
1677 }
1678
1679 ERTS_GLB_INLINE int
1680 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_trylock_x(erts_mtx_t * mtx,const char * file,unsigned int line)1681 erts_mtx_trylock_x(erts_mtx_t *mtx, const char *file, unsigned int line)
1682 #else
1683 erts_mtx_trylock(erts_mtx_t *mtx)
1684 #endif
1685 {
1686 int res;
1687
1688 #ifdef ERTS_ENABLE_LOCK_CHECK
1689 if (erts_lc_trylock_force_busy(&mtx->lc))
1690 return EBUSY; /* Make sure caller can handle the situation without
1691 causing a lock order violation */
1692 #endif
1693
1694 res = ethr_mutex_trylock(&mtx->mtx);
1695
1696 #ifdef ERTS_ENABLE_LOCK_CHECK
1697 #ifdef ERTS_ENABLE_LOCK_POSITION
1698 erts_lc_trylock_x(res == 0, &mtx->lc,file,line);
1699 #else
1700 erts_lc_trylock(res == 0, &mtx->lc);
1701 #endif
1702 #endif
1703 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1704 erts_dlc_trylock(&mtx->dlc, res == 0);
1705 #endif
1706 #ifdef ERTS_ENABLE_LOCK_COUNT
1707 erts_lcnt_trylock(&mtx->lcnt, res);
1708 #endif
1709 return res;
1710 }
1711
1712 ERTS_GLB_INLINE void
1713 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_lock_x(erts_mtx_t * mtx,const char * file,unsigned int line)1714 erts_mtx_lock_x(erts_mtx_t *mtx, const char *file, unsigned int line)
1715 #else
1716 erts_mtx_lock(erts_mtx_t *mtx)
1717 #endif
1718 {
1719 #ifdef ERTS_ENABLE_LOCK_CHECK
1720 #ifdef ERTS_ENABLE_LOCK_POSITION
1721 erts_lc_lock_x(&mtx->lc, file, line);
1722 #else
1723 erts_lc_lock(&mtx->lc);
1724 #endif
1725 #endif
1726 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1727 erts_dlc_lock(&mtx->dlc);
1728 #endif
1729 #ifdef ERTS_ENABLE_LOCK_COUNT
1730 erts_lcnt_lock(&mtx->lcnt);
1731 #endif
1732 ethr_mutex_lock(&mtx->mtx);
1733 #ifdef ERTS_ENABLE_LOCK_COUNT
1734 erts_lcnt_lock_post_x(&mtx->lcnt, file, line);
1735 #endif
1736 }
1737
1738 ERTS_GLB_INLINE void
erts_mtx_unlock(erts_mtx_t * mtx)1739 erts_mtx_unlock(erts_mtx_t *mtx)
1740 {
1741 #ifdef ERTS_ENABLE_LOCK_CHECK
1742 erts_lc_unlock(&mtx->lc);
1743 #endif
1744 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1745 erts_dlc_unlock(&mtx->dlc);
1746 #endif
1747 #ifdef ERTS_ENABLE_LOCK_COUNT
1748 erts_lcnt_unlock(&mtx->lcnt);
1749 #endif
1750 ethr_mutex_unlock(&mtx->mtx);
1751 }
1752
1753 ERTS_GLB_INLINE int
erts_lc_mtx_is_locked(erts_mtx_t * mtx)1754 erts_lc_mtx_is_locked(erts_mtx_t *mtx)
1755 {
1756 #if defined(ERTS_ENABLE_LOCK_CHECK)
1757 int res;
1758 erts_lc_lock_t lc = mtx->lc;
1759 lc.flags = ERTS_LOCK_FLAGS_TYPE_MUTEX;
1760 lc.taken_options = 0;
1761 erts_lc_have_locks(&res, &lc, 1);
1762 return res;
1763 #else
1764 return 0;
1765 #endif
1766 }
1767
1768 ERTS_GLB_INLINE void
erts_cnd_init(erts_cnd_t * cnd)1769 erts_cnd_init(erts_cnd_t *cnd)
1770 {
1771 int res = ethr_cond_init(cnd);
1772 if (res)
1773 erts_thr_fatal_error(res, "initialize condition variable");
1774 }
1775
1776 ERTS_GLB_INLINE void
erts_cnd_destroy(erts_cnd_t * cnd)1777 erts_cnd_destroy(erts_cnd_t *cnd)
1778 {
1779 int res = ethr_cond_destroy(cnd);
1780 if (res != 0) {
1781 #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
1782 if (res == EBUSY) {
1783 char *warn = "Ignoring busy cond destroy. "
1784 "Most likely a bug in pthread implementation.";
1785 erts_send_warning_to_logger_str_nogl(warn);
1786 }
1787 else
1788 #endif
1789 erts_thr_fatal_error(res, "destroy condition variable");
1790 }
1791 }
1792
1793 ERTS_GLB_INLINE void
erts_cnd_wait(erts_cnd_t * cnd,erts_mtx_t * mtx)1794 erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
1795 {
1796 int res;
1797 ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
1798 #ifdef ERTS_ENABLE_LOCK_CHECK
1799 erts_lc_unlock(&mtx->lc);
1800 #endif
1801 #ifdef ERTS_ENABLE_LOCK_COUNT
1802 erts_lcnt_unlock(&mtx->lcnt);
1803 #endif
1804 res = ethr_cond_wait(cnd, &mtx->mtx);
1805 #ifdef ERTS_ENABLE_LOCK_CHECK
1806 erts_lc_lock(&mtx->lc);
1807 #endif
1808 #ifdef ERTS_ENABLE_LOCK_COUNT
1809 erts_lcnt_lock(&mtx->lcnt);
1810 #endif
1811 #ifdef ERTS_ENABLE_LOCK_COUNT
1812 erts_lcnt_lock_post(&mtx->lcnt);
1813 #endif
1814 if (res != 0 && res != EINTR)
1815 erts_thr_fatal_error(res, "wait on condition variable");
1816 ERTS_MSACC_POP_STATE();
1817 }
1818
1819 /*
1820 * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast()
1821 *
1822 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
1823 * even though the associated mutex/mutexes isn't/aren't locked by the
1824 * caller. Our implementation do not allow that in order to avoid a
1825 * performance penalty. That is, all associated mutexes *need* to be
1826 * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()!
1827 */
1828
1829 ERTS_GLB_INLINE void
erts_cnd_signal(erts_cnd_t * cnd)1830 erts_cnd_signal(erts_cnd_t *cnd)
1831 {
1832 ethr_cond_signal(cnd);
1833 }
1834
1835
1836 ERTS_GLB_INLINE void
erts_cnd_broadcast(erts_cnd_t * cnd)1837 erts_cnd_broadcast(erts_cnd_t *cnd)
1838 {
1839 ethr_cond_broadcast(cnd);
1840 }
1841
1842 /* rwmutex */
1843
1844 ERTS_GLB_INLINE void
erts_rwmtx_set_reader_group(int no)1845 erts_rwmtx_set_reader_group(int no)
1846 {
1847 int res;
1848 #ifdef ERTS_ENABLE_LOCK_CHECK
1849 erts_lc_check_no_locked_of_type(ERTS_LOCK_TYPE_RWMUTEX);
1850 #endif
1851 res = ethr_rwmutex_set_reader_group(no);
1852 if (res != 0)
1853 erts_thr_fatal_error(res, "set reader group");
1854 }
1855
1856 ERTS_GLB_INLINE void
erts_rwmtx_init_opt(erts_rwmtx_t * rwmtx,erts_rwmtx_opt_t * opt,char * name,Eterm extra,erts_lock_flags_t flags)1857 erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_rwmtx_opt_t *opt,
1858 char *name, Eterm extra, erts_lock_flags_t flags) {
1859 int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
1860 if (res != 0) {
1861 erts_thr_fatal_error(res, "initialize rwmutex");
1862 }
1863
1864 flags |= ERTS_LOCK_TYPE_RWMUTEX;
1865 #ifdef DEBUG
1866 rwmtx->flags = flags;
1867 #endif
1868
1869 #ifdef ERTS_ENABLE_LOCK_CHECK
1870 erts_lc_init_lock_x(&rwmtx->lc, name, flags, extra);
1871 #endif
1872 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1873 erts_dlc_create_lock(&rwmtx->dlc, name);
1874 #endif
1875 #ifdef ERTS_ENABLE_LOCK_COUNT
1876 erts_lcnt_init_ref_x(&rwmtx->lcnt, name, extra, flags);
1877 #endif
1878 }
1879
1880 ERTS_GLB_INLINE void
erts_rwmtx_init(erts_rwmtx_t * rwmtx,char * name,Eterm extra,erts_lock_flags_t flags)1881 erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name, Eterm extra,
1882 erts_lock_flags_t flags) {
1883 erts_rwmtx_init_opt(rwmtx, NULL, name, extra, flags);
1884 }
1885
1886 ERTS_GLB_INLINE void
erts_rwmtx_destroy(erts_rwmtx_t * rwmtx)1887 erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
1888 {
1889 int res;
1890
1891 ASSERT(!(rwmtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
1892
1893 #ifdef ERTS_ENABLE_LOCK_CHECK
1894 erts_lc_destroy_lock(&rwmtx->lc);
1895 #endif
1896 #ifdef ERTS_ENABLE_LOCK_COUNT
1897 erts_lcnt_uninstall(&rwmtx->lcnt);
1898 #endif
1899 res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
1900 if (res != 0) {
1901 #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
1902 if (res == EBUSY) {
1903 char *warn = "Ignoring busy rwmutex destroy. "
1904 "Most likely a bug in pthread implementation.";
1905 erts_send_warning_to_logger_str_nogl(warn);
1906 }
1907 else
1908 #endif
1909 erts_thr_fatal_error(res, "destroy rwmutex");
1910 }
1911 }
1912
1913 ERTS_GLB_INLINE int
1914 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrlock_x(erts_rwmtx_t * rwmtx,const char * file,unsigned int line)1915 erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line)
1916 #else
1917 erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
1918 #endif
1919 {
1920 int res;
1921
1922 #ifdef ERTS_ENABLE_LOCK_CHECK
1923 if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ))
1924 return EBUSY; /* Make sure caller can handle the situation without
1925 causing a lock order violation */
1926 #endif
1927
1928 res = ethr_rwmutex_tryrlock(&rwmtx->rwmtx);
1929
1930 #ifdef ERTS_ENABLE_LOCK_CHECK
1931 #ifdef ERTS_ENABLE_LOCK_POSITION
1932 erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_READ,file,line);
1933 #else
1934 erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_READ);
1935 #endif
1936 #endif
1937 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1938 erts_dlc_trylock(&rwmtx->dlc, res == 0);
1939 #endif
1940 #ifdef ERTS_ENABLE_LOCK_COUNT
1941 erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTIONS_READ);
1942 #endif
1943
1944 return res;
1945 }
1946
1947 ERTS_GLB_INLINE void
1948 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rlock_x(erts_rwmtx_t * rwmtx,const char * file,unsigned int line)1949 erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line)
1950 #else
1951 erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
1952 #endif
1953 {
1954 #ifdef ERTS_ENABLE_LOCK_CHECK
1955 #ifdef ERTS_ENABLE_LOCK_POSITION
1956 erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ,file,line);
1957 #else
1958 erts_lc_lock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ);
1959 #endif
1960 #endif
1961 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1962 erts_dlc_lock(&rwmtx->dlc);
1963 #endif
1964 #ifdef ERTS_ENABLE_LOCK_COUNT
1965 erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ);
1966 #endif
1967 ethr_rwmutex_rlock(&rwmtx->rwmtx);
1968 #ifdef ERTS_ENABLE_LOCK_COUNT
1969 erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
1970 #endif
1971 }
1972
1973 ERTS_GLB_INLINE void
erts_rwmtx_runlock(erts_rwmtx_t * rwmtx)1974 erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
1975 {
1976 #ifdef ERTS_ENABLE_LOCK_CHECK
1977 erts_lc_unlock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ);
1978 #endif
1979 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
1980 erts_dlc_unlock(&rwmtx->dlc);
1981 #endif
1982 #ifdef ERTS_ENABLE_LOCK_COUNT
1983 erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ);
1984 #endif
1985 ethr_rwmutex_runlock(&rwmtx->rwmtx);
1986 }
1987
1988
1989 ERTS_GLB_INLINE int
1990 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrwlock_x(erts_rwmtx_t * rwmtx,const char * file,unsigned int line)1991 erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line)
1992 #else
1993 erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
1994 #endif
1995 {
1996 int res;
1997
1998 #ifdef ERTS_ENABLE_LOCK_CHECK
1999 if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR))
2000 return EBUSY; /* Make sure caller can handle the situation without
2001 causing a lock order violation */
2002 #endif
2003
2004 res = ethr_rwmutex_tryrwlock(&rwmtx->rwmtx);
2005
2006 #ifdef ERTS_ENABLE_LOCK_CHECK
2007 #ifdef ERTS_ENABLE_LOCK_POSITION
2008 erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR,file,line);
2009 #else
2010 erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR);
2011 #endif
2012 #endif
2013 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
2014 erts_dlc_trylock(&rwmtx->dlc, res == 0);
2015 #endif
2016 #ifdef ERTS_ENABLE_LOCK_COUNT
2017 erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTIONS_RDWR);
2018 #endif
2019
2020 return res;
2021 }
2022
2023 ERTS_GLB_INLINE void
2024 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rwlock_x(erts_rwmtx_t * rwmtx,const char * file,unsigned int line)2025 erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line)
2026 #else
2027 erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
2028 #endif
2029 {
2030 #ifdef ERTS_ENABLE_LOCK_CHECK
2031 #ifdef ERTS_ENABLE_LOCK_POSITION
2032 erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR,file,line);
2033 #else
2034 erts_lc_lock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR);
2035 #endif
2036 #endif
2037 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
2038 erts_dlc_lock(&rwmtx->dlc);
2039 #endif
2040 #ifdef ERTS_ENABLE_LOCK_COUNT
2041 erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR);
2042 #endif
2043 ethr_rwmutex_rwlock(&rwmtx->rwmtx);
2044 #ifdef ERTS_ENABLE_LOCK_COUNT
2045 erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
2046 #endif
2047 }
2048
2049 ERTS_GLB_INLINE void
erts_rwmtx_rwunlock(erts_rwmtx_t * rwmtx)2050 erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
2051 {
2052 #ifdef ERTS_ENABLE_LOCK_CHECK
2053 erts_lc_unlock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR);
2054 #endif
2055 #ifdef ERTS_DYN_LOCK_CHECK_INTERNAL
2056 erts_dlc_unlock(&rwmtx->dlc);
2057 #endif
2058 #ifdef ERTS_ENABLE_LOCK_COUNT
2059 erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR);
2060 #endif
2061 ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
2062 }
2063
2064 #if 0 /* The following rwmtx function names are
2065 reserved for potential future use. */
2066
2067 /* Try upgrade from r-locked state to rw-locked state */
2068 ERTS_GLB_INLINE int
2069 erts_rwmtx_trywlock(erts_rwmtx_t *rwmtx)
2070 {
2071 return 0;
2072 }
2073
2074 /* Upgrade from r-locked state to rw-locked state */
2075 ERTS_GLB_INLINE void
2076 erts_rwmtx_wlock(erts_rwmtx_t *rwmtx)
2077 {
2078
2079 }
2080
2081 /* Downgrade from rw-locked state to r-locked state */
2082 ERTS_GLB_INLINE void
2083 erts_rwmtx_wunlock(erts_rwmtx_t *rwmtx)
2084 {
2085
2086 }
2087
2088 #endif
2089
2090 ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rlocked(erts_rwmtx_t * mtx)2091 erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx)
2092 {
2093 #if defined(ERTS_ENABLE_LOCK_CHECK)
2094 int res;
2095 erts_lc_lock_t lc = mtx->lc;
2096 lc.flags = ERTS_LOCK_TYPE_RWMUTEX;
2097 lc.taken_options = ERTS_LOCK_OPTIONS_READ;
2098 erts_lc_have_locks(&res, &lc, 1);
2099 return res;
2100 #else
2101 return 0;
2102 #endif
2103 }
2104
2105 ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t * mtx)2106 erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
2107 {
2108 #if defined(ERTS_ENABLE_LOCK_CHECK)
2109 int res;
2110 erts_lc_lock_t lc = mtx->lc;
2111 lc.flags = ERTS_LOCK_TYPE_RWMUTEX;
2112 lc.taken_options = ERTS_LOCK_OPTIONS_RDWR;
2113 erts_lc_have_locks(&res, &lc, 1);
2114 return res;
2115 #else
2116 return 0;
2117 #endif
2118 }
2119
2120 /* spinlock */
2121
2122 ERTS_GLB_INLINE void
erts_spinlock_init(erts_spinlock_t * lock,char * name,Eterm extra,erts_lock_flags_t flags)2123 erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
2124 {
2125 int res = ethr_spinlock_init(&lock->slck);
2126 if (res) {
2127 erts_thr_fatal_error(res, "init spinlock");
2128 }
2129
2130 flags |= ERTS_LOCK_TYPE_SPINLOCK;
2131 #ifdef DEBUG
2132 lock->flags = flags;
2133 #endif
2134
2135 #ifdef ERTS_ENABLE_LOCK_CHECK
2136 erts_lc_init_lock_x(&lock->lc, name, flags, extra);
2137 #endif
2138 #ifdef ERTS_ENABLE_LOCK_COUNT
2139 erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags);
2140 #endif
2141 }
2142
2143 ERTS_GLB_INLINE void
erts_spinlock_destroy(erts_spinlock_t * lock)2144 erts_spinlock_destroy(erts_spinlock_t *lock)
2145 {
2146 int res;
2147
2148 ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
2149
2150 #ifdef ERTS_ENABLE_LOCK_CHECK
2151 erts_lc_destroy_lock(&lock->lc);
2152 #endif
2153 #ifdef ERTS_ENABLE_LOCK_COUNT
2154 erts_lcnt_uninstall(&lock->lcnt);
2155 #endif
2156 res = ethr_spinlock_destroy(&lock->slck);
2157 if (res != 0) {
2158 #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
2159 if (res == EBUSY) {
2160 char *warn = "Ignoring busy spinlock destroy. "
2161 "Most likely a bug in pthread implementation.";
2162 erts_send_warning_to_logger_str_nogl(warn);
2163 }
2164 else
2165 #endif
2166 erts_thr_fatal_error(res, "destroy rwlock");
2167 }
2168 }
2169
2170 ERTS_GLB_INLINE void
erts_spin_unlock(erts_spinlock_t * lock)2171 erts_spin_unlock(erts_spinlock_t *lock)
2172 {
2173 #ifdef ERTS_ENABLE_LOCK_CHECK
2174 erts_lc_unlock(&lock->lc);
2175 #endif
2176 #ifdef ERTS_ENABLE_LOCK_COUNT
2177 erts_lcnt_unlock(&lock->lcnt);
2178 #endif
2179 ethr_spin_unlock(&lock->slck);
2180 }
2181
2182 ERTS_GLB_INLINE void
2183 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_spin_lock_x(erts_spinlock_t * lock,const char * file,unsigned int line)2184 erts_spin_lock_x(erts_spinlock_t *lock, const char *file, unsigned int line)
2185 #else
2186 erts_spin_lock(erts_spinlock_t *lock)
2187 #endif
2188 {
2189 #ifdef ERTS_ENABLE_LOCK_CHECK
2190 #ifdef ERTS_ENABLE_LOCK_POSITION
2191 erts_lc_lock_x(&lock->lc,file,line);
2192 #else
2193 erts_lc_lock(&lock->lc);
2194 #endif
2195 #endif
2196 #ifdef ERTS_ENABLE_LOCK_COUNT
2197 erts_lcnt_lock(&lock->lcnt);
2198 #endif
2199 ethr_spin_lock(&lock->slck);
2200 #ifdef ERTS_ENABLE_LOCK_COUNT
2201 erts_lcnt_lock_post_x(&lock->lcnt, file, line);
2202 #endif
2203 }
2204
2205 ERTS_GLB_INLINE int
erts_lc_spinlock_is_locked(erts_spinlock_t * lock)2206 erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
2207 {
2208 #if defined(ERTS_ENABLE_LOCK_CHECK)
2209 int res;
2210 erts_lc_lock_t lc = lock->lc;
2211 lc.flags = ERTS_LOCK_TYPE_SPINLOCK;
2212 lc.taken_options = 0;
2213 erts_lc_have_locks(&res, &lc, 1);
2214 return res;
2215 #else
2216 return 0;
2217 #endif
2218 }
2219
2220 /* rwspinlock */
2221
2222 ERTS_GLB_INLINE void
erts_rwlock_init(erts_rwlock_t * lock,char * name,Eterm extra,erts_lock_flags_t flags)2223 erts_rwlock_init(erts_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
2224 {
2225 int res = ethr_rwlock_init(&lock->rwlck);
2226 if (res) {
2227 erts_thr_fatal_error(res, "init rwlock");
2228 }
2229
2230 flags |= ERTS_LOCK_TYPE_RWSPINLOCK;
2231 #ifdef DEBUG
2232 lock->flags = flags;
2233 #endif
2234
2235 #ifdef ERTS_ENABLE_LOCK_CHECK
2236 erts_lc_init_lock_x(&lock->lc, name, flags, extra);
2237 #endif
2238 #ifdef ERTS_ENABLE_LOCK_COUNT
2239 erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags);
2240 #endif
2241 }
2242
2243 ERTS_GLB_INLINE void
erts_rwlock_destroy(erts_rwlock_t * lock)2244 erts_rwlock_destroy(erts_rwlock_t *lock)
2245 {
2246 int res;
2247
2248 ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
2249
2250 #ifdef ERTS_ENABLE_LOCK_CHECK
2251 erts_lc_destroy_lock(&lock->lc);
2252 #endif
2253 #ifdef ERTS_ENABLE_LOCK_COUNT
2254 erts_lcnt_uninstall(&lock->lcnt);
2255 #endif
2256 res = ethr_rwlock_destroy(&lock->rwlck);
2257 if (res != 0) {
2258 #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
2259 if (res == EBUSY) {
2260 char *warn = "Ignoring busy rwlock destroy. "
2261 "Most likely a bug in pthread implementation.";
2262 erts_send_warning_to_logger_str_nogl(warn);
2263 }
2264 else
2265 #endif
2266 erts_thr_fatal_error(res, "destroy rwlock");
2267 }
2268 }
2269
2270 ERTS_GLB_INLINE void
erts_read_unlock(erts_rwlock_t * lock)2271 erts_read_unlock(erts_rwlock_t *lock)
2272 {
2273 #ifdef ERTS_ENABLE_LOCK_CHECK
2274 erts_lc_unlock_flg(&lock->lc, ERTS_LOCK_OPTIONS_READ);
2275 #endif
2276 #ifdef ERTS_ENABLE_LOCK_COUNT
2277 erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ);
2278 #endif
2279 ethr_read_unlock(&lock->rwlck);
2280 }
2281
2282 ERTS_GLB_INLINE void
2283 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_read_lock_x(erts_rwlock_t * lock,const char * file,unsigned int line)2284 erts_read_lock_x(erts_rwlock_t *lock, const char *file, unsigned int line)
2285 #else
2286 erts_read_lock(erts_rwlock_t *lock)
2287 #endif
2288 {
2289 #ifdef ERTS_ENABLE_LOCK_CHECK
2290 #ifdef ERTS_ENABLE_LOCK_POSITION
2291 erts_lc_lock_flg_x(&lock->lc, ERTS_LOCK_OPTIONS_READ,file,line);
2292 #else
2293 erts_lc_lock_flg(&lock->lc, ERTS_LOCK_OPTIONS_READ);
2294 #endif
2295 #endif
2296 #ifdef ERTS_ENABLE_LOCK_COUNT
2297 erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ);
2298 #endif
2299 ethr_read_lock(&lock->rwlck);
2300 #ifdef ERTS_ENABLE_LOCK_COUNT
2301 erts_lcnt_lock_post_x(&lock->lcnt, file, line);
2302 #endif
2303 }
2304
2305 ERTS_GLB_INLINE void
erts_write_unlock(erts_rwlock_t * lock)2306 erts_write_unlock(erts_rwlock_t *lock)
2307 {
2308 #ifdef ERTS_ENABLE_LOCK_CHECK
2309 erts_lc_unlock_flg(&lock->lc, ERTS_LOCK_OPTIONS_RDWR);
2310 #endif
2311 #ifdef ERTS_ENABLE_LOCK_COUNT
2312 erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR);
2313 #endif
2314 ethr_write_unlock(&lock->rwlck);
2315 }
2316
2317 ERTS_GLB_INLINE void
2318 #ifdef ERTS_ENABLE_LOCK_POSITION
erts_write_lock_x(erts_rwlock_t * lock,const char * file,unsigned int line)2319 erts_write_lock_x(erts_rwlock_t *lock, const char *file, unsigned int line)
2320 #else
2321 erts_write_lock(erts_rwlock_t *lock)
2322 #endif
2323 {
2324 #ifdef ERTS_ENABLE_LOCK_CHECK
2325 #ifdef ERTS_ENABLE_LOCK_POSITION
2326 erts_lc_lock_flg_x(&lock->lc, ERTS_LOCK_OPTIONS_RDWR,file,line);
2327 #else
2328 erts_lc_lock_flg(&lock->lc, ERTS_LOCK_OPTIONS_RDWR);
2329 #endif
2330 #endif
2331 #ifdef ERTS_ENABLE_LOCK_COUNT
2332 erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR);
2333 #endif
2334 ethr_write_lock(&lock->rwlck);
2335 #ifdef ERTS_ENABLE_LOCK_COUNT
2336 erts_lcnt_lock_post_x(&lock->lcnt, file, line);
2337 #endif
2338 }
2339
2340 ERTS_GLB_INLINE int
erts_lc_rwlock_is_rlocked(erts_rwlock_t * lock)2341 erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock)
2342 {
2343 #if defined(ERTS_ENABLE_LOCK_CHECK)
2344 int res;
2345 erts_lc_lock_t lc = lock->lc;
2346 lc.flags = ERTS_LOCK_TYPE_RWSPINLOCK;
2347 lc.taken_options = ERTS_LOCK_OPTIONS_READ;
2348 erts_lc_have_locks(&res, &lc, 1);
2349 return res;
2350 #else
2351 return 0;
2352 #endif
2353 }
2354
2355 ERTS_GLB_INLINE int
erts_lc_rwlock_is_rwlocked(erts_rwlock_t * lock)2356 erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
2357 {
2358 #if defined(ERTS_ENABLE_LOCK_CHECK)
2359 int res;
2360 erts_lc_lock_t lc = lock->lc;
2361 lc.flags = ERTS_LOCK_TYPE_RWSPINLOCK;
2362 lc.taken_options = ERTS_LOCK_OPTIONS_RDWR;
2363 erts_lc_have_locks(&res, &lc, 1);
2364 return res;
2365 #else
2366 return 0;
2367 #endif
2368 }
2369
2370 ERTS_GLB_INLINE void
erts_tsd_key_create(erts_tsd_key_t * keyp,char * keyname)2371 erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname)
2372 {
2373 int res = ethr_tsd_key_create(keyp, keyname);
2374 if (res)
2375 erts_thr_fatal_error(res, "create thread specific data key");
2376 }
2377
2378 ERTS_GLB_INLINE void
erts_tsd_key_delete(erts_tsd_key_t key)2379 erts_tsd_key_delete(erts_tsd_key_t key)
2380 {
2381 int res = ethr_tsd_key_delete(key);
2382 if (res)
2383 erts_thr_fatal_error(res, "delete thread specific data key");
2384 }
2385
2386 ERTS_GLB_INLINE void
erts_tsd_set(erts_tsd_key_t key,void * value)2387 erts_tsd_set(erts_tsd_key_t key, void *value)
2388 {
2389 int res = ethr_tsd_set(key, value);
2390 if (res)
2391 erts_thr_fatal_error(res, "set thread specific data");
2392 }
2393
2394 ERTS_GLB_INLINE void *
erts_tsd_get(erts_tsd_key_t key)2395 erts_tsd_get(erts_tsd_key_t key)
2396 {
2397 return ethr_tsd_get(key);
2398 }
2399
erts_tse_fetch(void)2400 ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void)
2401 {
2402 return (erts_tse_t *) ethr_get_ts_event();
2403 }
2404
erts_tse_use(erts_tse_t * ep)2405 ERTS_GLB_INLINE void erts_tse_use(erts_tse_t *ep)
2406 {
2407 /*
2408 * When enabling use on event from emulator
2409 * it *must* not already be in use...
2410 */
2411 #ifdef DEBUG
2412 erts_tse_t *tmp_ep;
2413 ASSERT(!(ep->iflgs & ETHR_TS_EV_BUSY));
2414 tmp_ep =
2415 #else
2416 (void)
2417 #endif
2418 ethr_use_ts_event(ep);
2419 ASSERT(ep == tmp_ep);
2420 }
2421
erts_tse_return(erts_tse_t * ep)2422 ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
2423 {
2424 ethr_leave_ts_event(ep);
2425 }
2426
erts_tse_prepare_timed(erts_tse_t * ep)2427 ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep)
2428 {
2429 int res;
2430 ETHR_ASSERT(ep->iflgs & ETHR_TS_EV_BUSY);
2431 res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event);
2432 if (res != 0)
2433 erts_thr_fatal_error(res, "prepare timed");
2434 }
2435
erts_tse_set(erts_tse_t * ep)2436 ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
2437 {
2438 ethr_event_set(&((ethr_ts_event *) ep)->event);
2439 }
2440
erts_tse_reset(erts_tse_t * ep)2441 ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
2442 {
2443 ETHR_ASSERT(ep->iflgs & ETHR_TS_EV_BUSY);
2444 ethr_event_reset(&((ethr_ts_event *) ep)->event);
2445 }
2446
erts_tse_wait(erts_tse_t * ep)2447 ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
2448 {
2449 int res;
2450 ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
2451 ETHR_ASSERT(ep->iflgs & ETHR_TS_EV_BUSY);
2452 res = ethr_event_wait(&((ethr_ts_event *) ep)->event);
2453 ERTS_MSACC_POP_STATE();
2454 return res;
2455 }
2456
erts_tse_swait(erts_tse_t * ep,int spincount)2457 ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
2458 {
2459 int res;
2460 ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
2461 ETHR_ASSERT(ep->iflgs & ETHR_TS_EV_BUSY);
2462 res = ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
2463 ERTS_MSACC_POP_STATE();
2464 return res;
2465 }
2466
erts_tse_twait(erts_tse_t * ep,Sint64 tmo)2467 ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
2468 {
2469 int res;
2470 ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
2471 ETHR_ASSERT(ep->iflgs & ETHR_TS_EV_BUSY);
2472 res = ethr_event_twait(&((ethr_ts_event *) ep)->event,
2473 (ethr_sint64_t) tmo);
2474 ERTS_MSACC_POP_STATE();
2475 return res;
2476 }
2477
erts_tse_stwait(erts_tse_t * ep,int spincount,Sint64 tmo)2478 ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
2479 {
2480 int res;
2481 ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
2482 ETHR_ASSERT(ep->iflgs & ETHR_TS_EV_BUSY);
2483 res = ethr_event_stwait(&((ethr_ts_event *) ep)->event,
2484 spincount,
2485 (ethr_sint64_t) tmo);
2486 ERTS_MSACC_POP_STATE();
2487 return res;
2488 }
2489
erts_tse_is_tmp(erts_tse_t * ep)2490 ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
2491 {
2492 return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP;
2493 }
2494
erts_thr_set_main_status(int on,int no)2495 ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no)
2496 {
2497 int res = ethr_set_main_thr_status(on, no);
2498 if (res != 0)
2499 erts_thr_fatal_error(res, "set thread main status");
2500 }
2501
erts_thr_get_main_status(void)2502 ERTS_GLB_INLINE int erts_thr_get_main_status(void)
2503 {
2504 int main_status;
2505 int res = ethr_get_main_thr_status(&main_status);
2506 if (res != 0)
2507 erts_thr_fatal_error(res, "get thread main status");
2508 return main_status;
2509 }
2510
erts_thr_yield(void)2511 ERTS_GLB_INLINE void erts_thr_yield(void)
2512 {
2513 int res = ETHR_YIELD();
2514 if (res != 0)
2515 erts_thr_fatal_error(res, "yield");
2516 }
2517
2518
2519 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
2520
2521 ERTS_GLB_INLINE void
erts_thr_kill(erts_tid_t tid,int sig)2522 erts_thr_kill(erts_tid_t tid, int sig) {
2523 int res = ethr_kill((ethr_tid)tid, sig);
2524 if (res)
2525 erts_thr_fatal_error(res, "killing thread");
2526 }
2527
2528 ERTS_GLB_INLINE void
erts_thr_sigmask(int how,const sigset_t * set,sigset_t * oset)2529 erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
2530 {
2531 int res = ethr_sigmask(how, set, oset);
2532 if (res)
2533 erts_thr_fatal_error(res, "get or set signal mask");
2534 }
2535
2536 ERTS_GLB_INLINE void
erts_thr_sigwait(const sigset_t * set,int * sig)2537 erts_thr_sigwait(const sigset_t *set, int *sig)
2538 {
2539 int res;
2540 do {
2541 res = ethr_sigwait(set, sig);
2542 } while (res == EINTR);
2543 if (res)
2544 erts_thr_fatal_error(res, "to wait for signal");
2545 }
2546
2547 #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
2548
2549 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
2550
2551 #endif /* #ifndef ERL_THREAD_H__ */
2552