1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2004-2017. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 /*
22 * Description: Thread library for use in the ERTS and other OTP
23 * applications.
24 * Author: Rickard Green
25 */
26
27 #ifndef ETHREAD_H__
28 #define ETHREAD_H__
29
30 #ifndef ETHR_HAVE_ETHREAD_DEFINES
31 # include "ethread_header_config.h"
32 #endif
33
34 #include <stdlib.h>
35 #include "ethread_inline.h"
36 #include "erl_errno.h"
37
38 #if defined(DEBUG)
39 # define ETHR_DEBUG
40 #endif
41
42 #if defined(__PPC__) || defined (__POWERPC)
43 /* OSE compiler should be fixed! */
44 #define __ppc__
45 #endif
46
47 #if defined(ETHR_DEBUG)
48 # undef ETHR_XCHK
49 # define ETHR_XCHK 1
50 #else
51 # ifndef ETHR_XCHK
52 # define ETHR_XCHK 0
53 # endif
54 #endif
55
56 #if defined(ETHR_DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \
57 || (defined(__GNUC__) && defined(ERTS_MIXED_VC))
58 # undef ETHR_INLINE
59 # define ETHR_INLINE
60 # undef ETHR_FORCE_INLINE
61 # define ETHR_FORCE_INLINE
62 # undef ETHR_TRY_INLINE_FUNCS
63 #endif
64
65 /* Assume 64-byte cache line size */
66 #define ETHR_CACHE_LINE_SIZE ASSUMED_CACHE_LINE_SIZE
67 #define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1)
68
69 #define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \
70 (((((SZ) - 1) / ETHR_CACHE_LINE_SIZE) + 1) * ETHR_CACHE_LINE_SIZE)
71
72 #ifndef ETHR_INLINE_FUNC_NAME_
73 # define ETHR_INLINE_FUNC_NAME_(X) X
74 #endif
75
76 #if !defined(__func__)
77 # if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
78 # if !defined(__GNUC__) || __GNUC__ < 2
79 # define __func__ "[unknown_function]"
80 # else
81 # define __func__ __FUNCTION__
82 # endif
83 # endif
84 #endif
85
86 #if defined(__GNUC__)
87 # define ETHR_PROTO_NORETURN__ void __attribute__((noreturn))
88 # define ETHR_IMPL_NORETURN__ void
89 #elif defined(__WIN32__) && defined(_MSC_VER)
90 # define ETHR_PROTO_NORETURN__ __declspec(noreturn) void
91 # define ETHR_IMPL_NORETURN__ __declspec(noreturn) void
92 #else
93 # define ETHR_PROTO_NORETURN__ void
94 # define ETHR_IMPL_NORETURN__ void
95 #endif
96
97 ETHR_PROTO_NORETURN__
98 ethr_assert_failed(const char *file, int line, const char *func, char *a);
99 #ifdef ETHR_DEBUG
100 #define ETHR_ASSERT(A) \
101 ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, __func__, #A)))
102 #else
103 #define ETHR_ASSERT(A) ((void) 1)
104 #endif
105
106
107 #if defined(ETHR_PTHREADS)
108 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
109 * The pthread implementation *
110 \* */
111
112 #if defined(__linux__) && !defined(_GNU_SOURCE)
113 #error "_GNU_SOURCE not defined. Please, compile all files with -D_GNU_SOURCE."
114 #endif
115
116 #ifdef ETHR_HAVE_PTHREAD_SETNAME_NP_1
117 #define _DARWIN_C_SOURCE
118 #endif
119
120 #if defined(ETHR_NEED_NPTL_PTHREAD_H)
121 #include <nptl/pthread.h>
122 #elif defined(ETHR_HAVE_MIT_PTHREAD_H)
123 #include <pthread/mit/pthread.h>
124 #elif defined(ETHR_HAVE_PTHREAD_H)
125 #include <pthread.h>
126 #endif
127
128 /* Types */
129
130 typedef pthread_t ethr_tid;
131
132 typedef pthread_key_t ethr_tsd_key;
133
134 #define ETHR_HAVE_ETHR_SIG_FUNCS 1
135
136 #if defined(VALGRIND)
137 # define ETHR_FORCE_PTHREAD_RWLOCK
138 # define ETHR_FORCE_PTHREAD_MUTEX
139 #endif
140
141 #if !defined(ETHR_FORCE_PTHREAD_RWLOCK)
142 # define ETHR_USE_OWN_RWMTX_IMPL__
143 #endif
144
145 #if !defined(ETHR_FORCE_PTHREAD_MUTEX) && 0
146 # define ETHR_USE_OWN_MTX_IMPL__
147 #endif
148
149 #elif defined(ETHR_WIN32_THREADS)
150 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
151 * The native win32 threads implementation *
152 \* */
153
154 #if !defined(_WIN32_WINNT)
155 #error "_WIN32_WINNT not defined. Please, compile all files with -D_WIN32_WINNT=0x0403"
156 #elif _WIN32_WINNT < 0x0403
157 #error "_WIN32_WINNT defined to a value less than 0x0403. Please, compile all files with -D_WIN32_WINNT=0x0403"
158 #endif
159
160 #ifdef WIN32_LEAN_AND_MEAN
161 # define ETHR_WIN32_LEAN_AND_MEAN_ALREADY_DEFINED
162 #else
163 # define WIN32_LEAN_AND_MEAN
164 #endif
165 #include <windows.h>
166 #ifndef ETHR_WIN32_LEAN_AND_MEAN_ALREADY_DEFINED
167 # undef WIN32_LEAN_AND_MEAN
168 #endif
169
170 #if defined(_MSC_VER)
171
172 #if ETHR_SIZEOF_LONG == 4
173 #define ETHR_HAVE_INT32_T 1
174 typedef long ethr_sint32_t;
175 typedef unsigned long ethr_uint32_t;
176 #endif
177
178 #if ETHR_SIZEOF___INT64 == 8
179 #define ETHR_HAVE_INT64_T 1
180 typedef __int64 ethr_sint64_t;
181 typedef unsigned __int64 ethr_uint64_t;
182 #endif
183
184 #endif
185
186 struct ethr_join_data_;
187
188 /* Types */
189 typedef struct {
190 long id;
191 struct ethr_join_data_ *jdata;
192 } ethr_tid; /* thread id type */
193
194 typedef DWORD ethr_tsd_key;
195
196 #undef ETHR_HAVE_ETHR_SIG_FUNCS
197
198 #define ETHR_USE_OWN_RWMTX_IMPL__
199
200 #define ETHR_YIELD() (Sleep(0), 0)
201
202 #else /* No supported thread lib found */
203
204 #ifdef ETHR_NO_SUPP_THR_LIB_NOT_FATAL
205 #define ETHR_NO_THREAD_LIB
206 #else
207 #error "No supported thread lib found"
208 #endif
209
210 #endif
211
212 #ifndef ETHR_HAVE_INT32_T
213 #if ETHR_SIZEOF_INT == 4
214 #define ETHR_HAVE_INT32_T 1
215 typedef int ethr_sint32_t;
216 typedef unsigned int ethr_uint32_t;
217 #elif ETHR_SIZEOF_LONG == 4
218 #define ETHR_HAVE_INT32_T 1
219 typedef long ethr_sint32_t;
220 typedef unsigned long ethr_uint32_t;
221 #endif
222 #endif
223
224 #ifndef ETHR_HAVE_INT64_T
225 #if ETHR_SIZEOF_INT == 8
226 #define ETHR_HAVE_INT64_T 1
227 typedef int ethr_sint64_t;
228 typedef unsigned int ethr_uint64_t;
229 #elif ETHR_SIZEOF_LONG == 8
230 #define ETHR_HAVE_INT64_T 1
231 typedef long ethr_sint64_t;
232 typedef unsigned long ethr_uint64_t;
233 #elif ETHR_SIZEOF_LONG_LONG == 8
234 #define ETHR_HAVE_INT64_T 1
235 typedef long long ethr_sint64_t;
236 typedef unsigned long long ethr_uint64_t;
237 #endif
238 #endif
239
240 #if ETHR_SIZEOF_PTR == 4
241 #ifndef ETHR_HAVE_INT32_T
242 #error "No 32-bit integer type found"
243 #endif
244 typedef ethr_sint32_t ethr_sint_t;
245 typedef ethr_uint32_t ethr_uint_t;
246 #elif ETHR_SIZEOF_PTR == 8
247 #ifndef ETHR_HAVE_INT64_T
248 #error "No 64-bit integer type found"
249 #endif
250 typedef ethr_sint64_t ethr_sint_t;
251 typedef ethr_uint64_t ethr_uint_t;
252 #endif
253
254 #if defined(ETHR_SIZEOF___INT128_T) && ETHR_SIZEOF___INT128_T == 16
255 #define ETHR_HAVE_INT128_T
256 typedef __int128_t ethr_sint128_t;
257 typedef __uint128_t ethr_uint128_t;
258 #endif
259
260 #define ETHR_FATAL_ERROR__(ERR) \
261 ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR))
262
263 ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file,
264 int line,
265 const char *func,
266 int err);
267
268 #if !ETHR_AT_LEAST_GCC_VSN__(2, 96, 0)
269 #define __builtin_expect(X, Y) (X)
270 #endif
271
272 #if ETHR_AT_LEAST_GCC_VSN__(3, 1, 1) && !defined __cplusplus
273 # define ETHR_CHOOSE_EXPR __builtin_choose_expr
274 #else
275 # define ETHR_CHOOSE_EXPR(B, E1, E2) ((B) ? (E1) : (E2))
276 #endif
277
278 #if ((defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) \
279 || (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))))
280 # define ETHR_X86_RUNTIME_CONF__
281
282 # define ETHR_X86_RUNTIME_CONF_HAVE_META(feature) \
283 (__builtin_expect(ethr_runtime__.conf.have_##feature != 0, 1))
284 # define ETHR_X86_RUNTIME_CONF_HAVE_NO_META(feature) \
285 (__builtin_expect(ethr_runtime__.conf.have_##feature == 0, 0))
286
287 # define ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ \
288 ETHR_X86_RUNTIME_CONF_HAVE_META(dw_cmpxchg)
289 # define ETHR_X86_RUNTIME_CONF_HAVE_NO_DW_CMPXCHG__ \
290 ETHR_X86_RUNTIME_CONF_HAVE_NO_META(dw_cmpxchg)
291 # define ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ \
292 ETHR_X86_RUNTIME_CONF_HAVE_META(sse2)
293 # define ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__ \
294 ETHR_X86_RUNTIME_CONF_HAVE_NO_META(sse2)
295 # define ETHR_X86_RUNTIME_CONF_HAVE_RDTSCP__ \
296 ETHR_X86_RUNTIME_CONF_HAVE_META(rdtscp)
297 # define ETHR_X86_RUNTIME_CONF_HAVE_NO_RDTSCP__ \
298 ETHR_X86_RUNTIME_CONF_HAVE_NO_META(rdtscp)
299 # define ETHR_X86_RUNTIME_CONF_HAVE_CONSTANT_TSC__ \
300 ETHR_X86_RUNTIME_CONF_HAVE_META(constant_tsc)
301 # define ETHR_X86_RUNTIME_CONF_HAVE_NO_CONSTANT_TSC__ \
302 ETHR_X86_RUNTIME_CONF_HAVE_NO_META(nonstop_tsc)
303 # define ETHR_X86_RUNTIME_CONF_HAVE_NONSTOP_TSC__ \
304 ETHR_X86_RUNTIME_CONF_HAVE_META(nonstop_tsc)
305 # define ETHR_X86_RUNTIME_CONF_HAVE_NO_NONSTOP_TSC__ \
306 ETHR_X86_RUNTIME_CONF_HAVE_NO_META(nonstop_tsc)
307 # define ETHR_X86_RUNTIME_CONF_HAVE_TSC_RELIABLE__ \
308 ETHR_X86_RUNTIME_CONF_HAVE_META(tsc_reliable)
309 # define ETHR_X86_RUNTIME_CONF_HAVE_NO_TSC_RELIABLE_TSC__ \
310 ETHR_X86_RUNTIME_CONF_HAVE_NO_META(tsc_reliable)
311 # define ETHR_X86_RUNTIME_CONF_HAVE_NONSTOP_TSC_S3__ \
312 ETHR_X86_RUNTIME_CONF_HAVE_META(nonstop_tsc_s3)
313 # define ETHR_X86_RUNTIME_CONF_HAVE_NO_NONSTOP_TSC_S3__ \
314 ETHR_X86_RUNTIME_CONF_HAVE_NO_META(nonstop_tsc_s3)
315
316 #endif
317
318 #if (defined(__GNUC__) \
319 && !defined(ETHR_PPC_HAVE_LWSYNC) \
320 && !defined(ETHR_PPC_HAVE_NO_LWSYNC) \
321 && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)))
322 # define ETHR_PPC_RUNTIME_CONF__
323
324 # define ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ \
325 (__builtin_expect(ethr_runtime__.conf.have_lwsync != 0, 1))
326 # define ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__ \
327 (__builtin_expect(ethr_runtime__.conf.have_lwsync == 0, 0))
328 #endif
329
330 typedef struct {
331 #if defined(ETHR_X86_RUNTIME_CONF__)
332 int have_dw_cmpxchg;
333 int have_sse2;
334 int have_rdtscp;
335 int have_constant_tsc;
336 int have_tsc_reliable;
337 int have_nonstop_tsc;
338 int have_nonstop_tsc_s3;
339 #endif
340 #if defined(ETHR_PPC_RUNTIME_CONF__)
341 int have_lwsync;
342 #endif
343 int dummy;
344 } ethr_runtime_conf_t;
345
346
347 typedef union {
348 ethr_runtime_conf_t conf;
349 char pad__[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_runtime_conf_t))+ETHR_CACHE_LINE_SIZE];
350 } ethr_runtime_t;
351
352
353 extern ethr_runtime_t ethr_runtime__;
354
355 /* For native CPU-optimised atomics, spinlocks, and rwlocks. */
356 #if !defined(ETHR_DISABLE_NATIVE_IMPLS)
357 # if defined(__GNUC__)
358 # if defined(ETHR_PREFER_GCC_NATIVE_IMPLS)
359 # include "gcc/ethread.h"
360 # elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS)
361 # include "libatomic_ops/ethread.h"
362 # endif
363 # if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64)
364 # if ETHR_SIZEOF_PTR == 4
365 # if defined(__i386__)
366 # include "i386/ethread.h"
367 # elif (defined(__powerpc__)||defined(__ppc__))&&!defined(__powerpc64__)
368 # include "ppc32/ethread.h"
369 # elif defined(__sparc__)
370 # include "sparc32/ethread.h"
371 # elif defined(__tile__)
372 # include "tile/ethread.h"
373 # endif
374 # elif ETHR_SIZEOF_PTR == 8
375 # if defined(__x86_64__)
376 # include "x86_64/ethread.h"
377 # elif defined(__sparc__) && defined(__arch64__)
378 # include "sparc64/ethread.h"
379 # endif
380 # endif
381 # if ETHR_HAVE_GCC___ATOMIC_BUILTINS
382 # include "gcc/ethread.h"
383 # endif
384 # include "libatomic_ops/ethread.h"
385 # include "gcc/ethread.h"
386 # endif
387 # elif defined(ETHR_HAVE_LIBATOMIC_OPS)
388 # include "libatomic_ops/ethread.h"
389 # elif defined(ETHR_WIN32_THREADS)
390 # include "win/ethread.h"
391 # endif
392 #endif /* !ETHR_DISABLE_NATIVE_IMPLS */
393
394 #if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64) && !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(ETHR_SMP_REQUIRE_NATIVE_IMPLS)
395 #error "No native ethread implementation found. If you want to use fallbacks you have to disable native ethread support with configure."
396 #endif
397
398 #include "ethr_atomics.h" /* The atomics API */
399
400 #if defined(__GNUC__)
401 # ifndef ETHR_SPIN_BODY
402 # if defined(__i386__) || defined(__x86_64__)
403 # define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory")
404 # elif defined(__ia64__)
405 # define ETHR_SPIN_BODY __asm__ __volatile__("hint @pause" : : : "memory")
406 # elif defined(__sparc__)
407 # define ETHR_SPIN_BODY __asm__ __volatile__("membar #LoadLoad")
408 # else
409 # define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
410 # endif
411 # endif
412 #elif defined(ETHR_WIN32_THREADS)
413 # ifndef ETHR_SPIN_BODY
414 # define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0)
415 # endif
416 #endif
417
418 #define ETHR_YIELD_AFTER_BUSY_LOOPS 50
419
420 #ifndef ETHR_SPIN_BODY
421 # define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
422 #endif
423
424 #ifndef ETHR_YIELD
425 # if defined(ETHR_HAVE_SCHED_YIELD)
426 # ifdef ETHR_HAVE_SCHED_H
427 # include <sched.h>
428 # endif
429 # include <errno.h>
430 # if defined(ETHR_SCHED_YIELD_RET_INT)
431 # define ETHR_YIELD() (sched_yield() < 0 ? errno : 0)
432 # else
433 # define ETHR_YIELD() (sched_yield(), 0)
434 # endif
435 # elif defined(ETHR_HAVE_PTHREAD_YIELD)
436 # if defined(ETHR_PTHREAD_YIELD_RET_INT)
437 # define ETHR_YIELD() pthread_yield()
438 # else
439 # define ETHR_YIELD() (pthread_yield(), 0)
440 # endif
441 # else
442 # define ETHR_YIELD() (ethr_compiler_barrier(), 0)
443 # endif
444 #endif
445
446 #if defined(VALGRIND)
447 /* mutex as fallback for spinlock for VALGRIND. */
448 # undef ETHR_HAVE_NATIVE_SPINLOCKS
449 # undef ETHR_HAVE_NATIVE_RWSPINLOCKS
450 #else
451 # include "ethr_optimized_fallbacks.h"
452 #endif
453
454 typedef struct {
455 void *(*thread_create_prepare_func)(void);
456 void (*thread_create_parent_func)(void *);
457 void (*thread_create_child_func)(void *);
458 } ethr_init_data;
459
460 #define ETHR_INIT_DATA_DEFAULT_INITER {NULL, NULL, NULL}
461
462 typedef struct {
463 void *(*alloc)(size_t);
464 void *(*realloc)(void *, size_t);
465 void (*free)(void *);
466 } ethr_memory_allocator;
467
468 #define ETHR_MEM_ALLOC_DEF_INITER__ {NULL, NULL, NULL}
469
470 typedef struct {
471 ethr_memory_allocator std;
472 ethr_memory_allocator sl;
473 ethr_memory_allocator ll;
474 } ethr_memory_allocators;
475
476 #define ETHR_MEM_ALLOCS_DEF_INITER__ \
477 {ETHR_MEM_ALLOC_DEF_INITER__, \
478 ETHR_MEM_ALLOC_DEF_INITER__, \
479 ETHR_MEM_ALLOC_DEF_INITER__}
480
481 typedef struct {
482 ethr_memory_allocators mem;
483 int reader_groups;
484 int main_threads;
485 } ethr_late_init_data;
486
487 #define ETHR_LATE_INIT_DATA_DEFAULT_INITER \
488 {ETHR_MEM_ALLOCS_DEF_INITER__, 0, 0}
489
490 typedef struct {
491 int detached; /* boolean (default false) */
492 int suggested_stack_size; /* kilo words (default sys dependent) */
493 char *name; /* max 14 char long (default no-name) */
494 } ethr_thr_opts;
495
496 #define ETHR_THR_OPTS_DEFAULT_INITER {0, -1, NULL}
497
498 #if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
499 # define ETHR_NEED_SPINLOCK_PROTOTYPES__
500 # define ETHR_NEED_RWSPINLOCK_PROTOTYPES__
501 #endif
502
503 #if defined(__WIN32__)
504 int ethr_win_get_errno__(void);
505 #endif
506
507 int ethr_init(ethr_init_data *);
508 int ethr_late_init(ethr_late_init_data *);
509 int ethr_install_exit_handler(void (*funcp)(void));
510 int ethr_thr_create(ethr_tid *, void * (*)(void *), void *, ethr_thr_opts *);
511 int ethr_thr_join(ethr_tid, void **);
512 int ethr_thr_detach(ethr_tid);
513 void ethr_thr_exit(void *);
514 ethr_tid ethr_self(void);
515 int ethr_getname(ethr_tid, char *, size_t);
516 void ethr_setname(char *);
517 int ethr_equal_tids(ethr_tid, ethr_tid);
518
519 int ethr_tsd_key_create(ethr_tsd_key *,char *);
520 int ethr_tsd_key_delete(ethr_tsd_key);
521 int ethr_tsd_set(ethr_tsd_key, void *);
522 void *ethr_tsd_get(ethr_tsd_key);
523
524 void *ethr_get_stacklimit(void);
525 int ethr_set_stacklimit(void *limit);
526
527 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
528 #include <signal.h>
529 int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset);
530 int ethr_sigwait(const sigset_t *set, int *sig);
531 int ethr_kill(const ethr_tid tid, const int sig);
532 #endif
533
534 void ethr_compiler_barrier(void);
535
536 #if defined(ETHR_HAVE_NATIVE_SPINLOCKS)
537 typedef ethr_native_spinlock_t ethr_spinlock_t;
538 #elif defined(__WIN32__)
539 typedef CRITICAL_SECTION ethr_spinlock_t;
540 #else
541 typedef pthread_mutex_t ethr_spinlock_t;
542 #endif
543
544 #ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__
545 int ethr_spinlock_init(ethr_spinlock_t *);
546 int ethr_spinlock_destroy(ethr_spinlock_t *);
547 void ethr_spin_unlock(ethr_spinlock_t *);
548 void ethr_spin_lock(ethr_spinlock_t *);
549 #endif
550
551 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
552
553 static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)554 ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
555 {
556 #ifdef ETHR_HAVE_NATIVE_SPINLOCKS
557 ethr_native_spinlock_init(lock);
558 return 0;
559 #elif defined(__WIN32__)
560 if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX))
561 return ethr_win_get_errno__();
562 return 0;
563 #else
564 return pthread_mutex_init((pthread_mutex_t *) lock, NULL);
565 #endif
566 }
567
568 static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)569 ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
570 {
571 #ifdef ETHR_HAVE_NATIVE_SPINLOCKS
572 return ethr_native_spinlock_destroy(lock);
573 #elif defined(__WIN32__)
574 DeleteCriticalSection((CRITICAL_SECTION *) lock);
575 return 0;
576 #else
577 return pthread_mutex_destroy((pthread_mutex_t *) lock);
578 #endif
579 }
580
581 static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)582 ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
583 {
584 #ifdef ETHR_HAVE_NATIVE_SPINLOCKS
585 ethr_native_spin_unlock(lock);
586 #elif defined(__WIN32__)
587 LeaveCriticalSection((CRITICAL_SECTION *) lock);
588 #else
589 int err = pthread_mutex_unlock((pthread_mutex_t *) lock);
590 if (err)
591 ETHR_FATAL_ERROR__(err);
592 #endif
593 }
594
595 static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)596 ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
597 {
598 #ifdef ETHR_HAVE_NATIVE_SPINLOCKS
599 ethr_native_spin_lock(lock);
600 #elif defined(__WIN32__)
601 EnterCriticalSection((CRITICAL_SECTION *) lock);
602 #else
603 int err = pthread_mutex_lock((pthread_mutex_t *) lock);
604 if (err)
605 ETHR_FATAL_ERROR__(err);
606 #endif
607 }
608
609 #endif /* ETHR_TRY_INLINE_FUNCS */
610
611 typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */
612
613 #if defined(ETHR_WIN32_THREADS)
614 # include "win/ethr_event.h"
615 #elif defined(ETHR_PTHREADS)
616 # include "pthread/ethr_event.h"
617 #endif
618
619 int ethr_set_main_thr_status(int, int);
620 int ethr_get_main_thr_status(int *);
621
622 struct ethr_ts_event_ {
623 ethr_ts_event *next;
624 ethr_ts_event *prev;
625 ethr_event event;
626 void *udata;
627 ethr_atomic32_t uaflgs;
628 unsigned uflgs;
629 unsigned iflgs; /* for ethr lib only */
630 short rgix; /* for ethr lib only */
631 short mtix; /* for ethr lib only */
632 };
633
634 #define ETHR_TS_EV_ETHREAD (((unsigned) 1) << 0)
635 #define ETHR_TS_EV_INITED (((unsigned) 1) << 1)
636 #define ETHR_TS_EV_TMP (((unsigned) 1) << 2)
637 #define ETHR_TS_EV_MAIN_THR (((unsigned) 1) << 3)
638 #define ETHR_TS_EV_BUSY (((unsigned) 1) << 4)
639 #define ETHR_TS_EV_PEEK (((unsigned) 1) << 5)
640
641 ethr_sint64_t ethr_no_used_tse(void);
642 int ethr_free_ts_event__(ethr_ts_event *tsep);
643 int ethr_make_ts_event__(ethr_ts_event **tsepp, int tmp);
644
645 #if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
646 ethr_ts_event *ethr_lookup_ts_event__(int busy_dup);
647 ethr_ts_event *ethr_peek_ts_event(void);
648 void ethr_unpeek_ts_event(ethr_ts_event *);
649 ethr_ts_event *ethr_use_ts_event(ethr_ts_event *tsep);
650 ethr_ts_event *ethr_get_ts_event(void);
651 void ethr_leave_ts_event(ethr_ts_event *);
652 #endif
653
654 #if defined(ETHR_PTHREADS)
655
656 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
657
658 extern pthread_key_t ethr_ts_event_key__;
659
660 static ETHR_INLINE ethr_ts_event *
ETHR_INLINE_FUNC_NAME_(ethr_lookup_ts_event__)661 ETHR_INLINE_FUNC_NAME_(ethr_lookup_ts_event__)(int busy_dup)
662 {
663 ethr_ts_event *tsep = (ethr_ts_event*)pthread_getspecific(ethr_ts_event_key__);
664 if (!tsep || (busy_dup && (tsep->iflgs & ETHR_TS_EV_BUSY))) {
665 int res = ethr_make_ts_event__(&tsep, 0);
666 if (res != 0)
667 ETHR_FATAL_ERROR__(res);
668 ETHR_ASSERT(tsep);
669 }
670 return tsep;
671 }
672
673 #endif
674
675 #elif defined(ETHR_WIN32_THREADS)
676
677 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
678
679 extern DWORD ethr_ts_event_key__;
680
681 static ETHR_INLINE ethr_ts_event *
ETHR_INLINE_FUNC_NAME_(ethr_lookup_ts_event__)682 ETHR_INLINE_FUNC_NAME_(ethr_lookup_ts_event__)(int busy_dup)
683 {
684 ethr_ts_event *tsep = (ethr_ts_event *) TlsGetValue(ethr_ts_event_key__);
685 if (!tsep || (busy_dup && (tsep->iflgs & ETHR_TS_EV_BUSY))) {
686 int res = ethr_make_ts_event__(&tsep, !0);
687 if (res != 0)
688 ETHR_FATAL_ERROR__(res);
689 ETHR_ASSERT(tsep);
690 }
691 return tsep;
692 }
693
694 #endif
695
696 #endif
697
698 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
699
700 static ETHR_INLINE ethr_ts_event *
ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)701 ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void)
702 {
703 ethr_ts_event *tsep = ethr_lookup_ts_event__(!0);
704 ETHR_ASSERT(!(tsep->iflgs & ETHR_TS_EV_BUSY));
705 tsep->iflgs |= ETHR_TS_EV_BUSY;
706 return tsep;
707 }
708
709 static ETHR_INLINE ethr_ts_event *
ETHR_INLINE_FUNC_NAME_(ethr_peek_ts_event)710 ETHR_INLINE_FUNC_NAME_(ethr_peek_ts_event)(void)
711 {
712 ethr_ts_event *tsep = ethr_lookup_ts_event__(0);
713 ETHR_ASSERT(!(tsep->iflgs & ETHR_TS_EV_PEEK));
714 tsep->iflgs |= ETHR_TS_EV_PEEK;
715 return tsep;
716 }
717
718 static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_unpeek_ts_event)719 ETHR_INLINE_FUNC_NAME_(ethr_unpeek_ts_event)(ethr_ts_event *tsep)
720 {
721 ETHR_ASSERT(tsep->iflgs & ETHR_TS_EV_PEEK);
722 tsep->iflgs &= ~ETHR_TS_EV_PEEK;
723 if ((tsep->iflgs & (ETHR_TS_EV_TMP|ETHR_TS_EV_BUSY)) == ETHR_TS_EV_TMP) {
724 int res = ethr_free_ts_event__(tsep);
725 if (res != 0)
726 ETHR_FATAL_ERROR__(res);
727 }
728 }
729
730 static ETHR_INLINE ethr_ts_event *
ETHR_INLINE_FUNC_NAME_(ethr_use_ts_event)731 ETHR_INLINE_FUNC_NAME_(ethr_use_ts_event)(ethr_ts_event *tsep)
732 {
733 ethr_ts_event *tmp_tsep = tsep;
734 if (tmp_tsep->iflgs & ETHR_TS_EV_BUSY) {
735 int res = ethr_make_ts_event__(&tmp_tsep, !0);
736 if (res != 0)
737 ETHR_FATAL_ERROR__(res);
738 ETHR_ASSERT(tmp_tsep && tsep != tmp_tsep);
739 }
740 ETHR_ASSERT(!(tmp_tsep->iflgs & ETHR_TS_EV_BUSY));
741 tmp_tsep->iflgs |= ETHR_TS_EV_BUSY;
742 return tmp_tsep;
743 }
744
745 static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)746 ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep)
747 {
748 ETHR_ASSERT(tsep->iflgs & ETHR_TS_EV_BUSY);
749 tsep->iflgs &= ~ETHR_TS_EV_BUSY;
750 if ((tsep->iflgs & (ETHR_TS_EV_TMP|ETHR_TS_EV_PEEK)) == ETHR_TS_EV_TMP) {
751 int res = ethr_free_ts_event__(tsep);
752 if (res != 0)
753 ETHR_FATAL_ERROR__(res);
754 }
755 }
756
757 #endif
758
759 #include "ethr_mutex.h" /* Need atomic declarations and tse */
760
761 #ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
762 typedef ethr_native_rwlock_t ethr_rwlock_t;
763 #else
764 typedef ethr_rwmutex ethr_rwlock_t;
765 #endif
766
767 #ifdef ETHR_NEED_RWSPINLOCK_PROTOTYPES__
768 int ethr_rwlock_init(ethr_rwlock_t *);
769 int ethr_rwlock_destroy(ethr_rwlock_t *);
770 void ethr_read_unlock(ethr_rwlock_t *);
771 void ethr_read_lock(ethr_rwlock_t *);
772 void ethr_write_unlock(ethr_rwlock_t *);
773 void ethr_write_lock(ethr_rwlock_t *);
774 #endif
775
776 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
777
778 static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)779 ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock)
780 {
781 #ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
782 ethr_native_rwlock_init(lock);
783 return 0;
784 #else
785 return ethr_rwmutex_init_opt((ethr_rwmutex *) lock, NULL);
786 #endif
787 }
788
789 static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)790 ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock)
791 {
792 #ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
793 return ethr_native_rwlock_destroy(lock);
794 #else
795 return ethr_rwmutex_destroy((ethr_rwmutex *) lock);
796 #endif
797 }
798
799 static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)800 ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock)
801 {
802 #ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
803 ethr_native_read_unlock(lock);
804 #else
805 ethr_rwmutex_runlock((ethr_rwmutex *) lock);
806 #endif
807 }
808
809 static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_read_lock)810 ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock)
811 {
812 #ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
813 ethr_native_read_lock(lock);
814 #else
815 ethr_rwmutex_rlock((ethr_rwmutex *) lock);
816 #endif
817 }
818
819 static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)820 ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock)
821 {
822 #ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
823 ethr_native_write_unlock(lock);
824 #else
825 ethr_rwmutex_rwunlock((ethr_rwmutex *) lock);
826 #endif
827 }
828
829 static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_write_lock)830 ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
831 {
832 #ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
833 ethr_native_write_lock(lock);
834 #else
835 ethr_rwmutex_rwlock((ethr_rwmutex *) lock);
836 #endif
837 }
838
839 #endif /* ETHR_TRY_INLINE_FUNCS */
840
841 #endif /* #ifndef ETHREAD_H__ */
842