1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2010-2016. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 /*
22 * Description: Mutex, rwmutex and condition variable implementation
23 * Author: Rickard Green
24 */
25
26 #ifdef HAVE_CONFIG_H
27 #include "config.h"
28 #endif
29
30 #define ETHR_INLINE_MTX_FUNC_NAME_(X) X ## __
31 #define ETHR_MUTEX_IMPL__
32 #define ETHR_TRY_INLINE_FUNCS
33
34 #include <limits.h>
35 #include "ethread.h"
36 #include "ethr_internal.h"
37
38 #define ETHR_SPIN_WITH_WAITERS 1
39
40 #define ETHR_MTX_MAX_FLGS_SPIN 10
41
42 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
43 static int default_rwmtx_main_spincount;
44 static int default_rwmtx_aux_spincount;
45 #endif
46 #ifdef ETHR_USE_OWN_MTX_IMPL__
47 static int default_mtx_main_spincount;
48 static int default_mtx_aux_spincount;
49 static int default_cnd_main_spincount;
50 static int default_cnd_aux_spincount;
51 #endif
52
53 static int no_spin;
54
55 #ifndef ETHR_USE_OWN_RWMTX_IMPL__
56 static pthread_rwlockattr_t write_pref_attr_data;
57 static pthread_rwlockattr_t *write_pref_attr;
58 #endif
59
60 #if defined(ETHR_MTX_Q_LOCK_SPINLOCK__)
61 # define ETHR_MTX_QLOCK_INIT ethr_spinlock_init
62 # define ETHR_MTX_QLOCK_DESTROY ethr_spinlock_destroy
63 # define ETHR_MTX_Q_LOCK ethr_spin_lock
64 # define ETHR_MTX_Q_UNLOCK ethr_spin_unlock
65 #elif defined(ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__)
66 # define ETHR_MTX_QLOCK_INIT(QL) pthread_mutex_init((QL), NULL)
67 # define ETHR_MTX_QLOCK_DESTROY pthread_mutex_destroy
68 # define ETHR_MTX_Q_LOCK(L) \
69 do { \
70 int res__ = pthread_mutex_lock(L); \
71 if (res__ != 0) \
72 ETHR_FATAL_ERROR__(res__); \
73 } while (0)
74 # define ETHR_MTX_Q_UNLOCK(L) \
75 do { \
76 int res__ = pthread_mutex_unlock(L); \
77 if (res__ != 0) \
78 ETHR_FATAL_ERROR__(res__); \
79 } while (0)
80 #elif defined(ETHR_MTX_Q_LOCK_CRITICAL_SECTION__)
81 # define ETHR_MTX_QLOCK_INIT(QL) (InitializeCriticalSection((QL)), 0)
82 # define ETHR_MTX_QLOCK_DESTROY(QL) (DeleteCriticalSection((QL)), 0)
83 # define ETHR_MTX_Q_LOCK(QL) EnterCriticalSection((QL))
84 # define ETHR_MTX_Q_UNLOCK(QL) LeaveCriticalSection((QL))
85 #endif
86
87 int
ethr_mutex_lib_init(int cpu_conf)88 ethr_mutex_lib_init(int cpu_conf)
89 {
90 int res = 0;
91
92 no_spin = cpu_conf == 1;
93
94 #ifdef ETHR_USE_OWN_MTX_IMPL__
95 default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE;
96 default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
97 default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
98 default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
99 #endif
100
101 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
102
103 default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE;
104 default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
105
106 #else
107
108 #if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \
109 && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
110 res = pthread_rwlockattr_init(&write_pref_attr_data);
111 if (res != 0)
112 return res;
113 res = pthread_rwlockattr_setkind_np(
114 &write_pref_attr_data,
115 PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
116 write_pref_attr = &write_pref_attr_data;
117 #else
118 write_pref_attr = NULL;
119 #endif
120
121 #endif
122
123 return res;
124 }
125
126 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
127
128 #ifdef ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS
129 #if 0 /*
130 * When inc and dec are real atomic instructions as on x86, the
131 * ETHR_RLOCK_WITH_INC_DEC implementations performs better with
132 * lots of read locks compared to the cmpxchg based implementation.
133 * It, however, performs worse with lots of mixed reads and writes.
134 * It could be used for rwlocks that are known to be read locked
135 * much, but the readers array based implementation outperforms it
136 * by far. Therefore, it has been disabled, and will probably be
137 * removed some time in the future.
138 */
139 # define ETHR_RLOCK_WITH_INC_DEC
140 #endif
141 #endif
142
143 static int reader_groups_array_size = 0;
144 static int main_threads_array_size = 0;
145
146 #endif
147
148 int
ethr_mutex_lib_late_init(int no_reader_groups,int no_main_threads)149 ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
150 {
151
152 #ifdef ETHR_USE_OWN_MTX_IMPL__
153 default_mtx_main_spincount += (no_main_threads
154 * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC);
155 if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX)
156 default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX;
157 #endif
158
159 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
160
161 default_rwmtx_main_spincount += (no_main_threads
162 * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC);
163 if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX)
164 default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX;
165
166 reader_groups_array_size = (no_reader_groups <= 1
167 ? 1
168 : no_reader_groups + 1);
169 main_threads_array_size = (no_main_threads <= 1
170 ? 1
171 : no_main_threads + 1);
172 #endif
173 return 0;
174 }
175
176 int
ethr_rwmutex_set_reader_group(int ix)177 ethr_rwmutex_set_reader_group(int ix)
178 {
179 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
180 ethr_ts_event *tse;
181
182 if (ix < 0 || reader_groups_array_size <= ix)
183 return EINVAL;
184
185 tse = ethr_get_ts_event();
186
187 if ((tse->iflgs & (ETHR_TS_EV_ETHREAD|ETHR_TS_EV_TMP)) == 0) {
188 ethr_leave_ts_event(tse);
189 return EINVAL;
190 }
191
192 tse->rgix = ix;
193
194 ethr_leave_ts_event(tse);
195 #endif
196 return 0;
197 }
198
199 #if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
200 static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int);
201 #define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) hard_debug_chk_q__(&(RWMTX)->mtxb,1)
202 #define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) hard_debug_chk_q__(&(MTX)->mtxb, 0)
203 #else
204 #define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX)
205 #define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX)
206 #endif
207
208 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
209 static void
210 rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
211 ethr_sint32_t initial,
212 int q_locked);
213 static void
214 rwmutex_unlock_wake(ethr_rwmutex *rwmtx,
215 int have_w,
216 ethr_sint32_t initial,
217 int transfer_read_lock);
218 static int
219 rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
220 ethr_sint32_t initial,
221 ethr_ts_event *tse,
222 int start_next_ix,
223 int check_before_try,
224 int try_write_lock);
225 #endif
226
227 /* -- Utilities used by multiple implementations -- */
228
229 #if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) \
230 || defined(ETHR_WIN32_THREADS)
231
232 static ETHR_INLINE void
enqueue(ethr_ts_event ** queue,ethr_ts_event * tse_start,ethr_ts_event * tse_end)233 enqueue(ethr_ts_event **queue,
234 ethr_ts_event *tse_start,
235 ethr_ts_event *tse_end)
236 {
237 if (!*queue) {
238 *queue = tse_start;
239 tse_start->prev = tse_end;
240 tse_end->next = tse_start;
241 }
242 else {
243 tse_end->next = *queue;
244 tse_start->prev = (*queue)->prev;
245 (*queue)->prev->next = tse_start;
246 (*queue)->prev = tse_end;
247 }
248 }
249
250
251 static ETHR_INLINE void
dequeue(ethr_ts_event ** queue,ethr_ts_event * tse_start,ethr_ts_event * tse_end)252 dequeue(ethr_ts_event **queue,
253 ethr_ts_event *tse_start,
254 ethr_ts_event *tse_end)
255 {
256 if (tse_start->prev == tse_end) {
257 ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start);
258 *queue = NULL;
259 }
260 else {
261 if (*queue == tse_start)
262 *queue = tse_end->next;
263 tse_end->next->prev = tse_start->prev;
264 tse_start->prev->next = tse_end->next;
265 }
266 }
267
268 #endif
269
270 #if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
271
272 static ETHR_INLINE void
insert(ethr_ts_event * tse_pred,ethr_ts_event * tse)273 insert(ethr_ts_event *tse_pred, ethr_ts_event *tse)
274 {
275 tse->next = tse_pred->next;
276 tse->prev = tse_pred;
277 tse_pred->next->prev = tse;
278 tse_pred->next = tse;
279 }
280
281 static ETHR_INLINE void
rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex * rwmtx,ethr_ts_event * tse)282 rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
283 {
284 int ix = (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
285 ? tse->rgix
286 : tse->mtix);
287 rwmtx->tdata.ra[ix].data.waiting_readers++;
288 }
289
290 static ETHR_INLINE void
rwmutex_freqread_rdrs_add(ethr_rwmutex * rwmtx,ethr_rwmutex_type type,int ix,int inc)291 rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx,
292 ethr_rwmutex_type type,
293 int ix,
294 int inc)
295 {
296 if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0)
297 ethr_atomic32_add(&rwmtx->tdata.ra[ix].data.readers, inc);
298 else {
299 ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
300 ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
301 ETHR_ASSERT(inc == 1);
302 ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
303 }
304 }
305
306 static ETHR_INLINE void
rwmutex_freqread_rdrs_inc(ethr_rwmutex * rwmtx,ethr_ts_event * tse)307 rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
308 {
309 int ix;
310 if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
311 ix = tse->rgix;
312 atomic_inc:
313 ethr_atomic32_inc(&rwmtx->tdata.ra[ix].data.readers);
314 }
315 else {
316 ix = tse->mtix;
317 if (ix == 0)
318 goto atomic_inc;
319 ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
320 ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
321 ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
322 }
323 }
324
325 #if 0 /* Not used */
326
327 static ETHR_INLINE void
328 rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
329 {
330 int ix;
331 if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
332 ix = tse->rgix;
333 atomic_dec:
334 ethr_atomic32_dec(&rwmtx->tdata.ra[ix].data.readers);
335 }
336 else {
337 ix = tse->mtix;
338 if (ix == 0)
339 goto atomic_dec;
340 ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
341 ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
342 ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
343 }
344 }
345
346 #endif
347
348 static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_dec_read(ethr_rwmutex * rwmtx,ethr_ts_event * tse)349 rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
350 {
351 int ix;
352 if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
353 ix = tse->rgix;
354 atomic_dec_read:
355 return ethr_atomic32_dec_read(&rwmtx->tdata.ra[ix].data.readers);
356 }
357 else {
358 ix = tse->mtix;
359 if (ix == 0)
360 goto atomic_dec_read;
361 ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
362 ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
363 ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
364 return (ethr_sint32_t) 0;
365 }
366 }
367
368 static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex * rwmtx,ethr_ts_event * tse)369 rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
370 {
371 int ix;
372 if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
373 ix = tse->rgix;
374 atomic_dec_read:
375 return ethr_atomic32_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
376 }
377 else {
378 ix = tse->mtix;
379 if (ix == 0)
380 goto atomic_dec_read;
381 ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
382 ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
383 ethr_atomic32_set_relb(&rwmtx->tdata.ra[ix].data.readers,
384 (ethr_sint32_t) 0);
385 return (ethr_sint32_t) 0;
386 }
387 }
388
389 static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_read(ethr_rwmutex * rwmtx,int ix)390 rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix)
391 {
392 ethr_sint32_t res = ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers);
393 #ifdef ETHR_DEBUG
394 switch (rwmtx->type) {
395 case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
396 ETHR_ASSERT(res >= 0);
397 break;
398 case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
399 ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1));
400 break;
401 default:
402 ETHR_ASSERT(0);
403 break;
404 }
405 #endif
406 return res;
407 }
408
409 static void
event_wait(struct ethr_mutex_base_ * mtxb,ethr_ts_event * tse,int spincount,ethr_sint32_t type,int is_rwmtx,int is_freq_read)410 event_wait(struct ethr_mutex_base_ *mtxb,
411 ethr_ts_event *tse,
412 int spincount,
413 ethr_sint32_t type,
414 int is_rwmtx,
415 int is_freq_read)
416 {
417 int locked = 0;
418 ethr_sint32_t act;
419 int need_try_complete_runlock = 0;
420 int transfer_read_lock = 0;
421
422 ETHR_ASSERT(tse->iflgs & ETHR_TS_EV_BUSY);
423
424 /* Need to enqueue and wait... */
425
426 tse->uflgs = type;
427 ethr_atomic32_set(&tse->uaflgs, type);
428
429 ETHR_MTX_Q_LOCK(&mtxb->qlck);
430 locked = 1;
431
432 #ifdef ETHR_MTX_HARD_DEBUG_Q
433 hard_debug_chk_q__(mtxb, is_rwmtx);
434 #endif
435
436 act = ethr_atomic32_read(&mtxb->flgs);
437
438 if (act & type) {
439
440 /* Wait bit already there; enqueue... */
441
442 ETHR_ASSERT(mtxb->q);
443 if (type == ETHR_RWMTX_W_WAIT_FLG__) {
444 enqueue(&mtxb->q, tse, tse);
445 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
446 mtxb->ws++;
447 #endif
448 }
449 else {
450 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
451 ETHR_ASSERT(is_rwmtx);
452 ETHR_ASSERT(rwmtx->rq_end);
453 insert(rwmtx->rq_end, tse);
454 rwmtx->rq_end = tse;
455 if (is_freq_read)
456 rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
457 else
458 rwmtx->tdata.rs++;
459 }
460 }
461 else {
462
463 /* Set wait bit */
464
465 while (1) {
466 ethr_sint32_t new, exp = act;
467 need_try_complete_runlock = 0;
468 transfer_read_lock = 0;
469
470 if (type == ETHR_RWMTX_W_WAIT_FLG__) {
471 if (is_freq_read && act == ETHR_RWMTX_R_FLG__)
472 need_try_complete_runlock = 1;
473 if (act != 0)
474 new = act | ETHR_RWMTX_W_WAIT_FLG__;
475 else
476 new = ETHR_RWMTX_W_FLG__; /* Try to get it */
477 }
478 else {
479 ETHR_ASSERT(is_rwmtx);
480
481 if (!is_freq_read) {
482 if (act & (ETHR_RWMTX_W_FLG__| ETHR_RWMTX_W_WAIT_FLG__))
483 new = act | ETHR_RWMTX_R_WAIT_FLG__;
484 else
485 new = act + 1; /* Try to get it */
486 }
487 else {
488 new = act | ETHR_RWMTX_R_WAIT_FLG__;
489 if ((act & (ETHR_RWMTX_W_FLG__
490 | ETHR_RWMTX_W_WAIT_FLG__)) == 0) {
491 /* Transfer read lock to this thread. */
492 transfer_read_lock = 1;
493 }
494 }
495 }
496
497 act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, new, exp);
498 if (exp == act) {
499 if (new & type) {
500 act = new;
501 break;
502 }
503 else {
504 /* Got it */
505 goto done;
506 }
507 }
508 }
509
510 /* Enqueue */
511
512 if (type == ETHR_RWMTX_R_WAIT_FLG__) {
513 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
514 ETHR_ASSERT(is_rwmtx);
515 ETHR_ASSERT(!rwmtx->rq_end);
516 rwmtx->rq_end = tse;
517 if (is_freq_read)
518 rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
519 else
520 rwmtx->tdata.rs++;
521 }
522 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
523 else {
524 mtxb->ws++;
525 }
526 #endif
527
528 enqueue(&mtxb->q, tse, tse);
529 }
530
531 #ifdef ETHR_MTX_HARD_DEBUG_Q
532 hard_debug_chk_q__(mtxb, is_rwmtx);
533 #endif
534
535 /* Wait */
536 locked = 0;
537
538 ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock));
539
540 if (transfer_read_lock) {
541 ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
542 != ETHR_RWMUTEX_TYPE_NORMAL);
543 /*
544 * We are the only one in the queue and we are not write
545 * locked; rwmutex_transfer_read_lock() will:
546 * - transfer a read lock to us (since we're first in q)
547 * - unlock the Q-lock
548 */
549 rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1);
550 }
551 else {
552 ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
553
554 if (need_try_complete_runlock) {
555 ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
556 != ETHR_RWMUTEX_TYPE_NORMAL);
557 /*
558 * We were the only one in queue when we enqueued, and it
559 * was seemingly read locked. We need to try to complete a
560 * runlock otherwise we might be hanging forever. If the
561 * runlock could be completed we will be dequeued and
562 * woken by ourselves.
563 */
564 rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
565 act, tse, 0, 1, 0);
566 }
567 }
568
569 while (1) {
570 ethr_event_reset(&tse->event);
571
572 act = ethr_atomic32_read_acqb(&tse->uaflgs);
573 if (!act)
574 goto done; /* Got it */
575
576 ETHR_ASSERT(act == type);
577 ethr_event_swait(&tse->event, spincount);
578 /* swait result: 0 || EINTR */
579
580 act = ethr_atomic32_read_acqb(&tse->uaflgs);
581 if (!act)
582 goto done; /* Got it */
583 }
584
585 done:
586 if (locked)
587 ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
588 }
589
590 static void
wake_writer(struct ethr_mutex_base_ * mtxb,int is_rwmtx)591 wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
592 {
593 ethr_ts_event *tse;
594
595 tse = mtxb->q;
596 ETHR_ASSERT(tse);
597 dequeue(&mtxb->q, tse, tse);
598
599 ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
600 ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
601 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
602 mtxb->ws--;
603 #endif
604 #if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
605 hard_debug_chk_q__(mtxb, is_rwmtx);
606 #endif
607
608 ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
609
610 ethr_atomic32_set(&tse->uaflgs, 0);
611 ethr_event_set(&tse->event);
612 }
613
614 static ETHR_INLINE int
initial_spincount(struct ethr_mutex_base_ * mtxb)615 initial_spincount(struct ethr_mutex_base_ *mtxb)
616 {
617 return (mtxb->aux_scnt < ETHR_MTX_MAX_FLGS_SPIN
618 ? mtxb->aux_scnt
619 : ETHR_MTX_MAX_FLGS_SPIN);
620 }
621
622 static ETHR_INLINE int
update_spincount(struct ethr_mutex_base_ * mtxb,ethr_ts_event * tse,int * scnt_state,int * scnt)623 update_spincount(struct ethr_mutex_base_ *mtxb,
624 ethr_ts_event *tse,
625 int *scnt_state,
626 int *scnt)
627 {
628 int state = *scnt_state;
629 if (state <= 0) {
630 /* Here state is max spincount to do on event negated */
631 *scnt = -state;
632 }
633 else {
634 /* Here state is initial spincount made on flags */
635 *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
636 ? mtxb->main_scnt
637 : mtxb->aux_scnt);
638 if (*scnt <= state)
639 *scnt = 0;
640 else {
641 if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN)
642 *scnt_state = 0; /* No spin on event */
643 else {
644 /* Spin on event after... */
645 *scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN);
646 /* ... we have spun on flags */
647 *scnt = ETHR_MTX_MAX_FLGS_SPIN;
648 }
649 *scnt -= state;
650 return 0;
651 }
652 }
653 return 1;
654 }
655
656 int check_readers_array(ethr_rwmutex *rwmtx,
657 int start_rix,
658 int length);
659
660 static ETHR_INLINE void
write_lock_wait(struct ethr_mutex_base_ * mtxb,ethr_sint32_t initial,int is_rwmtx,int is_freq_read)661 write_lock_wait(struct ethr_mutex_base_ *mtxb,
662 ethr_sint32_t initial,
663 int is_rwmtx,
664 int is_freq_read)
665 {
666 ethr_sint32_t act = initial;
667 int scnt, start_scnt;
668 ethr_ts_event *tse = NULL;
669 int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
670 int res;
671
672 ETHR_ASSERT(!is_freq_read || is_rwmtx);
673
674 start_scnt = scnt = initial_spincount(mtxb);
675
676 /*
677 * Spin trying to write lock for a while. If unsuccessful,
678 * wait on event.
679 */
680
681 while (1) {
682 while (act != 0) {
683
684 if (is_freq_read && act == ETHR_RWMTX_R_FLG__) {
685 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
686 scnt--;
687 if (!tse)
688 tse = ethr_peek_ts_event();
689 res = rwmutex_try_complete_runlock(rwmtx, act,
690 tse, 0, 0,
691 1);
692 if (res != EBUSY)
693 goto done; /* Got it */
694 if (scnt <= 0)
695 goto chk_spin;
696 if (--until_yield == 0) {
697 until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
698 (void) ETHR_YIELD();
699 }
700 }
701
702 if (scnt <= 0) {
703 chk_spin:
704 scnt = 0;
705
706 if (!tse)
707 tse = ethr_peek_ts_event();
708 if (update_spincount(mtxb, tse, &start_scnt, &scnt)) {
709 ethr_ts_event *tmp_tse = ethr_use_ts_event(tse);
710 event_wait(mtxb, tmp_tse, scnt, ETHR_RWMTX_W_WAIT_FLG__,
711 is_rwmtx, is_freq_read);
712 ethr_leave_ts_event(tmp_tse);
713 goto done;
714 }
715 }
716 ETHR_SPIN_BODY;
717 if (--until_yield == 0) {
718 until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
719 (void) ETHR_YIELD();
720 }
721 act = ethr_atomic32_read(&mtxb->flgs);
722 scnt--;
723 }
724
725 act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs,
726 ETHR_RWMTX_W_FLG__,
727 0);
728 if (act == 0)
729 goto done; /* Got it */
730 }
731
732 done:
733 if (tse)
734 ethr_unpeek_ts_event(tse);
735 }
736
737 static int
mtxb_init(struct ethr_mutex_base_ * mtxb,int def_main_scnt,int main_scnt,int def_aux_scnt,int aux_scnt)738 mtxb_init(struct ethr_mutex_base_ *mtxb,
739 int def_main_scnt,
740 int main_scnt,
741 int def_aux_scnt,
742 int aux_scnt)
743 {
744 ETHR_MTX_HARD_DEBUG_LFS_INIT(mtxb);
745 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
746 mtxb->ws = 0;
747 #endif
748 ETHR_MTX_CHK_EXCL_INIT(mtxb);
749 if (no_spin) {
750 mtxb->main_scnt = 0;
751 mtxb->aux_scnt = 0;
752 }
753 else {
754 if (main_scnt > SHRT_MAX)
755 mtxb->main_scnt = SHRT_MAX;
756 else if (main_scnt < 0)
757 mtxb->main_scnt = def_main_scnt;
758 else
759 mtxb->main_scnt = (short) main_scnt;
760 if (aux_scnt > SHRT_MAX)
761 mtxb->aux_scnt = SHRT_MAX;
762 else if (aux_scnt < 0)
763 mtxb->aux_scnt = def_aux_scnt;
764 else
765 mtxb->aux_scnt = (short) aux_scnt;
766 if (mtxb->main_scnt < mtxb->aux_scnt)
767 mtxb->main_scnt = mtxb->aux_scnt;
768
769 }
770 mtxb->q = NULL;
771 ethr_atomic32_init(&mtxb->flgs, 0);
772 return ETHR_MTX_QLOCK_INIT(&mtxb->qlck);
773 }
774
775 static int
mtxb_destroy(struct ethr_mutex_base_ * mtxb)776 mtxb_destroy(struct ethr_mutex_base_ *mtxb)
777 {
778 ethr_sint32_t act;
779 ETHR_MTX_Q_LOCK(&mtxb->qlck);
780 act = ethr_atomic32_read(&mtxb->flgs);
781 ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
782 if (act != 0)
783 return EINVAL;
784 return ETHR_MTX_QLOCK_DESTROY(&mtxb->qlck);
785 }
786
787
788 #endif /* ETHR_USE_OWN_RWMTX_IMPL__ || ETHR_USE_OWN_MTX_IMPL__ */
789
790 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
791 * Mutex and condition variable implementation *
792 \* */
793
794 #ifdef ETHR_USE_OWN_MTX_IMPL__
795
796 /* -- Mutex ---------------------------------------------------------------- */
797
798 int
ethr_mutex_init_opt(ethr_mutex * mtx,ethr_mutex_opt * opt)799 ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
800 {
801 int res;
802 #if ETHR_XCHK
803 if (!mtx) {
804 ETHR_ASSERT(0);
805 return EINVAL;
806 }
807 mtx->initialized = ETHR_MUTEX_INITIALIZED;
808 #endif
809 ETHR_MTX_HARD_DEBUG_FENCE_INIT(mtx);
810 res = mtxb_init(&mtx->mtxb,
811 default_mtx_main_spincount,
812 opt ? opt->main_spincount : -1,
813 default_mtx_aux_spincount,
814 opt ? opt->aux_spincount : -1);
815 #if ETHR_XCHK
816 if (res != 0)
817 mtx->initialized = 0;
818 #endif
819 return res;
820 }
821
822 int
ethr_mutex_init(ethr_mutex * mtx)823 ethr_mutex_init(ethr_mutex *mtx)
824 {
825 return ethr_mutex_init_opt(mtx, NULL);
826 }
827
828 int
ethr_mutex_destroy(ethr_mutex * mtx)829 ethr_mutex_destroy(ethr_mutex *mtx)
830 {
831 #if ETHR_XCHK
832 if (ethr_not_inited__) {
833 ETHR_ASSERT(0);
834 return EACCES;
835 }
836 if (!mtx) {
837 ETHR_ASSERT(0);
838 return EINVAL;
839 }
840 mtx->initialized = 0;
841 #endif
842 return mtxb_destroy(&mtx->mtxb);
843 }
844
845 void
ethr_mutex_lock_wait__(ethr_mutex * mtx,ethr_sint32_t initial)846 ethr_mutex_lock_wait__(ethr_mutex *mtx, ethr_sint32_t initial)
847 {
848 write_lock_wait(&mtx->mtxb, initial, 0, 0);
849 }
850
851 void
ethr_mutex_unlock_wake__(ethr_mutex * mtx,ethr_sint32_t initial)852 ethr_mutex_unlock_wake__(ethr_mutex *mtx, ethr_sint32_t initial)
853 {
854 ethr_ts_event *tse;
855
856 ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
857 tse = mtx->mtxb.q;
858
859 ETHR_ASSERT(tse);
860 ETHR_ASSERT(ethr_atomic32_read(&mtx->mtxb.flgs)
861 == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
862 ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__);
863 ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
864
865 /*
866 * If we have multiple waiters, there is no need to modify
867 * mtxb->flgs; otherwise, we need to clear the write wait bit...
868 */
869 if (tse->next == mtx->mtxb.q)
870 ethr_atomic32_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
871
872 wake_writer(&mtx->mtxb, 0);
873 }
874
875 /* -- Condition variables -------------------------------------------------- */
876
877 static void
enqueue_mtx(ethr_mutex * mtx,ethr_ts_event * tse_start,ethr_ts_event * tse_end)878 enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
879 {
880 ethr_sint32_t act;
881
882 /*
883 * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx'
884 * is not currently locked by current thread, we almost certainly have a
885 * hard to debug race condition. There might however be some (strange)
886 * use for it. POSIX also allow a call to `pthread_cond_signal' or
887 * `pthread_cond_broadcast' even though the the associated mutex isn't
888 * locked by the caller. Therefore, we also allow this kind of strange
889 * usage, but optimize for the case where the mutex is locked by the
890 * calling thread.
891 */
892
893 ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
894
895 ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
896
897 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
898 {
899 int dbg_nws__ = 0;
900 ethr_ts_event *dbg_tse__;
901 for (dbg_tse__ = tse_start;
902 dbg_tse__ != tse_end;
903 dbg_tse__ = dbg_tse__->next)
904 dbg_nws__++;
905 mtx->mtxb.ws += dbg_nws__ + 1;
906 }
907 #endif
908
909 act = ethr_atomic32_read(&mtx->mtxb.flgs);
910 ETHR_ASSERT(act == 0
911 || act == ETHR_RWMTX_W_FLG__
912 || act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
913 if (act & ETHR_RWMTX_W_FLG__) {
914 /* The normal sane case */
915 if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) {
916 ETHR_ASSERT(!mtx->mtxb.q);
917 act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs,
918 (ETHR_RWMTX_W_FLG__
919 | ETHR_RWMTX_W_WAIT_FLG__),
920 ETHR_RWMTX_W_FLG__);
921 if (act != ETHR_RWMTX_W_FLG__) {
922 /*
923 * Sigh... this wasn't so sane after all since, the mutex was
924 * obviously not locked by the current thread....
925 */
926 ETHR_ASSERT(act == 0);
927 goto mtx_unlocked;
928 }
929 }
930
931 #ifdef ETHR_DEBUG
932 if (act & ETHR_RWMTX_W_WAIT_FLG__)
933 ETHR_ASSERT(mtx->mtxb.q);
934 else
935 ETHR_ASSERT(!mtx->mtxb.q);
936 #endif
937
938 enqueue(&mtx->mtxb.q, tse_start, tse_end);
939
940 ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
941 ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
942
943 }
944 else {
945 int multi;
946 mtx_unlocked:
947 /* Sigh... mutex isn't locked... */
948
949 multi = tse_start != tse_end;
950
951 while (1) {
952 ethr_sint32_t new, exp = act;
953
954 if (multi || (act & ETHR_RWMTX_W_FLG__))
955 new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__;
956 else
957 new = ETHR_RWMTX_W_FLG__;
958
959 act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, new, exp);
960 if (exp == act) {
961 ETHR_ASSERT(!mtx->mtxb.q);
962 if (act & ETHR_RWMTX_W_FLG__) {
963 enqueue(&mtx->mtxb.q, tse_start, tse_end);
964
965 ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
966 ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
967
968 }
969 else {
970 ETHR_ASSERT(!mtx->mtxb.q);
971 /*
972 * Acquired the mutex on behalf of the
973 * first thread in the queue; wake
974 * it and enqueue the rest...
975 */
976 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
977 mtx->mtxb.ws--;
978 #endif
979 if (multi) {
980 enqueue(&mtx->mtxb.q, tse_start->next, tse_end);
981 ETHR_ASSERT(mtx->mtxb.q);
982 }
983
984 ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
985 ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
986
987 ethr_atomic32_set(&tse_start->uaflgs, 0);
988 ethr_event_set(&tse_start->event);
989 }
990 break;
991 }
992 }
993 }
994 }
995
996 int
ethr_cond_init_opt(ethr_cond * cnd,ethr_cond_opt * opt)997 ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
998 {
999 #if ETHR_XCHK
1000 if (!cnd) {
1001 ETHR_ASSERT(0);
1002 return EINVAL;
1003 }
1004 cnd->initialized = ETHR_COND_INITIALIZED;
1005 #endif
1006 ETHR_MTX_HARD_DEBUG_FENCE_INIT(cnd);
1007 cnd->q = NULL;
1008 if (no_spin) {
1009 cnd->main_scnt = 0;
1010 cnd->aux_scnt = 0;
1011 }
1012 else {
1013 if (!opt || opt->main_spincount < 0)
1014 cnd->main_scnt = default_cnd_main_spincount;
1015 else if (opt->main_spincount > SHRT_MAX)
1016 cnd->main_scnt = SHRT_MAX;
1017 else
1018 cnd->main_scnt = (short) opt->main_spincount;
1019 if (!opt || opt->aux_spincount < 0)
1020 cnd->aux_scnt = default_cnd_aux_spincount;
1021 else if (opt->aux_spincount > SHRT_MAX)
1022 cnd->aux_scnt = SHRT_MAX;
1023 else
1024 cnd->aux_scnt = (short) opt->aux_spincount;
1025 if (cnd->main_scnt < cnd->aux_scnt)
1026 cnd->main_scnt = cnd->aux_scnt;
1027 }
1028 ETHR_MTX_QLOCK_INIT(&cnd->qlck);
1029 return 0;
1030 }
1031
1032 int
ethr_cond_init(ethr_cond * cnd)1033 ethr_cond_init(ethr_cond *cnd)
1034 {
1035 return ethr_cond_init_opt(cnd, NULL);
1036 }
1037
1038 int
ethr_cond_destroy(ethr_cond * cnd)1039 ethr_cond_destroy(ethr_cond *cnd)
1040 {
1041 #if ETHR_XCHK
1042 if (ethr_not_inited__) {
1043 ETHR_ASSERT(0);
1044 return EACCES;
1045 }
1046 if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
1047 ETHR_ASSERT(0);
1048 return EINVAL;
1049 }
1050 cnd->initialized = 0;
1051 #endif
1052 return ETHR_MTX_QLOCK_DESTROY(&cnd->qlck);
1053 }
1054
1055 void
ethr_cond_signal(ethr_cond * cnd)1056 ethr_cond_signal(ethr_cond *cnd)
1057 {
1058 ethr_ts_event *tse;
1059
1060 ETHR_ASSERT(!ethr_not_inited__);
1061 ETHR_ASSERT(cnd != NULL);
1062 ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
1063 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1064
1065 ETHR_MTX_Q_LOCK(&cnd->qlck);
1066
1067 tse = cnd->q;
1068
1069 if (!tse) {
1070 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1071 ETHR_MTX_Q_UNLOCK(&cnd->qlck);
1072 }
1073 else {
1074 ethr_mutex *mtx = (ethr_mutex *) tse->udata;
1075
1076 ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
1077 ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
1078 ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
1079
1080 ethr_atomic32_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
1081
1082 dequeue(&cnd->q, tse, tse);
1083
1084 ETHR_MTX_Q_UNLOCK(&cnd->qlck);
1085
1086 tse->next = tse->prev = NULL;
1087
1088 enqueue_mtx(mtx, tse, tse);
1089
1090 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1091 ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
1092 }
1093 }
1094
1095 void
ethr_cond_broadcast(ethr_cond * cnd)1096 ethr_cond_broadcast(ethr_cond *cnd)
1097 {
1098 int got_all;
1099 ethr_ts_event *tse;
1100 ETHR_ASSERT(!ethr_not_inited__);
1101 ETHR_ASSERT(cnd);
1102 ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
1103
1104 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1105 do {
1106 got_all = 1;
1107
1108 ETHR_MTX_Q_LOCK(&cnd->qlck);
1109
1110 tse = cnd->q;
1111
1112 if (!tse) {
1113 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1114 ETHR_MTX_Q_UNLOCK(&cnd->qlck);
1115 }
1116 else {
1117 ethr_mutex *mtx = (ethr_mutex *) tse->udata;
1118 ethr_ts_event *tse_tmp, *tse_end;
1119
1120 ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
1121 tse_end = cnd->q->prev;
1122
1123 tse_tmp = tse;
1124
1125 do {
1126
1127 if (mtx == (ethr_mutex *) tse_tmp->udata) {
1128 /* The normal case */
1129
1130 ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
1131 ETHR_ASSERT(ethr_atomic32_read(&tse_tmp->uaflgs)
1132 == ETHR_CND_WAIT_FLG__);
1133
1134 ethr_atomic32_set(&tse_tmp->uaflgs,
1135 ETHR_RWMTX_W_WAIT_FLG__);
1136 }
1137 else {
1138 /* Should be very unusual */
1139 ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
1140 tse_end = tse_tmp->prev;
1141 got_all = 0;
1142 break;
1143 }
1144
1145 tse_tmp = tse_tmp->next;
1146
1147 } while (tse_tmp != cnd->q);
1148
1149 dequeue(&cnd->q, tse, tse_end);
1150
1151 ETHR_MTX_Q_UNLOCK(&cnd->qlck);
1152
1153 enqueue_mtx(mtx, tse, tse_end);
1154 }
1155
1156 } while (!got_all);
1157
1158 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1159 }
1160
1161 int
ethr_cond_wait(ethr_cond * cnd,ethr_mutex * mtx)1162 ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
1163 {
1164 int woken;
1165 int scnt;
1166 void *udata = NULL;
1167 ethr_ts_event *tse;
1168
1169 ETHR_ASSERT(!ethr_not_inited__);
1170 ETHR_ASSERT(cnd);
1171 ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
1172 ETHR_ASSERT(mtx);
1173 ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
1174
1175 tse = ethr_get_ts_event();
1176
1177 scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
1178 ? cnd->main_scnt
1179 : cnd->aux_scnt);
1180
1181 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1182 ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
1183
1184 udata = tse->udata; /* Got to restore udata before returning */
1185 tse->udata = (void *) mtx;
1186
1187 tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */
1188 ethr_atomic32_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
1189
1190 ETHR_MTX_Q_LOCK(&cnd->qlck);
1191
1192 enqueue(&cnd->q, tse, tse);
1193
1194 ETHR_MTX_Q_UNLOCK(&cnd->qlck);
1195
1196 ethr_mutex_unlock(mtx);
1197
1198 /* Wait */
1199 woken = 0;
1200 while (1) {
1201 ethr_sint32_t act;
1202
1203 ethr_event_reset(&tse->event);
1204
1205 act = ethr_atomic32_read_acqb(&tse->uaflgs);
1206 if (!act)
1207 break; /* Mtx locked */
1208
1209 /* First time, got EINTR, or spurious wakeup... */
1210
1211 ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
1212 || act == ETHR_RWMTX_W_WAIT_FLG__);
1213
1214 if (woken) {
1215 /*
1216 * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already been enqueued
1217 * on the mutex; continue wait until locked...
1218 */
1219 if (act == ETHR_CND_WAIT_FLG__) {
1220 ETHR_MTX_Q_LOCK(&cnd->qlck);
1221 act = ethr_atomic32_read(&tse->uaflgs);
1222 ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
1223 || act == ETHR_RWMTX_W_WAIT_FLG__);
1224 /*
1225 * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already
1226 * enqueued on the mutex; continue wait until locked...
1227 */
1228 if (act == ETHR_CND_WAIT_FLG__)
1229 dequeue(&cnd->q, tse, tse);
1230
1231 ETHR_MTX_Q_UNLOCK(&cnd->qlck);
1232
1233 if (act == ETHR_CND_WAIT_FLG__) {
1234 tse->udata = udata;
1235 ethr_leave_ts_event(tse);
1236 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1237 ethr_mutex_lock(mtx);
1238 return EINTR;
1239 }
1240 }
1241 ETHR_ASSERT(act == ETHR_RWMTX_W_WAIT_FLG__);
1242 }
1243 ETHR_ASSERT(tse->iflgs & ETHR_TS_EV_BUSY);
1244 ethr_event_swait(&tse->event, scnt);
1245 /* swait result: 0 || EINTR */
1246 woken = 1;
1247 }
1248
1249 ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
1250 ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
1251 ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
1252 ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
1253 tse->udata = udata;
1254 ethr_leave_ts_event(tse);
1255 return 0;
1256 }
1257
1258 #elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
1259 /* -- pthread mutex and condition variables -------------------------------- */
1260
1261 int
ethr_mutex_init(ethr_mutex * mtx)1262 ethr_mutex_init(ethr_mutex *mtx)
1263 {
1264 #if ETHR_XCHK
1265 if (!mtx) {
1266 ETHR_ASSERT(0);
1267 return EINVAL;
1268 }
1269 mtx->initialized = ETHR_MUTEX_INITIALIZED;
1270 #endif
1271 return pthread_mutex_init(&mtx->pt_mtx, NULL);
1272 }
1273
1274 int
ethr_mutex_init_opt(ethr_mutex * mtx,ethr_mutex_opt * opt)1275 ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
1276 {
1277 return ethr_mutex_init(mtx);
1278 }
1279
1280 int
ethr_mutex_destroy(ethr_mutex * mtx)1281 ethr_mutex_destroy(ethr_mutex *mtx)
1282 {
1283 #if ETHR_XCHK
1284 if (ethr_not_inited__) {
1285 ETHR_ASSERT(0);
1286 return EACCES;
1287 }
1288 if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
1289 ETHR_ASSERT(0);
1290 return EINVAL;
1291 }
1292 #endif
1293 #if ETHR_XCHK
1294 mtx->initialized = 0;
1295 #endif
1296 return pthread_mutex_destroy(&mtx->pt_mtx);
1297 }
1298
1299 int
ethr_cond_init(ethr_cond * cnd)1300 ethr_cond_init(ethr_cond *cnd)
1301 {
1302 #if ETHR_XCHK
1303 if (!cnd) {
1304 ETHR_ASSERT(0);
1305 return EINVAL;
1306 }
1307 cnd->initialized = ETHR_COND_INITIALIZED;
1308 #endif
1309 return pthread_cond_init(&cnd->pt_cnd, NULL);
1310 }
1311
1312 int
ethr_cond_init_opt(ethr_cond * cnd,ethr_cond_opt * opt)1313 ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
1314 {
1315 return ethr_cond_init(cnd);
1316 }
1317
1318 int
ethr_cond_destroy(ethr_cond * cnd)1319 ethr_cond_destroy(ethr_cond *cnd)
1320 {
1321 #if ETHR_XCHK
1322 if (ethr_not_inited__) {
1323 ETHR_ASSERT(0);
1324 return EACCES;
1325 }
1326 if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
1327 ETHR_ASSERT(0);
1328 return EINVAL;
1329 }
1330 cnd->initialized = 0;
1331 #endif
1332 return pthread_cond_destroy(&cnd->pt_cnd);
1333 }
1334
1335 void
ethr_cond_signal(ethr_cond * cnd)1336 ethr_cond_signal(ethr_cond *cnd)
1337 {
1338 int res;
1339
1340 ETHR_ASSERT(!ethr_not_inited__);
1341 ETHR_ASSERT(cnd);
1342 ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
1343
1344 res = pthread_cond_signal(&cnd->pt_cnd);
1345 if (res != 0)
1346 ETHR_FATAL_ERROR__(res);
1347 }
1348
1349 void
ethr_cond_broadcast(ethr_cond * cnd)1350 ethr_cond_broadcast(ethr_cond *cnd)
1351 {
1352 int res;
1353
1354 ETHR_ASSERT(!ethr_not_inited__);
1355 ETHR_ASSERT(cnd);
1356 ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
1357
1358 res = pthread_cond_broadcast(&cnd->pt_cnd);
1359 if (res != 0)
1360 ETHR_FATAL_ERROR__(res);
1361 }
1362
1363 int
ethr_cond_wait(ethr_cond * cnd,ethr_mutex * mtx)1364 ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
1365 {
1366 int res;
1367
1368 ETHR_ASSERT(!ethr_not_inited__);
1369 ETHR_ASSERT(cnd);
1370 ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
1371 ETHR_ASSERT(mtx);
1372 ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
1373
1374 res = pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
1375 if (res != 0 && res != EINTR)
1376 ETHR_FATAL_ERROR__(res);
1377 return res;
1378 }
1379
1380 #elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
1381
1382 /*
1383 * As of Vista/Server, 2008 Windows has condition variables that can be
1384 * used with critical sections. However, we need to be able to run on
1385 * older Windows versions too, so we need to implement condition variables
1386 * ourselves.
1387 */
1388
1389 #ifdef ETHR_DBG_WIN_MTX_WITH_PTHREADS
1390 /*
1391 * For debugging of this implementation on POSIX platforms...
1392 */
1393
1394 #define ethr_win_get_errno__() EINVAL
1395 #if defined(__GNUC__)
1396 #define __forceinline __inline__
1397 #else
1398 #define __forceinline
1399 #endif
1400
1401 static int
InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION * cs,int sc)1402 InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION *cs, int sc)
1403 {
1404 return 0 == pthread_mutex_init((pthread_mutex_t *) cs, NULL);
1405 }
1406
DeleteCriticalSection(CRITICAL_SECTION * cs)1407 static void DeleteCriticalSection(CRITICAL_SECTION *cs)
1408 {
1409 int res = pthread_mutex_destroy((pthread_mutex_t *) cs);
1410 if (res != 0)
1411 ETHR_FATAL_ERROR__(res);
1412 }
1413
TryEnterCriticalSection(CRITICAL_SECTION * cs)1414 int TryEnterCriticalSection(CRITICAL_SECTION *cs)
1415 {
1416 int res;
1417 res = pthread_mutex_trylock((pthread_mutex_t *) cs);
1418 if (res != 0 && res != EBUSY)
1419 ETHR_FATAL_ERROR__(res);
1420 return res == 0;
1421 }
1422
EnterCriticalSection(CRITICAL_SECTION * cs)1423 void EnterCriticalSection(CRITICAL_SECTION *cs)
1424 {
1425 int res = pthread_mutex_lock((pthread_mutex_t *) cs);
1426 if (res != 0)
1427 ETHR_FATAL_ERROR__(res);
1428 }
1429
LeaveCriticalSection(CRITICAL_SECTION * cs)1430 void LeaveCriticalSection(CRITICAL_SECTION *cs)
1431 {
1432 int res = pthread_mutex_unlock((pthread_mutex_t *) cs);
1433 if (res != 0)
1434 ETHR_FATAL_ERROR__(res);
1435 }
1436
1437 #endif
1438
1439 #define ETHR_CND_WAIT__ ((ethr_sint32_t) 0x11dead11)
1440 #define ETHR_CND_WAKEUP__ ((ethr_sint32_t) 0x11beef11)
1441
1442 static ETHR_FORCE_INLINE void
cond_wakeup(ethr_ts_event * tse)1443 cond_wakeup(ethr_ts_event *tse)
1444 {
1445 ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT__);
1446
1447 ethr_atomic32_set_relb(&tse->uaflgs, ETHR_CND_WAKEUP__);
1448 ethr_event_set(&tse->event);
1449 }
1450
1451 void
ethr_mutex_cond_wakeup__(ethr_mutex * mtx)1452 ethr_mutex_cond_wakeup__(ethr_mutex *mtx)
1453 {
1454 /*
1455 * Called by ethr_mutex_unlock() when we have
1456 * cond signal/broadcast wakeups waiting to
1457 * be completed.
1458 */
1459 ethr_ts_event *tse;
1460
1461 if (!mtx->posix_compliant) {
1462 tse = mtx->wakeups;
1463 dequeue(&mtx->wakeups, tse, tse);
1464 }
1465 else {
1466 ethr_spin_lock(&mtx->lock);
1467 tse = mtx->wakeups;
1468 if (tse)
1469 dequeue(&mtx->wakeups, tse, tse);
1470 if (!mtx->wakeups)
1471 ethr_atomic32_set_relb(&mtx->have_wakeups, 0);
1472 ethr_spin_unlock(&mtx->lock);
1473 }
1474
1475 LeaveCriticalSection(&mtx->cs);
1476
1477 ETHR_ASSERT(tse || mtx->posix_compliant);
1478
1479 /*
1480 * We delay actual condition variable wakeup until
1481 * this point when we have left the critical section.
1482 * This in order to avoid that the other thread is
1483 * woken and then right away have to go to sleep
1484 * waiting for the critical section that we are in.
1485 *
1486 * We also only wake one thread at a time even if
1487 * there are multiple threads waiting to be woken.
1488 * Otherwise all but one will be woken and then right
1489 * away have to go to sleep on the critical section.
1490 * Since each wakeup is guaranteed to generate at
1491 * least one lock/unlock sequence on this mutex, all
1492 * threads will eventually be woken.
1493 */
1494
1495 if (tse)
1496 cond_wakeup(tse);
1497 }
1498
1499 int
ethr_mutex_init_opt(ethr_mutex * mtx,ethr_mutex_opt * opt)1500 ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
1501 {
1502 int spincount;
1503 #if ETHR_XCHK
1504 if (!mtx) {
1505 ETHR_ASSERT(0);
1506 return EINVAL;
1507 }
1508 mtx->initialized = ETHR_MUTEX_INITIALIZED;
1509 #endif
1510
1511 spincount = opt ? opt->aux_spincount : 0;
1512 if (spincount < 0)
1513 spincount = 0;
1514
1515 if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, spincount)) {
1516 #if ETHR_XCHK
1517 mtx->initialized = 0;
1518 #endif
1519 return ethr_win_get_errno__();
1520 }
1521
1522 mtx->posix_compliant = opt ? opt->posix_compliant : 0;
1523 mtx->wakeups = NULL;
1524 if (mtx->posix_compliant) {
1525 ethr_atomic32_init(&mtx->locked, 0);
1526 ethr_atomic32_init(&mtx->have_wakeups, 0);
1527 ethr_spinlock_init(&mtx->lock);
1528 }
1529 return 0;
1530 }
1531
1532 int
ethr_mutex_init(ethr_mutex * mtx)1533 ethr_mutex_init(ethr_mutex *mtx)
1534 {
1535 return ethr_mutex_init_opt(mtx, NULL);
1536 }
1537
1538 int
ethr_mutex_destroy(ethr_mutex * mtx)1539 ethr_mutex_destroy(ethr_mutex *mtx)
1540 {
1541 DeleteCriticalSection(&mtx->cs);
1542 if (mtx->posix_compliant)
1543 return ethr_spinlock_destroy(&mtx->lock);
1544 else
1545 return 0;
1546 }
1547
1548 int
ethr_cond_wait(ethr_cond * cnd,ethr_mutex * mtx)1549 ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
1550 {
1551 void *udata;
1552 ethr_ts_event *tse = ethr_get_ts_event();
1553 int spincount;
1554
1555 udata = tse->udata;
1556 tse->udata = (void *) mtx;
1557 ethr_atomic32_set_relb(&tse->uaflgs, ETHR_CND_WAIT__);
1558
1559 EnterCriticalSection(&cnd->cs);
1560 enqueue(&cnd->waiters, tse, tse);
1561 LeaveCriticalSection(&cnd->cs);
1562
1563 ethr_mutex_unlock(mtx);
1564
1565 spincount = cnd->spincount;
1566
1567 while (ethr_atomic32_read_acqb(&tse->uaflgs) != ETHR_CND_WAKEUP__) {
1568 ethr_event_reset(&tse->event);
1569 if (ethr_atomic32_read_acqb(&tse->uaflgs) == ETHR_CND_WAKEUP__)
1570 break;
1571 ethr_event_swait(&tse->event, spincount);
1572 spincount = 0;
1573 }
1574
1575 tse->udata = udata;
1576 ethr_leave_ts_event(tse);
1577
1578 ethr_mutex_lock(mtx);
1579
1580 return 0;
1581 }
1582
1583 static ETHR_FORCE_INLINE void
posix_compliant_mtx_enqueue(ethr_mutex * mtx,ethr_ts_event * tse_start,ethr_ts_event * tse_end)1584 posix_compliant_mtx_enqueue(ethr_mutex *mtx,
1585 ethr_ts_event *tse_start,
1586 ethr_ts_event *tse_end)
1587 {
1588 ethr_ts_event *tse_wakeup = NULL; /* Avoid erroneous compiler warning... */
1589 /*
1590 * The associated mutex might not be locked, so we need to
1591 * check if it is. If locked, enqueue for wakeup at unlock;
1592 * otherwise, wakeup the first one now and enqueue the rest.
1593 */
1594 if (tse_start == tse_end && !ethr_atomic32_read(&mtx->locked)) {
1595 tse_wakeup = tse_start;
1596 wakeup:
1597 cond_wakeup(tse_wakeup);
1598 }
1599 else {
1600 int need_wakeup;
1601 ethr_spin_lock(&mtx->lock);
1602 if (!mtx->wakeups)
1603 ethr_atomic32_set_mb(&mtx->have_wakeups, 1);
1604 need_wakeup = !ethr_atomic32_read(&mtx->locked);
1605 if (need_wakeup) {
1606 if (tse_start == tse_end) {
1607 if (!mtx->wakeups)
1608 ethr_atomic32_set_relb(&mtx->have_wakeups, 0);
1609 ethr_spin_unlock(&mtx->lock);
1610 tse_wakeup = tse_start;
1611 goto wakeup;
1612 }
1613 tse_wakeup = tse_start;
1614 tse_start = tse_start->next;
1615 }
1616 enqueue(&mtx->wakeups, tse_start, tse_end);
1617 ethr_spin_unlock(&mtx->lock);
1618 if (need_wakeup)
1619 goto wakeup;
1620 }
1621 }
1622
1623 static ETHR_FORCE_INLINE void
enqueue_cond_wakeups(ethr_ts_event * queue,int posix_compliant)1624 enqueue_cond_wakeups(ethr_ts_event *queue, int posix_compliant)
1625 {
1626 if (queue) {
1627 int more;
1628 ethr_ts_event *q = queue;
1629
1630 /*
1631 * Waiters may be using different mutexes...
1632 */
1633
1634 do {
1635 ethr_mutex *mtx;
1636 ethr_ts_event *tse, *tse_start, *tse_end;
1637
1638 more = 0;
1639 tse_start = q;
1640 mtx = (ethr_mutex *) tse_start->udata;
1641
1642 ETHR_ASSERT(posix_compliant
1643 ? mtx->posix_compliant
1644 : !mtx->posix_compliant);
1645
1646 ETHR_ASSERT(ethr_atomic32_read(&tse_start->uaflgs)
1647 == ETHR_CND_WAIT__);
1648 ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
1649
1650 tse_end = tse_start->prev;
1651
1652 for (tse = tse_start->next; tse != tse_start; tse = tse->next) {
1653
1654 ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs)
1655 == ETHR_CND_WAIT__);
1656
1657 if (mtx != (ethr_mutex *) tse->udata) {
1658 tse_end = tse->prev;
1659 dequeue(&q, tse_start, tse_end);
1660 more = 1;
1661 break;
1662 }
1663 }
1664
1665 if (posix_compliant)
1666 posix_compliant_mtx_enqueue(mtx, tse_start, tse_end);
1667 else
1668 enqueue(&mtx->wakeups, tse_start, tse_end);
1669
1670 } while (more);
1671 }
1672 }
1673
1674 void
ethr_cond_broadcast(ethr_cond * cnd)1675 ethr_cond_broadcast(ethr_cond *cnd)
1676 {
1677 ethr_ts_event *waiters;
1678
1679 EnterCriticalSection(&cnd->cs);
1680 waiters = cnd->waiters;
1681 cnd->waiters = NULL;
1682 LeaveCriticalSection(&cnd->cs);
1683
1684 if (cnd->posix_compliant)
1685 enqueue_cond_wakeups(waiters, 1);
1686 else
1687 enqueue_cond_wakeups(waiters, 0);
1688 }
1689
1690 void
ethr_cond_signal(ethr_cond * cnd)1691 ethr_cond_signal(ethr_cond *cnd)
1692 {
1693 ethr_mutex *mtx;
1694 ethr_ts_event *tse;
1695
1696 EnterCriticalSection(&cnd->cs);
1697 tse = cnd->waiters;
1698 if (tse)
1699 dequeue(&cnd->waiters, tse, tse);
1700 LeaveCriticalSection(&cnd->cs);
1701
1702 if (tse) {
1703 mtx = (ethr_mutex *) tse->udata;
1704
1705 ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT__);
1706 ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
1707 ETHR_ASSERT(cnd->posix_compliant
1708 ? mtx->posix_compliant
1709 : !mtx->posix_compliant);
1710
1711 if (cnd->posix_compliant)
1712 posix_compliant_mtx_enqueue(mtx, tse, tse);
1713 else
1714 enqueue(&mtx->wakeups, tse, tse);
1715 }
1716 }
1717
1718 int
ethr_cond_init_opt(ethr_cond * cnd,ethr_cond_opt * opt)1719 ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
1720 {
1721 int spincount;
1722
1723 #if ETHR_XCHK
1724 if (!cnd) {
1725 ETHR_ASSERT(0);
1726 return EINVAL;
1727 }
1728 cnd->initialized = ETHR_COND_INITIALIZED;
1729 #endif
1730
1731 spincount = opt ? opt->aux_spincount : 0;
1732 if (spincount < 0)
1733 spincount = 0;
1734
1735 if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, spincount)) {
1736 #if ETHR_XCHK
1737 cnd->initialized = 0;
1738 #endif
1739 return ethr_win_get_errno__();
1740 }
1741
1742 cnd->posix_compliant = opt ? opt->posix_compliant : 0;
1743 cnd->waiters = NULL;
1744 cnd->spincount = spincount;
1745 return 0;
1746 }
1747
1748 int
ethr_cond_init(ethr_cond * cnd)1749 ethr_cond_init(ethr_cond *cnd)
1750 {
1751 return ethr_cond_init_opt(cnd, NULL);
1752 }
1753
1754 int
ethr_cond_destroy(ethr_cond * cnd)1755 ethr_cond_destroy(ethr_cond *cnd)
1756 {
1757 DeleteCriticalSection(&cnd->cs);
1758 return 0;
1759 }
1760
1761 #else
1762 #error "No mutex implementation found"
1763 #endif
1764
1765 /* -- Exported symbols of inline functions --------------------------------- */
1766
1767 int
ethr_mutex_trylock(ethr_mutex * mtx)1768 ethr_mutex_trylock(ethr_mutex *mtx)
1769 {
1770 ETHR_ASSERT(!ethr_not_inited__);
1771 ETHR_ASSERT(mtx);
1772 ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
1773
1774 return ethr_mutex_trylock__(mtx);
1775 }
1776
1777 void
ethr_mutex_lock(ethr_mutex * mtx)1778 ethr_mutex_lock(ethr_mutex *mtx)
1779 {
1780 ETHR_ASSERT(!ethr_not_inited__);
1781 ETHR_ASSERT(mtx);
1782 ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
1783
1784 ethr_mutex_lock__(mtx);
1785 }
1786
1787 void
ethr_mutex_unlock(ethr_mutex * mtx)1788 ethr_mutex_unlock(ethr_mutex *mtx)
1789 {
1790 ETHR_ASSERT(!ethr_not_inited__);
1791 ETHR_ASSERT(mtx);
1792 ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
1793
1794 ethr_mutex_unlock__(mtx);
1795 }
1796
1797
1798 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
1799 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
1800 * Read/Write Mutex *
1801 \* */
1802
1803 static void
wake_readers(ethr_rwmutex * rwmtx,int rs)1804 wake_readers(ethr_rwmutex *rwmtx, int rs)
1805 {
1806 ethr_ts_event *tse;
1807 #ifdef ETHR_DEBUG
1808 int drs = 0;
1809 #endif
1810
1811 tse = rwmtx->mtxb.q;
1812 ETHR_ASSERT(tse);
1813 ETHR_ASSERT(rwmtx->rq_end);
1814 dequeue(&rwmtx->mtxb.q, tse, rwmtx->rq_end);
1815 rwmtx->rq_end->next = NULL;
1816 rwmtx->rq_end = NULL;
1817
1818 ETHR_ASSERT(!rwmtx->mtxb.q
1819 || (ethr_atomic32_read(&rwmtx->mtxb.q->uaflgs)
1820 == ETHR_RWMTX_W_WAIT_FLG__));
1821
1822 ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
1823 ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
1824
1825 while (tse) {
1826 ethr_ts_event *tse_next;
1827
1828 #ifdef ETHR_DEBUG
1829 ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
1830 ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs)
1831 == ETHR_RWMTX_R_WAIT_FLG__);
1832 drs++;
1833 #endif
1834
1835 tse_next = tse->next; /* we aren't allowed to read tse->next
1836 after we have reset uaflgs */
1837
1838 ethr_atomic32_set(&tse->uaflgs, 0);
1839 ethr_event_set(&tse->event);
1840 tse = tse_next;
1841 }
1842
1843 ETHR_ASSERT(rs == drs);
1844 }
1845
1846 static ETHR_INLINE int
is_w_waiter(ethr_ts_event * tse)1847 is_w_waiter(ethr_ts_event *tse)
1848 {
1849 ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__
1850 || tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
1851 return tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__;
1852 }
1853
1854 static ETHR_INLINE int
multiple_w_waiters(ethr_rwmutex * rwmtx)1855 multiple_w_waiters(ethr_rwmutex *rwmtx)
1856 {
1857 ETHR_ASSERT(rwmtx->mtxb.q);
1858 ETHR_ASSERT(rwmtx->mtxb.q->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
1859
1860 if (!rwmtx->rq_end)
1861 return rwmtx->mtxb.q->next != rwmtx->mtxb.q;
1862 else {
1863 ETHR_ASSERT(rwmtx->mtxb.q->next != rwmtx->mtxb.q);
1864 if (rwmtx->mtxb.q->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__)
1865 return 1;
1866 ETHR_ASSERT(rwmtx->rq_end->next == rwmtx->mtxb.q
1867 || rwmtx->rq_end->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
1868 return rwmtx->rq_end->next != rwmtx->mtxb.q;
1869 }
1870 }
1871
check_readers_array(ethr_rwmutex * rwmtx,int start_rix,int length)1872 int check_readers_array(ethr_rwmutex *rwmtx,
1873 int start_rix,
1874 int length)
1875 {
1876 int ix = start_rix;
1877
1878 ETHR_MEMORY_BARRIER;
1879
1880 do {
1881 ethr_sint32_t act = rwmutex_freqread_rdrs_read(rwmtx, ix);
1882 if (act != 0)
1883 return EBUSY;
1884 ix++;
1885 if (ix == length)
1886 ix = 0;
1887 } while (ix != start_rix);
1888
1889 return 0;
1890 }
1891
1892 static void
rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex * rwmtx,ethr_ts_event * tse,ethr_sint32_t initial)1893 rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx,
1894 ethr_ts_event *tse,
1895 ethr_sint32_t initial)
1896 {
1897 ethr_sint32_t act = initial;
1898
1899 if ((act & (ETHR_RWMTX_W_FLG__|
1900 ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) {
1901 if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
1902 if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
1903 /*
1904 * We *need* to try to complete the runlock.
1905 * A writer that just enqueued (not seen by us
1906 * in flag field) may depend on someone else
1907 * completing the runlock. We just took over
1908 * that responsibilty since we modified reader
1909 * groups.
1910 */
1911 rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
1912 }
1913 }
1914 else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__)
1915 rwmutex_transfer_read_lock(rwmtx, act, 0);
1916 else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__)
1917 rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
1918 else {
1919 /*
1920 * Don't know if we got readers or writers
1921 * first in queue; need to peek
1922 */
1923 ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
1924 if (!rwmtx->mtxb.q)
1925 ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
1926 else if (is_w_waiter(rwmtx->mtxb.q)) {
1927 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
1928 ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
1929 if ((act & ETHR_RWMTX_W_FLG__) == 0)
1930 rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
1931 }
1932 else {
1933 /*
1934 * rwmutex_transfer_read_lock() will
1935 * unlock Q lock.
1936 */
1937 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
1938 if (act & ETHR_RWMTX_W_FLG__)
1939 ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
1940 else
1941 rwmutex_transfer_read_lock(rwmtx, act, 1);
1942 }
1943 }
1944 }
1945 }
1946
1947 static void
rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex * rwmtx,ethr_ts_event * tse)1948 rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
1949 ethr_ts_event *tse)
1950 {
1951 ethr_sint32_t act;
1952 /*
1953 * Restore failed increment
1954 */
1955 act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse);
1956
1957 ETHR_MEMORY_BARRIER;
1958
1959 if (act == 0) {
1960 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
1961 rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
1962 }
1963 }
1964
1965 static int
rwmutex_try_complete_runlock(ethr_rwmutex * rwmtx,ethr_sint32_t initial,ethr_ts_event * tse,int start_next_ix,int check_before_try,int try_write_lock)1966 rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
1967 ethr_sint32_t initial,
1968 ethr_ts_event *tse,
1969 int start_next_ix,
1970 int check_before_try,
1971 int try_write_lock)
1972 {
1973 ethr_ts_event *tse_tmp;
1974 ethr_sint32_t act = initial;
1975 int six, res, length;
1976
1977 ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
1978
1979 if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__)
1980 return try_write_lock ? EBUSY : 0;
1981
1982 tse_tmp = tse;
1983 if (!tse_tmp)
1984 tse_tmp = ethr_peek_ts_event();
1985
1986 if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
1987 goto check_waiters;
1988
1989 if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
1990 length = reader_groups_array_size;
1991 six = tse_tmp->rgix;
1992 }
1993 else {
1994 length = main_threads_array_size;
1995 six = tse_tmp->mtix;
1996 }
1997 if (start_next_ix) {
1998 six++;
1999 if (six >= length)
2000 six = 0;
2001 }
2002
2003 if (!tse)
2004 ethr_unpeek_ts_event(tse_tmp);
2005
2006 if (check_before_try) {
2007 res = check_readers_array(rwmtx, six, length);
2008
2009 ETHR_MEMORY_BARRIER;
2010
2011 if (res == EBUSY)
2012 return try_write_lock ? EBUSY : 0;
2013 }
2014
2015 restart:
2016
2017 while (1) {
2018 ethr_sint32_t exp = act;
2019 ethr_sint32_t new = act+1;
2020
2021 ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0);
2022
2023 ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
2024 < ETHR_RWMTX_R_PEND_UNLCK_MASK__);
2025
2026 act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
2027 if (exp == act) {
2028 act = new;
2029 break;
2030 }
2031
2032 if (!try_write_lock) {
2033 if (act == 0 || (act & (ETHR_RWMTX_W_FLG__
2034 | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)))
2035 return 0;
2036 if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
2037 if ((act & ETHR_RWMTX_R_FLG__) == 0)
2038 return 0;
2039 }
2040 else if ((act & ETHR_RWMTX_R_FLG__) == 0) {
2041 if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
2042 return 0;
2043 goto check_waiters;
2044 }
2045 }
2046 else {
2047 if (act == 0)
2048 goto tryrwlock;
2049 if (act & (ETHR_RWMTX_W_FLG__
2050 | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))
2051 return EBUSY;
2052 }
2053 }
2054
2055 res = check_readers_array(rwmtx, six, length);
2056
2057 ETHR_MEMORY_BARRIER;
2058
2059 ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
2060
2061 while (1) {
2062 int finished_abort = 0;
2063 ethr_sint32_t exp = act;
2064 ethr_sint32_t new = act;
2065
2066 new--;
2067 if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
2068 if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) {
2069 new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
2070 finished_abort = 1;
2071 }
2072 ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__);
2073 }
2074 else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) {
2075 new &= ~ETHR_RWMTX_R_FLG__;
2076 }
2077
2078 ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
2079
2080 act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
2081 if (exp == act) {
2082 act = new;
2083 if (act & ETHR_RWMTX_W_FLG__)
2084 return try_write_lock ? EBUSY : 0;
2085 if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__))
2086 goto restart;
2087 if (act & (ETHR_RWMTX_R_FLG__
2088 | ETHR_RWMTX_R_ABRT_UNLCK_FLG__
2089 | ETHR_RWMTX_R_PEND_UNLCK_MASK__))
2090 return try_write_lock ? EBUSY : 0;
2091 /* Read unlock completed */
2092 break;
2093 }
2094 }
2095
2096 /*
2097 * Read unlock completed, but we have to check if
2098 * threads have to be woken (or if we should try
2099 * to write lock it).
2100 */
2101
2102 if (act & ETHR_RWMTX_WAIT_FLGS__) {
2103 check_waiters:
2104 rwmutex_unlock_wake(rwmtx, 0, act, 0);
2105 return try_write_lock ? EBUSY : 0;
2106 }
2107
2108 if (!try_write_lock)
2109 return 0;
2110
2111 tryrwlock:
2112 /* Try to write lock it */
2113
2114 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
2115 ETHR_RWMTX_W_FLG__,
2116 0);
2117 return act == 0 ? 0 : EBUSY;
2118 }
2119
2120 #ifdef ETHR_RLOCK_WITH_INC_DEC
2121
2122 static ETHR_INLINE void
rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex * rwmtx)2123 rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx)
2124 {
2125 ethr_sint32_t act;
2126 /*
2127 * Restore failed increment
2128 */
2129 act = ethr_atomic32_dec_read(&rwmtx->mtxb.flgs);
2130 if ((act & ETHR_RWMTX_WAIT_FLGS__)
2131 && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
2132 rwmutex_unlock_wake(rwmtx, 0, act, 0);
2133 }
2134 }
2135
2136 #endif
2137
2138 static void
rwmutex_normal_rlock_wait(ethr_rwmutex * rwmtx,ethr_sint32_t initial)2139 rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
2140 {
2141 ethr_sint32_t act = initial, exp;
2142 int scnt, start_scnt;
2143 ethr_ts_event *tse = ethr_get_ts_event();
2144 int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
2145
2146 start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
2147
2148 /*
2149 * Spin trying to read lock for a while. If unsuccessful,
2150 * wait on event.
2151 */
2152
2153 while (1) {
2154
2155 #ifdef ETHR_RLOCK_WITH_INC_DEC
2156 rwmutex_incdec_restore_failed_tryrlock(rwmtx);
2157 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2158 #endif
2159
2160 while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
2161 if (scnt <= 0) {
2162 if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
2163 event_wait(&rwmtx->mtxb, tse, scnt,
2164 ETHR_RWMTX_R_WAIT_FLG__, 1, 0);
2165 goto done;
2166 }
2167 }
2168 ETHR_SPIN_BODY;
2169 if (--until_yield == 0) {
2170 until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
2171 (void) ETHR_YIELD();
2172 }
2173 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2174 scnt--;
2175 }
2176 exp = act;
2177
2178 #ifdef ETHR_RLOCK_WITH_INC_DEC
2179 act = ethr_atomic32_inc_read(&rwmtx->mtxb.flgs);
2180 if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0)
2181 goto done; /* Got it */
2182 #else
2183 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
2184 if (act == exp)
2185 goto done; /* Got it */
2186 #endif
2187 }
2188
2189 done:
2190 ethr_leave_ts_event(tse);
2191 }
2192
2193 static void
2194 rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
2195 ethr_ts_event *tse);
2196
2197 static int
rwmutex_freqread_rlock(ethr_rwmutex * rwmtx,ethr_ts_event * tse,int trylock)2198 rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock)
2199 {
2200 int res = 0;
2201 ethr_sint32_t act;
2202
2203 rwmutex_freqread_rdrs_inc(rwmtx, tse);
2204
2205 ETHR_MEMORY_BARRIER;
2206
2207 act = ethr_atomic32_read_acqb(&rwmtx->mtxb.flgs);
2208
2209 if (act != ETHR_RWMTX_R_FLG__) {
2210 int wake_other_readers;
2211
2212 while (1) {
2213 ethr_sint32_t exp, new;
2214
2215 wake_other_readers = 0;
2216
2217 if (act == 0)
2218 new = act | ETHR_RWMTX_R_FLG__;
2219 else if (act == ETHR_RWMTX_R_FLG__)
2220 break; /* Got it */
2221 else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
2222 rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
2223 if (trylock)
2224 res = EBUSY;
2225 else
2226 rwmutex_freqread_rlock_wait(rwmtx, tse);
2227 break;
2228 }
2229 else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
2230 if ((act & ETHR_RWMTX_R_FLG__) == 0)
2231 ETHR_FATAL_ERROR__(EFAULT);
2232 /*
2233 * An aborted runlock, not write locked, and no write
2234 * waiters, i.e., we got it...
2235 */
2236 if (act & ETHR_RWMTX_R_WAIT_FLG__)
2237 wake_other_readers = 1;
2238 break;
2239 }
2240 else {
2241 new = act | ETHR_RWMTX_R_FLG__;
2242 if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
2243 /*
2244 * Someone is doing tryrwlock (no writer and no
2245 * write waiters); we will try to abort that...
2246 */
2247 new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
2248 }
2249
2250 if (act & ETHR_RWMTX_R_WAIT_FLG__)
2251 wake_other_readers = 1;
2252 }
2253
2254 exp = act;
2255 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
2256 if (act == exp)
2257 break;
2258 }
2259
2260 if (wake_other_readers)
2261 rwmutex_transfer_read_lock(rwmtx, act, 0);
2262 }
2263
2264 return res;
2265 }
2266
2267 static void
rwmutex_freqread_rlock_wait(ethr_rwmutex * rwmtx,ethr_ts_event * tse)2268 rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
2269 ethr_ts_event *tse)
2270 {
2271 ethr_sint32_t act;
2272 int scnt, start_scnt;
2273 int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
2274
2275 start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
2276
2277 /*
2278 * Spin trying to read lock for a while. If unsuccessful,
2279 * wait on event.
2280 */
2281
2282 while (1) {
2283
2284 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2285
2286 while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
2287 if (scnt <= 0) {
2288 if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
2289 ethr_ts_event *tmp_tse = ethr_use_ts_event(tse);
2290 event_wait(&rwmtx->mtxb, tmp_tse, scnt,
2291 ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
2292 ethr_leave_ts_event(tmp_tse);
2293 return; /* Got it */
2294 }
2295 }
2296 ETHR_SPIN_BODY;
2297 if (--until_yield == 0) {
2298 until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
2299 (void) ETHR_YIELD();
2300 }
2301 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2302 scnt--;
2303 }
2304
2305 if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY)
2306 break; /* Got it */
2307 }
2308 }
2309
2310 static void
rwmutex_normal_rwlock_wait(ethr_rwmutex * rwmtx,ethr_sint32_t initial)2311 rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
2312 {
2313 write_lock_wait(&rwmtx->mtxb, initial, 1, 0);
2314 }
2315
2316 static void
rwmutex_freqread_rwlock_wait(ethr_rwmutex * rwmtx,ethr_sint32_t initial)2317 rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
2318 {
2319 write_lock_wait(&rwmtx->mtxb, initial, 1, 1);
2320 }
2321
2322 static ETHR_INLINE void
rwlock_wake_set_flags(ethr_rwmutex * rwmtx,ethr_sint32_t new_initial,ethr_sint32_t act_initial)2323 rwlock_wake_set_flags(ethr_rwmutex *rwmtx,
2324 ethr_sint32_t new_initial,
2325 ethr_sint32_t act_initial)
2326 {
2327 ethr_sint32_t act, act_mask;
2328 int chk_abrt_flg;
2329
2330 ETHR_MEMORY_BARRIER;
2331
2332 if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
2333 /* r pend unlock mask may vary and must be retained */
2334 act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
2335 if (new_initial & ETHR_RWMTX_R_FLG__)
2336 chk_abrt_flg = 1;
2337 else
2338 chk_abrt_flg = 0;
2339 }
2340 else {
2341 #ifdef ETHR_RLOCK_WITH_INC_DEC
2342 /* rs mask may vary and must be retained */
2343 act_mask = ETHR_RWMTX_RS_MASK__;
2344 chk_abrt_flg = 0;
2345 #else
2346 /* rs mask always zero */
2347 ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0);
2348 ethr_atomic32_set(&rwmtx->mtxb.flgs, new_initial);
2349 return;
2350 #endif
2351 }
2352
2353 act = act_initial;
2354 while (1) {
2355 ethr_sint32_t exp = act;
2356 ethr_sint32_t new = new_initial + (act & act_mask);
2357 if (chk_abrt_flg && (act & act_mask))
2358 new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
2359 act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
2360 if (act == exp)
2361 break;
2362 exp = act;
2363 }
2364 }
2365
2366 #ifdef ETHR_DEBUG
2367
2368 static void
dbg_unlock_wake(ethr_rwmutex * rwmtx,int have_w,ethr_ts_event * tse)2369 dbg_unlock_wake(ethr_rwmutex *rwmtx,
2370 int have_w,
2371 ethr_ts_event *tse)
2372 {
2373 ethr_sint32_t exp, act, imask;
2374
2375 exp = have_w ? ETHR_RWMTX_W_FLG__ : 0;
2376
2377 if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
2378 imask = ETHR_RWMTX_R_PEND_UNLCK_MASK__|ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
2379 else {
2380 #ifdef ETHR_RLOCK_WITH_INC_DEC
2381 imask = ETHR_RWMTX_RS_MASK__;
2382 #else
2383 imask = 0;
2384 #endif
2385 }
2386
2387 ETHR_ASSERT(tse);
2388
2389 if (is_w_waiter(tse)) {
2390
2391 exp |= ETHR_RWMTX_W_WAIT_FLG__;
2392 if (rwmtx->rq_end) {
2393 exp |= ETHR_RWMTX_R_WAIT_FLG__;
2394 }
2395 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2396 ETHR_ASSERT((exp & ~imask) == (act & ~imask));
2397
2398 ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
2399
2400 }
2401 else {
2402
2403 exp |= ETHR_RWMTX_R_WAIT_FLG__;
2404 if (rwmtx->rq_end->next != rwmtx->mtxb.q)
2405 exp |= ETHR_RWMTX_W_WAIT_FLG__;
2406 else if (exp == ETHR_RWMTX_R_WAIT_FLG__) {
2407 if (!have_w) {
2408 if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
2409 imask |= ETHR_RWMTX_R_FLG__;
2410 else
2411 imask |= ETHR_RWMTX_RS_MASK__;
2412 }
2413 }
2414 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2415 ETHR_ASSERT((exp & ~imask) == (act & ~imask));
2416
2417 ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
2418
2419 }
2420 }
2421
2422 #endif
2423
2424 static void
rwmutex_transfer_read_lock(ethr_rwmutex * rwmtx,ethr_sint32_t initial,int q_locked)2425 rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
2426 ethr_sint32_t initial,
2427 int q_locked)
2428 {
2429 ethr_sint32_t act = initial;
2430
2431 if (!q_locked) {
2432 ethr_ts_event *tse;
2433 ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__);
2434 ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0);
2435 ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
2436
2437 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2438 tse = rwmtx->mtxb.q;
2439 if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) {
2440 /* Someone else woke the readers up... */
2441 ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
2442 return;
2443 }
2444 }
2445
2446 rwmutex_unlock_wake(rwmtx, 0, initial, 1);
2447 }
2448
2449 static void
rwmutex_unlock_wake(ethr_rwmutex * rwmtx,int have_w,ethr_sint32_t initial,int transfer_read_lock)2450 rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, ethr_sint32_t initial,
2451 int transfer_read_lock)
2452 {
2453 ethr_sint32_t new, act = initial;
2454 ethr_ts_event *tse;
2455
2456 if (transfer_read_lock) {
2457 /*
2458 * - Q already locked
2459 * - Got R waiters first in Q
2460 * - Not W locked
2461 */
2462 tse = rwmtx->mtxb.q;
2463
2464 ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__);
2465 ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0);
2466 ETHR_ASSERT(tse && !is_w_waiter(tse));
2467 }
2468 else {
2469
2470 if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
2471 if (!have_w)
2472 return;
2473 else {
2474 while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
2475 ethr_sint32_t exp = act;
2476 new = exp & ~ETHR_RWMTX_W_FLG__;
2477 act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
2478 if (act == exp)
2479 return;
2480 }
2481 }
2482 }
2483
2484 ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
2485 tse = rwmtx->mtxb.q;
2486
2487 if (!have_w) {
2488 if (!tse) {
2489 #ifdef ETHR_DEBUG
2490 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2491 ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
2492 #endif
2493 goto already_served;
2494 }
2495 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2496 if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) {
2497 ETHR_ASSERT(tse && !is_w_waiter(tse));
2498 }
2499 else if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
2500 already_served:
2501 ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
2502 return;
2503 }
2504 }
2505 }
2506
2507 #ifdef ETHR_DEBUG
2508 dbg_unlock_wake(rwmtx, have_w, tse);
2509 #endif
2510
2511 if (is_w_waiter(tse)) {
2512
2513 if (!have_w) {
2514 act = ethr_atomic32_read_bor(&rwmtx->mtxb.flgs,
2515 ETHR_RWMTX_W_FLG__);
2516 ETHR_ASSERT((act & ~(ETHR_RWMTX_WAIT_FLGS__
2517 | (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL
2518 ? 0
2519 : ETHR_RWMTX_R_PEND_UNLCK_MASK__))) == 0);
2520 ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__);
2521 act |= ETHR_RWMTX_W_FLG__;
2522 }
2523
2524 /*
2525 * If we have multiple write waiters, there
2526 * is no need to modify mtxb->flgs; otherwise,
2527 * we need to clear the write wait bit...
2528 */
2529 if (!multiple_w_waiters(rwmtx)) {
2530 new = ETHR_RWMTX_W_FLG__;
2531 if (tse->next != rwmtx->mtxb.q) {
2532 ETHR_ASSERT(tse->next->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
2533 new |= ETHR_RWMTX_R_WAIT_FLG__;
2534 }
2535
2536 rwlock_wake_set_flags(rwmtx, new, act);
2537 }
2538
2539 wake_writer(&rwmtx->mtxb, 1);
2540 }
2541 else {
2542 int rs;
2543
2544 if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) {
2545 rs = rwmtx->tdata.rs;
2546 new = (ethr_sint32_t) rs;
2547 rwmtx->tdata.rs = 0;
2548 }
2549 else {
2550 ethr_rwmutex_type type = rwmtx->type;
2551 int length = (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
2552 ? reader_groups_array_size
2553 : main_threads_array_size);
2554 int ix;
2555
2556 rs = 0;
2557 for (ix = 0; ix < length; ix++) {
2558 int wrs = rwmtx->tdata.ra[ix].data.waiting_readers;
2559 rwmtx->tdata.ra[ix].data.waiting_readers = 0;
2560 ETHR_ASSERT(wrs >= 0);
2561 if (wrs) {
2562 rs += wrs;
2563 rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs);
2564 }
2565 }
2566
2567 new = ETHR_RWMTX_R_FLG__;
2568 }
2569
2570 if (rwmtx->rq_end->next != rwmtx->mtxb.q)
2571 new |= ETHR_RWMTX_W_WAIT_FLG__;
2572
2573 rwlock_wake_set_flags(rwmtx, new, act);
2574
2575 wake_readers(rwmtx, rs);
2576 }
2577 }
2578
2579 static ethr_rwmtx_readers_array__ *
alloc_readers_array(int length,ethr_rwmutex_lived lived)2580 alloc_readers_array(int length, ethr_rwmutex_lived lived)
2581 {
2582 ethr_rwmtx_readers_array__ *ra;
2583 size_t sz;
2584 void *mem;
2585
2586 sz = sizeof(ethr_rwmtx_readers_array__) * (length + 1);
2587
2588 switch (lived) {
2589 case ETHR_RWMUTEX_LONG_LIVED:
2590 mem = ethr_mem__.ll.alloc(sz);
2591 break;
2592 case ETHR_RWMUTEX_SHORT_LIVED:
2593 mem = ethr_mem__.sl.alloc(sz);
2594 break;
2595 default:
2596 mem = ethr_mem__.std.alloc(sz);
2597 break;
2598 }
2599 if (!mem)
2600 return NULL;
2601
2602 if ((((ethr_uint_t) mem) & ETHR_CACHE_LINE_MASK) == 0) {
2603 ra = (ethr_rwmtx_readers_array__ *) mem;
2604 ra->data.byte_offset = 0;
2605 }
2606 else {
2607 ra = ((ethr_rwmtx_readers_array__ *)
2608 ((((ethr_uint_t) mem) & ~ETHR_CACHE_LINE_MASK)
2609 + ETHR_CACHE_LINE_SIZE));
2610 ra->data.byte_offset = (int) ((ethr_uint_t) ra
2611 - (ethr_uint_t) mem);
2612 }
2613 ra->data.lived = lived;
2614 return ra;
2615 }
2616
2617 static void
free_readers_array(ethr_rwmtx_readers_array__ * ra)2618 free_readers_array(ethr_rwmtx_readers_array__ *ra)
2619 {
2620 void *ptr = (void *) (((char *) ra) - ra->data.byte_offset);
2621 switch (ra->data.lived) {
2622 case ETHR_RWMUTEX_LONG_LIVED:
2623 ethr_mem__.ll.free(ptr);
2624 break;
2625 case ETHR_RWMUTEX_SHORT_LIVED:
2626 ethr_mem__.sl.free(ptr);
2627 break;
2628 default:
2629 ethr_mem__.std.free(ptr);
2630 break;
2631 }
2632 }
2633
2634 int
ethr_rwmutex_init_opt(ethr_rwmutex * rwmtx,ethr_rwmutex_opt * opt)2635 ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
2636 {
2637 int res;
2638 ethr_rwmtx_readers_array__ *ra = NULL;
2639 #if ETHR_XCHK
2640 if (ethr_not_completely_inited__) {
2641 ETHR_ASSERT(0);
2642 return EACCES;
2643 }
2644 if (!rwmtx) {
2645 ETHR_ASSERT(0);
2646 return EINVAL;
2647 }
2648 rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
2649 #endif
2650 ETHR_MTX_HARD_DEBUG_FENCE_INIT(rwmtx);
2651 rwmtx->rq_end = NULL;
2652 rwmtx->type = opt ? opt->type : ETHR_RWMUTEX_TYPE_NORMAL;
2653 switch (rwmtx->type) {
2654 case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
2655 if (main_threads_array_size <= reader_groups_array_size) {
2656 /* No point using reader groups... */
2657 rwmtx->type = ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ;
2658 }
2659 /* Fall through */
2660 case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
2661 int length;
2662
2663 length = (rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
2664 ? main_threads_array_size
2665 : reader_groups_array_size);
2666
2667 if (length == 1) {
2668 /* No point using a frequent reader type... */
2669 rwmtx->type = ETHR_RWMUTEX_TYPE_NORMAL;
2670 }
2671 else {
2672 int ix;
2673 ra = alloc_readers_array(length,
2674 (opt
2675 ? opt->lived
2676 : ETHR_RWMUTEX_UNKNOWN_LIVED));
2677 if (!ra) {
2678 res = ENOMEM;
2679 goto error;
2680 }
2681
2682 rwmtx->tdata.ra = ra;
2683
2684 for (ix = 0; ix < length; ix++) {
2685 ethr_atomic32_init(&rwmtx->tdata.ra[ix].data.readers, 0);
2686 rwmtx->tdata.ra[ix].data.waiting_readers = 0;
2687 }
2688 break;
2689 }
2690 }
2691 case ETHR_RWMUTEX_TYPE_NORMAL:
2692 rwmtx->tdata.rs = 0;
2693 break;
2694 default:
2695 res = EINVAL;
2696 goto error;
2697 }
2698 res = mtxb_init(&rwmtx->mtxb,
2699 default_rwmtx_main_spincount,
2700 opt ? opt->main_spincount : -1,
2701 default_rwmtx_aux_spincount,
2702 opt ? opt->aux_spincount : -1);
2703 if (res == 0)
2704 return 0;
2705
2706 error:
2707
2708 if (ra)
2709 free_readers_array(ra);
2710
2711 #if ETHR_XCHK
2712 rwmtx->initialized = 0;
2713 #endif
2714 return res;
2715 }
2716
2717 int
ethr_rwmutex_init(ethr_rwmutex * rwmtx)2718 ethr_rwmutex_init(ethr_rwmutex *rwmtx)
2719 {
2720 return ethr_rwmutex_init_opt(rwmtx, NULL);
2721 }
2722
2723 int
ethr_rwmutex_destroy(ethr_rwmutex * rwmtx)2724 ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
2725 {
2726 int res;
2727 #if ETHR_XCHK
2728 if (ethr_not_inited__) {
2729 ETHR_ASSERT(0);
2730 return EACCES;
2731 }
2732 if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
2733 ETHR_ASSERT(0);
2734 return EINVAL;
2735 }
2736 #endif
2737 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2738 if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
2739 ethr_sint32_t act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2740 if (act == ETHR_RWMTX_R_FLG__)
2741 rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0);
2742 }
2743 res = mtxb_destroy(&rwmtx->mtxb);
2744 if (res != 0)
2745 return res;
2746 if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
2747 free_readers_array(rwmtx->tdata.ra);
2748 #if ETHR_XCHK
2749 rwmtx->initialized = 0;
2750 #endif
2751 return 0;
2752 }
2753
2754 #define ETHR_MAX_TRYRLOCK_TRIES 5
2755
2756 int
ethr_rwmutex_tryrlock(ethr_rwmutex * rwmtx)2757 ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
2758 {
2759 int res = 0;
2760 ethr_sint32_t act;
2761
2762 ETHR_ASSERT(!ethr_not_inited__);
2763 ETHR_ASSERT(rwmtx);
2764 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
2765
2766 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
2767
2768 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2769
2770 switch (rwmtx->type) {
2771 case ETHR_RWMUTEX_TYPE_NORMAL: {
2772 #ifdef ETHR_RLOCK_WITH_INC_DEC
2773 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2774 if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
2775 res = EBUSY;
2776 else {
2777 act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
2778 if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
2779 rwmutex_incdec_restore_failed_tryrlock(rwmtx);
2780 res = EBUSY;
2781 }
2782 }
2783 #else
2784 ethr_sint32_t exp = 0;
2785 int tries = 0;
2786
2787 while (1) {
2788 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
2789 if (act == exp) {
2790 res = 0;
2791 break;
2792 }
2793 if (tries > ETHR_MAX_TRYRLOCK_TRIES
2794 || (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))) {
2795 res = EBUSY;
2796 break;
2797 }
2798 tries++;
2799 exp = act;
2800 }
2801 #endif
2802 break;
2803 }
2804
2805 case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
2806 case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
2807 ethr_ts_event *tse = ethr_peek_ts_event();
2808 res = rwmutex_freqread_rlock(rwmtx, tse, 1);
2809 ethr_unpeek_ts_event(tse);
2810 break;
2811 }
2812 }
2813
2814 #ifdef ETHR_MTX_CHK_EXCL
2815 if (res == 0) {
2816 ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
2817 ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
2818 }
2819 #endif
2820
2821 ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res);
2822 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
2823
2824 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2825 return res;
2826 }
2827
2828 void
ethr_rwmutex_rlock(ethr_rwmutex * rwmtx)2829 ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
2830 {
2831 ethr_sint32_t act;
2832
2833 ETHR_ASSERT(!ethr_not_inited__);
2834 ETHR_ASSERT(rwmtx);
2835 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
2836
2837 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
2838
2839 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2840
2841 switch (rwmtx->type) {
2842 case ETHR_RWMUTEX_TYPE_NORMAL: {
2843 #ifdef ETHR_RLOCK_WITH_INC_DEC
2844 act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
2845 if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
2846 rwmutex_normal_rlock_wait(rwmtx, act);
2847 #else
2848 ethr_sint32_t exp = 0;
2849
2850 while (1) {
2851 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
2852 if (act == exp)
2853 break;
2854
2855 if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
2856 rwmutex_normal_rlock_wait(rwmtx, act);
2857 break;
2858 }
2859 exp = act;
2860 }
2861 #endif
2862 break;
2863 }
2864
2865 case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
2866 case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
2867 ethr_ts_event *tse = ethr_peek_ts_event();
2868 rwmutex_freqread_rlock(rwmtx, tse, 0);
2869 ethr_unpeek_ts_event(tse);
2870 break;
2871 }
2872 }
2873
2874 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2875 ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
2876 ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
2877 ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb);
2878 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
2879 }
2880
2881 void
ethr_rwmutex_runlock(ethr_rwmutex * rwmtx)2882 ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
2883 {
2884 ethr_sint32_t act;
2885
2886 ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
2887 ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb);
2888 ETHR_ASSERT(!ethr_not_inited__);
2889 ETHR_ASSERT(rwmtx);
2890 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
2891
2892 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
2893 ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb);
2894
2895 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2896
2897 switch (rwmtx->type) {
2898 case ETHR_RWMUTEX_TYPE_NORMAL:
2899 act = ethr_atomic32_dec_read_relb(&rwmtx->mtxb.flgs);
2900 if ((act & ETHR_RWMTX_WAIT_FLGS__)
2901 && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
2902 ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
2903 rwmutex_unlock_wake(rwmtx, 0, act, 0);
2904 }
2905 break;
2906
2907 case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
2908 case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
2909 ethr_ts_event *tse = ethr_peek_ts_event();
2910
2911 act = rwmutex_freqread_rdrs_dec_read_relb(rwmtx, tse);
2912
2913 ETHR_ASSERT(act >= 0);
2914
2915 ETHR_MEMORY_BARRIER;
2916
2917 if (act == 0) {
2918 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2919 if (act != ETHR_RWMTX_R_FLG__)
2920 rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
2921 }
2922
2923 ethr_unpeek_ts_event(tse);
2924 break;
2925 }
2926 }
2927
2928 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2929 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
2930 }
2931
2932 int
ethr_rwmutex_tryrwlock(ethr_rwmutex * rwmtx)2933 ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
2934 {
2935 int res = 0;
2936 ethr_sint32_t act;
2937
2938 ETHR_ASSERT(!ethr_not_inited__);
2939 ETHR_ASSERT(rwmtx);
2940 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
2941
2942 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
2943
2944 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2945
2946 switch (rwmtx->type) {
2947 case ETHR_RWMUTEX_TYPE_NORMAL:
2948 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
2949 ETHR_RWMTX_W_FLG__, 0);
2950 if (act != 0)
2951 res = EBUSY;
2952 break;
2953
2954 case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
2955 case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
2956
2957 res = 0;
2958 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
2959
2960 do {
2961
2962 if (act == 0)
2963 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
2964 ETHR_RWMTX_W_FLG__, 0);
2965 else if (act == ETHR_RWMTX_R_FLG__) {
2966 res = rwmutex_try_complete_runlock(rwmtx, act, NULL,
2967 0, 1, 1);
2968 break;
2969 }
2970 else {
2971 res = EBUSY;
2972 break;
2973 }
2974
2975 } while (act != 0);
2976
2977 break;
2978 }
2979
2980 #ifdef ETHR_MTX_CHK_EXCL
2981 if (res == 0) {
2982 ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
2983 ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
2984 }
2985 #endif
2986
2987 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
2988 ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res);
2989 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
2990
2991 return res;
2992 }
2993
2994 void
ethr_rwmutex_rwlock(ethr_rwmutex * rwmtx)2995 ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
2996 {
2997 ethr_sint32_t act;
2998 ETHR_ASSERT(!ethr_not_inited__);
2999 ETHR_ASSERT(rwmtx);
3000 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
3001
3002 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
3003
3004 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
3005
3006 switch (rwmtx->type) {
3007 case ETHR_RWMUTEX_TYPE_NORMAL:
3008 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
3009 ETHR_RWMTX_W_FLG__, 0);
3010 if (act != 0)
3011 rwmutex_normal_rwlock_wait(rwmtx, act);
3012 break;
3013
3014 case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
3015 case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
3016
3017 act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
3018
3019 do {
3020
3021 if (act != 0) {
3022 rwmutex_freqread_rwlock_wait(rwmtx, act);
3023 break;
3024 }
3025
3026 act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
3027 ETHR_RWMTX_W_FLG__, 0);
3028
3029 } while (act != 0);
3030
3031 break;
3032 }
3033
3034 ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
3035 ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
3036 ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb);
3037 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
3038 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
3039
3040 }
3041
3042 void
ethr_rwmutex_rwunlock(ethr_rwmutex * rwmtx)3043 ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
3044 {
3045 ethr_sint32_t act;
3046 ETHR_ASSERT(!ethr_not_inited__);
3047 ETHR_ASSERT(rwmtx);
3048 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
3049
3050 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
3051 ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb);
3052
3053 ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
3054 ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb);
3055
3056 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
3057
3058 switch (rwmtx->type) {
3059 case ETHR_RWMUTEX_TYPE_NORMAL:
3060 act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs,
3061 0, ETHR_RWMTX_W_FLG__);
3062 if (act != ETHR_RWMTX_W_FLG__)
3063 rwmutex_unlock_wake(rwmtx, 1, act, 0);
3064 break;
3065
3066 case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
3067 case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
3068 act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
3069 ETHR_RWMTX_W_FLG__);
3070 if (act != ETHR_RWMTX_W_FLG__)
3071 rwmutex_unlock_wake(rwmtx, 1, act, 0);
3072 break;
3073 }
3074
3075 ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
3076 ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
3077 }
3078
3079 #else
3080 /* -- pthread read/write mutex --------------------------------------------- */
3081
3082 int
ethr_rwmutex_init(ethr_rwmutex * rwmtx)3083 ethr_rwmutex_init(ethr_rwmutex *rwmtx)
3084 {
3085 #if ETHR_XCHK
3086 if (!rwmtx) {
3087 ETHR_ASSERT(0);
3088 return EINVAL;
3089 }
3090 rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
3091 #endif
3092 return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr);
3093 }
3094
3095 int
ethr_rwmutex_init_opt(ethr_rwmutex * rwmtx,ethr_rwmutex_opt * opt)3096 ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
3097 {
3098 return ethr_rwmutex_init(rwmtx);
3099 }
3100
3101 int
ethr_rwmutex_destroy(ethr_rwmutex * rwmtx)3102 ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
3103 {
3104 int res;
3105 #if ETHR_XCHK
3106 if (ethr_not_inited__) {
3107 ETHR_ASSERT(0);
3108 return EACCES;
3109 }
3110 if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
3111 ETHR_ASSERT(0);
3112 return EINVAL;
3113 }
3114 #endif
3115 res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
3116 #if ETHR_XCHK
3117 rwmtx->initialized = 0;
3118 #endif
3119 return res;
3120 }
3121
3122 /* -- Exported symbols of inline functions --------------------------------- */
3123
3124 int
ethr_rwmutex_tryrlock(ethr_rwmutex * rwmtx)3125 ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
3126 {
3127 ETHR_ASSERT(!ethr_not_inited__);
3128 ETHR_ASSERT(rwmtx);
3129 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
3130
3131 return ethr_rwmutex_tryrlock__(rwmtx);
3132 }
3133
3134 void
ethr_rwmutex_rlock(ethr_rwmutex * rwmtx)3135 ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
3136 {
3137 ETHR_ASSERT(!ethr_not_inited__);
3138 ETHR_ASSERT(rwmtx);
3139 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
3140
3141 ethr_rwmutex_rlock__(rwmtx);
3142 }
3143
3144 void
ethr_rwmutex_runlock(ethr_rwmutex * rwmtx)3145 ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
3146 {
3147 ETHR_ASSERT(!ethr_not_inited__);
3148 ETHR_ASSERT(rwmtx);
3149 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
3150
3151 ethr_rwmutex_runlock__(rwmtx);
3152 }
3153
3154 int
ethr_rwmutex_tryrwlock(ethr_rwmutex * rwmtx)3155 ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
3156 {
3157 ETHR_ASSERT(!ethr_not_inited__);
3158 ETHR_ASSERT(rwmtx);
3159 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
3160
3161 return ethr_rwmutex_tryrwlock__(rwmtx);
3162 }
3163
3164 void
ethr_rwmutex_rwlock(ethr_rwmutex * rwmtx)3165 ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
3166 {
3167 ETHR_ASSERT(!ethr_not_inited__);
3168 ETHR_ASSERT(rwmtx);
3169 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
3170
3171 return ethr_rwmutex_rwlock__(rwmtx);
3172 }
3173
3174 void
ethr_rwmutex_rwunlock(ethr_rwmutex * rwmtx)3175 ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
3176 {
3177 ETHR_ASSERT(!ethr_not_inited__);
3178 ETHR_ASSERT(rwmtx);
3179 ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
3180
3181 ethr_rwmutex_rwunlock__(rwmtx);
3182 }
3183
3184 #endif /* pthread */
3185
3186
3187 #if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
3188
3189 #ifdef ETHR_MTX_HARD_DEBUG_Q
3190 static void
hard_debug_chk_q__(struct ethr_mutex_base_ * mtxb,int is_rwmtx)3191 hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
3192 {
3193 int res;
3194 ethr_sint32_t flgs = ethr_atomic32_read(&mtxb->flgs);
3195
3196 ETHR_MTX_HARD_ASSERT(res == 0);
3197
3198 ETHR_MTX_HARD_ASSERT(!(flgs & ETHR_RWMTX_R_WAIT_FLG__) || is_rwmtx);
3199
3200 if (!(flgs & ETHR_RWMTX_WAIT_FLGS__)) {
3201 ETHR_MTX_HARD_ASSERT(!mtxb->q);
3202 if (is_rwmtx) {
3203 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
3204 ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
3205 ETHR_MTX_HARD_ASSERT(!rwmtx->rs);
3206 }
3207 }
3208 else {
3209 ethr_ts_event *tse;
3210 int ws = 0, rs = 0, rsf = 0, ref = 0;
3211
3212 ETHR_MTX_HARD_ASSERT(mtxb->q);
3213
3214 tse = mtxb->q;
3215
3216 do {
3217 ethr_sint32_t type;
3218
3219 ETHR_MTX_HARD_ASSERT(tse->next->prev == tse);
3220 ETHR_MTX_HARD_ASSERT(tse->prev->next == tse);
3221
3222 type = ethr_atomic32_read(&tse->uaflgs);
3223 ETHR_MTX_HARD_ASSERT(type == tse->uflgs);
3224 switch (type) {
3225 case ETHR_RWMTX_W_WAIT_FLG__:
3226 ws++;
3227 break;
3228 case ETHR_RWMTX_R_WAIT_FLG__: {
3229 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
3230 ETHR_MTX_HARD_ASSERT(is_rwmtx);
3231 if (!rsf)
3232 rsf = 1;
3233 ETHR_MTX_HARD_ASSERT(!ref);
3234 if (rwmtx->rq_end == tse) {
3235 ETHR_MTX_HARD_ASSERT(
3236 tse->next == rwmtx->mtxb.q
3237 || tse->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
3238 ref = 1;
3239 }
3240 rs++;
3241 break;
3242 }
3243 default:
3244 ETHR_MTX_HARD_ASSERT(! "invalid wait type found");
3245 }
3246
3247 tse = tse->next;
3248 } while (tse != mtxb->q);
3249
3250 if (is_rwmtx) {
3251 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
3252 ETHR_MTX_HARD_ASSERT(rs == rwmtx->rs);
3253 }
3254
3255 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
3256 ETHR_MTX_HARD_ASSERT(ws == mtxb->ws);
3257 #endif
3258
3259 if (flgs & ETHR_RWMTX_W_WAIT_FLG__)
3260 ETHR_MTX_HARD_ASSERT(ws);
3261 else
3262 ETHR_MTX_HARD_ASSERT(!ws);
3263
3264 if (flgs & ETHR_RWMTX_R_WAIT_FLG__) {
3265 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
3266 ETHR_MTX_HARD_ASSERT(is_rwmtx);
3267 ETHR_MTX_HARD_ASSERT(rwmtx->rq_end);
3268 ETHR_MTX_HARD_ASSERT(rsf);
3269 ETHR_MTX_HARD_ASSERT(ref);
3270 ETHR_MTX_HARD_ASSERT(rs);
3271 }
3272 else {
3273 if (is_rwmtx) {
3274 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
3275 ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
3276 }
3277 ETHR_MTX_HARD_ASSERT(!rsf);
3278 ETHR_MTX_HARD_ASSERT(!ref);
3279 ETHR_MTX_HARD_ASSERT(!rs);
3280 }
3281 }
3282 }
3283
3284 #elif defined(ETHR_MTX_HARD_DEBUG_WSQ)
3285
3286 static void
hard_debug_chk_q__(struct ethr_mutex_base_ * mtxb,int is_rwmtx)3287 hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
3288 {
3289 int ws = 0;
3290 int rs = 0;
3291
3292 if (mtxb->q) {
3293 ethr_ts_event *tse = mtxb->q;
3294 do {
3295 switch (tse->uflgs) {
3296 case ETHR_RWMTX_W_WAIT_FLG__:
3297 ws++;
3298 break;
3299 case ETHR_RWMTX_R_WAIT_FLG__:
3300 rs++;
3301 break;
3302 default:
3303 ETHR_MTX_HARD_ASSERT(0);
3304 break;
3305 }
3306 tse = tse->next;
3307 } while (tse != mtxb->q);
3308 }
3309
3310 ETHR_MTX_HARD_ASSERT(mtxb->ws == ws);
3311 if (is_rwmtx) {
3312 ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
3313 ETHR_MTX_HARD_ASSERT(rwmtx->rs == rs);
3314 }
3315 }
3316
3317 #endif
3318
3319 #endif
3320