1 /* $NetBSD: sleepq.h,v 1.24 2015/02/08 19:39:09 christos Exp $ */
2
3 /*-
4 * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #ifndef _SYS_SLEEPQ_H_
33 #define _SYS_SLEEPQ_H_
34
35 #include <sys/lwp.h>
36 #include <sys/mutex.h>
37 #include <sys/pool.h>
38 #include <sys/queue.h>
39 #include <sys/sched.h>
40 #include <sys/syncobj.h>
41
42 /*
43 * Generic sleep queues.
44 */
45
46 #define SLEEPTAB_HASH_SHIFT 7
47 #define SLEEPTAB_HASH_SIZE (1 << SLEEPTAB_HASH_SHIFT)
48 #define SLEEPTAB_HASH_MASK (SLEEPTAB_HASH_SIZE - 1)
49 #define SLEEPTAB_HASH(wchan) (((uintptr_t)(wchan) >> 8) & SLEEPTAB_HASH_MASK)
50
51 TAILQ_HEAD(sleepq, lwp);
52
53 typedef struct sleepq sleepq_t;
54
55 typedef struct sleeptab {
56 struct {
57 kmutex_t *st_mutex;
58 sleepq_t st_queue;
59 } st_queues[SLEEPTAB_HASH_SIZE];
60 } sleeptab_t;
61
62 void sleepq_init(sleepq_t *);
63 void sleepq_remove(sleepq_t *, lwp_t *);
64 void sleepq_enqueue(sleepq_t *, wchan_t, const char *, struct syncobj *);
65 void sleepq_unsleep(lwp_t *, bool);
66 void sleepq_timeout(void *);
67 void sleepq_wake(sleepq_t *, wchan_t, u_int, kmutex_t *);
68 int sleepq_abort(kmutex_t *, int);
69 void sleepq_changepri(lwp_t *, pri_t);
70 void sleepq_lendpri(lwp_t *, pri_t);
71 int sleepq_block(int, bool);
72
73 void sleeptab_init(sleeptab_t *);
74
75 extern sleeptab_t sleeptab;
76
77 #ifdef _KERNEL
78 /*
79 * Return non-zero if it is unsafe to sleep.
80 *
81 * XXX This only exists because panic() is broken.
82 */
83 static inline bool
sleepq_dontsleep(lwp_t * l)84 sleepq_dontsleep(lwp_t *l)
85 {
86 extern int cold;
87
88 return cold || (doing_shutdown && (panicstr || CURCPU_IDLE_P()));
89 }
90
91 /*
92 * Find the correct sleep queue for the specified wait channel. This
93 * acquires and holds the per-queue interlock.
94 */
95 static inline sleepq_t *
sleeptab_lookup(sleeptab_t * st,wchan_t wchan,kmutex_t ** mp)96 sleeptab_lookup(sleeptab_t *st, wchan_t wchan, kmutex_t **mp)
97 {
98 sleepq_t *sq;
99
100 sq = &st->st_queues[SLEEPTAB_HASH(wchan)].st_queue;
101 *mp = st->st_queues[SLEEPTAB_HASH(wchan)].st_mutex;
102 mutex_spin_enter(*mp);
103 return sq;
104 }
105
106 static inline kmutex_t *
sleepq_hashlock(wchan_t wchan)107 sleepq_hashlock(wchan_t wchan)
108 {
109 kmutex_t *mp;
110
111 mp = sleeptab.st_queues[SLEEPTAB_HASH(wchan)].st_mutex;
112 mutex_spin_enter(mp);
113 return mp;
114 }
115
116 /*
117 * Prepare to block on a sleep queue, after which any interlock can be
118 * safely released.
119 */
120 static inline void
sleepq_enter(sleepq_t * sq,lwp_t * l,kmutex_t * mp)121 sleepq_enter(sleepq_t *sq, lwp_t *l, kmutex_t *mp)
122 {
123
124 /*
125 * Acquire the per-LWP mutex and lend it ours sleep queue lock.
126 * Once interlocked, we can release the kernel lock.
127 */
128 lwp_lock(l);
129 lwp_unlock_to(l, mp);
130 KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks);
131 }
132 #endif
133
134 /*
135 * Turnstiles, specialized sleep queues for use by kernel locks.
136 */
137
138 typedef struct turnstile {
139 LIST_ENTRY(turnstile) ts_chain; /* link on hash chain */
140 struct turnstile *ts_free; /* turnstile free list */
141 wchan_t ts_obj; /* lock object */
142 sleepq_t ts_sleepq[2]; /* sleep queues */
143 u_int ts_waiters[2]; /* count of waiters */
144
145 /* priority inheritance */
146 pri_t ts_eprio;
147 lwp_t *ts_inheritor;
148 SLIST_ENTRY(turnstile) ts_pichain;
149 } turnstile_t;
150
151 typedef struct tschain {
152 kmutex_t *tc_mutex; /* mutex on structs & queues */
153 LIST_HEAD(, turnstile) tc_chain; /* turnstile chain */
154 } tschain_t;
155
156 #define TS_READER_Q 0 /* reader sleep queue */
157 #define TS_WRITER_Q 1 /* writer sleep queue */
158
159 #define TS_WAITERS(ts, q) \
160 (ts)->ts_waiters[(q)]
161
162 #define TS_ALL_WAITERS(ts) \
163 ((ts)->ts_waiters[TS_READER_Q] + \
164 (ts)->ts_waiters[TS_WRITER_Q])
165
166 #define TS_FIRST(ts, q) (TAILQ_FIRST(&(ts)->ts_sleepq[(q)]))
167
168 #ifdef _KERNEL
169
170 void turnstile_init(void);
171 turnstile_t *turnstile_lookup(wchan_t);
172 void turnstile_exit(wchan_t);
173 void turnstile_block(turnstile_t *, int, wchan_t, syncobj_t *);
174 void turnstile_wakeup(turnstile_t *, int, int, lwp_t *);
175 void turnstile_print(volatile void *, void (*)(const char *, ...)
176 __printflike(1, 2));
177 void turnstile_unsleep(lwp_t *, bool);
178 void turnstile_changepri(lwp_t *, pri_t);
179
180 extern pool_cache_t turnstile_cache;
181 extern turnstile_t turnstile0;
182
183 #endif /* _KERNEL */
184
185 #endif /* _SYS_SLEEPQ_H_ */
186