1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 */
29
30 #ifndef _SYS_UMTXVAR_H_
31 #define _SYS_UMTXVAR_H_
32
33 #ifdef _KERNEL
34
35 #include <sys/_timespec.h>
36
37 /*
38 * The umtx_key structure is used by both the Linux futex code and the
39 * umtx implementation to map userland addresses to unique keys.
40 */
41 enum {
42 TYPE_SIMPLE_WAIT,
43 TYPE_CV,
44 TYPE_SEM,
45 TYPE_SIMPLE_LOCK,
46 TYPE_NORMAL_UMUTEX,
47 TYPE_PI_UMUTEX,
48 TYPE_PP_UMUTEX,
49 TYPE_RWLOCK,
50 TYPE_FUTEX,
51 TYPE_SHM,
52 TYPE_PI_ROBUST_UMUTEX,
53 TYPE_PP_ROBUST_UMUTEX,
54 TYPE_PI_FUTEX,
55 };
56
57 /* Key to represent a unique userland synchronous object */
58 struct umtx_key {
59 int hash;
60 int type;
61 int shared;
62 union {
63 struct {
64 struct vm_object *object;
65 uintptr_t offset;
66 } shared;
67 struct {
68 struct vmspace *vs;
69 uintptr_t addr;
70 } private;
71 struct {
72 void *a;
73 uintptr_t b;
74 } both;
75 } info;
76 };
77
78 #define THREAD_SHARE 0
79 #define PROCESS_SHARE 1
80 #define AUTO_SHARE 2
81
82 struct umtx_abs_timeout {
83 int clockid;
84 bool is_abs_real; /* TIMER_ABSTIME && CLOCK_REALTIME* */
85 struct timespec cur;
86 struct timespec end;
87 };
88
89 struct thread;
90
91 /* Priority inheritance mutex info. */
92 struct umtx_pi {
93 /* Owner thread */
94 struct thread *pi_owner;
95
96 /* Reference count */
97 int pi_refcount;
98
99 /* List entry to link umtx holding by thread */
100 TAILQ_ENTRY(umtx_pi) pi_link;
101
102 /* List entry in hash */
103 TAILQ_ENTRY(umtx_pi) pi_hashlink;
104
105 /* List for waiters */
106 TAILQ_HEAD(,umtx_q) pi_blocked;
107
108 /* Identify a userland lock object */
109 struct umtx_key pi_key;
110 };
111
112 /* A userland synchronous object user. */
113 struct umtx_q {
114 /* Linked list for the hash. */
115 TAILQ_ENTRY(umtx_q) uq_link;
116
117 /* Umtx key. */
118 struct umtx_key uq_key;
119
120 /* Umtx flags. */
121 int uq_flags;
122 #define UQF_UMTXQ 0x0001
123
124 /* Futex bitset mask */
125 u_int uq_bitset;
126
127 /* The thread waits on. */
128 struct thread *uq_thread;
129
130 /*
131 * Blocked on PI mutex. read can use chain lock
132 * or umtx_lock, write must have both chain lock and
133 * umtx_lock being hold.
134 */
135 struct umtx_pi *uq_pi_blocked;
136
137 /* On blocked list */
138 TAILQ_ENTRY(umtx_q) uq_lockq;
139
140 /* Thread contending with us */
141 TAILQ_HEAD(,umtx_pi) uq_pi_contested;
142
143 /* Inherited priority from PP mutex */
144 u_char uq_inherited_pri;
145
146 /* Spare queue ready to be reused */
147 struct umtxq_queue *uq_spare_queue;
148
149 /* The queue we on */
150 struct umtxq_queue *uq_cur_queue;
151 };
152
153 TAILQ_HEAD(umtxq_head, umtx_q);
154
155 /* Per-key wait-queue */
156 struct umtxq_queue {
157 struct umtxq_head head;
158 struct umtx_key key;
159 LIST_ENTRY(umtxq_queue) link;
160 int length;
161 };
162
163 LIST_HEAD(umtxq_list, umtxq_queue);
164
165 /* Userland lock object's wait-queue chain */
166 struct umtxq_chain {
167 /* Lock for this chain. */
168 struct mtx uc_lock;
169
170 /* List of sleep queues. */
171 struct umtxq_list uc_queue[2];
172 #define UMTX_SHARED_QUEUE 0
173 #define UMTX_EXCLUSIVE_QUEUE 1
174
175 LIST_HEAD(, umtxq_queue) uc_spare_queue;
176
177 /* Busy flag */
178 char uc_busy;
179
180 /* Chain lock waiters */
181 int uc_waiters;
182
183 /* All PI in the list */
184 TAILQ_HEAD(,umtx_pi) uc_pi_list;
185
186 #ifdef UMTX_PROFILING
187 u_int length;
188 u_int max_length;
189 #endif
190 };
191
192 static inline int
umtx_key_match(const struct umtx_key * k1,const struct umtx_key * k2)193 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
194 {
195
196 return (k1->type == k2->type &&
197 k1->info.both.a == k2->info.both.a &&
198 k1->info.both.b == k2->info.both.b);
199 }
200
201 void umtx_abs_timeout_init(struct umtx_abs_timeout *, int, int,
202 const struct timespec *);
203 int umtx_copyin_timeout(const void *, struct timespec *);
204 void umtx_exec(struct proc *p);
205 int umtx_key_get(const void *, int, int, struct umtx_key *);
206 void umtx_key_release(struct umtx_key *);
207 struct umtx_q *umtxq_alloc(void);
208 void umtxq_busy(struct umtx_key *);
209 int umtxq_count(struct umtx_key *);
210 void umtxq_free(struct umtx_q *);
211 struct umtxq_chain *umtxq_getchain(struct umtx_key *);
212 void umtxq_insert_queue(struct umtx_q *, int);
213 void umtxq_remove_queue(struct umtx_q *, int);
214 int umtxq_requeue(struct umtx_key *, int, struct umtx_key *, int);
215 int umtxq_signal_mask(struct umtx_key *, int, u_int);
216 int umtxq_sleep(struct umtx_q *, const char *,
217 struct umtx_abs_timeout *);
218 int umtxq_sleep_pi(struct umtx_q *, struct umtx_pi *, uint32_t,
219 const char *, struct umtx_abs_timeout *, bool);
220 void umtxq_unbusy(struct umtx_key *);
221 void umtxq_unbusy_unlocked(struct umtx_key *);
222 int kern_umtx_wake(struct thread *, void *, int, int);
223 void umtx_pi_adjust(struct thread *, u_char);
224 struct umtx_pi *umtx_pi_alloc(int);
225 int umtx_pi_claim(struct umtx_pi *, struct thread *);
226 int umtx_pi_drop(struct thread *, struct umtx_key *, bool, int *);
227 void umtx_pi_free(struct umtx_pi *);
228 void umtx_pi_insert(struct umtx_pi *);
229 struct umtx_pi *umtx_pi_lookup(struct umtx_key *);
230 void umtx_pi_ref(struct umtx_pi *);
231 void umtx_pi_unref(struct umtx_pi *);
232 void umtx_thread_init(struct thread *);
233 void umtx_thread_fini(struct thread *);
234 void umtx_thread_alloc(struct thread *);
235 void umtx_thread_exit(struct thread *);
236
237 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
238 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
239
240 /*
241 * Lock a chain.
242 *
243 * The code is a macro so that file/line information is taken from the caller.
244 */
245 #define umtxq_lock(key) do { \
246 struct umtx_key *_key = (key); \
247 struct umtxq_chain *_uc; \
248 \
249 _uc = umtxq_getchain(_key); \
250 mtx_lock(&_uc->uc_lock); \
251 } while (0)
252
253 /*
254 * Unlock a chain.
255 */
256 static inline void
umtxq_unlock(struct umtx_key * key)257 umtxq_unlock(struct umtx_key *key)
258 {
259 struct umtxq_chain *uc;
260
261 uc = umtxq_getchain(key);
262 mtx_unlock(&uc->uc_lock);
263 }
264
265 #endif /* _KERNEL */
266 #endif /* !_SYS_UMTXVAR_H_ */
267