1 /* $NetBSD: locks_up.c,v 1.12 2023/04/12 06:35:40 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2010 Antti Kantee. All Rights Reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 /*
29 * Virtual uniprocessor rump kernel version of locks. Since the entire
30 * kernel is running on only one CPU in the system, there is no need
31 * to perform slow cache-coherent MP locking operations. This speeds
32 * up things quite dramatically and is a good example of that two
33 * disjoint kernels running simultaneously in an MP system can be
34 * massively faster than one with fine-grained locking.
35 */
36
37 #include <sys/cdefs.h>
38 __KERNEL_RCSID(0, "$NetBSD: locks_up.c,v 1.12 2023/04/12 06:35:40 riastradh Exp $");
39
40 #include <sys/param.h>
41 #include <sys/kernel.h>
42 #include <sys/kmem.h>
43 #include <sys/mutex.h>
44 #include <sys/rwlock.h>
45
46 #include <rump-sys/kern.h>
47
48 #include <rump/rumpuser.h>
49
50 struct upmtx {
51 struct lwp *upm_owner;
52 int upm_wanted;
53 struct rumpuser_cv *upm_rucv;
54 };
55 #define UPMTX(mtx) struct upmtx *upm = *(struct upmtx **)mtx
56
57 static inline void
checkncpu(void)58 checkncpu(void)
59 {
60
61 if (__predict_false(ncpu != 1))
62 panic("UP lock implementation requires RUMP_NCPU == 1");
63 }
64
65 void
mutex_init(kmutex_t * mtx,kmutex_type_t type,int ipl)66 mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl)
67 {
68 struct upmtx *upm;
69
70 CTASSERT(sizeof(kmutex_t) >= sizeof(void *));
71 checkncpu();
72
73 /*
74 * In uniprocessor locking we don't need to differentiate
75 * between spin mutexes and adaptive ones. We could
76 * replace mutex_enter() with a NOP for spin mutexes, but
77 * not bothering with that for now.
78 */
79
80 /*
81 * XXX: pool_cache would be nice, but not easily possible,
82 * as pool cache init wants to call mutex_init() ...
83 */
84 upm = rump_hypermalloc(sizeof(*upm), 0, true, "mutex_init");
85 memset(upm, 0, sizeof(*upm));
86 rumpuser_cv_init(&upm->upm_rucv);
87 memcpy(mtx, &upm, sizeof(void *));
88 }
89
90 void
mutex_destroy(kmutex_t * mtx)91 mutex_destroy(kmutex_t *mtx)
92 {
93 UPMTX(mtx);
94
95 KASSERT(upm->upm_owner == NULL);
96 KASSERT(upm->upm_wanted == 0);
97 rumpuser_cv_destroy(upm->upm_rucv);
98 rump_hyperfree(upm, sizeof(*upm));
99 }
100
101 void
mutex_enter(kmutex_t * mtx)102 mutex_enter(kmutex_t *mtx)
103 {
104 UPMTX(mtx);
105
106 /* fastpath? */
107 if (mutex_tryenter(mtx))
108 return;
109
110 /*
111 * No? bummer, do it the slow and painful way then.
112 */
113 upm->upm_wanted++;
114 while (!mutex_tryenter(mtx)) {
115 rump_schedlock_cv_wait(upm->upm_rucv);
116 }
117 upm->upm_wanted--;
118
119 KASSERT(upm->upm_wanted >= 0);
120 }
121
122 void
mutex_spin_enter(kmutex_t * mtx)123 mutex_spin_enter(kmutex_t *mtx)
124 {
125
126 mutex_enter(mtx);
127 }
128
129 int
mutex_tryenter(kmutex_t * mtx)130 mutex_tryenter(kmutex_t *mtx)
131 {
132 UPMTX(mtx);
133
134 if (upm->upm_owner)
135 return 0;
136
137 upm->upm_owner = curlwp;
138 return 1;
139 }
140
141 void
mutex_exit(kmutex_t * mtx)142 mutex_exit(kmutex_t *mtx)
143 {
144 UPMTX(mtx);
145
146 if (upm->upm_wanted) {
147 rumpuser_cv_signal(upm->upm_rucv); /* CPU is our interlock */
148 }
149 upm->upm_owner = NULL;
150 }
151
152 void
mutex_spin_exit(kmutex_t * mtx)153 mutex_spin_exit(kmutex_t *mtx)
154 {
155
156 mutex_exit(mtx);
157 }
158
159 int
mutex_owned(kmutex_t * mtx)160 mutex_owned(kmutex_t *mtx)
161 {
162 UPMTX(mtx);
163
164 return upm->upm_owner == curlwp;
165 }
166
167 struct uprw {
168 struct lwp *uprw_owner;
169 int uprw_readers;
170 uint16_t uprw_rwant;
171 uint16_t uprw_wwant;
172 struct rumpuser_cv *uprw_rucv_reader;
173 struct rumpuser_cv *uprw_rucv_writer;
174 };
175
176 #define UPRW(rw) struct uprw *uprw = *(struct uprw **)rw
177
178 /* reader/writer locks */
179
180 void
rw_init(krwlock_t * rw)181 rw_init(krwlock_t *rw)
182 {
183 struct uprw *uprw;
184
185 CTASSERT(sizeof(krwlock_t) >= sizeof(void *));
186 checkncpu();
187
188 uprw = rump_hypermalloc(sizeof(*uprw), 0, true, "rwinit");
189 memset(uprw, 0, sizeof(*uprw));
190 rumpuser_cv_init(&uprw->uprw_rucv_reader);
191 rumpuser_cv_init(&uprw->uprw_rucv_writer);
192 memcpy(rw, &uprw, sizeof(void *));
193 }
194
195 void
rw_destroy(krwlock_t * rw)196 rw_destroy(krwlock_t *rw)
197 {
198 UPRW(rw);
199
200 rumpuser_cv_destroy(uprw->uprw_rucv_reader);
201 rumpuser_cv_destroy(uprw->uprw_rucv_writer);
202 rump_hyperfree(uprw, sizeof(*uprw));
203 }
204
205 /* take rwlock. prefer writers over readers (see rw_tryenter and rw_exit) */
206 void
rw_enter(krwlock_t * rw,const krw_t op)207 rw_enter(krwlock_t *rw, const krw_t op)
208 {
209 UPRW(rw);
210 struct rumpuser_cv *rucv;
211 uint16_t *wp;
212
213 if (rw_tryenter(rw, op))
214 return;
215
216 /* lagpath */
217 if (op == RW_READER) {
218 rucv = uprw->uprw_rucv_reader;
219 wp = &uprw->uprw_rwant;
220 } else {
221 rucv = uprw->uprw_rucv_writer;
222 wp = &uprw->uprw_wwant;
223 }
224
225 (*wp)++;
226 while (!rw_tryenter(rw, op)) {
227 rump_schedlock_cv_wait(rucv);
228 }
229 (*wp)--;
230 }
231
232 int
rw_tryenter(krwlock_t * rw,const krw_t op)233 rw_tryenter(krwlock_t *rw, const krw_t op)
234 {
235 UPRW(rw);
236
237 switch (op) {
238 case RW_READER:
239 if (uprw->uprw_owner == NULL && uprw->uprw_wwant == 0) {
240 uprw->uprw_readers++;
241 return 1;
242 }
243 break;
244 case RW_WRITER:
245 if (uprw->uprw_owner == NULL && uprw->uprw_readers == 0) {
246 uprw->uprw_owner = curlwp;
247 return 1;
248 }
249 break;
250 }
251
252 return 0;
253 }
254
255 void
rw_exit(krwlock_t * rw)256 rw_exit(krwlock_t *rw)
257 {
258 UPRW(rw);
259
260 if (uprw->uprw_readers > 0) {
261 uprw->uprw_readers--;
262 } else {
263 KASSERT(uprw->uprw_owner == curlwp);
264 uprw->uprw_owner = NULL;
265 }
266
267 if (uprw->uprw_wwant) {
268 rumpuser_cv_signal(uprw->uprw_rucv_writer);
269 } else if (uprw->uprw_rwant) {
270 rumpuser_cv_signal(uprw->uprw_rucv_reader);
271 }
272 }
273
274 int
rw_tryupgrade(krwlock_t * rw)275 rw_tryupgrade(krwlock_t *rw)
276 {
277 UPRW(rw);
278
279 if (uprw->uprw_readers == 1 && uprw->uprw_owner == NULL) {
280 uprw->uprw_readers = 0;
281 uprw->uprw_owner = curlwp;
282 return 1;
283 } else {
284 return 0;
285 }
286 }
287
288 int
rw_write_held(krwlock_t * rw)289 rw_write_held(krwlock_t *rw)
290 {
291 UPRW(rw);
292
293 return uprw->uprw_owner == curlwp;
294 }
295
296 int
rw_read_held(krwlock_t * rw)297 rw_read_held(krwlock_t *rw)
298 {
299 UPRW(rw);
300
301 return uprw->uprw_readers > 0;
302 }
303
304 int
rw_lock_held(krwlock_t * rw)305 rw_lock_held(krwlock_t *rw)
306 {
307 UPRW(rw);
308
309 return uprw->uprw_owner || uprw->uprw_readers;
310 }
311
312 krw_t
rw_lock_op(krwlock_t * rw)313 rw_lock_op(krwlock_t *rw)
314 {
315
316 return rw_write_held(rw) ? RW_WRITER : RW_READER;
317 }
318
319 /*
320 * Condvars are almost the same as in the MP case except that we
321 * use the scheduler mutex as the pthread interlock instead of the
322 * mutex associated with the condvar.
323 */
324
325 #define RUMPCV(cv) (*(struct rumpuser_cv **)(cv))
326
327 void
cv_init(kcondvar_t * cv,const char * msg)328 cv_init(kcondvar_t *cv, const char *msg)
329 {
330
331 CTASSERT(sizeof(kcondvar_t) >= sizeof(void *));
332 checkncpu();
333
334 rumpuser_cv_init((struct rumpuser_cv **)cv);
335 }
336
337 void
cv_destroy(kcondvar_t * cv)338 cv_destroy(kcondvar_t *cv)
339 {
340
341 rumpuser_cv_destroy(RUMPCV(cv));
342 }
343
344 void
cv_wait(kcondvar_t * cv,kmutex_t * mtx)345 cv_wait(kcondvar_t *cv, kmutex_t *mtx)
346 {
347 #ifdef DIAGNOSTIC
348 UPMTX(mtx);
349 KASSERT(upm->upm_owner == curlwp);
350
351 if (rump_threads == 0)
352 panic("cv_wait without threads");
353 #endif
354
355 /*
356 * NOTE: we must atomically release the *CPU* here, i.e.
357 * nothing between mutex_exit and entering rumpuser condwait
358 * may preempt us from the virtual CPU.
359 */
360 mutex_exit(mtx);
361 rump_schedlock_cv_wait(RUMPCV(cv));
362 mutex_enter(mtx);
363 }
364
365 int
cv_wait_sig(kcondvar_t * cv,kmutex_t * mtx)366 cv_wait_sig(kcondvar_t *cv, kmutex_t *mtx)
367 {
368
369 cv_wait(cv, mtx);
370 return 0;
371 }
372
373 int
cv_timedwait(kcondvar_t * cv,kmutex_t * mtx,int ticks)374 cv_timedwait(kcondvar_t *cv, kmutex_t *mtx, int ticks)
375 {
376 struct timespec ts;
377
378 #ifdef DIAGNOSTIC
379 UPMTX(mtx);
380 KASSERT(upm->upm_owner == curlwp);
381 #endif
382
383 ts.tv_sec = ticks / hz;
384 ts.tv_nsec = (ticks % hz) * (1000000000/hz);
385
386 if (ticks == 0) {
387 cv_wait(cv, mtx);
388 return 0;
389 } else {
390 int rv;
391 mutex_exit(mtx);
392 rv = rump_schedlock_cv_timedwait(RUMPCV(cv), &ts);
393 mutex_enter(mtx);
394 if (rv)
395 return EWOULDBLOCK;
396 else
397 return 0;
398 }
399 }
400
401 int
cv_timedwait_sig(kcondvar_t * cv,kmutex_t * mtx,int ticks)402 cv_timedwait_sig(kcondvar_t *cv, kmutex_t *mtx, int ticks)
403 {
404
405 return cv_timedwait(cv, mtx, ticks);
406 }
407
408 void
cv_signal(kcondvar_t * cv)409 cv_signal(kcondvar_t *cv)
410 {
411
412 /* CPU == interlock */
413 rumpuser_cv_signal(RUMPCV(cv));
414 }
415
416 void
cv_broadcast(kcondvar_t * cv)417 cv_broadcast(kcondvar_t *cv)
418 {
419
420 /* CPU == interlock */
421 rumpuser_cv_broadcast(RUMPCV(cv));
422 }
423
424 bool
cv_has_waiters(kcondvar_t * cv)425 cv_has_waiters(kcondvar_t *cv)
426 {
427 int n;
428
429 rumpuser_cv_has_waiters(RUMPCV(cv), &n);
430
431 return n > 0;
432 }
433
434 /* this is not much of an attempt, but ... */
435 bool
cv_is_valid(kcondvar_t * cv)436 cv_is_valid(kcondvar_t *cv)
437 {
438
439 return RUMPCV(cv) != NULL;
440 }
441