1 /* 2 * Copyright (c) 2023 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /* 35 * DragonFly implementation of the FreeBSD sleepq*() API. FreeBSD's sleepq*() 36 * API differs from our tsleep*() API in several major ways. 37 * 38 * ONLY USE THIS FOR FREEBSD COMPATIBILITY, E.G. THE LINUX KPI 39 * 40 * - Explicit interlock via sleepq_lock(wchan) 41 * - Single global wchan hash table 42 * - Explicit blocking counts 43 * - Expects all blockers on the same wchan to be of the same type 44 * 45 * In addition, the API calls have a ton of unnecessary redundancy which 46 * creates problems with using on-stack structures for things. To make 47 * ends meat, DragonFly records information in the thread structure and 48 * only enacts the actual operation when the thread issues a *wait*() API 49 * call. Only the spin interlock is handled in real-time. 50 */ 51 52 #include "opt_ddb.h" 53 54 #include <sys/cdefs.h> 55 #include <sys/param.h> 56 #include <sys/systm.h> 57 #include <sys/lock.h> 58 #include <sys/caps.h> 59 #include <sys/malloc.h> 60 #include <sys/queue.h> 61 #include <sys/sleepqueue.h> 62 #include <sys/objcache.h> 63 #include <sys/sysctl.h> 64 65 #include <sys/time.h> 66 67 #include <machine/atomic.h> 68 69 #ifdef DDB 70 #include <ddb/ddb.h> 71 #endif 72 73 #include <sys/signal2.h> 74 #include <sys/thread2.h> 75 #include <sys/spinlock2.h> 76 #include <sys/mutex2.h> 77 78 #include <vm/vm_extern.h> 79 80 #define SLEEPQ_HSIZE 1024 81 #define SLEEPQ_HMASK (SLEEPQ_HSIZE - 1) 82 #define SLEEPQ_HASH(wchan) ((((uintptr_t)(wchan) >> 10) ^ \ 83 ((uintptr_t)(wchan) & SLEEPQ_HMASK))) 84 85 #define SLEEPQ_LOOKUP(wchan) &sleepq_chains[SLEEPQ_HASH(wchan)] 86 #define SLEEPQ_NRQUEUES 2 87 #define SLEEPQ_FREEPERSLOT 4 88 89 struct sleepqueue_wchan; 90 91 struct sleepqueue_chain { 92 struct spinlock sc_spin; 93 TAILQ_HEAD(, sleepqueue_wchan) sc_wchead; 94 u_int sc_free_count; 95 }; 96 97 struct sleepqueue_wchan { 98 TAILQ_ENTRY(sleepqueue_wchan) wc_entry; 99 const void *wc_wchan; 100 struct sleepqueue_chain *wc_sc; 101 u_int wc_refs; 102 int wc_type; 103 u_int wc_blocked[SLEEPQ_NRQUEUES]; 104 }; 105 106 static struct sleepqueue_chain sleepq_chains[SLEEPQ_HSIZE]; 107 static MALLOC_DEFINE(M_SLEEPQ, "sleepq", "fbsd sleepq api"); 108 static struct objcache *sleepq_wc_cache; 109 110 /* 111 * Return existing wchan, assert not NULL. 112 */ 113 static __inline 114 struct sleepqueue_wchan * 115 sleepq_wclookup(const void *wchan) 116 { 117 struct sleepqueue_chain *sc; 118 struct sleepqueue_wchan *wc; 119 120 sc = SLEEPQ_LOOKUP(wchan); 121 KKASSERT(spin_held(&sc->sc_spin)); 122 TAILQ_FOREACH(wc, &sc->sc_wchead, wc_entry) { 123 if (wc->wc_wchan == wchan) 124 return(wc); 125 } 126 panic("sleepq_wclookup: wchan %p not found\n", wc); 127 128 return NULL; /* not reached */ 129 } 130 131 /* 132 * Early initialization of sleep queues 133 */ 134 static void 135 init_sleepqueues(void) 136 { 137 int i; 138 139 for (i = 0; i < SLEEPQ_HSIZE; ++i) { 140 spin_init(&sleepq_chains[i].sc_spin, "sleepq"); 141 TAILQ_INIT(&sleepq_chains[i].sc_wchead); 142 } 143 /*thread0.td_sleepqueue = sleepq_alloc();*/ 144 145 sleepq_wc_cache = objcache_create_simple(M_SLEEPQ, 146 sizeof(struct sleepqueue_wchan)); 147 148 } 149 SYSINIT(sysinit_sleepqueues, SI_BOOT2_LWKT_INIT, SI_ORDER_SECOND, 150 init_sleepqueues, NULL); 151 152 /* 153 * DragonFlyBSD Interface Functions 154 * 155 * Setup and teardown sleepq related thread structures 156 * 157 * sleepq_alloc() - not applicable 158 * sleepq_free() - not applicable 159 */ 160 void 161 sleepq_setup_thread(struct thread *td) 162 { 163 } 164 165 void 166 sleepq_teardown_thread(struct thread *td) 167 { 168 } 169 170 /* 171 * Lock the wchan, creating the structure if necessary. Returns with 172 * the spin-lock held. 173 * 174 * Also used as an interlock, so we need to actually lock something here. 175 */ 176 void 177 sleepq_lock(const void *wchan) 178 { 179 struct sleepqueue_chain *sc; 180 struct sleepqueue_wchan *wc; 181 182 sc = SLEEPQ_LOOKUP(wchan); 183 spin_lock(&sc->sc_spin); 184 185 for (;;) { 186 /* 187 * Look for the wchan, if not found then allocate one from 188 * the existing in-list cache if available. 189 */ 190 TAILQ_FOREACH(wc, &sc->sc_wchead, wc_entry) { 191 if (wc->wc_wchan == wchan) { 192 ++wc->wc_refs; 193 return; 194 } 195 if (wc->wc_wchan == NULL) { 196 wc->wc_wchan = wchan; 197 ++wc->wc_refs; 198 --sc->sc_free_count; 199 return; 200 } 201 } 202 203 /* 204 * Not found and no free entries available, allocate 205 * a new, free wc, then relock and repeat the search. 206 */ 207 spin_unlock(&sc->sc_spin); 208 wc = objcache_get(sleepq_wc_cache, M_WAITOK); 209 KKASSERT(wc->wc_wchan == NULL && wc->wc_refs == 0); 210 wc->wc_sc = sc; 211 spin_lock(&sc->sc_spin); 212 TAILQ_INSERT_TAIL(&sc->sc_wchead, wc, wc_entry); 213 ++sc->sc_free_count; 214 /* loop re-search */ 215 } 216 } 217 218 /* 219 * Unlock the sleep queue chain associated with a given wait channel. 220 */ 221 void 222 sleepq_release(const void *wchan) 223 { 224 struct sleepqueue_chain *sc; 225 struct sleepqueue_wchan *wc; 226 227 wc = sleepq_wclookup(wchan); 228 sc = wc->wc_sc; 229 KKASSERT(wc->wc_refs > 0); 230 if (--wc->wc_refs == 0) { 231 /* just sanity-check one for brevity */ 232 KKASSERT(wc->wc_blocked[0] == 0); 233 wc->wc_wchan = NULL; 234 wc->wc_type = 0; 235 ++sc->sc_free_count; 236 } 237 238 spin_unlock(&sc->sc_spin); 239 } 240 241 /* 242 * Place the current thread on the specified wait channel and sub-queue. 243 * 244 * The caller must be holding the wchan lock and it will remain locked 245 * on return. The caller must follow-up with a sleepq_*wait*() call. 246 * 247 * If INVARIANTS is enabled, then it associates the passed in lock with 248 * the sleepq to make sure it is held when that sleep queue is woken up 249 * (XXX not implemented) 250 */ 251 void 252 sleepq_add(const void *wchan, struct lock_object *lock, const char *wmesg, 253 int flags, int queue) 254 { 255 struct sleepqueue_wchan *wc; 256 struct thread *td; 257 int domain; 258 259 /* 260 * Locate the wc (also asserts that the lock is held). Bump 261 * wc->wc_refs and wc->wc_blocked[queue] to indicate that the 262 * thread has been placed on the sleepq. 263 */ 264 wc = sleepq_wclookup(wchan); 265 ++wc->wc_refs; 266 ++wc->wc_blocked[queue]; 267 wc->wc_type = flags & SLEEPQ_TYPE; 268 269 /* 270 * tsleep_interlock() sets TDF_TSLEEPQ, sets td_wchan, and td_wdomain, 271 * and places the thread on the appropriate sleepq. 272 */ 273 td = curthread; 274 td->td_wmesg = wmesg; 275 domain = PDOMAIN_FBSD0 + queue * PDOMAIN_FBSDINC; 276 tsleep_interlock(wchan, domain); 277 278 /* 279 * Clear timeout to discern a possible later sleepq_set_timeout_sbt() 280 * call. 281 */ 282 td->td_sqwc = wc; 283 td->td_sqtimo = 0; 284 td->td_sqqueue = queue; 285 } 286 287 /* 288 * Set a timeout for the thread after it has been added to a wait channel. 289 */ 290 void 291 sleepq_set_timeout_sbt(const void *wchan, sbintime_t sbt, sbintime_t pr, 292 int flags) 293 { 294 struct sleepqueue_wchan *wc; 295 struct thread *td; 296 297 td = curthread; 298 wc = td->td_sqwc; 299 KKASSERT(wc && wc->wc_wchan == wchan); 300 td->td_sqtimo = sbticks + sbt; 301 } 302 303 /* 304 * Return the number of actual sleepers for the specified queue. 305 * 306 * Caller must be holding the wchan locked 307 */ 308 u_int 309 sleepq_sleepcnt(const void *wchan, int queue) 310 { 311 struct sleepqueue_wchan *wc; 312 313 wc = sleepq_wclookup(wchan); 314 return (wc->wc_blocked[queue]); 315 } 316 317 static __inline 318 int 319 _sleepq_wait_begin(struct thread *td, struct sleepqueue_chain *sc, 320 struct sleepqueue_wchan *wc, sbintime_t timo, int tflags) 321 { 322 int domain; 323 int ret; 324 325 spin_unlock(&sc->sc_spin); 326 327 domain = PDOMAIN_FBSD0 + td->td_sqqueue * PDOMAIN_FBSDINC; 328 if (timo) { 329 timo -= sbticks; 330 if (timo > 0) { 331 ret = tsleep(td->td_wchan, tflags, td->td_wmesg, timo); 332 } else { 333 ret = EWOULDBLOCK; 334 } 335 } else { 336 ret = tsleep(td->td_wchan, tflags, td->td_wmesg, 0); 337 } 338 339 return ret; 340 } 341 342 /* 343 * Wait for completion 344 */ 345 static __inline 346 void 347 _sleepq_wait_complete(struct thread *td, struct sleepqueue_chain *sc, 348 struct sleepqueue_wchan *wc) 349 { 350 struct sleepqueue_wchan *wcn; 351 352 td->td_sqwc = NULL; 353 spin_lock(&sc->sc_spin); 354 --wc->wc_blocked[td->td_sqqueue]; 355 if (--wc->wc_refs == 0) { 356 wc->wc_wchan = NULL; 357 wc->wc_type = 0; 358 ++sc->sc_free_count; 359 if (sc->sc_free_count <= SLEEPQ_FREEPERSLOT) { 360 wcn = TAILQ_NEXT(wc, wc_entry); 361 if (wcn && wcn->wc_wchan) { 362 TAILQ_REMOVE(&sc->sc_wchead, wc, wc_entry); 363 TAILQ_INSERT_TAIL(&sc->sc_wchead, wc, wc_entry); 364 } 365 spin_unlock(&sc->sc_spin); 366 } else { 367 --sc->sc_free_count; 368 TAILQ_REMOVE(&sc->sc_wchead, wc, wc_entry); 369 spin_unlock(&sc->sc_spin); 370 objcache_put(sleepq_wc_cache, wc); 371 } 372 } else { 373 spin_unlock(&sc->sc_spin); 374 } 375 } 376 377 /* 378 * Various combinations of wait until unblocked, with and without 379 * signaling and timeouts. 380 * 381 * The wchan lock must be held by the caller and will be released upon 382 * return. 383 * 384 * NOTE: tsleep_interlock() was already issued by sleepq_add(). 385 */ 386 void 387 sleepq_wait(const void *wchan, int pri) 388 { 389 struct sleepqueue_chain *sc; 390 struct sleepqueue_wchan *wc; 391 struct thread *td; 392 393 td = curthread; 394 wc = td->td_sqwc; 395 KKASSERT(wc != NULL && wc->wc_wchan == wchan); 396 sc = wc->wc_sc; 397 398 (void)_sleepq_wait_begin(td, sc, wc, 0, PINTERLOCKED); 399 _sleepq_wait_complete(td, sc, wc); 400 } 401 402 int 403 sleepq_wait_sig(const void *wchan, int pri) 404 { 405 struct sleepqueue_chain *sc; 406 struct sleepqueue_wchan *wc; 407 struct thread *td; 408 int ret; 409 410 td = curthread; 411 wc = td->td_sqwc; 412 KKASSERT(wc != NULL && wc->wc_wchan == wchan); 413 sc = wc->wc_sc; 414 415 ret = _sleepq_wait_begin(td, sc, wc, 0, PINTERLOCKED | PCATCH); 416 _sleepq_wait_complete(td, sc, wc); 417 418 return ret; 419 } 420 421 int 422 sleepq_timedwait(const void *wchan, int pri) 423 { 424 struct sleepqueue_chain *sc; 425 struct sleepqueue_wchan *wc; 426 struct thread *td; 427 int ret; 428 429 td = curthread; 430 wc = td->td_sqwc; 431 KKASSERT(wc != NULL && wc->wc_wchan == wchan); 432 sc = wc->wc_sc; 433 434 ret = _sleepq_wait_begin(td, sc, wc, td->td_sqtimo, PINTERLOCKED); 435 _sleepq_wait_complete(td, sc, wc); 436 437 return ret; 438 } 439 440 int 441 sleepq_timedwait_sig(const void *wchan, int pri) 442 { 443 struct sleepqueue_chain *sc; 444 struct sleepqueue_wchan *wc; 445 struct thread *td; 446 int ret; 447 448 td = curthread; 449 wc = td->td_sqwc; 450 KKASSERT(wc != NULL && wc->wc_wchan == wchan); 451 sc = wc->wc_sc; 452 453 ret = _sleepq_wait_begin(td, sc, wc, td->td_sqtimo, 454 PINTERLOCKED | PCATCH); 455 _sleepq_wait_complete(td, sc, wc); 456 457 return ret; 458 } 459 460 /* 461 * Returns the type of sleepqueue given a waitchannel. Returns 0 if 462 * sleepq_add() has not been called no the wchan. 463 * 464 * wchan must be locked. 465 */ 466 int 467 sleepq_type(const void *wchan) 468 { 469 struct sleepqueue_wchan *wc; 470 int type; 471 472 wc = sleepq_wclookup(wchan); 473 type = wc->wc_type; 474 475 return type; 476 } 477 478 /* 479 * Wakeup one thread sleeping on a wait channel. 480 * 481 * DragonFly: Presumably callers also deal with multiple wakeups, 482 * our wakeup_domaoin_one() function is a bit non-deterministic. 483 * 484 * Nominally returns whether the swapper should be woken up which 485 * is not applicable to DragonFlyBSD. 486 */ 487 int 488 sleepq_signal(const void *wchan, int flags, int pri, int queue) 489 { 490 int domain; 491 492 domain = PDOMAIN_FBSD0 + queue * PDOMAIN_FBSDINC; 493 wakeup_domain_one(wchan, domain); 494 495 return 0; 496 } 497 498 /* 499 * Resume everything sleeping on the specified wait channel and queue. 500 * 501 * Nominally returns whether the swapper should be woken up which is not 502 * applicable to DragonFlyBSD. 503 */ 504 int 505 sleepq_broadcast(const void *wchan, int flags, int pri, int queue) 506 { 507 int domain; 508 509 domain = PDOMAIN_FBSD0 + queue * PDOMAIN_FBSDINC; 510 wakeup_domain(wchan, domain); 511 512 return 0; 513 } 514