1 /* $NetBSD: locks_up.c,v 1.5 2010/12/01 17:22:51 pooka Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Antti Kantee. All Rights Reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* 29 * Virtual uniprocessor rump kernel version of locks. Since the entire 30 * kernel is running on only one CPU in the system, there is no need 31 * to perform slow cache-coherent MP locking operations. This speeds 32 * up things quite dramatically and is a good example of that two 33 * disjoint kernels running simultaneously in an MP system can be 34 * massively faster than one with fine-grained locking. 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: locks_up.c,v 1.5 2010/12/01 17:22:51 pooka Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/kernel.h> 42 #include <sys/kmem.h> 43 #include <sys/mutex.h> 44 #include <sys/rwlock.h> 45 46 #include <rump/rumpuser.h> 47 48 #include "rump_private.h" 49 50 struct upmtx { 51 struct lwp *upm_owner; 52 int upm_wanted; 53 struct rumpuser_cv *upm_rucv; 54 }; 55 #define UPMTX(mtx) struct upmtx *upm = *(struct upmtx **)mtx 56 57 static inline void 58 checkncpu(void) 59 { 60 61 if (__predict_false(ncpu != 1)) 62 panic("UP lock implementation requires RUMP_NCPU == 1"); 63 } 64 65 void 66 mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) 67 { 68 struct upmtx *upm; 69 70 CTASSERT(sizeof(kmutex_t) >= sizeof(void *)); 71 checkncpu(); 72 73 /* 74 * XXX: pool_cache would be nice, but not easily possible, 75 * as pool cache init wants to call mutex_init() ... 76 */ 77 upm = rump_hypermalloc(sizeof(*upm), 0, true, "mutex_init"); 78 memset(upm, 0, sizeof(*upm)); 79 rumpuser_cv_init(&upm->upm_rucv); 80 memcpy(mtx, &upm, sizeof(void *)); 81 } 82 83 void 84 mutex_destroy(kmutex_t *mtx) 85 { 86 UPMTX(mtx); 87 88 KASSERT(upm->upm_owner == NULL); 89 KASSERT(upm->upm_wanted == 0); 90 rumpuser_cv_destroy(upm->upm_rucv); 91 rump_hyperfree(upm, sizeof(*upm)); 92 } 93 94 void 95 mutex_enter(kmutex_t *mtx) 96 { 97 UPMTX(mtx); 98 99 /* fastpath? */ 100 if (mutex_tryenter(mtx)) 101 return; 102 103 /* 104 * No? bummer, do it the slow and painful way then. 105 */ 106 upm->upm_wanted++; 107 while (!mutex_tryenter(mtx)) { 108 rump_schedlock_cv_wait(upm->upm_rucv); 109 } 110 upm->upm_wanted--; 111 112 KASSERT(upm->upm_wanted >= 0); 113 } 114 115 void 116 mutex_spin_enter(kmutex_t *mtx) 117 { 118 119 mutex_enter(mtx); 120 } 121 122 int 123 mutex_tryenter(kmutex_t *mtx) 124 { 125 UPMTX(mtx); 126 127 if (upm->upm_owner) 128 return 0; 129 130 upm->upm_owner = curlwp; 131 return 1; 132 } 133 134 void 135 mutex_exit(kmutex_t *mtx) 136 { 137 UPMTX(mtx); 138 139 if (upm->upm_wanted) { 140 rumpuser_cv_signal(upm->upm_rucv); /* CPU is our interlock */ 141 } 142 upm->upm_owner = NULL; 143 } 144 145 void 146 mutex_spin_exit(kmutex_t *mtx) 147 { 148 149 mutex_exit(mtx); 150 } 151 152 int 153 mutex_owned(kmutex_t *mtx) 154 { 155 UPMTX(mtx); 156 157 return upm->upm_owner == curlwp; 158 } 159 160 struct lwp * 161 mutex_owner(kmutex_t *mtx) 162 { 163 164 return upm->upm_owner; 165 } 166 167 struct uprw { 168 struct lwp *uprw_owner; 169 int uprw_readers; 170 uint16_t uprw_rwant; 171 uint16_t uprw_wwant; 172 struct rumpuser_cv *uprw_rucv_reader; 173 struct rumpuser_cv *uprw_rucv_writer; 174 }; 175 176 #define UPRW(rw) struct uprw *uprw = *(struct uprw **)rw 177 178 /* reader/writer locks */ 179 180 void 181 rw_init(krwlock_t *rw) 182 { 183 struct uprw *uprw; 184 185 CTASSERT(sizeof(krwlock_t) >= sizeof(void *)); 186 checkncpu(); 187 188 uprw = rump_hypermalloc(sizeof(*uprw), 0, true, "rwinit"); 189 memset(uprw, 0, sizeof(*uprw)); 190 rumpuser_cv_init(&uprw->uprw_rucv_reader); 191 rumpuser_cv_init(&uprw->uprw_rucv_writer); 192 memcpy(rw, &uprw, sizeof(void *)); 193 } 194 195 void 196 rw_destroy(krwlock_t *rw) 197 { 198 UPRW(rw); 199 200 rumpuser_cv_destroy(uprw->uprw_rucv_reader); 201 rumpuser_cv_destroy(uprw->uprw_rucv_writer); 202 rump_hyperfree(uprw, sizeof(*uprw)); 203 } 204 205 /* take rwlock. prefer writers over readers (see rw_tryenter and rw_exit) */ 206 void 207 rw_enter(krwlock_t *rw, const krw_t op) 208 { 209 UPRW(rw); 210 struct rumpuser_cv *rucv; 211 uint16_t *wp; 212 213 if (rw_tryenter(rw, op)) 214 return; 215 216 /* lagpath */ 217 if (op == RW_READER) { 218 rucv = uprw->uprw_rucv_reader; 219 wp = &uprw->uprw_rwant; 220 } else { 221 rucv = uprw->uprw_rucv_writer; 222 wp = &uprw->uprw_wwant; 223 } 224 225 (*wp)++; 226 while (!rw_tryenter(rw, op)) { 227 rump_schedlock_cv_wait(rucv); 228 } 229 (*wp)--; 230 } 231 232 int 233 rw_tryenter(krwlock_t *rw, const krw_t op) 234 { 235 UPRW(rw); 236 237 switch (op) { 238 case RW_READER: 239 if (uprw->uprw_owner == NULL && uprw->uprw_wwant == 0) { 240 uprw->uprw_readers++; 241 return 1; 242 } 243 break; 244 case RW_WRITER: 245 if (uprw->uprw_owner == NULL && uprw->uprw_readers == 0) { 246 uprw->uprw_owner = curlwp; 247 return 1; 248 } 249 break; 250 } 251 252 return 0; 253 } 254 255 void 256 rw_exit(krwlock_t *rw) 257 { 258 UPRW(rw); 259 260 if (uprw->uprw_readers > 0) { 261 uprw->uprw_readers--; 262 } else { 263 KASSERT(uprw->uprw_owner == curlwp); 264 uprw->uprw_owner = NULL; 265 } 266 267 if (uprw->uprw_wwant) { 268 rumpuser_cv_signal(uprw->uprw_rucv_writer); 269 } else if (uprw->uprw_rwant) { 270 rumpuser_cv_signal(uprw->uprw_rucv_reader); 271 } 272 } 273 274 int 275 rw_tryupgrade(krwlock_t *rw) 276 { 277 UPRW(rw); 278 279 if (uprw->uprw_readers == 1 && uprw->uprw_owner == NULL) { 280 uprw->uprw_readers = 0; 281 uprw->uprw_owner = curlwp; 282 return 1; 283 } else { 284 return 0; 285 } 286 } 287 288 int 289 rw_write_held(krwlock_t *rw) 290 { 291 UPRW(rw); 292 293 return uprw->uprw_owner == curlwp; 294 } 295 296 int 297 rw_read_held(krwlock_t *rw) 298 { 299 UPRW(rw); 300 301 return uprw->uprw_readers > 0; 302 } 303 304 int 305 rw_lock_held(krwlock_t *rw) 306 { 307 UPRW(rw); 308 309 return uprw->uprw_owner || uprw->uprw_readers; 310 } 311 312 313 /* 314 * Condvars are almost the same as in the MP case except that we 315 * use the scheduler mutex as the pthread interlock instead of the 316 * mutex associated with the condvar. 317 */ 318 319 #define RUMPCV(cv) (*(struct rumpuser_cv **)(cv)) 320 321 void 322 cv_init(kcondvar_t *cv, const char *msg) 323 { 324 325 CTASSERT(sizeof(kcondvar_t) >= sizeof(void *)); 326 checkncpu(); 327 328 rumpuser_cv_init((struct rumpuser_cv **)cv); 329 } 330 331 void 332 cv_destroy(kcondvar_t *cv) 333 { 334 335 rumpuser_cv_destroy(RUMPCV(cv)); 336 } 337 338 void 339 cv_wait(kcondvar_t *cv, kmutex_t *mtx) 340 { 341 #ifdef DIAGNOSTIC 342 UPMTX(mtx); 343 KASSERT(upm->upm_owner == curlwp); 344 345 if (rump_threads == 0) 346 panic("cv_wait without threads"); 347 #endif 348 349 /* 350 * NOTE: we must atomically release the *CPU* here, i.e. 351 * nothing between mutex_exit and entering rumpuser condwait 352 * may preempt us from the virtual CPU. 353 */ 354 mutex_exit(mtx); 355 rump_schedlock_cv_wait(RUMPCV(cv)); 356 mutex_enter(mtx); 357 } 358 359 int 360 cv_wait_sig(kcondvar_t *cv, kmutex_t *mtx) 361 { 362 363 cv_wait(cv, mtx); 364 return 0; 365 } 366 367 int 368 cv_timedwait(kcondvar_t *cv, kmutex_t *mtx, int ticks) 369 { 370 struct timespec ts, tstick; 371 372 #ifdef DIAGNOSTIC 373 UPMTX(mtx); 374 KASSERT(upm->upm_owner == curlwp); 375 #endif 376 377 /* 378 * XXX: this fetches rump kernel time, but rumpuser_cv_timedwait 379 * uses host time. 380 */ 381 nanotime(&ts); 382 tstick.tv_sec = ticks / hz; 383 tstick.tv_nsec = (ticks % hz) * (1000000000/hz); 384 timespecadd(&ts, &tstick, &ts); 385 386 if (ticks == 0) { 387 cv_wait(cv, mtx); 388 return 0; 389 } else { 390 int rv; 391 mutex_exit(mtx); 392 rv = rump_schedlock_cv_timedwait(RUMPCV(cv), &ts); 393 mutex_enter(mtx); 394 if (rv) 395 return EWOULDBLOCK; 396 else 397 return 0; 398 } 399 } 400 401 int 402 cv_timedwait_sig(kcondvar_t *cv, kmutex_t *mtx, int ticks) 403 { 404 405 return cv_timedwait(cv, mtx, ticks); 406 } 407 408 void 409 cv_signal(kcondvar_t *cv) 410 { 411 412 /* CPU == interlock */ 413 rumpuser_cv_signal(RUMPCV(cv)); 414 } 415 416 void 417 cv_broadcast(kcondvar_t *cv) 418 { 419 420 /* CPU == interlock */ 421 rumpuser_cv_broadcast(RUMPCV(cv)); 422 } 423 424 bool 425 cv_has_waiters(kcondvar_t *cv) 426 { 427 428 return rumpuser_cv_has_waiters(RUMPCV(cv)); 429 } 430 431 /* this is not much of an attempt, but ... */ 432 bool 433 cv_is_valid(kcondvar_t *cv) 434 { 435 436 return RUMPCV(cv) != NULL; 437 } 438