1 /*- 2 * Copyright (c) 2017-2019 Hans Petter Selasky 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 #include <linux/workqueue.h> 29 #include <linux/wait.h> 30 #include <linux/compat.h> 31 #include <linux/spinlock.h> 32 #include <linux/rcupdate.h> 33 #include <linux/irq_work.h> 34 35 #include <sys/kernel.h> 36 37 /* 38 * Define all work struct states 39 */ 40 enum { 41 WORK_ST_IDLE, /* idle - not started */ 42 WORK_ST_TIMER, /* timer is being started */ 43 WORK_ST_TASK, /* taskqueue is being queued */ 44 WORK_ST_EXEC, /* callback is being called */ 45 WORK_ST_CANCEL, /* cancel is being requested */ 46 WORK_ST_MAX, 47 }; 48 49 /* 50 * Define global workqueues 51 */ 52 static struct workqueue_struct *linux_system_short_wq; 53 static struct workqueue_struct *linux_system_long_wq; 54 55 struct workqueue_struct *system_wq; 56 struct workqueue_struct *system_long_wq; 57 struct workqueue_struct *system_unbound_wq; 58 struct workqueue_struct *system_highpri_wq; 59 struct workqueue_struct *system_power_efficient_wq; 60 61 struct taskqueue *linux_irq_work_tq; 62 63 static int linux_default_wq_cpus = 4; 64 65 static void linux_delayed_work_timer_fn(void *); 66 67 /* 68 * This function atomically updates the work state and returns the 69 * previous state at the time of update. 70 */ 71 static uint8_t 72 linux_update_state(atomic_t *v, const uint8_t *pstate) 73 { 74 int c, old; 75 76 c = v->counter; 77 78 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c) 79 c = old; 80 81 return (c); 82 } 83 84 /* 85 * A LinuxKPI task is allowed to free itself inside the callback function 86 * and cannot safely be referred after the callback function has 87 * completed. This function gives the linux_work_fn() function a hint, 88 * that the task is not going away and can have its state checked 89 * again. Without this extra hint LinuxKPI tasks cannot be serialized 90 * across multiple worker threads. 91 */ 92 static bool 93 linux_work_exec_unblock(struct work_struct *work) 94 { 95 struct workqueue_struct *wq; 96 struct work_exec *exec; 97 bool retval = false; 98 99 wq = work->work_queue; 100 if (unlikely(wq == NULL)) 101 goto done; 102 103 WQ_EXEC_LOCK(wq); 104 TAILQ_FOREACH(exec, &wq->exec_head, entry) { 105 if (exec->target == work) { 106 exec->target = NULL; 107 retval = true; 108 break; 109 } 110 } 111 WQ_EXEC_UNLOCK(wq); 112 done: 113 return (retval); 114 } 115 116 static void 117 linux_delayed_work_enqueue(struct delayed_work *dwork) 118 { 119 struct taskqueue *tq; 120 121 tq = dwork->work.work_queue->taskqueue; 122 taskqueue_enqueue(tq, &dwork->work.work_task); 123 } 124 125 /* 126 * This function queues the given work structure on the given 127 * workqueue. It returns non-zero if the work was successfully 128 * [re-]queued. Else the work is already pending for completion. 129 */ 130 bool 131 linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq, 132 struct work_struct *work) 133 { 134 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 135 [WORK_ST_IDLE] = WORK_ST_TASK, /* start queuing task */ 136 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */ 137 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ 138 [WORK_ST_EXEC] = WORK_ST_TASK, /* queue task another time */ 139 [WORK_ST_CANCEL] = WORK_ST_TASK, /* start queuing task again */ 140 }; 141 142 if (atomic_read(&wq->draining) != 0) 143 return (!work_pending(work)); 144 145 switch (linux_update_state(&work->state, states)) { 146 case WORK_ST_EXEC: 147 case WORK_ST_CANCEL: 148 if (linux_work_exec_unblock(work) != 0) 149 return (true); 150 /* FALLTHROUGH */ 151 case WORK_ST_IDLE: 152 work->work_queue = wq; 153 taskqueue_enqueue(wq->taskqueue, &work->work_task); 154 return (true); 155 default: 156 return (false); /* already on a queue */ 157 } 158 } 159 160 /* 161 * Callback func for linux_queue_rcu_work 162 */ 163 static void 164 rcu_work_func(struct rcu_head *rcu) 165 { 166 struct rcu_work *rwork; 167 168 rwork = container_of(rcu, struct rcu_work, rcu); 169 linux_queue_work_on(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); 170 } 171 172 /* 173 * This function queue a work after a grace period 174 * If the work was already pending it returns false, 175 * if not it calls call_rcu and returns true. 176 */ 177 bool 178 linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) 179 { 180 181 if (!linux_work_pending(&rwork->work)) { 182 rwork->wq = wq; 183 linux_call_rcu(RCU_TYPE_REGULAR, &rwork->rcu, rcu_work_func); 184 return (true); 185 } 186 return (false); 187 } 188 189 /* 190 * This function waits for the last execution of a work and then 191 * flush the work. 192 * It returns true if the work was pending and we waited, it returns 193 * false otherwise. 194 */ 195 bool 196 linux_flush_rcu_work(struct rcu_work *rwork) 197 { 198 199 if (linux_work_pending(&rwork->work)) { 200 linux_rcu_barrier(RCU_TYPE_REGULAR); 201 linux_flush_work(&rwork->work); 202 return (true); 203 } 204 return (linux_flush_work(&rwork->work)); 205 } 206 207 /* 208 * This function queues the given work structure on the given 209 * workqueue after a given delay in ticks. It returns non-zero if the 210 * work was successfully [re-]queued. Else the work is already pending 211 * for completion. 212 */ 213 bool 214 linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq, 215 struct delayed_work *dwork, unsigned delay) 216 { 217 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 218 [WORK_ST_IDLE] = WORK_ST_TIMER, /* start timeout */ 219 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */ 220 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ 221 [WORK_ST_EXEC] = WORK_ST_TIMER, /* start timeout */ 222 [WORK_ST_CANCEL] = WORK_ST_TIMER, /* start timeout */ 223 }; 224 225 if (atomic_read(&wq->draining) != 0) 226 return (!work_pending(&dwork->work)); 227 228 switch (linux_update_state(&dwork->work.state, states)) { 229 case WORK_ST_EXEC: 230 case WORK_ST_CANCEL: 231 if (delay == 0 && linux_work_exec_unblock(&dwork->work) != 0) { 232 dwork->timer.expires = jiffies; 233 return (true); 234 } 235 /* FALLTHROUGH */ 236 case WORK_ST_IDLE: 237 dwork->work.work_queue = wq; 238 dwork->timer.expires = jiffies + delay; 239 240 if (delay == 0) { 241 linux_delayed_work_enqueue(dwork); 242 } else if (unlikely(cpu != WORK_CPU_UNBOUND)) { 243 mtx_lock(&dwork->timer.mtx); 244 callout_reset_on(&dwork->timer.callout, delay, 245 &linux_delayed_work_timer_fn, dwork, cpu); 246 mtx_unlock(&dwork->timer.mtx); 247 } else { 248 mtx_lock(&dwork->timer.mtx); 249 callout_reset(&dwork->timer.callout, delay, 250 &linux_delayed_work_timer_fn, dwork); 251 mtx_unlock(&dwork->timer.mtx); 252 } 253 return (true); 254 default: 255 return (false); /* already on a queue */ 256 } 257 } 258 259 void 260 linux_work_fn(void *context, int pending) 261 { 262 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 263 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 264 [WORK_ST_TIMER] = WORK_ST_EXEC, /* delayed work w/o timeout */ 265 [WORK_ST_TASK] = WORK_ST_EXEC, /* call callback */ 266 [WORK_ST_EXEC] = WORK_ST_IDLE, /* complete callback */ 267 [WORK_ST_CANCEL] = WORK_ST_EXEC, /* failed to cancel */ 268 }; 269 struct work_struct *work; 270 struct workqueue_struct *wq; 271 struct work_exec exec; 272 struct task_struct *task; 273 274 task = current; 275 276 /* setup local variables */ 277 work = context; 278 wq = work->work_queue; 279 280 /* store target pointer */ 281 exec.target = work; 282 283 /* insert executor into list */ 284 WQ_EXEC_LOCK(wq); 285 TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry); 286 while (1) { 287 switch (linux_update_state(&work->state, states)) { 288 case WORK_ST_TIMER: 289 case WORK_ST_TASK: 290 case WORK_ST_CANCEL: 291 WQ_EXEC_UNLOCK(wq); 292 293 /* set current work structure */ 294 task->work = work; 295 296 /* call work function */ 297 work->func(work); 298 299 /* set current work structure */ 300 task->work = NULL; 301 302 WQ_EXEC_LOCK(wq); 303 /* check if unblocked */ 304 if (exec.target != work) { 305 /* reapply block */ 306 exec.target = work; 307 break; 308 } 309 /* FALLTHROUGH */ 310 default: 311 goto done; 312 } 313 } 314 done: 315 /* remove executor from list */ 316 TAILQ_REMOVE(&wq->exec_head, &exec, entry); 317 WQ_EXEC_UNLOCK(wq); 318 } 319 320 void 321 linux_delayed_work_fn(void *context, int pending) 322 { 323 struct delayed_work *dwork = context; 324 325 /* 326 * Make sure the timer belonging to the delayed work gets 327 * drained before invoking the work function. Else the timer 328 * mutex may still be in use which can lead to use-after-free 329 * situations, because the work function might free the work 330 * structure before returning. 331 */ 332 callout_drain(&dwork->timer.callout); 333 334 linux_work_fn(&dwork->work, pending); 335 } 336 337 static void 338 linux_delayed_work_timer_fn(void *arg) 339 { 340 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 341 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 342 [WORK_ST_TIMER] = WORK_ST_TASK, /* start queueing task */ 343 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ 344 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */ 345 [WORK_ST_CANCEL] = WORK_ST_TASK, /* failed to cancel */ 346 }; 347 struct delayed_work *dwork = arg; 348 349 switch (linux_update_state(&dwork->work.state, states)) { 350 case WORK_ST_TIMER: 351 case WORK_ST_CANCEL: 352 linux_delayed_work_enqueue(dwork); 353 break; 354 default: 355 break; 356 } 357 } 358 359 /* 360 * This function cancels the given work structure in a synchronous 361 * fashion. It returns non-zero if the work was successfully 362 * cancelled. Else the work was already cancelled. 363 */ 364 bool 365 linux_cancel_work_sync(struct work_struct *work) 366 { 367 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 368 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 369 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */ 370 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */ 371 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */ 372 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */ 373 }; 374 struct taskqueue *tq; 375 bool retval = false; 376 377 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 378 "linux_cancel_work_sync() might sleep"); 379 retry: 380 switch (linux_update_state(&work->state, states)) { 381 case WORK_ST_IDLE: 382 case WORK_ST_TIMER: 383 return (retval); 384 case WORK_ST_EXEC: 385 tq = work->work_queue->taskqueue; 386 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0) 387 taskqueue_drain(tq, &work->work_task); 388 goto retry; /* work may have restarted itself */ 389 default: 390 tq = work->work_queue->taskqueue; 391 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0) 392 taskqueue_drain(tq, &work->work_task); 393 retval = true; 394 goto retry; 395 } 396 } 397 398 /* 399 * This function atomically stops the timer and callback. The timer 400 * callback will not be called after this function returns. This 401 * functions returns true when the timeout was cancelled. Else the 402 * timeout was not started or has already been called. 403 */ 404 static inline bool 405 linux_cancel_timer(struct delayed_work *dwork, bool drain) 406 { 407 bool cancelled; 408 409 mtx_lock(&dwork->timer.mtx); 410 cancelled = (callout_stop(&dwork->timer.callout) == 1); 411 mtx_unlock(&dwork->timer.mtx); 412 413 /* check if we should drain */ 414 if (drain) 415 callout_drain(&dwork->timer.callout); 416 return (cancelled); 417 } 418 419 /* 420 * This function cancels the given delayed work structure in a 421 * non-blocking fashion. It returns non-zero if the work was 422 * successfully cancelled. Else the work may still be busy or already 423 * cancelled. 424 */ 425 bool 426 linux_cancel_delayed_work(struct delayed_work *dwork) 427 { 428 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 429 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 430 [WORK_ST_TIMER] = WORK_ST_CANCEL, /* try to cancel */ 431 [WORK_ST_TASK] = WORK_ST_CANCEL, /* try to cancel */ 432 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */ 433 [WORK_ST_CANCEL] = WORK_ST_CANCEL, /* NOP */ 434 }; 435 struct taskqueue *tq; 436 bool cancelled; 437 438 mtx_lock(&dwork->timer.mtx); 439 switch (linux_update_state(&dwork->work.state, states)) { 440 case WORK_ST_TIMER: 441 case WORK_ST_CANCEL: 442 cancelled = (callout_stop(&dwork->timer.callout) == 1); 443 if (cancelled) { 444 atomic_cmpxchg(&dwork->work.state, 445 WORK_ST_CANCEL, WORK_ST_IDLE); 446 mtx_unlock(&dwork->timer.mtx); 447 return (true); 448 } 449 /* FALLTHROUGH */ 450 case WORK_ST_TASK: 451 tq = dwork->work.work_queue->taskqueue; 452 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0) { 453 atomic_cmpxchg(&dwork->work.state, 454 WORK_ST_CANCEL, WORK_ST_IDLE); 455 mtx_unlock(&dwork->timer.mtx); 456 return (true); 457 } 458 /* FALLTHROUGH */ 459 default: 460 mtx_unlock(&dwork->timer.mtx); 461 return (false); 462 } 463 } 464 465 /* 466 * This function cancels the given work structure in a synchronous 467 * fashion. It returns non-zero if the work was successfully 468 * cancelled. Else the work was already cancelled. 469 */ 470 bool 471 linux_cancel_delayed_work_sync(struct delayed_work *dwork) 472 { 473 static const uint8_t states[WORK_ST_MAX] __aligned(8) = { 474 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ 475 [WORK_ST_TIMER] = WORK_ST_IDLE, /* cancel and drain */ 476 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */ 477 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */ 478 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */ 479 }; 480 struct taskqueue *tq; 481 bool retval = false; 482 int ret, state; 483 bool cancelled; 484 485 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 486 "linux_cancel_delayed_work_sync() might sleep"); 487 mtx_lock(&dwork->timer.mtx); 488 489 state = linux_update_state(&dwork->work.state, states); 490 switch (state) { 491 case WORK_ST_IDLE: 492 mtx_unlock(&dwork->timer.mtx); 493 return (retval); 494 case WORK_ST_TIMER: 495 case WORK_ST_CANCEL: 496 cancelled = (callout_stop(&dwork->timer.callout) == 1); 497 498 tq = dwork->work.work_queue->taskqueue; 499 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL); 500 mtx_unlock(&dwork->timer.mtx); 501 502 callout_drain(&dwork->timer.callout); 503 taskqueue_drain(tq, &dwork->work.work_task); 504 return (cancelled || (ret != 0)); 505 default: 506 tq = dwork->work.work_queue->taskqueue; 507 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL); 508 mtx_unlock(&dwork->timer.mtx); 509 if (ret != 0) 510 taskqueue_drain(tq, &dwork->work.work_task); 511 return (ret != 0); 512 } 513 } 514 515 /* 516 * This function waits until the given work structure is completed. 517 * It returns non-zero if the work was successfully 518 * waited for. Else the work was not waited for. 519 */ 520 bool 521 linux_flush_work(struct work_struct *work) 522 { 523 struct taskqueue *tq; 524 bool retval; 525 526 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 527 "linux_flush_work() might sleep"); 528 529 switch (atomic_read(&work->state)) { 530 case WORK_ST_IDLE: 531 return (false); 532 default: 533 tq = work->work_queue->taskqueue; 534 retval = taskqueue_poll_is_busy(tq, &work->work_task); 535 taskqueue_drain(tq, &work->work_task); 536 return (retval); 537 } 538 } 539 540 /* 541 * This function waits until the given delayed work structure is 542 * completed. It returns non-zero if the work was successfully waited 543 * for. Else the work was not waited for. 544 */ 545 bool 546 linux_flush_delayed_work(struct delayed_work *dwork) 547 { 548 struct taskqueue *tq; 549 bool retval; 550 551 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 552 "linux_flush_delayed_work() might sleep"); 553 554 switch (atomic_read(&dwork->work.state)) { 555 case WORK_ST_IDLE: 556 return (false); 557 case WORK_ST_TIMER: 558 if (linux_cancel_timer(dwork, 1)) 559 linux_delayed_work_enqueue(dwork); 560 /* FALLTHROUGH */ 561 default: 562 tq = dwork->work.work_queue->taskqueue; 563 retval = taskqueue_poll_is_busy(tq, &dwork->work.work_task); 564 taskqueue_drain(tq, &dwork->work.work_task); 565 return (retval); 566 } 567 } 568 569 /* 570 * This function returns true if the given work is pending, and not 571 * yet executing: 572 */ 573 bool 574 linux_work_pending(struct work_struct *work) 575 { 576 switch (atomic_read(&work->state)) { 577 case WORK_ST_TIMER: 578 case WORK_ST_TASK: 579 case WORK_ST_CANCEL: 580 return (true); 581 default: 582 return (false); 583 } 584 } 585 586 /* 587 * This function returns true if the given work is busy. 588 */ 589 bool 590 linux_work_busy(struct work_struct *work) 591 { 592 struct taskqueue *tq; 593 594 switch (atomic_read(&work->state)) { 595 case WORK_ST_IDLE: 596 return (false); 597 case WORK_ST_EXEC: 598 tq = work->work_queue->taskqueue; 599 return (taskqueue_poll_is_busy(tq, &work->work_task)); 600 default: 601 return (true); 602 } 603 } 604 605 struct workqueue_struct * 606 linux_create_workqueue_common(const char *name, int cpus) 607 { 608 struct workqueue_struct *wq; 609 610 /* 611 * If zero CPUs are specified use the default number of CPUs: 612 */ 613 if (cpus == 0) 614 cpus = linux_default_wq_cpus; 615 616 wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO); 617 wq->taskqueue = taskqueue_create(name, M_WAITOK, 618 taskqueue_thread_enqueue, &wq->taskqueue); 619 atomic_set(&wq->draining, 0); 620 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); 621 TAILQ_INIT(&wq->exec_head); 622 mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF); 623 624 return (wq); 625 } 626 627 void 628 linux_destroy_workqueue(struct workqueue_struct *wq) 629 { 630 atomic_inc(&wq->draining); 631 drain_workqueue(wq); 632 taskqueue_free(wq->taskqueue); 633 mtx_destroy(&wq->exec_mtx); 634 kfree(wq); 635 } 636 637 void 638 linux_init_delayed_work(struct delayed_work *dwork, work_func_t func) 639 { 640 memset(dwork, 0, sizeof(*dwork)); 641 dwork->work.func = func; 642 TASK_INIT(&dwork->work.work_task, 0, linux_delayed_work_fn, dwork); 643 mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL, 644 MTX_DEF | MTX_NOWITNESS); 645 callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0); 646 } 647 648 struct work_struct * 649 linux_current_work(void) 650 { 651 return (current->work); 652 } 653 654 static void 655 linux_work_init(void *arg) 656 { 657 int max_wq_cpus = mp_ncpus + 1; 658 659 /* avoid deadlock when there are too few threads */ 660 if (max_wq_cpus < 4) 661 max_wq_cpus = 4; 662 663 /* set default number of CPUs */ 664 linux_default_wq_cpus = max_wq_cpus; 665 666 linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus); 667 linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus); 668 669 /* populate the workqueue pointers */ 670 system_long_wq = linux_system_long_wq; 671 system_wq = linux_system_short_wq; 672 system_power_efficient_wq = linux_system_short_wq; 673 system_unbound_wq = linux_system_short_wq; 674 system_highpri_wq = linux_system_short_wq; 675 } 676 SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL); 677 678 static void 679 linux_work_uninit(void *arg) 680 { 681 destroy_workqueue(linux_system_short_wq); 682 destroy_workqueue(linux_system_long_wq); 683 684 /* clear workqueue pointers */ 685 system_long_wq = NULL; 686 system_wq = NULL; 687 system_power_efficient_wq = NULL; 688 system_unbound_wq = NULL; 689 system_highpri_wq = NULL; 690 } 691 SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL); 692 693 void 694 linux_irq_work_fn(void *context, int pending) 695 { 696 struct irq_work *irqw = context; 697 698 irqw->func(irqw); 699 } 700 701 static void 702 linux_irq_work_init_fn(void *context, int pending) 703 { 704 /* 705 * LinuxKPI performs lazy allocation of memory structures required by 706 * current on the first access to it. As some irq_work clients read 707 * it with spinlock taken, we have to preallocate td_lkpi_task before 708 * first call to irq_work_queue(). As irq_work uses a single thread, 709 * it is enough to read current once at SYSINIT stage. 710 */ 711 if (current == NULL) 712 panic("irq_work taskqueue is not initialized"); 713 } 714 static struct task linux_irq_work_init_task = 715 TASK_INITIALIZER(0, linux_irq_work_init_fn, &linux_irq_work_init_task); 716 717 static void 718 linux_irq_work_init(void *arg) 719 { 720 linux_irq_work_tq = taskqueue_create_fast("linuxkpi_irq_wq", 721 M_WAITOK, taskqueue_thread_enqueue, &linux_irq_work_tq); 722 taskqueue_start_threads(&linux_irq_work_tq, 1, PWAIT, 723 "linuxkpi_irq_wq"); 724 taskqueue_enqueue(linux_irq_work_tq, &linux_irq_work_init_task); 725 } 726 SYSINIT(linux_irq_work_init, SI_SUB_TASKQ, SI_ORDER_SECOND, 727 linux_irq_work_init, NULL); 728 729 static void 730 linux_irq_work_uninit(void *arg) 731 { 732 taskqueue_drain_all(linux_irq_work_tq); 733 taskqueue_free(linux_irq_work_tq); 734 } 735 SYSUNINIT(linux_irq_work_uninit, SI_SUB_TASKQ, SI_ORDER_SECOND, 736 linux_irq_work_uninit, NULL); 737