1 /* 2 * Copyright (c) 2020 François Tigeot <ftigeot@wolfpond.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <linux/interrupt.h> 28 #include <linux/slab.h> 29 30 #include <sys/kthread.h> 31 32 /* 33 * Linux tasklet constraints: 34 * - tasklets that have the same type cannot be run on multiple processors at 35 * the same time 36 * - tasklets always run on the processor from which they were originally 37 * submitted 38 * - when a tasklet is scheduled, its state is set to TASKLET_STATE_SCHED, 39 * and the tasklet added to a queue 40 * - during the execution of its function, the tasklet state is set to 41 * TASKLET_STATE_RUN and the TASKLET_STATE_SCHED state is removed 42 */ 43 44 struct tasklet_entry { 45 struct tasklet_struct *ts; 46 STAILQ_ENTRY(tasklet_entry) tasklet_entries; 47 }; 48 49 static struct lock tasklet_lock = LOCK_INITIALIZER("dltll", 0, LK_CANRECURSE); 50 51 static struct thread *tasklet_td = NULL; 52 STAILQ_HEAD(tasklet_list_head, tasklet_entry) tlist = STAILQ_HEAD_INITIALIZER(tlist); 53 STAILQ_HEAD(tasklet_hi_list_head, tasklet_entry) tlist_hi = STAILQ_HEAD_INITIALIZER(tlist_hi); 54 55 static int tasklet_pending = 0; 56 57 /* 58 * Linux does: 59 * 1 - copy list locally 60 * 2 - empty global list 61 * 3 - process local list from head to tail 62 ***** 63 * local list processing: 64 * - if element cannot be run, put it at the tail 65 * - last element == null 66 */ 67 #define PROCESS_TASKLET_LIST(which_list) do { \ 68 STAILQ_FOREACH_MUTABLE(te, &which_list, tasklet_entries, tmp_te) { \ 69 struct tasklet_struct *t = te->ts; \ 70 \ 71 /* \ 72 This tasklet is dying, remove it from the list. \ 73 We allow to it to run one last time if it has \ 74 already been scheduled. \ 75 */ \ 76 if (test_bit(TASKLET_IS_DYING, &t->state)) { \ 77 STAILQ_REMOVE(&which_list, te, tasklet_entry, tasklet_entries); \ 78 kfree(te); \ 79 } \ 80 \ 81 /* This tasklet is not enabled, try the next one */ \ 82 if (atomic_read(&t->count) != 0) \ 83 continue; \ 84 \ 85 /* This tasklet is not scheduled, try the next one */ \ 86 if (!test_bit(TASKLET_STATE_SCHED, &t->state)) \ 87 continue; \ 88 \ 89 clear_bit(TASKLET_STATE_SCHED, &t->state); \ 90 set_bit(TASKLET_STATE_RUN, &t->state); \ 91 \ 92 lockmgr(&tasklet_lock, LK_RELEASE); \ 93 if (t->func) \ 94 t->func(t->data); \ 95 lockmgr(&tasklet_lock, LK_EXCLUSIVE); \ 96 \ 97 clear_bit(TASKLET_STATE_RUN, &t->state); \ 98 } \ 99 } while (0) 100 101 /* XXX runners should be CPU-specific */ 102 static void 103 tasklet_runner(void *arg) 104 { 105 struct tasklet_entry *te, *tmp_te; 106 107 lockmgr(&tasklet_lock, LK_EXCLUSIVE); 108 while (1) { 109 /* 110 Only sleep if we haven't been raced by a _schedule() 111 call during an unlock window 112 */ 113 if (tasklet_pending == 0) { 114 lksleep(&tasklet_runner, &tasklet_lock, 0, "tkidle", 0); 115 } 116 tasklet_pending = 0; 117 118 /* Process hi tasklets first */ 119 PROCESS_TASKLET_LIST(tlist_hi); 120 PROCESS_TASKLET_LIST(tlist); 121 } 122 lockmgr(&tasklet_lock, LK_RELEASE); 123 } 124 125 void 126 tasklet_init(struct tasklet_struct *t, 127 void (*func)(unsigned long), unsigned long data) 128 { 129 t->state = 0; 130 t->func = func; 131 t->data = data; 132 atomic_set(&t->count, 0); 133 } 134 135 #define TASKLET_SCHEDULE_COMMON(t, list) do { \ 136 struct tasklet_entry *te; \ 137 \ 138 lockmgr(&tasklet_lock, LK_EXCLUSIVE); \ 139 set_bit(TASKLET_STATE_SCHED, &t->state); \ 140 \ 141 STAILQ_FOREACH(te, &(list), tasklet_entries) { \ 142 if (te->ts == t) \ 143 goto found_and_done; \ 144 } \ 145 \ 146 te = kzalloc(sizeof(struct tasklet_entry), M_WAITOK); \ 147 te->ts = t; \ 148 STAILQ_INSERT_TAIL(&(list), te, tasklet_entries); \ 149 \ 150 found_and_done: \ 151 tasklet_pending = 1; \ 152 wakeup(&tasklet_runner); \ 153 lockmgr(&tasklet_lock, LK_RELEASE); \ 154 } while (0) 155 156 void 157 tasklet_schedule(struct tasklet_struct *t) 158 { 159 TASKLET_SCHEDULE_COMMON(t, tlist); 160 } 161 162 void 163 tasklet_hi_schedule(struct tasklet_struct *t) 164 { 165 TASKLET_SCHEDULE_COMMON(t, tlist_hi); 166 } 167 168 void 169 tasklet_kill(struct tasklet_struct *t) 170 { 171 set_bit(TASKLET_IS_DYING, &t->state); 172 wakeup(&tasklet_runner); 173 } 174 175 static int init_tasklets(void *arg) 176 { 177 kthread_create(tasklet_runner, NULL, &tasklet_td, "tasklet_runner"); 178 179 return 0; 180 } 181 182 SYSINIT(linux_tasklet_init, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, init_tasklets, NULL); 183