1 /*-
2  * Copyright (c) 2017 Hans Petter Selasky
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice unmodified, this list of conditions, and the following
10  *    disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <linux/workqueue.h>
31 #include <linux/wait.h>
32 #include <linux/compat.h>
33 #include <linux/spinlock.h>
34 
35 #include <sys/kernel.h>
36 
37 /*
38  * Define all work struct states
39  */
40 enum {
41 	WORK_ST_IDLE,			/* idle - not started */
42 	WORK_ST_TIMER,			/* timer is being started */
43 	WORK_ST_TASK,			/* taskqueue is being queued */
44 	WORK_ST_EXEC,			/* callback is being called */
45 	WORK_ST_CANCEL,			/* cancel is being requested */
46 	WORK_ST_MAX,
47 };
48 
49 /*
50  * Define global workqueues
51  */
52 static struct workqueue_struct *linux_system_short_wq;
53 static struct workqueue_struct *linux_system_long_wq;
54 
55 struct workqueue_struct *system_wq;
56 struct workqueue_struct *system_long_wq;
57 struct workqueue_struct *system_unbound_wq;
58 struct workqueue_struct *system_power_efficient_wq;
59 
60 static int linux_default_wq_cpus = 4;
61 
62 static void linux_delayed_work_timer_fn(void *);
63 
64 /*
65  * This function atomically updates the work state and returns the
66  * previous state at the time of update.
67  */
68 static uint8_t
69 linux_update_state(atomic_t *v, const uint8_t *pstate)
70 {
71 	int c, old;
72 
73 	c = v->counter;
74 
75 	while ((old = atomic_cmpxchg(v, c, pstate[c])) != c)
76 		c = old;
77 
78 	return (c);
79 }
80 
81 /*
82  * A LinuxKPI task is allowed to free itself inside the callback function
83  * and cannot safely be referred after the callback function has
84  * completed. This function gives the linux_work_fn() function a hint,
85  * that the task is not going away and can have its state checked
86  * again. Without this extra hint LinuxKPI tasks cannot be serialized
87  * accross multiple worker threads.
88  */
89 static bool
90 linux_work_exec_unblock(struct work_struct *work)
91 {
92 	struct workqueue_struct *wq;
93 	struct work_exec *exec;
94 	bool retval = 0;
95 
96 	wq = work->work_queue;
97 	if (unlikely(wq == NULL))
98 		goto done;
99 
100 	WQ_EXEC_LOCK(wq);
101 	TAILQ_FOREACH(exec, &wq->exec_head, entry) {
102 		if (exec->target == work) {
103 			exec->target = NULL;
104 			retval = 1;
105 			break;
106 		}
107 	}
108 	WQ_EXEC_UNLOCK(wq);
109 done:
110 	return (retval);
111 }
112 
113 static void
114 linux_delayed_work_enqueue(struct delayed_work *dwork)
115 {
116 	struct taskqueue *tq;
117 
118 	tq = dwork->work.work_queue->taskqueue;
119 	taskqueue_enqueue(tq, &dwork->work.work_task);
120 }
121 
122 /*
123  * This function queues the given work structure on the given
124  * workqueue. It returns non-zero if the work was successfully
125  * [re-]queued. Else the work is already pending for completion.
126  */
127 bool
128 linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq,
129     struct work_struct *work)
130 {
131 	static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
132 		[WORK_ST_IDLE] = WORK_ST_TASK,		/* start queuing task */
133 		[WORK_ST_TIMER] = WORK_ST_TIMER,	/* NOP */
134 		[WORK_ST_TASK] = WORK_ST_TASK,		/* NOP */
135 		[WORK_ST_EXEC] = WORK_ST_TASK,		/* queue task another time */
136 		[WORK_ST_CANCEL] = WORK_ST_TASK,	/* start queuing task again */
137 	};
138 
139 	if (atomic_read(&wq->draining) != 0)
140 		return (!work_pending(work));
141 
142 	switch (linux_update_state(&work->state, states)) {
143 	case WORK_ST_EXEC:
144 	case WORK_ST_CANCEL:
145 		if (linux_work_exec_unblock(work) != 0)
146 			return (1);
147 		/* FALLTHROUGH */
148 	case WORK_ST_IDLE:
149 		work->work_queue = wq;
150 		taskqueue_enqueue(wq->taskqueue, &work->work_task);
151 		return (1);
152 	default:
153 		return (0);		/* already on a queue */
154 	}
155 }
156 
157 /*
158  * This function queues the given work structure on the given
159  * workqueue after a given delay in ticks. It returns non-zero if the
160  * work was successfully [re-]queued. Else the work is already pending
161  * for completion.
162  */
163 bool
164 linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
165     struct delayed_work *dwork, unsigned delay)
166 {
167 	static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
168 		[WORK_ST_IDLE] = WORK_ST_TIMER,		/* start timeout */
169 		[WORK_ST_TIMER] = WORK_ST_TIMER,	/* NOP */
170 		[WORK_ST_TASK] = WORK_ST_TASK,		/* NOP */
171 		[WORK_ST_EXEC] = WORK_ST_TIMER,		/* start timeout */
172 		[WORK_ST_CANCEL] = WORK_ST_TIMER,	/* start timeout */
173 	};
174 
175 	if (atomic_read(&wq->draining) != 0)
176 		return (!work_pending(&dwork->work));
177 
178 	switch (linux_update_state(&dwork->work.state, states)) {
179 	case WORK_ST_EXEC:
180 	case WORK_ST_CANCEL:
181 		if (delay == 0 && linux_work_exec_unblock(&dwork->work) != 0) {
182 			dwork->timer.expires = jiffies;
183 			return (1);
184 		}
185 		/* FALLTHROUGH */
186 	case WORK_ST_IDLE:
187 		dwork->work.work_queue = wq;
188 		dwork->timer.expires = jiffies + delay;
189 
190 		if (delay == 0) {
191 			linux_delayed_work_enqueue(dwork);
192 		} else if (unlikely(cpu != WORK_CPU_UNBOUND)) {
193 			mtx_lock(&dwork->timer.mtx);
194 			callout_reset_on(&dwork->timer.callout, delay,
195 			    &linux_delayed_work_timer_fn, dwork, cpu);
196 			mtx_unlock(&dwork->timer.mtx);
197 		} else {
198 			mtx_lock(&dwork->timer.mtx);
199 			callout_reset(&dwork->timer.callout, delay,
200 			    &linux_delayed_work_timer_fn, dwork);
201 			mtx_unlock(&dwork->timer.mtx);
202 		}
203 		return (1);
204 	default:
205 		return (0);		/* already on a queue */
206 	}
207 }
208 
209 void
210 linux_work_fn(void *context, int pending)
211 {
212 	static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
213 		[WORK_ST_IDLE] = WORK_ST_IDLE,		/* NOP */
214 		[WORK_ST_TIMER] = WORK_ST_EXEC,		/* delayed work w/o timeout */
215 		[WORK_ST_TASK] = WORK_ST_EXEC,		/* call callback */
216 		[WORK_ST_EXEC] = WORK_ST_IDLE,		/* complete callback */
217 		[WORK_ST_CANCEL] = WORK_ST_IDLE,	/* complete cancel */
218 	};
219 	struct work_struct *work;
220 	struct workqueue_struct *wq;
221 	struct work_exec exec;
222 
223 	linux_set_current(curthread);
224 
225 	/* setup local variables */
226 	work = context;
227 	wq = work->work_queue;
228 
229 	/* store target pointer */
230 	exec.target = work;
231 
232 	/* insert executor into list */
233 	WQ_EXEC_LOCK(wq);
234 	TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry);
235 	while (1) {
236 		switch (linux_update_state(&work->state, states)) {
237 		case WORK_ST_TIMER:
238 		case WORK_ST_TASK:
239 			WQ_EXEC_UNLOCK(wq);
240 
241 			/* call work function */
242 			work->func(work);
243 
244 			WQ_EXEC_LOCK(wq);
245 			/* check if unblocked */
246 			if (exec.target != work) {
247 				/* reapply block */
248 				exec.target = work;
249 				break;
250 			}
251 			/* FALLTHROUGH */
252 		default:
253 			goto done;
254 		}
255 	}
256 done:
257 	/* remove executor from list */
258 	TAILQ_REMOVE(&wq->exec_head, &exec, entry);
259 	WQ_EXEC_UNLOCK(wq);
260 }
261 
262 static void
263 linux_delayed_work_timer_fn(void *arg)
264 {
265 	static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
266 		[WORK_ST_IDLE] = WORK_ST_IDLE,		/* NOP */
267 		[WORK_ST_TIMER] = WORK_ST_TASK,		/* start queueing task */
268 		[WORK_ST_TASK] = WORK_ST_TASK,		/* NOP */
269 		[WORK_ST_EXEC] = WORK_ST_TASK,		/* queue task another time */
270 		[WORK_ST_CANCEL] = WORK_ST_IDLE,	/* complete cancel */
271 	};
272 	struct delayed_work *dwork = arg;
273 
274 	switch (linux_update_state(&dwork->work.state, states)) {
275 	case WORK_ST_TIMER:
276 		linux_delayed_work_enqueue(dwork);
277 		break;
278 	default:
279 		break;
280 	}
281 }
282 
283 /*
284  * This function cancels the given work structure in a synchronous
285  * fashion. It returns non-zero if the work was successfully
286  * cancelled. Else the work was already cancelled.
287  */
288 bool
289 linux_cancel_work_sync(struct work_struct *work)
290 {
291 	static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
292 		[WORK_ST_IDLE] = WORK_ST_IDLE,		/* NOP */
293 		[WORK_ST_TIMER] = WORK_ST_IDLE,		/* idle */
294 		[WORK_ST_TASK] = WORK_ST_IDLE,		/* idle */
295 		[WORK_ST_EXEC] = WORK_ST_IDLE,		/* idle */
296 		[WORK_ST_CANCEL] = WORK_ST_IDLE,	/* idle */
297 	};
298 	struct taskqueue *tq;
299 
300 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
301 	    "linux_cancel_work_sync() might sleep");
302 
303 	switch (linux_update_state(&work->state, states)) {
304 	case WORK_ST_IDLE:
305 		return (0);
306 	default:
307 		tq = work->work_queue->taskqueue;
308 		if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
309 			taskqueue_drain(tq, &work->work_task);
310 		return (1);
311 	}
312 }
313 
314 /*
315  * This function atomically stops the timer and callback. The timer
316  * callback will not be called after this function returns. This
317  * functions returns true when the timeout was cancelled. Else the
318  * timeout was not started or has already been called.
319  */
320 static inline bool
321 linux_cancel_timer(struct delayed_work *dwork, bool drain)
322 {
323 	bool cancelled;
324 
325 	mtx_lock(&dwork->timer.mtx);
326 	cancelled = (callout_stop(&dwork->timer.callout) == 1);
327 	mtx_unlock(&dwork->timer.mtx);
328 
329 	/* check if we should drain */
330 	if (drain)
331 		callout_drain(&dwork->timer.callout);
332 	return (cancelled);
333 }
334 
335 /*
336  * This function cancels the given delayed work structure in a
337  * non-blocking fashion. It returns non-zero if the work was
338  * successfully cancelled. Else the work may still be busy or already
339  * cancelled.
340  */
341 bool
342 linux_cancel_delayed_work(struct delayed_work *dwork)
343 {
344 	static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
345 		[WORK_ST_IDLE] = WORK_ST_IDLE,		/* NOP */
346 		[WORK_ST_TIMER] = WORK_ST_CANCEL,	/* cancel */
347 		[WORK_ST_TASK] = WORK_ST_CANCEL,	/* cancel */
348 		[WORK_ST_EXEC] = WORK_ST_CANCEL,	/* cancel */
349 		[WORK_ST_CANCEL] = WORK_ST_CANCEL,	/* cancel */
350 	};
351 	struct taskqueue *tq;
352 
353 	switch (linux_update_state(&dwork->work.state, states)) {
354 	case WORK_ST_TIMER:
355 		if (linux_cancel_timer(dwork, 0))
356 			return (1);
357 		/* FALLTHROUGH */
358 	case WORK_ST_TASK:
359 	case WORK_ST_EXEC:
360 		tq = dwork->work.work_queue->taskqueue;
361 		if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0)
362 			return (1);
363 		/* FALLTHROUGH */
364 	default:
365 		return (0);
366 	}
367 }
368 
369 /*
370  * This function cancels the given work structure in a synchronous
371  * fashion. It returns non-zero if the work was successfully
372  * cancelled. Else the work was already cancelled.
373  */
374 bool
375 linux_cancel_delayed_work_sync(struct delayed_work *dwork)
376 {
377 	static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
378 		[WORK_ST_IDLE] = WORK_ST_IDLE,		/* NOP */
379 		[WORK_ST_TIMER] = WORK_ST_IDLE,		/* idle */
380 		[WORK_ST_TASK] = WORK_ST_IDLE,		/* idle */
381 		[WORK_ST_EXEC] = WORK_ST_IDLE,		/* idle */
382 		[WORK_ST_CANCEL] = WORK_ST_IDLE,	/* idle */
383 	};
384 	struct taskqueue *tq;
385 
386 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
387 	    "linux_cancel_delayed_work_sync() might sleep");
388 
389 	switch (linux_update_state(&dwork->work.state, states)) {
390 	case WORK_ST_IDLE:
391 		return (0);
392 	case WORK_ST_TIMER:
393 		if (linux_cancel_timer(dwork, 1)) {
394 			/*
395 			 * Make sure taskqueue is also drained before
396 			 * returning:
397 			 */
398 			tq = dwork->work.work_queue->taskqueue;
399 			taskqueue_drain(tq, &dwork->work.work_task);
400 			return (1);
401 		}
402 		/* FALLTHROUGH */
403 	default:
404 		tq = dwork->work.work_queue->taskqueue;
405 		if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) != 0)
406 			taskqueue_drain(tq, &dwork->work.work_task);
407 		return (1);
408 	}
409 }
410 
411 /*
412  * This function waits until the given work structure is completed.
413  * It returns non-zero if the work was successfully
414  * waited for. Else the work was not waited for.
415  */
416 bool
417 linux_flush_work(struct work_struct *work)
418 {
419 	struct taskqueue *tq;
420 
421 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
422 	    "linux_flush_work() might sleep");
423 
424 	switch (atomic_read(&work->state)) {
425 	case WORK_ST_IDLE:
426 		return (0);
427 	default:
428 		tq = work->work_queue->taskqueue;
429 		taskqueue_drain(tq, &work->work_task);
430 		return (1);
431 	}
432 }
433 
434 /*
435  * This function waits until the given delayed work structure is
436  * completed. It returns non-zero if the work was successfully waited
437  * for. Else the work was not waited for.
438  */
439 bool
440 linux_flush_delayed_work(struct delayed_work *dwork)
441 {
442 	struct taskqueue *tq;
443 
444 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
445 	    "linux_flush_delayed_work() might sleep");
446 
447 	switch (atomic_read(&dwork->work.state)) {
448 	case WORK_ST_IDLE:
449 		return (0);
450 	case WORK_ST_TIMER:
451 		if (linux_cancel_timer(dwork, 1))
452 			linux_delayed_work_enqueue(dwork);
453 		/* FALLTHROUGH */
454 	default:
455 		tq = dwork->work.work_queue->taskqueue;
456 		taskqueue_drain(tq, &dwork->work.work_task);
457 		return (1);
458 	}
459 }
460 
461 /*
462  * This function returns true if the given work is pending, and not
463  * yet executing:
464  */
465 bool
466 linux_work_pending(struct work_struct *work)
467 {
468 	switch (atomic_read(&work->state)) {
469 	case WORK_ST_TIMER:
470 	case WORK_ST_TASK:
471 		return (1);
472 	default:
473 		return (0);
474 	}
475 }
476 
477 /*
478  * This function returns true if the given work is busy.
479  */
480 bool
481 linux_work_busy(struct work_struct *work)
482 {
483 	struct taskqueue *tq;
484 
485 	switch (atomic_read(&work->state)) {
486 	case WORK_ST_IDLE:
487 		return (0);
488 	case WORK_ST_EXEC:
489 	case WORK_ST_CANCEL:
490 		tq = work->work_queue->taskqueue;
491 		return (taskqueue_poll_is_busy(tq, &work->work_task));
492 	default:
493 		return (1);
494 	}
495 }
496 
497 struct workqueue_struct *
498 linux_create_workqueue_common(const char *name, int cpus)
499 {
500 	struct workqueue_struct *wq;
501 
502 	/*
503 	 * If zero CPUs are specified use the default number of CPUs:
504 	 */
505 	if (cpus == 0)
506 		cpus = linux_default_wq_cpus;
507 
508 	wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO);
509 	wq->taskqueue = taskqueue_create(name, M_WAITOK,
510 	    taskqueue_thread_enqueue, &wq->taskqueue);
511 	atomic_set(&wq->draining, 0);
512 	taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
513 	TAILQ_INIT(&wq->exec_head);
514 	mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF);
515 
516 	return (wq);
517 }
518 
519 void
520 linux_destroy_workqueue(struct workqueue_struct *wq)
521 {
522 	atomic_inc(&wq->draining);
523 	drain_workqueue(wq);
524 	taskqueue_free(wq->taskqueue);
525 	mtx_destroy(&wq->exec_mtx);
526 	kfree(wq);
527 }
528 
529 void
530 linux_init_delayed_work(struct delayed_work *dwork, work_func_t func)
531 {
532 	memset(dwork, 0, sizeof(*dwork));
533 	INIT_WORK(&dwork->work, func);
534 	mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL,
535 	    MTX_DEF | MTX_NOWITNESS);
536 	callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0);
537 }
538 
539 static void
540 linux_work_init(void *arg)
541 {
542 	int max_wq_cpus = mp_ncpus + 1;
543 
544 	/* avoid deadlock when there are too few threads */
545 	if (max_wq_cpus < 4)
546 		max_wq_cpus = 4;
547 
548 	/* set default number of CPUs */
549 	linux_default_wq_cpus = max_wq_cpus;
550 
551 	linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus);
552 	linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus);
553 
554 	/* populate the workqueue pointers */
555 	system_long_wq = linux_system_long_wq;
556 	system_wq = linux_system_short_wq;
557 	system_power_efficient_wq = linux_system_short_wq;
558 	system_unbound_wq = linux_system_short_wq;
559 }
560 SYSINIT(linux_work_init, SI_SUB_INIT_IF, SI_ORDER_THIRD, linux_work_init, NULL);
561 
562 static void
563 linux_work_uninit(void *arg)
564 {
565 	destroy_workqueue(linux_system_short_wq);
566 	destroy_workqueue(linux_system_long_wq);
567 
568 	/* clear workqueue pointers */
569 	system_long_wq = NULL;
570 	system_wq = NULL;
571 	system_power_efficient_wq = NULL;
572 	system_unbound_wq = NULL;
573 }
574 SYSUNINIT(linux_work_uninit, SI_SUB_INIT_IF, SI_ORDER_THIRD, linux_work_uninit, NULL);
575