18d59ecb2SHans Petter Selasky /*-
28d59ecb2SHans Petter Selasky * Copyright (c) 2010 Isilon Systems, Inc.
38d59ecb2SHans Petter Selasky * Copyright (c) 2010 iX Systems, Inc.
48d59ecb2SHans Petter Selasky * Copyright (c) 2010 Panasas, Inc.
58d59ecb2SHans Petter Selasky * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
646565964SMark Johnston * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org>
78d59ecb2SHans Petter Selasky * All rights reserved.
88d59ecb2SHans Petter Selasky *
98d59ecb2SHans Petter Selasky * Redistribution and use in source and binary forms, with or without
108d59ecb2SHans Petter Selasky * modification, are permitted provided that the following conditions
118d59ecb2SHans Petter Selasky * are met:
128d59ecb2SHans Petter Selasky * 1. Redistributions of source code must retain the above copyright
138d59ecb2SHans Petter Selasky * notice unmodified, this list of conditions, and the following
148d59ecb2SHans Petter Selasky * disclaimer.
158d59ecb2SHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright
168d59ecb2SHans Petter Selasky * notice, this list of conditions and the following disclaimer in the
178d59ecb2SHans Petter Selasky * documentation and/or other materials provided with the distribution.
188d59ecb2SHans Petter Selasky *
198d59ecb2SHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
208d59ecb2SHans Petter Selasky * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
218d59ecb2SHans Petter Selasky * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
228d59ecb2SHans Petter Selasky * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
238d59ecb2SHans Petter Selasky * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
248d59ecb2SHans Petter Selasky * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
258d59ecb2SHans Petter Selasky * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
268d59ecb2SHans Petter Selasky * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
278d59ecb2SHans Petter Selasky * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
288d59ecb2SHans Petter Selasky * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
298d59ecb2SHans Petter Selasky */
3046565964SMark Johnston
31307f78f3SVladimir Kondratyev #ifndef _LINUXKPI_LINUX_WAIT_H_
32307f78f3SVladimir Kondratyev #define _LINUXKPI_LINUX_WAIT_H_
338d59ecb2SHans Petter Selasky
3485714218SHans Petter Selasky #include <linux/compiler.h>
358d59ecb2SHans Petter Selasky #include <linux/list.h>
3646565964SMark Johnston #include <linux/spinlock.h>
370e123c13SEmmanuel Vadot #include <linux/sched.h>
3846565964SMark Johnston
3946565964SMark Johnston #include <asm/atomic.h>
408d59ecb2SHans Petter Selasky
418d59ecb2SHans Petter Selasky #include <sys/param.h>
428d59ecb2SHans Petter Selasky #include <sys/systm.h>
438d59ecb2SHans Petter Selasky
4446565964SMark Johnston #define SKIP_SLEEP() (SCHEDULER_STOPPED() || kdb_active)
458d59ecb2SHans Petter Selasky
4646565964SMark Johnston #define might_sleep() \
4746565964SMark Johnston WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "might_sleep()")
488d59ecb2SHans Petter Selasky
497e95e98dSHans Petter Selasky #define might_sleep_if(cond) do { \
507e95e98dSHans Petter Selasky if (cond) { might_sleep(); } \
517e95e98dSHans Petter Selasky } while (0)
527e95e98dSHans Petter Selasky
5346565964SMark Johnston struct wait_queue;
5446565964SMark Johnston struct wait_queue_head;
558d59ecb2SHans Petter Selasky
56e6e028d0SHans Petter Selasky #define wait_queue_entry wait_queue
57e6e028d0SHans Petter Selasky
5846565964SMark Johnston typedef struct wait_queue wait_queue_t;
59e6e028d0SHans Petter Selasky typedef struct wait_queue_entry wait_queue_entry_t;
6046565964SMark Johnston typedef struct wait_queue_head wait_queue_head_t;
618d59ecb2SHans Petter Selasky
6246565964SMark Johnston typedef int wait_queue_func_t(wait_queue_t *, unsigned int, int, void *);
6346565964SMark Johnston
6446565964SMark Johnston /*
6546565964SMark Johnston * Many API consumers directly reference these fields and those of
6646565964SMark Johnston * wait_queue_head.
6746565964SMark Johnston */
6846565964SMark Johnston struct wait_queue {
6946565964SMark Johnston unsigned int flags; /* always 0 */
7046565964SMark Johnston void *private;
7146565964SMark Johnston wait_queue_func_t *func;
72ab98f1e8SHans Petter Selasky union {
73ab98f1e8SHans Petter Selasky struct list_head task_list; /* < v4.13 */
74ab98f1e8SHans Petter Selasky struct list_head entry; /* >= v4.13 */
75ab98f1e8SHans Petter Selasky };
7646565964SMark Johnston };
7746565964SMark Johnston
7846565964SMark Johnston struct wait_queue_head {
7946565964SMark Johnston spinlock_t lock;
80ab98f1e8SHans Petter Selasky union {
81ab98f1e8SHans Petter Selasky struct list_head task_list; /* < v4.13 */
82ab98f1e8SHans Petter Selasky struct list_head head; /* >= v4.13 */
83ab98f1e8SHans Petter Selasky };
8446565964SMark Johnston };
8546565964SMark Johnston
8646565964SMark Johnston /*
8746565964SMark Johnston * This function is referenced by at least one DRM driver, so it may not be
8846565964SMark Johnston * renamed and furthermore must be the default wait queue callback.
8946565964SMark Johnston */
9046565964SMark Johnston extern wait_queue_func_t autoremove_wake_function;
918f368d48SHans Petter Selasky extern wait_queue_func_t default_wake_function;
9246565964SMark Johnston
938f368d48SHans Petter Selasky #define DEFINE_WAIT_FUNC(name, function) \
9446565964SMark Johnston wait_queue_t name = { \
9546565964SMark Johnston .private = current, \
968f368d48SHans Petter Selasky .func = function, \
9746565964SMark Johnston .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \
988d59ecb2SHans Petter Selasky }
998d59ecb2SHans Petter Selasky
1008f368d48SHans Petter Selasky #define DEFINE_WAIT(name) \
1018f368d48SHans Petter Selasky DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1028f368d48SHans Petter Selasky
10346565964SMark Johnston #define DECLARE_WAITQUEUE(name, task) \
10446565964SMark Johnston wait_queue_t name = { \
10546565964SMark Johnston .private = task, \
10646565964SMark Johnston .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \
10746565964SMark Johnston }
1088d59ecb2SHans Petter Selasky
10946565964SMark Johnston #define DECLARE_WAIT_QUEUE_HEAD(name) \
11046565964SMark Johnston wait_queue_head_t name = { \
11146565964SMark Johnston .task_list = LINUX_LIST_HEAD_INIT(name.task_list), \
11246565964SMark Johnston }; \
11346565964SMark Johnston MTX_SYSINIT(name, &(name).lock.m, spin_lock_name("wqhead"), MTX_DEF)
11446565964SMark Johnston
11546565964SMark Johnston #define init_waitqueue_head(wqh) do { \
11646565964SMark Johnston mtx_init(&(wqh)->lock.m, spin_lock_name("wqhead"), \
11746565964SMark Johnston NULL, MTX_DEF | MTX_NEW | MTX_NOWITNESS); \
11846565964SMark Johnston INIT_LIST_HEAD(&(wqh)->task_list); \
1198d59ecb2SHans Petter Selasky } while (0)
1208d59ecb2SHans Petter Selasky
121ff443195SEmmanuel Vadot #define __init_waitqueue_head(wqh, name, lk) init_waitqueue_head(wqh)
122ff443195SEmmanuel Vadot
1231b092623SHans Petter Selasky void linux_init_wait_entry(wait_queue_t *, int);
12446565964SMark Johnston void linux_wake_up(wait_queue_head_t *, unsigned int, int, bool);
1258d59ecb2SHans Petter Selasky
1261b092623SHans Petter Selasky #define init_wait_entry(wq, flags) \
1271b092623SHans Petter Selasky linux_init_wait_entry(wq, flags)
12846565964SMark Johnston #define wake_up(wqh) \
12946565964SMark Johnston linux_wake_up(wqh, TASK_NORMAL, 1, false)
13046565964SMark Johnston #define wake_up_all(wqh) \
13146565964SMark Johnston linux_wake_up(wqh, TASK_NORMAL, 0, false)
13246565964SMark Johnston #define wake_up_locked(wqh) \
13346565964SMark Johnston linux_wake_up(wqh, TASK_NORMAL, 1, true)
13446565964SMark Johnston #define wake_up_all_locked(wqh) \
13546565964SMark Johnston linux_wake_up(wqh, TASK_NORMAL, 0, true)
13646565964SMark Johnston #define wake_up_interruptible(wqh) \
13746565964SMark Johnston linux_wake_up(wqh, TASK_INTERRUPTIBLE, 1, false)
13846565964SMark Johnston #define wake_up_interruptible_all(wqh) \
13946565964SMark Johnston linux_wake_up(wqh, TASK_INTERRUPTIBLE, 0, false)
14046565964SMark Johnston
14146565964SMark Johnston int linux_wait_event_common(wait_queue_head_t *, wait_queue_t *, int,
14246565964SMark Johnston unsigned int, spinlock_t *);
14346565964SMark Johnston
14446565964SMark Johnston /*
14546565964SMark Johnston * Returns -ERESTARTSYS for a signal, 0 if cond is false after timeout, 1 if
14646565964SMark Johnston * cond is true after timeout, remaining jiffies (> 0) if cond is true before
14746565964SMark Johnston * timeout.
14846565964SMark Johnston */
14946565964SMark Johnston #define __wait_event_common(wqh, cond, timeout, state, lock) ({ \
15046565964SMark Johnston DEFINE_WAIT(__wq); \
1514ef8a630SHans Petter Selasky const int __timeout = ((int)(timeout)) < 1 ? 1 : (timeout); \
15246565964SMark Johnston int __start = ticks; \
153b0338411SNavdeep Parhar int __ret = 0; \
154b0338411SNavdeep Parhar \
15546565964SMark Johnston for (;;) { \
15646565964SMark Johnston linux_prepare_to_wait(&(wqh), &__wq, state); \
1574ef8a630SHans Petter Selasky if (cond) \
158b0338411SNavdeep Parhar break; \
15946565964SMark Johnston __ret = linux_wait_event_common(&(wqh), &__wq, \
16046565964SMark Johnston __timeout, state, lock); \
16146565964SMark Johnston if (__ret != 0) \
16246565964SMark Johnston break; \
163b0338411SNavdeep Parhar } \
16446565964SMark Johnston linux_finish_wait(&(wqh), &__wq); \
16546565964SMark Johnston if (__timeout != MAX_SCHEDULE_TIMEOUT) { \
16646565964SMark Johnston if (__ret == -EWOULDBLOCK) \
16746565964SMark Johnston __ret = !!(cond); \
16846565964SMark Johnston else if (__ret != -ERESTARTSYS) { \
16946565964SMark Johnston __ret = __timeout + __start - ticks; \
17046565964SMark Johnston /* range check return value */ \
171b0338411SNavdeep Parhar if (__ret < 1) \
172b0338411SNavdeep Parhar __ret = 1; \
17346565964SMark Johnston else if (__ret > __timeout) \
17446565964SMark Johnston __ret = __timeout; \
17546565964SMark Johnston } \
176b0338411SNavdeep Parhar } \
177b0338411SNavdeep Parhar __ret; \
178b0338411SNavdeep Parhar })
179b0338411SNavdeep Parhar
1804ef8a630SHans Petter Selasky #define wait_event(wqh, cond) do { \
1814ef8a630SHans Petter Selasky (void) __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
18246565964SMark Johnston TASK_UNINTERRUPTIBLE, NULL); \
1834ef8a630SHans Petter Selasky } while (0)
184b0338411SNavdeep Parhar
18546565964SMark Johnston #define wait_event_timeout(wqh, cond, timeout) ({ \
18646565964SMark Johnston __wait_event_common(wqh, cond, timeout, TASK_UNINTERRUPTIBLE, \
18746565964SMark Johnston NULL); \
18846565964SMark Johnston })
1898d59ecb2SHans Petter Selasky
190bd40dea7SHans Petter Selasky #define wait_event_killable(wqh, cond) ({ \
191bd40dea7SHans Petter Selasky __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
192bd40dea7SHans Petter Selasky TASK_INTERRUPTIBLE, NULL); \
193bd40dea7SHans Petter Selasky })
194bd40dea7SHans Petter Selasky
19546565964SMark Johnston #define wait_event_interruptible(wqh, cond) ({ \
19646565964SMark Johnston __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
19746565964SMark Johnston TASK_INTERRUPTIBLE, NULL); \
19846565964SMark Johnston })
19946565964SMark Johnston
20046565964SMark Johnston #define wait_event_interruptible_timeout(wqh, cond, timeout) ({ \
20146565964SMark Johnston __wait_event_common(wqh, cond, timeout, TASK_INTERRUPTIBLE, \
20246565964SMark Johnston NULL); \
20346565964SMark Johnston })
20446565964SMark Johnston
20546565964SMark Johnston /*
20646565964SMark Johnston * Wait queue is already locked.
20746565964SMark Johnston */
20846565964SMark Johnston #define wait_event_interruptible_locked(wqh, cond) ({ \
20946565964SMark Johnston int __ret; \
21046565964SMark Johnston \
21146565964SMark Johnston spin_unlock(&(wqh).lock); \
21246565964SMark Johnston __ret = __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
21346565964SMark Johnston TASK_INTERRUPTIBLE, NULL); \
21446565964SMark Johnston spin_lock(&(wqh).lock); \
21546565964SMark Johnston __ret; \
21646565964SMark Johnston })
21746565964SMark Johnston
21846565964SMark Johnston /*
219c3bfe0deSHans Petter Selasky * The passed spinlock is held when testing the condition.
22046565964SMark Johnston */
22146565964SMark Johnston #define wait_event_interruptible_lock_irq(wqh, cond, lock) ({ \
22246565964SMark Johnston __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
22346565964SMark Johnston TASK_INTERRUPTIBLE, &(lock)); \
22446565964SMark Johnston })
2258d59ecb2SHans Petter Selasky
226c3bfe0deSHans Petter Selasky /*
227c3bfe0deSHans Petter Selasky * The passed spinlock is held when testing the condition.
228c3bfe0deSHans Petter Selasky */
229c3bfe0deSHans Petter Selasky #define wait_event_lock_irq(wqh, cond, lock) ({ \
230c3bfe0deSHans Petter Selasky __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
231c3bfe0deSHans Petter Selasky TASK_UNINTERRUPTIBLE, &(lock)); \
232c3bfe0deSHans Petter Selasky })
233c3bfe0deSHans Petter Selasky
2348d59ecb2SHans Petter Selasky static inline void
__add_wait_queue(wait_queue_head_t * wqh,wait_queue_t * wq)23546565964SMark Johnston __add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq)
2368d59ecb2SHans Petter Selasky {
23746565964SMark Johnston list_add(&wq->task_list, &wqh->task_list);
2388d59ecb2SHans Petter Selasky }
2398d59ecb2SHans Petter Selasky
2408d59ecb2SHans Petter Selasky static inline void
add_wait_queue(wait_queue_head_t * wqh,wait_queue_t * wq)24146565964SMark Johnston add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq)
2428d59ecb2SHans Petter Selasky {
24346565964SMark Johnston
24446565964SMark Johnston spin_lock(&wqh->lock);
24546565964SMark Johnston __add_wait_queue(wqh, wq);
24646565964SMark Johnston spin_unlock(&wqh->lock);
2478d59ecb2SHans Petter Selasky }
2488d59ecb2SHans Petter Selasky
24946565964SMark Johnston static inline void
__add_wait_queue_tail(wait_queue_head_t * wqh,wait_queue_t * wq)25046565964SMark Johnston __add_wait_queue_tail(wait_queue_head_t *wqh, wait_queue_t *wq)
25146565964SMark Johnston {
25246565964SMark Johnston list_add_tail(&wq->task_list, &wqh->task_list);
25346565964SMark Johnston }
25446565964SMark Johnston
25546565964SMark Johnston static inline void
__add_wait_queue_entry_tail(wait_queue_head_t * wqh,wait_queue_entry_t * wq)256e6e028d0SHans Petter Selasky __add_wait_queue_entry_tail(wait_queue_head_t *wqh, wait_queue_entry_t *wq)
257e6e028d0SHans Petter Selasky {
258e6e028d0SHans Petter Selasky list_add_tail(&wq->entry, &wqh->head);
259e6e028d0SHans Petter Selasky }
260e6e028d0SHans Petter Selasky
261e6e028d0SHans Petter Selasky static inline void
__remove_wait_queue(wait_queue_head_t * wqh,wait_queue_t * wq)26246565964SMark Johnston __remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq)
26346565964SMark Johnston {
26446565964SMark Johnston list_del(&wq->task_list);
26546565964SMark Johnston }
26646565964SMark Johnston
26746565964SMark Johnston static inline void
remove_wait_queue(wait_queue_head_t * wqh,wait_queue_t * wq)26846565964SMark Johnston remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq)
26946565964SMark Johnston {
27046565964SMark Johnston
27146565964SMark Johnston spin_lock(&wqh->lock);
27246565964SMark Johnston __remove_wait_queue(wqh, wq);
27346565964SMark Johnston spin_unlock(&wqh->lock);
27446565964SMark Johnston }
27546565964SMark Johnston
27646565964SMark Johnston bool linux_waitqueue_active(wait_queue_head_t *);
27746565964SMark Johnston
27846565964SMark Johnston #define waitqueue_active(wqh) linux_waitqueue_active(wqh)
27946565964SMark Johnston
28046565964SMark Johnston void linux_prepare_to_wait(wait_queue_head_t *, wait_queue_t *, int);
28146565964SMark Johnston void linux_finish_wait(wait_queue_head_t *, wait_queue_t *);
28246565964SMark Johnston
28346565964SMark Johnston #define prepare_to_wait(wqh, wq, state) linux_prepare_to_wait(wqh, wq, state)
28446565964SMark Johnston #define finish_wait(wqh, wq) linux_finish_wait(wqh, wq)
28546565964SMark Johnston
28646565964SMark Johnston void linux_wake_up_bit(void *, int);
28746565964SMark Johnston int linux_wait_on_bit_timeout(unsigned long *, int, unsigned int, int);
28846565964SMark Johnston void linux_wake_up_atomic_t(atomic_t *);
28946565964SMark Johnston int linux_wait_on_atomic_t(atomic_t *, unsigned int);
29046565964SMark Johnston
29146565964SMark Johnston #define wake_up_bit(word, bit) linux_wake_up_bit(word, bit)
292d901abf1SHans Petter Selasky #define wait_on_bit(word, bit, state) \
293d901abf1SHans Petter Selasky linux_wait_on_bit_timeout(word, bit, state, MAX_SCHEDULE_TIMEOUT)
29446565964SMark Johnston #define wait_on_bit_timeout(word, bit, state, timeout) \
29546565964SMark Johnston linux_wait_on_bit_timeout(word, bit, state, timeout)
29646565964SMark Johnston #define wake_up_atomic_t(a) linux_wake_up_atomic_t(a)
29746565964SMark Johnston /*
29846565964SMark Johnston * All existing callers have a cb that just schedule()s. To avoid adding
29946565964SMark Johnston * complexity, just emulate that internally. The prototype is different so that
30046565964SMark Johnston * callers must be manually modified; a cb that does something other than call
30146565964SMark Johnston * schedule() will require special treatment.
30246565964SMark Johnston */
30346565964SMark Johnston #define wait_on_atomic_t(a, state) linux_wait_on_atomic_t(a, state)
30446565964SMark Johnston
30546565964SMark Johnston struct task_struct;
30646565964SMark Johnston bool linux_wake_up_state(struct task_struct *, unsigned int);
30746565964SMark Johnston
30846565964SMark Johnston #define wake_up_process(task) linux_wake_up_state(task, TASK_NORMAL)
30946565964SMark Johnston #define wake_up_state(task, state) linux_wake_up_state(task, state)
31046565964SMark Johnston
311307f78f3SVladimir Kondratyev #endif /* _LINUXKPI_LINUX_WAIT_H_ */
312