1 /* SPDX-License-Identifier: GPL-2.0 */
5 * Linux wait queue related types and methods
7 #include <linux/list.h>
8 #include <linux/stddef.h>
9 #include <linux/spinlock.h>
11 #include <asm/current.h>
12 #include <uapi/linux/wait.h>
14 typedef struct wait_queue_entry wait_queue_entry_t;
16 typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
17 int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
19 /* wait_queue_entry::flags */
20 #define WQ_FLAG_EXCLUSIVE 0x01
21 #define WQ_FLAG_WOKEN 0x02
22 #define WQ_FLAG_BOOKMARK 0x04
23 #define WQ_FLAG_CUSTOM 0x08
24 #define WQ_FLAG_DONE 0x10
25 #define WQ_FLAG_PRIORITY 0x20
28 * A single wait-queue entry structure:
30 struct wait_queue_entry {
33 wait_queue_func_t func;
34 struct list_head entry;
37 struct wait_queue_head {
39 struct list_head head;
41 typedef struct wait_queue_head wait_queue_head_t;
46 * Macros for declaration and initialisaton of the datatypes
49 #define __WAITQUEUE_INITIALIZER(name, tsk) { \
51 .func = default_wake_function, \
52 .entry = { NULL, NULL } }
54 #define DECLARE_WAITQUEUE(name, tsk) \
55 struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
57 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
58 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
59 .head = LIST_HEAD_INIT(name.head) }
61 #define DECLARE_WAIT_QUEUE_HEAD(name) \
62 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
64 extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
66 #define init_waitqueue_head(wq_head) \
68 static struct lock_class_key __key; \
70 __init_waitqueue_head((wq_head), #wq_head, &__key); \
74 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
75 ({ init_waitqueue_head(&name); name; })
76 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
77 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
79 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
82 static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
85 wq_entry->private = p;
86 wq_entry->func = default_wake_function;
90 init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
93 wq_entry->private = NULL;
94 wq_entry->func = func;
98 * waitqueue_active -- locklessly test for waiters on the queue
99 * @wq_head: the waitqueue to test for waiters
101 * returns true if the wait list is not empty
103 * NOTE: this function is lockless and requires care, incorrect usage _will_
104 * lead to sporadic and non-obvious failure.
106 * Use either while holding wait_queue_head::lock or when used for wakeups
107 * with an extra smp_mb() like::
109 * CPU0 - waker CPU1 - waiter
112 * @cond = true; prepare_to_wait(&wq_head, &wait, state);
113 * smp_mb(); // smp_mb() from set_current_state()
114 * if (waitqueue_active(wq_head)) if (@cond)
115 * wake_up(wq_head); break;
118 * finish_wait(&wq_head, &wait);
120 * Because without the explicit smp_mb() it's possible for the
121 * waitqueue_active() load to get hoisted over the @cond store such that we'll
122 * observe an empty wait list while the waiter might not observe @cond.
124 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
125 * which (when the lock is uncontended) are of roughly equal cost.
127 static inline int waitqueue_active(struct wait_queue_head *wq_head)
129 return !list_empty(&wq_head->head);
133 * wq_has_single_sleeper - check if there is only one sleeper
134 * @wq_head: wait queue head
136 * Returns true of wq_head has only one sleeper on the list.
138 * Please refer to the comment for waitqueue_active.
140 static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
142 return list_is_singular(&wq_head->head);
146 * wq_has_sleeper - check if there are any waiting processes
147 * @wq_head: wait queue head
149 * Returns true if wq_head has waiting processes
151 * Please refer to the comment for waitqueue_active.
153 static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
156 * We need to be sure we are in sync with the
157 * add_wait_queue modifications to the wait queue.
159 * This memory barrier should be paired with one on the
163 return waitqueue_active(wq_head);
166 extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
167 extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
168 extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
169 extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
171 static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
173 struct list_head *head = &wq_head->head;
174 struct wait_queue_entry *wq;
176 list_for_each_entry(wq, &wq_head->head, entry) {
177 if (!(wq->flags & WQ_FLAG_PRIORITY))
181 list_add(&wq_entry->entry, head);
185 * Used for wake-one threads:
188 __add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
190 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
191 __add_wait_queue(wq_head, wq_entry);
194 static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
196 list_add_tail(&wq_entry->entry, &wq_head->head);
200 __add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
202 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
203 __add_wait_queue_entry_tail(wq_head, wq_entry);
207 __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
209 list_del(&wq_entry->entry);
212 void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
213 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
214 void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
215 unsigned int mode, void *key, wait_queue_entry_t *bookmark);
216 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
217 void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
218 void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
219 void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
220 void __wake_up_pollfree(struct wait_queue_head *wq_head);
222 #define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
223 #define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
224 #define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
225 #define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
226 #define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
228 #define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
229 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
230 #define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
231 #define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE)
234 * Wakeup macros to be used to report events to the targets.
236 #define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
237 #define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
238 #define wake_up_poll(x, m) \
239 __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
240 #define wake_up_locked_poll(x, m) \
241 __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
242 #define wake_up_interruptible_poll(x, m) \
243 __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
244 #define wake_up_interruptible_sync_poll(x, m) \
245 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
246 #define wake_up_interruptible_sync_poll_locked(x, m) \
247 __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
250 * wake_up_pollfree - signal that a polled waitqueue is going away
251 * @wq_head: the wait queue head
253 * In the very rare cases where a ->poll() implementation uses a waitqueue whose
254 * lifetime is tied to a task rather than to the 'struct file' being polled,
255 * this function must be called before the waitqueue is freed so that
256 * non-blocking polls (e.g. epoll) are notified that the queue is going away.
258 * The caller must also RCU-delay the freeing of the wait_queue_head, e.g. via
259 * an explicit synchronize_rcu() or call_rcu(), or via SLAB_TYPESAFE_BY_RCU.
261 static inline void wake_up_pollfree(struct wait_queue_head *wq_head)
264 * For performance reasons, we don't always take the queue lock here.
265 * Therefore, we might race with someone removing the last entry from
266 * the queue, and proceed while they still hold the queue lock.
267 * However, rcu_read_lock() is required to be held in such cases, so we
268 * can safely proceed with an RCU-delayed free.
270 if (waitqueue_active(wq_head))
271 __wake_up_pollfree(wq_head);
274 #define ___wait_cond_timeout(condition) \
276 bool __cond = (condition); \
277 if (__cond && !__ret) \
282 #define ___wait_is_interruptible(state) \
283 (!__builtin_constant_p(state) || \
284 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \
286 extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
289 * The below macro ___wait_event() has an explicit shadow of the __ret
290 * variable when used from the wait_event_*() macros.
292 * This is so that both can use the ___wait_cond_timeout() construct
293 * to wrap the condition.
295 * The type inconsistency of the wait_event_*() __ret variable is also
296 * on purpose; we use long where we can return timeout values and int
300 #define ___wait_event(wq_head, condition, state, exclusive, ret, cmd) \
303 struct wait_queue_entry __wq_entry; \
304 long __ret = ret; /* explicit shadow */ \
306 init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0); \
308 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
313 if (___wait_is_interruptible(state) && __int) { \
320 finish_wait(&wq_head, &__wq_entry); \
324 #define __wait_event(wq_head, condition) \
325 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
329 * wait_event - sleep until a condition gets true
330 * @wq_head: the waitqueue to wait on
331 * @condition: a C expression for the event to wait for
333 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
334 * @condition evaluates to true. The @condition is checked each time
335 * the waitqueue @wq_head is woken up.
337 * wake_up() has to be called after changing any variable that could
338 * change the result of the wait condition.
340 #define wait_event(wq_head, condition) \
345 __wait_event(wq_head, condition); \
348 #define __io_wait_event(wq_head, condition) \
349 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
353 * io_wait_event() -- like wait_event() but with io_schedule()
355 #define io_wait_event(wq_head, condition) \
360 __io_wait_event(wq_head, condition); \
363 #define __wait_event_freezable(wq_head, condition) \
364 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
365 freezable_schedule())
368 * wait_event_freezable - sleep (or freeze) until a condition gets true
369 * @wq_head: the waitqueue to wait on
370 * @condition: a C expression for the event to wait for
372 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
373 * to system load) until the @condition evaluates to true. The
374 * @condition is checked each time the waitqueue @wq_head is woken up.
376 * wake_up() has to be called after changing any variable that could
377 * change the result of the wait condition.
379 #define wait_event_freezable(wq_head, condition) \
384 __ret = __wait_event_freezable(wq_head, condition); \
388 #define __wait_event_timeout(wq_head, condition, timeout) \
389 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
390 TASK_UNINTERRUPTIBLE, 0, timeout, \
391 __ret = schedule_timeout(__ret))
394 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
395 * @wq_head: the waitqueue to wait on
396 * @condition: a C expression for the event to wait for
397 * @timeout: timeout, in jiffies
399 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
400 * @condition evaluates to true. The @condition is checked each time
401 * the waitqueue @wq_head is woken up.
403 * wake_up() has to be called after changing any variable that could
404 * change the result of the wait condition.
407 * 0 if the @condition evaluated to %false after the @timeout elapsed,
408 * 1 if the @condition evaluated to %true after the @timeout elapsed,
409 * or the remaining jiffies (at least 1) if the @condition evaluated
410 * to %true before the @timeout elapsed.
412 #define wait_event_timeout(wq_head, condition, timeout) \
414 long __ret = timeout; \
416 if (!___wait_cond_timeout(condition)) \
417 __ret = __wait_event_timeout(wq_head, condition, timeout); \
421 #define __wait_event_freezable_timeout(wq_head, condition, timeout) \
422 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
423 TASK_INTERRUPTIBLE, 0, timeout, \
424 __ret = freezable_schedule_timeout(__ret))
427 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
428 * increasing load and is freezable.
430 #define wait_event_freezable_timeout(wq_head, condition, timeout) \
432 long __ret = timeout; \
434 if (!___wait_cond_timeout(condition)) \
435 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
439 #define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
440 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0, \
441 cmd1; schedule(); cmd2)
443 * Just like wait_event_cmd(), except it sets exclusive flag
445 #define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
449 __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2); \
452 #define __wait_event_cmd(wq_head, condition, cmd1, cmd2) \
453 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
454 cmd1; schedule(); cmd2)
457 * wait_event_cmd - sleep until a condition gets true
458 * @wq_head: the waitqueue to wait on
459 * @condition: a C expression for the event to wait for
460 * @cmd1: the command will be executed before sleep
461 * @cmd2: the command will be executed after sleep
463 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
464 * @condition evaluates to true. The @condition is checked each time
465 * the waitqueue @wq_head is woken up.
467 * wake_up() has to be called after changing any variable that could
468 * change the result of the wait condition.
470 #define wait_event_cmd(wq_head, condition, cmd1, cmd2) \
474 __wait_event_cmd(wq_head, condition, cmd1, cmd2); \
477 #define __wait_event_interruptible(wq_head, condition) \
478 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
482 * wait_event_interruptible - sleep until a condition gets true
483 * @wq_head: the waitqueue to wait on
484 * @condition: a C expression for the event to wait for
486 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
487 * @condition evaluates to true or a signal is received.
488 * The @condition is checked each time the waitqueue @wq_head is woken up.
490 * wake_up() has to be called after changing any variable that could
491 * change the result of the wait condition.
493 * The function will return -ERESTARTSYS if it was interrupted by a
494 * signal and 0 if @condition evaluated to true.
496 #define wait_event_interruptible(wq_head, condition) \
501 __ret = __wait_event_interruptible(wq_head, condition); \
505 #define __wait_event_interruptible_timeout(wq_head, condition, timeout) \
506 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
507 TASK_INTERRUPTIBLE, 0, timeout, \
508 __ret = schedule_timeout(__ret))
511 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
512 * @wq_head: the waitqueue to wait on
513 * @condition: a C expression for the event to wait for
514 * @timeout: timeout, in jiffies
516 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
517 * @condition evaluates to true or a signal is received.
518 * The @condition is checked each time the waitqueue @wq_head is woken up.
520 * wake_up() has to be called after changing any variable that could
521 * change the result of the wait condition.
524 * 0 if the @condition evaluated to %false after the @timeout elapsed,
525 * 1 if the @condition evaluated to %true after the @timeout elapsed,
526 * the remaining jiffies (at least 1) if the @condition evaluated
527 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
528 * interrupted by a signal.
530 #define wait_event_interruptible_timeout(wq_head, condition, timeout) \
532 long __ret = timeout; \
534 if (!___wait_cond_timeout(condition)) \
535 __ret = __wait_event_interruptible_timeout(wq_head, \
536 condition, timeout); \
540 #define __wait_event_hrtimeout(wq_head, condition, timeout, state) \
543 struct hrtimer_sleeper __t; \
545 hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
547 if ((timeout) != KTIME_MAX) \
548 hrtimer_start_range_ns(&__t.timer, timeout, \
549 current->timer_slack_ns, \
552 __ret = ___wait_event(wq_head, condition, state, 0, 0, \
559 hrtimer_cancel(&__t.timer); \
560 destroy_hrtimer_on_stack(&__t.timer); \
565 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
566 * @wq_head: the waitqueue to wait on
567 * @condition: a C expression for the event to wait for
568 * @timeout: timeout, as a ktime_t
570 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
571 * @condition evaluates to true or a signal is received.
572 * The @condition is checked each time the waitqueue @wq_head is woken up.
574 * wake_up() has to be called after changing any variable that could
575 * change the result of the wait condition.
577 * The function returns 0 if @condition became true, or -ETIME if the timeout
580 #define wait_event_hrtimeout(wq_head, condition, timeout) \
585 __ret = __wait_event_hrtimeout(wq_head, condition, timeout, \
586 TASK_UNINTERRUPTIBLE); \
591 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
592 * @wq: the waitqueue to wait on
593 * @condition: a C expression for the event to wait for
594 * @timeout: timeout, as a ktime_t
596 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
597 * @condition evaluates to true or a signal is received.
598 * The @condition is checked each time the waitqueue @wq is woken up.
600 * wake_up() has to be called after changing any variable that could
601 * change the result of the wait condition.
603 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
604 * interrupted by a signal, or -ETIME if the timeout elapsed.
606 #define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
611 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
612 TASK_INTERRUPTIBLE); \
616 #define __wait_event_interruptible_exclusive(wq, condition) \
617 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
620 #define wait_event_interruptible_exclusive(wq, condition) \
625 __ret = __wait_event_interruptible_exclusive(wq, condition); \
629 #define __wait_event_killable_exclusive(wq, condition) \
630 ___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \
633 #define wait_event_killable_exclusive(wq, condition) \
638 __ret = __wait_event_killable_exclusive(wq, condition); \
643 #define __wait_event_freezable_exclusive(wq, condition) \
644 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
645 freezable_schedule())
647 #define wait_event_freezable_exclusive(wq, condition) \
652 __ret = __wait_event_freezable_exclusive(wq, condition); \
657 * wait_event_idle - wait for a condition without contributing to system load
658 * @wq_head: the waitqueue to wait on
659 * @condition: a C expression for the event to wait for
661 * The process is put to sleep (TASK_IDLE) until the
662 * @condition evaluates to true.
663 * The @condition is checked each time the waitqueue @wq_head is woken up.
665 * wake_up() has to be called after changing any variable that could
666 * change the result of the wait condition.
669 #define wait_event_idle(wq_head, condition) \
673 ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule()); \
677 * wait_event_idle_exclusive - wait for a condition with contributing to system load
678 * @wq_head: the waitqueue to wait on
679 * @condition: a C expression for the event to wait for
681 * The process is put to sleep (TASK_IDLE) until the
682 * @condition evaluates to true.
683 * The @condition is checked each time the waitqueue @wq_head is woken up.
685 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
686 * set thus if other processes wait on the same list, when this
687 * process is woken further processes are not considered.
689 * wake_up() has to be called after changing any variable that could
690 * change the result of the wait condition.
693 #define wait_event_idle_exclusive(wq_head, condition) \
697 ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule()); \
700 #define __wait_event_idle_timeout(wq_head, condition, timeout) \
701 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
702 TASK_IDLE, 0, timeout, \
703 __ret = schedule_timeout(__ret))
706 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
707 * @wq_head: the waitqueue to wait on
708 * @condition: a C expression for the event to wait for
709 * @timeout: timeout, in jiffies
711 * The process is put to sleep (TASK_IDLE) until the
712 * @condition evaluates to true. The @condition is checked each time
713 * the waitqueue @wq_head is woken up.
715 * wake_up() has to be called after changing any variable that could
716 * change the result of the wait condition.
719 * 0 if the @condition evaluated to %false after the @timeout elapsed,
720 * 1 if the @condition evaluated to %true after the @timeout elapsed,
721 * or the remaining jiffies (at least 1) if the @condition evaluated
722 * to %true before the @timeout elapsed.
724 #define wait_event_idle_timeout(wq_head, condition, timeout) \
726 long __ret = timeout; \
728 if (!___wait_cond_timeout(condition)) \
729 __ret = __wait_event_idle_timeout(wq_head, condition, timeout); \
733 #define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
734 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
735 TASK_IDLE, 1, timeout, \
736 __ret = schedule_timeout(__ret))
739 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
740 * @wq_head: the waitqueue to wait on
741 * @condition: a C expression for the event to wait for
742 * @timeout: timeout, in jiffies
744 * The process is put to sleep (TASK_IDLE) until the
745 * @condition evaluates to true. The @condition is checked each time
746 * the waitqueue @wq_head is woken up.
748 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
749 * set thus if other processes wait on the same list, when this
750 * process is woken further processes are not considered.
752 * wake_up() has to be called after changing any variable that could
753 * change the result of the wait condition.
756 * 0 if the @condition evaluated to %false after the @timeout elapsed,
757 * 1 if the @condition evaluated to %true after the @timeout elapsed,
758 * or the remaining jiffies (at least 1) if the @condition evaluated
759 * to %true before the @timeout elapsed.
761 #define wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
763 long __ret = timeout; \
765 if (!___wait_cond_timeout(condition)) \
766 __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
770 extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
771 extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
773 #define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
776 DEFINE_WAIT(__wait); \
778 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
780 __ret = fn(&(wq), &__wait); \
783 } while (!(condition)); \
784 __remove_wait_queue(&(wq), &__wait); \
785 __set_current_state(TASK_RUNNING); \
791 * wait_event_interruptible_locked - sleep until a condition gets true
792 * @wq: the waitqueue to wait on
793 * @condition: a C expression for the event to wait for
795 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
796 * @condition evaluates to true or a signal is received.
797 * The @condition is checked each time the waitqueue @wq is woken up.
799 * It must be called with wq.lock being held. This spinlock is
800 * unlocked while sleeping but @condition testing is done while lock
801 * is held and when this macro exits the lock is held.
803 * The lock is locked/unlocked using spin_lock()/spin_unlock()
804 * functions which must match the way they are locked/unlocked outside
807 * wake_up_locked() has to be called after changing any variable that could
808 * change the result of the wait condition.
810 * The function will return -ERESTARTSYS if it was interrupted by a
811 * signal and 0 if @condition evaluated to true.
813 #define wait_event_interruptible_locked(wq, condition) \
815 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
818 * wait_event_interruptible_locked_irq - sleep until a condition gets true
819 * @wq: the waitqueue to wait on
820 * @condition: a C expression for the event to wait for
822 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
823 * @condition evaluates to true or a signal is received.
824 * The @condition is checked each time the waitqueue @wq is woken up.
826 * It must be called with wq.lock being held. This spinlock is
827 * unlocked while sleeping but @condition testing is done while lock
828 * is held and when this macro exits the lock is held.
830 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
831 * functions which must match the way they are locked/unlocked outside
834 * wake_up_locked() has to be called after changing any variable that could
835 * change the result of the wait condition.
837 * The function will return -ERESTARTSYS if it was interrupted by a
838 * signal and 0 if @condition evaluated to true.
840 #define wait_event_interruptible_locked_irq(wq, condition) \
842 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
845 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
846 * @wq: the waitqueue to wait on
847 * @condition: a C expression for the event to wait for
849 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
850 * @condition evaluates to true or a signal is received.
851 * The @condition is checked each time the waitqueue @wq is woken up.
853 * It must be called with wq.lock being held. This spinlock is
854 * unlocked while sleeping but @condition testing is done while lock
855 * is held and when this macro exits the lock is held.
857 * The lock is locked/unlocked using spin_lock()/spin_unlock()
858 * functions which must match the way they are locked/unlocked outside
861 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
862 * set thus when other process waits process on the list if this
863 * process is awaken further processes are not considered.
865 * wake_up_locked() has to be called after changing any variable that could
866 * change the result of the wait condition.
868 * The function will return -ERESTARTSYS if it was interrupted by a
869 * signal and 0 if @condition evaluated to true.
871 #define wait_event_interruptible_exclusive_locked(wq, condition) \
873 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
876 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
877 * @wq: the waitqueue to wait on
878 * @condition: a C expression for the event to wait for
880 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
881 * @condition evaluates to true or a signal is received.
882 * The @condition is checked each time the waitqueue @wq is woken up.
884 * It must be called with wq.lock being held. This spinlock is
885 * unlocked while sleeping but @condition testing is done while lock
886 * is held and when this macro exits the lock is held.
888 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
889 * functions which must match the way they are locked/unlocked outside
892 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
893 * set thus when other process waits process on the list if this
894 * process is awaken further processes are not considered.
896 * wake_up_locked() has to be called after changing any variable that could
897 * change the result of the wait condition.
899 * The function will return -ERESTARTSYS if it was interrupted by a
900 * signal and 0 if @condition evaluated to true.
902 #define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
904 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
907 #define __wait_event_killable(wq, condition) \
908 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
911 * wait_event_killable - sleep until a condition gets true
912 * @wq_head: the waitqueue to wait on
913 * @condition: a C expression for the event to wait for
915 * The process is put to sleep (TASK_KILLABLE) until the
916 * @condition evaluates to true or a signal is received.
917 * The @condition is checked each time the waitqueue @wq_head is woken up.
919 * wake_up() has to be called after changing any variable that could
920 * change the result of the wait condition.
922 * The function will return -ERESTARTSYS if it was interrupted by a
923 * signal and 0 if @condition evaluated to true.
925 #define wait_event_killable(wq_head, condition) \
930 __ret = __wait_event_killable(wq_head, condition); \
934 #define __wait_event_killable_timeout(wq_head, condition, timeout) \
935 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
936 TASK_KILLABLE, 0, timeout, \
937 __ret = schedule_timeout(__ret))
940 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
941 * @wq_head: the waitqueue to wait on
942 * @condition: a C expression for the event to wait for
943 * @timeout: timeout, in jiffies
945 * The process is put to sleep (TASK_KILLABLE) until the
946 * @condition evaluates to true or a kill signal is received.
947 * The @condition is checked each time the waitqueue @wq_head is woken up.
949 * wake_up() has to be called after changing any variable that could
950 * change the result of the wait condition.
953 * 0 if the @condition evaluated to %false after the @timeout elapsed,
954 * 1 if the @condition evaluated to %true after the @timeout elapsed,
955 * the remaining jiffies (at least 1) if the @condition evaluated
956 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
957 * interrupted by a kill signal.
959 * Only kill signals interrupt this process.
961 #define wait_event_killable_timeout(wq_head, condition, timeout) \
963 long __ret = timeout; \
965 if (!___wait_cond_timeout(condition)) \
966 __ret = __wait_event_killable_timeout(wq_head, \
967 condition, timeout); \
972 #define __wait_event_lock_irq(wq_head, condition, lock, cmd) \
973 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
974 spin_unlock_irq(&lock); \
977 spin_lock_irq(&lock))
980 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
981 * condition is checked under the lock. This
982 * is expected to be called with the lock
984 * @wq_head: the waitqueue to wait on
985 * @condition: a C expression for the event to wait for
986 * @lock: a locked spinlock_t, which will be released before cmd
987 * and schedule() and reacquired afterwards.
988 * @cmd: a command which is invoked outside the critical section before
991 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
992 * @condition evaluates to true. The @condition is checked each time
993 * the waitqueue @wq_head is woken up.
995 * wake_up() has to be called after changing any variable that could
996 * change the result of the wait condition.
998 * This is supposed to be called while holding the lock. The lock is
999 * dropped before invoking the cmd and going to sleep and is reacquired
1002 #define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd) \
1006 __wait_event_lock_irq(wq_head, condition, lock, cmd); \
1010 * wait_event_lock_irq - sleep until a condition gets true. The
1011 * condition is checked under the lock. This
1012 * is expected to be called with the lock
1014 * @wq_head: the waitqueue to wait on
1015 * @condition: a C expression for the event to wait for
1016 * @lock: a locked spinlock_t, which will be released before schedule()
1017 * and reacquired afterwards.
1019 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
1020 * @condition evaluates to true. The @condition is checked each time
1021 * the waitqueue @wq_head is woken up.
1023 * wake_up() has to be called after changing any variable that could
1024 * change the result of the wait condition.
1026 * This is supposed to be called while holding the lock. The lock is
1027 * dropped before going to sleep and is reacquired afterwards.
1029 #define wait_event_lock_irq(wq_head, condition, lock) \
1033 __wait_event_lock_irq(wq_head, condition, lock, ); \
1037 #define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd) \
1038 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
1039 spin_unlock_irq(&lock); \
1042 spin_lock_irq(&lock))
1045 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
1046 * The condition is checked under the lock. This is expected to
1047 * be called with the lock taken.
1048 * @wq_head: the waitqueue to wait on
1049 * @condition: a C expression for the event to wait for
1050 * @lock: a locked spinlock_t, which will be released before cmd and
1051 * schedule() and reacquired afterwards.
1052 * @cmd: a command which is invoked outside the critical section before
1055 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1056 * @condition evaluates to true or a signal is received. The @condition is
1057 * checked each time the waitqueue @wq_head is woken up.
1059 * wake_up() has to be called after changing any variable that could
1060 * change the result of the wait condition.
1062 * This is supposed to be called while holding the lock. The lock is
1063 * dropped before invoking the cmd and going to sleep and is reacquired
1066 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1067 * and 0 if @condition evaluated to true.
1069 #define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd) \
1073 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1074 condition, lock, cmd); \
1079 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
1080 * The condition is checked under the lock. This is expected
1081 * to be called with the lock taken.
1082 * @wq_head: the waitqueue to wait on
1083 * @condition: a C expression for the event to wait for
1084 * @lock: a locked spinlock_t, which will be released before schedule()
1085 * and reacquired afterwards.
1087 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1088 * @condition evaluates to true or signal is received. The @condition is
1089 * checked each time the waitqueue @wq_head is woken up.
1091 * wake_up() has to be called after changing any variable that could
1092 * change the result of the wait condition.
1094 * This is supposed to be called while holding the lock. The lock is
1095 * dropped before going to sleep and is reacquired afterwards.
1097 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1098 * and 0 if @condition evaluated to true.
1100 #define wait_event_interruptible_lock_irq(wq_head, condition, lock) \
1104 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1105 condition, lock,); \
1109 #define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state) \
1110 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
1111 state, 0, timeout, \
1112 spin_unlock_irq(&lock); \
1113 __ret = schedule_timeout(__ret); \
1114 spin_lock_irq(&lock));
1117 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
1118 * true or a timeout elapses. The condition is checked under
1119 * the lock. This is expected to be called with the lock taken.
1120 * @wq_head: the waitqueue to wait on
1121 * @condition: a C expression for the event to wait for
1122 * @lock: a locked spinlock_t, which will be released before schedule()
1123 * and reacquired afterwards.
1124 * @timeout: timeout, in jiffies
1126 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1127 * @condition evaluates to true or signal is received. The @condition is
1128 * checked each time the waitqueue @wq_head is woken up.
1130 * wake_up() has to be called after changing any variable that could
1131 * change the result of the wait condition.
1133 * This is supposed to be called while holding the lock. The lock is
1134 * dropped before going to sleep and is reacquired afterwards.
1136 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
1137 * was interrupted by a signal, and the remaining jiffies otherwise
1138 * if the condition evaluated to true before the timeout elapsed.
1140 #define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock, \
1143 long __ret = timeout; \
1144 if (!___wait_cond_timeout(condition)) \
1145 __ret = __wait_event_lock_irq_timeout( \
1146 wq_head, condition, lock, timeout, \
1147 TASK_INTERRUPTIBLE); \
1151 #define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout) \
1153 long __ret = timeout; \
1154 if (!___wait_cond_timeout(condition)) \
1155 __ret = __wait_event_lock_irq_timeout( \
1156 wq_head, condition, lock, timeout, \
1157 TASK_UNINTERRUPTIBLE); \
1162 * Waitqueues which are removed from the waitqueue_head at wakeup time
1164 void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1165 bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1166 long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1167 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
1168 long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
1169 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1170 int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1172 #define DEFINE_WAIT_FUNC(name, function) \
1173 struct wait_queue_entry name = { \
1174 .private = current, \
1176 .entry = LIST_HEAD_INIT((name).entry), \
1179 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1181 #define init_wait(wait) \
1183 (wait)->private = current; \
1184 (wait)->func = autoremove_wake_function; \
1185 INIT_LIST_HEAD(&(wait)->entry); \
1186 (wait)->flags = 0; \
1189 typedef int (*task_call_f)(struct task_struct *p, void *arg);
1190 extern int task_call_func(struct task_struct *p, task_call_f func, void *arg);
1192 #endif /* _LINUX_WAIT_H */