1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_SCHED_WAKE_Q_H
3#define _LINUX_SCHED_WAKE_Q_H
4
5/*
6 * Wake-queues are lists of tasks with a pending wakeup, whose
7 * callers have already marked the task as woken internally,
8 * and can thus carry on. A common use case is being able to
9 * do the wakeups once the corresponding user lock as been
10 * released.
11 *
12 * We hold reference to each task in the list across the wakeup,
13 * thus guaranteeing that the memory is still valid by the time
14 * the actual wakeups are performed in wake_up_q().
15 *
16 * One per task suffices, because there's never a need for a task to be
17 * in two wake queues simultaneously; it is forbidden to abandon a task
18 * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is
19 * already in a wake queue, the wakeup will happen soon and the second
20 * waker can just skip it.
21 *
22 * The DEFINE_WAKE_Q macro declares and initializes the list head.
23 * wake_up_q() does NOT reinitialize the list; it's expected to be
24 * called near the end of a function. Otherwise, the list can be
25 * re-initialized for later re-use by wake_q_init().
26 *
27 * NOTE that this can cause spurious wakeups. schedule() callers
28 * must ensure the call is done inside a loop, confirming that the
29 * wakeup condition has in fact occurred.
30 *
31 * NOTE that there is no guarantee the wakeup will happen any later than the
32 * wake_q_add() location. Therefore task must be ready to be woken at the
33 * location of the wake_q_add().
34 */
35
36#include <linux/sched.h>
37
38struct wake_q_head {
39 struct wake_q_node *first;
40 struct wake_q_node **lastp;
41};
42
43#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)
44
45#define WAKE_Q_HEAD_INITIALIZER(name) \
46 { WAKE_Q_TAIL, &name.first }
47
48#define DEFINE_WAKE_Q(name) \
49 struct wake_q_head name = WAKE_Q_HEAD_INITIALIZER(name)
50
51static inline void wake_q_init(struct wake_q_head *head)
52{
53 head->first = WAKE_Q_TAIL;
54 head->lastp = &head->first;
55}
56
57static inline bool wake_q_empty(struct wake_q_head *head)
58{
59 return head->first == WAKE_Q_TAIL;
60}
61
62extern void wake_q_add(struct wake_q_head *head, struct task_struct *task);
63extern void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task);
64extern void wake_up_q(struct wake_q_head *head);
65
66/* Spin unlock helpers to unlock and call wake_up_q with preempt disabled */
67static inline
68void raw_spin_unlock_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q)
69{
70 guard(preempt)();
71 raw_spin_unlock(lock);
72 if (wake_q) {
73 wake_up_q(head: wake_q);
74 wake_q_init(head: wake_q);
75 }
76}
77
78static inline
79void raw_spin_unlock_irq_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q)
80{
81 guard(preempt)();
82 raw_spin_unlock_irq(lock);
83 if (wake_q) {
84 wake_up_q(head: wake_q);
85 wake_q_init(head: wake_q);
86 }
87}
88
89static inline
90void raw_spin_unlock_irqrestore_wake(raw_spinlock_t *lock, unsigned long flags,
91 struct wake_q_head *wake_q)
92{
93 guard(preempt)();
94 raw_spin_unlock_irqrestore(lock, flags);
95 if (wake_q) {
96 wake_up_q(head: wake_q);
97 wake_q_init(head: wake_q);
98 }
99}
100#endif /* _LINUX_SCHED_WAKE_Q_H */
101