1 // SPDX-License-Identifier: GPL-2.0
3 * Generic wait-for-completion handler;
5 * It differs from semaphores in that their default case is the opposite,
6 * wait_for_completion default blocks whereas semaphore default non-block. The
7 * interface also makes it easy to 'complete' multiple waiting threads,
8 * something which isn't entirely natural for semaphores.
10 * But more importantly, the primitive documents the usage. Semaphores would
11 * typically be used for exclusion which gives rise to priority inversion.
12 * Waiting for completion is a typically sync point, but not an exclusion point.
17 * complete: - signals a single thread waiting on this completion
18 * @x: holds the state of this particular completion
20 * This will wake up a single thread waiting on this completion. Threads will be
21 * awakened in the same order in which they were queued.
23 * See also complete_all(), wait_for_completion() and related routines.
25 * If this function wakes up a task, it executes a full memory barrier before
26 * accessing the task state.
28 void complete(struct completion *x)
32 raw_spin_lock_irqsave(&x->wait.lock, flags);
34 if (x->done != UINT_MAX)
36 swake_up_locked(&x->wait);
37 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
39 EXPORT_SYMBOL(complete);
42 * complete_all: - signals all threads waiting on this completion
43 * @x: holds the state of this particular completion
45 * This will wake up all threads waiting on this particular completion event.
47 * If this function wakes up a task, it executes a full memory barrier before
48 * accessing the task state.
50 * Since complete_all() sets the completion of @x permanently to done
51 * to allow multiple waiters to finish, a call to reinit_completion()
52 * must be used on @x if @x is to be used again. The code must make
53 * sure that all waiters have woken and finished before reinitializing
54 * @x. Also note that the function completion_done() can not be used
55 * to know if there are still waiters after complete_all() has been called.
57 void complete_all(struct completion *x)
61 lockdep_assert_RT_in_threaded_ctx();
63 raw_spin_lock_irqsave(&x->wait.lock, flags);
65 swake_up_all_locked(&x->wait);
66 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
68 EXPORT_SYMBOL(complete_all);
70 static inline long __sched
71 do_wait_for_common(struct completion *x,
72 long (*action)(long), long timeout, int state)
75 DECLARE_SWAITQUEUE(wait);
78 if (signal_pending_state(state, current)) {
79 timeout = -ERESTARTSYS;
82 __prepare_to_swait(&x->wait, &wait);
83 __set_current_state(state);
84 raw_spin_unlock_irq(&x->wait.lock);
85 timeout = action(timeout);
86 raw_spin_lock_irq(&x->wait.lock);
87 } while (!x->done && timeout);
88 __finish_swait(&x->wait, &wait);
92 if (x->done != UINT_MAX)
97 static inline long __sched
98 __wait_for_common(struct completion *x,
99 long (*action)(long), long timeout, int state)
105 raw_spin_lock_irq(&x->wait.lock);
106 timeout = do_wait_for_common(x, action, timeout, state);
107 raw_spin_unlock_irq(&x->wait.lock);
115 wait_for_common(struct completion *x, long timeout, int state)
117 return __wait_for_common(x, schedule_timeout, timeout, state);
121 wait_for_common_io(struct completion *x, long timeout, int state)
123 return __wait_for_common(x, io_schedule_timeout, timeout, state);
127 * wait_for_completion: - waits for completion of a task
128 * @x: holds the state of this particular completion
130 * This waits to be signaled for completion of a specific task. It is NOT
131 * interruptible and there is no timeout.
133 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
134 * and interrupt capability. Also see complete().
136 void __sched wait_for_completion(struct completion *x)
138 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
140 EXPORT_SYMBOL(wait_for_completion);
143 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
144 * @x: holds the state of this particular completion
145 * @timeout: timeout value in jiffies
147 * This waits for either a completion of a specific task to be signaled or for a
148 * specified timeout to expire. The timeout is in jiffies. It is not
151 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
152 * till timeout) if completed.
154 unsigned long __sched
155 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
157 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
159 EXPORT_SYMBOL(wait_for_completion_timeout);
162 * wait_for_completion_io: - waits for completion of a task
163 * @x: holds the state of this particular completion
165 * This waits to be signaled for completion of a specific task. It is NOT
166 * interruptible and there is no timeout. The caller is accounted as waiting
167 * for IO (which traditionally means blkio only).
169 void __sched wait_for_completion_io(struct completion *x)
171 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
173 EXPORT_SYMBOL(wait_for_completion_io);
176 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
177 * @x: holds the state of this particular completion
178 * @timeout: timeout value in jiffies
180 * This waits for either a completion of a specific task to be signaled or for a
181 * specified timeout to expire. The timeout is in jiffies. It is not
182 * interruptible. The caller is accounted as waiting for IO (which traditionally
185 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
186 * till timeout) if completed.
188 unsigned long __sched
189 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
191 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
193 EXPORT_SYMBOL(wait_for_completion_io_timeout);
196 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
197 * @x: holds the state of this particular completion
199 * This waits for completion of a specific task to be signaled. It is
202 * Return: -ERESTARTSYS if interrupted, 0 if completed.
204 int __sched wait_for_completion_interruptible(struct completion *x)
206 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
207 if (t == -ERESTARTSYS)
211 EXPORT_SYMBOL(wait_for_completion_interruptible);
214 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
215 * @x: holds the state of this particular completion
216 * @timeout: timeout value in jiffies
218 * This waits for either a completion of a specific task to be signaled or for a
219 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
221 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
222 * or number of jiffies left till timeout) if completed.
225 wait_for_completion_interruptible_timeout(struct completion *x,
226 unsigned long timeout)
228 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
230 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
233 * wait_for_completion_killable: - waits for completion of a task (killable)
234 * @x: holds the state of this particular completion
236 * This waits to be signaled for completion of a specific task. It can be
237 * interrupted by a kill signal.
239 * Return: -ERESTARTSYS if interrupted, 0 if completed.
241 int __sched wait_for_completion_killable(struct completion *x)
243 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
244 if (t == -ERESTARTSYS)
248 EXPORT_SYMBOL(wait_for_completion_killable);
251 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
252 * @x: holds the state of this particular completion
253 * @timeout: timeout value in jiffies
255 * This waits for either a completion of a specific task to be
256 * signaled or for a specified timeout to expire. It can be
257 * interrupted by a kill signal. The timeout is in jiffies.
259 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
260 * or number of jiffies left till timeout) if completed.
263 wait_for_completion_killable_timeout(struct completion *x,
264 unsigned long timeout)
266 return wait_for_common(x, timeout, TASK_KILLABLE);
268 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
271 * try_wait_for_completion - try to decrement a completion without blocking
272 * @x: completion structure
274 * Return: 0 if a decrement cannot be done without blocking
275 * 1 if a decrement succeeded.
277 * If a completion is being used as a counting completion,
278 * attempt to decrement the counter without blocking. This
279 * enables us to avoid waiting if the resource the completion
280 * is protecting is not available.
282 bool try_wait_for_completion(struct completion *x)
288 * Since x->done will need to be locked only
289 * in the non-blocking case, we check x->done
290 * first without taking the lock so we can
291 * return early in the blocking case.
293 if (!READ_ONCE(x->done))
296 raw_spin_lock_irqsave(&x->wait.lock, flags);
299 else if (x->done != UINT_MAX)
301 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
304 EXPORT_SYMBOL(try_wait_for_completion);
307 * completion_done - Test to see if a completion has any waiters
308 * @x: completion structure
310 * Return: 0 if there are waiters (wait_for_completion() in progress)
311 * 1 if there are no waiters.
313 * Note, this will always return true if complete_all() was called on @X.
315 bool completion_done(struct completion *x)
319 if (!READ_ONCE(x->done))
323 * If ->done, we need to wait for complete() to release ->wait.lock
324 * otherwise we can end up freeing the completion before complete()
325 * is done referencing it.
327 raw_spin_lock_irqsave(&x->wait.lock, flags);
328 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
331 EXPORT_SYMBOL(completion_done);