1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Generic wait-for-completion handler;
5 *
6 * It differs from semaphores in that their default case is the opposite,
7 * wait_for_completion default blocks whereas semaphore default non-block. The
8 * interface also makes it easy to 'complete' multiple waiting threads,
9 * something which isn't entirely natural for semaphores.
10 *
11 * But more importantly, the primitive documents the usage. Semaphores would
12 * typically be used for exclusion which gives rise to priority inversion.
13 * Waiting for completion is a typically sync point, but not an exclusion point.
14 */
15
16#include <linux/linkage.h>
17#include <linux/sched/debug.h>
18#include <linux/completion.h>
19#include "sched.h"
20
21static void complete_with_flags(struct completion *x, int wake_flags)
22{
23 unsigned long flags;
24
25 raw_spin_lock_irqsave(&x->wait.lock, flags);
26
27 if (x->done != UINT_MAX)
28 x->done++;
29 swake_up_locked(q: &x->wait, wake_flags);
30 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
31}
32
33void complete_on_current_cpu(struct completion *x)
34{
35 return complete_with_flags(x, WF_CURRENT_CPU);
36}
37
38/**
39 * complete: - signals a single thread waiting on this completion
40 * @x: holds the state of this particular completion
41 *
42 * This will wake up a single thread waiting on this completion. Threads will be
43 * awakened in the same order in which they were queued.
44 *
45 * See also complete_all(), wait_for_completion() and related routines.
46 *
47 * If this function wakes up a task, it executes a full memory barrier before
48 * accessing the task state.
49 */
50void complete(struct completion *x)
51{
52 complete_with_flags(x, wake_flags: 0);
53}
54EXPORT_SYMBOL(complete);
55
56/**
57 * complete_all: - signals all threads waiting on this completion
58 * @x: holds the state of this particular completion
59 *
60 * This will wake up all threads waiting on this particular completion event.
61 *
62 * If this function wakes up a task, it executes a full memory barrier before
63 * accessing the task state.
64 *
65 * Since complete_all() sets the completion of @x permanently to done
66 * to allow multiple waiters to finish, a call to reinit_completion()
67 * must be used on @x if @x is to be used again. The code must make
68 * sure that all waiters have woken and finished before reinitializing
69 * @x. Also note that the function completion_done() can not be used
70 * to know if there are still waiters after complete_all() has been called.
71 */
72void complete_all(struct completion *x)
73{
74 unsigned long flags;
75
76 lockdep_assert_RT_in_threaded_ctx();
77
78 raw_spin_lock_irqsave(&x->wait.lock, flags);
79 x->done = UINT_MAX;
80 swake_up_all_locked(q: &x->wait);
81 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
82}
83EXPORT_SYMBOL(complete_all);
84
85static inline long __sched
86do_wait_for_common(struct completion *x,
87 long (*action)(long), long timeout, int state)
88{
89 if (!x->done) {
90 DECLARE_SWAITQUEUE(wait);
91
92 do {
93 if (signal_pending_state(state, current)) {
94 timeout = -ERESTARTSYS;
95 break;
96 }
97 __prepare_to_swait(q: &x->wait, wait: &wait);
98 __set_current_state(state);
99 raw_spin_unlock_irq(&x->wait.lock);
100 timeout = action(timeout);
101 raw_spin_lock_irq(&x->wait.lock);
102 } while (!x->done && timeout);
103 __finish_swait(q: &x->wait, wait: &wait);
104 if (!x->done)
105 return timeout;
106 }
107 if (x->done != UINT_MAX)
108 x->done--;
109 return timeout ?: 1;
110}
111
112static inline long __sched
113__wait_for_common(struct completion *x,
114 long (*action)(long), long timeout, int state)
115{
116 might_sleep();
117
118 complete_acquire(x);
119
120 raw_spin_lock_irq(&x->wait.lock);
121 timeout = do_wait_for_common(x, action, timeout, state);
122 raw_spin_unlock_irq(&x->wait.lock);
123
124 complete_release(x);
125
126 return timeout;
127}
128
129static long __sched
130wait_for_common(struct completion *x, long timeout, int state)
131{
132 return __wait_for_common(x, action: schedule_timeout, timeout, state);
133}
134
135static long __sched
136wait_for_common_io(struct completion *x, long timeout, int state)
137{
138 return __wait_for_common(x, action: io_schedule_timeout, timeout, state);
139}
140
141/**
142 * wait_for_completion: - waits for completion of a task
143 * @x: holds the state of this particular completion
144 *
145 * This waits to be signaled for completion of a specific task. It is NOT
146 * interruptible and there is no timeout.
147 *
148 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
149 * and interrupt capability. Also see complete().
150 */
151void __sched wait_for_completion(struct completion *x)
152{
153 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
154}
155EXPORT_SYMBOL(wait_for_completion);
156
157/**
158 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
159 * @x: holds the state of this particular completion
160 * @timeout: timeout value in jiffies
161 *
162 * This waits for either a completion of a specific task to be signaled or for a
163 * specified timeout to expire. The timeout is in jiffies. It is not
164 * interruptible.
165 *
166 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
167 * till timeout) if completed.
168 */
169unsigned long __sched
170wait_for_completion_timeout(struct completion *x, unsigned long timeout)
171{
172 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
173}
174EXPORT_SYMBOL(wait_for_completion_timeout);
175
176/**
177 * wait_for_completion_io: - waits for completion of a task
178 * @x: holds the state of this particular completion
179 *
180 * This waits to be signaled for completion of a specific task. It is NOT
181 * interruptible and there is no timeout. The caller is accounted as waiting
182 * for IO (which traditionally means blkio only).
183 */
184void __sched wait_for_completion_io(struct completion *x)
185{
186 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
187}
188EXPORT_SYMBOL(wait_for_completion_io);
189
190/**
191 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
192 * @x: holds the state of this particular completion
193 * @timeout: timeout value in jiffies
194 *
195 * This waits for either a completion of a specific task to be signaled or for a
196 * specified timeout to expire. The timeout is in jiffies. It is not
197 * interruptible. The caller is accounted as waiting for IO (which traditionally
198 * means blkio only).
199 *
200 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
201 * till timeout) if completed.
202 */
203unsigned long __sched
204wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
205{
206 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
207}
208EXPORT_SYMBOL(wait_for_completion_io_timeout);
209
210/**
211 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
212 * @x: holds the state of this particular completion
213 *
214 * This waits for completion of a specific task to be signaled. It is
215 * interruptible.
216 *
217 * Return: -ERESTARTSYS if interrupted, 0 if completed.
218 */
219int __sched wait_for_completion_interruptible(struct completion *x)
220{
221 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
222
223 if (t == -ERESTARTSYS)
224 return t;
225 return 0;
226}
227EXPORT_SYMBOL(wait_for_completion_interruptible);
228
229/**
230 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
231 * @x: holds the state of this particular completion
232 * @timeout: timeout value in jiffies
233 *
234 * This waits for either a completion of a specific task to be signaled or for a
235 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
236 *
237 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
238 * or number of jiffies left till timeout) if completed.
239 */
240long __sched
241wait_for_completion_interruptible_timeout(struct completion *x,
242 unsigned long timeout)
243{
244 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
245}
246EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
247
248/**
249 * wait_for_completion_killable: - waits for completion of a task (killable)
250 * @x: holds the state of this particular completion
251 *
252 * This waits to be signaled for completion of a specific task. It can be
253 * interrupted by a kill signal.
254 *
255 * Return: -ERESTARTSYS if interrupted, 0 if completed.
256 */
257int __sched wait_for_completion_killable(struct completion *x)
258{
259 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
260
261 if (t == -ERESTARTSYS)
262 return t;
263 return 0;
264}
265EXPORT_SYMBOL(wait_for_completion_killable);
266
267int __sched wait_for_completion_state(struct completion *x, unsigned int state)
268{
269 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
270
271 if (t == -ERESTARTSYS)
272 return t;
273 return 0;
274}
275EXPORT_SYMBOL(wait_for_completion_state);
276
277/**
278 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
279 * @x: holds the state of this particular completion
280 * @timeout: timeout value in jiffies
281 *
282 * This waits for either a completion of a specific task to be
283 * signaled or for a specified timeout to expire. It can be
284 * interrupted by a kill signal. The timeout is in jiffies.
285 *
286 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
287 * or number of jiffies left till timeout) if completed.
288 */
289long __sched
290wait_for_completion_killable_timeout(struct completion *x,
291 unsigned long timeout)
292{
293 return wait_for_common(x, timeout, TASK_KILLABLE);
294}
295EXPORT_SYMBOL(wait_for_completion_killable_timeout);
296
297/**
298 * try_wait_for_completion - try to decrement a completion without blocking
299 * @x: completion structure
300 *
301 * Return: 0 if a decrement cannot be done without blocking
302 * 1 if a decrement succeeded.
303 *
304 * If a completion is being used as a counting completion,
305 * attempt to decrement the counter without blocking. This
306 * enables us to avoid waiting if the resource the completion
307 * is protecting is not available.
308 */
309bool try_wait_for_completion(struct completion *x)
310{
311 unsigned long flags;
312 bool ret = true;
313
314 /*
315 * Since x->done will need to be locked only
316 * in the non-blocking case, we check x->done
317 * first without taking the lock so we can
318 * return early in the blocking case.
319 */
320 if (!READ_ONCE(x->done))
321 return false;
322
323 raw_spin_lock_irqsave(&x->wait.lock, flags);
324 if (!x->done)
325 ret = false;
326 else if (x->done != UINT_MAX)
327 x->done--;
328 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
329 return ret;
330}
331EXPORT_SYMBOL(try_wait_for_completion);
332
333/**
334 * completion_done - Test to see if a completion has any waiters
335 * @x: completion structure
336 *
337 * Return: 0 if there are waiters (wait_for_completion() in progress)
338 * 1 if there are no waiters.
339 *
340 * Note, this will always return true if complete_all() was called on @X.
341 */
342bool completion_done(struct completion *x)
343{
344 unsigned long flags;
345
346 if (!READ_ONCE(x->done))
347 return false;
348
349 /*
350 * If ->done, we need to wait for complete() to release ->wait.lock
351 * otherwise we can end up freeing the completion before complete()
352 * is done referencing it.
353 */
354 raw_spin_lock_irqsave(&x->wait.lock, flags);
355 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
356 return true;
357}
358EXPORT_SYMBOL(completion_done);
359

source code of linux/kernel/sched/completion.c