1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/kernel.h> |
3 | #include <linux/errno.h> |
4 | #include <linux/file.h> |
5 | #include <linux/io_uring.h> |
6 | |
7 | #include <trace/events/io_uring.h> |
8 | |
9 | #include <uapi/linux/io_uring.h> |
10 | |
11 | #include "io_uring.h" |
12 | #include "refs.h" |
13 | #include "cancel.h" |
14 | #include "timeout.h" |
15 | |
16 | struct io_timeout { |
17 | struct file *file; |
18 | u32 off; |
19 | u32 target_seq; |
20 | u32 repeats; |
21 | struct list_head list; |
22 | /* head of the link, used by linked timeouts only */ |
23 | struct io_kiocb *head; |
24 | /* for linked completions */ |
25 | struct io_kiocb *prev; |
26 | }; |
27 | |
28 | struct io_timeout_rem { |
29 | struct file *file; |
30 | u64 addr; |
31 | |
32 | /* timeout update */ |
33 | struct timespec64 ts; |
34 | u32 flags; |
35 | bool ltimeout; |
36 | }; |
37 | |
38 | static inline bool io_is_timeout_noseq(struct io_kiocb *req) |
39 | { |
40 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
41 | struct io_timeout_data *data = req->async_data; |
42 | |
43 | return !timeout->off || data->flags & IORING_TIMEOUT_MULTISHOT; |
44 | } |
45 | |
46 | static inline void io_put_req(struct io_kiocb *req) |
47 | { |
48 | if (req_ref_put_and_test(req)) { |
49 | io_queue_next(req); |
50 | io_free_req(req); |
51 | } |
52 | } |
53 | |
54 | static inline bool io_timeout_finish(struct io_timeout *timeout, |
55 | struct io_timeout_data *data) |
56 | { |
57 | if (!(data->flags & IORING_TIMEOUT_MULTISHOT)) |
58 | return true; |
59 | |
60 | if (!timeout->off || (timeout->repeats && --timeout->repeats)) |
61 | return false; |
62 | |
63 | return true; |
64 | } |
65 | |
66 | static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer); |
67 | |
68 | static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts) |
69 | { |
70 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
71 | struct io_timeout_data *data = req->async_data; |
72 | struct io_ring_ctx *ctx = req->ctx; |
73 | |
74 | if (!io_timeout_finish(timeout, data)) { |
75 | bool filled; |
76 | filled = io_fill_cqe_req_aux(req, defer: ts->locked, res: -ETIME, |
77 | IORING_CQE_F_MORE); |
78 | if (filled) { |
79 | /* re-arm timer */ |
80 | spin_lock_irq(lock: &ctx->timeout_lock); |
81 | list_add(new: &timeout->list, head: ctx->timeout_list.prev); |
82 | data->timer.function = io_timeout_fn; |
83 | hrtimer_start(timer: &data->timer, tim: timespec64_to_ktime(ts: data->ts), mode: data->mode); |
84 | spin_unlock_irq(lock: &ctx->timeout_lock); |
85 | return; |
86 | } |
87 | } |
88 | |
89 | io_req_task_complete(req, ts); |
90 | } |
91 | |
92 | static bool io_kill_timeout(struct io_kiocb *req, int status) |
93 | __must_hold(&req->ctx->timeout_lock) |
94 | { |
95 | struct io_timeout_data *io = req->async_data; |
96 | |
97 | if (hrtimer_try_to_cancel(timer: &io->timer) != -1) { |
98 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
99 | |
100 | if (status) |
101 | req_set_fail(req); |
102 | atomic_set(v: &req->ctx->cq_timeouts, |
103 | i: atomic_read(v: &req->ctx->cq_timeouts) + 1); |
104 | list_del_init(entry: &timeout->list); |
105 | io_req_queue_tw_complete(req, res: status); |
106 | return true; |
107 | } |
108 | return false; |
109 | } |
110 | |
111 | __cold void io_flush_timeouts(struct io_ring_ctx *ctx) |
112 | { |
113 | u32 seq; |
114 | struct io_timeout *timeout, *tmp; |
115 | |
116 | spin_lock_irq(lock: &ctx->timeout_lock); |
117 | seq = ctx->cached_cq_tail - atomic_read(v: &ctx->cq_timeouts); |
118 | |
119 | list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { |
120 | struct io_kiocb *req = cmd_to_io_kiocb(timeout); |
121 | u32 events_needed, events_got; |
122 | |
123 | if (io_is_timeout_noseq(req)) |
124 | break; |
125 | |
126 | /* |
127 | * Since seq can easily wrap around over time, subtract |
128 | * the last seq at which timeouts were flushed before comparing. |
129 | * Assuming not more than 2^31-1 events have happened since, |
130 | * these subtractions won't have wrapped, so we can check if |
131 | * target is in [last_seq, current_seq] by comparing the two. |
132 | */ |
133 | events_needed = timeout->target_seq - ctx->cq_last_tm_flush; |
134 | events_got = seq - ctx->cq_last_tm_flush; |
135 | if (events_got < events_needed) |
136 | break; |
137 | |
138 | io_kill_timeout(req, status: 0); |
139 | } |
140 | ctx->cq_last_tm_flush = seq; |
141 | spin_unlock_irq(lock: &ctx->timeout_lock); |
142 | } |
143 | |
144 | static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts) |
145 | { |
146 | io_tw_lock(ctx: link->ctx, ts); |
147 | while (link) { |
148 | struct io_kiocb *nxt = link->link; |
149 | long res = -ECANCELED; |
150 | |
151 | if (link->flags & REQ_F_FAIL) |
152 | res = link->cqe.res; |
153 | link->link = NULL; |
154 | io_req_set_res(req: link, res, cflags: 0); |
155 | io_req_task_complete(req: link, ts); |
156 | link = nxt; |
157 | } |
158 | } |
159 | |
160 | static void io_fail_links(struct io_kiocb *req) |
161 | __must_hold(&req->ctx->completion_lock) |
162 | { |
163 | struct io_kiocb *link = req->link; |
164 | bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES; |
165 | |
166 | if (!link) |
167 | return; |
168 | |
169 | while (link) { |
170 | if (ignore_cqes) |
171 | link->flags |= REQ_F_CQE_SKIP; |
172 | else |
173 | link->flags &= ~REQ_F_CQE_SKIP; |
174 | trace_io_uring_fail_link(req, link); |
175 | link = link->link; |
176 | } |
177 | |
178 | link = req->link; |
179 | link->io_task_work.func = io_req_tw_fail_links; |
180 | io_req_task_work_add(req: link); |
181 | req->link = NULL; |
182 | } |
183 | |
184 | static inline void io_remove_next_linked(struct io_kiocb *req) |
185 | { |
186 | struct io_kiocb *nxt = req->link; |
187 | |
188 | req->link = nxt->link; |
189 | nxt->link = NULL; |
190 | } |
191 | |
192 | void io_disarm_next(struct io_kiocb *req) |
193 | __must_hold(&req->ctx->completion_lock) |
194 | { |
195 | struct io_kiocb *link = NULL; |
196 | |
197 | if (req->flags & REQ_F_ARM_LTIMEOUT) { |
198 | link = req->link; |
199 | req->flags &= ~REQ_F_ARM_LTIMEOUT; |
200 | if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { |
201 | io_remove_next_linked(req); |
202 | io_req_queue_tw_complete(req: link, res: -ECANCELED); |
203 | } |
204 | } else if (req->flags & REQ_F_LINK_TIMEOUT) { |
205 | struct io_ring_ctx *ctx = req->ctx; |
206 | |
207 | spin_lock_irq(lock: &ctx->timeout_lock); |
208 | link = io_disarm_linked_timeout(req); |
209 | spin_unlock_irq(lock: &ctx->timeout_lock); |
210 | if (link) |
211 | io_req_queue_tw_complete(req: link, res: -ECANCELED); |
212 | } |
213 | if (unlikely((req->flags & REQ_F_FAIL) && |
214 | !(req->flags & REQ_F_HARDLINK))) |
215 | io_fail_links(req); |
216 | } |
217 | |
218 | struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, |
219 | struct io_kiocb *link) |
220 | __must_hold(&req->ctx->completion_lock) |
221 | __must_hold(&req->ctx->timeout_lock) |
222 | { |
223 | struct io_timeout_data *io = link->async_data; |
224 | struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout); |
225 | |
226 | io_remove_next_linked(req); |
227 | timeout->head = NULL; |
228 | if (hrtimer_try_to_cancel(timer: &io->timer) != -1) { |
229 | list_del(entry: &timeout->list); |
230 | return link; |
231 | } |
232 | |
233 | return NULL; |
234 | } |
235 | |
236 | static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) |
237 | { |
238 | struct io_timeout_data *data = container_of(timer, |
239 | struct io_timeout_data, timer); |
240 | struct io_kiocb *req = data->req; |
241 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
242 | struct io_ring_ctx *ctx = req->ctx; |
243 | unsigned long flags; |
244 | |
245 | spin_lock_irqsave(&ctx->timeout_lock, flags); |
246 | list_del_init(entry: &timeout->list); |
247 | atomic_set(v: &req->ctx->cq_timeouts, |
248 | i: atomic_read(v: &req->ctx->cq_timeouts) + 1); |
249 | spin_unlock_irqrestore(lock: &ctx->timeout_lock, flags); |
250 | |
251 | if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS)) |
252 | req_set_fail(req); |
253 | |
254 | io_req_set_res(req, res: -ETIME, cflags: 0); |
255 | req->io_task_work.func = io_timeout_complete; |
256 | io_req_task_work_add(req); |
257 | return HRTIMER_NORESTART; |
258 | } |
259 | |
260 | static struct io_kiocb *(struct io_ring_ctx *ctx, |
261 | struct io_cancel_data *cd) |
262 | __must_hold(&ctx->timeout_lock) |
263 | { |
264 | struct io_timeout *timeout; |
265 | struct io_timeout_data *io; |
266 | struct io_kiocb *req = NULL; |
267 | |
268 | list_for_each_entry(timeout, &ctx->timeout_list, list) { |
269 | struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); |
270 | |
271 | if (io_cancel_req_match(req: tmp, cd)) { |
272 | req = tmp; |
273 | break; |
274 | } |
275 | } |
276 | if (!req) |
277 | return ERR_PTR(error: -ENOENT); |
278 | |
279 | io = req->async_data; |
280 | if (hrtimer_try_to_cancel(timer: &io->timer) == -1) |
281 | return ERR_PTR(error: -EALREADY); |
282 | timeout = io_kiocb_to_cmd(req, struct io_timeout); |
283 | list_del_init(entry: &timeout->list); |
284 | return req; |
285 | } |
286 | |
287 | int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) |
288 | __must_hold(&ctx->completion_lock) |
289 | { |
290 | struct io_kiocb *req; |
291 | |
292 | spin_lock_irq(lock: &ctx->timeout_lock); |
293 | req = io_timeout_extract(ctx, cd); |
294 | spin_unlock_irq(lock: &ctx->timeout_lock); |
295 | |
296 | if (IS_ERR(ptr: req)) |
297 | return PTR_ERR(ptr: req); |
298 | io_req_task_queue_fail(req, ret: -ECANCELED); |
299 | return 0; |
300 | } |
301 | |
302 | static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts) |
303 | { |
304 | unsigned issue_flags = ts->locked ? 0 : IO_URING_F_UNLOCKED; |
305 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
306 | struct io_kiocb *prev = timeout->prev; |
307 | int ret = -ENOENT; |
308 | |
309 | if (prev) { |
310 | if (!(req->task->flags & PF_EXITING)) { |
311 | struct io_cancel_data cd = { |
312 | .ctx = req->ctx, |
313 | .data = prev->cqe.user_data, |
314 | }; |
315 | |
316 | ret = io_try_cancel(tctx: req->task->io_uring, cd: &cd, issue_flags); |
317 | } |
318 | io_req_set_res(req, res: ret ?: -ETIME, cflags: 0); |
319 | io_req_task_complete(req, ts); |
320 | io_put_req(req: prev); |
321 | } else { |
322 | io_req_set_res(req, res: -ETIME, cflags: 0); |
323 | io_req_task_complete(req, ts); |
324 | } |
325 | } |
326 | |
327 | static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) |
328 | { |
329 | struct io_timeout_data *data = container_of(timer, |
330 | struct io_timeout_data, timer); |
331 | struct io_kiocb *prev, *req = data->req; |
332 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
333 | struct io_ring_ctx *ctx = req->ctx; |
334 | unsigned long flags; |
335 | |
336 | spin_lock_irqsave(&ctx->timeout_lock, flags); |
337 | prev = timeout->head; |
338 | timeout->head = NULL; |
339 | |
340 | /* |
341 | * We don't expect the list to be empty, that will only happen if we |
342 | * race with the completion of the linked work. |
343 | */ |
344 | if (prev) { |
345 | io_remove_next_linked(req: prev); |
346 | if (!req_ref_inc_not_zero(req: prev)) |
347 | prev = NULL; |
348 | } |
349 | list_del(entry: &timeout->list); |
350 | timeout->prev = prev; |
351 | spin_unlock_irqrestore(lock: &ctx->timeout_lock, flags); |
352 | |
353 | req->io_task_work.func = io_req_task_link_timeout; |
354 | io_req_task_work_add(req); |
355 | return HRTIMER_NORESTART; |
356 | } |
357 | |
358 | static clockid_t io_timeout_get_clock(struct io_timeout_data *data) |
359 | { |
360 | switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) { |
361 | case IORING_TIMEOUT_BOOTTIME: |
362 | return CLOCK_BOOTTIME; |
363 | case IORING_TIMEOUT_REALTIME: |
364 | return CLOCK_REALTIME; |
365 | default: |
366 | /* can't happen, vetted at prep time */ |
367 | WARN_ON_ONCE(1); |
368 | fallthrough; |
369 | case 0: |
370 | return CLOCK_MONOTONIC; |
371 | } |
372 | } |
373 | |
374 | static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, |
375 | struct timespec64 *ts, enum hrtimer_mode mode) |
376 | __must_hold(&ctx->timeout_lock) |
377 | { |
378 | struct io_timeout_data *io; |
379 | struct io_timeout *timeout; |
380 | struct io_kiocb *req = NULL; |
381 | |
382 | list_for_each_entry(timeout, &ctx->ltimeout_list, list) { |
383 | struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); |
384 | |
385 | if (user_data == tmp->cqe.user_data) { |
386 | req = tmp; |
387 | break; |
388 | } |
389 | } |
390 | if (!req) |
391 | return -ENOENT; |
392 | |
393 | io = req->async_data; |
394 | if (hrtimer_try_to_cancel(timer: &io->timer) == -1) |
395 | return -EALREADY; |
396 | hrtimer_init(timer: &io->timer, which_clock: io_timeout_get_clock(data: io), mode); |
397 | io->timer.function = io_link_timeout_fn; |
398 | hrtimer_start(timer: &io->timer, tim: timespec64_to_ktime(ts: *ts), mode); |
399 | return 0; |
400 | } |
401 | |
402 | static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, |
403 | struct timespec64 *ts, enum hrtimer_mode mode) |
404 | __must_hold(&ctx->timeout_lock) |
405 | { |
406 | struct io_cancel_data cd = { .ctx = ctx, .data = user_data, }; |
407 | struct io_kiocb *req = io_timeout_extract(ctx, cd: &cd); |
408 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
409 | struct io_timeout_data *data; |
410 | |
411 | if (IS_ERR(ptr: req)) |
412 | return PTR_ERR(ptr: req); |
413 | |
414 | timeout->off = 0; /* noseq */ |
415 | data = req->async_data; |
416 | list_add_tail(new: &timeout->list, head: &ctx->timeout_list); |
417 | hrtimer_init(timer: &data->timer, which_clock: io_timeout_get_clock(data), mode); |
418 | data->timer.function = io_timeout_fn; |
419 | hrtimer_start(timer: &data->timer, tim: timespec64_to_ktime(ts: *ts), mode); |
420 | return 0; |
421 | } |
422 | |
423 | int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
424 | { |
425 | struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); |
426 | |
427 | if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) |
428 | return -EINVAL; |
429 | if (sqe->buf_index || sqe->len || sqe->splice_fd_in) |
430 | return -EINVAL; |
431 | |
432 | tr->ltimeout = false; |
433 | tr->addr = READ_ONCE(sqe->addr); |
434 | tr->flags = READ_ONCE(sqe->timeout_flags); |
435 | if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) { |
436 | if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1) |
437 | return -EINVAL; |
438 | if (tr->flags & IORING_LINK_TIMEOUT_UPDATE) |
439 | tr->ltimeout = true; |
440 | if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS)) |
441 | return -EINVAL; |
442 | if (get_timespec64(ts: &tr->ts, u64_to_user_ptr(sqe->addr2))) |
443 | return -EFAULT; |
444 | if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0) |
445 | return -EINVAL; |
446 | } else if (tr->flags) { |
447 | /* timeout removal doesn't support flags */ |
448 | return -EINVAL; |
449 | } |
450 | |
451 | return 0; |
452 | } |
453 | |
454 | static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) |
455 | { |
456 | return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS |
457 | : HRTIMER_MODE_REL; |
458 | } |
459 | |
460 | /* |
461 | * Remove or update an existing timeout command |
462 | */ |
463 | int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) |
464 | { |
465 | struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); |
466 | struct io_ring_ctx *ctx = req->ctx; |
467 | int ret; |
468 | |
469 | if (!(tr->flags & IORING_TIMEOUT_UPDATE)) { |
470 | struct io_cancel_data cd = { .ctx = ctx, .data = tr->addr, }; |
471 | |
472 | spin_lock(lock: &ctx->completion_lock); |
473 | ret = io_timeout_cancel(ctx, cd: &cd); |
474 | spin_unlock(lock: &ctx->completion_lock); |
475 | } else { |
476 | enum hrtimer_mode mode = io_translate_timeout_mode(flags: tr->flags); |
477 | |
478 | spin_lock_irq(lock: &ctx->timeout_lock); |
479 | if (tr->ltimeout) |
480 | ret = io_linked_timeout_update(ctx, user_data: tr->addr, ts: &tr->ts, mode); |
481 | else |
482 | ret = io_timeout_update(ctx, user_data: tr->addr, ts: &tr->ts, mode); |
483 | spin_unlock_irq(lock: &ctx->timeout_lock); |
484 | } |
485 | |
486 | if (ret < 0) |
487 | req_set_fail(req); |
488 | io_req_set_res(req, res: ret, cflags: 0); |
489 | return IOU_OK; |
490 | } |
491 | |
492 | static int __io_timeout_prep(struct io_kiocb *req, |
493 | const struct io_uring_sqe *sqe, |
494 | bool is_timeout_link) |
495 | { |
496 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
497 | struct io_timeout_data *data; |
498 | unsigned flags; |
499 | u32 off = READ_ONCE(sqe->off); |
500 | |
501 | if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in) |
502 | return -EINVAL; |
503 | if (off && is_timeout_link) |
504 | return -EINVAL; |
505 | flags = READ_ONCE(sqe->timeout_flags); |
506 | if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK | |
507 | IORING_TIMEOUT_ETIME_SUCCESS | |
508 | IORING_TIMEOUT_MULTISHOT)) |
509 | return -EINVAL; |
510 | /* more than one clock specified is invalid, obviously */ |
511 | if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) |
512 | return -EINVAL; |
513 | /* multishot requests only make sense with rel values */ |
514 | if (!(~flags & (IORING_TIMEOUT_MULTISHOT | IORING_TIMEOUT_ABS))) |
515 | return -EINVAL; |
516 | |
517 | INIT_LIST_HEAD(list: &timeout->list); |
518 | timeout->off = off; |
519 | if (unlikely(off && !req->ctx->off_timeout_used)) |
520 | req->ctx->off_timeout_used = true; |
521 | /* |
522 | * for multishot reqs w/ fixed nr of repeats, repeats tracks the |
523 | * remaining nr |
524 | */ |
525 | timeout->repeats = 0; |
526 | if ((flags & IORING_TIMEOUT_MULTISHOT) && off > 0) |
527 | timeout->repeats = off; |
528 | |
529 | if (WARN_ON_ONCE(req_has_async_data(req))) |
530 | return -EFAULT; |
531 | if (io_alloc_async_data(req)) |
532 | return -ENOMEM; |
533 | |
534 | data = req->async_data; |
535 | data->req = req; |
536 | data->flags = flags; |
537 | |
538 | if (get_timespec64(ts: &data->ts, u64_to_user_ptr(sqe->addr))) |
539 | return -EFAULT; |
540 | |
541 | if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0) |
542 | return -EINVAL; |
543 | |
544 | INIT_LIST_HEAD(list: &timeout->list); |
545 | data->mode = io_translate_timeout_mode(flags); |
546 | hrtimer_init(timer: &data->timer, which_clock: io_timeout_get_clock(data), mode: data->mode); |
547 | |
548 | if (is_timeout_link) { |
549 | struct io_submit_link *link = &req->ctx->submit_state.link; |
550 | |
551 | if (!link->head) |
552 | return -EINVAL; |
553 | if (link->last->opcode == IORING_OP_LINK_TIMEOUT) |
554 | return -EINVAL; |
555 | timeout->head = link->last; |
556 | link->last->flags |= REQ_F_ARM_LTIMEOUT; |
557 | } |
558 | return 0; |
559 | } |
560 | |
561 | int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
562 | { |
563 | return __io_timeout_prep(req, sqe, is_timeout_link: false); |
564 | } |
565 | |
566 | int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
567 | { |
568 | return __io_timeout_prep(req, sqe, is_timeout_link: true); |
569 | } |
570 | |
571 | int io_timeout(struct io_kiocb *req, unsigned int issue_flags) |
572 | { |
573 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
574 | struct io_ring_ctx *ctx = req->ctx; |
575 | struct io_timeout_data *data = req->async_data; |
576 | struct list_head *entry; |
577 | u32 tail, off = timeout->off; |
578 | |
579 | spin_lock_irq(lock: &ctx->timeout_lock); |
580 | |
581 | /* |
582 | * sqe->off holds how many events that need to occur for this |
583 | * timeout event to be satisfied. If it isn't set, then this is |
584 | * a pure timeout request, sequence isn't used. |
585 | */ |
586 | if (io_is_timeout_noseq(req)) { |
587 | entry = ctx->timeout_list.prev; |
588 | goto add; |
589 | } |
590 | |
591 | tail = data_race(ctx->cached_cq_tail) - atomic_read(v: &ctx->cq_timeouts); |
592 | timeout->target_seq = tail + off; |
593 | |
594 | /* Update the last seq here in case io_flush_timeouts() hasn't. |
595 | * This is safe because ->completion_lock is held, and submissions |
596 | * and completions are never mixed in the same ->completion_lock section. |
597 | */ |
598 | ctx->cq_last_tm_flush = tail; |
599 | |
600 | /* |
601 | * Insertion sort, ensuring the first entry in the list is always |
602 | * the one we need first. |
603 | */ |
604 | list_for_each_prev(entry, &ctx->timeout_list) { |
605 | struct io_timeout *nextt = list_entry(entry, struct io_timeout, list); |
606 | struct io_kiocb *nxt = cmd_to_io_kiocb(nextt); |
607 | |
608 | if (io_is_timeout_noseq(req: nxt)) |
609 | continue; |
610 | /* nxt.seq is behind @tail, otherwise would've been completed */ |
611 | if (off >= nextt->target_seq - tail) |
612 | break; |
613 | } |
614 | add: |
615 | list_add(new: &timeout->list, head: entry); |
616 | data->timer.function = io_timeout_fn; |
617 | hrtimer_start(timer: &data->timer, tim: timespec64_to_ktime(ts: data->ts), mode: data->mode); |
618 | spin_unlock_irq(lock: &ctx->timeout_lock); |
619 | return IOU_ISSUE_SKIP_COMPLETE; |
620 | } |
621 | |
622 | void io_queue_linked_timeout(struct io_kiocb *req) |
623 | { |
624 | struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); |
625 | struct io_ring_ctx *ctx = req->ctx; |
626 | |
627 | spin_lock_irq(lock: &ctx->timeout_lock); |
628 | /* |
629 | * If the back reference is NULL, then our linked request finished |
630 | * before we got a chance to setup the timer |
631 | */ |
632 | if (timeout->head) { |
633 | struct io_timeout_data *data = req->async_data; |
634 | |
635 | data->timer.function = io_link_timeout_fn; |
636 | hrtimer_start(timer: &data->timer, tim: timespec64_to_ktime(ts: data->ts), |
637 | mode: data->mode); |
638 | list_add_tail(new: &timeout->list, head: &ctx->ltimeout_list); |
639 | } |
640 | spin_unlock_irq(lock: &ctx->timeout_lock); |
641 | /* drop submission reference */ |
642 | io_put_req(req); |
643 | } |
644 | |
645 | static bool io_match_task(struct io_kiocb *head, struct task_struct *task, |
646 | bool cancel_all) |
647 | __must_hold(&req->ctx->timeout_lock) |
648 | { |
649 | struct io_kiocb *req; |
650 | |
651 | if (task && head->task != task) |
652 | return false; |
653 | if (cancel_all) |
654 | return true; |
655 | |
656 | io_for_each_link(req, head) { |
657 | if (req->flags & REQ_F_INFLIGHT) |
658 | return true; |
659 | } |
660 | return false; |
661 | } |
662 | |
663 | /* Returns true if we found and killed one or more timeouts */ |
664 | __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, |
665 | bool cancel_all) |
666 | { |
667 | struct io_timeout *timeout, *tmp; |
668 | int canceled = 0; |
669 | |
670 | /* |
671 | * completion_lock is needed for io_match_task(). Take it before |
672 | * timeout_lockfirst to keep locking ordering. |
673 | */ |
674 | spin_lock(lock: &ctx->completion_lock); |
675 | spin_lock_irq(lock: &ctx->timeout_lock); |
676 | list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { |
677 | struct io_kiocb *req = cmd_to_io_kiocb(timeout); |
678 | |
679 | if (io_match_task(head: req, task: tsk, cancel_all) && |
680 | io_kill_timeout(req, status: -ECANCELED)) |
681 | canceled++; |
682 | } |
683 | spin_unlock_irq(lock: &ctx->timeout_lock); |
684 | spin_unlock(lock: &ctx->completion_lock); |
685 | return canceled != 0; |
686 | } |
687 | |