1 | #include <linux/kernel.h> |
2 | #include <linux/errno.h> |
3 | #include <linux/file.h> |
4 | #include <linux/slab.h> |
5 | #include <linux/net.h> |
6 | #include <linux/io_uring.h> |
7 | |
8 | #include "io_uring.h" |
9 | #include "notif.h" |
10 | #include "rsrc.h" |
11 | |
12 | static void io_notif_complete_tw_ext(struct io_kiocb *notif, struct io_tw_state *ts) |
13 | { |
14 | struct io_notif_data *nd = io_notif_to_data(notif); |
15 | struct io_ring_ctx *ctx = notif->ctx; |
16 | |
17 | if (nd->zc_report && (nd->zc_copied || !nd->zc_used)) |
18 | notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED; |
19 | |
20 | if (nd->account_pages && ctx->user) { |
21 | __io_unaccount_mem(user: ctx->user, nr_pages: nd->account_pages); |
22 | nd->account_pages = 0; |
23 | } |
24 | io_req_task_complete(req: notif, ts); |
25 | } |
26 | |
27 | static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg, |
28 | bool success) |
29 | { |
30 | struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); |
31 | struct io_kiocb *notif = cmd_to_io_kiocb(nd); |
32 | |
33 | if (refcount_dec_and_test(r: &uarg->refcnt)) |
34 | __io_req_task_work_add(req: notif, flags: IOU_F_TWQ_LAZY_WAKE); |
35 | } |
36 | |
37 | static void io_tx_ubuf_callback_ext(struct sk_buff *skb, struct ubuf_info *uarg, |
38 | bool success) |
39 | { |
40 | struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); |
41 | |
42 | if (nd->zc_report) { |
43 | if (success && !nd->zc_used && skb) |
44 | WRITE_ONCE(nd->zc_used, true); |
45 | else if (!success && !nd->zc_copied) |
46 | WRITE_ONCE(nd->zc_copied, true); |
47 | } |
48 | io_tx_ubuf_callback(skb, uarg, success); |
49 | } |
50 | |
51 | void io_notif_set_extended(struct io_kiocb *notif) |
52 | { |
53 | struct io_notif_data *nd = io_notif_to_data(notif); |
54 | |
55 | if (nd->uarg.callback != io_tx_ubuf_callback_ext) { |
56 | nd->account_pages = 0; |
57 | nd->zc_report = false; |
58 | nd->zc_used = false; |
59 | nd->zc_copied = false; |
60 | nd->uarg.callback = io_tx_ubuf_callback_ext; |
61 | notif->io_task_work.func = io_notif_complete_tw_ext; |
62 | } |
63 | } |
64 | |
65 | struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) |
66 | __must_hold(&ctx->uring_lock) |
67 | { |
68 | struct io_kiocb *notif; |
69 | struct io_notif_data *nd; |
70 | |
71 | if (unlikely(!io_alloc_req(ctx, ¬if))) |
72 | return NULL; |
73 | notif->opcode = IORING_OP_NOP; |
74 | notif->flags = 0; |
75 | notif->file = NULL; |
76 | notif->task = current; |
77 | io_get_task_refs(nr: 1); |
78 | notif->rsrc_node = NULL; |
79 | notif->io_task_work.func = io_req_task_complete; |
80 | |
81 | nd = io_notif_to_data(notif); |
82 | nd->uarg.flags = IO_NOTIF_UBUF_FLAGS; |
83 | nd->uarg.callback = io_tx_ubuf_callback; |
84 | refcount_set(r: &nd->uarg.refcnt, n: 1); |
85 | return notif; |
86 | } |
87 | |