| 1 | // SPDX-License-Identifier: GPL-2.0 |
|---|---|
| 2 | |
| 3 | #include <linux/net.h> |
| 4 | #include <linux/uio.h> |
| 5 | #include <net/sock.h> |
| 6 | #include <linux/nospec.h> |
| 7 | |
| 8 | #include "rsrc.h" |
| 9 | |
| 10 | #define IO_NOTIF_UBUF_FLAGS (SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN) |
| 11 | #define IO_NOTIF_SPLICE_BATCH 32 |
| 12 | |
| 13 | struct io_notif_data { |
| 14 | struct file *file; |
| 15 | struct ubuf_info uarg; |
| 16 | unsigned long account_pages; |
| 17 | bool zc_report; |
| 18 | bool zc_used; |
| 19 | bool zc_copied; |
| 20 | }; |
| 21 | |
| 22 | struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx); |
| 23 | void io_notif_set_extended(struct io_kiocb *notif); |
| 24 | |
| 25 | static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif) |
| 26 | { |
| 27 | return io_kiocb_to_cmd(notif, struct io_notif_data); |
| 28 | } |
| 29 | |
| 30 | static inline void io_notif_flush(struct io_kiocb *notif) |
| 31 | __must_hold(¬if->ctx->uring_lock) |
| 32 | { |
| 33 | struct io_notif_data *nd = io_notif_to_data(notif); |
| 34 | |
| 35 | /* drop slot's master ref */ |
| 36 | if (refcount_dec_and_test(r: &nd->uarg.refcnt)) |
| 37 | __io_req_task_work_add(req: notif, flags: IOU_F_TWQ_LAZY_WAKE); |
| 38 | } |
| 39 | |
| 40 | static inline int io_notif_account_mem(struct io_kiocb *notif, unsigned len) |
| 41 | { |
| 42 | struct io_ring_ctx *ctx = notif->ctx; |
| 43 | struct io_notif_data *nd = io_notif_to_data(notif); |
| 44 | unsigned nr_pages = (len >> PAGE_SHIFT) + 2; |
| 45 | int ret; |
| 46 | |
| 47 | if (ctx->user) { |
| 48 | ret = __io_account_mem(user: ctx->user, nr_pages); |
| 49 | if (ret) |
| 50 | return ret; |
| 51 | nd->account_pages += nr_pages; |
| 52 | } |
| 53 | return 0; |
| 54 | } |
| 55 |