Warning: This file is not a C or C++ file. It does not have highlighting.
1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
---|---|
2 | /* I/O iterator iteration building functions. |
3 | * |
4 | * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. |
5 | * Written by David Howells (dhowells@redhat.com) |
6 | */ |
7 | |
8 | #ifndef _LINUX_IOV_ITER_H |
9 | #define _LINUX_IOV_ITER_H |
10 | |
11 | #include <linux/uio.h> |
12 | #include <linux/bvec.h> |
13 | |
14 | typedef size_t (*iov_step_f)(void *iter_base, size_t progress, size_t len, |
15 | void *priv, void *priv2); |
16 | typedef size_t (*iov_ustep_f)(void __user *iter_base, size_t progress, size_t len, |
17 | void *priv, void *priv2); |
18 | |
19 | /* |
20 | * Handle ITER_UBUF. |
21 | */ |
22 | static __always_inline |
23 | size_t iterate_ubuf(struct iov_iter *iter, size_t len, void *priv, void *priv2, |
24 | iov_ustep_f step) |
25 | { |
26 | void __user *base = iter->ubuf; |
27 | size_t progress = 0, remain; |
28 | |
29 | remain = step(base + iter->iov_offset, 0, len, priv, priv2); |
30 | progress = len - remain; |
31 | iter->iov_offset += progress; |
32 | iter->count -= progress; |
33 | return progress; |
34 | } |
35 | |
36 | /* |
37 | * Handle ITER_IOVEC. |
38 | */ |
39 | static __always_inline |
40 | size_t iterate_iovec(struct iov_iter *iter, size_t len, void *priv, void *priv2, |
41 | iov_ustep_f step) |
42 | { |
43 | const struct iovec *p = iter->__iov; |
44 | size_t progress = 0, skip = iter->iov_offset; |
45 | |
46 | do { |
47 | size_t remain, consumed; |
48 | size_t part = min(len, p->iov_len - skip); |
49 | |
50 | if (likely(part)) { |
51 | remain = step(p->iov_base + skip, progress, part, priv, priv2); |
52 | consumed = part - remain; |
53 | progress += consumed; |
54 | skip += consumed; |
55 | len -= consumed; |
56 | if (skip < p->iov_len) |
57 | break; |
58 | } |
59 | p++; |
60 | skip = 0; |
61 | } while (len); |
62 | |
63 | iter->nr_segs -= p - iter->__iov; |
64 | iter->__iov = p; |
65 | iter->iov_offset = skip; |
66 | iter->count -= progress; |
67 | return progress; |
68 | } |
69 | |
70 | /* |
71 | * Handle ITER_KVEC. |
72 | */ |
73 | static __always_inline |
74 | size_t iterate_kvec(struct iov_iter *iter, size_t len, void *priv, void *priv2, |
75 | iov_step_f step) |
76 | { |
77 | const struct kvec *p = iter->kvec; |
78 | size_t progress = 0, skip = iter->iov_offset; |
79 | |
80 | do { |
81 | size_t remain, consumed; |
82 | size_t part = min(len, p->iov_len - skip); |
83 | |
84 | if (likely(part)) { |
85 | remain = step(p->iov_base + skip, progress, part, priv, priv2); |
86 | consumed = part - remain; |
87 | progress += consumed; |
88 | skip += consumed; |
89 | len -= consumed; |
90 | if (skip < p->iov_len) |
91 | break; |
92 | } |
93 | p++; |
94 | skip = 0; |
95 | } while (len); |
96 | |
97 | iter->nr_segs -= p - iter->kvec; |
98 | iter->kvec = p; |
99 | iter->iov_offset = skip; |
100 | iter->count -= progress; |
101 | return progress; |
102 | } |
103 | |
104 | /* |
105 | * Handle ITER_BVEC. |
106 | */ |
107 | static __always_inline |
108 | size_t iterate_bvec(struct iov_iter *iter, size_t len, void *priv, void *priv2, |
109 | iov_step_f step) |
110 | { |
111 | const struct bio_vec *p = iter->bvec; |
112 | size_t progress = 0, skip = iter->iov_offset; |
113 | |
114 | do { |
115 | size_t remain, consumed; |
116 | size_t offset = p->bv_offset + skip, part; |
117 | void *kaddr = kmap_local_page(p->bv_page + offset / PAGE_SIZE); |
118 | |
119 | part = min3(len, |
120 | (size_t)(p->bv_len - skip), |
121 | (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); |
122 | remain = step(kaddr + offset % PAGE_SIZE, progress, part, priv, priv2); |
123 | kunmap_local(kaddr); |
124 | consumed = part - remain; |
125 | len -= consumed; |
126 | progress += consumed; |
127 | skip += consumed; |
128 | if (skip >= p->bv_len) { |
129 | skip = 0; |
130 | p++; |
131 | } |
132 | if (remain) |
133 | break; |
134 | } while (len); |
135 | |
136 | iter->nr_segs -= p - iter->bvec; |
137 | iter->bvec = p; |
138 | iter->iov_offset = skip; |
139 | iter->count -= progress; |
140 | return progress; |
141 | } |
142 | |
143 | /* |
144 | * Handle ITER_XARRAY. |
145 | */ |
146 | static __always_inline |
147 | size_t iterate_xarray(struct iov_iter *iter, size_t len, void *priv, void *priv2, |
148 | iov_step_f step) |
149 | { |
150 | struct folio *folio; |
151 | size_t progress = 0; |
152 | loff_t start = iter->xarray_start + iter->iov_offset; |
153 | pgoff_t index = start / PAGE_SIZE; |
154 | XA_STATE(xas, iter->xarray, index); |
155 | |
156 | rcu_read_lock(); |
157 | xas_for_each(&xas, folio, ULONG_MAX) { |
158 | size_t remain, consumed, offset, part, flen; |
159 | |
160 | if (xas_retry(&xas, folio)) |
161 | continue; |
162 | if (WARN_ON(xa_is_value(folio))) |
163 | break; |
164 | if (WARN_ON(folio_test_hugetlb(folio))) |
165 | break; |
166 | |
167 | offset = offset_in_folio(folio, start + progress); |
168 | flen = min(folio_size(folio) - offset, len); |
169 | |
170 | while (flen) { |
171 | void *base = kmap_local_folio(folio, offset); |
172 | |
173 | part = min_t(size_t, flen, |
174 | PAGE_SIZE - offset_in_page(offset)); |
175 | remain = step(base, progress, part, priv, priv2); |
176 | kunmap_local(base); |
177 | |
178 | consumed = part - remain; |
179 | progress += consumed; |
180 | len -= consumed; |
181 | |
182 | if (remain || len == 0) |
183 | goto out; |
184 | flen -= consumed; |
185 | offset += consumed; |
186 | } |
187 | } |
188 | |
189 | out: |
190 | rcu_read_unlock(); |
191 | iter->iov_offset += progress; |
192 | iter->count -= progress; |
193 | return progress; |
194 | } |
195 | |
196 | /* |
197 | * Handle ITER_DISCARD. |
198 | */ |
199 | static __always_inline |
200 | size_t iterate_discard(struct iov_iter *iter, size_t len, void *priv, void *priv2, |
201 | iov_step_f step) |
202 | { |
203 | size_t progress = len; |
204 | |
205 | iter->count -= progress; |
206 | return progress; |
207 | } |
208 | |
209 | /** |
210 | * iterate_and_advance2 - Iterate over an iterator |
211 | * @iter: The iterator to iterate over. |
212 | * @len: The amount to iterate over. |
213 | * @priv: Data for the step functions. |
214 | * @priv2: More data for the step functions. |
215 | * @ustep: Function for UBUF/IOVEC iterators; given __user addresses. |
216 | * @step: Function for other iterators; given kernel addresses. |
217 | * |
218 | * Iterate over the next part of an iterator, up to the specified length. The |
219 | * buffer is presented in segments, which for kernel iteration are broken up by |
220 | * physical pages and mapped, with the mapped address being presented. |
221 | * |
222 | * Two step functions, @step and @ustep, must be provided, one for handling |
223 | * mapped kernel addresses and the other is given user addresses which have the |
224 | * potential to fault since no pinning is performed. |
225 | * |
226 | * The step functions are passed the address and length of the segment, @priv, |
227 | * @priv2 and the amount of data so far iterated over (which can, for example, |
228 | * be added to @priv to point to the right part of a second buffer). The step |
229 | * functions should return the amount of the segment they didn't process (ie. 0 |
230 | * indicates complete processsing). |
231 | * |
232 | * This function returns the amount of data processed (ie. 0 means nothing was |
233 | * processed and the value of @len means processes to completion). |
234 | */ |
235 | static __always_inline |
236 | size_t iterate_and_advance2(struct iov_iter *iter, size_t len, void *priv, |
237 | void *priv2, iov_ustep_f ustep, iov_step_f step) |
238 | { |
239 | if (unlikely(iter->count < len)) |
240 | len = iter->count; |
241 | if (unlikely(!len)) |
242 | return 0; |
243 | |
244 | if (likely(iter_is_ubuf(iter))) |
245 | return iterate_ubuf(iter, len, priv, priv2, ustep); |
246 | if (likely(iter_is_iovec(iter))) |
247 | return iterate_iovec(iter, len, priv, priv2, ustep); |
248 | if (iov_iter_is_bvec(iter)) |
249 | return iterate_bvec(iter, len, priv, priv2, step); |
250 | if (iov_iter_is_kvec(iter)) |
251 | return iterate_kvec(iter, len, priv, priv2, step); |
252 | if (iov_iter_is_xarray(iter)) |
253 | return iterate_xarray(iter, len, priv, priv2, step); |
254 | return iterate_discard(iter, len, priv, priv2, step); |
255 | } |
256 | |
257 | /** |
258 | * iterate_and_advance - Iterate over an iterator |
259 | * @iter: The iterator to iterate over. |
260 | * @len: The amount to iterate over. |
261 | * @priv: Data for the step functions. |
262 | * @ustep: Function for UBUF/IOVEC iterators; given __user addresses. |
263 | * @step: Function for other iterators; given kernel addresses. |
264 | * |
265 | * As iterate_and_advance2(), but priv2 is always NULL. |
266 | */ |
267 | static __always_inline |
268 | size_t iterate_and_advance(struct iov_iter *iter, size_t len, void *priv, |
269 | iov_ustep_f ustep, iov_step_f step) |
270 | { |
271 | return iterate_and_advance2(iter, len, priv, NULL, ustep, step); |
272 | } |
273 | |
274 | #endif /* _LINUX_IOV_ITER_H */ |
275 |
Warning: This file is not a C or C++ file. It does not have highlighting.