1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* Internal definitions for network filesystem support |
3 | * |
4 | * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. |
5 | * Written by David Howells (dhowells@redhat.com) |
6 | */ |
7 | |
8 | #include <linux/slab.h> |
9 | #include <linux/seq_file.h> |
10 | #include <linux/netfs.h> |
11 | #include <linux/fscache.h> |
12 | #include <linux/fscache-cache.h> |
13 | #include <trace/events/netfs.h> |
14 | #include <trace/events/fscache.h> |
15 | |
16 | #ifdef pr_fmt |
17 | #undef pr_fmt |
18 | #endif |
19 | |
20 | #define pr_fmt(fmt) "netfs: " fmt |
21 | |
22 | /* |
23 | * buffered_read.c |
24 | */ |
25 | void netfs_rreq_unlock_folios(struct netfs_io_request *rreq); |
26 | int netfs_prefetch_for_write(struct file *file, struct folio *folio, |
27 | size_t offset, size_t len); |
28 | |
29 | /* |
30 | * io.c |
31 | */ |
32 | int netfs_begin_read(struct netfs_io_request *rreq, bool sync); |
33 | |
34 | /* |
35 | * main.c |
36 | */ |
37 | extern unsigned int netfs_debug; |
38 | extern struct list_head netfs_io_requests; |
39 | extern spinlock_t netfs_proc_lock; |
40 | |
41 | #ifdef CONFIG_PROC_FS |
42 | static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) |
43 | { |
44 | spin_lock(lock: &netfs_proc_lock); |
45 | list_add_tail_rcu(new: &rreq->proc_link, head: &netfs_io_requests); |
46 | spin_unlock(lock: &netfs_proc_lock); |
47 | } |
48 | static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) |
49 | { |
50 | if (!list_empty(head: &rreq->proc_link)) { |
51 | spin_lock(lock: &netfs_proc_lock); |
52 | list_del_rcu(entry: &rreq->proc_link); |
53 | spin_unlock(lock: &netfs_proc_lock); |
54 | } |
55 | } |
56 | #else |
57 | static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) {} |
58 | static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {} |
59 | #endif |
60 | |
61 | /* |
62 | * misc.c |
63 | */ |
64 | #define NETFS_FLAG_PUT_MARK BIT(0) |
65 | #define NETFS_FLAG_PAGECACHE_MARK BIT(1) |
66 | int netfs_xa_store_and_mark(struct xarray *xa, unsigned long index, |
67 | struct folio *folio, unsigned int flags, |
68 | gfp_t gfp_mask); |
69 | int netfs_add_folios_to_buffer(struct xarray *buffer, |
70 | struct address_space *mapping, |
71 | pgoff_t index, pgoff_t to, gfp_t gfp_mask); |
72 | void netfs_clear_buffer(struct xarray *buffer); |
73 | |
74 | /* |
75 | * objects.c |
76 | */ |
77 | struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, |
78 | struct file *file, |
79 | loff_t start, size_t len, |
80 | enum netfs_io_origin origin); |
81 | void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what); |
82 | void netfs_clear_subrequests(struct netfs_io_request *rreq, bool was_async); |
83 | void netfs_put_request(struct netfs_io_request *rreq, bool was_async, |
84 | enum netfs_rreq_ref_trace what); |
85 | struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq); |
86 | |
87 | static inline void netfs_see_request(struct netfs_io_request *rreq, |
88 | enum netfs_rreq_ref_trace what) |
89 | { |
90 | trace_netfs_rreq_ref(rreq_debug_id: rreq->debug_id, ref: refcount_read(r: &rreq->ref), what); |
91 | } |
92 | |
93 | /* |
94 | * output.c |
95 | */ |
96 | int netfs_begin_write(struct netfs_io_request *wreq, bool may_wait, |
97 | enum netfs_write_trace what); |
98 | struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len); |
99 | int netfs_advance_writethrough(struct netfs_io_request *wreq, size_t copied, bool to_page_end); |
100 | int netfs_end_writethrough(struct netfs_io_request *wreq, struct kiocb *iocb); |
101 | |
102 | /* |
103 | * stats.c |
104 | */ |
105 | #ifdef CONFIG_NETFS_STATS |
106 | extern atomic_t netfs_n_rh_dio_read; |
107 | extern atomic_t netfs_n_rh_dio_write; |
108 | extern atomic_t netfs_n_rh_readahead; |
109 | extern atomic_t netfs_n_rh_readpage; |
110 | extern atomic_t netfs_n_rh_rreq; |
111 | extern atomic_t netfs_n_rh_sreq; |
112 | extern atomic_t netfs_n_rh_download; |
113 | extern atomic_t netfs_n_rh_download_done; |
114 | extern atomic_t netfs_n_rh_download_failed; |
115 | extern atomic_t netfs_n_rh_download_instead; |
116 | extern atomic_t netfs_n_rh_read; |
117 | extern atomic_t netfs_n_rh_read_done; |
118 | extern atomic_t netfs_n_rh_read_failed; |
119 | extern atomic_t netfs_n_rh_zero; |
120 | extern atomic_t netfs_n_rh_short_read; |
121 | extern atomic_t netfs_n_rh_write; |
122 | extern atomic_t netfs_n_rh_write_begin; |
123 | extern atomic_t netfs_n_rh_write_done; |
124 | extern atomic_t netfs_n_rh_write_failed; |
125 | extern atomic_t netfs_n_rh_write_zskip; |
126 | extern atomic_t netfs_n_wh_wstream_conflict; |
127 | extern atomic_t netfs_n_wh_upload; |
128 | extern atomic_t netfs_n_wh_upload_done; |
129 | extern atomic_t netfs_n_wh_upload_failed; |
130 | extern atomic_t netfs_n_wh_write; |
131 | extern atomic_t netfs_n_wh_write_done; |
132 | extern atomic_t netfs_n_wh_write_failed; |
133 | |
134 | int netfs_stats_show(struct seq_file *m, void *v); |
135 | |
136 | static inline void netfs_stat(atomic_t *stat) |
137 | { |
138 | atomic_inc(v: stat); |
139 | } |
140 | |
141 | static inline void netfs_stat_d(atomic_t *stat) |
142 | { |
143 | atomic_dec(v: stat); |
144 | } |
145 | |
146 | #else |
147 | #define netfs_stat(x) do {} while(0) |
148 | #define netfs_stat_d(x) do {} while(0) |
149 | #endif |
150 | |
151 | /* |
152 | * Miscellaneous functions. |
153 | */ |
154 | static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx) |
155 | { |
156 | #if IS_ENABLED(CONFIG_FSCACHE) |
157 | struct fscache_cookie *cookie = ctx->cache; |
158 | |
159 | return fscache_cookie_valid(cookie) && cookie->cache_priv && |
160 | fscache_cookie_enabled(cookie); |
161 | #else |
162 | return false; |
163 | #endif |
164 | } |
165 | |
166 | /* |
167 | * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap). |
168 | */ |
169 | static inline struct netfs_group *netfs_get_group(struct netfs_group *netfs_group) |
170 | { |
171 | if (netfs_group) |
172 | refcount_inc(r: &netfs_group->ref); |
173 | return netfs_group; |
174 | } |
175 | |
176 | /* |
177 | * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap). |
178 | */ |
179 | static inline void netfs_put_group(struct netfs_group *netfs_group) |
180 | { |
181 | if (netfs_group && refcount_dec_and_test(r: &netfs_group->ref)) |
182 | netfs_group->free(netfs_group); |
183 | } |
184 | |
185 | /* |
186 | * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap). |
187 | */ |
188 | static inline void netfs_put_group_many(struct netfs_group *netfs_group, int nr) |
189 | { |
190 | if (netfs_group && refcount_sub_and_test(i: nr, r: &netfs_group->ref)) |
191 | netfs_group->free(netfs_group); |
192 | } |
193 | |
194 | /* |
195 | * fscache-cache.c |
196 | */ |
197 | #ifdef CONFIG_PROC_FS |
198 | extern const struct seq_operations fscache_caches_seq_ops; |
199 | #endif |
200 | bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); |
201 | void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); |
202 | struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache); |
203 | void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where); |
204 | |
205 | static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache) |
206 | { |
207 | return smp_load_acquire(&cache->state); |
208 | } |
209 | |
210 | static inline bool fscache_cache_is_live(const struct fscache_cache *cache) |
211 | { |
212 | return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE; |
213 | } |
214 | |
215 | static inline void fscache_set_cache_state(struct fscache_cache *cache, |
216 | enum fscache_cache_state new_state) |
217 | { |
218 | smp_store_release(&cache->state, new_state); |
219 | |
220 | } |
221 | |
222 | static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, |
223 | enum fscache_cache_state old_state, |
224 | enum fscache_cache_state new_state) |
225 | { |
226 | return try_cmpxchg_release(&cache->state, &old_state, new_state); |
227 | } |
228 | |
229 | /* |
230 | * fscache-cookie.c |
231 | */ |
232 | extern struct kmem_cache *fscache_cookie_jar; |
233 | #ifdef CONFIG_PROC_FS |
234 | extern const struct seq_operations fscache_cookies_seq_ops; |
235 | #endif |
236 | extern struct timer_list fscache_cookie_lru_timer; |
237 | |
238 | extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); |
239 | extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie, |
240 | enum fscache_access_trace why); |
241 | |
242 | static inline void fscache_see_cookie(struct fscache_cookie *cookie, |
243 | enum fscache_cookie_trace where) |
244 | { |
245 | trace_fscache_cookie(cookie_debug_id: cookie->debug_id, ref: refcount_read(r: &cookie->ref), |
246 | where); |
247 | } |
248 | |
249 | /* |
250 | * fscache-main.c |
251 | */ |
252 | extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len); |
253 | #ifdef CONFIG_FSCACHE |
254 | int __init fscache_init(void); |
255 | void __exit fscache_exit(void); |
256 | #else |
257 | static inline int fscache_init(void) { return 0; } |
258 | static inline void fscache_exit(void) {} |
259 | #endif |
260 | |
261 | /* |
262 | * fscache-proc.c |
263 | */ |
264 | #ifdef CONFIG_PROC_FS |
265 | extern int __init fscache_proc_init(void); |
266 | extern void fscache_proc_cleanup(void); |
267 | #else |
268 | #define fscache_proc_init() (0) |
269 | #define fscache_proc_cleanup() do {} while (0) |
270 | #endif |
271 | |
272 | /* |
273 | * fscache-stats.c |
274 | */ |
275 | #ifdef CONFIG_FSCACHE_STATS |
276 | extern atomic_t fscache_n_volumes; |
277 | extern atomic_t fscache_n_volumes_collision; |
278 | extern atomic_t fscache_n_volumes_nomem; |
279 | extern atomic_t fscache_n_cookies; |
280 | extern atomic_t fscache_n_cookies_lru; |
281 | extern atomic_t fscache_n_cookies_lru_expired; |
282 | extern atomic_t fscache_n_cookies_lru_removed; |
283 | extern atomic_t fscache_n_cookies_lru_dropped; |
284 | |
285 | extern atomic_t fscache_n_acquires; |
286 | extern atomic_t fscache_n_acquires_ok; |
287 | extern atomic_t fscache_n_acquires_oom; |
288 | |
289 | extern atomic_t fscache_n_invalidates; |
290 | |
291 | extern atomic_t fscache_n_relinquishes; |
292 | extern atomic_t fscache_n_relinquishes_retire; |
293 | extern atomic_t fscache_n_relinquishes_dropped; |
294 | |
295 | extern atomic_t fscache_n_resizes; |
296 | extern atomic_t fscache_n_resizes_null; |
297 | |
298 | static inline void fscache_stat(atomic_t *stat) |
299 | { |
300 | atomic_inc(v: stat); |
301 | } |
302 | |
303 | static inline void fscache_stat_d(atomic_t *stat) |
304 | { |
305 | atomic_dec(v: stat); |
306 | } |
307 | |
308 | #define __fscache_stat(stat) (stat) |
309 | |
310 | int fscache_stats_show(struct seq_file *m); |
311 | #else |
312 | |
313 | #define __fscache_stat(stat) (NULL) |
314 | #define fscache_stat(stat) do {} while (0) |
315 | #define fscache_stat_d(stat) do {} while (0) |
316 | |
317 | static inline int fscache_stats_show(struct seq_file *m) { return 0; } |
318 | #endif |
319 | |
320 | /* |
321 | * fscache-volume.c |
322 | */ |
323 | #ifdef CONFIG_PROC_FS |
324 | extern const struct seq_operations fscache_volumes_seq_ops; |
325 | #endif |
326 | |
327 | struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, |
328 | enum fscache_volume_trace where); |
329 | void fscache_put_volume(struct fscache_volume *volume, |
330 | enum fscache_volume_trace where); |
331 | bool fscache_begin_volume_access(struct fscache_volume *volume, |
332 | struct fscache_cookie *cookie, |
333 | enum fscache_access_trace why); |
334 | void fscache_create_volume(struct fscache_volume *volume, bool wait); |
335 | |
336 | /*****************************************************************************/ |
337 | /* |
338 | * debug tracing |
339 | */ |
340 | #define dbgprintk(FMT, ...) \ |
341 | printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) |
342 | |
343 | #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) |
344 | #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) |
345 | #define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) |
346 | |
347 | #ifdef __KDEBUG |
348 | #define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) |
349 | #define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) |
350 | #define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) |
351 | |
352 | #elif defined(CONFIG_NETFS_DEBUG) |
353 | #define _enter(FMT, ...) \ |
354 | do { \ |
355 | if (netfs_debug) \ |
356 | kenter(FMT, ##__VA_ARGS__); \ |
357 | } while (0) |
358 | |
359 | #define _leave(FMT, ...) \ |
360 | do { \ |
361 | if (netfs_debug) \ |
362 | kleave(FMT, ##__VA_ARGS__); \ |
363 | } while (0) |
364 | |
365 | #define _debug(FMT, ...) \ |
366 | do { \ |
367 | if (netfs_debug) \ |
368 | kdebug(FMT, ##__VA_ARGS__); \ |
369 | } while (0) |
370 | |
371 | #else |
372 | #define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) |
373 | #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) |
374 | #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) |
375 | #endif |
376 | |
377 | /* |
378 | * assertions |
379 | */ |
380 | #if 1 /* defined(__KDEBUGALL) */ |
381 | |
382 | #define ASSERT(X) \ |
383 | do { \ |
384 | if (unlikely(!(X))) { \ |
385 | pr_err("\n"); \ |
386 | pr_err("Assertion failed\n"); \ |
387 | BUG(); \ |
388 | } \ |
389 | } while (0) |
390 | |
391 | #define ASSERTCMP(X, OP, Y) \ |
392 | do { \ |
393 | if (unlikely(!((X) OP (Y)))) { \ |
394 | pr_err("\n"); \ |
395 | pr_err("Assertion failed\n"); \ |
396 | pr_err("%lx " #OP " %lx is false\n", \ |
397 | (unsigned long)(X), (unsigned long)(Y)); \ |
398 | BUG(); \ |
399 | } \ |
400 | } while (0) |
401 | |
402 | #define ASSERTIF(C, X) \ |
403 | do { \ |
404 | if (unlikely((C) && !(X))) { \ |
405 | pr_err("\n"); \ |
406 | pr_err("Assertion failed\n"); \ |
407 | BUG(); \ |
408 | } \ |
409 | } while (0) |
410 | |
411 | #define ASSERTIFCMP(C, X, OP, Y) \ |
412 | do { \ |
413 | if (unlikely((C) && !((X) OP (Y)))) { \ |
414 | pr_err("\n"); \ |
415 | pr_err("Assertion failed\n"); \ |
416 | pr_err("%lx " #OP " %lx is false\n", \ |
417 | (unsigned long)(X), (unsigned long)(Y)); \ |
418 | BUG(); \ |
419 | } \ |
420 | } while (0) |
421 | |
422 | #else |
423 | |
424 | #define ASSERT(X) do {} while (0) |
425 | #define ASSERTCMP(X, OP, Y) do {} while (0) |
426 | #define ASSERTIF(C, X) do {} while (0) |
427 | #define ASSERTIFCMP(C, X, OP, Y) do {} while (0) |
428 | |
429 | #endif /* assert or not */ |
430 | |