1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/fs/nfs/pagelist.c
4	*
5	* A set of helper functions for managing NFS read and write requests.
6	* The main purpose of these routines is to provide support for the
7	* coalescing of several requests into a single RPC call.
8	*
9	* Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no>
10	*
11	*/
12
13	#include <linux/slab.h>
14	#include <linux/file.h>
15	#include <linux/sched.h>
16	#include <linux/sunrpc/clnt.h>
17	#include <linux/nfs.h>
18	#include <linux/nfs3.h>
19	#include <linux/nfs4.h>
20	#include <linux/nfs_fs.h>
21	#include <linux/nfs_page.h>
22	#include <linux/nfs_mount.h>
23	#include <linux/export.h>
24	#include <linux/filelock.h>
25
26	#include "internal.h"
27	#include "pnfs.h"
28	#include "nfstrace.h"
29	#include "fscache.h"
30
31	#define NFSDBG_FACILITY NFSDBG_PAGECACHE
32
33	static struct kmem_cache *nfs_page_cachep;
34	static const struct rpc_call_ops nfs_pgio_common_ops;
35
36	struct nfs_page_iter_page {
37	const struct nfs_page *req;
38	size_t count;
39	};
40
41	static void nfs_page_iter_page_init(struct nfs_page_iter_page *i,
42	const struct nfs_page *req)
43	{
44	i->req = req;
45	i->count = 0;
46	}
47
48	static void nfs_page_iter_page_advance(struct nfs_page_iter_page *i, size_t sz)
49	{
50	const struct nfs_page *req = i->req;
51	size_t tmp = i->count + sz;
52
53	i->count = (tmp < req->wb_bytes) ? tmp : req->wb_bytes;
54	}
55
56	static struct page *nfs_page_iter_page_get(struct nfs_page_iter_page *i)
57	{
58	const struct nfs_page *req = i->req;
59	struct page *page;
60
61	if (i->count != req->wb_bytes) {
62	size_t base = i->count + req->wb_pgbase;
63	size_t len = PAGE_SIZE - offset_in_page(base);
64
65	page = nfs_page_to_page(req, pgbase: base);
66	nfs_page_iter_page_advance(i, sz: len);
67	return page;
68	}
69	return NULL;
70	}
71
72	static struct nfs_pgio_mirror *
73	nfs_pgio_get_mirror(struct nfs_pageio_descriptor *desc, u32 idx)
74	{
75	if (desc->pg_ops->pg_get_mirror)
76	return desc->pg_ops->pg_get_mirror(desc, idx);
77	return &desc->pg_mirrors[0];
78	}
79
80	struct nfs_pgio_mirror *
81	nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc)
82	{
83	return nfs_pgio_get_mirror(desc, idx: desc->pg_mirror_idx);
84	}
85	EXPORT_SYMBOL_GPL(nfs_pgio_current_mirror);
86
87	static u32
88	nfs_pgio_set_current_mirror(struct nfs_pageio_descriptor *desc, u32 idx)
89	{
90	if (desc->pg_ops->pg_set_mirror)
91	return desc->pg_ops->pg_set_mirror(desc, idx);
92	return desc->pg_mirror_idx;
93	}
94
95	void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
96	struct nfs_pgio_header *hdr,
97	void (*release)(struct nfs_pgio_header *hdr))
98	{
99	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
100
101
102	hdr->req = nfs_list_entry(head: mirror->pg_list.next);
103	hdr->inode = desc->pg_inode;
104	hdr->cred = nfs_req_openctx(req: hdr->req)->cred;
105	hdr->io_start = req_offset(req: hdr->req);
106	hdr->good_bytes = mirror->pg_count;
107	hdr->io_completion = desc->pg_io_completion;
108	hdr->dreq = desc->pg_dreq;
109	nfs_netfs_set_pgio_header(hdr, desc);
110	hdr->release = release;
111	hdr->completion_ops = desc->pg_completion_ops;
112	if (hdr->completion_ops->init_hdr)
113	hdr->completion_ops->init_hdr(hdr);
114
115	hdr->pgio_mirror_idx = desc->pg_mirror_idx;
116	}
117	EXPORT_SYMBOL_GPL(nfs_pgheader_init);
118
119	void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos)
120	{
121	unsigned int new = pos - hdr->io_start;
122
123	trace_nfs_pgio_error(hdr, error, pos);
124	if (hdr->good_bytes > new) {
125	hdr->good_bytes = new;
126	clear_bit(nr: NFS_IOHDR_EOF, addr: &hdr->flags);
127	if (!test_and_set_bit(nr: NFS_IOHDR_ERROR, addr: &hdr->flags))
128	hdr->error = error;
129	}
130	}
131
132	static inline struct nfs_page *nfs_page_alloc(void)
133	{
134	struct nfs_page *p =
135	kmem_cache_zalloc(k: nfs_page_cachep, flags: nfs_io_gfp_mask());
136	if (p)
137	INIT_LIST_HEAD(list: &p->wb_list);
138	return p;
139	}
140
141	static inline void
142	nfs_page_free(struct nfs_page *p)
143	{
144	kmem_cache_free(s: nfs_page_cachep, objp: p);
145	}
146
147	/**
148	* nfs_iocounter_wait - wait for i/o to complete
149	* @l_ctx: nfs_lock_context with io_counter to use
150	*
151	* returns -ERESTARTSYS if interrupted by a fatal signal.
152	* Otherwise returns 0 once the io_count hits 0.
153	*/
154	int
155	nfs_iocounter_wait(struct nfs_lock_context *l_ctx)
156	{
157	return wait_var_event_killable(&l_ctx->io_count,
158	!atomic_read(&l_ctx->io_count));
159	}
160
161	/**
162	* nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O
163	* to complete
164	* @task: the rpc_task that should wait
165	* @l_ctx: nfs_lock_context with io_counter to check
166	*
167	* Returns true if there is outstanding I/O to wait on and the
168	* task has been put to sleep.
169	*/
170	bool
171	nfs_async_iocounter_wait(struct rpc_task *task, struct nfs_lock_context *l_ctx)
172	{
173	struct inode *inode = d_inode(dentry: l_ctx->open_context->dentry);
174	bool ret = false;
175
176	if (atomic_read(v: &l_ctx->io_count) > 0) {
177	rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL);
178	ret = true;
179	}
180
181	if (atomic_read(v: &l_ctx->io_count) == 0) {
182	rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task);
183	ret = false;
184	}
185
186	return ret;
187	}
188	EXPORT_SYMBOL_GPL(nfs_async_iocounter_wait);
189
190	/*
191	* nfs_page_lock_head_request - page lock the head of the page group
192	* @req: any member of the page group
193	*/
194	struct nfs_page *
195	nfs_page_group_lock_head(struct nfs_page *req)
196	{
197	struct nfs_page *head = req->wb_head;
198
199	while (!nfs_lock_request(req: head)) {
200	int ret = nfs_wait_on_request(head);
201	if (ret < 0)
202	return ERR_PTR(error: ret);
203	}
204	if (head != req)
205	kref_get(kref: &head->wb_kref);
206	return head;
207	}
208
209	/*
210	* nfs_unroll_locks - unlock all newly locked reqs and wait on @req
211	* @head: head request of page group, must be holding head lock
212	* @req: request that couldn't lock and needs to wait on the req bit lock
213	*
214	* This is a helper function for nfs_lock_and_join_requests
215	* returns 0 on success, < 0 on error.
216	*/
217	static void
218	nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req)
219	{
220	struct nfs_page *tmp;
221
222	/* relinquish all the locks successfully grabbed this run */
223	for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
224	if (!kref_read(kref: &tmp->wb_kref))
225	continue;
226	nfs_unlock_and_release_request(tmp);
227	}
228	}
229
230	/*
231	* nfs_page_group_lock_subreq - try to lock a subrequest
232	* @head: head request of page group
233	* @subreq: request to lock
234	*
235	* This is a helper function for nfs_lock_and_join_requests which
236	* must be called with the head request and page group both locked.
237	* On error, it returns with the page group unlocked.
238	*/
239	static int
240	nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq)
241	{
242	int ret;
243
244	if (!kref_get_unless_zero(kref: &subreq->wb_kref))
245	return 0;
246	while (!nfs_lock_request(req: subreq)) {
247	nfs_page_group_unlock(head);
248	ret = nfs_wait_on_request(subreq);
249	if (!ret)
250	ret = nfs_page_group_lock(head);
251	if (ret < 0) {
252	nfs_unroll_locks(head, req: subreq);
253	nfs_release_request(subreq);
254	return ret;
255	}
256	}
257	return 0;
258	}
259
260	/*
261	* nfs_page_group_lock_subrequests - try to lock the subrequests
262	* @head: head request of page group
263	*
264	* This is a helper function for nfs_lock_and_join_requests which
265	* must be called with the head request locked.
266	*/
267	int nfs_page_group_lock_subrequests(struct nfs_page *head)
268	{
269	struct nfs_page *subreq;
270	int ret;
271
272	ret = nfs_page_group_lock(head);
273	if (ret < 0)
274	return ret;
275	/* lock each request in the page group */
276	for (subreq = head->wb_this_page; subreq != head;
277	subreq = subreq->wb_this_page) {
278	ret = nfs_page_group_lock_subreq(head, subreq);
279	if (ret < 0)
280	return ret;
281	}
282	nfs_page_group_unlock(head);
283	return 0;
284	}
285
286	/*
287	* nfs_page_set_headlock - set the request PG_HEADLOCK
288	* @req: request that is to be locked
289	*
290	* this lock must be held when modifying req->wb_head
291	*
292	* return 0 on success, < 0 on error
293	*/
294	int
295	nfs_page_set_headlock(struct nfs_page *req)
296	{
297	if (!test_and_set_bit(nr: PG_HEADLOCK, addr: &req->wb_flags))
298	return 0;
299
300	set_bit(nr: PG_CONTENDED1, addr: &req->wb_flags);
301	smp_mb__after_atomic();
302	return wait_on_bit_lock(word: &req->wb_flags, bit: PG_HEADLOCK,
303	TASK_UNINTERRUPTIBLE);
304	}
305
306	/*
307	* nfs_page_clear_headlock - clear the request PG_HEADLOCK
308	* @req: request that is to be locked
309	*/
310	void
311	nfs_page_clear_headlock(struct nfs_page *req)
312	{
313	clear_bit_unlock(nr: PG_HEADLOCK, addr: &req->wb_flags);
314	smp_mb__after_atomic();
315	if (!test_bit(PG_CONTENDED1, &req->wb_flags))
316	return;
317	wake_up_bit(word: &req->wb_flags, bit: PG_HEADLOCK);
318	}
319
320	/*
321	* nfs_page_group_lock - lock the head of the page group
322	* @req: request in group that is to be locked
323	*
324	* this lock must be held when traversing or modifying the page
325	* group list
326	*
327	* return 0 on success, < 0 on error
328	*/
329	int
330	nfs_page_group_lock(struct nfs_page *req)
331	{
332	int ret;
333
334	ret = nfs_page_set_headlock(req);
335	if (ret || req->wb_head == req)
336	return ret;
337	return nfs_page_set_headlock(req: req->wb_head);
338	}
339
340	/*
341	* nfs_page_group_unlock - unlock the head of the page group
342	* @req: request in group that is to be unlocked
343	*/
344	void
345	nfs_page_group_unlock(struct nfs_page *req)
346	{
347	if (req != req->wb_head)
348	nfs_page_clear_headlock(req: req->wb_head);
349	nfs_page_clear_headlock(req);
350	}
351
352	/*
353	* nfs_page_group_sync_on_bit_locked
354	*
355	* must be called with page group lock held
356	*/
357	static bool
358	nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit)
359	{
360	struct nfs_page *head = req->wb_head;
361	struct nfs_page *tmp;
362
363	WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags));
364	WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));
365
366	tmp = req->wb_this_page;
367	while (tmp != req) {
368	if (!test_bit(bit, &tmp->wb_flags))
369	return false;
370	tmp = tmp->wb_this_page;
371	}
372
373	/* true! reset all bits */
374	tmp = req;
375	do {
376	clear_bit(nr: bit, addr: &tmp->wb_flags);
377	tmp = tmp->wb_this_page;
378	} while (tmp != req);
379
380	return true;
381	}
382
383	/*
384	* nfs_page_group_sync_on_bit - set bit on current request, but only
385	* return true if the bit is set for all requests in page group
386	* @req - request in page group
387	* @bit - PG_* bit that is used to sync page group
388	*/
389	bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit)
390	{
391	bool ret;
392
393	nfs_page_group_lock(req);
394	ret = nfs_page_group_sync_on_bit_locked(req, bit);
395	nfs_page_group_unlock(req);
396
397	return ret;
398	}
399
400	/*
401	* nfs_page_group_init - Initialize the page group linkage for @req
402	* @req - a new nfs request
403	* @prev - the previous request in page group, or NULL if @req is the first
404	* or only request in the group (the head).
405	*/
406	static inline void
407	nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev)
408	{
409	struct inode *inode;
410	WARN_ON_ONCE(prev == req);
411
412	if (!prev) {
413	/* a head request */
414	req->wb_head = req;
415	req->wb_this_page = req;
416	} else {
417	/* a subrequest */
418	WARN_ON_ONCE(prev->wb_this_page != prev->wb_head);
419	WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags));
420	req->wb_head = prev->wb_head;
421	req->wb_this_page = prev->wb_this_page;
422	prev->wb_this_page = req;
423
424	/* All subrequests take a ref on the head request until
425	* nfs_page_group_destroy is called */
426	kref_get(kref: &req->wb_head->wb_kref);
427
428	/* grab extra ref and bump the request count if head request
429	* has extra ref from the write/commit path to handle handoff
430	* between write and commit lists. */
431	if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) {
432	inode = nfs_page_to_inode(req);
433	set_bit(nr: PG_INODE_REF, addr: &req->wb_flags);
434	kref_get(kref: &req->wb_kref);
435	atomic_long_inc(v: &NFS_I(inode)->nrequests);
436	}
437	}
438	}
439
440	/*
441	* nfs_page_group_destroy - sync the destruction of page groups
442	* @req - request that no longer needs the page group
443	*
444	* releases the page group reference from each member once all
445	* members have called this function.
446	*/
447	static void
448	nfs_page_group_destroy(struct kref *kref)
449	{
450	struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);
451	struct nfs_page *head = req->wb_head;
452	struct nfs_page *tmp, *next;
453
454	if (!nfs_page_group_sync_on_bit(req, bit: PG_TEARDOWN))
455	goto out;
456
457	tmp = req;
458	do {
459	next = tmp->wb_this_page;
460	/* unlink and free */
461	tmp->wb_this_page = tmp;
462	tmp->wb_head = tmp;
463	nfs_free_request(req: tmp);
464	tmp = next;
465	} while (tmp != req);
466	out:
467	/* subrequests must release the ref on the head request */
468	if (head != req)
469	nfs_release_request(head);
470	}
471
472	static struct nfs_page *nfs_page_create(struct nfs_lock_context *l_ctx,
473	unsigned int pgbase, pgoff_t index,
474	unsigned int offset, unsigned int count)
475	{
476	struct nfs_page *req;
477	struct nfs_open_context *ctx = l_ctx->open_context;
478
479	if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))
480	return ERR_PTR(error: -EBADF);
481	/* try to allocate the request struct */
482	req = nfs_page_alloc();
483	if (req == NULL)
484	return ERR_PTR(error: -ENOMEM);
485
486	req->wb_lock_context = l_ctx;
487	refcount_inc(r: &l_ctx->count);
488	atomic_inc(v: &l_ctx->io_count);
489
490	/* Initialize the request struct. Initially, we assume a
491	* long write-back delay. This will be adjusted in
492	* update_nfs_request below if the region is not locked. */
493	req->wb_pgbase = pgbase;
494	req->wb_index = index;
495	req->wb_offset = offset;
496	req->wb_bytes = count;
497	kref_init(kref: &req->wb_kref);
498	req->wb_nio = 0;
499	return req;
500	}
501
502	static void nfs_page_assign_folio(struct nfs_page *req, struct folio *folio)
503	{
504	if (folio != NULL) {
505	req->wb_folio = folio;
506	folio_get(folio);
507	set_bit(nr: PG_FOLIO, addr: &req->wb_flags);
508	}
509	}
510
511	static void nfs_page_assign_page(struct nfs_page *req, struct page *page)
512	{
513	if (page != NULL) {
514	req->wb_page = page;
515	get_page(page);
516	}
517	}
518
519	/**
520	* nfs_page_create_from_page - Create an NFS read/write request.
521	* @ctx: open context to use
522	* @page: page to write
523	* @pgbase: starting offset within the page for the write
524	* @offset: file offset for the write
525	* @count: number of bytes to read/write
526	*
527	* The page must be locked by the caller. This makes sure we never
528	* create two different requests for the same page.
529	* User should ensure it is safe to sleep in this function.
530	*/
531	struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx,
532	struct page *page,
533	unsigned int pgbase, loff_t offset,
534	unsigned int count)
535	{
536	struct nfs_lock_context *l_ctx = nfs_get_lock_context(ctx);
537	struct nfs_page *ret;
538
539	if (IS_ERR(ptr: l_ctx))
540	return ERR_CAST(ptr: l_ctx);
541	ret = nfs_page_create(l_ctx, pgbase, index: offset >> PAGE_SHIFT,
542	offset_in_page(offset), count);
543	if (!IS_ERR(ptr: ret)) {
544	nfs_page_assign_page(req: ret, page);
545	nfs_page_group_init(req: ret, NULL);
546	}
547	nfs_put_lock_context(l_ctx);
548	return ret;
549	}
550
551	/**
552	* nfs_page_create_from_folio - Create an NFS read/write request.
553	* @ctx: open context to use
554	* @folio: folio to write
555	* @offset: starting offset within the folio for the write
556	* @count: number of bytes to read/write
557	*
558	* The page must be locked by the caller. This makes sure we never
559	* create two different requests for the same page.
560	* User should ensure it is safe to sleep in this function.
561	*/
562	struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx,
563	struct folio *folio,
564	unsigned int offset,
565	unsigned int count)
566	{
567	struct nfs_lock_context *l_ctx = nfs_get_lock_context(ctx);
568	struct nfs_page *ret;
569
570	if (IS_ERR(ptr: l_ctx))
571	return ERR_CAST(ptr: l_ctx);
572	ret = nfs_page_create(l_ctx, pgbase: offset, index: folio_index(folio), offset, count);
573	if (!IS_ERR(ptr: ret)) {
574	nfs_page_assign_folio(req: ret, folio);
575	nfs_page_group_init(req: ret, NULL);
576	}
577	nfs_put_lock_context(l_ctx);
578	return ret;
579	}
580
581	static struct nfs_page *
582	nfs_create_subreq(struct nfs_page *req,
583	unsigned int pgbase,
584	unsigned int offset,
585	unsigned int count)
586	{
587	struct nfs_page *last;
588	struct nfs_page *ret;
589	struct folio *folio = nfs_page_to_folio(req);
590	struct page *page = nfs_page_to_page(req, pgbase);
591
592	ret = nfs_page_create(l_ctx: req->wb_lock_context, pgbase, index: req->wb_index,
593	offset, count);
594	if (!IS_ERR(ptr: ret)) {
595	if (folio)
596	nfs_page_assign_folio(req: ret, folio);
597	else
598	nfs_page_assign_page(req: ret, page);
599	/* find the last request */
600	for (last = req->wb_head;
601	last->wb_this_page != req->wb_head;
602	last = last->wb_this_page)
603	;
604
605	nfs_lock_request(req: ret);
606	nfs_page_group_init(req: ret, prev: last);
607	ret->wb_nio = req->wb_nio;
608	}
609	return ret;
610	}
611
612	/**
613	* nfs_unlock_request - Unlock request and wake up sleepers.
614	* @req: pointer to request
615	*/
616	void nfs_unlock_request(struct nfs_page *req)
617	{
618	clear_bit_unlock(nr: PG_BUSY, addr: &req->wb_flags);
619	smp_mb__after_atomic();
620	if (!test_bit(PG_CONTENDED2, &req->wb_flags))
621	return;
622	wake_up_bit(word: &req->wb_flags, bit: PG_BUSY);
623	}
624
625	/**
626	* nfs_unlock_and_release_request - Unlock request and release the nfs_page
627	* @req: pointer to request
628	*/
629	void nfs_unlock_and_release_request(struct nfs_page *req)
630	{
631	nfs_unlock_request(req);
632	nfs_release_request(req);
633	}
634
635	/*
636	* nfs_clear_request - Free up all resources allocated to the request
637	* @req:
638	*
639	* Release page and open context resources associated with a read/write
640	* request after it has completed.
641	*/
642	static void nfs_clear_request(struct nfs_page *req)
643	{
644	struct folio *folio = nfs_page_to_folio(req);
645	struct page *page = req->wb_page;
646	struct nfs_lock_context *l_ctx = req->wb_lock_context;
647	struct nfs_open_context *ctx;
648
649	if (folio != NULL) {
650	folio_put(folio);
651	req->wb_folio = NULL;
652	clear_bit(nr: PG_FOLIO, addr: &req->wb_flags);
653	} else if (page != NULL) {
654	put_page(page);
655	req->wb_page = NULL;
656	}
657	if (l_ctx != NULL) {
658	if (atomic_dec_and_test(v: &l_ctx->io_count)) {
659	wake_up_var(var: &l_ctx->io_count);
660	ctx = l_ctx->open_context;
661	if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags))
662	rpc_wake_up(&NFS_SERVER(inode: d_inode(dentry: ctx->dentry))->uoc_rpcwaitq);
663	}
664	nfs_put_lock_context(l_ctx);
665	req->wb_lock_context = NULL;
666	}
667	}
668
669	/**
670	* nfs_free_request - Release the count on an NFS read/write request
671	* @req: request to release
672	*
673	* Note: Should never be called with the spinlock held!
674	*/
675	void nfs_free_request(struct nfs_page *req)
676	{
677	WARN_ON_ONCE(req->wb_this_page != req);
678
679	/* extra debug: make sure no sync bits are still set */
680	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));
681	WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));
682	WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));
683	WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));
684	WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));
685
686	/* Release struct file and open context */
687	nfs_clear_request(req);
688	nfs_page_free(p: req);
689	}
690
691	void nfs_release_request(struct nfs_page *req)
692	{
693	kref_put(kref: &req->wb_kref, release: nfs_page_group_destroy);
694	}
695	EXPORT_SYMBOL_GPL(nfs_release_request);
696
697	/**
698	* nfs_wait_on_request - Wait for a request to complete.
699	* @req: request to wait upon.
700	*
701	* Interruptible by fatal signals only.
702	* The user is responsible for holding a count on the request.
703	*/
704	int
705	nfs_wait_on_request(struct nfs_page *req)
706	{
707	if (!test_bit(PG_BUSY, &req->wb_flags))
708	return 0;
709	set_bit(nr: PG_CONTENDED2, addr: &req->wb_flags);
710	smp_mb__after_atomic();
711	return wait_on_bit_io(word: &req->wb_flags, bit: PG_BUSY,
712	TASK_UNINTERRUPTIBLE);
713	}
714	EXPORT_SYMBOL_GPL(nfs_wait_on_request);
715
716	/*
717	* nfs_generic_pg_test - determine if requests can be coalesced
718	* @desc: pointer to descriptor
719	* @prev: previous request in desc, or NULL
720	* @req: this request
721	*
722	* Returns zero if @req cannot be coalesced into @desc, otherwise it returns
723	* the size of the request.
724	*/
725	size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
726	struct nfs_page *prev, struct nfs_page *req)
727	{
728	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
729
730
731	if (mirror->pg_count > mirror->pg_bsize) {
732	/* should never happen */
733	WARN_ON_ONCE(1);
734	return 0;
735	}
736
737	/*
738	* Limit the request size so that we can still allocate a page array
739	* for it without upsetting the slab allocator.
740	*/
741	if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
742	sizeof(struct page *) > PAGE_SIZE)
743	return 0;
744
745	return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);
746	}
747	EXPORT_SYMBOL_GPL(nfs_generic_pg_test);
748
749	struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *ops)
750	{
751	struct nfs_pgio_header *hdr = ops->rw_alloc_header();
752
753	if (hdr) {
754	INIT_LIST_HEAD(list: &hdr->pages);
755	hdr->rw_ops = ops;
756	}
757	return hdr;
758	}
759	EXPORT_SYMBOL_GPL(nfs_pgio_header_alloc);
760
761	/**
762	* nfs_pgio_data_destroy - make @hdr suitable for reuse
763	*
764	* Frees memory and releases refs from nfs_generic_pgio, so that it may
765	* be called again.
766	*
767	* @hdr: A header that has had nfs_generic_pgio called
768	*/
769	static void nfs_pgio_data_destroy(struct nfs_pgio_header *hdr)
770	{
771	if (hdr->args.context)
772	put_nfs_open_context(ctx: hdr->args.context);
773	if (hdr->page_array.pagevec != hdr->page_array.page_array)
774	kfree(objp: hdr->page_array.pagevec);
775	}
776
777	/*
778	* nfs_pgio_header_free - Free a read or write header
779	* @hdr: The header to free
780	*/
781	void nfs_pgio_header_free(struct nfs_pgio_header *hdr)
782	{
783	nfs_pgio_data_destroy(hdr);
784	hdr->rw_ops->rw_free_header(hdr);
785	}
786	EXPORT_SYMBOL_GPL(nfs_pgio_header_free);
787
788	/**
789	* nfs_pgio_rpcsetup - Set up arguments for a pageio call
790	* @hdr: The pageio hdr
791	* @pgbase: base
792	* @count: Number of bytes to read
793	* @how: How to commit data (writes only)
794	* @cinfo: Commit information for the call (writes only)
795	*/
796	static void nfs_pgio_rpcsetup(struct nfs_pgio_header *hdr, unsigned int pgbase,
797	unsigned int count, int how,
798	struct nfs_commit_info *cinfo)
799	{
800	struct nfs_page *req = hdr->req;
801
802	/* Set up the RPC argument and reply structs
803	* NB: take care not to mess about with hdr->commit et al. */
804
805	hdr->args.fh = NFS_FH(inode: hdr->inode);
806	hdr->args.offset = req_offset(req);
807	/* pnfs_set_layoutcommit needs this */
808	hdr->mds_offset = hdr->args.offset;
809	hdr->args.pgbase = pgbase;
810	hdr->args.pages = hdr->page_array.pagevec;
811	hdr->args.count = count;
812	hdr->args.context = get_nfs_open_context(ctx: nfs_req_openctx(req));
813	hdr->args.lock_context = req->wb_lock_context;
814	hdr->args.stable = NFS_UNSTABLE;
815	switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
816	case 0:
817	break;
818	case FLUSH_COND_STABLE:
819	if (nfs_reqs_to_commit(cinfo))
820	break;
821	fallthrough;
822	default:
823	hdr->args.stable = NFS_FILE_SYNC;
824	}
825
826	hdr->res.fattr = &hdr->fattr;
827	hdr->res.count = 0;
828	hdr->res.eof = 0;
829	hdr->res.verf = &hdr->verf;
830	nfs_fattr_init(fattr: &hdr->fattr);
831	}
832
833	/**
834	* nfs_pgio_prepare - Prepare pageio hdr to go over the wire
835	* @task: The current task
836	* @calldata: pageio header to prepare
837	*/
838	static void nfs_pgio_prepare(struct rpc_task *task, void *calldata)
839	{
840	struct nfs_pgio_header *hdr = calldata;
841	int err;
842	err = NFS_PROTO(inode: hdr->inode)->pgio_rpc_prepare(task, hdr);
843	if (err)
844	rpc_exit(task, err);
845	}
846
847	int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
848	const struct cred *cred, const struct nfs_rpc_ops *rpc_ops,
849	const struct rpc_call_ops *call_ops, int how, int flags)
850	{
851	struct rpc_task *task;
852	struct rpc_message msg = {
853	.rpc_argp = &hdr->args,
854	.rpc_resp = &hdr->res,
855	.rpc_cred = cred,
856	};
857	struct rpc_task_setup task_setup_data = {
858	.rpc_client = clnt,
859	.task = &hdr->task,
860	.rpc_message = &msg,
861	.callback_ops = call_ops,
862	.callback_data = hdr,
863	.workqueue = nfsiod_workqueue,
864	.flags = RPC_TASK_ASYNC | flags,
865	};
866
867	if (nfs_server_capable(inode: hdr->inode, NFS_CAP_MOVEABLE))
868	task_setup_data.flags |= RPC_TASK_MOVEABLE;
869
870	hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);
871
872	dprintk("NFS: initiated pgio call "
873	"(req %s/%llu, %u bytes @ offset %llu)\n",
874	hdr->inode->i_sb->s_id,
875	(unsigned long long)NFS_FILEID(hdr->inode),
876	hdr->args.count,
877	(unsigned long long)hdr->args.offset);
878
879	task = rpc_run_task(&task_setup_data);
880	if (IS_ERR(ptr: task))
881	return PTR_ERR(ptr: task);
882	rpc_put_task(task);
883	return 0;
884	}
885	EXPORT_SYMBOL_GPL(nfs_initiate_pgio);
886
887	/**
888	* nfs_pgio_error - Clean up from a pageio error
889	* @hdr: pageio header
890	*/
891	static void nfs_pgio_error(struct nfs_pgio_header *hdr)
892	{
893	set_bit(nr: NFS_IOHDR_REDO, addr: &hdr->flags);
894	hdr->completion_ops->completion(hdr);
895	}
896
897	/**
898	* nfs_pgio_release - Release pageio data
899	* @calldata: The pageio header to release
900	*/
901	static void nfs_pgio_release(void *calldata)
902	{
903	struct nfs_pgio_header *hdr = calldata;
904	hdr->completion_ops->completion(hdr);
905	}
906
907	static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,
908	unsigned int bsize)
909	{
910	INIT_LIST_HEAD(list: &mirror->pg_list);
911	mirror->pg_bytes_written = 0;
912	mirror->pg_count = 0;
913	mirror->pg_bsize = bsize;
914	mirror->pg_base = 0;
915	mirror->pg_recoalesce = 0;
916	}
917
918	/**
919	* nfs_pageio_init - initialise a page io descriptor
920	* @desc: pointer to descriptor
921	* @inode: pointer to inode
922	* @pg_ops: pointer to pageio operations
923	* @compl_ops: pointer to pageio completion operations
924	* @rw_ops: pointer to nfs read/write operations
925	* @bsize: io block size
926	* @io_flags: extra parameters for the io function
927	*/
928	void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
929	struct inode *inode,
930	const struct nfs_pageio_ops *pg_ops,
931	const struct nfs_pgio_completion_ops *compl_ops,
932	const struct nfs_rw_ops *rw_ops,
933	size_t bsize,
934	int io_flags)
935	{
936	desc->pg_moreio = 0;
937	desc->pg_inode = inode;
938	desc->pg_ops = pg_ops;
939	desc->pg_completion_ops = compl_ops;
940	desc->pg_rw_ops = rw_ops;
941	desc->pg_ioflags = io_flags;
942	desc->pg_error = 0;
943	desc->pg_lseg = NULL;
944	desc->pg_io_completion = NULL;
945	desc->pg_dreq = NULL;
946	nfs_netfs_reset_pageio_descriptor(desc);
947	desc->pg_bsize = bsize;
948
949	desc->pg_mirror_count = 1;
950	desc->pg_mirror_idx = 0;
951
952	desc->pg_mirrors_dynamic = NULL;
953	desc->pg_mirrors = desc->pg_mirrors_static;
954	nfs_pageio_mirror_init(mirror: &desc->pg_mirrors[0], bsize);
955	desc->pg_maxretrans = 0;
956	}
957
958	/**
959	* nfs_pgio_result - Basic pageio error handling
960	* @task: The task that ran
961	* @calldata: Pageio header to check
962	*/
963	static void nfs_pgio_result(struct rpc_task *task, void *calldata)
964	{
965	struct nfs_pgio_header *hdr = calldata;
966	struct inode *inode = hdr->inode;
967
968	if (hdr->rw_ops->rw_done(task, hdr, inode) != 0)
969	return;
970	if (task->tk_status < 0)
971	nfs_set_pgio_error(hdr, error: task->tk_status, pos: hdr->args.offset);
972	else
973	hdr->rw_ops->rw_result(task, hdr);
974	}
975
976	/*
977	* Create an RPC task for the given read or write request and kick it.
978	* The page must have been locked by the caller.
979	*
980	* It may happen that the page we're passed is not marked dirty.
981	* This is the case if nfs_updatepage detects a conflicting request
982	* that has been written but not committed.
983	*/
984	int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
985	struct nfs_pgio_header *hdr)
986	{
987	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
988
989	struct nfs_page *req;
990	struct page **pages,
991	*last_page;
992	struct list_head *head = &mirror->pg_list;
993	struct nfs_commit_info cinfo;
994	struct nfs_page_array *pg_array = &hdr->page_array;
995	unsigned int pagecount, pageused;
996	unsigned int pg_base = offset_in_page(mirror->pg_base);
997	gfp_t gfp_flags = nfs_io_gfp_mask();
998
999	pagecount = nfs_page_array_len(base: pg_base, len: mirror->pg_count);
1000	pg_array->npages = pagecount;
1001
1002	if (pagecount <= ARRAY_SIZE(pg_array->page_array))
1003	pg_array->pagevec = pg_array->page_array;
1004	else {
1005	pg_array->pagevec = kcalloc(n: pagecount, size: sizeof(struct page *), flags: gfp_flags);
1006	if (!pg_array->pagevec) {
1007	pg_array->npages = 0;
1008	nfs_pgio_error(hdr);
1009	desc->pg_error = -ENOMEM;
1010	return desc->pg_error;
1011	}
1012	}
1013
1014	nfs_init_cinfo(cinfo: &cinfo, inode: desc->pg_inode, dreq: desc->pg_dreq);
1015	pages = hdr->page_array.pagevec;
1016	last_page = NULL;
1017	pageused = 0;
1018	while (!list_empty(head)) {
1019	struct nfs_page_iter_page i;
1020	struct page *page;
1021
1022	req = nfs_list_entry(head: head->next);
1023	nfs_list_move_request(req, head: &hdr->pages);
1024
1025	if (req->wb_pgbase == 0)
1026	last_page = NULL;
1027
1028	nfs_page_iter_page_init(i: &i, req);
1029	while ((page = nfs_page_iter_page_get(i: &i)) != NULL) {
1030	if (last_page != page) {
1031	pageused++;
1032	if (pageused > pagecount)
1033	goto full;
1034	*pages++ = last_page = page;
1035	}
1036	}
1037	}
1038	full:
1039	if (WARN_ON_ONCE(pageused != pagecount)) {
1040	nfs_pgio_error(hdr);
1041	desc->pg_error = -EINVAL;
1042	return desc->pg_error;
1043	}
1044
1045	if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
1046	(desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))
1047	desc->pg_ioflags &= ~FLUSH_COND_STABLE;
1048
1049	/* Set up the argument struct */
1050	nfs_pgio_rpcsetup(hdr, pgbase: pg_base, count: mirror->pg_count, how: desc->pg_ioflags,
1051	cinfo: &cinfo);
1052	desc->pg_rpc_callops = &nfs_pgio_common_ops;
1053	return 0;
1054	}
1055	EXPORT_SYMBOL_GPL(nfs_generic_pgio);
1056
1057	static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)
1058	{
1059	struct nfs_pgio_header *hdr;
1060	int ret;
1061	unsigned short task_flags = 0;
1062
1063	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
1064	if (!hdr) {
1065	desc->pg_error = -ENOMEM;
1066	return desc->pg_error;
1067	}
1068	nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);
1069	ret = nfs_generic_pgio(desc, hdr);
1070	if (ret == 0) {
1071	if (NFS_SERVER(inode: hdr->inode)->nfs_client->cl_minorversion)
1072	task_flags = RPC_TASK_MOVEABLE;
1073	ret = nfs_initiate_pgio(NFS_CLIENT(inode: hdr->inode),
1074	hdr,
1075	hdr->cred,
1076	NFS_PROTO(inode: hdr->inode),
1077	desc->pg_rpc_callops,
1078	desc->pg_ioflags,
1079	RPC_TASK_CRED_NOREF | task_flags);
1080	}
1081	return ret;
1082	}
1083
1084	static struct nfs_pgio_mirror *
1085	nfs_pageio_alloc_mirrors(struct nfs_pageio_descriptor *desc,
1086	unsigned int mirror_count)
1087	{
1088	struct nfs_pgio_mirror *ret;
1089	unsigned int i;
1090
1091	kfree(objp: desc->pg_mirrors_dynamic);
1092	desc->pg_mirrors_dynamic = NULL;
1093	if (mirror_count == 1)
1094	return desc->pg_mirrors_static;
1095	ret = kmalloc_array(n: mirror_count, size: sizeof(*ret), flags: nfs_io_gfp_mask());
1096	if (ret != NULL) {
1097	for (i = 0; i < mirror_count; i++)
1098	nfs_pageio_mirror_init(mirror: &ret[i], bsize: desc->pg_bsize);
1099	desc->pg_mirrors_dynamic = ret;
1100	}
1101	return ret;
1102	}
1103
1104	/*
1105	* nfs_pageio_setup_mirroring - determine if mirroring is to be used
1106	* by calling the pg_get_mirror_count op
1107	*/
1108	static void nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio,
1109	struct nfs_page *req)
1110	{
1111	unsigned int mirror_count = 1;
1112
1113	if (pgio->pg_ops->pg_get_mirror_count)
1114	mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
1115	if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0)
1116	return;
1117
1118	if (!mirror_count || mirror_count > NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX) {
1119	pgio->pg_error = -EINVAL;
1120	return;
1121	}
1122
1123	pgio->pg_mirrors = nfs_pageio_alloc_mirrors(desc: pgio, mirror_count);
1124	if (pgio->pg_mirrors == NULL) {
1125	pgio->pg_error = -ENOMEM;
1126	pgio->pg_mirrors = pgio->pg_mirrors_static;
1127	mirror_count = 1;
1128	}
1129	pgio->pg_mirror_count = mirror_count;
1130	}
1131
1132	static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio)
1133	{
1134	pgio->pg_mirror_count = 1;
1135	pgio->pg_mirror_idx = 0;
1136	pgio->pg_mirrors = pgio->pg_mirrors_static;
1137	kfree(objp: pgio->pg_mirrors_dynamic);
1138	pgio->pg_mirrors_dynamic = NULL;
1139	}
1140
1141	static bool nfs_match_lock_context(const struct nfs_lock_context *l1,
1142	const struct nfs_lock_context *l2)
1143	{
1144	return l1->lockowner == l2->lockowner;
1145	}
1146
1147	static bool nfs_page_is_contiguous(const struct nfs_page *prev,
1148	const struct nfs_page *req)
1149	{
1150	size_t prev_end = prev->wb_pgbase + prev->wb_bytes;
1151
1152	if (req_offset(req) != req_offset(req: prev) + prev->wb_bytes)
1153	return false;
1154	if (req->wb_pgbase == 0)
1155	return prev_end == nfs_page_max_length(req: prev);
1156	if (req->wb_pgbase == prev_end) {
1157	struct folio *folio = nfs_page_to_folio(req);
1158	if (folio)
1159	return folio == nfs_page_to_folio(req: prev);
1160	return req->wb_page == prev->wb_page;
1161	}
1162	return false;
1163	}
1164
1165	/**
1166	* nfs_coalesce_size - test two requests for compatibility
1167	* @prev: pointer to nfs_page
1168	* @req: pointer to nfs_page
1169	* @pgio: pointer to nfs_pagio_descriptor
1170	*
1171	* The nfs_page structures 'prev' and 'req' are compared to ensure that the
1172	* page data area they describe is contiguous, and that their RPC
1173	* credentials, NFSv4 open state, and lockowners are the same.
1174	*
1175	* Returns size of the request that can be coalesced
1176	*/
1177	static unsigned int nfs_coalesce_size(struct nfs_page *prev,
1178	struct nfs_page *req,
1179	struct nfs_pageio_descriptor *pgio)
1180	{
1181	struct file_lock_context *flctx;
1182
1183	if (prev) {
1184	if (!nfs_match_open_context(ctx1: nfs_req_openctx(req), ctx2: nfs_req_openctx(req: prev)))
1185	return 0;
1186	flctx = locks_inode_context(inode: d_inode(dentry: nfs_req_openctx(req)->dentry));
1187	if (flctx != NULL &&
1188	!(list_empty_careful(head: &flctx->flc_posix) &&
1189	list_empty_careful(head: &flctx->flc_flock)) &&
1190	!nfs_match_lock_context(l1: req->wb_lock_context,
1191	l2: prev->wb_lock_context))
1192	return 0;
1193	if (!nfs_page_is_contiguous(prev, req))
1194	return 0;
1195	}
1196	return pgio->pg_ops->pg_test(pgio, prev, req);
1197	}
1198
1199	/**
1200	* nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
1201	* @desc: destination io descriptor
1202	* @req: request
1203	*
1204	* If the request 'req' was successfully coalesced into the existing list
1205	* of pages 'desc', it returns the size of req.
1206	*/
1207	static unsigned int
1208	nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
1209	struct nfs_page *req)
1210	{
1211	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
1212	struct nfs_page *prev = NULL;
1213	unsigned int size;
1214
1215	if (list_empty(head: &mirror->pg_list)) {
1216	if (desc->pg_ops->pg_init)
1217	desc->pg_ops->pg_init(desc, req);
1218	if (desc->pg_error < 0)
1219	return 0;
1220	mirror->pg_base = req->wb_pgbase;
1221	mirror->pg_count = 0;
1222	mirror->pg_recoalesce = 0;
1223	} else
1224	prev = nfs_list_entry(head: mirror->pg_list.prev);
1225
1226	if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) {
1227	if (NFS_SERVER(inode: desc->pg_inode)->flags & NFS_MOUNT_SOFTERR)
1228	desc->pg_error = -ETIMEDOUT;
1229	else
1230	desc->pg_error = -EIO;
1231	return 0;
1232	}
1233
1234	size = nfs_coalesce_size(prev, req, pgio: desc);
1235	if (size < req->wb_bytes)
1236	return size;
1237	nfs_list_move_request(req, head: &mirror->pg_list);
1238	mirror->pg_count += req->wb_bytes;
1239	return req->wb_bytes;
1240	}
1241
1242	/*
1243	* Helper for nfs_pageio_add_request and nfs_pageio_complete
1244	*/
1245	static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
1246	{
1247	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
1248
1249	if (!list_empty(head: &mirror->pg_list)) {
1250	int error = desc->pg_ops->pg_doio(desc);
1251	if (error < 0)
1252	desc->pg_error = error;
1253	if (list_empty(head: &mirror->pg_list))
1254	mirror->pg_bytes_written += mirror->pg_count;
1255	}
1256	}
1257
1258	static void
1259	nfs_pageio_cleanup_request(struct nfs_pageio_descriptor *desc,
1260	struct nfs_page *req)
1261	{
1262	LIST_HEAD(head);
1263
1264	nfs_list_move_request(req, head: &head);
1265	desc->pg_completion_ops->error_cleanup(&head, desc->pg_error);
1266	}
1267
1268	/**
1269	* __nfs_pageio_add_request - Attempt to coalesce a request into a page list.
1270	* @desc: destination io descriptor
1271	* @req: request
1272	*
1273	* This may split a request into subrequests which are all part of the
1274	* same page group. If so, it will submit @req as the last one, to ensure
1275	* the pointer to @req is still valid in case of failure.
1276	*
1277	* Returns true if the request 'req' was successfully coalesced into the
1278	* existing list of pages 'desc'.
1279	*/
1280	static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
1281	struct nfs_page *req)
1282	{
1283	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
1284	struct nfs_page *subreq;
1285	unsigned int size, subreq_size;
1286
1287	nfs_page_group_lock(req);
1288
1289	subreq = req;
1290	subreq_size = subreq->wb_bytes;
1291	for(;;) {
1292	size = nfs_pageio_do_add_request(desc, req: subreq);
1293	if (size == subreq_size) {
1294	/* We successfully submitted a request */
1295	if (subreq == req)
1296	break;
1297	req->wb_pgbase += size;
1298	req->wb_bytes -= size;
1299	req->wb_offset += size;
1300	subreq_size = req->wb_bytes;
1301	subreq = req;
1302	continue;
1303	}
1304	if (WARN_ON_ONCE(subreq != req)) {
1305	nfs_page_group_unlock(req);
1306	nfs_pageio_cleanup_request(desc, req: subreq);
1307	subreq = req;
1308	subreq_size = req->wb_bytes;
1309	nfs_page_group_lock(req);
1310	}
1311	if (!size) {
1312	/* Can't coalesce any more, so do I/O */
1313	nfs_page_group_unlock(req);
1314	desc->pg_moreio = 1;
1315	nfs_pageio_doio(desc);
1316	if (desc->pg_error < 0 || mirror->pg_recoalesce)
1317	return 0;
1318	/* retry add_request for this subreq */
1319	nfs_page_group_lock(req);
1320	continue;
1321	}
1322	subreq = nfs_create_subreq(req, pgbase: req->wb_pgbase,
1323	offset: req->wb_offset, count: size);
1324	if (IS_ERR(ptr: subreq))
1325	goto err_ptr;
1326	subreq_size = size;
1327	}
1328
1329	nfs_page_group_unlock(req);
1330	return 1;
1331	err_ptr:
1332	desc->pg_error = PTR_ERR(ptr: subreq);
1333	nfs_page_group_unlock(req);
1334	return 0;
1335	}
1336
1337	static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
1338	{
1339	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
1340	LIST_HEAD(head);
1341
1342	do {
1343	list_splice_init(list: &mirror->pg_list, head: &head);
1344	mirror->pg_recoalesce = 0;
1345
1346	while (!list_empty(head: &head)) {
1347	struct nfs_page *req;
1348
1349	req = list_first_entry(&head, struct nfs_page, wb_list);
1350	if (__nfs_pageio_add_request(desc, req))
1351	continue;
1352	if (desc->pg_error < 0) {
1353	list_splice_tail(list: &head, head: &mirror->pg_list);
1354	mirror->pg_recoalesce = 1;
1355	return 0;
1356	}
1357	break;
1358	}
1359	} while (mirror->pg_recoalesce);
1360	return 1;
1361	}
1362
1363	static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc,
1364	struct nfs_page *req)
1365	{
1366	int ret;
1367
1368	do {
1369	ret = __nfs_pageio_add_request(desc, req);
1370	if (ret)
1371	break;
1372	if (desc->pg_error < 0)
1373	break;
1374	ret = nfs_do_recoalesce(desc);
1375	} while (ret);
1376
1377	return ret;
1378	}
1379
1380	static void nfs_pageio_error_cleanup(struct nfs_pageio_descriptor *desc)
1381	{
1382	u32 midx;
1383	struct nfs_pgio_mirror *mirror;
1384
1385	if (!desc->pg_error)
1386	return;
1387
1388	for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1389	mirror = nfs_pgio_get_mirror(desc, idx: midx);
1390	desc->pg_completion_ops->error_cleanup(&mirror->pg_list,
1391	desc->pg_error);
1392	}
1393	}
1394
1395	int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
1396	struct nfs_page *req)
1397	{
1398	u32 midx;
1399	unsigned int pgbase, offset, bytes;
1400	struct nfs_page *dupreq;
1401
1402	pgbase = req->wb_pgbase;
1403	offset = req->wb_offset;
1404	bytes = req->wb_bytes;
1405
1406	nfs_pageio_setup_mirroring(pgio: desc, req);
1407	if (desc->pg_error < 0)
1408	goto out_failed;
1409
1410	/* Create the mirror instances first, and fire them off */
1411	for (midx = 1; midx < desc->pg_mirror_count; midx++) {
1412	nfs_page_group_lock(req);
1413
1414	dupreq = nfs_create_subreq(req,
1415	pgbase, offset, count: bytes);
1416
1417	nfs_page_group_unlock(req);
1418	if (IS_ERR(ptr: dupreq)) {
1419	desc->pg_error = PTR_ERR(ptr: dupreq);
1420	goto out_failed;
1421	}
1422
1423	nfs_pgio_set_current_mirror(desc, idx: midx);
1424	if (!nfs_pageio_add_request_mirror(desc, req: dupreq))
1425	goto out_cleanup_subreq;
1426	}
1427
1428	nfs_pgio_set_current_mirror(desc, idx: 0);
1429	if (!nfs_pageio_add_request_mirror(desc, req))
1430	goto out_failed;
1431
1432	return 1;
1433
1434	out_cleanup_subreq:
1435	nfs_pageio_cleanup_request(desc, req: dupreq);
1436	out_failed:
1437	nfs_pageio_error_cleanup(desc);
1438	return 0;
1439	}
1440
1441	/*
1442	* nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
1443	* nfs_pageio_descriptor
1444	* @desc: pointer to io descriptor
1445	* @mirror_idx: pointer to mirror index
1446	*/
1447	static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,
1448	u32 mirror_idx)
1449	{
1450	struct nfs_pgio_mirror *mirror;
1451	u32 restore_idx;
1452
1453	restore_idx = nfs_pgio_set_current_mirror(desc, idx: mirror_idx);
1454	mirror = nfs_pgio_current_mirror(desc);
1455
1456	for (;;) {
1457	nfs_pageio_doio(desc);
1458	if (desc->pg_error < 0 || !mirror->pg_recoalesce)
1459	break;
1460	if (!nfs_do_recoalesce(desc))
1461	break;
1462	}
1463	nfs_pgio_set_current_mirror(desc, idx: restore_idx);
1464	}
1465
1466	/*
1467	* nfs_pageio_resend - Transfer requests to new descriptor and resend
1468	* @hdr - the pgio header to move request from
1469	* @desc - the pageio descriptor to add requests to
1470	*
1471	* Try to move each request (nfs_page) from @hdr to @desc then attempt
1472	* to send them.
1473	*
1474	* Returns 0 on success and < 0 on error.
1475	*/
1476	int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,
1477	struct nfs_pgio_header *hdr)
1478	{
1479	LIST_HEAD(pages);
1480
1481	desc->pg_io_completion = hdr->io_completion;
1482	desc->pg_dreq = hdr->dreq;
1483	nfs_netfs_set_pageio_descriptor(desc, hdr);
1484	list_splice_init(list: &hdr->pages, head: &pages);
1485	while (!list_empty(head: &pages)) {
1486	struct nfs_page *req = nfs_list_entry(head: pages.next);
1487
1488	if (!nfs_pageio_add_request(desc, req))
1489	break;
1490	}
1491	nfs_pageio_complete(desc);
1492	if (!list_empty(head: &pages)) {
1493	int err = desc->pg_error < 0 ? desc->pg_error : -EIO;
1494	hdr->completion_ops->error_cleanup(&pages, err);
1495	nfs_set_pgio_error(hdr, error: err, pos: hdr->io_start);
1496	return err;
1497	}
1498	return 0;
1499	}
1500	EXPORT_SYMBOL_GPL(nfs_pageio_resend);
1501
1502	/**
1503	* nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1504	* @desc: pointer to io descriptor
1505	*/
1506	void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
1507	{
1508	u32 midx;
1509
1510	for (midx = 0; midx < desc->pg_mirror_count; midx++)
1511	nfs_pageio_complete_mirror(desc, mirror_idx: midx);
1512
1513	if (desc->pg_error < 0)
1514	nfs_pageio_error_cleanup(desc);
1515	if (desc->pg_ops->pg_cleanup)
1516	desc->pg_ops->pg_cleanup(desc);
1517	nfs_pageio_cleanup_mirroring(pgio: desc);
1518	}
1519
1520	/**
1521	* nfs_pageio_cond_complete - Conditional I/O completion
1522	* @desc: pointer to io descriptor
1523	* @index: page index
1524	*
1525	* It is important to ensure that processes don't try to take locks
1526	* on non-contiguous ranges of pages as that might deadlock. This
1527	* function should be called before attempting to wait on a locked
1528	* nfs_page. It will complete the I/O if the page index 'index'
1529	* is not contiguous with the existing list of pages in 'desc'.
1530	*/
1531	void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
1532	{
1533	struct nfs_pgio_mirror *mirror;
1534	struct nfs_page *prev;
1535	struct folio *folio;
1536	u32 midx;
1537
1538	for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1539	mirror = nfs_pgio_get_mirror(desc, idx: midx);
1540	if (!list_empty(head: &mirror->pg_list)) {
1541	prev = nfs_list_entry(head: mirror->pg_list.prev);
1542	folio = nfs_page_to_folio(req: prev);
1543	if (folio) {
1544	if (index == folio_next_index(folio))
1545	continue;
1546	} else if (index == prev->wb_index + 1)
1547	continue;
1548	nfs_pageio_complete(desc);
1549	break;
1550	}
1551	}
1552	}
1553
1554	/*
1555	* nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
1556	*/
1557	void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio)
1558	{
1559	nfs_pageio_complete(desc: pgio);
1560	}
1561
1562	int __init nfs_init_nfspagecache(void)
1563	{
1564	nfs_page_cachep = kmem_cache_create(name: "nfs_page",
1565	size: sizeof(struct nfs_page),
1566	align: 0, SLAB_HWCACHE_ALIGN,
1567	NULL);
1568	if (nfs_page_cachep == NULL)
1569	return -ENOMEM;
1570
1571	return 0;
1572	}
1573
1574	void nfs_destroy_nfspagecache(void)
1575	{
1576	kmem_cache_destroy(s: nfs_page_cachep);
1577	}
1578
1579	static const struct rpc_call_ops nfs_pgio_common_ops = {
1580	.rpc_call_prepare = nfs_pgio_prepare,
1581	.rpc_call_done = nfs_pgio_result,
1582	.rpc_release = nfs_pgio_release,
1583	};
1584
1585	const struct nfs_pageio_ops nfs_pgio_rw_ops = {
1586	.pg_test = nfs_generic_pg_test,
1587	.pg_doio = nfs_generic_pg_pgios,
1588	};
1589